diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index df54c1ae..416f0092 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -86,11 +86,11 @@ jobs: - name: Run integration tests run: npm run test:integ:all - - name: Upload browser test screenshots + - name: Upload Artifacts if: always() uses: actions/upload-artifact@v5 with: - name: browser-test-screenshots - path: tests_integ/browser/__screenshots__/ + name: test-artifacts + path: ./test/.artifacts/ retention-days: 4 if-no-files-found: ignore diff --git a/.github/workflows/test-lint.yml b/.github/workflows/test-lint.yml index a7076070..2eeb229d 100644 --- a/.github/workflows/test-lint.yml +++ b/.github/workflows/test-lint.yml @@ -40,6 +40,23 @@ jobs: - name: Run unit tests run: npm run test:all:coverage + - name: List directory contents + if: always() + run: | + echo "Current directory:" + pwd + echo "Directory contents:" + find . -type f + + - name: Upload Artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: test-artifacts + path: ./test/.artifacts/ + retention-days: 4 + if-no-files-found: ignore + - name: Run linting run: npm run lint diff --git a/.gitignore b/.gitignore index 57b1943b..b79c3836 100644 --- a/.gitignore +++ b/.gitignore @@ -37,4 +37,6 @@ Thumbs.db .env.production.local # Github workflow artifacts -.artifact \ No newline at end of file +.artifact +# Test artifacts +test/.artifacts \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 737d7f59..6adacc7a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -23,6 +23,7 @@ "@vitest/browser": "^4.0.15", "@vitest/browser-playwright": "^4.0.15", "@vitest/coverage-v8": "^4.0.15", + "@vitest/ui": "^4.0.15", "eslint": "^9.0.0", "eslint-plugin-tsdoc": "^0.5.0", "husky": "^9.1.7", @@ -4323,6 +4324,28 @@ "url": "https://opencollective.com/vitest" } }, + "node_modules/@vitest/ui": { + "version": "4.0.15", + "resolved": "https://registry.npmjs.org/@vitest/ui/-/ui-4.0.15.tgz", + "integrity": "sha512-sxSyJMaKp45zI0u+lHrPuZM1ZJQ8FaVD35k+UxVrha1yyvQ+TZuUYllUixwvQXlB7ixoDc7skf3lQPopZIvaQw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/utils": "4.0.15", + "fflate": "^0.8.2", + "flatted": "^3.3.3", + "pathe": "^2.0.3", + "sirv": "^3.0.2", + "tinyglobby": "^0.2.15", + "tinyrainbow": "^3.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "vitest": "4.0.15" + } + }, "node_modules/@vitest/utils": { "version": "4.0.15", "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-4.0.15.tgz", @@ -5465,6 +5488,13 @@ } } }, + "node_modules/fflate": { + "version": "0.8.2", + "resolved": "https://registry.npmjs.org/fflate/-/fflate-0.8.2.tgz", + "integrity": "sha512-cPJU47OaAoCbg0pBvzsgpTPhmhqI5eJjh/JIu8tPj5q+T7iLvW/JAYUqmE7KOB4R1ZyEhzBaIQpQpardBF5z8A==", + "dev": true, + "license": "MIT" + }, "node_modules/file-entry-cache": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-8.0.0.tgz", diff --git a/package.json b/package.json index 1cc665c0..7f27eb0a 100644 --- a/package.json +++ b/package.json @@ -33,6 +33,10 @@ "./vended_tools/http_request": { "types": "./dist/vended_tools/http_request/index.d.ts", "default": "./dist/vended_tools/http_request/index.js" + }, + "./vended_tools/bash": { + "types": "./dist/vended_tools/bash/index.d.ts", + "default": "./dist/vended_tools/bash/index.js" } }, "scripts": { @@ -78,6 +82,7 @@ "@vitest/browser": "^4.0.15", "@vitest/browser-playwright": "^4.0.15", "@vitest/coverage-v8": "^4.0.15", + "@vitest/ui": "^4.0.15", "eslint": "^9.0.0", "eslint-plugin-tsdoc": "^0.5.0", "husky": "^9.1.7", diff --git a/src/models/openai.ts b/src/models/openai.ts index 3e325a22..14a9fc3b 100644 --- a/src/models/openai.ts +++ b/src/models/openai.ts @@ -265,7 +265,7 @@ export class OpenAIModel extends Model { // In Node.js, can use OPENAI_API_KEY environment variable as fallback const hasEnvKey = typeof process !== 'undefined' && typeof process.env !== 'undefined' && process.env.OPENAI_API_KEY - if (!apiKey && !hasEnvKey) { + if (!apiKey && !hasEnvKey && !clientConfig?.apiKey) { throw new Error( "OpenAI API key is required. Provide it via the 'apiKey' option or set the OPENAI_API_KEY environment variable." ) diff --git a/tests_integ/__fixtures__/model-test-helpers.ts b/tests_integ/__fixtures__/model-test-helpers.ts index 3c75ef52..ac530d97 100644 --- a/tests_integ/__fixtures__/model-test-helpers.ts +++ b/tests_integ/__fixtures__/model-test-helpers.ts @@ -1,5 +1,10 @@ -import { fromNodeProviderChain } from '@aws-sdk/credential-providers' -import type { Message, ContentBlock } from '../../src/types/messages.js' +import { BedrockModel, type BedrockModelOptions } from '@strands-agents/sdk' +import { OpenAIModel, type OpenAIModelOptions } from '@strands-agents/sdk/openai' + +import type { Message, ContentBlock } from '@strands-agents/sdk' +import { isInBrowser } from './test-helpers.js' + +export * from '../../src/__fixtures__/model-test-helpers.js' /** * Determines whether AWS integration tests should run based on environment and credentials. @@ -9,22 +14,71 @@ import type { Message, ContentBlock } from '../../src/types/messages.js' * * @returns Promise - true if tests should run, false if they should be skipped */ -export async function shouldRunTests(): Promise { +export async function shouldSkipBedrockTests(): Promise { // In a CI environment, we ALWAYS expect credentials to be configured. // A failure is better than a skip. - if (process.env.CI) { + if (globalThis.process?.env?.CI) { console.log('✅ Running in CI environment, integration tests will run.') - return true + return false } // In a local environment, we check for credentials as a convenience. try { - const credentialProvider = fromNodeProviderChain() - await credentialProvider() - console.log('✅ AWS credentials found locally, integration tests will run.') - return true + if (isInBrowser()) { + const { commands } = await import('vitest/browser') + await commands.getAwsCredentials() + console.log('✅ credentials found via vitest, integration tests will run.') + return false + } else { + const { fromNodeProviderChain } = await import('@aws-sdk/credential-providers') + const credentialProvider = fromNodeProviderChain() + await credentialProvider() + console.log('✅ AWS credentials found locally, integration tests will run.') + return false + } } catch { - console.log('⏭️ AWS credentials not available locally, integration tests will be skipped.') + console.log('⏭️ AWS credentials not available, integration tests will be skipped.') + return true + } +} + +/** + * Determines if OpenAI integration tests should be skipped. + * In CI environments, throws an error if API key is missing (tests should not be skipped). + * In local development, skips tests if API key is not available. + * + * @returns true if tests should be skipped, false if they should run + * @throws Error if running in CI and API key is missing + */ +export const shouldSkipOpenAITests = async (): Promise => { + const getApiKey = async (): Promise => { + if (isInBrowser()) { + const { commands } = await import('vitest/browser') + return await commands.getOpenAIAPIKey() + } else { + return globalThis.process.env.OPENAI_API_KEY as string + } + } + + // In a CI environment, we ALWAYS expect credentials to be configured. + // A failure is better than a skip. + if (globalThis.process?.env?.CI) { + console.log('✅ Running in CI environment, integration tests will run.') + const apiKey = await getApiKey() + + if (!apiKey) { + throw new Error('OpenAI API key must be available in CI environments') + } + + return false + } + + const apiKey = await getApiKey() + if (!apiKey) { + console.log('⏭️ OpenAI API key not available - integration tests will be skipped') + return true + } else { + console.log('⏭️ OpenAI API key available - integration tests will run') return false } } @@ -47,3 +101,38 @@ export const getMessageText = (message: Message): string => { .map((block) => block.text) .join('\n') } + +export function createBedrockModel(options: BedrockModelOptions = {}) { + if (isInBrowser()) { + return new BedrockModel({ + ...options, + clientConfig: { + ...(options.clientConfig ?? {}), + credentials: async () => { + const { commands } = await import('vitest/browser') + return await commands.getAwsCredentials() + }, + }, + }) + } else { + return new BedrockModel(options) + } +} + +export function createOpenAIModel(config: OpenAIModelOptions = {}) { + if (isInBrowser()) { + return new OpenAIModel({ + ...config, + clientConfig: { + ...(config.clientConfig ?? {}), + apiKey: async (): Promise => { + const { commands } = await import('vitest/browser') + return await commands.getOpenAIAPIKey() + }, + dangerouslyAllowBrowser: true, + }, + }) + } else { + return new OpenAIModel(config) + } +} diff --git a/tests_integ/__fixtures__/test-helpers.ts b/tests_integ/__fixtures__/test-helpers.ts index 7311bf50..29a4d5eb 100644 --- a/tests_integ/__fixtures__/test-helpers.ts +++ b/tests_integ/__fixtures__/test-helpers.ts @@ -1,5 +1,6 @@ -import { readFileSync } from 'node:fs' -import { join } from 'node:path' +export const isInBrowser = () => { + return globalThis?.process?.env == null +} /** * Helper to load fixture files from Vite URL imports. @@ -8,45 +9,16 @@ import { join } from 'node:path' * @param url - The URL from a Vite ?url import * @returns The file contents as a Uint8Array */ -export const loadFixture = (url: string): Uint8Array => { - const relativePath = url.startsWith('/') ? url.slice(1) : url - const filePath = join(process.cwd(), relativePath) - return new Uint8Array(readFileSync(filePath)) -} - -/** - * Determines if OpenAI integration tests should be skipped. - * In CI environments, throws an error if API key is missing (tests should not be skipped). - * In local development, skips tests if API key is not available. - * - * @returns true if tests should be skipped, false if they should run - * @throws Error if running in CI and API key is missing - */ -export const shouldSkipOpenAITests = (): boolean => { - try { - const isCI = !!process.env.CI - const hasKey = !!process.env.OPENAI_API_KEY - - if (isCI && !hasKey) { - throw new Error('OpenAI API key must be available in CI environments') - } - - if (hasKey) { - if (isCI) { - console.log('✅ Running in CI environment with OpenAI API key - tests will run') - } else { - console.log('✅ OpenAI API key found for integration tests') - } - return false - } else { - console.log('⏭️ OpenAI API key not available - integration tests will be skipped') - return true - } - } catch (error) { - if (error instanceof Error && error.message.includes('CI environments')) { - throw error - } - console.log('⏭️ OpenAI API key not available - integration tests will be skipped') - return true +export const loadFixture = async (url: string): Promise => { + if (isInBrowser()) { + const response = await globalThis.fetch(url) + const arrayBuffer = await response.arrayBuffer() + return new Uint8Array(arrayBuffer) + } else { + const { join } = await import('node:path') + const { readFile } = await import('node:fs/promises') + const relativePath = url.startsWith('/') ? url.slice(1) : url + const filePath = join(process.cwd(), relativePath) + return new Uint8Array(await readFile(filePath)) } } diff --git a/tests_integ/agent.test.ts b/tests_integ/agent.test.ts index 8fb9ac19..9d0fb7f6 100644 --- a/tests_integ/agent.test.ts +++ b/tests_integ/agent.test.ts @@ -1,15 +1,16 @@ import { describe, it, expect } from 'vitest' -import { Agent, DocumentBlock, ImageBlock, Message, TextBlock, tool } from '@strands-agents/sdk' -import { BedrockModel } from '@strands-agents/sdk/bedrock' +import { Agent, DocumentBlock, ImageBlock, Message, TextBlock, tool, type Model } from '@strands-agents/sdk' import { notebook } from '@strands-agents/sdk/vended_tools/notebook' import { httpRequest } from '@strands-agents/sdk/vended_tools/http_request' -import { OpenAIModel } from '@strands-agents/sdk/openai' import { z } from 'zod' - -// eslint-disable-next-line no-restricted-imports -import { collectGenerator } from '../src/__fixtures__/model-test-helpers.js' -import { shouldRunTests } from './__fixtures__/model-test-helpers.js' -import { loadFixture, shouldSkipOpenAITests } from './__fixtures__/test-helpers.js' +import { + createBedrockModel, + createOpenAIModel, + shouldSkipBedrockTests, + collectGenerator, + shouldSkipOpenAITests, +} from './__fixtures__/model-test-helpers.js' +import { loadFixture } from './__fixtures__/test-helpers.js' // Import fixtures using Vite's ?url suffix import yellowPngUrl from './__resources__/yellow.png?url' @@ -38,18 +39,22 @@ const calculatorTool = tool({ const providers = [ { name: 'BedrockModel', - skip: !(await shouldRunTests()), - createModel: () => new BedrockModel(), + skip: shouldSkipBedrockTests, + createModel: () => createBedrockModel(), }, { name: 'OpenAIModel', - skip: shouldSkipOpenAITests(), - createModel: () => new OpenAIModel(), + skip: shouldSkipOpenAITests, + createModel: () => createOpenAIModel(), }, -] - -describe.each(providers)('Agent with $name', ({ name, skip, createModel }) => { - describe.skipIf(skip)(`${name} Integration Tests`, () => { +] satisfies Array<{ + name: string + skip: () => Promise + createModel: () => Model +}> + +describe.each(providers)('Agent with $name', async ({ name, skip, createModel }) => { + describe(`${name} Integration Tests`, { skip: await skip(), timeout: 60000 }, () => { describe('Basic Functionality', () => { it('handles invocation, streaming, system prompts, and tool use', async () => { // Test basic invocation with system prompt and tool @@ -123,14 +128,14 @@ describe.each(providers)('Agent with $name', ({ name, skip, createModel }) => { expect(agent.messages).toHaveLength(4) // 2 user + 2 assistant // Verify message ordering - expect(agent.messages[0].role).toBe('user') - expect(agent.messages[1].role).toBe('assistant') - expect(agent.messages[2].role).toBe('user') - expect(agent.messages[3].role).toBe('assistant') + expect(agent.messages[0]?.role).toBe('user') + expect(agent.messages[1]?.role).toBe('assistant') + expect(agent.messages[2]?.role).toBe('user') + expect(agent.messages[3]?.role).toBe('assistant') // Verify conversation context is preserved - const lastMessage = agent.messages[agent.messages.length - 1] - const textContent = lastMessage.content.find((block) => block.type === 'textBlock') + const lastMessage = agent.messages.at(-1) + const textContent = lastMessage?.content.find((block) => block.type === 'textBlock') expect(textContent?.text).toMatch(/Alice/i) }) }) @@ -145,7 +150,7 @@ describe.each(providers)('Agent with $name', ({ name, skip, createModel }) => { }) // Create image block - const imageBytes = loadFixture(yellowPngUrl) + const imageBytes = await loadFixture(yellowPngUrl) const imageBlock = new ImageBlock({ format: 'png', source: { bytes: imageBytes }, @@ -170,7 +175,7 @@ describe.each(providers)('Agent with $name', ({ name, skip, createModel }) => { printer: false, }) - const result = await agent.invoke() + const result = await agent.invoke([]) expect(result.stopReason).toBe('endTurn') expect(result.lastMessage.role).toBe('assistant') @@ -243,7 +248,7 @@ describe.each(providers)('Agent with $name', ({ name, skip, createModel }) => { it('handles tool invocation', async () => { const agent = new Agent({ - model: await createModel(), + model: createModel(), tools: [notebook, httpRequest], printer: false, }) diff --git a/tests_integ/bash.test.ts b/tests_integ/bash.node.test.ts similarity index 80% rename from tests_integ/bash.test.ts rename to tests_integ/bash.node.test.ts index 85355384..9dffc2cc 100644 --- a/tests_integ/bash.test.ts +++ b/tests_integ/bash.node.test.ts @@ -1,12 +1,12 @@ -/* eslint-disable no-restricted-imports */ import { describe, it, expect } from 'vitest' -import { Agent, BedrockModel } from '../src/index.js' -import { bash } from '../vended_tools/bash/index.js' -import { getMessageText, shouldRunTests } from './__fixtures__/model-test-helpers.js' +import { Agent, BedrockModel } from '@strands-agents/sdk' +import { bash } from '@strands-agents/sdk/vended_tools/bash' +import { getMessageText, shouldSkipBedrockTests } from './__fixtures__/model-test-helpers.js' +import { isInBrowser } from './__fixtures__/test-helpers.js' -describe.skipIf(!(await shouldRunTests()) || process.platform === 'win32')( +describe.skipIf((await shouldSkipBedrockTests()) || globalThis.process.platform === 'win32')( 'Bash Tool Integration', - { timeout: 60000 }, + { timeout: 60000, skip: isInBrowser() }, () => { // Shared agent configuration for all tests const createAgent = () => diff --git a/tests_integ/bedrock.test.ts b/tests_integ/bedrock.test.ts index f584aa16..ddefe1bb 100644 --- a/tests_integ/bedrock.test.ts +++ b/tests_integ/bedrock.test.ts @@ -1,6 +1,5 @@ import { describe, it, expect, vi } from 'vitest' import { - BedrockModel, Message, Agent, TextBlock, @@ -8,15 +7,14 @@ import { SlidingWindowConversationManager, } from '@strands-agents/sdk' -// eslint-disable-next-line no-restricted-imports -import { collectIterator } from '../src/__fixtures__/model-test-helpers.js' -import { shouldRunTests } from './__fixtures__/model-test-helpers.js' +import { createBedrockModel, shouldSkipBedrockTests, collectIterator } from './__fixtures__/model-test-helpers.js' +import { isInBrowser } from './__fixtures__/test-helpers' -describe.skipIf(!(await shouldRunTests()))('BedrockModel Integration Tests', () => { +describe.skipIf(await shouldSkipBedrockTests())('BedrockModel Integration Tests', () => { describe('Streaming', () => { describe('Configuration', () => { it.concurrent('respects maxTokens configuration', async () => { - const provider = new BedrockModel({ maxTokens: 20 }) + const provider = createBedrockModel({ maxTokens: 20 }) const messages: Message[] = [ { type: 'message', @@ -35,7 +33,7 @@ describe.skipIf(!(await shouldRunTests()))('BedrockModel Integration Tests', () }) it.concurrent('uses system prompt cache on subsequent requests', async () => { - const provider = new BedrockModel({ maxTokens: 100 }) + const provider = createBedrockModel({ maxTokens: 100 }) const largeContext = `Context information: ${'hello '.repeat(2000)} [test-${Date.now()}-${Math.random()}]` const cachedSystemPrompt = [ { type: 'textBlock' as const, text: 'You are a helpful assistant.' }, @@ -63,7 +61,7 @@ describe.skipIf(!(await shouldRunTests()))('BedrockModel Integration Tests', () }) it.concurrent('uses message cache points on subsequent requests', async () => { - const provider = new BedrockModel({ maxTokens: 100 }) + const provider = createBedrockModel({ maxTokens: 100 }) const largeContext = `Context information: ${'hello '.repeat(2000)} [test-${Date.now()}-${Math.random()}]` const messagesWithCachePoint = (text: string): Message[] => [ { @@ -91,7 +89,7 @@ describe.skipIf(!(await shouldRunTests()))('BedrockModel Integration Tests', () describe('Error Handling', () => { it.concurrent('handles invalid model ID gracefully', async () => { - const provider = new BedrockModel({ modelId: 'invalid-model-id-that-does-not-exist' }) + const provider = createBedrockModel({ modelId: 'invalid-model-id-that-does-not-exist' }) const messages: Message[] = [{ type: 'message', role: 'user', content: [{ type: 'textBlock', text: 'Hello' }] }] await expect(collectIterator(provider.stream(messages))).rejects.toThrow() }) @@ -101,7 +99,7 @@ describe.skipIf(!(await shouldRunTests()))('BedrockModel Integration Tests', () describe('Agent with Conversation Manager', () => { it('manages conversation history with SlidingWindowConversationManager', async () => { const agent = new Agent({ - model: new BedrockModel({ maxTokens: 100 }), + model: createBedrockModel({ maxTokens: 100 }), conversationManager: new SlidingWindowConversationManager({ windowSize: 4 }), }) @@ -122,7 +120,7 @@ describe.skipIf(!(await shouldRunTests()))('BedrockModel Integration Tests', () it('throws ContextWindowOverflowError with NullConversationManager', async () => { const agent = new Agent({ - model: new BedrockModel({ maxTokens: 50 }), + model: createBedrockModel({ maxTokens: 50 }), conversationManager: new NullConversationManager(), }) @@ -136,7 +134,7 @@ describe.skipIf(!(await shouldRunTests()))('BedrockModel Integration Tests', () describe('Region Configuration', () => { it('uses explicit region when provided', async () => { - const provider = new BedrockModel({ + const provider = createBedrockModel({ region: 'us-east-1', maxTokens: 50, }) @@ -152,7 +150,7 @@ describe.skipIf(!(await shouldRunTests()))('BedrockModel Integration Tests', () vi.stubEnv('AWS_REGION', undefined) vi.stubEnv('AWS_DEFAULT_REGION', undefined) - const provider = new BedrockModel({ + const provider = createBedrockModel({ maxTokens: 50, }) @@ -172,11 +170,11 @@ describe.skipIf(!(await shouldRunTests()))('BedrockModel Integration Tests', () ) }) - it('uses AWS_REGION environment variable when set', async () => { + it('uses AWS_REGION environment variable when set', { skip: isInBrowser() }, async () => { // Use vitest to stub the environment variable vi.stubEnv('AWS_REGION', 'eu-central-1') - const provider = new BedrockModel({ + const provider = createBedrockModel({ maxTokens: 50, }) @@ -190,7 +188,7 @@ describe.skipIf(!(await shouldRunTests()))('BedrockModel Integration Tests', () // Use vitest to stub the environment variable vi.stubEnv('AWS_REGION', 'eu-west-1') - const provider = new BedrockModel({ + const provider = createBedrockModel({ region: 'ap-southeast-2', maxTokens: 50, }) @@ -202,7 +200,7 @@ describe.skipIf(!(await shouldRunTests()))('BedrockModel Integration Tests', () }) it('uses region from clientConfig when provided', async () => { - const provider = new BedrockModel({ + const provider = createBedrockModel({ clientConfig: { region: 'ap-northeast-1' }, maxTokens: 50, }) @@ -214,7 +212,7 @@ describe.skipIf(!(await shouldRunTests()))('BedrockModel Integration Tests', () }) }) - describe('Agent with String Model ID', () => { + describe('Agent with String Model ID', { skip: isInBrowser() }, () => { it.concurrent('accepts string model ID and creates functional Agent', async () => { // Create agent with string model ID const agent = new Agent({ diff --git a/tests_integ/browser/agent.browser.test.ts b/tests_integ/browser/agent.browser.test.ts deleted file mode 100644 index 675c2b5a..00000000 --- a/tests_integ/browser/agent.browser.test.ts +++ /dev/null @@ -1,166 +0,0 @@ -import { describe, it, expect } from 'vitest' -import { commands } from 'vitest/browser' -import { Agent, DocumentBlock, ImageBlock, Message, TextBlock, tool } from '@strands-agents/sdk' -import { BedrockModel } from '@strands-agents/sdk/bedrock' -import { OpenAIModel } from '@strands-agents/sdk/openai' -import { notebook } from '@strands-agents/sdk/vended_tools/notebook' -import { httpRequest } from '@strands-agents/sdk/vended_tools/http_request' -import { z } from 'zod' - -import { collectGenerator } from '../../src/__fixtures__/model-test-helpers.js' - -// Import fixtures -import yellowPngUrl from '../__resources__/yellow.png?url' - -// Environment detection for browser vs Node.js -const isNode = typeof process !== 'undefined' && typeof process.versions !== 'undefined' && !!process.versions.node - -// Browser-compatible fixture loader -const loadFixture = async (url: string): Promise => { - if (isNode) { - // In Node.js, use synchronous file reading - const { readFileSync } = await import('node:fs') - const { join } = await import('node:path') - const relativePath = url.startsWith('/') ? url.slice(1) : url - const filePath = join(process.cwd(), relativePath) - return new Uint8Array(readFileSync(filePath)) - } else { - // In browser, use fetch API - const response = await globalThis.fetch(url) - const arrayBuffer = await response.arrayBuffer() - return new Uint8Array(arrayBuffer) - } -} - -// Calculator tool for testing -const calculatorTool = tool({ - name: 'calculator', - description: 'Performs basic arithmetic operations', - inputSchema: z.object({ - operation: z.enum(['add', 'subtract', 'multiply', 'divide']), - a: z.number(), - b: z.number(), - }), - callback: async ({ operation, a, b }) => { - const ops = { - add: a + b, - subtract: a - b, - multiply: a * b, - divide: a / b, - } - return `Result: ${ops[operation]}` - }, -}) - -// Provider configurations with browser credential handling -const providers = [ - { - name: 'BedrockModel', - createModel: async () => { - const credentials = await commands.getAwsCredentials() - return new BedrockModel({ - region: 'us-east-1', - clientConfig: { - credentials, - }, - }) - }, - }, - { - name: 'OpenAIModel', - createModel: async () => - new OpenAIModel({ - apiKey: await commands.getOpenAIAPIKey(), - clientConfig: { - dangerouslyAllowBrowser: true, - }, - }), - }, -] - -describe.each(providers)('Agent Browser Tests with $name', async ({ name, createModel }) => { - describe(`${name} Browser Integration`, () => { - it('handles basic invocation', async () => { - const agent = new Agent({ model: await createModel(), printer: false }) - const result = await agent.invoke('Say hello in one word') - - expect(result.stopReason).toBe('endTurn') - expect(result.lastMessage.role).toBe('assistant') - expect(result.lastMessage.content.length).toBeGreaterThan(0) - }) - - it('handles tool use', async () => { - const agent = new Agent({ - model: await createModel(), - printer: false, - systemPrompt: 'Use the calculator tool to solve math problems. Respond with only the numeric result.', - tools: [calculatorTool], - }) - - const { result } = await collectGenerator(agent.stream('What is 123 * 456?')) - - // Verify tool was used - const toolUseMessage = agent.messages.find((msg) => msg.content.some((block) => block.type === 'toolUseBlock')) - expect(toolUseMessage).toBeDefined() - - // Verify final response - expect(result.stopReason).toBe('endTurn') - expect(result.lastMessage.role).toBe('assistant') - }) - - it('handles media blocks', async () => { - const docBlock = new DocumentBlock({ - name: 'test-document', - format: 'txt', - source: { text: 'The document contains the word ZEBRA.' }, - }) - - const imageBytes = await loadFixture(yellowPngUrl) - const imageBlock = new ImageBlock({ - format: 'png', - source: { bytes: imageBytes }, - }) - - const agent = new Agent({ - model: await createModel(), - messages: [ - new Message({ - role: 'user', - content: [ - docBlock, - imageBlock, - new TextBlock('What animal is in the document and what color is the image? Answer briefly.'), - ], - }), - ], - printer: false, - }) - - const result = await agent.invoke('Answer the question!') - - expect(result.stopReason).toBe('endTurn') - expect(result.lastMessage.role).toBe('assistant') - }) - }) - - it('handles tool invocation', async () => { - const agent = new Agent({ - model: await createModel(), - tools: [notebook, httpRequest], - printer: false, - }) - - await agent.invoke('Call Open-Meteo to get the weather in NYC, and take a note of it') - - expect( - agent.messages.some((message) => - message.content.some((block) => block.type == 'toolUseBlock' && block.name == 'notebook') - ) - ).toBe(true) - expect( - agent.messages.some((message) => - message.content.some((block) => block.type == 'toolUseBlock' && block.name == 'http_request') - ) - ).toBe(true) - }) -}) diff --git a/tests_integ/browser/bedrock.browser.test.ts b/tests_integ/browser/bedrock.browser.test.ts deleted file mode 100644 index 35dab68f..00000000 --- a/tests_integ/browser/bedrock.browser.test.ts +++ /dev/null @@ -1,62 +0,0 @@ -import { describe, it, expect } from 'vitest' -import { BedrockModel } from '@strands-agents/sdk/bedrock' -import { Message, TextBlock } from '@strands-agents/sdk' -import { commands } from 'vitest/browser' -import { collectIterator } from '../../src/__fixtures__/model-test-helpers' - -describe('Region Configuration', () => { - const sayHighMessage = Message.fromMessageData({ - role: 'user', - content: [new TextBlock('say hi')], - }) - - it('uses explicit region when provided', async () => { - const provider = new BedrockModel({ - region: 'us-east-1', - maxTokens: 50, - clientConfig: { - credentials: await commands.getAwsCredentials(), - }, - }) - - // Validate region configuration by checking config.region() directly - // Making an actual request doesn't guarantee the correct region is being used - const regionResult = await provider['_client'].config.region() - expect(regionResult).toBe('us-east-1') - - // ensure that invocation works - await collectIterator(provider.stream([sayHighMessage])) - }) - - it('defaults to us-west-2 when no region provided and AWS SDK does not resolve one', async () => { - const provider = new BedrockModel({ - maxTokens: 50, - clientConfig: { - credentials: await commands.getAwsCredentials(), - }, - }) - - // Validate region defaults to us-west-2 - // Making an actual request doesn't guarantee the correct region is being used - const regionResult = await provider['_client'].config.region() - expect(regionResult).toBe('us-west-2') - - // ensure that invocation works - await collectIterator(provider.stream([sayHighMessage])) - }) - - it('uses region from clientConfig when provided', async () => { - const provider = new BedrockModel({ - clientConfig: { region: 'ap-northeast-1', credentials: await commands.getAwsCredentials() }, - maxTokens: 50, - }) - - // Validate clientConfig region is used - // Making an actual request doesn't guarantee the correct region is being used - const regionResult = await provider['_client'].config.region() - expect(regionResult).toBe('ap-northeast-1') - - // ensure that invocation works - await collectIterator(provider.stream([sayHighMessage])) - }) -}) diff --git a/tests_integ/browser/environment.browser.test.ts b/tests_integ/browser/environment.browser.test.ts deleted file mode 100644 index 2b7e602b..00000000 --- a/tests_integ/browser/environment.browser.test.ts +++ /dev/null @@ -1,64 +0,0 @@ -import { describe, it, expect } from 'vitest' -import { isBrowser, isNode } from '../../src/__fixtures__/environment.js' - -describe('environment', () => { - describe('Browser compatibility', () => { - describe('when running in browser', () => { - it('isNode should resolve to false', () => { - expect(isNode).toBe(false) - }) - it('has window object with expected properties', () => { - expect(window).toBeDefined() - expect(typeof window).toBe('object') - expect(window.location).toBeDefined() - expect(window.navigator).toBeDefined() - }) - - it('has document object with DOM methods', () => { - expect(document).toBeDefined() - expect(typeof document).toBe('object') - expect(typeof document.createElement).toBe('function') - expect(typeof document.querySelector).toBe('function') - }) - - it('has navigator object with browser information', () => { - expect(navigator).toBeDefined() - expect(typeof navigator).toBe('object') - expect(typeof navigator.userAgent).toBe('string') - expect(navigator.userAgent.length).toBeGreaterThan(0) - }) - }) - - describe('environment detection', () => { - it('correctly identifies browser environment', () => { - expect(isBrowser).toBe(true) - expect(typeof window).toBe('object') - }) - }) - }) - - describe('JavaScript features', () => { - it('supports modern JavaScript features', () => { - // Test ES2022 features work - const testArray = [1, 2, 3] - const lastElement = testArray.at(-1) - expect(lastElement).toBe(3) - }) - - it('supports async/await functionality', async () => { - // Test async functionality works - const promise = Promise.resolve('test') - const result = await promise - expect(result).toBe('test') - }) - }) - - describe('TypeScript configuration', () => { - it('validates strict typing environment', () => { - // This test validates strict TypeScript configuration - // If this compiles and runs, strict typing is working - const testValue: string = 'test' - expect(typeof testValue).toBe('string') - }) - }) -}) diff --git a/tests_integ/environment.test.ts b/tests_integ/environment.test.ts index 1598c526..b7ba70cd 100644 --- a/tests_integ/environment.test.ts +++ b/tests_integ/environment.test.ts @@ -1,22 +1,61 @@ import { describe, it, expect } from 'vitest' // eslint-disable-next-line no-restricted-imports -import { isNode } from '../src/__fixtures__/environment.js' +import { isBrowser, isNode } from '../src/__fixtures__/environment.js' +import { isInBrowser } from './__fixtures__/test-helpers.js' describe('environment', () => { - describe('Node.js compatibility', () => { + describe('Node.js compatibility', { skip: isInBrowser() }, () => { it('works in Node.js environment', () => { // Test Node.js specific features are available expect(typeof process).toBe('object') expect(process.version).toBeDefined() }) - }) - describe('environment detection', () => { it('correctly identifies Node.js environment', () => { expect(isNode).toBe(true) expect(typeof process).toBe('object') }) + + it('correctly identifies browser environment', () => { + expect(isBrowser).toBe(false) + expect(typeof window).toBe('undefined') + }) + }) + + describe('Browser compatibility', { skip: !isInBrowser() }, () => { + describe('when running in browser', () => { + it('isNode should resolve to false', () => { + expect(isNode).toBe(false) + }) + it('has window object with expected properties', () => { + expect(window).toBeDefined() + expect(typeof window).toBe('object') + expect(window.location).toBeDefined() + expect(window.navigator).toBeDefined() + }) + + it('has document object with DOM methods', () => { + expect(document).toBeDefined() + expect(typeof document).toBe('object') + expect(typeof document.createElement).toBe('function') + expect(typeof document.querySelector).toBe('function') + }) + + it('has navigator object with browser information', () => { + expect(navigator).toBeDefined() + expect(typeof navigator).toBe('object') + expect(typeof navigator.userAgent).toBe('string') + expect(navigator.userAgent.length).toBeGreaterThan(0) + }) + }) + + describe('environment detection', () => { + it('correctly identifies browser environment', () => { + expect(isBrowser).toBe(true) + expect(typeof window).toBe('object') + }) + }) }) describe('JavaScript features', () => { diff --git a/tests_integ/file-editor.test.ts b/tests_integ/file-editor.node.test.ts similarity index 97% rename from tests_integ/file-editor.test.ts rename to tests_integ/file-editor.node.test.ts index 29654d42..d7cb81c4 100644 --- a/tests_integ/file-editor.test.ts +++ b/tests_integ/file-editor.node.test.ts @@ -3,12 +3,12 @@ import { describe, it, expect, beforeEach, afterEach } from 'vitest' import { Agent, BedrockModel } from '../src/index.js' import { fileEditor } from '../vended_tools/file_editor/index.js' import { collectGenerator } from '../src/__fixtures__/model-test-helpers.js' -import { shouldRunTests } from './__fixtures__/model-test-helpers.js' +import { shouldSkipBedrockTests } from './__fixtures__/model-test-helpers.js' import { promises as fs } from 'fs' import * as path from 'path' import { tmpdir } from 'os' -describe.skipIf(!(await shouldRunTests()))('FileEditor Tool Integration', () => { +describe.skipIf(await shouldSkipBedrockTests())('FileEditor Tool Integration', () => { let testDir: string // Shared agent configuration for all tests diff --git a/tests_integ/http-request.test.ts b/tests_integ/http-request.test.ts index 26bfaa63..5393c144 100644 --- a/tests_integ/http-request.test.ts +++ b/tests_integ/http-request.test.ts @@ -1,12 +1,12 @@ import { describe, it, expect } from 'vitest' import { httpRequest } from '@strands-agents/sdk/vended_tools/http_request' -import { Agent, BedrockModel } from '@strands-agents/sdk' -import { shouldRunTests } from './__fixtures__/model-test-helpers.js' +import { Agent } from '@strands-agents/sdk' +import { createBedrockModel, shouldSkipBedrockTests } from './__fixtures__/model-test-helpers.js' -describe.skipIf(!(await shouldRunTests()))('httpRequest tool (integration)', () => { +describe.skipIf(await shouldSkipBedrockTests())('httpRequest tool (integration)', () => { it('agent uses http_request tool to fetch weather from Open-Meteo', async () => { const agent = new Agent({ - model: new BedrockModel({ maxTokens: 500 }), + model: createBedrockModel({ maxTokens: 500 }), tools: [httpRequest], printer: false, }) diff --git a/tests_integ/integ-setup.ts b/tests_integ/integ-setup.ts index b9095260..ed5c34ea 100644 --- a/tests_integ/integ-setup.ts +++ b/tests_integ/integ-setup.ts @@ -51,7 +51,13 @@ async function loadApiKeysFromSecretsManager(): Promise { } } +let didSetup = false + export async function setup(): Promise { + if (didSetup) { + return + } + console.log('Global setup: Loading API keys from Secrets Manager...') try { @@ -59,5 +65,7 @@ export async function setup(): Promise { console.log('Global setup complete: API keys loaded into environment') } catch (error) { console.error('Global setup failed:', error) + } finally { + didSetup = true } } diff --git a/tests_integ/mcp.test.ts b/tests_integ/mcp.node.test.ts similarity index 100% rename from tests_integ/mcp.test.ts rename to tests_integ/mcp.node.test.ts diff --git a/tests_integ/notebook.test.ts b/tests_integ/notebook.test.ts index 814cd1e4..c1534028 100644 --- a/tests_integ/notebook.test.ts +++ b/tests_integ/notebook.test.ts @@ -1,17 +1,14 @@ /* eslint-disable no-restricted-imports */ import { describe, it, expect } from 'vitest' -import { Agent, BedrockModel } from '../src/index.js' -import type { AgentStreamEvent, AgentResult } from '../src/index.js' -import { notebook } from '../vended_tools/notebook/index.js' +import { Agent, type AgentStreamEvent, AgentResult } from '@strands-agents/sdk' +import { notebook } from '@strands-agents/sdk/vended_tools/notebook' import { collectGenerator } from '../src/__fixtures__/model-test-helpers.js' -import { shouldRunTests } from './__fixtures__/model-test-helpers.js' +import { createBedrockModel, shouldSkipBedrockTests } from './__fixtures__/model-test-helpers.js' -describe.skipIf(!(await shouldRunTests()))('Notebook Tool Integration', () => { +describe.skipIf(await shouldSkipBedrockTests())('Notebook Tool Integration', () => { // Shared agent configuration for all tests const agentParams = { - model: new BedrockModel({ - region: 'us-east-1', - }), + model: createBedrockModel(), tools: [notebook], } diff --git a/tests_integ/openai.test.ts b/tests_integ/openai.test.ts index 5cf8960b..73aaa5b0 100644 --- a/tests_integ/openai.test.ts +++ b/tests_integ/openai.test.ts @@ -1,16 +1,13 @@ import { describe, it, expect } from 'vitest' -import { OpenAIModel } from '@strands-agents/sdk/openai' import { Message } from '@strands-agents/sdk' import type { ToolSpec } from '@strands-agents/sdk' -// eslint-disable-next-line no-restricted-imports -import { collectIterator } from '../src/__fixtures__/model-test-helpers.js' -import { shouldSkipOpenAITests } from './__fixtures__/test-helpers.js' +import { collectIterator, createOpenAIModel, shouldSkipOpenAITests } from './__fixtures__/model-test-helpers.js' -describe.skipIf(shouldSkipOpenAITests())('OpenAIModel Integration Tests', () => { +describe.skipIf(await shouldSkipOpenAITests())('OpenAIModel Integration Tests', () => { describe('Configuration', () => { it.concurrent('respects maxTokens configuration', async () => { - const provider = new OpenAIModel({ + const provider = createOpenAIModel({ modelId: 'gpt-4o-mini', maxTokens: 20, // Very small limit }) @@ -34,7 +31,7 @@ describe.skipIf(shouldSkipOpenAITests())('OpenAIModel Integration Tests', () => }) it.concurrent('respects temperature configuration', async () => { - const provider = new OpenAIModel({ + const provider = createOpenAIModel({ modelId: 'gpt-4o-mini', temperature: 0, // Deterministic maxTokens: 50, @@ -77,7 +74,7 @@ describe.skipIf(shouldSkipOpenAITests())('OpenAIModel Integration Tests', () => describe('Error Handling', () => { it.concurrent('handles invalid model ID gracefully', async () => { - const provider = new OpenAIModel({ + const provider = createOpenAIModel({ modelId: 'invalid-model-id-that-does-not-exist-xyz', }) @@ -99,7 +96,7 @@ describe.skipIf(shouldSkipOpenAITests())('OpenAIModel Integration Tests', () => describe('Content Block Lifecycle', () => { it.concurrent('emits complete content block lifecycle events', async () => { - const provider = new OpenAIModel({ + const provider = createOpenAIModel({ modelId: 'gpt-4o-mini', maxTokens: 50, }) @@ -139,7 +136,7 @@ describe.skipIf(shouldSkipOpenAITests())('OpenAIModel Integration Tests', () => describe('Stop Reasons', () => { it.concurrent('returns endTurn stop reason for natural completion', async () => { - const provider = new OpenAIModel({ + const provider = createOpenAIModel({ modelId: 'gpt-4o-mini', maxTokens: 100, }) @@ -159,7 +156,7 @@ describe.skipIf(shouldSkipOpenAITests())('OpenAIModel Integration Tests', () => }) it.concurrent('returns maxTokens stop reason when token limit reached', async () => { - const provider = new OpenAIModel({ + const provider = createOpenAIModel({ modelId: 'gpt-4o-mini', maxTokens: 10, // Very small limit to force cutoff }) @@ -179,7 +176,7 @@ describe.skipIf(shouldSkipOpenAITests())('OpenAIModel Integration Tests', () => }) it.concurrent('returns toolUse stop reason when requesting tool use', async () => { - const provider = new OpenAIModel({ + const provider = createOpenAIModel({ modelId: 'gpt-4o-mini', maxTokens: 200, }) diff --git a/vitest.config.ts b/vitest.config.ts index 08a531cd..8664955a 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -1,8 +1,9 @@ import { defineConfig } from 'vitest/config' import { playwright } from '@vitest/browser-playwright' -import { AwsCredentialIdentity } from '@aws-sdk/types' +import { type AwsCredentialIdentity } from '@aws-sdk/types' import { fromNodeProviderChain } from '@aws-sdk/credential-providers' -import { BrowserCommand } from 'vitest/node' +import { type BrowserCommand } from 'vitest/node' +import { setup } from './tests_integ/integ-setup' // Conditionally exclude bash tool from coverage on Windows // since tests are skipped on Windows (bash not available) @@ -12,19 +13,27 @@ if (process.platform === 'win32') { } const getAwsCredentials: BrowserCommand<[], AwsCredentialIdentity> = async ({ testPath, provider }) => { + await setup() const credentialProvider = fromNodeProviderChain() return await credentialProvider() } const getOpenAIAPIKey: BrowserCommand<[], string | undefined> = async ({ testPath, provider }) => { + await setup() return process.env.OPENAI_API_KEY } export default defineConfig({ test: { unstubEnvs: true, + reporters: [ + 'default', + ['junit', { outputFile: 'test/.artifacts/test-report/junit/report.xml' }], + ['json', { outputFile: 'test/.artifacts/test-report/json/report.json' }], + ], projects: [ { + // Unit Tests (node) test: { include: ['src/**/__tests__/**/*.test.ts', 'vended_tools/**/__tests__/**/*.test.ts'], includeSource: ['src/**/*.{js,ts}'], @@ -36,13 +45,16 @@ export default defineConfig({ }, }, { + // Unit Tests (browser) test: { include: ['src/**/__tests__/**/*.test.ts', 'vended_tools/**/__tests__/**/*.test.ts'], exclude: ['vended_tools/file_editor/**/*.test.ts', 'vended_tools/bash/**/*.test.ts'], name: { label: 'unit-browser', color: 'cyan' }, browser: { enabled: true, + headless: true, provider: playwright(), + screenshotDirectory: 'test/.artifacts/browser-screenshots/', instances: [ { browser: 'chromium', @@ -52,6 +64,7 @@ export default defineConfig({ }, }, { + // Integ Tests (Node) test: { include: ['tests_integ/**/*.test.ts'], exclude: ['tests_integ/**/*.browser.test.ts'], @@ -65,13 +78,17 @@ export default defineConfig({ }, }, { + // Integ Tests (browser) test: { - include: ['tests_integ/**/*.browser.test.ts'], + include: ['tests_integ/**/*.test.ts'], + exclude: ['**/*.node.test.ts'], name: { label: 'integ-browser', color: 'yellow' }, testTimeout: 30000, browser: { enabled: true, + headless: true, provider: playwright(), + screenshotDirectory: 'test/.artifacts/browser-screenshots/', instances: [ { browser: 'chromium', @@ -97,7 +114,7 @@ export default defineConfig({ coverage: { provider: 'v8', reporter: ['text', 'json', 'html'], - include: ['src/**/*', 'vended_tools/**/*'], + include: ['src/**/*.{ts,js}', 'vended_tools/**/*.{ts,js}'], exclude: coverageExclude, thresholds: { lines: 80,