diff --git a/Makefile b/Makefile index a374c22db..be93e1de9 100644 --- a/Makefile +++ b/Makefile @@ -4,8 +4,7 @@ integration-setup: integration-cleanup .PHONY: integration-cleanup integration-cleanup: - docker stop instance || true - docker volume rm instance-rest-test-data || true + docker container rm --force --volumes instance || true .PHONY: integration integration: integration-setup diff --git a/test/integration/README.md b/test/integration/README.md index a52ae2e54..36b011975 100644 --- a/test/integration/README.md +++ b/test/integration/README.md @@ -37,7 +37,7 @@ make integration If Elasticsearch doesn't come up, run `make integration-cleanup` and then `DETACH=false .ci/run-elasticsearch.sh` manually to read the startup logs. -If you get an error about `vm.max_map_count` being too low, run `sudo sysctl -w vm.max_map_count=262144` to update the setting until the next reboot, or `sudo sysctl -w vm.max_map_count=262144 | sudo tee -a /etc/sysctl.conf` to update the setting permanently. +If you get an error about `vm.max_map_count` being too low, run `sudo sysctl -w vm.max_map_count=262144` to update the setting until the next reboot, or `sudo sysctl -w vm.max_map_count=262144; echo 'vm.max_map_count=262144' | sudo tee -a /etc/sysctl.conf` to update the setting permanently. ### Exit on the first failure diff --git a/test/integration/index.js b/test/integration/index.js index 5c4addcb7..5f92eee21 100644 --- a/test/integration/index.js +++ b/test/integration/index.js @@ -30,7 +30,6 @@ const yaml = require('js-yaml') const minimist = require('minimist') const ms = require('ms') const { Client } = require('../../index') -const { kProductCheck } = require('@elastic/transport/lib/symbols') const build = require('./test-runner') const { sleep } = require('./helper') const createJunitReporter = require('./reporter') @@ -44,7 +43,8 @@ const MAX_FILE_TIME = 1000 * 30 const MAX_TEST_TIME = 1000 * 3 const options = minimist(process.argv.slice(2), { - boolean: ['bail'] + boolean: ['bail'], + string: ['suite', 'test'], }) const freeSkips = { @@ -56,10 +56,6 @@ const freeSkips = { '/free/cluster.desired_nodes/20_dry_run.yml': ['*'], '/free/cluster.prevalidate_node_removal/10_basic.yml': ['*'], - '/free/health/30_feature.yml': ['*'], - '/free/health/40_useractions.yml': ['*'], - '/free/health/40_diagnosis.yml': ['Diagnosis'], - // the v8 client never sends the scroll_id in querystring, // the way the test is structured causes a security exception 'free/scroll/10_basic.yml': ['Body params override query string'], @@ -70,9 +66,6 @@ const freeSkips = { 'free/cat.allocation/10_basic.yml': ['*'], 'free/cat.snapshots/10_basic.yml': ['Test cat snapshots output'], - // TODO: remove this once 'arbitrary_key' is implemented - // https://github.com/elastic/elasticsearch/pull/41492 - 'indices.split/30_copy_settings.yml': ['*'], 'indices.stats/50_disk_usage.yml': ['Disk usage stats'], 'indices.stats/60_field_usage.yml': ['Field usage stats'], @@ -99,22 +92,6 @@ const platinumDenyList = { 'api_key/11_invalidation.yml': ['Test invalidate api key by realm name'], 'analytics/histogram.yml': ['Histogram requires values in increasing order'], - // this two test cases are broken, we should - // return on those in the future. - 'analytics/top_metrics.yml': [ - 'sort by keyword field fails', - 'sort by string script fails' - ], - - 'cat.aliases/10_basic.yml': ['Empty cluster'], - 'index/10_with_id.yml': ['Index with ID'], - 'indices.get_alias/10_basic.yml': ['Get alias against closed indices'], - 'indices.get_alias/20_empty.yml': ['Check empty aliases when getting all aliases via /_alias'], - 'text_structure/find_structure.yml': ['*'], - - // https://github.com/elastic/elasticsearch/pull/39400 - 'ml/jobs_crud.yml': ['Test put job with id that is already taken'], - // object keys must me strings, and `0.0.toString()` is `0` 'ml/evaluate_data_frame.yml': [ 'Test binary_soft_classifition precision', @@ -122,12 +99,6 @@ const platinumDenyList = { 'Test binary_soft_classifition confusion_matrix' ], - // it gets random failures on CI, must investigate - 'ml/set_upgrade_mode.yml': [ - 'Attempt to open job when upgrade_mode is enabled', - 'Setting upgrade mode to disabled from enabled' - ], - // The cleanup fails with a index not found when retrieving the jobs 'ml/get_datafeed_stats.yml': ['Test get datafeed stats when total_search_time_ms mapping is missing'], 'ml/bucket_correlation_agg.yml': ['Test correlation bucket agg simple'], @@ -209,13 +180,6 @@ const platinumDenyList = { // start should be a string in the yaml test 'platinum/ml/start_stop_datafeed.yml': ['*'], - - // health API not yet supported - '/platinum/health/10_usage.yml': ['*'], - - // ML update_trained_model_deployment not supported yet - '/platinum/ml/3rd_party_deployment.yml': ['Test update deployment'], - '/platinum/ml/update_trained_model_deployment.yml': ['Test with unknown model id'] } function runner (opts = {}) { @@ -227,8 +191,6 @@ function runner (opts = {}) { } } const client = new Client(options) - // TODO: remove the following line once https://github.com/elastic/elasticsearch/issues/82358 is fixed - client.transport[kProductCheck] = null log('Loading yaml suite') start({ client, isXPack: opts.isXPack }) .catch(err => { @@ -333,13 +295,21 @@ async function start ({ client, isXPack }) { } const cleanPath = file.slice(file.lastIndexOf(apiName)) + + // skip if --suite CLI arg doesn't match + if (options.suite && !cleanPath.endsWith(options.suite)) continue + log(' ' + cleanPath) const junitTestSuite = junitTestSuites.testsuite(apiName.slice(1) + ' - ' + cleanPath) for (const test of tests) { const testTime = now() const name = Object.keys(test)[0] + + // skip setups, teardowns and anything that doesn't match --test flag when present if (name === 'setup' || name === 'teardown') continue + if (options.test && !name.endsWith(options.test)) continue + const junitTestCase = junitTestSuite.testcase(name) stats.total += 1 @@ -439,6 +409,8 @@ if (require.main === module) { } const shouldSkip = (isXPack, file, name) => { + if (options.suite || options.test) return false + let list = Object.keys(freeSkips) for (let i = 0; i < list.length; i++) { const freeTest = freeSkips[list[i]]