diff --git a/.gitignore b/.gitignore index 455a6987..90a3908c 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,4 @@ *.log .idea package-lock.json - +yarn.lock diff --git a/bin/kcl-bootstrap b/bin/kcl-bootstrap index 6cf5cb37..3377a34d 100755 --- a/bin/kcl-bootstrap +++ b/bin/kcl-bootstrap @@ -7,7 +7,6 @@ SPDX-License-Identifier: Apache-2.0 'use strict'; - var fs = require('fs'); var http = require('http'); var https = require('https'); @@ -19,69 +18,115 @@ var util = require('util'); var MAVEN_PACKAGE_LIST = [ - getMavenPackageInfo('software.amazon.kinesis', 'amazon-kinesis-client-multilang', '2.1.2'), - getMavenPackageInfo('software.amazon.kinesis', 'amazon-kinesis-client', '2.1.2'), - getMavenPackageInfo('software.amazon.awssdk', 'kinesis', '2.4.0'), - getMavenPackageInfo('software.amazon.awssdk', 'aws-cbor-protocol', '2.4.0'), - getMavenPackageInfo('com.fasterxml.jackson.dataformat', 'jackson-dataformat-cbor', '2.9.8'), - getMavenPackageInfo('software.amazon.awssdk', 'aws-json-protocol', '2.4.0'), - getMavenPackageInfo('software.amazon.awssdk', 'dynamodb', '2.4.0'), - getMavenPackageInfo('software.amazon.awssdk', 'cloudwatch', '2.4.0'), - getMavenPackageInfo('software.amazon.awssdk', 'netty-nio-client', '2.4.0'), - getMavenPackageInfo('io.netty', 'netty-codec-http', '4.1.32.Final'), - getMavenPackageInfo('io.netty', 'netty-codec-http2', '4.1.32.Final'), - getMavenPackageInfo('io.netty', 'netty-codec', '4.1.32.Final'), - getMavenPackageInfo('io.netty', 'netty-transport', '4.1.32.Final'), - getMavenPackageInfo('io.netty', 'netty-resolver', '4.1.32.Final'), - getMavenPackageInfo('io.netty', 'netty-common', '4.1.32.Final'), - getMavenPackageInfo('io.netty', 'netty-buffer', '4.1.32.Final'), - getMavenPackageInfo('io.netty', 'netty-handler', '4.1.32.Final'), - getMavenPackageInfo('io.netty', 'netty-transport-native-epoll', '4.1.32.Final'), - getMavenPackageInfo('io.netty', 'netty-transport-native-unix-common', '4.1.32.Final'), - getMavenPackageInfo('com.typesafe.netty', 'netty-reactive-streams-http', '2.0.0'), - getMavenPackageInfo('com.typesafe.netty', 'netty-reactive-streams', '2.0.0'), - getMavenPackageInfo('org.reactivestreams', 'reactive-streams', '1.0.2'), - getMavenPackageInfo('com.google.guava', 'guava', '26.0-jre'), + getMavenPackageInfo('software.amazon.kinesis', 'amazon-kinesis-client-multilang', '2.3.9'), + getMavenPackageInfo('software.amazon.kinesis', 'amazon-kinesis-client', '2.3.9'), + getMavenPackageInfo('software.amazon.awssdk', 'kinesis', '2.17.52'), + getMavenPackageInfo('software.amazon.awssdk', 'aws-cbor-protocol', '2.17.52'), + getMavenPackageInfo('com.fasterxml.jackson.dataformat', 'jackson-dataformat-cbor', '2.12.3'), + getMavenPackageInfo('software.amazon.awssdk', 'aws-json-protocol', '2.17.52'), + getMavenPackageInfo('software.amazon.awssdk', 'dynamodb', '2.17.52'), + getMavenPackageInfo('software.amazon.awssdk', 'cloudwatch', '2.17.52'), + getMavenPackageInfo('software.amazon.awssdk', 'netty-nio-client', '2.17.52'), + getMavenPackageInfo('io.netty', 'netty-codec-http', '4.1.68.Final'), + getMavenPackageInfo('io.netty', 'netty-codec-http2', '4.1.68.Final'), + getMavenPackageInfo('io.netty', 'netty-codec', '4.1.68.Final'), + getMavenPackageInfo('io.netty', 'netty-transport', '4.1.68.Final'), + getMavenPackageInfo('io.netty', 'netty-resolver', '4.1.68.Final'), + getMavenPackageInfo('io.netty', 'netty-common', '4.1.68.Final'), + getMavenPackageInfo('io.netty', 'netty-buffer', '4.1.68.Final'), + getMavenPackageInfo('io.netty', 'netty-handler', '4.1.68.Final'), + getMavenPackageInfo('io.netty', 'netty-transport-native-epoll', '4.1.68.Final'), + getMavenPackageInfo('io.netty', 'netty-transport-native-unix-common', '4.1.68.Final'), + getMavenPackageInfo('com.typesafe.netty', 'netty-reactive-streams-http', '2.0.5'), + getMavenPackageInfo('com.typesafe.netty', 'netty-reactive-streams', '2.0.5'), + getMavenPackageInfo('org.reactivestreams', 'reactive-streams', '1.0.3'), + getMavenPackageInfo('com.google.guava', 'guava', '30.0-jre'), getMavenPackageInfo('com.google.code.findbugs', 'jsr305', '3.0.2'), - getMavenPackageInfo('org.checkerframework', 'checker-qual', '2.5.2'), - getMavenPackageInfo('com.google.errorprone', 'error_prone_annotations', '2.1.3'), - getMavenPackageInfo('com.google.j2objc', 'j2objc-annotations', '1.1'), - getMavenPackageInfo('org.codehaus.mojo', 'animal-sniffer-annotations', '1.14'), - getMavenPackageInfo('com.google.protobuf', 'protobuf-java', '2.6.1'), + getMavenPackageInfo('org.checkerframework', 'checker-qual', '3.5.0'), + getMavenPackageInfo('com.google.errorprone', 'error_prone_annotations', '2.3.4'), + getMavenPackageInfo('com.google.j2objc', 'j2objc-annotations', '1.3'), + getMavenPackageInfo('org.codehaus.mojo', 'animal-sniffer-annotations', '1.14'), // REMOVE ME + getMavenPackageInfo('com.google.protobuf', 'protobuf-java', '3.11.4'), getMavenPackageInfo('org.apache.commons', 'commons-lang3', '3.8.1'), - getMavenPackageInfo('org.slf4j', 'slf4j-api', '1.7.25'), + getMavenPackageInfo('org.slf4j', 'slf4j-api', '1.7.32'), getMavenPackageInfo('io.reactivex.rxjava2', 'rxjava', '2.1.14'), - getMavenPackageInfo('software.amazon.awssdk', 'sts', '2.4.0'), - getMavenPackageInfo('software.amazon.awssdk', 'aws-query-protocol', '2.4.0'), - getMavenPackageInfo('software.amazon.awssdk', 'protocol-core', '2.4.0'), - getMavenPackageInfo('software.amazon.awssdk', 'profiles', '2.4.0'), - getMavenPackageInfo('software.amazon.awssdk', 'sdk-core', '2.4.0'), - getMavenPackageInfo('com.fasterxml.jackson.core', 'jackson-core', '2.9.8'), - getMavenPackageInfo('com.fasterxml.jackson.core', 'jackson-databind', '2.9.8'), - getMavenPackageInfo('software.amazon.awssdk', 'auth', '2.4.0'), - getMavenPackageInfo('software.amazon', 'flow', '1.7'), - getMavenPackageInfo('software.amazon.awssdk', 'http-client-spi', '2.4.0'), - getMavenPackageInfo('software.amazon.awssdk', 'regions', '2.4.0'), - getMavenPackageInfo('com.fasterxml.jackson.core', 'jackson-annotations', '2.9.0'), - getMavenPackageInfo('software.amazon.awssdk', 'annotations', '2.4.0'), - getMavenPackageInfo('software.amazon.awssdk', 'utils', '2.4.0'), - getMavenPackageInfo('software.amazon.awssdk', 'aws-core', '2.4.0'), - getMavenPackageInfo('software.amazon.awssdk', 'apache-client', '2.4.0'), - getMavenPackageInfo('org.apache.httpcomponents', 'httpclient', '4.5.6'), - getMavenPackageInfo('commons-codec', 'commons-codec', '1.10'), - getMavenPackageInfo('org.apache.httpcomponents', 'httpcore', '4.4.10'), - getMavenPackageInfo('com.amazonaws', 'aws-java-sdk-core', '1.11.477'), - getMavenPackageInfo('commons-logging', 'commons-logging', '1.1.3'), + getMavenPackageInfo('software.amazon.awssdk', 'sts', '2.17.52'), + getMavenPackageInfo('software.amazon.awssdk', 'aws-query-protocol', '2.17.52'), + getMavenPackageInfo('software.amazon.awssdk', 'protocol-core', '2.17.52'), + getMavenPackageInfo('software.amazon.awssdk', 'profiles', '2.17.52'), + getMavenPackageInfo('software.amazon.awssdk', 'sdk-core', '2.17.52'), + getMavenPackageInfo('com.fasterxml.jackson.core', 'jackson-core', '2.12.3'), + getMavenPackageInfo('com.fasterxml.jackson.core', 'jackson-databind', '2.12.3'), + getMavenPackageInfo('software.amazon.awssdk', 'auth', '2.17.97'), + getMavenPackageInfo('software.amazon', 'flow', '1.7'), // REMOVE ME + getMavenPackageInfo('software.amazon.awssdk', 'http-client-spi', '2.17.52'), + getMavenPackageInfo('software.amazon.awssdk', 'regions', '2.17.52'), + getMavenPackageInfo('com.fasterxml.jackson.core', 'jackson-annotations', '2.12.3'), + getMavenPackageInfo('software.amazon.awssdk', 'annotations', '2.17.52'), + getMavenPackageInfo('software.amazon.awssdk', 'utils', '2.17.52'), + getMavenPackageInfo('software.amazon.awssdk', 'aws-core', '2.17.52'), + getMavenPackageInfo('software.amazon.awssdk', 'apache-client', '2.17.52'), + getMavenPackageInfo('org.apache.httpcomponents', 'httpclient', '4.5.13'), + getMavenPackageInfo('commons-codec', 'commons-codec', '1.15'), + getMavenPackageInfo('org.apache.httpcomponents', 'httpcore', '4.4.13'), + getMavenPackageInfo('com.amazonaws', 'aws-java-sdk-core', '1.12.3'), + getMavenPackageInfo('commons-logging', 'commons-logging', '1.2'), getMavenPackageInfo('software.amazon.ion', 'ion-java', '1.0.2'), - getMavenPackageInfo('joda-time', 'joda-time', '2.8.1'), - getMavenPackageInfo('ch.qos.logback', 'logback-classic', '1.2.3'), - getMavenPackageInfo('ch.qos.logback', 'logback-core', '1.2.3'), + getMavenPackageInfo('joda-time', 'joda-time', '2.10.2'), + getMavenPackageInfo('ch.qos.logback', 'logback-classic', '1.2.7'), + getMavenPackageInfo('ch.qos.logback', 'logback-core', '1.2.7'), getMavenPackageInfo('com.beust', 'jcommander', '1.72'), - getMavenPackageInfo('commons-io', 'commons-io', '2.6'), + getMavenPackageInfo('commons-io', 'commons-io', '2.7'), getMavenPackageInfo('org.apache.commons', 'commons-collections4', '4.2'), - getMavenPackageInfo('commons-beanutils', 'commons-beanutils', '1.9.3'), - getMavenPackageInfo('commons-collections', 'commons-collections', '3.2.2') -]; + getMavenPackageInfo('commons-beanutils', 'commons-beanutils', '1.9.4'), + getMavenPackageInfo('commons-collections', 'commons-collections', '3.2.2'), + // add + getMavenPackageInfo('software.amazon.awssdk', 'arns', '2.17.52'), + getMavenPackageInfo('com.amazonaws', 'aws-java-sdk-sts', '1.12.3'), + getMavenPackageInfo('com.amazonaws', 'jmespath-java', '1.12.3'), + getMavenPackageInfo('com.damnhandy', 'handy-uri-templates', '2.1.8'), + // getMavenPackageInfo('com.fasterxml.jackson', 'jackson-bom', '2.12.3'), // ??? + getMavenPackageInfo('com.github.erosb', 'everit-json-schema', '1.12.2'), + getMavenPackageInfo('com.github.luben', 'zstd-jni', '1.4.9-1'), + getMavenPackageInfo('com.google.guava', 'failureaccess', '1.0.1'), + getMavenPackageInfo('com.google.guava', 'listenablefuture', '9999.0-empty-to-avoid-conflict-with-guava'), + getMavenPackageInfo('com.google.re2j', 're2j', '1.3'), + getMavenPackageInfo('com.kjetland', 'mbknor-jackson-jsonschema_2.12', '1.0.39'), + getMavenPackageInfo('commons-digester', 'commons-digester', '1.8.1'), + getMavenPackageInfo('commons-validator', 'commons-validator', '1.6'), + getMavenPackageInfo('io.github.classgraph', 'classgraph', '4.8.120'), + getMavenPackageInfo('javax.validation', 'validation-api', '2.0.1.Final'), + getMavenPackageInfo('org.apache.avro', 'avro', '1.10.2'), + getMavenPackageInfo('org.apache.commons', 'commons-compress', '1.21'), + getMavenPackageInfo('org.apache.kafka', 'kafka-clients', '2.8.1'), + getMavenPackageInfo('org.jetbrains', 'annotations', '13.0'), + getMavenPackageInfo('org.jetbrains.kotlin', 'kotlin-reflect', '1.3.50'), + getMavenPackageInfo('org.jetbrains.kotlin', 'kotlin-script-runtime', '1.3.50'), + getMavenPackageInfo('org.jetbrains.kotlin', 'kotlin-scripting-common', '1.3.50'), + getMavenPackageInfo('org.jetbrains.kotlin', 'kotlin-scripting-compiler-embeddable', '1.3.50'), + getMavenPackageInfo('org.jetbrains.kotlin', 'kotlin-scripting-compiler-impl-embeddable', '1.3.50'), + getMavenPackageInfo('org.jetbrains.kotlin', 'kotlin-scripting-jvm', '1.3.50'), + getMavenPackageInfo('org.jetbrains.kotlin', 'kotlin-stdlib-common', '1.3.50'), + getMavenPackageInfo('org.jetbrains.kotlin', 'kotlin-stdlib', '1.3.50'), + getMavenPackageInfo('org.jetbrains.kotlinx', 'kotlinx-coroutines-core', '1.1.1'), + getMavenPackageInfo('org.json', 'json', '20201115'), + getMavenPackageInfo('org.lz4', 'lz4-java', '1.7.1'), + getMavenPackageInfo('org.projectlombok', 'lombok-utils', '1.18.12'), + getMavenPackageInfo('org.projectlombok', 'lombok', '1.18.20'), + getMavenPackageInfo('org.scala-lang', 'scala-library', '2.12.10'), + getMavenPackageInfo('org.xerial.snappy', 'snappy-java', '1.1.8.1'), + getMavenPackageInfo('software.amazon.awssdk', 'glue', '2.17.52'), + getMavenPackageInfo('software.amazon.awssdk', 'json-utils', '2.17.52'), + getMavenPackageInfo('software.amazon.awssdk', 'metrics-spi', '2.17.52'), + getMavenPackageInfo('software.amazon.awssdk', 'third-party-jackson-core', '2.17.52'), + getMavenPackageInfo('software.amazon.awssdk', 'third-party-jackson-dataformat-cbor', '2.17.52'), + getMavenPackageInfo('software.amazon.awssdk', 'url-connection-client', '2.17.52'), + getMavenPackageInfo('software.amazon.eventstream', 'eventstream', '1.0.1'), + getMavenPackageInfo('software.amazon.eventstream', 'eventstream', '1.0.1'), + getMavenPackageInfo('software.amazon.glue', 'schema-registry-build-tools', '1.1.5'), + getMavenPackageInfo('software.amazon.glue', 'schema-registry-common', '1.1.5'), + getMavenPackageInfo('software.amazon.glue', 'schema-registry-serde', '1.1.5'), +] var DEFAULT_JAR_PATH = path.resolve(path.join(__dirname, '..', 'lib', 'jars')); var MULTI_LANG_DAEMON_CLASS = 'software.amazon.kinesis.multilang.MultiLangDaemon'; @@ -90,7 +135,7 @@ var MAX_HTTP_REDIRECT_FOLLOW = 3; function bootstrap() { var args = parseArguments(); - downloadMavenPackages(MAVEN_PACKAGE_LIST, args.jarPath, function(err) { + downloadMavenPackages(MAVEN_PACKAGE_LIST, args.jarPath, function (err) { if (err) { errorExit(util.format('Unable to download MultiLangDaemon jar files from maven: %s', err)); } @@ -113,7 +158,7 @@ function parseArguments() { var args = { 'properties': program.properties, - 'logConfiguration': program.logConfiguration ? program.logConfiguration: null, + 'logConfiguration': program.logConfiguration ? program.logConfiguration : null, 'java': (program.java ? program.java : (process.env.JAVA_HOME ? createJavaHomeExecutablePath() : null)), 'jarPath': (program.jarPath ? program.jarPath : DEFAULT_JAR_PATH), 'execute': program.execute @@ -148,7 +193,7 @@ function startKinesisClientLibraryApplication(options) { var cmd = java + ' ' + args.join(' '); console.log("=========================================================="); - console.log(cmd); + // console.log(cmd); console.log("=========================================================="); if (options.execute) { console.log("Starting MultiLangDaemon ..."); @@ -174,11 +219,11 @@ function downloadMavenPackages(mavenPackages, destinationDirectory, callback) { var remainingPackages = mavenPackages.length; var callbackInvoked = false; - var downloadMavenPackageCallback = function(err, filePath) { + var downloadMavenPackageCallback = function (err, filePath) { remainingPackages = remainingPackages - 1; if (!callbackInvoked) { if (!err) { - console.log(filePath + ' downloaded. ' + remainingPackages + ' files remain.'); + console.log(remainingPackages + ' files remain. ' + filePath + ' downloaded. '); } if (err || remainingPackages === 0) { callbackInvoked = true; @@ -188,13 +233,13 @@ function downloadMavenPackages(mavenPackages, destinationDirectory, callback) { } }; - for (var i = 0 ; i < mavenPackages.length ; ++i) { + for (var i = 0; i < mavenPackages.length; ++i) { downloadMavenPackage(mavenPackages[i], destinationDirectory, downloadMavenPackageCallback); } } function downloadMavenPackage(mavenPackage, destinationDirectory, callback) { - process.nextTick(function() { + process.nextTick(function () { var mavenPackageUrlInfo = getMavenPackageUrlInfo(mavenPackage); var destinationFile = path.join(destinationDirectory, mavenPackageUrlInfo.fileName); if (fs.existsSync(destinationFile)) { @@ -216,7 +261,7 @@ function httpDownloadFile(requestUrl, destinationFile, redirectCount, callback) path: url.parse(requestUrl).path, agent: false }; - var request = protocol.get(options, function(response) { + var request = protocol.get(options, function (response) { // Non-2XX response. if (response.statusCode > 300) { if (response.statusCode > 300 && response.statusCode < 400 && response.headers.location) { @@ -233,7 +278,7 @@ function httpDownloadFile(requestUrl, destinationFile, redirectCount, callback) response.pipe(destinationFileStream); var callbackInvoked = false; - var destinationFileStreamFinishCallback = function() { + var destinationFileStreamFinishCallback = function () { if (callbackInvoked) { return; } @@ -244,7 +289,7 @@ function httpDownloadFile(requestUrl, destinationFile, redirectCount, callback) // Older Node.js version may not support 'finish' event. destinationFileStream.on('close', destinationFileStreamFinishCallback); } - }).on('error', function(err) { + }).on('error', function (err) { fs.unlink(destinationFile); callback(err); }); @@ -285,7 +330,7 @@ function isDirectory(path) { function createDirectory(path) { try { fs.mkdirSync(path); - } catch(e) { + } catch (e) { if (e.code !== 'EEXIST') { throw e; }