Skip to content

Updating KCL to v2.3.9 #91

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 3 commits into from
Closed

Updating KCL to v2.3.9 #91

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
67bb320
bump versions
pljarlegan Dec 6, 2021
8020d29
SAN: lint
pljarlegan Dec 9, 2021
736e54f
MOD: bump some versions
pljarlegan Dec 9, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,4 @@
*.log
.idea
package-lock.json

yarn.lock
183 changes: 114 additions & 69 deletions bin/kcl-bootstrap
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ SPDX-License-Identifier: Apache-2.0

'use strict';


var fs = require('fs');
var http = require('http');
var https = require('https');
Expand All @@ -19,69 +18,115 @@ var util = require('util');


var MAVEN_PACKAGE_LIST = [
getMavenPackageInfo('software.amazon.kinesis', 'amazon-kinesis-client-multilang', '2.1.2'),
getMavenPackageInfo('software.amazon.kinesis', 'amazon-kinesis-client', '2.1.2'),
getMavenPackageInfo('software.amazon.awssdk', 'kinesis', '2.4.0'),
getMavenPackageInfo('software.amazon.awssdk', 'aws-cbor-protocol', '2.4.0'),
getMavenPackageInfo('com.fasterxml.jackson.dataformat', 'jackson-dataformat-cbor', '2.9.8'),
getMavenPackageInfo('software.amazon.awssdk', 'aws-json-protocol', '2.4.0'),
getMavenPackageInfo('software.amazon.awssdk', 'dynamodb', '2.4.0'),
getMavenPackageInfo('software.amazon.awssdk', 'cloudwatch', '2.4.0'),
getMavenPackageInfo('software.amazon.awssdk', 'netty-nio-client', '2.4.0'),
getMavenPackageInfo('io.netty', 'netty-codec-http', '4.1.32.Final'),
getMavenPackageInfo('io.netty', 'netty-codec-http2', '4.1.32.Final'),
getMavenPackageInfo('io.netty', 'netty-codec', '4.1.32.Final'),
getMavenPackageInfo('io.netty', 'netty-transport', '4.1.32.Final'),
getMavenPackageInfo('io.netty', 'netty-resolver', '4.1.32.Final'),
getMavenPackageInfo('io.netty', 'netty-common', '4.1.32.Final'),
getMavenPackageInfo('io.netty', 'netty-buffer', '4.1.32.Final'),
getMavenPackageInfo('io.netty', 'netty-handler', '4.1.32.Final'),
getMavenPackageInfo('io.netty', 'netty-transport-native-epoll', '4.1.32.Final'),
getMavenPackageInfo('io.netty', 'netty-transport-native-unix-common', '4.1.32.Final'),
getMavenPackageInfo('com.typesafe.netty', 'netty-reactive-streams-http', '2.0.0'),
getMavenPackageInfo('com.typesafe.netty', 'netty-reactive-streams', '2.0.0'),
getMavenPackageInfo('org.reactivestreams', 'reactive-streams', '1.0.2'),
getMavenPackageInfo('com.google.guava', 'guava', '26.0-jre'),
getMavenPackageInfo('software.amazon.kinesis', 'amazon-kinesis-client-multilang', '2.3.9'),
getMavenPackageInfo('software.amazon.kinesis', 'amazon-kinesis-client', '2.3.9'),
getMavenPackageInfo('software.amazon.awssdk', 'kinesis', '2.17.52'),
getMavenPackageInfo('software.amazon.awssdk', 'aws-cbor-protocol', '2.17.52'),
getMavenPackageInfo('com.fasterxml.jackson.dataformat', 'jackson-dataformat-cbor', '2.12.3'),
getMavenPackageInfo('software.amazon.awssdk', 'aws-json-protocol', '2.17.52'),
getMavenPackageInfo('software.amazon.awssdk', 'dynamodb', '2.17.52'),
getMavenPackageInfo('software.amazon.awssdk', 'cloudwatch', '2.17.52'),
getMavenPackageInfo('software.amazon.awssdk', 'netty-nio-client', '2.17.52'),
getMavenPackageInfo('io.netty', 'netty-codec-http', '4.1.68.Final'),
getMavenPackageInfo('io.netty', 'netty-codec-http2', '4.1.68.Final'),
getMavenPackageInfo('io.netty', 'netty-codec', '4.1.68.Final'),
getMavenPackageInfo('io.netty', 'netty-transport', '4.1.68.Final'),
getMavenPackageInfo('io.netty', 'netty-resolver', '4.1.68.Final'),
getMavenPackageInfo('io.netty', 'netty-common', '4.1.68.Final'),
getMavenPackageInfo('io.netty', 'netty-buffer', '4.1.68.Final'),
getMavenPackageInfo('io.netty', 'netty-handler', '4.1.68.Final'),
getMavenPackageInfo('io.netty', 'netty-transport-native-epoll', '4.1.68.Final'),
getMavenPackageInfo('io.netty', 'netty-transport-native-unix-common', '4.1.68.Final'),
getMavenPackageInfo('com.typesafe.netty', 'netty-reactive-streams-http', '2.0.5'),
getMavenPackageInfo('com.typesafe.netty', 'netty-reactive-streams', '2.0.5'),
getMavenPackageInfo('org.reactivestreams', 'reactive-streams', '1.0.3'),
getMavenPackageInfo('com.google.guava', 'guava', '30.0-jre'),
getMavenPackageInfo('com.google.code.findbugs', 'jsr305', '3.0.2'),
getMavenPackageInfo('org.checkerframework', 'checker-qual', '2.5.2'),
getMavenPackageInfo('com.google.errorprone', 'error_prone_annotations', '2.1.3'),
getMavenPackageInfo('com.google.j2objc', 'j2objc-annotations', '1.1'),
getMavenPackageInfo('org.codehaus.mojo', 'animal-sniffer-annotations', '1.14'),
getMavenPackageInfo('com.google.protobuf', 'protobuf-java', '2.6.1'),
getMavenPackageInfo('org.checkerframework', 'checker-qual', '3.5.0'),
getMavenPackageInfo('com.google.errorprone', 'error_prone_annotations', '2.3.4'),
getMavenPackageInfo('com.google.j2objc', 'j2objc-annotations', '1.3'),
getMavenPackageInfo('org.codehaus.mojo', 'animal-sniffer-annotations', '1.14'), // REMOVE ME
getMavenPackageInfo('com.google.protobuf', 'protobuf-java', '3.11.4'),
getMavenPackageInfo('org.apache.commons', 'commons-lang3', '3.8.1'),
getMavenPackageInfo('org.slf4j', 'slf4j-api', '1.7.25'),
getMavenPackageInfo('org.slf4j', 'slf4j-api', '1.7.32'),
getMavenPackageInfo('io.reactivex.rxjava2', 'rxjava', '2.1.14'),
getMavenPackageInfo('software.amazon.awssdk', 'sts', '2.4.0'),
getMavenPackageInfo('software.amazon.awssdk', 'aws-query-protocol', '2.4.0'),
getMavenPackageInfo('software.amazon.awssdk', 'protocol-core', '2.4.0'),
getMavenPackageInfo('software.amazon.awssdk', 'profiles', '2.4.0'),
getMavenPackageInfo('software.amazon.awssdk', 'sdk-core', '2.4.0'),
getMavenPackageInfo('com.fasterxml.jackson.core', 'jackson-core', '2.9.8'),
getMavenPackageInfo('com.fasterxml.jackson.core', 'jackson-databind', '2.9.8'),
getMavenPackageInfo('software.amazon.awssdk', 'auth', '2.4.0'),
getMavenPackageInfo('software.amazon', 'flow', '1.7'),
getMavenPackageInfo('software.amazon.awssdk', 'http-client-spi', '2.4.0'),
getMavenPackageInfo('software.amazon.awssdk', 'regions', '2.4.0'),
getMavenPackageInfo('com.fasterxml.jackson.core', 'jackson-annotations', '2.9.0'),
getMavenPackageInfo('software.amazon.awssdk', 'annotations', '2.4.0'),
getMavenPackageInfo('software.amazon.awssdk', 'utils', '2.4.0'),
getMavenPackageInfo('software.amazon.awssdk', 'aws-core', '2.4.0'),
getMavenPackageInfo('software.amazon.awssdk', 'apache-client', '2.4.0'),
getMavenPackageInfo('org.apache.httpcomponents', 'httpclient', '4.5.6'),
getMavenPackageInfo('commons-codec', 'commons-codec', '1.10'),
getMavenPackageInfo('org.apache.httpcomponents', 'httpcore', '4.4.10'),
getMavenPackageInfo('com.amazonaws', 'aws-java-sdk-core', '1.11.477'),
getMavenPackageInfo('commons-logging', 'commons-logging', '1.1.3'),
getMavenPackageInfo('software.amazon.awssdk', 'sts', '2.17.52'),
getMavenPackageInfo('software.amazon.awssdk', 'aws-query-protocol', '2.17.52'),
getMavenPackageInfo('software.amazon.awssdk', 'protocol-core', '2.17.52'),
getMavenPackageInfo('software.amazon.awssdk', 'profiles', '2.17.52'),
getMavenPackageInfo('software.amazon.awssdk', 'sdk-core', '2.17.52'),
getMavenPackageInfo('com.fasterxml.jackson.core', 'jackson-core', '2.12.3'),
getMavenPackageInfo('com.fasterxml.jackson.core', 'jackson-databind', '2.12.3'),
getMavenPackageInfo('software.amazon.awssdk', 'auth', '2.17.97'),
getMavenPackageInfo('software.amazon', 'flow', '1.7'), // REMOVE ME
getMavenPackageInfo('software.amazon.awssdk', 'http-client-spi', '2.17.52'),
getMavenPackageInfo('software.amazon.awssdk', 'regions', '2.17.52'),
getMavenPackageInfo('com.fasterxml.jackson.core', 'jackson-annotations', '2.12.3'),
getMavenPackageInfo('software.amazon.awssdk', 'annotations', '2.17.52'),
getMavenPackageInfo('software.amazon.awssdk', 'utils', '2.17.52'),
getMavenPackageInfo('software.amazon.awssdk', 'aws-core', '2.17.52'),
getMavenPackageInfo('software.amazon.awssdk', 'apache-client', '2.17.52'),
getMavenPackageInfo('org.apache.httpcomponents', 'httpclient', '4.5.13'),
getMavenPackageInfo('commons-codec', 'commons-codec', '1.15'),
getMavenPackageInfo('org.apache.httpcomponents', 'httpcore', '4.4.13'),
getMavenPackageInfo('com.amazonaws', 'aws-java-sdk-core', '1.12.3'),
getMavenPackageInfo('commons-logging', 'commons-logging', '1.2'),
getMavenPackageInfo('software.amazon.ion', 'ion-java', '1.0.2'),
getMavenPackageInfo('joda-time', 'joda-time', '2.8.1'),
getMavenPackageInfo('ch.qos.logback', 'logback-classic', '1.2.3'),
getMavenPackageInfo('ch.qos.logback', 'logback-core', '1.2.3'),
getMavenPackageInfo('joda-time', 'joda-time', '2.10.2'),
getMavenPackageInfo('ch.qos.logback', 'logback-classic', '1.2.7'),
getMavenPackageInfo('ch.qos.logback', 'logback-core', '1.2.7'),
getMavenPackageInfo('com.beust', 'jcommander', '1.72'),
getMavenPackageInfo('commons-io', 'commons-io', '2.6'),
getMavenPackageInfo('commons-io', 'commons-io', '2.7'),
getMavenPackageInfo('org.apache.commons', 'commons-collections4', '4.2'),
getMavenPackageInfo('commons-beanutils', 'commons-beanutils', '1.9.3'),
getMavenPackageInfo('commons-collections', 'commons-collections', '3.2.2')
];
getMavenPackageInfo('commons-beanutils', 'commons-beanutils', '1.9.4'),
getMavenPackageInfo('commons-collections', 'commons-collections', '3.2.2'),
// add
getMavenPackageInfo('software.amazon.awssdk', 'arns', '2.17.52'),
getMavenPackageInfo('com.amazonaws', 'aws-java-sdk-sts', '1.12.3'),
getMavenPackageInfo('com.amazonaws', 'jmespath-java', '1.12.3'),
getMavenPackageInfo('com.damnhandy', 'handy-uri-templates', '2.1.8'),
// getMavenPackageInfo('com.fasterxml.jackson', 'jackson-bom', '2.12.3'), // ???
getMavenPackageInfo('com.github.erosb', 'everit-json-schema', '1.12.2'),
getMavenPackageInfo('com.github.luben', 'zstd-jni', '1.4.9-1'),
getMavenPackageInfo('com.google.guava', 'failureaccess', '1.0.1'),
getMavenPackageInfo('com.google.guava', 'listenablefuture', '9999.0-empty-to-avoid-conflict-with-guava'),
getMavenPackageInfo('com.google.re2j', 're2j', '1.3'),
getMavenPackageInfo('com.kjetland', 'mbknor-jackson-jsonschema_2.12', '1.0.39'),
getMavenPackageInfo('commons-digester', 'commons-digester', '1.8.1'),
getMavenPackageInfo('commons-validator', 'commons-validator', '1.6'),
getMavenPackageInfo('io.github.classgraph', 'classgraph', '4.8.120'),
getMavenPackageInfo('javax.validation', 'validation-api', '2.0.1.Final'),
getMavenPackageInfo('org.apache.avro', 'avro', '1.10.2'),
getMavenPackageInfo('org.apache.commons', 'commons-compress', '1.21'),
getMavenPackageInfo('org.apache.kafka', 'kafka-clients', '2.8.1'),
getMavenPackageInfo('org.jetbrains', 'annotations', '13.0'),
getMavenPackageInfo('org.jetbrains.kotlin', 'kotlin-reflect', '1.3.50'),
getMavenPackageInfo('org.jetbrains.kotlin', 'kotlin-script-runtime', '1.3.50'),
getMavenPackageInfo('org.jetbrains.kotlin', 'kotlin-scripting-common', '1.3.50'),
getMavenPackageInfo('org.jetbrains.kotlin', 'kotlin-scripting-compiler-embeddable', '1.3.50'),
getMavenPackageInfo('org.jetbrains.kotlin', 'kotlin-scripting-compiler-impl-embeddable', '1.3.50'),
getMavenPackageInfo('org.jetbrains.kotlin', 'kotlin-scripting-jvm', '1.3.50'),
getMavenPackageInfo('org.jetbrains.kotlin', 'kotlin-stdlib-common', '1.3.50'),
getMavenPackageInfo('org.jetbrains.kotlin', 'kotlin-stdlib', '1.3.50'),
getMavenPackageInfo('org.jetbrains.kotlinx', 'kotlinx-coroutines-core', '1.1.1'),
getMavenPackageInfo('org.json', 'json', '20201115'),
getMavenPackageInfo('org.lz4', 'lz4-java', '1.7.1'),
getMavenPackageInfo('org.projectlombok', 'lombok-utils', '1.18.12'),
getMavenPackageInfo('org.projectlombok', 'lombok', '1.18.20'),
getMavenPackageInfo('org.scala-lang', 'scala-library', '2.12.10'),
getMavenPackageInfo('org.xerial.snappy', 'snappy-java', '1.1.8.1'),
getMavenPackageInfo('software.amazon.awssdk', 'glue', '2.17.52'),
getMavenPackageInfo('software.amazon.awssdk', 'json-utils', '2.17.52'),
getMavenPackageInfo('software.amazon.awssdk', 'metrics-spi', '2.17.52'),
getMavenPackageInfo('software.amazon.awssdk', 'third-party-jackson-core', '2.17.52'),
getMavenPackageInfo('software.amazon.awssdk', 'third-party-jackson-dataformat-cbor', '2.17.52'),
getMavenPackageInfo('software.amazon.awssdk', 'url-connection-client', '2.17.52'),
getMavenPackageInfo('software.amazon.eventstream', 'eventstream', '1.0.1'),
getMavenPackageInfo('software.amazon.eventstream', 'eventstream', '1.0.1'),
getMavenPackageInfo('software.amazon.glue', 'schema-registry-build-tools', '1.1.5'),
getMavenPackageInfo('software.amazon.glue', 'schema-registry-common', '1.1.5'),
getMavenPackageInfo('software.amazon.glue', 'schema-registry-serde', '1.1.5'),
]

var DEFAULT_JAR_PATH = path.resolve(path.join(__dirname, '..', 'lib', 'jars'));
var MULTI_LANG_DAEMON_CLASS = 'software.amazon.kinesis.multilang.MultiLangDaemon';
Expand All @@ -90,7 +135,7 @@ var MAX_HTTP_REDIRECT_FOLLOW = 3;

function bootstrap() {
var args = parseArguments();
downloadMavenPackages(MAVEN_PACKAGE_LIST, args.jarPath, function(err) {
downloadMavenPackages(MAVEN_PACKAGE_LIST, args.jarPath, function (err) {
if (err) {
errorExit(util.format('Unable to download MultiLangDaemon jar files from maven: %s', err));
}
Expand All @@ -113,7 +158,7 @@ function parseArguments() {

var args = {
'properties': program.properties,
'logConfiguration': program.logConfiguration ? program.logConfiguration: null,
'logConfiguration': program.logConfiguration ? program.logConfiguration : null,
'java': (program.java ? program.java : (process.env.JAVA_HOME ? createJavaHomeExecutablePath() : null)),
'jarPath': (program.jarPath ? program.jarPath : DEFAULT_JAR_PATH),
'execute': program.execute
Expand Down Expand Up @@ -148,7 +193,7 @@ function startKinesisClientLibraryApplication(options) {
var cmd = java + ' ' + args.join(' ');

console.log("==========================================================");
console.log(cmd);
// console.log(cmd);
console.log("==========================================================");
if (options.execute) {
console.log("Starting MultiLangDaemon ...");
Expand All @@ -174,11 +219,11 @@ function downloadMavenPackages(mavenPackages, destinationDirectory, callback) {
var remainingPackages = mavenPackages.length;
var callbackInvoked = false;

var downloadMavenPackageCallback = function(err, filePath) {
var downloadMavenPackageCallback = function (err, filePath) {
remainingPackages = remainingPackages - 1;
if (!callbackInvoked) {
if (!err) {
console.log(filePath + ' downloaded. ' + remainingPackages + ' files remain.');
console.log(remainingPackages + ' files remain. ' + filePath + ' downloaded. ');
}
if (err || remainingPackages === 0) {
callbackInvoked = true;
Expand All @@ -188,13 +233,13 @@ function downloadMavenPackages(mavenPackages, destinationDirectory, callback) {
}
};

for (var i = 0 ; i < mavenPackages.length ; ++i) {
for (var i = 0; i < mavenPackages.length; ++i) {
downloadMavenPackage(mavenPackages[i], destinationDirectory, downloadMavenPackageCallback);
}
}

function downloadMavenPackage(mavenPackage, destinationDirectory, callback) {
process.nextTick(function() {
process.nextTick(function () {
var mavenPackageUrlInfo = getMavenPackageUrlInfo(mavenPackage);
var destinationFile = path.join(destinationDirectory, mavenPackageUrlInfo.fileName);
if (fs.existsSync(destinationFile)) {
Expand All @@ -216,7 +261,7 @@ function httpDownloadFile(requestUrl, destinationFile, redirectCount, callback)
path: url.parse(requestUrl).path,
agent: false
};
var request = protocol.get(options, function(response) {
var request = protocol.get(options, function (response) {
// Non-2XX response.
if (response.statusCode > 300) {
if (response.statusCode > 300 && response.statusCode < 400 && response.headers.location) {
Expand All @@ -233,7 +278,7 @@ function httpDownloadFile(requestUrl, destinationFile, redirectCount, callback)
response.pipe(destinationFileStream);

var callbackInvoked = false;
var destinationFileStreamFinishCallback = function() {
var destinationFileStreamFinishCallback = function () {
if (callbackInvoked) {
return;
}
Expand All @@ -244,7 +289,7 @@ function httpDownloadFile(requestUrl, destinationFile, redirectCount, callback)
// Older Node.js version may not support 'finish' event.
destinationFileStream.on('close', destinationFileStreamFinishCallback);
}
}).on('error', function(err) {
}).on('error', function (err) {
fs.unlink(destinationFile);
callback(err);
});
Expand Down Expand Up @@ -285,7 +330,7 @@ function isDirectory(path) {
function createDirectory(path) {
try {
fs.mkdirSync(path);
} catch(e) {
} catch (e) {
if (e.code !== 'EEXIST') {
throw e;
}
Expand Down