diff --git a/common.js b/common.js index a7c5238..edee6db 100644 --- a/common.js +++ b/common.js @@ -12,6 +12,8 @@ var async = require('async'); var uuid = require('uuid'); require('./constants'); var pjson = require('./package.json'); +const Credstash = require('credstash'); +const credstash = new Credstash(); // function which creates a string representation of now suitable for use in S3 // paths @@ -805,4 +807,23 @@ exports.reprocessFile = function (dynamoDB, s3, region, file, callback) { } } }); +} + +/** + * Get a value from credstash with given key + * + * @param {string} key The key to use + * + * @return {Promise} return a promise to be used with async/await + */ +exports.credstashValue = function(key) { + return new Promise((resolve, reject) => { + credstash.get(key, (e, secret) => { + if (e) { + reject(e); + } else { + resolve(secret); + } + }); + }) } \ No newline at end of file diff --git a/deploy.yaml b/deploy.yaml index 0846eee..280e1eb 100755 --- a/deploy.yaml +++ b/deploy.yaml @@ -1,8 +1,8 @@ #This is deploy.yaml V 0.9 -#This file will set up most of what is needed to configure the Lambda Redshift database +#This file will set up most of what is needed to configure the Lambda Redshift database #loader #It will not create the KMS key required for encrypting the database password -#The user will need to create that key and assign it the alias "LambdaRedshiftLoaderKey" +#The user will need to create that key and assign it the alias "LambdaRedshiftLoaderKey" #The config script will use the key by its alias # #Parameters and notes @@ -10,10 +10,10 @@ #AvailabilityZone - Note that the stack will create in the current region by default, and # the list of availability zones available will be created based on the # region in which the user is creating the CloudFormation stack -#SecurityGroup - sets up the networking. Will require SSH and default Redshift port +#SecurityGroup - sets up the networking. Will require SSH and default Redshift port # access. #KeyName - the access key for SSH access to the driver EC2 instance. -#SubnetId - The subnet within your selected availability zone to use for the +#SubnetId - The subnet within your selected availability zone to use for the # driver EC2 instance. Note that CloudFormation will *not* cross-check # at runtime to verify that the subnet and availability zone match. @@ -56,9 +56,9 @@ Mappings: HVM64: "ami-0ad42f4f66f6c1cc9" sa-east-1: HVM64: "ami-05145e0b28ad8e0b2" - -Parameters: - AvailabilityZone: + +Parameters: + AvailabilityZone: Description: The availability zone in which to set up the driver and trigger Type: AWS::EC2::AvailabilityZone::Name Default: us-east-1a @@ -71,31 +71,31 @@ Parameters: SubnetId: Description: "The existing Subnet for the EC2 driver instance" Type: AWS::EC2::Subnet::Id - -Resources: - EC2LambdaSetupRole: + +Resources: + EC2LambdaSetupRole: Type: "AWS::IAM::Role" - Properties: - AssumeRolePolicyDocument: + Properties: + AssumeRolePolicyDocument: Version: "2012-10-17" - Statement: - - + Statement: + - Effect: "Allow" - Principal: - Service: + Principal: + Service: - "ec2.amazonaws.com" - Action: + Action: - "sts:AssumeRole" Path: "/" - Policies: - - + Policies: + - PolicyName: "EC2-lambda-setup" - PolicyDocument: + PolicyDocument: Version: "2012-10-17" - Statement: - - + Statement: + - Effect: "Allow" - Action: + Action: - "dynamodb:CreateTable" - "dynamodb:PutItem" - "dynamodb:ListTables" @@ -115,7 +115,7 @@ Resources: - "lambda:GetFunctionConfiguration" - "lambda:DeleteFunction" - "lambda:GetPolicy" - - "s3:*" + - "s3:*" - "kms:Encrypt" - "kms:Decrypt" - "kms:ReEncrypt*" @@ -125,20 +125,20 @@ Resources: MyInstanceProfile: Type: AWS::IAM::InstanceProfile - Properties: + Properties: Path: '/' - Roles: + Roles: - !Ref EC2LambdaSetupRole InstanceProfileName: ec2-lambda-setup-instance-profile - - MyEC2Instance: + + MyEC2Instance: Type: AWS::EC2::Instance - Properties: + Properties: AvailabilityZone: !Ref AvailabilityZone IamInstanceProfile: 'ec2-lambda-setup-instance-profile' - BlockDeviceMappings: + BlockDeviceMappings: - DeviceName: "/dev/xvda" - Ebs: + Ebs: DeleteOnTermination: "true" VolumeSize: "10" ImageId: !FindInMap @@ -147,7 +147,7 @@ Resources: - HVM64 InstanceType: "t2.micro" KeyName: !Ref KeyName - SecurityGroupIds: + SecurityGroupIds: - !Ref SecurityGroup SubnetId: !Ref SubnetId UserData: @@ -158,7 +158,7 @@ Resources: yum install git -y sudo -u ec2-user git clone https://github.com/awslabs/aws-lambda-redshift-loader/ /home/ec2-user/aws-lambda-redshift-loader sudo -u ec2-user git clone https://github.com/creationix/nvm.git /home/ec2-user/.nvm - chmod 755 /home/ec2-user/.nvm/nvm.sh + chmod 755 /home/ec2-user/.nvm/nvm.sh sudo -u ec2-user [ -s "/home/ec2-user/.nvm/nvm.sh" ] && \. "/home/ec2-user/.nvm/nvm.sh" sudo -u ec2-user echo 'export NVM_DIR="$HOME/.nvm"' >> /home/ec2-user/.bashrc sudo -u ec2-user echo '[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"' >> /home/ec2-user/.bashrc @@ -172,35 +172,35 @@ Resources: sudo chown ec2-user:ec2-user /usr/bin/setup.sh sudo chmod +x /usr/bin/setup.sh sudo -u ec2-user setup.sh - Tags: - - Key: "Name" + Tags: + - Key: "Name" Value: "CF Created Instance" - Key: "Purpose" Value: "Lambda Trigger Setup" - - EC2LambdaTriggerRole: + + EC2LambdaTriggerRole: Type: "AWS::IAM::Role" - Properties: - AssumeRolePolicyDocument: + Properties: + AssumeRolePolicyDocument: Version: "2012-10-17" - Statement: - - + Statement: + - Effect: "Allow" - Principal: - Service: + Principal: + Service: - lambda.amazonaws.com - Action: + Action: - sts:AssumeRole Path: "/" - Policies: - - + Policies: + - PolicyName: "EC2-lambda-trigger-policy" - PolicyDocument: + PolicyDocument: Version: "2012-10-17" - Statement: - - + Statement: + - Effect: "Allow" - Action: + Action: - "dynamodb:DeleteItem" - "dynamodb:DescribeTable" - "dynamodb:GetItem" @@ -232,7 +232,6 @@ Resources: - Arn Code: S3Bucket: !Sub awslabs-code-${AWS::Region} - S3Key: 'LambdaRedshiftLoader/AWSLambdaRedshiftLoader-2.7.0.zip' + S3Key: 'LambdaRedshiftLoader/AWSLambdaRedshiftLoader-2.7.1.zip' Runtime: nodejs8.10 - \ No newline at end of file diff --git a/dist/AWSLambdaRedshiftLoader-2.7.1.zip b/dist/AWSLambdaRedshiftLoader-2.7.1.zip new file mode 100644 index 0000000..09e79dd Binary files /dev/null and b/dist/AWSLambdaRedshiftLoader-2.7.1.zip differ diff --git a/index.js b/index.js index b63e6bf..33aedeb 100644 --- a/index.js +++ b/index.js @@ -10,13 +10,14 @@ var debug = process.env['DEBUG'] !== undefined var pjson = require('./package.json'); var region = process.env['AWS_REGION']; +const awsXRay = require('aws-xray-sdk'); if (!region || region === null || region === "") { region = "us-east-1"; console.log("AWS Lambda Redshift Database Loader using default region " + region); } -var aws = require('aws-sdk'); +const aws = awsXRay.captureAWS(require('aws-sdk')); aws.config.update({ region: region }); @@ -47,9 +48,10 @@ kmsCrypto.setRegion(region); var common = require('./common'); var async = require('async'); var uuid = require('uuid'); -const {Client} = require('pg'); +const {Client} = awsXRay.capturePostgres(require('pg')); const maxRetryMS = 200; + // empty import/invocation of the keepalive fix for node-postgres module require('pg-ka-fix')(); @@ -1068,7 +1070,9 @@ exports.handler = function (event, context) { } // add the cluster password - encryptedItems[passwordKeyMapEntry] = new Buffer(clusterInfo.connectPassword.S, 'base64'); + if (clusterInfo.connectPassword) { + encryptedItems[passwordKeyMapEntry] = new Buffer(clusterInfo.connectPassword.S, 'base64'); + } // add the master encryption key to the list of items to be decrypted, // if there is one @@ -1077,7 +1081,7 @@ exports.handler = function (event, context) { } // decrypt the encrypted items - kmsCrypto.decryptMap(encryptedItems, function (err, decryptedConfigItems) { + kmsCrypto.decryptMap(encryptedItems, async function (err, decryptedConfigItems) { if (err) { callback(err, { status: ERROR, @@ -1184,8 +1188,10 @@ exports.handler = function (event, context) { console.log(copyCommand); } + var password = clusterInfo.credstashPassKey && clusterInfo.credstashPassKey.S ? await common.credstashValue(clusterInfo.credstashPassKey.S) : encodeURIComponent(decryptedConfigItems[passwordKeyMapEntry].toString()); + // build the connection string - var dbString = 'postgres://' + clusterInfo.connectUser.S + ":" + encodeURIComponent(decryptedConfigItems[passwordKeyMapEntry].toString()) + "@" + clusterInfo.clusterEndpoint.S + ":" + var dbString = 'postgres://' + clusterInfo.connectUser.S + ":" + password + "@" + clusterInfo.clusterEndpoint.S + ":" + clusterInfo.clusterPort.N; if (clusterInfo.clusterDB) { dbString = dbString + '/' + clusterInfo.clusterDB.S; diff --git a/package-lock.json b/package-lock.json index 48c99bc..f6f48fb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "aws-lambda-redshift-loader", - "version": "2.6.9", + "version": "2.7.1", "lockfileVersion": 1, "requires": true, "dependencies": { @@ -9,12 +9,17 @@ "resolved": "https://registry.npmjs.org/JSONSelect/-/JSONSelect-0.4.0.tgz", "integrity": "sha1-oI7cxn6z/L6Z7WMIVTRKDPKCu40=" }, + "aes-js": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/aes-js/-/aes-js-0.2.2.tgz", + "integrity": "sha1-5XxbCikyOyw/GOS9tvIMSrbi7Go=" + }, "agent-base": { "version": "4.2.1", "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-4.2.1.tgz", "integrity": "sha512-JVwXMr9nHYTUXsBFKUqhJwvlcYU/blreOEUkhNR2eXZIvwd+c+o5V4MgDPKWnMS/56awN3TRzIP+KoPn+roQtg==", "requires": { - "es6-promisify": "5.0.0" + "es6-promisify": "^5.0.0" } }, "async": { @@ -22,7 +27,7 @@ "resolved": "https://registry.npmjs.org/async/-/async-2.6.2.tgz", "integrity": "sha1-GDMOp+bjE4h/XS8qkEusb+TdU4E=", "requires": { - "lodash": "4.17.11" + "lodash": "^4.17.11" }, "dependencies": { "lodash": { @@ -32,15 +37,36 @@ } } }, + "async-listener": { + "version": "0.6.10", + "resolved": "https://registry.npmjs.org/async-listener/-/async-listener-0.6.10.tgz", + "integrity": "sha512-gpuo6xOyF4D5DE5WvyqZdPA3NGhiT6Qf07l7DCB0wwDEsLvDIbCr6j9S5aj5Ch96dLace5tXVzWBZkxU/c5ohw==", + "requires": { + "semver": "^5.3.0", + "shimmer": "^1.1.0" + }, + "dependencies": { + "semver": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==" + } + } + }, + "atomic-batcher": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/atomic-batcher/-/atomic-batcher-1.0.2.tgz", + "integrity": "sha1-0WkB0QzOxZUWwZe5zNiTBom4E7Q=" + }, "augment": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/augment/-/augment-3.2.1.tgz", "integrity": "sha1-7F2elFYUDvqxEQ/ds1LL5U5/sxw=" }, "aws-sdk": { - "version": "2.399.0", - "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.399.0.tgz", - "integrity": "sha1-Q8sKrHmh1TS69+U3uE+aa+zLX3s=", + "version": "2.467.0", + "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.467.0.tgz", + "integrity": "sha512-rbVYwpAkoicdgszVClxSkzmoIQwlsrMy/vBoYZesuhUrLOXMefr/mYtpyqkiupCIG9MsqpnYUJYYNlllq4sRPQ==", "requires": { "buffer": "4.9.1", "events": "1.1.1", @@ -53,28 +79,13 @@ "xml2js": "0.4.19" }, "dependencies": { - "buffer": { - "version": "4.9.1", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.1.tgz", - "integrity": "sha1-bRu2AbB6TvztlwlBMgkwJ8lbwpg=", - "requires": { - "base64-js": "1.3.0", - "ieee754": "1.1.8", - "isarray": "1.0.0" - } - }, - "ieee754": { - "version": "1.1.8", - "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.8.tgz", - "integrity": "sha1-vjPUCsEO8ZJnAfbwii2G+/0a0+Q=" - }, "xml2js": { "version": "0.4.19", "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.19.tgz", - "integrity": "sha1-aGwg8hMgnpSr8NG88e+qKRx4J6c=", + "integrity": "sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q==", "requires": { - "sax": "1.2.1", - "xmlbuilder": "9.0.7" + "sax": ">=0.6.0", + "xmlbuilder": "~9.0.1" } }, "xmlbuilder": { @@ -84,16 +95,131 @@ } } }, + "aws-xray-sdk": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/aws-xray-sdk/-/aws-xray-sdk-2.4.0.tgz", + "integrity": "sha512-h9Us0MDaSQCgW4m7inq3NFwWt+JWmJff4i3DiKvJgHgle7EdeDPwfhTvJ99lXLLgQGtAUwMTiGX0cVqlmXHFXw==", + "requires": { + "aws-xray-sdk-core": "^2.4.0", + "aws-xray-sdk-express": "^2.4.0", + "aws-xray-sdk-mysql": "^2.4.0", + "aws-xray-sdk-postgres": "^2.4.0", + "pkginfo": "^0.4.0" + } + }, + "aws-xray-sdk-core": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/aws-xray-sdk-core/-/aws-xray-sdk-core-2.4.0.tgz", + "integrity": "sha512-V9UZg4r3tn7njSOdL55e0GWzV2cPDVvvxmo1y/eOkyWs1cDXtCK7LzE9R+iuFCIQ9kCaxcu8usoF88cGLxXixg==", + "requires": { + "atomic-batcher": "^1.0.2", + "aws-sdk": "^2.304.0", + "continuation-local-storage": "^3.2.0", + "date-fns": "^1.29.0", + "pkginfo": "^0.4.0", + "semver": "^5.3.0", + "winston": "^2.4.4" + }, + "dependencies": { + "semver": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==" + } + } + }, + "aws-xray-sdk-express": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/aws-xray-sdk-express/-/aws-xray-sdk-express-2.4.0.tgz", + "integrity": "sha512-NPjkRik2KVD8AAnFYcCGeREshDumGpOIAnxNNmn5MWQS99tNZtQrTbdrZXbvaI/eql808SxMQdNo0/yN5jGqEA==" + }, + "aws-xray-sdk-mysql": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/aws-xray-sdk-mysql/-/aws-xray-sdk-mysql-2.4.0.tgz", + "integrity": "sha512-U75s1Uaw/VbNgJhrEGubBEegydlOckl0XHcOOmqpkZiV1BkT2H6kPX0s/o6XeVjdKpQiK/MYmt86Wu/SHErElQ==" + }, + "aws-xray-sdk-postgres": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/aws-xray-sdk-postgres/-/aws-xray-sdk-postgres-2.4.0.tgz", + "integrity": "sha512-oKC1bWQR9iHiRoVWtu2f//VnVoIWCxcVqGzXjT4AVjI/gjN74vLAwEppCX2fl5LpfpKTXfIR7H6v5XfDa3EGnA==" + }, "base64-js": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.0.tgz", - "integrity": "sha512-ccav/yGvoa80BQDljCxsmmQ3Xvx60/UpBIij5QN21W3wBi/hhIC9OoO+KLpu9IJTS9j4DRVJ3aDDF9cMSoa2lw==" + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.1.tgz", + "integrity": "sha512-mLQ4i2QO1ytvGWFWmcngKO//JXAQueZvwEKtjgQFM4jIK0kU+ytMfplL8j+n5mspOfjHwoAg+9yhb7BwAHm36g==" + }, + "buffer": { + "version": "4.9.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.1.tgz", + "integrity": "sha1-bRu2AbB6TvztlwlBMgkwJ8lbwpg=", + "requires": { + "base64-js": "^1.0.2", + "ieee754": "^1.1.4", + "isarray": "^1.0.0" + } + }, + "buffer-writer": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/buffer-writer/-/buffer-writer-2.0.0.tgz", + "integrity": "sha512-a7ZpuTZU1TRtnwyCNW3I5dc0wWNC3VR9S++Ewyk2HHZdrO3CQJqSpd+95Us590V6AL7JqUAH2IwZ/398PmNFgw==" }, "colors": { "version": "0.5.1", "resolved": "http://registry.npmjs.org/colors/-/colors-0.5.1.tgz", "integrity": "sha1-fQAj6usVTo7p/Oddy5I9DtFmd3Q=" }, + "continuation-local-storage": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/continuation-local-storage/-/continuation-local-storage-3.2.1.tgz", + "integrity": "sha512-jx44cconVqkCEEyLSKWwkvUXwO561jXMa3LPjTPsm5QR22PA0/mhe33FT4Xb5y74JDvt/Cq+5lm8S8rskLv9ZA==", + "requires": { + "async-listener": "^0.6.0", + "emitter-listener": "^1.1.1" + } + }, + "credstash": { + "version": "1.0.44", + "resolved": "https://registry.npmjs.org/credstash/-/credstash-1.0.44.tgz", + "integrity": "sha1-flh8ZL5F2p6y5iYofjXzZ7dgEFI=", + "requires": { + "aes-js": "0.2.2", + "async": "1.5.2", + "aws-sdk": "2.2.35", + "xtend": "4.0.1" + }, + "dependencies": { + "async": { + "version": "1.5.2", + "resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz", + "integrity": "sha1-7GphrlZIDAw8skHJVhjiCJL5Zyo=" + }, + "aws-sdk": { + "version": "2.2.35", + "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.2.35.tgz", + "integrity": "sha1-WISw/p5FCz0fnrskB2dlTtAhsfM=", + "requires": { + "sax": "1.1.5", + "xml2js": "0.4.16", + "xmlbuilder": "4.2.1" + } + }, + "sax": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.1.5.tgz", + "integrity": "sha1-HaUKjQDN7NWUBWWfX/hTSf53N0M=" + } + } + }, + "cycle": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/cycle/-/cycle-1.0.3.tgz", + "integrity": "sha1-IegLK+hYD5i0aPN5QwZisEbDStI=" + }, + "date-fns": { + "version": "1.30.1", + "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-1.30.1.tgz", + "integrity": "sha512-hBSVCvSmWC+QypYObzwGOd9wqdDpOt+0wl0KbU+R+uuZBS1jN8VsD1ss3irQDknRj5NvxiTF6oj/nDRnN/UQNw==" + }, "debug": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz", @@ -107,6 +233,14 @@ "resolved": "https://registry.npmjs.org/ebnf-parser/-/ebnf-parser-0.1.10.tgz", "integrity": "sha1-zR9rpHfFY4xAyX7ZtXLbW6tdgzE=" }, + "emitter-listener": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/emitter-listener/-/emitter-listener-1.1.2.tgz", + "integrity": "sha512-Bt1sBAGFHY9DKY+4/2cV6izcKJUf5T7/gkdmkxzX/qv9CcGH8xSwVRW5mtX03SWJtRTWSOpzCuWN9rBFYZepZQ==", + "requires": { + "shimmer": "^1.2.0" + } + }, "es6-promise": { "version": "4.2.4", "resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-4.2.4.tgz", @@ -117,7 +251,7 @@ "resolved": "https://registry.npmjs.org/es6-promisify/-/es6-promisify-5.0.0.tgz", "integrity": "sha1-UQnWLz5W6pZ8S2NQWu8IKRyKUgM=", "requires": { - "es6-promise": "4.2.4" + "es6-promise": "^4.0.3" } }, "escodegen": { @@ -125,9 +259,9 @@ "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-0.0.21.tgz", "integrity": "sha1-U9ZSz6EDA4gnlFilJmxf/HCcY8M=", "requires": { - "esprima": "1.0.4", - "estraverse": "0.0.4", - "source-map": "0.7.3" + "esprima": "~1.0.2", + "estraverse": "~0.0.4", + "source-map": ">= 0.1.2" } }, "esprima": { @@ -145,31 +279,46 @@ "resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz", "integrity": "sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ=" }, + "eyes": { + "version": "0.1.8", + "resolved": "https://registry.npmjs.org/eyes/-/eyes-0.1.8.tgz", + "integrity": "sha1-Ys8SAjTGg3hdkCNIqADvPgzCC8A=" + }, "https-proxy-agent": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-2.2.1.tgz", "integrity": "sha1-UVUpcPoE1yPgTFbQQXjD+SWSu8A=", "requires": { - "agent-base": "4.2.1", - "debug": "3.1.0" + "agent-base": "^4.1.0", + "debug": "^3.1.0" } }, + "ieee754": { + "version": "1.1.8", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.8.tgz", + "integrity": "sha1-vjPUCsEO8ZJnAfbwii2G+/0a0+Q=" + }, "isarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" }, + "isstream": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", + "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=" + }, "jison": { "version": "0.4.4", "resolved": "https://registry.npmjs.org/jison/-/jison-0.4.4.tgz", "integrity": "sha1-hrA7Ze/sAHqR9JPs6r5W+QqmtDk=", "requires": { "JSONSelect": "0.4.0", - "ebnf-parser": "0.1.10", + "ebnf-parser": "0.1.x", "escodegen": "0.0.21", - "esprima": "1.0.4", - "jison-lex": "0.2.1", - "lex-parser": "0.1.4", + "esprima": "1.0.x", + "jison-lex": "0.2.x", + "lex-parser": "0.1.x", "nomnom": "1.5.2" } }, @@ -178,7 +327,7 @@ "resolved": "https://registry.npmjs.org/jison-lex/-/jison-lex-0.2.1.tgz", "integrity": "sha1-rEuBXozOUTLrErXfz+jXB7iETf4=", "requires": { - "lex-parser": "0.1.4", + "lex-parser": "0.1.x", "nomnom": "1.5.2" } }, @@ -197,6 +346,11 @@ "resolved": "https://registry.npmjs.org/lex-parser/-/lex-parser-0.1.4.tgz", "integrity": "sha1-ZMTwJfF/1Tv7RXY/rrFvAVp0dVA=" }, + "lodash": { + "version": "4.17.15", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz", + "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==" + }, "minimist": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz", @@ -212,61 +366,27 @@ "resolved": "https://registry.npmjs.org/nomnom/-/nomnom-1.5.2.tgz", "integrity": "sha1-9DRUSKhTz71cDSYyDyR3qwUm/i8=", "requires": { - "colors": "0.5.1", - "underscore": "1.1.7" + "colors": "0.5.x", + "underscore": "1.1.x" } }, + "packet-reader": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/packet-reader/-/packet-reader-1.0.0.tgz", + "integrity": "sha512-HAKu/fG3HpHFO0AA8WE8q2g+gBJaZ9MG7fcKk+IJPLTGAD6Psw4443l+9DGRbOIh3/aXr7Phy0TjilYivJo5XQ==" + }, "pg": { - "version": "7.9.0", - "resolved": "https://registry.npmjs.org/pg/-/pg-7.9.0.tgz", - "integrity": "sha1-BPACTYEFREY/R9u1qtokhqp9zDY=", + "version": "7.11.0", + "resolved": "https://registry.npmjs.org/pg/-/pg-7.11.0.tgz", + "integrity": "sha512-YO4V7vCmEMGoF390LJaFaohWNKaA2ayoQOEZmiHVcAUF+YsRThpf/TaKCgSvsSE7cDm37Q/Cy3Gz41xiX/XjTw==", "requires": { "buffer-writer": "2.0.0", "packet-reader": "1.0.0", "pg-connection-string": "0.1.3", - "pg-pool": "2.0.6", - "pg-types": "2.0.1", - "pgpass": "1.0.2", + "pg-pool": "^2.0.4", + "pg-types": "~2.0.0", + "pgpass": "1.x", "semver": "4.3.2" - }, - "dependencies": { - "buffer-writer": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/buffer-writer/-/buffer-writer-2.0.0.tgz", - "integrity": "sha1-zn64Gjj3gp2wnIc/L7t5LAyY7AQ=" - }, - "packet-reader": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/packet-reader/-/packet-reader-1.0.0.tgz", - "integrity": "sha1-kjjlSA3tq6z+H+PydxBj8WQVfXQ=" - }, - "pg-pool": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-2.0.6.tgz", - "integrity": "sha1-e1YaSC/rCg5Zm1i1E3/S2zrYERw=" - }, - "pg-types": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.0.1.tgz", - "integrity": "sha512-b7y6QM1VF5nOeX9ukMQ0h8a9z89mojrBHXfJeSug4mhL0YpxNBm83ot2TROyoAmX/ZOX3UbwVO4EbH7i1ZZNiw==", - "requires": { - "pg-int8": "1.0.1", - "postgres-array": "2.0.0", - "postgres-bytea": "1.0.0", - "postgres-date": "1.0.4", - "postgres-interval": "1.1.2" - } - }, - "postgres-array": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", - "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==" - }, - "postgres-date": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.4.tgz", - "integrity": "sha512-bESRvKVuTrjoBluEcpv2346+6kgB7UlnqWZsnbnCccTNq/pqfj1j6oBaN5+b/NrDXepYUT/HKadqv3iS9lJuVA==" - } } }, "pg-connection-string": { @@ -284,25 +404,57 @@ "resolved": "https://registry.npmjs.org/pg-ka-fix/-/pg-ka-fix-1.0.0-rm-fix.tgz", "integrity": "sha1-eSiIWYQdkWO1pWA8FMg8KIUSu8M=" }, + "pg-pool": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-2.0.7.tgz", + "integrity": "sha512-UiJyO5B9zZpu32GSlP0tXy8J2NsJ9EFGFfz5v6PSbdz/1hBLX1rNiiy5+mAm5iJJYwfCv4A0EBcQLGWwjbpzZw==" + }, + "pg-types": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.0.1.tgz", + "integrity": "sha512-b7y6QM1VF5nOeX9ukMQ0h8a9z89mojrBHXfJeSug4mhL0YpxNBm83ot2TROyoAmX/ZOX3UbwVO4EbH7i1ZZNiw==", + "requires": { + "pg-int8": "1.0.1", + "postgres-array": "~2.0.0", + "postgres-bytea": "~1.0.0", + "postgres-date": "~1.0.4", + "postgres-interval": "^1.1.0" + } + }, "pgpass": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.2.tgz", "integrity": "sha1-Knu0G2BltnkH6R2hsHwYR8h3swY=", "requires": { - "split": "1.0.1" + "split": "^1.0.0" } }, + "pkginfo": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/pkginfo/-/pkginfo-0.4.1.tgz", + "integrity": "sha1-tUGO8EOd5UJfxJlQQtztFPsqhP8=" + }, + "postgres-array": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", + "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==" + }, "postgres-bytea": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.0.tgz", "integrity": "sha1-AntTPAqokOJtFy1Hz5zOzFIazTU=" }, + "postgres-date": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.4.tgz", + "integrity": "sha512-bESRvKVuTrjoBluEcpv2346+6kgB7UlnqWZsnbnCccTNq/pqfj1j6oBaN5+b/NrDXepYUT/HKadqv3iS9lJuVA==" + }, "postgres-interval": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.1.2.tgz", - "integrity": "sha512-fC3xNHeTskCxL1dC8KOtxXt7YeFmlbTYtn7ul8MkVERuTmf7pI4DrkAxcw3kh1fQ9uz4wQmd03a1mRiXUZChfQ==", + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz", + "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==", "requires": { - "xtend": "4.0.1" + "xtend": "^4.0.0" } }, "punycode": { @@ -336,6 +488,17 @@ "resolved": "https://registry.npmjs.org/semver/-/semver-4.3.2.tgz", "integrity": "sha1-x6BxWKgL7dBSNVt3DYLWZA+AO+c=" }, + "serverless-plugin-tracing": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/serverless-plugin-tracing/-/serverless-plugin-tracing-2.0.0.tgz", + "integrity": "sha1-32uLMWasm7cKN8f8h1AUsjaRWPY=", + "dev": true + }, + "shimmer": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/shimmer/-/shimmer-1.2.1.tgz", + "integrity": "sha512-sQTKC1Re/rM6XyFM6fIAGHRPVGvyXfgzIDvzoq608vM+jeyVD0Tu1E6Np0Kc2zAIFWIj963V2800iF/9LPieQw==" + }, "sorted-array": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/sorted-array/-/sorted-array-1.1.0.tgz", @@ -352,9 +515,14 @@ "resolved": "https://registry.npmjs.org/split/-/split-1.0.1.tgz", "integrity": "sha512-mTyOoPbrivtXnwnIxZRFYRrPNtEFKlpB2fvjSnCQUiAA6qAZzqwna5envK4uk6OIeP17CsdF3rSBGYVBsU0Tkg==", "requires": { - "through": "2.3.8" + "through": "2" } }, + "stack-trace": { + "version": "0.0.10", + "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.10.tgz", + "integrity": "sha1-VHxws0fo0ytOEI6hoqFZ5f3eGcA=" + }, "statemachines": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/statemachines/-/statemachines-0.1.0.tgz", @@ -388,6 +556,48 @@ "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", "integrity": "sha1-G0r0lV6zB3xQHCOHL8ZROBFYcTE=" }, + "winston": { + "version": "2.4.4", + "resolved": "https://registry.npmjs.org/winston/-/winston-2.4.4.tgz", + "integrity": "sha512-NBo2Pepn4hK4V01UfcWcDlmiVTs7VTB1h7bgnB0rgP146bYhMxX0ypCz3lBOfNxCO4Zuek7yeT+y/zM1OfMw4Q==", + "requires": { + "async": "~1.0.0", + "colors": "1.0.x", + "cycle": "1.0.x", + "eyes": "0.1.x", + "isstream": "0.1.x", + "stack-trace": "0.0.x" + }, + "dependencies": { + "async": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/async/-/async-1.0.0.tgz", + "integrity": "sha1-+PwEyjoTeErenhZBr5hXjPvWR6k=" + }, + "colors": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/colors/-/colors-1.0.3.tgz", + "integrity": "sha1-BDP0TYCWgP3rYO0mDxsMJi6CpAs=" + } + } + }, + "xml2js": { + "version": "0.4.16", + "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.16.tgz", + "integrity": "sha1-+C/M0vlUDX4Km12sFj50cRlcnbM=", + "requires": { + "sax": ">=0.6.0", + "xmlbuilder": "^4.1.0" + } + }, + "xmlbuilder": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-4.2.1.tgz", + "integrity": "sha1-qlijBBoGb5DqoWwvU4n/GfP0YaU=", + "requires": { + "lodash": "^4.0.0" + } + }, "xtend": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz", diff --git a/package.json b/package.json index fa2f502..685cd65 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "aws-lambda-redshift-loader", "description": "An Amazon Redshift Database Loader written for AWS Lambda", - "version": "2.7.0", + "version": "2.7.1", "homepage": "http://github.com/awslabs/aws-lambda-redshift-loader", "bugs": { "url": "http://github.com/awslabs/aws-lambda-redshift-loader/issues", @@ -10,6 +10,8 @@ "dependencies": { "async": "2.6.2", "aws-sdk": "2.467.0", + "aws-xray-sdk": "^2.4.0", + "credstash": "^1.0.44", "https-proxy-agent": "2.2.1", "minimist": "1.2.0", "pg": "7.11.0", @@ -54,5 +56,8 @@ "repository": { "type": "git", "url": "http://github.com/awslabs/aws-lambda-redshift-loader" + }, + "devDependencies": { + "serverless-plugin-tracing": "^2.0.0" } } diff --git a/setup-file.js b/setup-file.js index 400ad63..0178d93 100644 --- a/setup-file.js +++ b/setup-file.js @@ -156,9 +156,11 @@ q_userName = function (callback) { }; q_userPwd = function (callback) { - // the Database Password - common.validateNotNull(setupConfig.userPwd, 'You Must Provide a Password', rl); + if (!setupConfig.userPwd) { + return callback(null); + } + // the Database Password kmsCrypto.encrypt(setupConfig.userPwd, function (err, ciphertext) { if (err) { console.log(JSON.stringify(err)); @@ -172,6 +174,14 @@ q_userPwd = function (callback) { }); }; +q_credstashPassKey = function (callback) { + // the Credstash key value for Database Username + dynamoConfig.Item.loadClusters.L[0].M.credstashPassKey = { + S: setupConfig.credstashPassKey + }; + callback(null); +}; + q_table = function (callback) { // the Table to be Loaded common.validateNotNull(setupConfig.table, 'You Must Provide a Table Name', rl); @@ -443,6 +453,7 @@ qs.push(q_presql); qs.push(q_postsql); qs.push(q_userName); qs.push(q_userPwd); +qs.push(q_credstashPassKey); qs.push(q_df); qs.push(q_csvDelimiter); qs.push(q_ignoreCsvHeader); diff --git a/setup.js b/setup.js index 545c80a..368478e 100644 --- a/setup.js +++ b/setup.js @@ -161,7 +161,6 @@ q_userName = function (callback) { q_userPwd = function (callback) { rl.question('Enter the Database Password (will be displayed, but encrypted before storing) > ', function (answer) { - common.validateNotNull(answer, 'You Must Provide a Password', rl); kmsCrypto.encrypt(answer, function (err, ciphertext) { if (err) { @@ -177,6 +176,15 @@ q_userPwd = function (callback) { }); }; +q_credstashPassKey = function (callback) { + rl.question('Enter the Credstash key for Database Password > ', function (answer) { + dynamoConfig.Item.loadClusters.L[0].M.credstashPassKey = { + S: answer + }; + callback(null); + }); +}; + q_table = function (callback) { rl.question('Enter the Table to be Loaded > ', function (answer) { common.validateNotNull(answer, 'You Must Provide a Table Name', rl); @@ -431,6 +439,7 @@ qs.push(q_columnList); qs.push(q_truncateTable); qs.push(q_userName); qs.push(q_userPwd); +qs.push(q_credstashPassKey); qs.push(q_df); qs.push(q_csvDelimiter); qs.push(q_ignoreCsvHeader); diff --git a/upgrades.js b/upgrades.js index 0893b65..b4fcec2 100644 --- a/upgrades.js +++ b/upgrades.js @@ -5,7 +5,7 @@ http://aws.amazon.com/asl/ - or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions and limitations under the License. + or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions and limitations under the License. */ var pjson = require('./package.json'); var common = require('./common'); @@ -147,7 +147,7 @@ exports.fixEncryptedItemEncoding = function(err, s3Info, configPre, forwardCallb // upgrade each cluster entry's password for redshift if (configPre && configPre.loadClusters && configPre.loadClusters.L) { configPre.loadClusters.L.map(function (item) { - if (item.M.connectPassword.S.indexOf('[') > -1) { + if (item.M.connectPassword && item.M.connectPassword.S.indexOf('[') > -1) { willDoUpgrade = true; item.M.connectPassword.S = new Buffer(JSON.parse(item.M.connectPassword.S)).toString('base64'); }