From bd830dad5f71c5493ed924a8613f85024c270cf6 Mon Sep 17 00:00:00 2001 From: Gabriel MANOLACHE Date: Mon, 16 Nov 2020 17:40:48 +0100 Subject: [PATCH] Fixed an Parquet check and consistency Fixed an issue when generating the manifest for Parquet that didnt contain meta information about content length. (cherry picked from commit d1077988eb3c3ceeede72405733564f12dba2c9f) --- index.js | 7 +++++-- setup.js | 4 ++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/index.js b/index.js index 30e7528..e505950 100644 --- a/index.js +++ b/index.js @@ -840,7 +840,10 @@ function handler(event, context) { manifestContents.entries.push({ url: u, - mandatory: true + mandatory: true, + meta: { + content_length: s3Info.size + } }); } @@ -1170,7 +1173,7 @@ function handler(event, context) { } else { copyOptions = copyOptions + ' \'auto\' \n'; } - } else if (config.dataFormat.S === 'Parquet' || config.dataFormat.S === 'ORC') { + } else if (config.dataFormat.S === 'PARQUET' || config.dataFormat.S === 'ORC') { copyOptions = copyOptions + ' format as ' + config.dataFormat.S; } else { callback(null, { diff --git a/setup.js b/setup.js index 6028766..a58b88b 100644 --- a/setup.js +++ b/setup.js @@ -210,8 +210,8 @@ q_truncateTable = function (callback) { }; q_df = function (callback) { - rl.question('Enter the Data Format (CSV, JSON, AVRO, Parquet, and ORC) > ', function (answer) { - common.validateArrayContains(['CSV', 'JSON', 'AVRO', 'Parquet', 'ORC'], answer.toUpperCase(), rl); + rl.question('Enter the Data Format (CSV, JSON, AVRO, PARQUET, and ORC) > ', function (answer) { + common.validateArrayContains(['CSV', 'JSON', 'AVRO', 'PARQUET', 'ORC'], answer.toUpperCase(), rl); dynamoConfig.Item.dataFormat = { S: answer.toUpperCase() };