From c7613353f04cc22da31b3e6b39165d3904326a78 Mon Sep 17 00:00:00 2001 From: Jia Liu Date: Fri, 10 Jun 2022 23:35:15 +0800 Subject: [PATCH 1/2] ORC-1198: Add stream parameter constructor for PhysicalFsWriter. --- .../org/apache/orc/impl/PhysicalFsWriter.java | 29 +++++++++++++------ 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/java/core/src/java/org/apache/orc/impl/PhysicalFsWriter.java b/java/core/src/java/org/apache/orc/impl/PhysicalFsWriter.java index 47744dce23..6b87df73e9 100644 --- a/java/core/src/java/org/apache/orc/impl/PhysicalFsWriter.java +++ b/java/core/src/java/org/apache/orc/impl/PhysicalFsWriter.java @@ -57,7 +57,7 @@ public class PhysicalFsWriter implements PhysicalWriter { // a protobuf outStream around streamFactory private CodedOutputStream codedCompressStream; - private final Path path; + private Path path; private final HadoopShims shims; private final long blockSize; private final int maxPadding; @@ -86,12 +86,11 @@ public PhysicalFsWriter(FileSystem fs, this(fs, path, opts, new WriterEncryptionVariant[0]); } - public PhysicalFsWriter(FileSystem fs, - Path path, + public PhysicalFsWriter(FSDataOutputStream outputStream, OrcFile.WriterOptions opts, WriterEncryptionVariant[] encryption ) throws IOException { - this.path = path; + this.rawWriter = outputStream; long defaultStripeSize = opts.getStripeSize(); this.addBlockPadding = opts.getBlockPadding(); if (opts.isEnforceBufferSize()) { @@ -109,10 +108,6 @@ public PhysicalFsWriter(FileSystem fs, this.compressionStrategy = opts.getCompressionStrategy(); this.maxPadding = (int) (opts.getPaddingTolerance() * defaultStripeSize); this.blockSize = opts.getBlockSize(); - LOG.info("ORC writer created for path: {} with stripeSize: {} blockSize: {}" + - " compression: {}", path, defaultStripeSize, blockSize, compress); - rawWriter = fs.create(path, opts.getOverwrite(), HDFS_BUFFER_SIZE, - fs.getDefaultReplication(path), blockSize); blockOffset = 0; unencrypted = new VariantTracker(opts.getSchema(), compress); writeVariableLengthBlocks = opts.getWriteVariableLengthBlocks(); @@ -129,6 +124,18 @@ public PhysicalFsWriter(FileSystem fs, } } + public PhysicalFsWriter(FileSystem fs, + Path path, + OrcFile.WriterOptions opts, + WriterEncryptionVariant[] encryption + ) throws IOException { + this(fs.create(path, opts.getOverwrite(), HDFS_BUFFER_SIZE, + fs.getDefaultReplication(path), opts.getBlockSize()), opts, encryption); + this.path = path; + LOG.info("ORC writer created for path: {} with stripeSize: {} blockSize: {}" + + " compression: {}", path, opts.getStripeSize(), blockSize, compress); + } + /** * Record the information about each column encryption variant. * The unencrypted data and each encrypted column root are variants. @@ -763,6 +770,10 @@ public void writeStatistics(StreamName name, @Override public String toString() { - return path.toString(); + if (path != null) { + return path.toString(); + } else { + return ByteString.EMPTY.toString(); + } } } From 9ffdab40d3fe11332d5285bc292d26c0aeddde1f Mon Sep 17 00:00:00 2001 From: Jia Liu Date: Sat, 11 Jun 2022 01:02:40 +0800 Subject: [PATCH 2/2] ORC-1198: Fix format violations. --- .../org/apache/orc/impl/PhysicalFsWriter.java | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/java/core/src/java/org/apache/orc/impl/PhysicalFsWriter.java b/java/core/src/java/org/apache/orc/impl/PhysicalFsWriter.java index 6b87df73e9..ad6c19878e 100644 --- a/java/core/src/java/org/apache/orc/impl/PhysicalFsWriter.java +++ b/java/core/src/java/org/apache/orc/impl/PhysicalFsWriter.java @@ -86,6 +86,18 @@ public PhysicalFsWriter(FileSystem fs, this(fs, path, opts, new WriterEncryptionVariant[0]); } + public PhysicalFsWriter(FileSystem fs, + Path path, + OrcFile.WriterOptions opts, + WriterEncryptionVariant[] encryption + ) throws IOException { + this(fs.create(path, opts.getOverwrite(), HDFS_BUFFER_SIZE, + fs.getDefaultReplication(path), opts.getBlockSize()), opts, encryption); + this.path = path; + LOG.info("ORC writer created for path: {} with stripeSize: {} blockSize: {}" + + " compression: {}", path, opts.getStripeSize(), blockSize, compress); + } + public PhysicalFsWriter(FSDataOutputStream outputStream, OrcFile.WriterOptions opts, WriterEncryptionVariant[] encryption @@ -124,18 +136,6 @@ public PhysicalFsWriter(FSDataOutputStream outputStream, } } - public PhysicalFsWriter(FileSystem fs, - Path path, - OrcFile.WriterOptions opts, - WriterEncryptionVariant[] encryption - ) throws IOException { - this(fs.create(path, opts.getOverwrite(), HDFS_BUFFER_SIZE, - fs.getDefaultReplication(path), opts.getBlockSize()), opts, encryption); - this.path = path; - LOG.info("ORC writer created for path: {} with stripeSize: {} blockSize: {}" + - " compression: {}", path, opts.getStripeSize(), blockSize, compress); - } - /** * Record the information about each column encryption variant. * The unencrypted data and each encrypted column root are variants.