From 9c25c5efd5082fbcca591d73c85e930263c4b842 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Mon, 21 Jan 2019 19:14:33 +0100 Subject: [PATCH 01/17] PARQUET-1499: Enable JDK11 build --- .travis.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.travis.yml b/.travis.yml index d33051f57f..7bedb1e243 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,10 @@ language: java before_install: - bash dev/travis-before_install.sh +jdk: + - openjdk8 + - openjdk11 + env: - HADOOP_PROFILE=default TEST_CODECS=uncompressed,brotli - HADOOP_PROFILE=default TEST_CODECS=gzip,snappy From f0b95040543225c180758575799c2e8127a47dd3 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Fri, 14 Jun 2019 19:10:59 +0200 Subject: [PATCH 02/17] Don't compile the Java extension --- dev/travis-before_install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/travis-before_install.sh b/dev/travis-before_install.sh index c87c8b5cad..ae472de871 100644 --- a/dev/travis-before_install.sh +++ b/dev/travis-before_install.sh @@ -33,7 +33,7 @@ wget -nv https://archive.apache.org/dist/thrift/${THIFT_VERSION}/thrift-${THIFT_ tar zxf thrift-${THIFT_VERSION}.tar.gz cd thrift-${THIFT_VERSION} chmod +x ./configure -./configure --disable-gen-erl --disable-gen-hs --without-ruby --without-haskell --without-erlang --without-php --without-nodejs +./configure --disable-gen-erl --disable-gen-hs --without-ruby --without-haskell --without-erlang --without-php --without-nodejs --without-java sudo make install cd .. branch_specific_script="dev/travis-before_install-${TRAVIS_BRANCH}.sh" From 8be9494097535cbc52c8c0c8126622855c069e0e Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Tue, 18 Jun 2019 11:34:56 +0200 Subject: [PATCH 03/17] Bump the Javadoc plugin Got some weird warnings from generated code: [ERROR] Failed to execute goal org.apache.maven.plugins:maven-javadoc-plugin:2.9.1:javadoc (default-cli) on project parquet-format-structures: An error has occurred in JavaDocs report generation: [ERROR] Exit code: 1 - javadoc: error - The code being documented uses modules but the packages defined in http://docs.oracle.com/javase/7/docs/api/ are in the unnamed module. [ERROR] /home/travis/build/apache/parquet-mr/parquet-format-structures/target/generated-sources/thrift/org/apache/parquet/format/ColumnIndex.java:49: warning - invalid usage of tag < [ERROR] /home/travis/build/apache/parquet-mr/parquet-format-structures/target/generated-sources/thrift/org/apache/parquet/format/ColumnIndex.java:394: warning - invalid usage of tag < [ERROR] /home/travis/build/apache/parquet-mr/parquet-format-structures/target/generated-sources/thrift/org/apache/parquet/format/ColumnIndex.java:407: warning - invalid usage of tag < [ERROR] /home/travis/build/apache/parquet-mr/parquet-format-structures/target/generated-sources/thrift/org/apache/parquet/format/ColumnIndex.java:84: warning - invalid usage of tag < [ERROR] /home/travis/build/apache/parquet-mr/parquet-format-structures/target/generated-sources/thrift/org/apache/parquet/format/OffsetIndex.java:21: warning - invalid usage of tag < [ERROR] /home/travis/build/apache/parquet-mr/parquet-format-structures/target/generated-sources/thrift/org/apache/parquet/format/OffsetIndex.java:150: warning - invalid usage of tag < [ERROR] /home/travis/build/apache/parquet-mr/parquet-format-structures/target/generated-sources/thrift/org/apache/parquet/format/OffsetIndex.java:159: warning - invalid usage of tag < [ERROR] /home/travis/build/apache/parquet-mr/parquet-format-structures/target/generated-sources/thrift/org/apache/parquet/format/OffsetIndex.java:29: warning - invalid usage of tag < [ERROR] /home/travis/build/apache/parquet-mr/parquet-format-structures/target/generated-sources/thrift/org/apache/parquet/format/ColumnIndex.java:49: warning - invalid usage of tag < [ERROR] /home/travis/build/apache/parquet-mr/parquet-format-structures/target/generated-sources/thrift/org/apache/parquet/format/OffsetIndex.java:21: warning - invalid usage of tag < --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index ba98aba19a..75bc3c33c6 100644 --- a/pom.xml +++ b/pom.xml @@ -161,7 +161,7 @@ org.apache.maven.plugins maven-javadoc-plugin - 2.9 + 3.1.0 From 51b90e7356aae4fb5558ccc3239cbbba96d1c3d9 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Tue, 18 Jun 2019 14:16:40 +0200 Subject: [PATCH 04/17] Disable the lint --- parquet-format-structures/pom.xml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/parquet-format-structures/pom.xml b/parquet-format-structures/pom.xml index a5dbabbc1d..1e5dc2a850 100644 --- a/parquet-format-structures/pom.xml +++ b/parquet-format-structures/pom.xml @@ -70,7 +70,7 @@ org.apache.thrift thrift-maven-plugin - ${thrift-maven-plugin.version} + ${thrift-maven-plugin.version} ${parquet.thrift.path} ${format.thrift.executable} @@ -129,6 +129,7 @@ -Xdoclint:none + none @@ -142,6 +143,7 @@ -Xdoclint:none + none From 75a8845e9e03db3cab2519df2cc213b38baa019c Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Tue, 18 Jun 2019 14:42:53 +0200 Subject: [PATCH 05/17] Remove sudo from the setup script So we can run it in Docker as well --- dev/travis-before_install.sh | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/dev/travis-before_install.sh b/dev/travis-before_install.sh index ae472de871..60bb2c9f5f 100644 --- a/dev/travis-before_install.sh +++ b/dev/travis-before_install.sh @@ -23,18 +23,17 @@ export THIFT_VERSION=0.12.0 set -e date -sudo apt-get update -qq -sudo apt-get install -qq build-essential pv autoconf automake libtool curl make \ +apt-get update -qq +apt-get install -qq build-essential pv autoconf automake libtool curl make \ g++ unzip libboost-dev libboost-test-dev libboost-program-options-dev \ libevent-dev automake libtool flex bison pkg-config g++ libssl-dev xmlstarlet date pwd -wget -nv https://archive.apache.org/dist/thrift/${THIFT_VERSION}/thrift-${THIFT_VERSION}.tar.gz -tar zxf thrift-${THIFT_VERSION}.tar.gz +wget -qO- https://archive.apache.org/dist/thrift/0.12.0/thrift-0.12.0.tar.gz | tar zxf - cd thrift-${THIFT_VERSION} chmod +x ./configure ./configure --disable-gen-erl --disable-gen-hs --without-ruby --without-haskell --without-erlang --without-php --without-nodejs --without-java -sudo make install +make install cd .. branch_specific_script="dev/travis-before_install-${TRAVIS_BRANCH}.sh" if [[ -e "$branch_specific_script" ]] From e7664a9597e04e1e6a0a4e968b295beba3538d78 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Tue, 18 Jun 2019 14:50:21 +0200 Subject: [PATCH 06/17] Bump maven-javadoc-plugin --- parquet-format-structures/pom.xml | 3 ++- pom.xml | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/parquet-format-structures/pom.xml b/parquet-format-structures/pom.xml index 1e5dc2a850..446e10ba0c 100644 --- a/parquet-format-structures/pom.xml +++ b/parquet-format-structures/pom.xml @@ -126,10 +126,10 @@ org.apache.maven.plugins maven-javadoc-plugin + ${maven-javadoc-plugin.version} -Xdoclint:none - none @@ -140,6 +140,7 @@ org.apache.maven.plugins maven-javadoc-plugin + ${maven-javadoc-plugin.version} -Xdoclint:none diff --git a/pom.xml b/pom.xml index 75bc3c33c6..847fb4e5cf 100644 --- a/pom.xml +++ b/pom.xml @@ -101,6 +101,7 @@ 27.0.1-jre 0.1.1 1.10.19 + 3.1.0 2.3 @@ -161,7 +162,7 @@ org.apache.maven.plugins maven-javadoc-plugin - 3.1.0 + ${maven-javadoc-plugin.version} From ae7c372e1ada7679569b9b82b523a390e686d6ef Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Tue, 18 Jun 2019 15:08:43 +0200 Subject: [PATCH 07/17] Add root again --- dev/travis-before_install.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dev/travis-before_install.sh b/dev/travis-before_install.sh index 60bb2c9f5f..8200a73063 100644 --- a/dev/travis-before_install.sh +++ b/dev/travis-before_install.sh @@ -23,8 +23,8 @@ export THIFT_VERSION=0.12.0 set -e date -apt-get update -qq -apt-get install -qq build-essential pv autoconf automake libtool curl make \ +sudo apt-get update -qq +sudo apt-get install -qq --no-install-recommends build-essential pv autoconf automake libtool curl make \ g++ unzip libboost-dev libboost-test-dev libboost-program-options-dev \ libevent-dev automake libtool flex bison pkg-config g++ libssl-dev xmlstarlet date @@ -33,7 +33,7 @@ wget -qO- https://archive.apache.org/dist/thrift/0.12.0/thrift-0.12.0.tar.gz | t cd thrift-${THIFT_VERSION} chmod +x ./configure ./configure --disable-gen-erl --disable-gen-hs --without-ruby --without-haskell --without-erlang --without-php --without-nodejs --without-java -make install +sudo make install cd .. branch_specific_script="dev/travis-before_install-${TRAVIS_BRANCH}.sh" if [[ -e "$branch_specific_script" ]] From 9c4ea44ff3e48034f11e11ed11510aa8132345ab Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Tue, 18 Jun 2019 15:25:29 +0200 Subject: [PATCH 08/17] Ignore the Thrift doclint errors --- .travis.yml | 2 +- dev/travis-before_install.sh | 1 + parquet-format-structures/pom.xml | 3 +-- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7bedb1e243..49f96db7c8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,6 @@ language: java before_install: - - bash dev/travis-before_install.sh + - ./dev/travis-before_install.sh jdk: - openjdk8 diff --git a/dev/travis-before_install.sh b/dev/travis-before_install.sh index 8200a73063..0b6e3419da 100644 --- a/dev/travis-before_install.sh +++ b/dev/travis-before_install.sh @@ -1,3 +1,4 @@ +#!/usr/bin/env bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/parquet-format-structures/pom.xml b/parquet-format-structures/pom.xml index 446e10ba0c..63766f565e 100644 --- a/parquet-format-structures/pom.xml +++ b/parquet-format-structures/pom.xml @@ -129,7 +129,7 @@ ${maven-javadoc-plugin.version} - -Xdoclint:none + None @@ -143,7 +143,6 @@ ${maven-javadoc-plugin.version} - -Xdoclint:none none From bfae72b8af1108bbfeb9d3cd47930693b30227b5 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Tue, 18 Jun 2019 15:30:22 +0200 Subject: [PATCH 09/17] chmod +x --- dev/travis-before_install.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 dev/travis-before_install.sh diff --git a/dev/travis-before_install.sh b/dev/travis-before_install.sh old mode 100644 new mode 100755 From 013496d61821db44d020cb6bb2be1a1f3f4347e4 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Tue, 18 Jun 2019 15:43:18 +0200 Subject: [PATCH 10/17] Don't compile stuff we don't use --- dev/travis-before_install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/travis-before_install.sh b/dev/travis-before_install.sh index 0b6e3419da..5aac4c86f6 100755 --- a/dev/travis-before_install.sh +++ b/dev/travis-before_install.sh @@ -33,7 +33,7 @@ pwd wget -qO- https://archive.apache.org/dist/thrift/0.12.0/thrift-0.12.0.tar.gz | tar zxf - cd thrift-${THIFT_VERSION} chmod +x ./configure -./configure --disable-gen-erl --disable-gen-hs --without-ruby --without-haskell --without-erlang --without-php --without-nodejs --without-java +./configure --disable-gen-erl --disable-gen-hs --without-ruby --without-haskell --without-erlang --without-php --without-nodejs --without-java --without-go --without-c --without-cpp --without-python --without-py3 sudo make install cd .. branch_specific_script="dev/travis-before_install-${TRAVIS_BRANCH}.sh" From 4fa6f40c356a1c0f5ef3c8ecbf53f73741e51e24 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Tue, 18 Jun 2019 15:58:35 +0200 Subject: [PATCH 11/17] Bump to xenial --- .travis.yml | 2 ++ dev/travis-before_install.sh | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 49f96db7c8..cd85a1358e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,6 @@ language: java +dist: xenial + before_install: - ./dev/travis-before_install.sh diff --git a/dev/travis-before_install.sh b/dev/travis-before_install.sh index 5aac4c86f6..0cd3352e4a 100755 --- a/dev/travis-before_install.sh +++ b/dev/travis-before_install.sh @@ -33,7 +33,7 @@ pwd wget -qO- https://archive.apache.org/dist/thrift/0.12.0/thrift-0.12.0.tar.gz | tar zxf - cd thrift-${THIFT_VERSION} chmod +x ./configure -./configure --disable-gen-erl --disable-gen-hs --without-ruby --without-haskell --without-erlang --without-php --without-nodejs --without-java --without-go --without-c --without-cpp --without-python --without-py3 +./configure --disable-gen-erl --disable-gen-hs --without-ruby --without-haskell --without-erlang --without-php --without-nodejs --without-java --without-go --without-cpp --without-python --without-python3 sudo make install cd .. branch_specific_script="dev/travis-before_install-${TRAVIS_BRANCH}.sh" From 48f987b7b666506babb6951382b66c36eab4a979 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Tue, 18 Jun 2019 16:07:10 +0200 Subject: [PATCH 12/17] Add an upgrade step which wil hopefully upgrade OPENJDK 11.0.2 to 11.0.3 --- dev/travis-before_install.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dev/travis-before_install.sh b/dev/travis-before_install.sh index 0cd3352e4a..006a5c3565 100755 --- a/dev/travis-before_install.sh +++ b/dev/travis-before_install.sh @@ -25,6 +25,7 @@ export THIFT_VERSION=0.12.0 set -e date sudo apt-get update -qq +sudo apt-get upgrade -qq sudo apt-get install -qq --no-install-recommends build-essential pv autoconf automake libtool curl make \ g++ unzip libboost-dev libboost-test-dev libboost-program-options-dev \ libevent-dev automake libtool flex bison pkg-config g++ libssl-dev xmlstarlet @@ -33,7 +34,7 @@ pwd wget -qO- https://archive.apache.org/dist/thrift/0.12.0/thrift-0.12.0.tar.gz | tar zxf - cd thrift-${THIFT_VERSION} chmod +x ./configure -./configure --disable-gen-erl --disable-gen-hs --without-ruby --without-haskell --without-erlang --without-php --without-nodejs --without-java --without-go --without-cpp --without-python --without-python3 +./configure --disable-gen-erl --disable-gen-hs --without-ruby --without-haskell --without-erlang --without-php --without-nodejs --without-java --without-go --without-cpp --without-python sudo make install cd .. branch_specific_script="dev/travis-before_install-${TRAVIS_BRANCH}.sh" From 0e905f65b87b652de6c0744a5a684a2a77b8c599 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Tue, 18 Jun 2019 16:16:48 +0200 Subject: [PATCH 13/17] Add y --- dev/travis-before_install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/travis-before_install.sh b/dev/travis-before_install.sh index 006a5c3565..2a0de51541 100755 --- a/dev/travis-before_install.sh +++ b/dev/travis-before_install.sh @@ -25,7 +25,7 @@ export THIFT_VERSION=0.12.0 set -e date sudo apt-get update -qq -sudo apt-get upgrade -qq +sudo apt-get upgrade -y -qq sudo apt-get install -qq --no-install-recommends build-essential pv autoconf automake libtool curl make \ g++ unzip libboost-dev libboost-test-dev libboost-program-options-dev \ libevent-dev automake libtool flex bison pkg-config g++ libssl-dev xmlstarlet From b8fbb60c213ea0d82bc2cfacea5eb7a4b2a57fd6 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Tue, 18 Jun 2019 16:24:00 +0200 Subject: [PATCH 14/17] Remove the upgrade --- dev/travis-before_install.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/dev/travis-before_install.sh b/dev/travis-before_install.sh index 2a0de51541..4f6644b2d3 100755 --- a/dev/travis-before_install.sh +++ b/dev/travis-before_install.sh @@ -25,7 +25,6 @@ export THIFT_VERSION=0.12.0 set -e date sudo apt-get update -qq -sudo apt-get upgrade -y -qq sudo apt-get install -qq --no-install-recommends build-essential pv autoconf automake libtool curl make \ g++ unzip libboost-dev libboost-test-dev libboost-program-options-dev \ libevent-dev automake libtool flex bison pkg-config g++ libssl-dev xmlstarlet From 9d67e8f2c661e113a7782e952eb5ecc11a5830b2 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Tue, 18 Jun 2019 16:39:27 +0200 Subject: [PATCH 15/17] Set the source --- parquet-format-structures/pom.xml | 4 +++- pom.xml | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/parquet-format-structures/pom.xml b/parquet-format-structures/pom.xml index 63766f565e..0ccf2aea84 100644 --- a/parquet-format-structures/pom.xml +++ b/parquet-format-structures/pom.xml @@ -129,7 +129,8 @@ ${maven-javadoc-plugin.version} - None + none + 8 @@ -144,6 +145,7 @@ none + 8 diff --git a/pom.xml b/pom.xml index 847fb4e5cf..cf69763818 100644 --- a/pom.xml +++ b/pom.xml @@ -182,6 +182,7 @@ **/generated-sources/**/*.java + 8 From 6f36862fd74a94d46fad295c394baba7ee6bd2bf Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Tue, 18 Jun 2019 16:57:34 +0200 Subject: [PATCH 16/17] Bump to Javadoc 3.1.0 --- pom.xml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/pom.xml b/pom.xml index cf69763818..3a5c4a1f5b 100644 --- a/pom.xml +++ b/pom.xml @@ -365,6 +365,17 @@ + + org.apache.maven.plugins + maven-javadoc-plugin + ${maven-javadoc-plugin.version} + + + **/generated-sources/**/*.java + + 8 + + com.mycila.maven-license-plugin maven-license-plugin From ad4697e5e7372df899520cfb1fdca79587ea4462 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Wed, 19 Jun 2019 13:17:55 +0200 Subject: [PATCH 17/17] PARQUET-1550: CleanUtil does not work in Java 11 --- .../src/main/java/org/apache/parquet/hadoop/codec/CleanUtil.java | 1 - 1 file changed, 1 deletion(-) diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/codec/CleanUtil.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/codec/CleanUtil.java index 82b1414f29..fed2999234 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/codec/CleanUtil.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/codec/CleanUtil.java @@ -108,5 +108,4 @@ public static void cleanDirectBuffer(ByteBuffer buf) { } } } - }