diff --git a/.gitignore b/.gitignore index aa67d3d37a..a0cecb0414 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,6 @@ target/ .cache *~ mvn_install.log +.factorypath +bin/ + diff --git a/.travis.yml b/.travis.yml index d33051f57f..374d9aa820 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,10 +1,36 @@ -language: java -before_install: - - bash dev/travis-before_install.sh +language: generic + +services: + - docker env: - - HADOOP_PROFILE=default TEST_CODECS=uncompressed,brotli - - HADOOP_PROFILE=default TEST_CODECS=gzip,snappy + global: + - JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 + - IMAGE=parquet-mr + - CACHE_FOLDER=$HOME/docker-images + - CACHE_FILE=${CACHE_FOLDER}/parquet-mr.tgz + matrix: + - TEST_CODECS=uncompressed,brotli + - TEST_CODECS=gzip,snappy + +cache: + directories: + - "$HOME/.m2" + - ${CACHE_FOLDER} + +before_install: + - if [[ -f ${CACHE_FILE} ]]; then + docker load -i "${CACHE_FILE}"; + else + docker build -f dev/Dockerfile -t "${IMAGE}" "${PWD}"; + mkdir -p "${CACHE_FOLDER}"; + docker save "${IMAGE}" | gzip -c > "${CACHE_FILE}"; + fi + +install: docker run -v $HOME/.m2:/root/.m2 -v ${PWD}:/parquet-mr -e JAVA_HOME=$JAVA_HOME -e TEST_CODECS=$TEST_CODECS -i -t parquet-mr /bin/bash -c "cd /parquet-mr && mvn install --batch-mode -DskipTests=true -Dmaven.javadoc.skip=true -Dsource.skip=true | pv -fbi 60 > mvn_install.log || (cat mvn_install.log && false)" + +script: docker run -v $HOME/.m2:/root/.m2 -v ${PWD}:/parquet-mr -e JAVA_HOME=$JAVA_HOME -e TEST_CODECS=$TEST_CODECS -i -t parquet-mr /bin/bash -c "cd /parquet-mr && mvn verify javadoc:javadoc" -install: mvn install --batch-mode -DskipTests=true -Dmaven.javadoc.skip=true -Dsource.skip=true | pv -fbi 60 > mvn_install.log || (cat mvn_install.log && false) -script: mvn verify javadoc:javadoc -P $HADOOP_PROFILE +before_cache: + - rm -Rf $HOME/.m2/repository/org/apache/parquet + - find $HOME/.m2/repository/ -name *SNAPSHOT | xargs rm -Rf diff --git a/dev/Dockerfile b/dev/Dockerfile new file mode 100644 index 0000000000..cc4f679fba --- /dev/null +++ b/dev/Dockerfile @@ -0,0 +1,28 @@ +############################################################################### +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +############################################################################### + +FROM thrift:0.12.0 + +RUN set -ex; \ + apt-get update && \ + apt-get install -y --no-install-recommends \ + maven \ + openjdk-8-jdk-headless \ + openjdk-8-jre-headless \ + pv && \ + rm -rf /var/lib/apt/lists/* diff --git a/dev/travis-before_install-master.sh b/dev/travis-before_install-master.sh deleted file mode 100644 index 2c865ce30f..0000000000 --- a/dev/travis-before_install-master.sh +++ /dev/null @@ -1,50 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -################################################################################ -# This is a branch-specific script that gets invoked at the end of -# travis-before_install.sh. It is run for the master branch only. -################################################################################ - -fail_the_build= -reduced_pom="$(tempfile)" -shopt -s globstar # Enables ** to match files in subdirectories recursively -for pom in **/pom.xml -do - # Removes the project/version and project/parent/version elements, because - # those are allowed to have SNAPSHOT in them. Also removes comments. - xmlstarlet ed -N pom='http://maven.apache.org/POM/4.0.0' \ - -d '/pom:project/pom:version|/pom:project/pom:parent/pom:version|//comment()' "$pom" > "$reduced_pom" - if grep -q SNAPSHOT "$reduced_pom" - then - if [[ ! "$fail_the_build" ]] - then - printf "Error: POM files in the master branch can not refer to SNAPSHOT versions.\n" - fail_the_build=YES - fi - printf "\nOffending POM file: %s\nOffending content:\n" "$pom" - # Removes every element that does not have SNAPSHOT in it or its - # descendants. As a result, we get a skeleton of the POM file with only the - # offending parts. - xmlstarlet ed -d "//*[count((.|.//*)[contains(text(), 'SNAPSHOT')]) = 0]" "$reduced_pom" - fi -done -rm "$reduced_pom" -if [[ "$fail_the_build" ]] -then - exit 1 -fi diff --git a/dev/travis-before_install.sh b/dev/travis-before_install.sh deleted file mode 100644 index c87c8b5cad..0000000000 --- a/dev/travis-before_install.sh +++ /dev/null @@ -1,44 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -################################################################################ -# This script gets invoked by .travis.yml in the before_install step -################################################################################ - -export THIFT_VERSION=0.12.0 - -set -e -date -sudo apt-get update -qq -sudo apt-get install -qq build-essential pv autoconf automake libtool curl make \ - g++ unzip libboost-dev libboost-test-dev libboost-program-options-dev \ - libevent-dev automake libtool flex bison pkg-config g++ libssl-dev xmlstarlet -date -pwd -wget -nv https://archive.apache.org/dist/thrift/${THIFT_VERSION}/thrift-${THIFT_VERSION}.tar.gz -tar zxf thrift-${THIFT_VERSION}.tar.gz -cd thrift-${THIFT_VERSION} -chmod +x ./configure -./configure --disable-gen-erl --disable-gen-hs --without-ruby --without-haskell --without-erlang --without-php --without-nodejs -sudo make install -cd .. -branch_specific_script="dev/travis-before_install-${TRAVIS_BRANCH}.sh" -if [[ -e "$branch_specific_script" ]] -then - . "$branch_specific_script" -fi -date diff --git a/pom.xml b/pom.xml index 9db8c0cb13..4f9dff055e 100644 --- a/pom.xml +++ b/pom.xml @@ -480,6 +480,7 @@ **/*.md.vm **/.classpath **/.project + **/*.factorypath **/.settings/** **/build/** **/target/**