ORC-17. Support HDFS as a C++ plugin module.

Fixes #134

Signed-off-by: Owen O'Malley <[email protected]>

Author: Anatoli Shein

.travis.yml

@@ -12,6 +12,14 @@ matrix:
   - compiler: clang
     os: osx
     osx_image: xcode6.4
+  - compiler: clang
+    os: osx
+    osx_image: xcode8.3
+    script:
+    - mkdir build
+    - cd build
+    - cmake -DOPENSSL_ROOT_DIR=`brew --prefix openssl` ..
+    - make package test-out
 
 jdk:
   - openjdk7
@@ -22,4 +30,4 @@ script:
   - mkdir build
   - cd build
   - cmake ..
-  - make package test-out
+  - make package test-out

CMakeLists.txt

@@ -30,6 +30,10 @@ option (BUILD_JAVA
     "Include ORC Java library in the build process"
      ON)
 
+option (BUILD_LIBHDFSPP
+    "Include LIBHDFSPP library in the build process"
+     ON)
+
 # Make sure that a build type is selected
 if (NOT CMAKE_BUILD_TYPE)
   message(STATUS "No build type selected, default to ReleaseWithDebugInfo")
@@ -84,6 +88,7 @@ endif ()
 
 enable_testing()
 
+INCLUDE(CheckSourceCompiles)
 INCLUDE(ThirdpartyToolchain)
 
 set (EXAMPLE_DIRECTORY ${CMAKE_SOURCE_DIR}/examples)

c++/include/CMakeLists.txt

@@ -10,61 +10,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CXX11_FLAGS} ${WARN_FLAGS}")
-
-INCLUDE(CheckCXXSourceCompiles)
-
-CHECK_CXX_SOURCE_COMPILES("
-    #include <initializer_list>
-    struct A {
-      A(std::initializer_list<int> list);
-    };
-    int main(int,char*[]){
-    }"
-  ORC_CXX_HAS_INITIALIZER_LIST
-)
-
-CHECK_CXX_SOURCE_COMPILES("
-    int main(int,char*[]) noexcept {
-      return 0;
-    }"
-  ORC_CXX_HAS_NOEXCEPT
-)
-
-CHECK_CXX_SOURCE_COMPILES("
-    int main(int,char* argv[]){
-      return argv[0] != nullptr;
-    }"
-  ORC_CXX_HAS_NULLPTR
-)
-
-CHECK_CXX_SOURCE_COMPILES("
-    struct A {
-      virtual ~A();
-      virtual void foo();
-    };
-    struct B: public A {
-      virtual void foo() override;
-    };
-    int main(int,char*[]){
-    }"
-  ORC_CXX_HAS_OVERRIDE
-)
-
-CHECK_CXX_SOURCE_COMPILES("
-    #include<memory>
-    int main(int,char* []){
-      std::unique_ptr<int> ptr(new int);
-    }"
-  ORC_CXX_HAS_UNIQUE_PTR
-)
-
-CHECK_CXX_SOURCE_COMPILES("
-    #include <cstdint>
-    int main(int, char*[]) { }"
-  ORC_CXX_HAS_CSTDINT
-)
-
 configure_file (
   "orc/orc-config.hh.in"
   "${CMAKE_CURRENT_BINARY_DIR}/orc/orc-config.hh"

c++/include/orc/OrcFile.hh

@@ -102,12 +102,24 @@ namespace orc {
     virtual void close() = 0;
   };
 
+  /**
+   * Create a stream to a local file or HDFS file if path begins with "hdfs://"
+   * @param path the name of the file in the local file system or HDFS
+   */
+  ORC_UNIQUE_PTR<InputStream> readFile(const std::string& path);
+
   /**
    * Create a stream to a local file.
    * @param path the name of the file in the local file system
    */
   ORC_UNIQUE_PTR<InputStream> readLocalFile(const std::string& path);
 
+  /**
+   * Create a stream to an HDFS file.
+   * @param path the uri of the file in HDFS
+   */
+  ORC_UNIQUE_PTR<InputStream> readHdfsFile(const std::string& path);
+
   /**
    * Create a reader to the for the ORC file.
    * @param stream the stream to read

c++/libs/libhdfspp/imported_timestamp

@@ -0,0 +1,10 @@
+Wed Aug 30 10:56:51 EDT 2017
+HDFS-10787
+commit 9587bb04a818a2661e264f619b09c15ce10ff38e
+Author: Anatoli Shein <[email protected]>
+Date:   Wed Aug 30 10:49:42 2017 -0400
+
+    fixed warnings3
+diffs: --------------
+       --------------
+Wed Aug 30 10:56:51 EDT 2017

c++/libs/libhdfspp/libhdfspp.tar.gz

@@ -0,0 +1,32 @@
+if [ -z "$1" ]; then
+    echo "Usage: pull_hdfs [path_to_hdfs_git_root]"
+    exit 1;
+fi
+if [ ! -d "$1" ]; then
+    echo "$1 is not a directory"
+fi
+if [ ! -d "$1/hadoop-hdfs-project" ]; then
+    echo "$1 is not the root of a hadoop git checkout"
+fi
+
+HADOOP_ROOT=$1
+echo HADOOP_ROOT=$HADOOP_ROOT
+OUT=$(readlink -m `dirname $0`)
+echo OUT=$OUT
+TS=$OUT/imported_timestamp
+
+    cd $HADOOP_ROOT &&
+    mvn -pl :hadoop-hdfs-native-client -Pnative compile -Dnative_make_args="copy_hadoop_files"
+    (date > $TS; git rev-parse --abbrev-ref HEAD >> $TS; git log -n 1 >> $TS;  \
+        echo "diffs: --------------" >> $TS; git diff HEAD >> $TS; \
+        echo "       --------------" >> $TS)
+    cd $OUT &&
+    #Delete everything except for pull_hdfs.sh and imported_timestamp
+    find . ! -name 'pull_hdfs.sh' ! -name 'imported_timestamp' ! -name '.' ! -name '..' -exec rm -rf {} + &&
+    cp -R $HADOOP_ROOT/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp . &&
+    cp -R $HADOOP_ROOT/hadoop-hdfs-project/hadoop-hdfs-native-client/target/main/native/libhdfspp/extern libhdfspp/ &&
+    cd libhdfspp &&
+	tar -czf ../libhdfspp.tar.gz * &&
+	cd .. &&
+	rm -rf libhdfspp &&
+	date >> $TS

c++/libs/libhdfspp/pull_hdfs.sh

@@ -123,6 +123,7 @@ include_directories (
   ${ZLIB_INCLUDE_DIRS}
   ${SNAPPY_INCLUDE_DIRS}
   ${LZ4_INCLUDE_DIRS}
+  ${LIBHDFSPP_INCLUDE_DIRS}
   )
 
 add_custom_command(OUTPUT orc_proto.pb.h orc_proto.pb.cc
@@ -132,7 +133,7 @@ add_custom_command(OUTPUT orc_proto.pb.h orc_proto.pb.cc
         "${CMAKE_SOURCE_DIR}/proto/orc_proto.proto"
 )
 
-add_library (orc STATIC
+set(SOURCE_FILES
   "${CMAKE_CURRENT_BINARY_DIR}/Adaptor.hh"
   orc_proto.pb.h
   io/InputStream.cc
@@ -161,13 +162,24 @@ add_library (orc STATIC
   Writer.cc
   )
 
+if(ORC_CXX_HAS_THREAD_LOCAL AND BUILD_LIBHDFSPP)
+  set(SOURCE_FILES ${SOURCE_FILES} OrcHdfsFile.cc)
+endif(ORC_CXX_HAS_THREAD_LOCAL AND BUILD_LIBHDFSPP)
+
+add_library (orc STATIC ${SOURCE_FILES})
+
 install(TARGETS orc DESTINATION lib)
 
 target_link_libraries (orc
   ${PROTOBUF_LIBRARIES}
   ${ZLIB_LIBRARIES}
   ${SNAPPY_LIBRARIES}
   ${LZ4_LIBRARIES}
+  ${LIBHDFSPP_LIBRARIES}
   )
 
 add_dependencies(orc protobuf)
+
+if(ORC_CXX_HAS_THREAD_LOCAL AND BUILD_LIBHDFSPP)
+  add_definitions(-DBUILD_LIBHDFSPP)
+endif(ORC_CXX_HAS_THREAD_LOCAL AND BUILD_LIBHDFSPP)

c++/src/CMakeLists.txt

@@ -28,6 +28,7 @@
 #include <sys/stat.h>
 #include <sys/types.h>
 #include <unistd.h>
+#include <string.h>
 
 namespace orc {
 
@@ -86,8 +87,20 @@ namespace orc {
     close(file);
   }
 
+  std::unique_ptr<InputStream> readFile(const std::string& path) {
+#ifdef BUILD_LIBHDFSPP
+    if(strncmp (path.c_str(), "hdfs://", 7) == 0){
+      return orc::readHdfsFile(std::string(path));
+    } else {
+#endif
+      return orc::readLocalFile(std::string(path));
+#ifdef BUILD_LIBHDFSPP
+      }
+#endif
+  }
+
   std::unique_ptr<InputStream> readLocalFile(const std::string& path) {
-    return std::unique_ptr<InputStream>(new FileInputStream(path));
+      return std::unique_ptr<InputStream>(new FileInputStream(path));
   }
 
   OutputStream::~OutputStream() {