fix types; add licenses; more tests; more complex

Author: julienledem

java/vector/src/main/codegen/templates/UnionVector.java

@@ -42,9 +42,11 @@
 import java.util.Iterator;
 import org.apache.arrow.vector.complex.impl.ComplexCopier;
 import org.apache.arrow.vector.util.CallBack;
+import org.apache.arrow.vector.schema.ArrowFieldNode;
 
 import static org.apache.arrow.flatbuf.UnionMode.Sparse;
 
+
 /*
  * This class is generated using freemarker and the ${.template_name} template.
  */
@@ -59,7 +61,7 @@
  * For performance reasons, UnionVector stores a cached reference to each subtype vector, to avoid having to do the map lookup
  * each time the vector is accessed.
  */
-public class UnionVector implements ValueVector {
+public class UnionVector implements FieldVector {
 
   private String name;
   private BufferAllocator allocator;
@@ -97,6 +99,28 @@ public MinorType getMinorType() {
     return MinorType.UNION;
   }
 
+  @Override
+  public void initializeChildrenFromFields(List<Field> children) {
+    getMap().initializeChildrenFromFields(children);
+  }
+
+  @Override
+  public List<FieldVector> getChildrenFromFields() {
+    return getMap().getChildrenFromFields();
+  }
+
+  @Override
+  public void loadFieldBuffers(ArrowFieldNode fieldNode, List<ArrowBuf> ownBuffers) {
+    // TODO
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public List<ArrowBuf> getFieldBuffers() {
+    // TODO
+    throw new UnsupportedOperationException();
+  }
+
   public MapVector getMap() {
     if (mapVector == null) {
       int vectorCount = internalMap.size();
@@ -239,10 +263,10 @@ public void copyFromSafe(int inIndex, int outIndex, UnionVector from) {
     copyFrom(inIndex, outIndex, from);
   }
 
-  public ValueVector addVector(ValueVector v) {
+  public FieldVector addVector(FieldVector v) {
     String name = v.getMinorType().name().toLowerCase();
     Preconditions.checkState(internalMap.getChild(name) == null, String.format("%s vector already exists", name));
-    final ValueVector newVector = internalMap.addOrGet(name, v.getMinorType(), v.getClass());
+    final FieldVector newVector = internalMap.addOrGet(name, v.getMinorType(), v.getClass());
     v.makeTransferPair(newVector).transfer();
     internalMap.putChild(name, newVector);
     if (callBack != null) {

java/vector/src/main/java/org/apache/arrow/vector/FieldVector.java

@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 package org.apache.arrow.vector;
 
 import java.util.List;

java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java

@@ -19,14 +19,14 @@
 
 import java.io.Closeable;
 
-import io.netty.buffer.ArrowBuf;
-
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.OutOfMemoryException;
 import org.apache.arrow.vector.complex.reader.FieldReader;
 import org.apache.arrow.vector.types.Types.MinorType;
-import org.apache.arrow.vector.util.TransferPair;
 import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.util.TransferPair;
+
+import io.netty.buffer.ArrowBuf;
 
 /**
  * An abstraction that is used to store a sequence of values in an individual column.

java/vector/src/main/java/org/apache/arrow/vector/VectorLoader.java

@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 package org.apache.arrow.vector;
 
 import static com.google.common.base.Preconditions.checkArgument;

java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java

@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 package org.apache.arrow.vector;
 
 import java.util.ArrayList;

java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java

@@ -17,25 +17,23 @@
  */
 package org.apache.arrow.vector;
 
-import com.google.flatbuffers.FlatBufferBuilder;
-import io.netty.buffer.ArrowBuf;
-
 import java.util.Collections;
 import java.util.Iterator;
+import java.util.List;
 
-import org.apache.arrow.flatbuf.Type;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.OutOfMemoryException;
 import org.apache.arrow.vector.complex.impl.NullReader;
 import org.apache.arrow.vector.complex.reader.FieldReader;
+import org.apache.arrow.vector.schema.ArrowFieldNode;
 import org.apache.arrow.vector.types.Types.MinorType;
 import org.apache.arrow.vector.types.pojo.ArrowType.Null;
 import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.arrow.vector.util.TransferPair;
 
-import com.google.common.collect.Iterators;
+import io.netty.buffer.ArrowBuf;
 
-public class ZeroVector implements ValueVector {
+public class ZeroVector implements FieldVector {
   public final static ZeroVector INSTANCE = new ZeroVector();
 
   private final String name = "[DEFAULT]";
@@ -175,4 +173,28 @@ public Mutator getMutator() {
   public FieldReader getReader() {
     return NullReader.INSTANCE;
   }
+
+  @Override
+  public void initializeChildrenFromFields(List<Field> children) {
+    if (!children.isEmpty()) {
+      throw new IllegalArgumentException("Zero vector has no children");
+    }
+  }
+
+  @Override
+  public List<FieldVector> getChildrenFromFields() {
+    return Collections.emptyList();
+  }
+
+  @Override
+  public void loadFieldBuffers(ArrowFieldNode fieldNode, List<ArrowBuf> ownBuffers) {
+    if (!ownBuffers.isEmpty()) {
+      throw new IllegalArgumentException("Zero vector has no buffers");
+    }
+  }
+
+  @Override
+  public List<ArrowBuf> getFieldBuffers() {
+    return Collections.emptyList();
+  }
 }

java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java

@@ -17,22 +17,13 @@
  */
 package org.apache.arrow.vector.complex;
 
-import java.util.Collection;
-
-import javax.annotation.Nullable;
-
-import org.apache.arrow.flatbuf.Field;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.OutOfMemoryException;
+import org.apache.arrow.vector.FieldVector;
 import org.apache.arrow.vector.ValueVector;
 import org.apache.arrow.vector.types.Types.MinorType;
 import org.apache.arrow.vector.util.CallBack;
 
-import com.google.common.base.Function;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Sets;
-
 /**
  * Base class for composite vectors.
  *
@@ -65,8 +56,8 @@ public BufferAllocator getAllocator() {
   /**
    * Returns a {@link org.apache.arrow.vector.ValueVector} corresponding to the given field name if exists or null.
    */
-  public ValueVector getChild(String name) {
-    return getChild(name, ValueVector.class);
+  public FieldVector getChild(String name) {
+    return getChild(name, FieldVector.class);
   }
 
   /**
@@ -81,7 +72,7 @@ public void close() {
 
   protected <T extends ValueVector> T typeify(ValueVector v, Class<T> clazz) {
     if (clazz.isAssignableFrom(v.getClass())) {
-      return (T) v;
+      return clazz.cast(v);
     }
     throw new IllegalStateException(String.format("Vector requested [%s] was different than type stored [%s]. Arrow doesn't yet support hetergenous types.", clazz.getSimpleName(), v.getClass().getSimpleName()));
   }
@@ -94,10 +85,10 @@ protected boolean supportsDirectRead() {
   public abstract int size();
 
   // add a new vector with the input MajorType or return the existing vector if we already added one with the same type
-  public abstract <T extends ValueVector> T addOrGet(String name, MinorType minorType, Class<T> clazz, int... precisionScale);
+  public abstract <T extends FieldVector> T addOrGet(String name, MinorType minorType, Class<T> clazz, int... precisionScale);
 
   // return the child vector with the input name
-  public abstract <T extends ValueVector> T getChild(String name, Class<T> clazz);
+  public abstract <T extends FieldVector> T getChild(String name, Class<T> clazz);
 
   // return the child vector's ordinal in the composite container
   public abstract VectorWithOrdinal getChildVectorWithOrdinal(String name);

java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java

@@ -18,10 +18,12 @@
 package org.apache.arrow.vector.complex;
 
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
 
 import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.FieldVector;
 import org.apache.arrow.vector.ValueVector;
 import org.apache.arrow.vector.types.Types.MinorType;
 import org.apache.arrow.vector.util.CallBack;
@@ -40,7 +42,7 @@ public abstract class AbstractMapVector extends AbstractContainerVector {
   private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AbstractContainerVector.class);
 
   // Maintains a map with key as field name and value is the vector itself
-  private final MapWithOrdinal<String, ValueVector> vectors =  new MapWithOrdinal<>();
+  private final MapWithOrdinal<String, FieldVector> vectors =  new MapWithOrdinal<>();
 
   protected AbstractMapVector(String name, BufferAllocator allocator, CallBack callBack) {
     super(name, allocator, callBack);
@@ -108,7 +110,7 @@ public boolean allocateNewSafe() {
    * @return resultant {@link org.apache.arrow.vector.ValueVector}
    */
   @Override
-  public <T extends ValueVector> T addOrGet(String name, MinorType minorType, Class<T> clazz, int... precisionScale) {
+  public <T extends FieldVector> T addOrGet(String name, MinorType minorType, Class<T> clazz, int... precisionScale) {
     final ValueVector existing = getChild(name);
     boolean create = false;
     if (existing == null) {
@@ -152,7 +154,7 @@ public ValueVector getChildByOrdinal(int id) {
    * field name if exists or null.
    */
   @Override
-  public <T extends ValueVector> T getChild(String name, Class<T> clazz) {
+  public <T extends FieldVector> T getChild(String name, Class<T> clazz) {
     final ValueVector v = vectors.get(name.toLowerCase());
     if (v == null) {
       return null;
@@ -165,7 +167,7 @@ protected ValueVector add(String name, MinorType minorType, int... precisionScal
     if (existing != null) {
       throw new IllegalStateException(String.format("Vector already exists: Existing[%s], Requested[%s] ", existing.getClass().getSimpleName(), minorType));
     }
-    ValueVector vector = minorType.getNewVector(name, allocator, callBack, precisionScale);
+    FieldVector vector = minorType.getNewVector(name, allocator, callBack, precisionScale);
     putChild(name, vector);
     if (callBack!=null) {
       callBack.doWork();
@@ -178,7 +180,7 @@ protected ValueVector add(String name, MinorType minorType, int... precisionScal
    *
    * Note that this method does not enforce any vector type check nor throws a schema change exception.
    */
-  protected void putChild(String name, ValueVector vector) {
+  protected void putChild(String name, FieldVector vector) {
     putVector(name, vector);
   }
 
@@ -187,7 +189,7 @@ protected void putChild(String name, ValueVector vector) {
    * @param name  field name
    * @param vector  vector to be inserted
    */
-  protected void putVector(String name, ValueVector vector) {
+  protected void putVector(String name, FieldVector vector) {
     final ValueVector old = vectors.put(
         Preconditions.checkNotNull(name, "field name cannot be null").toLowerCase(),
         Preconditions.checkNotNull(vector, "vector cannot be null")
@@ -201,9 +203,9 @@ protected void putVector(String name, ValueVector vector) {
   /**
    * Returns a sequence of underlying child vectors.
    */
-  protected List<ValueVector> getChildren() {
+  protected List<FieldVector> getChildren() {
     int size = vectors.size();
-    List<ValueVector> children = new ArrayList<>();
+    List<FieldVector> children = new ArrayList<>();
     for (int i = 0; i < size; i++) {
       children.add(vectors.getByOrdinal(i));
     }
@@ -228,7 +230,7 @@ public int size() {
 
   @Override
   public Iterator<ValueVector> iterator() {
-    return vectors.values().iterator();
+    return Collections.<ValueVector>unmodifiableCollection(vectors.values()).iterator();
   }
 
   /**