Update FieldNode structure to be more explicit and reflect schema

Author: julienledem

format/Message.fbs

@@ -17,7 +17,7 @@ table Tuple {
 table List {
 }
 
-enum UnionMode:int { Sparse, Dense }
+enum UnionMode:short { Sparse, Dense }
 
 table Union {
   mode: UnionMode;
@@ -28,7 +28,7 @@ table Int {
   is_signed: bool;
 }
 
-enum Precision:int {SINGLE, DOUBLE}
+enum Precision:short {SINGLE, DOUBLE}
 
 table FloatingPoint {
   precision: Precision;
@@ -114,7 +114,7 @@ table Field {
 /// ----------------------------------------------------------------------
 /// Endianness of the platform that produces the RecordBatch
 
-enum Endianness:int { Little, Big }
+enum Endianness:short { Little, Big }
 
 /// ----------------------------------------------------------------------
 /// A Schema describes the columns in a row batch
@@ -133,8 +133,19 @@ table Schema {
 /// Data structures for describing a table row batch (a collection of
 /// equal-length Arrow arrays)
 
+enum VectorType: short {
+  /// used in List type Dense Union and variable length primitive types (String, Binary)
+  OFFSET,
+  /// fixed length primitive values
+  VALUES,
+  /// Bit vector indicated if each value is null
+  VALIDITY,
+  /// Type vector used in Union type
+  TYPE
+}
+
 /// A Buffer represents a single contiguous memory segment
-struct Buffer {
+table Buffer {
   /// The shared memory page id where this buffer is located. Currently this is
   /// not used
   page: int;
@@ -146,6 +157,9 @@ struct Buffer {
   /// The absolute length (in bytes) of the memory buffer. The memory is found
   /// from offset (inclusive) to offset + length (non-inclusive).
   length: long;
+
+  /// the type of the vector to be explicit
+  type: VectorType;
 }
 
 /// Metadata about a field at some level of a nested type tree (but not
@@ -154,7 +168,7 @@ struct Buffer {
 /// For example, a List<Int16> with values [[1, 2, 3], null, [4], [5, 6], null]
 /// would have {length: 5, null_count: 2} for its List node, and {length: 6,
 /// null_count: 0} for its Int16 node, as separate FieldNode structs
-struct FieldNode {
+table FieldNode {
   /// The number of value slots in the Arrow array at this level of a nested
   /// tree
   length: int;
@@ -163,26 +177,30 @@ struct FieldNode {
   /// to write their physical validity bitmap out as a materialized buffer,
   /// instead setting the length of the bitmap buffer to 0.
   null_count: int;
+
+  /// children according to the schema
+  children: [FieldNode];
+
+  /// Buffers correspond to the pre-ordered flattened buffer tree
+  ///
+  /// The number of buffers appended to this list depends on the field type and length.
+  /// For example, most primitive arrays will have 2 buffers, 1 for the validity
+  /// bitmap and 1 for the values. For struct arrays, there will only be a
+  /// single buffer for the validity (nulls) bitmap
+  vectors: [Buffer];
 }
 
 /// A data header describing the shared memory layout of a "record" or "row"
 /// batch. Some systems call this a "row batch" internally and others a "record
 /// batch".
 table RecordBatch {
-  /// number of records / rows. The arrays in the batch should all have this
+  /// number of records / rows. The root arrays in the batch should all have this
   /// length
   length: int;
 
-  /// Nodes correspond to the pre-ordered flattened logical schema
+  /// Nodes correspond to the first level of the logical schema
   nodes: [FieldNode];
 
-  /// Buffers correspond to the pre-ordered flattened buffer tree
-  ///
-  /// The number of buffers appended to this list depends on the schema. For
-  /// example, most primitive arrays will have 2 buffers, 1 for the validity
-  /// bitmap and 1 for the values. For struct arrays, there will only be a
-  /// single buffer for the validity (nulls) bitmap
-  buffers: [Buffer];
 }
 
 /// ----------------------------------------------------------------------