list) {
+ int max = 1;
+ for (long num : list) {
+ int bitWidth = 64 - Long.numberOfLeadingZeros(num);
+ max = bitWidth > max ? bitWidth : max;
}
+ return max;
+ }
- public static byte[] getUnsignedVarInt(int value) {
- int preValue = value;
- int length = 0;
- while ((value & 0xFFFFFF80) != 0L) {
- length++;
- value >>>= 7;
- }
- length++;
+ public static byte[] getUnsignedVarInt(int value) {
+ int preValue = value;
+ int length = 0;
+ while ((value & 0xFFFFFF80) != 0L) {
+ length++;
+ value >>>= 7;
+ }
+ length++;
- byte[] res = new byte[length];
- value = preValue;
- int i = 0;
- while ((value & 0xFFFFFF80) != 0L) {
- res[i] = (byte) ((value & 0x7F) | 0x80);
- value >>>= 7;
- i++;
- }
- res[i] = (byte) (value & 0x7F);
- return res;
+ byte[] res = new byte[length];
+ value = preValue;
+ int i = 0;
+ while ((value & 0xFFFFFF80) != 0L) {
+ res[i] = (byte) ((value & 0x7F) | 0x80);
+ value >>>= 7;
+ i++;
}
+ res[i] = (byte) (value & 0x7F);
+ return res;
+ }
- /**
- * read an unsigned var int in stream and transform it to int format
- *
- * @param in stream to read an unsigned var int
- * @return integer value
- * @throws IOException exception in IO
- */
- public static int readUnsignedVarInt(InputStream in) throws IOException {
- int value = 0;
- int i = 0;
- int b;
- while (((b = in.read()) & 0x80) != 0) {
- value |= (b & 0x7F) << i;
- i += 7;
- }
- return value | (b << i);
+ /**
+ * read an unsigned var int in stream and transform it to int format
+ *
+ * @param in stream to read an unsigned var int
+ * @return integer value
+ * @throws IOException exception in IO
+ */
+ public static int readUnsignedVarInt(InputStream in) throws IOException {
+ int value = 0;
+ int i = 0;
+ int b;
+ while (((b = in.read()) & 0x80) != 0) {
+ value |= (b & 0x7F) << i;
+ i += 7;
}
+ return value | (b << i);
+ }
- /**
- * write a value to stream using unsigned var int format. for example, int
- * 123456789 has its binary format 111010-1101111-0011010-0010101, function
- * writeUnsignedVarInt will split every seven bits and write them to stream
- * from low bit to high bit like: 1-0010101 1-0011010 1-1101111 0-0111010 1
- * represents has next byte to write, 0 represents number end
- *
- * @param value value to write into stream
- * @param out output stream
- * @throws IOException exception in IO
- */
- public static void writeUnsignedVarInt(int value, OutputStream out) throws IOException {
- while ((value & 0xFFFFFF80) != 0L) {
- out.write((value & 0x7F) | 0x80);
- value >>>= 7;
- }
- out.write(value & 0x7F);
+ /**
+ * write a value to stream using unsigned var int format. for example, int 123456789 has its
+ * binary format 111010-1101111-0011010-0010101, function writeUnsignedVarInt will split every
+ * seven bits and write them to stream from low bit to high bit like: 1-0010101 1-0011010
+ * 1-1101111 0-0111010 1 represents has next byte to write, 0 represents number end
+ *
+ * @param value value to write into stream
+ * @param out output stream
+ * @throws IOException exception in IO
+ */
+ public static void writeUnsignedVarInt(int value, OutputStream out) throws IOException {
+ while ((value & 0xFFFFFF80) != 0L) {
+ out.write((value & 0x7F) | 0x80);
+ value >>>= 7;
}
+ out.write(value & 0x7F);
+ }
- /**
- * write integer value using special bit to output stream
- *
- * @param value value to write to stream
- * @param out output stream
- * @param bitWidth bit length
- * @throws IOException exception in IO
- */
- public static void writeIntLittleEndianPaddedOnBitWidth(int value, OutputStream out, int bitWidth)
- throws IOException {
- int paddedByteNum = (bitWidth + 7) / 8;
- if (paddedByteNum > 4) {
- throw new IOException(String.format(
- "tsfile-common BytesUtils: encountered value (%d) that requires more than 4 bytes", paddedByteNum));
- }
- int offset = 0;
- while (paddedByteNum > 0) {
- out.write((value >>> offset) & 0xFF);
- offset += 8;
- paddedByteNum--;
- }
+ /**
+ * write integer value using special bit to output stream
+ *
+ * @param value value to write to stream
+ * @param out output stream
+ * @param bitWidth bit length
+ * @throws IOException exception in IO
+ */
+ public static void writeIntLittleEndianPaddedOnBitWidth(int value, OutputStream out, int bitWidth)
+ throws IOException {
+ int paddedByteNum = (bitWidth + 7) / 8;
+ if (paddedByteNum > 4) {
+ throw new IOException(String.format(
+ "tsfile-common BytesUtils: encountered value (%d) that requires more than 4 bytes",
+ paddedByteNum));
+ }
+ int offset = 0;
+ while (paddedByteNum > 0) {
+ out.write((value >>> offset) & 0xFF);
+ offset += 8;
+ paddedByteNum--;
}
+ }
- /**
- * write long value using special bit to output stream
- *
- * @param value value to write to stream
- * @param out output stream
- * @param bitWidth bit length
- * @throws IOException exception in IO
- */
- public static void writeLongLittleEndianPaddedOnBitWidth(long value, OutputStream out, int bitWidth)
- throws IOException {
- int paddedByteNum = (bitWidth + 7) / 8;
- if (paddedByteNum > 8) {
- throw new IOException(String.format(
- "tsfile-common BytesUtils: encountered value (%d) that requires more than 4 bytes", paddedByteNum));
- }
- out.write(BytesUtils.longToBytes(value, paddedByteNum));
+ /**
+ * write long value using special bit to output stream
+ *
+ * @param value value to write to stream
+ * @param out output stream
+ * @param bitWidth bit length
+ * @throws IOException exception in IO
+ */
+ public static void writeLongLittleEndianPaddedOnBitWidth(long value, OutputStream out,
+ int bitWidth) throws IOException {
+ int paddedByteNum = (bitWidth + 7) / 8;
+ if (paddedByteNum > 8) {
+ throw new IOException(String.format(
+ "tsfile-common BytesUtils: encountered value (%d) that requires more than 4 bytes",
+ paddedByteNum));
}
+ out.write(BytesUtils.longToBytes(value, paddedByteNum));
+ }
- /**
- * read integer value using special bit from input stream
- *
- * @param in input stream
- * @param bitWidth bit length
- * @return integer value
- * @throws IOException exception in IO
- */
- public static int readIntLittleEndianPaddedOnBitWidth(InputStream in, int bitWidth) throws IOException {
- int paddedByteNum = (bitWidth + 7) / 8;
- if (paddedByteNum > 4) {
- throw new IOException(String.format(
- "tsfile-common BytesUtils: encountered value (%d) that requires more than 4 bytes", paddedByteNum));
- }
- int result = 0;
- int offset = 0;
- while (paddedByteNum > 0) {
- int ch = in.read();
- result += ch << offset;
- offset += 8;
- paddedByteNum--;
- }
- return result;
+ /**
+ * read integer value using special bit from input stream
+ *
+ * @param in input stream
+ * @param bitWidth bit length
+ * @return integer value
+ * @throws IOException exception in IO
+ */
+ public static int readIntLittleEndianPaddedOnBitWidth(InputStream in, int bitWidth)
+ throws IOException {
+ int paddedByteNum = (bitWidth + 7) / 8;
+ if (paddedByteNum > 4) {
+ throw new IOException(String.format(
+ "tsfile-common BytesUtils: encountered value (%d) that requires more than 4 bytes",
+ paddedByteNum));
}
+ int result = 0;
+ int offset = 0;
+ while (paddedByteNum > 0) {
+ int ch = in.read();
+ result += ch << offset;
+ offset += 8;
+ paddedByteNum--;
+ }
+ return result;
+ }
- /**
- * read long value using special bit from input stream
- *
- * @param in input stream
- * @param bitWidth bit length
- * @return long long value
- * @throws IOException exception in IO
- */
- public static long readLongLittleEndianPaddedOnBitWidth(InputStream in, int bitWidth) throws IOException {
- int paddedByteNum = (bitWidth + 7) / 8;
- if (paddedByteNum > 8) {
- throw new IOException(String.format(
- "tsfile-common BytesUtils: encountered value (%d) that requires more than 4 bytes", paddedByteNum));
- }
- long result = 0;
- for (int i = 0; i < paddedByteNum; i++) {
- int ch = in.read();
- result <<= 8;
- result |= (ch & 0xff);
- }
- return result;
+ /**
+ * read long value using special bit from input stream
+ *
+ * @param in input stream
+ * @param bitWidth bit length
+ * @return long long value
+ * @throws IOException exception in IO
+ */
+ public static long readLongLittleEndianPaddedOnBitWidth(InputStream in, int bitWidth)
+ throws IOException {
+ int paddedByteNum = (bitWidth + 7) / 8;
+ if (paddedByteNum > 8) {
+ throw new IOException(String.format(
+ "tsfile-common BytesUtils: encountered value (%d) that requires more than 4 bytes",
+ paddedByteNum));
+ }
+ long result = 0;
+ for (int i = 0; i < paddedByteNum; i++) {
+ int ch = in.read();
+ result <<= 8;
+ result |= (ch & 0xff);
}
+ return result;
+ }
}
diff --git a/src/main/java/cn/edu/tsinghua/tsfile/common/utils/TsRandomAccessFileWriter.java b/src/main/java/cn/edu/tsinghua/tsfile/common/utils/TsRandomAccessFileWriter.java
index 37362be3..4ee5837d 100644
--- a/src/main/java/cn/edu/tsinghua/tsfile/common/utils/TsRandomAccessFileWriter.java
+++ b/src/main/java/cn/edu/tsinghua/tsfile/common/utils/TsRandomAccessFileWriter.java
@@ -6,7 +6,8 @@
import java.io.RandomAccessFile;
/**
- * RandomAccessOutputStream implements the tsfile file writer interface and extends OutputStream.
+ * RandomAccessOutputStream implements the tsfile file writer interface and extends OutputStream.
+ *
* The main difference between RandomAccessOutputStream and general OutputStream
* is:RandomAccessOutputStream provide method {@code getPos} for random accessing. It also
* implements {@code getOutputStream} to return an OutputStream supporting tsfile-format
@@ -14,64 +15,66 @@
* @author kangrong
*/
public class TsRandomAccessFileWriter implements ITsRandomAccessFileWriter {
- private static final String DEFAULT_FILE_MODE = "rw";
- private RandomAccessFile out;
- private OutputStream outputStream;
+ private static final String DEFAULT_FILE_MODE = "rw";
+ private RandomAccessFile out;
+ private OutputStream outputStream;
- public TsRandomAccessFileWriter(File file) throws IOException {
- this(file, DEFAULT_FILE_MODE);
- }
+ public TsRandomAccessFileWriter(File file) throws IOException {
+ this(file, DEFAULT_FILE_MODE);
+ }
- public TsRandomAccessFileWriter(File file, String mode) throws IOException {
- out = new RandomAccessFile(file, mode);
- outputStream=new OutputStream() {
- @Override
- public void write(int b) throws IOException {
- TsRandomAccessFileWriter.this.write(b);
- }
- @Override
- public void write(byte b[], int off, int len) throws IOException {
- out.write(b, off, len);
- }
- @Override
- public void write(byte b[]) throws IOException {
- TsRandomAccessFileWriter.this.write(b);
- }
+ public TsRandomAccessFileWriter(File file, String mode) throws IOException {
+ out = new RandomAccessFile(file, mode);
+ outputStream = new OutputStream() {
+ @Override
+ public void write(int b) throws IOException {
+ TsRandomAccessFileWriter.this.write(b);
+ }
- @Override
- public void close() throws IOException {
- TsRandomAccessFileWriter.this.close();
- }
- };
- }
-
- @Override
- public void write(int b) throws IOException {
- out.write(b);
- }
+ @Override
+ public void write(byte b[], int off, int len) throws IOException {
+ out.write(b, off, len);
+ }
- @Override
- public void write(byte b[]) throws IOException {
- out.write(b);
- }
+ @Override
+ public void write(byte b[]) throws IOException {
+ TsRandomAccessFileWriter.this.write(b);
+ }
- @Override
- public long getPos() throws IOException {
- return out.length();
- }
+ @Override
+ public void close() throws IOException {
+ TsRandomAccessFileWriter.this.close();
+ }
+ };
+ }
- @Override
- public void seek(long offset) throws IOException {
- out.seek(offset);
- }
+ @Override
+ public void write(int b) throws IOException {
+ out.write(b);
+ }
- @Override
- public void close() throws IOException {
- out.close();
- }
+ @Override
+ public void write(byte b[]) throws IOException {
+ out.write(b);
+ }
- @Override
- public OutputStream getOutputStream() {
- return outputStream;
- }
+ @Override
+ public long getPos() throws IOException {
+ return out.length();
+ }
+
+ @Override
+ public void seek(long offset) throws IOException {
+ out.seek(offset);
+ }
+
+ @Override
+ public void close() throws IOException {
+ out.close();
+ }
+
+ @Override
+ public OutputStream getOutputStream() {
+ return outputStream;
+ }
}
diff --git a/src/main/java/cn/edu/tsinghua/tsfile/compress/Compressor.java b/src/main/java/cn/edu/tsinghua/tsfile/compress/Compressor.java
index cccf1028..04ba977f 100644
--- a/src/main/java/cn/edu/tsinghua/tsfile/compress/Compressor.java
+++ b/src/main/java/cn/edu/tsinghua/tsfile/compress/Compressor.java
@@ -7,75 +7,74 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.xerial.snappy.Snappy;
-
import java.io.IOException;
/**
* compress data according to type in schema
*/
public abstract class Compressor {
- public static Compressor getCompressor(String name) {
- return getCompressor(CompressionTypeName.valueOf(name));
- }
+ public static Compressor getCompressor(String name) {
+ return getCompressor(CompressionTypeName.valueOf(name));
+ }
- public static Compressor getCompressor(CompressionTypeName name) {
- if (name == null) {
- throw new CompressionTypeNotSupportedException("NULL");
- }
- switch (name) {
- case UNCOMPRESSED:
- return new NoCompressor();
- case SNAPPY:
- return new SnappyCompressor();
- default:
- throw new CompressionTypeNotSupportedException(name.toString());
- }
+ public static Compressor getCompressor(CompressionTypeName name) {
+ if (name == null) {
+ throw new CompressionTypeNotSupportedException("NULL");
+ }
+ switch (name) {
+ case UNCOMPRESSED:
+ return new NoCompressor();
+ case SNAPPY:
+ return new SnappyCompressor();
+ default:
+ throw new CompressionTypeNotSupportedException(name.toString());
}
+ }
- public abstract ListByteArrayOutputStream compress(ListByteArrayOutputStream ListByteArray);
+ public abstract ListByteArrayOutputStream compress(ListByteArrayOutputStream ListByteArray);
- public abstract CompressionTypeName getCodecName();
+ public abstract CompressionTypeName getCodecName();
- /**
- * NoCompressor will do nothing for data and return the input data directly.
- *
- * @author kangrong
- */
- static public class NoCompressor extends Compressor {
+ /**
+ * NoCompressor will do nothing for data and return the input data directly.
+ *
+ * @author kangrong
+ */
+ static public class NoCompressor extends Compressor {
- @Override
- public ListByteArrayOutputStream compress(ListByteArrayOutputStream ListByteArray) {
- return ListByteArray;
- }
+ @Override
+ public ListByteArrayOutputStream compress(ListByteArrayOutputStream ListByteArray) {
+ return ListByteArray;
+ }
- @Override
- public CompressionTypeName getCodecName() {
- return CompressionTypeName.UNCOMPRESSED;
- }
+ @Override
+ public CompressionTypeName getCodecName() {
+ return CompressionTypeName.UNCOMPRESSED;
}
+ }
- static public class SnappyCompressor extends Compressor {
- private static final Logger LOGGER = LoggerFactory.getLogger(SnappyCompressor.class);
+ static public class SnappyCompressor extends Compressor {
+ private static final Logger LOGGER = LoggerFactory.getLogger(SnappyCompressor.class);
- @Override
- public ListByteArrayOutputStream compress(ListByteArrayOutputStream listByteArray) {
- if (listByteArray == null) {
- return null;
- }
- PublicBAOS out = new PublicBAOS();
- try {
- out.write(Snappy.compress(listByteArray.toByteArray()));
- } catch (IOException e) {
- LOGGER.error(
- "tsfile-compression SnappyCompressor: errors occurs when compress input byte, ListByteArray is {}, ByteArrayOutputStream is {}",
- listByteArray, out, e);
- }
- return ListByteArrayOutputStream.from(out);
- }
+ @Override
+ public ListByteArrayOutputStream compress(ListByteArrayOutputStream listByteArray) {
+ if (listByteArray == null) {
+ return null;
+ }
+ PublicBAOS out = new PublicBAOS();
+ try {
+ out.write(Snappy.compress(listByteArray.toByteArray()));
+ } catch (IOException e) {
+ LOGGER.error(
+ "tsfile-compression SnappyCompressor: errors occurs when compress input byte, ListByteArray is {}, ByteArrayOutputStream is {}",
+ listByteArray, out, e);
+ }
+ return ListByteArrayOutputStream.from(out);
+ }
- @Override
- public CompressionTypeName getCodecName() {
- return CompressionTypeName.SNAPPY;
- }
+ @Override
+ public CompressionTypeName getCodecName() {
+ return CompressionTypeName.SNAPPY;
}
+ }
}
diff --git a/src/main/java/cn/edu/tsinghua/tsfile/compress/UnCompressor.java b/src/main/java/cn/edu/tsinghua/tsfile/compress/UnCompressor.java
index f4bac317..636481bf 100644
--- a/src/main/java/cn/edu/tsinghua/tsfile/compress/UnCompressor.java
+++ b/src/main/java/cn/edu/tsinghua/tsfile/compress/UnCompressor.java
@@ -5,66 +5,65 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.xerial.snappy.Snappy;
-
import java.io.IOException;
/**
* uncompress data according to type in metadata
*/
public abstract class UnCompressor {
- public static UnCompressor getUnCompressor(CompressionTypeName name) {
- if (name == null) {
- throw new CompressionTypeNotSupportedException("NULL");
- }
- switch (name) {
- case UNCOMPRESSED:
- return new NoUnCompressor();
- case SNAPPY:
- return new SnappyUnCompressor();
- default:
- throw new CompressionTypeNotSupportedException(name.toString());
- }
+ public static UnCompressor getUnCompressor(CompressionTypeName name) {
+ if (name == null) {
+ throw new CompressionTypeNotSupportedException("NULL");
+ }
+ switch (name) {
+ case UNCOMPRESSED:
+ return new NoUnCompressor();
+ case SNAPPY:
+ return new SnappyUnCompressor();
+ default:
+ throw new CompressionTypeNotSupportedException(name.toString());
}
+ }
- public abstract byte[] uncompress(byte[] byteArray);
+ public abstract byte[] uncompress(byte[] byteArray);
- public abstract CompressionTypeName getCodecName();
+ public abstract CompressionTypeName getCodecName();
- static public class NoUnCompressor extends UnCompressor {
+ static public class NoUnCompressor extends UnCompressor {
- @Override
- public byte[] uncompress(byte[] byteArray) {
- return byteArray;
- }
+ @Override
+ public byte[] uncompress(byte[] byteArray) {
+ return byteArray;
+ }
- @Override
- public CompressionTypeName getCodecName() {
- return CompressionTypeName.UNCOMPRESSED;
- }
+ @Override
+ public CompressionTypeName getCodecName() {
+ return CompressionTypeName.UNCOMPRESSED;
}
+ }
- static public class SnappyUnCompressor extends UnCompressor {
- private static final Logger LOGGER = LoggerFactory.getLogger(SnappyUnCompressor.class);
+ static public class SnappyUnCompressor extends UnCompressor {
+ private static final Logger LOGGER = LoggerFactory.getLogger(SnappyUnCompressor.class);
- @Override
- public byte[] uncompress(byte[] bytes) {
- if (bytes == null) {
- return null;
- }
+ @Override
+ public byte[] uncompress(byte[] bytes) {
+ if (bytes == null) {
+ return null;
+ }
- try {
- return Snappy.uncompress(bytes);
- } catch (IOException e) {
- LOGGER.error(
- "tsfile-compression SnappyUnCompressor: errors occurs when uncompress input byte, bytes is {}",
- bytes, e);
- }
- return null;
- }
+ try {
+ return Snappy.uncompress(bytes);
+ } catch (IOException e) {
+ LOGGER.error(
+ "tsfile-compression SnappyUnCompressor: errors occurs when uncompress input byte, bytes is {}",
+ bytes, e);
+ }
+ return null;
+ }
- @Override
- public CompressionTypeName getCodecName() {
- return CompressionTypeName.SNAPPY;
- }
+ @Override
+ public CompressionTypeName getCodecName() {
+ return CompressionTypeName.SNAPPY;
}
+ }
}
diff --git a/src/main/java/cn/edu/tsinghua/tsfile/encoding/bitpacking/IntPacker.java b/src/main/java/cn/edu/tsinghua/tsfile/encoding/bitpacking/IntPacker.java
index 02ada119..fcc6042c 100644
--- a/src/main/java/cn/edu/tsinghua/tsfile/encoding/bitpacking/IntPacker.java
+++ b/src/main/java/cn/edu/tsinghua/tsfile/encoding/bitpacking/IntPacker.java
@@ -1,152 +1,154 @@
package cn.edu.tsinghua.tsfile.encoding.bitpacking;
/**
- * This class is used to encode(decode) Integer in Java with specified bit-width.
- * User need to guarantee that the length of every given Integer in binary mode
- * is less than or equal to the bit-width.
- *
- * e.g., if bit-width is 4, then Integer '16'(10000)b is not allowed but '15'(1111)b is allowed.
- *
- * For a full example,
- * Width: 3
- * Input: 5 4 7 3 0 1 3 2
- *
- * Output:
- *
- * +-----------------------+ +-----------------------+ +-----------------------+
- * |1 |0 |1 |1 |0 |0 |1 |1 | |1 |0 |1 |1 |0 |0 |0 |0 | |0 |1 |0 |1 |1 |0 |1 |0 |
- * +-----------------------+ +-----------------------+ +-----------------------+
- * +-----+ +-----+ +---------+ +-----+ +-----+ +---------+ +-----+ +-----+
- * 5 4 7 3 0 1 3 2
+ * This class is used to encode(decode) Integer in Java with specified bit-width. User need to
+ * guarantee that the length of every given Integer in binary mode is less than or equal to the
+ * bit-width.
+ *
+ * e.g., if bit-width is 4, then Integer '16'(10000)b is not allowed but '15'(1111)b is allowed.
+ *
+ * For a full example, Width: 3 Input: 5 4 7 3 0 1 3 2
+ *
+ * Output:
+ *
+ * +-----------------------+ +-----------------------+ +-----------------------+ |1 |0 |1 |1 |0 |0
+ * |1 |1 | |1 |0 |1 |1 |0 |0 |0 |0 | |0 |1 |0 |1 |1 |0 |1 |0 | +-----------------------+
+ * +-----------------------+ +-----------------------+ +-----+ +-----+ +---------+ +-----+ +-----+
+ * +---------+ +-----+ +-----+ 5 4 7 3 0 1 3 2
*
- * @author Zhang Jinrui
+ * @author Zhang Jinrui
*/
public class IntPacker {
- /**
- * Number of Integers for each pack operation
- */
- private static final int NUM_OF_INTS = 8;
- /**
- * bit-width
- */
- private int width;
+ /**
+ * Number of Integers for each pack operation
+ */
+ private static final int NUM_OF_INTS = 8;
+ /**
+ * bit-width
+ */
+ private int width;
- public IntPacker(int width) {
- this.width = width;
- }
+ public IntPacker(int width) {
+ this.width = width;
+ }
- /**
- * Encode 8 ({@link IntPacker#NUM_OF_INTS}) Integers from the array 'values' with specified bit-width to bytes
- *
- * @param values - array where '8 Integers' are in
- * @param offset - the offset of first Integer to be encoded
- * @param buf - encoded bytes, buf size must be equal to ({@link IntPacker#NUM_OF_INTS} * {@link IntPacker#width} / 8)
- */
- public void pack8Values(int[] values, int offset, byte[] buf) {
- int bufIdx = 0;
- int valueIdx = offset;
- //remaining bits for the current unfinished Integer
- int leftBit = 0;
+ /**
+ * Encode 8 ({@link IntPacker#NUM_OF_INTS}) Integers from the array 'values' with specified
+ * bit-width to bytes
+ *
+ * @param values - array where '8 Integers' are in
+ * @param offset - the offset of first Integer to be encoded
+ * @param buf - encoded bytes, buf size must be equal to ({@link IntPacker#NUM_OF_INTS} *
+ * {@link IntPacker#width} / 8)
+ */
+ public void pack8Values(int[] values, int offset, byte[] buf) {
+ int bufIdx = 0;
+ int valueIdx = offset;
+ // remaining bits for the current unfinished Integer
+ int leftBit = 0;
- while (valueIdx < NUM_OF_INTS + offset) {
- // buffer is used for saving 32 bits as a part of result
- int buffer = 0;
- // remaining size of bits in the 'buffer'
- int leftSize = 32;
+ while (valueIdx < NUM_OF_INTS + offset) {
+ // buffer is used for saving 32 bits as a part of result
+ int buffer = 0;
+ // remaining size of bits in the 'buffer'
+ int leftSize = 32;
- // encode the left bits of current Integer to 'buffer'
- if (leftBit > 0) {
- buffer |= (values[valueIdx] << (32 - leftBit));
- leftSize -= leftBit;
- leftBit = 0;
- valueIdx++;
- }
+ // encode the left bits of current Integer to 'buffer'
+ if (leftBit > 0) {
+ buffer |= (values[valueIdx] << (32 - leftBit));
+ leftSize -= leftBit;
+ leftBit = 0;
+ valueIdx++;
+ }
- while (leftSize >= width && valueIdx < NUM_OF_INTS + offset) {
- //encode one Integer to the 'buffer'
- buffer |= (values[valueIdx] << (leftSize - width));
- leftSize -= width;
- valueIdx++;
- }
- // If the remaining space of the buffer can not save the bits for one Integer,
- if (leftSize > 0 && valueIdx < NUM_OF_INTS + offset) {
- // put the first 'leftSize' bits of the Integer into remaining space of the buffer
- buffer |= (values[valueIdx] >>> (width - leftSize));
- leftBit = width - leftSize;
- leftSize = 0;
- }
+ while (leftSize >= width && valueIdx < NUM_OF_INTS + offset) {
+ // encode one Integer to the 'buffer'
+ buffer |= (values[valueIdx] << (leftSize - width));
+ leftSize -= width;
+ valueIdx++;
+ }
+ // If the remaining space of the buffer can not save the bits for one Integer,
+ if (leftSize > 0 && valueIdx < NUM_OF_INTS + offset) {
+ // put the first 'leftSize' bits of the Integer into remaining space of the buffer
+ buffer |= (values[valueIdx] >>> (width - leftSize));
+ leftBit = width - leftSize;
+ leftSize = 0;
+ }
- // put the buffer into the final result
- for (int j = 0; j < 4; j++) {
- buf[bufIdx] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF);
- bufIdx++;
- if (bufIdx >= width) {
- return;
- }
- }
+ // put the buffer into the final result
+ for (int j = 0; j < 4; j++) {
+ buf[bufIdx] = (byte) ((buffer >>> ((3 - j) * 8)) & 0xFF);
+ bufIdx++;
+ if (bufIdx >= width) {
+ return;
}
+ }
}
+ }
- /**
- * decode Integers from byte array.
- *
- * @param buf - array where bytes are in.
- * @param offset - offset of first byte to be decoded in buf
- * @param values - decoded result , the length of 'values' should be @{link IntPacker#NUM_OF_INTS}
- */
- public void unpack8Values(byte[] buf, int offset, int[] values) {
- int byteIdx = offset;
- long buffer = 0;
- //total bits which have read from 'buf' to 'buffer'. i.e., number of available bits to be decoded.
- int totalBits = 0;
- int valueIdx = 0;
+ /**
+ * decode Integers from byte array.
+ *
+ * @param buf - array where bytes are in.
+ * @param offset - offset of first byte to be decoded in buf
+ * @param values - decoded result , the length of 'values' should be @{link IntPacker#NUM_OF_INTS}
+ */
+ public void unpack8Values(byte[] buf, int offset, int[] values) {
+ int byteIdx = offset;
+ long buffer = 0;
+ // total bits which have read from 'buf' to 'buffer'. i.e., number of available bits to be
+ // decoded.
+ int totalBits = 0;
+ int valueIdx = 0;
- while (valueIdx < NUM_OF_INTS) {
- //If current available bits are not enough to decode one Integer, then add next byte from buf to 'buffer'
- //until totalBits >= width
- while (totalBits < width) {
- buffer = ((buffer << 8) | (buf[byteIdx] & 0xFF));
- byteIdx++;
- totalBits += 8;
- }
+ while (valueIdx < NUM_OF_INTS) {
+ // If current available bits are not enough to decode one Integer, then add next byte from buf
+ // to 'buffer'
+ // until totalBits >= width
+ while (totalBits < width) {
+ buffer = ((buffer << 8) | (buf[byteIdx] & 0xFF));
+ byteIdx++;
+ totalBits += 8;
+ }
- //If current available bits are enough to decode one Integer, then decode one Integer one by one
- //until left bits in 'buffer' is not enough to decode one Integer.
- while (totalBits >= width && valueIdx < 8) {
- values[valueIdx] = (int) (buffer >>> (totalBits - width));
- valueIdx++;
- totalBits -= width;
- buffer = (buffer & ((1 << totalBits) - 1));
- }
- }
+ // If current available bits are enough to decode one Integer, then decode one Integer one by
+ // one
+ // until left bits in 'buffer' is not enough to decode one Integer.
+ while (totalBits >= width && valueIdx < 8) {
+ values[valueIdx] = (int) (buffer >>> (totalBits - width));
+ valueIdx++;
+ totalBits -= width;
+ buffer = (buffer & ((1 << totalBits) - 1));
+ }
}
+ }
- /**
- * decode all values from 'buf' with specified offset and length
- * decoded result will be saved in the array named 'values'.
- *
- * @param buf: array where all bytes are in.
- * @param offset: the offset of first byte to be decoded in buf.
- * @param length: length of bytes to be decoded in buf.
- * @param values: decoded result.
- */
- public void unpackAllValues(byte[] buf, int offset, int length, int[] values) {
- int idx = 0;
- int k = 0;
- while (idx < length) {
- int[] tv = new int[8];
- //decode 8 values one time, current result will be saved in the array named 'tv'
- unpack8Values(buf, idx, tv);
- for (int i = 0; i < 8; i++) {
- values[k + i] = tv[i];
- }
- idx += width;
- k += 8;
- }
+ /**
+ * decode all values from 'buf' with specified offset and length decoded result will be saved in
+ * the array named 'values'.
+ *
+ * @param buf: array where all bytes are in.
+ * @param offset: the offset of first byte to be decoded in buf.
+ * @param length: length of bytes to be decoded in buf.
+ * @param values: decoded result.
+ */
+ public void unpackAllValues(byte[] buf, int offset, int length, int[] values) {
+ int idx = 0;
+ int k = 0;
+ while (idx < length) {
+ int[] tv = new int[8];
+ // decode 8 values one time, current result will be saved in the array named 'tv'
+ unpack8Values(buf, idx, tv);
+ for (int i = 0; i < 8; i++) {
+ values[k + i] = tv[i];
+ }
+ idx += width;
+ k += 8;
}
+ }
- public void setWidth(int width) {
- this.width = width;
- }
+ public void setWidth(int width) {
+ this.width = width;
+ }
}
diff --git a/src/main/java/cn/edu/tsinghua/tsfile/encoding/bitpacking/LongPacker.java b/src/main/java/cn/edu/tsinghua/tsfile/encoding/bitpacking/LongPacker.java
index a2bcfeff..289263f0 100644
--- a/src/main/java/cn/edu/tsinghua/tsfile/encoding/bitpacking/LongPacker.java
+++ b/src/main/java/cn/edu/tsinghua/tsfile/encoding/bitpacking/LongPacker.java
@@ -1,168 +1,168 @@
package cn.edu.tsinghua.tsfile.encoding.bitpacking;
/**
- * This class is used to encode(decode) Long in Java with specified bit-width.
- * User need to guarantee that the length of every given Long in binary mode
- * is less than or equal to the bit-width.
+ * This class is used to encode(decode) Long in Java with specified bit-width. User need to
+ * guarantee that the length of every given Long in binary mode is less than or equal to the
+ * bit-width.
*
- * e.g., if bit-width is 31, then Long '2147483648'(2^31) is not allowed but '2147483647'(2^31-1) is allowed.
+ * e.g., if bit-width is 31, then Long '2147483648'(2^31) is not allowed but '2147483647'(2^31-1) is
+ * allowed.
*
- * For a full example,
- * Width: 3
- * Input: 5 4 7 3 0 1 3 2
+ * For a full example, Width: 3 Input: 5 4 7 3 0 1 3 2
*
* Output:
*
- * +-----------------------+ +-----------------------+ +-----------------------+
- * |1 |0 |1 |1 |0 |0 |1 |1 | |1 |0 |1 |1 |0 |0 |0 |0 | |0 |1 |0 |1 |1 |0 |1 |0 |
- * +-----------------------+ +-----------------------+ +-----------------------+
- * +-----+ +-----+ +---------+ +-----+ +-----+ +---------+ +-----+ +-----+
- * 5 4 7 3 0 1 3 2
+ * +-----------------------+ +-----------------------+ +-----------------------+ |1 |0 |1 |1 |0 |0
+ * |1 |1 | |1 |0 |1 |1 |0 |0 |0 |0 | |0 |1 |0 |1 |1 |0 |1 |0 | +-----------------------+
+ * +-----------------------+ +-----------------------+ +-----+ +-----+ +---------+ +-----+ +-----+
+ * +---------+ +-----+ +-----+ 5 4 7 3 0 1 3 2
*
* @author Zhang Jinrui
*/
public class LongPacker {
- /**
- * Number of Long values for each pack operation
- */
- private static final int NUM_OF_LONGS = 8;
- /**
- * bit-width
- */
- private int width;
+ /**
+ * Number of Long values for each pack operation
+ */
+ private static final int NUM_OF_LONGS = 8;
+ /**
+ * bit-width
+ */
+ private int width;
- public LongPacker(int width) {
- this.width = width;
- }
+ public LongPacker(int width) {
+ this.width = width;
+ }
- /**
- * Encode 8 ({@link LongPacker#NUM_OF_LONGS}) Longs from the array 'values' with specified bit-width to bytes
- *
- * @param values - array where '8 Longs' are in
- * @param offset - the offset of first Long to be encoded
- * @param buf - encoded bytes, buf size must be equal to ({@link LongPacker#NUM_OF_LONGS}} * {@link IntPacker#width} / 8)
- */
- public void pack8Values(long[] values, int offset, byte[] buf) {
+ /**
+ * Encode 8 ({@link LongPacker#NUM_OF_LONGS}) Longs from the array 'values' with specified
+ * bit-width to bytes
+ *
+ * @param values - array where '8 Longs' are in
+ * @param offset - the offset of first Long to be encoded
+ * @param buf - encoded bytes, buf size must be equal to ({@link LongPacker#NUM_OF_LONGS}} *
+ * {@link IntPacker#width} / 8)
+ */
+ public void pack8Values(long[] values, int offset, byte[] buf) {
- int bufIdx = 0;
- int valueIdx = offset;
- //remaining bits for the current unfinished Integer
- int leftBit = 0;
+ int bufIdx = 0;
+ int valueIdx = offset;
+ // remaining bits for the current unfinished Integer
+ int leftBit = 0;
- while (valueIdx < NUM_OF_LONGS + offset) {
- // buffer is used for saving 64 bits as a part of result
- long buffer = 0;
- //remaining size of bits in the 'buffer'
- int leftSize = 64;
+ while (valueIdx < NUM_OF_LONGS + offset) {
+ // buffer is used for saving 64 bits as a part of result
+ long buffer = 0;
+ // remaining size of bits in the 'buffer'
+ int leftSize = 64;
- // encode the left bits of current Long to 'buffer'
- if (leftBit > 0) {
- buffer |= (values[valueIdx] << (64 - leftBit));
- leftSize -= leftBit;
- leftBit = 0;
- valueIdx++;
- }
+ // encode the left bits of current Long to 'buffer'
+ if (leftBit > 0) {
+ buffer |= (values[valueIdx] << (64 - leftBit));
+ leftSize -= leftBit;
+ leftBit = 0;
+ valueIdx++;
+ }
- while (leftSize >= width && valueIdx < NUM_OF_LONGS + offset) {
- //encode one Long to the 'buffer'
- buffer |= (values[valueIdx] << (leftSize - width));
- leftSize -= width;
- valueIdx++;
- }
- // If the remaining space of the buffer can not save the bits for one Long
- if (leftSize > 0 && valueIdx < NUM_OF_LONGS + offset) {
- // put the first 'leftSize' bits of the Long into remaining space of the buffer
- buffer |= (values[valueIdx] >>> (width - leftSize));
- leftBit = width - leftSize;
- leftSize = 0;
- }
+ while (leftSize >= width && valueIdx < NUM_OF_LONGS + offset) {
+ // encode one Long to the 'buffer'
+ buffer |= (values[valueIdx] << (leftSize - width));
+ leftSize -= width;
+ valueIdx++;
+ }
+ // If the remaining space of the buffer can not save the bits for one Long
+ if (leftSize > 0 && valueIdx < NUM_OF_LONGS + offset) {
+ // put the first 'leftSize' bits of the Long into remaining space of the buffer
+ buffer |= (values[valueIdx] >>> (width - leftSize));
+ leftBit = width - leftSize;
+ leftSize = 0;
+ }
- // put the buffer into the final result
- for (int j = 0; j < 8; j++) {
- buf[bufIdx] = (byte) ((buffer >>> ((8 - j - 1) * 8)) & 0xFF);
- bufIdx++;
- if (bufIdx >= width * 8 / 8) {
- return;
- }
- }
+ // put the buffer into the final result
+ for (int j = 0; j < 8; j++) {
+ buf[bufIdx] = (byte) ((buffer >>> ((8 - j - 1) * 8)) & 0xFF);
+ bufIdx++;
+ if (bufIdx >= width * 8 / 8) {
+ return;
}
+ }
}
+ }
- /**
- * decode values from byte array.
- *
- * @param buf - array where bytes are in.
- * @param offset - offset of first byte to be decoded in buf
- * @param values - decoded result , the size of values should be 8
- */
- public void unpack8Values(byte[] buf, int offset, long[] values) {
- int byteIdx = offset;
- int valueIdx = 0;
- //left bit(s) available for current byte in 'buf'
- int leftBits = 8;
- //bits that has been read for current long value which is to be decoded
- int totalBits = 0;
+ /**
+ * decode values from byte array.
+ *
+ * @param buf - array where bytes are in.
+ * @param offset - offset of first byte to be decoded in buf
+ * @param values - decoded result , the size of values should be 8
+ */
+ public void unpack8Values(byte[] buf, int offset, long[] values) {
+ int byteIdx = offset;
+ int valueIdx = 0;
+ // left bit(s) available for current byte in 'buf'
+ int leftBits = 8;
+ // bits that has been read for current long value which is to be decoded
+ int totalBits = 0;
- //decode long value one by one
- while (valueIdx < 8) {
- //set all the 64 bits in current value to '0'
- values[valueIdx] = 0;
- //read until 'totalBits' is equal to width
- while (totalBits < width) {
- //If 'leftBits' in current byte belongs to current long value
- if (width - totalBits >= leftBits) {
- //then put left bits in current byte to current long value
- values[valueIdx] = values[valueIdx] << leftBits;
- values[valueIdx] = (values[valueIdx] | ((((1L << leftBits) - 1)) & buf[byteIdx]));
- totalBits += leftBits;
- //get next byte
- byteIdx++;
- //set 'leftBits' in next byte to 8 because the next byte has not been used
- leftBits = 8;
- //Else take part of bits in 'leftBits' to current value.
- } else {
- //numbers of bits to be take
- int t = width - totalBits;
- values[valueIdx] = values[valueIdx] << t;
- values[valueIdx] = (values[valueIdx]
- | ((((1L << leftBits) - 1)) & buf[byteIdx]) >>> (leftBits - t));
- leftBits -= t;
- totalBits += t;
- }
- }
- //Start to decode next long value
- valueIdx++;
- totalBits = 0;
+ // decode long value one by one
+ while (valueIdx < 8) {
+ // set all the 64 bits in current value to '0'
+ values[valueIdx] = 0;
+ // read until 'totalBits' is equal to width
+ while (totalBits < width) {
+ // If 'leftBits' in current byte belongs to current long value
+ if (width - totalBits >= leftBits) {
+ // then put left bits in current byte to current long value
+ values[valueIdx] = values[valueIdx] << leftBits;
+ values[valueIdx] = (values[valueIdx] | ((((1L << leftBits) - 1)) & buf[byteIdx]));
+ totalBits += leftBits;
+ // get next byte
+ byteIdx++;
+ // set 'leftBits' in next byte to 8 because the next byte has not been used
+ leftBits = 8;
+ // Else take part of bits in 'leftBits' to current value.
+ } else {
+ // numbers of bits to be take
+ int t = width - totalBits;
+ values[valueIdx] = values[valueIdx] << t;
+ values[valueIdx] =
+ (values[valueIdx] | ((((1L << leftBits) - 1)) & buf[byteIdx]) >>> (leftBits - t));
+ leftBits -= t;
+ totalBits += t;
}
-
+ }
+ // Start to decode next long value
+ valueIdx++;
+ totalBits = 0;
}
- /**
- * decode all values from 'buf' with specified offset and length
- * decoded result will be saved in array named 'values'.
- *
- * @param buf: array where all bytes are in.
- * @param offset: the offset of first byte to be decoded in buf.
- * @param length: length of bytes to be decoded in buf.
- * @param values: decoded result
- */
- public void unpackAllValues(byte[] buf, int offset, int length, long[] values) {
- int idx = 0;
- int k = 0;
- while (idx < length) {
- long[] tv = new long[8];
- //decode 8 values one time, current result will be saved in the array named 'tv'
- unpack8Values(buf, idx, tv);
- for (int i = 0; i < 8; i++) {
- values[k + i] = tv[i];
- }
- idx += width;
- k += 8;
- }
- }
+ }
- public void setWidth(int width) {
- this.width = width;
+ /**
+ * decode all values from 'buf' with specified offset and length decoded result will be saved in
+ * array named 'values'.
+ *
+ * @param buf: array where all bytes are in.
+ * @param offset: the offset of first byte to be decoded in buf.
+ * @param length: length of bytes to be decoded in buf.
+ * @param values: decoded result
+ */
+ public void unpackAllValues(byte[] buf, int offset, int length, long[] values) {
+ int idx = 0;
+ int k = 0;
+ while (idx < length) {
+ long[] tv = new long[8];
+ // decode 8 values one time, current result will be saved in the array named 'tv'
+ unpack8Values(buf, idx, tv);
+ for (int i = 0; i < 8; i++) {
+ values[k + i] = tv[i];
+ }
+ idx += width;
+ k += 8;
}
+ }
+
+ public void setWidth(int width) {
+ this.width = width;
+ }
}
diff --git a/src/main/java/cn/edu/tsinghua/tsfile/encoding/common/EncodingConfig.java b/src/main/java/cn/edu/tsinghua/tsfile/encoding/common/EncodingConfig.java
index 1b4239d3..667dfdea 100644
--- a/src/main/java/cn/edu/tsinghua/tsfile/encoding/common/EncodingConfig.java
+++ b/src/main/java/cn/edu/tsinghua/tsfile/encoding/common/EncodingConfig.java
@@ -6,12 +6,12 @@
* @author xuyi
*/
public class EncodingConfig {
- // if number n repeats more than 8(>= 8), use rle encoding, otherwise use bit-packing
- public static final int RLE_MAX_REPEATED_NUM = 8;
+ // if number n repeats more than 8(>= 8), use rle encoding, otherwise use bit-packing
+ public static final int RLE_MAX_REPEATED_NUM = 8;
- // when to start a new bit-pacing group
- public static final int RLE_MAX_BIT_PACKED_NUM = 63;
+ // when to start a new bit-pacing group
+ public static final int RLE_MAX_BIT_PACKED_NUM = 63;
- // bit width for Bitmap Encoding
- public static final int BITMAP_BITWIDTH = 1;
+ // bit width for Bitmap Encoding
+ public static final int BITMAP_BITWIDTH = 1;
}
diff --git a/src/main/java/cn/edu/tsinghua/tsfile/encoding/common/EndianType.java b/src/main/java/cn/edu/tsinghua/tsfile/encoding/common/EndianType.java
index 1ba2dcf8..1f8826f7 100644
--- a/src/main/java/cn/edu/tsinghua/tsfile/encoding/common/EndianType.java
+++ b/src/main/java/cn/edu/tsinghua/tsfile/encoding/common/EndianType.java
@@ -6,5 +6,5 @@
* @author xuyi
*/
public enum EndianType {
- BIG_ENDIAN, LITTLE_ENDIAN
+ BIG_ENDIAN, LITTLE_ENDIAN
}
diff --git a/src/main/java/cn/edu/tsinghua/tsfile/encoding/decoder/BitmapDecoder.java b/src/main/java/cn/edu/tsinghua/tsfile/encoding/decoder/BitmapDecoder.java
index 70a1b00e..d5abaafc 100644
--- a/src/main/java/cn/edu/tsinghua/tsfile/encoding/decoder/BitmapDecoder.java
+++ b/src/main/java/cn/edu/tsinghua/tsfile/encoding/decoder/BitmapDecoder.java
@@ -8,7 +8,6 @@
import cn.edu.tsinghua.tsfile.file.metadata.enums.TSEncoding;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
@@ -23,202 +22,202 @@
* {@code }
*/
public class BitmapDecoder extends Decoder {
- private static final Logger LOGGER = LoggerFactory.getLogger(BitmapDecoder.class);
-
- /**
- * how many bytes for all encoded data in inputstream
- */
- private int length;
-
- /**
- * number of encoded data
- */
- private int number;
-
- /**
- * number of data left for reading in current buffer
- */
- private int currentCount;
-
- /**
- * each time decoder receives a inputstream, decoder creates a buffer to save all encoded data
- */
- private ByteArrayInputStream byteCache;
-
- /**
- * decoder reads all bitmap index from byteCache and save in Map
- */
- private Map buffer;
-
- /**
- * @param endianType deprecated
- */
- public BitmapDecoder(EndianType endianType) {
- super(TSEncoding.BITMAP);
- byteCache = new ByteArrayInputStream(new byte[0]);
- buffer = new HashMap<>();
- length = 0;
- number = 0;
- currentCount = 0;
- LOGGER.debug("tsfile-encoding BitmapDecoder: init bitmap decoder");
+ private static final Logger LOGGER = LoggerFactory.getLogger(BitmapDecoder.class);
+
+ /**
+ * how many bytes for all encoded data in inputstream
+ */
+ private int length;
+
+ /**
+ * number of encoded data
+ */
+ private int number;
+
+ /**
+ * number of data left for reading in current buffer
+ */
+ private int currentCount;
+
+ /**
+ * each time decoder receives a inputstream, decoder creates a buffer to save all encoded data
+ */
+ private ByteArrayInputStream byteCache;
+
+ /**
+ * decoder reads all bitmap index from byteCache and save in Map
+ */
+ private Map buffer;
+
+ /**
+ * @param endianType deprecated
+ */
+ public BitmapDecoder(EndianType endianType) {
+ super(TSEncoding.BITMAP);
+ byteCache = new ByteArrayInputStream(new byte[0]);
+ buffer = new HashMap<>();
+ length = 0;
+ number = 0;
+ currentCount = 0;
+ LOGGER.debug("tsfile-encoding BitmapDecoder: init bitmap decoder");
+ }
+
+ @Override
+ public int readInt(InputStream in) {
+ if (currentCount == 0) {
+ try {
+ reset();
+ getLengthAndNumber(in);
+ readNext();
+ } catch (IOException e) {
+ LOGGER.error(
+ "tsfile-encoding BitmapDecoder: error occurs when reading next number. lenght {}, number {}, current number {}, result buffer {}",
+ length, number, currentCount, buffer, e);
+ }
}
-
- @Override
- public int readInt(InputStream in) {
- if (currentCount == 0) {
- try {
- reset();
- getLengthAndNumber(in);
- readNext();
- } catch (IOException e) {
- LOGGER.error(
- "tsfile-encoding BitmapDecoder: error occurs when reading next number. lenght {}, number {}, current number {}, result buffer {}",
- length, number, currentCount, buffer, e);
- }
- }
- int result = 0;
- int index = (number - currentCount) / 8;
- int offset = 7 - ((number - currentCount) % 8);
- for (Map.Entry entry : buffer.entrySet()) {
- byte[] tmp = entry.getValue();
- if ((tmp[index] & ((byte) 1 << offset)) != 0) {
- result = entry.getKey();
- }
- }
- currentCount--;
- return result;
+ int result = 0;
+ int index = (number - currentCount) / 8;
+ int offset = 7 - ((number - currentCount) % 8);
+ for (Map.Entry entry : buffer.entrySet()) {
+ byte[] tmp = entry.getValue();
+ if ((tmp[index] & ((byte) 1 << offset)) != 0) {
+ result = entry.getKey();
+ }
}
-
- private void getLengthAndNumber(InputStream in) throws IOException {
- this.length = ReadWriteStreamUtils.readUnsignedVarInt(in);
- this.number = ReadWriteStreamUtils.readUnsignedVarInt(in);
- byte[] tmp = new byte[length];
- in.read(tmp, 0, length);
- this.byteCache = new ByteArrayInputStream(tmp);
- }
-
- /**
- * Decode all data from buffer and save them
- */
- private void readNext() throws IOException {
- int len = (this.number + 7) / 8;
- while (byteCache.available() > 0) {
- int value = ReadWriteStreamUtils.readUnsignedVarInt(byteCache);
- byte[] tmp = new byte[len];
- byteCache.read(tmp, 0, len);
- buffer.put(value, tmp);
- }
- currentCount = number;
+ currentCount--;
+ return result;
+ }
+
+ private void getLengthAndNumber(InputStream in) throws IOException {
+ this.length = ReadWriteStreamUtils.readUnsignedVarInt(in);
+ this.number = ReadWriteStreamUtils.readUnsignedVarInt(in);
+ byte[] tmp = new byte[length];
+ in.read(tmp, 0, length);
+ this.byteCache = new ByteArrayInputStream(tmp);
+ }
+
+ /**
+ * Decode all data from buffer and save them
+ */
+ private void readNext() throws IOException {
+ int len = (this.number + 7) / 8;
+ while (byteCache.available() > 0) {
+ int value = ReadWriteStreamUtils.readUnsignedVarInt(byteCache);
+ byte[] tmp = new byte[len];
+ byteCache.read(tmp, 0, len);
+ buffer.put(value, tmp);
}
-
- private void reset() {
- this.length = 0;
- this.number = 0;
- this.currentCount = 0;
- if (this.byteCache == null) {
- new ByteArrayInputStream(new byte[0]);
- } else {
- this.byteCache.reset();
- }
- if (this.buffer == null) {
- this.buffer = new HashMap<>();
- } else {
- this.buffer.clear();
- }
+ currentCount = number;
+ }
+
+ private void reset() {
+ this.length = 0;
+ this.number = 0;
+ this.currentCount = 0;
+ if (this.byteCache == null) {
+ new ByteArrayInputStream(new byte[0]);
+ } else {
+ this.byteCache.reset();
}
-
- /**
- * For special value in page list, get its bitmap index
- *
- * @param target value to get its bitmap index
- * @param pageList input page list
- * @return List(Pair of (length, bitmap index) )
- */
- public List> decodeAll(int target, List pageList) {
- List> resultList = new ArrayList<>();
- for (InputStream inputStream : pageList) {
- try {
- reset();
- getLengthAndNumber(inputStream);
- int byteArrayLength = (this.number + 7) / 8;
- byte[] tmp = new byte[byteArrayLength];
- while (byteCache.available() > 0) {
- int value = ReadWriteStreamUtils.readUnsignedVarInt(byteCache);
- if (value == target) {
- byteCache.read(tmp, 0, byteArrayLength);
- break;
- } else {
- byteCache.skip(byteArrayLength);
- }
- }
-
- resultList.add(new Pair<>(this.number, tmp));
- LOGGER.debug("tsfile-encoding BitmapDecoder: number {} in current page, byte length {}",
- this.number, byteArrayLength);
- } catch (IOException e) {
- LOGGER.error(
- "tsfile-encoding BitmapDecoder: error occurs when decoding all numbers in page {}, number {}",
- inputStream, this.number, e);
- }
- }
- return resultList;
+ if (this.buffer == null) {
+ this.buffer = new HashMap<>();
+ } else {
+ this.buffer.clear();
}
-
- /**
- * Check whether there is number left for reading
- *
- * @param in : decoded data saved in InputStream
- * @return true or false to indicate whether there is number left
- * @throws IOException cannot read next value
- * @see Decoder#hasNext(java.io.InputStream)
- */
- @Override
- public boolean hasNext(InputStream in) throws IOException {
- if (currentCount > 0 || in.available() > 0) {
- return true;
+ }
+
+ /**
+ * For special value in page list, get its bitmap index
+ *
+ * @param target value to get its bitmap index
+ * @param pageList input page list
+ * @return List(Pair of (length, bitmap index) )
+ */
+ public List> decodeAll(int target, List pageList) {
+ List> resultList = new ArrayList<>();
+ for (InputStream inputStream : pageList) {
+ try {
+ reset();
+ getLengthAndNumber(inputStream);
+ int byteArrayLength = (this.number + 7) / 8;
+ byte[] tmp = new byte[byteArrayLength];
+ while (byteCache.available() > 0) {
+ int value = ReadWriteStreamUtils.readUnsignedVarInt(byteCache);
+ if (value == target) {
+ byteCache.read(tmp, 0, byteArrayLength);
+ break;
+ } else {
+ byteCache.skip(byteArrayLength);
+ }
}
- return false;
- }
-
- /**
- * In current version, boolean value is equal to Enums value in schema
- *
- * @param in : decoded data saved in InputStream
- * @throws TSFileDecodingException cannot read next value
- * @see Decoder#readBoolean(java.io.InputStream)
- */
- @Override
- public boolean readBoolean(InputStream in) {
- throw new TSFileDecodingException("Method readBoolean is not supported by BitmapDecoder");
- }
-
- @Override
- public short readShort(InputStream in) {
- throw new TSFileDecodingException("Method readShort is not supported by BitmapDecoder");
- }
- @Override
- public long readLong(InputStream in) {
- throw new TSFileDecodingException("Method readLong is not supported by BitmapDecoder");
+ resultList.add(new Pair<>(this.number, tmp));
+ LOGGER.debug("tsfile-encoding BitmapDecoder: number {} in current page, byte length {}",
+ this.number, byteArrayLength);
+ } catch (IOException e) {
+ LOGGER.error(
+ "tsfile-encoding BitmapDecoder: error occurs when decoding all numbers in page {}, number {}",
+ inputStream, this.number, e);
+ }
}
-
- @Override
- public float readFloat(InputStream in) {
- throw new TSFileDecodingException("Method readFloat is not supported by BitmapDecoder");
- }
-
- @Override
- public double readDouble(InputStream in) {
- throw new TSFileDecodingException("Method readDouble is not supported by BitmapDecoder");
- }
-
- @Override
- public Binary readBinary(InputStream in) {
- throw new TSFileDecodingException("Method readBinary is not supported by BitmapDecoder");
- }
-
- @Override
- public BigDecimal readBigDecimal(InputStream in) {
- throw new TSFileDecodingException("Method readBigDecimal is not supported by BitmapDecoder");
+ return resultList;
+ }
+
+ /**
+ * Check whether there is number left for reading
+ *
+ * @param in : decoded data saved in InputStream
+ * @return true or false to indicate whether there is number left
+ * @throws IOException cannot read next value
+ * @see Decoder#hasNext(java.io.InputStream)
+ */
+ @Override
+ public boolean hasNext(InputStream in) throws IOException {
+ if (currentCount > 0 || in.available() > 0) {
+ return true;
}
+ return false;
+ }
+
+ /**
+ * In current version, boolean value is equal to Enums value in schema
+ *
+ * @param in : decoded data saved in InputStream
+ * @throws TSFileDecodingException cannot read next value
+ * @see Decoder#readBoolean(java.io.InputStream)
+ */
+ @Override
+ public boolean readBoolean(InputStream in) {
+ throw new TSFileDecodingException("Method readBoolean is not supported by BitmapDecoder");
+ }
+
+ @Override
+ public short readShort(InputStream in) {
+ throw new TSFileDecodingException("Method readShort is not supported by BitmapDecoder");
+ }
+
+ @Override
+ public long readLong(InputStream in) {
+ throw new TSFileDecodingException("Method readLong is not supported by BitmapDecoder");
+ }
+
+ @Override
+ public float readFloat(InputStream in) {
+ throw new TSFileDecodingException("Method readFloat is not supported by BitmapDecoder");
+ }
+
+ @Override
+ public double readDouble(InputStream in) {
+ throw new TSFileDecodingException("Method readDouble is not supported by BitmapDecoder");
+ }
+
+ @Override
+ public Binary readBinary(InputStream in) {
+ throw new TSFileDecodingException("Method readBinary is not supported by BitmapDecoder");
+ }
+
+ @Override
+ public BigDecimal readBigDecimal(InputStream in) {
+ throw new TSFileDecodingException("Method readBigDecimal is not supported by BitmapDecoder");
+ }
}
diff --git a/src/main/java/cn/edu/tsinghua/tsfile/encoding/decoder/Decoder.java b/src/main/java/cn/edu/tsinghua/tsfile/encoding/decoder/Decoder.java
index cd82a327..1506374a 100644
--- a/src/main/java/cn/edu/tsinghua/tsfile/encoding/decoder/Decoder.java
+++ b/src/main/java/cn/edu/tsinghua/tsfile/encoding/decoder/Decoder.java
@@ -6,7 +6,6 @@
import cn.edu.tsinghua.tsfile.file.metadata.enums.TSDataType;
import cn.edu.tsinghua.tsfile.file.metadata.enums.TSEncoding;
import cn.edu.tsinghua.tsfile.format.Encoding;
-
import java.io.IOException;
import java.io.InputStream;
import java.math.BigDecimal;
@@ -15,70 +14,72 @@
* @author Zhang Jinrui
*/
public abstract class Decoder {
- public TSEncoding type;
+ public TSEncoding type;
- public Decoder(TSEncoding type) {
- this.type = type;
- }
+ public Decoder(TSEncoding type) {
+ this.type = type;
+ }
- public static Decoder getDecoderByType(Encoding type, TSDataType dataType) {
- // PLA and DFT encoding are not supported in current version
- if (type == Encoding.PLAIN) {
- return new PlainDecoder(EndianType.LITTLE_ENDIAN);
- } else if (type == Encoding.RLE && dataType == TSDataType.BOOLEAN) {
- return new IntRleDecoder(EndianType.LITTLE_ENDIAN);
- } else if (type == Encoding.TS_2DIFF && dataType == TSDataType.INT32) {
- return new DeltaBinaryDecoder.IntDeltaDecoder();
- } else if (type == Encoding.TS_2DIFF && dataType == TSDataType.INT64) {
- return new DeltaBinaryDecoder.LongDeltaDecoder();
- } else if (type == Encoding.RLE && dataType == TSDataType.INT32) {
- return new IntRleDecoder(EndianType.LITTLE_ENDIAN);
- } else if (type == Encoding.RLE && dataType == TSDataType.INT64) {
- return new LongRleDecoder(EndianType.LITTLE_ENDIAN);
- } else if (type == Encoding.BITMAP && dataType == TSDataType.ENUMS) {
- return new BitmapDecoder(EndianType.LITTLE_ENDIAN);
- } else if ((dataType == TSDataType.FLOAT || dataType == TSDataType.DOUBLE) && (type == Encoding.RLE || type == Encoding.TS_2DIFF) ) {
- return new FloatDecoder(TSEncoding.valueOf(type.toString()), dataType);
- } else if (type == Encoding.GORILLA && dataType == TSDataType.FLOAT) {
- return new SinglePrecisionDecoder();
- } else if (type == Encoding.GORILLA && dataType == TSDataType.DOUBLE) {
- return new DoublePrecisionDecoder();
- } else {
- throw new TSFileDecodingException("Decoder not found:" + type + " , DataType is :" + dataType);
- }
+ public static Decoder getDecoderByType(Encoding type, TSDataType dataType) {
+ // PLA and DFT encoding are not supported in current version
+ if (type == Encoding.PLAIN) {
+ return new PlainDecoder(EndianType.LITTLE_ENDIAN);
+ } else if (type == Encoding.RLE && dataType == TSDataType.BOOLEAN) {
+ return new IntRleDecoder(EndianType.LITTLE_ENDIAN);
+ } else if (type == Encoding.TS_2DIFF && dataType == TSDataType.INT32) {
+ return new DeltaBinaryDecoder.IntDeltaDecoder();
+ } else if (type == Encoding.TS_2DIFF && dataType == TSDataType.INT64) {
+ return new DeltaBinaryDecoder.LongDeltaDecoder();
+ } else if (type == Encoding.RLE && dataType == TSDataType.INT32) {
+ return new IntRleDecoder(EndianType.LITTLE_ENDIAN);
+ } else if (type == Encoding.RLE && dataType == TSDataType.INT64) {
+ return new LongRleDecoder(EndianType.LITTLE_ENDIAN);
+ } else if (type == Encoding.BITMAP && dataType == TSDataType.ENUMS) {
+ return new BitmapDecoder(EndianType.LITTLE_ENDIAN);
+ } else if ((dataType == TSDataType.FLOAT || dataType == TSDataType.DOUBLE)
+ && (type == Encoding.RLE || type == Encoding.TS_2DIFF)) {
+ return new FloatDecoder(TSEncoding.valueOf(type.toString()), dataType);
+ } else if (type == Encoding.GORILLA && dataType == TSDataType.FLOAT) {
+ return new SinglePrecisionDecoder();
+ } else if (type == Encoding.GORILLA && dataType == TSDataType.DOUBLE) {
+ return new DoublePrecisionDecoder();
+ } else {
+ throw new TSFileDecodingException(
+ "Decoder not found:" + type + " , DataType is :" + dataType);
}
+ }
- public int readInt(InputStream in) {
- throw new TSFileDecodingException("Method readInt is not supproted by Decoder");
- }
+ public int readInt(InputStream in) {
+ throw new TSFileDecodingException("Method readInt is not supproted by Decoder");
+ }
- public boolean readBoolean(InputStream in) {
- throw new TSFileDecodingException("Method readBoolean is not supproted by Decoder");
- }
+ public boolean readBoolean(InputStream in) {
+ throw new TSFileDecodingException("Method readBoolean is not supproted by Decoder");
+ }
- public short readShort(InputStream in) {
- throw new TSFileDecodingException("Method readShort is not supproted by Decoder");
- }
+ public short readShort(InputStream in) {
+ throw new TSFileDecodingException("Method readShort is not supproted by Decoder");
+ }
- public long readLong(InputStream in) {
- throw new TSFileDecodingException("Method readLong is not supproted by Decoder");
- }
+ public long readLong(InputStream in) {
+ throw new TSFileDecodingException("Method readLong is not supproted by Decoder");
+ }
- public float readFloat(InputStream in) {
- throw new TSFileDecodingException("Method readFloat is not supproted by Decoder");
- }
+ public float readFloat(InputStream in) {
+ throw new TSFileDecodingException("Method readFloat is not supproted by Decoder");
+ }
- public double readDouble(InputStream in) {
- throw new TSFileDecodingException("Method readDouble is not supproted by Decoder");
- }
+ public double readDouble(InputStream in) {
+ throw new TSFileDecodingException("Method readDouble is not supproted by Decoder");
+ }
- public Binary readBinary(InputStream in) {
- throw new TSFileDecodingException("Method readBinary is not supproted by Decoder");
- }
+ public Binary readBinary(InputStream in) {
+ throw new TSFileDecodingException("Method readBinary is not supproted by Decoder");
+ }
- public BigDecimal readBigDecimal(InputStream in) {
- throw new TSFileDecodingException("Method readBigDecimal is not supproted by Decoder");
- }
+ public BigDecimal readBigDecimal(InputStream in) {
+ throw new TSFileDecodingException("Method readBigDecimal is not supproted by Decoder");
+ }
- public abstract boolean hasNext(InputStream in) throws IOException;
+ public abstract boolean hasNext(InputStream in) throws IOException;
}
diff --git a/src/main/java/cn/edu/tsinghua/tsfile/encoding/decoder/DeltaBinaryDecoder.java b/src/main/java/cn/edu/tsinghua/tsfile/encoding/decoder/DeltaBinaryDecoder.java
index eb464c0b..649642b1 100644
--- a/src/main/java/cn/edu/tsinghua/tsfile/encoding/decoder/DeltaBinaryDecoder.java
+++ b/src/main/java/cn/edu/tsinghua/tsfile/encoding/decoder/DeltaBinaryDecoder.java
@@ -5,7 +5,6 @@
import cn.edu.tsinghua.tsfile.file.metadata.enums.TSEncoding;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-
import java.io.IOException;
import java.io.InputStream;
@@ -17,225 +16,225 @@
* @see DeltaBinaryEncoder
*/
public abstract class DeltaBinaryDecoder extends Decoder {
- private static final Logger LOG = LoggerFactory.getLogger(DeltaBinaryDecoder.class);
- protected long count = 0;
- protected byte[] deltaBuf;
-
+ private static final Logger LOG = LoggerFactory.getLogger(DeltaBinaryDecoder.class);
+ protected long count = 0;
+ protected byte[] deltaBuf;
+
+ /**
+ * the first value in one pack.
+ */
+ protected int readIntTotalCount = 0;
+ protected int nextReadIndex = 0;
+ /**
+ * max bit length of all value in a pack
+ */
+ protected int packWidth;
+ /**
+ * data number in this pack
+ */
+ protected int packNum;
+
+ /**
+ * how many bytes data takes after encoding
+ */
+ protected int encodingLength;
+
+ public DeltaBinaryDecoder() {
+ super(TSEncoding.TS_2DIFF);
+ }
+
+ protected abstract void readHeader(InputStream in) throws IOException;
+
+ protected abstract void allocateDataArray();
+
+ protected abstract void readValue(int i);
+
+ /**
+ * calculate the bytes length containing v bits
+ *
+ * @param v - number of bits
+ * @return number of bytes
+ */
+ protected int ceil(int v) {
+ return (int) Math.ceil((double) (v) / 8.0);
+ }
+
+
+ @Override
+ public boolean hasNext(InputStream in) throws IOException {
+ return (nextReadIndex < readIntTotalCount) || in.available() > 0;
+ }
+
+
+ public static class IntDeltaDecoder extends DeltaBinaryDecoder {
+ private int firstValue;
+ private int[] data;
+ private int previous;
/**
- * the first value in one pack.
+ * minimum value for all difference
*/
- protected int readIntTotalCount = 0;
- protected int nextReadIndex = 0;
- /**
- * max bit length of all value in a pack
- */
- protected int packWidth;
+ private int minDeltaBase;
+
+ public IntDeltaDecoder() {
+ super();
+ }
+
/**
- * data number in this pack
+ * if there's no decoded data left, decode next pack into {@code data}
+ *
+ * @param in InputStream
+ * @return int
+ * @throws IOException cannot read T from InputStream
*/
- protected int packNum;
+ protected int readT(InputStream in) throws IOException {
+ if (nextReadIndex == readIntTotalCount)
+ return loadIntBatch(in);
+ return data[nextReadIndex++];
+ }
+
+ @Override
+ public int readInt(InputStream in) {
+ try {
+ return readT(in);
+ } catch (IOException e) {
+ LOG.warn("meet IOException when load batch from InputStream, return 0");
+ return 0;
+ }
+ }
/**
- * how many bytes data takes after encoding
+ * if remaining data has been run out, load next pack from InputStream
+ *
+ * @param in InputStream
+ * @return int
+ * @throws IOException cannot load batch from InputStream
*/
- protected int encodingLength;
+ protected int loadIntBatch(InputStream in) throws IOException {
+ packNum = BytesUtils.readInt(in);
+ packWidth = BytesUtils.readInt(in);
+ count++;
+ readHeader(in);
+
+ encodingLength = ceil(packNum * packWidth);
+ deltaBuf = BytesUtils.safeReadInputStreamToBytes(encodingLength, in);
+ allocateDataArray();
+
+ previous = firstValue;
+ readIntTotalCount = packNum;
+ nextReadIndex = 0;
+ readPack();
+ return firstValue;
+ }
+
+ private void readPack() throws IOException {
+ for (int i = 0; i < packNum; i++) {
+ readValue(i);
+ previous = data[i];
+ }
+ }
- public DeltaBinaryDecoder() {
- super(TSEncoding.TS_2DIFF);
+ @Override
+ protected void readHeader(InputStream in) throws IOException {
+ minDeltaBase = BytesUtils.readInt(in);
+ firstValue = BytesUtils.readInt(in);
+ }
+
+ @Override
+ protected void allocateDataArray() {
+ data = new int[packNum];
}
- protected abstract void readHeader(InputStream in) throws IOException;
+ @Override
+ protected void readValue(int i) {
+ int v = BytesUtils.bytesToInt(deltaBuf, packWidth * i, packWidth);
+ data[i] = previous + minDeltaBase + v;
+ }
+ }
- protected abstract void allocateDataArray();
+ public static class LongDeltaDecoder extends DeltaBinaryDecoder {
+ private long firstValue;
+ private long[] data;
+ private long previous;
+ /**
+ * minimum value for all difference
+ */
+ private long minDeltaBase;
- protected abstract void readValue(int i);
+ public LongDeltaDecoder() {
+ super();
+ }
/**
- * calculate the bytes length containing v bits
+ * if there's no decoded data left, decode next pack into {@code data}
*
- * @param v - number of bits
- * @return number of bytes
+ * @param in InputStream
+ * @return long value
+ * @throws IOException cannot read T from InputStream
*/
- protected int ceil(int v) {
- return (int) Math.ceil((double) (v) / 8.0);
+ protected long readT(InputStream in) throws IOException {
+ if (nextReadIndex == readIntTotalCount)
+ return loadIntBatch(in);
+ return data[nextReadIndex++];
+ }
+
+ /***
+ * if remaining data has been run out, load next pack from InputStream
+ *
+ * @param in InputStream
+ * @return long value
+ * @throws IOException cannot load batch from InputStream
+ */
+ protected long loadIntBatch(InputStream in) throws IOException {
+ packNum = BytesUtils.readInt(in);
+ packWidth = BytesUtils.readInt(in);
+ count++;
+ readHeader(in);
+
+ encodingLength = ceil(packNum * packWidth);
+ deltaBuf = BytesUtils.safeReadInputStreamToBytes(encodingLength, in);
+ allocateDataArray();
+
+ previous = firstValue;
+ readIntTotalCount = packNum;
+ nextReadIndex = 0;
+ readPack();
+ return firstValue;
+ }
+
+
+ private void readPack() throws IOException {
+ for (int i = 0; i < packNum; i++) {
+ readValue(i);
+ previous = data[i];
+ }
+ }
+
+ @Override
+ public long readLong(InputStream in) {
+ try {
+ return readT(in);
+ } catch (IOException e) {
+ LOG.warn("meet IOException when load batch from InputStream, return 0");
+ return 0;
+ }
}
+ @Override
+ protected void readHeader(InputStream in) throws IOException {
+ minDeltaBase = BytesUtils.readLong(in);
+ firstValue = BytesUtils.readLong(in);
+ }
@Override
- public boolean hasNext(InputStream in) throws IOException {
- return (nextReadIndex < readIntTotalCount) || in.available() > 0;
- }
-
-
- public static class IntDeltaDecoder extends DeltaBinaryDecoder {
- private int firstValue;
- private int[] data;
- private int previous;
- /**
- * minimum value for all difference
- */
- private int minDeltaBase;
-
- public IntDeltaDecoder() {
- super();
- }
-
- /**
- * if there's no decoded data left, decode next pack into {@code data}
- *
- * @param in InputStream
- * @return int
- * @throws IOException cannot read T from InputStream
- */
- protected int readT(InputStream in) throws IOException {
- if (nextReadIndex == readIntTotalCount)
- return loadIntBatch(in);
- return data[nextReadIndex++];
- }
-
- @Override
- public int readInt(InputStream in) {
- try {
- return readT(in);
- } catch (IOException e) {
- LOG.warn("meet IOException when load batch from InputStream, return 0");
- return 0;
- }
- }
-
- /**
- * if remaining data has been run out, load next pack from InputStream
- *
- * @param in InputStream
- * @return int
- * @throws IOException cannot load batch from InputStream
- */
- protected int loadIntBatch(InputStream in) throws IOException {
- packNum = BytesUtils.readInt(in);
- packWidth = BytesUtils.readInt(in);
- count++;
- readHeader(in);
-
- encodingLength = ceil(packNum * packWidth);
- deltaBuf = BytesUtils.safeReadInputStreamToBytes(encodingLength, in);
- allocateDataArray();
-
- previous = firstValue;
- readIntTotalCount = packNum;
- nextReadIndex = 0;
- readPack();
- return firstValue;
- }
-
- private void readPack() throws IOException {
- for (int i = 0; i < packNum; i++) {
- readValue(i);
- previous = data[i];
- }
- }
-
- @Override
- protected void readHeader(InputStream in) throws IOException {
- minDeltaBase = BytesUtils.readInt(in);
- firstValue = BytesUtils.readInt(in);
- }
-
- @Override
- protected void allocateDataArray() {
- data = new int[packNum];
- }
-
- @Override
- protected void readValue(int i) {
- int v = BytesUtils.bytesToInt(deltaBuf, packWidth * i, packWidth);
- data[i] = previous + minDeltaBase + v;
- }
- }
-
- public static class LongDeltaDecoder extends DeltaBinaryDecoder {
- private long firstValue;
- private long[] data;
- private long previous;
- /**
- * minimum value for all difference
- */
- private long minDeltaBase;
-
- public LongDeltaDecoder() {
- super();
- }
-
- /**
- * if there's no decoded data left, decode next pack into {@code data}
- *
- * @param in InputStream
- * @return long value
- * @throws IOException cannot read T from InputStream
- */
- protected long readT(InputStream in) throws IOException {
- if (nextReadIndex == readIntTotalCount)
- return loadIntBatch(in);
- return data[nextReadIndex++];
- }
-
- /***
- * if remaining data has been run out, load next pack from InputStream
- *
- * @param in InputStream
- * @return long value
- * @throws IOException cannot load batch from InputStream
- */
- protected long loadIntBatch(InputStream in) throws IOException {
- packNum = BytesUtils.readInt(in);
- packWidth = BytesUtils.readInt(in);
- count++;
- readHeader(in);
-
- encodingLength = ceil(packNum * packWidth);
- deltaBuf = BytesUtils.safeReadInputStreamToBytes(encodingLength, in);
- allocateDataArray();
-
- previous = firstValue;
- readIntTotalCount = packNum;
- nextReadIndex = 0;
- readPack();
- return firstValue;
- }
-
-
- private void readPack() throws IOException {
- for (int i = 0; i < packNum; i++) {
- readValue(i);
- previous = data[i];
- }
- }
-
- @Override
- public long readLong(InputStream in) {
- try {
- return readT(in);
- } catch (IOException e) {
- LOG.warn("meet IOException when load batch from InputStream, return 0");
- return 0;
- }
- }
-
- @Override
- protected void readHeader(InputStream in) throws IOException {
- minDeltaBase = BytesUtils.readLong(in);
- firstValue = BytesUtils.readLong(in);
- }
-
- @Override
- protected void allocateDataArray() {
- data = new long[packNum];
- }
-
- @Override
- protected void readValue(int i) {
- long v = BytesUtils.bytesToLong(deltaBuf, packWidth * i, packWidth);
- data[i] = previous + minDeltaBase + v;
- }
+ protected void allocateDataArray() {
+ data = new long[packNum];
+ }
+ @Override
+ protected void readValue(int i) {
+ long v = BytesUtils.bytesToLong(deltaBuf, packWidth * i, packWidth);
+ data[i] = previous + minDeltaBase + v;
}
+
+ }
}
diff --git a/src/main/java/cn/edu/tsinghua/tsfile/encoding/decoder/DoublePrecisionDecoder.java b/src/main/java/cn/edu/tsinghua/tsfile/encoding/decoder/DoublePrecisionDecoder.java
index 3c2385a2..a5614568 100644
--- a/src/main/java/cn/edu/tsinghua/tsfile/encoding/decoder/DoublePrecisionDecoder.java
+++ b/src/main/java/cn/edu/tsinghua/tsfile/encoding/decoder/DoublePrecisionDecoder.java
@@ -2,92 +2,91 @@
import java.io.IOException;
import java.io.InputStream;
-
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-
import cn.edu.tsinghua.tsfile.common.conf.TSFileConfig;
/**
* Decoder for value value using gorilla
*/
-public class DoublePrecisionDecoder extends GorillaDecoder{
- private static final Logger LOGGER = LoggerFactory.getLogger(DoublePrecisionDecoder.class);
- private long preValue;
-
- public DoublePrecisionDecoder() {
- }
-
- @Override
- public double readDouble(InputStream in) {
- if (!flag) {
- flag = true;
- try {
- int[] buf = new int[8];
- for (int i = 0; i < 8; i++)
- buf[i] = in.read();
- long res = 0L;
- for (int i = 0; i < 8; i++) {
- res += ((long) buf[i] << (i * 8));
- }
- preValue = res;
- double tmp = Double.longBitsToDouble(preValue);
- leadingZeroNum = Long.numberOfLeadingZeros(preValue);
- tailingZeroNum = Long.numberOfTrailingZeros(preValue);
- fillBuffer(in);
- getNextValue(in);
- return tmp;
- } catch (IOException e) {
- LOGGER.error("DoublePrecisionDecoder cannot read first double number because: {}", e.getMessage());
- }
- } else {
- try {
- double tmp = Double.longBitsToDouble(preValue);
- getNextValue(in);
- return tmp;
- } catch (IOException e) {
- LOGGER.error("DoublePrecisionDecoder cannot read following double number because: {}", e.getMessage());
- }
- }
- return Double.NaN;
- }
-
- /**
- * check whether there is any value to encode left
- *
- * @param in stream to read
- * @throws IOException cannot read from stream
- */
- private void getNextValue(InputStream in) throws IOException {
- nextFlag1 = readBit(in);
- // case: '0'
- if (!nextFlag1) {
- return;
- }
- nextFlag2 = readBit(in);
-
- if (!nextFlag2) {
- // case: '10'
- long tmp = 0;
- for (int i = 0; i < TSFileConfig.DOUBLE_LENGTH - leadingZeroNum - tailingZeroNum; i++) {
- long bit = readBit(in) ? 1 : 0;
- tmp |= (bit << (TSFileConfig.DOUBLE_LENGTH - 1 - leadingZeroNum - i));
- }
- tmp ^= preValue;
- preValue = tmp;
- } else {
- // case: '11'
- int leadingZeroNumTmp = readIntFromStream(in, TSFileConfig.DOUBLE_LEADING_ZERO_LENGTH);
- int lenTmp = readIntFromStream(in, TSFileConfig.DOUBLE_VALUE_LENGTH);
- long tmp = readLongFromStream(in, lenTmp);
- tmp <<= (TSFileConfig.DOUBLE_LENGTH - leadingZeroNumTmp - lenTmp);
- tmp ^= preValue;
- preValue = tmp;
- }
- leadingZeroNum = Long.numberOfLeadingZeros(preValue);
- tailingZeroNum = Long.numberOfTrailingZeros(preValue);
- if(Double.isNaN(Double.longBitsToDouble(preValue))){
- isEnd = true;
- }
- }
+public class DoublePrecisionDecoder extends GorillaDecoder {
+ private static final Logger LOGGER = LoggerFactory.getLogger(DoublePrecisionDecoder.class);
+ private long preValue;
+
+ public DoublePrecisionDecoder() {}
+
+ @Override
+ public double readDouble(InputStream in) {
+ if (!flag) {
+ flag = true;
+ try {
+ int[] buf = new int[8];
+ for (int i = 0; i < 8; i++)
+ buf[i] = in.read();
+ long res = 0L;
+ for (int i = 0; i < 8; i++) {
+ res += ((long) buf[i] << (i * 8));
+ }
+ preValue = res;
+ double tmp = Double.longBitsToDouble(preValue);
+ leadingZeroNum = Long.numberOfLeadingZeros(preValue);
+ tailingZeroNum = Long.numberOfTrailingZeros(preValue);
+ fillBuffer(in);
+ getNextValue(in);
+ return tmp;
+ } catch (IOException e) {
+ LOGGER.error("DoublePrecisionDecoder cannot read first double number because: {}",
+ e.getMessage());
+ }
+ } else {
+ try {
+ double tmp = Double.longBitsToDouble(preValue);
+ getNextValue(in);
+ return tmp;
+ } catch (IOException e) {
+ LOGGER.error("DoublePrecisionDecoder cannot read following double number because: {}",
+ e.getMessage());
+ }
+ }
+ return Double.NaN;
+ }
+
+ /**
+ * check whether there is any value to encode left
+ *
+ * @param in stream to read
+ * @throws IOException cannot read from stream
+ */
+ private void getNextValue(InputStream in) throws IOException {
+ nextFlag1 = readBit(in);
+ // case: '0'
+ if (!nextFlag1) {
+ return;
+ }
+ nextFlag2 = readBit(in);
+
+ if (!nextFlag2) {
+ // case: '10'
+ long tmp = 0;
+ for (int i = 0; i < TSFileConfig.DOUBLE_LENGTH - leadingZeroNum - tailingZeroNum; i++) {
+ long bit = readBit(in) ? 1 : 0;
+ tmp |= (bit << (TSFileConfig.DOUBLE_LENGTH - 1 - leadingZeroNum - i));
+ }
+ tmp ^= preValue;
+ preValue = tmp;
+ } else {
+ // case: '11'
+ int leadingZeroNumTmp = readIntFromStream(in, TSFileConfig.DOUBLE_LEADING_ZERO_LENGTH);
+ int lenTmp = readIntFromStream(in, TSFileConfig.DOUBLE_VALUE_LENGTH);
+ long tmp = readLongFromStream(in, lenTmp);
+ tmp <<= (TSFileConfig.DOUBLE_LENGTH - leadingZeroNumTmp - lenTmp);
+ tmp ^= preValue;
+ preValue = tmp;
+ }
+ leadingZeroNum = Long.numberOfLeadingZeros(preValue);
+ tailingZeroNum = Long.numberOfTrailingZeros(preValue);
+ if (Double.isNaN(Double.longBitsToDouble(preValue))) {
+ isEnd = true;
+ }
+ }
}
diff --git a/src/main/java/cn/edu/tsinghua/tsfile/encoding/decoder/FloatDecoder.java b/src/main/java/cn/edu/tsinghua/tsfile/encoding/decoder/FloatDecoder.java
index 620473f7..404a7471 100644
--- a/src/main/java/cn/edu/tsinghua/tsfile/encoding/decoder/FloatDecoder.java
+++ b/src/main/java/cn/edu/tsinghua/tsfile/encoding/decoder/FloatDecoder.java
@@ -9,122 +9,120 @@
import cn.edu.tsinghua.tsfile.file.metadata.enums.TSEncoding;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-
import java.io.IOException;
import java.io.InputStream;
/**
- * Decoder for float or double value using rle or two diff. For
- * more info about encoding pattern, see {@link FloatEncoder}
+ * Decoder for float or double value using rle or two diff. For more info about encoding pattern,
+ * see {@link FloatEncoder}
*/
public class FloatDecoder extends Decoder {
- private static final Logger LOGGER = LoggerFactory.getLogger(FloatDecoder.class);
- private Decoder decoder;
+ private static final Logger LOGGER = LoggerFactory.getLogger(FloatDecoder.class);
+ private Decoder decoder;
- /**
- * maxPointValue = 10^(maxPointNumer) maxPointNumber can be read from stream
- */
- private double maxPointValue;
+ /**
+ * maxPointValue = 10^(maxPointNumer) maxPointNumber can be read from stream
+ */
+ private double maxPointValue;
- /**
- * flag to indicate whether we have read maxPointNumber and calculate
- * maxPointValue
- */
- private boolean isMaxPointNumberRead;
+ /**
+ * flag to indicate whether we have read maxPointNumber and calculate maxPointValue
+ */
+ private boolean isMaxPointNumberRead;
- public FloatDecoder(TSEncoding encodingType, TSDataType dataType) {
- super(encodingType);
- if (encodingType == TSEncoding.RLE) {
- if (dataType == TSDataType.FLOAT) {
- decoder = new IntRleDecoder(EndianType.LITTLE_ENDIAN);
- LOGGER.debug("tsfile-encoding FloatDecoder: init decoder using int-rle and float");
- } else if (dataType == TSDataType.DOUBLE) {
- decoder = new LongRleDecoder(EndianType.LITTLE_ENDIAN);
- LOGGER.debug("tsfile-encoding FloatDecoder: init decoder using long-rle and double");
- } else {
- throw new TSFileDecodingException(
- String.format("data type %s is not supported by FloatDecoder", dataType));
- }
- } else if (encodingType == TSEncoding.TS_2DIFF) {
- if (dataType == TSDataType.FLOAT) {
- decoder = new DeltaBinaryDecoder.IntDeltaDecoder();
- LOGGER.debug("tsfile-encoding FloatDecoder: init decoder using int-delta and float");
- } else if (dataType == TSDataType.DOUBLE) {
- decoder = new DeltaBinaryDecoder.LongDeltaDecoder();
- LOGGER.debug("tsfile-encoding FloatDecoder: init decoder using long-delta and double");
- } else {
- throw new TSFileDecodingException(
- String.format("data type %s is not supported by FloatDecoder", dataType));
- }
- } else {
- throw new TSFileDecodingException(
- String.format("%s encoding is not supported by FloatDecoder", encodingType));
- }
- isMaxPointNumberRead = false;
+ public FloatDecoder(TSEncoding encodingType, TSDataType dataType) {
+ super(encodingType);
+ if (encodingType == TSEncoding.RLE) {
+ if (dataType == TSDataType.FLOAT) {
+ decoder = new IntRleDecoder(EndianType.LITTLE_ENDIAN);
+ LOGGER.debug("tsfile-encoding FloatDecoder: init decoder using int-rle and float");
+ } else if (dataType == TSDataType.DOUBLE) {
+ decoder = new LongRleDecoder(EndianType.LITTLE_ENDIAN);
+ LOGGER.debug("tsfile-encoding FloatDecoder: init decoder using long-rle and double");
+ } else {
+ throw new TSFileDecodingException(
+ String.format("data type %s is not supported by FloatDecoder", dataType));
+ }
+ } else if (encodingType == TSEncoding.TS_2DIFF) {
+ if (dataType == TSDataType.FLOAT) {
+ decoder = new DeltaBinaryDecoder.IntDeltaDecoder();
+ LOGGER.debug("tsfile-encoding FloatDecoder: init decoder using int-delta and float");
+ } else if (dataType == TSDataType.DOUBLE) {
+ decoder = new DeltaBinaryDecoder.LongDeltaDecoder();
+ LOGGER.debug("tsfile-encoding FloatDecoder: init decoder using long-delta and double");
+ } else {
+ throw new TSFileDecodingException(
+ String.format("data type %s is not supported by FloatDecoder", dataType));
+ }
+ } else {
+ throw new TSFileDecodingException(
+ String.format("%s encoding is not supported by FloatDecoder", encodingType));
}
+ isMaxPointNumberRead = false;
+ }
- @Override
- public float readFloat(InputStream in) {
- readMaxPointValue(in);
- int value = decoder.readInt(in);
- double result = value / maxPointValue;
- return (float) result;
- }
+ @Override
+ public float readFloat(InputStream in) {
+ readMaxPointValue(in);
+ int value = decoder.readInt(in);
+ double result = value / maxPointValue;
+ return (float) result;
+ }
- @Override
- public double readDouble(InputStream in) {
- readMaxPointValue(in);
- long value = decoder.readLong(in);
- double result = value / maxPointValue;
- return result;
- }
+ @Override
+ public double readDouble(InputStream in) {
+ readMaxPointValue(in);
+ long value = decoder.readLong(in);
+ double result = value / maxPointValue;
+ return result;
+ }
- private void readMaxPointValue(InputStream in) {
- try {
- if (!isMaxPointNumberRead) {
- int maxPointNumber = ReadWriteStreamUtils.readUnsignedVarInt(in);
- if (maxPointNumber <= 0) {
- maxPointValue = 1;
- } else {
- maxPointValue = Math.pow(10, maxPointNumber);
- }
- isMaxPointNumberRead = true;
- }
- } catch (IOException e) {
- LOGGER.error("tsfile-encoding FloatDecoder: error occurs when reading maxPointValue", e);
+ private void readMaxPointValue(InputStream in) {
+ try {
+ if (!isMaxPointNumberRead) {
+ int maxPointNumber = ReadWriteStreamUtils.readUnsignedVarInt(in);
+ if (maxPointNumber <= 0) {
+ maxPointValue = 1;
+ } else {
+ maxPointValue = Math.pow(10, maxPointNumber);
}
+ isMaxPointNumberRead = true;
+ }
+ } catch (IOException e) {
+ LOGGER.error("tsfile-encoding FloatDecoder: error occurs when reading maxPointValue", e);
}
+ }
- @Override
- public boolean hasNext(InputStream in) throws IOException {
- if (decoder == null) {
- return false;
- }
- return decoder.hasNext(in);
+ @Override
+ public boolean hasNext(InputStream in) throws IOException {
+ if (decoder == null) {
+ return false;
}
+ return decoder.hasNext(in);
+ }
- @Override
- public Binary readBinary(InputStream in) {
- throw new TSFileDecodingException("Method readBinary is not supproted by FloatDecoder");
- }
+ @Override
+ public Binary readBinary(InputStream in) {
+ throw new TSFileDecodingException("Method readBinary is not supproted by FloatDecoder");
+ }
- @Override
- public boolean readBoolean(InputStream in) {
- throw new TSFileDecodingException("Method readBoolean is not supproted by FloatDecoder");
- }
+ @Override
+ public boolean readBoolean(InputStream in) {
+ throw new TSFileDecodingException("Method readBoolean is not supproted by FloatDecoder");
+ }
- @Override
- public short readShort(InputStream in) {
- throw new TSFileDecodingException("Method readShort is not supproted by FloatDecoder");
- }
+ @Override
+ public short readShort(InputStream in) {
+ throw new TSFileDecodingException("Method readShort is not supproted by FloatDecoder");
+ }
- @Override
- public int readInt(InputStream in) {
- throw new TSFileDecodingException("Method readInt is not supproted by FloatDecoder");
- }
+ @Override
+ public int readInt(InputStream in) {
+ throw new TSFileDecodingException("Method readInt is not supproted by FloatDecoder");
+ }
- @Override
- public long readLong(InputStream in) {
- throw new TSFileDecodingException("Method readLong is not supproted by FloatDecoder");
- }
+ @Override
+ public long readLong(InputStream in) {
+ throw new TSFileDecodingException("Method readLong is not supproted by FloatDecoder");
+ }
}
diff --git a/src/main/java/cn/edu/tsinghua/tsfile/encoding/decoder/GorillaDecoder.java b/src/main/java/cn/edu/tsinghua/tsfile/encoding/decoder/GorillaDecoder.java
index 4a91301b..ddae4e69 100644
--- a/src/main/java/cn/edu/tsinghua/tsfile/encoding/decoder/GorillaDecoder.java
+++ b/src/main/java/cn/edu/tsinghua/tsfile/encoding/decoder/GorillaDecoder.java
@@ -2,98 +2,100 @@
import java.io.IOException;
import java.io.InputStream;
-
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-
import cn.edu.tsinghua.tsfile.file.metadata.enums.TSEncoding;
public abstract class GorillaDecoder extends Decoder {
- private static final Logger LOGGER = LoggerFactory.getLogger(GorillaDecoder.class);
- protected static final int EOF = -1;
- // flag to indicate whether the first value is read from stream
- protected boolean flag;
- protected int leadingZeroNum, tailingZeroNum;
- protected boolean isEnd;
- // 8-bit buffer of bits to write out
- protected int buffer;
- // number of bits remaining in buffer
- protected int numberLeftInBuffer;
-
- protected boolean nextFlag1;
- protected boolean nextFlag2;
+ private static final Logger LOGGER = LoggerFactory.getLogger(GorillaDecoder.class);
+ protected static final int EOF = -1;
+ // flag to indicate whether the first value is read from stream
+ protected boolean flag;
+ protected int leadingZeroNum, tailingZeroNum;
+ protected boolean isEnd;
+ // 8-bit buffer of bits to write out
+ protected int buffer;
+ // number of bits remaining in buffer
+ protected int numberLeftInBuffer;
- public GorillaDecoder() {
- super(TSEncoding.GORILLA);
- this.flag = false;
- this.isEnd = false;
- }
+ protected boolean nextFlag1;
+ protected boolean nextFlag2;
- @Override
- public boolean hasNext(InputStream in) throws IOException {
- if (in.available() > 0 || !isEnd) {
- return true;
- }
- return false;
- }
+ public GorillaDecoder() {
+ super(TSEncoding.GORILLA);
+ this.flag = false;
+ this.isEnd = false;
+ }
- protected boolean isEmpty() {
- return buffer == EOF;
+ @Override
+ public boolean hasNext(InputStream in) throws IOException {
+ if (in.available() > 0 || !isEnd) {
+ return true;
}
-
- protected boolean readBit(InputStream in) throws IOException {
- if(numberLeftInBuffer == 0 && !isEnd){
- fillBuffer(in);
- }
- if (isEmpty()) throw new IOException("Reading from empty input stream");
- numberLeftInBuffer--;
- return ((buffer >> numberLeftInBuffer) & 1) == 1;
+ return false;
+ }
+
+ protected boolean isEmpty() {
+ return buffer == EOF;
+ }
+
+ protected boolean readBit(InputStream in) throws IOException {
+ if (numberLeftInBuffer == 0 && !isEnd) {
+ fillBuffer(in);
}
-
- /**
- * read one byte and save in buffer
- * @param in stream to read
- */
- protected void fillBuffer(InputStream in) {
- try {
- buffer = in.read();
- numberLeftInBuffer = 8;
- } catch (IOException e) {
- LOGGER.error("Failed to fill a new buffer, because {}",e.getMessage());
- buffer = EOF;
- numberLeftInBuffer = -1;
- }
+ if (isEmpty())
+ throw new IOException("Reading from empty input stream");
+ numberLeftInBuffer--;
+ return ((buffer >> numberLeftInBuffer) & 1) == 1;
+ }
+
+ /**
+ * read one byte and save in buffer
+ *
+ * @param in stream to read
+ */
+ protected void fillBuffer(InputStream in) {
+ try {
+ buffer = in.read();
+ numberLeftInBuffer = 8;
+ } catch (IOException e) {
+ LOGGER.error("Failed to fill a new buffer, because {}", e.getMessage());
+ buffer = EOF;
+ numberLeftInBuffer = -1;
+ }
+ }
+
+ /**
+ * read some bits and convert them to a int value
+ *
+ * @param in stream to read
+ * @param len number of bit to read
+ * @return converted int value
+ * @throws IOException cannot read from stream
+ */
+ protected int readIntFromStream(InputStream in, int len) throws IOException {
+ int num = 0;
+ for (int i = 0; i < len; i++) {
+ int bit = readBit(in) ? 1 : 0;
+ num |= bit << (len - 1 - i);
+ }
+ return num;
+ }
+
+ /**
+ * read some bits and convert them to a long value
+ *
+ * @param in stream to read
+ * @param len number of bit to read
+ * @return converted long value
+ * @throws IOException cannot read from stream
+ */
+ protected long readLongFromStream(InputStream in, int len) throws IOException {
+ long num = 0;
+ for (int i = 0; i < len; i++) {
+ long bit = (long) (readBit(in) ? 1 : 0);
+ num |= bit << (len - 1 - i);
}
-
- /**
- * read some bits and convert them to a int value
- * @param in stream to read
- * @param len number of bit to read
- * @return converted int value
- * @throws IOException cannot read from stream
- */
- protected int readIntFromStream(InputStream in, int len) throws IOException{
- int num = 0;
- for (int i = 0; i < len; i++) {
- int bit = readBit(in) ? 1 : 0;
- num |= bit << (len - 1 - i);
- }
- return num;
- }
-
- /**
- * read some bits and convert them to a long value
- * @param in stream to read
- * @param len number of bit to read
- * @return converted long value
- * @throws IOException cannot read from stream
- */
- protected long readLongFromStream(InputStream in, int len) throws IOException{
- long num = 0;
- for (int i = 0; i < len; i++) {
- long bit = (long)(readBit(in) ? 1 : 0);
- num |= bit << (len - 1 - i);
- }
- return num;
- }
+ return num;
+ }
}
diff --git a/src/main/java/cn/edu/tsinghua/tsfile/encoding/decoder/IntRleDecoder.java b/src/main/java/cn/edu/tsinghua/tsfile/encoding/decoder/IntRleDecoder.java
index 4a79134a..05c267be 100644
--- a/src/main/java/cn/edu/tsinghua/tsfile/encoding/decoder/IntRleDecoder.java
+++ b/src/main/java/cn/edu/tsinghua/tsfile/encoding/decoder/IntRleDecoder.java
@@ -1,109 +1,112 @@
-package cn.edu.tsinghua.tsfile.encoding.decoder;
-
-import cn.edu.tsinghua.tsfile.common.exception.TSFileDecodingException;
-import cn.edu.tsinghua.tsfile.common.utils.ReadWriteStreamUtils;
-import cn.edu.tsinghua.tsfile.encoding.bitpacking.IntPacker;
-import cn.edu.tsinghua.tsfile.encoding.common.EndianType;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.io.InputStream;
-
-/**
- * Decoder for int value using rle or bit-packing
- */
-public class IntRleDecoder extends RleDecoder {
- private static final Logger LOGGER = LoggerFactory.getLogger(IntRleDecoder.class);
-
- /**
- * current value for rle repeated value
- */
- private int currentValue;
-
- /**
- * buffer to save all values in group using bit-packing
- */
- private int[] currentBuffer;
-
- /**
- * packer for unpacking int value
- */
- private IntPacker packer;
-
- public IntRleDecoder(EndianType endianType) {
- super(endianType);
- currentValue = 0;
- }
-
- @Override
- public boolean readBoolean(InputStream in) {
- return this.readInt(in) == 0 ? false : true;
- }
-
- /**
- * read a int value from InputStream
- *
- * @param in - InputStream
- * @return value - current valid value
- */
- @Override
- public int readInt(InputStream in) {
- if (!isLengthAndBitWidthReaded) {
- //start to read a new rle+bit-packing pattern
- try {
- readLengthAndBitWidth(in);
- } catch (IOException e) {
- LOGGER.error("tsfile-encoding IntRleDecoder: error occurs when reading length", e);
- }
- }
-
- if (currentCount == 0) {
- try {
- readNext();
- } catch (IOException e) {
- LOGGER.error("tsfile-encoding IntRleDecoder: error occurs when reading all encoding number, length is {}, bit width is {}", length, bitWidth, e);
- }
- }
- --currentCount;
- int result = 0;
- switch (mode) {
- case RLE:
- result = currentValue;
- break;
- case BIT_PACKED:
- result = currentBuffer[bitPackingNum - currentCount - 1];
- break;
- default:
- throw new TSFileDecodingException(String.format("tsfile-encoding IntRleDecoder: not a valid mode %s", mode));
- }
-
- if (!hasNextPackage()) {
- isLengthAndBitWidthReaded = false;
- }
- return result;
- }
-
- @Override
- protected void initPacker() {
- packer = new IntPacker(bitWidth);
- }
-
- @Override
- protected void readNumberInRLE() throws IOException {
- currentValue = ReadWriteStreamUtils.readIntLittleEndianPaddedOnBitWidth(byteCache, bitWidth);
- }
-
- @Override
- protected void readBitPackingBuffer(int bitPackedGroupCount, int lastBitPackedNum) throws IOException {
- currentBuffer = new int[bitPackedGroupCount * config.RLE_MIN_REPEATED_NUM];
- byte[] bytes = new byte[bitPackedGroupCount * bitWidth];
- int bytesToRead = bitPackedGroupCount * bitWidth;
- bytesToRead = Math.min(bytesToRead, byteCache.available());
-// new DataInputStream(byteCache).readFully(bytes, 0, bytesToRead);
- byteCache.read(bytes, 0, bytesToRead);
-
- // save all int values in currentBuffer
- packer.unpackAllValues(bytes, 0, bytesToRead, currentBuffer);
- }
-}
+package cn.edu.tsinghua.tsfile.encoding.decoder;
+
+import cn.edu.tsinghua.tsfile.common.exception.TSFileDecodingException;
+import cn.edu.tsinghua.tsfile.common.utils.ReadWriteStreamUtils;
+import cn.edu.tsinghua.tsfile.encoding.bitpacking.IntPacker;
+import cn.edu.tsinghua.tsfile.encoding.common.EndianType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import java.io.IOException;
+import java.io.InputStream;
+
+/**
+ * Decoder for int value using rle or bit-packing
+ */
+public class IntRleDecoder extends RleDecoder {
+ private static final Logger LOGGER = LoggerFactory.getLogger(IntRleDecoder.class);
+
+ /**
+ * current value for rle repeated value
+ */
+ private int currentValue;
+
+ /**
+ * buffer to save all values in group using bit-packing
+ */
+ private int[] currentBuffer;
+
+ /**
+ * packer for unpacking int value
+ */
+ private IntPacker packer;
+
+ public IntRleDecoder(EndianType endianType) {
+ super(endianType);
+ currentValue = 0;
+ }
+
+ @Override
+ public boolean readBoolean(InputStream in) {
+ return this.readInt(in) == 0 ? false : true;
+ }
+
+ /**
+ * read a int value from InputStream
+ *
+ * @param in - InputStream
+ * @return value - current valid value
+ */
+ @Override
+ public int readInt(InputStream in) {
+ if (!isLengthAndBitWidthReaded) {
+ // start to read a new rle+bit-packing pattern
+ try {
+ readLengthAndBitWidth(in);
+ } catch (IOException e) {
+ LOGGER.error("tsfile-encoding IntRleDecoder: error occurs when reading length", e);
+ }
+ }
+
+ if (currentCount == 0) {
+ try {
+ readNext();
+ } catch (IOException e) {
+ LOGGER.error(
+ "tsfile-encoding IntRleDecoder: error occurs when reading all encoding number, length is {}, bit width is {}",
+ length, bitWidth, e);
+ }
+ }
+ --currentCount;
+ int result = 0;
+ switch (mode) {
+ case RLE:
+ result = currentValue;
+ break;
+ case BIT_PACKED:
+ result = currentBuffer[bitPackingNum - currentCount - 1];
+ break;
+ default:
+ throw new TSFileDecodingException(
+ String.format("tsfile-encoding IntRleDecoder: not a valid mode %s", mode));
+ }
+
+ if (!hasNextPackage()) {
+ isLengthAndBitWidthReaded = false;
+ }
+ return result;
+ }
+
+ @Override
+ protected void initPacker() {
+ packer = new IntPacker(bitWidth);
+ }
+
+ @Override
+ protected void readNumberInRLE() throws IOException {
+ currentValue = ReadWriteStreamUtils.readIntLittleEndianPaddedOnBitWidth(byteCache, bitWidth);
+ }
+
+ @Override
+ protected void readBitPackingBuffer(int bitPackedGroupCount, int lastBitPackedNum)
+ throws IOException {
+ currentBuffer = new int[bitPackedGroupCount * config.RLE_MIN_REPEATED_NUM];
+ byte[] bytes = new byte[bitPackedGroupCount * bitWidth];
+ int bytesToRead = bitPackedGroupCount * bitWidth;
+ bytesToRead = Math.min(bytesToRead, byteCache.available());
+ // new DataInputStream(byteCache).readFully(bytes, 0, bytesToRead);
+ byteCache.read(bytes, 0, bytesToRead);
+
+ // save all int values in currentBuffer
+ packer.unpackAllValues(bytes, 0, bytesToRead, currentBuffer);
+ }
+}
diff --git a/src/main/java/cn/edu/tsinghua/tsfile/encoding/decoder/LongRleDecoder.java b/src/main/java/cn/edu/tsinghua/tsfile/encoding/decoder/LongRleDecoder.java
index 819076b9..6301e6f7 100644
--- a/src/main/java/cn/edu/tsinghua/tsfile/encoding/decoder/LongRleDecoder.java
+++ b/src/main/java/cn/edu/tsinghua/tsfile/encoding/decoder/LongRleDecoder.java
@@ -1,104 +1,107 @@
-package cn.edu.tsinghua.tsfile.encoding.decoder;
-
-import cn.edu.tsinghua.tsfile.common.exception.TSFileDecodingException;
-import cn.edu.tsinghua.tsfile.common.utils.ReadWriteStreamUtils;
-import cn.edu.tsinghua.tsfile.encoding.bitpacking.LongPacker;
-import cn.edu.tsinghua.tsfile.encoding.common.EndianType;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.io.InputStream;
-
-/**
- * Decoder for long value using rle or bit-packing
- */
-public class LongRleDecoder extends RleDecoder {
- private static final Logger LOGGER = LoggerFactory.getLogger(LongRleDecoder.class);
-
- /**
- * current value for rle repeated value
- */
- private long currentValue;
-
- /**
- * buffer to save all values in group using bit-packing
- */
- private long[] currentBuffer;
-
- /**
- * packer for unpacking long value
- */
- private LongPacker packer;
-
- public LongRleDecoder(EndianType endianType) {
- super(endianType);
- currentValue = 0;
- }
-
- /**
- * read a long value from InputStream
- *
- * @param in - InputStream
- * @return value - current valid value
- */
- @Override
- public long readLong(InputStream in) {
- if (!isLengthAndBitWidthReaded) {
- //start to read a new rle+bit-packing pattern
- try {
- readLengthAndBitWidth(in);
- } catch (IOException e) {
- LOGGER.error("tsfile-encoding IntRleDecoder: error occurs when reading length", e);
- }
- }
-
- if (currentCount == 0) {
- try {
- readNext();
- } catch (IOException e) {
- LOGGER.error("tsfile-encoding IntRleDecoder: error occurs when reading all encoding number, length is {}, bit width is {}", length, bitWidth, e);
- }
- }
- --currentCount;
- long result = 0;
- switch (mode) {
- case RLE:
- result = currentValue;
- break;
- case BIT_PACKED:
- result = currentBuffer[bitPackingNum - currentCount - 1];
- break;
- default:
- throw new TSFileDecodingException(String.format("tsfile-encoding LongRleDecoder: not a valid mode %s", mode));
- }
-
- if (!hasNextPackage()) {
- isLengthAndBitWidthReaded = false;
- }
- return result;
- }
-
- @Override
- protected void initPacker() {
- packer = new LongPacker(bitWidth);
- }
-
- @Override
- protected void readNumberInRLE() throws IOException {
- currentValue = ReadWriteStreamUtils.readLongLittleEndianPaddedOnBitWidth(byteCache, bitWidth);
- }
-
- @Override
- protected void readBitPackingBuffer(int bitPackedGroupCount, int lastBitPackedNum) throws IOException {
- currentBuffer = new long[bitPackedGroupCount * config.RLE_MIN_REPEATED_NUM];
- byte[] bytes = new byte[bitPackedGroupCount * bitWidth];
- int bytesToRead = bitPackedGroupCount * bitWidth;
- bytesToRead = Math.min(bytesToRead, byteCache.available());
-// new DataInputStream(byteCache).readFully(bytes, 0, bytesToRead);
- byteCache.read(bytes, 0, bytesToRead);
-
- // save all long values in currentBuffer
- packer.unpackAllValues(bytes, 0, bytesToRead, currentBuffer);
- }
-}
+package cn.edu.tsinghua.tsfile.encoding.decoder;
+
+import cn.edu.tsinghua.tsfile.common.exception.TSFileDecodingException;
+import cn.edu.tsinghua.tsfile.common.utils.ReadWriteStreamUtils;
+import cn.edu.tsinghua.tsfile.encoding.bitpacking.LongPacker;
+import cn.edu.tsinghua.tsfile.encoding.common.EndianType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import java.io.IOException;
+import java.io.InputStream;
+
+/**
+ * Decoder for long value using rle or bit-packing
+ */
+public class LongRleDecoder extends RleDecoder {
+ private static final Logger LOGGER = LoggerFactory.getLogger(LongRleDecoder.class);
+
+ /**
+ * current value for rle repeated value
+ */
+ private long currentValue;
+
+ /**
+ * buffer to save all values in group using bit-packing
+ */
+ private long[] currentBuffer;
+
+ /**
+ * packer for unpacking long value
+ */
+ private LongPacker packer;
+
+ public LongRleDecoder(EndianType endianType) {
+ super(endianType);
+ currentValue = 0;
+ }
+
+ /**
+ * read a long value from InputStream
+ *
+ * @param in - InputStream
+ * @return value - current valid value
+ */
+ @Override
+ public long readLong(InputStream in) {
+ if (!isLengthAndBitWidthReaded) {
+ // start to read a new rle+bit-packing pattern
+ try {
+ readLengthAndBitWidth(in);
+ } catch (IOException e) {
+ LOGGER.error("tsfile-encoding IntRleDecoder: error occurs when reading length", e);
+ }
+ }
+
+ if (currentCount == 0) {
+ try {
+ readNext();
+ } catch (IOException e) {
+ LOGGER.error(
+ "tsfile-encoding IntRleDecoder: error occurs when reading all encoding number, length is {}, bit width is {}",
+ length, bitWidth, e);
+ }
+ }
+ --currentCount;
+ long result = 0;
+ switch (mode) {
+ case RLE:
+ result = currentValue;
+ break;
+ case BIT_PACKED:
+ result = currentBuffer[bitPackingNum - currentCount - 1];
+ break;
+ default:
+ throw new TSFileDecodingException(
+ String.format("tsfile-encoding LongRleDecoder: not a valid mode %s", mode));
+ }
+
+ if (!hasNextPackage()) {
+ isLengthAndBitWidthReaded = false;
+ }
+ return result;
+ }
+
+ @Override
+ protected void initPacker() {
+ packer = new LongPacker(bitWidth);
+ }
+
+ @Override
+ protected void readNumberInRLE() throws IOException {
+ currentValue = ReadWriteStreamUtils.readLongLittleEndianPaddedOnBitWidth(byteCache, bitWidth);
+ }
+
+ @Override
+ protected void readBitPackingBuffer(int bitPackedGroupCount, int lastBitPackedNum)
+ throws IOException {
+ currentBuffer = new long[bitPackedGroupCount * config.RLE_MIN_REPEATED_NUM];
+ byte[] bytes = new byte[bitPackedGroupCount * bitWidth];
+ int bytesToRead = bitPackedGroupCount * bitWidth;
+ bytesToRead = Math.min(bytesToRead, byteCache.available());
+ // new DataInputStream(byteCache).readFully(bytes, 0, bytesToRead);
+ byteCache.read(bytes, 0, bytesToRead);
+
+ // save all long values in currentBuffer
+ packer.unpackAllValues(bytes, 0, bytesToRead, currentBuffer);
+ }
+}
diff --git a/src/main/java/cn/edu/tsinghua/tsfile/encoding/decoder/PlainDecoder.java b/src/main/java/cn/edu/tsinghua/tsfile/encoding/decoder/PlainDecoder.java
index 7b3a843d..0f6e0257 100644
--- a/src/main/java/cn/edu/tsinghua/tsfile/encoding/decoder/PlainDecoder.java
+++ b/src/main/java/cn/edu/tsinghua/tsfile/encoding/decoder/PlainDecoder.java
@@ -6,7 +6,6 @@
import cn.edu.tsinghua.tsfile.file.metadata.enums.TSEncoding;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-
import java.io.IOException;
import java.io.InputStream;
import java.math.BigDecimal;
@@ -16,111 +15,111 @@
*/
public class PlainDecoder extends Decoder {
- private static final Logger LOGGER = LoggerFactory.getLogger(PlainDecoder.class);
- public EndianType endianType;
+ private static final Logger LOGGER = LoggerFactory.getLogger(PlainDecoder.class);
+ public EndianType endianType;
- public PlainDecoder(EndianType endianType) {
- super(TSEncoding.PLAIN);
- this.endianType = endianType;
- }
+ public PlainDecoder(EndianType endianType) {
+ super(TSEncoding.PLAIN);
+ this.endianType = endianType;
+ }
- @Override
- public boolean readBoolean(InputStream in) {
- try {
- int ch1 = in.read();
- if (ch1 == 0) {
- return false;
- } else {
- return true;
- }
- } catch (IOException e) {
- LOGGER.error("tsfile-encoding PlainDecoder: errors whewn read boolean", e);
- }
+ @Override
+ public boolean readBoolean(InputStream in) {
+ try {
+ int ch1 = in.read();
+ if (ch1 == 0) {
return false;
+ } else {
+ return true;
+ }
+ } catch (IOException e) {
+ LOGGER.error("tsfile-encoding PlainDecoder: errors whewn read boolean", e);
}
+ return false;
+ }
- @Override
- public short readShort(InputStream in) {
- try {
- int ch1 = in.read();
- int ch2 = in.read();
- if (this.endianType == EndianType.LITTLE_ENDIAN) {
- return (short) ((ch2 << 8) + ch1);
- } else {
- LOGGER.error(
- "tsfile-encoding PlainEncoder: current version does not support short value decoding");
- }
- } catch (IOException e) {
- LOGGER.error("tsfile-encoding PlainDecoder: errors whewn read short", e);
- }
- return -1;
+ @Override
+ public short readShort(InputStream in) {
+ try {
+ int ch1 = in.read();
+ int ch2 = in.read();
+ if (this.endianType == EndianType.LITTLE_ENDIAN) {
+ return (short) ((ch2 << 8) + ch1);
+ } else {
+ LOGGER.error(
+ "tsfile-encoding PlainEncoder: current version does not support short value decoding");
+ }
+ } catch (IOException e) {
+ LOGGER.error("tsfile-encoding PlainDecoder: errors whewn read short", e);
}
+ return -1;
+ }
- @Override
- public int readInt(InputStream in) {
- try {
- int ch1 = in.read();
- int ch2 = in.read();
- int ch3 = in.read();
- int ch4 = in.read();
- if (this.endianType == EndianType.LITTLE_ENDIAN) {
- return ch1 + (ch2 << 8) + (ch3 << 16) + (ch4 << 24);
- } else {
- LOGGER.error(
- "tsfile-encoding PlainEncoder: current version does not support int value encoding");
- }
- } catch (IOException e) {
- LOGGER.error("tsfile-encoding PlainDecoder: errors whewn read int", e);
- }
- return -1;
+ @Override
+ public int readInt(InputStream in) {
+ try {
+ int ch1 = in.read();
+ int ch2 = in.read();
+ int ch3 = in.read();
+ int ch4 = in.read();
+ if (this.endianType == EndianType.LITTLE_ENDIAN) {
+ return ch1 + (ch2 << 8) + (ch3 << 16) + (ch4 << 24);
+ } else {
+ LOGGER.error(
+ "tsfile-encoding PlainEncoder: current version does not support int value encoding");
+ }
+ } catch (IOException e) {
+ LOGGER.error("tsfile-encoding PlainDecoder: errors whewn read int", e);
}
+ return -1;
+ }
- @Override
- public long readLong(InputStream in) {
- int[] buf = new int[8];
- try {
- for (int i = 0; i < 8; i++)
- buf[i] = in.read();
- } catch (IOException e) {
- LOGGER.error("tsfile-encoding PlainDecoder: errors whewn read long", e);
- }
-
- Long res = 0L;
- for (int i = 0; i < 8; i++) {
- res += ((long) buf[i] << (i * 8));
- }
- return res;
+ @Override
+ public long readLong(InputStream in) {
+ int[] buf = new int[8];
+ try {
+ for (int i = 0; i < 8; i++)
+ buf[i] = in.read();
+ } catch (IOException e) {
+ LOGGER.error("tsfile-encoding PlainDecoder: errors whewn read long", e);
}
- @Override
- public float readFloat(InputStream in) {
- return Float.intBitsToFloat(readInt(in));
+ Long res = 0L;
+ for (int i = 0; i < 8; i++) {
+ res += ((long) buf[i] << (i * 8));
}
+ return res;
+ }
- @Override
- public double readDouble(InputStream in) {
- return Double.longBitsToDouble(readLong(in));
- }
+ @Override
+ public float readFloat(InputStream in) {
+ return Float.intBitsToFloat(readInt(in));
+ }
- @Override
- public Binary readBinary(InputStream in) {
- int length = readInt(in);
- byte[] buf = new byte[length];
- try {
- in.read(buf, 0, buf.length);
- } catch (IOException e) {
- LOGGER.error("tsfile-encoding PlainDecoder: errors whewn read binary", e);
- }
- return new Binary(buf);
- }
+ @Override
+ public double readDouble(InputStream in) {
+ return Double.longBitsToDouble(readLong(in));
+ }
- @Override
- public boolean hasNext(InputStream in) throws IOException {
- return in.available() > 0;
+ @Override
+ public Binary readBinary(InputStream in) {
+ int length = readInt(in);
+ byte[] buf = new byte[length];
+ try {
+ in.read(buf, 0, buf.length);
+ } catch (IOException e) {
+ LOGGER.error("tsfile-encoding PlainDecoder: errors whewn read binary", e);
}
+ return new Binary(buf);
+ }
- @Override
- public BigDecimal readBigDecimal(InputStream in) {
- throw new TSFileDecodingException("Method readBigDecimal is not supproted by PlainDecoder");
- }
+ @Override
+ public boolean hasNext(InputStream in) throws IOException {
+ return in.available() > 0;
+ }
+
+ @Override
+ public BigDecimal readBigDecimal(InputStream in) {
+ throw new TSFileDecodingException("Method readBigDecimal is not supproted by PlainDecoder");
+ }
}
diff --git a/src/main/java/cn/edu/tsinghua/tsfile/encoding/decoder/RleDecoder.java b/src/main/java/cn/edu/tsinghua/tsfile/encoding/decoder/RleDecoder.java
index 69ba33fa..2b15f406 100644
--- a/src/main/java/cn/edu/tsinghua/tsfile/encoding/decoder/RleDecoder.java
+++ b/src/main/java/cn/edu/tsinghua/tsfile/encoding/decoder/RleDecoder.java
@@ -1,216 +1,214 @@
-package cn.edu.tsinghua.tsfile.encoding.decoder;
-
-import cn.edu.tsinghua.tsfile.common.conf.TSFileConfig;
-import cn.edu.tsinghua.tsfile.common.conf.TSFileDescriptor;
-import cn.edu.tsinghua.tsfile.common.exception.TSFileDecodingException;
-import cn.edu.tsinghua.tsfile.common.utils.Binary;
-import cn.edu.tsinghua.tsfile.common.utils.ReadWriteStreamUtils;
-import cn.edu.tsinghua.tsfile.encoding.common.EndianType;
-import cn.edu.tsinghua.tsfile.file.metadata.enums.TSEncoding;
-
-import java.io.ByteArrayInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.math.BigDecimal;
-
-/**
- * Abstract class for all rle decoder. Decoding values according to
- * following grammar: {@code }. For more
- * information about rle format, see RleEncoder
- */
-public abstract class RleDecoder extends Decoder {
- // private static final Logger LOGGER = LoggerFactory.getLogger(RleDecoder.class);
- public EndianType endianType;
- protected TSFileConfig config = TSFileDescriptor.getInstance().getConfig();
- /**
- * mode to indicate current encoding type
- * 0 - RLE
- * 1 - BIT_PACKED
- */
- protected MODE mode;
- /**
- * bit width for bit-packing and rle to decode
- */
- protected int bitWidth;
- /**
- * number of data left for reading in current buffer
- */
- protected int currentCount;
- /**
- * how many bytes for all encoded data like [{@code }] in
- * inputstream
- */
- protected int length;
- /**
- * a flag to indicate whether current pattern is end. false - need to start
- * reading a new page true - current page isn't over
- */
- protected boolean isLengthAndBitWidthReaded;
- /**
- * buffer to save data format like [{@code }] for decoder
- */
- protected ByteArrayInputStream byteCache;
- /**
- * number of bit-packing group in which is saved in header
- */
- protected int bitPackingNum;
-
- public RleDecoder(EndianType endianType) {
- super(TSEncoding.RLE);
- this.endianType = endianType;
- currentCount = 0;
- isLengthAndBitWidthReaded = false;
- bitPackingNum = 0;
- byteCache = new ByteArrayInputStream(new byte[0]);
- // LOGGER.debug("tsfile-encoding RleDecoder: init rle decoder");
- }
-
- /**
- * get header for both rle and bit-packing current encode mode which is
- * saved in first bit of header
- * @return int value
- * @throws IOException cannot get header
- */
- public int getHeader() throws IOException {
- int header = ReadWriteStreamUtils.readUnsignedVarInt(byteCache);
- mode = (header & 1) == 0 ? MODE.RLE : MODE.BIT_PACKED;
- return header;
- }
-
- /**
- * get all encoded data according to mode
- *
- * @throws IOException cannot read next value
- */
- protected void readNext() throws IOException {
- int header = getHeader();
- switch (mode) {
- case RLE:
- currentCount = header >> 1;
- readNumberInRLE();
- break;
- case BIT_PACKED:
- int bitPackedGroupCount = header >> 1;
- // in last bit-packing group, there may be some useless value,
- // lastBitPackedNum indicates how many values is useful
- int lastBitPackedNum = byteCache.read();
- if (bitPackedGroupCount > 0) {
-
- currentCount = (bitPackedGroupCount - 1) * config.RLE_MIN_REPEATED_NUM + lastBitPackedNum;
- bitPackingNum = currentCount;
- } else {
- throw new TSFileDecodingException(String.format(
- "tsfile-encoding IntRleDecoder: bitPackedGroupCount %d, smaller than 1", bitPackedGroupCount));
- }
- readBitPackingBuffer(bitPackedGroupCount, lastBitPackedNum);
- break;
- default:
- throw new TSFileDecodingException(
- String.format("tsfile-encoding IntRleDecoder: unknown encoding mode %s", mode));
- }
- }
-
- /**
- * read length and bit width of current package before we decode number
- *
- * @param in InputStream
- * @throws IOException cannot read length and bit-width
- */
- protected void readLengthAndBitWidth(InputStream in) throws IOException {
- // long st = System.currentTimeMillis();
- length = ReadWriteStreamUtils.readUnsignedVarInt(in);
- byte[] tmp = new byte[length];
- in.read(tmp, 0, length);
- byteCache = new ByteArrayInputStream(tmp);
- isLengthAndBitWidthReaded = true;
- bitWidth = byteCache.read();
- initPacker();
- // long et = System.currentTimeMillis();
- }
-
- /**
- * Check whether there is number left for reading
- *
- * @param in decoded data saved in InputStream
- * @return true or false to indicate whether there is number left
- * @throws IOException cannot check next value
- */
- @Override
- public boolean hasNext(InputStream in) throws IOException {
- if (currentCount > 0 || in.available() > 0 || hasNextPackage()) {
- return true;
- }
- return false;
- }
-
- /**
- * Check whether there is another pattern left for reading
- *
- * @return true or false to indicate whether there is another pattern left
- */
- protected boolean hasNextPackage() {
- return currentCount > 0 || byteCache.available() > 0;
- }
-
- protected abstract void initPacker();
-
- /**
- * Read rle package and save them in buffer
- *
- * @throws IOException cannot read number
- */
- protected abstract void readNumberInRLE() throws IOException;
-
- /**
- * Read bit-packing package and save them in buffer
- *
- * @param bitPackedGroupCount number of group number
- * @param lastBitPackedNum number of useful value in last group
- * @throws IOException cannot read bit pack
- */
- protected abstract void readBitPackingBuffer(int bitPackedGroupCount, int lastBitPackedNum) throws IOException;
-
- @Override
- public boolean readBoolean(InputStream in) {
- throw new TSFileDecodingException("Method readBoolean is not supproted by RleDecoder");
- }
-
- @Override
- public short readShort(InputStream in) {
- throw new TSFileDecodingException("Method readShort is not supproted by RleDecoder");
- }
-
- @Override
- public int readInt(InputStream in) {
- throw new TSFileDecodingException("Method readInt is not supproted by RleDecoder");
- }
-
- @Override
- public long readLong(InputStream in) {
- throw new TSFileDecodingException("Method readLong is not supproted by RleDecoder");
- }
-
- @Override
- public float readFloat(InputStream in) {
- throw new TSFileDecodingException("Method readFloat is not supproted by RleDecoder");
- }
-
- @Override
- public double readDouble(InputStream in) {
- throw new TSFileDecodingException("Method readDouble is not supproted by RleDecoder");
- }
-
- @Override
- public Binary readBinary(InputStream in) {
- throw new TSFileDecodingException("Method readBinary is not supproted by RleDecoder");
- }
-
- @Override
- public BigDecimal readBigDecimal(InputStream in) {
- throw new TSFileDecodingException("Method readBigDecimal is not supproted by RleDecoder");
- }
-
- protected static enum MODE {
- RLE, BIT_PACKED
- }
-}
+package cn.edu.tsinghua.tsfile.encoding.decoder;
+
+import cn.edu.tsinghua.tsfile.common.conf.TSFileConfig;
+import cn.edu.tsinghua.tsfile.common.conf.TSFileDescriptor;
+import cn.edu.tsinghua.tsfile.common.exception.TSFileDecodingException;
+import cn.edu.tsinghua.tsfile.common.utils.Binary;
+import cn.edu.tsinghua.tsfile.common.utils.ReadWriteStreamUtils;
+import cn.edu.tsinghua.tsfile.encoding.common.EndianType;
+import cn.edu.tsinghua.tsfile.file.metadata.enums.TSEncoding;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.math.BigDecimal;
+
+/**
+ * Abstract class for all rle decoder. Decoding values according to following grammar:
+ * {@code }. For more information about rle format, see RleEncoder
+ */
+public abstract class RleDecoder extends Decoder {
+ // private static final Logger LOGGER = LoggerFactory.getLogger(RleDecoder.class);
+ public EndianType endianType;
+ protected TSFileConfig config = TSFileDescriptor.getInstance().getConfig();
+ /**
+ * mode to indicate current encoding type 0 - RLE 1 - BIT_PACKED
+ */
+ protected MODE mode;
+ /**
+ * bit width for bit-packing and rle to decode
+ */
+ protected int bitWidth;
+ /**
+ * number of data left for reading in current buffer
+ */
+ protected int currentCount;
+ /**
+ * how many bytes for all encoded data like [{@code }] in inputstream
+ */
+ protected int length;
+ /**
+ * a flag to indicate whether current pattern is end. false - need to start reading a new page
+ * true - current page isn't over
+ */
+ protected boolean isLengthAndBitWidthReaded;
+ /**
+ * buffer to save data format like [{@code }] for decoder
+ */
+ protected ByteArrayInputStream byteCache;
+ /**
+ * number of bit-packing group in which is saved in header
+ */
+ protected int bitPackingNum;
+
+ public RleDecoder(EndianType endianType) {
+ super(TSEncoding.RLE);
+ this.endianType = endianType;
+ currentCount = 0;
+ isLengthAndBitWidthReaded = false;
+ bitPackingNum = 0;
+ byteCache = new ByteArrayInputStream(new byte[0]);
+ // LOGGER.debug("tsfile-encoding RleDecoder: init rle decoder");
+ }
+
+ /**
+ * get header for both rle and bit-packing current encode mode which is saved in first bit of
+ * header
+ *
+ * @return int value
+ * @throws IOException cannot get header
+ */
+ public int getHeader() throws IOException {
+ int header = ReadWriteStreamUtils.readUnsignedVarInt(byteCache);
+ mode = (header & 1) == 0 ? MODE.RLE : MODE.BIT_PACKED;
+ return header;
+ }
+
+ /**
+ * get all encoded data according to mode
+ *
+ * @throws IOException cannot read next value
+ */
+ protected void readNext() throws IOException {
+ int header = getHeader();
+ switch (mode) {
+ case RLE:
+ currentCount = header >> 1;
+ readNumberInRLE();
+ break;
+ case BIT_PACKED:
+ int bitPackedGroupCount = header >> 1;
+ // in last bit-packing group, there may be some useless value,
+ // lastBitPackedNum indicates how many values is useful
+ int lastBitPackedNum = byteCache.read();
+ if (bitPackedGroupCount > 0) {
+
+ currentCount = (bitPackedGroupCount - 1) * config.RLE_MIN_REPEATED_NUM + lastBitPackedNum;
+ bitPackingNum = currentCount;
+ } else {
+ throw new TSFileDecodingException(
+ String.format("tsfile-encoding IntRleDecoder: bitPackedGroupCount %d, smaller than 1",
+ bitPackedGroupCount));
+ }
+ readBitPackingBuffer(bitPackedGroupCount, lastBitPackedNum);
+ break;
+ default:
+ throw new TSFileDecodingException(
+ String.format("tsfile-encoding IntRleDecoder: unknown encoding mode %s", mode));
+ }
+ }
+
+ /**
+ * read length and bit width of current package before we decode number
+ *
+ * @param in InputStream
+ * @throws IOException cannot read length and bit-width
+ */
+ protected void readLengthAndBitWidth(InputStream in) throws IOException {
+ // long st = System.currentTimeMillis();
+ length = ReadWriteStreamUtils.readUnsignedVarInt(in);
+ byte[] tmp = new byte[length];
+ in.read(tmp, 0, length);
+ byteCache = new ByteArrayInputStream(tmp);
+ isLengthAndBitWidthReaded = true;
+ bitWidth = byteCache.read();
+ initPacker();
+ // long et = System.currentTimeMillis();
+ }
+
+ /**
+ * Check whether there is number left for reading
+ *
+ * @param in decoded data saved in InputStream
+ * @return true or false to indicate whether there is number left
+ * @throws IOException cannot check next value
+ */
+ @Override
+ public boolean hasNext(InputStream in) throws IOException {
+ if (currentCount > 0 || in.available() > 0 || hasNextPackage()) {
+ return true;
+ }
+ return false;
+ }
+
+ /**
+ * Check whether there is another pattern left for reading
+ *
+ * @return true or false to indicate whether there is another pattern left
+ */
+ protected boolean hasNextPackage() {
+ return currentCount > 0 || byteCache.available() > 0;
+ }
+
+ protected abstract void initPacker();
+
+ /**
+ * Read rle package and save them in buffer
+ *
+ * @throws IOException cannot read number
+ */
+ protected abstract void readNumberInRLE() throws IOException;
+
+ /**
+ * Read bit-packing package and save them in buffer
+ *
+ * @param bitPackedGroupCount number of group number
+ * @param lastBitPackedNum number of useful value in last group
+ * @throws IOException cannot read bit pack
+ */
+ protected abstract void readBitPackingBuffer(int bitPackedGroupCount, int lastBitPackedNum)
+ throws IOException;
+
+ @Override
+ public boolean readBoolean(InputStream in) {
+ throw new TSFileDecodingException("Method readBoolean is not supproted by RleDecoder");
+ }
+
+ @Override
+ public short readShort(InputStream in) {
+ throw new TSFileDecodingException("Method readShort is not supproted by RleDecoder");
+ }
+
+ @Override
+ public int readInt(InputStream in) {
+ throw new TSFileDecodingException("Method readInt is not supproted by RleDecoder");
+ }
+
+ @Override
+ public long readLong(InputStream in) {
+ throw new TSFileDecodingException("Method readLong is not supproted by RleDecoder");
+ }
+
+ @Override
+ public float readFloat(InputStream in) {
+ throw new TSFileDecodingException("Method readFloat is not supproted by RleDecoder");
+ }
+
+ @Override
+ public double readDouble(InputStream in) {
+ throw new TSFileDecodingException("Method readDouble is not supproted by RleDecoder");
+ }
+
+ @Override
+ public Binary readBinary(InputStream in) {
+ throw new TSFileDecodingException("Method readBinary is not supproted by RleDecoder");
+ }
+
+ @Override
+ public BigDecimal readBigDecimal(InputStream in) {
+ throw new TSFileDecodingException("Method readBigDecimal is not supproted by RleDecoder");
+ }
+
+ protected static enum MODE {
+ RLE, BIT_PACKED
+ }
+}
diff --git a/src/main/java/cn/edu/tsinghua/tsfile/encoding/decoder/SinglePrecisionDecoder.java b/src/main/java/cn/edu/tsinghua/tsfile/encoding/decoder/SinglePrecisionDecoder.java
index c5e1280c..ba31cdf2 100644
--- a/src/main/java/cn/edu/tsinghua/tsfile/encoding/decoder/SinglePrecisionDecoder.java
+++ b/src/main/java/cn/edu/tsinghua/tsfile/encoding/decoder/SinglePrecisionDecoder.java
@@ -2,89 +2,88 @@
import java.io.IOException;
import java.io.InputStream;
-
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-
import cn.edu.tsinghua.tsfile.common.conf.TSFileConfig;
/**
* Decoder for value value using gorilla
*/
public class SinglePrecisionDecoder extends GorillaDecoder {
- private static final Logger LOGGER = LoggerFactory.getLogger(SinglePrecisionDecoder.class);
- private int preValue;
+ private static final Logger LOGGER = LoggerFactory.getLogger(SinglePrecisionDecoder.class);
+ private int preValue;
- public SinglePrecisionDecoder() {
- }
+ public SinglePrecisionDecoder() {}
- @Override
- public float readFloat(InputStream in) {
- if (!flag) {
- flag = true;
- try {
- int ch1 = in.read();
- int ch2 = in.read();
- int ch3 = in.read();
- int ch4 = in.read();
- preValue = ch1 + (ch2 << 8) + (ch3 << 16) + (ch4 << 24);
- leadingZeroNum = Integer.numberOfLeadingZeros(preValue);
- tailingZeroNum = Integer.numberOfTrailingZeros(preValue);
- float tmp = Float.intBitsToFloat(preValue);
- fillBuffer(in);
- getNextValue(in);
- return tmp;
- } catch (IOException e) {
- LOGGER.error("SinglePrecisionDecoder cannot read first float number because: {}", e.getMessage());
- }
- } else {
- try {
- float tmp = Float.intBitsToFloat(preValue);
- getNextValue(in);
- return tmp;
- } catch (IOException e) {
- LOGGER.error("SinglePrecisionDecoder cannot read following float number because: {}", e.getMessage());
- }
- }
- return Float.NaN;
- }
+ @Override
+ public float readFloat(InputStream in) {
+ if (!flag) {
+ flag = true;
+ try {
+ int ch1 = in.read();
+ int ch2 = in.read();
+ int ch3 = in.read();
+ int ch4 = in.read();
+ preValue = ch1 + (ch2 << 8) + (ch3 << 16) + (ch4 << 24);
+ leadingZeroNum = Integer.numberOfLeadingZeros(preValue);
+ tailingZeroNum = Integer.numberOfTrailingZeros(preValue);
+ float tmp = Float.intBitsToFloat(preValue);
+ fillBuffer(in);
+ getNextValue(in);
+ return tmp;
+ } catch (IOException e) {
+ LOGGER.error("SinglePrecisionDecoder cannot read first float number because: {}",
+ e.getMessage());
+ }
+ } else {
+ try {
+ float tmp = Float.intBitsToFloat(preValue);
+ getNextValue(in);
+ return tmp;
+ } catch (IOException e) {
+ LOGGER.error("SinglePrecisionDecoder cannot read following float number because: {}",
+ e.getMessage());
+ }
+ }
+ return Float.NaN;
+ }
- /**
- * check whether there is any value to encode left
- *
- * @param in stream to read
- * @throws IOException cannot read from stream
- */
- private void getNextValue(InputStream in) throws IOException {
- nextFlag1 = readBit(in);
- // case: '0'
- if (!nextFlag1) {
- return;
- }
- nextFlag2 = readBit(in);
+ /**
+ * check whether there is any value to encode left
+ *
+ * @param in stream to read
+ * @throws IOException cannot read from stream
+ */
+ private void getNextValue(InputStream in) throws IOException {
+ nextFlag1 = readBit(in);
+ // case: '0'
+ if (!nextFlag1) {
+ return;
+ }
+ nextFlag2 = readBit(in);
- if (!nextFlag2) {
- // case: '10'
- int tmp = 0;
- for (int i = 0; i < TSFileConfig.FLOAT_LENGTH - leadingZeroNum - tailingZeroNum; i++) {
- int bit = readBit(in) ? 1 : 0;
- tmp |= bit << (TSFileConfig.FLOAT_LENGTH - 1 - leadingZeroNum - i);
- }
- tmp ^= preValue;
- preValue = tmp;
- } else {
- // case: '11'
- int leadingZeroNumTmp = readIntFromStream(in, TSFileConfig.FLAOT_LEADING_ZERO_LENGTH);
- int lenTmp = readIntFromStream(in, TSFileConfig.FLOAT_VALUE_LENGTH);
- int tmp = readIntFromStream(in, lenTmp);
- tmp <<= (TSFileConfig.FLOAT_LENGTH - leadingZeroNumTmp - lenTmp);
- tmp ^= preValue;
- preValue = tmp;
- }
- leadingZeroNum = Integer.numberOfLeadingZeros(preValue);
- tailingZeroNum = Integer.numberOfTrailingZeros(preValue);
- if(Float.isNaN(Float.intBitsToFloat(preValue))){
- isEnd = true;
- }
- }
+ if (!nextFlag2) {
+ // case: '10'
+ int tmp = 0;
+ for (int i = 0; i < TSFileConfig.FLOAT_LENGTH - leadingZeroNum - tailingZeroNum; i++) {
+ int bit = readBit(in) ? 1 : 0;
+ tmp |= bit << (TSFileConfig.FLOAT_LENGTH - 1 - leadingZeroNum - i);
+ }
+ tmp ^= preValue;
+ preValue = tmp;
+ } else {
+ // case: '11'
+ int leadingZeroNumTmp = readIntFromStream(in, TSFileConfig.FLAOT_LEADING_ZERO_LENGTH);
+ int lenTmp = readIntFromStream(in, TSFileConfig.FLOAT_VALUE_LENGTH);
+ int tmp = readIntFromStream(in, lenTmp);
+ tmp <<= (TSFileConfig.FLOAT_LENGTH - leadingZeroNumTmp - lenTmp);
+ tmp ^= preValue;
+ preValue = tmp;
+ }
+ leadingZeroNum = Integer.numberOfLeadingZeros(preValue);
+ tailingZeroNum = Integer.numberOfTrailingZeros(preValue);
+ if (Float.isNaN(Float.intBitsToFloat(preValue))) {
+ isEnd = true;
+ }
+ }
}
diff --git a/src/main/java/cn/edu/tsinghua/tsfile/encoding/encoder/BitmapEncoder.java b/src/main/java/cn/edu/tsinghua/tsfile/encoding/encoder/BitmapEncoder.java
index 72c39c4c..1228f669 100644
--- a/src/main/java/cn/edu/tsinghua/tsfile/encoding/encoder/BitmapEncoder.java
+++ b/src/main/java/cn/edu/tsinghua/tsfile/encoding/encoder/BitmapEncoder.java
@@ -5,7 +5,6 @@
import cn.edu.tsinghua.tsfile.file.metadata.enums.TSEncoding;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.ArrayList;
@@ -15,6 +14,7 @@
/**
* Encodes values using bitmap, according to the following grammar:
+ *
*
* {@code
* bitmap-encoding:
@@ -28,87 +28,88 @@
*
*/
public class BitmapEncoder extends Encoder {
- private static final Logger LOGGER = LoggerFactory.getLogger(BitmapEncoder.class);
+ private static final Logger LOGGER = LoggerFactory.getLogger(BitmapEncoder.class);
- /**
- * Bitmap Encoder stores all current values in a list temporally
- */
- private List values;
+ /**
+ * Bitmap Encoder stores all current values in a list temporally
+ */
+ private List values;
- /**
- * @param endianType deprecated
- */
- public BitmapEncoder(EndianType endianType) {
- super(TSEncoding.BITMAP);
- this.values = new ArrayList();
- LOGGER.debug("tsfile-encoding BitmapEncoder: init bitmap encoder");
- }
+ /**
+ * @param endianType deprecated
+ */
+ public BitmapEncoder(EndianType endianType) {
+ super(TSEncoding.BITMAP);
+ this.values = new ArrayList();
+ LOGGER.debug("tsfile-encoding BitmapEncoder: init bitmap encoder");
+ }
- /**
- * Each time encoder receives a value, encoder doesn't write it to OutputStream immediately.
- * Encoder stores current value in a list. When all value is received, flush() method will be
- * invoked. Encoder encodes all values and writes them to OutputStream
- *
- * @param value value to encode
- * @param out OutputStream to write encoded stream
- * @throws IOException cannot encode value
- * @see Encoder#encode(int, java.io.ByteArrayOutputStream)
- */
- @Override
- public void encode(int value, ByteArrayOutputStream out) throws IOException {
- values.add(value);
- }
+ /**
+ * Each time encoder receives a value, encoder doesn't write it to OutputStream immediately.
+ * Encoder stores current value in a list. When all value is received, flush() method will be
+ * invoked. Encoder encodes all values and writes them to OutputStream
+ *
+ * @param value value to encode
+ * @param out OutputStream to write encoded stream
+ * @throws IOException cannot encode value
+ * @see Encoder#encode(int, java.io.ByteArrayOutputStream)
+ */
+ @Override
+ public void encode(int value, ByteArrayOutputStream out) throws IOException {
+ values.add(value);
+ }
- /**
- * When all data received, encoder now encodes values in list and write them to OutputStream
- *
- * @param out OutputStream to write encoded stream
- * @throws IOException cannot flush to OutputStream
- * @see Encoder#flush(java.io.ByteArrayOutputStream)
- */
- @Override
- public void flush(ByteArrayOutputStream out) throws IOException {
- // byteCache stores all and we know its size
- ByteArrayOutputStream byteCache = new ByteArrayOutputStream();
- Set valueType = new HashSet(values);
- int byteNum = (values.size() + 7) / 8;
- if (byteNum == 0) {
- reset();
- return;
- }
- int len = values.size();
-// LOGGER.debug("tsfile-encoding BitmapEncoder: number of data in list is {}", len);
- for (int value : valueType) {
- byte[] buffer = new byte[byteNum];
- for (int i = 0; i < len; i++) {
- if (values.get(i) == value) {
- int index = i / 8;
- int offset = 7 - (i % 8);
- // Encoder use 1 bit in byte to indicate that value appears
- buffer[index] |= ((byte) 1 << offset);
- }
- }
- ReadWriteStreamUtils.writeUnsignedVarInt(value, byteCache);
- byteCache.write(buffer);
-// LOGGER.debug("tsfile-encoding BitmapEncoder: encode value {}, bitmap index {}", value, buffer);
+ /**
+ * When all data received, encoder now encodes values in list and write them to OutputStream
+ *
+ * @param out OutputStream to write encoded stream
+ * @throws IOException cannot flush to OutputStream
+ * @see Encoder#flush(java.io.ByteArrayOutputStream)
+ */
+ @Override
+ public void flush(ByteArrayOutputStream out) throws IOException {
+ // byteCache stores all and we know its size
+ ByteArrayOutputStream byteCache = new ByteArrayOutputStream();
+ Set valueType = new HashSet(values);
+ int byteNum = (values.size() + 7) / 8;
+ if (byteNum == 0) {
+ reset();
+ return;
+ }
+ int len = values.size();
+ // LOGGER.debug("tsfile-encoding BitmapEncoder: number of data in list is {}", len);
+ for (int value : valueType) {
+ byte[] buffer = new byte[byteNum];
+ for (int i = 0; i < len; i++) {
+ if (values.get(i) == value) {
+ int index = i / 8;
+ int offset = 7 - (i % 8);
+ // Encoder use 1 bit in byte to indicate that value appears
+ buffer[index] |= ((byte) 1 << offset);
}
- ReadWriteStreamUtils.writeUnsignedVarInt(byteCache.size(), out);
- ReadWriteStreamUtils.writeUnsignedVarInt(len, out);
- out.write(byteCache.toByteArray());
- reset();
+ }
+ ReadWriteStreamUtils.writeUnsignedVarInt(value, byteCache);
+ byteCache.write(buffer);
+ // LOGGER.debug("tsfile-encoding BitmapEncoder: encode value {}, bitmap index {}", value,
+ // buffer);
}
+ ReadWriteStreamUtils.writeUnsignedVarInt(byteCache.size(), out);
+ ReadWriteStreamUtils.writeUnsignedVarInt(len, out);
+ out.write(byteCache.toByteArray());
+ reset();
+ }
- private void reset() {
- values.clear();
- }
+ private void reset() {
+ values.clear();
+ }
- public int getOneItemMaxSize() {
- return 1;
- }
+ public int getOneItemMaxSize() {
+ return 1;
+ }
- public long getMaxByteSize() {
- //byteCacheSize + byteDictSize + (byte array + array length) * byteDictSize
- return 4 + 4 + ((values.size() + 7) / 8 + 4) * values.size();
- }
+ public long getMaxByteSize() {
+ // byteCacheSize + byteDictSize + (byte array + array length) * byteDictSize
+ return 4 + 4 + ((values.size() + 7) / 8 + 4) * values.size();
+ }
}
diff --git a/src/main/java/cn/edu/tsinghua/tsfile/encoding/encoder/DeltaBinaryEncoder.java b/src/main/java/cn/edu/tsinghua/tsfile/encoding/encoder/DeltaBinaryEncoder.java
index 0d7b5b6a..9fdbd378 100644
--- a/src/main/java/cn/edu/tsinghua/tsfile/encoding/encoder/DeltaBinaryEncoder.java
+++ b/src/main/java/cn/edu/tsinghua/tsfile/encoding/encoder/DeltaBinaryEncoder.java
@@ -4,7 +4,6 @@
import cn.edu.tsinghua.tsfile.file.metadata.enums.TSEncoding;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-
import java.io.ByteArrayOutputStream;
import java.io.IOException;
@@ -28,285 +27,285 @@
* @author kangrong
*/
abstract public class DeltaBinaryEncoder extends Encoder {
- protected static final int BLOCK_DEFAULT_SIZE = 128;
- private static final Logger LOG = LoggerFactory.getLogger(DeltaBinaryEncoder.class);
- protected ByteArrayOutputStream out;
- protected int blockSize;
- // input value is stored in deltaBlackBuffer temporarily
- protected byte[] encodingBlockBuffer;
+ protected static final int BLOCK_DEFAULT_SIZE = 128;
+ private static final Logger LOG = LoggerFactory.getLogger(DeltaBinaryEncoder.class);
+ protected ByteArrayOutputStream out;
+ protected int blockSize;
+ // input value is stored in deltaBlackBuffer temporarily
+ protected byte[] encodingBlockBuffer;
- protected int writeIndex = -1;
- protected int writeWidth = 0;
+ protected int writeIndex = -1;
+ protected int writeWidth = 0;
- /**
- * @param size - the number how many numbers to be packed into a block.
- */
- public DeltaBinaryEncoder(int size) {
- super(TSEncoding.TS_2DIFF);
- blockSize = size;
+ /**
+ * @param size - the number how many numbers to be packed into a block.
+ */
+ public DeltaBinaryEncoder(int size) {
+ super(TSEncoding.TS_2DIFF);
+ blockSize = size;
+ }
+
+ protected abstract void writeHeader() throws IOException;
+
+ protected abstract void writeValueToBytes(int i);
+
+ protected abstract void calcTwoDiff(int i);
+
+ protected abstract void reset();
+
+ protected abstract int calculateBitWidthsForDeltaBlockBuffer();
+
+
+ /**
+ * write all data into {@code encodingBlockBuffer}.
+ */
+ private void writeDataWithMinWidth() {
+ for (int i = 0; i < writeIndex; i++) {
+ writeValueToBytes(i);
+ }
+ int encodingLength = (int) Math.ceil((double) (writeIndex * writeWidth) / 8.0);
+ out.write(encodingBlockBuffer, 0, encodingLength);
+ }
+
+ private void writeHeaderToBytes() throws IOException {
+ out.write(BytesUtils.intToBytes(writeIndex));
+ out.write(BytesUtils.intToBytes(writeWidth));
+ writeHeader();
+ }
+
+
+ private void flushBlockBuffer(ByteArrayOutputStream out) throws IOException {
+ if (writeIndex == -1)
+ return;
+ // since we store the min delta, the deltas will be converted to be the
+ // difference to min delta and all positive
+ this.out = out;
+ for (int i = 0; i < writeIndex; i++) {
+ calcTwoDiff(i);
}
+ writeWidth = calculateBitWidthsForDeltaBlockBuffer();
+ // System.out.println("write width:"+writeWidth);
+ writeHeaderToBytes();
+ writeDataWithMinWidth();
+
+ reset();
+ writeIndex = -1;
+ }
+
+
+ /**
+ * calling this method to flush all values which haven't encoded to result byte array
+ */
+ @Override
+ public void flush(ByteArrayOutputStream out) {
+ try {
+ flushBlockBuffer(out);
+ } catch (IOException e) {
+ LOG.error("flush data to stream failed!");
+ }
+ }
- protected abstract void writeHeader() throws IOException;
+ static public class IntDeltaEncoder extends DeltaBinaryEncoder {
+ private int[] deltaBlockBuffer;
+ private int firstValue;
+ private int previousValue;
+ private int minDeltaBase;
- protected abstract void writeValueToBytes(int i);
+ public IntDeltaEncoder() {
+ this(BLOCK_DEFAULT_SIZE);
+ }
- protected abstract void calcTwoDiff(int i);
+ public IntDeltaEncoder(int size) {
+ super(size);
+ deltaBlockBuffer = new int[this.blockSize];
+ encodingBlockBuffer = new byte[blockSize * 4];
+ reset();
+ }
- protected abstract void reset();
+ protected int calculateBitWidthsForDeltaBlockBuffer() {
+ int width = 0;
+ for (int i = 0; i < writeIndex; i++) {
+ width = Math.max(width, getValueWidth(deltaBlockBuffer[i]));
+ }
+ return width;
+ }
- protected abstract int calculateBitWidthsForDeltaBlockBuffer();
+ private void calcDelta(Integer value) {
+ Integer delta = value - previousValue;// calculate delta
+ if (delta < minDeltaBase) {
+ minDeltaBase = delta;
+ }
+ deltaBlockBuffer[writeIndex++] = delta;
+ }
/**
- * write all data into {@code encodingBlockBuffer}.
+ * input a integer
+ *
+ * @param value value to encode
+ * @param out the ByteArrayOutputStream which data encode into
*/
- private void writeDataWithMinWidth() {
- for (int i = 0; i < writeIndex; i++) {
- writeValueToBytes(i);
- }
- int encodingLength = (int) Math.ceil((double) (writeIndex * writeWidth) / 8.0);
- out.write(encodingBlockBuffer, 0, encodingLength);
+ public void encodeValue(int value, ByteArrayOutputStream out) {
+ if (writeIndex == -1) {
+ writeIndex++;
+ firstValue = value;
+ previousValue = firstValue;
+ return;
+ }
+ calcDelta(value);
+ previousValue = value;
+ if (writeIndex == blockSize) {
+ flush(out);
+ }
}
- private void writeHeaderToBytes() throws IOException {
- out.write(BytesUtils.intToBytes(writeIndex));
- out.write(BytesUtils.intToBytes(writeWidth));
- writeHeader();
+ @Override
+ protected void reset() {
+ firstValue = 0;
+ previousValue = 0;
+ minDeltaBase = Integer.MAX_VALUE;
+ for (int i = 0; i < blockSize; i++) {
+ encodingBlockBuffer[i] = 0;
+ deltaBlockBuffer[i] = 0;
+ }
}
- private void flushBlockBuffer(ByteArrayOutputStream out) throws IOException {
- if (writeIndex == -1)
- return;
- // since we store the min delta, the deltas will be converted to be the
- // difference to min delta and all positive
- this.out = out;
- for (int i = 0; i < writeIndex; i++) {
- calcTwoDiff(i);
- }
- writeWidth = calculateBitWidthsForDeltaBlockBuffer();
- // System.out.println("write width:"+writeWidth);
- writeHeaderToBytes();
- writeDataWithMinWidth();
-
- reset();
- writeIndex = -1;
+ private int getValueWidth(int v) {
+ return 32 - Integer.numberOfLeadingZeros(v);
}
+ @Override
+ protected void writeValueToBytes(int i) {
+ BytesUtils.intToBytes(deltaBlockBuffer[i], encodingBlockBuffer, writeWidth * i, writeWidth);
+ }
+
+ @Override
+ protected void calcTwoDiff(int i) {
+ deltaBlockBuffer[i] = deltaBlockBuffer[i] - minDeltaBase;
+ }
+
+ @Override
+ protected void writeHeader() throws IOException {
+ out.write(BytesUtils.intToBytes(minDeltaBase));
+ out.write(BytesUtils.intToBytes(firstValue));
+ }
+
+ @Override
+ public void encode(int value, ByteArrayOutputStream out) {
+ encodeValue(value, out);
+ }
+
+ @Override
+ public int getOneItemMaxSize() {
+ return 4;
+ }
- /**
- * calling this method to flush all values which haven't encoded to result byte array
- */
@Override
- public void flush(ByteArrayOutputStream out) {
- try {
- flushBlockBuffer(out);
- } catch (IOException e) {
- LOG.error("flush data to stream failed!");
- }
+ public long getMaxByteSize() {
+ // The meaning of 24 is: index(4)+width(4)+minDeltaBase(4)+firstValue(4)
+ return 24 + writeIndex * 4;
}
+ }
- static public class IntDeltaEncoder extends DeltaBinaryEncoder {
- private int[] deltaBlockBuffer;
- private int firstValue;
- private int previousValue;
- private int minDeltaBase;
-
- public IntDeltaEncoder() {
- this(BLOCK_DEFAULT_SIZE);
- }
-
- public IntDeltaEncoder(int size) {
- super(size);
- deltaBlockBuffer = new int[this.blockSize];
- encodingBlockBuffer = new byte[blockSize * 4];
- reset();
- }
-
- protected int calculateBitWidthsForDeltaBlockBuffer() {
- int width = 0;
- for (int i = 0; i < writeIndex; i++) {
- width = Math.max(width, getValueWidth(deltaBlockBuffer[i]));
- }
- return width;
- }
-
- private void calcDelta(Integer value) {
- Integer delta = value - previousValue;// calculate delta
- if (delta < minDeltaBase) {
- minDeltaBase = delta;
- }
- deltaBlockBuffer[writeIndex++] = delta;
-
- }
-
- /**
- * input a integer
- *
- * @param value value to encode
- * @param out the ByteArrayOutputStream which data encode into
- */
- public void encodeValue(int value, ByteArrayOutputStream out) {
- if (writeIndex == -1) {
- writeIndex++;
- firstValue = value;
- previousValue = firstValue;
- return;
- }
- calcDelta(value);
- previousValue = value;
- if (writeIndex == blockSize) {
- flush(out);
- }
- }
-
- @Override
- protected void reset() {
- firstValue = 0;
- previousValue = 0;
- minDeltaBase = Integer.MAX_VALUE;
- for (int i = 0; i < blockSize; i++) {
- encodingBlockBuffer[i] = 0;
- deltaBlockBuffer[i] = 0;
- }
- }
-
-
- private int getValueWidth(int v) {
- return 32 - Integer.numberOfLeadingZeros(v);
- }
-
- @Override
- protected void writeValueToBytes(int i) {
- BytesUtils.intToBytes(deltaBlockBuffer[i], encodingBlockBuffer, writeWidth * i, writeWidth);
- }
-
- @Override
- protected void calcTwoDiff(int i) {
- deltaBlockBuffer[i] = deltaBlockBuffer[i] - minDeltaBase;
- }
-
- @Override
- protected void writeHeader() throws IOException {
- out.write(BytesUtils.intToBytes(minDeltaBase));
- out.write(BytesUtils.intToBytes(firstValue));
- }
-
- @Override
- public void encode(int value, ByteArrayOutputStream out) {
- encodeValue(value, out);
- }
-
- @Override
- public int getOneItemMaxSize() {
- return 4;
- }
-
- @Override
- public long getMaxByteSize() {
- // The meaning of 24 is: index(4)+width(4)+minDeltaBase(4)+firstValue(4)
- return 24 + writeIndex * 4;
- }
+ static public class LongDeltaEncoder extends DeltaBinaryEncoder {
+ private long[] deltaBlockBuffer;
+ private long firstValue;
+ private long previousValue;
+ private long minDeltaBase;
+
+ public LongDeltaEncoder() {
+ this(BLOCK_DEFAULT_SIZE);
+ }
+
+ public LongDeltaEncoder(int size) {
+ super(size);
+ deltaBlockBuffer = new long[this.blockSize];
+ encodingBlockBuffer = new byte[blockSize * 8];
+ reset();
+ }
+
+ private void calcDelta(Long value) {
+ Long delta = value - previousValue;// calculate delta
+ if (delta < minDeltaBase) {
+ minDeltaBase = delta;
+ }
+ deltaBlockBuffer[writeIndex++] = delta;
+ }
+
+ @Override
+ protected void reset() {
+ firstValue = 0l;
+ previousValue = 0l;
+ minDeltaBase = Long.MAX_VALUE;
+ for (int i = 0; i < blockSize; i++) {
+ encodingBlockBuffer[i] = 0;
+ deltaBlockBuffer[i] = 0l;
+ }
+ }
+
+ private int getValueWidth(Long v) {
+ return 64 - Long.numberOfLeadingZeros(v);
+ }
+
+ @Override
+ protected void writeValueToBytes(int i) {
+ BytesUtils.longToBytes(deltaBlockBuffer[i], encodingBlockBuffer, writeWidth * i, writeWidth);
+ }
+
+ @Override
+ protected void calcTwoDiff(int i) {
+ deltaBlockBuffer[i] = deltaBlockBuffer[i] - minDeltaBase;
+ }
+
+ @Override
+ protected void writeHeader() throws IOException {
+ out.write(BytesUtils.longToBytes(minDeltaBase));
+ out.write(BytesUtils.longToBytes(firstValue));
+ }
+
+ @Override
+ public void encode(long value, ByteArrayOutputStream out) {
+ encodeValue(value, out);
+ }
+
+ @Override
+ public int getOneItemMaxSize() {
+ return 8;
+ }
+
+ @Override
+ public long getMaxByteSize() {
+ // The meaning of 24 is: index(4)+width(4)+minDeltaBase(8)+firstValue(8)
+ return 24 + writeIndex * 8;
+ }
+
+ /**
+ * input a integer or long value.
+ *
+ * @param value value to encode
+ * @param out - the ByteArrayOutputStream which data encode into
+ */
+ public void encodeValue(long value, ByteArrayOutputStream out) {
+ if (writeIndex == -1) {
+ writeIndex++;
+ firstValue = value;
+ previousValue = firstValue;
+ return;
+ }
+ calcDelta(value);
+ previousValue = value;
+ if (writeIndex == blockSize) {
+ flush(out);
+ }
}
- static public class LongDeltaEncoder extends DeltaBinaryEncoder {
- private long[] deltaBlockBuffer;
- private long firstValue;
- private long previousValue;
- private long minDeltaBase;
-
- public LongDeltaEncoder() {
- this(BLOCK_DEFAULT_SIZE);
- }
-
- public LongDeltaEncoder(int size) {
- super(size);
- deltaBlockBuffer = new long[this.blockSize];
- encodingBlockBuffer = new byte[blockSize * 8];
- reset();
- }
-
- private void calcDelta(Long value) {
- Long delta = value - previousValue;// calculate delta
- if (delta < minDeltaBase) {
- minDeltaBase = delta;
- }
- deltaBlockBuffer[writeIndex++] = delta;
- }
-
- @Override
- protected void reset() {
- firstValue = 0l;
- previousValue = 0l;
- minDeltaBase = Long.MAX_VALUE;
- for (int i = 0; i < blockSize; i++) {
- encodingBlockBuffer[i] = 0;
- deltaBlockBuffer[i] = 0l;
- }
- }
-
- private int getValueWidth(Long v) {
- return 64 - Long.numberOfLeadingZeros(v);
- }
-
- @Override
- protected void writeValueToBytes(int i) {
- BytesUtils.longToBytes(deltaBlockBuffer[i], encodingBlockBuffer, writeWidth * i, writeWidth);
- }
-
- @Override
- protected void calcTwoDiff(int i) {
- deltaBlockBuffer[i] = deltaBlockBuffer[i] - minDeltaBase;
- }
-
- @Override
- protected void writeHeader() throws IOException {
- out.write(BytesUtils.longToBytes(minDeltaBase));
- out.write(BytesUtils.longToBytes(firstValue));
- }
-
- @Override
- public void encode(long value, ByteArrayOutputStream out) {
- encodeValue(value, out);
- }
-
- @Override
- public int getOneItemMaxSize() {
- return 8;
- }
-
- @Override
- public long getMaxByteSize() {
- // The meaning of 24 is: index(4)+width(4)+minDeltaBase(8)+firstValue(8)
- return 24 + writeIndex * 8;
- }
-
- /**
- * input a integer or long value.
- *
- * @param value value to encode
- * @param out - the ByteArrayOutputStream which data encode into
- */
- public void encodeValue(long value, ByteArrayOutputStream out) {
- if (writeIndex == -1) {
- writeIndex++;
- firstValue = value;
- previousValue = firstValue;
- return;
- }
- calcDelta(value);
- previousValue = value;
- if (writeIndex == blockSize) {
- flush(out);
- }
- }
-
- protected int calculateBitWidthsForDeltaBlockBuffer() {
- int width = 0;
- for (int i = 0; i < writeIndex; i++) {
- width = Math.max(width, getValueWidth(deltaBlockBuffer[i]));
- }
- return width;
- }
+ protected int calculateBitWidthsForDeltaBlockBuffer() {
+ int width = 0;
+ for (int i = 0; i < writeIndex; i++) {
+ width = Math.max(width, getValueWidth(deltaBlockBuffer[i]));
+ }
+ return width;
}
+ }
}
diff --git a/src/main/java/cn/edu/tsinghua/tsfile/encoding/encoder/DoublePrecisionEncoder.java b/src/main/java/cn/edu/tsinghua/tsfile/encoding/encoder/DoublePrecisionEncoder.java
index c47ef66f..a86fe8f2 100644
--- a/src/main/java/cn/edu/tsinghua/tsfile/encoding/encoder/DoublePrecisionEncoder.java
+++ b/src/main/java/cn/edu/tsinghua/tsfile/encoding/encoder/DoublePrecisionEncoder.java
@@ -2,7 +2,6 @@
import java.io.ByteArrayOutputStream;
import java.io.IOException;
-
import cn.edu.tsinghua.tsfile.common.conf.TSFileConfig;
/**
@@ -10,80 +9,83 @@
*
*/
public class DoublePrecisionEncoder extends GorillaEncoder {
- private long preValue;
-
- public DoublePrecisionEncoder() {
- }
+ private long preValue;
- @Override
- public void encode(double value, ByteArrayOutputStream out) throws IOException {
- if (!flag) {
- // case: write first 8 byte value without any encoding
- flag = true;
- preValue = Double.doubleToLongBits(value);
- leadingZeroNum = Long.numberOfLeadingZeros(preValue);
- tailingZeroNum = Long.numberOfTrailingZeros(preValue);
- byte[] bufferBig = new byte[8];
- byte[] bufferLittle = new byte[8];
+ public DoublePrecisionEncoder() {}
- for (int i = 0; i < 8; i++) {
- bufferLittle[i] = (byte) (((preValue) >> (i * 8)) & 0xFF);
- bufferBig[8 - i - 1] = (byte) (((preValue) >> (i * 8)) & 0xFF);
- }
- out.write(bufferLittle);
- } else {
- long nextValue = Double.doubleToLongBits(value);
- long tmp = nextValue ^ preValue;
- if (tmp == 0) {
- // case: write '0'
- writeBit(false, out);
- } else {
- int leadingZeroNumTmp = Long.numberOfLeadingZeros(tmp);
- int tailingZeroNumTmp = Long.numberOfTrailingZeros(tmp);
- if (leadingZeroNumTmp >= leadingZeroNum && tailingZeroNumTmp >= tailingZeroNum) {
- // case: write '10' and effective bits without first leadingZeroNum '0' and last tailingZeroNum '0'
- writeBit(true, out);
- writeBit(false, out);
- writeBits(tmp, out, TSFileConfig.DOUBLE_LENGTH - 1 - leadingZeroNum, tailingZeroNum);
- } else {
- // case: write '11', leading zero num of value, effective bits len and effective bit value
- writeBit(true, out);
- writeBit(true, out);
- writeBits(leadingZeroNumTmp, out, TSFileConfig.DOUBLE_LEADING_ZERO_LENGTH - 1, 0);
- writeBits(TSFileConfig.DOUBLE_LENGTH - leadingZeroNumTmp - tailingZeroNumTmp, out, TSFileConfig.DOUBLE_VALUE_LENGTH - 1, 0);
- writeBits(tmp, out, TSFileConfig.DOUBLE_LENGTH - 1 - leadingZeroNumTmp, tailingZeroNumTmp);
- }
- }
- preValue = nextValue;
- leadingZeroNum = Long.numberOfLeadingZeros(preValue);
- tailingZeroNum = Long.numberOfTrailingZeros(preValue);
- }
- }
+ @Override
+ public void encode(double value, ByteArrayOutputStream out) throws IOException {
+ if (!flag) {
+ // case: write first 8 byte value without any encoding
+ flag = true;
+ preValue = Double.doubleToLongBits(value);
+ leadingZeroNum = Long.numberOfLeadingZeros(preValue);
+ tailingZeroNum = Long.numberOfTrailingZeros(preValue);
+ byte[] bufferBig = new byte[8];
+ byte[] bufferLittle = new byte[8];
- private void writeBits(long num, ByteArrayOutputStream out, int start, int end) {
- for (int i = start; i >= end; i--) {
- long bit = num & (1L << i);
- writeBit(bit, out);
- }
- }
-
- @Override
- public void flush(ByteArrayOutputStream out) throws IOException {
- encode(Double.NaN, out);
- clearBuffer(out);
- reset();
- }
-
- @Override
- public int getOneItemMaxSize() {
- // case '11'
- // 2bit + 6bit + 7bit + 64bit = 79bit
- return 10;
+ for (int i = 0; i < 8; i++) {
+ bufferLittle[i] = (byte) (((preValue) >> (i * 8)) & 0xFF);
+ bufferBig[8 - i - 1] = (byte) (((preValue) >> (i * 8)) & 0xFF);
+ }
+ out.write(bufferLittle);
+ } else {
+ long nextValue = Double.doubleToLongBits(value);
+ long tmp = nextValue ^ preValue;
+ if (tmp == 0) {
+ // case: write '0'
+ writeBit(false, out);
+ } else {
+ int leadingZeroNumTmp = Long.numberOfLeadingZeros(tmp);
+ int tailingZeroNumTmp = Long.numberOfTrailingZeros(tmp);
+ if (leadingZeroNumTmp >= leadingZeroNum && tailingZeroNumTmp >= tailingZeroNum) {
+ // case: write '10' and effective bits without first leadingZeroNum '0' and last
+ // tailingZeroNum '0'
+ writeBit(true, out);
+ writeBit(false, out);
+ writeBits(tmp, out, TSFileConfig.DOUBLE_LENGTH - 1 - leadingZeroNum, tailingZeroNum);
+ } else {
+ // case: write '11', leading zero num of value, effective bits len and effective bit value
+ writeBit(true, out);
+ writeBit(true, out);
+ writeBits(leadingZeroNumTmp, out, TSFileConfig.DOUBLE_LEADING_ZERO_LENGTH - 1, 0);
+ writeBits(TSFileConfig.DOUBLE_LENGTH - leadingZeroNumTmp - tailingZeroNumTmp, out,
+ TSFileConfig.DOUBLE_VALUE_LENGTH - 1, 0);
+ writeBits(tmp, out, TSFileConfig.DOUBLE_LENGTH - 1 - leadingZeroNumTmp,
+ tailingZeroNumTmp);
+ }
+ }
+ preValue = nextValue;
+ leadingZeroNum = Long.numberOfLeadingZeros(preValue);
+ tailingZeroNum = Long.numberOfTrailingZeros(preValue);
}
+ }
- @Override
- public long getMaxByteSize() {
- // max(first 8 byte, case '11' 2bit + 6bit + 7bit + 64bit = 79bit ) + NaN(2bit + 6bit + 7bit + 64bit = 79bit) = 158bit
- return 20;
+ private void writeBits(long num, ByteArrayOutputStream out, int start, int end) {
+ for (int i = start; i >= end; i--) {
+ long bit = num & (1L << i);
+ writeBit(bit, out);
}
+ }
+
+ @Override
+ public void flush(ByteArrayOutputStream out) throws IOException {
+ encode(Double.NaN, out);
+ clearBuffer(out);
+ reset();
+ }
+
+ @Override
+ public int getOneItemMaxSize() {
+ // case '11'
+ // 2bit + 6bit + 7bit + 64bit = 79bit
+ return 10;
+ }
+
+ @Override
+ public long getMaxByteSize() {
+ // max(first 8 byte, case '11' 2bit + 6bit + 7bit + 64bit = 79bit ) + NaN(2bit + 6bit + 7bit +
+ // 64bit = 79bit) = 158bit
+ return 20;
+ }
}
diff --git a/src/main/java/cn/edu/tsinghua/tsfile/encoding/encoder/Encoder.java b/src/main/java/cn/edu/tsinghua/tsfile/encoding/encoder/Encoder.java
index f19183fe..c355d51e 100644
--- a/src/main/java/cn/edu/tsinghua/tsfile/encoding/encoder/Encoder.java
+++ b/src/main/java/cn/edu/tsinghua/tsfile/encoding/encoder/Encoder.java
@@ -3,7 +3,6 @@
import cn.edu.tsinghua.tsfile.common.exception.TSFileEncodingException;
import cn.edu.tsinghua.tsfile.common.utils.Binary;
import cn.edu.tsinghua.tsfile.file.metadata.enums.TSEncoding;
-
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.math.BigDecimal;
@@ -16,62 +15,62 @@
*/
public abstract class Encoder {
- public TSEncoding type;
-
- public Encoder(TSEncoding type) {
- this.type = type;
- }
-
- public void encode(boolean value, ByteArrayOutputStream out) throws IOException {
- throw new TSFileEncodingException("Method encode boolean is not supported by Encoder");
- }
-
- public void encode(short value, ByteArrayOutputStream out) throws IOException {
- throw new TSFileEncodingException("Method encode short is not supported by Encoder");
- }
-
- public void encode(int value, ByteArrayOutputStream out) throws IOException {
- throw new TSFileEncodingException("Method encode int is not supported by Encoder");
- }
-
- public void encode(long value, ByteArrayOutputStream out) throws IOException {
- throw new TSFileEncodingException("Method encode long is not supported by Encoder");
- }
-
- public void encode(float value, ByteArrayOutputStream out) throws IOException {
- throw new TSFileEncodingException("Method encode float is not supported by Encoder");
- }
-
- public void encode(double value, ByteArrayOutputStream out) throws IOException {
- throw new TSFileEncodingException("Method encode double is not supported by Encoder");
- }
-
- public void encode(Binary value, ByteArrayOutputStream out) throws IOException {
- throw new TSFileEncodingException("Method encode Binary is not supported by Encoder");
- }
-
- public void encode(BigDecimal value, ByteArrayOutputStream out) throws IOException {
- throw new TSFileEncodingException("Method encode BigDecimal is not supported by Encoder");
- }
-
- public abstract void flush(ByteArrayOutputStream out) throws IOException;
-
- /**
- * return the maximal possible size of one data item.
- *
- * @return the maximal possible size of one data item encoded by this encoder
- */
- public int getOneItemMaxSize() {
- throw new UnsupportedOperationException();
- }
-
- /**
- * this function returns the maximal possible memory size occupied by current Encoder. This statistic is extra
- * memory size for Encoder and doesn't involve OutputStream.
- *
- * @return the maximal size of possible memory occupied by current encoder
- */
- public long getMaxByteSize() {
- throw new UnsupportedOperationException();
- }
+ public TSEncoding type;
+
+ public Encoder(TSEncoding type) {
+ this.type = type;
+ }
+
+ public void encode(boolean value, ByteArrayOutputStream out) throws IOException {
+ throw new TSFileEncodingException("Method encode boolean is not supported by Encoder");
+ }
+
+ public void encode(short value, ByteArrayOutputStream out) throws IOException {
+ throw new TSFileEncodingException("Method encode short is not supported by Encoder");
+ }
+
+ public void encode(int value, ByteArrayOutputStream out) throws IOException {
+ throw new TSFileEncodingException("Method encode int is not supported by Encoder");
+ }
+
+ public void encode(long value, ByteArrayOutputStream out) throws IOException {
+ throw new TSFileEncodingException("Method encode long is not supported by Encoder");
+ }
+
+ public void encode(float value, ByteArrayOutputStream out) throws IOException {
+ throw new TSFileEncodingException("Method encode float is not supported by Encoder");
+ }
+
+ public void encode(double value, ByteArrayOutputStream out) throws IOException {
+ throw new TSFileEncodingException("Method encode double is not supported by Encoder");
+ }
+
+ public void encode(Binary value, ByteArrayOutputStream out) throws IOException {
+ throw new TSFileEncodingException("Method encode Binary is not supported by Encoder");
+ }
+
+ public void encode(BigDecimal value, ByteArrayOutputStream out) throws IOException {
+ throw new TSFileEncodingException("Method encode BigDecimal is not supported by Encoder");
+ }
+
+ public abstract void flush(ByteArrayOutputStream out) throws IOException;
+
+ /**
+ * return the maximal possible size of one data item.
+ *
+ * @return the maximal possible size of one data item encoded by this encoder
+ */
+ public int getOneItemMaxSize() {
+ throw new UnsupportedOperationException();
+ }
+
+ /**
+ * this function returns the maximal possible memory size occupied by current Encoder. This
+ * statistic is extra memory size for Encoder and doesn't involve OutputStream.
+ *
+ * @return the maximal size of possible memory occupied by current encoder
+ */
+ public long getMaxByteSize() {
+ throw new UnsupportedOperationException();
+ }
}
diff --git a/src/main/java/cn/edu/tsinghua/tsfile/encoding/encoder/FloatEncoder.java b/src/main/java/cn/edu/tsinghua/tsfile/encoding/encoder/FloatEncoder.java
index b8a5bacf..a6b14ed7 100644
--- a/src/main/java/cn/edu/tsinghua/tsfile/encoding/encoder/FloatEncoder.java
+++ b/src/main/java/cn/edu/tsinghua/tsfile/encoding/encoder/FloatEncoder.java
@@ -7,13 +7,12 @@
import cn.edu.tsinghua.tsfile.file.metadata.enums.TSEncoding;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-
import java.io.ByteArrayOutputStream;
import java.io.IOException;
/**
- * Encoder for float or double value using rle or two diff
- * according to following grammar:
+ * Encoder for float or double value using rle or two diff according to following grammar:
+ *
*
* {@code
* float encoder:
@@ -23,113 +22,113 @@
*
*/
public class FloatEncoder extends Encoder {
- private static final Logger LOGGER = LoggerFactory.getLogger(FloatEncoder.class);
- private Encoder encoder;
-
- /**
- * number for accuracy of decimal places
- */
- private int maxPointNumber;
-
- /**
- * maxPointValue = 10^(maxPointNumber)
- */
- private double maxPointValue;
-
- /**
- * flag to check whether maxPointNumber is saved in stream
- */
- private boolean isMaxPointNumberSaved;
-
- public FloatEncoder(TSEncoding encodingType, TSDataType dataType, int maxPointNumber) {
- super(encodingType);
- this.maxPointNumber = maxPointNumber;
- calculateMaxPonitNum();
- isMaxPointNumberSaved = false;
- if (encodingType == TSEncoding.RLE) {
- if (dataType == TSDataType.FLOAT) {
- encoder = new IntRleEncoder(EndianType.LITTLE_ENDIAN);
- LOGGER.debug("tsfile-encoding FloatEncoder: init encoder using int-rle and float");
- } else if (dataType == TSDataType.DOUBLE) {
- encoder = new LongRleEncoder(EndianType.LITTLE_ENDIAN);
- LOGGER.debug("tsfile-encoding FloatEncoder: init encoder using long-rle and double");
- } else {
- throw new TSFileEncodingException(
- String.format("data type %s is not supported by FloatEncoder", dataType));
- }
- } else if (encodingType == TSEncoding.TS_2DIFF) {
- if (dataType == TSDataType.FLOAT) {
- encoder = new DeltaBinaryEncoder.IntDeltaEncoder();
- LOGGER.debug("tsfile-encoding FloatEncoder: init encoder using int-delta and float");
- } else if (dataType == TSDataType.DOUBLE) {
- encoder = new DeltaBinaryEncoder.LongDeltaEncoder();
- LOGGER.debug("tsfile-encoding FloatEncoder: init encoder using long-delta and double");
- } else {
- throw new TSFileEncodingException(
- String.format("data type %s is not supported by FloatEncoder", dataType));
- }
- } else {
- throw new TSFileEncodingException(
- String.format("%s encoding is not supported by FloatEncoder", encodingType));
- }
+ private static final Logger LOGGER = LoggerFactory.getLogger(FloatEncoder.class);
+ private Encoder encoder;
+
+ /**
+ * number for accuracy of decimal places
+ */
+ private int maxPointNumber;
+
+ /**
+ * maxPointValue = 10^(maxPointNumber)
+ */
+ private double maxPointValue;
+
+ /**
+ * flag to check whether maxPointNumber is saved in stream
+ */
+ private boolean isMaxPointNumberSaved;
+
+ public FloatEncoder(TSEncoding encodingType, TSDataType dataType, int maxPointNumber) {
+ super(encodingType);
+ this.maxPointNumber = maxPointNumber;
+ calculateMaxPonitNum();
+ isMaxPointNumberSaved = false;
+ if (encodingType == TSEncoding.RLE) {
+ if (dataType == TSDataType.FLOAT) {
+ encoder = new IntRleEncoder(EndianType.LITTLE_ENDIAN);
+ LOGGER.debug("tsfile-encoding FloatEncoder: init encoder using int-rle and float");
+ } else if (dataType == TSDataType.DOUBLE) {
+ encoder = new LongRleEncoder(EndianType.LITTLE_ENDIAN);
+ LOGGER.debug("tsfile-encoding FloatEncoder: init encoder using long-rle and double");
+ } else {
+ throw new TSFileEncodingException(
+ String.format("data type %s is not supported by FloatEncoder", dataType));
+ }
+ } else if (encodingType == TSEncoding.TS_2DIFF) {
+ if (dataType == TSDataType.FLOAT) {
+ encoder = new DeltaBinaryEncoder.IntDeltaEncoder();
+ LOGGER.debug("tsfile-encoding FloatEncoder: init encoder using int-delta and float");
+ } else if (dataType == TSDataType.DOUBLE) {
+ encoder = new DeltaBinaryEncoder.LongDeltaEncoder();
+ LOGGER.debug("tsfile-encoding FloatEncoder: init encoder using long-delta and double");
+ } else {
+ throw new TSFileEncodingException(
+ String.format("data type %s is not supported by FloatEncoder", dataType));
+ }
+ } else {
+ throw new TSFileEncodingException(
+ String.format("%s encoding is not supported by FloatEncoder", encodingType));
}
-
- @Override
- public void encode(float value, ByteArrayOutputStream out) throws IOException {
- saveMaxPointNumber(out);
- int valueInt = convertFloatToInt(value);
- encoder.encode(valueInt, out);
+ }
+
+ @Override
+ public void encode(float value, ByteArrayOutputStream out) throws IOException {
+ saveMaxPointNumber(out);
+ int valueInt = convertFloatToInt(value);
+ encoder.encode(valueInt, out);
+ }
+
+ @Override
+ public void encode(double value, ByteArrayOutputStream out) throws IOException {
+ saveMaxPointNumber(out);
+ long valueLong = convertDoubleToLong(value);
+ encoder.encode(valueLong, out);
+ }
+
+ private void calculateMaxPonitNum() {
+ if (maxPointNumber <= 0) {
+ maxPointNumber = 0;
+ maxPointValue = 1;
+ } else {
+ maxPointValue = Math.pow(10, maxPointNumber);
}
-
- @Override
- public void encode(double value, ByteArrayOutputStream out) throws IOException {
- saveMaxPointNumber(out);
- long valueLong = convertDoubleToLong(value);
- encoder.encode(valueLong, out);
+ }
+
+ private int convertFloatToInt(float value) {
+ return (int) Math.round(value * maxPointValue);
+ }
+
+ private long convertDoubleToLong(double value) {
+ return Math.round(value * maxPointValue);
+ }
+
+ @Override
+ public void flush(ByteArrayOutputStream out) throws IOException {
+ encoder.flush(out);
+ reset();
+ }
+
+ private void reset() {
+ isMaxPointNumberSaved = false;
+ }
+
+ private void saveMaxPointNumber(ByteArrayOutputStream out) throws IOException {
+ if (!isMaxPointNumberSaved) {
+ ReadWriteStreamUtils.writeUnsignedVarInt(maxPointNumber, out);
+ isMaxPointNumberSaved = true;
}
+ }
- private void calculateMaxPonitNum() {
- if (maxPointNumber <= 0) {
- maxPointNumber = 0;
- maxPointValue = 1;
- } else {
- maxPointValue = Math.pow(10, maxPointNumber);
- }
- }
+ @Override
+ public int getOneItemMaxSize() {
+ return encoder.getOneItemMaxSize();
+ }
- private int convertFloatToInt(float value) {
- return (int) Math.round(value * maxPointValue);
- }
-
- private long convertDoubleToLong(double value) {
- return Math.round(value * maxPointValue);
- }
-
- @Override
- public void flush(ByteArrayOutputStream out) throws IOException {
- encoder.flush(out);
- reset();
- }
-
- private void reset() {
- isMaxPointNumberSaved = false;
- }
-
- private void saveMaxPointNumber(ByteArrayOutputStream out) throws IOException {
- if (!isMaxPointNumberSaved) {
- ReadWriteStreamUtils.writeUnsignedVarInt(maxPointNumber, out);
- isMaxPointNumberSaved = true;
- }
- }
-
- @Override
- public int getOneItemMaxSize() {
- return encoder.getOneItemMaxSize();
- }
-
- @Override
- public long getMaxByteSize() {
- return encoder.getMaxByteSize();
- }
+ @Override
+ public long getMaxByteSize() {
+ return encoder.getMaxByteSize();
+ }
}
diff --git a/src/main/java/cn/edu/tsinghua/tsfile/encoding/encoder/GorillaEncoder.java b/src/main/java/cn/edu/tsinghua/tsfile/encoding/encoder/GorillaEncoder.java
index 3feff798..62e29efe 100644
--- a/src/main/java/cn/edu/tsinghua/tsfile/encoding/encoder/GorillaEncoder.java
+++ b/src/main/java/cn/edu/tsinghua/tsfile/encoding/encoder/GorillaEncoder.java
@@ -1,64 +1,67 @@
package cn.edu.tsinghua.tsfile.encoding.encoder;
import java.io.ByteArrayOutputStream;
-
import cn.edu.tsinghua.tsfile.file.metadata.enums.TSEncoding;
/**
- * Gorilla encoding. For more information about how it works,
- * please see http://www.vldb.org/pvldb/vol8/p1816-teller.pdf
+ * Gorilla encoding. For more information about how it works, please see
+ * http://www.vldb.org/pvldb/vol8/p1816-teller.pdf
*/
-public abstract class GorillaEncoder extends Encoder{
- // flag to indicate whether the first value is saved
- protected boolean flag;
- protected int leadingZeroNum, tailingZeroNum;
- // 8-bit buffer of bits to write out
- protected byte buffer;
- // number of bits remaining in buffer
- protected int numberLeftInBuffer;
-
- public GorillaEncoder() {
- super(TSEncoding.GORILLA);
- this.flag = false;
- }
+public abstract class GorillaEncoder extends Encoder {
+ // flag to indicate whether the first value is saved
+ protected boolean flag;
+ protected int leadingZeroNum, tailingZeroNum;
+ // 8-bit buffer of bits to write out
+ protected byte buffer;
+ // number of bits remaining in buffer
+ protected int numberLeftInBuffer;
+
+ public GorillaEncoder() {
+ super(TSEncoding.GORILLA);
+ this.flag = false;
+ }
+
+ protected void writeBit(boolean b, ByteArrayOutputStream out) {
+ // add bit to buffer
+ buffer <<= 1;
+ if (b)
+ buffer |= 1;
+
+ // if buffer is full (8 bits), write out as a single byte
+ numberLeftInBuffer++;
+ if (numberLeftInBuffer == 8)
+ clearBuffer(out);
+ }
+
+ protected void writeBit(int i, ByteArrayOutputStream out) {
+ if (i == 0) {
+ writeBit(false, out);
+ } else {
+ writeBit(true, out);
+ }
+ }
+
+ protected void writeBit(long i, ByteArrayOutputStream out) {
+ if (i == 0) {
+ writeBit(false, out);
+ } else {
+ writeBit(true, out);
+ }
+ }
- protected void writeBit(boolean b, ByteArrayOutputStream out){
- // add bit to buffer
- buffer <<= 1;
- if (b) buffer |= 1;
+ protected void clearBuffer(ByteArrayOutputStream out) {
+ if (numberLeftInBuffer == 0)
+ return;
+ if (numberLeftInBuffer > 0)
+ buffer <<= (8 - numberLeftInBuffer);
+ out.write(buffer);
+ numberLeftInBuffer = 0;
+ buffer = 0;
+ }
- // if buffer is full (8 bits), write out as a single byte
- numberLeftInBuffer++;
- if (numberLeftInBuffer == 8) clearBuffer(out);
- }
-
- protected void writeBit(int i, ByteArrayOutputStream out){
- if(i == 0){
- writeBit(false, out);
- } else{
- writeBit(true, out);
- }
- }
-
- protected void writeBit(long i, ByteArrayOutputStream out){
- if(i == 0){
- writeBit(false, out);
- } else{
- writeBit(true, out);
- }
- }
-
- protected void clearBuffer(ByteArrayOutputStream out){
- if (numberLeftInBuffer == 0) return;
- if (numberLeftInBuffer > 0) buffer <<= (8 - numberLeftInBuffer);
- out.write(buffer);
- numberLeftInBuffer = 0;
- buffer = 0;
- }
-
- protected void reset(){
- this.flag = false;
- this.numberLeftInBuffer = 0;
- this.buffer = 0;
- }
+ protected void reset() {
+ this.flag = false;
+ this.numberLeftInBuffer = 0;
+ this.buffer = 0;
+ }
}
diff --git a/src/main/java/cn/edu/tsinghua/tsfile/encoding/encoder/IntRleEncoder.java b/src/main/java/cn/edu/tsinghua/tsfile/encoding/encoder/IntRleEncoder.java
index 5575e8fb..a79f34f9 100644
--- a/src/main/java/cn/edu/tsinghua/tsfile/encoding/encoder/IntRleEncoder.java
+++ b/src/main/java/cn/edu/tsinghua/tsfile/encoding/encoder/IntRleEncoder.java
@@ -1,118 +1,116 @@
-package cn.edu.tsinghua.tsfile.encoding.encoder;
-
-import cn.edu.tsinghua.tsfile.common.utils.ReadWriteStreamUtils;
-import cn.edu.tsinghua.tsfile.encoding.bitpacking.IntPacker;
-import cn.edu.tsinghua.tsfile.encoding.common.EndianType;
-
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.util.ArrayList;
-
-
-/**
- * Encoder for int value using rle or bit-packing
- */
-public class IntRleEncoder extends RleEncoder {
-
- /**
- * Packer for packing int value
- */
- private IntPacker packer;
-
- public IntRleEncoder(EndianType endianType) {
- super(endianType);
- bufferedValues = new Integer[config.RLE_MIN_REPEATED_NUM];
- preValue = 0;
- values = new ArrayList();
- }
-
- @Override
- public void encode(int value, ByteArrayOutputStream out) {
- values.add(value);
- }
-
- @Override
- public void encode(boolean value, ByteArrayOutputStream out) {
- if (value) {
- this.encode(1, out);
- } else {
- this.encode(0, out);
- }
- }
-
- /**
- * write all values buffered in cache to OutputStream
- *
- * @param out - byteArrayOutputStream
- * @throws IOException cannot flush to OutputStream
- */
- @Override
- public void flush(ByteArrayOutputStream out) throws IOException {
- // we get bit width after receiving all data
- this.bitWidth = ReadWriteStreamUtils.getIntMaxBitWidth(values);
- packer = new IntPacker(bitWidth);
- for (Integer value : values) {
- encodeValue(value);
- }
- super.flush(out);
- }
-
- @Override
- protected void reset() {
- super.reset();
- preValue = 0;
- }
-
- /**
- * write bytes to OutputStream using rle
- * rle format: [header][value]
- */
- @Override
- protected void writeRleRun() throws IOException {
- endPreviousBitPackedRun(config.RLE_MIN_REPEATED_NUM);
- ReadWriteStreamUtils.writeUnsignedVarInt(repeatCount << 1, byteCache);
- ReadWriteStreamUtils.writeIntLittleEndianPaddedOnBitWidth(preValue, byteCache, bitWidth);
- repeatCount = 0;
- numBufferedValues = 0;
- }
-
- @Override
- protected void clearBuffer() {
-
- for (int i = numBufferedValues; i < config.RLE_MIN_REPEATED_NUM; i++) {
- bufferedValues[i] = 0;
- }
- }
-
- @Override
- protected void convertBuffer() {
- byte[] bytes = new byte[bitWidth];
-
- int[] tmpBuffer = new int[config.RLE_MIN_REPEATED_NUM];
- for (int i = 0; i < config.RLE_MIN_REPEATED_NUM; i++) {
- tmpBuffer[i] = (int) bufferedValues[i];
- }
- packer.pack8Values(tmpBuffer, 0, bytes);
- // we'll not write bit-packing group to OutputStream immediately
- // we buffer them in list
- bytesBuffer.add(bytes);
- }
-
- @Override
- public int getOneItemMaxSize() {
- // The meaning of 45 is:
- // 4 + 4 + max(4+4,1 + 4 + 4 * 8)
- // length + bitwidth + max(rle-header + num, bit-header + lastNum + 8packer)
- return 45;
- }
-
- @Override
- public long getMaxByteSize() {
- if (values == null) {
- return 0;
- }
- // try to caculate max value
- int groupNum = (values.size() / 8 + 1) / 63 + 1;
- return 8 + groupNum * 5 + values.size() * 4;
- }
-}
\ No newline at end of file
+package cn.edu.tsinghua.tsfile.encoding.encoder;
+
+import cn.edu.tsinghua.tsfile.common.utils.ReadWriteStreamUtils;
+import cn.edu.tsinghua.tsfile.encoding.bitpacking.IntPacker;
+import cn.edu.tsinghua.tsfile.encoding.common.EndianType;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+
+
+/**
+ * Encoder for int value using rle or bit-packing
+ */
+public class IntRleEncoder extends RleEncoder {
+
+ /**
+ * Packer for packing int value
+ */
+ private IntPacker packer;
+
+ public IntRleEncoder(EndianType endianType) {
+ super(endianType);
+ bufferedValues = new Integer[config.RLE_MIN_REPEATED_NUM];
+ preValue = 0;
+ values = new ArrayList();
+ }
+
+ @Override
+ public void encode(int value, ByteArrayOutputStream out) {
+ values.add(value);
+ }
+
+ @Override
+ public void encode(boolean value, ByteArrayOutputStream out) {
+ if (value) {
+ this.encode(1, out);
+ } else {
+ this.encode(0, out);
+ }
+ }
+
+ /**
+ * write all values buffered in cache to OutputStream
+ *
+ * @param out - byteArrayOutputStream
+ * @throws IOException cannot flush to OutputStream
+ */
+ @Override
+ public void flush(ByteArrayOutputStream out) throws IOException {
+ // we get bit width after receiving all data
+ this.bitWidth = ReadWriteStreamUtils.getIntMaxBitWidth(values);
+ packer = new IntPacker(bitWidth);
+ for (Integer value : values) {
+ encodeValue(value);
+ }
+ super.flush(out);
+ }
+
+ @Override
+ protected void reset() {
+ super.reset();
+ preValue = 0;
+ }
+
+ /**
+ * write bytes to OutputStream using rle rle format: [header][value]
+ */
+ @Override
+ protected void writeRleRun() throws IOException {
+ endPreviousBitPackedRun(config.RLE_MIN_REPEATED_NUM);
+ ReadWriteStreamUtils.writeUnsignedVarInt(repeatCount << 1, byteCache);
+ ReadWriteStreamUtils.writeIntLittleEndianPaddedOnBitWidth(preValue, byteCache, bitWidth);
+ repeatCount = 0;
+ numBufferedValues = 0;
+ }
+
+ @Override
+ protected void clearBuffer() {
+
+ for (int i = numBufferedValues; i < config.RLE_MIN_REPEATED_NUM; i++) {
+ bufferedValues[i] = 0;
+ }
+ }
+
+ @Override
+ protected void convertBuffer() {
+ byte[] bytes = new byte[bitWidth];
+
+ int[] tmpBuffer = new int[config.RLE_MIN_REPEATED_NUM];
+ for (int i = 0; i < config.RLE_MIN_REPEATED_NUM; i++) {
+ tmpBuffer[i] = (int) bufferedValues[i];
+ }
+ packer.pack8Values(tmpBuffer, 0, bytes);
+ // we'll not write bit-packing group to OutputStream immediately
+ // we buffer them in list
+ bytesBuffer.add(bytes);
+ }
+
+ @Override
+ public int getOneItemMaxSize() {
+ // The meaning of 45 is:
+ // 4 + 4 + max(4+4,1 + 4 + 4 * 8)
+ // length + bitwidth + max(rle-header + num, bit-header + lastNum + 8packer)
+ return 45;
+ }
+
+ @Override
+ public long getMaxByteSize() {
+ if (values == null) {
+ return 0;
+ }
+ // try to caculate max value
+ int groupNum = (values.size() / 8 + 1) / 63 + 1;
+ return 8 + groupNum * 5 + values.size() * 4;
+ }
+}
diff --git a/src/main/java/cn/edu/tsinghua/tsfile/encoding/encoder/LongRleEncoder.java b/src/main/java/cn/edu/tsinghua/tsfile/encoding/encoder/LongRleEncoder.java
index 709497a2..5290fa2e 100644
--- a/src/main/java/cn/edu/tsinghua/tsfile/encoding/encoder/LongRleEncoder.java
+++ b/src/main/java/cn/edu/tsinghua/tsfile/encoding/encoder/LongRleEncoder.java
@@ -1,105 +1,105 @@
-package cn.edu.tsinghua.tsfile.encoding.encoder;
-
-import cn.edu.tsinghua.tsfile.common.utils.ReadWriteStreamUtils;
-import cn.edu.tsinghua.tsfile.encoding.bitpacking.LongPacker;
-import cn.edu.tsinghua.tsfile.encoding.common.EndianType;
-
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.util.ArrayList;
-
-/**
- * Encoder for long value using rle or bit-packing
- */
-public class LongRleEncoder extends RleEncoder {
-
- /**
- * Packer for packing long value
- */
- private LongPacker packer;
-
- public LongRleEncoder(EndianType endianType) {
- super(endianType);
- bufferedValues = new Long[config.RLE_MIN_REPEATED_NUM];
- preValue = (long) 0;
- values = new ArrayList();
- }
-
- @Override
- public void encode(long value, ByteArrayOutputStream out) {
- values.add(value);
- }
-
- /**
- * write all values buffered in cache to OutputStream
- *
- * @param out - byteArrayOutputStream
- * @throws IOException cannot flush to OutputStream
- */
- @Override
- public void flush(ByteArrayOutputStream out) throws IOException {
- // we get bit width after receiving all data
- this.bitWidth = ReadWriteStreamUtils.getLongMaxBitWidth(values);
- packer = new LongPacker(bitWidth);
- for (Long value : values) {
- encodeValue(value);
- }
- super.flush(out);
- }
-
- @Override
- protected void reset() {
- super.reset();
- preValue = (long) 0;
- }
-
- /**
- * write bytes to OutputStream using rle rle format: [header][value]
- * @throws IOException cannot write rle run
- */
- @Override
- protected void writeRleRun() throws IOException {
- endPreviousBitPackedRun(config.RLE_MIN_REPEATED_NUM);
- ReadWriteStreamUtils.writeUnsignedVarInt(repeatCount << 1, byteCache);
- ReadWriteStreamUtils.writeLongLittleEndianPaddedOnBitWidth(preValue, byteCache, bitWidth);
- repeatCount = 0;
- numBufferedValues = 0;
- }
-
- @Override
- protected void clearBuffer() {
- for (int i = numBufferedValues; i < config.RLE_MIN_REPEATED_NUM; i++) {
- bufferedValues[i] = (long) 0;
- }
- }
-
- @Override
- protected void convertBuffer() {
- byte[] bytes = new byte[bitWidth];
- long[] tmpBuffer = new long[config.RLE_MIN_REPEATED_NUM];
- for (int i = 0; i < config.RLE_MIN_REPEATED_NUM; i++) {
- tmpBuffer[i] = (long) bufferedValues[i];
- }
- packer.pack8Values(tmpBuffer, 0, bytes);
- // we'll not write bit-packing group to OutputStream immediately
- // we buffer them in list
- bytesBuffer.add(bytes);
- }
-
- @Override
- public int getOneItemMaxSize() {
- // 4 + 4 + max(4+8,1 + 4 + 8 * 8)
- // length + bitwidth + max(rle-header + num, bit-header + lastNum + 8packer)
- return 77;
- }
-
- @Override
- public long getMaxByteSize() {
- if (values == null) {
- return 0;
- }
- // try to caculate max value
- int groupNum = (values.size() / 8 + 1) / 63 + 1;
- return 8 + groupNum * 5 + values.size() * 8;
- }
-}
\ No newline at end of file
+package cn.edu.tsinghua.tsfile.encoding.encoder;
+
+import cn.edu.tsinghua.tsfile.common.utils.ReadWriteStreamUtils;
+import cn.edu.tsinghua.tsfile.encoding.bitpacking.LongPacker;
+import cn.edu.tsinghua.tsfile.encoding.common.EndianType;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+
+/**
+ * Encoder for long value using rle or bit-packing
+ */
+public class LongRleEncoder extends RleEncoder {
+
+ /**
+ * Packer for packing long value
+ */
+ private LongPacker packer;
+
+ public LongRleEncoder(EndianType endianType) {
+ super(endianType);
+ bufferedValues = new Long[config.RLE_MIN_REPEATED_NUM];
+ preValue = (long) 0;
+ values = new ArrayList();
+ }
+
+ @Override
+ public void encode(long value, ByteArrayOutputStream out) {
+ values.add(value);
+ }
+
+ /**
+ * write all values buffered in cache to OutputStream
+ *
+ * @param out - byteArrayOutputStream
+ * @throws IOException cannot flush to OutputStream
+ */
+ @Override
+ public void flush(ByteArrayOutputStream out) throws IOException {
+ // we get bit width after receiving all data
+ this.bitWidth = ReadWriteStreamUtils.getLongMaxBitWidth(values);
+ packer = new LongPacker(bitWidth);
+ for (Long value : values) {
+ encodeValue(value);
+ }
+ super.flush(out);
+ }
+
+ @Override
+ protected void reset() {
+ super.reset();
+ preValue = (long) 0;
+ }
+
+ /**
+ * write bytes to OutputStream using rle rle format: [header][value]
+ *
+ * @throws IOException cannot write rle run
+ */
+ @Override
+ protected void writeRleRun() throws IOException {
+ endPreviousBitPackedRun(config.RLE_MIN_REPEATED_NUM);
+ ReadWriteStreamUtils.writeUnsignedVarInt(repeatCount << 1, byteCache);
+ ReadWriteStreamUtils.writeLongLittleEndianPaddedOnBitWidth(preValue, byteCache, bitWidth);
+ repeatCount = 0;
+ numBufferedValues = 0;
+ }
+
+ @Override
+ protected void clearBuffer() {
+ for (int i = numBufferedValues; i < config.RLE_MIN_REPEATED_NUM; i++) {
+ bufferedValues[i] = (long) 0;
+ }
+ }
+
+ @Override
+ protected void convertBuffer() {
+ byte[] bytes = new byte[bitWidth];
+ long[] tmpBuffer = new long[config.RLE_MIN_REPEATED_NUM];
+ for (int i = 0; i < config.RLE_MIN_REPEATED_NUM; i++) {
+ tmpBuffer[i] = (long) bufferedValues[i];
+ }
+ packer.pack8Values(tmpBuffer, 0, bytes);
+ // we'll not write bit-packing group to OutputStream immediately
+ // we buffer them in list
+ bytesBuffer.add(bytes);
+ }
+
+ @Override
+ public int getOneItemMaxSize() {
+ // 4 + 4 + max(4+8,1 + 4 + 8 * 8)
+ // length + bitwidth + max(rle-header + num, bit-header + lastNum + 8packer)
+ return 77;
+ }
+
+ @Override
+ public long getMaxByteSize() {
+ if (values == null) {
+ return 0;
+ }
+ // try to caculate max value
+ int groupNum = (values.size() / 8 + 1) / 63 + 1;
+ return 8 + groupNum * 5 + values.size() * 8;
+ }
+}
diff --git a/src/main/java/cn/edu/tsinghua/tsfile/encoding/encoder/PlainEncoder.java b/src/main/java/cn/edu/tsinghua/tsfile/encoding/encoder/PlainEncoder.java
index 2750c5b1..db919796 100644
--- a/src/main/java/cn/edu/tsinghua/tsfile/encoding/encoder/PlainEncoder.java
+++ b/src/main/java/cn/edu/tsinghua/tsfile/encoding/encoder/PlainEncoder.java
@@ -8,7 +8,6 @@
import cn.edu.tsinghua.tsfile.file.metadata.enums.TSEncoding;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.math.BigDecimal;
@@ -18,144 +17,145 @@
* @author Zhang Jinrui
*/
public class PlainEncoder extends Encoder {
- private static final Logger LOGGER = LoggerFactory.getLogger(PlainEncoder.class);
- public EndianType endianType;
- private TSDataType dataType;
- private int maxStringLength;
-
- public PlainEncoder(EndianType endianType, TSDataType dataType, int maxStringLength) {
- super(TSEncoding.PLAIN);
- this.endianType = endianType;
- this.dataType = dataType;
- this.maxStringLength = maxStringLength;
- }
-
- @Override
- public void encode(boolean value, ByteArrayOutputStream out) {
- if (value) {
- out.write(1);
- } else {
- out.write(0);
- }
+ private static final Logger LOGGER = LoggerFactory.getLogger(PlainEncoder.class);
+ public EndianType endianType;
+ private TSDataType dataType;
+ private int maxStringLength;
+
+ public PlainEncoder(EndianType endianType, TSDataType dataType, int maxStringLength) {
+ super(TSEncoding.PLAIN);
+ this.endianType = endianType;
+ this.dataType = dataType;
+ this.maxStringLength = maxStringLength;
+ }
+
+ @Override
+ public void encode(boolean value, ByteArrayOutputStream out) {
+ if (value) {
+ out.write(1);
+ } else {
+ out.write(0);
}
-
- @Override
- public void encode(short value, ByteArrayOutputStream out) {
- if (this.endianType == EndianType.LITTLE_ENDIAN) {
- out.write((value >> 0) & 0xFF);
- out.write((value >> 8) & 0xFF);
- } else if (this.endianType == EndianType.BIG_ENDIAN) {
- LOGGER.error(
- "tsfile-encoding PlainEncoder: current version does not support short value encoding");
- throw new TSFileEncodingException(
- "tsfile-encoding PlainEncoder: current version does not support short value encoding");
- // out.write((value >> 8) & 0xFF);
- // out.write((value >> 0) & 0xFF);
- }
+ }
+
+ @Override
+ public void encode(short value, ByteArrayOutputStream out) {
+ if (this.endianType == EndianType.LITTLE_ENDIAN) {
+ out.write((value >> 0) & 0xFF);
+ out.write((value >> 8) & 0xFF);
+ } else if (this.endianType == EndianType.BIG_ENDIAN) {
+ LOGGER.error(
+ "tsfile-encoding PlainEncoder: current version does not support short value encoding");
+ throw new TSFileEncodingException(
+ "tsfile-encoding PlainEncoder: current version does not support short value encoding");
+ // out.write((value >> 8) & 0xFF);
+ // out.write((value >> 0) & 0xFF);
}
-
- @Override
- public void encode(int value, ByteArrayOutputStream out) {
- if (this.endianType == EndianType.LITTLE_ENDIAN) {
- out.write((value >> 0) & 0xFF);
- out.write((value >> 8) & 0xFF);
- out.write((value >> 16) & 0xFF);
- out.write((value >> 24) & 0xFF);
- } else if (this.endianType == EndianType.BIG_ENDIAN) {
- LOGGER.error(
- "tsfile-encoding PlainEncoder: current version does not support int value encoding");
- throw new TSFileEncodingException(
- "tsfile-encoding PlainEncoder: current version does not support int value encoding");
- // out.write((value >> 24) & 0xFF);
- // out.write((value >> 16) & 0xFF);
- // out.write((value >> 8) & 0xFF);
- // out.write((value >> 0) & 0xFF);
- }
+ }
+
+ @Override
+ public void encode(int value, ByteArrayOutputStream out) {
+ if (this.endianType == EndianType.LITTLE_ENDIAN) {
+ out.write((value >> 0) & 0xFF);
+ out.write((value >> 8) & 0xFF);
+ out.write((value >> 16) & 0xFF);
+ out.write((value >> 24) & 0xFF);
+ } else if (this.endianType == EndianType.BIG_ENDIAN) {
+ LOGGER.error(
+ "tsfile-encoding PlainEncoder: current version does not support int value encoding");
+ throw new TSFileEncodingException(
+ "tsfile-encoding PlainEncoder: current version does not support int value encoding");
+ // out.write((value >> 24) & 0xFF);
+ // out.write((value >> 16) & 0xFF);
+ // out.write((value >> 8) & 0xFF);
+ // out.write((value >> 0) & 0xFF);
}
+ }
- @Override
- public void encode(long value, ByteArrayOutputStream out) {
- byte[] bufferBig = new byte[8];
- byte[] bufferLittle = new byte[8];
-
- for (int i = 0; i < 8; i++) {
- bufferLittle[i] = (byte) (((value) >> (i * 8)) & 0xFF);
- bufferBig[8 - i - 1] = (byte) (((value) >> (i * 8)) & 0xFF);
- }
- try {
- if (this.endianType == EndianType.LITTLE_ENDIAN) {
- out.write(bufferLittle);
- } else if (this.endianType == EndianType.BIG_ENDIAN) {
- LOGGER.error(
- "tsfile-encoding PlainEncoder: current version does not support long value encoding");
- throw new TSFileEncodingException(
- "tsfile-encoding PlainEncoder: current version does not support long value encoding");
-// out.write(bufferBig);
- }
- } catch (IOException e) {
- LOGGER.error("tsfile-encoding PlainEncoder: error occurs when encode long value {}", value,
- e);
- }
- }
+ @Override
+ public void encode(long value, ByteArrayOutputStream out) {
+ byte[] bufferBig = new byte[8];
+ byte[] bufferLittle = new byte[8];
- @Override
- public void encode(float value, ByteArrayOutputStream out) {
- encode(Float.floatToIntBits(value), out);
+ for (int i = 0; i < 8; i++) {
+ bufferLittle[i] = (byte) (((value) >> (i * 8)) & 0xFF);
+ bufferBig[8 - i - 1] = (byte) (((value) >> (i * 8)) & 0xFF);
}
-
- @Override
- public void encode(double value, ByteArrayOutputStream out) {
- encode(Double.doubleToLongBits(value), out);
+ try {
+ if (this.endianType == EndianType.LITTLE_ENDIAN) {
+ out.write(bufferLittle);
+ } else if (this.endianType == EndianType.BIG_ENDIAN) {
+ LOGGER.error(
+ "tsfile-encoding PlainEncoder: current version does not support long value encoding");
+ throw new TSFileEncodingException(
+ "tsfile-encoding PlainEncoder: current version does not support long value encoding");
+ // out.write(bufferBig);
+ }
+ } catch (IOException e) {
+ LOGGER.error("tsfile-encoding PlainEncoder: error occurs when encode long value {}", value,
+ e);
}
-
- @Override
- public void encode(Binary value, ByteArrayOutputStream out) {
- try {
- // write the length of the bytes
- encode(value.getLength(), out);
- // write value
- out.write(value.values);
- } catch (IOException e) {
- LOGGER.error("tsfile-encoding PlainEncoder: error occurs when encode Binary value {}", value, e);
- }
+ }
+
+ @Override
+ public void encode(float value, ByteArrayOutputStream out) {
+ encode(Float.floatToIntBits(value), out);
+ }
+
+ @Override
+ public void encode(double value, ByteArrayOutputStream out) {
+ encode(Double.doubleToLongBits(value), out);
+ }
+
+ @Override
+ public void encode(Binary value, ByteArrayOutputStream out) {
+ try {
+ // write the length of the bytes
+ encode(value.getLength(), out);
+ // write value
+ out.write(value.values);
+ } catch (IOException e) {
+ LOGGER.error("tsfile-encoding PlainEncoder: error occurs when encode Binary value {}", value,
+ e);
}
-
- @Override
- public void flush(ByteArrayOutputStream out) {
- }
-
- @Override
- public int getOneItemMaxSize() {
- switch (dataType) {
- case BOOLEAN:
- return 1;
- case INT32:
- return 4;
- case INT64:
- return 8;
- case FLOAT:
- return 4;
- case DOUBLE:
- return 8;
- case TEXT:
- // refer to encode(Binary,ByteArrayOutputStream)
- return 4 + TSFileConfig.BYTE_SIZE_PER_CHAR * maxStringLength;
- case ENUMS:
- return 4;
- case BIGDECIMAL:
- return 8;
- default:
- throw new UnsupportedOperationException(dataType.toString());
- }
- }
-
- @Override
- public long getMaxByteSize() {
- return 0;
- }
-
- @Override
- public void encode(BigDecimal value, ByteArrayOutputStream out) throws IOException {
- throw new TSFileEncodingException("tsfile-encoding PlainEncoder: current version does not support BigDecimal value encoding");
+ }
+
+ @Override
+ public void flush(ByteArrayOutputStream out) {}
+
+ @Override
+ public int getOneItemMaxSize() {
+ switch (dataType) {
+ case BOOLEAN:
+ return 1;
+ case INT32:
+ return 4;
+ case INT64:
+ return 8;
+ case FLOAT:
+ return 4;
+ case DOUBLE:
+ return 8;
+ case TEXT:
+ // refer to encode(Binary,ByteArrayOutputStream)
+ return 4 + TSFileConfig.BYTE_SIZE_PER_CHAR * maxStringLength;
+ case ENUMS:
+ return 4;
+ case BIGDECIMAL:
+ return 8;
+ default:
+ throw new UnsupportedOperationException(dataType.toString());
}
+ }
+
+ @Override
+ public long getMaxByteSize() {
+ return 0;
+ }
+
+ @Override
+ public void encode(BigDecimal value, ByteArrayOutputStream out) throws IOException {
+ throw new TSFileEncodingException(
+ "tsfile-encoding PlainEncoder: current version does not support BigDecimal value encoding");
+ }
}
diff --git a/src/main/java/cn/edu/tsinghua/tsfile/encoding/encoder/RleEncoder.java b/src/main/java/cn/edu/tsinghua/tsfile/encoding/encoder/RleEncoder.java
index 5c79a239..c3a7d8d1 100644
--- a/src/main/java/cn/edu/tsinghua/tsfile/encoding/encoder/RleEncoder.java
+++ b/src/main/java/cn/edu/tsinghua/tsfile/encoding/encoder/RleEncoder.java
@@ -1,312 +1,315 @@
-package cn.edu.tsinghua.tsfile.encoding.encoder;
-
-import cn.edu.tsinghua.tsfile.common.conf.TSFileConfig;
-import cn.edu.tsinghua.tsfile.common.conf.TSFileDescriptor;
-import cn.edu.tsinghua.tsfile.common.exception.TSFileEncodingException;
-import cn.edu.tsinghua.tsfile.common.utils.Binary;
-import cn.edu.tsinghua.tsfile.common.utils.ReadWriteStreamUtils;
-import cn.edu.tsinghua.tsfile.encoding.common.EndianType;
-import cn.edu.tsinghua.tsfile.file.metadata.enums.TSEncoding;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.math.BigDecimal;
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Encodes values using a combination of run length encoding and bit packing,
- * according to the following grammar:
- *
- * {@code
- * rle-bit-packing-hybrid:
- * length := length of the in bytes stored as 4 bytes little endian
- * bitwidth := bitwidth for all encoded data in
- * encoded-data := *
- * run := |
- * bit-packed-run :=
- * bit-packed-header := varint-encode( << 1 | 1)
- * lastBitPackedNum := the number of useful value in last bit-pack may be less than 8, so lastBitPackedNum indicates how many values are useful
- * bit-packed-values := bit packed
- * rle-run :=
- * rle-header := varint-encode( (number of times repeated) << 1)
- * repeated-value := value that is repeated, using a fixed-width of round-up-to-next-byte(bit-width)
- * }
- *
- *
- * @param data type T for RLE
- */
-public abstract class RleEncoder> extends Encoder {
- private static final Logger LOGGER = LoggerFactory.getLogger(RleEncoder.class);
- public EndianType endianType;
-
- /**
- * we save all value in a list and calculate its bitwidth
- */
- protected List values;
-
- /**
- * the bit width used for bit-packing and rle
- */
- protected int bitWidth;
-
- /**
- * for a given value now buffered, how many times it occurs
- */
- protected int repeatCount;
-
- /**
- * the number of group which using bit packing, it is saved in header
- */
- protected int bitPackedGroupCount;
-
- /**
- * the number of buffered value in array
- */
- protected int numBufferedValues;
-
- /**
- * we will write all bytes using bit-packing to OutputStream once. Before that, all bytes are
- * saved in list
- */
- protected List bytesBuffer;
-
- /**
- * flag which indicate encoding mode false -- rle true -- bit-packing
- */
- protected boolean isBitPackRun;
-
- /**
- * previous value written, used to detect repeated values
- */
- protected T preValue;
-
- /**
- * array to buffer values temporarily
- */
- protected T[] bufferedValues;
-
- protected boolean isBitWidthSaved;
-
- /**
- * output stream to buffer {@code }
- */
- protected ByteArrayOutputStream byteCache;
-
- protected TSFileConfig config = TSFileDescriptor.getInstance().getConfig();
-
- public RleEncoder(EndianType endianType) {
- super(TSEncoding.RLE);
- this.endianType = endianType;
- bytesBuffer = new ArrayList();
- isBitPackRun = false;
- isBitWidthSaved = false;
- byteCache = new ByteArrayOutputStream();
- }
-
- protected void reset() {
- numBufferedValues = 0;
- repeatCount = 0;
- bitPackedGroupCount = 0;
- bytesBuffer.clear();
- isBitPackRun = false;
- isBitWidthSaved = false;
- byteCache.reset();
- values.clear();
- }
-
- /**
- * Write all values buffered in cache to OutputStream
- *
- * @param out - byteArrayOutputStream
- * @throws IOException cannot flush to OutputStream
- */
- @Override
- public void flush(ByteArrayOutputStream out) throws IOException {
- int lastBitPackedNum = numBufferedValues;
- if (repeatCount >= config.RLE_MIN_REPEATED_NUM) {
- try {
- writeRleRun();
- } catch (IOException e) {
- LOGGER.error(
- "tsfile-encoding RleEncoder : error occurs when writing nums to OutputStram when flushing left nums. "
- + "numBufferedValues {}, repeatCount {}, bitPackedGroupCount{}, isBitPackRun {}, isBitWidthSaved {}",
- numBufferedValues, repeatCount, bitPackedGroupCount, isBitPackRun, isBitWidthSaved, e);
- throw e;
- }
- } else if (numBufferedValues > 0) {
- clearBuffer();
- writeOrAppendBitPackedRun();
- endPreviousBitPackedRun(lastBitPackedNum);
- } else {
- endPreviousBitPackedRun(config.RLE_MIN_REPEATED_NUM);
- }
- //write length
- ReadWriteStreamUtils.writeUnsignedVarInt(byteCache.size(), out);
- out.write(byteCache.toByteArray());
- reset();
- }
-
- /**
- * Write bytes to OutputStream using rle.
- * rle format: {@code
- * [header][value]
- * header: (repeated value) << 1}
- * @throws IOException cannot write RLE run
- */
- protected abstract void writeRleRun() throws IOException;
-
- /**
- * Start a bit-packing run transform values to bytes and buffer them in cache
- */
- public void writeOrAppendBitPackedRun() {
- if (bitPackedGroupCount >= config.RLE_MAX_BIT_PACKED_NUM) {
- // we've packed as many values as we can for this run,
- // end it and start a new one
- endPreviousBitPackedRun(config.RLE_MIN_REPEATED_NUM);
- }
- if (!isBitPackRun) {
- isBitPackRun = true;
- }
-
- convertBuffer();
-
- numBufferedValues = 0;
- repeatCount = 0;
- ++bitPackedGroupCount;
- }
-
- /**
- * End a bit-packing run write all bit-packing group to OutputStream bit-packing format:
- * {@code
- * [header][lastBitPackedNum][bit-packing group]+
- * [bit-packing group]+ are saved in List bytesBuffer
- * }
- * @param lastBitPackedNum - in last bit-packing group, it may have useful values less than 8.
- * This param indicates how many values are useful
- */
- protected void endPreviousBitPackedRun(int lastBitPackedNum) {
- if (!isBitPackRun) {
- return;
- }
- byte bitPackHeader = (byte) ((bitPackedGroupCount << 1) | 1);
- byteCache.write(bitPackHeader);
- byteCache.write(lastBitPackedNum);
- for (byte[] bytes : bytesBuffer) {
- byteCache.write(bytes, 0, bytes.length);
- }
- bytesBuffer.clear();
- isBitPackRun = false;
- bitPackedGroupCount = 0;
- }
-
- /**
- * Encode T value using rle or bit-packing.
- * It may not write to OutputStream immediately
- *
- * @param value - value to encode
- */
- protected void encodeValue(T value) {
- if (!isBitWidthSaved) {
- // save bit width in header,
- // perpare for read
- byteCache.write(bitWidth);
- isBitWidthSaved = true;
- }
- if (value.equals(preValue)) {
- repeatCount++;
- if (repeatCount >= config.RLE_MIN_REPEATED_NUM && repeatCount <= config.RLE_MAX_REPEATED_NUM) {
- // value occurs more than RLE_MIN_REPEATED_NUM times but less than EncodingConfig.RLE_MAX_REPEATED_NUM
- // we'll use rle, so just keep on counting repeats for now
- // we'll write current value to OutputStream when we encounter a different value
- return;
- } else if (repeatCount == config.RLE_MAX_REPEATED_NUM + 1) {
- // value occurs more than EncodingConfig.RLE_MAX_REPEATED_NUM
- // we'll write current rle run to stream and keep on counting current value
- repeatCount = config.RLE_MAX_REPEATED_NUM;
- try {
- writeRleRun();
- LOGGER.debug("tsfile-encoding RleEncoder : write full rle run to stream");
- } catch (IOException e) {
- LOGGER.error(
- " error occurs when writing full rle run to OutputStram when repeatCount = {}."
- + "numBufferedValues {}, repeatCount {}, bitPackedGroupCount{}, isBitPackRun {}, isBitWidthSaved {}",
- config.RLE_MAX_REPEATED_NUM + 1, numBufferedValues, repeatCount, bitPackedGroupCount, isBitPackRun, isBitWidthSaved, e);
- }
- repeatCount = 1;
- preValue = value;
- }
-
- } else {
- // we encounter a differnt value
- if (repeatCount >= config.RLE_MIN_REPEATED_NUM) {
- try {
- writeRleRun();
- } catch (IOException e) {
- LOGGER.error(
- "tsfile-encoding RleEncoder : error occurs when writing num to OutputStram when repeatCount > {}."
- + "numBufferedValues {}, repeatCount {}, bitPackedGroupCount{}, isBitPackRun {}, isBitWidthSaved {}",
- config.RLE_MIN_REPEATED_NUM, numBufferedValues, repeatCount, bitPackedGroupCount, isBitPackRun, isBitWidthSaved, e);
- }
- }
- repeatCount = 1;
- preValue = value;
- }
- bufferedValues[numBufferedValues] = value;
- numBufferedValues++;
- // if none of value we encountered occurs more MAX_REPEATED_NUM times
- // we'll use bit-packing
- if (numBufferedValues == config.RLE_MIN_REPEATED_NUM) {
- writeOrAppendBitPackedRun();
- }
- }
-
- /**
- * clean all useless value in bufferedValues and set 0
- */
- protected abstract void clearBuffer();
-
- protected abstract void convertBuffer();
-
- @Override
- public void encode(boolean value, ByteArrayOutputStream out) {
- throw new TSFileEncodingException(getClass().getName());
- }
-
- @Override
- public void encode(short value, ByteArrayOutputStream out) {
- throw new TSFileEncodingException(getClass().getName());
- }
-
- @Override
- public void encode(int value, ByteArrayOutputStream out) {
- throw new TSFileEncodingException(getClass().getName());
- }
-
- @Override
- public void encode(long value, ByteArrayOutputStream out) {
- throw new TSFileEncodingException(getClass().getName());
- }
-
- @Override
- public void encode(float value, ByteArrayOutputStream out) {
- throw new TSFileEncodingException(getClass().getName());
- }
-
- @Override
- public void encode(double value, ByteArrayOutputStream out) {
- throw new TSFileEncodingException(getClass().getName());
- }
-
- @Override
- public void encode(Binary value, ByteArrayOutputStream out) {
- throw new TSFileEncodingException(getClass().getName());
- }
-
- @Override
- public void encode(BigDecimal value, ByteArrayOutputStream out) {
- throw new TSFileEncodingException(getClass().getName());
- }
-}
+package cn.edu.tsinghua.tsfile.encoding.encoder;
+
+import cn.edu.tsinghua.tsfile.common.conf.TSFileConfig;
+import cn.edu.tsinghua.tsfile.common.conf.TSFileDescriptor;
+import cn.edu.tsinghua.tsfile.common.exception.TSFileEncodingException;
+import cn.edu.tsinghua.tsfile.common.utils.Binary;
+import cn.edu.tsinghua.tsfile.common.utils.ReadWriteStreamUtils;
+import cn.edu.tsinghua.tsfile.encoding.common.EndianType;
+import cn.edu.tsinghua.tsfile.file.metadata.enums.TSEncoding;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Encodes values using a combination of run length encoding and bit packing, according to the
+ * following grammar:
+ *
+ *
+ * {@code
+ * rle-bit-packing-hybrid:
+ * length := length of the in bytes stored as 4 bytes little endian
+ * bitwidth := bitwidth for all encoded data in
+ * encoded-data := *
+ * run := |
+ * bit-packed-run :=
+ * bit-packed-header := varint-encode( << 1 | 1)
+ * lastBitPackedNum := the number of useful value in last bit-pack may be less than 8, so lastBitPackedNum indicates how many values are useful
+ * bit-packed-values := bit packed
+ * rle-run :=
+ * rle-header := varint-encode( (number of times repeated) << 1)
+ * repeated-value := value that is repeated, using a fixed-width of round-up-to-next-byte(bit-width)
+ * }
+ *
+ *
+ * @param data type T for RLE
+ */
+public abstract class RleEncoder> extends Encoder {
+ private static final Logger LOGGER = LoggerFactory.getLogger(RleEncoder.class);
+ public EndianType endianType;
+
+ /**
+ * we save all value in a list and calculate its bitwidth
+ */
+ protected List values;
+
+ /**
+ * the bit width used for bit-packing and rle
+ */
+ protected int bitWidth;
+
+ /**
+ * for a given value now buffered, how many times it occurs
+ */
+ protected int repeatCount;
+
+ /**
+ * the number of group which using bit packing, it is saved in header
+ */
+ protected int bitPackedGroupCount;
+
+ /**
+ * the number of buffered value in array
+ */
+ protected int numBufferedValues;
+
+ /**
+ * we will write all bytes using bit-packing to OutputStream once. Before that, all bytes are
+ * saved in list
+ */
+ protected List bytesBuffer;
+
+ /**
+ * flag which indicate encoding mode false -- rle true -- bit-packing
+ */
+ protected boolean isBitPackRun;
+
+ /**
+ * previous value written, used to detect repeated values
+ */
+ protected T preValue;
+
+ /**
+ * array to buffer values temporarily
+ */
+ protected T[] bufferedValues;
+
+ protected boolean isBitWidthSaved;
+
+ /**
+ * output stream to buffer {@code }
+ */
+ protected ByteArrayOutputStream byteCache;
+
+ protected TSFileConfig config = TSFileDescriptor.getInstance().getConfig();
+
+ public RleEncoder(EndianType endianType) {
+ super(TSEncoding.RLE);
+ this.endianType = endianType;
+ bytesBuffer = new ArrayList();
+ isBitPackRun = false;
+ isBitWidthSaved = false;
+ byteCache = new ByteArrayOutputStream();
+ }
+
+ protected void reset() {
+ numBufferedValues = 0;
+ repeatCount = 0;
+ bitPackedGroupCount = 0;
+ bytesBuffer.clear();
+ isBitPackRun = false;
+ isBitWidthSaved = false;
+ byteCache.reset();
+ values.clear();
+ }
+
+ /**
+ * Write all values buffered in cache to OutputStream
+ *
+ * @param out - byteArrayOutputStream
+ * @throws IOException cannot flush to OutputStream
+ */
+ @Override
+ public void flush(ByteArrayOutputStream out) throws IOException {
+ int lastBitPackedNum = numBufferedValues;
+ if (repeatCount >= config.RLE_MIN_REPEATED_NUM) {
+ try {
+ writeRleRun();
+ } catch (IOException e) {
+ LOGGER.error(
+ "tsfile-encoding RleEncoder : error occurs when writing nums to OutputStram when flushing left nums. "
+ + "numBufferedValues {}, repeatCount {}, bitPackedGroupCount{}, isBitPackRun {}, isBitWidthSaved {}",
+ numBufferedValues, repeatCount, bitPackedGroupCount, isBitPackRun, isBitWidthSaved, e);
+ throw e;
+ }
+ } else if (numBufferedValues > 0) {
+ clearBuffer();
+ writeOrAppendBitPackedRun();
+ endPreviousBitPackedRun(lastBitPackedNum);
+ } else {
+ endPreviousBitPackedRun(config.RLE_MIN_REPEATED_NUM);
+ }
+ // write length
+ ReadWriteStreamUtils.writeUnsignedVarInt(byteCache.size(), out);
+ out.write(byteCache.toByteArray());
+ reset();
+ }
+
+ /**
+ * Write bytes to OutputStream using rle. rle format: {@code
+ * [header][value]
+ * header: (repeated value) << 1}
+ *
+ * @throws IOException cannot write RLE run
+ */
+ protected abstract void writeRleRun() throws IOException;
+
+ /**
+ * Start a bit-packing run transform values to bytes and buffer them in cache
+ */
+ public void writeOrAppendBitPackedRun() {
+ if (bitPackedGroupCount >= config.RLE_MAX_BIT_PACKED_NUM) {
+ // we've packed as many values as we can for this run,
+ // end it and start a new one
+ endPreviousBitPackedRun(config.RLE_MIN_REPEATED_NUM);
+ }
+ if (!isBitPackRun) {
+ isBitPackRun = true;
+ }
+
+ convertBuffer();
+
+ numBufferedValues = 0;
+ repeatCount = 0;
+ ++bitPackedGroupCount;
+ }
+
+ /**
+ * End a bit-packing run write all bit-packing group to OutputStream bit-packing format: {@code
+ * [header][lastBitPackedNum][bit-packing group]+
+ * [bit-packing group]+ are saved in List bytesBuffer
+ * }
+ *
+ * @param lastBitPackedNum - in last bit-packing group, it may have useful values less than 8.
+ * This param indicates how many values are useful
+ */
+ protected void endPreviousBitPackedRun(int lastBitPackedNum) {
+ if (!isBitPackRun) {
+ return;
+ }
+ byte bitPackHeader = (byte) ((bitPackedGroupCount << 1) | 1);
+ byteCache.write(bitPackHeader);
+ byteCache.write(lastBitPackedNum);
+ for (byte[] bytes : bytesBuffer) {
+ byteCache.write(bytes, 0, bytes.length);
+ }
+ bytesBuffer.clear();
+ isBitPackRun = false;
+ bitPackedGroupCount = 0;
+ }
+
+ /**
+ * Encode T value using rle or bit-packing. It may not write to OutputStream immediately
+ *
+ * @param value - value to encode
+ */
+ protected void encodeValue(T value) {
+ if (!isBitWidthSaved) {
+ // save bit width in header,
+ // perpare for read
+ byteCache.write(bitWidth);
+ isBitWidthSaved = true;
+ }
+ if (value.equals(preValue)) {
+ repeatCount++;
+ if (repeatCount >= config.RLE_MIN_REPEATED_NUM
+ && repeatCount <= config.RLE_MAX_REPEATED_NUM) {
+ // value occurs more than RLE_MIN_REPEATED_NUM times but less than
+ // EncodingConfig.RLE_MAX_REPEATED_NUM
+ // we'll use rle, so just keep on counting repeats for now
+ // we'll write current value to OutputStream when we encounter a different value
+ return;
+ } else if (repeatCount == config.RLE_MAX_REPEATED_NUM + 1) {
+ // value occurs more than EncodingConfig.RLE_MAX_REPEATED_NUM
+ // we'll write current rle run to stream and keep on counting current value
+ repeatCount = config.RLE_MAX_REPEATED_NUM;
+ try {
+ writeRleRun();
+ LOGGER.debug("tsfile-encoding RleEncoder : write full rle run to stream");
+ } catch (IOException e) {
+ LOGGER.error(
+ " error occurs when writing full rle run to OutputStram when repeatCount = {}."
+ + "numBufferedValues {}, repeatCount {}, bitPackedGroupCount{}, isBitPackRun {}, isBitWidthSaved {}",
+ config.RLE_MAX_REPEATED_NUM + 1, numBufferedValues, repeatCount, bitPackedGroupCount,
+ isBitPackRun, isBitWidthSaved, e);
+ }
+ repeatCount = 1;
+ preValue = value;
+ }
+
+ } else {
+ // we encounter a differnt value
+ if (repeatCount >= config.RLE_MIN_REPEATED_NUM) {
+ try {
+ writeRleRun();
+ } catch (IOException e) {
+ LOGGER.error(
+ "tsfile-encoding RleEncoder : error occurs when writing num to OutputStram when repeatCount > {}."
+ + "numBufferedValues {}, repeatCount {}, bitPackedGroupCount{}, isBitPackRun {}, isBitWidthSaved {}",
+ config.RLE_MIN_REPEATED_NUM, numBufferedValues, repeatCount, bitPackedGroupCount,
+ isBitPackRun, isBitWidthSaved, e);
+ }
+ }
+ repeatCount = 1;
+ preValue = value;
+ }
+ bufferedValues[numBufferedValues] = value;
+ numBufferedValues++;
+ // if none of value we encountered occurs more MAX_REPEATED_NUM times
+ // we'll use bit-packing
+ if (numBufferedValues == config.RLE_MIN_REPEATED_NUM) {
+ writeOrAppendBitPackedRun();
+ }
+ }
+
+ /**
+ * clean all useless value in bufferedValues and set 0
+ */
+ protected abstract void clearBuffer();
+
+ protected abstract void convertBuffer();
+
+ @Override
+ public void encode(boolean value, ByteArrayOutputStream out) {
+ throw new TSFileEncodingException(getClass().getName());
+ }
+
+ @Override
+ public void encode(short value, ByteArrayOutputStream out) {
+ throw new TSFileEncodingException(getClass().getName());
+ }
+
+ @Override
+ public void encode(int value, ByteArrayOutputStream out) {
+ throw new TSFileEncodingException(getClass().getName());
+ }
+
+ @Override
+ public void encode(long value, ByteArrayOutputStream out) {
+ throw new TSFileEncodingException(getClass().getName());
+ }
+
+ @Override
+ public void encode(float value, ByteArrayOutputStream out) {
+ throw new TSFileEncodingException(getClass().getName());
+ }
+
+ @Override
+ public void encode(double value, ByteArrayOutputStream out) {
+ throw new TSFileEncodingException(getClass().getName());
+ }
+
+ @Override
+ public void encode(Binary value, ByteArrayOutputStream out) {
+ throw new TSFileEncodingException(getClass().getName());
+ }
+
+ @Override
+ public void encode(BigDecimal value, ByteArrayOutputStream out) {
+ throw new TSFileEncodingException(getClass().getName());
+ }
+}
diff --git a/src/main/java/cn/edu/tsinghua/tsfile/encoding/encoder/SinglePrecisionEncoder.java b/src/main/java/cn/edu/tsinghua/tsfile/encoding/encoder/SinglePrecisionEncoder.java
index c542d82e..53003fe6 100644
--- a/src/main/java/cn/edu/tsinghua/tsfile/encoding/encoder/SinglePrecisionEncoder.java
+++ b/src/main/java/cn/edu/tsinghua/tsfile/encoding/encoder/SinglePrecisionEncoder.java
@@ -2,83 +2,84 @@
import java.io.ByteArrayOutputStream;
import java.io.IOException;
-
import cn.edu.tsinghua.tsfile.common.conf.TSFileConfig;
/**
* Encoder for int value using gorilla encoding
*
*/
-public class SinglePrecisionEncoder extends GorillaEncoder{
- private int preValue;
+public class SinglePrecisionEncoder extends GorillaEncoder {
+ private int preValue;
+
+ public SinglePrecisionEncoder() {}
- public SinglePrecisionEncoder() {
- }
-
- @Override
- public void encode(float value, ByteArrayOutputStream out) throws IOException {
- if(!flag){
- flag = true;
- preValue = Float.floatToIntBits(value);
- leadingZeroNum = Integer.numberOfLeadingZeros(preValue);
- tailingZeroNum = Integer.numberOfTrailingZeros(preValue);
- out.write((preValue >> 0) & 0xFF);
- out.write((preValue >> 8) & 0xFF);
- out.write((preValue >> 16) & 0xFF);
- out.write((preValue >> 24) & 0xFF);
- } else{
- int nextValue = Float.floatToIntBits(value);
- int tmp = nextValue ^ preValue;
- if(tmp == 0){
- // case: write '0'
- writeBit(false, out);
- } else{
- int leadingZeroNumTmp = Integer.numberOfLeadingZeros(tmp);
- int tailingZeroNumTmp = Integer.numberOfTrailingZeros(tmp);
- if(leadingZeroNumTmp >= leadingZeroNum && tailingZeroNumTmp >= tailingZeroNum){
- // case: write '10' and effective bits without first leadingZeroNum '0' and last tailingZeroNum '0'
- writeBit(true, out);
- writeBit(false, out);
- writeBits(tmp, out, TSFileConfig.FLOAT_LENGTH - 1 - leadingZeroNum, tailingZeroNum);
- } else{
- // case: write '11', leading zero num of value, effective bits len and effective bit value
- writeBit(true, out);
- writeBit(true, out);
- writeBits(leadingZeroNumTmp, out, TSFileConfig.FLAOT_LEADING_ZERO_LENGTH - 1, 0);
- writeBits(TSFileConfig.FLOAT_LENGTH - leadingZeroNumTmp - tailingZeroNumTmp, out, TSFileConfig.FLOAT_VALUE_LENGTH - 1, 0);
- writeBits(tmp, out, TSFileConfig.FLOAT_LENGTH - 1 - leadingZeroNumTmp, tailingZeroNumTmp);
- }
- }
- preValue = nextValue;
- leadingZeroNum = Integer.numberOfLeadingZeros(preValue);
- tailingZeroNum = Integer.numberOfTrailingZeros(preValue);
+ @Override
+ public void encode(float value, ByteArrayOutputStream out) throws IOException {
+ if (!flag) {
+ flag = true;
+ preValue = Float.floatToIntBits(value);
+ leadingZeroNum = Integer.numberOfLeadingZeros(preValue);
+ tailingZeroNum = Integer.numberOfTrailingZeros(preValue);
+ out.write((preValue >> 0) & 0xFF);
+ out.write((preValue >> 8) & 0xFF);
+ out.write((preValue >> 16) & 0xFF);
+ out.write((preValue >> 24) & 0xFF);
+ } else {
+ int nextValue = Float.floatToIntBits(value);
+ int tmp = nextValue ^ preValue;
+ if (tmp == 0) {
+ // case: write '0'
+ writeBit(false, out);
+ } else {
+ int leadingZeroNumTmp = Integer.numberOfLeadingZeros(tmp);
+ int tailingZeroNumTmp = Integer.numberOfTrailingZeros(tmp);
+ if (leadingZeroNumTmp >= leadingZeroNum && tailingZeroNumTmp >= tailingZeroNum) {
+ // case: write '10' and effective bits without first leadingZeroNum '0' and last
+ // tailingZeroNum '0'
+ writeBit(true, out);
+ writeBit(false, out);
+ writeBits(tmp, out, TSFileConfig.FLOAT_LENGTH - 1 - leadingZeroNum, tailingZeroNum);
+ } else {
+ // case: write '11', leading zero num of value, effective bits len and effective bit value
+ writeBit(true, out);
+ writeBit(true, out);
+ writeBits(leadingZeroNumTmp, out, TSFileConfig.FLAOT_LEADING_ZERO_LENGTH - 1, 0);
+ writeBits(TSFileConfig.FLOAT_LENGTH - leadingZeroNumTmp - tailingZeroNumTmp, out,
+ TSFileConfig.FLOAT_VALUE_LENGTH - 1, 0);
+ writeBits(tmp, out, TSFileConfig.FLOAT_LENGTH - 1 - leadingZeroNumTmp, tailingZeroNumTmp);
}
+ }
+ preValue = nextValue;
+ leadingZeroNum = Integer.numberOfLeadingZeros(preValue);
+ tailingZeroNum = Integer.numberOfTrailingZeros(preValue);
}
-
- @Override
- public void flush(ByteArrayOutputStream out) throws IOException {
- encode(Float.NaN, out);
- clearBuffer(out);
- reset();
- }
-
- private void writeBits(int num, ByteArrayOutputStream out, int start, int end){
- for(int i = start; i >= end; i--){
- int bit = num & (1 << i);
- writeBit(bit, out);
- }
- }
-
- @Override
- public int getOneItemMaxSize() {
- // case '11'
- // 2bit + 5bit + 6bit + 32bit = 45bit
- return 6;
- }
+ }
- @Override
- public long getMaxByteSize() {
- // max(first 4 byte, case '11' bit + 5bit + 6bit + 32bit = 45bit) + NaN(case '11' bit + 5bit + 6bit + 32bit = 45bit) = 90bit
- return 12;
+ @Override
+ public void flush(ByteArrayOutputStream out) throws IOException {
+ encode(Float.NaN, out);
+ clearBuffer(out);
+ reset();
+ }
+
+ private void writeBits(int num, ByteArrayOutputStream out, int start, int end) {
+ for (int i = start; i >= end; i--) {
+ int bit = num & (1 << i);
+ writeBit(bit, out);
}
+ }
+
+ @Override
+ public int getOneItemMaxSize() {
+ // case '11'
+ // 2bit + 5bit + 6bit + 32bit = 45bit
+ return 6;
+ }
+
+ @Override
+ public long getMaxByteSize() {
+ // max(first 4 byte, case '11' bit + 5bit + 6bit + 32bit = 45bit) + NaN(case '11' bit + 5bit +
+ // 6bit + 32bit = 45bit) = 90bit
+ return 12;
+ }
}
diff --git a/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/RowGroupMetaData.java b/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/RowGroupMetaData.java
index 3fa078cc..af34b3fb 100644
--- a/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/RowGroupMetaData.java
+++ b/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/RowGroupMetaData.java
@@ -1,195 +1,197 @@
-package cn.edu.tsinghua.tsfile.file.metadata;
-
-import cn.edu.tsinghua.tsfile.file.metadata.converter.IConverter;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-
-/**
- * For more information, see RowGroupMetaData in cn.edu.thu.tsfile.format package
- */
-public class RowGroupMetaData implements IConverter {
- private static final Logger LOGGER = LoggerFactory.getLogger(RowGroupMetaData.class);
-
- private String deltaObjectID;
-
- /**
- * Number of rows in this row group
- */
- private long numOfRows;
-
- /**
- * Total byte size of all the uncompressed time series data in this row group
- */
- private long totalByteSize;
-
- /**
- * This path is relative to the current file.
- */
- private String path;
-
- private List timeSeriesChunkMetaDataList;
-
- /**
- * which schema/group does the delta object belongs to
- */
- private String deltaObjectType;
-
- /**
- * The time when endRowgroup() is called.
- */
- private long writtenTime;
-
- public RowGroupMetaData() {
- timeSeriesChunkMetaDataList = new ArrayList();
- }
-
- public RowGroupMetaData(String deltaObjectID, long numOfRows, long totalByteSize,
- List timeSeriesChunkMetaDataList, String deltaObjectType) {
- this.deltaObjectID = deltaObjectID;
- this.numOfRows = numOfRows;
- this.totalByteSize = totalByteSize;
- this.timeSeriesChunkMetaDataList = timeSeriesChunkMetaDataList;
- this.deltaObjectType = deltaObjectType;
- }
-
- /**
- * add time series chunk metadata to list. THREAD NOT SAFE
- *
- * @param metadata time series metadata to add
- */
- public void addTimeSeriesChunkMetaData(TimeSeriesChunkMetaData metadata) {
- if (timeSeriesChunkMetaDataList == null) {
- timeSeriesChunkMetaDataList = new ArrayList();
- }
- timeSeriesChunkMetaDataList.add(metadata);
- }
-
- public List getMetaDatas() {
- return timeSeriesChunkMetaDataList == null ? null
- : Collections.unmodifiableList(timeSeriesChunkMetaDataList);
- }
-
- @Override
- public cn.edu.tsinghua.tsfile.format.RowGroupMetaData convertToThrift() {
- try {
- List timeSeriesChunkMetaDataListInThrift = null;
- if (timeSeriesChunkMetaDataList != null) {
- timeSeriesChunkMetaDataListInThrift = new ArrayList<>();
- for (TimeSeriesChunkMetaData timeSeriesChunkMetaData : timeSeriesChunkMetaDataList) {
- timeSeriesChunkMetaDataListInThrift.add(timeSeriesChunkMetaData.convertToThrift());
- }
- }
- cn.edu.tsinghua.tsfile.format.RowGroupMetaData metaDataInThrift =
- new cn.edu.tsinghua.tsfile.format.RowGroupMetaData(timeSeriesChunkMetaDataListInThrift,
- deltaObjectID, totalByteSize, numOfRows, deltaObjectType, writtenTime);
- metaDataInThrift.setFile_path(path);
- return metaDataInThrift;
- } catch (Exception e) {
- if (LOGGER.isErrorEnabled())
- LOGGER.error(
- "tsfile-file RowGroupMetaData: failed to convert row group metadata from TSFile to thrift, row group metadata:{}",
- this, e);
- throw e;
- }
- }
-
- @Override
- public void convertToTSF(cn.edu.tsinghua.tsfile.format.RowGroupMetaData metaDataInThrift) {
- try {
- deltaObjectID = metaDataInThrift.getDelta_object_id();
- numOfRows = metaDataInThrift.getMax_num_rows();
- totalByteSize = metaDataInThrift.getTotal_byte_size();
- path = metaDataInThrift.getFile_path();
- deltaObjectType = metaDataInThrift.getDelta_object_type();
- writtenTime = metaDataInThrift.getWrittenTime();
- List timeSeriesChunkMetaDataListInThrift = metaDataInThrift.getTsc_metadata();
- if (timeSeriesChunkMetaDataListInThrift == null) {
- timeSeriesChunkMetaDataList = null;
- } else {
- if (timeSeriesChunkMetaDataList == null) {
- timeSeriesChunkMetaDataList = new ArrayList<>();
- }
- timeSeriesChunkMetaDataList.clear();
- for (cn.edu.tsinghua.tsfile.format.TimeSeriesChunkMetaData timeSeriesChunkMetaDataInThrift : timeSeriesChunkMetaDataListInThrift) {
- TimeSeriesChunkMetaData timeSeriesChunkMetaData = new TimeSeriesChunkMetaData();
- timeSeriesChunkMetaData.convertToTSF(timeSeriesChunkMetaDataInThrift);
- timeSeriesChunkMetaDataList.add(timeSeriesChunkMetaData);
- }
- }
- } catch (Exception e) {
- if (LOGGER.isErrorEnabled())
- LOGGER.error(
- "tsfile-file RowGroupMetaData: failed to convert row group metadata from thrift to TSFile, row group metadata:{}",
- metaDataInThrift, e);
- throw e;
- }
- }
-
- @Override
- public String toString() {
- return String.format(
- "RowGroupMetaData{ delta object id: %s, number of rows: %d, total byte size: %d, time series chunk list: %s }",
- deltaObjectID, numOfRows, totalByteSize, timeSeriesChunkMetaDataList);
- }
-
- public long getNumOfRows() {
- return numOfRows;
- }
-
- public void setNumOfRows(long numOfRows) {
- this.numOfRows = numOfRows;
- }
-
- public long getTotalByteSize() {
- return totalByteSize;
- }
-
- public void setTotalByteSize(long totalByteSize) {
- this.totalByteSize = totalByteSize;
- }
-
- public String getPath() {
- return path;
- }
-
- public void setPath(String path) {
- this.path = path;
- }
-
- public String getDeltaObjectID() {
- return deltaObjectID;
- }
-
- public void setDeltaObjectID(String deltaObjectUID) {
- this.deltaObjectID = deltaObjectUID;
- }
-
- public List getTimeSeriesChunkMetaDataList() {
- return timeSeriesChunkMetaDataList;
- }
-
- public void setTimeSeriesChunkMetaDataList(
- List timeSeriesChunkMetaDataList) {
- this.timeSeriesChunkMetaDataList = timeSeriesChunkMetaDataList;
- }
-
- public String getDeltaObjectType() {
- return deltaObjectType;
- }
-
- public void setDeltaObjectType(String deltaObjectType) {
- this.deltaObjectType = deltaObjectType;
- }
-
- public long getWrittenTime() {
- return writtenTime;
- }
-
- public void setWrittenTime(long writtenTime) {
- this.writtenTime = writtenTime;
- }
-}
+package cn.edu.tsinghua.tsfile.file.metadata;
+
+import cn.edu.tsinghua.tsfile.file.metadata.converter.IConverter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * For more information, see RowGroupMetaData in cn.edu.thu.tsfile.format package
+ */
+public class RowGroupMetaData
+ implements IConverter {
+ private static final Logger LOGGER = LoggerFactory.getLogger(RowGroupMetaData.class);
+
+ private String deltaObjectID;
+
+ /**
+ * Number of rows in this row group
+ */
+ private long numOfRows;
+
+ /**
+ * Total byte size of all the uncompressed time series data in this row group
+ */
+ private long totalByteSize;
+
+ /**
+ * This path is relative to the current file.
+ */
+ private String path;
+
+ private List timeSeriesChunkMetaDataList;
+
+ /**
+ * which schema/group does the delta object belongs to
+ */
+ private String deltaObjectType;
+
+ /**
+ * The time when endRowgroup() is called.
+ */
+ private long writtenTime;
+
+ public RowGroupMetaData() {
+ timeSeriesChunkMetaDataList = new ArrayList();
+ }
+
+ public RowGroupMetaData(String deltaObjectID, long numOfRows, long totalByteSize,
+ List timeSeriesChunkMetaDataList, String deltaObjectType) {
+ this.deltaObjectID = deltaObjectID;
+ this.numOfRows = numOfRows;
+ this.totalByteSize = totalByteSize;
+ this.timeSeriesChunkMetaDataList = timeSeriesChunkMetaDataList;
+ this.deltaObjectType = deltaObjectType;
+ }
+
+ /**
+ * add time series chunk metadata to list. THREAD NOT SAFE
+ *
+ * @param metadata time series metadata to add
+ */
+ public void addTimeSeriesChunkMetaData(TimeSeriesChunkMetaData metadata) {
+ if (timeSeriesChunkMetaDataList == null) {
+ timeSeriesChunkMetaDataList = new ArrayList();
+ }
+ timeSeriesChunkMetaDataList.add(metadata);
+ }
+
+ public List getMetaDatas() {
+ return timeSeriesChunkMetaDataList == null ? null
+ : Collections.unmodifiableList(timeSeriesChunkMetaDataList);
+ }
+
+ @Override
+ public cn.edu.tsinghua.tsfile.format.RowGroupMetaData convertToThrift() {
+ try {
+ List timeSeriesChunkMetaDataListInThrift =
+ null;
+ if (timeSeriesChunkMetaDataList != null) {
+ timeSeriesChunkMetaDataListInThrift = new ArrayList<>();
+ for (TimeSeriesChunkMetaData timeSeriesChunkMetaData : timeSeriesChunkMetaDataList) {
+ timeSeriesChunkMetaDataListInThrift.add(timeSeriesChunkMetaData.convertToThrift());
+ }
+ }
+ cn.edu.tsinghua.tsfile.format.RowGroupMetaData metaDataInThrift =
+ new cn.edu.tsinghua.tsfile.format.RowGroupMetaData(timeSeriesChunkMetaDataListInThrift,
+ deltaObjectID, totalByteSize, numOfRows, deltaObjectType, writtenTime);
+ metaDataInThrift.setFile_path(path);
+ return metaDataInThrift;
+ } catch (Exception e) {
+ if (LOGGER.isErrorEnabled())
+ LOGGER.error(
+ "tsfile-file RowGroupMetaData: failed to convert row group metadata from TSFile to thrift, row group metadata:{}",
+ this, e);
+ throw e;
+ }
+ }
+
+ @Override
+ public void convertToTSF(cn.edu.tsinghua.tsfile.format.RowGroupMetaData metaDataInThrift) {
+ try {
+ deltaObjectID = metaDataInThrift.getDelta_object_id();
+ numOfRows = metaDataInThrift.getMax_num_rows();
+ totalByteSize = metaDataInThrift.getTotal_byte_size();
+ path = metaDataInThrift.getFile_path();
+ deltaObjectType = metaDataInThrift.getDelta_object_type();
+ writtenTime = metaDataInThrift.getWrittenTime();
+ List timeSeriesChunkMetaDataListInThrift =
+ metaDataInThrift.getTsc_metadata();
+ if (timeSeriesChunkMetaDataListInThrift == null) {
+ timeSeriesChunkMetaDataList = null;
+ } else {
+ if (timeSeriesChunkMetaDataList == null) {
+ timeSeriesChunkMetaDataList = new ArrayList<>();
+ }
+ timeSeriesChunkMetaDataList.clear();
+ for (cn.edu.tsinghua.tsfile.format.TimeSeriesChunkMetaData timeSeriesChunkMetaDataInThrift : timeSeriesChunkMetaDataListInThrift) {
+ TimeSeriesChunkMetaData timeSeriesChunkMetaData = new TimeSeriesChunkMetaData();
+ timeSeriesChunkMetaData.convertToTSF(timeSeriesChunkMetaDataInThrift);
+ timeSeriesChunkMetaDataList.add(timeSeriesChunkMetaData);
+ }
+ }
+ } catch (Exception e) {
+ if (LOGGER.isErrorEnabled())
+ LOGGER.error(
+ "tsfile-file RowGroupMetaData: failed to convert row group metadata from thrift to TSFile, row group metadata:{}",
+ metaDataInThrift, e);
+ throw e;
+ }
+ }
+
+ @Override
+ public String toString() {
+ return String.format(
+ "RowGroupMetaData{ delta object id: %s, number of rows: %d, total byte size: %d, time series chunk list: %s }",
+ deltaObjectID, numOfRows, totalByteSize, timeSeriesChunkMetaDataList);
+ }
+
+ public long getNumOfRows() {
+ return numOfRows;
+ }
+
+ public void setNumOfRows(long numOfRows) {
+ this.numOfRows = numOfRows;
+ }
+
+ public long getTotalByteSize() {
+ return totalByteSize;
+ }
+
+ public void setTotalByteSize(long totalByteSize) {
+ this.totalByteSize = totalByteSize;
+ }
+
+ public String getPath() {
+ return path;
+ }
+
+ public void setPath(String path) {
+ this.path = path;
+ }
+
+ public String getDeltaObjectID() {
+ return deltaObjectID;
+ }
+
+ public void setDeltaObjectID(String deltaObjectUID) {
+ this.deltaObjectID = deltaObjectUID;
+ }
+
+ public List getTimeSeriesChunkMetaDataList() {
+ return timeSeriesChunkMetaDataList;
+ }
+
+ public void setTimeSeriesChunkMetaDataList(
+ List timeSeriesChunkMetaDataList) {
+ this.timeSeriesChunkMetaDataList = timeSeriesChunkMetaDataList;
+ }
+
+ public String getDeltaObjectType() {
+ return deltaObjectType;
+ }
+
+ public void setDeltaObjectType(String deltaObjectType) {
+ this.deltaObjectType = deltaObjectType;
+ }
+
+ public long getWrittenTime() {
+ return writtenTime;
+ }
+
+ public void setWrittenTime(long writtenTime) {
+ this.writtenTime = writtenTime;
+ }
+}
diff --git a/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/TInTimeSeriesChunkMetaData.java b/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/TInTimeSeriesChunkMetaData.java
index 732c304b..50c55001 100644
--- a/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/TInTimeSeriesChunkMetaData.java
+++ b/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/TInTimeSeriesChunkMetaData.java
@@ -1,134 +1,134 @@
-package cn.edu.tsinghua.tsfile.file.metadata;
-
-import cn.edu.tsinghua.tsfile.file.metadata.converter.IConverter;
-import cn.edu.tsinghua.tsfile.file.metadata.enums.TSDataType;
-import cn.edu.tsinghua.tsfile.file.metadata.enums.TSFreqType;
-import cn.edu.tsinghua.tsfile.format.DataType;
-import cn.edu.tsinghua.tsfile.format.FreqType;
-import cn.edu.tsinghua.tsfile.format.TimeInTimeSeriesChunkMetaData;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.List;
-
-/**
- * For more information, see TimeInTimeSeriesChunkMetaData
- * in cn.edu.thu.tsfile.format package
- */
-public class TInTimeSeriesChunkMetaData implements IConverter {
- private static final Logger LOGGER = LoggerFactory.getLogger(TInTimeSeriesChunkMetaData.class);
-
- private TSDataType dataType;
- private long startTime;
- private long endTime;
-
- private TSFreqType freqType;
- private List frequencies;
-
- /**
- * If values for data consist of enum values, metadata will store all possible values in time
- * series
- */
- private List enumValues;
-
- public TInTimeSeriesChunkMetaData() {
- }
-
- public TInTimeSeriesChunkMetaData(TSDataType dataType, long startTime, long endTime) {
- this.dataType = dataType;
- this.startTime = startTime;
- this.endTime = endTime;
- }
-
- @Override
- public TimeInTimeSeriesChunkMetaData convertToThrift() {
- try {
- TimeInTimeSeriesChunkMetaData tTimeSeriesChunkMetaDataInThrift =
- new TimeInTimeSeriesChunkMetaData(
- dataType == null ? null : DataType.valueOf(dataType.toString()), startTime, endTime);
- tTimeSeriesChunkMetaDataInThrift.setFreq_type(freqType == null ? null : FreqType.valueOf(freqType.toString()));
- tTimeSeriesChunkMetaDataInThrift.setFrequencies(frequencies);
- tTimeSeriesChunkMetaDataInThrift.setEnum_values(enumValues);
- return tTimeSeriesChunkMetaDataInThrift;
- } catch (Exception e) {
- if (LOGGER.isErrorEnabled())
- LOGGER.error(
- "tsfile-file TInTimeSeriesChunkMetaData: failed to convert TimeInTimeSeriesChunkMetaData from TSFile to thrift, content is {}",
- this, e);
- throw e;
- }
- }
-
- @Override
- public void convertToTSF(TimeInTimeSeriesChunkMetaData tTimeSeriesChunkMetaDataInThrift) {
- try {
- dataType = tTimeSeriesChunkMetaDataInThrift.getData_type() == null ? null : TSDataType.valueOf(tTimeSeriesChunkMetaDataInThrift.getData_type().toString());
- freqType = tTimeSeriesChunkMetaDataInThrift.getFreq_type() == null ? null : TSFreqType.valueOf(tTimeSeriesChunkMetaDataInThrift.getFreq_type().toString());
- frequencies = tTimeSeriesChunkMetaDataInThrift.getFrequencies();
- startTime = tTimeSeriesChunkMetaDataInThrift.getStartime();
- endTime = tTimeSeriesChunkMetaDataInThrift.getEndtime();
- enumValues = tTimeSeriesChunkMetaDataInThrift.getEnum_values();
- } catch (Exception e) {
- if (LOGGER.isErrorEnabled())
- LOGGER.error(
- "tsfile-file TInTimeSeriesChunkMetaData: failed to convert TimeInTimeSeriesChunkMetaData from thrift to TSFile, content is {}",
- tTimeSeriesChunkMetaDataInThrift, e);
- throw e;
- }
- }
-
- @Override
- public String toString() {
- return String.format(
- "TInTimeSeriesChunkMetaData{ TSDataType %s, TSFreqType %s, frequencies %s, starttime %d, endtime %d, enumValues %s }",
- dataType, freqType, frequencies, startTime, endTime, enumValues);
- }
-
- public TSDataType getDataType() {
- return dataType;
- }
-
- public void setDataType(TSDataType dataType) {
- this.dataType = dataType;
- }
-
- public TSFreqType getFreqType() {
- return freqType;
- }
-
- public void setFreqType(TSFreqType freqType) {
- this.freqType = freqType;
- }
-
- public List getFrequencies() {
- return frequencies;
- }
-
- public void setFrequencies(List frequencies) {
- this.frequencies = frequencies;
- }
-
- public long getStartTime() {
- return startTime;
- }
-
- public void setStartTime(long startTime) {
- this.startTime = startTime;
- }
-
- public long getEndTime() {
- return endTime;
- }
-
- public void setEndTime(long endTime) {
- this.endTime = endTime;
- }
-
- public List getEnumValues() {
- return enumValues;
- }
-
- public void setEnumValues(List enumValues) {
- this.enumValues = enumValues;
- }
-}
+package cn.edu.tsinghua.tsfile.file.metadata;
+
+import cn.edu.tsinghua.tsfile.file.metadata.converter.IConverter;
+import cn.edu.tsinghua.tsfile.file.metadata.enums.TSDataType;
+import cn.edu.tsinghua.tsfile.file.metadata.enums.TSFreqType;
+import cn.edu.tsinghua.tsfile.format.DataType;
+import cn.edu.tsinghua.tsfile.format.FreqType;
+import cn.edu.tsinghua.tsfile.format.TimeInTimeSeriesChunkMetaData;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import java.util.List;
+
+/**
+ * For more information, see TimeInTimeSeriesChunkMetaData in cn.edu.thu.tsfile.format package
+ */
+public class TInTimeSeriesChunkMetaData implements IConverter {
+ private static final Logger LOGGER = LoggerFactory.getLogger(TInTimeSeriesChunkMetaData.class);
+
+ private TSDataType dataType;
+ private long startTime;
+ private long endTime;
+
+ private TSFreqType freqType;
+ private List frequencies;
+
+ /**
+ * If values for data consist of enum values, metadata will store all possible values in time
+ * series
+ */
+ private List enumValues;
+
+ public TInTimeSeriesChunkMetaData() {}
+
+ public TInTimeSeriesChunkMetaData(TSDataType dataType, long startTime, long endTime) {
+ this.dataType = dataType;
+ this.startTime = startTime;
+ this.endTime = endTime;
+ }
+
+ @Override
+ public TimeInTimeSeriesChunkMetaData convertToThrift() {
+ try {
+ TimeInTimeSeriesChunkMetaData tTimeSeriesChunkMetaDataInThrift =
+ new TimeInTimeSeriesChunkMetaData(
+ dataType == null ? null : DataType.valueOf(dataType.toString()), startTime, endTime);
+ tTimeSeriesChunkMetaDataInThrift
+ .setFreq_type(freqType == null ? null : FreqType.valueOf(freqType.toString()));
+ tTimeSeriesChunkMetaDataInThrift.setFrequencies(frequencies);
+ tTimeSeriesChunkMetaDataInThrift.setEnum_values(enumValues);
+ return tTimeSeriesChunkMetaDataInThrift;
+ } catch (Exception e) {
+ if (LOGGER.isErrorEnabled())
+ LOGGER.error(
+ "tsfile-file TInTimeSeriesChunkMetaData: failed to convert TimeInTimeSeriesChunkMetaData from TSFile to thrift, content is {}",
+ this, e);
+ throw e;
+ }
+ }
+
+ @Override
+ public void convertToTSF(TimeInTimeSeriesChunkMetaData tTimeSeriesChunkMetaDataInThrift) {
+ try {
+ dataType = tTimeSeriesChunkMetaDataInThrift.getData_type() == null ? null
+ : TSDataType.valueOf(tTimeSeriesChunkMetaDataInThrift.getData_type().toString());
+ freqType = tTimeSeriesChunkMetaDataInThrift.getFreq_type() == null ? null
+ : TSFreqType.valueOf(tTimeSeriesChunkMetaDataInThrift.getFreq_type().toString());
+ frequencies = tTimeSeriesChunkMetaDataInThrift.getFrequencies();
+ startTime = tTimeSeriesChunkMetaDataInThrift.getStartime();
+ endTime = tTimeSeriesChunkMetaDataInThrift.getEndtime();
+ enumValues = tTimeSeriesChunkMetaDataInThrift.getEnum_values();
+ } catch (Exception e) {
+ if (LOGGER.isErrorEnabled())
+ LOGGER.error(
+ "tsfile-file TInTimeSeriesChunkMetaData: failed to convert TimeInTimeSeriesChunkMetaData from thrift to TSFile, content is {}",
+ tTimeSeriesChunkMetaDataInThrift, e);
+ throw e;
+ }
+ }
+
+ @Override
+ public String toString() {
+ return String.format(
+ "TInTimeSeriesChunkMetaData{ TSDataType %s, TSFreqType %s, frequencies %s, starttime %d, endtime %d, enumValues %s }",
+ dataType, freqType, frequencies, startTime, endTime, enumValues);
+ }
+
+ public TSDataType getDataType() {
+ return dataType;
+ }
+
+ public void setDataType(TSDataType dataType) {
+ this.dataType = dataType;
+ }
+
+ public TSFreqType getFreqType() {
+ return freqType;
+ }
+
+ public void setFreqType(TSFreqType freqType) {
+ this.freqType = freqType;
+ }
+
+ public List getFrequencies() {
+ return frequencies;
+ }
+
+ public void setFrequencies(List frequencies) {
+ this.frequencies = frequencies;
+ }
+
+ public long getStartTime() {
+ return startTime;
+ }
+
+ public void setStartTime(long startTime) {
+ this.startTime = startTime;
+ }
+
+ public long getEndTime() {
+ return endTime;
+ }
+
+ public void setEndTime(long endTime) {
+ this.endTime = endTime;
+ }
+
+ public List getEnumValues() {
+ return enumValues;
+ }
+
+ public void setEnumValues(List enumValues) {
+ this.enumValues = enumValues;
+ }
+}
diff --git a/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/TimeSeriesChunkMetaData.java b/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/TimeSeriesChunkMetaData.java
index 7c5965cf..294f3cc5 100644
--- a/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/TimeSeriesChunkMetaData.java
+++ b/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/TimeSeriesChunkMetaData.java
@@ -1,253 +1,254 @@
-package cn.edu.tsinghua.tsfile.file.metadata;
-
-import cn.edu.tsinghua.tsfile.file.metadata.converter.IConverter;
-import cn.edu.tsinghua.tsfile.file.metadata.enums.CompressionTypeName;
-import cn.edu.tsinghua.tsfile.file.metadata.enums.TSChunkType;
-import cn.edu.tsinghua.tsfile.format.CompressionType;
-import cn.edu.tsinghua.tsfile.format.TimeSeriesChunkType;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * For more information, see TimeSeriesChunkMetaData in cn.edu.thu.tsfile.format package
- */
-public class TimeSeriesChunkMetaData
- implements IConverter {
- private static final Logger LOGGER = LoggerFactory.getLogger(TimeSeriesChunkMetaData.class);
-
- private TimeSeriesChunkProperties properties;
-
- private long numRows;
-
- /**
- * total byte size of all uncompressed pages in this time series chunk (including the headers)
- */
- private long totalByteSize;
-
- /**
- * Optional json metadata
- */
- private List jsonMetaData;
-
- /**
- * Byte offset from beginning of file to first data page
- */
- private long dataPageOffset;
-
- /**
- * Byte offset from beginning of file to root index page
- */
- private long indexPageOffset;
-
- /**
- * Byte offset from the beginning of file to first (only) dictionary page
- */
- private long dictionaryPageOffset;
-
- /**
- * one of TSeriesMetaData and VSeriesMetaData is not null
- */
- private TInTimeSeriesChunkMetaData tInTimeSeriesChunkMetaData;
- private VInTimeSeriesChunkMetaData vInTimeSeriesChunkMetaData;
-
- /**
- * The maximum time of the tombstones that take effect on this chunk. Only data with larger timestamps than this
- * should be exposed to user.
- */
- private long maxTombstoneTime;
-
- /**
- * The time when the RowGroup of this chunk is closed. This will not be written out and will only be set when read together
- * with its RowGroup during querying.
- */
- private long writtenTime;
-
- public TimeSeriesChunkMetaData() {
- properties = new TimeSeriesChunkProperties();
- jsonMetaData = new ArrayList();
- }
-
- public TimeSeriesChunkMetaData(String measurementUID, TSChunkType tsChunkGroup, long fileOffset,
- CompressionTypeName compression) {
- this();
- this.properties = new TimeSeriesChunkProperties(measurementUID, tsChunkGroup, fileOffset, compression);
- }
-
- public TimeSeriesChunkProperties getProperties() {
- return properties;
- }
-
- public void setProperties(TimeSeriesChunkProperties properties) {
- this.properties = properties;
- }
-
- @Override
- public cn.edu.tsinghua.tsfile.format.TimeSeriesChunkMetaData convertToThrift() {
- try {
- cn.edu.tsinghua.tsfile.format.TimeSeriesChunkMetaData metadataInThrift = initTimeSeriesChunkMetaDataInThrift();
- if (tInTimeSeriesChunkMetaData != null) {
- metadataInThrift.setTime_tsc(tInTimeSeriesChunkMetaData.convertToThrift());
- }
- if (vInTimeSeriesChunkMetaData != null) {
- metadataInThrift.setValue_tsc(vInTimeSeriesChunkMetaData.convertToThrift());
- }
- return metadataInThrift;
- } catch (Exception e) {
- if (LOGGER.isErrorEnabled())
- LOGGER.error(
- "tsfile-file TimeSeriesChunkMetaData: failed to convert TimeSeriesChunkMetaData from TSFile to thrift, content is {}",
- this, e);
- }
- return null;
- }
-
- @Override
- public void convertToTSF(cn.edu.tsinghua.tsfile.format.TimeSeriesChunkMetaData metadataInThrift) {
- try {
- initTimeSeriesChunkMetaDataInTSFile(metadataInThrift);
- if (metadataInThrift.getTime_tsc() == null) {
- tInTimeSeriesChunkMetaData = null;
- } else {
- if (tInTimeSeriesChunkMetaData == null) {
- tInTimeSeriesChunkMetaData = new TInTimeSeriesChunkMetaData();
- }
- tInTimeSeriesChunkMetaData.convertToTSF(metadataInThrift.getTime_tsc());
- }
- if (metadataInThrift.getValue_tsc() == null) {
- vInTimeSeriesChunkMetaData = null;
- } else {
- if (vInTimeSeriesChunkMetaData == null) {
- vInTimeSeriesChunkMetaData = new VInTimeSeriesChunkMetaData();
- }
- vInTimeSeriesChunkMetaData.convertToTSF(metadataInThrift.getValue_tsc());
- }
- } catch (Exception e) {
- if (LOGGER.isErrorEnabled())
- LOGGER.error(
- "tsfile-file TimeSeriesChunkMetaData: failed to convert TimeSeriesChunkMetaData from thrift to TSFile, content is {}",
- metadataInThrift, e);
- }
- }
-
- private cn.edu.tsinghua.tsfile.format.TimeSeriesChunkMetaData initTimeSeriesChunkMetaDataInThrift() {
- cn.edu.tsinghua.tsfile.format.TimeSeriesChunkMetaData metadataInThrift =
- new cn.edu.tsinghua.tsfile.format.TimeSeriesChunkMetaData(
- properties.getMeasurementUID(),
- properties.getTsChunkType() == null ? null : TimeSeriesChunkType.valueOf(properties.getTsChunkType().toString()),
- properties.getFileOffset(),
- properties.getCompression() == null ? null : CompressionType.valueOf(properties.getCompression().toString()));
- metadataInThrift.setNum_rows(numRows);
- metadataInThrift.setTotal_byte_size(totalByteSize);
- metadataInThrift.setJson_metadata(jsonMetaData);
- metadataInThrift.setData_page_offset(dataPageOffset);
- metadataInThrift.setIndex_page_offset(indexPageOffset);
- metadataInThrift.setDictionary_page_offset(dictionaryPageOffset);
- return metadataInThrift;
- }
-
- private void initTimeSeriesChunkMetaDataInTSFile(
- cn.edu.tsinghua.tsfile.format.TimeSeriesChunkMetaData metadataInThrift) {
- properties = new TimeSeriesChunkProperties(
- metadataInThrift.getMeasurement_uid(),
- metadataInThrift.getTimeseries_chunk_type() == null ? null : TSChunkType.valueOf(metadataInThrift.getTimeseries_chunk_type().toString()),
- metadataInThrift.getFile_offset(),
- metadataInThrift.getCompression_type() == null ? null : CompressionTypeName.valueOf(metadataInThrift.getCompression_type().toString()));
- numRows = metadataInThrift.getNum_rows();
- totalByteSize = metadataInThrift.getTotal_byte_size();
- jsonMetaData = metadataInThrift.getJson_metadata();
- dataPageOffset = metadataInThrift.getData_page_offset();
- indexPageOffset = metadataInThrift.getIndex_page_offset();
- dictionaryPageOffset = metadataInThrift.getDictionary_page_offset();
- }
-
- @Override
- public String toString() {
- return String.format(
- "TimeSeriesChunkProperties %s, numRows %d, totalByteSize %d, jsonMetaData %s, dataPageOffset %d, indexPageOffset %d, dictionaryPageOffset %s",
- properties, numRows, totalByteSize, jsonMetaData, dataPageOffset, indexPageOffset,
- dictionaryPageOffset);
- }
-
- public long getNumRows() {
- return numRows;
- }
-
- public void setNumRows(long numRows) {
- this.numRows = numRows;
- }
-
- public long getTotalByteSize() {
- return totalByteSize;
- }
-
- public void setTotalByteSize(long totalByteSize) {
- this.totalByteSize = totalByteSize;
- }
-
- public List getJsonMetaData() {
- return jsonMetaData;
- }
-
- public void setJsonMetaData(List jsonMetaData) {
- this.jsonMetaData = jsonMetaData;
- }
-
- public long getDataPageOffset() {
- return dataPageOffset;
- }
-
- public void setDataPageOffset(long dataPageOffset) {
- this.dataPageOffset = dataPageOffset;
- }
-
- public long getIndexPageOffset() {
- return indexPageOffset;
- }
-
- public void setIndexPageOffset(long indexPageOffset) {
- this.indexPageOffset = indexPageOffset;
- }
-
- public long getDictionaryPageOffset() {
- return dictionaryPageOffset;
- }
-
- public void setDictionaryPageOffset(long dictionaryPageOffset) {
- this.dictionaryPageOffset = dictionaryPageOffset;
- }
-
- public TInTimeSeriesChunkMetaData getTInTimeSeriesChunkMetaData() {
- return tInTimeSeriesChunkMetaData;
- }
-
- public void setTInTimeSeriesChunkMetaData(TInTimeSeriesChunkMetaData tInTimeSeriesChunkMetaData) {
- this.tInTimeSeriesChunkMetaData = tInTimeSeriesChunkMetaData;
- }
-
- public VInTimeSeriesChunkMetaData getVInTimeSeriesChunkMetaData() {
- return vInTimeSeriesChunkMetaData;
- }
-
- public void setVInTimeSeriesChunkMetaData(VInTimeSeriesChunkMetaData vInTimeSeriesChunkMetaData) {
- this.vInTimeSeriesChunkMetaData = vInTimeSeriesChunkMetaData;
- }
-
- public long getMaxTombstoneTime() {
- return maxTombstoneTime;
- }
-
- public void setMaxTombstoneTime(long maxTombstoneTime) {
- this.maxTombstoneTime = maxTombstoneTime;
- }
-
- public long getWrittenTime() {
- return writtenTime;
- }
-
- public void setWrittenTime(long writtenTime) {
- this.writtenTime = writtenTime;
- }
-}
+package cn.edu.tsinghua.tsfile.file.metadata;
+
+import cn.edu.tsinghua.tsfile.file.metadata.converter.IConverter;
+import cn.edu.tsinghua.tsfile.file.metadata.enums.CompressionTypeName;
+import cn.edu.tsinghua.tsfile.file.metadata.enums.TSChunkType;
+import cn.edu.tsinghua.tsfile.format.CompressionType;
+import cn.edu.tsinghua.tsfile.format.TimeSeriesChunkType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * For more information, see TimeSeriesChunkMetaData in cn.edu.thu.tsfile.format package
+ */
+public class TimeSeriesChunkMetaData
+ implements IConverter {
+ private static final Logger LOGGER = LoggerFactory.getLogger(TimeSeriesChunkMetaData.class);
+
+ private TimeSeriesChunkProperties properties;
+
+ private long numRows;
+
+ /**
+ * total byte size of all uncompressed pages in this time series chunk (including the headers)
+ */
+ private long totalByteSize;
+
+ /**
+ * Optional json metadata
+ */
+ private List jsonMetaData;
+
+ /**
+ * Byte offset from beginning of file to first data page
+ */
+ private long dataPageOffset;
+
+ /**
+ * Byte offset from beginning of file to root index page
+ */
+ private long indexPageOffset;
+
+ /**
+ * Byte offset from the beginning of file to first (only) dictionary page
+ */
+ private long dictionaryPageOffset;
+
+ /**
+ * one of TSeriesMetaData and VSeriesMetaData is not null
+ */
+ private TInTimeSeriesChunkMetaData tInTimeSeriesChunkMetaData;
+ private VInTimeSeriesChunkMetaData vInTimeSeriesChunkMetaData;
+
+ /**
+ * The maximum time of the tombstones that take effect on this chunk. Only data with larger
+ * timestamps than this should be exposed to user.
+ */
+ private long maxTombstoneTime;
+
+ /**
+ * The time when the RowGroup of this chunk is closed. This will not be written out and will only
+ * be set when read together with its RowGroup during querying.
+ */
+ private long writtenTime;
+
+ public TimeSeriesChunkMetaData() {
+ properties = new TimeSeriesChunkProperties();
+ jsonMetaData = new ArrayList();
+ }
+
+ public TimeSeriesChunkMetaData(String measurementUID, TSChunkType tsChunkGroup, long fileOffset,
+ CompressionTypeName compression) {
+ this();
+ this.properties =
+ new TimeSeriesChunkProperties(measurementUID, tsChunkGroup, fileOffset, compression);
+ }
+
+ public TimeSeriesChunkProperties getProperties() {
+ return properties;
+ }
+
+ public void setProperties(TimeSeriesChunkProperties properties) {
+ this.properties = properties;
+ }
+
+ @Override
+ public cn.edu.tsinghua.tsfile.format.TimeSeriesChunkMetaData convertToThrift() {
+ try {
+ cn.edu.tsinghua.tsfile.format.TimeSeriesChunkMetaData metadataInThrift =
+ initTimeSeriesChunkMetaDataInThrift();
+ if (tInTimeSeriesChunkMetaData != null) {
+ metadataInThrift.setTime_tsc(tInTimeSeriesChunkMetaData.convertToThrift());
+ }
+ if (vInTimeSeriesChunkMetaData != null) {
+ metadataInThrift.setValue_tsc(vInTimeSeriesChunkMetaData.convertToThrift());
+ }
+ return metadataInThrift;
+ } catch (Exception e) {
+ if (LOGGER.isErrorEnabled())
+ LOGGER.error(
+ "tsfile-file TimeSeriesChunkMetaData: failed to convert TimeSeriesChunkMetaData from TSFile to thrift, content is {}",
+ this, e);
+ }
+ return null;
+ }
+
+ @Override
+ public void convertToTSF(cn.edu.tsinghua.tsfile.format.TimeSeriesChunkMetaData metadataInThrift) {
+ try {
+ initTimeSeriesChunkMetaDataInTSFile(metadataInThrift);
+ if (metadataInThrift.getTime_tsc() == null) {
+ tInTimeSeriesChunkMetaData = null;
+ } else {
+ if (tInTimeSeriesChunkMetaData == null) {
+ tInTimeSeriesChunkMetaData = new TInTimeSeriesChunkMetaData();
+ }
+ tInTimeSeriesChunkMetaData.convertToTSF(metadataInThrift.getTime_tsc());
+ }
+ if (metadataInThrift.getValue_tsc() == null) {
+ vInTimeSeriesChunkMetaData = null;
+ } else {
+ if (vInTimeSeriesChunkMetaData == null) {
+ vInTimeSeriesChunkMetaData = new VInTimeSeriesChunkMetaData();
+ }
+ vInTimeSeriesChunkMetaData.convertToTSF(metadataInThrift.getValue_tsc());
+ }
+ } catch (Exception e) {
+ if (LOGGER.isErrorEnabled())
+ LOGGER.error(
+ "tsfile-file TimeSeriesChunkMetaData: failed to convert TimeSeriesChunkMetaData from thrift to TSFile, content is {}",
+ metadataInThrift, e);
+ }
+ }
+
+ private cn.edu.tsinghua.tsfile.format.TimeSeriesChunkMetaData initTimeSeriesChunkMetaDataInThrift() {
+ cn.edu.tsinghua.tsfile.format.TimeSeriesChunkMetaData metadataInThrift =
+ new cn.edu.tsinghua.tsfile.format.TimeSeriesChunkMetaData(properties.getMeasurementUID(),
+ properties.getTsChunkType() == null ? null
+ : TimeSeriesChunkType.valueOf(properties.getTsChunkType().toString()),
+ properties.getFileOffset(), properties.getCompression() == null ? null
+ : CompressionType.valueOf(properties.getCompression().toString()));
+ metadataInThrift.setNum_rows(numRows);
+ metadataInThrift.setTotal_byte_size(totalByteSize);
+ metadataInThrift.setJson_metadata(jsonMetaData);
+ metadataInThrift.setData_page_offset(dataPageOffset);
+ metadataInThrift.setIndex_page_offset(indexPageOffset);
+ metadataInThrift.setDictionary_page_offset(dictionaryPageOffset);
+ return metadataInThrift;
+ }
+
+ private void initTimeSeriesChunkMetaDataInTSFile(
+ cn.edu.tsinghua.tsfile.format.TimeSeriesChunkMetaData metadataInThrift) {
+ properties = new TimeSeriesChunkProperties(metadataInThrift.getMeasurement_uid(),
+ metadataInThrift.getTimeseries_chunk_type() == null ? null
+ : TSChunkType.valueOf(metadataInThrift.getTimeseries_chunk_type().toString()),
+ metadataInThrift.getFile_offset(), metadataInThrift.getCompression_type() == null ? null
+ : CompressionTypeName.valueOf(metadataInThrift.getCompression_type().toString()));
+ numRows = metadataInThrift.getNum_rows();
+ totalByteSize = metadataInThrift.getTotal_byte_size();
+ jsonMetaData = metadataInThrift.getJson_metadata();
+ dataPageOffset = metadataInThrift.getData_page_offset();
+ indexPageOffset = metadataInThrift.getIndex_page_offset();
+ dictionaryPageOffset = metadataInThrift.getDictionary_page_offset();
+ }
+
+ @Override
+ public String toString() {
+ return String.format(
+ "TimeSeriesChunkProperties %s, numRows %d, totalByteSize %d, jsonMetaData %s, dataPageOffset %d, indexPageOffset %d, dictionaryPageOffset %s",
+ properties, numRows, totalByteSize, jsonMetaData, dataPageOffset, indexPageOffset,
+ dictionaryPageOffset);
+ }
+
+ public long getNumRows() {
+ return numRows;
+ }
+
+ public void setNumRows(long numRows) {
+ this.numRows = numRows;
+ }
+
+ public long getTotalByteSize() {
+ return totalByteSize;
+ }
+
+ public void setTotalByteSize(long totalByteSize) {
+ this.totalByteSize = totalByteSize;
+ }
+
+ public List getJsonMetaData() {
+ return jsonMetaData;
+ }
+
+ public void setJsonMetaData(List jsonMetaData) {
+ this.jsonMetaData = jsonMetaData;
+ }
+
+ public long getDataPageOffset() {
+ return dataPageOffset;
+ }
+
+ public void setDataPageOffset(long dataPageOffset) {
+ this.dataPageOffset = dataPageOffset;
+ }
+
+ public long getIndexPageOffset() {
+ return indexPageOffset;
+ }
+
+ public void setIndexPageOffset(long indexPageOffset) {
+ this.indexPageOffset = indexPageOffset;
+ }
+
+ public long getDictionaryPageOffset() {
+ return dictionaryPageOffset;
+ }
+
+ public void setDictionaryPageOffset(long dictionaryPageOffset) {
+ this.dictionaryPageOffset = dictionaryPageOffset;
+ }
+
+ public TInTimeSeriesChunkMetaData getTInTimeSeriesChunkMetaData() {
+ return tInTimeSeriesChunkMetaData;
+ }
+
+ public void setTInTimeSeriesChunkMetaData(TInTimeSeriesChunkMetaData tInTimeSeriesChunkMetaData) {
+ this.tInTimeSeriesChunkMetaData = tInTimeSeriesChunkMetaData;
+ }
+
+ public VInTimeSeriesChunkMetaData getVInTimeSeriesChunkMetaData() {
+ return vInTimeSeriesChunkMetaData;
+ }
+
+ public void setVInTimeSeriesChunkMetaData(VInTimeSeriesChunkMetaData vInTimeSeriesChunkMetaData) {
+ this.vInTimeSeriesChunkMetaData = vInTimeSeriesChunkMetaData;
+ }
+
+ public long getMaxTombstoneTime() {
+ return maxTombstoneTime;
+ }
+
+ public void setMaxTombstoneTime(long maxTombstoneTime) {
+ this.maxTombstoneTime = maxTombstoneTime;
+ }
+
+ public long getWrittenTime() {
+ return writtenTime;
+ }
+
+ public void setWrittenTime(long writtenTime) {
+ this.writtenTime = writtenTime;
+ }
+}
diff --git a/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/TimeSeriesChunkProperties.java b/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/TimeSeriesChunkProperties.java
index 2afefd4f..10227c83 100644
--- a/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/TimeSeriesChunkProperties.java
+++ b/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/TimeSeriesChunkProperties.java
@@ -1,56 +1,55 @@
-package cn.edu.tsinghua.tsfile.file.metadata;
-
-import cn.edu.tsinghua.tsfile.file.metadata.enums.CompressionTypeName;
-import cn.edu.tsinghua.tsfile.file.metadata.enums.TSChunkType;
-
-/**
- * store required members in TimeSeriesChunkMetaData
- */
-public class TimeSeriesChunkProperties {
- private String measurementUID;
-
- /**
- * Type of this time series
- */
- @Deprecated
- private TSChunkType tsChunkType;
-
- /**
- * Byte offset in file_path to the RowGroupMetaData
- */
- private long fileOffset;
- private CompressionTypeName compression;
-
- public TimeSeriesChunkProperties() {
- }
-
- public TimeSeriesChunkProperties(String measurementUID, TSChunkType tsChunkType, long fileOffset,
- CompressionTypeName compression) {
- this.measurementUID = measurementUID;
- this.tsChunkType = tsChunkType;
- this.fileOffset = fileOffset;
- this.compression = compression;
- }
-
- public TSChunkType getTsChunkType() {
- return tsChunkType;
- }
-
- public long getFileOffset() {
- return fileOffset;
- }
-
- public CompressionTypeName getCompression() {
- return compression;
- }
-
- public String getMeasurementUID() {
- return measurementUID;
- }
-
- @Override
- public String toString() {
- return String.format("measurementUID %s, TSChunkType %s, fileOffset %d, CompressionTypeName %s",
- measurementUID, tsChunkType, fileOffset, compression);
- }
-}
+package cn.edu.tsinghua.tsfile.file.metadata;
+
+import cn.edu.tsinghua.tsfile.file.metadata.enums.CompressionTypeName;
+import cn.edu.tsinghua.tsfile.file.metadata.enums.TSChunkType;
+
+/**
+ * store required members in TimeSeriesChunkMetaData
+ */
+public class TimeSeriesChunkProperties {
+ private String measurementUID;
+
+ /**
+ * Type of this time series
+ */
+ @Deprecated
+ private TSChunkType tsChunkType;
+
+ /**
+ * Byte offset in file_path to the RowGroupMetaData
+ */
+ private long fileOffset;
+ private CompressionTypeName compression;
+
+ public TimeSeriesChunkProperties() {}
+
+ public TimeSeriesChunkProperties(String measurementUID, TSChunkType tsChunkType, long fileOffset,
+ CompressionTypeName compression) {
+ this.measurementUID = measurementUID;
+ this.tsChunkType = tsChunkType;
+ this.fileOffset = fileOffset;
+ this.compression = compression;
+ }
+
+ public TSChunkType getTsChunkType() {
+ return tsChunkType;
+ }
+
+ public long getFileOffset() {
+ return fileOffset;
+ }
+
+ public CompressionTypeName getCompression() {
+ return compression;
+ }
+
+ public String getMeasurementUID() {
+ return measurementUID;
+ }
+
+ @Override
+ public String toString() {
+ return String.format("measurementUID %s, TSChunkType %s, fileOffset %d, CompressionTypeName %s",
+ measurementUID, tsChunkType, fileOffset, compression);
+ }
+}
diff --git a/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/TimeSeriesMetadata.java b/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/TimeSeriesMetadata.java
index 8503b563..a697627c 100644
--- a/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/TimeSeriesMetadata.java
+++ b/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/TimeSeriesMetadata.java
@@ -1,141 +1,141 @@
-package cn.edu.tsinghua.tsfile.file.metadata;
-
-import cn.edu.tsinghua.tsfile.file.metadata.converter.IConverter;
-import cn.edu.tsinghua.tsfile.file.metadata.enums.TSDataType;
-import cn.edu.tsinghua.tsfile.file.metadata.enums.TSFreqType;
-import cn.edu.tsinghua.tsfile.format.DataType;
-import cn.edu.tsinghua.tsfile.format.FreqType;
-import cn.edu.tsinghua.tsfile.format.TimeSeries;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.List;
-
-/**
- * For more information, see TimeSeries in cn.edu.thu.tsfile.format package
- */
-public class TimeSeriesMetadata implements IConverter {
- private static final Logger LOGGER = LoggerFactory.getLogger(TimeSeriesMetadata.class);
-
- private String measurementUID;
-
- private TSDataType type;
-
- /**
- * If type is FIXED_LEN_BYTE_ARRAY, this is the byte length of the values. Otherwise, if
- * specified, this is the maximum bit length to store any of the values. (e.g. a low cardinality
- * INT timeseries could have this set to 32). Note that this is in the schema, and therefore fixed
- * for the entire file.
- */
- private int typeLength;
-
- private TSFreqType freqType;
- private List frequencies;
-
- /**
- * If values for data consist of enum values, metadata will store all possible values in time
- * series
- */
- private List enumValues;
-
- public TimeSeriesMetadata() {
- }
-
- public TimeSeriesMetadata(String measurementUID, TSDataType dataType) {
- this.measurementUID = measurementUID;
- this.type = dataType;
- }
-
- @Override
- public TimeSeries convertToThrift() {
- try {
- TimeSeries timeSeriesInThrift = new TimeSeries(measurementUID,
- type == null ? null : DataType.valueOf(type.toString()), "");//FIXME remove deltaType from TimeSeries.java
- timeSeriesInThrift.setType_length(typeLength);
- timeSeriesInThrift.setFreq_type(freqType == null ? null : FreqType.valueOf(freqType.toString()));
- timeSeriesInThrift.setFrequencies(frequencies);
- timeSeriesInThrift.setEnum_values(enumValues);
- return timeSeriesInThrift;
- } catch (Exception e) {
- if (LOGGER.isErrorEnabled())
- LOGGER.error(
- "tsfile-file TimeSeriesMetadata: failed to convert TimeSeriesMetadata from TSFile to thrift, content is {}",
- this, e);
- throw e;
- }
- }
-
- @Override
- public void convertToTSF(TimeSeries timeSeriesInThrift) {
- try {
- measurementUID = timeSeriesInThrift.getMeasurement_uid();
- type = timeSeriesInThrift.getType() == null ? null
- : TSDataType.valueOf(timeSeriesInThrift.getType().toString());
- typeLength = timeSeriesInThrift.getType_length();
- freqType = timeSeriesInThrift.getFreq_type() == null ? null
- : TSFreqType.valueOf(timeSeriesInThrift.getFreq_type().toString());
- frequencies = timeSeriesInThrift.getFrequencies();
- enumValues = timeSeriesInThrift.getEnum_values();
- } catch (Exception e) {
- if (LOGGER.isErrorEnabled())
- LOGGER.error(
- "tsfile-file TimeSeriesMetadata: failed to convert TimeSeriesMetadata from TSFile to thrift, content is {}",
- timeSeriesInThrift, e);
- }
- }
-
- public String getMeasurementUID() {
- return measurementUID;
- }
-
- public void setMeasurementUID(String measurementUID) {
- this.measurementUID = measurementUID;
- }
-
- public int getTypeLength() {
- return typeLength;
- }
-
- public void setTypeLength(int typeLength) {
- this.typeLength = typeLength;
- }
-
- public TSDataType getType() {
- return type;
- }
-
- public void setType(TSDataType type) {
- this.type = type;
- }
-
- public TSFreqType getFreqType() {
- return freqType;
- }
-
- public void setFreqType(TSFreqType freqType) {
- this.freqType = freqType;
- }
-
- public List getFrequencies() {
- return frequencies;
- }
-
- public void setFrequencies(List frequencies) {
- this.frequencies = frequencies;
- }
-
- public List getEnumValues() {
- return enumValues;
- }
-
- public void setEnumValues(List enumValues) {
- this.enumValues = enumValues;
- }
-
- @Override
- public String toString() {
- return String.format(
- "TimeSeriesMetadata: measurementUID %s, type length %d, DataType %s, FreqType %s,frequencies %s",
- measurementUID, typeLength, type, freqType, frequencies);
- }
-}
+package cn.edu.tsinghua.tsfile.file.metadata;
+
+import cn.edu.tsinghua.tsfile.file.metadata.converter.IConverter;
+import cn.edu.tsinghua.tsfile.file.metadata.enums.TSDataType;
+import cn.edu.tsinghua.tsfile.file.metadata.enums.TSFreqType;
+import cn.edu.tsinghua.tsfile.format.DataType;
+import cn.edu.tsinghua.tsfile.format.FreqType;
+import cn.edu.tsinghua.tsfile.format.TimeSeries;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import java.util.List;
+
+/**
+ * For more information, see TimeSeries in cn.edu.thu.tsfile.format package
+ */
+public class TimeSeriesMetadata implements IConverter {
+ private static final Logger LOGGER = LoggerFactory.getLogger(TimeSeriesMetadata.class);
+
+ private String measurementUID;
+
+ private TSDataType type;
+
+ /**
+ * If type is FIXED_LEN_BYTE_ARRAY, this is the byte length of the values. Otherwise, if
+ * specified, this is the maximum bit length to store any of the values. (e.g. a low cardinality
+ * INT timeseries could have this set to 32). Note that this is in the schema, and therefore fixed
+ * for the entire file.
+ */
+ private int typeLength;
+
+ private TSFreqType freqType;
+ private List frequencies;
+
+ /**
+ * If values for data consist of enum values, metadata will store all possible values in time
+ * series
+ */
+ private List enumValues;
+
+ public TimeSeriesMetadata() {}
+
+ public TimeSeriesMetadata(String measurementUID, TSDataType dataType) {
+ this.measurementUID = measurementUID;
+ this.type = dataType;
+ }
+
+ @Override
+ public TimeSeries convertToThrift() {
+ try {
+ TimeSeries timeSeriesInThrift = new TimeSeries(measurementUID,
+ type == null ? null : DataType.valueOf(type.toString()), "");// FIXME remove deltaType
+ // from TimeSeries.java
+ timeSeriesInThrift.setType_length(typeLength);
+ timeSeriesInThrift
+ .setFreq_type(freqType == null ? null : FreqType.valueOf(freqType.toString()));
+ timeSeriesInThrift.setFrequencies(frequencies);
+ timeSeriesInThrift.setEnum_values(enumValues);
+ return timeSeriesInThrift;
+ } catch (Exception e) {
+ if (LOGGER.isErrorEnabled())
+ LOGGER.error(
+ "tsfile-file TimeSeriesMetadata: failed to convert TimeSeriesMetadata from TSFile to thrift, content is {}",
+ this, e);
+ throw e;
+ }
+ }
+
+ @Override
+ public void convertToTSF(TimeSeries timeSeriesInThrift) {
+ try {
+ measurementUID = timeSeriesInThrift.getMeasurement_uid();
+ type = timeSeriesInThrift.getType() == null ? null
+ : TSDataType.valueOf(timeSeriesInThrift.getType().toString());
+ typeLength = timeSeriesInThrift.getType_length();
+ freqType = timeSeriesInThrift.getFreq_type() == null ? null
+ : TSFreqType.valueOf(timeSeriesInThrift.getFreq_type().toString());
+ frequencies = timeSeriesInThrift.getFrequencies();
+ enumValues = timeSeriesInThrift.getEnum_values();
+ } catch (Exception e) {
+ if (LOGGER.isErrorEnabled())
+ LOGGER.error(
+ "tsfile-file TimeSeriesMetadata: failed to convert TimeSeriesMetadata from TSFile to thrift, content is {}",
+ timeSeriesInThrift, e);
+ }
+ }
+
+ public String getMeasurementUID() {
+ return measurementUID;
+ }
+
+ public void setMeasurementUID(String measurementUID) {
+ this.measurementUID = measurementUID;
+ }
+
+ public int getTypeLength() {
+ return typeLength;
+ }
+
+ public void setTypeLength(int typeLength) {
+ this.typeLength = typeLength;
+ }
+
+ public TSDataType getType() {
+ return type;
+ }
+
+ public void setType(TSDataType type) {
+ this.type = type;
+ }
+
+ public TSFreqType getFreqType() {
+ return freqType;
+ }
+
+ public void setFreqType(TSFreqType freqType) {
+ this.freqType = freqType;
+ }
+
+ public List getFrequencies() {
+ return frequencies;
+ }
+
+ public void setFrequencies(List frequencies) {
+ this.frequencies = frequencies;
+ }
+
+ public List getEnumValues() {
+ return enumValues;
+ }
+
+ public void setEnumValues(List enumValues) {
+ this.enumValues = enumValues;
+ }
+
+ @Override
+ public String toString() {
+ return String.format(
+ "TimeSeriesMetadata: measurementUID %s, type length %d, DataType %s, FreqType %s,frequencies %s",
+ measurementUID, typeLength, type, freqType, frequencies);
+ }
+}
diff --git a/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/TsDeltaObject.java b/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/TsDeltaObject.java
index 0e70407b..ab6e7578 100644
--- a/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/TsDeltaObject.java
+++ b/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/TsDeltaObject.java
@@ -3,36 +3,36 @@
import cn.edu.tsinghua.tsfile.file.metadata.converter.IConverter;
import cn.edu.tsinghua.tsfile.format.DeltaObject;
-public class TsDeltaObject implements IConverter{
- /** start position of RowGroupMetadataBlock in file **/
- public long offset;
-
- /** size of RowGroupMetadataBlock in byte **/
- public int metadataBlockSize;
-
- /** start time for a delta object **/
- public long startTime;
-
- /** end time for a delta object **/
- public long endTime;
-
- public TsDeltaObject(long offset, int metadataBlockSize, long startTime, long endTime){
- this.offset = offset;
- this.metadataBlockSize = metadataBlockSize;
- this.startTime = startTime;
- this.endTime = endTime;
- }
-
- @Override
- public DeltaObject convertToThrift() {
- return new DeltaObject(offset, metadataBlockSize, startTime, endTime);
- }
-
- @Override
- public void convertToTSF(DeltaObject metadata) {
- this.offset = metadata.getOffset();
- this.metadataBlockSize = metadata.getMetadata_block_size();
- this.startTime = metadata.getStart_time();
- this.endTime = metadata.getEnd_time();
- }
+public class TsDeltaObject implements IConverter {
+ /** start position of RowGroupMetadataBlock in file **/
+ public long offset;
+
+ /** size of RowGroupMetadataBlock in byte **/
+ public int metadataBlockSize;
+
+ /** start time for a delta object **/
+ public long startTime;
+
+ /** end time for a delta object **/
+ public long endTime;
+
+ public TsDeltaObject(long offset, int metadataBlockSize, long startTime, long endTime) {
+ this.offset = offset;
+ this.metadataBlockSize = metadataBlockSize;
+ this.startTime = startTime;
+ this.endTime = endTime;
+ }
+
+ @Override
+ public DeltaObject convertToThrift() {
+ return new DeltaObject(offset, metadataBlockSize, startTime, endTime);
+ }
+
+ @Override
+ public void convertToTSF(DeltaObject metadata) {
+ this.offset = metadata.getOffset();
+ this.metadataBlockSize = metadata.getMetadata_block_size();
+ this.startTime = metadata.getStart_time();
+ this.endTime = metadata.getEnd_time();
+ }
}
diff --git a/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/TsDigest.java b/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/TsDigest.java
index 0b0e8eaf..a8571e1f 100644
--- a/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/TsDigest.java
+++ b/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/TsDigest.java
@@ -1,79 +1,77 @@
-package cn.edu.tsinghua.tsfile.file.metadata;
-
-import cn.edu.tsinghua.tsfile.file.metadata.converter.IConverter;
-import cn.edu.tsinghua.tsfile.format.Digest;
-
-import java.nio.ByteBuffer;
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- * For more information, see Digest in cn.edu.thu.tsfile.format package
- */
-public class TsDigest implements IConverter {
- /**
- * Digest/statistics per row group and per page.
- */
- public Map statistics;
-
- public TsDigest() {
- }
-
- public TsDigest(Map statistics) {
- this.statistics = statistics;
- }
-
- public void setStatistics(Map statistics) {
- this.statistics = statistics;
- }
-
- public Map getStatistics(){
- return this.statistics;
- }
-
- public void addStatistics(String key, ByteBuffer value) {
- if(statistics == null) {
- statistics = new HashMap<>();
- }
- statistics.put(key, value);
- }
-
- @Override
- public String toString() {
- return statistics != null ? statistics.toString() : "";
- }
-
- @Override
- public Digest convertToThrift() {
- Digest digest = new Digest();
- if (statistics != null) {
- Map statisticsInThrift = new HashMap<>();
- for (String key : statistics.keySet()) {
- statisticsInThrift.put(key, statistics.get(key));
- }
- digest.setStatistics(statisticsInThrift);
- }
- return digest;
- }
-
- @Override
- public void convertToTSF(Digest digestInThrift) {
- if (digestInThrift != null) {
- Map statisticsInThrift = digestInThrift.getStatistics();
- if (statisticsInThrift != null) {
- statistics = new HashMap<>();
- for (String key : statisticsInThrift.keySet()) {
- statistics.put(key, byteBufferDeepCopy(statisticsInThrift.get(key)));
- }
- } else {
- statistics = null;
- }
- }
- }
-
- public ByteBuffer byteBufferDeepCopy(ByteBuffer src) {
- ByteBuffer copy = ByteBuffer.allocate(src.remaining()).put(src.slice());
- copy.flip();
- return copy;
- }
-}
+package cn.edu.tsinghua.tsfile.file.metadata;
+
+import cn.edu.tsinghua.tsfile.file.metadata.converter.IConverter;
+import cn.edu.tsinghua.tsfile.format.Digest;
+import java.nio.ByteBuffer;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * For more information, see Digest in cn.edu.thu.tsfile.format package
+ */
+public class TsDigest implements IConverter {
+ /**
+ * Digest/statistics per row group and per page.
+ */
+ public Map statistics;
+
+ public TsDigest() {}
+
+ public TsDigest(Map statistics) {
+ this.statistics = statistics;
+ }
+
+ public void setStatistics(Map statistics) {
+ this.statistics = statistics;
+ }
+
+ public Map getStatistics() {
+ return this.statistics;
+ }
+
+ public void addStatistics(String key, ByteBuffer value) {
+ if (statistics == null) {
+ statistics = new HashMap<>();
+ }
+ statistics.put(key, value);
+ }
+
+ @Override
+ public String toString() {
+ return statistics != null ? statistics.toString() : "";
+ }
+
+ @Override
+ public Digest convertToThrift() {
+ Digest digest = new Digest();
+ if (statistics != null) {
+ Map statisticsInThrift = new HashMap<>();
+ for (String key : statistics.keySet()) {
+ statisticsInThrift.put(key, statistics.get(key));
+ }
+ digest.setStatistics(statisticsInThrift);
+ }
+ return digest;
+ }
+
+ @Override
+ public void convertToTSF(Digest digestInThrift) {
+ if (digestInThrift != null) {
+ Map statisticsInThrift = digestInThrift.getStatistics();
+ if (statisticsInThrift != null) {
+ statistics = new HashMap<>();
+ for (String key : statisticsInThrift.keySet()) {
+ statistics.put(key, byteBufferDeepCopy(statisticsInThrift.get(key)));
+ }
+ } else {
+ statistics = null;
+ }
+ }
+ }
+
+ public ByteBuffer byteBufferDeepCopy(ByteBuffer src) {
+ ByteBuffer copy = ByteBuffer.allocate(src.remaining()).put(src.slice());
+ copy.flip();
+ return copy;
+ }
+}
diff --git a/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/TsFileMetaData.java b/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/TsFileMetaData.java
index 8d24c4a3..7287c77d 100644
--- a/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/TsFileMetaData.java
+++ b/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/TsFileMetaData.java
@@ -1,263 +1,271 @@
-package cn.edu.tsinghua.tsfile.file.metadata;
-
-import cn.edu.tsinghua.tsfile.file.metadata.converter.IConverter;
-import cn.edu.tsinghua.tsfile.file.metadata.enums.TSDataType;
-import cn.edu.tsinghua.tsfile.format.DeltaObject;
-import cn.edu.tsinghua.tsfile.format.FileMetaData;
-import cn.edu.tsinghua.tsfile.format.TimeSeries;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.*;
-
-/**
- * TSFileMetaData collects all metadata info and saves in its data structure
- */
-public class TsFileMetaData implements IConverter {
- private static final Logger LOGGER = LoggerFactory.getLogger(TsFileMetaData.class);
-
- private Map deltaObjectMap;
-
- /**
- * TSFile schema for this file. This schema contains metadata for all the time series. The schema
- * is represented as a list.
- */
- private List timeSeriesList;
-
- /**
- * Version of this file
- */
- private int currentVersion;
-
- /**
- * Optional json metadata
- */
- private List jsonMetaData;
-
- /**
- * String for application that wrote this file. This should be in the format version
- * (build ). e.g. impala version 1.0 (build SHA-1_hash_code)
- */
- private String createdBy;
-
- /**
- * User specified props
- */
- private Map props;
-
- public TsFileMetaData() {
- }
-
- /**
- * @param timeSeriesList - time series info list
- * @param currentVersion - current version
- */
- public TsFileMetaData(Map deltaObjectMap, List timeSeriesList, int currentVersion) {
- this.props = new HashMap<>();
- this.deltaObjectMap = deltaObjectMap;
- this.timeSeriesList = timeSeriesList;
- this.currentVersion = currentVersion;
- }
-
- /**
- * add time series metadata to list. THREAD NOT SAFE
- * @param timeSeries series metadata to add
- */
- public void addTimeSeriesMetaData(TimeSeriesMetadata timeSeries) {
- if (timeSeriesList == null) {
- timeSeriesList = new ArrayList<>();
- }
- timeSeriesList.add(timeSeries);
- }
-
-// /**
-// * get all delta object uid and their types
-// *
-// * @return set of {@code Pair}
-// */
-// public Set> getAllDeltaObjects() {
-// // Pair
-// Set> deltaObjectSet = new HashSet>();
-// if (rowGroupMetadataList != null) {
-// for (RowGroupMetaData rowGroup : rowGroupMetadataList) {
-// deltaObjectSet.add(
-// new Pair(rowGroup.getDeltaObjectUID(), rowGroup.getDeltaObjectType()));
-// }
-// }
-// return deltaObjectSet;
-// }
-
- @Override
- public String toString() {
- return String.format("TSFMetaData { DeltaOjectMap: %s, timeSeries list %s, current version %d }", deltaObjectMap,
- timeSeriesList, currentVersion);
- }
-
- /**
- * create file metadata in thrift format. For more information about file metadata
- * in cn.edu.thu.tsfile.format package, see FileMetaData in tsfile-format
- *
- * @return file metadata in thrift format
- */
- @Override
- public FileMetaData convertToThrift() {
- try {
- List timeSeriesListInThrift = null;
- if (timeSeriesList != null) {
- timeSeriesListInThrift = new ArrayList();
- for (TimeSeriesMetadata timeSeries : timeSeriesList) {
- timeSeriesListInThrift.add(timeSeries.convertToThrift());
- }
- }
-
- Map deltaObjectMapInThrift = null;
- if( deltaObjectMap != null){
- deltaObjectMapInThrift = new HashMap<>();
- for(Map.Entry entry : deltaObjectMap.entrySet()){
- TsDeltaObject object = entry.getValue();
- deltaObjectMapInThrift.put(entry.getKey(), new DeltaObject(object.offset,
- object.metadataBlockSize, object.startTime, object.endTime));
- }
- }
-
- FileMetaData metaDataInThrift = new FileMetaData(currentVersion, deltaObjectMapInThrift, timeSeriesListInThrift);
- metaDataInThrift.setCreated_by(createdBy);
- metaDataInThrift.setJson_metadata(jsonMetaData);
- metaDataInThrift.setProperties(props);
- return metaDataInThrift;
- } catch (Exception e) {
- LOGGER.error("TsFileMetaData: failed to convert file metadata from TSFile to thrift, content is {}", this, e);
- throw e;
- }
- }
-
- /**
- * receive file metadata in thrift format and convert it to tsfile format
- * @param metadataInThrift - file metadata in thrift format
- */
- @Override
- public void convertToTSF(FileMetaData metadataInThrift) {
- try {
- if (metadataInThrift.getTimeseries_list() == null) {
- timeSeriesList = null;
- } else {
- timeSeriesList = new ArrayList();
-
- for (TimeSeries timeSeriesInThrift : metadataInThrift.getTimeseries_list()) {
- TimeSeriesMetadata timeSeriesInTSFile = new TimeSeriesMetadata();
- timeSeriesInTSFile.convertToTSF(timeSeriesInThrift);
- timeSeriesList.add(timeSeriesInTSFile);
- }
- }
-
- if(metadataInThrift.getDelta_object_map() == null){
- deltaObjectMap = null;
- } else {
- deltaObjectMap = new HashMap<>();
- for (Map.Entry entry : metadataInThrift.getDelta_object_map().entrySet()){
- DeltaObject object = entry.getValue();
- deltaObjectMap.put(entry.getKey(), new TsDeltaObject(object.getOffset(),
- object.getMetadata_block_size(), object.getStart_time(), object.getEnd_time()));
- }
- }
-
- currentVersion = metadataInThrift.getVersion();
- createdBy = metadataInThrift.getCreated_by();
- jsonMetaData = metadataInThrift.getJson_metadata();
- props = metadataInThrift.getProperties();
- } catch (Exception e) {
- LOGGER.error("TsFileMetaData: failed to convert file metadata from thrift to TSFile, content is {}",metadataInThrift, e);
- throw e;
- }
-
- }
-
- public List getTimeSeriesList() {
- return timeSeriesList;
- }
-
- public void setTimeSeriesList(List timeSeriesList) {
- this.timeSeriesList = timeSeriesList;
- }
-
- public int getCurrentVersion() {
- return currentVersion;
- }
-
- public void setCurrentVersion(int currentVersion) {
- this.currentVersion = currentVersion;
- }
-
- public List getJsonMetaData() {
- return jsonMetaData;
- }
-
- public void setJsonMetaData(List jsonMetaData) {
- this.jsonMetaData = jsonMetaData;
- }
-
- public String getCreatedBy() {
- return createdBy;
- }
-
- public void setCreatedBy(String createdBy) {
- this.createdBy = createdBy;
- }
-
- public void addProp(String key, String value) {
- props.put(key, value);
- }
-
- public Map getProps() {
- return props;
- }
-
- public void setProps(Map properties) {
- this.props.clear();
- this.props.putAll(properties);
- }
-
- public String getProp(String key) {
- if (props.containsKey(key))
- return props.get(key);
- else
- return null;
- }
-
- public Map getDeltaObjectMap() {
- return deltaObjectMap;
- }
-
- public void setDeltaObjectMap(Map deltaObjectMap) {
- this.deltaObjectMap = deltaObjectMap;
- }
-
- public boolean containsDeltaObject(String DeltaObjUID) {
- return this.deltaObjectMap.containsKey(DeltaObjUID);
- }
-
- public TsDeltaObject getDeltaObject(String DeltaObjUID) {
- return this.deltaObjectMap.get(DeltaObjUID);
- }
-
- //For Tsfile-Spark-Connector
- public boolean containsMeasurement(String measurement) {
- for(TimeSeriesMetadata ts: timeSeriesList ){
- if(ts.getMeasurementUID().equals(measurement)) {
- return true;
- }
- }
- return false;
- }
-
- //For Tsfile-Spark-Connector
- public TSDataType getType(String measurement) throws IOException{
- for(TimeSeriesMetadata ts: timeSeriesList ){
- if(ts.getMeasurementUID().equals(measurement)) {
- return ts.getType();
- }
- }
- throw new IOException("Measurement " + measurement + " does not exist in the current file.");
- }
-}
+package cn.edu.tsinghua.tsfile.file.metadata;
+
+import cn.edu.tsinghua.tsfile.file.metadata.converter.IConverter;
+import cn.edu.tsinghua.tsfile.file.metadata.enums.TSDataType;
+import cn.edu.tsinghua.tsfile.format.DeltaObject;
+import cn.edu.tsinghua.tsfile.format.FileMetaData;
+import cn.edu.tsinghua.tsfile.format.TimeSeries;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import java.io.IOException;
+import java.util.*;
+
+/**
+ * TSFileMetaData collects all metadata info and saves in its data structure
+ */
+public class TsFileMetaData implements IConverter {
+ private static final Logger LOGGER = LoggerFactory.getLogger(TsFileMetaData.class);
+
+ private Map deltaObjectMap;
+
+ /**
+ * TSFile schema for this file. This schema contains metadata for all the time series. The schema
+ * is represented as a list.
+ */
+ private List timeSeriesList;
+
+ /**
+ * Version of this file
+ */
+ private int currentVersion;
+
+ /**
+ * Optional json metadata
+ */
+ private List jsonMetaData;
+
+ /**
+ * String for application that wrote this file. This should be in the format version
+ * (build ). e.g. impala version 1.0 (build SHA-1_hash_code)
+ */
+ private String createdBy;
+
+ /**
+ * User specified props
+ */
+ private Map props;
+
+ public TsFileMetaData() {}
+
+ /**
+ * @param timeSeriesList - time series info list
+ * @param currentVersion - current version
+ */
+ public TsFileMetaData(Map deltaObjectMap,
+ List timeSeriesList, int currentVersion) {
+ this.props = new HashMap<>();
+ this.deltaObjectMap = deltaObjectMap;
+ this.timeSeriesList = timeSeriesList;
+ this.currentVersion = currentVersion;
+ }
+
+ /**
+ * add time series metadata to list. THREAD NOT SAFE
+ *
+ * @param timeSeries series metadata to add
+ */
+ public void addTimeSeriesMetaData(TimeSeriesMetadata timeSeries) {
+ if (timeSeriesList == null) {
+ timeSeriesList = new ArrayList<>();
+ }
+ timeSeriesList.add(timeSeries);
+ }
+
+ // /**
+ // * get all delta object uid and their types
+ // *
+ // * @return set of {@code Pair}
+ // */
+ // public Set> getAllDeltaObjects() {
+ // // Pair
+ // Set> deltaObjectSet = new HashSet>();
+ // if (rowGroupMetadataList != null) {
+ // for (RowGroupMetaData rowGroup : rowGroupMetadataList) {
+ // deltaObjectSet.add(
+ // new Pair(rowGroup.getDeltaObjectUID(), rowGroup.getDeltaObjectType()));
+ // }
+ // }
+ // return deltaObjectSet;
+ // }
+
+ @Override
+ public String toString() {
+ return String.format(
+ "TSFMetaData { DeltaOjectMap: %s, timeSeries list %s, current version %d }", deltaObjectMap,
+ timeSeriesList, currentVersion);
+ }
+
+ /**
+ * create file metadata in thrift format. For more information about file metadata in
+ * cn.edu.thu.tsfile.format package, see FileMetaData in tsfile-format
+ *
+ * @return file metadata in thrift format
+ */
+ @Override
+ public FileMetaData convertToThrift() {
+ try {
+ List timeSeriesListInThrift = null;
+ if (timeSeriesList != null) {
+ timeSeriesListInThrift = new ArrayList();
+ for (TimeSeriesMetadata timeSeries : timeSeriesList) {
+ timeSeriesListInThrift.add(timeSeries.convertToThrift());
+ }
+ }
+
+ Map deltaObjectMapInThrift = null;
+ if (deltaObjectMap != null) {
+ deltaObjectMapInThrift = new HashMap<>();
+ for (Map.Entry entry : deltaObjectMap.entrySet()) {
+ TsDeltaObject object = entry.getValue();
+ deltaObjectMapInThrift.put(entry.getKey(), new DeltaObject(object.offset,
+ object.metadataBlockSize, object.startTime, object.endTime));
+ }
+ }
+
+ FileMetaData metaDataInThrift =
+ new FileMetaData(currentVersion, deltaObjectMapInThrift, timeSeriesListInThrift);
+ metaDataInThrift.setCreated_by(createdBy);
+ metaDataInThrift.setJson_metadata(jsonMetaData);
+ metaDataInThrift.setProperties(props);
+ return metaDataInThrift;
+ } catch (Exception e) {
+ LOGGER.error(
+ "TsFileMetaData: failed to convert file metadata from TSFile to thrift, content is {}",
+ this, e);
+ throw e;
+ }
+ }
+
+ /**
+ * receive file metadata in thrift format and convert it to tsfile format
+ *
+ * @param metadataInThrift - file metadata in thrift format
+ */
+ @Override
+ public void convertToTSF(FileMetaData metadataInThrift) {
+ try {
+ if (metadataInThrift.getTimeseries_list() == null) {
+ timeSeriesList = null;
+ } else {
+ timeSeriesList = new ArrayList();
+
+ for (TimeSeries timeSeriesInThrift : metadataInThrift.getTimeseries_list()) {
+ TimeSeriesMetadata timeSeriesInTSFile = new TimeSeriesMetadata();
+ timeSeriesInTSFile.convertToTSF(timeSeriesInThrift);
+ timeSeriesList.add(timeSeriesInTSFile);
+ }
+ }
+
+ if (metadataInThrift.getDelta_object_map() == null) {
+ deltaObjectMap = null;
+ } else {
+ deltaObjectMap = new HashMap<>();
+ for (Map.Entry entry : metadataInThrift.getDelta_object_map()
+ .entrySet()) {
+ DeltaObject object = entry.getValue();
+ deltaObjectMap.put(entry.getKey(), new TsDeltaObject(object.getOffset(),
+ object.getMetadata_block_size(), object.getStart_time(), object.getEnd_time()));
+ }
+ }
+
+ currentVersion = metadataInThrift.getVersion();
+ createdBy = metadataInThrift.getCreated_by();
+ jsonMetaData = metadataInThrift.getJson_metadata();
+ props = metadataInThrift.getProperties();
+ } catch (Exception e) {
+ LOGGER.error(
+ "TsFileMetaData: failed to convert file metadata from thrift to TSFile, content is {}",
+ metadataInThrift, e);
+ throw e;
+ }
+
+ }
+
+ public List getTimeSeriesList() {
+ return timeSeriesList;
+ }
+
+ public void setTimeSeriesList(List timeSeriesList) {
+ this.timeSeriesList = timeSeriesList;
+ }
+
+ public int getCurrentVersion() {
+ return currentVersion;
+ }
+
+ public void setCurrentVersion(int currentVersion) {
+ this.currentVersion = currentVersion;
+ }
+
+ public List getJsonMetaData() {
+ return jsonMetaData;
+ }
+
+ public void setJsonMetaData(List jsonMetaData) {
+ this.jsonMetaData = jsonMetaData;
+ }
+
+ public String getCreatedBy() {
+ return createdBy;
+ }
+
+ public void setCreatedBy(String createdBy) {
+ this.createdBy = createdBy;
+ }
+
+ public void addProp(String key, String value) {
+ props.put(key, value);
+ }
+
+ public Map getProps() {
+ return props;
+ }
+
+ public void setProps(Map properties) {
+ this.props.clear();
+ this.props.putAll(properties);
+ }
+
+ public String getProp(String key) {
+ if (props.containsKey(key))
+ return props.get(key);
+ else
+ return null;
+ }
+
+ public Map getDeltaObjectMap() {
+ return deltaObjectMap;
+ }
+
+ public void setDeltaObjectMap(Map deltaObjectMap) {
+ this.deltaObjectMap = deltaObjectMap;
+ }
+
+ public boolean containsDeltaObject(String DeltaObjUID) {
+ return this.deltaObjectMap.containsKey(DeltaObjUID);
+ }
+
+ public TsDeltaObject getDeltaObject(String DeltaObjUID) {
+ return this.deltaObjectMap.get(DeltaObjUID);
+ }
+
+ // For Tsfile-Spark-Connector
+ public boolean containsMeasurement(String measurement) {
+ for (TimeSeriesMetadata ts : timeSeriesList) {
+ if (ts.getMeasurementUID().equals(measurement)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ // For Tsfile-Spark-Connector
+ public TSDataType getType(String measurement) throws IOException {
+ for (TimeSeriesMetadata ts : timeSeriesList) {
+ if (ts.getMeasurementUID().equals(measurement)) {
+ return ts.getType();
+ }
+ }
+ throw new IOException("Measurement " + measurement + " does not exist in the current file.");
+ }
+}
diff --git a/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/TsRowGroupBlockMetaData.java b/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/TsRowGroupBlockMetaData.java
index 4e4798a9..43588a8e 100644
--- a/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/TsRowGroupBlockMetaData.java
+++ b/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/TsRowGroupBlockMetaData.java
@@ -2,84 +2,85 @@
import java.util.ArrayList;
import java.util.List;
-
import cn.edu.tsinghua.tsfile.file.metadata.converter.IConverter;
import cn.edu.tsinghua.tsfile.format.RowGroupBlockMetaData;
-public class TsRowGroupBlockMetaData implements IConverter{
- /**
- * Row groups in this file
- */
- private List rowGroupMetadataList;
-
- private String deltaObjectID;
-
- public TsRowGroupBlockMetaData(){
- rowGroupMetadataList = new ArrayList<>();
- }
-
- public TsRowGroupBlockMetaData(List rowGroupMetadataList){
- this.rowGroupMetadataList = rowGroupMetadataList;
- }
-
- /**
- * add row group metadata to rowGroups. THREAD NOT SAFE
- * @param rowGroup - row group metadata to add
- */
- public void addRowGroupMetaData(RowGroupMetaData rowGroup) {
- if (rowGroupMetadataList == null) {
- rowGroupMetadataList = new ArrayList();
- }
- rowGroupMetadataList.add(rowGroup);
- }
-
- public List getRowGroups() {
- return rowGroupMetadataList;
- }
+public class TsRowGroupBlockMetaData implements IConverter {
+ /**
+ * Row groups in this file
+ */
+ private List rowGroupMetadataList;
+
+ private String deltaObjectID;
- public void setRowGroups(List rowGroupMetadataList) {
- this.rowGroupMetadataList = rowGroupMetadataList;
+ public TsRowGroupBlockMetaData() {
+ rowGroupMetadataList = new ArrayList<>();
+ }
+
+ public TsRowGroupBlockMetaData(List rowGroupMetadataList) {
+ this.rowGroupMetadataList = rowGroupMetadataList;
+ }
+
+ /**
+ * add row group metadata to rowGroups. THREAD NOT SAFE
+ *
+ * @param rowGroup - row group metadata to add
+ */
+ public void addRowGroupMetaData(RowGroupMetaData rowGroup) {
+ if (rowGroupMetadataList == null) {
+ rowGroupMetadataList = new ArrayList();
}
+ rowGroupMetadataList.add(rowGroup);
+ }
+
+ public List getRowGroups() {
+ return rowGroupMetadataList;
+ }
- @Override
- public RowGroupBlockMetaData convertToThrift() {
-// long numOfRows = 0;
- List rowGroupMetaDataListInThrift = null;
- if (rowGroupMetadataList != null) {
- rowGroupMetaDataListInThrift =
- new ArrayList();
- for (RowGroupMetaData rowGroupMetaData : rowGroupMetadataList) {
-// numOfRows += rowGroupMetaData.getNumOfRows();
- rowGroupMetaDataListInThrift.add(rowGroupMetaData.convertToThrift());
- }
- }
- RowGroupBlockMetaData rowGroupBlockMetaData= new RowGroupBlockMetaData(rowGroupMetaDataListInThrift);
- rowGroupBlockMetaData.setDelta_object_id(deltaObjectID);
- return rowGroupBlockMetaData;
- }
+ public void setRowGroups(List rowGroupMetadataList) {
+ this.rowGroupMetadataList = rowGroupMetadataList;
+ }
- @Override
- public void convertToTSF(RowGroupBlockMetaData metadataInThrift) {
- List rowGroupMetaDataListInThrift =
- metadataInThrift.getRow_groups_metadata();
- if (rowGroupMetaDataListInThrift == null) {
- rowGroupMetadataList = null;
- } else {
- rowGroupMetadataList = new ArrayList();
- for (cn.edu.tsinghua.tsfile.format.RowGroupMetaData rowGroupMetaDataInThrift : rowGroupMetaDataListInThrift) {
- RowGroupMetaData rowGroupMetaDataInTSFile = new RowGroupMetaData();
- rowGroupMetaDataInTSFile.convertToTSF(rowGroupMetaDataInThrift);
- rowGroupMetadataList.add(rowGroupMetaDataInTSFile);
- }
- }
- this.deltaObjectID = metadataInThrift.getDelta_object_id();
- }
+ @Override
+ public RowGroupBlockMetaData convertToThrift() {
+ // long numOfRows = 0;
+ List rowGroupMetaDataListInThrift = null;
+ if (rowGroupMetadataList != null) {
+ rowGroupMetaDataListInThrift =
+ new ArrayList();
+ for (RowGroupMetaData rowGroupMetaData : rowGroupMetadataList) {
+ // numOfRows += rowGroupMetaData.getNumOfRows();
+ rowGroupMetaDataListInThrift.add(rowGroupMetaData.convertToThrift());
+ }
+ }
+ RowGroupBlockMetaData rowGroupBlockMetaData =
+ new RowGroupBlockMetaData(rowGroupMetaDataListInThrift);
+ rowGroupBlockMetaData.setDelta_object_id(deltaObjectID);
+ return rowGroupBlockMetaData;
+ }
+
+ @Override
+ public void convertToTSF(RowGroupBlockMetaData metadataInThrift) {
+ List rowGroupMetaDataListInThrift =
+ metadataInThrift.getRow_groups_metadata();
+ if (rowGroupMetaDataListInThrift == null) {
+ rowGroupMetadataList = null;
+ } else {
+ rowGroupMetadataList = new ArrayList();
+ for (cn.edu.tsinghua.tsfile.format.RowGroupMetaData rowGroupMetaDataInThrift : rowGroupMetaDataListInThrift) {
+ RowGroupMetaData rowGroupMetaDataInTSFile = new RowGroupMetaData();
+ rowGroupMetaDataInTSFile.convertToTSF(rowGroupMetaDataInThrift);
+ rowGroupMetadataList.add(rowGroupMetaDataInTSFile);
+ }
+ }
+ this.deltaObjectID = metadataInThrift.getDelta_object_id();
+ }
- public String getDeltaObjectID() {
- return deltaObjectID;
- }
+ public String getDeltaObjectID() {
+ return deltaObjectID;
+ }
- public void setDeltaObjectID(String deltaObjectID) {
- this.deltaObjectID = deltaObjectID;
- }
+ public void setDeltaObjectID(String deltaObjectID) {
+ this.deltaObjectID = deltaObjectID;
+ }
}
diff --git a/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/VInTimeSeriesChunkMetaData.java b/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/VInTimeSeriesChunkMetaData.java
index 3d3ce88c..37c0611a 100644
--- a/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/VInTimeSeriesChunkMetaData.java
+++ b/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/VInTimeSeriesChunkMetaData.java
@@ -1,113 +1,113 @@
-package cn.edu.tsinghua.tsfile.file.metadata;
-
-import cn.edu.tsinghua.tsfile.file.metadata.converter.IConverter;
-import cn.edu.tsinghua.tsfile.file.metadata.enums.TSDataType;
-import cn.edu.tsinghua.tsfile.format.DataType;
-import cn.edu.tsinghua.tsfile.format.ValueInTimeSeriesChunkMetaData;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.List;
-
-/**
- * For more information, see ValueInTimeSeriesChunkMetaData
- * in cn.edu.thu.tsfile.format package
- */
-public class VInTimeSeriesChunkMetaData implements IConverter {
- private static final Logger LOGGER = LoggerFactory.getLogger(VInTimeSeriesChunkMetaData.class);
-
- private TSDataType dataType;
-
- private TsDigest digest;
- private int maxError;
-
- /**
- * If values for data consist of enum values, metadata will store all possible values in time
- * series
- */
- private List enumValues;
-
- public VInTimeSeriesChunkMetaData() {
- }
-
- public VInTimeSeriesChunkMetaData(TSDataType dataType) {
- this.dataType = dataType;
- }
-
- @Override
- public ValueInTimeSeriesChunkMetaData convertToThrift() {
- try {
- ValueInTimeSeriesChunkMetaData vTimeSeriesChunkMetaDataInThrift = new ValueInTimeSeriesChunkMetaData(
- dataType == null ? null : DataType.valueOf(dataType.toString()));
- vTimeSeriesChunkMetaDataInThrift.setMax_error(maxError);
- vTimeSeriesChunkMetaDataInThrift.setEnum_values(enumValues);
- vTimeSeriesChunkMetaDataInThrift.setDigest(digest == null ? null : digest.convertToThrift());
- return vTimeSeriesChunkMetaDataInThrift;
- } catch (Exception e) {
- if (LOGGER.isErrorEnabled())
- LOGGER.error(
- "tsfile-file VInTimeSeriesChunkMetaData: failed to convert ValueInTimeSeriesChunkMetaData from TSFile to thrift, content is {}",
- this, e);
- throw e;
- }
- }
-
- @Override
- public void convertToTSF(ValueInTimeSeriesChunkMetaData vTimeSeriesChunkMetaDataInThrift) {
- try {
- this.dataType = vTimeSeriesChunkMetaDataInThrift.getData_type() == null ? null : TSDataType.valueOf(vTimeSeriesChunkMetaDataInThrift.getData_type().toString());
- this.maxError = vTimeSeriesChunkMetaDataInThrift.getMax_error();
- this.enumValues = vTimeSeriesChunkMetaDataInThrift.getEnum_values();
- if (vTimeSeriesChunkMetaDataInThrift.getDigest() == null) {
- this.digest = null;
- } else {
- this.digest = new TsDigest();
- this.digest.convertToTSF(vTimeSeriesChunkMetaDataInThrift.getDigest());
- }
- } catch (Exception e) {
- if (LOGGER.isErrorEnabled())
- LOGGER.error(
- "tsfile-file VInTimeSeriesChunkMetaData: failed to convert ValueInTimeSeriesChunkMetaData from thrift to TSFile, content is {}",
- vTimeSeriesChunkMetaDataInThrift, e);
- throw e;
- }
- }
-
- @Override
- public String toString() {
- return String.format("VInTimeSeriesChunkMetaData{ TSDataType %s, TSDigest %s, maxError %d, enumValues %s }", dataType, digest,
- maxError, enumValues);
- }
-
- public TSDataType getDataType() {
- return dataType;
- }
-
- public void setDataType(TSDataType dataType) {
- this.dataType = dataType;
- }
-
- public TsDigest getDigest() {
- return digest;
- }
-
- public void setDigest(TsDigest digest) {
- this.digest = digest;
- }
-
- public int getMaxError() {
- return maxError;
- }
-
- public void setMaxError(int maxError) {
- this.maxError = maxError;
- }
-
- public List getEnumValues() {
- return enumValues;
- }
-
- public void setEnumValues(List enumValues) {
- this.enumValues = enumValues;
- }
-}
+package cn.edu.tsinghua.tsfile.file.metadata;
+
+import cn.edu.tsinghua.tsfile.file.metadata.converter.IConverter;
+import cn.edu.tsinghua.tsfile.file.metadata.enums.TSDataType;
+import cn.edu.tsinghua.tsfile.format.DataType;
+import cn.edu.tsinghua.tsfile.format.ValueInTimeSeriesChunkMetaData;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import java.util.List;
+
+/**
+ * For more information, see ValueInTimeSeriesChunkMetaData in cn.edu.thu.tsfile.format package
+ */
+public class VInTimeSeriesChunkMetaData implements IConverter {
+ private static final Logger LOGGER = LoggerFactory.getLogger(VInTimeSeriesChunkMetaData.class);
+
+ private TSDataType dataType;
+
+ private TsDigest digest;
+ private int maxError;
+
+ /**
+ * If values for data consist of enum values, metadata will store all possible values in time
+ * series
+ */
+ private List enumValues;
+
+ public VInTimeSeriesChunkMetaData() {}
+
+ public VInTimeSeriesChunkMetaData(TSDataType dataType) {
+ this.dataType = dataType;
+ }
+
+ @Override
+ public ValueInTimeSeriesChunkMetaData convertToThrift() {
+ try {
+ ValueInTimeSeriesChunkMetaData vTimeSeriesChunkMetaDataInThrift =
+ new ValueInTimeSeriesChunkMetaData(
+ dataType == null ? null : DataType.valueOf(dataType.toString()));
+ vTimeSeriesChunkMetaDataInThrift.setMax_error(maxError);
+ vTimeSeriesChunkMetaDataInThrift.setEnum_values(enumValues);
+ vTimeSeriesChunkMetaDataInThrift.setDigest(digest == null ? null : digest.convertToThrift());
+ return vTimeSeriesChunkMetaDataInThrift;
+ } catch (Exception e) {
+ if (LOGGER.isErrorEnabled())
+ LOGGER.error(
+ "tsfile-file VInTimeSeriesChunkMetaData: failed to convert ValueInTimeSeriesChunkMetaData from TSFile to thrift, content is {}",
+ this, e);
+ throw e;
+ }
+ }
+
+ @Override
+ public void convertToTSF(ValueInTimeSeriesChunkMetaData vTimeSeriesChunkMetaDataInThrift) {
+ try {
+ this.dataType = vTimeSeriesChunkMetaDataInThrift.getData_type() == null ? null
+ : TSDataType.valueOf(vTimeSeriesChunkMetaDataInThrift.getData_type().toString());
+ this.maxError = vTimeSeriesChunkMetaDataInThrift.getMax_error();
+ this.enumValues = vTimeSeriesChunkMetaDataInThrift.getEnum_values();
+ if (vTimeSeriesChunkMetaDataInThrift.getDigest() == null) {
+ this.digest = null;
+ } else {
+ this.digest = new TsDigest();
+ this.digest.convertToTSF(vTimeSeriesChunkMetaDataInThrift.getDigest());
+ }
+ } catch (Exception e) {
+ if (LOGGER.isErrorEnabled())
+ LOGGER.error(
+ "tsfile-file VInTimeSeriesChunkMetaData: failed to convert ValueInTimeSeriesChunkMetaData from thrift to TSFile, content is {}",
+ vTimeSeriesChunkMetaDataInThrift, e);
+ throw e;
+ }
+ }
+
+ @Override
+ public String toString() {
+ return String.format(
+ "VInTimeSeriesChunkMetaData{ TSDataType %s, TSDigest %s, maxError %d, enumValues %s }",
+ dataType, digest, maxError, enumValues);
+ }
+
+ public TSDataType getDataType() {
+ return dataType;
+ }
+
+ public void setDataType(TSDataType dataType) {
+ this.dataType = dataType;
+ }
+
+ public TsDigest getDigest() {
+ return digest;
+ }
+
+ public void setDigest(TsDigest digest) {
+ this.digest = digest;
+ }
+
+ public int getMaxError() {
+ return maxError;
+ }
+
+ public void setMaxError(int maxError) {
+ this.maxError = maxError;
+ }
+
+ public List getEnumValues() {
+ return enumValues;
+ }
+
+ public void setEnumValues(List enumValues) {
+ this.enumValues = enumValues;
+ }
+}
diff --git a/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/converter/IConverter.java b/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/converter/IConverter.java
index d03324c6..ffcec759 100644
--- a/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/converter/IConverter.java
+++ b/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/converter/IConverter.java
@@ -1,22 +1,22 @@
-package cn.edu.tsinghua.tsfile.file.metadata.converter;
-
-/**
- * convert metadata between TSFile format and thrift format
- *
- * @param TsFile-defined type
- */
-public interface IConverter {
- /**
- * convert TSFile format metadata to thrift format
- *
- * @return metadata in thrift format
- */
- T convertToThrift();
-
- /**
- * convert thrift format metadata to TSFile format
- *
- * @param metadata metadata in thrift format
- */
- void convertToTSF(T metadata);
-}
+package cn.edu.tsinghua.tsfile.file.metadata.converter;
+
+/**
+ * convert metadata between TSFile format and thrift format
+ *
+ * @param TsFile-defined type
+ */
+public interface IConverter {
+ /**
+ * convert TSFile format metadata to thrift format
+ *
+ * @return metadata in thrift format
+ */
+ T convertToThrift();
+
+ /**
+ * convert thrift format metadata to TSFile format
+ *
+ * @param metadata metadata in thrift format
+ */
+ void convertToTSF(T metadata);
+}
diff --git a/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/converter/TsFileMetaDataConverter.java b/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/converter/TsFileMetaDataConverter.java
index 4e114b48..d3b66d7d 100644
--- a/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/converter/TsFileMetaDataConverter.java
+++ b/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/converter/TsFileMetaDataConverter.java
@@ -1,55 +1,58 @@
-package cn.edu.tsinghua.tsfile.file.metadata.converter;
-
-import cn.edu.tsinghua.tsfile.file.metadata.TsFileMetaData;
-import cn.edu.tsinghua.tsfile.format.FileMetaData;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * converter for file metadata
- */
-public class TsFileMetaDataConverter {
- private static final Logger LOGGER = LoggerFactory.getLogger(TsFileMetaDataConverter.class);
-
- /**
- * convert tsfile format file matadata to thrift format file matadata
- *
- * @param fileMetadataInTsFile file metadata in tsfile format
- * @return file metadata in thrift format
- */
- public FileMetaData toThriftFileMetadata(TsFileMetaData fileMetadataInTsFile) {
- try {
- return fileMetadataInTsFile.convertToThrift();
- } catch (Exception e) {
- LOGGER.error("TsFileMetaDataConverter: failed to convert metadata from TsFile to thrift, content is {}",
- fileMetadataInTsFile, e);
- }
- return null;
- }
-
- /**
- * convert thrift format file matadata to tsfile format file matadata
- *
- * @param fileMetaDataInThrift file metadata in thrift format
- * @return file metadata in tsfile format
- */
- public TsFileMetaData toTsFileMetadata(FileMetaData fileMetaDataInThrift) {
- TsFileMetaData fileMetaDataInTSFile = new TsFileMetaData();
- try {
- fileMetaDataInTSFile.convertToTSF(fileMetaDataInThrift);
- } catch (Exception e) {
- LOGGER.error("TsFileMetaDataConverter: failed to convert metadata from thrift to TSFile, content is {}",
- fileMetaDataInThrift, e);
- }
- return fileMetaDataInTSFile;
- }
-
- public int calculatePageHeaderSize(int digestSize) {
- //PageHeader: PageType--4, uncompressedSize--4,compressedSize--4
- //DatapageHeader: numValues--4, numNulls--4, numRows--4, Encoding--4, isCompressed--1, maxTimestamp--8, minTimestamp--8
- //Digest: max ByteBuffer, min ByteBuffer
- // * 2 to caculate max object size in memory
-
- return 2 * (45 + digestSize);
- }
-}
+package cn.edu.tsinghua.tsfile.file.metadata.converter;
+
+import cn.edu.tsinghua.tsfile.file.metadata.TsFileMetaData;
+import cn.edu.tsinghua.tsfile.format.FileMetaData;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * converter for file metadata
+ */
+public class TsFileMetaDataConverter {
+ private static final Logger LOGGER = LoggerFactory.getLogger(TsFileMetaDataConverter.class);
+
+ /**
+ * convert tsfile format file matadata to thrift format file matadata
+ *
+ * @param fileMetadataInTsFile file metadata in tsfile format
+ * @return file metadata in thrift format
+ */
+ public FileMetaData toThriftFileMetadata(TsFileMetaData fileMetadataInTsFile) {
+ try {
+ return fileMetadataInTsFile.convertToThrift();
+ } catch (Exception e) {
+ LOGGER.error(
+ "TsFileMetaDataConverter: failed to convert metadata from TsFile to thrift, content is {}",
+ fileMetadataInTsFile, e);
+ }
+ return null;
+ }
+
+ /**
+ * convert thrift format file matadata to tsfile format file matadata
+ *
+ * @param fileMetaDataInThrift file metadata in thrift format
+ * @return file metadata in tsfile format
+ */
+ public TsFileMetaData toTsFileMetadata(FileMetaData fileMetaDataInThrift) {
+ TsFileMetaData fileMetaDataInTSFile = new TsFileMetaData();
+ try {
+ fileMetaDataInTSFile.convertToTSF(fileMetaDataInThrift);
+ } catch (Exception e) {
+ LOGGER.error(
+ "TsFileMetaDataConverter: failed to convert metadata from thrift to TSFile, content is {}",
+ fileMetaDataInThrift, e);
+ }
+ return fileMetaDataInTSFile;
+ }
+
+ public int calculatePageHeaderSize(int digestSize) {
+ // PageHeader: PageType--4, uncompressedSize--4,compressedSize--4
+ // DatapageHeader: numValues--4, numNulls--4, numRows--4, Encoding--4, isCompressed--1,
+ // maxTimestamp--8, minTimestamp--8
+ // Digest: max ByteBuffer, min ByteBuffer
+ // * 2 to caculate max object size in memory
+
+ return 2 * (45 + digestSize);
+ }
+}
diff --git a/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/enums/CompressionTypeName.java b/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/enums/CompressionTypeName.java
index 4caee864..2ac7cf21 100644
--- a/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/enums/CompressionTypeName.java
+++ b/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/enums/CompressionTypeName.java
@@ -1,53 +1,50 @@
-package cn.edu.tsinghua.tsfile.file.metadata.enums;
-
-import cn.edu.tsinghua.tsfile.common.exception.CompressionTypeNotSupportedException;
-import cn.edu.tsinghua.tsfile.format.CompressionType;
-
-public enum CompressionTypeName {
- UNCOMPRESSED(CompressionType.UNCOMPRESSED, ""),
- SNAPPY(CompressionType.SNAPPY, ".snappy"),
- GZIP(CompressionType.GZIP, ".gz"),
- LZO(CompressionType.LZO, ".lzo"),
- SDT(CompressionType.SDT, ".sdt"),
- PAA(CompressionType.PAA, ".paa"),
- PLA(CompressionType.PLA, ".pla");
-
- private final CompressionType tsfileCompressionType;
- private final String extension;
- private CompressionTypeName(CompressionType tsfileCompressionType, String extension) {
- this.tsfileCompressionType = tsfileCompressionType;
- this.extension = extension;
- }
-
- public static CompressionTypeName fromConf(String name) {
- if (name == null) {
- return UNCOMPRESSED;
- }
- switch (name.trim().toUpperCase()) {
- case "UNCOMPRESSED":
- return UNCOMPRESSED;
- case "SNAPPY":
- return SNAPPY;
- case "GZIP":
- return GZIP;
- case "LZO":
- return LZO;
- case "SDT":
- return SDT;
- case "PAA":
- return PAA;
- case "PLA":
- return PLA;
- default:
- throw new CompressionTypeNotSupportedException(name);
- }
- }
-
- public CompressionType getTsfileCompressionCodec() {
- return tsfileCompressionType;
- }
-
- public String getExtension() {
- return extension;
- }
-}
\ No newline at end of file
+package cn.edu.tsinghua.tsfile.file.metadata.enums;
+
+import cn.edu.tsinghua.tsfile.common.exception.CompressionTypeNotSupportedException;
+import cn.edu.tsinghua.tsfile.format.CompressionType;
+
+public enum CompressionTypeName {
+ UNCOMPRESSED(CompressionType.UNCOMPRESSED, ""), SNAPPY(CompressionType.SNAPPY, ".snappy"), GZIP(
+ CompressionType.GZIP, ".gz"), LZO(CompressionType.LZO, ".lzo"), SDT(CompressionType.SDT,
+ ".sdt"), PAA(CompressionType.PAA, ".paa"), PLA(CompressionType.PLA, ".pla");
+
+ private final CompressionType tsfileCompressionType;
+ private final String extension;
+
+ private CompressionTypeName(CompressionType tsfileCompressionType, String extension) {
+ this.tsfileCompressionType = tsfileCompressionType;
+ this.extension = extension;
+ }
+
+ public static CompressionTypeName fromConf(String name) {
+ if (name == null) {
+ return UNCOMPRESSED;
+ }
+ switch (name.trim().toUpperCase()) {
+ case "UNCOMPRESSED":
+ return UNCOMPRESSED;
+ case "SNAPPY":
+ return SNAPPY;
+ case "GZIP":
+ return GZIP;
+ case "LZO":
+ return LZO;
+ case "SDT":
+ return SDT;
+ case "PAA":
+ return PAA;
+ case "PLA":
+ return PLA;
+ default:
+ throw new CompressionTypeNotSupportedException(name);
+ }
+ }
+
+ public CompressionType getTsfileCompressionCodec() {
+ return tsfileCompressionType;
+ }
+
+ public String getExtension() {
+ return extension;
+ }
+}
diff --git a/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/enums/TSChunkType.java b/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/enums/TSChunkType.java
index 30147cdd..9f9188ca 100644
--- a/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/enums/TSChunkType.java
+++ b/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/enums/TSChunkType.java
@@ -1,5 +1,5 @@
-package cn.edu.tsinghua.tsfile.file.metadata.enums;
-
-public enum TSChunkType {
- TIME, VALUE
-}
+package cn.edu.tsinghua.tsfile.file.metadata.enums;
+
+public enum TSChunkType {
+ TIME, VALUE
+}
diff --git a/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/enums/TSDataType.java b/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/enums/TSDataType.java
index b10caec1..3b8c0c43 100644
--- a/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/enums/TSDataType.java
+++ b/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/enums/TSDataType.java
@@ -1,5 +1,5 @@
-package cn.edu.tsinghua.tsfile.file.metadata.enums;
-
-public enum TSDataType {
- BOOLEAN, INT32, INT64, INT96, FLOAT, DOUBLE, TEXT, FIXED_LEN_BYTE_ARRAY, ENUMS, BIGDECIMAL
-}
+package cn.edu.tsinghua.tsfile.file.metadata.enums;
+
+public enum TSDataType {
+ BOOLEAN, INT32, INT64, INT96, FLOAT, DOUBLE, TEXT, FIXED_LEN_BYTE_ARRAY, ENUMS, BIGDECIMAL
+}
diff --git a/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/enums/TSEncoding.java b/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/enums/TSEncoding.java
index 6c6f3d07..222cc814 100644
--- a/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/enums/TSEncoding.java
+++ b/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/enums/TSEncoding.java
@@ -1,5 +1,5 @@
-package cn.edu.tsinghua.tsfile.file.metadata.enums;
-
-public enum TSEncoding {
- PLAIN, PLAIN_DICTIONARY, RLE, DIFF, TS_2DIFF, BITMAP, GORILLA
-}
+package cn.edu.tsinghua.tsfile.file.metadata.enums;
+
+public enum TSEncoding {
+ PLAIN, PLAIN_DICTIONARY, RLE, DIFF, TS_2DIFF, BITMAP, GORILLA
+}
diff --git a/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/enums/TSFreqType.java b/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/enums/TSFreqType.java
index e8a469a8..61e882d5 100644
--- a/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/enums/TSFreqType.java
+++ b/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/enums/TSFreqType.java
@@ -1,5 +1,5 @@
-package cn.edu.tsinghua.tsfile.file.metadata.enums;
-
-public enum TSFreqType {
- SINGLE_FREQ, MULTI_FREQ, IRREGULAR_FREQ
-}
+package cn.edu.tsinghua.tsfile.file.metadata.enums;
+
+public enum TSFreqType {
+ SINGLE_FREQ, MULTI_FREQ, IRREGULAR_FREQ
+}
diff --git a/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/enums/TSPageType.java b/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/enums/TSPageType.java
index 4c3b77ba..84cc1df4 100644
--- a/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/enums/TSPageType.java
+++ b/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/enums/TSPageType.java
@@ -1,5 +1,5 @@
-package cn.edu.tsinghua.tsfile.file.metadata.enums;
-
-public enum TSPageType {
- DATA, DICTIONARY, INDEX
-}
+package cn.edu.tsinghua.tsfile.file.metadata.enums;
+
+public enum TSPageType {
+ DATA, DICTIONARY, INDEX
+}
diff --git a/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/statistics/BigDecimalStatistics.java b/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/statistics/BigDecimalStatistics.java
index 758128a6..3b86c4e4 100644
--- a/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/statistics/BigDecimalStatistics.java
+++ b/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/statistics/BigDecimalStatistics.java
@@ -1,7 +1,6 @@
package cn.edu.tsinghua.tsfile.file.metadata.statistics;
import cn.edu.tsinghua.tsfile.common.utils.BytesUtils;
-
import java.math.BigDecimal;
/**
@@ -10,114 +9,116 @@
* @author kangrong
*/
public class BigDecimalStatistics extends Statistics {
- private BigDecimal max;
- private BigDecimal min;
- private BigDecimal first;
- private double sum;
- private BigDecimal last;
-
- @Override
- public void updateStats(BigDecimal value) {
- if (this.isEmpty) {
- initializeStats(value, value, value, value.doubleValue(), value);
- isEmpty = false;
- } else {
- updateStats(value, value, value, value.doubleValue(), value);
- }
- }
-
- private void updateStats(BigDecimal minValue, BigDecimal maxValue, BigDecimal firstValue, double sumValue,
- BigDecimal lastValue) {
- if (minValue.doubleValue() < min.doubleValue()) {
- min = minValue;
- }
- if (maxValue.doubleValue() > max.doubleValue()) {
- max = maxValue;
- }
- sum += sumValue;
- this.last = lastValue;
- }
-
- @Override
- public BigDecimal getMax() {
- return max;
- }
-
- @Override
- public BigDecimal getMin() {
- return min;
- }
-
- @Override
- public BigDecimal getFirst() {
- return first;
- }
-
- @Override
- public double getSum() {
- return sum;
- }
-
- @Override
- public BigDecimal getLast() {
- return last;
- }
-
- @Override
- protected void mergeStatisticsValue(Statistics stats) {
- BigDecimalStatistics bigDecimalStats = (BigDecimalStatistics) stats;
- if (this.isEmpty) {
- initializeStats(bigDecimalStats.getMin(), bigDecimalStats.getMax(), bigDecimalStats.getFirst(),
- bigDecimalStats.getSum(), bigDecimalStats.getLast());
- isEmpty = false;
- } else {
- updateStats(bigDecimalStats.getMin(), bigDecimalStats.getMax(), bigDecimalStats.getFirst(),
- bigDecimalStats.getSum(), bigDecimalStats.getLast());
- }
-
- }
-
- public void initializeStats(BigDecimal min, BigDecimal max, BigDecimal first, double sum, BigDecimal last) {
- this.min = min;
- this.max = max;
- this.first = first;
- this.sum = sum;
- this.last = last;
- }
-
- @Override
- public byte[] getMaxBytes() {
- return BytesUtils.doubleToBytes(max.doubleValue());
- }
-
- @Override
- public byte[] getMinBytes() {
- return BytesUtils.doubleToBytes(min.doubleValue());
- }
-
- @Override
- public byte[] getFirstBytes() {
- return BytesUtils.doubleToBytes(first.doubleValue());
- }
-
- @Override
- public byte[] getSumBytes() {
- return BytesUtils.doubleToBytes(sum);
- }
-
- @Override
- public byte[] getLastBytes(){
- return BytesUtils.doubleToBytes(last.doubleValue());
- }
-
- @Override
- public void setMinMaxFromBytes(byte[] minBytes, byte[] maxBytes) {
- max = new BigDecimal(BytesUtils.bytesToDouble(maxBytes));
- min = new BigDecimal(BytesUtils.bytesToDouble(minBytes));
- }
-
- @Override
- public String toString() {
- return "[max:" + max + ",min:" + min + ",first:" + first + ",sum:" + sum + ",last:" + last + "]";
- }
+ private BigDecimal max;
+ private BigDecimal min;
+ private BigDecimal first;
+ private double sum;
+ private BigDecimal last;
+
+ @Override
+ public void updateStats(BigDecimal value) {
+ if (this.isEmpty) {
+ initializeStats(value, value, value, value.doubleValue(), value);
+ isEmpty = false;
+ } else {
+ updateStats(value, value, value, value.doubleValue(), value);
+ }
+ }
+
+ private void updateStats(BigDecimal minValue, BigDecimal maxValue, BigDecimal firstValue,
+ double sumValue, BigDecimal lastValue) {
+ if (minValue.doubleValue() < min.doubleValue()) {
+ min = minValue;
+ }
+ if (maxValue.doubleValue() > max.doubleValue()) {
+ max = maxValue;
+ }
+ sum += sumValue;
+ this.last = lastValue;
+ }
+
+ @Override
+ public BigDecimal getMax() {
+ return max;
+ }
+
+ @Override
+ public BigDecimal getMin() {
+ return min;
+ }
+
+ @Override
+ public BigDecimal getFirst() {
+ return first;
+ }
+
+ @Override
+ public double getSum() {
+ return sum;
+ }
+
+ @Override
+ public BigDecimal getLast() {
+ return last;
+ }
+
+ @Override
+ protected void mergeStatisticsValue(Statistics stats) {
+ BigDecimalStatistics bigDecimalStats = (BigDecimalStatistics) stats;
+ if (this.isEmpty) {
+ initializeStats(bigDecimalStats.getMin(), bigDecimalStats.getMax(),
+ bigDecimalStats.getFirst(), bigDecimalStats.getSum(), bigDecimalStats.getLast());
+ isEmpty = false;
+ } else {
+ updateStats(bigDecimalStats.getMin(), bigDecimalStats.getMax(), bigDecimalStats.getFirst(),
+ bigDecimalStats.getSum(), bigDecimalStats.getLast());
+ }
+
+ }
+
+ public void initializeStats(BigDecimal min, BigDecimal max, BigDecimal first, double sum,
+ BigDecimal last) {
+ this.min = min;
+ this.max = max;
+ this.first = first;
+ this.sum = sum;
+ this.last = last;
+ }
+
+ @Override
+ public byte[] getMaxBytes() {
+ return BytesUtils.doubleToBytes(max.doubleValue());
+ }
+
+ @Override
+ public byte[] getMinBytes() {
+ return BytesUtils.doubleToBytes(min.doubleValue());
+ }
+
+ @Override
+ public byte[] getFirstBytes() {
+ return BytesUtils.doubleToBytes(first.doubleValue());
+ }
+
+ @Override
+ public byte[] getSumBytes() {
+ return BytesUtils.doubleToBytes(sum);
+ }
+
+ @Override
+ public byte[] getLastBytes() {
+ return BytesUtils.doubleToBytes(last.doubleValue());
+ }
+
+ @Override
+ public void setMinMaxFromBytes(byte[] minBytes, byte[] maxBytes) {
+ max = new BigDecimal(BytesUtils.bytesToDouble(maxBytes));
+ min = new BigDecimal(BytesUtils.bytesToDouble(minBytes));
+ }
+
+ @Override
+ public String toString() {
+ return "[max:" + max + ",min:" + min + ",first:" + first + ",sum:" + sum + ",last:" + last
+ + "]";
+ }
}
diff --git a/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/statistics/BinaryStatistics.java b/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/statistics/BinaryStatistics.java
index 2816f6b7..99f57641 100644
--- a/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/statistics/BinaryStatistics.java
+++ b/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/statistics/BinaryStatistics.java
@@ -9,110 +9,114 @@
* @author CGF
*/
public class BinaryStatistics extends Statistics {
- private Binary max = new Binary("");
- private Binary min = new Binary("");
- private Binary first = new Binary("");
- private double sum;
- private Binary last = new Binary("");
-
- @Override
- public void setMinMaxFromBytes(byte[] minBytes, byte[] maxBytes) {
- max = new Binary(maxBytes);
- min = new Binary(minBytes);
- }
-
- @Override
- public Binary getMin() {
- return min;
- }
-
- @Override
- public Binary getMax() {
- return max;
- }
-
- @Override
- public Binary getFirst() {
- return first;
- }
-
- @Override
- public double getSum() {
- return sum;
- }
-
- @Override
- public Binary getLast(){
- return last;
- }
-
- public void initializeStats(Binary min, Binary max, Binary first, double sum,Binary last) {
- this.min = min;
- this.max = max;
- this.first = first;
- this.sum = sum;
- this.last = last;
- }
-
- @Override
- protected void mergeStatisticsValue(Statistics stats) {
- BinaryStatistics stringStats = (BinaryStatistics) stats;
- if (isEmpty) {
- initializeStats(stringStats.getMin(), stringStats.getMax(), stringStats.getFirst(), stringStats.getSum(),stringStats.getLast());
- isEmpty = false;
- } else {
- updateStats(stringStats.getMin(), stringStats.getMax(), stringStats.getFirst(), stringStats.getSum(),stringStats.getLast());
- }
- }
-
- @Override
- public void updateStats(Binary value) {
- if (isEmpty) {
- initializeStats(value, value, value, 0,value);
- isEmpty = false;
- } else {
- updateStats(value, value, value, 0,value);
- isEmpty = false;
- }
- }
-
- private void updateStats(Binary minValue, Binary maxValue, Binary firstValue, double sum,Binary lastValue) {
- if (minValue.compareTo(min) < 0) {
- min = minValue;
- }
- if (maxValue.compareTo(max) > 0) {
- max = maxValue;
- }
- this.last = lastValue;
- }
-
- @Override
- public byte[] getMaxBytes() {
- return BytesUtils.StringToBytes(max.getStringValue());
- }
-
- @Override
- public byte[] getMinBytes() {
- return BytesUtils.StringToBytes(min.getStringValue());
- }
-
- @Override
- public byte[] getFirstBytes() {
- return BytesUtils.StringToBytes(first.getStringValue());
- }
-
- @Override
- public byte[] getSumBytes() {
- return BytesUtils.doubleToBytes(sum);
- }
-
- @Override
- public byte[] getLastBytes(){
- return BytesUtils.StringToBytes(last.getStringValue());
- }
-
- @Override
- public String toString(){
- return "[max:" + max + ",min:" + min + ",first:" + first + ",sum:" + sum + ",last:" + last + "]";
- }
+ private Binary max = new Binary("");
+ private Binary min = new Binary("");
+ private Binary first = new Binary("");
+ private double sum;
+ private Binary last = new Binary("");
+
+ @Override
+ public void setMinMaxFromBytes(byte[] minBytes, byte[] maxBytes) {
+ max = new Binary(maxBytes);
+ min = new Binary(minBytes);
+ }
+
+ @Override
+ public Binary getMin() {
+ return min;
+ }
+
+ @Override
+ public Binary getMax() {
+ return max;
+ }
+
+ @Override
+ public Binary getFirst() {
+ return first;
+ }
+
+ @Override
+ public double getSum() {
+ return sum;
+ }
+
+ @Override
+ public Binary getLast() {
+ return last;
+ }
+
+ public void initializeStats(Binary min, Binary max, Binary first, double sum, Binary last) {
+ this.min = min;
+ this.max = max;
+ this.first = first;
+ this.sum = sum;
+ this.last = last;
+ }
+
+ @Override
+ protected void mergeStatisticsValue(Statistics stats) {
+ BinaryStatistics stringStats = (BinaryStatistics) stats;
+ if (isEmpty) {
+ initializeStats(stringStats.getMin(), stringStats.getMax(), stringStats.getFirst(),
+ stringStats.getSum(), stringStats.getLast());
+ isEmpty = false;
+ } else {
+ updateStats(stringStats.getMin(), stringStats.getMax(), stringStats.getFirst(),
+ stringStats.getSum(), stringStats.getLast());
+ }
+ }
+
+ @Override
+ public void updateStats(Binary value) {
+ if (isEmpty) {
+ initializeStats(value, value, value, 0, value);
+ isEmpty = false;
+ } else {
+ updateStats(value, value, value, 0, value);
+ isEmpty = false;
+ }
+ }
+
+ private void updateStats(Binary minValue, Binary maxValue, Binary firstValue, double sum,
+ Binary lastValue) {
+ if (minValue.compareTo(min) < 0) {
+ min = minValue;
+ }
+ if (maxValue.compareTo(max) > 0) {
+ max = maxValue;
+ }
+ this.last = lastValue;
+ }
+
+ @Override
+ public byte[] getMaxBytes() {
+ return BytesUtils.StringToBytes(max.getStringValue());
+ }
+
+ @Override
+ public byte[] getMinBytes() {
+ return BytesUtils.StringToBytes(min.getStringValue());
+ }
+
+ @Override
+ public byte[] getFirstBytes() {
+ return BytesUtils.StringToBytes(first.getStringValue());
+ }
+
+ @Override
+ public byte[] getSumBytes() {
+ return BytesUtils.doubleToBytes(sum);
+ }
+
+ @Override
+ public byte[] getLastBytes() {
+ return BytesUtils.StringToBytes(last.getStringValue());
+ }
+
+ @Override
+ public String toString() {
+ return "[max:" + max + ",min:" + min + ",first:" + first + ",sum:" + sum + ",last:" + last
+ + "]";
+ }
}
diff --git a/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/statistics/BooleanStatistics.java b/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/statistics/BooleanStatistics.java
index df458335..36b169f6 100644
--- a/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/statistics/BooleanStatistics.java
+++ b/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/statistics/BooleanStatistics.java
@@ -6,112 +6,114 @@
* @author CGF
*/
public class BooleanStatistics extends Statistics {
- private boolean max;
- private boolean min;
- private boolean first;
- private double sum;
- private boolean last;
-
- @Override
- public void setMinMaxFromBytes(byte[] minBytes, byte[] maxBytes) {
- max = BytesUtils.bytesToBool(maxBytes);
- min = BytesUtils.bytesToBool(minBytes);
- }
-
- @Override
- public void updateStats(boolean value) {
- if (isEmpty) {
- initializeStats(value, value, value, 0, value);
- isEmpty = false;
- } else {
- updateStats(value, value, value, 0, value);
- isEmpty = false;
- }
- }
-
- private void updateStats(boolean minValue, boolean maxValue, boolean firstValue, double sumValue,
- boolean lastValue) {
- if (!minValue && min) {
- min = minValue;
- }
- if (maxValue && !max) {
- max = maxValue;
- }
- this.last = lastValue;
- }
-
- @Override
- public Boolean getMax() {
- return max;
- }
-
- @Override
- public Boolean getMin() {
- return min;
- }
-
- @Override
- public Boolean getFirst() {
- return first;
- }
-
- @Override
- public double getSum() {
- return sum;
- }
-
- @Override
- public Boolean getLast(){
- return last;
- }
-
- @Override
- protected void mergeStatisticsValue(Statistics stats) {
- BooleanStatistics boolStats = (BooleanStatistics) stats;
- if (isEmpty) {
- initializeStats(boolStats.getMin(), boolStats.getMax(), boolStats.getFirst(), boolStats.getSum(),
- boolStats.getLast());
- isEmpty = false;
- } else {
- updateStats(boolStats.getMin(), boolStats.getMax(), boolStats.getFirst(), boolStats.getSum(),
- boolStats.getLast());
- }
- }
-
- public void initializeStats(boolean min, boolean max, boolean firstValue, double sumValue, boolean lastValue) {
- this.min = min;
- this.max = max;
- this.first = firstValue;
- this.last = lastValue;
- }
-
- @Override
- public byte[] getMaxBytes() {
- return BytesUtils.boolToBytes(max);
- }
-
- @Override
- public byte[] getMinBytes() {
- return BytesUtils.boolToBytes(min);
- }
-
- @Override
- public byte[] getFirstBytes() {
- return BytesUtils.boolToBytes(first);
- }
-
- @Override
- public byte[] getSumBytes() {
- return BytesUtils.doubleToBytes(sum);
- }
-
- @Override
- public byte[] getLastBytes() {
- return BytesUtils.boolToBytes(last);
- }
-
- @Override
- public String toString() {
- return "[max:" + max + ",min:" + min + ",first:" + first + ",sum:" + sum + ",last:" + last + "]";
- }
+ private boolean max;
+ private boolean min;
+ private boolean first;
+ private double sum;
+ private boolean last;
+
+ @Override
+ public void setMinMaxFromBytes(byte[] minBytes, byte[] maxBytes) {
+ max = BytesUtils.bytesToBool(maxBytes);
+ min = BytesUtils.bytesToBool(minBytes);
+ }
+
+ @Override
+ public void updateStats(boolean value) {
+ if (isEmpty) {
+ initializeStats(value, value, value, 0, value);
+ isEmpty = false;
+ } else {
+ updateStats(value, value, value, 0, value);
+ isEmpty = false;
+ }
+ }
+
+ private void updateStats(boolean minValue, boolean maxValue, boolean firstValue, double sumValue,
+ boolean lastValue) {
+ if (!minValue && min) {
+ min = minValue;
+ }
+ if (maxValue && !max) {
+ max = maxValue;
+ }
+ this.last = lastValue;
+ }
+
+ @Override
+ public Boolean getMax() {
+ return max;
+ }
+
+ @Override
+ public Boolean getMin() {
+ return min;
+ }
+
+ @Override
+ public Boolean getFirst() {
+ return first;
+ }
+
+ @Override
+ public double getSum() {
+ return sum;
+ }
+
+ @Override
+ public Boolean getLast() {
+ return last;
+ }
+
+ @Override
+ protected void mergeStatisticsValue(Statistics stats) {
+ BooleanStatistics boolStats = (BooleanStatistics) stats;
+ if (isEmpty) {
+ initializeStats(boolStats.getMin(), boolStats.getMax(), boolStats.getFirst(),
+ boolStats.getSum(), boolStats.getLast());
+ isEmpty = false;
+ } else {
+ updateStats(boolStats.getMin(), boolStats.getMax(), boolStats.getFirst(), boolStats.getSum(),
+ boolStats.getLast());
+ }
+ }
+
+ public void initializeStats(boolean min, boolean max, boolean firstValue, double sumValue,
+ boolean lastValue) {
+ this.min = min;
+ this.max = max;
+ this.first = firstValue;
+ this.last = lastValue;
+ }
+
+ @Override
+ public byte[] getMaxBytes() {
+ return BytesUtils.boolToBytes(max);
+ }
+
+ @Override
+ public byte[] getMinBytes() {
+ return BytesUtils.boolToBytes(min);
+ }
+
+ @Override
+ public byte[] getFirstBytes() {
+ return BytesUtils.boolToBytes(first);
+ }
+
+ @Override
+ public byte[] getSumBytes() {
+ return BytesUtils.doubleToBytes(sum);
+ }
+
+ @Override
+ public byte[] getLastBytes() {
+ return BytesUtils.boolToBytes(last);
+ }
+
+ @Override
+ public String toString() {
+ return "[max:" + max + ",min:" + min + ",first:" + first + ",sum:" + sum + ",last:" + last
+ + "]";
+ }
}
diff --git a/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/statistics/DoubleStatistics.java b/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/statistics/DoubleStatistics.java
index 740fcce1..c1965bc3 100644
--- a/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/statistics/DoubleStatistics.java
+++ b/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/statistics/DoubleStatistics.java
@@ -8,112 +8,116 @@
* @author kangrong
*/
public class DoubleStatistics extends Statistics {
- private double max;
- private double min;
- private double first;
- private double sum;
- private double last;
-
- @Override
- public void setMinMaxFromBytes(byte[] minBytes, byte[] maxBytes) {
- max = BytesUtils.bytesToDouble(maxBytes);
- min = BytesUtils.bytesToDouble(minBytes);
- }
-
- @Override
- public void updateStats(double value) {
- if (this.isEmpty) {
- initializeStats(value, value, value, value,value);
- isEmpty = false;
- } else {
- updateStats(value, value, value, value,value);
- }
- }
-
- private void updateStats(double minValue, double maxValue, double firstValue, double sumValue,double lastValue) {
- if (minValue < min) {
- min = minValue;
- }
- if (maxValue > max) {
- max = maxValue;
- }
- sum += sumValue;
- this.last = lastValue;
- }
-
- @Override
- public Double getMax() {
- return max;
- }
-
- @Override
- public Double getMin() {
- return min;
- }
-
- @Override
- public Double getFirst() {
- return first;
- }
-
- @Override
- public double getSum() {
- return sum;
- }
-
- @Override
- public Double getLast(){
- return last;
- }
-
- @Override
- protected void mergeStatisticsValue(Statistics stats) {
- DoubleStatistics doubleStats = (DoubleStatistics) stats;
- if (this.isEmpty) {
- initializeStats(doubleStats.getMin(), doubleStats.getMax(), doubleStats.getFirst(), doubleStats.getSum(),doubleStats.getLast());
- isEmpty = false;
- } else {
- updateStats(doubleStats.getMin(), doubleStats.getMax(), doubleStats.getFirst(), doubleStats.getSum(),doubleStats.getLast());
- }
-
- }
-
- public void initializeStats(double min, double max, double first, double sum,double last) {
- this.min = min;
- this.max = max;
- this.first = first;
- this.sum = sum;
- this.last = last;
- }
-
- @Override
- public byte[] getMaxBytes() {
- return BytesUtils.doubleToBytes(max);
- }
-
- @Override
- public byte[] getMinBytes() {
- return BytesUtils.doubleToBytes(min);
- }
-
- @Override
- public byte[] getFirstBytes() {
- return BytesUtils.doubleToBytes(first);
- }
-
- @Override
- public byte[] getSumBytes() {
- return BytesUtils.doubleToBytes(sum);
- }
-
- @Override
- public byte[] getLastBytes(){
- return BytesUtils.doubleToBytes(last);
- }
-
- @Override
- public String toString() {
- return "[max:" + max + ",min:" + min + ",first:" + first + ",sum:" + sum + ",last:" + last + "]";
- }
+ private double max;
+ private double min;
+ private double first;
+ private double sum;
+ private double last;
+
+ @Override
+ public void setMinMaxFromBytes(byte[] minBytes, byte[] maxBytes) {
+ max = BytesUtils.bytesToDouble(maxBytes);
+ min = BytesUtils.bytesToDouble(minBytes);
+ }
+
+ @Override
+ public void updateStats(double value) {
+ if (this.isEmpty) {
+ initializeStats(value, value, value, value, value);
+ isEmpty = false;
+ } else {
+ updateStats(value, value, value, value, value);
+ }
+ }
+
+ private void updateStats(double minValue, double maxValue, double firstValue, double sumValue,
+ double lastValue) {
+ if (minValue < min) {
+ min = minValue;
+ }
+ if (maxValue > max) {
+ max = maxValue;
+ }
+ sum += sumValue;
+ this.last = lastValue;
+ }
+
+ @Override
+ public Double getMax() {
+ return max;
+ }
+
+ @Override
+ public Double getMin() {
+ return min;
+ }
+
+ @Override
+ public Double getFirst() {
+ return first;
+ }
+
+ @Override
+ public double getSum() {
+ return sum;
+ }
+
+ @Override
+ public Double getLast() {
+ return last;
+ }
+
+ @Override
+ protected void mergeStatisticsValue(Statistics stats) {
+ DoubleStatistics doubleStats = (DoubleStatistics) stats;
+ if (this.isEmpty) {
+ initializeStats(doubleStats.getMin(), doubleStats.getMax(), doubleStats.getFirst(),
+ doubleStats.getSum(), doubleStats.getLast());
+ isEmpty = false;
+ } else {
+ updateStats(doubleStats.getMin(), doubleStats.getMax(), doubleStats.getFirst(),
+ doubleStats.getSum(), doubleStats.getLast());
+ }
+
+ }
+
+ public void initializeStats(double min, double max, double first, double sum, double last) {
+ this.min = min;
+ this.max = max;
+ this.first = first;
+ this.sum = sum;
+ this.last = last;
+ }
+
+ @Override
+ public byte[] getMaxBytes() {
+ return BytesUtils.doubleToBytes(max);
+ }
+
+ @Override
+ public byte[] getMinBytes() {
+ return BytesUtils.doubleToBytes(min);
+ }
+
+ @Override
+ public byte[] getFirstBytes() {
+ return BytesUtils.doubleToBytes(first);
+ }
+
+ @Override
+ public byte[] getSumBytes() {
+ return BytesUtils.doubleToBytes(sum);
+ }
+
+ @Override
+ public byte[] getLastBytes() {
+ return BytesUtils.doubleToBytes(last);
+ }
+
+ @Override
+ public String toString() {
+ return "[max:" + max + ",min:" + min + ",first:" + first + ",sum:" + sum + ",last:" + last
+ + "]";
+ }
}
diff --git a/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/statistics/FloatStatistics.java b/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/statistics/FloatStatistics.java
index edea6889..b66ab7d5 100644
--- a/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/statistics/FloatStatistics.java
+++ b/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/statistics/FloatStatistics.java
@@ -8,111 +8,115 @@
* @author kangrong
*/
public class FloatStatistics extends Statistics {
- private float max;
- private float min;
- private float first;
- private double sum;
- private float last;
-
- @Override
- public void setMinMaxFromBytes(byte[] minBytes, byte[] maxBytes) {
- max = BytesUtils.bytesToFloat(maxBytes);
- min = BytesUtils.bytesToFloat(minBytes);
- }
-
- @Override
- public void updateStats(float value) {
- if (this.isEmpty) {
- initializeStats(value, value, value, value,value);
- isEmpty = false;
- } else {
- updateStats(value, value, value, value,value);
- }
- }
-
- private void updateStats(float minValue, float maxValue, float firstValue, double sumValue,float last) {
- if (minValue < min) {
- min = minValue;
- }
- if (maxValue > max) {
- max = maxValue;
- }
- sum += sumValue;
- this.last = last;
- }
-
- @Override
- public Float getMax() {
- return max;
- }
-
- @Override
- public Float getMin() {
- return min;
- }
-
- @Override
- public Float getFirst() {
- return first;
- }
-
- @Override
- public double getSum() {
- return sum;
- }
-
- @Override
- public Float getLast(){
- return last;
- }
-
- @Override
- protected void mergeStatisticsValue(Statistics stats) {
- FloatStatistics floatStats = (FloatStatistics) stats;
- if (isEmpty) {
- initializeStats(floatStats.getMin(), floatStats.getMax(), floatStats.getFirst(), floatStats.getSum(),floatStats.getLast());
- isEmpty = false;
- } else {
- updateStats(floatStats.getMin(), floatStats.getMax(), floatStats.getFirst(), floatStats.getSum(),floatStats.getLast());
- }
-
- }
-
- public void initializeStats(float min, float max, float first, double sum,float last) {
- this.min = min;
- this.max = max;
- this.first = first;
- this.sum = sum;
- this.last = last;
- }
-
- @Override
- public byte[] getMaxBytes() {
- return BytesUtils.floatToBytes(max);
- }
-
- @Override
- public byte[] getMinBytes() {
- return BytesUtils.floatToBytes(min);
- }
-
- @Override
- public byte[] getFirstBytes() {
- return BytesUtils.floatToBytes(first);
- }
-
- @Override
- public byte[] getSumBytes() {
- return BytesUtils.doubleToBytes(sum);
- }
-
- @Override
- public byte[] getLastBytes(){
- return BytesUtils.floatToBytes(last);
- }
-
- @Override
- public String toString() {
- return "[max:" + max + ",min:" + min + ",first:" + first + ",sum:" + sum + ",last:" + last + "]";
- }
+ private float max;
+ private float min;
+ private float first;
+ private double sum;
+ private float last;
+
+ @Override
+ public void setMinMaxFromBytes(byte[] minBytes, byte[] maxBytes) {
+ max = BytesUtils.bytesToFloat(maxBytes);
+ min = BytesUtils.bytesToFloat(minBytes);
+ }
+
+ @Override
+ public void updateStats(float value) {
+ if (this.isEmpty) {
+ initializeStats(value, value, value, value, value);
+ isEmpty = false;
+ } else {
+ updateStats(value, value, value, value, value);
+ }
+ }
+
+ private void updateStats(float minValue, float maxValue, float firstValue, double sumValue,
+ float last) {
+ if (minValue < min) {
+ min = minValue;
+ }
+ if (maxValue > max) {
+ max = maxValue;
+ }
+ sum += sumValue;
+ this.last = last;
+ }
+
+ @Override
+ public Float getMax() {
+ return max;
+ }
+
+ @Override
+ public Float getMin() {
+ return min;
+ }
+
+ @Override
+ public Float getFirst() {
+ return first;
+ }
+
+ @Override
+ public double getSum() {
+ return sum;
+ }
+
+ @Override
+ public Float getLast() {
+ return last;
+ }
+
+ @Override
+ protected void mergeStatisticsValue(Statistics stats) {
+ FloatStatistics floatStats = (FloatStatistics) stats;
+ if (isEmpty) {
+ initializeStats(floatStats.getMin(), floatStats.getMax(), floatStats.getFirst(),
+ floatStats.getSum(), floatStats.getLast());
+ isEmpty = false;
+ } else {
+ updateStats(floatStats.getMin(), floatStats.getMax(), floatStats.getFirst(),
+ floatStats.getSum(), floatStats.getLast());
+ }
+
+ }
+
+ public void initializeStats(float min, float max, float first, double sum, float last) {
+ this.min = min;
+ this.max = max;
+ this.first = first;
+ this.sum = sum;
+ this.last = last;
+ }
+
+ @Override
+ public byte[] getMaxBytes() {
+ return BytesUtils.floatToBytes(max);
+ }
+
+ @Override
+ public byte[] getMinBytes() {
+ return BytesUtils.floatToBytes(min);
+ }
+
+ @Override
+ public byte[] getFirstBytes() {
+ return BytesUtils.floatToBytes(first);
+ }
+
+ @Override
+ public byte[] getSumBytes() {
+ return BytesUtils.doubleToBytes(sum);
+ }
+
+ @Override
+ public byte[] getLastBytes() {
+ return BytesUtils.floatToBytes(last);
+ }
+
+ @Override
+ public String toString() {
+ return "[max:" + max + ",min:" + min + ",first:" + first + ",sum:" + sum + ",last:" + last
+ + "]";
+ }
}
diff --git a/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/statistics/IntegerStatistics.java b/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/statistics/IntegerStatistics.java
index a9a9c069..7b880717 100644
--- a/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/statistics/IntegerStatistics.java
+++ b/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/statistics/IntegerStatistics.java
@@ -8,114 +8,116 @@
* @author kangrong
*/
public class IntegerStatistics extends Statistics {
- private int max;
- private int min;
- private int first;
- private double sum;
- private int last;
-
- @Override
- public void setMinMaxFromBytes(byte[] minBytes, byte[] maxBytes) {
- max = BytesUtils.bytesToInt(maxBytes);
- min = BytesUtils.bytesToInt(minBytes);
- }
-
- @Override
- public void updateStats(int value) {
- if (isEmpty) {
- initializeStats(value, value, value, value, value);
- isEmpty = false;
- } else {
- updateStats(value, value, value, value, value);
- isEmpty = false;
- }
- }
-
- private void updateStats(int minValue, int maxValue, int firstValue, double sumValue, int lastValue) {
- if (minValue < min) {
- min = minValue;
- }
- if (maxValue > max) {
- max = maxValue;
- }
- sum += sumValue;
- this.last = lastValue;
- }
-
- @Override
- public Integer getMax() {
- return max;
- }
-
- @Override
- public Integer getMin() {
- return min;
- }
-
- @Override
- public Integer getFirst() {
- return first;
- }
-
- @Override
- public double getSum() {
- return sum;
- }
-
- @Override
- public Integer getLast() {
- return last;
- }
-
- @Override
- protected void mergeStatisticsValue(Statistics stats) {
- IntegerStatistics intStats = (IntegerStatistics) stats;
- if (isEmpty) {
- initializeStats(intStats.getMin(), intStats.getMax(), intStats.getFirst(), intStats.getSum(),
- intStats.getLast());
- isEmpty = false;
- } else {
- updateStats(intStats.getMin(), intStats.getMax(), intStats.getFirst(), intStats.getSum(),
- intStats.getLast());
- }
-
- }
-
- private void initializeStats(int min, int max, int first, double sum, int last) {
- this.min = min;
- this.max = max;
- this.first = first;
- this.sum = sum;
- this.last = last;
- }
-
- @Override
- public byte[] getMaxBytes() {
- return BytesUtils.intToBytes(max);
- }
-
- @Override
- public byte[] getMinBytes() {
- return BytesUtils.intToBytes(min);
- }
-
- @Override
- public byte[] getFirstBytes() {
- return BytesUtils.intToBytes(first);
- }
-
- @Override
- public byte[] getSumBytes() {
- return BytesUtils.doubleToBytes(sum);
- }
-
- @Override
- public byte[] getLastBytes() {
- return BytesUtils.intToBytes(last);
- }
-
- @Override
- public String toString() {
- return "[max:" + max + ",min:" + min + ",first:" + first + ",sum:" + sum + ",last:" + last + "]";
- }
+ private int max;
+ private int min;
+ private int first;
+ private double sum;
+ private int last;
+
+ @Override
+ public void setMinMaxFromBytes(byte[] minBytes, byte[] maxBytes) {
+ max = BytesUtils.bytesToInt(maxBytes);
+ min = BytesUtils.bytesToInt(minBytes);
+ }
+
+ @Override
+ public void updateStats(int value) {
+ if (isEmpty) {
+ initializeStats(value, value, value, value, value);
+ isEmpty = false;
+ } else {
+ updateStats(value, value, value, value, value);
+ isEmpty = false;
+ }
+ }
+
+ private void updateStats(int minValue, int maxValue, int firstValue, double sumValue,
+ int lastValue) {
+ if (minValue < min) {
+ min = minValue;
+ }
+ if (maxValue > max) {
+ max = maxValue;
+ }
+ sum += sumValue;
+ this.last = lastValue;
+ }
+
+ @Override
+ public Integer getMax() {
+ return max;
+ }
+
+ @Override
+ public Integer getMin() {
+ return min;
+ }
+
+ @Override
+ public Integer getFirst() {
+ return first;
+ }
+
+ @Override
+ public double getSum() {
+ return sum;
+ }
+
+ @Override
+ public Integer getLast() {
+ return last;
+ }
+
+ @Override
+ protected void mergeStatisticsValue(Statistics stats) {
+ IntegerStatistics intStats = (IntegerStatistics) stats;
+ if (isEmpty) {
+ initializeStats(intStats.getMin(), intStats.getMax(), intStats.getFirst(), intStats.getSum(),
+ intStats.getLast());
+ isEmpty = false;
+ } else {
+ updateStats(intStats.getMin(), intStats.getMax(), intStats.getFirst(), intStats.getSum(),
+ intStats.getLast());
+ }
+
+ }
+
+ private void initializeStats(int min, int max, int first, double sum, int last) {
+ this.min = min;
+ this.max = max;
+ this.first = first;
+ this.sum = sum;
+ this.last = last;
+ }
+
+ @Override
+ public byte[] getMaxBytes() {
+ return BytesUtils.intToBytes(max);
+ }
+
+ @Override
+ public byte[] getMinBytes() {
+ return BytesUtils.intToBytes(min);
+ }
+
+ @Override
+ public byte[] getFirstBytes() {
+ return BytesUtils.intToBytes(first);
+ }
+
+ @Override
+ public byte[] getSumBytes() {
+ return BytesUtils.doubleToBytes(sum);
+ }
+
+ @Override
+ public byte[] getLastBytes() {
+ return BytesUtils.intToBytes(last);
+ }
+
+ @Override
+ public String toString() {
+ return "[max:" + max + ",min:" + min + ",first:" + first + ",sum:" + sum + ",last:" + last
+ + "]";
+ }
}
diff --git a/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/statistics/LongStatistics.java b/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/statistics/LongStatistics.java
index 63776961..eaaac143 100644
--- a/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/statistics/LongStatistics.java
+++ b/src/main/java/cn/edu/tsinghua/tsfile/file/metadata/statistics/LongStatistics.java
@@ -8,124 +8,126 @@
* @author kangrong
*/
public class LongStatistics extends Statistics