Skip to content

Commit ac08a25

Browse files
HADOOP-11867. Add a high-performance vectored read API. (#3904)
part of HADOOP-18103. Add support for multiple ranged vectored read api in PositionedReadable. The default iterates through the ranges to read each synchronously, but the intent is that FSDataInputStream subclasses can make more efficient readers especially in object stores implementation. Also added implementation in S3A where smaller ranges are merged and sliced byte buffers are returned to the readers. All the merged ranged are fetched from S3 asynchronously. Contributed By: Owen O'Malley and Mukund Thakur
1 parent d8ab842 commit ac08a25

File tree

30 files changed

+2422
-67
lines changed

30 files changed

+2422
-67
lines changed

dev-support/Jenkinsfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ pipeline {
4747

4848
options {
4949
buildDiscarder(logRotator(numToKeepStr: '5'))
50-
timeout (time: 24, unit: 'HOURS')
50+
timeout (time: 48, unit: 'HOURS')
5151
timestamps()
5252
checkoutToSubdirectory('src')
5353
}

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/BufferedFSInputStream.java

Lines changed: 23 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
/**
1+
/*
22
* Licensed to the Apache Software Foundation (ASF) under one
33
* or more contributor license agreements. See the NOTICE file
44
* distributed with this work for additional information
@@ -22,6 +22,9 @@
2222
import java.io.FileDescriptor;
2323
import java.io.IOException;
2424
import java.util.StringJoiner;
25+
import java.nio.ByteBuffer;
26+
import java.util.List;
27+
import java.util.function.IntFunction;
2528

2629
import org.apache.hadoop.classification.InterfaceAudience;
2730
import org.apache.hadoop.classification.InterfaceStability;
@@ -158,8 +161,24 @@ public IOStatistics getIOStatistics() {
158161
@Override
159162
public String toString() {
160163
return new StringJoiner(", ",
161-
BufferedFSInputStream.class.getSimpleName() + "[", "]")
162-
.add("in=" + in)
163-
.toString();
164+
BufferedFSInputStream.class.getSimpleName() + "[", "]")
165+
.add("in=" + in)
166+
.toString();
167+
}
168+
169+
@Override
170+
public int minSeekForVectorReads() {
171+
return ((PositionedReadable) in).minSeekForVectorReads();
172+
}
173+
174+
@Override
175+
public int maxReadSizeForVectorReads() {
176+
return ((PositionedReadable) in).maxReadSizeForVectorReads();
177+
}
178+
179+
@Override
180+
public void readVectored(List<? extends FileRange> ranges,
181+
IntFunction<ByteBuffer> allocate) throws IOException {
182+
((PositionedReadable) in).readVectored(ranges, allocate);
164183
}
165184
}

0 commit comments

Comments
 (0)