Skip to content

Commit ce4ec74

Browse files
abhishekdas99abhishekdas99
authored
HADOOP-17024. ListStatus on ViewFS root (ls "/") should list the linkFallBack root (configured target root). Contributed by Abhishek Das.
1 parent bdbd59c commit ce4ec74

File tree

4 files changed

+209
-2
lines changed

4 files changed

+209
-2
lines changed

‎hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/InodeTree.java‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,7 @@ static class INodeDir<T> extends INode<T> {
123123
private final Map<String, INode<T>> children = new HashMap<>();
124124
private T internalDirFs = null; //filesystem of this internal directory
125125
private boolean isRoot = false;
126+
private INodeLink<T> fallbackLink = null;
126127

127128
INodeDir(final String pathToNode, final UserGroupInformation aUgi) {
128129
super(pathToNode, aUgi);
@@ -149,6 +150,17 @@ boolean isRoot() {
149150
return isRoot;
150151
}
151152

153+
INodeLink<T> getFallbackLink() {
154+
return fallbackLink;
155+
}
156+
157+
void addFallbackLink(INodeLink<T> link) throws IOException {
158+
if (!isRoot) {
159+
throw new IOException("Fallback link can only be added for root");
160+
}
161+
this.fallbackLink = link;
162+
}
163+
152164
Map<String, INode<T>> getChildren() {
153165
return Collections.unmodifiableMap(children);
154166
}
@@ -580,6 +592,7 @@ protected InodeTree(final Configuration config, final String viewName)
580592
}
581593
}
582594
rootFallbackLink = fallbackLink;
595+
getRootDir().addFallbackLink(rootFallbackLink);
583596
}
584597

585598
if (!gotMountTableEntry) {

‎hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java‎

Lines changed: 48 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1200,10 +1200,19 @@ public FileStatus getFileStatus(Path f) throws IOException {
12001200
}
12011201

12021202

1203+
/**
1204+
* {@inheritDoc}
1205+
*
1206+
* Note: listStatus on root("/") considers listing from fallbackLink if
1207+
* available. If the same directory name is present in configured mount
1208+
* path as well as in fallback link, then only the configured mount path
1209+
* will be listed in the returned result.
1210+
*/
12031211
@Override
12041212
public FileStatus[] listStatus(Path f) throws AccessControlException,
12051213
FileNotFoundException, IOException {
12061214
checkPathIsSlash(f);
1215+
FileStatus[] fallbackStatuses = listStatusForFallbackLink();
12071216
FileStatus[] result = new FileStatus[theInternalDir.getChildren().size()];
12081217
int i = 0;
12091218
for (Entry<String, INode<FileSystem>> iEntry :
@@ -1226,7 +1235,45 @@ public FileStatus[] listStatus(Path f) throws AccessControlException,
12261235
myUri, null));
12271236
}
12281237
}
1229-
return result;
1238+
if (fallbackStatuses.length > 0) {
1239+
return consolidateFileStatuses(fallbackStatuses, result);
1240+
} else {
1241+
return result;
1242+
}
1243+
}
1244+
1245+
private FileStatus[] consolidateFileStatuses(FileStatus[] fallbackStatuses,
1246+
FileStatus[] mountPointStatuses) {
1247+
ArrayList<FileStatus> result = new ArrayList<>();
1248+
Set<String> pathSet = new HashSet<>();
1249+
for (FileStatus status : mountPointStatuses) {
1250+
result.add(status);
1251+
pathSet.add(status.getPath().getName());
1252+
}
1253+
for (FileStatus status : fallbackStatuses) {
1254+
if (!pathSet.contains(status.getPath().getName())) {
1255+
result.add(status);
1256+
}
1257+
}
1258+
return result.toArray(new FileStatus[0]);
1259+
}
1260+
1261+
private FileStatus[] listStatusForFallbackLink() throws IOException {
1262+
if (theInternalDir.isRoot() &&
1263+
theInternalDir.getFallbackLink() != null) {
1264+
FileSystem linkedFs =
1265+
theInternalDir.getFallbackLink().getTargetFileSystem();
1266+
// Fallback link is only applicable for root
1267+
FileStatus[] statuses = linkedFs.listStatus(new Path("/"));
1268+
for (FileStatus status : statuses) {
1269+
// Fix the path back to viewfs scheme
1270+
status.setPath(
1271+
new Path(myUri.toString(), status.getPath().getName()));
1272+
}
1273+
return statuses;
1274+
} else {
1275+
return new FileStatus[0];
1276+
}
12301277
}
12311278

12321279
@Override

‎hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java‎

Lines changed: 50 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,10 +25,12 @@
2525
import java.net.URISyntaxException;
2626
import java.util.ArrayList;
2727
import java.util.EnumSet;
28+
import java.util.HashSet;
2829
import java.util.List;
2930
import java.util.Map;
3031
import java.util.Map.Entry;
3132

33+
import java.util.Set;
3234
import org.apache.hadoop.classification.InterfaceAudience;
3335
import org.apache.hadoop.classification.InterfaceStability;
3436
import org.apache.hadoop.conf.Configuration;
@@ -950,10 +952,19 @@ public int getUriDefaultPort() {
950952
return -1;
951953
}
952954

955+
/**
956+
* {@inheritDoc}
957+
*
958+
* Note: listStatus on root("/") considers listing from fallbackLink if
959+
* available. If the same directory name is present in configured mount
960+
* path as well as in fallback link, then only the configured mount path
961+
* will be listed in the returned result.
962+
*/
953963
@Override
954964
public FileStatus[] listStatus(final Path f) throws AccessControlException,
955965
IOException {
956966
checkPathIsSlash(f);
967+
FileStatus[] fallbackStatuses = listStatusForFallbackLink();
957968
FileStatus[] result = new FileStatus[theInternalDir.getChildren().size()];
958969
int i = 0;
959970
for (Entry<String, INode<AbstractFileSystem>> iEntry :
@@ -979,7 +990,45 @@ public FileStatus[] listStatus(final Path f) throws AccessControlException,
979990
myUri, null));
980991
}
981992
}
982-
return result;
993+
if (fallbackStatuses.length > 0) {
994+
return consolidateFileStatuses(fallbackStatuses, result);
995+
} else {
996+
return result;
997+
}
998+
}
999+
1000+
private FileStatus[] consolidateFileStatuses(FileStatus[] fallbackStatuses,
1001+
FileStatus[] mountPointStatuses) {
1002+
ArrayList<FileStatus> result = new ArrayList<>();
1003+
Set<String> pathSet = new HashSet<>();
1004+
for (FileStatus status : mountPointStatuses) {
1005+
result.add(status);
1006+
pathSet.add(status.getPath().getName());
1007+
}
1008+
for (FileStatus status : fallbackStatuses) {
1009+
if (!pathSet.contains(status.getPath().getName())) {
1010+
result.add(status);
1011+
}
1012+
}
1013+
return result.toArray(new FileStatus[0]);
1014+
}
1015+
1016+
private FileStatus[] listStatusForFallbackLink() throws IOException {
1017+
if (theInternalDir.isRoot() &&
1018+
theInternalDir.getFallbackLink() != null) {
1019+
AbstractFileSystem linkedFs =
1020+
theInternalDir.getFallbackLink().getTargetFileSystem();
1021+
// Fallback link is only applicable for root
1022+
FileStatus[] statuses = linkedFs.listStatus(new Path("/"));
1023+
for (FileStatus status : statuses) {
1024+
// Fix the path back to viewfs scheme
1025+
status.setPath(
1026+
new Path(myUri.toString(), status.getPath().getName()));
1027+
}
1028+
return statuses;
1029+
} else {
1030+
return new FileStatus[0];
1031+
}
9831032
}
9841033

9851034
@Override

‎hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemLinkFallback.java‎

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
import java.net.URI;
2727
import java.net.URISyntaxException;
2828

29+
import java.util.HashSet;
2930
import javax.security.auth.login.LoginException;
3031

3132
import org.apache.hadoop.conf.Configuration;
@@ -261,4 +262,101 @@ public void testConfLinkFallbackWithMountPoint() throws Exception {
261262
e.getMessage().contains(expectedErrorMsg));
262263
}
263264
}
265+
266+
/**
267+
* This tests whether the fallback link gets listed for list operation
268+
* of root directory of mount table.
269+
* @throws Exception
270+
*/
271+
@Test
272+
public void testListingWithFallbackLink() throws Exception {
273+
Path dir1 = new Path(targetTestRoot, "fallbackDir/dir1");
274+
fsTarget.mkdirs(dir1);
275+
String clusterName = Constants.CONFIG_VIEWFS_DEFAULT_MOUNT_TABLE;
276+
URI viewFsUri = new URI(FsConstants.VIEWFS_SCHEME, clusterName,
277+
"/", null, null);
278+
279+
HashSet<Path> beforeFallback = new HashSet<>();
280+
try(FileSystem vfs = FileSystem.get(viewFsUri, conf)) {
281+
for (FileStatus stat : vfs.listStatus(new Path(viewFsUri.toString()))) {
282+
beforeFallback.add(stat.getPath());
283+
}
284+
}
285+
286+
ConfigUtil.addLinkFallback(conf, clusterName,
287+
new Path(targetTestRoot, "fallbackDir").toUri());
288+
289+
try (FileSystem vfs = FileSystem.get(viewFsUri, conf)) {
290+
HashSet<Path> afterFallback = new HashSet<>();
291+
for (FileStatus stat : vfs.listStatus(new Path(viewFsUri.toString()))) {
292+
afterFallback.add(stat.getPath());
293+
}
294+
afterFallback.removeAll(beforeFallback);
295+
assertTrue("Listing didn't include fallback link",
296+
afterFallback.size() == 1);
297+
Path[] fallbackArray = new Path[afterFallback.size()];
298+
afterFallback.toArray(fallbackArray);
299+
Path expected = new Path(viewFsUri.toString(), "dir1");
300+
assertEquals("Path did not match",
301+
expected, fallbackArray[0]);
302+
303+
// Create a directory using the returned fallback path and verify
304+
Path childDir = new Path(fallbackArray[0], "child");
305+
vfs.mkdirs(childDir);
306+
FileStatus status = fsTarget.getFileStatus(new Path(dir1, "child"));
307+
assertTrue(status.isDirectory());
308+
assertTrue(vfs.getFileStatus(childDir).isDirectory());
309+
}
310+
}
311+
312+
/**
313+
* This tests whether fallback directory gets shaded during list operation
314+
* of root directory of mount table when the same directory name exists as
315+
* mount point as well as in the fallback linked directory.
316+
* @throws Exception
317+
*/
318+
@Test
319+
public void testListingWithFallbackLinkWithSameMountDirectories()
320+
throws Exception {
321+
// Creating two directories under the fallback directory.
322+
// "user" directory already exists as configured mount point.
323+
Path dir1 = new Path(targetTestRoot, "fallbackDir/user");
324+
Path dir2 = new Path(targetTestRoot, "fallbackDir/user1");
325+
fsTarget.mkdirs(dir1);
326+
fsTarget.mkdirs(dir2);
327+
String clusterName = Constants.CONFIG_VIEWFS_DEFAULT_MOUNT_TABLE;
328+
URI viewFsUri = new URI(FsConstants.VIEWFS_SCHEME, clusterName,
329+
"/", null, null);
330+
331+
HashSet<Path> beforeFallback = new HashSet<>();
332+
try(FileSystem vfs = FileSystem.get(viewFsUri, conf)) {
333+
for (FileStatus stat : vfs.listStatus(new Path(viewFsUri.toString()))) {
334+
beforeFallback.add(stat.getPath());
335+
}
336+
}
337+
ConfigUtil.addLinkFallback(conf, clusterName,
338+
new Path(targetTestRoot, "fallbackDir").toUri());
339+
340+
try (FileSystem vfs = FileSystem.get(viewFsUri, conf)) {
341+
HashSet<Path> afterFallback = new HashSet<>();
342+
for (FileStatus stat : vfs.listStatus(new Path(viewFsUri.toString()))) {
343+
afterFallback.add(stat.getPath());
344+
}
345+
afterFallback.removeAll(beforeFallback);
346+
assertTrue("The same directory name in fallback link should be shaded",
347+
afterFallback.size() == 1);
348+
Path[] fallbackArray = new Path[afterFallback.size()];
349+
// Only user1 should be listed as fallback link
350+
Path expected = new Path(viewFsUri.toString(), "user1");
351+
assertEquals("Path did not match",
352+
expected, afterFallback.toArray(fallbackArray)[0]);
353+
354+
// Create a directory using the returned fallback path and verify
355+
Path childDir = new Path(fallbackArray[0], "child");
356+
vfs.mkdirs(childDir);
357+
FileStatus status = fsTarget.getFileStatus(new Path(dir2, "child"));
358+
assertTrue(status.isDirectory());
359+
assertTrue(vfs.getFileStatus(childDir).isDirectory());
360+
}
361+
}
264362
}

0 commit comments

Comments
 (0)