Skip to content

Commit 7c02d18

Browse files
umamaheswararaoumamaheswararao
authored
HDFS-15427. Merged ListStatus with Fallback target filesystem and InternalDirViewFS. Contributed by Uma Maheswara Rao G.
1 parent 201d734 commit 7c02d18

File tree

4 files changed

+360
-78
lines changed

4 files changed

+360
-78
lines changed

‎hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/InodeTree.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -374,7 +374,7 @@ protected abstract T getTargetFileSystem(URI uri)
374374
throws UnsupportedFileSystemException, URISyntaxException, IOException;
375375

376376
protected abstract T getTargetFileSystem(INodeDir<T> dir)
377-
throws URISyntaxException;
377+
throws URISyntaxException, IOException;
378378

379379
protected abstract T getTargetFileSystem(String settings, URI[] mergeFsURIs)
380380
throws UnsupportedFileSystemException, URISyntaxException, IOException;
@@ -393,7 +393,7 @@ private boolean hasFallbackLink() {
393393
return rootFallbackLink != null;
394394
}
395395

396-
private INodeLink<T> getRootFallbackLink() {
396+
protected INodeLink<T> getRootFallbackLink() {
397397
Preconditions.checkState(root.isInternalDir());
398398
return rootFallbackLink;
399399
}

‎hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java‎

Lines changed: 50 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -290,8 +290,9 @@ protected FileSystem getTargetFileSystem(final URI uri)
290290

291291
@Override
292292
protected FileSystem getTargetFileSystem(final INodeDir<FileSystem> dir)
293-
throws URISyntaxException {
294-
return new InternalDirOfViewFs(dir, creationTime, ugi, myUri, config);
293+
throws URISyntaxException {
294+
return new InternalDirOfViewFs(dir, creationTime, ugi, myUri, config,
295+
this);
295296
}
296297

297298
@Override
@@ -518,10 +519,10 @@ public void access(Path path, FsAction mode) throws AccessControlException,
518519
/**
519520
* {@inheritDoc}
520521
*
521-
* Note: listStatus on root("/") considers listing from fallbackLink if
522-
* available. If the same directory name is present in configured mount path
523-
* as well as in fallback link, then only the configured mount path will be
524-
* listed in the returned result.
522+
* Note: listStatus considers listing from fallbackLink if available. If the
523+
* same directory path is present in configured mount path as well as in
524+
* fallback fs, then only the fallback path will be listed in the returned
525+
* result except for link.
525526
*
526527
* If any of the the immediate children of the given path f is a symlink(mount
527528
* link), the returned FileStatus object of that children would be represented
@@ -1125,11 +1126,13 @@ static class InternalDirOfViewFs extends FileSystem {
11251126
final UserGroupInformation ugi; // the user/group of user who created mtable
11261127
final URI myUri;
11271128
private final boolean showMountLinksAsSymlinks;
1129+
private InodeTree<FileSystem> fsState;
11281130

11291131
public InternalDirOfViewFs(final InodeTree.INodeDir<FileSystem> dir,
11301132
final long cTime, final UserGroupInformation ugi, URI uri,
1131-
Configuration config) throws URISyntaxException {
1133+
Configuration config, InodeTree fsState) throws URISyntaxException {
11321134
myUri = uri;
1135+
this.fsState = fsState;
11331136
try {
11341137
initialize(myUri, config);
11351138
} catch (IOException e) {
@@ -1225,7 +1228,8 @@ public FileStatus[] listStatus(Path f) throws AccessControlException,
12251228
FileNotFoundException, IOException {
12261229
checkPathIsSlash(f);
12271230
FileStatus[] fallbackStatuses = listStatusForFallbackLink();
1228-
FileStatus[] result = new FileStatus[theInternalDir.getChildren().size()];
1231+
Set<FileStatus> linkStatuses = new HashSet<>();
1232+
Set<FileStatus> internalDirStatuses = new HashSet<>();
12291233
int i = 0;
12301234
for (Entry<String, INode<FileSystem>> iEntry :
12311235
theInternalDir.getChildren().entrySet()) {
@@ -1238,11 +1242,10 @@ public FileStatus[] listStatus(Path f) throws AccessControlException,
12381242
// To maintain backward compatibility, with default option(showing
12391243
// mount links as symlinks), we will represent target link as
12401244
// symlink and rest other properties are belongs to mount link only.
1241-
result[i++] =
1245+
linkStatuses.add(
12421246
new FileStatus(0, false, 0, 0, creationTime, creationTime,
12431247
PERMISSION_555, ugi.getShortUserName(),
1244-
ugi.getPrimaryGroupName(), link.getTargetLink(),
1245-
path);
1248+
ugi.getPrimaryGroupName(), link.getTargetLink(), path));
12461249
continue;
12471250
}
12481251

@@ -1258,63 +1261,71 @@ public FileStatus[] listStatus(Path f) throws AccessControlException,
12581261
FileStatus status =
12591262
((ChRootedFileSystem)link.getTargetFileSystem())
12601263
.getMyFs().getFileStatus(new Path(linkedPath));
1261-
result[i++] = new FileStatus(status.getLen(), status.isDirectory(),
1262-
status.getReplication(), status.getBlockSize(),
1263-
status.getModificationTime(), status.getAccessTime(),
1264-
status.getPermission(), status.getOwner(), status.getGroup(),
1265-
null, path);
1264+
linkStatuses.add(
1265+
new FileStatus(status.getLen(), status.isDirectory(),
1266+
status.getReplication(), status.getBlockSize(),
1267+
status.getModificationTime(), status.getAccessTime(),
1268+
status.getPermission(), status.getOwner(),
1269+
status.getGroup(), null, path));
12661270
} catch (FileNotFoundException ex) {
12671271
LOG.warn("Cannot get one of the children's(" + path
12681272
+ ") target path(" + link.getTargetFileSystem().getUri()
12691273
+ ") file status.", ex);
12701274
throw ex;
12711275
}
12721276
} else {
1273-
result[i++] =
1277+
internalDirStatuses.add(
12741278
new FileStatus(0, true, 0, 0, creationTime, creationTime,
12751279
PERMISSION_555, ugi.getShortUserName(),
1276-
ugi.getPrimaryGroupName(), path);
1280+
ugi.getPrimaryGroupName(), path));
12771281
}
12781282
}
1283+
FileStatus[] internalDirStatusesMergedWithFallBack = internalDirStatuses
1284+
.toArray(new FileStatus[internalDirStatuses.size()]);
12791285
if (fallbackStatuses.length > 0) {
1280-
return consolidateFileStatuses(fallbackStatuses, result);
1281-
} else {
1282-
return result;
1286+
internalDirStatusesMergedWithFallBack =
1287+
merge(fallbackStatuses, internalDirStatusesMergedWithFallBack);
12831288
}
1289+
// Links will always have precedence than internalDir or fallback paths.
1290+
return merge(linkStatuses.toArray(new FileStatus[linkStatuses.size()]),
1291+
internalDirStatusesMergedWithFallBack);
12841292
}
12851293

1286-
private FileStatus[] consolidateFileStatuses(FileStatus[] fallbackStatuses,
1287-
FileStatus[] mountPointStatuses) {
1294+
private FileStatus[] merge(FileStatus[] toStatuses,
1295+
FileStatus[] fromStatuses) {
12881296
ArrayList<FileStatus> result = new ArrayList<>();
12891297
Set<String> pathSet = new HashSet<>();
1290-
for (FileStatus status : mountPointStatuses) {
1298+
for (FileStatus status : toStatuses) {
12911299
result.add(status);
12921300
pathSet.add(status.getPath().getName());
12931301
}
1294-
for (FileStatus status : fallbackStatuses) {
1302+
for (FileStatus status : fromStatuses) {
12951303
if (!pathSet.contains(status.getPath().getName())) {
12961304
result.add(status);
12971305
}
12981306
}
1299-
return result.toArray(new FileStatus[0]);
1307+
return result.toArray(new FileStatus[result.size()]);
13001308
}
13011309

13021310
private FileStatus[] listStatusForFallbackLink() throws IOException {
1303-
if (theInternalDir.isRoot() &&
1304-
theInternalDir.getFallbackLink() != null) {
1305-
FileSystem linkedFs =
1306-
theInternalDir.getFallbackLink().getTargetFileSystem();
1307-
// Fallback link is only applicable for root
1308-
FileStatus[] statuses = linkedFs.listStatus(new Path("/"));
1309-
for (FileStatus status : statuses) {
1310-
// Fix the path back to viewfs scheme
1311-
status.setPath(
1312-
new Path(myUri.toString(), status.getPath().getName()));
1311+
if (this.fsState.getRootFallbackLink() != null) {
1312+
FileSystem linkedFallbackFs =
1313+
this.fsState.getRootFallbackLink().getTargetFileSystem();
1314+
Path p = Path.getPathWithoutSchemeAndAuthority(
1315+
new Path(theInternalDir.fullPath));
1316+
if (theInternalDir.isRoot() || linkedFallbackFs.exists(p)) {
1317+
FileStatus[] statuses = linkedFallbackFs.listStatus(p);
1318+
for (FileStatus status : statuses) {
1319+
// Fix the path back to viewfs scheme
1320+
Path pathFromConfiguredFallbackRoot =
1321+
new Path(p, status.getPath().getName());
1322+
status.setPath(
1323+
new Path(myUri.toString(), pathFromConfiguredFallbackRoot));
1324+
}
1325+
return statuses;
13131326
}
1314-
return statuses;
1315-
} else {
1316-
return new FileStatus[0];
13171327
}
1328+
return new FileStatus[0];
13181329
}
13191330

13201331
@Override

‎hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java‎

Lines changed: 59 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@
4444
import org.apache.hadoop.fs.FSDataOutputStream;
4545
import org.apache.hadoop.fs.FileAlreadyExistsException;
4646
import org.apache.hadoop.fs.FileChecksum;
47+
import org.apache.hadoop.fs.FileContext;
4748
import org.apache.hadoop.fs.FileStatus;
4849
import org.apache.hadoop.fs.FsConstants;
4950
import org.apache.hadoop.fs.FsServerDefaults;
@@ -236,7 +237,8 @@ protected AbstractFileSystem getTargetFileSystem(final URI uri)
236237
@Override
237238
protected AbstractFileSystem getTargetFileSystem(
238239
final INodeDir<AbstractFileSystem> dir) throws URISyntaxException {
239-
return new InternalDirOfViewFs(dir, creationTime, ugi, getUri());
240+
return new InternalDirOfViewFs(dir, creationTime, ugi, getUri(), this,
241+
config);
240242
}
241243

242244
@Override
@@ -455,6 +457,11 @@ public LocatedFileStatus getViewFsFileStatus(LocatedFileStatus stat,
455457
/**
456458
* {@inheritDoc}
457459
*
460+
* Note: listStatus considers listing from fallbackLink if available. If the
461+
* same directory path is present in configured mount path as well as in
462+
* fallback fs, then only the fallback path will be listed in the returned
463+
* result except for link.
464+
*
458465
* If any of the the immediate children of the given path f is a symlink(mount
459466
* link), the returned FileStatus object of that children would be represented
460467
* as a symlink. It will not be resolved to the target path and will not get
@@ -880,15 +887,20 @@ static class InternalDirOfViewFs extends AbstractFileSystem {
880887
final long creationTime; // of the the mount table
881888
final UserGroupInformation ugi; // the user/group of user who created mtable
882889
final URI myUri; // the URI of the outer ViewFs
883-
890+
private InodeTree<AbstractFileSystem> fsState;
891+
private Configuration conf;
892+
884893
public InternalDirOfViewFs(final InodeTree.INodeDir<AbstractFileSystem> dir,
885-
final long cTime, final UserGroupInformation ugi, final URI uri)
894+
final long cTime, final UserGroupInformation ugi, final URI uri,
895+
InodeTree fsState, Configuration conf)
886896
throws URISyntaxException {
887897
super(FsConstants.VIEWFS_URI, FsConstants.VIEWFS_SCHEME, false, -1);
888898
theInternalDir = dir;
889899
creationTime = cTime;
890900
this.ugi = ugi;
891901
myUri = uri;
902+
this.fsState = fsState;
903+
this.conf = conf;
892904
}
893905

894906
static private void checkPathIsSlash(final Path f) throws IOException {
@@ -1015,7 +1027,8 @@ public int getUriDefaultPort() {
10151027
public FileStatus[] listStatus(final Path f) throws IOException {
10161028
checkPathIsSlash(f);
10171029
FileStatus[] fallbackStatuses = listStatusForFallbackLink();
1018-
FileStatus[] result = new FileStatus[theInternalDir.getChildren().size()];
1030+
Set<FileStatus> linkStatuses = new HashSet<>();
1031+
Set<FileStatus> internalDirStatuses = new HashSet<>();
10191032
int i = 0;
10201033
for (Entry<String, INode<AbstractFileSystem>> iEntry :
10211034
theInternalDir.getChildren().entrySet()) {
@@ -1029,11 +1042,10 @@ public FileStatus[] listStatus(final Path f) throws IOException {
10291042
// To maintain backward compatibility, with default option(showing
10301043
// mount links as symlinks), we will represent target link as
10311044
// symlink and rest other properties are belongs to mount link only.
1032-
result[i++] =
1045+
linkStatuses.add(
10331046
new FileStatus(0, false, 0, 0, creationTime, creationTime,
10341047
PERMISSION_555, ugi.getShortUserName(),
1035-
ugi.getPrimaryGroupName(), link.getTargetLink(),
1036-
path);
1048+
ugi.getPrimaryGroupName(), link.getTargetLink(), path));
10371049
continue;
10381050
}
10391051

@@ -1049,63 +1061,75 @@ public FileStatus[] listStatus(final Path f) throws IOException {
10491061
FileStatus status =
10501062
((ChRootedFs) link.getTargetFileSystem()).getMyFs()
10511063
.getFileStatus(new Path(linkedPath));
1052-
result[i++] = new FileStatus(status.getLen(), status.isDirectory(),
1053-
status.getReplication(), status.getBlockSize(),
1054-
status.getModificationTime(), status.getAccessTime(),
1055-
status.getPermission(), status.getOwner(), status.getGroup(),
1056-
null, path);
1064+
linkStatuses.add(
1065+
new FileStatus(status.getLen(), status.isDirectory(),
1066+
status.getReplication(), status.getBlockSize(),
1067+
status.getModificationTime(), status.getAccessTime(),
1068+
status.getPermission(), status.getOwner(),
1069+
status.getGroup(), null, path));
10571070
} catch (FileNotFoundException ex) {
10581071
LOG.warn("Cannot get one of the children's(" + path
10591072
+ ") target path(" + link.getTargetFileSystem().getUri()
10601073
+ ") file status.", ex);
10611074
throw ex;
10621075
}
10631076
} else {
1064-
result[i++] =
1077+
internalDirStatuses.add(
10651078
new FileStatus(0, true, 0, 0, creationTime, creationTime,
10661079
PERMISSION_555, ugi.getShortUserName(),
1067-
ugi.getGroupNames()[0], path);
1080+
ugi.getPrimaryGroupName(), path));
10681081
}
10691082
}
1083+
1084+
FileStatus[] internalDirStatusesMergedWithFallBack = internalDirStatuses
1085+
.toArray(new FileStatus[internalDirStatuses.size()]);
10701086
if (fallbackStatuses.length > 0) {
1071-
return consolidateFileStatuses(fallbackStatuses, result);
1072-
} else {
1073-
return result;
1087+
internalDirStatusesMergedWithFallBack =
1088+
merge(fallbackStatuses, internalDirStatusesMergedWithFallBack);
10741089
}
1090+
1091+
// Links will always have precedence than internalDir or fallback paths.
1092+
return merge(linkStatuses.toArray(new FileStatus[linkStatuses.size()]),
1093+
internalDirStatusesMergedWithFallBack);
10751094
}
10761095

1077-
private FileStatus[] consolidateFileStatuses(FileStatus[] fallbackStatuses,
1078-
FileStatus[] mountPointStatuses) {
1096+
private FileStatus[] merge(FileStatus[] toStatuses,
1097+
FileStatus[] fromStatuses) {
10791098
ArrayList<FileStatus> result = new ArrayList<>();
10801099
Set<String> pathSet = new HashSet<>();
1081-
for (FileStatus status : mountPointStatuses) {
1100+
for (FileStatus status : toStatuses) {
10821101
result.add(status);
10831102
pathSet.add(status.getPath().getName());
10841103
}
1085-
for (FileStatus status : fallbackStatuses) {
1104+
for (FileStatus status : fromStatuses) {
10861105
if (!pathSet.contains(status.getPath().getName())) {
10871106
result.add(status);
10881107
}
10891108
}
1090-
return result.toArray(new FileStatus[0]);
1109+
return result.toArray(new FileStatus[result.size()]);
10911110
}
10921111

10931112
private FileStatus[] listStatusForFallbackLink() throws IOException {
1094-
if (theInternalDir.isRoot() &&
1095-
theInternalDir.getFallbackLink() != null) {
1096-
AbstractFileSystem linkedFs =
1097-
theInternalDir.getFallbackLink().getTargetFileSystem();
1098-
// Fallback link is only applicable for root
1099-
FileStatus[] statuses = linkedFs.listStatus(new Path("/"));
1100-
for (FileStatus status : statuses) {
1101-
// Fix the path back to viewfs scheme
1102-
status.setPath(
1103-
new Path(myUri.toString(), status.getPath().getName()));
1113+
if (fsState.getRootFallbackLink() != null) {
1114+
AbstractFileSystem linkedFallbackFs =
1115+
fsState.getRootFallbackLink().getTargetFileSystem();
1116+
Path p = Path.getPathWithoutSchemeAndAuthority(
1117+
new Path(theInternalDir.fullPath));
1118+
if (theInternalDir.isRoot() || FileContext
1119+
.getFileContext(linkedFallbackFs, conf).util().exists(p)) {
1120+
// Fallback link is only applicable for root
1121+
FileStatus[] statuses = linkedFallbackFs.listStatus(p);
1122+
for (FileStatus status : statuses) {
1123+
// Fix the path back to viewfs scheme
1124+
Path pathFromConfiguredFallbackRoot =
1125+
new Path(p, status.getPath().getName());
1126+
status.setPath(
1127+
new Path(myUri.toString(), pathFromConfiguredFallbackRoot));
1128+
}
1129+
return statuses;
11041130
}
1105-
return statuses;
1106-
} else {
1107-
return new FileStatus[0];
11081131
}
1132+
return new FileStatus[0];
11091133
}
11101134

11111135
@Override

0 commit comments

Comments
 (0)