Skip to content

Commit 12b7059

Browse files
yangwweiyangwwei
committed
YARN-9440. Improve diagnostics for scheduler and app activities. Contributed by Tao Yang.
1 parent 1d70c8c commit 12b7059

30 files changed

+1477
-275
lines changed

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,18 +17,25 @@
1717
*/
1818
package org.apache.hadoop.yarn.util.resource;
1919

20+
import com.google.common.collect.ImmutableSet;
21+
import org.apache.hadoop.yarn.api.records.ResourceInformation;
2022
import org.slf4j.Logger;
2123
import org.slf4j.LoggerFactory;
2224
import org.apache.hadoop.classification.InterfaceAudience.Private;
2325
import org.apache.hadoop.classification.InterfaceStability.Unstable;
2426
import org.apache.hadoop.yarn.api.records.Resource;
2527

28+
import java.util.Set;
29+
2630
@Private
2731
@Unstable
2832
public class DefaultResourceCalculator extends ResourceCalculator {
2933
private static final Logger LOG =
3034
LoggerFactory.getLogger(DefaultResourceCalculator.class);
3135

36+
private static final Set<String> INSUFFICIENT_RESOURCE_NAME =
37+
ImmutableSet.of(ResourceInformation.MEMORY_URI);
38+
3239
@Override
3340
public int compare(Resource unused, Resource lhs, Resource rhs,
3441
boolean singleType) {
@@ -150,4 +157,13 @@ public boolean isAnyMajorResourceZeroOrNegative(Resource resource) {
150157
public boolean isAnyMajorResourceAboveZero(Resource resource) {
151158
return resource.getMemorySize() > 0;
152159
}
160+
161+
public Set<String> getInsufficientResourceNames(Resource required,
162+
Resource available) {
163+
if (required.getMemorySize() > available.getMemorySize()) {
164+
return INSUFFICIENT_RESOURCE_NAME;
165+
} else {
166+
return ImmutableSet.of();
167+
}
168+
}
153169
}

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,9 @@
2828
import java.io.PrintWriter;
2929
import java.io.StringWriter;
3030
import java.util.Arrays;
31+
import java.util.Set;
32+
import java.util.stream.Collectors;
33+
import java.util.stream.IntStream;
3134

3235
/**
3336
* A {@link ResourceCalculator} which uses the concept of
@@ -588,4 +591,15 @@ public boolean isAnyMajorResourceAboveZero(Resource resource) {
588591
}
589592
return false;
590593
}
594+
595+
@Override
596+
public Set<String> getInsufficientResourceNames(Resource required,
597+
Resource available) {
598+
int maxLength = ResourceUtils.getNumberOfCountableResourceTypes();
599+
return IntStream.range(0, maxLength).filter(
600+
i -> required.getResourceInformation(i).getValue() > available
601+
.getResourceInformation(i).getValue())
602+
.mapToObj(i -> ResourceUtils.getResourceTypesArray()[i].getName())
603+
.collect(Collectors.toSet());
604+
}
591605
}

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,8 @@
2121
import org.apache.hadoop.classification.InterfaceStability.Unstable;
2222
import org.apache.hadoop.yarn.api.records.Resource;
2323

24+
import java.util.Set;
25+
2426
/**
2527
* A set of {@link Resource} comparison and manipulation interfaces.
2628
*/
@@ -284,4 +286,15 @@ public abstract float divide(
284286
* @return returns true if any resource is {@literal >} 0
285287
*/
286288
public abstract boolean isAnyMajorResourceAboveZero(Resource resource);
289+
290+
/**
291+
* Get insufficient resource names via comparing required resource and
292+
* capacity resource.
293+
*
294+
* @param required - required resource
295+
* @param available - available resource
296+
* @return insufficient resource names
297+
*/
298+
public abstract Set<String> getInsufficientResourceNames(Resource required,
299+
Resource available);
287300
}

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceCalculator.java‎

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,10 @@
2121
import java.util.Arrays;
2222
import java.util.Collection;
2323

24+
import com.google.common.collect.ImmutableSet;
2425
import org.apache.hadoop.conf.Configuration;
2526
import org.apache.hadoop.yarn.api.records.Resource;
27+
import org.apache.hadoop.yarn.api.records.ResourceInformation;
2628
import org.apache.hadoop.yarn.conf.YarnConfiguration;
2729
import org.junit.Assert;
2830
import org.junit.Before;
@@ -406,4 +408,121 @@ public void testDivisionByZeroRatioNumeratorAndDenominatorIsZero() {
406408
0));
407409
assertEquals(0.0, ratio, 0.00001);
408410
}
411+
412+
@Test
413+
public void testFitsInDiagnosticsCollector() {
414+
if (resourceCalculator instanceof DefaultResourceCalculator) {
415+
// required-resource = (0, 0)
416+
assertEquals(ImmutableSet.of(),
417+
resourceCalculator.getInsufficientResourceNames(newResource(0, 0),
418+
newResource(0, 0)));
419+
assertEquals(ImmutableSet.of(),
420+
resourceCalculator.getInsufficientResourceNames(newResource(0, 0),
421+
newResource(0, 1)));
422+
assertEquals(ImmutableSet.of(),
423+
resourceCalculator.getInsufficientResourceNames(newResource(0, 0),
424+
newResource(1, 0)));
425+
assertEquals(ImmutableSet.of(),
426+
resourceCalculator.getInsufficientResourceNames(newResource(0, 0),
427+
newResource(1, 1)));
428+
429+
// required-resource = (0, 1)
430+
assertEquals(ImmutableSet.of(),
431+
resourceCalculator.getInsufficientResourceNames(newResource(0, 1),
432+
newResource(0, 0)));
433+
assertEquals(ImmutableSet.of(),
434+
resourceCalculator.getInsufficientResourceNames(newResource(0, 1),
435+
newResource(0, 1)));
436+
assertEquals(ImmutableSet.of(),
437+
resourceCalculator.getInsufficientResourceNames(newResource(0, 1),
438+
newResource(1, 0)));
439+
assertEquals(ImmutableSet.of(),
440+
resourceCalculator.getInsufficientResourceNames(newResource(0, 1),
441+
newResource(1, 1)));
442+
443+
// required-resource = (1, 0)
444+
assertEquals(ImmutableSet.of(ResourceInformation.MEMORY_URI),
445+
resourceCalculator.getInsufficientResourceNames(newResource(1, 0),
446+
newResource(0, 0)));
447+
assertEquals(ImmutableSet.of(ResourceInformation.MEMORY_URI),
448+
resourceCalculator.getInsufficientResourceNames(newResource(1, 0),
449+
newResource(0, 1)));
450+
assertEquals(ImmutableSet.of(),
451+
resourceCalculator.getInsufficientResourceNames(newResource(1, 0),
452+
newResource(1, 0)));
453+
assertEquals(ImmutableSet.of(),
454+
resourceCalculator.getInsufficientResourceNames(newResource(1, 0),
455+
newResource(1, 1)));
456+
457+
// required-resource = (1, 1)
458+
assertEquals(ImmutableSet.of(ResourceInformation.MEMORY_URI),
459+
resourceCalculator.getInsufficientResourceNames(newResource(1, 1),
460+
newResource(0, 0)));
461+
assertEquals(ImmutableSet.of(ResourceInformation.MEMORY_URI),
462+
resourceCalculator.getInsufficientResourceNames(newResource(1, 1),
463+
newResource(0, 1)));
464+
assertEquals(ImmutableSet.of(),
465+
resourceCalculator.getInsufficientResourceNames(newResource(1, 1),
466+
newResource(1, 0)));
467+
assertEquals(ImmutableSet.of(),
468+
resourceCalculator.getInsufficientResourceNames(newResource(1, 1),
469+
newResource(1, 1)));
470+
} else if (resourceCalculator instanceof DominantResourceCalculator) {
471+
// required-resource = (0, 0)
472+
assertEquals(ImmutableSet.of(),
473+
resourceCalculator.getInsufficientResourceNames(newResource(0, 0),
474+
newResource(0, 0)));
475+
assertEquals(ImmutableSet.of(),
476+
resourceCalculator.getInsufficientResourceNames(newResource(0, 0),
477+
newResource(0, 1)));
478+
assertEquals(ImmutableSet.of(),
479+
resourceCalculator.getInsufficientResourceNames(newResource(0, 0),
480+
newResource(1, 0)));
481+
assertEquals(ImmutableSet.of(),
482+
resourceCalculator.getInsufficientResourceNames(newResource(0, 0),
483+
newResource(1, 1)));
484+
485+
// required-resource = (0, 1)
486+
assertEquals(ImmutableSet.of(ResourceInformation.VCORES_URI),
487+
resourceCalculator.getInsufficientResourceNames(newResource(0, 1),
488+
newResource(0, 0)));
489+
assertEquals(ImmutableSet.of(),
490+
resourceCalculator.getInsufficientResourceNames(newResource(0, 1),
491+
newResource(0, 1)));
492+
assertEquals(ImmutableSet.of(ResourceInformation.VCORES_URI),
493+
resourceCalculator.getInsufficientResourceNames(newResource(0, 1),
494+
newResource(1, 0)));
495+
assertEquals(ImmutableSet.of(),
496+
resourceCalculator.getInsufficientResourceNames(newResource(0, 1),
497+
newResource(1, 1)));
498+
499+
// required-resource = (1, 0)
500+
assertEquals(ImmutableSet.of(ResourceInformation.MEMORY_URI),
501+
resourceCalculator.getInsufficientResourceNames(newResource(1, 0),
502+
newResource(0, 0)));
503+
assertEquals(ImmutableSet.of(ResourceInformation.MEMORY_URI),
504+
resourceCalculator.getInsufficientResourceNames(newResource(1, 0),
505+
newResource(0, 1)));
506+
assertEquals(ImmutableSet.of(),
507+
resourceCalculator.getInsufficientResourceNames(newResource(1, 0),
508+
newResource(1, 0)));
509+
assertEquals(ImmutableSet.of(),
510+
resourceCalculator.getInsufficientResourceNames(newResource(1, 0),
511+
newResource(1, 1)));
512+
513+
// required-resource = (1, 1)
514+
assertEquals(ImmutableSet.of(ResourceInformation.MEMORY_URI,
515+
ResourceInformation.VCORES_URI), resourceCalculator
516+
.getInsufficientResourceNames(newResource(1, 1), newResource(0, 0)));
517+
assertEquals(ImmutableSet.of(ResourceInformation.MEMORY_URI),
518+
resourceCalculator.getInsufficientResourceNames(newResource(1, 1),
519+
newResource(0, 1)));
520+
assertEquals(ImmutableSet.of(ResourceInformation.VCORES_URI),
521+
resourceCalculator.getInsufficientResourceNames(newResource(1, 1),
522+
newResource(1, 0)));
523+
assertEquals(ImmutableSet.of(),
524+
resourceCalculator.getInsufficientResourceNames(newResource(1, 1),
525+
newResource(1, 1)));
526+
}
527+
}
409528
}

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
import java.util.HashSet;
2525
import java.util.List;
2626
import java.util.Map;
27+
import java.util.Optional;
2728
import java.util.Set;
2829
import java.util.concurrent.ConcurrentHashMap;
2930
import java.util.concurrent.ConcurrentSkipListSet;
@@ -46,6 +47,7 @@
4647
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
4748
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
4849
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState;
50+
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.DiagnosticsCollector;
4951
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.SchedulingMode;
5052
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.ApplicationSchedulingConfig;
5153
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.ContainerRequest;
@@ -765,16 +767,18 @@ public boolean canDelayTo(
765767
* @param schedulerKey schedulerKey
766768
* @param schedulerNode schedulerNode
767769
* @param schedulingMode schedulingMode
770+
* @param dcOpt optional diagnostics collector
768771
* @return can use the node or not.
769772
*/
770773
public boolean precheckNode(SchedulerRequestKey schedulerKey,
771-
SchedulerNode schedulerNode, SchedulingMode schedulingMode) {
774+
SchedulerNode schedulerNode, SchedulingMode schedulingMode,
775+
Optional<DiagnosticsCollector> dcOpt) {
772776
this.readLock.lock();
773777
try {
774778
AppPlacementAllocator ap =
775779
schedulerKeyToAppPlacementAllocator.get(schedulerKey);
776780
return (ap != null) && ap.precheckNode(schedulerNode,
777-
schedulingMode);
781+
schedulingMode, dcOpt);
778782
} finally {
779783
this.readLock.unlock();
780784
}

0 commit comments

Comments
 (0)