YARN-6971. Clean up different ways to create resources. Contributed by Riya Khandelwal

Author: szilard-nemeth

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java

@@ -148,6 +148,7 @@
 import org.apache.hadoop.yarn.util.Clock;
 import org.apache.hadoop.yarn.util.ControlledClock;
 import org.apache.hadoop.yarn.util.Records;
+import org.apache.hadoop.yarn.util.resource.Resources;
 import org.apache.hadoop.yarn.util.SystemClock;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.Assertions;
@@ -502,8 +503,8 @@ public void testPreemptReducers() throws Exception {
             0, 0, 0, 0, 0, 0, "jobfile", null, false, ""));
     MyContainerAllocator allocator = new MyContainerAllocator(rm, conf,
         appAttemptId, mockJob, SystemClock.getInstance());
-    allocator.setMapResourceRequest(BuilderUtils.newResource(1024, 1));
-    allocator.setReduceResourceRequest(BuilderUtils.newResource(1024, 1));
+    allocator.setMapResourceRequest(Resources.createResource(1024));
+    allocator.setReduceResourceRequest(Resources.createResource(1024));
     RMContainerAllocator.AssignedRequests assignedRequests =
         allocator.getAssignedRequests();
     RMContainerAllocator.ScheduledRequests scheduledRequests =
@@ -557,8 +558,8 @@ public void testNonAggressivelyPreemptReducers() throws Exception {
     clock.setTime(1);
     MyContainerAllocator allocator = new MyContainerAllocator(rm, conf,
         appAttemptId, mockJob, clock);
-    allocator.setMapResourceRequest(BuilderUtils.newResource(1024, 1));
-    allocator.setReduceResourceRequest(BuilderUtils.newResource(1024, 1));
+    allocator.setMapResourceRequest(Resources.createResource(1024));
+    allocator.setReduceResourceRequest(Resources.createResource(1024));
     RMContainerAllocator.AssignedRequests assignedRequests =
         allocator.getAssignedRequests();
     RMContainerAllocator.ScheduledRequests scheduledRequests =
@@ -622,8 +623,8 @@ public void testUnconditionalPreemptReducers() throws Exception {
     clock.setTime(1);
     MyContainerAllocator allocator = new MyContainerAllocator(rm, conf,
         appAttemptId, mockJob, clock);
-    allocator.setMapResourceRequest(BuilderUtils.newResource(1024, 1));
-    allocator.setReduceResourceRequest(BuilderUtils.newResource(1024, 1));
+    allocator.setMapResourceRequest(Resources.createResource(1024));
+    allocator.setReduceResourceRequest(Resources.createResource(1024));
     RMContainerAllocator.AssignedRequests assignedRequests =
         allocator.getAssignedRequests();
     RMContainerAllocator.ScheduledRequests scheduledRequests =
@@ -2205,8 +2206,8 @@ public void testReduceScheduling() throws Exception {
     int scheduledReduces = 0;
     int assignedMaps = 2;
     int assignedReduces = 0;
-    Resource mapResourceReqt = BuilderUtils.newResource(1024, 1);
-    Resource reduceResourceReqt = BuilderUtils.newResource(2 * 1024, 1);
+    Resource mapResourceReqt = Resources.createResource(1024);
+    Resource reduceResourceReqt = Resources.createResource(2 * 1024);
     int numPendingReduces = 4;
     float maxReduceRampupLimit = 0.5f;
     float reduceSlowStart = 0.2f;
@@ -2241,7 +2242,7 @@ public void testReduceScheduling() throws Exception {
     verify(allocator, never()).scheduleAllReduces();
 
     succeededMaps = 3;
-    doReturn(BuilderUtils.newResource(0, 0)).when(allocator).getResourceLimit();
+    doReturn(Resources.createResource(0)).when(allocator).getResourceLimit();
     allocator.scheduleReduces(
         totalMaps, succeededMaps,
         scheduledMaps, scheduledReduces,
@@ -2252,7 +2253,7 @@ public void testReduceScheduling() throws Exception {
     verify(allocator, times(1)).setIsReduceStarted(true);
 
     // Test reduce ramp-up
-    doReturn(BuilderUtils.newResource(100 * 1024, 100 * 1)).when(allocator)
+    doReturn(Resources.createResource(100 * 1024, 100 * 1)).when(allocator)
       .getResourceLimit();
     allocator.scheduleReduces(
         totalMaps, succeededMaps,
@@ -2266,7 +2267,7 @@ public void testReduceScheduling() throws Exception {
 
     // Test reduce ramp-down
     scheduledReduces = 3;
-    doReturn(BuilderUtils.newResource(10 * 1024, 10 * 1)).when(allocator)
+    doReturn(Resources.createResource(10 * 1024, 10 * 1)).when(allocator)
       .getResourceLimit();
     allocator.scheduleReduces(
         totalMaps, succeededMaps,
@@ -2282,7 +2283,7 @@ public void testReduceScheduling() throws Exception {
     // should be invoked twice.
     scheduledMaps = 2;
     assignedReduces = 2;
-    doReturn(BuilderUtils.newResource(10 * 1024, 10 * 1)).when(allocator)
+    doReturn(Resources.createResource(10 * 1024, 10 * 1)).when(allocator)
       .getResourceLimit();
     allocator.scheduleReduces(
         totalMaps, succeededMaps,
@@ -2300,7 +2301,7 @@ public void testReduceScheduling() throws Exception {
     // Test ramp-down when enough memory but not enough cpu resource
     scheduledMaps = 10;
     assignedReduces = 0;
-    doReturn(BuilderUtils.newResource(100 * 1024, 5 * 1)).when(allocator)
+    doReturn(Resources.createResource(100 * 1024, 5 * 1)).when(allocator)
         .getResourceLimit();
     allocator.scheduleReduces(totalMaps, succeededMaps, scheduledMaps,
         scheduledReduces, assignedMaps, assignedReduces, mapResourceReqt,
@@ -2309,7 +2310,7 @@ public void testReduceScheduling() throws Exception {
     verify(allocator, times(3)).rampDownReduces(anyInt());
 
     // Test ramp-down when enough cpu but not enough memory resource
-    doReturn(BuilderUtils.newResource(10 * 1024, 100 * 1)).when(allocator)
+    doReturn(Resources.createResource(10 * 1024, 100 * 1)).when(allocator)
         .getResourceLimit();
     allocator.scheduleReduces(totalMaps, succeededMaps, scheduledMaps,
         scheduledReduces, assignedMaps, assignedReduces, mapResourceReqt,

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/uam/UnmanagedApplicationManager.java

@@ -62,9 +62,9 @@
 import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
 import org.apache.hadoop.yarn.server.AMHeartbeatRequestHandler;
 import org.apache.hadoop.yarn.server.AMRMClientRelayer;
-import org.apache.hadoop.yarn.server.utils.BuilderUtils;
 import org.apache.hadoop.yarn.util.AsyncCallback;
 import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.apache.hadoop.yarn.util.resource.Resources;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -426,7 +426,7 @@ private void submitUnmanagedApp(ApplicationId appId)
 
     ContainerLaunchContext amContainer =
         this.recordFactory.newRecordInstance(ContainerLaunchContext.class);
-    Resource resource = BuilderUtils.newResource(1024, 1);
+    Resource resource = Resources.createResource(1024);
     context.setResource(resource);
     context.setAMContainerSpec(amContainer);
     submitRequest.setApplicationSubmissionContext(context);

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java

@@ -118,6 +118,7 @@
 import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
 import org.apache.hadoop.yarn.server.utils.BuilderUtils;
 import org.apache.hadoop.yarn.server.utils.YarnServerBuilderUtils;
+import org.apache.hadoop.yarn.util.resource.Resources;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
@@ -243,7 +244,7 @@ public NodeHeartbeatResponse nodeHeartbeat(NodeHeartbeatRequest request)
             ContainerId.newContainerId(appAttemptID, heartBeatID.get());
         ContainerLaunchContext launchContext = recordFactory
             .newRecordInstance(ContainerLaunchContext.class);
-        Resource resource = BuilderUtils.newResource(2, 1);
+        Resource resource = Resources.createResource(2, 1);
         long currentTime = System.currentTimeMillis();
         String user = "testUser";
         ContainerTokenIdentifier containerToken = BuilderUtils
@@ -286,7 +287,7 @@ public NodeHeartbeatResponse nodeHeartbeat(NodeHeartbeatRequest request)
             .newRecordInstance(ContainerLaunchContext.class);
         long currentTime = System.currentTimeMillis();
         String user = "testUser";
-        Resource resource = BuilderUtils.newResource(3, 1);
+        Resource resource = Resources.createResource(3, 1);
         ContainerTokenIdentifier containerToken = BuilderUtils
             .newContainerTokenIdentifier(BuilderUtils.newContainerToken(
                 secondContainerID, 0, InetAddress.getByName("localhost")
@@ -990,7 +991,7 @@ public void testRemovePreviousCompletedContainersFromContext() throws Exception
     ContainerId cId = ContainerId.newContainerId(appAttemptId, 1);
     Token containerToken =
         BuilderUtils.newContainerToken(cId, 0, "anyHost", 1234, "anyUser",
-            BuilderUtils.newResource(1024, 1), 0, 123,
+            Resources.createResource(1024), 0, 123,
             "password".getBytes(), 0);
     Container anyCompletedContainer = new ContainerImpl(conf, null,
         null, null, null,
@@ -1012,7 +1013,7 @@ public org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Cont
         ContainerId.newContainerId(appAttemptId, 3);
     Token runningContainerToken =
         BuilderUtils.newContainerToken(runningContainerId, 0, "anyHost",
-          1234, "anyUser", BuilderUtils.newResource(1024, 1), 0, 123,
+          1234, "anyUser", Resources.createResource(1024), 0, 123,
           "password".getBytes(), 0);
     Container runningContainer =
         new ContainerImpl(conf, null, null, null, null,
@@ -1071,7 +1072,7 @@ public void testCompletedContainersIsRecentlyStopped() throws Exception {
     ContainerId containerId = ContainerId.newContainerId(appAttemptId, 1);
     Token containerToken =
         BuilderUtils.newContainerToken(containerId, 0, "host", 1234, "user",
-            BuilderUtils.newResource(1024, 1), 0, 123,
+            Resources.createResource(1024), 0, 123,
             "password".getBytes(), 0);
 
     Container completedContainer = new ContainerImpl(conf, null,
@@ -1110,7 +1111,7 @@ public void testCleanedupApplicationContainerCleanup() throws IOException {
     ContainerId cId = ContainerId.newContainerId(appAttemptId, 1);
     Token containerToken =
         BuilderUtils.newContainerToken(cId, 0, "anyHost", 1234, "anyUser",
-            BuilderUtils.newResource(1024, 1), 0, 123,
+            Resources.createResource(1024), 0, 123,
             "password".getBytes(), 0);
     Container anyCompletedContainer = new ContainerImpl(conf, null,
         null, null, null,

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java

@@ -93,6 +93,7 @@
 import org.apache.hadoop.yarn.server.nodemanager.security.NMTokenSecretManagerInNM;
 import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
 import org.apache.hadoop.yarn.server.utils.BuilderUtils;
+import org.apache.hadoop.yarn.util.resource.Resources;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
@@ -439,7 +440,7 @@ public static Token createContainerToken(ContainerId cId, long rmIdentifier,
       NMContainerTokenSecretManager containerTokenSecretManager,
       LogAggregationContext logAggregationContext)
       throws IOException {
-    Resource r = BuilderUtils.newResource(1024, 1);
+    Resource r = Resources.createResource(1024);
     return createContainerToken(cId, rmIdentifier, nodeId, user, r,
         containerTokenSecretManager, logAggregationContext);
   }
@@ -449,7 +450,7 @@ public static Token createContainerToken(ContainerId cId, long rmIdentifier,
       NMContainerTokenSecretManager containerTokenSecretManager,
       LogAggregationContext logAggregationContext, ContainerType containerType)
       throws IOException {
-    Resource r = BuilderUtils.newResource(1024, 1);
+    Resource r = Resources.createResource(1024);
     return createContainerToken(cId, rmIdentifier, nodeId, user, r,
         containerTokenSecretManager, logAggregationContext, containerType);
   }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java

@@ -106,6 +106,7 @@
 import org.apache.hadoop.yarn.server.nodemanager.recovery.NMNullStateStoreService;
 import org.apache.hadoop.yarn.server.utils.BuilderUtils;
 import org.apache.hadoop.yarn.util.ControlledClock;
+import org.apache.hadoop.yarn.util.resource.Resources;
 import org.junit.Assert;
 import org.junit.Test;
 import org.mockito.ArgumentMatcher;
@@ -1391,7 +1392,7 @@ protected void scheduleContainer(Container container) {
       cId = BuilderUtils.newContainerId(appId, 1, timestamp, id);
       when(mockContainer.getId()).thenReturn(cId);
 
-      Resource resource = BuilderUtils.newResource(1024, 1);
+      Resource resource = Resources.createResource(1024);
       when(mockContainer.getResource()).thenReturn(resource);
       String host = "127.0.0.1";
       int port = 1234;

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java

@@ -126,6 +126,7 @@
 import org.apache.hadoop.yarn.util.AuxiliaryServiceHelper;
 import org.apache.hadoop.yarn.util.LinuxResourceCalculatorPlugin;
 import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin;
+import org.apache.hadoop.yarn.util.resource.Resources;
 import org.junit.Assert;
 import org.junit.Assume;
 import org.junit.Before;
@@ -1481,7 +1482,7 @@ public void handle(Event event) {
 
   protected Token createContainerToken(ContainerId cId, Priority priority,
       long createTime) throws InvalidToken {
-    Resource r = BuilderUtils.newResource(1024, 1);
+    Resource r = Resources.createResource(1024);
     ContainerTokenIdentifier containerTokenIdentifier =
         new ContainerTokenIdentifier(cId, context.getNodeId().toString(), user,
           r, System.currentTimeMillis() + 10000L, 123, DUMMY_RM_IDENTIFIER,

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java

@@ -143,6 +143,7 @@
 import org.apache.hadoop.yarn.server.utils.BuilderUtils;
 import org.apache.hadoop.yarn.util.ConverterUtils;
 import org.apache.hadoop.yarn.util.Records;
+import org.apache.hadoop.yarn.util.resource.Resources;
 import org.junit.Assert;
 import org.junit.Test;
 import org.mockito.ArgumentCaptor;
@@ -2293,7 +2294,7 @@ private ContainerId createContainer(ApplicationAttemptId appAttemptId1,
       long cId, ContainerType containerType) {
     ContainerId containerId = BuilderUtils.newContainerId(appAttemptId1,
         cId);
-    Resource r = BuilderUtils.newResource(1024, 1);
+    Resource r = Resources.createResource(1024);
     ContainerTokenIdentifier containerToken = new ContainerTokenIdentifier(
         containerId, context.getNodeId().toString(), user, r,
         System.currentTimeMillis() + 100000L, 123, DUMMY_RM_IDENTIFIER,

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java

@@ -86,6 +86,7 @@
 import org.apache.hadoop.yarn.util.ProcfsBasedProcessTree;
 import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin;
 import org.apache.hadoop.yarn.util.TestProcfsBasedProcessTree;
+import org.apache.hadoop.yarn.util.resource.Resources;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
@@ -301,7 +302,7 @@ public void testContainerKillOnMemoryOverflow() throws IOException,
     commands.add("/bin/bash");
     commands.add(scriptFile.getAbsolutePath());
     containerLaunchContext.setCommands(commands);
-    Resource r = BuilderUtils.newResource(0, 0);
+    Resource r = Resources.createResource(0);
     ContainerTokenIdentifier containerIdentifier =
         new ContainerTokenIdentifier(cId, context.getNodeId().toString(), user,
           r, System.currentTimeMillis() + 120000, 123, DUMMY_RM_IDENTIFIER,

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerBehaviorCompatibility.java

@@ -25,7 +25,7 @@
 import org.apache.hadoop.yarn.api.records.ExecutionType;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.BaseContainerManagerTest;
-import org.apache.hadoop.yarn.server.utils.BuilderUtils;
+import org.apache.hadoop.yarn.util.resource.Resources;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
@@ -70,7 +70,7 @@ public void testForceStartGuaranteedContainersWhenOppContainerDisabled()
     // on the RM side it won't check vcores at all.
     list.add(StartContainerRequest.newInstance(containerLaunchContext,
         createContainerToken(createContainerId(0), DUMMY_RM_IDENTIFIER,
-            context.getNodeId(), user, BuilderUtils.newResource(2048, 4),
+            context.getNodeId(), user, Resources.createResource(2048, 4),
             context.getContainerTokenSecretManager(), null,
             ExecutionType.GUARANTEED)));
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerOppContainersByResources.java

@@ -33,7 +33,7 @@
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.BaseContainerManagerTest;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.BaseContainerSchedulerTest;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState;
-import org.apache.hadoop.yarn.server.utils.BuilderUtils;
+import org.apache.hadoop.yarn.util.resource.Resources;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -161,7 +161,7 @@ public void testKillOpportunisticWhenNoResourcesAvailable() throws Exception {
         recordFactory.newRecordInstance(ContainerLaunchContext.class),
         createContainerToken(createContainerId(0), DUMMY_RM_IDENTIFIER,
             context.getNodeId(),
-            user, BuilderUtils.newResource(2048, 1),
+            user, Resources.createResource(2048),
             context.getContainerTokenSecretManager(), null,
             ExecutionType.GUARANTEED)));
 
@@ -170,7 +170,7 @@ public void testKillOpportunisticWhenNoResourcesAvailable() throws Exception {
         recordFactory.newRecordInstance(ContainerLaunchContext.class),
         createContainerToken(createContainerId(1), DUMMY_RM_IDENTIFIER,
             context.getNodeId(),
-            user, BuilderUtils.newResource(2048, 1),
+            user, Resources.createResource(2048),
             context.getContainerTokenSecretManager(), null,
             ExecutionType.OPPORTUNISTIC)));
 
@@ -219,7 +219,7 @@ public void testOpportunisticRunsWhenResourcesAvailable() throws Exception {
           recordFactory.newRecordInstance(ContainerLaunchContext.class),
           createContainerToken(createContainerId(i), DUMMY_RM_IDENTIFIER,
               context.getNodeId(),
-              user, BuilderUtils.newResource(512, 1),
+              user, Resources.createResource(512),
               context.getContainerTokenSecretManager(), null,
               ExecutionType.OPPORTUNISTIC)));
     }