Skip to content

Commit d312991

Browse files
Prabhu JosephPrabhu Joseph
Prabhu Joseph
authored and
Prabhu Joseph
committed
YARN-10198. Remove Validation of Queue Exist under Managed Parent Queue
Contributed by Peter Bacsko. Reviewed by Manikandan R.
1 parent 62c8015 commit d312991

File tree

2 files changed

+150
-44
lines changed

2 files changed

+150
-44
lines changed

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java‎

Lines changed: 79 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -29,9 +29,9 @@
2929
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
3030
import org.apache.hadoop.yarn.conf.YarnConfiguration;
3131
import org.apache.hadoop.yarn.exceptions.YarnException;
32-
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
3332
import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMapping.MappingType;
3433
import org.apache.hadoop.yarn.server.resourcemanager.placement.QueueMapping.QueueMappingBuilder;
34+
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
3535
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AutoCreatedLeafQueue;
3636
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
3737
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
@@ -65,13 +65,18 @@ public UserGroupMappingPlacementRule(){
6565
this(false, null, null);
6666
}
6767

68-
public UserGroupMappingPlacementRule(boolean overrideWithQueueMappings,
68+
@VisibleForTesting
69+
UserGroupMappingPlacementRule(boolean overrideWithQueueMappings,
6970
List<QueueMapping> newMappings, Groups groups) {
7071
this.mappings = newMappings;
7172
this.overrideWithQueueMappings = overrideWithQueueMappings;
7273
this.groups = groups;
7374
}
7475

76+
private String getPrimaryGroup(String user) throws IOException {
77+
return groups.getGroups(user).get(0);
78+
}
79+
7580
private String getSecondaryGroup(String user) throws IOException {
7681
List<String> groupsList = groups.getGroups(user);
7782
String secondaryGroup = null;
@@ -100,60 +105,27 @@ private ApplicationPlacementContext getPlacementForUser(String user)
100105
if (mapping.getParentQueue() != null
101106
&& mapping.getParentQueue().equals(PRIMARY_GROUP_MAPPING)
102107
&& mapping.getQueue().equals(CURRENT_USER_MAPPING)) {
103-
if (this.queueManager
104-
.getQueue(groups.getGroups(user).get(0)) != null) {
105-
QueueMapping queueMapping =
106-
QueueMappingBuilder.create()
107-
.type(mapping.getType())
108-
.source(mapping.getSource()).queue(user)
109-
.parentQueue(groups.getGroups(user).get(0))
110-
.build();
111-
validateQueueMapping(queueMapping);
112-
return getPlacementContext(queueMapping, user);
113-
} else {
114-
return null;
115-
}
108+
return getContextForGroupParent(user, mapping,
109+
getPrimaryGroup(user));
116110
} else if (mapping.getParentQueue() != null
117111
&& mapping.getParentQueue().equals(SECONDARY_GROUP_MAPPING)
118112
&& mapping.getQueue().equals(CURRENT_USER_MAPPING)) {
119-
String secondaryGroup = getSecondaryGroup(user);
120-
if (secondaryGroup != null) {
121-
QueueMapping queueMapping =
122-
QueueMappingBuilder.create()
123-
.type(mapping.getType())
124-
.source(mapping.getSource())
125-
.queue(user)
126-
.parentQueue(secondaryGroup)
127-
.build();
128-
validateQueueMapping(queueMapping);
129-
return getPlacementContext(queueMapping, user);
130-
} else {
131-
return null;
132-
}
113+
return getContextForGroupParent(user, mapping,
114+
getSecondaryGroup(user));
133115
} else if (mapping.getQueue().equals(CURRENT_USER_MAPPING)) {
134116
return getPlacementContext(mapping, user);
135117
} else if (mapping.getQueue().equals(PRIMARY_GROUP_MAPPING)) {
136-
if (this.queueManager
137-
.getQueue(groups.getGroups(user).get(0)) != null) {
138-
return getPlacementContext(mapping,
139-
groups.getGroups(user).get(0));
140-
} else {
141-
return null;
142-
}
118+
return getContextForPrimaryGroup(user, mapping);
143119
} else if (mapping.getQueue().equals(SECONDARY_GROUP_MAPPING)) {
144-
String secondaryGroup = getSecondaryGroup(user);
145-
if (secondaryGroup != null) {
146-
return getPlacementContext(mapping, secondaryGroup);
147-
} else {
148-
return null;
149-
}
120+
return getContextForSecondaryGroup(user, mapping);
150121
} else {
151122
return getPlacementContext(mapping);
152123
}
153124
}
125+
154126
if (user.equals(mapping.getSource())) {
155127
if (mapping.getQueue().equals(PRIMARY_GROUP_MAPPING)) {
156-
return getPlacementContext(mapping, groups.getGroups(user).get(0));
128+
return getPlacementContext(mapping, getPrimaryGroup(user));
157129
} else if (mapping.getQueue().equals(SECONDARY_GROUP_MAPPING)) {
158130
String secondaryGroup = getSecondaryGroup(user);
159131
if (secondaryGroup != null) {
@@ -180,6 +152,70 @@ private ApplicationPlacementContext getPlacementForUser(String user)
180152
return null;
181153
}
182154

155+
// invoked for mappings:
156+
// u:%user:[parent].%primary_group
157+
// u:%user:%primary_group
158+
private ApplicationPlacementContext getContextForPrimaryGroup(
159+
String user,
160+
QueueMapping mapping) throws IOException {
161+
String group = getPrimaryGroup(user);
162+
163+
CSQueue parent = queueManager.getQueue(mapping.getParentQueue());
164+
165+
if (parent instanceof ManagedParentQueue) {
166+
return getPlacementContext(mapping, group);
167+
} else {
168+
if (this.queueManager.getQueue(group) != null) {
169+
return getPlacementContext(mapping, group);
170+
} else {
171+
return null;
172+
}
173+
}
174+
}
175+
176+
// invoked for mappings
177+
// u:%user:%secondary_group
178+
// u:%user:[parent].%secondary_group
179+
private ApplicationPlacementContext getContextForSecondaryGroup(
180+
String user,
181+
QueueMapping mapping) throws IOException {
182+
String secondaryGroup = getSecondaryGroup(user);
183+
184+
if (secondaryGroup != null) {
185+
if (this.queueManager.getQueue(secondaryGroup) != null) {
186+
return getPlacementContext(mapping, secondaryGroup);
187+
} else {
188+
return null;
189+
}
190+
} else {
191+
return null;
192+
}
193+
}
194+
195+
// invoked for mappings:
196+
// u:%user:%primary_group.%user
197+
// u:%user:%secondary_group.%user
198+
private ApplicationPlacementContext getContextForGroupParent(
199+
String user,
200+
QueueMapping mapping,
201+
String group) throws IOException {
202+
203+
if (this.queueManager.getQueue(group) != null) {
204+
// replace the group string
205+
QueueMapping resolvedGroupMapping =
206+
QueueMappingBuilder.create()
207+
.type(mapping.getType())
208+
.source(mapping.getSource())
209+
.queue(user)
210+
.parentQueue(group)
211+
.build();
212+
validateQueueMapping(resolvedGroupMapping);
213+
return getPlacementContext(resolvedGroupMapping, user);
214+
} else {
215+
return null;
216+
}
217+
}
218+
183219
@Override
184220
public ApplicationPlacementContext getPlacementForApp(
185221
ApplicationSubmissionContext asc, String user)

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestUserGroupMappingPlacementRule.java‎

Lines changed: 71 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@
3636
import org.apache.hadoop.yarn.server.resourcemanager.placement.TestUserGroupMappingPlacementRule.QueueMappingTestData.QueueMappingTestDataBuilder;
3737
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueManager;
3838
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue;
39+
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ManagedParentQueue;
3940
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ParentQueue;
4041
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.PrimaryGroupMapping;
4142
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.SimpleGroupsMapping;
@@ -74,6 +75,9 @@ private void verifyQueueMapping(QueueMappingTestData queueMappingTestData)
7475
ParentQueue bsubgroup2 = mock(ParentQueue.class);
7576
when(bsubgroup2.getQueueName()).thenReturn("bsubgroup2");
7677

78+
ManagedParentQueue managedParent = mock(ManagedParentQueue.class);
79+
when(managedParent.getQueueName()).thenReturn("managedParent");
80+
7781
LeafQueue a = mock(LeafQueue.class);
7882
when(a.getQueueName()).thenReturn("a");
7983
when(a.getParent()).thenReturn(agroup);
@@ -88,6 +92,7 @@ private void verifyQueueMapping(QueueMappingTestData queueMappingTestData)
8892
when(queueManager.getQueue("agroup")).thenReturn(agroup);
8993
when(queueManager.getQueue("bsubgroup2")).thenReturn(bsubgroup2);
9094
when(queueManager.getQueue("asubgroup2")).thenReturn(asubgroup2);
95+
when(queueManager.getQueue("managedParent")).thenReturn(managedParent);
9196

9297
rule.setQueueManager(queueManager);
9398
ApplicationSubmissionContext asc = Records.newRecord(
@@ -236,7 +241,72 @@ public void testMapping() throws YarnException {
236241
.inputUser("a")
237242
.expectedQueue("q1")
238243
.build());
239-
244+
245+
// "agroup" queue exists
246+
verifyQueueMapping(
247+
QueueMappingTestDataBuilder.create()
248+
.queueMapping(QueueMappingBuilder.create()
249+
.type(MappingType.USER)
250+
.source("%user")
251+
.queue("%primary_group")
252+
.parentQueue("bsubgroup2")
253+
.build())
254+
.inputUser("a")
255+
.expectedQueue("agroup")
256+
.build());
257+
258+
// "abcgroup" queue doesn't exist, %primary_group queue, not managed parent
259+
verifyQueueMapping(
260+
QueueMappingTestDataBuilder.create()
261+
.queueMapping(QueueMappingBuilder.create()
262+
.type(MappingType.USER)
263+
.source("%user")
264+
.queue("%primary_group")
265+
.parentQueue("bsubgroup2")
266+
.build())
267+
.inputUser("abc")
268+
.expectedQueue("default")
269+
.build());
270+
271+
// "abcgroup" queue doesn't exist, %primary_group queue, managed parent
272+
verifyQueueMapping(
273+
QueueMappingTestDataBuilder.create()
274+
.queueMapping(QueueMappingBuilder.create()
275+
.type(MappingType.USER)
276+
.source("%user")
277+
.queue("%primary_group")
278+
.parentQueue("managedParent")
279+
.build())
280+
.inputUser("abc")
281+
.expectedQueue("abcgroup")
282+
.build());
283+
284+
// "abcgroup" queue doesn't exist, %secondary_group queue
285+
verifyQueueMapping(
286+
QueueMappingTestDataBuilder.create()
287+
.queueMapping(QueueMappingBuilder.create()
288+
.type(MappingType.USER)
289+
.source("%user")
290+
.queue("%secondary_group")
291+
.parentQueue("bsubgroup2")
292+
.build())
293+
.inputUser("abc")
294+
.expectedQueue("default")
295+
.build());
296+
297+
// "asubgroup2" queue exists, %secondary_group queue
298+
verifyQueueMapping(
299+
QueueMappingTestDataBuilder.create()
300+
.queueMapping(QueueMappingBuilder.create()
301+
.type(MappingType.USER)
302+
.source("%user")
303+
.queue("%secondary_group")
304+
.parentQueue("bsubgroup2")
305+
.build())
306+
.inputUser("a")
307+
.expectedQueue("asubgroup2")
308+
.build());
309+
240310
// specify overwritten, and see if user specified a queue, and it will be
241311
// overridden
242312
verifyQueueMapping(

0 commit comments

Comments
 (0)