Merge branch 'apache:trunk' into YARN-11158-V3

Author: slfan1989

LICENSE-binary

@@ -241,8 +241,8 @@ com.google.guava:guava:27.0-jre
 com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava
 com.microsoft.azure:azure-storage:7.0.0
 com.nimbusds:nimbus-jose-jwt:9.8.1
-com.squareup.okhttp3:okhttp:4.9.3
-com.squareup.okio:okio:1.6.0
+com.squareup.okhttp3:okhttp:4.10.0
+com.squareup.okio:okio:3.2.0
 com.zaxxer:HikariCP:4.0.3
 commons-beanutils:commons-beanutils:1.9.3
 commons-cli:commons-cli:1.2

hadoop-client-modules/hadoop-client-runtime/pom.xml

@@ -148,6 +148,7 @@
                       <!-- Leave javax APIs that are stable -->
                       <!-- the jdk ships part of the javax.annotation namespace, so if we want to relocate this we'll have to care it out by class :( -->
                       <exclude>com.google.code.findbugs:jsr305</exclude>
+                      <exclude>io.netty:*</exclude>
                       <exclude>io.dropwizard.metrics:metrics-core</exclude>
                       <exclude>org.eclipse.jetty:jetty-servlet</exclude>
                       <exclude>org.eclipse.jetty:jetty-security</exclude>
@@ -156,6 +157,8 @@
                       <exclude>org.bouncycastle:*</exclude>
                       <!-- Leave snappy that includes native methods which cannot be relocated. -->
                       <exclude>org.xerial.snappy:*</exclude>
+                      <!-- leave out kotlin classes -->
+                      <exclude>org.jetbrains.kotlin:*</exclude>
                     </excludes>
                   </artifactSet>
                   <filters>

hadoop-common-project/hadoop-common/pom.xml

@@ -383,6 +383,11 @@
       <artifactId>mockwebserver</artifactId>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>com.squareup.okio</groupId>
+      <artifactId>okio-jvm</artifactId>
+      <scope>test</scope>
+    </dependency>
     <dependency>
       <groupId>dnsjava</groupId>
       <artifactId>dnsjava</artifactId>

hadoop-common-project/hadoop-common/src/site/markdown/DeprecatedProperties.md

@@ -208,7 +208,8 @@ The following table lists the configuration property names that are deprecated i
 | mapred.task.profile.params | mapreduce.task.profile.params |
 | mapred.task.profile.reduces | mapreduce.task.profile.reduces |
 | mapred.task.timeout | mapreduce.task.timeout |
-| mapred.tasktracker.indexcache.mb | mapreduce.tasktracker.indexcache.mb |
+| mapred.tasktracker.indexcache.mb | mapreduce.reduce.shuffle.indexcache.mb |
+| mapreduce.tasktracker.indexcache.mb | mapreduce.reduce.shuffle.indexcache.mb |
 | mapred.tasktracker.map.tasks.maximum | mapreduce.tasktracker.map.tasks.maximum |
 | mapred.tasktracker.memory\_calculator\_plugin | mapreduce.tasktracker.resourcecalculatorplugin |
 | mapred.tasktracker.memorycalculatorplugin | mapreduce.tasktracker.resourcecalculatorplugin |

hadoop-hdfs-project/hadoop-hdfs-client/pom.xml

@@ -37,6 +37,16 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
     <dependency>
       <groupId>com.squareup.okhttp3</groupId>
       <artifactId>okhttp</artifactId>
+      <exclusions>
+        <exclusion>
+          <groupId>com.squareup.okio</groupId>
+          <artifactId>okio-jvm</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+    <dependency>
+      <groupId>com.squareup.okio</groupId>
+      <artifactId>okio-jvm</artifactId>
     </dependency>
     <dependency>
       <groupId>org.jetbrains.kotlin</groupId>

hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/PoolAlignmentContext.java

@@ -71,8 +71,7 @@ public void receiveResponseState(RpcHeaderProtos.RpcResponseHeaderProto header)
    */
   @Override
   public void updateRequestState(RpcHeaderProtos.RpcRequestHeaderProto.Builder header) {
-    long maxStateId = Long.max(poolLocalStateId.get(), sharedGlobalStateId.get());
-    header.setStateId(maxStateId);
+    header.setStateId(poolLocalStateId.get());
   }
 
   /**

hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestPoolAlignmentContext.java

@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.ipc.protobuf.RpcHeaderProtos.RpcRequestHeaderProto;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+
+public class TestPoolAlignmentContext {
+  @Test
+  public void testNamenodeRequestsOnlyUsePoolLocalStateID() {
+    RouterStateIdContext routerStateIdContext = new RouterStateIdContext(new Configuration());
+    String namespaceId = "namespace1";
+    routerStateIdContext.getNamespaceStateId(namespaceId).accumulate(20L);
+    PoolAlignmentContext poolContext1 = new PoolAlignmentContext(routerStateIdContext, namespaceId);
+    PoolAlignmentContext poolContext2 = new PoolAlignmentContext(routerStateIdContext, namespaceId);
+
+    assertRequestHeaderStateId(poolContext1, Long.MIN_VALUE);
+    assertRequestHeaderStateId(poolContext2, Long.MIN_VALUE);
+    Assertions.assertEquals(20L, poolContext1.getLastSeenStateId());
+    Assertions.assertEquals(20L, poolContext2.getLastSeenStateId());
+
+    poolContext1.advanceClientStateId(30L);
+    assertRequestHeaderStateId(poolContext1, 30L);
+    assertRequestHeaderStateId(poolContext2, Long.MIN_VALUE);
+    Assertions.assertEquals(20L, poolContext1.getLastSeenStateId());
+    Assertions.assertEquals(20L, poolContext2.getLastSeenStateId());
+  }
+
+  private void assertRequestHeaderStateId(PoolAlignmentContext poolAlignmentContext,
+      Long expectedValue) {
+    RpcRequestHeaderProto.Builder builder = RpcRequestHeaderProto.newBuilder();
+    poolAlignmentContext.updateRequestState(builder);
+    Assertions.assertEquals(expectedValue, builder.getStateId());
+  }
+}

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java

@@ -520,7 +520,7 @@ public void addVolume(final StorageLocation location,
 
     for (final NamespaceInfo nsInfo : nsInfos) {
       String bpid = nsInfo.getBlockPoolID();
-      try (AutoCloseDataSetLock l = lockManager.writeLock(LockLevel.BLOCK_POOl, bpid)) {
+      try {
         fsVolume.addBlockPool(bpid, this.conf, this.timer);
         fsVolume.getVolumeMap(bpid, tempVolumeMap, ramDiskReplicaTracker);
       } catch (IOException e) {

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/IndexCache.java

@@ -23,7 +23,7 @@
 import java.util.concurrent.atomic.AtomicInteger;
 
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.mapreduce.server.tasktracker.TTConfig;
+import org.apache.hadoop.mapreduce.MRJobConfig;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -43,7 +43,7 @@ class IndexCache {
   public IndexCache(JobConf conf) {
     this.conf = conf;
     totalMemoryAllowed =
-      conf.getInt(TTConfig.TT_INDEX_CACHE, 10) * 1024 * 1024;
+      conf.getInt(MRJobConfig.SHUFFLE_INDEX_CACHE, 10) * 1024 * 1024;
     LOG.info("IndexCache created with max memory = " + totalMemoryAllowed);
   }
 

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java

@@ -577,6 +577,8 @@ public interface MRJobConfig {
   public static final String MAX_SHUFFLE_FETCH_HOST_FAILURES = "mapreduce.reduce.shuffle.max-host-failures";
   public static final int DEFAULT_MAX_SHUFFLE_FETCH_HOST_FAILURES = 5;
 
+  public static final String SHUFFLE_INDEX_CACHE = "mapreduce.reduce.shuffle.indexcache.mb";
+
   public static final String REDUCE_SKIP_INCR_PROC_COUNT = "mapreduce.reduce.skip.proc-count.auto-incr";
 
   public static final String REDUCE_SKIP_MAXGROUPS = "mapreduce.reduce.skip.maxgroups";