Merge branch 'apache:trunk' into YARN-11394

Author: slfan1989

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/WritableName.java

@@ -92,7 +92,7 @@ public static synchronized Class<?> getClass(String name, Configuration conf
                                             ) throws IOException {
     Class<?> writableClass = NAME_TO_CLASS.get(name);
     if (writableClass != null)
-      return writableClass.asSubclass(Writable.class);
+      return writableClass;
     try {
       return conf.getClassByName(name);
     } catch (ClassNotFoundException e) {

hadoop-common-project/hadoop-common/src/site/markdown/SecureMode.md

@@ -20,7 +20,9 @@ Hadoop in Secure Mode
 Introduction
 ------------
 
-This document describes how to configure authentication for Hadoop in secure mode. When Hadoop is configured to run in secure mode, each Hadoop service and each user must be authenticated by Kerberos.
+In its default configuration, we expect you to make sure attackers don't have access to your Hadoop cluster by restricting all network access. If you want any restrictions on who can remotely access data or submit work, you MUST secure authentication and access for your Hadoop cluster as described in this document.
+
+When Hadoop is configured to run in secure mode, each Hadoop service and each user must be authenticated by Kerberos.
 
 Forward and reverse host lookup for all service hosts must be configured correctly to allow services to authenticate with each other. Host lookups may be configured using either DNS or `/etc/hosts` files. Working knowledge of Kerberos and DNS is recommended before attempting to configure Hadoop services in Secure Mode.
 

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFile.java

@@ -26,6 +26,9 @@
 import org.apache.hadoop.io.SequenceFile.Metadata;
 import org.apache.hadoop.io.compress.CompressionCodec;
 import org.apache.hadoop.io.compress.DefaultCodec;
+import org.apache.hadoop.io.serializer.Deserializer;
+import org.apache.hadoop.io.serializer.Serialization;
+import org.apache.hadoop.io.serializer.Serializer;
 import org.apache.hadoop.io.serializer.avro.AvroReflectSerialization;
 import org.apache.hadoop.test.GenericTestUtils;
 import org.apache.hadoop.util.ReflectionUtils;
@@ -756,6 +759,122 @@ public void testSequenceFileWriter() throws Exception {
     }
   }
 
+  @Test
+  public void testSerializationUsingWritableNameAlias() throws IOException {
+    Configuration config = new Configuration();
+    config.set(CommonConfigurationKeys.IO_SERIALIZATIONS_KEY, SimpleSerializer.class.getName());
+    Path path = new Path(System.getProperty("test.build.data", "."),
+        "SerializationUsingWritableNameAlias");
+
+    // write with the original serializable class
+    SequenceFile.Writer writer = SequenceFile.createWriter(
+        config,
+        SequenceFile.Writer.file(path),
+        SequenceFile.Writer.keyClass(SimpleSerializable.class),
+        SequenceFile.Writer.valueClass(SimpleSerializable.class));
+
+    int max = 10;
+    try {
+      SimpleSerializable val = new SimpleSerializable();
+      val.setId(-1);
+      for (int i = 0; i < max; i++) {
+        SimpleSerializable key = new SimpleSerializable();
+        key.setId(i);
+        writer.append(key, val);
+      }
+    } finally {
+      writer.close();
+    }
+
+    // override name so it gets forced to the new serializable
+    WritableName.setName(AnotherSimpleSerializable.class, SimpleSerializable.class.getName());
+
+    // read and expect our new serializable, and all the correct values read
+    SequenceFile.Reader reader = new SequenceFile.Reader(
+        config,
+        SequenceFile.Reader.file(path));
+
+    AnotherSimpleSerializable key = new AnotherSimpleSerializable();
+    int count = 0;
+    while (true) {
+      key = (AnotherSimpleSerializable) reader.next(key);
+      if (key == null) {
+        // make sure we exhausted all the ints we wrote
+        assertEquals(count, max);
+        break;
+      }
+      assertEquals(count++, key.getId());
+    }
+  }
+
+  public static class SimpleSerializable implements Serializable {
+
+    private int id;
+
+    public int getId() {
+      return id;
+    }
+
+    public void setId(int id) {
+      this.id = id;
+    }
+  }
+
+  public static class AnotherSimpleSerializable extends SimpleSerializable {
+  }
+
+  public static class SimpleSerializer implements Serialization<SimpleSerializable> {
+
+    @Override
+    public boolean accept(Class<?> c) {
+      return SimpleSerializable.class.isAssignableFrom(c);
+    }
+
+    @Override
+    public Serializer<SimpleSerializable> getSerializer(Class<SimpleSerializable> c) {
+      return new Serializer<SimpleSerializable>() {
+        private DataOutputStream out;
+        @Override
+        public void open(OutputStream out) throws IOException {
+          this.out = new DataOutputStream(out);
+        }
+
+        @Override
+        public void serialize(SimpleSerializable simpleSerializable) throws IOException {
+          out.writeInt(simpleSerializable.getId());
+        }
+
+        @Override
+        public void close() throws IOException {
+          out.close();
+        }
+      };
+    }
+
+    @Override
+    public Deserializer<SimpleSerializable> getDeserializer(Class<SimpleSerializable> c) {
+      return new Deserializer<SimpleSerializable>() {
+        private DataInputStream dis;
+        @Override
+        public void open(InputStream in) throws IOException {
+          dis = new DataInputStream(in);
+        }
+
+        @Override
+        public SimpleSerializable deserialize(SimpleSerializable simpleSerializable)
+            throws IOException {
+          simpleSerializable.setId(dis.readInt());
+          return simpleSerializable;
+        }
+
+        @Override
+        public void close() throws IOException {
+          dis.close();
+        }
+      };
+    }
+  }
+
   /** For debugging and testing. */
   public static void main(String[] args) throws Exception {
     int count = 1024 * 1024;

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestWritableName.java

@@ -24,8 +24,14 @@
 import java.util.Random;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.io.serializer.Deserializer;
+import org.apache.hadoop.io.serializer.Serialization;
+import org.apache.hadoop.io.serializer.SerializationFactory;
+import org.apache.hadoop.io.serializer.Serializer;
 import org.junit.Test;
-
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
 
 /** Unit tests for WritableName. */
@@ -63,6 +69,28 @@ public boolean equals(Object o) {
     }
   }
 
+  private static class SimpleSerializable {
+
+  }
+
+  private static class SimpleSerializer implements Serialization<SimpleSerializable> {
+
+    @Override
+    public boolean accept(Class<?> c) {
+      return c.equals(SimpleSerializable.class);
+    }
+
+    @Override
+    public Serializer<SimpleSerializable> getSerializer(Class<SimpleSerializable> c) {
+      return null;
+    }
+
+    @Override
+    public Deserializer<SimpleSerializable> getDeserializer(Class<SimpleSerializable> c) {
+      return null;
+    }
+  }
+
   private static final String testName = "mystring";
 
   @Test
@@ -95,7 +123,27 @@ public void testAddName() throws Exception {
     // check original name still works
     test = WritableName.getClass(testName, conf);
     assertTrue(test.equals(SimpleWritable.class));
+  }
+
+  @Test
+  public void testAddNameSerializable() throws Exception {
+    Configuration conf = new Configuration();
+    conf.set(CommonConfigurationKeys.IO_SERIALIZATIONS_KEY, SimpleSerializer.class.getName());
+    SerializationFactory serializationFactory =
+        new SerializationFactory(conf);
 
+    String altName = testName + ".alt";
+
+    WritableName.addName(SimpleSerializable.class, altName);
+
+    Class<?> test = WritableName.getClass(altName, conf);
+    assertEquals(test, SimpleSerializable.class);
+    assertNotNull(serializationFactory.getSerialization(test));
+
+    // check original name still works
+    test = WritableName.getClass(SimpleSerializable.class.getName(), conf);
+    assertEquals(test, SimpleSerializable.class);
+    assertNotNull(serializationFactory.getSerialization(test));
   }
 
   @Test

hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/fs/TestXAttr.java

@@ -19,8 +19,8 @@
 package org.apache.hadoop.fs;
 
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotSame;
+import static org.junit.Assert.assertNotEquals;
 
 import org.junit.BeforeClass;
 import org.junit.Test;
@@ -77,18 +77,18 @@ public void testXAttrEquals() {
     assertEquals(XATTR3, XATTR3);
     assertEquals(XATTR4, XATTR4);
     assertEquals(XATTR5, XATTR5);
-    assertFalse(XATTR1.equals(XATTR2));
-    assertFalse(XATTR2.equals(XATTR3));
-    assertFalse(XATTR3.equals(XATTR4));
-    assertFalse(XATTR4.equals(XATTR5));
+    assertNotEquals(XATTR1, XATTR2);
+    assertNotEquals(XATTR2, XATTR3);
+    assertNotEquals(XATTR3, XATTR4);
+    assertNotEquals(XATTR4, XATTR5);
   }
 
   @Test
   public void testXAttrHashCode() {
     assertEquals(XATTR.hashCode(), XATTR1.hashCode());
-    assertFalse(XATTR1.hashCode() == XATTR2.hashCode());
-    assertFalse(XATTR2.hashCode() == XATTR3.hashCode());
-    assertFalse(XATTR3.hashCode() == XATTR4.hashCode());
-    assertFalse(XATTR4.hashCode() == XATTR5.hashCode());
+    assertNotEquals(XATTR1.hashCode(), XATTR2.hashCode());
+    assertNotEquals(XATTR2.hashCode(), XATTR3.hashCode());
+    assertNotEquals(XATTR3.hashCode(), XATTR4.hashCode());
+    assertNotEquals(XATTR4.hashCode(), XATTR5.hashCode());
   }
 }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java

@@ -1557,7 +1557,8 @@ public ReplicaHandler recoverAppend(
   @Override // FsDatasetSpi
   public Replica recoverClose(ExtendedBlock b, long newGS,
       long expectedBlockLen) throws IOException {
-    LOG.info("Recover failed close " + b);
+    LOG.info("Recover failed close {}, new GS:{}, expectedBlockLen:{}",
+        b, newGS, expectedBlockLen);
     while (true) {
       try {
         try (AutoCloseableLock lock = lockManager.writeLock(LockLevel.VOLUME,

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java

@@ -5939,6 +5939,8 @@ LocatedBlock bumpBlockGenerationStamp(ExtendedBlock block,
     }
     // Ensure we record the new generation stamp
     getEditLog().logSync();
+    LOG.info("bumpBlockGenerationStamp({}, client={}) success",
+        locatedBlock.getBlock(), clientName);
     return locatedBlock;
   }
 

hadoop-tools/hadoop-distcp/src/site/markdown/DistCp.md.vm

@@ -653,7 +653,7 @@ checksums if the checksum algorithm between the two stores is different.
 * `distcp.update.modification.time` would only be used if either of the two
   stores don't have checksum validation resulting in incompatible checksum
   comparison between the two. Even if the property is set to true, it won't
-  be used if their is valid checksum comparison between the two stores.
+  be used if there is valid checksum comparison between the two stores.
 
 To turn off the modification time check, set this in your core-site.xml
 ```xml

hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/contract/AbstractContractDistCpTest.java

@@ -899,8 +899,7 @@ public void testDistCpUpdateCheckFileSkip() throws Exception {
     // Creating a source file with certain dataset.
     byte[] sourceBlock = dataset(10, 'a', 'z');
 
-    // Write the dataset and as well create the target path.
-    ContractTestUtils.createFile(localFS, dest, true, sourceBlock);
+    // Write the dataset.
     ContractTestUtils
         .writeDataset(remoteFS, source, sourceBlock, sourceBlock.length,
             1024, true);

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml

@@ -30,7 +30,6 @@
     <!-- Basedir eeded for generating FindBugs warnings using parent pom -->
     <yarn.basedir>${project.parent.parent.basedir}</yarn.basedir>
     <container-executor.conf.dir>../etc/hadoop</container-executor.conf.dir>
-    <extra.libhadoop.rpath>../lib/native</extra.libhadoop.rpath>
     <container-executor.additional_cflags></container-executor.additional_cflags>
   </properties>
 
@@ -224,7 +223,6 @@
                   <source>${basedir}/src</source>
                   <vars>
                     <HADOOP_CONF_DIR>${container-executor.conf.dir}</HADOOP_CONF_DIR>
-                    <EXTRA_LIBHADOOP_RPATH>${extra.libhadoop.rpath}</EXTRA_LIBHADOOP_RPATH>
                     <JVM_ARCH_DATA_MODEL>${sun.arch.data.model}</JVM_ARCH_DATA_MODEL>
                   </vars>
                   <env>