Merge branch 'trunk' into HADOOP-19425-PART3

Author: slfan1989

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java

@@ -379,10 +379,6 @@ private static class DeprecatedKeyInfo {
       this.customMessage = customMessage;
     }
 
-    private final String getWarningMessage(String key) {
-      return getWarningMessage(key, null);
-    }
-
     /**
      * Method to provide the warning message. It gives the custom message if
      * non-null, and default message otherwise.
@@ -412,12 +408,9 @@ private String getWarningMessage(String key, String source) {
       return warningMessage;
     }
 
-    boolean getAndSetAccessed() {
-      return accessed.getAndSet(true);
-    }
-
-    public void clearAccessed() {
-      accessed.set(false);
+    void logDeprecation(String name, String source) {
+      LOG_DEPRECATION.info(getWarningMessage(name, source));
+      this.accessed.set(true);
     }
   }
 
@@ -728,12 +721,10 @@ private String[] handleDeprecation(DeprecationContext deprecations,
     }
     // Initialize the return value with requested name
     String[] names = new String[]{name};
-    // Deprecated keys are logged once and an updated names are returned
+    // Deprecated keys are logged and updated names are returned
     DeprecatedKeyInfo keyInfo = deprecations.getDeprecatedKeyMap().get(name);
     if (keyInfo != null) {
-      if (!keyInfo.getAndSetAccessed()) {
-        logDeprecation(keyInfo.getWarningMessage(name));
-      }
+      keyInfo.logDeprecation(name, null);
       // Override return value for deprecated keys
       names = keyInfo.newKeys;
     }
@@ -1462,13 +1453,6 @@ void logDeprecation(String message) {
     LOG_DEPRECATION.info(message);
   }
 
-  void logDeprecationOnce(String name, String source) {
-    DeprecatedKeyInfo keyInfo = getDeprecatedKeyInfo(name);
-    if (keyInfo != null && !keyInfo.getAndSetAccessed()) {
-      LOG_DEPRECATION.info(keyInfo.getWarningMessage(name, source));
-    }
-  }
-
   /**
    * Unset a previously set property.
    * @param name the property name
@@ -2448,7 +2432,10 @@ private CredentialEntry getCredentialEntry(CredentialProvider provider,
     if (oldName != null) {
       entry = provider.getCredentialEntry(oldName);
       if (entry != null) {
-        logDeprecationOnce(oldName, provider.toString());
+        DeprecatedKeyInfo ki = getDeprecatedKeyInfo(oldName);
+        if (ki != null) {
+          ki.logDeprecation(oldName, provider.toString());
+        }
         return entry;
       }
     }
@@ -2459,7 +2446,7 @@ private CredentialEntry getCredentialEntry(CredentialProvider provider,
       for (String newName : keyInfo.newKeys) {
         entry = provider.getCredentialEntry(newName);
         if (entry != null) {
-          logDeprecationOnce(name, null);
+          keyInfo.logDeprecation(name, null);
           return entry;
         }
       }
@@ -3433,8 +3420,7 @@ void handleEndProperty() {
           deprecations.getDeprecatedKeyMap().get(confName);
 
       if (keyInfo != null) {
-        logDeprecation(keyInfo.getWarningMessage(confName, wrapper.toString()));
-        keyInfo.clearAccessed();
+        keyInfo.logDeprecation(confName, wrapper.toString());
         for (String key : keyInfo.newKeys) {
           // update new keys with deprecated key's value
           results.add(new ParsedItem(

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/service/launcher/IrqHandler.java

@@ -21,10 +21,9 @@
 import java.util.concurrent.atomic.AtomicInteger;
 
 import org.apache.hadoop.util.Preconditions;
+import org.apache.hadoop.util.SignalUtil;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import sun.misc.Signal;
-import sun.misc.SignalHandler;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
@@ -38,8 +37,7 @@
  */
 @InterfaceAudience.Private
 @InterfaceStability.Unstable
-@SuppressWarnings("UseOfSunClasses")
-public final class IrqHandler implements SignalHandler {
+public final class IrqHandler implements SignalUtil.Handler {
   private static final Logger LOG = LoggerFactory.getLogger(IrqHandler.class);
 
   /**
@@ -68,7 +66,7 @@ public final class IrqHandler implements SignalHandler {
   /**
    * Stored signal.
    */
-  private Signal signal;
+  private SignalUtil.Signal signal;
 
   /**
    * Create an IRQ handler bound to the specific interrupt.
@@ -89,8 +87,8 @@ public IrqHandler(String name, Interrupted handler) {
   public void bind() {
     Preconditions.checkState(signal == null, "Handler already bound");
     try {
-      signal = new Signal(name);
-      Signal.handle(signal, this);
+      signal = new SignalUtil.Signal(name);
+      SignalUtil.handle(signal, this);
     } catch (IllegalArgumentException e) {
       throw new IllegalArgumentException(
           "Could not set handler for signal \"" + name + "\"."
@@ -110,7 +108,7 @@ public String getName() {
    * Raise the signal.
    */
   public void raise() {
-    Signal.raise(signal);
+    SignalUtil.raise(signal);
   }
 
   @Override
@@ -123,7 +121,7 @@ public String toString() {
    * @param s signal raised
    */
   @Override
-  public void handle(Signal s) {
+  public void handle(SignalUtil.Signal s) {
     signalCount.incrementAndGet();
     InterruptData data = new InterruptData(s.getName(), s.getNumber());
     LOG.info("Interrupted: {}", data);

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SignalLogger.java

@@ -19,8 +19,6 @@
 package org.apache.hadoop.util;
 
 import org.slf4j.Logger;
-import sun.misc.Signal;
-import sun.misc.SignalHandler;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
@@ -41,13 +39,13 @@ public enum SignalLogger {
   /**
    * Our signal handler.
    */
-  private static class Handler implements SignalHandler {
+  private static class Handler implements SignalUtil.Handler {
     final private Logger log;
-    final private SignalHandler prevHandler;
+    final private SignalUtil.Handler prevHandler;
 
     Handler(String name, Logger log) {
       this.log = log;
-      prevHandler = Signal.handle(new Signal(name), this);
+      prevHandler = SignalUtil.handle(new SignalUtil.Signal(name), this);
     }
 
     /**
@@ -56,9 +54,8 @@ private static class Handler implements SignalHandler {
      * @param signal    The incoming signal
      */
     @Override
-    public void handle(Signal signal) {
-      log.error("RECEIVED SIGNAL " + signal.getNumber() +
-          ": SIG" + signal.getName());
+    public void handle(SignalUtil.Signal signal) {
+      log.error("RECEIVED SIGNAL {}: SIG{}", signal.getNumber(), signal.getName());
       prevHandler.handle(signal);
     }
   }
@@ -75,7 +72,7 @@ public void register(final Logger log) {
     registered = true;
     StringBuilder bld = new StringBuilder();
     bld.append("registered UNIX signal handlers for [");
-    final String SIGNALS[] = { "TERM", "HUP", "INT" };
+    final String[] SIGNALS = {"TERM", "HUP", "INT"};
     String separator = "";
     for (String signalName : SIGNALS) {
       try {

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SignalUtil.java

@@ -0,0 +1,168 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.util;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+
+import org.apache.hadoop.util.dynamic.BindingUtils;
+import org.apache.hadoop.util.dynamic.DynConstructors;
+import org.apache.hadoop.util.dynamic.DynMethods;
+
+import java.lang.reflect.Method;
+import java.lang.reflect.Proxy;
+
+@InterfaceAudience.Private
+public class SignalUtil {
+
+  static final Class<?> JDK_SIGNAL_CLAZZ =
+      BindingUtils.loadClassSafely("sun.misc.Signal");
+  static final Class<?> JDK_SIGNAL_HANDLER_CLAZZ =
+      BindingUtils.loadClassSafely("sun.misc.SignalHandler");
+
+  static final DynConstructors.Ctor<?> JDK_SIGNAL_CTOR =
+      new DynConstructors.Builder()
+          .impl(JDK_SIGNAL_CLAZZ, String.class)
+          .build();
+
+  static final DynMethods.StaticMethod JDK_SIGNAL_HANDLE_STATIC_METHOD =
+      new DynMethods.Builder("handle")
+          .impl(JDK_SIGNAL_CLAZZ, JDK_SIGNAL_CLAZZ, JDK_SIGNAL_HANDLER_CLAZZ)
+          .buildStatic();
+
+  static final DynMethods.StaticMethod JDK_SIGNAL_RAISE_STATIC_METHOD =
+      new DynMethods.Builder("raise")
+          .impl(JDK_SIGNAL_CLAZZ, JDK_SIGNAL_CLAZZ)
+          .buildStatic();
+
+  static final DynMethods.UnboundMethod JDK_SIGNAL_HANDLER_HANDLE_UNBOUND_METHOD =
+      new DynMethods.Builder("handle")
+          .impl(JDK_SIGNAL_HANDLER_CLAZZ, JDK_SIGNAL_CLAZZ)
+          .build();
+
+  @InterfaceAudience.Private
+  public static class Signal {
+    private static final DynMethods.UnboundMethod GET_NUMBER_UNBOUND_METHOD =
+        new DynMethods.Builder("getNumber").impl(JDK_SIGNAL_CLAZZ).build();
+
+    private static final DynMethods.UnboundMethod GET_NAME_UNBOUND_METHOD =
+        new DynMethods.Builder("getName").impl(JDK_SIGNAL_CLAZZ).build();
+
+    private final Object/* sun.misc.Signal */ delegate;
+    private final DynMethods.BoundMethod getNumberMethod;
+    private final DynMethods.BoundMethod getNameMethod;
+
+    public Signal(String name) {
+      Preconditions.checkNotNull(name);
+      this.delegate = JDK_SIGNAL_CTOR.newInstance(name);
+      this.getNumberMethod = GET_NUMBER_UNBOUND_METHOD.bind(delegate);
+      this.getNameMethod = GET_NAME_UNBOUND_METHOD.bind(delegate);
+    }
+
+    public Signal(Object delegate) {
+      Preconditions.checkArgument(JDK_SIGNAL_CLAZZ.isInstance(delegate),
+          String.format("Expected class is '%s', but actual class is '%s'",
+              JDK_SIGNAL_CLAZZ.getName(), delegate.getClass().getName()));
+      this.delegate = delegate;
+      this.getNumberMethod = GET_NUMBER_UNBOUND_METHOD.bind(delegate);
+      this.getNameMethod = GET_NAME_UNBOUND_METHOD.bind(delegate);
+    }
+
+    public int getNumber() {
+      return getNumberMethod.invoke();
+    }
+
+    public String getName() {
+      return getNameMethod.invoke();
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+      if (this == obj) {
+        return true;
+      }
+      if (obj instanceof Signal) {
+        return delegate.equals(((Signal)obj).delegate);
+      }
+      return false;
+    }
+
+    @Override
+    public int hashCode() {
+      return delegate.hashCode();
+    }
+
+    @Override
+    public String toString() {
+      return delegate.toString();
+    }
+  }
+
+  @InterfaceAudience.Private
+  public interface Handler {
+    void handle(Signal sig);
+  }
+
+  static class JdkSignalHandlerImpl implements Handler {
+
+    private final Object/* sun.misc.SignalHandler */ delegate;
+    private final DynMethods.BoundMethod jdkSignalHandlerHandleMethod;
+
+    JdkSignalHandlerImpl(Handler handler) {
+      this.delegate = Proxy.newProxyInstance(
+          getClass().getClassLoader(),
+          new Class<?>[] {JDK_SIGNAL_HANDLER_CLAZZ},
+          (proxyObj, method, args) -> {
+            if ("handle".equals(method.getName()) && args.length == 1
+                && JDK_SIGNAL_CLAZZ.isInstance(args[0])) {
+              handler.handle(new Signal(args[0]));
+              return null;
+            } else {
+              Method delegateMethod = handler.getClass().getMethod(
+                  method.getName(), method.getParameterTypes());
+              return delegateMethod.invoke(handler, args);
+            }
+          }
+      );
+      this.jdkSignalHandlerHandleMethod = JDK_SIGNAL_HANDLER_HANDLE_UNBOUND_METHOD.bind(delegate);
+    }
+
+    JdkSignalHandlerImpl(Object delegate) {
+      Preconditions.checkArgument(JDK_SIGNAL_HANDLER_CLAZZ.isInstance(delegate),
+          String.format("Expected class is '%s', but actual class is '%s'",
+              JDK_SIGNAL_HANDLER_CLAZZ.getName(), delegate.getClass().getName()));
+      this.delegate = delegate;
+      this.jdkSignalHandlerHandleMethod = JDK_SIGNAL_HANDLER_HANDLE_UNBOUND_METHOD.bind(delegate);
+    }
+
+    @Override
+    public void handle(Signal sig) {
+      jdkSignalHandlerHandleMethod.invoke(sig.delegate);
+    }
+  }
+
+  public static Handler handle(Signal sig, Handler handler) {
+    Object preHandle = JDK_SIGNAL_HANDLE_STATIC_METHOD.invoke(
+        sig.delegate, new JdkSignalHandlerImpl(handler).delegate);
+    return new JdkSignalHandlerImpl(preHandle);
+  }
+
+  public static void raise(Signal sig) throws IllegalArgumentException {
+    JDK_SIGNAL_RAISE_STATIC_METHOD.invoke(sig.delegate);
+  }
+}

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java

@@ -384,6 +384,36 @@ public void testDeprecatedPropertyInXMLFileGeneratesLogMessage(@TempDir java.nio
     assertTrue(hasDeprecationMessage);
   }
 
+  @Test
+  public void testDeprecatedPropertyLogsWarningOnEveryUse(){
+    String oldProp = "test.deprecation.old.conf.b";
+    String newProp = "test.deprecation.new.conf.b";
+    Configuration.addDeprecation(oldProp, newProp);
+
+    TestAppender appender = new TestAppender();
+    Logger deprecationLogger = Logger.getLogger("org.apache.hadoop.conf.Configuration.deprecation");
+    deprecationLogger.addAppender(appender);
+
+    try {
+      conf.set(oldProp, "b1");
+      conf.get(oldProp);
+      conf.set(oldProp, "b2");
+      conf.get(oldProp);
+      // Using the new property should not log a warning
+      conf.set(newProp, "b3");
+      conf.get(newProp);
+      conf.set(newProp, "b4");
+      conf.get(newProp);
+    } finally {
+      deprecationLogger.removeAppender(appender);
+    }
+
+    Pattern deprecationMsgPattern = Pattern.compile(oldProp + " is deprecated");
+    long count = appender.log.stream().map(LoggingEvent::getRenderedMessage)
+            .filter(msg -> deprecationMsgPattern.matcher(msg).find()).count();
+    assertEquals(4, count, "Expected exactly four warnings for deprecated property usage");
+  }
+
   /**
    * A simple appender for white box testing.
    */