HADOOP-16830 + HADOOP-17271: S3A instrumentation

* delete duration now tracked
* and split into single/bulk delete calls
* delete/rename cost tests updated to match
* ...and to work correctly if bulk deletes are disabled.
* +moved most of the Statistic names into the hadoop-common constant files,
  for reuse.
* New method IOStatisticsLogging.ioStatisticsToSortedString(); sorts the
  metrics before printing. Adds cost of creation of and insertion into
  treemaps, so should be used with care. Used for test reporting.

Change-Id: I2c7378e504aa364a39ea7feb50c4c6b538bb1d8f

Author: steveloughran

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/FunctionsRaisingIOE.java

@@ -25,7 +25,7 @@
 
 /**
  * Support for functional programming/lambda-expressions.
- * @deprecated use {@code org.apache.hadoop.util.functional} classes
+ * @deprecated use {@code org.apache.hadoop.util.functional}
  */
 @InterfaceAudience.Private
 @InterfaceStability.Unstable

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/IOStatisticsLogging.java

@@ -20,6 +20,7 @@
 
 import javax.annotation.Nullable;
 import java.util.Map;
+import java.util.TreeMap;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -72,11 +73,35 @@ public static String ioStatisticsToString(
       @Nullable final IOStatistics statistics) {
     if (statistics != null) {
       StringBuilder sb = new StringBuilder();
-      mapToString(sb, "counters", statistics.counters());
-      mapToString(sb, "gauges", statistics.gauges());
-      mapToString(sb, "minimums", statistics.minimums());
-      mapToString(sb, "maximums", statistics.maximums());
-      mapToString(sb, "means", statistics.meanStatistics());
+      mapToString(sb, "counters", statistics.counters(), " ");
+      mapToString(sb, "gauges", statistics.gauges(), " ");
+      mapToString(sb, "minimums", statistics.minimums(), " ");
+      mapToString(sb, "maximums", statistics.maximums(), " ");
+      mapToString(sb, "means", statistics.meanStatistics(), " ");
+
+      return sb.toString();
+    } else {
+      return "";
+    }
+  }
+
+  /**
+   * Convert IOStatistics to a string form, with all the metrics sorted.
+   * This is more expensive than the simple conversion, so should only
+   * be used for logging/output where it's known/highly likely that the
+   * caller wants to see the values. Not for debug logging.
+   * @param statistics A statistics instance.
+   * @return string value or the empty string if null
+   */
+  public static String ioStatisticsToSortedString(
+      @Nullable final IOStatistics statistics) {
+    if (statistics != null) {
+      StringBuilder sb = new StringBuilder();
+      mapToSortedString(sb, "counters", statistics.counters());
+      mapToSortedString(sb, "\ngauges", statistics.gauges());
+      mapToSortedString(sb, "\nminimums", statistics.minimums());
+      mapToSortedString(sb, "\nmaximums", statistics.maximums());
+      mapToSortedString(sb, "\nmeans", statistics.meanStatistics());
 
       return sb.toString();
     } else {
@@ -86,26 +111,59 @@ public static String ioStatisticsToString(
 
   /**
    * Given a map, add its entryset to the string.
+   * The entries are only sorted if the source entryset
+   * iterator is sorted, such as from a TreeMap.
    * @param sb string buffer to append to
    * @param type type (for output)
    * @param map map to evaluate
+   * @param separator separator
    * @param <E> type of values of the map
    */
   private static <E> void mapToString(StringBuilder sb,
       final String type,
-      final Map<String, E> map) {
+      final Map<String, E> map,
+      final String separator) {
     int count = 0;
     sb.append(type);
     sb.append("=(");
     for (Map.Entry<String, E> entry : map.entrySet()) {
       if (count > 0) {
-        sb.append(' ');
+        sb.append(separator);
       }
       count++;
       sb.append(IOStatisticsBinding.entryToString(
           entry.getKey(), entry.getValue()));
     }
-    sb.append("); ");
+    sb.append(");\n");
+  }
+
+  /**
+   * Given a map, produce a string with all the values, sorted.
+   * Needs to create a treemap and insert all the entries.
+   * @param sb string buffer to append to
+   * @param type type (for output)
+   * @param map map to evaluate
+   * @param <E> type of values of the map
+   */
+  private static <E> void mapToSortedString(StringBuilder sb,
+      final String type,
+      final Map<String, E> map) {
+    mapToString(sb, type, sortedMap(map), "\n");
+  }
+
+  /**
+   * Create a sorted (tree) map from an unsorted map.
+   * This incurs the cost of creating a map and that
+   * of inserting every object into the tree.
+   * @param source source map
+   * @param <E> value type
+   * @return a treemap with all the entries.
+   */
+  private static <E> Map<String, E> sortedMap(
+      final Map<String, E> source) {
+    Map<String, E> tm = new TreeMap<>();
+    tm.putAll(source);
+    return tm;
   }
 
   /**

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/MeanStatistic.java

@@ -278,8 +278,8 @@ public MeanStatistic copy() {
 
   @Override
   public String toString() {
-    return String.format("(sum=%d, samples=%d, mean=%.4f)",
-        sum, samples, mean());
+    return String.format("(samples=%d, sum=%d, mean=%.4f)",
+        samples, sum, mean());
   }
 
 }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/StoreStatisticNames.java

@@ -22,10 +22,11 @@
 import org.apache.hadoop.classification.InterfaceStability;
 
 /**
- * These are common statistic names.
+ * Common statistic names for object store operations..
  * <p>
  * When adding new common statistic name constants, please make them unique.
- * By convention, they are implicitly unique:
+ * By convention:
+ * </p>
  * <ul>
  *   <li>the name of the constants are uppercase, words separated by
  *   underscores.</li>
@@ -160,13 +161,28 @@ public final class StoreStatisticNames {
   public static final String OBJECT_CONTINUE_LIST_REQUEST =
       "object_continue_list_request";
 
+  /**
+   * A bulk DELETE request was made: {@value}.
+   * A separate statistic from {@link #OBJECT_DELETE_REQUEST}
+   * so that metrics on duration of the operations can
+   * be distinguished.
+   */
+  public static final String OBJECT_BULK_DELETE_REQUEST
+      = "object_bulk_delete_request";
+
   /**
    * A store's equivalent of a DELETE request was made: {@value}.
    * This may be an HTTP DELETE verb, or it may be some custom
    * operation which takes a list of objects to delete.
    */
-  public static final String OP_HTTP_DELETE_REQUEST
-      = "op_http_delete_request";
+  public static final String OBJECT_DELETE_REQUEST
+      = "object_delete_request";
+
+  /**
+   * The count of objects deleted in delete requests
+   */
+  public static final String OBJECT_DELETE_OBJECTS
+      = "object_delete_objects";
 
   /**
    * Object multipart upload initiated.
@@ -285,6 +301,36 @@ public final class StoreStatisticNames {
   public static final String OBJECT_METADATA_REQUESTS
       = "object_metadata_request";
 
+  public static final String OBJECT_COPY_REQUESTS
+      = "object_copy_requests";
+
+  public static final String STORE_IO_THROTTLE_RATE
+      = "store_io_throttle_rate";
+
+  public static final String DELEGATION_TOKEN_ISSUED
+      = "delegation_token_issued";
+
+  public static final String MULTIPART_UPLOAD_INSTANTIATED
+      = "multipart_instantiated";
+
+  public static final String MULTIPART_UPLOAD_PART_PUT
+      = "multipart_upload_part_put";
+
+  public static final String MULTIPART_UPLOAD_PART_PUT_BYTES
+      = "multipart_upload_part_put_bytes";
+
+  public static final String MULTIPART_UPLOAD_ABORTED
+      = "multipart_upload_aborted";
+
+  public static final String MULTIPART_UPLOAD_ABORT_UNDER_PATH_INVOKED
+      = "multipart_upload_abort_under_path_invoked";
+
+  public static final String MULTIPART_UPLOAD_COMPLETED
+      = "multipart_upload_completed";
+
+  public static final String MULTIPART_UPLOAD_STARTED
+      = "multipart_upload_started";
+
   private StoreStatisticNames() {
   }
 

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/StreamStatisticNames.java

@@ -283,6 +283,9 @@ public final class StreamStatisticNames {
   public static final String STREAM_WRITE_QUEUE_DURATION
       = "stream_write_queue_duration";
 
+  public static final String STREAM_WRITE_TOTAL_DATA
+      = "stream_write_total_data";
+
   private StreamStatisticNames() {
   }
 

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/IOStatisticsStoreImpl.java

@@ -154,13 +154,16 @@ private void setAtomicLong(final AtomicLong aLong, final long value) {
    * increment an atomic long and return its value;
    * null long is no-op returning 0.
    * @param aLong atomic long; may be null
-   * @param increment amount to increment; -ve for a decrement
+   * param increment amount to increment; negative for a decrement
    * @return final value or 0 if the long is null
    */
   private long incAtomicLong(final AtomicLong aLong,
       final long increment) {
     if (aLong != null) {
-      return aLong.addAndGet(increment);
+      // optimization: zero is a get rather than addAndGet()
+      return increment != 0
+          ? aLong.addAndGet(increment)
+          : aLong.get();
     } else {
       return 0;
     }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/statistics/impl/StatisticDurationTracker.java

@@ -67,7 +67,7 @@ public StatisticDurationTracker(
    * of the key name is updated.
    * @param iostats statistics to update
    * @param key Key to use as prefix of values.
-   * @param count  #of times to increment the matching counter.
+   * @param count #of times to increment the matching counter.
    */
   public StatisticDurationTracker(
       final IOStatisticsStore iostats,
@@ -87,6 +87,10 @@ public void failed() {
 
   /**
    * Set the finished time and then update the statistics.
+   * If the operation failed then the key + .failures counter will be
+   * incremented by one.
+   * The operation min/mean/max values will be updated with the duration;
+   * on a failure these will all be the .failures metrics.
    */
   @Override
   public void close() {

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/InconsistentS3ClientFactory.java

@@ -41,9 +41,9 @@ public class InconsistentS3ClientFactory extends DefaultS3ClientFactory {
    * Logs a warning that this is being done.
    * @param credentials credentials to use
    * @param awsConf  AWS configuration
-   * @param metrics
-   * @param endpoint
-   * @param pathStyleAccess
+   * @param metrics metric collector
+   * @param endpoint AWS endpoint
+   * @param pathStyleAccess should path style access be supported?
    * @return an inconsistent client.
    */
   @Override