Skip to content

Commit 667292d

Browse files
slfan1989slfan1989
authored
Merge branch 'apache:trunk' into YARN-11424-V2
2 parents 420460b + 74ddf69 commit 667292d

File tree

9 files changed

+411
-73
lines changed

9 files changed

+411
-73
lines changed

‎hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MetricsRegistry.java‎

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -227,6 +227,29 @@ public synchronized MutableQuantiles newQuantiles(String name, String desc,
227227
return ret;
228228
}
229229

230+
/**
231+
* Create a mutable inverse metric that estimates inverse quantiles of a stream of values
232+
* @param name of the metric
233+
* @param desc metric description
234+
* @param sampleName of the metric (e.g., "Ops")
235+
* @param valueName of the metric (e.g., "Rate")
236+
* @param interval rollover interval of estimator in seconds
237+
* @return a new inverse quantile estimator object
238+
* @throws MetricsException if interval is not a positive integer
239+
*/
240+
public synchronized MutableQuantiles newInverseQuantiles(String name, String desc,
241+
String sampleName, String valueName, int interval) {
242+
checkMetricName(name);
243+
if (interval <= 0) {
244+
throw new MetricsException("Interval should be positive. Value passed" +
245+
" is: " + interval);
246+
}
247+
MutableQuantiles ret =
248+
new MutableInverseQuantiles(name, desc, sampleName, valueName, interval);
249+
metricsMap.put(name, ret);
250+
return ret;
251+
}
252+
230253
/**
231254
* Create a mutable metric with stats
232255
* @param name of the metric
@@ -278,7 +301,7 @@ public MutableRate newRate(String name, String description) {
278301
}
279302

280303
/**
281-
* Create a mutable rate metric (for throughput measurement)
304+
* Create a mutable rate metric (for throughput measurement).
282305
* @param name of the metric
283306
* @param desc description
284307
* @param extended produce extended stat (stdev/min/max etc.) if true

‎hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableInverseQuantiles.java‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
/**
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
package org.apache.hadoop.metrics2.lib;
19+
20+
import org.apache.hadoop.classification.InterfaceAudience;
21+
import org.apache.hadoop.classification.InterfaceStability;
22+
import org.apache.hadoop.classification.VisibleForTesting;
23+
import org.apache.hadoop.metrics2.util.Quantile;
24+
import org.apache.hadoop.metrics2.util.SampleQuantiles;
25+
import java.text.DecimalFormat;
26+
import static org.apache.hadoop.metrics2.lib.Interns.info;
27+
28+
/**
29+
* Watches a stream of long values, maintaining online estimates of specific
30+
* quantiles with provably low error bounds. Inverse quantiles are meant for
31+
* highly accurate low-percentile (e.g. 1st, 5th) metrics.
32+
* InverseQuantiles are used for metrics where higher the value better it is.
33+
* ( eg: data transfer rate ).
34+
* The 1st percentile here corresponds to the 99th inverse percentile metric,
35+
* 5th percentile to 95th and so on.
36+
*/
37+
@InterfaceAudience.Public
38+
@InterfaceStability.Evolving
39+
public class MutableInverseQuantiles extends MutableQuantiles{
40+
41+
static class InversePercentile extends Quantile {
42+
InversePercentile(double inversePercentile) {
43+
super(inversePercentile/100, inversePercentile/1000);
44+
}
45+
}
46+
47+
@VisibleForTesting
48+
public static final Quantile[] INVERSE_QUANTILES = {new InversePercentile(50),
49+
new InversePercentile(25), new InversePercentile(10),
50+
new InversePercentile(5), new InversePercentile(1)};
51+
52+
/**
53+
* Instantiates a new {@link MutableInverseQuantiles} for a metric that rolls itself
54+
* over on the specified time interval.
55+
*
56+
* @param name of the metric
57+
* @param description long-form textual description of the metric
58+
* @param sampleName type of items in the stream (e.g., "Ops")
59+
* @param valueName type of the values
60+
* @param intervalSecs rollover interval (in seconds) of the estimator
61+
*/
62+
public MutableInverseQuantiles(String name, String description, String sampleName,
63+
String valueName, int intervalSecs) {
64+
super(name, description, sampleName, valueName, intervalSecs);
65+
}
66+
67+
/**
68+
* Sets quantileInfo and estimator.
69+
*
70+
* @param ucName capitalized name of the metric
71+
* @param uvName capitalized type of the values
72+
* @param desc uncapitalized long-form textual description of the metric
73+
* @param lvName uncapitalized type of the values
74+
* @param df Number formatter for inverse percentile value
75+
*/
76+
void setQuantiles(String ucName, String uvName, String desc, String lvName, DecimalFormat df) {
77+
// Construct the MetricsInfos for inverse quantiles, converting to inverse percentiles
78+
setQuantileInfos(INVERSE_QUANTILES.length);
79+
for (int i = 0; i < INVERSE_QUANTILES.length; i++) {
80+
double inversePercentile = 100 * (1 - INVERSE_QUANTILES[i].quantile);
81+
String nameTemplate = ucName + df.format(inversePercentile) + "thInversePercentile" + uvName;
82+
String descTemplate = df.format(inversePercentile) + " inverse percentile " + lvName
83+
+ " with " + getInterval() + " second interval for " + desc;
84+
addQuantileInfo(i, info(nameTemplate, descTemplate));
85+
}
86+
87+
setEstimator(new SampleQuantiles(INVERSE_QUANTILES));
88+
}
89+
}

‎hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableQuantiles.java‎

Lines changed: 76 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020

2121
import static org.apache.hadoop.metrics2.lib.Interns.info;
2222

23+
import java.text.DecimalFormat;
2324
import java.util.Map;
2425
import java.util.concurrent.Executors;
2526
import java.util.concurrent.ScheduledExecutorService;
@@ -52,9 +53,10 @@ public class MutableQuantiles extends MutableMetric {
5253
new Quantile(0.75, 0.025), new Quantile(0.90, 0.010),
5354
new Quantile(0.95, 0.005), new Quantile(0.99, 0.001) };
5455

55-
private final MetricsInfo numInfo;
56-
private final MetricsInfo[] quantileInfos;
57-
private final int interval;
56+
private MetricsInfo numInfo;
57+
private MetricsInfo[] quantileInfos;
58+
private int intervalSecs;
59+
private static DecimalFormat decimalFormat = new DecimalFormat("###.####");
5860

5961
private QuantileEstimator estimator;
6062
private long previousCount = 0;
@@ -91,26 +93,39 @@ public MutableQuantiles(String name, String description, String sampleName,
9193
String lsName = StringUtils.uncapitalize(sampleName);
9294
String lvName = StringUtils.uncapitalize(valueName);
9395

94-
numInfo = info(ucName + "Num" + usName, String.format(
95-
"Number of %s for %s with %ds interval", lsName, desc, interval));
96+
setInterval(interval);
97+
setNumInfo(info(ucName + "Num" + usName, String.format(
98+
"Number of %s for %s with %ds interval", lsName, desc, interval)));
99+
scheduledTask = scheduler.scheduleWithFixedDelay(new RolloverSample(this),
100+
interval, interval, TimeUnit.SECONDS);
101+
setQuantiles(ucName, uvName, desc, lvName, decimalFormat);
102+
}
103+
104+
/**
105+
* Sets quantileInfo and estimator.
106+
*
107+
* @param ucName capitalized name of the metric
108+
* @param uvName capitalized type of the values
109+
* @param desc uncapitalized long-form textual description of the metric
110+
* @param lvName uncapitalized type of the values
111+
* @param pDecimalFormat Number formatter for percentile value
112+
*/
113+
void setQuantiles(String ucName, String uvName, String desc, String lvName, DecimalFormat pDecimalFormat) {
96114
// Construct the MetricsInfos for the quantiles, converting to percentiles
97-
quantileInfos = new MetricsInfo[quantiles.length];
98-
String nameTemplate = ucName + "%dthPercentile" + uvName;
99-
String descTemplate = "%d percentile " + lvName + " with " + interval
100-
+ " second interval for " + desc;
115+
setQuantileInfos(quantiles.length);
101116
for (int i = 0; i < quantiles.length; i++) {
102-
int percentile = (int) (100 * quantiles[i].quantile);
103-
quantileInfos[i] = info(String.format(nameTemplate, percentile),
104-
String.format(descTemplate, percentile));
117+
double percentile = 100 * quantiles[i].quantile;
118+
String nameTemplate = ucName + pDecimalFormat.format(percentile) + "thPercentile" + uvName;
119+
String descTemplate = pDecimalFormat.format(percentile) + " percentile " + lvName
120+
+ " with " + getInterval() + " second interval for " + desc;
121+
addQuantileInfo(i, info(nameTemplate, descTemplate));
105122
}
106123

107-
estimator = new SampleQuantiles(quantiles);
108-
109-
this.interval = interval;
110-
scheduledTask = scheduler.scheduleWithFixedDelay(new RolloverSample(this),
111-
interval, interval, TimeUnit.SECONDS);
124+
setEstimator(new SampleQuantiles(quantiles));
112125
}
113126

127+
public MutableQuantiles() {}
128+
114129
@Override
115130
public synchronized void snapshot(MetricsRecordBuilder builder, boolean all) {
116131
if (all || changed()) {
@@ -133,8 +148,50 @@ public synchronized void add(long value) {
133148
estimator.insert(value);
134149
}
135150

136-
public int getInterval() {
137-
return interval;
151+
/**
152+
* Set info about the metrics.
153+
*
154+
* @param pNumInfo info about the metrics.
155+
*/
156+
public synchronized void setNumInfo(MetricsInfo pNumInfo) {
157+
this.numInfo = pNumInfo;
158+
}
159+
160+
/**
161+
* Initialize quantileInfos array.
162+
*
163+
* @param length of the quantileInfos array.
164+
*/
165+
public synchronized void setQuantileInfos(int length) {
166+
this.quantileInfos = new MetricsInfo[length];
167+
}
168+
169+
/**
170+
* Add entry to quantileInfos array.
171+
*
172+
* @param i array index.
173+
* @param info info to be added to quantileInfos array.
174+
*/
175+
public synchronized void addQuantileInfo(int i, MetricsInfo info) {
176+
this.quantileInfos[i] = info;
177+
}
178+
179+
/**
180+
* Set the rollover interval (in seconds) of the estimator.
181+
*
182+
* @param pIntervalSecs of the estimator.
183+
*/
184+
public synchronized void setInterval(int pIntervalSecs) {
185+
this.intervalSecs = pIntervalSecs;
186+
}
187+
188+
/**
189+
* Get the rollover interval (in seconds) of the estimator.
190+
*
191+
* @return intervalSecs of the estimator.
192+
*/
193+
public synchronized int getInterval() {
194+
return intervalSecs;
138195
}
139196

140197
public void stop() {

‎hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/util/TestSampleQuantiles.java‎

Lines changed: 56 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
import java.util.Map;
2525
import java.util.Random;
2626

27+
import org.apache.hadoop.metrics2.lib.MutableInverseQuantiles;
2728
import org.junit.Before;
2829
import org.junit.Test;
2930

@@ -36,6 +37,7 @@ public class TestSampleQuantiles {
3637
new Quantile(0.95, 0.005), new Quantile(0.99, 0.001) };
3738

3839
SampleQuantiles estimator;
40+
final static int NUM_REPEATS = 10;
3941

4042
@Before
4143
public void init() {
@@ -91,28 +93,70 @@ public void testClear() throws IOException {
9193
@Test
9294
public void testQuantileError() throws IOException {
9395
final int count = 100000;
94-
Random r = new Random(0xDEADDEAD);
95-
Long[] values = new Long[count];
96+
Random rnd = new Random(0xDEADDEAD);
97+
int[] values = new int[count];
9698
for (int i = 0; i < count; i++) {
97-
values[i] = (long) (i + 1);
99+
values[i] = i + 1;
98100
}
99-
// Do 10 shuffle/insert/check cycles
100-
for (int i = 0; i < 10; i++) {
101-
System.out.println("Starting run " + i);
102-
Collections.shuffle(Arrays.asList(values), r);
101+
102+
// Repeat shuffle/insert/check cycles 10 times
103+
for (int i = 0; i < NUM_REPEATS; i++) {
104+
105+
// Shuffle
106+
Collections.shuffle(Arrays.asList(values), rnd);
103107
estimator.clear();
104-
for (int j = 0; j < count; j++) {
105-
estimator.insert(values[j]);
108+
109+
// Insert
110+
for (int value : values) {
111+
estimator.insert(value);
106112
}
107113
Map<Quantile, Long> snapshot;
108114
snapshot = estimator.snapshot();
115+
116+
// Check
109117
for (Quantile q : quantiles) {
110118
long actual = (long) (q.quantile * count);
111119
long error = (long) (q.error * count);
112120
long estimate = snapshot.get(q);
113-
System.out
114-
.println(String.format("Expected %d with error %d, estimated %d",
115-
actual, error, estimate));
121+
assertThat(estimate <= actual + error).isTrue();
122+
assertThat(estimate >= actual - error).isTrue();
123+
}
124+
}
125+
}
126+
127+
/**
128+
* Correctness test that checks that absolute error of the estimate for inverse quantiles
129+
* is within specified error bounds for some randomly permuted streams of items.
130+
*/
131+
@Test
132+
public void testInverseQuantiles() throws IOException {
133+
SampleQuantiles inverseQuantilesEstimator =
134+
new SampleQuantiles(MutableInverseQuantiles.INVERSE_QUANTILES);
135+
final int count = 100000;
136+
Random rnd = new Random(0xDEADDEAD);
137+
int[] values = new int[count];
138+
for (int i = 0; i < count; i++) {
139+
values[i] = i + 1;
140+
}
141+
142+
// Repeat shuffle/insert/check cycles 10 times
143+
for (int i = 0; i < NUM_REPEATS; i++) {
144+
// Shuffle
145+
Collections.shuffle(Arrays.asList(values), rnd);
146+
inverseQuantilesEstimator.clear();
147+
148+
// Insert
149+
for (int value : values) {
150+
inverseQuantilesEstimator.insert(value);
151+
}
152+
Map<Quantile, Long> snapshot;
153+
snapshot = inverseQuantilesEstimator.snapshot();
154+
155+
// Check
156+
for (Quantile q : MutableInverseQuantiles.INVERSE_QUANTILES) {
157+
long actual = (long) (q.quantile * count);
158+
long error = (long) (q.error * count);
159+
long estimate = snapshot.get(q);
116160
assertThat(estimate <= actual + error).isTrue();
117161
assertThat(estimate >= actual - error).isTrue();
118162
}

‎hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/MetricsAsserts.java‎

Lines changed: 23 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -392,13 +392,34 @@ public static void assertQuantileGauges(String prefix,
392392
*/
393393
public static void assertQuantileGauges(String prefix,
394394
MetricsRecordBuilder rb, String valueName) {
395-
verify(rb).addGauge(eqName(info(prefix + "NumOps", "")), geq(0l));
395+
verify(rb).addGauge(eqName(info(prefix + "NumOps", "")), geq(0L));
396396
for (Quantile q : MutableQuantiles.quantiles) {
397397
String nameTemplate = prefix + "%dthPercentile" + valueName;
398398
int percentile = (int) (100 * q.quantile);
399399
verify(rb).addGauge(
400400
eqName(info(String.format(nameTemplate, percentile), "")),
401-
geq(0l));
401+
geq(0L));
402+
}
403+
}
404+
405+
/**
406+
* Asserts that the NumOps and inverse quantiles for a metric have been changed at
407+
* some point to a non-zero value, for the specified value name of the
408+
* metrics (e.g., "Rate").
409+
*
410+
* @param prefix of the metric
411+
* @param rb MetricsRecordBuilder with the metric
412+
* @param valueName the value name for the metric
413+
*/
414+
public static void assertInverseQuantileGauges(String prefix,
415+
MetricsRecordBuilder rb, String valueName) {
416+
verify(rb).addGauge(eqName(info(prefix + "NumOps", "")), geq(0L));
417+
for (Quantile q : MutableQuantiles.quantiles) {
418+
String nameTemplate = prefix + "%dthInversePercentile" + valueName;
419+
int percentile = (int) (100 * q.quantile);
420+
verify(rb).addGauge(
421+
eqName(info(String.format(nameTemplate, percentile), "")),
422+
geq(0L));
402423
}
403424
}
404425

0 commit comments

Comments
 (0)