diff --git a/docs/monitoring/metrics.md b/docs/monitoring/metrics.md index 7963e1a015468aaab10cd03e3b0662232f280a5f..e4b5161010f7fe0160a504e5f18315cce8e72540 100644 --- a/docs/monitoring/metrics.md +++ b/docs/monitoring/metrics.md @@ -424,14 +424,17 @@ class MyMapper extends RichMapFunction[Long,Long] { ## Scope -Every metric is assigned an identifier under which it will be reported that is based on 3 components: the user-provided name when registering the metric, an optional user-defined scope and a system-provided scope. +Every metric is assigned an identifier and a set of key-value pairs under which the metric will be reported. + +THe identifier is based on 3 components: the user-defined name when registering the metric, an optional user-defined scope and a system-provided scope. For example, if `A.B` is the system scope, `C.D` the user scope and `E` the name, then the identifier for the metric will be `A.B.C.D.E`. You can configure which delimiter to use for the identifier (default: `.`) by setting the `metrics.scope.delimiter` key in `conf/flink-conf.yaml`. ### User Scope -You can define a user scope by calling either `MetricGroup#addGroup(String name)` or `MetricGroup#addGroup(int name)`. +You can define a user scope by calling `MetricGroup#addGroup(String name)`, `MetricGroup#addGroup(int name)` or `Metric#addGroup(String key, String value)`. +These methods affect what `MetricGroup#getMetricIdentifier` and `MetricGroup#getScopeComponents` return.
The only difference between calling this method and {@code group.addGroup(key).addGroup(value)} is that
+ * {@link #getAllVariables()} of the value group return an additional {@code " implements MetricGroup {
return parentMetricGroup.addGroup(name);
}
+ @Override
+ public final MetricGroup addGroup(String key, String value) {
+ return parentMetricGroup.addGroup(key, value);
+ }
+
@Override
public String[] getScopeComponents() {
return parentMetricGroup.getScopeComponents();
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointStatsTrackerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointStatsTrackerTest.java
index a47b497b4859f13886b86bc5736fbc741493ac13..e7a1d7b20e9ae49e71be7fe176c6ccfc93960762 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointStatsTrackerTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/CheckpointStatsTrackerTest.java
@@ -384,6 +384,11 @@ public class CheckpointStatsTrackerTest {
throw new UnsupportedOperationException("Not expected in this test");
}
+ @Override
+ public MetricGroup addGroup(String key, String value) {
+ throw new UnsupportedOperationException("Not expected in this test");
+ }
+
@Override
public String[] getScopeComponents() {
throw new UnsupportedOperationException("Not expected in this test");
diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/groups/MetricGroupTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/groups/MetricGroupTest.java
index 94760e614cb8ab33013efb4e69bc779aaf979515..0fced3316f96d714877c01dc8c425b98e3b3c1bb 100644
--- a/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/groups/MetricGroupTest.java
+++ b/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/groups/MetricGroupTest.java
@@ -27,7 +27,9 @@ import org.apache.flink.runtime.jobgraph.JobVertexID;
import org.apache.flink.runtime.metrics.MetricRegistry;
import org.apache.flink.runtime.metrics.MetricRegistryConfiguration;
import org.apache.flink.runtime.metrics.MetricRegistryImpl;
+import org.apache.flink.runtime.metrics.NoOpMetricRegistry;
import org.apache.flink.runtime.metrics.dump.QueryScopeInfo;
+import org.apache.flink.runtime.metrics.scope.ScopeFormat;
import org.apache.flink.runtime.metrics.util.DummyCharacterFilter;
import org.apache.flink.util.AbstractID;
import org.apache.flink.util.TestLogger;
@@ -39,6 +41,7 @@ import org.junit.Test;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -78,6 +81,141 @@ public class MetricGroupTest extends TestLogger {
assertTrue(subgroup1 == subgroup2);
}
+ /**
+ * Verifies the basic behavior when defining user-defined variables.
+ */
+ @Test
+ public void testUserDefinedVariable() {
+ MetricRegistry registry = new NoOpMetricRegistry();
+ GenericMetricGroup root = new GenericMetricGroup(registry, new DummyAbstractMetricGroup(registry), "root");
+
+ String key = "key";
+ String value = "value";
+ MetricGroup group = root.addGroup(key, value);
+
+ String variableValue = group.getAllVariables().get(ScopeFormat.asVariable("key"));
+ assertEquals(value, variableValue);
+
+ String identifier = group.getMetricIdentifier("metric");
+ assertTrue("Key is missing from metric identifier.", identifier.contains("key"));
+ assertTrue("Value is missing from metric identifier.", identifier.contains("value"));
+
+ String logicalScope = ((AbstractMetricGroup) group).getLogicalScope(new DummyCharacterFilter());
+ assertTrue("Key is missing from logical scope.", logicalScope.contains(key));
+ assertFalse("Value is present in logical scope.", logicalScope.contains(value));
+ }
+
+ /**
+ * Verifies that calling {@link MetricGroup#addGroup(String, String)} on a {@link GenericKeyMetricGroup} goes
+ * through the generic code path.
+ */
+ @Test
+ public void testUserDefinedVariableOnKeyGroup() {
+ MetricRegistry registry = new NoOpMetricRegistry();
+ GenericMetricGroup root = new GenericMetricGroup(registry, new DummyAbstractMetricGroup(registry), "root");
+
+ String key1 = "key1";
+ String value1 = "value1";
+ root.addGroup(key1, value1);
+
+ String key2 = "key2";
+ String value2 = "value2";
+ MetricGroup group = root.addGroup(key1).addGroup(key2, value2);
+
+ String variableValue = group.getAllVariables().get("value2");
+ assertNull(variableValue);
+
+ String identifier = group.getMetricIdentifier("metric");
+ assertTrue("Key1 is missing from metric identifier.", identifier.contains("key1"));
+ assertTrue("Key2 is missing from metric identifier.", identifier.contains("key2"));
+ assertTrue("Value2 is missing from metric identifier.", identifier.contains("value2"));
+
+ String logicalScope = ((AbstractMetricGroup) group).getLogicalScope(new DummyCharacterFilter());
+ assertTrue("Key1 is missing from logical scope.", logicalScope.contains(key1));
+ assertTrue("Key2 is missing from logical scope.", logicalScope.contains(key2));
+ assertTrue("Value2 is missing from logical scope.", logicalScope.contains(value2));
+ }
+
+ /**
+ * Verifies that calling {@link MetricGroup#addGroup(String, String)} if a generic group with the key name already
+ * exists goes through the generic code path.
+ */
+ @Test
+ public void testNameCollisionForKeyAfterGenericGroup() {
+ MetricRegistry registry = new NoOpMetricRegistry();
+ GenericMetricGroup root = new GenericMetricGroup(registry, new DummyAbstractMetricGroup(registry), "root");
+
+ String key = "key";
+ String value = "value";
+
+ root.addGroup(key);
+ MetricGroup group = root.addGroup(key, value);
+
+ String variableValue = group.getAllVariables().get(ScopeFormat.asVariable("key"));
+ assertNull(variableValue);
+
+ String identifier = group.getMetricIdentifier("metric");
+ assertTrue("Key is missing from metric identifier.", identifier.contains("key"));
+ assertTrue("Value is missing from metric identifier.", identifier.contains("value"));
+
+ String logicalScope = ((AbstractMetricGroup) group).getLogicalScope(new DummyCharacterFilter());
+ assertTrue("Key is missing from logical scope.", logicalScope.contains(key));
+ assertTrue("Value is missing from logical scope.", logicalScope.contains(value));
+ }
+
+ /**
+ * Verifies that calling {@link MetricGroup#addGroup(String, String)} if a generic group with the key and value name
+ * already exists goes through the generic code path.
+ */
+ @Test
+ public void testNameCollisionForKeyAndValueAfterGenericGroup() {
+ MetricRegistry registry = new NoOpMetricRegistry();
+ GenericMetricGroup root = new GenericMetricGroup(registry, new DummyAbstractMetricGroup(registry), "root");
+
+ String key = "key";
+ String value = "value";
+
+ root.addGroup(key).addGroup(value);
+ MetricGroup group = root.addGroup(key, value);
+
+ String variableValue = group.getAllVariables().get(ScopeFormat.asVariable("key"));
+ assertNull(variableValue);
+
+ String identifier = group.getMetricIdentifier("metric");
+ assertTrue("Key is missing from metric identifier.", identifier.contains("key"));
+ assertTrue("Value is missing from metric identifier.", identifier.contains("value"));
+
+ String logicalScope = ((AbstractMetricGroup) group).getLogicalScope(new DummyCharacterFilter());
+ assertTrue("Key is missing from logical scope.", logicalScope.contains(key));
+ assertTrue("Value is missing from logical scope.", logicalScope.contains(value));
+ }
+
+ /**
+ * Verifies that existing key/value groups are returned when calling {@link MetricGroup#addGroup(String)}.
+ */
+ @Test
+ public void testNameCollisionAfterKeyValueGroup() {
+ MetricRegistry registry = new NoOpMetricRegistry();
+ GenericMetricGroup root = new GenericMetricGroup(registry, new DummyAbstractMetricGroup(registry), "root");
+
+ String key = "key";
+ String value = "value";
+
+ root.addGroup(key, value);
+ MetricGroup group = root.addGroup(key).addGroup(value);
+
+ String variableValue = group.getAllVariables().get(ScopeFormat.asVariable("key"));
+ assertEquals(value, variableValue);
+
+ String identifier = group.getMetricIdentifier("metric");
+ assertTrue("Key is missing from metric identifier.", identifier.contains("key"));
+ assertTrue("Value is missing from metric identifier.", identifier.contains("value"));
+
+ String logicalScope = ((AbstractMetricGroup) group).getLogicalScope(new DummyCharacterFilter());
+ assertTrue("Key is missing from logical scope.", logicalScope.contains(key));
+ assertFalse("Value is present in logical scope.", logicalScope.contains(value));
+ }
+
@Test
public void closedGroupDoesNotRegisterMetrics() {
GenericMetricGroup group = new GenericMetricGroup(