From b732151955ed453532f9c5cad43a552318caa76b Mon Sep 17 00:00:00 2001 From: gniadeck <77535280+gniadeck@users.noreply.github.com> Date: Sun, 14 Dec 2025 14:13:45 +0100 Subject: [PATCH 1/5] Add GC pause duration histogram Signed-off-by: gniadeck <77535280+gniadeck@users.noreply.github.com> --- .../jvm/JvmGarbageCollectorMetrics.java | 46 +++++++++++++++++++ .../jvm/JvmGarbageCollectorMetricsTest.java | 4 +- 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/prometheus-metrics-instrumentation-jvm/src/main/java/io/prometheus/metrics/instrumentation/jvm/JvmGarbageCollectorMetrics.java b/prometheus-metrics-instrumentation-jvm/src/main/java/io/prometheus/metrics/instrumentation/jvm/JvmGarbageCollectorMetrics.java index 262e2df5f..8ac2d7c58 100644 --- a/prometheus-metrics-instrumentation-jvm/src/main/java/io/prometheus/metrics/instrumentation/jvm/JvmGarbageCollectorMetrics.java +++ b/prometheus-metrics-instrumentation-jvm/src/main/java/io/prometheus/metrics/instrumentation/jvm/JvmGarbageCollectorMetrics.java @@ -1,6 +1,8 @@ package io.prometheus.metrics.instrumentation.jvm; +import com.sun.management.GarbageCollectionNotificationInfo; import io.prometheus.metrics.config.PrometheusProperties; +import io.prometheus.metrics.core.metrics.Histogram; import io.prometheus.metrics.core.metrics.SummaryWithCallback; import io.prometheus.metrics.model.registry.PrometheusRegistry; import io.prometheus.metrics.model.snapshots.Quantiles; @@ -9,6 +11,8 @@ import java.lang.management.ManagementFactory; import java.util.List; import javax.annotation.Nullable; +import javax.management.NotificationEmitter; +import javax.management.openmbean.CompositeData; /** * JVM Garbage Collector metrics. The {@link JvmGarbageCollectorMetrics} are registered as part of @@ -39,6 +43,7 @@ public class JvmGarbageCollectorMetrics { private static final String JVM_GC_COLLECTION_SECONDS = "jvm_gc_collection_seconds"; + private static final String JVM_GC_DURATION_SECONDS = "jvm_gc_duration_seconds"; private final PrometheusProperties config; private final List garbageCollectorBeans; @@ -67,6 +72,47 @@ private void register(PrometheusRegistry registry) { } }) .register(registry); + + registerGCDurationHistogram(registry); + } + + private void registerGCDurationHistogram(PrometheusRegistry registry) { + double[] buckets = {0.01, 0.1, 1, 10}; + + Histogram gcDurationHistogram = + Histogram.builder(config) + .name(JVM_GC_DURATION_SECONDS) + .help("JVM GC pause duration histogram.") + .unit(Unit.SECONDS) + .labelNames("gc", "action", "cause") + .classicUpperBounds(buckets) + .register(registry); + + for (GarbageCollectorMXBean gcBean : garbageCollectorBeans) { + + if (!(gcBean instanceof NotificationEmitter)) { + continue; + } + + NotificationEmitter notificationEmitter = (NotificationEmitter) gcBean; + + notificationEmitter.addNotificationListener( + (notification, handback) -> { + if (!GarbageCollectionNotificationInfo.GARBAGE_COLLECTION_NOTIFICATION.equals( + notification.getType())) { + return; + } + + GarbageCollectionNotificationInfo info = + GarbageCollectionNotificationInfo.from((CompositeData) notification.getUserData()); + + gcDurationHistogram + .labelValues(info.getGcName(), info.getGcAction(), info.getGcCause()) + .observe(Unit.millisToSeconds(info.getGcInfo().getDuration())); + }, + null, + null); + } } public static Builder builder() { diff --git a/prometheus-metrics-instrumentation-jvm/src/test/java/io/prometheus/metrics/instrumentation/jvm/JvmGarbageCollectorMetricsTest.java b/prometheus-metrics-instrumentation-jvm/src/test/java/io/prometheus/metrics/instrumentation/jvm/JvmGarbageCollectorMetricsTest.java index 177f29d2e..e631574e3 100644 --- a/prometheus-metrics-instrumentation-jvm/src/test/java/io/prometheus/metrics/instrumentation/jvm/JvmGarbageCollectorMetricsTest.java +++ b/prometheus-metrics-instrumentation-jvm/src/test/java/io/prometheus/metrics/instrumentation/jvm/JvmGarbageCollectorMetricsTest.java @@ -58,7 +58,9 @@ public void testGoodCase() throws IOException { @Test public void testIgnoredMetricNotScraped() { MetricNameFilter filter = - MetricNameFilter.builder().nameMustNotBeEqualTo("jvm_gc_collection_seconds").build(); + MetricNameFilter.builder() + .nameMustNotBeEqualTo("jvm_gc_collection_seconds", "jvm_gc_duration_seconds") + .build(); PrometheusRegistry registry = new PrometheusRegistry(); JvmGarbageCollectorMetrics.builder() From 7f5ad1ba91b4c59eea1e276c3d37ff108bbcb78e Mon Sep 17 00:00:00 2001 From: gniadeck <77535280+gniadeck@users.noreply.github.com> Date: Sun, 14 Dec 2025 14:19:40 +0100 Subject: [PATCH 2/5] align gc label name with opentelemetry spec Signed-off-by: gniadeck <77535280+gniadeck@users.noreply.github.com> --- .../metrics/instrumentation/jvm/JvmGarbageCollectorMetrics.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prometheus-metrics-instrumentation-jvm/src/main/java/io/prometheus/metrics/instrumentation/jvm/JvmGarbageCollectorMetrics.java b/prometheus-metrics-instrumentation-jvm/src/main/java/io/prometheus/metrics/instrumentation/jvm/JvmGarbageCollectorMetrics.java index 8ac2d7c58..83bcedad9 100644 --- a/prometheus-metrics-instrumentation-jvm/src/main/java/io/prometheus/metrics/instrumentation/jvm/JvmGarbageCollectorMetrics.java +++ b/prometheus-metrics-instrumentation-jvm/src/main/java/io/prometheus/metrics/instrumentation/jvm/JvmGarbageCollectorMetrics.java @@ -84,7 +84,7 @@ private void registerGCDurationHistogram(PrometheusRegistry registry) { .name(JVM_GC_DURATION_SECONDS) .help("JVM GC pause duration histogram.") .unit(Unit.SECONDS) - .labelNames("gc", "action", "cause") + .labelNames("name", "action", "cause") .classicUpperBounds(buckets) .register(registry); From 42cf5ef553a3a327d1d5b0c56255e5f43457094e Mon Sep 17 00:00:00 2001 From: gniadeck <77535280+gniadeck@users.noreply.github.com> Date: Sun, 14 Dec 2025 14:23:53 +0100 Subject: [PATCH 3/5] simplification, spotless fixes Signed-off-by: gniadeck <77535280+gniadeck@users.noreply.github.com> --- .../jvm/JvmGarbageCollectorMetrics.java | 36 +++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/prometheus-metrics-instrumentation-jvm/src/main/java/io/prometheus/metrics/instrumentation/jvm/JvmGarbageCollectorMetrics.java b/prometheus-metrics-instrumentation-jvm/src/main/java/io/prometheus/metrics/instrumentation/jvm/JvmGarbageCollectorMetrics.java index 83bcedad9..bd58ab862 100644 --- a/prometheus-metrics-instrumentation-jvm/src/main/java/io/prometheus/metrics/instrumentation/jvm/JvmGarbageCollectorMetrics.java +++ b/prometheus-metrics-instrumentation-jvm/src/main/java/io/prometheus/metrics/instrumentation/jvm/JvmGarbageCollectorMetrics.java @@ -94,24 +94,24 @@ private void registerGCDurationHistogram(PrometheusRegistry registry) { continue; } - NotificationEmitter notificationEmitter = (NotificationEmitter) gcBean; - - notificationEmitter.addNotificationListener( - (notification, handback) -> { - if (!GarbageCollectionNotificationInfo.GARBAGE_COLLECTION_NOTIFICATION.equals( - notification.getType())) { - return; - } - - GarbageCollectionNotificationInfo info = - GarbageCollectionNotificationInfo.from((CompositeData) notification.getUserData()); - - gcDurationHistogram - .labelValues(info.getGcName(), info.getGcAction(), info.getGcCause()) - .observe(Unit.millisToSeconds(info.getGcInfo().getDuration())); - }, - null, - null); + ((NotificationEmitter) gcBean) + .addNotificationListener( + (notification, handback) -> { + if (!GarbageCollectionNotificationInfo.GARBAGE_COLLECTION_NOTIFICATION.equals( + notification.getType())) { + return; + } + + GarbageCollectionNotificationInfo info = + GarbageCollectionNotificationInfo.from( + (CompositeData) notification.getUserData()); + + gcDurationHistogram + .labelValues(info.getGcName(), info.getGcAction(), info.getGcCause()) + .observe(Unit.millisToSeconds(info.getGcInfo().getDuration())); + }, + null, + null); } } From 042e03ce28cad1555a8baa2b1d7a51982bf9e7c5 Mon Sep 17 00:00:00 2001 From: gniadeck <77535280+gniadeck@users.noreply.github.com> Date: Tue, 16 Dec 2025 20:56:42 +0100 Subject: [PATCH 4/5] align naming with opentelemetry semantic conventions Signed-off-by: gniadeck <77535280+gniadeck@users.noreply.github.com> --- .../jvm/JvmGarbageCollectorMetrics.java | 11 +++++------ .../jvm/JvmGarbageCollectorMetricsTest.java | 2 +- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/prometheus-metrics-instrumentation-jvm/src/main/java/io/prometheus/metrics/instrumentation/jvm/JvmGarbageCollectorMetrics.java b/prometheus-metrics-instrumentation-jvm/src/main/java/io/prometheus/metrics/instrumentation/jvm/JvmGarbageCollectorMetrics.java index bd58ab862..ab36ad40d 100644 --- a/prometheus-metrics-instrumentation-jvm/src/main/java/io/prometheus/metrics/instrumentation/jvm/JvmGarbageCollectorMetrics.java +++ b/prometheus-metrics-instrumentation-jvm/src/main/java/io/prometheus/metrics/instrumentation/jvm/JvmGarbageCollectorMetrics.java @@ -43,7 +43,7 @@ public class JvmGarbageCollectorMetrics { private static final String JVM_GC_COLLECTION_SECONDS = "jvm_gc_collection_seconds"; - private static final String JVM_GC_DURATION_SECONDS = "jvm_gc_duration_seconds"; + private static final String JVM_GC_DURATION = "jvm_gc_duration"; private final PrometheusProperties config; private final List garbageCollectorBeans; @@ -81,10 +81,9 @@ private void registerGCDurationHistogram(PrometheusRegistry registry) { Histogram gcDurationHistogram = Histogram.builder(config) - .name(JVM_GC_DURATION_SECONDS) - .help("JVM GC pause duration histogram.") - .unit(Unit.SECONDS) - .labelNames("name", "action", "cause") + .name(JVM_GC_DURATION) + .help("Duration of JVM garbage collection actions.") + .labelNames("jvm_gc_action", "jvm_gc_name", "jvm_gc_cause") .classicUpperBounds(buckets) .register(registry); @@ -107,7 +106,7 @@ private void registerGCDurationHistogram(PrometheusRegistry registry) { (CompositeData) notification.getUserData()); gcDurationHistogram - .labelValues(info.getGcName(), info.getGcAction(), info.getGcCause()) + .labelValues(info.getGcAction(), info.getGcName(), info.getGcCause()) .observe(Unit.millisToSeconds(info.getGcInfo().getDuration())); }, null, diff --git a/prometheus-metrics-instrumentation-jvm/src/test/java/io/prometheus/metrics/instrumentation/jvm/JvmGarbageCollectorMetricsTest.java b/prometheus-metrics-instrumentation-jvm/src/test/java/io/prometheus/metrics/instrumentation/jvm/JvmGarbageCollectorMetricsTest.java index e631574e3..8d8564996 100644 --- a/prometheus-metrics-instrumentation-jvm/src/test/java/io/prometheus/metrics/instrumentation/jvm/JvmGarbageCollectorMetricsTest.java +++ b/prometheus-metrics-instrumentation-jvm/src/test/java/io/prometheus/metrics/instrumentation/jvm/JvmGarbageCollectorMetricsTest.java @@ -59,7 +59,7 @@ public void testGoodCase() throws IOException { public void testIgnoredMetricNotScraped() { MetricNameFilter filter = MetricNameFilter.builder() - .nameMustNotBeEqualTo("jvm_gc_collection_seconds", "jvm_gc_duration_seconds") + .nameMustNotBeEqualTo("jvm_gc_collection_seconds", "jvm_gc_duration") .build(); PrometheusRegistry registry = new PrometheusRegistry(); From e5cc0c87e4f7fcf0081a1f4437eff48a4b08ebfd Mon Sep 17 00:00:00 2001 From: gniadeck <77535280+gniadeck@users.noreply.github.com> Date: Sat, 17 Jan 2026 17:16:15 +0100 Subject: [PATCH 5/5] Label test and migration to opentelemetry naming Signed-off-by: gniadeck <77535280+gniadeck@users.noreply.github.com> --- .../jvm/JvmGarbageCollectorMetrics.java | 4 +- .../jvm/JvmGarbageCollectorMetricsTest.java | 136 +++++++++++++++++- 2 files changed, 135 insertions(+), 5 deletions(-) diff --git a/prometheus-metrics-instrumentation-jvm/src/main/java/io/prometheus/metrics/instrumentation/jvm/JvmGarbageCollectorMetrics.java b/prometheus-metrics-instrumentation-jvm/src/main/java/io/prometheus/metrics/instrumentation/jvm/JvmGarbageCollectorMetrics.java index ab36ad40d..fd97b2b4d 100644 --- a/prometheus-metrics-instrumentation-jvm/src/main/java/io/prometheus/metrics/instrumentation/jvm/JvmGarbageCollectorMetrics.java +++ b/prometheus-metrics-instrumentation-jvm/src/main/java/io/prometheus/metrics/instrumentation/jvm/JvmGarbageCollectorMetrics.java @@ -43,7 +43,7 @@ public class JvmGarbageCollectorMetrics { private static final String JVM_GC_COLLECTION_SECONDS = "jvm_gc_collection_seconds"; - private static final String JVM_GC_DURATION = "jvm_gc_duration"; + private static final String JVM_GC_DURATION = "jvm.gc.duration"; private final PrometheusProperties config; private final List garbageCollectorBeans; @@ -83,7 +83,7 @@ private void registerGCDurationHistogram(PrometheusRegistry registry) { Histogram.builder(config) .name(JVM_GC_DURATION) .help("Duration of JVM garbage collection actions.") - .labelNames("jvm_gc_action", "jvm_gc_name", "jvm_gc_cause") + .labelNames("jvm.gc.action", "jvm.gc.name", "jvm.gc.cause") .classicUpperBounds(buckets) .register(registry); diff --git a/prometheus-metrics-instrumentation-jvm/src/test/java/io/prometheus/metrics/instrumentation/jvm/JvmGarbageCollectorMetricsTest.java b/prometheus-metrics-instrumentation-jvm/src/test/java/io/prometheus/metrics/instrumentation/jvm/JvmGarbageCollectorMetricsTest.java index 8d8564996..86a89801e 100644 --- a/prometheus-metrics-instrumentation-jvm/src/test/java/io/prometheus/metrics/instrumentation/jvm/JvmGarbageCollectorMetricsTest.java +++ b/prometheus-metrics-instrumentation-jvm/src/test/java/io/prometheus/metrics/instrumentation/jvm/JvmGarbageCollectorMetricsTest.java @@ -1,7 +1,10 @@ package io.prometheus.metrics.instrumentation.jvm; +import static com.sun.management.GarbageCollectionNotificationInfo.GARBAGE_COLLECTION_NOTIFICATION; import static io.prometheus.metrics.instrumentation.jvm.TestUtil.convertToOpenMetricsFormat; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentCaptor.forClass; +import static org.mockito.Mockito.*; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -12,15 +15,22 @@ import java.io.IOException; import java.lang.management.GarbageCollectorMXBean; import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; import java.util.concurrent.TimeUnit; +import javax.management.Notification; +import javax.management.NotificationEmitter; +import javax.management.NotificationListener; +import javax.management.openmbean.*; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.mockito.Mockito; +import org.mockito.ArgumentCaptor; class JvmGarbageCollectorMetricsTest { - private final GarbageCollectorMXBean mockGcBean1 = Mockito.mock(GarbageCollectorMXBean.class); - private final GarbageCollectorMXBean mockGcBean2 = Mockito.mock(GarbageCollectorMXBean.class); + private final GarbageCollectorMXBean mockGcBean1 = mock(GarbageCollectorMXBean.class); + private final GarbageCollectorMXBean mockGcBean2 = mock(GarbageCollectorMXBean.class); @BeforeEach public void setUp() { @@ -72,4 +82,124 @@ public void testIgnoredMetricNotScraped() { verify(mockGcBean1, times(0)).getCollectionCount(); assertThat(snapshots.size()).isZero(); } + + @Test + @SuppressWarnings("rawtypes") + public void testGCDurationHistogramLabels() throws Exception { + GarbageCollectorMXBean mockGcBean = + mock( + GarbageCollectorMXBean.class, + withSettings().extraInterfaces(NotificationEmitter.class)); + when(mockGcBean.getName()).thenReturn("MyGC"); + + PrometheusRegistry registry = new PrometheusRegistry(); + JvmGarbageCollectorMetrics.builder() + .garbageCollectorBeans(Collections.singletonList(mockGcBean)) + .register(registry); + + NotificationListener listener; + ArgumentCaptor captor = forClass(NotificationListener.class); + verify((NotificationEmitter) mockGcBean) + .addNotificationListener(captor.capture(), isNull(), isNull()); + listener = captor.getValue(); + + TabularType memoryTabularType = getMemoryTabularType(); + TabularData memoryBefore = new TabularDataSupport(memoryTabularType); + TabularData memoryAfter = new TabularDataSupport(memoryTabularType); + + CompositeType gcInfoType = + new CompositeType( + "sun.management.BaseGcInfoCompositeType", + "gcInfo", + new String[] { + "id", "startTime", "endTime", "duration", "memoryUsageBeforeGc", "memoryUsageAfterGc" + }, + new String[] { + "id", "startTime", "endTime", "duration", "memoryUsageBeforeGc", "memoryUsageAfterGc" + }, + new OpenType[] { + SimpleType.LONG, + SimpleType.LONG, + SimpleType.LONG, + SimpleType.LONG, + memoryTabularType, + memoryTabularType + }); + + java.util.Map gcInfoMap = new HashMap<>(); + gcInfoMap.put("id", 0L); + gcInfoMap.put("startTime", 100L); + gcInfoMap.put("endTime", 200L); + gcInfoMap.put("duration", 100L); + gcInfoMap.put("memoryUsageBeforeGc", memoryBefore); + gcInfoMap.put("memoryUsageAfterGc", memoryAfter); + + CompositeData notificationData = getGcNotificationData(gcInfoType, gcInfoMap); + + Notification notification = + new Notification( + GARBAGE_COLLECTION_NOTIFICATION, mockGcBean, 1, System.currentTimeMillis(), "gc"); + notification.setUserData(notificationData); + + listener.handleNotification(notification, null); + + MetricSnapshots snapshots = registry.scrape(); + + String expected = + """ + {"jvm.gc.duration_bucket","jvm.gc.action"="end of minor GC","jvm.gc.cause"="testCause","jvm.gc.name"="MyGC",le="0.01"} 0 + {"jvm.gc.duration_bucket","jvm.gc.action"="end of minor GC","jvm.gc.cause"="testCause","jvm.gc.name"="MyGC",le="0.1"} 1 + {"jvm.gc.duration_bucket","jvm.gc.action"="end of minor GC","jvm.gc.cause"="testCause","jvm.gc.name"="MyGC",le="1.0"} 1 + {"jvm.gc.duration_bucket","jvm.gc.action"="end of minor GC","jvm.gc.cause"="testCause","jvm.gc.name"="MyGC",le="10.0"} 1 + {"jvm.gc.duration_bucket","jvm.gc.action"="end of minor GC","jvm.gc.cause"="testCause","jvm.gc.name"="MyGC",le="+Inf"} 1 + {"jvm.gc.duration_count","jvm.gc.action"="end of minor GC","jvm.gc.cause"="testCause","jvm.gc.name"="MyGC"} 1 + {"jvm.gc.duration_sum","jvm.gc.action"="end of minor GC","jvm.gc.cause"="testCause","jvm.gc.name"="MyGC"} 0.1 + """; + + String metrics = convertToOpenMetricsFormat(snapshots); + + assertThat(metrics).contains(expected); + } + + private TabularType getMemoryTabularType() throws OpenDataException { + CompositeType memoryUsageType = + new CompositeType( + "java.lang.management.MemoryUsage", + "MemoryUsage", + new String[] {"init", "used", "committed", "max"}, + new String[] {"init", "used", "committed", "max"}, + new OpenType[] {SimpleType.LONG, SimpleType.LONG, SimpleType.LONG, SimpleType.LONG}); + + CompositeType memoryUsageEntryType = + new CompositeType( + "memoryUsageEntry", + "memoryUsageEntry", + new String[] {"key", "value"}, + new String[] {"key", "value"}, + new OpenType[] {SimpleType.STRING, memoryUsageType}); + + return new TabularType( + "memoryUsageTabular", "memoryUsageTabular", memoryUsageEntryType, new String[] {"key"}); + } + + private static CompositeData getGcNotificationData( + CompositeType gcInfoType, Map gcInfoMap) throws OpenDataException { + CompositeData gcInfoData = new CompositeDataSupport(gcInfoType, gcInfoMap); + + CompositeType notificationType = + new CompositeType( + "sun.management.BaseGarbageCollectionNotifInfoCompositeType", + "GarbageCollectionNotificationInfo", + new String[] {"gcAction", "gcName", "gcCause", "gcInfo"}, + new String[] {"gcAction", "gcName", "gcCause", "gcInfo"}, + new OpenType[] {SimpleType.STRING, SimpleType.STRING, SimpleType.STRING, gcInfoType}); + + Map notifMap = new HashMap<>(); + notifMap.put("gcAction", "end of minor GC"); + notifMap.put("gcName", "MyGC"); + notifMap.put("gcCause", "testCause"); + notifMap.put("gcInfo", gcInfoData); + + return new CompositeDataSupport(notificationType, notifMap); + } }