lukaszraczylo/kubemirror
1
0
Fork 0
mirror of https://github.com/lukaszraczylo/kubemirror.git synced 2026-07-11 21:11:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(monitoring): match real controller label values in dashboards and alerts

Browse Source 
kubemirror overrides controller names to <Kind>.<version>.<group> (e.g. Secret.v1.), but the dashboards, alerts, and recording rules filtered controller=~"secret|configmap" (the controller-runtime default name), so every panel and alert matched nothing. Switch all selectors to Secret.*|ConfigMap.* and document the naming convention.
This commit is contained in:
Lukasz Raczylo
2026-06-21 13:26:49 +01:00
parent 6d27d7dfe7
commit 28896ab1d6
3 changed files with 29 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files
monitoring/README.md
+6 -1
View File
@@ -159,9 +159,14 @@ histogram_quantile(0.99,
) )
# Current workqueue depth # Current workqueue depth
workqueue_depth{name=~"secret|configmap"} workqueue_depth{name=~"Secret.*|ConfigMap.*"}
``` ```
> **Controller label values:** kubemirror names each controller `<Kind>.<version>.<group>`
> (e.g. `Secret.v1.`, `ConfigMap.v1.`) plus a `-mirror` variant for orphan
> detection, so PromQL filters use prefixes like `controller=~"Secret.*"` rather
> than `controller="secret"`. Adjust the kind for other mirrored resource types.
### Using kubectl ### Using kubectl
```bash ```bash
monitoring/grafana-dashboard.json
+11 -11
View File
@@ -134,12 +134,12 @@
"pluginVersion": "8.0.0", "pluginVersion": "8.0.0",
"targets": [ "targets": [
{ {
"expr": "sum(rate(controller_runtime_reconcile_total{controller=\"secret\"}[5m])) by (result)", "expr": "sum(rate(controller_runtime_reconcile_total{controller=~\"Secret.*\"}[5m])) by (result)",
"legendFormat": "Secret - {{result}}", "legendFormat": "Secret - {{result}}",
"refId": "A" "refId": "A"
}, },
{ {
"expr": "sum(rate(controller_runtime_reconcile_total{controller=\"configmap\"}[5m])) by (result)", "expr": "sum(rate(controller_runtime_reconcile_total{controller=~\"ConfigMap.*\"}[5m])) by (result)",
"legendFormat": "ConfigMap - {{result}}", "legendFormat": "ConfigMap - {{result}}",
"refId": "B" "refId": "B"
} }
@@ -198,7 +198,7 @@
"pluginVersion": "8.0.0", "pluginVersion": "8.0.0",
"targets": [ "targets": [
{ {
"expr": "sum(workqueue_depth{name=~\"secret|configmap\"})", "expr": "sum(workqueue_depth{name=~\"Secret.*|ConfigMap.*\"})",
"refId": "A" "refId": "A"
} }
], ],
@@ -266,22 +266,22 @@
"pluginVersion": "8.0.0", "pluginVersion": "8.0.0",
"targets": [ "targets": [
{ {
"expr": "histogram_quantile(0.99, sum(rate(controller_runtime_reconcile_time_seconds_bucket{controller=\"secret\"}[5m])) by (le))", "expr": "histogram_quantile(0.99, sum(rate(controller_runtime_reconcile_time_seconds_bucket{controller=~\"Secret.*\"}[5m])) by (le))",
"legendFormat": "Secret p99", "legendFormat": "Secret p99",
"refId": "A" "refId": "A"
}, },
{ {
"expr": "histogram_quantile(0.95, sum(rate(controller_runtime_reconcile_time_seconds_bucket{controller=\"secret\"}[5m])) by (le))", "expr": "histogram_quantile(0.95, sum(rate(controller_runtime_reconcile_time_seconds_bucket{controller=~\"Secret.*\"}[5m])) by (le))",
"legendFormat": "Secret p95", "legendFormat": "Secret p95",
"refId": "B" "refId": "B"
}, },
{ {
"expr": "histogram_quantile(0.99, sum(rate(controller_runtime_reconcile_time_seconds_bucket{controller=\"configmap\"}[5m])) by (le))", "expr": "histogram_quantile(0.99, sum(rate(controller_runtime_reconcile_time_seconds_bucket{controller=~\"ConfigMap.*\"}[5m])) by (le))",
"legendFormat": "ConfigMap p99", "legendFormat": "ConfigMap p99",
"refId": "C" "refId": "C"
}, },
{ {
"expr": "histogram_quantile(0.95, sum(rate(controller_runtime_reconcile_time_seconds_bucket{controller=\"configmap\"}[5m])) by (le))", "expr": "histogram_quantile(0.95, sum(rate(controller_runtime_reconcile_time_seconds_bucket{controller=~\"ConfigMap.*\"}[5m])) by (le))",
"legendFormat": "ConfigMap p95", "legendFormat": "ConfigMap p95",
"refId": "D" "refId": "D"
} }
@@ -350,12 +350,12 @@
"pluginVersion": "8.0.0", "pluginVersion": "8.0.0",
"targets": [ "targets": [
{ {
"expr": "workqueue_depth{name=\"secret\"}", "expr": "workqueue_depth{name=~\"Secret.*\"}",
"legendFormat": "Secret", "legendFormat": "Secret",
"refId": "A" "refId": "A"
}, },
{ {
"expr": "workqueue_depth{name=\"configmap\"}", "expr": "workqueue_depth{name=~\"ConfigMap.*\"}",
"legendFormat": "ConfigMap", "legendFormat": "ConfigMap",
"refId": "B" "refId": "B"
} }
@@ -571,12 +571,12 @@
"pluginVersion": "8.0.0", "pluginVersion": "8.0.0",
"targets": [ "targets": [
{ {
"expr": "sum(rate(controller_runtime_reconcile_errors_total{controller=\"secret\"}[5m])) / sum(rate(controller_runtime_reconcile_total{controller=\"secret\"}[5m]))", "expr": "sum(rate(controller_runtime_reconcile_errors_total{controller=~\"Secret.*\"}[5m])) / sum(rate(controller_runtime_reconcile_total{controller=~\"Secret.*\"}[5m]))",
"legendFormat": "Secret Error Rate", "legendFormat": "Secret Error Rate",
"refId": "A" "refId": "A"
}, },
{ {
"expr": "sum(rate(controller_runtime_reconcile_errors_total{controller=\"configmap\"}[5m])) / sum(rate(controller_runtime_reconcile_total{controller=\"configmap\"}[5m]))", "expr": "sum(rate(controller_runtime_reconcile_errors_total{controller=~\"ConfigMap.*\"}[5m])) / sum(rate(controller_runtime_reconcile_total{controller=~\"ConfigMap.*\"}[5m]))",
"legendFormat": "ConfigMap Error Rate", "legendFormat": "ConfigMap Error Rate",
"refId": "B" "refId": "B"
} }
monitoring/prometheusrule.yaml
+12 -12
View File
@@ -25,9 +25,9 @@ spec:
- alert: KubeMirrorHighReconcileErrors - alert: KubeMirrorHighReconcileErrors
expr: | expr: |
rate(controller_runtime_reconcile_errors_total{controller="secret"}[5m]) > 0.1 rate(controller_runtime_reconcile_errors_total{controller=~"Secret.*"}[5m]) > 0.1
or or
rate(controller_runtime_reconcile_errors_total{controller="configmap"}[5m]) > 0.1 rate(controller_runtime_reconcile_errors_total{controller=~"ConfigMap.*"}[5m]) > 0.1
for: 10m for: 10m
labels: labels:
severity: warning severity: warning
@@ -39,7 +39,7 @@ spec:
- alert: KubeMirrorReconcileLatencyHigh - alert: KubeMirrorReconcileLatencyHigh
expr: | expr: |
histogram_quantile(0.99, histogram_quantile(0.99,
rate(controller_runtime_reconcile_time_seconds_bucket{controller=~"secret|configmap"}[5m]) rate(controller_runtime_reconcile_time_seconds_bucket{controller=~"Secret.*|ConfigMap.*"}[5m])
) > 5 ) > 5
for: 10m for: 10m
labels: labels:
@@ -51,7 +51,7 @@ spec:
- alert: KubeMirrorWorkqueueDepthHigh - alert: KubeMirrorWorkqueueDepthHigh
expr: | expr: |
workqueue_depth{name=~"secret|configmap"} > 100 workqueue_depth{name=~"Secret.*|ConfigMap.*"} > 100
for: 15m for: 15m
labels: labels:
severity: warning severity: warning
@@ -74,9 +74,9 @@ spec:
# Resource mirror alerts # Resource mirror alerts
- alert: KubeMirrorHighFailureRate - alert: KubeMirrorHighFailureRate
expr: | expr: |
sum(rate(controller_runtime_reconcile_errors_total{controller=~"secret|configmap"}[5m])) sum(rate(controller_runtime_reconcile_errors_total{controller=~"Secret.*|ConfigMap.*"}[5m]))
/ /
sum(rate(controller_runtime_reconcile_total{controller=~"secret|configmap"}[5m])) sum(rate(controller_runtime_reconcile_total{controller=~"Secret.*|ConfigMap.*"}[5m]))
> 0.05 > 0.05
for: 10m for: 10m
labels: labels:
@@ -118,29 +118,29 @@ spec:
- record: kubemirror:reconcile_duration_seconds:p99 - record: kubemirror:reconcile_duration_seconds:p99
expr: | expr: |
histogram_quantile(0.99, histogram_quantile(0.99,
rate(controller_runtime_reconcile_time_seconds_bucket{controller=~"secret|configmap"}[5m]) rate(controller_runtime_reconcile_time_seconds_bucket{controller=~"Secret.*|ConfigMap.*"}[5m])
) )
- record: kubemirror:reconcile_duration_seconds:p95 - record: kubemirror:reconcile_duration_seconds:p95
expr: | expr: |
histogram_quantile(0.95, histogram_quantile(0.95,
rate(controller_runtime_reconcile_time_seconds_bucket{controller=~"secret|configmap"}[5m]) rate(controller_runtime_reconcile_time_seconds_bucket{controller=~"Secret.*|ConfigMap.*"}[5m])
) )
- record: kubemirror:reconcile_duration_seconds:p50 - record: kubemirror:reconcile_duration_seconds:p50
expr: | expr: |
histogram_quantile(0.50, histogram_quantile(0.50,
rate(controller_runtime_reconcile_time_seconds_bucket{controller=~"secret|configmap"}[5m]) rate(controller_runtime_reconcile_time_seconds_bucket{controller=~"Secret.*|ConfigMap.*"}[5m])
) )
- record: kubemirror:reconcile_rate:5m - record: kubemirror:reconcile_rate:5m
expr: | expr: |
sum(rate(controller_runtime_reconcile_total{controller=~"secret|configmap"}[5m])) by (controller, result) sum(rate(controller_runtime_reconcile_total{controller=~"Secret.*|ConfigMap.*"}[5m])) by (controller, result)
- record: kubemirror:reconcile_errors:rate5m - record: kubemirror:reconcile_errors:rate5m
expr: | expr: |
sum(rate(controller_runtime_reconcile_errors_total{controller=~"secret|configmap"}[5m])) by (controller) sum(rate(controller_runtime_reconcile_errors_total{controller=~"Secret.*|ConfigMap.*"}[5m])) by (controller)
- record: kubemirror:workqueue_depth:max - record: kubemirror:workqueue_depth:max
expr: | expr: |
max(workqueue_depth{name=~"secret|configmap"}) by (name) max(workqueue_depth{name=~"Secret.*|ConfigMap.*"}) by (name)