lukaszraczylo/kubemirror
1
0
Fork 0
mirror of https://github.com/lukaszraczylo/kubemirror.git synced 2026-07-06 06:15:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(monitoring): match real controller label values in dashboards and alerts

Browse Source 
kubemirror overrides controller names to <Kind>.<version>.<group> (e.g. Secret.v1.), but the dashboards, alerts, and recording rules filtered controller=~"secret|configmap" (the controller-runtime default name), so every panel and alert matched nothing. Switch all selectors to Secret.*|ConfigMap.* and document the naming convention.
This commit is contained in:
Lukasz Raczylo
2026-06-21 13:26:49 +01:00
parent 6d27d7dfe7
commit 28896ab1d6
3 changed files with 29 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files
monitoring/README.md
+6 -1
View File
@@ -159,9 +159,14 @@ histogram_quantile(0.99,
)
# Current workqueue depth
workqueue_depth{name=~"secret|configmap"}
workqueue_depth{name=~"Secret.*|ConfigMap.*"}
```
> **Controller label values:** kubemirror names each controller `<Kind>.<version>.<group>`
> (e.g. `Secret.v1.`, `ConfigMap.v1.`) plus a `-mirror` variant for orphan
> detection, so PromQL filters use prefixes like `controller=~"Secret.*"` rather
> than `controller="secret"`. Adjust the kind for other mirrored resource types.
### Using kubectl
```bash
monitoring/grafana-dashboard.json
+11 -11
View File
@@ -134,12 +134,12 @@
"pluginVersion": "8.0.0",
"targets": [
{
"expr": "sum(rate(controller_runtime_reconcile_total{controller=\"secret\"}[5m])) by (result)",
"expr": "sum(rate(controller_runtime_reconcile_total{controller=~\"Secret.*\"}[5m])) by (result)",
"legendFormat": "Secret - {{result}}",
"refId": "A"
},
{
"expr": "sum(rate(controller_runtime_reconcile_total{controller=\"configmap\"}[5m])) by (result)",
"expr": "sum(rate(controller_runtime_reconcile_total{controller=~\"ConfigMap.*\"}[5m])) by (result)",
"legendFormat": "ConfigMap - {{result}}",
"refId": "B"
}
@@ -198,7 +198,7 @@
"pluginVersion": "8.0.0",
"targets": [
{
"expr": "sum(workqueue_depth{name=~\"secret|configmap\"})",
"expr": "sum(workqueue_depth{name=~\"Secret.*|ConfigMap.*\"})",
"refId": "A"
}
],
@@ -266,22 +266,22 @@
"pluginVersion": "8.0.0",
"targets": [
{
"expr": "histogram_quantile(0.99, sum(rate(controller_runtime_reconcile_time_seconds_bucket{controller=\"secret\"}[5m])) by (le))",
"expr": "histogram_quantile(0.99, sum(rate(controller_runtime_reconcile_time_seconds_bucket{controller=~\"Secret.*\"}[5m])) by (le))",
"legendFormat": "Secret p99",
"refId": "A"
},
{
"expr": "histogram_quantile(0.95, sum(rate(controller_runtime_reconcile_time_seconds_bucket{controller=\"secret\"}[5m])) by (le))",
"expr": "histogram_quantile(0.95, sum(rate(controller_runtime_reconcile_time_seconds_bucket{controller=~\"Secret.*\"}[5m])) by (le))",
"legendFormat": "Secret p95",
"refId": "B"
},
{
"expr": "histogram_quantile(0.99, sum(rate(controller_runtime_reconcile_time_seconds_bucket{controller=\"configmap\"}[5m])) by (le))",
"expr": "histogram_quantile(0.99, sum(rate(controller_runtime_reconcile_time_seconds_bucket{controller=~\"ConfigMap.*\"}[5m])) by (le))",
"legendFormat": "ConfigMap p99",
"refId": "C"
},
{
"expr": "histogram_quantile(0.95, sum(rate(controller_runtime_reconcile_time_seconds_bucket{controller=\"configmap\"}[5m])) by (le))",
"expr": "histogram_quantile(0.95, sum(rate(controller_runtime_reconcile_time_seconds_bucket{controller=~\"ConfigMap.*\"}[5m])) by (le))",
"legendFormat": "ConfigMap p95",
"refId": "D"
}
@@ -350,12 +350,12 @@
"pluginVersion": "8.0.0",
"targets": [
{
"expr": "workqueue_depth{name=\"secret\"}",
"expr": "workqueue_depth{name=~\"Secret.*\"}",
"legendFormat": "Secret",
"refId": "A"
},
{
"expr": "workqueue_depth{name=\"configmap\"}",
"expr": "workqueue_depth{name=~\"ConfigMap.*\"}",
"legendFormat": "ConfigMap",
"refId": "B"
}
@@ -571,12 +571,12 @@
"pluginVersion": "8.0.0",
"targets": [
{
"expr": "sum(rate(controller_runtime_reconcile_errors_total{controller=\"secret\"}[5m])) / sum(rate(controller_runtime_reconcile_total{controller=\"secret\"}[5m]))",
"expr": "sum(rate(controller_runtime_reconcile_errors_total{controller=~\"Secret.*\"}[5m])) / sum(rate(controller_runtime_reconcile_total{controller=~\"Secret.*\"}[5m]))",
"legendFormat": "Secret Error Rate",
"refId": "A"
},
{
"expr": "sum(rate(controller_runtime_reconcile_errors_total{controller=\"configmap\"}[5m])) / sum(rate(controller_runtime_reconcile_total{controller=\"configmap\"}[5m]))",
"expr": "sum(rate(controller_runtime_reconcile_errors_total{controller=~\"ConfigMap.*\"}[5m])) / sum(rate(controller_runtime_reconcile_total{controller=~\"ConfigMap.*\"}[5m]))",
"legendFormat": "ConfigMap Error Rate",
"refId": "B"
}
monitoring/prometheusrule.yaml
+12 -12
View File
@@ -25,9 +25,9 @@ spec:
- alert: KubeMirrorHighReconcileErrors
expr: |
rate(controller_runtime_reconcile_errors_total{controller="secret"}[5m]) > 0.1
rate(controller_runtime_reconcile_errors_total{controller=~"Secret.*"}[5m]) > 0.1
or
rate(controller_runtime_reconcile_errors_total{controller="configmap"}[5m]) > 0.1
rate(controller_runtime_reconcile_errors_total{controller=~"ConfigMap.*"}[5m]) > 0.1
for: 10m
labels:
severity: warning
@@ -39,7 +39,7 @@ spec:
- alert: KubeMirrorReconcileLatencyHigh
expr: |
histogram_quantile(0.99,
rate(controller_runtime_reconcile_time_seconds_bucket{controller=~"secret|configmap"}[5m])
rate(controller_runtime_reconcile_time_seconds_bucket{controller=~"Secret.*|ConfigMap.*"}[5m])
) > 5
for: 10m
labels:
@@ -51,7 +51,7 @@ spec:
- alert: KubeMirrorWorkqueueDepthHigh
expr: |
workqueue_depth{name=~"secret|configmap"} > 100
workqueue_depth{name=~"Secret.*|ConfigMap.*"} > 100
for: 15m
labels:
severity: warning
@@ -74,9 +74,9 @@ spec:
# Resource mirror alerts
- alert: KubeMirrorHighFailureRate
expr: |
sum(rate(controller_runtime_reconcile_errors_total{controller=~"secret|configmap"}[5m]))
sum(rate(controller_runtime_reconcile_errors_total{controller=~"Secret.*|ConfigMap.*"}[5m]))
/
sum(rate(controller_runtime_reconcile_total{controller=~"secret|configmap"}[5m]))
sum(rate(controller_runtime_reconcile_total{controller=~"Secret.*|ConfigMap.*"}[5m]))
> 0.05
for: 10m
labels:
@@ -118,29 +118,29 @@ spec:
- record: kubemirror:reconcile_duration_seconds:p99
expr: |
histogram_quantile(0.99,
rate(controller_runtime_reconcile_time_seconds_bucket{controller=~"secret|configmap"}[5m])
rate(controller_runtime_reconcile_time_seconds_bucket{controller=~"Secret.*|ConfigMap.*"}[5m])
)
- record: kubemirror:reconcile_duration_seconds:p95
expr: |
histogram_quantile(0.95,
rate(controller_runtime_reconcile_time_seconds_bucket{controller=~"secret|configmap"}[5m])
rate(controller_runtime_reconcile_time_seconds_bucket{controller=~"Secret.*|ConfigMap.*"}[5m])
)
- record: kubemirror:reconcile_duration_seconds:p50
expr: |
histogram_quantile(0.50,
rate(controller_runtime_reconcile_time_seconds_bucket{controller=~"secret|configmap"}[5m])
rate(controller_runtime_reconcile_time_seconds_bucket{controller=~"Secret.*|ConfigMap.*"}[5m])
)
- record: kubemirror:reconcile_rate:5m
expr: |
sum(rate(controller_runtime_reconcile_total{controller=~"secret|configmap"}[5m])) by (controller, result)
sum(rate(controller_runtime_reconcile_total{controller=~"Secret.*|ConfigMap.*"}[5m])) by (controller, result)
- record: kubemirror:reconcile_errors:rate5m
expr: |
sum(rate(controller_runtime_reconcile_errors_total{controller=~"secret|configmap"}[5m])) by (controller)
sum(rate(controller_runtime_reconcile_errors_total{controller=~"Secret.*|ConfigMap.*"}[5m])) by (controller)
- record: kubemirror:workqueue_depth:max
expr: |
max(workqueue_depth{name=~"secret|configmap"}) by (name)
max(workqueue_depth{name=~"Secret.*|ConfigMap.*"}) by (name)