diff --git a/monitoring/README.md b/monitoring/README.md index 1a11432..65a4e00 100644 --- a/monitoring/README.md +++ b/monitoring/README.md @@ -159,9 +159,14 @@ histogram_quantile(0.99, ) # Current workqueue depth -workqueue_depth{name=~"secret|configmap"} +workqueue_depth{name=~"Secret.*|ConfigMap.*"} ``` +> **Controller label values:** kubemirror names each controller `..` +> (e.g. `Secret.v1.`, `ConfigMap.v1.`) plus a `-mirror` variant for orphan +> detection, so PromQL filters use prefixes like `controller=~"Secret.*"` rather +> than `controller="secret"`. Adjust the kind for other mirrored resource types. + ### Using kubectl ```bash diff --git a/monitoring/grafana-dashboard.json b/monitoring/grafana-dashboard.json index a495a54..92e2929 100644 --- a/monitoring/grafana-dashboard.json +++ b/monitoring/grafana-dashboard.json @@ -134,12 +134,12 @@ "pluginVersion": "8.0.0", "targets": [ { - "expr": "sum(rate(controller_runtime_reconcile_total{controller=\"secret\"}[5m])) by (result)", + "expr": "sum(rate(controller_runtime_reconcile_total{controller=~\"Secret.*\"}[5m])) by (result)", "legendFormat": "Secret - {{result}}", "refId": "A" }, { - "expr": "sum(rate(controller_runtime_reconcile_total{controller=\"configmap\"}[5m])) by (result)", + "expr": "sum(rate(controller_runtime_reconcile_total{controller=~\"ConfigMap.*\"}[5m])) by (result)", "legendFormat": "ConfigMap - {{result}}", "refId": "B" } @@ -198,7 +198,7 @@ "pluginVersion": "8.0.0", "targets": [ { - "expr": "sum(workqueue_depth{name=~\"secret|configmap\"})", + "expr": "sum(workqueue_depth{name=~\"Secret.*|ConfigMap.*\"})", "refId": "A" } ], @@ -266,22 +266,22 @@ "pluginVersion": "8.0.0", "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(controller_runtime_reconcile_time_seconds_bucket{controller=\"secret\"}[5m])) by (le))", + "expr": "histogram_quantile(0.99, sum(rate(controller_runtime_reconcile_time_seconds_bucket{controller=~\"Secret.*\"}[5m])) by (le))", "legendFormat": "Secret p99", "refId": "A" }, { - "expr": "histogram_quantile(0.95, sum(rate(controller_runtime_reconcile_time_seconds_bucket{controller=\"secret\"}[5m])) by (le))", + "expr": "histogram_quantile(0.95, sum(rate(controller_runtime_reconcile_time_seconds_bucket{controller=~\"Secret.*\"}[5m])) by (le))", "legendFormat": "Secret p95", "refId": "B" }, { - "expr": "histogram_quantile(0.99, sum(rate(controller_runtime_reconcile_time_seconds_bucket{controller=\"configmap\"}[5m])) by (le))", + "expr": "histogram_quantile(0.99, sum(rate(controller_runtime_reconcile_time_seconds_bucket{controller=~\"ConfigMap.*\"}[5m])) by (le))", "legendFormat": "ConfigMap p99", "refId": "C" }, { - "expr": "histogram_quantile(0.95, sum(rate(controller_runtime_reconcile_time_seconds_bucket{controller=\"configmap\"}[5m])) by (le))", + "expr": "histogram_quantile(0.95, sum(rate(controller_runtime_reconcile_time_seconds_bucket{controller=~\"ConfigMap.*\"}[5m])) by (le))", "legendFormat": "ConfigMap p95", "refId": "D" } @@ -350,12 +350,12 @@ "pluginVersion": "8.0.0", "targets": [ { - "expr": "workqueue_depth{name=\"secret\"}", + "expr": "workqueue_depth{name=~\"Secret.*\"}", "legendFormat": "Secret", "refId": "A" }, { - "expr": "workqueue_depth{name=\"configmap\"}", + "expr": "workqueue_depth{name=~\"ConfigMap.*\"}", "legendFormat": "ConfigMap", "refId": "B" } @@ -571,12 +571,12 @@ "pluginVersion": "8.0.0", "targets": [ { - "expr": "sum(rate(controller_runtime_reconcile_errors_total{controller=\"secret\"}[5m])) / sum(rate(controller_runtime_reconcile_total{controller=\"secret\"}[5m]))", + "expr": "sum(rate(controller_runtime_reconcile_errors_total{controller=~\"Secret.*\"}[5m])) / sum(rate(controller_runtime_reconcile_total{controller=~\"Secret.*\"}[5m]))", "legendFormat": "Secret Error Rate", "refId": "A" }, { - "expr": "sum(rate(controller_runtime_reconcile_errors_total{controller=\"configmap\"}[5m])) / sum(rate(controller_runtime_reconcile_total{controller=\"configmap\"}[5m]))", + "expr": "sum(rate(controller_runtime_reconcile_errors_total{controller=~\"ConfigMap.*\"}[5m])) / sum(rate(controller_runtime_reconcile_total{controller=~\"ConfigMap.*\"}[5m]))", "legendFormat": "ConfigMap Error Rate", "refId": "B" } diff --git a/monitoring/prometheusrule.yaml b/monitoring/prometheusrule.yaml index d96e840..aa7ddd1 100644 --- a/monitoring/prometheusrule.yaml +++ b/monitoring/prometheusrule.yaml @@ -25,9 +25,9 @@ spec: - alert: KubeMirrorHighReconcileErrors expr: | - rate(controller_runtime_reconcile_errors_total{controller="secret"}[5m]) > 0.1 + rate(controller_runtime_reconcile_errors_total{controller=~"Secret.*"}[5m]) > 0.1 or - rate(controller_runtime_reconcile_errors_total{controller="configmap"}[5m]) > 0.1 + rate(controller_runtime_reconcile_errors_total{controller=~"ConfigMap.*"}[5m]) > 0.1 for: 10m labels: severity: warning @@ -39,7 +39,7 @@ spec: - alert: KubeMirrorReconcileLatencyHigh expr: | histogram_quantile(0.99, - rate(controller_runtime_reconcile_time_seconds_bucket{controller=~"secret|configmap"}[5m]) + rate(controller_runtime_reconcile_time_seconds_bucket{controller=~"Secret.*|ConfigMap.*"}[5m]) ) > 5 for: 10m labels: @@ -51,7 +51,7 @@ spec: - alert: KubeMirrorWorkqueueDepthHigh expr: | - workqueue_depth{name=~"secret|configmap"} > 100 + workqueue_depth{name=~"Secret.*|ConfigMap.*"} > 100 for: 15m labels: severity: warning @@ -74,9 +74,9 @@ spec: # Resource mirror alerts - alert: KubeMirrorHighFailureRate expr: | - sum(rate(controller_runtime_reconcile_errors_total{controller=~"secret|configmap"}[5m])) + sum(rate(controller_runtime_reconcile_errors_total{controller=~"Secret.*|ConfigMap.*"}[5m])) / - sum(rate(controller_runtime_reconcile_total{controller=~"secret|configmap"}[5m])) + sum(rate(controller_runtime_reconcile_total{controller=~"Secret.*|ConfigMap.*"}[5m])) > 0.05 for: 10m labels: @@ -118,29 +118,29 @@ spec: - record: kubemirror:reconcile_duration_seconds:p99 expr: | histogram_quantile(0.99, - rate(controller_runtime_reconcile_time_seconds_bucket{controller=~"secret|configmap"}[5m]) + rate(controller_runtime_reconcile_time_seconds_bucket{controller=~"Secret.*|ConfigMap.*"}[5m]) ) - record: kubemirror:reconcile_duration_seconds:p95 expr: | histogram_quantile(0.95, - rate(controller_runtime_reconcile_time_seconds_bucket{controller=~"secret|configmap"}[5m]) + rate(controller_runtime_reconcile_time_seconds_bucket{controller=~"Secret.*|ConfigMap.*"}[5m]) ) - record: kubemirror:reconcile_duration_seconds:p50 expr: | histogram_quantile(0.50, - rate(controller_runtime_reconcile_time_seconds_bucket{controller=~"secret|configmap"}[5m]) + rate(controller_runtime_reconcile_time_seconds_bucket{controller=~"Secret.*|ConfigMap.*"}[5m]) ) - record: kubemirror:reconcile_rate:5m expr: | - sum(rate(controller_runtime_reconcile_total{controller=~"secret|configmap"}[5m])) by (controller, result) + sum(rate(controller_runtime_reconcile_total{controller=~"Secret.*|ConfigMap.*"}[5m])) by (controller, result) - record: kubemirror:reconcile_errors:rate5m expr: | - sum(rate(controller_runtime_reconcile_errors_total{controller=~"secret|configmap"}[5m])) by (controller) + sum(rate(controller_runtime_reconcile_errors_total{controller=~"Secret.*|ConfigMap.*"}[5m])) by (controller) - record: kubemirror:workqueue_depth:max expr: | - max(workqueue_depth{name=~"secret|configmap"}) by (name) + max(workqueue_depth{name=~"Secret.*|ConfigMap.*"}) by (name)