Skip to content

Commit da2b620

Browse files
Jayclifford345Jayclifford345
authored
fix(mixins): fix label generation for Loki logs dashboard (#17412)
1 parent 50b12ef commit da2b620

11 files changed

Lines changed: 45 additions & 92 deletions

File tree

‎production/helm/meta-monitoring/values.yaml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -136,16 +136,16 @@ podLogs:
136136
- job
137137
- level
138138
- namespace
139-
- pod
140139
- service_name
141140
- cluster
142141
gatherMethod: kubernetesApi
143142
namespaces:
144143
- meta
145144
- loki
145+
structuredMetadata:
146+
pod:
146147

147148
# Collectors
148-
149149
# The Alloy Singleton is a single instance of the Alloy Collector that is deployed in the cluster.
150150
alloy-singleton:
151151
enabled: true

‎production/loki-mixin-compiled-ssd/dashboards/loki-deletion.json‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -379,7 +379,7 @@
379379
"span": 4,
380380
"targets": [
381381
{
382-
"expr": "node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=~\"$cluster\", namespace=~\"$namespace\", pod=~\"(.*compactor|(loki.*|enterprise-logs)-backend.*|loki-single-binary)\"}",
382+
"expr": "node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=~\"$cluster\", namespace=~\"$namespace\", pod=~\"(.*compactor.*|(loki.*|enterprise-logs)-backend.*|loki-single-binary)\"}",
383383
"format": "time_series",
384384
"legendFormat": "{{pod}}",
385385
"legendLink": null
@@ -426,7 +426,7 @@
426426
"span": 4,
427427
"targets": [
428428
{
429-
"expr": "go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", pod=~\"(.*compactor|(loki.*|enterprise-logs)-backend.*|loki-single-binary)\"} / 1024 / 1024 ",
429+
"expr": "go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", pod=~\"(.*compactor.*|(loki.*|enterprise-logs)-backend.*|loki-single-binary)\"} / 1024 / 1024 ",
430430
"format": "time_series",
431431
"legendFormat": " {{pod}} ",
432432
"legendLink": null
@@ -579,7 +579,7 @@
579579
"span": 6,
580580
"targets": [
581581
{
582-
"expr": "sum(rate(loki_compactor_deleted_lines{cluster=~\"$cluster\", namespace=~\"$namespace\", pod=~\"(.*compactor|(loki.*|enterprise-logs)-backend.*|loki-single-binary)\"}[$__rate_interval])) by (user)",
582+
"expr": "sum(rate(loki_compactor_deleted_lines{cluster=~\"$cluster\", namespace=~\"$namespace\", pod=~\"(.*compactor.*|(loki.*|enterprise-logs)-backend.*|loki-single-binary)\"}[$__rate_interval])) by (user)",
583583
"format": "time_series",
584584
"legendFormat": "{{user}}",
585585
"legendLink": null
@@ -606,7 +606,7 @@
606606
"span": 6,
607607
"targets": [
608608
{
609-
"expr": "{cluster=~\"$cluster\", namespace=~\"$namespace\", pod=~\"(.*compactor|(loki.*|enterprise-logs)-backend.*|loki-single-binary)\"} |~ \"Started processing delete request|delete request for user marked as processed\" | logfmt | line_format \"{{.ts}} user={{.user}} delete_request_id={{.delete_request_id}} msg={{.msg}}\" ",
609+
"expr": "{cluster=~\"$cluster\", namespace=~\"$namespace\", pod=~\"(.*compactor.*|(loki.*|enterprise-logs)-backend.*|loki-single-binary)\"} |~ \"Started processing delete request|delete request for user marked as processed\" | logfmt | line_format \"{{.ts}} user={{.user}} delete_request_id={{.delete_request_id}} msg={{.msg}}\" ",
610610
"refId": "A"
611611
}
612612
],
@@ -619,7 +619,7 @@
619619
"span": 6,
620620
"targets": [
621621
{
622-
"expr": "{cluster=~\"$cluster\", namespace=~\"$namespace\", pod=~\"(.*compactor|(loki.*|enterprise-logs)-backend.*|loki-single-binary)\"} |~ \"delete request for user added\" | logfmt | line_format \"{{.ts}} user={{.user}} query='{{.query}}'\"",
622+
"expr": "{cluster=~\"$cluster\", namespace=~\"$namespace\", pod=~\"(.*compactor.*|(loki.*|enterprise-logs)-backend.*|loki-single-binary)\"} |~ \"delete request for user added\" | logfmt | line_format \"{{.ts}} user={{.user}} query='{{.query}}'\"",
623623
"refId": "A"
624624
}
625625
],

‎production/loki-mixin-compiled-ssd/dashboards/loki-logs.json‎

Lines changed: 9 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@
6363
"steppedLine": false,
6464
"targets": [
6565
{
66-
"expr": "sum by (pod)(go_goroutines{cluster=\"$cluster\", namespace=\"$namespace\", container=~\"$container\", pod=~\"$pod\"})",
66+
"expr": "sum by (pod)(go_goroutines{cluster=\"$cluster\", namespace=\"$namespace\", container=~\"$container\" })",
6767
"refId": "A"
6868
}
6969
],
@@ -241,7 +241,7 @@
241241
"steppedLine": false,
242242
"targets": [
243243
{
244-
"expr": "sum by (pod)(rate(container_cpu_usage_seconds_total{cluster=\"$cluster\", namespace=\"$namespace\", container=~\"$container\", pod=~\"$pod\"}[$__rate_interval]))",
244+
"expr": "sum by (pod)(rate(container_cpu_usage_seconds_total{cluster=\"$cluster\", namespace=\"$namespace\", container=~\"$container\"}[$__rate_interval]))",
245245
"refId": "A"
246246
}
247247
],
@@ -332,7 +332,7 @@
332332
"steppedLine": false,
333333
"targets": [
334334
{
335-
"expr": "sum by (pod)(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container=~\"$container\", pod=~\"$pod\"})",
335+
"expr": "sum by (pod)(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container=~\"$container\"})",
336336
"refId": "A"
337337
}
338338
],
@@ -423,7 +423,7 @@
423423
"steppedLine": false,
424424
"targets": [
425425
{
426-
"expr": "sum by (pod)(rate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\", container=~\"$container\", pod=~\"$pod\"}[$__rate_interval]))",
426+
"expr": "sum by (pod)(rate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\", container=~\"$container\"}[$__rate_interval]))",
427427
"refId": "A"
428428
}
429429
],
@@ -514,7 +514,7 @@
514514
"steppedLine": false,
515515
"targets": [
516516
{
517-
"expr": "sum by (pod)(rate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval]))",
517+
"expr": "sum by (pod)(rate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))",
518518
"refId": "A"
519519
}
520520
],
@@ -600,7 +600,7 @@
600600
"steppedLine": false,
601601
"targets": [
602602
{
603-
"expr": "increase(kube_pod_container_status_last_terminated_reason{cluster=\"$cluster\", namespace=\"$namespace\", container=~\"$container\", pod=~\"$pod\"}[30m]) > 0",
603+
"expr": "increase(kube_pod_container_status_last_terminated_reason{cluster=\"$cluster\", namespace=\"$namespace\", container=~\"$container\"}[30m]) > 0",
604604
"legendFormat": "{{reason}}",
605605
"refId": "A"
606606
}
@@ -692,7 +692,7 @@
692692
"steppedLine": false,
693693
"targets": [
694694
{
695-
"expr": "sum(rate(promtail_custom_bad_words_total{cluster=\"$cluster\", exported_namespace=\"$namespace\", exported_pod=~\"$pod\", container=~\"$container\"}[$__rate_interval])) by (level)",
695+
"expr": "sum(rate(promtail_custom_bad_words_total{cluster=\"$cluster\", exported_namespace=\"$namespace\", container=~\"$container\"}[$__rate_interval])) by (level)",
696696
"legendFormat": "{{level}}",
697697
"refId": "A"
698698
}
@@ -801,7 +801,7 @@
801801
"steppedLine": false,
802802
"targets": [
803803
{
804-
"expr": "sum(rate({cluster=\"$cluster\", namespace=\"$namespace\", container=~\"$container\", pod=~\"$pod\" } |logfmt| level=~\"$level\" |= \"$filter\" | __error__=\"\" [$__auto])) by (level)",
804+
"expr": "sum(rate({cluster=\"$cluster\", namespace=\"$namespace\", container=~\"$container\" } |logfmt| level=~\"$level\" |= \"$filter\" | __error__=\"\" [$__auto])) by (level)",
805805
"intervalFactor": 3,
806806
"legendFormat": "{{level}}",
807807
"refId": "A"
@@ -866,7 +866,7 @@
866866
},
867867
"targets": [
868868
{
869-
"expr": "{cluster=\"$cluster\", namespace=\"$namespace\", container=~\"$container\", pod=~\"$pod\"} | logfmt | level=~\"$level\" |= \"$filter\"",
869+
"expr": "{cluster=\"$cluster\", namespace=\"$namespace\", container=~\"$container\"} | logfmt | level=~\"$level\" |= \"$filter\"",
870870
"refId": "A"
871871
}
872872
],
@@ -975,26 +975,6 @@
975975
"type": "query",
976976
"useTags": false
977977
},
978-
{
979-
"allValue": null,
980-
"current": { },
981-
"datasource": "$datasource",
982-
"hide": 0,
983-
"includeAll": false,
984-
"label": null,
985-
"multi": true,
986-
"name": "pod",
987-
"options": [ ],
988-
"query": "label_values(kube_pod_container_info{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$container\"}, pod)",
989-
"refresh": 0,
990-
"regex": "",
991-
"sort": 1,
992-
"tagValuesQuery": "",
993-
"tags": [ ],
994-
"tagsQuery": "",
995-
"type": "query",
996-
"useTags": false
997-
},
998978
{
999979
"allValue": null,
1000980
"current": {

‎production/loki-mixin-compiled-ssd/dashboards/loki-operational.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3578,7 +3578,7 @@
35783578
"steppedLine": false,
35793579
"targets": [
35803580
{
3581-
"expr": "go_memstats_heap_inuse_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\".*backend.*\"}",
3581+
"expr": "go_memstats_heap_inuse_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container=~\".*backend.*\"}",
35823582
"instant": false,
35833583
"intervalFactor": 3,
35843584
"legendFormat": "{{pod}}",

‎production/loki-mixin-compiled/dashboards/loki-deletion.json‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -379,7 +379,7 @@
379379
"span": 4,
380380
"targets": [
381381
{
382-
"expr": "node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=~\"$cluster\", namespace=~\"$namespace\", pod=~\"(.*compactor|(loki.*|enterprise-logs)-backend.*|loki-single-binary)\"}",
382+
"expr": "node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=~\"$cluster\", namespace=~\"$namespace\", pod=~\"(.*compactor.*|(loki.*|enterprise-logs)-backend.*|loki-single-binary)\"}",
383383
"format": "time_series",
384384
"legendFormat": "{{pod}}",
385385
"legendLink": null
@@ -426,7 +426,7 @@
426426
"span": 4,
427427
"targets": [
428428
{
429-
"expr": "go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", pod=~\"(.*compactor|(loki.*|enterprise-logs)-backend.*|loki-single-binary)\"} / 1024 / 1024 ",
429+
"expr": "go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", pod=~\"(.*compactor.*|(loki.*|enterprise-logs)-backend.*|loki-single-binary)\"} / 1024 / 1024 ",
430430
"format": "time_series",
431431
"legendFormat": " {{pod}} ",
432432
"legendLink": null
@@ -579,7 +579,7 @@
579579
"span": 6,
580580
"targets": [
581581
{
582-
"expr": "sum(rate(loki_compactor_deleted_lines{cluster=~\"$cluster\", namespace=~\"$namespace\", pod=~\"(.*compactor|(loki.*|enterprise-logs)-backend.*|loki-single-binary)\"}[$__rate_interval])) by (user)",
582+
"expr": "sum(rate(loki_compactor_deleted_lines{cluster=~\"$cluster\", namespace=~\"$namespace\", pod=~\"(.*compactor.*|(loki.*|enterprise-logs)-backend.*|loki-single-binary)\"}[$__rate_interval])) by (user)",
583583
"format": "time_series",
584584
"legendFormat": "{{user}}",
585585
"legendLink": null
@@ -606,7 +606,7 @@
606606
"span": 6,
607607
"targets": [
608608
{
609-
"expr": "{cluster=~\"$cluster\", namespace=~\"$namespace\", pod=~\"(.*compactor|(loki.*|enterprise-logs)-backend.*|loki-single-binary)\"} |~ \"Started processing delete request|delete request for user marked as processed\" | logfmt | line_format \"{{.ts}} user={{.user}} delete_request_id={{.delete_request_id}} msg={{.msg}}\" ",
609+
"expr": "{cluster=~\"$cluster\", namespace=~\"$namespace\", pod=~\"(.*compactor.*|(loki.*|enterprise-logs)-backend.*|loki-single-binary)\"} |~ \"Started processing delete request|delete request for user marked as processed\" | logfmt | line_format \"{{.ts}} user={{.user}} delete_request_id={{.delete_request_id}} msg={{.msg}}\" ",
610610
"refId": "A"
611611
}
612612
],
@@ -619,7 +619,7 @@
619619
"span": 6,
620620
"targets": [
621621
{
622-
"expr": "{cluster=~\"$cluster\", namespace=~\"$namespace\", pod=~\"(.*compactor|(loki.*|enterprise-logs)-backend.*|loki-single-binary)\"} |~ \"delete request for user added\" | logfmt | line_format \"{{.ts}} user={{.user}} query='{{.query}}'\"",
622+
"expr": "{cluster=~\"$cluster\", namespace=~\"$namespace\", pod=~\"(.*compactor.*|(loki.*|enterprise-logs)-backend.*|loki-single-binary)\"} |~ \"delete request for user added\" | logfmt | line_format \"{{.ts}} user={{.user}} query='{{.query}}'\"",
623623
"refId": "A"
624624
}
625625
],

‎production/loki-mixin-compiled/dashboards/loki-logs.json‎

Lines changed: 9 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@
6363
"steppedLine": false,
6464
"targets": [
6565
{
66-
"expr": "sum by (pod)(go_goroutines{cluster=\"$cluster\", namespace=\"$namespace\", container=~\"$container\", pod=~\"$pod\"})",
66+
"expr": "sum by (pod)(go_goroutines{cluster=\"$cluster\", namespace=\"$namespace\", container=~\"$container\" })",
6767
"refId": "A"
6868
}
6969
],
@@ -241,7 +241,7 @@
241241
"steppedLine": false,
242242
"targets": [
243243
{
244-
"expr": "sum by (pod)(rate(container_cpu_usage_seconds_total{cluster=\"$cluster\", namespace=\"$namespace\", container=~\"$container\", pod=~\"$pod\"}[$__rate_interval]))",
244+
"expr": "sum by (pod)(rate(container_cpu_usage_seconds_total{cluster=\"$cluster\", namespace=\"$namespace\", container=~\"$container\"}[$__rate_interval]))",
245245
"refId": "A"
246246
}
247247
],
@@ -332,7 +332,7 @@
332332
"steppedLine": false,
333333
"targets": [
334334
{
335-
"expr": "sum by (pod)(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container=~\"$container\", pod=~\"$pod\"})",
335+
"expr": "sum by (pod)(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container=~\"$container\"})",
336336
"refId": "A"
337337
}
338338
],
@@ -423,7 +423,7 @@
423423
"steppedLine": false,
424424
"targets": [
425425
{
426-
"expr": "sum by (pod)(rate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\", container=~\"$container\", pod=~\"$pod\"}[$__rate_interval]))",
426+
"expr": "sum by (pod)(rate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\", container=~\"$container\"}[$__rate_interval]))",
427427
"refId": "A"
428428
}
429429
],
@@ -514,7 +514,7 @@
514514
"steppedLine": false,
515515
"targets": [
516516
{
517-
"expr": "sum by (pod)(rate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\"}[$__rate_interval]))",
517+
"expr": "sum by (pod)(rate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))",
518518
"refId": "A"
519519
}
520520
],
@@ -600,7 +600,7 @@
600600
"steppedLine": false,
601601
"targets": [
602602
{
603-
"expr": "increase(kube_pod_container_status_last_terminated_reason{cluster=\"$cluster\", namespace=\"$namespace\", container=~\"$container\", pod=~\"$pod\"}[30m]) > 0",
603+
"expr": "increase(kube_pod_container_status_last_terminated_reason{cluster=\"$cluster\", namespace=\"$namespace\", container=~\"$container\"}[30m]) > 0",
604604
"legendFormat": "{{reason}}",
605605
"refId": "A"
606606
}
@@ -692,7 +692,7 @@
692692
"steppedLine": false,
693693
"targets": [
694694
{
695-
"expr": "sum(rate(promtail_custom_bad_words_total{cluster=\"$cluster\", exported_namespace=\"$namespace\", exported_pod=~\"$pod\", container=~\"$container\"}[$__rate_interval])) by (level)",
695+
"expr": "sum(rate(promtail_custom_bad_words_total{cluster=\"$cluster\", exported_namespace=\"$namespace\", container=~\"$container\"}[$__rate_interval])) by (level)",
696696
"legendFormat": "{{level}}",
697697
"refId": "A"
698698
}
@@ -801,7 +801,7 @@
801801
"steppedLine": false,
802802
"targets": [
803803
{
804-
"expr": "sum(rate({cluster=\"$cluster\", namespace=\"$namespace\", container=~\"$container\", pod=~\"$pod\" } |logfmt| level=~\"$level\" |= \"$filter\" | __error__=\"\" [$__auto])) by (level)",
804+
"expr": "sum(rate({cluster=\"$cluster\", namespace=\"$namespace\", container=~\"$container\" } |logfmt| level=~\"$level\" |= \"$filter\" | __error__=\"\" [$__auto])) by (level)",
805805
"intervalFactor": 3,
806806
"legendFormat": "{{level}}",
807807
"refId": "A"
@@ -866,7 +866,7 @@
866866
},
867867
"targets": [
868868
{
869-
"expr": "{cluster=\"$cluster\", namespace=\"$namespace\", container=~\"$container\", pod=~\"$pod\"} | logfmt | level=~\"$level\" |= \"$filter\"",
869+
"expr": "{cluster=\"$cluster\", namespace=\"$namespace\", container=~\"$container\"} | logfmt | level=~\"$level\" |= \"$filter\"",
870870
"refId": "A"
871871
}
872872
],
@@ -975,26 +975,6 @@
975975
"type": "query",
976976
"useTags": false
977977
},
978-
{
979-
"allValue": null,
980-
"current": { },
981-
"datasource": "$datasource",
982-
"hide": 0,
983-
"includeAll": false,
984-
"label": null,
985-
"multi": true,
986-
"name": "pod",
987-
"options": [ ],
988-
"query": "label_values(kube_pod_container_info{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$container\"}, pod)",
989-
"refresh": 0,
990-
"regex": "",
991-
"sort": 1,
992-
"tagValuesQuery": "",
993-
"tags": [ ],
994-
"tagsQuery": "",
995-
"type": "query",
996-
"useTags": false
997-
},
998978
{
999979
"allValue": null,
1000980
"current": {

‎production/loki-mixin/config.libsonnet‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,9 @@
66
// Tags for dashboards.
77
tags: ['loki'],
88

9+
// The label used to differentiate between different Loki components
10+
per_component_label: 'container',
11+
912
// The label used to differentiate between different application instances (i.e. 'pod' in a kubernetes install).
1013
per_instance_label: 'pod',
1114

0 commit comments

Comments
 (0)