helm: release a new helm charts to use k8s service for discover query-scheduler replicas (#9477)

**What this PR does / why we need it**:
Currently, we have a bug in our code when running Loki in SSD mode and
using the ring for query-scheduler discovery. It causes queries to not
be distributed to all the available read pods. I have explained the
issue in detail in [the PR which fixes the
code](#9471).

Since this bug causes a major query performance impact and code release
might take time, in this PR we are doing a new helm release which fixes
the issue by using the k8s service for discovering `query-scheduler`
replicas.

**Which issue(s) this PR fixes**:
Fixes #9195

Author: sandeepsukhani

docs/sources/installation/helm/reference.md

@@ -1707,6 +1707,24 @@ true
 			<td><pre lang="json">
 {}
 </pre>
+</td>
+		</tr>
+		<tr>
+			<td>loki.frontend.scheduler_address</td>
+			<td>string</td>
+			<td></td>
+			<td><pre lang="json">
+"{{ include \"loki.querySchedulerAddress\" . }}"
+</pre>
+</td>
+		</tr>
+		<tr>
+			<td>loki.frontend_worker.scheduler_address</td>
+			<td>string</td>
+			<td></td>
+			<td><pre lang="json">
+"{{ include \"loki.querySchedulerAddress\" . }}"
+</pre>
 </td>
 		</tr>
 		<tr>

production/helm/loki/CHANGELOG.md

@@ -15,6 +15,10 @@ Entries should include a reference to the pull request that introduced the chang
 
 - [CHANGE] Add clusterLabelOverride for alert label overrides.
 
+## 5.5.3
+
+- [BUGFIX] Fix issue in distribution of queries to available read pods by using k8s service for discovering query-scheduler replicas
+
 ## 5.5.2
 
 - [BUGFIX] Use $.Release.Namespace consistently

production/helm/loki/Chart.yaml

@@ -3,7 +3,7 @@ name: loki
 description: Helm chart for Grafana Loki in simple, scalable mode
 type: application
 appVersion: 2.8.2
-version: 5.5.2
+version: 5.5.3
 home: https://grafana.github.io/helm-charts
 sources:
   - https://github.com/grafana/loki

production/helm/loki/README.md

@@ -1,6 +1,6 @@
 # loki
 
-![Version: 5.5.2](https://img.shields.io/badge/Version-5.5.2-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 2.8.2](https://img.shields.io/badge/AppVersion-2.8.2-informational?style=flat-square)
+![Version: 5.5.3](https://img.shields.io/badge/Version-5.5.3-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 2.8.2](https://img.shields.io/badge/AppVersion-2.8.2-informational?style=flat-square)
 
 Helm chart for Grafana Loki in simple, scalable mode
 

production/helm/loki/templates/_helpers.tpl

@@ -762,3 +762,12 @@ enableServiceLinks: false
 {{- printf "%s" $compactorAddress }}
 {{- end }}
 
+{{/* Determine query-scheduler address */}}
+{{- define "loki.querySchedulerAddress" -}}
+{{- $isSimpleScalable := eq (include "loki.deployment.isScalable" .) "true" -}}
+{{- $schedulerAddress := ""}}
+{{- if and $isSimpleScalable (not .Values.read.legacyReadTarget ) -}}
+{{- $schedulerAddress = printf "query-scheduler-discovery.%s.svc.%s.:9095" .Release.Namespace .Values.global.clusterDomain -}}
+{{- end -}}
+{{- printf "%s" $schedulerAddress }}
+{{- end }}

production/helm/loki/templates/backend/query-scheduler-discovery.yaml

@@ -0,0 +1,26 @@
+{{- $isSimpleScalable := eq (include "loki.deployment.isScalable" .) "true" -}}
+{{- if and $isSimpleScalable (not .Values.read.legacyReadTarget ) }}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: query-scheduler-discovery
+  labels:
+    {{- include "loki.backendSelectorLabels" . | nindent 4 }}
+    prometheus.io/service-monitor: "false"
+spec:
+  type: ClusterIP
+  clusterIP: None
+  publishNotReadyAddresses: true
+  ports:
+    - name: http-metrics
+      port: 3100
+      targetPort: http-metrics
+      protocol: TCP
+    - name: grpc
+      port: 9095
+      targetPort: grpc
+      protocol: TCP
+  selector:
+    {{- include "loki.backendSelectorLabels" . | nindent 4 }}
+{{- end }}

production/helm/loki/values.yaml

@@ -207,6 +207,16 @@ loki:
     index_gateway:
       {{- tpl (. | toYaml) $ | nindent 4 }}
     {{- end }}
+
+    {{- with .Values.loki.frontend }}
+    frontend:
+      {{- tpl (. | toYaml) $ | nindent 4 }}
+    {{- end }}
+
+    {{- with .Values.loki.frontend_worker }}
+    frontend_worker:
+      {{- tpl (. | toYaml) $ | nindent 4 }}
+    {{- end }}
   # Should authentication be enabled
   auth_enabled: true
   #  Extra memberlist configuration
@@ -302,6 +312,10 @@ loki:
   # --  Optional index gateway configuration
   index_gateway:
     mode: ring
+  frontend:
+    scheduler_address: '{{ include "loki.querySchedulerAddress" . }}'
+  frontend_worker:
+    scheduler_address: '{{ include "loki.querySchedulerAddress" . }}'
 enterprise:
   # Enable enterprise features, license must be provided
   enabled: false