Add jsonpath support

Signed-off-by: Christian Haudum <christian.haudum@gmail.com>

Author: chaudum

pkg/distributor/field_detection.go

@@ -13,6 +13,8 @@ import (
 
 	"github.com/grafana/loki/v3/pkg/loghttp/push"
 	"github.com/grafana/loki/v3/pkg/logproto"
+	"github.com/grafana/loki/v3/pkg/logql/log"
+	"github.com/grafana/loki/v3/pkg/logql/log/jsonexpr"
 	"github.com/grafana/loki/v3/pkg/logql/log/logfmt"
 	"github.com/grafana/loki/v3/pkg/util/constants"
 )
@@ -141,9 +143,9 @@ func (l *FieldDetector) detectGenericFieldFromLogEntry(entry logproto.Entry, hin
 	lineBytes := unsafe.Slice(unsafe.StringData(entry.Line), len(entry.Line))
 	var v []byte
 	if isJSON(entry.Line) {
-		v = l.getValueUsingJSONParser(lineBytes, hints)
+		v = getValueUsingJSONParser(lineBytes, hints)
 	} else if isLogFmt(lineBytes) {
-		v = l.getValueUsingLogfmtParser(lineBytes, hints)
+		v = getValueUsingLogfmtParser(lineBytes, hints)
 	}
 	return string(v)
 }
@@ -152,9 +154,9 @@ func (l *FieldDetector) extractLogLevelFromLogLine(log string) string {
 	lineBytes := unsafe.Slice(unsafe.StringData(log), len(log))
 	var v []byte
 	if isJSON(log) {
-		v = l.getValueUsingJSONParser(lineBytes, l.allowedLevelLabels)
+		v = getValueUsingJSONParser(lineBytes, l.allowedLevelLabels)
 	} else if isLogFmt(lineBytes) {
-		v = l.getValueUsingLogfmtParser(lineBytes, l.allowedLevelLabels)
+		v = getValueUsingLogfmtParser(lineBytes, l.allowedLevelLabels)
 	} else {
 		return detectLevelFromLogLine(log)
 	}
@@ -179,7 +181,7 @@ func (l *FieldDetector) extractLogLevelFromLogLine(log string) string {
 	}
 }
 
-func (l *FieldDetector) getValueUsingLogfmtParser(line []byte, hints []string) []byte {
+func getValueUsingLogfmtParser(line []byte, hints []string) []byte {
 	d := logfmt.NewDecoder(line)
 	// In order to have the same behaviour as the JSON field extraction,
 	// the full line needs to be parsed to extract all possible matching fields.
@@ -201,14 +203,20 @@ func (l *FieldDetector) getValueUsingLogfmtParser(line []byte, hints []string) [
 	return res
 }
 
-func (l *FieldDetector) getValueUsingJSONParser(log []byte, hints []string) []byte {
+func getValueUsingJSONParser(line []byte, hints []string) []byte {
+	var res []byte
 	for _, allowedLabel := range hints {
-		l, _, _, err := jsonparser.Get(log, allowedLabel)
-		if err == nil {
-			return l
+		parsed, err := jsonexpr.Parse(allowedLabel, false)
+		if err != nil {
+			continue
 		}
+		l, _, _, err := jsonparser.Get(line, log.JSONPathsToStrings(parsed)...)
+		if err != nil {
+			continue
+		}
+		return l
 	}
-	return nil
+	return res
 }
 
 func isLogFmt(line []byte) bool {

pkg/distributor/field_detection_test.go

@@ -468,8 +468,9 @@ func Test_DetectGenericFields(t *testing.T) {
 	detector := newFieldDetector(
 		validationContext{
 			discoverGenericFields: map[string][]string{
-				"trace_id": []string{"trace_id"},
-				"org_id":   []string{"org_id", "user_id", "tenant_id"},
+				"trace_id":   []string{"trace_id"},
+				"org_id":     []string{"org_id", "user_id", "tenant_id"},
+				"product_id": []string{"product.id"}, // jsonpath
 			},
 			allowStructuredMetadata: true,
 		})
@@ -577,6 +578,19 @@ func Test_DetectGenericFields(t *testing.T) {
 				{Name: "org_id", Value: "fake_b"}, // first field from configuration that matches takes precedence
 			},
 		},
+		{
+			name: "logline matches jsonpath",
+			labels: labels.Labels{
+				{Name: "env", Value: "prod"},
+			},
+			entry: push.Entry{
+				Line:               `{"product": {"details": "product details", "id": "P2024/01"}}`,
+				StructuredMetadata: push.LabelsAdapter{},
+			},
+			expected: push.LabelsAdapter{
+				{Name: "product_id", Value: "P2024/01"},
+			},
+		},
 	} {
 		t.Run(tc.name, func(t *testing.T) {
 			extracted := push.LabelsAdapter{}

pkg/logql/log/parser.go

@@ -571,7 +571,7 @@ func NewJSONExpressionParser(expressions []LabelExtractionExpr) (*JSONExpression
 		}
 
 		ids = append(ids, exp.Identifier)
-		paths = append(paths, pathsToString(path))
+		paths = append(paths, JSONPathsToStrings(path))
 	}
 
 	return &JSONExpressionParser{
@@ -581,7 +581,7 @@ func NewJSONExpressionParser(expressions []LabelExtractionExpr) (*JSONExpression
 	}, nil
 }
 
-func pathsToString(paths []interface{}) []string {
+func JSONPathsToStrings(paths []interface{}) []string {
 	stingPaths := make([]string, 0, len(paths))
 	for _, p := range paths {
 		switch v := p.(type) {