Skip to content

Commit f6a6a4f

Browse files
trevorwhitneytrevorwhitney
authored
feat: add case-insensitive regex optimization (#20578)
1 parent a28d8b4 commit f6a6a4f

19 files changed

Lines changed: 1083 additions & 205 deletions

‎pkg/engine/internal/executor/dataobjscan_predicate.go‎

Lines changed: 91 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import (
1010
"github.com/apache/arrow-go/v18/arrow/scalar"
1111

1212
"github.com/grafana/loki/v3/pkg/dataobj/sections/logs"
13+
"github.com/grafana/loki/v3/pkg/engine/internal/executor/matchutil"
1314
"github.com/grafana/loki/v3/pkg/engine/internal/planner/physical"
1415
"github.com/grafana/loki/v3/pkg/engine/internal/types"
1516
)
@@ -86,18 +87,22 @@ func buildLogsUnaryPredicate(expr physical.UnaryExpression, columns []*logs.Colu
8687
}
8788

8889
var comparisonBinaryOps = map[types.BinaryOp]struct{}{
89-
types.BinaryOpEq: {},
90-
types.BinaryOpNeq: {},
91-
types.BinaryOpGt: {},
92-
types.BinaryOpGte: {},
93-
types.BinaryOpLt: {},
94-
types.BinaryOpLte: {},
95-
types.BinaryOpMatchSubstr: {},
96-
types.BinaryOpNotMatchSubstr: {},
97-
types.BinaryOpMatchRe: {},
98-
types.BinaryOpNotMatchRe: {},
99-
types.BinaryOpMatchPattern: {},
100-
types.BinaryOpNotMatchPattern: {},
90+
types.BinaryOpEq: {},
91+
types.BinaryOpNeq: {},
92+
types.BinaryOpGt: {},
93+
types.BinaryOpGte: {},
94+
types.BinaryOpLt: {},
95+
types.BinaryOpLte: {},
96+
types.BinaryOpMatchSubstr: {},
97+
types.BinaryOpNotMatchSubstr: {},
98+
types.BinaryOpMatchRe: {},
99+
types.BinaryOpNotMatchRe: {},
100+
types.BinaryOpMatchPattern: {},
101+
types.BinaryOpNotMatchPattern: {},
102+
types.BinaryOpEqCaseInsensitive: {},
103+
types.BinaryOpNotEqCaseInsensitive: {},
104+
types.BinaryOpMatchSubstrCaseInsensitive: {},
105+
types.BinaryOpNotMatchSubstrCaseInsensitive: {},
101106
}
102107

103108
func buildLogsBinaryPredicate(expr physical.BinaryExpression, columns []*logs.Column) (logs.Predicate, error) {
@@ -228,6 +233,38 @@ func buildLogsComparison(expr *physical.BinaryExpr, columns []*logs.Column) (log
228233
return logs.TruePredicate{}, nil // Not match operations against a non-existent column will always pass.
229234
}
230235
return buildLogsMatch(col, expr.Op, s)
236+
237+
case types.BinaryOpEqCaseInsensitive:
238+
if col == nil && s.IsValid() {
239+
return logs.FalsePredicate{}, nil // Column(NULL) == non-null: always fails
240+
} else if col == nil && !s.IsValid() {
241+
return logs.TruePredicate{}, nil // Column(NULL) == NULL: always passes
242+
}
243+
return buildLogsCaseInsensitiveMatch(col, expr.Op, s)
244+
245+
case types.BinaryOpNotEqCaseInsensitive:
246+
if col == nil && s.IsValid() {
247+
return logs.TruePredicate{}, nil // Column(NULL) != non-null: always passes
248+
} else if col == nil && !s.IsValid() {
249+
return logs.FalsePredicate{}, nil // Column(NULL) != NULL: always fails
250+
}
251+
inner, err := buildLogsCaseInsensitiveMatch(col, types.BinaryOpEqCaseInsensitive, s)
252+
if err != nil {
253+
return nil, err
254+
}
255+
return logs.NotPredicate{Inner: inner}, nil
256+
257+
case types.BinaryOpMatchSubstrCaseInsensitive:
258+
if col == nil {
259+
return logs.FalsePredicate{}, nil // Match operations against a non-existent column will always fail.
260+
}
261+
return buildLogsCaseInsensitiveMatch(col, expr.Op, s)
262+
263+
case types.BinaryOpNotMatchSubstrCaseInsensitive:
264+
if col == nil {
265+
return logs.TruePredicate{}, nil // Not match operations against a non-existent column will always pass.
266+
}
267+
return buildLogsCaseInsensitiveMatch(col, expr.Op, s)
231268
}
232269

233270
return nil, fmt.Errorf("unsupported binary operator %s in logs predicate", expr.Op)
@@ -371,3 +408,45 @@ func getBytes(value scalar.Scalar) []byte {
371408

372409
return nil
373410
}
411+
412+
func buildLogsCaseInsensitiveMatch(col *logs.Column, op types.BinaryOp, value scalar.Scalar) (logs.Predicate, error) {
413+
// All case-insensitive match operations require the value to be a string or binary.
414+
var find []byte
415+
416+
switch value := value.(type) {
417+
case *scalar.Binary:
418+
find = value.Data()
419+
case *scalar.String:
420+
find = value.Data()
421+
default:
422+
return nil, fmt.Errorf("unsupported scalar type %T for op %s, expected binary or string", value, op)
423+
}
424+
425+
switch op {
426+
case types.BinaryOpEqCaseInsensitive:
427+
return logs.FuncPredicate{
428+
Column: col,
429+
Keep: func(_ *logs.Column, value scalar.Scalar) bool {
430+
return matchutil.EqualUpper(getBytes(value), find)
431+
},
432+
}, nil
433+
434+
case types.BinaryOpMatchSubstrCaseInsensitive:
435+
return logs.FuncPredicate{
436+
Column: col,
437+
Keep: func(_ *logs.Column, value scalar.Scalar) bool {
438+
return matchutil.ContainsUpper(getBytes(value), find)
439+
},
440+
}, nil
441+
442+
case types.BinaryOpNotMatchSubstrCaseInsensitive:
443+
return logs.FuncPredicate{
444+
Column: col,
445+
Keep: func(_ *logs.Column, value scalar.Scalar) bool {
446+
return !matchutil.ContainsUpper(getBytes(value), find)
447+
},
448+
}, nil
449+
}
450+
451+
return nil, fmt.Errorf("unrecognized case-insensitive match operation %s", op)
452+
}

‎pkg/engine/internal/executor/dataobjscan_predicate_test.go‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -576,3 +576,103 @@ func columnRef(ty types.ColumnType, column string) *physical.ColumnExpr {
576576
},
577577
}
578578
}
579+
580+
func Test_buildLogsPredicate_CaseInsensitive_Substring(t *testing.T) {
581+
messageColumn := &logs.Column{Type: logs.ColumnTypeMessage}
582+
columns := []*logs.Column{messageColumn}
583+
584+
// Build a predicate for case-insensitive substring match
585+
expr := &physical.BinaryExpr{
586+
Op: types.BinaryOpMatchSubstrCaseInsensitive,
587+
Left: &physical.ColumnExpr{
588+
Ref: types.ColumnRef{
589+
Type: types.ColumnTypeBuiltin,
590+
Column: types.ColumnNameBuiltinMessage,
591+
},
592+
},
593+
Right: physical.NewLiteral("DURATION"), // Uppercased pattern
594+
}
595+
596+
predicate, err := buildLogsPredicate(expr, columns)
597+
require.NoError(t, err, "buildLogsPredicate should support case-insensitive operators")
598+
599+
funcPred, ok := predicate.(logs.FuncPredicate)
600+
require.True(t, ok, "expected logs.FuncPredicate")
601+
602+
// Test that it matches various cases
603+
testCases := []struct {
604+
name string
605+
input string
606+
expected bool
607+
}{
608+
{"lowercase", "duration", true},
609+
{"uppercase", "DURATION", true},
610+
{"mixed case", "Duration", true},
611+
{"mixed case 2", "DuRaTiOn", true},
612+
{"in sentence lowercase", "request duration was 100ms", true},
613+
{"in sentence uppercase", "REQUEST DURATION WAS 100MS", true},
614+
{"in sentence mixed", "Request Duration was 100ms", true},
615+
{"no match", "timeout occurred", false},
616+
{"partial match", "dura", false}, // "DURATION" is not in "dura"
617+
}
618+
619+
for _, tc := range testCases {
620+
t.Run(tc.name, func(t *testing.T) {
621+
buf := memory.NewBufferBytes([]byte(tc.input))
622+
inputScalar := scalar.NewBinaryScalar(buf, arrow.BinaryTypes.Binary)
623+
624+
result := funcPred.Keep(messageColumn, inputScalar)
625+
require.Equal(t, tc.expected, result,
626+
"case-insensitive match should correctly match %q", tc.input)
627+
})
628+
}
629+
}
630+
631+
func Test_buildLogsPredicate_CaseInsensitive_Equality(t *testing.T) {
632+
messageColumn := &logs.Column{Type: logs.ColumnTypeMessage}
633+
columns := []*logs.Column{messageColumn}
634+
635+
// Build a predicate for case-insensitive equality
636+
expr := &physical.BinaryExpr{
637+
Op: types.BinaryOpEqCaseInsensitive,
638+
Left: &physical.ColumnExpr{
639+
Ref: types.ColumnRef{
640+
Type: types.ColumnTypeBuiltin,
641+
Column: types.ColumnNameBuiltinMessage,
642+
},
643+
},
644+
Right: physical.NewLiteral("ERROR"), // Uppercased pattern
645+
}
646+
647+
predicate, err := buildLogsPredicate(expr, columns)
648+
require.NoError(t, err, "buildLogsPredicate should support case-insensitive operators")
649+
650+
funcPred, ok := predicate.(logs.FuncPredicate)
651+
require.True(t, ok, "expected logs.FuncPredicate")
652+
653+
// Test exact equality with various cases
654+
testCases := []struct {
655+
name string
656+
input string
657+
expected bool
658+
}{
659+
{"exact lowercase", "error", true},
660+
{"exact uppercase", "ERROR", true},
661+
{"exact mixed case", "Error", true},
662+
{"exact mixed case 2", "ErRoR", true},
663+
{"substring should not match", "error occurred", false},
664+
{"prefix should not match", "error!", false},
665+
{"different word", "warning", false},
666+
}
667+
668+
for _, tc := range testCases {
669+
t.Run(tc.name, func(t *testing.T) {
670+
buf := memory.NewBufferBytes([]byte(tc.input))
671+
inputScalar := scalar.NewBinaryScalar(buf, arrow.BinaryTypes.Binary)
672+
673+
result := funcPred.Keep(messageColumn, inputScalar)
674+
require.Equal(t, tc.expected, result,
675+
"case-insensitive equality should correctly match %q", tc.input)
676+
})
677+
}
678+
}

‎pkg/engine/internal/executor/functions.go‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import (
1111
"github.com/apache/arrow-go/v18/arrow/memory"
1212

1313
"github.com/grafana/loki/v3/pkg/engine/internal/errors"
14+
"github.com/grafana/loki/v3/pkg/engine/internal/executor/matchutil"
1415
"github.com/grafana/loki/v3/pkg/engine/internal/types"
1516
)
1617

@@ -111,6 +112,23 @@ func init() {
111112
return !reg.Match([]byte(a)), nil
112113
}})
113114

115+
// Functions for [types.BinaryOpEqCaseInsensitive]
116+
binaryFunctions.register(types.BinaryOpEqCaseInsensitive, arrow.BinaryTypes.String, &genericBoolFunction[*array.String, string]{eval: func(a, b string) (bool, error) {
117+
return matchutil.EqualUpper([]byte(a), []byte(b)), nil
118+
}})
119+
// Functions for [types.BinaryOpNotEqCaseInsensitive]
120+
binaryFunctions.register(types.BinaryOpNotEqCaseInsensitive, arrow.BinaryTypes.String, &genericBoolFunction[*array.String, string]{eval: func(a, b string) (bool, error) {
121+
return !matchutil.EqualUpper([]byte(a), []byte(b)), nil
122+
}})
123+
// Functions for [types.BinaryOpMatchSubstrCaseInsensitive]
124+
binaryFunctions.register(types.BinaryOpMatchSubstrCaseInsensitive, arrow.BinaryTypes.String, &genericBoolFunction[*array.String, string]{eval: func(a, b string) (bool, error) {
125+
return matchutil.ContainsUpper([]byte(a), []byte(b)), nil
126+
}})
127+
// Functions for [types.BinaryOpNotMatchSubstrCaseInsensitive]
128+
binaryFunctions.register(types.BinaryOpNotMatchSubstrCaseInsensitive, arrow.BinaryTypes.String, &genericBoolFunction[*array.String, string]{eval: func(a, b string) (bool, error) {
129+
return !matchutil.ContainsUpper([]byte(a), []byte(b)), nil
130+
}})
131+
114132
// Functions for [types.UnaryOpNot]
115133
unaryFunctions.register(types.UnaryOpNot, arrow.FixedWidthTypes.Boolean, UnaryFunc(func(input arrow.Array) (arrow.Array, error) {
116134
arr, ok := input.(*array.Boolean)

‎pkg/engine/internal/executor/functions_test.go‎

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -661,6 +661,124 @@ func TestStringMatchingFunctions(t *testing.T) {
661661
}
662662
}
663663

664+
func TestCaseInsensitiveStringFunctions(t *testing.T) {
665+
tests := []struct {
666+
name string
667+
op types.BinaryOp
668+
lhs []string
669+
rhs []string
670+
expected []bool
671+
}{
672+
{
673+
name: "case insensitive equality - lowercase pattern",
674+
op: types.BinaryOpEqCaseInsensitive,
675+
lhs: []string{"Hello", "WORLD", "TeSt", ""},
676+
rhs: []string{"HELLO", "WORLD", "TEST", ""},
677+
expected: []bool{true, true, true, true},
678+
},
679+
{
680+
name: "case insensitive equality - mixed case pattern",
681+
op: types.BinaryOpEqCaseInsensitive,
682+
lhs: []string{"ERROR", "Warning", "info"},
683+
rhs: []string{"ERROR", "WARNING", "INFO"},
684+
expected: []bool{true, true, true},
685+
},
686+
{
687+
name: "case insensitive equality - no match",
688+
op: types.BinaryOpEqCaseInsensitive,
689+
lhs: []string{"HELLO", "WORLD", "TEST"},
690+
rhs: []string{"GOODBYE", "EARTH", "EXAM"},
691+
expected: []bool{false, false, false},
692+
},
693+
{
694+
name: "case insensitive inequality - match",
695+
op: types.BinaryOpNotEqCaseInsensitive,
696+
lhs: []string{"Hello", "WORLD"},
697+
rhs: []string{"HELLO", "WORLD"},
698+
expected: []bool{false, false},
699+
},
700+
{
701+
name: "case insensitive inequality - no match",
702+
op: types.BinaryOpNotEqCaseInsensitive,
703+
lhs: []string{"HELLO", "WORLD"},
704+
rhs: []string{"GOODBYE", "EARTH"},
705+
expected: []bool{true, true},
706+
},
707+
{
708+
name: "case insensitive substring - basic",
709+
op: types.BinaryOpMatchSubstrCaseInsensitive,
710+
lhs: []string{"Hello World", "TEST STRING", "FooBar"},
711+
rhs: []string{"WORLD", "STRING", "FOOBAR"},
712+
expected: []bool{true, true, true},
713+
},
714+
{
715+
name: "case insensitive substring - mixed case lhs and rhs",
716+
op: types.BinaryOpMatchSubstrCaseInsensitive,
717+
lhs: []string{"Error Occurred", "WARNING message", "Info: log"},
718+
rhs: []string{"ERROR", "WARNING", "INFO"},
719+
expected: []bool{true, true, true},
720+
},
721+
{
722+
name: "case insensitive substring - no match",
723+
op: types.BinaryOpMatchSubstrCaseInsensitive,
724+
lhs: []string{"hello world", "test string"},
725+
rhs: []string{"GOODBYE", "EXAM"},
726+
expected: []bool{false, false},
727+
},
728+
{
729+
name: "case insensitive substring - empty string",
730+
op: types.BinaryOpMatchSubstrCaseInsensitive,
731+
lhs: []string{"anything", ""},
732+
rhs: []string{"", ""},
733+
expected: []bool{true, true},
734+
},
735+
{
736+
name: "case insensitive not substring - match",
737+
op: types.BinaryOpNotMatchSubstrCaseInsensitive,
738+
lhs: []string{"Hello World", "TEST"},
739+
rhs: []string{"WORLD", "TEST"},
740+
expected: []bool{false, false},
741+
},
742+
{
743+
name: "case insensitive not substring - no match",
744+
op: types.BinaryOpNotMatchSubstrCaseInsensitive,
745+
lhs: []string{"hello world", "test string"},
746+
rhs: []string{"GOODBYE", "EXAM"},
747+
expected: []bool{true, true},
748+
},
749+
{
750+
name: "case insensitive with unicode",
751+
op: types.BinaryOpMatchSubstrCaseInsensitive,
752+
lhs: []string{"Error in MÜNCHEN", "ΑΒΓΔ test"},
753+
rhs: []string{"MÜNCHEN", "ΑΒΓΔ"},
754+
expected: []bool{true, true},
755+
},
756+
{
757+
name: "case insensitive with numbers and special chars",
758+
op: types.BinaryOpEqCaseInsensitive,
759+
lhs: []string{"Error123!", "Test@456"},
760+
rhs: []string{"ERROR123!", "TEST@456"},
761+
expected: []bool{true, true},
762+
},
763+
}
764+
765+
for _, tt := range tests {
766+
t.Run(tt.name, func(t *testing.T) {
767+
lhsArray := createStringArray(tt.lhs, nil)
768+
rhsArray := createStringArray(tt.rhs, nil)
769+
770+
fn, err := binaryFunctions.GetForSignature(tt.op, arrow.BinaryTypes.String)
771+
require.NoError(t, err)
772+
773+
result, err := fn.Evaluate(lhsArray, rhsArray, false, false)
774+
require.NoError(t, err)
775+
776+
actual, _ := extractBoolValues(result)
777+
assert.Equal(t, tt.expected, actual)
778+
})
779+
}
780+
}
781+
664782
func TestCompileRegexMatchFunctions(t *testing.T) {
665783
tests := []struct {
666784
name string

0 commit comments

Comments
 (0)