Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
44 commits
Select commit Hold shift + click to select a range
c917201
feat: shard instant topk queries with count min sketch.
jeschkies Sep 23, 2024
66dc3f0
Make it compile
jeschkies Sep 23, 2024
215754c
Update range
jeschkies Sep 23, 2024
436302f
Fix a few testing bugs
jeschkies Sep 24, 2024
6e53380
Calculate width and depth
jeschkies Sep 24, 2024
91133e7
Format code
jeschkies Sep 24, 2024
9a07276
Remove redundant return
jeschkies Sep 24, 2024
6dab358
Keep a limited number of labels
jeschkies Sep 27, 2024
7849984
Save one metric to string
jeschkies Sep 27, 2024
c9726cc
Convert limited label vector
jeschkies Sep 27, 2024
947a924
Rename vector
jeschkies Sep 27, 2024
33acd3b
Remove todo
jeschkies Sep 27, 2024
9ac2030
Checkin heap test
jeschkies Sep 27, 2024
acb7754
Fix vector name
jeschkies Sep 30, 2024
ad742f1
Fix marshalling
jeschkies Oct 2, 2024
6762512
Remove commented code from shard_resolver.
jeschkies Oct 2, 2024
c7c9ad4
Actually merge metrics
jeschkies Oct 2, 2024
3c3024e
Validate that approx_topk is not using groups
jeschkies Oct 2, 2024
a6f9053
support unsharded approx_topk
jeschkies Oct 3, 2024
09f7ee1
Support per tenant activation
jeschkies Oct 3, 2024
71c02fa
Make the max heap size configurable
jeschkies Oct 3, 2024
7d71609
Test shard mapper
jeschkies Oct 3, 2024
ba2ecec
Test protobuf serialization
jeschkies Oct 3, 2024
5ae7998
Remove nil group
jeschkies Oct 3, 2024
385e161
Test approx_topk integration
jeschkies Oct 3, 2024
4ccead2
Merge remote-tracking branch 'grafana/main' into karsten/ptopk
jeschkies Oct 3, 2024
ac494bd
Satisfy linter
jeschkies Oct 3, 2024
43cf741
Define separate approx topk equivalence test
jeschkies Oct 3, 2024
ecc9050
Correct error string
jeschkies Oct 3, 2024
24edc7b
Add more test cases to show how accurate the result for each item in the
cstyan Oct 4, 2024
681cd2e
Add another test case
jeschkies Oct 7, 2024
de7d83c
Remove deadline
jeschkies Oct 7, 2024
2da4270
No timeout limit
jeschkies Oct 7, 2024
00a7e1f
Test intersectiion
jeschkies Oct 7, 2024
524a10f
Remove test
jeschkies Oct 8, 2024
026b573
add hll cardinality estimate to CMS sketch and proto
cstyan Oct 10, 2024
b122237
Format imports
jeschkies Oct 19, 2024
782a148
Merge branch 'main' into karsten/ptopk
jeschkies Oct 28, 2024
0b015bf
include cardinality estimate in metrics.go logging and change CMS
cstyan Oct 29, 2024
149f990
Merge branch 'main' into karsten/ptopk
jeschkies Oct 30, 2024
09c3fe8
Fix types in tests
jeschkies Oct 30, 2024
b8d1cbb
Fix heap test
jeschkies Oct 30, 2024
a8ccd9f
Merge branch 'main' into karsten/ptopk
jeschkies Nov 1, 2024
76cf279
Update docs
jeschkies Nov 4, 2024
File filter

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Fix a few testing bugs
  • Loading branch information
jeschkies committed Oct 2, 2024
commit 436302fd873efbac9bcf8c110cc5c51209a1273d
2 changes: 1 addition & 1 deletion pkg/logql/downstream.go
Original file line number Diff line number Diff line change
Expand Up @@ -652,7 +652,7 @@ func (ev *DownstreamEvaluator) NewStepEvaluator(
}

if len(results) != 1 {
return nil, fmt.Errorf("unexpected results length for sharded quantile: got (%d), want (1)", len(results))
return nil, fmt.Errorf("unexpected results length for sharded count min sketch: got (%d), want (1)", len(results))
}

vector, ok := results[0].Data.(*CountMinSketchVector)
Expand Down
18 changes: 13 additions & 5 deletions pkg/logql/downstream_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -191,7 +191,7 @@ func TestMappingEquivalenceSketches(t *testing.T) {
}{
//{`quantile_over_time(0.70, {a=~".+"} | logfmt | unwrap value [1s]) by (a)`, 0.05, []QueryRangeType{InstantType, RangeType}},
//{`quantile_over_time(0.99, {a=~".+"} | logfmt | unwrap value [1s]) by (a)`, 0.02, []QueryRangeType{InstantType, RangeType}},
{`approx_topk(3, sum by (a) (sum_over_time ({a=~".+"} | logfmt | unwrap value [100s])))`, 0.02, []QueryRangeType{InstantType}},
{`topk(3, sum by (a) (sum_over_time ({a=~".+"} | logfmt | unwrap value [1s])))`, 0.002, []QueryRangeType{InstantType}},
} {
q := NewMockQuerier(
shards,
Expand Down Expand Up @@ -251,8 +251,8 @@ func TestMappingEquivalenceSketches(t *testing.T) {
// plus set step and interval to 0
params, err := NewLiteralParams(
tc.query,
time.Unix(0, int64(rounds+1)),
time.Unix(0, int64(rounds+1)),
time.Unix(1, 0),
time.Unix(1, 0),
0,
0,
logproto.FORWARD,
Expand All @@ -261,11 +261,18 @@ func TestMappingEquivalenceSketches(t *testing.T) {
nil,
)
require.NoError(t, err)
qry := regular.Query(params)
qry := regular.Query(params.Copy())
ctx := user.InjectOrgID(context.Background(), "fake")

strategy := NewPowerOfTwoStrategy(ConstantShards(shards))
mapper := NewShardMapper(strategy, nilShardMetrics, []string{ShardQuantileOverTime})

// TODO: use different test
// Use approximated topk
params.queryString = "approx_" + params.queryString
params.queryExpr, err = syntax.ParseExpr(params.queryString)
require.NoError(t, err)

_, _, mapped, err := mapper.Parse(params.GetExpression())
require.NoError(t, err)

Expand All @@ -276,6 +283,7 @@ func TestMappingEquivalenceSketches(t *testing.T) {

res, err := qry.Exec(ctx)
require.NoError(t, err)
require.NotEmpty(t, res.Data.(promql.Vector))

shardedRes, err := shardedQry.Exec(ctx)
require.NoError(t, err)
Expand Down Expand Up @@ -680,7 +688,7 @@ func relativeErrorVector(t *testing.T, expected, actual promql.Vector, alpha flo
require.Equal(t, expected[i].Metric, actual[i].Metric)

e[i] = expected[i].F
a[i] = expected[i].F
a[i] = actual[i].F
}
require.InEpsilonSlice(t, e, a, alpha)
}
Expand Down
1 change: 1 addition & 0 deletions pkg/logql/shardmapper.go
Original file line number Diff line number Diff line change
Expand Up @@ -335,6 +335,7 @@ func (m ShardMapper) mapVectorAggregationExpr(expr *syntax.VectorAggregationExpr
Left: sharded,
Grouping: expr.Grouping,
Operation: syntax.OpTypeTopK,
Params: expr.Params,
}, bytesPerShard, nil
default:
// this should not be reachable. If an operation is shardable it should
Expand Down
2 changes: 2 additions & 0 deletions pkg/logql/syntax/ast.go
Original file line number Diff line number Diff line change
Expand Up @@ -2285,6 +2285,8 @@ var shardableOps = map[string]bool{
OpTypeMax: true,
OpTypeMin: true,

OpTypeApproxTopK: true,

// range vector ops
OpRangeTypeAvg: true,
OpRangeTypeCount: true,
Expand Down
21 changes: 7 additions & 14 deletions pkg/logql/test_utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -227,26 +227,19 @@ type MockDownstreamer struct {

func (m MockDownstreamer) Downstreamer(_ context.Context) Downstreamer { return m }

func (m MockDownstreamer) Downstream(ctx context.Context, queries []DownstreamQuery, _ Accumulator) ([]logqlmodel.Result, error) {
results := make([]logqlmodel.Result, 0, len(queries))
for _, query := range queries {
func (m MockDownstreamer) Downstream(ctx context.Context, queries []DownstreamQuery, acc Accumulator) ([]logqlmodel.Result, error) {
for i, query := range queries {
res, err := m.Query(query.Params).Exec(ctx)
if err != nil {
return nil, err
}
results = append(results, res)
}

if matrix, ok := results[0].Data.(ProbabilisticQuantileMatrix); ok {
if len(results) == 1 {
return results, nil
}
for _, m := range results[1:] {
matrix, _ = matrix.Merge(m.Data.(ProbabilisticQuantileMatrix))
err = acc.Accumulate(ctx, res, i)
if err != nil {
return nil, err
}
return []logqlmodel.Result{{Data: matrix}}, nil
}
return results, nil

return acc.Result(), nil
}

// create nStreams of nEntries with labelNames each where each label value
Expand Down