metrics-generator: truncate label names and values exceeding a configurable length

CHANGELOG.md

@@ -72,7 +72,7 @@ Old config will still work but will be removed in a future release. [#1735](http
 * [ENHANCEMENT] Add cli command an existing file to tempodb's current parquet schema. [#1706](https://github.com/grafana/tempo/pull/1707) (@joe-elliott)
 * [ENHANCEMENT] Add query parameter to search API for traceQL queries [#1729](https://github.com/grafana/tempo/pull/1729) (@kvrhdn)
 * [ENHANCEMENT] metrics-generator: filter out older spans before metrics are aggregated [#1612](https://github.com/grafana/tempo/pull/1612) (@ie-pham)
-* * [ENHANCEMENT] Add hedging to trace by ID lookups created by the frontend. [#1735](https://github.com/grafana/tempo/pull/1735) (@mapno)
+* [ENHANCEMENT] Add hedging to trace by ID lookups created by the frontend. [#1735](https://github.com/grafana/tempo/pull/1735) (@mapno)
     New config options and defaults:
 ```
 query_frontend:
@@ -88,6 +88,7 @@ Internal types are updated to use `scope` instead of `instrumentation_library`.
 * [ENHANCEMENT] metrics-generator: extract `status_message` field from spans [#1786](https://github.com/grafana/tempo/pull/1786), [#1794](https://github.com/grafana/tempo/pull/1794) (@stoewer)
 * [ENHANCEMENT] metrics-generator: handle collisions between user defined and default dimensions [#1794](https://github.com/grafana/tempo/pull/1794) (@stoewer)
 * [ENHANCEMENT] distributor: Log span names when `distributor.log_received_spans.include_all_attributes` is on [#1790](https://github.com/grafana/tempo/pull/1790) (@suraciii)
+* [ENHANCEMENT] metrics-generator: truncate label names and values exceeding a configurable length [#1897](https://github.com/grafana/tempo/pull/1897) (@kvrhdn)
 * [BUGFIX] Honor caching and buffering settings when finding traces by id [#1697](https://github.com/grafana/tempo/pull/1697) (@joe-elliott)
 * [BUGFIX] Correctly propagate errors from the iterator layer up through the queriers [#1723](https://github.com/grafana/tempo/pull/1723) (@joe-elliott)
 * [BUGFIX] Make multitenancy work with HTTP [#1781](https://github.com/grafana/tempo/pull/1781) (@gouthamve)

docs/tempo/website/configuration/_index.md

@@ -302,6 +302,12 @@ metrics_generator:
         # A list of labels that will be added to all generated metrics.
         [external_labels: <map>]
 
+        # The maximum length of label names. Label names exceeding this limit will be truncated.
+        [max_label_name_length: <int> | default = 1024]
+
+        # The maximum length of label values. Label values exceeding this limit will be truncated.
+        [max_label_value_length: <int> | default = 2048]
+
     # Storage and remote write configuration
     storage:
 

modules/generator/processor/servicegraphs/servicegraphs.go

@@ -60,7 +60,8 @@ func (t tooManySpansError) Error() string {
 type Processor struct {
 	Cfg Config
 
-	store store.Store
+	registry registry.Registry
+	store    store.Store
 
 	closeCh chan struct{}
 
@@ -82,7 +83,8 @@ func New(cfg Config, tenant string, registry registry.Registry, logger log.Logge
 	}
 
 	p := &Processor{
-		Cfg: cfg,
+		Cfg:      cfg,
+		registry: registry,
 
 		closeCh: make(chan struct{}, 1),
 
@@ -241,7 +243,7 @@ func (p *Processor) onComplete(e *store.Edge) {
 		labelValues = append(labelValues, e.Dimensions[dimension])
 	}
 
-	registryLabelValues := registry.NewLabelValues(labelValues)
+	registryLabelValues := p.registry.NewLabelValues(labelValues)
 
 	p.serviceGraphRequestTotal.Inc(registryLabelValues, 1)
 	if e.Failed {

modules/generator/processor/spanmetrics/spanmetrics.go

@@ -29,6 +29,8 @@ var (
 type Processor struct {
 	Cfg Config
 
+	registry registry.Registry
+
 	spanMetricsCallsTotal      registry.Counter
 	spanMetricsDurationSeconds registry.Histogram
 	spanMetricsSizeTotal       registry.Counter
@@ -46,6 +48,7 @@ func New(cfg Config, registry registry.Registry) gen.Processor {
 
 	return &Processor{
 		Cfg:                        cfg,
+		registry:                   registry,
 		spanMetricsCallsTotal:      registry.NewCounter(metricCallsTotal, labels),
 		spanMetricsDurationSeconds: registry.NewHistogram(metricDurationSeconds, labels, cfg.HistogramBuckets),
 		spanMetricsSizeTotal:       registry.NewCounter(metricSizeTotal, labels),
@@ -98,7 +101,7 @@ func (p *Processor) aggregateMetricsForSpan(svcName string, rs *v1.Resource, spa
 		labelValues = append(labelValues, value)
 	}
 
-	registryLabelValues := registry.NewLabelValues(labelValues)
+	registryLabelValues := p.registry.NewLabelValues(labelValues)
 
 	p.spanMetricsCallsTotal.Inc(registryLabelValues, 1)
 	p.spanMetricsSizeTotal.Inc(registryLabelValues, float64(span.Size()))

modules/generator/registry/config.go

@@ -17,10 +17,20 @@ type Config struct {
 
 	// ExternalLabels are added to any time series generated by this instance.
 	ExternalLabels map[string]string `yaml:"external_labels,omitempty"`
+
+	// MaxLabelNameLength configures the maximum length of label names. Label names exceeding
+	// this limit will be truncated.
+	MaxLabelNameLength int `yaml:"max_label_name_length"`
+
+	// MaxLabelValueLength configures the maximum length of label values. Label values exceeding
+	// this limit will be truncated.
+	MaxLabelValueLength int `yaml:"max_label_value_length"`
 }
 
 // RegisterFlagsAndApplyDefaults registers the flags.
 func (cfg *Config) RegisterFlagsAndApplyDefaults(prefix string, f *flag.FlagSet) {
 	cfg.CollectionInterval = 15 * time.Second
 	cfg.StaleDuration = 15 * time.Minute
+	cfg.MaxLabelNameLength = 1024
+	cfg.MaxLabelValueLength = 2048
 }

modules/generator/registry/counter_test.go

@@ -19,8 +19,8 @@ func Test_counter(t *testing.T) {
 
 	c := newCounter("my_counter", []string{"label"}, onAdd, nil)
 
-	c.Inc(NewLabelValues([]string{"value-1"}), 1.0)
-	c.Inc(NewLabelValues([]string{"value-2"}), 2.0)
+	c.Inc(newLabelValues([]string{"value-1"}), 1.0)
+	c.Inc(newLabelValues([]string{"value-2"}), 2.0)
 
 	assert.Equal(t, 2, seriesAdded)
 
@@ -31,8 +31,8 @@ func Test_counter(t *testing.T) {
 	}
 	collectMetricAndAssert(t, c, collectionTimeMs, nil, 2, expectedSamples, nil)
 
-	c.Inc(NewLabelValues([]string{"value-2"}), 2.0)
-	c.Inc(NewLabelValues([]string{"value-3"}), 3.0)
+	c.Inc(newLabelValues([]string{"value-2"}), 2.0)
+	c.Inc(newLabelValues([]string{"value-3"}), 3.0)
 
 	assert.Equal(t, 3, seriesAdded)
 
@@ -52,7 +52,7 @@ func Test_counter_invalidLabelValues(t *testing.T) {
 		c.Inc(nil, 1.0)
 	})
 	assert.Panics(t, func() {
-		c.Inc(NewLabelValues([]string{"value-1", "value-2"}), 1.0)
+		c.Inc(newLabelValues([]string{"value-1", "value-2"}), 1.0)
 	})
 }
 
@@ -68,8 +68,8 @@ func Test_counter_cantAdd(t *testing.T) {
 	// allow adding new series
 	canAdd = true
 
-	c.Inc(NewLabelValues([]string{"value-1"}), 1.0)
-	c.Inc(NewLabelValues([]string{"value-2"}), 2.0)
+	c.Inc(newLabelValues([]string{"value-1"}), 1.0)
+	c.Inc(newLabelValues([]string{"value-2"}), 2.0)
 
 	collectionTimeMs := time.Now().UnixMilli()
 	expectedSamples := []sample{
@@ -81,8 +81,8 @@ func Test_counter_cantAdd(t *testing.T) {
 	// block new series - existing series can still be updated
 	canAdd = false
 
-	c.Inc(NewLabelValues([]string{"value-2"}), 2.0)
-	c.Inc(NewLabelValues([]string{"value-3"}), 3.0)
+	c.Inc(newLabelValues([]string{"value-2"}), 2.0)
+	c.Inc(newLabelValues([]string{"value-3"}), 3.0)
 
 	collectionTimeMs = time.Now().UnixMilli()
 	expectedSamples = []sample{
@@ -102,8 +102,8 @@ func Test_counter_removeStaleSeries(t *testing.T) {
 	c := newCounter("my_counter", []string{"label"}, nil, onRemove)
 
 	timeMs := time.Now().UnixMilli()
-	c.Inc(NewLabelValues([]string{"value-1"}), 1.0)
-	c.Inc(NewLabelValues([]string{"value-2"}), 2.0)
+	c.Inc(newLabelValues([]string{"value-1"}), 1.0)
+	c.Inc(newLabelValues([]string{"value-2"}), 2.0)
 
 	c.removeStaleSeries(timeMs)
 
@@ -120,7 +120,7 @@ func Test_counter_removeStaleSeries(t *testing.T) {
 	timeMs = time.Now().UnixMilli()
 
 	// update value-2 series
-	c.Inc(NewLabelValues([]string{"value-2"}), 2.0)
+	c.Inc(newLabelValues([]string{"value-2"}), 2.0)
 
 	c.removeStaleSeries(timeMs)
 
@@ -136,8 +136,8 @@ func Test_counter_removeStaleSeries(t *testing.T) {
 func Test_counter_externalLabels(t *testing.T) {
 	c := newCounter("my_counter", []string{"label"}, nil, nil)
 
-	c.Inc(NewLabelValues([]string{"value-1"}), 1.0)
-	c.Inc(NewLabelValues([]string{"value-2"}), 2.0)
+	c.Inc(newLabelValues([]string{"value-1"}), 1.0)
+	c.Inc(newLabelValues([]string{"value-2"}), 2.0)
 
 	collectionTimeMs := time.Now().UnixMilli()
 	expectedSamples := []sample{
@@ -165,8 +165,8 @@ func Test_counter_concurrencyDataRace(t *testing.T) {
 
 	for i := 0; i < 4; i++ {
 		go accessor(func() {
-			c.Inc(NewLabelValues([]string{"value-1"}), 1.0)
-			c.Inc(NewLabelValues([]string{"value-2"}), 1.0)
+			c.Inc(newLabelValues([]string{"value-1"}), 1.0)
+			c.Inc(newLabelValues([]string{"value-2"}), 1.0)
 		})
 	}
 
@@ -177,7 +177,7 @@ func Test_counter_concurrencyDataRace(t *testing.T) {
 		for i := range s {
 			s[i] = letters[rand.Intn(len(letters))]
 		}
-		c.Inc(NewLabelValues([]string{string(s)}), 1.0)
+		c.Inc(newLabelValues([]string{string(s)}), 1.0)
 	})
 
 	go accessor(func() {
@@ -210,7 +210,7 @@ func Test_counter_concurrencyCorrectness(t *testing.T) {
 				case <-end:
 					return
 				default:
-					c.Inc(NewLabelValues([]string{"value-1"}), 1.0)
+					c.Inc(newLabelValues([]string{"value-1"}), 1.0)
 					totalCount.Inc()
 				}
 			}

modules/generator/registry/histogram_test.go

@@ -21,8 +21,8 @@ func Test_histogram(t *testing.T) {
 
 	h := newHistogram("my_histogram", []string{"label"}, []float64{1.0, 2.0}, onAdd, nil)
 
-	h.ObserveWithExemplar(NewLabelValues([]string{"value-1"}), 1.0, "trace-1")
-	h.ObserveWithExemplar(NewLabelValues([]string{"value-2"}), 1.5, "trace-2")
+	h.ObserveWithExemplar(newLabelValues([]string{"value-1"}), 1.0, "trace-1")
+	h.ObserveWithExemplar(newLabelValues([]string{"value-2"}), 1.5, "trace-2")
 
 	assert.Equal(t, 2, seriesAdded)
 
@@ -53,8 +53,8 @@ func Test_histogram(t *testing.T) {
 	}
 	collectMetricAndAssert(t, h, collectionTimeMs, nil, 10, expectedSamples, expectedExemplars)
 
-	h.ObserveWithExemplar(NewLabelValues([]string{"value-2"}), 2.5, "trace-2.2")
-	h.ObserveWithExemplar(NewLabelValues([]string{"value-3"}), 3.0, "trace-3")
+	h.ObserveWithExemplar(newLabelValues([]string{"value-2"}), 2.5, "trace-2.2")
+	h.ObserveWithExemplar(newLabelValues([]string{"value-3"}), 3.0, "trace-3")
 
 	assert.Equal(t, 3, seriesAdded)
 
@@ -98,7 +98,7 @@ func Test_histogram_invalidLabelValues(t *testing.T) {
 		h.ObserveWithExemplar(nil, 1.0, "")
 	})
 	assert.Panics(t, func() {
-		h.ObserveWithExemplar(NewLabelValues([]string{"value-1", "value-2"}), 1.0, "")
+		h.ObserveWithExemplar(newLabelValues([]string{"value-1", "value-2"}), 1.0, "")
 	})
 }
 
@@ -114,8 +114,8 @@ func Test_histogram_cantAdd(t *testing.T) {
 	// allow adding new series
 	canAdd = true
 
-	h.ObserveWithExemplar(NewLabelValues([]string{"value-1"}), 1.0, "")
-	h.ObserveWithExemplar(NewLabelValues([]string{"value-2"}), 1.5, "")
+	h.ObserveWithExemplar(newLabelValues([]string{"value-1"}), 1.0, "")
+	h.ObserveWithExemplar(newLabelValues([]string{"value-2"}), 1.5, "")
 
 	collectionTimeMs := time.Now().UnixMilli()
 	expectedSamples := []sample{
@@ -135,8 +135,8 @@ func Test_histogram_cantAdd(t *testing.T) {
 	// block new series - existing series can still be updated
 	canAdd = false
 
-	h.ObserveWithExemplar(NewLabelValues([]string{"value-2"}), 2.5, "")
-	h.ObserveWithExemplar(NewLabelValues([]string{"value-3"}), 3.0, "")
+	h.ObserveWithExemplar(newLabelValues([]string{"value-2"}), 2.5, "")
+	h.ObserveWithExemplar(newLabelValues([]string{"value-3"}), 3.0, "")
 
 	collectionTimeMs = time.Now().UnixMilli()
 	expectedSamples = []sample{
@@ -164,8 +164,8 @@ func Test_histogram_removeStaleSeries(t *testing.T) {
 	h := newHistogram("my_histogram", []string{"label"}, []float64{1.0, 2.0}, nil, onRemove)
 
 	timeMs := time.Now().UnixMilli()
-	h.ObserveWithExemplar(NewLabelValues([]string{"value-1"}), 1.0, "")
-	h.ObserveWithExemplar(NewLabelValues([]string{"value-2"}), 1.5, "")
+	h.ObserveWithExemplar(newLabelValues([]string{"value-1"}), 1.0, "")
+	h.ObserveWithExemplar(newLabelValues([]string{"value-2"}), 1.5, "")
 
 	h.removeStaleSeries(timeMs)
 
@@ -190,7 +190,7 @@ func Test_histogram_removeStaleSeries(t *testing.T) {
 	timeMs = time.Now().UnixMilli()
 
 	// update value-2 series
-	h.ObserveWithExemplar(NewLabelValues([]string{"value-2"}), 2.5, "")
+	h.ObserveWithExemplar(newLabelValues([]string{"value-2"}), 2.5, "")
 
 	h.removeStaleSeries(timeMs)
 
@@ -210,8 +210,8 @@ func Test_histogram_removeStaleSeries(t *testing.T) {
 func Test_histogram_externalLabels(t *testing.T) {
 	h := newHistogram("my_histogram", []string{"label"}, []float64{1.0, 2.0}, nil, nil)
 
-	h.ObserveWithExemplar(NewLabelValues([]string{"value-1"}), 1.0, "")
-	h.ObserveWithExemplar(NewLabelValues([]string{"value-2"}), 1.5, "")
+	h.ObserveWithExemplar(newLabelValues([]string{"value-1"}), 1.0, "")
+	h.ObserveWithExemplar(newLabelValues([]string{"value-2"}), 1.5, "")
 
 	collectionTimeMs := time.Now().UnixMilli()
 	expectedSamples := []sample{
@@ -247,8 +247,8 @@ func Test_histogram_concurrencyDataRace(t *testing.T) {
 
 	for i := 0; i < 4; i++ {
 		go accessor(func() {
-			h.ObserveWithExemplar(NewLabelValues([]string{"value-1"}), 1.0, "")
-			h.ObserveWithExemplar(NewLabelValues([]string{"value-2"}), 1.5, "")
+			h.ObserveWithExemplar(newLabelValues([]string{"value-1"}), 1.0, "")
+			h.ObserveWithExemplar(newLabelValues([]string{"value-2"}), 1.5, "")
 		})
 	}
 
@@ -259,7 +259,7 @@ func Test_histogram_concurrencyDataRace(t *testing.T) {
 		for i := range s {
 			s[i] = letters[rand.Intn(len(letters))]
 		}
-		h.ObserveWithExemplar(NewLabelValues([]string{string(s)}), 1.0, "")
+		h.ObserveWithExemplar(newLabelValues([]string{string(s)}), 1.0, "")
 	})
 
 	go accessor(func() {
@@ -292,7 +292,7 @@ func Test_histogram_concurrencyCorrectness(t *testing.T) {
 				case <-end:
 					return
 				default:
-					h.ObserveWithExemplar(NewLabelValues([]string{"value-1"}), 2.0, "")
+					h.ObserveWithExemplar(newLabelValues([]string{"value-1"}), 2.0, "")
 					totalCount.Inc()
 				}
 			}

modules/generator/registry/interface.go

@@ -2,6 +2,7 @@ package registry
 
 // Registry is a metrics store.
 type Registry interface {
+	NewLabelValues(values []string) *LabelValues
 	NewCounter(name string, labels []string) Counter
 	NewHistogram(name string, labels []string, buckets []float64) Histogram
 }
@@ -27,13 +28,18 @@ type LabelValues struct {
 	hash   uint64
 }
 
-func NewLabelValues(values []string) *LabelValues {
+func newLabelValues(values []string) *LabelValues {
 	return &LabelValues{
 		values: values,
 		hash:   0,
 	}
 }
 
+func newLabelValuesWithMax(values []string, maxLengthLabelValue int) *LabelValues {
+	truncateLength(values, maxLengthLabelValue)
+	return newLabelValues(values)
+}
+
 func (l *LabelValues) getValues() []string {
 	if l == nil {
 		return nil

modules/generator/registry/interface_test.go

@@ -0,0 +1,19 @@
+package registry
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_newLabelValuesWithMax(t *testing.T) {
+	labelValues := newLabelValuesWithMax([]string{"abc", "abcdef"}, 5)
+
+	assert.Equal(t, []string{"abc", "abcde"}, labelValues.getValues())
+}
+
+func Test_newLabelValuesWithMax_zeroLength(t *testing.T) {
+	labelValues := newLabelValuesWithMax([]string{"abc", "abcdef"}, 0)
+
+	assert.Equal(t, []string{"abc", "abcdef"}, labelValues.getValues())
+}