Skip to content

Commit 8958eb9

Browse files
ashwanthgoliJoaoBraveCoding
ashwanthgoli
and
JoaoBraveCoding
authored
feat(thanos): add support for named stores (#14638)
Co-authored-by: Joao Marcal <jmarcal@redhat.com>
1 parent 13ea254 commit 8958eb9

File tree

9 files changed

+520
-52
lines changed

9 files changed

+520
-52
lines changed

‎pkg/loki/config_wrapper.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -567,9 +567,9 @@ func applyStorageConfig(cfg, defaults *ConfigWrapper) error {
567567
}
568568
}
569569

570-
if !reflect.DeepEqual(cfg.Common.Storage.ObjectStore, defaults.StorageConfig.ObjectStore) {
570+
if !reflect.DeepEqual(cfg.Common.Storage.ObjectStore, defaults.StorageConfig.ObjectStore.Config) {
571571
applyConfig = func(r *ConfigWrapper) {
572-
r.StorageConfig.ObjectStore = r.Common.Storage.ObjectStore
572+
r.StorageConfig.ObjectStore.Config = r.Common.Storage.ObjectStore
573573
}
574574
}
575575

‎pkg/loki/config_wrapper_test.go‎

Lines changed: 134 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,10 @@ import (
1616

1717
"github.com/grafana/loki/v3/pkg/distributor"
1818
"github.com/grafana/loki/v3/pkg/loki/common"
19+
azurebucket "github.com/grafana/loki/v3/pkg/storage/bucket/azure"
20+
"github.com/grafana/loki/v3/pkg/storage/bucket/filesystem"
21+
"github.com/grafana/loki/v3/pkg/storage/bucket/gcs"
22+
"github.com/grafana/loki/v3/pkg/storage/bucket/s3"
1923
"github.com/grafana/loki/v3/pkg/storage/bucket/swift"
2024
"github.com/grafana/loki/v3/pkg/storage/chunk/client/alibaba"
2125
"github.com/grafana/loki/v3/pkg/storage/chunk/client/aws"
@@ -842,6 +846,48 @@ storage_config:
842846
assert.Equal(t, "789abc", config.StorageConfig.NamedStores.AWS["store-2"].S3Config.SecretAccessKey.String())
843847
})
844848

849+
t.Run("named storage config (thanos) provided via config file is preserved", func(t *testing.T) {
850+
namedStoresConfig := `common:
851+
storage:
852+
object_store:
853+
s3:
854+
endpoint: s3://common-bucket
855+
region: us-east1
856+
access_key_id: abc123
857+
secret_access_key: def789
858+
storage_config:
859+
object_store:
860+
named_stores:
861+
s3:
862+
store-1:
863+
endpoint: s3://foo-bucket
864+
region: us-west1
865+
access_key_id: 123abc
866+
secret_access_key: 789def
867+
store-2:
868+
endpoint: s3://bar-bucket
869+
region: us-west2
870+
access_key_id: 456def
871+
secret_access_key: 789abc`
872+
config, _ := testContext(namedStoresConfig, nil)
873+
874+
// should be set by common config
875+
assert.Equal(t, "s3://common-bucket", config.StorageConfig.ObjectStore.S3.Endpoint)
876+
assert.Equal(t, "us-east1", config.StorageConfig.ObjectStore.S3.Region)
877+
assert.Equal(t, "abc123", config.StorageConfig.ObjectStore.S3.AccessKeyID)
878+
assert.Equal(t, "def789", config.StorageConfig.ObjectStore.S3.SecretAccessKey.String())
879+
880+
assert.Equal(t, "s3://foo-bucket", config.StorageConfig.ObjectStore.NamedStores.S3["store-1"].Endpoint)
881+
assert.Equal(t, "us-west1", config.StorageConfig.ObjectStore.NamedStores.S3["store-1"].Region)
882+
assert.Equal(t, "123abc", config.StorageConfig.ObjectStore.NamedStores.S3["store-1"].AccessKeyID)
883+
assert.Equal(t, "789def", config.StorageConfig.ObjectStore.NamedStores.S3["store-1"].SecretAccessKey.String())
884+
885+
assert.Equal(t, "s3://bar-bucket", config.StorageConfig.ObjectStore.NamedStores.S3["store-2"].Endpoint)
886+
assert.Equal(t, "us-west2", config.StorageConfig.ObjectStore.NamedStores.S3["store-2"].Region)
887+
assert.Equal(t, "456def", config.StorageConfig.ObjectStore.NamedStores.S3["store-2"].AccessKeyID)
888+
assert.Equal(t, "789abc", config.StorageConfig.ObjectStore.NamedStores.S3["store-2"].SecretAccessKey.String())
889+
})
890+
845891
t.Run("partial ruler config from file is honored for overriding things like bucket names", func(t *testing.T) {
846892
specificRulerConfig := `common:
847893
storage:
@@ -2280,3 +2326,91 @@ func TestNamedStores_applyDefaults(t *testing.T) {
22802326
assert.Equal(t, expected, (alibaba.OssConfig)(nsCfg.AlibabaCloud["store-8"]))
22812327
})
22822328
}
2329+
2330+
func TestBucketNamedStores_applyDefaults(t *testing.T) {
2331+
namedStoresConfig := `storage_config:
2332+
object_store:
2333+
named_stores:
2334+
s3:
2335+
store-1:
2336+
endpoint: s3.test
2337+
bucket_name: foobar
2338+
dualstack_enabled: false
2339+
azure:
2340+
store-2:
2341+
account_name: foo
2342+
container_name: bar
2343+
max_retries: 3
2344+
gcs:
2345+
store-3:
2346+
bucket_name: foobar
2347+
filesystem:
2348+
store-4:
2349+
dir: foobar
2350+
swift:
2351+
store-5:
2352+
container_name: foobar
2353+
request_timeout: 30s
2354+
`
2355+
// make goconst happy
2356+
bucketName := "foobar"
2357+
2358+
config, defaults, err := configWrapperFromYAML(t, namedStoresConfig, nil)
2359+
require.NoError(t, err)
2360+
2361+
nsCfg := config.StorageConfig.ObjectStore.NamedStores
2362+
2363+
t.Run("s3", func(t *testing.T) {
2364+
assert.Len(t, nsCfg.S3, 1)
2365+
2366+
// expect the defaults to be set on named store config
2367+
expected := defaults.StorageConfig.ObjectStore.S3
2368+
expected.BucketName = bucketName
2369+
expected.Endpoint = "s3.test"
2370+
// override defaults
2371+
expected.DualstackEnabled = false
2372+
2373+
assert.Equal(t, expected, (s3.Config)(nsCfg.S3["store-1"]))
2374+
})
2375+
2376+
t.Run("azure", func(t *testing.T) {
2377+
assert.Len(t, nsCfg.Azure, 1)
2378+
2379+
expected := defaults.StorageConfig.ObjectStore.Azure
2380+
expected.StorageAccountName = "foo"
2381+
expected.ContainerName = "bar"
2382+
// overrides defaults
2383+
expected.MaxRetries = 3
2384+
2385+
assert.Equal(t, expected, (azurebucket.Config)(nsCfg.Azure["store-2"]))
2386+
})
2387+
2388+
t.Run("gcs", func(t *testing.T) {
2389+
assert.Len(t, nsCfg.GCS, 1)
2390+
2391+
expected := defaults.StorageConfig.ObjectStore.GCS
2392+
expected.BucketName = bucketName
2393+
2394+
assert.Equal(t, expected, (gcs.Config)(nsCfg.GCS["store-3"]))
2395+
})
2396+
2397+
t.Run("filesystem", func(t *testing.T) {
2398+
assert.Len(t, nsCfg.Filesystem, 1)
2399+
2400+
expected := defaults.StorageConfig.ObjectStore.Filesystem
2401+
expected.Directory = bucketName
2402+
2403+
assert.Equal(t, expected, (filesystem.Config)(nsCfg.Filesystem["store-4"]))
2404+
})
2405+
2406+
t.Run("swift", func(t *testing.T) {
2407+
assert.Len(t, nsCfg.Swift, 1)
2408+
2409+
expected := defaults.StorageConfig.ObjectStore.Swift
2410+
expected.ContainerName = bucketName
2411+
// override defaults
2412+
expected.RequestTimeout = 30 * time.Second
2413+
2414+
assert.Equal(t, expected, (swift.Config)(nsCfg.Swift["store-5"]))
2415+
})
2416+
}

‎pkg/loki/validation.go‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"fmt"
55
"strings"
66

7+
"github.com/grafana/loki/v3/pkg/storage/bucket"
78
"github.com/grafana/loki/v3/pkg/storage/chunk/cache"
89
"github.com/grafana/loki/v3/pkg/storage/config"
910
"github.com/grafana/loki/v3/pkg/storage/types"
@@ -97,7 +98,11 @@ func validateSchemaValues(c *Config) []error {
9798
errs = append(errs, fmt.Errorf("unrecognized `store` (index) type `%s`, choose one of: %s", cfg.IndexType, strings.Join(types.SupportedIndexTypes, ", ")))
9899
}
99100

100-
if !util.StringsContain(types.TestingStorageTypes, cfg.ObjectType) &&
101+
if c.StorageConfig.UseThanosObjstore {
102+
if !util.StringsContain(bucket.SupportedBackends, cfg.ObjectType) && !c.StorageConfig.ObjectStore.NamedStores.Exists(cfg.ObjectType) {
103+
errs = append(errs, fmt.Errorf("unrecognized `object_store` type `%s`, which also does not match any named_stores. Choose one of: %s. Or choose a named_store", cfg.ObjectType, strings.Join(bucket.SupportedBackends, ", ")))
104+
}
105+
} else if !util.StringsContain(types.TestingStorageTypes, cfg.ObjectType) &&
101106
!util.StringsContain(types.SupportedStorageTypes, cfg.ObjectType) &&
102107
!util.StringsContain(types.DeprecatedStorageTypes, cfg.ObjectType) {
103108
if !c.StorageConfig.NamedStores.Exists(cfg.ObjectType) {

‎pkg/storage/bucket/client.go‎

Lines changed: 28 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -73,8 +73,8 @@ func init() {
7373
metrics = objstore.BucketMetrics(prometheus.WrapRegistererWithPrefix("loki_", prometheus.DefaultRegisterer), "")
7474
}
7575

76-
// StorageBackendConfig holds configuration for accessing long-term storage.
77-
type StorageBackendConfig struct {
76+
// Config holds configuration for accessing long-term storage.
77+
type Config struct {
7878
// Backends
7979
S3 s3.Config `yaml:"s3"`
8080
GCS gcs.Config `yaml:"gcs"`
@@ -84,76 +84,68 @@ type StorageBackendConfig struct {
8484
Alibaba oss.Config `yaml:"alibaba"`
8585
BOS bos.Config `yaml:"bos"`
8686

87+
StoragePrefix string `yaml:"storage_prefix"`
88+
8789
// Used to inject additional backends into the config. Allows for this config to
8890
// be embedded in multiple contexts and support non-object storage based backends.
8991
ExtraBackends []string `yaml:"-"`
92+
93+
// Not used internally, meant to allow callers to wrap Buckets
94+
// created using this config
95+
Middlewares []func(objstore.InstrumentedBucket) (objstore.InstrumentedBucket, error) `yaml:"-"`
9096
}
9197

9298
// Returns the SupportedBackends for the package and any custom backends injected into the config.
93-
func (cfg *StorageBackendConfig) SupportedBackends() []string {
99+
func (cfg *Config) SupportedBackends() []string {
94100
return append(SupportedBackends, cfg.ExtraBackends...)
95101
}
96102

97103
// RegisterFlags registers the backend storage config.
98-
func (cfg *StorageBackendConfig) RegisterFlags(f *flag.FlagSet) {
104+
func (cfg *Config) RegisterFlags(f *flag.FlagSet) {
99105
cfg.RegisterFlagsWithPrefix("", f)
100106
}
101107

102-
func (cfg *StorageBackendConfig) RegisterFlagsWithPrefixAndDefaultDirectory(prefix, dir string, f *flag.FlagSet) {
108+
func (cfg *Config) RegisterFlagsWithPrefixAndDefaultDirectory(prefix, dir string, f *flag.FlagSet) {
103109
cfg.GCS.RegisterFlagsWithPrefix(prefix, f)
104110
cfg.S3.RegisterFlagsWithPrefix(prefix, f)
105111
cfg.Azure.RegisterFlagsWithPrefix(prefix, f)
106112
cfg.Swift.RegisterFlagsWithPrefix(prefix, f)
107113
cfg.Filesystem.RegisterFlagsWithPrefixAndDefaultDirectory(prefix, dir, f)
108114
cfg.Alibaba.RegisterFlagsWithPrefix(prefix, f)
109115
cfg.BOS.RegisterFlagsWithPrefix(prefix, f)
116+
f.StringVar(&cfg.StoragePrefix, prefix+"storage-prefix", "", "Prefix for all objects stored in the backend storage. For simplicity, it may only contain digits and English alphabet letters.")
110117
}
111118

112-
func (cfg *StorageBackendConfig) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) {
119+
func (cfg *Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) {
113120
cfg.RegisterFlagsWithPrefixAndDefaultDirectory(prefix, "", f)
114121
}
115122

116-
func (cfg *StorageBackendConfig) Validate() error {
123+
func (cfg *Config) Validate() error {
124+
if cfg.StoragePrefix != "" {
125+
acceptablePrefixCharacters := regexp.MustCompile(validPrefixCharactersRegex)
126+
if !acceptablePrefixCharacters.MatchString(cfg.StoragePrefix) {
127+
return ErrInvalidCharactersInStoragePrefix
128+
}
129+
}
130+
117131
if err := cfg.S3.Validate(); err != nil {
118132
return err
119133
}
120134

121135
return nil
122136
}
123137

124-
// Config holds configuration for accessing long-term storage.
125-
type Config struct {
126-
StorageBackendConfig `yaml:",inline"`
127-
StoragePrefix string `yaml:"storage_prefix"`
128-
129-
// Not used internally, meant to allow callers to wrap Buckets
130-
// created using this config
131-
Middlewares []func(objstore.InstrumentedBucket) (objstore.InstrumentedBucket, error) `yaml:"-"`
132-
}
133-
134-
// RegisterFlags registers the backend storage config.
135-
func (cfg *Config) RegisterFlags(f *flag.FlagSet) {
136-
cfg.RegisterFlagsWithPrefix("", f)
137-
}
138-
139-
func (cfg *Config) RegisterFlagsWithPrefixAndDefaultDirectory(prefix, dir string, f *flag.FlagSet) {
140-
cfg.StorageBackendConfig.RegisterFlagsWithPrefixAndDefaultDirectory(prefix, dir, f)
141-
f.StringVar(&cfg.StoragePrefix, prefix+"storage-prefix", "", "Prefix for all objects stored in the backend storage. For simplicity, it may only contain digits and English alphabet letters.")
138+
type ConfigWithNamedStores struct {
139+
Config `yaml:",inline"`
140+
NamedStores NamedStores `yaml:"named_stores"`
142141
}
143142

144-
func (cfg *Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) {
145-
cfg.RegisterFlagsWithPrefixAndDefaultDirectory(prefix, "", f)
146-
}
147-
148-
func (cfg *Config) Validate() error {
149-
if cfg.StoragePrefix != "" {
150-
acceptablePrefixCharacters := regexp.MustCompile(validPrefixCharactersRegex)
151-
if !acceptablePrefixCharacters.MatchString(cfg.StoragePrefix) {
152-
return ErrInvalidCharactersInStoragePrefix
153-
}
143+
func (cfg *ConfigWithNamedStores) Validate() error {
144+
if err := cfg.Config.Validate(); err != nil {
145+
return err
154146
}
155147

156-
return cfg.StorageBackendConfig.Validate()
148+
return cfg.NamedStores.Validate()
157149
}
158150

159151
func (cfg *Config) disableRetries(backend string) error {

0 commit comments

Comments
 (0)