Skip to content

Commit 32a9bc0

Browse files
loki-gh-app[bot]loki-gh-app[bot]
authored
fix(config): Copy Alibaba and IBM object storage configuration from common (#14315)
1 parent 6ac7522 commit 32a9bc0

File tree

2 files changed

+138
-0
lines changed

2 files changed

+138
-0
lines changed

‎pkg/loki/config_wrapper.go

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -570,6 +570,26 @@ func applyStorageConfig(cfg, defaults *ConfigWrapper) error {
570570
}
571571
}
572572

573+
if !reflect.DeepEqual(cfg.Common.Storage.AlibabaCloud, defaults.StorageConfig.AlibabaStorageConfig) {
574+
configsFound++
575+
576+
applyConfig = func(r *ConfigWrapper) {
577+
r.Ruler.StoreConfig.Type = "alibaba"
578+
r.Ruler.StoreConfig.AlibabaCloud = r.Common.Storage.AlibabaCloud
579+
r.StorageConfig.AlibabaStorageConfig = r.Common.Storage.AlibabaCloud
580+
}
581+
}
582+
583+
if !reflect.DeepEqual(cfg.Common.Storage.COS, defaults.StorageConfig.COSConfig) {
584+
configsFound++
585+
586+
applyConfig = func(r *ConfigWrapper) {
587+
r.Ruler.StoreConfig.Type = "cos"
588+
r.Ruler.StoreConfig.COS = r.Common.Storage.COS
589+
r.StorageConfig.COSConfig = r.Common.Storage.COS
590+
}
591+
}
592+
573593
if !reflect.DeepEqual(cfg.Common.Storage.CongestionControl, defaults.StorageConfig.CongestionControl) {
574594
applyConfig = func(r *ConfigWrapper) {
575595
r.StorageConfig.CongestionControl = r.Common.Storage.CongestionControl

‎pkg/loki/config_wrapper_test.go

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -219,12 +219,16 @@ memberlist:
219219
assert.EqualValues(t, defaults.Ruler.StoreConfig.Swift, config.Ruler.StoreConfig.Swift)
220220
assert.EqualValues(t, defaults.Ruler.StoreConfig.Local, config.Ruler.StoreConfig.Local)
221221
assert.EqualValues(t, defaults.Ruler.StoreConfig.BOS, config.Ruler.StoreConfig.BOS)
222+
assert.EqualValues(t, defaults.Ruler.StoreConfig.AlibabaCloud, config.Ruler.StoreConfig.AlibabaCloud)
223+
assert.EqualValues(t, defaults.Ruler.StoreConfig.COS, config.Ruler.StoreConfig.COS)
222224
assert.EqualValues(t, defaults.StorageConfig.AWSStorageConfig, config.StorageConfig.AWSStorageConfig)
223225
assert.EqualValues(t, defaults.StorageConfig.AzureStorageConfig, config.StorageConfig.AzureStorageConfig)
224226
assert.EqualValues(t, defaults.StorageConfig.GCSConfig, config.StorageConfig.GCSConfig)
225227
assert.EqualValues(t, defaults.StorageConfig.Swift, config.StorageConfig.Swift)
226228
assert.EqualValues(t, defaults.StorageConfig.FSConfig, config.StorageConfig.FSConfig)
227229
assert.EqualValues(t, defaults.StorageConfig.BOSStorageConfig, config.StorageConfig.BOSStorageConfig)
230+
assert.EqualValues(t, defaults.StorageConfig.AlibabaStorageConfig, config.StorageConfig.AlibabaStorageConfig)
231+
assert.EqualValues(t, defaults.StorageConfig.COSConfig, config.StorageConfig.COSConfig)
228232
})
229233

230234
t.Run("when multiple configs are provided, an error is returned", func(t *testing.T) {
@@ -296,12 +300,17 @@ memberlist:
296300
assert.EqualValues(t, defaults.Ruler.StoreConfig.Swift, config.Ruler.StoreConfig.Swift)
297301
assert.EqualValues(t, defaults.Ruler.StoreConfig.Local, config.Ruler.StoreConfig.Local)
298302
assert.EqualValues(t, defaults.Ruler.StoreConfig.BOS, config.Ruler.StoreConfig.BOS)
303+
assert.EqualValues(t, defaults.Ruler.StoreConfig.AlibabaCloud, config.Ruler.StoreConfig.AlibabaCloud)
304+
assert.EqualValues(t, defaults.Ruler.StoreConfig.COS, config.Ruler.StoreConfig.COS)
305+
299306
// should remain empty
300307
assert.EqualValues(t, defaults.StorageConfig.AzureStorageConfig, config.StorageConfig.AzureStorageConfig)
301308
assert.EqualValues(t, defaults.StorageConfig.GCSConfig, config.StorageConfig.GCSConfig)
302309
assert.EqualValues(t, defaults.StorageConfig.Swift, config.StorageConfig.Swift)
303310
assert.EqualValues(t, defaults.StorageConfig.FSConfig, config.StorageConfig.FSConfig)
304311
assert.EqualValues(t, defaults.StorageConfig.BOSStorageConfig, config.StorageConfig.BOSStorageConfig)
312+
assert.EqualValues(t, defaults.StorageConfig.AlibabaStorageConfig, config.StorageConfig.AlibabaStorageConfig)
313+
assert.EqualValues(t, defaults.StorageConfig.COSConfig, config.StorageConfig.COSConfig)
305314
})
306315

307316
t.Run("when common s3 storage config is provided (with session token), ruler and storage config are defaulted to use it", func(t *testing.T) {
@@ -356,12 +365,17 @@ memberlist:
356365
assert.EqualValues(t, defaults.Ruler.StoreConfig.Swift, config.Ruler.StoreConfig.Swift)
357366
assert.EqualValues(t, defaults.Ruler.StoreConfig.Local, config.Ruler.StoreConfig.Local)
358367
assert.EqualValues(t, defaults.Ruler.StoreConfig.BOS, config.Ruler.StoreConfig.BOS)
368+
assert.EqualValues(t, defaults.Ruler.StoreConfig.AlibabaCloud, config.Ruler.StoreConfig.AlibabaCloud)
369+
assert.EqualValues(t, defaults.Ruler.StoreConfig.COS, config.Ruler.StoreConfig.COS)
370+
359371
// should remain empty
360372
assert.EqualValues(t, defaults.StorageConfig.AzureStorageConfig, config.StorageConfig.AzureStorageConfig)
361373
assert.EqualValues(t, defaults.StorageConfig.GCSConfig, config.StorageConfig.GCSConfig)
362374
assert.EqualValues(t, defaults.StorageConfig.Swift, config.StorageConfig.Swift)
363375
assert.EqualValues(t, defaults.StorageConfig.FSConfig, config.StorageConfig.FSConfig)
364376
assert.EqualValues(t, defaults.StorageConfig.BOSStorageConfig, config.StorageConfig.BOSStorageConfig)
377+
assert.EqualValues(t, defaults.StorageConfig.AlibabaStorageConfig, config.StorageConfig.AlibabaStorageConfig)
378+
assert.EqualValues(t, defaults.StorageConfig.COSConfig, config.StorageConfig.COSConfig)
365379
})
366380

367381
t.Run("when common gcs storage config is provided, ruler and storage config are defaulted to use it", func(t *testing.T) {
@@ -392,12 +406,17 @@ memberlist:
392406
assert.EqualValues(t, defaults.Ruler.StoreConfig.Swift, config.Ruler.StoreConfig.Swift)
393407
assert.EqualValues(t, defaults.Ruler.StoreConfig.Local, config.Ruler.StoreConfig.Local)
394408
assert.EqualValues(t, defaults.Ruler.StoreConfig.BOS, config.Ruler.StoreConfig.BOS)
409+
assert.EqualValues(t, defaults.Ruler.StoreConfig.AlibabaCloud, config.Ruler.StoreConfig.AlibabaCloud)
410+
assert.EqualValues(t, defaults.Ruler.StoreConfig.COS, config.Ruler.StoreConfig.COS)
411+
395412
// should remain empty
396413
assert.EqualValues(t, defaults.StorageConfig.AzureStorageConfig, config.StorageConfig.AzureStorageConfig)
397414
assert.EqualValues(t, defaults.StorageConfig.AWSStorageConfig.S3Config, config.StorageConfig.AWSStorageConfig.S3Config)
398415
assert.EqualValues(t, defaults.StorageConfig.Swift, config.StorageConfig.Swift)
399416
assert.EqualValues(t, defaults.StorageConfig.FSConfig, config.StorageConfig.FSConfig)
400417
assert.EqualValues(t, defaults.StorageConfig.BOSStorageConfig, config.StorageConfig.BOSStorageConfig)
418+
assert.EqualValues(t, defaults.StorageConfig.AlibabaStorageConfig, config.StorageConfig.AlibabaStorageConfig)
419+
assert.EqualValues(t, defaults.StorageConfig.COSConfig, config.StorageConfig.COSConfig)
401420
})
402421

403422
t.Run("when common azure storage config is provided, ruler and storage config are defaulted to use it", func(t *testing.T) {
@@ -444,13 +463,17 @@ memberlist:
444463
assert.EqualValues(t, defaults.Ruler.StoreConfig.Swift, config.Ruler.StoreConfig.Swift)
445464
assert.EqualValues(t, defaults.Ruler.StoreConfig.Local, config.Ruler.StoreConfig.Local)
446465
assert.EqualValues(t, defaults.Ruler.StoreConfig.BOS, config.Ruler.StoreConfig.BOS)
466+
assert.EqualValues(t, defaults.Ruler.StoreConfig.AlibabaCloud, config.Ruler.StoreConfig.AlibabaCloud)
467+
assert.EqualValues(t, defaults.Ruler.StoreConfig.COS, config.Ruler.StoreConfig.COS)
447468

448469
// should remain empty
449470
assert.EqualValues(t, defaults.StorageConfig.GCSConfig, config.StorageConfig.GCSConfig)
450471
assert.EqualValues(t, defaults.StorageConfig.AWSStorageConfig.S3Config, config.StorageConfig.AWSStorageConfig.S3Config)
451472
assert.EqualValues(t, defaults.StorageConfig.Swift, config.StorageConfig.Swift)
452473
assert.EqualValues(t, defaults.StorageConfig.FSConfig, config.StorageConfig.FSConfig)
453474
assert.EqualValues(t, defaults.StorageConfig.BOSStorageConfig, config.StorageConfig.BOSStorageConfig)
475+
assert.EqualValues(t, defaults.StorageConfig.AlibabaStorageConfig, config.StorageConfig.AlibabaStorageConfig)
476+
assert.EqualValues(t, defaults.StorageConfig.COSConfig, config.StorageConfig.COSConfig)
454477
})
455478

456479
t.Run("when common bos storage config is provided, ruler and storage config are defaulted to use it", func(t *testing.T) {
@@ -482,13 +505,17 @@ memberlist:
482505
assert.EqualValues(t, defaults.Ruler.StoreConfig.S3, config.Ruler.StoreConfig.S3)
483506
assert.EqualValues(t, defaults.Ruler.StoreConfig.Swift, config.Ruler.StoreConfig.Swift)
484507
assert.EqualValues(t, defaults.Ruler.StoreConfig.Local, config.Ruler.StoreConfig.Local)
508+
assert.EqualValues(t, defaults.Ruler.StoreConfig.AlibabaCloud, config.Ruler.StoreConfig.AlibabaCloud)
509+
assert.EqualValues(t, defaults.Ruler.StoreConfig.COS, config.Ruler.StoreConfig.COS)
485510

486511
// should remain empty
487512
assert.EqualValues(t, defaults.StorageConfig.AzureStorageConfig, config.StorageConfig.AzureStorageConfig)
488513
assert.EqualValues(t, defaults.StorageConfig.GCSConfig, config.StorageConfig.GCSConfig)
489514
assert.EqualValues(t, defaults.StorageConfig.AWSStorageConfig.S3Config, config.StorageConfig.AWSStorageConfig.S3Config)
490515
assert.EqualValues(t, defaults.StorageConfig.Swift, config.StorageConfig.Swift)
491516
assert.EqualValues(t, defaults.StorageConfig.FSConfig, config.StorageConfig.FSConfig)
517+
assert.EqualValues(t, defaults.StorageConfig.AlibabaStorageConfig, config.StorageConfig.AlibabaStorageConfig)
518+
assert.EqualValues(t, defaults.StorageConfig.COSConfig, config.StorageConfig.COSConfig)
492519
})
493520

494521
t.Run("when common swift storage config is provided, ruler and storage config are defaulted to use it", func(t *testing.T) {
@@ -549,12 +576,103 @@ memberlist:
549576
assert.EqualValues(t, defaults.Ruler.StoreConfig.Azure, config.Ruler.StoreConfig.Azure)
550577
assert.EqualValues(t, defaults.Ruler.StoreConfig.Local, config.Ruler.StoreConfig.Local)
551578
assert.EqualValues(t, defaults.Ruler.StoreConfig.BOS, config.Ruler.StoreConfig.BOS)
579+
assert.EqualValues(t, defaults.Ruler.StoreConfig.AlibabaCloud, config.Ruler.StoreConfig.AlibabaCloud)
580+
assert.EqualValues(t, defaults.Ruler.StoreConfig.COS, config.Ruler.StoreConfig.COS)
581+
582+
// should remain empty
583+
assert.EqualValues(t, defaults.StorageConfig.GCSConfig, config.StorageConfig.GCSConfig)
584+
assert.EqualValues(t, defaults.StorageConfig.AWSStorageConfig.S3Config, config.StorageConfig.AWSStorageConfig.S3Config)
585+
assert.EqualValues(t, defaults.StorageConfig.AzureStorageConfig, config.StorageConfig.AzureStorageConfig)
586+
assert.EqualValues(t, defaults.StorageConfig.FSConfig, config.StorageConfig.FSConfig)
587+
assert.EqualValues(t, defaults.StorageConfig.BOSStorageConfig, config.StorageConfig.BOSStorageConfig)
588+
assert.EqualValues(t, defaults.StorageConfig.AlibabaStorageConfig, config.StorageConfig.AlibabaStorageConfig)
589+
assert.EqualValues(t, defaults.StorageConfig.COSConfig, config.StorageConfig.COSConfig)
590+
})
591+
592+
t.Run("when common alibaba storage config is provided, ruler and storage config are defaulted to use it", func(t *testing.T) {
593+
configInput := `common:
594+
storage:
595+
alibabacloud:
596+
bucket: testbucket
597+
endpoint: https://example.com
598+
access_key_id: abc123
599+
secret_access_key: def789`
600+
601+
config, defaults := testContext(configInput, nil)
602+
603+
assert.Equal(t, "alibaba", config.Ruler.StoreConfig.Type)
604+
605+
for _, actual := range []alibaba.OssConfig{
606+
config.Ruler.StoreConfig.AlibabaCloud,
607+
config.StorageConfig.AlibabaStorageConfig,
608+
} {
609+
assert.Equal(t, "testbucket", actual.Bucket)
610+
assert.Equal(t, "https://example.com", actual.Endpoint)
611+
assert.Equal(t, "abc123", actual.AccessKeyID)
612+
assert.Equal(t, "def789", actual.SecretAccessKey)
613+
}
614+
615+
// should remain empty
616+
assert.EqualValues(t, defaults.Ruler.StoreConfig.GCS, config.Ruler.StoreConfig.GCS)
617+
assert.EqualValues(t, defaults.Ruler.StoreConfig.S3, config.Ruler.StoreConfig.S3)
618+
assert.EqualValues(t, defaults.Ruler.StoreConfig.Azure, config.Ruler.StoreConfig.Azure)
619+
assert.EqualValues(t, defaults.Ruler.StoreConfig.Swift, config.Ruler.StoreConfig.Swift)
620+
assert.EqualValues(t, defaults.Ruler.StoreConfig.Local, config.Ruler.StoreConfig.Local)
621+
assert.EqualValues(t, defaults.Ruler.StoreConfig.BOS, config.Ruler.StoreConfig.BOS)
622+
assert.EqualValues(t, defaults.Ruler.StoreConfig.COS, config.Ruler.StoreConfig.COS)
623+
624+
// should remain empty
625+
assert.EqualValues(t, defaults.StorageConfig.GCSConfig, config.StorageConfig.GCSConfig)
626+
assert.EqualValues(t, defaults.StorageConfig.AWSStorageConfig.S3Config, config.StorageConfig.AWSStorageConfig.S3Config)
627+
assert.EqualValues(t, defaults.StorageConfig.AzureStorageConfig, config.StorageConfig.AzureStorageConfig)
628+
assert.EqualValues(t, defaults.StorageConfig.Swift, config.StorageConfig.Swift)
629+
assert.EqualValues(t, defaults.StorageConfig.FSConfig, config.StorageConfig.FSConfig)
630+
assert.EqualValues(t, defaults.StorageConfig.BOSStorageConfig, config.StorageConfig.BOSStorageConfig)
631+
assert.EqualValues(t, defaults.StorageConfig.COSConfig, config.StorageConfig.COSConfig)
632+
})
633+
634+
t.Run("when common cos storage config is provided, ruler and storage config are defaulted to use it", func(t *testing.T) {
635+
configInput := `common:
636+
storage:
637+
cos:
638+
bucketnames: testbucket
639+
endpoint: https://example.com
640+
region: test-region
641+
access_key_id: abc123
642+
secret_access_key: def789`
643+
644+
config, defaults := testContext(configInput, nil)
645+
646+
assert.Equal(t, "cos", config.Ruler.StoreConfig.Type)
647+
648+
for _, actual := range []ibmcloud.COSConfig{
649+
config.Ruler.StoreConfig.COS,
650+
config.StorageConfig.COSConfig,
651+
} {
652+
assert.Equal(t, "testbucket", actual.BucketNames)
653+
assert.Equal(t, "https://example.com", actual.Endpoint)
654+
assert.Equal(t, "test-region", actual.Region)
655+
assert.Equal(t, "abc123", actual.AccessKeyID)
656+
assert.Equal(t, flagext.SecretWithValue("def789"), actual.SecretAccessKey)
657+
}
658+
659+
// should remain empty
660+
assert.EqualValues(t, defaults.Ruler.StoreConfig.GCS, config.Ruler.StoreConfig.GCS)
661+
assert.EqualValues(t, defaults.Ruler.StoreConfig.S3, config.Ruler.StoreConfig.S3)
662+
assert.EqualValues(t, defaults.Ruler.StoreConfig.Azure, config.Ruler.StoreConfig.Azure)
663+
assert.EqualValues(t, defaults.Ruler.StoreConfig.Swift, config.Ruler.StoreConfig.Swift)
664+
assert.EqualValues(t, defaults.Ruler.StoreConfig.Local, config.Ruler.StoreConfig.Local)
665+
assert.EqualValues(t, defaults.Ruler.StoreConfig.BOS, config.Ruler.StoreConfig.BOS)
666+
assert.EqualValues(t, defaults.Ruler.StoreConfig.AlibabaCloud, config.Ruler.StoreConfig.AlibabaCloud)
667+
552668
// should remain empty
553669
assert.EqualValues(t, defaults.StorageConfig.GCSConfig, config.StorageConfig.GCSConfig)
554670
assert.EqualValues(t, defaults.StorageConfig.AWSStorageConfig.S3Config, config.StorageConfig.AWSStorageConfig.S3Config)
555671
assert.EqualValues(t, defaults.StorageConfig.AzureStorageConfig, config.StorageConfig.AzureStorageConfig)
672+
assert.EqualValues(t, defaults.StorageConfig.Swift, config.StorageConfig.Swift)
556673
assert.EqualValues(t, defaults.StorageConfig.FSConfig, config.StorageConfig.FSConfig)
557674
assert.EqualValues(t, defaults.StorageConfig.BOSStorageConfig, config.StorageConfig.BOSStorageConfig)
675+
assert.EqualValues(t, defaults.StorageConfig.AlibabaStorageConfig, config.StorageConfig.AlibabaStorageConfig)
558676
})
559677

560678
t.Run("when common filesystem/local config is provided, ruler and storage config are defaulted to use it", func(t *testing.T) {

0 commit comments

Comments
 (0)