feat(thanos): add support for aliyun oss and baidu bos (grafana#14891)

Author: ashwanthgoli

pkg/storage/bucket/bos/bucket_client.go

@@ -0,0 +1,17 @@
+package bos
+
+import (
+	"github.com/go-kit/log"
+	"github.com/thanos-io/objstore"
+	"github.com/thanos-io/objstore/providers/bos"
+)
+
+func NewBucketClient(cfg Config, name string, logger log.Logger) (objstore.Bucket, error) {
+	bosCfg := bos.Config{
+		Endpoint:  cfg.Endpoint,
+		Bucket:    cfg.Bucket,
+		SecretKey: cfg.SecretKey.String(),
+		AccessKey: cfg.AccessKey,
+	}
+	return bos.NewBucketWithConfig(logger, bosCfg, name)
+}

pkg/storage/bucket/bos/config.go

@@ -0,0 +1,26 @@
+package bos
+
+import (
+	"flag"
+
+	"github.com/grafana/dskit/flagext"
+)
+
+// Config holds the configuration for Baidu Cloud BOS client
+type Config struct {
+	Bucket    string         `yaml:"bucket"`
+	Endpoint  string         `yaml:"endpoint"`
+	AccessKey string         `yaml:"access_key"`
+	SecretKey flagext.Secret `yaml:"secret_key"`
+}
+
+func (cfg *Config) RegisterFlags(f *flag.FlagSet) {
+	cfg.RegisterFlagsWithPrefix("", f)
+}
+
+func (cfg *Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) {
+	f.StringVar(&cfg.Bucket, prefix+"bos.bucket", "", "Name of BOS bucket.")
+	f.StringVar(&cfg.Endpoint, prefix+"bos.endpoint", "", "BOS endpoint to connect to.")
+	f.StringVar(&cfg.AccessKey, prefix+"bos.access-key", "", "Baidu Cloud Engine (BCE) Access Key ID.")
+	f.Var(&cfg.SecretKey, prefix+"bos.secret-key", "Baidu Cloud Engine (BCE) Secret Access Key.")
+}

pkg/storage/bucket/client.go

@@ -15,8 +15,10 @@ import (
 	objstoretracing "github.com/thanos-io/objstore/tracing/opentracing"
 
 	"github.com/grafana/loki/v3/pkg/storage/bucket/azure"
+	"github.com/grafana/loki/v3/pkg/storage/bucket/bos"
 	"github.com/grafana/loki/v3/pkg/storage/bucket/filesystem"
 	"github.com/grafana/loki/v3/pkg/storage/bucket/gcs"
+	"github.com/grafana/loki/v3/pkg/storage/bucket/oss"
 	"github.com/grafana/loki/v3/pkg/storage/bucket/s3"
 	"github.com/grafana/loki/v3/pkg/storage/bucket/swift"
 )
@@ -37,12 +39,18 @@ const (
 	// Filesystem is the value for the filesystem storage backend.
 	Filesystem = "filesystem"
 
+	// Alibaba is the value for the Alibaba Cloud OSS storage backend
+	Alibaba = "alibabacloud"
+
+	// BOS is the value for the Baidu Cloud BOS storage backend
+	BOS = "bos"
+
 	// validPrefixCharactersRegex allows only alphanumeric characters to prevent subtle bugs and simplify validation
 	validPrefixCharactersRegex = `^[\da-zA-Z]+$`
 )
 
 var (
-	SupportedBackends = []string{S3, GCS, Azure, Swift, Filesystem}
+	SupportedBackends = []string{S3, GCS, Azure, Swift, Filesystem, Alibaba, BOS}
 
 	ErrUnsupportedStorageBackend        = errors.New("unsupported storage backend")
 	ErrInvalidCharactersInStoragePrefix = errors.New("storage prefix contains invalid characters, it may only contain digits and English alphabet letters")
@@ -73,6 +81,8 @@ type StorageBackendConfig struct {
 	Azure      azure.Config      `yaml:"azure"`
 	Swift      swift.Config      `yaml:"swift"`
 	Filesystem filesystem.Config `yaml:"filesystem"`
+	Alibaba    oss.Config        `yaml:"alibaba"`
+	BOS        bos.Config        `yaml:"bos"`
 
 	// Used to inject additional backends into the config. Allows for this config to
 	// be embedded in multiple contexts and support non-object storage based backends.
@@ -95,6 +105,8 @@ func (cfg *StorageBackendConfig) RegisterFlagsWithPrefixAndDefaultDirectory(pref
 	cfg.Azure.RegisterFlagsWithPrefix(prefix, f)
 	cfg.Swift.RegisterFlagsWithPrefix(prefix, f)
 	cfg.Filesystem.RegisterFlagsWithPrefixAndDefaultDirectory(prefix, dir, f)
+	cfg.Alibaba.RegisterFlagsWithPrefix(prefix, f)
+	cfg.BOS.RegisterFlagsWithPrefix(prefix, f)
 }
 
 func (cfg *StorageBackendConfig) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) {
@@ -154,7 +166,7 @@ func (cfg *Config) disableRetries(backend string) error {
 		cfg.Azure.MaxRetries = 1
 	case Swift:
 		cfg.Swift.MaxRetries = 1
-	case Filesystem:
+	case Filesystem, Alibaba, BOS:
 		// do nothing
 	default:
 		return fmt.Errorf("cannot disable retries for backend: %s", backend)
@@ -173,7 +185,7 @@ func (cfg *Config) configureTransport(backend string, rt http.RoundTripper) erro
 		cfg.Azure.Transport = rt
 	case Swift:
 		cfg.Swift.Transport = rt
-	case Filesystem:
+	case Filesystem, Alibaba, BOS:
 		// do nothing
 	default:
 		return fmt.Errorf("cannot configure transport for backend: %s", backend)
@@ -201,6 +213,10 @@ func NewClient(ctx context.Context, backend string, cfg Config, name string, log
 		client, err = swift.NewBucketClient(cfg.Swift, name, logger, instrumentTransport())
 	case Filesystem:
 		client, err = filesystem.NewBucketClient(cfg.Filesystem)
+	case Alibaba:
+		client, err = oss.NewBucketClient(cfg.Alibaba, name, logger)
+	case BOS:
+		client, err = bos.NewBucketClient(cfg.BOS, name, logger)
 	default:
 		return nil, ErrUnsupportedStorageBackend
 	}

pkg/storage/bucket/oss/bucket_client.go

@@ -0,0 +1,18 @@
+package oss
+
+import (
+	"github.com/go-kit/log"
+	"github.com/thanos-io/objstore"
+	"github.com/thanos-io/objstore/providers/oss"
+)
+
+// NewBucketClient creates a new Alibaba Cloud OSS bucket client
+func NewBucketClient(cfg Config, component string, logger log.Logger) (objstore.Bucket, error) {
+	ossCfg := oss.Config{
+		Endpoint:        cfg.Endpoint,
+		Bucket:          cfg.Bucket,
+		AccessKeyID:     cfg.AccessKeyID,
+		AccessKeySecret: cfg.AccessKeySecret.String(),
+	}
+	return oss.NewBucketWithConfig(logger, ossCfg, component, nil)
+}

pkg/storage/bucket/oss/config.go

@@ -0,0 +1,28 @@
+package oss
+
+import (
+	"flag"
+
+	"github.com/grafana/dskit/flagext"
+)
+
+// Config holds the configuration for Alibaba Cloud OSS client
+type Config struct {
+	Endpoint        string         `yaml:"endpoint"`
+	Bucket          string         `yaml:"bucket"`
+	AccessKeyID     string         `yaml:"access_key_id"`
+	AccessKeySecret flagext.Secret `yaml:"access_key_secret"`
+}
+
+// RegisterFlags registers the flags for Alibaba Cloud OSS storage config
+func (cfg *Config) RegisterFlags(f *flag.FlagSet) {
+	cfg.RegisterFlagsWithPrefix("", f)
+}
+
+// RegisterFlagsWithPrefix registers the flags for Alibaba Cloud OSS storage config with prefix
+func (cfg *Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) {
+	f.StringVar(&cfg.Bucket, prefix+"oss.bucketname", "", "Name of OSS bucket.")
+	f.StringVar(&cfg.Endpoint, prefix+"oss.endpoint", "", "Endpoint to connect to.")
+	f.StringVar(&cfg.AccessKeyID, prefix+"oss.access-key-id", "", "alibabacloud Access Key ID")
+	f.Var(&cfg.AccessKeySecret, prefix+"oss.access-key-secret", "alibabacloud Secret Access Key")
+}