feat: Add objstore support for Swift using thanos.io/objstore (#11672)

Co-authored-by: Joao Marcal <jmarcal@redhat.com>
Co-authored-by: Ashwanth Goli <iamashwanth@gmail.com>

Author: 3 people

docs/sources/shared/configuration.md

@@ -5976,6 +5976,12 @@ The `swift_storage_config` block configures the connection to OpenStack Object S
 # is received on a request.
 # CLI flag: -<prefix>.swift.request-timeout
 [request_timeout: <duration> | default = 5s]
+
+http:
+  # Path to the CA certificates to validate server certificate against. If not
+  # set, the host's root CA certificates are used.
+  # CLI flag: -<prefix>.swift.http.tls-ca-path
+  [tls_ca_path: <string> | default = ""]
 ```
 
 ### table_manager

pkg/logqlmodel/stats/context.go

@@ -108,7 +108,7 @@ func (c *Context) Index() Index {
 	return c.index
 }
 
-// Merge index stats from multiple respones in a concurrency-safe manner
+// Merge index stats from multiple response in a concurrency-safe manner
 func (c *Context) MergeIndex(i Index) {
 	c.mtx.Lock()
 	defer c.mtx.Unlock()

pkg/loki/config_wrapper_test.go

@@ -547,17 +547,17 @@ memberlist:
 
 			assert.Equal(t, "swift", config.Ruler.StoreConfig.Type)
 
-			for _, actual := range []swift.Config{
-				config.Ruler.StoreConfig.Swift.Config,
-				config.StorageConfig.Swift.Config,
+			for _, actual := range []openstack.SwiftConfig{
+				config.Ruler.StoreConfig.Swift,
+				config.StorageConfig.Swift,
 			} {
 				assert.Equal(t, 3, actual.AuthVersion)
 				assert.Equal(t, "http://example.com", actual.AuthURL)
 				assert.Equal(t, "steve", actual.Username)
 				assert.Equal(t, "example.com", actual.UserDomainName)
 				assert.Equal(t, "1", actual.UserDomainID)
 				assert.Equal(t, "27", actual.UserID)
-				assert.Equal(t, "supersecret", actual.Password)
+				assert.Equal(t, flagext.SecretWithValue("supersecret"), actual.Password)
 				assert.Equal(t, "2", actual.DomainID)
 				assert.Equal(t, "test.com", actual.DomainName)
 				assert.Equal(t, "13", actual.ProjectID)

pkg/storage/bucket/client.go

@@ -176,7 +176,7 @@ func (cfg *Config) configureTransport(backend string, rt http.RoundTripper) erro
 	case Azure:
 		cfg.Azure.Transport = rt
 	case Swift:
-		cfg.Swift.Transport = rt
+		cfg.Swift.HTTP.Transport = rt
 	case Filesystem, Alibaba, BOS:
 		// do nothing
 	default:

pkg/storage/bucket/http/config.go

@@ -2,6 +2,7 @@ package http
 
 import (
 	"flag"
+	"net/http"
 	"time"
 )
 
@@ -15,7 +16,19 @@ type Config struct {
 	MaxIdleConns          int           `yaml:"max_idle_connections"`
 	MaxIdleConnsPerHost   int           `yaml:"max_idle_connections_per_host"`
 	MaxConnsPerHost       int           `yaml:"max_connections_per_host"`
-	CAFile                string        `yaml:"ca_file"`
+
+	// Allow upstream callers to inject a round tripper
+	Transport http.RoundTripper `yaml:"-"`
+
+	TLSConfig TLSConfig `yaml:",inline"`
+}
+
+// TLSConfig configures the options for TLS connections.
+type TLSConfig struct {
+	CAPath     string `yaml:"tls_ca_path" category:"advanced"`
+	CertPath   string `yaml:"tls_cert_path" category:"advanced"`
+	KeyPath    string `yaml:"tls_key_path" category:"advanced"`
+	ServerName string `yaml:"tls_server_name" category:"advanced"`
 }
 
 // RegisterFlags registers the flags for the storage HTTP client.
@@ -25,13 +38,21 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet) {
 
 // RegisterFlagsWithPrefix registers the flags for the storage HTTP client with the provided prefix.
 func (cfg *Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) {
-	f.DurationVar(&cfg.IdleConnTimeout, prefix+"idle-conn-timeout", 90*time.Second, "The time an idle connection will remain idle before closing.")
-	f.DurationVar(&cfg.ResponseHeaderTimeout, prefix+"response-header-timeout", 2*time.Minute, "The amount of time the client will wait for a servers response headers.")
-	f.BoolVar(&cfg.InsecureSkipVerify, prefix+"insecure-skip-verify", false, "If the client connects via HTTPS and this option is enabled, the client will accept any certificate and hostname.")
+	f.DurationVar(&cfg.IdleConnTimeout, prefix+"http.idle-conn-timeout", 90*time.Second, "The time an idle connection will remain idle before closing.")
+	f.DurationVar(&cfg.ResponseHeaderTimeout, prefix+"http.response-header-timeout", 2*time.Minute, "The amount of time the client will wait for a servers response headers.")
+	f.BoolVar(&cfg.InsecureSkipVerify, prefix+"http.insecure-skip-verify", false, "If the client connects via HTTPS and this option is enabled, the client will accept any certificate and hostname.")
 	f.DurationVar(&cfg.TLSHandshakeTimeout, prefix+"tls-handshake-timeout", 10*time.Second, "Maximum time to wait for a TLS handshake. 0 means no limit.")
 	f.DurationVar(&cfg.ExpectContinueTimeout, prefix+"expect-continue-timeout", 1*time.Second, "The time to wait for a server's first response headers after fully writing the request headers if the request has an Expect header. 0 to send the request body immediately.")
 	f.IntVar(&cfg.MaxIdleConns, prefix+"max-idle-connections", 100, "Maximum number of idle (keep-alive) connections across all hosts. 0 means no limit.")
 	f.IntVar(&cfg.MaxIdleConnsPerHost, prefix+"max-idle-connections-per-host", 100, "Maximum number of idle (keep-alive) connections to keep per-host. If 0, a built-in default value is used.")
 	f.IntVar(&cfg.MaxConnsPerHost, prefix+"max-connections-per-host", 0, "Maximum number of connections per host. 0 means no limit.")
-	f.StringVar(&cfg.CAFile, prefix+"ca-file", "", "Path to the trusted CA file that signed the SSL certificate of the object storage endpoint.")
+	cfg.TLSConfig.RegisterFlagsWithPrefix(prefix, f)
+}
+
+// RegisterFlagsWithPrefix registers the flags for s3 storage with the provided prefix.
+func (cfg *TLSConfig) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) {
+	f.StringVar(&cfg.CAPath, prefix+"http.tls-ca-path", "", "Path to the CA certificates to validate server certificate against. If not set, the host's root CA certificates are used.")
+	f.StringVar(&cfg.CertPath, prefix+"http.tls-cert-path", "", "Path to the client certificate, which will be used for authenticating with the server. Also requires the key path to be configured.")
+	f.StringVar(&cfg.KeyPath, prefix+"http.tls-key-path", "", "Path to the key for the client certificate. Also requires the client certificate to be configured.")
+	f.StringVar(&cfg.ServerName, prefix+"http.tls-server-name", "", "Override the expected name on the server certificate.")
 }

pkg/storage/bucket/s3/config.go

@@ -4,17 +4,16 @@ import (
 	"encoding/json"
 	"flag"
 	"fmt"
-	"net/http"
 	"slices"
 	"strings"
-	"time"
 
 	s3_service "github.com/aws/aws-sdk-go/service/s3"
 	"github.com/grafana/dskit/flagext"
 	"github.com/minio/minio-go/v7/pkg/encrypt"
 	"github.com/pkg/errors"
 	"github.com/thanos-io/objstore/providers/s3"
 
+	"github.com/grafana/loki/v3/pkg/storage/bucket/http"
 	"github.com/grafana/loki/v3/pkg/util"
 )
 
@@ -55,52 +54,6 @@ func thanosS3BucketLookupTypesValues() (list []string) {
 	return list
 }
 
-// HTTPConfig stores the http.Transport configuration for the s3 minio client.
-type HTTPConfig struct {
-	IdleConnTimeout       time.Duration `yaml:"idle_conn_timeout" category:"advanced"`
-	ResponseHeaderTimeout time.Duration `yaml:"response_header_timeout" category:"advanced"`
-	InsecureSkipVerify    bool          `yaml:"insecure_skip_verify" category:"advanced"`
-	TLSHandshakeTimeout   time.Duration `yaml:"tls_handshake_timeout" category:"advanced"`
-	ExpectContinueTimeout time.Duration `yaml:"expect_continue_timeout" category:"advanced"`
-	MaxIdleConns          int           `yaml:"max_idle_connections" category:"advanced"`
-	MaxIdleConnsPerHost   int           `yaml:"max_idle_connections_per_host" category:"advanced"`
-	MaxConnsPerHost       int           `yaml:"max_connections_per_host" category:"advanced"`
-
-	// Allow upstream callers to inject a round tripper
-	Transport http.RoundTripper `yaml:"-"`
-
-	TLSConfig TLSConfig `yaml:",inline"`
-}
-
-// TLSConfig configures the options for TLS connections.
-type TLSConfig struct {
-	CAPath     string `yaml:"tls_ca_path" category:"advanced"`
-	CertPath   string `yaml:"tls_cert_path" category:"advanced"`
-	KeyPath    string `yaml:"tls_key_path" category:"advanced"`
-	ServerName string `yaml:"tls_server_name" category:"advanced"`
-}
-
-// RegisterFlagsWithPrefix registers the flags for s3 storage with the provided prefix
-func (cfg *HTTPConfig) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) {
-	f.DurationVar(&cfg.IdleConnTimeout, prefix+"s3.http.idle-conn-timeout", 90*time.Second, "The time an idle connection will remain idle before closing.")
-	f.DurationVar(&cfg.ResponseHeaderTimeout, prefix+"s3.http.response-header-timeout", 2*time.Minute, "The amount of time the client will wait for a servers response headers.")
-	f.BoolVar(&cfg.InsecureSkipVerify, prefix+"s3.http.insecure-skip-verify", false, "If the client connects to S3 via HTTPS and this option is enabled, the client will accept any certificate and hostname.")
-	f.DurationVar(&cfg.TLSHandshakeTimeout, prefix+"s3.tls-handshake-timeout", 10*time.Second, "Maximum time to wait for a TLS handshake. 0 means no limit.")
-	f.DurationVar(&cfg.ExpectContinueTimeout, prefix+"s3.expect-continue-timeout", 1*time.Second, "The time to wait for a server's first response headers after fully writing the request headers if the request has an Expect header. 0 to send the request body immediately.")
-	f.IntVar(&cfg.MaxIdleConns, prefix+"s3.max-idle-connections", 100, "Maximum number of idle (keep-alive) connections across all hosts. 0 means no limit.")
-	f.IntVar(&cfg.MaxIdleConnsPerHost, prefix+"s3.max-idle-connections-per-host", 100, "Maximum number of idle (keep-alive) connections to keep per-host. If 0, a built-in default value is used.")
-	f.IntVar(&cfg.MaxConnsPerHost, prefix+"s3.max-connections-per-host", 0, "Maximum number of connections per host. 0 means no limit.")
-	cfg.TLSConfig.RegisterFlagsWithPrefix(prefix, f)
-}
-
-// RegisterFlagsWithPrefix registers the flags for s3 storage with the provided prefix.
-func (cfg *TLSConfig) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) {
-	f.StringVar(&cfg.CAPath, prefix+"s3.http.tls-ca-path", "", "Path to the CA certificates to validate server certificate against. If not set, the host's root CA certificates are used.")
-	f.StringVar(&cfg.CertPath, prefix+"s3.http.tls-cert-path", "", "Path to the client certificate, which will be used for authenticating with the server. Also requires the key path to be configured.")
-	f.StringVar(&cfg.KeyPath, prefix+"s3.http.tls-key-path", "", "Path to the key for the client certificate. Also requires the client certificate to be configured.")
-	f.StringVar(&cfg.ServerName, prefix+"s3.http.tls-server-name", "", "Override the expected name on the server certificate.")
-}
-
 // Config holds the config options for an S3 backend
 type Config struct {
 	Endpoint             string              `yaml:"endpoint"`
@@ -121,7 +74,7 @@ type Config struct {
 	MaxRetries           int                 `yaml:"max_retries"`
 
 	SSE         SSEConfig   `yaml:"sse"`
-	HTTP        HTTPConfig  `yaml:"http"`
+	HTTP        http.Config `yaml:"http"`
 	TraceConfig TraceConfig `yaml:"trace"`
 }
 
@@ -149,7 +102,7 @@ func (cfg *Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) {
 	f.StringVar(&cfg.STSEndpoint, prefix+"s3.sts-endpoint", "", "Accessing S3 resources using temporary, secure credentials provided by AWS Security Token Service.")
 	f.IntVar(&cfg.MaxRetries, prefix+"s3.max-retries", 10, "The maximum number of retries for S3 requests that are retryable. Default is 10, set this to 1 to disable retries.")
 	cfg.SSE.RegisterFlagsWithPrefix(prefix+"s3.sse.", f)
-	cfg.HTTP.RegisterFlagsWithPrefix(prefix, f)
+	cfg.HTTP.RegisterFlagsWithPrefix(prefix+"s3.", f)
 	cfg.TraceConfig.RegisterFlagsWithPrefix(prefix+"s3.trace.", f)
 }
 

pkg/storage/bucket/swift/bucket_client.go

@@ -13,31 +13,49 @@ import (
 // NewBucketClient creates a new Swift bucket client
 func NewBucketClient(cfg Config, _ string, logger log.Logger, wrapper func(http.RoundTripper) http.RoundTripper) (objstore.Bucket, error) {
 	bucketConfig := swift.Config{
-		AuthVersion:       cfg.AuthVersion,
-		AuthUrl:           cfg.AuthURL,
-		Username:          cfg.Username,
-		UserDomainName:    cfg.UserDomainName,
-		UserDomainID:      cfg.UserDomainID,
-		UserId:            cfg.UserID,
-		Password:          cfg.Password,
-		DomainId:          cfg.DomainID,
-		DomainName:        cfg.DomainName,
-		ProjectID:         cfg.ProjectID,
-		ProjectName:       cfg.ProjectName,
-		ProjectDomainID:   cfg.ProjectDomainID,
-		ProjectDomainName: cfg.ProjectDomainName,
-		RegionName:        cfg.RegionName,
-		ContainerName:     cfg.ContainerName,
-		Retries:           cfg.MaxRetries,
-		ConnectTimeout:    model.Duration(cfg.ConnectTimeout),
-		Timeout:           model.Duration(cfg.RequestTimeout),
+		ApplicationCredentialID:     cfg.ApplicationCredentialID,
+		ApplicationCredentialName:   cfg.ApplicationCredentialName,
+		ApplicationCredentialSecret: cfg.ApplicationCredentialSecret.String(),
+		AuthVersion:                 cfg.AuthVersion,
+		AuthUrl:                     cfg.AuthURL,
+		Username:                    cfg.Username,
+		UserDomainName:              cfg.UserDomainName,
+		UserDomainID:                cfg.UserDomainID,
+		UserId:                      cfg.UserID,
+		Password:                    cfg.Password.String(),
+		DomainId:                    cfg.DomainID,
+		DomainName:                  cfg.DomainName,
+		ProjectID:                   cfg.ProjectID,
+		ProjectName:                 cfg.ProjectName,
+		ProjectDomainID:             cfg.ProjectDomainID,
+		ProjectDomainName:           cfg.ProjectDomainName,
+		RegionName:                  cfg.RegionName,
+		ContainerName:               cfg.ContainerName,
+		Retries:                     cfg.MaxRetries,
+		ConnectTimeout:              model.Duration(cfg.ConnectTimeout),
+		Timeout:                     model.Duration(cfg.RequestTimeout),
+		HTTPConfig: exthttp.HTTPConfig{
+			IdleConnTimeout:       model.Duration(cfg.HTTP.IdleConnTimeout),
+			ResponseHeaderTimeout: model.Duration(cfg.HTTP.ResponseHeaderTimeout),
+			InsecureSkipVerify:    cfg.HTTP.InsecureSkipVerify,
+			TLSHandshakeTimeout:   model.Duration(cfg.HTTP.TLSHandshakeTimeout),
+			ExpectContinueTimeout: model.Duration(cfg.HTTP.ExpectContinueTimeout),
+			MaxIdleConns:          cfg.HTTP.MaxIdleConns,
+			MaxIdleConnsPerHost:   cfg.HTTP.MaxIdleConnsPerHost,
+			MaxConnsPerHost:       cfg.HTTP.MaxConnsPerHost,
+			Transport:             cfg.HTTP.Transport,
+			TLSConfig: exthttp.TLSConfig{
+				CAFile:     cfg.HTTP.TLSConfig.CAPath,
+				CertFile:   cfg.HTTP.TLSConfig.CertPath,
+				KeyFile:    cfg.HTTP.TLSConfig.KeyPath,
+				ServerName: cfg.HTTP.TLSConfig.ServerName,
+			},
+		},
 
 		// Hard-coded defaults.
 		ChunkSize:              swift.DefaultConfig.ChunkSize,
 		UseDynamicLargeObjects: false,
-		HTTPConfig:             exthttp.DefaultHTTPConfig,
 	}
-	bucketConfig.HTTPConfig.Transport = cfg.Transport
 
 	return swift.NewContainerFromConfig(logger, &bucketConfig, false, wrapper)
 }

pkg/storage/bucket/swift/config.go

@@ -2,34 +2,37 @@ package swift
 
 import (
 	"flag"
-	"net/http"
 	"time"
+
+	"github.com/grafana/dskit/flagext"
+
+	"github.com/grafana/loki/v3/pkg/storage/bucket/http"
 )
 
 // Config holds the config options for Swift backend
 type Config struct {
-	AuthVersion       int           `yaml:"auth_version"`
-	AuthURL           string        `yaml:"auth_url"`
-	Internal          bool          `yaml:"internal"`
-	Username          string        `yaml:"username"`
-	UserDomainName    string        `yaml:"user_domain_name"`
-	UserDomainID      string        `yaml:"user_domain_id"`
-	UserID            string        `yaml:"user_id"`
-	Password          string        `yaml:"password"`
-	DomainID          string        `yaml:"domain_id"`
-	DomainName        string        `yaml:"domain_name"`
-	ProjectID         string        `yaml:"project_id"`
-	ProjectName       string        `yaml:"project_name"`
-	ProjectDomainID   string        `yaml:"project_domain_id"`
-	ProjectDomainName string        `yaml:"project_domain_name"`
-	RegionName        string        `yaml:"region_name"`
-	ContainerName     string        `yaml:"container_name"`
-	MaxRetries        int           `yaml:"max_retries"`
-	ConnectTimeout    time.Duration `yaml:"connect_timeout"`
-	RequestTimeout    time.Duration `yaml:"request_timeout"`
-
-	// Allow upstream callers to inject a round tripper
-	Transport http.RoundTripper `yaml:"-"`
+	ApplicationCredentialID     string         `yaml:"application_credential_id"`
+	ApplicationCredentialName   string         `yaml:"application_credential_name"`
+	ApplicationCredentialSecret flagext.Secret `yaml:"application_credential_secret"`
+	AuthVersion                 int            `yaml:"auth_version"`
+	AuthURL                     string         `yaml:"auth_url"`
+	Username                    string         `yaml:"username"`
+	UserDomainName              string         `yaml:"user_domain_name"`
+	UserDomainID                string         `yaml:"user_domain_id"`
+	UserID                      string         `yaml:"user_id"`
+	Password                    flagext.Secret `yaml:"password"`
+	DomainID                    string         `yaml:"domain_id"`
+	DomainName                  string         `yaml:"domain_name"`
+	ProjectID                   string         `yaml:"project_id"`
+	ProjectName                 string         `yaml:"project_name"`
+	ProjectDomainID             string         `yaml:"project_domain_id"`
+	ProjectDomainName           string         `yaml:"project_domain_name"`
+	RegionName                  string         `yaml:"region_name"`
+	ContainerName               string         `yaml:"container_name"`
+	MaxRetries                  int            `yaml:"max_retries" category:"advanced"`
+	ConnectTimeout              time.Duration  `yaml:"connect_timeout" category:"advanced"`
+	RequestTimeout              time.Duration  `yaml:"request_timeout" category:"advanced"`
+	HTTP                        http.Config    `yaml:"http"`
 }
 
 // RegisterFlags registers the flags for Swift storage
@@ -39,14 +42,16 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet) {
 
 // RegisterFlagsWithPrefix registers the flags for Swift storage with the provided prefix
 func (cfg *Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) {
+	f.StringVar(&cfg.ApplicationCredentialID, prefix+"swift.application-credential-id", "", "OpenStack Swift application credential id")
+	f.StringVar(&cfg.ApplicationCredentialName, prefix+"swift.application-credential-name", "", "OpenStack Swift application credential name")
+	f.Var(&cfg.ApplicationCredentialSecret, prefix+"swift.application-credential-secret", "OpenStack Swift application credential secret")
 	f.IntVar(&cfg.AuthVersion, prefix+"swift.auth-version", 0, "OpenStack Swift authentication API version. 0 to autodetect.")
 	f.StringVar(&cfg.AuthURL, prefix+"swift.auth-url", "", "OpenStack Swift authentication URL")
-	f.BoolVar(&cfg.Internal, prefix+"swift.internal", false, "Set this to true to use the internal OpenStack Swift endpoint URL")
 	f.StringVar(&cfg.Username, prefix+"swift.username", "", "OpenStack Swift username.")
 	f.StringVar(&cfg.UserDomainName, prefix+"swift.user-domain-name", "", "OpenStack Swift user's domain name.")
 	f.StringVar(&cfg.UserDomainID, prefix+"swift.user-domain-id", "", "OpenStack Swift user's domain ID.")
 	f.StringVar(&cfg.UserID, prefix+"swift.user-id", "", "OpenStack Swift user ID.")
-	f.StringVar(&cfg.Password, prefix+"swift.password", "", "OpenStack Swift API key.")
+	f.Var(&cfg.Password, prefix+"swift.password", "OpenStack Swift API key.")
 	f.StringVar(&cfg.DomainID, prefix+"swift.domain-id", "", "OpenStack Swift user's domain ID.")
 	f.StringVar(&cfg.DomainName, prefix+"swift.domain-name", "", "OpenStack Swift user's domain name.")
 	f.StringVar(&cfg.ProjectID, prefix+"swift.project-id", "", "OpenStack Swift project ID (v2,v3 auth only).")
@@ -58,8 +63,5 @@ func (cfg *Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) {
 	f.IntVar(&cfg.MaxRetries, prefix+"swift.max-retries", 3, "Max retries on requests error.")
 	f.DurationVar(&cfg.ConnectTimeout, prefix+"swift.connect-timeout", 10*time.Second, "Time after which a connection attempt is aborted.")
 	f.DurationVar(&cfg.RequestTimeout, prefix+"swift.request-timeout", 5*time.Second, "Time after which an idle request is aborted. The timeout watchdog is reset each time some data is received, so the timeout triggers after X time no data is received on a request.")
-}
-
-func (cfg *Config) Validate() error {
-	return nil
+	cfg.HTTP.RegisterFlagsWithPrefix(prefix+"swift.", f)
 }