add max_retries to gcs, obs, cos

Signed-off-by: Ashwanth Goli <iamashwanth@gmail.com>

providers/cos/cos.go

@@ -59,6 +59,7 @@ type Config struct {
 	Endpoint   string             `yaml:"endpoint"`
 	SecretKey  string             `yaml:"secret_key"`
 	SecretId   string             `yaml:"secret_id"`
+	MaxRetries int                `yaml:"max_retries"`
 	HTTPConfig exthttp.HTTPConfig `yaml:"http_config"`
 }
 
@@ -143,6 +144,10 @@ func NewBucketWithConfig(logger log.Logger, config Config, component string, rt
 		},
 	})
 
+	if config.MaxRetries > 0 {
+		client.Conf.RetryOpt.Count = config.MaxRetries
+	}
+
 	bkt := &Bucket{
 		logger: logger,
 		client: client,

providers/gcs/gcs.go

@@ -55,8 +55,10 @@ type Config struct {
 	ChunkSizeBytes int  `yaml:"chunk_size_bytes"`
 	noAuth         bool `yaml:"no_auth"`
 
-	// gcs client retries idempotent operations by default, this option disables retries.
-	DisableRetries bool `yaml:"disable_retries"`
+	// MaxRetries controls the number of retries for idempotent operations.
+	// Overrides the default gcs storage client behaviour if this value is greater than 0.
+	// Set this to 1 to disable retries.
+	MaxRetries int `yaml:"max_retries"`
 }
 
 // Bucket implements the store.Bucket and shipper.Bucket interfaces against GCS.
@@ -176,8 +178,8 @@ func newBucket(ctx context.Context, logger log.Logger, gc Config, opts []option.
 		chunkSize: gc.ChunkSizeBytes,
 	}
 
-	if gc.DisableRetries {
-		bkt.bkt = bkt.bkt.Retryer(storage.WithPolicy(storage.RetryNever))
+	if gc.MaxRetries > 0 {
+		bkt.bkt = bkt.bkt.Retryer(storage.WithMaxAttempts(gc.MaxRetries))
 	}
 
 	return bkt, nil

providers/obs/obs.go

@@ -46,6 +46,7 @@ type Config struct {
 	Endpoint   string             `yaml:"endpoint"`
 	AccessKey  string             `yaml:"access_key"`
 	SecretKey  string             `yaml:"secret_key"`
+	MaxRetries int                `yaml:"max_retries"`
 	HTTPConfig exthttp.HTTPConfig `yaml:"http_config"`
 }
 
@@ -102,7 +103,13 @@ func NewBucketWithConfig(logger log.Logger, config Config) (*Bucket, error) {
 		return nil, errors.Wrap(err, "get http transport err")
 	}
 
-	client, err := obs.New(config.AccessKey, config.SecretKey, config.Endpoint, obs.WithHttpTransport(rt))
+	var client *obs.ObsClient
+	if config.MaxRetries > 0 {
+		client, err = obs.New(config.AccessKey, config.SecretKey, config.Endpoint, obs.WithHttpTransport(rt), obs.WithMaxRetryCount(config.MaxRetries))
+	} else {
+		client, err = obs.New(config.AccessKey, config.SecretKey, config.Endpoint, obs.WithHttpTransport(rt))
+	}
+
 	if err != nil {
 		return nil, errors.Wrap(err, "initialize obs client err")
 	}