remove shared store

cmd/logcli/main.go

@@ -560,7 +560,8 @@ func newQuery(instant bool, cmd *kingpin.CmdClause) *query.Query {
 	cmd.Flag("include-label", "Include labels given the provided key during output.").StringsVar(&q.ShowLabelsKey)
 	cmd.Flag("labels-length", "Set a fixed padding to labels").Default("0").IntVar(&q.FixedLabelsLen)
 	cmd.Flag("store-config", "Execute the current query using a configured storage from a given Loki configuration file.").Default("").StringVar(&q.LocalConfig)
-	cmd.Flag("remote-schema", "Execute the current query using a remote schema retrieved using the configured storage in the given Loki configuration file.").Default("false").BoolVar(&q.FetchSchemaFromStorage)
+	cmd.Flag("remote-schema", "Execute the current query using a remote schema retrieved from the configured -schema-store.").Default("false").BoolVar(&q.FetchSchemaFromStorage)
+	cmd.Flag("schema-store", "Store used for retrieving remote schema.").Default("").StringVar(&q.SchemaStore)
 	cmd.Flag("colored-output", "Show output with colored labels").Default("false").BoolVar(&q.ColoredOutput)
 
 	return q

docs/sources/configure/_index.md

@@ -1988,17 +1988,6 @@ boltdb_shipper:
   # CLI flag: -boltdb.shipper.active-index-directory
   [active_index_directory: <string> | default = ""]
 
-  # Shared store for keeping index files. Supported types: gcs, s3, azure, cos,
-  # filesystem
-  # CLI flag: -boltdb.shipper.shared-store
-  [shared_store: <string> | default = ""]
-
-  # Prefix to add to Object Keys in Shared store. Path separator(if any) should
-  # always be a '/'. Prefix should never start with a separator but should
-  # always end with it
-  # CLI flag: -boltdb.shipper.shared-store.key-prefix
-  [shared_store_key_prefix: <string> | default = "index/"]
-
   # Cache location for restoring index files from storage for queries
   # CLI flag: -boltdb.shipper.cache-location
   [cache_location: <string> | default = ""]
@@ -2050,17 +2039,6 @@ tsdb_shipper:
   # CLI flag: -tsdb.shipper.active-index-directory
   [active_index_directory: <string> | default = ""]
 
-  # Shared store for keeping index files. Supported types: gcs, s3, azure, cos,
-  # filesystem
-  # CLI flag: -tsdb.shipper.shared-store
-  [shared_store: <string> | default = ""]
-
-  # Prefix to add to Object Keys in Shared store. Path separator(if any) should
-  # always be a '/'. Prefix should never start with a separator but should
-  # always end with it
-  # CLI flag: -tsdb.shipper.shared-store.key-prefix
-  [shared_store_key_prefix: <string> | default = "index/"]
-
   # Cache location for restoring index files from storage for queries
   # CLI flag: -tsdb.shipper.cache-location
   [cache_location: <string> | default = ""]
@@ -4121,8 +4099,8 @@ The `period_config` block configures what index schemas should be used for from
 
 # Configures how the index is updated and stored.
 index:
-  # path prefix for index tables. Defaults to "index/". The path prefix always
-  # needs to end with a path delimiter '/', except when the prefix is empty.
+  # Path prefix for index tables. Prefix always needs to end with a path
+  # delimiter '/', except when the prefix is empty.
   [path_prefix: <string> | default = "/index"]
 
   # Table prefix for all period tables.

docs/sources/configure/examples.md

@@ -38,7 +38,6 @@ storage_config:
   tsdb_shipper:
     active_index_directory: /tmp/loki/index
     cache_location: /tmp/loki/index_cache
-    shared_store: filesystem
   filesystem:
     directory: /tmp/loki/chunks
 
@@ -80,7 +79,6 @@ storage_config:
  tsdb_shipper:
    active_index_directory: /loki/index
    cache_location: /loki/index_cache
-   shared_store: s3
  aws:
    s3: s3://access_key:secret_access_key@custom_endpoint/bucket_name
    s3forcepathstyle: true
@@ -137,10 +135,8 @@ storage_config:
   tsdb_shipper:
     active_index_directory: /loki/index
     cache_location: /loki/index_cache
-    shared_store: gcs
   gcs:
     bucket_name: replace_by_your_bucked_name
-
 
 ```
 
@@ -166,7 +162,6 @@ storage_config:
   tsdb_shipper:
     active_index_directory: /loki/index
     cache_location: /loki/index_cache
-    shared_store: bos
   bos:
     bucket_name: bucket_name_1
     endpoint: bj.bcebos.com
@@ -217,7 +212,6 @@ schema_config:
     index:
         period: 24h
         prefix: index_
-
 
 ```
 
@@ -242,7 +236,6 @@ storage_config:
   tsdb_shipper:
     active_index_directory: /loki/index
     cache_location: /loki/index_cache
-    shared_store: alibabacloud
   alibabacloud:
     bucket: <bucket>
     endpoint: <endpoint>
@@ -313,7 +306,6 @@ storage_config:
   tsdb_shipper:
     active_index_directory: /loki/index
     cache_location: /loki/index_cache
-    shared_store: cos
   cos:
     bucketnames: <bucket1, bucket2>
     endpoint: <endpoint>
@@ -344,7 +336,6 @@ storage_config:
   tsdb_shipper:
     active_index_directory: /loki/index
     cache_location: /loki/index_cache
-    shared_store: cos
   cos:
     bucketnames: <bucket1, bucket2>
     endpoint: <endpoint>
@@ -382,7 +373,6 @@ storage_config:
   tsdb_shipper:
     active_index_directory: /loki/index
     cache_location: /loki/index_cache
-    shared_store: cos
   cos:
     bucketnames: <bucket1, bucket2>
     endpoint: <endpoint>

docs/sources/configure/examples/1-Local-Configuration-Example.yaml

@@ -28,6 +28,5 @@ storage_config:
   tsdb_shipper:
     active_index_directory: /tmp/loki/index
     cache_location: /tmp/loki/index_cache
-    shared_store: filesystem
   filesystem:
-    directory: /tmp/loki/chunks
+    directory: /tmp/loki/chunks

docs/sources/configure/examples/11-COS-HMAC-Example.yaml

@@ -14,7 +14,6 @@ storage_config:
   tsdb_shipper:
     active_index_directory: /loki/index
     cache_location: /loki/index_cache
-    shared_store: cos
   cos:
     bucketnames: <bucket1, bucket2>
     endpoint: <endpoint>

docs/sources/configure/examples/12-COS-APIKey-Example.yaml

@@ -14,7 +14,6 @@ storage_config:
   tsdb_shipper:
     active_index_directory: /loki/index
     cache_location: /loki/index_cache
-    shared_store: cos
   cos:
     bucketnames: <bucket1, bucket2>
     endpoint: <endpoint>

docs/sources/configure/examples/13-COS-Trusted-Profile-Example.yaml

@@ -20,7 +20,6 @@ storage_config:
   tsdb_shipper:
     active_index_directory: /loki/index
     cache_location: /loki/index_cache
-    shared_store: cos
   cos:
     bucketnames: <bucket1, bucket2>
     endpoint: <endpoint>

docs/sources/configure/examples/2-S3-Cluster-Example.yaml

@@ -29,7 +29,6 @@ storage_config:
  tsdb_shipper:
    active_index_directory: /loki/index
    cache_location: /loki/index_cache
-   shared_store: s3
  aws:
    s3: s3://access_key:secret_access_key@custom_endpoint/bucket_name
-   s3forcepathstyle: true
+   s3forcepathstyle: true

docs/sources/configure/examples/4-GCS-Example.yaml

@@ -28,7 +28,5 @@ storage_config:
   tsdb_shipper:
     active_index_directory: /loki/index
     cache_location: /loki/index_cache
-    shared_store: gcs
   gcs:
     bucket_name: replace_by_your_bucked_name
-

docs/sources/configure/examples/5-BOS-Example.yaml

@@ -15,7 +15,6 @@ storage_config:
   tsdb_shipper:
     active_index_directory: /loki/index
     cache_location: /loki/index_cache
-    shared_store: bos
   bos:
     bucket_name: bucket_name_1
     endpoint: bj.bcebos.com

docs/sources/configure/examples/7-Schema-Migration-Snippet.yaml

@@ -20,4 +20,3 @@ schema_config:
     index:
         period: 24h
         prefix: index_
-

docs/sources/configure/examples/8-alibaba-cloud-storage-Snippet.yaml

@@ -14,7 +14,6 @@ storage_config:
   tsdb_shipper:
     active_index_directory: /loki/index
     cache_location: /loki/index_cache
-    shared_store: alibabacloud
   alibabacloud:
     bucket: <bucket>
     endpoint: <endpoint>

docs/sources/operations/storage/boltdb-shipper.md

@@ -41,7 +41,6 @@ storage_config:
 
   boltdb_shipper:
     active_index_directory: /loki/index
-    shared_store: gcs
     cache_location: /loki/boltdb-cache
 ```
 

docs/sources/operations/storage/retention.md

@@ -78,7 +78,6 @@ storage_config:
     tsdb_shipper:
         active_index_directory: /data/index
         cache_location: /data/index_cache
-        shared_store: gcs
     gcs:
         bucket_name: loki
 ```
@@ -246,7 +245,6 @@ storage_config:
   tsdb_shipper:
     active_index_directory: /loki/index
     cache_location: /loki/index_cache
-    shared_store: gcs
   gcs:
     bucket_name: GCS_BUCKET_NAME
 

docs/sources/operations/storage/tsdb.md

@@ -42,7 +42,6 @@ storage_config:
       # only applicable if using microservices where index-gateways are independently deployed.
       # This example is using kubernetes-style naming.
       server_address: dns:///index-gateway.<namespace>.svc.cluster.local:9095
-    shared_store: gcs
   # New tsdb-shipper configuration
   tsdb_shipper:
     active_index_directory: /data/tsdb-index
@@ -51,7 +50,6 @@ storage_config:
       # only applicable if using microservices where index-gateways are independently deployed.
       # This example is using kubernetes-style naming.
       server_address: dns:///index-gateway.<namespace>.svc.cluster.local:9095
-    shared_store: gcs
 
 query_scheduler:
   # the TSDB index dispatches many more, but each individually smaller, requests. 

docs/sources/setup/install/tanka.md

@@ -48,7 +48,6 @@ Revise the YAML contents of `environments/loki/main.jsonnet`, updating these var
 - Update the `username`, `password`, and the relevant `htpasswd` variable values.
 - Update the S3 or GCS variable values, depending on your object storage type. See [storage_config](/docs/loki/latest/configuration/#storage_config) for more configuration details.
 - Remove from the configuration the S3 or GCS object storage variables that are not part of your setup.
-- Update the value of `boltdb_shipper_shared_store` or `tsdb_shipper_shared_store` to the type of object storage you are using. Options are `gcs`, `s3`, `azure`, `cos` or `filesystem`. Update the `object_store` variable under the `schema_config` section to the same value. 
 - Update the Promtail configuration `container_root_path` variable's value to reflect your root path for the Docker daemon. Run `docker info | grep "Root Dir"` to acquire your root path.
 - Update the `from` value in the Loki `schema_config` section to no more than 14 days prior to the current date. The `from` date represents the first day for which the `schema_config` section is valid. For example, if today is `2021-01-15`, set `from` to `2021-01-01`. This recommendation is based on Loki's default acceptance of log lines up to 14 days in the past. The `reject_old_samples_max_age` configuration variable controls the acceptance range.
 
@@ -77,9 +76,6 @@ loki + promtail + gateway {
     bigtable_project: 'project',
     gcs_bucket_name: 'bucket',
 
-    //Set this variable based on the type of object storage you're using.
-    boltdb_shipper_shared_store: 'my-object-storage-backend-type',
-
     //Update the object_store and from fields
     loki+: {
       schema_config: {

docs/sources/storage/_index.md

@@ -225,7 +225,6 @@ storage_config:
     active_index_directory: /loki/index
     cache_location: /loki/index_cache
     cache_ttl: 24h         # Can be increased for faster performance over longer query periods, uses more disk space
-    shared_store: gcs
   gcs:
       bucket_name: <bucket>
 
@@ -248,7 +247,6 @@ storage_config:
     active_index_directory: /loki/index
     cache_location: /loki/index_cache
     cache_ttl: 24h         # Can be increased for faster performance over longer query periods, uses more disk space
-    shared_store: s3
   aws:
     s3: s3://<access_key>:<uri-encoded-secret-access-key>@<region>
     bucketnames: <bucket1,bucket2>
@@ -366,7 +364,6 @@ storage_config:
     active_index_directory: /loki/index
     cache_location: /loki/index_cache
     cache_ttl: 24h
-    shared_store: azure
   filesystem:
     directory: /loki/chunks
 ```
@@ -399,7 +396,6 @@ storage_config:
     active_index_directory: /loki/index
     cache_location: /loki/index_cache
     cache_ttl: 24h
-    shared_store: azure
   filesystem:
     directory: /loki/chunks
 ```
@@ -421,7 +417,6 @@ storage_config:
   tsdb_shipper:
    active_index_directory: /loki/index
    cache_location: /loki/index_cache
-   shared_store: cos
   cos:
     bucketnames: <bucket1, bucket2>
     endpoint: <endpoint>
@@ -476,7 +471,6 @@ storage_config:
     active_index_directory: /loki/index
     cache_location: /loki/index_cache
     cache_ttl: 24h         # Can be increased for faster performance over longer query periods, uses more disk space
-    shared_store: s3
 
 schema_config:
   configs:

pkg/logcli/query/query.go

@@ -53,6 +53,7 @@ type Query struct {
 	ColoredOutput          bool
 	LocalConfig            string
 	FetchSchemaFromStorage bool
+	SchemaStore            string
 
 	// Parallelization parameters.
 
@@ -406,7 +407,7 @@ func (q *Query) DoLocalQuery(out output.LogOutput, statistics bool, orgID string
 
 	cm := storage.NewClientMetrics()
 	if useRemoteSchema {
-		client, err := GetObjectClient(conf, cm)
+		client, err := GetObjectClient(q.SchemaStore, conf, cm)
 		if err != nil {
 			return err
 		}
@@ -489,9 +490,9 @@ func (q *Query) DoLocalQuery(out output.LogOutput, statistics bool, orgID string
 	return nil
 }
 
-func GetObjectClient(conf loki.Config, cm storage.ClientMetrics) (chunk.ObjectClient, error) {
+func GetObjectClient(store string, conf loki.Config, cm storage.ClientMetrics) (chunk.ObjectClient, error) {
 	oc, err := storage.NewObjectClient(
-		conf.StorageConfig.BoltDBShipperConfig.SharedStoreType,
+		store,
 		conf.StorageConfig,
 		cm,
 	)

pkg/logcli/query/query_test.go

@@ -25,8 +25,6 @@ import (
 	"github.com/grafana/loki/pkg/storage"
 	"github.com/grafana/loki/pkg/storage/chunk/client/local"
 	"github.com/grafana/loki/pkg/storage/config"
-	"github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper"
-	"github.com/grafana/loki/pkg/storage/stores/shipper/indexshipper/boltdb"
 	"github.com/grafana/loki/pkg/util/marshal"
 )
 
@@ -514,17 +512,7 @@ func TestLoadFromURL(t *testing.T) {
 
 	// Missing SharedStoreType should error
 	cm := storage.NewClientMetrics()
-	client, err := GetObjectClient(conf, cm)
-	require.Error(t, err)
-	require.Nil(t, client)
-
-	conf.StorageConfig.BoltDBShipperConfig = boltdb.IndexCfg{
-		Config: indexshipper.Config{
-			SharedStoreType: config.StorageTypeFileSystem,
-		},
-	}
-
-	client, err = GetObjectClient(conf, cm)
+	client, err := GetObjectClient(config.StorageTypeFileSystem, conf, cm)
 	require.NoError(t, err)
 	require.NotNil(t, client)