grafana · ashwanthgoli · Oct 30, 2023 · Oct 6, 2023 · Oct 9, 2023 · Oct 9, 2023
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -23,6 +23,7 @@
 * [10709](https://github.com/grafana/loki/pull/10709) **chaudum**/**salvacorts** Remove `ingester.max-transfer-retries` configuration option in favor of using the WAL.
 * [10736](https://github.com/grafana/loki/pull/10736) **ashwanthgoli** Deprecate write dedupe cache as this is not required by the newer single store indexes (tsdb and boltdb-shipper).
 * [10693](https://github.com/grafana/loki/pull/10693) **ashwanthgoli** Embedded cache: Updates the metric prefix from `querier_cache_` to `loki_embeddedcache_` and removes duplicate metrics.
+* [10840](https://github.com/grafana/loki/pull/10840) **ashwanthgoli** Removes `shared_store` and `shared_store_key_prefix` from tsdb and boltdb shipper configs and their corresponding CLI flags.
 
 ##### Fixes
 

@@ -560,7 +560,8 @@ func newQuery(instant bool, cmd *kingpin.CmdClause) *query.Query {
 	cmd.Flag("include-label", "Include labels given the provided key during output.").StringsVar(&q.ShowLabelsKey)
 	cmd.Flag("labels-length", "Set a fixed padding to labels").Default("0").IntVar(&q.FixedLabelsLen)
 	cmd.Flag("store-config", "Execute the current query using a configured storage from a given Loki configuration file.").Default("").StringVar(&q.LocalConfig)
-	cmd.Flag("remote-schema", "Execute the current query using a remote schema retrieved using the configured storage in the given Loki configuration file.").Default("false").BoolVar(&q.FetchSchemaFromStorage)
+	cmd.Flag("remote-schema", "Execute the current query using a remote schema retrieved from the configured -schema-store.").Default("false").BoolVar(&q.FetchSchemaFromStorage)
+	cmd.Flag("schema-store", "Store used for retrieving remote schema.").Default("").StringVar(&q.SchemaStore)
 	cmd.Flag("colored-output", "Show output with colored labels").Default("false").BoolVar(&q.ColoredOutput)
 
 	return q

@@ -1988,17 +1988,6 @@ boltdb_shipper:
   # CLI flag: -boltdb.shipper.active-index-directory
   [active_index_directory: <string> | default = ""]
 
-  # Shared store for keeping index files. Supported types: gcs, s3, azure, cos,
-  # filesystem
-  # CLI flag: -boltdb.shipper.shared-store
-  [shared_store: <string> | default = ""]
-
-  # Prefix to add to Object Keys in Shared store. Path separator(if any) should
-  # always be a '/'. Prefix should never start with a separator but should
-  # always end with it
-  # CLI flag: -boltdb.shipper.shared-store.key-prefix
-  [shared_store_key_prefix: <string> | default = "index/"]
-
   # Cache location for restoring index files from storage for queries
   # CLI flag: -boltdb.shipper.cache-location
   [cache_location: <string> | default = ""]
@@ -2054,17 +2043,6 @@ tsdb_shipper:
   # CLI flag: -tsdb.shipper.active-index-directory
   [active_index_directory: <string> | default = ""]
 
-  # Shared store for keeping index files. Supported types: gcs, s3, azure, cos,
-  # filesystem
-  # CLI flag: -tsdb.shipper.shared-store
-  [shared_store: <string> | default = ""]
-
-  # Prefix to add to Object Keys in Shared store. Path separator(if any) should
-  # always be a '/'. Prefix should never start with a separator but should
-  # always end with it
-  # CLI flag: -tsdb.shipper.shared-store.key-prefix
-  [shared_store_key_prefix: <string> | default = "index/"]
-
   # Cache location for restoring index files from storage for queries
   # CLI flag: -tsdb.shipper.cache-location
   [cache_location: <string> | default = ""]
@@ -3354,7 +3332,7 @@ ring:
   # CLI flag: -common.storage.ring.instance-enable-ipv6
   [instance_enable_ipv6: <boolean> | default = false]
 
-[instance_interface_names: <list of strings>]
+[instance_interface_names: <list of strings> | default = [<private network interfaces>]]
 
 [instance_addr: <string> | default = ""]
 
@@ -4135,6 +4113,10 @@ The `period_config` block configures what index schemas should be used for from
 
 # Configures how the index is updated and stored.
 index:
+  # Path prefix for index tables. Prefix always needs to end with a path
+  # delimiter '/', except when the prefix is empty.
+  [path_prefix: <string> | default = "index/"]
+
   # Table prefix for all period tables.
   [prefix: <string> | default = ""]
 

diff --git a/docs/sources/configure/examples.md b/docs/sources/configure/examples.md
@@ -38,7 +38,6 @@ storage_config:
   tsdb_shipper:
     active_index_directory: /tmp/loki/index
     cache_location: /tmp/loki/index_cache
-    shared_store: filesystem
   filesystem:
     directory: /tmp/loki/chunks
 
@@ -80,7 +79,6 @@ storage_config:
  tsdb_shipper:
    active_index_directory: /loki/index
    cache_location: /loki/index_cache
-   shared_store: s3
  aws:
    s3: s3://access_key:secret_access_key@custom_endpoint/bucket_name
    s3forcepathstyle: true
@@ -137,10 +135,8 @@ storage_config:
   tsdb_shipper:
     active_index_directory: /loki/index
     cache_location: /loki/index_cache
-    shared_store: gcs
   gcs:
     bucket_name: replace_by_your_bucked_name
-
 
 ```
 
@@ -166,7 +162,6 @@ storage_config:
   tsdb_shipper:
     active_index_directory: /loki/index
     cache_location: /loki/index_cache
-    shared_store: bos
   bos:
     bucket_name: bucket_name_1
     endpoint: bj.bcebos.com
@@ -217,7 +212,6 @@ schema_config:
     index:
         period: 24h
         prefix: index_
-
 
 ```
 
@@ -242,7 +236,6 @@ storage_config:
   tsdb_shipper:
     active_index_directory: /loki/index
     cache_location: /loki/index_cache
-    shared_store: alibabacloud
   alibabacloud:
     bucket: <bucket>
     endpoint: <endpoint>
@@ -313,7 +306,6 @@ storage_config:
   tsdb_shipper:
     active_index_directory: /loki/index
     cache_location: /loki/index_cache
-    shared_store: cos
   cos:
     bucketnames: <bucket1, bucket2>
     endpoint: <endpoint>
@@ -344,7 +336,6 @@ storage_config:
   tsdb_shipper:
     active_index_directory: /loki/index
     cache_location: /loki/index_cache
-    shared_store: cos
   cos:
     bucketnames: <bucket1, bucket2>
     endpoint: <endpoint>
@@ -382,7 +373,6 @@ storage_config:
   tsdb_shipper:
     active_index_directory: /loki/index
     cache_location: /loki/index_cache
-    shared_store: cos
   cos:
     bucketnames: <bucket1, bucket2>
     endpoint: <endpoint>

diff --git a/docs/sources/configure/examples/1-Local-Configuration-Example.yaml b/docs/sources/configure/examples/1-Local-Configuration-Example.yaml
@@ -28,6 +28,5 @@ storage_config:
   tsdb_shipper:
     active_index_directory: /tmp/loki/index
     cache_location: /tmp/loki/index_cache
-    shared_store: filesystem
   filesystem:
-    directory: /tmp/loki/chunks
+    directory: /tmp/loki/chunks
diff --git a/docs/sources/configure/examples/11-COS-HMAC-Example.yaml b/docs/sources/configure/examples/11-COS-HMAC-Example.yaml
@@ -14,7 +14,6 @@ storage_config:
   tsdb_shipper:
     active_index_directory: /loki/index
     cache_location: /loki/index_cache
-    shared_store: cos
   cos:
     bucketnames: <bucket1, bucket2>
     endpoint: <endpoint>

diff --git a/docs/sources/configure/examples/12-COS-APIKey-Example.yaml b/docs/sources/configure/examples/12-COS-APIKey-Example.yaml
@@ -14,7 +14,6 @@ storage_config:
   tsdb_shipper:
     active_index_directory: /loki/index
     cache_location: /loki/index_cache
-    shared_store: cos
   cos:
     bucketnames: <bucket1, bucket2>
     endpoint: <endpoint>

diff --git a/docs/sources/configure/examples/13-COS-Trusted-Profile-Example.yaml b/docs/sources/configure/examples/13-COS-Trusted-Profile-Example.yaml
@@ -20,7 +20,6 @@ storage_config:
   tsdb_shipper:
     active_index_directory: /loki/index
     cache_location: /loki/index_cache
-    shared_store: cos
   cos:
     bucketnames: <bucket1, bucket2>
     endpoint: <endpoint>

diff --git a/docs/sources/configure/examples/2-S3-Cluster-Example.yaml b/docs/sources/configure/examples/2-S3-Cluster-Example.yaml
@@ -29,7 +29,6 @@ storage_config:
  tsdb_shipper:
    active_index_directory: /loki/index
    cache_location: /loki/index_cache
-   shared_store: s3
  aws:
    s3: s3://access_key:secret_access_key@custom_endpoint/bucket_name
-   s3forcepathstyle: true
+   s3forcepathstyle: true
diff --git a/docs/sources/configure/examples/4-GCS-Example.yaml b/docs/sources/configure/examples/4-GCS-Example.yaml
@@ -28,7 +28,5 @@ storage_config:
   tsdb_shipper:
     active_index_directory: /loki/index
     cache_location: /loki/index_cache
-    shared_store: gcs
   gcs:
     bucket_name: replace_by_your_bucked_name
-
diff --git a/docs/sources/configure/examples/5-BOS-Example.yaml b/docs/sources/configure/examples/5-BOS-Example.yaml
@@ -15,7 +15,6 @@ storage_config:
   tsdb_shipper:
     active_index_directory: /loki/index
     cache_location: /loki/index_cache
-    shared_store: bos
   bos:
     bucket_name: bucket_name_1
     endpoint: bj.bcebos.com

diff --git a/docs/sources/configure/examples/7-Schema-Migration-Snippet.yaml b/docs/sources/configure/examples/7-Schema-Migration-Snippet.yaml
@@ -20,4 +20,3 @@ schema_config:
     index:
         period: 24h
         prefix: index_
-
diff --git a/docs/sources/configure/examples/8-alibaba-cloud-storage-Snippet.yaml b/docs/sources/configure/examples/8-alibaba-cloud-storage-Snippet.yaml
@@ -14,7 +14,6 @@ storage_config:
   tsdb_shipper:
     active_index_directory: /loki/index
     cache_location: /loki/index_cache
-    shared_store: alibabacloud
   alibabacloud:
     bucket: <bucket>
     endpoint: <endpoint>

@@ -41,7 +41,6 @@ storage_config:
 
   boltdb_shipper:
     active_index_directory: /loki/index
-    shared_store: gcs
     cache_location: /loki/boltdb-cache
 ```
 

@@ -78,7 +78,6 @@ storage_config:
     tsdb_shipper:
         active_index_directory: /data/index
         cache_location: /data/index_cache
-        shared_store: gcs
     gcs:
         bucket_name: loki
 ```
@@ -246,7 +245,6 @@ storage_config:
   tsdb_shipper:
     active_index_directory: /loki/index
     cache_location: /loki/index_cache
-    shared_store: gcs
   gcs:
     bucket_name: GCS_BUCKET_NAME
 

@@ -42,7 +42,6 @@ storage_config:
       # only applicable if using microservices where index-gateways are independently deployed.
       # This example is using kubernetes-style naming.
       server_address: dns:///index-gateway.<namespace>.svc.cluster.local:9095
-    shared_store: gcs
   # New tsdb-shipper configuration
   tsdb_shipper:
     active_index_directory: /data/tsdb-index
@@ -51,7 +50,6 @@ storage_config:
       # only applicable if using microservices where index-gateways are independently deployed.
       # This example is using kubernetes-style naming.
       server_address: dns:///index-gateway.<namespace>.svc.cluster.local:9095
-    shared_store: gcs
 
 query_scheduler:
   # the TSDB index dispatches many more, but each individually smaller, requests. 

@@ -48,7 +48,6 @@ Revise the YAML contents of `environments/loki/main.jsonnet`, updating these var
 - Update the `username`, `password`, and the relevant `htpasswd` variable values.
 - Update the S3 or GCS variable values, depending on your object storage type. See [storage_config](/docs/loki/latest/configuration/#storage_config) for more configuration details.
 - Remove from the configuration the S3 or GCS object storage variables that are not part of your setup.
-- Update the value of `boltdb_shipper_shared_store` or `tsdb_shipper_shared_store` to the type of object storage you are using. Options are `gcs`, `s3`, `azure`, `cos` or `filesystem`. Update the `object_store` variable under the `schema_config` section to the same value. 
 - Update the Promtail configuration `container_root_path` variable's value to reflect your root path for the Docker daemon. Run `docker info | grep "Root Dir"` to acquire your root path.
 - Update the `from` value in the Loki `schema_config` section to no more than 14 days prior to the current date. The `from` date represents the first day for which the `schema_config` section is valid. For example, if today is `2021-01-15`, set `from` to `2021-01-01`. This recommendation is based on Loki's default acceptance of log lines up to 14 days in the past. The `reject_old_samples_max_age` configuration variable controls the acceptance range.
 
@@ -77,9 +76,6 @@ loki + promtail + gateway {
     bigtable_project: 'project',
     gcs_bucket_name: 'bucket',
 
-    //Set this variable based on the type of object storage you're using.
-    boltdb_shipper_shared_store: 'my-object-storage-backend-type',
-
     //Update the object_store and from fields
     loki+: {
       schema_config: {

@@ -38,6 +38,56 @@ The output is incredibly verbose as it shows the entire internal config struct u
 
 ### Loki
 
+#### Removed `shared_store` and `shared_store_key_prefix` from shipper configuration
+
+The following CLI flags and the corresponding YAML settings to configure shared store for TSDB and BoltDB shippers are now removed:
+- `-boltdb.shipper.shared-store`
+- `-tsdb.shipper.shared-store`
+
+Going forward `object_store` setting in the [period_config](/docs/loki/latest/configure/#period_config) will be used to configure store for the index.
+This enforces chunks and index files to reside together in the same storage bucket for a given period.
+
+We are removing the shared store setting in an effort to simplify storage configuration and reduce the possibility for misconfiguration.
+
+{{% admonition type="warning" %}}
+With this change Loki no longer allows storing chunks and indexes for a given period in different storage buckets.
+This is a breaking change for setups that store chunks and indexes in different storage buckets by setting `-boltdb.shipper.shared-store` or `-tsdb.shipper.shared-store` to a value
+different from `object_store` in `period_config`.
+{{% /admonition %}}
+
+- If you have not configured `-boltdb.shipper.shared-store`,`-tsdb.shipper.shared-store` or their corresponding YAML setting before, no changes are required as part of the upgrade.
+- If you have configured `-boltdb.shipper.shared-store` or its YAML setting:
+  - If it matches the value of `object_store` for all the periods that use `boltdb-shipper` as index type, no changes are required.
+  - If there is a mismatch, you lose access to the index for periods where `-boltdb.shipper.shared-store` does not match `object_store`.
+    - To make these indexes queryable, index tables need to moved or copied to the store configured in `object_store`.
+- If you have configured `-tsdb.shipper.shared-store` or its YAML setting:
+  - If it matches the value of `object_store` for all the periods that use `tsdb` as index type, no changes are required.
+  - If there is a mismatch, you lose access to the index for periods where `-tsdb.shipper.shared-store` does not match `object_store`.
+    - To make these indexes queryable, index tables need to moved or copied to the store configured in `object_store`.
+
+The following CLI flags and the corresponding YAML settings to configure a path prefix for TSDB and BoltDB shippers are now removed:
+- `-boltdb.shipper.shared-store.key-prefix`
+- `-tsdb.shipper.shared-store.key-prefix`
+
+Path prefix for storing the index can now be configured by setting `path_prefix` under `index` key in [period_config](/docs/loki/latest/configure/#period_config).
+This enables users to change the path prefix by adding a new period config.
+```
+period_config:
+  index:
+    path_prefix: "index/"
+    period: 24h
+```
+
+{{% admonition type="note" %}}
+`path_prefix` only applies to TSDB and BoltDB indexes. This setting has no effect on [legacy indexes]({{< relref "../../storage#index-storage" >}}).
+{{% /admonition %}}
+
+`path_prefix` defaults to `index/` which is same as the default value of the removed configurations.
+
+- No changes are required if you have not configured `-boltdb.shipper.shared-store.key-prefix`, `-tsdb.shipper.shared-store.key-prefix` or the corresponding YAML setting previously.
+- If you have configured `-boltdb.shipper.shared-store.key-prefix` or its YAML setting to a value other than `index/`, ensure that all the existing period configs that use `boltdb-shipper` as the index have `path_prefix` set to the value previously configured.
+- If you have configured `-tsdb.shipper.shared-store.key-prefix` or its YAML setting to a value other than `index/`, ensure that all the existing period configs that use `tsdb` as the index have the `path_prefix` set to the value previously configured.
+
 #### Configuration `use_boltdb_shipper_as_backup` is removed
 
 The setting `use_boltdb_shipper_as_backup` (`-tsdb.shipper.use-boltdb-shipper-as-backup`) was a remnant from the development of the TSDB storage.
@@ -97,6 +147,13 @@ If you using a [legacy index type]({{< relref "../../storage#index-storage" >}})
 
 - Already deprecated metric `querier_cache_stale_gets_total` is now removed.
 
+### LogCLI
+
+#### Store for retrieving remote schema
+
+Previously LogCLI used to fetch remote schema from the store configured in `-boltdb.shipper.shared-store` when `-remote-schema` is set to true.
+A new CLI flag `-schema-store` is introduced as a replacement to configure the store for retrieving remote schema.
+
 ## 2.9.0
 
 ### Loki

@@ -225,7 +225,6 @@ storage_config:
     active_index_directory: /loki/index
     cache_location: /loki/index_cache
     cache_ttl: 24h         # Can be increased for faster performance over longer query periods, uses more disk space
-    shared_store: gcs
   gcs:
       bucket_name: <bucket>
 
@@ -248,7 +247,6 @@ storage_config:
     active_index_directory: /loki/index
     cache_location: /loki/index_cache
     cache_ttl: 24h         # Can be increased for faster performance over longer query periods, uses more disk space
-    shared_store: s3
   aws:
     s3: s3://<access_key>:<uri-encoded-secret-access-key>@<region>
     bucketnames: <bucket1,bucket2>
@@ -366,7 +364,6 @@ storage_config:
     active_index_directory: /loki/index
     cache_location: /loki/index_cache
     cache_ttl: 24h
-    shared_store: azure
   filesystem:
     directory: /loki/chunks
 ```
@@ -399,7 +396,6 @@ storage_config:
     active_index_directory: /loki/index
     cache_location: /loki/index_cache
     cache_ttl: 24h
-    shared_store: azure
   filesystem:
     directory: /loki/chunks
 ```
@@ -421,7 +417,6 @@ storage_config:
   tsdb_shipper:
    active_index_directory: /loki/index
    cache_location: /loki/index_cache
-   shared_store: cos
   cos:
     bucketnames: <bucket1, bucket2>
     endpoint: <endpoint>
@@ -476,7 +471,6 @@ storage_config:
     active_index_directory: /loki/index
     cache_location: /loki/index_cache
     cache_ttl: 24h         # Can be increased for faster performance over longer query periods, uses more disk space
-    shared_store: s3
 
 schema_config:
   configs:
Original file line number	Diff line number	Diff line change
Expand Up		@@ -20,4 +20,3 @@ schema_config:
		index:
		period: 24h
		prefix: index_