introduce path prefix to index period config

docs/sources/configure/_index.md

@@ -3340,7 +3340,7 @@ ring:
   # CLI flag: -common.storage.ring.instance-enable-ipv6
   [instance_enable_ipv6: <boolean> | default = false]
 
-[instance_interface_names: <list of strings>]
+[instance_interface_names: <list of strings> | default = [<private network interfaces>]]
 
 [instance_addr: <string> | default = ""]
 
@@ -4121,6 +4121,10 @@ The `period_config` block configures what index schemas should be used for from
 
 # Configures how the index is updated and stored.
 index:
+  # path prefix for index tables. Defaults to "index/". The path prefix always
+  # needs to end with a path delimiter '/', except when the prefix is empty.
+  [path_prefix: <string> | default = "/index"]
+
   # Table prefix for all period tables.
   [prefix: <string> | default = ""]
 

pkg/compactor/compactor.go

@@ -144,8 +144,8 @@ func (cfg *Config) Validate() error {
 		return errors.New("interval for applying retention should either be set to a 0 or a multiple of compaction interval")
 	}
 
-	if err := storage.ValidateSharedStoreKeyPrefix(cfg.SharedStoreKeyPrefix); err != nil {
-		return err
+	if err := config.ValidatePathPrefix(cfg.SharedStoreKeyPrefix); err != nil {
+		return fmt.Errorf("invalid compactor.shared-store.key-prefix: %w", err)
 	}
 
 	if cfg.DeletionMode != "" {

pkg/compactor/compactor_test.go

@@ -73,10 +73,11 @@ func TestCompactor_RunCompaction(t *testing.T) {
 			From:       config.DayTime{Time: model.Time(0)},
 			IndexType:  "dummy",
 			ObjectType: "fs_01",
-			IndexTables: config.PeriodicTableConfig{
-				Prefix: indexTablePrefix,
-				Period: config.ObjectStorageIndexRequiredPeriod,
-			},
+			IndexTables: config.IndexPeriodicTableConfig{
+				PeriodicTableConfig: config.PeriodicTableConfig{
+					Prefix: indexTablePrefix,
+					Period: config.ObjectStorageIndexRequiredPeriod,
+				}},
 		},
 	}
 
@@ -123,19 +124,21 @@ func TestCompactor_RunCompactionMultipleStores(t *testing.T) {
 			From:       config.DayTime{Time: model.Time(0)},
 			IndexType:  "dummy",
 			ObjectType: "fs_01",
-			IndexTables: config.PeriodicTableConfig{
-				Prefix: indexTablePrefix,
-				Period: config.ObjectStorageIndexRequiredPeriod,
-			},
+			IndexTables: config.IndexPeriodicTableConfig{
+				PeriodicTableConfig: config.PeriodicTableConfig{
+					Prefix: indexTablePrefix,
+					Period: config.ObjectStorageIndexRequiredPeriod,
+				}},
 		},
 		{
 			From:       config.DayTime{Time: model.Time(periodTwoStart * daySeconds * 1000)},
 			IndexType:  "dummy",
 			ObjectType: "fs_02",
-			IndexTables: config.PeriodicTableConfig{
-				Prefix: indexTablePrefix,
-				Period: config.ObjectStorageIndexRequiredPeriod,
-			},
+			IndexTables: config.IndexPeriodicTableConfig{
+				PeriodicTableConfig: config.PeriodicTableConfig{
+					Prefix: indexTablePrefix,
+					Period: config.ObjectStorageIndexRequiredPeriod,
+				}},
 		},
 	}
 
@@ -200,43 +203,47 @@ func Test_schemaPeriodForTable(t *testing.T) {
 			IndexType:  "boltdb",
 			ObjectType: "filesystem",
 			Schema:     "v9",
-			IndexTables: config.PeriodicTableConfig{
-				Prefix: indexTablePrefix,
-				Period: time.Hour * 24,
-			},
+			IndexTables: config.IndexPeriodicTableConfig{
+				PeriodicTableConfig: config.PeriodicTableConfig{
+					Prefix: indexTablePrefix,
+					Period: time.Hour * 24,
+				}},
 			RowShards: 16,
 		},
 		{
 			From:       dayFromTime(start.Add(25 * time.Hour)),
 			IndexType:  "boltdb",
 			ObjectType: "filesystem",
 			Schema:     "v12",
-			IndexTables: config.PeriodicTableConfig{
-				Prefix: indexTablePrefix,
-				Period: time.Hour * 24,
-			},
+			IndexTables: config.IndexPeriodicTableConfig{
+				PeriodicTableConfig: config.PeriodicTableConfig{
+					Prefix: indexTablePrefix,
+					Period: time.Hour * 24,
+				}},
 			RowShards: 16,
 		},
 		{
 			From:       dayFromTime(start.Add(73 * time.Hour)),
 			IndexType:  "tsdb",
 			ObjectType: "filesystem",
 			Schema:     "v12",
-			IndexTables: config.PeriodicTableConfig{
-				Prefix: tsdbIndexTablePrefix,
-				Period: time.Hour * 24,
-			},
+			IndexTables: config.IndexPeriodicTableConfig{
+				PeriodicTableConfig: config.PeriodicTableConfig{
+					Prefix: tsdbIndexTablePrefix,
+					Period: time.Hour * 24,
+				}},
 			RowShards: 16,
 		},
 		{
 			From:       dayFromTime(start.Add(100 * time.Hour)),
 			IndexType:  "tsdb",
 			ObjectType: "filesystem",
 			Schema:     "v12",
-			IndexTables: config.PeriodicTableConfig{
-				Prefix: indexTablePrefix,
-				Period: time.Hour * 24,
-			},
+			IndexTables: config.IndexPeriodicTableConfig{
+				PeriodicTableConfig: config.PeriodicTableConfig{
+					Prefix: indexTablePrefix,
+					Period: time.Hour * 24,
+				}},
 			RowShards: 16,
 		},
 	}}

pkg/compactor/retention/util_test.go

@@ -45,54 +45,59 @@ var (
 				IndexType:  "boltdb",
 				ObjectType: "filesystem",
 				Schema:     "v9",
-				IndexTables: config.PeriodicTableConfig{
-					Prefix: "index_",
-					Period: time.Hour * 24,
-				},
+				IndexTables: config.IndexPeriodicTableConfig{
+					PeriodicTableConfig: config.PeriodicTableConfig{
+						Prefix: "index_",
+						Period: time.Hour * 24,
+					}},
 				RowShards: 16,
 			},
 			{
 				From:       dayFromTime(start.Add(25 * time.Hour)),
 				IndexType:  "boltdb",
 				ObjectType: "filesystem",
 				Schema:     "v10",
-				IndexTables: config.PeriodicTableConfig{
-					Prefix: "index_",
-					Period: time.Hour * 24,
-				},
+				IndexTables: config.IndexPeriodicTableConfig{
+					PeriodicTableConfig: config.PeriodicTableConfig{
+						Prefix: "index_",
+						Period: time.Hour * 24,
+					}},
 				RowShards: 16,
 			},
 			{
 				From:       dayFromTime(start.Add(73 * time.Hour)),
 				IndexType:  "boltdb",
 				ObjectType: "filesystem",
 				Schema:     "v11",
-				IndexTables: config.PeriodicTableConfig{
-					Prefix: "index_",
-					Period: time.Hour * 24,
-				},
+				IndexTables: config.IndexPeriodicTableConfig{
+					PeriodicTableConfig: config.PeriodicTableConfig{
+						Prefix: "index_",
+						Period: time.Hour * 24,
+					}},
 				RowShards: 16,
 			},
 			{
 				From:       dayFromTime(start.Add(100 * time.Hour)),
 				IndexType:  "boltdb",
 				ObjectType: "filesystem",
 				Schema:     "v12",
-				IndexTables: config.PeriodicTableConfig{
-					Prefix: "index_",
-					Period: time.Hour * 24,
-				},
+				IndexTables: config.IndexPeriodicTableConfig{
+					PeriodicTableConfig: config.PeriodicTableConfig{
+						Prefix: "index_",
+						Period: time.Hour * 24,
+					}},
 				RowShards: 16,
 			},
 			{
 				From:       dayFromTime(start.Add(125 * time.Hour)),
 				IndexType:  "tsdb",
 				ObjectType: "filesystem",
 				Schema:     "v12",
-				IndexTables: config.PeriodicTableConfig{
-					Prefix: "index_",
-					Period: time.Hour * 24,
-				},
+				IndexTables: config.IndexPeriodicTableConfig{
+					PeriodicTableConfig: config.PeriodicTableConfig{
+						Prefix: "index_",
+						Period: time.Hour * 24,
+					}},
 				RowShards: 16,
 			},
 		},

pkg/loki/modules_test.go

@@ -382,9 +382,10 @@ func minimalWorkingConfig(t *testing.T, dir, target string, cfgTransformers ...f
 			{
 				IndexType:  config.BoltDBShipperType,
 				ObjectType: config.StorageTypeFileSystem,
-				IndexTables: config.PeriodicTableConfig{
-					Period: time.Hour * 24,
-				},
+				IndexTables: config.IndexPeriodicTableConfig{
+					PeriodicTableConfig: config.PeriodicTableConfig{
+						Period: time.Hour * 24,
+					}},
 				RowShards: 16,
 				Schema:    "v11",
 				From: config.DayTime{

pkg/storage/chunk/client/aws/metrics_autoscaling_test.go

@@ -156,9 +156,10 @@ func TestTableManagerMetricsAutoScaling(t *testing.T) {
 		Configs: []config.PeriodConfig{
 			{
 				IndexType: "aws-dynamo",
-				IndexTables: config.PeriodicTableConfig{
-					Prefix: "a",
-				},
+				IndexTables: config.IndexPeriodicTableConfig{
+					PeriodicTableConfig: config.PeriodicTableConfig{
+						Prefix: "a",
+					}},
 			},
 			{
 				IndexType:   "aws-dynamo",
@@ -315,9 +316,10 @@ func TestTableManagerMetricsReadAutoScaling(t *testing.T) {
 		Configs: []config.PeriodConfig{
 			{
 				IndexType: "aws-dynamo",
-				IndexTables: config.PeriodicTableConfig{
-					Prefix: "a",
-				},
+				IndexTables: config.IndexPeriodicTableConfig{
+					PeriodicTableConfig: config.PeriodicTableConfig{
+						Prefix: "a",
+					}},
 			},
 			{
 				IndexType:   "aws-dynamo",

pkg/storage/chunk/client/grpc/grpc_client_test.go

@@ -27,11 +27,12 @@ func TestGrpcStore(t *testing.T) {
 			IndexType:  "grpc-store",
 			ObjectType: "grpc-store",
 			Schema:     "v10",
-			IndexTables: config.PeriodicTableConfig{
-				Prefix: "index_",
-				Period: 604800000000000,
-				Tags:   nil,
-			},
+			IndexTables: config.IndexPeriodicTableConfig{
+				PeriodicTableConfig: config.PeriodicTableConfig{
+					Prefix: "index_",
+					Period: 604800000000000,
+					Tags:   nil,
+				}},
 			RowShards: 16,
 		},
 	}}

pkg/storage/chunk/client/local/fixtures.go

@@ -58,10 +58,11 @@ func (f *fixture) Clients() (
 				Prefix: "chunks",
 				Period: 10 * time.Minute,
 			},
-			IndexTables: config.PeriodicTableConfig{
-				Prefix: "index",
-				Period: 10 * time.Minute,
-			},
+			IndexTables: config.IndexPeriodicTableConfig{
+				PeriodicTableConfig: config.PeriodicTableConfig{
+					Prefix: "index",
+					Period: 10 * time.Minute,
+				}},
 		}},
 	}
 

pkg/storage/chunk/client/testutils/testutils.go

@@ -120,10 +120,11 @@ func SchemaConfig(store, schema string, from model.Time) config.SchemaConfig {
 				Prefix: "cortex",
 				Period: 7 * 24 * time.Hour,
 			},
-			IndexTables: config.PeriodicTableConfig{
-				Prefix: "cortex_chunks",
-				Period: 7 * 24 * time.Hour,
-			},
+			IndexTables: config.IndexPeriodicTableConfig{
+				PeriodicTableConfig: config.PeriodicTableConfig{
+					Prefix: "cortex_chunks",
+					Period: 7 * 24 * time.Hour,
+				}},
 		}},
 	}
 	if err := s.Validate(); err != nil {

pkg/storage/config/schema_config.go

@@ -51,6 +51,8 @@ const (
 
 	// ObjectStorageIndexRequiredPeriod defines the required index period for object storage based index stores like boltdb-shipper and tsdb
 	ObjectStorageIndexRequiredPeriod = 24 * time.Hour
+
+	pathPrefixDelimiter = "/"
 )
 
 var (
@@ -158,11 +160,11 @@ type PeriodConfig struct {
 	// type of index client to use.
 	IndexType string `yaml:"store" doc:"description=store and object_store below affect which <storage_config> key is used. Which index to use. Either tsdb or boltdb-shipper. Following stores are deprecated: aws, aws-dynamo, gcp, gcp-columnkey, bigtable, bigtable-hashed, cassandra, grpc."`
 	// type of object client to use.
-	ObjectType  string              `yaml:"object_store" doc:"description=Which store to use for the chunks. Either aws (alias s3), azure, gcs, alibabacloud, bos, cos, swift, filesystem, or a named_store (refer to named_stores_config). Following stores are deprecated: aws-dynamo, gcp, gcp-columnkey, bigtable, bigtable-hashed, cassandra, grpc."`
-	Schema      string              `yaml:"schema" doc:"description=The schema version to use, current recommended schema is v12."`
-	IndexTables PeriodicTableConfig `yaml:"index" doc:"description=Configures how the index is updated and stored."`
-	ChunkTables PeriodicTableConfig `yaml:"chunks" doc:"description=Configured how the chunks are updated and stored."`
-	RowShards   uint32              `yaml:"row_shards" doc:"description=How many shards will be created. Only used if schema is v10 or greater."`
+	ObjectType  string                   `yaml:"object_store" doc:"description=Which store to use for the chunks. Either aws (alias s3), azure, gcs, alibabacloud, bos, cos, swift, filesystem, or a named_store (refer to named_stores_config). Following stores are deprecated: aws-dynamo, gcp, gcp-columnkey, bigtable, bigtable-hashed, cassandra, grpc."`
+	Schema      string                   `yaml:"schema" doc:"description=The schema version to use, current recommended schema is v12."`
+	IndexTables IndexPeriodicTableConfig `yaml:"index" doc:"description=Configures how the index is updated and stored."`
+	ChunkTables PeriodicTableConfig      `yaml:"chunks" doc:"description=Configured how the chunks are updated and stored."`
+	RowShards   uint32                   `yaml:"row_shards" doc:"description=How many shards will be created. Only used if schema is v10 or greater."`
 
 	// Integer representation of schema used for hot path calculation. Populated on unmarshaling.
 	schemaInt *int `yaml:"-"`
@@ -368,6 +370,10 @@ func validateChunks(cfg PeriodConfig) error {
 }
 
 func (cfg *PeriodConfig) applyDefaults() {
+	if cfg.IndexTables.PathPrefix == "" {
+		cfg.IndexTables.PathPrefix = "/index"
+	}
+
 	if cfg.RowShards == 0 {
 		cfg.RowShards = defaultRowShards(cfg.Schema)
 	}
@@ -416,9 +422,8 @@ func (cfg PeriodConfig) validate() error {
 		return errTSDBNon24HoursIndexPeriod
 	}
 
-	// Ensure the tables period is a multiple of the bucket period
-	if cfg.IndexTables.Period > 0 && cfg.IndexTables.Period%(24*time.Hour) != 0 {
-		return errInvalidTablePeriod
+	if err := cfg.IndexTables.Validate(); err != nil {
+		return err
 	}
 
 	if cfg.ChunkTables.Period > 0 && cfg.ChunkTables.Period%(24*time.Hour) != 0 {
@@ -472,6 +477,36 @@ func (cfg *PeriodConfig) VersionAsInt() (int, error) {
 	return n, err
 }
 
+type IndexPeriodicTableConfig struct {
+	PathPrefix          string `yaml:"path_prefix" doc:"default=/index|description=Path prefix for index tables. Prefix always needs to end with a path delimiter '/', except when the prefix is empty."`
+	PeriodicTableConfig `yaml:",inline"`
+}
+
+func (cfg *IndexPeriodicTableConfig) Validate() error {
+	// Ensure the tables period is a multiple of the bucket period
+	if cfg.Period > 0 && cfg.Period%(24*time.Hour) != 0 {
+		return errInvalidTablePeriod
+	}
+
+	return ValidatePathPrefix(cfg.PathPrefix)
+}
+
+func ValidatePathPrefix(prefix string) error {
+	if prefix == "" {
+		return errors.New("prefix must be set")
+	} else if strings.Contains(prefix, "\\") {
+		// When using windows filesystem as object store the implementation of ObjectClient in Cortex takes care of conversion of separator.
+		// We just need to always use `/` as a path separator.
+		return fmt.Errorf("prefix should only have '%s' as a path separator", pathPrefixDelimiter)
+	} else if strings.HasPrefix(prefix, pathPrefixDelimiter) {
+		return errors.New("prefix should never start with a path separator i.e '/'")
+	} else if !strings.HasSuffix(prefix, pathPrefixDelimiter) {
+		return errors.New("prefix should end with a path separator i.e '/'")
+	}
+
+	return nil
+}
+
 // PeriodicTableConfig is configuration for a set of time-sharded tables.
 type PeriodicTableConfig struct {
 	Prefix string        `yaml:"prefix" doc:"description=Table prefix for all period tables."`