Open
Description
Describe the bug
The upgrade from v3.1.1 to v3.3.0 fails with the following error message:
INF ring.go:316 > | component=distributor msg=ring doesn't exist in KV store yet
ERR ratestore.go:109 > | err=empty ring msg=error getting ingester clients ts=2024-11-20T07:28:06.085153173Z
WRN logging.go:128 > | msg=POST /loki/api/v1/push (500) 504.95µs Response: "empty ring\n" ws: false; Content-Length: 633; Content-Type: application/x-protobuf; User-Agent: Alloy/v1.5.0 (linux; rpm)
loki confg:
auth_enabled: false
analytics:
reporting_enabled: false
server:
http_listen_port:3100
http_server_write_timeout:10m
http_server_read_timeout:10m
http_server_idle_timeout:10m
graceful_shutdown_timeout:2m
grpc_listen_port:9095
grpc_server_max_recv_msg_size:838866080
grpc_server_max_send_msg_size:838866080
log_level:info
log_format:json
common:
compactor_address:http://loki:3100
path_prefix:/loki
replication_factor:1
storage:
s3:
bucketnames:loki-data
endpoint: ${LOKI_S3_ENDPOINT:-minio:9000}
access_key_id: ${LOKI_S3_ACCESS_KEY_ID:-lgtmp}
secret_access_key: ${LOKI_S3_SECRET_ACCESS_KEY:-supersecret}
insecure: ${LOKI_S3_INSECURE:-true}
s3forcepathstyle: true
http_config:
insecure_skip_verify: true
ring:
kvstore:
store: inmemory
bloom_gateway:
enabled: true
client:
addresses: "dns+loki:9095"
cache_results: true
results_cache:
cache:
memcached_client:
addresses: "dns+memcached:11211"
index_gateway:
mode:simple
compactor:
working_directory:/tmp/compactor
memberlist:
join_members:
-loki-memberlist:7946
# https://github.com/grafana/loki/blob/main/docs/sources/configure/_index.md#query_range
query_range:
align_queries_with_step: true
cache_results: true
results_cache:
cache:
memcached_client:
addresses: "dns+memcached:11211"
cache_index_stats_results: true
index_stats_results_cache:
cache:
memcached_client:
addresses: "dns+memcached:11211"
# cache_volume_results: true
# cache_series_results: true
# instant_metric_query_split_align: true
# cache_instant_metric_results: true
# instant_metric_results_cache:
# cache:
# memcached_client:
# addresses: "dns+memcached:11211"
# series_results_cache:
# cache:
# memcached_client:
# addresses: dns+memcached:11211"
# volume_results_cache:
# cache:
# memcached_client:
# addresses: "dns+memcached:11211"
pattern_ingester:
enabled: true
limits_config:
max_global_streams_per_user:0
ingestion_rate_mb:50000
ingestion_burst_size_mb:50000
volume_enabled: true
allow_structured_metadata: true
reject_old_samples: false
retention_period:672h
reject_old_samples_max_age:672h
split_queries_by_interval:24h
max_query_parallelism:32
ruler:
storage:
s3:
bucketnames:loki-ruler
type:s3
runtime_config:
file:/etc/loki/configs/runtime.yaml
schema_config:
configs:
- from: "2024-10-24"
index:
prefix:loki_index_
period:24h
object_store:s3
store:tsdb
schema:v13
storage_config:
tsdb_shipper:
active_index_directory:/loki/index
cache_location:/loki/cache
index_gateway_client:
server_address: "dns+loki:9095"
chunk_store_config:
chunk_cache_config:
memcached_client:
addresses: "dns+memcached:11211"
To Reproduce
Steps to reproduce the behavior:
Upgrade attempt v3.3.3 with this configuration
Expected behavior
Flawless upgrade to v3.3.3