Skip to content

Upgrade fails v3.1.1 to v3.3.0 with err=empty ring msg=error getting ingester clients, status unhealthy #15030

Open
@StefanSa

Description

@StefanSa

Describe the bug
The upgrade from v3.1.1 to v3.3.0 fails with the following error message:

INF ring.go:316 > | component=distributor msg=ring doesn't exist in KV store yet
ERR ratestore.go:109 > | err=empty ring msg=error getting ingester clients ts=2024-11-20T07:28:06.085153173Z
WRN logging.go:128 > | msg=POST /loki/api/v1/push (500) 504.95µs Response: "empty ring\n" ws: false; Content-Length: 633; Content-Type: application/x-protobuf; User-Agent: Alloy/v1.5.0 (linux; rpm)

loki confg:

auth_enabled: false

analytics:
 reporting_enabled: false

server:
http_listen_port:3100
http_server_write_timeout:10m
http_server_read_timeout:10m
http_server_idle_timeout:10m
graceful_shutdown_timeout:2m
grpc_listen_port:9095
grpc_server_max_recv_msg_size:838866080
grpc_server_max_send_msg_size:838866080
log_level:info
log_format:json

common:
compactor_address:http://loki:3100
path_prefix:/loki
replication_factor:1
  storage:
    s3:
bucketnames:loki-data
      endpoint: ${LOKI_S3_ENDPOINT:-minio:9000}
      access_key_id: ${LOKI_S3_ACCESS_KEY_ID:-lgtmp}
      secret_access_key: ${LOKI_S3_SECRET_ACCESS_KEY:-supersecret}
      insecure: ${LOKI_S3_INSECURE:-true}
      s3forcepathstyle: true
      http_config:
        insecure_skip_verify: true
  ring:
    kvstore:
       store: inmemory

bloom_gateway:
  enabled: true
  client:
    addresses: "dns+loki:9095"
    cache_results: true
    results_cache:
      cache:
        memcached_client:
          addresses: "dns+memcached:11211"

index_gateway:
mode:simple

compactor:
working_directory:/tmp/compactor

memberlist:
  join_members:
  -loki-memberlist:7946

# https://github.com/grafana/loki/blob/main/docs/sources/configure/_index.md#query_range
query_range:
  align_queries_with_step: true

  cache_results: true
  results_cache:
    cache:
      memcached_client:
        addresses: "dns+memcached:11211"

  cache_index_stats_results: true
  index_stats_results_cache:
    cache:
      memcached_client:
        addresses: "dns+memcached:11211"
  # cache_volume_results: true
  # cache_series_results: true
  # instant_metric_query_split_align: true
  # cache_instant_metric_results: true
  # instant_metric_results_cache:
  #   cache:
  #     memcached_client:
  #       addresses: "dns+memcached:11211"
  # series_results_cache:
  #   cache:
  #     memcached_client:
  #       addresses: dns+memcached:11211"
  # volume_results_cache:
  #   cache:
  #     memcached_client:
  #       addresses: "dns+memcached:11211"

pattern_ingester:
  enabled: true


limits_config:
max_global_streams_per_user:0
ingestion_rate_mb:50000
ingestion_burst_size_mb:50000
  volume_enabled: true
  allow_structured_metadata: true
  reject_old_samples: false
retention_period:672h
reject_old_samples_max_age:672h
split_queries_by_interval:24h
max_query_parallelism:32

ruler:
  storage:
    s3:
bucketnames:loki-ruler
type:s3

runtime_config:
file:/etc/loki/configs/runtime.yaml

schema_config:
  configs:
  - from: "2024-10-24"
    index:
prefix:loki_index_
period:24h
object_store:s3
store:tsdb
schema:v13

storage_config:
  tsdb_shipper:
active_index_directory:/loki/index
cache_location:/loki/cache
    index_gateway_client:
      server_address: "dns+loki:9095"

chunk_store_config:
  chunk_cache_config:
    memcached_client:
      addresses: "dns+memcached:11211"

To Reproduce
Steps to reproduce the behavior:
Upgrade attempt v3.3.3 with this configuration

Expected behavior
Flawless upgrade to v3.3.3

Metadata

Metadata

Assignees

No one assigned

    Type

    No type

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions