diff --git a/.github/workflows/test-build-deploy.yml b/.github/workflows/test-build-deploy.yml index 469f98d4f0e..faa04fa59fb 100644 --- a/.github/workflows/test-build-deploy.yml +++ b/.github/workflows/test-build-deploy.yml @@ -140,6 +140,7 @@ jobs: docker pull quay.io/cortexproject/cortex:v1.14.0 fi docker pull memcached:1.6.1 + docker pull redis:7.0.4-alpine env: TEST_TAGS: ${{ matrix.tags }} - name: Integration Tests diff --git a/CHANGELOG.md b/CHANGELOG.md index 91a959e6050..2911d8ca82c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,7 @@ * [FEATURE] Querier/Ruler: Support the new thanos promql engine. This is an experimental feature and might change in the future. #5093 * [FEATURE] Added zstd as an option for grpc compression #5092 * [FEATURE] Ring: Add new kv store option `dynamodb`. #5026 +* [FEATURE] Cache: Support redis as backend for caching bucket and index cache. #5057 * [BUGFIX] Updated `golang.org/x/net` dependency to fix CVE-2022-27664. #5008 * [BUGFIX] Fix panic when otel and xray tracing is enabled. #5044 * [BUGFIX] Fixed no compact block got grouped in shuffle sharding grouper. #5055 diff --git a/docs/blocks-storage/querier.md b/docs/blocks-storage/querier.md index 96215459656..7dee1d32fa7 100644 --- a/docs/blocks-storage/querier.md +++ b/docs/blocks-storage/querier.md @@ -519,7 +519,8 @@ blocks_storage: [consistency_delay: | default = 0s] index_cache: - # The index cache backend type. Supported values: inmemory, memcached. + # The index cache backend type. Supported values: inmemory, memcached, + # redis. # CLI flag: -blocks-storage.bucket-store.index-cache.backend [backend: | default = "inmemory"] @@ -576,6 +577,113 @@ blocks_storage: # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.auto-discovery [auto_discovery: | default = false] + redis: + # Comma separated list of redis addresses. Supported prefixes are: dns+ + # (looked up as an A/AAAA query), dnssrv+ (looked up as a SRV query, + # dnssrvnoa+ (looked up as a SRV query, with no A/AAAA lookup made after + # that). + # CLI flag: -blocks-storage.bucket-store.index-cache.redis.addresses + [addresses: | default = ""] + + # Redis username. + # CLI flag: -blocks-storage.bucket-store.index-cache.redis.username + [username: | default = ""] + + # Redis password. + # CLI flag: -blocks-storage.bucket-store.index-cache.redis.password + [password: | default = ""] + + # Database to be selected after connecting to the server. + # CLI flag: -blocks-storage.bucket-store.index-cache.redis.db + [db: | default = 0] + + # Specifies the master's name. Must be not empty for Redis Sentinel. + # CLI flag: -blocks-storage.bucket-store.index-cache.redis.master-name + [master_name: | default = ""] + + # Maximum number of socket connections. + # CLI flag: -blocks-storage.bucket-store.index-cache.redis.pool-size + [pool_size: | default = 100] + + # Specifies the minimum number of idle connections, which is useful when + # it is slow to establish new connections. + # CLI flag: -blocks-storage.bucket-store.index-cache.redis.min-idle-conns + [min_idle_conns: | default = 10] + + # The maximum number of concurrent GetMulti() operations. If set to 0, + # concurrency is unlimited. + # CLI flag: -blocks-storage.bucket-store.index-cache.redis.max-get-multi-concurrency + [max_get_multi_concurrency: | default = 100] + + # The maximum size per batch for mget. + # CLI flag: -blocks-storage.bucket-store.index-cache.redis.get-multi-batch-size + [get_multi_batch_size: | default = 100] + + # The maximum number of concurrent SetMulti() operations. If set to 0, + # concurrency is unlimited. + # CLI flag: -blocks-storage.bucket-store.index-cache.redis.max-set-multi-concurrency + [max_set_multi_concurrency: | default = 100] + + # The maximum size per batch for pipeline set. + # CLI flag: -blocks-storage.bucket-store.index-cache.redis.set-multi-batch-size + [set_multi_batch_size: | default = 100] + + # Client dial timeout. + # CLI flag: -blocks-storage.bucket-store.index-cache.redis.dial-timeout + [dial_timeout: | default = 5s] + + # Client read timeout. + # CLI flag: -blocks-storage.bucket-store.index-cache.redis.read-timeout + [read_timeout: | default = 3s] + + # Client write timeout. + # CLI flag: -blocks-storage.bucket-store.index-cache.redis.write-timeout + [write_timeout: | default = 3s] + + # Amount of time after which client closes idle connections. Should be + # less than server's timeout. -1 disables idle timeout check. + # CLI flag: -blocks-storage.bucket-store.index-cache.redis.idle-timeout + [idle_timeout: | default = 5m] + + # Connection age at which client retires (closes) the connection. + # Default 0 is to not close aged connections. + # CLI flag: -blocks-storage.bucket-store.index-cache.redis.max-conn-age + [max_conn_age: | default = 0s] + + # Whether to enable tls for redis connection. + # CLI flag: -blocks-storage.bucket-store.index-cache.redis.tls-enabled + [tls_enabled: | default = false] + + # Path to the client certificate file, which will be used for + # authenticating with the server. Also requires the key path to be + # configured. + # CLI flag: -blocks-storage.bucket-store.index-cache.redis..tls-cert-path + [tls_cert_path: | default = ""] + + # Path to the key file for the client certificate. Also requires the + # client certificate to be configured. + # CLI flag: -blocks-storage.bucket-store.index-cache.redis..tls-key-path + [tls_key_path: | default = ""] + + # Path to the CA certificates file to validate server certificate + # against. If not set, the host's root CA certificates are used. + # CLI flag: -blocks-storage.bucket-store.index-cache.redis..tls-ca-path + [tls_ca_path: | default = ""] + + # Override the expected name on the server certificate. + # CLI flag: -blocks-storage.bucket-store.index-cache.redis..tls-server-name + [tls_server_name: | default = ""] + + # Skip validating server certificate. + # CLI flag: -blocks-storage.bucket-store.index-cache.redis..tls-insecure-skip-verify + [tls_insecure_skip_verify: | default = false] + + # If not zero then client-side caching is enabled. Client-side caching + # is when data is stored in memory instead of fetching data each time. + # See https://redis.io/docs/manual/client-side-caching/ for more info. + # CLI flag: -blocks-storage.bucket-store.index-cache.redis.cache-size + [cache_size: | default = 0] + chunks_cache: # Backend for chunks cache, if not empty. Supported values: memcached. # CLI flag: -blocks-storage.bucket-store.chunks-cache.backend @@ -628,6 +736,113 @@ blocks_storage: # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.auto-discovery [auto_discovery: | default = false] + redis: + # Comma separated list of redis addresses. Supported prefixes are: dns+ + # (looked up as an A/AAAA query), dnssrv+ (looked up as a SRV query, + # dnssrvnoa+ (looked up as a SRV query, with no A/AAAA lookup made after + # that). + # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.addresses + [addresses: | default = ""] + + # Redis username. + # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.username + [username: | default = ""] + + # Redis password. + # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.password + [password: | default = ""] + + # Database to be selected after connecting to the server. + # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.db + [db: | default = 0] + + # Specifies the master's name. Must be not empty for Redis Sentinel. + # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.master-name + [master_name: | default = ""] + + # Maximum number of socket connections. + # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.pool-size + [pool_size: | default = 100] + + # Specifies the minimum number of idle connections, which is useful when + # it is slow to establish new connections. + # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.min-idle-conns + [min_idle_conns: | default = 10] + + # The maximum number of concurrent GetMulti() operations. If set to 0, + # concurrency is unlimited. + # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.max-get-multi-concurrency + [max_get_multi_concurrency: | default = 100] + + # The maximum size per batch for mget. + # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.get-multi-batch-size + [get_multi_batch_size: | default = 100] + + # The maximum number of concurrent SetMulti() operations. If set to 0, + # concurrency is unlimited. + # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.max-set-multi-concurrency + [max_set_multi_concurrency: | default = 100] + + # The maximum size per batch for pipeline set. + # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.set-multi-batch-size + [set_multi_batch_size: | default = 100] + + # Client dial timeout. + # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.dial-timeout + [dial_timeout: | default = 5s] + + # Client read timeout. + # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.read-timeout + [read_timeout: | default = 3s] + + # Client write timeout. + # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.write-timeout + [write_timeout: | default = 3s] + + # Amount of time after which client closes idle connections. Should be + # less than server's timeout. -1 disables idle timeout check. + # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.idle-timeout + [idle_timeout: | default = 5m] + + # Connection age at which client retires (closes) the connection. + # Default 0 is to not close aged connections. + # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.max-conn-age + [max_conn_age: | default = 0s] + + # Whether to enable tls for redis connection. + # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.tls-enabled + [tls_enabled: | default = false] + + # Path to the client certificate file, which will be used for + # authenticating with the server. Also requires the key path to be + # configured. + # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis..tls-cert-path + [tls_cert_path: | default = ""] + + # Path to the key file for the client certificate. Also requires the + # client certificate to be configured. + # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis..tls-key-path + [tls_key_path: | default = ""] + + # Path to the CA certificates file to validate server certificate + # against. If not set, the host's root CA certificates are used. + # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis..tls-ca-path + [tls_ca_path: | default = ""] + + # Override the expected name on the server certificate. + # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis..tls-server-name + [tls_server_name: | default = ""] + + # Skip validating server certificate. + # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis..tls-insecure-skip-verify + [tls_insecure_skip_verify: | default = false] + + # If not zero then client-side caching is enabled. Client-side caching + # is when data is stored in memory instead of fetching data each time. + # See https://redis.io/docs/manual/client-side-caching/ for more info. + # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.cache-size + [cache_size: | default = 0] + # Size of each subrange that bucket object is split into for better # caching. # CLI flag: -blocks-storage.bucket-store.chunks-cache.subrange-size @@ -699,6 +914,113 @@ blocks_storage: # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.auto-discovery [auto_discovery: | default = false] + redis: + # Comma separated list of redis addresses. Supported prefixes are: dns+ + # (looked up as an A/AAAA query), dnssrv+ (looked up as a SRV query, + # dnssrvnoa+ (looked up as a SRV query, with no A/AAAA lookup made after + # that). + # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.addresses + [addresses: | default = ""] + + # Redis username. + # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.username + [username: | default = ""] + + # Redis password. + # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.password + [password: | default = ""] + + # Database to be selected after connecting to the server. + # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.db + [db: | default = 0] + + # Specifies the master's name. Must be not empty for Redis Sentinel. + # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.master-name + [master_name: | default = ""] + + # Maximum number of socket connections. + # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.pool-size + [pool_size: | default = 100] + + # Specifies the minimum number of idle connections, which is useful when + # it is slow to establish new connections. + # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.min-idle-conns + [min_idle_conns: | default = 10] + + # The maximum number of concurrent GetMulti() operations. If set to 0, + # concurrency is unlimited. + # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.max-get-multi-concurrency + [max_get_multi_concurrency: | default = 100] + + # The maximum size per batch for mget. + # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.get-multi-batch-size + [get_multi_batch_size: | default = 100] + + # The maximum number of concurrent SetMulti() operations. If set to 0, + # concurrency is unlimited. + # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.max-set-multi-concurrency + [max_set_multi_concurrency: | default = 100] + + # The maximum size per batch for pipeline set. + # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.set-multi-batch-size + [set_multi_batch_size: | default = 100] + + # Client dial timeout. + # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.dial-timeout + [dial_timeout: | default = 5s] + + # Client read timeout. + # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.read-timeout + [read_timeout: | default = 3s] + + # Client write timeout. + # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.write-timeout + [write_timeout: | default = 3s] + + # Amount of time after which client closes idle connections. Should be + # less than server's timeout. -1 disables idle timeout check. + # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.idle-timeout + [idle_timeout: | default = 5m] + + # Connection age at which client retires (closes) the connection. + # Default 0 is to not close aged connections. + # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.max-conn-age + [max_conn_age: | default = 0s] + + # Whether to enable tls for redis connection. + # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.tls-enabled + [tls_enabled: | default = false] + + # Path to the client certificate file, which will be used for + # authenticating with the server. Also requires the key path to be + # configured. + # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis..tls-cert-path + [tls_cert_path: | default = ""] + + # Path to the key file for the client certificate. Also requires the + # client certificate to be configured. + # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis..tls-key-path + [tls_key_path: | default = ""] + + # Path to the CA certificates file to validate server certificate + # against. If not set, the host's root CA certificates are used. + # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis..tls-ca-path + [tls_ca_path: | default = ""] + + # Override the expected name on the server certificate. + # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis..tls-server-name + [tls_server_name: | default = ""] + + # Skip validating server certificate. + # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis..tls-insecure-skip-verify + [tls_insecure_skip_verify: | default = false] + + # If not zero then client-side caching is enabled. Client-side caching + # is when data is stored in memory instead of fetching data each time. + # See https://redis.io/docs/manual/client-side-caching/ for more info. + # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.cache-size + [cache_size: | default = 0] + # How long to cache list of tenants in the bucket. # CLI flag: -blocks-storage.bucket-store.metadata-cache.tenants-list-ttl [tenants_list_ttl: | default = 15m] diff --git a/docs/blocks-storage/store-gateway.md b/docs/blocks-storage/store-gateway.md index 26f645bb775..23b8295f35a 100644 --- a/docs/blocks-storage/store-gateway.md +++ b/docs/blocks-storage/store-gateway.md @@ -117,6 +117,7 @@ The store-gateway can use a cache to speed up lookups of postings and series fro - `inmemory` - `memcached` +- `redis` #### In-memory index cache @@ -139,20 +140,26 @@ The Memcached client uses a jump hash algorithm to shard cached entries across a For example, if you're running Memcached in Kubernetes, you may: 1. Deploy your Memcached cluster using a [StatefulSet](https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/) -2. Create an [headless service](https://kubernetes.io/docs/concepts/services-networking/service/#headless-services) for Memcached StatefulSet +2. Create a [headless service](https://kubernetes.io/docs/concepts/services-networking/service/#headless-services) for Memcached StatefulSet 3. Configure the Cortex's Memcached client address using the `dnssrvnoa+` [service discovery](../configuration/arguments.md#dns-service-discovery) +#### Redis index cache + +The `redis` index cache allows to use [Redis](https://memcached.org/) as cache backend. This cache backend is configured using `-blocks-storage.bucket-store.index-cache.backend=redis` and requires the Redis server(s) addresses via `-blocks-storage.bucket-store.index-cache.redis.addresses` (or config file). + +Using `redis` as the cache backend has similar trade-offs as using `memcached` cache backend. However, client side caching can be enabled when using `redis` backend to avoid Store Gateway fetching data from cache each time. See [here](https://redis.io/docs/manual/client-side-caching/) for more info and it can be enabled by setting flag `-blocks-storage.bucket-store.index-cache.redis.cache-size` > 0. + ### Chunks cache Store-gateway can also use a cache for storing chunks fetched from the storage. Chunks contain actual samples, and can be reused if user query hits the same series for the same time range. -To enable chunks cache, please set `-blocks-storage.bucket-store.chunks-cache.backend`. Chunks can currently only be stored into Memcached cache. Memcached client can be configured via flags with `-blocks-storage.bucket-store.chunks-cache.memcached.*` prefix. +To enable chunks cache, please set `-blocks-storage.bucket-store.chunks-cache.backend`. Chunks can be stored into Memcached or Redis cache. Memcached client can be configured via flags with `-blocks-storage.bucket-store.chunks-cache.memcached.*` prefix. Redis client can be configured via flags with `-blocks-storage.bucket-store.chunks-cache.redis.*` prefix. There are additional low-level options for configuring chunks cache. Please refer to other flags with `-blocks-storage.bucket-store.chunks-cache.*` prefix. ### Metadata cache -Store-gateway and [querier](./querier.md) can use memcached for caching bucket metadata: +Store-gateway and [querier](./querier.md) can use memcached or redis for caching bucket metadata: - List of tenants - List of blocks per tenant @@ -162,11 +169,11 @@ Store-gateway and [querier](./querier.md) can use memcached for caching bucket m Using the metadata cache can significantly reduce the number of API calls to object storage and protects from linearly scale the number of these API calls with the number of querier and store-gateway instances (because the bucket is periodically scanned and synched by each querier and store-gateway). -To enable metadata cache, please set `-blocks-storage.bucket-store.metadata-cache.backend`. Only `memcached` backend is supported currently. Memcached client has additional configuration available via flags with `-blocks-storage.bucket-store.metadata-cache.memcached.*` prefix. +To enable metadata cache, please set `-blocks-storage.bucket-store.metadata-cache.backend`. `memcached` and `redis` backend are supported currently. Memcached client has additional configuration available via flags with `-blocks-storage.bucket-store.metadata-cache.memcached.*` prefix. Redis client has additional configuration available via flags with `-blocks-storage.bucket-store.metadata-cache.redis.*` prefix. Additional options for configuring metadata cache have `-blocks-storage.bucket-store.metadata-cache.*` prefix. By configuring TTL to zero or negative value, caching of given item type is disabled. -_The same memcached backend cluster should be shared between store-gateways and queriers._ +_The same cache backend deployment should be shared between store-gateways and queriers._ ## Store-gateway HTTP endpoints @@ -589,7 +596,8 @@ blocks_storage: [consistency_delay: | default = 0s] index_cache: - # The index cache backend type. Supported values: inmemory, memcached. + # The index cache backend type. Supported values: inmemory, memcached, + # redis. # CLI flag: -blocks-storage.bucket-store.index-cache.backend [backend: | default = "inmemory"] @@ -646,6 +654,113 @@ blocks_storage: # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.auto-discovery [auto_discovery: | default = false] + redis: + # Comma separated list of redis addresses. Supported prefixes are: dns+ + # (looked up as an A/AAAA query), dnssrv+ (looked up as a SRV query, + # dnssrvnoa+ (looked up as a SRV query, with no A/AAAA lookup made after + # that). + # CLI flag: -blocks-storage.bucket-store.index-cache.redis.addresses + [addresses: | default = ""] + + # Redis username. + # CLI flag: -blocks-storage.bucket-store.index-cache.redis.username + [username: | default = ""] + + # Redis password. + # CLI flag: -blocks-storage.bucket-store.index-cache.redis.password + [password: | default = ""] + + # Database to be selected after connecting to the server. + # CLI flag: -blocks-storage.bucket-store.index-cache.redis.db + [db: | default = 0] + + # Specifies the master's name. Must be not empty for Redis Sentinel. + # CLI flag: -blocks-storage.bucket-store.index-cache.redis.master-name + [master_name: | default = ""] + + # Maximum number of socket connections. + # CLI flag: -blocks-storage.bucket-store.index-cache.redis.pool-size + [pool_size: | default = 100] + + # Specifies the minimum number of idle connections, which is useful when + # it is slow to establish new connections. + # CLI flag: -blocks-storage.bucket-store.index-cache.redis.min-idle-conns + [min_idle_conns: | default = 10] + + # The maximum number of concurrent GetMulti() operations. If set to 0, + # concurrency is unlimited. + # CLI flag: -blocks-storage.bucket-store.index-cache.redis.max-get-multi-concurrency + [max_get_multi_concurrency: | default = 100] + + # The maximum size per batch for mget. + # CLI flag: -blocks-storage.bucket-store.index-cache.redis.get-multi-batch-size + [get_multi_batch_size: | default = 100] + + # The maximum number of concurrent SetMulti() operations. If set to 0, + # concurrency is unlimited. + # CLI flag: -blocks-storage.bucket-store.index-cache.redis.max-set-multi-concurrency + [max_set_multi_concurrency: | default = 100] + + # The maximum size per batch for pipeline set. + # CLI flag: -blocks-storage.bucket-store.index-cache.redis.set-multi-batch-size + [set_multi_batch_size: | default = 100] + + # Client dial timeout. + # CLI flag: -blocks-storage.bucket-store.index-cache.redis.dial-timeout + [dial_timeout: | default = 5s] + + # Client read timeout. + # CLI flag: -blocks-storage.bucket-store.index-cache.redis.read-timeout + [read_timeout: | default = 3s] + + # Client write timeout. + # CLI flag: -blocks-storage.bucket-store.index-cache.redis.write-timeout + [write_timeout: | default = 3s] + + # Amount of time after which client closes idle connections. Should be + # less than server's timeout. -1 disables idle timeout check. + # CLI flag: -blocks-storage.bucket-store.index-cache.redis.idle-timeout + [idle_timeout: | default = 5m] + + # Connection age at which client retires (closes) the connection. + # Default 0 is to not close aged connections. + # CLI flag: -blocks-storage.bucket-store.index-cache.redis.max-conn-age + [max_conn_age: | default = 0s] + + # Whether to enable tls for redis connection. + # CLI flag: -blocks-storage.bucket-store.index-cache.redis.tls-enabled + [tls_enabled: | default = false] + + # Path to the client certificate file, which will be used for + # authenticating with the server. Also requires the key path to be + # configured. + # CLI flag: -blocks-storage.bucket-store.index-cache.redis..tls-cert-path + [tls_cert_path: | default = ""] + + # Path to the key file for the client certificate. Also requires the + # client certificate to be configured. + # CLI flag: -blocks-storage.bucket-store.index-cache.redis..tls-key-path + [tls_key_path: | default = ""] + + # Path to the CA certificates file to validate server certificate + # against. If not set, the host's root CA certificates are used. + # CLI flag: -blocks-storage.bucket-store.index-cache.redis..tls-ca-path + [tls_ca_path: | default = ""] + + # Override the expected name on the server certificate. + # CLI flag: -blocks-storage.bucket-store.index-cache.redis..tls-server-name + [tls_server_name: | default = ""] + + # Skip validating server certificate. + # CLI flag: -blocks-storage.bucket-store.index-cache.redis..tls-insecure-skip-verify + [tls_insecure_skip_verify: | default = false] + + # If not zero then client-side caching is enabled. Client-side caching + # is when data is stored in memory instead of fetching data each time. + # See https://redis.io/docs/manual/client-side-caching/ for more info. + # CLI flag: -blocks-storage.bucket-store.index-cache.redis.cache-size + [cache_size: | default = 0] + chunks_cache: # Backend for chunks cache, if not empty. Supported values: memcached. # CLI flag: -blocks-storage.bucket-store.chunks-cache.backend @@ -698,6 +813,113 @@ blocks_storage: # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.auto-discovery [auto_discovery: | default = false] + redis: + # Comma separated list of redis addresses. Supported prefixes are: dns+ + # (looked up as an A/AAAA query), dnssrv+ (looked up as a SRV query, + # dnssrvnoa+ (looked up as a SRV query, with no A/AAAA lookup made after + # that). + # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.addresses + [addresses: | default = ""] + + # Redis username. + # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.username + [username: | default = ""] + + # Redis password. + # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.password + [password: | default = ""] + + # Database to be selected after connecting to the server. + # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.db + [db: | default = 0] + + # Specifies the master's name. Must be not empty for Redis Sentinel. + # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.master-name + [master_name: | default = ""] + + # Maximum number of socket connections. + # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.pool-size + [pool_size: | default = 100] + + # Specifies the minimum number of idle connections, which is useful when + # it is slow to establish new connections. + # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.min-idle-conns + [min_idle_conns: | default = 10] + + # The maximum number of concurrent GetMulti() operations. If set to 0, + # concurrency is unlimited. + # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.max-get-multi-concurrency + [max_get_multi_concurrency: | default = 100] + + # The maximum size per batch for mget. + # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.get-multi-batch-size + [get_multi_batch_size: | default = 100] + + # The maximum number of concurrent SetMulti() operations. If set to 0, + # concurrency is unlimited. + # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.max-set-multi-concurrency + [max_set_multi_concurrency: | default = 100] + + # The maximum size per batch for pipeline set. + # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.set-multi-batch-size + [set_multi_batch_size: | default = 100] + + # Client dial timeout. + # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.dial-timeout + [dial_timeout: | default = 5s] + + # Client read timeout. + # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.read-timeout + [read_timeout: | default = 3s] + + # Client write timeout. + # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.write-timeout + [write_timeout: | default = 3s] + + # Amount of time after which client closes idle connections. Should be + # less than server's timeout. -1 disables idle timeout check. + # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.idle-timeout + [idle_timeout: | default = 5m] + + # Connection age at which client retires (closes) the connection. + # Default 0 is to not close aged connections. + # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.max-conn-age + [max_conn_age: | default = 0s] + + # Whether to enable tls for redis connection. + # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.tls-enabled + [tls_enabled: | default = false] + + # Path to the client certificate file, which will be used for + # authenticating with the server. Also requires the key path to be + # configured. + # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis..tls-cert-path + [tls_cert_path: | default = ""] + + # Path to the key file for the client certificate. Also requires the + # client certificate to be configured. + # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis..tls-key-path + [tls_key_path: | default = ""] + + # Path to the CA certificates file to validate server certificate + # against. If not set, the host's root CA certificates are used. + # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis..tls-ca-path + [tls_ca_path: | default = ""] + + # Override the expected name on the server certificate. + # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis..tls-server-name + [tls_server_name: | default = ""] + + # Skip validating server certificate. + # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis..tls-insecure-skip-verify + [tls_insecure_skip_verify: | default = false] + + # If not zero then client-side caching is enabled. Client-side caching + # is when data is stored in memory instead of fetching data each time. + # See https://redis.io/docs/manual/client-side-caching/ for more info. + # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.cache-size + [cache_size: | default = 0] + # Size of each subrange that bucket object is split into for better # caching. # CLI flag: -blocks-storage.bucket-store.chunks-cache.subrange-size @@ -769,6 +991,113 @@ blocks_storage: # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.auto-discovery [auto_discovery: | default = false] + redis: + # Comma separated list of redis addresses. Supported prefixes are: dns+ + # (looked up as an A/AAAA query), dnssrv+ (looked up as a SRV query, + # dnssrvnoa+ (looked up as a SRV query, with no A/AAAA lookup made after + # that). + # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.addresses + [addresses: | default = ""] + + # Redis username. + # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.username + [username: | default = ""] + + # Redis password. + # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.password + [password: | default = ""] + + # Database to be selected after connecting to the server. + # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.db + [db: | default = 0] + + # Specifies the master's name. Must be not empty for Redis Sentinel. + # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.master-name + [master_name: | default = ""] + + # Maximum number of socket connections. + # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.pool-size + [pool_size: | default = 100] + + # Specifies the minimum number of idle connections, which is useful when + # it is slow to establish new connections. + # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.min-idle-conns + [min_idle_conns: | default = 10] + + # The maximum number of concurrent GetMulti() operations. If set to 0, + # concurrency is unlimited. + # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.max-get-multi-concurrency + [max_get_multi_concurrency: | default = 100] + + # The maximum size per batch for mget. + # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.get-multi-batch-size + [get_multi_batch_size: | default = 100] + + # The maximum number of concurrent SetMulti() operations. If set to 0, + # concurrency is unlimited. + # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.max-set-multi-concurrency + [max_set_multi_concurrency: | default = 100] + + # The maximum size per batch for pipeline set. + # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.set-multi-batch-size + [set_multi_batch_size: | default = 100] + + # Client dial timeout. + # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.dial-timeout + [dial_timeout: | default = 5s] + + # Client read timeout. + # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.read-timeout + [read_timeout: | default = 3s] + + # Client write timeout. + # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.write-timeout + [write_timeout: | default = 3s] + + # Amount of time after which client closes idle connections. Should be + # less than server's timeout. -1 disables idle timeout check. + # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.idle-timeout + [idle_timeout: | default = 5m] + + # Connection age at which client retires (closes) the connection. + # Default 0 is to not close aged connections. + # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.max-conn-age + [max_conn_age: | default = 0s] + + # Whether to enable tls for redis connection. + # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.tls-enabled + [tls_enabled: | default = false] + + # Path to the client certificate file, which will be used for + # authenticating with the server. Also requires the key path to be + # configured. + # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis..tls-cert-path + [tls_cert_path: | default = ""] + + # Path to the key file for the client certificate. Also requires the + # client certificate to be configured. + # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis..tls-key-path + [tls_key_path: | default = ""] + + # Path to the CA certificates file to validate server certificate + # against. If not set, the host's root CA certificates are used. + # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis..tls-ca-path + [tls_ca_path: | default = ""] + + # Override the expected name on the server certificate. + # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis..tls-server-name + [tls_server_name: | default = ""] + + # Skip validating server certificate. + # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis..tls-insecure-skip-verify + [tls_insecure_skip_verify: | default = false] + + # If not zero then client-side caching is enabled. Client-side caching + # is when data is stored in memory instead of fetching data each time. + # See https://redis.io/docs/manual/client-side-caching/ for more info. + # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.cache-size + [cache_size: | default = 0] + # How long to cache list of tenants in the bucket. # CLI flag: -blocks-storage.bucket-store.metadata-cache.tenants-list-ttl [tenants_list_ttl: | default = 15m] diff --git a/docs/blocks-storage/store-gateway.template b/docs/blocks-storage/store-gateway.template index 68b787f1ab4..b4cb6a1fe03 100644 --- a/docs/blocks-storage/store-gateway.template +++ b/docs/blocks-storage/store-gateway.template @@ -117,6 +117,7 @@ The store-gateway can use a cache to speed up lookups of postings and series fro - `inmemory` - `memcached` +- `redis` #### In-memory index cache @@ -139,20 +140,26 @@ The Memcached client uses a jump hash algorithm to shard cached entries across a For example, if you're running Memcached in Kubernetes, you may: 1. Deploy your Memcached cluster using a [StatefulSet](https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/) -2. Create an [headless service](https://kubernetes.io/docs/concepts/services-networking/service/#headless-services) for Memcached StatefulSet +2. Create a [headless service](https://kubernetes.io/docs/concepts/services-networking/service/#headless-services) for Memcached StatefulSet 3. Configure the Cortex's Memcached client address using the `dnssrvnoa+` [service discovery](../configuration/arguments.md#dns-service-discovery) +#### Redis index cache + +The `redis` index cache allows to use [Redis](https://memcached.org/) as cache backend. This cache backend is configured using `-blocks-storage.bucket-store.index-cache.backend=redis` and requires the Redis server(s) addresses via `-blocks-storage.bucket-store.index-cache.redis.addresses` (or config file). + +Using `redis` as the cache backend has similar trade-offs as using `memcached` cache backend. However, client side caching can be enabled when using `redis` backend to avoid Store Gateway fetching data from cache each time. See [here](https://redis.io/docs/manual/client-side-caching/) for more info and it can be enabled by setting flag `-blocks-storage.bucket-store.index-cache.redis.cache-size` > 0. + ### Chunks cache Store-gateway can also use a cache for storing chunks fetched from the storage. Chunks contain actual samples, and can be reused if user query hits the same series for the same time range. -To enable chunks cache, please set `-blocks-storage.bucket-store.chunks-cache.backend`. Chunks can currently only be stored into Memcached cache. Memcached client can be configured via flags with `-blocks-storage.bucket-store.chunks-cache.memcached.*` prefix. +To enable chunks cache, please set `-blocks-storage.bucket-store.chunks-cache.backend`. Chunks can be stored into Memcached or Redis cache. Memcached client can be configured via flags with `-blocks-storage.bucket-store.chunks-cache.memcached.*` prefix. Redis client can be configured via flags with `-blocks-storage.bucket-store.chunks-cache.redis.*` prefix. There are additional low-level options for configuring chunks cache. Please refer to other flags with `-blocks-storage.bucket-store.chunks-cache.*` prefix. ### Metadata cache -Store-gateway and [querier](./querier.md) can use memcached for caching bucket metadata: +Store-gateway and [querier](./querier.md) can use memcached or redis for caching bucket metadata: - List of tenants - List of blocks per tenant @@ -162,11 +169,11 @@ Store-gateway and [querier](./querier.md) can use memcached for caching bucket m Using the metadata cache can significantly reduce the number of API calls to object storage and protects from linearly scale the number of these API calls with the number of querier and store-gateway instances (because the bucket is periodically scanned and synched by each querier and store-gateway). -To enable metadata cache, please set `-blocks-storage.bucket-store.metadata-cache.backend`. Only `memcached` backend is supported currently. Memcached client has additional configuration available via flags with `-blocks-storage.bucket-store.metadata-cache.memcached.*` prefix. +To enable metadata cache, please set `-blocks-storage.bucket-store.metadata-cache.backend`. `memcached` and `redis` backend are supported currently. Memcached client has additional configuration available via flags with `-blocks-storage.bucket-store.metadata-cache.memcached.*` prefix. Redis client has additional configuration available via flags with `-blocks-storage.bucket-store.metadata-cache.redis.*` prefix. Additional options for configuring metadata cache have `-blocks-storage.bucket-store.metadata-cache.*` prefix. By configuring TTL to zero or negative value, caching of given item type is disabled. -_The same memcached backend cluster should be shared between store-gateways and queriers._ +_The same cache backend deployment should be shared between store-gateways and queriers._ ## Store-gateway HTTP endpoints diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md index 4659d4b4784..a85cc9957fe 100644 --- a/docs/configuration/config-file-reference.md +++ b/docs/configuration/config-file-reference.md @@ -3460,7 +3460,8 @@ bucket_store: [consistency_delay: | default = 0s] index_cache: - # The index cache backend type. Supported values: inmemory, memcached. + # The index cache backend type. Supported values: inmemory, memcached, + # redis. # CLI flag: -blocks-storage.bucket-store.index-cache.backend [backend: | default = "inmemory"] @@ -3517,6 +3518,113 @@ bucket_store: # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.auto-discovery [auto_discovery: | default = false] + redis: + # Comma separated list of redis addresses. Supported prefixes are: dns+ + # (looked up as an A/AAAA query), dnssrv+ (looked up as a SRV query, + # dnssrvnoa+ (looked up as a SRV query, with no A/AAAA lookup made after + # that). + # CLI flag: -blocks-storage.bucket-store.index-cache.redis.addresses + [addresses: | default = ""] + + # Redis username. + # CLI flag: -blocks-storage.bucket-store.index-cache.redis.username + [username: | default = ""] + + # Redis password. + # CLI flag: -blocks-storage.bucket-store.index-cache.redis.password + [password: | default = ""] + + # Database to be selected after connecting to the server. + # CLI flag: -blocks-storage.bucket-store.index-cache.redis.db + [db: | default = 0] + + # Specifies the master's name. Must be not empty for Redis Sentinel. + # CLI flag: -blocks-storage.bucket-store.index-cache.redis.master-name + [master_name: | default = ""] + + # Maximum number of socket connections. + # CLI flag: -blocks-storage.bucket-store.index-cache.redis.pool-size + [pool_size: | default = 100] + + # Specifies the minimum number of idle connections, which is useful when + # it is slow to establish new connections. + # CLI flag: -blocks-storage.bucket-store.index-cache.redis.min-idle-conns + [min_idle_conns: | default = 10] + + # The maximum number of concurrent GetMulti() operations. If set to 0, + # concurrency is unlimited. + # CLI flag: -blocks-storage.bucket-store.index-cache.redis.max-get-multi-concurrency + [max_get_multi_concurrency: | default = 100] + + # The maximum size per batch for mget. + # CLI flag: -blocks-storage.bucket-store.index-cache.redis.get-multi-batch-size + [get_multi_batch_size: | default = 100] + + # The maximum number of concurrent SetMulti() operations. If set to 0, + # concurrency is unlimited. + # CLI flag: -blocks-storage.bucket-store.index-cache.redis.max-set-multi-concurrency + [max_set_multi_concurrency: | default = 100] + + # The maximum size per batch for pipeline set. + # CLI flag: -blocks-storage.bucket-store.index-cache.redis.set-multi-batch-size + [set_multi_batch_size: | default = 100] + + # Client dial timeout. + # CLI flag: -blocks-storage.bucket-store.index-cache.redis.dial-timeout + [dial_timeout: | default = 5s] + + # Client read timeout. + # CLI flag: -blocks-storage.bucket-store.index-cache.redis.read-timeout + [read_timeout: | default = 3s] + + # Client write timeout. + # CLI flag: -blocks-storage.bucket-store.index-cache.redis.write-timeout + [write_timeout: | default = 3s] + + # Amount of time after which client closes idle connections. Should be + # less than server's timeout. -1 disables idle timeout check. + # CLI flag: -blocks-storage.bucket-store.index-cache.redis.idle-timeout + [idle_timeout: | default = 5m] + + # Connection age at which client retires (closes) the connection. Default + # 0 is to not close aged connections. + # CLI flag: -blocks-storage.bucket-store.index-cache.redis.max-conn-age + [max_conn_age: | default = 0s] + + # Whether to enable tls for redis connection. + # CLI flag: -blocks-storage.bucket-store.index-cache.redis.tls-enabled + [tls_enabled: | default = false] + + # Path to the client certificate file, which will be used for + # authenticating with the server. Also requires the key path to be + # configured. + # CLI flag: -blocks-storage.bucket-store.index-cache.redis..tls-cert-path + [tls_cert_path: | default = ""] + + # Path to the key file for the client certificate. Also requires the + # client certificate to be configured. + # CLI flag: -blocks-storage.bucket-store.index-cache.redis..tls-key-path + [tls_key_path: | default = ""] + + # Path to the CA certificates file to validate server certificate against. + # If not set, the host's root CA certificates are used. + # CLI flag: -blocks-storage.bucket-store.index-cache.redis..tls-ca-path + [tls_ca_path: | default = ""] + + # Override the expected name on the server certificate. + # CLI flag: -blocks-storage.bucket-store.index-cache.redis..tls-server-name + [tls_server_name: | default = ""] + + # Skip validating server certificate. + # CLI flag: -blocks-storage.bucket-store.index-cache.redis..tls-insecure-skip-verify + [tls_insecure_skip_verify: | default = false] + + # If not zero then client-side caching is enabled. Client-side caching is + # when data is stored in memory instead of fetching data each time. See + # https://redis.io/docs/manual/client-side-caching/ for more info. + # CLI flag: -blocks-storage.bucket-store.index-cache.redis.cache-size + [cache_size: | default = 0] + chunks_cache: # Backend for chunks cache, if not empty. Supported values: memcached. # CLI flag: -blocks-storage.bucket-store.chunks-cache.backend @@ -3569,6 +3677,113 @@ bucket_store: # CLI flag: -blocks-storage.bucket-store.chunks-cache.memcached.auto-discovery [auto_discovery: | default = false] + redis: + # Comma separated list of redis addresses. Supported prefixes are: dns+ + # (looked up as an A/AAAA query), dnssrv+ (looked up as a SRV query, + # dnssrvnoa+ (looked up as a SRV query, with no A/AAAA lookup made after + # that). + # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.addresses + [addresses: | default = ""] + + # Redis username. + # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.username + [username: | default = ""] + + # Redis password. + # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.password + [password: | default = ""] + + # Database to be selected after connecting to the server. + # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.db + [db: | default = 0] + + # Specifies the master's name. Must be not empty for Redis Sentinel. + # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.master-name + [master_name: | default = ""] + + # Maximum number of socket connections. + # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.pool-size + [pool_size: | default = 100] + + # Specifies the minimum number of idle connections, which is useful when + # it is slow to establish new connections. + # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.min-idle-conns + [min_idle_conns: | default = 10] + + # The maximum number of concurrent GetMulti() operations. If set to 0, + # concurrency is unlimited. + # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.max-get-multi-concurrency + [max_get_multi_concurrency: | default = 100] + + # The maximum size per batch for mget. + # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.get-multi-batch-size + [get_multi_batch_size: | default = 100] + + # The maximum number of concurrent SetMulti() operations. If set to 0, + # concurrency is unlimited. + # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.max-set-multi-concurrency + [max_set_multi_concurrency: | default = 100] + + # The maximum size per batch for pipeline set. + # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.set-multi-batch-size + [set_multi_batch_size: | default = 100] + + # Client dial timeout. + # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.dial-timeout + [dial_timeout: | default = 5s] + + # Client read timeout. + # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.read-timeout + [read_timeout: | default = 3s] + + # Client write timeout. + # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.write-timeout + [write_timeout: | default = 3s] + + # Amount of time after which client closes idle connections. Should be + # less than server's timeout. -1 disables idle timeout check. + # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.idle-timeout + [idle_timeout: | default = 5m] + + # Connection age at which client retires (closes) the connection. Default + # 0 is to not close aged connections. + # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.max-conn-age + [max_conn_age: | default = 0s] + + # Whether to enable tls for redis connection. + # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.tls-enabled + [tls_enabled: | default = false] + + # Path to the client certificate file, which will be used for + # authenticating with the server. Also requires the key path to be + # configured. + # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis..tls-cert-path + [tls_cert_path: | default = ""] + + # Path to the key file for the client certificate. Also requires the + # client certificate to be configured. + # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis..tls-key-path + [tls_key_path: | default = ""] + + # Path to the CA certificates file to validate server certificate against. + # If not set, the host's root CA certificates are used. + # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis..tls-ca-path + [tls_ca_path: | default = ""] + + # Override the expected name on the server certificate. + # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis..tls-server-name + [tls_server_name: | default = ""] + + # Skip validating server certificate. + # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis..tls-insecure-skip-verify + [tls_insecure_skip_verify: | default = false] + + # If not zero then client-side caching is enabled. Client-side caching is + # when data is stored in memory instead of fetching data each time. See + # https://redis.io/docs/manual/client-side-caching/ for more info. + # CLI flag: -blocks-storage.bucket-store.chunks-cache.redis.cache-size + [cache_size: | default = 0] + # Size of each subrange that bucket object is split into for better caching. # CLI flag: -blocks-storage.bucket-store.chunks-cache.subrange-size [subrange_size: | default = 16000] @@ -3639,6 +3854,113 @@ bucket_store: # CLI flag: -blocks-storage.bucket-store.metadata-cache.memcached.auto-discovery [auto_discovery: | default = false] + redis: + # Comma separated list of redis addresses. Supported prefixes are: dns+ + # (looked up as an A/AAAA query), dnssrv+ (looked up as a SRV query, + # dnssrvnoa+ (looked up as a SRV query, with no A/AAAA lookup made after + # that). + # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.addresses + [addresses: | default = ""] + + # Redis username. + # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.username + [username: | default = ""] + + # Redis password. + # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.password + [password: | default = ""] + + # Database to be selected after connecting to the server. + # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.db + [db: | default = 0] + + # Specifies the master's name. Must be not empty for Redis Sentinel. + # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.master-name + [master_name: | default = ""] + + # Maximum number of socket connections. + # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.pool-size + [pool_size: | default = 100] + + # Specifies the minimum number of idle connections, which is useful when + # it is slow to establish new connections. + # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.min-idle-conns + [min_idle_conns: | default = 10] + + # The maximum number of concurrent GetMulti() operations. If set to 0, + # concurrency is unlimited. + # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.max-get-multi-concurrency + [max_get_multi_concurrency: | default = 100] + + # The maximum size per batch for mget. + # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.get-multi-batch-size + [get_multi_batch_size: | default = 100] + + # The maximum number of concurrent SetMulti() operations. If set to 0, + # concurrency is unlimited. + # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.max-set-multi-concurrency + [max_set_multi_concurrency: | default = 100] + + # The maximum size per batch for pipeline set. + # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.set-multi-batch-size + [set_multi_batch_size: | default = 100] + + # Client dial timeout. + # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.dial-timeout + [dial_timeout: | default = 5s] + + # Client read timeout. + # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.read-timeout + [read_timeout: | default = 3s] + + # Client write timeout. + # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.write-timeout + [write_timeout: | default = 3s] + + # Amount of time after which client closes idle connections. Should be + # less than server's timeout. -1 disables idle timeout check. + # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.idle-timeout + [idle_timeout: | default = 5m] + + # Connection age at which client retires (closes) the connection. Default + # 0 is to not close aged connections. + # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.max-conn-age + [max_conn_age: | default = 0s] + + # Whether to enable tls for redis connection. + # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.tls-enabled + [tls_enabled: | default = false] + + # Path to the client certificate file, which will be used for + # authenticating with the server. Also requires the key path to be + # configured. + # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis..tls-cert-path + [tls_cert_path: | default = ""] + + # Path to the key file for the client certificate. Also requires the + # client certificate to be configured. + # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis..tls-key-path + [tls_key_path: | default = ""] + + # Path to the CA certificates file to validate server certificate against. + # If not set, the host's root CA certificates are used. + # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis..tls-ca-path + [tls_ca_path: | default = ""] + + # Override the expected name on the server certificate. + # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis..tls-server-name + [tls_server_name: | default = ""] + + # Skip validating server certificate. + # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis..tls-insecure-skip-verify + [tls_insecure_skip_verify: | default = false] + + # If not zero then client-side caching is enabled. Client-side caching is + # when data is stored in memory instead of fetching data each time. See + # https://redis.io/docs/manual/client-side-caching/ for more info. + # CLI flag: -blocks-storage.bucket-store.metadata-cache.redis.cache-size + [cache_size: | default = 0] + # How long to cache list of tenants in the bucket. # CLI flag: -blocks-storage.bucket-store.metadata-cache.tenants-list-ttl [tenants_list_ttl: | default = 15m] diff --git a/integration/e2e/cache/cache.go b/integration/e2e/cache/cache.go index 7598c307701..1a165a0a91d 100644 --- a/integration/e2e/cache/cache.go +++ b/integration/e2e/cache/cache.go @@ -7,6 +7,7 @@ import ( const ( MemcachedPort = 11211 + RedisPort = 6379 ) func NewMemcached() *e2e.ConcreteService { @@ -18,3 +19,13 @@ func NewMemcached() *e2e.ConcreteService { MemcachedPort, ) } + +func NewRedis() *e2e.ConcreteService { + return e2e.NewConcreteService( + "redis", + images.Redis, + e2e.NewCommand("redis-server", "*:6379"), + e2e.NewTCPReadinessProbe(RedisPort), + RedisPort, + ) +} diff --git a/integration/e2e/images/images.go b/integration/e2e/images/images.go index 63deb2fc733..c7993192854 100644 --- a/integration/e2e/images/images.go +++ b/integration/e2e/images/images.go @@ -7,6 +7,7 @@ package images var ( Memcached = "memcached:1.6.1" + Redis = "docker.io/redis:7.0.4-alpine" Minio = "minio/minio:RELEASE.2021-10-13T00-23-17Z" KES = "minio/kes:v0.17.1" Consul = "consul:1.8.4" diff --git a/integration/querier_test.go b/integration/querier_test.go index 551a15ac465..651003a9ecd 100644 --- a/integration/querier_test.go +++ b/integration/querier_test.go @@ -35,10 +35,12 @@ func TestQuerierWithBlocksStorageRunningInMicroservicesMode(t *testing.T) { "blocks sharding disabled, ingester gRPC streaming disabled, memcached index cache": { blocksShardingStrategy: "", ingesterStreamingEnabled: false, - // Memcached index cache is required to avoid flaky tests when the blocks sharding is disabled - // because two different requests may hit two different store-gateways, so if the cache is not - // shared there's no guarantee we'll have a cache hit. - indexCacheBackend: tsdb.IndexCacheBackendMemcached, + indexCacheBackend: tsdb.IndexCacheBackendMemcached, + }, + "blocks sharding disabled, ingester gRPC streaming disabled, redis index cache": { + blocksShardingStrategy: "", + ingesterStreamingEnabled: false, + indexCacheBackend: tsdb.IndexCacheBackendRedis, }, "blocks default sharding, ingester gRPC streaming disabled, inmemory index cache": { blocksShardingStrategy: "default", @@ -74,6 +76,19 @@ func TestQuerierWithBlocksStorageRunningInMicroservicesMode(t *testing.T) { indexCacheBackend: tsdb.IndexCacheBackendInMemory, bucketIndexEnabled: true, }, + "blocks default sharding, ingester gRPC streaming enabled, redis index cache, bucket index enabled": { + blocksShardingStrategy: "default", + ingesterStreamingEnabled: true, + indexCacheBackend: tsdb.IndexCacheBackendRedis, + bucketIndexEnabled: true, + }, + "blocks shuffle sharding, ingester gRPC streaming enabled, redis index cache, bucket index enabled": { + blocksShardingStrategy: "shuffle-sharding", + tenantShardSize: 1, + ingesterStreamingEnabled: true, + indexCacheBackend: tsdb.IndexCacheBackendRedis, + bucketIndexEnabled: true, + }, } for testName, testCfg := range tests { @@ -106,10 +121,15 @@ func TestQuerierWithBlocksStorageRunningInMicroservicesMode(t *testing.T) { consul := e2edb.NewConsul() minio := e2edb.NewMinio(9000, flags["-blocks-storage.s3.bucket-name"]) memcached := e2ecache.NewMemcached() - require.NoError(t, s.StartAndWaitReady(consul, minio, memcached)) - - // Add the memcached address to the flags. - flags["-blocks-storage.bucket-store.index-cache.memcached.addresses"] = "dns+" + memcached.NetworkEndpoint(e2ecache.MemcachedPort) + redis := e2ecache.NewRedis() + require.NoError(t, s.StartAndWaitReady(consul, minio, memcached, redis)) + + // Add the cache address to the flags. + if testCfg.indexCacheBackend == tsdb.IndexCacheBackendMemcached { + flags["-blocks-storage.bucket-store.index-cache.memcached.addresses"] = "dns+" + memcached.NetworkEndpoint(e2ecache.MemcachedPort) + } else if testCfg.indexCacheBackend == tsdb.IndexCacheBackendRedis { + flags["-blocks-storage.bucket-store.index-cache.redis.addresses"] = redis.NetworkEndpoint(e2ecache.RedisPort) + } // Start Cortex components. distributor := e2ecortex.NewDistributor("distributor", e2ecortex.RingStoreConsul, consul.NetworkHTTPEndpoint(), flags, "") @@ -280,10 +300,7 @@ func TestQuerierWithBlocksStorageRunningInSingleBinaryMode(t *testing.T) { "blocks sharding disabled, ingester gRPC streaming disabled, memcached index cache": { blocksShardingEnabled: false, ingesterStreamingEnabled: false, - // Memcached index cache is required to avoid flaky tests when the blocks sharding is disabled - // because two different requests may hit two different store-gateways, so if the cache is not - // shared there's no guarantee we'll have a cache hit. - indexCacheBackend: tsdb.IndexCacheBackendMemcached, + indexCacheBackend: tsdb.IndexCacheBackendMemcached, }, "blocks sharding enabled, ingester gRPC streaming enabled, memcached index cache": { blocksShardingEnabled: true, @@ -296,6 +313,22 @@ func TestQuerierWithBlocksStorageRunningInSingleBinaryMode(t *testing.T) { indexCacheBackend: tsdb.IndexCacheBackendMemcached, bucketIndexEnabled: true, }, + "blocks sharding disabled, ingester gRPC streaming disabled, redis index cache": { + blocksShardingEnabled: false, + ingesterStreamingEnabled: false, + indexCacheBackend: tsdb.IndexCacheBackendRedis, + }, + "blocks sharding enabled, ingester gRPC streaming enabled, redis index cache": { + blocksShardingEnabled: true, + ingesterStreamingEnabled: true, + indexCacheBackend: tsdb.IndexCacheBackendRedis, + }, + "blocks sharding enabled, ingester gRPC streaming enabled, redis index cache, bucket index enabled": { + blocksShardingEnabled: true, + ingesterStreamingEnabled: true, + indexCacheBackend: tsdb.IndexCacheBackendRedis, + bucketIndexEnabled: true, + }, } for testName, testCfg := range tests { @@ -311,7 +344,8 @@ func TestQuerierWithBlocksStorageRunningInSingleBinaryMode(t *testing.T) { consul := e2edb.NewConsul() minio := e2edb.NewMinio(9000, bucketName) memcached := e2ecache.NewMemcached() - require.NoError(t, s.StartAndWaitReady(consul, minio, memcached)) + redis := e2ecache.NewRedis() + require.NoError(t, s.StartAndWaitReady(consul, minio, memcached, redis)) // Setting the replication factor equal to the number of Cortex replicas // make sure each replica creates the same blocks, so the total number of @@ -321,16 +355,15 @@ func TestQuerierWithBlocksStorageRunningInSingleBinaryMode(t *testing.T) { // Configure the blocks storage to frequently compact TSDB head // and ship blocks to the storage. flags := mergeFlags(BlocksStorageFlags(), map[string]string{ - "-blocks-storage.tsdb.block-ranges-period": blockRangePeriod.String(), - "-blocks-storage.tsdb.ship-interval": "1s", - "-blocks-storage.bucket-store.sync-interval": "1s", - "-blocks-storage.tsdb.retention-period": ((blockRangePeriod * 2) - 1).String(), - "-blocks-storage.bucket-store.index-cache.backend": testCfg.indexCacheBackend, - "-blocks-storage.bucket-store.index-cache.memcached.addresses": "dns+" + memcached.NetworkEndpoint(e2ecache.MemcachedPort), - "-blocks-storage.bucket-store.bucket-index.enabled": strconv.FormatBool(testCfg.bucketIndexEnabled), - "-querier.ingester-streaming": strconv.FormatBool(testCfg.ingesterStreamingEnabled), - "-querier.query-store-for-labels-enabled": "true", - "-querier.thanos-engine": strconv.FormatBool(thanosEngine), + "-blocks-storage.tsdb.block-ranges-period": blockRangePeriod.String(), + "-blocks-storage.tsdb.ship-interval": "1s", + "-blocks-storage.bucket-store.sync-interval": "1s", + "-blocks-storage.tsdb.retention-period": ((blockRangePeriod * 2) - 1).String(), + "-blocks-storage.bucket-store.index-cache.backend": testCfg.indexCacheBackend, + "-blocks-storage.bucket-store.bucket-index.enabled": strconv.FormatBool(testCfg.bucketIndexEnabled), + "-querier.ingester-streaming": strconv.FormatBool(testCfg.ingesterStreamingEnabled), + "-querier.query-store-for-labels-enabled": "true", + "-querier.thanos-engine": strconv.FormatBool(thanosEngine), // Ingester. "-ring.store": "consul", "-consul.hostname": consul.NetworkHTTPEndpoint(), @@ -343,6 +376,13 @@ func TestQuerierWithBlocksStorageRunningInSingleBinaryMode(t *testing.T) { "-store-gateway.sharding-ring.replication-factor": "1", }) + // Add the cache address to the flags. + if testCfg.indexCacheBackend == tsdb.IndexCacheBackendMemcached { + flags["-blocks-storage.bucket-store.index-cache.memcached.addresses"] = "dns+" + memcached.NetworkEndpoint(e2ecache.MemcachedPort) + } else if testCfg.indexCacheBackend == tsdb.IndexCacheBackendRedis { + flags["-blocks-storage.bucket-store.index-cache.redis.addresses"] = redis.NetworkEndpoint(e2ecache.RedisPort) + } + // Start Cortex replicas. cortex1 := e2ecortex.NewSingleBinary("cortex-1", flags, "") cortex2 := e2ecortex.NewSingleBinary("cortex-2", flags, "") diff --git a/pkg/cortexpb/compat_test.go b/pkg/cortexpb/compat_test.go index 662ad0800ce..63200d6ae1f 100644 --- a/pkg/cortexpb/compat_test.go +++ b/pkg/cortexpb/compat_test.go @@ -7,7 +7,6 @@ import ( "testing" "unsafe" - "github.com/efficientgo/core/testutil" jsoniter "github.com/json-iterator/go" "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/model/textparse" @@ -99,7 +98,7 @@ func TestMetricMetadataToMetricTypeToMetricType(t *testing.T) { for _, tt := range tc { t.Run(tt.desc, func(t *testing.T) { m := MetricMetadataMetricTypeToMetricType(tt.input) - testutil.Equals(t, tt.expected, m) + assert.Equal(t, tt.expected, m) }) } } diff --git a/pkg/storage/tsdb/caching_bucket.go b/pkg/storage/tsdb/caching_bucket.go index a0454adc19c..b468e31d533 100644 --- a/pkg/storage/tsdb/caching_bucket.go +++ b/pkg/storage/tsdb/caching_bucket.go @@ -23,25 +23,26 @@ import ( const ( CacheBackendMemcached = "memcached" + CacheBackendRedis = "redis" ) type CacheBackend struct { Backend string `yaml:"backend"` Memcached MemcachedClientConfig `yaml:"memcached"` + Redis RedisClientConfig `yaml:"redis"` } // Validate the config. func (cfg *CacheBackend) Validate() error { - if cfg.Backend != "" && cfg.Backend != CacheBackendMemcached { + switch cfg.Backend { + case CacheBackendMemcached: + return cfg.Memcached.Validate() + case CacheBackendRedis: + return cfg.Redis.Validate() + case "": + default: return fmt.Errorf("unsupported cache backend: %s", cfg.Backend) } - - if cfg.Backend == CacheBackendMemcached { - if err := cfg.Memcached.Validate(); err != nil { - return err - } - } - return nil } @@ -58,6 +59,7 @@ func (cfg *ChunksCacheConfig) RegisterFlagsWithPrefix(f *flag.FlagSet, prefix st f.StringVar(&cfg.Backend, prefix+"backend", "", fmt.Sprintf("Backend for chunks cache, if not empty. Supported values: %s.", CacheBackendMemcached)) cfg.Memcached.RegisterFlagsWithPrefix(f, prefix+"memcached.") + cfg.Redis.RegisterFlagsWithPrefix(f, prefix+"redis.") f.Int64Var(&cfg.SubrangeSize, prefix+"subrange-size", 16000, "Size of each subrange that bucket object is split into for better caching.") f.IntVar(&cfg.MaxGetRangeRequests, prefix+"max-get-range-requests", 3, "Maximum number of sub-GetRange requests that a single GetRange request can be split into when fetching chunks. Zero or negative value = unlimited number of sub-requests.") @@ -89,6 +91,7 @@ func (cfg *MetadataCacheConfig) RegisterFlagsWithPrefix(f *flag.FlagSet, prefix f.StringVar(&cfg.Backend, prefix+"backend", "", fmt.Sprintf("Backend for metadata cache, if not empty. Supported values: %s.", CacheBackendMemcached)) cfg.Memcached.RegisterFlagsWithPrefix(f, prefix+"memcached.") + cfg.Redis.RegisterFlagsWithPrefix(f, prefix+"redis.") f.DurationVar(&cfg.TenantsListTTL, prefix+"tenants-list-ttl", 15*time.Minute, "How long to cache list of tenants in the bucket.") f.DurationVar(&cfg.TenantBlocksListTTL, prefix+"tenant-blocks-list-ttl", 5*time.Minute, "How long to cache list of blocks for each tenant.") @@ -111,7 +114,7 @@ func CreateCachingBucket(chunksConfig ChunksCacheConfig, metadataConfig Metadata cfg := cache.NewCachingBucketConfig() cachingConfigured := false - chunksCache, err := createCache("chunks-cache", chunksConfig.Backend, chunksConfig.Memcached, logger, reg) + chunksCache, err := createCache("chunks-cache", &chunksConfig.CacheBackend, logger, reg) if err != nil { return nil, errors.Wrapf(err, "chunks-cache") } @@ -121,7 +124,7 @@ func CreateCachingBucket(chunksConfig ChunksCacheConfig, metadataConfig Metadata cfg.CacheGetRange("chunks", chunksCache, isTSDBChunkFile, chunksConfig.SubrangeSize, chunksConfig.AttributesTTL, chunksConfig.SubrangeTTL, chunksConfig.MaxGetRangeRequests) } - metadataCache, err := createCache("metadata-cache", metadataConfig.Backend, metadataConfig.Memcached, logger, reg) + metadataCache, err := createCache("metadata-cache", &metadataConfig.CacheBackend, logger, reg) if err != nil { return nil, errors.Wrapf(err, "metadata-cache") } @@ -149,22 +152,29 @@ func CreateCachingBucket(chunksConfig ChunksCacheConfig, metadataConfig Metadata return storecache.NewCachingBucket(bkt, cfg, logger, reg) } -func createCache(cacheName string, backend string, memcached MemcachedClientConfig, logger log.Logger, reg prometheus.Registerer) (cache.Cache, error) { - switch backend { +func createCache(cacheName string, cacheBackend *CacheBackend, logger log.Logger, reg prometheus.Registerer) (cache.Cache, error) { + switch cacheBackend.Backend { case "": // No caching. return nil, nil case CacheBackendMemcached: var client cacheutil.MemcachedClient - client, err := cacheutil.NewMemcachedClientWithConfig(logger, cacheName, memcached.ToMemcachedClientConfig(), reg) + client, err := cacheutil.NewMemcachedClientWithConfig(logger, cacheName, cacheBackend.Memcached.ToMemcachedClientConfig(), reg) if err != nil { return nil, errors.Wrapf(err, "failed to create memcached client") } return cache.NewMemcachedCache(cacheName, logger, client, reg), nil + case CacheBackendRedis: + redisCache, err := cacheutil.NewRedisClientWithConfig(logger, cacheName, cacheBackend.Redis.ToRedisClientConfig(), reg) + if err != nil { + return nil, errors.Wrapf(err, "failed to create redis client") + } + return cache.NewRedisCache(cacheName, logger, redisCache, reg), nil + default: - return nil, errors.Errorf("unsupported cache type for cache %s: %s", cacheName, backend) + return nil, errors.Errorf("unsupported cache type for cache %s: %s", cacheName, cacheBackend.Backend) } } diff --git a/pkg/storage/tsdb/index_cache.go b/pkg/storage/tsdb/index_cache.go index 60b347d142f..ca4a5484208 100644 --- a/pkg/storage/tsdb/index_cache.go +++ b/pkg/storage/tsdb/index_cache.go @@ -23,6 +23,9 @@ const ( // IndexCacheBackendMemcached is the value for the memcached index cache backend. IndexCacheBackendMemcached = "memcached" + // IndexCacheBackendRedis is the value for the redis index cache backend. + IndexCacheBackendRedis = "redis" + // IndexCacheBackendDefault is the value for the default index cache backend. IndexCacheBackendDefault = IndexCacheBackendInMemory @@ -30,7 +33,7 @@ const ( ) var ( - supportedIndexCacheBackends = []string{IndexCacheBackendInMemory, IndexCacheBackendMemcached} + supportedIndexCacheBackends = []string{IndexCacheBackendInMemory, IndexCacheBackendMemcached, IndexCacheBackendRedis} errUnsupportedIndexCacheBackend = errors.New("unsupported index cache backend") errNoIndexCacheAddresses = errors.New("no index cache backend addresses") @@ -40,6 +43,7 @@ type IndexCacheConfig struct { Backend string `yaml:"backend"` InMemory InMemoryIndexCacheConfig `yaml:"inmemory"` Memcached MemcachedClientConfig `yaml:"memcached"` + Redis RedisClientConfig `yaml:"redis"` } func (cfg *IndexCacheConfig) RegisterFlags(f *flag.FlagSet) { @@ -51,6 +55,7 @@ func (cfg *IndexCacheConfig) RegisterFlagsWithPrefix(f *flag.FlagSet, prefix str cfg.InMemory.RegisterFlagsWithPrefix(f, prefix+"inmemory.") cfg.Memcached.RegisterFlagsWithPrefix(f, prefix+"memcached.") + cfg.Redis.RegisterFlagsWithPrefix(f, prefix+"redis.") } // Validate the config. @@ -63,6 +68,10 @@ func (cfg *IndexCacheConfig) Validate() error { if err := cfg.Memcached.Validate(); err != nil { return err } + } else if cfg.Backend == IndexCacheBackendRedis { + if err := cfg.Redis.Validate(); err != nil { + return err + } } return nil @@ -83,6 +92,8 @@ func NewIndexCache(cfg IndexCacheConfig, logger log.Logger, registerer prometheu return newInMemoryIndexCache(cfg.InMemory, logger, registerer) case IndexCacheBackendMemcached: return newMemcachedIndexCache(cfg.Memcached, logger, registerer) + case IndexCacheBackendRedis: + return newRedisIndexCache(cfg.Redis, logger, registerer) default: return nil, errUnsupportedIndexCacheBackend } @@ -111,3 +122,12 @@ func newMemcachedIndexCache(cfg MemcachedClientConfig, logger log.Logger, regist return storecache.NewMemcachedIndexCache(logger, client, registerer) } + +func newRedisIndexCache(cfg RedisClientConfig, logger log.Logger, registerer prometheus.Registerer) (storecache.IndexCache, error) { + client, err := cacheutil.NewRedisClientWithConfig(logger, "index-cache", cfg.ToRedisClientConfig(), registerer) + if err != nil { + return nil, errors.Wrapf(err, "create index cache redis client") + } + + return storecache.NewRemoteIndexCache(logger, client, registerer) +} diff --git a/pkg/storage/tsdb/redis_client_config.go b/pkg/storage/tsdb/redis_client_config.go new file mode 100644 index 00000000000..9c0937d431a --- /dev/null +++ b/pkg/storage/tsdb/redis_client_config.go @@ -0,0 +1,107 @@ +package tsdb + +import ( + "flag" + "time" + + "github.com/pkg/errors" + "github.com/thanos-io/thanos/pkg/cacheutil" + + "github.com/cortexproject/cortex/pkg/util/tls" +) + +type RedisClientConfig struct { + Addresses string `yaml:"addresses"` + Username string `yaml:"username"` + Password string `yaml:"password"` + DB int `yaml:"db"` + MasterName string `yaml:"master_name"` + + PoolSize int `yaml:"pool_size"` + MinIdleConns int `yaml:"min_idle_conns"` + MaxGetMultiConcurrency int `yaml:"max_get_multi_concurrency"` + GetMultiBatchSize int `yaml:"get_multi_batch_size"` + MaxSetMultiConcurrency int `yaml:"max_set_multi_concurrency"` + SetMultiBatchSize int `yaml:"set_multi_batch_size"` + + DialTimeout time.Duration `yaml:"dial_timeout"` + ReadTimeout time.Duration `yaml:"read_timeout"` + WriteTimeout time.Duration `yaml:"write_timeout"` + IdleTimeout time.Duration `yaml:"idle_timeout"` + MaxConnAge time.Duration `yaml:"max_conn_age"` + + TLSEnabled bool `yaml:"tls_enabled"` + TLS tls.ClientConfig `yaml:",inline"` + + // If not zero then client-side caching is enabled. + // Client-side caching is when data is stored in memory + // instead of fetching data each time. + // See https://redis.io/docs/manual/client-side-caching/ for info. + CacheSize int `yaml:"cache_size"` +} + +func (cfg *RedisClientConfig) RegisterFlagsWithPrefix(f *flag.FlagSet, prefix string) { + f.StringVar(&cfg.Addresses, prefix+"addresses", "", "Comma separated list of redis addresses. Supported prefixes are: dns+ (looked up as an A/AAAA query), dnssrv+ (looked up as a SRV query, dnssrvnoa+ (looked up as a SRV query, with no A/AAAA lookup made after that).") + f.StringVar(&cfg.Username, prefix+"username", "", "Redis username.") + f.StringVar(&cfg.Password, prefix+"password", "", "Redis password.") + f.IntVar(&cfg.DB, prefix+"db", 0, "Database to be selected after connecting to the server.") + f.DurationVar(&cfg.DialTimeout, prefix+"dial-timeout", time.Second*5, "Client dial timeout.") + f.DurationVar(&cfg.ReadTimeout, prefix+"read-timeout", time.Second*3, "Client read timeout.") + f.DurationVar(&cfg.WriteTimeout, prefix+"write-timeout", time.Second*3, "Client write timeout.") + f.DurationVar(&cfg.IdleTimeout, prefix+"idle-timeout", time.Minute*5, "Amount of time after which client closes idle connections. Should be less than server's timeout. -1 disables idle timeout check.") + f.DurationVar(&cfg.MaxConnAge, prefix+"max-conn-age", 0, "Connection age at which client retires (closes) the connection. Default 0 is to not close aged connections.") + f.IntVar(&cfg.PoolSize, prefix+"pool-size", 100, "Maximum number of socket connections.") + f.IntVar(&cfg.MinIdleConns, prefix+"min-idle-conns", 10, "Specifies the minimum number of idle connections, which is useful when it is slow to establish new connections.") + f.IntVar(&cfg.MaxGetMultiConcurrency, prefix+"max-get-multi-concurrency", 100, "The maximum number of concurrent GetMulti() operations. If set to 0, concurrency is unlimited.") + f.IntVar(&cfg.GetMultiBatchSize, prefix+"get-multi-batch-size", 100, "The maximum size per batch for mget.") + f.IntVar(&cfg.MaxSetMultiConcurrency, prefix+"max-set-multi-concurrency", 100, "The maximum number of concurrent SetMulti() operations. If set to 0, concurrency is unlimited.") + f.IntVar(&cfg.SetMultiBatchSize, prefix+"set-multi-batch-size", 100, "The maximum size per batch for pipeline set.") + f.StringVar(&cfg.MasterName, prefix+"master-name", "", "Specifies the master's name. Must be not empty for Redis Sentinel.") + f.IntVar(&cfg.CacheSize, prefix+"cache-size", 0, "If not zero then client-side caching is enabled. Client-side caching is when data is stored in memory instead of fetching data each time. See https://redis.io/docs/manual/client-side-caching/ for more info.") + f.BoolVar(&cfg.TLSEnabled, prefix+"tls-enabled", false, "Whether to enable tls for redis connection.") + cfg.TLS.RegisterFlagsWithPrefix(prefix, f) +} + +// Validate the config. +func (cfg *RedisClientConfig) Validate() error { + if cfg.Addresses == "" { + return errNoIndexCacheAddresses + } + + if cfg.TLSEnabled { + if (cfg.TLS.CertPath != "") != (cfg.TLS.KeyPath != "") { + return errors.New("both client key and certificate must be provided") + } + } + + return nil +} + +func (cfg *RedisClientConfig) ToRedisClientConfig() cacheutil.RedisClientConfig { + return cacheutil.RedisClientConfig{ + Addr: cfg.Addresses, + Username: cfg.Username, + Password: cfg.Password, + DB: cfg.DB, + MasterName: cfg.MasterName, + DialTimeout: cfg.DialTimeout, + ReadTimeout: cfg.ReadTimeout, + WriteTimeout: cfg.WriteTimeout, + PoolSize: cfg.PoolSize, + MinIdleConns: cfg.MinIdleConns, + IdleTimeout: cfg.IdleTimeout, + MaxConnAge: cfg.MaxConnAge, + MaxGetMultiConcurrency: cfg.MaxGetMultiConcurrency, + GetMultiBatchSize: cfg.GetMultiBatchSize, + MaxSetMultiConcurrency: cfg.MaxSetMultiConcurrency, + SetMultiBatchSize: cfg.SetMultiBatchSize, + TLSEnabled: cfg.TLSEnabled, + TLSConfig: cacheutil.TLSConfig{ + CAFile: cfg.TLS.CAPath, + KeyFile: cfg.TLS.KeyPath, + CertFile: cfg.TLS.CertPath, + ServerName: cfg.TLS.ServerName, + InsecureSkipVerify: cfg.TLS.InsecureSkipVerify, + }, + } +} diff --git a/pkg/storage/tsdb/redis_client_config_test.go b/pkg/storage/tsdb/redis_client_config_test.go new file mode 100644 index 00000000000..3d5c52d5079 --- /dev/null +++ b/pkg/storage/tsdb/redis_client_config_test.go @@ -0,0 +1,73 @@ +package tsdb + +import ( + "testing" + + "github.com/pkg/errors" + "github.com/stretchr/testify/require" + + "github.com/cortexproject/cortex/pkg/util/tls" +) + +func TestRedisIndexCacheConfigValidate(t *testing.T) { + for _, tc := range []struct { + name string + conf *RedisClientConfig + err error + }{ + { + name: "empty address", + conf: &RedisClientConfig{}, + err: errNoIndexCacheAddresses, + }, + { + name: "provide TLS cert path but not key path", + conf: &RedisClientConfig{ + Addresses: "aaa", + TLSEnabled: true, + TLS: tls.ClientConfig{ + CertPath: "foo", + }, + }, + err: errors.New("both client key and certificate must be provided"), + }, + { + name: "provide TLS key path but not cert path", + conf: &RedisClientConfig{ + Addresses: "aaa", + TLSEnabled: true, + TLS: tls.ClientConfig{ + KeyPath: "foo", + }, + }, + err: errors.New("both client key and certificate must be provided"), + }, + { + name: "success when TLS enabled", + conf: &RedisClientConfig{ + Addresses: "aaa", + TLSEnabled: true, + TLS: tls.ClientConfig{ + KeyPath: "foo", + CertPath: "bar", + }, + }, + }, + { + name: "success when TLS disabled", + conf: &RedisClientConfig{ + Addresses: "aaa", + TLSEnabled: false, + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + err := tc.conf.Validate() + if tc.err == nil { + require.NoError(t, err) + } else { + require.Equal(t, tc.err.Error(), err.Error()) + } + }) + } +}