Reduce distributor memory usage during periods of high error volume

Signed-off-by: Xiaochao Dong (@damnever) <[email protected]>

Author: damnever

pkg/ring/batch.go

@@ -152,8 +152,7 @@ func (b *batchTracker) record(instance instance, err error) {
 		if err != nil {
 			// Track the number of errors by error family, and if it exceeds maxFailures
 			// shortcut the waiting rpc.
-			wrappedErr := httpgrpcutil.WrapHTTPGrpcError(err, "addr=%s state=%s zone=%s", instance.desc.Addr, instance.desc.State, instance.desc.Zone)
-			errCount := instance.itemTrackers[i].recordError(wrappedErr)
+			errCount := instance.itemTrackers[i].recordError(err)
 			// We should return an error if we reach the maxFailure (quorum) on a given error family OR
 			// we dont have any remaining ingesters to try
 			// Ex: 2xx, 4xx, 5xx -> return 4xx
@@ -162,13 +161,17 @@ func (b *batchTracker) record(instance instance, err error) {
 			// Ex: 5xx, _, 5xx -> return 5xx
 			if errCount > int32(sampleTrackers[i].maxFailures) {
 				if b.rpcsFailed.Inc() == 1 {
-					b.err <- httpgrpcutil.WrapHTTPGrpcError(sampleTrackers[i].getError(), "maxFailure (quorum) on a given error family")
+					b.err <- httpgrpcutil.WrapHTTPGrpcError(
+						sampleTrackers[i].getError(), "maxFailure (quorum) on a given error family, addr=%s state=%s zone=%s",
+						instance.desc.Addr, instance.desc.State, instance.desc.Zone)
 				}
 				continue
 			}
 			if sampleTrackers[i].remaining.Dec() == 0 {
 				if b.rpcsFailed.Inc() == 1 {
-					b.err <- httpgrpcutil.WrapHTTPGrpcError(sampleTrackers[i].getError(), "not enough remaining instances to try")
+					b.err <- httpgrpcutil.WrapHTTPGrpcError(
+						sampleTrackers[i].getError(), "not enough remaining instances to try, addr=%s state=%s zone=%s",
+						instance.desc.Addr, instance.desc.State, instance.desc.Zone)
 				}
 				continue
 			}
@@ -187,7 +190,9 @@ func (b *batchTracker) record(instance instance, err error) {
 			// Ex: 4xx, 5xx, 2xx
 			if sampleTrackers[i].remaining.Dec() == 0 {
 				if b.rpcsFailed.Inc() == 1 {
-					b.err <- httpgrpcutil.WrapHTTPGrpcError(sampleTrackers[i].getError(), "not enough remaining instances to try")
+					b.err <- httpgrpcutil.WrapHTTPGrpcError(
+						sampleTrackers[i].getError(), "not enough remaining instances to try, addr=%s state=%s zone=%s",
+						instance.desc.Addr, instance.desc.State, instance.desc.Zone)
 				}
 			}
 		}