Upgrade Prometheus to latest master

CHANGELOG.md

@@ -89,6 +89,9 @@
 * [ENHANCEMENT] Blocks storage: reduced number of bucket listing operations to list block content (applies to newly created blocks only). #3363
 * [ENHANCEMENT] Ruler: Include the tenant ID on the notifier logs. #3372
 * [ENHANCEMENT] Blocks storage Compactor: Added `-compactor.enabled-tenants` and `-compactor.disabled-tenants` to explicitly enable or disable compaction of specific tenants. #3385
+* [ENHANCEMENT] Blocks storage ingester: Creating checkpoint only once even when there are multiple Head compactions in a single `Compact()` call. #3373
+* [BUGFIX] Blocks storage ingester: Read repair memory-mapped chunks file which can end up being empty on abrupt shutdowns combined with faulty disks. #3373
+* [BUGFIX] Blocks storage ingester: Close TSDB resources on failed startup preventing ingester OOMing. #3373
 * [BUGFIX] No-longer-needed ingester operations for queries triggered by queriers and rulers are now canceled. #3178
 * [BUGFIX] Ruler: directories in the configured `rules-path` will be removed on startup and shutdown in order to ensure they don't persist between runs. #3195
 * [BUGFIX] Handle hash-collisions in the query path. #3192

go.mod

@@ -47,7 +47,7 @@ require (
 	github.com/prometheus/client_golang v1.7.1
 	github.com/prometheus/client_model v0.2.0
 	github.com/prometheus/common v0.14.0
-	github.com/prometheus/prometheus v1.8.2-0.20201014093524-73e2ce1bd643
+	github.com/prometheus/prometheus v1.8.2-0.20201028100903-3245b3267b24
 	github.com/segmentio/fasthash v0.0.0-20180216231524-a72b379d632e
 	github.com/sony/gobreaker v0.4.1
 	github.com/spf13/afero v1.2.2

go.sum

@@ -1081,8 +1081,8 @@ github.com/prometheus/prometheus v1.8.2-0.20200805082714-e0cf219f0de2/go.mod h1:
 github.com/prometheus/prometheus v1.8.2-0.20200819132913-cb830b0a9c78 h1:tHIAD+hgCIb86T0/Du7vGyfHa6J1+XsImQoY8Ete+c8=
 github.com/prometheus/prometheus v1.8.2-0.20200819132913-cb830b0a9c78/go.mod h1:zfAqy/MwhMFajB9E2n12/9gG2fvofIE9uKDtlZCDxqs=
 github.com/prometheus/prometheus v1.8.2-0.20200923143134-7e2db3d092f3/go.mod h1:9VNWoDFHOMovlubld5uKKxfCDcPBj2GMOCjcUFXkYaM=
-github.com/prometheus/prometheus v1.8.2-0.20201014093524-73e2ce1bd643 h1:BDAexvKlOVjE5A8MlqRxzwkEpPl1/v6ydU1/J7kJtZc=
-github.com/prometheus/prometheus v1.8.2-0.20201014093524-73e2ce1bd643/go.mod h1:XYjkJiog7fyQu3puQNivZPI2pNq1C/775EIoHfDvuvY=
+github.com/prometheus/prometheus v1.8.2-0.20201028100903-3245b3267b24 h1:V/4Cj2GytqdqK7OMEz6c4LNjey3SNyfw3pg5jPKtJvQ=
+github.com/prometheus/prometheus v1.8.2-0.20201028100903-3245b3267b24/go.mod h1:MDRkz271loM/PrYN+wUNEaTMDGSP760MQzB0yEjdgSQ=
 github.com/rafaeljusto/redigomock v0.0.0-20190202135759-257e089e14a1 h1:+kGqA4dNN5hn7WwvKdzHl0rdN5AEkbNZd0VjRltAiZg=
 github.com/rafaeljusto/redigomock v0.0.0-20190202135759-257e089e14a1/go.mod h1:JaY6n2sDr+z2WTsXkOmNRUfDy6FN0L6Nk7x06ndm4tY=
 github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
@@ -1590,8 +1590,8 @@ golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc
 golang.org/x/tools v0.0.0-20200822203824-307de81be3f4/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
 golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
 golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE=
-golang.org/x/tools v0.0.0-20201008025239-9df69603baec h1:RY2OghEV/7X1MLaecgm1mwFd3sGvUddm5pGVSxQvX0c=
-golang.org/x/tools v0.0.0-20201008025239-9df69603baec/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU=
+golang.org/x/tools v0.0.0-20201020161133-226fd2f889ca h1:pvScuB+UnCGDas2naNKUOXruM08MjwVcEdaweeynIqQ=
+golang.org/x/tools v0.0.0-20201020161133-226fd2f889ca/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=

integration/e2e/scenario_test.go

@@ -10,7 +10,7 @@ import (
 	"time"
 
 	"github.com/go-kit/kit/log"
-	"github.com/prometheus/prometheus/util/testutil"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"github.com/thanos-io/thanos/pkg/objstore/s3"
 	"gopkg.in/yaml.v2"
@@ -23,7 +23,7 @@ const bktName = "cheesecake"
 
 func spinup(t *testing.T, networkName string) (*e2e.Scenario, *e2e.HTTPService, *e2e.HTTPService) {
 	s, err := e2e.NewScenario(networkName)
-	testutil.Ok(t, err)
+	assert.NoError(t, err)
 
 	m1 := e2edb.NewMinio(9000, bktName)
 	m2 := e2edb.NewMinio(9001, bktName)

pkg/configs/legacy_promql/functions_test.go

@@ -20,7 +20,7 @@ import (
 
 	"github.com/prometheus/prometheus/pkg/labels"
 	"github.com/prometheus/prometheus/pkg/timestamp"
-	"github.com/prometheus/prometheus/util/testutil"
+	"github.com/stretchr/testify/assert"
 )
 
 func TestDeriv(t *testing.T) {
@@ -35,21 +35,21 @@ func TestDeriv(t *testing.T) {
 
 	metric := labels.FromStrings("__name__", "foo")
 	_, err := a.Add(metric, 1493712816939, 1.0)
-	testutil.Ok(t, err)
+	assert.NoError(t, err)
 
 	_, err = a.Add(metric, 1493712846939, 1.0)
-	testutil.Ok(t, err)
+	assert.NoError(t, err)
 
 	err = a.Commit()
-	testutil.Ok(t, err)
+	assert.NoError(t, err)
 
 	query, err := engine.NewInstantQuery(storage, "deriv(foo[30m])", timestamp.Time(1493712846939))
-	testutil.Ok(t, err)
+	assert.NoError(t, err)
 
 	result := query.Exec(context.Background())
-	testutil.Ok(t, result.Err)
+	assert.NoError(t, result.Err)
 
 	vec, _ := result.Vector()
-	testutil.Assert(t, len(vec) == 1, "Expected 1 result, got %d", len(vec))
-	testutil.Assert(t, vec[0].V == 0.0, "Expected 0.0 as value, got %f", vec[0].V)
+	assert.True(t, len(vec) == 1, "Expected 1 result, got %d", len(vec))
+	assert.True(t, vec[0].V == 0.0, "Expected 0.0 as value, got %f", vec[0].V)
 }

pkg/ingester/ingester_v2_test.go

@@ -1667,11 +1667,12 @@ func TestIngester_v2OpenExistingTSDBOnStartup(t *testing.T) {
 		"should fail and rollback if an error occur while loading a TSDB on concurrency > number of TSDBs": {
 			concurrency: 10,
 			setup: func(t *testing.T, dir string) {
-				// Create a fake TSDB on disk with an empty chunks head segment file (it's invalid and
-				// opening TSDB should fail).
+				// Create a fake TSDB on disk with an empty chunks head segment file (it's invalid unless
+				// it's the last one and opening TSDB should fail).
 				require.NoError(t, os.MkdirAll(filepath.Join(dir, "user0", "wal", ""), 0700))
 				require.NoError(t, os.MkdirAll(filepath.Join(dir, "user0", "chunks_head", ""), 0700))
 				require.NoError(t, ioutil.WriteFile(filepath.Join(dir, "user0", "chunks_head", "00000001"), nil, 0700))
+				require.NoError(t, ioutil.WriteFile(filepath.Join(dir, "user0", "chunks_head", "00000002"), nil, 0700))
 
 				require.NoError(t, os.MkdirAll(filepath.Join(dir, "user1", "dummy"), 0700))
 			},
@@ -1690,11 +1691,12 @@ func TestIngester_v2OpenExistingTSDBOnStartup(t *testing.T) {
 				require.NoError(t, os.MkdirAll(filepath.Join(dir, "user3", "dummy"), 0700))
 				require.NoError(t, os.MkdirAll(filepath.Join(dir, "user4", "dummy"), 0700))
 
-				// Create a fake TSDB on disk with an empty chunks head segment file (it's invalid and
-				// opening TSDB should fail).
+				// Create a fake TSDB on disk with an empty chunks head segment file (it's invalid unless
+				// it's the last one and opening TSDB should fail).
 				require.NoError(t, os.MkdirAll(filepath.Join(dir, "user2", "wal", ""), 0700))
 				require.NoError(t, os.MkdirAll(filepath.Join(dir, "user2", "chunks_head", ""), 0700))
 				require.NoError(t, ioutil.WriteFile(filepath.Join(dir, "user2", "chunks_head", "00000001"), nil, 0700))
+				require.NoError(t, ioutil.WriteFile(filepath.Join(dir, "user2", "chunks_head", "00000002"), nil, 0700))
 			},
 			check: func(t *testing.T, i *Ingester) {
 				require.Equal(t, 0, len(i.TSDBState.dbs))

pkg/querier/chunk_tar_test.go

@@ -14,7 +14,7 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/prometheus/prometheus/promql"
-	"github.com/prometheus/prometheus/util/testutil"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"github.com/weaveworks/common/user"
 
@@ -53,7 +53,7 @@ func getTarDataFromEnv(t testing.TB) (query string, from, through time.Time, ste
 
 func runRangeQuery(t testing.TB, query string, from, through time.Time, step time.Duration, store chunkstore.ChunkStore) {
 	dir, err := ioutil.TempDir("", t.Name())
-	testutil.Ok(t, err)
+	assert.NoError(t, err)
 	defer os.RemoveAll(dir)
 	queryTracker := promql.NewActiveQueryTracker(dir, 1, util.Logger)
 

pkg/querier/querier_test.go

@@ -23,7 +23,6 @@ import (
 	"github.com/prometheus/prometheus/promql"
 	"github.com/prometheus/prometheus/scrape"
 	"github.com/prometheus/prometheus/storage"
-	"github.com/prometheus/prometheus/util/testutil"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"github.com/weaveworks/common/user"
@@ -262,7 +261,7 @@ func TestNoHistoricalQueryToIngester(t *testing.T) {
 	}
 
 	dir, err := ioutil.TempDir("", t.Name())
-	testutil.Ok(t, err)
+	assert.NoError(t, err)
 	defer os.RemoveAll(dir)
 	queryTracker := promql.NewActiveQueryTracker(dir, 10, util.Logger)
 
@@ -495,7 +494,7 @@ func mockDistibutorFor(t *testing.T, cs mockChunkStore, through model.Time) *moc
 
 func testQuery(t testing.TB, queryable storage.Queryable, end model.Time, q query) *promql.Result {
 	dir, err := ioutil.TempDir("", "test_query")
-	testutil.Ok(t, err)
+	assert.NoError(t, err)
 	defer os.RemoveAll(dir)
 	queryTracker := promql.NewActiveQueryTracker(dir, 10, util.Logger)
 
@@ -646,7 +645,7 @@ func TestShortTermQueryToLTS(t *testing.T) {
 	}
 
 	dir, err := ioutil.TempDir("", t.Name())
-	testutil.Ok(t, err)
+	assert.NoError(t, err)
 	defer os.RemoveAll(dir)
 	queryTracker := promql.NewActiveQueryTracker(dir, 10, util.Logger)
 

pkg/ruler/ruler_test.go

@@ -23,7 +23,6 @@ import (
 	"github.com/prometheus/prometheus/promql"
 	promRules "github.com/prometheus/prometheus/rules"
 	"github.com/prometheus/prometheus/storage"
-	"github.com/prometheus/prometheus/util/testutil"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"github.com/weaveworks/common/user"
@@ -87,7 +86,7 @@ func (r ruleLimits) RulerMaxRulesPerRuleGroup(_ string) int {
 
 func testSetup(t *testing.T, cfg Config) (*promql.Engine, storage.QueryableFunc, Pusher, log.Logger, RulesLimits, func()) {
 	dir, err := ioutil.TempDir("", filepath.Base(t.Name()))
-	testutil.Ok(t, err)
+	assert.NoError(t, err)
 	cleanup := func() {
 		os.RemoveAll(dir)
 	}