From b31ec6ef09241eee679cbac275253d6eeeb2206f Mon Sep 17 00:00:00 2001 From: Ganesh Vernekar Date: Wed, 23 Oct 2019 11:50:21 +0200 Subject: [PATCH 01/18] Flush tokens to file Signed-off-by: Ganesh Vernekar --- pkg/ingester/ingester.go | 2 +- pkg/ingester/tokens | 1 + pkg/ring/lifecycler.go | 30 +++++++++++++++++++++++++++++- pkg/ring/lifecycler_test.go | 10 +++++----- pkg/ring/tokens | Bin 0 -> 4 bytes pkg/ruler/ruler.go | 2 +- pkg/ruler/tokens | 1 + 7 files changed, 38 insertions(+), 8 deletions(-) create mode 100644 pkg/ingester/tokens create mode 100644 pkg/ring/tokens create mode 100644 pkg/ruler/tokens diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index 4f5cc0ecbee..5490348f020 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -190,7 +190,7 @@ func New(cfg Config, clientConfig client.Config, limits *validation.Overrides, c } var err error - i.lifecycler, err = ring.NewLifecycler(cfg.LifecyclerConfig, i, "ingester") + i.lifecycler, err = ring.NewLifecycler(cfg.LifecyclerConfig, i, "ingester", "") if err != nil { return nil, err } diff --git a/pkg/ingester/tokens b/pkg/ingester/tokens new file mode 100644 index 00000000000..7303e56e9fb --- /dev/null +++ b/pkg/ingester/tokens @@ -0,0 +1 @@ +��$ \ No newline at end of file diff --git a/pkg/ring/lifecycler.go b/pkg/ring/lifecycler.go index 669ab94989b..9ddfdea935f 100644 --- a/pkg/ring/lifecycler.go +++ b/pkg/ring/lifecycler.go @@ -2,9 +2,11 @@ package ring import ( "context" + "encoding/binary" "flag" "fmt" "os" + "path" "sort" "sync" "time" @@ -38,6 +40,10 @@ var ( }, []string{"op", "status", "name"}) ) +const ( + tokensFileName = "tokens" +) + // LifecyclerConfig is the config to build a Lifecycler. type LifecyclerConfig struct { RingConfig Config `yaml:"ring,omitempty"` @@ -108,6 +114,7 @@ type Lifecycler struct { cfg LifecyclerConfig flushTransferer FlushTransferer KVStore kv.Client + tokenDir string // Controls the lifecycle of the ingester quit chan struct{} @@ -132,7 +139,7 @@ type Lifecycler struct { } // NewLifecycler makes and starts a new Lifecycler. -func NewLifecycler(cfg LifecyclerConfig, flushTransferer FlushTransferer, name string) (*Lifecycler, error) { +func NewLifecycler(cfg LifecyclerConfig, flushTransferer FlushTransferer, name, tokenDir string) (*Lifecycler, error) { addr := cfg.Addr if addr == "" { var err error @@ -155,6 +162,7 @@ func NewLifecycler(cfg LifecyclerConfig, flushTransferer FlushTransferer, name s cfg: cfg, flushTransferer: flushTransferer, KVStore: store, + tokenDir: tokenDir, Addr: fmt.Sprintf("%s:%d", addr, port), ID: cfg.ID, @@ -242,12 +250,32 @@ func (i *Lifecycler) getTokens() []uint32 { return i.tokens } +func (i *Lifecycler) flushTokensToFile() { + tokenFilePath := path.Join(i.tokenDir, tokensFileName) + f, err := os.OpenFile(tokenFilePath, os.O_WRONLY|os.O_CREATE, 0666) + if err != nil { + level.Error(util.Logger).Log("msg", "error in creating token file", "err", err) + return + } + + b := make([]byte, 4*len(i.tokens)) + for idx, token := range i.tokens { + binary.BigEndian.PutUint32(b[idx*4:], token) + } + + if _, err = f.Write(b); err != nil { + level.Error(util.Logger).Log("msg", "error in writing token file", "err", err) + return + } +} + func (i *Lifecycler) setTokens(tokens []uint32) { tokensOwned.WithLabelValues(i.RingName).Set(float64(len(tokens))) i.stateMtx.Lock() defer i.stateMtx.Unlock() i.tokens = tokens + i.flushTokensToFile() } // ClaimTokensFor takes all the tokens for the supplied ingester and assigns them to this ingester. diff --git a/pkg/ring/lifecycler_test.go b/pkg/ring/lifecycler_test.go index 50c8a84373e..d51d45f1b66 100644 --- a/pkg/ring/lifecycler_test.go +++ b/pkg/ring/lifecycler_test.go @@ -68,7 +68,7 @@ func TestRingNormaliseMigration(t *testing.T) { lifecyclerConfig1 := testLifecyclerConfig(ringConfig, "ing1") ft := &flushTransferer{} - l1, err := NewLifecycler(lifecyclerConfig1, ft, "ingester") + l1, err := NewLifecycler(lifecyclerConfig1, ft, "ingester", "") require.NoError(t, err) // Check this ingester joined, is active, and has one token. @@ -85,7 +85,7 @@ func TestRingNormaliseMigration(t *testing.T) { lifecyclerConfig2.JoinAfter = 100 * time.Second lifecyclerConfig2.NormaliseTokens = true - l2, err := NewLifecycler(lifecyclerConfig2, &flushTransferer{}, "ingester") + l2, err := NewLifecycler(lifecyclerConfig2, &flushTransferer{}, "ingester", "") require.NoError(t, err) // This will block until l1 has successfully left the ring. @@ -122,7 +122,7 @@ func TestRingRestart(t *testing.T) { // Add an 'ingester' with normalised tokens. lifecyclerConfig1 := testLifecyclerConfig(ringConfig, "ing1") lifecyclerConfig1.NormaliseTokens = true - l1, err := NewLifecycler(lifecyclerConfig1, &nopFlushTransferer{}, "ingester") + l1, err := NewLifecycler(lifecyclerConfig1, &nopFlushTransferer{}, "ingester", "") require.NoError(t, err) // Check this ingester joined, is active, and has one token. @@ -135,7 +135,7 @@ func TestRingRestart(t *testing.T) { token := l1.tokens[0] // Add a second ingester with the same settings, so it will think it has restarted - l2, err := NewLifecycler(lifecyclerConfig1, &nopFlushTransferer{}, "ingester") + l2, err := NewLifecycler(lifecyclerConfig1, &nopFlushTransferer{}, "ingester", "") require.NoError(t, err) // Check the new ingester picked up the same token @@ -195,7 +195,7 @@ func TestCheckReady(t *testing.T) { defer r.Stop() cfg := testLifecyclerConfig(ringConfig, "ring1") cfg.MinReadyDuration = 1 * time.Nanosecond - l1, err := NewLifecycler(cfg, &nopFlushTransferer{}, "ingester") + l1, err := NewLifecycler(cfg, &nopFlushTransferer{}, "ingester", "") l1.setTokens([]uint32{1}) require.NoError(t, err) diff --git a/pkg/ring/tokens b/pkg/ring/tokens new file mode 100644 index 0000000000000000000000000000000000000000..720d64f4baafc33efdf971f02084aca5f25b34a5 GIT binary patch literal 4 LcmZQzU|<9Q00jU7 literal 0 HcmV?d00001 diff --git a/pkg/ruler/ruler.go b/pkg/ruler/ruler.go index b6e8f174fbe..c174c6c0f5f 100644 --- a/pkg/ruler/ruler.go +++ b/pkg/ruler/ruler.go @@ -159,7 +159,7 @@ func NewRuler(cfg Config, engine *promql.Engine, queryable storage.Queryable, d // If sharding is enabled, create/join a ring to distribute tokens to // the ruler if cfg.EnableSharding { - ruler.lifecycler, err = ring.NewLifecycler(cfg.LifecyclerConfig, ruler, "ruler") + ruler.lifecycler, err = ring.NewLifecycler(cfg.LifecyclerConfig, ruler, "ruler", "") if err != nil { return nil, err } diff --git a/pkg/ruler/tokens b/pkg/ruler/tokens new file mode 100644 index 00000000000..9dc4fb90cb4 --- /dev/null +++ b/pkg/ruler/tokens @@ -0,0 +1 @@ +��z� \ No newline at end of file From 1d3eeaf9007111e4583f98f8c9f6f6c3d245b7d1 Mon Sep 17 00:00:00 2001 From: Ganesh Vernekar Date: Wed, 23 Oct 2019 17:53:48 +0200 Subject: [PATCH 02/18] Read tokens from file on startup Signed-off-by: Ganesh Vernekar --- pkg/ring/lifecycler.go | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/pkg/ring/lifecycler.go b/pkg/ring/lifecycler.go index 9ddfdea935f..6e2ec3ff020 100644 --- a/pkg/ring/lifecycler.go +++ b/pkg/ring/lifecycler.go @@ -3,8 +3,10 @@ package ring import ( "context" "encoding/binary" + "errors" "flag" "fmt" + "io/ioutil" "os" "path" "sort" @@ -269,6 +271,28 @@ func (i *Lifecycler) flushTokensToFile() { } } +func (i *Lifecycler) getTokensFromFile() ([]uint32, error) { + tokenFilePath := path.Join(i.tokenDir, tokensFileName) + b, err := ioutil.ReadFile(tokenFilePath) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, err + } + if len(b)%4 != 0 { + return nil, errors.New("token data is not 4 byte aligned") + } + + numTokens := len(b) >> 2 + tokens := make([]uint32, 0, numTokens) + for i := 0; i < numTokens; i++ { + tokens = append(tokens, binary.BigEndian.Uint32(b[i<<2:])) + } + + return tokens, nil +} + func (i *Lifecycler) setTokens(tokens []uint32) { tokensOwned.WithLabelValues(i.RingName).Set(float64(len(tokens))) @@ -417,6 +441,19 @@ func (i *Lifecycler) initRing(ctx context.Context) error { ingesterDesc, ok := ringDesc.Ingesters[i.ID] if !ok { + // We load the tokens from the file only if it does not exist in the ring yet. + tokens, err := i.getTokensFromFile() + if err != nil { + level.Error(util.Logger).Log("msg", "error in getting tokens from file", "err", err) + } else if len(tokens) > 0 { + level.Info(util.Logger).Log("msg", "adding tokens from file", "num_tokens", len(tokens)) + if len(tokens) == i.cfg.NumTokens { + i.setState(ACTIVE) + } + ringDesc.AddIngester(i.ID, i.Addr, tokens, i.GetState(), i.cfg.NormaliseTokens) + return ringDesc, true, nil + } + // Either we are a new ingester, or consul must have restarted level.Info(util.Logger).Log("msg", "entry not found in ring, adding with no tokens") ringDesc.AddIngester(i.ID, i.Addr, []uint32{}, i.GetState(), i.cfg.NormaliseTokens) From e8df9b6f002291f9144e5d3e360cdb1047de2194 Mon Sep 17 00:00:00 2001 From: Ganesh Vernekar Date: Wed, 23 Oct 2019 18:58:00 +0200 Subject: [PATCH 03/18] Fix tests Signed-off-by: Ganesh Vernekar --- pkg/ingester/ingester.go | 2 +- pkg/ingester/lifecycle_test.go | 15 +++++++++++++++ pkg/ingester/tokens | 1 - pkg/ring/lifecycler.go | 14 ++++++++------ pkg/ring/lifecycler_test.go | 27 +++++++++++++++++---------- pkg/ruler/ruler.go | 2 +- 6 files changed, 42 insertions(+), 19 deletions(-) delete mode 100644 pkg/ingester/tokens diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index 5490348f020..4f5cc0ecbee 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -190,7 +190,7 @@ func New(cfg Config, clientConfig client.Config, limits *validation.Overrides, c } var err error - i.lifecycler, err = ring.NewLifecycler(cfg.LifecyclerConfig, i, "ingester", "") + i.lifecycler, err = ring.NewLifecycler(cfg.LifecyclerConfig, i, "ingester") if err != nil { return nil, err } diff --git a/pkg/ingester/lifecycle_test.go b/pkg/ingester/lifecycle_test.go index 6cabb4e0b6f..c9896e31b22 100644 --- a/pkg/ingester/lifecycle_test.go +++ b/pkg/ingester/lifecycle_test.go @@ -1,8 +1,11 @@ package ingester import ( + "fmt" "io" + "io/ioutil" "math" + "os" "testing" "time" @@ -91,11 +94,17 @@ func TestIngesterTransfer(t *testing.T) { limits, err := validation.NewOverrides(defaultLimitsTestConfig()) require.NoError(t, err) + tokenDir1, err := ioutil.TempDir(os.TempDir(), "ingester_transfer") + require.NoError(t, err) + tokenDir2, err := ioutil.TempDir(os.TempDir(), "ingester_transfer") + require.NoError(t, err) + // Start the first ingester, and get it into ACTIVE state. cfg1 := defaultIngesterTestConfig() cfg1.LifecyclerConfig.ID = "ingester1" cfg1.LifecyclerConfig.Addr = "ingester1" cfg1.LifecyclerConfig.JoinAfter = 0 * time.Second + cfg1.LifecyclerConfig.TokenFileDir = tokenDir1 cfg1.MaxTransferRetries = 10 ing1, err := New(cfg1, defaultClientTestConfig(), limits, nil, nil) require.NoError(t, err) @@ -139,6 +148,7 @@ func TestIngesterTransfer(t *testing.T) { cfg2.LifecyclerConfig.ID = "ingester2" cfg2.LifecyclerConfig.Addr = "ingester2" cfg2.LifecyclerConfig.JoinAfter = 100 * time.Second + cfg2.LifecyclerConfig.TokenFileDir = tokenDir2 ing2, err := New(cfg2, defaultClientTestConfig(), limits, nil, nil) require.NoError(t, err) @@ -178,11 +188,15 @@ func TestIngesterBadTransfer(t *testing.T) { limits, err := validation.NewOverrides(defaultLimitsTestConfig()) require.NoError(t, err) + tokenDir, err := ioutil.TempDir(os.TempDir(), "ingester_bad_transfer") + require.NoError(t, err) + // Start ingester in PENDING. cfg := defaultIngesterTestConfig() cfg.LifecyclerConfig.ID = "ingester1" cfg.LifecyclerConfig.Addr = "ingester1" cfg.LifecyclerConfig.JoinAfter = 100 * time.Second + cfg.LifecyclerConfig.TokenFileDir = tokenDir ing, err := New(cfg, defaultClientTestConfig(), limits, nil, nil) require.NoError(t, err) @@ -198,6 +212,7 @@ func TestIngesterBadTransfer(t *testing.T) { require.Error(t, err) // Check the ingester is still waiting. + fmt.Println(ing.lifecycler.GetState().String()) require.Equal(t, ring.PENDING, ing.lifecycler.GetState()) } diff --git a/pkg/ingester/tokens b/pkg/ingester/tokens deleted file mode 100644 index 7303e56e9fb..00000000000 --- a/pkg/ingester/tokens +++ /dev/null @@ -1 +0,0 @@ -��$ \ No newline at end of file diff --git a/pkg/ring/lifecycler.go b/pkg/ring/lifecycler.go index 6e2ec3ff020..7a5ec4a4ea3 100644 --- a/pkg/ring/lifecycler.go +++ b/pkg/ring/lifecycler.go @@ -60,6 +60,7 @@ type LifecyclerConfig struct { NormaliseTokens bool `yaml:"normalise_tokens,omitempty"` InfNames []string `yaml:"interface_names"` FinalSleep time.Duration `yaml:"final_sleep"` + TokenFileDir string `yaml:"token_file_dir,omitempty"` // For testing, you can override the address and ID of this ingester Addr string `yaml:"address"` @@ -90,6 +91,7 @@ func (cfg *LifecyclerConfig) RegisterFlagsWithPrefix(prefix string, f *flag.Flag flagext.DeprecatedFlag(f, prefix+"claim-on-rollout", "DEPRECATED. This feature is no longer optional.") f.BoolVar(&cfg.NormaliseTokens, prefix+"normalise-tokens", false, "Store tokens in a normalised fashion to reduce allocations.") f.DurationVar(&cfg.FinalSleep, prefix+"final-sleep", 30*time.Second, "Duration to sleep for before exiting, to ensure metrics are scraped.") + f.StringVar(&cfg.TokenFileDir, "ingester.token-file-dir", "", "Directory in which the token file is to be stored.") hostname, err := os.Hostname() if err != nil { @@ -116,7 +118,6 @@ type Lifecycler struct { cfg LifecyclerConfig flushTransferer FlushTransferer KVStore kv.Client - tokenDir string // Controls the lifecycle of the ingester quit chan struct{} @@ -141,7 +142,7 @@ type Lifecycler struct { } // NewLifecycler makes and starts a new Lifecycler. -func NewLifecycler(cfg LifecyclerConfig, flushTransferer FlushTransferer, name, tokenDir string) (*Lifecycler, error) { +func NewLifecycler(cfg LifecyclerConfig, flushTransferer FlushTransferer, name string) (*Lifecycler, error) { addr := cfg.Addr if addr == "" { var err error @@ -164,7 +165,6 @@ func NewLifecycler(cfg LifecyclerConfig, flushTransferer FlushTransferer, name, cfg: cfg, flushTransferer: flushTransferer, KVStore: store, - tokenDir: tokenDir, Addr: fmt.Sprintf("%s:%d", addr, port), ID: cfg.ID, @@ -253,7 +253,7 @@ func (i *Lifecycler) getTokens() []uint32 { } func (i *Lifecycler) flushTokensToFile() { - tokenFilePath := path.Join(i.tokenDir, tokensFileName) + tokenFilePath := path.Join(i.cfg.TokenFileDir, tokensFileName) f, err := os.OpenFile(tokenFilePath, os.O_WRONLY|os.O_CREATE, 0666) if err != nil { level.Error(util.Logger).Log("msg", "error in creating token file", "err", err) @@ -272,7 +272,7 @@ func (i *Lifecycler) flushTokensToFile() { } func (i *Lifecycler) getTokensFromFile() ([]uint32, error) { - tokenFilePath := path.Join(i.tokenDir, tokensFileName) + tokenFilePath := path.Join(i.cfg.TokenFileDir, tokensFileName) b, err := ioutil.ReadFile(tokenFilePath) if err != nil { if os.IsNotExist(err) { @@ -280,7 +280,9 @@ func (i *Lifecycler) getTokensFromFile() ([]uint32, error) { } return nil, err } - if len(b)%4 != 0 { + if len(b) == 0 { + return nil, nil + } else if len(b)%4 != 0 { return nil, errors.New("token data is not 4 byte aligned") } diff --git a/pkg/ring/lifecycler_test.go b/pkg/ring/lifecycler_test.go index d51d45f1b66..f48d8021569 100644 --- a/pkg/ring/lifecycler_test.go +++ b/pkg/ring/lifecycler_test.go @@ -2,6 +2,8 @@ package ring import ( "context" + "io/ioutil" + "os" "testing" "time" @@ -25,7 +27,10 @@ func (f *flushTransferer) TransferOut(ctx context.Context) error { return f.lifecycler.ChangeState(ctx, ACTIVE) } -func testLifecyclerConfig(ringConfig Config, id string) LifecyclerConfig { +func testLifecyclerConfig(t *testing.T, ringConfig Config, id string) LifecyclerConfig { + tokenDir, err := ioutil.TempDir(os.TempDir(), "ingester_bad_transfer") + require.NoError(t, err) + var lifecyclerConfig LifecyclerConfig flagext.DefaultValues(&lifecyclerConfig) lifecyclerConfig.Addr = "0.0.0.0" @@ -34,6 +39,8 @@ func testLifecyclerConfig(ringConfig Config, id string) LifecyclerConfig { lifecyclerConfig.NumTokens = 1 lifecyclerConfig.ID = id lifecyclerConfig.FinalSleep = 0 + lifecyclerConfig.TokenFileDir = tokenDir + return lifecyclerConfig } @@ -65,10 +72,10 @@ func TestRingNormaliseMigration(t *testing.T) { defer r.Stop() // Add an 'ingester' with denormalised tokens. - lifecyclerConfig1 := testLifecyclerConfig(ringConfig, "ing1") + lifecyclerConfig1 := testLifecyclerConfig(t, ringConfig, "ing1") ft := &flushTransferer{} - l1, err := NewLifecycler(lifecyclerConfig1, ft, "ingester", "") + l1, err := NewLifecycler(lifecyclerConfig1, ft, "ingester") require.NoError(t, err) // Check this ingester joined, is active, and has one token. @@ -81,11 +88,11 @@ func TestRingNormaliseMigration(t *testing.T) { token := l1.tokens[0] // Add a second ingester with normalised tokens. - var lifecyclerConfig2 = testLifecyclerConfig(ringConfig, "ing2") + var lifecyclerConfig2 = testLifecyclerConfig(t, ringConfig, "ing2") lifecyclerConfig2.JoinAfter = 100 * time.Second lifecyclerConfig2.NormaliseTokens = true - l2, err := NewLifecycler(lifecyclerConfig2, &flushTransferer{}, "ingester", "") + l2, err := NewLifecycler(lifecyclerConfig2, &flushTransferer{}, "ingester") require.NoError(t, err) // This will block until l1 has successfully left the ring. @@ -120,9 +127,9 @@ func TestRingRestart(t *testing.T) { defer r.Stop() // Add an 'ingester' with normalised tokens. - lifecyclerConfig1 := testLifecyclerConfig(ringConfig, "ing1") + lifecyclerConfig1 := testLifecyclerConfig(t, ringConfig, "ing1") lifecyclerConfig1.NormaliseTokens = true - l1, err := NewLifecycler(lifecyclerConfig1, &nopFlushTransferer{}, "ingester", "") + l1, err := NewLifecycler(lifecyclerConfig1, &nopFlushTransferer{}, "ingester") require.NoError(t, err) // Check this ingester joined, is active, and has one token. @@ -135,7 +142,7 @@ func TestRingRestart(t *testing.T) { token := l1.tokens[0] // Add a second ingester with the same settings, so it will think it has restarted - l2, err := NewLifecycler(lifecyclerConfig1, &nopFlushTransferer{}, "ingester", "") + l2, err := NewLifecycler(lifecyclerConfig1, &nopFlushTransferer{}, "ingester") require.NoError(t, err) // Check the new ingester picked up the same token @@ -193,9 +200,9 @@ func TestCheckReady(t *testing.T) { r, err := New(ringConfig, "ingester") require.NoError(t, err) defer r.Stop() - cfg := testLifecyclerConfig(ringConfig, "ring1") + cfg := testLifecyclerConfig(t, ringConfig, "ring1") cfg.MinReadyDuration = 1 * time.Nanosecond - l1, err := NewLifecycler(cfg, &nopFlushTransferer{}, "ingester", "") + l1, err := NewLifecycler(cfg, &nopFlushTransferer{}, "ingester") l1.setTokens([]uint32{1}) require.NoError(t, err) diff --git a/pkg/ruler/ruler.go b/pkg/ruler/ruler.go index c174c6c0f5f..b6e8f174fbe 100644 --- a/pkg/ruler/ruler.go +++ b/pkg/ruler/ruler.go @@ -159,7 +159,7 @@ func NewRuler(cfg Config, engine *promql.Engine, queryable storage.Queryable, d // If sharding is enabled, create/join a ring to distribute tokens to // the ruler if cfg.EnableSharding { - ruler.lifecycler, err = ring.NewLifecycler(cfg.LifecyclerConfig, ruler, "ruler", "") + ruler.lifecycler, err = ring.NewLifecycler(cfg.LifecyclerConfig, ruler, "ruler") if err != nil { return nil, err } From c2c43dcd6ed866798bf2dfa1ce00c09c6fdd5d5b Mon Sep 17 00:00:00 2001 From: Ganesh Vernekar Date: Wed, 23 Oct 2019 20:41:09 +0200 Subject: [PATCH 04/18] Don't flush on empty directory flag Signed-off-by: Ganesh Vernekar --- pkg/ingester/tokens | 1 + pkg/ring/lifecycler.go | 3 +++ 2 files changed, 4 insertions(+) create mode 100644 pkg/ingester/tokens diff --git a/pkg/ingester/tokens b/pkg/ingester/tokens new file mode 100644 index 00000000000..5af9289303a --- /dev/null +++ b/pkg/ingester/tokens @@ -0,0 +1 @@ +�\�� \ No newline at end of file diff --git a/pkg/ring/lifecycler.go b/pkg/ring/lifecycler.go index 7a5ec4a4ea3..c93c67cfee7 100644 --- a/pkg/ring/lifecycler.go +++ b/pkg/ring/lifecycler.go @@ -253,6 +253,9 @@ func (i *Lifecycler) getTokens() []uint32 { } func (i *Lifecycler) flushTokensToFile() { + if i.cfg.TokenFileDir == "" { + return + } tokenFilePath := path.Join(i.cfg.TokenFileDir, tokensFileName) f, err := os.OpenFile(tokenFilePath, os.O_WRONLY|os.O_CREATE, 0666) if err != nil { From 7e2ad44beb0b95dee81ac270a30b8b041ffdcb1d Mon Sep 17 00:00:00 2001 From: Ganesh Vernekar Date: Wed, 23 Oct 2019 21:12:10 +0200 Subject: [PATCH 05/18] Unit test Signed-off-by: Ganesh Vernekar --- pkg/ring/lifecycler_test.go | 70 +++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/pkg/ring/lifecycler_test.go b/pkg/ring/lifecycler_test.go index f48d8021569..661197ab283 100644 --- a/pkg/ring/lifecycler_test.go +++ b/pkg/ring/lifecycler_test.go @@ -210,3 +210,73 @@ func TestCheckReady(t *testing.T) { err = l1.CheckReady(context.Background()) require.Error(t, err) } + +type noopFlushTransferer struct { + lifecycler *Lifecycler +} + +func (f *noopFlushTransferer) StopIncomingRequests() {} +func (f *noopFlushTransferer) Flush() {} +func (f *noopFlushTransferer) TransferOut(ctx context.Context) error { return nil } + +func TestTokensOnDisk(t *testing.T) { + var ringConfig Config + flagext.DefaultValues(&ringConfig) + ringConfig.KVStore.Mock = consul.NewInMemoryClient(GetCodec()) + + r, err := New(ringConfig, "ingester") + require.NoError(t, err) + defer r.Stop() + + lifecyclerConfig := testLifecyclerConfig(t, ringConfig, "ing1") + lifecyclerConfig.NumTokens = 512 + + // Start first ingester. + ft := &noopFlushTransferer{} + l1, err := NewLifecycler(lifecyclerConfig, ft, "ingester") + require.NoError(t, err) + // Check this ingester joined, is active, and has 512 token. + var expTokens []TokenDesc + test.Poll(t, 1000*time.Millisecond, true, func() interface{} { + d, err := r.KVClient.Get(context.Background(), ConsulKey) + require.NoError(t, err) + desc, ok := d.(*Desc) + if ok { + expTokens = desc.Tokens + } + return ok && + len(desc.Ingesters) == 1 && + desc.Ingesters["ing1"].State == ACTIVE && + len(desc.Ingesters["ing1"].Tokens) == 0 && + len(desc.Tokens) == 512 + }) + + l1.Shutdown() + + // Start new ingester at same token directory. + lifecyclerConfig.ID = "ing2" + l2, err := NewLifecycler(lifecyclerConfig, &noopFlushTransferer{}, "ingester") + require.NoError(t, err) + defer l2.Shutdown() + + // Check this ingester joined, is active, and has 512 token. + var actTokens []TokenDesc + test.Poll(t, 1000*time.Millisecond, true, func() interface{} { + d, err := r.KVClient.Get(context.Background(), ConsulKey) + require.NoError(t, err) + desc, ok := d.(*Desc) + if ok { + actTokens = desc.Tokens + } + return ok && + len(desc.Ingesters) == 1 && + desc.Ingesters["ing2"].State == ACTIVE && + len(desc.Ingesters["ing2"].Tokens) == 0 && + len(desc.Tokens) == 512 + }) + + // Check for same tokens. + for i := 0; i < 512; i++ { + require.Equal(t, expTokens[i].Token, actTokens[i].Token) + } +} From 4fbd89d67c6a221067699cc286736712c5e7bfb9 Mon Sep 17 00:00:00 2001 From: Ganesh Vernekar Date: Thu, 24 Oct 2019 15:38:39 +0200 Subject: [PATCH 06/18] Fix some bugs Signed-off-by: Ganesh Vernekar --- pkg/ingester/lifecycle_test.go | 2 -- pkg/ring/lifecycler.go | 1 + 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/ingester/lifecycle_test.go b/pkg/ingester/lifecycle_test.go index c9896e31b22..76c30b3053b 100644 --- a/pkg/ingester/lifecycle_test.go +++ b/pkg/ingester/lifecycle_test.go @@ -1,7 +1,6 @@ package ingester import ( - "fmt" "io" "io/ioutil" "math" @@ -212,7 +211,6 @@ func TestIngesterBadTransfer(t *testing.T) { require.Error(t, err) // Check the ingester is still waiting. - fmt.Println(ing.lifecycler.GetState().String()) require.Equal(t, ring.PENDING, ing.lifecycler.GetState()) } diff --git a/pkg/ring/lifecycler.go b/pkg/ring/lifecycler.go index c93c67cfee7..7fe7dc62b77 100644 --- a/pkg/ring/lifecycler.go +++ b/pkg/ring/lifecycler.go @@ -456,6 +456,7 @@ func (i *Lifecycler) initRing(ctx context.Context) error { i.setState(ACTIVE) } ringDesc.AddIngester(i.ID, i.Addr, tokens, i.GetState(), i.cfg.NormaliseTokens) + i.setTokens(tokens) return ringDesc, true, nil } From 8aa452fa341bd06d0fd30764f2b8198374c51bb4 Mon Sep 17 00:00:00 2001 From: Ganesh Vernekar Date: Mon, 28 Oct 2019 15:02:13 +0100 Subject: [PATCH 07/18] Fix integration test Signed-off-by: Ganesh Vernekar --- pkg/ring/lifecycler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/ring/lifecycler.go b/pkg/ring/lifecycler.go index 7fe7dc62b77..e57770ff4ad 100644 --- a/pkg/ring/lifecycler.go +++ b/pkg/ring/lifecycler.go @@ -91,7 +91,7 @@ func (cfg *LifecyclerConfig) RegisterFlagsWithPrefix(prefix string, f *flag.Flag flagext.DeprecatedFlag(f, prefix+"claim-on-rollout", "DEPRECATED. This feature is no longer optional.") f.BoolVar(&cfg.NormaliseTokens, prefix+"normalise-tokens", false, "Store tokens in a normalised fashion to reduce allocations.") f.DurationVar(&cfg.FinalSleep, prefix+"final-sleep", 30*time.Second, "Duration to sleep for before exiting, to ensure metrics are scraped.") - f.StringVar(&cfg.TokenFileDir, "ingester.token-file-dir", "", "Directory in which the token file is to be stored.") + f.StringVar(&cfg.TokenFileDir, prefix+"token-file-dir", "", "Directory in which the token file is to be stored.") hostname, err := os.Hostname() if err != nil { From aa67711333821f6142a50684ef5a64474b1e4ccf Mon Sep 17 00:00:00 2001 From: Ganesh Vernekar Date: Fri, 1 Nov 2019 11:31:00 +0100 Subject: [PATCH 08/18] Fix review comments Signed-off-by: Ganesh Vernekar --- pkg/ingester/lifecycle_test.go | 6 +++--- pkg/ingester/tokens | 1 - pkg/ring/lifecycler.go | 13 +++++++------ pkg/ring/lifecycler_test.go | 2 +- pkg/ring/tokens | Bin 4 -> 0 bytes 5 files changed, 11 insertions(+), 11 deletions(-) delete mode 100644 pkg/ingester/tokens delete mode 100644 pkg/ring/tokens diff --git a/pkg/ingester/lifecycle_test.go b/pkg/ingester/lifecycle_test.go index 76c30b3053b..547fe8a5495 100644 --- a/pkg/ingester/lifecycle_test.go +++ b/pkg/ingester/lifecycle_test.go @@ -103,7 +103,7 @@ func TestIngesterTransfer(t *testing.T) { cfg1.LifecyclerConfig.ID = "ingester1" cfg1.LifecyclerConfig.Addr = "ingester1" cfg1.LifecyclerConfig.JoinAfter = 0 * time.Second - cfg1.LifecyclerConfig.TokenFileDir = tokenDir1 + cfg1.LifecyclerConfig.TokensFileDir = tokenDir1 cfg1.MaxTransferRetries = 10 ing1, err := New(cfg1, defaultClientTestConfig(), limits, nil, nil) require.NoError(t, err) @@ -147,7 +147,7 @@ func TestIngesterTransfer(t *testing.T) { cfg2.LifecyclerConfig.ID = "ingester2" cfg2.LifecyclerConfig.Addr = "ingester2" cfg2.LifecyclerConfig.JoinAfter = 100 * time.Second - cfg2.LifecyclerConfig.TokenFileDir = tokenDir2 + cfg2.LifecyclerConfig.TokensFileDir = tokenDir2 ing2, err := New(cfg2, defaultClientTestConfig(), limits, nil, nil) require.NoError(t, err) @@ -195,7 +195,7 @@ func TestIngesterBadTransfer(t *testing.T) { cfg.LifecyclerConfig.ID = "ingester1" cfg.LifecyclerConfig.Addr = "ingester1" cfg.LifecyclerConfig.JoinAfter = 100 * time.Second - cfg.LifecyclerConfig.TokenFileDir = tokenDir + cfg.LifecyclerConfig.TokensFileDir = tokenDir ing, err := New(cfg, defaultClientTestConfig(), limits, nil, nil) require.NoError(t, err) diff --git a/pkg/ingester/tokens b/pkg/ingester/tokens deleted file mode 100644 index 5af9289303a..00000000000 --- a/pkg/ingester/tokens +++ /dev/null @@ -1 +0,0 @@ -�\�� \ No newline at end of file diff --git a/pkg/ring/lifecycler.go b/pkg/ring/lifecycler.go index e57770ff4ad..70be7a444c5 100644 --- a/pkg/ring/lifecycler.go +++ b/pkg/ring/lifecycler.go @@ -60,7 +60,7 @@ type LifecyclerConfig struct { NormaliseTokens bool `yaml:"normalise_tokens,omitempty"` InfNames []string `yaml:"interface_names"` FinalSleep time.Duration `yaml:"final_sleep"` - TokenFileDir string `yaml:"token_file_dir,omitempty"` + TokensFileDir string `yaml:"token_file_dir,omitempty"` // For testing, you can override the address and ID of this ingester Addr string `yaml:"address"` @@ -91,7 +91,7 @@ func (cfg *LifecyclerConfig) RegisterFlagsWithPrefix(prefix string, f *flag.Flag flagext.DeprecatedFlag(f, prefix+"claim-on-rollout", "DEPRECATED. This feature is no longer optional.") f.BoolVar(&cfg.NormaliseTokens, prefix+"normalise-tokens", false, "Store tokens in a normalised fashion to reduce allocations.") f.DurationVar(&cfg.FinalSleep, prefix+"final-sleep", 30*time.Second, "Duration to sleep for before exiting, to ensure metrics are scraped.") - f.StringVar(&cfg.TokenFileDir, prefix+"token-file-dir", "", "Directory in which the token file is to be stored.") + f.StringVar(&cfg.TokensFileDir, prefix+"token-file-dir", "", "Directory in which the token file is to be stored.") hostname, err := os.Hostname() if err != nil { @@ -253,10 +253,10 @@ func (i *Lifecycler) getTokens() []uint32 { } func (i *Lifecycler) flushTokensToFile() { - if i.cfg.TokenFileDir == "" { + if i.cfg.TokensFileDir == "" { return } - tokenFilePath := path.Join(i.cfg.TokenFileDir, tokensFileName) + tokenFilePath := path.Join(i.cfg.TokensFileDir, tokensFileName) f, err := os.OpenFile(tokenFilePath, os.O_WRONLY|os.O_CREATE, 0666) if err != nil { level.Error(util.Logger).Log("msg", "error in creating token file", "err", err) @@ -275,7 +275,7 @@ func (i *Lifecycler) flushTokensToFile() { } func (i *Lifecycler) getTokensFromFile() ([]uint32, error) { - tokenFilePath := path.Join(i.cfg.TokenFileDir, tokensFileName) + tokenFilePath := path.Join(i.cfg.TokensFileDir, tokensFileName) b, err := ioutil.ReadFile(tokenFilePath) if err != nil { if os.IsNotExist(err) { @@ -302,8 +302,9 @@ func (i *Lifecycler) setTokens(tokens []uint32) { tokensOwned.WithLabelValues(i.RingName).Set(float64(len(tokens))) i.stateMtx.Lock() - defer i.stateMtx.Unlock() i.tokens = tokens + i.stateMtx.Unlock() + i.flushTokensToFile() } diff --git a/pkg/ring/lifecycler_test.go b/pkg/ring/lifecycler_test.go index 661197ab283..74842de4ec7 100644 --- a/pkg/ring/lifecycler_test.go +++ b/pkg/ring/lifecycler_test.go @@ -39,7 +39,7 @@ func testLifecyclerConfig(t *testing.T, ringConfig Config, id string) Lifecycler lifecyclerConfig.NumTokens = 1 lifecyclerConfig.ID = id lifecyclerConfig.FinalSleep = 0 - lifecyclerConfig.TokenFileDir = tokenDir + lifecyclerConfig.TokensFileDir = tokenDir return lifecyclerConfig } diff --git a/pkg/ring/tokens b/pkg/ring/tokens deleted file mode 100644 index 720d64f4baafc33efdf971f02084aca5f25b34a5..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4 LcmZQzU|<9Q00jU7 From 7ab02ffe583ea85795353e8db0909d68b9794378 Mon Sep 17 00:00:00 2001 From: Ganesh Vernekar Date: Fri, 1 Nov 2019 18:25:14 +0100 Subject: [PATCH 09/18] Add interface for tokens with a SimpleListTokens type Signed-off-by: Ganesh Vernekar --- pkg/ring/lifecycler.go | 64 ++++++++++------------- pkg/ring/lifecycler_test.go | 11 ++-- pkg/ring/model.go | 22 ++++---- pkg/ring/tokens.go | 100 ++++++++++++++++++++++++++++++++++++ 4 files changed, 143 insertions(+), 54 deletions(-) create mode 100644 pkg/ring/tokens.go diff --git a/pkg/ring/lifecycler.go b/pkg/ring/lifecycler.go index 70be7a444c5..0b2f7a5b3e4 100644 --- a/pkg/ring/lifecycler.go +++ b/pkg/ring/lifecycler.go @@ -2,8 +2,6 @@ package ring import ( "context" - "encoding/binary" - "errors" "flag" "fmt" "io/ioutil" @@ -133,7 +131,7 @@ type Lifecycler struct { // back empty. And it changes during lifecycle of ingester. stateMtx sync.Mutex state IngesterState - tokens []uint32 + tokens Tokens // Controls the ready-reporting readyLock sync.Mutex @@ -207,7 +205,7 @@ func (i *Lifecycler) CheckReady(ctx context.Context) error { return fmt.Errorf("error talking to consul: %s", err) } - if len(i.getTokens()) == 0 { + if i.getTokens().Len() == 0 { return fmt.Errorf("this ingester owns no tokens") } @@ -246,9 +244,14 @@ func (i *Lifecycler) ChangeState(ctx context.Context, state IngesterState) error return <-err } -func (i *Lifecycler) getTokens() []uint32 { +var emptyTokens Tokens = &SimpleListTokens{} + +func (i *Lifecycler) getTokens() Tokens { i.stateMtx.Lock() defer i.stateMtx.Unlock() + if i.tokens == nil { + return emptyTokens + } return i.tokens } @@ -263,18 +266,14 @@ func (i *Lifecycler) flushTokensToFile() { return } - b := make([]byte, 4*len(i.tokens)) - for idx, token := range i.tokens { - binary.BigEndian.PutUint32(b[idx*4:], token) - } - + b := i.tokens.Marshal(nil) if _, err = f.Write(b); err != nil { level.Error(util.Logger).Log("msg", "error in writing token file", "err", err) return } } -func (i *Lifecycler) getTokensFromFile() ([]uint32, error) { +func (i *Lifecycler) getTokensFromFile() (Tokens, error) { tokenFilePath := path.Join(i.cfg.TokensFileDir, tokensFileName) b, err := ioutil.ReadFile(tokenFilePath) if err != nil { @@ -283,23 +282,12 @@ func (i *Lifecycler) getTokensFromFile() ([]uint32, error) { } return nil, err } - if len(b) == 0 { - return nil, nil - } else if len(b)%4 != 0 { - return nil, errors.New("token data is not 4 byte aligned") - } - - numTokens := len(b) >> 2 - tokens := make([]uint32, 0, numTokens) - for i := 0; i < numTokens; i++ { - tokens = append(tokens, binary.BigEndian.Uint32(b[i<<2:])) - } - return tokens, nil + return UnmarshalTokens(b) } -func (i *Lifecycler) setTokens(tokens []uint32) { - tokensOwned.WithLabelValues(i.RingName).Set(float64(len(tokens))) +func (i *Lifecycler) setTokens(tokens Tokens) { + tokensOwned.WithLabelValues(i.RingName).Set(float64(tokens.Len())) i.stateMtx.Lock() i.tokens = tokens @@ -313,7 +301,7 @@ func (i *Lifecycler) ClaimTokensFor(ctx context.Context, ingesterID string) erro err := make(chan error) i.actorChan <- func() { - var tokens []uint32 + var tokens Tokens claimTokens := func(in interface{}) (out interface{}, retry bool, err error) { ringDesc, ok := in.(*Desc) @@ -451,12 +439,12 @@ func (i *Lifecycler) initRing(ctx context.Context) error { tokens, err := i.getTokensFromFile() if err != nil { level.Error(util.Logger).Log("msg", "error in getting tokens from file", "err", err) - } else if len(tokens) > 0 { - level.Info(util.Logger).Log("msg", "adding tokens from file", "num_tokens", len(tokens)) - if len(tokens) == i.cfg.NumTokens { + } else if tokens != nil && tokens.Len() > 0 { + level.Info(util.Logger).Log("msg", "adding tokens from file", "num_tokens", tokens.Len()) + if tokens.Len() == i.cfg.NumTokens { i.setState(ACTIVE) } - ringDesc.AddIngester(i.ID, i.Addr, tokens, i.GetState(), i.cfg.NormaliseTokens) + ringDesc.AddIngester(i.ID, i.Addr, tokens.Tokens(), i.GetState(), i.cfg.NormaliseTokens) i.setTokens(tokens) return ringDesc, true, nil } @@ -472,7 +460,7 @@ func (i *Lifecycler) initRing(ctx context.Context) error { tokens, _ := ringDesc.TokensFor(i.ID) i.setTokens(tokens) - level.Info(util.Logger).Log("msg", "existing entry found in ring", "state", i.GetState(), "tokens", len(tokens)) + level.Info(util.Logger).Log("msg", "existing entry found in ring", "state", i.GetState(), "tokens", tokens.Len()) return ringDesc, true, nil }) } @@ -489,17 +477,17 @@ func (i *Lifecycler) autoJoin(ctx context.Context) error { // At this point, we should not have any tokens, and we should be in PENDING state. myTokens, takenTokens := ringDesc.TokensFor(i.ID) - if len(myTokens) > 0 { - level.Error(util.Logger).Log("msg", "tokens already exist for this ingester - wasn't expecting any!", "num_tokens", len(myTokens)) + if myTokens.Len() > 0 { + level.Error(util.Logger).Log("msg", "tokens already exist for this ingester - wasn't expecting any!", "num_tokens", myTokens.Len()) } - newTokens := GenerateTokens(i.cfg.NumTokens-len(myTokens), takenTokens) + newTokens := GenerateTokens(i.cfg.NumTokens-myTokens.Len(), takenTokens.Tokens()) i.setState(ACTIVE) ringDesc.AddIngester(i.ID, i.Addr, newTokens, i.GetState(), i.cfg.NormaliseTokens) - tokens := append(myTokens, newTokens...) - sort.Sort(sortableUint32(tokens)) - i.setTokens(tokens) + myTokens.Add(newTokens...) + sort.Sort(sortableUint32(myTokens.Tokens())) + i.setTokens(myTokens) return ringDesc, true, nil }) @@ -520,7 +508,7 @@ func (i *Lifecycler) updateConsul(ctx context.Context) error { if !ok { // consul must have restarted level.Info(util.Logger).Log("msg", "found empty ring, inserting tokens") - ringDesc.AddIngester(i.ID, i.Addr, i.getTokens(), i.GetState(), i.cfg.NormaliseTokens) + ringDesc.AddIngester(i.ID, i.Addr, i.getTokens().Tokens(), i.GetState(), i.cfg.NormaliseTokens) } else { ingesterDesc.Timestamp = time.Now().Unix() ingesterDesc.State = i.GetState() diff --git a/pkg/ring/lifecycler_test.go b/pkg/ring/lifecycler_test.go index 74842de4ec7..cbbba02653e 100644 --- a/pkg/ring/lifecycler_test.go +++ b/pkg/ring/lifecycler_test.go @@ -85,7 +85,7 @@ func TestRingNormaliseMigration(t *testing.T) { return checkDenormalised(d, "ing1") }) - token := l1.tokens[0] + token := l1.tokens.Tokens()[0] // Add a second ingester with normalised tokens. var lifecyclerConfig2 = testLifecyclerConfig(t, ringConfig, "ing2") @@ -139,7 +139,7 @@ func TestRingRestart(t *testing.T) { return checkNormalised(d, "ing1") }) - token := l1.tokens[0] + token := l1.tokens.Tokens()[0] // Add a second ingester with the same settings, so it will think it has restarted l2, err := NewLifecycler(lifecyclerConfig1, &nopFlushTransferer{}, "ingester") @@ -151,8 +151,8 @@ func TestRingRestart(t *testing.T) { require.NoError(t, err) l2Tokens := l2.getTokens() return checkNormalised(d, "ing1") && - len(l2Tokens) == 1 && - l2Tokens[0] == token + l2Tokens.Len() == 1 && + l2Tokens.Tokens()[0] == token }) } @@ -197,13 +197,14 @@ func TestCheckReady(t *testing.T) { flagext.DefaultValues(&ringConfig) ringConfig.KVStore.Mock = &MockClient{} + tokens := SimpleListTokens([]uint32{1}) r, err := New(ringConfig, "ingester") require.NoError(t, err) defer r.Stop() cfg := testLifecyclerConfig(t, ringConfig, "ring1") cfg.MinReadyDuration = 1 * time.Nanosecond l1, err := NewLifecycler(cfg, &nopFlushTransferer{}, "ingester") - l1.setTokens([]uint32{1}) + l1.setTokens(&tokens) require.NoError(t, err) // Delete the ring key before checking ready diff --git a/pkg/ring/model.go b/pkg/ring/model.go index dc5739b7a82..2c1412d4f2b 100644 --- a/pkg/ring/model.go +++ b/pkg/ring/model.go @@ -75,14 +75,14 @@ func (d *Desc) RemoveIngester(id string) { // ClaimTokens transfers all the tokens from one ingester to another, // returning the claimed token. -func (d *Desc) ClaimTokens(from, to string, normaliseTokens bool) []uint32 { - var result []uint32 +func (d *Desc) ClaimTokens(from, to string, normaliseTokens bool) Tokens { + result := &SimpleListTokens{} if normaliseTokens { // If the ingester we are claiming from is normalising, get its tokens then erase them from the ring. if fromDesc, found := d.Ingesters[from]; found { - result = fromDesc.Tokens + result.Add(fromDesc.Tokens...) fromDesc.Tokens = nil d.Ingesters[from] = fromDesc } @@ -93,16 +93,16 @@ func (d *Desc) ClaimTokens(from, to string, normaliseTokens bool) []uint32 { // When all ingesters are in normalised mode, d.Tokens is empty here for i := 0; i < len(d.Tokens); { if d.Tokens[i].Ingester == from { - result = append(result, d.Tokens[i].Token) + result.Add(d.Tokens[i].Token) d.Tokens = d.Tokens[:i+copy(d.Tokens[i:], d.Tokens[i+1:])] continue } i++ } - sort.Sort(uint32s(result)) + sort.Sort(uint32s(result.Tokens())) ing := d.Ingesters[to] - ing.Tokens = result + ing.Tokens = result.Tokens() d.Ingesters[to] = ing } else { @@ -110,7 +110,7 @@ func (d *Desc) ClaimTokens(from, to string, normaliseTokens bool) []uint32 { for i := 0; i < len(d.Tokens); i++ { if d.Tokens[i].Ingester == from { d.Tokens[i].Ingester = to - result = append(result, d.Tokens[i].Token) + result.Add(d.Tokens[i].Token) } } } @@ -148,12 +148,12 @@ func (d *Desc) Ready(heartbeatTimeout time.Duration) error { } // TokensFor partitions the tokens into those for the given ID, and those for others. -func (d *Desc) TokensFor(id string) (tokens, other []uint32) { - var takenTokens, myTokens []uint32 +func (d *Desc) TokensFor(id string) (tokens, other Tokens) { + takenTokens, myTokens := &SimpleListTokens{}, &SimpleListTokens{} for _, token := range migrateRing(d) { - takenTokens = append(takenTokens, token.Token) + takenTokens.Add(token.Token) if token.Ingester == id { - myTokens = append(myTokens, token.Token) + myTokens.Add(token.Token) } } return myTokens, takenTokens diff --git a/pkg/ring/tokens.go b/pkg/ring/tokens.go new file mode 100644 index 00000000000..d50f5d7a18d --- /dev/null +++ b/pkg/ring/tokens.go @@ -0,0 +1,100 @@ +package ring + +import ( + "encoding/binary" + "errors" +) + +// Tokens is the interface to store Token. +type Tokens interface { + // Encoding return the type of the token implementation. + Encoding() TokenType + // Add adds a new token. + Add(...uint32) + // Tokens returns a slice of uint32 tokens. + Tokens() []uint32 + // Len returns the number of tokens. + Len() int + // Marshal marshalls all the tokens into byte stream. + Marshal([]byte) []byte + // Unmarshal reads the tokens from the byte stream. + Unmarshal([]byte) error +} + +// TokenType is the type identifier for the token. +type TokenType byte + +const ( + // SimpleList is the type for Type1Tokens. + SimpleList TokenType = 1 +) + +// SimpleListTokens is a simple list of tokens. +type SimpleListTokens []uint32 + +// Encoding implements the Tokens interface. +func (slt SimpleListTokens) Encoding() TokenType { + return SimpleList +} + +// Tokens implements the Tokens interface. +func (slt SimpleListTokens) Tokens() []uint32 { + return slt +} + +// Len implements the Tokens interface. +func (slt SimpleListTokens) Len() int { + return len(slt) +} + +// Add implements the Tokens interface. +func (slt *SimpleListTokens) Add(tokens ...uint32) { + *slt = append(*slt, tokens...) +} + +// Marshal implements the Tokens interface. +func (slt SimpleListTokens) Marshal(b []byte) []byte { + if cap(b) < 1+(4*len(slt)) { + b = make([]byte, 1+(4*len(slt))) + } + b = b[:1+(4*len(slt))] + b[0] = byte(SimpleList) + for idx, token := range slt { + binary.BigEndian.PutUint32(b[1+(idx*4):], uint32(token)) + } + return b +} + +// Unmarshal implements the Tokens interface. +func (slt *SimpleListTokens) Unmarshal(b []byte) error { + *slt = (*slt)[:0] + if len(b) == 0 { + return nil + } else if len(b)%4 != 0 { + return errors.New("token data is not 4 byte aligned") + } + + numTokens := len(b) >> 2 + for i := 0; i < numTokens; i++ { + *slt = append(*slt, binary.BigEndian.Uint32(b[i<<2:])) + } + + return nil +} + +// UnmarshalTokens converts the byte stream into Tokens. +// The first byte should be the token type follwed by the +// actual token data. +func UnmarshalTokens(b []byte) (Tokens, error) { + if len(b) == 0 { + return nil, errors.New("empty bytes") + } + switch TokenType(b[0]) { + case SimpleList: + tokens := &SimpleListTokens{} + err := tokens.Unmarshal(b[1:]) + return tokens, err + default: + return nil, errors.New("invalid token type") + } +} From 5c33c1cdaef1526c485e65a32ae5dd148fbc56aa Mon Sep 17 00:00:00 2001 From: Ganesh Vernekar Date: Mon, 4 Nov 2019 13:39:11 +0100 Subject: [PATCH 10/18] Remove Tokens interface Signed-off-by: Ganesh Vernekar --- pkg/ring/lifecycler.go | 88 ++++++++++++++++++++++--------- pkg/ring/lifecycler_test.go | 8 +-- pkg/ring/model.go | 20 ++++---- pkg/ring/model_test.go | 8 +-- pkg/ring/tokens.go | 100 ------------------------------------ 5 files changed, 82 insertions(+), 142 deletions(-) delete mode 100644 pkg/ring/tokens.go diff --git a/pkg/ring/lifecycler.go b/pkg/ring/lifecycler.go index 760eb979815..5235cba5561 100644 --- a/pkg/ring/lifecycler.go +++ b/pkg/ring/lifecycler.go @@ -2,6 +2,8 @@ package ring import ( "context" + "encoding/json" + "errors" "flag" "fmt" "io/ioutil" @@ -254,17 +256,17 @@ func (i *Lifecycler) ChangeState(ctx context.Context, state IngesterState) error return <-err } -func (i *Lifecycler) getTokens() []uint32 { +func (i *Lifecycler) getTokens() Tokens { i.stateMtx.Lock() defer i.stateMtx.Unlock() if i.tokens == nil { return nil } - return i.tokens.Tokens() + return i.tokens } func (i *Lifecycler) flushTokensToFile() { - if i.cfg.TokensFileDir == "" || i.tokens == nil || i.tokens.Len() == 0 { + if i.cfg.TokensFileDir == "" || i.tokens == nil || len(i.tokens) == 0 { return } tokenFilePath := path.Join(i.cfg.TokensFileDir, tokensFileName) @@ -274,7 +276,11 @@ func (i *Lifecycler) flushTokensToFile() { return } - b := i.tokens.Marshal(nil) + b, err := MarshalTokens(i.tokens) + if err != nil { + level.Error(util.Logger).Log("msg", "error in marshalling tokens", "err", err) + return + } if _, err = f.Write(b); err != nil { level.Error(util.Logger).Log("msg", "error in writing token file", "err", err) return @@ -295,7 +301,7 @@ func (i *Lifecycler) getTokensFromFile() (Tokens, error) { } func (i *Lifecycler) setTokens(tokens Tokens) { - tokensOwned.WithLabelValues(i.RingName).Set(float64(tokens.Len())) + tokensOwned.WithLabelValues(i.RingName).Set(float64(len(tokens))) i.stateMtx.Lock() i.tokens = tokens @@ -504,12 +510,12 @@ func (i *Lifecycler) initRing(ctx context.Context) error { tokens, err := i.getTokensFromFile() if err != nil { level.Error(util.Logger).Log("msg", "error in getting tokens from file", "err", err) - } else if tokens != nil && tokens.Len() > 0 { - level.Info(util.Logger).Log("msg", "adding tokens from file", "num_tokens", tokens.Len()) - if tokens.Len() == i.cfg.NumTokens { + } else if tokens != nil && len(tokens) > 0 { + level.Info(util.Logger).Log("msg", "adding tokens from file", "num_tokens", len(tokens)) + if len(tokens) == i.cfg.NumTokens { i.setState(ACTIVE) } - ringDesc.AddIngester(i.ID, i.Addr, tokens.Tokens(), i.GetState(), i.cfg.NormaliseTokens) + ringDesc.AddIngester(i.ID, i.Addr, tokens, i.GetState(), i.cfg.NormaliseTokens) i.setTokens(tokens) return ringDesc, true, nil } @@ -525,7 +531,7 @@ func (i *Lifecycler) initRing(ctx context.Context) error { tokens, _ := ringDesc.TokensFor(i.ID) i.setTokens(tokens) - level.Info(util.Logger).Log("msg", "existing entry found in ring", "state", i.GetState(), "tokens", tokens.Len()) + level.Info(util.Logger).Log("msg", "existing entry found in ring", "state", i.GetState(), "tokens", len(tokens)) // we haven't modified the ring, don't try to store it. return nil, true, nil }) @@ -557,15 +563,15 @@ func (i *Lifecycler) verifyTokens(ctx context.Context) bool { if !i.compareTokens(ringTokens) { // uh, oh... our tokens are not our anymore. Let's try new ones. - needTokens := i.cfg.NumTokens - ringTokens.Len() + needTokens := i.cfg.NumTokens - len(ringTokens) level.Info(util.Logger).Log("msg", "generating new tokens", "count", needTokens) - newTokens := GenerateTokens(needTokens, takenTokens.Tokens()) + newTokens := GenerateTokens(needTokens, takenTokens) - ringTokens.Add(newTokens...) - sort.Sort(sortableUint32(ringTokens.Tokens())) + ringTokens = append(ringTokens, newTokens...) + sort.Sort(sortableUint32(ringTokens)) - ringDesc.AddIngester(i.ID, i.Addr, ringTokens.Tokens(), i.GetState(), i.cfg.NormaliseTokens) + ringDesc.AddIngester(i.ID, i.Addr, ringTokens, i.GetState(), i.cfg.NormaliseTokens) i.setTokens(ringTokens) @@ -586,18 +592,17 @@ func (i *Lifecycler) verifyTokens(ctx context.Context) bool { } func (i *Lifecycler) compareTokens(fromRing Tokens) bool { - sort.Sort(sortableUint32(fromRing.Tokens())) + sort.Sort(sortableUint32(fromRing)) tokens := i.getTokens() sort.Sort(sortableUint32(tokens)) - if len(tokens) != fromRing.Len() { + if len(tokens) != len(fromRing) { return false } - tokensFromRing := fromRing.Tokens() for i := 0; i < len(tokens); i++ { - if tokens[i] != tokensFromRing[i] { + if tokens[i] != fromRing[i] { return false } } @@ -617,17 +622,17 @@ func (i *Lifecycler) autoJoin(ctx context.Context, targetState IngesterState) er // At this point, we should not have any tokens, and we should be in PENDING state. myTokens, takenTokens := ringDesc.TokensFor(i.ID) - if myTokens.Len() > 0 { - level.Error(util.Logger).Log("msg", "tokens already exist for this ingester - wasn't expecting any!", "num_tokens", myTokens.Len()) + if len(myTokens) > 0 { + level.Error(util.Logger).Log("msg", "tokens already exist for this ingester - wasn't expecting any!", "num_tokens", len(myTokens)) } - newTokens := GenerateTokens(i.cfg.NumTokens-myTokens.Len(), takenTokens.Tokens()) + newTokens := GenerateTokens(i.cfg.NumTokens-len(myTokens), takenTokens) i.setState(targetState) ringDesc.AddIngester(i.ID, i.Addr, newTokens, i.GetState(), i.cfg.NormaliseTokens) - myTokens.Add(newTokens...) - sort.Sort(sortableUint32(myTokens.Tokens())) + myTokens = append(myTokens, newTokens...) + sort.Sort(sortableUint32(myTokens)) i.setTokens(myTokens) return ringDesc, true, nil @@ -747,3 +752,38 @@ func (i *Lifecycler) unregister(ctx context.Context) error { return ringDesc, true, nil }) } + +// TokenVersion1 is the version is a simple list of tokens. +const TokenVersion1 = 1 + +// Tokens is a simple list of tokens. +type Tokens []uint32 + +// MarshalTokens encodes given tokens into JSON. +func MarshalTokens(tokens Tokens) ([]byte, error) { + data := tokensJSON{ + Version: TokenVersion1, + Tokens: tokens, + } + return json.Marshal(data) +} + +// UnmarshalTokens converts the JSON byte stream into Tokens. +func UnmarshalTokens(b []byte) (Tokens, error) { + tj := tokensJSON{} + if err := json.Unmarshal(b, &tj); err != nil { + return nil, err + } + switch tj.Version { + case TokenVersion1: + tokens := Tokens(tj.Tokens) + return tokens, nil + default: + return nil, errors.New("invalid token type") + } +} + +type tokensJSON struct { + Version int `json:"version"` + Tokens []uint32 `json:"tokens"` +} diff --git a/pkg/ring/lifecycler_test.go b/pkg/ring/lifecycler_test.go index 0f8d3778a30..b16df44e67a 100644 --- a/pkg/ring/lifecycler_test.go +++ b/pkg/ring/lifecycler_test.go @@ -87,7 +87,7 @@ func TestRingNormaliseMigration(t *testing.T) { return checkDenormalised(d, "ing1") }) - token := l1.tokens.Tokens()[0] + token := l1.tokens[0] // Add a second ingester with normalised tokens. var lifecyclerConfig2 = testLifecyclerConfig(t, ringConfig, "ing2") @@ -190,7 +190,7 @@ func TestRingRestart(t *testing.T) { return checkNormalised(d, "ing1") }) - token := l1.tokens.Tokens()[0] + token := l1.tokens[0] // Add a second ingester with the same settings, so it will think it has restarted l2, err := NewLifecycler(lifecyclerConfig1, &nopFlushTransferer{}, "ingester") @@ -253,14 +253,14 @@ func TestCheckReady(t *testing.T) { flagext.DefaultValues(&ringConfig) ringConfig.KVStore.Mock = &MockClient{} - tokens := SimpleListTokens([]uint32{1}) + tokens := Tokens([]uint32{1}) r, err := New(ringConfig, "ingester") require.NoError(t, err) defer r.Stop() cfg := testLifecyclerConfig(t, ringConfig, "ring1") cfg.MinReadyDuration = 1 * time.Nanosecond l1, err := NewLifecycler(cfg, &nopFlushTransferer{}, "ingester") - l1.setTokens(&tokens) + l1.setTokens(tokens) l1.Start() require.NoError(t, err) diff --git a/pkg/ring/model.go b/pkg/ring/model.go index 69d26c30f8c..fd802672a07 100644 --- a/pkg/ring/model.go +++ b/pkg/ring/model.go @@ -80,13 +80,13 @@ func (d *Desc) RemoveIngester(id string) { // and 'to' ingester uses either normalised or non-normalised tokens, but not both. Tokens list must // be sorted properly. If all of this is true, everything will be fine. func (d *Desc) ClaimTokens(from, to string, normaliseTokens bool) Tokens { - result := &SimpleListTokens{} + result := Tokens{} if normaliseTokens { // If the ingester we are claiming from is normalising, get its tokens then erase them from the ring. if fromDesc, found := d.Ingesters[from]; found { - result.Add(fromDesc.Tokens...) + result = append(result, fromDesc.Tokens...) fromDesc.Tokens = nil d.Ingesters[from] = fromDesc } @@ -97,7 +97,7 @@ func (d *Desc) ClaimTokens(from, to string, normaliseTokens bool) Tokens { // When all ingesters are in normalised mode, d.Tokens is empty here for i := 0; i < len(d.Tokens); { if d.Tokens[i].Ingester == from { - result.Add(d.Tokens[i].Token) + result = append(result, d.Tokens[i].Token) d.Tokens = append(d.Tokens[:i], d.Tokens[i+1:]...) continue } @@ -105,17 +105,17 @@ func (d *Desc) ClaimTokens(from, to string, normaliseTokens bool) Tokens { } ing := d.Ingesters[to] - ing.Tokens = result.Tokens() + ing.Tokens = result d.Ingesters[to] = ing } else { // If source ingester is normalising, copy its tokens to d.Tokens, and set new owner if fromDesc, found := d.Ingesters[from]; found { - result.Add(fromDesc.Tokens...) + result = append(result, fromDesc.Tokens...) fromDesc.Tokens = nil d.Ingesters[from] = fromDesc - for _, t := range result.Tokens() { + for _, t := range result { d.Tokens = append(d.Tokens, TokenDesc{Ingester: to, Token: t}) } @@ -126,7 +126,7 @@ func (d *Desc) ClaimTokens(from, to string, normaliseTokens bool) Tokens { for i := 0; i < len(d.Tokens); i++ { if d.Tokens[i].Ingester == from { d.Tokens[i].Ingester = to - result.Add(d.Tokens[i].Token) + result = append(result, d.Tokens[i].Token) } } } @@ -170,11 +170,11 @@ func (d *Desc) Ready(now time.Time, heartbeatTimeout time.Duration) error { // TokensFor partitions the tokens into those for the given ID, and those for others. func (d *Desc) TokensFor(id string) (tokens, other Tokens) { - takenTokens, myTokens := &SimpleListTokens{}, &SimpleListTokens{} + takenTokens, myTokens := Tokens{}, Tokens{} for _, token := range migrateRing(d) { - takenTokens.Add(token.Token) + takenTokens = append(takenTokens, token.Token) if token.Ingester == id { - myTokens.Add(token.Token) + myTokens = append(myTokens, token.Token) } } return myTokens, takenTokens diff --git a/pkg/ring/model_test.go b/pkg/ring/model_test.go index d3436b90b2e..e4656271ced 100644 --- a/pkg/ring/model_test.go +++ b/pkg/ring/model_test.go @@ -104,7 +104,7 @@ func TestClaimTokensFromNormalizedToNormalized(t *testing.T) { r := normalizedSource() result := r.ClaimTokens("first", "second", true) - assert.Equal(t, []uint32{100, 200, 300}, result.Tokens()) + assert.Equal(t, []uint32{100, 200, 300}, []uint32(result)) assert.Equal(t, normalizedOutput(), r) } @@ -112,7 +112,7 @@ func TestClaimTokensFromNormalizedToUnnormalized(t *testing.T) { r := normalizedSource() result := r.ClaimTokens("first", "second", false) - assert.Equal(t, []uint32{100, 200, 300}, result.Tokens()) + assert.Equal(t, []uint32{100, 200, 300}, []uint32(result)) assert.Equal(t, unnormalizedOutput(), r) } @@ -120,7 +120,7 @@ func TestClaimTokensFromUnnormalizedToUnnormalized(t *testing.T) { r := unnormalizedSource() result := r.ClaimTokens("first", "second", false) - assert.Equal(t, []uint32{100, 200, 300}, result.Tokens()) + assert.Equal(t, []uint32{100, 200, 300}, []uint32(result)) assert.Equal(t, unnormalizedOutput(), r) } @@ -129,7 +129,7 @@ func TestClaimTokensFromUnnormalizedToNormalized(t *testing.T) { result := r.ClaimTokens("first", "second", true) - assert.Equal(t, []uint32{100, 200, 300}, result.Tokens()) + assert.Equal(t, []uint32{100, 200, 300}, []uint32(result)) assert.Equal(t, normalizedOutput(), r) } diff --git a/pkg/ring/tokens.go b/pkg/ring/tokens.go deleted file mode 100644 index d50f5d7a18d..00000000000 --- a/pkg/ring/tokens.go +++ /dev/null @@ -1,100 +0,0 @@ -package ring - -import ( - "encoding/binary" - "errors" -) - -// Tokens is the interface to store Token. -type Tokens interface { - // Encoding return the type of the token implementation. - Encoding() TokenType - // Add adds a new token. - Add(...uint32) - // Tokens returns a slice of uint32 tokens. - Tokens() []uint32 - // Len returns the number of tokens. - Len() int - // Marshal marshalls all the tokens into byte stream. - Marshal([]byte) []byte - // Unmarshal reads the tokens from the byte stream. - Unmarshal([]byte) error -} - -// TokenType is the type identifier for the token. -type TokenType byte - -const ( - // SimpleList is the type for Type1Tokens. - SimpleList TokenType = 1 -) - -// SimpleListTokens is a simple list of tokens. -type SimpleListTokens []uint32 - -// Encoding implements the Tokens interface. -func (slt SimpleListTokens) Encoding() TokenType { - return SimpleList -} - -// Tokens implements the Tokens interface. -func (slt SimpleListTokens) Tokens() []uint32 { - return slt -} - -// Len implements the Tokens interface. -func (slt SimpleListTokens) Len() int { - return len(slt) -} - -// Add implements the Tokens interface. -func (slt *SimpleListTokens) Add(tokens ...uint32) { - *slt = append(*slt, tokens...) -} - -// Marshal implements the Tokens interface. -func (slt SimpleListTokens) Marshal(b []byte) []byte { - if cap(b) < 1+(4*len(slt)) { - b = make([]byte, 1+(4*len(slt))) - } - b = b[:1+(4*len(slt))] - b[0] = byte(SimpleList) - for idx, token := range slt { - binary.BigEndian.PutUint32(b[1+(idx*4):], uint32(token)) - } - return b -} - -// Unmarshal implements the Tokens interface. -func (slt *SimpleListTokens) Unmarshal(b []byte) error { - *slt = (*slt)[:0] - if len(b) == 0 { - return nil - } else if len(b)%4 != 0 { - return errors.New("token data is not 4 byte aligned") - } - - numTokens := len(b) >> 2 - for i := 0; i < numTokens; i++ { - *slt = append(*slt, binary.BigEndian.Uint32(b[i<<2:])) - } - - return nil -} - -// UnmarshalTokens converts the byte stream into Tokens. -// The first byte should be the token type follwed by the -// actual token data. -func UnmarshalTokens(b []byte) (Tokens, error) { - if len(b) == 0 { - return nil, errors.New("empty bytes") - } - switch TokenType(b[0]) { - case SimpleList: - tokens := &SimpleListTokens{} - err := tokens.Unmarshal(b[1:]) - return tokens, err - default: - return nil, errors.New("invalid token type") - } -} From d00ff977897c04d0c3fdf5f316a747074da55bfd Mon Sep 17 00:00:00 2001 From: Ganesh Vernekar Date: Wed, 6 Nov 2019 13:40:12 +0100 Subject: [PATCH 11/18] Fix review comments Signed-off-by: Ganesh Vernekar --- pkg/ring/lifecycler.go | 89 ++++-------------------------------------- pkg/ring/tokens.go | 83 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 90 insertions(+), 82 deletions(-) create mode 100644 pkg/ring/tokens.go diff --git a/pkg/ring/lifecycler.go b/pkg/ring/lifecycler.go index 5235cba5561..3037a8e2c02 100644 --- a/pkg/ring/lifecycler.go +++ b/pkg/ring/lifecycler.go @@ -2,13 +2,9 @@ package ring import ( "context" - "encoding/json" - "errors" "flag" "fmt" - "io/ioutil" "os" - "path" "sort" "sync" "time" @@ -42,10 +38,6 @@ var ( }, []string{"op", "status", "name"}) ) -const ( - tokensFileName = "tokens" -) - // LifecyclerConfig is the config to build a Lifecycler. type LifecyclerConfig struct { RingConfig Config `yaml:"ring,omitempty"` @@ -61,7 +53,7 @@ type LifecyclerConfig struct { NormaliseTokens bool `yaml:"normalise_tokens,omitempty"` InfNames []string `yaml:"interface_names"` FinalSleep time.Duration `yaml:"final_sleep"` - TokensFileDir string `yaml:"token_file_dir,omitempty"` + TokensFileDir string `yaml:"tokens_file_dir,omitempty"` // For testing, you can override the address and ID of this ingester Addr string `yaml:"address"` @@ -93,7 +85,7 @@ func (cfg *LifecyclerConfig) RegisterFlagsWithPrefix(prefix string, f *flag.Flag flagext.DeprecatedFlag(f, prefix+"claim-on-rollout", "DEPRECATED. This feature is no longer optional.") f.BoolVar(&cfg.NormaliseTokens, prefix+"normalise-tokens", false, "Store tokens in a normalised fashion to reduce allocations.") f.DurationVar(&cfg.FinalSleep, prefix+"final-sleep", 30*time.Second, "Duration to sleep for before exiting, to ensure metrics are scraped.") - f.StringVar(&cfg.TokensFileDir, prefix+"token-file-dir", "", "Directory in which the token file is to be stored.") + f.StringVar(&cfg.TokensFileDir, prefix+"tokens-file-dir", "", "Directory in which the tokens file is stored. If empty, tokens are not stored at shutdown and restored at startup.") hostname, err := os.Hostname() if err != nil { @@ -265,41 +257,6 @@ func (i *Lifecycler) getTokens() Tokens { return i.tokens } -func (i *Lifecycler) flushTokensToFile() { - if i.cfg.TokensFileDir == "" || i.tokens == nil || len(i.tokens) == 0 { - return - } - tokenFilePath := path.Join(i.cfg.TokensFileDir, tokensFileName) - f, err := os.OpenFile(tokenFilePath, os.O_WRONLY|os.O_CREATE, 0666) - if err != nil { - level.Error(util.Logger).Log("msg", "error in creating token file", "err", err) - return - } - - b, err := MarshalTokens(i.tokens) - if err != nil { - level.Error(util.Logger).Log("msg", "error in marshalling tokens", "err", err) - return - } - if _, err = f.Write(b); err != nil { - level.Error(util.Logger).Log("msg", "error in writing token file", "err", err) - return - } -} - -func (i *Lifecycler) getTokensFromFile() (Tokens, error) { - tokenFilePath := path.Join(i.cfg.TokensFileDir, tokensFileName) - b, err := ioutil.ReadFile(tokenFilePath) - if err != nil { - if os.IsNotExist(err) { - return nil, nil - } - return nil, err - } - - return UnmarshalTokens(b) -} - func (i *Lifecycler) setTokens(tokens Tokens) { tokensOwned.WithLabelValues(i.RingName).Set(float64(len(tokens))) @@ -307,7 +264,9 @@ func (i *Lifecycler) setTokens(tokens Tokens) { i.tokens = tokens i.stateMtx.Unlock() - i.flushTokensToFile() + if err := i.tokens.StoreToFile(i.cfg.TokensFileDir); err != nil { + level.Error(util.Logger).Log("msg", "error storing tokens to disk", "err", err) + } } // ClaimTokensFor takes all the tokens for the supplied ingester and assigns them to this ingester. @@ -506,8 +465,9 @@ func (i *Lifecycler) initRing(ctx context.Context) error { ingesterDesc, ok := ringDesc.Ingesters[i.ID] if !ok { + var tokens Tokens // We load the tokens from the file only if it does not exist in the ring yet. - tokens, err := i.getTokensFromFile() + err := tokens.LoadFromFile(i.cfg.TokensFileDir) if err != nil { level.Error(util.Logger).Log("msg", "error in getting tokens from file", "err", err) } else if tokens != nil && len(tokens) > 0 { @@ -752,38 +712,3 @@ func (i *Lifecycler) unregister(ctx context.Context) error { return ringDesc, true, nil }) } - -// TokenVersion1 is the version is a simple list of tokens. -const TokenVersion1 = 1 - -// Tokens is a simple list of tokens. -type Tokens []uint32 - -// MarshalTokens encodes given tokens into JSON. -func MarshalTokens(tokens Tokens) ([]byte, error) { - data := tokensJSON{ - Version: TokenVersion1, - Tokens: tokens, - } - return json.Marshal(data) -} - -// UnmarshalTokens converts the JSON byte stream into Tokens. -func UnmarshalTokens(b []byte) (Tokens, error) { - tj := tokensJSON{} - if err := json.Unmarshal(b, &tj); err != nil { - return nil, err - } - switch tj.Version { - case TokenVersion1: - tokens := Tokens(tj.Tokens) - return tokens, nil - default: - return nil, errors.New("invalid token type") - } -} - -type tokensJSON struct { - Version int `json:"version"` - Tokens []uint32 `json:"tokens"` -} diff --git a/pkg/ring/tokens.go b/pkg/ring/tokens.go new file mode 100644 index 00000000000..93258fe7a0c --- /dev/null +++ b/pkg/ring/tokens.go @@ -0,0 +1,83 @@ +package ring + +import ( + "encoding/json" + "errors" + "io/ioutil" + "os" + "path" +) + +const ( + // TokenVersion1 is the version is a simple list of tokens. + TokenVersion1 = 1 + + // Name of the file in which tokens is stored. + tokensFileName = "tokens" +) + +// Tokens is a simple list of tokens. +type Tokens []uint32 + +// StoreToFile stores the tokens in the given directory. +func (t Tokens) StoreToFile(dir string) error { + if dir == "" { + return errors.New("directory is empty") + } + tokenFilePath := path.Join(dir, tokensFileName) + f, err := os.OpenFile(tokenFilePath, os.O_WRONLY|os.O_CREATE, 0666) + if err != nil { + return err + } + + b, err := t.Marshal() + if err != nil { + return err + } + if _, err = f.Write(b); err != nil { + return err + } + return nil +} + +// LoadFromFile loads tokens from given directory. +func (t *Tokens) LoadFromFile(dir string) error { + tokenFilePath := path.Join(dir, tokensFileName) + b, err := ioutil.ReadFile(tokenFilePath) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + return t.Unmarshal(b) +} + +// Marshal encodes the tokens into JSON. +func (t Tokens) Marshal() ([]byte, error) { + data := tokensJSON{ + Version: TokenVersion1, + Tokens: t, + } + return json.Marshal(data) +} + +// Unmarshal reads the tokens from JSON byte stream. +func (t *Tokens) Unmarshal(b []byte) error { + tj := tokensJSON{} + if err := json.Unmarshal(b, &tj); err != nil { + return err + } + switch tj.Version { + case TokenVersion1: + *t = Tokens(tj.Tokens) + return nil + default: + return errors.New("invalid token type") + } +} + +type tokensJSON struct { + Version int `json:"version"` + Tokens []uint32 `json:"tokens"` +} From f4f886c9a0fb5252a26867c16443830e4d500dbb Mon Sep 17 00:00:00 2001 From: Ganesh Vernekar Date: Mon, 11 Nov 2019 11:56:44 +0100 Subject: [PATCH 12/18] Fix review comments Signed-off-by: Ganesh Vernekar --- pkg/ingester/lifecycle_test.go | 12 +++++++++--- pkg/ring/lifecycler.go | 8 ++++---- pkg/ring/lifecycler_test.go | 2 +- pkg/ring/model.go | 6 +++--- pkg/ring/tokens.go | 36 ++++++++++++++++++++++------------ pkg/ruler/tokens | 1 - 6 files changed, 41 insertions(+), 24 deletions(-) delete mode 100644 pkg/ruler/tokens diff --git a/pkg/ingester/lifecycle_test.go b/pkg/ingester/lifecycle_test.go index b6af1fcf07f..b64164e37b7 100644 --- a/pkg/ingester/lifecycle_test.go +++ b/pkg/ingester/lifecycle_test.go @@ -97,15 +97,21 @@ func TestIngesterTransfer(t *testing.T) { tokenDir1, err := ioutil.TempDir(os.TempDir(), "ingester_transfer") require.NoError(t, err) + defer func() { + require.NoError(t, os.RemoveAll(tokenDir1)) + }() tokenDir2, err := ioutil.TempDir(os.TempDir(), "ingester_transfer") require.NoError(t, err) + defer func() { + require.NoError(t, os.RemoveAll(tokenDir2)) + }() // Start the first ingester, and get it into ACTIVE state. cfg1 := defaultIngesterTestConfig() cfg1.LifecyclerConfig.ID = "ingester1" cfg1.LifecyclerConfig.Addr = "ingester1" cfg1.LifecyclerConfig.JoinAfter = 0 * time.Second - cfg1.LifecyclerConfig.TokensFileDir = tokenDir1 + cfg1.LifecyclerConfig.TokensFilePath = tokenDir1 cfg1.MaxTransferRetries = 10 ing1, err := New(cfg1, defaultClientTestConfig(), limits, nil, nil) require.NoError(t, err) @@ -149,7 +155,7 @@ func TestIngesterTransfer(t *testing.T) { cfg2.LifecyclerConfig.ID = "ingester2" cfg2.LifecyclerConfig.Addr = "ingester2" cfg2.LifecyclerConfig.JoinAfter = 100 * time.Second - cfg2.LifecyclerConfig.TokensFileDir = tokenDir2 + cfg2.LifecyclerConfig.TokensFilePath = tokenDir2 ing2, err := New(cfg2, defaultClientTestConfig(), limits, nil, nil) require.NoError(t, err) @@ -197,7 +203,7 @@ func TestIngesterBadTransfer(t *testing.T) { cfg.LifecyclerConfig.ID = "ingester1" cfg.LifecyclerConfig.Addr = "ingester1" cfg.LifecyclerConfig.JoinAfter = 100 * time.Second - cfg.LifecyclerConfig.TokensFileDir = tokenDir + cfg.LifecyclerConfig.TokensFilePath = tokenDir ing, err := New(cfg, defaultClientTestConfig(), limits, nil, nil) require.NoError(t, err) diff --git a/pkg/ring/lifecycler.go b/pkg/ring/lifecycler.go index 3037a8e2c02..f562475727d 100644 --- a/pkg/ring/lifecycler.go +++ b/pkg/ring/lifecycler.go @@ -53,7 +53,7 @@ type LifecyclerConfig struct { NormaliseTokens bool `yaml:"normalise_tokens,omitempty"` InfNames []string `yaml:"interface_names"` FinalSleep time.Duration `yaml:"final_sleep"` - TokensFileDir string `yaml:"tokens_file_dir,omitempty"` + TokensFilePath string `yaml:"tokens_file_path,omitempty"` // For testing, you can override the address and ID of this ingester Addr string `yaml:"address"` @@ -85,7 +85,7 @@ func (cfg *LifecyclerConfig) RegisterFlagsWithPrefix(prefix string, f *flag.Flag flagext.DeprecatedFlag(f, prefix+"claim-on-rollout", "DEPRECATED. This feature is no longer optional.") f.BoolVar(&cfg.NormaliseTokens, prefix+"normalise-tokens", false, "Store tokens in a normalised fashion to reduce allocations.") f.DurationVar(&cfg.FinalSleep, prefix+"final-sleep", 30*time.Second, "Duration to sleep for before exiting, to ensure metrics are scraped.") - f.StringVar(&cfg.TokensFileDir, prefix+"tokens-file-dir", "", "Directory in which the tokens file is stored. If empty, tokens are not stored at shutdown and restored at startup.") + f.StringVar(&cfg.TokensFilePath, prefix+"tokens-file-path", "", "File path in which the tokens is stored. If empty, tokens are not stored at shutdown and restored at startup.") hostname, err := os.Hostname() if err != nil { @@ -264,7 +264,7 @@ func (i *Lifecycler) setTokens(tokens Tokens) { i.tokens = tokens i.stateMtx.Unlock() - if err := i.tokens.StoreToFile(i.cfg.TokensFileDir); err != nil { + if err := i.tokens.StoreToFile(i.cfg.TokensFilePath); err != nil { level.Error(util.Logger).Log("msg", "error storing tokens to disk", "err", err) } } @@ -467,7 +467,7 @@ func (i *Lifecycler) initRing(ctx context.Context) error { if !ok { var tokens Tokens // We load the tokens from the file only if it does not exist in the ring yet. - err := tokens.LoadFromFile(i.cfg.TokensFileDir) + err := tokens.LoadFromFile(i.cfg.TokensFilePath) if err != nil { level.Error(util.Logger).Log("msg", "error in getting tokens from file", "err", err) } else if tokens != nil && len(tokens) > 0 { diff --git a/pkg/ring/lifecycler_test.go b/pkg/ring/lifecycler_test.go index b16df44e67a..3b934383270 100644 --- a/pkg/ring/lifecycler_test.go +++ b/pkg/ring/lifecycler_test.go @@ -40,7 +40,7 @@ func testLifecyclerConfig(t *testing.T, ringConfig Config, id string) Lifecycler lifecyclerConfig.NumTokens = 1 lifecyclerConfig.ID = id lifecyclerConfig.FinalSleep = 0 - lifecyclerConfig.TokensFileDir = tokenDir + lifecyclerConfig.TokensFilePath = tokenDir return lifecyclerConfig } diff --git a/pkg/ring/model.go b/pkg/ring/model.go index fd802672a07..8b1f7200175 100644 --- a/pkg/ring/model.go +++ b/pkg/ring/model.go @@ -80,13 +80,13 @@ func (d *Desc) RemoveIngester(id string) { // and 'to' ingester uses either normalised or non-normalised tokens, but not both. Tokens list must // be sorted properly. If all of this is true, everything will be fine. func (d *Desc) ClaimTokens(from, to string, normaliseTokens bool) Tokens { - result := Tokens{} + var result Tokens if normaliseTokens { // If the ingester we are claiming from is normalising, get its tokens then erase them from the ring. if fromDesc, found := d.Ingesters[from]; found { - result = append(result, fromDesc.Tokens...) + result = fromDesc.Tokens fromDesc.Tokens = nil d.Ingesters[from] = fromDesc } @@ -111,7 +111,7 @@ func (d *Desc) ClaimTokens(from, to string, normaliseTokens bool) Tokens { } else { // If source ingester is normalising, copy its tokens to d.Tokens, and set new owner if fromDesc, found := d.Ingesters[from]; found { - result = append(result, fromDesc.Tokens...) + result = fromDesc.Tokens fromDesc.Tokens = nil d.Ingesters[from] = fromDesc diff --git a/pkg/ring/tokens.go b/pkg/ring/tokens.go index 93258fe7a0c..becbce4ce32 100644 --- a/pkg/ring/tokens.go +++ b/pkg/ring/tokens.go @@ -5,31 +5,38 @@ import ( "errors" "io/ioutil" "os" - "path" + + "github.com/cortexproject/cortex/pkg/util" + "github.com/go-kit/kit/log/level" + "github.com/prometheus/tsdb/fileutil" ) const ( // TokenVersion1 is the version is a simple list of tokens. TokenVersion1 = 1 - - // Name of the file in which tokens is stored. - tokensFileName = "tokens" ) // Tokens is a simple list of tokens. type Tokens []uint32 // StoreToFile stores the tokens in the given directory. -func (t Tokens) StoreToFile(dir string) error { - if dir == "" { - return errors.New("directory is empty") +func (t Tokens) StoreToFile(tokenFilePath string) error { + if tokenFilePath == "" { + return errors.New("path is empty") } - tokenFilePath := path.Join(dir, tokensFileName) - f, err := os.OpenFile(tokenFilePath, os.O_WRONLY|os.O_CREATE, 0666) + tmp := tokenFilePath + ".tmp" + f, err := os.OpenFile(tmp, os.O_WRONLY|os.O_CREATE, 0666) if err != nil { return err } + defer func() { + // RemoveAll returns no error when tmp doesn't exist so it is safe to always run it. + if err := os.RemoveAll(tmp); err != nil { + level.Error(util.Logger).Log("msg", "error deleting temporary file", "err", err) + } + }() + b, err := t.Marshal() if err != nil { return err @@ -37,12 +44,17 @@ func (t Tokens) StoreToFile(dir string) error { if _, err = f.Write(b); err != nil { return err } - return nil + + if err := f.Close(); err != nil { + return err + } + + // Block successfully written, make visible and remove old ones. + return fileutil.Replace(tmp, tokenFilePath) } // LoadFromFile loads tokens from given directory. -func (t *Tokens) LoadFromFile(dir string) error { - tokenFilePath := path.Join(dir, tokensFileName) +func (t *Tokens) LoadFromFile(tokenFilePath string) error { b, err := ioutil.ReadFile(tokenFilePath) if err != nil { if os.IsNotExist(err) { diff --git a/pkg/ruler/tokens b/pkg/ruler/tokens deleted file mode 100644 index 9dc4fb90cb4..00000000000 --- a/pkg/ruler/tokens +++ /dev/null @@ -1 +0,0 @@ -��z� \ No newline at end of file From 39a24853dc656e1125a98d263001add18853b3e5 Mon Sep 17 00:00:00 2001 From: Ganesh Vernekar Date: Mon, 11 Nov 2019 11:59:34 +0100 Subject: [PATCH 13/18] Vendor fix Signed-off-by: Ganesh Vernekar --- go.mod | 1 + go.sum | 3 + vendor/github.com/prometheus/tsdb/LICENSE | 201 ++++++++++++++++++ .../prometheus/tsdb/fileutil/dir_unix.go | 22 ++ .../prometheus/tsdb/fileutil/dir_windows.go | 46 ++++ .../prometheus/tsdb/fileutil/fileutil.go | 159 ++++++++++++++ .../prometheus/tsdb/fileutil/flock.go | 41 ++++ .../prometheus/tsdb/fileutil/flock_plan9.go | 32 +++ .../prometheus/tsdb/fileutil/flock_solaris.go | 59 +++++ .../prometheus/tsdb/fileutil/flock_unix.go | 54 +++++ .../prometheus/tsdb/fileutil/flock_windows.go | 36 ++++ .../prometheus/tsdb/fileutil/mmap.go | 61 ++++++ .../prometheus/tsdb/fileutil/mmap_386.go | 18 ++ .../prometheus/tsdb/fileutil/mmap_amd64.go | 18 ++ .../prometheus/tsdb/fileutil/mmap_unix.go | 30 +++ .../prometheus/tsdb/fileutil/mmap_windows.go | 46 ++++ .../prometheus/tsdb/fileutil/preallocate.go | 54 +++++ .../tsdb/fileutil/preallocate_darwin.go | 41 ++++ .../tsdb/fileutil/preallocate_linux.go | 47 ++++ .../tsdb/fileutil/preallocate_other.go | 25 +++ .../prometheus/tsdb/fileutil/sync.go | 24 +++ .../prometheus/tsdb/fileutil/sync_darwin.go | 27 +++ .../prometheus/tsdb/fileutil/sync_linux.go | 29 +++ vendor/modules.txt | 2 + 24 files changed, 1076 insertions(+) create mode 100644 vendor/github.com/prometheus/tsdb/LICENSE create mode 100644 vendor/github.com/prometheus/tsdb/fileutil/dir_unix.go create mode 100644 vendor/github.com/prometheus/tsdb/fileutil/dir_windows.go create mode 100644 vendor/github.com/prometheus/tsdb/fileutil/fileutil.go create mode 100644 vendor/github.com/prometheus/tsdb/fileutil/flock.go create mode 100644 vendor/github.com/prometheus/tsdb/fileutil/flock_plan9.go create mode 100644 vendor/github.com/prometheus/tsdb/fileutil/flock_solaris.go create mode 100644 vendor/github.com/prometheus/tsdb/fileutil/flock_unix.go create mode 100644 vendor/github.com/prometheus/tsdb/fileutil/flock_windows.go create mode 100644 vendor/github.com/prometheus/tsdb/fileutil/mmap.go create mode 100644 vendor/github.com/prometheus/tsdb/fileutil/mmap_386.go create mode 100644 vendor/github.com/prometheus/tsdb/fileutil/mmap_amd64.go create mode 100644 vendor/github.com/prometheus/tsdb/fileutil/mmap_unix.go create mode 100644 vendor/github.com/prometheus/tsdb/fileutil/mmap_windows.go create mode 100644 vendor/github.com/prometheus/tsdb/fileutil/preallocate.go create mode 100644 vendor/github.com/prometheus/tsdb/fileutil/preallocate_darwin.go create mode 100644 vendor/github.com/prometheus/tsdb/fileutil/preallocate_linux.go create mode 100644 vendor/github.com/prometheus/tsdb/fileutil/preallocate_other.go create mode 100644 vendor/github.com/prometheus/tsdb/fileutil/sync.go create mode 100644 vendor/github.com/prometheus/tsdb/fileutil/sync_darwin.go create mode 100644 vendor/github.com/prometheus/tsdb/fileutil/sync_linux.go diff --git a/go.mod b/go.mod index 26175b29ca2..d005bf0a825 100644 --- a/go.mod +++ b/go.mod @@ -59,6 +59,7 @@ require ( github.com/prometheus/client_golang v1.1.0 github.com/prometheus/common v0.7.0 github.com/prometheus/prometheus v1.8.2-0.20190918104050-8744afdd1ea0 + github.com/prometheus/tsdb v0.10.0 github.com/rafaeljusto/redigomock v0.0.0-20190202135759-257e089e14a1 github.com/satori/go.uuid v1.2.0 // indirect github.com/segmentio/fasthash v0.0.0-20180216231524-a72b379d632e diff --git a/go.sum b/go.sum index 2e87440bd20..edfa3b23e0b 100644 --- a/go.sum +++ b/go.sum @@ -122,6 +122,7 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/dgrijalva/jwt-go v0.0.0-20160705203006-01aeca54ebda/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dgryski/go-sip13 v0.0.0-20190329191031-25c5027a8c7b/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= @@ -537,6 +538,8 @@ github.com/prometheus/prometheus v0.0.0-20190818123050-43acd0e2e93f/go.mod h1:rM github.com/prometheus/prometheus v1.8.2-0.20190819201610-48b2c9c8eae2/go.mod h1:rMTlmxGCvukf2KMu3fClMDKLLoJ5hl61MhcJ7xKakf0= github.com/prometheus/prometheus v1.8.2-0.20190918104050-8744afdd1ea0 h1:W4dTblzSVIBNfDimJhh70OpZQQMwLVpwK50scXdH94w= github.com/prometheus/prometheus v1.8.2-0.20190918104050-8744afdd1ea0/go.mod h1:elNqjVbwD3sCZJqKzyN7uEuwGcCpeJvv67D6BrHsDbw= +github.com/prometheus/tsdb v0.10.0 h1:If5rVCMTp6W2SiRAQFlbpJNgVlgMEd+U2GZckwK38ic= +github.com/prometheus/tsdb v0.10.0/go.mod h1:oi49uRhEe9dPUTlS3JRZOwJuVi6tmh10QSgwXEyGCt4= github.com/rafaeljusto/redigomock v0.0.0-20190202135759-257e089e14a1 h1:+kGqA4dNN5hn7WwvKdzHl0rdN5AEkbNZd0VjRltAiZg= github.com/rafaeljusto/redigomock v0.0.0-20190202135759-257e089e14a1/go.mod h1:JaY6n2sDr+z2WTsXkOmNRUfDy6FN0L6Nk7x06ndm4tY= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= diff --git a/vendor/github.com/prometheus/tsdb/LICENSE b/vendor/github.com/prometheus/tsdb/LICENSE new file mode 100644 index 00000000000..261eeb9e9f8 --- /dev/null +++ b/vendor/github.com/prometheus/tsdb/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/vendor/github.com/prometheus/tsdb/fileutil/dir_unix.go b/vendor/github.com/prometheus/tsdb/fileutil/dir_unix.go new file mode 100644 index 00000000000..58a77dfc1a9 --- /dev/null +++ b/vendor/github.com/prometheus/tsdb/fileutil/dir_unix.go @@ -0,0 +1,22 @@ +// Copyright 2016 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build !windows + +package fileutil + +import "os" + +// OpenDir opens a directory for syncing. +func OpenDir(path string) (*os.File, error) { return os.Open(path) } diff --git a/vendor/github.com/prometheus/tsdb/fileutil/dir_windows.go b/vendor/github.com/prometheus/tsdb/fileutil/dir_windows.go new file mode 100644 index 00000000000..c123395c004 --- /dev/null +++ b/vendor/github.com/prometheus/tsdb/fileutil/dir_windows.go @@ -0,0 +1,46 @@ +// Copyright 2016 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build windows + +package fileutil + +import ( + "os" + "syscall" +) + +// OpenDir opens a directory in windows with write access for syncing. +func OpenDir(path string) (*os.File, error) { + fd, err := openDir(path) + if err != nil { + return nil, err + } + return os.NewFile(uintptr(fd), path), nil +} + +func openDir(path string) (fd syscall.Handle, err error) { + if len(path) == 0 { + return syscall.InvalidHandle, syscall.ERROR_FILE_NOT_FOUND + } + pathp, err := syscall.UTF16PtrFromString(path) + if err != nil { + return syscall.InvalidHandle, err + } + access := uint32(syscall.GENERIC_READ | syscall.GENERIC_WRITE) + sharemode := uint32(syscall.FILE_SHARE_READ | syscall.FILE_SHARE_WRITE) + createmode := uint32(syscall.OPEN_EXISTING) + fl := uint32(syscall.FILE_FLAG_BACKUP_SEMANTICS) + return syscall.CreateFile(pathp, access, sharemode, nil, createmode, fl, 0) +} diff --git a/vendor/github.com/prometheus/tsdb/fileutil/fileutil.go b/vendor/github.com/prometheus/tsdb/fileutil/fileutil.go new file mode 100644 index 00000000000..4088f522aeb --- /dev/null +++ b/vendor/github.com/prometheus/tsdb/fileutil/fileutil.go @@ -0,0 +1,159 @@ +// Copyright 2018 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package fileutil provides utility methods used when dealing with the filesystem in tsdb. +// It is largely copied from github.com/coreos/etcd/pkg/fileutil to avoid the +// dependency chain it brings with it. +// Please check github.com/coreos/etcd for licensing information. +package fileutil + +import ( + "io/ioutil" + "os" + "path/filepath" + "sort" + "strings" +) + +// CopyDirs copies all directories, subdirectories and files recursively including the empty folders. +// Source and destination must be full paths. +func CopyDirs(src, dest string) error { + if err := os.MkdirAll(dest, 0777); err != nil { + return err + } + files, err := readDirs(src) + if err != nil { + return err + } + + for _, f := range files { + dp := filepath.Join(dest, f) + sp := filepath.Join(src, f) + + stat, err := os.Stat(sp) + if err != nil { + return err + } + + // Empty directories are also created. + if stat.IsDir() { + if err := os.MkdirAll(dp, 0777); err != nil { + return err + } + continue + } + + if err := copyFile(sp, dp); err != nil { + return err + } + } + return nil +} + +func copyFile(src, dest string) error { + data, err := ioutil.ReadFile(src) + if err != nil { + return err + } + + err = ioutil.WriteFile(dest, data, 0644) + if err != nil { + return err + } + return nil +} + +// readDirs reads the source directory recursively and +// returns relative paths to all files and empty directories. +func readDirs(src string) ([]string, error) { + var files []string + + err := filepath.Walk(src, func(path string, f os.FileInfo, err error) error { + relativePath := strings.TrimPrefix(path, src) + if len(relativePath) > 0 { + files = append(files, relativePath) + } + return nil + }) + if err != nil { + return nil, err + } + return files, nil +} + +// ReadDir returns the filenames in the given directory in sorted order. +func ReadDir(dirpath string) ([]string, error) { + dir, err := os.Open(dirpath) + if err != nil { + return nil, err + } + defer dir.Close() + names, err := dir.Readdirnames(-1) + if err != nil { + return nil, err + } + sort.Strings(names) + return names, nil +} + +// Rename safely renames a file. +func Rename(from, to string) error { + if err := os.Rename(from, to); err != nil { + return err + } + + // Directory was renamed; sync parent dir to persist rename. + pdir, err := OpenDir(filepath.Dir(to)) + if err != nil { + return err + } + + if err = pdir.Sync(); err != nil { + pdir.Close() + return err + } + return pdir.Close() +} + +// Replace moves a file or directory to a new location and deletes any previous data. +// It is not atomic. +func Replace(from, to string) error { + // Remove destination only if it is a dir otherwise leave it to os.Rename + // as it replaces the destination file and is atomic. + { + f, err := os.Stat(to) + if !os.IsNotExist(err) { + if err == nil && f.IsDir() { + if err := os.RemoveAll(to); err != nil { + return err + } + } + } + } + + if err := os.Rename(from, to); err != nil { + return err + } + + // Directory was renamed; sync parent dir to persist rename. + pdir, err := OpenDir(filepath.Dir(to)) + if err != nil { + return err + } + + if err = pdir.Sync(); err != nil { + pdir.Close() + return err + } + return pdir.Close() +} diff --git a/vendor/github.com/prometheus/tsdb/fileutil/flock.go b/vendor/github.com/prometheus/tsdb/fileutil/flock.go new file mode 100644 index 00000000000..d5eaa7ca2ad --- /dev/null +++ b/vendor/github.com/prometheus/tsdb/fileutil/flock.go @@ -0,0 +1,41 @@ +// Copyright 2016 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fileutil + +import ( + "os" + "path/filepath" +) + +// Releaser provides the Release method to release a file lock. +type Releaser interface { + Release() error +} + +// Flock locks the file with the provided name. If the file does not exist, it is +// created. The returned Releaser is used to release the lock. existed is true +// if the file to lock already existed. A non-nil error is returned if the +// locking has failed. Neither this function nor the returned Releaser is +// goroutine-safe. +func Flock(fileName string) (r Releaser, existed bool, err error) { + if err = os.MkdirAll(filepath.Dir(fileName), 0755); err != nil { + return nil, false, err + } + + _, err = os.Stat(fileName) + existed = err == nil + + r, err = newLock(fileName) + return r, existed, err +} diff --git a/vendor/github.com/prometheus/tsdb/fileutil/flock_plan9.go b/vendor/github.com/prometheus/tsdb/fileutil/flock_plan9.go new file mode 100644 index 00000000000..8a3d44c5e14 --- /dev/null +++ b/vendor/github.com/prometheus/tsdb/fileutil/flock_plan9.go @@ -0,0 +1,32 @@ +// Copyright 2016 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fileutil + +import "os" + +type plan9Lock struct { + f *os.File +} + +func (l *plan9Lock) Release() error { + return l.f.Close() +} + +func newLock(fileName string) (Releaser, error) { + f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, os.ModeExclusive|0644) + if err != nil { + return nil, err + } + return &plan9Lock{f}, nil +} diff --git a/vendor/github.com/prometheus/tsdb/fileutil/flock_solaris.go b/vendor/github.com/prometheus/tsdb/fileutil/flock_solaris.go new file mode 100644 index 00000000000..7f527ae6c49 --- /dev/null +++ b/vendor/github.com/prometheus/tsdb/fileutil/flock_solaris.go @@ -0,0 +1,59 @@ +// Copyright 2016 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build solaris + +package fileutil + +import ( + "os" + "syscall" +) + +type unixLock struct { + f *os.File +} + +func (l *unixLock) Release() error { + if err := l.set(false); err != nil { + return err + } + return l.f.Close() +} + +func (l *unixLock) set(lock bool) error { + flock := syscall.Flock_t{ + Type: syscall.F_UNLCK, + Start: 0, + Len: 0, + Whence: 1, + } + if lock { + flock.Type = syscall.F_WRLCK + } + return syscall.FcntlFlock(l.f.Fd(), syscall.F_SETLK, &flock) +} + +func newLock(fileName string) (Releaser, error) { + f, err := os.OpenFile(fileName, os.O_RDWR|os.O_CREATE, 0644) + if err != nil { + return nil, err + } + l := &unixLock{f} + err = l.set(true) + if err != nil { + f.Close() + return nil, err + } + return l, nil +} diff --git a/vendor/github.com/prometheus/tsdb/fileutil/flock_unix.go b/vendor/github.com/prometheus/tsdb/fileutil/flock_unix.go new file mode 100644 index 00000000000..f493fbd831d --- /dev/null +++ b/vendor/github.com/prometheus/tsdb/fileutil/flock_unix.go @@ -0,0 +1,54 @@ +// Copyright 2016 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build darwin dragonfly freebsd linux netbsd openbsd + +package fileutil + +import ( + "os" + "syscall" +) + +type unixLock struct { + f *os.File +} + +func (l *unixLock) Release() error { + if err := l.set(false); err != nil { + return err + } + return l.f.Close() +} + +func (l *unixLock) set(lock bool) error { + how := syscall.LOCK_UN + if lock { + how = syscall.LOCK_EX + } + return syscall.Flock(int(l.f.Fd()), how|syscall.LOCK_NB) +} + +func newLock(fileName string) (Releaser, error) { + f, err := os.OpenFile(fileName, os.O_RDWR|os.O_CREATE, 0644) + if err != nil { + return nil, err + } + l := &unixLock{f} + err = l.set(true) + if err != nil { + f.Close() + return nil, err + } + return l, nil +} diff --git a/vendor/github.com/prometheus/tsdb/fileutil/flock_windows.go b/vendor/github.com/prometheus/tsdb/fileutil/flock_windows.go new file mode 100644 index 00000000000..1c17ff4ea30 --- /dev/null +++ b/vendor/github.com/prometheus/tsdb/fileutil/flock_windows.go @@ -0,0 +1,36 @@ +// Copyright 2016 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fileutil + +import "syscall" + +type windowsLock struct { + fd syscall.Handle +} + +func (fl *windowsLock) Release() error { + return syscall.Close(fl.fd) +} + +func newLock(fileName string) (Releaser, error) { + pathp, err := syscall.UTF16PtrFromString(fileName) + if err != nil { + return nil, err + } + fd, err := syscall.CreateFile(pathp, syscall.GENERIC_READ|syscall.GENERIC_WRITE, 0, nil, syscall.CREATE_ALWAYS, syscall.FILE_ATTRIBUTE_NORMAL, 0) + if err != nil { + return nil, err + } + return &windowsLock{fd}, nil +} diff --git a/vendor/github.com/prometheus/tsdb/fileutil/mmap.go b/vendor/github.com/prometheus/tsdb/fileutil/mmap.go new file mode 100644 index 00000000000..26fc80c5850 --- /dev/null +++ b/vendor/github.com/prometheus/tsdb/fileutil/mmap.go @@ -0,0 +1,61 @@ +// Copyright 2018 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fileutil + +import ( + "os" + + "github.com/pkg/errors" +) + +type MmapFile struct { + f *os.File + b []byte +} + +func OpenMmapFile(path string) (*MmapFile, error) { + f, err := os.Open(path) + if err != nil { + return nil, errors.Wrap(err, "try lock file") + } + info, err := f.Stat() + if err != nil { + return nil, errors.Wrap(err, "stat") + } + + b, err := mmap(f, int(info.Size())) + if err != nil { + return nil, errors.Wrap(err, "mmap") + } + + return &MmapFile{f: f, b: b}, nil +} + +func (f *MmapFile) Close() error { + err0 := munmap(f.b) + err1 := f.f.Close() + + if err0 != nil { + return err0 + } + return err1 +} + +func (f *MmapFile) File() *os.File { + return f.f +} + +func (f *MmapFile) Bytes() []byte { + return f.b +} diff --git a/vendor/github.com/prometheus/tsdb/fileutil/mmap_386.go b/vendor/github.com/prometheus/tsdb/fileutil/mmap_386.go new file mode 100644 index 00000000000..66b9d368034 --- /dev/null +++ b/vendor/github.com/prometheus/tsdb/fileutil/mmap_386.go @@ -0,0 +1,18 @@ +// Copyright 2018 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build windows + +package fileutil + +const maxMapSize = 0x7FFFFFFF // 2GB diff --git a/vendor/github.com/prometheus/tsdb/fileutil/mmap_amd64.go b/vendor/github.com/prometheus/tsdb/fileutil/mmap_amd64.go new file mode 100644 index 00000000000..4b523bc67c2 --- /dev/null +++ b/vendor/github.com/prometheus/tsdb/fileutil/mmap_amd64.go @@ -0,0 +1,18 @@ +// Copyright 2018 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build windows + +package fileutil + +const maxMapSize = 0xFFFFFFFFFFFF // 256TB diff --git a/vendor/github.com/prometheus/tsdb/fileutil/mmap_unix.go b/vendor/github.com/prometheus/tsdb/fileutil/mmap_unix.go new file mode 100644 index 00000000000..043f4d408cc --- /dev/null +++ b/vendor/github.com/prometheus/tsdb/fileutil/mmap_unix.go @@ -0,0 +1,30 @@ +// Copyright 2017 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build !windows,!plan9 + +package fileutil + +import ( + "os" + + "golang.org/x/sys/unix" +) + +func mmap(f *os.File, length int) ([]byte, error) { + return unix.Mmap(int(f.Fd()), 0, length, unix.PROT_READ, unix.MAP_SHARED) +} + +func munmap(b []byte) (err error) { + return unix.Munmap(b) +} diff --git a/vendor/github.com/prometheus/tsdb/fileutil/mmap_windows.go b/vendor/github.com/prometheus/tsdb/fileutil/mmap_windows.go new file mode 100644 index 00000000000..b9422641239 --- /dev/null +++ b/vendor/github.com/prometheus/tsdb/fileutil/mmap_windows.go @@ -0,0 +1,46 @@ +// Copyright 2017 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fileutil + +import ( + "os" + "syscall" + "unsafe" +) + +func mmap(f *os.File, size int) ([]byte, error) { + low, high := uint32(size), uint32(size>>32) + h, errno := syscall.CreateFileMapping(syscall.Handle(f.Fd()), nil, syscall.PAGE_READONLY, high, low, nil) + if h == 0 { + return nil, os.NewSyscallError("CreateFileMapping", errno) + } + + addr, errno := syscall.MapViewOfFile(h, syscall.FILE_MAP_READ, 0, 0, uintptr(size)) + if addr == 0 { + return nil, os.NewSyscallError("MapViewOfFile", errno) + } + + if err := syscall.CloseHandle(syscall.Handle(h)); err != nil { + return nil, os.NewSyscallError("CloseHandle", err) + } + + return (*[maxMapSize]byte)(unsafe.Pointer(addr))[:size], nil +} + +func munmap(b []byte) error { + if err := syscall.UnmapViewOfFile((uintptr)(unsafe.Pointer(&b[0]))); err != nil { + return os.NewSyscallError("UnmapViewOfFile", err) + } + return nil +} diff --git a/vendor/github.com/prometheus/tsdb/fileutil/preallocate.go b/vendor/github.com/prometheus/tsdb/fileutil/preallocate.go new file mode 100644 index 00000000000..c747b7cf81f --- /dev/null +++ b/vendor/github.com/prometheus/tsdb/fileutil/preallocate.go @@ -0,0 +1,54 @@ +// Copyright 2015 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fileutil + +import ( + "io" + "os" +) + +// Preallocate tries to allocate the space for given +// file. This operation is only supported on linux by a +// few filesystems (btrfs, ext4, etc.). +// If the operation is unsupported, no error will be returned. +// Otherwise, the error encountered will be returned. +func Preallocate(f *os.File, sizeInBytes int64, extendFile bool) error { + if sizeInBytes == 0 { + // fallocate will return EINVAL if length is 0; skip + return nil + } + if extendFile { + return preallocExtend(f, sizeInBytes) + } + return preallocFixed(f, sizeInBytes) +} + +func preallocExtendTrunc(f *os.File, sizeInBytes int64) error { + curOff, err := f.Seek(0, io.SeekCurrent) + if err != nil { + return err + } + size, err := f.Seek(sizeInBytes, io.SeekEnd) + if err != nil { + return err + } + if _, err = f.Seek(curOff, io.SeekStart); err != nil { + return err + } + if sizeInBytes > size { + return nil + } + return f.Truncate(sizeInBytes) +} diff --git a/vendor/github.com/prometheus/tsdb/fileutil/preallocate_darwin.go b/vendor/github.com/prometheus/tsdb/fileutil/preallocate_darwin.go new file mode 100644 index 00000000000..c9fa1a6c226 --- /dev/null +++ b/vendor/github.com/prometheus/tsdb/fileutil/preallocate_darwin.go @@ -0,0 +1,41 @@ +// Copyright 2015 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fileutil + +import ( + "os" + "syscall" + "unsafe" +) + +func preallocExtend(f *os.File, sizeInBytes int64) error { + if err := preallocFixed(f, sizeInBytes); err != nil { + return err + } + return preallocExtendTrunc(f, sizeInBytes) +} + +func preallocFixed(f *os.File, sizeInBytes int64) error { + fstore := &syscall.Fstore_t{ + Flags: syscall.F_ALLOCATEALL, + Posmode: syscall.F_PEOFPOSMODE, + Length: sizeInBytes} + p := unsafe.Pointer(fstore) + _, _, errno := syscall.Syscall(syscall.SYS_FCNTL, f.Fd(), uintptr(syscall.F_PREALLOCATE), uintptr(p)) + if errno == 0 || errno == syscall.ENOTSUP { + return nil + } + return errno +} diff --git a/vendor/github.com/prometheus/tsdb/fileutil/preallocate_linux.go b/vendor/github.com/prometheus/tsdb/fileutil/preallocate_linux.go new file mode 100644 index 00000000000..ada0462213e --- /dev/null +++ b/vendor/github.com/prometheus/tsdb/fileutil/preallocate_linux.go @@ -0,0 +1,47 @@ +// Copyright 2015 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fileutil + +import ( + "os" + "syscall" +) + +func preallocExtend(f *os.File, sizeInBytes int64) error { + // use mode = 0 to change size + err := syscall.Fallocate(int(f.Fd()), 0, 0, sizeInBytes) + if err != nil { + errno, ok := err.(syscall.Errno) + // not supported; fallback + // fallocate EINTRs frequently in some environments; fallback + if ok && (errno == syscall.ENOTSUP || errno == syscall.EINTR) { + return preallocExtendTrunc(f, sizeInBytes) + } + } + return err +} + +func preallocFixed(f *os.File, sizeInBytes int64) error { + // use mode = 1 to keep size; see FALLOC_FL_KEEP_SIZE + err := syscall.Fallocate(int(f.Fd()), 1, 0, sizeInBytes) + if err != nil { + errno, ok := err.(syscall.Errno) + // treat not supported as nil error + if ok && errno == syscall.ENOTSUP { + return nil + } + } + return err +} diff --git a/vendor/github.com/prometheus/tsdb/fileutil/preallocate_other.go b/vendor/github.com/prometheus/tsdb/fileutil/preallocate_other.go new file mode 100644 index 00000000000..162fbc5f782 --- /dev/null +++ b/vendor/github.com/prometheus/tsdb/fileutil/preallocate_other.go @@ -0,0 +1,25 @@ +// Copyright 2015 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build !linux,!darwin + +package fileutil + +import "os" + +func preallocExtend(f *os.File, sizeInBytes int64) error { + return preallocExtendTrunc(f, sizeInBytes) +} + +func preallocFixed(f *os.File, sizeInBytes int64) error { return nil } diff --git a/vendor/github.com/prometheus/tsdb/fileutil/sync.go b/vendor/github.com/prometheus/tsdb/fileutil/sync.go new file mode 100644 index 00000000000..2e64a40880a --- /dev/null +++ b/vendor/github.com/prometheus/tsdb/fileutil/sync.go @@ -0,0 +1,24 @@ +// Copyright 2016 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build !linux,!darwin + +package fileutil + +import "os" + +// Fdatasync is a wrapper around file.Sync(). Special handling is needed on linux platform. +func Fdatasync(f *os.File) error { + return f.Sync() +} diff --git a/vendor/github.com/prometheus/tsdb/fileutil/sync_darwin.go b/vendor/github.com/prometheus/tsdb/fileutil/sync_darwin.go new file mode 100644 index 00000000000..2af1b0f4119 --- /dev/null +++ b/vendor/github.com/prometheus/tsdb/fileutil/sync_darwin.go @@ -0,0 +1,27 @@ +// Copyright 2016 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build darwin + +package fileutil + +import ( + "os" +) + +// Fdatasync on darwin platform invokes fcntl(F_FULLFSYNC) for actual persistence +// on physical drive media. +func Fdatasync(f *os.File) error { + return f.Sync() +} diff --git a/vendor/github.com/prometheus/tsdb/fileutil/sync_linux.go b/vendor/github.com/prometheus/tsdb/fileutil/sync_linux.go new file mode 100644 index 00000000000..8b4fc8268e5 --- /dev/null +++ b/vendor/github.com/prometheus/tsdb/fileutil/sync_linux.go @@ -0,0 +1,29 @@ +// Copyright 2016 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build linux + +package fileutil + +import ( + "os" + "syscall" +) + +// Fdatasync is similar to fsync(), but does not flush modified metadata +// unless that metadata is needed in order to allow a subsequent data retrieval +// to be correctly handled. +func Fdatasync(f *os.File) error { + return syscall.Fdatasync(int(f.Fd())) +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 7cbc3fb3f40..5cbb62ac166 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -456,6 +456,8 @@ github.com/prometheus/prometheus/util/teststorage github.com/prometheus/prometheus/util/testutil github.com/prometheus/prometheus/util/treecache github.com/prometheus/prometheus/web/api/v1 +# github.com/prometheus/tsdb v0.10.0 +github.com/prometheus/tsdb/fileutil # github.com/rafaeljusto/redigomock v0.0.0-20190202135759-257e089e14a1 github.com/rafaeljusto/redigomock # github.com/rs/cors v1.6.0 From 2e9e6f3eaa0ec2e9dbf29e499a4d7946737277d1 Mon Sep 17 00:00:00 2001 From: Ganesh Vernekar Date: Mon, 11 Nov 2019 12:10:21 +0100 Subject: [PATCH 14/18] Fix review comments Signed-off-by: Ganesh Vernekar --- pkg/ingester/lifecycle_test.go | 3 +++ pkg/ring/lifecycler_test.go | 3 +++ 2 files changed, 6 insertions(+) diff --git a/pkg/ingester/lifecycle_test.go b/pkg/ingester/lifecycle_test.go index b64164e37b7..06e810b459b 100644 --- a/pkg/ingester/lifecycle_test.go +++ b/pkg/ingester/lifecycle_test.go @@ -197,6 +197,9 @@ func TestIngesterBadTransfer(t *testing.T) { tokenDir, err := ioutil.TempDir(os.TempDir(), "ingester_bad_transfer") require.NoError(t, err) + defer func() { + require.NoError(t, os.RemoveAll(tokenDir)) + }() // Start ingester in PENDING. cfg := defaultIngesterTestConfig() diff --git a/pkg/ring/lifecycler_test.go b/pkg/ring/lifecycler_test.go index 3b934383270..32cf603d2e5 100644 --- a/pkg/ring/lifecycler_test.go +++ b/pkg/ring/lifecycler_test.go @@ -31,6 +31,9 @@ func (f *flushTransferer) TransferOut(ctx context.Context) error { func testLifecyclerConfig(t *testing.T, ringConfig Config, id string) LifecyclerConfig { tokenDir, err := ioutil.TempDir(os.TempDir(), "ingester_bad_transfer") require.NoError(t, err) + defer func() { + require.NoError(t, os.RemoveAll(tokenDir)) + }() var lifecyclerConfig LifecyclerConfig flagext.DefaultValues(&lifecyclerConfig) From 7d73e0706afeb9e6d1bbdd12b923c5b777abe239 Mon Sep 17 00:00:00 2001 From: Ganesh Vernekar Date: Mon, 11 Nov 2019 12:24:07 +0100 Subject: [PATCH 15/18] Fix vendoring Signed-off-by: Ganesh Vernekar --- go.mod | 1 - go.sum | 3 - pkg/ring/lifecycler.go | 2 +- pkg/ring/tokens.go | 2 +- vendor/github.com/prometheus/tsdb/LICENSE | 201 ------------------ .../prometheus/tsdb/fileutil/dir_unix.go | 22 -- .../prometheus/tsdb/fileutil/dir_windows.go | 46 ---- .../prometheus/tsdb/fileutil/fileutil.go | 159 -------------- .../prometheus/tsdb/fileutil/flock.go | 41 ---- .../prometheus/tsdb/fileutil/flock_plan9.go | 32 --- .../prometheus/tsdb/fileutil/flock_solaris.go | 59 ----- .../prometheus/tsdb/fileutil/flock_unix.go | 54 ----- .../prometheus/tsdb/fileutil/flock_windows.go | 36 ---- .../prometheus/tsdb/fileutil/mmap.go | 61 ------ .../prometheus/tsdb/fileutil/mmap_386.go | 18 -- .../prometheus/tsdb/fileutil/mmap_amd64.go | 18 -- .../prometheus/tsdb/fileutil/mmap_unix.go | 30 --- .../prometheus/tsdb/fileutil/mmap_windows.go | 46 ---- .../prometheus/tsdb/fileutil/preallocate.go | 54 ----- .../tsdb/fileutil/preallocate_darwin.go | 41 ---- .../tsdb/fileutil/preallocate_linux.go | 47 ---- .../tsdb/fileutil/preallocate_other.go | 25 --- .../prometheus/tsdb/fileutil/sync.go | 24 --- .../prometheus/tsdb/fileutil/sync_darwin.go | 27 --- .../prometheus/tsdb/fileutil/sync_linux.go | 29 --- vendor/modules.txt | 2 - 26 files changed, 2 insertions(+), 1078 deletions(-) delete mode 100644 vendor/github.com/prometheus/tsdb/LICENSE delete mode 100644 vendor/github.com/prometheus/tsdb/fileutil/dir_unix.go delete mode 100644 vendor/github.com/prometheus/tsdb/fileutil/dir_windows.go delete mode 100644 vendor/github.com/prometheus/tsdb/fileutil/fileutil.go delete mode 100644 vendor/github.com/prometheus/tsdb/fileutil/flock.go delete mode 100644 vendor/github.com/prometheus/tsdb/fileutil/flock_plan9.go delete mode 100644 vendor/github.com/prometheus/tsdb/fileutil/flock_solaris.go delete mode 100644 vendor/github.com/prometheus/tsdb/fileutil/flock_unix.go delete mode 100644 vendor/github.com/prometheus/tsdb/fileutil/flock_windows.go delete mode 100644 vendor/github.com/prometheus/tsdb/fileutil/mmap.go delete mode 100644 vendor/github.com/prometheus/tsdb/fileutil/mmap_386.go delete mode 100644 vendor/github.com/prometheus/tsdb/fileutil/mmap_amd64.go delete mode 100644 vendor/github.com/prometheus/tsdb/fileutil/mmap_unix.go delete mode 100644 vendor/github.com/prometheus/tsdb/fileutil/mmap_windows.go delete mode 100644 vendor/github.com/prometheus/tsdb/fileutil/preallocate.go delete mode 100644 vendor/github.com/prometheus/tsdb/fileutil/preallocate_darwin.go delete mode 100644 vendor/github.com/prometheus/tsdb/fileutil/preallocate_linux.go delete mode 100644 vendor/github.com/prometheus/tsdb/fileutil/preallocate_other.go delete mode 100644 vendor/github.com/prometheus/tsdb/fileutil/sync.go delete mode 100644 vendor/github.com/prometheus/tsdb/fileutil/sync_darwin.go delete mode 100644 vendor/github.com/prometheus/tsdb/fileutil/sync_linux.go diff --git a/go.mod b/go.mod index d005bf0a825..26175b29ca2 100644 --- a/go.mod +++ b/go.mod @@ -59,7 +59,6 @@ require ( github.com/prometheus/client_golang v1.1.0 github.com/prometheus/common v0.7.0 github.com/prometheus/prometheus v1.8.2-0.20190918104050-8744afdd1ea0 - github.com/prometheus/tsdb v0.10.0 github.com/rafaeljusto/redigomock v0.0.0-20190202135759-257e089e14a1 github.com/satori/go.uuid v1.2.0 // indirect github.com/segmentio/fasthash v0.0.0-20180216231524-a72b379d632e diff --git a/go.sum b/go.sum index edfa3b23e0b..2e87440bd20 100644 --- a/go.sum +++ b/go.sum @@ -122,7 +122,6 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/dgrijalva/jwt-go v0.0.0-20160705203006-01aeca54ebda/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dgryski/go-sip13 v0.0.0-20190329191031-25c5027a8c7b/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= @@ -538,8 +537,6 @@ github.com/prometheus/prometheus v0.0.0-20190818123050-43acd0e2e93f/go.mod h1:rM github.com/prometheus/prometheus v1.8.2-0.20190819201610-48b2c9c8eae2/go.mod h1:rMTlmxGCvukf2KMu3fClMDKLLoJ5hl61MhcJ7xKakf0= github.com/prometheus/prometheus v1.8.2-0.20190918104050-8744afdd1ea0 h1:W4dTblzSVIBNfDimJhh70OpZQQMwLVpwK50scXdH94w= github.com/prometheus/prometheus v1.8.2-0.20190918104050-8744afdd1ea0/go.mod h1:elNqjVbwD3sCZJqKzyN7uEuwGcCpeJvv67D6BrHsDbw= -github.com/prometheus/tsdb v0.10.0 h1:If5rVCMTp6W2SiRAQFlbpJNgVlgMEd+U2GZckwK38ic= -github.com/prometheus/tsdb v0.10.0/go.mod h1:oi49uRhEe9dPUTlS3JRZOwJuVi6tmh10QSgwXEyGCt4= github.com/rafaeljusto/redigomock v0.0.0-20190202135759-257e089e14a1 h1:+kGqA4dNN5hn7WwvKdzHl0rdN5AEkbNZd0VjRltAiZg= github.com/rafaeljusto/redigomock v0.0.0-20190202135759-257e089e14a1/go.mod h1:JaY6n2sDr+z2WTsXkOmNRUfDy6FN0L6Nk7x06ndm4tY= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= diff --git a/pkg/ring/lifecycler.go b/pkg/ring/lifecycler.go index f562475727d..c12886a2e0a 100644 --- a/pkg/ring/lifecycler.go +++ b/pkg/ring/lifecycler.go @@ -470,7 +470,7 @@ func (i *Lifecycler) initRing(ctx context.Context) error { err := tokens.LoadFromFile(i.cfg.TokensFilePath) if err != nil { level.Error(util.Logger).Log("msg", "error in getting tokens from file", "err", err) - } else if tokens != nil && len(tokens) > 0 { + } else if len(tokens) > 0 { level.Info(util.Logger).Log("msg", "adding tokens from file", "num_tokens", len(tokens)) if len(tokens) == i.cfg.NumTokens { i.setState(ACTIVE) diff --git a/pkg/ring/tokens.go b/pkg/ring/tokens.go index becbce4ce32..af8a7819638 100644 --- a/pkg/ring/tokens.go +++ b/pkg/ring/tokens.go @@ -8,7 +8,7 @@ import ( "github.com/cortexproject/cortex/pkg/util" "github.com/go-kit/kit/log/level" - "github.com/prometheus/tsdb/fileutil" + "github.com/prometheus/prometheus/tsdb/fileutil" ) const ( diff --git a/vendor/github.com/prometheus/tsdb/LICENSE b/vendor/github.com/prometheus/tsdb/LICENSE deleted file mode 100644 index 261eeb9e9f8..00000000000 --- a/vendor/github.com/prometheus/tsdb/LICENSE +++ /dev/null @@ -1,201 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/vendor/github.com/prometheus/tsdb/fileutil/dir_unix.go b/vendor/github.com/prometheus/tsdb/fileutil/dir_unix.go deleted file mode 100644 index 58a77dfc1a9..00000000000 --- a/vendor/github.com/prometheus/tsdb/fileutil/dir_unix.go +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright 2016 The etcd Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// +build !windows - -package fileutil - -import "os" - -// OpenDir opens a directory for syncing. -func OpenDir(path string) (*os.File, error) { return os.Open(path) } diff --git a/vendor/github.com/prometheus/tsdb/fileutil/dir_windows.go b/vendor/github.com/prometheus/tsdb/fileutil/dir_windows.go deleted file mode 100644 index c123395c004..00000000000 --- a/vendor/github.com/prometheus/tsdb/fileutil/dir_windows.go +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright 2016 The etcd Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// +build windows - -package fileutil - -import ( - "os" - "syscall" -) - -// OpenDir opens a directory in windows with write access for syncing. -func OpenDir(path string) (*os.File, error) { - fd, err := openDir(path) - if err != nil { - return nil, err - } - return os.NewFile(uintptr(fd), path), nil -} - -func openDir(path string) (fd syscall.Handle, err error) { - if len(path) == 0 { - return syscall.InvalidHandle, syscall.ERROR_FILE_NOT_FOUND - } - pathp, err := syscall.UTF16PtrFromString(path) - if err != nil { - return syscall.InvalidHandle, err - } - access := uint32(syscall.GENERIC_READ | syscall.GENERIC_WRITE) - sharemode := uint32(syscall.FILE_SHARE_READ | syscall.FILE_SHARE_WRITE) - createmode := uint32(syscall.OPEN_EXISTING) - fl := uint32(syscall.FILE_FLAG_BACKUP_SEMANTICS) - return syscall.CreateFile(pathp, access, sharemode, nil, createmode, fl, 0) -} diff --git a/vendor/github.com/prometheus/tsdb/fileutil/fileutil.go b/vendor/github.com/prometheus/tsdb/fileutil/fileutil.go deleted file mode 100644 index 4088f522aeb..00000000000 --- a/vendor/github.com/prometheus/tsdb/fileutil/fileutil.go +++ /dev/null @@ -1,159 +0,0 @@ -// Copyright 2018 The Prometheus Authors -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package fileutil provides utility methods used when dealing with the filesystem in tsdb. -// It is largely copied from github.com/coreos/etcd/pkg/fileutil to avoid the -// dependency chain it brings with it. -// Please check github.com/coreos/etcd for licensing information. -package fileutil - -import ( - "io/ioutil" - "os" - "path/filepath" - "sort" - "strings" -) - -// CopyDirs copies all directories, subdirectories and files recursively including the empty folders. -// Source and destination must be full paths. -func CopyDirs(src, dest string) error { - if err := os.MkdirAll(dest, 0777); err != nil { - return err - } - files, err := readDirs(src) - if err != nil { - return err - } - - for _, f := range files { - dp := filepath.Join(dest, f) - sp := filepath.Join(src, f) - - stat, err := os.Stat(sp) - if err != nil { - return err - } - - // Empty directories are also created. - if stat.IsDir() { - if err := os.MkdirAll(dp, 0777); err != nil { - return err - } - continue - } - - if err := copyFile(sp, dp); err != nil { - return err - } - } - return nil -} - -func copyFile(src, dest string) error { - data, err := ioutil.ReadFile(src) - if err != nil { - return err - } - - err = ioutil.WriteFile(dest, data, 0644) - if err != nil { - return err - } - return nil -} - -// readDirs reads the source directory recursively and -// returns relative paths to all files and empty directories. -func readDirs(src string) ([]string, error) { - var files []string - - err := filepath.Walk(src, func(path string, f os.FileInfo, err error) error { - relativePath := strings.TrimPrefix(path, src) - if len(relativePath) > 0 { - files = append(files, relativePath) - } - return nil - }) - if err != nil { - return nil, err - } - return files, nil -} - -// ReadDir returns the filenames in the given directory in sorted order. -func ReadDir(dirpath string) ([]string, error) { - dir, err := os.Open(dirpath) - if err != nil { - return nil, err - } - defer dir.Close() - names, err := dir.Readdirnames(-1) - if err != nil { - return nil, err - } - sort.Strings(names) - return names, nil -} - -// Rename safely renames a file. -func Rename(from, to string) error { - if err := os.Rename(from, to); err != nil { - return err - } - - // Directory was renamed; sync parent dir to persist rename. - pdir, err := OpenDir(filepath.Dir(to)) - if err != nil { - return err - } - - if err = pdir.Sync(); err != nil { - pdir.Close() - return err - } - return pdir.Close() -} - -// Replace moves a file or directory to a new location and deletes any previous data. -// It is not atomic. -func Replace(from, to string) error { - // Remove destination only if it is a dir otherwise leave it to os.Rename - // as it replaces the destination file and is atomic. - { - f, err := os.Stat(to) - if !os.IsNotExist(err) { - if err == nil && f.IsDir() { - if err := os.RemoveAll(to); err != nil { - return err - } - } - } - } - - if err := os.Rename(from, to); err != nil { - return err - } - - // Directory was renamed; sync parent dir to persist rename. - pdir, err := OpenDir(filepath.Dir(to)) - if err != nil { - return err - } - - if err = pdir.Sync(); err != nil { - pdir.Close() - return err - } - return pdir.Close() -} diff --git a/vendor/github.com/prometheus/tsdb/fileutil/flock.go b/vendor/github.com/prometheus/tsdb/fileutil/flock.go deleted file mode 100644 index d5eaa7ca2ad..00000000000 --- a/vendor/github.com/prometheus/tsdb/fileutil/flock.go +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright 2016 The Prometheus Authors -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package fileutil - -import ( - "os" - "path/filepath" -) - -// Releaser provides the Release method to release a file lock. -type Releaser interface { - Release() error -} - -// Flock locks the file with the provided name. If the file does not exist, it is -// created. The returned Releaser is used to release the lock. existed is true -// if the file to lock already existed. A non-nil error is returned if the -// locking has failed. Neither this function nor the returned Releaser is -// goroutine-safe. -func Flock(fileName string) (r Releaser, existed bool, err error) { - if err = os.MkdirAll(filepath.Dir(fileName), 0755); err != nil { - return nil, false, err - } - - _, err = os.Stat(fileName) - existed = err == nil - - r, err = newLock(fileName) - return r, existed, err -} diff --git a/vendor/github.com/prometheus/tsdb/fileutil/flock_plan9.go b/vendor/github.com/prometheus/tsdb/fileutil/flock_plan9.go deleted file mode 100644 index 8a3d44c5e14..00000000000 --- a/vendor/github.com/prometheus/tsdb/fileutil/flock_plan9.go +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright 2016 The Prometheus Authors -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package fileutil - -import "os" - -type plan9Lock struct { - f *os.File -} - -func (l *plan9Lock) Release() error { - return l.f.Close() -} - -func newLock(fileName string) (Releaser, error) { - f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, os.ModeExclusive|0644) - if err != nil { - return nil, err - } - return &plan9Lock{f}, nil -} diff --git a/vendor/github.com/prometheus/tsdb/fileutil/flock_solaris.go b/vendor/github.com/prometheus/tsdb/fileutil/flock_solaris.go deleted file mode 100644 index 7f527ae6c49..00000000000 --- a/vendor/github.com/prometheus/tsdb/fileutil/flock_solaris.go +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright 2016 The Prometheus Authors -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// +build solaris - -package fileutil - -import ( - "os" - "syscall" -) - -type unixLock struct { - f *os.File -} - -func (l *unixLock) Release() error { - if err := l.set(false); err != nil { - return err - } - return l.f.Close() -} - -func (l *unixLock) set(lock bool) error { - flock := syscall.Flock_t{ - Type: syscall.F_UNLCK, - Start: 0, - Len: 0, - Whence: 1, - } - if lock { - flock.Type = syscall.F_WRLCK - } - return syscall.FcntlFlock(l.f.Fd(), syscall.F_SETLK, &flock) -} - -func newLock(fileName string) (Releaser, error) { - f, err := os.OpenFile(fileName, os.O_RDWR|os.O_CREATE, 0644) - if err != nil { - return nil, err - } - l := &unixLock{f} - err = l.set(true) - if err != nil { - f.Close() - return nil, err - } - return l, nil -} diff --git a/vendor/github.com/prometheus/tsdb/fileutil/flock_unix.go b/vendor/github.com/prometheus/tsdb/fileutil/flock_unix.go deleted file mode 100644 index f493fbd831d..00000000000 --- a/vendor/github.com/prometheus/tsdb/fileutil/flock_unix.go +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright 2016 The Prometheus Authors -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// +build darwin dragonfly freebsd linux netbsd openbsd - -package fileutil - -import ( - "os" - "syscall" -) - -type unixLock struct { - f *os.File -} - -func (l *unixLock) Release() error { - if err := l.set(false); err != nil { - return err - } - return l.f.Close() -} - -func (l *unixLock) set(lock bool) error { - how := syscall.LOCK_UN - if lock { - how = syscall.LOCK_EX - } - return syscall.Flock(int(l.f.Fd()), how|syscall.LOCK_NB) -} - -func newLock(fileName string) (Releaser, error) { - f, err := os.OpenFile(fileName, os.O_RDWR|os.O_CREATE, 0644) - if err != nil { - return nil, err - } - l := &unixLock{f} - err = l.set(true) - if err != nil { - f.Close() - return nil, err - } - return l, nil -} diff --git a/vendor/github.com/prometheus/tsdb/fileutil/flock_windows.go b/vendor/github.com/prometheus/tsdb/fileutil/flock_windows.go deleted file mode 100644 index 1c17ff4ea30..00000000000 --- a/vendor/github.com/prometheus/tsdb/fileutil/flock_windows.go +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright 2016 The Prometheus Authors -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package fileutil - -import "syscall" - -type windowsLock struct { - fd syscall.Handle -} - -func (fl *windowsLock) Release() error { - return syscall.Close(fl.fd) -} - -func newLock(fileName string) (Releaser, error) { - pathp, err := syscall.UTF16PtrFromString(fileName) - if err != nil { - return nil, err - } - fd, err := syscall.CreateFile(pathp, syscall.GENERIC_READ|syscall.GENERIC_WRITE, 0, nil, syscall.CREATE_ALWAYS, syscall.FILE_ATTRIBUTE_NORMAL, 0) - if err != nil { - return nil, err - } - return &windowsLock{fd}, nil -} diff --git a/vendor/github.com/prometheus/tsdb/fileutil/mmap.go b/vendor/github.com/prometheus/tsdb/fileutil/mmap.go deleted file mode 100644 index 26fc80c5850..00000000000 --- a/vendor/github.com/prometheus/tsdb/fileutil/mmap.go +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright 2018 The Prometheus Authors -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package fileutil - -import ( - "os" - - "github.com/pkg/errors" -) - -type MmapFile struct { - f *os.File - b []byte -} - -func OpenMmapFile(path string) (*MmapFile, error) { - f, err := os.Open(path) - if err != nil { - return nil, errors.Wrap(err, "try lock file") - } - info, err := f.Stat() - if err != nil { - return nil, errors.Wrap(err, "stat") - } - - b, err := mmap(f, int(info.Size())) - if err != nil { - return nil, errors.Wrap(err, "mmap") - } - - return &MmapFile{f: f, b: b}, nil -} - -func (f *MmapFile) Close() error { - err0 := munmap(f.b) - err1 := f.f.Close() - - if err0 != nil { - return err0 - } - return err1 -} - -func (f *MmapFile) File() *os.File { - return f.f -} - -func (f *MmapFile) Bytes() []byte { - return f.b -} diff --git a/vendor/github.com/prometheus/tsdb/fileutil/mmap_386.go b/vendor/github.com/prometheus/tsdb/fileutil/mmap_386.go deleted file mode 100644 index 66b9d368034..00000000000 --- a/vendor/github.com/prometheus/tsdb/fileutil/mmap_386.go +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright 2018 The Prometheus Authors -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// +build windows - -package fileutil - -const maxMapSize = 0x7FFFFFFF // 2GB diff --git a/vendor/github.com/prometheus/tsdb/fileutil/mmap_amd64.go b/vendor/github.com/prometheus/tsdb/fileutil/mmap_amd64.go deleted file mode 100644 index 4b523bc67c2..00000000000 --- a/vendor/github.com/prometheus/tsdb/fileutil/mmap_amd64.go +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright 2018 The Prometheus Authors -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// +build windows - -package fileutil - -const maxMapSize = 0xFFFFFFFFFFFF // 256TB diff --git a/vendor/github.com/prometheus/tsdb/fileutil/mmap_unix.go b/vendor/github.com/prometheus/tsdb/fileutil/mmap_unix.go deleted file mode 100644 index 043f4d408cc..00000000000 --- a/vendor/github.com/prometheus/tsdb/fileutil/mmap_unix.go +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright 2017 The Prometheus Authors -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// +build !windows,!plan9 - -package fileutil - -import ( - "os" - - "golang.org/x/sys/unix" -) - -func mmap(f *os.File, length int) ([]byte, error) { - return unix.Mmap(int(f.Fd()), 0, length, unix.PROT_READ, unix.MAP_SHARED) -} - -func munmap(b []byte) (err error) { - return unix.Munmap(b) -} diff --git a/vendor/github.com/prometheus/tsdb/fileutil/mmap_windows.go b/vendor/github.com/prometheus/tsdb/fileutil/mmap_windows.go deleted file mode 100644 index b9422641239..00000000000 --- a/vendor/github.com/prometheus/tsdb/fileutil/mmap_windows.go +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright 2017 The Prometheus Authors -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package fileutil - -import ( - "os" - "syscall" - "unsafe" -) - -func mmap(f *os.File, size int) ([]byte, error) { - low, high := uint32(size), uint32(size>>32) - h, errno := syscall.CreateFileMapping(syscall.Handle(f.Fd()), nil, syscall.PAGE_READONLY, high, low, nil) - if h == 0 { - return nil, os.NewSyscallError("CreateFileMapping", errno) - } - - addr, errno := syscall.MapViewOfFile(h, syscall.FILE_MAP_READ, 0, 0, uintptr(size)) - if addr == 0 { - return nil, os.NewSyscallError("MapViewOfFile", errno) - } - - if err := syscall.CloseHandle(syscall.Handle(h)); err != nil { - return nil, os.NewSyscallError("CloseHandle", err) - } - - return (*[maxMapSize]byte)(unsafe.Pointer(addr))[:size], nil -} - -func munmap(b []byte) error { - if err := syscall.UnmapViewOfFile((uintptr)(unsafe.Pointer(&b[0]))); err != nil { - return os.NewSyscallError("UnmapViewOfFile", err) - } - return nil -} diff --git a/vendor/github.com/prometheus/tsdb/fileutil/preallocate.go b/vendor/github.com/prometheus/tsdb/fileutil/preallocate.go deleted file mode 100644 index c747b7cf81f..00000000000 --- a/vendor/github.com/prometheus/tsdb/fileutil/preallocate.go +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright 2015 The etcd Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package fileutil - -import ( - "io" - "os" -) - -// Preallocate tries to allocate the space for given -// file. This operation is only supported on linux by a -// few filesystems (btrfs, ext4, etc.). -// If the operation is unsupported, no error will be returned. -// Otherwise, the error encountered will be returned. -func Preallocate(f *os.File, sizeInBytes int64, extendFile bool) error { - if sizeInBytes == 0 { - // fallocate will return EINVAL if length is 0; skip - return nil - } - if extendFile { - return preallocExtend(f, sizeInBytes) - } - return preallocFixed(f, sizeInBytes) -} - -func preallocExtendTrunc(f *os.File, sizeInBytes int64) error { - curOff, err := f.Seek(0, io.SeekCurrent) - if err != nil { - return err - } - size, err := f.Seek(sizeInBytes, io.SeekEnd) - if err != nil { - return err - } - if _, err = f.Seek(curOff, io.SeekStart); err != nil { - return err - } - if sizeInBytes > size { - return nil - } - return f.Truncate(sizeInBytes) -} diff --git a/vendor/github.com/prometheus/tsdb/fileutil/preallocate_darwin.go b/vendor/github.com/prometheus/tsdb/fileutil/preallocate_darwin.go deleted file mode 100644 index c9fa1a6c226..00000000000 --- a/vendor/github.com/prometheus/tsdb/fileutil/preallocate_darwin.go +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright 2015 The etcd Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package fileutil - -import ( - "os" - "syscall" - "unsafe" -) - -func preallocExtend(f *os.File, sizeInBytes int64) error { - if err := preallocFixed(f, sizeInBytes); err != nil { - return err - } - return preallocExtendTrunc(f, sizeInBytes) -} - -func preallocFixed(f *os.File, sizeInBytes int64) error { - fstore := &syscall.Fstore_t{ - Flags: syscall.F_ALLOCATEALL, - Posmode: syscall.F_PEOFPOSMODE, - Length: sizeInBytes} - p := unsafe.Pointer(fstore) - _, _, errno := syscall.Syscall(syscall.SYS_FCNTL, f.Fd(), uintptr(syscall.F_PREALLOCATE), uintptr(p)) - if errno == 0 || errno == syscall.ENOTSUP { - return nil - } - return errno -} diff --git a/vendor/github.com/prometheus/tsdb/fileutil/preallocate_linux.go b/vendor/github.com/prometheus/tsdb/fileutil/preallocate_linux.go deleted file mode 100644 index ada0462213e..00000000000 --- a/vendor/github.com/prometheus/tsdb/fileutil/preallocate_linux.go +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright 2015 The etcd Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package fileutil - -import ( - "os" - "syscall" -) - -func preallocExtend(f *os.File, sizeInBytes int64) error { - // use mode = 0 to change size - err := syscall.Fallocate(int(f.Fd()), 0, 0, sizeInBytes) - if err != nil { - errno, ok := err.(syscall.Errno) - // not supported; fallback - // fallocate EINTRs frequently in some environments; fallback - if ok && (errno == syscall.ENOTSUP || errno == syscall.EINTR) { - return preallocExtendTrunc(f, sizeInBytes) - } - } - return err -} - -func preallocFixed(f *os.File, sizeInBytes int64) error { - // use mode = 1 to keep size; see FALLOC_FL_KEEP_SIZE - err := syscall.Fallocate(int(f.Fd()), 1, 0, sizeInBytes) - if err != nil { - errno, ok := err.(syscall.Errno) - // treat not supported as nil error - if ok && errno == syscall.ENOTSUP { - return nil - } - } - return err -} diff --git a/vendor/github.com/prometheus/tsdb/fileutil/preallocate_other.go b/vendor/github.com/prometheus/tsdb/fileutil/preallocate_other.go deleted file mode 100644 index 162fbc5f782..00000000000 --- a/vendor/github.com/prometheus/tsdb/fileutil/preallocate_other.go +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright 2015 The etcd Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// +build !linux,!darwin - -package fileutil - -import "os" - -func preallocExtend(f *os.File, sizeInBytes int64) error { - return preallocExtendTrunc(f, sizeInBytes) -} - -func preallocFixed(f *os.File, sizeInBytes int64) error { return nil } diff --git a/vendor/github.com/prometheus/tsdb/fileutil/sync.go b/vendor/github.com/prometheus/tsdb/fileutil/sync.go deleted file mode 100644 index 2e64a40880a..00000000000 --- a/vendor/github.com/prometheus/tsdb/fileutil/sync.go +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright 2016 The etcd Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// +build !linux,!darwin - -package fileutil - -import "os" - -// Fdatasync is a wrapper around file.Sync(). Special handling is needed on linux platform. -func Fdatasync(f *os.File) error { - return f.Sync() -} diff --git a/vendor/github.com/prometheus/tsdb/fileutil/sync_darwin.go b/vendor/github.com/prometheus/tsdb/fileutil/sync_darwin.go deleted file mode 100644 index 2af1b0f4119..00000000000 --- a/vendor/github.com/prometheus/tsdb/fileutil/sync_darwin.go +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright 2016 The etcd Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// +build darwin - -package fileutil - -import ( - "os" -) - -// Fdatasync on darwin platform invokes fcntl(F_FULLFSYNC) for actual persistence -// on physical drive media. -func Fdatasync(f *os.File) error { - return f.Sync() -} diff --git a/vendor/github.com/prometheus/tsdb/fileutil/sync_linux.go b/vendor/github.com/prometheus/tsdb/fileutil/sync_linux.go deleted file mode 100644 index 8b4fc8268e5..00000000000 --- a/vendor/github.com/prometheus/tsdb/fileutil/sync_linux.go +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright 2016 The etcd Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// +build linux - -package fileutil - -import ( - "os" - "syscall" -) - -// Fdatasync is similar to fsync(), but does not flush modified metadata -// unless that metadata is needed in order to allow a subsequent data retrieval -// to be correctly handled. -func Fdatasync(f *os.File) error { - return syscall.Fdatasync(int(f.Fd())) -} diff --git a/vendor/modules.txt b/vendor/modules.txt index 5cbb62ac166..7cbc3fb3f40 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -456,8 +456,6 @@ github.com/prometheus/prometheus/util/teststorage github.com/prometheus/prometheus/util/testutil github.com/prometheus/prometheus/util/treecache github.com/prometheus/prometheus/web/api/v1 -# github.com/prometheus/tsdb v0.10.0 -github.com/prometheus/tsdb/fileutil # github.com/rafaeljusto/redigomock v0.0.0-20190202135759-257e089e14a1 github.com/rafaeljusto/redigomock # github.com/rs/cors v1.6.0 From 7a161423f580693af0b80271705be2445f4c970d Mon Sep 17 00:00:00 2001 From: Ganesh Vernekar Date: Wed, 13 Nov 2019 18:04:21 -0500 Subject: [PATCH 16/18] Fix Marco's comments Signed-off-by: Ganesh Vernekar --- pkg/ingester/lifecycle_test.go | 21 ------------ pkg/ring/lifecycler.go | 11 +++---- pkg/ring/lifecycler_test.go | 59 +++++++++++++++++----------------- pkg/ring/model_test.go | 8 ++--- pkg/ring/tokens.go | 24 ++++++++------ pkg/ring/tokens_test.go | 22 +++++++++++++ 6 files changed, 74 insertions(+), 71 deletions(-) create mode 100644 pkg/ring/tokens_test.go diff --git a/pkg/ingester/lifecycle_test.go b/pkg/ingester/lifecycle_test.go index 06e810b459b..6d7677d52df 100644 --- a/pkg/ingester/lifecycle_test.go +++ b/pkg/ingester/lifecycle_test.go @@ -4,7 +4,6 @@ import ( "io" "io/ioutil" "math" - "os" "testing" "time" @@ -95,23 +94,11 @@ func TestIngesterTransfer(t *testing.T) { limits, err := validation.NewOverrides(defaultLimitsTestConfig()) require.NoError(t, err) - tokenDir1, err := ioutil.TempDir(os.TempDir(), "ingester_transfer") - require.NoError(t, err) - defer func() { - require.NoError(t, os.RemoveAll(tokenDir1)) - }() - tokenDir2, err := ioutil.TempDir(os.TempDir(), "ingester_transfer") - require.NoError(t, err) - defer func() { - require.NoError(t, os.RemoveAll(tokenDir2)) - }() - // Start the first ingester, and get it into ACTIVE state. cfg1 := defaultIngesterTestConfig() cfg1.LifecyclerConfig.ID = "ingester1" cfg1.LifecyclerConfig.Addr = "ingester1" cfg1.LifecyclerConfig.JoinAfter = 0 * time.Second - cfg1.LifecyclerConfig.TokensFilePath = tokenDir1 cfg1.MaxTransferRetries = 10 ing1, err := New(cfg1, defaultClientTestConfig(), limits, nil, nil) require.NoError(t, err) @@ -155,7 +142,6 @@ func TestIngesterTransfer(t *testing.T) { cfg2.LifecyclerConfig.ID = "ingester2" cfg2.LifecyclerConfig.Addr = "ingester2" cfg2.LifecyclerConfig.JoinAfter = 100 * time.Second - cfg2.LifecyclerConfig.TokensFilePath = tokenDir2 ing2, err := New(cfg2, defaultClientTestConfig(), limits, nil, nil) require.NoError(t, err) @@ -195,18 +181,11 @@ func TestIngesterBadTransfer(t *testing.T) { limits, err := validation.NewOverrides(defaultLimitsTestConfig()) require.NoError(t, err) - tokenDir, err := ioutil.TempDir(os.TempDir(), "ingester_bad_transfer") - require.NoError(t, err) - defer func() { - require.NoError(t, os.RemoveAll(tokenDir)) - }() - // Start ingester in PENDING. cfg := defaultIngesterTestConfig() cfg.LifecyclerConfig.ID = "ingester1" cfg.LifecyclerConfig.Addr = "ingester1" cfg.LifecyclerConfig.JoinAfter = 100 * time.Second - cfg.LifecyclerConfig.TokensFilePath = tokenDir ing, err := New(cfg, defaultClientTestConfig(), limits, nil, nil) require.NoError(t, err) diff --git a/pkg/ring/lifecycler.go b/pkg/ring/lifecycler.go index c12886a2e0a..54751a5efc9 100644 --- a/pkg/ring/lifecycler.go +++ b/pkg/ring/lifecycler.go @@ -85,7 +85,7 @@ func (cfg *LifecyclerConfig) RegisterFlagsWithPrefix(prefix string, f *flag.Flag flagext.DeprecatedFlag(f, prefix+"claim-on-rollout", "DEPRECATED. This feature is no longer optional.") f.BoolVar(&cfg.NormaliseTokens, prefix+"normalise-tokens", false, "Store tokens in a normalised fashion to reduce allocations.") f.DurationVar(&cfg.FinalSleep, prefix+"final-sleep", 30*time.Second, "Duration to sleep for before exiting, to ensure metrics are scraped.") - f.StringVar(&cfg.TokensFilePath, prefix+"tokens-file-path", "", "File path in which the tokens is stored. If empty, tokens are not stored at shutdown and restored at startup.") + f.StringVar(&cfg.TokensFilePath, prefix+"tokens-file-path", "", "File path where tokens are stored. If empty, tokens are not stored at shutdown and restored at startup.") hostname, err := os.Hostname() if err != nil { @@ -251,9 +251,6 @@ func (i *Lifecycler) ChangeState(ctx context.Context, state IngesterState) error func (i *Lifecycler) getTokens() Tokens { i.stateMtx.Lock() defer i.stateMtx.Unlock() - if i.tokens == nil { - return nil - } return i.tokens } @@ -261,11 +258,11 @@ func (i *Lifecycler) setTokens(tokens Tokens) { tokensOwned.WithLabelValues(i.RingName).Set(float64(len(tokens))) i.stateMtx.Lock() - i.tokens = tokens - i.stateMtx.Unlock() + defer i.stateMtx.Unlock() + i.tokens = tokens if err := i.tokens.StoreToFile(i.cfg.TokensFilePath); err != nil { - level.Error(util.Logger).Log("msg", "error storing tokens to disk", "err", err) + level.Error(util.Logger).Log("msg", "error storing tokens to disk", "path", i.cfg.TokensFilePath, "err", err) } } diff --git a/pkg/ring/lifecycler_test.go b/pkg/ring/lifecycler_test.go index 32cf603d2e5..a69fdd67d17 100644 --- a/pkg/ring/lifecycler_test.go +++ b/pkg/ring/lifecycler_test.go @@ -4,6 +4,7 @@ import ( "context" "io/ioutil" "os" + "sort" "testing" "time" @@ -28,13 +29,7 @@ func (f *flushTransferer) TransferOut(ctx context.Context) error { return f.lifecycler.ChangeState(ctx, ACTIVE) } -func testLifecyclerConfig(t *testing.T, ringConfig Config, id string) LifecyclerConfig { - tokenDir, err := ioutil.TempDir(os.TempDir(), "ingester_bad_transfer") - require.NoError(t, err) - defer func() { - require.NoError(t, os.RemoveAll(tokenDir)) - }() - +func testLifecyclerConfig(ringConfig Config, id string) LifecyclerConfig { var lifecyclerConfig LifecyclerConfig flagext.DefaultValues(&lifecyclerConfig) lifecyclerConfig.Addr = "0.0.0.0" @@ -43,8 +38,6 @@ func testLifecyclerConfig(t *testing.T, ringConfig Config, id string) Lifecycler lifecyclerConfig.NumTokens = 1 lifecyclerConfig.ID = id lifecyclerConfig.FinalSleep = 0 - lifecyclerConfig.TokensFilePath = tokenDir - return lifecyclerConfig } @@ -76,7 +69,7 @@ func TestRingNormaliseMigration(t *testing.T) { defer r.Stop() // Add an 'ingester' with denormalised tokens. - lifecyclerConfig1 := testLifecyclerConfig(t, ringConfig, "ing1") + lifecyclerConfig1 := testLifecyclerConfig(ringConfig, "ing1") ft := &flushTransferer{} l1, err := NewLifecycler(lifecyclerConfig1, ft, "ingester") @@ -93,7 +86,7 @@ func TestRingNormaliseMigration(t *testing.T) { token := l1.tokens[0] // Add a second ingester with normalised tokens. - var lifecyclerConfig2 = testLifecyclerConfig(t, ringConfig, "ing2") + var lifecyclerConfig2 = testLifecyclerConfig(ringConfig, "ing2") lifecyclerConfig2.JoinAfter = 100 * time.Second lifecyclerConfig2.NormaliseTokens = true @@ -124,7 +117,7 @@ func TestLifecycler_HealthyInstancesCount(t *testing.T) { defer r.Stop() // Add the first ingester to the ring - lifecyclerConfig1 := testLifecyclerConfig(t, ringConfig, "ing1") + lifecyclerConfig1 := testLifecyclerConfig(ringConfig, "ing1") lifecyclerConfig1.HeartbeatPeriod = 100 * time.Millisecond lifecyclerConfig1.JoinAfter = 100 * time.Millisecond @@ -140,7 +133,7 @@ func TestLifecycler_HealthyInstancesCount(t *testing.T) { }) // Add the second ingester to the ring - lifecyclerConfig2 := testLifecyclerConfig(t, ringConfig, "ing2") + lifecyclerConfig2 := testLifecyclerConfig(ringConfig, "ing2") lifecyclerConfig2.HeartbeatPeriod = 100 * time.Millisecond lifecyclerConfig2.JoinAfter = 100 * time.Millisecond @@ -180,7 +173,7 @@ func TestRingRestart(t *testing.T) { defer r.Stop() // Add an 'ingester' with normalised tokens. - lifecyclerConfig1 := testLifecyclerConfig(t, ringConfig, "ing1") + lifecyclerConfig1 := testLifecyclerConfig(ringConfig, "ing1") lifecyclerConfig1.NormaliseTokens = true l1, err := NewLifecycler(lifecyclerConfig1, &nopFlushTransferer{}, "ingester") require.NoError(t, err) @@ -256,14 +249,13 @@ func TestCheckReady(t *testing.T) { flagext.DefaultValues(&ringConfig) ringConfig.KVStore.Mock = &MockClient{} - tokens := Tokens([]uint32{1}) r, err := New(ringConfig, "ingester") require.NoError(t, err) defer r.Stop() - cfg := testLifecyclerConfig(t, ringConfig, "ring1") + cfg := testLifecyclerConfig(ringConfig, "ring1") cfg.MinReadyDuration = 1 * time.Nanosecond l1, err := NewLifecycler(cfg, &nopFlushTransferer{}, "ingester") - l1.setTokens(tokens) + l1.setTokens(Tokens([]uint32{1})) l1.Start() require.NoError(t, err) @@ -289,28 +281,35 @@ func TestTokensOnDisk(t *testing.T) { require.NoError(t, err) defer r.Stop() - lifecyclerConfig := testLifecyclerConfig(t, ringConfig, "ing1") + tokenDir, err := ioutil.TempDir(os.TempDir(), "tokens_on_disk") + require.NoError(t, err) + defer func() { + require.NoError(t, os.RemoveAll(tokenDir)) + }() + + lifecyclerConfig := testLifecyclerConfig(ringConfig, "ing1") lifecyclerConfig.NumTokens = 512 + lifecyclerConfig.TokensFilePath = tokenDir + lifecyclerConfig.NormaliseTokens = true // Start first ingester. - ft := &noopFlushTransferer{} - l1, err := NewLifecycler(lifecyclerConfig, ft, "ingester") + l1, err := NewLifecycler(lifecyclerConfig, &noopFlushTransferer{}, "ingester") require.NoError(t, err) l1.Start() // Check this ingester joined, is active, and has 512 token. - var expTokens []TokenDesc + var expTokens []uint32 test.Poll(t, 1000*time.Millisecond, true, func() interface{} { d, err := r.KVClient.Get(context.Background(), ConsulKey) require.NoError(t, err) desc, ok := d.(*Desc) if ok { - expTokens = desc.Tokens + expTokens = desc.Ingesters["ing1"].Tokens } return ok && len(desc.Ingesters) == 1 && desc.Ingesters["ing1"].State == ACTIVE && - len(desc.Ingesters["ing1"].Tokens) == 0 && - len(desc.Tokens) == 512 + len(desc.Ingesters["ing1"].Tokens) == 512 && + len(desc.Tokens) == 0 }) l1.Shutdown() @@ -323,23 +322,25 @@ func TestTokensOnDisk(t *testing.T) { defer l2.Shutdown() // Check this ingester joined, is active, and has 512 token. - var actTokens []TokenDesc + var actTokens []uint32 test.Poll(t, 1000*time.Millisecond, true, func() interface{} { d, err := r.KVClient.Get(context.Background(), ConsulKey) require.NoError(t, err) desc, ok := d.(*Desc) if ok { - actTokens = desc.Tokens + actTokens = desc.Ingesters["ing2"].Tokens } return ok && len(desc.Ingesters) == 1 && desc.Ingesters["ing2"].State == ACTIVE && - len(desc.Ingesters["ing2"].Tokens) == 0 && - len(desc.Tokens) == 512 + len(desc.Ingesters["ing2"].Tokens) == 512 && + len(desc.Tokens) == 0 }) // Check for same tokens. + sort.Slice(expTokens, func(i, j int) bool { return expTokens[i] < expTokens[j] }) + sort.Slice(actTokens, func(i, j int) bool { return actTokens[i] < actTokens[j] }) for i := 0; i < 512; i++ { - require.Equal(t, expTokens[i].Token, actTokens[i].Token) + require.Equal(t, expTokens, actTokens) } } diff --git a/pkg/ring/model_test.go b/pkg/ring/model_test.go index e4656271ced..97e954ad06c 100644 --- a/pkg/ring/model_test.go +++ b/pkg/ring/model_test.go @@ -104,7 +104,7 @@ func TestClaimTokensFromNormalizedToNormalized(t *testing.T) { r := normalizedSource() result := r.ClaimTokens("first", "second", true) - assert.Equal(t, []uint32{100, 200, 300}, []uint32(result)) + assert.Equal(t, Tokens{100, 200, 300}, result) assert.Equal(t, normalizedOutput(), r) } @@ -112,7 +112,7 @@ func TestClaimTokensFromNormalizedToUnnormalized(t *testing.T) { r := normalizedSource() result := r.ClaimTokens("first", "second", false) - assert.Equal(t, []uint32{100, 200, 300}, []uint32(result)) + assert.Equal(t, Tokens{100, 200, 300}, result) assert.Equal(t, unnormalizedOutput(), r) } @@ -120,7 +120,7 @@ func TestClaimTokensFromUnnormalizedToUnnormalized(t *testing.T) { r := unnormalizedSource() result := r.ClaimTokens("first", "second", false) - assert.Equal(t, []uint32{100, 200, 300}, []uint32(result)) + assert.Equal(t, Tokens{100, 200, 300}, result) assert.Equal(t, unnormalizedOutput(), r) } @@ -129,7 +129,7 @@ func TestClaimTokensFromUnnormalizedToNormalized(t *testing.T) { result := r.ClaimTokens("first", "second", true) - assert.Equal(t, []uint32{100, 200, 300}, []uint32(result)) + assert.Equal(t, Tokens{100, 200, 300}, result) assert.Equal(t, normalizedOutput(), r) } diff --git a/pkg/ring/tokens.go b/pkg/ring/tokens.go index af8a7819638..3649ba93774 100644 --- a/pkg/ring/tokens.go +++ b/pkg/ring/tokens.go @@ -12,8 +12,8 @@ import ( ) const ( - // TokenVersion1 is the version is a simple list of tokens. - TokenVersion1 = 1 + // TokensVersion1 is the version is a simple list of tokens. + TokensVersion1 = 1 ) // Tokens is a simple list of tokens. @@ -24,16 +24,20 @@ func (t Tokens) StoreToFile(tokenFilePath string) error { if tokenFilePath == "" { return errors.New("path is empty") } - tmp := tokenFilePath + ".tmp" - f, err := os.OpenFile(tmp, os.O_WRONLY|os.O_CREATE, 0666) + + f, err := ioutil.TempFile(os.TempDir(), "tokens") if err != nil { return err } defer func() { + // If the file was not closed, then there must already be an error, hence ignore + // the error (if any) from f.Close(). If the file was already closed, then + // we would ignore the error in that case too. + f.Close() // RemoveAll returns no error when tmp doesn't exist so it is safe to always run it. - if err := os.RemoveAll(tmp); err != nil { - level.Error(util.Logger).Log("msg", "error deleting temporary file", "err", err) + if err := os.RemoveAll(f.Name()); err != nil { + level.Warn(util.Logger).Log("msg", "error deleting temporary file", "err", err) } }() @@ -50,7 +54,7 @@ func (t Tokens) StoreToFile(tokenFilePath string) error { } // Block successfully written, make visible and remove old ones. - return fileutil.Replace(tmp, tokenFilePath) + return fileutil.Replace(f.Name(), tokenFilePath) } // LoadFromFile loads tokens from given directory. @@ -68,7 +72,7 @@ func (t *Tokens) LoadFromFile(tokenFilePath string) error { // Marshal encodes the tokens into JSON. func (t Tokens) Marshal() ([]byte, error) { data := tokensJSON{ - Version: TokenVersion1, + Version: TokensVersion1, Tokens: t, } return json.Marshal(data) @@ -81,11 +85,11 @@ func (t *Tokens) Unmarshal(b []byte) error { return err } switch tj.Version { - case TokenVersion1: + case TokensVersion1: *t = Tokens(tj.Tokens) return nil default: - return errors.New("invalid token type") + return errors.New("invalid token version") } } diff --git a/pkg/ring/tokens_test.go b/pkg/ring/tokens_test.go new file mode 100644 index 00000000000..702b600a9eb --- /dev/null +++ b/pkg/ring/tokens_test.go @@ -0,0 +1,22 @@ +package ring + +import ( + "math/rand" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestTokenSerialization(t *testing.T) { + tokens := make(Tokens, 512) + for i := 0; i < 512; i++ { + tokens = append(tokens, uint32(rand.Int31())) + } + + b, err := tokens.Marshal() + require.NoError(t, err) + + var unmarshaledTokens Tokens + require.NoError(t, unmarshaledTokens.Unmarshal(b)) + require.Equal(t, tokens, unmarshaledTokens) +} From 41d37d8f58c8ca5ba0ea44666fcac8e2a90e5290 Mon Sep 17 00:00:00 2001 From: Ganesh Vernekar Date: Thu, 21 Nov 2019 15:55:39 -0800 Subject: [PATCH 17/18] Fix Peter's comments Signed-off-by: Ganesh Vernekar --- pkg/ring/lifecycler.go | 39 ++++++++++++++++++----------- pkg/ring/lifecycler_test.go | 2 +- pkg/ring/tokens.go | 49 ++++++++++++------------------------- 3 files changed, 41 insertions(+), 49 deletions(-) diff --git a/pkg/ring/lifecycler.go b/pkg/ring/lifecycler.go index 54751a5efc9..695296706ed 100644 --- a/pkg/ring/lifecycler.go +++ b/pkg/ring/lifecycler.go @@ -261,8 +261,10 @@ func (i *Lifecycler) setTokens(tokens Tokens) { defer i.stateMtx.Unlock() i.tokens = tokens - if err := i.tokens.StoreToFile(i.cfg.TokensFilePath); err != nil { - level.Error(util.Logger).Log("msg", "error storing tokens to disk", "path", i.cfg.TokensFilePath, "err", err) + if i.cfg.TokensFilePath != "" { + if err := i.tokens.StoreToFile(i.cfg.TokensFilePath); err != nil { + level.Error(util.Logger).Log("msg", "error storing tokens to disk", "path", i.cfg.TokensFilePath, "err", err) + } } } @@ -451,9 +453,22 @@ heartbeatLoop: // - add an ingester entry to the ring // - copies out our state and tokens if they exist func (i *Lifecycler) initRing(ctx context.Context) error { - var ringDesc *Desc + var ( + ringDesc *Desc + tokensFromFile Tokens + err error + ) + + if i.cfg.TokensFilePath != "" { + tokensFromFile, err = LoadTokensFromFile(i.cfg.TokensFilePath) + if err != nil { + level.Error(util.Logger).Log("msg", "error in getting tokens from file", "err", err) + } + } else { + level.Warn(util.Logger).Log("msg", "not loading tokens from file, tokens file path is empty") + } - err := i.KVStore.CAS(ctx, ConsulKey, func(in interface{}) (out interface{}, retry bool, err error) { + err = i.KVStore.CAS(ctx, ConsulKey, func(in interface{}) (out interface{}, retry bool, err error) { if in == nil { ringDesc = NewDesc() } else { @@ -462,18 +477,14 @@ func (i *Lifecycler) initRing(ctx context.Context) error { ingesterDesc, ok := ringDesc.Ingesters[i.ID] if !ok { - var tokens Tokens - // We load the tokens from the file only if it does not exist in the ring yet. - err := tokens.LoadFromFile(i.cfg.TokensFilePath) - if err != nil { - level.Error(util.Logger).Log("msg", "error in getting tokens from file", "err", err) - } else if len(tokens) > 0 { - level.Info(util.Logger).Log("msg", "adding tokens from file", "num_tokens", len(tokens)) - if len(tokens) == i.cfg.NumTokens { + // We use the tokens from the file only if it does not exist in the ring yet. + if len(tokensFromFile) > 0 { + level.Info(util.Logger).Log("msg", "adding tokens from file", "num_tokens", len(tokensFromFile)) + if len(tokensFromFile) == i.cfg.NumTokens { i.setState(ACTIVE) } - ringDesc.AddIngester(i.ID, i.Addr, tokens, i.GetState(), i.cfg.NormaliseTokens) - i.setTokens(tokens) + ringDesc.AddIngester(i.ID, i.Addr, tokensFromFile, i.GetState(), i.cfg.NormaliseTokens) + i.setTokens(tokensFromFile) return ringDesc, true, nil } diff --git a/pkg/ring/lifecycler_test.go b/pkg/ring/lifecycler_test.go index a69fdd67d17..06854caee03 100644 --- a/pkg/ring/lifecycler_test.go +++ b/pkg/ring/lifecycler_test.go @@ -289,7 +289,7 @@ func TestTokensOnDisk(t *testing.T) { lifecyclerConfig := testLifecyclerConfig(ringConfig, "ing1") lifecyclerConfig.NumTokens = 512 - lifecyclerConfig.TokensFilePath = tokenDir + lifecyclerConfig.TokensFilePath = tokenDir + "/tokens" lifecyclerConfig.NormaliseTokens = true // Start first ingester. diff --git a/pkg/ring/tokens.go b/pkg/ring/tokens.go index 3649ba93774..3a8fd180a87 100644 --- a/pkg/ring/tokens.go +++ b/pkg/ring/tokens.go @@ -5,15 +5,6 @@ import ( "errors" "io/ioutil" "os" - - "github.com/cortexproject/cortex/pkg/util" - "github.com/go-kit/kit/log/level" - "github.com/prometheus/prometheus/tsdb/fileutil" -) - -const ( - // TokensVersion1 is the version is a simple list of tokens. - TokensVersion1 = 1 ) // Tokens is a simple list of tokens. @@ -25,6 +16,8 @@ func (t Tokens) StoreToFile(tokenFilePath string) error { return errors.New("path is empty") } + // If any operations failed further in the function, we keep the temporary + // file hanging around for debugging. f, err := ioutil.TempFile(os.TempDir(), "tokens") if err != nil { return err @@ -35,10 +28,6 @@ func (t Tokens) StoreToFile(tokenFilePath string) error { // the error (if any) from f.Close(). If the file was already closed, then // we would ignore the error in that case too. f.Close() - // RemoveAll returns no error when tmp doesn't exist so it is safe to always run it. - if err := os.RemoveAll(f.Name()); err != nil { - level.Warn(util.Logger).Log("msg", "error deleting temporary file", "err", err) - } }() b, err := t.Marshal() @@ -53,29 +42,27 @@ func (t Tokens) StoreToFile(tokenFilePath string) error { return err } - // Block successfully written, make visible and remove old ones. - return fileutil.Replace(f.Name(), tokenFilePath) + // Tokens successfully written, replace the temporary file with the actual file path. + return os.Rename(f.Name(), tokenFilePath) } -// LoadFromFile loads tokens from given directory. -func (t *Tokens) LoadFromFile(tokenFilePath string) error { +// LoadTokensFromFile loads tokens from given file path. +func LoadTokensFromFile(tokenFilePath string) (Tokens, error) { b, err := ioutil.ReadFile(tokenFilePath) if err != nil { if os.IsNotExist(err) { - return nil + return nil, nil } - return err + return nil, err } - return t.Unmarshal(b) + var t Tokens + err = t.Unmarshal(b) + return t, err } // Marshal encodes the tokens into JSON. func (t Tokens) Marshal() ([]byte, error) { - data := tokensJSON{ - Version: TokensVersion1, - Tokens: t, - } - return json.Marshal(data) + return json.Marshal(tokensJSON{Tokens: t}) } // Unmarshal reads the tokens from JSON byte stream. @@ -84,16 +71,10 @@ func (t *Tokens) Unmarshal(b []byte) error { if err := json.Unmarshal(b, &tj); err != nil { return err } - switch tj.Version { - case TokensVersion1: - *t = Tokens(tj.Tokens) - return nil - default: - return errors.New("invalid token version") - } + *t = Tokens(tj.Tokens) + return nil } type tokensJSON struct { - Version int `json:"version"` - Tokens []uint32 `json:"tokens"` + Tokens []uint32 `json:"tokens"` } From 8838d977f9deab9bdb6d5f5ba31285359664c013 Mon Sep 17 00:00:00 2001 From: Ganesh Vernekar Date: Mon, 2 Dec 2019 18:27:08 +0530 Subject: [PATCH 18/18] Fix Goutham's and Peter's comment Signed-off-by: Ganesh Vernekar --- pkg/ring/lifecycler.go | 13 ++++++------- pkg/ring/model.go | 6 +++--- pkg/ring/tokens.go | 9 +++++---- pkg/ring/util.go | 6 ------ 4 files changed, 14 insertions(+), 20 deletions(-) diff --git a/pkg/ring/lifecycler.go b/pkg/ring/lifecycler.go index 695296706ed..c1e7172295f 100644 --- a/pkg/ring/lifecycler.go +++ b/pkg/ring/lifecycler.go @@ -465,7 +465,7 @@ func (i *Lifecycler) initRing(ctx context.Context) error { level.Error(util.Logger).Log("msg", "error in getting tokens from file", "err", err) } } else { - level.Warn(util.Logger).Log("msg", "not loading tokens from file, tokens file path is empty") + level.Info(util.Logger).Log("msg", "not loading tokens from file, tokens file path is empty") } err = i.KVStore.CAS(ctx, ConsulKey, func(in interface{}) (out interface{}, retry bool, err error) { @@ -480,7 +480,7 @@ func (i *Lifecycler) initRing(ctx context.Context) error { // We use the tokens from the file only if it does not exist in the ring yet. if len(tokensFromFile) > 0 { level.Info(util.Logger).Log("msg", "adding tokens from file", "num_tokens", len(tokensFromFile)) - if len(tokensFromFile) == i.cfg.NumTokens { + if len(tokensFromFile) >= i.cfg.NumTokens { i.setState(ACTIVE) } ringDesc.AddIngester(i.ID, i.Addr, tokensFromFile, i.GetState(), i.cfg.NormaliseTokens) @@ -537,7 +537,7 @@ func (i *Lifecycler) verifyTokens(ctx context.Context) bool { newTokens := GenerateTokens(needTokens, takenTokens) ringTokens = append(ringTokens, newTokens...) - sort.Sort(sortableUint32(ringTokens)) + sort.Sort(ringTokens) ringDesc.AddIngester(i.ID, i.Addr, ringTokens, i.GetState(), i.cfg.NormaliseTokens) @@ -560,10 +560,10 @@ func (i *Lifecycler) verifyTokens(ctx context.Context) bool { } func (i *Lifecycler) compareTokens(fromRing Tokens) bool { - sort.Sort(sortableUint32(fromRing)) + sort.Sort(fromRing) tokens := i.getTokens() - sort.Sort(sortableUint32(tokens)) + sort.Sort(tokens) if len(tokens) != len(fromRing) { return false @@ -596,11 +596,10 @@ func (i *Lifecycler) autoJoin(ctx context.Context, targetState IngesterState) er newTokens := GenerateTokens(i.cfg.NumTokens-len(myTokens), takenTokens) i.setState(targetState) - ringDesc.AddIngester(i.ID, i.Addr, newTokens, i.GetState(), i.cfg.NormaliseTokens) myTokens = append(myTokens, newTokens...) - sort.Sort(sortableUint32(myTokens)) + sort.Sort(myTokens) i.setTokens(myTokens) return ringDesc, true, nil diff --git a/pkg/ring/model.go b/pkg/ring/model.go index 8b1f7200175..43ae3140ff1 100644 --- a/pkg/ring/model.go +++ b/pkg/ring/model.go @@ -319,8 +319,8 @@ func buildNormalizedIngestersMap(inputRing *Desc) map[string]IngesterDesc { continue } - if !sort.IsSorted(sortableUint32(ing.Tokens)) { - sort.Sort(sortableUint32(ing.Tokens)) + if !sort.IsSorted(Tokens(ing.Tokens)) { + sort.Sort(Tokens(ing.Tokens)) } seen := make(map[uint32]bool) @@ -407,7 +407,7 @@ func resolveConflicts(normalizedIngesters map[string]IngesterDesc) { } } - sort.Sort(sortableUint32(tokens)) + sort.Sort(Tokens(tokens)) // let's store the resolved result back newTokenLists := map[string][]uint32{} diff --git a/pkg/ring/tokens.go b/pkg/ring/tokens.go index 3a8fd180a87..b8ec5e323b9 100644 --- a/pkg/ring/tokens.go +++ b/pkg/ring/tokens.go @@ -10,6 +10,10 @@ import ( // Tokens is a simple list of tokens. type Tokens []uint32 +func (t Tokens) Len() int { return len(t) } +func (t Tokens) Swap(i, j int) { t[i], t[j] = t[j], t[i] } +func (t Tokens) Less(i, j int) bool { return t[i] < t[j] } + // StoreToFile stores the tokens in the given directory. func (t Tokens) StoreToFile(tokenFilePath string) error { if tokenFilePath == "" { @@ -27,7 +31,7 @@ func (t Tokens) StoreToFile(tokenFilePath string) error { // If the file was not closed, then there must already be an error, hence ignore // the error (if any) from f.Close(). If the file was already closed, then // we would ignore the error in that case too. - f.Close() + _ = f.Close() }() b, err := t.Marshal() @@ -50,9 +54,6 @@ func (t Tokens) StoreToFile(tokenFilePath string) error { func LoadTokensFromFile(tokenFilePath string) (Tokens, error) { b, err := ioutil.ReadFile(tokenFilePath) if err != nil { - if os.IsNotExist(err) { - return nil, nil - } return nil, err } var t Tokens diff --git a/pkg/ring/util.go b/pkg/ring/util.go index d072e549322..e30166fa7bc 100644 --- a/pkg/ring/util.go +++ b/pkg/ring/util.go @@ -27,9 +27,3 @@ func GenerateTokens(numTokens int, takenTokens []uint32) []uint32 { } return tokens } - -type sortableUint32 []uint32 - -func (ts sortableUint32) Len() int { return len(ts) } -func (ts sortableUint32) Swap(i, j int) { ts[i], ts[j] = ts[j], ts[i] } -func (ts sortableUint32) Less(i, j int) bool { return ts[i] < ts[j] }