Skip to content

Commit e2e5bcf

Browse files
authored
Handling CMK AccessDenied errors (#5420)
* Handling CMK Errors Signed-off-by: Alan Protasio <[email protected]> * lint Signed-off-by: Alan Protasio <[email protected]> * add test on BucketStores Signed-off-by: Alan Protasio <[email protected]> * fixing race Signed-off-by: Alan Protasio <[email protected]> * lint Signed-off-by: Alan Protasio <[email protected]> * Implementing error handling on labels apis Signed-off-by: Alan Protasio <[email protected]> * handling errros from thanos SG Signed-off-by: Alan Protasio <[email protected]> * creating IsOneOfTheExpectedErrors func Signed-off-by: Alan Protasio <[email protected]> --------- Signed-off-by: Alan Protasio <[email protected]>
1 parent 1dcb72d commit e2e5bcf

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

41 files changed

+767
-38
lines changed

go.mod

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ require (
5151
github.com/sony/gobreaker v0.5.0
5252
github.com/spf13/afero v1.9.5
5353
github.com/stretchr/testify v1.8.4
54-
github.com/thanos-io/objstore v0.0.0-20230522103316-23ebe2eacadd
54+
github.com/thanos-io/objstore v0.0.0-20230629211010-ff1b35b9841a
5555
github.com/thanos-io/promql-engine v0.0.0-20230526105742-791d78b260ea
5656
github.com/thanos-io/thanos v0.31.1-0.20230627154113-7cfaf3fe2d43
5757
github.com/uber/jaeger-client-go v2.30.0+incompatible

go.sum

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1160,8 +1160,8 @@ github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXl
11601160
github.com/tencentyun/cos-go-sdk-v5 v0.7.40 h1:W6vDGKCHe4wBACI1d2UgE6+50sJFhRWU4O8IB2ozzxM=
11611161
github.com/thanos-community/galaxycache v0.0.0-20211122094458-3a32041a1f1e h1:f1Zsv7OAU9iQhZwigp50Yl38W10g/vd5NC8Rdk1Jzng=
11621162
github.com/thanos-community/galaxycache v0.0.0-20211122094458-3a32041a1f1e/go.mod h1:jXcofnrSln/cLI6/dhlBxPQZEEQHVPCcFaH75M+nSzM=
1163-
github.com/thanos-io/objstore v0.0.0-20230522103316-23ebe2eacadd h1:asQ0HomkaUXZuR3J7daBEusMS++3hkYsYM6u8gpmPWM=
1164-
github.com/thanos-io/objstore v0.0.0-20230522103316-23ebe2eacadd/go.mod h1:5V7lzXuaxwt6XFQoA/zJrhdnQrxq1+r0bwQ1iYOq3gM=
1163+
github.com/thanos-io/objstore v0.0.0-20230629211010-ff1b35b9841a h1:tXcVeuval1nzdHn1JXqaBmyjuEUcpDI9huPrUF04nR4=
1164+
github.com/thanos-io/objstore v0.0.0-20230629211010-ff1b35b9841a/go.mod h1:5V7lzXuaxwt6XFQoA/zJrhdnQrxq1+r0bwQ1iYOq3gM=
11651165
github.com/thanos-io/promql-engine v0.0.0-20230526105742-791d78b260ea h1:kzK8sBn2+mo3NAxP+XjAjAqr1hwfxxFUy5CybaBkjAI=
11661166
github.com/thanos-io/promql-engine v0.0.0-20230526105742-791d78b260ea/go.mod h1:eIgPaXWgOhNAv6CPPrgu09r0AtT7byBTZy+7WkX0D18=
11671167
github.com/thanos-io/thanos v0.31.1-0.20230627154113-7cfaf3fe2d43 h1:UHyTPGdDHAoNHuSce5cJ2vEi6g1v8D5ZFBWZ61uTHSM=

pkg/compactor/blocks_cleaner.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -325,6 +325,10 @@ func (c *BlocksCleaner) cleanUser(ctx context.Context, userID string, firstRun b
325325
idx, err := bucketindex.ReadIndex(ctx, c.bucketClient, userID, c.cfgProvider, c.logger)
326326
if errors.Is(err, bucketindex.ErrIndexCorrupted) {
327327
level.Warn(userLogger).Log("msg", "found a corrupted bucket index, recreating it")
328+
} else if errors.Is(err, bucket.ErrCustomerManagedKeyAccessDenied) {
329+
// Give up cleaning if we get access denied
330+
level.Warn(userLogger).Log("msg", err.Error())
331+
return nil
328332
} else if err != nil && !errors.Is(err, bucketindex.ErrIndexNotFound) {
329333
return err
330334
}

pkg/compactor/blocks_cleaner_test.go

Lines changed: 32 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@ package compactor
33
import (
44
"context"
55
"crypto/rand"
6-
"errors"
76
"fmt"
87
"path"
98
"strings"
@@ -17,14 +16,12 @@ import (
1716
prom_testutil "github.com/prometheus/client_golang/prometheus/testutil"
1817
"github.com/stretchr/testify/assert"
1918
"github.com/stretchr/testify/require"
20-
"github.com/thanos-io/objstore"
2119
"github.com/thanos-io/thanos/pkg/block"
2220
"github.com/thanos-io/thanos/pkg/block/metadata"
2321

2422
"github.com/cortexproject/cortex/pkg/storage/tsdb"
2523
"github.com/cortexproject/cortex/pkg/storage/tsdb/bucketindex"
2624
cortex_testutil "github.com/cortexproject/cortex/pkg/storage/tsdb/testutil"
27-
"github.com/cortexproject/cortex/pkg/util"
2825
"github.com/cortexproject/cortex/pkg/util/services"
2926
)
3027

@@ -57,6 +54,37 @@ func TestBlocksCleaner(t *testing.T) {
5754
}
5855
}
5956

57+
func TestBlockCleaner_KeyPermissionDenied(t *testing.T) {
58+
const userID = "user-1"
59+
60+
bucketClient, _ := cortex_testutil.PrepareFilesystemBucket(t)
61+
bucketClient = bucketindex.BucketWithGlobalMarkers(bucketClient)
62+
63+
// Create blocks.
64+
ctx := context.Background()
65+
deletionDelay := 12 * time.Hour
66+
bucketClient = &cortex_testutil.MockBucketFailure{
67+
Bucket: bucketClient,
68+
GetFailures: map[string]error{
69+
path.Join(userID, "bucket-index.json.gz"): cortex_testutil.ErrKeyAccessDeniedError,
70+
},
71+
}
72+
73+
cfg := BlocksCleanerConfig{
74+
DeletionDelay: deletionDelay,
75+
CleanupInterval: time.Minute,
76+
CleanupConcurrency: 1,
77+
}
78+
79+
logger := log.NewNopLogger()
80+
scanner := tsdb.NewUsersScanner(bucketClient, tsdb.AllUsers, logger)
81+
cfgProvider := newMockConfigProvider()
82+
83+
cleaner := NewBlocksCleaner(cfg, bucketClient, scanner, cfgProvider, logger, nil)
84+
err := cleaner.cleanUser(ctx, userID, true)
85+
require.NoError(t, err)
86+
}
87+
6088
func testBlocksCleanerWithOptions(t *testing.T, options testBlocksCleanerOptions) {
6189
bucketClient, _ := cortex_testutil.PrepareFilesystemBucket(t)
6290

@@ -254,7 +282,7 @@ func TestBlocksCleaner_ShouldContinueOnBlockDeletionFailure(t *testing.T) {
254282
createDeletionMark(t, bucketClient, userID, block4, now.Add(-deletionDelay).Add(-time.Hour))
255283

256284
// To emulate a failure deleting a block, we wrap the bucket client in a mocked one.
257-
bucketClient = &mockBucketFailure{
285+
bucketClient = &cortex_testutil.MockBucketFailure{
258286
Bucket: bucketClient,
259287
DeleteFailures: []string{path.Join(userID, block3.String(), metadata.MetaFilename)},
260288
}
@@ -658,19 +686,6 @@ func TestBlocksCleaner_ShouldRemoveBlocksOutsideRetentionPeriod(t *testing.T) {
658686
}
659687
}
660688

661-
type mockBucketFailure struct {
662-
objstore.Bucket
663-
664-
DeleteFailures []string
665-
}
666-
667-
func (m *mockBucketFailure) Delete(ctx context.Context, name string) error {
668-
if util.StringsContain(m.DeleteFailures, name) {
669-
return errors.New("mocked delete failure")
670-
}
671-
return m.Bucket.Delete(ctx, name)
672-
}
673-
674689
type mockConfigProvider struct {
675690
userRetentionPeriods map[string]time.Duration
676691
}

pkg/querier/blocks_finder_bucket_index.go

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,8 @@ import (
1010
"github.com/prometheus/client_golang/prometheus"
1111
"github.com/thanos-io/objstore"
1212

13+
"github.com/cortexproject/cortex/pkg/util/validation"
14+
1315
"github.com/cortexproject/cortex/pkg/storage/bucket"
1416
"github.com/cortexproject/cortex/pkg/storage/tsdb/bucketindex"
1517
"github.com/cortexproject/cortex/pkg/util/services"
@@ -62,6 +64,11 @@ func (f *BucketIndexBlocksFinder) GetBlocks(ctx context.Context, userID string,
6264
// so the bucket index hasn't been created yet.
6365
return nil, nil, nil
6466
}
67+
68+
if errors.Is(err, bucket.ErrCustomerManagedKeyAccessDenied) {
69+
return nil, nil, validation.AccessDeniedError(err.Error())
70+
}
71+
6572
if err != nil {
6673
return nil, nil, err
6774
}

pkg/querier/blocks_finder_bucket_index_test.go

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,8 @@ import (
1313
"github.com/stretchr/testify/require"
1414
"github.com/thanos-io/objstore"
1515

16+
"github.com/cortexproject/cortex/pkg/util/validation"
17+
1618
"github.com/cortexproject/cortex/pkg/storage/tsdb/bucketindex"
1719
cortex_testutil "github.com/cortexproject/cortex/pkg/storage/tsdb/testutil"
1820
"github.com/cortexproject/cortex/pkg/util/services"
@@ -241,3 +243,21 @@ func prepareBucketIndexBlocksFinder(t testing.TB, bkt objstore.Bucket) *BucketIn
241243

242244
return finder
243245
}
246+
247+
func TestBucketIndexBlocksFinder_GetBlocks_KeyPermissionDenied(t *testing.T) {
248+
const userID = "user-1"
249+
bkt, _ := cortex_testutil.PrepareFilesystemBucket(t)
250+
251+
bkt = &cortex_testutil.MockBucketFailure{
252+
Bucket: bkt,
253+
GetFailures: map[string]error{
254+
path.Join(userID, "bucket-index.json.gz"): cortex_testutil.ErrKeyAccessDeniedError,
255+
},
256+
}
257+
258+
finder := prepareBucketIndexBlocksFinder(t, bkt)
259+
260+
_, _, err := finder.GetBlocks(context.Background(), userID, 0, 100)
261+
expected := validation.AccessDeniedError("error")
262+
require.IsType(t, expected, err)
263+
}

pkg/querier/error_translate_queryable.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,9 @@ func TranslateToPromqlAPIError(err error) error {
3939
case validation.LimitError:
4040
// This will be returned with status code 422 by Prometheus API.
4141
return err
42+
case validation.AccessDeniedError:
43+
// This will be returned with status code 422 by Prometheus API.
44+
return err
4245
default:
4346
if errors.Is(err, context.Canceled) {
4447
return err // 422

pkg/querier/error_translate_queryable_test.go

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,12 @@ func TestApiStatusCodes(t *testing.T) {
4343
expectedCode: 422,
4444
},
4545

46+
{
47+
err: validation.AccessDeniedError("access denied"),
48+
expectedString: "access denied",
49+
expectedCode: 422,
50+
},
51+
4652
{
4753
err: promql.ErrTooManySamples("query execution"),
4854
expectedString: "too many samples",

pkg/storage/bucket/client.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,8 @@ var (
4040
SupportedBackends = []string{S3, GCS, Azure, Swift, Filesystem}
4141

4242
ErrUnsupportedStorageBackend = errors.New("unsupported storage backend")
43+
44+
ErrCustomerManagedKeyAccessDenied = errors.New("access denied: customer key")
4345
)
4446

4547
// Config holds configuration for accessing long-term storage.

pkg/storage/bucket/client_mock.go

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,10 @@ import (
1212
"github.com/thanos-io/objstore"
1313
)
1414

15-
var errObjectDoesNotExist = errors.New("object does not exist")
15+
var (
16+
errObjectDoesNotExist = errors.New("object does not exist")
17+
errKeyPermissionDenied = errors.New("object key permission denied")
18+
)
1619

1720
// ClientMock mocks objstore.Bucket
1821
type ClientMock struct {
@@ -175,6 +178,11 @@ func (m *ClientMock) IsObjNotFoundErr(err error) bool {
175178
return err == errObjectDoesNotExist
176179
}
177180

181+
// IsCustomerManagedKeyError mocks objstore.Bucket.IsCustomerManagedKeyError()
182+
func (m *ClientMock) IsCustomerManagedKeyError(err error) bool {
183+
return err == errKeyPermissionDenied
184+
}
185+
178186
// ObjectSize mocks objstore.Bucket.Attributes()
179187
func (m *ClientMock) Attributes(ctx context.Context, name string) (objstore.ObjectAttributes, error) {
180188
args := m.Called(ctx, name)

0 commit comments

Comments
 (0)