From 40dbc29984d67a3f4946a0b30d53f3db19952bf0 Mon Sep 17 00:00:00 2001 From: wangyingbin Date: Wed, 16 Sep 2020 15:15:45 +0800 Subject: [PATCH 0001/2866] rgw:When KMS encryption is used and the key does not exist, we should not throw ERR_ INVALID_ ACCESS_ Key error code. When kms encryption is used, the key_id is null or the actual_key size is wrong, we should not throw "ERR_INVALID_ACCESS_KEY " error code, instead of "EINVAL"error code, is used to indicate parameter error. Signed-off-by: wangyingbin --- src/rgw/rgw_crypt.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/rgw/rgw_crypt.cc b/src/rgw/rgw_crypt.cc index 176a830430e09..3a040da942cca 100644 --- a/src/rgw/rgw_crypt.cc +++ b/src/rgw/rgw_crypt.cc @@ -782,7 +782,7 @@ int rgw_s3_prepare_encrypt(struct req_state* s, ldout(s->cct, 5) << "ERROR: not provide a valid key id" << dendl; s->err.message = "Server Side Encryption with KMS managed key requires " "HTTP header x-amz-server-side-encryption-aws-kms-key-id"; - return -ERR_INVALID_ACCESS_KEY; + return -EINVAL; } /* try to retrieve actual key */ std::string key_selector = create_random_key_selector(s->cct); @@ -797,7 +797,7 @@ int rgw_s3_prepare_encrypt(struct req_state* s, ldout(s->cct, 5) << "ERROR: key obtained from key_id:" << key_id << " is not 256 bit size" << dendl; s->err.message = "KMS provided an invalid key for the given kms-keyid."; - return -ERR_INVALID_ACCESS_KEY; + return -EINVAL; } set_attr(attrs, RGW_ATTR_CRYPT_MODE, "SSE-KMS"); set_attr(attrs, RGW_ATTR_CRYPT_KEYID, key_id); From b8d7d356120dc3698def0b88deb5616448cfd3c0 Mon Sep 17 00:00:00 2001 From: Seena Fallah Date: Wed, 14 Oct 2020 10:43:15 +0330 Subject: [PATCH 0002/2866] rgw: add infile arg to role-policy put Allow passing policy-doc with infile arg Signed-off-by: Seena Fallah --- doc/radosgw/role.rst | 4 ++++ src/rgw/rgw_admin.cc | 14 ++++++++++++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/doc/radosgw/role.rst b/doc/radosgw/role.rst index 4511e2fe13311..66ef0ac9c1f84 100644 --- a/doc/radosgw/role.rst +++ b/doc/radosgw/role.rst @@ -203,6 +203,10 @@ For example:: radosgw-admin role-policy put --role-name=S3Access1 --policy-name=Policy1 --policy-doc=\{\"Version\":\"2012-10-17\",\"Statement\":\[\{\"Effect\":\"Allow\",\"Action\":\[\"s3:*\"\],\"Resource\":\"arn:aws:s3:::example_bucket\"\}\]\} +For passing ``policy-doc`` as a file:: + + radosgw-admin role-policy put --role-name=S3Access1 --policy-name=Policy1 --infile policy-document.json + In the above example, we are attaching a policy 'Policy1' to role 'S3Access1', which allows all s3 actions on 'example_bucket'. List Permission Policy Names attached to a Role diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc index 9f4ef12dbff0a..376f4b5489130 100644 --- a/src/rgw/rgw_admin.cc +++ b/src/rgw/rgw_admin.cc @@ -5834,12 +5834,22 @@ int main(int argc, const char **argv) return -EINVAL; } - if (perm_policy_doc.empty()) { + if (perm_policy_doc.empty() && infile.empty()) { cerr << "permission policy document is empty" << std::endl; return -EINVAL; } - bufferlist bl = bufferlist::static_from_string(perm_policy_doc); + bufferlist bl; + if (!infile.empty()) { + int ret = read_input(infile, bl); + if (ret < 0) { + cerr << "ERROR: failed to read input policy document: " << cpp_strerror(-ret) << std::endl; + return -ret; + } + perm_policy_doc = bl.to_str(); + } else { + bl = bufferlist::static_from_string(perm_policy_doc); + } try { const rgw::IAM::Policy p(g_ceph_context, tenant, bl); } catch (rgw::IAM::PolicyParseException& e) { From 3ae77c51a74f441d1e9a078fc7bf61103f783b5e Mon Sep 17 00:00:00 2001 From: Liu Lan Date: Thu, 10 Dec 2020 17:00:56 +0800 Subject: [PATCH 0003/2866] rgw: an empty tagset is allowed by S3 Signed-off-by: Liu Lan --- src/rgw/rgw_tag.cc | 3 +++ src/rgw/rgw_tag_s3.cc | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/rgw/rgw_tag.cc b/src/rgw/rgw_tag.cc index f43a182486856..6f030c3577ccb 100644 --- a/src/rgw/rgw_tag.cc +++ b/src/rgw/rgw_tag.cc @@ -35,6 +35,9 @@ int RGWObjTags::check_and_add_tag(const string&key, const string& val){ } int RGWObjTags::set_from_string(const string& input){ + if (input.empty()) { + return 0; + } int ret=0; vector kvs; boost::split(kvs, input, boost::is_any_of("&")); diff --git a/src/rgw/rgw_tag_s3.cc b/src/rgw/rgw_tag_s3.cc index b03607f73a7e6..b54265f90bff5 100644 --- a/src/rgw/rgw_tag_s3.cc +++ b/src/rgw/rgw_tag_s3.cc @@ -30,7 +30,8 @@ void RGWObjTagEntry_S3::dump_xml(Formatter *f) const { void RGWObjTagSet_S3::decode_xml(XMLObj *obj) { vector entries; - RGWXMLDecoder::decode_xml("Tag", entries, obj, true); + bool mandatory{false}; + RGWXMLDecoder::decode_xml("Tag", entries, obj, mandatory); for (auto& entry : entries) { const std::string& key = entry.get_key(); From 7c7b6f4ed6bcb9be7456537204b7b374c7df7115 Mon Sep 17 00:00:00 2001 From: Adam Kupczyk Date: Thu, 25 Feb 2021 13:16:01 +0100 Subject: [PATCH 0004/2866] os/bluestore: Set new compression blob size to 64K Signed-off-by: Adam Kupczyk --- src/common/options/global.yaml.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/options/global.yaml.in b/src/common/options/global.yaml.in index deb15388ccbff..172caf2f6b288 100644 --- a/src/common/options/global.yaml.in +++ b/src/common/options/global.yaml.in @@ -5446,7 +5446,7 @@ options: state) media fmt_desc: Default value of ``bluestore compression min blob size`` for non-rotational (solid state) media. - default: 8_K + default: 64_K see_also: - bluestore_compression_min_blob_size flags: From 6380481f0538f0b0655f0a1c1c05db623447ec03 Mon Sep 17 00:00:00 2001 From: Paul Reece Date: Fri, 4 Jun 2021 14:38:39 -0400 Subject: [PATCH 0005/2866] rgw: allow CompletedMultipartUpload in XML Bit of a strange one here It seems some S3 libraries like nodejs and golang have been using CompletedMultipartUpload instead of CompleteMultipartUpload in their XML, which some things speaking the S3 protocol allow, while others do not See also: https://github.com/aws/aws-sdk-js-v3/issues/1814 I have no idea if the Ceph Project wants to allow this undocumented behavior or not, but if you do, this code works for me :) Signed-off-by: Paul Reece --- src/rgw/rgw_multi.cc | 1 + src/rgw/rgw_op.cc | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/src/rgw/rgw_multi.cc b/src/rgw/rgw_multi.cc index e3369f64b641f..82c5d0ed2cd97 100644 --- a/src/rgw/rgw_multi.cc +++ b/src/rgw/rgw_multi.cc @@ -57,6 +57,7 @@ bool RGWMultiCompleteUpload::xml_end(const char *el) { XMLObj *RGWMultiXMLParser::alloc_obj(const char *el) { XMLObj *obj = NULL; if (strcmp(el, "CompleteMultipartUpload") == 0 || + strcmp(el, "CompletedMultipartUpload") == 0 || strcmp(el, "MultipartUpload") == 0) { obj = new RGWMultiCompleteUpload(); } else if (strcmp(el, "Part") == 0) { diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc index ad1b0a82c9b29..5e6f13aed0996 100644 --- a/src/rgw/rgw_op.cc +++ b/src/rgw/rgw_op.cc @@ -5887,11 +5887,16 @@ void RGWCompleteMultipart::execute(optional_yield y) } parts = static_cast(parser.find_first("CompleteMultipartUpload")); + if (!parts || parts->parts.empty()) { + parts = static_cast(parser.find_first("CompletedMultipartUpload")); + } + if (!parts || parts->parts.empty()) { op_ret = -ERR_MALFORMED_XML; return; } + if ((int)parts->parts.size() > s->cct->_conf->rgw_multipart_part_upload_limit) { op_ret = -ERANGE; From be0842f2dd5731f0f6eaab55471662dfd2865ccf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20H=C3=BCttner?= Date: Thu, 6 May 2021 12:18:00 -0400 Subject: [PATCH 0006/2866] rgw: RGWSwiftWebsiteHandler::is_web_dir checks empty subdir_name checking for empty name avoids later assertion in RGWObjectCtx::set_atomic Fixes: CVE-2021-3531 (cherry picked from commit f44a8ae8aa27ecef69528db9aec220f12492810e) Signed-off-by: Felix Huettner --- src/rgw/rgw_rest_swift.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/rgw/rgw_rest_swift.cc b/src/rgw/rgw_rest_swift.cc index e2a2db57d4ae5..1623dd07f3cf6 100644 --- a/src/rgw/rgw_rest_swift.cc +++ b/src/rgw/rgw_rest_swift.cc @@ -2551,6 +2551,9 @@ bool RGWSwiftWebsiteHandler::is_web_dir() const return false; } else if (subdir_name.back() == '/') { subdir_name.pop_back(); + if (subdir_name.empty()) { + return false; + } } std::unique_ptr obj = s->bucket->get_object(rgw_obj_key(std::move(subdir_name))); From 399a45e113322f3b2f8be9af83044c3aee8d96b9 Mon Sep 17 00:00:00 2001 From: Sunny Kumar Date: Wed, 2 Jun 2021 16:14:45 +0100 Subject: [PATCH 0007/2866] librbd/object_map: rbd diff between two snapshots lists entire image content Problem: rbd diff between two snapshots lists entire image content with `whole-object` switch. Solution: While computing resize diff during object map do not mark state as new data unless diff state is valid. Fixes: https://tracker.ceph.com/issues/50787 Signed-off-by: Sunny Kumar --- src/librbd/object_map/DiffRequest.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/librbd/object_map/DiffRequest.cc b/src/librbd/object_map/DiffRequest.cc index ca341f78c2f28..606d48bbf33c4 100644 --- a/src/librbd/object_map/DiffRequest.cc +++ b/src/librbd/object_map/DiffRequest.cc @@ -222,7 +222,9 @@ void DiffRequest::handle_load_object_map(int r) { uint8_t object_map_state = *it; if (object_map_state == OBJECT_NONEXISTENT) { *diff_it = DIFF_STATE_HOLE; - } else if (diff_from_start || object_map_state != OBJECT_EXISTS_CLEAN) { + } else if (diff_from_start || + (m_object_diff_state_valid && + object_map_state != OBJECT_EXISTS_CLEAN)) { *diff_it = DIFF_STATE_DATA_UPDATED; } else { *diff_it = DIFF_STATE_DATA; From 8d62c4adf74d2aee02f12bc1955490175b55bfda Mon Sep 17 00:00:00 2001 From: Sunny Kumar Date: Tue, 15 Jun 2021 14:19:44 +0100 Subject: [PATCH 0008/2866] test/librbd: add test condition for whole-object switch with fast diff enable This patch adds a test condition to test fast diff between two snapshots with whole-object switch. `TestMockObjectMapDiffRequest.EndDelta` expects diff state `DIFF_STATE_DATA_UPDATED` which is an incorrect expectation when diff is calculated between two snapshots. Fixes: https://tracker.ceph.com/issues/50787 Signed-off-by: Sunny Kumar --- src/test/librbd/object_map/test_mock_DiffRequest.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/librbd/object_map/test_mock_DiffRequest.cc b/src/test/librbd/object_map/test_mock_DiffRequest.cc index f90d488a555bb..c25ae4a95c5e6 100644 --- a/src/test/librbd/object_map/test_mock_DiffRequest.cc +++ b/src/test/librbd/object_map/test_mock_DiffRequest.cc @@ -212,7 +212,7 @@ TEST_F(TestMockObjectMapDiffRequest, IntermediateDelta) { BitVector<2> object_map_1; object_map_1.resize(object_count); - object_map_1[1] = OBJECT_EXISTS_CLEAN; + object_map_1[1] = OBJECT_EXISTS; object_map_1[2] = OBJECT_EXISTS_CLEAN; expect_load_map(mock_image_ctx, 1U, object_map_1, 0); @@ -281,7 +281,7 @@ TEST_F(TestMockObjectMapDiffRequest, EndDelta) { BitVector<2> expected_diff_state; expected_diff_state.resize(object_count); expected_diff_state[1] = DIFF_STATE_DATA; - expected_diff_state[2] = DIFF_STATE_DATA_UPDATED; + expected_diff_state[2] = DIFF_STATE_DATA; expected_diff_state[3] = DIFF_STATE_HOLE_UPDATED; ASSERT_EQ(expected_diff_state, m_object_diff_state); } From 7511f9f675ea4e43992605dc03109bc5f356a5e1 Mon Sep 17 00:00:00 2001 From: Peng Zhang Date: Fri, 2 Jul 2021 14:13:50 +0800 Subject: [PATCH 0009/2866] rgw: update last_added_entry when count == num_entries RGWRados::cls_bucket_list_unordered() will produce one redundent entry every time is_truncated is true.The issue could be easily reproduced when a bucket is filled with amounts of incomplete multipart upload. To be more specific, the number of incomplete multipart upload objects should be greater than 1100. Signed-off-by: Peng Zhang --- src/rgw/rgw_rados.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc index b62e3a8a6098f..6ea6b025d3c86 100644 --- a/src/rgw/rgw_rados.cc +++ b/src/rgw/rgw_rados.cc @@ -8780,6 +8780,7 @@ int RGWRados::cls_bucket_list_unordered(const DoutPrefixProvider *dpp, ent_list.emplace_back(std::move(dirent)); ++count; } else { + last_added_entry = dirent.key; *is_truncated = true; goto check_updates; } From f241a330dcb5968f9ec1de1a382572258cb6daac Mon Sep 17 00:00:00 2001 From: Lei Zhang <1091517373@qq.com> Date: Wed, 14 Jul 2021 17:30:48 +0800 Subject: [PATCH 0010/2866] rgw: RGWPostObj::execute() may lost data. Signed-off-by: Lei Zhang <1091517373@qq.com> --- src/rgw/rgw_op.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc index 167ad3593e446..b0945742c6faf 100644 --- a/src/rgw/rgw_op.cc +++ b/src/rgw/rgw_op.cc @@ -4284,6 +4284,9 @@ void RGWPostObj::execute(optional_yield y) hash.Update((const unsigned char *)data.c_str(), data.length()); op_ret = filter->process(std::move(data), ofs); + if (op_ret < 0) { + return; + } ofs += len; From 34082b7ee48a33e566348395395858e1e0db3013 Mon Sep 17 00:00:00 2001 From: Arthur Outhenin-Chalandre Date: Fri, 4 Jun 2021 18:29:37 +0200 Subject: [PATCH 0011/2866] rbd-mirror: fix mirror image removal Invoke ImageRemoveRequest instead of calling directly mirror_image_remove so that the MirrroringWatcher can pick up local image deletion. Fixes: https://tracker.ceph.com/issues/51031 Signed-off-by: Arthur Outhenin-Chalandre --- .../test_mock_TrashMoveRequest.cc | 59 +++++++++++++++---- .../image_deleter/TrashMoveRequest.cc | 12 ++-- 2 files changed, 54 insertions(+), 17 deletions(-) diff --git a/src/test/rbd_mirror/image_deleter/test_mock_TrashMoveRequest.cc b/src/test/rbd_mirror/image_deleter/test_mock_TrashMoveRequest.cc index c3702a9cc33bf..e1a52c87602c8 100644 --- a/src/test/rbd_mirror/image_deleter/test_mock_TrashMoveRequest.cc +++ b/src/test/rbd_mirror/image_deleter/test_mock_TrashMoveRequest.cc @@ -9,6 +9,7 @@ #include "librbd/TrashWatcher.h" #include "librbd/journal/ResetRequest.h" #include "librbd/mirror/GetInfoRequest.h" +#include "librbd/mirror/ImageRemoveRequest.h" #include "librbd/trash/MoveRequest.h" #include "tools/rbd_mirror/Threads.h" #include "tools/rbd_mirror/image_deleter/TrashMoveRequest.h" @@ -131,6 +132,37 @@ struct GetInfoRequest { GetInfoRequest* GetInfoRequest::s_instance = nullptr; +template<> +struct ImageRemoveRequest { + static ImageRemoveRequest* s_instance; + std::string global_image_id; + std::string image_id; + Context* on_finish; + + static ImageRemoveRequest *create(librados::IoCtx& io_ctx, + const std::string& global_image_id, + const std::string& image_id, + Context* on_finish) { + ceph_assert(s_instance != nullptr); + s_instance->global_image_id = global_image_id; + s_instance->image_id = image_id; + s_instance->on_finish = on_finish; + return s_instance; + } + + ImageRemoveRequest() { + ceph_assert(s_instance == nullptr); + s_instance = this; + } + ~ImageRemoveRequest() { + s_instance = nullptr; + } + + MOCK_METHOD0(send, void()); +}; + +ImageRemoveRequest* ImageRemoveRequest::s_instance = nullptr; + } // namespace mirror namespace trash { @@ -185,6 +217,7 @@ class TestMockImageDeleterTrashMoveRequest : public TestMockFixture { typedef TrashMoveRequest MockTrashMoveRequest; typedef librbd::journal::ResetRequest MockJournalResetRequest; typedef librbd::mirror::GetInfoRequest MockGetMirrorInfoRequest; + typedef librbd::mirror::ImageRemoveRequest MockImageRemoveRequest; typedef librbd::trash::MoveRequest MockLibrbdTrashMoveRequest; typedef librbd::TrashWatcher MockTrashWatcher; @@ -273,11 +306,12 @@ class TestMockImageDeleterTrashMoveRequest : public TestMockFixture { .WillOnce(Return(r)); } - void expect_mirror_image_remove(librados::IoCtx &ioctx, int r) { - EXPECT_CALL(get_mock_io_ctx(ioctx), - exec(StrEq("rbd_mirroring"), _, StrEq("rbd"), - StrEq("mirror_image_remove"), _, _, _, _)) - .WillOnce(Return(r)); + void expect_mirror_image_remove_request( + MockImageRemoveRequest& mock_image_remove_request, int r) { + EXPECT_CALL(mock_image_remove_request, send()) + .WillOnce(Invoke([this, &mock_image_remove_request, r]() { + m_threads->work_queue->queue(mock_image_remove_request.on_finish, r); + })); } void expect_journal_reset(MockJournalResetRequest& mock_journal_reset_request, @@ -355,7 +389,8 @@ TEST_F(TestMockImageDeleterTrashMoveRequest, SuccessJournal) { MockLibrbdTrashMoveRequest mock_librbd_trash_move_request; expect_trash_move(mock_librbd_trash_move_request, m_image_name, "image id", {}, 0); - expect_mirror_image_remove(m_local_io_ctx, 0); + MockImageRemoveRequest mock_image_remove_request; + expect_mirror_image_remove_request(mock_image_remove_request, 0); expect_close(mock_image_ctx, 0); @@ -396,7 +431,8 @@ TEST_F(TestMockImageDeleterTrashMoveRequest, SuccessSnapshot) { MockLibrbdTrashMoveRequest mock_librbd_trash_move_request; expect_trash_move(mock_librbd_trash_move_request, m_image_name, "image id", {}, 0); - expect_mirror_image_remove(m_local_io_ctx, 0); + MockImageRemoveRequest mock_image_remove_request; + expect_mirror_image_remove_request(mock_image_remove_request, 0); expect_close(mock_image_ctx, 0); @@ -742,7 +778,8 @@ TEST_F(TestMockImageDeleterTrashMoveRequest, RemoveMirrorImageError) { MockLibrbdTrashMoveRequest mock_librbd_trash_move_request; expect_trash_move(mock_librbd_trash_move_request, m_image_name, "image id", {}, 0); - expect_mirror_image_remove(m_local_io_ctx, -EINVAL); + MockImageRemoveRequest mock_image_remove_request; + expect_mirror_image_remove_request(mock_image_remove_request, -EINVAL); expect_close(mock_image_ctx, 0); @@ -791,7 +828,8 @@ TEST_F(TestMockImageDeleterTrashMoveRequest, CloseImageError) { MockLibrbdTrashMoveRequest mock_librbd_trash_move_request; expect_trash_move(mock_librbd_trash_move_request, m_image_name, "image id", {}, 0); - expect_mirror_image_remove(m_local_io_ctx, 0); + MockImageRemoveRequest mock_image_remove_request; + expect_mirror_image_remove_request(mock_image_remove_request, 0); expect_close(mock_image_ctx, -EINVAL); @@ -842,7 +880,8 @@ TEST_F(TestMockImageDeleterTrashMoveRequest, DelayedDelation) { expect_trash_move(mock_librbd_trash_move_request, m_image_name, "image id", 600, 0); - expect_mirror_image_remove(m_local_io_ctx, 0); + MockImageRemoveRequest mock_image_remove_request; + expect_mirror_image_remove_request(mock_image_remove_request, 0); expect_close(mock_image_ctx, 0); MockTrashWatcher mock_trash_watcher; diff --git a/src/tools/rbd_mirror/image_deleter/TrashMoveRequest.cc b/src/tools/rbd_mirror/image_deleter/TrashMoveRequest.cc index 7f718cb9c66e2..234bb69129666 100644 --- a/src/tools/rbd_mirror/image_deleter/TrashMoveRequest.cc +++ b/src/tools/rbd_mirror/image_deleter/TrashMoveRequest.cc @@ -15,6 +15,7 @@ #include "librbd/Utils.h" #include "librbd/asio/ContextWQ.h" #include "librbd/journal/ResetRequest.h" +#include "librbd/mirror/ImageRemoveRequest.h" #include "librbd/mirror/GetInfoRequest.h" #include "librbd/trash/MoveRequest.h" #include "tools/rbd_mirror/image_deleter/Types.h" @@ -315,15 +316,12 @@ template void TrashMoveRequest::remove_mirror_image() { dout(10) << dendl; - librados::ObjectWriteOperation op; - librbd::cls_client::mirror_image_remove(&op, m_image_id); - - auto aio_comp = create_rados_callback< + auto ctx = create_context_callback< TrashMoveRequest, &TrashMoveRequest::handle_remove_mirror_image>(this); - int r = m_io_ctx.aio_operate(RBD_MIRRORING, aio_comp, &op); - ceph_assert(r == 0); - aio_comp->release(); + auto req = librbd::mirror::ImageRemoveRequest::create( + m_io_ctx, m_global_image_id, m_image_id, ctx); + req->send(); } template From 0e147f779d73d0688c2b89428db6012ed1560f20 Mon Sep 17 00:00:00 2001 From: Arthur Outhenin-Chalandre Date: Mon, 7 Jun 2021 12:53:48 +0200 Subject: [PATCH 0012/2866] cls/rbd: add mirror_image_status_remove on client Signed-off-by: Arthur Outhenin-Chalandre --- src/cls/rbd/cls_rbd.cc | 1 - src/cls/rbd/cls_rbd_client.cc | 14 ++++++++++++++ src/cls/rbd/cls_rbd_client.h | 4 ++++ src/test/cls_rbd/test_cls_rbd.cc | 5 +++++ 4 files changed, 23 insertions(+), 1 deletion(-) diff --git a/src/cls/rbd/cls_rbd.cc b/src/cls/rbd/cls_rbd.cc index e3e05d85ed500..ffc19267cf400 100644 --- a/src/cls/rbd/cls_rbd.cc +++ b/src/cls/rbd/cls_rbd.cc @@ -6366,7 +6366,6 @@ int mirror_image_status_set(cls_method_context_t hctx, bufferlist *in, * Output: * @returns 0 on success, negative error code on failure * - * NOTE: deprecated - remove this method after Octopus is unsupported */ int mirror_image_status_remove(cls_method_context_t hctx, bufferlist *in, bufferlist *out) { diff --git a/src/cls/rbd/cls_rbd_client.cc b/src/cls/rbd/cls_rbd_client.cc index fee3ac8923cc3..2f1f37eaa9f72 100644 --- a/src/cls/rbd/cls_rbd_client.cc +++ b/src/cls/rbd/cls_rbd_client.cc @@ -2310,6 +2310,20 @@ int mirror_image_status_get_summary_finish( return 0; } +int mirror_image_status_remove(librados::IoCtx *ioctx, + const std::string &global_image_id) { + librados::ObjectWriteOperation op; + mirror_image_status_remove(&op, global_image_id); + return ioctx->operate(RBD_MIRRORING, &op); +} + +void mirror_image_status_remove(librados::ObjectWriteOperation *op, + const std::string &global_image_id) { + bufferlist bl; + encode(global_image_id, bl); + op->exec("rbd", "mirror_image_status_remove", bl); +} + int mirror_image_status_remove_down(librados::IoCtx *ioctx) { librados::ObjectWriteOperation op; mirror_image_status_remove_down(&op); diff --git a/src/cls/rbd/cls_rbd_client.h b/src/cls/rbd/cls_rbd_client.h index ef2b05fd8449e..38098805e98c4 100644 --- a/src/cls/rbd/cls_rbd_client.h +++ b/src/cls/rbd/cls_rbd_client.h @@ -478,6 +478,10 @@ void mirror_image_status_get_summary_start( int mirror_image_status_get_summary_finish( ceph::buffer::list::const_iterator *iter, std::map *states); +int mirror_image_status_remove(librados::IoCtx *ioctx, + const std::string &global_image_id); +void mirror_image_status_remove(librados::ObjectWriteOperation *op, + const std::string &global_image_id); int mirror_image_status_remove_down(librados::IoCtx *ioctx); void mirror_image_status_remove_down(librados::ObjectWriteOperation *op); diff --git a/src/test/cls_rbd/test_cls_rbd.cc b/src/test/cls_rbd/test_cls_rbd.cc index d01e04704ff64..5b33597455643 100644 --- a/src/test/cls_rbd/test_cls_rbd.cc +++ b/src/test/cls_rbd/test_cls_rbd.cc @@ -1889,6 +1889,11 @@ TEST_F(TestClsRbd, mirror_image_status) { ASSERT_EQ(1U, states.size()); ASSERT_EQ(3, states[cls::rbd::MIRROR_IMAGE_STATUS_STATE_UNKNOWN]); + // Test remove of status + ASSERT_EQ(0, mirror_image_status_set(&ioctx, "uuid1", status1)); + ASSERT_EQ(0, mirror_image_status_remove(&ioctx, "uuid1")); + ASSERT_EQ(-ENOENT, mirror_image_instance_get(&ioctx, "uuid1", &read_instance)); + // Test statuses are not down after watcher is started ASSERT_EQ(0, mirror_image_status_set(&ioctx, "uuid1", status1)); From a538c5d279c90397d375668baddd65776d2462b0 Mon Sep 17 00:00:00 2001 From: Arthur Outhenin-Chalandre Date: Mon, 7 Jun 2021 14:58:03 +0200 Subject: [PATCH 0013/2866] rbd-mirror: add mirror status removal on ImageReplayer shutdown In a scenario where you have rbd-mirror daemons on both clusters. The rbd-mirror daemon on the primary site will not properly cleanup his status on image removal. This commit add a path for direct removal at the shut_down of the ImageReplayer to properly cleanup the metadata. Signed-off-by: Arthur Outhenin-Chalandre --- .../rbd_mirror/test_mock_ImageReplayer.cc | 5 +- .../test_mock_MirrorStatusUpdater.cc | 138 +++++++++++++++++- src/tools/rbd_mirror/ImageReplayer.cc | 65 +++++++-- src/tools/rbd_mirror/ImageReplayer.h | 5 + src/tools/rbd_mirror/MirrorStatusUpdater.cc | 38 ++++- src/tools/rbd_mirror/MirrorStatusUpdater.h | 5 +- 6 files changed, 228 insertions(+), 28 deletions(-) diff --git a/src/test/rbd_mirror/test_mock_ImageReplayer.cc b/src/test/rbd_mirror/test_mock_ImageReplayer.cc index 20693e3223b7b..57c2639a00ced 100644 --- a/src/test/rbd_mirror/test_mock_ImageReplayer.cc +++ b/src/test/rbd_mirror/test_mock_ImageReplayer.cc @@ -63,7 +63,10 @@ struct MirrorStatusUpdater { MOCK_METHOD3(set_mirror_image_status, void(const std::string&, const cls::rbd::MirrorImageSiteStatus&, bool)); - MOCK_METHOD2(remove_mirror_image_status, void(const std::string&, Context*)); + MOCK_METHOD2(remove_refresh_mirror_image_status, void(const std::string&, + Context*)); + MOCK_METHOD3(remove_mirror_image_status, void(const std::string&, bool, + Context*)); }; template <> diff --git a/src/test/rbd_mirror/test_mock_MirrorStatusUpdater.cc b/src/test/rbd_mirror/test_mock_MirrorStatusUpdater.cc index 057159ceb4b88..ac8009df1379e 100644 --- a/src/test/rbd_mirror/test_mock_MirrorStatusUpdater.cc +++ b/src/test/rbd_mirror/test_mock_MirrorStatusUpdater.cc @@ -186,6 +186,38 @@ class TestMockMirrorStatusUpdater : public TestMockFixture { } } + void expect_mirror_status_remove(const std::string& global_image_id, int r) { + EXPECT_CALL(*m_mock_local_io_ctx, + exec(RBD_MIRRORING, _, StrEq("rbd"), + StrEq("mirror_image_status_remove"), _, _, _, _)) + .WillOnce(WithArg<4>(Invoke( + [r, global_image_id](bufferlist& in_bl) { + auto bl_it = in_bl.cbegin(); + std::string decode_global_image_id; + decode(decode_global_image_id, bl_it); + EXPECT_EQ(global_image_id, decode_global_image_id); + + return r; + }))); + } + + void expect_mirror_status_removes(const std::set& mirror_images, + int r) { + EXPECT_CALL(*m_mock_local_io_ctx, aio_operate(_, _, _, _, _)) + .WillOnce(Invoke([this](auto&&... args) { + int r = m_mock_local_io_ctx->do_aio_operate(decltype(args)(args)...); + m_mock_local_io_ctx->aio_flush(); + return r; + })); + + for (auto global_image_id : mirror_images) { + expect_mirror_status_remove(global_image_id, r); + if (r < 0) { + break; + } + } + } + void fire_timer_event(Context** timer_event, Context** update_task) { expect_timer_add_event(timer_event); @@ -385,6 +417,78 @@ TEST_F(TestMockMirrorStatusUpdater, OverwriteStatus) { *mock_mirror_status_watcher); } +TEST_F(TestMockMirrorStatusUpdater, RemoveStatus) { + MockMirrorStatusUpdater mock_mirror_status_updater(m_local_io_ctx, + m_mock_threads, ""); + MockMirrorStatusWatcher* mock_mirror_status_watcher = + new MockMirrorStatusWatcher(); + + InSequence seq; + + Context* timer_event = nullptr; + init_mirror_status_updater(mock_mirror_status_updater, + *mock_mirror_status_watcher, &timer_event); + + C_SaferCond ctx; + mock_mirror_status_updater.set_mirror_image_status("1", {}, false); + expect_work_queue(false); + mock_mirror_status_updater.remove_mirror_image_status("1", false, &ctx); + ASSERT_EQ(0, ctx.wait()); + + Context* update_task = nullptr; + fire_timer_event(&timer_event, &update_task); + + C_SaferCond remove_flush_ctx; + EXPECT_CALL(*m_mock_local_io_ctx, aio_operate(_, _, _, _, _)) + .WillOnce(Invoke([this, &remove_flush_ctx](auto&&... args) { + int r = m_mock_local_io_ctx->do_aio_operate(decltype(args)(args)...); + m_mock_local_io_ctx->aio_flush(); + remove_flush_ctx.complete(r); + return r; + })); + expect_mirror_status_remove("1", 0); + update_task->complete(0); + ASSERT_EQ(0, remove_flush_ctx.wait()); + + shut_down_mirror_status_updater(mock_mirror_status_updater, + *mock_mirror_status_watcher); +} + +TEST_F(TestMockMirrorStatusUpdater, OverwriteRemoveStatus) { + MockMirrorStatusUpdater mock_mirror_status_updater(m_local_io_ctx, + m_mock_threads, ""); + MockMirrorStatusWatcher* mock_mirror_status_watcher = + new MockMirrorStatusWatcher(); + + InSequence seq; + + Context* timer_event = nullptr; + init_mirror_status_updater(mock_mirror_status_updater, + *mock_mirror_status_watcher, &timer_event); + + C_SaferCond ctx; + mock_mirror_status_updater.set_mirror_image_status("1", {}, false); + expect_work_queue(false); + mock_mirror_status_updater.remove_mirror_image_status("1", false, &ctx); + ASSERT_EQ(0, ctx.wait()); + mock_mirror_status_updater.set_mirror_image_status( + "1", {"", cls::rbd::MIRROR_IMAGE_STATUS_STATE_REPLAYING, "description"}, + false); + + + Context* update_task = nullptr; + fire_timer_event(&timer_event, &update_task); + + expect_mirror_status_update( + {{"1", cls::rbd::MirrorImageSiteStatus{ + "", cls::rbd::MIRROR_IMAGE_STATUS_STATE_REPLAYING, "description"}}}, + "", 0); + update_task->complete(0); + + shut_down_mirror_status_updater(mock_mirror_status_updater, + *mock_mirror_status_watcher); +} + TEST_F(TestMockMirrorStatusUpdater, OverwriteStatusInFlight) { MockMirrorStatusUpdater mock_mirror_status_updater(m_local_io_ctx, m_mock_threads, ""); @@ -447,7 +551,32 @@ TEST_F(TestMockMirrorStatusUpdater, ImmediateUpdate) { *mock_mirror_status_watcher); } -TEST_F(TestMockMirrorStatusUpdater, RemoveIdleStatus) { +TEST_F(TestMockMirrorStatusUpdater, RemoveImmediateUpdate) { + MockMirrorStatusUpdater mock_mirror_status_updater(m_local_io_ctx, + m_mock_threads, ""); + MockMirrorStatusWatcher* mock_mirror_status_watcher = + new MockMirrorStatusWatcher(); + + InSequence seq; + + Context* timer_event = nullptr; + init_mirror_status_updater(mock_mirror_status_updater, + *mock_mirror_status_watcher, &timer_event); + + mock_mirror_status_updater.set_mirror_image_status("1", {}, false); + + C_SaferCond ctx; + expect_work_queue(true); + expect_work_queue(true); + expect_mirror_status_removes({"1"}, 0); + mock_mirror_status_updater.remove_mirror_image_status("1", true, &ctx); + ASSERT_EQ(0, ctx.wait()); + + shut_down_mirror_status_updater(mock_mirror_status_updater, + *mock_mirror_status_watcher); +} + +TEST_F(TestMockMirrorStatusUpdater, RemoveRefreshIdleStatus) { MockMirrorStatusUpdater mock_mirror_status_updater(m_local_io_ctx, m_mock_threads, ""); MockMirrorStatusWatcher* mock_mirror_status_watcher = @@ -463,14 +592,14 @@ TEST_F(TestMockMirrorStatusUpdater, RemoveIdleStatus) { C_SaferCond ctx; expect_work_queue(true); - mock_mirror_status_updater.remove_mirror_image_status("1", &ctx); + mock_mirror_status_updater.remove_refresh_mirror_image_status("1", &ctx); ASSERT_EQ(0, ctx.wait()); shut_down_mirror_status_updater(mock_mirror_status_updater, *mock_mirror_status_watcher); } -TEST_F(TestMockMirrorStatusUpdater, RemoveInFlightStatus) { +TEST_F(TestMockMirrorStatusUpdater, RemoveRefreshInFlightStatus) { MockMirrorStatusUpdater mock_mirror_status_updater(m_local_io_ctx, m_mock_threads, ""); MockMirrorStatusWatcher* mock_mirror_status_watcher = @@ -491,7 +620,8 @@ TEST_F(TestMockMirrorStatusUpdater, RemoveInFlightStatus) { EXPECT_CALL(*m_mock_local_io_ctx, aio_operate(_, _, _, _, _)) .WillOnce(Invoke( [this, &mock_mirror_status_updater, &on_removed](auto&&... args) { - mock_mirror_status_updater.remove_mirror_image_status("1", &on_removed); + mock_mirror_status_updater.remove_refresh_mirror_image_status( + "1", &on_removed); int r = m_mock_local_io_ctx->do_aio_operate(decltype(args)(args)...); m_mock_local_io_ctx->aio_flush(); diff --git a/src/tools/rbd_mirror/ImageReplayer.cc b/src/tools/rbd_mirror/ImageReplayer.cc index f5cd65fe821ec..9546d09fce1a2 100644 --- a/src/tools/rbd_mirror/ImageReplayer.cc +++ b/src/tools/rbd_mirror/ImageReplayer.cc @@ -312,6 +312,7 @@ void ImageReplayer::start(Context *on_finish, bool manual, bool restart) m_manual_stop = false; m_delete_requested = false; m_restart_requested = false; + m_status_removed = false; if (on_finish != nullptr) { ceph_assert(m_on_start_finish == nullptr); @@ -930,6 +931,7 @@ void ImageReplayer::handle_shut_down(int r) { dout(0) << "remote image no longer exists: scheduling deletion" << dendl; unregister_asok_hook = true; std::swap(delete_requested, m_delete_requested); + m_delete_in_progress = true; } std::swap(resync_requested, m_resync_requested); @@ -965,23 +967,12 @@ void ImageReplayer::handle_shut_down(int r) { return; } - if (m_local_status_updater->exists(m_global_image_id)) { - dout(15) << "removing local mirror image status" << dendl; - auto ctx = new LambdaContext([this, r](int) { - handle_shut_down(r); - }); - m_local_status_updater->remove_mirror_image_status(m_global_image_id, ctx); - return; - } - - if (m_remote_image_peer.mirror_status_updater != nullptr && - m_remote_image_peer.mirror_status_updater->exists(m_global_image_id)) { - dout(15) << "removing remote mirror image status" << dendl; + if (!m_status_removed) { auto ctx = new LambdaContext([this, r](int) { - handle_shut_down(r); - }); - m_remote_image_peer.mirror_status_updater->remove_mirror_image_status( - m_global_image_id, ctx); + m_status_removed = true; + handle_shut_down(r); + }); + remove_image_status(m_delete_in_progress, ctx); return; } @@ -1137,6 +1128,48 @@ void ImageReplayer::reregister_admin_socket_hook() { register_admin_socket_hook(); } +template +void ImageReplayer::remove_image_status(bool force, Context *on_finish) +{ + auto ctx = new LambdaContext([this, force, on_finish](int) { + remove_image_status_remote(force, on_finish); + }); + + if (m_local_status_updater->exists(m_global_image_id)) { + dout(15) << "removing local mirror image status" << dendl; + if (force) { + m_local_status_updater->remove_mirror_image_status( + m_global_image_id, true, ctx); + } else { + m_local_status_updater->remove_refresh_mirror_image_status( + m_global_image_id, ctx); + } + return; + } + + ctx->complete(0); +} + +template +void ImageReplayer::remove_image_status_remote(bool force, Context *on_finish) +{ + if (m_remote_image_peer.mirror_status_updater != nullptr && + m_remote_image_peer.mirror_status_updater->exists(m_global_image_id)) { + dout(15) << "removing remote mirror image status" << dendl; + if (force) { + m_remote_image_peer.mirror_status_updater->remove_mirror_image_status( + m_global_image_id, true, on_finish); + } else { + m_remote_image_peer.mirror_status_updater->remove_refresh_mirror_image_status( + m_global_image_id, on_finish); + } + return; + } + if (on_finish) { + on_finish->complete(0); + } +} + template std::ostream &operator<<(std::ostream &os, const ImageReplayer &replayer) { diff --git a/src/tools/rbd_mirror/ImageReplayer.h b/src/tools/rbd_mirror/ImageReplayer.h index 47f221278706b..feb5ae546a0e5 100644 --- a/src/tools/rbd_mirror/ImageReplayer.h +++ b/src/tools/rbd_mirror/ImageReplayer.h @@ -204,10 +204,13 @@ class ImageReplayer { BootstrapProgressContext m_progress_cxt; bool m_finished = false; + bool m_delete_in_progress = false; bool m_delete_requested = false; bool m_resync_requested = false; bool m_restart_requested = false; + bool m_status_removed = false; + image_replayer::StateBuilder* m_state_builder = nullptr; image_replayer::Replayer* m_replayer = nullptr; ReplayerListener* m_replayer_listener = nullptr; @@ -258,6 +261,8 @@ class ImageReplayer { void register_admin_socket_hook(); void unregister_admin_socket_hook(); void reregister_admin_socket_hook(); + void remove_image_status(bool force, Context *on_finish); + void remove_image_status_remote(bool force, Context *on_finish); }; diff --git a/src/tools/rbd_mirror/MirrorStatusUpdater.cc b/src/tools/rbd_mirror/MirrorStatusUpdater.cc index 0530db601f3bc..257cb1df27cf7 100644 --- a/src/tools/rbd_mirror/MirrorStatusUpdater.cc +++ b/src/tools/rbd_mirror/MirrorStatusUpdater.cc @@ -189,18 +189,33 @@ void MirrorStatusUpdater::set_mirror_image_status( } } +template +void MirrorStatusUpdater::remove_refresh_mirror_image_status( + const std::string& global_image_id, + Context* on_finish) { + if (try_remove_mirror_image_status(global_image_id, false, false, + on_finish)) { + m_threads->work_queue->queue(on_finish, 0); + } +} + template void MirrorStatusUpdater::remove_mirror_image_status( - const std::string& global_image_id, Context* on_finish) { - if (try_remove_mirror_image_status(global_image_id, on_finish)) { + const std::string& global_image_id, bool immediate_update, + Context* on_finish) { + if (try_remove_mirror_image_status(global_image_id, true, immediate_update, + on_finish)) { m_threads->work_queue->queue(on_finish, 0); } } template bool MirrorStatusUpdater::try_remove_mirror_image_status( - const std::string& global_image_id, Context* on_finish) { - dout(15) << "global_image_id=" << global_image_id << dendl; + const std::string& global_image_id, bool queue_update, + bool immediate_update, Context* on_finish) { + dout(15) << "global_image_id=" << global_image_id << ", " + << "queue_update=" << queue_update << ", " + << "immediate_update=" << immediate_update << dendl; std::unique_lock locker(m_lock); if ((m_update_in_flight && @@ -209,8 +224,10 @@ bool MirrorStatusUpdater::try_remove_mirror_image_status( m_update_global_image_ids.count(global_image_id) > 0)) { // if update is scheduled/in-progress, wait for it to complete on_finish = new LambdaContext( - [this, global_image_id, on_finish](int r) { - if (try_remove_mirror_image_status(global_image_id, on_finish)) { + [this, global_image_id, queue_update, immediate_update, + on_finish](int r) { + if (try_remove_mirror_image_status(global_image_id, queue_update, + immediate_update, on_finish)) { on_finish->complete(0); } }); @@ -219,6 +236,13 @@ bool MirrorStatusUpdater::try_remove_mirror_image_status( } m_global_image_status.erase(global_image_id); + if (queue_update) { + m_update_global_image_ids.insert(global_image_id); + if (immediate_update) { + queue_update_task(std::move(locker)); + } + } + return true; } @@ -314,6 +338,8 @@ void MirrorStatusUpdater::update_task(int r) { auto status_it = global_image_status.find(global_image_id); if (status_it == global_image_status.end()) { + librbd::cls_client::mirror_image_status_remove(&op, global_image_id); + ++op_count; continue; } diff --git a/src/tools/rbd_mirror/MirrorStatusUpdater.h b/src/tools/rbd_mirror/MirrorStatusUpdater.h index 60ae68ce2c7cd..783b818fc56e8 100644 --- a/src/tools/rbd_mirror/MirrorStatusUpdater.h +++ b/src/tools/rbd_mirror/MirrorStatusUpdater.h @@ -44,7 +44,9 @@ class MirrorStatusUpdater { const cls::rbd::MirrorImageSiteStatus& mirror_image_site_status, bool immediate_update); void remove_mirror_image_status(const std::string& global_image_id, - Context* on_finish); + bool immediate_update, Context* on_finish); + void remove_refresh_mirror_image_status(const std::string& global_image_id, + Context* on_finish); private: /** @@ -90,6 +92,7 @@ class MirrorStatusUpdater { GlobalImageIds m_updating_global_image_ids; bool try_remove_mirror_image_status(const std::string& global_image_id, + bool queue_update, bool immediate_update, Context* on_finish); void init_mirror_status_watcher(Context* on_finish); From dd425ed5aad1c60304b5097016e2ac0a0a7db883 Mon Sep 17 00:00:00 2001 From: "cao.leilc" Date: Fri, 13 Aug 2021 10:00:48 +0800 Subject: [PATCH 0014/2866] rgw : add return when access key exist in subuser create Fixes: https://tracker.ceph.com/issues/51559 Signed-off-by: caolei --- src/rgw/rgw_user.cc | 10 ++++++++-- src/rgw/rgw_user.h | 6 ++++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/rgw/rgw_user.cc b/src/rgw/rgw_user.cc index 9d45170a54c7a..a627bdd9c02c6 100644 --- a/src/rgw/rgw_user.cc +++ b/src/rgw/rgw_user.cc @@ -636,8 +636,9 @@ int RGWAccessKeyPool::check_op(RGWUserAdminOpState& op_state, // don't check for secret key because we may be doing a removal - check_existing_key(op_state); - + if (check_existing_key(op_state)) { + op_state.set_access_key_exist(); + } return 0; } @@ -1180,6 +1181,11 @@ int RGWSubUserPool::add(const DoutPrefixProvider *dpp, RGWUserAdminOpState& op_s return ret; } + if (op_state.get_access_key_exist()) { + set_err_msg(err_msg, "cannot create existing key"); + return -ERR_KEY_EXIST; + } + if (key_type == KEY_TYPE_S3 && op_state.get_access_key().empty()) { op_state.set_gen_access(); } diff --git a/src/rgw/rgw_user.h b/src/rgw/rgw_user.h index 542efc2092561..b40abc6f19b4d 100644 --- a/src/rgw/rgw_user.h +++ b/src/rgw/rgw_user.h @@ -121,6 +121,7 @@ struct RGWUserAdminOpState { std::string id; // access key std::string key; // secret key int32_t key_type{-1}; + bool access_key_exist = false; std::set mfa_ids; @@ -254,6 +255,10 @@ struct RGWUserAdminOpState { type_specified = true; } + void set_access_key_exist() { + access_key_exist = true; + } + void set_suspension(__u8 is_suspended) { suspended = is_suspended; suspension_op = true; @@ -372,6 +377,7 @@ struct RGWUserAdminOpState { void set_generate_subuser(bool flag) { gen_subuser = flag; } __u8 get_suspension_status() { return suspended; } int32_t get_key_type() {return key_type; } + bool get_access_key_exist() {return access_key_exist; } uint32_t get_subuser_perm() { return perm_mask; } int32_t get_max_buckets() { return max_buckets; } uint32_t get_op_mask() { return op_mask; } From 04b8721dd4b3f02abcb9fce7fcfbcbefb44fbe85 Mon Sep 17 00:00:00 2001 From: Xiubo Li Date: Sat, 15 Aug 2020 07:51:59 +0800 Subject: [PATCH 0015/2866] client: minor cleanup to Client related code Remove the extra while spaces to clean up the code style. Signed-off-by: Xiubo Li --- src/client/Client.cc | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/src/client/Client.cc b/src/client/Client.cc index ddf42266a5c78..dad50d2cb0df3 100644 --- a/src/client/Client.cc +++ b/src/client/Client.cc @@ -3585,7 +3585,7 @@ void Client::send_cap(Inode *in, MetaSession *session, Cap *cap, if (flush) follows = in->snaprealm->get_snap_context().seq; - + auto m = make_message(op, in->ino, 0, @@ -3604,14 +3604,14 @@ void Client::send_cap(Inode *in, MetaSession *session, Cap *cap, m->head.uid = in->uid; m->head.gid = in->gid; m->head.mode = in->mode; - + m->head.nlink = in->nlink; - + if (flush & CEPH_CAP_XATTR_EXCL) { encode(in->xattrs, m->xattrbl); m->head.xattr_version = in->xattr_version; } - + m->size = in->size; m->max_size = in->max_size; m->truncate_seq = in->truncate_seq; @@ -3761,11 +3761,7 @@ void Client::check_caps(Inode *in, unsigned flags) used &= ~(CEPH_CAP_FILE_CACHE | CEPH_CAP_FILE_LAZYIO); } - - for (auto &p : in->caps) { - mds_rank_t mds = p.first; - Cap &cap = p.second; - + for (auto &[mds, cap] : in->caps) { MetaSession *session = &mds_sessions.at(mds); cap_used = used; @@ -3773,7 +3769,7 @@ void Client::check_caps(Inode *in, unsigned flags) cap_used &= ~in->auth_cap->issued; revoking = cap.implemented & ~cap.issued; - + ldout(cct, 10) << " cap mds." << mds << " issued " << ccap_string(cap.issued) << " implemented " << ccap_string(cap.implemented) From 60338473403de8c7dcbbdce24808821dd3260c6f Mon Sep 17 00:00:00 2001 From: Xiubo Li Date: Fri, 14 Aug 2020 11:38:03 +0800 Subject: [PATCH 0016/2866] client: minor cleanup to MetaRequest class To make the code more readable and there is no need to care about the members order when initlizeing them. Signed-off-by: Xiubo Li --- src/client/MetaRequest.h | 67 ++++++++++++++++------------------------ 1 file changed, 27 insertions(+), 40 deletions(-) diff --git a/src/client/MetaRequest.h b/src/client/MetaRequest.h index fa97bf0f31626..630f5ee15bc87 100644 --- a/src/client/MetaRequest.h +++ b/src/client/MetaRequest.h @@ -21,73 +21,60 @@ class dir_result_t; struct MetaRequest { private: InodeRef _inode, _old_inode, _other_inode; - Dentry *_dentry; //associated with path - Dentry *_old_dentry; //associated with path2 - int abort_rc; + Dentry *_dentry = NULL; //associated with path + Dentry *_old_dentry = NULL; //associated with path2 + int abort_rc = 0; public: - uint64_t tid; + uint64_t tid = 0; utime_t op_stamp; ceph_mds_request_head head; filepath path, path2; std::string alternate_name; bufferlist data; - int inode_drop; //the inode caps this operation will drop - int inode_unless; //unless we have these caps already - int old_inode_drop, old_inode_unless; - int dentry_drop, dentry_unless; - int old_dentry_drop, old_dentry_unless; - int other_inode_drop, other_inode_unless; + int inode_drop = 0; //the inode caps this operation will drop + int inode_unless = 0; //unless we have these caps already + int old_inode_drop = 0, old_inode_unless = 0; + int dentry_drop = 0, dentry_unless = 0; + int old_dentry_drop = 0, old_dentry_unless = 0; + int other_inode_drop = 0, other_inode_unless = 0; std::vector cap_releases; - int regetattr_mask; // getattr mask if i need to re-stat after a traceless reply + int regetattr_mask = 0; // getattr mask if i need to re-stat after a traceless reply utime_t sent_stamp; - mds_rank_t mds; // who i am asking - mds_rank_t resend_mds; // someone wants you to (re)send the request here - bool send_to_auth; // must send to auth mds - __u32 sent_on_mseq; // mseq at last submission of this request - int num_fwd; // # of times i've been forwarded - int retry_attempt; + mds_rank_t mds = -1; // who i am asking + mds_rank_t resend_mds = -1; // someone wants you to (re)send the request here + bool send_to_auth = false; // must send to auth mds + __u32 sent_on_mseq = 0; // mseq at last submission of this request + int num_fwd = 0; // # of times i've been forwarded + int retry_attempt = 0; std::atomic ref = { 1 }; - ceph::cref_t reply; // the reply - bool kick; - bool success; - + ceph::cref_t reply = NULL; // the reply + bool kick = false; + bool success = false; + // readdir result - dir_result_t *dirp; + dir_result_t *dirp = NULL; //possible responses - bool got_unsafe; + bool got_unsafe = false; xlist::item item; xlist::item unsafe_item; xlist::item unsafe_dir_item; xlist::item unsafe_target_item; - ceph::condition_variable *caller_cond; // who to take up - ceph::condition_variable *dispatch_cond; // who to kick back + ceph::condition_variable *caller_cond = NULL; // who to take up + ceph::condition_variable *dispatch_cond = NULL; // who to kick back std::list waitfor_safe; InodeRef target; UserPerm perms; explicit MetaRequest(int op) : - _dentry(NULL), _old_dentry(NULL), abort_rc(0), - tid(0), - inode_drop(0), inode_unless(0), - old_inode_drop(0), old_inode_unless(0), - dentry_drop(0), dentry_unless(0), - old_dentry_drop(0), old_dentry_unless(0), - other_inode_drop(0), other_inode_unless(0), - regetattr_mask(0), - mds(-1), resend_mds(-1), send_to_auth(false), sent_on_mseq(0), - num_fwd(0), retry_attempt(0), - reply(0), - kick(false), success(false), dirp(NULL), - got_unsafe(false), item(this), unsafe_item(this), - unsafe_dir_item(this), unsafe_target_item(this), - caller_cond(0), dispatch_cond(0) { + item(this), unsafe_item(this), unsafe_dir_item(this), + unsafe_target_item(this) { memset(&head, 0, sizeof(head)); head.op = op; } From 5ce296c23a6ef2225345c9114cac3976f2898949 Mon Sep 17 00:00:00 2001 From: Xiubo Li Date: Tue, 9 Feb 2021 11:07:44 +0800 Subject: [PATCH 0017/2866] client: minor cleanup to Fh related code To make the code more readable and there is no need to care about the members order when initlizeing them. Signed-off-by: Xiubo Li --- src/client/Fh.cc | 4 ++-- src/client/Fh.h | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/client/Fh.cc b/src/client/Fh.cc index 62bd261404e21..e72390df714c6 100644 --- a/src/client/Fh.cc +++ b/src/client/Fh.cc @@ -19,8 +19,8 @@ #include "Fh.h" Fh::Fh(InodeRef in, int flags, int cmode, uint64_t _gen, const UserPerm &perms) : - inode(in), _ref(1), pos(0), mds(0), mode(cmode), gen(_gen), flags(flags), - pos_locked(false), actor_perms(perms), readahead() + inode(in), mode(cmode), gen(_gen), flags(flags), actor_perms(perms), + readahead() { inode->add_fh(this); } diff --git a/src/client/Fh.h b/src/client/Fh.h index c3355ba6c5266..3448f5b6ac88e 100644 --- a/src/client/Fh.h +++ b/src/client/Fh.h @@ -13,14 +13,14 @@ class Inode; struct Fh { InodeRef inode; - int _ref; - loff_t pos; - int mds; // have to talk to mds we opened with (for now) + int _ref = 1; + loff_t pos = 0; + int mds = 0; // have to talk to mds we opened with (for now) int mode; // the mode i opened the file with uint64_t gen; int flags; - bool pos_locked; // pos is currently in use + bool pos_locked = false; // pos is currently in use std::list pos_waiters; // waiters for pos UserPerm actor_perms; // perms I opened the file with From 730dd8a87a3dbeba348dd419869ac110c57ed47a Mon Sep 17 00:00:00 2001 From: Xiubo Li Date: Mon, 17 Aug 2020 22:10:42 +0800 Subject: [PATCH 0018/2866] client: switch to use shared_ptr for mds_sessions To break the `client_lock`, when we need to unlock it and lock it later, during the gap we must make sure that the `session` won't be released. Signed-off-by: Xiubo Li --- src/client/Client.cc | 198 +++++++++++++++++++-------------------- src/client/Client.h | 8 +- src/client/MetaSession.h | 1 + 3 files changed, 103 insertions(+), 104 deletions(-) diff --git a/src/client/Client.cc b/src/client/Client.cc index dad50d2cb0df3..00f38421f4b03 100644 --- a/src/client/Client.cc +++ b/src/client/Client.cc @@ -1652,7 +1652,7 @@ void Client::dump_mds_sessions(Formatter *f, bool cap_dump) f->open_array_section("sessions"); for (const auto &p : mds_sessions) { f->open_object_section("session"); - p.second.dump(f, cap_dump); + p.second->dump(f, cap_dump); f->close_section(); } f->close_section(); @@ -1808,7 +1808,7 @@ int Client::make_request(MetaRequest *request, if (use_mds >= 0) request->resend_mds = use_mds; - MetaSession *session = NULL; + MetaSessionRef session = NULL; while (1) { if (request->aborted()) break; @@ -1859,11 +1859,11 @@ int Client::make_request(MetaRequest *request, if (!have_open_session(mds)) continue; } else { - session = &mds_sessions.at(mds); + session = mds_sessions.at(mds); } // send request. - send_request(request, session); + send_request(request, session.get()); // wait for signal ldout(cct, 20) << "awaiting reply|forward|kick on " << &caller_cond << dendl; @@ -1905,7 +1905,7 @@ int Client::make_request(MetaRequest *request, request->dispatch_cond = 0; if (r >= 0 && ptarget) - r = verify_reply_trace(r, session, request, reply, ptarget, pcreated, perms); + r = verify_reply_trace(r, session.get(), request, reply, ptarget, pcreated, perms); if (pdirbl) *pdirbl = reply->get_extra_bl(); @@ -2069,24 +2069,24 @@ bool Client::have_open_session(mds_rank_t mds) { const auto &it = mds_sessions.find(mds); return it != mds_sessions.end() && - (it->second.state == MetaSession::STATE_OPEN || - it->second.state == MetaSession::STATE_STALE); + (it->second->state == MetaSession::STATE_OPEN || + it->second->state == MetaSession::STATE_STALE); } -MetaSession *Client::_get_mds_session(mds_rank_t mds, Connection *con) +MetaSessionRef Client::_get_mds_session(mds_rank_t mds, Connection *con) { const auto &it = mds_sessions.find(mds); - if (it == mds_sessions.end() || it->second.con != con) { + if (it == mds_sessions.end() || it->second->con != con) { return NULL; } else { - return &it->second; + return it->second; } } -MetaSession *Client::_get_or_open_mds_session(mds_rank_t mds) +MetaSessionRef Client::_get_or_open_mds_session(mds_rank_t mds) { auto it = mds_sessions.find(mds); - return it == mds_sessions.end() ? _open_mds_session(mds) : &it->second; + return it == mds_sessions.end() ? _open_mds_session(mds) : it->second; } /** @@ -2160,15 +2160,15 @@ void Client::update_metadata(std::string const &k, std::string const &v) metadata[k] = v; } -MetaSession *Client::_open_mds_session(mds_rank_t mds) +MetaSessionRef Client::_open_mds_session(mds_rank_t mds) { ldout(cct, 10) << __func__ << " mds." << mds << dendl; auto addrs = mdsmap->get_addrs(mds); auto em = mds_sessions.emplace(std::piecewise_construct, std::forward_as_tuple(mds), - std::forward_as_tuple(mds, messenger->connect_to_mds(addrs), addrs)); + std::forward_as_tuple(new MetaSession(mds, messenger->connect_to_mds(addrs), addrs))); ceph_assert(em.second); /* not already present */ - MetaSession *session = &em.first->second; + auto session = em.first->second; auto m = make_message(CEPH_SESSION_REQUEST_OPEN); m->metadata = metadata; @@ -2208,7 +2208,7 @@ void Client::handle_client_session(const MConstRef& m) ldout(cct, 10) << __func__ << " " << *m << " from mds." << from << dendl; std::scoped_lock cl(client_lock); - MetaSession *session = _get_mds_session(from, m->get_connection().get()); + auto session = _get_mds_session(from, m->get_connection().get()); if (!session) { ldout(cct, 10) << " discarding session message from sessionless mds " << m->get_source_inst() << dendl; return; @@ -2222,13 +2222,13 @@ void Client::handle_client_session(const MConstRef& m) if (!missing_features.empty()) { lderr(cct) << "mds." << from << " lacks required features '" << missing_features << "', closing session " << dendl; - _close_mds_session(session); - _closed_mds_session(session, -CEPHFS_EPERM, true); + _close_mds_session(session.get()); + _closed_mds_session(session.get(), -CEPHFS_EPERM, true); break; } session->mds_features = std::move(m->supported_features); - renew_caps(session); + renew_caps(session.get()); session->state = MetaSession::STATE_OPEN; if (is_unmounting()) mount_cond.notify_all(); @@ -2239,7 +2239,7 @@ void Client::handle_client_session(const MConstRef& m) } case CEPH_SESSION_CLOSE: - _closed_mds_session(session); + _closed_mds_session(session.get()); break; case CEPH_SESSION_RENEWCAPS: @@ -2248,7 +2248,7 @@ void Client::handle_client_session(const MConstRef& m) session->cap_ttl = session->last_cap_renew_request + mdsmap->get_session_timeout(); if (was_stale) - wake_up_session_caps(session, false); + wake_up_session_caps(session.get(), false); } break; @@ -2257,7 +2257,7 @@ void Client::handle_client_session(const MConstRef& m) session->cap_gen++; session->cap_ttl = ceph_clock_now(); session->cap_ttl -= 1; - renew_caps(session); + renew_caps(session.get()); break; case CEPH_SESSION_RECALL_STATE: @@ -2273,7 +2273,7 @@ void Client::handle_client_session(const MConstRef& m) * by tick(). */ renew_and_flush_cap_releases(); - trim_caps(session, m->get_max_caps()); + trim_caps(session.get(), m->get_max_caps()); break; case CEPH_SESSION_FLUSHMSG: @@ -2285,7 +2285,7 @@ void Client::handle_client_session(const MConstRef& m) break; case CEPH_SESSION_FORCE_RO: - force_session_readonly(session); + force_session_readonly(session.get()); break; case CEPH_SESSION_REJECT: @@ -2298,7 +2298,7 @@ void Client::handle_client_session(const MConstRef& m) error_str = "unknown error"; lderr(cct) << "mds." << from << " rejected us (" << error_str << ")" << dendl; - _closed_mds_session(session, -CEPHFS_EPERM, true); + _closed_mds_session(session.get(), -CEPHFS_EPERM, true); } break; @@ -2312,7 +2312,7 @@ bool Client::_any_stale_sessions() const ceph_assert(ceph_mutex_is_locked_by_me(client_lock)); for (const auto &p : mds_sessions) { - if (p.second.state == MetaSession::STATE_STALE) { + if (p.second->state == MetaSession::STATE_STALE) { return true; } } @@ -2325,14 +2325,13 @@ void Client::_kick_stale_sessions() ldout(cct, 1) << __func__ << dendl; for (auto it = mds_sessions.begin(); it != mds_sessions.end(); ) { - MetaSession &s = it->second; - if (s.state == MetaSession::STATE_REJECTED) { - mds_sessions.erase(it++); + auto s = it->second; + if (s->state == MetaSession::STATE_REJECTED) { + mds_sessions.erase(it->first); continue; } - ++it; - if (s.state == MetaSession::STATE_STALE) - _closed_mds_session(&s); + if (s->state == MetaSession::STATE_STALE) + _closed_mds_session(s.get()); } } @@ -2431,7 +2430,7 @@ void Client::handle_client_request_forward(const MConstRefget_source().num()); std::scoped_lock cl(client_lock); - MetaSession *session = _get_mds_session(mds, fwd->get_connection().get()); + auto session = _get_mds_session(mds, fwd->get_connection().get()); if (!session) { return; } @@ -2479,7 +2478,7 @@ void Client::handle_client_reply(const MConstRef& reply) mds_rank_t mds_num = mds_rank_t(reply->get_source().num()); std::scoped_lock cl(client_lock); - MetaSession *session = _get_mds_session(mds_num, reply->get_connection().get()); + auto session = _get_mds_session(mds_num, reply->get_connection().get()); if (!session) { return; } @@ -2525,7 +2524,7 @@ void Client::handle_client_reply(const MConstRef& reply) ceph_assert(!request->reply); request->reply = reply; - insert_trace(request, session); + insert_trace(request, session.get()); // Handle unsafe reply if (!is_safe) { @@ -2867,10 +2866,9 @@ void Client::handle_mds_map(const MConstRef& m) _mdsmap.swap(mdsmap); // reset session - for (auto p = mds_sessions.begin(); p != mds_sessions.end(); ) { - mds_rank_t mds = p->first; - MetaSession *session = &p->second; - ++p; + for (auto &p : mds_sessions) { + mds_rank_t mds = p.first; + auto session = p.second; int oldstate = _mdsmap->get_state(mds); int newstate = mdsmap->get_state(mds); @@ -2889,33 +2887,33 @@ void Client::handle_mds_map(const MConstRef& m) // When new MDS starts to take over, notify kernel to trim unused entries // in its dcache/icache. Hopefully, the kernel will release some unused // inodes before the new MDS enters reconnect state. - trim_cache_for_reconnect(session); + trim_cache_for_reconnect(session.get()); } else if (oldstate == newstate) continue; // no change session->mds_state = newstate; if (newstate == MDSMap::STATE_RECONNECT) { session->con = messenger->connect_to_mds(session->addrs); - send_reconnect(session); + send_reconnect(session.get()); } else if (newstate > MDSMap::STATE_RECONNECT) { if (oldstate < MDSMap::STATE_RECONNECT) { ldout(cct, 1) << "we may miss the MDSMap::RECONNECT, close mds session ... " << dendl; - _closed_mds_session(session); + _closed_mds_session(session.get()); continue; } if (newstate >= MDSMap::STATE_ACTIVE) { if (oldstate < MDSMap::STATE_ACTIVE) { // kick new requests - kick_requests(session); - kick_flushing_caps(session); + kick_requests(session.get()); + kick_flushing_caps(session.get()); signal_context_list(session->waiting_for_open); - wake_up_session_caps(session, true); + wake_up_session_caps(session.get(), true); } connect_mds_targets(mds); } } else if (newstate == MDSMap::STATE_NULL && mds >= mdsmap->get_max_mds()) { - _closed_mds_session(session); + _closed_mds_session(session.get()); } } @@ -3072,9 +3070,9 @@ void Client::wait_unsafe_requests() { list last_unsafe_reqs; for (const auto &p : mds_sessions) { - const MetaSession &s = p.second; - if (!s.unsafe_requests.empty()) { - MetaRequest *req = s.unsafe_requests.back(); + const auto s = p.second; + if (!s->unsafe_requests.empty()) { + MetaRequest *req = s->unsafe_requests.back(); req->get(); last_unsafe_reqs.push_back(req); } @@ -3154,12 +3152,12 @@ void Client::handle_lease(const MConstRef& m) mds_rank_t mds = mds_rank_t(m->get_source().num()); std::scoped_lock cl(client_lock); - MetaSession *session = _get_mds_session(mds, m->get_connection().get()); + auto session = _get_mds_session(mds, m->get_connection().get()); if (!session) { return; } - got_mds_push(session); + got_mds_push(session.get()); ceph_seq_t seq = m->get_seq(); @@ -3762,7 +3760,7 @@ void Client::check_caps(Inode *in, unsigned flags) } for (auto &[mds, cap] : in->caps) { - MetaSession *session = &mds_sessions.at(mds); + auto session = mds_sessions.at(mds); cap_used = used; if (in->auth_cap && &cap != in->auth_cap) @@ -3817,7 +3815,7 @@ void Client::check_caps(Inode *in, unsigned flags) if (in->flags & I_KICK_FLUSH) { ldout(cct, 20) << " reflushing caps (check_caps) on " << *in << " to mds." << mds << dendl; - kick_flushing_caps(in, session); + kick_flushing_caps(in, session.get()); } if (!in->cap_snaps.empty() && in->cap_snaps.rbegin()->second.flush_tid == 0) @@ -3836,7 +3834,7 @@ void Client::check_caps(Inode *in, unsigned flags) flush_tid = 0; } - send_cap(in, session, &cap, msg_flags, cap_used, wanted, retain, + send_cap(in, session.get(), &cap, msg_flags, cap_used, wanted, retain, flushing, flush_tid); } } @@ -4704,7 +4702,7 @@ void Client::wait_sync_caps(ceph_tid_t want) ldout(cct, 10) << __func__ << " want " << want << " (last is " << last_flush_tid << ", " << num_flushing_caps << " total flushing)" << dendl; for (auto &p : mds_sessions) { - MetaSession *s = &p.second; + auto s = p.second; if (s->flushing_caps_tids.empty()) continue; ceph_tid_t oldest_tid = *s->flushing_caps_tids.begin(); @@ -4967,12 +4965,12 @@ void Client::handle_snap(const MConstRef& m) mds_rank_t mds = mds_rank_t(m->get_source().num()); std::scoped_lock cl(client_lock); - MetaSession *session = _get_mds_session(mds, m->get_connection().get()); + auto session = _get_mds_session(mds, m->get_connection().get()); if (!session) { return; } - got_mds_push(session); + got_mds_push(session.get()); map to_move; SnapRealm *realm = 0; @@ -5039,12 +5037,12 @@ void Client::handle_quota(const MConstRef& m) mds_rank_t mds = mds_rank_t(m->get_source().num()); std::scoped_lock cl(client_lock); - MetaSession *session = _get_mds_session(mds, m->get_connection().get()); + auto session = _get_mds_session(mds, m->get_connection().get()); if (!session) { return; } - got_mds_push(session); + got_mds_push(session.get()); ldout(cct, 10) << __func__ << " " << *m << " from mds." << mds << dendl; @@ -5065,7 +5063,7 @@ void Client::handle_caps(const MConstRef& m) mds_rank_t mds = mds_rank_t(m->get_source().num()); std::scoped_lock cl(client_lock); - MetaSession *session = _get_mds_session(mds, m->get_connection().get()); + auto session = _get_mds_session(mds, m->get_connection().get()); if (!session) { return; } @@ -5080,7 +5078,7 @@ void Client::handle_caps(const MConstRef& m) set_cap_epoch_barrier(m->osd_epoch_barrier); } - got_mds_push(session); + got_mds_push(session.get()); Inode *in; vinodeno_t vino(m->get_ino(), CEPH_NOSNAP); @@ -5105,20 +5103,20 @@ void Client::handle_caps(const MConstRef& m) } switch (m->get_op()) { - case CEPH_CAP_OP_EXPORT: return handle_cap_export(session, in, m); - case CEPH_CAP_OP_FLUSHSNAP_ACK: return handle_cap_flushsnap_ack(session, in, m); - case CEPH_CAP_OP_IMPORT: /* no return */ handle_cap_import(session, in, m); + case CEPH_CAP_OP_EXPORT: return handle_cap_export(session.get(), in, m); + case CEPH_CAP_OP_FLUSHSNAP_ACK: return handle_cap_flushsnap_ack(session.get(), in, m); + case CEPH_CAP_OP_IMPORT: /* no return */ handle_cap_import(session.get(), in, m); } if (auto it = in->caps.find(mds); it != in->caps.end()) { Cap &cap = in->caps.at(mds); switch (m->get_op()) { - case CEPH_CAP_OP_TRUNC: return handle_cap_trunc(session, in, m); + case CEPH_CAP_OP_TRUNC: return handle_cap_trunc(session.get(), in, m); case CEPH_CAP_OP_IMPORT: case CEPH_CAP_OP_REVOKE: - case CEPH_CAP_OP_GRANT: return handle_cap_grant(session, in, &cap, m); - case CEPH_CAP_OP_FLUSH_ACK: return handle_cap_flush_ack(session, in, &cap, m); + case CEPH_CAP_OP_GRANT: return handle_cap_grant(session.get(), in, &cap, m); + case CEPH_CAP_OP_FLUSH_ACK: return handle_cap_flush_ack(session.get(), in, &cap, m); } } else { ldout(cct, 5) << __func__ << " don't have " << *in << " cap on mds." << mds << dendl; @@ -5182,7 +5180,7 @@ void Client::handle_cap_export(MetaSession *session, Inode *in, const MConstRef< if (cap.cap_id == m->get_cap_id()) { if (m->peer.cap_id) { const auto peer_mds = mds_rank_t(m->peer.mds); - MetaSession *tsession = _get_or_open_mds_session(peer_mds); + auto tsession = _get_or_open_mds_session(peer_mds); auto it = in->caps.find(peer_mds); if (it != in->caps.end()) { Cap &tcap = it->second; @@ -5196,10 +5194,10 @@ void Client::handle_cap_export(MetaSession *session, Inode *in, const MConstRef< if (&cap == in->auth_cap) in->auth_cap = &tcap; if (in->auth_cap == &tcap && in->flushing_cap_item.is_on_list()) - adjust_session_flushing_caps(in, session, tsession); + adjust_session_flushing_caps(in, session, tsession.get()); } } else { - add_update_cap(in, tsession, m->peer.cap_id, cap.issued, 0, + add_update_cap(in, tsession.get(), m->peer.cap_id, cap.issued, 0, m->peer.seq - 1, m->peer.mseq, (uint64_t)-1, &cap == in->auth_cap ? CEPH_CAP_FLAG_AUTH : 0, cap.latest_perms); @@ -6253,7 +6251,7 @@ int Client::mount(const std::string &mount_root, const UserPerm& perms, void Client::_close_sessions() { for (auto it = mds_sessions.begin(); it != mds_sessions.end(); ) { - if (it->second.state == MetaSession::STATE_REJECTED) + if (it->second->state == MetaSession::STATE_REJECTED) mds_sessions.erase(it++); else ++it; @@ -6262,8 +6260,8 @@ void Client::_close_sessions() while (!mds_sessions.empty()) { // send session closes! for (auto &p : mds_sessions) { - if (p.second.state != MetaSession::STATE_CLOSING) { - _close_mds_session(&p.second); + if (p.second->state != MetaSession::STATE_CLOSING) { + _close_mds_session(p.second.get()); mds_ranks_closing.insert(p.first); } } @@ -6280,7 +6278,7 @@ void Client::_close_sessions() while (!mds_ranks_closing.empty()) { auto session = mds_sessions.at(*mds_ranks_closing.begin()); // this prunes entry from mds_sessions and mds_ranks_closing - _closed_mds_session(&session, -CEPHFS_ETIMEDOUT); + _closed_mds_session(session.get(), -CEPHFS_ETIMEDOUT); } } @@ -6305,7 +6303,7 @@ void Client::flush_mdlog_sync(Inode *in) for (auto &rank : anchor) { auto session = &mds_sessions.at(rank); - flush_mdlog(session); + flush_mdlog(session->get()); } } @@ -6314,7 +6312,7 @@ void Client::flush_mdlog_sync() if (mds_requests.empty()) return; for (auto &p : mds_sessions) { - flush_mdlog(&p.second); + flush_mdlog(p.second.get()); } } @@ -6353,8 +6351,8 @@ void Client::_abort_mds_sessions(int err) // Force-close all sessions while(!mds_sessions.empty()) { - auto& session = mds_sessions.begin()->second; - _closed_mds_session(&session, err); + auto session = mds_sessions.begin()->second; + _closed_mds_session(session.get(), err); } } @@ -6518,16 +6516,16 @@ void Client::flush_cap_releases() // send any cap releases for (auto &p : mds_sessions) { - auto &session = p.second; - if (session.release && mdsmap->is_clientreplay_or_active_or_stopping( + auto session = p.second; + if (session->release && mdsmap->is_clientreplay_or_active_or_stopping( p.first)) { - nr_caps += session.release->caps.size(); + nr_caps += session->release->caps.size(); if (cct->_conf->client_inject_release_failure) { ldout(cct, 20) << __func__ << " injecting failure to send cap release message" << dendl; } else { - session.con->send_message2(std::move(session.release)); + session->con->send_message2(std::move(session->release)); } - session.release.reset(); + session->release.reset(); } } @@ -6570,7 +6568,7 @@ void Client::tick() } signal_cond_list(waiting_for_mdsmap); for (auto &p : mds_sessions) { - signal_context_list(p.second.waiting_for_open); + signal_context_list(p.second->waiting_for_open); } } } @@ -6719,7 +6717,7 @@ void Client::renew_caps() for (auto &p : mds_sessions) { ldout(cct, 15) << "renew_caps requesting from mds." << p.first << dendl; if (mdsmap->get_state(p.first) >= MDSMap::STATE_REJOIN) - renew_caps(&p.second); + renew_caps(p.second.get()); } } @@ -6764,13 +6762,13 @@ bool Client::_dentry_valid(const Dentry *dn) utime_t now = ceph_clock_now(); if (dn->lease_mds >= 0 && dn->lease_ttl > now && mds_sessions.count(dn->lease_mds)) { - MetaSession &s = mds_sessions.at(dn->lease_mds); - if (s.cap_ttl > now && s.cap_gen == dn->lease_gen) { + auto s = mds_sessions.at(dn->lease_mds); + if (s->cap_ttl > now && s->cap_gen == dn->lease_gen) { dlease_hit(); return true; } - ldout(cct, 20) << " bad lease, cap_ttl " << s.cap_ttl << ", cap_gen " << s.cap_gen + ldout(cct, 20) << " bad lease, cap_ttl " << s->cap_ttl << ", cap_gen " << s->cap_gen << " vs lease_gen " << dn->lease_gen << dendl; } @@ -14995,11 +14993,11 @@ void Client::ms_handle_remote_reset(Connection *con) { // kludge to figure out which mds this is; fixme with a Connection* state mds_rank_t mds = MDS_RANK_NONE; - MetaSession *s = NULL; + MetaSessionRef s = NULL; for (auto &p : mds_sessions) { if (mdsmap->have_inst(p.first) && mdsmap->get_addrs(p.first) == con->get_peer_addrs()) { mds = p.first; - s = &p.second; + s = p.second; } } if (mds >= 0) { @@ -15007,7 +15005,7 @@ void Client::ms_handle_remote_reset(Connection *con) switch (s->state) { case MetaSession::STATE_CLOSING: ldout(cct, 1) << "reset from mds we were closing; we'll call that closed" << dendl; - _closed_mds_session(s); + _closed_mds_session(s.get()); break; case MetaSession::STATE_OPENING: @@ -15015,8 +15013,8 @@ void Client::ms_handle_remote_reset(Connection *con) ldout(cct, 1) << "reset from mds we were opening; retrying" << dendl; list waiters; waiters.swap(s->waiting_for_open); - _closed_mds_session(s); - MetaSession *news = _get_or_open_mds_session(mds); + _closed_mds_session(s.get()); + auto news = _get_or_open_mds_session(mds); news->waiting_for_open.swap(waiters); } break; @@ -15026,7 +15024,7 @@ void Client::ms_handle_remote_reset(Connection *con) objecter->maybe_request_map(); /* to check if we are blocklisted */ if (cct->_conf.get_val("client_reconnect_stale")) { ldout(cct, 1) << "reset from mds we were open; close mds session for reconnect" << dendl; - _closed_mds_session(s); + _closed_mds_session(s.get()); } else { ldout(cct, 1) << "reset from mds we were open; mark session as stale" << dendl; s->state = MetaSession::STATE_STALE; @@ -15414,7 +15412,7 @@ int Client::start_reclaim(const std::string& uuid, unsigned flags, continue; } - MetaSession *session; + MetaSessionRef session; if (!have_open_session(mds)) { session = _get_or_open_mds_session(mds); if (session->state == MetaSession::STATE_REJECTED) @@ -15428,7 +15426,7 @@ int Client::start_reclaim(const std::string& uuid, unsigned flags, continue; } - session = &mds_sessions.at(mds); + session = mds_sessions.at(mds); if (!session->mds_features.test(CEPHFS_FEATURE_RECLAIM_CLIENT)) return -CEPHFS_EOPNOTSUPP; @@ -15482,14 +15480,14 @@ void Client::finish_reclaim() auto it = metadata.find("reclaiming_uuid"); if (it == metadata.end()) { for (auto &p : mds_sessions) - p.second.reclaim_state = MetaSession::RECLAIM_NULL; + p.second->reclaim_state = MetaSession::RECLAIM_NULL; return; } for (auto &p : mds_sessions) { - p.second.reclaim_state = MetaSession::RECLAIM_NULL; + p.second->reclaim_state = MetaSession::RECLAIM_NULL; auto m = make_message("", MClientReclaim::FLAG_FINISH); - p.second.con->send_message2(std::move(m)); + p.second->con->send_message2(std::move(m)); } metadata["uuid"] = it->second; @@ -15502,7 +15500,7 @@ void Client::handle_client_reclaim_reply(const MConstRef& r ldout(cct, 10) << __func__ << " " << *reply << " from mds." << from << dendl; std::scoped_lock cl(client_lock); - MetaSession *session = _get_mds_session(from, reply->get_connection().get()); + auto session = _get_mds_session(from, reply->get_connection().get()); if (!session) { ldout(cct, 10) << " discarding reclaim reply from sessionless mds." << from << dendl; return; diff --git a/src/client/Client.h b/src/client/Client.h index bbf1619fedfdd..b6a76faca5083 100644 --- a/src/client/Client.h +++ b/src/client/Client.h @@ -901,9 +901,9 @@ class Client : public Dispatcher, public md_config_obs_t { void get_session_metadata(std::map *meta) const; bool have_open_session(mds_rank_t mds); void got_mds_push(MetaSession *s); - MetaSession *_get_mds_session(mds_rank_t mds, Connection *con); ///< return session for mds *and* con; null otherwise - MetaSession *_get_or_open_mds_session(mds_rank_t mds); - MetaSession *_open_mds_session(mds_rank_t mds); + MetaSessionRef _get_mds_session(mds_rank_t mds, Connection *con); ///< return session for mds *and* con; null otherwise + MetaSessionRef _get_or_open_mds_session(mds_rank_t mds); + MetaSessionRef _open_mds_session(mds_rank_t mds); void _close_mds_session(MetaSession *s); void _closed_mds_session(MetaSession *s, int err=0, bool rejected=false); bool _any_stale_sessions() const; @@ -1476,7 +1476,7 @@ class Client : public Dispatcher, public md_config_obs_t { epoch_t cap_epoch_barrier = 0; // mds sessions - map mds_sessions; // mds -> push seq + map mds_sessions; // mds -> push seq std::set mds_ranks_closing; // mds ranks currently tearing down sessions std::list waiting_for_mdsmap; diff --git a/src/client/MetaSession.h b/src/client/MetaSession.h index 775912b53a1a2..cfeb26a471c90 100644 --- a/src/client/MetaSession.h +++ b/src/client/MetaSession.h @@ -68,4 +68,5 @@ struct MetaSession { ceph_seq_t mseq, epoch_t osd_barrier); }; +using MetaSessionRef = std::shared_ptr; #endif From d4688848fa4710d50db5488eaefd05defcb78263 Mon Sep 17 00:00:00 2001 From: Xiubo Li Date: Wed, 10 Feb 2021 14:17:11 +0800 Subject: [PATCH 0019/2866] client: adjust the Fh class members order Some members once ininitalized in the Fh constructor they won't change in the whole lifetime, and putting them under the client_lock makes no sense. And this could help simplify the inode lock code in some cases. Signed-off-by: Xiubo Li --- src/client/Fh.cc | 2 +- src/client/Fh.h | 12 ++++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/client/Fh.cc b/src/client/Fh.cc index e72390df714c6..e7dd10b366641 100644 --- a/src/client/Fh.cc +++ b/src/client/Fh.cc @@ -19,7 +19,7 @@ #include "Fh.h" Fh::Fh(InodeRef in, int flags, int cmode, uint64_t _gen, const UserPerm &perms) : - inode(in), mode(cmode), gen(_gen), flags(flags), actor_perms(perms), + inode(in), flags(flags), gen(_gen), actor_perms(perms), mode(cmode), readahead() { inode->add_fh(this); diff --git a/src/client/Fh.h b/src/client/Fh.h index 3448f5b6ac88e..4657fac8457dd 100644 --- a/src/client/Fh.h +++ b/src/client/Fh.h @@ -13,18 +13,22 @@ class Inode; struct Fh { InodeRef inode; + int flags; + uint64_t gen; + UserPerm actor_perms; // perms I opened the file with + + // the members above once ininitalized in the constructor + // they won't change, and putting them under the client_lock + // makes no sense. + int _ref = 1; loff_t pos = 0; int mds = 0; // have to talk to mds we opened with (for now) int mode; // the mode i opened the file with - uint64_t gen; - int flags; bool pos_locked = false; // pos is currently in use std::list pos_waiters; // waiters for pos - UserPerm actor_perms; // perms I opened the file with - Readahead readahead; // file lock From 3d48ecfc65c2e8df635fa404e774028e366b21e3 Mon Sep 17 00:00:00 2001 From: Xiubo Li Date: Wed, 10 Feb 2021 14:19:35 +0800 Subject: [PATCH 0020/2866] client: remove useless mds member in Fh class Signed-off-by: Xiubo Li --- src/client/Fh.h | 1 - 1 file changed, 1 deletion(-) diff --git a/src/client/Fh.h b/src/client/Fh.h index 4657fac8457dd..434716fee2bac 100644 --- a/src/client/Fh.h +++ b/src/client/Fh.h @@ -23,7 +23,6 @@ struct Fh { int _ref = 1; loff_t pos = 0; - int mds = 0; // have to talk to mds we opened with (for now) int mode; // the mode i opened the file with bool pos_locked = false; // pos is currently in use From 7a3198ef6052676fa98bc2d7dd2805ca056007b9 Mon Sep 17 00:00:00 2001 From: wzbxqt Date: Fri, 6 Aug 2021 23:09:05 +0800 Subject: [PATCH 0021/2866] rgw,doc: fix command problem Signed-off-by: wangzhong --- doc/radosgw/STS.rst | 10 +++++----- doc/radosgw/role.rst | 8 ++++---- src/rgw/rgw_admin.cc | 4 ++-- src/test/cli/radosgw-admin/help.t | 4 ++-- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/doc/radosgw/STS.rst b/doc/radosgw/STS.rst index d843ea5d37c1a..6cfb28dd4f1ca 100644 --- a/doc/radosgw/STS.rst +++ b/doc/radosgw/STS.rst @@ -67,7 +67,7 @@ The trust between the IDP and the role is created by adding a Condition to the r allows access only to applications with the app id given in the trust policy document. The Condition is of the form:: - "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"Federated\":[\"arn:aws:iam:::oidc-provider/\"]},\"Action\":[\"sts:AssumeRoleWithWebIdentity\"],\"Condition\":{\"StringEquals\":{\" :app_id\":\"\"\}\}\}\]\}" + '''{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"Federated":["arn:aws:iam:::oidc-provider/"]},"Action":["sts:AssumeRoleWithWebIdentity"],"Condition":{"StringEquals":{" :app_id":""}}}]}''' The app_id in the condition above must match the 'aud' field of the incoming token. @@ -106,7 +106,7 @@ according to the permission policy attached to the role. region_name='' ) - policy_document = "{\"Version\":\"2012-10-17\",\"Statement\":{\"Effect\":\"Allow\",\"Principal\":{\"AWS\":[\"arn:aws:iam:::user/TESTER1\"]},\"Action\":[\"sts:AssumeRole\"]}]}" + policy_document = '''{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["arn:aws:iam:::user/TESTER1"]},"Action":["sts:AssumeRole"]}]}''' role_response = iam_client.create_role( AssumeRolePolicyDocument=policy_document, @@ -114,7 +114,7 @@ according to the permission policy attached to the role. RoleName='S3Access', ) - role_policy = "{\"Version\":\"2012-10-17\",\"Statement\":{\"Effect\":\"Allow\",\"Action\":\"s3:*\",\"Resource\":\"arn:aws:s3:::*\"}}" + role_policy = '''{"Version":"2012-10-17","Statement":{"Effect":"Allow","Action":"s3:*","Resource":"arn:aws:s3:::*"}}''' response = iam_client.put_role_policy( RoleName='S3Access', @@ -171,14 +171,14 @@ according to permission policy of the role. ] ) - policy_document = "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"Federated\":[\"arn:aws:iam:::oidc-provider/localhost:8080/auth/realms/demo\"]},\"Action\":[\"sts:AssumeRoleWithWebIdentity\"],\"Condition\":{\"StringEquals\":{\"localhost:8080/auth/realms/demo:app_id\":\"customer-portal\"}}}]}" + policy_document = '''{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"Federated":["arn:aws:iam:::oidc-provider/localhost:8080/auth/realms/demo"]},"Action":["sts:AssumeRoleWithWebIdentity"],"Condition":{"StringEquals":{"localhost:8080/auth/realms/demo:app_id":"customer-portal"}}}]}''' role_response = iam_client.create_role( AssumeRolePolicyDocument=policy_document, Path='/', RoleName='S3Access', ) - role_policy = "{\"Version\":\"2012-10-17\",\"Statement\":{\"Effect\":\"Allow\",\"Action\":\"s3:*\",\"Resource\":\"arn:aws:s3:::*\"}}" + role_policy = '''{"Version":"2012-10-17","Statement":{"Effect":"Allow","Action":"s3:*","Resource":"arn:aws:s3:::*"}}''' response = iam_client.put_role_policy( RoleName='S3Access', diff --git a/doc/radosgw/role.rst b/doc/radosgw/role.rst index a774624315a32..2954fd0022aaa 100644 --- a/doc/radosgw/role.rst +++ b/doc/radosgw/role.rst @@ -53,7 +53,7 @@ Delete a Role To delete a role, execute the following:: - radosgw-admin role rm --role-name={role-name} + radosgw-admin role delete --role-name={role-name} Request Parameters ~~~~~~~~~~~~~~~~~~ @@ -65,7 +65,7 @@ Request Parameters For example:: - radosgw-admin role rm --role-name=S3Access1 + radosgw-admin role delete --role-name=S3Access1 Note: A role can be deleted only when it doesn't have any permission policy attached to it. @@ -267,7 +267,7 @@ Delete Policy attached to a Role To delete permission policy attached to a role, execute the following:: - radosgw-admin role policy rm --role-name={role-name} --policy-name={policy-name} + radosgw-admin role policy delete --role-name={role-name} --policy-name={policy-name} Request Parameters ~~~~~~~~~~~~~~~~~~ @@ -284,7 +284,7 @@ Request Parameters For example:: - radosgw-admin role-policy get --role-name=S3Access1 --policy-name=Policy1 + radosgw-admin role-policy delete --role-name=S3Access1 --policy-name=Policy1 REST APIs for Manipulating a Role diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc index b202e9d9c31db..b246aa3e4544f 100644 --- a/src/rgw/rgw_admin.cc +++ b/src/rgw/rgw_admin.cc @@ -250,14 +250,14 @@ void usage() cout << " orphans list-jobs deprecated -- list the current job-ids for orphans search\n"; cout << " * the three 'orphans' sub-commands are now deprecated; consider using the `rgw-orphan-list` tool\n"; cout << " role create create a AWS role for use with STS\n"; - cout << " role rm remove a role\n"; + cout << " role delete remove a role\n"; cout << " role get get a role\n"; cout << " role list list roles with specified path prefix\n"; cout << " role modify modify the assume role policy of an existing role\n"; cout << " role-policy put add/update permission policy to role\n"; cout << " role-policy list list policies attached to a role\n"; cout << " role-policy get get the specified inline policy document embedded with the given role\n"; - cout << " role-policy rm remove policy attached to a role\n"; + cout << " role-policy delete remove policy attached to a role\n"; cout << " reshard add schedule a resharding of a bucket\n"; cout << " reshard list list all bucket resharding or scheduled to be resharded\n"; cout << " reshard status read bucket resharding status\n"; diff --git a/src/test/cli/radosgw-admin/help.t b/src/test/cli/radosgw-admin/help.t index c63c63cb55e2c..e4afb655db54d 100644 --- a/src/test/cli/radosgw-admin/help.t +++ b/src/test/cli/radosgw-admin/help.t @@ -144,14 +144,14 @@ orphans list-jobs deprecated -- list the current job-ids for orphans search * the three 'orphans' sub-commands are now deprecated; consider using the `rgw-orphan-list` tool role create create a AWS role for use with STS - role rm remove a role + role delete remove a role role get get a role role list list roles with specified path prefix role modify modify the assume role policy of an existing role role-policy put add/update permission policy to role role-policy list list policies attached to a role role-policy get get the specified inline policy document embedded with the given role - role-policy rm remove policy attached to a role + role-policy delete remove policy attached to a role reshard add schedule a resharding of a bucket reshard list list all bucket resharding or scheduled to be resharded reshard status read bucket resharding status From 0c1c7fb886fcaaff5f00937cf62cf69feb8d4deb Mon Sep 17 00:00:00 2001 From: Arthur Outhenin-Chalandre Date: Fri, 25 Jun 2021 10:15:23 +0200 Subject: [PATCH 0022/2866] rbd-mirror: remove mirror image at shut_down when there is no images Some cases makes the ImageReplayer to be eternally restarted if there is no local and remote images. If both images are absent and that the local image id exists, the ImageReplayer shutdown will request a mirror image removal. Signed-off-by: Arthur Outhenin-Chalandre --- .../rbd_mirror/test_mock_ImageReplayer.cc | 67 +++++++++++++++---- .../image_deleter/TrashMoveRequest.cc | 11 +++ .../image_replayer/BootstrapRequest.cc | 8 +-- .../rbd_mirror/image_replayer/StateBuilder.h | 4 +- 4 files changed, 71 insertions(+), 19 deletions(-) diff --git a/src/test/rbd_mirror/test_mock_ImageReplayer.cc b/src/test/rbd_mirror/test_mock_ImageReplayer.cc index 57c2639a00ced..e75fa0ac4907c 100644 --- a/src/test/rbd_mirror/test_mock_ImageReplayer.cc +++ b/src/test/rbd_mirror/test_mock_ImageReplayer.cc @@ -283,10 +283,11 @@ class TestMockImageReplayer : public TestMockFixture { void expect_send(MockBootstrapRequest& mock_bootstrap_request, MockStateBuilder& mock_state_builder, librbd::MockTestImageCtx& mock_local_image_ctx, - bool do_resync, int r) { + bool do_resync, bool set_local_image, int r) { EXPECT_CALL(mock_bootstrap_request, send()) .WillOnce(Invoke([this, &mock_bootstrap_request, &mock_state_builder, - &mock_local_image_ctx, do_resync, r]() { + &mock_local_image_ctx, set_local_image, do_resync, + r]() { if (r == 0 || r == -ENOLINK) { mock_state_builder.local_image_id = mock_local_image_ctx.id; mock_state_builder.remote_image_id = m_remote_image_ctx->id; @@ -296,9 +297,15 @@ class TestMockImageReplayer : public TestMockFixture { mock_state_builder.local_image_ctx = &mock_local_image_ctx; *mock_bootstrap_request.do_resync = do_resync; } - if (r < 0) { + if (r < 0 && r != -ENOENT) { mock_state_builder.remote_image_id = ""; } + if (r == -ENOENT) { + *mock_bootstrap_request.state_builder = &mock_state_builder; + } + if (set_local_image) { + mock_state_builder.local_image_id = mock_local_image_ctx.id; + } mock_bootstrap_request.on_finish->complete(r); })); } @@ -405,7 +412,7 @@ TEST_F(TestMockImageReplayer, StartStop) { MockBootstrapRequest mock_bootstrap_request; MockStateBuilder mock_state_builder; expect_send(mock_bootstrap_request, mock_state_builder, mock_local_image_ctx, - false, 0); + false, false, 0); expect_create_replayer(mock_state_builder, mock_replayer); expect_init(mock_replayer, 0); @@ -447,8 +454,42 @@ TEST_F(TestMockImageReplayer, LocalImagePrimary) { MockStateBuilder mock_state_builder; expect_send(mock_bootstrap_request, mock_state_builder, mock_local_image_ctx, - false, -ENOMSG); + false, false, -ENOMSG); + + expect_mirror_image_status_exists(false); + + create_image_replayer(mock_threads); + + C_SaferCond start_ctx; + m_image_replayer->start(&start_ctx); + ASSERT_EQ(0, start_ctx.wait()); +} + +TEST_F(TestMockImageReplayer, MetadataCleanup) { + // START + + create_local_image(); + librbd::MockTestImageCtx mock_local_image_ctx(*m_local_image_ctx); + + MockThreads mock_threads(m_threads); + expect_work_queue_repeatedly(mock_threads); + expect_add_event_after_repeatedly(mock_threads); + + MockImageDeleter mock_image_deleter; + MockBootstrapRequest mock_bootstrap_request; + MockReplayer mock_replayer; + + expect_get_replay_status(mock_replayer); + expect_set_mirror_image_status_repeatedly(); + + InSequence seq; + + MockStateBuilder mock_state_builder; + expect_send(mock_bootstrap_request, mock_state_builder, mock_local_image_ctx, + false, true, -ENOLINK); + expect_close(mock_state_builder, 0); + expect_trash_move(mock_image_deleter, "global image id", false, 0); expect_mirror_image_status_exists(false); create_image_replayer(mock_threads); @@ -475,7 +516,7 @@ TEST_F(TestMockImageReplayer, BootstrapRemoteDeleted) { MockBootstrapRequest mock_bootstrap_request; MockStateBuilder mock_state_builder; expect_send(mock_bootstrap_request, mock_state_builder, mock_local_image_ctx, - false, -ENOLINK); + false, false, -ENOLINK); expect_close(mock_state_builder, 0); @@ -506,7 +547,7 @@ TEST_F(TestMockImageReplayer, BootstrapResyncRequested) { MockBootstrapRequest mock_bootstrap_request; MockStateBuilder mock_state_builder; expect_send(mock_bootstrap_request, mock_state_builder, mock_local_image_ctx, - true, 0); + true, false, 0); expect_close(mock_state_builder, 0); @@ -536,7 +577,7 @@ TEST_F(TestMockImageReplayer, BootstrapError) { InSequence seq; MockStateBuilder mock_state_builder; expect_send(mock_bootstrap_request, mock_state_builder, mock_local_image_ctx, - false, -EINVAL); + false, false, -EINVAL); expect_mirror_image_status_exists(false); @@ -599,7 +640,7 @@ TEST_F(TestMockImageReplayer, StopError) { InSequence seq; MockStateBuilder mock_state_builder; expect_send(mock_bootstrap_request, mock_state_builder, mock_local_image_ctx, - false, 0); + false, false, 0); expect_create_replayer(mock_state_builder, mock_replayer); expect_init(mock_replayer, 0); @@ -638,7 +679,7 @@ TEST_F(TestMockImageReplayer, ReplayerError) { InSequence seq; MockStateBuilder mock_state_builder; expect_send(mock_bootstrap_request, mock_state_builder, mock_local_image_ctx, - false, 0); + false, false, 0); expect_create_replayer(mock_state_builder, mock_replayer); expect_init(mock_replayer, -EINVAL); @@ -675,7 +716,7 @@ TEST_F(TestMockImageReplayer, ReplayerResync) { InSequence seq; MockStateBuilder mock_state_builder; expect_send(mock_bootstrap_request, mock_state_builder, mock_local_image_ctx, - false, 0); + false, false, 0); expect_create_replayer(mock_state_builder, mock_replayer); expect_init(mock_replayer, 0); @@ -718,7 +759,7 @@ TEST_F(TestMockImageReplayer, ReplayerInterrupted) { InSequence seq; MockStateBuilder mock_state_builder; expect_send(mock_bootstrap_request, mock_state_builder, mock_local_image_ctx, - false, 0); + false, false, 0); expect_create_replayer(mock_state_builder, mock_replayer); expect_init(mock_replayer, 0); @@ -766,7 +807,7 @@ TEST_F(TestMockImageReplayer, ReplayerRenamed) { InSequence seq; MockStateBuilder mock_state_builder; expect_send(mock_bootstrap_request, mock_state_builder, mock_local_image_ctx, - false, 0); + false, false, 0); expect_create_replayer(mock_state_builder, mock_replayer); expect_init(mock_replayer, 0); diff --git a/src/tools/rbd_mirror/image_deleter/TrashMoveRequest.cc b/src/tools/rbd_mirror/image_deleter/TrashMoveRequest.cc index 234bb69129666..e53923ef38730 100644 --- a/src/tools/rbd_mirror/image_deleter/TrashMoveRequest.cc +++ b/src/tools/rbd_mirror/image_deleter/TrashMoveRequest.cc @@ -182,6 +182,13 @@ template void TrashMoveRequest::handle_open_image(int r) { dout(10) << "r=" << r << dendl; + if (r == -ENOENT) { + dout(5) << "mirror image does not exist, removing orphaned metadata" << dendl; + m_image_ctx = nullptr; + remove_mirror_image(); + return; + } + if (r < 0) { derr << "failed to open image: " << cpp_strerror(r) << dendl; m_image_ctx = nullptr; @@ -343,6 +350,10 @@ template void TrashMoveRequest::close_image() { dout(10) << dendl; + if (m_image_ctx == nullptr) { + handle_close_image(0); + return; + } Context *ctx = create_context_callback< TrashMoveRequest, &TrashMoveRequest::handle_close_image>(this); m_image_ctx->state->close(ctx); diff --git a/src/tools/rbd_mirror/image_replayer/BootstrapRequest.cc b/src/tools/rbd_mirror/image_replayer/BootstrapRequest.cc index 1671bb9f6a65b..a34f39c15b34a 100644 --- a/src/tools/rbd_mirror/image_replayer/BootstrapRequest.cc +++ b/src/tools/rbd_mirror/image_replayer/BootstrapRequest.cc @@ -194,10 +194,10 @@ void BootstrapRequest::handle_prepare_remote_image(int r) { // TODO need to support multiple remote images if (state_builder != nullptr && state_builder->remote_image_id.empty() && - !state_builder->local_image_id.empty() && - state_builder->is_linked()) { - // local image exists and is non-primary and linked to the missing - // remote image + (state_builder->local_image_id.empty() || + state_builder->is_linked())) { + // both images doesn't exist or local image exists and is non-primary + // and linked to the missing remote image finish(-ENOLINK); } else { finish(-ENOENT); diff --git a/src/tools/rbd_mirror/image_replayer/StateBuilder.h b/src/tools/rbd_mirror/image_replayer/StateBuilder.h index d055c84e02068..807fe7f9162e4 100644 --- a/src/tools/rbd_mirror/image_replayer/StateBuilder.h +++ b/src/tools/rbd_mirror/image_replayer/StateBuilder.h @@ -81,13 +81,13 @@ class StateBuilder { std::string global_image_id; - std::string local_image_id; + std::string local_image_id{}; librbd::mirror::PromotionState local_promotion_state = librbd::mirror::PROMOTION_STATE_PRIMARY; ImageCtxT* local_image_ctx = nullptr; std::string remote_mirror_uuid; - std::string remote_image_id; + std::string remote_image_id{}; librbd::mirror::PromotionState remote_promotion_state = librbd::mirror::PROMOTION_STATE_NON_PRIMARY; ImageCtxT* remote_image_ctx = nullptr; From 4db66da51211504ba0a2353180ae084ba1ab3fcf Mon Sep 17 00:00:00 2001 From: Arthur Outhenin-Chalandre Date: Fri, 11 Jun 2021 09:29:59 +0200 Subject: [PATCH 0023/2866] qa/rbd-mirror: add OMAP cleanup checks This make sure that all images are deleted in the existing qa scripts and checks if all rbd-mirror metadata in OMAP are correctly deleted. Signed-off-by: Arthur Outhenin-Chalandre --- qa/workunits/rbd/rbd_mirror_helpers.sh | 70 +++++++++++++++++++++++++ qa/workunits/rbd/rbd_mirror_journal.sh | 49 +++++++++++++++++ qa/workunits/rbd/rbd_mirror_snapshot.sh | 48 +++++++++++++++++ qa/workunits/rbd/rbd_mirror_stress.sh | 5 ++ 4 files changed, 172 insertions(+) diff --git a/qa/workunits/rbd/rbd_mirror_helpers.sh b/qa/workunits/rbd/rbd_mirror_helpers.sh index 82299715e231e..ca715d854c176 100755 --- a/qa/workunits/rbd/rbd_mirror_helpers.sh +++ b/qa/workunits/rbd/rbd_mirror_helpers.sh @@ -549,6 +549,8 @@ status() echo "${cluster} ${image_pool} ${image_ns} rbd_mirroring omap vals" rados --cluster ${cluster} -p ${image_pool} --namespace "${image_ns}" listomapvals rbd_mirroring + echo "${cluster} ${image_pool} ${image_ns} rbd_mirror_leader omap vals" + rados --cluster ${cluster} -p ${image_pool} --namespace "${image_ns}" listomapvals rbd_mirror_leader echo done done @@ -1096,6 +1098,20 @@ unprotect_snapshot() rbd --cluster ${cluster} snap unprotect ${pool}/${image}@${snap} } +unprotect_snapshot_retry() +{ + local cluster=$1 + local pool=$2 + local image=$3 + local snap=$4 + + for s in 0 1 2 4 8 16 32; do + sleep ${s} + unprotect_snapshot ${cluster} ${pool} ${image} ${snap} && return 0 + done + return 1 +} + wait_for_snap_present() { local cluster=$1 @@ -1292,6 +1308,8 @@ enable_mirror() local mode=${4:-${MIRROR_IMAGE_MODE}} rbd --cluster=${cluster} mirror image enable ${pool}/${image} ${mode} + # Display image info including the global image id for debugging purpose + rbd --cluster=${cluster} info ${pool}/${image} } test_image_present() @@ -1389,6 +1407,58 @@ get_clone_format() }' } +list_omap_keys() +{ + local cluster=$1 + local pool=$2 + local obj_name=$3 + + rados --cluster ${cluster} -p ${pool} listomapkeys ${obj_name} +} + +count_omap_keys_with_filter() +{ + local cluster=$1 + local pool=$2 + local obj_name=$3 + local filter=$4 + + list_omap_keys ${cluster} ${pool} ${obj_name} | grep -c ${filter} +} + +wait_for_omap_keys() +{ + local cluster=$1 + local pool=$2 + local obj_name=$3 + local filter=$4 + + for s in 0 1 2 2 4 4 8 8 8 16 16 32; do + sleep $s + + set +e + test "$(count_omap_keys_with_filter ${cluster} ${pool} ${obj_name} ${filter})" = 0 + error_code=$? + set -e + + if [ $error_code -eq 0 ]; then + return 0 + fi + done + + return 1 +} + +wait_for_image_in_omap() +{ + local cluster=$1 + local pool=$2 + + wait_for_omap_keys ${cluster} ${pool} rbd_mirroring status_global + wait_for_omap_keys ${cluster} ${pool} rbd_mirroring image_ + wait_for_omap_keys ${cluster} ${pool} rbd_mirror_leader image_map +} + # # Main # diff --git a/qa/workunits/rbd/rbd_mirror_journal.sh b/qa/workunits/rbd/rbd_mirror_journal.sh index 84fd2424f4ba8..75c60123639ba 100755 --- a/qa/workunits/rbd/rbd_mirror_journal.sh +++ b/qa/workunits/rbd/rbd_mirror_journal.sh @@ -119,6 +119,8 @@ if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then all_admin_daemons ${CLUSTER1} rbd mirror status fi +remove_image_retry ${CLUSTER2} ${POOL} ${image1} + testlog "TEST: test image rename" new_name="${image}_RENAMED" rename_image ${CLUSTER2} ${POOL} ${image} ${new_name} @@ -138,6 +140,18 @@ wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' trash_restore ${CLUSTER2} ${POOL} ${image_id} wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} +testlog "TEST: check if removed images' OMAP are removed (with rbd-mirror on one cluster)" +remove_image_retry ${CLUSTER2} ${POOL} ${image} + +wait_for_image_in_omap ${CLUSTER1} ${POOL} +wait_for_image_in_omap ${CLUSTER2} ${POOL} + +create_image_and_enable_mirror ${CLUSTER2} ${POOL} ${image} +wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} +write_image ${CLUSTER2} ${POOL} ${image} 100 +wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' + testlog "TEST: failover and failback" start_mirrors ${CLUSTER2} @@ -216,6 +230,8 @@ wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${force_promote_image} 'up+stopp wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${force_promote_image} 'up+stopped' write_image ${CLUSTER1} ${POOL} ${force_promote_image} 100 write_image ${CLUSTER2} ${POOL} ${force_promote_image} 100 +remove_image_retry ${CLUSTER1} ${POOL} ${force_promote_image} +remove_image_retry ${CLUSTER2} ${POOL} ${force_promote_image} testlog "TEST: cloned images" testlog " - default" @@ -240,6 +256,7 @@ wait_for_image_replay_started ${CLUSTER1} ${POOL} ${clone_image} wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${clone_image} wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${clone_image} 'up+replaying' 'primary_position' compare_images ${POOL} ${clone_image} +remove_image_retry ${CLUSTER2} ${POOL} ${clone_image} testlog " - clone v1" clone_image ${CLUSTER1} ${PARENT_POOL} ${parent_image} ${parent_snap} ${POOL} ${clone_image}1 @@ -249,6 +266,10 @@ clone_image ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} ${POOL} \ test $(get_clone_format ${CLUSTER2} ${POOL} ${clone_image}_v1) = 1 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${clone_image}_v1 test $(get_clone_format ${CLUSTER1} ${POOL} ${clone_image}_v1) = 1 +remove_image_retry ${CLUSTER2} ${POOL} ${clone_image}_v1 +remove_image_retry ${CLUSTER1} ${POOL} ${clone_image}1 +unprotect_snapshot_retry ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} +remove_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} testlog " - clone v2" parent_snap=snap_v2 @@ -277,6 +298,7 @@ test_snap_removed_from_trash ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent wait_for_snap_moved_to_trash ${CLUSTER1} ${PARENT_POOL} ${parent_image} ${parent_snap} remove_image_retry ${CLUSTER1} ${POOL} ${clone_image}_v2 wait_for_snap_removed_from_trash ${CLUSTER1} ${PARENT_POOL} ${parent_image} ${parent_snap} +remove_image_retry ${CLUSTER2} ${PARENT_POOL} ${parent_image} testlog "TEST: data pool" dp_image=test_data_pool @@ -295,6 +317,7 @@ wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${dp_image} 'up+replaying' 'prim compare_images ${POOL} ${dp_image}@snap1 compare_images ${POOL} ${dp_image}@snap2 compare_images ${POOL} ${dp_image} +remove_image_retry ${CLUSTER2} ${POOL} ${dp_image} testlog "TEST: disable mirroring / delete non-primary image" image2=test2 @@ -379,7 +402,18 @@ for i in `seq 1 20`; do done wait_for_snap_present ${CLUSTER1} ${POOL} ${image2} "${snap_name}_${i}" +unprotect_snapshot ${CLUSTER2} ${POOL} ${image4} 'snap1' +unprotect_snapshot ${CLUSTER2} ${POOL} ${image4} 'snap2' +for i in ${image2} ${image4}; do + remove_image_retry ${CLUSTER2} ${POOL} ${i} +done + testlog "TEST: disable mirror while daemon is stopped" +# TODO: workaround for the daemon to ack the deletion, to remove when +# image_map cleanup is fixed +for i in ${image2} ${image4}; do + wait_for_image_present ${CLUSTER1} ${POOL} ${i} 'deleted' +done stop_mirrors ${CLUSTER1} stop_mirrors ${CLUSTER2} set_pool_mirror_mode ${CLUSTER2} ${POOL} 'image' @@ -388,6 +422,11 @@ if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then test_image_present ${CLUSTER1} ${POOL} ${image} 'present' fi start_mirrors ${CLUSTER1} +start_mirrors ${CLUSTER2} # TODO: remove start/stop of cluster2 deamons when + # image_map cleanup at startup is resolved +wait_for_image_in_omap ${CLUSTER1} ${POOL} +wait_for_image_in_omap ${CLUSTER2} ${POOL} +stop_mirrors ${CLUSTER2} wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' set_pool_mirror_mode ${CLUSTER2} ${POOL} 'pool' enable_journaling ${CLUSTER2} ${POOL} ${image} @@ -415,6 +454,7 @@ remove_image_retry ${CLUSTER2} ${POOL}/${NS1} ${image} disable_mirror ${CLUSTER2} ${POOL}/${NS2} ${image} wait_for_image_present ${CLUSTER1} ${POOL}/${NS1} ${image} 'deleted' wait_for_image_present ${CLUSTER1} ${POOL}/${NS2} ${image} 'deleted' +remove_image_retry ${CLUSTER2} ${POOL}/${NS2} ${image} testlog " - data pool" dp_image=test_data_pool @@ -428,6 +468,7 @@ write_image ${CLUSTER2} ${POOL}/${NS1} ${dp_image} 100 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL}/${NS1} ${dp_image} wait_for_status_in_pool_dir ${CLUSTER1} ${POOL}/${NS1} ${dp_image} 'up+replaying' 'primary_position' compare_images ${POOL}/${NS1} ${dp_image} +remove_image_retry ${CLUSTER2} ${POOL}/${NS1} ${dp_image} testlog "TEST: simple image resync" request_resync_image ${CLUSTER1} ${POOL} ${image} image_id @@ -460,6 +501,7 @@ wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present' wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'primary_position' compare_images ${POOL} ${image} +remove_image_retry ${CLUSTER2} ${POOL} ${image} testlog "TEST: client disconnect" image=laggy @@ -531,6 +573,7 @@ disconnect_image ${CLUSTER2} ${POOL} ${image} test -z "$(get_mirror_journal_position ${CLUSTER2} ${POOL} ${image})" wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image} wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+error' 'disconnected' +remove_image_retry ${CLUSTER2} ${POOL} ${image} testlog "TEST: split-brain" image=split-brain @@ -544,6 +587,12 @@ demote_image ${CLUSTER1} ${POOL} ${image} wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+error' 'split-brain' request_resync_image ${CLUSTER1} ${POOL} ${image} image_id wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' 'primary_position' +remove_image_retry ${CLUSTER2} ${POOL} ${image} + +testlog "TEST: check if removed images' OMAP are removed" +start_mirrors ${CLUSTER2} +wait_for_image_in_omap ${CLUSTER1} ${POOL} +wait_for_image_in_omap ${CLUSTER2} ${POOL} if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then # teuthology will trash the daemon diff --git a/qa/workunits/rbd/rbd_mirror_snapshot.sh b/qa/workunits/rbd/rbd_mirror_snapshot.sh index 6452739026638..f83b166984e1f 100755 --- a/qa/workunits/rbd/rbd_mirror_snapshot.sh +++ b/qa/workunits/rbd/rbd_mirror_snapshot.sh @@ -122,6 +122,8 @@ if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then all_admin_daemons ${CLUSTER1} rbd mirror status fi +remove_image_retry ${CLUSTER2} ${POOL} ${image1} + testlog "TEST: test image rename" new_name="${image}_RENAMED" rename_image ${CLUSTER2} ${POOL} ${image} ${new_name} @@ -144,6 +146,18 @@ trash_restore ${CLUSTER2} ${POOL} ${image_id} enable_mirror ${CLUSTER2} ${POOL} ${image} snapshot wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} +testlog "TEST: check if removed images' OMAP are removed (with rbd-mirror on one cluster)" +remove_image_retry ${CLUSTER2} ${POOL} ${image} + +wait_for_image_in_omap ${CLUSTER1} ${POOL} +wait_for_image_in_omap ${CLUSTER2} ${POOL} + +create_image_and_enable_mirror ${CLUSTER2} ${POOL} ${image} +wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} +write_image ${CLUSTER2} ${POOL} ${image} 100 +wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image} +wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' + testlog "TEST: failover and failback" start_mirrors ${CLUSTER2} @@ -222,6 +236,8 @@ wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${force_promote_image} 'up+stopp wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${force_promote_image} 'up+stopped' write_image ${CLUSTER1} ${POOL} ${force_promote_image} 100 write_image ${CLUSTER2} ${POOL} ${force_promote_image} 100 +remove_image_retry ${CLUSTER1} ${POOL} ${force_promote_image} +remove_image_retry ${CLUSTER2} ${POOL} ${force_promote_image} testlog "TEST: cloned images" testlog " - default" @@ -246,6 +262,7 @@ wait_for_image_replay_started ${CLUSTER1} ${POOL} ${clone_image} wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${clone_image} wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${clone_image} 'up+replaying' compare_images ${POOL} ${clone_image} +remove_image_retry ${CLUSTER2} ${POOL} ${clone_image} testlog " - clone v1" clone_image_and_enable_mirror ${CLUSTER1} ${PARENT_POOL} ${parent_image} \ @@ -256,6 +273,10 @@ clone_image_and_enable_mirror ${CLUSTER2} ${PARENT_POOL} ${parent_image} \ test $(get_clone_format ${CLUSTER2} ${POOL} ${clone_image}_v1) = 1 wait_for_image_replay_started ${CLUSTER1} ${POOL} ${clone_image}_v1 test $(get_clone_format ${CLUSTER1} ${POOL} ${clone_image}_v1) = 1 +remove_image_retry ${CLUSTER2} ${POOL} ${clone_image}_v1 +remove_image_retry ${CLUSTER1} ${POOL} ${clone_image}1 +unprotect_snapshot_retry ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} +remove_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} ${parent_snap} testlog " - clone v2" parent_snap=snap_v2 @@ -288,6 +309,7 @@ mirror_image_snapshot ${CLUSTER2} ${PARENT_POOL} ${parent_image} wait_for_snap_moved_to_trash ${CLUSTER1} ${PARENT_POOL} ${parent_image} ${parent_snap} remove_image_retry ${CLUSTER1} ${POOL} ${clone_image}_v2 wait_for_snap_removed_from_trash ${CLUSTER1} ${PARENT_POOL} ${parent_image} ${parent_snap} +remove_image_retry ${CLUSTER2} ${PARENT_POOL} ${parent_image} testlog "TEST: data pool" dp_image=test_data_pool @@ -306,6 +328,7 @@ wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${dp_image} 'up+replaying' compare_images ${POOL} ${dp_image}@snap1 compare_images ${POOL} ${dp_image}@snap2 compare_images ${POOL} ${dp_image} +remove_image_retry ${CLUSTER2} ${POOL} ${dp_image} testlog "TEST: disable mirroring / delete non-primary image" image2=test2 @@ -354,7 +377,18 @@ done mirror_image_snapshot ${CLUSTER2} ${POOL} ${image2} wait_for_snap_present ${CLUSTER1} ${POOL} ${image2} "${snap_name}_${i}" +unprotect_snapshot ${CLUSTER2} ${POOL} ${image4} 'snap1' +unprotect_snapshot ${CLUSTER2} ${POOL} ${image4} 'snap2' +for i in ${image2} ${image4}; do + remove_image_retry ${CLUSTER2} ${POOL} ${i} +done + testlog "TEST: disable mirror while daemon is stopped" +# TODO: workaround for the daemon to ack the deletion, to remove when +# image_map cleanup is fixed +for i in ${image2} ${image4}; do + wait_for_image_present ${CLUSTER1} ${POOL} ${i} 'deleted' +done stop_mirrors ${CLUSTER1} stop_mirrors ${CLUSTER2} disable_mirror ${CLUSTER2} ${POOL} ${image} @@ -362,6 +396,11 @@ if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then test_image_present ${CLUSTER1} ${POOL} ${image} 'present' fi start_mirrors ${CLUSTER1} +start_mirrors ${CLUSTER2} # TODO: remove start/stop of cluster2 deamons when + # image_map cleanup at startup is resolved +wait_for_image_in_omap ${CLUSTER1} ${POOL} +wait_for_image_in_omap ${CLUSTER2} ${POOL} +stop_mirrors ${CLUSTER2} wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' enable_mirror ${CLUSTER2} ${POOL} ${image} wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present' @@ -387,6 +426,7 @@ remove_image_retry ${CLUSTER2} ${POOL}/${NS1} ${image} disable_mirror ${CLUSTER2} ${POOL}/${NS2} ${image} wait_for_image_present ${CLUSTER1} ${POOL}/${NS1} ${image} 'deleted' wait_for_image_present ${CLUSTER1} ${POOL}/${NS2} ${image} 'deleted' +remove_image_retry ${CLUSTER2} ${POOL}/${NS2} ${image} testlog " - data pool" dp_image=test_data_pool @@ -400,6 +440,7 @@ write_image ${CLUSTER2} ${POOL}/${NS1} ${dp_image} 100 wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL}/${NS1} ${dp_image} wait_for_status_in_pool_dir ${CLUSTER1} ${POOL}/${NS1} ${dp_image} 'up+replaying' compare_images ${POOL}/${NS1} ${dp_image} +remove_image_retry ${CLUSTER2} ${POOL}/${NS1} ${dp_image} testlog "TEST: simple image resync" request_resync_image ${CLUSTER1} ${POOL} ${image} image_id @@ -432,6 +473,7 @@ wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present' wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image} wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' compare_images ${POOL} ${image} +remove_image_retry ${CLUSTER2} ${POOL} ${image} testlog "TEST: split-brain" image=split-brain @@ -445,6 +487,12 @@ demote_image ${CLUSTER1} ${POOL} ${image} wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+error' 'split-brain' request_resync_image ${CLUSTER1} ${POOL} ${image} image_id wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' +remove_image_retry ${CLUSTER2} ${POOL} ${image} + +testlog "TEST: check if removed images' OMAP are removed" +start_mirrors ${CLUSTER2} +wait_for_image_in_omap ${CLUSTER1} ${POOL} +wait_for_image_in_omap ${CLUSTER2} ${POOL} if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then # teuthology will trash the daemon diff --git a/qa/workunits/rbd/rbd_mirror_stress.sh b/qa/workunits/rbd/rbd_mirror_stress.sh index a17ad75e1b58e..cb79aba7ebc9c 100755 --- a/qa/workunits/rbd/rbd_mirror_stress.sh +++ b/qa/workunits/rbd/rbd_mirror_stress.sh @@ -214,3 +214,8 @@ do purge_snapshots ${CLUSTER2} ${POOL} ${image} remove_image_retry ${CLUSTER2} ${POOL} ${image} done + +testlog "TEST: check if removed images' OMAP are removed" + +wait_for_image_in_omap ${CLUSTER1} ${POOL} +wait_for_image_in_omap ${CLUSTER2} ${POOL} From e135403c736295b63fe1c8a861af40de302b8b57 Mon Sep 17 00:00:00 2001 From: Arthur Outhenin-Chalandre Date: Tue, 13 Jul 2021 14:19:49 +0200 Subject: [PATCH 0024/2866] rbd-mirror: add image_map cleanup in LoadRequest In the LoadRequest in the ImageMap class add initial cleanup to remove stale entries. To cleanup the LoadRequest will query the mirror image list and remove all the image_map that are notin the list. Signed-off-by: Arthur Outhenin-Chalandre --- qa/workunits/rbd/rbd_mirror_journal.sh | 10 --- qa/workunits/rbd/rbd_mirror_snapshot.sh | 10 --- src/tools/rbd_mirror/image_map/LoadRequest.cc | 78 ++++++++++++++++++- src/tools/rbd_mirror/image_map/LoadRequest.h | 13 ++++ 4 files changed, 90 insertions(+), 21 deletions(-) diff --git a/qa/workunits/rbd/rbd_mirror_journal.sh b/qa/workunits/rbd/rbd_mirror_journal.sh index 75c60123639ba..56a8b13a9a777 100755 --- a/qa/workunits/rbd/rbd_mirror_journal.sh +++ b/qa/workunits/rbd/rbd_mirror_journal.sh @@ -409,11 +409,6 @@ for i in ${image2} ${image4}; do done testlog "TEST: disable mirror while daemon is stopped" -# TODO: workaround for the daemon to ack the deletion, to remove when -# image_map cleanup is fixed -for i in ${image2} ${image4}; do - wait_for_image_present ${CLUSTER1} ${POOL} ${i} 'deleted' -done stop_mirrors ${CLUSTER1} stop_mirrors ${CLUSTER2} set_pool_mirror_mode ${CLUSTER2} ${POOL} 'image' @@ -422,11 +417,6 @@ if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then test_image_present ${CLUSTER1} ${POOL} ${image} 'present' fi start_mirrors ${CLUSTER1} -start_mirrors ${CLUSTER2} # TODO: remove start/stop of cluster2 deamons when - # image_map cleanup at startup is resolved -wait_for_image_in_omap ${CLUSTER1} ${POOL} -wait_for_image_in_omap ${CLUSTER2} ${POOL} -stop_mirrors ${CLUSTER2} wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' set_pool_mirror_mode ${CLUSTER2} ${POOL} 'pool' enable_journaling ${CLUSTER2} ${POOL} ${image} diff --git a/qa/workunits/rbd/rbd_mirror_snapshot.sh b/qa/workunits/rbd/rbd_mirror_snapshot.sh index f83b166984e1f..0060440fb8d0d 100755 --- a/qa/workunits/rbd/rbd_mirror_snapshot.sh +++ b/qa/workunits/rbd/rbd_mirror_snapshot.sh @@ -384,11 +384,6 @@ for i in ${image2} ${image4}; do done testlog "TEST: disable mirror while daemon is stopped" -# TODO: workaround for the daemon to ack the deletion, to remove when -# image_map cleanup is fixed -for i in ${image2} ${image4}; do - wait_for_image_present ${CLUSTER1} ${POOL} ${i} 'deleted' -done stop_mirrors ${CLUSTER1} stop_mirrors ${CLUSTER2} disable_mirror ${CLUSTER2} ${POOL} ${image} @@ -396,11 +391,6 @@ if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then test_image_present ${CLUSTER1} ${POOL} ${image} 'present' fi start_mirrors ${CLUSTER1} -start_mirrors ${CLUSTER2} # TODO: remove start/stop of cluster2 deamons when - # image_map cleanup at startup is resolved -wait_for_image_in_omap ${CLUSTER1} ${POOL} -wait_for_image_in_omap ${CLUSTER2} ${POOL} -stop_mirrors ${CLUSTER2} wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted' enable_mirror ${CLUSTER2} ${POOL} ${image} wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'present' diff --git a/src/tools/rbd_mirror/image_map/LoadRequest.cc b/src/tools/rbd_mirror/image_map/LoadRequest.cc index 7387b476cc0ff..46564a1607a2b 100644 --- a/src/tools/rbd_mirror/image_map/LoadRequest.cc +++ b/src/tools/rbd_mirror/image_map/LoadRequest.cc @@ -8,6 +8,7 @@ #include "include/rbd_types.h" #include "cls/rbd/cls_rbd_client.h" +#include "UpdateRequest.h" #include "LoadRequest.h" #define dout_context g_ceph_context @@ -23,6 +24,7 @@ namespace image_map { static const uint32_t MAX_RETURN = 1024; using librbd::util::create_rados_callback; +using librbd::util::create_context_callback; template LoadRequest::LoadRequest(librados::IoCtx &ioctx, @@ -80,7 +82,81 @@ void LoadRequest::handle_image_map_list(int r) { return; } - finish(0); + mirror_image_list(); +} + +template +void LoadRequest::mirror_image_list() { + dout(20) << dendl; + + librados::ObjectReadOperation op; + librbd::cls_client::mirror_image_list_start(&op, m_start_after, MAX_RETURN); + + m_out_bl.clear(); + librados::AioCompletion *aio_comp = create_rados_callback< + LoadRequest, + &LoadRequest::handle_mirror_image_list>(this); + int r = m_ioctx.aio_operate(RBD_MIRRORING, aio_comp, &op, &m_out_bl); + ceph_assert(r == 0); + aio_comp->release(); +} + +template +void LoadRequest::handle_mirror_image_list(int r) { + dout(20) << ": r=" << r << dendl; + + std::map ids; + if (r == 0) { + auto it = m_out_bl.cbegin(); + r = librbd::cls_client::mirror_image_list_finish(&it, &ids); + } + + if (r < 0 && r != -ENOENT) { + derr << "failed to list mirrored images: " << cpp_strerror(r) << dendl; + finish(r); + return; + } + + for (auto &id : ids) { + m_global_image_ids.emplace(id.second); + } + + if (ids.size() == MAX_RETURN) { + m_start_after = ids.rbegin()->first; + mirror_image_list(); + return; + } + + cleanup_image_map(); +} + +template +void LoadRequest::cleanup_image_map() { + dout(20) << dendl; + + std::set map_removals; + + auto it = m_image_mapping->begin(); + while (it != m_image_mapping->end()) { + if (m_global_image_ids.count(it->first) > 0) { + ++it; + continue; + } + map_removals.emplace(it->first); + it = m_image_mapping->erase(it); + } + + if (map_removals.size() == 0) { + finish(0); + return; + } + + auto ctx = create_context_callback< + LoadRequest, + &LoadRequest::finish>(this); + image_map::UpdateRequest *req = image_map::UpdateRequest::create( + m_ioctx, {}, std::move(map_removals), ctx); + req->send(); } template diff --git a/src/tools/rbd_mirror/image_map/LoadRequest.h b/src/tools/rbd_mirror/image_map/LoadRequest.h index 7657e1108a068..9b1be96857aeb 100644 --- a/src/tools/rbd_mirror/image_map/LoadRequest.h +++ b/src/tools/rbd_mirror/image_map/LoadRequest.h @@ -36,6 +36,12 @@ class LoadRequest { * IMAGE_MAP_LIST. . . . . . . * | * v + * MIRROR_IMAGE_LIST + * | + * v + * CLEANUP_IMAGE_MAP + * | + * v * * * @endverbatim @@ -48,12 +54,19 @@ class LoadRequest { std::map *m_image_mapping; Context *m_on_finish; + std::set m_global_image_ids; + bufferlist m_out_bl; std::string m_start_after; void image_map_list(); void handle_image_map_list(int r); + void mirror_image_list(); + void handle_mirror_image_list(int r); + + void cleanup_image_map(); + void finish(int r); }; From 416e25794af0252ec45b35c897c8cf2e91aca383 Mon Sep 17 00:00:00 2001 From: Arthur Outhenin-Chalandre Date: Thu, 22 Jul 2021 16:37:47 +0200 Subject: [PATCH 0025/2866] cls/rbd: prevent image_status when mirror image is not created This prevent image_status_set to succeed when there is no mirror image yet. This solves some stale entries that were not removed in rbd-mirror and prevent to add entries that would not be visible from the rbd cli. Signed-off-by: Arthur Outhenin-Chalandre --- src/cls/rbd/cls_rbd.cc | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/cls/rbd/cls_rbd.cc b/src/cls/rbd/cls_rbd.cc index ffc19267cf400..f9b30619c145b 100644 --- a/src/cls/rbd/cls_rbd.cc +++ b/src/cls/rbd/cls_rbd.cc @@ -5059,7 +5059,22 @@ int image_status_set(cls_method_context_t hctx, const string &global_image_id, ondisk_status.up = false; ondisk_status.last_update = ceph_clock_now(); - int r = cls_get_request_origin(hctx, &ondisk_status.origin); + std::string global_id_key = global_key(global_image_id); + std::string image_id; + int r = read_key(hctx, global_id_key, &image_id); + if (r < 0) { + return 0; + } + cls::rbd::MirrorImage mirror_image; + r = image_get(hctx, image_id, &mirror_image); + if (r < 0) { + return 0; + } + if (mirror_image.state != cls::rbd::MIRROR_IMAGE_STATE_ENABLED) { + return 0; + } + + r = cls_get_request_origin(hctx, &ondisk_status.origin); ceph_assert(r == 0); bufferlist bl; From 35398a5e17dc5a536ccd63417c937f2efe742654 Mon Sep 17 00:00:00 2001 From: Arthur Outhenin-Chalandre Date: Thu, 22 Jul 2021 18:53:16 +0200 Subject: [PATCH 0026/2866] rbd-mirror: remove image_mapped condition to remove image_map In some split-brain scenario the image is removed while the image_mapped is false. This prevents the removal of image_map in OMAP and thus the entry will not be removed until the daemon is restarted. Signed-off-by: Arthur Outhenin-Chalandre --- src/tools/rbd_mirror/ImageMap.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tools/rbd_mirror/ImageMap.cc b/src/tools/rbd_mirror/ImageMap.cc index 4493e073b98bc..d352fcb2c1403 100644 --- a/src/tools/rbd_mirror/ImageMap.cc +++ b/src/tools/rbd_mirror/ImageMap.cc @@ -416,7 +416,7 @@ void ImageMap::update_images_removed( to_remove.emplace_back(global_image_id, info.instance_id); } - if (image_mapped && image_removed) { + if (image_removed) { // local and peer images have been deleted if (m_policy->remove_image(global_image_id)) { schedule_action(global_image_id); From ff60aec2d9efa1842383ba0a5c3bd6b5a29389c6 Mon Sep 17 00:00:00 2001 From: Arthur Outhenin-Chalandre Date: Wed, 28 Jul 2021 14:14:47 +0200 Subject: [PATCH 0027/2866] rbd-mirror: fix bootstrap sequence while the image is removed If the image is being removed the PrepareRemoteImageRequest was returning the same error if the image was disabled or non primary which doesn't allow the BootstrapRequest to have the correct error handling. This commit fix this behavior by considering that the remote image is already deleted if the image is in disabling state. Signed-off-by: Arthur Outhenin-Chalandre --- src/test/rbd_mirror/test_ImageReplayer.cc | 2 +- .../rbd_mirror/image_replayer/PrepareRemoteImageRequest.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/rbd_mirror/test_ImageReplayer.cc b/src/test/rbd_mirror/test_ImageReplayer.cc index cb21fc7239619..76a6919d5adbb 100644 --- a/src/test/rbd_mirror/test_ImageReplayer.cc +++ b/src/test/rbd_mirror/test_ImageReplayer.cc @@ -664,7 +664,7 @@ TYPED_TEST(TestImageReplayer, BootstrapMirrorDisabling) this->create_replayer(); C_SaferCond cond; this->m_replayer->start(&cond); - ASSERT_EQ(-EREMOTEIO, cond.wait()); + ASSERT_EQ(-ENOENT, cond.wait()); ASSERT_TRUE(this->m_replayer->is_stopped()); } diff --git a/src/tools/rbd_mirror/image_replayer/PrepareRemoteImageRequest.cc b/src/tools/rbd_mirror/image_replayer/PrepareRemoteImageRequest.cc index 54a9dbf1650d5..08d3dd7806fc3 100644 --- a/src/tools/rbd_mirror/image_replayer/PrepareRemoteImageRequest.cc +++ b/src/tools/rbd_mirror/image_replayer/PrepareRemoteImageRequest.cc @@ -112,7 +112,7 @@ void PrepareRemoteImageRequest::handle_get_mirror_info(int r) { return; } else if (m_mirror_image.state == cls::rbd::MIRROR_IMAGE_STATE_DISABLING) { dout(5) << "remote image mirroring is being disabled" << dendl; - finish(-EREMOTEIO); + finish(-ENOENT); return; } else if (m_promotion_state != librbd::mirror::PROMOTION_STATE_PRIMARY && (state_builder == nullptr || From 965bc4150eafc8e3bbe69f63beea9c7fbb20ceb6 Mon Sep 17 00:00:00 2001 From: Arthur Outhenin-Chalandre Date: Thu, 29 Jul 2021 11:54:45 +0200 Subject: [PATCH 0028/2866] rbd-mirror: handle disabling/creating image in PrepareLocalImageRequest Signed-off-by: Arthur Outhenin-Chalandre --- .../test_mock_PrepareLocalImageRequest.cc | 100 ++++++++++++++++++ src/tools/rbd_mirror/ImageReplayer.cc | 3 + .../PrepareLocalImageRequest.cc | 32 +++++- .../image_replayer/PrepareLocalImageRequest.h | 7 ++ 4 files changed, 139 insertions(+), 3 deletions(-) diff --git a/src/test/rbd_mirror/image_replayer/test_mock_PrepareLocalImageRequest.cc b/src/test/rbd_mirror/image_replayer/test_mock_PrepareLocalImageRequest.cc index 0b0a87e8de52d..54c3b24efac96 100644 --- a/src/test/rbd_mirror/image_replayer/test_mock_PrepareLocalImageRequest.cc +++ b/src/test/rbd_mirror/image_replayer/test_mock_PrepareLocalImageRequest.cc @@ -5,6 +5,7 @@ #include "cls/rbd/cls_rbd_types.h" #include "librbd/journal/TypeTraits.h" #include "librbd/mirror/GetInfoRequest.h" +#include "tools/rbd_mirror/ImageDeleter.h" #include "tools/rbd_mirror/image_replayer/GetMirrorImageIdRequest.h" #include "tools/rbd_mirror/image_replayer/PrepareLocalImageRequest.h" #include "tools/rbd_mirror/image_replayer/StateBuilder.h" @@ -70,6 +71,28 @@ GetInfoRequest* GetInfoRequest +struct ImageDeleter { + static ImageDeleter* s_instance; + + static void trash_move(librados::IoCtx& local_io_ctx, + const std::string& global_image_id, bool resync, + librbd::asio::ContextWQ* work_queue, + Context* on_finish) { + ceph_assert(s_instance != nullptr); + s_instance->trash_move(global_image_id, resync, on_finish); + } + + MOCK_METHOD3(trash_move, void(const std::string&, bool, Context*)); + + ImageDeleter() { + s_instance = this; + } +}; + +ImageDeleter* ImageDeleter::s_instance = nullptr; + namespace image_replayer { template <> @@ -176,6 +199,7 @@ using ::testing::WithArgs; class TestMockImageReplayerPrepareLocalImageRequest : public TestMockFixture { public: + typedef ImageDeleter MockImageDeleter; typedef PrepareLocalImageRequest MockPrepareLocalImageRequest; typedef GetMirrorImageIdRequest MockGetMirrorImageIdRequest; typedef StateBuilder MockStateBuilder; @@ -222,6 +246,17 @@ class TestMockImageReplayerPrepareLocalImageRequest : public TestMockFixture { mock_get_mirror_info_request.on_finish, r); })); } + + void expect_trash_move(MockImageDeleter& mock_image_deleter, + const std::string& global_image_id, + bool ignore_orphan, int r) { + EXPECT_CALL(mock_image_deleter, + trash_move(global_image_id, ignore_orphan, _)) + .WillOnce(WithArg<2>(Invoke([this, r](Context* ctx) { + m_threads->work_queue->queue(ctx, r); + }))); + } + }; TEST_F(TestMockImageReplayerPrepareLocalImageRequest, SuccessJournal) { @@ -400,6 +435,71 @@ TEST_F(TestMockImageReplayerPrepareLocalImageRequest, MirrorImageInfoError) { ASSERT_EQ(-EINVAL, ctx.wait()); } +TEST_F(TestMockImageReplayerPrepareLocalImageRequest, ImageCreating) { + InSequence seq; + MockGetMirrorImageIdRequest mock_get_mirror_image_id_request; + expect_get_mirror_image_id(mock_get_mirror_image_id_request, "local image id", + 0); + expect_dir_get_name(m_local_io_ctx, "local image name", 0); + + MockGetMirrorInfoRequest mock_get_mirror_info_request; + expect_get_mirror_info(mock_get_mirror_info_request, + {cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT, + "global image id", + cls::rbd::MIRROR_IMAGE_STATE_CREATING}, + librbd::mirror::PROMOTION_STATE_NON_PRIMARY, + "remote mirror uuid", 0); + + MockImageDeleter mock_image_deleter; + expect_trash_move(mock_image_deleter, "global image id", false, 0); + + MockSnapshotStateBuilder mock_journal_state_builder; + MockStateBuilder* mock_state_builder = nullptr; + std::string local_image_name; + C_SaferCond ctx; + auto req = MockPrepareLocalImageRequest::create(m_local_io_ctx, + "global image id", + &local_image_name, + &mock_state_builder, + m_threads->work_queue, + &ctx); + req->send(); + + ASSERT_EQ(-ENOENT, ctx.wait()); + ASSERT_TRUE(mock_state_builder == nullptr); +} + +TEST_F(TestMockImageReplayerPrepareLocalImageRequest, ImageDisabling) { + InSequence seq; + MockGetMirrorImageIdRequest mock_get_mirror_image_id_request; + expect_get_mirror_image_id(mock_get_mirror_image_id_request, "local image id", + 0); + expect_dir_get_name(m_local_io_ctx, "local image name", 0); + + MockGetMirrorInfoRequest mock_get_mirror_info_request; + expect_get_mirror_info(mock_get_mirror_info_request, + {cls::rbd::MIRROR_IMAGE_MODE_SNAPSHOT, + "global image id", + cls::rbd::MIRROR_IMAGE_STATE_DISABLING}, + librbd::mirror::PROMOTION_STATE_NON_PRIMARY, + "remote mirror uuid", 0); + + MockSnapshotStateBuilder mock_journal_state_builder; + MockStateBuilder* mock_state_builder = nullptr; + std::string local_image_name; + C_SaferCond ctx; + auto req = MockPrepareLocalImageRequest::create(m_local_io_ctx, + "global image id", + &local_image_name, + &mock_state_builder, + m_threads->work_queue, + &ctx); + req->send(); + + ASSERT_EQ(-ERESTART, ctx.wait()); + ASSERT_TRUE(mock_state_builder == nullptr); +} + } // namespace image_replayer } // namespace mirror } // namespace rbd diff --git a/src/tools/rbd_mirror/ImageReplayer.cc b/src/tools/rbd_mirror/ImageReplayer.cc index 9546d09fce1a2..f04dc68d68fa4 100644 --- a/src/tools/rbd_mirror/ImageReplayer.cc +++ b/src/tools/rbd_mirror/ImageReplayer.cc @@ -397,6 +397,9 @@ void ImageReplayer::handle_bootstrap(int r) { m_delete_requested = true; on_start_fail(0, "remote image no longer exists"); return; + } else if (r == -ERESTART) { + on_start_fail(r, "image in transient state, try again"); + return; } else if (r < 0) { on_start_fail(r, "error bootstrapping replay"); return; diff --git a/src/tools/rbd_mirror/image_replayer/PrepareLocalImageRequest.cc b/src/tools/rbd_mirror/image_replayer/PrepareLocalImageRequest.cc index a64117425d93f..b1fef7254760a 100644 --- a/src/tools/rbd_mirror/image_replayer/PrepareLocalImageRequest.cc +++ b/src/tools/rbd_mirror/image_replayer/PrepareLocalImageRequest.cc @@ -10,6 +10,7 @@ #include "librbd/Journal.h" #include "librbd/Utils.h" #include "librbd/mirror/GetInfoRequest.h" +#include "tools/rbd_mirror/ImageDeleter.h" #include "tools/rbd_mirror/Threads.h" #include "tools/rbd_mirror/image_replayer/GetMirrorImageIdRequest.h" #include "tools/rbd_mirror/image_replayer/journal/StateBuilder.h" @@ -124,9 +125,16 @@ void PrepareLocalImageRequest::handle_get_mirror_info(int r) { return; } - // TODO save current mirror state to determine if we should - // delete a partially formed image - // (e.g. MIRROR_IMAGE_STATE_CREATING/DELETING) + if (m_mirror_image.state == cls::rbd::MIRROR_IMAGE_STATE_CREATING) { + dout(5) << "local image is still in creating state, issuing a removal" + << dendl; + move_to_trash(); + return; + } else if (m_mirror_image.state == cls::rbd::MIRROR_IMAGE_STATE_DISABLING) { + dout(5) << "local image mirroring is in disabling state" << dendl; + finish(-ERESTART); + return; + } switch (m_mirror_image.mode) { case cls::rbd::MIRROR_IMAGE_MODE_JOURNAL: @@ -156,6 +164,24 @@ void PrepareLocalImageRequest::handle_get_mirror_info(int r) { finish(0); } +template +void PrepareLocalImageRequest::move_to_trash() { + dout(10) << dendl; + + Context *ctx = create_context_callback< + PrepareLocalImageRequest, + &PrepareLocalImageRequest::handle_move_to_trash>(this); + ImageDeleter::trash_move(m_io_ctx, m_global_image_id, + false, m_work_queue, ctx); +} + +template +void PrepareLocalImageRequest::handle_move_to_trash(int r) { + dout(10) << ": r=" << r << dendl; + + finish(-ENOENT); +} + template void PrepareLocalImageRequest::finish(int r) { dout(10) << "r=" << r << dendl; diff --git a/src/tools/rbd_mirror/image_replayer/PrepareLocalImageRequest.h b/src/tools/rbd_mirror/image_replayer/PrepareLocalImageRequest.h index 8285ca060b3e3..6372169ff0550 100644 --- a/src/tools/rbd_mirror/image_replayer/PrepareLocalImageRequest.h +++ b/src/tools/rbd_mirror/image_replayer/PrepareLocalImageRequest.h @@ -67,6 +67,10 @@ class PrepareLocalImageRequest { * v * GET_MIRROR_INFO * | + * | (if the image mirror state is CREATING) + * v + * TRASH_MOVE + * | * v * * @@ -95,6 +99,9 @@ class PrepareLocalImageRequest { void get_mirror_info(); void handle_get_mirror_info(int r); + void move_to_trash(); + void handle_move_to_trash(int r); + void finish(int r); }; From b664a95889b35d0d3afbd8428e3af4976d7f81eb Mon Sep 17 00:00:00 2001 From: Arthur Outhenin-Chalandre Date: Fri, 6 Aug 2021 15:54:38 +0200 Subject: [PATCH 0029/2866] rbd-mirror: remove image_map next_state if sets to the same state In some cases, set_state is called with DISSOCIATING, then ASSOCIATING and DISSOCIATING again. In this case the state DISSOCIATING is processed to remove the image and then schedule the next action which is associating. To fix this case, this commit removes the next_state if the state is sets to the same state. Signed-off-by: Arthur Outhenin-Chalandre --- src/tools/rbd_mirror/image_map/Policy.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/tools/rbd_mirror/image_map/Policy.cc b/src/tools/rbd_mirror/image_map/Policy.cc index ccf188742a107..62fbd12dc0bc8 100644 --- a/src/tools/rbd_mirror/image_map/Policy.cc +++ b/src/tools/rbd_mirror/image_map/Policy.cc @@ -380,6 +380,7 @@ bool Policy::can_shuffle_image(const std::string &global_image_id) { bool Policy::set_state(ImageState* image_state, StateTransition::State state, bool ignore_current_state) { if (!ignore_current_state && image_state->state == state) { + image_state->next_state = boost::none; return false; } else if (StateTransition::is_idle(image_state->state)) { image_state->state = state; From 6a4e328161e4d692bf45605d62ca8fab2fee7670 Mon Sep 17 00:00:00 2001 From: Igor Fedotov Date: Mon, 23 Aug 2021 17:54:03 +0300 Subject: [PATCH 0030/2866] os/bluestore: dump alloc unit size on bluefs allocation failure. Signed-off-by: Igor Fedotov --- src/os/bluestore/BlueFS.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/os/bluestore/BlueFS.cc b/src/os/bluestore/BlueFS.cc index c991d66c3f7fb..914da608f91f2 100644 --- a/src/os/bluestore/BlueFS.cc +++ b/src/os/bluestore/BlueFS.cc @@ -3017,6 +3017,7 @@ int BlueFS::_allocate_without_fallback(uint8_t id, uint64_t len, << ", allocator type " << alloc[id]->get_type() << ", capacity 0x" << alloc[id]->get_capacity() << ", block size 0x" << alloc[id]->get_block_size() + << ", alloc size 0x" << alloc_size[id] << ", free 0x" << alloc[id]->get_free() << ", fragmentation " << alloc[id]->get_fragmentation() << ", allocated 0x" << (alloc_len > 0 ? alloc_len : 0) @@ -3060,6 +3061,7 @@ int BlueFS::_allocate(uint8_t id, uint64_t len, << ", allocator type " << alloc[id]->get_type() << ", capacity 0x" << alloc[id]->get_capacity() << ", block size 0x" << alloc[id]->get_block_size() + << ", alloc size 0x" << alloc_size[id] << ", free 0x" << alloc[id]->get_free() << ", fragmentation " << alloc[id]->get_fragmentation() << ", allocated 0x" << (alloc_len > 0 ? alloc_len : 0) From b9f462f97848630d1cd85c943c42f161894b2efe Mon Sep 17 00:00:00 2001 From: Mumuni Mohammed <43080869+Kalkulus1@users.noreply.github.com> Date: Sun, 22 Aug 2021 16:25:20 +0000 Subject: [PATCH 0031/2866] doc: adds a way to allow users set_contents_from_filename for rgw Signed-off-by: Mumuni Mohammed --- doc/radosgw/s3/python.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/doc/radosgw/s3/python.rst b/doc/radosgw/s3/python.rst index 9f9367e0fcb03..6fb314378aced 100644 --- a/doc/radosgw/s3/python.rst +++ b/doc/radosgw/s3/python.rst @@ -106,6 +106,17 @@ This creates a file ``hello.txt`` with the string ``"Hello World!"`` key = bucket.new_key('hello.txt') key.set_contents_from_string('Hello World!') + + +Uploading an Object or a File +----------------------------- + +This creates a file ``logo.png`` with the contents from the file ``"logo.png"`` + +.. code-block:: python + + key = bucket.new_key('logo.png') + key.set_contents_from_filename('logo.png') Change an Object's ACL From 8ed8e8cd485c01dd9c5eb94f62dd5d1868881d3a Mon Sep 17 00:00:00 2001 From: Mark Kogan Date: Mon, 30 Aug 2021 19:01:53 +0300 Subject: [PATCH 0032/2866] rgw: d3n: fix d3n cached objects on disk lru eviction entering infinite loop fixes: https://tracker.ceph.com/issues/52457 Signed-off-by: Mark Kogan --- src/common/options/rgw.yaml.in | 2 +- src/rgw/rgw_d3n_cacherequest.h | 2 +- src/rgw/rgw_d3n_datacache.cc | 43 ++++++++++++++++++++-------------- 3 files changed, 28 insertions(+), 19 deletions(-) diff --git a/src/common/options/rgw.yaml.in b/src/common/options/rgw.yaml.in index c7498481d1981..f8d935854b6a2 100644 --- a/src/common/options/rgw.yaml.in +++ b/src/common/options/rgw.yaml.in @@ -3134,7 +3134,7 @@ options: type: size level: advanced desc: datacache maximum size on disk in bytes - default: 1048576 + default: 1_G services: - rgw with_legacy: true diff --git a/src/rgw/rgw_d3n_cacherequest.h b/src/rgw/rgw_d3n_cacherequest.h index 41e026bcd3453..d4acc4c0b1f8f 100644 --- a/src/rgw/rgw_d3n_cacherequest.h +++ b/src/rgw/rgw_d3n_cacherequest.h @@ -26,7 +26,7 @@ struct D3nGetObjData { struct D3nL1CacheRequest { ~D3nL1CacheRequest() { - lsubdout(g_ceph_context, rgw_datacache, 30) << "D3nDataCache: " << __func__ << "(): Read From Cache, comlete" << dendl; + lsubdout(g_ceph_context, rgw_datacache, 30) << "D3nDataCache: " << __func__ << "(): Read From Cache, complete" << dendl; } // unique_ptr with custom deleter for struct aiocb diff --git a/src/rgw/rgw_d3n_datacache.cc b/src/rgw/rgw_d3n_datacache.cc index e2ad497dae977..3ca7e11ea2ad7 100644 --- a/src/rgw/rgw_d3n_datacache.cc +++ b/src/rgw/rgw_d3n_datacache.cc @@ -161,9 +161,8 @@ void D3nDataCache::d3n_libaio_write_completion_cb(D3nCacheAioWriteRequest* c) { // update cache_map entries for new chunk in cache const std::lock_guard l(d3n_cache_lock); auto it = d3n_outstanding_write_list.find(c->oid); - if (it != d3n_outstanding_write_list.end()) { + if (it != d3n_outstanding_write_list.end()) d3n_outstanding_write_list.erase(it); - } chunk_info = new D3nChunkDataInfo; chunk_info->oid = c->oid; chunk_info->set_ctx(cct); @@ -211,10 +210,15 @@ int D3nDataCache::d3n_libaio_create_write_request(bufferlist& bl, unsigned int l void D3nDataCache::put(bufferlist& bl, unsigned int len, std::string& oid) { - int r = 0; + size_t sr = 0; uint64_t freed_size = 0, _free_data_cache_size = 0, _outstanding_write_size = 0; - ldout(cct, 10) << "D3nDataCache::" << __func__ << "(): oid=" << oid << dendl; + ldout(cct, 10) << "D3nDataCache::" << __func__ << "(): oid=" << oid << ", len=" << len << dendl; + if (len > cct->_conf->rgw_d3n_l1_datacache_size) { + ldout(cct, 2) << "D3nDataCache: Warning: object oid=" << oid << ", length=" << len << + " is larger than the datacache size " << cct->_conf->rgw_d3n_l1_datacache_size << ", not writing to cache" << dendl; + return; + } { const std::lock_guard l(d3n_cache_lock); std::unordered_map::iterator iter = d3n_cache_map.find(oid); @@ -235,27 +239,33 @@ void D3nDataCache::put(bufferlist& bl, unsigned int len, std::string& oid) _outstanding_write_size = outstanding_write_size; } ldout(cct, 20) << "D3nDataCache: Before eviction _free_data_cache_size:" << _free_data_cache_size << ", _outstanding_write_size:" << _outstanding_write_size << ", freed_size:" << freed_size << dendl; - while (len >= (_free_data_cache_size - _outstanding_write_size + freed_size)) { - ldout(cct, 20) << "D3nDataCache: enter eviction, r=" << r << dendl; + while (len > (_free_data_cache_size - _outstanding_write_size + freed_size)) { + ldout(cct, 20) << "D3nDataCache: enter eviction" << dendl; if (eviction_policy == _eviction_policy::LRU) { - r = lru_eviction(); + sr = lru_eviction(); } else if (eviction_policy == _eviction_policy::RANDOM) { - r = random_eviction(); + sr = random_eviction(); } else { ldout(cct, 0) << "D3nDataCache: Warning: unknown cache eviction policy, defaulting to lru eviction" << dendl; - r = lru_eviction(); + sr = lru_eviction(); } - if (r < 0) + if (sr == 0) { + ldout(cct, 2) << "D3nDataCache: Warning: eviction was not able to free disk space, not writing to cache" << dendl; + auto it = d3n_outstanding_write_list.find(oid); + if (it != d3n_outstanding_write_list.end()) + d3n_outstanding_write_list.erase(it); return; - freed_size += r; + } + ldout(cct, 20) << "D3nDataCache: completed eviction of " << sr << " bytes" << dendl; + freed_size += sr; } + int r = 0; r = d3n_libaio_create_write_request(bl, len, oid); if (r < 0) { const std::lock_guard l(d3n_cache_lock); auto it = d3n_outstanding_write_list.find(oid); - if (it != d3n_outstanding_write_list.end()) { + if (it != d3n_outstanding_write_list.end()) d3n_outstanding_write_list.erase(it); - } ldout(cct, 1) << "D3nDataCache: create_aio_write_request fail, r=" << r << dendl; return; } @@ -324,7 +334,7 @@ size_t D3nDataCache::random_eviction() } location = cache_location + del_oid; - remove(location.c_str()); + ::remove(location.c_str()); return freed_size; } @@ -356,13 +366,12 @@ size_t D3nDataCache::lru_eviction() del_oid = del_entry->oid; ldout(cct, 20) << "D3nDataCache: lru_eviction: oid to remove: " << del_oid << dendl; std::unordered_map::iterator iter = d3n_cache_map.find(del_oid); - if (iter != d3n_cache_map.end()) { + if (iter != d3n_cache_map.end()) d3n_cache_map.erase(iter); // oid - } } freed_size = del_entry->size; delete del_entry; location = cache_location + del_oid; - remove(location.c_str()); + ::remove(location.c_str()); return freed_size; } From 0facfac91fd8f71e5a8b869d818e7c2b07b93516 Mon Sep 17 00:00:00 2001 From: Daniel Pivonka Date: Thu, 12 Aug 2021 14:25:23 -0400 Subject: [PATCH 0033/2866] mgr/cephadm: update host when adding host that already exists update addr and labels of existing host this prevents labels being removed if a host is readded or addr being set to 127.0.... if readding host active mgr is on Fixes: https://tracker.ceph.com/issues/51667 Signed-off-by: Daniel Pivonka --- src/pybind/mgr/cephadm/inventory.py | 23 +++++++++++++++++++++-- src/pybind/mgr/cephadm/module.py | 8 ++++++++ 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/src/pybind/mgr/cephadm/inventory.py b/src/pybind/mgr/cephadm/inventory.py index cd398221123e5..d4db7761c8a9e 100644 --- a/src/pybind/mgr/cephadm/inventory.py +++ b/src/pybind/mgr/cephadm/inventory.py @@ -93,8 +93,16 @@ def assert_host(self, host: str) -> None: raise OrchestratorError('host %s does not exist' % host) def add_host(self, spec: HostSpec) -> None: - self._inventory[spec.hostname] = spec.to_json() - self.save() + if spec.hostname in self._inventory: + # addr + if self.get_addr(spec.hostname) != spec.addr: + self.set_addr(spec.hostname, spec.addr) + # labels + for label in spec.labels: + self.add_label(spec.hostname, label) + else: + self._inventory[spec.hostname] = spec.to_json() + self.save() def rm_host(self, host: str) -> None: self.assert_host(host) @@ -602,6 +610,17 @@ def prime_empty_host(self, host): self.registry_login_queue.add(host) self.last_client_files[host] = {} + def refresh_all_host_info(self, host): + # type: (str) -> None + + self.last_host_check.pop(host, None) + self.daemon_refresh_queue.append(host) + self.registry_login_queue.add(host) + self.device_refresh_queue.append(host) + self.last_facts_update.pop(host, None) + self.osdspec_previews_refresh_queue.append(host) + self.last_autotune.pop(host, None) + def invalidate_host_daemons(self, host): # type: (str) -> None self.daemon_refresh_queue.append(host) diff --git a/src/pybind/mgr/cephadm/module.py b/src/pybind/mgr/cephadm/module.py index e56987c93c55f..c6fddc271b795 100644 --- a/src/pybind/mgr/cephadm/module.py +++ b/src/pybind/mgr/cephadm/module.py @@ -1398,6 +1398,11 @@ def _schedulable_hosts(self) -> List[HostSpec]: ] def _check_valid_addr(self, host: str, addr: str) -> str: + # make sure mgr is not resolving own ip + if addr in self.get_mgr_id(): + raise OrchestratorError( + "Can not automatically resolve ip address of host where active mgr is running. Please explicitly provide the address.") + # make sure hostname is resolvable before trying to make a connection try: ip_addr = utils.resolve_ip(addr) @@ -1445,6 +1450,9 @@ def _add_host(self, spec): if spec.addr == spec.hostname and ip_addr: spec.addr = ip_addr + if spec.hostname in self.inventory and self.inventory.get_addr(spec.hostname) != spec.addr: + self.cache.refresh_all_host_info(spec.hostname) + # prime crush map? if spec.location: self.check_mon_command({ From bf8e5fe4f066589ddfb1afb345ef2cc3079feb31 Mon Sep 17 00:00:00 2001 From: Joseph Sawaya Date: Mon, 30 Aug 2021 12:00:58 -0400 Subject: [PATCH 0034/2866] mgr/rook: fix orch ls to display OSD info This commit adds OSDs to the describe_service method in the RookOrchestrator class to display the information about running OSDs when a user does `ceph orch ls`. Signed-off-by: Joseph Sawaya --- src/pybind/mgr/rook/module.py | 14 +++++++++++++- src/pybind/mgr/rook/rook_cluster.py | 4 ++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/pybind/mgr/rook/module.py b/src/pybind/mgr/rook/module.py index bf6f673b302d9..0b6badcff08b9 100644 --- a/src/pybind/mgr/rook/module.py +++ b/src/pybind/mgr/rook/module.py @@ -348,7 +348,19 @@ def describe_service(self, size=active, last_refresh=now, ) - + if service_type == 'osd' or service_type is None: + # OSDs + all_osds = self.rook_cluster.get_osds() + svc = 'osd' + spec[svc] = orchestrator.ServiceDescription( + spec=DriveGroupSpec( + service_type='osd', + placement=PlacementSpec(count=len(all_osds), hosts=[osd.metadata.labels['topology-location-host'] for osd in all_osds]), + ), + size=len(all_osds), + last_refresh=now, + running= sum(osd.status.phase == 'Running' for osd in all_osds) + ) for dd in self._list_daemons(): if dd.service_name() not in spec: continue diff --git a/src/pybind/mgr/rook/rook_cluster.py b/src/pybind/mgr/rook/rook_cluster.py index 5a13d39294877..2fa86c9138a8f 100644 --- a/src/pybind/mgr/rook/rook_cluster.py +++ b/src/pybind/mgr/rook/rook_cluster.py @@ -703,6 +703,10 @@ def get_discovered_devices(self, nodenames: Optional[List[str]] = None) -> Dict[ self.fetcher.fetch() return self.fetcher.devices() + def get_osds(self) -> List: + osd_pods: KubernetesResource = KubernetesResource(self.coreV1_api.list_namespaced_pod, namespace='rook-ceph', label_selector='app=rook-ceph-osd') + return list(osd_pods.items) + def get_nfs_conf_url(self, nfs_cluster: str, instance: str) -> Optional[str]: # # Fetch cephnfs object for "nfs_cluster" and then return a rados:// From 82c9fff0f8d5e4fc41066979738a4eb6b38324e4 Mon Sep 17 00:00:00 2001 From: Mark Kogan Date: Wed, 1 Sep 2021 12:25:41 +0300 Subject: [PATCH 0035/2866] rgw: d3n: d3n on disk lru eviction:addressing comments Signed-off-by: Mark Kogan --- src/rgw/rgw_d3n_datacache.cc | 21 ++++----------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/src/rgw/rgw_d3n_datacache.cc b/src/rgw/rgw_d3n_datacache.cc index 3ca7e11ea2ad7..28c6a2cb2da3d 100644 --- a/src/rgw/rgw_d3n_datacache.cc +++ b/src/rgw/rgw_d3n_datacache.cc @@ -160,9 +160,7 @@ void D3nDataCache::d3n_libaio_write_completion_cb(D3nCacheAioWriteRequest* c) { // update cache_map entries for new chunk in cache const std::lock_guard l(d3n_cache_lock); - auto it = d3n_outstanding_write_list.find(c->oid); - if (it != d3n_outstanding_write_list.end()) - d3n_outstanding_write_list.erase(it); + d3n_outstanding_write_list.erase(c->oid); chunk_info = new D3nChunkDataInfo; chunk_info->oid = c->oid; chunk_info->set_ctx(cct); @@ -214,11 +212,6 @@ void D3nDataCache::put(bufferlist& bl, unsigned int len, std::string& oid) uint64_t freed_size = 0, _free_data_cache_size = 0, _outstanding_write_size = 0; ldout(cct, 10) << "D3nDataCache::" << __func__ << "(): oid=" << oid << ", len=" << len << dendl; - if (len > cct->_conf->rgw_d3n_l1_datacache_size) { - ldout(cct, 2) << "D3nDataCache: Warning: object oid=" << oid << ", length=" << len << - " is larger than the datacache size " << cct->_conf->rgw_d3n_l1_datacache_size << ", not writing to cache" << dendl; - return; - } { const std::lock_guard l(d3n_cache_lock); std::unordered_map::iterator iter = d3n_cache_map.find(oid); @@ -251,9 +244,7 @@ void D3nDataCache::put(bufferlist& bl, unsigned int len, std::string& oid) } if (sr == 0) { ldout(cct, 2) << "D3nDataCache: Warning: eviction was not able to free disk space, not writing to cache" << dendl; - auto it = d3n_outstanding_write_list.find(oid); - if (it != d3n_outstanding_write_list.end()) - d3n_outstanding_write_list.erase(it); + d3n_outstanding_write_list.erase(oid); return; } ldout(cct, 20) << "D3nDataCache: completed eviction of " << sr << " bytes" << dendl; @@ -263,9 +254,7 @@ void D3nDataCache::put(bufferlist& bl, unsigned int len, std::string& oid) r = d3n_libaio_create_write_request(bl, len, oid); if (r < 0) { const std::lock_guard l(d3n_cache_lock); - auto it = d3n_outstanding_write_list.find(oid); - if (it != d3n_outstanding_write_list.end()) - d3n_outstanding_write_list.erase(it); + d3n_outstanding_write_list.erase(oid); ldout(cct, 1) << "D3nDataCache: create_aio_write_request fail, r=" << r << dendl; return; } @@ -365,9 +354,7 @@ size_t D3nDataCache::lru_eviction() } del_oid = del_entry->oid; ldout(cct, 20) << "D3nDataCache: lru_eviction: oid to remove: " << del_oid << dendl; - std::unordered_map::iterator iter = d3n_cache_map.find(del_oid); - if (iter != d3n_cache_map.end()) - d3n_cache_map.erase(iter); // oid + d3n_cache_map.erase(del_oid); // oid } freed_size = del_entry->size; delete del_entry; From 870b0292352f62e7ab714a00b71262df2ec8cae8 Mon Sep 17 00:00:00 2001 From: wangyunqing Date: Tue, 30 Jun 2020 20:43:12 +0800 Subject: [PATCH 0036/2866] rgw: add try catch for user policies decode Signed-off-by: wangyunqing --- src/rgw/rgw_rest_user_policy.cc | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/src/rgw/rgw_rest_user_policy.cc b/src/rgw/rgw_rest_user_policy.cc index a552f3b2b2a2e..86e30a30411c1 100644 --- a/src/rgw/rgw_rest_user_policy.cc +++ b/src/rgw/rgw_rest_user_policy.cc @@ -156,6 +156,9 @@ void RGWPutUserPolicy::execute(optional_yield y) if (op_ret < 0) { op_ret = -ERR_INTERNAL_ERROR; } + } catch (buffer::error& err) { + ldpp_dout(this, 0) << "ERROR: failed to decode user policies" << dendl; + op_ret = -EIO; } catch (rgw::IAM::PolicyParseException& e) { ldpp_dout(this, 20) << "failed to parse policy: " << e.what() << dendl; op_ret = -ERR_MALFORMED_DOC; @@ -213,7 +216,13 @@ void RGWGetUserPolicy::execute(optional_yield y) map policies; if (auto it = user->get_attrs().find(RGW_ATTR_USER_POLICY); it != user->get_attrs().end()) { bufferlist bl = it->second; - decode(policies, bl); + try { + decode(policies, bl); + } catch (buffer::error& err) { + ldpp_dout(this, 0) << "ERROR: failed to decode user policies" << dendl; + op_ret = -EIO; + return; + } if (auto it = policies.find(policy_name); it != policies.end()) { policy = policies[policy_name]; dump(s->formatter); @@ -276,7 +285,13 @@ void RGWListUserPolicies::execute(optional_yield y) s->formatter->close_section(); s->formatter->open_object_section("ListUserPoliciesResult"); bufferlist bl = it->second; - decode(policies, bl); + try { + decode(policies, bl); + } catch (buffer::error& err) { + ldpp_dout(this, 0) << "ERROR: failed to decode user policies" << dendl; + op_ret = -EIO; + return; + } for (const auto& p : policies) { s->formatter->open_object_section("PolicyNames"); s->formatter->dump_string("member", p.first); @@ -348,7 +363,13 @@ void RGWDeleteUserPolicy::execute(optional_yield y) map policies; if (auto it = user->get_attrs().find(RGW_ATTR_USER_POLICY); it != user->get_attrs().end()) { bufferlist out_bl = it->second; - decode(policies, out_bl); + try { + decode(policies, out_bl); + } catch (buffer::error& err) { + ldpp_dout(this, 0) << "ERROR: failed to decode user policies" << dendl; + op_ret = -EIO; + return; + } if (auto p = policies.find(policy_name); p != policies.end()) { bufferlist in_bl; From 7e961d3ebd6cc39b38691bf6b4b7074db175c37b Mon Sep 17 00:00:00 2001 From: wangyunqing Date: Mon, 23 Nov 2020 11:19:55 +0800 Subject: [PATCH 0037/2866] rgw : check the object size when copy obj Fixes: https://tracker.ceph.com/issues/48322 Signed-off-by: wangyunqing --- src/rgw/rgw_op.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc index 64f4fc0e64030..def5c1dba1ff9 100644 --- a/src/rgw/rgw_op.cc +++ b/src/rgw/rgw_op.cc @@ -5414,6 +5414,10 @@ void RGWCopyObj::execute(optional_yield y) obj_size = astate->size; if (!s->system_request) { // no quota enforcement for system requests + if (astate->accounted_size > static_cast(s->cct->_conf->rgw_max_put_size)) { + op_ret = -ERR_TOO_LARGE; + return; + } // enforce quota against the destination bucket owner op_ret = dest_bucket->check_quota(this, user_quota, bucket_quota, astate->accounted_size, y); From 465f7e8fa34b37cf09e4d280da5239086924b942 Mon Sep 17 00:00:00 2001 From: jianglong01 Date: Tue, 7 Sep 2021 15:49:04 +0800 Subject: [PATCH 0038/2866] mgr/cephadm: the display of 'orch osd rm status' is incorrect After exec 'ceph orch osd rm --replace', 'orch osd rm status' has an wrong display. wrong display: OSD_ID HOST STATE PG_COUNT REPLACE FORCE DRAIN_STARTED_AT 15 cephqa08.cpp.zzbm.qianxin-inc.cn draining 0 True True 2021-09-07 07:54:05.776906 correct display: OSD_ID HOST STATE PG_COUNT REPLACE FORCE DRAIN_STARTED_AT 15 cephqa08.cpp.zzbm.qianxin-inc.cn draining 0 True False 2021-09-07 07:35:34.731417 Signed-off-by: jianglong01 --- src/pybind/mgr/orchestrator/module.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pybind/mgr/orchestrator/module.py b/src/pybind/mgr/orchestrator/module.py index d75271fbff6fc..234f8c50f66af 100644 --- a/src/pybind/mgr/orchestrator/module.py +++ b/src/pybind/mgr/orchestrator/module.py @@ -858,7 +858,7 @@ def _osd_rm_status(self, format: Format = Format.plain) -> HandleCommandResult: table.right_padding_width = 2 for osd in sorted(report, key=lambda o: o.osd_id): table.add_row([osd.osd_id, osd.hostname, osd.drain_status_human(), - osd.get_pg_count(), osd.replace, osd.replace, osd.drain_started_at]) + osd.get_pg_count(), osd.replace, osd.force, osd.drain_started_at]) out = table.get_string() return HandleCommandResult(stdout=out) From f83e0ac544a77440b538b0be303b3859125915dd Mon Sep 17 00:00:00 2001 From: Adam Kupczyk Date: Sat, 5 Jun 2021 08:55:14 +0200 Subject: [PATCH 0039/2866] os/bluestore/bluefs: Refactor flush_and_sync_log This refactor prepares flush_and_sync_log and compact_log_async for fine-grain locks in BlueFS. There is no new logic introduced, but refactor is accompanied by some new comments. Signed-off-by: Adam Kupczyk --- src/os/bluestore/BlueFS.cc | 162 +++++++++++++++++++++++++------------ src/os/bluestore/BlueFS.h | 13 ++- 2 files changed, 120 insertions(+), 55 deletions(-) diff --git a/src/os/bluestore/BlueFS.cc b/src/os/bluestore/BlueFS.cc index f34996be3e000..5a2c50359fcc4 100644 --- a/src/os/bluestore/BlueFS.cc +++ b/src/os/bluestore/BlueFS.cc @@ -2309,6 +2309,7 @@ void BlueFS::_compact_log_async(std::unique_lock& l) // 1. allocate new log space and jump to it. old_log_jump_to = log_file->fnode.get_allocated(); + uint64_t runway = log_file->fnode.get_allocated() - log_writer->get_effective_write_pos(); dout(10) << __func__ << " old_log_jump_to 0x" << std::hex << old_log_jump_to << " need 0x" << (old_log_jump_to + cct->_conf->bluefs_max_log_runway) << std::dec << dendl; int r = _allocate(vselector->select_prefer_bdev(log_file->vselector_hint), @@ -2324,9 +2325,12 @@ void BlueFS::_compact_log_async(std::unique_lock& l) log_t.op_file_update(log_file->fnode); log_t.op_jump(log_seq, old_log_jump_to); - flush_bdev(); // FIXME? + // we need to flush all bdev because we will be streaming all dirty files to log + // TODO - think - if _flush_and_sync_log_jump will not add dirty files nor release pending allocations + // then flush_bdev() will not be necessary + flush_bdev(); - _flush_and_sync_log(l, 0, old_log_jump_to); + _flush_and_sync_log_jump(old_log_jump_to, runway); // 2. prepare compacted log bluefs_transaction_t t; @@ -2455,38 +2459,17 @@ void BlueFS::_pad_bl(bufferlist& bl) } } - -int BlueFS::_flush_and_sync_log(std::unique_lock& l, - uint64_t want_seq, - uint64_t jump_to) +// Adds to log_t file modifications mentioned in `dirty_files`. +// Note: some bluefs ops may have already been stored in log_t transaction. +uint64_t BlueFS::_consume_dirty() { - while (log_flushing) { - dout(10) << __func__ << " want_seq " << want_seq - << " log is currently flushing, waiting" << dendl; - ceph_assert(!jump_to); - log_cond.wait(l); - } - if (want_seq && want_seq <= log_seq_stable) { - dout(10) << __func__ << " want_seq " << want_seq << " <= log_seq_stable " - << log_seq_stable << ", done" << dendl; - ceph_assert(!jump_to); - return 0; - } - if (log_t.empty() && dirty_files.empty()) { - dout(10) << __func__ << " want_seq " << want_seq - << " " << log_t << " not dirty, dirty_files empty, no-op" << dendl; - ceph_assert(!jump_to); - return 0; - } - - vector> to_release(pending_release.size()); - to_release.swap(pending_release); - + //acquire new seq + // this will became log_seq_stable once we write uint64_t seq = log_t.seq = ++log_seq; - ceph_assert(want_seq == 0 || want_seq <= seq); log_t.uuid = super.uuid; // log dirty files + // we just incremented log_seq. It is now illegal to add to dirty_files[log_seq] auto lsi = dirty_files.find(seq); if (lsi != dirty_files.end()) { dout(20) << __func__ << " " << lsi->second.size() << " dirty_files" << dendl; @@ -2495,21 +2478,37 @@ int BlueFS::_flush_and_sync_log(std::unique_lock& l, log_t.op_file_update(f.fnode); } } + return seq; +} - dout(10) << __func__ << " " << log_t << dendl; - ceph_assert(!log_t.empty()); - +// Extends log if its free space is smaller then bluefs_min_log_runway. +// Returns space available *BEFORE* adding new space. Signed for additional <0 detection. +int64_t BlueFS::_maybe_extend_log() +{ // allocate some more space (before we run out)? // BTW: this triggers `flush()` in the `page_aligned_appender` of `log_writer`. int64_t runway = log_writer->file->fnode.get_allocated() - log_writer->get_effective_write_pos(); - bool just_expanded_log = false; if (runway < (int64_t)cct->_conf->bluefs_min_log_runway) { dout(10) << __func__ << " allocating more log runway (0x" << std::hex << runway << std::dec << " remaining)" << dendl; - while (new_log_writer) { - dout(10) << __func__ << " waiting for async compaction" << dendl; - log_cond.wait(l); + /* + * Usually, when we are low on space in log, we just allocate new extent, + * put update op(log) to log and we are fine. + * Problem - it interferes with log compaction: + * New log produced in compaction will include - as last op - jump into some offset (anchor) of current log. + * It is assumed that log region (anchor - end) will contain all changes made by bluefs since + * full state capture into new log. + * Putting log update into (anchor - end) region is illegal, because any update there must be compatible with + * both logs, but old log is different then new log. + * + * Possible solutions: + * - stall extending log until we finish compacting and switch log (CURRENT) + * - re-run compaction with more runway for old log + * - add OP_FILE_ADDEXT that adds extent; will be compatible with both logs + */ + if (new_log_writer) { + return -EWOULDBLOCK; } vselector->sub_usage(log_writer->file->vselector_hint, log_writer->file->fnode); int r = _allocate( @@ -2519,8 +2518,13 @@ int BlueFS::_flush_and_sync_log(std::unique_lock& l, ceph_assert(r == 0); vselector->add_usage(log_writer->file->vselector_hint, log_writer->file->fnode); log_t.op_file_update(log_writer->file->fnode); - just_expanded_log = true; } + return runway; +} + +void BlueFS::_flush_and_sync_log_core(int64_t runway) +{ + dout(10) << __func__ << " " << log_t << dendl; bufferlist bl; bl.reserve(super.block_size); @@ -2532,38 +2536,29 @@ int BlueFS::_flush_and_sync_log(std::unique_lock& l, logger->inc(l_bluefs_logged_bytes, bl.length()); - if (just_expanded_log) { + if (true) { ceph_assert(bl.length() <= runway); // if we write this, we will have an unrecoverable data loss + // transaction will not fit extents before growth -> data loss on _replay } log_writer->append(bl); log_t.clear(); log_t.seq = 0; // just so debug output is less confusing - log_flushing = true; int r = _flush(log_writer, true); ceph_assert(r == 0); +} - if (jump_to) { - dout(10) << __func__ << " jumping log offset from 0x" << std::hex - << log_writer->pos << " -> 0x" << jump_to << std::dec << dendl; - log_writer->pos = jump_to; - vselector->sub_usage(log_writer->file->vselector_hint, log_writer->file->fnode.size); - log_writer->file->fnode.size = jump_to; - vselector->add_usage(log_writer->file->vselector_hint, log_writer->file->fnode.size); - } - - _flush_bdev_safely(log_writer); - - log_flushing = false; - log_cond.notify_all(); - +// Clears dirty_files up to (including) seq_stable. +void BlueFS::_clear_dirty_set_stable(uint64_t seq) +{ // clean dirty files if (seq > log_seq_stable) { log_seq_stable = seq; dout(20) << __func__ << " log_seq_stable " << log_seq_stable << dendl; + // undirty all files that were already streamed to log auto p = dirty_files.begin(); while (p != dirty_files.end()) { if (p->first > log_seq_stable) { @@ -2589,7 +2584,10 @@ int BlueFS::_flush_and_sync_log(std::unique_lock& l, << " already >= out seq " << seq << ", we lost a race against another log flush, done" << dendl; } +} +void BlueFS::_release_pending_allocations(vector>& to_release) +{ for (unsigned i = 0; i < to_release.size(); ++i) { if (!to_release[i].empty()) { /* OK, now we have the guarantee alloc[i] won't be null. */ @@ -2609,9 +2607,67 @@ int BlueFS::_flush_and_sync_log(std::unique_lock& l, } } } +} + +int BlueFS::_flush_and_sync_log(std::unique_lock& l, + uint64_t want_seq) +{ + if (want_seq && want_seq <= log_seq_stable) { + dout(10) << __func__ << " want_seq " << want_seq << " <= log_seq_stable " + << log_seq_stable << ", done" << dendl; + return 0; + } + int64_t available_runway; + do { + available_runway = _maybe_extend_log(); + if (available_runway == -EWOULDBLOCK) { + while (new_log_writer) { + dout(10) << __func__ << " waiting for async compaction" << dendl; + log_cond.wait(l); + } + } + } while (available_runway == -EWOULDBLOCK); + + ceph_assert(want_seq == 0 || want_seq <= log_seq + 1); // illegal to request seq that was not created yet + + uint64_t seq = _consume_dirty(); + vector> to_release(pending_release.size()); + to_release.swap(pending_release); + + _flush_and_sync_log_core(available_runway); + _flush_bdev_safely(log_writer); + + _clear_dirty_set_stable(seq); + _release_pending_allocations(to_release); _update_logger_stats(); + return 0; +} + +// Flushes log and immediately adjusts log_writer pos. +int BlueFS::_flush_and_sync_log_jump(uint64_t jump_to, + int64_t available_runway) +{ + ceph_assert(jump_to); + uint64_t seq = _consume_dirty(); + vector> to_release(pending_release.size()); + to_release.swap(pending_release); + + _flush_and_sync_log_core(available_runway); + + dout(10) << __func__ << " jumping log offset from 0x" << std::hex + << log_writer->pos << " -> 0x" << jump_to << std::dec << dendl; + log_writer->pos = jump_to; + vselector->sub_usage(log_writer->file->vselector_hint, log_writer->file->fnode.size); + log_writer->file->fnode.size = jump_to; + vselector->add_usage(log_writer->file->vselector_hint, log_writer->file->fnode.size); + _flush_bdev_safely(log_writer); + + _clear_dirty_set_stable(seq); + _release_pending_allocations(to_release); + + _update_logger_stats(); return 0; } diff --git a/src/os/bluestore/BlueFS.h b/src/os/bluestore/BlueFS.h index b285592b41df4..d8cbdbe4f661e 100644 --- a/src/os/bluestore/BlueFS.h +++ b/src/os/bluestore/BlueFS.h @@ -403,9 +403,18 @@ class BlueFS { void wait_for_aio(FileWriter *h); // safe to call without a lock #endif + int64_t _maybe_extend_log(); + void _extend_log(); + uint64_t _consume_dirty(); + void _clear_dirty_set_stable(uint64_t seq_stable); + void _release_pending_allocations(std::vector>& to_release); + + void _flush_and_sync_log_core(int64_t available_runway); + int _flush_and_sync_log_jump(uint64_t jump_to, + int64_t available_runway); int _flush_and_sync_log(std::unique_lock& l, - uint64_t want_seq = 0, - uint64_t jump_to = 0); + uint64_t want_seq = 0); + uint64_t _estimate_log_size(); bool _should_compact_log(); From 6b6a76a961db949c222de72f8fd5203ffac450d0 Mon Sep 17 00:00:00 2001 From: Adam Kupczyk Date: Sat, 5 Jun 2021 20:50:46 +0200 Subject: [PATCH 0040/2866] os/bluestore/bluefs: Refactor _flush This refactor prepares _flush for fine-grain locks in BlueFS. Introduced _flush_special, a flush dedicated to bluefs special files (ino=1) and (ino=0). Function _flush no longer accepts these special files. Signed-off-by: Adam Kupczyk --- src/os/bluestore/BlueFS.cc | 42 ++++++++++++++++++++++++-------------- src/os/bluestore/BlueFS.h | 2 ++ 2 files changed, 29 insertions(+), 15 deletions(-) diff --git a/src/os/bluestore/BlueFS.cc b/src/os/bluestore/BlueFS.cc index 5a2c50359fcc4..1d6ec6fd80c37 100644 --- a/src/os/bluestore/BlueFS.cc +++ b/src/os/bluestore/BlueFS.cc @@ -2230,7 +2230,7 @@ void BlueFS::_rewrite_log_and_layout_sync(bool allocate_with_fallback, log_writer = _create_writer(log_file); log_writer->append(bl); - r = _flush(log_writer, true); + r = _flush_special(log_writer); ceph_assert(r == 0); #ifdef HAVE_LIBAIO if (!cct->_conf->bluefs_sync_write) { @@ -2367,7 +2367,7 @@ void BlueFS::_compact_log_async(std::unique_lock& l) new_log_writer->append(bl); // 3. flush - r = _flush(new_log_writer, true); + r = _flush_special(new_log_writer); ceph_assert(r == 0); // 4. wait @@ -2546,7 +2546,7 @@ void BlueFS::_flush_and_sync_log_core(int64_t runway) log_t.clear(); log_t.seq = 0; // just so debug output is less confusing - int r = _flush(log_writer, true); + int r = _flush_special(log_writer); ceph_assert(r == 0); } @@ -2752,11 +2752,8 @@ int BlueFS::_flush_range(FileWriter *h, uint64_t offset, uint64_t length) ceph_assert(h->file->num_readers.load() == 0); - bool buffered; - if (h->file->fnode.ino == 1) - buffered = false; - else - buffered = cct->_conf->bluefs_buffered_io; + bool buffered = cct->_conf->bluefs_buffered_io; + ceph_assert(h->file->fnode.ino > 1); if (offset + length <= h->pos) return 0; @@ -2777,7 +2774,6 @@ int BlueFS::_flush_range(FileWriter *h, uint64_t offset, uint64_t length) if (allocated < offset + length) { // we should never run out of log space here; see the min runway check // in _flush_and_sync_log. - ceph_assert(h->file->fnode.ino != 1); int r = _allocate(vselector->select_prefer_bdev(h->file->vselector_hint), offset + length - allocated, &h->file->fnode); @@ -2793,15 +2789,14 @@ int BlueFS::_flush_range(FileWriter *h, uint64_t offset, uint64_t length) } if (h->file->fnode.size < offset + length) { h->file->fnode.size = offset + length; - if (h->file->fnode.ino > 1) { - // we do not need to dirty the log file (or it's compacting - // replacement) when the file size changes because replay is - // smart enough to discover it on its own. - h->file->is_dirty = true; - } + h->file->is_dirty = true; } dout(20) << __func__ << " file now, unflushed " << h->file->fnode << dendl; + return _flush_data(h, offset, length, buffered); +} +int BlueFS::_flush_data(FileWriter *h, uint64_t offset, uint64_t length, bool buffered) +{ uint64_t x_off = 0; auto p = h->file->fnode.seek(offset, &x_off); ceph_assert(p != h->file->fnode.extents.end()); @@ -2948,6 +2943,23 @@ int BlueFS::_flush(FileWriter *h, bool force, bool *flushed) return r; } +// Flush for bluefs special files. +// Does not add extents to h. +// Does not mark h as dirty. +// we do not need to dirty the log file (or it's compacting +// replacement) when the file size changes because replay is +// smart enough to discover it on its own. +int BlueFS::_flush_special(FileWriter *h) +{ + uint64_t length = h->get_buffer_length(); + uint64_t offset = h->pos; + ceph_assert(length + offset <= h->file->fnode.get_allocated()); + if (h->file->fnode.size < offset + length) { + h->file->fnode.size = offset + length; + } + return _flush_data(h, offset, length, false); +} + int BlueFS::_truncate(FileWriter *h, uint64_t offset) { dout(10) << __func__ << " 0x" << std::hex << offset << std::dec diff --git a/src/os/bluestore/BlueFS.h b/src/os/bluestore/BlueFS.h index d8cbdbe4f661e..a4cd65e4def29 100644 --- a/src/os/bluestore/BlueFS.h +++ b/src/os/bluestore/BlueFS.h @@ -394,8 +394,10 @@ class BlueFS { /* signal replay log to include h->file in nearest log flush */ int _signal_dirty_to_log(FileWriter *h); int _flush_range(FileWriter *h, uint64_t offset, uint64_t length); + int _flush_data(FileWriter *h, uint64_t offset, uint64_t length, bool buffered); int _flush(FileWriter *h, bool force, std::unique_lock& l); int _flush(FileWriter *h, bool force, bool *flushed = nullptr); + int _flush_special(FileWriter *h); int _fsync(FileWriter *h, std::unique_lock& l); #ifdef HAVE_LIBAIO From 08cf6d04a198a8dec3fc920a180208742c6fc965 Mon Sep 17 00:00:00 2001 From: Adam Kupczyk Date: Tue, 29 Jun 2021 13:03:56 +0200 Subject: [PATCH 0041/2866] os/bluestore/bluefs: Split single bluefs lock into multiple smaller locks Splits bluefs lock into log, dirty, dirs, file and writer locks. This breaks severe locking issues, and makes bluefs more multithreaded. Signed-off-by: Adam Kupczyk --- src/os/bluestore/BlueFS.cc | 418 +++++++++++++++++++++++++------------ src/os/bluestore/BlueFS.h | 113 ++++------ 2 files changed, 326 insertions(+), 205 deletions(-) diff --git a/src/os/bluestore/BlueFS.cc b/src/os/bluestore/BlueFS.cc index 1d6ec6fd80c37..8934520d4710f 100644 --- a/src/os/bluestore/BlueFS.cc +++ b/src/os/bluestore/BlueFS.cc @@ -148,7 +148,7 @@ class BlueFS::SocketHook : public AdminSocketHook { out.append(ss); } else if (command == "bluefs files list") { const char* devnames[3] = {"wal","db","slow"}; - std::lock_guard l(bluefs->lock); + std::lock_guard l(bluefs->dirs_lock); f->open_array_section("files"); for (auto &d : bluefs->dir_map) { std::string dir = d.first; @@ -310,6 +310,7 @@ void BlueFS::_shutdown_logger() delete logger; } +//AK - TODO - locking needed but not certain void BlueFS::_update_logger_stats() { // we must be holding the lock @@ -392,7 +393,6 @@ void BlueFS::handle_discard(unsigned id, interval_set& to_release) uint64_t BlueFS::get_used() { - std::lock_guard l(lock); uint64_t used = 0; for (unsigned id = 0; id < MAX_BDEV; ++id) { used += _get_used(id); @@ -418,7 +418,6 @@ uint64_t BlueFS::get_used(unsigned id) { ceph_assert(id < alloc.size()); ceph_assert(alloc[id]); - std::lock_guard l(lock); return _get_used(id); } @@ -431,13 +430,11 @@ uint64_t BlueFS::_get_total(unsigned id) const uint64_t BlueFS::get_total(unsigned id) { - std::lock_guard l(lock); return _get_total(id); } uint64_t BlueFS::get_free(unsigned id) { - std::lock_guard l(lock); ceph_assert(id < alloc.size()); return alloc[id]->get_free(); } @@ -467,7 +464,7 @@ void BlueFS::dump_block_extents(ostream& out) int BlueFS::get_block_extents(unsigned id, interval_set *extents) { - std::lock_guard l(lock); + std::lock_guard dirl(dirs_lock); dout(10) << __func__ << " bdev " << id << dendl; ceph_assert(id < alloc.size()); for (auto& p : file_map) { @@ -482,7 +479,6 @@ int BlueFS::get_block_extents(unsigned id, interval_set *extents) int BlueFS::mkfs(uuid_d osd_uuid, const bluefs_layout_t& layout) { - std::unique_lock l(lock); dout(1) << __func__ << " osd_uuid " << osd_uuid << dendl; @@ -519,7 +515,7 @@ int BlueFS::mkfs(uuid_d osd_uuid, const bluefs_layout_t& layout) // initial txn log_t.op_init(); - _flush_and_sync_log(l); + flush_and_sync_log(); // write supers super.log_fnode = log_file->fnode; @@ -868,7 +864,7 @@ int BlueFS::prepare_new_device(int id, const bluefs_layout_t& layout) new_log_dev_cur = BDEV_NEWDB; new_log_dev_next = BDEV_DB; } - _rewrite_log_and_layout_sync(false, + rewrite_log_and_layout_sync(false, BDEV_NEWDB, new_log_dev_cur, new_log_dev_next, @@ -876,7 +872,7 @@ int BlueFS::prepare_new_device(int id, const bluefs_layout_t& layout) layout); //} } else if(id == BDEV_NEWWAL) { - _rewrite_log_and_layout_sync(false, + rewrite_log_and_layout_sync(false, BDEV_DB, BDEV_NEWWAL, BDEV_WAL, @@ -907,7 +903,6 @@ void BlueFS::get_devices(set *ls) int BlueFS::fsck() { - std::lock_guard l(lock); dout(1) << __func__ << dendl; // hrm, i think we check everything on mount... return 0; @@ -1648,7 +1643,7 @@ int BlueFS::device_migrate_to_existing( new_log_dev_next; } - _rewrite_log_and_layout_sync( + rewrite_log_and_layout_sync( false, (flags & REMOVE_DB) ? BDEV_SLOW : BDEV_DB, new_log_dev_cur, @@ -1783,7 +1778,7 @@ int BlueFS::device_migrate_to_new( BDEV_DB : BDEV_SLOW; - _rewrite_log_and_layout_sync( + rewrite_log_and_layout_sync( false, super_dev, new_log_dev_cur, @@ -1813,12 +1808,16 @@ void BlueFS::_drop_link(FileRef file) dout(20) << __func__ << " had refs " << file->refs << " on " << file->fnode << dendl; ceph_assert(file->refs > 0); + ceph_assert(ceph_mutex_is_locked(log_lock)); + --file->refs; if (file->refs == 0) { dout(20) << __func__ << " destroying " << file->fnode << dendl; ceph_assert(file->num_reading.load() == 0); vselector->sub_usage(file->vselector_hint, file->fnode); log_t.op_file_remove(file->fnode.ino); + + std::lock_guard dl(dirty_lock); for (auto& r : file->fnode.extents) { pending_release[r.bdev].insert(r.offset, r.length); } @@ -1826,6 +1825,7 @@ void BlueFS::_drop_link(FileRef file) file->deleted = true; if (file->dirty_seq) { + // retract request to serialize changes ceph_assert(file->dirty_seq > log_seq_stable); ceph_assert(dirty_files.count(file->dirty_seq)); auto it = dirty_files[file->dirty_seq].iterator_to(*file); @@ -2049,8 +2049,9 @@ int64_t BlueFS::_read( return ret; } -void BlueFS::_invalidate_cache(FileRef f, uint64_t offset, uint64_t length) +void BlueFS::invalidate_cache(FileRef f, uint64_t offset, uint64_t length) { + std::lock_guard l(f->lock); dout(10) << __func__ << " file " << f->fnode << " 0x" << std::hex << offset << "~" << length << std::dec << dendl; @@ -2070,8 +2071,9 @@ void BlueFS::_invalidate_cache(FileRef f, uint64_t offset, uint64_t length) } } -uint64_t BlueFS::_estimate_log_size() +uint64_t BlueFS::estimate_log_size() { + std::lock_guard dirl(dirs_lock); int avg_dir_size = 40; // fixme int avg_file_size = 12; uint64_t size = 4096 * 2; @@ -2083,37 +2085,44 @@ uint64_t BlueFS::_estimate_log_size() void BlueFS::compact_log() { - std::unique_lock l(lock); if (!cct->_conf->bluefs_replay_recovery_disable_compact) { if (cct->_conf->bluefs_compact_log_sync) { - _compact_log_sync(); + compact_log_sync(); } else { - _compact_log_async(l); + compact_log_async(); } } } -bool BlueFS::_should_compact_log() +bool BlueFS::should_start_compact_log() { - uint64_t current = log_writer->file->fnode.size; - uint64_t expected = _estimate_log_size(); + if (log_is_compacting.load() == true) { + // compaction is already running + return false; + } + uint64_t current; + { + std::lock_guard dirl(log_lock); + current = log_writer->file->fnode.size; + } + uint64_t expected = estimate_log_size(); float ratio = (float)current / (float)expected; dout(10) << __func__ << " current 0x" << std::hex << current << " expected " << expected << std::dec << " ratio " << ratio - << (new_log ? " (async compaction in progress)" : "") << dendl; - if (new_log || - current < cct->_conf->bluefs_log_compact_min_size || + if (current < cct->_conf->bluefs_log_compact_min_size || ratio < cct->_conf->bluefs_log_compact_min_ratio) { return false; } return true; } -void BlueFS::_compact_log_dump_metadata(bluefs_transaction_t *t, +void BlueFS::compact_log_dump_metadata(bluefs_transaction_t *t, int flags) { + std::lock_guard dirl(dirs_lock); + t->seq = 1; t->uuid = super.uuid; dout(20) << __func__ << " op_init" << dendl; @@ -2123,7 +2132,7 @@ void BlueFS::_compact_log_dump_metadata(bluefs_transaction_t *t, if (ino == 1) continue; ceph_assert(ino > 1); - + //AK - TODO - touching fnode - need to lock for(auto& e : file_ref->fnode.extents) { auto bdev = e.bdev; auto bdev_new = bdev; @@ -2160,12 +2169,12 @@ void BlueFS::_compact_log_dump_metadata(bluefs_transaction_t *t, } } -void BlueFS::_compact_log_sync() +void BlueFS::compact_log_sync() { dout(10) << __func__ << dendl; auto prefer_bdev = vselector->select_prefer_bdev(log_writer->file->vselector_hint); - _rewrite_log_and_layout_sync(true, + rewrite_log_and_layout_sync(true, BDEV_DB, prefer_bdev, prefer_bdev, @@ -2174,13 +2183,16 @@ void BlueFS::_compact_log_sync() logger->inc(l_bluefs_log_compactions); } -void BlueFS::_rewrite_log_and_layout_sync(bool allocate_with_fallback, - int super_dev, - int log_dev, - int log_dev_new, - int flags, - std::optional layout) +void BlueFS::rewrite_log_and_layout_sync(bool allocate_with_fallback, + int super_dev, + int log_dev, + int log_dev_new, + int flags, + std::optional layout) { + //ceph_assert(ceph_mutex_is_notlocked(log_lock)); + std::lock_guard ll(log_lock); + File *log_file = log_writer->file.get(); // clear out log (be careful who calls us!!!) @@ -2192,7 +2204,7 @@ void BlueFS::_rewrite_log_and_layout_sync(bool allocate_with_fallback, << " flags:" << flags << dendl; bluefs_transaction_t t; - _compact_log_dump_metadata(&t, flags); + compact_log_dump_metadata(&t, flags); dout(20) << __func__ << " op_jump_seq " << log_seq << dendl; t.op_jump_seq(log_seq); @@ -2258,6 +2270,7 @@ void BlueFS::_rewrite_log_and_layout_sync(bool allocate_with_fallback, flush_bdev(); dout(10) << __func__ << " release old log extents " << old_fnode.extents << dendl; + std::lock_guard dl(dirty_lock); for (auto& r : old_fnode.extents) { pending_release[r.bdev].insert(r.offset, r.length); } @@ -2285,29 +2298,42 @@ void BlueFS::_rewrite_log_and_layout_sync(bool allocate_with_fallback, * * 8. Release the old log space. Clean up. */ -void BlueFS::_compact_log_async(std::unique_lock& l) + +void BlueFS::compact_log_async() { dout(10) << __func__ << dendl; + // only one compaction allowed at one time + bool old_is_comp = std::atomic_exchange(&log_is_compacting, true); + if (old_is_comp) { + dout(10) << __func__ << " ongoing" <file.get(); - ceph_assert(!new_log); - ceph_assert(!new_log_writer); + FileWriter *new_log_writer = nullptr; + FileRef new_log = nullptr; + uint64_t new_log_jump_to = 0; + uint64_t old_log_jump_to = 0; - // create a new log [writer] so that we know compaction is in progress - // (see _should_compact_log) new_log = ceph::make_ref(); - new_log->fnode.ino = 0; // so that _flush_range won't try to log the fnode + new_log->fnode.ino = 0; // we use _flush_special to avoid log of the fnode - // 0. wait for any racing flushes to complete. (We do not want to block - // in _flush_sync_log with jump_to set or else a racing thread might flush - // our entries and our jump_to update won't be correct.) - while (log_flushing) { - dout(10) << __func__ << " log is currently flushing, waiting" << dendl; - log_cond.wait(l); - } + // Part 1. + // Prepare current log for jumping into it. + // 1. Allocate extent + // 2. Update op to log + // 3. Jump op to log + // During that, no one else can write to log, otherwise we risk jumping backwards. + // We need to sync log, because we are injecting discontinuity, and writer is not prepared for that. + + //signal _maybe_extend_log that expansion of log is temporary inacceptable + bool old_forbidden = atomic_exchange(&log_forbidden_to_expand, true); + ceph_assert(old_forbidden == false); vselector->sub_usage(log_file->vselector_hint, log_file->fnode); - // 1. allocate new log space and jump to it. + // 1.1 allocate new log space and jump to it. old_log_jump_to = log_file->fnode.get_allocated(); uint64_t runway = log_file->fnode.get_allocated() - log_writer->get_effective_write_pos(); dout(10) << __func__ << " old_log_jump_to 0x" << std::hex << old_log_jump_to @@ -2322,21 +2348,24 @@ void BlueFS::_compact_log_async(std::unique_lock& l) // update the log file change and log a jump to the offset where we want to // write the new entries - log_t.op_file_update(log_file->fnode); - log_t.op_jump(log_seq, old_log_jump_to); + log_t.op_file_update(log_file->fnode); // 1.2 + log_t.op_jump(log_seq, old_log_jump_to); // 1.3 // we need to flush all bdev because we will be streaming all dirty files to log // TODO - think - if _flush_and_sync_log_jump will not add dirty files nor release pending allocations // then flush_bdev() will not be necessary flush_bdev(); - _flush_and_sync_log_jump(old_log_jump_to, runway); + log_lock.unlock(); + // out of jump section - now log can be used to write to + // 2. prepare compacted log bluefs_transaction_t t; - //avoid record two times in log_t and _compact_log_dump_metadata. - log_t.clear(); - _compact_log_dump_metadata(&t, 0); + //this needs files lock + //what will happen, if a file is modified *twice* before we stream it to log? + //the later state that we capture will be seen earlier and replay will see a temporary retraction (!) + compact_log_dump_metadata(&t, 0); uint64_t max_alloc_size = std::max(alloc_size[BDEV_WAL], std::max(alloc_size[BDEV_DB], @@ -2354,7 +2383,7 @@ void BlueFS::_compact_log_async(std::unique_lock& l) ceph_assert(r == 0); // we might have some more ops in log_t due to _allocate call - t.claim_ops(log_t); + // t.claim_ops(log_t); no we no longer track allocations in log bufferlist bl; encode(t, bl); @@ -2364,15 +2393,16 @@ void BlueFS::_compact_log_async(std::unique_lock& l) << std::dec << dendl; new_log_writer = _create_writer(new_log); - new_log_writer->append(bl); + new_log_writer->append(bl); + new_log_writer->lock.lock(); // 3. flush r = _flush_special(new_log_writer); ceph_assert(r == 0); // 4. wait - _flush_bdev_safely(new_log_writer); - + _flush_bdev(new_log_writer); + new_log_writer->lock.unlock(); // 5. update our log fnode // discard first old_log_jump_to extents @@ -2424,29 +2454,42 @@ void BlueFS::_compact_log_async(std::unique_lock& l) ++super.version; _write_super(BDEV_DB); - lock.unlock(); flush_bdev(); - lock.lock(); + + old_forbidden = atomic_exchange(&log_forbidden_to_expand, false); + ceph_assert(old_forbidden == true); + //to wake up if someone was in need of expanding log + log_cond.notify_all(); // 7. release old space dout(10) << __func__ << " release old log extents " << old_extents << dendl; - for (auto& r : old_extents) { - pending_release[r.bdev].insert(r.offset, r.length); + { + std::lock_guard dl(dirty_lock); + for (auto& r : old_extents) { + pending_release[r.bdev].insert(r.offset, r.length); + } } // delete the new log, remove from the dirty files list _close_writer(new_log_writer); + // flush_special does not dirty files + /* if (new_log->dirty_seq) { + std::lock_guard dl(dirty_lock); ceph_assert(dirty_files.count(new_log->dirty_seq)); auto it = dirty_files[new_log->dirty_seq].iterator_to(*new_log); dirty_files[new_log->dirty_seq].erase(it); } + */ new_log_writer = nullptr; new_log = nullptr; log_cond.notify_all(); dout(10) << __func__ << " log extents " << log_file->fnode.extents << dendl; logger->inc(l_bluefs_log_compactions); + + old_is_comp = atomic_exchange(&log_is_compacting, false); + ceph_assert(old_is_comp); } void BlueFS::_pad_bl(bufferlist& bl) @@ -2459,10 +2502,26 @@ void BlueFS::_pad_bl(bufferlist& bl) } } +/** + * Adds file modifications from `dirty_files` to bluefs transactions + * already stored in `log_t`. Writes them to disk and waits until are stable. + * Guarantees that file modifications with `want_seq` are already stable on disk. + * In addition may insert jump forward transaction to log write position `jump_to`. + * + * it should lock ability to: + * 1) add to log_t + * 2) modify dirty_files + * 3) add to pending_release + * + * pending_release and log_t go with same lock + */ + // Adds to log_t file modifications mentioned in `dirty_files`. // Note: some bluefs ops may have already been stored in log_t transaction. uint64_t BlueFS::_consume_dirty() { + ceph_assert(ceph_mutex_is_locked(dirty_lock)); + ceph_assert(ceph_mutex_is_locked(log_lock)); //acquire new seq // this will became log_seq_stable once we write uint64_t seq = log_t.seq = ++log_seq; @@ -2485,6 +2544,7 @@ uint64_t BlueFS::_consume_dirty() // Returns space available *BEFORE* adding new space. Signed for additional <0 detection. int64_t BlueFS::_maybe_extend_log() { + ceph_assert(ceph_mutex_is_locked(log_lock)); // allocate some more space (before we run out)? // BTW: this triggers `flush()` in the `page_aligned_appender` of `log_writer`. int64_t runway = log_writer->file->fnode.get_allocated() - @@ -2507,7 +2567,7 @@ int64_t BlueFS::_maybe_extend_log() * - re-run compaction with more runway for old log * - add OP_FILE_ADDEXT that adds extent; will be compatible with both logs */ - if (new_log_writer) { + if (log_forbidden_to_expand.load() == true) { return -EWOULDBLOCK; } vselector->sub_usage(log_writer->file->vselector_hint, log_writer->file->fnode); @@ -2524,6 +2584,7 @@ int64_t BlueFS::_maybe_extend_log() void BlueFS::_flush_and_sync_log_core(int64_t runway) { + ceph_assert(ceph_mutex_is_locked(log_lock)); dout(10) << __func__ << " " << log_t << dendl; bufferlist bl; @@ -2551,8 +2612,10 @@ void BlueFS::_flush_and_sync_log_core(int64_t runway) } // Clears dirty_files up to (including) seq_stable. -void BlueFS::_clear_dirty_set_stable(uint64_t seq) +void BlueFS::clear_dirty_set_stable(uint64_t seq) { + std::lock_guard lg(dirty_lock); + // clean dirty files if (seq > log_seq_stable) { log_seq_stable = seq; @@ -2586,7 +2649,7 @@ void BlueFS::_clear_dirty_set_stable(uint64_t seq) } } -void BlueFS::_release_pending_allocations(vector>& to_release) +void BlueFS::release_pending_allocations(vector>& to_release) { for (unsigned i = 0; i < to_release.size(); ++i) { if (!to_release[i].empty()) { @@ -2609,36 +2672,49 @@ void BlueFS::_release_pending_allocations(vector>& to_rel } } -int BlueFS::_flush_and_sync_log(std::unique_lock& l, - uint64_t want_seq) +int BlueFS::flush_and_sync_log(uint64_t want_seq) { - if (want_seq && want_seq <= log_seq_stable) { - dout(10) << __func__ << " want_seq " << want_seq << " <= log_seq_stable " - << log_seq_stable << ", done" << dendl; - return 0; - } + // we synchronize writing to log, by lock to log_lock int64_t available_runway; do { + log_lock.lock(); + dirty_lock.lock(); + if (want_seq && want_seq <= log_seq_stable) { + dout(10) << __func__ << " want_seq " << want_seq << " <= log_seq_stable " + << log_seq_stable << ", done" << dendl; + dirty_lock.unlock(); + log_lock.unlock(); + return 0; + } + available_runway = _maybe_extend_log(); if (available_runway == -EWOULDBLOCK) { - while (new_log_writer) { - dout(10) << __func__ << " waiting for async compaction" << dendl; - log_cond.wait(l); + // we are in need of adding runway, but we are during log-switch from compaction + dirty_lock.unlock(); + //instead log_lock_unlock() do move ownership + std::unique_lock ll(log_lock, std::adopt_lock); + while (log_forbidden_to_expand.load()) { + log_cond.wait(ll); } + } else { + ceph_assert(available_runway >= 0); } - } while (available_runway == -EWOULDBLOCK); + } while (available_runway < 0); ceph_assert(want_seq == 0 || want_seq <= log_seq + 1); // illegal to request seq that was not created yet uint64_t seq = _consume_dirty(); vector> to_release(pending_release.size()); to_release.swap(pending_release); + dirty_lock.unlock(); _flush_and_sync_log_core(available_runway); - _flush_bdev_safely(log_writer); + _flush_bdev(log_writer); + //now log_lock is no longer needed + log_lock.unlock(); - _clear_dirty_set_stable(seq); - _release_pending_allocations(to_release); + clear_dirty_set_stable(seq); + release_pending_allocations(to_release); _update_logger_stats(); return 0; @@ -2648,11 +2724,16 @@ int BlueFS::_flush_and_sync_log(std::unique_lock& l, int BlueFS::_flush_and_sync_log_jump(uint64_t jump_to, int64_t available_runway) { + ceph_assert(ceph_mutex_is_locked(log_lock)); + ceph_assert(jump_to); + // we synchronize writing to log, by lock to log_lock + + dirty_lock.lock(); uint64_t seq = _consume_dirty(); vector> to_release(pending_release.size()); to_release.swap(pending_release); - + dirty_lock.unlock(); _flush_and_sync_log_core(available_runway); dout(10) << __func__ << " jumping log offset from 0x" << std::hex @@ -2662,10 +2743,10 @@ int BlueFS::_flush_and_sync_log_jump(uint64_t jump_to, log_writer->file->fnode.size = jump_to; vselector->add_usage(log_writer->file->vselector_hint, log_writer->file->fnode.size); - _flush_bdev_safely(log_writer); + _flush_bdev(log_writer); - _clear_dirty_set_stable(seq); - _release_pending_allocations(to_release); + clear_dirty_set_stable(seq); + release_pending_allocations(to_release); _update_logger_stats(); return 0; @@ -2677,6 +2758,7 @@ ceph::bufferlist BlueFS::FileWriter::flush_buffer( const unsigned length, const bluefs_super_t& super) { + ceph_assert(ceph_mutex_is_locked(this->lock) || file->fnode.ino == 1); ceph::bufferlist bl; if (partial) { tail_block.splice(0, tail_block.length(), &bl); @@ -2715,6 +2797,8 @@ ceph::bufferlist BlueFS::FileWriter::flush_buffer( int BlueFS::_signal_dirty_to_log(FileWriter *h) { + ceph_assert(ceph_mutex_is_locked(h->lock)); + std::lock_guard dl(dirty_lock); h->file->fnode.mtime = ceph_clock_now(); ceph_assert(h->file->fnode.ino >= 1); if (h->file->dirty_seq == 0) { @@ -2740,8 +2824,14 @@ int BlueFS::_signal_dirty_to_log(FileWriter *h) return 0; } +void BlueFS::flush_range(FileWriter *h, uint64_t offset, uint64_t length) { + std::unique_lock hl(h->lock); + _flush_range(h, offset, length); +} + int BlueFS::_flush_range(FileWriter *h, uint64_t offset, uint64_t length) { + ceph_assert(ceph_mutex_is_locked(h->lock)); dout(10) << __func__ << " " << h << " pos 0x" << std::hex << h->pos << " 0x" << offset << "~" << length << std::dec << " to " << h->file->fnode << dendl; @@ -2764,13 +2854,13 @@ int BlueFS::_flush_range(FileWriter *h, uint64_t offset, uint64_t length) << std::hex << offset << "~" << length << std::dec << dendl; } + std::lock_guard file_lock(h->file->lock); ceph_assert(offset <= h->file->fnode.size); uint64_t allocated = h->file->fnode.get_allocated(); vselector->sub_usage(h->file->vselector_hint, h->file->fnode); // do not bother to dirty the file if we are overwriting // previously allocated extents. - if (allocated < offset + length) { // we should never run out of log space here; see the min runway check // in _flush_and_sync_log. @@ -2791,12 +2881,15 @@ int BlueFS::_flush_range(FileWriter *h, uint64_t offset, uint64_t length) h->file->fnode.size = offset + length; h->file->is_dirty = true; } + dout(20) << __func__ << " file now, unflushed " << h->file->fnode << dendl; return _flush_data(h, offset, length, buffered); } int BlueFS::_flush_data(FileWriter *h, uint64_t offset, uint64_t length, bool buffered) { + //ceph_assert(ceph_mutex_is_locked(h->lock)); + //ceph_assert(ceph_mutex_is_locked(h->file->lock)); uint64_t x_off = 0; auto p = h->file->fnode.seek(offset, &x_off); ceph_assert(p != h->file->fnode.extents.end()); @@ -2903,18 +2996,49 @@ void BlueFS::wait_for_aio(FileWriter *h) } #endif -int BlueFS::_flush(FileWriter *h, bool force, std::unique_lock& l) + +void BlueFS::append_try_flush(FileWriter *h, const char* buf, size_t len) +{ + std::unique_lock hl(h->lock); + size_t max_size = 1ull << 30; // cap to 1GB + while (len > 0) { + bool need_flush = true; + auto l0 = h->get_buffer_length(); + if (l0 < max_size) { + size_t l = std::min(len, max_size - l0); + h->append(buf, l); + buf += l; + len -= l; + need_flush = h->get_buffer_length() >= cct->_conf->bluefs_min_flush_size; + } + if (need_flush) { + bool flushed = false; + int r = _flush(h, true, &flushed); + ceph_assert(r == 0); + if (r == 0 && flushed) { + maybe_compact_log(); + } + // make sure we've made any progress with flush hence the + // loop doesn't iterate forever + ceph_assert(h->get_buffer_length() < max_size); + } + } +} + +void BlueFS::flush(FileWriter *h, bool force) { + std::unique_lock hl(h->lock); bool flushed = false; int r = _flush(h, force, &flushed); + ceph_assert(r == 0); if (r == 0 && flushed) { - _maybe_compact_log(l); + maybe_compact_log(); } - return r; } int BlueFS::_flush(FileWriter *h, bool force, bool *flushed) { + ceph_assert(ceph_mutex_is_locked(h->lock)); uint64_t length = h->get_buffer_length(); uint64_t offset = h->pos; if (flushed) { @@ -2951,6 +3075,8 @@ int BlueFS::_flush(FileWriter *h, bool force, bool *flushed) // smart enough to discover it on its own. int BlueFS::_flush_special(FileWriter *h) { + //ceph_assert(ceph_mutex_is_locked(h->lock)); + //ceph_assert(ceph_mutex_is_locked(h->file->lock)); uint64_t length = h->get_buffer_length(); uint64_t offset = h->pos; ceph_assert(length + offset <= h->file->fnode.get_allocated()); @@ -2960,8 +3086,9 @@ int BlueFS::_flush_special(FileWriter *h) return _flush_data(h, offset, length, false); } -int BlueFS::_truncate(FileWriter *h, uint64_t offset) +int BlueFS::truncate(FileWriter *h, uint64_t offset) { + std::lock_guard hl(h->lock); dout(10) << __func__ << " 0x" << std::hex << offset << std::dec << " file " << h->file->fnode << dendl; if (h->file->deleted) { @@ -2994,12 +3121,14 @@ int BlueFS::_truncate(FileWriter *h, uint64_t offset) vselector->sub_usage(h->file->vselector_hint, h->file->fnode.size); h->file->fnode.size = offset; vselector->add_usage(h->file->vselector_hint, h->file->fnode.size); + std::lock_guard ll(log_lock); log_t.op_file_update(h->file->fnode); return 0; } -int BlueFS::_fsync(FileWriter *h, std::unique_lock& l) +int BlueFS::fsync(FileWriter *h) { + std::unique_lock hl(h->lock); dout(10) << __func__ << " " << h << " " << h->file->fnode << dendl; int r = _flush(h, true); if (r < 0) @@ -3010,39 +3139,40 @@ int BlueFS::_fsync(FileWriter *h, std::unique_lock& l) } uint64_t old_dirty_seq = h->file->dirty_seq; - _flush_bdev_safely(h); + _flush_bdev(h); if (old_dirty_seq) { uint64_t s = log_seq; dout(20) << __func__ << " file metadata was dirty (" << old_dirty_seq << ") on " << h->file->fnode << ", flushing log" << dendl; - _flush_and_sync_log(l, old_dirty_seq); + flush_and_sync_log(old_dirty_seq); + // AK - TODO - think - how can dirty_seq change if we are under h lock? ceph_assert(h->file->dirty_seq == 0 || // cleaned - h->file->dirty_seq > s); // or redirtied by someone else + h->file->dirty_seq > s); // or redirtied by someone else } + maybe_compact_log(); return 0; } -void BlueFS::_flush_bdev_safely(FileWriter *h) +// be careful - either h->file->lock or log_lock must be taken +void BlueFS::_flush_bdev(FileWriter *h) { + if (h->file->fnode.ino != 1) { + ceph_assert(ceph_mutex_is_locked(h->lock)); + } else { + ceph_assert(ceph_mutex_is_locked(log_lock)); + } std::array flush_devs = h->dirty_devs; h->dirty_devs.fill(false); #ifdef HAVE_LIBAIO if (!cct->_conf->bluefs_sync_write) { list completed_ios; _claim_completed_aios(h, &completed_ios); - lock.unlock(); wait_for_aio(h); completed_ios.clear(); - flush_bdev(flush_devs); - lock.lock(); - } else -#endif - { - lock.unlock(); - flush_bdev(flush_devs); - lock.lock(); } +#endif + flush_bdev(flush_devs); } void BlueFS::flush_bdev(std::array& dirty_bdevs) @@ -3176,8 +3306,9 @@ int BlueFS::_allocate(uint8_t id, uint64_t len, return 0; } -int BlueFS::_preallocate(FileRef f, uint64_t off, uint64_t len) +int BlueFS::preallocate(FileRef f, uint64_t off, uint64_t len) { + std::lock_guard fl(f->lock); dout(10) << __func__ << " file " << f->fnode << " 0x" << std::hex << off << "~" << len << std::dec << dendl; if (f->deleted) { @@ -3196,6 +3327,7 @@ int BlueFS::_preallocate(FileRef f, uint64_t off, uint64_t len) vselector->add_usage(f->vselector_hint, f->fnode); if (r < 0) return r; + std::lock_guard ll(log_lock); log_t.op_file_update(f->fnode); } return 0; @@ -3203,8 +3335,13 @@ int BlueFS::_preallocate(FileRef f, uint64_t off, uint64_t len) void BlueFS::sync_metadata(bool avoid_compact) { - std::unique_lock l(lock); - if (log_t.empty() && dirty_files.empty()) { + bool can_skip_flush; + { + std::lock_guard ll(log_lock); + std::lock_guard dl(dirty_lock); + can_skip_flush = log_t.empty() && dirty_files.empty(); + } + if (can_skip_flush) { dout(10) << __func__ << " - no pending log events" << dendl; } else { utime_t start; @@ -3212,23 +3349,23 @@ void BlueFS::sync_metadata(bool avoid_compact) start = ceph_clock_now(); *_dout << dendl; flush_bdev(); // FIXME? - _flush_and_sync_log(l); + flush_and_sync_log(); dout(10) << __func__ << " done in " << (ceph_clock_now() - start) << dendl; } if (!avoid_compact) { - _maybe_compact_log(l); + maybe_compact_log(); } } -void BlueFS::_maybe_compact_log(std::unique_lock& l) +void BlueFS::maybe_compact_log() { if (!cct->_conf->bluefs_replay_recovery_disable_compact && - _should_compact_log()) { + should_start_compact_log()) { if (cct->_conf->bluefs_compact_log_sync) { - _compact_log_sync(); + compact_log_sync(); } else { - _compact_log_async(l); + compact_log_async(); } } } @@ -3239,7 +3376,7 @@ int BlueFS::open_for_write( FileWriter **h, bool overwrite) { - std::lock_guard l(lock); + std::lock_guard dirl(dirs_lock); dout(10) << __func__ << " " << dirname << "/" << filename << dendl; map::iterator p = dir_map.find(dirname); DirRef dir; @@ -3301,11 +3438,12 @@ int BlueFS::open_for_write( dout(20) << __func__ << " mapping " << dirname << "/" << filename << " vsel_hint " << file->vselector_hint << dendl; - - log_t.op_file_update(file->fnode); - if (create) - log_t.op_dir_link(dirname, filename, file->fnode.ino); - + { + std::lock_guard ll(log_lock); + log_t.op_file_update(file->fnode); + if (create) + log_t.op_dir_link(dirname, filename, file->fnode.ino); + } *h = _create_writer(file); if (boost::algorithm::ends_with(filename, ".log")) { @@ -3335,7 +3473,7 @@ BlueFS::FileWriter *BlueFS::_create_writer(FileRef f) return w; } -void BlueFS::_close_writer(FileWriter *h) +void BlueFS::_drain_writer(FileWriter *h) { dout(10) << __func__ << " " << h << " type " << h->writer_type << dendl; //h->buffer.reassign_to_mempool(mempool::mempool_bluefs_file_writer); @@ -3351,18 +3489,31 @@ void BlueFS::_close_writer(FileWriter *h) if (h->file->fnode.size >= (1ull << 30)) { dout(10) << __func__ << " file is unexpectedly large:" << h->file->fnode << dendl; } +} + +void BlueFS::_close_writer(FileWriter *h) +{ + _drain_writer(h); + delete h; +} +void BlueFS::close_writer(FileWriter *h) +{ + { + std::lock_guard l(h->lock); + _drain_writer(h); + } delete h; } uint64_t BlueFS::debug_get_dirty_seq(FileWriter *h) { - std::lock_guard l(lock); + std::lock_guard l(h->lock); return h->file->dirty_seq; } bool BlueFS::debug_get_is_dev_dirty(FileWriter *h, uint8_t dev) { - std::lock_guard l(lock); + std::lock_guard l(h->lock); return h->dirty_devs[dev]; } @@ -3372,7 +3523,7 @@ int BlueFS::open_for_read( FileReader **h, bool random) { - std::lock_guard l(lock); + std::lock_guard dirl(dirs_lock); dout(10) << __func__ << " " << dirname << "/" << filename << (random ? " (random)":" (sequential)") << dendl; map::iterator p = dir_map.find(dirname); @@ -3401,7 +3552,8 @@ int BlueFS::rename( std::string_view old_dirname, std::string_view old_filename, std::string_view new_dirname, std::string_view new_filename) { - std::lock_guard l(lock); + std::lock_guard dirl(dirs_lock); + std::lock_guard ll(log_lock); dout(10) << __func__ << " " << old_dirname << "/" << old_filename << " -> " << new_dirname << "/" << new_filename << dendl; map::iterator p = dir_map.find(old_dirname); @@ -3448,7 +3600,8 @@ int BlueFS::rename( int BlueFS::mkdir(std::string_view dirname) { - std::lock_guard l(lock); + std::lock_guard dirl(dirs_lock); + std::lock_guard ll(log_lock); dout(10) << __func__ << " " << dirname << dendl; map::iterator p = dir_map.find(dirname); if (p != dir_map.end()) { @@ -3462,7 +3615,8 @@ int BlueFS::mkdir(std::string_view dirname) int BlueFS::rmdir(std::string_view dirname) { - std::lock_guard l(lock); + std::lock_guard dirl(dirs_lock); + std::lock_guard ll(log_lock); dout(10) << __func__ << " " << dirname << dendl; auto p = dir_map.find(dirname); if (p == dir_map.end()) { @@ -3481,7 +3635,7 @@ int BlueFS::rmdir(std::string_view dirname) bool BlueFS::dir_exists(std::string_view dirname) { - std::lock_guard l(lock); + std::lock_guard dirl(dirs_lock); map::iterator p = dir_map.find(dirname); bool exists = p != dir_map.end(); dout(10) << __func__ << " " << dirname << " = " << (int)exists << dendl; @@ -3491,7 +3645,7 @@ bool BlueFS::dir_exists(std::string_view dirname) int BlueFS::stat(std::string_view dirname, std::string_view filename, uint64_t *size, utime_t *mtime) { - std::lock_guard l(lock); + std::lock_guard dirl(dirs_lock); dout(10) << __func__ << " " << dirname << "/" << filename << dendl; map::iterator p = dir_map.find(dirname); if (p == dir_map.end()) { @@ -3519,7 +3673,7 @@ int BlueFS::stat(std::string_view dirname, std::string_view filename, int BlueFS::lock_file(std::string_view dirname, std::string_view filename, FileLock **plock) { - std::lock_guard l(lock); + std::lock_guard dirl(dirs_lock); dout(10) << __func__ << " " << dirname << "/" << filename << dendl; map::iterator p = dir_map.find(dirname); if (p == dir_map.end()) { @@ -3539,6 +3693,7 @@ int BlueFS::lock_file(std::string_view dirname, std::string_view filename, file_map[ino_last] = file; dir->file_map[string{filename}] = file; ++file->refs; + std::lock_guard ll(log_lock); log_t.op_file_update(file->fnode); log_t.op_dir_link(dirname, filename, file->fnode.ino); } else { @@ -3557,7 +3712,7 @@ int BlueFS::lock_file(std::string_view dirname, std::string_view filename, int BlueFS::unlock_file(FileLock *fl) { - std::lock_guard l(lock); + std::lock_guard dirl(dirs_lock); dout(10) << __func__ << " " << fl << " on " << fl->file->fnode << dendl; ceph_assert(fl->file->locked); fl->file->locked = false; @@ -3571,7 +3726,7 @@ int BlueFS::readdir(std::string_view dirname, vector *ls) if (!dirname.empty() && dirname.back() == '/') { dirname.remove_suffix(1); } - std::lock_guard l(lock); + std::lock_guard dirl(dirs_lock); dout(10) << __func__ << " " << dirname << dendl; if (dirname.empty()) { // list dirs @@ -3599,7 +3754,8 @@ int BlueFS::readdir(std::string_view dirname, vector *ls) int BlueFS::unlink(std::string_view dirname, std::string_view filename) { - std::lock_guard l(lock); + std::lock_guard dirl(dirs_lock); + std::lock_guard ll(log_lock); dout(10) << __func__ << " " << dirname << "/" << filename << dendl; map::iterator p = dir_map.find(dirname); if (p == dir_map.end()) { diff --git a/src/os/bluestore/BlueFS.h b/src/os/bluestore/BlueFS.h index a4cd65e4def29..4f368841ff4d9 100644 --- a/src/os/bluestore/BlueFS.h +++ b/src/os/bluestore/BlueFS.h @@ -121,6 +121,11 @@ class BlueFS { std::atomic_int num_reading; void* vselector_hint = nullptr; + /* lock protects fnode and other the parts that can be modified during read & write operations. + Does not protect values that are fixed + Does not need to be taken when doing one-time operations: + _replay, device_migrate_to_existing, device_migrate_to_new */ + ceph::mutex lock = ceph::make_mutex("BlueFS::File::lock"); private: FRIEND_MAKE_REF(File); @@ -304,8 +309,9 @@ class BlueFS { }; private: - ceph::mutex lock = ceph::make_mutex("BlueFS::lock"); - + ceph::mutex log_lock = ceph::make_mutex("BlueFS::log_lock"); + ceph::mutex dirs_lock = ceph::make_mutex("BlueFS::dirs_lock"); + ceph::mutex dirty_lock = ceph::make_mutex("BlueFS::dirty_lock"); PerfCounters *logger = nullptr; uint64_t max_bytes[MAX_BDEV] = {0}; @@ -329,13 +335,12 @@ class BlueFS { FileWriter *log_writer = 0; ///< writer for the log bluefs_transaction_t log_t; ///< pending, unwritten log transaction bool log_flushing = false; ///< true while flushing the log - ceph::condition_variable log_cond; - - uint64_t new_log_jump_to = 0; - uint64_t old_log_jump_to = 0; - FileRef new_log = nullptr; - FileWriter *new_log_writer = nullptr; + ceph::condition_variable log_cond; ///< used for state control between log flush / log compaction + ceph::mutex cond_lock = ceph::make_mutex("BlueFS::cond_lock"); ///< .... + std::atomic log_is_compacting{false}; ///< signals that bluefs log is already ongoing compaction + std::atomic log_forbidden_to_expand{false}; ///< used to signal that async compaction is in state + /// that prohibits expansion of bluefs log /* * There are up to 3 block devices: * @@ -349,6 +354,11 @@ class BlueFS { std::vector alloc; ///< allocators for bdevs std::vector alloc_size; ///< alloc size for each device std::vector> pending_release; ///< extents to release + // TODO: it should be examined what makes pending_release immune to + // eras in a way similar to dirty_files. Hints: + // 1) we have actually only 2 eras: log_seq and log_seq+1 + // 2) we usually not remove extents from files. And when we do, we force log-syncing. + //std::vector> block_unused_too_granular; BlockDevice::aio_callback_t discard_cb[3]; //discard callbacks for each dev @@ -395,10 +405,9 @@ class BlueFS { int _signal_dirty_to_log(FileWriter *h); int _flush_range(FileWriter *h, uint64_t offset, uint64_t length); int _flush_data(FileWriter *h, uint64_t offset, uint64_t length, bool buffered); - int _flush(FileWriter *h, bool force, std::unique_lock& l); int _flush(FileWriter *h, bool force, bool *flushed = nullptr); int _flush_special(FileWriter *h); - int _fsync(FileWriter *h, std::unique_lock& l); + int _fsync(FileWriter *h); #ifdef HAVE_LIBAIO void _claim_completed_aios(FileWriter *h, std::list *ls); @@ -414,11 +423,10 @@ class BlueFS { void _flush_and_sync_log_core(int64_t available_runway); int _flush_and_sync_log_jump(uint64_t jump_to, int64_t available_runway); - int _flush_and_sync_log(std::unique_lock& l, - uint64_t want_seq = 0); + int flush_and_sync_log(uint64_t want_seq = 0); - uint64_t _estimate_log_size(); - bool _should_compact_log(); + uint64_t estimate_log_size(); + bool should_start_compact_log(); enum { REMOVE_DB = 1, @@ -426,12 +434,12 @@ class BlueFS { RENAME_SLOW2DB = 4, RENAME_DB2SLOW = 8, }; - void _compact_log_dump_metadata(bluefs_transaction_t *t, - int flags); - void _compact_log_sync(); - void _compact_log_async(std::unique_lock& l); + void compact_log_dump_metadata(bluefs_transaction_t *t, + int flags); + void compact_log_sync(); + void compact_log_async(); - void _rewrite_log_and_layout_sync(bool allocate_with_fallback, + void rewrite_log_and_layout_sync(bool allocate_with_fallback, int super_dev, int log_dev, int new_log_dev, @@ -440,7 +448,7 @@ class BlueFS { //void _aio_finish(void *priv); - void _flush_bdev_safely(FileWriter *h); + void _flush_bdev(FileWriter *h); void flush_bdev(); // this is safe to call without a lock void flush_bdev(std::array& dirty_bdevs); // this is safe to call without a lock @@ -459,8 +467,6 @@ class BlueFS { uint64_t len, ///< [in] this many bytes char *out); ///< [out] optional: or copy it here - void _invalidate_cache(FileRef f, uint64_t offset, uint64_t length); - int _open_super(); int _write_super(int dev); int _check_allocations(const bluefs_fnode_t& fnode, @@ -473,6 +479,7 @@ class BlueFS { int _replay(bool noop, bool to_stdout = false); ///< replay journal FileWriter *_create_writer(FileRef f); + void _drain_writer(FileWriter *h); void _close_writer(FileWriter *h); // always put the super in the second 4k block. FIXME should this be @@ -538,10 +545,8 @@ class BlueFS { FileReader **h, bool random = false); - void close_writer(FileWriter *h) { - std::lock_guard l(lock); - _close_writer(h); - } + // data added after last fsync() is lost + void close_writer(FileWriter *h); int rename(std::string_view old_dir, std::string_view old_file, std::string_view new_dir, std::string_view new_file); @@ -565,7 +570,7 @@ class BlueFS { /// sync any uncommitted state to disk void sync_metadata(bool avoid_compact); /// test and compact log, if necessary - void _maybe_compact_log(std::unique_lock& l); + void maybe_compact_log(); void set_volume_selector(BlueFSVolumeSelector* s) { vselector.reset(s); @@ -587,42 +592,11 @@ class BlueFS { // handler for discard event void handle_discard(unsigned dev, interval_set& to_release); - void flush(FileWriter *h, bool force = false) { - std::unique_lock l(lock); - int r = _flush(h, force, l); - ceph_assert(r == 0); - } + void flush(FileWriter *h, bool force = false); - void append_try_flush(FileWriter *h, const char* buf, size_t len) { - size_t max_size = 1ull << 30; // cap to 1GB - while (len > 0) { - bool need_flush = true; - auto l0 = h->get_buffer_length(); - if (l0 < max_size) { - size_t l = std::min(len, max_size - l0); - h->append(buf, l); - buf += l; - len -= l; - need_flush = h->get_buffer_length() >= cct->_conf->bluefs_min_flush_size; - } - if (need_flush) { - flush(h, true); - // make sure we've made any progress with flush hence the - // loop doesn't iterate forever - ceph_assert(h->get_buffer_length() < max_size); - } - } - } - void flush_range(FileWriter *h, uint64_t offset, uint64_t length) { - std::lock_guard l(lock); - _flush_range(h, offset, length); - } - int fsync(FileWriter *h) { - std::unique_lock l(lock); - int r = _fsync(h, l); - _maybe_compact_log(l); - return r; - } + void append_try_flush(FileWriter *h, const char* buf, size_t len); + void flush_range(FileWriter *h, uint64_t offset, uint64_t length); + int fsync(FileWriter *h); int64_t read(FileReader *h, uint64_t offset, size_t len, ceph::buffer::list *outbl, char *out) { // no need to hold the global lock here; we only touch h and @@ -637,18 +611,9 @@ class BlueFS { // atomics and asserts). return _read_random(h, offset, len, out); } - void invalidate_cache(FileRef f, uint64_t offset, uint64_t len) { - std::lock_guard l(lock); - _invalidate_cache(f, offset, len); - } - int preallocate(FileRef f, uint64_t offset, uint64_t len) { - std::lock_guard l(lock); - return _preallocate(f, offset, len); - } - int truncate(FileWriter *h, uint64_t offset) { - std::lock_guard l(lock); - return _truncate(h, offset); - } + void invalidate_cache(FileRef f, uint64_t offset, uint64_t len); + int preallocate(FileRef f, uint64_t offset, uint64_t len); + int truncate(FileWriter *h, uint64_t offset); int do_replay_recovery_read(FileReader *log, size_t log_pos, size_t read_offset, From 45b3c5d0ea6c737ae6bf742387f5746b65eeb88d Mon Sep 17 00:00:00 2001 From: Adam Kupczyk Date: Tue, 22 Jun 2021 13:15:21 +0200 Subject: [PATCH 0042/2866] os/bluestore/bluefs: Reorganize BlueFS state variables Reorganize BlueFS state variables into separate domains: 1) log, 2) dirty, 3) nodes. Each has separate lock. This change is intended to make it easier to control which locks need to be held when specific elements are modified. Signed-off-by: Adam Kupczyk --- src/os/bluestore/BlueFS.cc | 468 +++++++++++++++++++------------------ src/os/bluestore/BlueFS.h | 31 ++- 2 files changed, 257 insertions(+), 242 deletions(-) diff --git a/src/os/bluestore/BlueFS.cc b/src/os/bluestore/BlueFS.cc index 8934520d4710f..e501159b2953c 100644 --- a/src/os/bluestore/BlueFS.cc +++ b/src/os/bluestore/BlueFS.cc @@ -148,9 +148,9 @@ class BlueFS::SocketHook : public AdminSocketHook { out.append(ss); } else if (command == "bluefs files list") { const char* devnames[3] = {"wal","db","slow"}; - std::lock_guard l(bluefs->dirs_lock); + std::lock_guard l(bluefs->nodes.lock); f->open_array_section("files"); - for (auto &d : bluefs->dir_map) { + for (auto &d : bluefs->nodes.dir_map) { std::string dir = d.first; for (auto &r : d.second->file_map) { f->open_object_section("file"); @@ -314,8 +314,8 @@ void BlueFS::_shutdown_logger() void BlueFS::_update_logger_stats() { // we must be holding the lock - logger->set(l_bluefs_num_files, file_map.size()); - logger->set(l_bluefs_log_bytes, log_writer->file->fnode.size); + logger->set(l_bluefs_num_files, nodes.file_map.size()); + logger->set(l_bluefs_log_bytes, log.writer->file->fnode.size); if (alloc[BDEV_WAL]) { logger->set(l_bluefs_wal_total_bytes, _get_total(BDEV_WAL)); @@ -464,10 +464,10 @@ void BlueFS::dump_block_extents(ostream& out) int BlueFS::get_block_extents(unsigned id, interval_set *extents) { - std::lock_guard dirl(dirs_lock); + std::lock_guard dirl(nodes.lock); dout(10) << __func__ << " bdev " << id << dendl; ceph_assert(id < alloc.size()); - for (auto& p : file_map) { + for (auto& p : nodes.file_map) { for (auto& q : p.second->fnode.extents) { if (q.bdev == id) { extents->insert(q.offset, q.length); @@ -511,10 +511,10 @@ int BlueFS::mkfs(uuid_d osd_uuid, const bluefs_layout_t& layout) &log_file->fnode); vselector->add_usage(log_file->vselector_hint, log_file->fnode); ceph_assert(r == 0); - log_writer = _create_writer(log_file); + log.writer = _create_writer(log_file); // initial txn - log_t.op_init(); + log.t.op_init(); flush_and_sync_log(); // write supers @@ -525,8 +525,8 @@ int BlueFS::mkfs(uuid_d osd_uuid, const bluefs_layout_t& layout) // clean up super = bluefs_super_t(); - _close_writer(log_writer); - log_writer = NULL; + _close_writer(log.writer); + log.writer = NULL; vselector.reset(nullptr); _stop_alloc(); _shutdown_logger(); @@ -781,7 +781,7 @@ int BlueFS::mount() } // init freelist - for (auto& p : file_map) { + for (auto& p : nodes.file_map) { dout(30) << __func__ << " noting alloc for " << p.second->fnode << dendl; for (auto& q : p.second->fnode.extents) { bool is_shared = is_shared_alloc(q.bdev); @@ -804,11 +804,11 @@ int BlueFS::mount() } // set up the log for future writes - log_writer = _create_writer(_get_file(1)); - ceph_assert(log_writer->file->fnode.ino == 1); - log_writer->pos = log_writer->file->fnode.size; + log.writer = _create_writer(_get_file(1)); + ceph_assert(log.writer->file->fnode.ino == 1); + log.writer->pos = log.writer->file->fnode.size; dout(10) << __func__ << " log write pos set to 0x" - << std::hex << log_writer->pos << std::dec + << std::hex << log.writer->pos << std::dec << dendl; return 0; @@ -841,15 +841,15 @@ void BlueFS::umount(bool avoid_compact) sync_metadata(avoid_compact); - _close_writer(log_writer); - log_writer = NULL; + _close_writer(log.writer); + log.writer = NULL; vselector.reset(nullptr); _stop_alloc(); - file_map.clear(); - dir_map.clear(); + nodes.file_map.clear(); + nodes.dir_map.clear(); super = bluefs_super_t(); - log_t.clear(); + log.t.clear(); _shutdown_logger(); } @@ -1033,7 +1033,7 @@ int BlueFS::_replay(bool noop, bool to_stdout) { dout(10) << __func__ << (noop ? " NO-OP" : "") << dendl; ino_last = 1; // by the log - log_seq = 0; + uint64_t log_seq = 0; FileRef log_file; log_file = _get_file(1); @@ -1297,8 +1297,8 @@ int BlueFS::_replay(bool noop, bool to_stdout) if (!noop) { FileRef file = _get_file(ino); ceph_assert(file->fnode.ino); - map::iterator q = dir_map.find(dirname); - ceph_assert(q != dir_map.end()); + map::iterator q = nodes.dir_map.find(dirname); + ceph_assert(q != nodes.dir_map.end()); map::iterator r = q->second->file_map.find(filename); ceph_assert(r == q->second->file_map.end()); @@ -1328,8 +1328,8 @@ int BlueFS::_replay(bool noop, bool to_stdout) } if (!noop) { - map::iterator q = dir_map.find(dirname); - ceph_assert(q != dir_map.end()); + map::iterator q = nodes.dir_map.find(dirname); + ceph_assert(q != nodes.dir_map.end()); map::iterator r = q->second->file_map.find(filename); ceph_assert(r != q->second->file_map.end()); ceph_assert(r->second->refs > 0); @@ -1351,9 +1351,9 @@ int BlueFS::_replay(bool noop, bool to_stdout) } if (!noop) { - map::iterator q = dir_map.find(dirname); - ceph_assert(q == dir_map.end()); - dir_map[dirname] = ceph::make_ref(); + map::iterator q = nodes.dir_map.find(dirname); + ceph_assert(q == nodes.dir_map.end()); + nodes.dir_map[dirname] = ceph::make_ref(); } } break; @@ -1370,10 +1370,10 @@ int BlueFS::_replay(bool noop, bool to_stdout) } if (!noop) { - map::iterator q = dir_map.find(dirname); - ceph_assert(q != dir_map.end()); + map::iterator q = nodes.dir_map.find(dirname); + ceph_assert(q != nodes.dir_map.end()); ceph_assert(q->second->file_map.empty()); - dir_map.erase(q); + nodes.dir_map.erase(q); } } break; @@ -1434,8 +1434,8 @@ int BlueFS::_replay(bool noop, bool to_stdout) } if (!noop) { - auto p = file_map.find(ino); - ceph_assert(p != file_map.end()); + auto p = nodes.file_map.find(ino); + ceph_assert(p != nodes.file_map.end()); vselector->sub_usage(p->second->vselector_hint, p->second->fnode); if (cct->_conf->bluefs_log_replay_check_allocations) { int r = _check_allocations(p->second->fnode, @@ -1444,7 +1444,7 @@ int BlueFS::_replay(bool noop, bool to_stdout) return r; } } - file_map.erase(p); + nodes.file_map.erase(p); } } break; @@ -1464,6 +1464,9 @@ int BlueFS::_replay(bool noop, bool to_stdout) } if (!noop) { vselector->add_usage(log_file->vselector_hint, log_file->fnode); + log.seq_live = log_seq; + dirty.seq_next = log_seq + 1; + dirty.seq_stable = log_seq; } dout(10) << __func__ << " log file size was 0x" @@ -1477,7 +1480,7 @@ int BlueFS::_replay(bool noop, bool to_stdout) if (!noop) { // verify file link counts are all >0 - for (auto& p : file_map) { + for (auto& p : nodes.file_map) { if (p.second->refs == 0 && p.second->fnode.ino > 1) { derr << __func__ << " file with link count 0: " << p.second->fnode @@ -1494,7 +1497,7 @@ int BlueFS::_replay(bool noop, bool to_stdout) int BlueFS::log_dump() { // only dump log file's content - ceph_assert(log_writer == nullptr && "cannot log_dump on mounted BlueFS"); + ceph_assert(log.writer == nullptr && "cannot log_dump on mounted BlueFS"); int r = _open_super(); if (r < 0) { derr << __func__ << " failed to open super: " << cpp_strerror(r) << dendl; @@ -1536,7 +1539,7 @@ int BlueFS::device_migrate_to_existing( dout(0) << __func__ << " super to be written to " << dev_target << dendl; } - for (auto& [ino, file_ref] : file_map) { + for (auto& [ino, file_ref] : nodes.file_map) { //do not copy log if (file_ref->fnode.ino == 1) { continue; @@ -1674,7 +1677,7 @@ int BlueFS::device_migrate_to_new( flags |= devs_source.count(BDEV_WAL) ? REMOVE_WAL : 0; int dev_target_new = dev_target; //FIXME: remove, makes no sense - for (auto& p : file_map) { + for (auto& p : nodes.file_map) { //do not copy log if (p.second->fnode.ino == 1) { continue; @@ -1790,10 +1793,10 @@ int BlueFS::device_migrate_to_new( BlueFS::FileRef BlueFS::_get_file(uint64_t ino) { - auto p = file_map.find(ino); - if (p == file_map.end()) { + auto p = nodes.file_map.find(ino); + if (p == nodes.file_map.end()) { FileRef f = ceph::make_ref(); - file_map[ino] = f; + nodes.file_map[ino] = f; dout(30) << __func__ << " ino " << ino << " = " << f << " (new)" << dendl; return f; @@ -1808,28 +1811,28 @@ void BlueFS::_drop_link(FileRef file) dout(20) << __func__ << " had refs " << file->refs << " on " << file->fnode << dendl; ceph_assert(file->refs > 0); - ceph_assert(ceph_mutex_is_locked(log_lock)); + ceph_assert(ceph_mutex_is_locked(log.lock)); --file->refs; if (file->refs == 0) { dout(20) << __func__ << " destroying " << file->fnode << dendl; ceph_assert(file->num_reading.load() == 0); vselector->sub_usage(file->vselector_hint, file->fnode); - log_t.op_file_remove(file->fnode.ino); + log.t.op_file_remove(file->fnode.ino); - std::lock_guard dl(dirty_lock); + std::lock_guard dl(dirty.lock); for (auto& r : file->fnode.extents) { pending_release[r.bdev].insert(r.offset, r.length); } - file_map.erase(file->fnode.ino); + nodes.file_map.erase(file->fnode.ino); file->deleted = true; if (file->dirty_seq) { // retract request to serialize changes - ceph_assert(file->dirty_seq > log_seq_stable); - ceph_assert(dirty_files.count(file->dirty_seq)); - auto it = dirty_files[file->dirty_seq].iterator_to(*file); - dirty_files[file->dirty_seq].erase(it); + ceph_assert(file->dirty_seq > dirty.seq_stable); + ceph_assert(dirty.files.count(file->dirty_seq)); + auto it = dirty.files[file->dirty_seq].iterator_to(*file); + dirty.files[file->dirty_seq].erase(it); file->dirty_seq = 0; } } @@ -2073,13 +2076,13 @@ void BlueFS::invalidate_cache(FileRef f, uint64_t offset, uint64_t length) uint64_t BlueFS::estimate_log_size() { - std::lock_guard dirl(dirs_lock); + std::lock_guard dirl(nodes.lock); int avg_dir_size = 40; // fixme int avg_file_size = 12; uint64_t size = 4096 * 2; - size += file_map.size() * (1 + sizeof(bluefs_fnode_t)); - size += dir_map.size() + (1 + avg_dir_size); - size += file_map.size() * (1 + avg_dir_size + avg_file_size); + size += nodes.file_map.size() * (1 + sizeof(bluefs_fnode_t)); + size += nodes.dir_map.size() + (1 + avg_dir_size); + size += nodes.file_map.size() * (1 + avg_dir_size + avg_file_size); return round_up_to(size, super.block_size); } @@ -2102,8 +2105,8 @@ bool BlueFS::should_start_compact_log() } uint64_t current; { - std::lock_guard dirl(log_lock); - current = log_writer->file->fnode.size; + std::lock_guard dirl(log.lock); + current = log.writer->file->fnode.size; } uint64_t expected = estimate_log_size(); float ratio = (float)current / (float)expected; @@ -2121,14 +2124,14 @@ bool BlueFS::should_start_compact_log() void BlueFS::compact_log_dump_metadata(bluefs_transaction_t *t, int flags) { - std::lock_guard dirl(dirs_lock); + std::lock_guard dirl(nodes.lock); t->seq = 1; t->uuid = super.uuid; dout(20) << __func__ << " op_init" << dendl; t->op_init(); - for (auto& [ino, file_ref] : file_map) { + for (auto& [ino, file_ref] : nodes.file_map) { if (ino == 1) continue; ceph_assert(ino > 1); @@ -2158,7 +2161,7 @@ void BlueFS::compact_log_dump_metadata(bluefs_transaction_t *t, dout(20) << __func__ << " op_file_update " << file_ref->fnode << dendl; t->op_file_update(file_ref->fnode); } - for (auto& [path, dir_ref] : dir_map) { + for (auto& [path, dir_ref] : nodes.dir_map) { dout(20) << __func__ << " op_dir_create " << path << dendl; t->op_dir_create(path); for (auto& [fname, file_ref] : dir_ref->file_map) { @@ -2173,7 +2176,7 @@ void BlueFS::compact_log_sync() { dout(10) << __func__ << dendl; auto prefer_bdev = - vselector->select_prefer_bdev(log_writer->file->vselector_hint); + vselector->select_prefer_bdev(log.writer->file->vselector_hint); rewrite_log_and_layout_sync(true, BDEV_DB, prefer_bdev, @@ -2190,13 +2193,13 @@ void BlueFS::rewrite_log_and_layout_sync(bool allocate_with_fallback, int flags, std::optional layout) { - //ceph_assert(ceph_mutex_is_notlocked(log_lock)); - std::lock_guard ll(log_lock); + //ceph_assert(ceph_mutex_is_notlocked(log.lock)); + std::lock_guard ll(log.lock); - File *log_file = log_writer->file.get(); + File *log_file = log.writer->file.get(); // clear out log (be careful who calls us!!!) - log_t.clear(); + log.t.clear(); dout(20) << __func__ << " super_dev:" << super_dev << " log_dev:" << log_dev @@ -2206,8 +2209,8 @@ void BlueFS::rewrite_log_and_layout_sync(bool allocate_with_fallback, bluefs_transaction_t t; compact_log_dump_metadata(&t, flags); - dout(20) << __func__ << " op_jump_seq " << log_seq << dendl; - t.op_jump_seq(log_seq); + dout(20) << __func__ << " op_jump_seq " << log.seq_live << dendl; + t.op_jump_seq(log.seq_live); bufferlist bl; encode(t, bl); @@ -2234,21 +2237,21 @@ void BlueFS::rewrite_log_and_layout_sync(bool allocate_with_fallback, } } - _close_writer(log_writer); + _close_writer(log.writer); log_file->fnode.size = bl.length(); vselector->sub_usage(log_file->vselector_hint, old_fnode); vselector->add_usage(log_file->vselector_hint, log_file->fnode); - log_writer = _create_writer(log_file); - log_writer->append(bl); - r = _flush_special(log_writer); + log.writer = _create_writer(log_file); + log.writer->append(bl); + r = _flush_special(log.writer); ceph_assert(r == 0); #ifdef HAVE_LIBAIO if (!cct->_conf->bluefs_sync_write) { list completed_ios; - _claim_completed_aios(log_writer, &completed_ios); - wait_for_aio(log_writer); + _claim_completed_aios(log.writer, &completed_ios); + wait_for_aio(log.writer); completed_ios.clear(); } #endif @@ -2270,7 +2273,7 @@ void BlueFS::rewrite_log_and_layout_sync(bool allocate_with_fallback, flush_bdev(); dout(10) << __func__ << " release old log extents " << old_fnode.extents << dendl; - std::lock_guard dl(dirty_lock); + std::lock_guard dl(dirty.lock); for (auto& r : old_fnode.extents) { pending_release[r.bdev].insert(r.offset, r.length); } @@ -2309,8 +2312,8 @@ void BlueFS::compact_log_async() return; } - log_lock.lock(); - File *log_file = log_writer->file.get(); + log.lock.lock(); + File *log_file = log.writer->file.get(); FileWriter *new_log_writer = nullptr; FileRef new_log = nullptr; uint64_t new_log_jump_to = 0; @@ -2335,7 +2338,7 @@ void BlueFS::compact_log_async() // 1.1 allocate new log space and jump to it. old_log_jump_to = log_file->fnode.get_allocated(); - uint64_t runway = log_file->fnode.get_allocated() - log_writer->get_effective_write_pos(); + uint64_t runway = log_file->fnode.get_allocated() - log.writer->get_effective_write_pos(); dout(10) << __func__ << " old_log_jump_to 0x" << std::hex << old_log_jump_to << " need 0x" << (old_log_jump_to + cct->_conf->bluefs_max_log_runway) << std::dec << dendl; int r = _allocate(vselector->select_prefer_bdev(log_file->vselector_hint), @@ -2348,8 +2351,8 @@ void BlueFS::compact_log_async() // update the log file change and log a jump to the offset where we want to // write the new entries - log_t.op_file_update(log_file->fnode); // 1.2 - log_t.op_jump(log_seq, old_log_jump_to); // 1.3 + log.t.op_file_update(log_file->fnode); // 1.2 + log.t.op_jump(log.seq_live, old_log_jump_to); // 1.3 // we need to flush all bdev because we will be streaming all dirty files to log // TODO - think - if _flush_and_sync_log_jump will not add dirty files nor release pending allocations @@ -2357,7 +2360,7 @@ void BlueFS::compact_log_async() flush_bdev(); _flush_and_sync_log_jump(old_log_jump_to, runway); - log_lock.unlock(); + log.lock.unlock(); // out of jump section - now log can be used to write to // 2. prepare compacted log @@ -2374,7 +2377,7 @@ void BlueFS::compact_log_async() // conservative estimate for final encoded size new_log_jump_to = round_up_to(t.op_bl.length() + super.block_size * 2, max_alloc_size); - t.op_jump(log_seq, new_log_jump_to); + t.op_jump(log.seq_live, new_log_jump_to); // allocate //FIXME: check if we want DB here? @@ -2443,8 +2446,8 @@ void BlueFS::compact_log_async() // swap the log files. New log file is the log file now. new_log->fnode.swap_extents(log_file->fnode); - log_writer->pos = log_writer->file->fnode.size = - log_writer->pos - old_log_jump_to + new_log_jump_to; + log.writer->pos = log.writer->file->fnode.size = + log.writer->pos - old_log_jump_to + new_log_jump_to; vselector->add_usage(log_file->vselector_hint, log_file->fnode); @@ -2464,7 +2467,7 @@ void BlueFS::compact_log_async() // 7. release old space dout(10) << __func__ << " release old log extents " << old_extents << dendl; { - std::lock_guard dl(dirty_lock); + std::lock_guard dl(dirty.lock); for (auto& r : old_extents) { pending_release[r.bdev].insert(r.offset, r.length); } @@ -2475,10 +2478,10 @@ void BlueFS::compact_log_async() // flush_special does not dirty files /* if (new_log->dirty_seq) { - std::lock_guard dl(dirty_lock); - ceph_assert(dirty_files.count(new_log->dirty_seq)); - auto it = dirty_files[new_log->dirty_seq].iterator_to(*new_log); - dirty_files[new_log->dirty_seq].erase(it); + std::lock_guard dl(dirty.lock); + ceph_assert(dirty.files.count(new_log->dirty_seq)); + auto it = dirty.files[new_log->dirty_seq].iterator_to(*new_log); + dirty.files[new_log->dirty_seq].erase(it); } */ new_log_writer = nullptr; @@ -2503,38 +2506,41 @@ void BlueFS::_pad_bl(bufferlist& bl) } /** - * Adds file modifications from `dirty_files` to bluefs transactions - * already stored in `log_t`. Writes them to disk and waits until are stable. + * Adds file modifications from `dirty.files` to bluefs transactions + * already stored in `log.t`. Writes them to disk and waits until are stable. * Guarantees that file modifications with `want_seq` are already stable on disk. * In addition may insert jump forward transaction to log write position `jump_to`. * * it should lock ability to: - * 1) add to log_t - * 2) modify dirty_files + * 1) add to log.t + * 2) modify dirty.files * 3) add to pending_release * - * pending_release and log_t go with same lock + * pending_release and log.t go with same lock */ -// Adds to log_t file modifications mentioned in `dirty_files`. -// Note: some bluefs ops may have already been stored in log_t transaction. +// Adds to log.t file modifications mentioned in `dirty.files`. +// Note: some bluefs ops may have already been stored in log.t transaction. uint64_t BlueFS::_consume_dirty() { - ceph_assert(ceph_mutex_is_locked(dirty_lock)); - ceph_assert(ceph_mutex_is_locked(log_lock)); + ceph_assert(ceph_mutex_is_locked(dirty.lock)); + ceph_assert(ceph_mutex_is_locked(log.lock)); //acquire new seq - // this will became log_seq_stable once we write - uint64_t seq = log_t.seq = ++log_seq; - log_t.uuid = super.uuid; + // this will became seq_stable once we write + ++dirty.seq_next; + ++log.seq_live; + ceph_assert(dirty.seq_next == log.seq_live + 1); + uint64_t seq = log.t.seq = log.seq_live; + log.t.uuid = super.uuid; // log dirty files - // we just incremented log_seq. It is now illegal to add to dirty_files[log_seq] - auto lsi = dirty_files.find(seq); - if (lsi != dirty_files.end()) { - dout(20) << __func__ << " " << lsi->second.size() << " dirty_files" << dendl; + // we just incremented log_seq. It is now illegal to add to dirty.files[log_seq] + auto lsi = dirty.files.find(seq); + if (lsi != dirty.files.end()) { + dout(20) << __func__ << " " << lsi->second.size() << " dirty.files" << dendl; for (auto &f : lsi->second) { dout(20) << __func__ << " op_file_update " << f.fnode << dendl; - log_t.op_file_update(f.fnode); + log.t.op_file_update(f.fnode); } } return seq; @@ -2544,11 +2550,11 @@ uint64_t BlueFS::_consume_dirty() // Returns space available *BEFORE* adding new space. Signed for additional <0 detection. int64_t BlueFS::_maybe_extend_log() { - ceph_assert(ceph_mutex_is_locked(log_lock)); + ceph_assert(ceph_mutex_is_locked(log.lock)); // allocate some more space (before we run out)? - // BTW: this triggers `flush()` in the `page_aligned_appender` of `log_writer`. - int64_t runway = log_writer->file->fnode.get_allocated() - - log_writer->get_effective_write_pos(); + // BTW: this triggers `flush()` in the `page_aligned_appender` of `log.writer`. + int64_t runway = log.writer->file->fnode.get_allocated() - + log.writer->get_effective_write_pos(); if (runway < (int64_t)cct->_conf->bluefs_min_log_runway) { dout(10) << __func__ << " allocating more log runway (0x" << std::hex << runway << std::dec << " remaining)" << dendl; @@ -2570,26 +2576,26 @@ int64_t BlueFS::_maybe_extend_log() if (log_forbidden_to_expand.load() == true) { return -EWOULDBLOCK; } - vselector->sub_usage(log_writer->file->vselector_hint, log_writer->file->fnode); + vselector->sub_usage(log.writer->file->vselector_hint, log.writer->file->fnode); int r = _allocate( - vselector->select_prefer_bdev(log_writer->file->vselector_hint), + vselector->select_prefer_bdev(log.writer->file->vselector_hint), cct->_conf->bluefs_max_log_runway, - &log_writer->file->fnode); + &log.writer->file->fnode); ceph_assert(r == 0); - vselector->add_usage(log_writer->file->vselector_hint, log_writer->file->fnode); - log_t.op_file_update(log_writer->file->fnode); + vselector->add_usage(log.writer->file->vselector_hint, log.writer->file->fnode); + log.t.op_file_update(log.writer->file->fnode); } return runway; } void BlueFS::_flush_and_sync_log_core(int64_t runway) { - ceph_assert(ceph_mutex_is_locked(log_lock)); - dout(10) << __func__ << " " << log_t << dendl; + ceph_assert(ceph_mutex_is_locked(log.lock)); + dout(10) << __func__ << " " << log.t << dendl; bufferlist bl; bl.reserve(super.block_size); - encode(log_t, bl); + encode(log.t, bl); // pad to block boundary size_t realign = super.block_size - (bl.length() % super.block_size); if (realign && realign != super.block_size) @@ -2602,29 +2608,29 @@ void BlueFS::_flush_and_sync_log_core(int64_t runway) // transaction will not fit extents before growth -> data loss on _replay } - log_writer->append(bl); + log.writer->append(bl); - log_t.clear(); - log_t.seq = 0; // just so debug output is less confusing + log.t.clear(); + log.t.seq = 0; // just so debug output is less confusing - int r = _flush_special(log_writer); + int r = _flush_special(log.writer); ceph_assert(r == 0); } -// Clears dirty_files up to (including) seq_stable. +// Clears dirty.files up to (including) seq_stable. void BlueFS::clear_dirty_set_stable(uint64_t seq) { - std::lock_guard lg(dirty_lock); + std::lock_guard lg(dirty.lock); // clean dirty files - if (seq > log_seq_stable) { - log_seq_stable = seq; - dout(20) << __func__ << " log_seq_stable " << log_seq_stable << dendl; + if (seq > dirty.seq_stable) { + dirty.seq_stable = seq; + dout(20) << __func__ << " seq_stable " << dirty.seq_stable << dendl; // undirty all files that were already streamed to log - auto p = dirty_files.begin(); - while (p != dirty_files.end()) { - if (p->first > log_seq_stable) { + auto p = dirty.files.begin(); + while (p != dirty.files.end()) { + if (p->first > dirty.seq_stable) { dout(20) << __func__ << " done cleaning up dirty files" << dendl; break; } @@ -2633,17 +2639,17 @@ void BlueFS::clear_dirty_set_stable(uint64_t seq) while (l != p->second.end()) { File *file = &*l; ceph_assert(file->dirty_seq > 0); - ceph_assert(file->dirty_seq <= log_seq_stable); + ceph_assert(file->dirty_seq <= dirty.seq_stable); dout(20) << __func__ << " cleaned file " << file->fnode << dendl; file->dirty_seq = 0; p->second.erase(l++); } ceph_assert(p->second.empty()); - dirty_files.erase(p++); + dirty.files.erase(p++); } } else { - dout(20) << __func__ << " log_seq_stable " << log_seq_stable + dout(20) << __func__ << " seq_stable " << dirty.seq_stable << " already >= out seq " << seq << ", we lost a race against another log flush, done" << dendl; } @@ -2677,22 +2683,22 @@ int BlueFS::flush_and_sync_log(uint64_t want_seq) // we synchronize writing to log, by lock to log_lock int64_t available_runway; do { - log_lock.lock(); - dirty_lock.lock(); - if (want_seq && want_seq <= log_seq_stable) { - dout(10) << __func__ << " want_seq " << want_seq << " <= log_seq_stable " - << log_seq_stable << ", done" << dendl; - dirty_lock.unlock(); - log_lock.unlock(); + log.lock.lock(); + dirty.lock.lock(); + if (want_seq && want_seq <= dirty.seq_stable) { + dout(10) << __func__ << " want_seq " << want_seq << " <= seq_stable " + << dirty.seq_stable << ", done" << dendl; + dirty.lock.unlock(); + log.lock.unlock(); return 0; } available_runway = _maybe_extend_log(); if (available_runway == -EWOULDBLOCK) { // we are in need of adding runway, but we are during log-switch from compaction - dirty_lock.unlock(); - //instead log_lock_unlock() do move ownership - std::unique_lock ll(log_lock, std::adopt_lock); + dirty.lock.unlock(); + //instead log.lock_unlock() do move ownership + std::unique_lock ll(log.lock, std::adopt_lock); while (log_forbidden_to_expand.load()) { log_cond.wait(ll); } @@ -2706,12 +2712,12 @@ int BlueFS::flush_and_sync_log(uint64_t want_seq) uint64_t seq = _consume_dirty(); vector> to_release(pending_release.size()); to_release.swap(pending_release); - dirty_lock.unlock(); + dirty.lock.unlock(); _flush_and_sync_log_core(available_runway); - _flush_bdev(log_writer); - //now log_lock is no longer needed - log_lock.unlock(); + _flush_bdev(log.writer); + //now log.lock is no longer needed + log.lock.unlock(); clear_dirty_set_stable(seq); release_pending_allocations(to_release); @@ -2724,26 +2730,26 @@ int BlueFS::flush_and_sync_log(uint64_t want_seq) int BlueFS::_flush_and_sync_log_jump(uint64_t jump_to, int64_t available_runway) { - ceph_assert(ceph_mutex_is_locked(log_lock)); + ceph_assert(ceph_mutex_is_locked(log.lock)); ceph_assert(jump_to); // we synchronize writing to log, by lock to log_lock - dirty_lock.lock(); + dirty.lock.lock(); uint64_t seq = _consume_dirty(); vector> to_release(pending_release.size()); to_release.swap(pending_release); - dirty_lock.unlock(); + dirty.lock.unlock(); _flush_and_sync_log_core(available_runway); dout(10) << __func__ << " jumping log offset from 0x" << std::hex - << log_writer->pos << " -> 0x" << jump_to << std::dec << dendl; - log_writer->pos = jump_to; - vselector->sub_usage(log_writer->file->vselector_hint, log_writer->file->fnode.size); - log_writer->file->fnode.size = jump_to; - vselector->add_usage(log_writer->file->vselector_hint, log_writer->file->fnode.size); + << log.writer->pos << " -> 0x" << jump_to << std::dec << dendl; + log.writer->pos = jump_to; + vselector->sub_usage(log.writer->file->vselector_hint, log.writer->file->fnode.size); + log.writer->file->fnode.size = jump_to; + vselector->add_usage(log.writer->file->vselector_hint, log.writer->file->fnode.size); - _flush_bdev(log_writer); + _flush_bdev(log.writer); clear_dirty_set_stable(seq); release_pending_allocations(to_release); @@ -2798,26 +2804,26 @@ ceph::bufferlist BlueFS::FileWriter::flush_buffer( int BlueFS::_signal_dirty_to_log(FileWriter *h) { ceph_assert(ceph_mutex_is_locked(h->lock)); - std::lock_guard dl(dirty_lock); + std::lock_guard dl(dirty.lock); h->file->fnode.mtime = ceph_clock_now(); ceph_assert(h->file->fnode.ino >= 1); if (h->file->dirty_seq == 0) { - h->file->dirty_seq = log_seq + 1; - dirty_files[h->file->dirty_seq].push_back(*h->file); - dout(20) << __func__ << " dirty_seq = " << log_seq + 1 + h->file->dirty_seq = dirty.seq_next; + dirty.files[h->file->dirty_seq].push_back(*h->file); + dout(20) << __func__ << " dirty_seq = " << dirty.seq_next << " (was clean)" << dendl; } else { - if (h->file->dirty_seq != log_seq + 1) { + if (h->file->dirty_seq != dirty.seq_next) { // need re-dirty, erase from list first - ceph_assert(dirty_files.count(h->file->dirty_seq)); - auto it = dirty_files[h->file->dirty_seq].iterator_to(*h->file); - dirty_files[h->file->dirty_seq].erase(it); - h->file->dirty_seq = log_seq + 1; - dirty_files[h->file->dirty_seq].push_back(*h->file); - dout(20) << __func__ << " dirty_seq = " << log_seq + 1 + ceph_assert(dirty.files.count(h->file->dirty_seq)); + auto it = dirty.files[h->file->dirty_seq].iterator_to(*h->file); + dirty.files[h->file->dirty_seq].erase(it); + h->file->dirty_seq = dirty.seq_next; + dirty.files[h->file->dirty_seq].push_back(*h->file); + dout(20) << __func__ << " dirty_seq = " << dirty.seq_next << " (was " << h->file->dirty_seq << ")" << dendl; } else { - dout(20) << __func__ << " dirty_seq = " << log_seq + 1 + dout(20) << __func__ << " dirty_seq = " << dirty.seq_next << " (unchanged, do nothing) " << dendl; } } @@ -3121,8 +3127,8 @@ int BlueFS::truncate(FileWriter *h, uint64_t offset) vselector->sub_usage(h->file->vselector_hint, h->file->fnode.size); h->file->fnode.size = offset; vselector->add_usage(h->file->vselector_hint, h->file->fnode.size); - std::lock_guard ll(log_lock); - log_t.op_file_update(h->file->fnode); + std::lock_guard ll(log.lock); + log.t.op_file_update(h->file->fnode); return 0; } @@ -3142,7 +3148,7 @@ int BlueFS::fsync(FileWriter *h) _flush_bdev(h); if (old_dirty_seq) { - uint64_t s = log_seq; + uint64_t s = log.seq_live; // AKAK !!! locks! dout(20) << __func__ << " file metadata was dirty (" << old_dirty_seq << ") on " << h->file->fnode << ", flushing log" << dendl; flush_and_sync_log(old_dirty_seq); @@ -3154,13 +3160,13 @@ int BlueFS::fsync(FileWriter *h) return 0; } -// be careful - either h->file->lock or log_lock must be taken +// be careful - either h->file->lock or log.lock must be taken void BlueFS::_flush_bdev(FileWriter *h) { if (h->file->fnode.ino != 1) { ceph_assert(ceph_mutex_is_locked(h->lock)); } else { - ceph_assert(ceph_mutex_is_locked(log_lock)); + ceph_assert(ceph_mutex_is_locked(log.lock)); } std::array flush_devs = h->dirty_devs; h->dirty_devs.fill(false); @@ -3327,8 +3333,8 @@ int BlueFS::preallocate(FileRef f, uint64_t off, uint64_t len) vselector->add_usage(f->vselector_hint, f->fnode); if (r < 0) return r; - std::lock_guard ll(log_lock); - log_t.op_file_update(f->fnode); + std::lock_guard ll(log.lock); + log.t.op_file_update(f->fnode); } return 0; } @@ -3337,9 +3343,9 @@ void BlueFS::sync_metadata(bool avoid_compact) { bool can_skip_flush; { - std::lock_guard ll(log_lock); - std::lock_guard dl(dirty_lock); - can_skip_flush = log_t.empty() && dirty_files.empty(); + std::lock_guard ll(log.lock); + std::lock_guard dl(dirty.lock); + can_skip_flush = log.t.empty() && dirty.files.empty(); } if (can_skip_flush) { dout(10) << __func__ << " - no pending log events" << dendl; @@ -3376,11 +3382,11 @@ int BlueFS::open_for_write( FileWriter **h, bool overwrite) { - std::lock_guard dirl(dirs_lock); + std::lock_guard dirl(nodes.lock); dout(10) << __func__ << " " << dirname << "/" << filename << dendl; - map::iterator p = dir_map.find(dirname); + map::iterator p = nodes.dir_map.find(dirname); DirRef dir; - if (p == dir_map.end()) { + if (p == nodes.dir_map.end()) { // implicitly create the dir dout(20) << __func__ << " dir " << dirname << " does not exist" << dendl; @@ -3402,7 +3408,7 @@ int BlueFS::open_for_write( } file = ceph::make_ref(); file->fnode.ino = ++ino_last; - file_map[ino_last] = file; + nodes.file_map[ino_last] = file; dir->file_map[string{filename}] = file; ++file->refs; create = true; @@ -3439,10 +3445,10 @@ int BlueFS::open_for_write( << " vsel_hint " << file->vselector_hint << dendl; { - std::lock_guard ll(log_lock); - log_t.op_file_update(file->fnode); + std::lock_guard ll(log.lock); + log.t.op_file_update(file->fnode); if (create) - log_t.op_dir_link(dirname, filename, file->fnode.ino); + log.t.op_dir_link(dirname, filename, file->fnode.ino); } *h = _create_writer(file); @@ -3523,11 +3529,11 @@ int BlueFS::open_for_read( FileReader **h, bool random) { - std::lock_guard dirl(dirs_lock); + std::lock_guard dirl(nodes.lock); dout(10) << __func__ << " " << dirname << "/" << filename << (random ? " (random)":" (sequential)") << dendl; - map::iterator p = dir_map.find(dirname); - if (p == dir_map.end()) { + map::iterator p = nodes.dir_map.find(dirname); + if (p == nodes.dir_map.end()) { dout(20) << __func__ << " dir " << dirname << " not found" << dendl; return -ENOENT; } @@ -3552,12 +3558,12 @@ int BlueFS::rename( std::string_view old_dirname, std::string_view old_filename, std::string_view new_dirname, std::string_view new_filename) { - std::lock_guard dirl(dirs_lock); - std::lock_guard ll(log_lock); + std::lock_guard dirl(nodes.lock); + std::lock_guard ll(log.lock); dout(10) << __func__ << " " << old_dirname << "/" << old_filename << " -> " << new_dirname << "/" << new_filename << dendl; - map::iterator p = dir_map.find(old_dirname); - if (p == dir_map.end()) { + map::iterator p = nodes.dir_map.find(old_dirname); + if (p == nodes.dir_map.end()) { dout(20) << __func__ << " dir " << old_dirname << " not found" << dendl; return -ENOENT; } @@ -3571,8 +3577,8 @@ int BlueFS::rename( } FileRef file = q->second; - p = dir_map.find(new_dirname); - if (p == dir_map.end()) { + p = nodes.dir_map.find(new_dirname); + if (p == nodes.dir_map.end()) { dout(20) << __func__ << " dir " << new_dirname << " not found" << dendl; return -ENOENT; } @@ -3583,7 +3589,7 @@ int BlueFS::rename( << ") file " << new_filename << " already exists, unlinking" << dendl; ceph_assert(q->second != file); - log_t.op_dir_unlink(new_dirname, new_filename); + log.t.op_dir_unlink(new_dirname, new_filename); _drop_link(q->second); } @@ -3593,33 +3599,33 @@ int BlueFS::rename( new_dir->file_map[string{new_filename}] = file; old_dir->file_map.erase(string{old_filename}); - log_t.op_dir_link(new_dirname, new_filename, file->fnode.ino); - log_t.op_dir_unlink(old_dirname, old_filename); + log.t.op_dir_link(new_dirname, new_filename, file->fnode.ino); + log.t.op_dir_unlink(old_dirname, old_filename); return 0; } int BlueFS::mkdir(std::string_view dirname) { - std::lock_guard dirl(dirs_lock); - std::lock_guard ll(log_lock); + std::lock_guard dirl(nodes.lock); + std::lock_guard ll(log.lock); dout(10) << __func__ << " " << dirname << dendl; - map::iterator p = dir_map.find(dirname); - if (p != dir_map.end()) { + map::iterator p = nodes.dir_map.find(dirname); + if (p != nodes.dir_map.end()) { dout(20) << __func__ << " dir " << dirname << " exists" << dendl; return -EEXIST; } - dir_map[string{dirname}] = ceph::make_ref(); - log_t.op_dir_create(dirname); + nodes.dir_map[string{dirname}] = ceph::make_ref(); + log.t.op_dir_create(dirname); return 0; } int BlueFS::rmdir(std::string_view dirname) { - std::lock_guard dirl(dirs_lock); - std::lock_guard ll(log_lock); + std::lock_guard dirl(nodes.lock); + std::lock_guard ll(log.lock); dout(10) << __func__ << " " << dirname << dendl; - auto p = dir_map.find(dirname); - if (p == dir_map.end()) { + auto p = nodes.dir_map.find(dirname); + if (p == nodes.dir_map.end()) { dout(20) << __func__ << " dir " << dirname << " does not exist" << dendl; return -ENOENT; } @@ -3628,16 +3634,16 @@ int BlueFS::rmdir(std::string_view dirname) dout(20) << __func__ << " dir " << dirname << " not empty" << dendl; return -ENOTEMPTY; } - dir_map.erase(string{dirname}); - log_t.op_dir_remove(dirname); + nodes.dir_map.erase(string{dirname}); + log.t.op_dir_remove(dirname); return 0; } bool BlueFS::dir_exists(std::string_view dirname) { - std::lock_guard dirl(dirs_lock); - map::iterator p = dir_map.find(dirname); - bool exists = p != dir_map.end(); + std::lock_guard dirl(nodes.lock); + map::iterator p = nodes.dir_map.find(dirname); + bool exists = p != nodes.dir_map.end(); dout(10) << __func__ << " " << dirname << " = " << (int)exists << dendl; return exists; } @@ -3645,10 +3651,10 @@ bool BlueFS::dir_exists(std::string_view dirname) int BlueFS::stat(std::string_view dirname, std::string_view filename, uint64_t *size, utime_t *mtime) { - std::lock_guard dirl(dirs_lock); + std::lock_guard dirl(nodes.lock); dout(10) << __func__ << " " << dirname << "/" << filename << dendl; - map::iterator p = dir_map.find(dirname); - if (p == dir_map.end()) { + map::iterator p = nodes.dir_map.find(dirname); + if (p == nodes.dir_map.end()) { dout(20) << __func__ << " dir " << dirname << " not found" << dendl; return -ENOENT; } @@ -3673,10 +3679,10 @@ int BlueFS::stat(std::string_view dirname, std::string_view filename, int BlueFS::lock_file(std::string_view dirname, std::string_view filename, FileLock **plock) { - std::lock_guard dirl(dirs_lock); + std::lock_guard dirl(nodes.lock); dout(10) << __func__ << " " << dirname << "/" << filename << dendl; - map::iterator p = dir_map.find(dirname); - if (p == dir_map.end()) { + map::iterator p = nodes.dir_map.find(dirname); + if (p == nodes.dir_map.end()) { dout(20) << __func__ << " dir " << dirname << " not found" << dendl; return -ENOENT; } @@ -3690,12 +3696,12 @@ int BlueFS::lock_file(std::string_view dirname, std::string_view filename, file = ceph::make_ref(); file->fnode.ino = ++ino_last; file->fnode.mtime = ceph_clock_now(); - file_map[ino_last] = file; + nodes.file_map[ino_last] = file; dir->file_map[string{filename}] = file; ++file->refs; - std::lock_guard ll(log_lock); - log_t.op_file_update(file->fnode); - log_t.op_dir_link(dirname, filename, file->fnode.ino); + std::lock_guard ll(log.lock); + log.t.op_file_update(file->fnode); + log.t.op_dir_link(dirname, filename, file->fnode.ino); } else { file = q->second; if (file->locked) { @@ -3712,7 +3718,7 @@ int BlueFS::lock_file(std::string_view dirname, std::string_view filename, int BlueFS::unlock_file(FileLock *fl) { - std::lock_guard dirl(dirs_lock); + std::lock_guard dirl(nodes.lock); dout(10) << __func__ << " " << fl << " on " << fl->file->fnode << dendl; ceph_assert(fl->file->locked); fl->file->locked = false; @@ -3726,18 +3732,18 @@ int BlueFS::readdir(std::string_view dirname, vector *ls) if (!dirname.empty() && dirname.back() == '/') { dirname.remove_suffix(1); } - std::lock_guard dirl(dirs_lock); + std::lock_guard dirl(nodes.lock); dout(10) << __func__ << " " << dirname << dendl; if (dirname.empty()) { // list dirs - ls->reserve(dir_map.size() + 2); - for (auto& q : dir_map) { + ls->reserve(nodes.dir_map.size() + 2); + for (auto& q : nodes.dir_map) { ls->push_back(q.first); } } else { // list files in dir - map::iterator p = dir_map.find(dirname); - if (p == dir_map.end()) { + map::iterator p = nodes.dir_map.find(dirname); + if (p == nodes.dir_map.end()) { dout(20) << __func__ << " dir " << dirname << " not found" << dendl; return -ENOENT; } @@ -3754,11 +3760,11 @@ int BlueFS::readdir(std::string_view dirname, vector *ls) int BlueFS::unlink(std::string_view dirname, std::string_view filename) { - std::lock_guard dirl(dirs_lock); - std::lock_guard ll(log_lock); + std::lock_guard dirl(nodes.lock); + std::lock_guard ll(log.lock); dout(10) << __func__ << " " << dirname << "/" << filename << dendl; - map::iterator p = dir_map.find(dirname); - if (p == dir_map.end()) { + map::iterator p = nodes.dir_map.find(dirname); + if (p == nodes.dir_map.end()) { dout(20) << __func__ << " dir " << dirname << " not found" << dendl; return -ENOENT; } @@ -3776,7 +3782,7 @@ int BlueFS::unlink(std::string_view dirname, std::string_view filename) return -EBUSY; } dir->file_map.erase(string{filename}); - log_t.op_dir_unlink(dirname, filename); + log.t.op_dir_unlink(dirname, filename); _drop_link(file); return 0; } @@ -3855,7 +3861,7 @@ int BlueFS::do_replay_recovery_read(FileReader *log_reader, dout(2) << __func__ << " processing " << get_device_name(dev) << dendl; interval_set disk_regions; disk_regions.insert(0, bdev[dev]->get_size()); - for (auto f : file_map) { + for (auto f : nodes.file_map) { auto& e = f.second->fnode.extents; for (auto& p : e) { if (p.bdev == dev) { diff --git a/src/os/bluestore/BlueFS.h b/src/os/bluestore/BlueFS.h index 4f368841ff4d9..710f1809bca3e 100644 --- a/src/os/bluestore/BlueFS.h +++ b/src/os/bluestore/BlueFS.h @@ -322,22 +322,31 @@ class BlueFS { }; // cache - mempool::bluefs::map> dir_map; ///< dirname -> Dir - mempool::bluefs::unordered_map file_map; ///< ino -> File - - // map of dirty files, files of same dirty_seq are grouped into list. - std::map dirty_files; + struct { + ceph::mutex lock = ceph::make_mutex("BlueFS::nodes.lock"); + mempool::bluefs::map> dir_map; ///< dirname -> Dir + mempool::bluefs::unordered_map file_map; ///< ino -> File + } nodes; bluefs_super_t super; ///< latest superblock (as last written) uint64_t ino_last = 0; ///< last assigned ino (this one is in use) - uint64_t log_seq = 0; ///< last used log seq (by current pending log_t) - uint64_t log_seq_stable = 0; ///< last stable/synced log seq - FileWriter *log_writer = 0; ///< writer for the log - bluefs_transaction_t log_t; ///< pending, unwritten log transaction - bool log_flushing = false; ///< true while flushing the log + + struct { + ceph::mutex lock = ceph::make_mutex("BlueFS::log.lock"); + uint64_t seq_live = 0; //seq that log is currently writing to + FileWriter *writer = 0; + bluefs_transaction_t t; + } log; + + struct { + ceph::mutex lock = ceph::make_mutex("BlueFS::dirty.lock"); + uint64_t seq_stable = 0; //seq that is now stable on disk + uint64_t seq_next = 1; //seq that is ongoing and not yet stable + // map of dirty files, files of same dirty_seq are grouped into list. + std::map files; + } dirty; ceph::condition_variable log_cond; ///< used for state control between log flush / log compaction - ceph::mutex cond_lock = ceph::make_mutex("BlueFS::cond_lock"); ///< .... std::atomic log_is_compacting{false}; ///< signals that bluefs log is already ongoing compaction std::atomic log_forbidden_to_expand{false}; ///< used to signal that async compaction is in state /// that prohibits expansion of bluefs log From f833b461ad6f180838a0682019417abcfd9103b8 Mon Sep 17 00:00:00 2001 From: Adam Kupczyk Date: Thu, 24 Jun 2021 11:08:42 +0200 Subject: [PATCH 0043/2866] os/bluestore/bluefs: Clean up log seq progression Splits seq into seq_live and seq_stable. Cleans up log sequencing. Signed-off-by: Adam Kupczyk --- src/os/bluestore/BlueFS.cc | 84 ++++++++++++++++++++++++-------------- src/os/bluestore/BlueFS.h | 12 +++--- 2 files changed, 61 insertions(+), 35 deletions(-) diff --git a/src/os/bluestore/BlueFS.cc b/src/os/bluestore/BlueFS.cc index e501159b2953c..3a202c65df95f 100644 --- a/src/os/bluestore/BlueFS.cc +++ b/src/os/bluestore/BlueFS.cc @@ -514,6 +514,8 @@ int BlueFS::mkfs(uuid_d osd_uuid, const bluefs_layout_t& layout) log.writer = _create_writer(log_file); // initial txn + ceph_assert(log.seq_live == 1); + log.t.seq = 1; log.t.op_init(); flush_and_sync_log(); @@ -1221,7 +1223,7 @@ int BlueFS::_replay(bool noop, bool to_stdout) << std::endl; } - ceph_assert(next_seq >= log_seq); + ceph_assert(next_seq > log_seq); log_seq = next_seq - 1; // we will increment it below uint64_t skip = offset - read_pos; if (skip) { @@ -1249,7 +1251,7 @@ int BlueFS::_replay(bool noop, bool to_stdout) << ": op_jump_seq " << next_seq << std::endl; } - ceph_assert(next_seq >= log_seq); + ceph_assert(next_seq > log_seq); log_seq = next_seq - 1; // we will increment it below } break; @@ -1464,8 +1466,10 @@ int BlueFS::_replay(bool noop, bool to_stdout) } if (!noop) { vselector->add_usage(log_file->vselector_hint, log_file->fnode); - log.seq_live = log_seq; - dirty.seq_next = log_seq + 1; + log.seq_live = log_seq + 1; + dirty.seq_live = log_seq + 1; + log.t.seq = log.seq_live; + //log.seq_stable = log_seq; dirty.seq_stable = log_seq; } @@ -2198,8 +2202,13 @@ void BlueFS::rewrite_log_and_layout_sync(bool allocate_with_fallback, File *log_file = log.writer->file.get(); - // clear out log (be careful who calls us!!!) + // log.t.seq is always set to current live seq + ceph_assert(log.t.seq == log.seq_live); + // Capturing entire state. Dump anything that has been stored there. log.t.clear(); + log.t.seq = log.seq_live; + // From now on, no changes to log.t are permitted until we finish rewriting log. + // Can allow dirty to remain dirty - log.seq_live will not change. dout(20) << __func__ << " super_dev:" << super_dev << " log_dev:" << log_dev @@ -2351,9 +2360,10 @@ void BlueFS::compact_log_async() // update the log file change and log a jump to the offset where we want to // write the new entries - log.t.op_file_update(log_file->fnode); // 1.2 - log.t.op_jump(log.seq_live, old_log_jump_to); // 1.3 - + log.t.op_file_update(log_file->fnode); + // jump to new position should mean next seq + log.t.op_jump(log.seq_live + 1, old_log_jump_to); + uint64_t seq_now = log.seq_live; // we need to flush all bdev because we will be streaming all dirty files to log // TODO - think - if _flush_and_sync_log_jump will not add dirty files nor release pending allocations // then flush_bdev() will not be necessary @@ -2377,7 +2387,8 @@ void BlueFS::compact_log_async() // conservative estimate for final encoded size new_log_jump_to = round_up_to(t.op_bl.length() + super.block_size * 2, max_alloc_size); - t.op_jump(log.seq_live, new_log_jump_to); + //newly constructed log head will jump to what we had before + t.op_jump(seq_now, new_log_jump_to); // allocate //FIXME: check if we want DB here? @@ -2519,20 +2530,32 @@ void BlueFS::_pad_bl(bufferlist& bl) * pending_release and log.t go with same lock */ -// Adds to log.t file modifications mentioned in `dirty.files`. -// Note: some bluefs ops may have already been stored in log.t transaction. -uint64_t BlueFS::_consume_dirty() +// Returns log seq that was live before advance. +uint64_t BlueFS::_log_advance_seq() { ceph_assert(ceph_mutex_is_locked(dirty.lock)); ceph_assert(ceph_mutex_is_locked(log.lock)); //acquire new seq // this will became seq_stable once we write - ++dirty.seq_next; - ++log.seq_live; - ceph_assert(dirty.seq_next == log.seq_live + 1); - uint64_t seq = log.t.seq = log.seq_live; + ceph_assert(dirty.seq_stable < dirty.seq_live); + ceph_assert(log.t.seq == log.seq_live); + uint64_t seq = log.seq_live; log.t.uuid = super.uuid; + ++dirty.seq_live; + ++log.seq_live; + ceph_assert(dirty.seq_live == log.seq_live); + return seq; +} + + +// Adds to log.t file modifications mentioned in `dirty.files`. +// Note: some bluefs ops may have already been stored in log.t transaction. +void BlueFS::_consume_dirty(uint64_t seq) +{ + ceph_assert(ceph_mutex_is_locked(dirty.lock)); + ceph_assert(ceph_mutex_is_locked(log.lock)); + // log dirty files // we just incremented log_seq. It is now illegal to add to dirty.files[log_seq] auto lsi = dirty.files.find(seq); @@ -2543,7 +2566,6 @@ uint64_t BlueFS::_consume_dirty() log.t.op_file_update(f.fnode); } } - return seq; } // Extends log if its free space is smaller then bluefs_min_log_runway. @@ -2610,8 +2632,9 @@ void BlueFS::_flush_and_sync_log_core(int64_t runway) log.writer->append(bl); + // prepare log for new transactions log.t.clear(); - log.t.seq = 0; // just so debug output is less confusing + log.t.seq = log.seq_live; int r = _flush_special(log.writer); ceph_assert(r == 0); @@ -2706,10 +2729,10 @@ int BlueFS::flush_and_sync_log(uint64_t want_seq) ceph_assert(available_runway >= 0); } } while (available_runway < 0); - - ceph_assert(want_seq == 0 || want_seq <= log_seq + 1); // illegal to request seq that was not created yet - - uint64_t seq = _consume_dirty(); + + ceph_assert(want_seq == 0 || want_seq <= dirty.seq_live); // illegal to request seq that was not created yet + uint64_t seq =_log_advance_seq(); + _consume_dirty(seq); vector> to_release(pending_release.size()); to_release.swap(pending_release); dirty.lock.unlock(); @@ -2736,7 +2759,8 @@ int BlueFS::_flush_and_sync_log_jump(uint64_t jump_to, // we synchronize writing to log, by lock to log_lock dirty.lock.lock(); - uint64_t seq = _consume_dirty(); + uint64_t seq =_log_advance_seq(); + _consume_dirty(seq); vector> to_release(pending_release.size()); to_release.swap(pending_release); dirty.lock.unlock(); @@ -2808,22 +2832,22 @@ int BlueFS::_signal_dirty_to_log(FileWriter *h) h->file->fnode.mtime = ceph_clock_now(); ceph_assert(h->file->fnode.ino >= 1); if (h->file->dirty_seq == 0) { - h->file->dirty_seq = dirty.seq_next; + h->file->dirty_seq = dirty.seq_live; dirty.files[h->file->dirty_seq].push_back(*h->file); - dout(20) << __func__ << " dirty_seq = " << dirty.seq_next + dout(20) << __func__ << " dirty_seq = " << dirty.seq_live << " (was clean)" << dendl; } else { - if (h->file->dirty_seq != dirty.seq_next) { + if (h->file->dirty_seq != dirty.seq_live) { // need re-dirty, erase from list first ceph_assert(dirty.files.count(h->file->dirty_seq)); auto it = dirty.files[h->file->dirty_seq].iterator_to(*h->file); dirty.files[h->file->dirty_seq].erase(it); - h->file->dirty_seq = dirty.seq_next; + h->file->dirty_seq = dirty.seq_live; dirty.files[h->file->dirty_seq].push_back(*h->file); - dout(20) << __func__ << " dirty_seq = " << dirty.seq_next + dout(20) << __func__ << " dirty_seq = " << dirty.seq_live << " (was " << h->file->dirty_seq << ")" << dendl; } else { - dout(20) << __func__ << " dirty_seq = " << dirty.seq_next + dout(20) << __func__ << " dirty_seq = " << dirty.seq_live << " (unchanged, do nothing) " << dendl; } } @@ -3154,7 +3178,7 @@ int BlueFS::fsync(FileWriter *h) flush_and_sync_log(old_dirty_seq); // AK - TODO - think - how can dirty_seq change if we are under h lock? ceph_assert(h->file->dirty_seq == 0 || // cleaned - h->file->dirty_seq > s); // or redirtied by someone else + h->file->dirty_seq >= s); // or redirtied by someone else } maybe_compact_log(); return 0; diff --git a/src/os/bluestore/BlueFS.h b/src/os/bluestore/BlueFS.h index 710f1809bca3e..a6e2cab95bdc4 100644 --- a/src/os/bluestore/BlueFS.h +++ b/src/os/bluestore/BlueFS.h @@ -333,7 +333,8 @@ class BlueFS { struct { ceph::mutex lock = ceph::make_mutex("BlueFS::log.lock"); - uint64_t seq_live = 0; //seq that log is currently writing to + //uint64_t seq_stable = 0; //seq that is now stable on disk AK - consider also mirroring this + uint64_t seq_live = 1; //seq that log is currently writing to; mirrors dirty.seq_live FileWriter *writer = 0; bluefs_transaction_t t; } log; @@ -341,7 +342,7 @@ class BlueFS { struct { ceph::mutex lock = ceph::make_mutex("BlueFS::dirty.lock"); uint64_t seq_stable = 0; //seq that is now stable on disk - uint64_t seq_next = 1; //seq that is ongoing and not yet stable + uint64_t seq_live = 1; //seq that is ongoing and dirty files will be written to // map of dirty files, files of same dirty_seq are grouped into list. std::map files; } dirty; @@ -425,9 +426,10 @@ class BlueFS { int64_t _maybe_extend_log(); void _extend_log(); - uint64_t _consume_dirty(); - void _clear_dirty_set_stable(uint64_t seq_stable); - void _release_pending_allocations(std::vector>& to_release); + uint64_t _log_advance_seq(); + void _consume_dirty(uint64_t seq); + void clear_dirty_set_stable(uint64_t seq_stable); + void release_pending_allocations(std::vector>& to_release); void _flush_and_sync_log_core(int64_t available_runway); int _flush_and_sync_log_jump(uint64_t jump_to, From 9ad938f8156e57e0b4073d5718cd8dabab07f691 Mon Sep 17 00:00:00 2001 From: Adam Kupczyk Date: Mon, 28 Jun 2021 12:56:36 +0200 Subject: [PATCH 0044/2866] os/bluestore/bluefs: Rename functions to reflect lock that are used This is modification that only changes names of functions, so tracking of potential deadlocks is simpler. All internal functions start with _. Signed-off-by: Adam Kupczyk --- src/os/bluestore/BlueFS.cc | 204 +++++++++++++++++++------------------ src/os/bluestore/BlueFS.h | 65 +++++++----- 2 files changed, 148 insertions(+), 121 deletions(-) diff --git a/src/os/bluestore/BlueFS.cc b/src/os/bluestore/BlueFS.cc index 3a202c65df95f..f8145ce889000 100644 --- a/src/os/bluestore/BlueFS.cc +++ b/src/os/bluestore/BlueFS.cc @@ -464,7 +464,7 @@ void BlueFS::dump_block_extents(ostream& out) int BlueFS::get_block_extents(unsigned id, interval_set *extents) { - std::lock_guard dirl(nodes.lock); + std::lock_guard nl(nodes.lock); dout(10) << __func__ << " bdev " << id << dendl; ceph_assert(id < alloc.size()); for (auto& p : nodes.file_map) { @@ -517,13 +517,13 @@ int BlueFS::mkfs(uuid_d osd_uuid, const bluefs_layout_t& layout) ceph_assert(log.seq_live == 1); log.t.seq = 1; log.t.op_init(); - flush_and_sync_log(); + _flush_and_sync_log_LD(); // write supers super.log_fnode = log_file->fnode; super.memorized_layout = layout; _write_super(BDEV_DB); - flush_bdev(); + _flush_bdev(); // clean up super = bluefs_super_t(); @@ -866,7 +866,7 @@ int BlueFS::prepare_new_device(int id, const bluefs_layout_t& layout) new_log_dev_cur = BDEV_NEWDB; new_log_dev_next = BDEV_DB; } - rewrite_log_and_layout_sync(false, + _rewrite_log_and_layout_sync_LN_LD(false, BDEV_NEWDB, new_log_dev_cur, new_log_dev_next, @@ -874,7 +874,7 @@ int BlueFS::prepare_new_device(int id, const bluefs_layout_t& layout) layout); //} } else if(id == BDEV_NEWWAL) { - rewrite_log_and_layout_sync(false, + _rewrite_log_and_layout_sync_LN_LD(false, BDEV_DB, BDEV_NEWWAL, BDEV_WAL, @@ -1089,7 +1089,7 @@ int BlueFS::_replay(bool noop, bool to_stdout) int r = _read(log_reader, read_pos, super.block_size, &bl, NULL); if (r != (int)super.block_size && cct->_conf->bluefs_replay_recovery) { - r += do_replay_recovery_read(log_reader, pos, read_pos + r, super.block_size - r, &bl); + r += _do_replay_recovery_read(log_reader, pos, read_pos + r, super.block_size - r, &bl); } assert(r == (int)super.block_size); read_pos += r; @@ -1149,7 +1149,7 @@ int BlueFS::_replay(bool noop, bool to_stdout) << ", which is past eof" << dendl; if (cct->_conf->bluefs_replay_recovery) { //try to search for more data - r += do_replay_recovery_read(log_reader, pos, read_pos + r, more - r, &t); + r += _do_replay_recovery_read(log_reader, pos, read_pos + r, more - r, &t); if (r < (int)more) { //in normal mode we must read r==more, for recovery it is too strict break; @@ -1650,7 +1650,7 @@ int BlueFS::device_migrate_to_existing( new_log_dev_next; } - rewrite_log_and_layout_sync( + _rewrite_log_and_layout_sync_LN_LD( false, (flags & REMOVE_DB) ? BDEV_SLOW : BDEV_DB, new_log_dev_cur, @@ -1785,7 +1785,7 @@ int BlueFS::device_migrate_to_new( BDEV_DB : BDEV_SLOW; - rewrite_log_and_layout_sync( + _rewrite_log_and_layout_sync_LN_LD( false, super_dev, new_log_dev_cur, @@ -1810,7 +1810,7 @@ BlueFS::FileRef BlueFS::_get_file(uint64_t ino) } } -void BlueFS::_drop_link(FileRef file) +void BlueFS::_drop_link_D(FileRef file) { dout(20) << __func__ << " had refs " << file->refs << " on " << file->fnode << dendl; @@ -2078,9 +2078,9 @@ void BlueFS::invalidate_cache(FileRef f, uint64_t offset, uint64_t length) } } -uint64_t BlueFS::estimate_log_size() +uint64_t BlueFS::_estimate_log_size_N() { - std::lock_guard dirl(nodes.lock); + std::lock_guard nl(nodes.lock); int avg_dir_size = 40; // fixme int avg_file_size = 12; uint64_t size = 4096 * 2; @@ -2094,14 +2094,14 @@ void BlueFS::compact_log() { if (!cct->_conf->bluefs_replay_recovery_disable_compact) { if (cct->_conf->bluefs_compact_log_sync) { - compact_log_sync(); + _compact_log_sync_LN_LD(); } else { - compact_log_async(); + _compact_log_async_LD_NF_D(); } } } -bool BlueFS::should_start_compact_log() +bool BlueFS::_should_start_compact_log_L_N() { if (log_is_compacting.load() == true) { // compaction is already running @@ -2109,10 +2109,10 @@ bool BlueFS::should_start_compact_log() } uint64_t current; { - std::lock_guard dirl(log.lock); + std::lock_guard ll(log.lock); current = log.writer->file->fnode.size; } - uint64_t expected = estimate_log_size(); + uint64_t expected = _estimate_log_size_N(); float ratio = (float)current / (float)expected; dout(10) << __func__ << " current 0x" << std::hex << current << " expected " << expected << std::dec @@ -2125,10 +2125,10 @@ bool BlueFS::should_start_compact_log() return true; } -void BlueFS::compact_log_dump_metadata(bluefs_transaction_t *t, +void BlueFS::_compact_log_dump_metadata_N(bluefs_transaction_t *t, int flags) { - std::lock_guard dirl(nodes.lock); + std::lock_guard nl(nodes.lock); t->seq = 1; t->uuid = super.uuid; @@ -2176,12 +2176,12 @@ void BlueFS::compact_log_dump_metadata(bluefs_transaction_t *t, } } -void BlueFS::compact_log_sync() +void BlueFS::_compact_log_sync_LN_LD() { dout(10) << __func__ << dendl; auto prefer_bdev = vselector->select_prefer_bdev(log.writer->file->vselector_hint); - rewrite_log_and_layout_sync(true, + _rewrite_log_and_layout_sync_LN_LD(true, BDEV_DB, prefer_bdev, prefer_bdev, @@ -2190,7 +2190,7 @@ void BlueFS::compact_log_sync() logger->inc(l_bluefs_log_compactions); } -void BlueFS::rewrite_log_and_layout_sync(bool allocate_with_fallback, +void BlueFS::_rewrite_log_and_layout_sync_LN_LD(bool allocate_with_fallback, int super_dev, int log_dev, int log_dev_new, @@ -2216,7 +2216,7 @@ void BlueFS::rewrite_log_and_layout_sync(bool allocate_with_fallback, << " flags:" << flags << dendl; bluefs_transaction_t t; - compact_log_dump_metadata(&t, flags); + _compact_log_dump_metadata_N(&t, flags); dout(20) << __func__ << " op_jump_seq " << log.seq_live << dendl; t.op_jump_seq(log.seq_live); @@ -2260,11 +2260,11 @@ void BlueFS::rewrite_log_and_layout_sync(bool allocate_with_fallback, if (!cct->_conf->bluefs_sync_write) { list completed_ios; _claim_completed_aios(log.writer, &completed_ios); - wait_for_aio(log.writer); + _wait_for_aio(log.writer); completed_ios.clear(); } #endif - flush_bdev(); + _flush_bdev(); super.memorized_layout = layout; super.log_fnode = log_file->fnode; @@ -2279,7 +2279,7 @@ void BlueFS::rewrite_log_and_layout_sync(bool allocate_with_fallback, ++super.version; _write_super(super_dev); - flush_bdev(); + _flush_bdev(); dout(10) << __func__ << " release old log extents " << old_fnode.extents << dendl; std::lock_guard dl(dirty.lock); @@ -2311,7 +2311,7 @@ void BlueFS::rewrite_log_and_layout_sync(bool allocate_with_fallback, * 8. Release the old log space. Clean up. */ -void BlueFS::compact_log_async() +void BlueFS::_compact_log_async_LD_NF_D() //also locks FW for new_writer { dout(10) << __func__ << dendl; // only one compaction allowed at one time @@ -2367,8 +2367,8 @@ void BlueFS::compact_log_async() // we need to flush all bdev because we will be streaming all dirty files to log // TODO - think - if _flush_and_sync_log_jump will not add dirty files nor release pending allocations // then flush_bdev() will not be necessary - flush_bdev(); - _flush_and_sync_log_jump(old_log_jump_to, runway); + _flush_bdev(); + _flush_and_sync_log_jump_D(old_log_jump_to, runway); log.lock.unlock(); // out of jump section - now log can be used to write to @@ -2378,7 +2378,7 @@ void BlueFS::compact_log_async() //this needs files lock //what will happen, if a file is modified *twice* before we stream it to log? //the later state that we capture will be seen earlier and replay will see a temporary retraction (!) - compact_log_dump_metadata(&t, 0); + compact_log_dump_metadata_N(&t, 0); uint64_t max_alloc_size = std::max(alloc_size[BDEV_WAL], std::max(alloc_size[BDEV_DB], @@ -2409,6 +2409,7 @@ void BlueFS::compact_log_async() new_log_writer = _create_writer(new_log); new_log_writer->append(bl); + new_log->lock.lock(); new_log_writer->lock.lock(); // 3. flush r = _flush_special(new_log_writer); @@ -2417,6 +2418,7 @@ void BlueFS::compact_log_async() // 4. wait _flush_bdev(new_log_writer); new_log_writer->lock.unlock(); + new_log->lock.unlock(); // 5. update our log fnode // discard first old_log_jump_to extents @@ -2468,7 +2470,7 @@ void BlueFS::compact_log_async() ++super.version; _write_super(BDEV_DB); - flush_bdev(); + _flush_bdev(); old_forbidden = atomic_exchange(&log_forbidden_to_expand, false); ceph_assert(old_forbidden == true); @@ -2641,9 +2643,9 @@ void BlueFS::_flush_and_sync_log_core(int64_t runway) } // Clears dirty.files up to (including) seq_stable. -void BlueFS::clear_dirty_set_stable(uint64_t seq) +void BlueFS::_clear_dirty_set_stable_D(uint64_t seq) { - std::lock_guard lg(dirty.lock); + std::lock_guard dl(dirty.lock); // clean dirty files if (seq > dirty.seq_stable) { @@ -2678,7 +2680,7 @@ void BlueFS::clear_dirty_set_stable(uint64_t seq) } } -void BlueFS::release_pending_allocations(vector>& to_release) +void BlueFS::_release_pending_allocations(vector>& to_release) { for (unsigned i = 0; i < to_release.size(); ++i) { if (!to_release[i].empty()) { @@ -2701,9 +2703,8 @@ void BlueFS::release_pending_allocations(vector>& to_rele } } -int BlueFS::flush_and_sync_log(uint64_t want_seq) +int BlueFS::_flush_and_sync_log_LD(uint64_t want_seq) { - // we synchronize writing to log, by lock to log_lock int64_t available_runway; do { log.lock.lock(); @@ -2742,15 +2743,15 @@ int BlueFS::flush_and_sync_log(uint64_t want_seq) //now log.lock is no longer needed log.lock.unlock(); - clear_dirty_set_stable(seq); - release_pending_allocations(to_release); + _clear_dirty_set_stable_D(seq); + _release_pending_allocations(to_release); _update_logger_stats(); return 0; } // Flushes log and immediately adjusts log_writer pos. -int BlueFS::_flush_and_sync_log_jump(uint64_t jump_to, +int BlueFS::_flush_and_sync_log_jump_D(uint64_t jump_to, int64_t available_runway) { ceph_assert(ceph_mutex_is_locked(log.lock)); @@ -2775,8 +2776,8 @@ int BlueFS::_flush_and_sync_log_jump(uint64_t jump_to, _flush_bdev(log.writer); - clear_dirty_set_stable(seq); - release_pending_allocations(to_release); + _clear_dirty_set_stable_D(seq); + _release_pending_allocations(to_release); _update_logger_stats(); return 0; @@ -2825,7 +2826,7 @@ ceph::bufferlist BlueFS::FileWriter::flush_buffer( return bl; } -int BlueFS::_signal_dirty_to_log(FileWriter *h) +int BlueFS::_signal_dirty_to_log_D(FileWriter *h) { ceph_assert(ceph_mutex_is_locked(h->lock)); std::lock_guard dl(dirty.lock); @@ -2854,12 +2855,12 @@ int BlueFS::_signal_dirty_to_log(FileWriter *h) return 0; } -void BlueFS::flush_range(FileWriter *h, uint64_t offset, uint64_t length) { +void BlueFS::flush_range/*WF*/(FileWriter *h, uint64_t offset, uint64_t length) { std::unique_lock hl(h->lock); - _flush_range(h, offset, length); + _flush_range_F(h, offset, length); } -int BlueFS::_flush_range(FileWriter *h, uint64_t offset, uint64_t length) +int BlueFS::_flush_range_F(FileWriter *h, uint64_t offset, uint64_t length) { ceph_assert(ceph_mutex_is_locked(h->lock)); dout(10) << __func__ << " " << h << " pos 0x" << std::hex << h->pos @@ -2918,8 +2919,10 @@ int BlueFS::_flush_range(FileWriter *h, uint64_t offset, uint64_t length) int BlueFS::_flush_data(FileWriter *h, uint64_t offset, uint64_t length, bool buffered) { - //ceph_assert(ceph_mutex_is_locked(h->lock)); - //ceph_assert(ceph_mutex_is_locked(h->file->lock)); + if (h->file->fnode.ino != 1) { + ceph_assert(ceph_mutex_is_locked(h->lock)); + ceph_assert(ceph_mutex_is_locked(h->file->lock)); + } uint64_t x_off = 0; auto p = h->file->fnode.seek(offset, &x_off); ceph_assert(p != h->file->fnode.extents.end()); @@ -3009,7 +3012,7 @@ void BlueFS::_claim_completed_aios(FileWriter *h, list *ls) dout(10) << __func__ << " got " << ls->size() << " aios" << dendl; } -void BlueFS::wait_for_aio(FileWriter *h) +void BlueFS::_wait_for_aio(FileWriter *h) { // NOTE: this is safe to call without a lock, as long as our reference is // stable. @@ -3027,7 +3030,7 @@ void BlueFS::wait_for_aio(FileWriter *h) #endif -void BlueFS::append_try_flush(FileWriter *h, const char* buf, size_t len) +void BlueFS::append_try_flush/*_WFL_WFN*/(FileWriter *h, const char* buf, size_t len) { std::unique_lock hl(h->lock); size_t max_size = 1ull << 30; // cap to 1GB @@ -3043,10 +3046,10 @@ void BlueFS::append_try_flush(FileWriter *h, const char* buf, size_t len) } if (need_flush) { bool flushed = false; - int r = _flush(h, true, &flushed); + int r = _flush_F(h, true, &flushed); ceph_assert(r == 0); if (r == 0 && flushed) { - maybe_compact_log(); + _maybe_compact_log_LN_N_LD_D(); } // make sure we've made any progress with flush hence the // loop doesn't iterate forever @@ -3059,14 +3062,14 @@ void BlueFS::flush(FileWriter *h, bool force) { std::unique_lock hl(h->lock); bool flushed = false; - int r = _flush(h, force, &flushed); + int r = _flush_F(h, force, &flushed); ceph_assert(r == 0); if (r == 0 && flushed) { - maybe_compact_log(); + _maybe_compact_log_LN_NF_LD_D(); } } -int BlueFS::_flush(FileWriter *h, bool force, bool *flushed) +int BlueFS::_flush_F(FileWriter *h, bool force, bool *flushed) { ceph_assert(ceph_mutex_is_locked(h->lock)); uint64_t length = h->get_buffer_length(); @@ -3090,7 +3093,7 @@ int BlueFS::_flush(FileWriter *h, bool force, bool *flushed) << std::hex << offset << "~" << length << std::dec << " to " << h->file->fnode << dendl; ceph_assert(h->pos <= h->file->fnode.size); - int r = _flush_range(h, offset, length); + int r = _flush_range_F(h, offset, length); if (flushed) { *flushed = true; } @@ -3105,8 +3108,7 @@ int BlueFS::_flush(FileWriter *h, bool force, bool *flushed) // smart enough to discover it on its own. int BlueFS::_flush_special(FileWriter *h) { - //ceph_assert(ceph_mutex_is_locked(h->lock)); - //ceph_assert(ceph_mutex_is_locked(h->file->lock)); + ceph_assert(h->file->fnode.ino <= 1); uint64_t length = h->get_buffer_length(); uint64_t offset = h->pos; ceph_assert(length + offset <= h->file->fnode.get_allocated()); @@ -3137,7 +3139,7 @@ int BlueFS::truncate(FileWriter *h, uint64_t offset) ceph_abort_msg("actually this shouldn't happen"); } if (h->get_buffer_length()) { - int r = _flush(h, true); + int r = _flush_F(h, true); if (r < 0) return r; } @@ -3159,28 +3161,36 @@ int BlueFS::truncate(FileWriter *h, uint64_t offset) int BlueFS::fsync(FileWriter *h) { std::unique_lock hl(h->lock); - dout(10) << __func__ << " " << h << " " << h->file->fnode << dendl; - int r = _flush(h, true); - if (r < 0) - return r; - if (h->file->is_dirty) { - _signal_dirty_to_log(h); - h->file->is_dirty = false; + uint64_t old_dirty_seq = 0; + { + dout(10) << __func__ << " " << h << " " << h->file->fnode << dendl; + int r = _flush_F(h, true); + if (r < 0) + return r; + _flush_bdev(h); + if (h->file->is_dirty) { + _signal_dirty_to_log_D(h); + h->file->is_dirty = false; + } + { + std::lock_guard dl(dirty.lock); + if (dirty.seq_stable < h->file->dirty_seq) { + old_dirty_seq = h->file->dirty_seq; + dout(20) << __func__ << " file metadata was dirty (" << old_dirty_seq + << ") on " << h->file->fnode << ", flushing log" << dendl; + } + } } - uint64_t old_dirty_seq = h->file->dirty_seq; - - _flush_bdev(h); - if (old_dirty_seq) { uint64_t s = log.seq_live; // AKAK !!! locks! dout(20) << __func__ << " file metadata was dirty (" << old_dirty_seq << ") on " << h->file->fnode << ", flushing log" << dendl; - flush_and_sync_log(old_dirty_seq); + _flush_and_sync_log_LD(old_dirty_seq); // AK - TODO - think - how can dirty_seq change if we are under h lock? ceph_assert(h->file->dirty_seq == 0 || // cleaned h->file->dirty_seq >= s); // or redirtied by someone else } - maybe_compact_log(); + _maybe_compact_log_LN_NF_LD_D(); return 0; } @@ -3198,14 +3208,14 @@ void BlueFS::_flush_bdev(FileWriter *h) if (!cct->_conf->bluefs_sync_write) { list completed_ios; _claim_completed_aios(h, &completed_ios); - wait_for_aio(h); + _wait_for_aio(h); completed_ios.clear(); } #endif - flush_bdev(flush_devs); + _flush_bdev(flush_devs); } -void BlueFS::flush_bdev(std::array& dirty_bdevs) +void BlueFS::_flush_bdev(std::array& dirty_bdevs) { // NOTE: this is safe to call without a lock. dout(20) << __func__ << dendl; @@ -3215,7 +3225,7 @@ void BlueFS::flush_bdev(std::array& dirty_bdevs) } } -void BlueFS::flush_bdev() +void BlueFS::_flush_bdev() { // NOTE: this is safe to call without a lock. dout(20) << __func__ << dendl; @@ -3378,24 +3388,24 @@ void BlueFS::sync_metadata(bool avoid_compact) lgeneric_subdout(cct, bluefs, 10) << __func__; start = ceph_clock_now(); *_dout << dendl; - flush_bdev(); // FIXME? - flush_and_sync_log(); + _flush_bdev(); // FIXME? + _flush_and_sync_log_LD(); dout(10) << __func__ << " done in " << (ceph_clock_now() - start) << dendl; } if (!avoid_compact) { - maybe_compact_log(); + _maybe_compact_log_LN_NF_LD_D(); } } -void BlueFS::maybe_compact_log() +void BlueFS::_maybe_compact_log_LN_NF_LD_D() { if (!cct->_conf->bluefs_replay_recovery_disable_compact && - should_start_compact_log()) { + _should_start_compact_log_L_N()) { if (cct->_conf->bluefs_compact_log_sync) { - compact_log_sync(); + _compact_log_sync_LN_LD(); } else { - compact_log_async(); + _compact_log_async_LD_NF_D(); } } } @@ -3406,7 +3416,7 @@ int BlueFS::open_for_write( FileWriter **h, bool overwrite) { - std::lock_guard dirl(nodes.lock); + std::lock_guard nl(nodes.lock); dout(10) << __func__ << " " << dirname << "/" << filename << dendl; map::iterator p = nodes.dir_map.find(dirname); DirRef dir; @@ -3553,7 +3563,7 @@ int BlueFS::open_for_read( FileReader **h, bool random) { - std::lock_guard dirl(nodes.lock); + std::lock_guard nl(nodes.lock); dout(10) << __func__ << " " << dirname << "/" << filename << (random ? " (random)":" (sequential)") << dendl; map::iterator p = nodes.dir_map.find(dirname); @@ -3582,8 +3592,8 @@ int BlueFS::rename( std::string_view old_dirname, std::string_view old_filename, std::string_view new_dirname, std::string_view new_filename) { - std::lock_guard dirl(nodes.lock); std::lock_guard ll(log.lock); + std::lock_guard nl(nodes.lock); dout(10) << __func__ << " " << old_dirname << "/" << old_filename << " -> " << new_dirname << "/" << new_filename << dendl; map::iterator p = nodes.dir_map.find(old_dirname); @@ -3614,7 +3624,7 @@ int BlueFS::rename( << " already exists, unlinking" << dendl; ceph_assert(q->second != file); log.t.op_dir_unlink(new_dirname, new_filename); - _drop_link(q->second); + _drop_link_D(q->second); } dout(10) << __func__ << " " << new_dirname << "/" << new_filename << " " @@ -3630,8 +3640,8 @@ int BlueFS::rename( int BlueFS::mkdir(std::string_view dirname) { - std::lock_guard dirl(nodes.lock); std::lock_guard ll(log.lock); + std::lock_guard nl(nodes.lock); dout(10) << __func__ << " " << dirname << dendl; map::iterator p = nodes.dir_map.find(dirname); if (p != nodes.dir_map.end()) { @@ -3645,8 +3655,8 @@ int BlueFS::mkdir(std::string_view dirname) int BlueFS::rmdir(std::string_view dirname) { - std::lock_guard dirl(nodes.lock); std::lock_guard ll(log.lock); + std::lock_guard nl(nodes.lock); dout(10) << __func__ << " " << dirname << dendl; auto p = nodes.dir_map.find(dirname); if (p == nodes.dir_map.end()) { @@ -3665,7 +3675,7 @@ int BlueFS::rmdir(std::string_view dirname) bool BlueFS::dir_exists(std::string_view dirname) { - std::lock_guard dirl(nodes.lock); + std::lock_guard nl(nodes.lock); map::iterator p = nodes.dir_map.find(dirname); bool exists = p != nodes.dir_map.end(); dout(10) << __func__ << " " << dirname << " = " << (int)exists << dendl; @@ -3675,7 +3685,7 @@ bool BlueFS::dir_exists(std::string_view dirname) int BlueFS::stat(std::string_view dirname, std::string_view filename, uint64_t *size, utime_t *mtime) { - std::lock_guard dirl(nodes.lock); + std::lock_guard nl(nodes.lock); dout(10) << __func__ << " " << dirname << "/" << filename << dendl; map::iterator p = nodes.dir_map.find(dirname); if (p == nodes.dir_map.end()) { @@ -3703,7 +3713,8 @@ int BlueFS::stat(std::string_view dirname, std::string_view filename, int BlueFS::lock_file(std::string_view dirname, std::string_view filename, FileLock **plock) { - std::lock_guard dirl(nodes.lock); + std::lock_guard ll(log.lock); + std::lock_guard nl(nodes.lock); dout(10) << __func__ << " " << dirname << "/" << filename << dendl; map::iterator p = nodes.dir_map.find(dirname); if (p == nodes.dir_map.end()) { @@ -3723,7 +3734,6 @@ int BlueFS::lock_file(std::string_view dirname, std::string_view filename, nodes.file_map[ino_last] = file; dir->file_map[string{filename}] = file; ++file->refs; - std::lock_guard ll(log.lock); log.t.op_file_update(file->fnode); log.t.op_dir_link(dirname, filename, file->fnode.ino); } else { @@ -3742,7 +3752,7 @@ int BlueFS::lock_file(std::string_view dirname, std::string_view filename, int BlueFS::unlock_file(FileLock *fl) { - std::lock_guard dirl(nodes.lock); + std::lock_guard nl(nodes.lock); dout(10) << __func__ << " " << fl << " on " << fl->file->fnode << dendl; ceph_assert(fl->file->locked); fl->file->locked = false; @@ -3756,7 +3766,7 @@ int BlueFS::readdir(std::string_view dirname, vector *ls) if (!dirname.empty() && dirname.back() == '/') { dirname.remove_suffix(1); } - std::lock_guard dirl(nodes.lock); + std::lock_guard nl(nodes.lock); dout(10) << __func__ << " " << dirname << dendl; if (dirname.empty()) { // list dirs @@ -3784,8 +3794,8 @@ int BlueFS::readdir(std::string_view dirname, vector *ls) int BlueFS::unlink(std::string_view dirname, std::string_view filename) { - std::lock_guard dirl(nodes.lock); std::lock_guard ll(log.lock); + std::lock_guard nl(nodes.lock); dout(10) << __func__ << " " << dirname << "/" << filename << dendl; map::iterator p = nodes.dir_map.find(dirname); if (p == nodes.dir_map.end()) { @@ -3807,7 +3817,7 @@ int BlueFS::unlink(std::string_view dirname, std::string_view filename) } dir->file_map.erase(string{filename}); log.t.op_dir_unlink(dirname, filename); - _drop_link(file); + _drop_link_D(file); return 0; } @@ -3830,7 +3840,7 @@ bool BlueFS::wal_is_rotational() When we find it, we decode following bytes as extent. We read that whole extent and then check if merged with existing log part gives a proper bluefs transaction. */ -int BlueFS::do_replay_recovery_read(FileReader *log_reader, +int BlueFS::_do_replay_recovery_read(FileReader *log_reader, size_t replay_pos, size_t read_offset, size_t read_len, diff --git a/src/os/bluestore/BlueFS.h b/src/os/bluestore/BlueFS.h index a6e2cab95bdc4..3730e8469bc3d 100644 --- a/src/os/bluestore/BlueFS.h +++ b/src/os/bluestore/BlueFS.h @@ -400,7 +400,7 @@ class BlueFS { FileRef _get_file(uint64_t ino); - void _drop_link(FileRef f); + void _drop_link_D(FileRef f); unsigned _get_slow_device_id() { return bdev[BDEV_SLOW] ? BDEV_SLOW : BDEV_DB; @@ -412,32 +412,32 @@ class BlueFS { PExtentVector* extents); /* signal replay log to include h->file in nearest log flush */ - int _signal_dirty_to_log(FileWriter *h); - int _flush_range(FileWriter *h, uint64_t offset, uint64_t length); + int _signal_dirty_to_log_D(FileWriter *h); + int _flush_range_F(FileWriter *h, uint64_t offset, uint64_t length); int _flush_data(FileWriter *h, uint64_t offset, uint64_t length, bool buffered); - int _flush(FileWriter *h, bool force, bool *flushed = nullptr); + int _flush_F(FileWriter *h, bool force, bool *flushed = nullptr); int _flush_special(FileWriter *h); int _fsync(FileWriter *h); #ifdef HAVE_LIBAIO void _claim_completed_aios(FileWriter *h, std::list *ls); - void wait_for_aio(FileWriter *h); // safe to call without a lock + void _wait_for_aio(FileWriter *h); // safe to call without a lock #endif int64_t _maybe_extend_log(); void _extend_log(); uint64_t _log_advance_seq(); void _consume_dirty(uint64_t seq); - void clear_dirty_set_stable(uint64_t seq_stable); - void release_pending_allocations(std::vector>& to_release); + void _clear_dirty_set_stable_D(uint64_t seq_stable); + void _release_pending_allocations(std::vector>& to_release); void _flush_and_sync_log_core(int64_t available_runway); - int _flush_and_sync_log_jump(uint64_t jump_to, + int _flush_and_sync_log_jump_D(uint64_t jump_to, int64_t available_runway); - int flush_and_sync_log(uint64_t want_seq = 0); + int _flush_and_sync_log_LD(uint64_t want_seq = 0); - uint64_t estimate_log_size(); - bool should_start_compact_log(); + uint64_t _estimate_log_size_N(); + bool _should_start_compact_log_L_N(); enum { REMOVE_DB = 1, @@ -445,12 +445,12 @@ class BlueFS { RENAME_SLOW2DB = 4, RENAME_DB2SLOW = 8, }; - void compact_log_dump_metadata(bluefs_transaction_t *t, + void _compact_log_dump_metadata_N(bluefs_transaction_t *t, int flags); - void compact_log_sync(); - void compact_log_async(); + void _compact_log_sync_LN_LD(); + void _compact_log_async_LD_NF_D(); - void rewrite_log_and_layout_sync(bool allocate_with_fallback, + void _rewrite_log_and_layout_sync_LN_LD(bool allocate_with_fallback, int super_dev, int log_dev, int new_log_dev, @@ -460,8 +460,8 @@ class BlueFS { //void _aio_finish(void *priv); void _flush_bdev(FileWriter *h); - void flush_bdev(); // this is safe to call without a lock - void flush_bdev(std::array& dirty_bdevs); // this is safe to call without a lock + void _flush_bdev(); // this is safe to call without a lock + void _flush_bdev(std::array& dirty_bdevs); // this is safe to call without a lock int _preallocate(FileRef f, uint64_t off, uint64_t len); int _truncate(FileWriter *h, uint64_t off); @@ -581,7 +581,7 @@ class BlueFS { /// sync any uncommitted state to disk void sync_metadata(bool avoid_compact); /// test and compact log, if necessary - void maybe_compact_log(); + void _maybe_compact_log_LN_NF_LD_D(); void set_volume_selector(BlueFSVolumeSelector* s) { vselector.reset(s); @@ -625,11 +625,11 @@ class BlueFS { void invalidate_cache(FileRef f, uint64_t offset, uint64_t len); int preallocate(FileRef f, uint64_t offset, uint64_t len); int truncate(FileWriter *h, uint64_t offset); - int do_replay_recovery_read(FileReader *log, - size_t log_pos, - size_t read_offset, - size_t read_len, - bufferlist* bl); + int _do_replay_recovery_read(FileReader *log, + size_t log_pos, + size_t read_offset, + size_t read_len, + bufferlist* bl); size_t probe_alloc_avail(int dev, uint64_t alloc_size); @@ -695,5 +695,22 @@ class FitToFastVolumeSelector : public OriginalVolumeSelector { void get_paths(const std::string& base, paths& res) const override; }; - +/** + * Directional graph of locks. + * Vertices - Locks. Edges (directed) - locking progression. + * Edge A->B exist if last taken lock was A and next taken lock is B. + * + * Column represents last lock taken. + * Row represents next lock taken. + * + * < | L | D | N | F | W + * -------------|---|---|---|---|--- + * log L | < < + * dirty D | + * nodes N | < + * File F | < + * FileWriter W | < < < + * + * Claim: Deadlock is possible IFF graph contains cycles. + */ #endif From d482ed16b5a2ddfc22b06a33348fab66be5ed2ca Mon Sep 17 00:00:00 2001 From: Adam Kupczyk Date: Mon, 28 Jun 2021 14:57:31 +0200 Subject: [PATCH 0045/2866] os/bluestore/bluefs: Modify dirty_seq progression logic Modified File.dirty_seq to capture dirty.seq_stable instead of 0. This is used to distunguish files already serialized for compact_log_async_dump_metadata() function. Signed-off-by: Adam Kupczyk --- src/os/bluestore/BlueFS.cc | 52 ++++++++++++++++++++++++++++---------- src/os/bluestore/BlueFS.h | 3 +++ 2 files changed, 42 insertions(+), 13 deletions(-) diff --git a/src/os/bluestore/BlueFS.cc b/src/os/bluestore/BlueFS.cc index f8145ce889000..895c82619d86e 100644 --- a/src/os/bluestore/BlueFS.cc +++ b/src/os/bluestore/BlueFS.cc @@ -1831,13 +1831,12 @@ void BlueFS::_drop_link_D(FileRef file) nodes.file_map.erase(file->fnode.ino); file->deleted = true; - if (file->dirty_seq) { + if (file->dirty_seq > dirty.seq_stable) { // retract request to serialize changes - ceph_assert(file->dirty_seq > dirty.seq_stable); ceph_assert(dirty.files.count(file->dirty_seq)); auto it = dirty.files[file->dirty_seq].iterator_to(*file); dirty.files[file->dirty_seq].erase(it); - file->dirty_seq = 0; + file->dirty_seq = dirty.seq_stable; } } } @@ -2175,6 +2174,40 @@ void BlueFS::_compact_log_dump_metadata_N(bluefs_transaction_t *t, } } } +/* Streams to t files modified before *capture_before_seq* and all dirs */ +void BlueFS::compact_log_async_dump_metadata_NF(bluefs_transaction_t *t, + uint64_t capture_before_seq) +{ + std::lock_guard nl(nodes.lock); + + t->seq = 1; + t->uuid = super.uuid; + dout(20) << __func__ << " op_init" << dendl; + + t->op_init(); + for (auto& [ino, file_ref] : nodes.file_map) { + if (ino == 1) + continue; + ceph_assert(ino > 1); + std::lock_guard fl(file_ref->lock); + if (file_ref->dirty_seq < capture_before_seq) { + dout(20) << __func__ << " op_file_update " << file_ref->fnode << dendl; + t->op_file_update(file_ref->fnode); + } else { + dout(20) << __func__ << " skipping just modified, dirty_seq=" + << file_ref->dirty_seq << " " << file_ref->fnode << dendl; + } + } + for (auto& [path, dir_ref] : nodes.dir_map) { + dout(20) << __func__ << " op_dir_create " << path << dendl; + t->op_dir_create(path); + for (auto& [fname, file_ref] : dir_ref->file_map) { + dout(20) << __func__ << " op_dir_link " << path << "/" << fname + << " to " << file_ref->fnode.ino << dendl; + t->op_dir_link(path, fname, file_ref->fnode.ino); + } + } +} void BlueFS::_compact_log_sync_LN_LD() { @@ -2378,7 +2411,7 @@ void BlueFS::_compact_log_async_LD_NF_D() //also locks FW for new_writer //this needs files lock //what will happen, if a file is modified *twice* before we stream it to log? //the later state that we capture will be seen earlier and replay will see a temporary retraction (!) - compact_log_dump_metadata_N(&t, 0); + compact_log_async_dump_metadata_NF(&t, seq_now); uint64_t max_alloc_size = std::max(alloc_size[BDEV_WAL], std::max(alloc_size[BDEV_DB], @@ -2663,10 +2696,9 @@ void BlueFS::_clear_dirty_set_stable_D(uint64_t seq) auto l = p->second.begin(); while (l != p->second.end()) { File *file = &*l; - ceph_assert(file->dirty_seq > 0); ceph_assert(file->dirty_seq <= dirty.seq_stable); dout(20) << __func__ << " cleaned file " << file->fnode << dendl; - file->dirty_seq = 0; + file->dirty_seq = dirty.seq_stable; p->second.erase(l++); } @@ -2832,7 +2864,7 @@ int BlueFS::_signal_dirty_to_log_D(FileWriter *h) std::lock_guard dl(dirty.lock); h->file->fnode.mtime = ceph_clock_now(); ceph_assert(h->file->fnode.ino >= 1); - if (h->file->dirty_seq == 0) { + if (h->file->dirty_seq <= dirty.seq_stable) { h->file->dirty_seq = dirty.seq_live; dirty.files[h->file->dirty_seq].push_back(*h->file); dout(20) << __func__ << " dirty_seq = " << dirty.seq_live @@ -3182,13 +3214,7 @@ int BlueFS::fsync(FileWriter *h) } } if (old_dirty_seq) { - uint64_t s = log.seq_live; // AKAK !!! locks! - dout(20) << __func__ << " file metadata was dirty (" << old_dirty_seq - << ") on " << h->file->fnode << ", flushing log" << dendl; _flush_and_sync_log_LD(old_dirty_seq); - // AK - TODO - think - how can dirty_seq change if we are under h lock? - ceph_assert(h->file->dirty_seq == 0 || // cleaned - h->file->dirty_seq >= s); // or redirtied by someone else } _maybe_compact_log_LN_NF_LD_D(); return 0; diff --git a/src/os/bluestore/BlueFS.h b/src/os/bluestore/BlueFS.h index 3730e8469bc3d..444d2cadd90da 100644 --- a/src/os/bluestore/BlueFS.h +++ b/src/os/bluestore/BlueFS.h @@ -447,6 +447,9 @@ class BlueFS { }; void _compact_log_dump_metadata_N(bluefs_transaction_t *t, int flags); + void compact_log_async_dump_metadata_NF(bluefs_transaction_t *t, + uint64_t capture_before_seq); + void _compact_log_sync_LN_LD(); void _compact_log_async_LD_NF_D(); From a5a0a74a254eb5de16f87c9a199d3437304aa58a Mon Sep 17 00:00:00 2001 From: Adam Kupczyk Date: Tue, 29 Jun 2021 13:24:04 +0200 Subject: [PATCH 0046/2866] os/bluestore/bluefs: Weaken locks in append_try_flush Extracted _maybe_compact_log outside of file lock. The sequence FL could deadlock with LNF that is executed in _async_dump_metadata. Signed-off-by: Adam Kupczyk --- src/os/bluestore/BlueFS.cc | 74 +++++++++++++++++--------------------- 1 file changed, 33 insertions(+), 41 deletions(-) diff --git a/src/os/bluestore/BlueFS.cc b/src/os/bluestore/BlueFS.cc index 895c82619d86e..3a3d3d74299e7 100644 --- a/src/os/bluestore/BlueFS.cc +++ b/src/os/bluestore/BlueFS.cc @@ -2230,7 +2230,6 @@ void BlueFS::_rewrite_log_and_layout_sync_LN_LD(bool allocate_with_fallback, int flags, std::optional layout) { - //ceph_assert(ceph_mutex_is_notlocked(log.lock)); std::lock_guard ll(log.lock); File *log_file = log.writer->file.get(); @@ -2408,11 +2407,7 @@ void BlueFS::_compact_log_async_LD_NF_D() //also locks FW for new_writer // 2. prepare compacted log bluefs_transaction_t t; - //this needs files lock - //what will happen, if a file is modified *twice* before we stream it to log? - //the later state that we capture will be seen earlier and replay will see a temporary retraction (!) compact_log_async_dump_metadata_NF(&t, seq_now); - uint64_t max_alloc_size = std::max(alloc_size[BDEV_WAL], std::max(alloc_size[BDEV_DB], alloc_size[BDEV_SLOW])); @@ -2521,15 +2516,6 @@ void BlueFS::_compact_log_async_LD_NF_D() //also locks FW for new_writer // delete the new log, remove from the dirty files list _close_writer(new_log_writer); - // flush_special does not dirty files - /* - if (new_log->dirty_seq) { - std::lock_guard dl(dirty.lock); - ceph_assert(dirty.files.count(new_log->dirty_seq)); - auto it = dirty.files[new_log->dirty_seq].iterator_to(*new_log); - dirty.files[new_log->dirty_seq].erase(it); - } - */ new_log_writer = nullptr; new_log = nullptr; log_cond.notify_all(); @@ -2789,7 +2775,7 @@ int BlueFS::_flush_and_sync_log_jump_D(uint64_t jump_to, ceph_assert(ceph_mutex_is_locked(log.lock)); ceph_assert(jump_to); - // we synchronize writing to log, by lock to log_lock + // we synchronize writing to log, by lock to log.lock dirty.lock.lock(); uint64_t seq =_log_advance_seq(); @@ -3061,41 +3047,47 @@ void BlueFS::_wait_for_aio(FileWriter *h) } #endif - -void BlueFS::append_try_flush/*_WFL_WFN*/(FileWriter *h, const char* buf, size_t len) +void BlueFS::append_try_flush(FileWriter *h, const char* buf, size_t len) { - std::unique_lock hl(h->lock); - size_t max_size = 1ull << 30; // cap to 1GB - while (len > 0) { - bool need_flush = true; - auto l0 = h->get_buffer_length(); - if (l0 < max_size) { - size_t l = std::min(len, max_size - l0); - h->append(buf, l); - buf += l; - len -= l; - need_flush = h->get_buffer_length() >= cct->_conf->bluefs_min_flush_size; - } - if (need_flush) { - bool flushed = false; - int r = _flush_F(h, true, &flushed); - ceph_assert(r == 0); - if (r == 0 && flushed) { - _maybe_compact_log_LN_N_LD_D(); + bool flushed_sum = false; + { + std::unique_lock hl(h->lock); + size_t max_size = 1ull << 30; // cap to 1GB + while (len > 0) { + bool need_flush = true; + auto l0 = h->get_buffer_length(); + if (l0 < max_size) { + size_t l = std::min(len, max_size - l0); + h->append(buf, l); + buf += l; + len -= l; + need_flush = h->get_buffer_length() >= cct->_conf->bluefs_min_flush_size; + } + if (need_flush) { + bool flushed = false; + int r = _flush_F(h, true, &flushed); + ceph_assert(r == 0); + flushed_sum |= flushed; + // make sure we've made any progress with flush hence the + // loop doesn't iterate forever + ceph_assert(h->get_buffer_length() < max_size); } - // make sure we've made any progress with flush hence the - // loop doesn't iterate forever - ceph_assert(h->get_buffer_length() < max_size); } } + if (flushed_sum) { + _maybe_compact_log_LN_NF_LD_D(); + } } void BlueFS::flush(FileWriter *h, bool force) { - std::unique_lock hl(h->lock); bool flushed = false; - int r = _flush_F(h, force, &flushed); - ceph_assert(r == 0); + int r; + { + std::unique_lock hl(h->lock); + r = _flush_F(h, force, &flushed); + ceph_assert(r == 0); + } if (r == 0 && flushed) { _maybe_compact_log_LN_NF_LD_D(); } From aa9dbdd8e8a31fc1c233b30fde39a995cdc1663c Mon Sep 17 00:00:00 2001 From: Adam Kupczyk Date: Thu, 1 Jul 2021 11:33:20 +0200 Subject: [PATCH 0047/2866] os/bluestore/bluefs: Weaken locks in open_for_write Signed-off-by: Adam Kupczyk --- src/os/bluestore/BlueFS.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/os/bluestore/BlueFS.cc b/src/os/bluestore/BlueFS.cc index 3a3d3d74299e7..ed255cb1b266c 100644 --- a/src/os/bluestore/BlueFS.cc +++ b/src/os/bluestore/BlueFS.cc @@ -3434,7 +3434,7 @@ int BlueFS::open_for_write( FileWriter **h, bool overwrite) { - std::lock_guard nl(nodes.lock); + std::unique_lock nl(nodes.lock); dout(10) << __func__ << " " << dirname << "/" << filename << dendl; map::iterator p = nodes.dir_map.find(dirname); DirRef dir; @@ -3496,6 +3496,7 @@ int BlueFS::open_for_write( dout(20) << __func__ << " mapping " << dirname << "/" << filename << " vsel_hint " << file->vselector_hint << dendl; + nl.unlock(); { std::lock_guard ll(log.lock); log.t.op_file_update(file->fnode); From 13b637c32e29ff99b7a0a4596c71fd1f722c26aa Mon Sep 17 00:00:00 2001 From: Adam Kupczyk Date: Tue, 10 Aug 2021 15:15:52 +0200 Subject: [PATCH 0048/2866] os/bluestore/bluefs: Rearrange locks in prealocate Rearranged locks in preallocate to avoid possible deadlock with compact_log_async_dump_metadata_NF. Cycle was: L->N rename/mkdir N->F compact_log_async_dump_metadata_NF F->L preallocate Signed-off-by: Adam Kupczyk --- src/os/bluestore/BlueFS.cc | 2 +- src/os/bluestore/BlueFS.h | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/os/bluestore/BlueFS.cc b/src/os/bluestore/BlueFS.cc index ed255cb1b266c..7add76833705f 100644 --- a/src/os/bluestore/BlueFS.cc +++ b/src/os/bluestore/BlueFS.cc @@ -3366,6 +3366,7 @@ int BlueFS::_allocate(uint8_t id, uint64_t len, int BlueFS::preallocate(FileRef f, uint64_t off, uint64_t len) { + std::lock_guard ll(log.lock); std::lock_guard fl(f->lock); dout(10) << __func__ << " file " << f->fnode << " 0x" << std::hex << off << "~" << len << std::dec << dendl; @@ -3385,7 +3386,6 @@ int BlueFS::preallocate(FileRef f, uint64_t off, uint64_t len) vselector->add_usage(f->vselector_hint, f->fnode); if (r < 0) return r; - std::lock_guard ll(log.lock); log.t.op_file_update(f->fnode); } return 0; diff --git a/src/os/bluestore/BlueFS.h b/src/os/bluestore/BlueFS.h index 444d2cadd90da..a802443b60c63 100644 --- a/src/os/bluestore/BlueFS.h +++ b/src/os/bluestore/BlueFS.h @@ -711,8 +711,8 @@ class FitToFastVolumeSelector : public OriginalVolumeSelector { * log L | < < * dirty D | * nodes N | < - * File F | < - * FileWriter W | < < < + * File F | + * FileWriter W | < < < * * Claim: Deadlock is possible IFF graph contains cycles. */ From 95b662fa176cd841e26b76341f91bf2e6c266003 Mon Sep 17 00:00:00 2001 From: "J. Eric Ivancich" Date: Wed, 8 Sep 2021 14:58:02 -0400 Subject: [PATCH 0049/2866] rgw: fix rgw.none statistics Only unaccount for objects being removed if they exist. Also, clean up some formatting. Signed-off-by: J. Eric Ivancich --- src/cls/rgw/cls_rgw.cc | 56 +++++++++++++++++++++++++++--------------- 1 file changed, 36 insertions(+), 20 deletions(-) diff --git a/src/cls/rgw/cls_rgw.cc b/src/cls/rgw/cls_rgw.cc index 222803e341423..4b9a17cf29dfe 100644 --- a/src/cls/rgw/cls_rgw.cc +++ b/src/cls/rgw/cls_rgw.cc @@ -903,11 +903,14 @@ int rgw_bucket_prepare_op(cls_method_context_t hctx, bufferlist *in, bufferlist static void unaccount_entry(rgw_bucket_dir_header& header, rgw_bucket_dir_entry& entry) { - rgw_bucket_category_stats& stats = header.stats[entry.meta.category]; - stats.num_entries--; - stats.total_size -= entry.meta.accounted_size; - stats.total_size_rounded -= cls_rgw_get_rounded_size(entry.meta.accounted_size); - stats.actual_size -= entry.meta.size; + if (entry.exists) { + rgw_bucket_category_stats& stats = header.stats[entry.meta.category]; + stats.num_entries--; + stats.total_size -= entry.meta.accounted_size; + stats.total_size_rounded -= + cls_rgw_get_rounded_size(entry.meta.accounted_size); + stats.actual_size -= entry.meta.size; + } } static void log_entry(const char *func, const char *str, rgw_bucket_dir_entry *entry) @@ -993,6 +996,7 @@ static int read_key_entry(cls_method_context_t hctx, cls_rgw_obj_key& key, int rgw_bucket_complete_op(cls_method_context_t hctx, bufferlist *in, bufferlist *out) { CLS_LOG(10, "entered %s", __func__); + // decode request rgw_cls_obj_complete_op op; auto iter = in->cbegin(); @@ -1002,6 +1006,7 @@ int rgw_bucket_complete_op(cls_method_context_t hctx, bufferlist *in, bufferlist CLS_LOG(1, "ERROR: rgw_bucket_complete_op(): failed to decode request\n"); return -EINVAL; } + CLS_LOG(1, "rgw_bucket_complete_op(): request: op=%d name=%s instance=%s ver=%lu:%llu tag=%s", op.op, op.key.name.c_str(), op.key.instance.c_str(), (unsigned long)op.ver.pool, (unsigned long long)op.ver.epoch, @@ -1017,7 +1022,7 @@ int rgw_bucket_complete_op(cls_method_context_t hctx, bufferlist *in, bufferlist rgw_bucket_dir_entry entry; bool ondisk = true; - string idx; + std::string idx; rc = read_key_entry(hctx, op.key, &idx, &entry); if (rc == -ENOENT) { entry.key = op.key; @@ -1065,9 +1070,7 @@ int rgw_bucket_complete_op(cls_method_context_t hctx, bufferlist *in, bufferlist return 0; } - if (entry.exists) { - unaccount_entry(header, entry); - } + unaccount_entry(header, entry); entry.ver = op.ver; switch ((int)op.op) { @@ -1112,19 +1115,26 @@ int rgw_bucket_complete_op(cls_method_context_t hctx, bufferlist *in, bufferlist } if (op.log_op && !header.syncstopped) { - rc = log_index_operation(hctx, op.key, op.op, op.tag, entry.meta.mtime, entry.ver, - CLS_RGW_STATE_COMPLETE, header.ver, header.max_marker, op.bilog_flags, NULL, NULL, &op.zones_trace); - if (rc < 0) + rc = log_index_operation(hctx, op.key, op.op, op.tag, entry.meta.mtime, + entry.ver, CLS_RGW_STATE_COMPLETE, header.ver, + header.max_marker, op.bilog_flags, NULL, NULL, + &op.zones_trace); + if (rc < 0) { return rc; + } } - CLS_LOG(20, "rgw_bucket_complete_op(): remove_objs.size()=%d", (int)op.remove_objs.size()); - for (auto remove_iter = op.remove_objs.begin(); remove_iter != op.remove_objs.end(); ++remove_iter) { + CLS_LOG(20, "rgw_bucket_complete_op(): remove_objs.size()=%d", + int(op.remove_objs.size())); + + for (auto remove_iter = op.remove_objs.begin(); + remove_iter != op.remove_objs.end(); + ++remove_iter) { cls_rgw_obj_key& remove_key = *remove_iter; CLS_LOG(1, "rgw_bucket_complete_op(): removing entries, read_index_entry name=%s instance=%s", remove_key.name.c_str(), remove_key.instance.c_str()); rgw_bucket_dir_entry remove_entry; - string k; + std::string k; int ret = read_key_entry(hctx, remove_key, &k, &remove_entry); if (ret < 0) { CLS_LOG(1, "rgw_bucket_complete_op(): removing entries, read_index_entry name=%s instance=%s ret=%d", @@ -1136,25 +1146,31 @@ int rgw_bucket_complete_op(cls_method_context_t hctx, bufferlist *in, bufferlist remove_entry.key.name.c_str(), remove_entry.key.instance.c_str(), int(remove_entry.meta.category)); + unaccount_entry(header, remove_entry); if (op.log_op && !header.syncstopped) { ++header.ver; // increment index version, or we'll overwrite keys previously written - rc = log_index_operation(hctx, remove_key, CLS_RGW_OP_DEL, op.tag, remove_entry.meta.mtime, - remove_entry.ver, CLS_RGW_STATE_COMPLETE, header.ver, header.max_marker, op.bilog_flags, NULL, NULL, &op.zones_trace); - if (rc < 0) + rc = log_index_operation(hctx, remove_key, CLS_RGW_OP_DEL, op.tag, + remove_entry.meta.mtime, remove_entry.ver, + CLS_RGW_STATE_COMPLETE, header.ver, + header.max_marker, op.bilog_flags, NULL, + NULL, &op.zones_trace); + if (rc < 0) { continue; + } } ret = cls_cxx_map_remove_key(hctx, k); if (ret < 0) { - CLS_LOG(1, "rgw_bucket_complete_op(): cls_cxx_map_remove_key, failed to remove entry, name=%s instance=%s read_index_entry ret=%d", remove_key.name.c_str(), remove_key.instance.c_str(), rc); + CLS_LOG(1, "rgw_bucket_complete_op(): cls_cxx_map_remove_key, failed to remove entry, name=%s instance=%s read_index_entry ret=%d", + remove_key.name.c_str(), remove_key.instance.c_str(), rc); continue; } } return write_bucket_header(hctx, &header); -} +} // rgw_bucket_complete_op template static int write_entry(cls_method_context_t hctx, T& entry, const string& key) From 0cb06740a98e8f841bbf905d2df162146b8a8ae7 Mon Sep 17 00:00:00 2001 From: Xiubo Li Date: Mon, 4 Jan 2021 13:26:19 +0800 Subject: [PATCH 0050/2866] qa: enable dynamic debug support to kclient Add a 'kmount_count' counter in ctx to make sure the dynamic debug log won't be disabled until the last kernel mounter is unmounted. Fixes: https://tracker.ceph.com/issues/48736 Signed-off-by: Xiubo Li --- qa/tasks/cephfs/kernel_mount.py | 40 +++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/qa/tasks/cephfs/kernel_mount.py b/qa/tasks/cephfs/kernel_mount.py index 55fdae5d9e84d..bb0562471a755 100644 --- a/qa/tasks/cephfs/kernel_mount.py +++ b/qa/tasks/cephfs/kernel_mount.py @@ -27,6 +27,7 @@ def __init__(self, ctx, test_dir, client_id, client_remote, client_keyring_path=client_keyring_path, hostfs_mntpt=hostfs_mntpt, cephfs_name=cephfs_name, cephfs_mntpt=cephfs_mntpt, brxnet=brxnet) + self.dynamic_debug = config.get('dynamic_debug', False) self.rbytes = config.get('rbytes', False) self.inst = None self.addr = None @@ -50,6 +51,12 @@ def mount(self, mntopts=[], check_status=True, **kwargs): self._set_filemode_on_mntpt() + if self.dynamic_debug: + kmount_count = self.ctx.get(f'kmount_count.{self.client_remote.hostname}', 0) + if kmount_count == 0: + self.enable_dynamic_debug() + self.ctx[f'kmount_count.{self.client_remote.hostname}'] = kmount_count + 1 + self.mounted = True def _run_mount_cmd(self, mntopts, check_status): @@ -113,6 +120,13 @@ def umount(self, force=False): timeout=(15*60), omit_sudo=False) raise e + if self.dynamic_debug: + kmount_count = self.ctx.get(f'kmount_count.{self.client_remote.hostname}') + assert kmount_count + if kmount_count == 1: + self.disable_dynamic_debug() + self.ctx[f'kmount_count.{self.client_remote.hostname}'] = kmount_count - 1 + self.mounted = False self.cleanup() @@ -221,6 +235,32 @@ def get_id_to_dir(): )) raise + def _dynamic_debug_control(self, enable): + """ + Write to dynamic debug control file. + """ + if enable: + fdata = "module ceph +p" + else: + fdata = "module ceph -p" + + self.run_shell_payload(f""" +sudo modprobe ceph +echo '{fdata}' | sudo tee /sys/kernel/debug/dynamic_debug/control +""") + + def enable_dynamic_debug(self): + """ + Enable the dynamic debug. + """ + self._dynamic_debug_control(True) + + def disable_dynamic_debug(self): + """ + Disable the dynamic debug. + """ + self._dynamic_debug_control(False) + def get_global_id(self): """ Look up the CephFS client ID for this mount, using debugfs. From 8ff162e7f9f88fb312f24a8a6f7ad75eab229250 Mon Sep 17 00:00:00 2001 From: Jiffin Tony Thottan Date: Wed, 15 Sep 2021 15:17:30 +0530 Subject: [PATCH 0051/2866] rgw: remove user-caps from RGWOp_User_Modify The adminOps API for Modify User checks for user-caps value in the request but does handle that in later code path. Caps have seperate apis add_cap and remove_cap already present to modify exisitng caps in user. Hence removing unused cap variables in RGWOp_User_Modify::execute(). Signed-off-by: Jiffin Tony Thottan --- doc/radosgw/adminops.rst | 7 ------- src/rgw/rgw_rest_user.cc | 3 --- 2 files changed, 10 deletions(-) diff --git a/doc/radosgw/adminops.rst b/doc/radosgw/adminops.rst index 36719887a0f3a..2affe8ba181f3 100644 --- a/doc/radosgw/adminops.rst +++ b/doc/radosgw/adminops.rst @@ -603,13 +603,6 @@ Request Parameters :Example: ``s3`` :Required: No -``user-caps`` - -:Description: User capabilities. -:Type: String -:Example: ``usage=read, write; users=read`` -:Required: No - ``max-buckets`` :Description: Specify the maximum number of buckets the user can own. diff --git a/src/rgw/rgw_rest_user.cc b/src/rgw/rgw_rest_user.cc index 37c13003ec536..d6297f5472311 100644 --- a/src/rgw/rgw_rest_user.cc +++ b/src/rgw/rgw_rest_user.cc @@ -257,7 +257,6 @@ void RGWOp_User_Modify::execute(optional_yield y) std::string access_key; std::string secret_key; std::string key_type_str; - std::string caps; std::string op_mask_str; std::string default_placement_str; std::string placement_tags_str; @@ -278,7 +277,6 @@ void RGWOp_User_Modify::execute(optional_yield y) RESTArgs::get_string(s, "email", email, &email, &email_set); RESTArgs::get_string(s, "access-key", access_key, &access_key); RESTArgs::get_string(s, "secret-key", secret_key, &secret_key); - RESTArgs::get_string(s, "user-caps", caps, &caps); RESTArgs::get_bool(s, "generate-key", false, &gen_key); RESTArgs::get_bool(s, "suspended", false, &suspended); RESTArgs::get_int32(s, "max-buckets", RGW_DEFAULT_MAX_BUCKETS, &max_buckets, "a_set); @@ -301,7 +299,6 @@ void RGWOp_User_Modify::execute(optional_yield y) if (email_set) op_state.set_user_email(email); - op_state.set_caps(caps); op_state.set_access_key(access_key); op_state.set_secret_key(secret_key); From 980bf59d45963d7282588671c12ef81c2443cec4 Mon Sep 17 00:00:00 2001 From: Daniel Gryniewicz Date: Thu, 9 Sep 2021 11:58:43 -0400 Subject: [PATCH 0052/2866] RGW Zipper - load user attributes with user Signed-off-by: Daniel Gryniewicz --- src/rgw/rgw_sal_rados.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rgw/rgw_sal_rados.cc b/src/rgw/rgw_sal_rados.cc index ae11e36a6d6ee..00d9b6443dbc5 100644 --- a/src/rgw/rgw_sal_rados.cc +++ b/src/rgw/rgw_sal_rados.cc @@ -205,7 +205,7 @@ int RadosUser::trim_usage(const DoutPrefixProvider *dpp, uint64_t start_epoch, u int RadosUser::load_user(const DoutPrefixProvider* dpp, optional_yield y) { - return store->ctl()->user->get_info_by_uid(dpp, info.user_id, &info, y, RGWUserCtl::GetParams().set_objv_tracker(&objv_tracker)); + return store->ctl()->user->get_info_by_uid(dpp, info.user_id, &info, y, RGWUserCtl::GetParams().set_objv_tracker(&objv_tracker).set_attrs(&attrs)); } int RadosUser::store_user(const DoutPrefixProvider* dpp, optional_yield y, bool exclusive, RGWUserInfo* old_info) From ffd4d1c34abdfc56dd14402ddcdd0463ed343090 Mon Sep 17 00:00:00 2001 From: Mykola Golub Date: Tue, 21 Sep 2021 11:26:21 +0100 Subject: [PATCH 0053/2866] osd: don't crash in backfill due to inconsistent hash info Instead fail the pull so it will try to recover from other shards (or will mark the object missing). Fixes: https://tracker.ceph.com/issues/48959 Signed-off-by: Mykola Golub --- src/osd/ECBackend.cc | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/osd/ECBackend.cc b/src/osd/ECBackend.cc index 5388391c4f0d3..d9d2b5ae1be73 100644 --- a/src/osd/ECBackend.cc +++ b/src/osd/ECBackend.cc @@ -605,7 +605,16 @@ void ECBackend::continue_recovery_op( if (op.recovery_progress.first && op.obc) { /* We've got the attrs and the hinfo, might as well use them */ op.hinfo = get_hash_info(op.hoid); - ceph_assert(op.hinfo); + if (!op.hinfo) { + derr << __func__ << ": " << op.hoid << " has inconsistent hinfo" + << dendl; + ceph_assert(recovery_ops.count(op.hoid)); + eversion_t v = recovery_ops[op.hoid].v; + recovery_ops.erase(op.hoid); + get_parent()->on_failed_pull({get_parent()->whoami_shard()}, + op.hoid, v); + return; + } op.xattrs = op.obc->attr_cache; encode(*(op.hinfo), op.xattrs[ECUtil::get_hinfo_key()]); } From 104d8df5f44cf522422914b254536f42bbe31172 Mon Sep 17 00:00:00 2001 From: Mykola Golub Date: Tue, 21 Sep 2021 11:28:31 +0100 Subject: [PATCH 0054/2866] osd: no need to crash in deep scrub if hash info is invalid Signed-off-by: Mykola Golub --- src/osd/ECBackend.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/osd/ECBackend.cc b/src/osd/ECBackend.cc index d9d2b5ae1be73..15bb059033abf 100644 --- a/src/osd/ECBackend.cc +++ b/src/osd/ECBackend.cc @@ -2588,7 +2588,11 @@ int ECBackend::be_deep_scrub( return 0; } else { if (!get_parent()->get_pool().allows_ecoverwrites()) { - ceph_assert(hinfo->has_chunk_hash()); + if (!hinfo->has_chunk_hash()) { + dout(0) << "_scan_list " << poid << " got invalid hash info" << dendl; + o.ec_size_mismatch = true; + return 0; + } if (hinfo->get_total_chunk_size() != (unsigned)pos.data_pos) { dout(0) << "_scan_list " << poid << " got incorrect size on read 0x" << std::hex << pos From 7d7be31f722d7bf165c69b223b39a600d9fec543 Mon Sep 17 00:00:00 2001 From: wangyunqing Date: Wed, 22 Sep 2021 11:05:40 +0800 Subject: [PATCH 0055/2866] doc/rbd/rbd-mirroring.rst: fix typos Signed-off-by: wangyunqing --- doc/rbd/rbd-mirroring.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/rbd/rbd-mirroring.rst b/doc/rbd/rbd-mirroring.rst index 575d0648d6a4d..464ed2ce92958 100644 --- a/doc/rbd/rbd-mirroring.rst +++ b/doc/rbd/rbd-mirroring.rst @@ -310,7 +310,7 @@ For example:: section of the local or centralized configuration. Note that these settings may allow ``rbd-mirror`` to present a substantial write workload to the destination cluster: monitor cluster performance closely during - migrations and test carefuly before running multiple migrations in parallel. + migrations and test carefully before running multiple migrations in parallel. Create Image Mirror-Snapshots ----------------------------- From 2f2fa761b400debe1c6c210162b91e3c2c3dc3f9 Mon Sep 17 00:00:00 2001 From: Paul Reece Date: Wed, 22 Sep 2021 09:37:23 -0400 Subject: [PATCH 0056/2866] Added comment explaining the change Signed-off-by: Paul Reece --- src/rgw/rgw_multi.cc | 1 + src/rgw/rgw_op.cc | 1 + 2 files changed, 2 insertions(+) diff --git a/src/rgw/rgw_multi.cc b/src/rgw/rgw_multi.cc index 82c5d0ed2cd97..e2f69ac60a7d4 100644 --- a/src/rgw/rgw_multi.cc +++ b/src/rgw/rgw_multi.cc @@ -56,6 +56,7 @@ bool RGWMultiCompleteUpload::xml_end(const char *el) { XMLObj *RGWMultiXMLParser::alloc_obj(const char *el) { XMLObj *obj = NULL; + // CompletedMultipartUpload is incorrect but some versions of some libraries use it, see PR #41700 if (strcmp(el, "CompleteMultipartUpload") == 0 || strcmp(el, "CompletedMultipartUpload") == 0 || strcmp(el, "MultipartUpload") == 0) { diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc index 5e6f13aed0996..2370213e2f394 100644 --- a/src/rgw/rgw_op.cc +++ b/src/rgw/rgw_op.cc @@ -5888,6 +5888,7 @@ void RGWCompleteMultipart::execute(optional_yield y) parts = static_cast(parser.find_first("CompleteMultipartUpload")); if (!parts || parts->parts.empty()) { + // CompletedMultipartUpload is incorrect but some versions of some libraries use it, see PR #41700 parts = static_cast(parser.find_first("CompletedMultipartUpload")); } From 22c09dee82084e974489b8a4cd19b238f2f2593b Mon Sep 17 00:00:00 2001 From: Adam Kupczyk Date: Wed, 22 Sep 2021 16:36:21 +0200 Subject: [PATCH 0057/2866] os/bluestore/bluefs: Removed unused fields Signed-off-by: Adam Kupczyk --- src/os/bluestore/BlueFS.h | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/os/bluestore/BlueFS.h b/src/os/bluestore/BlueFS.h index a802443b60c63..085b131d0dcc2 100644 --- a/src/os/bluestore/BlueFS.h +++ b/src/os/bluestore/BlueFS.h @@ -309,9 +309,6 @@ class BlueFS { }; private: - ceph::mutex log_lock = ceph::make_mutex("BlueFS::log_lock"); - ceph::mutex dirs_lock = ceph::make_mutex("BlueFS::dirs_lock"); - ceph::mutex dirty_lock = ceph::make_mutex("BlueFS::dirty_lock"); PerfCounters *logger = nullptr; uint64_t max_bytes[MAX_BDEV] = {0}; From f9213ad9cf438afc81a1bf0a53c00184dfaf7b1b Mon Sep 17 00:00:00 2001 From: Paul Cuzner Date: Thu, 23 Sep 2021 09:43:57 +1200 Subject: [PATCH 0058/2866] monitoring:Adding the Ceph MIB The ceph MIB has been created and maintained in a a separate repo: https://github.com/SUSE/prometheus-webhook-snmp This patch brings this MIB into the main ceph repo, so alert changes can target prometheus and potentially SNMP environments within the same PR. Kudos to Volker Theile for creating the MIB. Fixes: https://tracker.ceph.com/issues/52708 Signed-off-by: Paul Cuzner --- monitoring/prometheus/README.md | 7 + monitoring/snmp/CEPH-PROMETHEUS-ALERT-MIB.txt | 377 ++++++++++++++++++ monitoring/snmp/README.md | 24 ++ 3 files changed, 408 insertions(+) create mode 100644 monitoring/snmp/CEPH-PROMETHEUS-ALERT-MIB.txt create mode 100644 monitoring/snmp/README.md diff --git a/monitoring/prometheus/README.md b/monitoring/prometheus/README.md index fde63a35fe2ee..205e373b19c0e 100644 --- a/monitoring/prometheus/README.md +++ b/monitoring/prometheus/README.md @@ -5,3 +5,10 @@ In monitoring/prometheus/alerts you'll find a set of Prometheus alert rules that should provide a decent set of default alerts for a Ceph cluster. Just put this file in a place according to your Prometheus configuration (wherever the `rules` configuration stanza points). + +### SNMP +Ceph provides a MIB (CEPH-PROMETHEUS-ALERT-MIB.txt) to support sending prometheus +alerts through to an SNMP management platform. The translation from prometheus +alert to SNMP trap requires the prometheus alert to contain an OID that maps to +a definition within the MIB. When making changes to the prometheus alert rules +file, developers should include any necessary changes to the MIB. diff --git a/monitoring/snmp/CEPH-PROMETHEUS-ALERT-MIB.txt b/monitoring/snmp/CEPH-PROMETHEUS-ALERT-MIB.txt new file mode 100644 index 0000000000000..0eeda4471cb7d --- /dev/null +++ b/monitoring/snmp/CEPH-PROMETHEUS-ALERT-MIB.txt @@ -0,0 +1,377 @@ +CEPH-PROMETHEUS-ALERT-MIB DEFINITIONS ::= BEGIN + +IMPORTS + MODULE-IDENTITY, OBJECT-TYPE, NOTIFICATION-TYPE, enterprises, TimeTicks + FROM SNMPv2-SMI + DisplayString + FROM SNMPv2-TC +; + +ceph OBJECT IDENTIFIER ::= { enterprises 50495 } +prometheus OBJECT IDENTIFIER ::= { ceph 15 } + +prometheusAlert MODULE-IDENTITY + LAST-UPDATED "201904010000Z" -- 1. Apr 2019 + ORGANIZATION "The Ceph Project" + CONTACT-INFO "https://ceph.com" + DESCRIPTION "Prometheus Alert SNMP MIB" + REVISION "201904010000Z" -- 1. Apr 2019 + DESCRIPTION "Initial version." + ::= { prometheus 1 } + +prometheusAlertObjects OBJECT IDENTIFIER ::= { prometheusAlert 1 } +prometheusAlertTraps OBJECT IDENTIFIER ::= { prometheusAlert 2 } + +-- +-- Objects +-- + +prometheusAlertNotificationAlertName OBJECT-TYPE + SYNTAX DisplayString + MAX-ACCESS accessible-for-notify + STATUS current + DESCRIPTION "The name of the Prometheus alert." +::= { prometheusAlertObjects 1 } + +prometheusAlertNotificationStatus OBJECT-TYPE + SYNTAX DisplayString + MAX-ACCESS accessible-for-notify + STATUS current + DESCRIPTION "The status of the Prometheus alert." +::= { prometheusAlertObjects 2 } + +prometheusAlertNotificationSeverity OBJECT-TYPE + SYNTAX DisplayString + MAX-ACCESS accessible-for-notify + STATUS current + DESCRIPTION "The severity of the Prometheus alert." +::= { prometheusAlertObjects 3 } + +prometheusAlertNotificationInstance OBJECT-TYPE + SYNTAX DisplayString + MAX-ACCESS accessible-for-notify + STATUS current + DESCRIPTION "Unique identifier for the Prometheus instance." +::= { prometheusAlertObjects 4 } + +prometheusAlertNotificationJob OBJECT-TYPE + SYNTAX DisplayString + MAX-ACCESS accessible-for-notify + STATUS current + DESCRIPTION "The name of the Prometheus job." +::= { prometheusAlertObjects 5 } + +prometheusAlertNotificationDescription OBJECT-TYPE + SYNTAX DisplayString + MAX-ACCESS accessible-for-notify + STATUS current + DESCRIPTION "The Prometheus alert description field." +::= { prometheusAlertObjects 6 } + +prometheusAlertNotificationLabels OBJECT-TYPE + SYNTAX DisplayString + MAX-ACCESS accessible-for-notify + STATUS current + DESCRIPTION "Additional Prometheus alert labels as JSON string." +::= { prometheusAlertObjects 7 } + +prometheusAlertNotificationTimestamp OBJECT-TYPE + SYNTAX TimeTicks + MAX-ACCESS accessible-for-notify + STATUS current + DESCRIPTION "The time when the Prometheus alert occurred." +::= { prometheusAlertObjects 8 } + +prometheusAlertNotificationRawData OBJECT-TYPE + SYNTAX DisplayString + MAX-ACCESS accessible-for-notify + STATUS current + DESCRIPTION "The raw Prometheus alert as JSON string." +::= { prometheusAlertObjects 9 } + +-- +-- Traps +-- + +prometheusAlertTrapDefault NOTIFICATION-TYPE + OBJECTS { + prometheusAlertNotificationAlertName, + prometheusAlertNotificationStatus, + prometheusAlertNotificationSeverity, + prometheusAlertNotificationInstance, + prometheusAlertNotificationJob, + prometheusAlertNotificationDescription, + prometheusAlertNotificationLabels, + prometheusAlertNotificationTimestamp, + prometheusAlertNotificationRawData + } + STATUS current + DESCRIPTION "Default trap." +::= { prometheusAlertTraps 1 } + +prometheusAlertClusterHealthTraps OBJECT IDENTIFIER ::= { prometheusAlertTraps 2 } +prometheusAlertMonTraps OBJECT IDENTIFIER ::= { prometheusAlertTraps 3 } +prometheusAlertOsdTraps OBJECT IDENTIFIER ::= { prometheusAlertTraps 4 } +prometheusAlertMdsTraps OBJECT IDENTIFIER ::= { prometheusAlertTraps 5 } +prometheusAlertMgrTraps OBJECT IDENTIFIER ::= { prometheusAlertTraps 6 } +prometheusAlertPgsTraps OBJECT IDENTIFIER ::= { prometheusAlertTraps 7 } +prometheusAlertNodesTraps OBJECT IDENTIFIER ::= { prometheusAlertTraps 8 } +prometheusAlertPoolsTraps OBJECT IDENTIFIER ::= { prometheusAlertTraps 9 } + +prometheusAlertClusterHealthTrapHealthError NOTIFICATION-TYPE + OBJECTS { + prometheusAlertNotificationAlertName, + prometheusAlertNotificationStatus, + prometheusAlertNotificationSeverity, + prometheusAlertNotificationInstance, + prometheusAlertNotificationJob, + prometheusAlertNotificationDescription, + prometheusAlertNotificationLabels, + prometheusAlertNotificationTimestamp, + prometheusAlertNotificationRawData + } + STATUS current + DESCRIPTION "Ceph in health_error state for more than 5m." +::= { prometheusAlertClusterHealthTraps 1 } + +prometheusAlertClusterHealthTrapHealthWarn NOTIFICATION-TYPE + OBJECTS { + prometheusAlertNotificationAlertName, + prometheusAlertNotificationStatus, + prometheusAlertNotificationSeverity, + prometheusAlertNotificationInstance, + prometheusAlertNotificationJob, + prometheusAlertNotificationDescription, + prometheusAlertNotificationLabels, + prometheusAlertNotificationTimestamp, + prometheusAlertNotificationRawData + } + STATUS current + DESCRIPTION "Ceph in health_warn for more than 15m." +::= { prometheusAlertClusterHealthTraps 2 } + +prometheusAlertMonTrapLowMonitorQuorumCount NOTIFICATION-TYPE + OBJECTS { + prometheusAlertNotificationAlertName, + prometheusAlertNotificationStatus, + prometheusAlertNotificationSeverity, + prometheusAlertNotificationInstance, + prometheusAlertNotificationJob, + prometheusAlertNotificationDescription, + prometheusAlertNotificationLabels, + prometheusAlertNotificationTimestamp, + prometheusAlertNotificationRawData + } + STATUS current + DESCRIPTION "Monitor count in quorum is low." +::= { prometheusAlertMonTraps 1 } + +prometheusAlertOsdTrap10PercentOsdsDown NOTIFICATION-TYPE + OBJECTS { + prometheusAlertNotificationAlertName, + prometheusAlertNotificationStatus, + prometheusAlertNotificationSeverity, + prometheusAlertNotificationInstance, + prometheusAlertNotificationJob, + prometheusAlertNotificationDescription, + prometheusAlertNotificationLabels, + prometheusAlertNotificationTimestamp, + prometheusAlertNotificationRawData + } + STATUS current + DESCRIPTION "More than 10% of OSDs are down." +::= { prometheusAlertOsdTraps 1 } + +prometheusAlertOsdTrapOsdDown NOTIFICATION-TYPE + OBJECTS { + prometheusAlertNotificationAlertName, + prometheusAlertNotificationStatus, + prometheusAlertNotificationSeverity, + prometheusAlertNotificationInstance, + prometheusAlertNotificationJob, + prometheusAlertNotificationDescription, + prometheusAlertNotificationLabels, + prometheusAlertNotificationTimestamp, + prometheusAlertNotificationRawData + } + STATUS current + DESCRIPTION "One or more OSDs down for more than 15 minutes." +::= { prometheusAlertOsdTraps 2 } + +prometheusAlertOsdTrapOsdsNearFull NOTIFICATION-TYPE + OBJECTS { + prometheusAlertNotificationAlertName, + prometheusAlertNotificationStatus, + prometheusAlertNotificationSeverity, + prometheusAlertNotificationInstance, + prometheusAlertNotificationJob, + prometheusAlertNotificationDescription, + prometheusAlertNotificationLabels, + prometheusAlertNotificationTimestamp, + prometheusAlertNotificationRawData + } + STATUS current + DESCRIPTION "An OSD is dangerously full, over 80%." +::= { prometheusAlertOsdTraps 3 } + +prometheusAlertOsdTrapFlapOsd NOTIFICATION-TYPE + OBJECTS { + prometheusAlertNotificationAlertName, + prometheusAlertNotificationStatus, + prometheusAlertNotificationSeverity, + prometheusAlertNotificationInstance, + prometheusAlertNotificationJob, + prometheusAlertNotificationDescription, + prometheusAlertNotificationLabels, + prometheusAlertNotificationTimestamp, + prometheusAlertNotificationRawData + } + STATUS current + DESCRIPTION "An OSD was marked down at back up at least once a minute for 5 minutes." +::= { prometheusAlertOsdTraps 4 } + +prometheusAlertOsdTrapHighPgCountDeviation NOTIFICATION-TYPE + OBJECTS { + prometheusAlertNotificationAlertName, + prometheusAlertNotificationStatus, + prometheusAlertNotificationSeverity, + prometheusAlertNotificationInstance, + prometheusAlertNotificationJob, + prometheusAlertNotificationDescription, + prometheusAlertNotificationLabels, + prometheusAlertNotificationTimestamp, + prometheusAlertNotificationRawData + } + STATUS current + DESCRIPTION "An OSD deviates by more then 30% from average PG count." +::= { prometheusAlertOsdTraps 5 } + +prometheusAlertPgsTrapPgsInactive NOTIFICATION-TYPE + OBJECTS { + prometheusAlertNotificationAlertName, + prometheusAlertNotificationStatus, + prometheusAlertNotificationSeverity, + prometheusAlertNotificationInstance, + prometheusAlertNotificationJob, + prometheusAlertNotificationDescription, + prometheusAlertNotificationLabels, + prometheusAlertNotificationTimestamp, + prometheusAlertNotificationRawData + } + STATUS current + DESCRIPTION "One or more PGs are inactive for more than 5 minutes." +::= { prometheusAlertPgsTraps 1 } + +prometheusAlertPgsTrapPgsUnclean NOTIFICATION-TYPE + OBJECTS { + prometheusAlertNotificationAlertName, + prometheusAlertNotificationStatus, + prometheusAlertNotificationSeverity, + prometheusAlertNotificationInstance, + prometheusAlertNotificationJob, + prometheusAlertNotificationDescription, + prometheusAlertNotificationLabels, + prometheusAlertNotificationTimestamp, + prometheusAlertNotificationRawData + } + STATUS current + DESCRIPTION "One or more PGs are not clean for more than 15 minutes." +::= { prometheusAlertPgsTraps 2 } + +prometheusAlertNodesTrapRootVolumeFull NOTIFICATION-TYPE + OBJECTS { + prometheusAlertNotificationAlertName, + prometheusAlertNotificationStatus, + prometheusAlertNotificationSeverity, + prometheusAlertNotificationInstance, + prometheusAlertNotificationJob, + prometheusAlertNotificationDescription, + prometheusAlertNotificationLabels, + prometheusAlertNotificationTimestamp, + prometheusAlertNotificationRawData + } + STATUS current + DESCRIPTION "Root volume (OSD and MON store) is dangerously full (< 5% free)." +::= { prometheusAlertNodesTraps 1 } + +prometheusAlertNodesTrapNetworkPacketsDropped NOTIFICATION-TYPE + OBJECTS { + prometheusAlertNotificationAlertName, + prometheusAlertNotificationStatus, + prometheusAlertNotificationSeverity, + prometheusAlertNotificationInstance, + prometheusAlertNotificationJob, + prometheusAlertNotificationDescription, + prometheusAlertNotificationLabels, + prometheusAlertNotificationTimestamp, + prometheusAlertNotificationRawData + } + STATUS current + DESCRIPTION "A node experiences packet drop > 1 packet/s on an interface." +::= { prometheusAlertNodesTraps 2 } + +prometheusAlertNodesTrapNetworkPacketErrors NOTIFICATION-TYPE + OBJECTS { + prometheusAlertNotificationAlertName, + prometheusAlertNotificationStatus, + prometheusAlertNotificationSeverity, + prometheusAlertNotificationInstance, + prometheusAlertNotificationJob, + prometheusAlertNotificationDescription, + prometheusAlertNotificationLabels, + prometheusAlertNotificationTimestamp, + prometheusAlertNotificationRawData + } + STATUS current + DESCRIPTION "A node experiences packet errors > 1 packet/s on an interface." +::= { prometheusAlertNodesTraps 3 } + +prometheusAlertNodesTrapStorageFilling NOTIFICATION-TYPE + OBJECTS { + prometheusAlertNotificationAlertName, + prometheusAlertNotificationStatus, + prometheusAlertNotificationSeverity, + prometheusAlertNotificationInstance, + prometheusAlertNotificationJob, + prometheusAlertNotificationDescription, + prometheusAlertNotificationLabels, + prometheusAlertNotificationTimestamp, + prometheusAlertNotificationRawData + } + STATUS current + DESCRIPTION "A mountpoint will be full in less then 5 days assuming the average fillup rate of the past 48 hours." +::= { prometheusAlertNodesTraps 4 } + +prometheusAlertPoolsTrapPoolFull NOTIFICATION-TYPE + OBJECTS { + prometheusAlertNotificationAlertName, + prometheusAlertNotificationStatus, + prometheusAlertNotificationSeverity, + prometheusAlertNotificationInstance, + prometheusAlertNotificationJob, + prometheusAlertNotificationDescription, + prometheusAlertNotificationLabels, + prometheusAlertNotificationTimestamp, + prometheusAlertNotificationRawData + } + STATUS current + DESCRIPTION "A pool is at 90% capacity or over." +::= { prometheusAlertPoolsTraps 1 } + +prometheusAlertPoolsTrapPoolFillingUp NOTIFICATION-TYPE + OBJECTS { + prometheusAlertNotificationAlertName, + prometheusAlertNotificationStatus, + prometheusAlertNotificationSeverity, + prometheusAlertNotificationInstance, + prometheusAlertNotificationJob, + prometheusAlertNotificationDescription, + prometheusAlertNotificationLabels, + prometheusAlertNotificationTimestamp, + prometheusAlertNotificationRawData + } + STATUS current + DESCRIPTION "A pool will be full in less then 5 days assuming the average fillup rate of the past 48 hours." +::= { prometheusAlertPoolsTraps 2 } + +END diff --git a/monitoring/snmp/README.md b/monitoring/snmp/README.md new file mode 100644 index 0000000000000..dccef1908f89d --- /dev/null +++ b/monitoring/snmp/README.md @@ -0,0 +1,24 @@ +# SNMP schema + +## Traps + +| OID | Description | +| :--- | :--- | +| 1.3.6.1.4.1.50495.15.1.2.1 | The default trap. This is used if no OID is specified in the alert labels. | +| 1.3.6.1.4.1.50495.15.1.2.[2...N] | Custom traps. | + +## Objects + +The following objects are appended as variable binds to an SNMP trap. + +| OID | Type | Description | +| :--- | :---: | :--- | +| 1.3.6.1.4.1.50495.15.1.1.1 | String | The name of the Prometheus alert. | +| 1.3.6.1.4.1.50495.15.1.1.2 | String | The status of the Prometheus alert. | +| 1.3.6.1.4.1.50495.15.1.1.3 | String | The severity of the Prometheus alert. | +| 1.3.6.1.4.1.50495.15.1.1.4 | String | Unique identifier for the Prometheus instance. | +| 1.3.6.1.4.1.50495.15.1.1.5 | String | The name of the Prometheus job. | +| 1.3.6.1.4.1.50495.15.1.1.6 | String | The Prometheus alert description field. | +| 1.3.6.1.4.1.50495.15.1.1.7 | String | Additional Prometheus alert labels as JSON string. | +| 1.3.6.1.4.1.50495.15.1.1.8 | Unix timestamp | The time when the Prometheus alert occurred. | +| 1.3.6.1.4.1.50495.15.1.1.9 | String | The raw Prometheus alert as JSON string. | \ No newline at end of file From bfbd279875f6395e203f17d2f22686f6f9ad9125 Mon Sep 17 00:00:00 2001 From: Elena Chernikova Date: Mon, 27 Sep 2021 01:13:00 +0300 Subject: [PATCH 0059/2866] sysvinit: create /var/run/ceph before pid /var/run is mounted as tmpfs and will be cleared after reboot. On sysvinit ceph init service creates this directory for pid with root ownership. Fixes: https://tracker.ceph.com/issues/19242 Signed-off-by: Elena Chernikova --- src/init-ceph.in | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/init-ceph.in b/src/init-ceph.in index f60c59293181d..9d93e6b85c1d4 100755 --- a/src/init-ceph.in +++ b/src/init-ceph.in @@ -295,9 +295,12 @@ for name in $what; do get_conf pid_file "$run_dir/$type.$id.pid" "pid file" + if [ ! -d $run_dir ]; then + install -d -m0770 -o ceph -g ceph /var/run/ceph + fi + if [ "$command" = "start" -o "$command" = "onestart" ]; then if [ -n "$pid_file" ]; then - do_cmd "mkdir -p "`dirname $pid_file` cmd="$cmd --pid-file $pid_file" fi @@ -423,10 +426,6 @@ for name in $what; do fi echo Starting Ceph $name on $host... - if [ ! -d $run_dir ]; then - # assume /var/run exists - install -d -m0770 -o ceph -g ceph /var/run/ceph - fi get_conf pre_start_eval "" "pre start eval" [ -n "$pre_start_eval" ] && $pre_start_eval get_conf pre_start "" "pre start command" From 2260e9d711fcf61e5530f7dd9d420b14e174f03a Mon Sep 17 00:00:00 2001 From: Prashant D Date: Tue, 28 Sep 2021 06:30:07 +0100 Subject: [PATCH 0060/2866] doc/dev/config: Replace invalid config debug-pg There is no such '--debug-pg' configuration option available, use '--debug-monc' instead. Signed-off-by: Prashant D --- doc/dev/config.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/dev/config.rst b/doc/dev/config.rst index a42cf3c29f1df..9cb20aee7e864 100644 --- a/doc/dev/config.rst +++ b/doc/dev/config.rst @@ -12,7 +12,7 @@ How can the configuration be set? Well, there are several sources: - command line arguments:: --debug-ms=1 - --debug-pg=10 + --debug-monc=10 etc. - arguments injected at runtime using ``injectargs`` or ``config set`` From e303dc1cf543fa467d6e0a16c122f77570865e21 Mon Sep 17 00:00:00 2001 From: Mykola Golub Date: Tue, 21 Sep 2021 16:17:17 +0100 Subject: [PATCH 0061/2866] osd: handle case when stat returns with error in get_hash_info Previously in the case of the error we stored in the cache and returned HashInfo(ec_impl->get_chunk_count()), which e.g. could propagate to non-primary shards, introducing inconsistency. The function's `checks` flag is replaced with `create` flag, which seems to have more clear meaning here. In be_deep_scrub the get_hash_info is still called with the second argument false (i.e. with `create=false`, while previously it was `checks=false`), which is done intentionally. Fixes: https://tracker.ceph.com/issues/48959 Signed-off-by: Mykola Golub --- src/osd/ECBackend.cc | 11 +++++++---- src/osd/ECBackend.h | 2 +- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/osd/ECBackend.cc b/src/osd/ECBackend.cc index 15bb059033abf..b08be6e6f709d 100644 --- a/src/osd/ECBackend.cc +++ b/src/osd/ECBackend.cc @@ -1827,7 +1827,7 @@ void ECBackend::do_read_op(ReadOp &op) } ECUtil::HashInfoRef ECBackend::get_hash_info( - const hobject_t &hoid, bool checks, const map> *attrs) + const hobject_t &hoid, bool create, const map> *attrs) { dout(10) << __func__ << ": Getting attr on " << hoid << dendl; ECUtil::HashInfoRef ref = unstable_hashinfo_registry.lookup(hoid); @@ -1839,7 +1839,6 @@ ECUtil::HashInfoRef ECBackend::get_hash_info( ghobject_t(hoid, ghobject_t::NO_GEN, get_parent()->whoami_shard().shard), &st); ECUtil::HashInfo hinfo(ec_impl->get_chunk_count()); - // XXX: What does it mean if there is no object on disk? if (r >= 0) { dout(10) << __func__ << ": found on disk, size " << st.st_size << dendl; bufferlist bl; @@ -1869,7 +1868,7 @@ ECUtil::HashInfoRef ECBackend::get_hash_info( dout(0) << __func__ << ": Can't decode hinfo for " << hoid << dendl; return ECUtil::HashInfoRef(); } - if (checks && hinfo.get_total_chunk_size() != (uint64_t)st.st_size) { + if (hinfo.get_total_chunk_size() != (uint64_t)st.st_size) { dout(0) << __func__ << ": Mismatch of total_chunk_size " << hinfo.get_total_chunk_size() << dendl; return ECUtil::HashInfoRef(); @@ -1877,6 +1876,10 @@ ECUtil::HashInfoRef ECBackend::get_hash_info( } else if (st.st_size > 0) { // If empty object and no hinfo, create it return ECUtil::HashInfoRef(); } + } else if (r != -ENOENT || !create) { + derr << __func__ << ": stat " << hoid << " failed: " << cpp_strerror(r) + << dendl; + return ECUtil::HashInfoRef(); } ref = unstable_hashinfo_registry.lookup_or_create(hoid, hinfo); } @@ -1891,7 +1894,7 @@ void ECBackend::start_rmw(Op *op, PGTransactionUPtr &&t) sinfo, std::move(t), [&](const hobject_t &i) { - ECUtil::HashInfoRef ref = get_hash_info(i, false); + ECUtil::HashInfoRef ref = get_hash_info(i, true); if (!ref) { derr << __func__ << ": get_hash_info(" << i << ")" << " returned a null pointer and there is no " diff --git a/src/osd/ECBackend.h b/src/osd/ECBackend.h index e5e4acee87002..296c1202afa37 100644 --- a/src/osd/ECBackend.h +++ b/src/osd/ECBackend.h @@ -629,7 +629,7 @@ class ECBackend : public PGBackend { const ECUtil::stripe_info_t sinfo; /// If modified, ensure that the ref is held until the update is applied SharedPtrRegistry unstable_hashinfo_registry; - ECUtil::HashInfoRef get_hash_info(const hobject_t &hoid, bool checks = true, + ECUtil::HashInfoRef get_hash_info(const hobject_t &hoid, bool create = false, const std::map> *attr = NULL); public: From 179124e80fa3e792f2ace9bfcec827edb4caba85 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Tue, 28 Sep 2021 11:27:20 -0400 Subject: [PATCH 0062/2866] .github/CODEOWNERS: tag rook devs on rook related PRs Signed-off-by: Sage Weil --- .github/CODEOWNERS | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 50c0be9a8a6aa..88c6da6c62d4e 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -35,6 +35,10 @@ /doc/dev/cephadm @ceph/orchestrators /doc/man/8/cephadm.rst @ceph/orchestrators +/qa/suites/orch/rook @ceph/rook +/src/pybind/mgr/rook @ceph/rook +/doc/mgr/rook.rst @ceph/rook + #ceph-volume /src/ceph-volume @ceph/ceph-volume /doc/ceph-volume @ceph/ceph-volume From d35920da5e58fb6c116d4e46aa63bce1adc91bff Mon Sep 17 00:00:00 2001 From: Mykola Golub Date: Wed, 22 Sep 2021 10:43:24 +0100 Subject: [PATCH 0063/2866] qa/suites/rados: add inconsistent hinfo test Signed-off-by: Mykola Golub --- .../singleton/all/ec-inconsistent-hinfo.yaml | 36 +++ qa/tasks/ec_inconsistent_hinfo.py | 225 ++++++++++++++++++ 2 files changed, 261 insertions(+) create mode 100644 qa/suites/rados/singleton/all/ec-inconsistent-hinfo.yaml create mode 100644 qa/tasks/ec_inconsistent_hinfo.py diff --git a/qa/suites/rados/singleton/all/ec-inconsistent-hinfo.yaml b/qa/suites/rados/singleton/all/ec-inconsistent-hinfo.yaml new file mode 100644 index 0000000000000..d71eab1499260 --- /dev/null +++ b/qa/suites/rados/singleton/all/ec-inconsistent-hinfo.yaml @@ -0,0 +1,36 @@ +roles: +- - mon.a + - mon.b + - mon.c + - mgr.x + - osd.0 + - osd.1 + - osd.2 + - osd.3 +openstack: + - volumes: # attached to each instance + count: 4 + size: 10 # GB +tasks: +- install: +- ceph: + create_rbd_pool: false + pre-mgr-commands: + - sudo ceph config set mgr mgr/devicehealth/enable_monitoring false --force + log-ignorelist: + - \(OBJECT_ + - \(OSDMAP_FLAGS\) + - \(OSD_ + - \(PG_ + - \(SLOW_OPS\) + - deep-scrub + - missing + - overall HEALTH_ + - repair + - slow request + - unfound + conf: + osd: + osd min pg log entries: 5 + osd max pg log entries: 5 +- ec_inconsistent_hinfo: diff --git a/qa/tasks/ec_inconsistent_hinfo.py b/qa/tasks/ec_inconsistent_hinfo.py new file mode 100644 index 0000000000000..fa10f2c45ba06 --- /dev/null +++ b/qa/tasks/ec_inconsistent_hinfo.py @@ -0,0 +1,225 @@ +""" +Inconsistent_hinfo +""" +import logging +import time +from dateutil.parser import parse +from tasks import ceph_manager +from tasks.util.rados import rados +from teuthology import misc as teuthology + +log = logging.getLogger(__name__) + +def wait_for_deep_scrub_complete(manager, pgid, check_time_now, inconsistent): + log.debug("waiting for pg %s deep-scrub complete (check_time_now=%s)" % + (pgid, check_time_now)) + for i in range(300): + time.sleep(5) + manager.flush_pg_stats([0, 1, 2, 3]) + pgs = manager.get_pg_stats() + pg = next((pg for pg in pgs if pg['pgid'] == pgid), None) + log.debug('pg=%s' % pg); + assert pg + + last_deep_scrub_time = parse(pg['last_deep_scrub_stamp']).strftime('%s') + if last_deep_scrub_time < check_time_now: + log.debug('not scrubbed') + continue + + status = pg['state'].split('+') + if inconsistent: + assert 'inconsistent' in status + else: + assert 'inconsistent' not in status + return + + assert False, 'not scrubbed' + + +def wait_for_backfilling_complete(manager, pgid, from_osd, to_osd): + log.debug("waiting for pg %s backfill from osd.%s to osd.%s complete" % + (pgid, from_osd, to_osd)) + for i in range(300): + time.sleep(5) + manager.flush_pg_stats([0, 1, 2, 3]) + pgs = manager.get_pg_stats() + pg = next((pg for pg in pgs if pg['pgid'] == pgid), None) + log.info('pg=%s' % pg); + assert pg + status = pg['state'].split('+') + if 'active' not in status: + log.debug('not active') + continue + if 'backfilling' in status: + assert from_osd in pg['acting'] and to_osd in pg['up'] + log.debug('backfilling') + continue + if to_osd not in pg['up']: + log.debug('backfill not started yet') + continue + log.debug('backfilled!') + break + +def task(ctx, config): + """ + Test handling of objects with inconsistent hash info during backfill and deep-scrub. + + A pretty rigid cluster is brought up and tested by this task + """ + if config is None: + config = {} + assert isinstance(config, dict), \ + 'ec_inconsistent_hinfo task only accepts a dict for configuration' + first_mon = teuthology.get_first_mon(ctx, config) + (mon,) = ctx.cluster.only(first_mon).remotes.keys() + + manager = ceph_manager.CephManager( + mon, + ctx=ctx, + logger=log.getChild('ceph_manager'), + ) + + profile = config.get('erasure_code_profile', { + 'k': '2', + 'm': '1', + 'crush-failure-domain': 'osd' + }) + profile_name = profile.get('name', 'backfill_unfound') + manager.create_erasure_code_profile(profile_name, profile) + pool = manager.create_pool_with_unique_name( + pg_num=1, + erasure_code_profile_name=profile_name, + min_size=2) + manager.raw_cluster_cmd('osd', 'pool', 'set', pool, + 'pg_autoscale_mode', 'off') + + manager.flush_pg_stats([0, 1, 2, 3]) + manager.wait_for_clean() + + pool_id = manager.get_pool_num(pool) + pgid = '%d.0' % pool_id + pgs = manager.get_pg_stats() + acting = next((pg['acting'] for pg in pgs if pg['pgid'] == pgid), None) + log.info("acting=%s" % acting) + assert acting + primary = acting[0] + + # something that is always there, readable and never empty + dummyfile = '/etc/group' + + # kludge to make sure they get a map + rados(ctx, mon, ['-p', pool, 'put', 'dummy', dummyfile]) + + manager.flush_pg_stats([0, 1]) + manager.wait_for_recovery() + + log.debug("create test object") + obj = 'test' + rados(ctx, mon, ['-p', pool, 'put', obj, dummyfile]) + + victim = acting[1] + + log.info("remove test object hash info from osd.%s shard and test deep-scrub and repair" + % victim) + + manager.objectstore_tool(pool, options='', args='rm-attr hinfo_key', + object_name=obj, osd=victim) + check_time_now = time.strftime('%s') + manager.raw_cluster_cmd('pg', 'deep-scrub', pgid) + wait_for_deep_scrub_complete(manager, pgid, check_time_now, True) + + check_time_now = time.strftime('%s') + manager.raw_cluster_cmd('pg', 'repair', pgid) + wait_for_deep_scrub_complete(manager, pgid, check_time_now, False) + + log.info("remove test object hash info from primary osd.%s shard and test backfill" + % primary) + + log.debug("write some data") + rados(ctx, mon, ['-p', pool, 'bench', '30', 'write', '-b', '4096', + '--no-cleanup']) + + manager.objectstore_tool(pool, options='', args='rm-attr hinfo_key', + object_name=obj, osd=primary) + + # mark the osd out to trigger a rebalance/backfill + source = acting[1] + target = [x for x in [0, 1, 2, 3] if x not in acting][0] + manager.mark_out_osd(source) + + # wait for everything to peer, backfill and recover + wait_for_backfilling_complete(manager, pgid, source, target) + manager.wait_for_clean() + + manager.flush_pg_stats([0, 1, 2, 3]) + pgs = manager.get_pg_stats() + pg = next((pg for pg in pgs if pg['pgid'] == pgid), None) + log.debug('pg=%s' % pg) + assert pg + assert 'clean' in pg['state'].split('+') + assert 'inconsistent' not in pg['state'].split('+') + unfound = manager.get_num_unfound_objects() + log.debug("there are %d unfound objects" % unfound) + assert unfound == 0 + + source, target = target, source + log.info("remove test object hash info from non-primary osd.%s shard and test backfill" + % source) + + manager.objectstore_tool(pool, options='', args='rm-attr hinfo_key', + object_name=obj, osd=source) + + # mark the osd in to trigger a rebalance/backfill + manager.mark_in_osd(target) + + # wait for everything to peer, backfill and recover + wait_for_backfilling_complete(manager, pgid, source, target) + manager.wait_for_clean() + + manager.flush_pg_stats([0, 1, 2, 3]) + pgs = manager.get_pg_stats() + pg = next((pg for pg in pgs if pg['pgid'] == pgid), None) + log.debug('pg=%s' % pg) + assert pg + assert 'clean' in pg['state'].split('+') + assert 'inconsistent' not in pg['state'].split('+') + unfound = manager.get_num_unfound_objects() + log.debug("there are %d unfound objects" % unfound) + assert unfound == 0 + + log.info("remove hash info from two shards and test backfill") + + source = acting[2] + target = [x for x in [0, 1, 2, 3] if x not in acting][0] + manager.objectstore_tool(pool, options='', args='rm-attr hinfo_key', + object_name=obj, osd=primary) + manager.objectstore_tool(pool, options='', args='rm-attr hinfo_key', + object_name=obj, osd=source) + + # mark the osd out to trigger a rebalance/backfill + manager.mark_out_osd(source) + + # wait for everything to peer, backfill and detect unfound object + wait_for_backfilling_complete(manager, pgid, source, target) + + # verify that there is unfound object + manager.flush_pg_stats([0, 1, 2, 3]) + pgs = manager.get_pg_stats() + pg = next((pg for pg in pgs if pg['pgid'] == pgid), None) + log.debug('pg=%s' % pg) + assert pg + assert 'backfill_unfound' in pg['state'].split('+') + unfound = manager.get_num_unfound_objects() + log.debug("there are %d unfound objects" % unfound) + assert unfound == 1 + m = manager.list_pg_unfound(pgid) + log.debug('list_pg_unfound=%s' % m) + assert m['num_unfound'] == pg['stat_sum']['num_objects_unfound'] + + # mark stuff lost + pgs = manager.get_pg_stats() + manager.raw_cluster_cmd('pg', pgid, 'mark_unfound_lost', 'delete') + + # wait for everything to peer and be happy... + manager.flush_pg_stats([0, 1, 2, 3]) + manager.wait_for_recovery() From b20637714e366b44ec604a76bab9095d5f4a1ef2 Mon Sep 17 00:00:00 2001 From: Adam King Date: Tue, 28 Sep 2021 22:10:07 -0400 Subject: [PATCH 0064/2866] mgr/cephadm: use json-pretty for ceph-volume calls To avoid a single line json string that is too long and causes a chunk longer than limit error Fixes: https://tracker.ceph.com/issues/52745 Signed-off-by: Adam King --- src/pybind/mgr/cephadm/serve.py | 2 +- src/pybind/mgr/cephadm/tests/test_cephadm.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pybind/mgr/cephadm/serve.py b/src/pybind/mgr/cephadm/serve.py index f8d4398b56f6b..7297b9dd2e879 100644 --- a/src/pybind/mgr/cephadm/serve.py +++ b/src/pybind/mgr/cephadm/serve.py @@ -470,7 +470,7 @@ def _refresh_facts(self, host: str) -> Optional[str]: def _refresh_host_devices(self, host: str) -> Optional[str]: with_lsm = self.mgr.get_module_option('device_enhanced_scan') inventory_args = ['--', 'inventory', - '--format=json', + '--format=json-pretty', '--filter-for-batch'] if with_lsm: inventory_args.insert(-1, "--with-lsm") diff --git a/src/pybind/mgr/cephadm/tests/test_cephadm.py b/src/pybind/mgr/cephadm/tests/test_cephadm.py index 50628ca3f78b9..ba5ebe7cf539c 100644 --- a/src/pybind/mgr/cephadm/tests/test_cephadm.py +++ b/src/pybind/mgr/cephadm/tests/test_cephadm.py @@ -1355,10 +1355,10 @@ def test_ceph_volume_no_filter_for_batch(self, _run_cephadm, cephadm_module: Cep assert _run_cephadm.mock_calls == [ mock.call('test', 'osd', 'ceph-volume', - ['--', 'inventory', '--format=json', '--filter-for-batch'], image='', + ['--', 'inventory', '--format=json-pretty', '--filter-for-batch'], image='', no_fsid=False), mock.call('test', 'osd', 'ceph-volume', - ['--', 'inventory', '--format=json'], image='', + ['--', 'inventory', '--format=json-pretty'], image='', no_fsid=False), ] From 97219591dab8f54f239ccfb1891c45b01e7bec79 Mon Sep 17 00:00:00 2001 From: Guillaume Abrioux Date: Wed, 22 Sep 2021 09:35:54 +0200 Subject: [PATCH 0065/2866] cephadm: fix osd size reported in 'orch ls' The current code enforces ('running osds' + 1), see [1]. Let's simply print the number of running osds instead of a meaningless calculation. [1] https://github.com/ceph/ceph/commit/1c0a0a6f6153dc555085f263f161666cd171964a#diff-4f2fb7d330e74b64ac41457b7c7a723cd78db86433e0b0c398874531e5a7e39eR1852-R1859 Fixes: https://tracker.ceph.com/issues/50928 Signed-off-by: Guillaume Abrioux --- src/pybind/mgr/orchestrator/module.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/pybind/mgr/orchestrator/module.py b/src/pybind/mgr/orchestrator/module.py index d75271fbff6fc..be9818a1fe0bc 100644 --- a/src/pybind/mgr/orchestrator/module.py +++ b/src/pybind/mgr/orchestrator/module.py @@ -604,10 +604,15 @@ def ukn(s: Optional[str]) -> str: else: refreshed = nice_delta(now, s.last_refresh, ' ago') + if s.spec.service_type == 'osd': + running = str(s.running) + else: + running = '{}/{}'.format(s.running, s.size) + table.add_row(( s.spec.service_name(), s.get_port_summary(), - '%d/%d' % (s.running, s.size), + running, refreshed, nice_delta(now, s.created), pl, From 1d848dc447fd4df18893903a053d58129fb8c13a Mon Sep 17 00:00:00 2001 From: Guillaume Abrioux Date: Wed, 29 Sep 2021 08:42:33 +0200 Subject: [PATCH 0066/2866] cephadm: add a test on 'ceph orch ls' This adds a unittest in order to cover the output of 'orch ls' command and check the reporting about the running osd number is displayed as expected. Signed-off-by: Guillaume Abrioux Co-Authored-by: Sebastian Wagner --- .../orchestrator/tests/test_orchestrator.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/src/pybind/mgr/orchestrator/tests/test_orchestrator.py b/src/pybind/mgr/orchestrator/tests/test_orchestrator.py index 35154270449f8..73a6d9b288908 100644 --- a/src/pybind/mgr/orchestrator/tests/test_orchestrator.py +++ b/src/pybind/mgr/orchestrator/tests/test_orchestrator.py @@ -11,9 +11,10 @@ from test_orchestrator import TestOrchestrator as _TestOrchestrator -from orchestrator import InventoryHost, DaemonDescription, ServiceDescription, DaemonDescriptionStatus +from orchestrator import InventoryHost, DaemonDescription, ServiceDescription, DaemonDescriptionStatus, OrchResult from orchestrator import OrchestratorValidationError from orchestrator.module import to_format, Format, OrchestratorCli, preview_table_osd +from unittest import mock def _test_resource(data, resource_class, extra=None): @@ -140,6 +141,21 @@ def test_handle_command(): retval=-2, stdout='', stderr='No orchestrator configured (try `ceph orch set backend`)') +r = OrchResult([ServiceDescription(spec=ServiceSpec(service_type='osd'), running=123)]) + + +@mock.patch("orchestrator.OrchestratorCli.describe_service", return_value=r) +def test_orch_ls(_describe_service): + cmd = { + 'prefix': 'orch ls', + } + m = OrchestratorCli('orchestrator', 0, 0) + r = m._handle_command(None, cmd) + out = 'NAME PORTS RUNNING REFRESHED AGE PLACEMENT \n' \ + 'osd 123 - - ' + assert r == HandleCommandResult(retval=0, stdout=out, stderr='') + + def test_preview_table_osd_smoke(): data = [ { From baf0c10bb9374f53a6538cc0d9b0390782f457f3 Mon Sep 17 00:00:00 2001 From: wangyingbin Date: Thu, 30 Sep 2021 15:34:30 +0800 Subject: [PATCH 0067/2866] rgw:word spelling error in rgw/rgw_pubsub.h file In line 362, mistakenly write "setting" as "settign" Signed-off-by: wangyingbin --- src/rgw/rgw_pubsub.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rgw/rgw_pubsub.h b/src/rgw/rgw_pubsub.h index ba86bbe845ee2..fb3005a0ab5ed 100644 --- a/src/rgw/rgw_pubsub.h +++ b/src/rgw/rgw_pubsub.h @@ -358,7 +358,7 @@ struct rgw_pubsub_event { }; WRITE_CLASS_ENCODER(rgw_pubsub_event) -// settign a unique ID for an event based on object hash and timestamp +// setting a unique ID for an event based on object hash and timestamp void set_event_id(std::string& id, const std::string& hash, const utime_t& ts); struct rgw_pubsub_sub_dest { From 8ef77a0bbcef710185c7ee6452d6ca8a133843c1 Mon Sep 17 00:00:00 2001 From: Sebastian Wagner Date: Tue, 28 Sep 2021 16:57:41 +0200 Subject: [PATCH 0068/2866] doc/cephadm: use sphinx autoclass to document RGWSpec Signed-off-by: Sebastian Wagner --- doc/cephadm/services/index.rst | 25 ++------- doc/cephadm/services/rgw.rst | 10 +++- doc/mgr/orchestrator_modules.rst | 4 ++ .../ceph/deployment/service_spec.py | 55 +++++++++++++++++-- 4 files changed, 68 insertions(+), 26 deletions(-) diff --git a/doc/cephadm/services/index.rst b/doc/cephadm/services/index.rst index 00eacdfce2f7e..f34180eb24fc6 100644 --- a/doc/cephadm/services/index.rst +++ b/doc/cephadm/services/index.rst @@ -99,28 +99,15 @@ deployment of services. Here is an example of a service specification in YAML: unmanaged: false networks: - 192.169.142.0/24 - ... + spec: + # Additional service specific attributes. In this example, the properties of this service specification are: -* ``service_type`` - The type of the service. Needs to be either a Ceph - service (``mon``, ``crash``, ``mds``, ``mgr``, ``osd`` or - ``rbd-mirror``), a gateway (``nfs`` or ``rgw``), part of the - monitoring stack (``alertmanager``, ``grafana``, ``node-exporter`` or - ``prometheus``) or (``container``) for custom containers. -* ``service_id`` - The name of the service. -* ``placement`` - See :ref:`orchestrator-cli-placement-spec`. -* ``networks``: A list of network identities instructing the daemons to only bind - on the particular networks in that list. In case the cluster is distributed across multiple - networks, you can add multiple networks. See :ref:`cephadm-monitoring-networks-ports`, - :ref:`cephadm-rgw-networks` and :ref:`cephadm-mgr-networks`. -* ``unmanaged`` If set to ``true``, the orchestrator will not deploy nor remove - any daemon associated with this service. Placement and all other properties - will be ignored. This is useful, if you do not want this service to be - managed temporarily. For cephadm, See :ref:`cephadm-spec-unmanaged` +.. py:currentmodule:: ceph.deployment.service_spec + +.. autoclass:: ServiceSpec + :members: Each service type can have additional service-specific properties. diff --git a/doc/cephadm/services/rgw.rst b/doc/cephadm/services/rgw.rst index 2914042cc375b..47017139b76c3 100644 --- a/doc/cephadm/services/rgw.rst +++ b/doc/cephadm/services/rgw.rst @@ -140,7 +140,15 @@ Then apply this yaml document: ceph orch apply -i myrgw.yaml Note the value of ``rgw_frontend_ssl_certificate`` is a literal string as -indicated by a ``|`` character preserving newline characters. +indicated by a ``|`` character preserving newline characters. + +Service specification +--------------------- + +.. py:currentmodule:: ceph.deployment.service_spec + +.. autoclass:: RGWSpec + :members: .. _orchestrator-haproxy-service-spec: diff --git a/doc/mgr/orchestrator_modules.rst b/doc/mgr/orchestrator_modules.rst index 65e2e4981d133..5991afe613656 100644 --- a/doc/mgr/orchestrator_modules.rst +++ b/doc/mgr/orchestrator_modules.rst @@ -233,6 +233,9 @@ Services .. py:currentmodule:: ceph.deployment.service_spec .. autoclass:: ServiceSpec + :members: + :private-members: + :noindex: .. py:currentmodule:: orchestrator @@ -291,6 +294,7 @@ Services .. py:currentmodule:: ceph.deployment.service_spec .. autoclass:: RGWSpec + :noindex: .. py:currentmodule:: orchestrator diff --git a/src/python-common/ceph/deployment/service_spec.py b/src/python-common/ceph/deployment/service_spec.py index 98246376e947a..698676798513f 100644 --- a/src/python-common/ceph/deployment/service_spec.py +++ b/src/python-common/ceph/deployment/service_spec.py @@ -411,7 +411,6 @@ class ServiceSpec(object): This structure is supposed to be enough information to start the services. - """ KNOWN_SERVICE_TYPES = 'alertmanager crash grafana iscsi mds mgr mon nfs ' \ 'node-exporter osd prometheus rbd-mirror rgw agent ' \ @@ -466,15 +465,37 @@ def __init__(self, preview_only: bool = False, networks: Optional[List[str]] = None, ): + + #: See :ref:`orchestrator-cli-placement-spec`. self.placement = PlacementSpec() if placement is None else placement # type: PlacementSpec assert service_type in ServiceSpec.KNOWN_SERVICE_TYPES, service_type + #: The type of the service. Needs to be either a Ceph + #: service (``mon``, ``crash``, ``mds``, ``mgr``, ``osd`` or + #: ``rbd-mirror``), a gateway (``nfs`` or ``rgw``), part of the + #: monitoring stack (``alertmanager``, ``grafana``, ``node-exporter`` or + #: ``prometheus``) or (``container``) for custom containers. self.service_type = service_type + + #: The name of the service. Required for ``iscsi``, ``mds``, ``nfs``, ``osd``, ``rgw``, + #: ``container``, ``ingress`` self.service_id = None + if self.service_type in self.REQUIRES_SERVICE_ID: self.service_id = service_id + + #: If set to ``true``, the orchestrator will not deploy nor remove + #: any daemon associated with this service. Placement and all other properties + #: will be ignored. This is useful, if you do not want this service to be + #: managed temporarily. For cephadm, See :ref:`cephadm-spec-unmanaged` self.unmanaged = unmanaged self.preview_only = preview_only + + #: A list of network identities instructing the daemons to only bind + #: on the particular networks in that list. In case the cluster is distributed + #: across multiple networks, you can add multiple networks. See + #: :ref:`cephadm-monitoring-networks-ports`, + #: :ref:`cephadm-rgw-networks` and :ref:`cephadm-mgr-networks`. self.networks: List[str] = networks or [] self.config: Optional[Dict[str, str]] = None @@ -519,6 +540,8 @@ def from_json(cls: Type[ServiceSpecT], json_spec: Dict) -> ServiceSpecT: the next two major releases (octoups, pacific). :param json_spec: A valid dict with ServiceSpec + + :meta private: """ if not isinstance(json_spec, dict): @@ -698,7 +721,21 @@ class RGWSpec(ServiceSpec): """ Settings to configure a (multisite) Ceph RGW + .. code-block:: yaml + + service_type: rgw + service_id: myrealm.myzone + spec: + rgw_realm: myrealm + rgw_zone: myzone + ssl: true + rgw_frontend_port: 1234 + rgw_frontend_type: beast + rgw_frontend_ssl_certificate: ... + + See also: :ref:`orchestrator-cli-service-spec` """ + MANAGED_CONFIG_OPTIONS = ServiceSpec.MANAGED_CONFIG_OPTIONS + [ 'rgw_zone', 'rgw_realm', @@ -732,11 +769,17 @@ def __init__(self, placement=placement, unmanaged=unmanaged, preview_only=preview_only, config=config, networks=networks) - self.rgw_realm = rgw_realm - self.rgw_zone = rgw_zone - self.rgw_frontend_port = rgw_frontend_port - self.rgw_frontend_ssl_certificate = rgw_frontend_ssl_certificate - self.rgw_frontend_type = rgw_frontend_type + #: The RGW realm associated with this service. Needs to be manually created + self.rgw_realm: Optional[str] = rgw_realm + #: The RGW zone associated with this service. Needs to be manually created + self.rgw_zone: Optional[str] = rgw_zone + #: Port of the RGW daemons + self.rgw_frontend_port: Optional[int] = rgw_frontend_port + #: List of SSL certificates + self.rgw_frontend_ssl_certificate: Optional[List[str]] = rgw_frontend_ssl_certificate + #: civetweb or beast (default: beast). See :ref:`rgw_frontends` + self.rgw_frontend_type: Optional[str] = rgw_frontend_type + #: enable SSL self.ssl = ssl def get_port_start(self) -> List[int]: From 2e7b369373236c32e429463512521da1fc7e159d Mon Sep 17 00:00:00 2001 From: Sebastian Wagner Date: Wed, 29 Sep 2021 12:00:42 +0200 Subject: [PATCH 0069/2866] doc/cephadm: use sphinx autoclass to document IscsiServiceSpec Signed-off-by: Sebastian Wagner --- doc/cephadm/services/iscsi.rst | 8 +++++++- src/python-common/ceph/deployment/service_spec.py | 7 +++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/doc/cephadm/services/iscsi.rst b/doc/cephadm/services/iscsi.rst index 581d2c9d40612..e039e8d9a3467 100644 --- a/doc/cephadm/services/iscsi.rst +++ b/doc/cephadm/services/iscsi.rst @@ -44,7 +44,6 @@ For example: trusted_ip_list: "IP_ADDRESS_1,IP_ADDRESS_2,IP_ADDRESS_3,..." api_user: API_USERNAME api_password: API_PASSWORD - api_secure: true ssl_cert: | -----BEGIN CERTIFICATE----- MIIDtTCCAp2gAwIBAgIYMC4xNzc1NDQxNjEzMzc2MjMyXzxvQ7EcMA0GCSqGSIb3 @@ -58,6 +57,11 @@ For example: [...] -----END PRIVATE KEY----- +.. py:currentmodule:: ceph.deployment.service_spec + +.. autoclass:: IscsiServiceSpec + :members: + The specification can then be applied using: @@ -68,6 +72,8 @@ The specification can then be applied using: See :ref:`orchestrator-cli-placement-spec` for details of the placement specification. +See also: :ref:`orchestrator-cli-service-spec`. + Further Reading =============== diff --git a/src/python-common/ceph/deployment/service_spec.py b/src/python-common/ceph/deployment/service_spec.py index 698676798513f..21c16026500a1 100644 --- a/src/python-common/ceph/deployment/service_spec.py +++ b/src/python-common/ceph/deployment/service_spec.py @@ -833,12 +833,19 @@ def __init__(self, #: RADOS pool where ceph-iscsi config data is stored. self.pool = pool + #: list of trusted IP addresses self.trusted_ip_list = trusted_ip_list + #: ``api_port`` as defined in the ``iscsi-gateway.cfg`` self.api_port = api_port + #: ``api_user`` as defined in the ``iscsi-gateway.cfg`` self.api_user = api_user + #: ``api_password`` as defined in the ``iscsi-gateway.cfg`` self.api_password = api_password + #: ``api_secure`` as defined in the ``iscsi-gateway.cfg`` self.api_secure = api_secure + #: SSL certificate self.ssl_cert = ssl_cert + #: SSL private key self.ssl_key = ssl_key if not self.api_secure and self.ssl_cert and self.ssl_key: From ce73047d25599b247ab889614fa62684b0c7aca0 Mon Sep 17 00:00:00 2001 From: Sebastian Wagner Date: Wed, 29 Sep 2021 12:02:28 +0200 Subject: [PATCH 0070/2866] doc/cephadm: update example of custom container spec file Signed-off-by: Sebastian Wagner --- doc/cephadm/services/custom-container.rst | 35 ++++++++++++----------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/doc/cephadm/services/custom-container.rst b/doc/cephadm/services/custom-container.rst index 542fcf16261d8..3ece248c5a4c9 100644 --- a/doc/cephadm/services/custom-container.rst +++ b/doc/cephadm/services/custom-container.rst @@ -11,32 +11,33 @@ A corresponding :ref:`orchestrator-cli-service-spec` must look like: service_id: foo placement: ... - image: docker.io/library/foo:latest - entrypoint: /usr/bin/foo - uid: 1000 - gid: 1000 - args: + spec: + image: docker.io/library/foo:latest + entrypoint: /usr/bin/foo + uid: 1000 + gid: 1000 + args: - "--net=host" - "--cpus=2" - ports: + ports: - 8080 - 8443 - envs: + envs: - SECRET=mypassword - PORT=8080 - PUID=1000 - PGID=1000 - volume_mounts: + volume_mounts: CONFIG_DIR: /etc/foo - bind_mounts: - - ['type=bind', 'source=lib/modules', 'destination=/lib/modules', 'ro=true'] - dirs: - - CONFIG_DIR - files: - CONFIG_DIR/foo.conf: - - refresh=true - - username=xyz - - "port: 1234" + bind_mounts: + - ['type=bind', 'source=lib/modules', 'destination=/lib/modules', 'ro=true'] + dirs: + - CONFIG_DIR + files: + CONFIG_DIR/foo.conf: + - refresh=true + - username=xyz + - "port: 1234" where the properties of a service specification are: From bdc802354554ea9c5e7c0377e3fe695f719921a7 Mon Sep 17 00:00:00 2001 From: Adam King Date: Thu, 30 Sep 2021 08:34:58 -0400 Subject: [PATCH 0071/2866] mgr/cephadm: put list_servers output in debug logs Over time, seen a number of cases where daemons are being marked stray for unknown reasons and it is always difficult to debug due to the difficulty of finding what list_servers is outputting. I think it would help to include this in the debug logs. That way we have that side of the info available and can easily compare it to daemon/host info cephadm is storing. Signed-off-by: Adam King --- src/pybind/mgr/cephadm/serve.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/pybind/mgr/cephadm/serve.py b/src/pybind/mgr/cephadm/serve.py index f8d4398b56f6b..019d847f65988 100644 --- a/src/pybind/mgr/cephadm/serve.py +++ b/src/pybind/mgr/cephadm/serve.py @@ -537,6 +537,7 @@ def _check_for_strays(self) -> None: del self.mgr.health_checks[k] if self.mgr.warn_on_stray_hosts or self.mgr.warn_on_stray_daemons: ls = self.mgr.list_servers() + self.log.debug(ls) managed = self.mgr.cache.get_daemon_names() host_detail = [] # type: List[str] host_num_daemons = 0 From 1b74a3db0c23423bb5a188eafb4346711d0b13a2 Mon Sep 17 00:00:00 2001 From: Adam King Date: Thu, 30 Sep 2021 09:32:31 -0400 Subject: [PATCH 0072/2866] mgr/cephadm: move handling of use_agent setting into agent.py The serve function itself should only really be calling other functions. The actual handling of the setting is better placed in the agent's own file. Signed-off-by: Adam King --- src/pybind/mgr/cephadm/agent.py | 23 +++++++++++++++++++++++ src/pybind/mgr/cephadm/serve.py | 19 ++----------------- 2 files changed, 25 insertions(+), 17 deletions(-) diff --git a/src/pybind/mgr/cephadm/agent.py b/src/pybind/mgr/cephadm/agent.py index d43f639264855..39f091a496b89 100644 --- a/src/pybind/mgr/cephadm/agent.py +++ b/src/pybind/mgr/cephadm/agent.py @@ -319,6 +319,29 @@ def _apply_agent(self) -> None: ) self.mgr.spec_store.save(spec) + def _handle_use_agent_setting(self) -> bool: + need_apply = False + if self.mgr.use_agent: + # on the off chance there are still agents hanging around from + # when we turned the config option off, we need to redeploy them + # we can tell they're in that state if we don't have a keyring for + # them in the host cache + for agent in self.mgr.cache.get_daemons_by_service('agent'): + if agent.hostname not in self.mgr.cache.agent_keys: + self.mgr._schedule_daemon_action(agent.name(), 'redeploy') + if 'agent' not in self.mgr.spec_store: + self.mgr.agent_helpers._apply_agent() + need_apply = True + else: + if 'agent' in self.mgr.spec_store: + self.mgr.spec_store.rm('agent') + need_apply = True + self.mgr.cache.agent_counter = {} + self.mgr.cache.agent_timestamp = {} + self.mgr.cache.agent_keys = {} + self.mgr.cache.agent_ports = {} + return need_apply + class SSLCerts: def __init__(self, mgr: "CephadmOrchestrator") -> None: diff --git a/src/pybind/mgr/cephadm/serve.py b/src/pybind/mgr/cephadm/serve.py index f8d4398b56f6b..b3048b1fb5641 100644 --- a/src/pybind/mgr/cephadm/serve.py +++ b/src/pybind/mgr/cephadm/serve.py @@ -91,23 +91,8 @@ def serve(self) -> None: self._purge_deleted_services() - if self.mgr.use_agent: - # on the off chance there are still agents hanging around from - # when we turned the config option off, we need to redeploy them - # we can tell they're in that state if we don't have a keyring for - # them in the host cache - for agent in self.mgr.cache.get_daemons_by_service('agent'): - if agent.hostname not in self.mgr.cache.agent_keys: - self.mgr._schedule_daemon_action(agent.name(), 'redeploy') - if 'agent' not in self.mgr.spec_store: - self.mgr.agent_helpers._apply_agent() - else: - if 'agent' in self.mgr.spec_store: - self.mgr.spec_store.rm('agent') - self.mgr.cache.agent_counter = {} - self.mgr.cache.agent_timestamp = {} - self.mgr.cache.agent_keys = {} - self.mgr.cache.agent_ports = {} + if self.mgr.agent_helpers._handle_use_agent_setting(): + continue if self.mgr.upgrade.continue_upgrade(): continue From 50ac0ec38d5d4e3c2e2ce2dafcccc3ea686ecba0 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Thu, 30 Sep 2021 10:12:24 -0500 Subject: [PATCH 0073/2866] mgr/rook: describe_services: name rgw svc appropriately Set rgw_zone if present. Do not set rgw_realm because we don't know what it is from the CephObjectstore CRD. Signed-off-by: Sage Weil --- src/pybind/mgr/rook/module.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/pybind/mgr/rook/module.py b/src/pybind/mgr/rook/module.py index 5a7f177d3a7bd..cea5ad8ef2835 100644 --- a/src/pybind/mgr/rook/module.py +++ b/src/pybind/mgr/rook/module.py @@ -299,9 +299,7 @@ def describe_service(self, # CephObjectstores all_zones = self.rook_cluster.get_resource("cephobjectstores") for zone in all_zones: - rgw_realm = zone['metadata']['name'] - rgw_zone = rgw_realm - svc = 'rgw.' + rgw_realm + svc = 'rgw.' + zone['metadata']['name'] if svc in spec: continue active = zone['spec']['gateway']['instances']; @@ -311,10 +309,10 @@ def describe_service(self, else: ssl = False port = zone['spec']['gateway']['port'] or 80 + rgw_zone = zone['spec'].get('zone', {}).get('name') or None spec[svc] = orchestrator.ServiceDescription( spec=RGWSpec( service_id=zone['metadata']['name'], - rgw_realm=rgw_realm, rgw_zone=rgw_zone, ssl=ssl, rgw_frontend_port=port, From f3d072b885bb9377876df6f7ab161e1a16fed39d Mon Sep 17 00:00:00 2001 From: Laura Flores Date: Thu, 30 Sep 2021 20:05:59 +0000 Subject: [PATCH 0074/2866] mgr/telemetry: remove pool name from telemetry perf report It is best to remove the pool names from the telemetry perf report, as these names could potentially count as sensitive information. Signed-off-by: Laura Flores --- src/pybind/mgr/telemetry/module.py | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/src/pybind/mgr/telemetry/module.py b/src/pybind/mgr/telemetry/module.py index 91596436a9d86..a0f7eb0b40a84 100644 --- a/src/pybind/mgr/telemetry/module.py +++ b/src/pybind/mgr/telemetry/module.py @@ -306,18 +306,16 @@ def get_stat_sum_per_pool(self) -> List[dict]: # Initialize 'result' list result: List[dict] = [] - # Create a list of tuples containing pool ids and their associated names - # that will later act as a queue, i.e.: - # pool_queue = [('1', '.mgr'), ('2', 'cephfs.a.meta'), ('3', 'cephfs.a.data')] + # Create a list of pool ids that will later act as a queue, i.e.: + # pool_queue = [1, 2, 3] osd_map = self.get('osd_map') - pool_queue: List[tuple] = [] + pool_queue = [] for pool in osd_map['pools']: - pool_queue.append((str(pool['pool']), pool['pool_name'])) + pool_queue.append(str(pool['pool'])) # Populate 'result', i.e.: # { # 'pool_id': '1' - # 'pool_name': '.mgr' # 'stats_sum': { # 'num_bytes': 36, # 'num_bytes_hit_set_archive': 0, @@ -327,11 +325,8 @@ def get_stat_sum_per_pool(self) -> List[dict]: # } # } while pool_queue: - # Pop a pool out of pool_queue - curr_pool = pool_queue.pop(0) - - # Get the current pool's id and name - curr_pool_id, curr_pool_name = curr_pool[0], curr_pool[1] + # Pop the current pool id out of pool_queue + curr_pool_id = pool_queue.pop(0) # Initialize a dict that will hold aggregated stats for the current pool compiled_stats_dict: Dict[str, Any] = defaultdict(lambda: defaultdict(int)) @@ -343,7 +338,6 @@ def get_stat_sum_per_pool(self) -> List[dict]: pool_id = pg['pgid'].split('.')[0] if pool_id == curr_pool_id: compiled_stats_dict['pool_id'] = int(pool_id) - compiled_stats_dict['pool_name'] = curr_pool_name for metric in pg['stat_sum']: compiled_stats_dict['stats_sum'][metric] += pg['stat_sum'][metric] else: From 8a16d95ae8f7b55af170e012373df5159f4b5973 Mon Sep 17 00:00:00 2001 From: Eunice Lee Date: Fri, 1 Oct 2021 13:00:03 -0700 Subject: [PATCH 0075/2866] doc: fix reference to non-existent stable branch Fixes: #50499 Signed-off-by: Eunice Lee (eunice337@gmail.com) --- doc/install/clone-source.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/install/clone-source.rst b/doc/install/clone-source.rst index da62ee93fa679..eac972d327308 100644 --- a/doc/install/clone-source.rst +++ b/doc/install/clone-source.rst @@ -93,7 +93,7 @@ will be on the ``master`` branch by default, which is the unstable development branch. You may choose other branches too. - ``master``: The unstable development branch. -- ``stable``: The bugfix branch. +- ``stable-release-name``: The name of the stable, :ref:`active release `. e.g. ``Pacific`` - ``next``: The release candidate branch. :: From 2a10be53475ce370e5b66d2ccc2f75ca5f36af8d Mon Sep 17 00:00:00 2001 From: Laura Flores Date: Mon, 4 Oct 2021 22:39:23 +0000 Subject: [PATCH 0076/2866] mon: simplify 'mgr module ls' output Fixes: https://tracker.ceph.com/issues/45322 Signed-off-by: Laura Flores --- doc/man/8/ceph.rst | 2 +- doc/mgr/administrator.rst | 3 ++- src/mon/MgrMonitor.cc | 15 ++++++++++++--- src/mon/MonCommands.h | 4 ++-- 4 files changed, 17 insertions(+), 7 deletions(-) diff --git a/doc/man/8/ceph.rst b/doc/man/8/ceph.rst index 639d85e565d6f..af381df92ea8a 100644 --- a/doc/man/8/ceph.rst +++ b/doc/man/8/ceph.rst @@ -580,7 +580,7 @@ Subcommand ``module ls`` will list currently enabled manager modules (plugins). Usage:: - ceph mgr module ls + ceph mgr module ls {detail} Subcommand ``module enable`` will enable a manager module. Available modules are included in MgrMap and visible via ``mgr dump``. diff --git a/doc/mgr/administrator.rst b/doc/mgr/administrator.rst index 6fa563c62ad5e..6115d171c014c 100644 --- a/doc/mgr/administrator.rst +++ b/doc/mgr/administrator.rst @@ -57,7 +57,8 @@ Using modules ------------- Use the command ``ceph mgr module ls`` to see which modules are -available, and which are currently enabled. Enable or disable modules +available, and which are currently enabled. Use ``ceph mgr module ls detail`` +to view detailed metadata about disabled modules. Enable or disable modules using the commands ``ceph mgr module enable `` and ``ceph mgr module disable `` respectively. diff --git a/src/mon/MgrMonitor.cc b/src/mon/MgrMonitor.cc index ef246e648c6c1..b2dae3ce8412a 100644 --- a/src/mon/MgrMonitor.cc +++ b/src/mon/MgrMonitor.cc @@ -953,6 +953,7 @@ bool MgrMonitor::preprocess_command(MonOpRequestRef op) string prefix; cmd_getval(cmdmap, "prefix", prefix); + int r = 0; if (prefix == "mgr stat") { @@ -982,6 +983,9 @@ bool MgrMonitor::preprocess_command(MonOpRequestRef op) } f->flush(rdata); } else if (prefix == "mgr module ls") { + string detail; + cmd_getval(cmdmap, "detail", detail); + f->open_object_section("modules"); { f->open_array_section("always_on_modules"); @@ -1002,9 +1006,14 @@ bool MgrMonitor::preprocess_command(MonOpRequestRef op) for (auto& p : map.available_modules) { if (map.modules.count(p.name) == 0 && map.get_always_on_modules().count(p.name) == 0) { - // For disabled modules, we show the full info, to - // give a hint about whether enabling it will work - p.dump(f.get()); + if (detail == "detail") { + // For disabled modules, we show the full info if the detail + // parameter is enabled, to give a hint about whether enabling it will work + p.dump(f.get()); + } else { + // Otherwise, we give a shortened summary by default + f->dump_string("module", p.name); + } } } f->close_section(); diff --git a/src/mon/MonCommands.h b/src/mon/MonCommands.h index 1db9879873364..78129f336e256 100644 --- a/src/mon/MonCommands.h +++ b/src/mon/MonCommands.h @@ -1231,8 +1231,8 @@ COMMAND("mgr dump " "mgr", "r") COMMAND("mgr fail name=who,type=CephString,req=false", "treat the named manager daemon as failed", "mgr", "rw") -COMMAND("mgr module ls", - "list active mgr modules", "mgr", "r") +COMMAND("mgr module ls name=detail,type=CephChoices,strings=detail,req=false", + "list active mgr modules", "mgr", "r") COMMAND("mgr services", "list service endpoints provided by mgr modules", "mgr", "r") From 05902d943bba4a64abbd943270b56cbdd1650e62 Mon Sep 17 00:00:00 2001 From: Yaarit Hatuka Date: Mon, 4 Oct 2021 18:50:01 +0000 Subject: [PATCH 0077/2866] mgr/devicehealth: fix missing timezone from time delta calculation An error occurs when subtracting a datetime object that is offset-naive (i.e. unaware of timezone) from a datetime object which is offset-aware. datetime.utcnow() is missing timezone info, e.g.: '2021-09-22 13:18:45.021712', while life_expectancy_max is in the format of: '2021-09-28 00:00:00.000000+00:00', hence we need to add timezone info to the former when calculating their time delta. Please note that we calculate time delta using `datetime.utcnow()` in `serve()` in this module, but there we refer to the delta in seconds, which works fine. Fixes: https://tracker.ceph.com/issues/52327 Signed-off-by: Yaarit Hatuka --- src/pybind/mgr/devicehealth/module.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pybind/mgr/devicehealth/module.py b/src/pybind/mgr/devicehealth/module.py index 9812c91a87cc7..9b6c1fe431c50 100644 --- a/src/pybind/mgr/devicehealth/module.py +++ b/src/pybind/mgr/devicehealth/module.py @@ -9,7 +9,7 @@ import rados import re from threading import Event -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from typing import cast, Any, Dict, List, Optional, Sequence, Tuple, TYPE_CHECKING, Union TIME_FORMAT = '%Y%m%d-%H%M%S' @@ -588,7 +588,7 @@ def check_health(self) -> Tuple[int, str, str]: devs = self.get("devices") osds_in = {} osds_out = {} - now = datetime.utcnow() + now = datetime.now(timezone.utc) # e.g. '2021-09-22 13:18:45.021712+00:00' osdmap = self.get("osd_map") assert osdmap is not None for dev in devs['devices']: @@ -602,7 +602,7 @@ def check_health(self) -> Tuple[int, str, str]: continue # life_expectancy_(min/max) is in the format of: # '%Y-%m-%dT%H:%M:%S.%f%z', e.g.: - # '2019-01-20T21:12:12.000000Z' + # '2019-01-20 21:12:12.000000+00:00' life_expectancy_max = datetime.strptime( dev['life_expectancy_max'], '%Y-%m-%dT%H:%M:%S.%f%z') From c4c9f391aaa6983cf4ce0af144b70ecd27e76f45 Mon Sep 17 00:00:00 2001 From: Cory Snyder Date: Wed, 22 Sep 2021 10:14:12 -0400 Subject: [PATCH 0078/2866] rgw: add abstraction for ops log destination and add file logger Adds an OpsLogSink abstraction for ops log destinations. Also implements this abstraction for a file logger since it's easier to use files vs. sockets with containers. Fixes: https://tracker.ceph.com/issues/48752 Signed-off-by: Cory Snyder --- src/common/options/rgw.yaml.in | 13 +++ src/rgw/librgw.cc | 19 ++- src/rgw/rgw_lib.h | 4 +- src/rgw/rgw_log.cc | 203 ++++++++++++++++++++++++++------- src/rgw/rgw_log.h | 69 ++++++++++- src/rgw/rgw_lua_request.cc | 20 ++-- src/rgw/rgw_lua_request.h | 4 +- src/rgw/rgw_main.cc | 16 ++- src/rgw/rgw_process.cc | 4 +- src/rgw/rgw_process.h | 6 +- src/test/fio/fio_librgw.cc | 2 +- 11 files changed, 282 insertions(+), 78 deletions(-) diff --git a/src/common/options/rgw.yaml.in b/src/common/options/rgw.yaml.in index 961d22b9e6d71..d116b05b67f85 100644 --- a/src/common/options/rgw.yaml.in +++ b/src/common/options/rgw.yaml.in @@ -1438,6 +1438,7 @@ options: - rgw_log_object_name - rgw_ops_log_rados - rgw_ops_log_socket_path + - rgw_ops_log_file_path with_legacy: true # enable logging bandwidth usage - name: rgw_enable_usage_log @@ -1478,6 +1479,18 @@ options: - rgw_enable_ops_log - rgw_ops_log_data_backlog with_legacy: true +# path to file where ops log can go +- name: rgw_ops_log_file_path + type: str + level: advanced + desc: File-system path for ops log. + long_desc: Path to file that RGW will log ops logs to. + fmt_desc: The file-system path for writing operations logs. + services: + - rgw + see_also: + - rgw_enable_ops_log + with_legacy: true # max data backlog for ops log - name: rgw_ops_log_data_backlog type: size diff --git a/src/rgw/librgw.cc b/src/rgw/librgw.cc index e22010cb452d5..18c4140e60b31 100644 --- a/src/rgw/librgw.cc +++ b/src/rgw/librgw.cc @@ -340,8 +340,7 @@ namespace rgw { << e.what() << dendl; } if (should_log) { - rgw_log_op(store, nullptr /* !rest */, s, - (op ? op->name() : "unknown"), olog); + rgw_log_op(nullptr /* !rest */, s, (op ? op->name() : "unknown"), olog); } int http_ret = s->err.http_ret; @@ -589,10 +588,20 @@ namespace rgw { // XXX ex-RGWRESTMgr_lib, mgr->set_logging(true) + OpsLogManifold* olog_manifold = new OpsLogManifold(); if (!g_conf()->rgw_ops_log_socket_path.empty()) { - olog = new OpsLogSocket(g_ceph_context, g_conf()->rgw_ops_log_data_backlog); - olog->init(g_conf()->rgw_ops_log_socket_path); + OpsLogSocket* olog_socket = new OpsLogSocket(g_ceph_context, g_conf()->rgw_ops_log_data_backlog); + olog_socket->init(g_conf()->rgw_ops_log_socket_path); + olog_manifold->add_sink(olog_socket); } + OpsLogFile* ops_log_file; + if (!g_conf()->rgw_ops_log_file_path.empty()) { + ops_log_file = new OpsLogFile(g_ceph_context, g_conf()->rgw_ops_log_file_path, g_conf()->rgw_ops_log_data_backlog); + ops_log_file->start(); + olog_manifold->add_sink(ops_log_file); + } + olog_manifold->add_sink(new OpsLogRados(store)); + olog = olog_manifold; int port = 80; RGWProcessEnv env = { store, &rest, olog, port }; @@ -653,7 +662,7 @@ namespace rgw { shutdown_async_signal_handler(); rgw_log_usage_finalize(); - + delete olog; StoreManager::close_storage(store); diff --git a/src/rgw/rgw_lib.h b/src/rgw/rgw_lib.h index 710c76c803158..0c5e9e721685f 100644 --- a/src/rgw/rgw_lib.h +++ b/src/rgw/rgw_lib.h @@ -20,7 +20,7 @@ #define dout_subsys ceph_subsys_rgw -class OpsLogSocket; +class OpsLogSink; namespace rgw { @@ -29,7 +29,7 @@ namespace rgw { class RGWLib : public DoutPrefixProvider { RGWFrontendConfig* fec; RGWLibFrontend* fe; - OpsLogSocket* olog; + OpsLogSink* olog; rgw::LDAPHelper* ldh{nullptr}; RGWREST rest; // XXX needed for RGWProcessEnv rgw::sal::Store* store; diff --git a/src/rgw/rgw_log.cc b/src/rgw/rgw_log.cc index da6eb1e6ad337..aee079daf7f0e 100644 --- a/src/rgw/rgw_log.cc +++ b/src/rgw/rgw_log.cc @@ -17,6 +17,9 @@ #include "services/svc_zone.h" +#include +#include + #define dout_subsys ceph_subsys_rgw using namespace std; @@ -317,32 +320,131 @@ void rgw_format_ops_log_entry(struct rgw_log_entry& entry, Formatter *formatter) formatter->close_section(); } -void OpsLogSocket::formatter_to_bl(bufferlist& bl) +OpsLogManifold::~OpsLogManifold() { - stringstream ss; - formatter->flush(ss); - const string& s = ss.str(); + for (const auto &sink : sinks) { + delete sink; + } +} - bl.append(s); +void OpsLogManifold::add_sink(OpsLogSink* sink) +{ + sinks.push_back(sink); } -void OpsLogSocket::init_connection(bufferlist& bl) +int OpsLogManifold::log(struct req_state* s, struct rgw_log_entry& entry) { - bl.append("["); + int ret = 0; + for (const auto &sink : sinks) { + if (sink->log(s, entry) < 0) { + ret = -1; + } + } + return ret; } -OpsLogSocket::OpsLogSocket(CephContext *cct, uint64_t _backlog) : OutputDataSocket(cct, _backlog) +OpsLogFile::OpsLogFile(CephContext* cct, std::string& path, uint64_t max_data_size) : + cct(cct), file(path, std::ofstream::app), data_size(0), max_data_size(max_data_size) { - formatter = new JSONFormatter; - delim.append(",\n"); } -OpsLogSocket::~OpsLogSocket() +void OpsLogFile::flush() +{ + std::scoped_lock flush_lock(flush_mutex); + { + std::scoped_lock log_lock(log_mutex); + assert(flush_buffer.empty()); + flush_buffer.swap(log_buffer); + data_size = 0; + } + for (auto bl : flush_buffer) { + int try_num = 0; + while (true) { + bl.write_stream(file); + if (!file) { + ldpp_dout(this, 0) << "ERROR: failed to log RGW ops log file entry" << dendl; + file.clear(); + if (stopped) { + break; + } + int sleep_time_secs = std::min((int) pow(2, try_num), 60); + std::this_thread::sleep_for(std::chrono::seconds(sleep_time_secs)); + try_num++; + } else { + break; + } + } + } + flush_buffer.clear(); + file << std::endl; +} + +void* OpsLogFile::entry() { + std::unique_lock lock(log_mutex); + while (!stopped) { + if (!log_buffer.empty()) { + lock.unlock(); + flush(); + lock.lock(); + continue; + } + cond_flush.wait(lock); + } + flush(); + return NULL; +} + +void OpsLogFile::start() { + stopped = false; + create("ops_log_file"); +} + +void OpsLogFile::stop() { + { + cond_flush.notify_one(); + stopped = true; + } + join(); +} + +OpsLogFile::~OpsLogFile() { + if (!stopped) { + stop(); + } + file.close(); +} + +int OpsLogFile::log_json(struct req_state* s, bufferlist& bl) +{ + std::unique_lock lock(log_mutex); + if (data_size + bl.length() >= max_data_size) { + ldout(s->cct, 0) << "ERROR: RGW ops log file buffer too full, dropping log for txn: " << s->trans_id << dendl; + return -1; + } + log_buffer.push_back(bl); + data_size += bl.length(); + cond_flush.notify_all(); + return 0; +} + +JsonOpsLogSink::JsonOpsLogSink() { + formatter = new JSONFormatter; +} + +JsonOpsLogSink::~JsonOpsLogSink() { delete formatter; } -void OpsLogSocket::log(struct rgw_log_entry& entry) +void JsonOpsLogSink::formatter_to_bl(bufferlist& bl) +{ + stringstream ss; + formatter->flush(ss); + const string& s = ss.str(); + bl.append(s); +} + +int JsonOpsLogSink::log(struct req_state* s, struct rgw_log_entry& entry) { bufferlist bl; @@ -351,11 +453,53 @@ void OpsLogSocket::log(struct rgw_log_entry& entry) formatter_to_bl(bl); lock.unlock(); + return log_json(s, bl); +} + +void OpsLogSocket::init_connection(bufferlist& bl) +{ + bl.append("["); +} + +OpsLogSocket::OpsLogSocket(CephContext *cct, uint64_t _backlog) : OutputDataSocket(cct, _backlog) +{ + delim.append(",\n"); +} + +int OpsLogSocket::log_json(struct req_state* s, bufferlist& bl) +{ append_output(bl); + return 0; } -int rgw_log_op(rgw::sal::Store* store, RGWREST* const rest, struct req_state *s, - const string& op_name, OpsLogSocket *olog) +OpsLogRados::OpsLogRados(rgw::sal::Store* store): store(store) +{ +} + +int OpsLogRados::log(struct req_state* s, struct rgw_log_entry& entry) +{ + if (!s->cct->_conf->rgw_ops_log_rados) { + return 0; + } + bufferlist bl; + encode(entry, bl); + + struct tm bdt; + time_t t = req_state::Clock::to_time_t(entry.time); + if (s->cct->_conf->rgw_log_object_name_utc) + gmtime_r(&t, &bdt); + else + localtime_r(&t, &bdt); + string oid = render_log_object_name(s->cct->_conf->rgw_log_object_name, &bdt, + entry.bucket_id, entry.bucket); + if (store->log_op(s, oid, bl) < 0) { + ldpp_dout(s, 0) << "ERROR: failed to log RADOS RGW ops log entry for txn: " << s->trans_id << dendl; + return -1; + } + return 0; +} + +int rgw_log_op(RGWREST* const rest, struct req_state *s, const string& op_name, OpsLogSink *olog) { struct rgw_log_entry entry; string bucket_id; @@ -473,37 +617,14 @@ int rgw_log_op(rgw::sal::Store* store, RGWREST* const rest, struct req_state *s, char buf[16]; snprintf(buf, sizeof(buf), "%d", s->err.http_ret); entry.http_status = buf; - } else + } else { entry.http_status = "200"; // default - + } entry.error_code = s->err.err_code; entry.bucket_id = bucket_id; entry.trans_id = s->trans_id; - - bufferlist bl; - encode(entry, bl); - - struct tm bdt; - time_t t = req_state::Clock::to_time_t(entry.time); - if (s->cct->_conf->rgw_log_object_name_utc) - gmtime_r(&t, &bdt); - else - localtime_r(&t, &bdt); - - int ret = 0; - - if (s->cct->_conf->rgw_ops_log_rados) { - string oid = render_log_object_name(s->cct->_conf->rgw_log_object_name, &bdt, - entry.bucket_id, entry.bucket); - ret = store->log_op(s, oid, bl); - } - if (olog) { - olog->log(entry); + return olog->log(s, entry); } - if (ret < 0) - ldpp_dout(s, 0) << "ERROR: failed to log entry" << dendl; - - return ret; + return 0; } - diff --git a/src/rgw/rgw_log.h b/src/rgw/rgw_log.h index 2469282b33012..5ee411e8cfaec 100644 --- a/src/rgw/rgw_log.h +++ b/src/rgw/rgw_log.h @@ -7,6 +7,10 @@ #include #include "rgw_common.h" #include "common/OutputDataSocket.h" +#include +#include + +#define dout_subsys ceph_subsys_rgw namespace rgw { namespace sal { class Store; @@ -132,26 +136,79 @@ struct rgw_log_entry { }; WRITE_CLASS_ENCODER(rgw_log_entry) -class OpsLogSocket : public OutputDataSocket { +class OpsLogSink { +public: + virtual int log(struct req_state* s, struct rgw_log_entry& entry) = 0; + virtual ~OpsLogSink() = default; +}; + +class OpsLogManifold: public OpsLogSink { + std::vector sinks; +public: + ~OpsLogManifold() override; + void add_sink(OpsLogSink* sink); + int log(struct req_state* s, struct rgw_log_entry& entry) override; +}; + +class JsonOpsLogSink : public OpsLogSink { ceph::Formatter *formatter; - ceph::mutex lock = ceph::make_mutex("OpsLogSocket"); + ceph::mutex lock = ceph::make_mutex("JsonOpsLogSink"); void formatter_to_bl(bufferlist& bl); +protected: + virtual int log_json(struct req_state* s, bufferlist& bl) = 0; +public: + JsonOpsLogSink(); + ~JsonOpsLogSink() override; + int log(struct req_state* s, struct rgw_log_entry& entry) override; +}; + +class OpsLogFile : public JsonOpsLogSink, public Thread, public DoutPrefixProvider { + CephContext* cct; + ceph::mutex log_mutex = ceph::make_mutex("OpsLogFile_log"); + ceph::mutex flush_mutex = ceph::make_mutex("OpsLogFile_flush"); + std::vector log_buffer; + std::vector flush_buffer; + std::condition_variable cond_flush; + std::ofstream file; + bool stopped; + uint64_t data_size; + uint64_t max_data_size; + + void flush(); +protected: + int log_json(struct req_state* s, bufferlist& bl) override; + void *entry() override; +public: + OpsLogFile(CephContext* cct, std::string& path, uint64_t max_data_size); + ~OpsLogFile() override; + CephContext *get_cct() const override { return cct; } + unsigned get_subsys() const override { return dout_subsys; } + std::ostream& gen_prefix(std::ostream& out) const override { return out << "rgw OpsLogFile: "; } + void start(); + void stop(); +}; +class OpsLogSocket : public OutputDataSocket, public JsonOpsLogSink { protected: + int log_json(struct req_state* s, bufferlist& bl) override; void init_connection(bufferlist& bl) override; public: OpsLogSocket(CephContext *cct, uint64_t _backlog); - ~OpsLogSocket() override; +}; - void log(struct rgw_log_entry& entry); +class OpsLogRados : public OpsLogSink { + rgw::sal::Store* store; +public: + OpsLogRados(rgw::sal::Store* store); + int log(struct req_state* s, struct rgw_log_entry& entry) override; }; class RGWREST; -int rgw_log_op(rgw::sal::Store* store, RGWREST* const rest, struct req_state* s, - const std::string& op_name, OpsLogSocket* olog); +int rgw_log_op(RGWREST* const rest, struct req_state* s, + const std::string& op_name, OpsLogSink* olog); void rgw_log_usage_init(CephContext* cct, rgw::sal::Store* store); void rgw_log_usage_finalize(); void rgw_format_ops_log_entry(struct rgw_log_entry& entry, diff --git a/src/rgw/rgw_lua_request.cc b/src/rgw/rgw_lua_request.cc index a8ea4ee19b2c9..7d6d72e0368b7 100644 --- a/src/rgw/rgw_lua_request.cc +++ b/src/rgw/rgw_lua_request.cc @@ -24,16 +24,15 @@ constexpr const char* RequestLogAction{"Log"}; int RequestLog(lua_State* L) { - const auto store = reinterpret_cast(lua_touserdata(L, lua_upvalueindex(1))); - const auto rest = reinterpret_cast(lua_touserdata(L, lua_upvalueindex(2))); - const auto olog = reinterpret_cast(lua_touserdata(L, lua_upvalueindex(3))); - const auto s = reinterpret_cast(lua_touserdata(L, lua_upvalueindex(4))); - const std::string op_name(reinterpret_cast(lua_touserdata(L, lua_upvalueindex(5)))); - if (store && s) { - const auto rc = rgw_log_op(store, rest, s, op_name, olog); + const auto rest = reinterpret_cast(lua_touserdata(L, lua_upvalueindex(1))); + const auto olog = reinterpret_cast(lua_touserdata(L, lua_upvalueindex(2))); + const auto s = reinterpret_cast(lua_touserdata(L, lua_upvalueindex(3))); + const std::string op_name(reinterpret_cast(lua_touserdata(L, lua_upvalueindex(4)))); + if (s) { + const auto rc = rgw_log_op(rest, s, op_name, olog); lua_pushinteger(L, rc); } else { - ldpp_dout(s, 1) << "Lua ERROR: missing rados store, cannot use ops log" << dendl; + ldpp_dout(s, 1) << "Lua ERROR: missing request state, cannot use ops log" << dendl; lua_pushinteger(L, -EINVAL); } @@ -777,7 +776,7 @@ struct RequestMetaTable : public EmptyMetaTable { int execute( rgw::sal::Store* store, RGWREST* rest, - OpsLogSocket* olog, + OpsLogSink* olog, req_state* s, const char* op_name, const std::string& script) @@ -799,12 +798,11 @@ int execute( lua_getglobal(L, RequestMetaTable::TableName().c_str()); ceph_assert(lua_istable(L, -1)); pushstring(L, RequestLogAction); - lua_pushlightuserdata(L, store); lua_pushlightuserdata(L, rest); lua_pushlightuserdata(L, olog); lua_pushlightuserdata(L, s); lua_pushlightuserdata(L, const_cast(op_name)); - lua_pushcclosure(L, RequestLog, FIVE_UPVALS); + lua_pushcclosure(L, RequestLog, FOUR_UPVALS); lua_rawset(L, -3); try { diff --git a/src/rgw/rgw_lua_request.h b/src/rgw/rgw_lua_request.h index d31e81b5a4fd5..fcf0673f3ba7b 100644 --- a/src/rgw/rgw_lua_request.h +++ b/src/rgw/rgw_lua_request.h @@ -5,7 +5,7 @@ class req_state; class RGWREST; -class OpsLogSocket; +class OpsLogSink; namespace rgw::sal { class Store; } @@ -16,7 +16,7 @@ namespace rgw::lua::request { int execute( rgw::sal::Store* store, RGWREST* rest, - OpsLogSocket* olog, + OpsLogSink* olog, req_state *s, const char* op_name, const std::string& script); diff --git a/src/rgw/rgw_main.cc b/src/rgw/rgw_main.cc index c44b0cba61cab..507c07ef2f972 100644 --- a/src/rgw/rgw_main.cc +++ b/src/rgw/rgw_main.cc @@ -518,12 +518,19 @@ int radosgw_Main(int argc, const char **argv) rgw::dmclock::SchedulerCtx sched_ctx{cct.get()}; - OpsLogSocket *olog = NULL; - + OpsLogManifold *olog = new OpsLogManifold(); if (!g_conf()->rgw_ops_log_socket_path.empty()) { - olog = new OpsLogSocket(g_ceph_context, g_conf()->rgw_ops_log_data_backlog); - olog->init(g_conf()->rgw_ops_log_socket_path); + OpsLogSocket* olog_socket = new OpsLogSocket(g_ceph_context, g_conf()->rgw_ops_log_data_backlog); + olog_socket->init(g_conf()->rgw_ops_log_socket_path); + olog->add_sink(olog_socket); + } + OpsLogFile* ops_log_file; + if (!g_conf()->rgw_ops_log_file_path.empty()) { + ops_log_file = new OpsLogFile(g_ceph_context, g_conf()->rgw_ops_log_file_path, g_conf()->rgw_ops_log_data_backlog); + ops_log_file->start(); + olog->add_sink(ops_log_file); } + olog->add_sink(new OpsLogRados(store)); r = signal_fd_init(); if (r < 0) { @@ -671,7 +678,6 @@ int radosgw_Main(int argc, const char **argv) shutdown_async_signal_handler(); rgw_log_usage_finalize(); - delete olog; StoreManager::close_storage(store); diff --git a/src/rgw/rgw_process.cc b/src/rgw/rgw_process.cc index 9cf78fe8809b7..3f8c9c33efe6a 100644 --- a/src/rgw/rgw_process.cc +++ b/src/rgw/rgw_process.cc @@ -189,7 +189,7 @@ int process_request(rgw::sal::Store* const store, const std::string& frontend_prefix, const rgw_auth_registry_t& auth_registry, RGWRestfulIO* const client_io, - OpsLogSocket* const olog, + OpsLogSink* const olog, optional_yield yield, rgw::dmclock::Scheduler *scheduler, string* user, @@ -356,7 +356,7 @@ int process_request(rgw::sal::Store* const store, } if (should_log) { - rgw_log_op(store, rest, s, (op ? op->name() : "unknown"), olog); + rgw_log_op(rest, s, (op ? op->name() : "unknown"), olog); } if (http_ret != nullptr) { diff --git a/src/rgw/rgw_process.h b/src/rgw/rgw_process.h index f8a97d16c29d4..1aac4a6cdf1bc 100644 --- a/src/rgw/rgw_process.h +++ b/src/rgw/rgw_process.h @@ -34,7 +34,7 @@ namespace rgw::dmclock { struct RGWProcessEnv { rgw::sal::Store* store; RGWREST *rest; - OpsLogSocket *olog; + OpsLogSink *olog; int port; std::string uri_prefix; std::shared_ptr auth_registry; @@ -49,7 +49,7 @@ class RGWProcess { CephContext *cct; rgw::sal::Store* store; rgw_auth_registry_ptr_t auth_registry; - OpsLogSocket* olog; + OpsLogSink* olog; ThreadPool m_tp; Throttle req_throttle; RGWREST* rest; @@ -169,7 +169,7 @@ extern int process_request(rgw::sal::Store* store, const std::string& frontend_prefix, const rgw_auth_registry_t& auth_registry, RGWRestfulIO* client_io, - OpsLogSocket* olog, + OpsLogSink* olog, optional_yield y, rgw::dmclock::Scheduler *scheduler, std::string* user, diff --git a/src/test/fio/fio_librgw.cc b/src/test/fio/fio_librgw.cc index 0def8aac7240b..b088b68f96c5c 100644 --- a/src/test/fio/fio_librgw.cc +++ b/src/test/fio/fio_librgw.cc @@ -380,7 +380,7 @@ namespace { { librgw_data* data = static_cast(td->io_ops_data); const char* object = io_u->file->file_name; - struct rgw_file_handle* object_fh; + struct rgw_file_handle* object_fh = nullptr; size_t nbytes; int r = 0; From f929556bdd756c9258e6283a29d85b82395296cc Mon Sep 17 00:00:00 2001 From: Ronen Friedman Date: Tue, 5 Oct 2021 12:27:44 +0000 Subject: [PATCH 0079/2866] common/ceph_time: fix wrong seconds output in exact_timespan_str() For example: 170.567s fixed from "2m500.567s" to "2m50.567s" Fixes: https://tracker.ceph.com/issues/52815 Signed-off-by: Ronen Friedman --- src/common/ceph_time.cc | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/src/common/ceph_time.cc b/src/common/ceph_time.cc index 1b8c3ae08a5e9..4af48a8a2a2fe 100644 --- a/src/common/ceph_time.cc +++ b/src/common/ceph_time.cc @@ -149,11 +149,11 @@ std::string timespan_str(timespan t) // that isn't as lame as this one! uint64_t nsec = std::chrono::nanoseconds(t).count(); std::ostringstream ss; - if (nsec < 2000000000) { - ss << ((float)nsec / 1000000000) << "s"; + if (nsec < 2'000'000'000) { + ss << ((float)nsec / 1'000'000'000) << "s"; return ss.str(); } - uint64_t sec = nsec / 1000000000; + uint64_t sec = nsec / 1'000'000'000; if (sec < 120) { ss << sec << "s"; return ss.str(); @@ -191,8 +191,8 @@ std::string timespan_str(timespan t) std::string exact_timespan_str(timespan t) { uint64_t nsec = std::chrono::nanoseconds(t).count(); - uint64_t sec = nsec / 1000000000; - nsec %= 1000000000; + uint64_t sec = nsec / 1'000'000'000; + nsec %= 1'000'000'000; uint64_t yr = sec / (60 * 60 * 24 * 365); std::ostringstream ss; if (yr) { @@ -224,14 +224,12 @@ std::string exact_timespan_str(timespan t) ss << min << "m"; sec -= min * 60; } - if (sec) { - ss << sec; - } - if (nsec) { - ss << ((float)nsec / 1000000000); - } if (sec || nsec) { - ss << "s"; + if (nsec) { + ss << (((float)nsec / 1'000'000'000) + sec) << "s"; + } else { + ss << sec << "s"; + } } return ss.str(); } From 4affb5c7029f6b83d640aa7b7206d9cf61e75f1d Mon Sep 17 00:00:00 2001 From: Patrick Donnelly Date: Thu, 23 Sep 2021 19:49:31 -0400 Subject: [PATCH 0080/2866] pybind/mgr/cephadm: always do mds upgrade sequence Minor versions also require this sequence. Signed-off-by: Patrick Donnelly --- src/pybind/mgr/cephadm/upgrade.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/src/pybind/mgr/cephadm/upgrade.py b/src/pybind/mgr/cephadm/upgrade.py index d7162941985d3..e9148a595f7eb 100644 --- a/src/pybind/mgr/cephadm/upgrade.py +++ b/src/pybind/mgr/cephadm/upgrade.py @@ -356,24 +356,6 @@ def _prepare_for_mds_upgrade( target_major: str, need_upgrade: List[DaemonDescription] ) -> bool: - # are any daemons running a different major version? - scale_down = False - for name, info in self.mgr.get("mds_metadata").items(): - version = info.get("ceph_version_short") - major_version = None - if version: - major_version = version.split('.')[0] - if not major_version: - self.mgr.log.info('Upgrade: mds.%s version is not known, will retry' % name) - time.sleep(5) - return False - if int(major_version) < int(target_major): - scale_down = True - - if not scale_down: - self.mgr.log.debug('Upgrade: All MDS daemons run same major version') - return True - # scale down all filesystems to 1 MDS assert self.upgrade_state if not self.upgrade_state.fs_original_max_mds: From bca21f01ce3bb32e0951f0fe15da88a81750a191 Mon Sep 17 00:00:00 2001 From: Patrick Donnelly Date: Fri, 17 Sep 2021 20:15:01 -0400 Subject: [PATCH 0081/2866] pybind/mgr/cephadm: disable allow_standby_replay during CephFS upgrade Following procedure in [1]. Also: harden checks for active. Ensure "up" and "in" are both [0]. There should be no standby-replay daemon. [1] https://docs.ceph.com/en/pacific/cephfs/upgrading/ Fixes: https://tracker.ceph.com/issues/52654 Signed-off-by: Patrick Donnelly --- src/pybind/mgr/cephadm/upgrade.py | 110 +++++++++++++++++++++--------- 1 file changed, 78 insertions(+), 32 deletions(-) diff --git a/src/pybind/mgr/cephadm/upgrade.py b/src/pybind/mgr/cephadm/upgrade.py index e9148a595f7eb..19f377bbb456c 100644 --- a/src/pybind/mgr/cephadm/upgrade.py +++ b/src/pybind/mgr/cephadm/upgrade.py @@ -16,6 +16,9 @@ logger = logging.getLogger(__name__) +# from ceph_fs.h +CEPH_MDSMAP_ALLOW_STANDBY_REPLAY = (1 << 5) + def normalize_image_digest(digest: str, default_registry: str) -> str: # normal case: @@ -41,6 +44,7 @@ def __init__(self, error: Optional[str] = None, paused: Optional[bool] = None, fs_original_max_mds: Optional[Dict[str, int]] = None, + fs_original_allow_standby_replay: Optional[Dict[str, bool]] = None ): self._target_name: str = target_name # Use CephadmUpgrade.target_image instead. self.progress_id: str = progress_id @@ -50,6 +54,7 @@ def __init__(self, self.error: Optional[str] = error self.paused: bool = paused or False self.fs_original_max_mds: Optional[Dict[str, int]] = fs_original_max_mds + self.fs_original_allow_standby_replay: Optional[Dict[str, bool]] = fs_original_allow_standby_replay def to_json(self) -> dict: return { @@ -59,6 +64,7 @@ def to_json(self) -> dict: 'target_digests': self.target_digests, 'target_version': self.target_version, 'fs_original_max_mds': self.fs_original_max_mds, + 'fs_original_allow_standby_replay': self.fs_original_allow_standby_replay, 'error': self.error, 'paused': self.paused, } @@ -360,20 +366,39 @@ def _prepare_for_mds_upgrade( assert self.upgrade_state if not self.upgrade_state.fs_original_max_mds: self.upgrade_state.fs_original_max_mds = {} + if not self.upgrade_state.fs_original_allow_standby_replay: + self.upgrade_state.fs_original_allow_standby_replay = {} fsmap = self.mgr.get("fs_map") continue_upgrade = True - for i in fsmap.get('filesystems', []): - fs = i["mdsmap"] - fs_id = i["id"] - fs_name = fs["fs_name"] + for fs in fsmap.get('filesystems', []): + fscid = fs["id"] + mdsmap = fs["mdsmap"] + fs_name = mdsmap["fs_name"] + + # disable allow_standby_replay? + if mdsmap['flags'] & CEPH_MDSMAP_ALLOW_STANDBY_REPLAY: + self.mgr.log.info('Upgrade: Disabling standby-replay for filesystem %s' % ( + fs_name + )) + if fscid not in self.upgrade_state.fs_original_allow_standby_replay: + self.upgrade_state.fs_original_allow_standby_replay[fscid] = True + self._save_upgrade_state() + ret, out, err = self.mgr.check_mon_command({ + 'prefix': 'fs set', + 'fs_name': fs_name, + 'var': 'allow_standby_replay', + 'val': '0', + }) + continue_upgrade = False + continue # scale down this filesystem? - if fs["max_mds"] > 1: + if mdsmap["max_mds"] > 1: self.mgr.log.info('Upgrade: Scaling down filesystem %s' % ( fs_name )) - if fs_id not in self.upgrade_state.fs_original_max_mds: - self.upgrade_state.fs_original_max_mds[fs_id] = fs['max_mds'] + if fscid not in self.upgrade_state.fs_original_max_mds: + self.upgrade_state.fs_original_max_mds[fscid] = mdsmap['max_mds'] self._save_upgrade_state() ret, out, err = self.mgr.check_mon_command({ 'prefix': 'fs set', @@ -384,13 +409,15 @@ def _prepare_for_mds_upgrade( continue_upgrade = False continue - if len(fs['info']) > 1: - self.mgr.log.info('Upgrade: Waiting for fs %s to scale down to 1 MDS' % (fs_name)) + if not (mdsmap['in'] == [0] and len(mdsmap['up']) == 1): + self.mgr.log.info('Upgrade: Waiting for fs %s to scale down to reach 1 MDS' % (fs_name)) time.sleep(10) continue_upgrade = False continue - lone_mds = list(fs['info'].values())[0] + mdss = list(mdsmap['info'].values()) + assert len(mdss) == 1 + lone_mds = mdss[0] if lone_mds['state'] != 'up:active': self.mgr.log.info('Upgrade: Waiting for mds.%s to be up:active (currently %s)' % ( lone_mds['name'], @@ -420,9 +447,9 @@ def _enough_mds_for_ok_to_stop(self, mds_daemon: DaemonDescription) -> bool: # find fs this mds daemon belongs to fsmap = self.mgr.get("fs_map") - for i in fsmap.get('filesystems', []): - fs = i["mdsmap"] - fs_name = fs["fs_name"] + for fs in fsmap.get('filesystems', []): + mdsmap = fs["mdsmap"] + fs_name = mdsmap["fs_name"] assert mds_daemon.daemon_id if fs_name != mds_daemon.service_name().split('.', 1)[1]: @@ -434,7 +461,7 @@ def _enough_mds_for_ok_to_stop(self, mds_daemon: DaemonDescription) -> bool: [daemon for daemon in self.mgr.cache.get_daemons_by_service(mds_daemon.service_name())]) # standby mds daemons for this fs? - if fs["max_mds"] < mds_count: + if mdsmap["max_mds"] < mds_count: return True return False @@ -761,24 +788,43 @@ def _do_upgrade(self): }) # complete mds upgrade? - if daemon_type == 'mds' and self.upgrade_state.fs_original_max_mds: - for i in self.mgr.get("fs_map")['filesystems']: - fs_id = i["id"] - fs_name = i['mdsmap']['fs_name'] - new_max = self.upgrade_state.fs_original_max_mds.get(fs_id) - if new_max: - self.mgr.log.info('Upgrade: Scaling up filesystem %s max_mds to %d' % ( - fs_name, new_max - )) - ret, _, err = self.mgr.check_mon_command({ - 'prefix': 'fs set', - 'fs_name': fs_name, - 'var': 'max_mds', - 'val': str(new_max), - }) - - self.upgrade_state.fs_original_max_mds = {} - self._save_upgrade_state() + if daemon_type == 'mds': + if self.upgrade_state.fs_original_max_mds: + for fs in self.mgr.get("fs_map")['filesystems']: + fscid = fs["id"] + fs_name = fs['mdsmap']['fs_name'] + new_max = self.upgrade_state.fs_original_max_mds.get(fscid, 1) + if new_max > 1: + self.mgr.log.info('Upgrade: Scaling up filesystem %s max_mds to %d' % ( + fs_name, new_max + )) + ret, _, err = self.mgr.check_mon_command({ + 'prefix': 'fs set', + 'fs_name': fs_name, + 'var': 'max_mds', + 'val': str(new_max), + }) + + self.upgrade_state.fs_original_max_mds = {} + self._save_upgrade_state() + if self.upgrade_state.fs_original_allow_standby_replay: + for fs in self.mgr.get("fs_map")['filesystems']: + fscid = fs["id"] + fs_name = fs['mdsmap']['fs_name'] + asr = self.upgrade_state.fs_original_allow_standby_replay.get(fscid, False) + if asr: + self.mgr.log.info('Upgrade: Enabling allow_standby_replay on filesystem %s' % ( + fs_name + )) + ret, _, err = self.mgr.check_mon_command({ + 'prefix': 'fs set', + 'fs_name': fs_name, + 'var': 'allow_standby_replay', + 'val': '1' + }) + + self.upgrade_state.fs_original_allow_standby_replay = {} + self._save_upgrade_state() # clean up logger.info('Upgrade: Finalizing container_image settings') From 7812cfb6744fc3bce50e26aa7dd6a4e47a43bb23 Mon Sep 17 00:00:00 2001 From: Patrick Donnelly Date: Tue, 5 Oct 2021 13:31:02 -0400 Subject: [PATCH 0082/2866] qa: move CephManager cluster instantiation to subtask This needs to be available for the cephfs_setup task so administration mounts can run ceph commands, potentially through `cephadm shell`. Signed-off-by: Patrick Donnelly --- qa/tasks/ceph.py | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/qa/tasks/ceph.py b/qa/tasks/ceph.py index 87a0f84eb9337..db86d6d53cc46 100644 --- a/qa/tasks/ceph.py +++ b/qa/tasks/ceph.py @@ -376,6 +376,20 @@ def crush_setup(ctx, config): yield +@contextlib.contextmanager +def setup_manager(ctx, config): + first_mon = teuthology.get_first_mon(ctx, config, config['cluster']) + (mon,) = ctx.cluster.only(first_mon).remotes.keys() + if not hasattr(ctx, 'managers'): + ctx.managers = {} + ctx.managers[config['cluster']] = CephManager( + mon, + ctx=ctx, + logger=log.getChild('ceph_manager.' + config['cluster']), + cluster=config['cluster'], + ) + yield + @contextlib.contextmanager def create_rbd_pool(ctx, config): cluster_name = config['cluster'] @@ -1866,6 +1880,7 @@ def task(ctx, config): lambda: run_daemon(ctx=ctx, config=config, type_='mgr'), lambda: crush_setup(ctx=ctx, config=config), lambda: run_daemon(ctx=ctx, config=config, type_='osd'), + lambda: setup_manager(ctx=ctx, config=config), lambda: create_rbd_pool(ctx=ctx, config=config), lambda: run_daemon(ctx=ctx, config=config, type_='mds'), lambda: cephfs_setup(ctx=ctx, config=config), @@ -1873,17 +1888,6 @@ def task(ctx, config): ] with contextutil.nested(*subtasks): - first_mon = teuthology.get_first_mon(ctx, config, config['cluster']) - (mon,) = ctx.cluster.only(first_mon).remotes.keys() - if not hasattr(ctx, 'managers'): - ctx.managers = {} - ctx.managers[config['cluster']] = CephManager( - mon, - ctx=ctx, - logger=log.getChild('ceph_manager.' + config['cluster']), - cluster=config['cluster'], - ) - try: if config.get('wait-for-healthy', True): healthy(ctx=ctx, config=dict(cluster=config['cluster'])) From 24bb450d39761ed526a6b6eb285c8c528ab90b5a Mon Sep 17 00:00:00 2001 From: Patrick Donnelly Date: Fri, 1 Oct 2021 12:02:14 -0400 Subject: [PATCH 0083/2866] qa: use ctx's ceph_manager to run ceph commands by mount This allows hooks for `cephadm shell` to function so that this code works with cephadm deployments. Signed-off-by: Patrick Donnelly --- qa/tasks/cephfs/mount.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/qa/tasks/cephfs/mount.py b/qa/tasks/cephfs/mount.py index 6224a7eae7b08..0f9c9397125a6 100644 --- a/qa/tasks/cephfs/mount.py +++ b/qa/tasks/cephfs/mount.py @@ -56,6 +56,8 @@ def __init__(self, ctx, test_dir, client_id, client_remote, self.cephfs_name = cephfs_name self.cephfs_mntpt = cephfs_mntpt + self.cluster_name = 'ceph' # TODO: use config['cluster'] + self.fs = None self._netns_name = None @@ -124,8 +126,7 @@ def netns_name(self, name): self._netns_name = name def assert_that_ceph_fs_exists(self): - output = self.client_remote.run(args='ceph fs ls', stdout=StringIO()).\ - stdout.getvalue() + output = self.ctx.managers[self.cluster_name].raw_cluster_cmd("fs", "ls") if self.cephfs_name: assert self.cephfs_name in output, \ 'expected ceph fs is not present on the cluster' From dbe5573ed4781cb4b214e701c77be7bc2cddabf3 Mon Sep 17 00:00:00 2001 From: Patrick Donnelly Date: Fri, 1 Oct 2021 12:05:12 -0400 Subject: [PATCH 0084/2866] qa: add note about where caps are generated Signed-off-by: Patrick Donnelly --- qa/tasks/cephfs/mount.py | 1 + 1 file changed, 1 insertion(+) diff --git a/qa/tasks/cephfs/mount.py b/qa/tasks/cephfs/mount.py index 0f9c9397125a6..02deb6f8fe04b 100644 --- a/qa/tasks/cephfs/mount.py +++ b/qa/tasks/cephfs/mount.py @@ -566,6 +566,7 @@ def wait_until_mounted(self): raise NotImplementedError() def get_keyring_path(self): + # N.B.: default keyring is /etc/ceph/ceph.keyring; see ceph.py and generate_caps return '/etc/ceph/ceph.client.{id}.keyring'.format(id=self.client_id) def get_key_from_keyfile(self): From 5a7382214fe4dbd4b79773c6e732512ade22793a Mon Sep 17 00:00:00 2001 From: Patrick Donnelly Date: Fri, 1 Oct 2021 12:05:42 -0400 Subject: [PATCH 0085/2866] qa: add tasks to check mds upgrade state Signed-off-by: Patrick Donnelly --- qa/tasks/fs.py | 68 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/qa/tasks/fs.py b/qa/tasks/fs.py index 71d7b22e81462..f7a9330e29b50 100644 --- a/qa/tasks/fs.py +++ b/qa/tasks/fs.py @@ -9,6 +9,74 @@ log = logging.getLogger(__name__) +# Everything up to CEPH_MDSMAP_ALLOW_STANDBY_REPLAY +CEPH_MDSMAP_ALLOW_STANDBY_REPLAY = (1<<5) +CEPH_MDSMAP_LAST = CEPH_MDSMAP_ALLOW_STANDBY_REPLAY +UPGRADE_FLAGS_MASK = ((CEPH_MDSMAP_LAST<<1) - 1) +def pre_upgrade_save(ctx, config): + """ + That the upgrade procedure doesn't clobber state: save state. + """ + + mdsc = MDSCluster(ctx) + status = mdsc.status() + + state = {} + ctx['mds-upgrade-state'] = state + + for fs in list(status.get_filesystems()): + fscid = fs['id'] + mdsmap = fs['mdsmap'] + fs_state = {} + fs_state['epoch'] = mdsmap['epoch'] + fs_state['max_mds'] = mdsmap['max_mds'] + fs_state['flags'] = mdsmap['flags'] & UPGRADE_FLAGS_MASK + state[fscid] = fs_state + log.debug(f"fs fscid={fscid},name={mdsmap['fs_name']} state = {fs_state}") + + +def post_upgrade_checks(ctx, config): + """ + That the upgrade procedure doesn't clobber state. + """ + + state = ctx['mds-upgrade-state'] + + mdsc = MDSCluster(ctx) + status = mdsc.status() + + for fs in list(status.get_filesystems()): + fscid = fs['id'] + mdsmap = fs['mdsmap'] + fs_state = state[fscid] + log.debug(f"checking fs fscid={fscid},name={mdsmap['fs_name']} state = {fs_state}") + + # check state was restored to previous values + assert fs_state['max_mds'] == mdsmap['max_mds'] + assert fs_state['flags'] == (mdsmap['flags'] & UPGRADE_FLAGS_MASK) + + # now confirm that the upgrade procedure was followed + epoch = mdsmap['epoch'] + pre_upgrade_epoch = fs_state['epoch'] + assert pre_upgrade_epoch < epoch + should_decrease_max_mds = fs_state['max_mds'] > 1 + did_decrease_max_mds = False + should_disable_allow_standby_replay = fs_state['flags'] & CEPH_MDSMAP_ALLOW_STANDBY_REPLAY + did_disable_allow_standby_replay = False + for i in range(pre_upgrade_epoch+1, mdsmap['epoch']): + old_status = mdsc.status(epoch=i) + old_fs = old_status.get_fsmap(fscid) + old_mdsmap = old_fs['mdsmap'] + if should_decrease_max_mds and old_mdsmap['max_mds'] == 1: + log.debug(f"max_mds reduced in epoch {i}") + did_decrease_max_mds = True + if should_disable_allow_standby_replay and not (old_mdsmap['flags'] & CEPH_MDSMAP_ALLOW_STANDBY_REPLAY): + log.debug(f"allow_standby_replay disabled in epoch {i}") + did_disable_allow_standby_replay = True + assert not should_decrease_max_mds or did_decrease_max_mds + assert not should_disable_allow_standby_replay or did_disable_allow_standby_replay + + def ready(ctx, config): """ That the file system is ready for clients. From b1420e5771927f5c659e0e5edbc5714035f3df09 Mon Sep 17 00:00:00 2001 From: Patrick Donnelly Date: Fri, 1 Oct 2021 12:06:50 -0400 Subject: [PATCH 0086/2866] qa: add test for cephfs upgrade sequence This also checks max_mds>1 and allow_standby_replay are restored to previous values. Future work can add tests for multiple file systems (or volumes). Signed-off-by: Patrick Donnelly --- qa/suites/fs/upgrade/mds_upgrade_sequence/% | 0 qa/suites/fs/upgrade/mds_upgrade_sequence/.qa | 1 + .../bluestore-bitmap.yaml | 1 + .../centos_8.3_container_tools_3.0.yaml | 1 + .../fs/upgrade/mds_upgrade_sequence/conf | 1 + .../upgrade/mds_upgrade_sequence/overrides/% | 0 .../mds_upgrade_sequence/overrides/.qa | 1 + .../overrides/pg-warn.yaml | 5 +++ .../overrides/whitelist_health.yaml | 1 + .../whitelist_wrongly_marked_down.yaml | 1 + .../upgrade/mds_upgrade_sequence/roles.yaml | 11 +++++++ .../fs/upgrade/mds_upgrade_sequence/tasks/% | 0 .../fs/upgrade/mds_upgrade_sequence/tasks/.qa | 1 + .../mds_upgrade_sequence/tasks/0-v16.2.4.yaml | 25 ++++++++++++++ .../mds_upgrade_sequence/tasks/1-volume/% | 0 .../mds_upgrade_sequence/tasks/1-volume/.qa | 1 + .../tasks/1-volume/0-create.yaml | 5 +++ .../tasks/1-volume/1-ranks/.qa | 1 + .../tasks/1-volume/1-ranks/1.yaml | 4 +++ .../tasks/1-volume/1-ranks/2.yaml | 4 +++ .../tasks/1-volume/2-allow_standby_replay/.qa | 1 + .../1-volume/2-allow_standby_replay/no.yaml | 4 +++ .../1-volume/2-allow_standby_replay/yes.yaml | 4 +++ .../tasks/1-volume/3-verify.yaml | 7 ++++ .../mds_upgrade_sequence/tasks/2-client.yaml | 3 ++ .../tasks/3-upgrade-with-workload.yaml | 33 +++++++++++++++++++ .../mds_upgrade_sequence/tasks/4-verify.yaml | 5 +++ qa/suites/orch/cephadm/mds_upgrade_sequence | 1 + 28 files changed, 122 insertions(+) create mode 100644 qa/suites/fs/upgrade/mds_upgrade_sequence/% create mode 120000 qa/suites/fs/upgrade/mds_upgrade_sequence/.qa create mode 120000 qa/suites/fs/upgrade/mds_upgrade_sequence/bluestore-bitmap.yaml create mode 120000 qa/suites/fs/upgrade/mds_upgrade_sequence/centos_8.3_container_tools_3.0.yaml create mode 120000 qa/suites/fs/upgrade/mds_upgrade_sequence/conf create mode 100644 qa/suites/fs/upgrade/mds_upgrade_sequence/overrides/% create mode 120000 qa/suites/fs/upgrade/mds_upgrade_sequence/overrides/.qa create mode 100644 qa/suites/fs/upgrade/mds_upgrade_sequence/overrides/pg-warn.yaml create mode 120000 qa/suites/fs/upgrade/mds_upgrade_sequence/overrides/whitelist_health.yaml create mode 120000 qa/suites/fs/upgrade/mds_upgrade_sequence/overrides/whitelist_wrongly_marked_down.yaml create mode 100644 qa/suites/fs/upgrade/mds_upgrade_sequence/roles.yaml create mode 100644 qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/% create mode 120000 qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/.qa create mode 100644 qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/0-v16.2.4.yaml create mode 100644 qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/1-volume/% create mode 120000 qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/1-volume/.qa create mode 100644 qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/1-volume/0-create.yaml create mode 120000 qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/1-volume/1-ranks/.qa create mode 100644 qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/1-volume/1-ranks/1.yaml create mode 100644 qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/1-volume/1-ranks/2.yaml create mode 120000 qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/1-volume/2-allow_standby_replay/.qa create mode 100644 qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/1-volume/2-allow_standby_replay/no.yaml create mode 100644 qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/1-volume/2-allow_standby_replay/yes.yaml create mode 100644 qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/1-volume/3-verify.yaml create mode 100644 qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/2-client.yaml create mode 100644 qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/3-upgrade-with-workload.yaml create mode 100644 qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/4-verify.yaml create mode 120000 qa/suites/orch/cephadm/mds_upgrade_sequence diff --git a/qa/suites/fs/upgrade/mds_upgrade_sequence/% b/qa/suites/fs/upgrade/mds_upgrade_sequence/% new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/qa/suites/fs/upgrade/mds_upgrade_sequence/.qa b/qa/suites/fs/upgrade/mds_upgrade_sequence/.qa new file mode 120000 index 0000000000000..a602a0353e751 --- /dev/null +++ b/qa/suites/fs/upgrade/mds_upgrade_sequence/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/qa/suites/fs/upgrade/mds_upgrade_sequence/bluestore-bitmap.yaml b/qa/suites/fs/upgrade/mds_upgrade_sequence/bluestore-bitmap.yaml new file mode 120000 index 0000000000000..fb603bc9a64c8 --- /dev/null +++ b/qa/suites/fs/upgrade/mds_upgrade_sequence/bluestore-bitmap.yaml @@ -0,0 +1 @@ +.qa/cephfs/objectstore-ec/bluestore-bitmap.yaml \ No newline at end of file diff --git a/qa/suites/fs/upgrade/mds_upgrade_sequence/centos_8.3_container_tools_3.0.yaml b/qa/suites/fs/upgrade/mds_upgrade_sequence/centos_8.3_container_tools_3.0.yaml new file mode 120000 index 0000000000000..479a5c26e451b --- /dev/null +++ b/qa/suites/fs/upgrade/mds_upgrade_sequence/centos_8.3_container_tools_3.0.yaml @@ -0,0 +1 @@ +.qa/distros/podman/centos_8.3_container_tools_3.0.yaml \ No newline at end of file diff --git a/qa/suites/fs/upgrade/mds_upgrade_sequence/conf b/qa/suites/fs/upgrade/mds_upgrade_sequence/conf new file mode 120000 index 0000000000000..6d47129847fad --- /dev/null +++ b/qa/suites/fs/upgrade/mds_upgrade_sequence/conf @@ -0,0 +1 @@ +.qa/cephfs/conf/ \ No newline at end of file diff --git a/qa/suites/fs/upgrade/mds_upgrade_sequence/overrides/% b/qa/suites/fs/upgrade/mds_upgrade_sequence/overrides/% new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/qa/suites/fs/upgrade/mds_upgrade_sequence/overrides/.qa b/qa/suites/fs/upgrade/mds_upgrade_sequence/overrides/.qa new file mode 120000 index 0000000000000..a602a0353e751 --- /dev/null +++ b/qa/suites/fs/upgrade/mds_upgrade_sequence/overrides/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/qa/suites/fs/upgrade/mds_upgrade_sequence/overrides/pg-warn.yaml b/qa/suites/fs/upgrade/mds_upgrade_sequence/overrides/pg-warn.yaml new file mode 100644 index 0000000000000..4ae54a40d3195 --- /dev/null +++ b/qa/suites/fs/upgrade/mds_upgrade_sequence/overrides/pg-warn.yaml @@ -0,0 +1,5 @@ +overrides: + ceph: + conf: + global: + mon pg warn min per osd: 0 diff --git a/qa/suites/fs/upgrade/mds_upgrade_sequence/overrides/whitelist_health.yaml b/qa/suites/fs/upgrade/mds_upgrade_sequence/overrides/whitelist_health.yaml new file mode 120000 index 0000000000000..74f39a49b27ed --- /dev/null +++ b/qa/suites/fs/upgrade/mds_upgrade_sequence/overrides/whitelist_health.yaml @@ -0,0 +1 @@ +.qa/cephfs/overrides/whitelist_health.yaml \ No newline at end of file diff --git a/qa/suites/fs/upgrade/mds_upgrade_sequence/overrides/whitelist_wrongly_marked_down.yaml b/qa/suites/fs/upgrade/mds_upgrade_sequence/overrides/whitelist_wrongly_marked_down.yaml new file mode 120000 index 0000000000000..b4528c0f8c096 --- /dev/null +++ b/qa/suites/fs/upgrade/mds_upgrade_sequence/overrides/whitelist_wrongly_marked_down.yaml @@ -0,0 +1 @@ +.qa/cephfs/overrides/whitelist_wrongly_marked_down.yaml \ No newline at end of file diff --git a/qa/suites/fs/upgrade/mds_upgrade_sequence/roles.yaml b/qa/suites/fs/upgrade/mds_upgrade_sequence/roles.yaml new file mode 100644 index 0000000000000..bce4ecd34ccb3 --- /dev/null +++ b/qa/suites/fs/upgrade/mds_upgrade_sequence/roles.yaml @@ -0,0 +1,11 @@ +roles: +- - host.a + - client.0 + - osd.0 + - osd.1 + - osd.2 +- - host.b + - client.1 + - osd.3 + - osd.4 + - osd.5 diff --git a/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/% b/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/% new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/.qa b/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/.qa new file mode 120000 index 0000000000000..a602a0353e751 --- /dev/null +++ b/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/0-v16.2.4.yaml b/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/0-v16.2.4.yaml new file mode 100644 index 0000000000000..e61d622669715 --- /dev/null +++ b/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/0-v16.2.4.yaml @@ -0,0 +1,25 @@ +meta: +- desc: | + setup ceph/pacific v16.2.4 + +tasks: +- install: + tag: v16.2.4 + exclude_packages: + - ceph-volume +- cephadm: + roleless: true + image: docker.io/ceph/ceph:v16.2.4 + cephadm_branch: v16.2.4 + cephadm_git_url: https://github.com/ceph/ceph + # needed for v16.2.4 due to --skip-admin-label + avoid_pacific_features: true +- print: "**** done starting v16.2.4" +- cephadm.shell: + host.a: + - ceph orch status + - ceph orch ps + - ceph orch ls + - ceph orch host ls + - ceph orch device ls + diff --git a/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/1-volume/% b/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/1-volume/% new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/1-volume/.qa b/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/1-volume/.qa new file mode 120000 index 0000000000000..a602a0353e751 --- /dev/null +++ b/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/1-volume/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/1-volume/0-create.yaml b/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/1-volume/0-create.yaml new file mode 100644 index 0000000000000..5ee0022c6bd6e --- /dev/null +++ b/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/1-volume/0-create.yaml @@ -0,0 +1,5 @@ +tasks: +- cephadm.shell: + host.a: + - ceph fs volume create cephfs --placement=4 + - ceph fs dump diff --git a/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/1-volume/1-ranks/.qa b/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/1-volume/1-ranks/.qa new file mode 120000 index 0000000000000..a602a0353e751 --- /dev/null +++ b/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/1-volume/1-ranks/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/1-volume/1-ranks/1.yaml b/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/1-volume/1-ranks/1.yaml new file mode 100644 index 0000000000000..8c1cd2fe0a1cd --- /dev/null +++ b/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/1-volume/1-ranks/1.yaml @@ -0,0 +1,4 @@ +tasks: +- cephadm.shell: + host.a: + - ceph fs set cephfs max_mds 1 diff --git a/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/1-volume/1-ranks/2.yaml b/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/1-volume/1-ranks/2.yaml new file mode 100644 index 0000000000000..fcd3b1ea42ea7 --- /dev/null +++ b/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/1-volume/1-ranks/2.yaml @@ -0,0 +1,4 @@ +tasks: +- cephadm.shell: + host.a: + - ceph fs set cephfs max_mds 2 diff --git a/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/1-volume/2-allow_standby_replay/.qa b/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/1-volume/2-allow_standby_replay/.qa new file mode 120000 index 0000000000000..a602a0353e751 --- /dev/null +++ b/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/1-volume/2-allow_standby_replay/.qa @@ -0,0 +1 @@ +../.qa/ \ No newline at end of file diff --git a/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/1-volume/2-allow_standby_replay/no.yaml b/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/1-volume/2-allow_standby_replay/no.yaml new file mode 100644 index 0000000000000..3dbc810899465 --- /dev/null +++ b/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/1-volume/2-allow_standby_replay/no.yaml @@ -0,0 +1,4 @@ +tasks: +- cephadm.shell: + host.a: + - ceph fs set cephfs allow_standby_replay false diff --git a/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/1-volume/2-allow_standby_replay/yes.yaml b/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/1-volume/2-allow_standby_replay/yes.yaml new file mode 100644 index 0000000000000..fb894425e990c --- /dev/null +++ b/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/1-volume/2-allow_standby_replay/yes.yaml @@ -0,0 +1,4 @@ +tasks: +- cephadm.shell: + host.a: + - ceph fs set cephfs allow_standby_replay true diff --git a/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/1-volume/3-verify.yaml b/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/1-volume/3-verify.yaml new file mode 100644 index 0000000000000..ec2a2a54f29de --- /dev/null +++ b/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/1-volume/3-verify.yaml @@ -0,0 +1,7 @@ +tasks: +- cephadm.shell: + host.a: + - ceph fs dump + - ceph --format=json fs dump | jq -e ".filesystems | length == 1" + - ceph --format=json mds versions | jq -e ". | add == 4" +- fs.pre_upgrade_save: diff --git a/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/2-client.yaml b/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/2-client.yaml new file mode 100644 index 0000000000000..d7352c6bf5c2d --- /dev/null +++ b/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/2-client.yaml @@ -0,0 +1,3 @@ +tasks: +- kclient: +- print: "**** done kclient client" diff --git a/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/3-upgrade-with-workload.yaml b/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/3-upgrade-with-workload.yaml new file mode 100644 index 0000000000000..200c4dcb58bb3 --- /dev/null +++ b/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/3-upgrade-with-workload.yaml @@ -0,0 +1,33 @@ +tasks: +- parallel: + - upgrade-tasks + - workload-tasks + +upgrade-tasks: + sequential: + - cephadm.shell: + env: [sha1] + host.a: + - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force + - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force + - ceph config set global log_to_journald false --force + - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 + - cephadm.shell: + env: [sha1] + host.a: + - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; ceph fs dump; sleep 30 ; done + - ceph orch ps + - ceph versions + - echo "wait for servicemap items w/ changing names to refresh" + - sleep 60 + - ceph orch ps + - ceph versions + - ceph versions | jq -e '.overall | length == 1' + - ceph versions | jq -e '.overall | keys' | grep $sha1 + +workload-tasks: + sequential: + - workunit: + clients: + all: + - suites/fsstress.sh diff --git a/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/4-verify.yaml b/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/4-verify.yaml new file mode 100644 index 0000000000000..c2b657e5a3b8a --- /dev/null +++ b/qa/suites/fs/upgrade/mds_upgrade_sequence/tasks/4-verify.yaml @@ -0,0 +1,5 @@ +tasks: +- cephadm.shell: + host.a: + - ceph fs dump +- fs.post_upgrade_checks: diff --git a/qa/suites/orch/cephadm/mds_upgrade_sequence b/qa/suites/orch/cephadm/mds_upgrade_sequence new file mode 120000 index 0000000000000..24aa41c1039f9 --- /dev/null +++ b/qa/suites/orch/cephadm/mds_upgrade_sequence @@ -0,0 +1 @@ +.qa/suites/fs/upgrade/mds_upgrade_sequence/ \ No newline at end of file From 485841b255e1bf59a64c285b6b3971577ce87811 Mon Sep 17 00:00:00 2001 From: Rishabh Dave Date: Tue, 15 Jun 2021 18:25:22 +0530 Subject: [PATCH 0087/2866] qa: import CommandFailedError from exceptions not run Stop importing CommandFailedError from teuthology.orchestra.run, it is actually defined in teuthology.exception. Fixes: https://tracker.ceph.com/issues/51226 Signed-off-by: Rishabh Dave --- qa/tasks/ceph_test_case.py | 2 +- qa/tasks/cephfs/cephfs_test_case.py | 2 +- qa/tasks/cephfs/fuse_mount.py | 2 +- qa/tasks/cephfs/kernel_mount.py | 2 +- qa/tasks/cephfs/mount.py | 3 ++- qa/tasks/cephfs/test_admin.py | 3 ++- qa/tasks/cephfs/test_auto_repair.py | 2 +- qa/tasks/cephfs/test_cephfs_shell.py | 2 +- qa/tasks/cephfs/test_client_recovery.py | 2 +- qa/tasks/cephfs/test_data_scan.py | 2 +- qa/tasks/cephfs/test_exports.py | 2 +- qa/tasks/cephfs/test_forward_scrub.py | 2 +- qa/tasks/cephfs/test_full.py | 2 +- qa/tasks/cephfs/test_misc.py | 2 +- qa/tasks/cephfs/test_multifs_auth.py | 2 +- qa/tasks/cephfs/test_multimds_misc.py | 2 +- qa/tasks/cephfs/test_recovery_pool.py | 2 +- qa/tasks/cephfs/test_snapshots.py | 3 ++- qa/tasks/cephfs/test_strays.py | 3 ++- qa/tasks/kclient.py | 2 +- qa/tasks/mds_creation_failure.py | 3 ++- qa/tasks/vstart_runner.py | 2 +- qa/tasks/workunit.py | 2 +- 23 files changed, 28 insertions(+), 23 deletions(-) diff --git a/qa/tasks/ceph_test_case.py b/qa/tasks/ceph_test_case.py index 7040853dfb437..0ce9e81f1c138 100644 --- a/qa/tasks/ceph_test_case.py +++ b/qa/tasks/ceph_test_case.py @@ -3,7 +3,7 @@ import time import logging -from teuthology.orchestra.run import CommandFailedError +from teuthology.exceptions import CommandFailedError if TYPE_CHECKING: from tasks.mgr.mgr_test_case import MgrCluster diff --git a/qa/tasks/cephfs/cephfs_test_case.py b/qa/tasks/cephfs/cephfs_test_case.py index 6b3460ff248e2..8383359d5aa7f 100644 --- a/qa/tasks/cephfs/cephfs_test_case.py +++ b/qa/tasks/cephfs/cephfs_test_case.py @@ -9,7 +9,7 @@ from teuthology import contextutil from teuthology.orchestra import run -from teuthology.orchestra.run import CommandFailedError +from teuthology.exceptions import CommandFailedError log = logging.getLogger(__name__) diff --git a/qa/tasks/cephfs/fuse_mount.py b/qa/tasks/cephfs/fuse_mount.py index 5f4978e57155d..8ecaab7b78bba 100644 --- a/qa/tasks/cephfs/fuse_mount.py +++ b/qa/tasks/cephfs/fuse_mount.py @@ -8,7 +8,7 @@ from teuthology.contextutil import MaxWhileTries from teuthology.contextutil import safe_while from teuthology.orchestra import run -from teuthology.orchestra.run import CommandFailedError +from teuthology.exceptions import CommandFailedError from tasks.ceph_manager import get_valgrind_args from tasks.cephfs.mount import CephFSMount diff --git a/qa/tasks/cephfs/kernel_mount.py b/qa/tasks/cephfs/kernel_mount.py index 55fdae5d9e84d..9d14f4d749a2c 100644 --- a/qa/tasks/cephfs/kernel_mount.py +++ b/qa/tasks/cephfs/kernel_mount.py @@ -6,7 +6,7 @@ from io import StringIO from textwrap import dedent -from teuthology.orchestra.run import CommandFailedError +from teuthology.exceptions import CommandFailedError from teuthology.orchestra import run from teuthology.contextutil import MaxWhileTries diff --git a/qa/tasks/cephfs/mount.py b/qa/tasks/cephfs/mount.py index 6224a7eae7b08..dccac187db659 100644 --- a/qa/tasks/cephfs/mount.py +++ b/qa/tasks/cephfs/mount.py @@ -14,7 +14,8 @@ from teuthology.contextutil import safe_while from teuthology.misc import get_file, write_file from teuthology.orchestra import run -from teuthology.orchestra.run import CommandFailedError, ConnectionLostError, Raw +from teuthology.orchestra.run import Raw +from teuthology.exceptions import CommandFailedError, ConnectionLostError from tasks.cephfs.filesystem import Filesystem diff --git a/qa/tasks/cephfs/test_admin.py b/qa/tasks/cephfs/test_admin.py index aff83b86797ba..d7a083d72ec1f 100644 --- a/qa/tasks/cephfs/test_admin.py +++ b/qa/tasks/cephfs/test_admin.py @@ -6,7 +6,8 @@ from io import StringIO from os.path import join as os_path_join -from teuthology.orchestra.run import CommandFailedError, Raw +from teuthology.orchestra.run import Raw +from teuthology.exceptions import CommandFailedError from tasks.cephfs.cephfs_test_case import CephFSTestCase from tasks.cephfs.filesystem import FileLayout, FSMissing diff --git a/qa/tasks/cephfs/test_auto_repair.py b/qa/tasks/cephfs/test_auto_repair.py index 00c86b68bf60d..e6f0a8f0bd4cb 100644 --- a/qa/tasks/cephfs/test_auto_repair.py +++ b/qa/tasks/cephfs/test_auto_repair.py @@ -6,7 +6,7 @@ import logging import time -from teuthology.orchestra.run import CommandFailedError +from teuthology.exceptions import CommandFailedError from tasks.cephfs.cephfs_test_case import CephFSTestCase diff --git a/qa/tasks/cephfs/test_cephfs_shell.py b/qa/tasks/cephfs/test_cephfs_shell.py index 83ee3991196e0..1ee5c7f84645d 100644 --- a/qa/tasks/cephfs/test_cephfs_shell.py +++ b/qa/tasks/cephfs/test_cephfs_shell.py @@ -10,7 +10,7 @@ import math from time import sleep from tasks.cephfs.cephfs_test_case import CephFSTestCase -from teuthology.orchestra.run import CommandFailedError +from teuthology.exceptions import CommandFailedError log = logging.getLogger(__name__) diff --git a/qa/tasks/cephfs/test_client_recovery.py b/qa/tasks/cephfs/test_client_recovery.py index 24726b369f9fd..eea93939d24b5 100644 --- a/qa/tasks/cephfs/test_client_recovery.py +++ b/qa/tasks/cephfs/test_client_recovery.py @@ -11,7 +11,7 @@ import os from teuthology.orchestra import run -from teuthology.orchestra.run import CommandFailedError +from teuthology.exceptions import CommandFailedError from tasks.cephfs.fuse_mount import FuseMount from tasks.cephfs.cephfs_test_case import CephFSTestCase from teuthology.packaging import get_package_version diff --git a/qa/tasks/cephfs/test_data_scan.py b/qa/tasks/cephfs/test_data_scan.py index dcb7eda40a56d..177ffe01eaa08 100644 --- a/qa/tasks/cephfs/test_data_scan.py +++ b/qa/tasks/cephfs/test_data_scan.py @@ -13,7 +13,7 @@ from collections import namedtuple, defaultdict from textwrap import dedent -from teuthology.orchestra.run import CommandFailedError +from teuthology.exceptions import CommandFailedError from tasks.cephfs.cephfs_test_case import CephFSTestCase, for_teuthology log = logging.getLogger(__name__) diff --git a/qa/tasks/cephfs/test_exports.py b/qa/tasks/cephfs/test_exports.py index d2421bedcbbc4..4961edb520d01 100644 --- a/qa/tasks/cephfs/test_exports.py +++ b/qa/tasks/cephfs/test_exports.py @@ -3,7 +3,7 @@ import time from tasks.cephfs.fuse_mount import FuseMount from tasks.cephfs.cephfs_test_case import CephFSTestCase -from teuthology.orchestra.run import CommandFailedError +from teuthology.exceptions import CommandFailedError log = logging.getLogger(__name__) diff --git a/qa/tasks/cephfs/test_forward_scrub.py b/qa/tasks/cephfs/test_forward_scrub.py index 82630e069587b..ab97a6f596ca1 100644 --- a/qa/tasks/cephfs/test_forward_scrub.py +++ b/qa/tasks/cephfs/test_forward_scrub.py @@ -14,7 +14,7 @@ from io import BytesIO from textwrap import dedent -from teuthology.orchestra.run import CommandFailedError +from teuthology.exceptions import CommandFailedError from tasks.cephfs.cephfs_test_case import CephFSTestCase import struct diff --git a/qa/tasks/cephfs/test_full.py b/qa/tasks/cephfs/test_full.py index 2b7166d6b2be2..962a3d036d1fc 100644 --- a/qa/tasks/cephfs/test_full.py +++ b/qa/tasks/cephfs/test_full.py @@ -7,7 +7,7 @@ except: # make it work for python2 pass -from teuthology.orchestra.run import CommandFailedError +from teuthology.exceptions import CommandFailedError from tasks.cephfs.fuse_mount import FuseMount from tasks.cephfs.cephfs_test_case import CephFSTestCase diff --git a/qa/tasks/cephfs/test_misc.py b/qa/tasks/cephfs/test_misc.py index 85b11cc6ca045..0367c141f29cc 100644 --- a/qa/tasks/cephfs/test_misc.py +++ b/qa/tasks/cephfs/test_misc.py @@ -2,7 +2,7 @@ from tasks.cephfs.fuse_mount import FuseMount from tasks.cephfs.cephfs_test_case import CephFSTestCase -from teuthology.orchestra.run import CommandFailedError +from teuthology.exceptions import CommandFailedError import errno import time import json diff --git a/qa/tasks/cephfs/test_multifs_auth.py b/qa/tasks/cephfs/test_multifs_auth.py index 2b28c61ca043e..6768d57cfc0af 100644 --- a/qa/tasks/cephfs/test_multifs_auth.py +++ b/qa/tasks/cephfs/test_multifs_auth.py @@ -8,7 +8,7 @@ # CapsHelper is subclassed from CephFSTestCase from tasks.cephfs.caps_helper import CapsHelper -from teuthology.orchestra.run import CommandFailedError +from teuthology.exceptions import CommandFailedError log = logging.getLogger(__name__) diff --git a/qa/tasks/cephfs/test_multimds_misc.py b/qa/tasks/cephfs/test_multimds_misc.py index 3c464e91d0a28..2bb6257c7eb8a 100644 --- a/qa/tasks/cephfs/test_multimds_misc.py +++ b/qa/tasks/cephfs/test_multimds_misc.py @@ -2,7 +2,7 @@ import errno from tasks.cephfs.cephfs_test_case import CephFSTestCase from teuthology.contextutil import safe_while -from teuthology.orchestra.run import CommandFailedError +from teuthology.exceptions import CommandFailedError log = logging.getLogger(__name__) diff --git a/qa/tasks/cephfs/test_recovery_pool.py b/qa/tasks/cephfs/test_recovery_pool.py index 9926b3670cac3..5dd5960add1f2 100644 --- a/qa/tasks/cephfs/test_recovery_pool.py +++ b/qa/tasks/cephfs/test_recovery_pool.py @@ -6,7 +6,7 @@ import traceback from collections import namedtuple -from teuthology.orchestra.run import CommandFailedError +from teuthology.exceptions import CommandFailedError from tasks.cephfs.cephfs_test_case import CephFSTestCase log = logging.getLogger(__name__) diff --git a/qa/tasks/cephfs/test_snapshots.py b/qa/tasks/cephfs/test_snapshots.py index fa4d9431fefdf..e651bb1c6b103 100644 --- a/qa/tasks/cephfs/test_snapshots.py +++ b/qa/tasks/cephfs/test_snapshots.py @@ -4,7 +4,8 @@ from textwrap import dedent from tasks.cephfs.fuse_mount import FuseMount from tasks.cephfs.cephfs_test_case import CephFSTestCase -from teuthology.orchestra.run import CommandFailedError, Raw +from teuthology.orchestra.run import Raw +from teuthology.exceptions import CommandFailedError log = logging.getLogger(__name__) diff --git a/qa/tasks/cephfs/test_strays.py b/qa/tasks/cephfs/test_strays.py index f8313badd0358..44ca0b013d840 100644 --- a/qa/tasks/cephfs/test_strays.py +++ b/qa/tasks/cephfs/test_strays.py @@ -5,7 +5,8 @@ import datetime import gevent -from teuthology.orchestra.run import CommandFailedError, Raw +from teuthology.exceptions import CommandFailedError +from teuthology.orchestra.run import Raw from tasks.cephfs.cephfs_test_case import CephFSTestCase, for_teuthology log = logging.getLogger(__name__) diff --git a/qa/tasks/kclient.py b/qa/tasks/kclient.py index d7bc9fa835b55..2749f01daeef1 100644 --- a/qa/tasks/kclient.py +++ b/qa/tasks/kclient.py @@ -5,7 +5,7 @@ import logging from teuthology.misc import deep_merge -from teuthology.orchestra.run import CommandFailedError +from teuthology.exceptions import CommandFailedError from teuthology import misc from teuthology.contextutil import MaxWhileTries from tasks.cephfs.kernel_mount import KernelMount diff --git a/qa/tasks/mds_creation_failure.py b/qa/tasks/mds_creation_failure.py index 58314086c0337..2ab8f70dd9d2f 100644 --- a/qa/tasks/mds_creation_failure.py +++ b/qa/tasks/mds_creation_failure.py @@ -5,7 +5,8 @@ import time from tasks import ceph_manager from teuthology import misc -from teuthology.orchestra.run import CommandFailedError, Raw +from teuthology.exceptions import CommandFailedError +from teuthology.orchestra.run import Raw log = logging.getLogger(__name__) diff --git a/qa/tasks/vstart_runner.py b/qa/tasks/vstart_runner.py index eb85d5ab43a2d..916256495d454 100644 --- a/qa/tasks/vstart_runner.py +++ b/qa/tasks/vstart_runner.py @@ -56,7 +56,7 @@ from teuthology.config import config as teuth_config from teuthology.contextutil import safe_while from teuthology.contextutil import MaxWhileTries -from teuthology.orchestra.run import CommandFailedError +from teuthology.exceptions import CommandFailedError try: import urllib3 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) diff --git a/qa/tasks/workunit.py b/qa/tasks/workunit.py index 371d2a2dddb5a..6412c99bc04d0 100644 --- a/qa/tasks/workunit.py +++ b/qa/tasks/workunit.py @@ -12,7 +12,7 @@ from teuthology import misc from teuthology.config import config as teuth_config -from teuthology.orchestra.run import CommandFailedError +from teuthology.exceptions import CommandFailedError from teuthology.parallel import parallel from teuthology.orchestra import run From eb25549b8c39a3ad2ee3152a35338e90f0693a13 Mon Sep 17 00:00:00 2001 From: Rishabh Dave Date: Tue, 5 Oct 2021 23:57:36 +0530 Subject: [PATCH 0088/2866] qa/cephfs: update xfstests_dev for centos stream Fixes: https://tracker.ceph.com/issues/52821 Signed-off-by: Rishabh Dave --- qa/tasks/cephfs/xfstests_dev.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qa/tasks/cephfs/xfstests_dev.py b/qa/tasks/cephfs/xfstests_dev.py index c0bdf338e1671..fe957b0045e65 100644 --- a/qa/tasks/cephfs/xfstests_dev.py +++ b/qa/tasks/cephfs/xfstests_dev.py @@ -95,7 +95,7 @@ def install_deps(self): # we keep fedora here so that right deps are installed when this test # is run locally by a dev. if distro in ('redhatenterpriseserver', 'redhatenterprise', 'fedora', - 'centos'): + 'centos', 'centosstream'): deps = """acl attr automake bc dbench dump e2fsprogs fio \ gawk gcc indent libtool lvm2 make psmisc quota sed \ xfsdump xfsprogs \ From 97f9f5362bea855628c2f82c31176bd129db19b7 Mon Sep 17 00:00:00 2001 From: Patrick Donnelly Date: Tue, 5 Oct 2021 13:49:17 -0400 Subject: [PATCH 0089/2866] qa: skip c-v install on pacific start Continuation of 4e525127fbb710c1ac074cf61b448055781a69e3. Fixes: https://tracker.ceph.com/issues/52822 Signed-off-by: Patrick Donnelly --- .../fs/upgrade/featureful_client/old_client/tasks/0-octopus.yaml | 1 + .../featureful_client/upgraded_client/tasks/0-octopus.yaml | 1 + qa/suites/fs/upgrade/nofs/tasks/0-octopus.yaml | 1 + 3 files changed, 3 insertions(+) diff --git a/qa/suites/fs/upgrade/featureful_client/old_client/tasks/0-octopus.yaml b/qa/suites/fs/upgrade/featureful_client/old_client/tasks/0-octopus.yaml index 40d34753da1b8..e7774423fa544 100644 --- a/qa/suites/fs/upgrade/featureful_client/old_client/tasks/0-octopus.yaml +++ b/qa/suites/fs/upgrade/featureful_client/old_client/tasks/0-octopus.yaml @@ -11,6 +11,7 @@ tasks: - ceph-mgr-rook - ceph-mgr-cephadm - cephadm + - ceph-volume extra_packages: ['librados2'] - print: "**** done installing octopus" - ceph: diff --git a/qa/suites/fs/upgrade/featureful_client/upgraded_client/tasks/0-octopus.yaml b/qa/suites/fs/upgrade/featureful_client/upgraded_client/tasks/0-octopus.yaml index 40d34753da1b8..e7774423fa544 100644 --- a/qa/suites/fs/upgrade/featureful_client/upgraded_client/tasks/0-octopus.yaml +++ b/qa/suites/fs/upgrade/featureful_client/upgraded_client/tasks/0-octopus.yaml @@ -11,6 +11,7 @@ tasks: - ceph-mgr-rook - ceph-mgr-cephadm - cephadm + - ceph-volume extra_packages: ['librados2'] - print: "**** done installing octopus" - ceph: diff --git a/qa/suites/fs/upgrade/nofs/tasks/0-octopus.yaml b/qa/suites/fs/upgrade/nofs/tasks/0-octopus.yaml index 40d34753da1b8..e7774423fa544 100644 --- a/qa/suites/fs/upgrade/nofs/tasks/0-octopus.yaml +++ b/qa/suites/fs/upgrade/nofs/tasks/0-octopus.yaml @@ -11,6 +11,7 @@ tasks: - ceph-mgr-rook - ceph-mgr-cephadm - cephadm + - ceph-volume extra_packages: ['librados2'] - print: "**** done installing octopus" - ceph: From b96aa5d1844e334c01e0e0dd17179468e881e883 Mon Sep 17 00:00:00 2001 From: Paul Cuzner Date: Wed, 6 Oct 2021 14:32:47 +1300 Subject: [PATCH 0090/2866] monitoring:Updated README Signed-off-by: Paul Cuzner --- monitoring/prometheus/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/monitoring/prometheus/README.md b/monitoring/prometheus/README.md index 205e373b19c0e..7ec8f309adcf5 100644 --- a/monitoring/prometheus/README.md +++ b/monitoring/prometheus/README.md @@ -7,8 +7,8 @@ file in a place according to your Prometheus configuration (wherever the `rules` configuration stanza points). ### SNMP -Ceph provides a MIB (CEPH-PROMETHEUS-ALERT-MIB.txt) to support sending prometheus -alerts through to an SNMP management platform. The translation from prometheus -alert to SNMP trap requires the prometheus alert to contain an OID that maps to -a definition within the MIB. When making changes to the prometheus alert rules +Ceph provides a MIB (CEPH-PROMETHEUS-ALERT-MIB.txt) to support sending Prometheus +alerts through to an SNMP management platform. The translation from Prometheus +alert to SNMP trap requires the Prometheus alert to contain an OID that maps to +a definition within the MIB. When making changes to the Prometheus alert rules file, developers should include any necessary changes to the MIB. From ff88d7de52b347486a626375816fc119ee8fd72f Mon Sep 17 00:00:00 2001 From: Venky Shankar Date: Thu, 7 Oct 2021 00:40:13 -0400 Subject: [PATCH 0091/2866] qa: skip internal metadata directory when scanning ceph debugfs directory kclient patchset https://patchwork.kernel.org/project/ceph-devel/list/?series=556049 introduces `meta` directory to add debugging entries. This needs to be filtered when scanning ceph debugfs directory. Fixes: https://tracker.ceph.com/issues/52824 Signed-off-by: Venky Shankar --- qa/tasks/cephfs/kernel_mount.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/qa/tasks/cephfs/kernel_mount.py b/qa/tasks/cephfs/kernel_mount.py index 55fdae5d9e84d..e8018eeec3640 100644 --- a/qa/tasks/cephfs/kernel_mount.py +++ b/qa/tasks/cephfs/kernel_mount.py @@ -16,7 +16,8 @@ UMOUNT_TIMEOUT = 300 - +# internal metadata directory +DEBUGFS_META_DIR = 'meta' class KernelMount(CephFSMount): def __init__(self, ctx, test_dir, client_id, client_remote, @@ -200,6 +201,8 @@ def _get_global_id(self): def get_id_to_dir(): result = {} for dir in glob.glob("/sys/kernel/debug/ceph/*"): + if os.path.basename(dir) == DEBUGFS_META_DIR: + continue mds_sessions_lines = open(os.path.join(dir, "mds_sessions")).readlines() global_id = mds_sessions_lines[0].split()[1].strip('"') client_id = mds_sessions_lines[1].split()[1].strip('"') From 6d6236dc8d15636af8060057e6e69c26c473f987 Mon Sep 17 00:00:00 2001 From: Venky Shankar Date: Fri, 1 Oct 2021 04:55:40 -0400 Subject: [PATCH 0092/2866] mds: skip journaling blocklisted clients when in `replay` state When a standby MDS is transitioning to active, it passes through `replay` state. When the MDS is in this state, there are no journal segments available for recording journal updates. If the MDS receives an OSDMap update in this state, journaling blocklisted clients causes a crash since no journal segments are available. This is a bit hard to reproduce as it requires correct timing of an OSDMap update along with various other factors. Note that, when the MDS reaches `reconnect` state, it will journal the blocklisted clients anyway. This partially fixes tracker: https://tracker.ceph.com/issues/51589 which mentions a similar crash but in `reconnect` state. However, that crash was seen in nautilus. A couple of minor changes include removing hardcoded function names and carving out reusable parts into a separate function. Partially-fixes: https://tracker.ceph.com/issues/51589 Signed-off-by: Venky Shankar --- src/mds/MDSRank.cc | 38 ++++++++++++++++++++++---------------- src/mds/MDSRank.h | 4 ++++ 2 files changed, 26 insertions(+), 16 deletions(-) diff --git a/src/mds/MDSRank.cc b/src/mds/MDSRank.cc index 190a53706455d..f5f0585060dab 100644 --- a/src/mds/MDSRank.cc +++ b/src/mds/MDSRank.cc @@ -1930,6 +1930,17 @@ void MDSRank::resolve_done() snapclient->sync(new C_MDSInternalNoop); } +void MDSRank::apply_blocklist(const std::set &addrs, epoch_t epoch) { + auto victims = server->apply_blocklist(addrs); + dout(4) << __func__ << ": killed " << victims << " blocklisted sessions (" + << addrs.size() << " blocklist entries, " + << sessionmap.get_sessions().size() << ")" << dendl; + if (victims) { + set_osd_epoch_barrier(epoch); + } +} + + void MDSRank::reconnect_start() { dout(1) << "reconnect_start" << dendl; @@ -1947,13 +1958,8 @@ void MDSRank::reconnect_start() o.get_blocklist(&blocklist); epoch = o.get_epoch(); }); - auto killed = server->apply_blocklist(blocklist); - dout(4) << "reconnect_start: killed " << killed << " blocklisted sessions (" - << blocklist.size() << " blocklist entries, " - << sessionmap.get_sessions().size() << ")" << dendl; - if (killed) { - set_osd_epoch_barrier(epoch); - } + + apply_blocklist(blocklist, epoch); server->reconnect_clients(new C_MDS_VoidFn(this, &MDSRank::reconnect_done)); finish_contexts(g_ceph_context, waiting_for_reconnect); @@ -3455,17 +3461,17 @@ void MDSRankDispatcher::handle_osd_map() purge_queue.update_op_limit(*mdsmap); - std::set newly_blocklisted; - objecter->consume_blocklist_events(&newly_blocklisted); - auto epoch = objecter->with_osdmap([](const OSDMap &o){return o.get_epoch();}); - dout(4) << "handle_osd_map epoch " << epoch << ", " - << newly_blocklisted.size() << " new blocklist entries" << dendl; - auto victims = server->apply_blocklist(newly_blocklisted); - if (victims) { - set_osd_epoch_barrier(epoch); + // it's ok if replay state is reached via standby-replay, the + // reconnect state will journal blocklisted clients (journal + // is opened for writing in `replay_done` before moving to + // up:resolve). + if (!is_replay()) { + std::set newly_blocklisted; + objecter->consume_blocklist_events(&newly_blocklisted); + auto epoch = objecter->with_osdmap([](const OSDMap &o){return o.get_epoch();}); + apply_blocklist(newly_blocklisted, epoch); } - // By default the objecter only requests OSDMap updates on use, // we would like to always receive the latest maps in order to // apply policy based on the FULL flag. diff --git a/src/mds/MDSRank.h b/src/mds/MDSRank.h index 6ba2001932867..21b5ac7924b62 100644 --- a/src/mds/MDSRank.h +++ b/src/mds/MDSRank.h @@ -546,6 +546,10 @@ class MDSRank { Context *create_async_exec_context(C_ExecAndReply *ctx); + // blocklist the provided addrs and set OSD epoch barrier + // with the provided epoch. + void apply_blocklist(const std::set &addrs, epoch_t epoch); + // Incarnation as seen in MDSMap at the point where a rank is // assigned. int incarnation = 0; From bce34dd68634d241b451111dcf2e931837eb4bfd Mon Sep 17 00:00:00 2001 From: Cory Snyder Date: Tue, 5 Oct 2021 19:07:11 +0000 Subject: [PATCH 0093/2866] rgw/rgw_rados: make RGW request IDs non-deterministic Use a random number vs. incremental counter for first component of request ID. Fixes: https://tracker.ceph.com/issues/52818 Signed-off-by: Cory Snyder --- src/rgw/rgw_rados.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h index 26b603b128b01..cb96b9dbb2e59 100644 --- a/src/rgw/rgw_rados.h +++ b/src/rgw/rgw_rados.h @@ -9,6 +9,7 @@ #include "include/rados/librados.hpp" #include "include/Context.h" +#include "include/random.h" #include "common/RefCountedObj.h" #include "common/ceph_time.h" #include "common/Timer.h" @@ -355,7 +356,7 @@ class RGWRados int open_pool_ctx(const DoutPrefixProvider *dpp, const rgw_pool& pool, librados::IoCtx& io_ctx, bool mostly_omap); - std::atomic max_req_id = { 0 }; + ceph::mutex lock = ceph::make_mutex("rados_timer_lock"); SafeTimer *timer; @@ -506,7 +507,7 @@ class RGWRados } uint64_t get_new_req_id() { - return ++max_req_id; + return ceph::util::generate_random_number(); } librados::IoCtx* get_lc_pool_ctx() { From 008b41bfc7a80d215314ece61af9291094e3f274 Mon Sep 17 00:00:00 2001 From: Daniel Pivonka Date: Thu, 7 Oct 2021 15:52:05 -0400 Subject: [PATCH 0094/2866] mgr/cephadm: track daemon deployment post actions by daemon name instead of type in rare cases tracking by type can cause a daemon to be missed when doing post actions if it is not in the cache yet Signed-off-by: Daniel Pivonka --- src/pybind/mgr/cephadm/serve.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/pybind/mgr/cephadm/serve.py b/src/pybind/mgr/cephadm/serve.py index 1cf024fac310a..59191999532c2 100644 --- a/src/pybind/mgr/cephadm/serve.py +++ b/src/pybind/mgr/cephadm/serve.py @@ -1004,8 +1004,12 @@ def _check_daemons(self) -> None: # do daemon post actions for daemon_type, daemon_descs in daemons_post.items(): - if daemon_type in self.mgr.requires_post_actions: - self.mgr.requires_post_actions.remove(daemon_type) + run_post = False + for d in daemon_descs: + if d.name() in self.mgr.requires_post_actions: + self.mgr.requires_post_actions.remove(d.name()) + run_post = True + if run_post: self.mgr._get_cephadm_service(daemon_type_to_service( daemon_type)).daemon_check_post(daemon_descs) @@ -1125,7 +1129,7 @@ def _create_daemon(self, DaemonDescriptionStatus.running, 'starting') self.mgr.cache.add_daemon(daemon_spec.host, sd) if daemon_spec.daemon_type in REQUIRES_POST_ACTIONS: - self.mgr.requires_post_actions.add(daemon_spec.daemon_type) + self.mgr.requires_post_actions.add(daemon_spec.name()) self.mgr.cache.invalidate_host_daemons(daemon_spec.host) self.mgr.cache.update_daemon_config_deps( From 00e5e5d5cdb56b7e249ffc98aeb456c9ac1c5f48 Mon Sep 17 00:00:00 2001 From: Dai Zhiwei Date: Fri, 8 Oct 2021 14:26:43 +0800 Subject: [PATCH 0095/2866] qa/tasks: replace iterkeys() with keys() in Python 3 Python 2.7 reached the end of its lifetime, the pr fixes teuthology task error in Python 3.x Fixes: https://tracker.ceph.com/issues/52878 Signed-off-by: Dai Zhiwei --- qa/tasks/netsplit.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/qa/tasks/netsplit.py b/qa/tasks/netsplit.py index b6614dc5061b4..0a9484a89a5c0 100644 --- a/qa/tasks/netsplit.py +++ b/qa/tasks/netsplit.py @@ -35,8 +35,8 @@ def disconnect(ctx, config): (ip1, _) = get_ip_and_ports(ctx, config[0]) (ip2, _) = get_ip_and_ports(ctx, config[1]) - (host1,) = ctx.cluster.only(config[0]).remotes.iterkeys() - (host2,) = ctx.cluster.only(config[1]).remotes.iterkeys() + (host1,) = ctx.cluster.only(config[0]).remotes.keys() + (host2,) = ctx.cluster.only(config[1]).remotes.keys() assert host1 is not None assert host2 is not None @@ -58,8 +58,8 @@ def reconnect(ctx, config): (ip1, _) = get_ip_and_ports(ctx, config[0]) (ip2, _) = get_ip_and_ports(ctx, config[1]) - (host1,) = ctx.cluster.only(config[0]).remotes.iterkeys() - (host2,) = ctx.cluster.only(config[1]).remotes.iterkeys() + (host1,) = ctx.cluster.only(config[0]).remotes.keys() + (host2,) = ctx.cluster.only(config[1]).remotes.keys() assert host1 is not None assert host2 is not None From 0be0772e8af4caab8f63271f3e6a908fd2a217aa Mon Sep 17 00:00:00 2001 From: Anthony D'Atri Date: Sun, 10 Oct 2021 12:31:09 -0700 Subject: [PATCH 0096/2866] ceph: document rgw_multipart_min_part_size Signed-off-by: Anthony D'Atri --- doc/radosgw/config-ref.rst | 1 + src/common/options/rgw.yaml.in | 8 +++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/doc/radosgw/config-ref.rst b/doc/radosgw/config-ref.rst index 371ce89d25d74..295fa8ce2d77e 100644 --- a/doc/radosgw/config-ref.rst +++ b/doc/radosgw/config-ref.rst @@ -36,6 +36,7 @@ instances or all radosgw-admin options can be put into the ``[global]`` or the .. confval:: rgw_exit_timeout_secs .. confval:: rgw_get_obj_window_size .. confval:: rgw_get_obj_max_req_size +.. confval:: rgw_multipart_min_part_size .. confval:: rgw_relaxed_s3_bucket_names .. confval:: rgw_list_buckets_max_chunk .. confval:: rgw_override_bucket_index_max_shards diff --git a/src/common/options/rgw.yaml.in b/src/common/options/rgw.yaml.in index 0cbf397783eeb..4bfe1c596c3dc 100644 --- a/src/common/options/rgw.yaml.in +++ b/src/common/options/rgw.yaml.in @@ -62,9 +62,11 @@ options: desc: Set RGW max chunk size long_desc: The chunk size is the size of RADOS I/O requests that RGW sends when accessing data objects. RGW read and write operations will never request more than - this amount in a single request. This also defines the RGW object head size, as + this amount in a single request. This also defines the RGW head object size, as head operations need to be atomic, and anything larger than this would require - more than a single operation. + more than a single operation. When RGW objects are written to the default + storage class, up to this amount of payload data will be stored alongside + metadata in the head object. default: 4_M services: - rgw @@ -2119,7 +2121,7 @@ options: type: size level: advanced desc: Minimum S3 multipart-upload part size - long_desc: When doing a multipart upload, each part (other than the last part) should + long_desc: When doing a multipart upload, each part (other than the last part) must be at least this size. default: 5_M services: From d388c5e958ddf5447c78db50ca2061bb443d2227 Mon Sep 17 00:00:00 2001 From: Avan Thakkar Date: Fri, 1 Oct 2021 03:56:42 +0530 Subject: [PATCH 0097/2866] mgr/dashboard: replace Client connections with active-stdby mgrs Fixes: https://tracker.ceph.com/issues/52121 Signed-off-by: Avan Thakkar --- .../grafana/dashboards/ceph-cluster.json | 25 +++++++++++++++---- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/monitoring/grafana/dashboards/ceph-cluster.json b/monitoring/grafana/dashboards/ceph-cluster.json index e4e367efdd8b3..1b91bfe418512 100644 --- a/monitoring/grafana/dashboards/ceph-cluster.json +++ b/monitoring/grafana/dashboards/ceph-cluster.json @@ -678,7 +678,7 @@ "type": "vonage-status-panel" }, { - "colorMode": "Disabled", + "colorMode": "Panel", "colors": { "crit": "rgba(245, 54, 54, 0.9)", "disable": "rgba(128, 128, 128, 0.9)", @@ -706,21 +706,36 @@ "targets": [ { "aggregation": "Last", - "alias": "Clients", + "alias": "Active", "decimals": 2, "displayAliasType": "Always", "displayType": "Regular", "displayValueWithAlias": "When Alias Displayed", - "expr": "ceph_mds_server_handle_client_session", + "expr": "count(ceph_mgr_status == 1) or vector(0)", "format": "time_series", "intervalFactor": 1, - "legendFormat": "Clients", + "legendFormat": "Active", "refId": "A", "units": "none", "valueHandler": "Number Threshold" + }, + { + "aggregation": "Last", + "alias": "Standby", + "decimals": 2, + "displayAliasType": "Always", + "displayType": "Regular", + "displayValueWithAlias": "When Alias Displayed", + "expr": "count(ceph_mgr_status == 0) or vector(0)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Standby", + "refId": "B", + "units": "none", + "valueHandler": "Number Threshold" } ], - "title": "Client connections", + "title": "MGRs", "type": "vonage-status-panel" }, { From e517882f2138c0e2aab1c08c8b3c882b92bcaf37 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Mon, 11 Oct 2021 11:57:40 -0500 Subject: [PATCH 0098/2866] mgr/rook: factor out replica/failureDomain calc Signed-off-by: Sage Weil --- src/pybind/mgr/rook/module.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/src/pybind/mgr/rook/module.py b/src/pybind/mgr/rook/module.py index 0e53aea449285..27b80538b6734 100644 --- a/src/pybind/mgr/rook/module.py +++ b/src/pybind/mgr/rook/module.py @@ -416,6 +416,20 @@ def _list_daemons(self, return result + def _get_pool_params(self) -> Tuple[str, str]: + num_replicas = self.get_ceph_option('osd_pool_default_size') + assert type(num_replicas) is int + + leaf_type_id = self.get_ceph_option('osd_crush_chooseleaf_type') + assert type(leaf_type_id) is int + crush = self.get('osd_map_crush') + leaf_type = 'host' + for t in crush['types']: + if t['type_id'] == leaf_type_id: + leaf_type = t['name'] + break + return num_replicas, leaf_type + @handle_orch_error def remove_service(self, service_name: str) -> str: service_type, service_name = service_name.split('.', 1) @@ -444,16 +458,7 @@ def apply_mds(self, spec): @handle_orch_error def apply_rgw(self, spec): # type: (RGWSpec) -> str - num_replicas = self.get_ceph_option('osd_pool_default_size') - assert type(num_replicas) is int - leaf_type_id = self.get_ceph_option('osd_crush_chooseleaf_type') - assert type(leaf_type_id) is int - crush = self.get('osd_map_crush') - leaf_type = 'host' - for t in crush['types']: - if t['type_id'] == leaf_type_id: - leaf_type = t['name'] - break + num_replicas, leaf_type = self._get_pool_params() return self.rook_cluster.apply_objectstore(spec, num_replicas, leaf_type) @handle_orch_error From 1a67d3e559c50bad1ccc42bf1706326ffa18da80 Mon Sep 17 00:00:00 2001 From: Joseph Sawaya Date: Fri, 3 Sep 2021 13:30:43 -0400 Subject: [PATCH 0099/2866] mgr/rook: apply mds using placement spec and osd_pool_default_size This commit changes the apply_mds command in the rook orchestrator to support some placement specs and also sets the replica size according to the osd_pool_default_size ceph option. This commit also adds `orch apply mds` to the QA to test if the command runs. Signed-off-by: Joseph Sawaya --- qa/suites/orch/rook/smoke/3-final.yaml | 1 + src/pybind/mgr/rook/module.py | 5 +-- src/pybind/mgr/rook/rook_cluster.py | 47 ++++++++++++++++++++++---- 3 files changed, 45 insertions(+), 8 deletions(-) diff --git a/qa/suites/orch/rook/smoke/3-final.yaml b/qa/suites/orch/rook/smoke/3-final.yaml index 90bbafad6e0f2..91a43f4acc8b5 100644 --- a/qa/suites/orch/rook/smoke/3-final.yaml +++ b/qa/suites/orch/rook/smoke/3-final.yaml @@ -7,3 +7,4 @@ tasks: - ceph orch host ls - ceph orch device ls - ceph orch apply rgw foo + - ceph orch apply mds foo diff --git a/src/pybind/mgr/rook/module.py b/src/pybind/mgr/rook/module.py index 27b80538b6734..d1412a708374d 100644 --- a/src/pybind/mgr/rook/module.py +++ b/src/pybind/mgr/rook/module.py @@ -416,7 +416,7 @@ def _list_daemons(self, return result - def _get_pool_params(self) -> Tuple[str, str]: + def _get_pool_params(self) -> Tuple[int, str]: num_replicas = self.get_ceph_option('osd_pool_default_size') assert type(num_replicas) is int @@ -453,7 +453,8 @@ def apply_mon(self, spec): @handle_orch_error def apply_mds(self, spec): # type: (ServiceSpec) -> str - return self.rook_cluster.apply_filesystem(spec) + num_replicas, leaf_type = self._get_pool_params() + return self.rook_cluster.apply_filesystem(spec, num_replicas, leaf_type) @handle_orch_error def apply_rgw(self, spec): diff --git a/src/pybind/mgr/rook/rook_cluster.py b/src/pybind/mgr/rook/rook_cluster.py index cf302c84550d4..1a28093509386 100644 --- a/src/pybind/mgr/rook/rook_cluster.py +++ b/src/pybind/mgr/rook/rook_cluster.py @@ -842,30 +842,65 @@ def ignore_409(self, what: str) -> Iterator[None]: else: raise - def apply_filesystem(self, spec: ServiceSpec) -> str: + def apply_filesystem(self, spec: ServiceSpec, num_replicas: int, + leaf_type: str) -> str: # TODO use spec.placement # TODO warn if spec.extended has entries we don't kow how # to action. + all_hosts = self.get_hosts() def _update_fs(new: cfs.CephFilesystem) -> cfs.CephFilesystem: new.spec.metadataServer.activeCount = spec.placement.count or 1 + new.spec.metadataServer.placement = cfs.Placement( + nodeAffinity=cfs.NodeAffinity( + requiredDuringSchedulingIgnoredDuringExecution=cfs.RequiredDuringSchedulingIgnoredDuringExecution( + nodeSelectorTerms=cfs.NodeSelectorTermsList( + [placement_spec_to_node_selector(spec.placement, all_hosts)] + ) + ) + ) + ) return new - def _create_fs() -> cfs.CephFilesystem: - return cfs.CephFilesystem( + fs = cfs.CephFilesystem( apiVersion=self.rook_env.api_name, metadata=dict( name=spec.service_id, namespace=self.rook_env.namespace, ), spec=cfs.Spec( - None, - None, + dataPools=cfs.DataPoolsList( + { + cfs.DataPoolsItem( + failureDomain=leaf_type, + replicated=cfs.Replicated( + size=num_replicas + ) + ) + } + ), + metadataPool=cfs.MetadataPool( + failureDomain=leaf_type, + replicated=cfs.Replicated( + size=num_replicas + ) + ), metadataServer=cfs.MetadataServer( activeCount=spec.placement.count or 1, - activeStandby=True + activeStandby=True, + placement= + cfs.Placement( + nodeAffinity=cfs.NodeAffinity( + requiredDuringSchedulingIgnoredDuringExecution=cfs.RequiredDuringSchedulingIgnoredDuringExecution( + nodeSelectorTerms=cfs.NodeSelectorTermsList( + [placement_spec_to_node_selector(spec.placement, all_hosts)] + ) + ) + ) + ) ) ) ) + return fs assert spec.service_id is not None return self._create_or_patch( cfs.CephFilesystem, 'cephfilesystems', spec.service_id, From 952c7c844acee5fe73e3f70737606b700b67238c Mon Sep 17 00:00:00 2001 From: Casey Bodley Date: Mon, 11 Oct 2021 14:25:37 -0400 Subject: [PATCH 0100/2866] radosgw-admin: 'reshard list' doesn't log ENOENT errors ENOENT errors are expected, especially in fresh clusters, before we've written any entries to the reshard list shards. avoid logging these non-fatal ERROR messages: > -1 ERROR: failed to list reshard log entries, oid=reshard.0000000000 marker= (2) No such file or directory Fixes: https://tracker.ceph.com/issues/52873 Signed-off-by: Casey Bodley --- src/rgw/rgw_reshard.cc | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/src/rgw/rgw_reshard.cc b/src/rgw/rgw_reshard.cc index 78c96d14c3932..5d07c1b2a374f 100644 --- a/src/rgw/rgw_reshard.cc +++ b/src/rgw/rgw_reshard.cc @@ -890,18 +890,17 @@ int RGWReshard::list(const DoutPrefixProvider *dpp, int logshard_num, string& ma int ret = cls_rgw_reshard_list(store->getRados()->reshard_pool_ctx, logshard_oid, marker, max, entries, is_truncated); - if (ret < 0) { - ldpp_dout(dpp, -1) << "ERROR: failed to list reshard log entries, oid=" << logshard_oid << " " - << "marker=" << marker << " " << cpp_strerror(ret) << dendl; - if (ret == -ENOENT) { - *is_truncated = false; - ret = 0; - } else { - if (ret == -EACCES) { - ldpp_dout(dpp, -1) << "access denied to pool " << store->svc()->zone->get_zone_params().reshard_pool - << ". Fix the pool access permissions of your client" << dendl; - } - } + if (ret == -ENOENT) { + // these shard objects aren't created until we actually write something to + // them, so treat ENOENT as a successful empty listing + *is_truncated = false; + ret = 0; + } else if (ret == -EACCES) { + ldpp_dout(dpp, -1) << "ERROR: access denied to pool " << store->svc()->zone->get_zone_params().reshard_pool + << ". Fix the pool access permissions of your client" << dendl; + } else if (ret < 0) { + ldpp_dout(dpp, -1) << "ERROR: failed to list reshard log entries, oid=" + << logshard_oid << " marker=" << marker << " " << cpp_strerror(ret) << dendl; } return ret; From f9baa5c71e3e7fbcb89d203e95a92b82661f31e4 Mon Sep 17 00:00:00 2001 From: Adam King Date: Mon, 11 Oct 2021 15:07:49 -0400 Subject: [PATCH 0101/2866] mgr/cephadm: update CEPHADM_AGENT_DOWN when agent reports in Right now if an agent considered down reports in the health warning will reamin until the serve loop runs again. This change will give better responsiveness on setting/removing this health warning. Signed-off-by: Adam King --- src/pybind/mgr/cephadm/agent.py | 27 ++++++++++++++++++++++++++- src/pybind/mgr/cephadm/serve.py | 17 +---------------- 2 files changed, 27 insertions(+), 17 deletions(-) diff --git a/src/pybind/mgr/cephadm/agent.py b/src/pybind/mgr/cephadm/agent.py index ead4c0a3bee5a..bbed1918f742a 100644 --- a/src/pybind/mgr/cephadm/agent.py +++ b/src/pybind/mgr/cephadm/agent.py @@ -21,7 +21,7 @@ from cryptography.hazmat.primitives import hashes, serialization from cryptography.hazmat.backends import default_backend -from typing import Any, Dict, Set, Tuple, TYPE_CHECKING +from typing import Any, Dict, List, Set, Tuple, TYPE_CHECKING if TYPE_CHECKING: from cephadm.module import CephadmOrchestrator @@ -164,6 +164,12 @@ def handle_metadata(self, data: Dict[str, Any]) -> None: # update timestamp of most recent agent update self.mgr.cache.agent_timestamp[host] = datetime_now() + agents_down = [] + for h in self.mgr.cache.get_hosts(): + if self.mgr.agent_helpers._agent_down(h): + agents_down.append(h) + self.mgr.agent_helpers._update_agent_down_healthcheck(agents_down) + up_to_date = False int_ack = int(data['ack']) @@ -261,6 +267,7 @@ def run(self) -> None: secure_agent_socket.sendall(msg.encode('utf-8')) agent_response = secure_agent_socket.recv(1024).decode() self.mgr.log.info(f'Received "{agent_response}" from agent on host {self.host}') + self.mgr.cache.sending_agent_message[self.host] = False return except ConnectionError as e: # if it's a connection error, possibly try to connect again. @@ -312,6 +319,24 @@ def _agent_down(self, host: str) -> bool: return True return False + def _update_agent_down_healthcheck(self, down_agent_hosts: List[str]) -> None: + if 'CEPHADM_AGENT_DOWN' in self.mgr.health_checks: + del self.mgr.health_checks['CEPHADM_AGENT_DOWN'] + if down_agent_hosts: + detail: List[str] = [] + for agent in down_agent_hosts: + detail.append((f'Cephadm agent on host {agent} has not reported in ' + f'{2.5 * self.mgr.agent_refresh_rate} seconds. Agent is assumed ' + 'down and host may be offline.')) + self.mgr.health_checks['CEPHADM_AGENT_DOWN'] = { + 'severity': 'warning', + 'summary': '%d Cephadm Agent(s) are not reporting. ' + 'Hosts may be offline' % (len(down_agent_hosts)), + 'count': len(down_agent_hosts), + 'detail': detail, + } + self.mgr.set_health_checks(self.mgr.health_checks) + # this function probably seems very unnecessary, but it makes it considerably easier # to get the unit tests working. All unit tests that check which daemons were deployed # or services setup would have to be individually changed to expect an agent service or diff --git a/src/pybind/mgr/cephadm/serve.py b/src/pybind/mgr/cephadm/serve.py index e3e9c57088385..a51aac9abc6e9 100644 --- a/src/pybind/mgr/cephadm/serve.py +++ b/src/pybind/mgr/cephadm/serve.py @@ -366,22 +366,7 @@ def refresh(host: str) -> None: refresh(self.mgr.cache.get_hosts()) - if 'CEPHADM_AGENT_DOWN' in self.mgr.health_checks: - del self.mgr.health_checks['CEPHADM_AGENT_DOWN'] - if agents_down: - detail: List[str] = [] - for agent in agents_down: - detail.append((f'Cephadm agent on host {agent} has not reported in ' - f'{2.5 * self.mgr.agent_refresh_rate} seconds. Agent is assumed ' - 'down and host may be offline.')) - self.mgr.health_checks['CEPHADM_AGENT_DOWN'] = { - 'severity': 'warning', - 'summary': '%d Cephadm Agent(s) are not reporting. ' - 'Hosts may be offline' % (len(agents_down)), - 'count': len(agents_down), - 'detail': detail, - } - self.mgr.set_health_checks(self.mgr.health_checks) + self.mgr.agent_helpers._update_agent_down_healthcheck(agents_down) self.mgr.config_checker.run_checks() From 1f43d8e618b68a8a1698745950bb095aa27886c5 Mon Sep 17 00:00:00 2001 From: Adam King Date: Wed, 29 Sep 2021 15:56:00 -0400 Subject: [PATCH 0102/2866] mgr/cephadm: update CEPHADM_FAILED_DAEMON after receiving agent metadata otherwise, there can be a period where a daemon will be marked incorrectly as failed or not failed even though mgr/cephadm knows it is or isn't in an error state Signed-off-by: Adam King --- src/pybind/mgr/cephadm/agent.py | 2 ++ src/pybind/mgr/cephadm/module.py | 15 ++++++++- src/pybind/mgr/cephadm/serve.py | 34 ++++++++++---------- src/pybind/mgr/cephadm/tests/test_cephadm.py | 18 +++++++---- 4 files changed, 45 insertions(+), 24 deletions(-) diff --git a/src/pybind/mgr/cephadm/agent.py b/src/pybind/mgr/cephadm/agent.py index ead4c0a3bee5a..1350d079794de 100644 --- a/src/pybind/mgr/cephadm/agent.py +++ b/src/pybind/mgr/cephadm/agent.py @@ -178,6 +178,7 @@ def handle_metadata(self, data: Dict[str, Any]) -> None: if 'ls' in data and data['ls']: self.mgr._process_ls_output(host, data['ls']) + self.mgr.update_failed_daemon_health_check() if 'networks' in data and data['networks']: self.mgr.cache.update_host_networks(host, data['networks']) if 'facts' in data and data['facts']: @@ -261,6 +262,7 @@ def run(self) -> None: secure_agent_socket.sendall(msg.encode('utf-8')) agent_response = secure_agent_socket.recv(1024).decode() self.mgr.log.info(f'Received "{agent_response}" from agent on host {self.host}') + self.mgr.cache.sending_agent_message[self.host] = False return except ConnectionError as e: # if it's a connection error, possibly try to connect again. diff --git a/src/pybind/mgr/cephadm/module.py b/src/pybind/mgr/cephadm/module.py index 41476c1c99207..238b1757ec133 100644 --- a/src/pybind/mgr/cephadm/module.py +++ b/src/pybind/mgr/cephadm/module.py @@ -729,6 +729,18 @@ def offline_hosts_remove(self, host: str) -> None: if host in self.offline_hosts: self.offline_hosts.remove(host) + def update_failed_daemon_health_check(self) -> None: + self.remove_health_warning('CEPHADM_FAILED_DAEMON') + failed_daemons = [] + for dd in self.cache.get_daemons(): + if dd.status is not None and dd.status == DaemonDescriptionStatus.error: + failed_daemons.append('daemon %s on %s is in %s state' % ( + dd.name(), dd.hostname, dd.status_desc + )) + if failed_daemons: + self.set_health_warning('CEPHADM_FAILED_DAEMON', f'{len(failed_daemons)} failed cephadm daemon(s)', len( + failed_daemons), failed_daemons) + @staticmethod def can_run() -> Tuple[bool, str]: if asyncssh is not None: @@ -1503,7 +1515,8 @@ def _set_maintenance_healthcheck(self) -> None: self.remove_health_warning('HOST_IN_MAINTENANCE') else: s = "host is" if len(in_maintenance) == 1 else "hosts are" - self.set_health_warning("HOST_IN_MAINTENANCE", f"{len(in_maintenance)} {s} in maintenance mode", 1, [f"{h} is in maintenance" for h in in_maintenance]) + self.set_health_warning("HOST_IN_MAINTENANCE", f"{len(in_maintenance)} {s} in maintenance mode", 1, [ + f"{h} is in maintenance" for h in in_maintenance]) @handle_orch_error @host_exists() diff --git a/src/pybind/mgr/cephadm/serve.py b/src/pybind/mgr/cephadm/serve.py index c93e95cc26c04..f42465647fae7 100644 --- a/src/pybind/mgr/cephadm/serve.py +++ b/src/pybind/mgr/cephadm/serve.py @@ -135,7 +135,8 @@ def _serve_sleep(self) -> None: def _update_paused_health(self) -> None: if self.mgr.paused: - self.mgr.set_health_warning('CEPHADM_PAUSED', 'cephadm background work is paused', 1, ["'ceph orch resume' to resume"]) + self.mgr.set_health_warning('CEPHADM_PAUSED', 'cephadm background work is paused', 1, [ + "'ceph orch resume' to resume"]) else: self.mgr.remove_health_warning('CEPHADM_PAUSED') @@ -379,22 +380,16 @@ def refresh(host: str) -> None: for k in [ 'CEPHADM_HOST_CHECK_FAILED', - 'CEPHADM_FAILED_DAEMON', 'CEPHADM_REFRESH_FAILED', ]: self.mgr.remove_health_warning(k) if bad_hosts: - self.mgr.set_health_warning('CEPHADM_HOST_CHECK_FAILED', f'{len(bad_hosts)} hosts fail cephadm check', len(bad_hosts), bad_hosts) + self.mgr.set_health_warning( + 'CEPHADM_HOST_CHECK_FAILED', f'{len(bad_hosts)} hosts fail cephadm check', len(bad_hosts), bad_hosts) if failures: - self.mgr.set_health_warning('CEPHADM_REFRESH_FAILED', 'failed to probe daemons or devices', len(failures), failures) - failed_daemons = [] - for dd in self.mgr.cache.get_daemons(): - if dd.status is not None and dd.status == DaemonDescriptionStatus.error: - failed_daemons.append('daemon %s on %s is in %s state' % ( - dd.name(), dd.hostname, dd.status_desc - )) - if failed_daemons: - self.mgr.set_health_warning('CEPHADM_FAILED_DAEMON', f'{len(failed_daemons)} failed cephadm daemon(s)', len(failed_daemons), failed_daemons) + self.mgr.set_health_warning( + 'CEPHADM_REFRESH_FAILED', 'failed to probe daemons or devices', len(failures), failures) + self.mgr.update_failed_daemon_health_check() def _check_host(self, host: str) -> Optional[str]: if host not in self.mgr.inventory: @@ -547,9 +542,11 @@ def _check_for_strays(self) -> None: 'stray host %s has %d stray daemons: %s' % ( host, len(missing_names), missing_names)) if self.mgr.warn_on_stray_hosts and host_detail: - self.mgr.set_health_warning('CEPHADM_STRAY_HOST', f'{len(host_detail)} stray host(s) with {host_num_daemons} daemon(s) not managed by cephadm', len(host_detail), host_detail) + self.mgr.set_health_warning( + 'CEPHADM_STRAY_HOST', f'{len(host_detail)} stray host(s) with {host_num_daemons} daemon(s) not managed by cephadm', len(host_detail), host_detail) if self.mgr.warn_on_stray_daemons and daemon_detail: - self.mgr.set_health_warning('CEPHADM_STRAY_DAEMON', f'{len(daemon_detail)} stray daemon(s) not managed by cephadm', len(daemon_detail), daemon_detail) + self.mgr.set_health_warning( + 'CEPHADM_STRAY_DAEMON', f'{len(daemon_detail)} stray daemon(s) not managed by cephadm', len(daemon_detail), daemon_detail) def _apply_all_services(self) -> bool: r = False @@ -624,9 +621,11 @@ def _apply_service_config(self, spec: ServiceSpec) -> None: options_failed_to_set.append(msg) if invalid_config_options: - self.mgr.set_health_warning('CEPHADM_INVALID_CONFIG_OPTION', f'Ignoring {len(invalid_config_options)} invalid config option(s)', len(invalid_config_options), invalid_config_options) + self.mgr.set_health_warning('CEPHADM_INVALID_CONFIG_OPTION', f'Ignoring {len(invalid_config_options)} invalid config option(s)', len( + invalid_config_options), invalid_config_options) if options_failed_to_set: - self.mgr.set_health_warning('CEPHADM_FAILED_SET_OPTION', f'Failed to set {len(options_failed_to_set)} option(s)', len(options_failed_to_set), options_failed_to_set) + self.mgr.set_health_warning('CEPHADM_FAILED_SET_OPTION', f'Failed to set {len(options_failed_to_set)} option(s)', len( + options_failed_to_set), options_failed_to_set) def _apply_service(self, spec: ServiceSpec) -> bool: """ @@ -867,7 +866,8 @@ def update_progress() -> None: daemons.append(sd) if daemon_place_fails: - self.mgr.set_health_warning('CEPHADM_DAEMON_PLACE_FAIL', f'Failed to place {len(daemon_place_fails)} daemon(s)', len(daemon_place_fails), daemon_place_fails) + self.mgr.set_health_warning('CEPHADM_DAEMON_PLACE_FAIL', f'Failed to place {len(daemon_place_fails)} daemon(s)', len( + daemon_place_fails), daemon_place_fails) # remove any? def _ok_to_stop(remove_daemons: List[orchestrator.DaemonDescription]) -> bool: diff --git a/src/pybind/mgr/cephadm/tests/test_cephadm.py b/src/pybind/mgr/cephadm/tests/test_cephadm.py index 63d7eb67fee8c..f3ccf69904f19 100644 --- a/src/pybind/mgr/cephadm/tests/test_cephadm.py +++ b/src/pybind/mgr/cephadm/tests/test_cephadm.py @@ -899,8 +899,10 @@ def test_daemon_place_fail_health_warning(self, _run_cephadm, cephadm_module): assert not r assert cephadm_module.health_checks.get('CEPHADM_DAEMON_PLACE_FAIL') is not None assert cephadm_module.health_checks['CEPHADM_DAEMON_PLACE_FAIL']['count'] == 1 - assert 'Failed to place 1 daemon(s)' in cephadm_module.health_checks['CEPHADM_DAEMON_PLACE_FAIL']['summary'] - assert 'Failed while placing mgr.a on test: fail' in cephadm_module.health_checks['CEPHADM_DAEMON_PLACE_FAIL']['detail'] + assert 'Failed to place 1 daemon(s)' in cephadm_module.health_checks[ + 'CEPHADM_DAEMON_PLACE_FAIL']['summary'] + assert 'Failed while placing mgr.a on test: fail' in cephadm_module.health_checks[ + 'CEPHADM_DAEMON_PLACE_FAIL']['detail'] @mock.patch("cephadm.serve.CephadmServe._run_cephadm") def test_apply_spec_fail_health_warning(self, _run_cephadm, cephadm_module: CephadmOrchestrator): @@ -913,7 +915,8 @@ def test_apply_spec_fail_health_warning(self, _run_cephadm, cephadm_module: Ceph assert cephadm_module.apply_spec_fails assert cephadm_module.health_checks.get('CEPHADM_APPLY_SPEC_FAIL') is not None assert cephadm_module.health_checks['CEPHADM_APPLY_SPEC_FAIL']['count'] == 1 - assert 'Failed to apply 1 service(s)' in cephadm_module.health_checks['CEPHADM_APPLY_SPEC_FAIL']['summary'] + assert 'Failed to apply 1 service(s)' in cephadm_module.health_checks[ + 'CEPHADM_APPLY_SPEC_FAIL']['summary'] @mock.patch("cephadm.module.CephadmOrchestrator.get_foreign_ceph_option") @mock.patch("cephadm.serve.CephadmServe._run_cephadm") @@ -922,11 +925,14 @@ def test_invalid_config_option_health_warning(self, _run_cephadm, get_foreign_ce with with_host(cephadm_module, 'test'): ps = PlacementSpec(hosts=['test:0.0.0.0=a'], count=1) get_foreign_ceph_option.side_effect = KeyError - CephadmServe(cephadm_module)._apply_service_config(ServiceSpec('mgr', placement=ps, config={'test': 'foo'})) + CephadmServe(cephadm_module)._apply_service_config( + ServiceSpec('mgr', placement=ps, config={'test': 'foo'})) assert cephadm_module.health_checks.get('CEPHADM_INVALID_CONFIG_OPTION') is not None assert cephadm_module.health_checks['CEPHADM_INVALID_CONFIG_OPTION']['count'] == 1 - assert 'Ignoring 1 invalid config option(s)' in cephadm_module.health_checks['CEPHADM_INVALID_CONFIG_OPTION']['summary'] - assert 'Ignoring invalid mgr config option test' in cephadm_module.health_checks['CEPHADM_INVALID_CONFIG_OPTION']['detail'] + assert 'Ignoring 1 invalid config option(s)' in cephadm_module.health_checks[ + 'CEPHADM_INVALID_CONFIG_OPTION']['summary'] + assert 'Ignoring invalid mgr config option test' in cephadm_module.health_checks[ + 'CEPHADM_INVALID_CONFIG_OPTION']['detail'] @mock.patch("cephadm.serve.CephadmServe._run_cephadm", _run_cephadm('{}')) @mock.patch("cephadm.services.nfs.NFSService.run_grace_tool", mock.MagicMock()) From 460f6ca0478c8320247f96a8d308197a0d9c432e Mon Sep 17 00:00:00 2001 From: Xuehan Xu Date: Tue, 12 Oct 2021 09:55:21 +0800 Subject: [PATCH 0103/2866] crimson/os/seastore: set ExtentPlacementManager::allocated_to before rolling segments There are circumstances in which a transaction that are supposed to roll the current segment is invalidated after it finished writing and before it rolls the segment. If we don't set ExtentPlacementManager::allocated_to in this situation, another transaction can try to write to the old "allocated_to" position, which would cause an invalid write error Fixes: https://tracker.ceph.com/issues/52894 Signed-off-by: Xuehan Xu --- src/crimson/os/seastore/extent_placement_manager.cc | 12 +++++------- src/crimson/os/seastore/extent_placement_manager.h | 3 +-- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/src/crimson/os/seastore/extent_placement_manager.cc b/src/crimson/os/seastore/extent_placement_manager.cc index 48a89ea273ae9..f3670f3c2eab2 100644 --- a/src/crimson/os/seastore/extent_placement_manager.cc +++ b/src/crimson/os/seastore/extent_placement_manager.cc @@ -69,9 +69,10 @@ SegmentedAllocator::Writer::finish_write( SegmentedAllocator::Writer::write_iertr::future<> SegmentedAllocator::Writer::_write( Transaction& t, - ool_record_t& record, - const record_size_t& record_size) + ool_record_t& record) { + record_size_t record_size = record.get_encoded_record_length(); + allocated_to += record_size.mdlength + record_size.dlength; bufferlist bl = record.encode( record_size, current_segment->segment->get_segment_id(), @@ -165,9 +166,8 @@ SegmentedAllocator::Writer::write( num_extents, current_segment->segment->get_segment_id(), allocated_to); - auto rsize = record.get_encoded_record_length(); return (num_extents ? - _write(t, record, rsize) : + _write(t, record) : write_iertr::now() ).si_then([this]() mutable { return roll_segment(false); @@ -179,7 +179,6 @@ SegmentedAllocator::Writer::write( add_extent_to_write(record, extent); it = extents.erase(it); } - record_size_t rsize = record.get_encoded_record_length(); DEBUGT( "writing {} extents to segment {} at {}", @@ -187,8 +186,7 @@ SegmentedAllocator::Writer::write( record.get_num_extents(), current_segment->segment->get_segment_id(), allocated_to); - allocated_to += rsize.mdlength + rsize.dlength; - return _write(t, record, rsize); + return _write(t, record); } ).si_then([]() -> write_iertr::future { diff --git a/src/crimson/os/seastore/extent_placement_manager.h b/src/crimson/os/seastore/extent_placement_manager.h index d9a15ea5577a8..690d817c2ed31 100644 --- a/src/crimson/os/seastore/extent_placement_manager.h +++ b/src/crimson/os/seastore/extent_placement_manager.h @@ -217,8 +217,7 @@ class SegmentedAllocator : public ExtentAllocator { write_iertr::future<> _write( Transaction& t, - ool_record_t& record, - const record_size_t& record_size); + ool_record_t& record); using roll_segment_ertr = crimson::errorator< crimson::ct_error::input_output_error>; From f7714de294dd7376a9a8ae5131aa429322b459c3 Mon Sep 17 00:00:00 2001 From: Aashish Sharma Date: Fri, 8 Oct 2021 15:33:13 +0530 Subject: [PATCH 0104/2866] mgr/dashboard: monitoring: grafonnet refactoring for hosts dashboards This PR intends to refactor hosts dashboards using grafonnet Fixes:https://tracker.ceph.com/issues/52777 Signed-off-by: Aashish Sharma --- .../grafana/dashboards/host-details.json | 2366 ++++++++--------- .../grafana/dashboards/hosts-overview.json | 1658 ++++++------ .../jsonnet/grafana_dashboards.jsonnet | 176 +- 3 files changed, 2150 insertions(+), 2050 deletions(-) diff --git a/monitoring/grafana/dashboards/host-details.json b/monitoring/grafana/dashboards/host-details.json index b002ad2f09dd8..1d071d4ec336d 100644 --- a/monitoring/grafana/dashboards/host-details.json +++ b/monitoring/grafana/dashboards/host-details.json @@ -1,1215 +1,1193 @@ { - "__inputs": [], - "__requires": [ - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "5.3.2" - }, - { - "type": "panel", - "id": "graph", - "name": "Graph", - "version": "5.0.0" - }, - { - "type": "panel", - "id": "singlestat", - "name": "Singlestat", - "version": "5.0.0" - } - ], - "annotations": { - "list": [ + "__inputs": [ ], + "__requires": [ { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": false, - "gnetId": null, - "graphTooltip": 0, - "id": null, - "iteration": 1615564911000, - "links": [], - "panels": [ - { - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 16, - "title": "$ceph_hosts System Overview", - "type": "row" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "datasource": "$datasource", - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 0, - "y": 1 - }, - "height": "160", - "id": 1, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": "", - "minSpan": 4, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "count(sum by (ceph_daemon) (ceph_osd_metadata{hostname='$ceph_hosts'}))", - "format": "time_series", - "intervalFactor": 2, - "refId": "A", - "step": 40, - "textEditor": true - } - ], - "thresholds": "", - "title": "OSDs", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "aliasColors": { - "interrupt": "#447EBC", - "steal": "#6D1F62", - "system": "#890F02", - "user": "#3F6833", - "wait": "#C15C17" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "description": "Shows the CPU breakdown. When multiple servers are selected, only the first host's cpu data is shown", - "fill": 1, - "gridPos": { - "h": 10, - "w": 6, - "x": 3, - "y": 1 - }, - "id": 9, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "minSpan": 12, - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum by (mode) (\n irate(node_cpu{instance=~\"($ceph_hosts)([\\\\.:].*)?\", mode=~\"(irq|nice|softirq|steal|system|user|iowait)\"}[1m]) or\n irate(node_cpu_seconds_total{instance=~\"($ceph_hosts)([\\\\.:].*)?\", mode=~\"(irq|nice|softirq|steal|system|user|iowait)\"}[1m])\n) / scalar(\n sum(irate(node_cpu{instance=~\"($ceph_hosts)([\\\\.:].*)?\"}[1m]) or\n irate(node_cpu_seconds_total{instance=~\"($ceph_hosts)([\\\\.:].*)?\"}[1m]))\n) * 100", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{mode}}", - "refId": "A", - "step": 10, - "textEditor": true - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "CPU Utilization", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percent", - "label": "% Utilization", - "logBase": 1, - "max": "100", - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": { - "Available": "#508642", - "Free": "#508642", - "Total": "#bf1b00", - "Used": "#bf1b00", - "total": "#bf1b00", - "used": "#0a50a1" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "gridPos": { - "h": 10, - "w": 6, - "x": 9, - "y": 1 - }, - "id": 14, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "total", - "color": "#bf1b00", - "fill": 0, - "linewidth": 2, - "stack": false - } - ], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "(node_memory_MemTotal{instance=~\"$ceph_hosts([\\\\.:].*)?\"} or node_memory_MemTotal_bytes{instance=~\"$ceph_hosts([\\\\.:].*)?\"})- (\n (node_memory_MemFree{instance=~\"$ceph_hosts([\\\\.:].*)?\"} or node_memory_MemFree_bytes{instance=~\"$ceph_hosts([\\\\.:].*)?\"}) + \n (node_memory_Cached{instance=~\"$ceph_hosts([\\\\.:].*)?\"} or node_memory_Cached_bytes{instance=~\"$ceph_hosts([\\\\.:].*)?\"}) + \n (node_memory_Buffers{instance=~\"$ceph_hosts([\\\\.:].*)?\"} or node_memory_Buffers_bytes{instance=~\"$ceph_hosts([\\\\.:].*)?\"}) +\n (node_memory_Slab{instance=~\"$ceph_hosts([\\\\.:].*)?\"} or node_memory_Slab_bytes{instance=~\"$ceph_hosts([\\\\.:].*)?\"})\n )\n \n", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "used", - "refId": "D" - }, - { - "expr": "node_memory_MemFree{instance=~\"$ceph_hosts([\\\\.:].*)?\"} or node_memory_MemFree_bytes{instance=~\"$ceph_hosts([\\\\.:].*)?\"} ", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "Free", - "refId": "A" - }, - { - "expr": "(node_memory_Cached{instance=~\"$ceph_hosts([\\\\.:].*)?\"} or node_memory_Cached_bytes{instance=~\"$ceph_hosts([\\\\.:].*)?\"}) + \n(node_memory_Buffers{instance=~\"$ceph_hosts([\\\\.:].*)?\"} or node_memory_Buffers_bytes{instance=~\"$ceph_hosts([\\\\.:].*)?\"}) +\n(node_memory_Slab{instance=~\"$ceph_hosts([\\\\.:].*)?\"} or node_memory_Slab_bytes{instance=~\"$ceph_hosts([\\\\.:].*)?\"}) \n", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "buffers/cache", - "refId": "C" - }, - { - "expr": "node_memory_MemTotal{instance=~\"$ceph_hosts([\\\\.:].*)?\"} or node_memory_MemTotal_bytes{instance=~\"$ceph_hosts([\\\\.:].*)?\"} ", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "total", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "RAM Usage", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" + "id": "grafana", + "name": "Grafana", + "type": "grafana", + "version": "5.3.2" }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "label": "RAM used", - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "description": "Show the network load (rx,tx) across all interfaces (excluding loopback 'lo')", - "fill": 0, - "gridPos": { - "h": 10, - "w": 6, - "x": 15, - "y": 1 - }, - "id": 10, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "hideZero": true, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "minSpan": 12, - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "/.*tx/", - "transform": "negative-Y" - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum by (device) (\n irate(node_network_receive_bytes{instance=~\"($ceph_hosts)([\\\\.:].*)?\",device!=\"lo\"}[1m]) or \n irate(node_network_receive_bytes_total{instance=~\"($ceph_hosts)([\\\\.:].*)?\",device!=\"lo\"}[1m])\n)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{device}}.rx", - "refId": "A", - "step": 10, - "textEditor": true - }, - { - "expr": "sum by (device) (\n irate(node_network_transmit_bytes{instance=~\"($ceph_hosts)([\\\\.:].*)?\",device!=\"lo\"}[1m]) or\n irate(node_network_transmit_bytes_total{instance=~\"($ceph_hosts)([\\\\.:].*)?\",device!=\"lo\"}[1m])\n)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{device}}.tx", - "refId": "B", - "step": 10 - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Network Load", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "decbytes", - "label": "Send (-) / Receive (+)", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 0, - "gridPos": { - "h": 5, - "w": 3, - "x": 21, - "y": 1 - }, - "hideTimeOverride": true, - "id": 18, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "/.*tx/", - "transform": "negative-Y" - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "irate(node_network_receive_drop{instance=~\"$ceph_hosts([\\\\.:].*)?\"}[1m]) or irate(node_network_receive_drop_total{instance=~\"$ceph_hosts([\\\\.:].*)?\"}[1m])", - "format": "time_series", - "instant": false, - "intervalFactor": 1, - "legendFormat": "{{device}}.rx", - "refId": "A" - }, - { - "expr": "irate(node_network_transmit_drop{instance=~\"$ceph_hosts([\\\\.:].*)?\"}[1m]) or irate(node_network_transmit_drop_total{instance=~\"$ceph_hosts([\\\\.:].*)?\"}[1m])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{device}}.tx", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Network drop rate", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "pps", - "label": "Send (-) / Receive (+)", - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "datasource": "$datasource", - "decimals": 0, - "description": "Each OSD consists of a Journal/WAL partition and a data partition. The RAW Capacity shown is the sum of the data partitions across all OSDs on the selected OSD hosts.", - "format": "bytes", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 5, - "w": 3, - "x": 0, - "y": 6 - }, - "height": "160", - "id": 2, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": "", - "minSpan": 4, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "sum(ceph_osd_stat_bytes and on (ceph_daemon) ceph_disk_occupation{instance=~\"($ceph_hosts)([\\\\.:].*)?\"})", - "format": "time_series", - "intervalFactor": 2, - "refId": "A", - "step": 40, - "textEditor": true - } - ], - "thresholds": "", - "title": "Raw Capacity", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 0, - "gridPos": { - "h": 5, - "w": 3, - "x": 21, - "y": 6 - }, - "hideTimeOverride": true, - "id": 19, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "/.*tx/", - "transform": "negative-Y" - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "irate(node_network_receive_errs{instance=~\"$ceph_hosts([\\\\.:].*)?\"}[1m]) or irate(node_network_receive_errs_total{instance=~\"$ceph_hosts([\\\\.:].*)?\"}[1m])", - "format": "time_series", - "instant": false, - "intervalFactor": 1, - "legendFormat": "{{device}}.rx", - "refId": "A" - }, - { - "expr": "irate(node_network_transmit_errs{instance=~\"$ceph_hosts([\\\\.:].*)?\"}[1m]) or irate(node_network_transmit_errs_total{instance=~\"$ceph_hosts([\\\\.:].*)?\"}[1m])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{device}}.tx", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Network error rate", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "pps", - "label": "Send (-) / Receive (+)", - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 11 - }, - "id": 12, - "panels": [], - "repeat": null, - "title": "OSD Disk Performance Statistics", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "description": "For any OSD devices on the host, this chart shows the iops per physical device. Each device is shown by it's name and corresponding OSD id value", - "fill": 1, - "gridPos": { - "h": 9, - "w": 11, - "x": 0, - "y": 12 - }, - "id": 6, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "minSpan": 12, - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "/.*reads/", - "transform": "negative-Y" - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "label_replace(\n (\n irate(node_disk_writes_completed{instance=~\"($ceph_hosts)([\\\\.:].*)?\"}[5m]) or\n irate(node_disk_writes_completed_total{instance=~\"($ceph_hosts)([\\\\.:].*)?\"}[5m])\n ),\n \"instance\",\n \"$1\",\n \"instance\",\n \"([^:.]*).*\"\n)\n* on(instance, device, ceph_daemon) group_left\n label_replace(\n label_replace(\n ceph_disk_occupation,\n \"device\",\n \"$1\",\n \"device\",\n \"/dev/(.*)\"\n ),\n \"instance\",\n \"$1\",\n \"instance\",\n \"([^:.]*).*\"\n )", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{device}}({{ceph_daemon}}) writes", - "refId": "A", - "step": 10, - "textEditor": true - }, - { - "expr": "label_replace(\n (irate(node_disk_reads_completed{instance=~\"($ceph_hosts)([\\\\.:].*)?\"}[5m]) or irate(node_disk_reads_completed_total{instance=~\"($ceph_hosts)([\\\\.:].*)?\"}[5m])),\n \"instance\",\n \"$1\",\n \"instance\",\n \"([^:.]*).*\"\n)\n* on(instance, device, ceph_daemon) group_left\n label_replace(\n label_replace(\n ceph_disk_occupation,\n \"device\",\n \"$1\",\n \"device\",\n \"/dev/(.*)\"\n ),\n \"instance\",\n \"$1\",\n \"instance\",\n \"([^:.]*).*\"\n )", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{device}}({{ceph_daemon}}) reads", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "$ceph_hosts Disk IOPS", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] + { + "id": "graph", + "name": "Graph", + "type": "panel", + "version": "5.0.0" }, - "yaxes": [ - { - "format": "ops", - "label": "Read (-) / Write (+)", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null + { + "id": "singlestat", + "name": "Singlestat", + "type": "panel", + "version": "5.0.0" } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "description": "For OSD hosts, this chart shows the disk bandwidth (read bytes/sec + write bytes/sec) of the physical OSD device. Each device is shown by device name, and corresponding OSD id", - "fill": 1, - "gridPos": { - "h": 9, - "w": 11, - "x": 12, - "y": 12 - }, - "id": 8, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "minSpan": 12, - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "/.*read/", - "transform": "negative-Y" - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "label_replace((irate(node_disk_bytes_written{instance=~\"($ceph_hosts)([\\\\.:].*)?\"}[5m]) or irate(node_disk_written_bytes_total{instance=~\"($ceph_hosts)([\\\\.:].*)?\"}[5m])), \"instance\", \"$1\", \"instance\", \"([^:.]*).*\") * on(instance, device, ceph_daemon) group_left label_replace(label_replace(ceph_disk_occupation, \"device\", \"$1\", \"device\", \"/dev/(.*)\"), \"instance\", \"$1\", \"instance\", \"([^:.]*).*\")", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{device}}({{ceph_daemon}}) write", - "refId": "B" - }, - { - "expr": "label_replace((irate(node_disk_bytes_read{instance=~\"($ceph_hosts)([\\\\.:].*)?\"}[5m]) or irate(node_disk_read_bytes_total{instance=~\"($ceph_hosts)([\\\\.:].*)?\"}[5m])), \"instance\", \"$1\", \"instance\", \"([^:.]*).*\") * on(instance, device, ceph_daemon) group_left label_replace(label_replace(ceph_disk_occupation, \"device\", \"$1\", \"device\", \"/dev/(.*)\"), \"instance\", \"$1\", \"instance\", \"([^:.]*).*\")", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{device}}({{ceph_daemon}}) read", - "refId": "C" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "$ceph_hosts Throughput by Disk", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "showIn": 0, + "tags": [ ], + "type": "dashboard" + } + ] + }, + "description": "", + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ ], + "panels": [ + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "panels": [ ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "$ceph_hosts System Overview", + "titleSize": "h6", + "type": "row" }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 0, + "y": 1 + }, + "id": 3, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "count(sum by (ceph_daemon) (ceph_osd_metadata{hostname='$ceph_hosts'}))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "OSDs", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" }, - "yaxes": [ - { - "format": "Bps", - "label": "Read (-) / Write (+)", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "description": "For OSD hosts, this chart shows the latency at the physical drive. Each drive is shown by device name, with it's corresponding OSD id", - "fill": 1, - "gridPos": { - "h": 9, - "w": 11, - "x": 0, - "y": 21 + { + "aliasColors": { + "interrupt": "#447EBC", + "steal": "#6D1F62", + "system": "#890F02", + "user": "#3F6833", + "wait": "#C15C17" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Shows the CPU breakdown. When multiple servers are selected, only the first host's cpu data is shown", + "fill": 1, + "gridPos": { + "h": 10, + "w": 6, + "x": 3, + "y": 1 + }, + "id": 4, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (mode) (\n irate(node_cpu{instance=~\"($ceph_hosts)([\\\\.:].*)?\", mode=~\"(irq|nice|softirq|steal|system|user|iowait)\"}[1m]) or\n irate(node_cpu_seconds_total{instance=~\"($ceph_hosts)([\\\\.:].*)?\", mode=~\"(irq|nice|softirq|steal|system|user|iowait)\"}[1m])\n) / scalar(\n sum(irate(node_cpu{instance=~\"($ceph_hosts)([\\\\.:].*)?\"}[1m]) or\n irate(node_cpu_seconds_total{instance=~\"($ceph_hosts)([\\\\.:].*)?\"}[1m]))\n) * 100", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{mode}}", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "percent", + "label": "% Utilization", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] }, - "id": 7, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + { + "aliasColors": { + "Available": "#508642", + "Free": "#508642", + "Total": "#bf1b00", + "Used": "#bf1b00", + "total": "#bf1b00", + "used": "#0a50a1" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "", + "fill": 1, + "gridPos": { + "h": 10, + "w": 6, + "x": 9, + "y": 1 + }, + "id": 5, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "total", + "color": "#bf1b00", + "fill": 0, + "linewidth": 2, + "stack": false + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_memory_MemFree{instance=~\"$ceph_hosts([\\\\.:].*)?\"} or node_memory_MemFree_bytes{instance=~\"$ceph_hosts([\\\\.:].*)?\"} ", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Free", + "refId": "A" + }, + { + "expr": "node_memory_MemTotal{instance=~\"$ceph_hosts([\\\\.:].*)?\"} or node_memory_MemTotal_bytes{instance=~\"$ceph_hosts([\\\\.:].*)?\"} ", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "total", + "refId": "B" + }, + { + "expr": "(node_memory_Cached{instance=~\"$ceph_hosts([\\\\.:].*)?\"} or node_memory_Cached_bytes{instance=~\"$ceph_hosts([\\\\.:].*)?\"}) + \n(node_memory_Buffers{instance=~\"$ceph_hosts([\\\\.:].*)?\"} or node_memory_Buffers_bytes{instance=~\"$ceph_hosts([\\\\.:].*)?\"}) +\n(node_memory_Slab{instance=~\"$ceph_hosts([\\\\.:].*)?\"} or node_memory_Slab_bytes{instance=~\"$ceph_hosts([\\\\.:].*)?\"}) \n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "buffers/cache", + "refId": "C" + }, + { + "expr": "(node_memory_MemTotal{instance=~\"$ceph_hosts([\\\\.:].*)?\"} or node_memory_MemTotal_bytes{instance=~\"$ceph_hosts([\\\\.:].*)?\"})- (\n (node_memory_MemFree{instance=~\"$ceph_hosts([\\\\.:].*)?\"} or node_memory_MemFree_bytes{instance=~\"$ceph_hosts([\\\\.:].*)?\"}) + \n (node_memory_Cached{instance=~\"$ceph_hosts([\\\\.:].*)?\"} or node_memory_Cached_bytes{instance=~\"$ceph_hosts([\\\\.:].*)?\"}) + \n (node_memory_Buffers{instance=~\"$ceph_hosts([\\\\.:].*)?\"} or node_memory_Buffers_bytes{instance=~\"$ceph_hosts([\\\\.:].*)?\"}) +\n (node_memory_Slab{instance=~\"$ceph_hosts([\\\\.:].*)?\"} or node_memory_Slab_bytes{instance=~\"$ceph_hosts([\\\\.:].*)?\"})\n )\n \n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "used", + "refId": "D" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "RAM Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "bytes", + "label": "RAM used", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] }, - "lines": true, - "linewidth": 1, - "links": [], - "minSpan": 12, - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "max by(instance,device) (label_replace((irate(node_disk_write_time_seconds_total{ instance=~\"($ceph_hosts)([\\\\.:].*)?\"}[5m]) ) / clamp_min(irate(node_disk_writes_completed_total{ instance=~\"($ceph_hosts)([\\\\.:].*)?\"}[5m]), 0.001) or (irate(node_disk_read_time_seconds_total{ instance=~\"($ceph_hosts)([\\\\.:].*)?\"}[5m]) ) / clamp_min(irate(node_disk_reads_completed_total{ instance=~\"($ceph_hosts)([\\\\.:].*)?\"}[5m]), 0.001), \"instance\", \"$1\", \"instance\", \"([^:.]*).*\")) * on(instance, device, ceph_daemon) group_left label_replace(label_replace(ceph_disk_occupation{instance=~\"($ceph_hosts)([\\\\.:].*)?\"}, \"device\", \"$1\", \"device\", \"/dev/(.*)\"), \"instance\", \"$1\", \"instance\", \"([^:.]*).*\")", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{device}}({{ceph_daemon}})", - "refId": "D" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "$ceph_hosts Disk Latency", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Show the network load (rx,tx) across all interfaces (excluding loopback 'lo')", + "fill": 1, + "gridPos": { + "h": 10, + "w": 6, + "x": 15, + "y": 1 + }, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "/.*tx/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (device) (\n irate(node_network_receive_bytes{instance=~\"($ceph_hosts)([\\\\.:].*)?\",device!=\"lo\"}[1m]) or \n irate(node_network_receive_bytes_total{instance=~\"($ceph_hosts)([\\\\.:].*)?\",device!=\"lo\"}[1m])\n)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}}.rx", + "refId": "A" + }, + { + "expr": "sum by (device) (\n irate(node_network_transmit_bytes{instance=~\"($ceph_hosts)([\\\\.:].*)?\",device!=\"lo\"}[1m]) or\n irate(node_network_transmit_bytes_total{instance=~\"($ceph_hosts)([\\\\.:].*)?\",device!=\"lo\"}[1m])\n)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}}.tx", + "refId": "B" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Network Load", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "decbytes", + "label": "Send (-) / Receive (+)", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "", + "fill": 1, + "gridPos": { + "h": 5, + "w": 3, + "x": 21, + "y": 1 + }, + "id": 7, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "/.*tx/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_network_receive_drop{instance=~\"$ceph_hosts([\\\\.:].*)?\"}[1m]) or irate(node_network_receive_drop_total{instance=~\"$ceph_hosts([\\\\.:].*)?\"}[1m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}}.rx", + "refId": "A" + }, + { + "expr": "irate(node_network_transmit_drop{instance=~\"$ceph_hosts([\\\\.:].*)?\"}[1m]) or irate(node_network_transmit_drop_total{instance=~\"$ceph_hosts([\\\\.:].*)?\"}[1m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}}.tx", + "refId": "B" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Network drop rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "pps", + "label": "Send (-) / Receive (+)", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] }, - "yaxes": [ - { - "format": "s", - "label": "", - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "description": "Show disk utilization % (util) of any OSD devices on the host by the physical device name and associated OSD id.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 9, - "w": 11, - "x": 12, - "y": 21 + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "description": "Each OSD consists of a Journal/WAL partition and a data partition. The RAW Capacity shown is the sum of the data partitions across all OSDs on the selected OSD hosts.", + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 0, + "y": 6 + }, + "id": 8, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(ceph_osd_stat_bytes and on (ceph_daemon) ceph_disk_occupation{instance=~\"($ceph_hosts)([\\\\.:].*)?\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Raw Capacity", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" }, - "id": 5, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "", + "fill": 1, + "gridPos": { + "h": 5, + "w": 3, + "x": 21, + "y": 6 + }, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "/.*tx/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_network_receive_errs{instance=~\"$ceph_hosts([\\\\.:].*)?\"}[1m]) or irate(node_network_receive_errs_total{instance=~\"$ceph_hosts([\\\\.:].*)?\"}[1m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}}.rx", + "refId": "A" + }, + { + "expr": "irate(node_network_transmit_errs{instance=~\"$ceph_hosts([\\\\.:].*)?\"}[1m]) or irate(node_network_transmit_errs_total{instance=~\"$ceph_hosts([\\\\.:].*)?\"}[1m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}}.tx", + "refId": "B" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Network error rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "pps", + "label": "Send (-) / Receive (+)", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] }, - "lines": true, - "linewidth": 1, - "links": [], - "maxPerRow": 2, - "nullPointMode": "connected", - "options": { - "dataLinks": [] + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 10, + "panels": [ ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "OSD Disk Performance Statistics", + "titleSize": "h6", + "type": "row" }, - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "label_replace(((irate(node_disk_io_time_ms{instance=~\"($ceph_hosts)([\\\\.:].*)?\"}[5m]) / 10 ) or irate(node_disk_io_time_seconds_total{instance=~\"($ceph_hosts)([\\\\.:].*)?\"}[5m]) * 100), \"instance\", \"$1\", \"instance\", \"([^:.]*).*\") * on(instance, device, ceph_daemon) group_left label_replace(label_replace(ceph_disk_occupation{instance=~\"($ceph_hosts)([\\\\.:].*)?\"}, \"device\", \"$1\", \"device\", \"/dev/(.*)\"), \"instance\", \"$1\", \"instance\", \"([^:.]*).*\")", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{device}}({{ceph_daemon}})", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "$ceph_hosts Disk utilization", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "For any OSD devices on the host, this chart shows the iops per physical device. Each device is shown by it's name and corresponding OSD id value", + "fill": 1, + "gridPos": { + "h": 9, + "w": 11, + "x": 0, + "y": 12 + }, + "id": 11, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "/.*reads/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "label_replace(\n (\n irate(node_disk_writes_completed{instance=~\"($ceph_hosts)([\\\\.:].*)?\"}[5m]) or\n irate(node_disk_writes_completed_total{instance=~\"($ceph_hosts)([\\\\.:].*)?\"}[5m])\n ),\n \"instance\",\n \"$1\",\n \"instance\",\n \"([^:.]*).*\"\n)\n* on(instance, device, ceph_daemon) group_left\n label_replace(\n label_replace(\n ceph_disk_occupation,\n \"device\",\n \"$1\",\n \"device\",\n \"/dev/(.*)\"\n ),\n \"instance\",\n \"$1\",\n \"instance\",\n \"([^:.]*).*\"\n )", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}}({{ceph_daemon}}) writes", + "refId": "A" + }, + { + "expr": "label_replace(\n (irate(node_disk_reads_completed{instance=~\"($ceph_hosts)([\\\\.:].*)?\"}[5m]) or irate(node_disk_reads_completed_total{instance=~\"($ceph_hosts)([\\\\.:].*)?\"}[5m])),\n \"instance\",\n \"$1\",\n \"instance\",\n \"([^:.]*).*\"\n)\n* on(instance, device, ceph_daemon) group_left\n label_replace(\n label_replace(\n ceph_disk_occupation,\n \"device\",\n \"$1\",\n \"device\",\n \"/dev/(.*)\"\n ),\n \"instance\",\n \"$1\",\n \"instance\",\n \"([^:.]*).*\"\n )", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}}({{ceph_daemon}}) reads", + "refId": "B" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "$ceph_hosts Disk IOPS", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "ops", + "label": "Read (-) / Write (+)", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "For OSD hosts, this chart shows the disk bandwidth (read bytes/sec + write bytes/sec) of the physical OSD device. Each device is shown by device name, and corresponding OSD id", + "fill": 1, + "gridPos": { + "h": 9, + "w": 11, + "x": 12, + "y": 12 + }, + "id": 12, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "/.*read/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "label_replace((irate(node_disk_bytes_written{instance=~\"($ceph_hosts)([\\\\.:].*)?\"}[5m]) or irate(node_disk_written_bytes_total{instance=~\"($ceph_hosts)([\\\\.:].*)?\"}[5m])), \"instance\", \"$1\", \"instance\", \"([^:.]*).*\") * on(instance, device, ceph_daemon) group_left label_replace(label_replace(ceph_disk_occupation, \"device\", \"$1\", \"device\", \"/dev/(.*)\"), \"instance\", \"$1\", \"instance\", \"([^:.]*).*\")", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}}({{ceph_daemon}}) write", + "refId": "A" + }, + { + "expr": "label_replace((irate(node_disk_bytes_read{instance=~\"($ceph_hosts)([\\\\.:].*)?\"}[5m]) or irate(node_disk_read_bytes_total{instance=~\"($ceph_hosts)([\\\\.:].*)?\"}[5m])), \"instance\", \"$1\", \"instance\", \"([^:.]*).*\") * on(instance, device, ceph_daemon) group_left label_replace(label_replace(ceph_disk_occupation, \"device\", \"$1\", \"device\", \"/dev/(.*)\"), \"instance\", \"$1\", \"instance\", \"([^:.]*).*\")", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}}({{ceph_daemon}}) read", + "refId": "B" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "$ceph_hosts Throughput by Disk", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "Bps", + "label": "Read (-) / Write (+)", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] }, - "yaxes": [ - { - "format": "percent", - "label": "%Util", - "logBase": 1, - "max": "100", - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - } - ], - "refresh": "10s", - "schemaVersion": 16, - "style": "dark", - "tags": [ - "overview" - ], - "templating": { - "list": [ { - "current": { - "text": "default", - "value": "default" - }, - "hide": 0, - "label": "Data Source", - "name": "datasource", - "options": [], - "query": "prometheus", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "type": "datasource" + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "For OSD hosts, this chart shows the latency at the physical drive. Each drive is shown by device name, with it's corresponding OSD id", + "fill": 1, + "gridPos": { + "h": 9, + "w": 11, + "x": 0, + "y": 21 + }, + "id": 13, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "max by(instance,device) (label_replace((irate(node_disk_write_time_seconds_total{ instance=~\"($ceph_hosts)([\\\\.:].*)?\"}[5m]) ) / clamp_min(irate(node_disk_writes_completed_total{ instance=~\"($ceph_hosts)([\\\\.:].*)?\"}[5m]), 0.001) or (irate(node_disk_read_time_seconds_total{ instance=~\"($ceph_hosts)([\\\\.:].*)?\"}[5m]) ) / clamp_min(irate(node_disk_reads_completed_total{ instance=~\"($ceph_hosts)([\\\\.:].*)?\"}[5m]), 0.001), \"instance\", \"$1\", \"instance\", \"([^:.]*).*\")) * on(instance, device, ceph_daemon) group_left label_replace(label_replace(ceph_disk_occupation{instance=~\"($ceph_hosts)([\\\\.:].*)?\"}, \"device\", \"$1\", \"device\", \"/dev/(.*)\"), \"instance\", \"$1\", \"instance\", \"([^:.]*).*\")", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}}({{ceph_daemon}})", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "$ceph_hosts Disk Latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "s", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] }, { - "allValue": null, - "current": {}, - "datasource": "$datasource", - "hide": 0, - "includeAll": false, - "label": "Hostname", - "multi": false, - "name": "ceph_hosts", - "options": [], - "query": "label_values(node_scrape_collector_success, instance) ", - "refresh": 1, - "regex": "([^.:]*).*", - "skipUrlSync": false, - "sort": 3, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Show disk utilization % (util) of any OSD devices on the host by the physical device name and associated OSD id.", + "fill": 1, + "gridPos": { + "h": 9, + "w": 11, + "x": 12, + "y": 21 + }, + "id": 14, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "label_replace(((irate(node_disk_io_time_ms{instance=~\"($ceph_hosts)([\\\\.:].*)?\"}[5m]) / 10 ) or irate(node_disk_io_time_seconds_total{instance=~\"($ceph_hosts)([\\\\.:].*)?\"}[5m]) * 100), \"instance\", \"$1\", \"instance\", \"([^:.]*).*\") * on(instance, device, ceph_daemon) group_left label_replace(label_replace(ceph_disk_occupation{instance=~\"($ceph_hosts)([\\\\.:].*)?\"}, \"device\", \"$1\", \"device\", \"/dev/(.*)\"), \"instance\", \"$1\", \"instance\", \"([^:.]*).*\")", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{device}}({{ceph_daemon}})", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "$ceph_hosts Disk utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "percent", + "label": "%Util", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "", - "title": "Host Details", - "uid": "rtOg0AiWz", - "version": 4 + ], + "refresh": "10s", + "rows": [ ], + "schemaVersion": 16, + "style": "dark", + "tags": [ + "overview" + ], + "templating": { + "list": [ + { + "current": { + "text": "default", + "value": "default" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "Hostname", + "multi": false, + "name": "ceph_hosts", + "options": [ ], + "query": "label_values(node_scrape_collector_success, instance) ", + "refresh": 1, + "regex": "([^.:]*).*", + "sort": 3, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Host Details", + "uid": "rtOg0AiWz", + "version": 0 } diff --git a/monitoring/grafana/dashboards/hosts-overview.json b/monitoring/grafana/dashboards/hosts-overview.json index b179d5717d536..b1e2d857dd142 100644 --- a/monitoring/grafana/dashboards/hosts-overview.json +++ b/monitoring/grafana/dashboards/hosts-overview.json @@ -1,852 +1,844 @@ { - "__inputs": [], - "__requires": [ - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "5.3.2" - }, - { - "type": "panel", - "id": "graph", - "name": "Graph", - "version": "5.0.0" - }, - { - "type": "panel", - "id": "singlestat", - "name": "Singlestat", - "version": "5.0.0" - } - ], - "annotations": { - "list": [ + "__inputs": [ ], + "__requires": [ { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": false, - "gnetId": null, - "graphTooltip": 0, - "id": null, - "iteration": 1557393917915, - "links": [], - "panels": [ - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "$datasource", - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 0, - "y": 0 - }, - "id": 5, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "count(sum by (hostname) (ceph_osd_metadata))", - "format": "time_series", - "instant": true, - "intervalFactor": 1, - "refId": "A" - } - ], - "thresholds": "", - "title": "OSD Hosts", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "$datasource", - "description": "Average CPU busy across all hosts (OSD, RGW, MON etc) within the cluster", - "decimals": 2, - "format": "percentunit", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 4, - "y": 0 - }, - "id": 6, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "avg(\n 1 - (\n avg by(instance) \n (irate(node_cpu_seconds_total{mode='idle',instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\"}[1m]) or\n irate(node_cpu{mode='idle',instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\"}[1m]))\n )\n )", - "format": "time_series", - "instant": true, - "intervalFactor": 1, - "refId": "A" - } - ], - "thresholds": "", - "title": "AVG CPU Busy", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "$datasource", - "description": "Average Memory Usage across all hosts in the cluster (excludes buffer/cache usage)", - "decimals": 2, - "format": "percentunit", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 8, - "y": 0 - }, - "id": 9, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "avg (((node_memory_MemTotal{instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\"} or node_memory_MemTotal_bytes{instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\"})- (\n (node_memory_MemFree{instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\"} or node_memory_MemFree_bytes{instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\"}) + \n (node_memory_Cached{instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\"} or node_memory_Cached_bytes{instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\"}) + \n (node_memory_Buffers{instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\"} or node_memory_Buffers_bytes{instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\"}) +\n (node_memory_Slab{instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\"} or node_memory_Slab_bytes{instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\"})\n )) /\n (node_memory_MemTotal{instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\"} or node_memory_MemTotal_bytes{instance=~\"($osd_hosts|$rgw_hosts|$mon_hosts|$mds_hosts).*\"} ))", - "format": "time_series", - "instant": true, - "intervalFactor": 1, - "refId": "A" - } - ], - "thresholds": "", - "title": "AVG RAM Utilization", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "$datasource", - "description": "IOPS Load at the device as reported by the OS on all OSD hosts", - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 12, - "y": 0 - }, - "id": 2, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "sum ((irate(node_disk_reads_completed{instance=~\"($osd_hosts).*\"}[5m]) or irate(node_disk_reads_completed_total{instance=~\"($osd_hosts).*\"}[5m]) ) + \n(irate(node_disk_writes_completed{instance=~\"($osd_hosts).*\"}[5m]) or irate(node_disk_writes_completed_total{instance=~\"($osd_hosts).*\"}[5m])))", - "format": "time_series", - "instant": true, - "intervalFactor": 1, - "refId": "A" - } - ], - "thresholds": "", - "title": "Physical IOPS", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "$datasource", - "description": "Average Disk utilization for all OSD data devices (i.e. excludes journal/WAL)", - "format": "percent", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 16, - "y": 0 - }, - "id": 20, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr" : "avg (\n label_replace((irate(node_disk_io_time_ms[5m]) / 10 ) or\n (irate(node_disk_io_time_seconds_total[5m]) * 100), \"instance\", \"$1\", \"instance\", \"([^.:]*).*\"\n ) *\n on(instance, device, ceph_daemon) label_replace(label_replace(ceph_disk_occupation{instance=~\"($osd_hosts).*\"}, \"device\", \"$1\", \"device\", \"/dev/(.*)\"), \"instance\", \"$1\", \"instance\", \"([^.:]*).*\")\n)", - "format": "time_series", - "instant": true, - "intervalFactor": 1, - "refId": "A" - } - ], - "thresholds": "", - "title": "AVG Disk Utilization", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "$datasource", - "decimals": 0, - "description": "Total send/receive network load across all hosts in the ceph cluster", - "format": "bytes", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 20, - "y": 0 + "id": "grafana", + "name": "Grafana", + "type": "grafana", + "version": "5.3.2" }, - "id": 18, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "sum (\n irate(node_network_receive_bytes{instance=~\"($osd_hosts|mon_hosts|mds_hosts|rgw_hosts).*\",device!=\"lo\"}[1m]) or\n irate(node_network_receive_bytes_total{instance=~\"($osd_hosts|mon_hosts|mds_hosts|rgw_hosts).*\",device!=\"lo\"}[1m])\n ) +\nsum (\n irate(node_network_transmit_bytes{instance=~\"($osd_hosts|mon_hosts|mds_hosts|rgw_hosts).*\",device!=\"lo\"}[1m]) or\n irate(node_network_transmit_bytes_total{instance=~\"($osd_hosts|mon_hosts|mds_hosts|rgw_hosts).*\",device!=\"lo\"}[1m])\n )", - "format": "time_series", - "instant": true, - "intervalFactor": 1, - "refId": "A" - } - ], - "thresholds": "", - "title": "Network Load", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "description": "Show the top 10 busiest hosts by cpu", - "fill": 0, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 5 - }, - "id": 13, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "topk(10,100 * ( 1 - (\n avg by(instance) \n (irate(node_cpu_seconds_total{mode='idle',instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\"}[1m]) or\n irate(node_cpu{mode='idle',instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\"}[1m]))\n )\n )\n)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "CPU Busy - Top 10 Hosts", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] + { + "id": "graph", + "name": "Graph", + "type": "panel", + "version": "5.0.0" }, - "yaxes": [ - { - "decimals": 1, - "format": "percent", - "label": null, - "logBase": 1, - "max": "100", - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null + { + "id": "singlestat", + "name": "Singlestat", + "type": "panel", + "version": "5.0.0" } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "description": "Top 10 hosts by network load", - "fill": 0, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 5 - }, - "id": 19, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "showIn": 0, + "tags": [ ], + "type": "dashboard" + } + ] + }, + "description": "", + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ ], + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 0, + "y": 0 + }, + "id": 2, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "count(sum by (hostname) (ceph_osd_metadata))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "OSD Hosts", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "topk(10, (sum by(instance) (\n (\n irate(node_network_receive_bytes{instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\",device!=\"lo\"}[1m]) or\n irate(node_network_receive_bytes_total{instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\",device!=\"lo\"}[1m])\n ) +\n (\n irate(node_network_transmit_bytes{instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\",device!=\"lo\"}[1m]) or\n irate(node_network_transmit_bytes_total{instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\",device!=\"lo\"}[1m])\n ))\n )\n)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Network Load - Top 10 Hosts", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "description": "Average CPU busy across all hosts (OSD, RGW, MON etc) within the cluster", + "format": "percentunit", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 4, + "y": 0 + }, + "id": 3, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "avg(\n 1 - (\n avg by(instance) \n (irate(node_cpu_seconds_total{mode='idle',instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\"}[1m]) or\n irate(node_cpu{mode='idle',instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\"}[1m]))\n )\n )", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "AVG CPU Busy", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "description": "Average Memory Usage across all hosts in the cluster (excludes buffer/cache usage)", + "format": "percentunit", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 8, + "y": 0 + }, + "id": 4, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "avg (((node_memory_MemTotal{instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\"} or node_memory_MemTotal_bytes{instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\"})- (\n (node_memory_MemFree{instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\"} or node_memory_MemFree_bytes{instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\"}) + \n (node_memory_Cached{instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\"} or node_memory_Cached_bytes{instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\"}) + \n (node_memory_Buffers{instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\"} or node_memory_Buffers_bytes{instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\"}) +\n (node_memory_Slab{instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\"} or node_memory_Slab_bytes{instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\"})\n )) /\n (node_memory_MemTotal{instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\"} or node_memory_MemTotal_bytes{instance=~\"($osd_hosts|$rgw_hosts|$mon_hosts|$mds_hosts).*\"} ))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "AVG RAM Utilization", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" }, - "yaxes": [ - { - "decimals": 1, - "format": "Bps", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - } - ], - "refresh": "10s", - "schemaVersion": 16, - "style": "dark", - "tags": [], - "templating": { - "list": [ { - "current": { - "text": "default", - "value": "default" - }, - "hide": 0, - "label": "Data Source", - "name": "datasource", - "options": [], - "query": "prometheus", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "type": "datasource" + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "description": "IOPS Load at the device as reported by the OS on all OSD hosts", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 12, + "y": 0 + }, + "id": 5, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum ((irate(node_disk_reads_completed{instance=~\"($osd_hosts).*\"}[5m]) or irate(node_disk_reads_completed_total{instance=~\"($osd_hosts).*\"}[5m]) ) + \n(irate(node_disk_writes_completed{instance=~\"($osd_hosts).*\"}[5m]) or irate(node_disk_writes_completed_total{instance=~\"($osd_hosts).*\"}[5m])))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Physical IOPS", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" }, { - "allValue": "", - "current": {}, - "datasource": "$datasource", - "hide": 2, - "includeAll": true, - "label": null, - "multi": false, - "name": "osd_hosts", - "options": [], - "query": "label_values(ceph_disk_occupation, exported_instance)", - "refresh": 1, - "regex": "([^.]*).*", - "skipUrlSync": false, - "sort": 1, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "ceph", - "type": "query", - "useTags": false + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "description": "Average Disk utilization for all OSD data devices (i.e. excludes journal/WAL)", + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 16, + "y": 0 + }, + "id": 6, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "avg (\n label_replace((irate(node_disk_io_time_ms[5m]) / 10 ) or\n (irate(node_disk_io_time_seconds_total[5m]) * 100), \"instance\", \"$1\", \"instance\", \"([^.:]*).*\"\n ) *\n on(instance, device, ceph_daemon) label_replace(label_replace(ceph_disk_occupation{instance=~\"($osd_hosts).*\"}, \"device\", \"$1\", \"device\", \"/dev/(.*)\"), \"instance\", \"$1\", \"instance\", \"([^.:]*).*\")\n)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "AVG Disk Utilization", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" }, { - "allValue": null, - "current": {}, - "datasource": "$datasource", - "hide": 2, - "includeAll": true, - "label": null, - "multi": false, - "name": "mon_hosts", - "options": [], - "query": "label_values(ceph_mon_metadata, ceph_daemon)", - "refresh": 1, - "regex": "mon.(.*)", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "description": "Total send/receive network load across all hosts in the ceph cluster", + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 20, + "y": 0 + }, + "id": 7, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum (\n irate(node_network_receive_bytes{instance=~\"($osd_hosts|mon_hosts|mds_hosts|rgw_hosts).*\",device!=\"lo\"}[1m]) or\n irate(node_network_receive_bytes_total{instance=~\"($osd_hosts|mon_hosts|mds_hosts|rgw_hosts).*\",device!=\"lo\"}[1m])\n ) +\nsum (\n irate(node_network_transmit_bytes{instance=~\"($osd_hosts|mon_hosts|mds_hosts|rgw_hosts).*\",device!=\"lo\"}[1m]) or\n irate(node_network_transmit_bytes_total{instance=~\"($osd_hosts|mon_hosts|mds_hosts|rgw_hosts).*\",device!=\"lo\"}[1m])\n )", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Network Load", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" }, { - "allValue": null, - "current": {}, - "datasource": "$datasource", - "hide": 2, - "includeAll": true, - "label": null, - "multi": false, - "name": "mds_hosts", - "options": [], - "query": "label_values(ceph_mds_inodes, ceph_daemon)", - "refresh": 1, - "regex": "mds.(.*)", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Show the top 10 busiest hosts by cpu", + "fill": 1, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 5 + }, + "id": 8, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "topk(10,100 * ( 1 - (\n avg by(instance) \n (irate(node_cpu_seconds_total{mode='idle',instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\"}[1m]) or\n irate(node_cpu{mode='idle',instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\"}[1m]))\n )\n )\n)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Busy - Top 10 Hosts", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] }, { - "allValue": null, - "current": {}, - "datasource": "$datasource", - "hide": 2, - "includeAll": true, - "label": null, - "multi": false, - "name": "rgw_hosts", - "options": [], - "query": "label_values(ceph_rgw_qlen, ceph_daemon)", - "refresh": 1, - "regex": "rgw.(.*)", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Top 10 hosts by network load", + "fill": 1, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 5 + }, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "topk(10, (sum by(instance) (\n (\n irate(node_network_receive_bytes{instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\",device!=\"lo\"}[1m]) or\n irate(node_network_receive_bytes_total{instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\",device!=\"lo\"}[1m])\n ) +\n (\n irate(node_network_transmit_bytes{instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\",device!=\"lo\"}[1m]) or\n irate(node_network_transmit_bytes_total{instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\",device!=\"lo\"}[1m])\n ))\n )\n)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Network Load - Top 10 Hosts", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "", - "title": "Host Overview", - "uid": "y0KGL0iZz", - "version": 3 + ], + "refresh": "10s", + "rows": [ ], + "schemaVersion": 16, + "style": "dark", + "tags": [ ], + "templating": { + "list": [ + { + "current": { + "text": "default", + "value": "default" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "osd_hosts", + "options": [ ], + "query": "label_values(ceph_disk_occupation, exported_instance)", + "refresh": 1, + "regex": "([^.]*).*", + "sort": 1, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "mon_hosts", + "options": [ ], + "query": "label_values(ceph_mon_metadata, ceph_daemon)", + "refresh": 1, + "regex": "mon.(.*)", + "sort": 1, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "mds_hosts", + "options": [ ], + "query": "label_values(ceph_mds_inodes, ceph_daemon)", + "refresh": 1, + "regex": "mds.(.*)", + "sort": 1, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "rgw_hosts", + "options": [ ], + "query": "label_values(ceph_rgw_qlen, ceph_daemon)", + "refresh": 1, + "regex": "rgw.(.*)", + "sort": 1, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Host Overview", + "uid": "y0KGL0iZz", + "version": 0 } diff --git a/monitoring/grafana/dashboards/jsonnet/grafana_dashboards.jsonnet b/monitoring/grafana/dashboards/jsonnet/grafana_dashboards.jsonnet index 11c89b212552f..6594cfe415fa2 100644 --- a/monitoring/grafana/dashboards/jsonnet/grafana_dashboards.jsonnet +++ b/monitoring/grafana/dashboards/jsonnet/grafana_dashboards.jsonnet @@ -1,54 +1,184 @@ local g = import 'grafana.libsonnet'; -local dashboardSchema(title, uid, time_from, refresh, schemaVersion, tags,timezone, timepicker) = - g.dashboard.new(title=title, uid=uid, time_from=time_from, refresh=refresh, schemaVersion=schemaVersion, tags=tags, timezone=timezone, timepicker=timepicker); +local dashboardSchema(title, description, uid, time_from, refresh, schemaVersion, tags, timezone, timepicker) = + g.dashboard.new(title=title, description=description, uid=uid, time_from=time_from, refresh=refresh, schemaVersion=schemaVersion, tags=tags, timezone=timezone, timepicker=timepicker); -local graphPanelSchema(title, nullPointMode, stack, formatY1, formatY2, labelY1, labelY2, min, fill, datasource) = - g.graphPanel.new(title=title, nullPointMode=nullPointMode, stack=stack, formatY1=formatY1, formatY2=formatY2, labelY1=labelY1, labelY2=labelY2, min=min, fill=fill, datasource=datasource); +local graphPanelSchema(aliasColors, title, description, nullPointMode, stack, formatY1, formatY2, labelY1, labelY2, min, fill, datasource) = + g.graphPanel.new(aliasColors=aliasColors, title=title, description=description, nullPointMode=nullPointMode, stack=stack, formatY1=formatY1, formatY2=formatY2, labelY1=labelY1, labelY2=labelY2, min=min, fill=fill, datasource=datasource); local addTargetSchema(expr, intervalFactor, format, legendFormat) = g.prometheus.target(expr=expr, intervalFactor=intervalFactor, format=format, legendFormat=legendFormat); -local addTemplateSchema(name, datasource, query, refresh, hide, includeAll, sort) = - g.template.new(name=name, datasource=datasource, query=query, refresh=refresh, hide=hide, includeAll=includeAll, sort=sort); +local addTemplateSchema(name, datasource, query, refresh, includeAll, sort, label, regex) = + g.template.new(name=name, datasource=datasource, query=query, refresh=refresh, includeAll=includeAll, sort=sort, label=label, regex=regex); local addAnnotationSchema(builtIn, datasource, enable, hide, iconColor, name, type) = g.annotation.datasource(builtIn=builtIn, datasource=datasource, enable=enable, hide=hide, iconColor=iconColor, name=name, type=type); +local addRowSchema(collapse, showTitle, title) = + g.row.new(collapse=collapse, showTitle=showTitle, title=title); + +local addSingelStatSchema(datasource, format, title, description, valueName, colorValue, gaugeMaxValue, gaugeShow, sparklineShow, thresholds) = + g.singlestat.new(datasource=datasource, format=format, title=title, description=description, valueName=valueName, colorValue=colorValue, gaugeMaxValue=gaugeMaxValue, gaugeShow=gaugeShow, sparklineShow=sparklineShow, thresholds=thresholds); + +local addPieChartSchema(aliasColors, datasource, description, legendType, pieType, title, valueName) = + g.pieChartPanel.new(aliasColors=aliasColors, datasource=datasource, description=description, legendType=legendType, pieType=pieType, title=title, valueName=valueName); + +local addTableSchema(datasource, description, sort, styles, title, transform) = + g.tablePanel.new(datasource=datasource, description=description, sort=sort, styles=styles, title=title, transform=transform); + +local addStyle(alias, colorMode, colors, dateFormat, decimals, mappingType, pattern, thresholds, type, unit, valueMaps) = + {'alias': alias, 'colorMode': colorMode, 'colors':colors, 'dateFormat':dateFormat, 'decimals':decimals, 'mappingType':mappingType, 'pattern':pattern, 'thresholds':thresholds, 'type':type, 'unit':unit, 'valueMaps':valueMaps}; + { - "radosgw-sync-overview.json": - local RgwSyncOverviewPanel(title, formatY1, labelY1, rgwMetric, x, y, w, h) = - graphPanelSchema(title, 'null as zero', true, formatY1, 'short', labelY1, null, 0, 1, '$datasource') + "hosts-overview.json": + local HostsOverviewSingleStatPanel(format, title, description, valueName, expr, targetFormat, x, y, w, h) = + addSingelStatSchema('$datasource', format, title, description, valueName, false, 100, false, false, '') + .addTarget(addTargetSchema(expr, 1, targetFormat, '')) + {gridPos: {x: x, y: y, w: w, h: h}}; + + local HostsOverviewGraphPanel(title, description, formatY1, expr, legendFormat, x, y, w, h) = + graphPanelSchema({}, title, description, 'null', false, formatY1, 'short', null, null, 0, 1, '$datasource') .addTargets( - [addTargetSchema('sum by (source_zone) (rate(%s[30s]))' % rgwMetric, 1, 'time_series', '{{source_zone}}')]) + {gridPos: {x: x, y: y, w: w, h: h}}; + [addTargetSchema(expr, 1, 'time_series', legendFormat)]) + {gridPos: {x: x, y: y, w: w, h: h}}; dashboardSchema( - 'RGW Sync Overview', 'rgw-sync-overview', 'now-1h', '15s', 16, ["overview"], '', {refresh_intervals:['5s','10s','15s','30s','1m','5m','15m','30m','1h','2h','1d'],time_options:['5m','15m','1h','6h','12h','24h','2d','7d','30d']} + 'Host Overview', '', 'y0KGL0iZz', 'now-1h', '10s', 16, [], '', {refresh_intervals:['5s','10s','30s','1m','5m','15m','30m','1h','2h','1d'],time_options:['5m','15m','1h','6h','12h','24h','2d','7d','30d']} + ) + .addRequired( + type='grafana', id='grafana', name='Grafana', version='5.3.2' + ) + .addRequired( + type='panel', id='graph', name='Graph', version='5.0.0' + ) + .addRequired( + type='panel', id='singlestat', name='Singlestat', version='5.0.0' ) .addAnnotation( addAnnotationSchema( 1, '-- Grafana --', true, true, 'rgba(0, 211, 255, 1)', 'Annotations & Alerts', 'dashboard') ) + .addTemplate( + g.template.datasource('datasource', 'prometheus', 'default', label='Data Source') + ) + .addTemplate( + addTemplateSchema('osd_hosts', '$datasource', 'label_values(ceph_disk_occupation, exported_instance)', 1, true, 1, null, '([^.]*).*') + ) + .addTemplate( + addTemplateSchema('mon_hosts', '$datasource', 'label_values(ceph_mon_metadata, ceph_daemon)', 1, true, 1, null, 'mon.(.*)') + ) + .addTemplate( + addTemplateSchema('mds_hosts', '$datasource', 'label_values(ceph_mds_inodes, ceph_daemon)', 1, true, 1, null, 'mds.(.*)') + ) + .addTemplate( + addTemplateSchema('rgw_hosts', '$datasource', 'label_values(ceph_rgw_qlen, ceph_daemon)', 1, true, 1, null, 'rgw.(.*)') + ) + .addPanels([ + HostsOverviewSingleStatPanel( + 'none', 'OSD Hosts', '', 'current', 'count(sum by (hostname) (ceph_osd_metadata))', 'time_series', 0, 0, 4, 5), + HostsOverviewSingleStatPanel( + 'percentunit', 'AVG CPU Busy', 'Average CPU busy across all hosts (OSD, RGW, MON etc) within the cluster', 'current', 'avg(\n 1 - (\n avg by(instance) \n (irate(node_cpu_seconds_total{mode=\'idle\',instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\"}[1m]) or\n irate(node_cpu{mode=\'idle\',instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\"}[1m]))\n )\n )', 'time_series', 4, 0, 4, 5), + HostsOverviewSingleStatPanel( + 'percentunit', 'AVG RAM Utilization', 'Average Memory Usage across all hosts in the cluster (excludes buffer/cache usage)', 'current', 'avg (((node_memory_MemTotal{instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\"} or node_memory_MemTotal_bytes{instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\"})- (\n (node_memory_MemFree{instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\"} or node_memory_MemFree_bytes{instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\"}) + \n (node_memory_Cached{instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\"} or node_memory_Cached_bytes{instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\"}) + \n (node_memory_Buffers{instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\"} or node_memory_Buffers_bytes{instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\"}) +\n (node_memory_Slab{instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\"} or node_memory_Slab_bytes{instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\"})\n )) /\n (node_memory_MemTotal{instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\"} or node_memory_MemTotal_bytes{instance=~\"($osd_hosts|$rgw_hosts|$mon_hosts|$mds_hosts).*\"} ))', 'time_series', 8, 0, 4, 5), + HostsOverviewSingleStatPanel( + 'none', 'Physical IOPS', 'IOPS Load at the device as reported by the OS on all OSD hosts', 'current', 'sum ((irate(node_disk_reads_completed{instance=~\"($osd_hosts).*\"}[5m]) or irate(node_disk_reads_completed_total{instance=~\"($osd_hosts).*\"}[5m]) ) + \n(irate(node_disk_writes_completed{instance=~\"($osd_hosts).*\"}[5m]) or irate(node_disk_writes_completed_total{instance=~\"($osd_hosts).*\"}[5m])))', 'time_series', 12, 0, 4, 5), + HostsOverviewSingleStatPanel( + 'percent', 'AVG Disk Utilization', 'Average Disk utilization for all OSD data devices (i.e. excludes journal/WAL)', 'current', 'avg (\n label_replace((irate(node_disk_io_time_ms[5m]) / 10 ) or\n (irate(node_disk_io_time_seconds_total[5m]) * 100), \"instance\", \"$1\", \"instance\", \"([^.:]*).*\"\n ) *\n on(instance, device, ceph_daemon) label_replace(label_replace(ceph_disk_occupation{instance=~\"($osd_hosts).*\"}, \"device\", \"$1\", \"device\", \"/dev/(.*)\"), \"instance\", \"$1\", \"instance\", \"([^.:]*).*\")\n)', 'time_series', 16, 0, 4, 5), + HostsOverviewSingleStatPanel( + 'bytes', 'Network Load', 'Total send/receive network load across all hosts in the ceph cluster', 'current', 'sum (\n irate(node_network_receive_bytes{instance=~\"($osd_hosts|mon_hosts|mds_hosts|rgw_hosts).*\",device!=\"lo\"}[1m]) or\n irate(node_network_receive_bytes_total{instance=~\"($osd_hosts|mon_hosts|mds_hosts|rgw_hosts).*\",device!=\"lo\"}[1m])\n ) +\nsum (\n irate(node_network_transmit_bytes{instance=~\"($osd_hosts|mon_hosts|mds_hosts|rgw_hosts).*\",device!=\"lo\"}[1m]) or\n irate(node_network_transmit_bytes_total{instance=~\"($osd_hosts|mon_hosts|mds_hosts|rgw_hosts).*\",device!=\"lo\"}[1m])\n )', 'time_series', 20, 0, 4, 5), + HostsOverviewGraphPanel( + 'CPU Busy - Top 10 Hosts', 'Show the top 10 busiest hosts by cpu', 'percent', 'topk(10,100 * ( 1 - (\n avg by(instance) \n (irate(node_cpu_seconds_total{mode=\'idle\',instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\"}[1m]) or\n irate(node_cpu{mode=\'idle\',instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\"}[1m]))\n )\n )\n)', '{{instance}}', 0, 5, 12, 9), + HostsOverviewGraphPanel( + 'Network Load - Top 10 Hosts', 'Top 10 hosts by network load', 'Bps', 'topk(10, (sum by(instance) (\n (\n irate(node_network_receive_bytes{instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\",device!=\"lo\"}[1m]) or\n irate(node_network_receive_bytes_total{instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\",device!=\"lo\"}[1m])\n ) +\n (\n irate(node_network_transmit_bytes{instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\",device!=\"lo\"}[1m]) or\n irate(node_network_transmit_bytes_total{instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\",device!=\"lo\"}[1m])\n ))\n )\n)', '{{instance}}', 12, 5, 12, 9), + ]) +} +{ + "host-details.json": + local HostDetailsSingleStatPanel(format, title, description, valueName, expr, targetFormat, x, y, w, h) = + addSingelStatSchema('$datasource', format, title, description, valueName, false, 100, false, false, '') + .addTarget(addTargetSchema(expr, 1, targetFormat, '')) + {gridPos: {x: x, y: y, w: w, h: h}}; + + local HostDetailsGraphPanel(alias, title, description, nullPointMode, formatY1, labelY1, expr, legendFormat, x, y, w, h) = + graphPanelSchema(alias, title, description, nullPointMode, false, formatY1, 'short', labelY1, null, null, 1, '$datasource') + .addTargets( + [addTargetSchema(expr, 1, 'time_series', legendFormat)]) + {gridPos: {x: x, y: y, w: w, h: h}}; + + dashboardSchema( + 'Host Details', '', 'rtOg0AiWz', 'now-1h', '10s', 16, ['overview'], '', {refresh_intervals:['5s','10s','30s','1m','5m','15m','30m','1h','2h','1d'],time_options:['5m','15m','1h','6h','12h','24h','2d','7d','30d']} + ) .addRequired( - type='grafana', id='grafana', name='Grafana', version='5.0.0' + type='grafana', id='grafana', name='Grafana', version='5.3.2' ) .addRequired( type='panel', id='graph', name='Graph', version='5.0.0' ) - .addTemplate( - addTemplateSchema('rgw_servers', '$datasource', 'prometehus', 1, 2, true, 1) + .addRequired( + type='panel', id='singlestat', name='Singlestat', version='5.0.0' + ) + .addAnnotation( + addAnnotationSchema( + 1, '-- Grafana --', true, true, 'rgba(0, 211, 255, 1)', 'Annotations & Alerts', 'dashboard') ) .addTemplate( g.template.datasource('datasource', 'prometheus', 'default', label='Data Source') ) + .addTemplate( + addTemplateSchema('ceph_hosts', '$datasource', 'label_values(node_scrape_collector_success, instance) ', 1, false, 3, 'Hostname', '([^.:]*).*') + ) .addPanels([ - RgwSyncOverviewPanel( - 'Replication (throughput) from Source Zone', 'Bps', null, 'ceph_data_sync_from_zone_fetch_bytes_sum', 0, 0, 8, 7), - RgwSyncOverviewPanel( - 'Replication (objects) from Source Zone', 'short', 'Objects/s', 'ceph_data_sync_from_zone_fetch_bytes_count', 8, 0, 8, 7), - RgwSyncOverviewPanel( - 'Polling Request Latency from Source Zone', 'ms', null, 'ceph_data_sync_from_zone_poll_latency_sum', 16, 0, 8, 7), - RgwSyncOverviewPanel( - 'Unsuccessful Object Replications from Source Zone', 'short', 'Count/s', 'ceph_data_sync_from_zone_fetch_errors', 0, 7, 8, 7) + addRowSchema(false, true, '$ceph_hosts System Overview') + {gridPos: {x: 0, y: 0, w: 24, h: 1}}, + HostDetailsSingleStatPanel( + 'none', 'OSDs', '', 'current', 'count(sum by (ceph_daemon) (ceph_osd_metadata{hostname=\'$ceph_hosts\'}))', 'time_series', 0, 1, 3, 5 + ), + HostDetailsGraphPanel( + {"interrupt": "#447EBC","steal": "#6D1F62","system": "#890F02","user": "#3F6833","wait": "#C15C17"},'CPU Utilization', 'Shows the CPU breakdown. When multiple servers are selected, only the first host\'s cpu data is shown', 'null', 'percent', '% Utilization', 'sum by (mode) (\n irate(node_cpu{instance=~\"($ceph_hosts)([\\\\.:].*)?\", mode=~\"(irq|nice|softirq|steal|system|user|iowait)\"}[1m]) or\n irate(node_cpu_seconds_total{instance=~\"($ceph_hosts)([\\\\.:].*)?\", mode=~\"(irq|nice|softirq|steal|system|user|iowait)\"}[1m])\n) / scalar(\n sum(irate(node_cpu{instance=~\"($ceph_hosts)([\\\\.:].*)?\"}[1m]) or\n irate(node_cpu_seconds_total{instance=~\"($ceph_hosts)([\\\\.:].*)?\"}[1m]))\n) * 100', '{{mode}}', 3, 1, 6, 10 + ), + HostDetailsGraphPanel( + {"Available": "#508642","Free": "#508642","Total": "#bf1b00","Used": "#bf1b00","total": "#bf1b00","used": "#0a50a1"},'RAM Usage', '', 'null', 'bytes', 'RAM used', 'node_memory_MemFree{instance=~\"$ceph_hosts([\\\\.:].*)?\"} or node_memory_MemFree_bytes{instance=~\"$ceph_hosts([\\\\.:].*)?\"} ', 'Free', 9, 1, 6, 10) + .addTargets( + [addTargetSchema('node_memory_MemTotal{instance=~\"$ceph_hosts([\\\\.:].*)?\"} or node_memory_MemTotal_bytes{instance=~\"$ceph_hosts([\\\\.:].*)?\"} ', 1, 'time_series', 'total'), + addTargetSchema('(node_memory_Cached{instance=~\"$ceph_hosts([\\\\.:].*)?\"} or node_memory_Cached_bytes{instance=~\"$ceph_hosts([\\\\.:].*)?\"}) + \n(node_memory_Buffers{instance=~\"$ceph_hosts([\\\\.:].*)?\"} or node_memory_Buffers_bytes{instance=~\"$ceph_hosts([\\\\.:].*)?\"}) +\n(node_memory_Slab{instance=~\"$ceph_hosts([\\\\.:].*)?\"} or node_memory_Slab_bytes{instance=~\"$ceph_hosts([\\\\.:].*)?\"}) \n', 1, 'time_series', 'buffers/cache'), + addTargetSchema('(node_memory_MemTotal{instance=~\"$ceph_hosts([\\\\.:].*)?\"} or node_memory_MemTotal_bytes{instance=~\"$ceph_hosts([\\\\.:].*)?\"})- (\n (node_memory_MemFree{instance=~\"$ceph_hosts([\\\\.:].*)?\"} or node_memory_MemFree_bytes{instance=~\"$ceph_hosts([\\\\.:].*)?\"}) + \n (node_memory_Cached{instance=~\"$ceph_hosts([\\\\.:].*)?\"} or node_memory_Cached_bytes{instance=~\"$ceph_hosts([\\\\.:].*)?\"}) + \n (node_memory_Buffers{instance=~\"$ceph_hosts([\\\\.:].*)?\"} or node_memory_Buffers_bytes{instance=~\"$ceph_hosts([\\\\.:].*)?\"}) +\n (node_memory_Slab{instance=~\"$ceph_hosts([\\\\.:].*)?\"} or node_memory_Slab_bytes{instance=~\"$ceph_hosts([\\\\.:].*)?\"})\n )\n \n', 1, 'time_series', 'used')]) + .addSeriesOverride({"alias": "total","color": "#bf1b00","fill": 0,"linewidth": 2,"stack": false} + ), + HostDetailsGraphPanel( + {},'Network Load', 'Show the network load (rx,tx) across all interfaces (excluding loopback \'lo\')', 'null', 'decbytes', 'Send (-) / Receive (+)', 'sum by (device) (\n irate(node_network_receive_bytes{instance=~\"($ceph_hosts)([\\\\.:].*)?\",device!=\"lo\"}[1m]) or \n irate(node_network_receive_bytes_total{instance=~\"($ceph_hosts)([\\\\.:].*)?\",device!=\"lo\"}[1m])\n)', '{{device}}.rx', 15, 1, 6, 10) + .addTargets( + [addTargetSchema('sum by (device) (\n irate(node_network_transmit_bytes{instance=~\"($ceph_hosts)([\\\\.:].*)?\",device!=\"lo\"}[1m]) or\n irate(node_network_transmit_bytes_total{instance=~\"($ceph_hosts)([\\\\.:].*)?\",device!=\"lo\"}[1m])\n)', 1, 'time_series', '{{device}}.tx')]) + .addSeriesOverride({"alias": "/.*tx/","transform": "negative-Y"} + ), + HostDetailsGraphPanel( + {},'Network drop rate', '', 'null', 'pps', 'Send (-) / Receive (+)', 'irate(node_network_receive_drop{instance=~\"$ceph_hosts([\\\\.:].*)?\"}[1m]) or irate(node_network_receive_drop_total{instance=~\"$ceph_hosts([\\\\.:].*)?\"}[1m])', '{{device}}.rx', 21, 1, 3, 5) + .addTargets( + [addTargetSchema('irate(node_network_transmit_drop{instance=~\"$ceph_hosts([\\\\.:].*)?\"}[1m]) or irate(node_network_transmit_drop_total{instance=~\"$ceph_hosts([\\\\.:].*)?\"}[1m])', 1, 'time_series', '{{device}}.tx')]) + .addSeriesOverride({"alias": "/.*tx/","transform": "negative-Y"} + ), + HostDetailsSingleStatPanel( + 'bytes', 'Raw Capacity', 'Each OSD consists of a Journal/WAL partition and a data partition. The RAW Capacity shown is the sum of the data partitions across all OSDs on the selected OSD hosts.', 'current', 'sum(ceph_osd_stat_bytes and on (ceph_daemon) ceph_disk_occupation{instance=~\"($ceph_hosts)([\\\\.:].*)?\"})', 'time_series', 0, 6, 3, 5 + ), + HostDetailsGraphPanel( + {},'Network error rate', '', 'null', 'pps', 'Send (-) / Receive (+)', 'irate(node_network_receive_errs{instance=~\"$ceph_hosts([\\\\.:].*)?\"}[1m]) or irate(node_network_receive_errs_total{instance=~\"$ceph_hosts([\\\\.:].*)?\"}[1m])', '{{device}}.rx', 21, 6, 3, 5) + .addTargets( + [addTargetSchema('irate(node_network_transmit_errs{instance=~\"$ceph_hosts([\\\\.:].*)?\"}[1m]) or irate(node_network_transmit_errs_total{instance=~\"$ceph_hosts([\\\\.:].*)?\"}[1m])', 1, 'time_series', '{{device}}.tx')]) + .addSeriesOverride({"alias": "/.*tx/","transform": "negative-Y"} + ), + addRowSchema(false, true, 'OSD Disk Performance Statistics') + {gridPos: {x: 0, y: 11, w: 24, h: 1}}, + HostDetailsGraphPanel( + {},'$ceph_hosts Disk IOPS', 'For any OSD devices on the host, this chart shows the iops per physical device. Each device is shown by it\'s name and corresponding OSD id value', 'connected', 'ops', 'Read (-) / Write (+)', 'label_replace(\n (\n irate(node_disk_writes_completed{instance=~\"($ceph_hosts)([\\\\.:].*)?\"}[5m]) or\n irate(node_disk_writes_completed_total{instance=~\"($ceph_hosts)([\\\\.:].*)?\"}[5m])\n ),\n \"instance\",\n \"$1\",\n \"instance\",\n \"([^:.]*).*\"\n)\n* on(instance, device, ceph_daemon) group_left\n label_replace(\n label_replace(\n ceph_disk_occupation,\n \"device\",\n \"$1\",\n \"device\",\n \"/dev/(.*)\"\n ),\n \"instance\",\n \"$1\",\n \"instance\",\n \"([^:.]*).*\"\n )', '{{device}}({{ceph_daemon}}) writes', 0, 12, 11, 9) + .addTargets( + [addTargetSchema('label_replace(\n (irate(node_disk_reads_completed{instance=~\"($ceph_hosts)([\\\\.:].*)?\"}[5m]) or irate(node_disk_reads_completed_total{instance=~\"($ceph_hosts)([\\\\.:].*)?\"}[5m])),\n \"instance\",\n \"$1\",\n \"instance\",\n \"([^:.]*).*\"\n)\n* on(instance, device, ceph_daemon) group_left\n label_replace(\n label_replace(\n ceph_disk_occupation,\n \"device\",\n \"$1\",\n \"device\",\n \"/dev/(.*)\"\n ),\n \"instance\",\n \"$1\",\n \"instance\",\n \"([^:.]*).*\"\n )', 1, 'time_series', '{{device}}({{ceph_daemon}}) reads')]) + .addSeriesOverride({"alias": "/.*reads/","transform": "negative-Y"} + ), + HostDetailsGraphPanel( + {},'$ceph_hosts Throughput by Disk', 'For OSD hosts, this chart shows the disk bandwidth (read bytes/sec + write bytes/sec) of the physical OSD device. Each device is shown by device name, and corresponding OSD id', 'connected', 'Bps', 'Read (-) / Write (+)', 'label_replace((irate(node_disk_bytes_written{instance=~\"($ceph_hosts)([\\\\.:].*)?\"}[5m]) or irate(node_disk_written_bytes_total{instance=~\"($ceph_hosts)([\\\\.:].*)?\"}[5m])), \"instance\", \"$1\", \"instance\", \"([^:.]*).*\") * on(instance, device, ceph_daemon) group_left label_replace(label_replace(ceph_disk_occupation, \"device\", \"$1\", \"device\", \"/dev/(.*)\"), \"instance\", \"$1\", \"instance\", \"([^:.]*).*\")', '{{device}}({{ceph_daemon}}) write', 12, 12, 11, 9) + .addTargets( + [addTargetSchema('label_replace((irate(node_disk_bytes_read{instance=~\"($ceph_hosts)([\\\\.:].*)?\"}[5m]) or irate(node_disk_read_bytes_total{instance=~\"($ceph_hosts)([\\\\.:].*)?\"}[5m])), \"instance\", \"$1\", \"instance\", \"([^:.]*).*\") * on(instance, device, ceph_daemon) group_left label_replace(label_replace(ceph_disk_occupation, \"device\", \"$1\", \"device\", \"/dev/(.*)\"), \"instance\", \"$1\", \"instance\", \"([^:.]*).*\")', 1, 'time_series', '{{device}}({{ceph_daemon}}) read')]) + .addSeriesOverride({"alias": "/.*read/","transform": "negative-Y"} + ), + HostDetailsGraphPanel( + {},'$ceph_hosts Disk Latency', 'For OSD hosts, this chart shows the latency at the physical drive. Each drive is shown by device name, with it\'s corresponding OSD id', 'null as zero', 's', '', 'max by(instance,device) (label_replace((irate(node_disk_write_time_seconds_total{ instance=~\"($ceph_hosts)([\\\\.:].*)?\"}[5m]) ) / clamp_min(irate(node_disk_writes_completed_total{ instance=~\"($ceph_hosts)([\\\\.:].*)?\"}[5m]), 0.001) or (irate(node_disk_read_time_seconds_total{ instance=~\"($ceph_hosts)([\\\\.:].*)?\"}[5m]) ) / clamp_min(irate(node_disk_reads_completed_total{ instance=~\"($ceph_hosts)([\\\\.:].*)?\"}[5m]), 0.001), \"instance\", \"$1\", \"instance\", \"([^:.]*).*\")) * on(instance, device, ceph_daemon) group_left label_replace(label_replace(ceph_disk_occupation{instance=~\"($ceph_hosts)([\\\\.:].*)?\"}, \"device\", \"$1\", \"device\", \"/dev/(.*)\"), \"instance\", \"$1\", \"instance\", \"([^:.]*).*\")', '{{device}}({{ceph_daemon}})', 0, 21, 11, 9 + ), + HostDetailsGraphPanel( + {},'$ceph_hosts Disk utilization', 'Show disk utilization % (util) of any OSD devices on the host by the physical device name and associated OSD id.', 'connected', 'percent', '%Util', 'label_replace(((irate(node_disk_io_time_ms{instance=~\"($ceph_hosts)([\\\\.:].*)?\"}[5m]) / 10 ) or irate(node_disk_io_time_seconds_total{instance=~\"($ceph_hosts)([\\\\.:].*)?\"}[5m]) * 100), \"instance\", \"$1\", \"instance\", \"([^:.]*).*\") * on(instance, device, ceph_daemon) group_left label_replace(label_replace(ceph_disk_occupation{instance=~\"($ceph_hosts)([\\\\.:].*)?\"}, \"device\", \"$1\", \"device\", \"/dev/(.*)\"), \"instance\", \"$1\", \"instance\", \"([^:.]*).*\")', '{{device}}({{ceph_daemon}})', 12, 21, 11, 9 + ) ]) -} +} \ No newline at end of file From f41eae16af9fa6725d5b714f068fab3f946d2927 Mon Sep 17 00:00:00 2001 From: Avan Thakkar Date: Fri, 11 Jun 2021 16:37:10 +0530 Subject: [PATCH 0105/2866] cephadm: expose gather-facts api method Fixes: https://tracker.ceph.com/issues/51209 This PR intends to expose host metadata(gather-facts) api method Signed-off-by: Avan Thakkar Signed-off-by: Aashish Sharma --- src/pybind/mgr/cephadm/module.py | 13 +++++++++++++ src/pybind/mgr/cephadm/tests/test_facts.py | 12 ++++++++++++ src/pybind/mgr/dashboard/services/orchestrator.py | 4 ++++ src/pybind/mgr/orchestrator/_interface.py | 6 ++++++ 4 files changed, 35 insertions(+) create mode 100644 src/pybind/mgr/cephadm/tests/test_facts.py diff --git a/src/pybind/mgr/cephadm/module.py b/src/pybind/mgr/cephadm/module.py index 41476c1c99207..fcf2ac174c9b3 100644 --- a/src/pybind/mgr/cephadm/module.py +++ b/src/pybind/mgr/cephadm/module.py @@ -1434,6 +1434,19 @@ def get_hosts(self): """ return list(self.inventory.all_specs()) + @handle_orch_error + def get_facts(self, hostname: Optional[str] = None) -> List[Dict[str, Any]]: + """ + Return a list of hosts metadata(gather_facts) managed by the orchestrator. + + Notes: + - skip async: manager reads from cache. + """ + if hostname: + return [self.cache.get_facts(hostname)] + + return [self.cache.get_facts(hostname) for hostname in self.cache.get_hosts()] + @handle_orch_error def add_host_label(self, host: str, label: str) -> str: self.inventory.add_label(host, label) diff --git a/src/pybind/mgr/cephadm/tests/test_facts.py b/src/pybind/mgr/cephadm/tests/test_facts.py new file mode 100644 index 0000000000000..79e6db00ce83c --- /dev/null +++ b/src/pybind/mgr/cephadm/tests/test_facts.py @@ -0,0 +1,12 @@ +import pytest + +from ..inventory import HostCache +from ..import CephadmOrchestrator + + +@pytest.fixture() +def test_facts(): + facts = {'node-1.ceph.com', {'bios_version': 'F2', 'cpu_cores': 16}} + HostCache.facts = facts + ret_facts = CephadmOrchestrator.get_facts('node-1.ceph.com') + assert ret_facts == [{'bios_version': 'F2', 'cpu_cores': 16}] diff --git a/src/pybind/mgr/dashboard/services/orchestrator.py b/src/pybind/mgr/dashboard/services/orchestrator.py index 44309a8493917..88a6cb3a86085 100644 --- a/src/pybind/mgr/dashboard/services/orchestrator.py +++ b/src/pybind/mgr/dashboard/services/orchestrator.py @@ -66,6 +66,10 @@ def get(self, hostname: str) -> Optional[HostSpec]: def add(self, hostname: str, addr: str, labels: List[str]): return self.api.add_host(HostSpec(hostname, addr=addr, labels=labels)) + @wait_api_result + def get_facts(self, hostname: Optional[str] = None) -> List[Dict[str, Any]]: + return self.api.get_facts(hostname) + @wait_api_result def remove(self, hostname: str): return self.api.remove_host(hostname) diff --git a/src/pybind/mgr/orchestrator/_interface.py b/src/pybind/mgr/orchestrator/_interface.py index 158299885b1cb..91925a59e6843 100644 --- a/src/pybind/mgr/orchestrator/_interface.py +++ b/src/pybind/mgr/orchestrator/_interface.py @@ -376,6 +376,12 @@ def get_hosts(self) -> OrchResult[List[HostSpec]]: """ raise NotImplementedError() + def get_facts(self, hostname: Optional[str] = None) -> OrchResult[List[Dict[str, Any]]]: + """ + Return hosts metadata(gather_facts). + """ + raise NotImplementedError() + def add_host_label(self, host: str, label: str) -> OrchResult[str]: """ Add a host label From ed8dd300a88173b1e5efafb6bb061a15ea296c29 Mon Sep 17 00:00:00 2001 From: dheart Date: Tue, 12 Oct 2021 16:46:34 +0800 Subject: [PATCH 0106/2866] os/bluestore: _do_write_small fix head_pad Signed-off-by: dheart --- src/os/bluestore/BlueStore.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc index c7406b995424b..9c8aa6d39814b 100644 --- a/src/os/bluestore/BlueStore.cc +++ b/src/os/bluestore/BlueStore.cc @@ -13654,7 +13654,7 @@ void BlueStore::_do_write_small( end_offs - offset + head_pad + tail_pad); } if (head_pad && - o->extent_map.has_any_lextents(offset - head_pad, chunk_size)) { + o->extent_map.has_any_lextents(offset - head_pad, head_pad)) { head_pad = 0; } if (tail_pad && o->extent_map.has_any_lextents(end_offs, tail_pad)) { From e3ad392fd04e9ade3d0fb3fd69f07c3d6bc6c366 Mon Sep 17 00:00:00 2001 From: Radoslaw Zarzynski Date: Tue, 12 Oct 2021 09:38:23 +0000 Subject: [PATCH 0107/2866] crimson: osd operations respect interruptor's InterruptCondition. For the sake of DRY. Signed-off-by: Radoslaw Zarzynski --- src/crimson/common/interruptible_future.h | 2 ++ .../osd/osd_operations/background_recovery.cc | 6 +++--- .../osd/osd_operations/client_request.cc | 18 +++++++++--------- .../osd_operations/internal_client_request.cc | 10 +++++----- .../osd/osd_operations/peering_event.cc | 8 ++++---- 5 files changed, 23 insertions(+), 21 deletions(-) diff --git a/src/crimson/common/interruptible_future.h b/src/crimson/common/interruptible_future.h index f41bc4f3ededd..b490945f0f76d 100644 --- a/src/crimson/common/interruptible_future.h +++ b/src/crimson/common/interruptible_future.h @@ -1034,6 +1034,8 @@ template struct interruptor { public: + using condition = InterruptCond; + template [[gnu::always_inline]] static interruptible_future_detail diff --git a/src/crimson/osd/osd_operations/background_recovery.cc b/src/crimson/osd/osd_operations/background_recovery.cc index 7614cc9102a6c..50089dbeda684 100644 --- a/src/crimson/osd/osd_operations/background_recovery.cc +++ b/src/crimson/osd/osd_operations/background_recovery.cc @@ -70,7 +70,7 @@ UrgentRecovery::do_recovery() { logger().debug("{}: {}", __func__, *this); if (!pg->has_reset_since(epoch_started)) { - return with_blocking_future_interruptible( + return with_blocking_future_interruptible( pg->get_recovery_handler()->recover_missing(soid, need) ).then_interruptible([] { return seastar::make_ready_future(false); @@ -113,7 +113,7 @@ PglogBasedRecovery::do_recovery() { if (pg->has_reset_since(epoch_started)) return seastar::make_ready_future(false); - return with_blocking_future_interruptible( + return with_blocking_future_interruptible( pg->get_recovery_handler()->start_recovery_ops( crimson::common::local_conf()->osd_recovery_max_single_start)); } @@ -134,7 +134,7 @@ BackfillRecovery::do_recovery() return seastar::make_ready_future(false); } // TODO: limits - return with_blocking_future_interruptible( + return with_blocking_future_interruptible( // process_event() of our boost::statechart machine is non-reentrant. // with the backfill_pipeline we protect it from a second entry from // the implementation of BackfillListener. diff --git a/src/crimson/osd/osd_operations/client_request.cc b/src/crimson/osd/osd_operations/client_request.cc index d5d90f4a2061f..3a5111ce0d417 100644 --- a/src/crimson/osd/osd_operations/client_request.cc +++ b/src/crimson/osd/osd_operations/client_request.cc @@ -104,16 +104,16 @@ seastar::future<> ClientRequest::start() return interruptor::now(); }); } - return with_blocking_future_interruptible( + return with_blocking_future_interruptible( handle.enter(pp(pg).await_map) ).then_interruptible([this, &pg] { - return with_blocking_future_interruptible( + return with_blocking_future_interruptible( pg.osdmap_gate.wait_for_map(m->get_min_epoch())); }).then_interruptible([this, &pg](auto map) { - return with_blocking_future_interruptible( + return with_blocking_future_interruptible( handle.enter(pp(pg).wait_for_active)); }).then_interruptible([this, &pg]() { - return with_blocking_future_interruptible( + return with_blocking_future_interruptible( pg.wait_for_active_blocker.wait()); }).then_interruptible([this, pgref=std::move(pgref)]() mutable { if (is_pg_op()) { @@ -157,7 +157,7 @@ ClientRequest::process_pg_op( ClientRequest::interruptible_future<> ClientRequest::process_op(Ref &pg) { - return with_blocking_future_interruptible( + return with_blocking_future_interruptible( handle.enter(pp(*pg).recover_missing)) .then_interruptible( [this, pg]() mutable { @@ -172,14 +172,14 @@ ClientRequest::process_op(Ref &pg) CEPH_OSD_FLAG_ACK | CEPH_OSD_FLAG_ONDISK, false); return conn->send(std::move(reply)); } else { - return with_blocking_future_interruptible( + return with_blocking_future_interruptible( handle.enter(pp(*pg).get_obc)).then_interruptible( [this, pg]() mutable -> PG::load_obc_iertr::future<> { logger().debug("{}: got obc lock", *this); op_info.set_from_op(&*m, *pg->get_osdmap()); return pg->with_locked_obc(m->get_hobj(), op_info, [this, pg](auto obc) mutable { - return with_blocking_future_interruptible( + return with_blocking_future_interruptible( handle.enter(pp(*pg).process) ).then_interruptible([this, pg, obc]() mutable { return do_process(pg, obc); @@ -217,13 +217,13 @@ ClientRequest::do_process(Ref& pg, crimson::osd::ObjectContextRef obc) [this, pg](auto submitted, auto all_completed) mutable { return submitted.then_interruptible( [this, pg] { - return with_blocking_future_interruptible( + return with_blocking_future_interruptible( handle.enter(pp(*pg).wait_repop)); }).then_interruptible( [this, pg, all_completed=std::move(all_completed)]() mutable { return all_completed.safe_then_interruptible( [this, pg](MURef reply) { - return with_blocking_future_interruptible( + return with_blocking_future_interruptible( handle.enter(pp(*pg).send_reply)).then_interruptible( [this, reply=std::move(reply)]() mutable{ return conn->send(std::move(reply)); diff --git a/src/crimson/osd/osd_operations/internal_client_request.cc b/src/crimson/osd/osd_operations/internal_client_request.cc index 3859b388a8cb2..8b15005ef3d20 100644 --- a/src/crimson/osd/osd_operations/internal_client_request.cc +++ b/src/crimson/osd/osd_operations/internal_client_request.cc @@ -43,18 +43,18 @@ seastar::future<> InternalClientRequest::start() return seastar::repeat([this] { logger().debug("{}: in repeat", *this); return interruptor::with_interruption([this]() mutable { - return with_blocking_future_interruptible( + return with_blocking_future_interruptible( handle.enter(pp().wait_for_active) ).then_interruptible([this] { - return with_blocking_future_interruptible( + return with_blocking_future_interruptible( pg->wait_for_active_blocker.wait()); }).then_interruptible([this] { - return with_blocking_future_interruptible( + return with_blocking_future_interruptible( handle.enter(pp().recover_missing) ).then_interruptible([this] { return do_recover_missing(pg, {}); }).then_interruptible([this] { - return with_blocking_future_interruptible( + return with_blocking_future_interruptible( handle.enter(pp().get_obc) ).then_interruptible([this] () -> PG::load_obc_iertr::future<> { logger().debug("{}: getting obc lock", *this); @@ -67,7 +67,7 @@ seastar::future<> InternalClientRequest::start() assert(ret == 0); return pg->with_locked_obc(get_target_oid(), op_info, [&osd_ops, this](auto obc) { - return with_blocking_future_interruptible( + return with_blocking_future_interruptible( handle.enter(pp().process) ).then_interruptible( [obc=std::move(obc), &osd_ops, this] { diff --git a/src/crimson/osd/osd_operations/peering_event.cc b/src/crimson/osd/osd_operations/peering_event.cc index 194876b1d5e13..398fc945a099f 100644 --- a/src/crimson/osd/osd_operations/peering_event.cc +++ b/src/crimson/osd/osd_operations/peering_event.cc @@ -70,18 +70,18 @@ seastar::future<> PeeringEvent::start() return complete_rctx(pg); } logger().debug("{}: pg present", *this); - return with_blocking_future_interruptible( + return with_blocking_future_interruptible( handle.enter(pp(*pg).await_map) ).then_interruptible([this, pg] { - return with_blocking_future_interruptible( + return with_blocking_future_interruptible( pg->osdmap_gate.wait_for_map(evt.get_epoch_sent())); }).then_interruptible([this, pg](auto) { - return with_blocking_future_interruptible( + return with_blocking_future_interruptible( handle.enter(pp(*pg).process)); }).then_interruptible([this, pg] { // TODO: likely we should synchronize also with the pg log-based // recovery. - return with_blocking_future_interruptible( + return with_blocking_future_interruptible( handle.enter(BackfillRecovery::bp(*pg).process)); }).then_interruptible([this, pg] { pg->do_peering_event(evt, ctx); From f41676bee53f0ed0562faea9d358d2b873e20bfa Mon Sep 17 00:00:00 2001 From: Radoslaw Zarzynski Date: Tue, 12 Oct 2021 11:23:45 +0000 Subject: [PATCH 0108/2866] crimson/osd: rectify a coding style issue in PglogBasedRecovery::do_recovery(). Signed-off-by: Radoslaw Zarzynski --- src/crimson/osd/osd_operations/background_recovery.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/crimson/osd/osd_operations/background_recovery.cc b/src/crimson/osd/osd_operations/background_recovery.cc index 50089dbeda684..a8ac12cfc495c 100644 --- a/src/crimson/osd/osd_operations/background_recovery.cc +++ b/src/crimson/osd/osd_operations/background_recovery.cc @@ -111,8 +111,9 @@ PglogBasedRecovery::PglogBasedRecovery( PglogBasedRecovery::interruptible_future PglogBasedRecovery::do_recovery() { - if (pg->has_reset_since(epoch_started)) + if (pg->has_reset_since(epoch_started)) { return seastar::make_ready_future(false); + } return with_blocking_future_interruptible( pg->get_recovery_handler()->start_recovery_ops( crimson::common::local_conf()->osd_recovery_max_single_start)); From 7b75040d257f599d760d830c0431e55615654026 Mon Sep 17 00:00:00 2001 From: Radoslaw Zarzynski Date: Tue, 12 Oct 2021 11:25:20 +0000 Subject: [PATCH 0109/2866] crimson: respect the CEPH_ARGS environment variable. Rook uses `CEPH_ARGS` to convey the `mon_host` configurable. Lack of support for it was the root cause of inability to reach out to monitors. See: https://gist.github.com/rzarzynski/95746aa73a48e811749a3b0aaeb31680#gistcomment-3924233. Signed-off-by: Radoslaw Zarzynski --- src/crimson/common/config_proxy.h | 8 ++++++++ src/crimson/osd/main.cc | 1 + 2 files changed, 9 insertions(+) diff --git a/src/crimson/common/config_proxy.h b/src/crimson/common/config_proxy.h index e56a6825f41c3..e27db9b5bedf3 100644 --- a/src/crimson/common/config_proxy.h +++ b/src/crimson/common/config_proxy.h @@ -179,6 +179,14 @@ class ConfigProxy : public seastar::peering_sharded_service }); } + seastar::future<> parse_env() { + return do_change([this](ConfigValues& values) { + get_config().parse_env(CEPH_ENTITY_TYPE_OSD, + values, + obs_mgr); + }); + } + seastar::future<> parse_config_files(const std::string& conf_files); using ShardedConfig = seastar::sharded; diff --git a/src/crimson/osd/main.cc b/src/crimson/osd/main.cc index 596dea0b1f4e4..18b7d520b8892 100644 --- a/src/crimson/osd/main.cc +++ b/src/crimson/osd/main.cc @@ -262,6 +262,7 @@ int main(int argc, char* argv[]) sharded_perf_coll().stop().get(); }); local_conf().parse_config_files(conf_file_list).get(); + local_conf().parse_env().get(); local_conf().parse_argv(ceph_args).get(); if (const auto ret = pidfile_write(local_conf()->pid_file); ret == -EACCES || ret == -EAGAIN) { From ad0b6094cebd2460795ac739ea5d6fddf16e9a79 Mon Sep 17 00:00:00 2001 From: Adam King Date: Tue, 12 Oct 2021 07:22:57 -0400 Subject: [PATCH 0110/2866] mgr/cephadm: remove host addr from agent deps Had no need to be a dependency anyway and actually was casuing issues in clusters with FQDNs due to the assumption that the daemon-id would match the hostname (which isn't necessarily true if it is an FQDN). Signed-off-by: Adam King --- src/pybind/mgr/cephadm/agent.py | 3 ++- src/pybind/mgr/cephadm/module.py | 4 ++-- src/pybind/mgr/cephadm/services/cephadmservice.py | 4 ++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/pybind/mgr/cephadm/agent.py b/src/pybind/mgr/cephadm/agent.py index ead4c0a3bee5a..e987add4ee26f 100644 --- a/src/pybind/mgr/cephadm/agent.py +++ b/src/pybind/mgr/cephadm/agent.py @@ -205,7 +205,7 @@ class AgentMessageThread(threading.Thread): def __init__(self, host: str, port: int, data: Dict[Any, Any], mgr: "CephadmOrchestrator") -> None: self.mgr = mgr self.host = host - self.addr = self.mgr.inventory.get_addr(host) + self.addr = self.mgr.inventory.get_addr(host) if host in self.mgr.inventory else host self.port = port self.data: str = json.dumps(data) super(AgentMessageThread, self).__init__(target=self.run) @@ -261,6 +261,7 @@ def run(self) -> None: secure_agent_socket.sendall(msg.encode('utf-8')) agent_response = secure_agent_socket.recv(1024).decode() self.mgr.log.info(f'Received "{agent_response}" from agent on host {self.host}') + self.mgr.cache.sending_agent_message[self.host] = False return except ConnectionError as e: # if it's a connection error, possibly try to connect again. diff --git a/src/pybind/mgr/cephadm/module.py b/src/pybind/mgr/cephadm/module.py index 41476c1c99207..8364696c9a95a 100644 --- a/src/pybind/mgr/cephadm/module.py +++ b/src/pybind/mgr/cephadm/module.py @@ -2119,8 +2119,8 @@ def _calc_daemon_deps(self, root_cert = self.cherrypy_thread.ssl_certs.get_root_cert() except Exception: pass - deps = sorted([self.get_mgr_ip(), self.inventory.get_addr(daemon_id), str(self.endpoint_port), - root_cert, str(self.get_module_option('device_enhanced_scan'))]) + deps = sorted([self.get_mgr_ip(), str(self.endpoint_port), root_cert, + str(self.get_module_option('device_enhanced_scan'))]) elif daemon_type == 'iscsi': deps = [self.get_mgr_ip()] else: diff --git a/src/pybind/mgr/cephadm/services/cephadmservice.py b/src/pybind/mgr/cephadm/services/cephadmservice.py index f80bf34669d25..da353bd5257ee 100644 --- a/src/pybind/mgr/cephadm/services/cephadmservice.py +++ b/src/pybind/mgr/cephadm/services/cephadmservice.py @@ -1043,6 +1043,6 @@ def generate_config(self, daemon_spec: CephadmDaemonDeploySpec) -> Tuple[Dict[st 'listener.key': listener_key, } - return config, sorted([str(self.mgr.get_mgr_ip()), self.mgr.inventory.get_addr(daemon_spec.host), - str(self.mgr.endpoint_port), self.mgr.cherrypy_thread.ssl_certs.get_root_cert(), + return config, sorted([str(self.mgr.get_mgr_ip()), str(self.mgr.endpoint_port), + self.mgr.cherrypy_thread.ssl_certs.get_root_cert(), str(self.mgr.get_module_option('device_enhanced_scan'))]) From db5cfb15e55dadf7bd5c381f53a4ea548fcea152 Mon Sep 17 00:00:00 2001 From: Nizamudeen A Date: Thu, 7 Oct 2021 21:06:29 +0530 Subject: [PATCH 0111/2866] mgr/dashboard: Fix orchestrator/01-hosts.e2e-spec.ts failure The test is failing on deleting a host because the agent daemon is present in that host. Its not possible to simply delete a host. We need to drain it first and then delete it. Fixes: https://tracker.ceph.com/issues/52764 Signed-off-by: Nizamudeen A --- .../integration/orchestrator/01-hosts.e2e-spec.ts | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/01-hosts.e2e-spec.ts b/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/01-hosts.e2e-spec.ts index cf85642a1b1d2..db72ddbd396cd 100644 --- a/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/01-hosts.e2e-spec.ts +++ b/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/01-hosts.e2e-spec.ts @@ -21,8 +21,20 @@ describe('Hosts page', () => { hosts.add(hostname, true); }); - it('should delete a host and add it back', function () { + it('should drain and delete a host and then add it back', function () { const host = Cypress._.last(this.hosts)['name']; + + // should drain the host first before deleting + hosts.editLabels(host, ['_no_schedule'], true); + hosts.clickHostTab(host, 'Daemons'); + cy.get('cd-host-details').within(() => { + // draining will take some time to complete. + // since we don't know how many daemons will be + // running in this host in future putting the wait + // to 15s + cy.wait(15000); + hosts.getTableCount('total').should('be.eq', 0); + }); hosts.delete(host); // add it back From 631360a41ab2ad93445d35a9730200beef2bebab Mon Sep 17 00:00:00 2001 From: Patrick Donnelly Date: Tue, 12 Oct 2021 14:00:50 -0400 Subject: [PATCH 0112/2866] mon/MDSMonitor: avoid crash when decoding old FSMap epochs Fixes: https://tracker.ceph.com/issues/52820 Signed-off-by: Patrick Donnelly --- src/mds/FSMap.cc | 1 + src/mon/MDSMonitor.cc | 7 ++++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/mds/FSMap.cc b/src/mds/FSMap.cc index d82f8c439cc57..9fe50244ef138 100644 --- a/src/mds/FSMap.cc +++ b/src/mds/FSMap.cc @@ -651,6 +651,7 @@ void FSMap::encode(bufferlist& bl, uint64_t features) const void FSMap::decode(bufferlist::const_iterator& p) { + struct_version = 0; DECODE_START(STRUCT_VERSION, p); DECODE_OLDEST(7); struct_version = struct_v; diff --git a/src/mon/MDSMonitor.cc b/src/mon/MDSMonitor.cc index c3a3d0046e4a0..b64671176f5ee 100644 --- a/src/mon/MDSMonitor.cc +++ b/src/mon/MDSMonitor.cc @@ -2335,7 +2335,12 @@ void MDSMonitor::tick() derr << "could not get version " << v << dendl; ceph_abort(); } - fsmap.decode(bl); + try { + fsmap.decode(bl); + } catch (const ceph::buffer::malformed_input& e) { + dout(5) << "flushing old fsmap struct because unable to decode FSMap: " << e.what() << dendl; + } + /* N.B. FSMap::is_struct_old is also true for undecoded (failed to decode) FSMap */ if (fsmap.is_struct_old()) { dout(5) << "fsmap struct is too old; proposing to flush out old versions" << dendl; do_propose = true; From eb3142710f17cbeec6393031fbb63ff6da7beb51 Mon Sep 17 00:00:00 2001 From: Casey Bodley Date: Tue, 12 Oct 2021 14:22:10 -0400 Subject: [PATCH 0113/2866] rgw: raise log level of realm/zonegroup/zone output on startup this output shows up in all radosgw-admin commands, and isn't relevant to most of them Signed-off-by: Casey Bodley --- src/rgw/services/svc_zone.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/rgw/services/svc_zone.cc b/src/rgw/services/svc_zone.cc index a3502b3b5abe2..3bcdbfd0eea83 100644 --- a/src/rgw/services/svc_zone.cc +++ b/src/rgw/services/svc_zone.cc @@ -271,12 +271,12 @@ int RGWSI_Zone::do_start(optional_yield y, const DoutPrefixProvider *dpp) *zone_public_config = zone_iter->second; ldout(cct, 20) << "zone " << zone_params->get_name() << " found" << dendl; - ldpp_dout(dpp, 1) << "Realm: " << std::left << setw(20) << realm->get_name() << " (" << realm->get_id() << ")" << dendl; - ldpp_dout(dpp, 1) << "ZoneGroup: " << std::left << setw(20) << zonegroup->get_name() << " (" << zonegroup->get_id() << ")" << dendl; - ldpp_dout(dpp, 1) << "Zone: " << std::left << setw(20) << zone_params->get_name() << " (" << zone_params->get_id() << ")" << dendl; + ldpp_dout(dpp, 4) << "Realm: " << std::left << setw(20) << realm->get_name() << " (" << realm->get_id() << ")" << dendl; + ldpp_dout(dpp, 4) << "ZoneGroup: " << std::left << setw(20) << zonegroup->get_name() << " (" << zonegroup->get_id() << ")" << dendl; + ldpp_dout(dpp, 4) << "Zone: " << std::left << setw(20) << zone_params->get_name() << " (" << zone_params->get_id() << ")" << dendl; if (init_from_period) { - ldpp_dout(dpp, 1) << "using period configuration: " << current_period->get_id() << ":" << current_period->get_epoch() << dendl; + ldpp_dout(dpp, 4) << "using period configuration: " << current_period->get_id() << ":" << current_period->get_epoch() << dendl; ret = init_zg_from_period(dpp, y); if (ret < 0) { return ret; From 845d21505713b8c1f48fb530d225e47a4c76f842 Mon Sep 17 00:00:00 2001 From: Casey Bodley Date: Tue, 12 Oct 2021 14:30:28 -0400 Subject: [PATCH 0114/2866] rgw: raise log level of http client headers these messages are especially spammy because the http client's dout prefix includes the entire request path: 2021-10-12T04:53:39.281+0000 7f66a0ff9700 10 http_client[POST/http://example.com/admin/log?type=data¬ify2&source-zone=3a571642-9f5e-46d8-8186-9eca8cc79ac6&rgwx-zonegroup=8f3b29b1-ffc6-4c90-9d0c-9bd258028cd8]receive_http_header 2021-10-12T04:53:39.281+0000 7f66a0ff9700 10 http_client[POST/http://example.com/admin/log?type=data¬ify2&source-zone=3a571642-9f5e-46d8-8186-9eca8cc79ac6&rgwx-zonegroup=8f3b29b1-ffc6-4c90-9d0c-9bd258028cd8]received header:HTTP/1.1 501 Not Implemented 2021-10-12T04:53:39.281+0000 7f66a0ff9700 10 http_client[POST/http://example.com/admin/log?type=data¬ify2&source-zone=3a571642-9f5e-46d8-8186-9eca8cc79ac6&rgwx-zonegroup=8f3b29b1-ffc6-4c90-9d0c-9bd258028cd8]receive_http_header 2021-10-12T04:53:39.281+0000 7f66a0ff9700 10 http_client[POST/http://example.com/admin/log?type=data¬ify2&source-zone=3a571642-9f5e-46d8-8186-9eca8cc79ac6&rgwx-zonegroup=8f3b29b1-ffc6-4c90-9d0c-9bd258028cd8]received header:Content-Length: 108 2021-10-12T04:53:39.281+0000 7f66a0ff9700 10 http_client[POST/http://example.com/admin/log?type=data¬ify2&source-zone=3a571642-9f5e-46d8-8186-9eca8cc79ac6&rgwx-zonegroup=8f3b29b1-ffc6-4c90-9d0c-9bd258028cd8]receive_http_header 2021-10-12T04:53:39.281+0000 7f66a0ff9700 10 http_client[POST/http://example.com/admin/log?type=data¬ify2&source-zone=3a571642-9f5e-46d8-8186-9eca8cc79ac6&rgwx-zonegroup=8f3b29b1-ffc6-4c90-9d0c-9bd258028cd8]received header:x-amz-request-id: tx00000000000000027e9ba-00616514d3-70a3-east 2021-10-12T04:53:39.281+0000 7f66a0ff9700 10 http_client[POST/http://example.com/admin/log?type=data¬ify2&source-zone=3a571642-9f5e-46d8-8186-9eca8cc79ac6&rgwx-zonegroup=8f3b29b1-ffc6-4c90-9d0c-9bd258028cd8]receive_http_header 2021-10-12T04:53:39.281+0000 7f66a0ff9700 10 http_client[POST/http://example.com/admin/log?type=data¬ify2&source-zone=3a571642-9f5e-46d8-8186-9eca8cc79ac6&rgwx-zonegroup=8f3b29b1-ffc6-4c90-9d0c-9bd258028cd8]received header:Accept-Ranges: bytes 2021-10-12T04:53:39.281+0000 7f66a0ff9700 10 http_client[POST/http://example.com/admin/log?type=data¬ify2&source-zone=3a571642-9f5e-46d8-8186-9eca8cc79ac6&rgwx-zonegroup=8f3b29b1-ffc6-4c90-9d0c-9bd258028cd8]receive_http_header 2021-10-12T04:53:39.281+0000 7f66a0ff9700 10 http_client[POST/http://example.com/admin/log?type=data¬ify2&source-zone=3a571642-9f5e-46d8-8186-9eca8cc79ac6&rgwx-zonegroup=8f3b29b1-ffc6-4c90-9d0c-9bd258028cd8]received header:Content-Type: application/json 2021-10-12T04:53:39.281+0000 7f66a0ff9700 10 http_client[POST/http://example.com/admin/log?type=data¬ify2&source-zone=3a571642-9f5e-46d8-8186-9eca8cc79ac6&rgwx-zonegroup=8f3b29b1-ffc6-4c90-9d0c-9bd258028cd8]receive_http_header 2021-10-12T04:53:39.281+0000 7f66a0ff9700 10 http_client[POST/http://example.com/admin/log?type=data¬ify2&source-zone=3a571642-9f5e-46d8-8186-9eca8cc79ac6&rgwx-zonegroup=8f3b29b1-ffc6-4c90-9d0c-9bd258028cd8]received header:Date: Tue, 12 Oct 2021 04:53:39 GMT 2021-10-12T04:53:39.281+0000 7f66a0ff9700 10 http_client[POST/http://example.com/admin/log?type=data¬ify2&source-zone=3a571642-9f5e-46d8-8186-9eca8cc79ac6&rgwx-zonegroup=8f3b29b1-ffc6-4c90-9d0c-9bd258028cd8]receive_http_header raise to 30 so they're only enabled for super-verbose logging Signed-off-by: Casey Bodley --- src/rgw/rgw_rest_client.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/rgw/rgw_rest_client.cc b/src/rgw/rgw_rest_client.cc index 54d185516eda7..68b40324ce268 100644 --- a/src/rgw/rgw_rest_client.cc +++ b/src/rgw/rgw_rest_client.cc @@ -49,7 +49,7 @@ int RGWHTTPSimpleRequest::receive_header(void *ptr, size_t len) char *s = (char *)ptr, *end = (char *)ptr + len; char *p = line; - ldpp_dout(this, 10) << "receive_http_header" << dendl; + ldpp_dout(this, 30) << "receive_http_header" << dendl; while (s != end) { if (*s == '\r') { @@ -58,7 +58,7 @@ int RGWHTTPSimpleRequest::receive_header(void *ptr, size_t len) } if (*s == '\n') { *p = '\0'; - ldpp_dout(this, 10) << "received header:" << line << dendl; + ldpp_dout(this, 30) << "received header:" << line << dendl; // TODO: fill whatever data required here char *l = line; char *tok = strsep(&l, " \t:"); From 5655329b3ad6a46514ca89ef1ce8436b50d09b94 Mon Sep 17 00:00:00 2001 From: Patrick Donnelly Date: Tue, 12 Oct 2021 14:39:42 -0400 Subject: [PATCH 0115/2866] mds/FSMap: do not assert allow_standby_replay on old FSMaps Octopus and older may have standby-replay daemons with allow_standby_replay==false. Fixes: https://tracker.ceph.com/issues/52874 Signed-off-by: Patrick Donnelly --- src/mds/FSMap.cc | 8 ++++++-- src/mds/FSMap.h | 2 +- src/mon/MDSMonitor.cc | 2 +- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/mds/FSMap.cc b/src/mds/FSMap.cc index d82f8c439cc57..1c9d3f52bccdc 100644 --- a/src/mds/FSMap.cc +++ b/src/mds/FSMap.cc @@ -850,8 +850,12 @@ const MDSMap::mds_info_t* FSMap::find_replacement_for(mds_role_t role) const return get_available_standby(*fs); } -void FSMap::sanity() const +void FSMap::sanity(bool pending) const { + /* Only do some sanity checks on **new** FSMaps. Older versions may not be + * compliant. + */ + if (legacy_client_fscid != FS_CLUSTER_ID_NONE) { ceph_assert(filesystems.count(legacy_client_fscid) == 1); } @@ -868,7 +872,7 @@ void FSMap::sanity() const ceph_assert(fs->mds_map.failed.count(info.rank) == 0); ceph_assert(fs->mds_map.damaged.count(info.rank) == 0); } else { - ceph_assert(fs->mds_map.allows_standby_replay()); + ceph_assert(!pending || fs->mds_map.allows_standby_replay()); } ceph_assert(info.compat.writeable(fs->mds_map.compat)); } diff --git a/src/mds/FSMap.h b/src/mds/FSMap.h index 62fa0a414f934..197d5141c17d5 100644 --- a/src/mds/FSMap.h +++ b/src/mds/FSMap.h @@ -568,7 +568,7 @@ class FSMap { * Assert that the FSMap, Filesystem, MDSMap, mds_info_t relations are * all self-consistent. */ - void sanity() const; + void sanity(bool pending=false) const; void encode(ceph::buffer::list& bl, uint64_t features) const; void decode(ceph::buffer::list::const_iterator& p); diff --git a/src/mon/MDSMonitor.cc b/src/mon/MDSMonitor.cc index c3a3d0046e4a0..7818e4258c6bc 100644 --- a/src/mon/MDSMonitor.cc +++ b/src/mon/MDSMonitor.cc @@ -190,7 +190,7 @@ void MDSMonitor::encode_pending(MonitorDBStore::TransactionRef t) // print map iff 'debug mon = 30' or higher print_map<30>(pending); if (!g_conf()->mon_mds_skip_sanity) { - pending.sanity(); + pending.sanity(true); } // Set 'modified' on maps modified this epoch From b57539dc947fd8a9f7164b6b4044288adcac2d8d Mon Sep 17 00:00:00 2001 From: Zack Cerza Date: Tue, 12 Oct 2021 12:43:34 -0600 Subject: [PATCH 0116/2866] Revert "qa: support isal ec test for aarch64" This commit has been causing scheduled jobs to request e.g. aarch64 smithi machines, which don't exist. The dispatcher then tries to find them forever, requiring the dispatcher to be killed and restarted. The queue will sit idle until someone notices the problem. Signed-off-by: Zack Cerza --- qa/standalone/erasure-code/test-erasure-code-plugins.sh | 2 +- qa/suites/rados/thrash-erasure-code-isa/arch/aarch64.yaml | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) delete mode 100644 qa/suites/rados/thrash-erasure-code-isa/arch/aarch64.yaml diff --git a/qa/standalone/erasure-code/test-erasure-code-plugins.sh b/qa/standalone/erasure-code/test-erasure-code-plugins.sh index 677a766863939..b5648d472b973 100755 --- a/qa/standalone/erasure-code/test-erasure-code-plugins.sh +++ b/qa/standalone/erasure-code/test-erasure-code-plugins.sh @@ -14,7 +14,7 @@ case $arch in aarch64*|arm*) legacy_jerasure_plugins=(jerasure_generic jerasure_neon) legacy_shec_plugins=(shec_generic shec_neon) - plugins=(jerasure shec lrc isa) + plugins=(jerasure shec lrc) ;; *) echo "unsupported platform ${arch}." diff --git a/qa/suites/rados/thrash-erasure-code-isa/arch/aarch64.yaml b/qa/suites/rados/thrash-erasure-code-isa/arch/aarch64.yaml deleted file mode 100644 index 6399b9959b595..0000000000000 --- a/qa/suites/rados/thrash-erasure-code-isa/arch/aarch64.yaml +++ /dev/null @@ -1 +0,0 @@ -arch: aarch64 From 869d1a5d9d6c3ebd56586b8e8b854ea242ea5e0d Mon Sep 17 00:00:00 2001 From: Jingya Su Date: Mon, 11 Oct 2021 22:04:23 -0400 Subject: [PATCH 0117/2866] mgr/volumes/fs/operations/versions: fix various flake8 issues Fixes: https://tracker.ceph.com/issues/51402 Signed-off-by: Jingya Su --- .../fs/operations/versions/subvolume_base.py | 170 ++++++++++++------ 1 file changed, 119 insertions(+), 51 deletions(-) diff --git a/src/pybind/mgr/volumes/fs/operations/versions/subvolume_base.py b/src/pybind/mgr/volumes/fs/operations/versions/subvolume_base.py index 2840a9f2ea354..346751b221092 100644 --- a/src/pybind/mgr/volumes/fs/operations/versions/subvolume_base.py +++ b/src/pybind/mgr/volumes/fs/operations/versions/subvolume_base.py @@ -1,6 +1,6 @@ import os import stat -import uuid + import errno import logging from hashlib import md5 @@ -9,16 +9,16 @@ import cephfs from ..pin_util import pin -from .subvolume_attrs import SubvolumeTypes, SubvolumeStates +from .subvolume_attrs import SubvolumeTypes from .metadata_manager import MetadataManager from ..trash import create_trashcan, open_trashcan from ...fs_util import get_ancestor_xattr from ...exception import MetadataMgrException, VolumeException -from .op_sm import SubvolumeOpSm from .auth_metadata import AuthMetadataManager log = logging.getLogger(__name__) + class SubvolumeBase(object): LEGACY_CONF_DIR = "_legacy" @@ -69,7 +69,8 @@ def config_path(self): @property def legacy_dir(self): - return os.path.join(self.vol_spec.base_dir.encode('utf-8'), SubvolumeBase.LEGACY_CONF_DIR.encode('utf-8')) + return (os.path.join(self.vol_spec.base_dir.encode('utf-8'), + SubvolumeBase.LEGACY_CONF_DIR.encode('utf-8'))) @property def legacy_config_path(self): @@ -105,7 +106,10 @@ def path(self): @property def features(self): - """ List of features supported by the subvolume, containing items from SubvolumeFeatures """ + """ + List of features supported by the subvolume, + containing items from SubvolumeFeatures + """ raise NotImplementedError @property @@ -115,7 +119,8 @@ def state(self): @property def subvol_type(self): - return SubvolumeTypes.from_value(self.metadata_mgr.get_global_option(MetadataManager.GLOBAL_META_KEY_TYPE)) + return (SubvolumeTypes.from_value(self.metadata_mgr.get_global_option + (MetadataManager.GLOBAL_META_KEY_TYPE))) @property def purgeable(self): @@ -124,15 +129,19 @@ def purgeable(self): def load_config(self): if self.legacy_mode: - self.metadata_mgr = MetadataManager(self.fs, self.legacy_config_path, 0o640) + self.metadata_mgr = MetadataManager(self.fs, + self.legacy_config_path, + 0o640) else: - self.metadata_mgr = MetadataManager(self.fs, self.config_path, 0o640) + self.metadata_mgr = MetadataManager(self.fs, + self.config_path, 0o640) def get_attrs(self, pathname): # get subvolume attributes - attrs = {} # type: Dict[str, Union[int, str, None]] + attrs = {} # type: Dict[str, Union[int, str, None]] stx = self.fs.statx(pathname, - cephfs.CEPH_STATX_UID | cephfs.CEPH_STATX_GID | cephfs.CEPH_STATX_MODE, + cephfs.CEPH_STATX_UID | cephfs.CEPH_STATX_GID + | cephfs.CEPH_STATX_MODE, cephfs.AT_SYMLINK_NOFOLLOW) attrs["uid"] = int(stx["uid"]) @@ -140,17 +149,24 @@ def get_attrs(self, pathname): attrs["mode"] = int(int(stx["mode"]) & ~stat.S_IFMT(stx["mode"])) try: - attrs["data_pool"] = self.fs.getxattr(pathname, 'ceph.dir.layout.pool').decode('utf-8') + attrs["data_pool"] = self.fs.getxattr(pathname, + 'ceph.dir.layout.pool' + ).decode('utf-8') except cephfs.NoData: attrs["data_pool"] = None try: - attrs["pool_namespace"] = self.fs.getxattr(pathname, 'ceph.dir.layout.pool_namespace').decode('utf-8') + attrs["pool_namespace"] = self.fs.getxattr(pathname, + 'ceph.dir.layout' + '.pool_namespace' + ).decode('utf-8') except cephfs.NoData: attrs["pool_namespace"] = None try: - attrs["quota"] = int(self.fs.getxattr(pathname, 'ceph.quota.max_bytes').decode('utf-8')) + attrs["quota"] = int(self.fs.getxattr(pathname, + 'ceph.quota.max_bytes' + ).decode('utf-8')) except cephfs.NoData: attrs["quota"] = None @@ -162,9 +178,11 @@ def set_attrs(self, path, attrs): quota = attrs.get("quota") if quota is not None: try: - self.fs.setxattr(path, 'ceph.quota.max_bytes', str(quota).encode('utf-8'), 0) - except cephfs.InvalidValue as e: - raise VolumeException(-errno.EINVAL, "invalid size specified: '{0}'".format(quota)) + self.fs.setxattr(path, 'ceph.quota.max_bytes', + str(quota).encode('utf-8'), 0) + except cephfs.InvalidValue: + raise VolumeException(-errno.EINVAL, + "invalid size specified: '{0}'".format(quota)) except cephfs.Error as e: raise VolumeException(-e.args[0], e.args[1]) @@ -172,10 +190,13 @@ def set_attrs(self, path, attrs): data_pool = attrs.get("data_pool") if data_pool is not None: try: - self.fs.setxattr(path, 'ceph.dir.layout.pool', data_pool.encode('utf-8'), 0) + self.fs.setxattr(path, 'ceph.dir.layout.pool', + data_pool.encode('utf-8'), 0) except cephfs.InvalidValue: raise VolumeException(-errno.EINVAL, - "invalid pool layout '{0}' -- need a valid data pool".format(data_pool)) + "invalid pool layout '{0}'" + "--need a valid data pool" + .format(data_pool)) except cephfs.Error as e: raise VolumeException(-e.args[0], e.args[1]) @@ -183,19 +204,22 @@ def set_attrs(self, path, attrs): xattr_key = xattr_val = None pool_namespace = attrs.get("pool_namespace") if pool_namespace is not None: - # enforce security isolation, use separate namespace for this subvolume + # enforce security isolation, use separate namespace + # for this subvolume xattr_key = 'ceph.dir.layout.pool_namespace' xattr_val = pool_namespace elif not data_pool: - # If subvolume's namespace layout is not set, then the subvolume's pool + # If subvolume's namespace layout is not set, + # then the subvolume's pool # layout remains unset and will undesirably change with ancestor's # pool layout changes. xattr_key = 'ceph.dir.layout.pool' xattr_val = None try: self.fs.getxattr(path, 'ceph.dir.layout.pool').decode('utf-8') - except cephfs.NoData as e: - xattr_val = get_ancestor_xattr(self.fs, os.path.split(path)[0], "ceph.dir.layout.pool") + except cephfs.NoData: + xattr_val = get_ancestor_xattr(self.fs, os.path.split(path)[0], + "ceph.dir.layout.pool") if xattr_key and xattr_val: try: self.fs.setxattr(path, xattr_key, xattr_val.encode('utf-8'), 0) @@ -230,16 +254,21 @@ def _resize(self, path, newsize, noshrink): try: newsize = int(newsize) if newsize <= 0: - raise VolumeException(-errno.EINVAL, "Invalid subvolume size") + raise VolumeException(-errno.EINVAL, + "Invalid subvolume size") except ValueError: newsize = newsize.lower() if not (newsize == "inf" or newsize == "infinite"): - raise VolumeException(-errno.EINVAL, "invalid size option '{0}'".format(newsize)) + raise (VolumeException(-errno.EINVAL, + "invalid size option '{0}'" + .format(newsize))) newsize = 0 noshrink = False try: - maxbytes = int(self.fs.getxattr(path, 'ceph.quota.max_bytes').decode('utf-8')) + maxbytes = int(self.fs.getxattr(path, + 'ceph.quota.max_bytes' + ).decode('utf-8')) except cephfs.NoData: maxbytes = 0 except cephfs.Error as e: @@ -248,25 +277,38 @@ def _resize(self, path, newsize, noshrink): subvolstat = self.fs.stat(path) if newsize > 0 and newsize < subvolstat.st_size: if noshrink: - raise VolumeException(-errno.EINVAL, "Can't resize the subvolume. The new size '{0}' would be lesser than the current " - "used size '{1}'".format(newsize, subvolstat.st_size)) + raise VolumeException(-errno.EINVAL, + "Can't resize the subvolume. " + "The new size '{0}' would be " + "lesser than the current " + "used size '{1}'" + .format(newsize, + subvolstat.st_size)) if not newsize == maxbytes: try: - self.fs.setxattr(path, 'ceph.quota.max_bytes', str(newsize).encode('utf-8'), 0) + self.fs.setxattr(path, 'ceph.quota.max_bytes', + str(newsize).encode('utf-8'), 0) except cephfs.Error as e: - raise VolumeException(-e.args[0], "Cannot set new size for the subvolume. '{0}'".format(e.args[1])) + raise (VolumeException(-e.args[0], + "Cannot set new size" + "for the subvolume. '{0}'" + .format(e.args[1]))) return newsize, subvolstat.st_size def pin(self, pin_type, pin_setting): return pin(self.fs, self.base_path, pin_type, pin_setting) - def init_config(self, version, subvolume_type, subvolume_path, subvolume_state): - self.metadata_mgr.init(version, subvolume_type.value, subvolume_path, subvolume_state.value) + def init_config(self, version, subvolume_type, + subvolume_path, subvolume_state): + self.metadata_mgr.init(version, subvolume_type.value, + subvolume_path, subvolume_state.value) self.metadata_mgr.flush() def discover(self): - log.debug("discovering subvolume '{0}' [mode: {1}]".format(self.subvolname, "legacy" if self.legacy_mode else "new")) + log.debug("discovering subvolume " + "'{0}' [mode: {1}]".format(self.subvolname, "legacy" + if self.legacy_mode else "new")) try: self.fs.stat(self.base_path) self.metadata_mgr.refresh() @@ -280,8 +322,13 @@ def discover(self): raise except cephfs.Error as e: if e.args[0] == errno.ENOENT: - raise VolumeException(-errno.ENOENT, "subvolume '{0}' does not exist".format(self.subvolname)) - raise VolumeException(-e.args[0], "error accessing subvolume '{0}'".format(self.subvolname)) + raise (VolumeException(-errno.ENOENT, + "subvolume '{0}' " + "does not exist" + .format(self.subvolname))) + raise VolumeException(-e.args[0], + "error accessing subvolume '{0}'" + .format(self.subvolname)) def _trash_dir(self, path): create_trashcan(self.fs, self.vol_spec) @@ -293,7 +340,8 @@ def _link_dir(self, path, bname): create_trashcan(self.fs, self.vol_spec) with open_trashcan(self.fs, self.vol_spec) as trashcan: trashcan.link(path, bname) - log.info("subvolume path '{0}' linked in trashcan bname {1}".format(path, bname)) + log.info("subvolume path '{0}' " + "linked in trashcan bname {1}".format(path, bname)) def trash_base_dir(self): if self.legacy_mode: @@ -306,29 +354,49 @@ def create_base_dir(self, mode): except cephfs.Error as e: raise VolumeException(-e.args[0], e.args[1]) - def info (self): - subvolpath = self.metadata_mgr.get_global_option(MetadataManager.GLOBAL_META_KEY_PATH) + def info(self): + subvolpath = (self.metadata_mgr.get_global_option( + MetadataManager.GLOBAL_META_KEY_PATH)) etype = self.subvol_type - st = self.fs.statx(subvolpath, cephfs.CEPH_STATX_BTIME | cephfs.CEPH_STATX_SIZE | - cephfs.CEPH_STATX_UID | cephfs.CEPH_STATX_GID | - cephfs.CEPH_STATX_MODE | cephfs.CEPH_STATX_ATIME | - cephfs.CEPH_STATX_MTIME | cephfs.CEPH_STATX_CTIME, - cephfs.AT_SYMLINK_NOFOLLOW) + st = self.fs.statx(subvolpath, cephfs.CEPH_STATX_BTIME + | cephfs.CEPH_STATX_SIZE + | cephfs.CEPH_STATX_UID | cephfs.CEPH_STATX_GID + | cephfs.CEPH_STATX_MODE | cephfs.CEPH_STATX_ATIME + | cephfs.CEPH_STATX_MTIME + | cephfs.CEPH_STATX_CTIME, + cephfs.AT_SYMLINK_NOFOLLOW) usedbytes = st["size"] try: - nsize = int(self.fs.getxattr(subvolpath, 'ceph.quota.max_bytes').decode('utf-8')) + nsize = int(self.fs.getxattr(subvolpath, + 'ceph.quota.max_bytes' + ).decode('utf-8')) except cephfs.NoData: nsize = 0 try: - data_pool = self.fs.getxattr(subvolpath, 'ceph.dir.layout.pool').decode('utf-8') - pool_namespace = self.fs.getxattr(subvolpath, 'ceph.dir.layout.pool_namespace').decode('utf-8') + data_pool = self.fs.getxattr(subvolpath, + 'ceph.dir.layout.pool' + ).decode('utf-8') + pool_namespace = self.fs.getxattr(subvolpath, + 'ceph.dir.layout.pool_namespace' + ).decode('utf-8') except cephfs.Error as e: raise VolumeException(-e.args[0], e.args[1]) - return {'path': subvolpath, 'type': etype.value, 'uid': int(st["uid"]), 'gid': int(st["gid"]), - 'atime': str(st["atime"]), 'mtime': str(st["mtime"]), 'ctime': str(st["ctime"]), - 'mode': int(st["mode"]), 'data_pool': data_pool, 'created_at': str(st["btime"]), - 'bytes_quota': "infinite" if nsize == 0 else nsize, 'bytes_used': int(usedbytes), - 'bytes_pcent': "undefined" if nsize == 0 else '{0:.2f}'.format((float(usedbytes) / nsize) * 100.0), - 'pool_namespace': pool_namespace, 'features': self.features, 'state': self.state.value} + return {'path': subvolpath, + 'type': etype.value, + 'uid': int(st["uid"]), + 'gid': int(st["gid"]), + 'atime': str(st["atime"]), + 'mtime': str(st["mtime"]), + 'ctime': str(st["ctime"]), + 'mode': int(st["mode"]), + 'data_pool': data_pool, + 'created_at': str(st["btime"]), + 'bytes_quota': "infinite" if nsize == 0 else nsize, + 'bytes_used': int(usedbytes), + 'bytes_pcent': "undefined" + if nsize == 0 + else '{0:.2f}'.format((float(usedbytes) / nsize) * 100.0), + 'pool_namespace': pool_namespace, + 'features': self.features, 'state': self.state.value} From 70f1c33707fe0c3bbf00a3ded5d6678b445e4e92 Mon Sep 17 00:00:00 2001 From: Jianpeng Ma Date: Wed, 13 Oct 2021 08:35:24 +0800 Subject: [PATCH 0118/2866] test/librbd: re-enable BlockGuard test. This patch partly revert PR#16255. Signed-off-by: Jianpeng Ma --- src/test/librbd/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/librbd/CMakeLists.txt b/src/test/librbd/CMakeLists.txt index a110016efd2eb..df6e8c5e92da8 100644 --- a/src/test/librbd/CMakeLists.txt +++ b/src/test/librbd/CMakeLists.txt @@ -11,7 +11,6 @@ set(librbd_test test_ImageWatcher.cc test_internal.cc test_mirroring.cc - test_BlockGuard.cc test_DeepCopy.cc test_Groups.cc test_Migration.cc @@ -41,6 +40,7 @@ target_link_libraries(rbd_test_mock PUBLIC # unittest_librbd # doesn't use add_ceph_test because it is called by run-rbd-unit-tests.sh set(unittest_librbd_srcs + test_BlockGuard.cc test_main.cc test_mock_fixture.cc test_mock_ConfigWatcher.cc From 82bcf9b57fc4939677ef1af20a1a3fa258243a37 Mon Sep 17 00:00:00 2001 From: myoungwon oh Date: Tue, 12 Oct 2021 12:09:22 +0900 Subject: [PATCH 0119/2866] test: increase retry duration when calculating manifest ref. count In situation where the object is degraded and delayed, retry time can expire before the object is recovered ---it takes almost 6 minutes to be recovered according to the log. Fixes: https://tracker.ceph.com/issues/52872 Signed-off-by: Myoungwon Oh --- src/test/librados/tier_cxx.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/test/librados/tier_cxx.cc b/src/test/librados/tier_cxx.cc index 4239084b6c714..287c6fa2ae6ac 100644 --- a/src/test/librados/tier_cxx.cc +++ b/src/test/librados/tier_cxx.cc @@ -177,7 +177,8 @@ void is_intended_refcount_state(librados::IoCtx& src_ioctx, } dst_refcount = refs.count(); } - for (int tries = 0; tries < 15; ++tries) { + int tries = 0; + for (; tries < 30; ++tries) { r = cls_cas_references_chunk(src_ioctx, src_oid, dst_oid); if (r == -ENOENT || r == -ENOLINK) { src_refcount = 0; @@ -189,6 +190,7 @@ void is_intended_refcount_state(librados::IoCtx& src_ioctx, } break; } + ASSERT_TRUE(tries < 30); ASSERT_TRUE(src_refcount >= 0); ASSERT_TRUE(src_refcount == expected_refcount); ASSERT_TRUE(src_refcount <= dst_refcount); From 4e6a50baeb37831bbd09cea7cc71428c47c5bbaa Mon Sep 17 00:00:00 2001 From: Michael Fritch Date: Mon, 11 Oct 2021 12:10:17 -0600 Subject: [PATCH 0120/2866] cephadm: use the current cephadm binary for the agent 78983ad0d0cc introduced passing the cephadm binary via config json, when the currently deployed cephadm binary could be used in the agent's unit.run file instead Signed-off-by: Michael Fritch --- src/cephadm/cephadm | 6 +++--- src/pybind/mgr/cephadm/services/cephadmservice.py | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/cephadm/cephadm b/src/cephadm/cephadm index ed392bd187d75..95d9fbc246135 100755 --- a/src/cephadm/cephadm +++ b/src/cephadm/cephadm @@ -3505,7 +3505,7 @@ class CephadmAgent(): loop_interval = 30 stop = False - required_files = ['cephadm', 'agent.json', 'keyring'] + required_files = ['agent.json', 'keyring'] def __init__(self, ctx: CephadmContext, fsid: str, daemon_id: Union[int, str] = ''): self.ctx = ctx @@ -3515,7 +3515,6 @@ class CephadmAgent(): self.target_port = '' self.host = '' self.daemon_dir = os.path.join(ctx.data_dir, self.fsid, f'{self.daemon_type}.{self.daemon_id}') - self.binary_path = os.path.join(self.daemon_dir, 'cephadm') self.config_path = os.path.join(self.daemon_dir, 'agent.json') self.keyring_path = os.path.join(self.daemon_dir, 'keyring') self.ca_path = os.path.join(self.daemon_dir, 'root_cert.pem') @@ -3559,7 +3558,8 @@ class CephadmAgent(): def unit_run(self) -> str: py3 = shutil.which('python3') - return ('set -e\n' + f'{py3} {self.binary_path} agent --fsid {self.fsid} --daemon-id {self.daemon_id} &\n') + binary_path = os.path.realpath(sys.argv[0]) + return ('set -e\n' + f'{py3} {binary_path} agent --fsid {self.fsid} --daemon-id {self.daemon_id} &\n') def unit_file(self) -> str: return """#generated by cephadm diff --git a/src/pybind/mgr/cephadm/services/cephadmservice.py b/src/pybind/mgr/cephadm/services/cephadmservice.py index f80bf34669d25..0cf78cb82a61b 100644 --- a/src/pybind/mgr/cephadm/services/cephadmservice.py +++ b/src/pybind/mgr/cephadm/services/cephadmservice.py @@ -1036,7 +1036,6 @@ def generate_config(self, daemon_spec: CephadmDaemonDeploySpec) -> Tuple[Dict[st self.mgr.inventory.get_addr(daemon_spec.host)) config = { 'agent.json': json.dumps(cfg), - 'cephadm': self.mgr._cephadm, 'keyring': daemon_spec.keyring, 'root_cert.pem': self.mgr.cherrypy_thread.ssl_certs.get_root_cert(), 'listener.crt': listener_cert, From ffe7de88530dda661fca5fd0075434d4c8a349ac Mon Sep 17 00:00:00 2001 From: Kalpesh Pandya Date: Wed, 22 Sep 2021 15:56:21 +0530 Subject: [PATCH 0121/2866] src/test: HTTP POST object upload This solves the tracker: https://tracker.ceph.com/issues/51724 Basically it is using 'generate_presigned_post()' boto3 API. This is verified under AMQP endpoint. Signed-off-by: Kalpesh Pandya --- src/test/rgw/bucket_notification/test_bn.py | 73 ++++++++++++++++++++- 1 file changed, 72 insertions(+), 1 deletion(-) diff --git a/src/test/rgw/bucket_notification/test_bn.py b/src/test/rgw/bucket_notification/test_bn.py index ccecfbaf9ce22..ec6b74d661714 100644 --- a/src/test/rgw/bucket_notification/test_bn.py +++ b/src/test/rgw/bucket_notification/test_bn.py @@ -1728,6 +1728,77 @@ def test_ps_s3_creation_triggers_on_master_ssl(): del os.environ['RABBITMQ_CONFIG_FILE'] +@attr('amqp_test') +def test_http_post_object_upload(): + """ test that uploads object using HTTP POST """ + + import boto3 + from collections import OrderedDict + import requests + + hostname = get_ip() + zonegroup = 'default' + conn = connection() + + endpoint = "http://%s:%d" % (get_config_host(), get_config_port()) + + conn1 = boto3.client(service_name='s3', + aws_access_key_id=get_access_key(), + aws_secret_access_key=get_secret_key(), + endpoint_url=endpoint, + ) + + bucket_name = gen_bucket_name() + topic_name = bucket_name + TOPIC_SUFFIX + + key_name = 'foo.txt' + + resp = conn1.generate_presigned_post(Bucket=bucket_name, Key=key_name,) + + url = resp['url'] + + bucket = conn1.create_bucket(ACL='public-read-write', Bucket=bucket_name) + + # start amqp receivers + exchange = 'ex1' + task1, receiver1 = create_amqp_receiver_thread(exchange, topic_name+'_1') + task1.start() + + # create s3 topics + endpoint_address = 'amqp://' + hostname + endpoint_args = 'push-endpoint=' + endpoint_address + '&amqp-exchange=' + exchange + '&amqp-ack-level=broker' + topic_conf1 = PSTopicS3(conn, topic_name+'_1', zonegroup, endpoint_args=endpoint_args) + topic_arn1 = topic_conf1.set_config() + + # create s3 notifications + notification_name = bucket_name + NOTIFICATION_SUFFIX + topic_conf_list = [{'Id': notification_name+'_1', 'TopicArn': topic_arn1, + 'Events': ['s3:ObjectCreated:Post'] + }] + s3_notification_conf = PSNotificationS3(conn, bucket_name, topic_conf_list) + response, status = s3_notification_conf.set_config() + assert_equal(status/100, 2) + + payload = OrderedDict([("key" , "foo.txt"),("acl" , "public-read"),\ + ("Content-Type" , "text/plain"),('file', ('bar'))]) + + # POST upload + r = requests.post(url, files=payload, verify=True) + assert_equal(r.status_code, 204) + + # check amqp receiver + events = receiver1.get_and_reset_events() + assert_equal(len(events), 1) + + # cleanup + stop_amqp_receiver(receiver1, task1) + s3_notification_conf.del_config() + topic_conf1.del_config() + conn1.delete_object(Bucket=bucket_name, Key=key_name) + # delete the bucket + conn1.delete_bucket(Bucket=bucket_name) + + @attr('amqp_test') def test_ps_s3_multipart_on_master(): """ test multipart object upload on master""" @@ -1898,7 +1969,7 @@ def test_ps_s3_metadata_on_master(): time.sleep(5) # check amqp receiver events = receiver.get_and_reset_events() - assert_equal(len(events), 3) # PUT, COPY, Multipart start, Multipart End + assert_equal(len(events), 3) # PUT, COPY, Multipart Complete for event in events: assert(event['Records'][0]['s3']['object']['key'] in expected_keys) From 29fa2cec790b5a68d9a8edf48cb47ca69d08c6cc Mon Sep 17 00:00:00 2001 From: Yongseok Oh Date: Fri, 24 Sep 2021 06:04:55 +0000 Subject: [PATCH 0122/2866] mds: improve mds_bal_fragment_size_max config option mds_bal_fragment_size_max is maintained as bal_fragment_size_max cached variable Fixes: https://tracker.ceph.com/issues/52723 Signed-off-by: Yongseok Oh --- src/mds/MDSRank.cc | 1 + src/mds/Server.cc | 8 +++++++- src/mds/Server.h | 1 + 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/mds/MDSRank.cc b/src/mds/MDSRank.cc index 190a53706455d..f870c61fe5d91 100644 --- a/src/mds/MDSRank.cc +++ b/src/mds/MDSRank.cc @@ -3672,6 +3672,7 @@ const char** MDSRankDispatcher::get_tracked_conf_keys() const "host", "mds_bal_fragment_dirs", "mds_bal_fragment_interval", + "mds_bal_fragment_size_max", "mds_cache_memory_limit", "mds_cache_mid", "mds_cache_reservation", diff --git a/src/mds/Server.cc b/src/mds/Server.cc index 095001076220f..1ed15937013c7 100644 --- a/src/mds/Server.cc +++ b/src/mds/Server.cc @@ -263,6 +263,7 @@ Server::Server(MDSRank *m, MetricsHandler *metrics_handler) : max_caps_throttle_ratio = g_conf().get_val("mds_session_max_caps_throttle_ratio"); caps_throttle_retry_request_timeout = g_conf().get_val("mds_cap_acquisition_throttle_retry_request_timeout"); dir_max_entries = g_conf().get_val("mds_dir_max_entries"); + bal_fragment_size_max = g_conf().get_val("mds_bal_fragment_size_max"); supported_features = feature_bitset_t(CEPHFS_FEATURES_MDS_SUPPORTED); } @@ -1263,6 +1264,11 @@ void Server::handle_conf_change(const std::set& changed) { dout(20) << __func__ << " max entries per directory changed to " << dir_max_entries << dendl; } + if (changed.count("mds_bal_fragment_size_max")) { + bal_fragment_size_max = g_conf().get_val("mds_bal_fragment_size_max"); + dout(20) << __func__ << " max fragment size changed to " + << bal_fragment_size_max << dendl; + } } /* @@ -3205,7 +3211,7 @@ bool Server::check_access(MDRequestRef& mdr, CInode *in, unsigned mask) bool Server::check_fragment_space(MDRequestRef &mdr, CDir *dir) { const auto size = dir->get_frag_size(); - const auto max = g_conf()->mds_bal_fragment_size_max; + const auto max = bal_fragment_size_max; if (size >= max) { dout(10) << "fragment " << *dir << " size exceeds " << max << " (CEPHFS_ENOSPC)" << dendl; respond_to_request(mdr, -CEPHFS_ENOSPC); diff --git a/src/mds/Server.h b/src/mds/Server.h index 31ed68c45aeb9..76f38717c7265 100644 --- a/src/mds/Server.h +++ b/src/mds/Server.h @@ -461,6 +461,7 @@ class Server { uint64_t max_snaps_per_dir = 100; unsigned delegate_inos_pct = 0; uint64_t dir_max_entries = 0; + int64_t bal_fragment_size_max = 0; DecayCounter recall_throttle; time last_recall_state; From b9f38cadc46338c9f1527beeaaba3d42639badf9 Mon Sep 17 00:00:00 2001 From: Avan Thakkar Date: Tue, 1 Jun 2021 18:25:15 +0530 Subject: [PATCH 0123/2866] mgr/dashboard: Create Cluster Workflow welcome screen and e2e tests A module option called CLUSTER_STATUS has two option. INSTALLED AND POST_INSTALLED. When CLUSTER_STATUS is INSTALLED it will allow to show the create-cluster-wizard after login the initial time. After the cluster creation is succesfull this option is set to POST_INSTALLED Also has the e2e codes for the Review Section Fixes: https://tracker.ceph.com/issues/50336 Signed-off-by: Avan Thakkar Signed-off-by: Nizamudeen A --- qa/tasks/mgr/dashboard/test_auth.py | 6 +- qa/tasks/mgr/dashboard/test_cluster.py | 23 +++++++ src/pybind/mgr/dashboard/controllers/auth.py | 2 + .../mgr/dashboard/controllers/cluster.py | 20 ++++++ .../cluster/cluster-welcome-page.po.ts | 22 +++++++ .../cluster/create-cluster-review.po.ts | 11 ++++ .../06-cluster-welcome-page.e2e-spec.ts | 19 ++++++ .../08-create-cluster-review.e2e-spec.ts | 61 ++++++++++++++++++ .../frontend/src/app/app-routing.module.ts | 14 +++++ .../src/app/ceph/cluster/cluster.module.ts | 4 +- .../create-cluster.component.html | 29 +++++++++ .../create-cluster.component.scss | 0 .../create-cluster.component.spec.ts | 45 +++++++++++++ .../create-cluster.component.ts | 44 +++++++++++++ .../core/auth/login/login.component.spec.ts | 26 ++++++++ .../app/core/auth/login/login.component.ts | 8 ++- .../app/shared/api/cluster.service.spec.ts | 42 +++++++++++++ .../src/app/shared/api/cluster.service.ts | 27 ++++++++ .../src/app/shared/constants/app.constants.ts | 1 + .../module-status-guard.service.spec.ts | 19 +++++- .../services/module-status-guard.service.ts | 17 ++++- src/pybind/mgr/dashboard/openapi.yaml | 63 +++++++++++++++++++ src/pybind/mgr/dashboard/services/cluster.py | 26 ++++++++ 23 files changed, 520 insertions(+), 9 deletions(-) create mode 100644 qa/tasks/mgr/dashboard/test_cluster.py create mode 100644 src/pybind/mgr/dashboard/controllers/cluster.py create mode 100644 src/pybind/mgr/dashboard/frontend/cypress/integration/cluster/cluster-welcome-page.po.ts create mode 100644 src/pybind/mgr/dashboard/frontend/cypress/integration/cluster/create-cluster-review.po.ts create mode 100644 src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/06-cluster-welcome-page.e2e-spec.ts create mode 100644 src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/08-create-cluster-review.e2e-spec.ts create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster.component.html create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster.component.scss create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster.component.spec.ts create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster.component.ts create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/shared/api/cluster.service.spec.ts create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/shared/api/cluster.service.ts create mode 100644 src/pybind/mgr/dashboard/services/cluster.py diff --git a/qa/tasks/mgr/dashboard/test_auth.py b/qa/tasks/mgr/dashboard/test_auth.py index 8fc7cd1992e6d..98566344444f7 100644 --- a/qa/tasks/mgr/dashboard/test_auth.py +++ b/qa/tasks/mgr/dashboard/test_auth.py @@ -335,7 +335,8 @@ def test_check_wo_token(self): self.assertStatus(200) data = self.jsonBody() self.assertSchema(data, JObj(sub_elems={ - "login_url": JLeaf(str) + "login_url": JLeaf(str), + "cluster_status": JLeaf(str) }, allow_unknown=False)) self.logout() @@ -345,6 +346,7 @@ def test_check_wo_token(self): self.assertStatus(200) data = self.jsonBody() self.assertSchema(data, JObj(sub_elems={ - "login_url": JLeaf(str) + "login_url": JLeaf(str), + "cluster_status": JLeaf(str) }, allow_unknown=False)) self.logout(set_cookies=True) diff --git a/qa/tasks/mgr/dashboard/test_cluster.py b/qa/tasks/mgr/dashboard/test_cluster.py new file mode 100644 index 0000000000000..14f8542796c55 --- /dev/null +++ b/qa/tasks/mgr/dashboard/test_cluster.py @@ -0,0 +1,23 @@ +from .helper import DashboardTestCase, JLeaf, JObj + + +class ClusterTest(DashboardTestCase): + + def setUp(self): + super().setUp() + self.reset_session() + + def test_get_status(self): + data = self._get('/api/cluster', version='0.1') + self.assertStatus(200) + self.assertSchema(data, JObj(sub_elems={ + "status": JLeaf(str) + }, allow_unknown=False)) + + def test_update_status(self): + req = {'status': 'POST_INSTALLED'} + self._put('/api/cluster', req, version='0.1') + self.assertStatus(200) + data = self._get('/api/cluster', version='0.1') + self.assertStatus(200) + self.assertEqual(data, req) diff --git a/src/pybind/mgr/dashboard/controllers/auth.py b/src/pybind/mgr/dashboard/controllers/auth.py index 353d5d72bb9e8..196f027b293ee 100644 --- a/src/pybind/mgr/dashboard/controllers/auth.py +++ b/src/pybind/mgr/dashboard/controllers/auth.py @@ -7,6 +7,7 @@ from .. import mgr from ..exceptions import InvalidCredentialsError, UserDoesNotExist from ..services.auth import AuthManager, JwtManager +from ..services.cluster import ClusterModel from ..settings import Settings from . import APIDoc, APIRouter, ControllerAuthMixin, EndpointDoc, RESTController, allow_empty_body @@ -117,4 +118,5 @@ def check(self, token): } return { 'login_url': self._get_login_url(), + 'cluster_status': ClusterModel.from_db().dict()['status'] } diff --git a/src/pybind/mgr/dashboard/controllers/cluster.py b/src/pybind/mgr/dashboard/controllers/cluster.py new file mode 100644 index 0000000000000..5ec49e39b1c22 --- /dev/null +++ b/src/pybind/mgr/dashboard/controllers/cluster.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- + +from ..security import Scope +from ..services.cluster import ClusterModel +from . import ApiController, ControllerDoc, EndpointDoc, RESTController + + +@ApiController('/cluster', Scope.CONFIG_OPT) +@ControllerDoc("Get Cluster Details", "Cluster") +class Cluster(RESTController): + @RESTController.MethodMap(version='0.1') + @EndpointDoc("Get the cluster status") + def list(self): + return ClusterModel.from_db().dict() + + @RESTController.MethodMap(version='0.1') + @EndpointDoc("Update the cluster status", + parameters={'status': (str, 'Cluster Status')}) + def singleton_set(self, status: str): + ClusterModel(status).to_db() diff --git a/src/pybind/mgr/dashboard/frontend/cypress/integration/cluster/cluster-welcome-page.po.ts b/src/pybind/mgr/dashboard/frontend/cypress/integration/cluster/cluster-welcome-page.po.ts new file mode 100644 index 0000000000000..5615b0369d025 --- /dev/null +++ b/src/pybind/mgr/dashboard/frontend/cypress/integration/cluster/cluster-welcome-page.po.ts @@ -0,0 +1,22 @@ +import { PageHelper } from '../page-helper.po'; +import { NotificationSidebarPageHelper } from '../ui/notification.po'; + +export class CreateClusterWelcomePageHelper extends PageHelper { + pages = { + index: { url: '#/create-cluster', id: 'cd-create-cluster' } + }; + + createCluster() { + cy.get('cd-create-cluster').should('contain.text', 'Welcome to Ceph'); + cy.get('[name=create-cluster]').click(); + } + + doSkip() { + cy.get('[name=skip-cluster-creation]').click(); + + cy.get('cd-dashboard').should('exist'); + const notification = new NotificationSidebarPageHelper(); + notification.open(); + notification.getNotifications().should('contain', 'Cluster creation skipped by user'); + } +} diff --git a/src/pybind/mgr/dashboard/frontend/cypress/integration/cluster/create-cluster-review.po.ts b/src/pybind/mgr/dashboard/frontend/cypress/integration/cluster/create-cluster-review.po.ts new file mode 100644 index 0000000000000..58844e39afec8 --- /dev/null +++ b/src/pybind/mgr/dashboard/frontend/cypress/integration/cluster/create-cluster-review.po.ts @@ -0,0 +1,11 @@ +import { PageHelper } from '../page-helper.po'; + +export class CreateClusterReviewPageHelper extends PageHelper { + pages = { + index: { url: '#/create-cluster', id: 'cd-create-cluster-review' } + }; + + checkDefaultHostName() { + this.getTableCell(1, 'ceph-node-00.cephlab.com').should('exist'); + } +} diff --git a/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/06-cluster-welcome-page.e2e-spec.ts b/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/06-cluster-welcome-page.e2e-spec.ts new file mode 100644 index 0000000000000..bd0470b86700a --- /dev/null +++ b/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/06-cluster-welcome-page.e2e-spec.ts @@ -0,0 +1,19 @@ +import { CreateClusterWelcomePageHelper } from '../cluster/cluster-welcome-page.po'; + +describe('Create cluster page', () => { + const createCluster = new CreateClusterWelcomePageHelper(); + + beforeEach(() => { + cy.login(); + Cypress.Cookies.preserveOnce('token'); + createCluster.navigateTo(); + }); + + it('should fail to create cluster', () => { + createCluster.createCluster(); + }); + + it('should skip to dashboard landing page', () => { + createCluster.doSkip(); + }); +}); diff --git a/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/08-create-cluster-review.e2e-spec.ts b/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/08-create-cluster-review.e2e-spec.ts new file mode 100644 index 0000000000000..a472810e6e6c5 --- /dev/null +++ b/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/08-create-cluster-review.e2e-spec.ts @@ -0,0 +1,61 @@ +import { CreateClusterWelcomePageHelper } from 'cypress/integration/cluster/cluster-welcome-page.po'; +import { CreateClusterReviewPageHelper } from 'cypress/integration/cluster/create-cluster-review.po'; + +describe('Create Cluster Review page', () => { + const reviewPage = new CreateClusterReviewPageHelper(); + const createCluster = new CreateClusterWelcomePageHelper(); + + beforeEach(() => { + cy.login(); + Cypress.Cookies.preserveOnce('token'); + createCluster.navigateTo(); + createCluster.createCluster(); + + cy.get('button[aria-label="Next"]').click(); + }); + + describe('navigation link and title test', () => { + it('should check if nav-link and title contains Review', () => { + cy.get('.nav-link').should('contain.text', 'Review'); + + cy.get('.title').should('contain.text', 'Review'); + }); + }); + + describe('fields check', () => { + it('should check cluster resources table is present', () => { + // check for table header 'Status' + reviewPage.getLegends().its(0).should('have.text', 'Cluster Resources'); + + // check for fields in table + reviewPage.getStatusTables().should('contain.text', 'Hosts'); + }); + + it('should check Hosts Per Label and Host Details tables are present', () => { + // check for there to be two tables + reviewPage.getDataTables().should('have.length', 2); + + // check for table header 'Hosts Per Label' + reviewPage.getLegends().its(1).should('have.text', 'Hosts Per Label'); + + // check for table header 'Host Details' + reviewPage.getLegends().its(2).should('have.text', 'Host Details'); + + // verify correct columns on Hosts Per Label table + reviewPage.getDataTableHeaders(0).contains('Label'); + + reviewPage.getDataTableHeaders(0).contains('Number of Hosts'); + + // verify correct columns on Host Details table + reviewPage.getDataTableHeaders(1).contains('Host Name'); + + reviewPage.getDataTableHeaders(1).contains('Labels'); + }); + + it('should check hosts count and default host name are present', () => { + reviewPage.getStatusTables().should('contain.text', '1'); + + reviewPage.checkDefaultHostName(); + }); + }); +}); diff --git a/src/pybind/mgr/dashboard/frontend/src/app/app-routing.module.ts b/src/pybind/mgr/dashboard/frontend/src/app/app-routing.module.ts index ebbe6f6651c1c..099b31efbda50 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/app-routing.module.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/app-routing.module.ts @@ -6,6 +6,7 @@ import _ from 'lodash'; import { CephfsListComponent } from './ceph/cephfs/cephfs-list/cephfs-list.component'; import { ConfigurationFormComponent } from './ceph/cluster/configuration/configuration-form/configuration-form.component'; import { ConfigurationComponent } from './ceph/cluster/configuration/configuration.component'; +import { CreateClusterComponent } from './ceph/cluster/create-cluster/create-cluster.component'; import { CrushmapComponent } from './ceph/cluster/crushmap/crushmap.component'; import { HostFormComponent } from './ceph/cluster/hosts/host-form/host-form.component'; import { HostsComponent } from './ceph/cluster/hosts/hosts.component'; @@ -89,6 +90,19 @@ const routes: Routes = [ { path: 'error', component: ErrorComponent }, // Cluster + { + path: 'create-cluster', + component: CreateClusterComponent, + canActivate: [ModuleStatusGuardService], + data: { + moduleStatusGuardConfig: { + apiPath: 'orchestrator', + redirectTo: 'dashboard', + backend: 'cephadm' + }, + breadcrumbs: 'Create Cluster' + } + }, { path: 'hosts', data: { breadcrumbs: 'Cluster/Hosts' }, diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/cluster.module.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/cluster.module.ts index cc58c38b8dc9c..a2c1e6d2f89ec 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/cluster.module.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/cluster.module.ts @@ -21,6 +21,7 @@ import { CephSharedModule } from '../shared/ceph-shared.module'; import { ConfigurationDetailsComponent } from './configuration/configuration-details/configuration-details.component'; import { ConfigurationFormComponent } from './configuration/configuration-form/configuration-form.component'; import { ConfigurationComponent } from './configuration/configuration.component'; +import { CreateClusterComponent } from './create-cluster/create-cluster.component'; import { CrushmapComponent } from './crushmap/crushmap.component'; import { HostDetailsComponent } from './hosts/host-details/host-details.component'; import { HostFormComponent } from './hosts/host-form/host-form.component'; @@ -112,7 +113,8 @@ import { TelemetryComponent } from './telemetry/telemetry.component'; PrometheusTabsComponent, ServiceFormComponent, OsdFlagsIndivModalComponent, - PlacementPipe + PlacementPipe, + CreateClusterComponent ] }) export class ClusterModule {} diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster.component.html new file mode 100644 index 0000000000000..661c13fc931c9 --- /dev/null +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster.component.html @@ -0,0 +1,29 @@ +
+
+
+ + Ceph +

Welcome to {{ projectConstants.projectName }}

+ +
+

Please proceed to complete the cluster creation

+
+ + +
+
+
+
+
diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster.component.scss b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster.component.scss new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster.component.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster.component.spec.ts new file mode 100644 index 0000000000000..7e061b2e25c9a --- /dev/null +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster.component.spec.ts @@ -0,0 +1,45 @@ +import { HttpClientTestingModule } from '@angular/common/http/testing'; +import { ComponentFixture, TestBed } from '@angular/core/testing'; +import { By } from '@angular/platform-browser'; +import { RouterTestingModule } from '@angular/router/testing'; + +import { ToastrModule } from 'ngx-toastr'; + +import { ClusterService } from '~/app/shared/api/cluster.service'; +import { SharedModule } from '~/app/shared/shared.module'; +import { configureTestBed } from '~/testing/unit-test-helper'; +import { CreateClusterComponent } from './create-cluster.component'; + +describe('CreateClusterComponent', () => { + let component: CreateClusterComponent; + let fixture: ComponentFixture; + let clusterService: ClusterService; + + configureTestBed({ + declarations: [CreateClusterComponent], + imports: [HttpClientTestingModule, RouterTestingModule, ToastrModule.forRoot(), SharedModule] + }); + + beforeEach(() => { + fixture = TestBed.createComponent(CreateClusterComponent); + component = fixture.componentInstance; + clusterService = TestBed.inject(ClusterService); + fixture.detectChanges(); + }); + + it('should create', () => { + expect(component).toBeTruthy(); + }); + + it('should have the heading "Welcome to Ceph Dashboard"', () => { + const heading = fixture.debugElement.query(By.css('h3')).nativeElement; + expect(heading.innerHTML).toBe('Welcome to Ceph Dashboard'); + }); + + it('should call updateStatus when cluster creation is skipped', () => { + const clusterServiceSpy = spyOn(clusterService, 'updateStatus').and.callThrough(); + expect(clusterServiceSpy).not.toHaveBeenCalled(); + component.skipClusterCreation(); + expect(clusterServiceSpy).toHaveBeenCalledTimes(1); + }); +}); diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster.component.ts new file mode 100644 index 0000000000000..239a4f13ca7f0 --- /dev/null +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster.component.ts @@ -0,0 +1,44 @@ +import { Component } from '@angular/core'; + +import { ClusterService } from '~/app/shared/api/cluster.service'; +import { AppConstants } from '~/app/shared/constants/app.constants'; +import { NotificationType } from '~/app/shared/enum/notification-type.enum'; +import { Permission } from '~/app/shared/models/permissions'; +import { AuthStorageService } from '~/app/shared/services/auth-storage.service'; +import { NotificationService } from '~/app/shared/services/notification.service'; + +@Component({ + selector: 'cd-create-cluster', + templateUrl: './create-cluster.component.html', + styleUrls: ['./create-cluster.component.scss'] +}) +export class CreateClusterComponent { + permission: Permission; + orchStatus = false; + featureAvailable = false; + projectConstants: typeof AppConstants = AppConstants; + + constructor( + private authStorageService: AuthStorageService, + private clusterService: ClusterService, + private notificationService: NotificationService + ) { + this.permission = this.authStorageService.getPermissions().configOpt; + } + + createCluster() { + this.notificationService.show( + NotificationType.error, + $localize`Cluster creation feature not implemented` + ); + } + + skipClusterCreation() { + this.clusterService.updateStatus('POST_INSTALLED').subscribe(() => { + this.notificationService.show( + NotificationType.info, + $localize`Cluster creation skipped by user` + ); + }); + } +} diff --git a/src/pybind/mgr/dashboard/frontend/src/app/core/auth/login/login.component.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/core/auth/login/login.component.spec.ts index 15a7275739f36..3cbfab4ebaac3 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/core/auth/login/login.component.spec.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/core/auth/login/login.component.spec.ts @@ -1,7 +1,11 @@ import { HttpClientTestingModule } from '@angular/common/http/testing'; import { ComponentFixture, TestBed } from '@angular/core/testing'; +import { Router } from '@angular/router'; import { RouterTestingModule } from '@angular/router/testing'; +import { of } from 'rxjs'; + +import { AuthService } from '~/app/shared/api/auth.service'; import { configureTestBed } from '~/testing/unit-test-helper'; import { AuthModule } from '../auth.module'; import { LoginComponent } from './login.component'; @@ -9,6 +13,8 @@ import { LoginComponent } from './login.component'; describe('LoginComponent', () => { let component: LoginComponent; let fixture: ComponentFixture; + let routerNavigateSpy: jasmine.Spy; + let authServiceLoginSpy: jasmine.Spy; configureTestBed({ imports: [RouterTestingModule, HttpClientTestingModule, AuthModule] @@ -17,6 +23,10 @@ describe('LoginComponent', () => { beforeEach(() => { fixture = TestBed.createComponent(LoginComponent); component = fixture.componentInstance; + routerNavigateSpy = spyOn(TestBed.inject(Router), 'navigate'); + routerNavigateSpy.and.returnValue(true); + authServiceLoginSpy = spyOn(TestBed.inject(AuthService), 'login'); + authServiceLoginSpy.and.returnValue(of(null)); fixture.detectChanges(); }); @@ -29,4 +39,20 @@ describe('LoginComponent', () => { component.ngOnInit(); expect(component['modalService'].hasOpenModals()).toBeFalsy(); }); + + it('should not show create cluster wizard if cluster creation was successful', () => { + component.postInstalled = true; + component.login(); + + expect(routerNavigateSpy).toHaveBeenCalledTimes(1); + expect(routerNavigateSpy).toHaveBeenCalledWith(['/']); + }); + + it('should show create cluster wizard if cluster creation was failed', () => { + component.postInstalled = false; + component.login(); + + expect(routerNavigateSpy).toHaveBeenCalledTimes(1); + expect(routerNavigateSpy).toHaveBeenCalledWith(['/create-cluster']); + }); }); diff --git a/src/pybind/mgr/dashboard/frontend/src/app/core/auth/login/login.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/core/auth/login/login.component.ts index 868ba66a00248..77bafd99c82e0 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/core/auth/login/login.component.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/core/auth/login/login.component.ts @@ -17,6 +17,7 @@ export class LoginComponent implements OnInit { model = new Credentials(); isLoginActive = false; returnUrl: string; + postInstalled = false; constructor( private authService: AuthService, @@ -43,6 +44,7 @@ export class LoginComponent implements OnInit { } this.authService.check(token).subscribe((login: any) => { if (login.login_url) { + this.postInstalled = login.cluster_status === 'POST_INSTALLED'; if (login.login_url === '#/login') { this.isLoginActive = true; } else { @@ -63,7 +65,11 @@ export class LoginComponent implements OnInit { login() { this.authService.login(this.model).subscribe(() => { - const url = _.get(this.route.snapshot.queryParams, 'returnUrl', '/'); + const urlPath = this.postInstalled ? '/' : '/create-cluster'; + let url = _.get(this.route.snapshot.queryParams, 'returnUrl', urlPath); + if (!this.postInstalled && this.route.snapshot.queryParams['returnUrl'] === '/dashboard') { + url = '/create-cluster'; + } this.router.navigate([url]); }); } diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/cluster.service.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/cluster.service.spec.ts new file mode 100644 index 0000000000000..758f670eec66b --- /dev/null +++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/cluster.service.spec.ts @@ -0,0 +1,42 @@ +import { HttpClientTestingModule, HttpTestingController } from '@angular/common/http/testing'; +import { fakeAsync, TestBed } from '@angular/core/testing'; + +import { configureTestBed } from '~/testing/unit-test-helper'; +import { ClusterService } from './cluster.service'; + +describe('ClusterService', () => { + let service: ClusterService; + let httpTesting: HttpTestingController; + + configureTestBed({ + imports: [HttpClientTestingModule], + providers: [ClusterService] + }); + + beforeEach(() => { + TestBed.configureTestingModule({}); + service = TestBed.inject(ClusterService); + httpTesting = TestBed.inject(HttpTestingController); + }); + + afterEach(() => { + httpTesting.verify(); + }); + + it('should be created', () => { + expect(service).toBeTruthy(); + }); + + it('should call getStatus', () => { + service.getStatus().subscribe(); + const req = httpTesting.expectOne('api/cluster'); + expect(req.request.method).toBe('GET'); + }); + + it('should update cluster status', fakeAsync(() => { + service.updateStatus('fakeStatus').subscribe(); + const req = httpTesting.expectOne('api/cluster'); + expect(req.request.method).toBe('PUT'); + expect(req.request.body).toEqual({ status: 'fakeStatus' }); + })); +}); diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/cluster.service.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/cluster.service.ts new file mode 100644 index 0000000000000..6b435d6ffed1d --- /dev/null +++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/cluster.service.ts @@ -0,0 +1,27 @@ +import { HttpClient } from '@angular/common/http'; +import { Injectable } from '@angular/core'; + +import { Observable } from 'rxjs'; + +@Injectable({ + providedIn: 'root' +}) +export class ClusterService { + baseURL = 'api/cluster'; + + constructor(private http: HttpClient) {} + + getStatus(): Observable { + return this.http.get(`${this.baseURL}`, { + headers: { Accept: 'application/vnd.ceph.api.v0.1+json' } + }); + } + + updateStatus(status: string) { + return this.http.put( + `${this.baseURL}`, + { status: status }, + { headers: { Accept: 'application/vnd.ceph.api.v0.1+json' } } + ); + } +} diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/constants/app.constants.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/constants/app.constants.ts index 05d6b5c53a9ad..5b668ad9000de 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/shared/constants/app.constants.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/constants/app.constants.ts @@ -7,6 +7,7 @@ export class AppConstants { public static readonly projectName = 'Ceph Dashboard'; public static readonly license = 'Free software (LGPL 2.1).'; public static readonly copyright = 'Copyright(c) ' + environment.year + ' Ceph contributors.'; + public static readonly cephLogo = 'assets/Ceph_Logo.svg'; } export enum URLVerbs { diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/services/module-status-guard.service.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/services/module-status-guard.service.spec.ts index 0948fc878a9e0..ebacc06c15192 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/shared/services/module-status-guard.service.spec.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/services/module-status-guard.service.spec.ts @@ -7,6 +7,7 @@ import { RouterTestingModule } from '@angular/router/testing'; import { of as observableOf } from 'rxjs'; import { configureTestBed } from '~/testing/unit-test-helper'; +import { MgrModuleService } from '../api/mgr-module.service'; import { ModuleStatusGuardService } from './module-status-guard.service'; describe('ModuleStatusGuardService', () => { @@ -15,6 +16,7 @@ describe('ModuleStatusGuardService', () => { let router: Router; let route: ActivatedRouteSnapshot; let ngZone: NgZone; + let mgrModuleService: MgrModuleService; @Component({ selector: 'cd-foo', template: '' }) class FooComponent {} @@ -25,9 +27,16 @@ describe('ModuleStatusGuardService', () => { const routes: Routes = [{ path: '**', component: FooComponent }]; - const testCanActivate = (getResult: {}, activateResult: boolean, urlResult: string) => { + const testCanActivate = ( + getResult: {}, + activateResult: boolean, + urlResult: string, + backend = 'cephadm' + ) => { let result: boolean; spyOn(httpClient, 'get').and.returnValue(observableOf(getResult)); + const test = { orchestrator: backend }; + spyOn(mgrModuleService, 'getConfig').and.returnValue(observableOf(test)); ngZone.run(() => { service.canActivateChild(route).subscribe((resp) => { result = resp; @@ -48,13 +57,15 @@ describe('ModuleStatusGuardService', () => { beforeEach(() => { service = TestBed.inject(ModuleStatusGuardService); httpClient = TestBed.inject(HttpClient); + mgrModuleService = TestBed.inject(MgrModuleService); router = TestBed.inject(Router); route = new ActivatedRouteSnapshot(); route.url = []; route.data = { moduleStatusGuardConfig: { apiPath: 'bar', - redirectTo: '/foo' + redirectTo: '/foo', + backend: 'rook' } }; ngZone = TestBed.inject(NgZone); @@ -76,4 +87,8 @@ describe('ModuleStatusGuardService', () => { it('should test canActivateChild with status unavailable', fakeAsync(() => { testCanActivate(null, false, '/foo'); })); + + it('should redirect normally if the backend provided matches the current backend', fakeAsync(() => { + testCanActivate({ available: true, message: 'foo' }, true, '/', 'rook'); + })); }); diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/services/module-status-guard.service.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/services/module-status-guard.service.ts index 171f34adfe6b4..3162afd232933 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/shared/services/module-status-guard.service.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/services/module-status-guard.service.ts @@ -5,7 +5,8 @@ import { ActivatedRouteSnapshot, CanActivate, CanActivateChild, Router } from '@ import { of as observableOf } from 'rxjs'; import { catchError, map } from 'rxjs/operators'; -import { Icons } from '../enum/icons.enum'; +import { MgrModuleService } from '~/app/shared/api/mgr-module.service'; +import { Icons } from '~/app/shared/enum/icons.enum'; /** * This service checks if a route can be activated by executing a @@ -39,7 +40,11 @@ export class ModuleStatusGuardService implements CanActivate, CanActivateChild { // TODO: Hotfix - remove ALLOWLIST'ing when a generic ErrorComponent is implemented static readonly ALLOWLIST: string[] = ['501']; - constructor(private http: HttpClient, private router: Router) {} + constructor( + private http: HttpClient, + private router: Router, + private mgrModuleService: MgrModuleService + ) {} canActivate(route: ActivatedRouteSnapshot) { return this.doCheck(route); @@ -54,9 +59,15 @@ export class ModuleStatusGuardService implements CanActivate, CanActivateChild { return observableOf(true); } const config = route.data['moduleStatusGuardConfig']; + let backendCheck = false; + if (config.backend) { + this.mgrModuleService.getConfig('orchestrator').subscribe((resp) => { + backendCheck = config.backend === resp['orchestrator']; + }); + } return this.http.get(`api/${config.apiPath}/status`).pipe( map((resp: any) => { - if (!resp.available) { + if (!resp.available && !backendCheck) { this.router.navigate([config.redirectTo || ''], { state: { header: config.header, diff --git a/src/pybind/mgr/dashboard/openapi.yaml b/src/pybind/mgr/dashboard/openapi.yaml index 844457b0a72d3..f03102599b0df 100644 --- a/src/pybind/mgr/dashboard/openapi.yaml +++ b/src/pybind/mgr/dashboard/openapi.yaml @@ -2038,6 +2038,67 @@ paths: - jwt: [] tags: - Cephfs + /api/cluster: + get: + parameters: [] + responses: + '200': + content: + application/vnd.ceph.api.v0.1+json: + type: object + description: OK + '400': + description: Operation exception. Please check the response body for details. + '401': + description: Unauthenticated access. Please login first. + '403': + description: Unauthorized access. Please check your permissions. + '500': + description: Unexpected error. Please check the response body for the stack + trace. + security: + - jwt: [] + summary: Get the cluster status + tags: + - Cluster + put: + parameters: [] + requestBody: + content: + application/json: + schema: + properties: + status: + description: Cluster Status + type: string + required: + - status + type: object + responses: + '200': + content: + application/vnd.ceph.api.v0.1+json: + type: object + description: Resource updated. + '202': + content: + application/vnd.ceph.api.v0.1+json: + type: object + description: Operation is still executing. Please check the task queue. + '400': + description: Operation exception. Please check the response body for details. + '401': + description: Unauthenticated access. Please login first. + '403': + description: Unauthorized access. Please check your permissions. + '500': + description: Unexpected error. Please check the response body for the stack + trace. + security: + - jwt: [] + summary: Update the cluster status + tags: + - Cluster /api/cluster_conf: get: parameters: [] @@ -10388,6 +10449,8 @@ tags: name: Auth - description: Cephfs Management API name: Cephfs +- description: Get Cluster Details + name: Cluster - description: Manage Cluster Configurations name: ClusterConfiguration - description: Crush Rule Management API diff --git a/src/pybind/mgr/dashboard/services/cluster.py b/src/pybind/mgr/dashboard/services/cluster.py new file mode 100644 index 0000000000000..aad517a21d68a --- /dev/null +++ b/src/pybind/mgr/dashboard/services/cluster.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +from enum import Enum + +from .. import mgr + + +class ClusterModel: + + class Status(Enum): + INSTALLED = 0 + POST_INSTALLED = 1 + + status: Status + + def __init__(self, status=Status.INSTALLED.name): + self.status = self.Status[status] + + def dict(self): + return {'status': self.status.name} + + def to_db(self): + mgr.set_store('cluster/status', self.status.name) + + @classmethod + def from_db(cls): + return cls(status=mgr.get_store('cluster/status', cls.Status.INSTALLED.name)) From 59cbf97e6ceedee0f9d682a94a749fbb01e9f787 Mon Sep 17 00:00:00 2001 From: Nizamudeen A Date: Sun, 4 Jul 2021 18:46:45 +0530 Subject: [PATCH 0124/2866] mgr/dashboard: Cluster Creation Add Host Section and e2es Add host section of the cluster creation workflow. 1. Fix bug in the modal where going forward one step on the wizard and coming back opens up the add host modal. 2. Rename Create Cluster to Expand Cluster as per the discussions 3. A skip confirmation modal to warn the user when he tries to skip the cluster creation 4. Adapted all the tests 5. Did some UI improvements like fixing and aligning the styles, colors.. - Used routed modal for host Additon form - Renamed the Create to Add in Host Form Fixes: https://tracker.ceph.com/issues/51517 Fixes: https://tracker.ceph.com/issues/51640 Fixes: https://tracker.ceph.com/issues/50336 Fixes: https://tracker.ceph.com/issues/50565 Signed-off-by: Avan Thakkar Signed-off-by: Aashish Sharma Signed-off-by: Nizamudeen A --- .../rados/dashboard/tasks/dashboard.yaml | 1 + src/pybind/mgr/dashboard/controllers/host.py | 2 +- .../cluster/cluster-welcome-page.po.ts | 22 --- .../cluster/create-cluster-review.po.ts | 11 -- .../integration/cluster/create-cluster.po.ts | 113 +++++++++++++ .../cypress/integration/cluster/hosts.po.ts | 17 +- .../orchestrator/01-hosts.e2e-spec.ts | 4 +- ...1-create-cluster-welcome-page.e2e-spec.ts} | 4 +- .../workflow/01-hosts.e2e-spec.ts | 4 +- .../02-create-cluster-add-host.e2e-spec.ts | 48 ++++++ .../03-create-cluster-review.e2e-spec.ts | 59 +++++++ .../workflow/04-cluster-check.e2e-spec.ts | 49 ++++++ .../08-create-cluster-review.e2e-spec.ts | 61 ------- .../frontend/src/app/app-routing.module.ts | 10 +- .../src/app/ceph/cluster/cluster.module.ts | 5 +- .../create-cluster.component.html | 60 ++++++- .../create-cluster.component.scss | 26 +++ .../create-cluster.component.spec.ts | 92 ++++++++-- .../create-cluster.component.ts | 128 +++++++++++--- .../hosts/host-form/host-form.component.html | 160 +++++++++--------- .../host-form/host-form.component.spec.ts | 10 +- .../hosts/host-form/host-form.component.ts | 22 ++- .../ceph/cluster/hosts/hosts.component.html | 8 +- .../cluster/hosts/hosts.component.spec.ts | 19 ++- .../app/ceph/cluster/hosts/hosts.component.ts | 43 +++-- .../osd-flags-indiv-modal.component.spec.ts | 4 +- .../core/auth/login/login.component.spec.ts | 2 +- .../app/core/auth/login/login.component.ts | 4 +- .../shared/components/components.module.ts | 7 +- .../components/modal/modal.component.html | 30 ++-- .../components/modal/modal.component.spec.ts | 15 +- .../components/modal/modal.component.ts | 9 +- .../components/wizard/wizard.component.html | 19 +++ .../components/wizard/wizard.component.scss | 30 ++++ .../wizard/wizard.component.spec.ts | 25 +++ .../components/wizard/wizard.component.ts | 39 +++++ .../src/app/shared/models/wizard-steps.ts | 4 + .../shared/services/task-message.service.ts | 4 +- .../services/wizard-steps.service.spec.ts | 16 ++ .../shared/services/wizard-steps.service.ts | 58 +++++++ .../src/styles/ceph-custom/_forms.scss | 11 ++ 41 files changed, 966 insertions(+), 289 deletions(-) delete mode 100644 src/pybind/mgr/dashboard/frontend/cypress/integration/cluster/cluster-welcome-page.po.ts delete mode 100644 src/pybind/mgr/dashboard/frontend/cypress/integration/cluster/create-cluster-review.po.ts create mode 100644 src/pybind/mgr/dashboard/frontend/cypress/integration/cluster/create-cluster.po.ts rename src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/{06-cluster-welcome-page.e2e-spec.ts => workflow/01-create-cluster-welcome-page.e2e-spec.ts} (69%) create mode 100644 src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/02-create-cluster-add-host.e2e-spec.ts create mode 100644 src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/03-create-cluster-review.e2e-spec.ts create mode 100644 src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/04-cluster-check.e2e-spec.ts delete mode 100644 src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/08-create-cluster-review.e2e-spec.ts create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/shared/components/wizard/wizard.component.html create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/shared/components/wizard/wizard.component.scss create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/shared/components/wizard/wizard.component.spec.ts create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/shared/components/wizard/wizard.component.ts create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/shared/models/wizard-steps.ts create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/shared/services/wizard-steps.service.spec.ts create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/shared/services/wizard-steps.service.ts diff --git a/qa/suites/rados/dashboard/tasks/dashboard.yaml b/qa/suites/rados/dashboard/tasks/dashboard.yaml index 1d0ec7395eae3..db28999ece922 100644 --- a/qa/suites/rados/dashboard/tasks/dashboard.yaml +++ b/qa/suites/rados/dashboard/tasks/dashboard.yaml @@ -39,6 +39,7 @@ tasks: - tasks.mgr.dashboard.test_auth - tasks.mgr.dashboard.test_cephfs - tasks.mgr.dashboard.test_cluster_configuration + - tasks.mgr.dashboard.test_cluster - tasks.mgr.dashboard.test_crush_rule - tasks.mgr.dashboard.test_erasure_code_profile - tasks.mgr.dashboard.test_ganesha diff --git a/src/pybind/mgr/dashboard/controllers/host.py b/src/pybind/mgr/dashboard/controllers/host.py index a72c4b34052d5..6ba1a5e0f1cbb 100644 --- a/src/pybind/mgr/dashboard/controllers/host.py +++ b/src/pybind/mgr/dashboard/controllers/host.py @@ -285,7 +285,7 @@ def list(self, sources=None): @raise_if_no_orchestrator([OrchFeature.HOST_LIST, OrchFeature.HOST_CREATE]) @handle_orchestrator_error('host') - @host_task('create', {'hostname': '{hostname}'}) + @host_task('add', {'hostname': '{hostname}'}) @EndpointDoc('', parameters={ 'hostname': (str, 'Hostname'), diff --git a/src/pybind/mgr/dashboard/frontend/cypress/integration/cluster/cluster-welcome-page.po.ts b/src/pybind/mgr/dashboard/frontend/cypress/integration/cluster/cluster-welcome-page.po.ts deleted file mode 100644 index 5615b0369d025..0000000000000 --- a/src/pybind/mgr/dashboard/frontend/cypress/integration/cluster/cluster-welcome-page.po.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { PageHelper } from '../page-helper.po'; -import { NotificationSidebarPageHelper } from '../ui/notification.po'; - -export class CreateClusterWelcomePageHelper extends PageHelper { - pages = { - index: { url: '#/create-cluster', id: 'cd-create-cluster' } - }; - - createCluster() { - cy.get('cd-create-cluster').should('contain.text', 'Welcome to Ceph'); - cy.get('[name=create-cluster]').click(); - } - - doSkip() { - cy.get('[name=skip-cluster-creation]').click(); - - cy.get('cd-dashboard').should('exist'); - const notification = new NotificationSidebarPageHelper(); - notification.open(); - notification.getNotifications().should('contain', 'Cluster creation skipped by user'); - } -} diff --git a/src/pybind/mgr/dashboard/frontend/cypress/integration/cluster/create-cluster-review.po.ts b/src/pybind/mgr/dashboard/frontend/cypress/integration/cluster/create-cluster-review.po.ts deleted file mode 100644 index 58844e39afec8..0000000000000 --- a/src/pybind/mgr/dashboard/frontend/cypress/integration/cluster/create-cluster-review.po.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { PageHelper } from '../page-helper.po'; - -export class CreateClusterReviewPageHelper extends PageHelper { - pages = { - index: { url: '#/create-cluster', id: 'cd-create-cluster-review' } - }; - - checkDefaultHostName() { - this.getTableCell(1, 'ceph-node-00.cephlab.com').should('exist'); - } -} diff --git a/src/pybind/mgr/dashboard/frontend/cypress/integration/cluster/create-cluster.po.ts b/src/pybind/mgr/dashboard/frontend/cypress/integration/cluster/create-cluster.po.ts new file mode 100644 index 0000000000000..22941b28e7cb7 --- /dev/null +++ b/src/pybind/mgr/dashboard/frontend/cypress/integration/cluster/create-cluster.po.ts @@ -0,0 +1,113 @@ +import { PageHelper } from '../page-helper.po'; +import { NotificationSidebarPageHelper } from '../ui/notification.po'; + +const pages = { + index: { url: '#/expand-cluster', id: 'cd-create-cluster' } +}; + +export class CreateClusterWizardHelper extends PageHelper { + pages = pages; + columnIndex = { + hostname: 1, + labels: 2, + status: 3 + }; + + createCluster() { + cy.get('cd-create-cluster').should('contain.text', 'Please expand your cluster first'); + cy.get('[name=expand-cluster]').click(); + } + + doSkip() { + cy.get('[name=skip-cluster-creation]').click(); + cy.contains('cd-modal button', 'Continue').click(); + + cy.get('cd-dashboard').should('exist'); + const notification = new NotificationSidebarPageHelper(); + notification.open(); + notification.getNotifications().should('contain', 'Cluster expansion skipped by user'); + } + + check_for_host() { + this.getTableCount('total').should('not.be.eq', 0); + } + + clickHostTab(hostname: string, tabName: string) { + this.getExpandCollapseElement(hostname).click(); + cy.get('cd-host-details').within(() => { + this.getTab(tabName).click(); + }); + } + + add(hostname: string, exist?: boolean, maintenance?: boolean) { + cy.get('.btn.btn-accent').first().click({ force: true }); + + cy.get('cd-modal').should('exist'); + cy.get('cd-modal').within(() => { + cy.get('#hostname').type(hostname); + if (maintenance) { + cy.get('label[for=maintenance]').click(); + } + if (exist) { + cy.get('#hostname').should('have.class', 'ng-invalid'); + } + cy.get('cd-submit-button').click(); + }); + // back to host list + cy.get(`${this.pages.index.id}`); + } + + checkExist(hostname: string, exist: boolean) { + this.clearTableSearchInput(); + this.getTableCell(this.columnIndex.hostname, hostname).should(($elements) => { + const hosts = $elements.map((_, el) => el.textContent).get(); + if (exist) { + expect(hosts).to.include(hostname); + } else { + expect(hosts).to.not.include(hostname); + } + }); + } + + delete(hostname: string) { + super.delete(hostname, this.columnIndex.hostname); + } + + // Add or remove labels on a host, then verify labels in the table + editLabels(hostname: string, labels: string[], add: boolean) { + this.getTableCell(this.columnIndex.hostname, hostname).click(); + this.clickActionButton('edit'); + + // add or remove label badges + if (add) { + cy.get('cd-modal').find('.select-menu-edit').click(); + for (const label of labels) { + cy.contains('cd-modal .badge', new RegExp(`^${label}$`)).should('not.exist'); + cy.get('.popover-body input').type(`${label}{enter}`); + } + } else { + for (const label of labels) { + cy.contains('cd-modal .badge', new RegExp(`^${label}$`)) + .find('.badge-remove') + .click(); + } + } + cy.get('cd-modal cd-submit-button').click(); + + // Verify labels are added or removed from Labels column + // First find row with hostname, then find labels in the row + this.getTableCell(this.columnIndex.hostname, hostname) + .parent() + .find(`datatable-body-cell:nth-child(${this.columnIndex.labels}) .badge`) + .should(($ele) => { + const newLabels = $ele.toArray().map((v) => v.innerText); + for (const label of labels) { + if (add) { + expect(newLabels).to.include(label); + } else { + expect(newLabels).to.not.include(label); + } + } + }); + } +} diff --git a/src/pybind/mgr/dashboard/frontend/cypress/integration/cluster/hosts.po.ts b/src/pybind/mgr/dashboard/frontend/cypress/integration/cluster/hosts.po.ts index 6752fe9e7870c..7a7e00d6648ac 100644 --- a/src/pybind/mgr/dashboard/frontend/cypress/integration/cluster/hosts.po.ts +++ b/src/pybind/mgr/dashboard/frontend/cypress/integration/cluster/hosts.po.ts @@ -2,7 +2,7 @@ import { PageHelper } from '../page-helper.po'; const pages = { index: { url: '#/hosts', id: 'cd-hosts' }, - create: { url: '#/hosts/create', id: 'cd-host-form' } + add: { url: '#/hosts/(modal:add)', id: 'cd-host-form' } }; export class HostsPageHelper extends PageHelper { @@ -49,21 +49,20 @@ export class HostsPageHelper extends PageHelper { }); } - @PageHelper.restrictTo(pages.create.url) + @PageHelper.restrictTo(pages.add.url) add(hostname: string, exist?: boolean, maintenance?: boolean) { - cy.get(`${this.pages.create.id}`).within(() => { + cy.get(`${this.pages.add.id}`).within(() => { cy.get('#hostname').type(hostname); if (maintenance) { cy.get('label[for=maintenance]').click(); } + if (exist) { + cy.get('#hostname').should('have.class', 'ng-invalid'); + } cy.get('cd-submit-button').click(); }); - if (exist) { - cy.get('#hostname').should('have.class', 'ng-invalid'); - } else { - // back to host list - cy.get(`${this.pages.index.id}`); - } + // back to host list + cy.get(`${this.pages.index.id}`); } @PageHelper.restrictTo(pages.index.url) diff --git a/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/01-hosts.e2e-spec.ts b/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/01-hosts.e2e-spec.ts index cf85642a1b1d2..6c79a74662dff 100644 --- a/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/01-hosts.e2e-spec.ts +++ b/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/01-hosts.e2e-spec.ts @@ -17,7 +17,7 @@ describe('Hosts page', () => { it('should not add an exsiting host', function () { const hostname = Cypress._.sample(this.hosts).name; - hosts.navigateTo('create'); + hosts.navigateTo('add'); hosts.add(hostname, true); }); @@ -26,7 +26,7 @@ describe('Hosts page', () => { hosts.delete(host); // add it back - hosts.navigateTo('create'); + hosts.navigateTo('add'); hosts.add(host); hosts.checkExist(host, true); }); diff --git a/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/06-cluster-welcome-page.e2e-spec.ts b/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/01-create-cluster-welcome-page.e2e-spec.ts similarity index 69% rename from src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/06-cluster-welcome-page.e2e-spec.ts rename to src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/01-create-cluster-welcome-page.e2e-spec.ts index bd0470b86700a..d776a210a57ee 100644 --- a/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/06-cluster-welcome-page.e2e-spec.ts +++ b/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/01-create-cluster-welcome-page.e2e-spec.ts @@ -1,7 +1,7 @@ -import { CreateClusterWelcomePageHelper } from '../cluster/cluster-welcome-page.po'; +import { CreateClusterWizardHelper } from 'cypress/integration/cluster/create-cluster.po'; describe('Create cluster page', () => { - const createCluster = new CreateClusterWelcomePageHelper(); + const createCluster = new CreateClusterWizardHelper(); beforeEach(() => { cy.login(); diff --git a/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/01-hosts.e2e-spec.ts b/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/01-hosts.e2e-spec.ts index 7dd0c1104334c..e8dea8e30080f 100644 --- a/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/01-hosts.e2e-spec.ts +++ b/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/01-hosts.e2e-spec.ts @@ -12,7 +12,7 @@ describe('Hosts page', () => { 'ceph-node-02.cephlab.com' ]; const addHost = (hostname: string, exist?: boolean, maintenance?: boolean) => { - hosts.navigateTo('create'); + hosts.navigateTo('add'); hosts.add(hostname, exist, maintenance); hosts.checkExist(hostname, true); }; @@ -49,7 +49,7 @@ describe('Hosts page', () => { }); it('should not add an existing host', function () { - hosts.navigateTo('create'); + hosts.navigateTo('add'); hosts.add(hostnames[0], true); }); diff --git a/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/02-create-cluster-add-host.e2e-spec.ts b/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/02-create-cluster-add-host.e2e-spec.ts new file mode 100644 index 0000000000000..7bf5b5be3d496 --- /dev/null +++ b/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/02-create-cluster-add-host.e2e-spec.ts @@ -0,0 +1,48 @@ +import { CreateClusterWizardHelper } from 'cypress/integration/cluster/create-cluster.po'; + +describe('Create cluster add host page', () => { + const createCluster = new CreateClusterWizardHelper(); + const hostnames = [ + 'ceph-node-00.cephlab.com', + 'ceph-node-01.cephlab.com', + 'ceph-node-02.cephlab.com' + ]; + const addHost = (hostname: string, exist?: boolean) => { + createCluster.add(hostname, exist, true); + createCluster.checkExist(hostname, true); + }; + + beforeEach(() => { + cy.login(); + Cypress.Cookies.preserveOnce('token'); + createCluster.navigateTo(); + createCluster.createCluster(); + }); + + it('should check if nav-link and title contains Add Hosts', () => { + cy.get('.nav-link').should('contain.text', 'Add Hosts'); + + cy.get('.title').should('contain.text', 'Add Hosts'); + }); + + it('should check existing host and add new hosts into maintenance mode', () => { + createCluster.checkExist(hostnames[0], true); + + addHost(hostnames[1], false); + addHost(hostnames[2], false); + }); + + it('should not add an existing host', () => { + createCluster.add(hostnames[0], true); + }); + + it('should edit host labels', () => { + const labels = ['foo', 'bar']; + createCluster.editLabels(hostnames[0], labels, true); + createCluster.editLabels(hostnames[0], labels, false); + }); + + it('should delete a host', () => { + createCluster.delete(hostnames[1]); + }); +}); diff --git a/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/03-create-cluster-review.e2e-spec.ts b/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/03-create-cluster-review.e2e-spec.ts new file mode 100644 index 0000000000000..17dd84994d60e --- /dev/null +++ b/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/03-create-cluster-review.e2e-spec.ts @@ -0,0 +1,59 @@ +import { CreateClusterWizardHelper } from 'cypress/integration/cluster/create-cluster.po'; + +describe('Create Cluster Review page', () => { + const createCluster = new CreateClusterWizardHelper(); + + beforeEach(() => { + cy.login(); + Cypress.Cookies.preserveOnce('token'); + createCluster.navigateTo(); + createCluster.createCluster(); + + cy.get('button[aria-label="Next"]').click(); + }); + + describe('navigation link and title test', () => { + it('should check if nav-link and title contains Review', () => { + cy.get('.nav-link').should('contain.text', 'Review'); + + cy.get('.title').should('contain.text', 'Review'); + }); + }); + + describe('fields check', () => { + it('should check cluster resources table is present', () => { + // check for table header 'Cluster Resources' + createCluster.getLegends().its(0).should('have.text', 'Cluster Resources'); + + // check for fields in table + createCluster.getStatusTables().should('contain.text', 'Hosts'); + }); + + it('should check Hosts by Label and Host Details tables are present', () => { + // check for there to be two tables + createCluster.getDataTables().should('have.length', 2); + + // check for table header 'Hosts by Label' + createCluster.getLegends().its(1).should('have.text', 'Hosts by Label'); + + // check for table header 'Host Details' + createCluster.getLegends().its(2).should('have.text', 'Host Details'); + + // verify correct columns on Hosts by Label table + createCluster.getDataTableHeaders(0).contains('Label'); + + createCluster.getDataTableHeaders(0).contains('Number of Hosts'); + + // verify correct columns on Host Details table + createCluster.getDataTableHeaders(1).contains('Host Name'); + + createCluster.getDataTableHeaders(1).contains('Labels'); + }); + + it('should check hosts count and default host name are present', () => { + createCluster.getStatusTables().contains(2); + + createCluster.check_for_host(); + }); + }); +}); diff --git a/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/04-cluster-check.e2e-spec.ts b/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/04-cluster-check.e2e-spec.ts new file mode 100644 index 0000000000000..9717dd8b7dbf8 --- /dev/null +++ b/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/04-cluster-check.e2e-spec.ts @@ -0,0 +1,49 @@ +import { CreateClusterWizardHelper } from 'cypress/integration/cluster/create-cluster.po'; +import { HostsPageHelper } from 'cypress/integration/cluster/hosts.po'; + +describe('when cluster creation is completed', () => { + const createCluster = new CreateClusterWizardHelper(); + + beforeEach(() => { + cy.login(); + Cypress.Cookies.preserveOnce('token'); + }); + + it('should redirect to dashboard landing page after cluster creation', () => { + createCluster.navigateTo(); + createCluster.createCluster(); + + cy.get('button[aria-label="Next"]').click(); + cy.get('button[aria-label="Next"]').click(); + + cy.get('cd-dashboard').should('exist'); + }); + + describe('Hosts page', () => { + const hosts = new HostsPageHelper(); + const hostnames = ['ceph-node-00.cephlab.com', 'ceph-node-02.cephlab.com']; + + beforeEach(() => { + hosts.navigateTo(); + }); + it('should have already exited from maintenance', () => { + for (let host = 0; host < hostnames.length; host++) { + cy.get('datatable-row-wrapper').should('not.have.text', 'maintenance'); + } + }); + + it('should display inventory', () => { + hosts.clickHostTab(hostnames[1], 'Physical Disks'); + cy.get('cd-host-details').within(() => { + hosts.getTableCount('total').should('be.gte', 0); + }); + }); + + it('should display daemons', () => { + hosts.clickHostTab(hostnames[1], 'Daemons'); + cy.get('cd-host-details').within(() => { + hosts.getTableCount('total').should('be.gte', 0); + }); + }); + }); +}); diff --git a/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/08-create-cluster-review.e2e-spec.ts b/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/08-create-cluster-review.e2e-spec.ts deleted file mode 100644 index a472810e6e6c5..0000000000000 --- a/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/08-create-cluster-review.e2e-spec.ts +++ /dev/null @@ -1,61 +0,0 @@ -import { CreateClusterWelcomePageHelper } from 'cypress/integration/cluster/cluster-welcome-page.po'; -import { CreateClusterReviewPageHelper } from 'cypress/integration/cluster/create-cluster-review.po'; - -describe('Create Cluster Review page', () => { - const reviewPage = new CreateClusterReviewPageHelper(); - const createCluster = new CreateClusterWelcomePageHelper(); - - beforeEach(() => { - cy.login(); - Cypress.Cookies.preserveOnce('token'); - createCluster.navigateTo(); - createCluster.createCluster(); - - cy.get('button[aria-label="Next"]').click(); - }); - - describe('navigation link and title test', () => { - it('should check if nav-link and title contains Review', () => { - cy.get('.nav-link').should('contain.text', 'Review'); - - cy.get('.title').should('contain.text', 'Review'); - }); - }); - - describe('fields check', () => { - it('should check cluster resources table is present', () => { - // check for table header 'Status' - reviewPage.getLegends().its(0).should('have.text', 'Cluster Resources'); - - // check for fields in table - reviewPage.getStatusTables().should('contain.text', 'Hosts'); - }); - - it('should check Hosts Per Label and Host Details tables are present', () => { - // check for there to be two tables - reviewPage.getDataTables().should('have.length', 2); - - // check for table header 'Hosts Per Label' - reviewPage.getLegends().its(1).should('have.text', 'Hosts Per Label'); - - // check for table header 'Host Details' - reviewPage.getLegends().its(2).should('have.text', 'Host Details'); - - // verify correct columns on Hosts Per Label table - reviewPage.getDataTableHeaders(0).contains('Label'); - - reviewPage.getDataTableHeaders(0).contains('Number of Hosts'); - - // verify correct columns on Host Details table - reviewPage.getDataTableHeaders(1).contains('Host Name'); - - reviewPage.getDataTableHeaders(1).contains('Labels'); - }); - - it('should check hosts count and default host name are present', () => { - reviewPage.getStatusTables().should('contain.text', '1'); - - reviewPage.checkDefaultHostName(); - }); - }); -}); diff --git a/src/pybind/mgr/dashboard/frontend/src/app/app-routing.module.ts b/src/pybind/mgr/dashboard/frontend/src/app/app-routing.module.ts index 099b31efbda50..bdd2cb8978234 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/app-routing.module.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/app-routing.module.ts @@ -91,7 +91,7 @@ const routes: Routes = [ // Cluster { - path: 'create-cluster', + path: 'expand-cluster', component: CreateClusterComponent, canActivate: [ModuleStatusGuardService], data: { @@ -100,18 +100,18 @@ const routes: Routes = [ redirectTo: 'dashboard', backend: 'cephadm' }, - breadcrumbs: 'Create Cluster' + breadcrumbs: 'Expand Cluster' } }, { path: 'hosts', + component: HostsComponent, data: { breadcrumbs: 'Cluster/Hosts' }, children: [ - { path: '', component: HostsComponent }, { - path: URLVerbs.CREATE, + path: URLVerbs.ADD, component: HostFormComponent, - data: { breadcrumbs: ActionLabels.CREATE } + outlet: 'modal' } ] }, diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/cluster.module.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/cluster.module.ts index a2c1e6d2f89ec..185c34b27504d 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/cluster.module.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/cluster.module.ts @@ -5,6 +5,7 @@ import { RouterModule } from '@angular/router'; import { TreeModule } from '@circlon/angular-tree-component'; import { + NgbActiveModal, NgbDatepickerModule, NgbDropdownModule, NgbNavModule, @@ -106,7 +107,6 @@ import { TelemetryComponent } from './telemetry/telemetry.component'; OsdCreationPreviewModalComponent, RulesListComponent, ActiveAlertListComponent, - HostFormComponent, ServiceDetailsComponent, ServiceDaemonListComponent, TelemetryComponent, @@ -115,6 +115,7 @@ import { TelemetryComponent } from './telemetry/telemetry.component'; OsdFlagsIndivModalComponent, PlacementPipe, CreateClusterComponent - ] + ], + providers: [NgbActiveModal] }) export class ClusterModule {} diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster.component.html index 661c13fc931c9..fb006ec1beb10 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster.component.html +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster.component.html @@ -1,4 +1,5 @@ -
+
@@ -10,16 +11,14 @@

Please proceed to complete the cluster creation

-
- + i18n>Expand Cluster
@@ -27,3 +26,48 @@
+ +
+
Expand Cluster
+
+ +
+ +
+

Add Hosts

+
+ +
+
+

Review

+
+

To be implemented

+
+ +
+
+
+ + +
+
+ + + You are about to skip the cluster expansion process. + You’ll need to navigate through the menu to add hosts and services. + +
Are you sure you want to continue?
+ diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster.component.scss b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster.component.scss index e69de29bb2d1d..580c5219e9f2c 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster.component.scss +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster.component.scss @@ -0,0 +1,26 @@ +@use './src/styles/vendor/variables' as vv; + +.container-fluid { + align-items: flex-start; + display: flex; + padding-left: 0; + width: 100%; +} + +.card-body { + max-width: 85%; +} + +.vertical-line { + border-left: 1px solid vv.$gray-400; +} + +cd-wizard { + width: 15%; +} + +cd-hosts { + ::ng-deep .nav { + display: none; + } +} diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster.component.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster.component.spec.ts index 7e061b2e25c9a..1ebdfb3a59d2f 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster.component.spec.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster.component.spec.ts @@ -5,7 +5,13 @@ import { RouterTestingModule } from '@angular/router/testing'; import { ToastrModule } from 'ngx-toastr'; -import { ClusterService } from '~/app/shared/api/cluster.service'; +import { CephModule } from '~/app/ceph/ceph.module'; +import { CoreModule } from '~/app/core/core.module'; +import { HostService } from '~/app/shared/api/host.service'; +import { ConfirmationModalComponent } from '~/app/shared/components/confirmation-modal/confirmation-modal.component'; +import { AppConstants } from '~/app/shared/constants/app.constants'; +import { ModalService } from '~/app/shared/services/modal.service'; +import { WizardStepsService } from '~/app/shared/services/wizard-steps.service'; import { SharedModule } from '~/app/shared/shared.module'; import { configureTestBed } from '~/testing/unit-test-helper'; import { CreateClusterComponent } from './create-cluster.component'; @@ -13,17 +19,31 @@ import { CreateClusterComponent } from './create-cluster.component'; describe('CreateClusterComponent', () => { let component: CreateClusterComponent; let fixture: ComponentFixture; - let clusterService: ClusterService; + let wizardStepService: WizardStepsService; + let hostService: HostService; + let modalServiceShowSpy: jasmine.Spy; + const projectConstants: typeof AppConstants = AppConstants; configureTestBed({ - declarations: [CreateClusterComponent], - imports: [HttpClientTestingModule, RouterTestingModule, ToastrModule.forRoot(), SharedModule] + imports: [ + HttpClientTestingModule, + RouterTestingModule, + ToastrModule.forRoot(), + SharedModule, + CoreModule, + CephModule + ] }); beforeEach(() => { fixture = TestBed.createComponent(CreateClusterComponent); component = fixture.componentInstance; - clusterService = TestBed.inject(ClusterService); + wizardStepService = TestBed.inject(WizardStepsService); + hostService = TestBed.inject(HostService); + modalServiceShowSpy = spyOn(TestBed.inject(ModalService), 'show').and.returnValue({ + // mock the close function, it might be called if there are async tests. + close: jest.fn() + }); fixture.detectChanges(); }); @@ -31,15 +51,65 @@ describe('CreateClusterComponent', () => { expect(component).toBeTruthy(); }); - it('should have the heading "Welcome to Ceph Dashboard"', () => { + it('should have project name as heading in welcome screen', () => { const heading = fixture.debugElement.query(By.css('h3')).nativeElement; - expect(heading.innerHTML).toBe('Welcome to Ceph Dashboard'); + expect(heading.innerHTML).toBe(`Welcome to ${projectConstants.projectName}`); }); - it('should call updateStatus when cluster creation is skipped', () => { - const clusterServiceSpy = spyOn(clusterService, 'updateStatus').and.callThrough(); - expect(clusterServiceSpy).not.toHaveBeenCalled(); + it('should show confirmation modal when cluster creation is skipped', () => { component.skipClusterCreation(); - expect(clusterServiceSpy).toHaveBeenCalledTimes(1); + expect(modalServiceShowSpy.calls.any()).toBeTruthy(); + expect(modalServiceShowSpy.calls.first().args[0]).toBe(ConfirmationModalComponent); + }); + + it('should show the wizard when cluster creation is started', () => { + component.createCluster(); + fixture.detectChanges(); + const nativeEl = fixture.debugElement.nativeElement; + expect(nativeEl.querySelector('cd-wizard')).not.toBe(null); + }); + + it('should have title Add Hosts', () => { + component.createCluster(); + fixture.detectChanges(); + const heading = fixture.debugElement.query(By.css('.title')).nativeElement; + expect(heading.innerHTML).toBe('Add Hosts'); + }); + + it('should show the host list when cluster creation as first step', () => { + component.createCluster(); + fixture.detectChanges(); + const nativeEl = fixture.debugElement.nativeElement; + expect(nativeEl.querySelector('cd-hosts')).not.toBe(null); + }); + + it('should move to next step and show the second page', () => { + const wizardStepServiceSpy = spyOn(wizardStepService, 'moveToNextStep').and.callThrough(); + const hostServiceSpy = spyOn(hostService, 'list').and.callThrough(); + component.createCluster(); + fixture.detectChanges(); + component.onNextStep(); + fixture.detectChanges(); + const heading = fixture.debugElement.query(By.css('.title')).nativeElement; + expect(wizardStepServiceSpy).toHaveBeenCalledTimes(1); + expect(hostServiceSpy).toBeCalledTimes(1); + expect(heading.innerHTML).toBe('Review'); + }); + + it('should show the button labels correctly', () => { + component.createCluster(); + fixture.detectChanges(); + let submitBtnLabel = component.showSubmitButtonLabel(); + expect(submitBtnLabel).toEqual('Next'); + let cancelBtnLabel = component.showCancelButtonLabel(); + expect(cancelBtnLabel).toEqual('Cancel'); + + // Last page of the wizard + component.onNextStep(); + fixture.detectChanges(); + submitBtnLabel = component.showSubmitButtonLabel(); + expect(submitBtnLabel).toEqual('Expand Cluster'); + cancelBtnLabel = component.showCancelButtonLabel(); + expect(cancelBtnLabel).toEqual('Back'); }); }); diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster.component.ts index 239a4f13ca7f0..b47a63e8cec00 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster.component.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster.component.ts @@ -1,44 +1,134 @@ -import { Component } from '@angular/core'; +import { Component, OnDestroy, TemplateRef, ViewChild } from '@angular/core'; +import { Router } from '@angular/router'; + +import { NgbModalRef } from '@ng-bootstrap/ng-bootstrap'; +import { forkJoin, Subscription } from 'rxjs'; +import { finalize } from 'rxjs/operators'; import { ClusterService } from '~/app/shared/api/cluster.service'; -import { AppConstants } from '~/app/shared/constants/app.constants'; +import { HostService } from '~/app/shared/api/host.service'; +import { ConfirmationModalComponent } from '~/app/shared/components/confirmation-modal/confirmation-modal.component'; +import { ActionLabelsI18n, AppConstants } from '~/app/shared/constants/app.constants'; import { NotificationType } from '~/app/shared/enum/notification-type.enum'; -import { Permission } from '~/app/shared/models/permissions'; +import { Permissions } from '~/app/shared/models/permissions'; +import { WizardStepModel } from '~/app/shared/models/wizard-steps'; import { AuthStorageService } from '~/app/shared/services/auth-storage.service'; +import { ModalService } from '~/app/shared/services/modal.service'; import { NotificationService } from '~/app/shared/services/notification.service'; +import { WizardStepsService } from '~/app/shared/services/wizard-steps.service'; @Component({ selector: 'cd-create-cluster', templateUrl: './create-cluster.component.html', styleUrls: ['./create-cluster.component.scss'] }) -export class CreateClusterComponent { - permission: Permission; - orchStatus = false; - featureAvailable = false; +export class CreateClusterComponent implements OnDestroy { + @ViewChild('skipConfirmTpl', { static: true }) + skipConfirmTpl: TemplateRef; + currentStep: WizardStepModel; + currentStepSub: Subscription; + permissions: Permissions; projectConstants: typeof AppConstants = AppConstants; + stepTitles = ['Add Hosts', 'Review']; + startClusterCreation = false; + observables: any = []; + modalRef: NgbModalRef; constructor( private authStorageService: AuthStorageService, + private stepsService: WizardStepsService, + private router: Router, + private hostService: HostService, + private notificationService: NotificationService, + private actionLabels: ActionLabelsI18n, private clusterService: ClusterService, - private notificationService: NotificationService + private modalService: ModalService ) { - this.permission = this.authStorageService.getPermissions().configOpt; + this.permissions = this.authStorageService.getPermissions(); + this.currentStepSub = this.stepsService.getCurrentStep().subscribe((step: WizardStepModel) => { + this.currentStep = step; + }); + this.currentStep.stepIndex = 1; } createCluster() { - this.notificationService.show( - NotificationType.error, - $localize`Cluster creation feature not implemented` - ); + this.startClusterCreation = true; } skipClusterCreation() { - this.clusterService.updateStatus('POST_INSTALLED').subscribe(() => { - this.notificationService.show( - NotificationType.info, - $localize`Cluster creation skipped by user` - ); - }); + const modalVariables = { + titleText: $localize`Warning`, + buttonText: $localize`Continue`, + warning: true, + bodyTpl: this.skipConfirmTpl, + showSubmit: true, + onSubmit: () => { + this.clusterService.updateStatus('POST_INSTALLED').subscribe({ + error: () => this.modalRef.close(), + complete: () => { + this.notificationService.show( + NotificationType.info, + $localize`Cluster expansion skipped by user` + ); + this.router.navigate(['/dashboard']); + this.modalRef.close(); + } + }); + } + }; + this.modalRef = this.modalService.show(ConfirmationModalComponent, modalVariables); + } + + onSubmit() { + forkJoin(this.observables) + .pipe( + finalize(() => + this.clusterService.updateStatus('POST_INSTALLED').subscribe(() => { + this.notificationService.show( + NotificationType.success, + $localize`Cluster expansion was successful` + ); + this.router.navigate(['/dashboard']); + }) + ) + ) + .subscribe({ + error: (error) => error.preventDefault() + }); + } + + onNextStep() { + if (!this.stepsService.isLastStep()) { + this.hostService.list().subscribe((hosts) => { + hosts.forEach((host) => { + if (host['status'] === 'maintenance') { + this.observables.push(this.hostService.update(host['hostname'], false, [], true)); + } + }); + }); + this.stepsService.moveToNextStep(); + } else { + this.onSubmit(); + } + } + + onPreviousStep() { + if (!this.stepsService.isFirstStep()) { + this.stepsService.moveToPreviousStep(); + } else { + this.router.navigate(['/dashboard']); + } + } + + showSubmitButtonLabel() { + return !this.stepsService.isLastStep() ? this.actionLabels.NEXT : $localize`Expand Cluster`; + } + + showCancelButtonLabel() { + return !this.stepsService.isFirstStep() ? this.actionLabels.BACK : this.actionLabels.CANCEL; + } + + ngOnDestroy(): void { + this.currentStepSub.unsubscribe(); } } diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/host-form/host-form.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/host-form/host-form.component.html index 487caf85b12de..a3477b9bd4102 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/host-form/host-form.component.html +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/host-form/host-form.component.html @@ -1,91 +1,95 @@ -
-
-
-
{{ action | titlecase }} {{ resource | upperFirst }}
+ + {{ action | titlecase }} {{ resource | upperFirst }} -
+ - -
- -
- - This field is required. - The chosen hostname is already in use. +
+ + +
+ + +
+ +
+ + This field is required. + The chosen hostname is already in use. +
-
- -
- -
- - The value is not a valid IP address. + +
+ +
+ + The value is not a valid IP address. +
-
- -
- -
- - + +
+ +
+ + +
-
- -
-
-
- - + +
+
+
+ + +
-
-
- -
+
+ +
+
- -
+ + diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/host-form/host-form.component.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/host-form/host-form.component.spec.ts index dbb834ea8c82c..ed3daf1e4b49b 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/host-form/host-form.component.spec.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/host-form/host-form.component.spec.ts @@ -3,6 +3,7 @@ import { ComponentFixture, fakeAsync, TestBed, tick } from '@angular/core/testin import { ReactiveFormsModule } from '@angular/forms'; import { RouterTestingModule } from '@angular/router/testing'; +import { NgbActiveModal } from '@ng-bootstrap/ng-bootstrap'; import { ToastrModule } from 'ngx-toastr'; import { LoadingPanelComponent } from '~/app/shared/components/loading-panel/loading-panel.component'; @@ -24,7 +25,8 @@ describe('HostFormComponent', () => { ReactiveFormsModule, ToastrModule.forRoot() ], - declarations: [HostFormComponent] + declarations: [HostFormComponent], + providers: [NgbActiveModal] }, [LoadingPanelComponent] ); @@ -32,6 +34,7 @@ describe('HostFormComponent', () => { beforeEach(() => { fixture = TestBed.createComponent(HostFormComponent); component = fixture.componentInstance; + component.ngOnInit(); formHelper = new FormHelper(component.hostForm); fixture.detectChanges(); }); @@ -40,6 +43,11 @@ describe('HostFormComponent', () => { expect(component).toBeTruthy(); }); + it('should open the form in a modal', () => { + const nativeEl = fixture.debugElement.nativeElement; + expect(nativeEl.querySelector('cd-modal')).not.toBe(null); + }); + it('should validate the network address is valid', fakeAsync(() => { formHelper.setValue('addr', '115.42.150.37', true); tick(); diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/host-form/host-form.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/host-form/host-form.component.ts index b90312ff855f2..2fc8b13b540e6 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/host-form/host-form.component.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/host-form/host-form.component.ts @@ -2,6 +2,8 @@ import { Component, OnInit } from '@angular/core'; import { FormControl, Validators } from '@angular/forms'; import { Router } from '@angular/router'; +import { NgbActiveModal } from '@ng-bootstrap/ng-bootstrap'; + import { HostService } from '~/app/shared/api/host.service'; import { SelectMessages } from '~/app/shared/components/select/select-messages.model'; import { ActionLabelsI18n, URLVerbs } from '~/app/shared/constants/app.constants'; @@ -24,6 +26,7 @@ export class HostFormComponent extends CdForm implements OnInit { addr: string; status: string; allLabels: any; + pageURL: string; messages = new SelectMessages({ empty: $localize`There are no labels.`, @@ -35,15 +38,19 @@ export class HostFormComponent extends CdForm implements OnInit { private router: Router, private actionLabels: ActionLabelsI18n, private hostService: HostService, - private taskWrapper: TaskWrapperService + private taskWrapper: TaskWrapperService, + public activeModal: NgbActiveModal ) { super(); this.resource = $localize`host`; - this.action = this.actionLabels.CREATE; - this.createForm(); + this.action = this.actionLabels.ADD; } ngOnInit() { + if (this.router.url.includes('hosts')) { + this.pageURL = 'hosts'; + } + this.createForm(); this.hostService.list().subscribe((resp: any[]) => { this.hostnames = resp.map((host) => { return host['hostname']; @@ -53,6 +60,7 @@ export class HostFormComponent extends CdForm implements OnInit { } private createForm() { + const disableMaintenance = this.pageURL !== 'hosts'; this.hostForm = new CdFormGroup({ hostname: new FormControl('', { validators: [ @@ -66,7 +74,7 @@ export class HostFormComponent extends CdForm implements OnInit { validators: [CdValidators.ip()] }), labels: new FormControl([]), - maintenance: new FormControl(false) + maintenance: new FormControl({ value: disableMaintenance, disabled: disableMaintenance }) }); } @@ -77,7 +85,7 @@ export class HostFormComponent extends CdForm implements OnInit { this.allLabels = this.hostForm.get('labels').value; this.taskWrapper .wrapTaskAroundCall({ - task: new FinishedTask('host/' + URLVerbs.CREATE, { + task: new FinishedTask('host/' + URLVerbs.ADD, { hostname: hostname }), call: this.hostService.create(hostname, this.addr, this.allLabels, this.status) @@ -87,7 +95,9 @@ export class HostFormComponent extends CdForm implements OnInit { this.hostForm.setErrors({ cdSubmitButton: true }); }, complete: () => { - this.router.navigate(['/hosts']); + this.pageURL === 'hosts' + ? this.router.navigate([this.pageURL, { outlets: { modal: null } }]) + : this.activeModal.close(); } }); } diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/hosts.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/hosts.component.html index f31adf9e5c0e7..59bab46d72d23 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/hosts.component.html +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/hosts.component.html @@ -5,12 +5,13 @@ Hosts List -
@@ -29,7 +30,7 @@
  • + *ngIf="permissions.grafana.read && !clusterCreation"> Overall Performance @@ -66,3 +67,4 @@ Are you sure you want to continue? + diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/hosts.component.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/hosts.component.spec.ts index f4d400f3230c8..049aceba0994a 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/hosts.component.spec.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/hosts.component.spec.ts @@ -72,6 +72,7 @@ describe('HostsComponent', () => { showForceMaintenanceModal = new MockShowForceMaintenanceModal(); fixture = TestBed.createComponent(HostsComponent); component = fixture.componentInstance; + component.clusterCreation = false; hostListSpy = spyOn(TestBed.inject(HostService), 'list'); orchService = TestBed.inject(OrchestratorService); }); @@ -182,7 +183,7 @@ describe('HostsComponent', () => { const tests = [ { expectResults: { - Create: { disabled: false, disableDesc: '' }, + Add: { disabled: false, disableDesc: '' }, Edit: { disabled: true, disableDesc: '' }, Delete: { disabled: true, disableDesc: '' } } @@ -190,7 +191,7 @@ describe('HostsComponent', () => { { selectRow: fakeHosts[0], // non-orchestrator host expectResults: { - Create: { disabled: false, disableDesc: '' }, + Add: { disabled: false, disableDesc: '' }, Edit: { disabled: true, disableDesc: component.messages.nonOrchHost }, Delete: { disabled: true, disableDesc: component.messages.nonOrchHost } } @@ -198,7 +199,7 @@ describe('HostsComponent', () => { { selectRow: fakeHosts[1], // orchestrator host expectResults: { - Create: { disabled: false, disableDesc: '' }, + Add: { disabled: false, disableDesc: '' }, Edit: { disabled: false, disableDesc: '' }, Delete: { disabled: false, disableDesc: '' } } @@ -222,7 +223,7 @@ describe('HostsComponent', () => { const tests = [ { expectResults: { - Create: resultNoOrchestrator, + Add: resultNoOrchestrator, Edit: { disabled: true, disableDesc: '' }, Delete: { disabled: true, disableDesc: '' } } @@ -230,7 +231,7 @@ describe('HostsComponent', () => { { selectRow: fakeHosts[0], // non-orchestrator host expectResults: { - Create: resultNoOrchestrator, + Add: resultNoOrchestrator, Edit: { disabled: true, disableDesc: component.messages.nonOrchHost }, Delete: { disabled: true, disableDesc: component.messages.nonOrchHost } } @@ -238,7 +239,7 @@ describe('HostsComponent', () => { { selectRow: fakeHosts[1], // orchestrator host expectResults: { - Create: resultNoOrchestrator, + Add: resultNoOrchestrator, Edit: resultNoOrchestrator, Delete: resultNoOrchestrator } @@ -255,7 +256,7 @@ describe('HostsComponent', () => { const tests = [ { expectResults: { - Create: resultMissingFeatures, + Add: resultMissingFeatures, Edit: { disabled: true, disableDesc: '' }, Delete: { disabled: true, disableDesc: '' } } @@ -263,7 +264,7 @@ describe('HostsComponent', () => { { selectRow: fakeHosts[0], // non-orchestrator host expectResults: { - Create: resultMissingFeatures, + Add: resultMissingFeatures, Edit: { disabled: true, disableDesc: component.messages.nonOrchHost }, Delete: { disabled: true, disableDesc: component.messages.nonOrchHost } } @@ -271,7 +272,7 @@ describe('HostsComponent', () => { { selectRow: fakeHosts[1], // orchestrator host expectResults: { - Create: resultMissingFeatures, + Add: resultMissingFeatures, Edit: resultMissingFeatures, Delete: resultMissingFeatures } diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/hosts.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/hosts.component.ts index 8c559a748b05d..1a3798b8ff9c0 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/hosts.component.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/hosts.component.ts @@ -1,4 +1,4 @@ -import { Component, OnInit, TemplateRef, ViewChild } from '@angular/core'; +import { Component, Input, OnInit, TemplateRef, ViewChild } from '@angular/core'; import { Router } from '@angular/router'; import { NgbModalRef } from '@ng-bootstrap/ng-bootstrap'; @@ -11,7 +11,7 @@ import { ConfirmationModalComponent } from '~/app/shared/components/confirmation import { CriticalConfirmationModalComponent } from '~/app/shared/components/critical-confirmation-modal/critical-confirmation-modal.component'; import { FormModalComponent } from '~/app/shared/components/form-modal/form-modal.component'; import { SelectMessages } from '~/app/shared/components/select/select-messages.model'; -import { ActionLabelsI18n } from '~/app/shared/constants/app.constants'; +import { ActionLabelsI18n, URLVerbs } from '~/app/shared/constants/app.constants'; import { TableComponent } from '~/app/shared/datatable/table/table.component'; import { CellTemplate } from '~/app/shared/enum/cell-template.enum'; import { Icons } from '~/app/shared/enum/icons.enum'; @@ -30,6 +30,7 @@ import { ModalService } from '~/app/shared/services/modal.service'; import { NotificationService } from '~/app/shared/services/notification.service'; import { TaskWrapperService } from '~/app/shared/services/task-wrapper.service'; import { URLBuilderService } from '~/app/shared/services/url-builder.service'; +import { HostFormComponent } from './host-form/host-form.component'; const BASE_URL = 'hosts'; @@ -46,6 +47,8 @@ export class HostsComponent extends ListWithDetails implements OnInit { public servicesTpl: TemplateRef; @ViewChild('maintenanceConfirmTpl', { static: true }) maintenanceConfirmTpl: TemplateRef; + @Input() + clusterCreation = false; permissions: Permissions; columns: Array = []; @@ -58,6 +61,8 @@ export class HostsComponent extends ListWithDetails implements OnInit { isExecuting = false; errorMessage: string; enableButton: boolean; + pageURL: string; + bsModalRef: NgbModalRef; icons = Icons; @@ -67,7 +72,7 @@ export class HostsComponent extends ListWithDetails implements OnInit { orchStatus: OrchestratorStatus; actionOrchFeatures = { - create: [OrchestratorFeature.HOST_CREATE], + add: [OrchestratorFeature.HOST_CREATE], edit: [OrchestratorFeature.HOST_LABEL_ADD, OrchestratorFeature.HOST_LABEL_REMOVE], delete: [OrchestratorFeature.HOST_DELETE], maintenance: [ @@ -80,7 +85,6 @@ export class HostsComponent extends ListWithDetails implements OnInit { private authStorageService: AuthStorageService, private hostService: HostService, private cephShortVersionPipe: CephShortVersionPipe, - private urlBuilder: URLBuilderService, private actionLabels: ActionLabelsI18n, private modalService: ModalService, private taskWrapper: TaskWrapperService, @@ -91,13 +95,6 @@ export class HostsComponent extends ListWithDetails implements OnInit { super(); this.permissions = this.authStorageService.getPermissions(); this.tableActions = [ - { - name: this.actionLabels.CREATE, - permission: 'create', - icon: Icons.add, - click: () => this.router.navigate([this.urlBuilder.getCreate()]), - disable: (selection: CdTableSelection) => this.getDisable('create', selection) - }, { name: this.actionLabels.EDIT, permission: 'update', @@ -118,7 +115,10 @@ export class HostsComponent extends ListWithDetails implements OnInit { icon: Icons.enter, click: () => this.hostMaintenance(), disable: (selection: CdTableSelection) => - this.getDisable('maintenance', selection) || this.isExecuting || this.enableButton + this.getDisable('maintenance', selection) || + this.isExecuting || + this.enableButton || + this.clusterCreation }, { name: this.actionLabels.EXIT_MAINTENANCE, @@ -126,12 +126,25 @@ export class HostsComponent extends ListWithDetails implements OnInit { icon: Icons.exit, click: () => this.hostMaintenance(), disable: (selection: CdTableSelection) => - this.getDisable('maintenance', selection) || this.isExecuting || !this.enableButton + this.getDisable('maintenance', selection) || + this.isExecuting || + !this.enableButton || + this.clusterCreation } ]; } ngOnInit() { + this.tableActions.unshift({ + name: this.actionLabels.ADD, + permission: 'create', + icon: Icons.add, + click: () => + this.clusterCreation + ? (this.bsModalRef = this.modalService.show(HostFormComponent)) + : this.router.navigate([BASE_URL, { outlets: { modal: [URLVerbs.ADD] } }]), + disable: (selection: CdTableSelection) => this.getDisable('add', selection) + }); this.columns = [ { name: $localize`Hostname`, @@ -141,6 +154,7 @@ export class HostsComponent extends ListWithDetails implements OnInit { { name: $localize`Services`, prop: 'services', + isHidden: this.clusterCreation, flexGrow: 3, cellTemplate: this.servicesTpl }, @@ -167,6 +181,7 @@ export class HostsComponent extends ListWithDetails implements OnInit { { name: $localize`Version`, prop: 'ceph_version', + isHidden: this.clusterCreation, flexGrow: 1, pipe: this.cephShortVersionPipe } @@ -287,7 +302,7 @@ export class HostsComponent extends ListWithDetails implements OnInit { } getDisable( - action: 'create' | 'edit' | 'delete' | 'maintenance', + action: 'add' | 'edit' | 'delete' | 'maintenance', selection: CdTableSelection ): boolean | string { if (action === 'delete' || action === 'edit' || action === 'maintenance') { diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/osd/osd-flags-indiv-modal/osd-flags-indiv-modal.component.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/osd/osd-flags-indiv-modal/osd-flags-indiv-modal.component.spec.ts index e85223c80e39c..93c9e9adcbbf0 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/osd/osd-flags-indiv-modal/osd-flags-indiv-modal.component.spec.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/osd/osd-flags-indiv-modal/osd-flags-indiv-modal.component.spec.ts @@ -1,6 +1,7 @@ import { HttpClientTestingModule, HttpTestingController } from '@angular/common/http/testing'; import { ComponentFixture, TestBed } from '@angular/core/testing'; import { ReactiveFormsModule } from '@angular/forms'; +import { RouterTestingModule } from '@angular/router/testing'; import { NgbActiveModal, NgbTooltipModule } from '@ng-bootstrap/ng-bootstrap'; import { ToastrModule } from 'ngx-toastr'; @@ -26,7 +27,8 @@ describe('OsdFlagsIndivModalComponent', () => { ReactiveFormsModule, SharedModule, ToastrModule.forRoot(), - NgbTooltipModule + NgbTooltipModule, + RouterTestingModule ], declarations: [OsdFlagsIndivModalComponent], providers: [NgbActiveModal] diff --git a/src/pybind/mgr/dashboard/frontend/src/app/core/auth/login/login.component.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/core/auth/login/login.component.spec.ts index 3cbfab4ebaac3..fc02e9bdeeefb 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/core/auth/login/login.component.spec.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/core/auth/login/login.component.spec.ts @@ -53,6 +53,6 @@ describe('LoginComponent', () => { component.login(); expect(routerNavigateSpy).toHaveBeenCalledTimes(1); - expect(routerNavigateSpy).toHaveBeenCalledWith(['/create-cluster']); + expect(routerNavigateSpy).toHaveBeenCalledWith(['/expand-cluster']); }); }); diff --git a/src/pybind/mgr/dashboard/frontend/src/app/core/auth/login/login.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/core/auth/login/login.component.ts index 77bafd99c82e0..a98548f94c766 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/core/auth/login/login.component.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/core/auth/login/login.component.ts @@ -65,10 +65,10 @@ export class LoginComponent implements OnInit { login() { this.authService.login(this.model).subscribe(() => { - const urlPath = this.postInstalled ? '/' : '/create-cluster'; + const urlPath = this.postInstalled ? '/' : '/expand-cluster'; let url = _.get(this.route.snapshot.queryParams, 'returnUrl', urlPath); if (!this.postInstalled && this.route.snapshot.queryParams['returnUrl'] === '/dashboard') { - url = '/create-cluster'; + url = '/expand-cluster'; } this.router.navigate([url]); }); diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/components.module.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/components.module.ts index ef8b423a3a70c..a6d0624d31885 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/components.module.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/components.module.ts @@ -45,6 +45,7 @@ import { SparklineComponent } from './sparkline/sparkline.component'; import { SubmitButtonComponent } from './submit-button/submit-button.component'; import { TelemetryNotificationComponent } from './telemetry-notification/telemetry-notification.component'; import { UsageBarComponent } from './usage-bar/usage-bar.component'; +import { WizardComponent } from './wizard/wizard.component'; @NgModule({ imports: [ @@ -93,7 +94,8 @@ import { UsageBarComponent } from './usage-bar/usage-bar.component'; Copy2ClipboardButtonComponent, DownloadButtonComponent, FormButtonPanelComponent, - MotdComponent + MotdComponent, + WizardComponent ], providers: [], exports: [ @@ -120,7 +122,8 @@ import { UsageBarComponent } from './usage-bar/usage-bar.component'; Copy2ClipboardButtonComponent, DownloadButtonComponent, FormButtonPanelComponent, - MotdComponent + MotdComponent, + WizardComponent ] }) export class ComponentsModule {} diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/modal/modal.component.html b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/modal/modal.component.html index 5cbd4f58c52fa..657e0d6053f89 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/modal/modal.component.html +++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/modal/modal.component.html @@ -1,13 +1,19 @@ -
    -

    - -

    - -
    +
    +
    +
    +
    +

    + +

    + +
    - + +
    +
    +
    diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/modal/modal.component.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/modal/modal.component.spec.ts index d3ee1ca2abd19..cf08bef10090d 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/modal/modal.component.spec.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/modal/modal.component.spec.ts @@ -1,4 +1,6 @@ import { ComponentFixture, TestBed } from '@angular/core/testing'; +import { Router } from '@angular/router'; +import { RouterTestingModule } from '@angular/router/testing'; import { NgbActiveModal } from '@ng-bootstrap/ng-bootstrap'; @@ -8,14 +10,18 @@ import { ModalComponent } from './modal.component'; describe('ModalComponent', () => { let component: ModalComponent; let fixture: ComponentFixture; + let routerNavigateSpy: jasmine.Spy; configureTestBed({ - declarations: [ModalComponent] + declarations: [ModalComponent], + imports: [RouterTestingModule] }); beforeEach(() => { fixture = TestBed.createComponent(ModalComponent); component = fixture.componentInstance; + routerNavigateSpy = spyOn(TestBed.inject(Router), 'navigate'); + routerNavigateSpy.and.returnValue(true); fixture.detectChanges(); }); @@ -38,4 +44,11 @@ describe('ModalComponent', () => { component.close(); expect(component.modalRef.close).toHaveBeenCalled(); }); + + it('should hide the routed modal', () => { + component.pageURL = 'hosts'; + component.close(); + expect(routerNavigateSpy).toHaveBeenCalledTimes(1); + expect(routerNavigateSpy).toHaveBeenCalledWith(['hosts', { outlets: { modal: null } }]); + }); }); diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/modal/modal.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/modal/modal.component.ts index 730da6d62527b..25e06e62af188 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/modal/modal.component.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/modal/modal.component.ts @@ -1,4 +1,5 @@ import { Component, EventEmitter, Input, Output } from '@angular/core'; +import { Router } from '@angular/router'; import { NgbActiveModal } from '@ng-bootstrap/ng-bootstrap'; @@ -10,6 +11,8 @@ import { NgbActiveModal } from '@ng-bootstrap/ng-bootstrap'; export class ModalComponent { @Input() modalRef: NgbActiveModal; + @Input() + pageURL: string; /** * Should be a function that is triggered when the modal is hidden. @@ -17,8 +20,12 @@ export class ModalComponent { @Output() hide = new EventEmitter(); + constructor(private router: Router) {} + close() { - this.modalRef?.close(); + this.pageURL + ? this.router.navigate([this.pageURL, { outlets: { modal: null } }]) + : this.modalRef?.close(); this.hide.emit(); } } diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/wizard/wizard.component.html b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/wizard/wizard.component.html new file mode 100644 index 0000000000000..25aa3e1df855e --- /dev/null +++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/wizard/wizard.component.html @@ -0,0 +1,19 @@ +
    +
    + +
    +
    diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/wizard/wizard.component.scss b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/wizard/wizard.component.scss new file mode 100644 index 0000000000000..80e3550cd68a5 --- /dev/null +++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/wizard/wizard.component.scss @@ -0,0 +1,30 @@ +@use './src/styles/vendor/variables' as vv; + +.card-body { + padding-left: 0; +} + +span.circle-step { + background: vv.$gray-500; + border-radius: 0.8em; + color: vv.$white; + display: inline-block; + font-weight: bold; + line-height: 1.6em; + margin-right: 5px; + text-align: center; + width: 1.6em; + + &.active { + background-color: vv.$primary; + } +} + +.nav-pills .nav-link { + background-color: vv.$white; + color: vv.$gray-800; + + &.active { + color: vv.$primary; + } +} diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/wizard/wizard.component.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/wizard/wizard.component.spec.ts new file mode 100644 index 0000000000000..b42578fb71193 --- /dev/null +++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/wizard/wizard.component.spec.ts @@ -0,0 +1,25 @@ +import { ComponentFixture, TestBed } from '@angular/core/testing'; + +import { SharedModule } from '~/app/shared/shared.module'; +import { configureTestBed } from '~/testing/unit-test-helper'; +import { WizardComponent } from './wizard.component'; + +describe('WizardComponent', () => { + let component: WizardComponent; + let fixture: ComponentFixture; + + configureTestBed({ + imports: [SharedModule] + }); + + beforeEach(() => { + fixture = TestBed.createComponent(WizardComponent); + component = fixture.componentInstance; + component.stepsTitle = ['Add Hosts', 'Review']; + fixture.detectChanges(); + }); + + it('should create', () => { + expect(component).toBeTruthy(); + }); +}); diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/components/wizard/wizard.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/wizard/wizard.component.ts new file mode 100644 index 0000000000000..d46aa480e7918 --- /dev/null +++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/components/wizard/wizard.component.ts @@ -0,0 +1,39 @@ +import { Component, Input, OnDestroy, OnInit } from '@angular/core'; + +import * as _ from 'lodash'; +import { Observable, Subscription } from 'rxjs'; + +import { WizardStepModel } from '~/app/shared/models/wizard-steps'; +import { WizardStepsService } from '~/app/shared/services/wizard-steps.service'; + +@Component({ + selector: 'cd-wizard', + templateUrl: './wizard.component.html', + styleUrls: ['./wizard.component.scss'] +}) +export class WizardComponent implements OnInit, OnDestroy { + @Input() + stepsTitle: string[]; + + steps: Observable; + currentStep: WizardStepModel; + currentStepSub: Subscription; + + constructor(private stepsService: WizardStepsService) {} + + ngOnInit(): void { + this.stepsService.setTotalSteps(this.stepsTitle.length); + this.steps = this.stepsService.getSteps(); + this.currentStepSub = this.stepsService.getCurrentStep().subscribe((step: WizardStepModel) => { + this.currentStep = step; + }); + } + + onStepClick(step: WizardStepModel) { + this.stepsService.setCurrentStep(step); + } + + ngOnDestroy(): void { + this.currentStepSub.unsubscribe(); + } +} diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/models/wizard-steps.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/models/wizard-steps.ts new file mode 100644 index 0000000000000..177feb486d1a1 --- /dev/null +++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/models/wizard-steps.ts @@ -0,0 +1,4 @@ +export interface WizardStepModel { + stepIndex: number; + isComplete: boolean; +} diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/services/task-message.service.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/services/task-message.service.ts index c39bb0c26b695..44eb9bd30978a 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/shared/services/task-message.service.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/services/task-message.service.ts @@ -114,9 +114,7 @@ export class TaskMessageService { messages = { // Host tasks - 'host/create': this.newTaskMessage(this.commonOperations.create, (metadata) => - this.host(metadata) - ), + 'host/add': this.newTaskMessage(this.commonOperations.add, (metadata) => this.host(metadata)), 'host/delete': this.newTaskMessage(this.commonOperations.delete, (metadata) => this.host(metadata) ), diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/services/wizard-steps.service.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/services/wizard-steps.service.spec.ts new file mode 100644 index 0000000000000..47c2149756703 --- /dev/null +++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/services/wizard-steps.service.spec.ts @@ -0,0 +1,16 @@ +import { TestBed } from '@angular/core/testing'; + +import { WizardStepsService } from './wizard-steps.service'; + +describe('WizardStepsService', () => { + let service: WizardStepsService; + + beforeEach(() => { + TestBed.configureTestingModule({}); + service = TestBed.inject(WizardStepsService); + }); + + it('should be created', () => { + expect(service).toBeTruthy(); + }); +}); diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/services/wizard-steps.service.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/services/wizard-steps.service.ts new file mode 100644 index 0000000000000..e0fb2be944de0 --- /dev/null +++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/services/wizard-steps.service.ts @@ -0,0 +1,58 @@ +import { Injectable } from '@angular/core'; + +import { BehaviorSubject, Observable } from 'rxjs'; + +import { WizardStepModel } from '~/app/shared/models/wizard-steps'; + +const initialStep = [{ stepIndex: 1, isComplete: false }]; + +@Injectable({ + providedIn: 'root' +}) +export class WizardStepsService { + steps$: BehaviorSubject; + currentStep$: BehaviorSubject = new BehaviorSubject(null); + + constructor() { + this.steps$ = new BehaviorSubject(initialStep); + this.currentStep$.next(this.steps$.value[0]); + } + + setTotalSteps(step: number) { + const steps: WizardStepModel[] = []; + for (let i = 1; i <= step; i++) { + steps.push({ stepIndex: i, isComplete: false }); + } + this.steps$ = new BehaviorSubject(steps); + } + + setCurrentStep(step: WizardStepModel): void { + this.currentStep$.next(step); + } + + getCurrentStep(): Observable { + return this.currentStep$.asObservable(); + } + + getSteps(): Observable { + return this.steps$.asObservable(); + } + + moveToNextStep(): void { + const index = this.currentStep$.value.stepIndex; + this.currentStep$.next(this.steps$.value[index]); + } + + moveToPreviousStep(): void { + const index = this.currentStep$.value.stepIndex - 1; + this.currentStep$.next(this.steps$.value[index - 1]); + } + + isLastStep(): boolean { + return this.currentStep$.value.stepIndex === this.steps$.value.length; + } + + isFirstStep(): boolean { + return this.currentStep$.value?.stepIndex - 1 === 0; + } +} diff --git a/src/pybind/mgr/dashboard/frontend/src/styles/ceph-custom/_forms.scss b/src/pybind/mgr/dashboard/frontend/src/styles/ceph-custom/_forms.scss index cca9bd5d5d9d8..3c6ddbf80c998 100644 --- a/src/pybind/mgr/dashboard/frontend/src/styles/ceph-custom/_forms.scss +++ b/src/pybind/mgr/dashboard/frontend/src/styles/ceph-custom/_forms.scss @@ -53,6 +53,17 @@ } cd-modal { + .modal { + /* stylelint-disable */ + background-color: rgba(0, 0, 0, 0.4); + /* stylelint-enable */ + display: block; + } + + .modal-dialog { + max-width: 70vh; + } + .cd-col-form-label { @extend .col-lg-4; } From 2e31ba54273732db5d93a843d22345b676fcecf7 Mon Sep 17 00:00:00 2001 From: Avan Thakkar Date: Fri, 9 Jul 2021 18:57:35 +0530 Subject: [PATCH 0125/2866] mgr/dashboard: Review Section for the Create Cluster Workflow Fixes: https://tracker.ceph.com/issues/50566 Signed-off-by: Avan Thakkar --- .../src/app/ceph/cluster/cluster.module.ts | 4 +- .../create-cluster-review.component.html | 31 +++++++ .../create-cluster-review.component.scss | 0 .../create-cluster-review.component.spec.ts | 61 +++++++++++++ .../create-cluster-review.component.ts | 86 +++++++++++++++++++ .../create-cluster.component.html | 5 +- .../create-cluster.component.spec.ts | 4 +- 7 files changed, 183 insertions(+), 8 deletions(-) create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster-review.component.html create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster-review.component.scss create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster-review.component.spec.ts create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster-review.component.ts diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/cluster.module.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/cluster.module.ts index 185c34b27504d..610bb79baebc7 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/cluster.module.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/cluster.module.ts @@ -22,6 +22,7 @@ import { CephSharedModule } from '../shared/ceph-shared.module'; import { ConfigurationDetailsComponent } from './configuration/configuration-details/configuration-details.component'; import { ConfigurationFormComponent } from './configuration/configuration-form/configuration-form.component'; import { ConfigurationComponent } from './configuration/configuration.component'; +import { CreateClusterReviewComponent } from './create-cluster/create-cluster-review.component'; import { CreateClusterComponent } from './create-cluster/create-cluster.component'; import { CrushmapComponent } from './crushmap/crushmap.component'; import { HostDetailsComponent } from './hosts/host-details/host-details.component'; @@ -114,7 +115,8 @@ import { TelemetryComponent } from './telemetry/telemetry.component'; ServiceFormComponent, OsdFlagsIndivModalComponent, PlacementPipe, - CreateClusterComponent + CreateClusterComponent, + CreateClusterReviewComponent ], providers: [NgbActiveModal] }) diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster-review.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster-review.component.html new file mode 100644 index 0000000000000..fa010fdcd8ede --- /dev/null +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster-review.component.html @@ -0,0 +1,31 @@ +
    +
    +
    + Cluster Resources + + + + + +
    Hosts{{ hostsCount }}
    +
    +
    + +
    + Hosts by Label + + + + Host Details + + +
    +
    diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster-review.component.scss b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster-review.component.scss new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster-review.component.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster-review.component.spec.ts new file mode 100644 index 0000000000000..e823932c09732 --- /dev/null +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster-review.component.spec.ts @@ -0,0 +1,61 @@ +import { HttpClientTestingModule } from '@angular/common/http/testing'; +import { ComponentFixture, TestBed } from '@angular/core/testing'; + +import _ from 'lodash'; +import { of } from 'rxjs'; + +import { CephModule } from '~/app/ceph/ceph.module'; +import { CoreModule } from '~/app/core/core.module'; +import { HostService } from '~/app/shared/api/host.service'; +import { SharedModule } from '~/app/shared/shared.module'; +import { configureTestBed } from '~/testing/unit-test-helper'; +import { CreateClusterReviewComponent } from './create-cluster-review.component'; + +describe('CreateClusterReviewComponent', () => { + let component: CreateClusterReviewComponent; + let fixture: ComponentFixture; + let hostService: HostService; + let hostListSpy: jasmine.Spy; + + configureTestBed({ + imports: [HttpClientTestingModule, SharedModule, CoreModule, CephModule] + }); + + beforeEach(() => { + fixture = TestBed.createComponent(CreateClusterReviewComponent); + component = fixture.componentInstance; + hostService = TestBed.inject(HostService); + hostListSpy = spyOn(hostService, 'list'); + }); + + it('should create', () => { + expect(component).toBeTruthy(); + }); + + it('should verify host metadata calculations', () => { + const hostnames = ['ceph.test1', 'ceph.test2']; + const payload = [ + { + hostname: hostnames[0], + ceph_version: 'ceph version Development', + labels: ['foo', 'bar'] + }, + { + hostname: hostnames[1], + ceph_version: 'ceph version Development', + labels: ['foo1', 'bar1'] + } + ]; + hostListSpy.and.callFake(() => of(payload)); + fixture.detectChanges(); + expect(hostListSpy).toHaveBeenCalled(); + + expect(component.hostsCount).toBe(2); + expect(component.uniqueLabels.size).toBe(4); + const labels = ['foo', 'bar', 'foo1', 'bar1']; + + labels.forEach((label) => { + expect(component.labelOccurrences[label]).toBe(1); + }); + }); +}); diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster-review.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster-review.component.ts new file mode 100644 index 0000000000000..c78e8f910ece6 --- /dev/null +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster-review.component.ts @@ -0,0 +1,86 @@ +import { Component, OnInit } from '@angular/core'; + +import _ from 'lodash'; + +import { HostService } from '~/app/shared/api/host.service'; +import { CellTemplate } from '~/app/shared/enum/cell-template.enum'; + +@Component({ + selector: 'cd-create-cluster-review', + templateUrl: './create-cluster-review.component.html', + styleUrls: ['./create-cluster-review.component.scss'] +}) +export class CreateClusterReviewComponent implements OnInit { + hosts: object[] = []; + hostsDetails: object; + hostsByLabel: object; + hostsCount: number; + labelOccurrences = {}; + hostsCountPerLabel: object[] = []; + uniqueLabels: Set = new Set(); + + constructor(private hostService: HostService) {} + + ngOnInit() { + this.hostsDetails = { + columns: [ + { + prop: 'hostname', + name: $localize`Host Name`, + flexGrow: 2 + }, + { + name: $localize`Labels`, + prop: 'labels', + flexGrow: 1, + cellTransformation: CellTemplate.badge, + customTemplateConfig: { + class: 'badge-dark' + } + } + ] + }; + + this.hostsByLabel = { + columns: [ + { + prop: 'label', + name: $localize`Labels`, + flexGrow: 1, + cellTransformation: CellTemplate.badge, + customTemplateConfig: { + class: 'badge-dark' + } + }, + { + name: $localize`Number of Hosts`, + prop: 'hosts_per_label', + flexGrow: 1 + } + ] + }; + + this.hostService.list().subscribe((resp: object[]) => { + this.hosts = resp; + this.hostsCount = this.hosts.length; + + _.forEach(this.hosts, (hostKey) => { + const labels = hostKey['labels']; + _.forEach(labels, (label) => { + this.labelOccurrences[label] = (this.labelOccurrences[label] || 0) + 1; + this.uniqueLabels.add(label); + }); + }); + + this.uniqueLabels.forEach((label) => { + this.hostsCountPerLabel.push({ + label: label, + hosts_per_label: this.labelOccurrences[label] + }); + }); + + this.hostsByLabel['data'] = [...this.hostsCountPerLabel]; + this.hostsDetails['data'] = [...this.hosts]; + }); + } +} diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster.component.html index fb006ec1beb10..38887328ec3ab 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster.component.html +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster.component.html @@ -44,10 +44,7 @@
    • -

    Review

    -
    -

    To be implemented

    +
    diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster.component.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster.component.spec.ts index 1ebdfb3a59d2f..b0564703840d4 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster.component.spec.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster.component.spec.ts @@ -90,10 +90,8 @@ describe('CreateClusterComponent', () => { fixture.detectChanges(); component.onNextStep(); fixture.detectChanges(); - const heading = fixture.debugElement.query(By.css('.title')).nativeElement; expect(wizardStepServiceSpy).toHaveBeenCalledTimes(1); - expect(hostServiceSpy).toBeCalledTimes(1); - expect(heading.innerHTML).toBe('Review'); + expect(hostServiceSpy).toBeCalledTimes(2); }); it('should show the button labels correctly', () => { From ad1535eea465f5e9f096600e01df1e95813ebb18 Mon Sep 17 00:00:00 2001 From: Nizamudeen A Date: Tue, 17 Aug 2021 19:04:31 +0530 Subject: [PATCH 0126/2866] mgr/dashboard: Cluster Creation Create OSDs Section Create OSDs section in cluster creation wizard Fixes: https://tracker.ceph.com/issues/51991 Fixes: https://tracker.ceph.com/issues/52298 Signed-off-by: Nizamudeen A Signed-off-by: Aashish Sharma --- .../integration/cluster/create-cluster.po.ts | 17 +++++++ .../02-create-cluster-add-host.e2e-spec.ts | 9 +++- .../03-create-cluster-create-osds.e2e-spec.ts | 38 ++++++++++++++ .../03-create-cluster-review.e2e-spec.ts | 4 +- .../workflow/04-cluster-check.e2e-spec.ts | 18 ++++++- .../create-cluster-review.component.html | 7 +++ .../create-cluster-review.component.ts | 9 +++- .../create-cluster.component.html | 9 ++++ .../create-cluster.component.scss | 4 ++ .../create-cluster.component.spec.ts | 33 ++++++++---- .../create-cluster.component.ts | 50 ++++++++++++++++--- .../hosts/host-form/host-form.component.ts | 8 +-- .../app/ceph/cluster/hosts/hosts.component.ts | 19 ++++--- .../inventory-devices.component.html | 2 +- .../inventory-devices.component.spec.ts | 12 +++++ .../inventory-devices.component.ts | 17 +++++++ .../inventory/inventory.component.spec.ts | 7 ++- ...osd-devices-selection-modal.component.html | 1 + .../osd-devices-selection-modal.component.ts | 6 ++- .../osd/osd-form/osd-form.component.html | 8 +-- .../osd/osd-form/osd-form.component.ts | 11 +++- .../shared/services/wizard-steps.service.ts | 5 ++ 22 files changed, 248 insertions(+), 46 deletions(-) create mode 100644 src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/03-create-cluster-create-osds.e2e-spec.ts diff --git a/src/pybind/mgr/dashboard/frontend/cypress/integration/cluster/create-cluster.po.ts b/src/pybind/mgr/dashboard/frontend/cypress/integration/cluster/create-cluster.po.ts index 22941b28e7cb7..4ae03f4aaae29 100644 --- a/src/pybind/mgr/dashboard/frontend/cypress/integration/cluster/create-cluster.po.ts +++ b/src/pybind/mgr/dashboard/frontend/cypress/integration/cluster/create-cluster.po.ts @@ -93,7 +93,10 @@ export class CreateClusterWizardHelper extends PageHelper { } } cy.get('cd-modal cd-submit-button').click(); + this.checkLabelExists(hostname, labels, add); + } + checkLabelExists(hostname: string, labels: string[], add: boolean) { // Verify labels are added or removed from Labels column // First find row with hostname, then find labels in the row this.getTableCell(this.columnIndex.hostname, hostname) @@ -110,4 +113,18 @@ export class CreateClusterWizardHelper extends PageHelper { } }); } + + createOSD(deviceType: 'hdd' | 'ssd') { + // Click Primary devices Add button + cy.get('cd-osd-devices-selection-groups[name="Primary"]').as('primaryGroups'); + cy.get('@primaryGroups').find('button').click(); + + // Select all devices with `deviceType` + cy.get('cd-osd-devices-selection-modal').within(() => { + cy.get('.modal-footer .tc_submitButton').as('addButton').should('be.disabled'); + this.filterTable('Type', deviceType); + this.getTableCount('total').should('be.gte', 1); + cy.get('@addButton').click(); + }); + } } diff --git a/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/02-create-cluster-add-host.e2e-spec.ts b/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/02-create-cluster-add-host.e2e-spec.ts index 7bf5b5be3d496..3b64c2987c9e5 100644 --- a/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/02-create-cluster-add-host.e2e-spec.ts +++ b/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/02-create-cluster-add-host.e2e-spec.ts @@ -8,7 +8,7 @@ describe('Create cluster add host page', () => { 'ceph-node-02.cephlab.com' ]; const addHost = (hostname: string, exist?: boolean) => { - createCluster.add(hostname, exist, true); + createCluster.add(hostname, exist, false); createCluster.checkExist(hostname, true); }; @@ -25,13 +25,18 @@ describe('Create cluster add host page', () => { cy.get('.title').should('contain.text', 'Add Hosts'); }); - it('should check existing host and add new hosts into maintenance mode', () => { + it('should check existing host and add new hosts', () => { createCluster.checkExist(hostnames[0], true); addHost(hostnames[1], false); addHost(hostnames[2], false); }); + it('should verify "_no_schedule" label is added', () => { + createCluster.checkLabelExists(hostnames[1], ['_no_schedule'], true); + createCluster.checkLabelExists(hostnames[2], ['_no_schedule'], true); + }); + it('should not add an existing host', () => { createCluster.add(hostnames[0], true); }); diff --git a/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/03-create-cluster-create-osds.e2e-spec.ts b/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/03-create-cluster-create-osds.e2e-spec.ts new file mode 100644 index 0000000000000..92c0739c5ede4 --- /dev/null +++ b/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/03-create-cluster-create-osds.e2e-spec.ts @@ -0,0 +1,38 @@ +import { CreateClusterWizardHelper } from 'cypress/integration/cluster/create-cluster.po'; +import { OSDsPageHelper } from 'cypress/integration/cluster/osds.po'; + +const osds = new OSDsPageHelper(); + +describe('Create cluster create osds page', () => { + const createCluster = new CreateClusterWizardHelper(); + + beforeEach(() => { + cy.login(); + Cypress.Cookies.preserveOnce('token'); + createCluster.navigateTo(); + createCluster.createCluster(); + cy.get('button[aria-label="Next"]').click(); + }); + + it('should check if nav-link and title contains Create OSDs', () => { + cy.get('.nav-link').should('contain.text', 'Create OSDs'); + + cy.get('.title').should('contain.text', 'Create OSDs'); + }); + + describe('when Orchestrator is available', () => { + it('should create OSDs', () => { + osds.navigateTo(); + osds.getTableCount('total').as('initOSDCount'); + + createCluster.navigateTo(); + createCluster.createCluster(); + cy.get('button[aria-label="Next"]').click(); + + createCluster.createOSD('hdd'); + + cy.get('button[aria-label="Next"]').click(); + cy.get('button[aria-label="Next"]').click(); + }); + }); +}); diff --git a/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/03-create-cluster-review.e2e-spec.ts b/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/03-create-cluster-review.e2e-spec.ts index 17dd84994d60e..624f457458ff2 100644 --- a/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/03-create-cluster-review.e2e-spec.ts +++ b/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/03-create-cluster-review.e2e-spec.ts @@ -10,13 +10,12 @@ describe('Create Cluster Review page', () => { createCluster.createCluster(); cy.get('button[aria-label="Next"]').click(); + cy.get('button[aria-label="Next"]').click(); }); describe('navigation link and title test', () => { it('should check if nav-link and title contains Review', () => { cy.get('.nav-link').should('contain.text', 'Review'); - - cy.get('.title').should('contain.text', 'Review'); }); }); @@ -27,6 +26,7 @@ describe('Create Cluster Review page', () => { // check for fields in table createCluster.getStatusTables().should('contain.text', 'Hosts'); + createCluster.getStatusTables().should('contain.text', 'Storage Capacity'); }); it('should check Hosts by Label and Host Details tables are present', () => { diff --git a/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/04-cluster-check.e2e-spec.ts b/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/04-cluster-check.e2e-spec.ts index 9717dd8b7dbf8..116cbd789c8c0 100644 --- a/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/04-cluster-check.e2e-spec.ts +++ b/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/04-cluster-check.e2e-spec.ts @@ -1,5 +1,6 @@ import { CreateClusterWizardHelper } from 'cypress/integration/cluster/create-cluster.po'; import { HostsPageHelper } from 'cypress/integration/cluster/hosts.po'; +import { OSDsPageHelper } from 'cypress/integration/cluster/osds.po'; describe('when cluster creation is completed', () => { const createCluster = new CreateClusterWizardHelper(); @@ -13,6 +14,7 @@ describe('when cluster creation is completed', () => { createCluster.navigateTo(); createCluster.createCluster(); + cy.get('button[aria-label="Next"]').click(); cy.get('button[aria-label="Next"]').click(); cy.get('button[aria-label="Next"]').click(); @@ -26,9 +28,9 @@ describe('when cluster creation is completed', () => { beforeEach(() => { hosts.navigateTo(); }); - it('should have already exited from maintenance', () => { + it('should have removed "_no_schedule" label', () => { for (let host = 0; host < hostnames.length; host++) { - cy.get('datatable-row-wrapper').should('not.have.text', 'maintenance'); + cy.get('datatable-row-wrapper').should('not.have.text', '_no_schedule'); } }); @@ -46,4 +48,16 @@ describe('when cluster creation is completed', () => { }); }); }); + + describe('OSDs page', () => { + const osds = new OSDsPageHelper(); + + beforeEach(() => { + osds.navigateTo(); + }); + + it('should check if osds are created', { retries: 1 }, () => { + osds.expectTableCount('total', 2); + }); + }); }); diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster-review.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster-review.component.html index fa010fdcd8ede..f95dfdb910f81 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster-review.component.html +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster-review.component.html @@ -9,6 +9,13 @@ class="bold">Hosts {{ hostsCount }} + + Storage Capacity + Number of devices: {{ filteredDevices.length }}. Raw capacity: + {{ capacity | dimlessBinary }}. +
    diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster-review.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster-review.component.ts index c78e8f910ece6..3abbcb122865e 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster-review.component.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster-review.component.ts @@ -4,6 +4,8 @@ import _ from 'lodash'; import { HostService } from '~/app/shared/api/host.service'; import { CellTemplate } from '~/app/shared/enum/cell-template.enum'; +import { WizardStepsService } from '~/app/shared/services/wizard-steps.service'; +import { InventoryDevice } from '../inventory/inventory-devices/inventory-device.model'; @Component({ selector: 'cd-create-cluster-review', @@ -18,8 +20,10 @@ export class CreateClusterReviewComponent implements OnInit { labelOccurrences = {}; hostsCountPerLabel: object[] = []; uniqueLabels: Set = new Set(); + filteredDevices: InventoryDevice[] = []; + capacity = 0; - constructor(private hostService: HostService) {} + constructor(private hostService: HostService, public wizardStepService: WizardStepsService) {} ngOnInit() { this.hostsDetails = { @@ -82,5 +86,8 @@ export class CreateClusterReviewComponent implements OnInit { this.hostsByLabel['data'] = [...this.hostsCountPerLabel]; this.hostsDetails['data'] = [...this.hosts]; }); + + this.filteredDevices = this.wizardStepService.osdDevices; + this.capacity = this.wizardStepService.osdCapacity; } } diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster.component.html index 38887328ec3ab..d9e8ec43d9cc9 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster.component.html +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster.component.html @@ -44,6 +44,15 @@
    +

    Create OSDs

    +
    +
    + +
    +
    +
    diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster.component.scss b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster.component.scss index 580c5219e9f2c..a2e88899a66e2 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster.component.scss +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster.component.scss @@ -24,3 +24,7 @@ cd-hosts { display: none; } } + +.alignForm { + margin-left: -1%; +} diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster.component.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster.component.spec.ts index b0564703840d4..8f9d7328e34ee 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster.component.spec.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster.component.spec.ts @@ -9,6 +9,7 @@ import { CephModule } from '~/app/ceph/ceph.module'; import { CoreModule } from '~/app/core/core.module'; import { HostService } from '~/app/shared/api/host.service'; import { ConfirmationModalComponent } from '~/app/shared/components/confirmation-modal/confirmation-modal.component'; +import { LoadingPanelComponent } from '~/app/shared/components/loading-panel/loading-panel.component'; import { AppConstants } from '~/app/shared/constants/app.constants'; import { ModalService } from '~/app/shared/services/modal.service'; import { WizardStepsService } from '~/app/shared/services/wizard-steps.service'; @@ -24,16 +25,19 @@ describe('CreateClusterComponent', () => { let modalServiceShowSpy: jasmine.Spy; const projectConstants: typeof AppConstants = AppConstants; - configureTestBed({ - imports: [ - HttpClientTestingModule, - RouterTestingModule, - ToastrModule.forRoot(), - SharedModule, - CoreModule, - CephModule - ] - }); + configureTestBed( + { + imports: [ + HttpClientTestingModule, + RouterTestingModule, + ToastrModule.forRoot(), + SharedModule, + CoreModule, + CephModule + ] + }, + [LoadingPanelComponent] + ); beforeEach(() => { fixture = TestBed.createComponent(CreateClusterComponent); @@ -91,7 +95,7 @@ describe('CreateClusterComponent', () => { component.onNextStep(); fixture.detectChanges(); expect(wizardStepServiceSpy).toHaveBeenCalledTimes(1); - expect(hostServiceSpy).toBeCalledTimes(2); + expect(hostServiceSpy).toBeCalledTimes(1); }); it('should show the button labels correctly', () => { @@ -102,6 +106,13 @@ describe('CreateClusterComponent', () => { let cancelBtnLabel = component.showCancelButtonLabel(); expect(cancelBtnLabel).toEqual('Cancel'); + component.onNextStep(); + fixture.detectChanges(); + submitBtnLabel = component.showSubmitButtonLabel(); + expect(submitBtnLabel).toEqual('Next'); + cancelBtnLabel = component.showCancelButtonLabel(); + expect(cancelBtnLabel).toEqual('Back'); + // Last page of the wizard component.onNextStep(); fixture.detectChanges(); diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster.component.ts index b47a63e8cec00..6d78a2110b461 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster.component.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster.component.ts @@ -1,21 +1,26 @@ -import { Component, OnDestroy, TemplateRef, ViewChild } from '@angular/core'; +import { Component, EventEmitter, OnDestroy, Output, TemplateRef, ViewChild } from '@angular/core'; import { Router } from '@angular/router'; import { NgbModalRef } from '@ng-bootstrap/ng-bootstrap'; +import _ from 'lodash'; import { forkJoin, Subscription } from 'rxjs'; import { finalize } from 'rxjs/operators'; import { ClusterService } from '~/app/shared/api/cluster.service'; import { HostService } from '~/app/shared/api/host.service'; +import { OsdService } from '~/app/shared/api/osd.service'; import { ConfirmationModalComponent } from '~/app/shared/components/confirmation-modal/confirmation-modal.component'; -import { ActionLabelsI18n, AppConstants } from '~/app/shared/constants/app.constants'; +import { ActionLabelsI18n, AppConstants, URLVerbs } from '~/app/shared/constants/app.constants'; import { NotificationType } from '~/app/shared/enum/notification-type.enum'; +import { FinishedTask } from '~/app/shared/models/finished-task'; import { Permissions } from '~/app/shared/models/permissions'; import { WizardStepModel } from '~/app/shared/models/wizard-steps'; import { AuthStorageService } from '~/app/shared/services/auth-storage.service'; import { ModalService } from '~/app/shared/services/modal.service'; import { NotificationService } from '~/app/shared/services/notification.service'; +import { TaskWrapperService } from '~/app/shared/services/task-wrapper.service'; import { WizardStepsService } from '~/app/shared/services/wizard-steps.service'; +import { DriveGroup } from '../osd/osd-form/drive-group.model'; @Component({ selector: 'cd-create-cluster', @@ -29,10 +34,15 @@ export class CreateClusterComponent implements OnDestroy { currentStepSub: Subscription; permissions: Permissions; projectConstants: typeof AppConstants = AppConstants; - stepTitles = ['Add Hosts', 'Review']; + stepTitles = ['Add Hosts', 'Create OSDs', 'Review']; startClusterCreation = false; observables: any = []; modalRef: NgbModalRef; + driveGroup = new DriveGroup(); + driveGroups: Object[] = []; + + @Output() + submitAction = new EventEmitter(); constructor( private authStorageService: AuthStorageService, @@ -42,7 +52,10 @@ export class CreateClusterComponent implements OnDestroy { private notificationService: NotificationService, private actionLabels: ActionLabelsI18n, private clusterService: ClusterService, - private modalService: ModalService + private modalService: ModalService, + private taskWrapper: TaskWrapperService, + private osdService: OsdService, + private wizardStepService: WizardStepsService ) { this.permissions = this.authStorageService.getPermissions(); this.currentStepSub = this.stepsService.getCurrentStep().subscribe((step: WizardStepModel) => { @@ -95,17 +108,42 @@ export class CreateClusterComponent implements OnDestroy { .subscribe({ error: (error) => error.preventDefault() }); + + this.taskWrapper + .wrapTaskAroundCall({ + task: new FinishedTask('osd/' + URLVerbs.CREATE, { + tracking_id: _.join(_.map(this.driveGroups, 'service_id'), ', ') + }), + call: this.osdService.create(this.driveGroups) + }) + .subscribe({ + error: (error) => error.preventDefault(), + complete: () => { + this.submitAction.emit(); + } + }); } onNextStep() { if (!this.stepsService.isLastStep()) { this.hostService.list().subscribe((hosts) => { hosts.forEach((host) => { - if (host['status'] === 'maintenance') { - this.observables.push(this.hostService.update(host['hostname'], false, [], true)); + const index = host['labels'].indexOf('_no_schedule', 0); + if (index > -1) { + host['labels'].splice(index, 1); + this.observables.push(this.hostService.update(host['hostname'], true, host['labels'])); } }); }); + this.driveGroup = this.wizardStepService.sharedData; + this.stepsService.getCurrentStep().subscribe((step: WizardStepModel) => { + this.currentStep = step; + }); + if (this.currentStep.stepIndex === 2 && this.driveGroup) { + const user = this.authStorageService.getUsername(); + this.driveGroup.setName(`dashboard-${user}-${_.now()}`); + this.driveGroups.push(this.driveGroup.spec); + } this.stepsService.moveToNextStep(); } else { this.onSubmit(); diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/host-form/host-form.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/host-form/host-form.component.ts index 2fc8b13b540e6..99313a5923aab 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/host-form/host-form.component.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/host-form/host-form.component.ts @@ -25,7 +25,7 @@ export class HostFormComponent extends CdForm implements OnInit { hostnames: string[]; addr: string; status: string; - allLabels: any; + allLabels: string[]; pageURL: string; messages = new SelectMessages({ @@ -60,7 +60,6 @@ export class HostFormComponent extends CdForm implements OnInit { } private createForm() { - const disableMaintenance = this.pageURL !== 'hosts'; this.hostForm = new CdFormGroup({ hostname: new FormControl('', { validators: [ @@ -74,7 +73,7 @@ export class HostFormComponent extends CdForm implements OnInit { validators: [CdValidators.ip()] }), labels: new FormControl([]), - maintenance: new FormControl({ value: disableMaintenance, disabled: disableMaintenance }) + maintenance: new FormControl({ value: false, disabled: this.pageURL !== 'hosts' }) }); } @@ -83,6 +82,9 @@ export class HostFormComponent extends CdForm implements OnInit { this.addr = this.hostForm.get('addr').value; this.status = this.hostForm.get('maintenance').value ? 'maintenance' : ''; this.allLabels = this.hostForm.get('labels').value; + if (this.pageURL !== 'hosts' && !this.allLabels.includes('_no_schedule')) { + this.allLabels.push('_no_schedule'); + } this.taskWrapper .wrapTaskAroundCall({ task: new FinishedTask('host/' + URLVerbs.ADD, { diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/hosts.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/hosts.component.ts index 1a3798b8ff9c0..c70f755f799e8 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/hosts.component.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/hosts.component.ts @@ -115,10 +115,8 @@ export class HostsComponent extends ListWithDetails implements OnInit { icon: Icons.enter, click: () => this.hostMaintenance(), disable: (selection: CdTableSelection) => - this.getDisable('maintenance', selection) || - this.isExecuting || - this.enableButton || - this.clusterCreation + this.getDisable('maintenance', selection) || this.isExecuting || this.enableButton, + visible: () => !this.clusterCreation }, { name: this.actionLabels.EXIT_MAINTENANCE, @@ -126,10 +124,8 @@ export class HostsComponent extends ListWithDetails implements OnInit { icon: Icons.exit, click: () => this.hostMaintenance(), disable: (selection: CdTableSelection) => - this.getDisable('maintenance', selection) || - this.isExecuting || - !this.enableButton || - this.clusterCreation + this.getDisable('maintenance', selection) || this.isExecuting || this.enableButton, + visible: () => !this.clusterCreation } ]; } @@ -189,6 +185,13 @@ export class HostsComponent extends ListWithDetails implements OnInit { this.orchService.status().subscribe((status: OrchestratorStatus) => { this.orchStatus = status; }); + + if (this.clusterCreation) { + const hiddenColumns = ['services', 'ceph_version']; + this.columns = this.columns.filter((col: any) => { + return !hiddenColumns.includes(col.prop); + }); + } } updateSelection(selection: CdTableSelection) { diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/inventory/inventory-devices/inventory-devices.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/inventory/inventory-devices/inventory-devices.component.html index 244bd8162cf4d..54cee708d2634 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/inventory/inventory-devices/inventory-devices.component.html +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/inventory/inventory-devices/inventory-devices.component.html @@ -4,7 +4,7 @@ [forceIdentifier]="true" [selectionType]="selectionType" columnMode="flex" - [autoReload]="false" + (fetchData)="getDevices()" [searchField]="false" (updateSelection)="updateSelection($event)" (columnFiltersChanged)="onColumnFiltersChanged($event)"> diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/inventory/inventory-devices/inventory-devices.component.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/inventory/inventory-devices/inventory-devices.component.spec.ts index 5fce0381cd2a5..29a3ece96d8ae 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/inventory/inventory-devices/inventory-devices.component.spec.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/inventory/inventory-devices/inventory-devices.component.spec.ts @@ -7,6 +7,7 @@ import { RouterTestingModule } from '@angular/router/testing'; import { ToastrModule } from 'ngx-toastr'; +import { HostService } from '~/app/shared/api/host.service'; import { OrchestratorService } from '~/app/shared/api/orchestrator.service'; import { TableActionsComponent } from '~/app/shared/datatable/table-actions/table-actions.component'; import { CdTableAction } from '~/app/shared/models/cd-table-action'; @@ -23,6 +24,7 @@ describe('InventoryDevicesComponent', () => { let component: InventoryDevicesComponent; let fixture: ComponentFixture; let orchService: OrchestratorService; + let hostService: HostService; const fakeAuthStorageService = { getPermissions: () => { @@ -59,6 +61,7 @@ describe('InventoryDevicesComponent', () => { beforeEach(() => { fixture = TestBed.createComponent(InventoryDevicesComponent); component = fixture.componentInstance; + hostService = TestBed.inject(HostService); orchService = TestBed.inject(OrchestratorService); }); @@ -70,6 +73,15 @@ describe('InventoryDevicesComponent', () => { expect(component.columns.every((column) => Boolean(column.prop))).toBeTruthy(); }); + it('should call inventoryDataList only when showOnlyAvailableData is true', () => { + const hostServiceSpy = spyOn(hostService, 'inventoryDeviceList').and.callThrough(); + component.getDevices(); + expect(hostServiceSpy).toBeCalledTimes(0); + component.showAvailDeviceOnly = true; + component.getDevices(); + expect(hostServiceSpy).toBeCalledTimes(1); + }); + describe('table actions', () => { const fakeDevices = require('./fixtures/inventory_list_response.json'); diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/inventory/inventory-devices/inventory-devices.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/inventory/inventory-devices/inventory-devices.component.ts index fa778d5b4f29b..e0d82cb1975db 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/inventory/inventory-devices/inventory-devices.component.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/inventory/inventory-devices/inventory-devices.component.ts @@ -43,6 +43,7 @@ export class InventoryDevicesComponent implements OnInit, OnDestroy { // Devices @Input() devices: InventoryDevice[] = []; + @Input() showAvailDeviceOnly = false; // Do not display these columns @Input() hiddenColumns: string[] = []; @@ -175,6 +176,22 @@ export class InventoryDevicesComponent implements OnInit, OnDestroy { } } + getDevices() { + if (this.showAvailDeviceOnly) { + this.hostService.inventoryDeviceList().subscribe( + (devices: InventoryDevice[]) => { + this.devices = _.filter(devices, 'available'); + this.devices = [...this.devices]; + }, + () => { + this.devices = []; + } + ); + } else { + this.devices = [...this.devices]; + } + } + ngOnDestroy() { if (this.fetchInventorySub) { this.fetchInventorySub.unsubscribe(); diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/inventory/inventory.component.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/inventory/inventory.component.spec.ts index da24403dee1e1..dd60f7959fd9d 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/inventory/inventory.component.spec.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/inventory/inventory.component.spec.ts @@ -52,17 +52,16 @@ describe('InventoryComponent', () => { describe('after ngOnInit', () => { it('should load devices', () => { fixture.detectChanges(); - expect(hostService.inventoryDeviceList).toHaveBeenNthCalledWith(1, undefined, false); component.refresh(); // click refresh button - expect(hostService.inventoryDeviceList).toHaveBeenNthCalledWith(2, undefined, true); + expect(hostService.inventoryDeviceList).toHaveBeenNthCalledWith(1, undefined, false); const newHost = 'host0'; component.hostname = newHost; fixture.detectChanges(); component.ngOnChanges(); - expect(hostService.inventoryDeviceList).toHaveBeenNthCalledWith(3, newHost, false); + expect(hostService.inventoryDeviceList).toHaveBeenNthCalledWith(2, newHost, false); component.refresh(); // click refresh button - expect(hostService.inventoryDeviceList).toHaveBeenNthCalledWith(4, newHost, true); + expect(hostService.inventoryDeviceList).toHaveBeenNthCalledWith(3, newHost, true); }); }); }); diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/osd/osd-devices-selection-modal/osd-devices-selection-modal.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/osd/osd-devices-selection-modal/osd-devices-selection-modal.component.html index 30effc21b53e2..3e53d5c410cd2 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/osd/osd-devices-selection-modal/osd-devices-selection-modal.component.html +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/osd/osd-devices-selection-modal/osd-devices-selection-modal.component.html @@ -20,6 +20,7 @@ diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/osd/osd-devices-selection-modal/osd-devices-selection-modal.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/osd/osd-devices-selection-modal/osd-devices-selection-modal.component.ts index fe77a4fe4d8a0..1909803dc3380 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/osd/osd-devices-selection-modal/osd-devices-selection-modal.component.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/osd/osd-devices-selection-modal/osd-devices-selection-modal.component.ts @@ -11,6 +11,7 @@ import { Icons } from '~/app/shared/enum/icons.enum'; import { CdFormBuilder } from '~/app/shared/forms/cd-form-builder'; import { CdFormGroup } from '~/app/shared/forms/cd-form-group'; import { CdTableColumnFiltersChange } from '~/app/shared/models/cd-table-column-filters-change'; +import { WizardStepsService } from '~/app/shared/services/wizard-steps.service'; @Component({ selector: 'cd-osd-devices-selection-modal', @@ -42,7 +43,8 @@ export class OsdDevicesSelectionModalComponent implements AfterViewInit { constructor( private formBuilder: CdFormBuilder, public activeModal: NgbActiveModal, - public actionLabels: ActionLabelsI18n + public actionLabels: ActionLabelsI18n, + public wizardStepService: WizardStepsService ) { this.action = actionLabels.ADD; this.createForm(); @@ -80,6 +82,8 @@ export class OsdDevicesSelectionModalComponent implements AfterViewInit { this.filteredDevices = event.data; this.capacity = _.sumBy(this.filteredDevices, 'sys_api.size'); this.event = event; + this.wizardStepService.osdDevices = this.filteredDevices; + this.wizardStepService.osdCapacity = this.capacity; } } diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/osd/osd-form/osd-form.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/osd/osd-form/osd-form.component.html index 390f673bdaf5b..675e20fcf59a6 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/osd/osd-form/osd-form.component.html +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/osd/osd-form/osd-form.component.html @@ -7,8 +7,9 @@ [formGroup]="form" novalidate>
    -
    {{ action | titlecase }} {{ resource | upperFirst }}
    +
    {{ action | titlecase }} {{ resource | upperFirst }}
    -
    +
    ; currentStep$: BehaviorSubject = new BehaviorSubject(null); + sharedData = new DriveGroup(); + osdDevices: InventoryDevice[] = []; + osdCapacity = 0; constructor() { this.steps$ = new BehaviorSubject(initialStep); From b914f59ff123b9e006d7d11dc786e967c0271dbf Mon Sep 17 00:00:00 2001 From: Aashish Sharma Date: Tue, 7 Sep 2021 12:00:45 +0530 Subject: [PATCH 0127/2866] mgr/dashboard: Cluster Creation Add Services Section Add Services section in cluster creation wizard Create Cluster OSD Section Followups 1. The device preview disappearing when going to next step and coming back to the previous step 2. Even when clearing the device preview, the Storage Capacity count and the drive group spec doesn't get cleared. 3. Expanding the cluster without selecting any devices gives a 400 error. 4. Renamed "Delete Host" to "Remove Host" 5. Generalizing most of the sub component code Fixes: https://tracker.ceph.com/issues/52499 Fixes: https://tracker.ceph.com/issues/51991 Signed-off-by: Nizamudeen A Signed-off-by: Aashish Sharma --- src/pybind/mgr/dashboard/controllers/host.py | 6 +- .../integration/cluster/create-cluster.po.ts | 63 ++++++++- .../cypress/integration/cluster/hosts.po.ts | 2 +- .../integration/cluster/services.po.ts | 2 +- .../03-create-cluster-create-osds.e2e-spec.ts | 1 + ...create-cluster-create-services.e2e-spec.ts | 36 +++++ ...s => 05-create-cluster-review.e2e-spec.ts} | 11 +- ...e-spec.ts => 06-cluster-check.e2e-spec.ts} | 14 ++ .../cypress/integration/page-helper.po.ts | 13 +- .../frontend/src/app/app-routing.module.ts | 6 +- .../create-cluster-review.component.html | 11 +- .../create-cluster-review.component.spec.ts | 29 ++--- .../create-cluster-review.component.ts | 90 +++++++++---- .../create-cluster.component.html | 20 ++- .../create-cluster.component.spec.ts | 33 ++++- .../create-cluster.component.ts | 123 ++++++++++-------- .../ceph/cluster/hosts/hosts.component.html | 4 +- .../cluster/hosts/hosts.component.spec.ts | 23 ++-- .../app/ceph/cluster/hosts/hosts.component.ts | 69 +++++----- ...sd-devices-selection-groups.component.html | 2 +- ...devices-selection-groups.component.spec.ts | 6 +- .../osd-devices-selection-groups.component.ts | 34 ++++- .../osd-devices-selection-modal.component.ts | 2 - .../osd/osd-form/osd-form.component.html | 4 +- .../osd/osd-form/osd-form.component.ts | 17 ++- .../service-form/service-form.component.html | 25 ++-- .../service-form.component.spec.ts | 4 +- .../service-form/service-form.component.ts | 27 ++-- .../cluster/services/services.component.html | 3 +- .../cluster/services/services.component.ts | 26 +++- .../src/app/shared/api/osd.service.ts | 2 + .../models/inventory-device-type.model.ts | 9 ++ .../app/shared/models/orchestrator.enum.ts | 4 +- .../shared/services/task-message.service.ts | 2 +- .../shared/services/wizard-steps.service.ts | 5 - .../mgr/dashboard/services/orchestrator.py | 4 +- 36 files changed, 505 insertions(+), 227 deletions(-) create mode 100644 src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/04-create-cluster-create-services.e2e-spec.ts rename src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/{03-create-cluster-review.e2e-spec.ts => 05-create-cluster-review.e2e-spec.ts} (84%) rename src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/{04-cluster-check.e2e-spec.ts => 06-cluster-check.e2e-spec.ts} (83%) create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/shared/models/inventory-device-type.model.ts diff --git a/src/pybind/mgr/dashboard/controllers/host.py b/src/pybind/mgr/dashboard/controllers/host.py index 6ba1a5e0f1cbb..c246de8f9f12c 100644 --- a/src/pybind/mgr/dashboard/controllers/host.py +++ b/src/pybind/mgr/dashboard/controllers/host.py @@ -283,7 +283,7 @@ def list(self, sources=None): from_orchestrator = 'orchestrator' in _sources return get_hosts(from_ceph, from_orchestrator) - @raise_if_no_orchestrator([OrchFeature.HOST_LIST, OrchFeature.HOST_CREATE]) + @raise_if_no_orchestrator([OrchFeature.HOST_LIST, OrchFeature.HOST_ADD]) @handle_orchestrator_error('host') @host_task('add', {'hostname': '{hostname}'}) @EndpointDoc('', @@ -301,9 +301,9 @@ def create(self, hostname: str, status: Optional[str] = None): # pragma: no cover - requires realtime env add_host(hostname, addr, labels, status) - @raise_if_no_orchestrator([OrchFeature.HOST_LIST, OrchFeature.HOST_DELETE]) + @raise_if_no_orchestrator([OrchFeature.HOST_LIST, OrchFeature.HOST_REMOVE]) @handle_orchestrator_error('host') - @host_task('delete', {'hostname': '{hostname}'}) + @host_task('remove', {'hostname': '{hostname}'}) @allow_empty_body def delete(self, hostname): # pragma: no cover - requires realtime env orch_client = OrchClient.instance() diff --git a/src/pybind/mgr/dashboard/frontend/cypress/integration/cluster/create-cluster.po.ts b/src/pybind/mgr/dashboard/frontend/cypress/integration/cluster/create-cluster.po.ts index 4ae03f4aaae29..a528c0fc12e05 100644 --- a/src/pybind/mgr/dashboard/frontend/cypress/integration/cluster/create-cluster.po.ts +++ b/src/pybind/mgr/dashboard/frontend/cypress/integration/cluster/create-cluster.po.ts @@ -13,6 +13,11 @@ export class CreateClusterWizardHelper extends PageHelper { status: 3 }; + serviceColumnIndex = { + service_name: 1, + placement: 2 + }; + createCluster() { cy.get('cd-create-cluster').should('contain.text', 'Please expand your cluster first'); cy.get('[name=expand-cluster]').click(); @@ -41,7 +46,6 @@ export class CreateClusterWizardHelper extends PageHelper { add(hostname: string, exist?: boolean, maintenance?: boolean) { cy.get('.btn.btn-accent').first().click({ force: true }); - cy.get('cd-modal').should('exist'); cy.get('cd-modal').within(() => { cy.get('#hostname').type(hostname); @@ -70,7 +74,7 @@ export class CreateClusterWizardHelper extends PageHelper { } delete(hostname: string) { - super.delete(hostname, this.columnIndex.hostname); + super.delete(hostname, this.columnIndex.hostname, 'hosts'); } // Add or remove labels on a host, then verify labels in the table @@ -127,4 +131,59 @@ export class CreateClusterWizardHelper extends PageHelper { cy.get('@addButton').click(); }); } + + private selectServiceType(serviceType: string) { + return this.selectOption('service_type', serviceType); + } + + addService(serviceType: string) { + cy.get('.btn.btn-accent').first().click({ force: true }); + cy.get('cd-modal').should('exist'); + cy.get('cd-modal').within(() => { + this.selectServiceType(serviceType); + if (serviceType === 'rgw') { + cy.get('#service_id').type('rgw'); + cy.get('#count').type('1'); + } else if (serviceType === 'ingress') { + this.selectOption('backend_service', 'rgw.rgw'); + cy.get('#service_id').should('have.value', 'rgw.rgw'); + cy.get('#virtual_ip').type('192.168.20.1/24'); + cy.get('#frontend_port').type('8081'); + cy.get('#monitor_port').type('8082'); + } + + cy.get('cd-submit-button').click(); + }); + } + + checkServiceExist(serviceName: string, exist: boolean) { + this.getTableCell(this.serviceColumnIndex.service_name, serviceName).should(($elements) => { + const services = $elements.map((_, el) => el.textContent).get(); + if (exist) { + expect(services).to.include(serviceName); + } else { + expect(services).to.not.include(serviceName); + } + }); + } + + deleteService(serviceName: string, wait: number) { + const getRow = this.getTableCell.bind(this, this.serviceColumnIndex.service_name); + getRow(serviceName).click(); + + // Clicks on table Delete button + this.clickActionButton('delete'); + + // Confirms deletion + cy.get('cd-modal .custom-control-label').click(); + cy.contains('cd-modal button', 'Delete').click(); + + // Wait for modal to close + cy.get('cd-modal').should('not.exist'); + + // wait for delete operation to complete: tearing down the service daemons + cy.wait(wait); + + this.checkServiceExist(serviceName, false); + } } diff --git a/src/pybind/mgr/dashboard/frontend/cypress/integration/cluster/hosts.po.ts b/src/pybind/mgr/dashboard/frontend/cypress/integration/cluster/hosts.po.ts index 7a7e00d6648ac..5f3d39dcedc50 100644 --- a/src/pybind/mgr/dashboard/frontend/cypress/integration/cluster/hosts.po.ts +++ b/src/pybind/mgr/dashboard/frontend/cypress/integration/cluster/hosts.po.ts @@ -79,7 +79,7 @@ export class HostsPageHelper extends PageHelper { @PageHelper.restrictTo(pages.index.url) delete(hostname: string) { - super.delete(hostname, this.columnIndex.hostname); + super.delete(hostname, this.columnIndex.hostname, 'hosts'); } // Add or remove labels on a host, then verify labels in the table diff --git a/src/pybind/mgr/dashboard/frontend/cypress/integration/cluster/services.po.ts b/src/pybind/mgr/dashboard/frontend/cypress/integration/cluster/services.po.ts index e28a8fbd57c26..0a943d4b0553f 100644 --- a/src/pybind/mgr/dashboard/frontend/cypress/integration/cluster/services.po.ts +++ b/src/pybind/mgr/dashboard/frontend/cypress/integration/cluster/services.po.ts @@ -2,7 +2,7 @@ import { PageHelper } from '../page-helper.po'; const pages = { index: { url: '#/services', id: 'cd-services' }, - create: { url: '#/services/create', id: 'cd-service-form' } + create: { url: '#/services/(modal:create)', id: 'cd-service-form' } }; export class ServicesPageHelper extends PageHelper { diff --git a/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/03-create-cluster-create-osds.e2e-spec.ts b/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/03-create-cluster-create-osds.e2e-spec.ts index 92c0739c5ede4..ee272bcf943b2 100644 --- a/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/03-create-cluster-create-osds.e2e-spec.ts +++ b/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/03-create-cluster-create-osds.e2e-spec.ts @@ -33,6 +33,7 @@ describe('Create cluster create osds page', () => { cy.get('button[aria-label="Next"]').click(); cy.get('button[aria-label="Next"]').click(); + cy.get('button[aria-label="Next"]').click(); }); }); }); diff --git a/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/04-create-cluster-create-services.e2e-spec.ts b/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/04-create-cluster-create-services.e2e-spec.ts new file mode 100644 index 0000000000000..0a95474c91d3f --- /dev/null +++ b/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/04-create-cluster-create-services.e2e-spec.ts @@ -0,0 +1,36 @@ +import { CreateClusterWizardHelper } from 'cypress/integration/cluster/create-cluster.po'; + +describe('Create cluster create services page', () => { + const createCluster = new CreateClusterWizardHelper(); + + beforeEach(() => { + cy.login(); + Cypress.Cookies.preserveOnce('token'); + createCluster.navigateTo(); + createCluster.createCluster(); + cy.get('button[aria-label="Next"]').click(); + cy.get('button[aria-label="Next"]').click(); + }); + + it('should check if nav-link and title contains Create Services', () => { + cy.get('.nav-link').should('contain.text', 'Create Services'); + + cy.get('.title').should('contain.text', 'Create Services'); + }); + + describe('when Orchestrator is available', () => { + it('should create an rgw service', () => { + createCluster.addService('rgw'); + + createCluster.checkExist('rgw.rgw', true); + }); + + it('should create and delete an ingress service', () => { + createCluster.addService('ingress'); + + createCluster.checkExist('ingress.rgw.rgw', true); + + createCluster.deleteService('ingress.rgw.rgw', 60000); + }); + }); +}); diff --git a/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/03-create-cluster-review.e2e-spec.ts b/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/05-create-cluster-review.e2e-spec.ts similarity index 84% rename from src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/03-create-cluster-review.e2e-spec.ts rename to src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/05-create-cluster-review.e2e-spec.ts index 624f457458ff2..9097034e1a88c 100644 --- a/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/03-create-cluster-review.e2e-spec.ts +++ b/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/05-create-cluster-review.e2e-spec.ts @@ -11,6 +11,7 @@ describe('Create Cluster Review page', () => { cy.get('button[aria-label="Next"]').click(); cy.get('button[aria-label="Next"]').click(); + cy.get('button[aria-label="Next"]').click(); }); describe('navigation link and title test', () => { @@ -29,18 +30,18 @@ describe('Create Cluster Review page', () => { createCluster.getStatusTables().should('contain.text', 'Storage Capacity'); }); - it('should check Hosts by Label and Host Details tables are present', () => { + it('should check Hosts by Services and Host Details tables are present', () => { // check for there to be two tables createCluster.getDataTables().should('have.length', 2); - // check for table header 'Hosts by Label' - createCluster.getLegends().its(1).should('have.text', 'Hosts by Label'); + // check for table header 'Hosts by Services' + createCluster.getLegends().its(1).should('have.text', 'Hosts by Services'); // check for table header 'Host Details' createCluster.getLegends().its(2).should('have.text', 'Host Details'); - // verify correct columns on Hosts by Label table - createCluster.getDataTableHeaders(0).contains('Label'); + // verify correct columns on Hosts by Services table + createCluster.getDataTableHeaders(0).contains('Services'); createCluster.getDataTableHeaders(0).contains('Number of Hosts'); diff --git a/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/04-cluster-check.e2e-spec.ts b/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/06-cluster-check.e2e-spec.ts similarity index 83% rename from src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/04-cluster-check.e2e-spec.ts rename to src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/06-cluster-check.e2e-spec.ts index 116cbd789c8c0..a5b54786b8738 100644 --- a/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/04-cluster-check.e2e-spec.ts +++ b/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/06-cluster-check.e2e-spec.ts @@ -1,6 +1,7 @@ import { CreateClusterWizardHelper } from 'cypress/integration/cluster/create-cluster.po'; import { HostsPageHelper } from 'cypress/integration/cluster/hosts.po'; import { OSDsPageHelper } from 'cypress/integration/cluster/osds.po'; +import { ServicesPageHelper } from 'cypress/integration/cluster/services.po'; describe('when cluster creation is completed', () => { const createCluster = new CreateClusterWizardHelper(); @@ -17,6 +18,7 @@ describe('when cluster creation is completed', () => { cy.get('button[aria-label="Next"]').click(); cy.get('button[aria-label="Next"]').click(); cy.get('button[aria-label="Next"]').click(); + cy.get('button[aria-label="Next"]').click(); cy.get('cd-dashboard').should('exist'); }); @@ -60,4 +62,16 @@ describe('when cluster creation is completed', () => { osds.expectTableCount('total', 2); }); }); + + describe('Services page', () => { + const services = new ServicesPageHelper(); + + beforeEach(() => { + services.navigateTo(); + }); + + it('should check if services are created', () => { + services.checkExist('rgw.rgw', true); + }); + }); }); diff --git a/src/pybind/mgr/dashboard/frontend/cypress/integration/page-helper.po.ts b/src/pybind/mgr/dashboard/frontend/cypress/integration/page-helper.po.ts index 7b099fd98acf2..4e15088a2c9c9 100644 --- a/src/pybind/mgr/dashboard/frontend/cypress/integration/page-helper.po.ts +++ b/src/pybind/mgr/dashboard/frontend/cypress/integration/page-helper.po.ts @@ -256,19 +256,22 @@ export abstract class PageHelper { * @param name The string to search in table cells. * @param columnIndex If provided, search string in columnIndex column. */ - delete(name: string, columnIndex?: number) { + delete(name: string, columnIndex?: number, section?: string) { // Selects row const getRow = columnIndex ? this.getTableCell.bind(this, columnIndex) : this.getFirstTableCell.bind(this); getRow(name).click(); + let action: string; + section === 'hosts' ? (action = 'remove') : (action = 'delete'); - // Clicks on table Delete button - this.clickActionButton('delete'); + // Clicks on table Delete/Remove button + this.clickActionButton(action); - // Confirms deletion + // Convert action to SentenceCase and Confirms deletion + const actionUpperCase = action.charAt(0).toUpperCase() + action.slice(1); cy.get('cd-modal .custom-control-label').click(); - cy.contains('cd-modal button', 'Delete').click(); + cy.contains('cd-modal button', actionUpperCase).click(); // Wait for modal to close cy.get('cd-modal').should('not.exist'); diff --git a/src/pybind/mgr/dashboard/frontend/src/app/app-routing.module.ts b/src/pybind/mgr/dashboard/frontend/src/app/app-routing.module.ts index bdd2cb8978234..e1eb456ee8aa0 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/app-routing.module.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/app-routing.module.ts @@ -122,7 +122,8 @@ const routes: Routes = [ }, { path: 'services', - canActivateChild: [ModuleStatusGuardService], + component: ServicesComponent, + canActivate: [ModuleStatusGuardService], data: { moduleStatusGuardConfig: { apiPath: 'orchestrator', @@ -134,11 +135,10 @@ const routes: Routes = [ breadcrumbs: 'Cluster/Services' }, children: [ - { path: '', component: ServicesComponent }, { path: URLVerbs.CREATE, component: ServiceFormComponent, - data: { breadcrumbs: ActionLabels.CREATE } + outlet: 'modal' } ] }, diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster-review.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster-review.component.html index f95dfdb910f81..e6f31dc9e74f8 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster-review.component.html +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster-review.component.html @@ -12,9 +12,8 @@ Storage Capacity - Number of devices: {{ filteredDevices.length }}. Raw capacity: - {{ capacity | dimlessBinary }}. + Number of devices: {{ totalDevices }}. Raw capacity: + {{ totalCapacity | dimlessBinary }}. @@ -22,9 +21,9 @@
    Hosts by Label - Hosts by Services + diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster-review.component.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster-review.component.spec.ts index e823932c09732..8122cb682f0ba 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster-review.component.spec.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster-review.component.spec.ts @@ -6,7 +6,7 @@ import { of } from 'rxjs'; import { CephModule } from '~/app/ceph/ceph.module'; import { CoreModule } from '~/app/core/core.module'; -import { HostService } from '~/app/shared/api/host.service'; +import { CephServiceService } from '~/app/shared/api/ceph-service.service'; import { SharedModule } from '~/app/shared/shared.module'; import { configureTestBed } from '~/testing/unit-test-helper'; import { CreateClusterReviewComponent } from './create-cluster-review.component'; @@ -14,8 +14,8 @@ import { CreateClusterReviewComponent } from './create-cluster-review.component' describe('CreateClusterReviewComponent', () => { let component: CreateClusterReviewComponent; let fixture: ComponentFixture; - let hostService: HostService; - let hostListSpy: jasmine.Spy; + let cephServiceService: CephServiceService; + let serviceListSpy: jasmine.Spy; configureTestBed({ imports: [HttpClientTestingModule, SharedModule, CoreModule, CephModule] @@ -24,8 +24,8 @@ describe('CreateClusterReviewComponent', () => { beforeEach(() => { fixture = TestBed.createComponent(CreateClusterReviewComponent); component = fixture.componentInstance; - hostService = TestBed.inject(HostService); - hostListSpy = spyOn(hostService, 'list'); + cephServiceService = TestBed.inject(CephServiceService); + serviceListSpy = spyOn(cephServiceService, 'list'); }); it('should create', () => { @@ -37,25 +37,18 @@ describe('CreateClusterReviewComponent', () => { const payload = [ { hostname: hostnames[0], - ceph_version: 'ceph version Development', - labels: ['foo', 'bar'] + service_type: ['mgr', 'mon'] }, { hostname: hostnames[1], - ceph_version: 'ceph version Development', - labels: ['foo1', 'bar1'] + service_type: ['mgr', 'alertmanager'] } ]; - hostListSpy.and.callFake(() => of(payload)); + serviceListSpy.and.callFake(() => of(payload)); fixture.detectChanges(); - expect(hostListSpy).toHaveBeenCalled(); + expect(serviceListSpy).toHaveBeenCalled(); - expect(component.hostsCount).toBe(2); - expect(component.uniqueLabels.size).toBe(4); - const labels = ['foo', 'bar', 'foo1', 'bar1']; - - labels.forEach((label) => { - expect(component.labelOccurrences[label]).toBe(1); - }); + expect(component.serviceCount).toBe(2); + expect(component.uniqueServices.size).toBe(2); }); }); diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster-review.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster-review.component.ts index 3abbcb122865e..90e25d1162dc7 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster-review.component.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster-review.component.ts @@ -2,10 +2,12 @@ import { Component, OnInit } from '@angular/core'; import _ from 'lodash'; +import { CephServiceService } from '~/app/shared/api/ceph-service.service'; import { HostService } from '~/app/shared/api/host.service'; +import { OsdService } from '~/app/shared/api/osd.service'; import { CellTemplate } from '~/app/shared/enum/cell-template.enum'; +import { CephServiceSpec } from '~/app/shared/models/service.interface'; import { WizardStepsService } from '~/app/shared/services/wizard-steps.service'; -import { InventoryDevice } from '../inventory/inventory-devices/inventory-device.model'; @Component({ selector: 'cd-create-cluster-review', @@ -15,17 +17,30 @@ import { InventoryDevice } from '../inventory/inventory-devices/inventory-device export class CreateClusterReviewComponent implements OnInit { hosts: object[] = []; hostsDetails: object; - hostsByLabel: object; + hostsByService: object; hostsCount: number; - labelOccurrences = {}; - hostsCountPerLabel: object[] = []; - uniqueLabels: Set = new Set(); - filteredDevices: InventoryDevice[] = []; - capacity = 0; + serviceCount: number; + serviceOccurrences = {}; + hostsCountPerService: object[] = []; + uniqueServices: Set = new Set(); + totalDevices: number; + totalCapacity = 0; + services: Array = []; - constructor(private hostService: HostService, public wizardStepService: WizardStepsService) {} + constructor( + public wizardStepsService: WizardStepsService, + public cephServiceService: CephServiceService, + public hostService: HostService, + private osdService: OsdService + ) {} ngOnInit() { + let dataDevices = 0; + let dataDeviceCapacity = 0; + let walDevices = 0; + let walDeviceCapacity = 0; + let dbDevices = 0; + let dbDeviceCapacity = 0; this.hostsDetails = { columns: [ { @@ -45,11 +60,11 @@ export class CreateClusterReviewComponent implements OnInit { ] }; - this.hostsByLabel = { + this.hostsByService = { columns: [ { - prop: 'label', - name: $localize`Labels`, + prop: 'service_type', + name: $localize`Services`, flexGrow: 1, cellTransformation: CellTemplate.badge, customTemplateConfig: { @@ -58,36 +73,55 @@ export class CreateClusterReviewComponent implements OnInit { }, { name: $localize`Number of Hosts`, - prop: 'hosts_per_label', + prop: 'hosts_per_service', flexGrow: 1 } ] }; - this.hostService.list().subscribe((resp: object[]) => { - this.hosts = resp; - this.hostsCount = this.hosts.length; + this.cephServiceService.list().subscribe((resp: Array) => { + this.services = resp; + this.serviceCount = this.services.length; - _.forEach(this.hosts, (hostKey) => { - const labels = hostKey['labels']; - _.forEach(labels, (label) => { - this.labelOccurrences[label] = (this.labelOccurrences[label] || 0) + 1; - this.uniqueLabels.add(label); - }); + _.forEach(this.services, (serviceKey) => { + this.serviceOccurrences[serviceKey['service_type']] = + (this.serviceOccurrences[serviceKey['service_type']] || 0) + 1; + this.uniqueServices.add(serviceKey['service_type']); }); - this.uniqueLabels.forEach((label) => { - this.hostsCountPerLabel.push({ - label: label, - hosts_per_label: this.labelOccurrences[label] + this.uniqueServices.forEach((serviceType) => { + this.hostsCountPerService.push({ + service_type: serviceType, + hosts_per_service: this.serviceOccurrences[serviceType] }); }); - this.hostsByLabel['data'] = [...this.hostsCountPerLabel]; + this.hostsByService['data'] = [...this.hostsCountPerService]; + }); + + this.hostService.list().subscribe((resp: object[]) => { + this.hosts = resp; + this.hostsCount = this.hosts.length; this.hostsDetails['data'] = [...this.hosts]; }); - this.filteredDevices = this.wizardStepService.osdDevices; - this.capacity = this.wizardStepService.osdCapacity; + if (this.osdService.osdDevices['data']) { + dataDevices = this.osdService.osdDevices['data']?.length; + dataDeviceCapacity = this.osdService.osdDevices['data']['capacity']; + } + + if (this.osdService.osdDevices['wal']) { + walDevices = this.osdService.osdDevices['wal']?.length; + walDeviceCapacity = this.osdService.osdDevices['wal']['capacity']; + } + + if (this.osdService.osdDevices['db']) { + dbDevices = this.osdService.osdDevices['db']?.length; + dbDeviceCapacity = this.osdService.osdDevices['db']['capacity']; + } + + this.totalDevices = dataDevices + walDevices + dbDevices; + this.osdService.osdDevices['totalDevices'] = this.totalDevices; + this.totalCapacity = dataDeviceCapacity + walDeviceCapacity + dbDeviceCapacity; } } diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster.component.html index d9e8ec43d9cc9..9ae80f4c0218d 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster.component.html +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster.component.html @@ -40,7 +40,11 @@

    Add Hosts


    - +
    @@ -48,11 +52,23 @@ i18n>Create OSDs
    - +
    +

    Create Services

    +
    + +
    +
    diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster.component.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster.component.spec.ts index 8f9d7328e34ee..3e0b7f7bdcce7 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster.component.spec.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster.component.spec.ts @@ -8,6 +8,7 @@ import { ToastrModule } from 'ngx-toastr'; import { CephModule } from '~/app/ceph/ceph.module'; import { CoreModule } from '~/app/core/core.module'; import { HostService } from '~/app/shared/api/host.service'; +import { OsdService } from '~/app/shared/api/osd.service'; import { ConfirmationModalComponent } from '~/app/shared/components/confirmation-modal/confirmation-modal.component'; import { LoadingPanelComponent } from '~/app/shared/components/loading-panel/loading-panel.component'; import { AppConstants } from '~/app/shared/constants/app.constants'; @@ -22,6 +23,7 @@ describe('CreateClusterComponent', () => { let fixture: ComponentFixture; let wizardStepService: WizardStepsService; let hostService: HostService; + let osdService: OsdService; let modalServiceShowSpy: jasmine.Spy; const projectConstants: typeof AppConstants = AppConstants; @@ -44,6 +46,7 @@ describe('CreateClusterComponent', () => { component = fixture.componentInstance; wizardStepService = TestBed.inject(WizardStepsService); hostService = TestBed.inject(HostService); + osdService = TestBed.inject(OsdService); modalServiceShowSpy = spyOn(TestBed.inject(ModalService), 'show').and.returnValue({ // mock the close function, it might be called if there are async tests. close: jest.fn() @@ -89,13 +92,11 @@ describe('CreateClusterComponent', () => { it('should move to next step and show the second page', () => { const wizardStepServiceSpy = spyOn(wizardStepService, 'moveToNextStep').and.callThrough(); - const hostServiceSpy = spyOn(hostService, 'list').and.callThrough(); component.createCluster(); fixture.detectChanges(); component.onNextStep(); fixture.detectChanges(); expect(wizardStepServiceSpy).toHaveBeenCalledTimes(1); - expect(hostServiceSpy).toBeCalledTimes(1); }); it('should show the button labels correctly', () => { @@ -113,6 +114,13 @@ describe('CreateClusterComponent', () => { cancelBtnLabel = component.showCancelButtonLabel(); expect(cancelBtnLabel).toEqual('Back'); + component.onNextStep(); + fixture.detectChanges(); + submitBtnLabel = component.showSubmitButtonLabel(); + expect(submitBtnLabel).toEqual('Next'); + cancelBtnLabel = component.showCancelButtonLabel(); + expect(cancelBtnLabel).toEqual('Back'); + // Last page of the wizard component.onNextStep(); fixture.detectChanges(); @@ -121,4 +129,25 @@ describe('CreateClusterComponent', () => { cancelBtnLabel = component.showCancelButtonLabel(); expect(cancelBtnLabel).toEqual('Back'); }); + + it('should ensure osd creation did not happen when no devices are selected', () => { + const osdServiceSpy = spyOn(osdService, 'create').and.callThrough(); + component.onSubmit(); + fixture.detectChanges(); + expect(osdServiceSpy).toBeCalledTimes(0); + }); + + it('should ensure osd creation did happen when devices are selected', () => { + const osdServiceSpy = spyOn(osdService, 'create').and.callThrough(); + osdService.osdDevices['totalDevices'] = 1; + component.onSubmit(); + fixture.detectChanges(); + expect(osdServiceSpy).toBeCalledTimes(1); + }); + + it('should ensure host list call happened', () => { + const hostServiceSpy = spyOn(hostService, 'list').and.callThrough(); + component.onSubmit(); + expect(hostServiceSpy).toHaveBeenCalledTimes(1); + }); }); diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster.component.ts index 6d78a2110b461..7d973119dfea2 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster.component.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster.component.ts @@ -34,7 +34,7 @@ export class CreateClusterComponent implements OnDestroy { currentStepSub: Subscription; permissions: Permissions; projectConstants: typeof AppConstants = AppConstants; - stepTitles = ['Add Hosts', 'Create OSDs', 'Review']; + stepTitles = ['Add Hosts', 'Create OSDs', 'Create Services', 'Review']; startClusterCreation = false; observables: any = []; modalRef: NgbModalRef; @@ -46,7 +46,7 @@ export class CreateClusterComponent implements OnDestroy { constructor( private authStorageService: AuthStorageService, - private stepsService: WizardStepsService, + private wizardStepsService: WizardStepsService, private router: Router, private hostService: HostService, private notificationService: NotificationService, @@ -54,13 +54,14 @@ export class CreateClusterComponent implements OnDestroy { private clusterService: ClusterService, private modalService: ModalService, private taskWrapper: TaskWrapperService, - private osdService: OsdService, - private wizardStepService: WizardStepsService + private osdService: OsdService ) { this.permissions = this.authStorageService.getPermissions(); - this.currentStepSub = this.stepsService.getCurrentStep().subscribe((step: WizardStepModel) => { - this.currentStep = step; - }); + this.currentStepSub = this.wizardStepsService + .getCurrentStep() + .subscribe((step: WizardStepModel) => { + this.currentStep = step; + }); this.currentStep.stepIndex = 1; } @@ -93,77 +94,87 @@ export class CreateClusterComponent implements OnDestroy { } onSubmit() { - forkJoin(this.observables) - .pipe( - finalize(() => - this.clusterService.updateStatus('POST_INSTALLED').subscribe(() => { - this.notificationService.show( - NotificationType.success, - $localize`Cluster expansion was successful` - ); - this.router.navigate(['/dashboard']); - }) - ) - ) - .subscribe({ - error: (error) => error.preventDefault() - }); - - this.taskWrapper - .wrapTaskAroundCall({ - task: new FinishedTask('osd/' + URLVerbs.CREATE, { - tracking_id: _.join(_.map(this.driveGroups, 'service_id'), ', ') - }), - call: this.osdService.create(this.driveGroups) - }) - .subscribe({ - error: (error) => error.preventDefault(), - complete: () => { - this.submitAction.emit(); + this.hostService.list().subscribe((hosts) => { + hosts.forEach((host) => { + const index = host['labels'].indexOf('_no_schedule', 0); + if (index > -1) { + host['labels'].splice(index, 1); + this.observables.push(this.hostService.update(host['hostname'], true, host['labels'])); } }); - } + forkJoin(this.observables) + .pipe( + finalize(() => + this.clusterService.updateStatus('POST_INSTALLED').subscribe(() => { + this.notificationService.show( + NotificationType.success, + $localize`Cluster expansion was successful` + ); + this.router.navigate(['/dashboard']); + }) + ) + ) + .subscribe({ + error: (error) => error.preventDefault() + }); + }); + if (this.driveGroup) { + const user = this.authStorageService.getUsername(); + this.driveGroup.setName(`dashboard-${user}-${_.now()}`); + this.driveGroups.push(this.driveGroup.spec); + } - onNextStep() { - if (!this.stepsService.isLastStep()) { - this.hostService.list().subscribe((hosts) => { - hosts.forEach((host) => { - const index = host['labels'].indexOf('_no_schedule', 0); - if (index > -1) { - host['labels'].splice(index, 1); - this.observables.push(this.hostService.update(host['hostname'], true, host['labels'])); + if (this.osdService.osdDevices['totalDevices'] > 0) { + this.taskWrapper + .wrapTaskAroundCall({ + task: new FinishedTask('osd/' + URLVerbs.CREATE, { + tracking_id: _.join(_.map(this.driveGroups, 'service_id'), ', ') + }), + call: this.osdService.create(this.driveGroups) + }) + .subscribe({ + error: (error) => error.preventDefault(), + complete: () => { + this.submitAction.emit(); + this.osdService.osdDevices = []; } }); - }); - this.driveGroup = this.wizardStepService.sharedData; - this.stepsService.getCurrentStep().subscribe((step: WizardStepModel) => { + } + } + + getDriveGroup(driveGroup: DriveGroup) { + this.driveGroup = driveGroup; + } + + onNextStep() { + if (!this.wizardStepsService.isLastStep()) { + this.wizardStepsService.getCurrentStep().subscribe((step: WizardStepModel) => { this.currentStep = step; }); - if (this.currentStep.stepIndex === 2 && this.driveGroup) { - const user = this.authStorageService.getUsername(); - this.driveGroup.setName(`dashboard-${user}-${_.now()}`); - this.driveGroups.push(this.driveGroup.spec); - } - this.stepsService.moveToNextStep(); + this.wizardStepsService.moveToNextStep(); } else { this.onSubmit(); } } onPreviousStep() { - if (!this.stepsService.isFirstStep()) { - this.stepsService.moveToPreviousStep(); + if (!this.wizardStepsService.isFirstStep()) { + this.wizardStepsService.moveToPreviousStep(); } else { this.router.navigate(['/dashboard']); } } showSubmitButtonLabel() { - return !this.stepsService.isLastStep() ? this.actionLabels.NEXT : $localize`Expand Cluster`; + return !this.wizardStepsService.isLastStep() + ? this.actionLabels.NEXT + : $localize`Expand Cluster`; } showCancelButtonLabel() { - return !this.stepsService.isFirstStep() ? this.actionLabels.BACK : this.actionLabels.CANCEL; + return !this.wizardStepsService.isFirstStep() + ? this.actionLabels.BACK + : this.actionLabels.CANCEL; } ngOnDestroy(): void { diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/hosts.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/hosts.component.html index 59bab46d72d23..cb0ebca6bcd6d 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/hosts.component.html +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/hosts.component.html @@ -11,7 +11,7 @@ columnMode="flex" (fetchData)="getHosts($event)" selectionType="single" - [hasDetails]="!clusterCreation" + [hasDetails]="hasTableDetails" (setExpandedRow)="setExpandedRow($event)" (updateSelection)="updateSelection($event)">
    @@ -30,7 +30,7 @@
  • + *ngIf="permissions.grafana.read"> Overall Performance diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/hosts.component.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/hosts.component.spec.ts index 049aceba0994a..288f62e7a3d29 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/hosts.component.spec.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/hosts.component.spec.ts @@ -72,7 +72,6 @@ describe('HostsComponent', () => { showForceMaintenanceModal = new MockShowForceMaintenanceModal(); fixture = TestBed.createComponent(HostsComponent); component = fixture.componentInstance; - component.clusterCreation = false; hostListSpy = spyOn(TestBed.inject(HostService), 'list'); orchService = TestBed.inject(OrchestratorService); }); @@ -185,7 +184,7 @@ describe('HostsComponent', () => { expectResults: { Add: { disabled: false, disableDesc: '' }, Edit: { disabled: true, disableDesc: '' }, - Delete: { disabled: true, disableDesc: '' } + Remove: { disabled: true, disableDesc: '' } } }, { @@ -193,7 +192,7 @@ describe('HostsComponent', () => { expectResults: { Add: { disabled: false, disableDesc: '' }, Edit: { disabled: true, disableDesc: component.messages.nonOrchHost }, - Delete: { disabled: true, disableDesc: component.messages.nonOrchHost } + Remove: { disabled: true, disableDesc: component.messages.nonOrchHost } } }, { @@ -201,15 +200,15 @@ describe('HostsComponent', () => { expectResults: { Add: { disabled: false, disableDesc: '' }, Edit: { disabled: false, disableDesc: '' }, - Delete: { disabled: false, disableDesc: '' } + Remove: { disabled: false, disableDesc: '' } } } ]; const features = [ - OrchestratorFeature.HOST_CREATE, + OrchestratorFeature.HOST_ADD, OrchestratorFeature.HOST_LABEL_ADD, - OrchestratorFeature.HOST_DELETE, + OrchestratorFeature.HOST_REMOVE, OrchestratorFeature.HOST_LABEL_REMOVE ]; await testTableActions(true, features, tests); @@ -225,7 +224,7 @@ describe('HostsComponent', () => { expectResults: { Add: resultNoOrchestrator, Edit: { disabled: true, disableDesc: '' }, - Delete: { disabled: true, disableDesc: '' } + Remove: { disabled: true, disableDesc: '' } } }, { @@ -233,7 +232,7 @@ describe('HostsComponent', () => { expectResults: { Add: resultNoOrchestrator, Edit: { disabled: true, disableDesc: component.messages.nonOrchHost }, - Delete: { disabled: true, disableDesc: component.messages.nonOrchHost } + Remove: { disabled: true, disableDesc: component.messages.nonOrchHost } } }, { @@ -241,7 +240,7 @@ describe('HostsComponent', () => { expectResults: { Add: resultNoOrchestrator, Edit: resultNoOrchestrator, - Delete: resultNoOrchestrator + Remove: resultNoOrchestrator } } ]; @@ -258,7 +257,7 @@ describe('HostsComponent', () => { expectResults: { Add: resultMissingFeatures, Edit: { disabled: true, disableDesc: '' }, - Delete: { disabled: true, disableDesc: '' } + Remove: { disabled: true, disableDesc: '' } } }, { @@ -266,7 +265,7 @@ describe('HostsComponent', () => { expectResults: { Add: resultMissingFeatures, Edit: { disabled: true, disableDesc: component.messages.nonOrchHost }, - Delete: { disabled: true, disableDesc: component.messages.nonOrchHost } + Remove: { disabled: true, disableDesc: component.messages.nonOrchHost } } }, { @@ -274,7 +273,7 @@ describe('HostsComponent', () => { expectResults: { Add: resultMissingFeatures, Edit: resultMissingFeatures, - Delete: resultMissingFeatures + Remove: resultMissingFeatures } } ]; diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/hosts.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/hosts.component.ts index c70f755f799e8..6bd8dbbdc2520 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/hosts.component.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/hosts.component.ts @@ -47,8 +47,21 @@ export class HostsComponent extends ListWithDetails implements OnInit { public servicesTpl: TemplateRef; @ViewChild('maintenanceConfirmTpl', { static: true }) maintenanceConfirmTpl: TemplateRef; + + @Input() + hiddenColumns: string[] = []; + + @Input() + hideTitle = false; + + @Input() + hideSubmitBtn = false; + + @Input() + hasTableDetails = true; + @Input() - clusterCreation = false; + showGeneralActionsOnly = false; permissions: Permissions; columns: Array = []; @@ -61,7 +74,6 @@ export class HostsComponent extends ListWithDetails implements OnInit { isExecuting = false; errorMessage: string; enableButton: boolean; - pageURL: string; bsModalRef: NgbModalRef; icons = Icons; @@ -72,9 +84,9 @@ export class HostsComponent extends ListWithDetails implements OnInit { orchStatus: OrchestratorStatus; actionOrchFeatures = { - add: [OrchestratorFeature.HOST_CREATE], + add: [OrchestratorFeature.HOST_ADD], edit: [OrchestratorFeature.HOST_LABEL_ADD, OrchestratorFeature.HOST_LABEL_REMOVE], - delete: [OrchestratorFeature.HOST_DELETE], + remove: [OrchestratorFeature.HOST_REMOVE], maintenance: [ OrchestratorFeature.HOST_MAINTENANCE_ENTER, OrchestratorFeature.HOST_MAINTENANCE_EXIT @@ -95,6 +107,16 @@ export class HostsComponent extends ListWithDetails implements OnInit { super(); this.permissions = this.authStorageService.getPermissions(); this.tableActions = [ + { + name: this.actionLabels.ADD, + permission: 'create', + icon: Icons.add, + click: () => + this.router.url.includes('/hosts') + ? this.router.navigate([BASE_URL, { outlets: { modal: [URLVerbs.ADD] } }]) + : (this.bsModalRef = this.modalService.show(HostFormComponent)), + disable: (selection: CdTableSelection) => this.getDisable('add', selection) + }, { name: this.actionLabels.EDIT, permission: 'update', @@ -103,11 +125,11 @@ export class HostsComponent extends ListWithDetails implements OnInit { disable: (selection: CdTableSelection) => this.getDisable('edit', selection) }, { - name: this.actionLabels.DELETE, + name: this.actionLabels.REMOVE, permission: 'delete', icon: Icons.destroy, click: () => this.deleteAction(), - disable: (selection: CdTableSelection) => this.getDisable('delete', selection) + disable: (selection: CdTableSelection) => this.getDisable('remove', selection) }, { name: this.actionLabels.ENTER_MAINTENANCE, @@ -116,7 +138,7 @@ export class HostsComponent extends ListWithDetails implements OnInit { click: () => this.hostMaintenance(), disable: (selection: CdTableSelection) => this.getDisable('maintenance', selection) || this.isExecuting || this.enableButton, - visible: () => !this.clusterCreation + visible: () => !this.showGeneralActionsOnly }, { name: this.actionLabels.EXIT_MAINTENANCE, @@ -124,23 +146,13 @@ export class HostsComponent extends ListWithDetails implements OnInit { icon: Icons.exit, click: () => this.hostMaintenance(), disable: (selection: CdTableSelection) => - this.getDisable('maintenance', selection) || this.isExecuting || this.enableButton, - visible: () => !this.clusterCreation + this.getDisable('maintenance', selection) || this.isExecuting || !this.enableButton, + visible: () => !this.showGeneralActionsOnly } ]; } ngOnInit() { - this.tableActions.unshift({ - name: this.actionLabels.ADD, - permission: 'create', - icon: Icons.add, - click: () => - this.clusterCreation - ? (this.bsModalRef = this.modalService.show(HostFormComponent)) - : this.router.navigate([BASE_URL, { outlets: { modal: [URLVerbs.ADD] } }]), - disable: (selection: CdTableSelection) => this.getDisable('add', selection) - }); this.columns = [ { name: $localize`Hostname`, @@ -150,7 +162,6 @@ export class HostsComponent extends ListWithDetails implements OnInit { { name: $localize`Services`, prop: 'services', - isHidden: this.clusterCreation, flexGrow: 3, cellTemplate: this.servicesTpl }, @@ -177,7 +188,6 @@ export class HostsComponent extends ListWithDetails implements OnInit { { name: $localize`Version`, prop: 'ceph_version', - isHidden: this.clusterCreation, flexGrow: 1, pipe: this.cephShortVersionPipe } @@ -186,12 +196,9 @@ export class HostsComponent extends ListWithDetails implements OnInit { this.orchStatus = status; }); - if (this.clusterCreation) { - const hiddenColumns = ['services', 'ceph_version']; - this.columns = this.columns.filter((col: any) => { - return !hiddenColumns.includes(col.prop); - }); - } + this.columns = this.columns.filter((col: any) => { + return !this.hiddenColumns.includes(col.prop); + }); } updateSelection(selection: CdTableSelection) { @@ -305,10 +312,10 @@ export class HostsComponent extends ListWithDetails implements OnInit { } getDisable( - action: 'add' | 'edit' | 'delete' | 'maintenance', + action: 'add' | 'edit' | 'remove' | 'maintenance', selection: CdTableSelection ): boolean | string { - if (action === 'delete' || action === 'edit' || action === 'maintenance') { + if (action === 'remove' || action === 'edit' || action === 'maintenance') { if (!selection?.hasSingleSelection) { return true; } @@ -327,10 +334,10 @@ export class HostsComponent extends ListWithDetails implements OnInit { this.modalRef = this.modalService.show(CriticalConfirmationModalComponent, { itemDescription: 'Host', itemNames: [hostname], - actionDescription: 'delete', + actionDescription: 'remove', submitActionObservable: () => this.taskWrapper.wrapTaskAroundCall({ - task: new FinishedTask('host/delete', { hostname: hostname }), + task: new FinishedTask('host/remove', { hostname: hostname }), call: this.hostService.delete(hostname) }) }); diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/osd/osd-devices-selection-groups/osd-devices-selection-groups.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/osd/osd-devices-selection-groups/osd-devices-selection-groups.component.html index bfbf3dcf143b4..8b1f59eac371c 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/osd/osd-devices-selection-groups/osd-devices-selection-groups.component.html +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/osd/osd-devices-selection-groups/osd-devices-selection-groups.component.html @@ -19,7 +19,7 @@ (click)="showSelectionModal()" data-toggle="tooltip" [title]="addButtonTooltip" - [disabled]="availDevices.length === 0 || !canSelect"> + [disabled]="availDevices.length === 0 || !canSelect || expansionCanSelect"> Add diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/osd/osd-devices-selection-groups/osd-devices-selection-groups.component.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/osd/osd-devices-selection-groups/osd-devices-selection-groups.component.spec.ts index 0d9250000616c..dea6746cf9512 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/osd/osd-devices-selection-groups/osd-devices-selection-groups.component.spec.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/osd/osd-devices-selection-groups/osd-devices-selection-groups.component.spec.ts @@ -2,6 +2,7 @@ import { HttpClientTestingModule } from '@angular/common/http/testing'; import { ComponentFixture, TestBed } from '@angular/core/testing'; import { FormsModule } from '@angular/forms'; import { BrowserAnimationsModule } from '@angular/platform-browser/animations'; +import { RouterTestingModule } from '@angular/router/testing'; import { ToastrModule } from 'ngx-toastr'; @@ -34,7 +35,8 @@ describe('OsdDevicesSelectionGroupsComponent', () => { FormsModule, HttpClientTestingModule, SharedModule, - ToastrModule.forRoot() + ToastrModule.forRoot(), + RouterTestingModule ], declarations: [OsdDevicesSelectionGroupsComponent, InventoryDevicesComponent] }); @@ -92,8 +94,10 @@ describe('OsdDevicesSelectionGroupsComponent', () => { describe('with devices selected', () => { beforeEach(() => { + component.isOsdPage = true; component.availDevices = []; component.devices = devices; + component.ngOnInit(); fixture.detectChanges(); }); diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/osd/osd-devices-selection-groups/osd-devices-selection-groups.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/osd/osd-devices-selection-groups/osd-devices-selection-groups.component.ts index bbeb8b0f0cee3..cff0cbc0563fc 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/osd/osd-devices-selection-groups/osd-devices-selection-groups.component.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/osd/osd-devices-selection-groups/osd-devices-selection-groups.component.ts @@ -1,8 +1,10 @@ import { Component, EventEmitter, Input, OnChanges, OnInit, Output } from '@angular/core'; +import { Router } from '@angular/router'; import _ from 'lodash'; import { InventoryDevice } from '~/app/ceph/cluster/inventory/inventory-devices/inventory-device.model'; +import { OsdService } from '~/app/shared/api/osd.service'; import { Icons } from '~/app/shared/enum/icons.enum'; import { CdTableColumnFiltersChange } from '~/app/shared/models/cd-table-column-filters-change'; import { ModalService } from '~/app/shared/services/modal.service'; @@ -37,7 +39,9 @@ export class OsdDevicesSelectionGroupsComponent implements OnInit, OnChanges { icons = Icons; devices: InventoryDevice[] = []; capacity = 0; - appliedFilters: any[] = []; + appliedFilters = new Array(); + expansionCanSelect = false; + isOsdPage: boolean; addButtonTooltip: String; tooltips = { @@ -46,9 +50,24 @@ export class OsdDevicesSelectionGroupsComponent implements OnInit, OnChanges { addByFilters: $localize`Add devices by using filters` }; - constructor(private modalService: ModalService) {} + constructor( + private modalService: ModalService, + public osdService: OsdService, + private router: Router + ) { + this.isOsdPage = this.router.url.includes('/osd'); + } ngOnInit() { + if (!this.isOsdPage) { + this.osdService?.osdDevices[this.type] + ? (this.devices = this.osdService.osdDevices[this.type]) + : (this.devices = []); + this.capacity = _.sumBy(this.devices, 'sys_api.size'); + this.osdService?.osdDevices + ? (this.expansionCanSelect = this.osdService?.osdDevices['disableSelect']) + : (this.expansionCanSelect = false); + } this.updateAddButtonTooltip(); } @@ -75,6 +94,12 @@ export class OsdDevicesSelectionGroupsComponent implements OnInit, OnChanges { this.capacity = _.sumBy(this.devices, 'sys_api.size'); this.appliedFilters = result.filters; const event = _.assign({ type: this.type }, result); + if (!this.isOsdPage) { + this.osdService.osdDevices[this.type] = this.devices; + this.osdService.osdDevices['disableSelect'] = + this.canSelect || this.devices.length === this.availDevices.length; + this.osdService.osdDevices[this.type]['capacity'] = this.capacity; + } this.selected.emit(event); }); } @@ -95,6 +120,11 @@ export class OsdDevicesSelectionGroupsComponent implements OnInit, OnChanges { } clearDevices() { + if (!this.isOsdPage) { + this.expansionCanSelect = false; + this.osdService.osdDevices['disableSelect'] = false; + this.osdService.osdDevices = []; + } const event = { type: this.type, clearedDevices: [...this.devices] diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/osd/osd-devices-selection-modal/osd-devices-selection-modal.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/osd/osd-devices-selection-modal/osd-devices-selection-modal.component.ts index 1909803dc3380..edfe9d6a7c3e2 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/osd/osd-devices-selection-modal/osd-devices-selection-modal.component.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/osd/osd-devices-selection-modal/osd-devices-selection-modal.component.ts @@ -82,8 +82,6 @@ export class OsdDevicesSelectionModalComponent implements AfterViewInit { this.filteredDevices = event.data; this.capacity = _.sumBy(this.filteredDevices, 'sys_api.size'); this.event = event; - this.wizardStepService.osdDevices = this.filteredDevices; - this.wizardStepService.osdCapacity = this.capacity; } } diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/osd/osd-form/osd-form.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/osd/osd-form/osd-form.component.html index 675e20fcf59a6..59a17362f6f83 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/osd/osd-form/osd-form.component.html +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/osd/osd-form/osd-form.component.html @@ -9,7 +9,7 @@
    {{ action | titlecase }} {{ resource | upperFirst }}
    + *ngIf="!hideTitle">{{ action | titlecase }} {{ resource | upperFirst }}
    + *ngIf="!hideSubmitBtn"> = new EventEmitter(); + icons = Icons; form: CdFormGroup; @@ -62,8 +70,6 @@ export class OsdFormComponent extends CdForm implements OnInit { featureList: OsdFeature[] = []; hasOrchestrator = true; - @Input() - clusterCreation = false; constructor( public actionLabels: ActionLabelsI18n, @@ -186,9 +192,8 @@ export class OsdFormComponent extends CdForm implements OnInit { this.enableFeatures(); } this.driveGroup.setDeviceSelection(event.type, event.filters); - if (this.clusterCreation) { - this.wizardStepService.sharedData = this.driveGroup; - } + + this.emitDriveGroup.emit(this.driveGroup); } onDevicesCleared(event: DevicesSelectionClearEvent) { diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/services/service-form/service-form.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/services/service-form/service-form.component.html index 3bda005110391..e8e340a7dc8dc 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/services/service-form/service-form.component.html +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/services/service-form/service-form.component.html @@ -1,12 +1,13 @@ -
    -
    -
    -
    {{ action | titlecase }} {{ resource | upperFirst }}
    + + {{ action | titlecase }} {{ resource | upperFirst }} + + +
    -
    -
    +
    -
    - -
    + + + diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/services/service-form/service-form.component.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/services/service-form/service-form.component.spec.ts index 11c0663efd246..78863435ea328 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/services/service-form/service-form.component.spec.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/services/service-form/service-form.component.spec.ts @@ -4,7 +4,7 @@ import { ReactiveFormsModule } from '@angular/forms'; import { By } from '@angular/platform-browser'; import { RouterTestingModule } from '@angular/router/testing'; -import { NgbTypeaheadModule } from '@ng-bootstrap/ng-bootstrap'; +import { NgbActiveModal, NgbTypeaheadModule } from '@ng-bootstrap/ng-bootstrap'; import _ from 'lodash'; import { ToastrModule } from 'ngx-toastr'; @@ -23,6 +23,7 @@ describe('ServiceFormComponent', () => { configureTestBed({ declarations: [ServiceFormComponent], + providers: [NgbActiveModal], imports: [ HttpClientTestingModule, NgbTypeaheadModule, @@ -36,6 +37,7 @@ describe('ServiceFormComponent', () => { beforeEach(() => { fixture = TestBed.createComponent(ServiceFormComponent); component = fixture.componentInstance; + component.ngOnInit(); form = component.serviceForm; formHelper = new FormHelper(form); fixture.detectChanges(); diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/services/service-form/service-form.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/services/service-form/service-form.component.ts index cb80a2571c0ae..941eabfb67c62 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/services/service-form/service-form.component.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/services/service-form/service-form.component.ts @@ -1,8 +1,8 @@ -import { Component, OnInit, ViewChild } from '@angular/core'; +import { Component, Input, OnInit, ViewChild } from '@angular/core'; import { AbstractControl, Validators } from '@angular/forms'; import { Router } from '@angular/router'; -import { NgbTypeahead } from '@ng-bootstrap/ng-bootstrap'; +import { NgbActiveModal, NgbTypeahead } from '@ng-bootstrap/ng-bootstrap'; import _ from 'lodash'; import { merge, Observable, Subject } from 'rxjs'; import { debounceTime, distinctUntilChanged, filter, map } from 'rxjs/operators'; @@ -31,6 +31,8 @@ export class ServiceFormComponent extends CdForm implements OnInit { @ViewChild(NgbTypeahead, { static: false }) typeahead: NgbTypeahead; + @Input() public hiddenServices: string[] = []; + serviceForm: CdFormGroup; action: string; resource: string; @@ -41,6 +43,7 @@ export class ServiceFormComponent extends CdForm implements OnInit { labelFocus = new Subject(); pools: Array; services: Array = []; + pageURL: string; constructor( public actionLabels: ActionLabelsI18n, @@ -49,7 +52,8 @@ export class ServiceFormComponent extends CdForm implements OnInit { private hostService: HostService, private poolService: PoolService, private router: Router, - private taskWrapperService: TaskWrapperService + private taskWrapperService: TaskWrapperService, + public activeModal: NgbActiveModal ) { super(); this.resource = $localize`service`; @@ -215,12 +219,17 @@ export class ServiceFormComponent extends CdForm implements OnInit { } ngOnInit(): void { + if (this.router.url.includes('services')) { + this.pageURL = 'services'; + } this.action = this.actionLabels.CREATE; this.cephServiceService.getKnownTypes().subscribe((resp: Array) => { // Remove service types: // osd - This is deployed a different way. // container - This should only be used in the CLI. - this.serviceTypes = _.difference(resp, ['container', 'osd']).sort(); + this.hiddenServices.push('osd', 'container'); + + this.serviceTypes = _.difference(resp, this.hiddenServices).sort(); }); this.hostService.list().subscribe((resp: object[]) => { const options: SelectOption[] = []; @@ -243,10 +252,6 @@ export class ServiceFormComponent extends CdForm implements OnInit { }); } - goToListView() { - this.router.navigate(['/services']); - } - searchLabels = (text$: Observable) => { return merge( text$.pipe(debounceTime(200), distinctUntilChanged()), @@ -385,8 +390,10 @@ export class ServiceFormComponent extends CdForm implements OnInit { error() { self.serviceForm.setErrors({ cdSubmitButton: true }); }, - complete() { - self.goToListView(); + complete: () => { + this.pageURL === 'services' + ? this.router.navigate([this.pageURL, { outlets: { modal: null } }]) + : this.activeModal.close(); } }); } diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/services/services.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/services/services.component.html index f6f66ee51485f..36ab431fc96b3 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/services/services.component.html +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/services/services.component.html @@ -8,7 +8,7 @@ selectionType="single" [autoReload]="5000" (fetchData)="getServices($event)" - [hasDetails]="true" + [hasDetails]="hasDetails" (setExpandedRow)="setExpandedRow($event)" (updateSelection)="updateSelection($event)"> + diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/services/services.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/services/services.component.ts index 7cba81ee49eab..c2346c2747990 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/services/services.component.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/services/services.component.ts @@ -1,5 +1,7 @@ import { Component, Input, OnChanges, OnInit, ViewChild } from '@angular/core'; +import { Router } from '@angular/router'; +import { NgbModalRef } from '@ng-bootstrap/ng-bootstrap'; import { delay } from 'rxjs/operators'; import { CephServiceService } from '~/app/shared/api/ceph-service.service'; @@ -24,6 +26,7 @@ import { ModalService } from '~/app/shared/services/modal.service'; import { TaskWrapperService } from '~/app/shared/services/task-wrapper.service'; import { URLBuilderService } from '~/app/shared/services/url-builder.service'; import { PlacementPipe } from './placement.pipe'; +import { ServiceFormComponent } from './service-form/service-form.component'; const BASE_URL = 'services'; @@ -42,9 +45,16 @@ export class ServicesComponent extends ListWithDetails implements OnChanges, OnI // Do not display these columns @Input() hiddenColumns: string[] = []; + @Input() hiddenServices: string[] = []; + + @Input() hasDetails = true; + + @Input() modal = true; + permissions: Permissions; tableActions: CdTableAction[]; showDocPanel = false; + bsModalRef: NgbModalRef; orchStatus: OrchestratorStatus; actionOrchFeatures = { @@ -65,7 +75,7 @@ export class ServicesComponent extends ListWithDetails implements OnChanges, OnI private cephServiceService: CephServiceService, private relativeDatePipe: RelativeDatePipe, private taskWrapperService: TaskWrapperService, - private urlBuilder: URLBuilderService + private router: Router ) { super(); this.permissions = this.authStorageService.getPermissions(); @@ -73,7 +83,7 @@ export class ServicesComponent extends ListWithDetails implements OnChanges, OnI { permission: 'create', icon: Icons.add, - routerLink: () => this.urlBuilder.getCreate(), + click: () => this.openModal(), name: this.actionLabels.CREATE, canBePrimary: (selection: CdTableSelection) => !selection.hasSelection, disable: (selection: CdTableSelection) => this.getDisable('create', selection) @@ -88,6 +98,15 @@ export class ServicesComponent extends ListWithDetails implements OnChanges, OnI ]; } + openModal() { + if (this.modal) { + this.router.navigate([BASE_URL, { outlets: { modal: [URLVerbs.CREATE] } }]); + } else { + this.bsModalRef = this.modalService.show(ServiceFormComponent); + this.bsModalRef.componentInstance.hiddenServices = this.hiddenServices; + } + } + ngOnInit() { const columns = [ { @@ -156,6 +175,9 @@ export class ServicesComponent extends ListWithDetails implements OnChanges, OnI this.cephServiceService.list().subscribe( (services: CephServiceSpec[]) => { this.services = services; + this.services = this.services.filter((col: any) => { + return !this.hiddenServices.includes(col.service_name); + }); this.isLoadingServices = false; }, () => { diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/osd.service.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/osd.service.ts index 9bff8083936e1..e33f2c3fc0fa1 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/osd.service.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/osd.service.ts @@ -5,6 +5,7 @@ import _ from 'lodash'; import { map } from 'rxjs/operators'; import { CdDevice } from '../models/devices'; +import { InventoryDeviceType } from '../models/inventory-device-type.model'; import { SmartDataResponseV1 } from '../models/smart'; import { DeviceService } from '../services/device.service'; @@ -13,6 +14,7 @@ import { DeviceService } from '../services/device.service'; }) export class OsdService { private path = 'api/osd'; + osdDevices: InventoryDeviceType[] = []; osdRecvSpeedModalPriorities = { KNOWN_PRIORITIES: [ diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/models/inventory-device-type.model.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/models/inventory-device-type.model.ts new file mode 100644 index 0000000000000..2155c2d874b45 --- /dev/null +++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/models/inventory-device-type.model.ts @@ -0,0 +1,9 @@ +import { InventoryDevice } from '~/app/ceph/cluster/inventory/inventory-devices/inventory-device.model'; + +export interface InventoryDeviceType { + type: string; + capacity: number; + devices: InventoryDevice[]; + canSelect: boolean; + totalDevices: number; +} diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/models/orchestrator.enum.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/models/orchestrator.enum.ts index 8c740f4e01756..cf4ba76a9cf09 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/shared/models/orchestrator.enum.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/models/orchestrator.enum.ts @@ -1,7 +1,7 @@ export enum OrchestratorFeature { HOST_LIST = 'get_hosts', - HOST_CREATE = 'add_host', - HOST_DELETE = 'remove_host', + HOST_ADD = 'add_host', + HOST_REMOVE = 'remove_host', HOST_LABEL_ADD = 'add_host_label', HOST_LABEL_REMOVE = 'remove_host_label', HOST_MAINTENANCE_ENTER = 'enter_host_maintenance', diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/services/task-message.service.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/services/task-message.service.ts index 44eb9bd30978a..8705f29991665 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/shared/services/task-message.service.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/services/task-message.service.ts @@ -115,7 +115,7 @@ export class TaskMessageService { messages = { // Host tasks 'host/add': this.newTaskMessage(this.commonOperations.add, (metadata) => this.host(metadata)), - 'host/delete': this.newTaskMessage(this.commonOperations.delete, (metadata) => + 'host/remove': this.newTaskMessage(this.commonOperations.remove, (metadata) => this.host(metadata) ), 'host/identify_device': this.newTaskMessage( diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/services/wizard-steps.service.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/services/wizard-steps.service.ts index 1c8114033c15e..e0fb2be944de0 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/shared/services/wizard-steps.service.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/services/wizard-steps.service.ts @@ -2,8 +2,6 @@ import { Injectable } from '@angular/core'; import { BehaviorSubject, Observable } from 'rxjs'; -import { InventoryDevice } from '~/app/ceph/cluster/inventory/inventory-devices/inventory-device.model'; -import { DriveGroup } from '~/app/ceph/cluster/osd/osd-form/drive-group.model'; import { WizardStepModel } from '~/app/shared/models/wizard-steps'; const initialStep = [{ stepIndex: 1, isComplete: false }]; @@ -14,9 +12,6 @@ const initialStep = [{ stepIndex: 1, isComplete: false }]; export class WizardStepsService { steps$: BehaviorSubject; currentStep$: BehaviorSubject = new BehaviorSubject(null); - sharedData = new DriveGroup(); - osdDevices: InventoryDevice[] = []; - osdCapacity = 0; constructor() { this.steps$ = new BehaviorSubject(initialStep); diff --git a/src/pybind/mgr/dashboard/services/orchestrator.py b/src/pybind/mgr/dashboard/services/orchestrator.py index 88a6cb3a86085..671ebdb751a39 100644 --- a/src/pybind/mgr/dashboard/services/orchestrator.py +++ b/src/pybind/mgr/dashboard/services/orchestrator.py @@ -192,8 +192,8 @@ def blink_device_light(self, hostname, device, ident_fault, on): class OrchFeature(object): HOST_LIST = 'get_hosts' - HOST_CREATE = 'add_host' - HOST_DELETE = 'remove_host' + HOST_ADD = 'add_host' + HOST_REMOVE = 'remove_host' HOST_LABEL_ADD = 'add_host_label' HOST_LABEL_REMOVE = 'remove_host_label' HOST_MAINTENANCE_ENTER = 'enter_host_maintenance' From 6644a00a2c6d6a7fa709ee82f5ada0d73a0929a8 Mon Sep 17 00:00:00 2001 From: Avan Thakkar Date: Tue, 3 Aug 2021 14:31:57 +0530 Subject: [PATCH 0128/2866] mgr/dashboard: introduce gather facts in host list Fixes: https://tracker.ceph.com/issues/52017 Signed-off-by: Avan Thakkar --- qa/tasks/mgr/dashboard/test_auth.py | 36 ++--- qa/tasks/mgr/dashboard/test_host.py | 14 +- .../mgr/dashboard/controllers/cluster.py | 11 +- src/pybind/mgr/dashboard/controllers/host.py | 35 +++-- .../05-create-cluster-review.e2e-spec.ts | 26 +++- .../cypress/integration/page-helper.po.ts | 2 +- .../create-cluster-review.component.html | 26 +++- .../create-cluster-review.component.scss | 5 + .../create-cluster-review.component.spec.ts | 3 +- .../create-cluster-review.component.ts | 33 ++--- .../create-cluster.component.html | 2 +- .../create-cluster.component.ts | 2 +- .../hosts/host-form/host-form.component.ts | 2 +- .../ceph/cluster/hosts/hosts.component.html | 12 ++ .../cluster/hosts/hosts.component.spec.ts | 75 ++++++++-- .../app/ceph/cluster/hosts/hosts.component.ts | 135 ++++++++++++++---- .../service-form/service-form.component.ts | 2 +- .../src/app/shared/api/host.service.spec.ts | 4 +- .../src/app/shared/api/host.service.ts | 7 +- src/pybind/mgr/dashboard/openapi.yaml | 8 +- src/pybind/mgr/dashboard/tests/test_host.py | 104 ++++++++++++-- src/pybind/mgr/dashboard/tests/test_tools.py | 13 +- src/pybind/mgr/dashboard/tools.py | 10 ++ 23 files changed, 428 insertions(+), 139 deletions(-) diff --git a/qa/tasks/mgr/dashboard/test_auth.py b/qa/tasks/mgr/dashboard/test_auth.py index 98566344444f7..a2266229bef7f 100644 --- a/qa/tasks/mgr/dashboard/test_auth.py +++ b/qa/tasks/mgr/dashboard/test_auth.py @@ -154,7 +154,7 @@ def test_logout(self): self.assertJsonBody({ "redirect_url": "#/login" }) - self._get("/api/host") + self._get("/api/host", version='1.1') self.assertStatus(401) self.set_jwt_token(None) @@ -169,7 +169,7 @@ def test_logout(self): self.assertJsonBody({ "redirect_url": "#/login" }) - self._get("/api/host", set_cookies=True) + self._get("/api/host", set_cookies=True, version='1.1') self.assertStatus(401) self.set_jwt_token(None) @@ -179,10 +179,10 @@ def test_token_ttl(self): self._post("/api/auth", {'username': 'admin', 'password': 'admin'}) self.assertStatus(201) self.set_jwt_token(self.jsonBody()['token']) - self._get("/api/host") + self._get("/api/host", version='1.1') self.assertStatus(200) time.sleep(6) - self._get("/api/host") + self._get("/api/host", version='1.1') self.assertStatus(401) self._ceph_cmd(['dashboard', 'set-jwt-token-ttl', '28800']) self.set_jwt_token(None) @@ -192,10 +192,10 @@ def test_token_ttl(self): self._post("/api/auth", {'username': 'admin', 'password': 'admin'}, set_cookies=True) self.assertStatus(201) self.set_jwt_token(self.jsonBody()['token']) - self._get("/api/host", set_cookies=True) + self._get("/api/host", set_cookies=True, version='1.1') self.assertStatus(200) time.sleep(6) - self._get("/api/host", set_cookies=True) + self._get("/api/host", set_cookies=True, version='1.1') self.assertStatus(401) self._ceph_cmd(['dashboard', 'set-jwt-token-ttl', '28800']) self.set_jwt_token(None) @@ -209,7 +209,7 @@ def test_remove_from_blocklist(self): # the following call adds the token to the blocklist self._post("/api/auth/logout") self.assertStatus(200) - self._get("/api/host") + self._get("/api/host", version='1.1') self.assertStatus(401) time.sleep(6) self._ceph_cmd(['dashboard', 'set-jwt-token-ttl', '28800']) @@ -229,7 +229,7 @@ def test_remove_from_blocklist(self): # the following call adds the token to the blocklist self._post("/api/auth/logout", set_cookies=True) self.assertStatus(200) - self._get("/api/host", set_cookies=True) + self._get("/api/host", set_cookies=True, version='1.1') self.assertStatus(401) time.sleep(6) self._ceph_cmd(['dashboard', 'set-jwt-token-ttl', '28800']) @@ -243,61 +243,61 @@ def test_remove_from_blocklist(self): def test_unauthorized(self): # test with Authorization header - self._get("/api/host") + self._get("/api/host", version='1.1') self.assertStatus(401) # test with Cookies set - self._get("/api/host", set_cookies=True) + self._get("/api/host", set_cookies=True, version='1.1') self.assertStatus(401) def test_invalidate_token_by_admin(self): # test with Authorization header - self._get("/api/host") + self._get("/api/host", version='1.1') self.assertStatus(401) self.create_user('user', 'user', ['read-only']) time.sleep(1) self._post("/api/auth", {'username': 'user', 'password': 'user'}) self.assertStatus(201) self.set_jwt_token(self.jsonBody()['token']) - self._get("/api/host") + self._get("/api/host", version='1.1') self.assertStatus(200) time.sleep(1) self._ceph_cmd_with_secret(['dashboard', 'ac-user-set-password', '--force-password', 'user'], 'user2') time.sleep(1) - self._get("/api/host") + self._get("/api/host", version='1.1') self.assertStatus(401) self.set_jwt_token(None) self._post("/api/auth", {'username': 'user', 'password': 'user2'}) self.assertStatus(201) self.set_jwt_token(self.jsonBody()['token']) - self._get("/api/host") + self._get("/api/host", version='1.1') self.assertStatus(200) self.delete_user("user") # test with Cookies set - self._get("/api/host", set_cookies=True) + self._get("/api/host", set_cookies=True, version='1.1') self.assertStatus(401) self.create_user('user', 'user', ['read-only']) time.sleep(1) self._post("/api/auth", {'username': 'user', 'password': 'user'}, set_cookies=True) self.assertStatus(201) self.set_jwt_token(self.jsonBody()['token']) - self._get("/api/host", set_cookies=True) + self._get("/api/host", set_cookies=True, version='1.1') self.assertStatus(200) time.sleep(1) self._ceph_cmd_with_secret(['dashboard', 'ac-user-set-password', '--force-password', 'user'], 'user2') time.sleep(1) - self._get("/api/host", set_cookies=True) + self._get("/api/host", set_cookies=True, version='1.1') self.assertStatus(401) self.set_jwt_token(None) self._post("/api/auth", {'username': 'user', 'password': 'user2'}, set_cookies=True) self.assertStatus(201) self.set_jwt_token(self.jsonBody()['token']) - self._get("/api/host", set_cookies=True) + self._get("/api/host", set_cookies=True, version='1.1') self.assertStatus(200) self.delete_user("user") diff --git a/qa/tasks/mgr/dashboard/test_host.py b/qa/tasks/mgr/dashboard/test_host.py index 124fff8d1544f..78d784473f3ca 100644 --- a/qa/tasks/mgr/dashboard/test_host.py +++ b/qa/tasks/mgr/dashboard/test_host.py @@ -32,11 +32,11 @@ def test_data_daemons(self): @DashboardTestCase.RunAs('test', 'test', ['block-manager']) def test_access_permissions(self): - self._get(self.URL_HOST) + self._get(self.URL_HOST, version='1.1') self.assertStatus(403) def test_host_list(self): - data = self._get(self.URL_HOST) + data = self._get(self.URL_HOST, version='1.1') self.assertStatus(200) orch_hostnames = {inventory_node['name'] for inventory_node in @@ -65,14 +65,14 @@ def test_host_list(self): self.assertIn(server['hostname'], orch_hostnames) def test_host_list_with_sources(self): - data = self._get('{}?sources=orchestrator'.format(self.URL_HOST)) + data = self._get('{}?sources=orchestrator'.format(self.URL_HOST), version='1.1') self.assertStatus(200) test_hostnames = {inventory_node['name'] for inventory_node in self.ORCHESTRATOR_TEST_DATA['inventory']} resp_hostnames = {host['hostname'] for host in data} self.assertEqual(test_hostnames, resp_hostnames) - data = self._get('{}?sources=ceph'.format(self.URL_HOST)) + data = self._get('{}?sources=ceph'.format(self.URL_HOST), version='1.1') self.assertStatus(200) test_hostnames = {inventory_node['name'] for inventory_node in self.ORCHESTRATOR_TEST_DATA['inventory']} @@ -80,7 +80,7 @@ def test_host_list_with_sources(self): self.assertEqual(len(test_hostnames.intersection(resp_hostnames)), 0) def test_host_devices(self): - hosts = self._get('{}'.format(self.URL_HOST)) + hosts = self._get('{}'.format(self.URL_HOST), version='1.1') hosts = [host['hostname'] for host in hosts if host['hostname'] != ''] assert hosts[0] data = self._get('{}/devices'.format('{}/{}'.format(self.URL_HOST, hosts[0]))) @@ -88,7 +88,7 @@ def test_host_devices(self): self.assertSchema(data, devices_schema) def test_host_daemons(self): - hosts = self._get('{}'.format(self.URL_HOST)) + hosts = self._get('{}'.format(self.URL_HOST), version='1.1') hosts = [host['hostname'] for host in hosts if host['hostname'] != ''] assert hosts[0] data = self._get('{}/daemons'.format('{}/{}'.format(self.URL_HOST, hosts[0]))) @@ -100,7 +100,7 @@ def test_host_daemons(self): }))) def test_host_smart(self): - hosts = self._get('{}'.format(self.URL_HOST)) + hosts = self._get('{}'.format(self.URL_HOST), version='1.1') hosts = [host['hostname'] for host in hosts if host['hostname'] != ''] assert hosts[0] self._get('{}/smart'.format('{}/{}'.format(self.URL_HOST, hosts[0]))) diff --git a/src/pybind/mgr/dashboard/controllers/cluster.py b/src/pybind/mgr/dashboard/controllers/cluster.py index 5ec49e39b1c22..d8170e672e992 100644 --- a/src/pybind/mgr/dashboard/controllers/cluster.py +++ b/src/pybind/mgr/dashboard/controllers/cluster.py @@ -2,18 +2,19 @@ from ..security import Scope from ..services.cluster import ClusterModel -from . import ApiController, ControllerDoc, EndpointDoc, RESTController +from . import APIDoc, APIRouter, EndpointDoc, RESTController +from ._version import APIVersion -@ApiController('/cluster', Scope.CONFIG_OPT) -@ControllerDoc("Get Cluster Details", "Cluster") +@APIRouter('/cluster', Scope.CONFIG_OPT) +@APIDoc("Get Cluster Details", "Cluster") class Cluster(RESTController): - @RESTController.MethodMap(version='0.1') + @RESTController.MethodMap(version=APIVersion.EXPERIMENTAL) @EndpointDoc("Get the cluster status") def list(self): return ClusterModel.from_db().dict() - @RESTController.MethodMap(version='0.1') + @RESTController.MethodMap(version=APIVersion.EXPERIMENTAL) @EndpointDoc("Update the cluster status", parameters={'status': (str, 'Cluster Status')}) def singleton_set(self, status: str): diff --git a/src/pybind/mgr/dashboard/controllers/host.py b/src/pybind/mgr/dashboard/controllers/host.py index c246de8f9f12c..0a897002a4a1c 100644 --- a/src/pybind/mgr/dashboard/controllers/host.py +++ b/src/pybind/mgr/dashboard/controllers/host.py @@ -15,7 +15,7 @@ from ..services.ceph_service import CephService from ..services.exception import handle_orchestrator_error from ..services.orchestrator import OrchClient, OrchFeature -from ..tools import TaskManager, str_to_bool +from ..tools import TaskManager, merge_list_of_dicts_by_key, str_to_bool from . import APIDoc, APIRouter, BaseController, Endpoint, EndpointDoc, \ ReadPermission, RESTController, Task, UIRouter, UpdatePermission, \ allow_empty_body @@ -154,10 +154,17 @@ def merge_hosts_by_hostname(ceph_hosts, orch_hosts): return hosts -def get_hosts(from_ceph=True, from_orchestrator=True): +def get_hosts(sources=None): """ Get hosts from various sources. """ + from_ceph = True + from_orchestrator = True + if sources: + _sources = sources.split(',') + from_ceph = 'ceph' in _sources + from_orchestrator = 'orchestrator' in _sources + ceph_hosts = [] if from_ceph: ceph_hosts = [ @@ -165,7 +172,6 @@ def get_hosts(from_ceph=True, from_orchestrator=True): server, { 'addr': '', 'labels': [], - 'service_type': '', 'sources': { 'ceph': True, 'orchestrator': False @@ -273,15 +279,24 @@ class Host(RESTController): @EndpointDoc("List Host Specifications", parameters={ 'sources': (str, 'Host Sources'), + 'facts': (bool, 'Host Facts') }, responses={200: LIST_HOST_SCHEMA}) - def list(self, sources=None): - if sources is None: - return get_hosts() - _sources = sources.split(',') - from_ceph = 'ceph' in _sources - from_orchestrator = 'orchestrator' in _sources - return get_hosts(from_ceph, from_orchestrator) + @RESTController.MethodMap(version=APIVersion(1, 1)) + def list(self, sources=None, facts=False): + hosts = get_hosts(sources) + orch = OrchClient.instance() + if str_to_bool(facts): + if orch.available(): + hosts_facts = orch.hosts.get_facts() + return merge_list_of_dicts_by_key(hosts, hosts_facts, 'hostname') + + raise DashboardException(code='orchestrator_status_unavailable', # pragma: no cover + msg="Please configure and enable the orchestrator if you " + "really want to gather facts from hosts", + component='orchestrator', + http_status_code=400) + return hosts @raise_if_no_orchestrator([OrchFeature.HOST_LIST, OrchFeature.HOST_ADD]) @handle_orchestrator_error('host') diff --git a/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/05-create-cluster-review.e2e-spec.ts b/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/05-create-cluster-review.e2e-spec.ts index 9097034e1a88c..29fd3a27f8edf 100644 --- a/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/05-create-cluster-review.e2e-spec.ts +++ b/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/05-create-cluster-review.e2e-spec.ts @@ -14,8 +14,8 @@ describe('Create Cluster Review page', () => { cy.get('button[aria-label="Next"]').click(); }); - describe('navigation link and title test', () => { - it('should check if nav-link and title contains Review', () => { + describe('navigation link test', () => { + it('should check if nav-link contains Review', () => { cy.get('.nav-link').should('contain.text', 'Review'); }); }); @@ -28,6 +28,8 @@ describe('Create Cluster Review page', () => { // check for fields in table createCluster.getStatusTables().should('contain.text', 'Hosts'); createCluster.getStatusTables().should('contain.text', 'Storage Capacity'); + createCluster.getStatusTables().should('contain.text', 'CPUs'); + createCluster.getStatusTables().should('contain.text', 'Memory'); }); it('should check Hosts by Services and Host Details tables are present', () => { @@ -46,14 +48,26 @@ describe('Create Cluster Review page', () => { createCluster.getDataTableHeaders(0).contains('Number of Hosts'); // verify correct columns on Host Details table - createCluster.getDataTableHeaders(1).contains('Host Name'); + createCluster.getDataTableHeaders(1).contains('Hostname'); createCluster.getDataTableHeaders(1).contains('Labels'); - }); - it('should check hosts count and default host name are present', () => { - createCluster.getStatusTables().contains(2); + createCluster.getDataTableHeaders(1).contains('CPUs'); + + createCluster.getDataTableHeaders(1).contains('Cores'); + + createCluster.getDataTableHeaders(1).contains('Total Memory'); + + createCluster.getDataTableHeaders(1).contains('Raw Capacity'); + + createCluster.getDataTableHeaders(1).contains('HDDs'); + + createCluster.getDataTableHeaders(1).contains('Flash'); + + createCluster.getDataTableHeaders(1).contains('NICs'); + }); + it('should check default host name is present', () => { createCluster.check_for_host(); }); }); diff --git a/src/pybind/mgr/dashboard/frontend/cypress/integration/page-helper.po.ts b/src/pybind/mgr/dashboard/frontend/cypress/integration/page-helper.po.ts index 4e15088a2c9c9..5405da94f0128 100644 --- a/src/pybind/mgr/dashboard/frontend/cypress/integration/page-helper.po.ts +++ b/src/pybind/mgr/dashboard/frontend/cypress/integration/page-helper.po.ts @@ -206,7 +206,7 @@ export abstract class PageHelper { getDataTableHeaders(index = 0) { this.waitDataTableToLoad(); - return cy.get('.datatable-header').its(index).find('.datatable-header-cell-label'); + return cy.get('.datatable-header').its(index).find('.datatable-header-cell'); } /** diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster-review.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster-review.component.html index e6f31dc9e74f8..f59cf6baceb51 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster-review.component.html +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster-review.component.html @@ -15,6 +15,16 @@ Number of devices: {{ totalDevices }}. Raw capacity: {{ totalCapacity | dimlessBinary }}. + + CPUs + {{ totalCPUs }} + + + Memory + {{ totalMemory }} + @@ -26,12 +36,14 @@ [columns]="hostsByService['columns']" [toolHeader]="false"> - - Host Details - - + +Host Details + + diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster-review.component.scss b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster-review.component.scss index e69de29bb2d1d..beecca0967162 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster-review.component.scss +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster-review.component.scss @@ -0,0 +1,5 @@ +cd-hosts { + ::ng-deep .nav { + display: none; + } +} diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster-review.component.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster-review.component.spec.ts index 8122cb682f0ba..3d9652be434ed 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster-review.component.spec.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster-review.component.spec.ts @@ -2,6 +2,7 @@ import { HttpClientTestingModule } from '@angular/common/http/testing'; import { ComponentFixture, TestBed } from '@angular/core/testing'; import _ from 'lodash'; +import { ToastrModule } from 'ngx-toastr'; import { of } from 'rxjs'; import { CephModule } from '~/app/ceph/ceph.module'; @@ -18,7 +19,7 @@ describe('CreateClusterReviewComponent', () => { let serviceListSpy: jasmine.Spy; configureTestBed({ - imports: [HttpClientTestingModule, SharedModule, CoreModule, CephModule] + imports: [HttpClientTestingModule, SharedModule, ToastrModule.forRoot(), CephModule, CoreModule] }); beforeEach(() => { diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster-review.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster-review.component.ts index 90e25d1162dc7..3e0f44f958a83 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster-review.component.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster-review.component.ts @@ -7,6 +7,7 @@ import { HostService } from '~/app/shared/api/host.service'; import { OsdService } from '~/app/shared/api/osd.service'; import { CellTemplate } from '~/app/shared/enum/cell-template.enum'; import { CephServiceSpec } from '~/app/shared/models/service.interface'; +import { DimlessBinaryPipe } from '~/app/shared/pipes/dimless-binary.pipe'; import { WizardStepsService } from '~/app/shared/services/wizard-steps.service'; @Component({ @@ -16,7 +17,6 @@ import { WizardStepsService } from '~/app/shared/services/wizard-steps.service'; }) export class CreateClusterReviewComponent implements OnInit { hosts: object[] = []; - hostsDetails: object; hostsByService: object; hostsCount: number; serviceCount: number; @@ -26,10 +26,13 @@ export class CreateClusterReviewComponent implements OnInit { totalDevices: number; totalCapacity = 0; services: Array = []; + totalCPUs = 0; + totalMemory = 0; constructor( public wizardStepsService: WizardStepsService, public cephServiceService: CephServiceService, + private dimlessBinary: DimlessBinaryPipe, public hostService: HostService, private osdService: OsdService ) {} @@ -41,24 +44,6 @@ export class CreateClusterReviewComponent implements OnInit { let walDeviceCapacity = 0; let dbDevices = 0; let dbDeviceCapacity = 0; - this.hostsDetails = { - columns: [ - { - prop: 'hostname', - name: $localize`Host Name`, - flexGrow: 2 - }, - { - name: $localize`Labels`, - prop: 'labels', - flexGrow: 1, - cellTransformation: CellTemplate.badge, - customTemplateConfig: { - class: 'badge-dark' - } - } - ] - }; this.hostsByService = { columns: [ @@ -88,6 +73,7 @@ export class CreateClusterReviewComponent implements OnInit { (this.serviceOccurrences[serviceKey['service_type']] || 0) + 1; this.uniqueServices.add(serviceKey['service_type']); }); + this.totalMemory = this.dimlessBinary.transform(this.totalMemory); this.uniqueServices.forEach((serviceType) => { this.hostsCountPerService.push({ @@ -99,10 +85,15 @@ export class CreateClusterReviewComponent implements OnInit { this.hostsByService['data'] = [...this.hostsCountPerService]; }); - this.hostService.list().subscribe((resp: object[]) => { + this.hostService.list('true').subscribe((resp: object[]) => { this.hosts = resp; this.hostsCount = this.hosts.length; - this.hostsDetails['data'] = [...this.hosts]; + _.forEach(this.hosts, (hostKey) => { + this.totalCPUs = this.totalCPUs + hostKey['cpu_count']; + // convert to bytes + this.totalMemory = this.totalMemory + hostKey['memory_total_kb'] * 1024; + }); + this.totalMemory = this.dimlessBinary.transform(this.totalMemory); }); if (this.osdService.osdDevices['data']) { diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster.component.html index 9ae80f4c0218d..5394dc1d33b21 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster.component.html +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster.component.html @@ -40,7 +40,7 @@

    Add Hosts


    - { + this.hostService.list('false').subscribe((hosts) => { hosts.forEach((host) => { const index = host['labels'].indexOf('_no_schedule', 0); if (index > -1) { diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/host-form/host-form.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/host-form/host-form.component.ts index 99313a5923aab..704b659127f26 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/host-form/host-form.component.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/host-form/host-form.component.ts @@ -51,7 +51,7 @@ export class HostFormComponent extends CdForm implements OnInit { this.pageURL = 'hosts'; } this.createForm(); - this.hostService.list().subscribe((resp: any[]) => { + this.hostService.list('false').subscribe((resp: any[]) => { this.hostnames = resp.map((host) => { return host['hostname']; }); diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/hosts.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/hosts.component.html index cb0ebca6bcd6d..a4d30918cc562 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/hosts.component.html +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/hosts.component.html @@ -67,4 +67,16 @@ Are you sure you want to continue? + + + Unavailable + + + + Flash + diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/hosts.component.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/hosts.component.spec.ts index 288f62e7a3d29..2c6a4db6983a1 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/hosts.component.spec.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/hosts.component.spec.ts @@ -12,6 +12,7 @@ import { CoreModule } from '~/app/core/core.module'; import { HostService } from '~/app/shared/api/host.service'; import { OrchestratorService } from '~/app/shared/api/orchestrator.service'; import { TableActionsComponent } from '~/app/shared/datatable/table-actions/table-actions.component'; +import { CdTableFetchDataContext } from '~/app/shared/models/cd-table-fetch-data-context'; import { CdTableSelection } from '~/app/shared/models/cd-table-selection'; import { OrchestratorFeature } from '~/app/shared/models/orchestrator.enum'; import { Permissions } from '~/app/shared/models/permissions'; @@ -99,7 +100,6 @@ describe('HostsComponent', () => { } ], hostname: hostname, - ceph_version: 'ceph version Development', labels: ['foo', 'bar'] } ]; @@ -108,14 +108,73 @@ describe('HostsComponent', () => { hostListSpy.and.callFake(() => of(payload)); fixture.detectChanges(); - return fixture.whenStable().then(() => { - fixture.detectChanges(); + component.getHosts(new CdTableFetchDataContext(() => undefined)); + fixture.detectChanges(); - const spans = fixture.debugElement.nativeElement.querySelectorAll( - '.datatable-body-cell-label span' - ); - expect(spans[0].textContent).toBe(hostname); - }); + const spans = fixture.debugElement.nativeElement.querySelectorAll( + '.datatable-body-cell-label span' + ); + expect(spans[0].textContent).toBe(hostname); + }); + + it('should test if host facts are tranformed correctly if orch available', () => { + const payload = [ + { + hostname: 'host_test', + services: [ + { + type: 'osd', + id: '0' + } + ], + cpu_count: 2, + cpu_cores: 1, + memory_total_kb: 1024, + hdd_count: 4, + hdd_capacity_bytes: 1024, + flash_count: 4, + flash_capacity_bytes: 1024, + nic_count: 1 + } + ]; + OrchestratorHelper.mockStatus(true); + hostListSpy.and.callFake(() => of(payload)); + fixture.detectChanges(); + + component.getHosts(new CdTableFetchDataContext(() => undefined)); + expect(hostListSpy).toHaveBeenCalled(); + expect(component.hosts[0]['cpu_count']).toEqual(2); + expect(component.hosts[0]['memory_total_bytes']).toEqual(1048576); + expect(component.hosts[0]['raw_capacity']).toEqual(2048); + expect(component.hosts[0]['hdd_count']).toEqual(4); + expect(component.hosts[0]['flash_count']).toEqual(4); + expect(component.hosts[0]['cpu_cores']).toEqual(1); + expect(component.hosts[0]['nic_count']).toEqual(1); + }); + + it('should test if host facts are unavailable if no orch available', () => { + const payload = [ + { + hostname: 'host_test', + services: [ + { + type: 'osd', + id: '0' + } + ] + } + ]; + OrchestratorHelper.mockStatus(false); + hostListSpy.and.callFake(() => of(payload)); + fixture.detectChanges(); + + component.getHosts(new CdTableFetchDataContext(() => undefined)); + fixture.detectChanges(); + + const spans = fixture.debugElement.nativeElement.querySelectorAll( + '.datatable-body-cell-label span' + ); + expect(spans[7].textContent).toBe('Unavailable'); }); it('should show force maintenance modal when it is safe to stop host', () => { diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/hosts.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/hosts.component.ts index 6bd8dbbdc2520..fe0b797bd1d7c 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/hosts.component.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/hosts.component.ts @@ -1,8 +1,10 @@ -import { Component, Input, OnInit, TemplateRef, ViewChild } from '@angular/core'; +import { Component, Input, OnDestroy, OnInit, TemplateRef, ViewChild } from '@angular/core'; import { Router } from '@angular/router'; import { NgbModalRef } from '@ng-bootstrap/ng-bootstrap'; import _ from 'lodash'; +import { Subscription } from 'rxjs'; +import { map, mergeMap } from 'rxjs/operators'; import { HostService } from '~/app/shared/api/host.service'; import { OrchestratorService } from '~/app/shared/api/orchestrator.service'; @@ -24,7 +26,7 @@ import { FinishedTask } from '~/app/shared/models/finished-task'; import { OrchestratorFeature } from '~/app/shared/models/orchestrator.enum'; import { OrchestratorStatus } from '~/app/shared/models/orchestrator.interface'; import { Permissions } from '~/app/shared/models/permissions'; -import { CephShortVersionPipe } from '~/app/shared/pipes/ceph-short-version.pipe'; +import { DimlessBinaryPipe } from '~/app/shared/pipes/dimless-binary.pipe'; import { AuthStorageService } from '~/app/shared/services/auth-storage.service'; import { ModalService } from '~/app/shared/services/modal.service'; import { NotificationService } from '~/app/shared/services/notification.service'; @@ -40,13 +42,19 @@ const BASE_URL = 'hosts'; styleUrls: ['./hosts.component.scss'], providers: [{ provide: URLBuilderService, useValue: new URLBuilderService(BASE_URL) }] }) -export class HostsComponent extends ListWithDetails implements OnInit { +export class HostsComponent extends ListWithDetails implements OnDestroy, OnInit { + private sub = new Subscription(); + @ViewChild(TableComponent) table: TableComponent; @ViewChild('servicesTpl', { static: true }) public servicesTpl: TemplateRef; @ViewChild('maintenanceConfirmTpl', { static: true }) maintenanceConfirmTpl: TemplateRef; + @ViewChild('orchTmpl', { static: true }) + orchTmpl: TemplateRef; + @ViewChild('flashTmpl', { static: true }) + flashTmpl: TemplateRef; @Input() hiddenColumns: string[] = []; @@ -95,8 +103,8 @@ export class HostsComponent extends ListWithDetails implements OnInit { constructor( private authStorageService: AuthStorageService, + private dimlessBinary: DimlessBinaryPipe, private hostService: HostService, - private cephShortVersionPipe: CephShortVersionPipe, private actionLabels: ActionLabelsI18n, private modalService: ModalService, private taskWrapper: TaskWrapperService, @@ -162,7 +170,7 @@ export class HostsComponent extends ListWithDetails implements OnInit { { name: $localize`Services`, prop: 'services', - flexGrow: 3, + flexGrow: 2, cellTemplate: this.servicesTpl }, { @@ -186,21 +194,59 @@ export class HostsComponent extends ListWithDetails implements OnInit { } }, { - name: $localize`Version`, - prop: 'ceph_version', - flexGrow: 1, - pipe: this.cephShortVersionPipe + name: $localize`Model`, + prop: 'model', + flexGrow: 1 + }, + { + name: $localize`CPUs`, + prop: 'cpu_count', + flexGrow: 0.3 + }, + { + name: $localize`Cores`, + prop: 'cpu_cores', + flexGrow: 0.3 + }, + { + name: $localize`Total Memory`, + prop: 'memory_total_bytes', + pipe: this.dimlessBinary, + flexGrow: 0.4 + }, + { + name: $localize`Raw Capacity`, + prop: 'raw_capacity', + pipe: this.dimlessBinary, + flexGrow: 0.5 + }, + { + name: $localize`HDDs`, + prop: 'hdd_count', + flexGrow: 0.3 + }, + { + name: $localize`Flash`, + prop: 'flash_count', + headerTemplate: this.flashTmpl, + flexGrow: 0.3 + }, + { + name: $localize`NICs`, + prop: 'nic_count', + flexGrow: 0.3 } ]; - this.orchService.status().subscribe((status: OrchestratorStatus) => { - this.orchStatus = status; - }); this.columns = this.columns.filter((col: any) => { return !this.hiddenColumns.includes(col.prop); }); } + ngOnDestroy() { + this.sub.unsubscribe(); + } + updateSelection(selection: CdTableSelection) { this.selection = selection; this.enableButton = false; @@ -343,6 +389,21 @@ export class HostsComponent extends ListWithDetails implements OnInit { }); } + transformHostsData() { + if (this.orchStatus?.available) { + _.forEach(this.hosts, (hostKey) => { + hostKey['memory_total_bytes'] = hostKey['memory_total_kb'] * 1024; + hostKey['raw_capacity'] = hostKey['hdd_capacity_bytes'] + hostKey['flash_capacity_bytes']; + }); + } else { + // mark host facts columns unavailable + for (let column = 4; column < this.columns.length; column++) { + this.columns[column]['prop'] = ''; + this.columns[column]['cellTemplate'] = this.orchTmpl; + } + } + } + getHosts(context: CdTableFetchDataContext) { if (this.isLoadingHosts) { return; @@ -357,24 +418,36 @@ export class HostsComponent extends ListWithDetails implements OnInit { 'tcmu-runner': 'iscsi' }; this.isLoadingHosts = true; - this.hostService.list().subscribe( - (resp: any[]) => { - resp.map((host) => { - host.services.map((service: any) => { - service.cdLink = `/perf_counters/${service.type}/${encodeURIComponent(service.id)}`; - const permission = this.permissions[typeToPermissionKey[service.type]]; - service.canRead = permission ? permission.read : false; - return service; - }); - return host; - }); - this.hosts = resp; - this.isLoadingHosts = false; - }, - () => { - this.isLoadingHosts = false; - context.error(); - } - ); + this.sub = this.orchService + .status() + .pipe( + mergeMap((orchStatus) => { + this.orchStatus = orchStatus; + + return this.hostService.list(`${this.orchStatus?.available}`); + }), + map((hostList: object[]) => + hostList.map((host) => { + host['services'].map((service: any) => { + service.cdLink = `/perf_counters/${service.type}/${encodeURIComponent(service.id)}`; + const permission = this.permissions[typeToPermissionKey[service.type]]; + service.canRead = permission ? permission.read : false; + return service; + }); + return host; + }) + ) + ) + .subscribe( + (hostList) => { + this.hosts = hostList; + this.transformHostsData(); + this.isLoadingHosts = false; + }, + () => { + this.isLoadingHosts = false; + context.error(); + } + ); } } diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/services/service-form/service-form.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/services/service-form/service-form.component.ts index 941eabfb67c62..2b424d7f26a35 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/services/service-form/service-form.component.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/services/service-form/service-form.component.ts @@ -231,7 +231,7 @@ export class ServiceFormComponent extends CdForm implements OnInit { this.serviceTypes = _.difference(resp, this.hiddenServices).sort(); }); - this.hostService.list().subscribe((resp: object[]) => { + this.hostService.list('false').subscribe((resp: object[]) => { const options: SelectOption[] = []; _.forEach(resp, (host: object) => { if (_.get(host, 'sources.orchestrator', false)) { diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/host.service.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/host.service.spec.ts index 8a7de5e25d634..797f94455a252 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/host.service.spec.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/host.service.spec.ts @@ -28,8 +28,8 @@ describe('HostService', () => { it('should call list', fakeAsync(() => { let result; - service.list().subscribe((resp) => (result = resp)); - const req = httpTesting.expectOne('api/host'); + service.list('true').subscribe((resp) => (result = resp)); + const req = httpTesting.expectOne('api/host?facts=true'); expect(req.request.method).toBe('GET'); req.flush(['foo', 'bar']); tick(); diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/host.service.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/host.service.ts index 984ee88fff11b..a1f2497b5a709 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/host.service.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/host.service.ts @@ -22,8 +22,11 @@ export class HostService { constructor(private http: HttpClient, private deviceService: DeviceService) {} - list(): Observable { - return this.http.get(this.baseURL); + list(facts: string): Observable { + return this.http.get(this.baseURL, { + headers: { Accept: 'application/vnd.ceph.api.v1.1+json' }, + params: { facts: facts } + }); } create(hostname: string, addr: string, labels: string[], status: string) { diff --git a/src/pybind/mgr/dashboard/openapi.yaml b/src/pybind/mgr/dashboard/openapi.yaml index f03102599b0df..ecf34a2bf4ee2 100644 --- a/src/pybind/mgr/dashboard/openapi.yaml +++ b/src/pybind/mgr/dashboard/openapi.yaml @@ -3293,10 +3293,16 @@ paths: name: sources schema: type: string + - default: false + description: Host Facts + in: query + name: facts + schema: + type: boolean responses: '200': content: - application/vnd.ceph.api.v1.0+json: + application/vnd.ceph.api.v1.1+json: schema: properties: addr: diff --git a/src/pybind/mgr/dashboard/tests/test_host.py b/src/pybind/mgr/dashboard/tests/test_host.py index 8b1e03b0ce317..a74d7d8e0aa02 100644 --- a/src/pybind/mgr/dashboard/tests/test_host.py +++ b/src/pybind/mgr/dashboard/tests/test_host.py @@ -52,7 +52,7 @@ def tearDownClass(cls): NotificationQueue.stop() @mock.patch('dashboard.controllers.host.get_hosts') - def test_host_list(self, mock_get_hosts): + def test_host_list_with_sources(self, mock_get_hosts): hosts = [{ 'hostname': 'host-0', 'sources': { @@ -73,33 +73,109 @@ def test_host_list(self, mock_get_hosts): } }] - def _get_hosts(from_ceph=True, from_orchestrator=True): - _hosts = [] - if from_ceph: - _hosts.append(hosts[0]) - if from_orchestrator: - _hosts.append(hosts[1]) - _hosts.append(hosts[2]) - return _hosts + def _get_hosts(sources=None): + if sources == 'ceph': + return hosts[0] + if sources == 'orchestrator': + return hosts[1:] + if sources == 'ceph, orchestrator': + return hosts[2] + return hosts mock_get_hosts.side_effect = _get_hosts - self._get(self.URL_HOST) + self._get(self.URL_HOST, version=APIVersion(1, 1)) self.assertStatus(200) self.assertJsonBody(hosts) - self._get('{}?sources=ceph'.format(self.URL_HOST)) + self._get('{}?sources=ceph'.format(self.URL_HOST), version=APIVersion(1, 1)) self.assertStatus(200) - self.assertJsonBody([hosts[0]]) + self.assertJsonBody(hosts[0]) - self._get('{}?sources=orchestrator'.format(self.URL_HOST)) + self._get('{}?sources=orchestrator'.format(self.URL_HOST), version=APIVersion(1, 1)) self.assertStatus(200) self.assertJsonBody(hosts[1:]) - self._get('{}?sources=ceph,orchestrator'.format(self.URL_HOST)) + self._get('{}?sources=ceph,orchestrator'.format(self.URL_HOST), version=APIVersion(1, 1)) self.assertStatus(200) self.assertJsonBody(hosts) + @mock.patch('dashboard.controllers.host.get_hosts') + def test_host_list_with_facts(self, mock_get_hosts): + hosts_without_facts = [{ + 'hostname': 'host-0', + 'sources': { + 'ceph': True, + 'orchestrator': False + } + }, { + 'hostname': 'host-1', + 'sources': { + 'ceph': False, + 'orchestrator': True + } + }] + + hosts_facts = [{ + 'hostname': 'host-0', + 'cpu_count': 1, + 'memory_total_kb': 1024 + }, { + 'hostname': 'host-1', + 'cpu_count': 2, + 'memory_total_kb': 1024 + }] + + hosts_with_facts = [{ + 'hostname': 'host-0', + 'sources': { + 'ceph': True, + 'orchestrator': False + }, + 'cpu_count': 1, + 'memory_total_kb': 1024 + }, { + 'hostname': 'host-1', + 'sources': { + 'ceph': False, + 'orchestrator': True + }, + 'cpu_count': 2, + 'memory_total_kb': 1024 + }] + # test with orchestrator available + with patch_orch(True, hosts=hosts_without_facts) as fake_client: + mock_get_hosts.return_value = hosts_without_facts + fake_client.hosts.get_facts.return_value = hosts_facts + # test with ?facts=true + self._get('{}?facts=true'.format(self.URL_HOST), version=APIVersion(1, 1)) + self.assertStatus(200) + self.assertHeader('Content-Type', + 'application/vnd.ceph.api.v1.1+json') + self.assertJsonBody(hosts_with_facts) + + # test with ?facts=false + self._get('{}?facts=false'.format(self.URL_HOST), version=APIVersion(1, 1)) + self.assertStatus(200) + self.assertHeader('Content-Type', + 'application/vnd.ceph.api.v1.1+json') + self.assertJsonBody(hosts_without_facts) + + # test with no orchestrator available + with patch_orch(False): + mock_get_hosts.return_value = hosts_without_facts + + # test with ?facts=true + self._get('{}?facts=true'.format(self.URL_HOST), version=APIVersion(1, 1)) + self.assertStatus(400) + + # test with ?facts=false + self._get('{}?facts=false'.format(self.URL_HOST), version=APIVersion(1, 1)) + self.assertStatus(200) + self.assertHeader('Content-Type', + 'application/vnd.ceph.api.v1.1+json') + self.assertJsonBody(hosts_without_facts) + def test_get_1(self): mgr.list_servers.return_value = [] diff --git a/src/pybind/mgr/dashboard/tests/test_tools.py b/src/pybind/mgr/dashboard/tests/test_tools.py index 51f9216a19c4d..fa14195e71e3f 100644 --- a/src/pybind/mgr/dashboard/tests/test_tools.py +++ b/src/pybind/mgr/dashboard/tests/test_tools.py @@ -13,7 +13,8 @@ from ..controllers import APIRouter, BaseController, Proxy, RESTController, Router from ..controllers._version import APIVersion from ..services.exception import handle_rados_error -from ..tools import dict_contains_path, dict_get, json_str_to_object, partial_dict +from ..tools import dict_contains_path, dict_get, json_str_to_object, \ + merge_list_of_dicts_by_key, partial_dict from . import ControllerTestCase # pylint: disable=no-name-in-module @@ -197,3 +198,13 @@ def test_dict_get(self): self.assertFalse(dict_get({'foo': {'bar': False}}, 'foo.bar')) self.assertIsNone(dict_get({'foo': {'bar': False}}, 'foo.bar.baz')) self.assertEqual(dict_get({'foo': {'bar': False}, 'baz': 'xyz'}, 'baz'), 'xyz') + + def test_merge_list_of_dicts_by_key(self): + expected_result = [{'a': 1, 'b': 2, 'c': 3}, {'a': 4, 'b': 5, 'c': 6}] + self.assertEqual(expected_result, merge_list_of_dicts_by_key( + [{'a': 1, 'b': 2}, {'a': 4, 'b': 5}], [{'a': 1, 'c': 3}, {'a': 4, 'c': 6}], 'a')) + + expected_result = [{'a': 1, 'b': 2}, {'a': 4, 'b': 5, 'c': 6}] + self.assertEqual(expected_result, merge_list_of_dicts_by_key( + [{'a': 1, 'b': 2}, {'a': 4, 'b': 5}], [{}, {'a': 4, 'c': 6}], 'a')) + self.assertRaises(TypeError, merge_list_of_dicts_by_key, None) diff --git a/src/pybind/mgr/dashboard/tools.py b/src/pybind/mgr/dashboard/tools.py index 1092534ea1864..4e4837d9323e0 100644 --- a/src/pybind/mgr/dashboard/tools.py +++ b/src/pybind/mgr/dashboard/tools.py @@ -828,3 +828,13 @@ def find_object_in_list(key, value, iterable): if key in obj and obj[key] == value: return obj return None + + +def merge_list_of_dicts_by_key(target_list: list, source_list: list, key: str): + target_list = {d[key]: d for d in target_list} + for sdict in source_list: + if bool(sdict): + if sdict[key] in target_list: + target_list[sdict[key]].update(sdict) + target_list = [value for value in target_list.values()] + return target_list From 3276147049511a9a8d45d84c95154a4ec014cd83 Mon Sep 17 00:00:00 2001 From: Nizamudeen A Date: Fri, 17 Sep 2021 18:43:01 +0530 Subject: [PATCH 0129/2866] mgr/dashboard: Cluster expansion e2e cleanups Fixes: https://tracker.ceph.com/issues/52645 Signed-off-by: Nizamudeen A --- .../integration/cluster/create-cluster.po.ts | 179 +++--------------- .../cypress/integration/cluster/hosts.po.ts | 16 +- .../integration/cluster/services.po.ts | 4 - .../orchestrator/01-hosts.e2e-spec.ts | 4 +- .../02-hosts-inventory.e2e-spec.ts | 2 +- .../orchestrator/05-services.e2e-spec.ts | 6 +- ...01-create-cluster-welcome-page.e2e-spec.ts | 4 +- .../workflow/01-hosts.e2e-spec.ts | 91 --------- .../02-create-cluster-add-host.e2e-spec.ts | 35 ++-- .../03-create-cluster-create-osds.e2e-spec.ts | 15 +- ...create-cluster-create-services.e2e-spec.ts | 26 +-- .../05-create-cluster-review.e2e-spec.ts | 16 +- .../workflow/06-cluster-check.e2e-spec.ts | 39 ++-- .../cypress/integration/page-helper.po.ts | 13 ++ 14 files changed, 133 insertions(+), 317 deletions(-) delete mode 100644 src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/01-hosts.e2e-spec.ts diff --git a/src/pybind/mgr/dashboard/frontend/cypress/integration/cluster/create-cluster.po.ts b/src/pybind/mgr/dashboard/frontend/cypress/integration/cluster/create-cluster.po.ts index a528c0fc12e05..52be703af28eb 100644 --- a/src/pybind/mgr/dashboard/frontend/cypress/integration/cluster/create-cluster.po.ts +++ b/src/pybind/mgr/dashboard/frontend/cypress/integration/cluster/create-cluster.po.ts @@ -1,26 +1,18 @@ -import { PageHelper } from '../page-helper.po'; -import { NotificationSidebarPageHelper } from '../ui/notification.po'; +import { HostsPageHelper } from 'cypress/integration/cluster/hosts.po'; +import { ServicesPageHelper } from 'cypress/integration/cluster/services.po'; +import { PageHelper } from 'cypress/integration/page-helper.po'; +import { NotificationSidebarPageHelper } from 'cypress/integration/ui/notification.po'; const pages = { index: { url: '#/expand-cluster', id: 'cd-create-cluster' } }; - export class CreateClusterWizardHelper extends PageHelper { pages = pages; - columnIndex = { - hostname: 1, - labels: 2, - status: 3 - }; - - serviceColumnIndex = { - service_name: 1, - placement: 2 - }; createCluster() { cy.get('cd-create-cluster').should('contain.text', 'Please expand your cluster first'); cy.get('[name=expand-cluster]').click(); + cy.get('cd-wizard').should('exist'); } doSkip() { @@ -33,91 +25,6 @@ export class CreateClusterWizardHelper extends PageHelper { notification.getNotifications().should('contain', 'Cluster expansion skipped by user'); } - check_for_host() { - this.getTableCount('total').should('not.be.eq', 0); - } - - clickHostTab(hostname: string, tabName: string) { - this.getExpandCollapseElement(hostname).click(); - cy.get('cd-host-details').within(() => { - this.getTab(tabName).click(); - }); - } - - add(hostname: string, exist?: boolean, maintenance?: boolean) { - cy.get('.btn.btn-accent').first().click({ force: true }); - cy.get('cd-modal').should('exist'); - cy.get('cd-modal').within(() => { - cy.get('#hostname').type(hostname); - if (maintenance) { - cy.get('label[for=maintenance]').click(); - } - if (exist) { - cy.get('#hostname').should('have.class', 'ng-invalid'); - } - cy.get('cd-submit-button').click(); - }); - // back to host list - cy.get(`${this.pages.index.id}`); - } - - checkExist(hostname: string, exist: boolean) { - this.clearTableSearchInput(); - this.getTableCell(this.columnIndex.hostname, hostname).should(($elements) => { - const hosts = $elements.map((_, el) => el.textContent).get(); - if (exist) { - expect(hosts).to.include(hostname); - } else { - expect(hosts).to.not.include(hostname); - } - }); - } - - delete(hostname: string) { - super.delete(hostname, this.columnIndex.hostname, 'hosts'); - } - - // Add or remove labels on a host, then verify labels in the table - editLabels(hostname: string, labels: string[], add: boolean) { - this.getTableCell(this.columnIndex.hostname, hostname).click(); - this.clickActionButton('edit'); - - // add or remove label badges - if (add) { - cy.get('cd-modal').find('.select-menu-edit').click(); - for (const label of labels) { - cy.contains('cd-modal .badge', new RegExp(`^${label}$`)).should('not.exist'); - cy.get('.popover-body input').type(`${label}{enter}`); - } - } else { - for (const label of labels) { - cy.contains('cd-modal .badge', new RegExp(`^${label}$`)) - .find('.badge-remove') - .click(); - } - } - cy.get('cd-modal cd-submit-button').click(); - this.checkLabelExists(hostname, labels, add); - } - - checkLabelExists(hostname: string, labels: string[], add: boolean) { - // Verify labels are added or removed from Labels column - // First find row with hostname, then find labels in the row - this.getTableCell(this.columnIndex.hostname, hostname) - .parent() - .find(`datatable-body-cell:nth-child(${this.columnIndex.labels}) .badge`) - .should(($ele) => { - const newLabels = $ele.toArray().map((v) => v.innerText); - for (const label of labels) { - if (add) { - expect(newLabels).to.include(label); - } else { - expect(newLabels).to.not.include(label); - } - } - }); - } - createOSD(deviceType: 'hdd' | 'ssd') { // Click Primary devices Add button cy.get('cd-osd-devices-selection-groups[name="Primary"]').as('primaryGroups'); @@ -131,59 +38,33 @@ export class CreateClusterWizardHelper extends PageHelper { cy.get('@addButton').click(); }); } +} - private selectServiceType(serviceType: string) { - return this.selectOption('service_type', serviceType); - } - - addService(serviceType: string) { - cy.get('.btn.btn-accent').first().click({ force: true }); - cy.get('cd-modal').should('exist'); - cy.get('cd-modal').within(() => { - this.selectServiceType(serviceType); - if (serviceType === 'rgw') { - cy.get('#service_id').type('rgw'); - cy.get('#count').type('1'); - } else if (serviceType === 'ingress') { - this.selectOption('backend_service', 'rgw.rgw'); - cy.get('#service_id').should('have.value', 'rgw.rgw'); - cy.get('#virtual_ip').type('192.168.20.1/24'); - cy.get('#frontend_port').type('8081'); - cy.get('#monitor_port').type('8082'); - } - - cy.get('cd-submit-button').click(); - }); - } - - checkServiceExist(serviceName: string, exist: boolean) { - this.getTableCell(this.serviceColumnIndex.service_name, serviceName).should(($elements) => { - const services = $elements.map((_, el) => el.textContent).get(); - if (exist) { - expect(services).to.include(serviceName); - } else { - expect(services).to.not.include(serviceName); - } - }); - } - - deleteService(serviceName: string, wait: number) { - const getRow = this.getTableCell.bind(this, this.serviceColumnIndex.service_name); - getRow(serviceName).click(); - - // Clicks on table Delete button - this.clickActionButton('delete'); - - // Confirms deletion - cy.get('cd-modal .custom-control-label').click(); - cy.contains('cd-modal button', 'Delete').click(); +export class CreateClusterHostPageHelper extends HostsPageHelper { + pages = { + index: { url: '#/expand-cluster', id: 'cd-wizard' }, + add: { url: '', id: 'cd-host-form' } + }; - // Wait for modal to close - cy.get('cd-modal').should('not.exist'); + columnIndex = { + hostname: 1, + labels: 2, + status: 3, + services: 0 + }; +} - // wait for delete operation to complete: tearing down the service daemons - cy.wait(wait); +export class CreateClusterServicePageHelper extends ServicesPageHelper { + pages = { + index: { url: '#/expand-cluster', id: 'cd-wizard' }, + create: { url: '', id: 'cd-service-form' } + }; - this.checkServiceExist(serviceName, false); - } + columnIndex = { + service_name: 1, + placement: 2, + running: 0, + size: 0, + last_refresh: 0 + }; } diff --git a/src/pybind/mgr/dashboard/frontend/cypress/integration/cluster/hosts.po.ts b/src/pybind/mgr/dashboard/frontend/cypress/integration/cluster/hosts.po.ts index 5f3d39dcedc50..c615b945a5d6a 100644 --- a/src/pybind/mgr/dashboard/frontend/cypress/integration/cluster/hosts.po.ts +++ b/src/pybind/mgr/dashboard/frontend/cypress/integration/cluster/hosts.po.ts @@ -41,15 +41,6 @@ export class HostsPageHelper extends PageHelper { }); } - @PageHelper.restrictTo(pages.index.url) - clickHostTab(hostname: string, tabName: string) { - this.getExpandCollapseElement(hostname).click(); - cy.get('cd-host-details').within(() => { - this.getTab(tabName).click(); - }); - } - - @PageHelper.restrictTo(pages.add.url) add(hostname: string, exist?: boolean, maintenance?: boolean) { cy.get(`${this.pages.add.id}`).within(() => { cy.get('#hostname').type(hostname); @@ -65,8 +56,8 @@ export class HostsPageHelper extends PageHelper { cy.get(`${this.pages.index.id}`); } - @PageHelper.restrictTo(pages.index.url) checkExist(hostname: string, exist: boolean) { + this.clearTableSearchInput(); this.getTableCell(this.columnIndex.hostname, hostname).should(($elements) => { const hosts = $elements.map((_, el) => el.textContent).get(); if (exist) { @@ -77,13 +68,11 @@ export class HostsPageHelper extends PageHelper { }); } - @PageHelper.restrictTo(pages.index.url) delete(hostname: string) { super.delete(hostname, this.columnIndex.hostname, 'hosts'); } // Add or remove labels on a host, then verify labels in the table - @PageHelper.restrictTo(pages.index.url) editLabels(hostname: string, labels: string[], add: boolean) { this.getTableCell(this.columnIndex.hostname, hostname).click(); this.clickActionButton('edit'); @@ -103,7 +92,10 @@ export class HostsPageHelper extends PageHelper { } } cy.get('cd-modal cd-submit-button').click(); + this.checkLabelExists(hostname, labels, add); + } + checkLabelExists(hostname: string, labels: string[], add: boolean) { // Verify labels are added or removed from Labels column // First find row with hostname, then find labels in the row this.getTableCell(this.columnIndex.hostname, hostname) diff --git a/src/pybind/mgr/dashboard/frontend/cypress/integration/cluster/services.po.ts b/src/pybind/mgr/dashboard/frontend/cypress/integration/cluster/services.po.ts index 0a943d4b0553f..4265329db042a 100644 --- a/src/pybind/mgr/dashboard/frontend/cypress/integration/cluster/services.po.ts +++ b/src/pybind/mgr/dashboard/frontend/cypress/integration/cluster/services.po.ts @@ -30,7 +30,6 @@ export class ServicesPageHelper extends PageHelper { return this.selectOption('service_type', serviceType); } - @PageHelper.restrictTo(pages.index.url) clickServiceTab(serviceName: string, tabName: string) { this.getExpandCollapseElement(serviceName).click(); cy.get('cd-service-details').within(() => { @@ -38,7 +37,6 @@ export class ServicesPageHelper extends PageHelper { }); } - @PageHelper.restrictTo(pages.create.url) addService(serviceType: string, exist?: boolean, count = '1') { cy.get(`${this.pages.create.id}`).within(() => { this.selectServiceType(serviceType); @@ -73,7 +71,6 @@ export class ServicesPageHelper extends PageHelper { }); } - @PageHelper.restrictTo(pages.index.url) checkExist(serviceName: string, exist: boolean) { this.getTableCell(this.columnIndex.service_name, serviceName).should(($elements) => { const services = $elements.map((_, el) => el.textContent).get(); @@ -85,7 +82,6 @@ export class ServicesPageHelper extends PageHelper { }); } - @PageHelper.restrictTo(pages.index.url) deleteService(serviceName: string) { const getRow = this.getTableCell.bind(this, this.columnIndex.service_name); getRow(serviceName).click(); diff --git a/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/01-hosts.e2e-spec.ts b/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/01-hosts.e2e-spec.ts index 6c79a74662dff..ffc5fa83ab430 100644 --- a/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/01-hosts.e2e-spec.ts +++ b/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/01-hosts.e2e-spec.ts @@ -33,7 +33,7 @@ describe('Hosts page', () => { it('should display inventory', function () { for (const host of this.hosts) { - hosts.clickHostTab(host.name, 'Physical Disks'); + hosts.clickTab('cd-host-details', host.name, 'Physical Disks'); cy.get('cd-host-details').within(() => { hosts.getTableCount('total').should('be.gte', 0); }); @@ -42,7 +42,7 @@ describe('Hosts page', () => { it('should display daemons', function () { for (const host of this.hosts) { - hosts.clickHostTab(host.name, 'Daemons'); + hosts.clickTab('cd-host-details', host.name, 'Daemons'); cy.get('cd-host-details').within(() => { hosts.getTableCount('total').should('be.gte', 0); }); diff --git a/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/02-hosts-inventory.e2e-spec.ts b/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/02-hosts-inventory.e2e-spec.ts index 43e8aee595d8c..f26228510f5e9 100644 --- a/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/02-hosts-inventory.e2e-spec.ts +++ b/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/02-hosts-inventory.e2e-spec.ts @@ -16,7 +16,7 @@ describe('Hosts page', () => { it('should display correct inventory', function () { for (const host of this.hosts) { - hosts.clickHostTab(host.name, 'Physical Disks'); + hosts.clickTab('cd-host-details', host.name, 'Physical Disks'); cy.get('cd-host-details').within(() => { hosts.getTableCount('total').should('be.eq', host.devices.length); }); diff --git a/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/05-services.e2e-spec.ts b/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/05-services.e2e-spec.ts index d36ffc17eed16..9f5e8ceda862b 100644 --- a/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/05-services.e2e-spec.ts +++ b/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/05-services.e2e-spec.ts @@ -14,16 +14,16 @@ describe('Services page', () => { services.navigateTo('create'); services.addService('rgw'); - services.checkExist('rgw.rgw.foo', true); + services.checkExist('rgw.foo', true); }); it('should create and delete an ingress service', () => { services.navigateTo('create'); services.addService('ingress'); - services.checkExist('ingress.rgw.rgw.foo', true); + services.checkExist('ingress.rgw.foo', true); - services.deleteService('ingress.rgw.rgw.foo'); + services.deleteService('ingress.rgw.foo'); }); }); }); diff --git a/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/01-create-cluster-welcome-page.e2e-spec.ts b/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/01-create-cluster-welcome-page.e2e-spec.ts index d776a210a57ee..02ebb184b39d6 100644 --- a/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/01-create-cluster-welcome-page.e2e-spec.ts +++ b/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/01-create-cluster-welcome-page.e2e-spec.ts @@ -9,11 +9,11 @@ describe('Create cluster page', () => { createCluster.navigateTo(); }); - it('should fail to create cluster', () => { + it('should open the wizard when Expand Cluster is clicked', () => { createCluster.createCluster(); }); - it('should skip to dashboard landing page', () => { + it('should skip to dashboard landing page when Skip is clicked', () => { createCluster.doSkip(); }); }); diff --git a/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/01-hosts.e2e-spec.ts b/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/01-hosts.e2e-spec.ts deleted file mode 100644 index e8dea8e30080f..0000000000000 --- a/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/01-hosts.e2e-spec.ts +++ /dev/null @@ -1,91 +0,0 @@ -import { HostsPageHelper } from 'cypress/integration/cluster/hosts.po'; -import { ServicesPageHelper } from 'cypress/integration/cluster/services.po'; - -describe('Hosts page', () => { - const hosts = new HostsPageHelper(); - const services = new ServicesPageHelper(); - - const serviceName = 'rgw.foo'; - const hostnames = [ - 'ceph-node-00.cephlab.com', - 'ceph-node-01.cephlab.com', - 'ceph-node-02.cephlab.com' - ]; - const addHost = (hostname: string, exist?: boolean, maintenance?: boolean) => { - hosts.navigateTo('add'); - hosts.add(hostname, exist, maintenance); - hosts.checkExist(hostname, true); - }; - - beforeEach(() => { - cy.login(); - Cypress.Cookies.preserveOnce('token'); - hosts.navigateTo(); - }); - - describe('when Orchestrator is available', () => { - it('should add a host', () => { - addHost(hostnames[2], false, false); - }); - - it('should display inventory', function () { - hosts.clickHostTab(hostnames[0], 'Physical Disks'); - cy.get('cd-host-details').within(() => { - hosts.getTableCount('total').should('be.gte', 0); - }); - }); - - it('should display daemons', function () { - hosts.clickHostTab(hostnames[0], 'Daemons'); - cy.get('cd-host-details').within(() => { - hosts.getTableCount('total').should('be.gte', 0); - }); - }); - - it('should edit host labels', function () { - const labels = ['foo', 'bar']; - hosts.editLabels(hostnames[0], labels, true); - hosts.editLabels(hostnames[0], labels, false); - }); - - it('should not add an existing host', function () { - hosts.navigateTo('add'); - hosts.add(hostnames[0], true); - }); - - it('should add a host in maintenance mode', function () { - addHost(hostnames[1], false, true); - }); - - it('should delete a host and add it back', function () { - hosts.delete(hostnames[1]); - addHost(hostnames[1], false, true); - }); - - it('should exit host from maintenance', function () { - hosts.maintenance(hostnames[1], true); - }); - - it('should check if mon service is running', () => { - hosts.clickHostTab(hostnames[1], 'Daemons'); - cy.get('cd-host-details').within(() => { - services.checkServiceStatus('mon'); - }); - }); - - it('should create rgw services', () => { - services.navigateTo('create'); - services.addService('rgw', false, '3'); - services.checkExist(serviceName, true); - hosts.navigateTo(); - hosts.clickHostTab(hostnames[1], 'Daemons'); - cy.get('cd-host-details').within(() => { - services.checkServiceStatus('rgw'); - }); - }); - - it('should force maintenance', () => { - hosts.maintenance(hostnames[1], true, true); - }); - }); -}); diff --git a/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/02-create-cluster-add-host.e2e-spec.ts b/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/02-create-cluster-add-host.e2e-spec.ts index 3b64c2987c9e5..a35bd6c10748e 100644 --- a/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/02-create-cluster-add-host.e2e-spec.ts +++ b/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/02-create-cluster-add-host.e2e-spec.ts @@ -1,15 +1,20 @@ -import { CreateClusterWizardHelper } from 'cypress/integration/cluster/create-cluster.po'; +import { + CreateClusterHostPageHelper, + CreateClusterWizardHelper +} from 'cypress/integration/cluster/create-cluster.po'; describe('Create cluster add host page', () => { const createCluster = new CreateClusterWizardHelper(); + const createClusterHostPage = new CreateClusterHostPageHelper(); const hostnames = [ 'ceph-node-00.cephlab.com', 'ceph-node-01.cephlab.com', 'ceph-node-02.cephlab.com' ]; const addHost = (hostname: string, exist?: boolean) => { - createCluster.add(hostname, exist, false); - createCluster.checkExist(hostname, true); + cy.get('.btn.btn-accent').first().click({ force: true }); + createClusterHostPage.add(hostname, exist, false); + createClusterHostPage.checkExist(hostname, true); }; beforeEach(() => { @@ -19,35 +24,37 @@ describe('Create cluster add host page', () => { createCluster.createCluster(); }); - it('should check if nav-link and title contains Add Hosts', () => { + it('should check if title contains Add Hosts', () => { cy.get('.nav-link').should('contain.text', 'Add Hosts'); cy.get('.title').should('contain.text', 'Add Hosts'); }); it('should check existing host and add new hosts', () => { - createCluster.checkExist(hostnames[0], true); + createClusterHostPage.checkExist(hostnames[0], true); addHost(hostnames[1], false); addHost(hostnames[2], false); }); + it('should delete a host and add it back', () => { + createClusterHostPage.delete(hostnames[1]); + addHost(hostnames[1], false); + }); + it('should verify "_no_schedule" label is added', () => { - createCluster.checkLabelExists(hostnames[1], ['_no_schedule'], true); - createCluster.checkLabelExists(hostnames[2], ['_no_schedule'], true); + createClusterHostPage.checkLabelExists(hostnames[1], ['_no_schedule'], true); + createClusterHostPage.checkLabelExists(hostnames[2], ['_no_schedule'], true); }); it('should not add an existing host', () => { - createCluster.add(hostnames[0], true); + cy.get('.btn.btn-accent').first().click({ force: true }); + createClusterHostPage.add(hostnames[0], true); }); it('should edit host labels', () => { const labels = ['foo', 'bar']; - createCluster.editLabels(hostnames[0], labels, true); - createCluster.editLabels(hostnames[0], labels, false); - }); - - it('should delete a host', () => { - createCluster.delete(hostnames[1]); + createClusterHostPage.editLabels(hostnames[0], labels, true); + createClusterHostPage.editLabels(hostnames[0], labels, false); }); }); diff --git a/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/03-create-cluster-create-osds.e2e-spec.ts b/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/03-create-cluster-create-osds.e2e-spec.ts index ee272bcf943b2..c8d93b4796230 100644 --- a/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/03-create-cluster-create-osds.e2e-spec.ts +++ b/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/03-create-cluster-create-osds.e2e-spec.ts @@ -11,12 +11,10 @@ describe('Create cluster create osds page', () => { Cypress.Cookies.preserveOnce('token'); createCluster.navigateTo(); createCluster.createCluster(); - cy.get('button[aria-label="Next"]').click(); + cy.get('.nav-link').contains('Create OSDs').click(); }); - it('should check if nav-link and title contains Create OSDs', () => { - cy.get('.nav-link').should('contain.text', 'Create OSDs'); - + it('should check if title contains Create OSDs', () => { cy.get('.title').should('contain.text', 'Create OSDs'); }); @@ -27,13 +25,16 @@ describe('Create cluster create osds page', () => { createCluster.navigateTo(); createCluster.createCluster(); - cy.get('button[aria-label="Next"]').click(); + cy.get('.nav-link').contains('Create OSDs').click(); createCluster.createOSD('hdd'); + // Go to the Review section and Expand the cluster + // because the drive group spec is only stored + // in frontend and will be lost when refreshed + cy.get('.nav-link').contains('Review').click(); cy.get('button[aria-label="Next"]').click(); - cy.get('button[aria-label="Next"]').click(); - cy.get('button[aria-label="Next"]').click(); + cy.get('cd-dashboard').should('exist'); }); }); }); diff --git a/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/04-create-cluster-create-services.e2e-spec.ts b/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/04-create-cluster-create-services.e2e-spec.ts index 0a95474c91d3f..5060a22649b57 100644 --- a/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/04-create-cluster-create-services.e2e-spec.ts +++ b/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/04-create-cluster-create-services.e2e-spec.ts @@ -1,36 +1,38 @@ -import { CreateClusterWizardHelper } from 'cypress/integration/cluster/create-cluster.po'; +import { + CreateClusterServicePageHelper, + CreateClusterWizardHelper +} from 'cypress/integration/cluster/create-cluster.po'; describe('Create cluster create services page', () => { const createCluster = new CreateClusterWizardHelper(); + const createClusterServicePage = new CreateClusterServicePageHelper(); beforeEach(() => { cy.login(); Cypress.Cookies.preserveOnce('token'); createCluster.navigateTo(); createCluster.createCluster(); - cy.get('button[aria-label="Next"]').click(); - cy.get('button[aria-label="Next"]').click(); + cy.get('.nav-link').contains('Create Services').click(); }); - it('should check if nav-link and title contains Create Services', () => { - cy.get('.nav-link').should('contain.text', 'Create Services'); - + it('should check if title contains Create Services', () => { cy.get('.title').should('contain.text', 'Create Services'); }); describe('when Orchestrator is available', () => { it('should create an rgw service', () => { - createCluster.addService('rgw'); + cy.get('.btn.btn-accent').first().click({ force: true }); - createCluster.checkExist('rgw.rgw', true); + createClusterServicePage.addService('rgw', false, '3'); + createClusterServicePage.checkExist('rgw.foo', true); }); it('should create and delete an ingress service', () => { - createCluster.addService('ingress'); - - createCluster.checkExist('ingress.rgw.rgw', true); + cy.get('.btn.btn-accent').first().click({ force: true }); - createCluster.deleteService('ingress.rgw.rgw', 60000); + createClusterServicePage.addService('ingress'); + createClusterServicePage.checkExist('ingress.rgw.foo', true); + createClusterServicePage.deleteService('ingress.rgw.foo'); }); }); }); diff --git a/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/05-create-cluster-review.e2e-spec.ts b/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/05-create-cluster-review.e2e-spec.ts index 29fd3a27f8edf..411522290259a 100644 --- a/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/05-create-cluster-review.e2e-spec.ts +++ b/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/05-create-cluster-review.e2e-spec.ts @@ -1,7 +1,11 @@ -import { CreateClusterWizardHelper } from 'cypress/integration/cluster/create-cluster.po'; +import { + CreateClusterHostPageHelper, + CreateClusterWizardHelper +} from 'cypress/integration/cluster/create-cluster.po'; describe('Create Cluster Review page', () => { const createCluster = new CreateClusterWizardHelper(); + const createClusterHostPage = new CreateClusterHostPageHelper(); beforeEach(() => { cy.login(); @@ -9,14 +13,12 @@ describe('Create Cluster Review page', () => { createCluster.navigateTo(); createCluster.createCluster(); - cy.get('button[aria-label="Next"]').click(); - cy.get('button[aria-label="Next"]').click(); - cy.get('button[aria-label="Next"]').click(); + cy.get('.nav-link').contains('Review').click(); }); describe('navigation link test', () => { - it('should check if nav-link contains Review', () => { - cy.get('.nav-link').should('contain.text', 'Review'); + it('should check if active nav-link is of Review section', () => { + cy.get('.nav-link.active').should('contain.text', 'Review'); }); }); @@ -68,7 +70,7 @@ describe('Create Cluster Review page', () => { }); it('should check default host name is present', () => { - createCluster.check_for_host(); + createClusterHostPage.check_for_host(); }); }); }); diff --git a/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/06-cluster-check.e2e-spec.ts b/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/06-cluster-check.e2e-spec.ts index a5b54786b8738..ac21714d321cb 100644 --- a/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/06-cluster-check.e2e-spec.ts +++ b/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/06-cluster-check.e2e-spec.ts @@ -5,6 +5,8 @@ import { ServicesPageHelper } from 'cypress/integration/cluster/services.po'; describe('when cluster creation is completed', () => { const createCluster = new CreateClusterWizardHelper(); + const services = new ServicesPageHelper(); + const serviceName = 'rgw.foo'; beforeEach(() => { cy.login(); @@ -15,40 +17,53 @@ describe('when cluster creation is completed', () => { createCluster.navigateTo(); createCluster.createCluster(); + cy.get('.nav-link').contains('Review').click(); cy.get('button[aria-label="Next"]').click(); - cy.get('button[aria-label="Next"]').click(); - cy.get('button[aria-label="Next"]').click(); - cy.get('button[aria-label="Next"]').click(); - cy.get('cd-dashboard').should('exist'); }); describe('Hosts page', () => { const hosts = new HostsPageHelper(); - const hostnames = ['ceph-node-00.cephlab.com', 'ceph-node-02.cephlab.com']; + const hostnames = [ + 'ceph-node-00.cephlab.com', + 'ceph-node-01.cephlab.com', + 'ceph-node-02.cephlab.com' + ]; beforeEach(() => { hosts.navigateTo(); }); + it('should have removed "_no_schedule" label', () => { - for (let host = 0; host < hostnames.length; host++) { - cy.get('datatable-row-wrapper').should('not.have.text', '_no_schedule'); + for (const hostname of hostnames) { + hosts.checkLabelExists(hostname, ['_no_schedule'], false); } }); it('should display inventory', () => { - hosts.clickHostTab(hostnames[1], 'Physical Disks'); + hosts.clickTab('cd-host-details', hostnames[1], 'Physical Disks'); cy.get('cd-host-details').within(() => { hosts.getTableCount('total').should('be.gte', 0); }); }); it('should display daemons', () => { - hosts.clickHostTab(hostnames[1], 'Daemons'); + hosts.clickTab('cd-host-details', hostnames[1], 'Daemons'); cy.get('cd-host-details').within(() => { hosts.getTableCount('total').should('be.gte', 0); }); }); + + it('should check if rgw service is running', () => { + hosts.clickTab('cd-host-details', hostnames[1], 'Daemons'); + cy.get('cd-host-details').within(() => { + services.checkServiceStatus('rgw'); + }); + }); + + it('should force maintenance and exit', { retries: 1 }, () => { + hosts.maintenance(hostnames[1], true, true); + }); }); describe('OSDs page', () => { @@ -59,19 +74,17 @@ describe('when cluster creation is completed', () => { }); it('should check if osds are created', { retries: 1 }, () => { - osds.expectTableCount('total', 2); + osds.getTableCount('total').should('be.gte', 1); }); }); describe('Services page', () => { - const services = new ServicesPageHelper(); - beforeEach(() => { services.navigateTo(); }); it('should check if services are created', () => { - services.checkExist('rgw.rgw', true); + services.checkExist(serviceName, true); }); }); }); diff --git a/src/pybind/mgr/dashboard/frontend/cypress/integration/page-helper.po.ts b/src/pybind/mgr/dashboard/frontend/cypress/integration/page-helper.po.ts index 5405da94f0128..6395128c94734 100644 --- a/src/pybind/mgr/dashboard/frontend/cypress/integration/page-helper.po.ts +++ b/src/pybind/mgr/dashboard/frontend/cypress/integration/page-helper.po.ts @@ -85,6 +85,19 @@ export abstract class PageHelper { return this.getTabs().its('length'); } + /** + * Helper method to navigate/click a tab inside the expanded table row. + * @param selector The selector of the expanded table row. + * @param name The name of the row which should expand. + * @param tabName Name of the tab to be navigated/clicked. + */ + clickTab(selector: string, name: string, tabName: string) { + this.getExpandCollapseElement(name).click(); + cy.get(selector).within(() => { + this.getTab(tabName).click(); + }); + } + /** * Helper method to select an option inside a select element. * This method will also expect that the option was set. From 56f6e9588f29b977fe42d76355590e3878cc77c2 Mon Sep 17 00:00:00 2001 From: haoyixing Date: Wed, 13 Oct 2021 20:16:58 +0800 Subject: [PATCH 0130/2866] rbd: when trash mv, show expiration time if it's not now Signed-off-by: haoyixing --- src/tools/rbd/action/Trash.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/tools/rbd/action/Trash.cc b/src/tools/rbd/action/Trash.cc index 05cc724589b37..cb4c8b8a93e50 100644 --- a/src/tools/rbd/action/Trash.cc +++ b/src/tools/rbd/action/Trash.cc @@ -101,6 +101,9 @@ int execute_move(const po::variables_map &vm, << std::endl; } + if (expires_at != "now") { + std::cout << "rbd: image " << image_name << " will expire at " << exp_time << std::endl; + } return r; } From 8790f04c5469d86daebb3bbe1eb780a86b7404ce Mon Sep 17 00:00:00 2001 From: Laura Flores Date: Mon, 4 Oct 2021 04:41:10 +0000 Subject: [PATCH 0131/2866] os/bluestore: update priorities and nicks of bluestore perf counters These perf counters do not show up in telemetry unless they are set to a "useful" priority or higher. Fetching these counters in telemetry may help to diagnose problems with RocksDB / BlueFS prefetching / insufficient cache sizes. Signed-off-by: Laura Flores --- src/os/bluestore/BlueStore.cc | 89 +++++++++++++++++++++++------------ 1 file changed, 58 insertions(+), 31 deletions(-) diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc index f9f74828e30ea..422eadecabf72 100644 --- a/src/os/bluestore/BlueStore.cc +++ b/src/os/bluestore/BlueStore.cc @@ -4884,38 +4884,49 @@ void BlueStore::_init_logger() l_bluestore_first, l_bluestore_last); b.add_time_avg(l_bluestore_kv_flush_lat, "kv_flush_lat", "Average kv_thread flush latency", - "fl_l", PerfCountersBuilder::PRIO_INTERESTING); + "kfll", PerfCountersBuilder::PRIO_INTERESTING); b.add_time_avg(l_bluestore_kv_commit_lat, "kv_commit_lat", - "Average kv_thread commit latency"); + "Average kv_thread commit latency", + "kcol", PerfCountersBuilder::PRIO_USEFUL); b.add_time_avg(l_bluestore_kv_sync_lat, "kv_sync_lat", "Average kv_sync thread latency", - "ks_l", PerfCountersBuilder::PRIO_INTERESTING); + "kscl", PerfCountersBuilder::PRIO_INTERESTING); b.add_time_avg(l_bluestore_kv_final_lat, "kv_final_lat", "Average kv_finalize thread latency", - "kf_l", PerfCountersBuilder::PRIO_INTERESTING); + "kfll", PerfCountersBuilder::PRIO_INTERESTING); b.add_time_avg(l_bluestore_state_prepare_lat, "state_prepare_lat", - "Average prepare state latency"); + "Average prepare state latency", + "sprl", PerfCountersBuilder::PRIO_USEFUL); b.add_time_avg(l_bluestore_state_aio_wait_lat, "state_aio_wait_lat", "Average aio_wait state latency", - "io_l", PerfCountersBuilder::PRIO_INTERESTING); + "sawl", PerfCountersBuilder::PRIO_INTERESTING); b.add_time_avg(l_bluestore_state_io_done_lat, "state_io_done_lat", - "Average io_done state latency"); + "Average io_done state latency", + "sidl", PerfCountersBuilder::PRIO_USEFUL); b.add_time_avg(l_bluestore_state_kv_queued_lat, "state_kv_queued_lat", - "Average kv_queued state latency"); + "Average kv_queued state latency", + "skql", PerfCountersBuilder::PRIO_USEFUL); b.add_time_avg(l_bluestore_state_kv_committing_lat, "state_kv_commiting_lat", - "Average kv_commiting state latency"); + "Average kv_commiting state latency", + "skcl", PerfCountersBuilder::PRIO_USEFUL); b.add_time_avg(l_bluestore_state_kv_done_lat, "state_kv_done_lat", - "Average kv_done state latency"); + "Average kv_done state latency", + "skdl", PerfCountersBuilder::PRIO_USEFUL); b.add_time_avg(l_bluestore_state_deferred_queued_lat, "state_deferred_queued_lat", - "Average deferred_queued state latency"); + "Average deferred_queued state latency", + "sdql", PerfCountersBuilder::PRIO_USEFUL); b.add_time_avg(l_bluestore_state_deferred_aio_wait_lat, "state_deferred_aio_wait_lat", - "Average aio_wait state latency"); + "Average aio_wait state latency", + "sdal", PerfCountersBuilder::PRIO_USEFUL); b.add_time_avg(l_bluestore_state_deferred_cleanup_lat, "state_deferred_cleanup_lat", - "Average cleanup state latency"); + "Average cleanup state latency", + "sdcl", PerfCountersBuilder::PRIO_USEFUL); b.add_time_avg(l_bluestore_state_finishing_lat, "state_finishing_lat", - "Average finishing state latency"); + "Average finishing state latency", + "sfnl", PerfCountersBuilder::PRIO_USEFUL); b.add_time_avg(l_bluestore_state_done_lat, "state_done_lat", - "Average done state latency"); + "Average done state latency", + "sdnl", PerfCountersBuilder::PRIO_USEFUL); b.add_time_avg(l_bluestore_throttle_lat, "throttle_lat", "Average submit throttle latency", "th_l", PerfCountersBuilder::PRIO_CRITICAL); @@ -4929,15 +4940,20 @@ void BlueStore::_init_logger() "Average read latency", "r_l", PerfCountersBuilder::PRIO_CRITICAL); b.add_time_avg(l_bluestore_read_onode_meta_lat, "read_onode_meta_lat", - "Average read onode metadata latency"); + "Average read onode metadata latency", + "roml", PerfCountersBuilder::PRIO_USEFUL); b.add_time_avg(l_bluestore_read_wait_aio_lat, "read_wait_aio_lat", - "Average read latency"); + "Average read latency", + "rwal", PerfCountersBuilder::PRIO_USEFUL); b.add_time_avg(l_bluestore_compress_lat, "compress_lat", - "Average compress latency"); + "Average compress latency", + "_cpl", PerfCountersBuilder::PRIO_USEFUL); b.add_time_avg(l_bluestore_decompress_lat, "decompress_lat", - "Average decompress latency"); + "Average decompress latency", + "dcpl", PerfCountersBuilder::PRIO_USEFUL); b.add_time_avg(l_bluestore_csum_lat, "csum_lat", - "Average checksum latency"); + "Average checksum latency", + "csml", PerfCountersBuilder::PRIO_USEFUL); b.add_u64_counter(l_bluestore_compress_success_count, "compress_success_count", "Sum for beneficial compress ops"); b.add_u64_counter(l_bluestore_compress_rejected_count, "compress_rejected_count", @@ -4968,9 +4984,11 @@ void BlueStore::_init_logger() b.add_u64(l_bluestore_pinned_onodes, "bluestore_pinned_onodes", "Number of pinned onodes in cache"); b.add_u64_counter(l_bluestore_onode_hits, "bluestore_onode_hits", - "Sum for onode-lookups hit in the cache"); + "Count of onode cache lookup hits", + "o_ht", PerfCountersBuilder::PRIO_USEFUL); b.add_u64_counter(l_bluestore_onode_misses, "bluestore_onode_misses", - "Sum for onode-lookups missed in the cache"); + "Count of onode cache lookup misses", + "o_ms", PerfCountersBuilder::PRIO_USEFUL); b.add_u64_counter(l_bluestore_onode_shard_hits, "bluestore_onode_shard_hits", "Sum for onode-shard lookups hit in the cache"); b.add_u64_counter(l_bluestore_onode_shard_misses, @@ -5031,25 +5049,34 @@ void BlueStore::_init_logger() b.add_u64_counter(l_bluestore_read_eio, "bluestore_read_eio", "Read EIO errors propagated to high level callers"); b.add_u64_counter(l_bluestore_reads_with_retries, "bluestore_reads_with_retries", - "Read operations that required at least one retry due to failed checksum validation"); + "Read operations that required at least one retry due to failed checksum validation", + "rd_r", PerfCountersBuilder::PRIO_USEFUL); b.add_u64(l_bluestore_fragmentation, "bluestore_fragmentation_micros", "How fragmented bluestore free space is (free extents / max possible number of free extents) * 1000"); b.add_time_avg(l_bluestore_omap_seek_to_first_lat, "omap_seek_to_first_lat", - "Average omap iterator seek_to_first call latency"); + "Average omap iterator seek_to_first call latency", + "osfl", PerfCountersBuilder::PRIO_USEFUL); b.add_time_avg(l_bluestore_omap_upper_bound_lat, "omap_upper_bound_lat", - "Average omap iterator upper_bound call latency"); + "Average omap iterator upper_bound call latency", + "oubl", PerfCountersBuilder::PRIO_USEFUL); b.add_time_avg(l_bluestore_omap_lower_bound_lat, "omap_lower_bound_lat", - "Average omap iterator lower_bound call latency"); + "Average omap iterator lower_bound call latency", + "olbl", PerfCountersBuilder::PRIO_USEFUL); b.add_time_avg(l_bluestore_omap_next_lat, "omap_next_lat", - "Average omap iterator next call latency"); + "Average omap iterator next call latency", + "onxl", PerfCountersBuilder::PRIO_USEFUL); b.add_time_avg(l_bluestore_omap_get_keys_lat, "omap_get_keys_lat", - "Average omap get_keys call latency"); + "Average omap get_keys call latency", + "ogkl", PerfCountersBuilder::PRIO_USEFUL); b.add_time_avg(l_bluestore_omap_get_values_lat, "omap_get_values_lat", - "Average omap get_values call latency"); + "Average omap get_values call latency", + "ogvl", PerfCountersBuilder::PRIO_USEFUL); b.add_time_avg(l_bluestore_clist_lat, "clist_lat", - "Average collection listing latency"); + "Average collection listing latency", + "cl_l", PerfCountersBuilder::PRIO_USEFUL); b.add_time_avg(l_bluestore_remove_lat, "remove_lat", - "Average removal latency"); + "Average removal latency", + "rm_l", PerfCountersBuilder::PRIO_USEFUL); // Resulting size axis configuration for op histograms, values are in bytes PerfHistogramCommon::axis_config_d alloc_hist_x_axis_config{ From bafbeb6fca21374aebace7ae65b964c875d87d01 Mon Sep 17 00:00:00 2001 From: Daniel Pivonka Date: Mon, 11 Oct 2021 17:15:35 -0400 Subject: [PATCH 0132/2866] doc/cephadm: document setting ssh-user during adopt Signed-off-by: Daniel Pivonka --- doc/cephadm/adoption.rst | 6 ++++++ doc/cephadm/host-management.rst | 2 ++ 2 files changed, 8 insertions(+) diff --git a/doc/cephadm/adoption.rst b/doc/cephadm/adoption.rst index db4fded39816a..e06422a716cc2 100644 --- a/doc/cephadm/adoption.rst +++ b/doc/cephadm/adoption.rst @@ -118,6 +118,12 @@ Adoption process document for instructions that describe how to import existing ssh keys. + .. note:: + It is also possible to have cephadm use a non-root user to ssh + into cluster hosts. This user needs to have passwordless sudo access. + Use ``ceph cephadm set-user `` and copy the ssh key to that user. + See :ref:`cephadm-ssh-user` + #. Tell cephadm which hosts to manage: .. prompt:: bash # diff --git a/doc/cephadm/host-management.rst b/doc/cephadm/host-management.rst index 621f2a753b1ed..2253cb6e14df6 100644 --- a/doc/cephadm/host-management.rst +++ b/doc/cephadm/host-management.rst @@ -233,6 +233,8 @@ You will then need to restart the mgr daemon to reload the configuration with:: ceph mgr fail +.. _cephadm-ssh-user: + Configuring a different SSH user ---------------------------------- From b72704834f07f1fc94f4c51c29efa56639fdb020 Mon Sep 17 00:00:00 2001 From: Yuri Weinstein Date: Wed, 13 Oct 2021 09:04:57 -0700 Subject: [PATCH 0133/2866] doc: 15.2.15 Release Notes Signed-off-by: Yuri Weinstein --- doc/releases/index.rst | 1 + doc/releases/octopus.rst | 67 +++++++++++++++++++++++++++++++++++++++ doc/releases/releases.yml | 2 ++ 3 files changed, 70 insertions(+) diff --git a/doc/releases/index.rst b/doc/releases/index.rst index dca734adbfe5b..9daf9965a6add 100644 --- a/doc/releases/index.rst +++ b/doc/releases/index.rst @@ -68,6 +68,7 @@ Release timeline .. _16.2.0: pacific#v16-2-0-pacific .. _Octopus: octopus +.. _15.2.15: octopus#v15-2-15-octopus .. _15.2.14: octopus#v15-2-14-octopus .. _15.2.13: octopus#v15-2-13-octopus .. _15.2.12: octopus#v15-2-12-octopus diff --git a/doc/releases/octopus.rst b/doc/releases/octopus.rst index 90936175afecc..1de3f793f80ab 100644 --- a/doc/releases/octopus.rst +++ b/doc/releases/octopus.rst @@ -5,6 +5,73 @@ Octopus Octopus is the 15th stable release of Ceph. It is named after an order of 8-limbed cephalopods. +v15.2.15 Octopus +================ + +This is the 15th backport release in the Octopus series. We recommend all +users update to this release. + +Notable Changes +--------------- + +* auth,mon: don't log "unable to find a keyring" error when key is given (`pr#43312 `_, Ilya Dryomov) +* ceph-monstore-tool: use a large enough paxos/{first,last}_committed (`issue#38219 `_, `pr#43263 `_, Kefu Chai) +* ceph-volume/tests: retry when destroying osd (`pr#42547 `_, Guillaume Abrioux) +* ceph-volume: disable cache for blkid calls (`pr#41115 `_, Rafał Wądołowski) +* ceph-volume: fix batch report and respect ceph.conf config values (`pr#41715 `_, Andrew Schoen) +* ceph-volume: fix lvm activate --all --no-systemd (`pr#43268 `_, Dimitri Savineau) +* ceph-volume: fix lvm activate arguments (`pr#43117 `_, Dimitri Savineau) +* ceph-volume: fix lvm migrate without args (`pr#43111 `_, Dimitri Savineau) +* ceph-volume: fix raw list with logical partition (`pr#43088 `_, Guillaume Abrioux, Dimitri Savineau) +* ceph-volume: lvm batch: fast_allocations(): avoid ZeroDivisionError (`pr#42494 `_, Jonas Zeiger) +* ceph-volume: pvs --noheadings replace pvs --no-heading (`pr#43077 `_, FengJiankui) +* ceph-volume: remove --all ref from deactivate help (`pr#43097 `_, Dimitri Savineau) +* ceph-volume: support no_systemd with lvm migrate (`pr#43092 `_, Dimitri Savineau) +* ceph-volume: work around phantom atari partitions (`pr#42752 `_, Blaine Gardner) +* ceph.spec: selinux scripts respect CEPH_AUTO_RESTART_ON_UPGRADE (`pr#43234 `_, Dan van der Ster) +* cephadm: add thread ident to log messages (`pr#43133 `_, Michael Fritch) +* cephadm: default to quay.io, not docker.io (`pr#42533 `_, Sage Weil) +* cephadm: use quay, not docker (`pr#43094 `_, Sage Weil, Juan Miguel Olmo Martínez) +* cmake: Replace boost download url (`pr#42694 `_, Rafał Wądołowski) +* cmake: s/Python_EXECUTABLE/Python3_EXECUTABLE/ (`pr#43265 `_, Michael Fritch) +* common/buffer: fix SIGABRT in rebuild_aligned_size_and_memory (`pr#42975 `_, Yin Congmin) +* common/options: Set osd_client_message_cap to 256 (`pr#42616 `_, Mark Nelson) +* doc/ceph-volume: add lvm migrate/new-db/new-wal (`pr#43090 `_, Dimitri Savineau) +* Don't persist report data (`pr#42670 `_, Brad Hubbard) +* krbd: escape udev_enumerate_add_match_sysattr values (`pr#42968 `_, Ilya Dryomov) +* mgr/cephadm: pass --container-init to cephadm if specified (`pr#42666 `_, Tim Serong) +* mgr/dashboard: cephadm e2e start script: add --expanded option (`pr#42794 `_, Alfonso Martínez) +* mgr/dashboard: deprecated variable usage in Grafana dashboards (`pr#43189 `_, Patrick Seidensal) +* mgr/dashboard: Incorrect MTU mismatch warning (`pr#43186 `_, Aashish Sharma) +* mgr/dashboard: stats=false not working when listing buckets (`pr#42892 `_, Avan Thakkar) +* mgr/influx: use "N/A" for unknown hostname (`pr#43369 `_, Kefu Chai) +* mgr/prometheus: Fix metric types from gauge to counter (`pr#42674 `_, Patrick Seidensal) +* mon/OSDMonitor: account for PG merging in epoch_by_pg accounting (`pr#42837 `_, Dan van der Ster) +* mon/PGMap: remove DIRTY field in `ceph df detail` when cache tiering is not in use (`pr#42862 `_, Deepika Upadhyay) +* mon: return -EINVAL when handling unknown option in 'ceph osd pool get' (`pr#43266 `_, Zhao Cuicui) +* monitoring/grafana/cluster: use per-unit max and limit values (`pr#42675 `_, David Caro) +* monitoring: fix Physical Device Latency unit (`pr#42676 `_, Seena Fallah) +* os/bluestore: accept undecodable multi-block bluefs transactions on log (`pr#43024 `_, Igor Fedotov) +* os/bluestore: fix bluefs migrate command (`pr#43140 `_, Igor Fedotov) +* os/bluestore: fix using incomplete bluefs log when dumping it (`pr#43008 `_, Igor Fedotov) +* osd/OSD: mkfs need wait for transcation completely finish (`pr#43418 `_, Chen Fan) +* pybind/rbd: fix mirror_image_get_status (`pr#42971 `_, Ilya Dryomov, Will Smith) +* qa/mgr/dashboard: add extra wait to test (`pr#43352 `_, Ernesto Puerta) +* qa/suites/rados: use centos_8.3_container_tools_3.0.yaml (`pr#43102 `_, Sebastian Wagner) +* qa/tests: advanced version to 15.2.14 to match the latest release (`pr#42761 `_, Yuri Weinstein) +* qa/workunits/mon/test_mon_config_key: use subprocess.run() instead of proc.communicate() (`pr#42498 `_, Kefu Chai) +* rbd-mirror: add perf counters to snapshot replayed (`pr#42986 `_, Arthur Outhenin-Chalandre) +* rbd-mirror: fix potential async op tracker leak in start_image_replayers (`pr#42978 `_, Mykola Golub) +* rbd-mirror: unbreak one-way snapshot-based mirroring (`pr#43314 `_, Ilya Dryomov) +* rgw : add check for tenant provided in RGWCreateRole (`pr#43270 `_, caolei) +* rgw: avoid infinite loop when deleting a bucket (`issue#49206 `_, `pr#43272 `_, Jeegn Chen) +* rgw: fail as expected when set/delete-bucket-website attempted on a non-exis… (`pr#43424 `_, xiangrui meng) +* rgw: fix sts memory leak (`pr#43349 `_, yuliyang_yewu) +* rgw: remove quota soft threshold (`pr#43271 `_, Zulai Wang) +* rgw: when deleted obj removed in versioned bucket, extra del-marker added (`pr#43273 `_, J. Eric Ivancich) +* run-make-check.sh: Increase failure output log size (`pr#42849 `_, David Galloway) +* tools/erasure-code: new tool to encode/decode files (`pr#43407 `_, Mykola Golub) + v15.2.14 Octopus ================ diff --git a/doc/releases/releases.yml b/doc/releases/releases.yml index a1c2ed4fad846..c16c77fb257d2 100644 --- a/doc/releases/releases.yml +++ b/doc/releases/releases.yml @@ -33,6 +33,8 @@ releases: octopus: target_eol: 2022-06-01 releases: + - version: 15.2.15 + released: 2021-10-18 - version: 15.2.14 released: 2021-08-05 - version: 15.2.13 From 7120d5dc6feaa307c0dd8cdb30a375d6cf986ce5 Mon Sep 17 00:00:00 2001 From: Adam King Date: Wed, 13 Oct 2021 12:33:47 -0400 Subject: [PATCH 0134/2866] mgr/cephadm: disable agent by default to avoid issues while it is stabilized Signed-off-by: Adam King --- src/pybind/mgr/cephadm/module.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pybind/mgr/cephadm/module.py b/src/pybind/mgr/cephadm/module.py index 41476c1c99207..14bf025199cea 100644 --- a/src/pybind/mgr/cephadm/module.py +++ b/src/pybind/mgr/cephadm/module.py @@ -334,7 +334,7 @@ class CephadmOrchestrator(orchestrator.Orchestrator, MgrModule, Option( 'use_agent', type='bool', - default=True, + default=False, desc='Use cephadm agent on each host to gather and send metadata' ), Option( From 7f750fe31aa62f0055e43e86d4461d48baaaa7d3 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Tue, 12 Oct 2021 15:22:14 -0400 Subject: [PATCH 0135/2866] mgr/cephadm: add 'orch upgrade ls' to list available versions Optionally pass image name to query an alternative image/registry, or --tags to list tags instead of ceph versions. Signed-off-by: Sage Weil --- src/pybind/mgr/cephadm/module.py | 4 ++ src/pybind/mgr/cephadm/registry.py | 61 +++++++++++++++++++++++ src/pybind/mgr/cephadm/upgrade.py | 29 +++++++++++ src/pybind/mgr/orchestrator/_interface.py | 3 ++ src/pybind/mgr/orchestrator/module.py | 10 ++++ 5 files changed, 107 insertions(+) create mode 100644 src/pybind/mgr/cephadm/registry.py diff --git a/src/pybind/mgr/cephadm/module.py b/src/pybind/mgr/cephadm/module.py index 41476c1c99207..720422902cc07 100644 --- a/src/pybind/mgr/cephadm/module.py +++ b/src/pybind/mgr/cephadm/module.py @@ -2469,6 +2469,10 @@ def upgrade_check(self, image: str, version: str) -> str: def upgrade_status(self) -> orchestrator.UpgradeStatusSpec: return self.upgrade.upgrade_status() + @handle_orch_error + def upgrade_ls(self, image: Optional[str], tags: bool) -> List[str]: + return self.upgrade.upgrade_ls(image, tags) + @handle_orch_error def upgrade_start(self, image: str, version: str) -> str: if self.inventory.get_host_with_state("maintenance"): diff --git a/src/pybind/mgr/cephadm/registry.py b/src/pybind/mgr/cephadm/registry.py new file mode 100644 index 0000000000000..7b293a4c1a528 --- /dev/null +++ b/src/pybind/mgr/cephadm/registry.py @@ -0,0 +1,61 @@ +import requests +from typing import List, Dict, Tuple +from requests import Response + + +class Registry: + + def __init__(self, url: str): + self._url: str = url + + @property + def api_domain(self) -> str: + if self._url == 'docker.io': + return 'registry-1.docker.io' + return self._url + + def get_token(self, response: Response) -> str: + realm, params = self.parse_www_authenticate(response.headers['Www-Authenticate']) + r = requests.get(realm, params=params) + r.raise_for_status() + ret = r.json() + if 'access_token' in ret: + return ret['access_token'] + if 'token' in ret: + return ret['token'] + raise ValueError(f'Unknown token reply {ret}') + + def parse_www_authenticate(self, text: str) -> Tuple[str, Dict[str, str]]: + # 'Www-Authenticate': 'Bearer realm="https://auth.docker.io/token",service="registry.docker.io",scope="repository:ceph/ceph:pull"' + r: Dict[str, str] = {} + for token in text.split(','): + key, value = token.split('=', 1) + r[key] = value.strip('"') + realm = r.pop('Bearer realm') + return realm, r + + def get_tags(self, image: str) -> List[str]: + tags = [] + headers = {'Accept': 'application/json'} + url = f'https://{self.api_domain}/v2/{image}/tags/list' + while True: + r = requests.get(url, headers=headers) + if r.status_code == 401: + if 'Authorization' in headers: + raise ValueError('failed authentication') + token = self.get_token(r) + headers['Authorization'] = f'Bearer {token}' + continue + r.raise_for_status() + + new_tags = r.json()['tags'] + tags.extend(new_tags) + + if 'Link' not in r.headers: + break + + # strip < > brackets off and prepend the domain + url = f'https://{self.api_domain}' + r.headers['Link'].split(';')[0][1:-1] + continue + + return tags diff --git a/src/pybind/mgr/cephadm/upgrade.py b/src/pybind/mgr/cephadm/upgrade.py index d7162941985d3..e71c9c8bf3a1c 100644 --- a/src/pybind/mgr/cephadm/upgrade.py +++ b/src/pybind/mgr/cephadm/upgrade.py @@ -5,6 +5,7 @@ from typing import TYPE_CHECKING, Optional, Dict, List, Tuple import orchestrator +from cephadm.registry import Registry from cephadm.serve import CephadmServe from cephadm.services.cephadmservice import CephadmDaemonDeploySpec from cephadm.utils import ceph_release_to_major, name_to_config_section, CEPH_UPGRADE_ORDER, MONITORING_STACK_TYPES @@ -176,6 +177,34 @@ def _check_target_version(self, version: str) -> Optional[str]: return None + def upgrade_ls(self, image: Optional[str], tags: bool) -> List[str]: + if image: + reg_name, image = image.split('/', 1) + else: + reg_name, image = self.mgr.container_image_base.split('/', 1) + self.mgr.log.info(f'reg_name {reg_name} image {image}') + reg = Registry(reg_name) + versions = [] + ls = reg.get_tags(image) + if not tags: + for t in ls: + if t[0] != 'v': + continue + v = t[1:].split('.') + if len(v) != 3: + continue + if '-' in v[2]: + continue + versions.append('.'.join(v)) + ls = sorted( + versions, + key=lambda k: list(map(int, k.split('.'))), + reverse=True + ) + else: + ls = sorted(ls) + return ls + def upgrade_start(self, image: str, version: str) -> str: if self.mgr.mode != 'root': raise OrchestratorError('upgrade is not supported in %s mode' % ( diff --git a/src/pybind/mgr/orchestrator/_interface.py b/src/pybind/mgr/orchestrator/_interface.py index 158299885b1cb..ffc77784c3013 100644 --- a/src/pybind/mgr/orchestrator/_interface.py +++ b/src/pybind/mgr/orchestrator/_interface.py @@ -642,6 +642,9 @@ def apply_alertmanager(self, spec: ServiceSpec) -> OrchResult[str]: def upgrade_check(self, image: Optional[str], version: Optional[str]) -> OrchResult[str]: raise NotImplementedError() + def upgrade_ls(self, image: Optional[str], tags: bool) -> OrchResult[List[str]]: + raise NotImplementedError() + def upgrade_start(self, image: Optional[str], version: Optional[str]) -> OrchResult[str]: raise NotImplementedError() diff --git a/src/pybind/mgr/orchestrator/module.py b/src/pybind/mgr/orchestrator/module.py index a9130c7f044f5..6332f70449244 100644 --- a/src/pybind/mgr/orchestrator/module.py +++ b/src/pybind/mgr/orchestrator/module.py @@ -1327,6 +1327,16 @@ def _upgrade_check(self, raise_if_exception(completion) return HandleCommandResult(stdout=completion.result_str()) + @_cli_read_command('orch upgrade ls') + def _upgrade_ls(self, + image: Optional[str] = None, + tags: bool = False) -> HandleCommandResult: + """Check for available versions (or tags) we can upgrade to""" + completion = self.upgrade_ls(image, tags) + r = raise_if_exception(completion) + out = json.dumps(r, indent=4) + return HandleCommandResult(stdout=out) + @_cli_write_command('orch upgrade status') def _upgrade_status(self) -> HandleCommandResult: """Check service versions vs available and target containers""" From 8661ba180759e16daa550b43537d2bbc56a82551 Mon Sep 17 00:00:00 2001 From: Kefu Chai Date: Thu, 14 Oct 2021 07:38:09 +0800 Subject: [PATCH 0136/2866] common/pick_address: use string_view when appropriate we don't need a string when string_view is good enough. Signed-off-by: Kefu Chai --- src/common/pick_address.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/pick_address.cc b/src/common/pick_address.cc index 203e1d63f170b..1b7831c060634 100644 --- a/src/common/pick_address.cc +++ b/src/common/pick_address.cc @@ -308,7 +308,7 @@ static int fill_in_one_address( interfaces, numa_node); if (!found) { - std::string ip_type = ""; + std::string_view ip_type; if ((ipv & CEPH_PICK_ADDRESS_IPV4) && (ipv & CEPH_PICK_ADDRESS_IPV6)) { ip_type = "IPv4 or IPv6"; } else if (ipv & CEPH_PICK_ADDRESS_IPV4) { From 3d97d6d98f0a72272362c154b5c74d158c572eaa Mon Sep 17 00:00:00 2001 From: Venky Shankar Date: Wed, 13 Oct 2021 11:02:15 +0530 Subject: [PATCH 0137/2866] doc / cephfs: health message codes should be permalinks ... so that such links can be included in alert warnings. Additionally, document some other health warnings. Credit to @pcuzner to point out that not all health warnings have been documented. Signed-off-by: Venky Shankar --- doc/cephfs/health-messages.rst | 86 ++++++++++++++++++++++++++++++---- 1 file changed, 78 insertions(+), 8 deletions(-) diff --git a/doc/cephfs/health-messages.rst b/doc/cephfs/health-messages.rst index 28ceb704a9d03..790fc3fdedbaf 100644 --- a/doc/cephfs/health-messages.rst +++ b/doc/cephfs/health-messages.rst @@ -72,7 +72,8 @@ slow requests. This page lists the health checks raised by MDS daemons. For the checks from other daemons, please see :ref:`health-checks`. -* ``MDS_TRIM`` +``MDS_TRIM`` +------------ Message "Behind on trimming..." @@ -85,7 +86,9 @@ other daemons, please see :ref:`health-checks`. too slowly, or a software bug is preventing trimming, then this health message may appear. The threshold for this message to appear is controlled by the config option ``mds_log_warn_factor``, the default is 2.0. -* ``MDS_HEALTH_CLIENT_LATE_RELEASE``, ``MDS_HEALTH_CLIENT_LATE_RELEASE_MANY`` + +``MDS_HEALTH_CLIENT_LATE_RELEASE``, ``MDS_HEALTH_CLIENT_LATE_RELEASE_MANY`` +--------------------------------------------------------------------------- Message "Client *name* failing to respond to capability release" @@ -96,7 +99,9 @@ other daemons, please see :ref:`health-checks`. is unresponsive or buggy, it might fail to do so promptly or fail to do so at all. This message appears if a client has taken longer than ``session_timeout`` (default 60s) to comply. -* ``MDS_CLIENT_RECALL``, ``MDS_HEALTH_CLIENT_RECALL_MANY`` + +``MDS_CLIENT_RECALL``, ``MDS_HEALTH_CLIENT_RECALL_MANY`` +-------------------------------------------------------- Message "Client *name* failing to respond to cache pressure" @@ -111,7 +116,9 @@ other daemons, please see :ref:`health-checks`. ``mds_recall_warning_threshold`` capabilities (decaying with a half-life of ``mds_recall_max_decay_rate``) within the last ``mds_recall_warning_decay_rate`` second. -* ``MDS_CLIENT_OLDEST_TID``, ``MDS_CLIENT_OLDEST_TID_MANY`` + +``MDS_CLIENT_OLDEST_TID``, ``MDS_CLIENT_OLDEST_TID_MANY`` +--------------------------------------------------------- Message "Client *name* failing to advance its oldest client/flush tid" @@ -124,7 +131,9 @@ other daemons, please see :ref:`health-checks`. appears if a client appears to have more than ``max_completed_requests`` (default 100000) requests that are complete on the MDS side but haven't yet been accounted for in the client's *oldest tid* value. -* ``MDS_DAMAGE`` + +``MDS_DAMAGE`` +-------------- Message "Metadata damage detected" @@ -135,7 +144,9 @@ other daemons, please see :ref:`health-checks`. client accesses to the damaged subtree will return IO errors. Use the ``damage ls`` admin socket command to get more detail on the damage. This message appears as soon as any damage is encountered. -* ``MDS_HEALTH_READ_ONLY`` + +``MDS_HEALTH_READ_ONLY`` +------------------------ Message "MDS in read-only mode" @@ -145,7 +156,9 @@ other daemons, please see :ref:`health-checks`. MDS will go into readonly mode if it encounters a write error while writing to the metadata pool, or if forced to by an administrator using the *force_readonly* admin socket command. -* ``MDS_SLOW_REQUEST`` + +``MDS_SLOW_REQUEST`` +-------------------- Message "*N* slow requests are blocked" @@ -157,7 +170,9 @@ other daemons, please see :ref:`health-checks`. Use the ``ops`` admin socket command to list outstanding metadata operations. This message appears if any client requests have taken longer than ``mds_op_complaint_time`` (default 30s). -* ``MDS_CACHE_OVERSIZED`` + +``MDS_CACHE_OVERSIZED`` +----------------------- Message "Too many inodes in cache" @@ -168,3 +183,58 @@ other daemons, please see :ref:`health-checks`. the actual cache size (in memory) is at least 50% greater than ``mds_cache_memory_limit`` (default 1GB). Modify ``mds_health_cache_threshold`` to set the warning ratio. + +``FS_WITH_FAILED_MDS`` +---------------------- + + Message + "Some MDS ranks do not have standby replacements" + + Description + Normally, a failed MDS rank will be replaced by a standby MDS. This situation + is transient and is not considered critical. However, if there are no standby + MDSs available to replace an active MDS rank, this health warning is generated. + +``MDS_INSUFFICIENT_STANDBY`` +---------------------------- + + Message + "Insufficient number of available standby(-replay) MDS daemons than configured" + + Description + The minimum number of standby(-replay) MDS daemons can be configured by setting + ``standby_count_wanted`` configuration variable. This health warning is generated + when the configured value mismatches the number of standby(-replay) MDS daemons + available. + +``FS_DEGRADED`` +---------------------------- + + Message + "Some MDS ranks have been marked failed or damaged" + + Description + When one or more MDS rank ends up in failed or damaged state due to + an unrecoverable error. The file system may be partially or fully + unavailable when one (or more) ranks are offline. + +``MDS_UP_LESS_THAN_MAX`` +---------------------------- + + Message + "Number of active ranks are less than configured number of maximum MDSs" + + Description + The maximum number of MDS ranks can be configured by setting ``max_mds`` + configuration variable. This health warning is generated when the number + of MDS ranks falls below this configured value. + +``MDS_ALL_DOWN`` +---------------------------- + + Message + "None of the MDS ranks are available (file system offline)" + + Description + All MDS ranks are unavailable resulting in the file system to be completely + offline. From 45d9eff5e4faf9f2fe91a48ec71e7147c589e771 Mon Sep 17 00:00:00 2001 From: Kefu Chai Date: Thu, 14 Oct 2021 07:51:31 +0800 Subject: [PATCH 0138/2866] common/pick_address: refactor pick_addresses() * consolidate the logic handling CEPH_PICK_ADDRESS_PREFER_IPV4 using std::sort(). this might be overkill. but it helps to explain what CEPH_PICK_ADDRESS_PREFER_IPV4 is for, and helps to dedup the code to order the addresses. * let fill_in_one_address() return an optional. more readable this way * early return if the required address is not found, instead of checking variables like ipv4_r * rename fill_in_one_address() to get_one_address() to reflect the change of the function's return value's type Signed-off-by: Kefu Chai --- src/common/pick_address.cc | 79 ++++++++++++++++---------------------- 1 file changed, 33 insertions(+), 46 deletions(-) diff --git a/src/common/pick_address.cc b/src/common/pick_address.cc index 1b7831c060634..0ab19292582b5 100644 --- a/src/common/pick_address.cc +++ b/src/common/pick_address.cc @@ -294,13 +294,12 @@ void pick_addresses(CephContext *cct, int needs) } #endif // !WITH_SEASTAR -static int fill_in_one_address( +static std::optional get_one_address( CephContext *cct, const struct ifaddrs *ifa, unsigned ipv, const string &networks, const string &interfaces, - entity_addrvec_t *addrs, int numa_node = -1) { const struct sockaddr *found = find_ip_in_subnet_list(cct, ifa, ipv, @@ -318,7 +317,7 @@ static int fill_in_one_address( } lderr(cct) << "unable to find any " << ip_type << " address in networks '" << networks << "' interfaces '" << interfaces << "'" << dendl; - return -1; + return {}; } char buf[INET6_ADDRSTRLEN]; @@ -334,16 +333,15 @@ static int fill_in_one_address( NI_NUMERICHOST); if (err != 0) { lderr(cct) << "unable to convert chosen address to string: " << gai_strerror(err) << dendl; - return -1; + return {}; } entity_addr_t addr; - bool r = addr.parse(buf); - if (!r) { - return -1; + if (addr.parse(buf)) { + return addr; + } else { + return {}; } - addrs->v.push_back(addr); - return 0; } int pick_addresses( @@ -417,31 +415,19 @@ int pick_addresses( } if (addr.is_blank_ip() && !networks.empty()) { - int ipv4_r = !(ipv & CEPH_PICK_ADDRESS_IPV4) ? 0 : -1; - int ipv6_r = !(ipv & CEPH_PICK_ADDRESS_IPV6) ? 0 : -1; // note: pass in ipv to filter the matching addresses - if ((ipv & CEPH_PICK_ADDRESS_IPV4) && - (flags & CEPH_PICK_ADDRESS_PREFER_IPV4)) { - ipv4_r = fill_in_one_address(cct, ifa, CEPH_PICK_ADDRESS_IPV4, - networks, interfaces, - addrs, - preferred_numa_node); - } - if (ipv & CEPH_PICK_ADDRESS_IPV6) { - ipv6_r = fill_in_one_address(cct, ifa, CEPH_PICK_ADDRESS_IPV6, - networks, interfaces, - addrs, - preferred_numa_node); - } - if ((ipv & CEPH_PICK_ADDRESS_IPV4) && - !(flags & CEPH_PICK_ADDRESS_PREFER_IPV4)) { - ipv4_r = fill_in_one_address(cct, ifa, CEPH_PICK_ADDRESS_IPV4, - networks, interfaces, - addrs, - preferred_numa_node); - } - if (ipv4_r < 0 || ipv6_r < 0) { - return -1; + for (auto pick_mask : {CEPH_PICK_ADDRESS_IPV4, CEPH_PICK_ADDRESS_IPV6}) { + if (ipv & pick_mask) { + auto ip_addr = get_one_address(cct, ifa, pick_mask, + networks, interfaces, + preferred_numa_node); + if (ip_addr) { + addrs->v.push_back(*ip_addr); + } else { + // picked but not found + return -1; + } + } } } @@ -450,22 +436,23 @@ int pick_addresses( // ipv4 and/or ipv6? if (addrs->v.empty()) { addr.set_type(entity_addr_t::TYPE_MSGR2); - if ((ipv & CEPH_PICK_ADDRESS_IPV4) && - (flags & CEPH_PICK_ADDRESS_PREFER_IPV4)) { - addr.set_family(AF_INET); - addrs->v.push_back(addr); - } - if (ipv & CEPH_PICK_ADDRESS_IPV6) { - addr.set_family(AF_INET6); - addrs->v.push_back(addr); - } - if ((ipv & CEPH_PICK_ADDRESS_IPV4) && - !(flags & CEPH_PICK_ADDRESS_PREFER_IPV4)) { - addr.set_family(AF_INET); - addrs->v.push_back(addr); + for (auto pick_mask : {CEPH_PICK_ADDRESS_IPV4, CEPH_PICK_ADDRESS_IPV6}) { + if (ipv & pick_mask) { + addr.set_family(pick_mask == CEPH_PICK_ADDRESS_IPV4 ? AF_INET : AF_INET6); + addrs->v.push_back(addr); + } } } + std::sort(addrs->v.begin(), addrs->v.end(), + [flags] (entity_addr_t& lhs, entity_addr_t& rhs) { + if (flags & CEPH_PICK_ADDRESS_PREFER_IPV4) { + return lhs.is_ipv4() && rhs.is_ipv6(); + } else { + return lhs.is_ipv6() && rhs.is_ipv4(); + } + }); + // msgr2 or legacy or both? if (msgrv == (CEPH_PICK_ADDRESS_MSGR1 | CEPH_PICK_ADDRESS_MSGR2)) { vector v; From f2477d97c2541b398a1266cedf7f05a62dcea862 Mon Sep 17 00:00:00 2001 From: Kefu Chai Date: Thu, 14 Oct 2021 14:39:55 +0800 Subject: [PATCH 0139/2866] cmake: only pass --verbose when VERBOSE env variable is set so that the setuptools is less chatty by default when building python bindings. Signed-off-by: Kefu Chai --- cmake/modules/Distutils.cmake | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/cmake/modules/Distutils.cmake b/cmake/modules/Distutils.cmake index 8dc69f0af5185..2c0df43dea6a3 100644 --- a/cmake/modules/Distutils.cmake +++ b/cmake/modules/Distutils.cmake @@ -75,6 +75,9 @@ function(distutils_add_cython_module target name src) endif() set(output_dir "${CYTHON_MODULE_DIR}/lib.3") set(setup_py ${CMAKE_CURRENT_SOURCE_DIR}/setup.py) + if(DEFINED ENV{VERBOSE}) + set(maybe_verbose --verbose) + endif() add_custom_command( OUTPUT ${output_dir}/${name}${ext_suffix} COMMAND @@ -89,7 +92,7 @@ function(distutils_add_cython_module target name src) CYTHON_BUILD_DIR=${CMAKE_CURRENT_BINARY_DIR} CEPH_LIBDIR=${CMAKE_LIBRARY_OUTPUT_DIRECTORY} ${Python3_EXECUTABLE} ${setup_py} - build --verbose --build-base ${CYTHON_MODULE_DIR} + build ${maybe_verbose} --build-base ${CYTHON_MODULE_DIR} --build-platlib ${output_dir} MAIN_DEPENDENCY ${src} DEPENDS ${setup_py} @@ -107,6 +110,9 @@ function(distutils_install_cython_module name) if(DU_DISABLE_VTA AND HAS_VTA) set(CFLAG_DISABLE_VTA -fno-var-tracking-assignments) endif() + if(DEFINED ENV{VERBOSE}) + set(maybe_verbose --verbose) + endif() install(CODE " set(ENV{CC} \"${PY_CC}\") set(ENV{LDSHARED} \"${PY_LDSHARED}\") @@ -130,12 +136,12 @@ function(distutils_install_cython_module name) execute_process( COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/setup.py - build --verbose --build-base ${CYTHON_MODULE_DIR} + build ${maybe_verbose} --build-base ${CYTHON_MODULE_DIR} --build-platlib ${CYTHON_MODULE_DIR}/lib.3 build_ext --cython-c-in-temp --build-temp ${CMAKE_CURRENT_BINARY_DIR} --cython-include-dirs ${PROJECT_SOURCE_DIR}/src/pybind/rados install \${options} --single-version-externally-managed --record /dev/null egg_info --egg-base ${CMAKE_CURRENT_BINARY_DIR} - --verbose + ${maybe_verbose} WORKING_DIRECTORY \"${CMAKE_CURRENT_SOURCE_DIR}\" RESULT_VARIABLE install_res) if(NOT \"\${install_res}\" STREQUAL 0) From 55eae0bd99d31d13df02fba93d1b7fe9aa7de32d Mon Sep 17 00:00:00 2001 From: haoyixing Date: Thu, 14 Oct 2021 15:08:16 +0800 Subject: [PATCH 0140/2866] rbd: correct image_spec() output when rbd-nbd Detach/Unmap error Signed-off-by: haoyixing --- src/tools/rbd_nbd/rbd-nbd.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tools/rbd_nbd/rbd-nbd.cc b/src/tools/rbd_nbd/rbd-nbd.cc index 60642edc11c79..1f8eb54fa016a 100644 --- a/src/tools/rbd_nbd/rbd-nbd.cc +++ b/src/tools/rbd_nbd/rbd-nbd.cc @@ -128,7 +128,7 @@ struct Config { std::string spec = poolname + "/"; if (!nsname.empty()) { - spec += "/" + nsname; + spec += nsname + "/"; } spec += imgname; From bba537988b09e5f9cce181570b06c4c8edbc0f3d Mon Sep 17 00:00:00 2001 From: Radoslaw Zarzynski Date: Wed, 13 Oct 2021 12:21:04 +0000 Subject: [PATCH 0141/2866] crimson/osd: print the actual reason on bind failure. Signed-off-by: Radoslaw Zarzynski --- src/crimson/osd/heartbeat.cc | 2 +- src/crimson/osd/osd.cc | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/crimson/osd/heartbeat.cc b/src/crimson/osd/heartbeat.cc index dc9669e016ffa..3c38eb8636a52 100644 --- a/src/crimson/osd/heartbeat.cc +++ b/src/crimson/osd/heartbeat.cc @@ -77,7 +77,7 @@ Heartbeat::start_messenger(crimson::net::Messenger& msgr, return msgr.start({this}); }, crimson::net::Messenger::bind_ertr::all_same_way( [] (const std::error_code& e) { - logger().error("heartbeat messenger bind(): address range is unavailable."); + logger().error("heartbeat messenger bind(): {}", e); ceph_abort(); })); } diff --git a/src/crimson/osd/osd.cc b/src/crimson/osd/osd.cc index f2b4cde54bcc2..02ef518bb3197 100644 --- a/src/crimson/osd/osd.cc +++ b/src/crimson/osd/osd.cc @@ -341,7 +341,7 @@ seastar::future<> OSD::start() return cluster_msgr->start(dispatchers); }, crimson::net::Messenger::bind_ertr::all_same_way( [] (const std::error_code& e) { - logger().error("cluster messenger bind(): address range is unavailable."); + logger().error("cluster messenger bind(): {}", e); ceph_abort(); })), public_msgr->bind(pick_addresses(CEPH_PICK_ADDRESS_PUBLIC)) @@ -349,7 +349,7 @@ seastar::future<> OSD::start() return public_msgr->start(dispatchers); }, crimson::net::Messenger::bind_ertr::all_same_way( [] (const std::error_code& e) { - logger().error("public messenger bind(): address range is unavailable."); + logger().error("public messenger bind(): {}", e); ceph_abort(); }))); }).then_unpack([this] { From 86ec66a9b5cb9f59eb25a58dfa7fd21b592480e9 Mon Sep 17 00:00:00 2001 From: Chunsong Feng Date: Wed, 13 Oct 2021 03:33:02 +0000 Subject: [PATCH 0142/2866] msgs/async/dpdk: fix dpdk build error Signed-off-by: Chunsong Feng --- src/msg/async/dpdk/DPDK.h | 6 +++--- src/msg/async/dpdk/DPDKStack.cc | 13 +++++++------ src/msg/async/dpdk/DPDKStack.h | 2 +- src/msg/async/dpdk/EventDPDK.cc | 2 +- src/msg/async/dpdk/EventDPDK.h | 2 +- src/msg/async/dpdk/TCP.cc | 2 +- src/msg/async/dpdk/TCP.h | 4 ++-- src/msg/async/dpdk/dpdk_rte.cc | 2 +- 8 files changed, 17 insertions(+), 16 deletions(-) diff --git a/src/msg/async/dpdk/DPDK.h b/src/msg/async/dpdk/DPDK.h index eadba40fb4912..47c09340d63e4 100644 --- a/src/msg/async/dpdk/DPDK.h +++ b/src/msg/async/dpdk/DPDK.h @@ -44,7 +44,6 @@ #include "net.h" #include "toeplitz.h" - struct free_deleter { void operator()(void* p) { ::free(p); } }; @@ -814,7 +813,7 @@ class DPDKDevice { if (ret != 0) { rte_exit(EXIT_FAILURE, "Cannot initialise port %u\n", _port_idx); } - string name(std::string("port") + std::to_string(port_idx)); + std::string name(std::string("port") + std::to_string(port_idx)); PerfCountersBuilder plb(cct, name, l_dpdk_dev_first, l_dpdk_dev_last); plb.add_u64_counter(l_dpdk_dev_rx_mcast, "dpdk_device_receive_multicast_packets", "DPDK received multicast packets"); @@ -855,7 +854,8 @@ class DPDKDevice { } const rss_key_type& rss_key() const { return _rss_key; } uint16_t hw_queues_count() { return _num_queues; } - std::unique_ptr init_local_queue(CephContext *c, EventCenter *center, string hugepages, uint16_t qid) { + std::unique_ptr init_local_queue(CephContext *c, + EventCenter *center, std::string hugepages, uint16_t qid) { std::unique_ptr qp; qp = std::unique_ptr(new DPDKQueuePair(c, center, this, qid)); return qp; diff --git a/src/msg/async/dpdk/DPDKStack.cc b/src/msg/async/dpdk/DPDKStack.cc index 2bfe45e9b44c3..a3451eec76966 100644 --- a/src/msg/async/dpdk/DPDKStack.cc +++ b/src/msg/async/dpdk/DPDKStack.cc @@ -142,11 +142,12 @@ void DPDKWorker::initialize() } } -using AvailableIPAddress = std::tuple; +using AvailableIPAddress = std::tuple; static bool parse_available_address( - const string &ips, const string &gates, const string &masks, vector &res) + const std::string &ips, const std::string &gates, + const std::string &masks, std::vector &res) { - vector ip_vec, gate_vec, mask_vec; + std::vector ip_vec, gate_vec, mask_vec; string_to_vec(ip_vec, ips); string_to_vec(gate_vec, gates); string_to_vec(mask_vec, masks); @@ -159,7 +160,7 @@ static bool parse_available_address( return true; } -static bool match_available_address(const vector &avails, +static bool match_available_address(const std::vector &avails, const entity_addr_t &ip, int &res) { for (size_t i = 0; i < avails.size(); ++i) { @@ -178,7 +179,7 @@ static bool match_available_address(const vector &avails, DPDKWorker::Impl::Impl(CephContext *cct, unsigned i, EventCenter *c, std::shared_ptr dev) : id(i), _netif(cct, dev, c), _dev(dev), _inet(cct, c, &_netif) { - vector tuples; + std::vector tuples; bool parsed = parse_available_address(cct->_conf.get_val("ms_dpdk_host_ipv4_addr"), cct->_conf.get_val("ms_dpdk_gateway_ipv4_addr"), cct->_conf.get_val("ms_dpdk_netmask_ipv4_addr"), tuples); @@ -263,7 +264,7 @@ void DPDKStack::spawn_worker(std::function &&func) break; } } - void *adapted_func = static_cast(funcs.back()); + void *adapted_func = static_cast(&funcs.back()); dpdk::eal::execute_on_master([adapted_func, core_id, this]() { int r = rte_eal_remote_launch(dpdk_thread_adaptor, adapted_func, core_id); if (r < 0) { diff --git a/src/msg/async/dpdk/DPDKStack.h b/src/msg/async/dpdk/DPDKStack.h index 2bd52d00245ec..acbac7b2011ba 100644 --- a/src/msg/async/dpdk/DPDKStack.h +++ b/src/msg/async/dpdk/DPDKStack.h @@ -247,7 +247,7 @@ class DPDKWorker : public Worker { }; class DPDKStack : public NetworkStack { - vector > funcs; + std::vector > funcs; virtual Worker* create_worker(CephContext *c, unsigned worker_id) override { return new DPDKWorker(c, worker_id); diff --git a/src/msg/async/dpdk/EventDPDK.cc b/src/msg/async/dpdk/EventDPDK.cc index 5d291716c719f..a38ddcc99b089 100644 --- a/src/msg/async/dpdk/EventDPDK.cc +++ b/src/msg/async/dpdk/EventDPDK.cc @@ -67,7 +67,7 @@ int DPDKDriver::resize_events(int newsize) return 0; } -int DPDKDriver::event_wait(vector &fired_events, struct timeval *tvp) +int DPDKDriver::event_wait(std::vector &fired_events, struct timeval *tvp) { int num_events = 512; int events[num_events]; diff --git a/src/msg/async/dpdk/EventDPDK.h b/src/msg/async/dpdk/EventDPDK.h index 541c2210e01da..ccf2cd28d47ac 100644 --- a/src/msg/async/dpdk/EventDPDK.h +++ b/src/msg/async/dpdk/EventDPDK.h @@ -33,7 +33,7 @@ class DPDKDriver : public EventDriver { int add_event(int fd, int cur_mask, int add_mask) override; int del_event(int fd, int cur_mask, int del_mask) override; int resize_events(int newsize) override; - int event_wait(vector &fired_events, struct timeval *tp) override; + int event_wait(std::vector &fired_events, struct timeval *tp) override; bool need_wakeup() override { return false; } }; diff --git a/src/msg/async/dpdk/TCP.cc b/src/msg/async/dpdk/TCP.cc index 26f29e10f75ac..86c80487c4611 100644 --- a/src/msg/async/dpdk/TCP.cc +++ b/src/msg/async/dpdk/TCP.cc @@ -222,7 +222,7 @@ void tcp::respond_with_reset(tcp_hdr* rth, ipaddr local_ip, ipaddr f #undef dout_prefix #define dout_prefix _prefix(_dout) template -ostream& tcp::tcb::_prefix(std::ostream *_dout) { +std::ostream& tcp::tcb::_prefix(std::ostream *_dout) { return *_dout << "tcp " << _local_ip << ":" << _local_port << " -> " << _foreign_ip << ":" << _foreign_port << " tcb(" << this << " fd=" << fd << " s=" << _state << ")."; } diff --git a/src/msg/async/dpdk/TCP.h b/src/msg/async/dpdk/TCP.h index 79fd6fbb13096..cf76d3279c03a 100644 --- a/src/msg/async/dpdk/TCP.h +++ b/src/msg/async/dpdk/TCP.h @@ -395,7 +395,7 @@ class tcp { public: // callback void close_final_cleanup(); - ostream& _prefix(std::ostream *_dout); + std::ostream& _prefix(std::ostream *_dout); public: tcb(tcp& t, connid id); @@ -685,7 +685,7 @@ class tcp { uint16_t _port; int _fd = -1; int16_t _errno; - queue _q; + std::queue _q; size_t _q_max_length; private: diff --git a/src/msg/async/dpdk/dpdk_rte.cc b/src/msg/async/dpdk/dpdk_rte.cc index 0937040a00b43..872d39d519de4 100644 --- a/src/msg/async/dpdk/dpdk_rte.cc +++ b/src/msg/async/dpdk/dpdk_rte.cc @@ -92,7 +92,7 @@ namespace dpdk { t = std::thread([&]() { // TODO: Inherit these from the app parameters - "opts" std::vector> args { - string2vector(string("ceph")), + string2vector("ceph"), string2vector("-c"), string2vector(c->_conf.get_val("ms_dpdk_coremask")), string2vector("-n"), string2vector(c->_conf->ms_dpdk_memory_channel), }; From f696c365a2e8d4c6d46a0a104751754da182ac7c Mon Sep 17 00:00:00 2001 From: Chunsong Feng Date: Wed, 13 Oct 2021 03:55:08 +0000 Subject: [PATCH 0143/2866] src/msg/dpdk: reserve funcs capacity to avoid reallocation When a new vector is added larger than then current vector capacity, it reallocates space. lamda function accesses the previous adress will cause a segment fault. Therefore, reserve sufficient funcs space to avoid reallocation. Fixes: https://tracker.ceph.com/issues/52907 Signed-off-by: Chunsong Feng --- src/msg/async/dpdk/DPDKStack.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/msg/async/dpdk/DPDKStack.h b/src/msg/async/dpdk/DPDKStack.h index acbac7b2011ba..4eb8604c7fbed 100644 --- a/src/msg/async/dpdk/DPDKStack.h +++ b/src/msg/async/dpdk/DPDKStack.h @@ -254,8 +254,9 @@ class DPDKStack : public NetworkStack { } public: - explicit DPDKStack(CephContext *cct): NetworkStack(cct) - {} + explicit DPDKStack(CephContext *cct): NetworkStack(cct) { + funcs.reserve(cct->_conf->ms_async_op_threads); + } virtual bool support_local_listen_table() const override { return true; } virtual void spawn_worker(std::function &&func) override; From f86c146cda26d1eb9b1334c1f8339ba622d00b80 Mon Sep 17 00:00:00 2001 From: Or Ozeri Date: Thu, 14 Oct 2021 12:56:09 +0300 Subject: [PATCH 0144/2866] msg/async: support disabling data crc for protocol v2 In protocol v1, the ms_crc_data config option allowed for globally disabling the crc calculation and verification for data segments. This option was so far ignored in protocol v2 implementation. This commit extends v2 to respect this configuration option. When the data crc is disabled, zeros will be sent instead of the supposed calculated crc. Signed-off-by: Or Ozeri --- src/crimson/net/ProtocolV2.h | 8 ++++++-- src/msg/async/ProtocolV2.cc | 6 ++++-- src/msg/async/frames_v2.cc | 19 +++++++++++++------ src/msg/async/frames_v2.h | 6 ++++-- src/test/msgr/test_frames_v2.cc | 6 ++++-- 5 files changed, 31 insertions(+), 14 deletions(-) diff --git a/src/crimson/net/ProtocolV2.h b/src/crimson/net/ProtocolV2.h index ab6ad86c1bfb9..2ad0e4941366f 100644 --- a/src/crimson/net/ProtocolV2.h +++ b/src/crimson/net/ProtocolV2.h @@ -124,8 +124,12 @@ class ProtocolV2 final : public Protocol { ceph::crypto::onwire::rxtx_t session_stream_handlers; ceph::compression::onwire::rxtx_t session_comp_handlers; - ceph::msgr::v2::FrameAssembler tx_frame_asm{&session_stream_handlers, false, &session_comp_handlers}; - ceph::msgr::v2::FrameAssembler rx_frame_asm{&session_stream_handlers, false, &session_comp_handlers}; + ceph::msgr::v2::FrameAssembler tx_frame_asm{ + &session_stream_handlers, false, common::local_conf()->ms_crc_data, + &session_comp_handlers}; + ceph::msgr::v2::FrameAssembler rx_frame_asm{ + &session_stream_handlers, false, common::local_conf()->ms_crc_data, + &session_comp_handlers}; ceph::bufferlist rx_preamble; ceph::msgr::v2::segment_bls_t rx_segments_data; diff --git a/src/msg/async/ProtocolV2.cc b/src/msg/async/ProtocolV2.cc index d13f0ead3aced..a176fc2c808ac 100644 --- a/src/msg/async/ProtocolV2.cc +++ b/src/msg/async/ProtocolV2.cc @@ -96,8 +96,10 @@ ProtocolV2::ProtocolV2(AsyncConnection *connection) replacing(false), can_write(false), bannerExchangeCallback(nullptr), - tx_frame_asm(&session_stream_handlers, false, &session_compression_handlers), - rx_frame_asm(&session_stream_handlers, false, &session_compression_handlers), + tx_frame_asm(&session_stream_handlers, false, cct->_conf->ms_crc_data, + &session_compression_handlers), + rx_frame_asm(&session_stream_handlers, false, cct->_conf->ms_crc_data, + &session_compression_handlers), next_tag(static_cast(0)), keepalive(false) { } diff --git a/src/msg/async/frames_v2.cc b/src/msg/async/frames_v2.cc index 8a29753476364..e0c41fdb64c17 100644 --- a/src/msg/async/frames_v2.cc +++ b/src/msg/async/frames_v2.cc @@ -108,7 +108,7 @@ bufferlist FrameAssembler::asm_crc_rev0(const preamble_block_t& preamble, frame_bl.append(reinterpret_cast(&preamble), sizeof(preamble)); for (size_t i = 0; i < m_descs.size(); i++) { ceph_assert(segment_bls[i].length() == m_descs[i].logical_len); - epilogue.crc_values[i] = segment_bls[i].crc32c(-1); + epilogue.crc_values[i] = m_with_data_crc ? segment_bls[i].crc32c(-1) : 0; if (segment_bls[i].length() > 0) { frame_bl.claim_append(segment_bls[i]); } @@ -161,7 +161,7 @@ bufferlist FrameAssembler::asm_crc_rev1(const preamble_block_t& preamble, ceph_assert(segment_bls[0].length() == m_descs[0].logical_len); if (segment_bls[0].length() > 0) { - uint32_t crc = segment_bls[0].crc32c(-1); + uint32_t crc = m_with_data_crc ? segment_bls[0].crc32c(-1) : 0; frame_bl.claim_append(segment_bls[0]); encode(crc, frame_bl); } @@ -171,7 +171,8 @@ bufferlist FrameAssembler::asm_crc_rev1(const preamble_block_t& preamble, for (size_t i = 1; i < m_descs.size(); i++) { ceph_assert(segment_bls[i].length() == m_descs[i].logical_len); - epilogue.crc_values[i - 1] = segment_bls[i].crc32c(-1); + epilogue.crc_values[i - 1] = + m_with_data_crc ? segment_bls[i].crc32c(-1) : 0; if (segment_bls[i].length() > 0) { frame_bl.claim_append(segment_bls[i]); } @@ -341,7 +342,9 @@ bool FrameAssembler::disasm_all_crc_rev0(bufferlist segment_bls[], for (size_t i = 0; i < m_descs.size(); i++) { ceph_assert(segment_bls[i].length() == m_descs[i].logical_len); - check_segment_crc(segment_bls[i], epilogue->crc_values[i]); + if (m_with_data_crc) { + check_segment_crc(segment_bls[i], epilogue->crc_values[i]); + } } return !(epilogue->late_flags & FRAME_LATE_FLAG_ABORTED); } @@ -374,7 +377,9 @@ void FrameAssembler::disasm_first_crc_rev1(bufferlist& preamble_bl, uint32_t expected_crc; decode(expected_crc, it); segment_bl.splice(m_descs[0].logical_len, FRAME_CRC_SIZE); - check_segment_crc(segment_bl, expected_crc); + if (m_with_data_crc) { + check_segment_crc(segment_bl, expected_crc); + } } else { ceph_assert(segment_bl.length() == 0); } @@ -388,7 +393,9 @@ bool FrameAssembler::disasm_remaining_crc_rev1(bufferlist segment_bls[], for (size_t i = 1; i < m_descs.size(); i++) { ceph_assert(segment_bls[i].length() == m_descs[i].logical_len); - check_segment_crc(segment_bls[i], epilogue->crc_values[i - 1]); + if (m_with_data_crc) { + check_segment_crc(segment_bls[i], epilogue->crc_values[i - 1]); + } } return check_epilogue_late_status(epilogue->late_status); } diff --git a/src/msg/async/frames_v2.h b/src/msg/async/frames_v2.h index 5b786e1aff881..9431d6e2db3ed 100644 --- a/src/msg/async/frames_v2.h +++ b/src/msg/async/frames_v2.h @@ -188,8 +188,9 @@ class FrameAssembler { public: // crypto must be non-null FrameAssembler(const ceph::crypto::onwire::rxtx_t* crypto, bool is_rev1, - const ceph::compression::onwire::rxtx_t* compression) - : m_crypto(crypto), m_is_rev1(is_rev1), m_compression(compression) {} + bool with_data_crc, const ceph::compression::onwire::rxtx_t* compression) + : m_crypto(crypto), m_is_rev1(is_rev1), m_with_data_crc(with_data_crc), + m_compression(compression) {} void set_is_rev1(bool is_rev1) { m_descs.clear(); @@ -401,6 +402,7 @@ class FrameAssembler { __u8 m_flags; const ceph::crypto::onwire::rxtx_t* m_crypto; bool m_is_rev1; // msgr2.1? + bool m_with_data_crc; const ceph::compression::onwire::rxtx_t* m_compression; }; diff --git a/src/test/msgr/test_frames_v2.cc b/src/test/msgr/test_frames_v2.cc index b164ceb4ad241..c5be32a9c2142 100644 --- a/src/test/msgr/test_frames_v2.cc +++ b/src/test/msgr/test_frames_v2.cc @@ -176,8 +176,10 @@ class RoundTripTestBase : public ::testing::TestWithParam< std::tuple> { protected: RoundTripTestBase() - : m_tx_frame_asm(&m_tx_crypto, std::get<1>(GetParam()).is_rev1, &m_tx_comp), - m_rx_frame_asm(&m_rx_crypto, std::get<1>(GetParam()).is_rev1, &m_rx_comp), + : m_tx_frame_asm(&m_tx_crypto, std::get<1>(GetParam()).is_rev1, true, + &m_tx_comp), + m_rx_frame_asm(&m_rx_crypto, std::get<1>(GetParam()).is_rev1, true, + &m_rx_comp), m_header(make_bufferlist(std::get<0>(GetParam()).header_len, 'H')), m_front(make_bufferlist(std::get<0>(GetParam()).front_len, 'F')), m_middle(make_bufferlist(std::get<0>(GetParam()).middle_len, 'M')), From 1e5f89fa4a991e1b5e5478a8730feb379fbb5102 Mon Sep 17 00:00:00 2001 From: Chunsong Feng Date: Thu, 14 Oct 2021 11:23:27 +0000 Subject: [PATCH 0145/2866] test/msgr: remove DPDK Non-runtime configure items The non-runtime configure items of the DPDK must be configured in the ceph.conf to avoid assert in the _conf.set_val_or_die. Signed-off-by: Chunsong Feng --- src/test/msgr/test_async_networkstack.cc | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/test/msgr/test_async_networkstack.cc b/src/test/msgr/test_async_networkstack.cc index d369833b846b4..14923fd7dfbfd 100644 --- a/src/test/msgr/test_async_networkstack.cc +++ b/src/test/msgr/test_async_networkstack.cc @@ -76,15 +76,11 @@ class NetworkWorkerTest : public ::testing::TestWithParam { addr = "127.0.0.1:15000"; port_addr = "127.0.0.1:15001"; } else { - g_ceph_context->_conf.set_val_or_die("ms_type", "async+dpdk"); g_ceph_context->_conf.set_val_or_die("ms_dpdk_debug_allow_loopback", "true"); g_ceph_context->_conf.set_val_or_die("ms_async_op_threads", "2"); - g_ceph_context->_conf.set_val_or_die("ms_dpdk_coremask", "0x7"); - g_ceph_context->_conf.set_val_or_die("ms_dpdk_host_ipv4_addr", "172.16.218.3"); - g_ceph_context->_conf.set_val_or_die("ms_dpdk_gateway_ipv4_addr", "172.16.218.2"); - g_ceph_context->_conf.set_val_or_die("ms_dpdk_netmask_ipv4_addr", "255.255.255.0"); - addr = "172.16.218.3:15000"; - port_addr = "172.16.218.3:15001"; + string ipv4_addr = g_ceph_context->_conf.get_val("ms_dpdk_host_ipv4_addr"); + addr = ipv4_addr + std::string(":15000"); + port_addr = ipv4_addr + std::string(":15001"); } stack = NetworkStack::create(g_ceph_context, GetParam()); stack->start(); From 6fdb468b8574a04a09e6267122857b9013723afc Mon Sep 17 00:00:00 2001 From: Chunsong Feng Date: Thu, 14 Oct 2021 12:17:50 +0000 Subject: [PATCH 0146/2866] cmake: link DPDK shared libraries to avoid DPDK initialization failure The common_async_dpdk library depends on the dpdk librte_eal file. If the static library is used, the dependency is transferred to libceph-common. By default, libceph-common uses the PUBLIC keyword, the keyword PUBLIC determines that both the application and libceph-common link to dpdk librte_eal.a. As a result, the global variables in dpdk librte_eal.a has two copies. So link DPDK shared libs to avoid DPDK initialization failure. Fixes: https://tracker.ceph.com/issues/42861 Signed-off-by: Chunsong Feng Signed-off-by: luorixin --- cmake/modules/Finddpdk.cmake | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/cmake/modules/Finddpdk.cmake b/cmake/modules/Finddpdk.cmake index fae4691a8865d..270db8fa2fed4 100644 --- a/cmake/modules/Finddpdk.cmake +++ b/cmake/modules/Finddpdk.cmake @@ -48,7 +48,6 @@ set(components mbuf mempool mempool_ring - mempool_stack net pci pmd_af_packet @@ -66,6 +65,8 @@ set(components pmd_ring pmd_sfc_efx pmd_vmxnet3_uio + pmd_hns3 + pmd_hinic ring timer) @@ -79,11 +80,7 @@ foreach(c ${components}) get_target_property(DPDK_rte_${c}_LIBRARY ${dpdk_lib} IMPORTED_LOCATION) else() - find_library(DPDK_rte_${c}_LIBRARY - NAMES - # use static library - ${CMAKE_STATIC_LIBRARY_PREFIX}rte_${c}${CMAKE_STATIC_LIBRARY_SUFFIX} - rte_${c} + find_library(DPDK_rte_${c}_LIBRARY rte_${c} HINTS ENV DPDK_DIR ${dpdk_LIBRARY_DIRS} @@ -135,7 +132,8 @@ if(dpdk_FOUND) find_package(Threads QUIET) list(APPEND _dpdk_libs Threads::Threads - dpdk::cflags) + dpdk::cflags + numa) set_target_properties(dpdk::dpdk PROPERTIES INTERFACE_LINK_LIBRARIES "${_dpdk_libs}" INTERFACE_INCLUDE_DIRECTORIES "${dpdk_INCLUDE_DIRS}") From a5df0cfcbe9b04832095f1f8ef0ce9f178888efd Mon Sep 17 00:00:00 2001 From: Mark Kogan Date: Tue, 12 Oct 2021 14:07:54 +0000 Subject: [PATCH 0147/2866] rgw: under fips, set flag to allow md5 in select rgw ops Fixes: https://tracker.ceph.com/issues/52900 Signed-off-by: Mark Kogan --- src/common/ceph_crypto.cc | 5 +++++ src/common/ceph_crypto.h | 1 + src/rgw/rgw_bucket.cc | 2 ++ src/rgw/rgw_etag_verifier.h | 10 ++++++++-- src/rgw/rgw_file.h | 2 ++ src/rgw/rgw_op.cc | 14 ++++++++++++++ src/rgw/rgw_putobj_processor.cc | 2 ++ src/rgw/rgw_rados.cc | 2 ++ src/rgw/rgw_rest_swift.cc | 2 ++ src/rgw/rgw_sal_rados.cc | 2 ++ src/rgw/rgw_tools.h | 7 ++++++- src/rgw/rgw_torrent.cc | 1 - src/rgw/rgw_zone.cc | 2 ++ src/rgw/services/svc_zone.cc | 2 ++ 14 files changed, 50 insertions(+), 4 deletions(-) diff --git a/src/common/ceph_crypto.cc b/src/common/ceph_crypto.cc index cbeb1b06df615..e1f8705c9e654 100644 --- a/src/common/ceph_crypto.cc +++ b/src/common/ceph_crypto.cc @@ -196,6 +196,11 @@ void ssl::OpenSSLDigest::Restart() { EVP_DigestInit_ex(mpContext, mpType, NULL); } +void ssl::OpenSSLDigest::SetFlags(int flags) { + EVP_MD_CTX_set_flags(mpContext, flags); + this->Restart(); +} + void ssl::OpenSSLDigest::Update(const unsigned char *input, size_t length) { if (length) { EVP_DigestUpdate(mpContext, const_cast(reinterpret_cast(input)), length); diff --git a/src/common/ceph_crypto.h b/src/common/ceph_crypto.h index 6bf344aa8f13f..2feced03a99a5 100644 --- a/src/common/ceph_crypto.h +++ b/src/common/ceph_crypto.h @@ -52,6 +52,7 @@ namespace TOPNSPC::crypto { OpenSSLDigest (const EVP_MD *_type); ~OpenSSLDigest (); void Restart(); + void SetFlags(int flags); void Update (const unsigned char *input, size_t length); void Final (unsigned char *digest); }; diff --git a/src/rgw/rgw_bucket.cc b/src/rgw/rgw_bucket.cc index edc7f8027311f..21aa6ff419e61 100644 --- a/src/rgw/rgw_bucket.cc +++ b/src/rgw/rgw_bucket.cc @@ -1909,6 +1909,8 @@ static void get_md5_digest(const RGWBucketEntryPoint *be, string& md5_digest) { f->flush(bl); MD5 hash; + // Allow use of MD5 digest in FIPS mode for non-cryptographic purposes + hash.SetFlags(EVP_MD_CTX_FLAG_NON_FIPS_ALLOW); hash.Update((const unsigned char *)bl.c_str(), bl.length()); hash.Final(m); diff --git a/src/rgw/rgw_etag_verifier.h b/src/rgw/rgw_etag_verifier.h index 75a0e3e4724da..56a679ebddd31 100644 --- a/src/rgw/rgw_etag_verifier.h +++ b/src/rgw/rgw_etag_verifier.h @@ -30,7 +30,10 @@ class ETagVerifier : public rgw::putobj::Pipe public: ETagVerifier(CephContext* cct_, rgw::sal::DataProcessor *next) - : Pipe(next), cct(cct_) {} + : Pipe(next), cct(cct_) { + // Allow use of MD5 digest in FIPS mode for non-cryptographic purposes + hash.SetFlags(EVP_MD_CTX_FLAG_NON_FIPS_ALLOW); + } virtual void calculate_etag() = 0; std::string get_calculated_etag() { return calculated_etag;} @@ -62,7 +65,10 @@ class ETagVerifier_MPU : public ETagVerifier rgw::sal::DataProcessor *next) : ETagVerifier(cct, next), part_ofs(std::move(part_ofs)) - {} + { + // Allow use of MD5 digest in FIPS mode for non-cryptographic purposes + hash.SetFlags(EVP_MD_CTX_FLAG_NON_FIPS_ALLOW); + } int process(bufferlist&& data, uint64_t logical_offset) override; void calculate_etag() override; diff --git a/src/rgw/rgw_file.h b/src/rgw/rgw_file.h index c05562881d654..b5ab94986dcd6 100644 --- a/src/rgw/rgw_file.h +++ b/src/rgw/rgw_file.h @@ -2517,6 +2517,8 @@ class RGWWriteRequest : public RGWLibContinuedReq, // invoking this classes's header_init() (void) RGWWriteRequest::header_init(); op = this; + // Allow use of MD5 digest in FIPS mode for non-cryptographic purposes + hash.SetFlags(EVP_MD_CTX_FLAG_NON_FIPS_ALLOW); } bool only_bucket() override { return true; } diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc index 25bf48daff905..a1f99507504d2 100644 --- a/src/rgw/rgw_op.cc +++ b/src/rgw/rgw_op.cc @@ -1674,6 +1674,8 @@ static int iterate_user_manifest_parts(const DoutPrefixProvider *dpp, rgw::sal::Bucket::ListResults results; MD5 etag_sum; + // Allow use of MD5 digest in FIPS mode for non-cryptographic purposes + etag_sum.SetFlags(EVP_MD_CTX_FLAG_NON_FIPS_ALLOW); do { static constexpr auto MAX_LIST_OBJS = 100u; int r = bucket->list(dpp, params, MAX_LIST_OBJS, results, y); @@ -1952,6 +1954,8 @@ int RGWGetObj::handle_slo_manifest(bufferlist& bl, optional_yield y) map slo_parts; MD5 etag_sum; + // Allow use of MD5 digest in FIPS mode for non-cryptographic purposes + etag_sum.SetFlags(EVP_MD_CTX_FLAG_NON_FIPS_ALLOW); total_len = 0; for (const auto& entry : slo_info.entries) { @@ -3829,6 +3833,8 @@ void RGWPutObj::execute(optional_yield y) char calc_md5[CEPH_CRYPTO_MD5_DIGESTSIZE * 2 + 1]; unsigned char m[CEPH_CRYPTO_MD5_DIGESTSIZE]; MD5 hash; + // Allow use of MD5 digest in FIPS mode for non-cryptographic purposes + hash.SetFlags(EVP_MD_CTX_FLAG_NON_FIPS_ALLOW); bufferlist bl, aclbl, bs; int len; @@ -4300,6 +4306,8 @@ void RGWPostObj::execute(optional_yield y) char calc_md5[CEPH_CRYPTO_MD5_DIGESTSIZE * 2 + 1]; unsigned char m[CEPH_CRYPTO_MD5_DIGESTSIZE]; MD5 hash; + // Allow use of MD5 digest in FIPS mode for non-cryptographic purposes + hash.SetFlags(EVP_MD_CTX_FLAG_NON_FIPS_ALLOW); ceph::buffer::list bl, aclbl; int len = 0; @@ -5769,6 +5777,8 @@ void RGWPutLC::execute(optional_yield y) ldpp_dout(this, 15) << "read len=" << data.length() << " data=" << (buf ? buf : "") << dendl; MD5 data_hash; + // Allow use of MD5 digest in FIPS mode for non-cryptographic purposes + data_hash.SetFlags(EVP_MD_CTX_FLAG_NON_FIPS_ALLOW); unsigned char data_hash_res[CEPH_CRYPTO_MD5_DIGESTSIZE]; data_hash.Update(reinterpret_cast(buf), data.length()); data_hash.Final(data_hash_res); @@ -6362,6 +6372,8 @@ bool RGWCompleteMultipart::check_previously_completed(const RGWMultiCompleteUplo string oetag = sattrs[RGW_ATTR_ETAG].to_str(); MD5 hash; + // Allow use of MD5 digest in FIPS mode for non-cryptographic purposes + hash.SetFlags(EVP_MD_CTX_FLAG_NON_FIPS_ALLOW); for (const auto& [index, part] : parts->parts) { std::string partetag = rgw_string_unquote(part); char petag[CEPH_CRYPTO_MD5_DIGESTSIZE]; @@ -7390,6 +7402,8 @@ int RGWBulkUploadOp::handle_file(const std::string_view path, ssize_t len = 0; size_t ofs = 0; MD5 hash; + // Allow use of MD5 digest in FIPS mode for non-cryptographic purposes + hash.SetFlags(EVP_MD_CTX_FLAG_NON_FIPS_ALLOW); do { ceph::bufferlist data; len = body.get_at_most(s->cct->_conf->rgw_max_chunk_size, data); diff --git a/src/rgw/rgw_putobj_processor.cc b/src/rgw/rgw_putobj_processor.cc index 58cff29802885..3e95b984e1447 100644 --- a/src/rgw/rgw_putobj_processor.cc +++ b/src/rgw/rgw_putobj_processor.cc @@ -639,6 +639,8 @@ int AppendObjectProcessor::complete(size_t accounted_size, const string &etag, c //calculate the etag if (!cur_etag.empty()) { MD5 hash; + // Allow use of MD5 digest in FIPS mode for non-cryptographic purposes + hash.SetFlags(EVP_MD_CTX_FLAG_NON_FIPS_ALLOW); char petag[CEPH_CRYPTO_MD5_DIGESTSIZE]; char final_etag[CEPH_CRYPTO_MD5_DIGESTSIZE]; char final_etag_str[CEPH_CRYPTO_MD5_DIGESTSIZE * 2 + 16]; diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc index 9b4f77a8ca9a8..4574f4c32588e 100644 --- a/src/rgw/rgw_rados.cc +++ b/src/rgw/rgw_rados.cc @@ -5316,6 +5316,8 @@ static void generate_fake_tag(const DoutPrefixProvider *dpp, rgw::sal::Store* st unsigned char md5[CEPH_CRYPTO_MD5_DIGESTSIZE]; char md5_str[CEPH_CRYPTO_MD5_DIGESTSIZE * 2 + 1]; MD5 hash; + // Allow use of MD5 digest in FIPS mode for non-cryptographic purposes + hash.SetFlags(EVP_MD_CTX_FLAG_NON_FIPS_ALLOW); hash.Update((const unsigned char *)manifest_bl.c_str(), manifest_bl.length()); map::iterator iter = attrset.find(RGW_ATTR_ETAG); diff --git a/src/rgw/rgw_rest_swift.cc b/src/rgw/rgw_rest_swift.cc index 124ce202f1d0d..0a651bbb96a20 100644 --- a/src/rgw/rgw_rest_swift.cc +++ b/src/rgw/rgw_rest_swift.cc @@ -977,6 +977,8 @@ int RGWPutObj_ObjStore_SWIFT::get_params(optional_yield y) } MD5 etag_sum; + // Allow use of MD5 digest in FIPS mode for non-cryptographic purposes + etag_sum.SetFlags(EVP_MD_CTX_FLAG_NON_FIPS_ALLOW); uint64_t total_size = 0; for (auto& entry : slo_info->entries) { etag_sum.Update((const unsigned char *)entry.etag.c_str(), diff --git a/src/rgw/rgw_sal_rados.cc b/src/rgw/rgw_sal_rados.cc index 5ce3b2f000939..de3ade7558841 100644 --- a/src/rgw/rgw_sal_rados.cc +++ b/src/rgw/rgw_sal_rados.cc @@ -2225,6 +2225,8 @@ int RadosMultipartUpload::complete(const DoutPrefixProvider *dpp, std::string etag; bufferlist etag_bl; MD5 hash; + // Allow use of MD5 digest in FIPS mode for non-cryptographic purposes + hash.SetFlags(EVP_MD_CTX_FLAG_NON_FIPS_ALLOW); bool truncated; int ret; diff --git a/src/rgw/rgw_tools.h b/src/rgw/rgw_tools.h index df17f3914df06..b842916fd8767 100644 --- a/src/rgw/rgw_tools.h +++ b/src/rgw/rgw_tools.h @@ -111,7 +111,12 @@ class RGWEtag H hash; public: - RGWEtag() {} + RGWEtag() { + if constexpr (std::is_same_v) { + // Allow use of MD5 digest in FIPS mode for non-cryptographic purposes + hash.SetFlags(EVP_MD_CTX_FLAG_NON_FIPS_ALLOW); + } + } void update(const char *buf, size_t len) { hash.Update((const unsigned char *)buf, len); diff --git a/src/rgw/rgw_torrent.cc b/src/rgw/rgw_torrent.cc index 887b13a92b4d1..41a4a4d91a510 100644 --- a/src/rgw/rgw_torrent.cc +++ b/src/rgw/rgw_torrent.cc @@ -19,7 +19,6 @@ using namespace std; using namespace librados; using namespace boost; -using ceph::crypto::MD5; using ceph::crypto::SHA1; seed::seed() diff --git a/src/rgw/rgw_zone.cc b/src/rgw/rgw_zone.cc index 9dd98d43f5dff..013793c7e63eb 100644 --- a/src/rgw/rgw_zone.cc +++ b/src/rgw/rgw_zone.cc @@ -1905,6 +1905,8 @@ static uint32_t gen_short_zone_id(const std::string zone_id) { unsigned char md5[CEPH_CRYPTO_MD5_DIGESTSIZE]; MD5 hash; + // Allow use of MD5 digest in FIPS mode for non-cryptographic purposes + hash.SetFlags(EVP_MD_CTX_FLAG_NON_FIPS_ALLOW); hash.Update((const unsigned char *)zone_id.c_str(), zone_id.size()); hash.Final(md5); diff --git a/src/rgw/services/svc_zone.cc b/src/rgw/services/svc_zone.cc index a3502b3b5abe2..995abe606864b 100644 --- a/src/rgw/services/svc_zone.cc +++ b/src/rgw/services/svc_zone.cc @@ -565,6 +565,8 @@ int RGWSI_Zone::replace_region_with_zonegroup(const DoutPrefixProvider *dpp, opt unsigned char md5[CEPH_CRYPTO_MD5_DIGESTSIZE]; char md5_str[CEPH_CRYPTO_MD5_DIGESTSIZE * 2 + 1]; MD5 hash; + // Allow use of MD5 digest in FIPS mode for non-cryptographic purposes + hash.SetFlags(EVP_MD_CTX_FLAG_NON_FIPS_ALLOW); hash.Update((const unsigned char *)new_realm_name.c_str(), new_realm_name.length()); hash.Final(md5); buf_to_hex(md5, CEPH_CRYPTO_MD5_DIGESTSIZE, md5_str); From 551e0c8f38f3f646dbfb5fbfde51d3107ca90cc6 Mon Sep 17 00:00:00 2001 From: Mark Kogan Date: Thu, 14 Oct 2021 14:32:31 +0000 Subject: [PATCH 0148/2866] rgw: under fips, set flag to allow md5 in select rgw ops - for review the ovrrides for review and relevant md5 usage: ``` src/rgw/rgw_crypt.cc:975: unsigned char key_hash_res[CEPH_CRYPTO_MD5_DIGESTSIZE]; int rgw_s3_prepare_encrypt(...) crypt_http_responses["x-amz-server-side-encryption-customer-algorithm"] = "AES256"; crypt_http_responses["x-amz-server-side-encryption-customer-key-MD5"] = std::string(keymd5); ~~~~~~~~~~~~~~~~ src/rgw/rgw_crypt.cc:1225: uint8_t key_hash_res[CEPH_CRYPTO_MD5_DIGESTSIZE]; int rgw_s3_prepare_decrypt(...) crypt_http_responses["x-amz-server-side-encryption-customer-algorithm"] = "AES256"; crypt_http_responses["x-amz-server-side-encryption-customer-key-MD5"] = keymd5; ~~~~~~~~~~~~~~~~ src/rgw/rgw_keystone.cc:40: unsigned char m[CEPH_CRYPTO_MD5_DIGESTSIZE]; void TokenCache::add_admin(...) rgw_get_token_id(token.token.id, admin_token_id); ~~~~~~~~~~~~~~ md5 add_locked(admin_token_id, token); void TokenCache::add_barbican(...) rgw_get_token_id(token.token.id, barbican_token_id); ~~~~~~~~~~~~~~~~~ md5 add_locked(barbican_token_id, token); ``` Signed-off-by: Mark Kogan --- src/rgw/rgw_crypt.cc | 4 ++++ src/rgw/rgw_keystone.cc | 2 ++ 2 files changed, 6 insertions(+) diff --git a/src/rgw/rgw_crypt.cc b/src/rgw/rgw_crypt.cc index 243d2badf556e..373f36eea1b73 100644 --- a/src/rgw/rgw_crypt.cc +++ b/src/rgw/rgw_crypt.cc @@ -972,6 +972,8 @@ int rgw_s3_prepare_encrypt(struct req_state* s, } MD5 key_hash; + // Allow use of MD5 digest in FIPS mode for non-cryptographic purposes + key_hash.SetFlags(EVP_MD_CTX_FLAG_NON_FIPS_ALLOW); unsigned char key_hash_res[CEPH_CRYPTO_MD5_DIGESTSIZE]; key_hash.Update(reinterpret_cast(key_bin.c_str()), key_bin.size()); key_hash.Final(key_hash_res); @@ -1222,6 +1224,8 @@ int rgw_s3_prepare_decrypt(struct req_state* s, } MD5 key_hash; + // Allow use of MD5 digest in FIPS mode for non-cryptographic purposes + key_hash.SetFlags(EVP_MD_CTX_FLAG_NON_FIPS_ALLOW); uint8_t key_hash_res[CEPH_CRYPTO_MD5_DIGESTSIZE]; key_hash.Update(reinterpret_cast(key_bin.c_str()), key_bin.size()); key_hash.Final(key_hash_res); diff --git a/src/rgw/rgw_keystone.cc b/src/rgw/rgw_keystone.cc index 32502bec06f0d..6c045afa5f320 100644 --- a/src/rgw/rgw_keystone.cc +++ b/src/rgw/rgw_keystone.cc @@ -40,6 +40,8 @@ void rgw_get_token_id(const string& token, string& token_id) unsigned char m[CEPH_CRYPTO_MD5_DIGESTSIZE]; MD5 hash; + // Allow use of MD5 digest in FIPS mode for non-cryptographic purposes + hash.SetFlags(EVP_MD_CTX_FLAG_NON_FIPS_ALLOW); hash.Update((const unsigned char *)token.c_str(), token.size()); hash.Final(m); From 1020192170329aa015ccdcb2f4cc40b172a99616 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Tue, 12 Oct 2021 09:43:05 -0500 Subject: [PATCH 0149/2866] mgr/rook: fix mds service list Signed-off-by: Sage Weil --- src/pybind/mgr/rook/module.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/pybind/mgr/rook/module.py b/src/pybind/mgr/rook/module.py index 104780d93f77b..06f4ef42062a8 100644 --- a/src/pybind/mgr/rook/module.py +++ b/src/pybind/mgr/rook/module.py @@ -284,16 +284,16 @@ def describe_service(self, total_mds = active if fs['spec'].get('metadataServer', {}).get('activeStandby', False): total_mds = active * 2 - spec[svc] = orchestrator.ServiceDescription( - spec=ServiceSpec( - service_type='mds', - service_id=fs['metadata']['name'], - placement=PlacementSpec(count=active), - ), - size=total_mds, - container_image_name=image_name, - last_refresh=now, - ) + spec[svc] = orchestrator.ServiceDescription( + spec=ServiceSpec( + service_type='mds', + service_id=fs['metadata']['name'], + placement=PlacementSpec(count=active), + ), + size=total_mds, + container_image_name=image_name, + last_refresh=now, + ) if service_type == 'rgw' or service_type is None: # CephObjectstores From e4e9477dd9fc96dd7f96ed71ecc7cfc4292b68e5 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Wed, 13 Oct 2021 16:16:03 -0500 Subject: [PATCH 0150/2866] qa/suites/orch/rook/smoke: use nvme-loop Signed-off-by: Sage Weil --- qa/suites/orch/rook/smoke/0-nvme-loop.yaml | 1 + 1 file changed, 1 insertion(+) create mode 120000 qa/suites/orch/rook/smoke/0-nvme-loop.yaml diff --git a/qa/suites/orch/rook/smoke/0-nvme-loop.yaml b/qa/suites/orch/rook/smoke/0-nvme-loop.yaml new file mode 120000 index 0000000000000..5206b6eddee88 --- /dev/null +++ b/qa/suites/orch/rook/smoke/0-nvme-loop.yaml @@ -0,0 +1 @@ +.qa/overrides/nvme_loop.yaml \ No newline at end of file From 3205a835f6bfa1cea4415dc8a95888ac7dade181 Mon Sep 17 00:00:00 2001 From: Adam King Date: Thu, 14 Oct 2021 14:56:51 -0400 Subject: [PATCH 0151/2866] cephadm: do not log commands sent to "call" function They could potentially log sensitive info such as registry login passwords Signed-off-by: Adam King --- src/cephadm/cephadm | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/cephadm/cephadm b/src/cephadm/cephadm index ed392bd187d75..28e5b9d58db1e 100755 --- a/src/cephadm/cephadm +++ b/src/cephadm/cephadm @@ -1413,8 +1413,6 @@ def call(ctx: CephadmContext, prefix += ': ' timeout = timeout or ctx.timeout - logger.debug('Running command: %s' % ' '.join(command)) - async def tee(reader: asyncio.StreamReader) -> str: collected = StringIO() async for line in reader: From d7413341119d9b8eabba8c11b60c0312cd1d5002 Mon Sep 17 00:00:00 2001 From: Samuel Just Date: Thu, 14 Oct 2021 14:51:38 -0700 Subject: [PATCH 0152/2866] crimson/os/seastore/segment_manager/block: open with dsync 67efc4 appears to be simply incorrect, I don't see any calls to flush(), so we do need to open with dsync until we implement a smarter flushing scheme. Also, refactor open_device to remove mode param -- we always pass the same value. Reverts: 67efc48332cebb69497c5676be6ed2a6e5365e03 Signed-off-by: Samuel Just --- src/crimson/os/seastore/segment_manager/block.cc | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/crimson/os/seastore/segment_manager/block.cc b/src/crimson/os/seastore/segment_manager/block.cc index 5b395fac589b6..907b089897669 100644 --- a/src/crimson/os/seastore/segment_manager/block.cc +++ b/src/crimson/os/seastore/segment_manager/block.cc @@ -245,12 +245,14 @@ using open_device_ret = >; static open_device_ret open_device( - const std::string &path, - seastar::open_flags mode) + const std::string &path) { return seastar::file_stat(path, seastar::follow_symlink::yes - ).then([mode, &path](auto stat) mutable { - return seastar::open_file_dma(path, mode).then([=](auto file) { + ).then([&path](auto stat) mutable { + return seastar::open_file_dma( + path, + seastar::open_flags::rw | seastar::open_flags::dsync + ).then([=](auto file) { logger().error( "open_device: open successful, size {}", stat.size @@ -389,7 +391,7 @@ BlockSegmentManager::~BlockSegmentManager() BlockSegmentManager::mount_ret BlockSegmentManager::mount() { return open_device( - device_path, seastar::open_flags::rw + device_path ).safe_then([=](auto p) { device = std::move(p.first); auto sd = p.second; @@ -440,7 +442,7 @@ BlockSegmentManager::mkfs_ret BlockSegmentManager::mkfs( } return maybe_create.safe_then([this] { - return open_device(device_path, seastar::open_flags::rw); + return open_device(device_path); }).safe_then([&, sm_config](auto p) { device = p.first; stat = p.second; From 38ddd5e8a355941a39302d38143578a75e2ace1a Mon Sep 17 00:00:00 2001 From: Laura Flores Date: Thu, 14 Oct 2021 15:59:34 +0000 Subject: [PATCH 0153/2866] mon: create default tabular format for the output of `mgr module ls` Signed-off-by: Laura Flores --- src/mon/MgrMonitor.cc | 105 ++++++++++++++++++++++++++++++------------ 1 file changed, 76 insertions(+), 29 deletions(-) diff --git a/src/mon/MgrMonitor.cc b/src/mon/MgrMonitor.cc index b2dae3ce8412a..c9458aab4eeb9 100644 --- a/src/mon/MgrMonitor.cc +++ b/src/mon/MgrMonitor.cc @@ -24,6 +24,10 @@ #include "ConfigMonitor.h" #include "HealthMonitor.h" +#include "common/TextTable.h" +#include "common/cmdparse.h" +#include "include/stringify.h" + #include "MgrMonitor.h" #define MGR_METADATA_PREFIX "mgr_metadata" @@ -946,10 +950,8 @@ bool MgrMonitor::preprocess_command(MonOpRequestRef op) return true; } - string format; - cmd_getval(cmdmap, "format", format); - boost::scoped_ptr f(Formatter::create(format, "json-pretty", - "json-pretty")); + string format = cmd_getval_or(cmdmap, "format", "plain"); + boost::scoped_ptr f(Formatter::create(format)); string prefix; cmd_getval(cmdmap, "prefix", prefix); @@ -957,6 +959,9 @@ bool MgrMonitor::preprocess_command(MonOpRequestRef op) int r = 0; if (prefix == "mgr stat") { + if (!f) { + f.reset(Formatter::create(format, "json-pretty", "json-pretty")); + } f->open_object_section("stat"); f->dump_unsigned("epoch", map.get_epoch()); f->dump_bool("available", map.get_available()); @@ -965,6 +970,9 @@ bool MgrMonitor::preprocess_command(MonOpRequestRef op) f->close_section(); f->flush(rdata); } else if (prefix == "mgr dump") { + if (!f) { + f.reset(Formatter::create(format, "json-pretty", "json-pretty")); + } int64_t epoch = cmd_getval_or(cmdmap, "epoch", map.get_epoch()); if (epoch == (int64_t)map.get_epoch()) { f->dump_object("mgrmap", map); @@ -985,42 +993,75 @@ bool MgrMonitor::preprocess_command(MonOpRequestRef op) } else if (prefix == "mgr module ls") { string detail; cmd_getval(cmdmap, "detail", detail); + if (f || (detail == "detail")) { + if (!f) { + f.reset(Formatter::create(format, "json-pretty", "json-pretty")); + } + f->open_object_section("modules"); + { + f->open_array_section("always_on_modules"); + for (auto& p : map.get_always_on_modules()) { + f->dump_string("module", p); + } + f->close_section(); + f->open_array_section("enabled_modules"); + for (auto& p : map.modules) { + if (map.get_always_on_modules().count(p) > 0) + continue; + // We only show the name for enabled modules. The any errors + // etc will show up as a health checks. + f->dump_string("module", p); + } + f->close_section(); + f->open_array_section("disabled_modules"); + for (auto& p : map.available_modules) { + if (map.modules.count(p.name) == 0 && + map.get_always_on_modules().count(p.name) == 0) { + if (detail == "detail") { + // For disabled modules, we show the full info if the detail + // parameter is enabled, to give a hint about whether enabling it will work + p.dump(f.get()); + } else { + // Otherwise, we give a shortened summary by default + f->dump_string("module", p.name); + } + } + } + f->close_section(); + } + f->close_section(); + f->flush(rdata); + } else { + TextTable tbl; + tbl.define_column("MODULE", TextTable::LEFT, TextTable::LEFT); + tbl.define_column(" ", TextTable::LEFT, TextTable::LEFT); - f->open_object_section("modules"); - { - f->open_array_section("always_on_modules"); for (auto& p : map.get_always_on_modules()) { - f->dump_string("module", p); + tbl << p; + tbl << "on (always on)"; + tbl << TextTable::endrow; } - f->close_section(); - f->open_array_section("enabled_modules"); for (auto& p : map.modules) { if (map.get_always_on_modules().count(p) > 0) continue; - // We only show the name for enabled modules. The any errors - // etc will show up as a health checks. - f->dump_string("module", p); + tbl << p; + tbl << "on"; + tbl << TextTable::endrow; } - f->close_section(); - f->open_array_section("disabled_modules"); for (auto& p : map.available_modules) { if (map.modules.count(p.name) == 0 && map.get_always_on_modules().count(p.name) == 0) { - if (detail == "detail") { - // For disabled modules, we show the full info if the detail - // parameter is enabled, to give a hint about whether enabling it will work - p.dump(f.get()); - } else { - // Otherwise, we give a shortened summary by default - f->dump_string("module", p.name); - } + tbl << p.name; + tbl << "-"; + tbl << TextTable::endrow; } } - f->close_section(); + rdata.append(stringify(tbl)); } - f->close_section(); - f->flush(rdata); } else if (prefix == "mgr services") { + if (!f) { + f.reset(Formatter::create(format, "json-pretty", "json-pretty")); + } f->open_object_section("services"); for (const auto &i : map.services) { f->dump_string(i.first.c_str(), i.second); @@ -1028,6 +1069,9 @@ bool MgrMonitor::preprocess_command(MonOpRequestRef op) f->close_section(); f->flush(rdata); } else if (prefix == "mgr metadata") { + if (!f) { + f.reset(Formatter::create(format, "json-pretty", "json-pretty")); + } string name; cmd_getval(cmdmap, "who", name); if (name.size() > 0 && !map.have_name(name)) { @@ -1035,9 +1079,6 @@ bool MgrMonitor::preprocess_command(MonOpRequestRef op) r = -ENOENT; goto reply; } - string format; - cmd_getval(cmdmap, "format", format); - boost::scoped_ptr f(Formatter::create(format, "json-pretty", "json-pretty")); if (name.size()) { f->open_object_section("mgr_metadata"); f->dump_string("name", name); @@ -1066,10 +1107,16 @@ bool MgrMonitor::preprocess_command(MonOpRequestRef op) } f->flush(rdata); } else if (prefix == "mgr versions") { + if (!f) { + f.reset(Formatter::create(format, "json-pretty", "json-pretty")); + } count_metadata("ceph_version", f.get()); f->flush(rdata); r = 0; } else if (prefix == "mgr count-metadata") { + if (!f) { + f.reset(Formatter::create(format, "json-pretty", "json-pretty")); + } string field; cmd_getval(cmdmap, "property", field); count_metadata(field, f.get()); From bb0a39b7d953bf05eef8d24ba4efb7a3ead3c17b Mon Sep 17 00:00:00 2001 From: Laura Flores Date: Thu, 14 Oct 2021 23:59:47 +0000 Subject: [PATCH 0154/2866] qa/tasks/mgr: update tests that use `mgr module ls` to specify the json format Signed-off-by: Laura Flores --- qa/tasks/mgr/mgr_test_case.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/qa/tasks/mgr/mgr_test_case.py b/qa/tasks/mgr/mgr_test_case.py index f5392d3ba38a3..5fe5e0759856c 100644 --- a/qa/tasks/mgr/mgr_test_case.py +++ b/qa/tasks/mgr/mgr_test_case.py @@ -74,7 +74,7 @@ def setup_mgrs(cls): # Unload all non-default plugins loaded = json.loads(cls.mgr_cluster.mon_manager.raw_cluster_cmd( - "mgr", "module", "ls"))['enabled_modules'] + "mgr", "module", "ls", "--format=json-pretty"))['enabled_modules'] unload_modules = set(loaded) - {"cephadm", "restful"} for m in unload_modules: @@ -111,7 +111,7 @@ def setUpClass(cls): def _unload_module(cls, module_name): def is_disabled(): enabled_modules = json.loads(cls.mgr_cluster.mon_manager.raw_cluster_cmd( - 'mgr', 'module', 'ls'))['enabled_modules'] + 'mgr', 'module', 'ls', "--format=json-pretty"))['enabled_modules'] return module_name not in enabled_modules if is_disabled(): @@ -124,7 +124,7 @@ def is_disabled(): @classmethod def _load_module(cls, module_name): loaded = json.loads(cls.mgr_cluster.mon_manager.raw_cluster_cmd( - "mgr", "module", "ls"))['enabled_modules'] + "mgr", "module", "ls", "--format=json-pretty"))['enabled_modules'] if module_name in loaded: # The enable command is idempotent, but our wait for a restart # isn't, so let's return now if it's already loaded From 522c184f4257c44f2e85205a48ccbecf9d1a9bdb Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Tue, 12 Oct 2021 11:13:04 -0500 Subject: [PATCH 0155/2866] mgr/nfs: add 'nfs cluster config get' Fixes: https://tracker.ceph.com/issues/52942 Signed-off-by: Sage Weil --- doc/mgr/nfs.rst | 9 +++++++++ src/pybind/mgr/nfs/cluster.py | 10 ++++++++++ src/pybind/mgr/nfs/export.py | 8 ++++++++ src/pybind/mgr/nfs/module.py | 5 +++++ 4 files changed, 32 insertions(+) diff --git a/doc/mgr/nfs.rst b/doc/mgr/nfs.rst index 2c70c5c2f2501..54ca19fdd2e26 100644 --- a/doc/mgr/nfs.rst +++ b/doc/mgr/nfs.rst @@ -204,6 +204,15 @@ Example use cases include: # ceph auth get-or-create client. mon 'allow r' osd 'allow rw pool=.nfs namespace=, allow rw tag cephfs data=' mds 'allow rw path=' +View Customized NFS Ganesha Configuration +----------------------------------------- + +.. code:: bash + + $ ceph nfs cluster config get + +This will output the user defined configuration (if any). + Reset NFS Ganesha Configuration ------------------------------- diff --git a/src/pybind/mgr/nfs/cluster.py b/src/pybind/mgr/nfs/cluster.py index 4ccf37d2b0572..156b57e4c9478 100644 --- a/src/pybind/mgr/nfs/cluster.py +++ b/src/pybind/mgr/nfs/cluster.py @@ -213,6 +213,16 @@ def show_nfs_cluster_info(self, cluster_id: Optional[str] = None) -> Tuple[int, except Exception as e: return exception_handler(e, "Failed to show info for cluster") + def get_nfs_cluster_config(self, cluster_id: str) -> Tuple[int, str, str]: + try: + if cluster_id in available_clusters(self.mgr): + rados_obj = NFSRados(self.mgr, cluster_id) + conf = rados_obj.read_obj(self._get_user_conf_obj_name(cluster_id)) + return 0, conf or "", "" + raise ClusterNotFound() + except Exception as e: + return exception_handler(e, f"Fetching NFS-Ganesha Config failed for {cluster_id}") + def set_nfs_cluster_config(self, cluster_id: str, nfs_config: str) -> Tuple[int, str, str]: try: if cluster_id in available_clusters(self.mgr): diff --git a/src/pybind/mgr/nfs/export.py b/src/pybind/mgr/nfs/export.py index 59c94a4dc0ab9..bfc779a9a5247 100644 --- a/src/pybind/mgr/nfs/export.py +++ b/src/pybind/mgr/nfs/export.py @@ -74,6 +74,14 @@ def write_obj(self, conf_block: str, obj: str, config_obj: str = '') -> None: ExportMgr._check_rados_notify(ioctx, config_obj) log.debug("Added %s url to %s", obj, config_obj) + def read_obj(self, obj: str) -> Optional[str]: + with self.mgr.rados.open_ioctx(self.pool) as ioctx: + ioctx.set_namespace(self.namespace) + try: + return ioctx.read(obj, 1048576).decode() + except ObjectNotFound: + return None + def update_obj(self, conf_block: str, obj: str, config_obj: str) -> None: with self.mgr.rados.open_ioctx(self.pool) as ioctx: ioctx.set_namespace(self.namespace) diff --git a/src/pybind/mgr/nfs/module.py b/src/pybind/mgr/nfs/module.py index 6e97d48f83436..d134e55eab22e 100644 --- a/src/pybind/mgr/nfs/module.py +++ b/src/pybind/mgr/nfs/module.py @@ -120,6 +120,11 @@ def _cmd_nfs_cluster_info(self, cluster_id: Optional[str] = None) -> Tuple[int, """Displays NFS Cluster info""" return self.nfs.show_nfs_cluster_info(cluster_id=cluster_id) + @CLICommand('nfs cluster config get', perm='r') + def _cmd_nfs_cluster_config_get(self, cluster_id: str) -> Tuple[int, str, str]: + """Fetch NFS-Ganesha config""" + return self.nfs.get_nfs_cluster_config(cluster_id=cluster_id) + @CLICommand('nfs cluster config set', perm='rw') @CLICheckNonemptyFileInput(desc='NFS-Ganesha Configuration') def _cmd_nfs_cluster_config_set(self, cluster_id: str, inbuf: str) -> Tuple[int, str, str]: From 03ba3ceb5d2dbb947de287e036e6651e55293a57 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Thu, 14 Oct 2021 13:21:40 -0400 Subject: [PATCH 0156/2866] mgr/nfs: add some test coverage for cluster info, cluster config Signed-off-by: Sage Weil --- src/pybind/mgr/nfs/tests/test_nfs.py | 72 ++++++++++++++++++++++++++-- src/pybind/mgr/tests/__init__.py | 7 ++- 2 files changed, 75 insertions(+), 4 deletions(-) diff --git a/src/pybind/mgr/nfs/tests/test_nfs.py b/src/pybind/mgr/nfs/tests/test_nfs.py index 04394fedf7bbe..479e3c81366b2 100644 --- a/src/pybind/mgr/nfs/tests/test_nfs.py +++ b/src/pybind/mgr/nfs/tests/test_nfs.py @@ -8,10 +8,13 @@ from unittest.mock import MagicMock from mgr_module import NFS_POOL_NAME +from rados import ObjectNotFound + from ceph.deployment.service_spec import NFSServiceSpec from nfs import Module from nfs.export import ExportMgr from nfs.export_utils import GaneshaConfParser, Export, RawBlock +from nfs.cluster import NFSCluster from orchestrator import ServiceDescription, DaemonDescription, OrchResult @@ -123,6 +126,8 @@ def _ioctl_stat_mock(self, key): return self.temp_store[self.temp_store_namespace][key].stat() def _ioctl_read_mock(self, key: str, size: Optional[Any] = None) -> bytes: + if key not in self.temp_store[self.temp_store_namespace]: + raise ObjectNotFound return self.temp_store[self.temp_store_namespace][key].read(size) def _ioctx_set_namespace_mock(self, namespace: str) -> None: @@ -153,6 +158,10 @@ def _mock_orchestrator(self, enable: bool) -> Iterator: ServiceDescription(spec=NFSServiceSpec(service_id=self.cluster_id)) ] if enable else [] + orch_nfs_daemons = [ + DaemonDescription('nfs', 'foo.mydaemon', 'myhostname') + ] if enable else [] + def mock_exec(cls, args): u = { "user_id": "abc", @@ -196,11 +205,23 @@ def mock_exec(cls, args): return 0, json.dumps([u]), '' return 0, json.dumps(u), '' - with mock.patch('nfs.module.Module.describe_service') as describe_service, \ + def mock_describe_service(cls, *args, **kwargs): + if kwargs['service_type'] == 'nfs': + return OrchResult(orch_nfs_services) + return OrchResult([]) + + def mock_list_daemons(cls, *args, **kwargs): + if kwargs['daemon_type'] == 'nfs': + return OrchResult(orch_nfs_daemons) + return OrchResult([]) + + with mock.patch('nfs.module.Module.describe_service', mock_describe_service) as describe_service, \ + mock.patch('nfs.module.Module.list_daemons', mock_list_daemons) as list_daemons, \ mock.patch('nfs.module.Module.rados') as rados, \ mock.patch('nfs.export.available_clusters', return_value=[self.cluster_id]), \ mock.patch('nfs.export.restart_nfs_service'), \ + mock.patch('nfs.cluster.restart_nfs_service'), \ mock.patch('nfs.export.ExportMgr._exec', mock_exec), \ mock.patch('nfs.export.check_fs', return_value=True), \ mock.patch('nfs.export_utils.check_fs', return_value=True), \ @@ -210,8 +231,6 @@ def mock_exec(cls, args): rados.open_ioctx.return_value.__enter__.return_value = self.io_mock rados.open_ioctx.return_value.__exit__ = mock.Mock(return_value=None) - describe_service.return_value = OrchResult(orch_nfs_services) - self._reset_temp_store() yield @@ -855,3 +874,50 @@ def _do_test_create_export_cephfs(self): assert export.clients[0].access_type == 'rw' assert export.clients[0].addresses == ["192.168.1.0/8"] assert export.cluster_id == self.cluster_id + + def _do_test_cluster_ls(self): + nfs_mod = Module('nfs', '', '') + cluster = NFSCluster(nfs_mod) + + rc, out, err = cluster.list_nfs_cluster() + assert rc == 0 + assert out == self.cluster_id + + def test_cluster_ls(self): + self._do_mock_test(self._do_test_cluster_ls) + + def _do_test_cluster_info(self): + nfs_mod = Module('nfs', '', '') + cluster = NFSCluster(nfs_mod) + + rc, out, err = cluster.show_nfs_cluster_info(self.cluster_id) + assert rc == 0 + assert json.loads(out) == {"foo": {"virtual_ip": None, "backend": []}} + + def test_cluster_info(self): + self._do_mock_test(self._do_test_cluster_info) + + def _do_test_cluster_config(self): + nfs_mod = Module('nfs', '', '') + cluster = NFSCluster(nfs_mod) + + rc, out, err = cluster.get_nfs_cluster_config(self.cluster_id) + assert rc == 0 + assert out == "" + + rc, out, err = cluster.set_nfs_cluster_config(self.cluster_id, '# foo\n') + assert rc == 0 + + rc, out, err = cluster.get_nfs_cluster_config(self.cluster_id) + assert rc == 0 + assert out == "# foo\n" + + rc, out, err = cluster.reset_nfs_cluster_config(self.cluster_id) + assert rc == 0 + + rc, out, err = cluster.get_nfs_cluster_config(self.cluster_id) + assert rc == 0 + assert out == "" + + def test_cluster_config(self): + self._do_mock_test(self._do_test_cluster_config) diff --git a/src/pybind/mgr/tests/__init__.py b/src/pybind/mgr/tests/__init__.py index 16286dd2e76aa..68b61e7ec5233 100644 --- a/src/pybind/mgr/tests/__init__.py +++ b/src/pybind/mgr/tests/__init__.py @@ -212,9 +212,14 @@ def __str__(self): return msg return '[errno {0}] {1}'.format(self.errno, msg) + class MockObjectNotFound(Exception): + pass sys.modules.update({ - 'rados': mock.MagicMock(Error=MockRadosError, OSError=MockRadosError), + 'rados': mock.MagicMock( + Error=MockRadosError, + OSError=MockRadosError, + ObjectNotFound=MockObjectNotFound), 'rbd': mock.Mock(), 'cephfs': mock.Mock(), }) From b7782084ac9657be9b2da6ebd56b5029cf859225 Mon Sep 17 00:00:00 2001 From: Paul Cuzner Date: Thu, 14 Oct 2021 13:06:30 +1300 Subject: [PATCH 0157/2866] mgr/cephadm:improve validation of orch device zap Improve the logic in the function to : 1. check that the host exists 2. check the host isn't in maintenance 3. check the disk exists on the host 4. check the disk isn't supporting a known OSD 5. check that the disk doesn't belong to another cluster In addition the command now returns a completion message so you know it worked. Fixes: https://tracker.ceph.com/issues/52919 Signed-off-by: Paul Cuzner --- src/pybind/mgr/cephadm/module.py | 89 ++++++++++++++++++++++++++------ 1 file changed, 73 insertions(+), 16 deletions(-) diff --git a/src/pybind/mgr/cephadm/module.py b/src/pybind/mgr/cephadm/module.py index 14bf025199cea..102a4cf63fbed 100644 --- a/src/pybind/mgr/cephadm/module.py +++ b/src/pybind/mgr/cephadm/module.py @@ -1503,7 +1503,8 @@ def _set_maintenance_healthcheck(self) -> None: self.remove_health_warning('HOST_IN_MAINTENANCE') else: s = "host is" if len(in_maintenance) == 1 else "hosts are" - self.set_health_warning("HOST_IN_MAINTENANCE", f"{len(in_maintenance)} {s} in maintenance mode", 1, [f"{h} is in maintenance" for h in in_maintenance]) + self.set_health_warning("HOST_IN_MAINTENANCE", f"{len(in_maintenance)} {s} in maintenance mode", 1, [ + f"{h} is in maintenance" for h in in_maintenance]) @handle_orch_error @host_exists() @@ -1941,33 +1942,89 @@ def get_inventory(self, host_filter: Optional[orchestrator.InventoryFilter] = No @handle_orch_error def zap_device(self, host: str, path: str) -> str: + """Zap a device on a managed host. + + Use ceph-volume zap to return a device to an unused/free state + + Args: + host (str): hostname of the cluster host + path (str): device path + + Raises: + OrchestratorError: host is not a cluster host + OrchestratorError: host is in maintenance and therefore unavailable + OrchestratorError: device path not found on the host + OrchestratorError: device is known to a different ceph cluster + OrchestratorError: device holds active osd + OrchestratorError: device cache hasn't been populated yet.. + + Returns: + str: output from the zap command + """ + self.log.info('Zap device %s:%s' % (host, path)) - out, err, code = CephadmServe(self)._run_cephadm( - host, 'osd.', 'ceph-volume', - ['--', 'lvm', 'list', path, '--format', 'json'], - error_ok=True) - if code: - raise OrchestratorError('Zap failed: ceph-volume lvm list %s' % '\n'.join(path)) - osd_info = json.loads('\n'.join(out)) + if host not in self.inventory.keys(): + raise OrchestratorError( + f"Host '{host}' is not a member of the cluster") - for osd_id in osd_info.keys(): - out, err, code = CephadmServe(self)._run_cephadm( - host, 'osd.', 'unit', - ['is-active', '--name', 'osd.%s' % osd_id], - error_ok=True) - if code == 0: - raise OrchestratorError('The systemctl unit of osd is active. Rm osd first') + host_info = self.inventory._inventory.get(host, {}) + if host_info.get('status', '').lower() == 'maintenance': + raise OrchestratorError( + f"Host '{host}' is in maintenance mode, which prevents any actions against it.") + + if host not in self.cache.devices: + raise OrchestratorError( + f"Host '{host} hasn't been scanned yet to determine it's inventory. Please try again later.") + + host_devices = self.cache.devices[host] + path_found = False + osd_id_list: List[str] = [] + + for dev in host_devices: + if dev.path == path: + # match, so look a little deeper + if dev.lvs: + for lv in cast(List[Dict[str, str]], dev.lvs): + if lv.get('osd_id', ''): + lv_fsid = lv.get('cluster_fsid') + if lv_fsid != self._cluster_fsid: + raise OrchestratorError( + f"device {path} has lv's from a different Ceph cluster ({lv_fsid})") + osd_id_list.append(lv.get('osd_id', '')) + path_found = True + break + if not path_found: + raise OrchestratorError( + f"Device path '{path}' not found on host '{host}'") + + if osd_id_list: + dev_name = os.path.basename(path) + active_osds: List[str] = [] + for osd_id in osd_id_list: + metadata = self.get_metadata('osd', str(osd_id)) + if metadata: + if metadata.get('hostname', '') == host and dev_name in metadata.get('devices', '').split(','): + active_osds.append("osd." + osd_id) + if active_osds: + raise OrchestratorError( + f"Unable to zap: device '{path}' on {host} has {len(active_osds)} active " + f"OSD{'s' if len(active_osds) > 1 else ''}" + f" ({', '.join(active_osds)}). Use 'ceph orch osd rm' first.") out, err, code = CephadmServe(self)._run_cephadm( host, 'osd', 'ceph-volume', ['--', 'lvm', 'zap', '--destroy', path], error_ok=True) + self.cache.invalidate_host_devices(host) self.cache.invalidate_host_networks(host) if code: raise OrchestratorError('Zap failed: %s' % '\n'.join(out + err)) - return '\n'.join(out + err) + msg = f'zap successful for {path} on {host}' + self.log.info(msg) + + return msg + '\n' @handle_orch_error def blink_device_light(self, ident_fault: str, on: bool, locs: List[orchestrator.DeviceLightLoc]) -> List[str]: From 5d80867f2e3eb46e36bb168638193940a9060bbd Mon Sep 17 00:00:00 2001 From: Josh Durgin Date: Thu, 14 Oct 2021 20:52:50 -0400 Subject: [PATCH 0158/2866] doc/governance.rst: update clt membership This now matches the mailing list which Sage audited. Signed-off-by: Josh Durgin --- doc/governance.rst | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/doc/governance.rst b/doc/governance.rst index 3de380c5f8503..e9db8eb41a0b6 100644 --- a/doc/governance.rst +++ b/doc/governance.rst @@ -52,23 +52,32 @@ the CLT itself. Current CLT members are: - * Abhishek Lekshmanan * Casey Bodley + * Dan van der Ster + * David Galloway + * David Orman * Ernesto Puerta * Gregory Farnum * Haomai Wang - * Jason Dillaman + * Ilya Dryomov + * Igor Fedotov + * Jeff Layton * Josh Durgin * João Eduardo Luis * Ken Dreyer + * Mark Nelson * Matt Benjamin + * Mike Perez * Myoungwon Oh * Neha Ojha * Patrick Donnelly * Sage Weil + * Sam Just * Sebastian Wagner * Xie Xingguo * Yehuda Sadeh + * Yuri Weinstein + * Zac Dover Component Leads --------------- From 6517d942c7f90a24355c659023ddf62385504c7f Mon Sep 17 00:00:00 2001 From: "J. Eric Ivancich" Date: Wed, 13 Oct 2021 17:58:55 -0400 Subject: [PATCH 0159/2866] rgw: allow force-check filter to pass through the SAL layer The zipper updates did not pass a force-check filter from RGWRados::Bucket::List through to RGWRados::cls_bucket_list_ordered. This filter is necessary for the "radosgw-admin bucket check --fix..." functionality. Declares type RGWBucketListNameFilter to encapsualte the filter type. Renames some fields so the two filters can be distinguished. Signed-off-by: J. Eric Ivancich --- src/rgw/rgw_bucket.cc | 12 +++++++----- src/rgw/rgw_bucket.h | 2 +- src/rgw/rgw_lc.cc | 2 +- src/rgw/rgw_rados.cc | 26 ++++++++++++++++++-------- src/rgw/rgw_rados.h | 11 +++++------ src/rgw/rgw_sal.h | 7 ++++++- src/rgw/rgw_sal_dbstore.cc | 3 ++- src/rgw/rgw_sal_rados.cc | 5 +++-- src/rgw/store/dbstore/common/dbstore.h | 7 ++++--- 9 files changed, 47 insertions(+), 28 deletions(-) diff --git a/src/rgw/rgw_bucket.cc b/src/rgw/rgw_bucket.cc index edc7f8027311f..ac9656aea8256 100644 --- a/src/rgw/rgw_bucket.cc +++ b/src/rgw/rgw_bucket.cc @@ -295,12 +295,13 @@ void check_bad_user_bucket_mapping(rgw::sal::Store* store, rgw::sal::User* user, } while (user_buckets.is_truncated()); } -// note: function type conforms to RGWRados::check_filter_t -bool rgw_bucket_object_check_filter(const string& oid) +// returns true if entry is in the empty namespace. note: function +// type conforms to type RGWBucketListNameFilter +bool rgw_bucket_object_check_filter(const std::string& oid) { - rgw_obj_key key; - string ns; - return rgw_obj_key::oid_to_key_in_ns(oid, &key, ns); + const static std::string empty_ns; + rgw_obj_key key; // thrown away but needed for parsing + return rgw_obj_key::oid_to_key_in_ns(oid, &key, empty_ns); } int rgw_remove_object(const DoutPrefixProvider *dpp, rgw::sal::Store* store, rgw::sal::Bucket* bucket, rgw_obj_key& key) @@ -604,6 +605,7 @@ int RGWBucket::check_object_index(const DoutPrefixProvider *dpp, while (results.is_truncated) { rgw::sal::Bucket::ListParams params; params.marker = results.next_marker; + params.force_check_filter = rgw_bucket_object_check_filter; int r = bucket->list(dpp, params, listing_max_entries, results, y); diff --git a/src/rgw/rgw_bucket.h b/src/rgw/rgw_bucket.h index 6ec09b9f667e6..0dbc4904f2acd 100644 --- a/src/rgw/rgw_bucket.h +++ b/src/rgw/rgw_bucket.h @@ -50,7 +50,7 @@ extern void rgw_parse_url_bucket(const std::string& bucket, std::string &tenant_name, std::string &bucket_name); // this is used as a filter to RGWRados::cls_bucket_list_ordered; it -// conforms to the type declaration of RGWRados::check_filter_t. +// conforms to the type RGWBucketListNameFilter extern bool rgw_bucket_object_check_filter(const std::string& oid); void init_default_bucket_layout(CephContext *cct, rgw::BucketLayout& layout, diff --git a/src/rgw/rgw_lc.cc b/src/rgw/rgw_lc.cc index 485848d22975a..8e19e36bc4781 100644 --- a/src/rgw/rgw_lc.cc +++ b/src/rgw/rgw_lc.cc @@ -837,7 +837,7 @@ int RGWLC::handle_multipart_expiration(rgw::sal::Bucket* target, * operating on one shard at a time */ params.allow_unordered = true; params.ns = RGW_OBJ_NS_MULTIPART; - params.filter = &mp_filter; + params.access_list_filter = &mp_filter; auto pf = [&](RGWLC::LCWorker* wk, WorkQ* wq, WorkItem& wi) { auto wt = boost::get>(wi); diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc index 9b4f77a8ca9a8..2c2b0a57ee529 100644 --- a/src/rgw/rgw_rados.cc +++ b/src/rgw/rgw_rados.cc @@ -110,6 +110,7 @@ static RGWObjCategory main_category = RGWObjCategory::Main; #define RGW_USAGE_OBJ_PREFIX "usage." +// returns true on success, false on failure static bool rgw_get_obj_data_pool(const RGWZoneGroup& zonegroup, const RGWZoneParams& zone_params, const rgw_placement_rule& head_placement_rule, const rgw_obj& obj, rgw_pool *pool) @@ -1866,7 +1867,8 @@ int RGWRados::Bucket::List::list_objects_ordered( &truncated, &cls_filtered, &cur_marker, - y); + y, + params.force_check_filter); if (r < 0) { return r; } @@ -1918,8 +1920,8 @@ int RGWRados::Bucket::List::list_objects_ordered( next_marker = index_key; } - if (params.filter && - ! params.filter->filter(obj.name, index_key.name)) { + if (params.access_list_filter && + ! params.access_list_filter->filter(obj.name, index_key.name)) { continue; } @@ -2168,12 +2170,15 @@ int RGWRados::Bucket::List::list_objects_unordered(const DoutPrefixProvider *dpp continue; } - if (params.filter && !params.filter->filter(obj.name, index_key.name)) + if (params.access_list_filter && + !params.access_list_filter->filter(obj.name, index_key.name)) { continue; + } if (params.prefix.size() && - (0 != obj.name.compare(0, params.prefix.size(), params.prefix))) + (0 != obj.name.compare(0, params.prefix.size(), params.prefix))) { continue; + } if (count >= max) { truncated = true; @@ -2329,6 +2334,7 @@ int RGWRados::create_bucket(const RGWUserInfo& owner, rgw_bucket& bucket, return -ENOENT; } +// returns true on success, false on failure bool RGWRados::get_obj_data_pool(const rgw_placement_rule& placement_rule, const rgw_obj& obj, rgw_pool *pool) { return rgw_get_obj_data_pool(svc.zone->get_zonegroup(), svc.zone->get_zone_params(), placement_rule, obj, pool); @@ -8377,7 +8383,7 @@ int RGWRados::cls_bucket_list_ordered(const DoutPrefixProvider *dpp, bool* cls_filtered, rgw_obj_index_key *last_entry, optional_yield y, - check_filter_t force_check_filter) + RGWBucketListNameFilter force_check_filter) { /* expansion_factor allows the number of entries to read to grow * exponentially; this is used when earlier reads are producing too @@ -8670,7 +8676,7 @@ int RGWRados::cls_bucket_list_unordered(const DoutPrefixProvider *dpp, bool *is_truncated, rgw_obj_index_key *last_entry, optional_yield y, - check_filter_t force_check_filter) { + RGWBucketListNameFilter force_check_filter) { ldpp_dout(dpp, 10) << __func__ << " " << bucket_info.bucket << " start_after=\"" << start_after << "\", prefix=\"" << prefix << @@ -9049,12 +9055,16 @@ int RGWRados::check_disk_state(const DoutPrefixProvider *dpp, list_state.meta.category = main_category; list_state.meta.etag = etag; list_state.meta.content_type = content_type; - if (astate->obj_tag.length() > 0) + + if (astate->obj_tag.length() > 0) { list_state.tag = astate->obj_tag.c_str(); + } + list_state.meta.owner = owner.get_id().to_str(); list_state.meta.owner_display_name = owner.get_display_name(); list_state.exists = true; + cls_rgw_encode_suggestion(CEPH_RGW_UPDATE | suggest_flag, list_state, suggested_updates); return 0; } diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h index 26b603b128b01..f335ecc00babf 100644 --- a/src/rgw/rgw_rados.h +++ b/src/rgw/rgw_rados.h @@ -986,13 +986,14 @@ class RGWRados rgw_obj_key end_marker; std::string ns; bool enforce_ns; - RGWAccessListFilter *filter; + RGWAccessListFilter* access_list_filter; + RGWBucketListNameFilter force_check_filter; bool list_versions; bool allow_unordered; Params() : enforce_ns(true), - filter(NULL), + access_list_filter(nullptr), list_versions(false), allow_unordered(false) {} @@ -1361,8 +1362,6 @@ class RGWRados using ent_map_t = boost::container::flat_map; - using check_filter_t = bool (*)(const std::string&); - int cls_bucket_list_ordered(const DoutPrefixProvider *dpp, RGWBucketInfo& bucket_info, const int shard_id, @@ -1377,7 +1376,7 @@ class RGWRados bool* cls_filtered, rgw_obj_index_key *last_entry, optional_yield y, - check_filter_t force_check_filter = nullptr); + RGWBucketListNameFilter force_check_filter = {}); int cls_bucket_list_unordered(const DoutPrefixProvider *dpp, RGWBucketInfo& bucket_info, int shard_id, @@ -1389,7 +1388,7 @@ class RGWRados bool *is_truncated, rgw_obj_index_key *last_entry, optional_yield y, - check_filter_t = nullptr); + RGWBucketListNameFilter force_check_filter = {}); int cls_bucket_head(const DoutPrefixProvider *dpp, const RGWBucketInfo& bucket_info, int shard_id, diff --git a/src/rgw/rgw_sal.h b/src/rgw/rgw_sal.h index 68120239214bf..7bd96a7bc4100 100644 --- a/src/rgw/rgw_sal.h +++ b/src/rgw/rgw_sal.h @@ -38,6 +38,10 @@ class RGWSyncModuleInstance; typedef std::shared_ptr RGWSyncModuleInstanceRef; class RGWCompressionInfo; + +using RGWBucketListNameFilter = std::function; + + namespace rgw { class Aio; namespace IAM { struct Policy; } @@ -378,7 +382,8 @@ class Bucket { rgw_obj_key end_marker; std::string ns; bool enforce_ns{true}; - RGWAccessListFilter* filter{nullptr}; + RGWAccessListFilter* access_list_filter{nullptr}; + RGWBucketListNameFilter force_check_filter; bool list_versions{false}; bool allow_unordered{false}; int shard_id{RGW_NO_SHARD}; diff --git a/src/rgw/rgw_sal_dbstore.cc b/src/rgw/rgw_sal_dbstore.cc index 67cbdad666d37..cd0484939828f 100644 --- a/src/rgw/rgw_sal_dbstore.cc +++ b/src/rgw/rgw_sal_dbstore.cc @@ -349,7 +349,8 @@ namespace rgw::sal { list_op.params.end_marker = params.end_marker; list_op.params.ns = params.ns; list_op.params.enforce_ns = params.enforce_ns; - list_op.params.filter = params.filter; + list_op.params.access_list_filter = params.access_list_filter; + list_op.params.force_check_filter = params.force_check_filter; list_op.params.list_versions = params.list_versions; list_op.params.allow_unordered = params.allow_unordered; diff --git a/src/rgw/rgw_sal_rados.cc b/src/rgw/rgw_sal_rados.cc index 5ce3b2f000939..e36c901a4fa91 100644 --- a/src/rgw/rgw_sal_rados.cc +++ b/src/rgw/rgw_sal_rados.cc @@ -736,7 +736,8 @@ int RadosBucket::list(const DoutPrefixProvider* dpp, ListParams& params, int max list_op.params.end_marker = params.end_marker; list_op.params.ns = params.ns; list_op.params.enforce_ns = params.enforce_ns; - list_op.params.filter = params.filter; + list_op.params.access_list_filter = params.access_list_filter; + list_op.params.force_check_filter = params.force_check_filter; list_op.params.list_versions = params.list_versions; list_op.params.allow_unordered = params.allow_unordered; @@ -766,7 +767,7 @@ int RadosBucket::list_multiparts(const DoutPrefixProvider *dpp, params.delim = delim; params.marker = marker; params.ns = RGW_OBJ_NS_MULTIPART; - params.filter = &mp_filter; + params.access_list_filter = &mp_filter; int ret = list(dpp, params, max_uploads, results, null_yield); diff --git a/src/rgw/store/dbstore/common/dbstore.h b/src/rgw/store/dbstore/common/dbstore.h index 3515fbe718bc0..4619ff55d2a97 100644 --- a/src/rgw/store/dbstore/common/dbstore.h +++ b/src/rgw/store/dbstore/common/dbstore.h @@ -1378,13 +1378,14 @@ class DB { rgw_obj_key end_marker; string ns; bool enforce_ns; - RGWAccessListFilter *filter; + RGWAccessListFilter* access_list_filter; + RGWBucketListNameFilter force_check_filter; bool list_versions; - bool allow_unordered; + bool allow_unordered; Params() : enforce_ns(true), - filter(NULL), + access_list_filter(nullptr), list_versions(false), allow_unordered(false) {} From edde516a26cedc705af793501e86ad3f1094ca5c Mon Sep 17 00:00:00 2001 From: Feng Hualong Date: Tue, 10 Aug 2021 16:22:33 +0800 Subject: [PATCH 0160/2866] rgw: osd compression bypass after rgw compression In particular, when rgw is itself doing compression, rgw should send a incompressible hint to OSD. Because the OSD has little effect on the data compressed in the condition. So there bypassing the osd compression is to avoid repeated compression calculation. Signed-off-by: Feng Hualong --- src/rgw/rgw_obj_manifest.h | 1 + src/rgw/rgw_op.cc | 2 ++ src/rgw/rgw_putobj_processor.cc | 14 ++++++++++++++ src/rgw/rgw_putobj_processor.h | 3 +++ src/rgw/rgw_rados.cc | 15 ++++++++++++++- src/rgw/rgw_rados.h | 5 +++++ src/rgw/rgw_sal.h | 1 + src/rgw/rgw_sal_dbstore.cc | 7 +++++++ src/rgw/rgw_sal_dbstore.h | 1 + src/rgw/rgw_sal_rados.cc | 5 +++++ src/rgw/rgw_sal_rados.h | 1 + 11 files changed, 54 insertions(+), 1 deletion(-) diff --git a/src/rgw/rgw_obj_manifest.h b/src/rgw/rgw_obj_manifest.h index 98b43e51a92d1..45f3ac8adf7e0 100644 --- a/src/rgw/rgw_obj_manifest.h +++ b/src/rgw/rgw_obj_manifest.h @@ -563,6 +563,7 @@ struct RGWObjState { bufferlist olh_tag; uint64_t pg_ver{false}; uint32_t zone_short_id{0}; + bool compressed{false}; /* important! don't forget to update copy constructor */ diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc index 25bf48daff905..61a44802ba9b2 100644 --- a/src/rgw/rgw_op.cc +++ b/src/rgw/rgw_op.cc @@ -4019,6 +4019,8 @@ void RGWPutObj::execute(optional_yield y) } else { compressor.emplace(s->cct, plugin, filter); filter = &*compressor; + // always send incompressible hint when rgw is itself doing compression + s->object->set_compressed(s->obj_ctx); } } } diff --git a/src/rgw/rgw_putobj_processor.cc b/src/rgw/rgw_putobj_processor.cc index 58cff29802885..3b87e8efb354c 100644 --- a/src/rgw/rgw_putobj_processor.cc +++ b/src/rgw/rgw_putobj_processor.cc @@ -76,6 +76,18 @@ static int process_completed(const AioResultList& completed, RawObjSet *written) return error.value_or(0); } +void RadosWriter::add_write_hint(librados::ObjectWriteOperation& op) { + const rgw_obj obj = head_obj->get_obj(); + const RGWObjState *obj_state = obj_ctx.get_state(obj); + const bool compressed = obj_state->compressed; + uint32_t alloc_hint_flags = 0; + if (compressed) { + alloc_hint_flags |= librados::ALLOC_HINT_FLAG_INCOMPRESSIBLE; + } + + op.set_alloc_hint2(0, 0, alloc_hint_flags); +} + int RadosWriter::set_stripe_obj(const rgw_raw_obj& raw_obj) { stripe_obj = store->svc()->rados->obj(raw_obj); @@ -90,6 +102,7 @@ int RadosWriter::process(bufferlist&& bl, uint64_t offset) return 0; } librados::ObjectWriteOperation op; + add_write_hint(op); if (offset == 0) { op.write_full(data); } else { @@ -106,6 +119,7 @@ int RadosWriter::write_exclusive(const bufferlist& data) librados::ObjectWriteOperation op; op.create(true); // exclusive create + add_write_hint(op); op.write_full(data); constexpr uint64_t id = 0; // unused diff --git a/src/rgw/rgw_putobj_processor.h b/src/rgw/rgw_putobj_processor.h index b376b26ce16b6..5aa4d62339871 100644 --- a/src/rgw/rgw_putobj_processor.h +++ b/src/rgw/rgw_putobj_processor.h @@ -88,6 +88,9 @@ class RadosWriter : public rgw::sal::DataProcessor { ~RadosWriter(); + // add alloc hint to osd + void add_write_hint(librados::ObjectWriteOperation& op); + // change the current stripe object int set_stripe_obj(const rgw_raw_obj& obj); diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc index 9b4f77a8ca9a8..3bf45ebb6546c 100644 --- a/src/rgw/rgw_rados.cc +++ b/src/rgw/rgw_rados.cc @@ -233,6 +233,7 @@ RGWObjState::RGWObjState(const RGWObjState& rhs) : obj (rhs.obj) { is_olh = rhs.is_olh; objv_tracker = rhs.objv_tracker; pg_ver = rhs.pg_ver; + compressed = rhs.compressed; } RGWObjState *RGWObjectCtx::get_state(const rgw_obj& obj) { @@ -253,6 +254,12 @@ RGWObjState *RGWObjectCtx::get_state(const rgw_obj& obj) { return result; } +void RGWObjectCtx::set_compressed(const rgw_obj& obj) { + std::unique_lock wl{lock}; + assert (!obj.empty()); + objs_state[obj].compressed = true; +} + void RGWObjectCtx::set_atomic(rgw_obj& obj) { std::unique_lock wl{lock}; assert (!obj.empty()); @@ -272,13 +279,15 @@ void RGWObjectCtx::invalidate(const rgw_obj& obj) { } bool is_atomic = iter->second.is_atomic; bool prefetch_data = iter->second.prefetch_data; + bool compressed = iter->second.compressed; objs_state.erase(iter); - if (is_atomic || prefetch_data) { + if (is_atomic || prefetch_data || compressed) { auto& state = objs_state[obj]; state.is_atomic = is_atomic; state.prefetch_data = prefetch_data; + state.compressed = compressed; } } @@ -3090,6 +3099,10 @@ int RGWRados::Object::Write::_do_write_meta(const DoutPrefixProvider *dpp, /* if we want to overwrite the data, we also want to overwrite the xattrs, so just remove the object */ op.write_full(*meta.data); + if (state->compressed) { + uint32_t alloc_hint_flags = librados::ALLOC_HINT_FLAG_INCOMPRESSIBLE; + op.set_alloc_hint2(0, 0, alloc_hint_flags); + } } string etag; diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h index 26b603b128b01..8e74bacdc3793 100644 --- a/src/rgw/rgw_rados.h +++ b/src/rgw/rgw_rados.h @@ -203,6 +203,7 @@ class RGWObjectCtx { RGWObjState *get_state(const rgw_obj& obj); + void set_compressed(const rgw_obj& obj); void set_atomic(rgw_obj& obj); void set_prefetch_data(const rgw_obj& obj); void invalidate(const rgw_obj& obj); @@ -1317,6 +1318,10 @@ class RGWRados RGWObjectCtx *rctx = static_cast(ctx); rctx->set_prefetch_data(obj); } + void set_compressed(void *ctx, const rgw_obj& obj) { + RGWObjectCtx *rctx = static_cast(ctx); + rctx->set_compressed(obj); + } int decode_policy(const DoutPrefixProvider *dpp, bufferlist& bl, ACLOwner *owner); int get_bucket_stats(const DoutPrefixProvider *dpp, RGWBucketInfo& bucket_info, int shard_id, std::string *bucket_ver, std::string *master_ver, std::map& stats, std::string *max_marker, bool* syncstopped = NULL); diff --git a/src/rgw/rgw_sal.h b/src/rgw/rgw_sal.h index 68120239214bf..b46b3656046b2 100644 --- a/src/rgw/rgw_sal.h +++ b/src/rgw/rgw_sal.h @@ -685,6 +685,7 @@ class Object { virtual int set_acl(const RGWAccessControlPolicy& acl) = 0; virtual void set_atomic(RGWObjectCtx* rctx) const = 0; virtual void set_prefetch_data(RGWObjectCtx* rctx) = 0; + virtual void set_compressed(RGWObjectCtx* rctx) = 0; bool empty() const { return key.empty(); } const std::string &get_name() const { return key.name; } diff --git a/src/rgw/rgw_sal_dbstore.cc b/src/rgw/rgw_sal_dbstore.cc index 67cbdad666d37..38fac3ba7c108 100644 --- a/src/rgw/rgw_sal_dbstore.cc +++ b/src/rgw/rgw_sal_dbstore.cc @@ -525,6 +525,13 @@ namespace rgw::sal { return; } + /* RGWObjectCtx will be moved out of sal */ + /* XXX: Placeholder. Should not be needed later after Dan's patch */ + void DBObject::set_compressed(RGWObjectCtx* rctx) + { + return; + } + bool DBObject::is_expired() { return false; } diff --git a/src/rgw/rgw_sal_dbstore.h b/src/rgw/rgw_sal_dbstore.h index 2f3689fa66304..dd8b8aa05d00a 100644 --- a/src/rgw/rgw_sal_dbstore.h +++ b/src/rgw/rgw_sal_dbstore.h @@ -337,6 +337,7 @@ class DBNotification : public Notification { virtual int set_acl(const RGWAccessControlPolicy& acl) override { acls = acl; return 0; } virtual void set_atomic(RGWObjectCtx* rctx) const override; virtual void set_prefetch_data(RGWObjectCtx* rctx) override; + virtual void set_compressed(RGWObjectCtx* rctx) override; virtual int get_obj_state(const DoutPrefixProvider* dpp, RGWObjectCtx* rctx, RGWObjState **state, optional_yield y, bool follow_olh = true) override; virtual int set_obj_attrs(const DoutPrefixProvider* dpp, RGWObjectCtx* rctx, Attrs* setattrs, Attrs* delattrs, optional_yield y, rgw_obj* target_obj = NULL) override; diff --git a/src/rgw/rgw_sal_rados.cc b/src/rgw/rgw_sal_rados.cc index 5ce3b2f000939..94b736999ee22 100644 --- a/src/rgw/rgw_sal_rados.cc +++ b/src/rgw/rgw_sal_rados.cc @@ -1573,6 +1573,11 @@ int RadosObject::copy_obj_data(RGWObjectCtx& rctx, Bucket* dest_bucket, real_time(), NULL, dpp, y); } +void RadosObject::set_compressed(RGWObjectCtx* rctx) { + rgw_obj obj = get_obj(); + store->getRados()->set_compressed(rctx, obj); +} + void RadosObject::set_atomic(RGWObjectCtx* rctx) const { rgw_obj obj = get_obj(); diff --git a/src/rgw/rgw_sal_rados.h b/src/rgw/rgw_sal_rados.h index 14b486b1df27c..849f1856a66b7 100644 --- a/src/rgw/rgw_sal_rados.h +++ b/src/rgw/rgw_sal_rados.h @@ -150,6 +150,7 @@ class RadosObject : public Object { virtual int set_acl(const RGWAccessControlPolicy& acl) override { acls = acl; return 0; } virtual void set_atomic(RGWObjectCtx* rctx) const override; virtual void set_prefetch_data(RGWObjectCtx* rctx) override; + virtual void set_compressed(RGWObjectCtx* rctx) override; virtual int get_obj_state(const DoutPrefixProvider* dpp, RGWObjectCtx* rctx, RGWObjState **state, optional_yield y, bool follow_olh = true) override; virtual int set_obj_attrs(const DoutPrefixProvider* dpp, RGWObjectCtx* rctx, Attrs* setattrs, Attrs* delattrs, optional_yield y, rgw_obj* target_obj = NULL) override; From 055dfa8d39137e4a0cd18865624e10be7c2b828c Mon Sep 17 00:00:00 2001 From: Xuehan Xu Date: Thu, 14 Oct 2021 17:41:18 +0800 Subject: [PATCH 0161/2866] crimson/common: extract parallel_for_each into errorator-loop.h otherwise, crimson::errorator::parallel_for_each would be referencing incomplete crimson::parallel_for_each_state type Signed-off-by: Xuehan Xu --- src/crimson/common/errorator-loop.h | 30 ++++++++++++++++++ src/crimson/common/errorator.h | 47 ++++++++++------------------- 2 files changed, 46 insertions(+), 31 deletions(-) diff --git a/src/crimson/common/errorator-loop.h b/src/crimson/common/errorator-loop.h index d2da4775dbcc7..b17217276e892 100644 --- a/src/crimson/common/errorator-loop.h +++ b/src/crimson/common/errorator-loop.h @@ -57,4 +57,34 @@ class parallel_for_each_state final : private seastar::continuation_base<> { } }; +template +static inline typename errorator::template future<> +parallel_for_each(Iterator first, Iterator last, Func&& func) noexcept { + parallel_for_each_state* s = nullptr; + // Process all elements, giving each future the following treatment: + // - available, not failed: do nothing + // - available, failed: collect exception in ex + // - not available: collect in s (allocating it if needed) + for (;first != last; ++first) { + auto f = seastar::futurize_invoke(std::forward(func), *first); + if (!f.available() || f.failed()) { + if (!s) { + using itraits = std::iterator_traits; + auto n = (seastar::internal::iterator_range_estimate_vector_capacity( + first, last, typename itraits::iterator_category()) + 1); + s = new parallel_for_each_state(n); + } + s->add_future(std::move(f)); + } + } + // If any futures were not available, hand off to parallel_for_each_state::start(). + // Otherwise we can return a result immediately. + if (s) { + // s->get_future() takes ownership of s (and chains it to one of the futures it contains) + // so this isn't a leak + return s->get_future(); + } + return seastar::make_ready_future<>(); +} + } // namespace crimson diff --git a/src/crimson/common/errorator.h b/src/crimson/common/errorator.h index a3c26a6128c5d..8472ad00eedfb 100644 --- a/src/crimson/common/errorator.h +++ b/src/crimson/common/errorator.h @@ -331,6 +331,13 @@ class parallel_for_each_state; template static inline constexpr bool is_error_v = std::is_base_of_v, T>; +template +struct errorator; + +template +static inline typename errorator::template future<> +parallel_for_each(Iterator first, Iterator last, Func&& func) noexcept; + template struct errorator { @@ -894,44 +901,22 @@ struct errorator { return make_ready_future<>(); } - template - static inline errorator::future<> - parallel_for_each(Iterator first, Iterator last, Func&& func) noexcept { - parallel_for_each_state* s = nullptr; - // Process all elements, giving each future the following treatment: - // - available, not failed: do nothing - // - available, failed: collect exception in ex - // - not available: collect in s (allocating it if needed) - for (;first != last; ++first) { - auto f = seastar::futurize_invoke(std::forward(func), *first); - if (!f.available() || f.failed()) { - if (!s) { - using itraits = std::iterator_traits; - auto n = (seastar::internal::iterator_range_estimate_vector_capacity( - first, last, typename itraits::iterator_category()) + 1); - s = new parallel_for_each_state(n); - } - s->add_future(std::move(f)); - } - } - // If any futures were not available, hand off to parallel_for_each_state::start(). - // Otherwise we can return a result immediately. - if (s) { - // s->get_future() takes ownership of s (and chains it to one of the futures it contains) - // so this isn't a leak - return s->get_future(); - } - return seastar::make_ready_future<>(); - } - template static inline auto parallel_for_each(Container&& container, Func&& func) noexcept { - return parallel_for_each( + return crimson::parallel_for_each( std::begin(container), std::end(container), std::forward(func)); } + template + static inline errorator::future<> + parallel_for_each(Iterator first, Iterator last, Func&& func) noexcept { + return crimson::parallel_for_each( + first, + last, + std::forward(func)); + } private: template class futurize { From b93d748d295e5d213bfc35abfa799e31a1bcd850 Mon Sep 17 00:00:00 2001 From: Xuehan Xu Date: Fri, 15 Oct 2021 13:42:41 +0800 Subject: [PATCH 0162/2866] common: add prepare_iovs to bufferlist Enable bufferlist that have more than IOV_MAX buffers to build a vector of iov. Signed-off-by: Xuehan Xu --- src/include/buffer.h | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/src/include/buffer.h b/src/include/buffer.h index 7c8f90e9fb5d3..adf56400ecae0 100644 --- a/src/include/buffer.h +++ b/src/include/buffer.h @@ -1196,6 +1196,40 @@ struct error_code; ++n; } } + + // vector, iov> + using iov_vec_t = + std::vector, std::vector>>; + + iov_vec_t prepare_iovs() + { + iov_vec_t iovs; + size_t iovs_i = 0; + size_t index = 0; + uint64_t off = 0; + iovs.resize(_num / IOV_MAX + 1); + iovs[iovs_i].second.resize( + std::min(_num - IOV_MAX * iovs_i, (size_t)IOV_MAX)); + iovs[iovs_i].first.first = off; + iovs[iovs_i].first.second = 0; + for (auto& bp : _buffers) { + if (index == IOV_MAX) { + iovs_i++; + index = 0; + iovs[iovs_i].first.first = off; + iovs[iovs_i].first.second = 0; + iovs[iovs_i].second.resize( + std::min(_num - IOV_MAX * iovs_i, (size_t)IOV_MAX)); + } + iovs[iovs_i].second[index].iov_base = (void*)bp.c_str(); + iovs[iovs_i].second[index++].iov_len = bp.length(); + off += bp.length(); + iovs[iovs_i].first.second += bp.length(); + } + + return iovs; + } + uint32_t crc32c(uint32_t crc) const; void invalidate_crc(); From a6bb0e9771eec7067ba03582583c78ded6eed8ee Mon Sep 17 00:00:00 2001 From: Kefu Chai Date: Fri, 15 Oct 2021 06:44:55 +0800 Subject: [PATCH 0163/2866] mgr/dashboard: s/hosts.clickHostTab/hosts.clickTab/ this change is a follow-up fix of 3276147049511a9a8d45d84c95154a4ec014cd83 otherwise we have following test failure: [lint:tsc ] cypress/integration/orchestrator/01-hosts.e2e-spec.ts(29,13): error TS2339: Property 'clickHostTab' does not exist on type 'HostsPageHelper'. also change "host" to "hostname" to be more consistent Fixes: https://tracker.ceph.com/issues/52645 Signed-off-by: Kefu Chai --- .../integration/orchestrator/01-hosts.e2e-spec.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/01-hosts.e2e-spec.ts b/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/01-hosts.e2e-spec.ts index 465ca311002f7..5344fcaed3bed 100644 --- a/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/01-hosts.e2e-spec.ts +++ b/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/01-hosts.e2e-spec.ts @@ -22,11 +22,11 @@ describe('Hosts page', () => { }); it('should drain and delete a host and then add it back', function () { - const host = Cypress._.last(this.hosts)['name']; + const hostname = Cypress._.last(this.hosts)['name']; // should drain the host first before deleting - hosts.editLabels(host, ['_no_schedule'], true); - hosts.clickHostTab(host, 'Daemons'); + hosts.editLabels(hostname, ['_no_schedule'], true); + hosts.clickTab('cd-host-details', hostname, 'Daemons'); cy.get('cd-host-details').within(() => { // draining will take some time to complete. // since we don't know how many daemons will be @@ -35,12 +35,12 @@ describe('Hosts page', () => { cy.wait(15000); hosts.getTableCount('total').should('be.eq', 0); }); - hosts.delete(host); + hosts.delete(hostname); // add it back hosts.navigateTo('add'); - hosts.add(host); - hosts.checkExist(host, true); + hosts.add(hostname); + hosts.checkExist(hostname, true); }); it('should display inventory', function () { From 2c49202027bc1b8dfbfa69aadc42b7edaf8db60a Mon Sep 17 00:00:00 2001 From: Chunsong Feng Date: Wed, 20 Nov 2019 09:42:11 +0800 Subject: [PATCH 0164/2866] msg/async/dpdk: exit condition waiting when DPDKStack is destructed exit() will call pthread_cond_destroy attempting to destroy dpdk::eal::cond upon which other threads are currently blocked results in undefine behavior. Link different libc version test, libc-2.17 can exit, libc-2.27 will deadlock, the call stack is as follows: Thread 3 (Thread 0xffff7e5749f0 (LWP 62213)): #0 0x0000ffff7f3c422c in futex_wait_cancelable (private=, expected=0, futex_word=0xaaaadc0e30f4 ) at ../sysdeps/unix/sysv/linux/futex-internal.h:88 #1 __pthread_cond_wait_common (abstime=0x0, mutex=0xaaaadc0e30f8 , cond=0xaaaadc0e30c8 ) at pthread_cond_wait.c:502 #2 __pthread_cond_wait (cond=0xaaaadc0e30c8 , mutex=0xaaaadc0e30f8 ) at pthread_cond_wait.c:655 #3 0x0000ffff7f1f1f80 in std::condition_variable::wait(std::unique_lock&) () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #4 0x0000aaaad37f5078 in dpdk::eal::::operator()(void) const (__closure=, __closure=) at ./src/msg/async/dpdk/dpdk_rte.cc:136 #5 0x0000ffff7f1f7ed4 in ?? () from /usr/lib/aarch64-linux-gnu/libstdc++.so.6 #6 0x0000ffff7f3be088 in start_thread (arg=0xffffe73e197f) at pthread_create.c:463 #7 0x0000ffff7efc74ec in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:78 Thread 1 (Thread 0xffff7ee3b010 (LWP 62200)): #0 0x0000ffff7f3c3c38 in futex_wait (private=, expected=12, futex_word=0xaaaadc0e30ec ) at ../sysdeps/unix/sysv/linux/futex-internal.h:61 #1 futex_wait_simple (private=, expected=12, futex_word=0xaaaadc0e30ec ) at ../sysdeps/nptl/futex-internal.h:135 #2 __pthread_cond_destroy (cond=0xaaaadc0e30c8 ) at pthread_cond_destroy.c:54 #3 0x0000ffff7ef2be34 in __run_exit_handlers (status=-6, listp=0xffff7f04a5a0 <__exit_funcs>, run_list_atexit=255, run_list_atexit@entry=true, run_dtors=run_dtors@entry=true) at exit.c:108 #4 0x0000ffff7ef2bf6c in __GI_exit (status=) at exit.c:139 #5 0x0000ffff7ef176e4 in __libc_start_main (main=0x0, argc=0, argv=0x0, init=, fini=, rtld_fini=, stack_end=) at ../csu/libc-start.c:344 #6 0x0000aaaad2939db0 in _start () at ./src/include/buffer.h:642 Fixes: https://tracker.ceph.com/issues/42890 Signed-off-by: Chunsong Feng Signed-off-by: luo rixin --- src/msg/async/dpdk/DPDKStack.cc | 12 ++++---- src/msg/async/dpdk/DPDKStack.h | 6 +++- src/msg/async/dpdk/dpdk_rte.cc | 52 ++++++++++++++++++--------------- src/msg/async/dpdk/dpdk_rte.h | 23 +++++++++------ 4 files changed, 55 insertions(+), 38 deletions(-) diff --git a/src/msg/async/dpdk/DPDKStack.cc b/src/msg/async/dpdk/DPDKStack.cc index a3451eec76966..1543a530df9da 100644 --- a/src/msg/async/dpdk/DPDKStack.cc +++ b/src/msg/async/dpdk/DPDKStack.cc @@ -249,12 +249,12 @@ void DPDKStack::spawn_worker(std::function &&func) // funcs.push_back(std::move(func)); int r = 0; - r = dpdk::eal::init(cct); + r = eal.start(); if (r < 0) { - lderr(cct) << __func__ << " init dpdk rte failed, r=" << r << dendl; + lderr(cct) << __func__ << " start dpdk rte failed, r=" << r << dendl; ceph_abort(); } - // if dpdk::eal::init already called by NVMEDevice, we will select 1..n + // if eal.start already called by NVMEDevice, we will select 1..n // cores unsigned nr_worker = funcs.size(); ceph_assert(rte_lcore_count() >= nr_worker); @@ -265,7 +265,7 @@ void DPDKStack::spawn_worker(std::function &&func) } } void *adapted_func = static_cast(&funcs.back()); - dpdk::eal::execute_on_master([adapted_func, core_id, this]() { + eal.execute_on_master([adapted_func, core_id, this]() { int r = rte_eal_remote_launch(dpdk_thread_adaptor, adapted_func, core_id); if (r < 0) { lderr(cct) << __func__ << " remote launch failed, r=" << r << dendl; @@ -276,7 +276,9 @@ void DPDKStack::spawn_worker(std::function &&func) void DPDKStack::join_worker(unsigned i) { - dpdk::eal::execute_on_master([&]() { + eal.execute_on_master([&]() { rte_eal_wait_lcore(i+1); }); + if (i+1 == get_num_worker()) + eal.stop(); } diff --git a/src/msg/async/dpdk/DPDKStack.h b/src/msg/async/dpdk/DPDKStack.h index 4eb8604c7fbed..e46bb6f51039e 100644 --- a/src/msg/async/dpdk/DPDKStack.h +++ b/src/msg/async/dpdk/DPDKStack.h @@ -25,6 +25,7 @@ #include "const.h" #include "IP.h" #include "Packet.h" +#include "dpdk_rte.h" class interface; @@ -246,6 +247,7 @@ class DPDKWorker : public Worker { friend class DPDKServerSocketImpl; }; +using namespace dpdk; class DPDKStack : public NetworkStack { std::vector > funcs; @@ -254,13 +256,15 @@ class DPDKStack : public NetworkStack { } public: - explicit DPDKStack(CephContext *cct): NetworkStack(cct) { + explicit DPDKStack(CephContext *cct): NetworkStack(cct), eal(cct) { funcs.reserve(cct->_conf->ms_async_op_threads); } virtual bool support_local_listen_table() const override { return true; } virtual void spawn_worker(std::function &&func) override; virtual void join_worker(unsigned i) override; + private: + dpdk::eal eal; }; #endif diff --git a/src/msg/async/dpdk/dpdk_rte.cc b/src/msg/async/dpdk/dpdk_rte.cc index 872d39d519de4..3ebb64360b04a 100644 --- a/src/msg/async/dpdk/dpdk_rte.cc +++ b/src/msg/async/dpdk/dpdk_rte.cc @@ -34,12 +34,6 @@ namespace dpdk { return v; } - bool eal::initialized = false; - std::thread eal::t; - std::mutex eal::lock; - std::condition_variable eal::cond; - std::list> eal::funcs; - static int bitcount(unsigned long long n) { return std::bitset{n}.count(); @@ -75,31 +69,33 @@ namespace dpdk { return count; } - int eal::init(CephContext *c) + bool eal::rte_initialized = false; + + int eal::start() { if (initialized) { return 1; } bool done = false; - auto coremask = c->_conf.get_val("ms_dpdk_coremask"); + auto coremask = cct->_conf.get_val("ms_dpdk_coremask"); int coremaskbit = coremask_bitcount(coremask.c_str()); if (coremaskbit <= 0 - || static_cast(coremaskbit) < c->_conf->ms_async_op_threads) + || static_cast(coremaskbit) < cct->_conf->ms_async_op_threads) return -EINVAL; t = std::thread([&]() { // TODO: Inherit these from the app parameters - "opts" std::vector> args { string2vector("ceph"), - string2vector("-c"), string2vector(c->_conf.get_val("ms_dpdk_coremask")), - string2vector("-n"), string2vector(c->_conf->ms_dpdk_memory_channel), + string2vector("-c"), string2vector(cct->_conf.get_val("ms_dpdk_coremask")), + string2vector("-n"), string2vector(cct->_conf->ms_dpdk_memory_channel), }; std::optional hugepages_path; - if (!c->_conf->ms_dpdk_hugepages.empty()) { - hugepages_path.emplace(c->_conf->ms_dpdk_hugepages); + if (!cct->_conf->ms_dpdk_hugepages.empty()) { + hugepages_path.emplace(cct->_conf->ms_dpdk_hugepages); } // If "hugepages" is not provided and DPDK PMD drivers mode is requested - @@ -123,13 +119,13 @@ namespace dpdk { args.push_back(string2vector("-m")); args.push_back(string2vector(size_MB_str.str())); - } else if (!c->_conf->ms_dpdk_pmd.empty()) { + } else if (!cct->_conf->ms_dpdk_pmd.empty()) { args.push_back(string2vector("--no-huge")); } std::string rte_file_prefix; rte_file_prefix = "rte_"; - rte_file_prefix += c->_conf->name.to_str(); + rte_file_prefix += cct->_conf->name.to_str(); args.push_back(string2vector("--file-prefix")); args.push_back(string2vector(rte_file_prefix)); @@ -138,27 +134,28 @@ namespace dpdk { for (auto&& a: args) { cargs.push_back(a.data()); } - /* initialise the EAL for all */ - int ret = rte_eal_init(cargs.size(), cargs.data()); - if (ret < 0) - return ret; + if (!rte_initialized) { + /* initialise the EAL for all */ + int ret = rte_eal_init(cargs.size(), cargs.data()); + if (ret < 0) + return; + rte_initialized = true; + } std::unique_lock l(lock); initialized = true; done = true; cond.notify_all(); - while (true) { + while (!stopped) { + cond.wait(l, [this] { return !funcs.empty() || stopped; }); if (!funcs.empty()) { auto f = std::move(funcs.front()); funcs.pop_front(); f(); cond.notify_all(); - } else { - cond.wait(l); } } }); - t.detach(); std::unique_lock l(lock); while (!done) cond.wait(l); @@ -182,4 +179,13 @@ namespace dpdk { return memsize; } + void eal::stop() + { + assert(initialized); + assert(!stopped); + stopped = true; + cond.notify_all(); + t.join(); + } + } // namespace dpdk diff --git a/src/msg/async/dpdk/dpdk_rte.h b/src/msg/async/dpdk/dpdk_rte.h index 4aa8389949241..6784af6d4fa5b 100644 --- a/src/msg/async/dpdk/dpdk_rte.h +++ b/src/msg/async/dpdk/dpdk_rte.h @@ -46,12 +46,10 @@ namespace dpdk { class eal { public: using cpuset = std::bitset; - - static std::mutex lock; - static std::condition_variable cond; - static std::list> funcs; - static int init(CephContext *c); - static void execute_on_master(std::function &&f) { + explicit eal(CephContext *cct) : cct(cct) {} + int start(); + void stop(); + void execute_on_master(std::function &&f) { bool done = false; std::unique_lock l(lock); funcs.emplace_back([&]() { f(); done = true; }); @@ -65,9 +63,16 @@ class eal { * * @return */ - static size_t mem_size(int num_cpus); - static bool initialized; - static std::thread t; + size_t mem_size(int num_cpus); + static bool rte_initialized; + private: + CephContext *cct; + bool initialized = false; + bool stopped = false; + std::thread t; + std::mutex lock; + std::condition_variable cond; + std::list> funcs; }; } // namespace dpdk From 9f9b021087c669d931d3d5801090af87fa0fe606 Mon Sep 17 00:00:00 2001 From: Xuehan Xu Date: Thu, 14 Oct 2021 17:43:23 +0800 Subject: [PATCH 0165/2866] crimson/os/seastore: deal with scenarios in which bufferlist of do_writev has more than IOV_MAX buffers Signed-off-by: Xuehan Xu --- .../os/seastore/segment_manager/block.cc | 58 ++++++++++++------- 1 file changed, 38 insertions(+), 20 deletions(-) diff --git a/src/crimson/os/seastore/segment_manager/block.cc b/src/crimson/os/seastore/segment_manager/block.cc index 5b395fac589b6..2fb9e4e41931c 100644 --- a/src/crimson/os/seastore/segment_manager/block.cc +++ b/src/crimson/os/seastore/segment_manager/block.cc @@ -5,6 +5,7 @@ #include #include "crimson/common/config_proxy.h" +#include "crimson/common/errorator-loop.h" #include "crimson/common/log.h" #include "include/buffer.h" @@ -53,31 +54,43 @@ static write_ertr::future<> do_writev( size_t block_size) { logger().debug( - "block: do_writev offset {} len {}", + "block: do_writev offset {} len {}, {} buffers", offset, - bl.length()); + bl.length(), + bl.get_num_buffers()); // writev requires each buffer to be aligned to the disks' block // size, we need to rebuild here bl.rebuild_aligned(block_size); - std::vector iov; - bl.prepare_iov(&iov); - return device.dma_write( - offset, - std::move(iov) - ).handle_exception([](auto e) -> write_ertr::future { - logger().error( - "do_writev: dma_write got error {}", - e); - return crimson::ct_error::input_output_error::make(); - }).then( - [bl=std::move(bl)/* hold the buf until the end of io */](size_t written) - -> write_ertr::future<> { - if (written != bl.length()) { - return crimson::ct_error::input_output_error::make(); - } - return write_ertr::now(); + return seastar::do_with( + bl.prepare_iovs(), + std::move(bl), + [&device, offset] (auto& iovs, auto& bl) { + return write_ertr::parallel_for_each( + iovs, + [&device, offset](auto& p) mutable { + auto off = offset + p.first.first; + auto len = p.first.second; + auto& iov = p.second; + logger().debug("do_writev: dma_write to {}, length {}", off, len); + return device.dma_write(off, std::move(iov)) + .handle_exception([](auto e) -> write_ertr::future { + logger().error( + "do_writev: dma_write got error {}", + e); + return crimson::ct_error::input_output_error::make(); + }).then( + [off, len](size_t written) -> write_ertr::future<> { + if (written != len) { + logger().error( + "do_writev: dma_write to {}, failed, written {} != iov len {}", + off, written, len); + return crimson::ct_error::input_output_error::make(); + } + return write_ertr::now(); + }); + }); }); } @@ -96,7 +109,12 @@ static read_ertr::future<> do_read( offset, bptr.c_str(), len - ).handle_exception([](auto e) -> read_ertr::future { + ).handle_exception( + //FIXME: this is a little bit tricky, since seastar::future::handle_exception + // returns seastar::future, to return an crimson::ct_error, we have to create + // a seastar::future holding that crimson::ct_error. This is not necessary + // once seastar::future::handle_exception() returns seastar::futurize_t + [](auto e) -> read_ertr::future { logger().error( "do_read: dma_read got error {}", e); From 0a86107dccfdffa7476c61312f0c3b5aae7da7b7 Mon Sep 17 00:00:00 2001 From: Joao Eduardo Luis Date: Mon, 11 Oct 2021 17:20:47 +0000 Subject: [PATCH 0166/2866] cephadm: allow pulling from insecure registries Signed-off-by: Joao Eduardo Luis --- src/cephadm/cephadm | 19 ++++++++++++++----- src/cephadm/tests/test_cephadm.py | 1 + src/pybind/mgr/cephadm/module.py | 7 +++++++ src/pybind/mgr/cephadm/serve.py | 6 +++++- 4 files changed, 27 insertions(+), 6 deletions(-) diff --git a/src/cephadm/cephadm b/src/cephadm/cephadm index ed392bd187d75..0b487b489aec3 100755 --- a/src/cephadm/cephadm +++ b/src/cephadm/cephadm @@ -3725,12 +3725,12 @@ def command_version(ctx): def command_pull(ctx): # type: (CephadmContext) -> int - _pull_image(ctx, ctx.image) + _pull_image(ctx, ctx.image, ctx.insecure) return command_inspect_image(ctx) -def _pull_image(ctx, image): - # type: (CephadmContext, str) -> None +def _pull_image(ctx, image, insecure=False): + # type: (CephadmContext, str, bool) -> None logger.info('Pulling container image %s...' % image) ignorelist = [ @@ -3740,8 +3740,12 @@ def _pull_image(ctx, image): ] cmd = [ctx.container_engine.path, 'pull', image] - if isinstance(ctx.container_engine, Podman) and os.path.exists('/etc/ceph/podman-auth.json'): - cmd.append('--authfile=/etc/ceph/podman-auth.json') + if isinstance(ctx.container_engine, Podman): + if insecure: + cmd.append('--tls-verify=false') + + if os.path.exists('/etc/ceph/podman-auth.json'): + cmd.append('--authfile=/etc/ceph/podman-auth.json') cmd_str = ' '.join(cmd) for sleep_secs in [1, 4, 25]: @@ -7401,6 +7405,11 @@ def _get_parser(): parser_pull = subparsers.add_parser( 'pull', help='pull latest image version') parser_pull.set_defaults(func=command_pull) + parser_pull.add_argument( + '--insecure', + action='store_true', + help=argparse.SUPPRESS, + ) parser_inspect_image = subparsers.add_parser( 'inspect-image', help='inspect local container image') diff --git a/src/cephadm/tests/test_cephadm.py b/src/cephadm/tests/test_cephadm.py index d95f76ea1b8f4..af1d086c5a7c1 100644 --- a/src/cephadm/tests/test_cephadm.py +++ b/src/cephadm/tests/test_cephadm.py @@ -1362,6 +1362,7 @@ class TestPull: def test_error(self, get_image_info_from_inspect, call, sleep): ctx = cd.CephadmContext() ctx.container_engine = mock_podman() + ctx.insecure = False call.return_value = ('', '', 0) retval = cd.command_pull(ctx) diff --git a/src/pybind/mgr/cephadm/module.py b/src/pybind/mgr/cephadm/module.py index 41476c1c99207..4742e01932e61 100644 --- a/src/pybind/mgr/cephadm/module.py +++ b/src/pybind/mgr/cephadm/module.py @@ -295,6 +295,12 @@ class CephadmOrchestrator(orchestrator.Orchestrator, MgrModule, default=None, desc='Custom repository password' ), + Option( + 'registry_insecure', + type='bool', + default=False, + desc='Registry is to be considered insecure (no TLS available). Only for development purposes.' + ), Option( 'use_repo_digest', type='bool', @@ -402,6 +408,7 @@ def __init__(self, *args: Any, **kwargs: Any): self.registry_url: Optional[str] = None self.registry_username: Optional[str] = None self.registry_password: Optional[str] = None + self.registry_insecure: bool = False self.use_repo_digest = True self.default_registry = '' self.autotune_memory_target_ratio = 0.0 diff --git a/src/pybind/mgr/cephadm/serve.py b/src/pybind/mgr/cephadm/serve.py index c93e95cc26c04..e8734cfd21fce 100644 --- a/src/pybind/mgr/cephadm/serve.py +++ b/src/pybind/mgr/cephadm/serve.py @@ -1309,7 +1309,11 @@ def _get_container_image_info(self, image_name: str) -> ContainerInspectInfo: self._registry_login(host, self.mgr.registry_url, self.mgr.registry_username, self.mgr.registry_password) - j = self._run_cephadm_json(host, '', 'pull', [], image=image_name, no_fsid=True) + pullargs: List[str] = [] + if self.mgr.registry_insecure: + pullargs.append("--insecure") + + j = self._run_cephadm_json(host, '', 'pull', pullargs, image=image_name, no_fsid=True) r = ContainerInspectInfo( j['image_id'], From 8e4f9ea46ceec267748d60c6ed009b209d2352ab Mon Sep 17 00:00:00 2001 From: Sebastian Wagner Date: Fri, 15 Oct 2021 13:13:02 +0200 Subject: [PATCH 0167/2866] doc/cephadm: Hosts: Add "initial CRUSH location" Followup on d2a9a35993 Signed-off-by: Sebastian Wagner --- doc/cephadm/host-management.rst | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/doc/cephadm/host-management.rst b/doc/cephadm/host-management.rst index 2253cb6e14df6..c109e2dc73b32 100644 --- a/doc/cephadm/host-management.rst +++ b/doc/cephadm/host-management.rst @@ -169,13 +169,14 @@ Where the force flag when entering maintenance allows the user to bypass warning See also :ref:`cephadm-fqdn` -Host Specification -================== +Creating many hosts at once +=========================== Many hosts can be added at once using -``ceph orch apply -i`` by submitting a multi-document YAML file:: +``ceph orch apply -i`` by submitting a multi-document YAML file: + +.. code-block:: yaml - --- service_type: host hostname: node-00 addr: 192.168.0.10 @@ -197,6 +198,26 @@ This can be combined with service specifications (below) to create a cluster spe file to deploy a whole cluster in one command. see ``cephadm bootstrap --apply-spec`` also to do this during bootstrap. Cluster SSH Keys must be copied to hosts prior to adding them. +Setting the initial CRUSH location of host +========================================== + +Hosts can contain a ``location`` identifier which will instruct cephadm to +create a new CRUSH host located in the specified hierachy. + +.. code-block:: yaml + + service_type: host + hostname: node-00 + addr: 192.168.0.10 + location: + rack: rack1 + +.. note:: + + The ``location`` attribute will be only affect the initial CRUSH location. Subsequent + changes of the ``location`` property will be ignored. Also, removing a host will no remove + any CRUSH buckets. + SSH Configuration ================= From 53cc27fde8ad8131cb00a9a3bd281ca2673241b3 Mon Sep 17 00:00:00 2001 From: Laura Flores Date: Thu, 30 Sep 2021 22:08:51 +0000 Subject: [PATCH 0168/2866] mgr/telemetry: add mempool stats to the perf report Adding mempool stats to the telemetry perf report for a deeper look at memory usage. Signed-off-by: Laura Flores --- src/pybind/mgr/telemetry/module.py | 39 ++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/src/pybind/mgr/telemetry/module.py b/src/pybind/mgr/telemetry/module.py index 91596436a9d86..dd120ff9d0ae1 100644 --- a/src/pybind/mgr/telemetry/module.py +++ b/src/pybind/mgr/telemetry/module.py @@ -302,6 +302,42 @@ def gather_configs(self) -> Dict[str, List[str]]: 'active_changed': sorted(list(active)), } + def get_mempool(self, mode: str = 'separated') -> Dict[str, dict]: + # Initialize result dict + result: Dict[str, dict] = defaultdict(lambda: defaultdict(int)) + + # Get list of osd ids from the metadata + osd_metadata = self.get('osd_metadata') + + # Grab output from the "osd.x dump_mempools" command + for osd_id in osd_metadata: + cmd_dict = { + 'prefix': 'dump_mempools', + 'id': str(osd_id), + 'format': 'json' + } + r, outb, outs = self.osd_command(cmd_dict) + if r != 0: + self.log.debug("Invalid command dictionary.") + continue + else: + try: + # This is where the mempool will land. + dump = json.loads(outb) + if mode == 'separated': + result["osd." + str(osd_id)] = dump['mempool']['by_pool'] + elif mode == 'aggregated': + for mem_type in dump['mempool']['by_pool']: + result[mem_type]['bytes'] += dump['mempool']['by_pool'][mem_type]['bytes'] + result[mem_type]['items'] += dump['mempool']['by_pool'][mem_type]['items'] + else: + self.log.debug("Incorrect mode specified in get_mempool") + except (json.decoder.JSONDecodeError, KeyError) as e: + self.log.debug("Error caught: {}".format(e)) + return {} + + return result + def get_stat_sum_per_pool(self) -> List[dict]: # Initialize 'result' list result: List[dict] = [] @@ -895,6 +931,9 @@ def compile_report(self, channels: Optional[List[str]] = None) -> Dict[str, Any] report['io_rate'] = self.get_io_rate() report['osd_perf_histograms'] = self.get_osd_histograms() + report['mempool_aggregated'] = self.get_mempool('aggregated') + report['mempool_separated'] = self.get_mempool('separated') + # NOTE: We do not include the 'device' channel in this report; it is # sent to a different endpoint. From be3fd05fc59bd523c1171d18cb0a0784f6a6d5c1 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Fri, 15 Oct 2021 11:09:07 -0400 Subject: [PATCH 0169/2866] mgr/cephadm: make upgrade ls output structured Signed-off-by: Sage Weil --- src/pybind/mgr/cephadm/module.py | 2 +- src/pybind/mgr/cephadm/upgrade.py | 25 +++++++++++++---------- src/pybind/mgr/orchestrator/_interface.py | 2 +- 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/src/pybind/mgr/cephadm/module.py b/src/pybind/mgr/cephadm/module.py index 720422902cc07..08361e47622df 100644 --- a/src/pybind/mgr/cephadm/module.py +++ b/src/pybind/mgr/cephadm/module.py @@ -2470,7 +2470,7 @@ def upgrade_status(self) -> orchestrator.UpgradeStatusSpec: return self.upgrade.upgrade_status() @handle_orch_error - def upgrade_ls(self, image: Optional[str], tags: bool) -> List[str]: + def upgrade_ls(self, image: Optional[str], tags: bool) -> Dict[Any, Any]: return self.upgrade.upgrade_ls(image, tags) @handle_orch_error diff --git a/src/pybind/mgr/cephadm/upgrade.py b/src/pybind/mgr/cephadm/upgrade.py index e71c9c8bf3a1c..4e23e0d121ea8 100644 --- a/src/pybind/mgr/cephadm/upgrade.py +++ b/src/pybind/mgr/cephadm/upgrade.py @@ -2,7 +2,7 @@ import logging import time import uuid -from typing import TYPE_CHECKING, Optional, Dict, List, Tuple +from typing import TYPE_CHECKING, Optional, Dict, List, Tuple, Any import orchestrator from cephadm.registry import Registry @@ -177,15 +177,18 @@ def _check_target_version(self, version: str) -> Optional[str]: return None - def upgrade_ls(self, image: Optional[str], tags: bool) -> List[str]: - if image: - reg_name, image = image.split('/', 1) - else: - reg_name, image = self.mgr.container_image_base.split('/', 1) - self.mgr.log.info(f'reg_name {reg_name} image {image}') + def upgrade_ls(self, image: Optional[str], tags: bool) -> Dict: + if not image: + image = self.mgr.container_image_base + reg_name, bare_image = image.split('/', 1) reg = Registry(reg_name) versions = [] - ls = reg.get_tags(image) + r: Dict[Any, Any] = { + "image": image, + "registry": reg_name, + "bare_image": bare_image, + } + ls = reg.get_tags(bare_image) if not tags: for t in ls: if t[0] != 'v': @@ -196,14 +199,14 @@ def upgrade_ls(self, image: Optional[str], tags: bool) -> List[str]: if '-' in v[2]: continue versions.append('.'.join(v)) - ls = sorted( + r["versions"] = sorted( versions, key=lambda k: list(map(int, k.split('.'))), reverse=True ) else: - ls = sorted(ls) - return ls + r["tags"] = sorted(ls) + return r def upgrade_start(self, image: str, version: str) -> str: if self.mgr.mode != 'root': diff --git a/src/pybind/mgr/orchestrator/_interface.py b/src/pybind/mgr/orchestrator/_interface.py index ffc77784c3013..04b62022b9690 100644 --- a/src/pybind/mgr/orchestrator/_interface.py +++ b/src/pybind/mgr/orchestrator/_interface.py @@ -642,7 +642,7 @@ def apply_alertmanager(self, spec: ServiceSpec) -> OrchResult[str]: def upgrade_check(self, image: Optional[str], version: Optional[str]) -> OrchResult[str]: raise NotImplementedError() - def upgrade_ls(self, image: Optional[str], tags: bool) -> OrchResult[List[str]]: + def upgrade_ls(self, image: Optional[str], tags: bool) -> OrchResult[Dict[Any, Any]]: raise NotImplementedError() def upgrade_start(self, image: Optional[str], version: Optional[str]) -> OrchResult[str]: From ca8997acaf01f185da1707f677bf1f7f23b5260f Mon Sep 17 00:00:00 2001 From: Casey Bodley Date: Fri, 15 Oct 2021 11:57:13 -0400 Subject: [PATCH 0170/2866] rgw: silence error message for nonexistent realm in a normal vstart cluster without a realm or period, every command prints this error message: 0 RGWPeriod::init failed to init realm id : (2) No such file or directory Signed-off-by: Casey Bodley --- src/rgw/rgw_zone.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rgw/rgw_zone.cc b/src/rgw/rgw_zone.cc index 9dd98d43f5dff..a9f6ecfd9c9cf 100644 --- a/src/rgw/rgw_zone.cc +++ b/src/rgw/rgw_zone.cc @@ -1009,7 +1009,7 @@ int RGWPeriod::init(const DoutPrefixProvider *dpp, RGWRealm realm(realm_id, realm_name); int ret = realm.init(dpp, cct, sysobj_svc, y); if (ret < 0) { - ldpp_dout(dpp, 0) << "RGWPeriod::init failed to init realm " << realm_name << " id " << realm_id << " : " << + ldpp_dout(dpp, 4) << "RGWPeriod::init failed to init realm " << realm_name << " id " << realm_id << " : " << cpp_strerror(-ret) << dendl; return ret; } From 46236139f9f3aa722ee533deb41f7c651b2df5b9 Mon Sep 17 00:00:00 2001 From: Laura Flores Date: Fri, 15 Oct 2021 16:07:01 +0000 Subject: [PATCH 0171/2866] mon: remove detail option for `mgr module ls` command Signed-off-by: Laura Flores --- doc/man/8/ceph.rst | 2 +- doc/mgr/administrator.rst | 2 +- src/mon/MgrMonitor.cc | 20 ++++---------------- src/mon/MonCommands.h | 2 +- 4 files changed, 7 insertions(+), 19 deletions(-) diff --git a/doc/man/8/ceph.rst b/doc/man/8/ceph.rst index af381df92ea8a..639d85e565d6f 100644 --- a/doc/man/8/ceph.rst +++ b/doc/man/8/ceph.rst @@ -580,7 +580,7 @@ Subcommand ``module ls`` will list currently enabled manager modules (plugins). Usage:: - ceph mgr module ls {detail} + ceph mgr module ls Subcommand ``module enable`` will enable a manager module. Available modules are included in MgrMap and visible via ``mgr dump``. diff --git a/doc/mgr/administrator.rst b/doc/mgr/administrator.rst index 6115d171c014c..411ad8c38b7c1 100644 --- a/doc/mgr/administrator.rst +++ b/doc/mgr/administrator.rst @@ -57,7 +57,7 @@ Using modules ------------- Use the command ``ceph mgr module ls`` to see which modules are -available, and which are currently enabled. Use ``ceph mgr module ls detail`` +available, and which are currently enabled. Use ``ceph mgr module ls --format=json-pretty`` to view detailed metadata about disabled modules. Enable or disable modules using the commands ``ceph mgr module enable `` and ``ceph mgr module disable `` respectively. diff --git a/src/mon/MgrMonitor.cc b/src/mon/MgrMonitor.cc index c9458aab4eeb9..ed44af17d0203 100644 --- a/src/mon/MgrMonitor.cc +++ b/src/mon/MgrMonitor.cc @@ -25,7 +25,6 @@ #include "HealthMonitor.h" #include "common/TextTable.h" -#include "common/cmdparse.h" #include "include/stringify.h" #include "MgrMonitor.h" @@ -955,7 +954,6 @@ bool MgrMonitor::preprocess_command(MonOpRequestRef op) string prefix; cmd_getval(cmdmap, "prefix", prefix); - int r = 0; if (prefix == "mgr stat") { @@ -991,12 +989,7 @@ bool MgrMonitor::preprocess_command(MonOpRequestRef op) } f->flush(rdata); } else if (prefix == "mgr module ls") { - string detail; - cmd_getval(cmdmap, "detail", detail); - if (f || (detail == "detail")) { - if (!f) { - f.reset(Formatter::create(format, "json-pretty", "json-pretty")); - } + if (f) { f->open_object_section("modules"); { f->open_array_section("always_on_modules"); @@ -1017,14 +1010,9 @@ bool MgrMonitor::preprocess_command(MonOpRequestRef op) for (auto& p : map.available_modules) { if (map.modules.count(p.name) == 0 && map.get_always_on_modules().count(p.name) == 0) { - if (detail == "detail") { - // For disabled modules, we show the full info if the detail - // parameter is enabled, to give a hint about whether enabling it will work - p.dump(f.get()); - } else { - // Otherwise, we give a shortened summary by default - f->dump_string("module", p.name); - } + // For disabled modules, we show the full info if the detail + // parameter is enabled, to give a hint about whether enabling it will work + p.dump(f.get()); } } f->close_section(); diff --git a/src/mon/MonCommands.h b/src/mon/MonCommands.h index 78129f336e256..cd2bc434a24c8 100644 --- a/src/mon/MonCommands.h +++ b/src/mon/MonCommands.h @@ -1231,7 +1231,7 @@ COMMAND("mgr dump " "mgr", "r") COMMAND("mgr fail name=who,type=CephString,req=false", "treat the named manager daemon as failed", "mgr", "rw") -COMMAND("mgr module ls name=detail,type=CephChoices,strings=detail,req=false", +COMMAND("mgr module ls", "list active mgr modules", "mgr", "r") COMMAND("mgr services", "list service endpoints provided by mgr modules", From 0dcea098cf4d51bed31d8646dc3b533514c08a72 Mon Sep 17 00:00:00 2001 From: Ilya Dryomov Date: Fri, 15 Oct 2021 18:13:55 +0200 Subject: [PATCH 0172/2866] librbd: honor FUA op flag for write_same() in write-around cache WriteAroundObjectDispatch::write_same() should pass op_flags through to dispatch_io() so that it can bypass the cache if needed. Fixes: https://tracker.ceph.com/issues/52956 Signed-off-by: Ilya Dryomov --- src/librbd/cache/WriteAroundObjectDispatch.cc | 4 +-- .../test_mock_WriteAroundObjectDispatch.cc | 26 ++++++++++++++++++- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/src/librbd/cache/WriteAroundObjectDispatch.cc b/src/librbd/cache/WriteAroundObjectDispatch.cc index adffeacfb0fa3..fafb73f409ae3 100644 --- a/src/librbd/cache/WriteAroundObjectDispatch.cc +++ b/src/librbd/cache/WriteAroundObjectDispatch.cc @@ -113,8 +113,8 @@ bool WriteAroundObjectDispatch::write_same( ldout(cct, 20) << data_object_name(m_image_ctx, object_no) << " " << object_off << "~" << object_len << dendl; - return dispatch_io(object_no, object_off, object_len, 0, dispatch_result, - on_finish, on_dispatched); + return dispatch_io(object_no, object_off, object_len, op_flags, + dispatch_result, on_finish, on_dispatched); } template diff --git a/src/test/librbd/cache/test_mock_WriteAroundObjectDispatch.cc b/src/test/librbd/cache/test_mock_WriteAroundObjectDispatch.cc index 52d2bd295069f..abfd185e31db4 100644 --- a/src/test/librbd/cache/test_mock_WriteAroundObjectDispatch.cc +++ b/src/test/librbd/cache/test_mock_WriteAroundObjectDispatch.cc @@ -651,7 +651,7 @@ TEST_F(TestMockCacheWriteAroundObjectDispatch, UnoptimizedIOBlockedIO) { finish_ctx_ptr2->complete(0); } -TEST_F(TestMockCacheWriteAroundObjectDispatch, FUA) { +TEST_F(TestMockCacheWriteAroundObjectDispatch, WriteFUA) { librbd::ImageCtx *ictx; ASSERT_EQ(0, open_image(m_image_name, &ictx)); @@ -675,5 +675,29 @@ TEST_F(TestMockCacheWriteAroundObjectDispatch, FUA) { ASSERT_EQ(finish_ctx_ptr, &finish_ctx); } +TEST_F(TestMockCacheWriteAroundObjectDispatch, WriteSameFUA) { + librbd::ImageCtx *ictx; + ASSERT_EQ(0, open_image(m_image_name, &ictx)); + + MockTestImageCtx mock_image_ctx(*ictx); + MockWriteAroundObjectDispatch object_dispatch(&mock_image_ctx, 16384, false); + + InSequence seq; + + bufferlist data; + data.append(std::string(512, '1')); + + io::DispatchResult dispatch_result; + MockContext finish_ctx; + MockContext dispatch_ctx; + Context* finish_ctx_ptr = &finish_ctx; + ASSERT_FALSE(object_dispatch.write_same(0, 0, 8192, {{0, 8192}}, + std::move(data), {}, + LIBRADOS_OP_FLAG_FADVISE_FUA, {}, + nullptr, nullptr, &dispatch_result, + &finish_ctx_ptr, &dispatch_ctx)); + ASSERT_EQ(finish_ctx_ptr, &finish_ctx); +} + } // namespace cache } // namespace librbd From 02bec8fba40518ed78a09a8d300bd70b1109315d Mon Sep 17 00:00:00 2001 From: Anthony D'Atri Date: Sun, 10 Oct 2021 13:25:00 -0700 Subject: [PATCH 0173/2866] common/options: document rgw_lc_debug_interval configuration option Signed-off-by: Anthony D'Atri --- src/common/options/rgw.yaml.in | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/common/options/rgw.yaml.in b/src/common/options/rgw.yaml.in index 0cbf397783eeb..f1800dea48484 100644 --- a/src/common/options/rgw.yaml.in +++ b/src/common/options/rgw.yaml.in @@ -406,6 +406,10 @@ options: - name: rgw_lc_debug_interval type: int level: dev + desc: Lifecycle processing interval + long_desc: When set to greater than 0, treat this many seconds as a day. + This facilitates debugging and testing of lifecycle code without lengthy + delays. default: -1 services: - rgw From 62bfc8ba71ca488df3c95dec8b8cc2c07c116972 Mon Sep 17 00:00:00 2001 From: myoungwon oh Date: Sun, 10 Oct 2021 12:07:12 +0900 Subject: [PATCH 0174/2866] tools/ceph_dedup_tool: add repair command Until now, ceph-dedup-tool only provides chunk-scrub ,which reports how many dangling references the object has, but there is no cli command to fix the reference mismach. Therefore, this commit adds repair command to correct detected reference mismatch. Signed-off-by: Myoungwon Oh --- src/tools/ceph_dedup_tool.cc | 94 ++++++++++++++++++++++++++++++------ 1 file changed, 80 insertions(+), 14 deletions(-) diff --git a/src/tools/ceph_dedup_tool.cc b/src/tools/ceph_dedup_tool.cc index 2c117529e0ff6..99e36e0347eb4 100644 --- a/src/tools/ceph_dedup_tool.cc +++ b/src/tools/ceph_dedup_tool.cc @@ -133,7 +133,7 @@ ceph::mutex glock = ceph::make_mutex("glock"); void usage() { - cout << " usage: [--op ] [--pool ] " << std::endl; + cout << " usage: [--op ] [--pool ] " << std::endl; cout << " --object " << std::endl; cout << " --chunk-size chunk-size (byte) " << std::endl; cout << " --chunk-algorithm " << std::endl; @@ -767,7 +767,8 @@ int chunk_scrub_common(const std::map < std::string, std::string > &opts, } if (op_name == "chunk-get-ref" || - op_name == "chunk-put-ref") { + op_name == "chunk-put-ref" || + op_name == "chunk-repair") { string target_object_name; uint64_t pool_id; i = opts.find("object"); @@ -801,17 +802,83 @@ int chunk_scrub_common(const std::map < std::string, std::string > &opts, } hobject_t oid(sobject_t(target_object_name, CEPH_NOSNAP), "", hash, pool_id, ""); + auto run_op = [] (ObjectWriteOperation& op, hobject_t& oid, + string& object_name, IoCtx& chunk_io_ctx) -> int { + int ret = chunk_io_ctx.operate(object_name, &op); + if (ret < 0) { + cerr << " operate fail : " << cpp_strerror(ret) << std::endl; + } + return ret; + }; + ObjectWriteOperation op; if (op_name == "chunk-get-ref") { cls_cas_chunk_get_ref(op, oid); - } else { + ret = run_op(op, oid, object_name, chunk_io_ctx); + } else if (op_name == "chunk-put-ref") { cls_cas_chunk_put_ref(op, oid); + ret = run_op(op, oid, object_name, chunk_io_ctx); + } else if (op_name == "chunk-repair") { + ret = rados.ioctx_create2(pool_id, io_ctx); + if (ret < 0) { + cerr << oid << " ref " << pool_id + << ": referencing pool does not exist" << std::endl; + return ret; + } + int chunk_ref = -1, base_ref = -1; + // read object on chunk pool to know how many reference the object has + bufferlist t; + ret = chunk_io_ctx.getxattr(object_name, CHUNK_REFCOUNT_ATTR, t); + if (ret < 0) { + return ret; + } + chunk_refs_t refs; + auto p = t.cbegin(); + decode(refs, p); + if (refs.get_type() != chunk_refs_t::TYPE_BY_OBJECT) { + cerr << " does not supported chunk type " << std::endl; + return -1; + } + chunk_ref = + static_cast(refs.r.get())->by_object.count(oid); + if (chunk_ref < 0) { + cerr << object_name << " has no reference of " << target_object_name + << std::endl; + return chunk_ref; + } + cout << object_name << " has " << chunk_ref << " references for " + << target_object_name << std::endl; + + // read object on base pool to know the number of chunk object's references + base_ref = cls_cas_references_chunk(io_ctx, target_object_name, object_name); + if (base_ref < 0) { + if (base_ref == -ENOENT || base_ref == -ENOLINK) { + base_ref = 0; + } else { + return base_ref; + } + } + cout << target_object_name << " has " << base_ref << " references for " + << object_name << std::endl; + if (chunk_ref != base_ref) { + if (base_ref > chunk_ref) { + cerr << "error : " << target_object_name << "'s ref. < " << object_name + << "' ref. " << std::endl; + return -EINVAL; + } + cout << " fix dangling reference from " << chunk_ref << " to " << base_ref + << std::endl; + while (base_ref != chunk_ref) { + ObjectWriteOperation op; + cls_cas_chunk_put_ref(op, oid); + chunk_ref--; + ret = run_op(op, oid, object_name, chunk_io_ctx); + if (ret < 0) { + return ret; + } + } + } } - ret = chunk_io_ctx.operate(object_name, &op); - if (ret < 0) { - cerr << " operate fail : " << cpp_strerror(ret) << std::endl; - } - return ret; } else if (op_name == "dump-chunk-refs") { @@ -945,12 +1012,11 @@ int main(int argc, const char **argv) if (op_name == "estimate") { return estimate_dedup_ratio(opts, args); - } else if (op_name == "chunk-scrub") { - return chunk_scrub_common(opts, args); - } else if (op_name == "chunk-get-ref" || - op_name == "chunk-put-ref") { - return chunk_scrub_common(opts, args); - } else if (op_name == "dump-chunk-refs") { + } else if (op_name == "chunk-scrub" || + op_name == "chunk-get-ref" || + op_name == "chunk-put-ref" || + op_name == "chunk-repair" || + op_name == "dump-chunk-refs") { return chunk_scrub_common(opts, args); } else { cerr << "unrecognized op " << op_name << std::endl; From c1d2119d5e69c644a3de68de4b6af030f592ad2f Mon Sep 17 00:00:00 2001 From: myoungwon oh Date: Sun, 10 Oct 2021 22:07:54 +0900 Subject: [PATCH 0175/2866] qa: add a test for chunk-repair command Signed-off-by: Myoungwon Oh --- qa/workunits/rados/test_dedup_tool.sh | 70 +++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/qa/workunits/rados/test_dedup_tool.sh b/qa/workunits/rados/test_dedup_tool.sh index bdd14698ff997..d0f995c748923 100755 --- a/qa/workunits/rados/test_dedup_tool.sh +++ b/qa/workunits/rados/test_dedup_tool.sh @@ -162,8 +162,78 @@ function test_dedup_chunk_scrub() $RADOS_TOOL -p $POOL rm bar } +function test_dedup_chunk_repair() +{ + + CHUNK_POOL=dedup_chunk_pool + run_expect_succ "$CEPH_TOOL" osd pool create "$CHUNK_POOL" 8 + + echo -n "hi there" > foo + + echo -n "hi there" > bar + + echo -n "there" > foo-chunk + + echo -n "CHUNK" > bar-chunk + + $CEPH_TOOL osd pool set $POOL fingerprint_algorithm sha1 --yes-i-really-mean-it + $CEPH_TOOL osd pool set $POOL dedup_chunk_algorithm fastcdc --yes-i-really-mean-it + $CEPH_TOOL osd pool set $POOL dedup_cdc_chunk_size 4096 --yes-i-really-mean-it + $CEPH_TOOL osd pool set $POOL dedup_tier $CHUNK_POOL --yes-i-really-mean-it + + $RADOS_TOOL -p $POOL put foo ./foo + $RADOS_TOOL -p $POOL put bar ./bar + + $RADOS_TOOL -p $CHUNK_POOL put bar-chunk ./bar-chunk + $RADOS_TOOL -p $CHUNK_POOL put foo-chunk ./foo-chunk + + $RADOS_TOOL ls -p $CHUNK_POOL + CHUNK_OID=$(echo -n "hi there" | sha1sum | awk '{print $1}') + + POOL_ID=$($CEPH_TOOL osd pool ls detail | grep $POOL | awk '{print$2}') + + + # increase ref count by two, resuling in mismatch + $DEDUP_TOOL --op chunk-get-ref --chunk-pool $CHUNK_POOL --object $CHUNK_OID --target-ref foo --target-ref-pool-id $POOL_ID + $DEDUP_TOOL --op chunk-get-ref --chunk-pool $CHUNK_POOL --object $CHUNK_OID --target-ref foo --target-ref-pool-id $POOL_ID + $DEDUP_TOOL --op chunk-get-ref --chunk-pool $CHUNK_POOL --object bar-chunk --target-ref bar --target-ref-pool-id $POOL_ID + $DEDUP_TOOL --op chunk-get-ref --chunk-pool $CHUNK_POOL --object bar-chunk --target-ref bar --target-ref-pool-id $POOL_ID + + RESULT=$($DEDUP_TOOL --op dump-chunk-refs --chunk-pool $CHUNK_POOL --object $CHUNK_OID) + RESULT=$($DEDUP_TOOL --op chunk-scrub --chunk-pool $CHUNK_POOL | grep "Damaged object" | awk '{print$4}') + if [ $RESULT -ne "1" ] ; then + $CEPH_TOOL osd pool delete $POOL $POOL --yes-i-really-really-mean-it + $CEPH_TOOL osd pool delete $CHUNK_POOL $CHUNK_POOL --yes-i-really-really-mean-it + die "Chunk-scrub failed expecting damaged objects is not 1" + fi + + $DEDUP_TOOL --op chunk-repair --chunk-pool $CHUNK_POOL --object $CHUNK_OID --target-ref foo --target-ref-pool-id $POOL_ID + $DEDUP_TOOL --op chunk-repair --chunk-pool $CHUNK_POOL --object bar-chunk --target-ref bar --target-ref-pool-id $POOL_ID + + $DEDUP_TOOL --op dump-chunk-refs --chunk-pool $CHUNK_POOL --object $CHUNK_OID | grep foo + RESULT=$($DEDUP_TOOL --op dump-chunk-refs --chunk-pool $CHUNK_POOL --object $CHUNK_OID | grep foo | wc -l) + if [ -n "$RESULT" ] ; then + $CEPH_TOOL osd pool delete $POOL $POOL --yes-i-really-really-mean-it + $CEPH_TOOL osd pool delete $CHUNK_POOL $CHUNK_POOL --yes-i-really-really-mean-it + die "Scrub failed expecting bar is removed" + fi + RESULT=$($DEDUP_TOOL --op dump-chunk-refs --chunk-pool $CHUNK_POOL --object bar-chunk | grep bar) + if [ -n "$RESULT" ] ; then + $CEPH_TOOL osd pool delete $POOL $POOL --yes-i-really-really-mean-it + $CEPH_TOOL osd pool delete $CHUNK_POOL $CHUNK_POOL --yes-i-really-really-mean-it + die "Scrub failed expecting bar is removed" + fi + + $CEPH_TOOL osd pool delete $CHUNK_POOL $CHUNK_POOL --yes-i-really-really-mean-it + + rm -rf ./foo ./bar ./foo-chunk ./bar-chunk ./test_obj + $RADOS_TOOL -p $POOL rm foo + $RADOS_TOOL -p $POOL rm bar +} + test_dedup_ratio_fixed test_dedup_chunk_scrub +test_dedup_chunk_repair $CEPH_TOOL osd pool delete $POOL $POOL --yes-i-really-really-mean-it From e903554a7fbdb3756513dc06ef3634c1ed539946 Mon Sep 17 00:00:00 2001 From: myoungwon oh Date: Sun, 10 Oct 2021 22:45:52 +0900 Subject: [PATCH 0176/2866] tools/ceph_dedup_tool: elaborate explanation of commands Signed-off-by: Myoungwon Oh --- src/tools/ceph_dedup_tool.cc | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/tools/ceph_dedup_tool.cc b/src/tools/ceph_dedup_tool.cc index 99e36e0347eb4..adfaa3839f099 100644 --- a/src/tools/ceph_dedup_tool.cc +++ b/src/tools/ceph_dedup_tool.cc @@ -133,7 +133,17 @@ ceph::mutex glock = ceph::make_mutex("glock"); void usage() { - cout << " usage: [--op ] [--pool ] " << std::endl; + cout << +"usage: \n" +" ceph-dedup-tool \n" +" [--op estimate --pool POOL --chunk-size CHUNK_SIZE --chunk-algorithm ALGO --fingerprint-algorithm FP_ALGO] \n" +" [--op chunk-scrub --op chunk-scrub --chunk-pool POOL] \n" +" [--op chunk-get-ref --chunk-pool POOL --object OID --target-ref OID --target-ref-pool-id POOL_ID] \n" +" [--op chunk-put-ref --chunk-pool POOL --object OID --target-ref OID --target-ref-pool-id POOL_ID] \n" +" [--op chunk-repair --chunk-pool POOL --object OID --target-ref OID --target-ref-pool-id POOL_ID] \n" +" [--op dump-chunk-refs --chunk-pool POOL --object OID] \n" + << std::endl; + cout << "optional arguments: " << std::endl; cout << " --object " << std::endl; cout << " --chunk-size chunk-size (byte) " << std::endl; cout << " --chunk-algorithm " << std::endl; From c2f82d7228a6ebbca9500f66fc118e0f67a0528a Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Tue, 3 Aug 2021 13:25:26 -0400 Subject: [PATCH 0177/2866] mgr/orchestrator: clean up 'orch device ls' output - headings in CAPS - DEVICE ID, not just serial or model/vendor - drop the health, LED fields from non-wide view - sort devices - 'Yes' or '', for easier visual parsing Signed-off-by: Sage Weil --- src/pybind/mgr/orchestrator/module.py | 37 ++++++++++----------------- 1 file changed, 14 insertions(+), 23 deletions(-) diff --git a/src/pybind/mgr/orchestrator/module.py b/src/pybind/mgr/orchestrator/module.py index 8e918960b02bb..2d179f65542bb 100644 --- a/src/pybind/mgr/orchestrator/module.py +++ b/src/pybind/mgr/orchestrator/module.py @@ -461,27 +461,27 @@ def _list_devices(self, "On": "On", "Off": "Off", True: "Yes", - False: "No", + False: "", } out = [] if wide: table = PrettyTable( - ['Hostname', 'Path', 'Type', 'Transport', 'RPM', 'Vendor', 'Model', - 'Serial', 'Size', 'Health', 'Ident', 'Fault', 'Available', - 'Reject Reasons'], + ['HOST', 'PATH', 'TYPE', 'TRANSPORT', 'RPM', 'DEVICE ID', 'SIZE', + 'HEALTH', 'IDENT', 'FAULT', + 'AVAILABLE', 'REJECT REASONS'], border=False) else: table = PrettyTable( - ['Hostname', 'Path', 'Type', 'Serial', 'Size', - 'Health', 'Ident', 'Fault', 'Available'], + ['HOST', 'PATH', 'TYPE', 'DEVICE ID', 'SIZE', + 'AVAILABLE', 'REJECT REASONS'], border=False) table.align = 'l' table._align['SIZE'] = 'r' table.left_padding_width = 0 table.right_padding_width = 2 for host_ in sorted(inv_hosts, key=lambda h: h.name): # type: InventoryHost - for d in host_.devices.devices: # type: Device + for d in sorted(host_.devices.devices, key=lambda d: d.path): # type: Device led_ident = 'N/A' led_fail = 'N/A' @@ -489,24 +489,17 @@ def _list_devices(self, led_ident = d.lsm_data['ledSupport']['IDENTstatus'] led_fail = d.lsm_data['ledSupport']['FAILstatus'] - if d.device_id is not None: - fallback_serial = d.device_id.split('_')[-1] - else: - fallback_serial = "" - if wide: table.add_row( ( host_.name, d.path, d.human_readable_type, - d.lsm_data.get('transport', 'Unknown'), - d.lsm_data.get('rpm', 'Unknown'), - d.sys_api.get('vendor') or 'N/A', - d.sys_api.get('model') or 'N/A', - d.lsm_data.get('serialNum', fallback_serial), + d.lsm_data.get('transport', ''), + d.lsm_data.get('rpm', ''), + d.device_id, format_dimless(d.sys_api.get('size', 0), 5), - d.lsm_data.get('health', 'Unknown'), + d.lsm_data.get('health', ''), display_map[led_ident], display_map[led_fail], display_map[d.available], @@ -519,12 +512,10 @@ def _list_devices(self, host_.name, d.path, d.human_readable_type, - d.lsm_data.get('serialNum', fallback_serial), + d.device_id, format_dimless(d.sys_api.get('size', 0), 5), - d.lsm_data.get('health', 'Unknown'), - display_map[led_ident], - display_map[led_fail], - display_map[d.available] + display_map[d.available], + ', '.join(d.rejected_reasons) ) ) out.append(table.get_string()) From 573661de503e627de007100bad0ce7f6226d0774 Mon Sep 17 00:00:00 2001 From: Kefu Chai Date: Sat, 16 Oct 2021 07:35:21 +0800 Subject: [PATCH 0178/2866] common/bl: move bl::prepare_iovs() to .cc file to reduce the compilation time by having a smaller header file. Signed-off-by: Kefu Chai --- src/common/buffer.cc | 28 ++++++++++++++++++++++++++++ src/include/buffer.h | 29 +---------------------------- 2 files changed, 29 insertions(+), 28 deletions(-) diff --git a/src/common/buffer.cc b/src/common/buffer.cc index 1b36b0543c750..fe9b1e09f373a 100644 --- a/src/common/buffer.cc +++ b/src/common/buffer.cc @@ -2030,6 +2030,34 @@ int buffer::list::write_fd(int fd, uint64_t offset) const } #endif +buffer::list::iov_vec_t buffer::list::prepare_iovs() +{ + iov_vec_t iovs; + size_t iovs_i = 0; + size_t index = 0; + uint64_t off = 0; + iovs.resize(_num / IOV_MAX + 1); + iovs[iovs_i].second.resize( + std::min(_num - IOV_MAX * iovs_i, (size_t)IOV_MAX)); + iovs[iovs_i].first.first = off; + iovs[iovs_i].first.second = 0; + for (auto& bp : _buffers) { + if (index == IOV_MAX) { + iovs_i++; + index = 0; + iovs[iovs_i].first.first = off; + iovs[iovs_i].first.second = 0; + iovs[iovs_i].second.resize( + std::min(_num - IOV_MAX * iovs_i, (size_t)IOV_MAX)); + } + iovs[iovs_i].second[index].iov_base = (void*)bp.c_str(); + iovs[iovs_i].second[index++].iov_len = bp.length(); + off += bp.length(); + iovs[iovs_i].first.second += bp.length(); + } + return iovs; +} + __u32 buffer::list::crc32c(__u32 crc) const { int cache_misses = 0; diff --git a/src/include/buffer.h b/src/include/buffer.h index adf56400ecae0..de9ef75b3b2bc 100644 --- a/src/include/buffer.h +++ b/src/include/buffer.h @@ -1201,34 +1201,7 @@ struct error_code; using iov_vec_t = std::vector, std::vector>>; - iov_vec_t prepare_iovs() - { - iov_vec_t iovs; - size_t iovs_i = 0; - size_t index = 0; - uint64_t off = 0; - iovs.resize(_num / IOV_MAX + 1); - iovs[iovs_i].second.resize( - std::min(_num - IOV_MAX * iovs_i, (size_t)IOV_MAX)); - iovs[iovs_i].first.first = off; - iovs[iovs_i].first.second = 0; - for (auto& bp : _buffers) { - if (index == IOV_MAX) { - iovs_i++; - index = 0; - iovs[iovs_i].first.first = off; - iovs[iovs_i].first.second = 0; - iovs[iovs_i].second.resize( - std::min(_num - IOV_MAX * iovs_i, (size_t)IOV_MAX)); - } - iovs[iovs_i].second[index].iov_base = (void*)bp.c_str(); - iovs[iovs_i].second[index++].iov_len = bp.length(); - off += bp.length(); - iovs[iovs_i].first.second += bp.length(); - } - - return iovs; - } + iov_vec_t prepare_iovs(); uint32_t crc32c(uint32_t crc) const; void invalidate_crc(); From f04a6cea549ef06fabe9289024246ca2a1a99251 Mon Sep 17 00:00:00 2001 From: Kefu Chai Date: Sat, 16 Oct 2021 07:48:55 +0800 Subject: [PATCH 0179/2866] common/bl: define iov_vec_t using a dedicated struct more readable this way Signed-off-by: Kefu Chai --- src/common/buffer.cc | 18 +++++++++--------- .../os/seastore/segment_manager/block.cc | 6 +++--- src/include/buffer.h | 10 ++++++---- 3 files changed, 18 insertions(+), 16 deletions(-) diff --git a/src/common/buffer.cc b/src/common/buffer.cc index fe9b1e09f373a..6e6bf41b11871 100644 --- a/src/common/buffer.cc +++ b/src/common/buffer.cc @@ -2037,23 +2037,23 @@ buffer::list::iov_vec_t buffer::list::prepare_iovs() size_t index = 0; uint64_t off = 0; iovs.resize(_num / IOV_MAX + 1); - iovs[iovs_i].second.resize( + iovs[iovs_i].iov.resize( std::min(_num - IOV_MAX * iovs_i, (size_t)IOV_MAX)); - iovs[iovs_i].first.first = off; - iovs[iovs_i].first.second = 0; + iovs[iovs_i].offset = off; + iovs[iovs_i].length = 0; for (auto& bp : _buffers) { if (index == IOV_MAX) { iovs_i++; index = 0; - iovs[iovs_i].first.first = off; - iovs[iovs_i].first.second = 0; - iovs[iovs_i].second.resize( + iovs[iovs_i].offset = off; + iovs[iovs_i].length = 0; + iovs[iovs_i].iov.resize( std::min(_num - IOV_MAX * iovs_i, (size_t)IOV_MAX)); } - iovs[iovs_i].second[index].iov_base = (void*)bp.c_str(); - iovs[iovs_i].second[index++].iov_len = bp.length(); + iovs[iovs_i].iov[index].iov_base = (void*)bp.c_str(); + iovs[iovs_i].iov[index++].iov_len = bp.length(); off += bp.length(); - iovs[iovs_i].first.second += bp.length(); + iovs[iovs_i].length += bp.length(); } return iovs; } diff --git a/src/crimson/os/seastore/segment_manager/block.cc b/src/crimson/os/seastore/segment_manager/block.cc index 0e71ee0ed1e86..f623cee7cbf4c 100644 --- a/src/crimson/os/seastore/segment_manager/block.cc +++ b/src/crimson/os/seastore/segment_manager/block.cc @@ -70,9 +70,9 @@ static write_ertr::future<> do_writev( return write_ertr::parallel_for_each( iovs, [&device, offset](auto& p) mutable { - auto off = offset + p.first.first; - auto len = p.first.second; - auto& iov = p.second; + auto off = offset + p.offset; + auto len = p.length; + auto& iov = p.iov; logger().debug("do_writev: dma_write to {}, length {}", off, len); return device.dma_write(off, std::move(iov)) .handle_exception([](auto e) -> write_ertr::future { diff --git a/src/include/buffer.h b/src/include/buffer.h index de9ef75b3b2bc..b0b8c440dec07 100644 --- a/src/include/buffer.h +++ b/src/include/buffer.h @@ -1197,10 +1197,12 @@ struct error_code; } } - // vector, iov> - using iov_vec_t = - std::vector, std::vector>>; - + struct iovec_t { + uint64_t offset; + uint64_t length; + std::vector iov; + }; + using iov_vec_t = std::vector; iov_vec_t prepare_iovs(); uint32_t crc32c(uint32_t crc) const; From e9678d0b67ad44fcf1674eea3b6014351bd28f2a Mon Sep 17 00:00:00 2001 From: Kefu Chai Date: Sat, 16 Oct 2021 07:58:50 +0800 Subject: [PATCH 0180/2866] common/bl: simplify bl::prepare_iovs() * consolidate the code to initialize a vector * use an iterator to track the current vector. less repeating this way. Signed-off-by: Kefu Chai --- src/common/buffer.cc | 33 ++++++++++++++++----------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/src/common/buffer.cc b/src/common/buffer.cc index 6e6bf41b11871..0fd90606862ec 100644 --- a/src/common/buffer.cc +++ b/src/common/buffer.cc @@ -2032,28 +2032,27 @@ int buffer::list::write_fd(int fd, uint64_t offset) const buffer::list::iov_vec_t buffer::list::prepare_iovs() { - iov_vec_t iovs; - size_t iovs_i = 0; size_t index = 0; uint64_t off = 0; - iovs.resize(_num / IOV_MAX + 1); - iovs[iovs_i].iov.resize( - std::min(_num - IOV_MAX * iovs_i, (size_t)IOV_MAX)); - iovs[iovs_i].offset = off; - iovs[iovs_i].length = 0; + iov_vec_t iovs{_num / IOV_MAX + 1}; + auto it = iovs.begin(); for (auto& bp : _buffers) { - if (index == IOV_MAX) { - iovs_i++; + if (index == 0) { + it->offset = off; + it->length = 0; + size_t nr_iov_created = std::distance(iovs.begin(), it); + it->iov.resize( + std::min(_num - IOV_MAX * nr_iov_created, (size_t)IOV_MAX)); + } + it->iov[index].iov_base = (void*)bp.c_str(); + it->iov[index].iov_len = bp.length(); + off += bp.length(); + it->length += bp.length(); + if (++index == IOV_MAX) { + // continue with a new vector if we have more buf + ++it; index = 0; - iovs[iovs_i].offset = off; - iovs[iovs_i].length = 0; - iovs[iovs_i].iov.resize( - std::min(_num - IOV_MAX * iovs_i, (size_t)IOV_MAX)); } - iovs[iovs_i].iov[index].iov_base = (void*)bp.c_str(); - iovs[iovs_i].iov[index++].iov_len = bp.length(); - off += bp.length(); - iovs[iovs_i].length += bp.length(); } return iovs; } From 4e8173424731acf89f9cc8a8f36c63361c243907 Mon Sep 17 00:00:00 2001 From: Kefu Chai Date: Sat, 16 Oct 2021 08:31:08 +0800 Subject: [PATCH 0181/2866] common/bl: mark bl::prepare_iovs() const Signed-off-by: Kefu Chai --- src/common/buffer.cc | 2 +- src/include/buffer.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/common/buffer.cc b/src/common/buffer.cc index 0fd90606862ec..4b1bc08936cda 100644 --- a/src/common/buffer.cc +++ b/src/common/buffer.cc @@ -2030,7 +2030,7 @@ int buffer::list::write_fd(int fd, uint64_t offset) const } #endif -buffer::list::iov_vec_t buffer::list::prepare_iovs() +buffer::list::iov_vec_t buffer::list::prepare_iovs() const { size_t index = 0; uint64_t off = 0; diff --git a/src/include/buffer.h b/src/include/buffer.h index b0b8c440dec07..0c89367dd3b2c 100644 --- a/src/include/buffer.h +++ b/src/include/buffer.h @@ -1203,7 +1203,7 @@ struct error_code; std::vector iov; }; using iov_vec_t = std::vector; - iov_vec_t prepare_iovs(); + iov_vec_t prepare_iovs() const; uint32_t crc32c(uint32_t crc) const; void invalidate_crc(); From 429ac06cbb44b8a8263beb0d0780a01cedb517ba Mon Sep 17 00:00:00 2001 From: Mykola Golub Date: Wed, 13 Oct 2021 18:22:09 +0300 Subject: [PATCH 0182/2866] qa/tasks/backfill_toofull: make test work when compression on The osd backfill reservation does not take compression into account so we need to operate with "uncompressed" bytes when calculating nearfull ratio. Signed-off-by: Mykola Golub --- qa/tasks/backfill_toofull.py | 16 ++++++++++++++-- qa/tasks/ceph_manager.py | 9 +++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/qa/tasks/backfill_toofull.py b/qa/tasks/backfill_toofull.py index f2156bea6017c..1a866595d2695 100644 --- a/qa/tasks/backfill_toofull.py +++ b/qa/tasks/backfill_toofull.py @@ -139,10 +139,22 @@ def task(ctx, config): # data, so if the osd backfill reservation incorrectly calculates "toofull" # the test will detect this (fail). # + # Note, we need to operate with "uncompressed" bytes because currently + # osd backfill reservation does not take compression into account. + # # We also need to update nearfull ratio to prevent "full ratio(s) out of order". - backfillfull = min(used_kb + primary_used_kb, total_kb * 0.9) / total_kb - nearfull_min = max(used_kb, primary_used_kb) / total_kb + pdf = manager.get_pool_df(pool) + log.debug("pool %s df: %s" % (pool, df)) + assert pdf + compress_ratio = 1.0 * pdf['compress_under_bytes'] / pdf['compress_bytes_used'] \ + if pdf['compress_bytes_used'] > 0 else 1.0 + log.debug("compress_ratio: %s" % compress_ratio) + + backfillfull = (used_kb + primary_used_kb) * compress_ratio / total_kb + assert backfillfull < 0.9 + nearfull_min = max(used_kb, primary_used_kb) * compress_ratio / total_kb + assert nearfull_min < backfillfull delta = backfillfull - nearfull_min nearfull = nearfull_min + delta * 0.1 backfillfull = nearfull_min + delta * 0.2 diff --git a/qa/tasks/ceph_manager.py b/qa/tasks/ceph_manager.py index 2524635ff0518..b5b1099e9101e 100644 --- a/qa/tasks/ceph_manager.py +++ b/qa/tasks/ceph_manager.py @@ -2341,6 +2341,15 @@ def get_osd_df(self, osdid): j = json.loads('\n'.join(out.split('\n')[1:])) return j['nodes'][0] + def get_pool_df(self, name): + """ + Get the pool df stats + """ + out = self.raw_cluster_cmd('df', 'detail', '--format=json') + j = json.loads('\n'.join(out.split('\n')[1:])) + return next((p['stats'] for p in j['pools'] if p['name'] == name), + None) + def get_pgids_to_force(self, backfill): """ Return the randomized list of PGs that can have their recovery/backfill forced From 4cdbba3344fe26b6351e88ce00a8655890a02115 Mon Sep 17 00:00:00 2001 From: Taha Jahangir Date: Sun, 17 Oct 2021 08:30:27 +0330 Subject: [PATCH 0183/2866] ceph-volume: fix a typo causing AttributeError Signed-off-by: Taha Jahangir --- src/ceph-volume/ceph_volume/devices/lvm/migrate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ceph-volume/ceph_volume/devices/lvm/migrate.py b/src/ceph-volume/ceph_volume/devices/lvm/migrate.py index 811ff63ee9198..dc982f153b4ed 100644 --- a/src/ceph-volume/ceph_volume/devices/lvm/migrate.py +++ b/src/ceph-volume/ceph_volume/devices/lvm/migrate.py @@ -414,7 +414,7 @@ def migrate_osd(self): target_lv = api.get_lv_by_fullname(self.args.target) if not target_lv: mlogger.error( - 'Target path "{}" is not a Logical Volume'.formaat( + 'Target path "{}" is not a Logical Volume'.format( self.args.target)) raise SystemExit( 'Unable to migrate to : {}'.format(self.args.target)) From 7cc7efae2a64196db1866ce1c5a5a3305d0f5771 Mon Sep 17 00:00:00 2001 From: Ilya Dryomov Date: Sat, 16 Oct 2021 21:08:55 +0200 Subject: [PATCH 0184/2866] librbd: preserve CEPH_OSD_FLAG_FULL_TRY in create_ioctx() The obvious use case is an image with a separate data pool but it could be useful in other places too. While at it, set_namespace() call in handle_v2_get_data_pool() is redundant since create_ioctx() already takes care of it. Fixes: https://tracker.ceph.com/issues/52961 Signed-off-by: Ilya Dryomov --- src/librbd/Utils.cc | 3 ++ src/librbd/image/OpenRequest.cc | 1 - src/test/librbd/test_librbd.cc | 63 ++++++++++++++++++++++++--------- 3 files changed, 49 insertions(+), 18 deletions(-) diff --git a/src/librbd/Utils.cc b/src/librbd/Utils.cc index 112f342bd0262..0ea31fc1c9d43 100644 --- a/src/librbd/Utils.cc +++ b/src/librbd/Utils.cc @@ -149,6 +149,9 @@ int create_ioctx(librados::IoCtx& src_io_ctx, const std::string& pool_desc, dst_io_ctx->set_namespace( pool_namespace ? *pool_namespace : src_io_ctx.get_namespace()); + if (src_io_ctx.get_pool_full_try()) { + dst_io_ctx->set_pool_full_try(); + } return 0; } diff --git a/src/librbd/image/OpenRequest.cc b/src/librbd/image/OpenRequest.cc index d34dd9ec89793..70008d712ca69 100644 --- a/src/librbd/image/OpenRequest.cc +++ b/src/librbd/image/OpenRequest.cc @@ -483,7 +483,6 @@ Context *OpenRequest::handle_v2_get_data_pool(int *result) { } m_image_ctx->data_ctx.close(); } else { - m_image_ctx->data_ctx.set_namespace(m_image_ctx->md_ctx.get_namespace()); m_image_ctx->rebuild_data_io_context(); } } else { diff --git a/src/test/librbd/test_librbd.cc b/src/test/librbd/test_librbd.cc index 53bee26f70160..1ea88f1ca68e0 100644 --- a/src/test/librbd/test_librbd.cc +++ b/src/test/librbd/test_librbd.cc @@ -2118,32 +2118,22 @@ TEST_F(TestLibRBD, ConcurrentCreatesUnvalidatedPool) rados_ioctx_destroy(ioctx); } -TEST_F(TestLibRBD, CreateThickRemoveFullTry) +static void remove_full_try(rados_ioctx_t ioctx, const std::string& image_name, + const std::string& data_pool_name) { - REQUIRE(!is_librados_test_stub(_rados)); - - rados_ioctx_t ioctx; - auto pool_name = create_pool(true); - ASSERT_EQ(0, rados_ioctx_create(_cluster, pool_name.c_str(), &ioctx)); - int order = 0; - auto image_name = get_temp_image_name(); uint64_t quota = 10 << 20; uint64_t size = 5 * quota; ASSERT_EQ(0, create_image(ioctx, image_name.c_str(), size, &order)); - // FIXME: this is a workaround for rbd_trash object being created - // on the first remove -- pre-create it to avoid bumping into quota - ASSERT_EQ(0, rbd_remove(ioctx, image_name.c_str())); - ASSERT_EQ(0, create_image(ioctx, image_name.c_str(), size, &order)); - std::string cmdstr = "{\"prefix\": \"osd pool set-quota\", \"pool\": \"" + - pool_name + "\", \"field\": \"max_bytes\", \"val\": \"" + + data_pool_name + "\", \"field\": \"max_bytes\", \"val\": \"" + std::to_string(quota) + "\"}"; char *cmd[1]; cmd[0] = (char *)cmdstr.c_str(); - ASSERT_EQ(0, rados_mon_command(_cluster, (const char **)cmd, 1, "", 0, - nullptr, 0, nullptr, 0)); + ASSERT_EQ(0, rados_mon_command(rados_ioctx_get_cluster(ioctx), + (const char **)cmd, 1, "", 0, nullptr, 0, + nullptr, 0)); rados_set_pool_full_try(ioctx); @@ -2168,8 +2158,47 @@ TEST_F(TestLibRBD, CreateThickRemoveFullTry) ASSERT_EQ(0, rbd_close(image)); // make sure we have latest map that marked the pool full - ASSERT_EQ(0, rados_wait_for_latest_osdmap(_cluster)); + ASSERT_EQ(0, rados_wait_for_latest_osdmap(rados_ioctx_get_cluster(ioctx))); ASSERT_EQ(0, rbd_remove(ioctx, image_name.c_str())); +} + +TEST_F(TestLibRBD, RemoveFullTry) +{ + REQUIRE(!is_librados_test_stub(_rados)); + + rados_ioctx_t ioctx; + auto pool_name = create_pool(true); + ASSERT_EQ(0, rados_ioctx_create(_cluster, pool_name.c_str(), &ioctx)); + // cancel out rbd_default_data_pool -- we need an image without + // a separate data pool + ASSERT_EQ(0, rbd_pool_metadata_set(ioctx, "conf_rbd_default_data_pool", + pool_name.c_str())); + + int order = 0; + auto image_name = get_temp_image_name(); + // FIXME: this is a workaround for rbd_trash object being created + // on the first remove -- pre-create it to avoid bumping into quota + ASSERT_EQ(0, create_image(ioctx, image_name.c_str(), 0, &order)); + ASSERT_EQ(0, rbd_remove(ioctx, image_name.c_str())); + remove_full_try(ioctx, image_name, pool_name); + + rados_ioctx_destroy(ioctx); +} + +TEST_F(TestLibRBD, RemoveFullTryDataPool) +{ + REQUIRE_FORMAT_V2(); + REQUIRE(!is_librados_test_stub(_rados)); + + rados_ioctx_t ioctx; + auto pool_name = create_pool(true); + auto data_pool_name = create_pool(true); + ASSERT_EQ(0, rados_ioctx_create(_cluster, pool_name.c_str(), &ioctx)); + ASSERT_EQ(0, rbd_pool_metadata_set(ioctx, "conf_rbd_default_data_pool", + data_pool_name.c_str())); + + auto image_name = get_temp_image_name(); + remove_full_try(ioctx, image_name, data_pool_name); rados_ioctx_destroy(ioctx); } From b387fdeeb3a56d7d2137bf32dab4ade083f6f096 Mon Sep 17 00:00:00 2001 From: Ilya Dryomov Date: Sun, 17 Oct 2021 13:09:14 +0200 Subject: [PATCH 0185/2866] test/librados_test_stub: add IoCtx::{set,get}_pool_full_try() stubs Because unittest_librbd links not just with librados_test_stub.a but also with librados.so (why?), these were sneakily picked up from the latter. This resulted in get_pool_full_try() reading from invalid memory and every once in a while returning true, prompting librbd to write to invalid memory through set_pool_full_try() in create_ioctx() and cause random crashes. Signed-off-by: Ilya Dryomov --- src/test/librados_test_stub/LibradosTestStub.cc | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/test/librados_test_stub/LibradosTestStub.cc b/src/test/librados_test_stub/LibradosTestStub.cc index b5ceb53eae92b..9e430057faabc 100644 --- a/src/test/librados_test_stub/LibradosTestStub.cc +++ b/src/test/librados_test_stub/LibradosTestStub.cc @@ -791,6 +791,13 @@ std::string IoCtx::get_namespace() const { return ctx->get_namespace(); } +void IoCtx::set_pool_full_try() { +} + +bool IoCtx::get_pool_full_try() { + return false; +} + static int save_operation_result(int result, int *pval) { if (pval != NULL) { *pval = result; From db366cab01035da1b531a8f873169943878f0409 Mon Sep 17 00:00:00 2001 From: Ilya Dryomov Date: Sun, 17 Oct 2021 14:38:22 +0200 Subject: [PATCH 0186/2866] test: fix valgrind.supp path in run-rbd-valgrind-unit-tests.sh Signed-off-by: Ilya Dryomov --- src/test/run-rbd-valgrind-unit-tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/run-rbd-valgrind-unit-tests.sh b/src/test/run-rbd-valgrind-unit-tests.sh index da2f00d0cceb1..46340392323dd 100755 --- a/src/test/run-rbd-valgrind-unit-tests.sh +++ b/src/test/run-rbd-valgrind-unit-tests.sh @@ -7,6 +7,6 @@ set -ex source $(dirname $0)/detect-build-env-vars.sh RBD_FEATURES=13 valgrind --tool=memcheck --leak-check=full --error-exitcode=1 \ - --suppressions=${CEPH_ROOT}/src/valgrind.supp unittest_librbd + --suppressions=${CEPH_ROOT}/qa/valgrind.supp unittest_librbd echo OK From abd19aa215eeef6da66c522c9fbeec870acee85a Mon Sep 17 00:00:00 2001 From: Patrick Donnelly Date: Fri, 15 Oct 2021 11:08:28 -0400 Subject: [PATCH 0187/2866] qa: reduce frag split confs for dir_split counter test Fixes: https://tracker.ceph.com/issues/52949 Fixes: a5675535ba532cfe782238d995751f9bc91f5078 Signed-off-by: Patrick Donnelly --- qa/suites/fs/libcephfs/tasks/libcephfs/+ | 0 qa/suites/fs/libcephfs/tasks/libcephfs/.qa | 1 + qa/suites/fs/libcephfs/tasks/libcephfs/frag.yaml | 1 + .../fs/libcephfs/tasks/{libcephfs.yaml => libcephfs/test.yaml} | 0 qa/suites/fs/traceless/overrides/frag.yaml | 1 + 5 files changed, 3 insertions(+) create mode 100644 qa/suites/fs/libcephfs/tasks/libcephfs/+ create mode 120000 qa/suites/fs/libcephfs/tasks/libcephfs/.qa create mode 120000 qa/suites/fs/libcephfs/tasks/libcephfs/frag.yaml rename qa/suites/fs/libcephfs/tasks/{libcephfs.yaml => libcephfs/test.yaml} (100%) create mode 120000 qa/suites/fs/traceless/overrides/frag.yaml diff --git a/qa/suites/fs/libcephfs/tasks/libcephfs/+ b/qa/suites/fs/libcephfs/tasks/libcephfs/+ new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/qa/suites/fs/libcephfs/tasks/libcephfs/.qa b/qa/suites/fs/libcephfs/tasks/libcephfs/.qa new file mode 120000 index 0000000000000..fea2489fdf6d9 --- /dev/null +++ b/qa/suites/fs/libcephfs/tasks/libcephfs/.qa @@ -0,0 +1 @@ +../.qa \ No newline at end of file diff --git a/qa/suites/fs/libcephfs/tasks/libcephfs/frag.yaml b/qa/suites/fs/libcephfs/tasks/libcephfs/frag.yaml new file mode 120000 index 0000000000000..5e5cdaed8c7ec --- /dev/null +++ b/qa/suites/fs/libcephfs/tasks/libcephfs/frag.yaml @@ -0,0 +1 @@ +.qa/cephfs/overrides/frag.yaml \ No newline at end of file diff --git a/qa/suites/fs/libcephfs/tasks/libcephfs.yaml b/qa/suites/fs/libcephfs/tasks/libcephfs/test.yaml similarity index 100% rename from qa/suites/fs/libcephfs/tasks/libcephfs.yaml rename to qa/suites/fs/libcephfs/tasks/libcephfs/test.yaml diff --git a/qa/suites/fs/traceless/overrides/frag.yaml b/qa/suites/fs/traceless/overrides/frag.yaml new file mode 120000 index 0000000000000..5e5cdaed8c7ec --- /dev/null +++ b/qa/suites/fs/traceless/overrides/frag.yaml @@ -0,0 +1 @@ +.qa/cephfs/overrides/frag.yaml \ No newline at end of file From a5523129590c4ecf900161f89521b07b73aef615 Mon Sep 17 00:00:00 2001 From: Dai Zhiwei Date: Mon, 18 Oct 2021 11:34:50 +0800 Subject: [PATCH 0188/2866] common: avoid pthread_mutex_unlock twice modified: src/common/cohort_lru.h Signed-off-by: Dai Zhiwei --- src/common/cohort_lru.h | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/common/cohort_lru.h b/src/common/cohort_lru.h index 2383fc95d7a3b..b105c80ccd537 100644 --- a/src/common/cohort_lru.h +++ b/src/common/cohort_lru.h @@ -139,15 +139,15 @@ namespace cohort { for (int ix = 0; ix < n_lanes; ++ix, lane_ix = next_evict_lane()) { Lane& lane = qlane[lane_ix]; - lane.lock.lock(); + std::unique_lock lane_lock{lane.lock}; /* if object at LRU has refcnt==1, it may be reclaimable */ Object* o = &(lane.q.back()); if (can_reclaim(o)) { ++(o->lru_refcnt); o->lru_flags |= FLAG_EVICTING; - lane.lock.unlock(); + lane_lock.unlock(); if (o->reclaim(newobj_fac)) { - lane.lock.lock(); + lane_lock.lock(); --(o->lru_refcnt); /* assertions that o state has not changed across * relock */ @@ -156,16 +156,13 @@ namespace cohort { Object::Queue::iterator it = Object::Queue::s_iterator_to(*o); lane.q.erase(it); - lane.lock.unlock(); return o; } else { - // XXX can't make unreachable (means what?) --(o->lru_refcnt); o->lru_flags &= ~FLAG_EVICTING; /* unlock in next block */ } } /* can_reclaim(o) */ - lane.lock.unlock(); } /* each lane */ return nullptr; } /* evict_block */ From 2916f2439eb2f62bc08c3e283b13391302b3e497 Mon Sep 17 00:00:00 2001 From: Kalpesh Pandya Date: Thu, 14 Oct 2021 01:29:06 +0530 Subject: [PATCH 0189/2866] src/rgw: Fix for malformed url This PR solves: https://tracker.ceph.com/issues/52738 It is solved by making changes to rgw_url.cc A test is also added to check it's working. Signed-off-by: Kalpesh Pandya --- src/rgw/rgw_url.cc | 2 +- src/test/rgw/test_rgw_url.cc | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/rgw/rgw_url.cc b/src/rgw/rgw_url.cc index 58f7b5490538c..7fd4788d77c6d 100644 --- a/src/rgw/rgw_url.cc +++ b/src/rgw/rgw_url.cc @@ -14,7 +14,7 @@ namespace { const std::string schema_re = "([[:alpha:]]+:\\/\\/)"; const std::string user_pass_re = "(([^:\\s]+):([^@\\s]+)@)?"; const std::string host_port_re = "([[:alnum:].:-]+)"; - const std::string path_re = "(/[[:print:]]+)?"; + const std::string path_re = "(/[[:print:]]*)?"; } bool parse_url_authority(const std::string& url, std::string& host, std::string& user, std::string& password) { diff --git a/src/test/rgw/test_rgw_url.cc b/src/test/rgw/test_rgw_url.cc index d38457b815d2e..b61e1d5959210 100644 --- a/src/test/rgw/test_rgw_url.cc +++ b/src/test/rgw/test_rgw_url.cc @@ -19,6 +19,18 @@ TEST(TestURL, SimpleAuthority) EXPECT_STREQ(host.c_str(), "example.com"); } +TEST(TestURL, SimpleAuthority_1) +{ + std::string host; + std::string user; + std::string password; + const std::string url = "http://example.com/"; + ASSERT_TRUE(parse_url_authority(url, host, user, password)); + ASSERT_TRUE(user.empty()); + ASSERT_TRUE(password.empty()); + EXPECT_STREQ(host.c_str(), "example.com"); +} + TEST(TestURL, IPAuthority) { std::string host; From 3f5dca49773bfe3b686c4c1449ed922bda640e92 Mon Sep 17 00:00:00 2001 From: Maya Gilad Date: Mon, 13 Sep 2021 20:11:39 +0300 Subject: [PATCH 0190/2866] doc added msgr on-wire-compression Signed-off-by: Maya Gilad --- doc/dev/msgr2.rst | 65 ++++++++++++++++++++++++++++++- doc/rados/configuration/msgr2.rst | 35 +++++++++++++++++ 2 files changed, 99 insertions(+), 1 deletion(-) diff --git a/doc/dev/msgr2.rst b/doc/dev/msgr2.rst index db8f3fa2e194c..af1421a3b878f 100644 --- a/doc/dev/msgr2.rst +++ b/doc/dev/msgr2.rst @@ -104,7 +104,8 @@ Each frame has a 32-byte preamble:: __le32 segment length __le16 segment alignment } * 4 - reserved (2 bytes) + __u8 flags + reserved (1 byte) __le32 preamble crc An empty frame has one empty segment. A non-empty frame can have @@ -114,6 +115,10 @@ empty. If there are less than four segments, unused (trailing) segment length and segment alignment fields are zeroed. +### Currently supported flags + + 1. FRAME_EARLY_DATA_COMPRESSED (see :ref:`msgr-post-compression`) + The reserved bytes are zeroed. The preamble checksum is CRC32-C. It covers everything up to @@ -528,6 +533,64 @@ where epilogue is:: late_status has the same meaning as in msgr2.1-crc mode. +Compression +----------- +Compression handshake is implemented using msgr2 feature-based handshaking. +In this phase, the client will indicate the server if on-wire-compression can be used for message transmitting, +in addition to the list of supported compression methods. If on-wire-compression is enabled for both client and server, +the server will choose a compression method based on client's request and its' own preferences. +Once the handshake is completed, both peers have setup their compression handlers (if desired). + +* TAG_COMPRESSION_REQUEST (client->server): declares compression capabilities and requirements:: + + bool is_compress + std::vector preferred_methods + + - if the client identifies that both peers support compression feature, it initiates the handshake. + - is_compress flag indicates whether the client's configuration is to use compression. + - preferred_methods is a list of compression algorithms that are supported by the client. + +* TAG_COMPRESSION_DONE (server->client) : determines on compression settings:: + + bool is_compress + uint32_t method + + - the server determines whether compression is possible according to its' configuration. + - if it is possible, it will pick its' most prioritizied compression method that is also supprorted by the client. + - if none exists, it will determine that session between the peers will be handled without compression. + +.. ditaa:: + + +---------+ +--------+ + | Client | | Server | + +---------+ +--------+ + | compression request | + |----------------------->| + |<-----------------------| + | compression done | + +# msgr2.x-secure mode + +Combining compression with encryption introduces security implications. +Compression will not be possible when using secure mode, unless configured specifically by an admin. + +.. _msgr-post-compression: + +Post-compression frame format +----------------------------- +Depending on the negotiated connection mode from TAG_COMPRESSION_DONE, the connection is able to acccept/send compressed frames or process all frames as decompressed. + +# msgr2.x-force mode + +All subsequent frames that will be sent via the connection will be compressed if compression requirements are met (e.g, the frames size). + +For compressed frames, the sending peer will enable the FRAME_EARLY_DATA_COMPRESSED flag, thus allowing the accepting peer to detect it and decompress the frame. + +# msgr2.x-none mode + +FRAME_EARLY_DATA_COMPRESSED flag will be disabled in preamble. + + Message flow handshake ---------------------- diff --git a/doc/rados/configuration/msgr2.rst b/doc/rados/configuration/msgr2.rst index 685422720cfe2..e1beaf5dc4cc3 100644 --- a/doc/rados/configuration/msgr2.rst +++ b/doc/rados/configuration/msgr2.rst @@ -146,6 +146,41 @@ secure) requirements on communication with the monitors. .. confval:: ms_mon_service_mode .. confval:: ms_mon_client_mode + +Compression modes +----------------- + +The v2 protocol supports two compression modes: + +* *force* mode provides: + + - In multi-availability zones deployment, compressing replication messages between OSDs saves latency. + - In the public cloud, inter-AZ communications are expensive. Thus, minimizing message + size reduces network costs to cloud provider. + - When using instance storage on AWS (probably other public clouds as well) the instances with NVMe + provide low network bandwidth relative to the device bandwidth. + In this case, NW compression can improve the overall performance since this is clearly + the bottleneck. + +* *none* mode provides: + + - messages are transmitted without compression. + + +Compression mode configuration options +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +For all connections, there is an option that controls compression usage in secure mode + +.. confval:: ms_compress_secure + +There is a parallel set of options that apply specifically to OSDs, +allowing administrators to set different requirements on communication between OSDs. + +.. confval:: ms_osd_compress_mode +.. confval:: ms_osd_compress_min_size +.. confval:: ms_osd_compression_algorithm + Transitioning from v1-only to v2-plus-v1 ---------------------------------------- From 8a35516294f7b9284f05e33948f2ac47760b57c1 Mon Sep 17 00:00:00 2001 From: Deepika Upadhyay Date: Wed, 13 Oct 2021 11:33:39 +0530 Subject: [PATCH 0191/2866] doc: add iSCSI initiator client configuration section for cephadm Signed-off-by: Deepika Upadhyay --- doc/cephadm/services/iscsi.rst | 9 ++++++++- doc/rbd/iscsi-initiator-linux.rst | 6 ++++++ doc/rbd/iscsi-initiators.rst | 2 ++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/doc/cephadm/services/iscsi.rst b/doc/cephadm/services/iscsi.rst index e039e8d9a3467..bb2de79215053 100644 --- a/doc/cephadm/services/iscsi.rst +++ b/doc/cephadm/services/iscsi.rst @@ -74,7 +74,14 @@ See :ref:`orchestrator-cli-placement-spec` for details of the placement specific See also: :ref:`orchestrator-cli-service-spec`. +Configuring iSCSI client +======================== + +The containerized iscsi service can be used from any host by +:ref:`configuring-the-iscsi-initiators`, which will use TCP/IP to send SCSI +commands to the iSCSI target (gateway). + Further Reading =============== -* RBD: :ref:`ceph-iscsi` +* Ceph iSCSI Overview: :ref:`ceph-iscsi` diff --git a/doc/rbd/iscsi-initiator-linux.rst b/doc/rbd/iscsi-initiator-linux.rst index 058cc4ed661df..e15e81bb98554 100644 --- a/doc/rbd/iscsi-initiator-linux.rst +++ b/doc/rbd/iscsi-initiator-linux.rst @@ -90,3 +90,9 @@ group for each path. You should now be able to use the RBD image like you would a normal multipath’d iSCSI disk. + +4. Logout from target: + + :: + + # iscsiadm -m node -T iqn.2003-01.org.linux-iscsi.rheln1 -u diff --git a/doc/rbd/iscsi-initiators.rst b/doc/rbd/iscsi-initiators.rst index 67447789d2b8b..b2433895211e3 100644 --- a/doc/rbd/iscsi-initiators.rst +++ b/doc/rbd/iscsi-initiators.rst @@ -1,3 +1,5 @@ +.. _configuring-the-iscsi-initiators: + -------------------------------- Configuring the iSCSI Initiators -------------------------------- From a7952949a874ca886c8ebacaba9b1d34a69e5bae Mon Sep 17 00:00:00 2001 From: Deepika Upadhyay Date: Wed, 13 Oct 2021 11:47:11 +0530 Subject: [PATCH 0192/2866] qa/suites/rbd: remove baremetal based setup needed for iscsi testing * replace ceph baremetal deployment with cephadm based deployment Signed-off-by: Deepika Upadhyay --- qa/suites/rbd/iscsi/base/install.yaml | 23 +++++++------------ qa/suites/rbd/iscsi/cluster/fixed-3.yaml | 21 ++++++++++++++--- qa/suites/rbd/iscsi/pool/.qa | 1 - qa/suites/rbd/iscsi/pool/datapool.yaml | 12 ---------- .../{ceph_iscsi.yaml => cephadm_iscsi.yaml} | 17 +++++++------- 5 files changed, 34 insertions(+), 40 deletions(-) delete mode 120000 qa/suites/rbd/iscsi/pool/.qa delete mode 100644 qa/suites/rbd/iscsi/pool/datapool.yaml rename qa/suites/rbd/iscsi/workloads/{ceph_iscsi.yaml => cephadm_iscsi.yaml} (67%) diff --git a/qa/suites/rbd/iscsi/base/install.yaml b/qa/suites/rbd/iscsi/base/install.yaml index e35affe7e24f4..5c5a6c31f60b4 100644 --- a/qa/suites/rbd/iscsi/base/install.yaml +++ b/qa/suites/rbd/iscsi/base/install.yaml @@ -1,21 +1,14 @@ use_shaman: True tasks: +- cephadm: +- cephadm.shell: + host.a: + - ceph orch status + - ceph orch ps + - ceph orch ls + - ceph orch host ls + - ceph orch device ls - install: extra_packages: - iscsi-initiator-utils - device-mapper-multipath -- ceph: -- install: - packages: - rpm: - - tcmu-runner - project: tcmu-runner - sha1: latest -- install: - packages: - rpm: - - ceph-iscsi - shaman: - force_noarch: True - project: ceph-iscsi - sha1: latest diff --git a/qa/suites/rbd/iscsi/cluster/fixed-3.yaml b/qa/suites/rbd/iscsi/cluster/fixed-3.yaml index bf1eb6018b40a..426ea32cf81f8 100644 --- a/qa/suites/rbd/iscsi/cluster/fixed-3.yaml +++ b/qa/suites/rbd/iscsi/cluster/fixed-3.yaml @@ -1,4 +1,19 @@ roles: -- [mon.a, osd.0, osd.1, osd.2, a_gateway.0] -- [mon.b, osd.3, osd.4, osd.5, b_client.0] -- [mon.c, mgr.x, osd.6, osd.7, c_gateway.1] +- - host.a + - mon.a + - mgr.x + - osd.0 + - osd.1 + - client.0 + - ceph.iscsi.iscsi.a +- - mon.b + - osd.2 + - osd.3 + - osd.4 + - client.1 +- - mon.c + - osd.5 + - osd.6 + - osd.7 + - client.2 + - ceph.iscsi.iscsi.b diff --git a/qa/suites/rbd/iscsi/pool/.qa b/qa/suites/rbd/iscsi/pool/.qa deleted file mode 120000 index a602a0353e751..0000000000000 --- a/qa/suites/rbd/iscsi/pool/.qa +++ /dev/null @@ -1 +0,0 @@ -../.qa/ \ No newline at end of file diff --git a/qa/suites/rbd/iscsi/pool/datapool.yaml b/qa/suites/rbd/iscsi/pool/datapool.yaml deleted file mode 100644 index 45b00378b83c2..0000000000000 --- a/qa/suites/rbd/iscsi/pool/datapool.yaml +++ /dev/null @@ -1,12 +0,0 @@ -overrides: - ceph: - log-ignorelist: - - overall HEALTH_ - - \(CACHE_POOL_NEAR_FULL\) - - \(CACHE_POOL_NO_HIT_SET\) -tasks: -- exec: - a_gateway.0: - - sudo ceph osd pool create rbd 4 - - sudo ceph osd pool create datapool 4 - - sudo ceph osd pool application enable datapool rbd diff --git a/qa/suites/rbd/iscsi/workloads/ceph_iscsi.yaml b/qa/suites/rbd/iscsi/workloads/cephadm_iscsi.yaml similarity index 67% rename from qa/suites/rbd/iscsi/workloads/ceph_iscsi.yaml rename to qa/suites/rbd/iscsi/workloads/cephadm_iscsi.yaml index f91a35eafd22e..202e6d8b5ec84 100644 --- a/qa/suites/rbd/iscsi/workloads/ceph_iscsi.yaml +++ b/qa/suites/rbd/iscsi/workloads/cephadm_iscsi.yaml @@ -1,22 +1,21 @@ tasks: -- ceph_iscsi: - gateways: [a_gateway.0, c_gateway.1] - clients: [b_client.0] +- ceph_iscsi_client: + clients: [client.1] - cram: parallel: False clients: - a_gateway.0: + client.0: - src/test/cli-integration/rbd/gwcli_create.t - b_client.0: + client.1: - src/test/cli-integration/rbd/iscsi_client.t - c_gateway.1: + client.2: - src/test/cli-integration/rbd/gwcli_delete.t - cram: parallel: False clients: - a_gateway.0: + client.0: - src/test/cli-integration/rbd/rest_api_create.t - b_client.0: + client.1: - src/test/cli-integration/rbd/iscsi_client.t - c_gateway.1: + client.2: - src/test/cli-integration/rbd/rest_api_delete.t From cbd2c71398bca7b7c21d1c9307d0abf5db6acb3c Mon Sep 17 00:00:00 2001 From: Deepika Upadhyay Date: Wed, 13 Oct 2021 11:49:02 +0530 Subject: [PATCH 0193/2866] qa/tasks: adapt ceph_iscsi.py task to ceph_iscsi_client * we use setup_iscsi_client.py to deploy iscsi client services, configuring intiator and mulitpath this is done by qa task ceph_iscsi_client * qa/cephadm: adds remotes ip addresses to iscsi gateway, * rename poolname: iscsi >> datapool, which we usually use for tests and expresses type of pool more clearly. Signed-off-by: Deepika Upadhyay --- qa/tasks/ceph_iscsi.py | 141 ---------------------------------- qa/tasks/ceph_iscsi_client.py | 56 ++++++++++++++ qa/tasks/cephadm.py | 39 ++++++++-- 3 files changed, 88 insertions(+), 148 deletions(-) delete mode 100644 qa/tasks/ceph_iscsi.py create mode 100644 qa/tasks/ceph_iscsi_client.py diff --git a/qa/tasks/ceph_iscsi.py b/qa/tasks/ceph_iscsi.py deleted file mode 100644 index a8870743a09f1..0000000000000 --- a/qa/tasks/ceph_iscsi.py +++ /dev/null @@ -1,141 +0,0 @@ -""" -Run ceph-iscsi cluster setup -""" -import logging -import contextlib -from io import StringIO -from teuthology.exceptions import CommandFailedError, ConnectionLostError -from teuthology.orchestra import run -from textwrap import dedent - -log = logging.getLogger(__name__) - -class IscsiSetup(object): - def __init__(self, ctx, config): - self.ctx = ctx - self.config = config - self.target_iqn = "iqn.2003-01.com.redhat.iscsi-gw:ceph-gw" - self.client_iqn = "iqn.1994-05.com.redhat:client" - self.trusted_ip_list = [] - self.background_procs = [] - - def run_daemon(self, remote, cmds): - p = remote.run(args=['sudo', 'adjust-ulimits', 'daemon-helper', 'kill', cmds], - wait=False, stdin=run.PIPE, stdout=StringIO()) - self.background_procs.append(p) - - def _kill_background(self, p): - if p.stdin: - p.stdin.close() - try: - p.wait() - except (CommandFailedError, ConnectionLostError): - pass - - def kill_backgrounds(self): - for p in self.background_procs: - self._kill_background(p) - self.background_procs = [] - - def _setup_iscsi_gateway_cfg(self, role): - # setup the iscsi-gateway.cfg file, we only set the - # clust_name and trusted_ip_list and all the others - # as default - ips = ','.join(self.trusted_ip_list) - conf = dedent(f'''\ -[config] -cluster_name = ceph -pool = rbd -api_secure = false -api_port = 5000 -trusted_ip_list = {ips} -''') - path = "/etc/ceph/iscsi-gateway.cfg" - (remote,) = (self.ctx.cluster.only(role).remotes.keys()) - remote.sudo_write_file(path, conf) - - def _setup_gateway(self, role): - """Spawned task that setups the gateway""" - (remote,) = (self.ctx.cluster.only(role).remotes.keys()) - - self._setup_iscsi_gateway_cfg(role) - - self.run_daemon(remote, "/usr/bin/tcmu-runner") - self.run_daemon(remote, "/usr/bin/rbd-target-gw") - self.run_daemon(remote, "/usr/bin/rbd-target-api") - - def setup_gateways(self): - for role in self.config['gateways']: - (remote,) = (self.ctx.cluster.only(role).remotes.keys()) - self.trusted_ip_list.append(remote.ip_address) - - for role in self.config['gateways']: - self._setup_gateway(role) - - def _setup_client(self, role): - """Spawned task that setups the gateway""" - (remote,) = (self.ctx.cluster.only(role).remotes.keys()) - - # copy the "iscsi-gateway.cfg" to client and will be - # used to get the IPs - self._setup_iscsi_gateway_cfg(role) - - conf = dedent(f''' -InitiatorName={self.client_iqn} -''') - path = "/etc/iscsi/initiatorname.iscsi" - remote.sudo_write_file(path, conf, mkdir=True) - - # the restart is needed after the above change is applied - remote.run(args=['sudo', 'systemctl', 'restart', 'iscsid']) - - remote.run(args=['sudo', 'modprobe', 'dm_multipath']) - remote.run(args=['sudo', 'mpathconf', '--enable']) - conf = dedent('''\ -devices { - device { - vendor "LIO-ORG" - product "LIO-ORG" - hardware_handler "1 alua" - path_grouping_policy "failover" - path_selector "queue-length 0" - failback 60 - path_checker tur - prio alua - prio_args exclusive_pref_bit - fast_io_fail_tmo 25 - no_path_retry queue - } -} -''') - path = "/etc/multipath.conf" - remote.sudo_write_file(path, conf, append=True) - remote.run(args=['sudo', 'systemctl', 'start', 'multipathd']) - - def setup_clients(self): - for role in self.config['clients']: - self._setup_client(role) - -@contextlib.contextmanager -def task(ctx, config): - """ - Run ceph iscsi setup. - - Specify the list of gateways to run :: - - tasks: - ceph_iscsi: - gateways: [a_gateway.0, c_gateway.1] - clients: [b_client.0] - - """ - log.info('Setting ceph iscsi cluster...') - iscsi = IscsiSetup(ctx, config) - iscsi.setup_gateways() - iscsi.setup_clients() - - try: - yield - finally: - log.info('Ending ceph iscsi daemons') - iscsi.kill_backgrounds() diff --git a/qa/tasks/ceph_iscsi_client.py b/qa/tasks/ceph_iscsi_client.py new file mode 100644 index 0000000000000..189b7fa31fef8 --- /dev/null +++ b/qa/tasks/ceph_iscsi_client.py @@ -0,0 +1,56 @@ +""" +Set up ceph-iscsi client. +""" +import logging +import contextlib +from textwrap import dedent + +log = logging.getLogger(__name__) + + +@contextlib.contextmanager +def task(ctx, config): + """ + Set up ceph-iscsi client. + + tasks: + ceph_iscsi_client: + clients: [client.1] + """ + log.info('Setting up ceph-iscsi client...') + for role in config['clients']: + (remote,) = (ctx.cluster.only(role).remotes.keys()) + + conf = dedent(''' + InitiatorName=iqn.1994-05.com.redhat:client + ''') + path = "/etc/iscsi/initiatorname.iscsi" + remote.sudo_write_file(path, conf, mkdir=True) + + # the restart is needed after the above change is applied + remote.run(args=['sudo', 'systemctl', 'restart', 'iscsid']) + + remote.run(args=['sudo', 'modprobe', 'dm_multipath']) + remote.run(args=['sudo', 'mpathconf', '--enable']) + conf = dedent(''' + devices { + device { + vendor "LIO-ORG" + product "TCMU device" + hardware_handler "1 alua" + path_grouping_policy "failover" + path_selector "queue-length 0" + failback 60 + path_checker tur + prio alua + prio_args exclusive_pref_bit + fast_io_fail_tmo 25 + no_path_retry queue + } + } + ''') + path = "/etc/multipath.conf" + remote.sudo_write_file(path, conf, append=True) + remote.run(args=['sudo', 'systemctl', 'start', 'multipathd']) + + yield diff --git a/qa/tasks/cephadm.py b/qa/tasks/cephadm.py index 8bd7f220c82f1..b72e8b0e2a674 100644 --- a/qa/tasks/cephadm.py +++ b/qa/tasks/cephadm.py @@ -19,6 +19,7 @@ from teuthology.orchestra import run from teuthology.orchestra.daemon import DaemonGroup from teuthology.config import config as teuth_config +from textwrap import dedent # these items we use from ceph.py should probably eventually move elsewhere from tasks.ceph import get_mons, healthy @@ -67,6 +68,18 @@ def build_initial_config(ctx, config): return conf +def distribute_iscsi_gateway_cfg(ctx, conf_data): + """ + Distribute common gateway config to get the IPs. + These will help in iscsi clients with finding trusted_ip_list. + """ + log.info('Distributing iscsi-gateway.cfg...') + for remote, roles in ctx.cluster.remotes.items(): + remote.write_file( + path='/etc/ceph/iscsi-gateway.cfg', + data=conf_data, + sudo=True) + def update_archive_setting(ctx, key, value): """ Add logs directory to job's info log file @@ -905,32 +918,44 @@ def ceph_iscsi(ctx, config): nodes = [] daemons = {} + ips = [] + for remote, roles in ctx.cluster.remotes.items(): for role in [r for r in roles - if teuthology.is_type('iscsi', cluster_name)(r)]: + if teuthology.is_type('iscsi', cluster_name)(r)]: c_, _, id_ = teuthology.split_role(role) log.info('Adding %s on %s' % (role, remote.shortname)) nodes.append(remote.shortname + '=' + id_) daemons[role] = (remote, id_) + ips.append(remote.ip_address) + trusted_ip_list = ','.join(ips) if nodes: - poolname = 'iscsi' - # ceph osd pool create iscsi 3 3 replicated + poolname = 'datapool' + # ceph osd pool create datapool 3 3 replicated _shell(ctx, cluster_name, remote, [ 'ceph', 'osd', 'pool', 'create', poolname, '3', '3', 'replicated'] ) _shell(ctx, cluster_name, remote, [ - 'ceph', 'osd', 'pool', 'application', 'enable', - poolname, 'rbd'] + 'rbd', 'pool', 'init', poolname] ) - # ceph orch apply iscsi iscsi user password + # ceph orch apply iscsi datapool (admin)user (admin)password _shell(ctx, cluster_name, remote, [ 'ceph', 'orch', 'apply', 'iscsi', - poolname, 'user', 'password', + poolname, 'admin', 'admin', + '--trusted_ip_list', trusted_ip_list, '--placement', str(len(nodes)) + ';' + ';'.join(nodes)] ) + + # used by iscsi client to identify valid gateway ip's + conf_data = dedent(f""" + [config] + trusted_ip_list = {trusted_ip_list} + """) + distribute_iscsi_gateway_cfg(ctx, conf_data) + for role, i in daemons.items(): remote, id_ = i ctx.daemons.register_daemon( From 3e9b8c8f8bc12287e93146e3d6a6bca37cb5ac07 Mon Sep 17 00:00:00 2001 From: Deepika Upadhyay Date: Wed, 13 Oct 2021 12:00:38 +0530 Subject: [PATCH 0194/2866] src/test/cli-integration/rbd: update gwcli_* to use podman exec * since iscsi now resides inside the container, for testing we use the hack: - find the iscsi container/ - use found container and podman exec command to test gateway creation deletion from host itself Signed-off-by: Deepika Upadhyay --- src/test/cli-integration/rbd/gwcli_create.t | 56 +++++++++++---------- src/test/cli-integration/rbd/gwcli_delete.t | 22 ++++---- 2 files changed, 43 insertions(+), 35 deletions(-) diff --git a/src/test/cli-integration/rbd/gwcli_create.t b/src/test/cli-integration/rbd/gwcli_create.t index e23ed895e7b29..63d3b305244e0 100644 --- a/src/test/cli-integration/rbd/gwcli_create.t +++ b/src/test/cli-integration/rbd/gwcli_create.t @@ -1,74 +1,78 @@ +Podman find iSCSI container +=========================== + $ ISCSI_CONTAINER=$(sudo podman ps -a | grep -F 'iscsi' | grep -Fv 'tcmu' | awk '{print $1}') + Dismiss the "could not load preferences file .gwcli/prefs.bin" warning ====================================================================== - $ sudo gwcli ls >/dev/null 2>&1 + $ sudo podman exec $ISCSI_CONTAINER gwcli ls >/dev/null 2>&1 Create a datapool/block0 disk ============================= - $ sudo gwcli disks/ create pool=datapool image=block0 size=300M wwn=36001405da17b74481464e9fa968746d3 - $ sudo gwcli ls disks/ | grep 'o- disks' | awk -F'[' '{print $2}' + $ sudo podman exec $ISCSI_CONTAINER gwcli disks/ create pool=datapool image=block0 size=300M wwn=36001405da17b74481464e9fa968746d3 + $ sudo podman exec $ISCSI_CONTAINER gwcli ls disks/ | grep 'o- disks' | awk -F'[' '{print $2}' 300M, Disks: 1] - $ sudo gwcli ls disks/ | grep 'o- datapool' | awk -F'[' '{print $2}' + $ sudo podman exec $ISCSI_CONTAINER gwcli ls disks/ | grep 'o- datapool' | awk -F'[' '{print $2}' datapool (300M)] - $ sudo gwcli ls disks/ | grep 'o- block0' | awk -F'[' '{print $2}' + $ sudo podman exec $ISCSI_CONTAINER gwcli ls disks/ | grep 'o- block0' | awk -F'[' '{print $2}' datapool/block0 (Unknown, 300M)] Create the target IQN ===================== - $ sudo gwcli iscsi-targets/ create target_iqn=iqn.2003-01.com.redhat.iscsi-gw:ceph-gw - $ sudo gwcli ls iscsi-targets/ | grep 'o- iscsi-targets' | awk -F'[' '{print $2}' + $ sudo podman exec $ISCSI_CONTAINER gwcli iscsi-targets/ create target_iqn=iqn.2003-01.com.redhat.iscsi-gw:ceph-gw + $ sudo podman exec $ISCSI_CONTAINER gwcli ls iscsi-targets/ | grep 'o- iscsi-targets' | awk -F'[' '{print $2}' DiscoveryAuth: None, Targets: 1] - $ sudo gwcli ls iscsi-targets/ | grep 'o- iqn.2003-01.com.redhat.iscsi-gw:ceph-gw' | awk -F'[' '{print $2}' + $ sudo podman exec $ISCSI_CONTAINER gwcli ls iscsi-targets/ | grep 'o- iqn.2003-01.com.redhat.iscsi-gw:ceph-gw' | awk -F'[' '{print $2}' Auth: None, Gateways: 0] - $ sudo gwcli ls iscsi-targets/ | grep 'o- disks' | awk -F'[' '{print $2}' + $ sudo podman exec $ISCSI_CONTAINER gwcli ls iscsi-targets/ | grep 'o- disks' | awk -F'[' '{print $2}' Disks: 0] - $ sudo gwcli ls iscsi-targets/ | grep 'o- gateways' | awk -F'[' '{print $2}' + $ sudo podman exec $ISCSI_CONTAINER gwcli ls iscsi-targets/ | grep 'o- gateways' | awk -F'[' '{print $2}' Up: 0/0, Portals: 0] - $ sudo gwcli ls iscsi-targets/ | grep 'o- host-groups' | awk -F'[' '{print $2}' + $ sudo podman exec $ISCSI_CONTAINER gwcli ls iscsi-targets/ | grep 'o- host-groups' | awk -F'[' '{print $2}' Groups : 0] - $ sudo gwcli ls iscsi-targets/ | grep 'o- hosts' | awk -F'[' '{print $2}' + $ sudo podman exec $ISCSI_CONTAINER gwcli ls iscsi-targets/ | grep 'o- hosts' | awk -F'[' '{print $2}' Auth: ACL_ENABLED, Hosts: 0] Create the first gateway ======================== $ HOST=`python3 -c "import socket; print(socket.getfqdn())"` > IP=`hostname -i | awk '{print $1}'` - > sudo gwcli iscsi-targets/iqn.2003-01.com.redhat.iscsi-gw:ceph-gw/gateways create ip_addresses=$IP gateway_name=$HOST - $ sudo gwcli ls iscsi-targets/ | grep 'o- gateways' | awk -F'[' '{print $2}' + > sudo podman exec $ISCSI_CONTAINER gwcli iscsi-targets/iqn.2003-01.com.redhat.iscsi-gw:ceph-gw/gateways create ip_addresses=$IP gateway_name=$HOST + $ sudo podman exec $ISCSI_CONTAINER gwcli ls iscsi-targets/ | grep 'o- gateways' | awk -F'[' '{print $2}' Up: 1/1, Portals: 1] Create the second gateway ======================== $ IP=`cat /etc/ceph/iscsi-gateway.cfg |grep 'trusted_ip_list' | awk -F'[, ]' '{print $3}'` > if [ "$IP" != `hostname -i | awk '{print $1}'` ]; then - > HOST=`python3 -c "import socket; print(socket.getfqdn('$IP'))"` - > sudo gwcli iscsi-targets/iqn.2003-01.com.redhat.iscsi-gw:ceph-gw/gateways create ip_addresses=$IP gateway_name=$HOST + > HOST=$(python3 -c "import socket; print(socket.getfqdn('$IP'))") + > sudo podman exec $ISCSI_CONTAINER gwcli iscsi-targets/iqn.2003-01.com.redhat.iscsi-gw:ceph-gw/gateways create ip_addresses=$IP gateway_name=$HOST > fi $ IP=`cat /etc/ceph/iscsi-gateway.cfg |grep 'trusted_ip_list' | awk -F'[, ]' '{print $4}'` > if [ "$IP" != `hostname -i | awk '{print $1}'` ]; then > HOST=`python3 -c "import socket; print(socket.getfqdn('$IP'))"` - > sudo gwcli iscsi-targets/iqn.2003-01.com.redhat.iscsi-gw:ceph-gw/gateways create ip_addresses=$IP gateway_name=$HOST + > sudo podman exec $ISCSI_CONTAINER gwcli iscsi-targets/iqn.2003-01.com.redhat.iscsi-gw:ceph-gw/gateways create ip_addresses=$IP gateway_name=$HOST > fi - $ sudo gwcli ls iscsi-targets/ | grep 'o- gateways' | awk -F'[' '{print $2}' + $ sudo podman exec $ISCSI_CONTAINER gwcli ls iscsi-targets/ | grep 'o- gateways' | awk -F'[' '{print $2}' Up: 2/2, Portals: 2] Attach the disk =============== - $ sudo gwcli iscsi-targets/iqn.2003-01.com.redhat.iscsi-gw:ceph-gw/disks/ add disk=datapool/block0 - $ sudo gwcli ls iscsi-targets/ | grep 'o- disks' | awk -F'[' '{print $2}' + $ sudo podman exec $ISCSI_CONTAINER gwcli iscsi-targets/iqn.2003-01.com.redhat.iscsi-gw:ceph-gw/disks/ add disk=datapool/block0 + $ sudo podman exec $ISCSI_CONTAINER gwcli ls iscsi-targets/ | grep 'o- disks' | awk -F'[' '{print $2}' Disks: 1] Create a host ============= - $ sudo gwcli iscsi-targets/iqn.2003-01.com.redhat.iscsi-gw:ceph-gw/hosts create client_iqn=iqn.1994-05.com.redhat:client - $ sudo gwcli ls iscsi-targets/ | grep 'o- hosts' | awk -F'[' '{print $2}' + $ sudo podman exec $ISCSI_CONTAINER gwcli iscsi-targets/iqn.2003-01.com.redhat.iscsi-gw:ceph-gw/hosts create client_iqn=iqn.1994-05.com.redhat:client + $ sudo podman exec $ISCSI_CONTAINER gwcli ls iscsi-targets/ | grep 'o- hosts' | awk -F'[' '{print $2}' Auth: ACL_ENABLED, Hosts: 1] - $ sudo gwcli ls iscsi-targets/ | grep 'o- iqn.1994-05.com.redhat:client' | awk -F'[' '{print $2}' + $ sudo podman exec $ISCSI_CONTAINER gwcli ls iscsi-targets/ | grep 'o- iqn.1994-05.com.redhat:client' | awk -F'[' '{print $2}' Auth: None, Disks: 0(0.00Y)] Map the LUN =========== - $ sudo gwcli iscsi-targets/iqn.2003-01.com.redhat.iscsi-gw:ceph-gw/hosts/iqn.1994-05.com.redhat:client disk disk=datapool/block0 - $ sudo gwcli ls iscsi-targets/ | grep 'o- hosts' | awk -F'[' '{print $2}' + $ sudo podman exec $ISCSI_CONTAINER gwcli iscsi-targets/iqn.2003-01.com.redhat.iscsi-gw:ceph-gw/hosts/iqn.1994-05.com.redhat:client disk disk=datapool/block0 + $ sudo podman exec $ISCSI_CONTAINER gwcli ls iscsi-targets/ | grep 'o- hosts' | awk -F'[' '{print $2}' Auth: ACL_ENABLED, Hosts: 1] - $ sudo gwcli ls iscsi-targets/ | grep 'o- iqn.1994-05.com.redhat:client' | awk -F'[' '{print $2}' + $ sudo podman exec $ISCSI_CONTAINER gwcli ls iscsi-targets/ | grep 'o- iqn.1994-05.com.redhat:client' | awk -F'[' '{print $2}' Auth: None, Disks: 1(300M)] diff --git a/src/test/cli-integration/rbd/gwcli_delete.t b/src/test/cli-integration/rbd/gwcli_delete.t index 5aef9d0e960ec..e973d87a39f0b 100644 --- a/src/test/cli-integration/rbd/gwcli_delete.t +++ b/src/test/cli-integration/rbd/gwcli_delete.t @@ -1,27 +1,31 @@ +Podman find iSCSI container +=========================== + $ ISCSI_CONTAINER=$(sudo podman ps -a | grep -F 'iscsi' | grep -Fv 'tcmu' | awk '{print $1}') + Dismiss the "could not load preferences file .gwcli/prefs.bin" warning ====================================================================== - $ sudo gwcli ls >/dev/null 2>&1 + $ sudo podman exec $ISCSI_CONTAINER gwcli ls >/dev/null 2>&1 Delete the host =============== - $ sudo gwcli iscsi-targets/iqn.2003-01.com.redhat.iscsi-gw:ceph-gw/hosts delete client_iqn=iqn.1994-05.com.redhat:client - $ sudo gwcli ls iscsi-targets/ | grep 'o- hosts' | awk -F'[' '{print $2}' + $ sudo podman exec $ISCSI_CONTAINER gwcli iscsi-targets/iqn.2003-01.com.redhat.iscsi-gw:ceph-gw/hosts delete client_iqn=iqn.1994-05.com.redhat:client + $ sudo podman exec $ISCSI_CONTAINER gwcli ls iscsi-targets/ | grep 'o- hosts' | awk -F'[' '{print $2}' Auth: ACL_ENABLED, Hosts: 0] Delete the iscsi-targets disk ============================= - $ sudo gwcli iscsi-targets/iqn.2003-01.com.redhat.iscsi-gw:ceph-gw/disks/ delete disk=datapool/block0 - $ sudo gwcli ls iscsi-targets/ | grep 'o- disks' | awk -F'[' '{print $2}' + $ sudo podman exec $ISCSI_CONTAINER gwcli iscsi-targets/iqn.2003-01.com.redhat.iscsi-gw:ceph-gw/disks/ delete disk=datapool/block0 + $ sudo podman exec $ISCSI_CONTAINER gwcli ls iscsi-targets/ | grep 'o- disks' | awk -F'[' '{print $2}' Disks: 0] Delete the target IQN ===================== - $ sudo gwcli iscsi-targets/ delete target_iqn=iqn.2003-01.com.redhat.iscsi-gw:ceph-gw - $ sudo gwcli ls iscsi-targets/ | grep 'o- iscsi-targets' | awk -F'[' '{print $2}' + $ sudo podman exec $ISCSI_CONTAINER gwcli iscsi-targets/ delete target_iqn=iqn.2003-01.com.redhat.iscsi-gw:ceph-gw + $ sudo podman exec $ISCSI_CONTAINER gwcli ls iscsi-targets/ | grep 'o- iscsi-targets' | awk -F'[' '{print $2}' DiscoveryAuth: None, Targets: 0] Delete the disks ================ - $ sudo gwcli disks/ delete image_id=datapool/block0 - $ sudo gwcli ls disks/ | grep 'o- disks' | awk -F'[' '{print $2}' + $ sudo podman exec $ISCSI_CONTAINER gwcli disks/ delete image_id=datapool/block0 + $ sudo podman exec $ISCSI_CONTAINER gwcli ls disks/ | grep 'o- disks' | awk -F'[' '{print $2}' 0.00Y, Disks: 0] From b09c6ca2cc639d702283d17e49366e191843dafc Mon Sep 17 00:00:00 2001 From: Deepika Upadhyay Date: Thu, 14 Oct 2021 12:40:14 +0530 Subject: [PATCH 0195/2866] src/test/cli-integration: correct hostname for iscsi gateway iscsi accepts first word of hostname as local hostname[0] instead of smithi061.front.sepia.ceph.com we use: smithi061 otherwise gateway creation will error out with: ```The first gateway defined must be the local machine``` [0] https://github.com/ceph/ceph-iscsi-cli/blob/ff3e430b70a62257a560b8988bd54bf845c01fb7/gwcli/utils.py#L36 Signed-off-by: Deepika Upadhyay --- src/test/cli-integration/rbd/gwcli_create.t | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/test/cli-integration/rbd/gwcli_create.t b/src/test/cli-integration/rbd/gwcli_create.t index 63d3b305244e0..67fdceb8855c7 100644 --- a/src/test/cli-integration/rbd/gwcli_create.t +++ b/src/test/cli-integration/rbd/gwcli_create.t @@ -34,7 +34,7 @@ Create the target IQN Create the first gateway ======================== - $ HOST=`python3 -c "import socket; print(socket.getfqdn())"` + $ HOST=$(python3 -c "import socket; print(socket.getfqdn())" | cut -d. -f1) > IP=`hostname -i | awk '{print $1}'` > sudo podman exec $ISCSI_CONTAINER gwcli iscsi-targets/iqn.2003-01.com.redhat.iscsi-gw:ceph-gw/gateways create ip_addresses=$IP gateway_name=$HOST $ sudo podman exec $ISCSI_CONTAINER gwcli ls iscsi-targets/ | grep 'o- gateways' | awk -F'[' '{print $2}' @@ -44,12 +44,12 @@ Create the second gateway ======================== $ IP=`cat /etc/ceph/iscsi-gateway.cfg |grep 'trusted_ip_list' | awk -F'[, ]' '{print $3}'` > if [ "$IP" != `hostname -i | awk '{print $1}'` ]; then - > HOST=$(python3 -c "import socket; print(socket.getfqdn('$IP'))") + > HOST=$(python3 -c "import socket; print(socket.getfqdn('$IP'))" | cut -d. -f1) > sudo podman exec $ISCSI_CONTAINER gwcli iscsi-targets/iqn.2003-01.com.redhat.iscsi-gw:ceph-gw/gateways create ip_addresses=$IP gateway_name=$HOST > fi $ IP=`cat /etc/ceph/iscsi-gateway.cfg |grep 'trusted_ip_list' | awk -F'[, ]' '{print $4}'` > if [ "$IP" != `hostname -i | awk '{print $1}'` ]; then - > HOST=`python3 -c "import socket; print(socket.getfqdn('$IP'))"` + > HOST=$(python3 -c "import socket; print(socket.getfqdn('$IP'))" | cut -d. -f1) > sudo podman exec $ISCSI_CONTAINER gwcli iscsi-targets/iqn.2003-01.com.redhat.iscsi-gw:ceph-gw/gateways create ip_addresses=$IP gateway_name=$HOST > fi $ sudo podman exec $ISCSI_CONTAINER gwcli ls iscsi-targets/ | grep 'o- gateways' | awk -F'[' '{print $2}' From 6e5a1eefd0b48614f7c95f86fd9fadab1b2b03fe Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Wed, 28 Jul 2021 10:29:47 -0400 Subject: [PATCH 0196/2866] mgr/dashboard: consume mgr/nfs via mgr.remote() Stop using the dashboard version of the Ganesha config classes; consume mgr/nfs instead via remote(). mgr/nfs/export: return Export from _apply_export Future callers will want this. mgr/nfs: new module methods for dashboard consumption Add some new methods that are easy for the dashboard API to consume. These are very similar to the CLI methods but do now have the @CLICommand and related decorators, and have slightly different interfaces (e.g., returning the created/modified Export dict). mgr/dashboard: remove old ganesha code (and tests) Fixes: https://tracker.ceph.com/issues/46493 Signed-off-by: Sage Weil --- .../mgr/dashboard/controllers/nfsganesha.py | 89 +- src/pybind/mgr/dashboard/services/ganesha.py | 1128 ----------------- .../mgr/dashboard/tests/test_ganesha.py | 956 +------------- src/pybind/mgr/nfs/cluster.py | 14 + src/pybind/mgr/nfs/export.py | 41 +- src/pybind/mgr/nfs/module.py | 38 +- 6 files changed, 123 insertions(+), 2143 deletions(-) delete mode 100644 src/pybind/mgr/dashboard/services/ganesha.py diff --git a/src/pybind/mgr/dashboard/controllers/nfsganesha.py b/src/pybind/mgr/dashboard/controllers/nfsganesha.py index 7c740587381b6..5571bbd00b5db 100644 --- a/src/pybind/mgr/dashboard/controllers/nfsganesha.py +++ b/src/pybind/mgr/dashboard/controllers/nfsganesha.py @@ -7,17 +7,21 @@ import cephfs import cherrypy +from .. import mgr from ..security import Scope from ..services.cephfs import CephFS -from ..services.cephx import CephX from ..services.exception import DashboardException, serialize_dashboard_exception -from ..services.ganesha import Ganesha, GaneshaConf, NFSException from ..services.rgw_client import NoCredentialsException, \ NoRgwDaemonsException, RequestException, RgwClient from . import APIDoc, APIRouter, BaseController, Endpoint, EndpointDoc, \ ReadPermission, RESTController, Task, UIRouter -logger = logging.getLogger('controllers.ganesha') +logger = logging.getLogger('controllers.nfs') + + +class NFSException(DashboardException): + def __init__(self, msg): + super(NFSException, self).__init__(component="nfs", msg=msg) # documentation helpers @@ -100,8 +104,8 @@ class NFSGanesha(RESTController): def status(self): status = {'available': True, 'message': None} try: - Ganesha.get_ganesha_clusters() - except NFSException as e: + mgr.remote('nfs', 'is_active') + except (NameError, ImportError) as e: status['message'] = str(e) # type: ignore status['available'] = False @@ -116,12 +120,7 @@ class NFSGaneshaExports(RESTController): @EndpointDoc("List all NFS-Ganesha exports", responses={200: [EXPORT_SCHEMA]}) def list(self): - result = [] - for cluster_id in Ganesha.get_ganesha_clusters(): - result.extend( - [export.to_dict() - for export in GaneshaConf.instance(cluster_id).list_exports()]) - return result + return mgr.remote('nfs', 'export_ls') @NfsTask('create', {'path': '{path}', 'fsal': '{fsal.name}', 'cluster_id': '{cluster_id}'}, 2.0) @@ -131,18 +130,18 @@ def list(self): def create(self, path, cluster_id, daemons, pseudo, tag, access_type, squash, security_label, protocols, transports, fsal, clients, reload_daemons=True): - if fsal['name'] not in Ganesha.fsals_available(): + if fsal['name'] not in mgr.remote('nfs', 'cluster_fsals'): raise NFSException("Cannot create this export. " "FSAL '{}' cannot be managed by the dashboard." .format(fsal['name'])) - ganesha_conf = GaneshaConf.instance(cluster_id) - ex_id = ganesha_conf.create_export({ + fsal.pop('user_id') # mgr/nfs does not let you customize user_id + # FIXME: what was this? 'tag': tag, + raw_ex = { 'path': path, 'pseudo': pseudo, 'cluster_id': cluster_id, 'daemons': daemons, - 'tag': tag, 'access_type': access_type, 'squash': squash, 'security_label': security_label, @@ -150,10 +149,9 @@ def create(self, path, cluster_id, daemons, pseudo, tag, access_type, 'transports': transports, 'fsal': fsal, 'clients': clients - }) - if reload_daemons: - ganesha_conf.reload_daemons(daemons) - return ganesha_conf.get_export(ex_id).to_dict() + } + export = mgr.remote('nfs', 'export_apply', cluster_id, raw_ex) + return export @EndpointDoc("Get an NFS-Ganesha export", parameters={ @@ -162,11 +160,7 @@ def create(self, path, cluster_id, daemons, pseudo, tag, access_type, }, responses={200: EXPORT_SCHEMA}) def get(self, cluster_id, export_id): - export_id = int(export_id) - ganesha_conf = GaneshaConf.instance(cluster_id) - if not ganesha_conf.has_export(export_id): - raise cherrypy.HTTPError(404) - return ganesha_conf.get_export(export_id).to_dict() + return mgr.remote('nfs', 'export_get', cluster_id, export_id) @NfsTask('edit', {'cluster_id': '{cluster_id}', 'export_id': '{export_id}'}, 2.0) @@ -178,23 +172,22 @@ def set(self, cluster_id, export_id, path, daemons, pseudo, tag, access_type, squash, security_label, protocols, transports, fsal, clients, reload_daemons=True): export_id = int(export_id) - ganesha_conf = GaneshaConf.instance(cluster_id) - if not ganesha_conf.has_export(export_id): + if not mgr.remote('nfs', 'export_get', export_id): raise cherrypy.HTTPError(404) # pragma: no cover - the handling is too obvious - if fsal['name'] not in Ganesha.fsals_available(): + if fsal['name'] not in mgr.remote('nfs', 'cluster_fsals'): raise NFSException("Cannot make modifications to this export. " "FSAL '{}' cannot be managed by the dashboard." .format(fsal['name'])) - old_export = ganesha_conf.update_export({ - 'export_id': export_id, + fsal.pop('user_id') # mgr/nfs does not let you customize user_id + # FIXME: what was this? 'tag': tag, + raw_ex = { 'path': path, + 'pseudo': pseudo, 'cluster_id': cluster_id, 'daemons': daemons, - 'pseudo': pseudo, - 'tag': tag, 'access_type': access_type, 'squash': squash, 'security_label': security_label, @@ -202,14 +195,9 @@ def set(self, cluster_id, export_id, path, daemons, pseudo, tag, access_type, 'transports': transports, 'fsal': fsal, 'clients': clients - }) - daemons = list(daemons) - for d_id in old_export.daemons: - if d_id not in daemons: - daemons.append(d_id) - if reload_daemons: - ganesha_conf.reload_daemons(daemons) - return ganesha_conf.get_export(export_id).to_dict() + } + export = mgr.remote('nfs', 'export_apply', cluster_id, raw_ex) + return export @NfsTask('delete', {'cluster_id': '{cluster_id}', 'export_id': '{export_id}'}, 2.0) @@ -224,13 +212,11 @@ def set(self, cluster_id, export_id, path, daemons, pseudo, tag, access_type, }) def delete(self, cluster_id, export_id, reload_daemons=True): export_id = int(export_id) - ganesha_conf = GaneshaConf.instance(cluster_id) - if not ganesha_conf.has_export(export_id): + export = mgr.remote('nfs', 'export_get', cluster_id, export_id) + if not export: raise cherrypy.HTTPError(404) # pragma: no cover - the handling is too obvious - export = ganesha_conf.remove_export(export_id) - if reload_daemons: - ganesha_conf.reload_daemons(export.daemons) + mgr.remote('nfs', 'export_rm', cluster_id, export['pseudo']) @APIRouter('/nfs-ganesha/daemon', Scope.NFS_GANESHA) @@ -241,15 +227,13 @@ class NFSGaneshaService(RESTController): responses={200: [{ 'daemon_id': (str, 'Daemon identifier'), 'cluster_id': (str, 'Cluster identifier'), - 'cluster_type': (str, 'Cluster type'), + 'cluster_type': (str, 'Cluster type'), # FIXME: remove this property 'status': (int, 'Status of daemon', True), 'desc': (str, 'Status description', True) }]}) def list(self): - result = [] - for cluster_id in Ganesha.get_ganesha_clusters(): - result.extend(GaneshaConf.instance(cluster_id).list_daemons()) - return result + # FIXME: remove this; dashboard should only care about clusters. + return mgr.remote('nfs', 'daemon_ls') @UIRouter('/nfs-ganesha', Scope.NFS_GANESHA) @@ -257,12 +241,13 @@ class NFSGaneshaUi(BaseController): @Endpoint('GET', '/cephx/clients') @ReadPermission def cephx_clients(self): - return list(CephX.list_clients()) + # FIXME: remove this; cephx users/creds are managed by mgr/nfs + return ['admin'] @Endpoint('GET', '/fsals') @ReadPermission def fsals(self): - return Ganesha.fsals_available() + return mgr.remote('nfs', 'cluster_fsals') @Endpoint('GET', '/lsdir') @ReadPermission @@ -316,4 +301,4 @@ def buckets(self, user_id=None): @Endpoint('GET', '/clusters') @ReadPermission def clusters(self): - return Ganesha.get_ganesha_clusters() + return mgr.remote('nfs', 'cluster_ls') diff --git a/src/pybind/mgr/dashboard/services/ganesha.py b/src/pybind/mgr/dashboard/services/ganesha.py deleted file mode 100644 index 59ff711db17e8..0000000000000 --- a/src/pybind/mgr/dashboard/services/ganesha.py +++ /dev/null @@ -1,1128 +0,0 @@ -# -*- coding: utf-8 -*- -# pylint: disable=too-many-lines - -import logging -import os -import re -from typing import Any, Dict, List, Optional, cast - -from ceph.deployment.service_spec import NFSServiceSpec -from orchestrator import DaemonDescription, OrchestratorError, ServiceDescription - -from .. import mgr -from ..exceptions import DashboardException -from ..settings import Settings -from .cephfs import CephFS -from .cephx import CephX -from .orchestrator import OrchClient -from .rgw_client import NoCredentialsException, NoRgwDaemonsException, RequestException, RgwClient - -logger = logging.getLogger('ganesha') - - -class NFSException(DashboardException): - def __init__(self, msg): - super(NFSException, self).__init__(component="nfs", msg=msg) - - -class Ganesha(object): - @classmethod - def _get_clusters_locations(cls): - # pylint: disable=too-many-branches - # Get Orchestrator clusters - orch_result = cls._get_orch_clusters_locations() - - # Get user-defined clusters - location_list_str = Settings.GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE - if not orch_result and not location_list_str: - raise NFSException("NFS-Ganesha cluster is not detected. " - "Please set the GANESHA_RADOS_POOL_NAMESPACE " - "setting or deploy an NFS-Ganesha cluster with the Orchestrator.") - result = {} # type: ignore - location_list = [loc.strip() for loc in location_list_str.split( - ",")] if location_list_str else [] - for location in location_list: - if not location: - raise NFSException("Invalid Ganesha cluster RADOS " - "[cluster_id:]pool/namespace setting: {}" - .format(location)) - if location.count(':') < 1: - # default cluster_id - if location.count('/') > 1: - raise NFSException("Invalid Ganesha RADOS pool/namespace " - "setting: {}".format(location)) - # in this case accept pool/namespace only - cluster = "_default_" - if location.count('/') == 0: - pool, namespace = location, None - else: - pool, namespace = location.split('/', 1) - else: - cluster = location[:location.find(':')] - pool_nm = location[location.find(':')+1:] - if pool_nm.count('/') == 0: - pool, namespace = pool_nm, None - else: - pool, namespace = pool_nm.split('/', 1) - - # Check pool/namespace collision. - for clusters in [orch_result, result]: - for cluster_name, cluster_data in clusters.items(): - if cluster_data['pool'] == pool and cluster_data['namespace'] == namespace: - raise NFSException( - f'Pool `{pool}` and namespace `{namespace}` are already in use by ' - f"""NFS-Ganesha cluster called `{cluster_name}`{" that is deployed by " - "the Orchestrator" if cluster_data['type'] == ClusterType.ORCHESTRATOR - else ''}. """ - 'Please update GANESHA_RADOS_POOL_NAMESPACE setting.' - ) - - if cluster in orch_result: - # cephadm might have set same cluster settings, ask the user to remove it. - raise NFSException( - 'Detected a conflicting NFS-Ganesha cluster name `{0}`. There exists an ' - 'NFS-Ganesha cluster called `{0}` that is deployed by the Orchestrator. ' - 'Please remove or rename the cluster from the GANESHA_RADOS_POOL_NAMESPACE ' - 'setting.'.format(cluster)) - - if cluster in result: - raise NFSException("Duplicate Ganesha cluster definition in " - "the setting: {}".format(location_list_str)) - result[cluster] = { - 'pool': pool, - 'namespace': namespace, - 'type': ClusterType.USER, - 'daemon_conf': None - } - return {**orch_result, **result} - - @classmethod - def _get_orch_clusters_locations(cls): - orch_result = {} # type: ignore - services = cls._get_orch_nfs_services() - for service in services: - spec = cast(NFSServiceSpec, service.spec) - try: - orch_result[spec.service_id] = { - 'pool': 'nfs-ganesha', - 'namespace': spec.service_id, - 'type': ClusterType.ORCHESTRATOR, - 'daemon_conf': spec.rados_config_name() - } - except AttributeError as ex: - logger.warning('Error when getting NFS service from the Orchestrator. %s', str(ex)) - continue - return orch_result - - @classmethod - def get_ganesha_clusters(cls): - return list(cls._get_clusters_locations()) - - @staticmethod - def _get_orch_nfs_services() -> List[ServiceDescription]: - try: - return OrchClient.instance().services.list('nfs') - except (RuntimeError, OrchestratorError, ImportError): - return [] - - @classmethod - def parse_rados_url(cls, rados_url): - if not rados_url.startswith("rados://"): - raise NFSException("Invalid NFS Ganesha RADOS configuration URL: {}" - .format(rados_url)) - rados_url = rados_url[8:] - url_comps = rados_url.split("/") - if len(url_comps) < 2 or len(url_comps) > 3: - raise NFSException("Invalid NFS Ganesha RADOS configuration URL: " - "rados://{}".format(rados_url)) - if len(url_comps) == 2: - return url_comps[0], None, url_comps[1] - return url_comps - - @classmethod - def make_rados_url(cls, pool, namespace, obj): - if namespace: - return "rados://{}/{}/{}".format(pool, namespace, obj) - return "rados://{}/{}".format(pool, obj) - - @classmethod - def get_cluster(cls, cluster_id): - locations = cls._get_clusters_locations() - if cluster_id not in locations: - raise NFSException("Cluster not found: cluster_id={}" - .format(cluster_id)) - return locations[cluster_id] - - @classmethod - def fsals_available(cls): - result = [] - if CephFS.list_filesystems(): - result.append("CEPH") - try: - if RgwClient.admin_instance().is_service_online() and \ - RgwClient.admin_instance().is_system_user(): - result.append("RGW") - except (DashboardException, NoCredentialsException, RequestException, - NoRgwDaemonsException): - pass - return result - - -class GaneshaConfParser(object): - def __init__(self, raw_config): - self.pos = 0 - self.text = "" - self.clean_config(raw_config) - - def clean_config(self, raw_config): - for line in raw_config.split("\n"): - cardinal_idx = line.find('#') - if cardinal_idx == -1: - self.text += line - else: - # remove comments - self.text += line[:cardinal_idx] - if line.startswith("%"): - self.text += "\n" - - def remove_all_whitespaces(self): - new_text = "" - in_string = False - in_section = False - for i, cha in enumerate(self.text): - if in_section: - if cha != '"' and self.text[i-1] != '\\': - new_text += cha - elif cha == '\n': - new_text += cha - in_section = False - elif i == (len(self.text)-1): - if cha != '"' and self.text[i-1] != '\\': - new_text += cha - in_section = False - elif not in_section and (i == 0 or self.text[i-1] == '\n') and cha == '%': - in_section = True - new_text += cha - elif in_string or cha not in [' ', '\n', '\t']: - new_text += cha - elif cha == '"' and self.text[i-1] != '\\': - in_string = not in_string - self.text = new_text - - def stream(self): - return self.text[self.pos:] - - def parse_block_name(self): - idx = self.stream().find('{') - if idx == -1: - raise Exception("Cannot find block name") - block_name = self.stream()[:idx] - self.pos += idx+1 - return block_name - - def parse_block_or_section(self): - if self.stream().startswith("%url"): - # section line - self.pos += self.stream().find('rados://') - idx = self.stream().find('\n') - if idx == -1: - value = self.stream() - self.pos += len(self.stream()) - else: - value = self.stream()[:idx] - self.pos += idx+1 - block_dict = {'block_name': '%url', 'value': value} - return block_dict - - block_name = self.parse_block_name().upper() - block_dict = {'block_name': block_name} - self.parse_block_body(block_dict) - if self.stream()[0] != '}': - raise Exception("No closing bracket '}' found at the end of block") - self.pos += 1 - return block_dict - - def parse_parameter_value(self, raw_value): - colon_idx = raw_value.find(',') - - if colon_idx == -1: - try: - return int(raw_value) - except ValueError: - if raw_value == "true": - return True - if raw_value == "false": - return False - if raw_value.find('"') == 0: - return raw_value[1:-1] - return raw_value - else: - return [self.parse_parameter_value(v.strip()) - for v in raw_value.split(',')] - - def parse_stanza(self, block_dict): - equal_idx = self.stream().find('=') - semicolon_idx = self.stream().find(';') - if equal_idx == -1: - raise Exception("Malformed stanza: no equal symbol found.") - parameter_name = self.stream()[:equal_idx].lower() - parameter_value = self.stream()[equal_idx+1:semicolon_idx] - block_dict[parameter_name] = self.parse_parameter_value( - parameter_value) - self.pos += semicolon_idx+1 - - def parse_block_body(self, block_dict): - last_pos = self.pos - while True: - semicolon_idx = self.stream().find(';') - lbracket_idx = self.stream().find('{') - rbracket_idx = self.stream().find('}') - - if rbracket_idx == 0: - # block end - return - - if (semicolon_idx != -1 and lbracket_idx != -1 - and semicolon_idx < lbracket_idx) \ - or (semicolon_idx != -1 and lbracket_idx == -1): - self.parse_stanza(block_dict) - elif (semicolon_idx != -1 and lbracket_idx != -1 - and semicolon_idx > lbracket_idx) or ( - semicolon_idx == -1 and lbracket_idx != -1): - if '_blocks_' not in block_dict: - block_dict['_blocks_'] = [] - block_dict['_blocks_'].append(self.parse_block_or_section()) - else: - raise Exception("Malformed stanza: no semicolon found.") - - if last_pos == self.pos: - raise Exception("Infinite loop while parsing block content") - last_pos = self.pos - - def parse(self): - self.remove_all_whitespaces() - blocks = [] - while self.stream(): - block_dict = self.parse_block_or_section() - blocks.append(block_dict) - return blocks - - @staticmethod - def _indentation(depth, size=4): - conf_str = "" - for _ in range(0, depth*size): - conf_str += " " - return conf_str - - @staticmethod - def write_block_body(block, depth=0): - def format_val(key, val): - if isinstance(val, list): - return ', '.join([format_val(key, v) for v in val]) - if isinstance(val, bool): - return str(val).lower() - if isinstance(val, int) or (block['block_name'] == 'CLIENT' - and key == 'clients'): - return '{}'.format(val) - return '"{}"'.format(val) - - conf_str = "" - for key, val in block.items(): - if key == 'block_name': - continue - if key == '_blocks_': - for blo in val: - conf_str += GaneshaConfParser.write_block(blo, depth) - elif val: - conf_str += GaneshaConfParser._indentation(depth) - conf_str += '{} = {};\n'.format(key, format_val(key, val)) - return conf_str - - @staticmethod - def write_block(block, depth): - if block['block_name'] == "%url": - return '%url "{}"\n\n'.format(block['value']) - - conf_str = "" - conf_str += GaneshaConfParser._indentation(depth) - conf_str += format(block['block_name']) - conf_str += " {\n" - conf_str += GaneshaConfParser.write_block_body(block, depth+1) - conf_str += GaneshaConfParser._indentation(depth) - conf_str += "}\n\n" - return conf_str - - @staticmethod - def write_conf(blocks): - if not isinstance(blocks, list): - blocks = [blocks] - conf_str = "" - for block in blocks: - conf_str += GaneshaConfParser.write_block(block, 0) - return conf_str - - -class FSal(object): - def __init__(self, name): - self.name = name - - @classmethod - def validate_path(cls, _): - raise NotImplementedError() - - def validate(self): - raise NotImplementedError() - - def fill_keys(self): - raise NotImplementedError() - - def create_path(self, path): - raise NotImplementedError() - - @staticmethod - def from_fsal_block(fsal_block): - if fsal_block['name'] == "CEPH": - return CephFSFSal.from_fsal_block(fsal_block) - if fsal_block['name'] == 'RGW': - return RGWFSal.from_fsal_block(fsal_block) - return None - - def to_fsal_block(self): - raise NotImplementedError() - - @staticmethod - def from_dict(fsal_dict): - if fsal_dict['name'] == "CEPH": - return CephFSFSal.from_dict(fsal_dict) - if fsal_dict['name'] == 'RGW': - return RGWFSal.from_dict(fsal_dict) - return None - - def to_dict(self): - raise NotImplementedError() - - -class RGWFSal(FSal): - def __init__(self, name, rgw_user_id, access_key, secret_key): - super(RGWFSal, self).__init__(name) - self.rgw_user_id = rgw_user_id - self.access_key = access_key - self.secret_key = secret_key - - @classmethod - def validate_path(cls, path): - return path == "/" or re.match(r'^[^/><|&()#?]+$', path) - - def validate(self): - if not self.rgw_user_id: - raise NFSException('RGW user must be specified') - - if not RgwClient.admin_instance().user_exists(self.rgw_user_id): - raise NFSException("RGW user '{}' does not exist" - .format(self.rgw_user_id)) - - def fill_keys(self): - keys = RgwClient.admin_instance().get_user_keys(self.rgw_user_id) - self.access_key = keys['access_key'] - self.secret_key = keys['secret_key'] - - def create_path(self, path): - if path == '/': # nothing to do - return - rgw = RgwClient.instance(self.rgw_user_id) - try: - exists = rgw.bucket_exists(path, self.rgw_user_id) - logger.debug('Checking existence of RGW bucket "%s" for user "%s": %s', - path, self.rgw_user_id, exists) - except RequestException as exp: - if exp.status_code == 403: - raise NFSException('Cannot create bucket "{}" as it already ' - 'exists, and belongs to other user.' - .format(path)) - raise exp - if not exists: - logger.info('Creating new RGW bucket "%s" for user "%s"', path, - self.rgw_user_id) - rgw.create_bucket(path) - - @classmethod - def from_fsal_block(cls, fsal_block): - return cls(fsal_block['name'], - fsal_block['user_id'], - fsal_block['access_key_id'], - fsal_block['secret_access_key']) - - def to_fsal_block(self): - return { - 'block_name': 'FSAL', - 'name': self.name, - 'user_id': self.rgw_user_id, - 'access_key_id': self.access_key, - 'secret_access_key': self.secret_key - } - - @classmethod - def from_dict(cls, fsal_dict): - return cls(fsal_dict['name'], fsal_dict['rgw_user_id'], None, None) - - def to_dict(self): - return { - 'name': self.name, - 'rgw_user_id': self.rgw_user_id - } - - -class CephFSFSal(FSal): - def __init__(self, name, user_id=None, fs_name=None, sec_label_xattr=None, - cephx_key=None): - super(CephFSFSal, self).__init__(name) - self.fs_name = fs_name - self.user_id = user_id - self.sec_label_xattr = sec_label_xattr - self.cephx_key = cephx_key - - @classmethod - def validate_path(cls, path): - return re.match(r'^/[^><|&()?]*$', path) - - def validate(self): - if self.user_id and self.user_id not in CephX.list_clients(): - raise NFSException("cephx user '{}' does not exist" - .format(self.user_id)) - - def fill_keys(self): - if self.user_id: - self.cephx_key = CephX.get_client_key(self.user_id) - - def create_path(self, path): - cfs = CephFS(self.fs_name) - if path == os.sep: - return - cfs.mk_dirs(path) - - @classmethod - def from_fsal_block(cls, fsal_block): - return cls(fsal_block['name'], - fsal_block.get('user_id', None), - fsal_block.get('filesystem', None), - fsal_block.get('sec_label_xattr', None), - fsal_block.get('secret_access_key', None)) - - def to_fsal_block(self): - result = { - 'block_name': 'FSAL', - 'name': self.name, - } - if self.user_id: - result['user_id'] = self.user_id - if self.fs_name: - result['filesystem'] = self.fs_name - if self.sec_label_xattr: - result['sec_label_xattr'] = self.sec_label_xattr - if self.cephx_key: - result['secret_access_key'] = self.cephx_key - return result - - @classmethod - def from_dict(cls, fsal_dict): - return cls(fsal_dict['name'], fsal_dict['user_id'], - fsal_dict['fs_name'], fsal_dict['sec_label_xattr'], None) - - def to_dict(self): - return { - 'name': self.name, - 'user_id': self.user_id, - 'fs_name': self.fs_name, - 'sec_label_xattr': self.sec_label_xattr - } - - -class Client(object): - def __init__(self, addresses, access_type=None, squash=None): - self.addresses = addresses - self.access_type = access_type - self.squash = GaneshaConf.format_squash(squash) - - @classmethod - def from_client_block(cls, client_block): - addresses = client_block['clients'] - if not isinstance(addresses, list): - addresses = [addresses] - return cls(addresses, - client_block.get('access_type', None), - client_block.get('squash', None)) - - def to_client_block(self): - result = { - 'block_name': 'CLIENT', - 'clients': self.addresses, - } - if self.access_type: - result['access_type'] = self.access_type - if self.squash: - result['squash'] = self.squash - return result - - @classmethod - def from_dict(cls, client_dict): - return cls(client_dict['addresses'], client_dict['access_type'], - client_dict['squash']) - - def to_dict(self): - return { - 'addresses': self.addresses, - 'access_type': self.access_type, - 'squash': self.squash - } - - -class Export(object): - # pylint: disable=R0902 - def __init__(self, export_id, path, fsal, cluster_id, daemons, pseudo=None, - tag=None, access_type=None, squash=None, - attr_expiration_time=None, security_label=False, - protocols=None, transports=None, clients=None): - self.export_id = export_id - self.path = GaneshaConf.format_path(path) - self.fsal = fsal - self.cluster_id = cluster_id - self.daemons = set(daemons) - self.pseudo = GaneshaConf.format_path(pseudo) - self.tag = tag - self.access_type = access_type - self.squash = GaneshaConf.format_squash(squash) - if attr_expiration_time is None: - self.attr_expiration_time = 0 - else: - self.attr_expiration_time = attr_expiration_time - self.security_label = security_label - self.protocols = {GaneshaConf.format_protocol(p) for p in protocols} - self.transports = set(transports) - self.clients = clients - - def validate(self): - # pylint: disable=R0912 - if not self.fsal.validate_path(self.path): - raise NFSException("Export path ({}) is invalid.".format(self.path)) - - if not self.protocols: - raise NFSException( - "No NFS protocol version specified for the export.") - - if not self.transports: - raise NFSException( - "No network transport type specified for the export.") - - for t in self.transports: - match = re.match(r'^TCP$|^UDP$', t) - if not match: - raise NFSException( - "'{}' is an invalid network transport type identifier" - .format(t)) - - self.fsal.validate() - - if 4 in self.protocols: - if not self.pseudo: - raise NFSException( - "Pseudo path is required when NFSv4 protocol is used") - match = re.match(r'^/[^><|&()]*$', self.pseudo) - if not match: - raise NFSException( - "Export pseudo path ({}) is invalid".format(self.pseudo)) - - if self.tag: - match = re.match(r'^[^/><|:&()]+$', self.tag) - if not match: - raise NFSException( - "Export tag ({}) is invalid".format(self.tag)) - - if self.fsal.name == 'RGW' and 4 not in self.protocols and not self.tag: - raise NFSException( - "Tag is mandatory for RGW export when using only NFSv3") - - @classmethod - def from_export_block(cls, export_block, cluster_id, defaults): - logger.debug("parsing export block: %s", export_block) - - fsal_block = [b for b in export_block['_blocks_'] - if b['block_name'] == "FSAL"] - - protocols = export_block.get('protocols', defaults['protocols']) - if not isinstance(protocols, list): - protocols = [protocols] - - transports = export_block.get('transports', defaults['transports']) - if not isinstance(transports, list): - transports = [transports] - - client_blocks = [b for b in export_block['_blocks_'] - if b['block_name'] == "CLIENT"] - - return cls(export_block['export_id'], - export_block['path'], - FSal.from_fsal_block(fsal_block[0]), - cluster_id, - [], - export_block.get('pseudo', None), - export_block.get('tag', None), - export_block.get('access_type', defaults['access_type']), - export_block.get('squash', defaults['squash']), - export_block.get('attr_expiration_time', None), - export_block.get('security_label', False), - protocols, - transports, - [Client.from_client_block(client) - for client in client_blocks]) - - def to_export_block(self, defaults): - # pylint: disable=too-many-branches - result = { - 'block_name': 'EXPORT', - 'export_id': self.export_id, - 'path': self.path - } - if self.pseudo: - result['pseudo'] = self.pseudo - if self.tag: - result['tag'] = self.tag - if 'access_type' not in defaults \ - or self.access_type != defaults['access_type']: - result['access_type'] = self.access_type - if 'squash' not in defaults or self.squash != defaults['squash']: - result['squash'] = self.squash - if self.fsal.name == 'CEPH': - result['attr_expiration_time'] = self.attr_expiration_time - result['security_label'] = self.security_label - if 'protocols' not in defaults: - result['protocols'] = list(self.protocols) - else: - def_proto = defaults['protocols'] - if not isinstance(def_proto, list): - def_proto = set([def_proto]) - if self.protocols != def_proto: - result['protocols'] = list(self.protocols) - if 'transports' not in defaults: - result['transports'] = list(self.transports) - else: - def_transp = defaults['transports'] - if not isinstance(def_transp, list): - def_transp = set([def_transp]) - if self.transports != def_transp: - result['transports'] = list(self.transports) - - result['_blocks_'] = [self.fsal.to_fsal_block()] - result['_blocks_'].extend([client.to_client_block() - for client in self.clients]) - return result - - @classmethod - def from_dict(cls, export_id, ex_dict, old_export=None): - return cls(export_id, - ex_dict['path'], - FSal.from_dict(ex_dict['fsal']), - ex_dict['cluster_id'], - ex_dict['daemons'], - ex_dict['pseudo'], - ex_dict['tag'], - ex_dict['access_type'], - ex_dict['squash'], - old_export.attr_expiration_time if old_export else None, - ex_dict['security_label'], - ex_dict['protocols'], - ex_dict['transports'], - [Client.from_dict(client) for client in ex_dict['clients']]) - - def to_dict(self): - return { - 'export_id': self.export_id, - 'path': self.path, - 'fsal': self.fsal.to_dict(), - 'cluster_id': self.cluster_id, - 'daemons': sorted(list(self.daemons)), - 'pseudo': self.pseudo, - 'tag': self.tag, - 'access_type': self.access_type, - 'squash': self.squash, - 'security_label': self.security_label, - 'protocols': sorted(list(self.protocols)), - 'transports': sorted(list(self.transports)), - 'clients': [client.to_dict() for client in self.clients] - } - - -class ClusterType(object): - - # Ganesha clusters deployed by the Orchestrator. - ORCHESTRATOR = 'orchestrator' - - # Ganesha clusters deployed manually by the user. Specified by using the - # GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE setting. - USER = 'user' - - -class GaneshaConf(object): - # pylint: disable=R0902 - - def __init__(self, cluster_id, rados_pool, rados_namespace, daemon_confs=None): - self.cluster_id = cluster_id - self.rados_pool = rados_pool - self.rados_namespace = rados_namespace - self.daemon_confs = daemon_confs if daemon_confs is not None else [] - self.export_conf_blocks = [] # type: ignore - self.daemons_conf_blocks = {} # type: ignore - self._defaults = {} - self.exports = {} - - self._read_raw_config() - - # load defaults - def_block = [b for b in self.export_conf_blocks - if b['block_name'] == "EXPORT_DEFAULTS"] - self.export_defaults = def_block[0] if def_block else {} - self._defaults = self.ganesha_defaults(self.export_defaults) - - for export_block in [block for block in self.export_conf_blocks - if block['block_name'] == "EXPORT"]: - export = Export.from_export_block(export_block, cluster_id, - self._defaults) - self.exports[export.export_id] = export - - # link daemons to exports - self._link_daemons_to_exports() - - def _link_daemons_to_exports(self): - raise NotImplementedError() - - @classmethod - def instance(cls, cluster_id): - cluster = Ganesha.get_cluster(cluster_id) - if cluster['type'] == ClusterType.ORCHESTRATOR: - return GaneshaConfOrchestrator(cluster_id, cluster['pool'], cluster['namespace'], - [cluster['daemon_conf']]) - if cluster['type'] == ClusterType.USER: - return GaneshaConfUser(cluster_id, cluster['pool'], cluster['namespace']) - raise NFSException('Unknown cluster type `{}` for cluster `{}`'.format( - cluster['type'], cluster_id)) - - def _read_raw_config(self): - - def _read_rados_obj(_obj): - size, _ = _obj.stat() - return _obj.read(size).decode("utf-8") - - with mgr.rados.open_ioctx(self.rados_pool) as ioctx: - if self.rados_namespace: - ioctx.set_namespace(self.rados_namespace) - objs = ioctx.list_objects() - for obj in objs: - if obj.key.startswith("export-"): - raw_config = _read_rados_obj(obj) - logger.debug("read export configuration from rados " - "object %s/%s/%s:\n%s", self.rados_pool, - self.rados_namespace, obj.key, raw_config) - self.export_conf_blocks.extend( - GaneshaConfParser(raw_config).parse()) - elif not self.daemon_confs and obj.key.startswith("conf-"): - # Read all `conf-xxx` for daemon configs. - raw_config = _read_rados_obj(obj) - logger.debug("read daemon configuration from rados " - "object %s/%s/%s:\n%s", self.rados_pool, - self.rados_namespace, obj.key, raw_config) - idx = obj.key.find('-') - self.daemons_conf_blocks[obj.key[idx+1:]] = \ - GaneshaConfParser(raw_config).parse() - - if self.daemon_confs: - # When daemon configs are provided. - for conf in self.daemon_confs: - size, _ = ioctx.stat(conf) - raw_config = ioctx.read(conf, size).decode("utf-8") - logger.debug("read daemon configuration from rados " - "object %s/%s/%s:\n%s", self.rados_pool, - self.rados_namespace, conf, raw_config) - self.daemons_conf_blocks[conf] = \ - GaneshaConfParser(raw_config).parse() - - def _write_raw_config(self, conf_block, obj): - raw_config = GaneshaConfParser.write_conf(conf_block) - with mgr.rados.open_ioctx(self.rados_pool) as ioctx: - if self.rados_namespace: - ioctx.set_namespace(self.rados_namespace) - ioctx.write_full(obj, raw_config.encode('utf-8')) - logger.debug( - "write configuration into rados object %s/%s/%s:\n%s", - self.rados_pool, self.rados_namespace, obj, raw_config) - - @classmethod - def ganesha_defaults(cls, export_defaults): - """ - According to - https://github.com/nfs-ganesha/nfs-ganesha/blob/next/src/config_samples/export.txt - """ - return { - 'access_type': export_defaults.get('access_type', 'NONE'), - 'protocols': export_defaults.get('protocols', [3, 4]), - 'transports': export_defaults.get('transports', ['TCP', 'UDP']), - 'squash': export_defaults.get('squash', 'root_squash') - } - - @classmethod - def format_squash(cls, squash): - if squash is None: - return None - if squash.lower() in ["no_root_squash", "noidsquash", "none"]: - return "no_root_squash" - if squash.lower() in ["rootid", "root_id_squash", "rootidsquash"]: - return "root_id_squash" - if squash.lower() in ["root", "root_squash", "rootsquash"]: - return "root_squash" - if squash.lower() in ["all", "all_squash", "allsquash", - "all_anonymous", "allanonymous"]: - return "all_squash" - logger.error("could not parse squash value: %s", squash) - raise NFSException("'{}' is an invalid squash option".format(squash)) - - @classmethod - def format_protocol(cls, protocol): - if str(protocol) in ["NFSV3", "3", "V3", "NFS3"]: - return 3 - if str(protocol) in ["NFSV4", "4", "V4", "NFS4"]: - return 4 - logger.error("could not parse protocol value: %s", protocol) - raise NFSException("'{}' is an invalid NFS protocol version identifier" - .format(protocol)) - - @classmethod - def format_path(cls, path): - if path is not None: - path = path.strip() - if len(path) > 1 and path[-1] == '/': - path = path[:-1] - return path - - def validate(self, export: Export): - export.validate() - - if 4 in export.protocols: # NFSv4 protocol - len_prefix = 1 - parent_export = None - for ex in self.list_exports(): - if export.tag and ex.tag == export.tag: - raise NFSException( - "Another export exists with the same tag: {}" - .format(export.tag)) - - if export.pseudo and ex.pseudo == export.pseudo: - raise NFSException( - "Another export exists with the same pseudo path: {}" - .format(export.pseudo)) - - if not ex.pseudo: - continue - - if export.pseudo[:export.pseudo.rfind('/')+1].startswith(ex.pseudo): - if export.pseudo[len(ex.pseudo)] == '/': - if len(ex.pseudo) > len_prefix: - len_prefix = len(ex.pseudo) - parent_export = ex - - if len_prefix > 1: - # validate pseudo path - idx = len(parent_export.pseudo) # type: ignore - idx = idx + 1 if idx > 1 else idx - real_path = "{}/{}".format( - parent_export.path # type: ignore - if len(parent_export.path) > 1 else "", # type: ignore - export.pseudo[idx:]) - if export.fsal.name == 'CEPH': - cfs = CephFS() - if export.path != real_path and not cfs.dir_exists(real_path): - raise NFSException( - "Pseudo path ({}) invalid, path {} does not exist." - .format(export.pseudo, real_path)) - - def _gen_export_id(self): - exports = sorted(self.exports) - nid = 1 - for e_id in exports: - if e_id == nid: - nid += 1 - else: - break - return nid - - def _persist_daemon_configuration(self): - raise NotImplementedError() - - def _save_export(self, export): - self.validate(export) - export.fsal.create_path(export.path) - export.fsal.fill_keys() - self.exports[export.export_id] = export - conf_block = export.to_export_block(self.export_defaults) - self._write_raw_config(conf_block, "export-{}".format(export.export_id)) - self._persist_daemon_configuration() - - def _delete_export(self, export_id): - self._persist_daemon_configuration() - with mgr.rados.open_ioctx(self.rados_pool) as ioctx: - if self.rados_namespace: - ioctx.set_namespace(self.rados_namespace) - ioctx.remove_object("export-{}".format(export_id)) - - def list_exports(self): - return [ex for _, ex in self.exports.items()] - - def create_export(self, ex_dict): - ex_id = self._gen_export_id() - export = Export.from_dict(ex_id, ex_dict) - self._save_export(export) - return ex_id - - def has_export(self, export_id): - return export_id in self.exports - - def update_export(self, ex_dict): - if ex_dict['export_id'] not in self.exports: - return None - old_export = self.exports[ex_dict['export_id']] - del self.exports[ex_dict['export_id']] - export = Export.from_dict(ex_dict['export_id'], ex_dict, old_export) - self._save_export(export) - self.exports[export.export_id] = export - return old_export - - def remove_export(self, export_id): - if export_id not in self.exports: - return None - export = self.exports[export_id] - del self.exports[export_id] - self._delete_export(export_id) - return export - - def get_export(self, export_id): - if export_id in self.exports: - return self.exports[export_id] - return None - - def list_daemons(self) -> List[Dict[str, Any]]: - raise NotImplementedError() - - def list_daemon_confs(self): - return self.daemons_conf_blocks.keys() - - def reload_daemons(self, daemons): - with mgr.rados.open_ioctx(self.rados_pool) as ioctx: - if self.rados_namespace: - ioctx.set_namespace(self.rados_namespace) - for daemon_id in daemons: - ioctx.notify("conf-{}".format(daemon_id)) - - -class GaneshaConfOrchestrator(GaneshaConf): - @classmethod - def _get_orch_nfs_instances(cls, - service_name: Optional[str] = None) -> List[DaemonDescription]: - try: - return OrchClient.instance().services.\ - list_daemons(service_name=service_name, daemon_type="nfs") - except (RuntimeError, OrchestratorError, ImportError): - return [] - - def _link_daemons_to_exports(self): - instances = self._get_orch_nfs_instances('nfs.{}'.format(self.cluster_id)) - daemon_ids = {instance.daemon_id for instance in instances} - for _, daemon_blocks in self.daemons_conf_blocks.items(): - for block in daemon_blocks: - if block['block_name'] == "%url": - rados_url = block['value'] - _, _, obj = Ganesha.parse_rados_url(rados_url) - if obj.startswith("export-"): - export_id = int(obj[obj.find('-')+1:]) - self.exports[export_id].daemons.update(daemon_ids) - - def validate(self, export: Export): - daemons_list = {d['daemon_id'] for d in self.list_daemons()} - if export.daemons and set(export.daemons) != daemons_list: - raise NFSException('Export should be linked to all daemons.') - super().validate(export) - - def _persist_daemon_configuration(self): - daemon_map = {} # type: ignore - for daemon_id in self.list_daemon_confs(): - daemon_map[daemon_id] = [] - - for daemon_id in self.list_daemon_confs(): - for _, ex in self.exports.items(): - if ex.daemons: - daemon_map[daemon_id].append({ - 'block_name': "%url", - 'value': Ganesha.make_rados_url( - self.rados_pool, self.rados_namespace, - "export-{}".format(ex.export_id)) - }) - for daemon_id, conf_blocks in daemon_map.items(): - self._write_raw_config(conf_blocks, daemon_id) - - def list_daemons(self) -> List[Dict[str, Any]]: - instances = self._get_orch_nfs_instances('nfs.{}'.format(self.cluster_id)) - return [{ - 'cluster_id': self.cluster_id, - 'daemon_id': instance.daemon_id, - 'cluster_type': ClusterType.ORCHESTRATOR, - 'status': instance.status, - 'status_desc': instance.status_desc - } for instance in instances] - - def reload_daemons(self, daemons): - with mgr.rados.open_ioctx(self.rados_pool) as ioctx: - if self.rados_namespace: - ioctx.set_namespace(self.rados_namespace) - for daemon_id in self.list_daemon_confs(): - ioctx.notify(daemon_id) - - -class GaneshaConfUser(GaneshaConf): - - def _link_daemons_to_exports(self): - for daemon_id, daemon_blocks in self.daemons_conf_blocks.items(): - for block in daemon_blocks: - if block['block_name'] == "%url": - rados_url = block['value'] - _, _, obj = Ganesha.parse_rados_url(rados_url) - if obj.startswith("export-"): - export_id = int(obj[obj.find('-')+1:]) - self.exports[export_id].daemons.add(daemon_id) - - def validate(self, export: Export): - daemons_list = [d['daemon_id'] for d in self.list_daemons()] - for daemon_id in export.daemons: - if daemon_id not in daemons_list: - raise NFSException("Daemon '{}' does not exist".format(daemon_id)) - super().validate(export) - - def _persist_daemon_configuration(self): - daemon_map = {} # type: ignore - for daemon_id in self.list_daemon_confs(): - daemon_map[daemon_id] = [] - - for _, ex in self.exports.items(): - for daemon in ex.daemons: - daemon_map[daemon].append({ - 'block_name': "%url", - 'value': Ganesha.make_rados_url( - self.rados_pool, self.rados_namespace, - "export-{}".format(ex.export_id)) - }) - for daemon_id, conf_blocks in daemon_map.items(): - self._write_raw_config(conf_blocks, "conf-{}".format(daemon_id)) - - def list_daemons(self) -> List[Dict[str, Any]]: - return [{ - 'cluster_id': self.cluster_id, - 'cluster_type': ClusterType.USER, - 'daemon_id': daemon_id, - 'status': 1, - 'status_desc': 'running' - } for daemon_id in self.list_daemon_confs()] diff --git a/src/pybind/mgr/dashboard/tests/test_ganesha.py b/src/pybind/mgr/dashboard/tests/test_ganesha.py index a77d6b4c64591..28eab96f5e6ba 100644 --- a/src/pybind/mgr/dashboard/tests/test_ganesha.py +++ b/src/pybind/mgr/dashboard/tests/test_ganesha.py @@ -1,965 +1,11 @@ # -*- coding: utf-8 -*- # pylint: disable=too-many-lines -import unittest -from unittest.mock import MagicMock, Mock, patch +from unittest.mock import patch from urllib.parse import urlencode -from ceph.deployment.service_spec import NFSServiceSpec -from orchestrator import DaemonDescription, ServiceDescription - -from .. import mgr from ..controllers.nfsganesha import NFSGaneshaUi -from ..services import ganesha -from ..services.ganesha import ClusterType, Export, GaneshaConf, GaneshaConfParser, NFSException -from ..settings import Settings from . import ControllerTestCase # pylint: disable=no-name-in-module -from . import KVStoreMockMixin # pylint: disable=no-name-in-module - - -class GaneshaConfTest(unittest.TestCase, KVStoreMockMixin): - daemon_raw_config = """ -NFS_CORE_PARAM { - Enable_NLM = false; - Enable_RQUOTA = false; - Protocols = 4; - NFS_Port = 14000; - } - - MDCACHE { - Dir_Chunk = 0; - } - - NFSv4 { - RecoveryBackend = rados_cluster; - Minor_Versions = 1, 2; - } - - RADOS_KV { - pool = nfs-ganesha; - namespace = vstart; - UserId = vstart; - nodeid = a; - } - - RADOS_URLS { - Userid = vstart; - watch_url = 'rados://nfs-ganesha/vstart/conf-nfs.vstart'; - } - - %url rados://nfs-ganesha/vstart/conf-nfs.vstart -""" - - export_1 = """ -EXPORT { - Export_ID=1; - Protocols = 4; - Path = /; - Pseudo = /cephfs_a/; - Access_Type = RW; - Protocols = 4; - Attr_Expiration_Time = 0; - # Delegations = R; - # Squash = root; - - FSAL { - Name = CEPH; - Filesystem = "a"; - User_Id = "ganesha"; - # Secret_Access_Key = "YOUR SECRET KEY HERE"; - } - - CLIENT - { - Clients = 192.168.0.10, 192.168.1.0/8; - Squash = None; - } - - CLIENT - { - Clients = 192.168.0.0/16; - Squash = All; - Access_Type = RO; - } -} -""" - - export_2 = """ -EXPORT -{ - Export_ID=2; - - Path = "/"; - - Pseudo = "/rgw"; - - Access_Type = RW; - - squash = AllAnonymous; - - Protocols = 4, 3; - - Transports = TCP, UDP; - - FSAL { - Name = RGW; - User_Id = "testuser"; - Access_Key_Id ="access_key"; - Secret_Access_Key = "secret_key"; - } -} -""" - - conf_nodea = ''' -%url "rados://nfs-ganesha/bar/export-2" - -%url "rados://nfs-ganesha/bar/export-1"''' - - conf_nodeb = '%url "rados://nfs-ganesha/bar/export-1"' - - conf_nfs_foo = ''' -%url "rados://nfs-ganesha/foo/export-1" - -%url "rados://nfs-ganesha/foo/export-2"''' - - class RObject(object): - def __init__(self, key, raw): - self.key = key - self.raw = raw - - def read(self, _): - return self.raw.encode('utf-8') - - def stat(self): - return len(self.raw), None - - def _ioctx_write_full_mock(self, key, content): - if key not in self.temp_store[self.temp_store_namespace]: - self.temp_store[self.temp_store_namespace][key] = \ - GaneshaConfTest.RObject(key, content.decode('utf-8')) - else: - self.temp_store[self.temp_store_namespace][key].raw = content.decode('utf-8') - - def _ioctx_remove_mock(self, key): - del self.temp_store[self.temp_store_namespace][key] - - def _ioctx_list_objects_mock(self): - return [obj for _, obj in self.temp_store[self.temp_store_namespace].items()] - - def _ioctl_stat_mock(self, key): - return self.temp_store[self.temp_store_namespace][key].stat() - - def _ioctl_read_mock(self, key, size): - return self.temp_store[self.temp_store_namespace][key].read(size) - - def _ioctx_set_namespace_mock(self, namespace): - self.temp_store_namespace = namespace - - @staticmethod - def _set_user_defined_clusters_location(clusters_pool_namespace='nfs-ganesha/bar'): - Settings.GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE = clusters_pool_namespace - - def setUp(self): - self.mock_kv_store() - - self.clusters = { - 'foo': { - 'pool': 'nfs-ganesha', - 'namespace': 'foo', - 'type': ClusterType.ORCHESTRATOR, - 'daemon_conf': 'conf-nfs.foo', - 'daemons': ['foo.host_a', 'foo.host_b'], - 'exports': { - 1: ['foo.host_a', 'foo.host_b'], - 2: ['foo.host_a', 'foo.host_b'], - 3: ['foo.host_a', 'foo.host_b'] # for new-added export - } - } - } - - # Unset user-defined location. - self._set_user_defined_clusters_location('') - - self.temp_store_namespace = None - self._reset_temp_store() - - self.io_mock = MagicMock() - self.io_mock.set_namespace.side_effect = self._ioctx_set_namespace_mock - self.io_mock.read = self._ioctl_read_mock - self.io_mock.stat = self._ioctl_stat_mock - self.io_mock.list_objects.side_effect = self._ioctx_list_objects_mock - self.io_mock.write_full.side_effect = self._ioctx_write_full_mock - self.io_mock.remove_object.side_effect = self._ioctx_remove_mock - - ioctx_mock = MagicMock() - ioctx_mock.__enter__ = Mock(return_value=(self.io_mock)) - ioctx_mock.__exit__ = Mock(return_value=None) - - mgr.rados = MagicMock() - mgr.rados.open_ioctx.return_value = ioctx_mock - - self._mock_orchestrator(True) - - ganesha.CephX = MagicMock() - ganesha.CephX.list_clients.return_value = ['ganesha'] - ganesha.CephX.get_client_key.return_value = 'ganesha' - - ganesha.CephFS = MagicMock() - - def _reset_temp_store(self): - self.temp_store_namespace = None - self.temp_store = { - 'bar': { - 'export-1': GaneshaConfTest.RObject("export-1", self.export_1), - 'export-2': GaneshaConfTest.RObject("export-2", self.export_2), - 'conf-nodea': GaneshaConfTest.RObject("conf-nodea", self.conf_nodea), - 'conf-nodeb': GaneshaConfTest.RObject("conf-nodeb", self.conf_nodeb), - }, - 'foo': { - 'export-1': GaneshaConfTest.RObject("export-1", self.export_1), - 'export-2': GaneshaConfTest.RObject("export-2", self.export_2), - 'conf-nfs.foo': GaneshaConfTest.RObject("conf-nfs.foo", self.conf_nfs_foo) - } - } - - def _mock_orchestrator(self, enable): - # mock nfs services - orch_nfs_services = [ - ServiceDescription(spec=NFSServiceSpec(service_id='foo')) - ] if enable else [] - # pylint: disable=protected-access - ganesha.Ganesha._get_orch_nfs_services = Mock(return_value=orch_nfs_services) - - # mock nfs daemons - def _get_nfs_instances(service_name=None): - if not enable: - return [] - instances = { - 'nfs.foo': [ - DaemonDescription(daemon_id='foo.host_a', status=1), - DaemonDescription(daemon_id='foo.host_b', status=1) - ], - 'nfs.bar': [ - DaemonDescription(daemon_id='bar.host_c', status=1) - ] - } - if service_name is not None: - return instances[service_name] - result = [] - for _, daemons in instances.items(): - result.extend(daemons) - return result - ganesha.GaneshaConfOrchestrator._get_orch_nfs_instances = Mock( - side_effect=_get_nfs_instances) - - def test_parse_daemon_raw_config(self): - expected_daemon_config = [ - { - "block_name": "NFS_CORE_PARAM", - "enable_nlm": False, - "enable_rquota": False, - "protocols": 4, - "nfs_port": 14000 - }, - { - "block_name": "MDCACHE", - "dir_chunk": 0 - }, - { - "block_name": "NFSV4", - "recoverybackend": "rados_cluster", - "minor_versions": [1, 2] - }, - { - "block_name": "RADOS_KV", - "pool": "nfs-ganesha", - "namespace": "vstart", - "userid": "vstart", - "nodeid": "a" - }, - { - "block_name": "RADOS_URLS", - "userid": "vstart", - "watch_url": "'rados://nfs-ganesha/vstart/conf-nfs.vstart'" - }, - { - "block_name": "%url", - "value": "rados://nfs-ganesha/vstart/conf-nfs.vstart" - } - ] - daemon_config = GaneshaConfParser(self.daemon_raw_config).parse() - self.assertEqual(daemon_config, expected_daemon_config) - - def test_export_parser_1(self): - blocks = GaneshaConfParser(self.export_1).parse() - self.assertIsInstance(blocks, list) - self.assertEqual(len(blocks), 1) - export = Export.from_export_block(blocks[0], '_default_', - GaneshaConf.ganesha_defaults({})) - - self.assertEqual(export.export_id, 1) - self.assertEqual(export.path, "/") - self.assertEqual(export.pseudo, "/cephfs_a") - self.assertIsNone(export.tag) - self.assertEqual(export.access_type, "RW") - self.assertEqual(export.squash, "root_squash") - self.assertEqual(export.protocols, {4}) - self.assertEqual(export.transports, {"TCP", "UDP"}) - self.assertEqual(export.fsal.name, "CEPH") - self.assertEqual(export.fsal.user_id, "ganesha") - self.assertEqual(export.fsal.fs_name, "a") - self.assertEqual(export.fsal.sec_label_xattr, None) - self.assertEqual(len(export.clients), 2) - self.assertEqual(export.clients[0].addresses, - ["192.168.0.10", "192.168.1.0/8"]) - self.assertEqual(export.clients[0].squash, "no_root_squash") - self.assertIsNone(export.clients[0].access_type) - self.assertEqual(export.clients[1].addresses, ["192.168.0.0/16"]) - self.assertEqual(export.clients[1].squash, "all_squash") - self.assertEqual(export.clients[1].access_type, "RO") - self.assertEqual(export.cluster_id, '_default_') - self.assertEqual(export.attr_expiration_time, 0) - self.assertEqual(export.security_label, False) - - def test_export_parser_2(self): - blocks = GaneshaConfParser(self.export_2).parse() - self.assertIsInstance(blocks, list) - self.assertEqual(len(blocks), 1) - export = Export.from_export_block(blocks[0], '_default_', - GaneshaConf.ganesha_defaults({})) - - self.assertEqual(export.export_id, 2) - self.assertEqual(export.path, "/") - self.assertEqual(export.pseudo, "/rgw") - self.assertIsNone(export.tag) - self.assertEqual(export.access_type, "RW") - self.assertEqual(export.squash, "all_squash") - self.assertEqual(export.protocols, {4, 3}) - self.assertEqual(export.transports, {"TCP", "UDP"}) - self.assertEqual(export.fsal.name, "RGW") - self.assertEqual(export.fsal.rgw_user_id, "testuser") - self.assertEqual(export.fsal.access_key, "access_key") - self.assertEqual(export.fsal.secret_key, "secret_key") - self.assertEqual(len(export.clients), 0) - self.assertEqual(export.cluster_id, '_default_') - - def test_daemon_conf_parser_a(self): - blocks = GaneshaConfParser(self.conf_nodea).parse() - self.assertIsInstance(blocks, list) - self.assertEqual(len(blocks), 2) - self.assertEqual(blocks[0]['block_name'], "%url") - self.assertEqual(blocks[0]['value'], "rados://nfs-ganesha/bar/export-2") - self.assertEqual(blocks[1]['block_name'], "%url") - self.assertEqual(blocks[1]['value'], "rados://nfs-ganesha/bar/export-1") - - def test_daemon_conf_parser_b(self): - blocks = GaneshaConfParser(self.conf_nodeb).parse() - self.assertIsInstance(blocks, list) - self.assertEqual(len(blocks), 1) - self.assertEqual(blocks[0]['block_name'], "%url") - self.assertEqual(blocks[0]['value'], "rados://nfs-ganesha/bar/export-1") - - def test_ganesha_conf(self): - for cluster_id, info in self.clusters.items(): - self._do_test_ganesha_conf(cluster_id, info['exports']) - self._reset_temp_store() - - def _do_test_ganesha_conf(self, cluster, expected_exports): - ganesha_conf = GaneshaConf.instance(cluster) - exports = ganesha_conf.exports - - self.assertEqual(len(exports.items()), 2) - self.assertIn(1, exports) - self.assertIn(2, exports) - - # export_id = 1 asserts - export = exports[1] - self.assertEqual(export.export_id, 1) - self.assertEqual(export.path, "/") - self.assertEqual(export.pseudo, "/cephfs_a") - self.assertIsNone(export.tag) - self.assertEqual(export.access_type, "RW") - self.assertEqual(export.squash, "root_squash") - self.assertEqual(export.protocols, {4}) - self.assertEqual(export.transports, {"TCP", "UDP"}) - self.assertEqual(export.fsal.name, "CEPH") - self.assertEqual(export.fsal.user_id, "ganesha") - self.assertEqual(export.fsal.fs_name, "a") - self.assertEqual(export.fsal.sec_label_xattr, None) - self.assertEqual(len(export.clients), 2) - self.assertEqual(export.clients[0].addresses, - ["192.168.0.10", "192.168.1.0/8"]) - self.assertEqual(export.clients[0].squash, "no_root_squash") - self.assertIsNone(export.clients[0].access_type) - self.assertEqual(export.clients[1].addresses, ["192.168.0.0/16"]) - self.assertEqual(export.clients[1].squash, "all_squash") - self.assertEqual(export.clients[1].access_type, "RO") - self.assertEqual(export.attr_expiration_time, 0) - self.assertEqual(export.security_label, False) - self.assertSetEqual(export.daemons, set(expected_exports[1])) - - # export_id = 2 asserts - export = exports[2] - self.assertEqual(export.export_id, 2) - self.assertEqual(export.path, "/") - self.assertEqual(export.pseudo, "/rgw") - self.assertIsNone(export.tag) - self.assertEqual(export.access_type, "RW") - self.assertEqual(export.squash, "all_squash") - self.assertEqual(export.protocols, {4, 3}) - self.assertEqual(export.transports, {"TCP", "UDP"}) - self.assertEqual(export.fsal.name, "RGW") - self.assertEqual(export.fsal.rgw_user_id, "testuser") - self.assertEqual(export.fsal.access_key, "access_key") - self.assertEqual(export.fsal.secret_key, "secret_key") - self.assertEqual(len(export.clients), 0) - self.assertSetEqual(export.daemons, set(expected_exports[2])) - - def test_config_dict(self): - for cluster_id, info in self.clusters.items(): - self._do_test_config_dict(cluster_id, info['exports']) - self._reset_temp_store() - - def _do_test_config_dict(self, cluster, expected_exports): - conf = GaneshaConf.instance(cluster) - export = conf.exports[1] - ex_dict = export.to_dict() - self.assertDictEqual(ex_dict, { - 'daemons': expected_exports[1], - 'export_id': 1, - 'path': '/', - 'pseudo': '/cephfs_a', - 'cluster_id': cluster, - 'tag': None, - 'access_type': 'RW', - 'squash': 'root_squash', - 'security_label': False, - 'protocols': [4], - 'transports': ['TCP', 'UDP'], - 'clients': [{ - 'addresses': ["192.168.0.10", "192.168.1.0/8"], - 'access_type': None, - 'squash': 'no_root_squash' - }, { - 'addresses': ["192.168.0.0/16"], - 'access_type': 'RO', - 'squash': 'all_squash' - }], - 'fsal': { - 'name': 'CEPH', - 'user_id': 'ganesha', - 'fs_name': 'a', - 'sec_label_xattr': None - } - }) - - export = conf.exports[2] - ex_dict = export.to_dict() - self.assertDictEqual(ex_dict, { - 'daemons': expected_exports[2], - 'export_id': 2, - 'path': '/', - 'pseudo': '/rgw', - 'cluster_id': cluster, - 'tag': None, - 'access_type': 'RW', - 'squash': 'all_squash', - 'security_label': False, - 'protocols': [3, 4], - 'transports': ['TCP', 'UDP'], - 'clients': [], - 'fsal': { - 'name': 'RGW', - 'rgw_user_id': 'testuser' - } - }) - - def test_config_from_dict(self): - for cluster_id, info in self.clusters.items(): - self._do_test_config_from_dict(cluster_id, info['exports']) - self._reset_temp_store() - - def _do_test_config_from_dict(self, cluster_id, expected_exports): - export = Export.from_dict(1, { - 'daemons': expected_exports[1], - 'export_id': 1, - 'path': '/', - 'cluster_id': cluster_id, - 'pseudo': '/cephfs_a', - 'tag': None, - 'access_type': 'RW', - 'squash': 'root_squash', - 'security_label': True, - 'protocols': [4], - 'transports': ['TCP', 'UDP'], - 'clients': [{ - 'addresses': ["192.168.0.10", "192.168.1.0/8"], - 'access_type': None, - 'squash': 'no_root_squash' - }, { - 'addresses': ["192.168.0.0/16"], - 'access_type': 'RO', - 'squash': 'all_squash' - }], - 'fsal': { - 'name': 'CEPH', - 'user_id': 'ganesha', - 'fs_name': 'a', - 'sec_label_xattr': 'security.selinux' - } - }) - - self.assertEqual(export.export_id, 1) - self.assertEqual(export.path, "/") - self.assertEqual(export.pseudo, "/cephfs_a") - self.assertIsNone(export.tag) - self.assertEqual(export.access_type, "RW") - self.assertEqual(export.squash, "root_squash") - self.assertEqual(export.protocols, {4}) - self.assertEqual(export.transports, {"TCP", "UDP"}) - self.assertEqual(export.fsal.name, "CEPH") - self.assertEqual(export.fsal.user_id, "ganesha") - self.assertEqual(export.fsal.fs_name, "a") - self.assertEqual(export.fsal.sec_label_xattr, 'security.selinux') - self.assertEqual(len(export.clients), 2) - self.assertEqual(export.clients[0].addresses, - ["192.168.0.10", "192.168.1.0/8"]) - self.assertEqual(export.clients[0].squash, "no_root_squash") - self.assertIsNone(export.clients[0].access_type) - self.assertEqual(export.clients[1].addresses, ["192.168.0.0/16"]) - self.assertEqual(export.clients[1].squash, "all_squash") - self.assertEqual(export.clients[1].access_type, "RO") - self.assertEqual(export.daemons, set(expected_exports[1])) - self.assertEqual(export.cluster_id, cluster_id) - self.assertEqual(export.attr_expiration_time, 0) - self.assertEqual(export.security_label, True) - - export = Export.from_dict(2, { - 'daemons': expected_exports[2], - 'export_id': 2, - 'path': '/', - 'pseudo': '/rgw', - 'cluster_id': cluster_id, - 'tag': None, - 'access_type': 'RW', - 'squash': 'all_squash', - 'security_label': False, - 'protocols': [4, 3], - 'transports': ['TCP', 'UDP'], - 'clients': [], - 'fsal': { - 'name': 'RGW', - 'rgw_user_id': 'testuser' - } - }) - - self.assertEqual(export.export_id, 2) - self.assertEqual(export.path, "/") - self.assertEqual(export.pseudo, "/rgw") - self.assertIsNone(export.tag) - self.assertEqual(export.access_type, "RW") - self.assertEqual(export.squash, "all_squash") - self.assertEqual(export.protocols, {4, 3}) - self.assertEqual(export.transports, {"TCP", "UDP"}) - self.assertEqual(export.fsal.name, "RGW") - self.assertEqual(export.fsal.rgw_user_id, "testuser") - self.assertIsNone(export.fsal.access_key) - self.assertIsNone(export.fsal.secret_key) - self.assertEqual(len(export.clients), 0) - self.assertEqual(export.daemons, set(expected_exports[2])) - self.assertEqual(export.cluster_id, cluster_id) - - def test_gen_raw_config(self): - for cluster_id, info in self.clusters.items(): - self._do_test_gen_raw_config(cluster_id, info['exports']) - self._reset_temp_store() - - def _do_test_gen_raw_config(self, cluster_id, expected_exports): - conf = GaneshaConf.instance(cluster_id) - # pylint: disable=W0212 - export = conf.exports[1] - del conf.exports[1] - conf._save_export(export) - conf = GaneshaConf.instance(cluster_id) - exports = conf.exports - self.assertEqual(len(exports.items()), 2) - self.assertIn(1, exports) - self.assertIn(2, exports) - - # export_id = 1 asserts - export = exports[1] - self.assertEqual(export.export_id, 1) - self.assertEqual(export.path, "/") - self.assertEqual(export.pseudo, "/cephfs_a") - self.assertIsNone(export.tag) - self.assertEqual(export.access_type, "RW") - self.assertEqual(export.squash, "root_squash") - self.assertEqual(export.protocols, {4}) - self.assertEqual(export.transports, {"TCP", "UDP"}) - self.assertEqual(export.fsal.name, "CEPH") - self.assertEqual(export.fsal.user_id, "ganesha") - self.assertEqual(export.fsal.fs_name, "a") - self.assertEqual(export.fsal.sec_label_xattr, None) - self.assertEqual(len(export.clients), 2) - self.assertEqual(export.clients[0].addresses, - ["192.168.0.10", "192.168.1.0/8"]) - self.assertEqual(export.clients[0].squash, "no_root_squash") - self.assertIsNone(export.clients[0].access_type) - self.assertEqual(export.clients[1].addresses, ["192.168.0.0/16"]) - self.assertEqual(export.clients[1].squash, "all_squash") - self.assertEqual(export.clients[1].access_type, "RO") - self.assertEqual(export.daemons, set(expected_exports[1])) - self.assertEqual(export.cluster_id, cluster_id) - self.assertEqual(export.attr_expiration_time, 0) - self.assertEqual(export.security_label, False) - - # export_id = 2 asserts - export = exports[2] - self.assertEqual(export.export_id, 2) - self.assertEqual(export.path, "/") - self.assertEqual(export.pseudo, "/rgw") - self.assertIsNone(export.tag) - self.assertEqual(export.access_type, "RW") - self.assertEqual(export.squash, "all_squash") - self.assertEqual(export.protocols, {4, 3}) - self.assertEqual(export.transports, {"TCP", "UDP"}) - self.assertEqual(export.fsal.name, "RGW") - self.assertEqual(export.fsal.rgw_user_id, "testuser") - self.assertEqual(export.fsal.access_key, "access_key") - self.assertEqual(export.fsal.secret_key, "secret_key") - self.assertEqual(len(export.clients), 0) - self.assertEqual(export.daemons, set(expected_exports[2])) - self.assertEqual(export.cluster_id, cluster_id) - - def test_update_export(self): - for cluster_id, info in self.clusters.items(): - self._do_test_update_export(cluster_id, info['exports']) - self._reset_temp_store() - - def _do_test_update_export(self, cluster_id, expected_exports): - ganesha.RgwClient = MagicMock() - admin_inst_mock = MagicMock() - admin_inst_mock.get_user_keys.return_value = { - 'access_key': 'access_key', - 'secret_key': 'secret_key' - } - ganesha.RgwClient.admin_instance.return_value = admin_inst_mock - - conf = GaneshaConf.instance(cluster_id) - conf.update_export({ - 'export_id': 2, - 'daemons': expected_exports[2], - 'path': 'bucket', - 'pseudo': '/rgw/bucket', - 'cluster_id': cluster_id, - 'tag': 'bucket_tag', - 'access_type': 'RW', - 'squash': 'all_squash', - 'security_label': False, - 'protocols': [4, 3], - 'transports': ['TCP', 'UDP'], - 'clients': [{ - 'addresses': ["192.168.0.0/16"], - 'access_type': None, - 'squash': None - }], - 'fsal': { - 'name': 'RGW', - 'rgw_user_id': 'testuser' - } - }) - - conf = GaneshaConf.instance(cluster_id) - export = conf.get_export(2) - self.assertEqual(export.export_id, 2) - self.assertEqual(export.path, "bucket") - self.assertEqual(export.pseudo, "/rgw/bucket") - self.assertEqual(export.tag, "bucket_tag") - self.assertEqual(export.access_type, "RW") - self.assertEqual(export.squash, "all_squash") - self.assertEqual(export.protocols, {4, 3}) - self.assertEqual(export.transports, {"TCP", "UDP"}) - self.assertEqual(export.fsal.name, "RGW") - self.assertEqual(export.fsal.rgw_user_id, "testuser") - self.assertEqual(export.fsal.access_key, "access_key") - self.assertEqual(export.fsal.secret_key, "secret_key") - self.assertEqual(len(export.clients), 1) - self.assertEqual(export.clients[0].addresses, ["192.168.0.0/16"]) - self.assertIsNone(export.clients[0].squash) - self.assertIsNone(export.clients[0].access_type) - self.assertEqual(export.daemons, set(expected_exports[2])) - self.assertEqual(export.cluster_id, cluster_id) - - def test_remove_export(self): - for cluster_id, info in self.clusters.items(): - self._do_test_remove_export(cluster_id, info['exports']) - self._reset_temp_store() - - def _do_test_remove_export(self, cluster_id, expected_exports): - conf = GaneshaConf.instance(cluster_id) - conf.remove_export(1) - exports = conf.list_exports() - self.assertEqual(len(exports), 1) - self.assertEqual(2, exports[0].export_id) - export = conf.get_export(2) - self.assertEqual(export.export_id, 2) - self.assertEqual(export.path, "/") - self.assertEqual(export.pseudo, "/rgw") - self.assertIsNone(export.tag) - self.assertEqual(export.access_type, "RW") - self.assertEqual(export.squash, "all_squash") - self.assertEqual(export.protocols, {4, 3}) - self.assertEqual(export.transports, {"TCP", "UDP"}) - self.assertEqual(export.fsal.name, "RGW") - self.assertEqual(export.fsal.rgw_user_id, "testuser") - self.assertEqual(export.fsal.access_key, "access_key") - self.assertEqual(export.fsal.secret_key, "secret_key") - self.assertEqual(len(export.clients), 0) - self.assertEqual(export.daemons, set(expected_exports[2])) - self.assertEqual(export.cluster_id, cluster_id) - - def test_create_export_rgw(self): - for cluster_id, info in self.clusters.items(): - self._do_test_create_export_rgw(cluster_id, info['exports']) - self._reset_temp_store() - - def _do_test_create_export_rgw(self, cluster_id, expected_exports): - ganesha.RgwClient = MagicMock() - admin_inst_mock = MagicMock() - admin_inst_mock.get_user_keys.return_value = { - 'access_key': 'access_key2', - 'secret_key': 'secret_key2' - } - ganesha.RgwClient.admin_instance.return_value = admin_inst_mock - - conf = GaneshaConf.instance(cluster_id) - ex_id = conf.create_export({ - 'daemons': expected_exports[3], - 'path': 'bucket', - 'pseudo': '/rgw/bucket', - 'tag': 'bucket_tag', - 'cluster_id': cluster_id, - 'access_type': 'RW', - 'squash': 'all_squash', - 'security_label': False, - 'protocols': [4, 3], - 'transports': ['TCP', 'UDP'], - 'clients': [{ - 'addresses': ["192.168.0.0/16"], - 'access_type': None, - 'squash': None - }], - 'fsal': { - 'name': 'RGW', - 'rgw_user_id': 'testuser' - } - }) - - conf = GaneshaConf.instance(cluster_id) - exports = conf.list_exports() - self.assertEqual(len(exports), 3) - export = conf.get_export(ex_id) - self.assertEqual(export.export_id, ex_id) - self.assertEqual(export.path, "bucket") - self.assertEqual(export.pseudo, "/rgw/bucket") - self.assertEqual(export.tag, "bucket_tag") - self.assertEqual(export.access_type, "RW") - self.assertEqual(export.squash, "all_squash") - self.assertEqual(export.protocols, {4, 3}) - self.assertEqual(export.transports, {"TCP", "UDP"}) - self.assertEqual(export.fsal.name, "RGW") - self.assertEqual(export.fsal.rgw_user_id, "testuser") - self.assertEqual(export.fsal.access_key, "access_key2") - self.assertEqual(export.fsal.secret_key, "secret_key2") - self.assertEqual(len(export.clients), 1) - self.assertEqual(export.clients[0].addresses, ["192.168.0.0/16"]) - self.assertIsNone(export.clients[0].squash) - self.assertIsNone(export.clients[0].access_type) - self.assertEqual(export.daemons, set(expected_exports[3])) - self.assertEqual(export.cluster_id, cluster_id) - - def test_create_export_cephfs(self): - for cluster_id, info in self.clusters.items(): - self._do_test_create_export_cephfs(cluster_id, info['exports']) - self._reset_temp_store() - - def _do_test_create_export_cephfs(self, cluster_id, expected_exports): - ganesha.CephX = MagicMock() - ganesha.CephX.list_clients.return_value = ["fs"] - ganesha.CephX.get_client_key.return_value = "fs_key" - - ganesha.CephFS = MagicMock() - ganesha.CephFS.dir_exists.return_value = True - - conf = GaneshaConf.instance(cluster_id) - ex_id = conf.create_export({ - 'daemons': expected_exports[3], - 'path': '/', - 'pseudo': '/cephfs2', - 'cluster_id': cluster_id, - 'tag': None, - 'access_type': 'RW', - 'squash': 'all_squash', - 'security_label': True, - 'protocols': [4], - 'transports': ['TCP'], - 'clients': [], - 'fsal': { - 'name': 'CEPH', - 'user_id': 'fs', - 'fs_name': None, - 'sec_label_xattr': 'security.selinux' - } - }) - - conf = GaneshaConf.instance(cluster_id) - exports = conf.list_exports() - self.assertEqual(len(exports), 3) - export = conf.get_export(ex_id) - self.assertEqual(export.export_id, ex_id) - self.assertEqual(export.path, "/") - self.assertEqual(export.pseudo, "/cephfs2") - self.assertIsNone(export.tag) - self.assertEqual(export.access_type, "RW") - self.assertEqual(export.squash, "all_squash") - self.assertEqual(export.protocols, {4}) - self.assertEqual(export.transports, {"TCP"}) - self.assertEqual(export.fsal.name, "CEPH") - self.assertEqual(export.fsal.user_id, "fs") - self.assertEqual(export.fsal.cephx_key, "fs_key") - self.assertEqual(export.fsal.sec_label_xattr, "security.selinux") - self.assertIsNone(export.fsal.fs_name) - self.assertEqual(len(export.clients), 0) - self.assertEqual(export.daemons, set(expected_exports[3])) - self.assertEqual(export.cluster_id, cluster_id) - self.assertEqual(export.attr_expiration_time, 0) - self.assertEqual(export.security_label, True) - - def test_reload_daemons(self): - # Fail to import call in Python 3.8, see https://bugs.python.org/issue35753 - mock_call = unittest.mock.call - - # Orchestrator cluster: reload all daemon config objects. - conf = GaneshaConf.instance('foo') - calls = [mock_call(conf) for conf in conf.list_daemon_confs()] - for daemons in [[], ['a', 'b']]: - conf.reload_daemons(daemons) - self.io_mock.notify.assert_has_calls(calls) - self.io_mock.reset_mock() - - # User-defined cluster: reload daemons in the parameter - self._set_user_defined_clusters_location() - conf = GaneshaConf.instance('_default_') - calls = [mock_call('conf-{}'.format(daemon)) for daemon in ['nodea', 'nodeb']] - conf.reload_daemons(['nodea', 'nodeb']) - self.io_mock.notify.assert_has_calls(calls) - - def test_list_daemons(self): - for cluster_id, info in self.clusters.items(): - instance = GaneshaConf.instance(cluster_id) - daemons = instance.list_daemons() - for daemon in daemons: - self.assertEqual(daemon['cluster_id'], cluster_id) - self.assertEqual(daemon['cluster_type'], info['type']) - self.assertIn('daemon_id', daemon) - self.assertIn('status', daemon) - self.assertIn('status_desc', daemon) - self.assertEqual([daemon['daemon_id'] for daemon in daemons], info['daemons']) - - def test_validate_orchestrator(self): - cluster_id = 'foo' - cluster_info = self.clusters[cluster_id] - instance = GaneshaConf.instance(cluster_id) - export = MagicMock() - - # export can be linked to none or all daemons - export_daemons = [[], cluster_info['daemons']] - for daemons in export_daemons: - export.daemons = daemons - instance.validate(export) - - # raise if linking to partial or non-exist daemons - export_daemons = [cluster_info['daemons'][:1], 'xxx'] - for daemons in export_daemons: - with self.assertRaises(NFSException): - export.daemons = daemons - instance.validate(export) - - def test_validate_user(self): - self._set_user_defined_clusters_location() - cluster_id = '_default_' - instance = GaneshaConf.instance(cluster_id) - export = MagicMock() - - # export can be linked to none, partial, or all daemons - fake_daemons = ['nodea', 'nodeb'] - export_daemons = [[], fake_daemons[:1], fake_daemons] - for daemons in export_daemons: - export.daemons = daemons - instance.validate(export) - - # raise if linking to non-exist daemons - export_daemons = ['xxx'] - for daemons in export_daemons: - with self.assertRaises(NFSException): - export.daemons = daemons - instance.validate(export) - - def _verify_locations(self, locations, cluster_ids): - for cluster_id in cluster_ids: - self.assertIn(cluster_id, locations) - cluster = locations.pop(cluster_id) - self.assertDictEqual(cluster, {key: cluster[key] for key in [ - 'pool', 'namespace', 'type', 'daemon_conf']}) - self.assertDictEqual(locations, {}) - - def test_get_cluster_locations(self): - # pylint: disable=protected-access - - # There is only a Orchestrator cluster. - self._mock_orchestrator(True) - locations = ganesha.Ganesha._get_clusters_locations() - self._verify_locations(locations, ['foo']) - - # No cluster. - self._mock_orchestrator(False) - with self.assertRaises(NFSException): - ganesha.Ganesha._get_clusters_locations() - - # There is only a user-defined cluster. - self._set_user_defined_clusters_location() - self._mock_orchestrator(False) - locations = ganesha.Ganesha._get_clusters_locations() - self._verify_locations(locations, ['_default_']) - - # There are both Orchestrator cluster and user-defined cluster. - self._set_user_defined_clusters_location() - self._mock_orchestrator(True) - locations = ganesha.Ganesha._get_clusters_locations() - self._verify_locations(locations, ['foo', '_default_']) - - def test_get_cluster_locations_conflict(self): - # pylint: disable=protected-access - - # Pool/namespace collision. - self._set_user_defined_clusters_location('nfs-ganesha/foo') - with self.assertRaises(NFSException) as ctx: - ganesha.Ganesha._get_clusters_locations() - self.assertIn('already in use', str(ctx.exception)) - - # Cluster name collision with orch. cluster. - self._set_user_defined_clusters_location('foo:nfs-ganesha/bar') - with self.assertRaises(NFSException) as ctx: - ganesha.Ganesha._get_clusters_locations() - self.assertIn('Detected a conflicting NFS-Ganesha cluster', str(ctx.exception)) - - # Cluster name collision with user-defined cluster. - self._set_user_defined_clusters_location( - 'cluster1:nfs-ganesha/bar,cluster1:fake-pool/fake-ns' - ) - with self.assertRaises(NFSException) as ctx: - ganesha.Ganesha._get_clusters_locations() - self.assertIn('Duplicate Ganesha cluster definition', str(ctx.exception)) class NFSGaneshaUiControllerTest(ControllerTestCase): diff --git a/src/pybind/mgr/nfs/cluster.py b/src/pybind/mgr/nfs/cluster.py index 4ccf37d2b0572..9989d4bc8785e 100644 --- a/src/pybind/mgr/nfs/cluster.py +++ b/src/pybind/mgr/nfs/cluster.py @@ -149,6 +149,20 @@ def list_nfs_cluster(self) -> Tuple[int, str, str]: except Exception as e: return exception_handler(e, "Failed to list NFS Cluster") + def list_daemons(self): + completion = self.mgr.list_daemons(daemon_type='nfs') + # Here completion.result is a list DaemonDescription objects + daemons = orchestrator.raise_if_exception(completion) + return [ + { + 'cluster_id': instance.service_id(), + 'daemon_id': instance.daemon_id, + 'cluster_type': 'orchestrator', + 'status': instance.status, + 'status_desc': instance.status_desc + } for instance in daemons + ] + def _show_nfs_cluster_info(self, cluster_id: str) -> Dict[str, Any]: completion = self.mgr.list_daemons(daemon_type='nfs') # Here completion.result is a list DaemonDescription objects diff --git a/src/pybind/mgr/nfs/export.py b/src/pybind/mgr/nfs/export.py index 59c94a4dc0ab9..ec79274c2bc0d 100644 --- a/src/pybind/mgr/nfs/export.py +++ b/src/pybind/mgr/nfs/export.py @@ -172,6 +172,20 @@ def _fetch_export( log.info('no exports for cluster %s', cluster_id) return None + def _fetch_export_id( + self, + cluster_id: str, + export_id: int + ) -> Optional[Export]: + try: + for ex in self.exports[cluster_id]: + if ex.export_id == export_id: + return ex + return None + except KeyError: + log.info(f'no exports for cluster {cluster_id}') + return None + def _delete_export_user(self, export: Export) -> None: if isinstance(export.fsal, CephFSFSAL): assert export.fsal.user_id @@ -364,6 +378,12 @@ def delete_all_exports(self, cluster_id: str) -> None: raise NFSException(f"Failed to delete exports: {err} and {ret}") log.info("All exports successfully deleted for cluster id: %s", cluster_id) + def list_all_exports(self): + r = [] + for cluster_id, ls in self.exports.items(): + r.extend([e.to_dict() for e in ls]) + return r + @export_cluster_checker def list_exports(self, cluster_id: str, @@ -397,6 +417,14 @@ def get_export( except Exception as e: return exception_handler(e, f"Failed to get {pseudo_path} export for {cluster_id}") + def get_export_by_id( + self, + cluster_id: str, + export_id: int + ) -> Dict[Any, Any]: + export = self._fetch_export_id(cluster_id, export_id) + return export.to_dict() if export else None + def apply_export(self, cluster_id: str, export_config: str) -> Tuple[int, str, str]: try: if not export_config: @@ -420,9 +448,9 @@ def apply_export(self, cluster_id: str, export_config: str) -> Tuple[int, str, s ret, out, err = (0, '', '') for export in j: try: - r, o, e = self._apply_export(cluster_id, export) + r, o, e, ex = self._apply_export(cluster_id, export) except Exception as ex: - r, o, e = exception_handler(ex, f'Failed to apply export: {ex}') + r, o, e, ex = exception_handler(ex, f'Failed to apply export: {ex}') if r: ret = r if o: @@ -431,7 +459,8 @@ def apply_export(self, cluster_id: str, export_config: str) -> Tuple[int, str, s err += e + '\n' return ret, out, err else: - return self._apply_export(cluster_id, j) + r, o, e, ex = self._apply_export(cluster_id, j) + return r, o, e except NotImplementedError: return 0, " Manual Restart of NFS PODS required for successful update of exports", "" except Exception as e: @@ -623,7 +652,7 @@ def _apply_export( self, cluster_id: str, new_export_dict: Dict, - ) -> Tuple[int, str, str]: + ) -> Tuple[int, str, str, Export]: for k in ['path', 'pseudo']: if k not in new_export_dict: raise NFSInvalidOperation(f'Export missing required field {k}') @@ -661,7 +690,7 @@ def _apply_export( if not old_export: self._create_export_user(new_export) self._save_export(cluster_id, new_export) - return 0, f'Added export {new_export.pseudo}', '' + return 0, f'Added export {new_export.pseudo}', '', new_export if old_export.fsal.name != new_export.fsal.name: raise NFSInvalidOperation('FSAL change not allowed') @@ -706,4 +735,4 @@ def _apply_export( # TODO: detect whether the update is such that a reload is sufficient restart_nfs_service(self.mgr, new_export.cluster_id) - return 0, f"Updated export {new_export.pseudo}", "" + return 0, f"Updated export {new_export.pseudo}", "", new_export diff --git a/src/pybind/mgr/nfs/module.py b/src/pybind/mgr/nfs/module.py index 6e97d48f83436..b6fff9c37d72f 100644 --- a/src/pybind/mgr/nfs/module.py +++ b/src/pybind/mgr/nfs/module.py @@ -1,13 +1,13 @@ import logging import threading -from typing import Tuple, Optional, List +from typing import Tuple, Optional, List, Dict, Any from mgr_module import MgrModule, CLICommand, Option, CLICheckNonemptyFileInput import orchestrator from .export import ExportMgr from .cluster import NFSCluster -from typing import Any +from .utils import available_clusters log = logging.getLogger(__name__) @@ -130,3 +130,37 @@ def _cmd_nfs_cluster_config_set(self, cluster_id: str, inbuf: str) -> Tuple[int, def _cmd_nfs_cluster_config_reset(self, cluster_id: str) -> Tuple[int, str, str]: """Reset NFS-Ganesha Config to default""" return self.nfs.reset_nfs_cluster_config(cluster_id=cluster_id) + + def is_active(self) -> bool: + return True + + def export_ls(self) -> List[Dict[Any, Any]]: + return self.export_mgr.list_all_exports() + + def export_get(self, cluster_id: str, export_id: int) -> Dict[Any, Any]: + return self.export_mgr.get_export_by_id(cluster_id, export_id) + + def export_rm(self, cluster_id: str, pseudo: str) -> None: + self.export_mgr.delete_export(cluster_id=cluster_id, pseudo_path=pseudo) + + def daemon_ls(self) -> List[Dict[Any, Any]]: + return self.nfs.list_daemons() + + def cluster_ls(self) -> List[str]: + return [ + { + 'pool': NFS_POOL_NAME, + 'namespace': cluster_id, + 'type': 'orchestrator', + 'daemon_conf': None, + } for cluster_id in available_clusters() + ] + + def cluster_fsals(self) -> List[str]: + return ['CEPH', 'RGW'] + + def export_apply(self, cluster_id: str, export: Dict[Any, Any]) -> Dict[Any, Any]: + ret, out, err, export = self.export_mgr._apply_export(cluster_id, export) + if ret: + return None + return export.to_dict() From 64178e1f5b0f26f5c17898e85c3d5b439e5f9dbf Mon Sep 17 00:00:00 2001 From: Varsha Rao Date: Tue, 10 Aug 2021 00:30:16 +0530 Subject: [PATCH 0197/2866] mgr/dashboard: directly use ExportMgr and NFSCluster objects Using the objects directly provides access to other methods and helps in avoiding repeatition. mgr/dashboard/nfsganesha: remove tag Since NFS v3 is no longer supported. We can remove tag. mgr/nfs: define global constant to list supported FSALs mgr/dashboard: directly list nfs clusters by directly importing available_cluster() method The current dashboard api returns a list of following dictionary { 'pool': 'nfs-ganesha', 'namespace': cluster_id, 'type': 'orchestrator', 'daemon_conf': None } None of these values are required for listing nfs cluster by mgr/nfs module. Instead directly list available cluster names mgr/dashboard: add comment to remove listing of daemons As the configs are per cluster. There is no need to list daemons per cluster. mgr/dashboard/controllers/nfsganesha: Add comments to update/remove status endpoint This endpoint can be updated in suggested way or even removed. As it was initially[1] introduced to check if dashboard pool and namespace configuration was set. [1] https://github.com/ceph/ceph/commit/824726393b185b8e5a8f17e66487dfde9f3c8b5c mgr/nfs: remove fetch_cluster_obj() There is no need to fetch NFSCluster class object. Directly available_clusters() can be imported to list nfs clusters. mgr/dashboard/controllers/nfsganesha: list exports based on cluster id As mgr/nfs module lists based on cluster id. mgr/dashboard/nfs: get and delete export by export id Fixes: https://tracker.ceph.com/issues/46493 Signed-off-by: Varsha Rao --- .../mgr/dashboard/controllers/nfsganesha.py | 102 +++++++++++++----- src/pybind/mgr/nfs/cluster.py | 1 + src/pybind/mgr/nfs/export.py | 42 +++++--- src/pybind/mgr/nfs/export_utils.py | 13 +-- src/pybind/mgr/nfs/module.py | 14 +-- 5 files changed, 109 insertions(+), 63 deletions(-) diff --git a/src/pybind/mgr/dashboard/controllers/nfsganesha.py b/src/pybind/mgr/dashboard/controllers/nfsganesha.py index 5571bbd00b5db..ae3c903c056f4 100644 --- a/src/pybind/mgr/dashboard/controllers/nfsganesha.py +++ b/src/pybind/mgr/dashboard/controllers/nfsganesha.py @@ -2,10 +2,15 @@ import logging import os +import json from functools import partial import cephfs import cherrypy +# Importing from nfs module throws Attribute Error +# https://gist.github.com/varshar16/61ac26426bbe5f5f562ebb14bcd0f548 +#from nfs.export_utils import NFS_GANESHA_SUPPORTED_FSALS +#from nfs.utils import available_clusters from .. import mgr from ..security import Scope @@ -23,6 +28,8 @@ class NFSException(DashboardException): def __init__(self, msg): super(NFSException, self).__init__(component="nfs", msg=msg) +# Remove this once attribute error is fixed +NFS_GANESHA_SUPPORTED_FSALS = ['CEPH', 'RGW'] # documentation helpers EXPORT_SCHEMA = { @@ -31,7 +38,6 @@ def __init__(self, msg): 'cluster_id': (str, 'Cluster identifier'), 'daemons': ([str], 'List of NFS Ganesha daemons identifiers'), 'pseudo': (str, 'Pseudo FS path'), - 'tag': (str, 'NFSv3 export tag'), 'access_type': (str, 'Export access type'), 'squash': (str, 'Export squash policy'), 'security_label': (str, 'Security label'), @@ -57,7 +63,6 @@ def __init__(self, msg): 'cluster_id': (str, 'Cluster identifier'), 'daemons': ([str], 'List of NFS Ganesha daemons identifiers'), 'pseudo': (str, 'Pseudo FS path'), - 'tag': (str, 'NFSv3 export tag'), 'access_type': (str, 'Export access type'), 'squash': (str, 'Export squash policy'), 'security_label': (str, 'Security label'), @@ -102,14 +107,19 @@ class NFSGanesha(RESTController): @Endpoint() @ReadPermission def status(self): - status = {'available': True, 'message': None} + ''' + FIXME: update this to check if any nfs cluster is available. Otherwise this endpoint can be safely removed too. + As it was introduced to check dashboard pool and namespace configuration. try: - mgr.remote('nfs', 'is_active') + cluster_ls = available_clusters(mgr) + if not cluster_ls: + raise NFSException('Please deploy a cluster using `nfs cluster create ... or orch apply nfs ..') except (NameError, ImportError) as e: status['message'] = str(e) # type: ignore status['available'] = False - return status + ''' + return {'available': True, 'message': None} @APIRouter('/nfs-ganesha/export', Scope.NFS_GANESHA) @@ -120,6 +130,14 @@ class NFSGaneshaExports(RESTController): @EndpointDoc("List all NFS-Ganesha exports", responses={200: [EXPORT_SCHEMA]}) def list(self): + ''' + list exports based on cluster_id ? + export_mgr = mgr.remote('nfs', 'fetch_nfs_export_obj') + ret, out, err = export_mgr.list_exports(cluster_id=cluster_id, detailed=True) + if ret == 0: + return json.loads(out) + raise NFSException(f"Failed to list exports: {err}") + ''' return mgr.remote('nfs', 'export_ls') @NfsTask('create', {'path': '{path}', 'fsal': '{fsal.name}', @@ -127,16 +145,10 @@ def list(self): @EndpointDoc("Creates a new NFS-Ganesha export", parameters=CREATE_EXPORT_SCHEMA, responses={201: EXPORT_SCHEMA}) - def create(self, path, cluster_id, daemons, pseudo, tag, access_type, + def create(self, path, cluster_id, daemons, pseudo, access_type, squash, security_label, protocols, transports, fsal, clients, reload_daemons=True): - if fsal['name'] not in mgr.remote('nfs', 'cluster_fsals'): - raise NFSException("Cannot create this export. " - "FSAL '{}' cannot be managed by the dashboard." - .format(fsal['name'])) - fsal.pop('user_id') # mgr/nfs does not let you customize user_id - # FIXME: what was this? 'tag': tag, raw_ex = { 'path': path, 'pseudo': pseudo, @@ -150,8 +162,11 @@ def create(self, path, cluster_id, daemons, pseudo, tag, access_type, 'fsal': fsal, 'clients': clients } - export = mgr.remote('nfs', 'export_apply', cluster_id, raw_ex) - return export + export_mgr = mgr.remote('nfs', 'fetch_nfs_export_obj') + ret, out, err = export_mgr.apply_export(cluster_id, json.dumps(raw_ex)) + if ret == 0: + return export_mgr._get_export_dict(cluster_id, pseudo) + raise NFSException(f"Export creation failed {err}") @EndpointDoc("Get an NFS-Ganesha export", parameters={ @@ -160,6 +175,15 @@ def create(self, path, cluster_id, daemons, pseudo, tag, access_type, }, responses={200: EXPORT_SCHEMA}) def get(self, cluster_id, export_id): + ''' + Get export by pseudo path? + export_mgr = mgr.remote('nfs', 'fetch_nfs_export_obj') + return export_mgr._get_export_dict(cluster_id, pseudo) + + Get export by id + export_mgr = mgr.remote('nfs', 'fetch_nfs_export_obj') + return export_mgr.get_export_by_id(cluster_id, export_id) + ''' return mgr.remote('nfs', 'export_get', cluster_id, export_id) @NfsTask('edit', {'cluster_id': '{cluster_id}', 'export_id': '{export_id}'}, @@ -168,21 +192,11 @@ def get(self, cluster_id, export_id): parameters=dict(export_id=(int, "Export ID"), **CREATE_EXPORT_SCHEMA), responses={200: EXPORT_SCHEMA}) - def set(self, cluster_id, export_id, path, daemons, pseudo, tag, access_type, + def set(self, cluster_id, export_id, path, daemons, pseudo, access_type, squash, security_label, protocols, transports, fsal, clients, reload_daemons=True): - export_id = int(export_id) - - if not mgr.remote('nfs', 'export_get', export_id): - raise cherrypy.HTTPError(404) # pragma: no cover - the handling is too obvious - - if fsal['name'] not in mgr.remote('nfs', 'cluster_fsals'): - raise NFSException("Cannot make modifications to this export. " - "FSAL '{}' cannot be managed by the dashboard." - .format(fsal['name'])) fsal.pop('user_id') # mgr/nfs does not let you customize user_id - # FIXME: what was this? 'tag': tag, raw_ex = { 'path': path, 'pseudo': pseudo, @@ -196,8 +210,12 @@ def set(self, cluster_id, export_id, path, daemons, pseudo, tag, access_type, 'fsal': fsal, 'clients': clients } - export = mgr.remote('nfs', 'export_apply', cluster_id, raw_ex) - return export + + export_mgr = mgr.remote('nfs', 'fetch_nfs_export_obj') + ret, out, err = export_mgr.apply_export(cluster_id, json.dumps(raw_ex)) + if ret == 0: + return export_mgr._get_export_dict(cluster_id, pseudo) + raise NFSException(f"Failed to update export: {err}") @NfsTask('delete', {'cluster_id': '{cluster_id}', 'export_id': '{export_id}'}, 2.0) @@ -211,6 +229,18 @@ def set(self, cluster_id, export_id, path, daemons, pseudo, tag, access_type, True) }) def delete(self, cluster_id, export_id, reload_daemons=True): + ''' + Delete by pseudo path + export_mgr = mgr.remote('nfs', 'fetch_nfs_export_obj') + export_mgr.delete_export(cluster_id, pseudo) + + if deleting by export id + export_mgr = mgr.remote('nfs', 'fetch_nfs_export_obj') + export = export_mgr.get_export_by_id(cluster_id, export_id) + ret, out, err = export_mgr.delete_export(cluster_id=cluster_id, pseudo_path=export['pseudo']) + if ret != 0: + raise NFSException(err) + ''' export_id = int(export_id) export = mgr.remote('nfs', 'export_get', cluster_id, export_id) @@ -219,6 +249,7 @@ def delete(self, cluster_id, export_id, reload_daemons=True): mgr.remote('nfs', 'export_rm', cluster_id, export['pseudo']) +# FIXME: remove this; dashboard should only care about clusters. @APIRouter('/nfs-ganesha/daemon', Scope.NFS_GANESHA) @APIDoc(group="NFS-Ganesha") class NFSGaneshaService(RESTController): @@ -232,7 +263,6 @@ class NFSGaneshaService(RESTController): 'desc': (str, 'Status description', True) }]}) def list(self): - # FIXME: remove this; dashboard should only care about clusters. return mgr.remote('nfs', 'daemon_ls') @@ -247,7 +277,7 @@ def cephx_clients(self): @Endpoint('GET', '/fsals') @ReadPermission def fsals(self): - return mgr.remote('nfs', 'cluster_fsals') + return NFS_GANESHA_SUPPORTED_FSALS @Endpoint('GET', '/lsdir') @ReadPermission @@ -301,4 +331,18 @@ def buckets(self, user_id=None): @Endpoint('GET', '/clusters') @ReadPermission def clusters(self): + ''' + Remove this remote call instead directly use available_cluster() method. It returns list of cluster names: ['vstart'] + The current dashboard api needs to changed from following to simply list of strings + [ + { + 'pool': 'nfs-ganesha', + 'namespace': cluster_id, + 'type': 'orchestrator', + 'daemon_conf': None + } for cluster_id in available_clusters() + ] + As pool, namespace, cluster type and daemon_conf are not required for listing cluster by mgr/nfs module + return available_cluster(mgr) + ''' return mgr.remote('nfs', 'cluster_ls') diff --git a/src/pybind/mgr/nfs/cluster.py b/src/pybind/mgr/nfs/cluster.py index 9989d4bc8785e..9902ee308a904 100644 --- a/src/pybind/mgr/nfs/cluster.py +++ b/src/pybind/mgr/nfs/cluster.py @@ -149,6 +149,7 @@ def list_nfs_cluster(self) -> Tuple[int, str, str]: except Exception as e: return exception_handler(e, "Failed to list NFS Cluster") + # FIXME: Remove this method. It was added for dashboard integration with mgr/nfs module. def list_daemons(self): completion = self.mgr.list_daemons(daemon_type='nfs') # Here completion.result is a list DaemonDescription objects diff --git a/src/pybind/mgr/nfs/export.py b/src/pybind/mgr/nfs/export.py index ec79274c2bc0d..9e24203f8dd8e 100644 --- a/src/pybind/mgr/nfs/export.py +++ b/src/pybind/mgr/nfs/export.py @@ -9,7 +9,8 @@ from mgr_module import NFS_POOL_NAME as POOL_NAME -from .export_utils import GaneshaConfParser, Export, RawBlock, CephFSFSAL, RGWFSAL +from .export_utils import GaneshaConfParser, Export, RawBlock, CephFSFSAL, RGWFSAL, \ + NFS_GANESHA_SUPPORTED_FSALS from .exception import NFSException, NFSInvalidOperation, FSNotFound, \ ClusterNotFound from .utils import available_clusters, check_fs, restart_nfs_service @@ -402,6 +403,12 @@ def list_exports(self, except Exception as e: return exception_handler(e, f"Failed to list exports for {cluster_id}") + def _get_export_dict(self, cluster_id: str, pseudo_path: str) -> Optional[Dict[str, Any]]: + export = self._fetch_export(cluster_id, pseudo_path) + if export: + return export.to_dict() + log.warning(f"No {pseudo_path} export to show for {cluster_id}") + @export_cluster_checker def get_export( self, @@ -409,9 +416,9 @@ def get_export( pseudo_path: str, ) -> Tuple[int, str, str]: try: - export = self._fetch_export(cluster_id, pseudo_path) - if export: - return 0, json.dumps(export.to_dict(), indent=2), '' + export_dict = self._get_export_dict(cluster_id, pseudo_path) + if export_dict: + return 0, json.dumps(export_dict, indent=2), '' log.warning("No %s export to show for %s", pseudo_path, cluster_id) return 0, '', '' except Exception as e: @@ -448,9 +455,9 @@ def apply_export(self, cluster_id: str, export_config: str) -> Tuple[int, str, s ret, out, err = (0, '', '') for export in j: try: - r, o, e, ex = self._apply_export(cluster_id, export) + r, o, e = self._apply_export(cluster_id, export) except Exception as ex: - r, o, e, ex = exception_handler(ex, f'Failed to apply export: {ex}') + r, o, e = exception_handler(ex, f'Failed to apply export: {ex}') if r: ret = r if o: @@ -459,7 +466,7 @@ def apply_export(self, cluster_id: str, export_config: str) -> Tuple[int, str, s err += e + '\n' return ret, out, err else: - r, o, e, ex = self._apply_export(cluster_id, j) + r, o, e = self._apply_export(cluster_id, j) return r, o, e except NotImplementedError: return 0, " Manual Restart of NFS PODS required for successful update of exports", "" @@ -546,13 +553,13 @@ def create_export_from_dict(self, fsal = ex_dict.get("fsal", {}) fsal_type = fsal.get("name") - if fsal_type == 'RGW': + if fsal_type == NFS_GANESHA_SUPPORTED_FSALS[1]: if '/' in path: raise NFSInvalidOperation('"/" is not allowed in path (bucket name)') uid = f'nfs.{cluster_id}.{path}' if "user_id" in fsal and fsal["user_id"] != uid: raise NFSInvalidOperation(f"export FSAL user_id must be '{uid}'") - elif fsal_type == 'CEPH': + elif fsal_type == NFS_GANESHA_SUPPORTED_FSALS[0]: fs_name = fsal.get("fs_name") if not fs_name: raise NFSInvalidOperation("export FSAL must specify fs_name") @@ -563,7 +570,8 @@ def create_export_from_dict(self, if "user_id" in fsal and fsal["user_id"] != user_id: raise NFSInvalidOperation(f"export FSAL user_id must be '{user_id}'") else: - raise NFSInvalidOperation("export must specify FSAL name of 'CEPH' or 'RGW'") + raise NFSInvalidOperation(f"NFS Ganesha supported FSALs are {NFS_GANESHA_SUPPORTED_FSALS}." + "Export must specify any one of it.") ex_dict["fsal"] = fsal ex_dict["cluster_id"] = cluster_id @@ -593,7 +601,7 @@ def create_cephfs_export(self, "access_type": access_type, "squash": squash, "fsal": { - "name": "CEPH", + "name": NFS_GANESHA_SUPPORTED_FSALS[0], "fs_name": fs_name, }, "clients": clients, @@ -631,7 +639,7 @@ def create_rgw_export(self, "path": bucket, "access_type": access_type, "squash": squash, - "fsal": {"name": "RGW"}, + "fsal": {"name": NFS_GANESHA_SUPPORTED_FSALS[1]}, "clients": clients, } ) @@ -652,7 +660,7 @@ def _apply_export( self, cluster_id: str, new_export_dict: Dict, - ) -> Tuple[int, str, str, Export]: + ) -> Tuple[int, str, str]: for k in ['path', 'pseudo']: if k not in new_export_dict: raise NFSInvalidOperation(f'Export missing required field {k}') @@ -690,7 +698,7 @@ def _apply_export( if not old_export: self._create_export_user(new_export) self._save_export(cluster_id, new_export) - return 0, f'Added export {new_export.pseudo}', '', new_export + return 0, f'Added export {new_export.pseudo}', '' if old_export.fsal.name != new_export.fsal.name: raise NFSInvalidOperation('FSAL change not allowed') @@ -698,7 +706,7 @@ def _apply_export( log.debug('export %s pseudo %s -> %s', new_export.export_id, old_export.pseudo, new_export.pseudo) - if old_export.fsal.name == 'CEPH': + if old_export.fsal.name == NFS_GANESHA_SUPPORTED_FSALS[0]: old_fsal = cast(CephFSFSAL, old_export.fsal) new_fsal = cast(CephFSFSAL, new_export.fsal) if old_fsal.user_id != new_fsal.user_id: @@ -718,7 +726,7 @@ def _apply_export( new_fsal.cephx_key = old_fsal.cephx_key else: new_fsal.cephx_key = old_fsal.cephx_key - if old_export.fsal.name == 'RGW': + if old_export.fsal.name == NFS_GANESHA_SUPPORTED_FSALS[1]: old_rgw_fsal = cast(RGWFSAL, old_export.fsal) new_rgw_fsal = cast(RGWFSAL, new_export.fsal) if old_rgw_fsal.user_id != new_rgw_fsal.user_id: @@ -735,4 +743,4 @@ def _apply_export( # TODO: detect whether the update is such that a reload is sufficient restart_nfs_service(self.mgr, new_export.cluster_id) - return 0, f"Updated export {new_export.pseudo}", "", new_export + return 0, f"Updated export {new_export.pseudo}", "" diff --git a/src/pybind/mgr/nfs/export_utils.py b/src/pybind/mgr/nfs/export_utils.py index b035b242beff8..6967108fae4c8 100644 --- a/src/pybind/mgr/nfs/export_utils.py +++ b/src/pybind/mgr/nfs/export_utils.py @@ -7,6 +7,7 @@ if TYPE_CHECKING: from nfs.module import Module +NFS_GANESHA_SUPPORTED_FSALS = ['CEPH', 'RGW'] class RawBlock(): def __init__(self, block_name: str, blocks: List['RawBlock'] = [], values: Dict[str, Any] = {}): @@ -182,17 +183,17 @@ def __init__(self, name: str) -> None: @classmethod def from_dict(cls, fsal_dict: Dict[str, Any]) -> 'FSAL': - if fsal_dict.get('name') == 'CEPH': + if fsal_dict.get('name') == NFS_GANESHA_SUPPORTED_FSALS[0]: return CephFSFSAL.from_dict(fsal_dict) - if fsal_dict.get('name') == 'RGW': + if fsal_dict.get('name') == NFS_GANESHA_SUPPORTED_FSALS[1]: return RGWFSAL.from_dict(fsal_dict) raise NFSInvalidOperation(f'Unknown FSAL {fsal_dict.get("name")}') @classmethod def from_fsal_block(cls, fsal_block: RawBlock) -> 'FSAL': - if fsal_block.values.get('name') == 'CEPH': + if fsal_block.values.get('name') == NFS_GANESHA_SUPPORTED_FSALS[0]: return CephFSFSAL.from_fsal_block(fsal_block) - if fsal_block.values.get('name') == 'RGW': + if fsal_block.values.get('name') == NFS_GANESHA_SUPPORTED_FSALS[1]: return RGWFSAL.from_fsal_block(fsal_block) raise NFSInvalidOperation(f'Unknown FSAL {fsal_block.values.get("name")}') @@ -503,11 +504,11 @@ def validate(self, mgr: 'Module') -> None: if client.access_type: self.validate_access_type(client.access_type) - if self.fsal.name == 'CEPH': + if self.fsal.name == NFS_GANESHA_SUPPORTED_FSALS[0]: fs = cast(CephFSFSAL, self.fsal) if not fs.fs_name or not check_fs(mgr, fs.fs_name): raise FSNotFound(fs.fs_name) - elif self.fsal.name == 'RGW': + elif self.fsal.name == NFS_GANESHA_SUPPORTED_FSALS[1]: rgw = cast(RGWFSAL, self.fsal) # noqa pass else: diff --git a/src/pybind/mgr/nfs/module.py b/src/pybind/mgr/nfs/module.py index b6fff9c37d72f..8fc8558d09840 100644 --- a/src/pybind/mgr/nfs/module.py +++ b/src/pybind/mgr/nfs/module.py @@ -131,8 +131,8 @@ def _cmd_nfs_cluster_config_reset(self, cluster_id: str) -> Tuple[int, str, str] """Reset NFS-Ganesha Config to default""" return self.nfs.reset_nfs_cluster_config(cluster_id=cluster_id) - def is_active(self) -> bool: - return True + def fetch_nfs_export_obj(self): + return self.export_mgr def export_ls(self) -> List[Dict[Any, Any]]: return self.export_mgr.list_all_exports() @@ -146,6 +146,7 @@ def export_rm(self, cluster_id: str, pseudo: str) -> None: def daemon_ls(self) -> List[Dict[Any, Any]]: return self.nfs.list_daemons() + # Remove this method after fixing attribute error def cluster_ls(self) -> List[str]: return [ { @@ -155,12 +156,3 @@ def cluster_ls(self) -> List[str]: 'daemon_conf': None, } for cluster_id in available_clusters() ] - - def cluster_fsals(self) -> List[str]: - return ['CEPH', 'RGW'] - - def export_apply(self, cluster_id: str, export: Dict[Any, Any]) -> Dict[Any, Any]: - ret, out, err, export = self.export_mgr._apply_export(cluster_id, export) - if ret: - return None - return export.to_dict() From 023c32c99b2b05d640b943214f6b27f99bd3a032 Mon Sep 17 00:00:00 2001 From: Thomas Lamprecht Date: Mon, 18 Oct 2021 12:10:24 +0200 Subject: [PATCH 0198/2866] docs: fix case and spelling of Proxmox VE There's no such thing as ProxMox, the hypervisor product with build in ceph server management and RBD client access is named `Proxmox VE` and the company behind it is named Proxmox Server Solutions GmbH Signed-off-by: Thomas Lamprecht --- doc/glossary.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/glossary.rst b/doc/glossary.rst index 1460235649423..e473895762ea1 100644 --- a/doc/glossary.rst +++ b/doc/glossary.rst @@ -76,7 +76,7 @@ reflect either technical terms or legacy ways of referring to Ceph systems. Cloud Platforms Cloud Stacks Third party cloud provisioning platforms such as OpenStack, CloudStack, - OpenNebula, ProxMox, etc. + OpenNebula, Proxmox VE, etc. Object Storage Device OSD From 58a6ab2147c34d5b3f14bf48f9b47b03bea8a672 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alfonso=20Mart=C3=ADnez?= Date: Thu, 26 Aug 2021 12:05:54 +0200 Subject: [PATCH 0199/2866] mgr/dashboard: NFS exports: API + UI: integration with mgr/nfs; cleanups MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit mgr/dashboard: move NFS_GANESHA_SUPPORTED_FSALS to mgr_module.py Importing from nfs module throws AttributeError because as a side effect the dashboard module is impersonating the nfs module. https://gist.github.com/varshar16/61ac26426bbe5f5f562ebb14bcd0f548 mgr/dashboard: 'Create NFS export' form: list clusters from nfs module mgr/dashboard: frontend+backend cleanups for NFS export Removed all code and references related to daemons. UI cleanup and adopted unit-testing for nfs-epxort create form for CEPHFS backend. Cleanup for export list/get/create/set/delete endpoints. mgr/dashboard: rm set-ganesha ref + update docs Remove existing set-ganesha-clusters-rados-pool-namespace references as they are no longer required. Moreover, nfs doc in dashboard doc is updated accordingly to the current nfs status. mgr/dashboard: add nfs-export e2e test coverage mgr/dashboard: 'Create NFS export' form: remove RGW user id field. - Improve bucket typeahead behavior. - Increase version for bucket list endpoint. - Some refactoring. mgr/dashboard: 'Create NFS export' form: allow RGW backend only when default realm is selected. When RGW multisite is configured, the NFS module can only handle buckets in the default realm. mgr/dashboard: 'Create service' form: fix NFS service creation. After https://github.com/ceph/ceph/pull/42073, NFS pool and namespace are not customizable. mgr/dashboard: 'Create NFS export' form: add bucket validation. - Allow only existing buckets. - Refactoring: - Moved bucket validator from bucket form to cd-validators.ts - Split bucket validator into 2: bucket name validator and bucket existence (that checks either existence or non-existence). mgr/dashboard: 'Create NFS export' form: path validation refactor: allow only existing paths. Fixes: https://tracker.ceph.com/issues/46493 Fixes: https://tracker.ceph.com/issues/51479 Signed-off-by: Alfonso Martínez Signed-off-by: Avan Thakkar Signed-off-by: Pere Diaz Bou --- doc/mgr/dashboard.rst | 106 +----- qa/tasks/mgr/dashboard/test_ganesha.py | 208 ----------- qa/tasks/mgr/dashboard/test_rgw.py | 12 +- .../dashboard/ci/cephadm/bootstrap-cluster.sh | 5 +- .../mgr/dashboard/controllers/nfsganesha.py | 211 ++++------- src/pybind/mgr/dashboard/controllers/rgw.py | 15 +- .../integration/block/images.e2e-spec.ts | 4 +- .../integration/block/mirroring.e2e-spec.ts | 2 +- .../integration/cluster/logs.e2e-spec.ts | 2 +- .../integration/cluster/services.po.ts | 27 +- .../workflow/07-nfs-exports.e2e-spec.ts | 81 ++++ .../workflow/nfs/nfs-export.po.ts | 45 +++ .../cypress/integration/page-helper.po.ts | 7 +- .../integration/pools/pools.e2e-spec.ts | 8 +- .../cypress/integration/pools/pools.po.ts | 6 - .../service-form/service-form.component.html | 44 --- .../service-form.component.spec.ts | 18 +- .../service-form/service-form.component.ts | 14 +- .../src/app/ceph/nfs/models/nfs.fsal.ts | 5 + .../src/app/ceph/nfs/nfs-cluster-type.enum.ts | 4 - .../nfs-details/nfs-details.component.spec.ts | 19 +- .../nfs/nfs-details/nfs-details.component.ts | 3 +- .../nfs-form-client.component.html | 13 +- .../nfs-form-client.component.ts | 4 +- .../ceph/nfs/nfs-form/nfs-form.component.html | 257 ++++--------- .../ceph/nfs/nfs-form/nfs-form.component.scss | 8 + .../nfs/nfs-form/nfs-form.component.spec.ts | 152 +++----- .../ceph/nfs/nfs-form/nfs-form.component.ts | 352 +++++++----------- .../nfs/nfs-list/nfs-list.component.spec.ts | 4 - .../ceph/nfs/nfs-list/nfs-list.component.ts | 39 +- .../src/app/ceph/rgw/models/rgw-daemon.ts | 1 + .../rgw-bucket-form.component.html | 6 +- .../rgw-bucket-form.component.spec.ts | 88 +---- .../rgw-bucket-form.component.ts | 115 +----- .../rgw-bucket-list.component.ts | 2 +- .../rgw-daemon-list.component.ts | 3 +- .../src/app/shared/api/api-client.spec.ts | 11 + .../frontend/src/app/shared/api/api-client.ts | 5 + .../src/app/shared/api/host.service.ts | 9 +- .../src/app/shared/api/nfs.service.spec.ts | 28 +- .../src/app/shared/api/nfs.service.ts | 74 ++-- .../app/shared/api/rgw-bucket.service.spec.ts | 10 +- .../src/app/shared/api/rgw-bucket.service.ts | 19 +- .../src/app/shared/api/rgw-site.service.ts | 14 + .../app/shared/forms/cd-validators.spec.ts | 152 +++++++- .../src/app/shared/forms/cd-validators.ts | 119 +++++- src/pybind/mgr/dashboard/openapi.yaml | 183 ++------- .../mgr/dashboard/plugins/feature_toggles.py | 4 +- src/pybind/mgr/dashboard/services/cephx.py | 28 -- .../mgr/dashboard/services/rgw_client.py | 13 + .../mgr/dashboard/tests/test_ganesha.py | 36 +- src/pybind/mgr/dashboard/tests/test_rgw.py | 4 + src/pybind/mgr/mgr_module.py | 1 + src/pybind/mgr/nfs/cluster.py | 16 - src/pybind/mgr/nfs/export.py | 12 +- src/pybind/mgr/nfs/export_utils.py | 3 +- src/pybind/mgr/nfs/module.py | 17 +- src/pybind/mgr/nfs/tests/test_nfs.py | 18 +- src/vstart.sh | 4 - 59 files changed, 1045 insertions(+), 1625 deletions(-) delete mode 100644 qa/tasks/mgr/dashboard/test_ganesha.py create mode 100644 src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/07-nfs-exports.e2e-spec.ts create mode 100644 src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/nfs/nfs-export.po.ts create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/ceph/nfs/models/nfs.fsal.ts delete mode 100644 src/pybind/mgr/dashboard/frontend/src/app/ceph/nfs/nfs-cluster-type.enum.ts create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/shared/api/api-client.spec.ts create mode 100644 src/pybind/mgr/dashboard/frontend/src/app/shared/api/api-client.ts delete mode 100644 src/pybind/mgr/dashboard/services/cephx.py diff --git a/doc/mgr/dashboard.rst b/doc/mgr/dashboard.rst index 73764ea86df5b..9d9d1afba8cfa 100644 --- a/doc/mgr/dashboard.rst +++ b/doc/mgr/dashboard.rst @@ -1179,97 +1179,8 @@ A log entry may look like this:: NFS-Ganesha Management ---------------------- -Support for NFS-Ganesha Clusters Deployed by the Orchestrator -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The Ceph Dashboard can be used to manage NFS-Ganesha clusters deployed by the -Orchestrator and will detect them automatically. For more details -on deploying NFS-Ganesha clusters with the Orchestrator, please see: - -- Cephadm backend: :ref:`orchestrator-cli-stateless-services`. Or particularly, see - :ref:`deploy-cephadm-nfs-ganesha`. -- Rook backend: `Ceph NFS Gateway CRD `_. - -Support for NFS-Ganesha Clusters Defined by the User -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. note:: - - This configuration only applies for user-defined clusters, - NOT for Orchestrator-deployed clusters. - -The Ceph Dashboard can manage `NFS Ganesha `_ exports that use -CephFS or RGW as their backstore. - -To enable this feature in Ceph Dashboard there are some assumptions that need -to be met regarding the way NFS-Ganesha services are configured. - -The dashboard manages NFS-Ganesha config files stored in RADOS objects on the Ceph Cluster. -NFS-Ganesha must store part of their configuration in the Ceph cluster. - -These configuration files follow the below conventions. -Each export block must be stored in its own RADOS object named -``export-``, where ```` must match the ``Export_ID`` attribute of the -export configuration. Then, for each NFS-Ganesha service daemon there should -exist a RADOS object named ``conf-``, where ```` is an -arbitrary string that should uniquely identify the daemon instance (e.g., the -hostname where the daemon is running). -Each ``conf-`` object contains the RADOS URLs to the exports that -the NFS-Ganesha daemon should serve. These URLs are of the form:: - - %url rados://[/]/export- - -Both the ``conf-`` and ``export-`` objects must be stored in the -same RADOS pool/namespace. - - -Configuring NFS-Ganesha in the Dashboard -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -To enable management of NFS-Ganesha exports in the Ceph Dashboard, we -need to tell the Dashboard the RADOS pool and namespace in which -configuration objects are stored. The Ceph Dashboard can then access them -by following the naming convention described above. - -The Dashboard command to configure the NFS-Ganesha configuration objects -location is:: - - $ ceph dashboard set-ganesha-clusters-rados-pool-namespace [/] - -After running the above command, the Ceph Dashboard is able to find the NFS-Ganesha -configuration objects and we can manage exports through the Web UI. - -.. note:: - - A dedicated pool for the NFS shares should be used. Otherwise it can cause the - `known issue `_ with listing of shares - if the NFS objects are stored together with a lot of other objects in a single - pool. - - -Support for Multiple NFS-Ganesha Clusters -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The Ceph Dashboard also supports management of NFS-Ganesha exports belonging -to other NFS-Ganesha clusters. An NFS-Ganesha cluster is a group of -NFS-Ganesha service daemons sharing the same exports. NFS-Ganesha -clusters are independent and don't share the exports configuration among each -other. - -Each NFS-Ganesha cluster should store its configuration objects in a -unique RADOS pool/namespace to isolate the configuration. - -To specify the the configuration location of each NFS-Ganesha cluster we -can use the same command as above but with a different value pattern:: - - $ ceph dashboard set-ganesha-clusters-rados-pool-namespace :[/](,:[/])* - -The ```` is an arbitrary string that should uniquely identify the -NFS-Ganesha cluster. - -When configuring the Ceph Dashboard with multiple NFS-Ganesha clusters, the -Web UI will allow you to choose to which cluster an export belongs. - +The dashboard requires enabling the NFS module which will be used to manage +NFS clusters and NFS exports. For more information check :ref:`mgr-nfs`. Plug-ins -------- @@ -1462,9 +1373,9 @@ something like this:: Reporting issues from Dashboard """"""""""""""""""""""""""""""" -Ceph-Dashboard provides two ways to create an issue in the Ceph Issue Tracker, +Ceph-Dashboard provides two ways to create an issue in the Ceph Issue Tracker, either using the Ceph command line interface or by using the Ceph Dashboard -user interface. +user interface. To create an issue in the Ceph Issue Tracker, a user needs to have an account on the issue tracker. Under the ``my account`` tab in the Ceph Issue Tracker, @@ -1480,8 +1391,8 @@ Then on successful update, you can create an issue using: The available projects to create an issue on are: #. dashboard #. block -#. object -#. file_system +#. object +#. file_system #. ceph_manager #. orchestrator #. ceph_volume @@ -1491,11 +1402,10 @@ The available tracker types are: #. bug #. feature -The subject and description are then set by the user. +The subject and description are then set by the user. The user can also create an issue using the Dashboard user interface. The settings icon drop down menu on the top right of the navigation bar has the option to ``Raise an issue``. On clicking it, a modal dialog opens that has the option to -select the project and tracker from their respective drop down menus. The subject +select the project and tracker from their respective drop down menus. The subject and multiline description are added by the user. The user can then submit the issue. - diff --git a/qa/tasks/mgr/dashboard/test_ganesha.py b/qa/tasks/mgr/dashboard/test_ganesha.py deleted file mode 100644 index 6868e0cb32495..0000000000000 --- a/qa/tasks/mgr/dashboard/test_ganesha.py +++ /dev/null @@ -1,208 +0,0 @@ -# -*- coding: utf-8 -*- -# pylint: disable=too-many-public-methods - -from __future__ import absolute_import - -from .helper import DashboardTestCase, JList, JObj - - -class GaneshaTest(DashboardTestCase): - CEPHFS = True - AUTH_ROLES = ['pool-manager', 'ganesha-manager'] - - @classmethod - def setUpClass(cls): - super(GaneshaTest, cls).setUpClass() - cls.create_pool('ganesha', 2**2, 'replicated') - cls._rados_cmd(['-p', 'ganesha', '-N', 'ganesha1', 'create', 'conf-node1']) - cls._rados_cmd(['-p', 'ganesha', '-N', 'ganesha1', 'create', 'conf-node2']) - cls._rados_cmd(['-p', 'ganesha', '-N', 'ganesha1', 'create', 'conf-node3']) - cls._rados_cmd(['-p', 'ganesha', '-N', 'ganesha2', 'create', 'conf-node1']) - cls._rados_cmd(['-p', 'ganesha', '-N', 'ganesha2', 'create', 'conf-node2']) - cls._rados_cmd(['-p', 'ganesha', '-N', 'ganesha2', 'create', 'conf-node3']) - cls._ceph_cmd(['dashboard', 'set-ganesha-clusters-rados-pool-namespace', - 'cluster1:ganesha/ganesha1,cluster2:ganesha/ganesha2']) - - # RGW setup - cls._radosgw_admin_cmd([ - 'user', 'create', '--uid', 'admin', '--display-name', 'admin', - '--system', '--access-key', 'admin', '--secret', 'admin' - ]) - cls._ceph_cmd_with_secret(['dashboard', 'set-rgw-api-secret-key'], 'admin') - cls._ceph_cmd_with_secret(['dashboard', 'set-rgw-api-access-key'], 'admin') - - @classmethod - def tearDownClass(cls): - super(GaneshaTest, cls).tearDownClass() - cls._radosgw_admin_cmd(['user', 'rm', '--uid', 'admin', '--purge-data']) - cls._ceph_cmd(['osd', 'pool', 'delete', 'ganesha', 'ganesha', - '--yes-i-really-really-mean-it']) - - @DashboardTestCase.RunAs('test', 'test', [{'rbd-image': ['create', 'update', 'delete']}]) - def test_read_access_permissions(self): - self._get('/api/nfs-ganesha/export') - self.assertStatus(403) - - def test_list_daemons(self): - daemons = self._get("/api/nfs-ganesha/daemon") - self.assertEqual(len(daemons), 6) - daemons = [(d['daemon_id'], d['cluster_id']) for d in daemons] - self.assertIn(('node1', 'cluster1'), daemons) - self.assertIn(('node2', 'cluster1'), daemons) - self.assertIn(('node3', 'cluster1'), daemons) - self.assertIn(('node1', 'cluster2'), daemons) - self.assertIn(('node2', 'cluster2'), daemons) - self.assertIn(('node3', 'cluster2'), daemons) - - @classmethod - def create_export(cls, path, cluster_id, daemons, fsal, sec_label_xattr=None): - if fsal == 'CEPH': - fsal = {"name": "CEPH", "user_id": "admin", "fs_name": None, - "sec_label_xattr": sec_label_xattr} - pseudo = "/cephfs{}".format(path) - else: - fsal = {"name": "RGW", "rgw_user_id": "admin"} - pseudo = "/rgw/{}".format(path if path[0] != '/' else "") - ex_json = { - "path": path, - "fsal": fsal, - "cluster_id": cluster_id, - "daemons": daemons, - "pseudo": pseudo, - "tag": None, - "access_type": "RW", - "squash": "no_root_squash", - "security_label": sec_label_xattr is not None, - "protocols": [4], - "transports": ["TCP"], - "clients": [{ - "addresses": ["10.0.0.0/8"], - "access_type": "RO", - "squash": "root" - }] - } - return cls._task_post('/api/nfs-ganesha/export', ex_json) - - def tearDown(self): - super(GaneshaTest, self).tearDown() - exports = self._get("/api/nfs-ganesha/export") - if self._resp.status_code != 200: - return - self.assertIsInstance(exports, list) - for exp in exports: - self._task_delete("/api/nfs-ganesha/export/{}/{}" - .format(exp['cluster_id'], exp['export_id'])) - - def _test_create_export(self, cephfs_path): - exports = self._get("/api/nfs-ganesha/export") - self.assertEqual(len(exports), 0) - - data = self.create_export(cephfs_path, 'cluster1', ['node1', 'node2'], 'CEPH', - "security.selinux") - - exports = self._get("/api/nfs-ganesha/export") - self.assertEqual(len(exports), 1) - self.assertDictEqual(exports[0], data) - return data - - def test_create_export(self): - self._test_create_export('/foo') - - def test_create_export_for_cephfs_root(self): - self._test_create_export('/') - - def test_update_export(self): - export = self._test_create_export('/foo') - export['access_type'] = 'RO' - export['daemons'] = ['node1', 'node3'] - export['security_label'] = True - data = self._task_put('/api/nfs-ganesha/export/{}/{}' - .format(export['cluster_id'], export['export_id']), - export) - exports = self._get("/api/nfs-ganesha/export") - self.assertEqual(len(exports), 1) - self.assertDictEqual(exports[0], data) - self.assertEqual(exports[0]['daemons'], ['node1', 'node3']) - self.assertEqual(exports[0]['security_label'], True) - - def test_delete_export(self): - export = self._test_create_export('/foo') - self._task_delete("/api/nfs-ganesha/export/{}/{}" - .format(export['cluster_id'], export['export_id'])) - self.assertStatus(204) - - def test_get_export(self): - exports = self._get("/api/nfs-ganesha/export") - self.assertEqual(len(exports), 0) - - data1 = self.create_export("/foo", 'cluster2', ['node1', 'node2'], 'CEPH') - data2 = self.create_export("mybucket", 'cluster2', ['node2', 'node3'], 'RGW') - - export1 = self._get("/api/nfs-ganesha/export/cluster2/1") - self.assertDictEqual(export1, data1) - - export2 = self._get("/api/nfs-ganesha/export/cluster2/2") - self.assertDictEqual(export2, data2) - - def test_invalid_status(self): - self._ceph_cmd(['dashboard', 'set-ganesha-clusters-rados-pool-namespace', '']) - - data = self._get('/api/nfs-ganesha/status') - self.assertStatus(200) - self.assertIn('available', data) - self.assertIn('message', data) - self.assertFalse(data['available']) - self.assertIn(("NFS-Ganesha cluster is not detected. " - "Please set the GANESHA_RADOS_POOL_NAMESPACE " - "setting or deploy an NFS-Ganesha cluster with the Orchestrator."), - data['message']) - - self._ceph_cmd(['dashboard', 'set-ganesha-clusters-rados-pool-namespace', - 'cluster1:ganesha/ganesha1,cluster2:ganesha/ganesha2']) - - def test_valid_status(self): - data = self._get('/api/nfs-ganesha/status') - self.assertStatus(200) - self.assertIn('available', data) - self.assertIn('message', data) - self.assertTrue(data['available']) - - def test_ganesha_fsals(self): - data = self._get('/ui-api/nfs-ganesha/fsals') - self.assertStatus(200) - self.assertIn('CEPH', data) - - def test_ganesha_filesystems(self): - data = self._get('/ui-api/nfs-ganesha/cephfs/filesystems') - self.assertStatus(200) - self.assertSchema(data, JList(JObj({ - 'id': int, - 'name': str - }))) - - def test_ganesha_lsdir(self): - fss = self._get('/ui-api/nfs-ganesha/cephfs/filesystems') - self.assertStatus(200) - for fs in fss: - data = self._get('/ui-api/nfs-ganesha/lsdir/{}'.format(fs['name'])) - self.assertStatus(200) - self.assertSchema(data, JObj({'paths': JList(str)})) - self.assertEqual(data['paths'][0], '/') - - def test_ganesha_buckets(self): - data = self._get('/ui-api/nfs-ganesha/rgw/buckets') - self.assertStatus(200) - schema = JList(str) - self.assertSchema(data, schema) - - def test_ganesha_clusters(self): - data = self._get('/ui-api/nfs-ganesha/clusters') - self.assertStatus(200) - schema = JList(str) - self.assertSchema(data, schema) - - def test_ganesha_cephx_clients(self): - data = self._get('/ui-api/nfs-ganesha/cephx/clients') - self.assertStatus(200) - schema = JList(str) - self.assertSchema(data, schema) diff --git a/qa/tasks/mgr/dashboard/test_rgw.py b/qa/tasks/mgr/dashboard/test_rgw.py index 1bfb995065968..dc972d3ed0a41 100644 --- a/qa/tasks/mgr/dashboard/test_rgw.py +++ b/qa/tasks/mgr/dashboard/test_rgw.py @@ -183,13 +183,13 @@ def test_all(self): self.assertEqual(data['tenant'], '') # List all buckets. - data = self._get('/api/rgw/bucket') + data = self._get('/api/rgw/bucket', version='1.1') self.assertStatus(200) self.assertEqual(len(data), 1) self.assertIn('teuth-test-bucket', data) # List all buckets with stats. - data = self._get('/api/rgw/bucket?stats=true') + data = self._get('/api/rgw/bucket?stats=true', version='1.1') self.assertStatus(200) self.assertEqual(len(data), 1) self.assertSchema(data[0], JObj(sub_elems={ @@ -203,7 +203,7 @@ def test_all(self): }, allow_unknown=True)) # List all buckets names without stats. - data = self._get('/api/rgw/bucket?stats=false') + data = self._get('/api/rgw/bucket?stats=false', version='1.1') self.assertStatus(200) self.assertEqual(data, ['teuth-test-bucket']) @@ -283,7 +283,7 @@ def test_all(self): # Delete the bucket. self._delete('/api/rgw/bucket/teuth-test-bucket') self.assertStatus(204) - data = self._get('/api/rgw/bucket') + data = self._get('/api/rgw/bucket', version='1.1') self.assertStatus(200) self.assertEqual(len(data), 0) @@ -306,7 +306,7 @@ def test_crud_w_tenant(self): self.assertIsNone(data) # List all buckets. - data = self._get('/api/rgw/bucket') + data = self._get('/api/rgw/bucket', version='1.1') self.assertStatus(200) self.assertEqual(len(data), 1) self.assertIn('testx/teuth-test-bucket', data) @@ -379,7 +379,7 @@ def _verify_tenant_bucket(bucket, tenant, uid): self._delete('/api/rgw/bucket/{}'.format( parse.quote_plus('testx/teuth-test-bucket'))) self.assertStatus(204) - data = self._get('/api/rgw/bucket') + data = self._get('/api/rgw/bucket', version='1.1') self.assertStatus(200) self.assertEqual(len(data), 0) diff --git a/src/pybind/mgr/dashboard/ci/cephadm/bootstrap-cluster.sh b/src/pybind/mgr/dashboard/ci/cephadm/bootstrap-cluster.sh index 10a060a9bceb6..2c451f7864c10 100755 --- a/src/pybind/mgr/dashboard/ci/cephadm/bootstrap-cluster.sh +++ b/src/pybind/mgr/dashboard/ci/cephadm/bootstrap-cluster.sh @@ -11,14 +11,15 @@ mon_ip=$(ifconfig eth0 | grep 'inet ' | awk '{ print $2}') cephadm bootstrap --mon-ip $mon_ip --initial-dashboard-password {{ admin_password }} --allow-fqdn-hostname --skip-monitoring-stack --dashboard-password-noupdate --shared_ceph_folder /mnt/{{ ceph_dev_folder }} fsid=$(cat /etc/ceph/ceph.conf | grep fsid | awk '{ print $3}') +cephadm_shell="cephadm shell --fsid ${fsid} -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring" {% for number in range(1, nodes) %} ssh-copy-id -f -i /etc/ceph/ceph.pub -o StrictHostKeyChecking=no root@{{ prefix }}-node-0{{ number }}.{{ domain }} {% if expanded_cluster is defined %} - cephadm shell --fsid $fsid -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring ceph orch host add {{ prefix }}-node-0{{ number }}.{{ domain }} + ${cephadm_shell} ceph orch host add {{ prefix }}-node-0{{ number }}.{{ domain }} {% endif %} {% endfor %} {% if expanded_cluster is defined %} - cephadm shell --fsid $fsid -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring ceph orch apply osd --all-available-devices + ${cephadm_shell} ceph orch apply osd --all-available-devices {% endif %} diff --git a/src/pybind/mgr/dashboard/controllers/nfsganesha.py b/src/pybind/mgr/dashboard/controllers/nfsganesha.py index ae3c903c056f4..c48b0667b8afb 100644 --- a/src/pybind/mgr/dashboard/controllers/nfsganesha.py +++ b/src/pybind/mgr/dashboard/controllers/nfsganesha.py @@ -1,25 +1,22 @@ # -*- coding: utf-8 -*- +import json import logging import os -import json from functools import partial +from typing import Any, Dict, List, Optional import cephfs import cherrypy -# Importing from nfs module throws Attribute Error -# https://gist.github.com/varshar16/61ac26426bbe5f5f562ebb14bcd0f548 -#from nfs.export_utils import NFS_GANESHA_SUPPORTED_FSALS -#from nfs.utils import available_clusters +from mgr_module import NFS_GANESHA_SUPPORTED_FSALS from .. import mgr from ..security import Scope from ..services.cephfs import CephFS from ..services.exception import DashboardException, serialize_dashboard_exception -from ..services.rgw_client import NoCredentialsException, \ - NoRgwDaemonsException, RequestException, RgwClient from . import APIDoc, APIRouter, BaseController, Endpoint, EndpointDoc, \ ReadPermission, RESTController, Task, UIRouter +from ._version import APIVersion logger = logging.getLogger('controllers.nfs') @@ -28,15 +25,12 @@ class NFSException(DashboardException): def __init__(self, msg): super(NFSException, self).__init__(component="nfs", msg=msg) -# Remove this once attribute error is fixed -NFS_GANESHA_SUPPORTED_FSALS = ['CEPH', 'RGW'] # documentation helpers EXPORT_SCHEMA = { 'export_id': (int, 'Export ID'), 'path': (str, 'Export path'), 'cluster_id': (str, 'Cluster identifier'), - 'daemons': ([str], 'List of NFS Ganesha daemons identifiers'), 'pseudo': (str, 'Pseudo FS path'), 'access_type': (str, 'Export access type'), 'squash': (str, 'Export squash policy'), @@ -45,10 +39,9 @@ def __init__(self, msg): 'transports': ([str], 'List of transport types'), 'fsal': ({ 'name': (str, 'name of FSAL'), - 'user_id': (str, 'CephX user id', True), - 'filesystem': (str, 'CephFS filesystem ID', True), + 'fs_name': (str, 'CephFS filesystem name', True), 'sec_label_xattr': (str, 'Name of xattr for security label', True), - 'rgw_user_id': (str, 'RGW user id', True) + 'user_id': (str, 'User id', True) }, 'FSAL configuration'), 'clients': ([{ 'addresses': ([str], 'list of IP addresses'), @@ -61,7 +54,6 @@ def __init__(self, msg): CREATE_EXPORT_SCHEMA = { 'path': (str, 'Export path'), 'cluster_id': (str, 'Cluster identifier'), - 'daemons': ([str], 'List of NFS Ganesha daemons identifiers'), 'pseudo': (str, 'Pseudo FS path'), 'access_type': (str, 'Export access type'), 'squash': (str, 'Export squash policy'), @@ -70,19 +62,14 @@ def __init__(self, msg): 'transports': ([str], 'List of transport types'), 'fsal': ({ 'name': (str, 'name of FSAL'), - 'user_id': (str, 'CephX user id', True), - 'filesystem': (str, 'CephFS filesystem ID', True), - 'sec_label_xattr': (str, 'Name of xattr for security label', True), - 'rgw_user_id': (str, 'RGW user id', True) + 'fs_name': (str, 'CephFS filesystem name', True), + 'sec_label_xattr': (str, 'Name of xattr for security label', True) }, 'FSAL configuration'), 'clients': ([{ 'addresses': ([str], 'list of IP addresses'), 'access_type': (str, 'Client access type'), 'squash': (str, 'Client squash policy') - }], 'List of client configurations'), - 'reload_daemons': (bool, - 'Trigger reload of NFS-Ganesha daemons configuration', - True) + }], 'List of client configurations') } @@ -96,7 +83,7 @@ def composed_decorator(func): @APIRouter('/nfs-ganesha', Scope.NFS_GANESHA) -@APIDoc("NFS-Ganesha Management API", "NFS-Ganesha") +@APIDoc("NFS-Ganesha Cluster Management API", "NFS-Ganesha") class NFSGanesha(RESTController): @EndpointDoc("Status of NFS-Ganesha management feature", @@ -107,19 +94,24 @@ class NFSGanesha(RESTController): @Endpoint() @ReadPermission def status(self): - ''' - FIXME: update this to check if any nfs cluster is available. Otherwise this endpoint can be safely removed too. - As it was introduced to check dashboard pool and namespace configuration. + status = {'available': True, 'message': None} try: - cluster_ls = available_clusters(mgr) - if not cluster_ls: - raise NFSException('Please deploy a cluster using `nfs cluster create ... or orch apply nfs ..') - except (NameError, ImportError) as e: - status['message'] = str(e) # type: ignore + mgr.remote('nfs', 'cluster_ls') + except ImportError as error: + logger.exception(error) status['available'] = False + status['message'] = str(error) # type: ignore + return status - ''' - return {'available': True, 'message': None} + + +@APIRouter('/nfs-ganesha/cluster', Scope.NFS_GANESHA) +@APIDoc(group="NFS-Ganesha") +class NFSGaneshaCluster(RESTController): + @ReadPermission + @RESTController.MethodMap(version=APIVersion.EXPERIMENTAL) + def list(self): + return mgr.remote('nfs', 'cluster_ls') @APIRouter('/nfs-ganesha/export', Scope.NFS_GANESHA) @@ -127,33 +119,43 @@ def status(self): class NFSGaneshaExports(RESTController): RESOURCE_ID = "cluster_id/export_id" + @staticmethod + def _get_schema_export(export: Dict[str, Any]) -> Dict[str, Any]: + """ + Method that avoids returning export info not exposed in the export schema + e.g., rgw user access/secret keys. + """ + schema_fsal_info = {} + for key in export['fsal'].keys(): + if key in EXPORT_SCHEMA['fsal'][0].keys(): # type: ignore + schema_fsal_info[key] = export['fsal'][key] + export['fsal'] = schema_fsal_info + return export + @EndpointDoc("List all NFS-Ganesha exports", responses={200: [EXPORT_SCHEMA]}) - def list(self): - ''' - list exports based on cluster_id ? - export_mgr = mgr.remote('nfs', 'fetch_nfs_export_obj') - ret, out, err = export_mgr.list_exports(cluster_id=cluster_id, detailed=True) - if ret == 0: - return json.loads(out) - raise NFSException(f"Failed to list exports: {err}") - ''' - return mgr.remote('nfs', 'export_ls') + def list(self) -> List[Dict[str, Any]]: + exports = [] + for export in mgr.remote('nfs', 'export_ls'): + exports.append(self._get_schema_export(export)) + + return exports @NfsTask('create', {'path': '{path}', 'fsal': '{fsal.name}', 'cluster_id': '{cluster_id}'}, 2.0) @EndpointDoc("Creates a new NFS-Ganesha export", parameters=CREATE_EXPORT_SCHEMA, responses={201: EXPORT_SCHEMA}) - def create(self, path, cluster_id, daemons, pseudo, access_type, - squash, security_label, protocols, transports, fsal, clients, - reload_daemons=True): - fsal.pop('user_id') # mgr/nfs does not let you customize user_id + @RESTController.MethodMap(version=APIVersion(2, 0)) # type: ignore + def create(self, path, cluster_id, pseudo, access_type, + squash, security_label, protocols, transports, fsal, clients) -> Dict[str, Any]: + + if hasattr(fsal, 'user_id'): + fsal.pop('user_id') # mgr/nfs does not let you customize user_id raw_ex = { 'path': path, 'pseudo': pseudo, 'cluster_id': cluster_id, - 'daemons': daemons, 'access_type': access_type, 'squash': squash, 'security_label': security_label, @@ -163,28 +165,25 @@ def create(self, path, cluster_id, daemons, pseudo, access_type, 'clients': clients } export_mgr = mgr.remote('nfs', 'fetch_nfs_export_obj') - ret, out, err = export_mgr.apply_export(cluster_id, json.dumps(raw_ex)) + ret, _, err = export_mgr.apply_export(cluster_id, json.dumps(raw_ex)) if ret == 0: - return export_mgr._get_export_dict(cluster_id, pseudo) + return self._get_schema_export( + export_mgr._get_export_dict(cluster_id, pseudo)) # pylint: disable=W0212 raise NFSException(f"Export creation failed {err}") @EndpointDoc("Get an NFS-Ganesha export", parameters={ 'cluster_id': (str, 'Cluster identifier'), - 'export_id': (int, "Export ID") + 'export_id': (str, "Export ID") }, responses={200: EXPORT_SCHEMA}) - def get(self, cluster_id, export_id): - ''' - Get export by pseudo path? - export_mgr = mgr.remote('nfs', 'fetch_nfs_export_obj') - return export_mgr._get_export_dict(cluster_id, pseudo) - - Get export by id - export_mgr = mgr.remote('nfs', 'fetch_nfs_export_obj') - return export_mgr.get_export_by_id(cluster_id, export_id) - ''' - return mgr.remote('nfs', 'export_get', cluster_id, export_id) + def get(self, cluster_id, export_id) -> Optional[Dict[str, Any]]: + export_id = int(export_id) + export = mgr.remote('nfs', 'export_get', cluster_id, export_id) + if export: + export = self._get_schema_export(export) + + return export @NfsTask('edit', {'cluster_id': '{cluster_id}', 'export_id': '{export_id}'}, 2.0) @@ -192,16 +191,17 @@ def get(self, cluster_id, export_id): parameters=dict(export_id=(int, "Export ID"), **CREATE_EXPORT_SCHEMA), responses={200: EXPORT_SCHEMA}) - def set(self, cluster_id, export_id, path, daemons, pseudo, access_type, - squash, security_label, protocols, transports, fsal, clients, - reload_daemons=True): + @RESTController.MethodMap(version=APIVersion(2, 0)) # type: ignore + def set(self, cluster_id, export_id, path, pseudo, access_type, + squash, security_label, protocols, transports, fsal, clients) -> Dict[str, Any]: - fsal.pop('user_id') # mgr/nfs does not let you customize user_id + if hasattr(fsal, 'user_id'): + fsal.pop('user_id') # mgr/nfs does not let you customize user_id raw_ex = { 'path': path, 'pseudo': pseudo, 'cluster_id': cluster_id, - 'daemons': daemons, + 'export_id': export_id, 'access_type': access_type, 'squash': squash, 'security_label': security_label, @@ -212,9 +212,10 @@ def set(self, cluster_id, export_id, path, daemons, pseudo, access_type, } export_mgr = mgr.remote('nfs', 'fetch_nfs_export_obj') - ret, out, err = export_mgr.apply_export(cluster_id, json.dumps(raw_ex)) + ret, _, err = export_mgr.apply_export(cluster_id, json.dumps(raw_ex)) if ret == 0: - return export_mgr._get_export_dict(cluster_id, pseudo) + return self._get_schema_export( + export_mgr._get_export_dict(cluster_id, pseudo)) # pylint: disable=W0212 raise NFSException(f"Failed to update export: {err}") @NfsTask('delete', {'cluster_id': '{cluster_id}', @@ -222,25 +223,10 @@ def set(self, cluster_id, export_id, path, daemons, pseudo, access_type, @EndpointDoc("Deletes an NFS-Ganesha export", parameters={ 'cluster_id': (str, 'Cluster identifier'), - 'export_id': (int, "Export ID"), - 'reload_daemons': (bool, - 'Trigger reload of NFS-Ganesha daemons' - ' configuration', - True) + 'export_id': (int, "Export ID") }) - def delete(self, cluster_id, export_id, reload_daemons=True): - ''' - Delete by pseudo path - export_mgr = mgr.remote('nfs', 'fetch_nfs_export_obj') - export_mgr.delete_export(cluster_id, pseudo) - - if deleting by export id - export_mgr = mgr.remote('nfs', 'fetch_nfs_export_obj') - export = export_mgr.get_export_by_id(cluster_id, export_id) - ret, out, err = export_mgr.delete_export(cluster_id=cluster_id, pseudo_path=export['pseudo']) - if ret != 0: - raise NFSException(err) - ''' + @RESTController.MethodMap(version=APIVersion(2, 0)) # type: ignore + def delete(self, cluster_id, export_id): export_id = int(export_id) export = mgr.remote('nfs', 'export_get', cluster_id, export_id) @@ -249,31 +235,8 @@ def delete(self, cluster_id, export_id, reload_daemons=True): mgr.remote('nfs', 'export_rm', cluster_id, export['pseudo']) -# FIXME: remove this; dashboard should only care about clusters. -@APIRouter('/nfs-ganesha/daemon', Scope.NFS_GANESHA) -@APIDoc(group="NFS-Ganesha") -class NFSGaneshaService(RESTController): - - @EndpointDoc("List NFS-Ganesha daemons information", - responses={200: [{ - 'daemon_id': (str, 'Daemon identifier'), - 'cluster_id': (str, 'Cluster identifier'), - 'cluster_type': (str, 'Cluster type'), # FIXME: remove this property - 'status': (int, 'Status of daemon', True), - 'desc': (str, 'Status description', True) - }]}) - def list(self): - return mgr.remote('nfs', 'daemon_ls') - - @UIRouter('/nfs-ganesha', Scope.NFS_GANESHA) class NFSGaneshaUi(BaseController): - @Endpoint('GET', '/cephx/clients') - @ReadPermission - def cephx_clients(self): - # FIXME: remove this; cephx users/creds are managed by mgr/nfs - return ['admin'] - @Endpoint('GET', '/fsals') @ReadPermission def fsals(self): @@ -318,31 +281,3 @@ def lsdir(self, fs_name, root_dir=None, depth=1): # pragma: no cover @ReadPermission def filesystems(self): return CephFS.list_filesystems() - - @Endpoint('GET', '/rgw/buckets') - @ReadPermission - def buckets(self, user_id=None): - try: - return RgwClient.instance(user_id).get_buckets() - except (DashboardException, NoCredentialsException, RequestException, - NoRgwDaemonsException): - return [] - - @Endpoint('GET', '/clusters') - @ReadPermission - def clusters(self): - ''' - Remove this remote call instead directly use available_cluster() method. It returns list of cluster names: ['vstart'] - The current dashboard api needs to changed from following to simply list of strings - [ - { - 'pool': 'nfs-ganesha', - 'namespace': cluster_id, - 'type': 'orchestrator', - 'daemon_conf': None - } for cluster_id in available_clusters() - ] - As pool, namespace, cluster type and daemon_conf are not required for listing cluster by mgr/nfs module - return available_cluster(mgr) - ''' - return mgr.remote('nfs', 'cluster_ls') diff --git a/src/pybind/mgr/dashboard/controllers/rgw.py b/src/pybind/mgr/dashboard/controllers/rgw.py index 640a531f749b2..713294d8f977a 100644 --- a/src/pybind/mgr/dashboard/controllers/rgw.py +++ b/src/pybind/mgr/dashboard/controllers/rgw.py @@ -14,9 +14,10 @@ from ..tools import json_str_to_object, str_to_bool from . import APIDoc, APIRouter, BaseController, Endpoint, EndpointDoc, \ ReadPermission, RESTController, allow_empty_body +from ._version import APIVersion try: - from typing import Any, List, Optional + from typing import Any, Dict, List, Optional, Union except ImportError: # pragma: no cover pass # Just for type checking @@ -100,6 +101,7 @@ def list(self) -> List[dict]: 'service_map_id': service['id'], 'version': metadata['ceph_version'], 'server_hostname': hostname, + 'realm_name': metadata['realm_name'], 'zonegroup_name': metadata['zonegroup_name'], 'zone_name': metadata['zone_name'], 'default': instance.daemon.name == metadata['id'] @@ -157,6 +159,8 @@ def list(self, query=None, daemon_name=None): return RgwClient.admin_instance(daemon_name=daemon_name).get_placement_targets() if query == 'realms': return RgwClient.admin_instance(daemon_name=daemon_name).get_realms() + if query == 'default-realm': + return RgwClient.admin_instance(daemon_name=daemon_name).get_default_realm() # @TODO: for multisite: by default, retrieve cluster topology/map. raise DashboardException(http_status_code=501, component='rgw', msg='Not Implemented') @@ -231,9 +235,12 @@ def get_s3_bucket_name(bucket_name, tenant=None): bucket_name = '{}:{}'.format(tenant, bucket_name) return bucket_name - def list(self, stats=False, daemon_name=None): - # type: (bool, Optional[str]) -> List[Any] - query_params = '?stats' if str_to_bool(stats) else '' + @RESTController.MethodMap(version=APIVersion(1, 1)) # type: ignore + def list(self, stats: bool = False, daemon_name: Optional[str] = None, + uid: Optional[str] = None) -> List[Union[str, Dict[str, Any]]]: + query_params = f'?stats={str_to_bool(stats)}' + if uid and uid.strip(): + query_params = f'{query_params}&uid={uid.strip()}' result = self.proxy(daemon_name, 'GET', 'bucket{}'.format(query_params)) if stats: diff --git a/src/pybind/mgr/dashboard/frontend/cypress/integration/block/images.e2e-spec.ts b/src/pybind/mgr/dashboard/frontend/cypress/integration/block/images.e2e-spec.ts index cf8832bb9bbc0..5c89359db790e 100644 --- a/src/pybind/mgr/dashboard/frontend/cypress/integration/block/images.e2e-spec.ts +++ b/src/pybind/mgr/dashboard/frontend/cypress/integration/block/images.e2e-spec.ts @@ -13,7 +13,7 @@ describe('Images page', () => { // Need pool for image testing pools.navigateTo('create'); pools.create(poolName, 8, 'rbd'); - pools.exist(poolName, true); + pools.existTableCell(poolName); }); after(() => { @@ -21,7 +21,7 @@ describe('Images page', () => { pools.navigateTo(); pools.delete(poolName); pools.navigateTo(); - pools.exist(poolName, false); + pools.existTableCell(poolName, false); }); beforeEach(() => { diff --git a/src/pybind/mgr/dashboard/frontend/cypress/integration/block/mirroring.e2e-spec.ts b/src/pybind/mgr/dashboard/frontend/cypress/integration/block/mirroring.e2e-spec.ts index ddee817e18ef1..120956579d8d1 100644 --- a/src/pybind/mgr/dashboard/frontend/cypress/integration/block/mirroring.e2e-spec.ts +++ b/src/pybind/mgr/dashboard/frontend/cypress/integration/block/mirroring.e2e-spec.ts @@ -32,7 +32,7 @@ describe('Mirroring page', () => { pools.navigateTo('create'); // Need pool for mirroring testing pools.create(poolName, 8, 'rbd'); pools.navigateTo(); - pools.exist(poolName, true); + pools.existTableCell(poolName, true); }); it('tests editing mode for pools', () => { diff --git a/src/pybind/mgr/dashboard/frontend/cypress/integration/cluster/logs.e2e-spec.ts b/src/pybind/mgr/dashboard/frontend/cypress/integration/cluster/logs.e2e-spec.ts index 731275e26d1c1..9868b89aedbc5 100644 --- a/src/pybind/mgr/dashboard/frontend/cypress/integration/cluster/logs.e2e-spec.ts +++ b/src/pybind/mgr/dashboard/frontend/cypress/integration/cluster/logs.e2e-spec.ts @@ -45,7 +45,7 @@ describe('Logs page', () => { pools.navigateTo('create'); pools.create(poolname, 8); pools.navigateTo(); - pools.exist(poolname, true); + pools.existTableCell(poolname, true); logs.checkAuditForPoolFunction(poolname, 'create', hour, minute); }); diff --git a/src/pybind/mgr/dashboard/frontend/cypress/integration/cluster/services.po.ts b/src/pybind/mgr/dashboard/frontend/cypress/integration/cluster/services.po.ts index 4265329db042a..457b759ead39f 100644 --- a/src/pybind/mgr/dashboard/frontend/cypress/integration/cluster/services.po.ts +++ b/src/pybind/mgr/dashboard/frontend/cypress/integration/cluster/services.po.ts @@ -40,15 +40,24 @@ export class ServicesPageHelper extends PageHelper { addService(serviceType: string, exist?: boolean, count = '1') { cy.get(`${this.pages.create.id}`).within(() => { this.selectServiceType(serviceType); - if (serviceType === 'rgw') { - cy.get('#service_id').type('foo'); - cy.get('#count').type(count); - } else if (serviceType === 'ingress') { - this.selectOption('backend_service', 'rgw.foo'); - cy.get('#service_id').should('have.value', 'rgw.foo'); - cy.get('#virtual_ip').type('192.168.20.1/24'); - cy.get('#frontend_port').type('8081'); - cy.get('#monitor_port').type('8082'); + switch (serviceType) { + case 'rgw': + cy.get('#service_id').type('foo'); + cy.get('#count').type(count); + break; + + case 'ingress': + this.selectOption('backend_service', 'rgw.foo'); + cy.get('#service_id').should('have.value', 'rgw.foo'); + cy.get('#virtual_ip').type('192.168.20.1/24'); + cy.get('#frontend_port').type('8081'); + cy.get('#monitor_port').type('8082'); + break; + + case 'nfs': + cy.get('#service_id').type('testnfs'); + cy.get('#count').type(count); + break; } cy.get('cd-submit-button').click(); diff --git a/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/07-nfs-exports.e2e-spec.ts b/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/07-nfs-exports.e2e-spec.ts new file mode 100644 index 0000000000000..2d92075298e19 --- /dev/null +++ b/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/07-nfs-exports.e2e-spec.ts @@ -0,0 +1,81 @@ +import { ServicesPageHelper } from 'cypress/integration/cluster/services.po'; +import { NFSPageHelper } from 'cypress/integration/orchestrator/workflow/nfs/nfs-export.po'; +import { BucketsPageHelper } from 'cypress/integration/rgw/buckets.po'; + +describe('nfsExport page', () => { + const nfsExport = new NFSPageHelper(); + const services = new ServicesPageHelper(); + const buckets = new BucketsPageHelper(); + const bucketName = 'e2e.nfs.bucket'; + // @TODO: uncomment this when a CephFS volume can be created through Dashboard. + // const fsPseudo = '/fsPseudo'; + const rgwPseudo = '/rgwPseudo'; + const editPseudo = '/editPseudo'; + const backends = ['CephFS', 'Object Gateway']; + const squash = 'no_root_squash'; + const client: object = { addresses: '192.168.0.10' }; + + beforeEach(() => { + cy.login(); + Cypress.Cookies.preserveOnce('token'); + nfsExport.navigateTo(); + }); + + describe('breadcrumb test', () => { + it('should open and show breadcrumb', () => { + nfsExport.expectBreadcrumbText('NFS'); + }); + }); + + describe('Create, edit and delete', () => { + it('should create an NFS cluster', () => { + services.navigateTo('create'); + + services.addService('nfs'); + + services.checkExist('nfs.testnfs', true); + services.getExpandCollapseElement().click(); + services.checkServiceStatus('nfs'); + }); + + it('should create a nfs-export with RGW backend', () => { + buckets.navigateTo('create'); + buckets.create(bucketName, 'dashboard', 'default-placement'); + + nfsExport.navigateTo(); + nfsExport.existTableCell(rgwPseudo, false); + nfsExport.navigateTo('create'); + nfsExport.create(backends[1], squash, client, rgwPseudo, bucketName); + nfsExport.existTableCell(rgwPseudo); + }); + + // @TODO: uncomment this when a CephFS volume can be created through Dashboard. + // it('should create a nfs-export with CephFS backend', () => { + // nfsExport.navigateTo(); + // nfsExport.existTableCell(fsPseudo, false); + // nfsExport.navigateTo('create'); + // nfsExport.create(backends[0], squash, client, fsPseudo); + // nfsExport.existTableCell(fsPseudo); + // }); + + it('should show Clients', () => { + nfsExport.clickTab('cd-nfs-details', rgwPseudo, 'Clients (1)'); + cy.get('cd-nfs-details').within(() => { + nfsExport.getTableCount('total').should('be.gte', 0); + }); + }); + + it('should edit an export', () => { + nfsExport.editExport(rgwPseudo, editPseudo); + + nfsExport.existTableCell(editPseudo); + }); + + it('should delete exports and bucket', () => { + nfsExport.delete(editPseudo); + + buckets.navigateTo(); + buckets.delete(bucketName); + }); + }); +}); diff --git a/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/nfs/nfs-export.po.ts b/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/nfs/nfs-export.po.ts new file mode 100644 index 0000000000000..91dfdf48d1015 --- /dev/null +++ b/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/nfs/nfs-export.po.ts @@ -0,0 +1,45 @@ +import { PageHelper } from 'cypress/integration/page-helper.po'; + +const pages = { + index: { url: '#/nfs', id: 'cd-nfs-list' }, + create: { url: '#/nfs/create', id: 'cd-nfs-form' } +}; + +export class NFSPageHelper extends PageHelper { + pages = pages; + + @PageHelper.restrictTo(pages.create.url) + create(backend: string, squash: string, client: object, pseudo: string, rgwPath?: string) { + this.selectOption('cluster_id', 'testnfs'); + // select a storage backend + this.selectOption('name', backend); + if (backend === 'CephFS') { + this.selectOption('fs_name', 'myfs'); + + cy.get('#security_label').click({ force: true }); + } else { + cy.get('input[data-testid=rgw_path]').type(rgwPath); + } + + cy.get('input[name=pseudo]').type(pseudo); + this.selectOption('squash', squash); + + // Add clients + cy.get('button[name=add_client]').click({ force: true }); + cy.get('input[name=addresses]').type(client['addresses']); + + cy.get('cd-submit-button').click(); + } + + editExport(pseudo: string, editPseudo: string) { + this.navigateEdit(pseudo); + + cy.get('input[name=pseudo]').clear().type(editPseudo); + + cy.get('cd-submit-button').click(); + + // Click the export and check its details table for updated content + this.getExpandCollapseElement(editPseudo).click(); + cy.get('.active.tab-pane').should('contain.text', editPseudo); + } +} diff --git a/src/pybind/mgr/dashboard/frontend/cypress/integration/page-helper.po.ts b/src/pybind/mgr/dashboard/frontend/cypress/integration/page-helper.po.ts index 6395128c94734..176bca5a14774 100644 --- a/src/pybind/mgr/dashboard/frontend/cypress/integration/page-helper.po.ts +++ b/src/pybind/mgr/dashboard/frontend/cypress/integration/page-helper.po.ts @@ -74,7 +74,7 @@ export abstract class PageHelper { } getTab(tabName: string) { - return cy.contains('.nav.nav-tabs li', new RegExp(`^${tabName}$`)); + return cy.contains('.nav.nav-tabs li', tabName); } getTabText(index: number) { @@ -203,6 +203,11 @@ export abstract class PageHelper { ); } + existTableCell(name: string, oughtToBePresent = true) { + const waitRule = oughtToBePresent ? 'be.visible' : 'not.exist'; + this.getFirstTableCell(name).should(waitRule); + } + getExpandCollapseElement(content?: string) { this.waitDataTableToLoad(); diff --git a/src/pybind/mgr/dashboard/frontend/cypress/integration/pools/pools.e2e-spec.ts b/src/pybind/mgr/dashboard/frontend/cypress/integration/pools/pools.e2e-spec.ts index e5a28bfd4e20c..b4c3c75ac5b85 100644 --- a/src/pybind/mgr/dashboard/frontend/cypress/integration/pools/pools.e2e-spec.ts +++ b/src/pybind/mgr/dashboard/frontend/cypress/integration/pools/pools.e2e-spec.ts @@ -30,19 +30,19 @@ describe('Pools page', () => { describe('Create, update and destroy', () => { it('should create a pool', () => { - pools.exist(poolName, false); + pools.existTableCell(poolName, false); pools.navigateTo('create'); pools.create(poolName, 8, 'rbd'); - pools.exist(poolName, true); + pools.existTableCell(poolName); }); it('should edit a pools placement group', () => { - pools.exist(poolName, true); + pools.existTableCell(poolName); pools.edit_pool_pg(poolName, 32); }); it('should show updated configuration field values', () => { - pools.exist(poolName, true); + pools.existTableCell(poolName); const bpsLimit = '4 B/s'; pools.edit_pool_configuration(poolName, bpsLimit); }); diff --git a/src/pybind/mgr/dashboard/frontend/cypress/integration/pools/pools.po.ts b/src/pybind/mgr/dashboard/frontend/cypress/integration/pools/pools.po.ts index ccf858b41206d..98cee470eda99 100644 --- a/src/pybind/mgr/dashboard/frontend/cypress/integration/pools/pools.po.ts +++ b/src/pybind/mgr/dashboard/frontend/cypress/integration/pools/pools.po.ts @@ -13,12 +13,6 @@ export class PoolPageHelper extends PageHelper { return expect((n & (n - 1)) === 0, `Placement groups ${n} are not a power of 2`).to.be.true; } - @PageHelper.restrictTo(pages.index.url) - exist(name: string, oughtToBePresent = true) { - const waitRule = oughtToBePresent ? 'be.visible' : 'not.exist'; - this.getFirstTableCell(name).should(waitRule); - } - @PageHelper.restrictTo(pages.create.url) create(name: string, placement_groups: number, ...apps: string[]) { cy.get('input[name=name]').clear().type(name); diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/services/service-form/service-form.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/services/service-form/service-form.component.html index e8e340a7dc8dc..99c0903dacf49 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/services/service-form/service-form.component.html +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/services/service-form/service-form.component.html @@ -173,50 +173,6 @@ - - - -
    - -
    - - This field is required. -
    -
    - - -
    - -
    - -
    -
    -     - diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/services/service-form/service-form.component.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/services/service-form/service-form.component.spec.ts index 78863435ea328..fd3bc8025dbe0 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/services/service-form/service-form.component.spec.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/services/service-form/service-form.component.spec.ts @@ -144,28 +144,14 @@ describe('ServiceFormComponent', () => { describe('should test service nfs', () => { beforeEach(() => { formHelper.setValue('service_type', 'nfs'); - formHelper.setValue('pool', 'foo'); }); - it('should submit nfs with namespace', () => { - formHelper.setValue('namespace', 'bar'); + it('should submit nfs', () => { component.onSubmit(); expect(cephServiceService.create).toHaveBeenCalledWith({ service_type: 'nfs', placement: {}, - unmanaged: false, - pool: 'foo', - namespace: 'bar' - }); - }); - - it('should submit nfs w/o namespace', () => { - component.onSubmit(); - expect(cephServiceService.create).toHaveBeenCalledWith({ - service_type: 'nfs', - placement: {}, - unmanaged: false, - pool: 'foo' + unmanaged: false }); }); }); diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/services/service-form/service-form.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/services/service-form/service-form.component.ts index 2b424d7f26a35..da4daf9c1f5f4 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/services/service-form/service-form.component.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/services/service-form/service-form.component.ts @@ -115,22 +115,16 @@ export class ServiceFormComponent extends CdForm implements OnInit { hosts: [[]], count: [null, [CdValidators.number(false), Validators.min(1)]], unmanaged: [false], - // NFS & iSCSI + // iSCSI pool: [ null, [ - CdValidators.requiredIf({ - service_type: 'nfs', - unmanaged: false - }), CdValidators.requiredIf({ service_type: 'iscsi', unmanaged: false }) ] ], - // NFS - namespace: [null], // RGW rgw_frontend_port: [ null, @@ -327,12 +321,6 @@ export class ServiceFormComponent extends CdForm implements OnInit { serviceSpec['placement']['count'] = values['count']; } switch (serviceType) { - case 'nfs': - serviceSpec['pool'] = values['pool']; - if (_.isString(values['namespace']) && !_.isEmpty(values['namespace'])) { - serviceSpec['namespace'] = values['namespace']; - } - break; case 'rgw': if (_.isNumber(values['rgw_frontend_port']) && values['rgw_frontend_port'] > 0) { serviceSpec['rgw_frontend_port'] = values['rgw_frontend_port']; diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/nfs/models/nfs.fsal.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/nfs/models/nfs.fsal.ts new file mode 100644 index 0000000000000..f204ac6d8b6b5 --- /dev/null +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/nfs/models/nfs.fsal.ts @@ -0,0 +1,5 @@ +export interface NfsFSAbstractionLayer { + value: string; + descr: string; + disabled: boolean; +} diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/nfs/nfs-cluster-type.enum.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/nfs/nfs-cluster-type.enum.ts deleted file mode 100644 index 7a775e5ab2dbd..0000000000000 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/nfs/nfs-cluster-type.enum.ts +++ /dev/null @@ -1,4 +0,0 @@ -export enum NFSClusterType { - user = 'user', - orchestrator = 'orchestrator' -} diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/nfs/nfs-details/nfs-details.component.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/nfs/nfs-details/nfs-details.component.spec.ts index 3abae2ee88e99..fcf5305393cba 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/nfs/nfs-details/nfs-details.component.spec.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/nfs/nfs-details/nfs-details.component.spec.ts @@ -25,18 +25,15 @@ describe('NfsDetailsComponent', () => { fixture = TestBed.createComponent(NfsDetailsComponent); component = fixture.componentInstance; - component.selection = undefined; component.selection = { export_id: 1, path: '/qwe', fsal: { name: 'CEPH', user_id: 'fs', fs_name: 1 }, cluster_id: 'cluster1', - daemons: ['node1', 'node2'], pseudo: '/qwe', - tag: 'asd', access_type: 'RW', squash: 'no_root_squash', - protocols: [3, 4], + protocols: [4], transports: ['TCP', 'UDP'], clients: [ { @@ -44,9 +41,7 @@ describe('NfsDetailsComponent', () => { access_type: 'RW', squash: 'root_id_squash' } - ], - id: 'cluster1:1', - state: 'LOADING' + ] }; component.ngOnChanges(); fixture.detectChanges(); @@ -62,8 +57,7 @@ describe('NfsDetailsComponent', () => { 'CephFS Filesystem': 1, 'CephFS User': 'fs', Cluster: 'cluster1', - Daemons: ['node1', 'node2'], - 'NFS Protocol': ['NFSv3', 'NFSv4'], + 'NFS Protocol': ['NFSv4'], Path: '/qwe', Pseudo: '/qwe', 'Security Label': undefined, @@ -77,7 +71,7 @@ describe('NfsDetailsComponent', () => { const newData = _.assignIn(component.selection, { fsal: { name: 'RGW', - rgw_user_id: 'rgw_user_id' + user_id: 'user-id' } }); component.selection = newData; @@ -85,9 +79,8 @@ describe('NfsDetailsComponent', () => { expect(component.data).toEqual({ 'Access Type': 'RW', Cluster: 'cluster1', - Daemons: ['node1', 'node2'], - 'NFS Protocol': ['NFSv3', 'NFSv4'], - 'Object Gateway User': 'rgw_user_id', + 'NFS Protocol': ['NFSv4'], + 'Object Gateway User': 'user-id', Path: '/qwe', Pseudo: '/qwe', Squash: 'no_root_squash', diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/nfs/nfs-details/nfs-details.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/nfs/nfs-details/nfs-details.component.ts index 25a42416f7e36..5a84bd52e9da9 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/nfs/nfs-details/nfs-details.component.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/nfs/nfs-details/nfs-details.component.ts @@ -45,7 +45,6 @@ export class NfsDetailsComponent implements OnChanges { this.data = {}; this.data[$localize`Cluster`] = this.selectedItem.cluster_id; - this.data[$localize`Daemons`] = this.selectedItem.daemons; this.data[$localize`NFS Protocol`] = this.selectedItem.protocols.map( (protocol: string) => 'NFSv' + protocol ); @@ -62,7 +61,7 @@ export class NfsDetailsComponent implements OnChanges { this.data[$localize`Security Label`] = this.selectedItem.fsal.sec_label_xattr; } else { this.data[$localize`Storage Backend`] = $localize`Object Gateway`; - this.data[$localize`Object Gateway User`] = this.selectedItem.fsal.rgw_user_id; + this.data[$localize`Object Gateway User`] = this.selectedItem.fsal.user_id; } } } diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/nfs/nfs-form-client/nfs-form-client.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/nfs/nfs-form-client/nfs-form-client.component.html index 4f84f8e03b75e..137cc43fa4b4f 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/nfs/nfs-form-client/nfs-form-client.component.html +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/nfs/nfs-form-client/nfs-form-client.component.html @@ -26,7 +26,7 @@
    - +
    + id="cluster_id"> @@ -34,66 +37,10 @@ [value]="cluster.cluster_id">{{ cluster.cluster_id }} This field is required. -
    -
    - - -
    - -
    - -
    - - - - -
    -     - -
    -
    - - - Add daemon - -
    -
    - -
    -
    - -
    -
    + *ngIf="nfsForm.showError('cluster_id', formDir, 'required') || allClusters?.length === 0" + i18n>This field is required. + To create a new NFS cluster, add a new NFS Service.
    @@ -120,70 +67,15 @@ value="" i18n>-- Select the storage backend -- + [value]="fsal.value" + [disabled]="fsal.disabled">{{ fsal.descr }} This field is required. -
    -
    - - -
    - -
    - This field is required. -
    -
    - - -
    - -
    - - This field is required. + *ngIf="fsalAvailabilityError" + i18n>{{ fsalAvailabilityError }}
    @@ -192,13 +84,13 @@ *ngIf="nfsForm.getValue('name') === 'CEPH'"> + i18n>Volume
    Path need to start with a '/' and can be followed by a word New directory will be created + *ngIf="nfsForm.showError('path', formDir, 'pathNameNotAllowed')" + i18n>The path does not exist.
    - +
    + [ngbTypeahead]="bucketDataSource"> This field is required. - Path can only be a single '/' or a word - - New bucket will be created + *ngIf="nfsForm.showError('path', formDir, 'bucketNameNotAllowed')" + i18n>The bucket does not exist or is not in the default realm (if multiple realms are configured). + To continue, create a new bucket.
    @@ -317,55 +213,23 @@ for="protocols" i18n>NFS Protocol
    -
    - - -
    + id="protocolNfsv4" + disabled>
    This field is required.
    - -
    - -
    - -
    -
    -
    @@ -435,9 +299,12 @@
    - +
    + autofocus + (keyup)="checkHostNameValue()"> This field is required. @@ -36,7 +46,8 @@
    -
    +
    diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/host-form/host-form.component.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/host-form/host-form.component.spec.ts index ed3daf1e4b49b..ed85d96cb1bab 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/host-form/host-form.component.spec.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/host-form/host-form.component.spec.ts @@ -81,4 +81,88 @@ describe('HostFormComponent', () => { component.submit(); expect(component.status).toBe('maintenance'); }); + + it('should expand the hostname correctly', () => { + component.hostForm.get('hostname').setValue('ceph-node-00.cephlab.com'); + fixture.detectChanges(); + component.submit(); + expect(component.hostnameArray).toStrictEqual(['ceph-node-00.cephlab.com']); + + component.hostnameArray = []; + + component.hostForm.get('hostname').setValue('ceph-node-[00-10].cephlab.com'); + fixture.detectChanges(); + component.submit(); + expect(component.hostnameArray).toStrictEqual([ + 'ceph-node-00.cephlab.com', + 'ceph-node-01.cephlab.com', + 'ceph-node-02.cephlab.com', + 'ceph-node-03.cephlab.com', + 'ceph-node-04.cephlab.com', + 'ceph-node-05.cephlab.com', + 'ceph-node-06.cephlab.com', + 'ceph-node-07.cephlab.com', + 'ceph-node-08.cephlab.com', + 'ceph-node-09.cephlab.com', + 'ceph-node-10.cephlab.com' + ]); + + component.hostnameArray = []; + + component.hostForm.get('hostname').setValue('ceph-node-00.cephlab.com,ceph-node-1.cephlab.com'); + fixture.detectChanges(); + component.submit(); + expect(component.hostnameArray).toStrictEqual([ + 'ceph-node-00.cephlab.com', + 'ceph-node-1.cephlab.com' + ]); + + component.hostnameArray = []; + + component.hostForm + .get('hostname') + .setValue('ceph-mon-[01-05].lab.com,ceph-osd-[1-4].lab.com,ceph-rgw-[001-006].lab.com'); + fixture.detectChanges(); + component.submit(); + expect(component.hostnameArray).toStrictEqual([ + 'ceph-mon-01.lab.com', + 'ceph-mon-02.lab.com', + 'ceph-mon-03.lab.com', + 'ceph-mon-04.lab.com', + 'ceph-mon-05.lab.com', + 'ceph-osd-1.lab.com', + 'ceph-osd-2.lab.com', + 'ceph-osd-3.lab.com', + 'ceph-osd-4.lab.com', + 'ceph-rgw-001.lab.com', + 'ceph-rgw-002.lab.com', + 'ceph-rgw-003.lab.com', + 'ceph-rgw-004.lab.com', + 'ceph-rgw-005.lab.com', + 'ceph-rgw-006.lab.com' + ]); + + component.hostnameArray = []; + + component.hostForm + .get('hostname') + .setValue('ceph-(mon-[00-04],osd-[001-005],rgw-[1-3]).lab.com'); + fixture.detectChanges(); + component.submit(); + expect(component.hostnameArray).toStrictEqual([ + 'ceph-mon-00.lab.com', + 'ceph-mon-01.lab.com', + 'ceph-mon-02.lab.com', + 'ceph-mon-03.lab.com', + 'ceph-mon-04.lab.com', + 'ceph-osd-001.lab.com', + 'ceph-osd-002.lab.com', + 'ceph-osd-003.lab.com', + 'ceph-osd-004.lab.com', + 'ceph-osd-005.lab.com', + 'ceph-rgw-1.lab.com', + 'ceph-rgw-2.lab.com', + 'ceph-rgw-3.lab.com' + ]); + }); }); diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/host-form/host-form.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/host-form/host-form.component.ts index 704b659127f26..6e84da688e581 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/host-form/host-form.component.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/host-form/host-form.component.ts @@ -3,6 +3,7 @@ import { FormControl, Validators } from '@angular/forms'; import { Router } from '@angular/router'; import { NgbActiveModal } from '@ng-bootstrap/ng-bootstrap'; +import expand from 'brace-expansion'; import { HostService } from '~/app/shared/api/host.service'; import { SelectMessages } from '~/app/shared/components/select/select-messages.model'; @@ -23,10 +24,12 @@ export class HostFormComponent extends CdForm implements OnInit { action: string; resource: string; hostnames: string[]; + hostnameArray: string[] = []; addr: string; status: string; allLabels: string[]; pageURL: string; + hostPattern = false; messages = new SelectMessages({ empty: $localize`There are no labels.`, @@ -59,6 +62,12 @@ export class HostFormComponent extends CdForm implements OnInit { }); } + // check if hostname is a single value or pattern to hide network address field + checkHostNameValue() { + const hostNames = this.hostForm.get('hostname').value; + hostNames.match(/[()\[\]{},]/g) ? (this.hostPattern = true) : (this.hostPattern = false); + } + private createForm() { this.hostForm = new CdFormGroup({ hostname: new FormControl('', { @@ -77,30 +86,77 @@ export class HostFormComponent extends CdForm implements OnInit { }); } + private isCommaSeparatedPattern(hostname: string) { + // eg. ceph-node-01.cephlab.com,ceph-node-02.cephlab.com + return hostname.includes(','); + } + + private isRangeTypePattern(hostname: string) { + // check if it is a range expression or comma separated range expression + // eg. ceph-mon-[01-05].lab.com,ceph-osd-[02-08].lab.com,ceph-rgw-[01-09] + return hostname.includes('[') && hostname.includes(']') && !hostname.match(/(?![^(]*\)),/g); + } + + private replaceBraces(hostname: string) { + // pattern to replace range [0-5] to [0..5](valid expression for brace expansion) + // replace any kind of brackets with curly braces + return hostname + .replace(/(?<=\d)\s*-\s*(?=\d)/g, '..') + .replace(/\(/g, '{') + .replace(/\)/g, '}') + .replace(/\[/g, '{') + .replace(/]/g, '}'); + } + + // expand hostnames in case hostname is a pattern + private checkHostNamePattern(hostname: string) { + if (this.isRangeTypePattern(hostname)) { + const hostnameRange = this.replaceBraces(hostname); + this.hostnameArray = expand(hostnameRange); + } else if (this.isCommaSeparatedPattern(hostname)) { + let hostArray = []; + hostArray = hostname.split(','); + hostArray.forEach((host: string) => { + if (this.isRangeTypePattern(host)) { + const hostnameRange = this.replaceBraces(host); + this.hostnameArray = this.hostnameArray.concat(expand(hostnameRange)); + } else { + this.hostnameArray.push(host); + } + }); + } else { + // single hostname + this.hostnameArray.push(hostname); + } + } + submit() { const hostname = this.hostForm.get('hostname').value; + this.checkHostNamePattern(hostname); this.addr = this.hostForm.get('addr').value; this.status = this.hostForm.get('maintenance').value ? 'maintenance' : ''; this.allLabels = this.hostForm.get('labels').value; if (this.pageURL !== 'hosts' && !this.allLabels.includes('_no_schedule')) { this.allLabels.push('_no_schedule'); } - this.taskWrapper - .wrapTaskAroundCall({ - task: new FinishedTask('host/' + URLVerbs.ADD, { - hostname: hostname - }), - call: this.hostService.create(hostname, this.addr, this.allLabels, this.status) - }) - .subscribe({ - error: () => { - this.hostForm.setErrors({ cdSubmitButton: true }); - }, - complete: () => { - this.pageURL === 'hosts' - ? this.router.navigate([this.pageURL, { outlets: { modal: null } }]) - : this.activeModal.close(); - } - }); + this.hostnameArray.forEach((hostName: string) => { + this.taskWrapper + .wrapTaskAroundCall({ + task: new FinishedTask('host/' + URLVerbs.ADD, { + hostname: hostName + }), + call: this.hostService.create(hostName, this.addr, this.allLabels, this.status) + }) + .subscribe({ + error: () => { + this.hostForm.setErrors({ cdSubmitButton: true }); + }, + complete: () => { + this.pageURL === 'hosts' + ? this.router.navigate([this.pageURL, { outlets: { modal: null } }]) + : this.activeModal.close(); + } + }); + }); } } From 441869220c97d739cbe2186a9e4ee0d8a2911f1d Mon Sep 17 00:00:00 2001 From: Daniel Pivonka Date: Thu, 21 Oct 2021 13:05:05 -0400 Subject: [PATCH 0299/2866] doc/cephadm: note that replacment osd needs to be on same host as removed osd Signed-off-by: Daniel Pivonka --- doc/cephadm/services/osd.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/cephadm/services/osd.rst b/doc/cephadm/services/osd.rst index 4347d05aff5b7..cdc9dda5ae392 100644 --- a/doc/cephadm/services/osd.rst +++ b/doc/cephadm/services/osd.rst @@ -305,6 +305,10 @@ This follows the same procedure as the procedure in the "Remove OSD" section, wi one exception: the OSD is not permanently removed from the CRUSH hierarchy, but is instead assigned a 'destroyed' flag. +.. note:: + The new OSD that will replace the removed OSD must be created on the same host + as the OSD that was removed. + **Preserving the OSD ID** The 'destroyed' flag is used to determine which OSD ids will be reused in the From 212f88cbbc2b4edb7976a2abff27ed9b45c5ae3b Mon Sep 17 00:00:00 2001 From: Adam King Date: Mon, 25 Oct 2021 13:40:19 -0400 Subject: [PATCH 0300/2866] mgr/cephadm: inform users if limit set for data devices is not met Signed-off-by: Adam King --- src/pybind/mgr/cephadm/services/osd.py | 16 ++++++++++++---- src/pybind/mgr/orchestrator/module.py | 5 ++++- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/src/pybind/mgr/cephadm/services/osd.py b/src/pybind/mgr/cephadm/services/osd.py index f5996eeae764b..3c0b654bacdea 100644 --- a/src/pybind/mgr/cephadm/services/osd.py +++ b/src/pybind/mgr/cephadm/services/osd.py @@ -204,12 +204,14 @@ def generate_previews(self, osdspecs: List[DriveGroupSpec], for_host: str) -> Li [ {'data': {}, 'osdspec': , - 'host': + 'host': , + 'notes': }, {'data': ..., 'osdspec': .., - 'host': .. + 'host': ..., + 'notes': ... } ] @@ -246,10 +248,16 @@ def generate_previews(self, osdspecs: List[DriveGroupSpec], for_host: str) -> Li except ValueError: logger.exception('Cannot decode JSON: \'%s\'' % ' '.join(out)) concat_out = {} - + notes = [] + if osdspec.data_devices is not None and osdspec.data_devices.limit and len(concat_out) < osdspec.data_devices.limit: + found = len(concat_out) + limit = osdspec.data_devices.limit + notes.append( + f'NOTE: Did not find enough disks matching filter on host {host} to reach data device limit (Found: {found} | Limit: {limit})') ret_all.append({'data': concat_out, 'osdspec': osdspec.service_id, - 'host': host}) + 'host': host, + 'notes': notes}) return ret_all def resolve_hosts_for_osdspecs(self, diff --git a/src/pybind/mgr/orchestrator/module.py b/src/pybind/mgr/orchestrator/module.py index 1b81a736e6399..c74a73413d917 100644 --- a/src/pybind/mgr/orchestrator/module.py +++ b/src/pybind/mgr/orchestrator/module.py @@ -158,6 +158,7 @@ def preview_table_osd(data: List) -> str: table.align = 'l' table.left_padding_width = 0 table.right_padding_width = 2 + notes = '' for osd_data in data: if osd_data.get('service_type') != 'osd': continue @@ -166,6 +167,8 @@ def preview_table_osd(data: List) -> str: if spec.get('error'): return spec.get('message') dg_name = spec.get('osdspec') + if spec.get('notes', []): + notes += '\n'.join(spec.get('notes')) + '\n' for osd in spec.get('data', []): db_path = osd.get('block_db', '-') wal_path = osd.get('block_wal', '-') @@ -173,7 +176,7 @@ def preview_table_osd(data: List) -> str: if not block_data: continue table.add_row(('osd', dg_name, host, block_data, db_path, wal_path)) - return table.get_string() + return notes + table.get_string() def preview_table_services(data: List) -> str: From b1debef085f10b6babf17408daba9b354ceb0480 Mon Sep 17 00:00:00 2001 From: "J. Eric Ivancich" Date: Mon, 13 Sep 2021 16:25:09 -0400 Subject: [PATCH 0301/2866] rgw: add logging to bucket listing so calls are better understood The logic to bucket listing is complex due to the sharded bucket indicies. When a log is examined, there should be enough information to follow the logic. Logging that touches every entry listed is at 20. Logging that describes exiting of the listing are at level 10. Additionally make logging more uniform and streamlined. Remove "INFO" as that seems redundant (but keep "ERROR" and "WARNING"). Dispense with class name decorating function name since there aren't any similar function names. Signed-off-by: J. Eric Ivancich --- src/rgw/rgw_rados.cc | 198 +++++++++++++++++++++++++++++++------------ 1 file changed, 142 insertions(+), 56 deletions(-) diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc index 1331f5c68f548..2458b639155ce 100644 --- a/src/rgw/rgw_rados.cc +++ b/src/rgw/rgw_rados.cc @@ -1846,15 +1846,20 @@ int RGWRados::Bucket::List::list_objects_ordered( } } + // we'll stop after this many attempts as long we return at least + // one entry; but we will also go beyond this number of attempts + // until we return at least one entry + constexpr uint16_t SOFT_MAX_ATTEMPTS = 8; + rgw_obj_index_key prev_marker; for (uint16_t attempt = 1; /* empty */; ++attempt) { - ldpp_dout(dpp, 20) << "RGWRados::Bucket::List::" << __func__ << - " starting attempt " << attempt << dendl; + ldpp_dout(dpp, 20) << __PRETTY_FUNCTION__ << + ": starting attempt " << attempt << dendl; if (attempt > 1 && !(prev_marker < cur_marker)) { // we've failed to make forward progress - ldpp_dout(dpp, 0) << "RGWRados::Bucket::List::" << __func__ << - ": ERROR marker failed to make forward progress; attempt=" << attempt << + ldpp_dout(dpp, 0) << "ERROR: " << __PRETTY_FUNCTION__ << + " marker failed to make forward progress; attempt=" << attempt << ", prev_marker=" << prev_marker << ", cur_marker=" << cur_marker << dendl; break; @@ -1887,8 +1892,8 @@ int RGWRados::Bucket::List::list_objects_ordered( rgw_obj_index_key index_key = entry.key; rgw_obj_key obj(index_key); - ldpp_dout(dpp, 20) << "RGWRados::Bucket::List::" << __func__ << - " considering entry " << entry.key << dendl; + ldpp_dout(dpp, 20) << __PRETTY_FUNCTION__ << + ": considering entry " << entry.key << dendl; /* note that parse_raw_oid() here will not set the correct * object's instance, as rgw_obj_index_key encodes that @@ -1898,13 +1903,15 @@ int RGWRados::Bucket::List::list_objects_ordered( */ bool valid = rgw_obj_key::parse_raw_oid(index_key.name, &obj); if (!valid) { - ldpp_dout(dpp, 0) << "ERROR: could not parse object name: " << - obj.name << dendl; + ldpp_dout(dpp, 0) << "ERROR: " << __PRETTY_FUNCTION__ << + " could not parse object name: " << obj.name << dendl; continue; } bool matched_ns = (obj.ns == params.ns); if (!params.list_versions && !entry.is_visible()) { + ldpp_dout(dpp, 10) << __PRETTY_FUNCTION__ << + ": skipping not visible entry \"" << entry.key << "\"" << dendl; continue; } @@ -1912,15 +1919,24 @@ int RGWRados::Bucket::List::list_objects_ordered( if (!params.ns.empty()) { /* we've iterated past the namespace we're searching -- done now */ truncated = false; + ldpp_dout(dpp, 10) << __PRETTY_FUNCTION__ << + ": finished due to getting past requested namespace \"" << + params.ns << "\"" << dendl; goto done; } - /* we're not looking at the namespace this object is in, next! */ + /* we're skipping past namespaced objects */ + ldpp_dout(dpp, 20) << __PRETTY_FUNCTION__ << + ": skipping past namespaced objects, including \"" << entry.key << + "\"" << dendl; continue; } if (cur_end_marker_valid && cur_end_marker <= index_key) { truncated = false; + ldpp_dout(dpp, 10) << __PRETTY_FUNCTION__ << + ": finished due to gitting end marker of \"" << cur_end_marker << + "\" with \"" << entry.key << "\"" << dendl; goto done; } @@ -1931,11 +1947,17 @@ int RGWRados::Bucket::List::list_objects_ordered( if (params.access_list_filter && ! params.access_list_filter->filter(obj.name, index_key.name)) { + ldpp_dout(dpp, 20) << __PRETTY_FUNCTION__ << + ": skipping past namespaced objects, including \"" << entry.key << + "\"" << dendl; continue; } if (params.prefix.size() && 0 != obj.name.compare(0, params.prefix.size(), params.prefix)) { + ldpp_dout(dpp, 20) << __PRETTY_FUNCTION__ << + ": skipping object \"" << entry.key << + "\" that doesn't match prefix \"" << params.prefix << "\"" << dendl; continue; } @@ -1946,20 +1968,23 @@ int RGWRados::Bucket::List::list_objects_ordered( // in the OSD/CLS or b) here. if (cls_filtered) { // NOTE: this condition is for the newer versions of the - // OSD that does filtering on the CLS side - - // should only find one delimiter at the end if it finds any - // after the prefix + // OSD that does filtering on the CLS side should only + // find one delimiter at the end if it finds any after the + // prefix if (delim_pos != int(obj.name.length() - params.delim.length())) { - ldpp_dout(dpp, 0) << - "WARNING: found delimiter in place other than the end of " + ldpp_dout(dpp, 0) << "WARNING: " << __PRETTY_FUNCTION__ << + " found delimiter in place other than the end of " "the prefix; obj.name=" << obj.name << ", prefix=" << params.prefix << dendl; } if (common_prefixes) { if (count >= max) { truncated = true; + ldpp_dout(dpp, 10) << __PRETTY_FUNCTION__ << + ": stopping early with common prefix \"" << entry.key << + "\" because requested number (" << max << + ") reached (cls filtered)" << dendl; goto done; } @@ -1967,6 +1992,9 @@ int RGWRados::Bucket::List::list_objects_ordered( count++; } + ldpp_dout(dpp, 20) << __PRETTY_FUNCTION__ << + ": finished entry with common prefix \"" << entry.key << + "\" so continuing loop (cls filtered)" << dendl; continue; } else { // NOTE: this condition is for older versions of the OSD @@ -1983,6 +2011,10 @@ int RGWRados::Bucket::List::list_objects_ordered( common_prefixes->find(prefix_key) == common_prefixes->end()) { if (count >= max) { truncated = true; + ldpp_dout(dpp, 10) << __PRETTY_FUNCTION__ << + ": stopping early with common prefix \"" << entry.key << + "\" because requested number (" << max << + ") reached (not cls filtered)" << dendl; goto done; } next_marker = prefix_key; @@ -1991,6 +2023,9 @@ int RGWRados::Bucket::List::list_objects_ordered( count++; } + ldpp_dout(dpp, 20) << __PRETTY_FUNCTION__ << + ": finished entry with common prefix \"" << entry.key << + "\" so continuing loop (not cls filtered)" << dendl; continue; } // if we're running an older OSD version } // if a delimiter was found after prefix @@ -1998,11 +2033,15 @@ int RGWRados::Bucket::List::list_objects_ordered( if (count >= max) { truncated = true; + ldpp_dout(dpp, 10) << __PRETTY_FUNCTION__ << + ": stopping early with entry \"" << entry.key << + "\" because requested number (" << max << + ") reached" << dendl; goto done; } - ldpp_dout(dpp, 20) << "RGWRados::Bucket::List::" << __func__ << - " adding entry " << entry.key << " to result" << dendl; + ldpp_dout(dpp, 20) << __PRETTY_FUNCTION__ << + ": adding entry " << entry.key << " to result" << dendl; result->emplace_back(std::move(entry)); count++; @@ -2020,30 +2059,37 @@ int RGWRados::Bucket::List::list_objects_ordered( cur_marker.name.substr(0, marker_delim_pos); skip_after_delim.append(after_delim_s); - ldpp_dout(dpp, 20) << "skip_after_delim=" << skip_after_delim << dendl; + ldpp_dout(dpp, 20) << __PRETTY_FUNCTION__ << + ": skip_after_delim=" << skip_after_delim << dendl; if (skip_after_delim > cur_marker.name) { cur_marker = skip_after_delim; - ldpp_dout(dpp, 20) << "setting cur_marker=" - << cur_marker.name - << "[" << cur_marker.instance << "]" - << dendl; + ldpp_dout(dpp, 20) << __PRETTY_FUNCTION__ << + ": setting cur_marker=" << cur_marker.name << + "[" << cur_marker.instance << "]" << dendl; } } } // if older osd didn't do delimiter filtering - ldpp_dout(dpp, 20) << "RGWRados::Bucket::List::" << __func__ << - " INFO end of outer loop, truncated=" << truncated << + ldpp_dout(dpp, 10) << __PRETTY_FUNCTION__ << + ": end of outer loop, truncated=" << truncated << ", count=" << count << ", attempt=" << attempt << dendl; if (!truncated || count >= (max + 1) / 2) { // if we finished listing, or if we're returning at least half the // requested entries, that's enough; S3 and swift protocols allow // returning fewer than max entries + ldpp_dout(dpp, 10) << __PRETTY_FUNCTION__ << + ": exiting attempt loop because we reached end (" << truncated << + ") or we're returning half the requested entries (" << count << + " of " << max << ")" << dendl; break; - } else if (attempt > 8 && count >= 1) { + } else if (attempt > SOFT_MAX_ATTEMPTS && count >= 1) { // if we've made at least 8 attempts and we have some, but very // few, results, return with what we have + ldpp_dout(dpp, 10) << __PRETTY_FUNCTION__ << + ": exiting attempt loop because we made " << attempt << + " attempts and we're returning " << count << " entries" << dendl; break; } } // for (uint16_t attempt... @@ -2136,6 +2182,9 @@ int RGWRados::Bucket::List::list_objects_unordered(const DoutPrefixProvider *dpp &cur_marker, y); if (r < 0) { + ldpp_dout(dpp, 0) << "ERROR: " << __PRETTY_FUNCTION__ << + " cls_bucket_list_unordered returned " << r << " for " << + target->get_bucket_info().bucket << dendl; return r; } @@ -2160,33 +2209,48 @@ int RGWRados::Bucket::List::list_objects_unordered(const DoutPrefixProvider *dpp */ bool valid = rgw_obj_key::parse_raw_oid(index_key.name, &obj); if (!valid) { - ldpp_dout(dpp, 0) << "ERROR: could not parse object name: " << - obj.name << dendl; + ldpp_dout(dpp, 0) << "ERROR: " << __PRETTY_FUNCTION__ << + " could not parse object name: " << obj.name << dendl; continue; } if (!params.list_versions && !entry.is_visible()) { + ldpp_dout(dpp, 20) << __PRETTY_FUNCTION__ << + ": skippping \"" << index_key << + "\" because not listing versions and entry not visibile" << dendl; continue; } if (params.enforce_ns && obj.ns != params.ns) { + ldpp_dout(dpp, 20) << __PRETTY_FUNCTION__ << + ": skippping \"" << index_key << + "\" because namespace does not match" << dendl; continue; } if (cur_end_marker_valid && cur_end_marker <= index_key) { // we're not guaranteed items will come in order, so we have // to loop through all + ldpp_dout(dpp, 20) << __PRETTY_FUNCTION__ << + ": skippping \"" << index_key << + "\" because after end_marker" << dendl; continue; } if (params.access_list_filter && !params.access_list_filter->filter(obj.name, index_key.name)) { + ldpp_dout(dpp, 20) << __PRETTY_FUNCTION__ << + ": skippping \"" << index_key << + "\" because doesn't match filter" << dendl; continue; } if (params.prefix.size() && (0 != obj.name.compare(0, params.prefix.size(), params.prefix))) { - continue; + ldpp_dout(dpp, 20) << __PRETTY_FUNCTION__ << + ": skippping \"" << index_key << + "\" because doesn't match prefix" << dendl; + continue; } if (count >= max) { @@ -6812,23 +6876,23 @@ int RGWRados::block_while_resharding(RGWRados::BucketShard *bs, RGWBucketReshardLock reshard_lock(this->store, bucket_info, true); ret = reshard_lock.lock(dpp); if (ret < 0) { - ldpp_dout(dpp, 20) << __func__ << - " INFO: failed to take reshard lock for bucket " << + ldpp_dout(dpp, 20) << __PRETTY_FUNCTION__ << + ": failed to take reshard lock for bucket " << bucket_id << "; expected if resharding underway" << dendl; } else { - ldpp_dout(dpp, 10) << __func__ << - " INFO: was able to take reshard lock for bucket " << + ldpp_dout(dpp, 10) << __PRETTY_FUNCTION__ << + ": was able to take reshard lock for bucket " << bucket_id << dendl; ret = RGWBucketReshard::clear_resharding(dpp, this->store, bucket_info); if (ret < 0) { reshard_lock.unlock(); - ldpp_dout(dpp, 0) << __func__ << + ldpp_dout(dpp, 0) << __PRETTY_FUNCTION__ << " ERROR: failed to clear resharding flags for bucket " << bucket_id << dendl; } else { reshard_lock.unlock(); - ldpp_dout(dpp, 5) << __func__ << - " INFO: apparently successfully cleared resharding flags for " + ldpp_dout(dpp, 5) << __PRETTY_FUNCTION__ << + ": apparently successfully cleared resharding flags for " "bucket " << bucket_id << dendl; continue; // if we apparently succeed immediately test again } // if clear resharding succeeded @@ -6837,13 +6901,13 @@ int RGWRados::block_while_resharding(RGWRados::BucketShard *bs, ret = reshard_wait->wait(y); if (ret < 0) { - ldpp_dout(dpp, 0) << __func__ << + ldpp_dout(dpp, 0) << __PRETTY_FUNCTION__ << " ERROR: bucket is still resharding, please retry" << dendl; return ret; } } // for loop - ldpp_dout(dpp, 0) << __func__ << + ldpp_dout(dpp, 0) << __PRETTY_FUNCTION__ << " ERROR: bucket is still resharding, please retry" << dendl; return -ERR_BUSY_RESHARDING; } @@ -8405,7 +8469,7 @@ int RGWRados::cls_bucket_list_ordered(const DoutPrefixProvider *dpp, * few results, perhaps due to filtering or to a series of * namespaced entries */ - ldpp_dout(dpp, 10) << "RGWRados::" << __func__ << ": " << + ldpp_dout(dpp, 10) << __PRETTY_FUNCTION__ << ": " << bucket_info.bucket << " start_after=\"" << start_after << "\", prefix=\"" << prefix << @@ -8428,6 +8492,8 @@ int RGWRados::cls_bucket_list_ordered(const DoutPrefixProvider *dpp, &index_pool, &shard_oids, nullptr); if (r < 0) { + ldpp_dout(dpp, 0) << __PRETTY_FUNCTION__ << + ": open_bucket_index for " << bucket_info.bucket << " failed" << dendl; return r; } @@ -8446,8 +8512,8 @@ int RGWRados::cls_bucket_list_ordered(const DoutPrefixProvider *dpp, num_entries_per_shard = num_entries; } - ldpp_dout(dpp, 10) << "RGWRados::" << __func__ << - " request from each of " << shard_count << + ldpp_dout(dpp, 10) << __PRETTY_FUNCTION__ << + ": request from each of " << shard_count << " shard(s) for " << num_entries_per_shard << " entries to get " << num_entries << " total entries" << dendl; @@ -8459,6 +8525,9 @@ int RGWRados::cls_bucket_list_ordered(const DoutPrefixProvider *dpp, list_versions, shard_oids, shard_list_results, cct->_conf->rgw_bucket_index_max_aio)(); if (r < 0) { + ldpp_dout(dpp, 0) << __PRETTY_FUNCTION__ << + ": CLSRGWIssueBucketList for " << bucket_info.bucket << + " failed" << dendl; return r; } @@ -8558,7 +8627,7 @@ int RGWRados::cls_bucket_list_ordered(const DoutPrefixProvider *dpp, const string& name = tracker.entry_name(); rgw_bucket_dir_entry& dirent = tracker.dir_entry(); - ldpp_dout(dpp, 20) << "RGWRados::" << __func__ << " currently processing " << + ldpp_dout(dpp, 20) << __PRETTY_FUNCTION__ << ": currently processing " << dirent.key << " from shard " << tracker.shard_idx << dendl; const bool force_check = @@ -8577,6 +8646,9 @@ int RGWRados::cls_bucket_list_ordered(const DoutPrefixProvider *dpp, r = check_disk_state(dpp, sub_ctx, bucket_info, dirent, dirent, updates[tracker.oid_name], y); if (r < 0 && r != -ENOENT) { + ldpp_dout(dpp, 0) << __PRETTY_FUNCTION__ << + ": check_disk_state for \"" << dirent.key << + "\" failed with r=" << r << dendl; return r; } } else { @@ -8585,7 +8657,7 @@ int RGWRados::cls_bucket_list_ordered(const DoutPrefixProvider *dpp, // at this point either r >= 0 or r == -ENOENT if (r >= 0) { // i.e., if r != -ENOENT - ldpp_dout(dpp, 10) << "RGWRados::" << __func__ << ": got " << + ldpp_dout(dpp, 10) << __PRETTY_FUNCTION__ << ": got " << dirent.key << dendl; auto [it, inserted] = m.insert_or_assign(name, std::move(dirent)); @@ -8593,12 +8665,12 @@ int RGWRados::cls_bucket_list_ordered(const DoutPrefixProvider *dpp, if (inserted) { ++count; } else { - ldpp_dout(dpp, 0) << "WARNING: RGWRados::" << __func__ << - ": reassigned map value at \"" << name << + ldpp_dout(dpp, 0) << "WARNING: " << __PRETTY_FUNCTION__ << + " reassigned map value at \"" << name << "\", which should not happen" << dendl; } } else { - ldpp_dout(dpp, 10) << "RGWRados::" << __func__ << ": skipping " << + ldpp_dout(dpp, 10) << __PRETTY_FUNCTION__ << ": skipping " << dirent.key.name << "[" << dirent.key.instance << "]" << dendl; last_entry_visited = &tracker.dir_entry(); } @@ -8614,6 +8686,10 @@ int RGWRados::cls_bucket_list_ordered(const DoutPrefixProvider *dpp, // as we cannot be certain that one of the next entries needs to // come from that shard; S3 and swift protocols allow returning // fewer than what was requested + ldpp_dout(dpp, 10) << __PRETTY_FUNCTION__ << + ": stopped accumulating results at count=" << count << + ", dirent=\"" << dirent.key << + "\", because its shard is untruncated and exhaused" << dendl; break; } } // while we haven't provided requested # of result entries @@ -8641,22 +8717,22 @@ int RGWRados::cls_bucket_list_ordered(const DoutPrefixProvider *dpp, } } - ldpp_dout(dpp, 20) << "RGWRados::" << __func__ << + ldpp_dout(dpp, 20) << __PRETTY_FUNCTION__ << ": returning, count=" << count << ", is_truncated=" << *is_truncated << dendl; if (*is_truncated && count < num_entries) { - ldpp_dout(dpp, 10) << "RGWRados::" << __func__ << - ": INFO requested " << num_entries << " entries but returning " << + ldpp_dout(dpp, 10) << __PRETTY_FUNCTION__ << + ": requested " << num_entries << " entries but returning " << count << ", which is truncated" << dendl; } if (last_entry_visited != nullptr && last_entry) { *last_entry = last_entry_visited->key; - ldpp_dout(dpp, 20) << "RGWRados::" << __func__ << + ldpp_dout(dpp, 20) << __PRETTY_FUNCTION__ << ": returning, last_entry=" << *last_entry << dendl; } else { - ldpp_dout(dpp, 20) << "RGWRados::" << __func__ << + ldpp_dout(dpp, 20) << __PRETTY_FUNCTION__ << ": returning, last_entry NOT SET" << dendl; } @@ -8692,7 +8768,8 @@ int RGWRados::cls_bucket_list_unordered(const DoutPrefixProvider *dpp, rgw_obj_index_key *last_entry, optional_yield y, RGWBucketListNameFilter force_check_filter) { - ldpp_dout(dpp, 10) << __func__ << " " << bucket_info.bucket << + ldpp_dout(dpp, 10) << __PRETTY_FUNCTION__ << " " << + bucket_info.bucket << " start_after=\"" << start_after << "\", prefix=\"" << prefix << "\", shard_id=" << shard_id << @@ -8734,9 +8811,9 @@ int RGWRados::cls_bucket_list_unordered(const DoutPrefixProvider *dpp, rgw_obj_key obj_key; bool parsed = rgw_obj_key::parse_raw_oid(start_after.name, &obj_key); if (!parsed) { - ldpp_dout(dpp, 0) << - "ERROR: RGWRados::cls_bucket_list_unordered received an invalid " - "start marker: '" << start_after << "'" << dendl; + ldpp_dout(dpp, 0) << "ERROR: " << __PRETTY_FUNCTION__ << + " received an invalid start marker: \"" << start_after << "\"" << + dendl; return -EINVAL; } else if (obj_key.name.empty()) { // if the name is empty that means the object name came in with @@ -8755,6 +8832,8 @@ int RGWRados::cls_bucket_list_unordered(const DoutPrefixProvider *dpp, string index_hash_source; r = parse_index_hash_source(obj_key.name, &index_hash_source); if (r < 0) { + ldpp_dout(dpp, 0) << "ERROR: " << __PRETTY_FUNCTION__ << + " error in parse_index_hash_source, r=" << r << dendl; return r; } current_shard = svc.bi_rados->bucket_shard_index(index_hash_source, num_shards); @@ -8774,12 +8853,14 @@ int RGWRados::cls_bucket_list_unordered(const DoutPrefixProvider *dpp, rgw_cls_list_ret result; librados::ObjectReadOperation op; - string empty_delimiter; + const std::string empty_delimiter; cls_rgw_bucket_list_op(op, marker, prefix, empty_delimiter, num_entries, list_versions, &result); r = rgw_rados_operate(dpp, ioctx, oid, &op, nullptr, null_yield); if (r < 0) { + ldpp_dout(dpp, 0) << "ERROR: " << __PRETTY_FUNCTION__ << + " error in rgw_rados_operate (bucket list op), r=" << r << dendl; return r; } @@ -8797,6 +8878,8 @@ int RGWRados::cls_bucket_list_unordered(const DoutPrefixProvider *dpp, sub_ctx.dup(ioctx); r = check_disk_state(dpp, sub_ctx, bucket_info, dirent, dirent, updates[oid], y); if (r < 0 && r != -ENOENT) { + ldpp_dout(dpp, 0) << "ERROR: " << __PRETTY_FUNCTION__ << + " error in check_disk_state, r=" << r << dendl; return r; } } else { @@ -8805,7 +8888,7 @@ int RGWRados::cls_bucket_list_unordered(const DoutPrefixProvider *dpp, // at this point either r >= 0 or r == -ENOENT if (r >= 0) { // i.e., if r != -ENOENT - ldpp_dout(dpp, 10) << "RGWRados::cls_bucket_list_unordered: got " << + ldpp_dout(dpp, 10) << __PRETTY_FUNCTION__ << ": got " << dirent.key << dendl; if (count < num_entries) { @@ -8815,6 +8898,9 @@ int RGWRados::cls_bucket_list_unordered(const DoutPrefixProvider *dpp, } else { last_added_entry = dirent.key; *is_truncated = true; + ldpp_dout(dpp, 10) << "ERROR: " << __PRETTY_FUNCTION__ << + ": reached max entries (" << num_entries << ") to return at \"" << + dirent.key << "\"" << dendl; goto check_updates; } } else { // r == -ENOENT From 4edde6a2c949f3e9efd119232c18c42738c0192d Mon Sep 17 00:00:00 2001 From: "J. Eric Ivancich" Date: Mon, 13 Sep 2021 16:46:27 -0400 Subject: [PATCH 0302/2866] rgw: explicitly use std namespace in bucket listing functions Signed-off-by: J. Eric Ivancich --- src/rgw/rgw_rados.cc | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc index 2458b639155ce..cf16234849ac5 100644 --- a/src/rgw/rgw_rados.cc +++ b/src/rgw/rgw_rados.cc @@ -1796,8 +1796,8 @@ int RGWRados::Bucket::update_bucket_id(const string& new_bucket_id, const DoutPr int RGWRados::Bucket::List::list_objects_ordered( const DoutPrefixProvider *dpp, int64_t max_p, - vector *result, - map *common_prefixes, + std::vector *result, + std::map *common_prefixes, bool *is_truncated, optional_yield y) { @@ -1831,8 +1831,8 @@ int RGWRados::Bucket::List::list_objects_ordered( rgw_obj_key prefix_obj(params.prefix); prefix_obj.set_ns(params.ns); - string cur_prefix = prefix_obj.get_index_key_name(); - string after_delim_s; /* needed in !params.delim.empty() AND later */ + std::string cur_prefix = prefix_obj.get_index_key_name(); + std::string after_delim_s; /* needed in !params.delim.empty() AND later */ if (!params.delim.empty()) { after_delim_s = cls_rgw_after_delim(params.delim); @@ -2125,9 +2125,9 @@ int RGWRados::Bucket::List::list_objects_ordered( */ int RGWRados::Bucket::List::list_objects_unordered(const DoutPrefixProvider *dpp, int64_t max_p, - vector *result, - map *common_prefixes, - bool *is_truncated, + std::vector* result, + std::map* common_prefixes, + bool* is_truncated, optional_yield y) { RGWRados *store = target->get_store(); @@ -2164,7 +2164,7 @@ int RGWRados::Bucket::List::list_objects_unordered(const DoutPrefixProvider *dpp rgw_obj_key prefix_obj(params.prefix); prefix_obj.set_ns(params.ns); - string cur_prefix = prefix_obj.get_index_key_name(); + std::string cur_prefix = prefix_obj.get_index_key_name(); while (truncated && count <= max) { std::vector ent_list; @@ -2249,7 +2249,7 @@ int RGWRados::Bucket::List::list_objects_unordered(const DoutPrefixProvider *dpp (0 != obj.name.compare(0, params.prefix.size(), params.prefix))) { ldpp_dout(dpp, 20) << __PRETTY_FUNCTION__ << ": skippping \"" << index_key << - "\" because doesn't match prefix" << dendl; + "\" because doesn't match prefix" << dendl; continue; } @@ -8452,15 +8452,15 @@ int RGWRados::cls_bucket_list_ordered(const DoutPrefixProvider *dpp, RGWBucketInfo& bucket_info, const int shard_id, const rgw_obj_index_key& start_after, - const string& prefix, - const string& delimiter, + const std::string& prefix, + const std::string& delimiter, const uint32_t num_entries, const bool list_versions, const uint16_t expansion_factor, ent_map_t& m, bool* is_truncated, bool* cls_filtered, - rgw_obj_index_key *last_entry, + rgw_obj_index_key* last_entry, optional_yield y, RGWBucketListNameFilter force_check_filter) { @@ -8487,7 +8487,7 @@ int RGWRados::cls_bucket_list_ordered(const DoutPrefixProvider *dpp, // key - oid (for different shards if there is any) // value - list result for the corresponding oid (shard), it is filled by // the AIO callback - map shard_oids; + std::map shard_oids; int r = svc.bi_rados->open_bucket_index(dpp, bucket_info, shard_id, &index_pool, &shard_oids, nullptr); @@ -8518,7 +8518,7 @@ int RGWRados::cls_bucket_list_ordered(const DoutPrefixProvider *dpp, num_entries << " total entries" << dendl; auto& ioctx = index_pool.ioctx(); - map shard_list_results; + std::map shard_list_results; cls_rgw_obj_key start_after_key(start_after.name, start_after.instance); r = CLSRGWIssueBucketList(ioctx, start_after_key, prefix, delimiter, num_entries_per_shard, @@ -8614,7 +8614,7 @@ int RGWRados::cls_bucket_list_ordered(const DoutPrefixProvider *dpp, rgw_bucket_dir_entry* last_entry_visited = nullptr; // to set last_entry (marker) - map updates; + std::map updates; uint32_t count = 0; while (count < num_entries && !candidates.empty()) { r = 0; @@ -8624,7 +8624,7 @@ int RGWRados::cls_bucket_list_ordered(const DoutPrefixProvider *dpp, tracker_idx = candidates.begin()->second; auto& tracker = results_trackers.at(tracker_idx); - const string& name = tracker.entry_name(); + const std::string& name = tracker.entry_name(); rgw_bucket_dir_entry& dirent = tracker.dir_entry(); ldpp_dout(dpp, 20) << __PRETTY_FUNCTION__ << ": currently processing " << @@ -8760,7 +8760,7 @@ int RGWRados::cls_bucket_list_unordered(const DoutPrefixProvider *dpp, RGWBucketInfo& bucket_info, int shard_id, const rgw_obj_index_key& start_after, - const string& prefix, + const std::string& prefix, uint32_t num_entries, bool list_versions, std::vector& ent_list, @@ -8784,7 +8784,7 @@ int RGWRados::cls_bucket_list_unordered(const DoutPrefixProvider *dpp, *is_truncated = false; RGWSI_RADOS::Pool index_pool; - map oids; + std::map oids; int r = svc.bi_rados->open_bucket_index(dpp, bucket_info, shard_id, &index_pool, &oids, nullptr); if (r < 0) { return r; @@ -8844,7 +8844,7 @@ int RGWRados::cls_bucket_list_unordered(const DoutPrefixProvider *dpp, } uint32_t count = 0u; - map updates; + std::map updates; rgw_obj_index_key last_added_entry; while (count <= num_entries && ((shard_id >= 0 && current_shard == uint32_t(shard_id)) || @@ -8920,7 +8920,7 @@ int RGWRados::cls_bucket_list_unordered(const DoutPrefixProvider *dpp, check_updates: // suggest updates if there is any - map::iterator miter = updates.begin(); + std::map::iterator miter = updates.begin(); for (; miter != updates.end(); ++miter) { if (miter->second.length()) { ObjectWriteOperation o; From 1c7e53d778b07549d4183e1c18165d5cd09efb14 Mon Sep 17 00:00:00 2001 From: Samuel Just Date: Thu, 21 Oct 2021 23:15:33 -0700 Subject: [PATCH 0303/2866] test/crimson/seastore/test_btree_lba_manager: add LBABtree level unit tests Signed-off-by: Samuel Just --- .../seastore/test_btree_lba_manager.cc | 148 ++++++++++++++++++ 1 file changed, 148 insertions(+) diff --git a/src/test/crimson/seastore/test_btree_lba_manager.cc b/src/test/crimson/seastore/test_btree_lba_manager.cc index 121b4c4d6e3ca..f3e34e6f0c4a4 100644 --- a/src/test/crimson/seastore/test_btree_lba_manager.cc +++ b/src/test/crimson/seastore/test_btree_lba_manager.cc @@ -114,6 +114,154 @@ struct btree_test_base : } }; +struct lba_btree_test : btree_test_base { + std::map check; + + auto get_op_context(Transaction &t) { + return op_context_t{cache, t}; + } + + LBAManager::mkfs_ret test_structure_setup(Transaction &t) final { + return cache.get_root( + t + ).si_then([this, &t](RootBlockRef croot) { + auto mut_croot = cache.duplicate_for_write( + t, croot + )->cast(); + mut_croot->root.lba_root = LBABtree::mkfs(get_op_context(t)); + }); + } + + void update_if_dirty(Transaction &t, LBABtree &btree, RootBlockRef croot) { + if (btree.is_root_dirty()) { + auto mut_croot = cache.duplicate_for_write( + t, croot + )->cast(); + mut_croot->root.lba_root = btree.get_root_undirty(); + } + } + + template + auto lba_btree_update(F &&f) { + auto tref = cache.create_transaction(Transaction::src_t::MUTATE); + auto &t = *tref; + with_trans_intr( + t, + [this, tref=std::move(tref), f=std::forward(f)](auto &t) mutable { + return cache.get_root( + t + ).si_then([this, f=std::move(f), &t](RootBlockRef croot) { + return seastar::do_with( + LBABtree(croot->root.lba_root), + [this, croot, f=std::move(f), &t](auto &btree) mutable { + return std::invoke( + std::move(f), btree, t + ).si_then([this, croot, &t, &btree] { + update_if_dirty(t, btree, croot); + return seastar::now(); + }); + }); + }).si_then([this, tref=std::move(tref)]() mutable { + return submit_transaction(std::move(tref)); + }); + }).unsafe_get0(); + } + + template + auto lba_btree_read(F &&f) { + auto t = cache.create_transaction(Transaction::src_t::READ); + return with_trans_intr( + *t, + [this, f=std::forward(f)](auto &t) mutable { + return cache.get_root( + t + ).si_then([f=std::move(f), &t](RootBlockRef croot) mutable { + return seastar::do_with( + LBABtree(croot->root.lba_root), + [f=std::move(f), &t](auto &btree) mutable { + return std::invoke( + std::move(f), btree, t + ); + }); + }); + }).unsafe_get0(); + } + + static auto get_map_val(extent_len_t len) { + return lba_map_val_t{0, P_ADDR_NULL, len, 0}; + } + + void insert(laddr_t addr, extent_len_t len) { + ceph_assert(check.count(addr) == 0); + check.emplace(addr, get_map_val(len)); + lba_btree_update([=](auto &btree, auto &t) { + return btree.insert( + get_op_context(t), addr, get_map_val(len) + ).si_then([](auto){}); + }); + } + + void remove(laddr_t addr) { + auto iter = check.find(addr); + ceph_assert(iter != check.end()); + auto len = iter->second.len; + check.erase(iter++); + lba_btree_update([=](auto &btree, auto &t) { + return btree.lower_bound( + get_op_context(t), addr + ).si_then([this, len, addr, &btree, &t](auto iter) { + EXPECT_FALSE(iter.is_end()); + EXPECT_TRUE(iter.get_key() == addr); + EXPECT_TRUE(iter.get_val().len == len); + return btree.remove( + get_op_context(t), iter + ); + }); + }); + } + + void check_lower_bound(laddr_t addr) { + auto iter = check.lower_bound(addr); + auto result = lba_btree_read([=](auto &btree, auto &t) { + return btree.lower_bound( + get_op_context(t), addr + ).si_then([](auto iter) + -> std::optional> { + if (iter.is_end()) { + return std::nullopt; + } else { + return std::make_optional( + std::make_pair(iter.get_key(), iter.get_val())); + } + }); + }); + if (iter == check.end()) { + EXPECT_FALSE(result); + } else { + EXPECT_TRUE(result); + decltype(result) to_check = *iter; + EXPECT_EQ(to_check, *result); + } + } +}; + +TEST_F(lba_btree_test, basic) +{ + run_async([this] { + constexpr unsigned total = 16<<10; + for (unsigned i = 0; i < total; i += 16) { + insert(i, 8); + } + + for (unsigned i = 0; i < total; i += 16) { + check_lower_bound(i); + check_lower_bound(i + 4); + check_lower_bound(i + 8); + check_lower_bound(i + 12); + } + }); +} + struct btree_lba_manager_test : btree_test_base { BtreeLBAManagerRef lba_manager; From eb79d8a3bc9985fafbb661e77016bf3f41f4aa62 Mon Sep 17 00:00:00 2001 From: Samuel Just Date: Sun, 24 Oct 2021 22:39:49 -0700 Subject: [PATCH 0304/2866] crimson/test/seastore/test_btree_lba_manager: shorten tests a bit Signed-off-by: Samuel Just --- src/test/crimson/seastore/test_btree_lba_manager.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/crimson/seastore/test_btree_lba_manager.cc b/src/test/crimson/seastore/test_btree_lba_manager.cc index f3e34e6f0c4a4..34f0cd7c8d3ea 100644 --- a/src/test/crimson/seastore/test_btree_lba_manager.cc +++ b/src/test/crimson/seastore/test_btree_lba_manager.cc @@ -573,7 +573,7 @@ TEST_F(btree_lba_manager_test, single_transaction_split_merge) run_async([this] { { auto t = create_transaction(); - for (unsigned i = 0; i < 600; ++i) { + for (unsigned i = 0; i < 400; ++i) { alloc_mapping(t, 0, block_size, get_paddr()); } check_mappings(t); @@ -614,7 +614,7 @@ TEST_F(btree_lba_manager_test, split_merge_multi) { run_async([this] { auto iterate = [&](auto f) { - for (uint64_t i = 0; i < (1<<12); ++i) { + for (uint64_t i = 0; i < (1<<10); ++i) { auto t = create_transaction(false); logger().debug("opened transaction"); for (unsigned j = 0; j < 5; ++j) { From dd2c6998927dc457231c5378f9ba277b7572c88f Mon Sep 17 00:00:00 2001 From: Samuel Just Date: Sun, 24 Oct 2021 23:03:33 -0700 Subject: [PATCH 0305/2866] crimson/os/seastore/lba_manager/lba_btree: add at_boundary helper distinct from is_end From an external caller, the condition is identical. However, internally iterators may be at a leaf boundary without being at end(). For those checks, use at_boundary() instead. Signed-off-by: Samuel Just --- src/crimson/os/seastore/lba_manager/btree/lba_btree.cc | 4 ++-- src/crimson/os/seastore/lba_manager/btree/lba_btree.h | 9 +++++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/crimson/os/seastore/lba_manager/btree/lba_btree.cc b/src/crimson/os/seastore/lba_manager/btree/lba_btree.cc index 31770ed688eb8..a1a324f62d20b 100644 --- a/src/crimson/os/seastore/lba_manager/btree/lba_btree.cc +++ b/src/crimson/os/seastore/lba_manager/btree/lba_btree.cc @@ -166,7 +166,7 @@ LBABtree::insert_ret LBABtree::insert( return find_insertion( c, laddr, ret ).si_then([this, c, laddr, val, &ret] { - if (!ret.is_end() && ret.get_key() == laddr) { + if (!ret.at_boundary() && ret.get_key() == laddr) { return insert_ret( interruptible::ready_future_marker{}, std::make_pair(ret, false)); @@ -488,7 +488,7 @@ LBABtree::find_insertion_ret LBABtree::find_insertion( // invariant that pos is a valid index for the node in the event // that the insertion point is at the end of a node. p.leaf.pos++; - assert(p.is_end()); + assert(p.at_boundary()); iter = p; return seastar::now(); }); diff --git a/src/crimson/os/seastore/lba_manager/btree/lba_btree.h b/src/crimson/os/seastore/lba_manager/btree/lba_btree.h index 6afa31f4ccde6..3364446862b22 100644 --- a/src/crimson/os/seastore/lba_manager/btree/lba_btree.h +++ b/src/crimson/os/seastore/lba_manager/btree/lba_btree.h @@ -87,8 +87,8 @@ class LBABtree { } bool is_end() const { - assert(leaf.pos <= leaf.node->get_size()); - return leaf.pos == leaf.node->get_size(); + // external methods may only resolve at a boundary if at end + return at_boundary(); } bool is_begin() const { @@ -134,6 +134,11 @@ class LBABtree { node_position_t, MAX_DEPTH> internal; node_position_t leaf; + bool at_boundary() const { + assert(leaf.pos <= leaf.node->get_size()); + return leaf.pos == leaf.node->get_size(); + } + depth_t check_split() const { if (!leaf.node->at_max_capacity()) { return 0; From 505492844c0ebf8567d33296b75e402f286ebe57 Mon Sep 17 00:00:00 2001 From: Samuel Just Date: Mon, 25 Oct 2021 06:14:08 +0000 Subject: [PATCH 0306/2866] crimson/os/seastore/lba_manager/lba_btree: factor out iterator::handle_boundary Factor out helper to adjust iterators at_boundary(). Signed-off-by: Samuel Just --- .../seastore/lba_manager/btree/lba_btree.cc | 54 +++++++++++-------- .../os/seastore/lba_manager/btree/lba_btree.h | 6 +++ 2 files changed, 38 insertions(+), 22 deletions(-) diff --git a/src/crimson/os/seastore/lba_manager/btree/lba_btree.cc b/src/crimson/os/seastore/lba_manager/btree/lba_btree.cc index a1a324f62d20b..ba166fb127ad0 100644 --- a/src/crimson/os/seastore/lba_manager/btree/lba_btree.cc +++ b/src/crimson/os/seastore/lba_manager/btree/lba_btree.cc @@ -21,21 +21,11 @@ LBABtree::mkfs_ret LBABtree::mkfs(op_context_t c) return lba_root_t{root_leaf->get_paddr(), 1u}; } -LBABtree::iterator_fut LBABtree::iterator::next( +LBABtree::iterator::handle_boundary_ret LBABtree::iterator::handle_boundary( op_context_t c, - mapped_space_visitor_t *visitor) const + mapped_space_visitor_t *visitor) { - assert_valid(); - assert(!is_end()); - - if ((leaf.pos + 1) < leaf.node->get_size()) { - auto ret = *this; - ret.leaf.pos++; - return iterator_fut( - interruptible::ready_future_marker{}, - ret); - } - + assert(at_boundary()); depth_t depth_with_space = 2; for (; depth_with_space <= get_depth(); ++depth_with_space) { if ((get_internal(depth_with_space).pos + 1) < @@ -46,29 +36,49 @@ LBABtree::iterator_fut LBABtree::iterator::next( if (depth_with_space <= get_depth()) { return seastar::do_with( - *this, [](const LBAInternalNode &internal) { return internal.begin(); }, [](const LBALeafNode &leaf) { return leaf.begin(); }, - [c, depth_with_space, visitor](auto &ret, auto &li, auto &ll) { + [this, c, depth_with_space, visitor](auto &li, auto &ll) { for (depth_t depth = 2; depth < depth_with_space; ++depth) { - ret.get_internal(depth).reset(); + get_internal(depth).reset(); } - ret.leaf.reset(); - ret.get_internal(depth_with_space).pos++; + leaf.reset(); + get_internal(depth_with_space).pos++; return lookup_depth_range( - c, ret, depth_with_space - 1, 0, li, ll, visitor + c, *this, depth_with_space - 1, 0, li, ll, visitor + ); + }); + } else { + // end + return seastar::now(); + } +} + +LBABtree::iterator_fut LBABtree::iterator::next( + op_context_t c, + mapped_space_visitor_t *visitor) const +{ + assert_valid(); + assert(!is_end()); + + auto ret = *this; + ret.leaf.pos++; + if (ret.at_boundary()) { + return seastar::do_with( + ret, + [c, visitor](auto &ret) mutable { + return ret.handle_boundary( + c, visitor ).si_then([&ret] { return std::move(ret); }); }); } else { - // end - auto ret = *this; - ret.leaf.pos = ret.leaf.node->get_size(); return iterator_fut( interruptible::ready_future_marker{}, ret); } + } LBABtree::iterator_fut LBABtree::iterator::prev(op_context_t c) const diff --git a/src/crimson/os/seastore/lba_manager/btree/lba_btree.h b/src/crimson/os/seastore/lba_manager/btree/lba_btree.h index 3364446862b22..10ec1a6b04c62 100644 --- a/src/crimson/os/seastore/lba_manager/btree/lba_btree.h +++ b/src/crimson/os/seastore/lba_manager/btree/lba_btree.h @@ -139,6 +139,12 @@ class LBABtree { return leaf.pos == leaf.node->get_size(); } + using handle_boundary_ertr = base_iertr; + using handle_boundary_ret = handle_boundary_ertr::future<>; + handle_boundary_ret handle_boundary( + op_context_t c, + mapped_space_visitor_t *visitor); + depth_t check_split() const { if (!leaf.node->at_max_capacity()) { return 0; From 542d237ffef70b60f851320ac7432c331859d47e Mon Sep 17 00:00:00 2001 From: Samuel Just Date: Mon, 25 Oct 2021 06:26:05 +0000 Subject: [PATCH 0307/2866] crimson/os/seastore/.../lba_btree: fix lower_bound at_boundary() Clarify that lookup_depth_range may return an iterator at_boundary() and audity users. Fix LBABtree::lookup to correctly handle at_boundary() after the call to lookup_depth_range. Fixes: https://tracker.ceph.com/issues/52805 Signed-off-by: Samuel Just --- .../os/seastore/lba_manager/btree/lba_btree.cc | 3 +++ .../os/seastore/lba_manager/btree/lba_btree.h | 17 ++++++++++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/src/crimson/os/seastore/lba_manager/btree/lba_btree.cc b/src/crimson/os/seastore/lba_manager/btree/lba_btree.cc index ba166fb127ad0..3190d59d48b7a 100644 --- a/src/crimson/os/seastore/lba_manager/btree/lba_btree.cc +++ b/src/crimson/os/seastore/lba_manager/btree/lba_btree.cc @@ -44,6 +44,7 @@ LBABtree::iterator::handle_boundary_ret LBABtree::iterator::handle_boundary( } leaf.reset(); get_internal(depth_with_space).pos++; + // note, cannot result in at_boundary() by construction return lookup_depth_range( c, *this, depth_with_space - 1, 0, li, ll, visitor ); @@ -113,9 +114,11 @@ LBABtree::iterator_fut LBABtree::iterator::prev(op_context_t c) const } ret.leaf.reset(); ret.get_internal(depth_with_space).pos--; + // note, cannot result in at_boundary() by construction return lookup_depth_range( c, ret, depth_with_space - 1, 0, li, ll, nullptr ).si_then([&ret] { + assert(!ret.at_boundary()); return std::move(ret); }); }); diff --git a/src/crimson/os/seastore/lba_manager/btree/lba_btree.h b/src/crimson/os/seastore/lba_manager/btree/lba_btree.h index 10ec1a6b04c62..dedf8e239e42a 100644 --- a/src/crimson/os/seastore/lba_manager/btree/lba_btree.h +++ b/src/crimson/os/seastore/lba_manager/btree/lba_btree.h @@ -494,6 +494,14 @@ class LBABtree { }); } + /** + * lookup_depth_range + * + * Performs node lookups on depths [from, to) using li and ll to + * specific target at each level. Note, may leave the iterator + * at_boundary(), call handle_boundary() prior to returning out + * lf LBABtree. + */ using lookup_depth_range_iertr = base_iertr; using lookup_depth_range_ret = lookup_depth_range_iertr::future<>; template @@ -579,7 +587,14 @@ class LBABtree { 0, li, ll, - visitor); + visitor + ).si_then([c, visitor, &iter] { + if (iter.at_boundary()) { + return iter.handle_boundary(c, visitor); + } else { + return lookup_iertr::now(); + } + }); }).si_then([&iter] { return std::move(iter); }); From 617b87228f7f8b87dcaf406e39abe59691676b12 Mon Sep 17 00:00:00 2001 From: Samuel Just Date: Mon, 25 Oct 2021 07:47:16 +0000 Subject: [PATCH 0308/2866] crimson/os/seastore/.../lba_btree: fix min_capacity condition Reducing the size of split_merge_multi has an interesting side effect of causing removes to happen on some leaf nodes immediately after split. After split, child nodes would have size 72 or 73. At size 72, the node would be at_min_size() and a remove would put it below causing the at_min_size() condition to fail and hande_merge to misbehave. Replace at_min_capacity() with below_min_capacity(). below_min_capacity() will not be true for any child of a split, and asserts that the child is below capacity by no more than 1. Signed-off-by: Samuel Just --- .../seastore/lba_manager/btree/lba_btree.cc | 8 ++++---- .../os/seastore/lba_manager/btree/lba_btree.h | 4 ++-- .../lba_manager/btree/lba_btree_node.h | 20 +++++++++++++++---- 3 files changed, 22 insertions(+), 10 deletions(-) diff --git a/src/crimson/os/seastore/lba_manager/btree/lba_btree.cc b/src/crimson/os/seastore/lba_manager/btree/lba_btree.cc index 3190d59d48b7a..0263698b06968 100644 --- a/src/crimson/os/seastore/lba_manager/btree/lba_btree.cc +++ b/src/crimson/os/seastore/lba_manager/btree/lba_btree.cc @@ -666,7 +666,7 @@ LBABtree::handle_merge_ret merge_level( auto [liter, riter] = donor_is_left ? std::make_pair(donor_iter, iter) : std::make_pair(iter, donor_iter); - if (donor->at_min_capacity()) { + if (donor->below_min_capacity()) { auto replacement = l->make_full_merge(c, r); parent_pos.node->update( @@ -727,8 +727,8 @@ LBABtree::handle_merge_ret LBABtree::handle_merge( iterator &iter) { LOG_PREFIX(LBATree::handle_merge); - if (!iter.leaf.node->at_min_capacity() || - iter.get_depth() == 1) { + if (iter.get_depth() == 1 || + !iter.leaf.node->below_min_capacity()) { DEBUGT( "no need to merge leaf, leaf size {}, depth {}", c.trans, @@ -775,7 +775,7 @@ LBABtree::handle_merge_ret LBABtree::handle_merge( DEBUGT("no need to collapse root", c.trans); } return seastar::stop_iteration::yes; - } else if (pos.node->at_min_capacity()) { + } else if (pos.node->below_min_capacity()) { DEBUGT( "continuing, next node {} depth {} at min", c.trans, diff --git a/src/crimson/os/seastore/lba_manager/btree/lba_btree.h b/src/crimson/os/seastore/lba_manager/btree/lba_btree.h index dedf8e239e42a..078113c09d425 100644 --- a/src/crimson/os/seastore/lba_manager/btree/lba_btree.h +++ b/src/crimson/os/seastore/lba_manager/btree/lba_btree.h @@ -157,11 +157,11 @@ class LBABtree { } depth_t check_merge() const { - if (!leaf.node->at_min_capacity()) { + if (!leaf.node->below_min_capacity()) { return 0; } for (depth_t merge_from = 1; merge_from < get_depth(); ++merge_from) { - if (!get_internal(merge_from + 1).node->at_min_capacity()) + if (!get_internal(merge_from + 1).node->below_min_capacity()) return merge_from; } return get_depth(); diff --git a/src/crimson/os/seastore/lba_manager/btree/lba_btree_node.h b/src/crimson/os/seastore/lba_manager/btree/lba_btree_node.h index 72aebaa385950..09a9a1f44c665 100644 --- a/src/crimson/os/seastore/lba_manager/btree/lba_btree_node.h +++ b/src/crimson/os/seastore/lba_manager/btree/lba_btree_node.h @@ -313,12 +313,18 @@ struct LBAInternalNode resolve_relative_addrs(base); } + constexpr static size_t get_min_capacity() { + return (get_capacity() - 1) / 2; + } + bool at_max_capacity() const { + assert(get_size() <= get_capacity()); return get_size() == get_capacity(); } - bool at_min_capacity() const { - return get_size() == (get_capacity() / 2); + bool below_min_capacity() const { + assert(get_size() >= (get_min_capacity() - 1)); + return get_size() < get_min_capacity(); } }; using LBAInternalNodeRef = LBAInternalNode::Ref; @@ -530,12 +536,18 @@ struct LBALeafNode std::ostream &print_detail(std::ostream &out) const final; + constexpr static size_t get_min_capacity() { + return (get_capacity() - 1) / 2; + } + bool at_max_capacity() const { + assert(get_size() <= get_capacity()); return get_size() == get_capacity(); } - bool at_min_capacity() const { - return get_size() == (get_capacity() / 2); + bool below_min_capacity() const { + assert(get_size() >= (get_min_capacity() - 1)); + return get_size() < get_min_capacity(); } }; using LBALeafNodeRef = TCachedExtentRef; From e6b17cde8b849b5463e06c2e16bb5944f3081021 Mon Sep 17 00:00:00 2001 From: Paul Cuzner Date: Tue, 26 Oct 2021 10:42:49 +1300 Subject: [PATCH 0309/2866] mgr/prometheus: update promtool testcase location Signed-off-by: Paul Cuzner --- src/test/CMakeLists.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/test/CMakeLists.txt b/src/test/CMakeLists.txt index 70b814c9ce206..0278e74f9900f 100644 --- a/src/test/CMakeLists.txt +++ b/src/test/CMakeLists.txt @@ -1,6 +1,6 @@ include(AddCephTest) -set(UNITTEST_LIBS GMock::Main GMock::GMock GTest::GTest ${CMAKE_THREAD_LIBS_INIT} +set(UNITTEST_LIBS GMock::Main GMock::GMock GTest::GTest ${CMAKE_THREAD_LIBS_INIT} ${GSSAPI_LIBRARIES} ${CMAKE_DL_LIBS}) if(WIN32) @@ -163,7 +163,7 @@ endif() add_executable(ceph_test_mutate test_mutate.cc ) -target_link_libraries(ceph_test_mutate global librados ${BLKID_LIBRARIES} +target_link_libraries(ceph_test_mutate global librados ${BLKID_LIBRARIES} ${CMAKE_DL_LIBS}) if(NOT WIN32) @@ -620,7 +620,7 @@ if(PROMTOOL_EXECUTABLE) OUTPUT_QUIET) if(NOT rc) add_ceph_test(run-promtool-unittests - ${PROMTOOL_EXECUTABLE} test rules ${CMAKE_SOURCE_DIR}/monitoring/prometheus/alerts/test_alerts.yml) + ${PROMTOOL_EXECUTABLE} test rules ${CMAKE_SOURCE_DIR}/monitoring/prometheus/tests/test_alerts.yml) elseif(NOT promtool_executable_checked) message(WARNING "'${PROMTOOL_EXECUTABLE} test rules' does not work, " "please use a newer prometheus") From f127e0829b3c512a380bedca46b868fc0c0c7f56 Mon Sep 17 00:00:00 2001 From: Sebastian Wagner Date: Wed, 20 Oct 2021 15:41:18 +0200 Subject: [PATCH 0310/2866] mgr/cephadm: Fix RGW ipv6 frontend configuration Signed-off-by: Sebastian Wagner --- .../mgr/cephadm/services/cephadmservice.py | 11 ++++-- src/pybind/mgr/cephadm/tests/test_services.py | 39 ++++++++++++++++++- 2 files changed, 44 insertions(+), 6 deletions(-) diff --git a/src/pybind/mgr/cephadm/services/cephadmservice.py b/src/pybind/mgr/cephadm/services/cephadmservice.py index f611a2ec2f913..2dbfb67190761 100644 --- a/src/pybind/mgr/cephadm/services/cephadmservice.py +++ b/src/pybind/mgr/cephadm/services/cephadmservice.py @@ -10,6 +10,7 @@ from ceph.deployment.service_spec import ServiceSpec, RGWSpec from ceph.deployment.utils import is_ipv6, unwrap_ipv6 +from mgr_util import build_url from orchestrator import OrchestratorError, DaemonDescription, DaemonDescriptionStatus from orchestrator._interface import daemon_type_to_service from cephadm import utils @@ -822,25 +823,27 @@ def prepare_create(self, daemon_spec: CephadmDaemonDeploySpec) -> CephadmDaemonD if ftype == 'beast': if spec.ssl: if daemon_spec.ip: - args.append(f"ssl_endpoint={daemon_spec.ip}:{port}") + args.append( + f"ssl_endpoint={build_url(host=daemon_spec.ip, port=port).lstrip('/')}") else: args.append(f"ssl_port={port}") args.append(f"ssl_certificate=config://rgw/cert/{spec.service_name()}") else: if daemon_spec.ip: - args.append(f"endpoint={daemon_spec.ip}:{port}") + args.append(f"endpoint={build_url(host=daemon_spec.ip, port=port).lstrip('/')}") else: args.append(f"port={port}") elif ftype == 'civetweb': if spec.ssl: if daemon_spec.ip: - args.append(f"port={daemon_spec.ip}:{port}s") # note the 's' suffix on port + # note the 's' suffix on port + args.append(f"port={build_url(host=daemon_spec.ip, port=port).lstrip('/')}s") else: args.append(f"port={port}s") # note the 's' suffix on port args.append(f"ssl_certificate=config://rgw/cert/{spec.service_name()}") else: if daemon_spec.ip: - args.append(f"port={daemon_spec.ip}:{port}") + args.append(f"port={build_url(host=daemon_spec.ip, port=port).lstrip('/')}") else: args.append(f"port={port}") frontend = f'{ftype} {" ".join(args)}' diff --git a/src/pybind/mgr/cephadm/tests/test_services.py b/src/pybind/mgr/cephadm/tests/test_services.py index b47e6f732f4e3..5180d363d0d02 100644 --- a/src/pybind/mgr/cephadm/tests/test_services.py +++ b/src/pybind/mgr/cephadm/tests/test_services.py @@ -15,8 +15,9 @@ from cephadm.services.monitoring import GrafanaService, AlertmanagerService, PrometheusService, \ NodeExporterService from cephadm.module import CephadmOrchestrator -from ceph.deployment.service_spec import IscsiServiceSpec, MonitoringSpec, AlertManagerSpec, ServiceSpec -from cephadm.tests.fixtures import with_host, with_service +from ceph.deployment.service_spec import IscsiServiceSpec, MonitoringSpec, AlertManagerSpec, \ + ServiceSpec, RGWSpec +from cephadm.tests.fixtures import with_host, with_service, _run_cephadm from orchestrator import OrchestratorError from orchestrator._interface import DaemonDescription @@ -419,3 +420,37 @@ def test_monitoring_ports(self, _run_cephadm, cephadm_module: CephadmOrchestrato ], stdin='{}', image='') + + +class TestRGWService: + + @pytest.mark.parametrize( + "frontend, ssl, expected", + [ + ('beast', False, 'beast endpoint=[fd00:fd00:fd00:3000::1]:80'), + ('beast', True, + 'beast ssl_endpoint=[fd00:fd00:fd00:3000::1]:443 ssl_certificate=config://rgw/cert/rgw.foo'), + ('civetweb', False, 'civetweb port=[fd00:fd00:fd00:3000::1]:80'), + ('civetweb', True, + 'civetweb port=[fd00:fd00:fd00:3000::1]:443s ssl_certificate=config://rgw/cert/rgw.foo'), + ] + ) + @patch("cephadm.serve.CephadmServe._run_cephadm", _run_cephadm('{}')) + def test_rgw_update(self, frontend, ssl, expected, cephadm_module: CephadmOrchestrator): + with with_host(cephadm_module, 'host1'): + cephadm_module.cache.update_host_networks('host1', { + 'fd00:fd00:fd00:3000::/64': { + 'if0': ['fd00:fd00:fd00:3000::1'] + } + }) + s = RGWSpec(service_id="foo", + networks=['fd00:fd00:fd00:3000::/64'], + ssl=ssl, + rgw_frontend_type=frontend) + with with_service(cephadm_module, s) as dds: + _, f, _ = cephadm_module.check_mon_command({ + 'prefix': 'config get', + 'who': f'client.{dds[0]}', + 'key': 'rgw_frontends', + }) + assert f == expected From 84dce06aa85c2a20ec81b7be38aeed8e83615415 Mon Sep 17 00:00:00 2001 From: Sebastian Wagner Date: Wed, 20 Oct 2021 14:45:03 +0200 Subject: [PATCH 0311/2866] doc/cephadm: indent yaml specs properly conform with other service types Signed-off-by: Sebastian Wagner --- doc/cephadm/services/osd.rst | 107 +++++++++++++++++++---------------- 1 file changed, 59 insertions(+), 48 deletions(-) diff --git a/doc/cephadm/services/osd.rst b/doc/cephadm/services/osd.rst index 4347d05aff5b7..ede66efc85b4e 100644 --- a/doc/cephadm/services/osd.rst +++ b/doc/cephadm/services/osd.rst @@ -437,11 +437,12 @@ Create a file called (for example) ``osd_spec.yml``: .. code-block:: yaml service_type: osd - service_id: default_drive_group <- name of the drive_group (name can be custom) + service_id: default_drive_group # custom name of the osd spec placement: - host_pattern: '*' <- which hosts to target, currently only supports globs - data_devices: <- the type of devices you are applying specs to - all: true <- a filter, check below for a full list + host_pattern: '*' # which hosts to target + spec: + data_devices: # the type of devices you are applying specs to + all: true # a filter, check below for a full list This means : @@ -621,9 +622,10 @@ This example would deploy all OSDs with encryption enabled. service_id: example_osd_spec placement: host_pattern: '*' - data_devices: - all: true - encrypted: true + spec: + data_devices: + all: true + encrypted: true See a full list in the DriveGroupSpecs @@ -661,10 +663,11 @@ This is a common setup and can be described quite easily: service_id: osd_spec_default placement: host_pattern: '*' - data_devices: - model: HDD-123-foo <- note that HDD-123 would also be valid - db_devices: - model: MC-55-44-XZ <- same here, MC-55-44 is valid + spec: + data_devices: + model: HDD-123-foo <- note that HDD-123 would also be valid + db_devices: + model: MC-55-44-XZ <- same here, MC-55-44 is valid However, we can improve it by reducing the filters on core properties of the drives: @@ -674,10 +677,11 @@ However, we can improve it by reducing the filters on core properties of the dri service_id: osd_spec_default placement: host_pattern: '*' - data_devices: - rotational: 1 - db_devices: - rotational: 0 + spec: + data_devices: + rotational: 1 + db_devices: + rotational: 0 Now, we enforce all rotating devices to be declared as 'data devices' and all non-rotating devices will be used as shared_devices (wal, db) @@ -689,10 +693,11 @@ If you know that drives with more than 2TB will always be the slower data device service_id: osd_spec_default placement: host_pattern: '*' - data_devices: - size: '2TB:' - db_devices: - size: ':2TB' + spec: + data_devices: + size: '2TB:' + db_devices: + size: ':2TB' Note: All of the above DriveGroups are equally valid. Which of those you want to use depends on taste and on how much you expect your node layout to change. @@ -731,20 +736,22 @@ This can be described with two layouts. service_id: osd_spec_hdd placement: host_pattern: '*' - data_devices: - rotational: 0 - db_devices: - model: MC-55-44-XZ - limit: 2 (db_slots is actually to be favoured here, but it's not implemented yet) + spec: + data_devices: + rotational: 0 + db_devices: + model: MC-55-44-XZ + limit: 2 (db_slots is actually to be favoured here, but it's not implemented yet) --- service_type: osd service_id: osd_spec_ssd placement: host_pattern: '*' - data_devices: - model: MC-55-44-XZ - db_devices: - vendor: VendorC + spec: + data_devices: + model: MC-55-44-XZ + db_devices: + vendor: VendorC This would create the desired layout by using all HDDs as data_devices with two SSD assigned as dedicated db/wal devices. The remaining SSDs(8) will be data_devices that have the 'VendorC' NVMEs assigned as dedicated db/wal devices. @@ -789,19 +796,21 @@ You can use the 'host_pattern' key in the layout to target certain nodes. Salt t service_id: osd_spec_node_one_to_five placement: host_pattern: 'node[1-5]' - data_devices: - rotational: 1 - db_devices: - rotational: 0 + spec: + data_devices: + rotational: 1 + db_devices: + rotational: 0 --- service_type: osd service_id: osd_spec_six_to_ten placement: host_pattern: 'node[6-10]' - data_devices: - model: MC-55-44-XZ - db_devices: - model: SSD-123-foo + spec: + data_devices: + model: MC-55-44-XZ + db_devices: + model: SSD-123-foo This applies different OSD specs to different hosts depending on the `host_pattern` key. @@ -837,12 +846,13 @@ The OSD spec for this case would look like the following (using the `model` filt service_id: osd_spec_default placement: host_pattern: '*' - data_devices: - model: MC-55-44-XZ - db_devices: - model: SSD-123-foo - wal_devices: - model: NVME-QQQQ-987 + spec: + data_devices: + model: MC-55-44-XZ + db_devices: + model: SSD-123-foo + wal_devices: + model: NVME-QQQQ-987 It is also possible to specify directly device paths in specific hosts like the following: @@ -855,14 +865,15 @@ It is also possible to specify directly device paths in specific hosts like the hosts: - Node01 - Node02 - data_devices: - paths: + spec: + data_devices: + paths: - /dev/sdb - db_devices: - paths: + db_devices: + paths: - /dev/sdc - wal_devices: - paths: + wal_devices: + paths: - /dev/sdd From 358a5ae588d0d98ba3a7f69d753aae2a639a6700 Mon Sep 17 00:00:00 2001 From: Sebastian Wagner Date: Wed, 20 Oct 2021 14:38:40 +0200 Subject: [PATCH 0312/2866] doc/cephadm: Add "Deploying OSDs on multiple similar hosts" Signed-off-by: Sebastian Wagner --- doc/cephadm/services/osd.rst | 44 ++++++++++++++++++++++++------------ 1 file changed, 29 insertions(+), 15 deletions(-) diff --git a/doc/cephadm/services/osd.rst b/doc/cephadm/services/osd.rst index ede66efc85b4e..0899af2edb85d 100644 --- a/doc/cephadm/services/osd.rst +++ b/doc/cephadm/services/osd.rst @@ -636,10 +636,10 @@ See a full list in the DriveGroupSpecs :exclude-members: from_json Examples --------- +======== The simple case -^^^^^^^^^^^^^^^ +--------------- All nodes with the same setup @@ -702,8 +702,8 @@ If you know that drives with more than 2TB will always be the slower data device Note: All of the above DriveGroups are equally valid. Which of those you want to use depends on taste and on how much you expect your node layout to change. -The advanced case -^^^^^^^^^^^^^^^^^ +Multiple OSD specs for a single host +------------------------------------ Here we have two distinct setups @@ -756,10 +756,18 @@ This can be described with two layouts. This would create the desired layout by using all HDDs as data_devices with two SSD assigned as dedicated db/wal devices. The remaining SSDs(8) will be data_devices that have the 'VendorC' NVMEs assigned as dedicated db/wal devices. -The advanced case (with non-uniform nodes) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Multiple hosts with the same disk layout +---------------------------------------- + +Assuming the cluster has different kinds of hosts each with similar disk +layout, it is recommended to apply different OSD specs matching only one +set of hosts. Typically you will have a spec for multiple hosts with the +same layout. -The examples above assumed that all nodes have the same drives. That's however not always the case. +The sevice id as the unique key: In case a new OSD spec with an already +applied service id is applied, the existing OSD spec will be superseeded. +cephadm will now create new OSD daemons based on the new spec +definition. Existing OSD daemons will not be affected. See :ref:`cephadm-osd-declarative`. Node1-5 @@ -787,15 +795,14 @@ Node6-10 Model: MC-55-44-ZX Size: 512GB -You can use the 'host_pattern' key in the layout to target certain nodes. Salt target notation helps to keep things easy. - +You can use the 'placement' key in the layout to target certain nodes. .. code-block:: yaml service_type: osd - service_id: osd_spec_node_one_to_five + service_id: disk_layout_a placement: - host_pattern: 'node[1-5]' + label: disk_layout_a spec: data_devices: rotational: 1 @@ -803,19 +810,26 @@ You can use the 'host_pattern' key in the layout to target certain nodes. Salt t rotational: 0 --- service_type: osd - service_id: osd_spec_six_to_ten + service_id: disk_layout_b placement: - host_pattern: 'node[6-10]' + label: disk_layout_b spec: data_devices: model: MC-55-44-XZ db_devices: model: SSD-123-foo -This applies different OSD specs to different hosts depending on the `host_pattern` key. +This applies different OSD specs to different hosts depending on the `placement` key. +See :ref:`orchestrator-cli-placement-spec` + +.. note:: + + Assuming each host has a unique disk layout, each OSD + spec needs to have a different service id + Dedicated wal + db -^^^^^^^^^^^^^^^^^^ +------------------ All previous cases co-located the WALs with the DBs. It's however possible to deploy the WAL on a dedicated device as well, if it makes sense. From 3b6ce664c3ed71aae9d5d4e182d42efe402a3161 Mon Sep 17 00:00:00 2001 From: Sebastian Wagner Date: Tue, 26 Oct 2021 11:27:51 +0200 Subject: [PATCH 0313/2866] doc/cephadm: osd.rst: Fix yaml comments Signed-off-by: Sebastian Wagner --- doc/cephadm/services/osd.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/cephadm/services/osd.rst b/doc/cephadm/services/osd.rst index 0899af2edb85d..aff13a2cef5ea 100644 --- a/doc/cephadm/services/osd.rst +++ b/doc/cephadm/services/osd.rst @@ -665,9 +665,9 @@ This is a common setup and can be described quite easily: host_pattern: '*' spec: data_devices: - model: HDD-123-foo <- note that HDD-123 would also be valid + model: HDD-123-foo # Note, HDD-123 would also be valid db_devices: - model: MC-55-44-XZ <- same here, MC-55-44 is valid + model: MC-55-44-XZ # Same here, MC-55-44 is valid However, we can improve it by reducing the filters on core properties of the drives: @@ -741,7 +741,7 @@ This can be described with two layouts. rotational: 0 db_devices: model: MC-55-44-XZ - limit: 2 (db_slots is actually to be favoured here, but it's not implemented yet) + limit: 2 # db_slots is actually to be favoured here, but it's not implemented yet --- service_type: osd service_id: osd_spec_ssd From 61fe2a21b7d3869a156b1994696b38fbb6587be9 Mon Sep 17 00:00:00 2001 From: Sebastian Wagner Date: Tue, 26 Oct 2021 11:31:14 +0200 Subject: [PATCH 0314/2866] doc/cephadm: osd.rst: s/DriveGroup/OSD spec/ Signed-off-by: Sebastian Wagner --- doc/cephadm/services/osd.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/cephadm/services/osd.rst b/doc/cephadm/services/osd.rst index aff13a2cef5ea..8739ad11c5f20 100644 --- a/doc/cephadm/services/osd.rst +++ b/doc/cephadm/services/osd.rst @@ -358,7 +358,7 @@ Example command: .. note:: If the unmanaged flag is unset, cephadm automatically deploys drives that - match the DriveGroup in your OSDSpec. For example, if you use the + match the OSDSpec. For example, if you use the ``all-available-devices`` option when creating OSDs, when you ``zap`` a device the cephadm orchestrator automatically creates a new OSD in the device. To disable this behavior, see :ref:`cephadm-osd-declarative`. @@ -612,7 +612,7 @@ Additional Options ------------------ There are multiple optional settings you can use to change the way OSDs are deployed. -You can add these options to the base level of a DriveGroup for it to take effect. +You can add these options to the base level of an OSD spec for it to take effect. This example would deploy all OSDs with encryption enabled. @@ -699,7 +699,7 @@ If you know that drives with more than 2TB will always be the slower data device db_devices: size: ':2TB' -Note: All of the above DriveGroups are equally valid. Which of those you want to use depends on taste and on how much you expect your node layout to change. +Note: All of the above OSD specs are equally valid. Which of those you want to use depends on taste and on how much you expect your node layout to change. Multiple OSD specs for a single host From d749453ce4f9cd8f88bb70cb1421fb1963881248 Mon Sep 17 00:00:00 2001 From: Sebastian Wagner Date: Tue, 26 Oct 2021 11:37:40 +0200 Subject: [PATCH 0315/2866] doc/cephadm: osd.rst: s/Note, /.. note:: / Signed-off-by: Sebastian Wagner --- doc/cephadm/services/osd.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/cephadm/services/osd.rst b/doc/cephadm/services/osd.rst index 8739ad11c5f20..0afe9f8d587f4 100644 --- a/doc/cephadm/services/osd.rst +++ b/doc/cephadm/services/osd.rst @@ -580,7 +580,7 @@ All This will take all disks that are 'available' -Note: This is exclusive for the data_devices section. +.. note:: This is exclusive for the data_devices section. .. code-block:: yaml @@ -605,7 +605,7 @@ but want to use only the first two, you could use `limit`: vendor: VendorA limit: 2 -Note: `limit` is a last resort and shouldn't be used if it can be avoided. +.. note:: `limit` is a last resort and shouldn't be used if it can be avoided. Additional Options @@ -699,7 +699,7 @@ If you know that drives with more than 2TB will always be the slower data device db_devices: size: ':2TB' -Note: All of the above OSD specs are equally valid. Which of those you want to use depends on taste and on how much you expect your node layout to change. +.. note:: All of the above OSD specs are equally valid. Which of those you want to use depends on taste and on how much you expect your node layout to change. Multiple OSD specs for a single host From eff5c76d885c8ebb3199db362730a7b02796c367 Mon Sep 17 00:00:00 2001 From: Sebastian Wagner Date: Tue, 28 Sep 2021 14:15:13 +0200 Subject: [PATCH 0316/2866] doc/cephadm/operations: extend CEPHADM_STRAY_DAEMON Signed-off-by: Sebastian Wagner --- doc/cephadm/operations.rst | 58 +++++++++++++++++++++++--------------- 1 file changed, 36 insertions(+), 22 deletions(-) diff --git a/doc/cephadm/operations.rst b/doc/cephadm/operations.rst index 45ae1575b7540..5ccbc8c4bc12a 100644 --- a/doc/cephadm/operations.rst +++ b/doc/cephadm/operations.rst @@ -178,30 +178,30 @@ running, but are not registered as hosts managed by *cephadm*. This means that those services cannot currently be managed by cephadm (e.g., restarted, upgraded, included in `ceph orch ps`). -You can manage the host(s) by running the following command: +* You can manage the host(s) by running the following command: -.. prompt:: bash # + .. prompt:: bash # - ceph orch host add ** + ceph orch host add ** -.. note:: + .. note:: - You might need to configure SSH access to the remote host - before this will work. + You might need to configure SSH access to the remote host + before this will work. -Alternatively, you can manually connect to the host and ensure that -services on that host are removed or migrated to a host that is -managed by *cephadm*. +* See :ref:`cephadm-fqdn` for more information about host names and + domain names. -This warning can be disabled entirely by running the following -command: +* Alternatively, you can manually connect to the host and ensure that + services on that host are removed or migrated to a host that is + managed by *cephadm*. -.. prompt:: bash # +* This warning can be disabled entirely by running the following + command: - ceph config set mgr mgr/cephadm/warn_on_stray_hosts false + .. prompt:: bash # -See :ref:`cephadm-fqdn` for more information about host names and -domain names. + ceph config set mgr mgr/cephadm/warn_on_stray_hosts false CEPHADM_STRAY_DAEMON ~~~~~~~~~~~~~~~~~~~~ @@ -212,16 +212,30 @@ tool, or because they were started manually. Those services cannot currently be managed by cephadm (e.g., restarted, upgraded, or included in `ceph orch ps`). -If the daemon is a stateful one (monitor or OSD), it should be adopted -by cephadm; see :ref:`cephadm-adoption`. For stateless daemons, it is -usually easiest to provision a new daemon with the ``ceph orch apply`` -command and then stop the unmanaged daemon. +* If the daemon is a stateful one (monitor or OSD), it should be adopted + by cephadm; see :ref:`cephadm-adoption`. For stateless daemons, it is + usually easiest to provision a new daemon with the ``ceph orch apply`` + command and then stop the unmanaged daemon. -This warning can be disabled entirely by running the following command: +* If the stray daemon(s) are running on hosts not managed by cephadm, you can manage the host(s) by running the following command: -.. prompt:: bash # + .. prompt:: bash # + + ceph orch host add ** + + .. note:: + + You might need to configure SSH access to the remote host + before this will work. + +* See :ref:`cephadm-fqdn` for more information about host names and + domain names. + +* This warning can be disabled entirely by running the following command: + + .. prompt:: bash # - ceph config set mgr mgr/cephadm/warn_on_stray_daemons false + ceph config set mgr mgr/cephadm/warn_on_stray_daemons false CEPHADM_HOST_CHECK_FAILED ~~~~~~~~~~~~~~~~~~~~~~~~~ From 81430251edbb07e73155c7b194cfe3d6d60c7905 Mon Sep 17 00:00:00 2001 From: Nizamudeen A Date: Tue, 26 Oct 2021 13:16:51 +0530 Subject: [PATCH 0317/2866] mgr/dashboard: Cluster expansion review page minor bug fixes Fix the Total Memory coming up as NaN Undefined Remove `Add` Button in the Cluster Details table Fixes: https://tracker.ceph.com/issues/53038 Signed-off-by: Nizamudeen A --- .../create-cluster/create-cluster-review.component.html | 1 + .../cluster/create-cluster/create-cluster-review.component.ts | 1 - .../frontend/src/app/ceph/cluster/hosts/hosts.component.html | 3 ++- .../frontend/src/app/ceph/cluster/hosts/hosts.component.ts | 3 +++ 4 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster-review.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster-review.component.html index f59cf6baceb51..812598f6ccab0 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster-review.component.html +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/create-cluster/create-cluster-review.component.html @@ -42,6 +42,7 @@ Host Details { this.hostsCountPerService.push({ diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/hosts.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/hosts.component.html index a4d30918cc562..3b310d4565027 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/hosts.component.html +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/hosts.component.html @@ -13,7 +13,8 @@ selectionType="single" [hasDetails]="hasTableDetails" (setExpandedRow)="setExpandedRow($event)" - (updateSelection)="updateSelection($event)"> + (updateSelection)="updateSelection($event)" + [toolHeader]="!hideToolHeader">
    Date: Wed, 20 Oct 2021 19:37:26 +0530 Subject: [PATCH 0318/2866] mgr/dashboard: gather facts should only be fetched when orch backend is cephadm Fixes: https://tracker.ceph.com/issues/52981 Signed-off-by: Avan Thakkar Gather facts in UI should only be fetched if there is orch available and get_facts feature is available for that orch backend. --- src/pybind/mgr/dashboard/controllers/host.py | 15 ++++++++-- .../cluster/hosts/hosts.component.spec.ts | 30 +++++++++++++++++-- .../app/ceph/cluster/hosts/hosts.component.ts | 17 +++++++++-- .../app/shared/models/orchestrator.enum.ts | 1 + src/pybind/mgr/dashboard/tests/test_host.py | 12 ++++++-- 5 files changed, 65 insertions(+), 10 deletions(-) diff --git a/src/pybind/mgr/dashboard/controllers/host.py b/src/pybind/mgr/dashboard/controllers/host.py index 0a897002a4a1c..9d63d0a96ef1f 100644 --- a/src/pybind/mgr/dashboard/controllers/host.py +++ b/src/pybind/mgr/dashboard/controllers/host.py @@ -287,9 +287,18 @@ def list(self, sources=None, facts=False): hosts = get_hosts(sources) orch = OrchClient.instance() if str_to_bool(facts): - if orch.available(): - hosts_facts = orch.hosts.get_facts() - return merge_list_of_dicts_by_key(hosts, hosts_facts, 'hostname') + if orch.available(['get_facts']): + try: + hosts_facts = orch.hosts.get_facts() + return merge_list_of_dicts_by_key(hosts, hosts_facts, 'hostname') + + except Exception: + raise DashboardException( + code='invalid_orchestrator_backend', # pragma: no cover + msg="Please enable the cephadm orchestrator backend " + "(try `ceph orch set backend cephadm`)", + component='orchestrator', + http_status_code=400) raise DashboardException(code='orchestrator_status_unavailable', # pragma: no cover msg="Please configure and enable the orchestrator if you " diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/hosts.component.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/hosts.component.spec.ts index 2c6a4db6983a1..4535a55911b87 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/hosts.component.spec.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/hosts.component.spec.ts @@ -104,7 +104,7 @@ describe('HostsComponent', () => { } ]; - OrchestratorHelper.mockStatus(true); + OrchestratorHelper.mockStatus(false); hostListSpy.and.callFake(() => of(payload)); fixture.detectChanges(); @@ -118,6 +118,7 @@ describe('HostsComponent', () => { }); it('should test if host facts are tranformed correctly if orch available', () => { + const features = [OrchestratorFeature.HOST_FACTS]; const payload = [ { hostname: 'host_test', @@ -137,7 +138,7 @@ describe('HostsComponent', () => { nic_count: 1 } ]; - OrchestratorHelper.mockStatus(true); + OrchestratorHelper.mockStatus(true, features); hostListSpy.and.callFake(() => of(payload)); fixture.detectChanges(); @@ -177,6 +178,31 @@ describe('HostsComponent', () => { expect(spans[7].textContent).toBe('Unavailable'); }); + it('should test if host facts are unavailable if get_fatcs orch feature is not available', () => { + const payload = [ + { + hostname: 'host_test', + services: [ + { + type: 'osd', + id: '0' + } + ] + } + ]; + OrchestratorHelper.mockStatus(true); + hostListSpy.and.callFake(() => of(payload)); + fixture.detectChanges(); + + component.getHosts(new CdTableFetchDataContext(() => undefined)); + fixture.detectChanges(); + + const spans = fixture.debugElement.nativeElement.querySelectorAll( + '.datatable-body-cell-label span' + ); + expect(spans[7].textContent).toBe('Unavailable'); + }); + it('should show force maintenance modal when it is safe to stop host', () => { const errorMsg = `WARNING: Stopping 1 out of 1 daemons in Grafana service. Service will not be operational with no daemons left. At diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/hosts.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/hosts.component.ts index fe0b797bd1d7c..4e9c36b8e5c09 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/hosts.component.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/hosts.component.ts @@ -389,8 +389,19 @@ export class HostsComponent extends ListWithDetails implements OnDestroy, OnInit }); } + checkHostsFactsAvailable() { + const orchFeatures = this.orchStatus.features; + if (!_.isEmpty(orchFeatures)) { + if (orchFeatures.get_facts.available) { + return true; + } + return false; + } + return false; + } + transformHostsData() { - if (this.orchStatus?.available) { + if (this.checkHostsFactsAvailable()) { _.forEach(this.hosts, (hostKey) => { hostKey['memory_total_bytes'] = hostKey['memory_total_kb'] * 1024; hostKey['raw_capacity'] = hostKey['hdd_capacity_bytes'] + hostKey['flash_capacity_bytes']; @@ -423,8 +434,8 @@ export class HostsComponent extends ListWithDetails implements OnDestroy, OnInit .pipe( mergeMap((orchStatus) => { this.orchStatus = orchStatus; - - return this.hostService.list(`${this.orchStatus?.available}`); + const factsAvailable = this.checkHostsFactsAvailable(); + return this.hostService.list(`${factsAvailable}`); }), map((hostList: object[]) => hostList.map((host) => { diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/models/orchestrator.enum.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/models/orchestrator.enum.ts index cf4ba76a9cf09..545d072fc45f4 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/shared/models/orchestrator.enum.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/models/orchestrator.enum.ts @@ -6,6 +6,7 @@ export enum OrchestratorFeature { HOST_LABEL_REMOVE = 'remove_host_label', HOST_MAINTENANCE_ENTER = 'enter_host_maintenance', HOST_MAINTENANCE_EXIT = 'exit_host_maintenance', + HOST_FACTS = 'get_facts', SERVICE_LIST = 'describe_service', SERVICE_CREATE = 'apply', diff --git a/src/pybind/mgr/dashboard/tests/test_host.py b/src/pybind/mgr/dashboard/tests/test_host.py index a74d7d8e0aa02..e15885a56951a 100644 --- a/src/pybind/mgr/dashboard/tests/test_host.py +++ b/src/pybind/mgr/dashboard/tests/test_host.py @@ -13,12 +13,13 @@ @contextlib.contextmanager -def patch_orch(available: bool, hosts: Optional[List[HostSpec]] = None, +def patch_orch(available: bool, missing_features: Optional[List[str]] = None, + hosts: Optional[List[HostSpec]] = None, inventory: Optional[List[dict]] = None): with mock.patch('dashboard.controllers.orchestrator.OrchClient.instance') as instance: fake_client = mock.Mock() fake_client.available.return_value = available - fake_client.get_missing_features.return_value = [] + fake_client.get_missing_features.return_value = missing_features if hosts is not None: fake_client.hosts.list.return_value = hosts @@ -161,6 +162,13 @@ def test_host_list_with_facts(self, mock_get_hosts): 'application/vnd.ceph.api.v1.1+json') self.assertJsonBody(hosts_without_facts) + # test with orchestrator available but orch backend!=cephadm + with patch_orch(True, missing_features=['get_facts']) as fake_client: + mock_get_hosts.return_value = hosts_without_facts + # test with ?facts=true + self._get('{}?facts=true'.format(self.URL_HOST), version=APIVersion(1, 1)) + self.assertStatus(400) + # test with no orchestrator available with patch_orch(False): mock_get_hosts.return_value = hosts_without_facts From 8a1aa9c4dbddb9ccc4331c9310fb2b685b9d8122 Mon Sep 17 00:00:00 2001 From: Adam King Date: Tue, 26 Oct 2021 07:58:15 -0400 Subject: [PATCH 0319/2866] mgr/cephadm: add host to get_daemons_by_type, metadata_up_to_date to refresh functions Signed-off-by: Adam King --- src/pybind/mgr/cephadm/agent.py | 8 +++----- src/pybind/mgr/cephadm/inventory.py | 18 +++++++++++++----- src/pybind/mgr/cephadm/serve.py | 27 ++++++--------------------- 3 files changed, 22 insertions(+), 31 deletions(-) diff --git a/src/pybind/mgr/cephadm/agent.py b/src/pybind/mgr/cephadm/agent.py index 8998c1e63f561..cb5f9acdf05eb 100644 --- a/src/pybind/mgr/cephadm/agent.py +++ b/src/pybind/mgr/cephadm/agent.py @@ -325,7 +325,7 @@ def _agent_down(self, host: str) -> bool: if host not in [h.hostname for h in self.mgr.cache.get_non_draining_hosts()]: return False # if we haven't deployed an agent on the host yet, don't say an agent is down - if not [a for a in self.mgr.cache.get_daemons_by_type('agent') if a.hostname == host]: + if not self.mgr.cache.get_daemons_by_type('agent', host=host): return False # if we don't have a timestamp, it's likely because of a mgr fail over. # just set the timestamp to now. However, if host was offline before, we @@ -421,8 +421,7 @@ def _check_agent(self, host: str) -> bool: try: # try to schedule redeploy of agent in case it is individually down - agent = [a for a in self.mgr.cache.get_daemons_by_type( - 'agent') if a.hostname == host][0] + agent = self.mgr.cache.get_daemons_by_type('agent', host=host)[0] with self.mgr.agent_helpers.agent_lock(host): daemon_spec = CephadmDaemonDeploySpec.from_daemon_description(agent) self.mgr._daemon_action(daemon_spec, action='redeploy') @@ -432,8 +431,7 @@ def _check_agent(self, host: str) -> bool: return True else: try: - agent = [a for a in self.mgr.cache.get_daemons_by_type( - 'agent') if a.hostname == host][0] + agent = self.mgr.cache.get_daemons_by_type('agent', host=host)[0] assert agent.daemon_id is not None assert agent.hostname is not None except Exception as e: diff --git a/src/pybind/mgr/cephadm/inventory.py b/src/pybind/mgr/cephadm/inventory.py index 2336706ac589f..62a055d6bbc85 100644 --- a/src/pybind/mgr/cephadm/inventory.py +++ b/src/pybind/mgr/cephadm/inventory.py @@ -817,8 +817,7 @@ def get_daemons(self): r.append(dd) return r - def get_error_daemons(self): - # type: () -> List[orchestrator.DaemonDescription] + def get_error_daemons(self) -> List[orchestrator.DaemonDescription]: r = [] for host, dm in self.daemons.items(): for name, dd in dm.items(): @@ -866,12 +865,13 @@ def get_daemons_by_service(self, service_name): result.append(d) return result - def get_daemons_by_type(self, service_type): - # type: (str) -> List[orchestrator.DaemonDescription] + def get_daemons_by_type(self, service_type: str, host: str = '') -> List[orchestrator.DaemonDescription]: assert service_type not in ['keepalived', 'haproxy'] result = [] # type: List[orchestrator.DaemonDescription] - for host, dm in self.daemons.items(): + for h, dm in self.daemons.items(): + if host and h != host: + continue for name, d in dm.items(): if d.daemon_type in service_to_daemon_types(service_type): result.append(d) @@ -915,6 +915,8 @@ def host_needs_daemon_refresh(self, host): seconds=self.mgr.daemon_cache_timeout) if host not in self.last_daemon_update or self.last_daemon_update[host] < cutoff: return True + if not self.mgr.cache.host_metadata_up_to_date(host): + return True return False def host_needs_facts_refresh(self, host): @@ -926,6 +928,8 @@ def host_needs_facts_refresh(self, host): seconds=self.mgr.facts_cache_timeout) if host not in self.last_facts_update or self.last_facts_update[host] < cutoff: return True + if not self.mgr.cache.host_metadata_up_to_date(host): + return True return False def host_needs_autotune_memory(self, host): @@ -961,6 +965,8 @@ def host_needs_device_refresh(self, host): seconds=self.mgr.device_cache_timeout) if host not in self.last_device_update or self.last_device_update[host] < cutoff: return True + if not self.mgr.cache.host_metadata_up_to_date(host): + return True return False def host_needs_network_refresh(self, host): @@ -975,6 +981,8 @@ def host_needs_network_refresh(self, host): seconds=self.mgr.device_cache_timeout) if host not in self.last_network_update or self.last_network_update[host] < cutoff: return True + if not self.mgr.cache.host_metadata_up_to_date(host): + return True return False def host_needs_osdspec_preview_refresh(self, host: str) -> bool: diff --git a/src/pybind/mgr/cephadm/serve.py b/src/pybind/mgr/cephadm/serve.py index 4a721b50fa301..e95f4b0c8e6aa 100644 --- a/src/pybind/mgr/cephadm/serve.py +++ b/src/pybind/mgr/cephadm/serve.py @@ -261,47 +261,32 @@ def refresh(host: str) -> None: or host not in [h.hostname for h in self.mgr.cache.get_non_draining_hosts()] or host in agents_down ): - if ( - self.mgr.cache.host_needs_daemon_refresh(host) - or not self.mgr.cache.host_metadata_up_to_date(host) - ): + if self.mgr.cache.host_needs_daemon_refresh(host): self.log.debug('refreshing %s daemons' % host) r = self._refresh_host_daemons(host) if r: failures.append(r) - if ( - self.mgr.cache.host_needs_facts_refresh(host) - or not self.mgr.cache.host_metadata_up_to_date(host) - ): + if self.mgr.cache.host_needs_facts_refresh(host): self.log.debug(('Refreshing %s facts' % host)) r = self._refresh_facts(host) if r: failures.append(r) - if ( - self.mgr.cache.host_needs_network_refresh(host) - or not self.mgr.cache.host_metadata_up_to_date(host) - ): + if self.mgr.cache.host_needs_network_refresh(host): self.log.debug(('Refreshing %s networks' % host)) r = self._refresh_host_networks(host) if r: failures.append(r) - if ( - self.mgr.cache.host_needs_device_refresh(host) - or not self.mgr.cache.host_metadata_up_to_date(host) - ): + if self.mgr.cache.host_needs_device_refresh(host): self.log.debug('refreshing %s devices' % host) r = self._refresh_host_devices(host) if r: failures.append(r) self.mgr.cache.metadata_up_to_date[host] = True - elif not [a for a in self.mgr.cache.get_daemons_by_type('agent') if a.hostname == host]: - if ( - self.mgr.cache.host_needs_daemon_refresh(host) - or not self.mgr.cache.host_metadata_up_to_date(host) - ): + elif not self.mgr.cache.get_daemons_by_type('agent', host=host): + if self.mgr.cache.host_needs_daemon_refresh(host): self.log.debug('refreshing %s daemons' % host) r = self._refresh_host_daemons(host) if r: From edbb900d39c82424ee6d1b6db5be8efbe2a4c794 Mon Sep 17 00:00:00 2001 From: Adam King Date: Tue, 26 Oct 2021 08:30:11 -0400 Subject: [PATCH 0320/2866] mgr/cephadm: new exception type for when agent's lock in use Signed-off-by: Adam King --- src/pybind/mgr/cephadm/agent.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/pybind/mgr/cephadm/agent.py b/src/pybind/mgr/cephadm/agent.py index cb5f9acdf05eb..1de205b71df08 100644 --- a/src/pybind/mgr/cephadm/agent.py +++ b/src/pybind/mgr/cephadm/agent.py @@ -302,6 +302,10 @@ def run(self) -> None: return +class AgentLockException(Exception): + pass + + class CephadmAgentHelpers: def __init__(self, mgr: "CephadmOrchestrator"): self.mgr: "CephadmOrchestrator" = mgr @@ -398,7 +402,7 @@ def agent_lock(self, host: str) -> Iterator[threading.Lock]: self.mgr.agent_helpers.agent_locks[host] = threading.Lock() lock = self.mgr.agent_helpers.agent_locks[host] if not lock.acquire(False): - raise Exception('Agent lock in use') + raise AgentLockException() try: yield lock finally: @@ -425,6 +429,9 @@ def _check_agent(self, host: str) -> bool: with self.mgr.agent_helpers.agent_lock(host): daemon_spec = CephadmDaemonDeploySpec.from_daemon_description(agent) self.mgr._daemon_action(daemon_spec, action='redeploy') + except AgentLockException: + self.mgr.log.debug( + f'Could not redeploy agent on host {host}. Someone else holds agent\'s lock') except Exception as e: self.mgr.log.debug( f'Failed to redeploy agent on host {host}. Agent possibly never deployed: {e}') @@ -448,6 +455,9 @@ def _check_agent(self, host: str) -> bool: with self.mgr.agent_helpers.agent_lock(host): self.mgr._daemon_action(daemon_spec, action='reconfig') return False + except AgentLockException: + self.mgr.log.debug( + f'Could not reconfig agent on host {host}. Someone else holds agent\'s lock') except Exception as e: self.mgr.log.debug( f'Agent on host {host} not ready to have config and deps checked: {e}') @@ -458,6 +468,9 @@ def _check_agent(self, host: str) -> bool: daemon_spec = CephadmDaemonDeploySpec.from_daemon_description(agent) self.mgr._daemon_action(daemon_spec, action=action) self.mgr.cache.rm_scheduled_daemon_action(agent.hostname, agent.name()) + except AgentLockException: + self.mgr.log.debug( + f'Could not {action} agent on host {host}. Someone else holds agent\'s lock') except Exception as e: self.mgr.log.debug( f'Agent on host {host} not ready to {action}: {e}') From f54ea99cabdaaa93a8fd8dfa6f47c807ebb33452 Mon Sep 17 00:00:00 2001 From: Nizamudeen A Date: Tue, 26 Oct 2021 17:03:33 +0530 Subject: [PATCH 0321/2866] mgr/k8sevents: pin the kubernetes version to 11.0.0 Fixes: https://tracker.ceph.com/issues/53044 Signed-off-by: Nizamudeen A --- src/pybind/mgr/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pybind/mgr/requirements.txt b/src/pybind/mgr/requirements.txt index 617b6dee2d357..8a7f30c8d7637 100644 --- a/src/pybind/mgr/requirements.txt +++ b/src/pybind/mgr/requirements.txt @@ -1,3 +1,3 @@ -rrequirements-required.txt asyncssh -kubernetes +kubernetes==11.0.0 From ea507d6c86f6a6ed5850edf24116e12088b24036 Mon Sep 17 00:00:00 2001 From: Cory Snyder Date: Fri, 10 Sep 2021 06:59:35 -0400 Subject: [PATCH 0322/2866] mgr/orchestrator: add --zap flag to 'orch osd rm' Adds the ability to zap OSD devices after removal, implemented as a flag on the 'orch osd rm' command. Fixes: https://tracker.ceph.com/issues/43692 Signed-off-by: Cory Snyder --- .../orch/cephadm/osds/2-ops/rm-zap-flag.yaml | 15 ++++++ src/pybind/mgr/cephadm/inventory.py | 7 +++ src/pybind/mgr/cephadm/module.py | 11 +++- src/pybind/mgr/cephadm/services/osd.py | 54 ++++++++++++++++--- .../mgr/cephadm/tests/test_osd_removal.py | 29 ++++++++++ src/pybind/mgr/orchestrator/_interface.py | 4 +- src/pybind/mgr/orchestrator/module.py | 5 +- src/pybind/mgr/rook/module.py | 4 +- 8 files changed, 118 insertions(+), 11 deletions(-) create mode 100644 qa/suites/orch/cephadm/osds/2-ops/rm-zap-flag.yaml diff --git a/qa/suites/orch/cephadm/osds/2-ops/rm-zap-flag.yaml b/qa/suites/orch/cephadm/osds/2-ops/rm-zap-flag.yaml new file mode 100644 index 0000000000000..8f07f6d537439 --- /dev/null +++ b/qa/suites/orch/cephadm/osds/2-ops/rm-zap-flag.yaml @@ -0,0 +1,15 @@ +tasks: +- cephadm.shell: + host.a: + - | + set -e + set -x + ceph orch ps + ceph orch device ls + DEVID=$(ceph device ls | grep osd.1 | awk '{print $1}') + HOST=$(ceph orch device ls | grep "$DEVID" | awk '{print $1}') + DEV=$(ceph orch device ls | grep "$DEVID" | awk '{print $2}') + echo "host $HOST, dev $DEV, devid $DEVID" + ceph orch osd rm --zap --replace 1 + while ceph orch osd rm status | grep ^1 ; do sleep 5 ; done + while ! ceph osd dump | grep osd.1 | grep "up\s*in" ; do sleep 5 ; done diff --git a/src/pybind/mgr/cephadm/inventory.py b/src/pybind/mgr/cephadm/inventory.py index 67ccc612ff784..06f2494c2aade 100644 --- a/src/pybind/mgr/cephadm/inventory.py +++ b/src/pybind/mgr/cephadm/inventory.py @@ -828,6 +828,13 @@ def get_daemon(self, daemon_name: str) -> orchestrator.DaemonDescription: return dd raise orchestrator.OrchestratorError(f'Unable to find {daemon_name} daemon(s)') + def has_daemon(self, daemon_name: str) -> bool: + try: + self.get_daemon(daemon_name) + except orchestrator.OrchestratorError: + return False + return True + def get_daemons_with_volatile_status(self) -> Iterator[Tuple[str, Dict[str, orchestrator.DaemonDescription]]]: def alter(host: str, dd_orig: orchestrator.DaemonDescription) -> orchestrator.DaemonDescription: dd = copy(dd_orig) diff --git a/src/pybind/mgr/cephadm/module.py b/src/pybind/mgr/cephadm/module.py index 711ddc69b54af..5b82e625a2955 100644 --- a/src/pybind/mgr/cephadm/module.py +++ b/src/pybind/mgr/cephadm/module.py @@ -361,6 +361,12 @@ class CephadmOrchestrator(orchestrator.Orchestrator, MgrModule, default=4721, desc='First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)' ), + Option( + 'max_osd_draining_count', + type='int', + default=10, + desc='max number of osds that will be drained simultaneously when osds are removed' + ), ] def __init__(self, *args: Any, **kwargs: Any): @@ -426,6 +432,7 @@ def __init__(self, *args: Any, **kwargs: Any): self.endpoint_port = 0 self.agent_starting_port = 0 self.apply_spec_fails: List[Tuple[str, str]] = [] + self.max_osd_draining_count = 10 self.notify('mon_map', None) self.config_notify() @@ -2591,7 +2598,8 @@ def upgrade_stop(self) -> str: @handle_orch_error def remove_osds(self, osd_ids: List[str], replace: bool = False, - force: bool = False) -> str: + force: bool = False, + zap: bool = False) -> str: """ Takes a list of OSDs and schedules them for removal. The function that takes care of the actual removal is @@ -2612,6 +2620,7 @@ def remove_osds(self, osd_ids: List[str], self.to_remove_osds.enqueue(OSD(osd_id=int(daemon.daemon_id), replace=replace, force=force, + zap=zap, hostname=daemon.hostname, process_started_at=datetime_now(), remove_util=self.to_remove_osds.rm_util)) diff --git a/src/pybind/mgr/cephadm/services/osd.py b/src/pybind/mgr/cephadm/services/osd.py index f5996eeae764b..94cb6fa076de7 100644 --- a/src/pybind/mgr/cephadm/services/osd.py +++ b/src/pybind/mgr/cephadm/services/osd.py @@ -467,6 +467,19 @@ def reweight_osd(self, osd: "OSD", weight: float) -> bool: self.mgr.log.info(f"{osd} weight is now {weight}") return True + def zap_osd(self, osd: "OSD") -> str: + "Zaps all devices that are associated with an OSD" + if osd.hostname is not None: + out, err, code = CephadmServe(self.mgr)._run_cephadm( + osd.hostname, 'osd', 'ceph-volume', + ['--', 'lvm', 'zap', '--destroy', '--osd-id', str(osd.osd_id)], + error_ok=True) + self.mgr.cache.invalidate_host_devices(osd.hostname) + if code: + raise OrchestratorError('Zap failed: %s' % '\n'.join(out + err)) + return '\n'.join(out + err) + raise OrchestratorError(f"Failed to zap OSD {osd.osd_id} because host was unknown") + def safe_to_destroy(self, osd_ids: List[int]) -> bool: """ Queries the safe-to-destroy flag for OSDs """ cmd_args = {'prefix': 'osd safe-to-destroy', @@ -521,7 +534,7 @@ def __init__(self, replace: bool = False, force: bool = False, hostname: Optional[str] = None, - ): + zap: bool = False): # the ID of the OSD self.osd_id = osd_id @@ -558,6 +571,9 @@ def __init__(self, self.original_weight: Optional[float] = None + # Whether devices associated with the OSD should be zapped (DATA ERASED) + self.zap = zap + def start(self) -> None: if self.started: logger.debug(f"Already started draining {self}") @@ -628,6 +644,9 @@ def down(self) -> bool: def destroy(self) -> bool: return self.rm_util.destroy_osd(self.osd_id) + def do_zap(self) -> str: + return self.rm_util.zap_osd(self) + def purge(self) -> bool: return self.rm_util.purge_osd(self.osd_id) @@ -656,6 +675,7 @@ def to_json(self) -> dict: out['stopped'] = self.stopped out['replace'] = self.replace out['force'] = self.force + out['zap'] = self.zap out['hostname'] = self.hostname # type: ignore for k in ['drain_started_at', 'drain_stopped_at', 'drain_done_at', 'process_started_at']: @@ -713,10 +733,10 @@ def process_removal_queue(self) -> None: self.cleanup() # find osds that are ok-to-stop and not yet draining - ok_to_stop_osds = self.rm_util.find_osd_stop_threshold(self.idling_osds()) - if ok_to_stop_osds: + ready_to_drain_osds = self._ready_to_drain_osds() + if ready_to_drain_osds: # start draining those - _ = [osd.start_draining() for osd in ok_to_stop_osds] + _ = [osd.start_draining() for osd in ready_to_drain_osds] all_osds = self.all_osds() @@ -748,8 +768,12 @@ def process_removal_queue(self) -> None: # stop and remove daemon assert osd.hostname is not None - CephadmServe(self.mgr)._remove_daemon(f'osd.{osd.osd_id}', osd.hostname) - logger.info(f"Successfully removed {osd} on {osd.hostname}") + + if self.mgr.cache.has_daemon(f'osd.{osd.osd_id}'): + CephadmServe(self.mgr)._remove_daemon(f'osd.{osd.osd_id}', osd.hostname) + logger.info(f"Successfully removed {osd} on {osd.hostname}") + else: + logger.info(f"Daemon {osd} on {osd.hostname} was already removed") if osd.replace: # mark destroyed in osdmap @@ -764,6 +788,12 @@ def process_removal_queue(self) -> None: raise orchestrator.OrchestratorError(f"Could not purge {osd}") logger.info(f"Successfully purged {osd} on {osd.hostname}") + if osd.zap: + # throws an exception if the zap fails + logger.info(f"Zapping devices for {osd} on {osd.hostname}") + osd.do_zap() + logger.info(f"Successfully zapped devices for {osd} on {osd.hostname}") + logger.debug(f"Removing {osd} from the queue.") # self could change while this is processing (osds get added from the CLI) @@ -779,6 +809,18 @@ def cleanup(self) -> None: for osd in self._not_in_cluster(): self.osds.remove(osd) + def _ready_to_drain_osds(self) -> List["OSD"]: + """ + Returns OSDs that are ok to stop and not yet draining. Only returns as many OSDs as can + be accomodated by the 'max_osd_draining_count' config value, considering the number of OSDs + that are already draining. + """ + draining_limit = max(1, self.mgr.max_osd_draining_count) + num_already_draining = len(self.draining_osds()) + num_to_start_draining = max(0, draining_limit - num_already_draining) + stoppable_osds = self.rm_util.find_osd_stop_threshold(self.idling_osds()) + return [] if stoppable_osds is None else stoppable_osds[:num_to_start_draining] + def _save_to_store(self) -> None: osd_queue = [osd.to_json() for osd in self.osds] logger.debug(f"Saving {osd_queue} to store") diff --git a/src/pybind/mgr/cephadm/tests/test_osd_removal.py b/src/pybind/mgr/cephadm/tests/test_osd_removal.py index cffdcaf03fbcb..9347678b84daa 100644 --- a/src/pybind/mgr/cephadm/tests/test_osd_removal.py +++ b/src/pybind/mgr/cephadm/tests/test_osd_removal.py @@ -3,6 +3,7 @@ from cephadm.services.osd import OSDRemovalQueue, OSD import pytest from tests import mock +from .fixtures import with_cephadm_module from datetime import datetime @@ -54,6 +55,33 @@ def test_process_removal_queue(self, rm_util): # rm_util.process_removal_queue() pass + @pytest.mark.parametrize( + "max_osd_draining_count, draining_osds, idling_osds, ok_to_stop, expected", + [ + # drain one at a time, one already draining + (1, [1], [1], [True], 0), + # drain one at a time, none draining yet + (1, [], [1, 2, 3], [True, True, True], 1), + # drain one at a time, one already draining, none ok-to-stop + (1, [1], [1], [False], 0), + # drain one at a time, none draining, one ok-to-stop + (1, [], [1, 2, 3], [False, False, True], 1), + # drain three at a time, one already draining, all ok-to-stop + (3, [1], [1, 2, 3], [True, True, True], 2), + # drain two at a time, none already draining, none ok-to-stop + (2, [], [1, 2, 3], [False, False, False], 0), + # drain two at a time, none already draining, none idling + (2, [], [], [], 0), + ] + ) + def test_ready_to_drain_osds(self, max_osd_draining_count, draining_osds, idling_osds, ok_to_stop, expected): + with with_cephadm_module({'max_osd_draining_count': max_osd_draining_count}) as m: + with mock.patch("cephadm.services.osd.OSDRemovalQueue.draining_osds", return_value=draining_osds): + with mock.patch("cephadm.services.osd.OSDRemovalQueue.idling_osds", return_value=idling_osds): + with mock.patch("cephadm.services.osd.RemoveUtil.ok_to_stop", side_effect=ok_to_stop): + removal_queue = OSDRemovalQueue(m) + assert len(removal_queue._ready_to_drain_osds()) == expected + def test_ok_to_stop(self, rm_util): rm_util.ok_to_stop([MockOSD(1)]) rm_util._run_mon_cmd.assert_called_with({'prefix': 'osd ok-to-stop', 'ids': ['1']}) @@ -81,6 +109,7 @@ def test_load(self, cephadm_module, rm_util): "stopped": False, "replace": False, "force": False, + "zap": False, "nodename": "node2", "drain_started_at": "2020-09-14T11:41:53.960463", "drain_stopped_at": None, diff --git a/src/pybind/mgr/orchestrator/_interface.py b/src/pybind/mgr/orchestrator/_interface.py index 1497f80aaa67e..fdb994136414f 100644 --- a/src/pybind/mgr/orchestrator/_interface.py +++ b/src/pybind/mgr/orchestrator/_interface.py @@ -553,11 +553,13 @@ def preview_osdspecs(self, def remove_osds(self, osd_ids: List[str], replace: bool = False, - force: bool = False) -> OrchResult[str]: + force: bool = False, + zap: bool = False) -> OrchResult[str]: """ :param osd_ids: list of OSD IDs :param replace: marks the OSD as being destroyed. See :ref:`orchestrator-osd-replace` :param force: Forces the OSD removal process without waiting for the data to be drained first. + :param zap: Zap/Erase all devices associated with the OSDs (DESTROYS DATA) Note that this can only remove OSDs that were successfully created (i.e. got an OSD ID). """ diff --git a/src/pybind/mgr/orchestrator/module.py b/src/pybind/mgr/orchestrator/module.py index 1b81a736e6399..6d28d59fc7c82 100644 --- a/src/pybind/mgr/orchestrator/module.py +++ b/src/pybind/mgr/orchestrator/module.py @@ -819,9 +819,10 @@ def _daemon_add_osd(self, svc_arg: Optional[str] = None) -> HandleCommandResult: def _osd_rm_start(self, osd_id: List[str], replace: bool = False, - force: bool = False) -> HandleCommandResult: + force: bool = False, + zap: bool = False) -> HandleCommandResult: """Remove OSD daemons""" - completion = self.remove_osds(osd_id, replace=replace, force=force) + completion = self.remove_osds(osd_id, replace=replace, force=force, zap=zap) raise_if_exception(completion) return HandleCommandResult(stdout=completion.result_str()) diff --git a/src/pybind/mgr/rook/module.py b/src/pybind/mgr/rook/module.py index 4fedaec8ae409..958277851e6fe 100644 --- a/src/pybind/mgr/rook/module.py +++ b/src/pybind/mgr/rook/module.py @@ -509,8 +509,10 @@ def apply_drivegroups(self, specs: List[DriveGroupSpec]) -> OrchResult[List[str] result_list.append(self.rook_cluster.add_osds(drive_group, matching_hosts)) return OrchResult(result_list) - def remove_osds(self, osd_ids: List[str], replace: bool = False, force: bool = False) -> OrchResult[str]: + def remove_osds(self, osd_ids: List[str], replace: bool = False, force: bool = False, zap: bool = False) -> OrchResult[str]: assert self._rook_cluster is not None + if zap: + raise RuntimeError("Rook does not support zapping devices during OSD removal.") res = self._rook_cluster.remove_osds(osd_ids, replace, force, self.mon_command) return OrchResult(res) From eee8d8a32e3f39440ac417b2e9ea46d93d4a1d73 Mon Sep 17 00:00:00 2001 From: Prasanna Kumar Kalever Date: Wed, 12 May 2021 18:30:46 +0530 Subject: [PATCH 0323/2866] rbd-nbd: generate and send device cookie with netlink connect request [root@linux-vm1]# rbd-nbd map rbd-pool/image0 --try-netlink /dev/nbd0 [root@linux-vm1]# cat /sys/block/nbd0/backend c704cb91-c6cf-466e-a335-0e935c0d5e47 Signed-off-by: Prasanna Kumar Kalever --- src/tools/rbd_nbd/nbd-netlink.h | 1 + src/tools/rbd_nbd/rbd-nbd.cc | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/src/tools/rbd_nbd/nbd-netlink.h b/src/tools/rbd_nbd/nbd-netlink.h index c5d0ef7aa7d5c..2d0b90964227a 100644 --- a/src/tools/rbd_nbd/nbd-netlink.h +++ b/src/tools/rbd_nbd/nbd-netlink.h @@ -35,6 +35,7 @@ enum { NBD_ATTR_SOCKETS, NBD_ATTR_DEAD_CONN_TIMEOUT, NBD_ATTR_DEVICE_LIST, + NBD_ATTR_BACKEND_IDENTIFIER, __NBD_ATTR_MAX, }; #define NBD_ATTR_MAX (__NBD_ATTR_MAX - 1) diff --git a/src/tools/rbd_nbd/rbd-nbd.cc b/src/tools/rbd_nbd/rbd-nbd.cc index 1f8eb54fa016a..5356dfb5798c3 100644 --- a/src/tools/rbd_nbd/rbd-nbd.cc +++ b/src/tools/rbd_nbd/rbd-nbd.cc @@ -123,6 +123,7 @@ struct Config { Command command = None; int pid = 0; + std::string cookie; std::string image_spec() const { std::string spec = poolname + "/"; @@ -1325,6 +1326,8 @@ static int netlink_connect(Config *cfg, struct nl_sock *sock, int nl_id, int fd, NLA_PUT_U64(msg, NBD_ATTR_BLOCK_SIZE_BYTES, RBD_NBD_BLKSIZE); NLA_PUT_U64(msg, NBD_ATTR_SERVER_FLAGS, flags); NLA_PUT_U64(msg, NBD_ATTR_DEAD_CONN_TIMEOUT, cfg->reattach_timeout); + if (!cfg->cookie.empty()) + NLA_PUT_STRING(msg, NBD_ATTR_BACKEND_IDENTIFIER, cfg->cookie.c_str()); sock_attr = nla_nest_start(msg, NBD_ATTR_SOCKETS); if (!sock_attr) { @@ -1719,6 +1722,11 @@ static int do_map(int argc, const char *argv[], Config *cfg, bool reconnect) use_netlink = cfg->try_netlink || reconnect; if (use_netlink) { + if (!reconnect) { + uuid_d uuid_gen; + uuid_gen.generate_random(); + cfg->cookie = uuid_gen.to_string(); + } r = try_netlink_setup(cfg, fd[0], size, flags, reconnect); if (r < 0) { goto free_server; From 59c08eabf12fdcf155b54dc422fb0f67d6ad747e Mon Sep 17 00:00:00 2001 From: Prasanna Kumar Kalever Date: Wed, 12 May 2021 18:33:54 +0530 Subject: [PATCH 0324/2866] rbd-nbd: provide a flag to show device cookie with map command [root@linux-vm1]# rbd-nbd map rbd-pool/image0 --try-netlink --show-cookie /dev/nbd0 c704cb91-c6cf-466e-a335-0e935c0d5e47 Signed-off-by: Prasanna Kumar Kalever --- src/tools/rbd_nbd/rbd-nbd.cc | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/src/tools/rbd_nbd/rbd-nbd.cc b/src/tools/rbd_nbd/rbd-nbd.cc index 5356dfb5798c3..ccf9b0d2d421d 100644 --- a/src/tools/rbd_nbd/rbd-nbd.cc +++ b/src/tools/rbd_nbd/rbd-nbd.cc @@ -107,6 +107,7 @@ struct Config { bool readonly = false; bool set_max_part = false; bool try_netlink = false; + bool show_cookie = false; std::string poolname; std::string nsname; @@ -165,6 +166,7 @@ static void usage() << " --reattach-timeout Set nbd re-attach timeout\n" << " (default: " << Config().reattach_timeout << ")\n" << " --try-netlink Use the nbd netlink interface\n" + << " --show-cookie Show device cookie\n" << "\n" << "List options:\n" << " --format plain|json|xml Output format (default: plain)\n" @@ -191,6 +193,8 @@ static int run_quiesce_hook(const std::string &quiesce_hook, const std::string &devpath, const std::string &command); +static std::string get_cookie(const std::string &devpath); + class NBDServer { public: @@ -930,6 +934,20 @@ class NBDListIterator { } }; +static std::string get_cookie(const std::string &devpath) +{ + std::string cookie; + std::ifstream ifs; + std::string path = "/sys/block/" + devpath.substr(sizeof("/dev/") - 1) + "/backend"; + + ifs.open(path, std::ifstream::in); + if (ifs.is_open()) { + std::getline(ifs, cookie); + ifs.close(); + } + return cookie; +} + static int load_module(Config *cfg) { ostringstream param; @@ -1769,7 +1787,16 @@ static int do_map(int argc, const char *argv[], Config *cfg, bool reconnect) if (r < 0) goto close_nbd; - cout << cfg->devpath << std::endl; + std::string cookie; + if (use_netlink && !cfg->cookie.empty()) { + cookie = get_cookie(cfg->devpath); + ceph_assert(cookie == cfg->cookie); + } + if (cfg->show_cookie && !cookie.empty()) { + cout << cfg->devpath << " " << cookie << std::endl; + } else { + cout << cfg->devpath << std::endl; + } run_server(forker, server, use_netlink); @@ -2064,6 +2091,8 @@ static int parse_args(vector& args, std::ostream *err_msg, cfg->pretty_format = true; } else if (ceph_argparse_flag(args, i, "--try-netlink", (char *)NULL)) { cfg->try_netlink = true; + } else if (ceph_argparse_flag(args, i, "--show-cookie", (char *)NULL)) { + cfg->show_cookie = true; } else if (ceph_argparse_witharg(args, i, &arg_value, "--encryption-format", (char *)NULL)) { if (arg_value == "luks1") { From 6ca70ee4f078aba86a6edd013bd8d95f690a46a1 Mon Sep 17 00:00:00 2001 From: Prasanna Kumar Kalever Date: Thu, 13 May 2021 13:52:25 +0530 Subject: [PATCH 0325/2866] rbd-nbd: mandate device cookie for attach command [root@linux-vm1]# rbd-nbd attach rbd-pool/image0 --device /dev/nbd0 \ --cookie c704cb91-c6cf-466e-a335-0e935c0d5e47 /dev/nbd0 Signed-off-by: Prasanna Kumar Kalever --- src/tools/rbd_nbd/rbd-nbd.cc | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/tools/rbd_nbd/rbd-nbd.cc b/src/tools/rbd_nbd/rbd-nbd.cc index ccf9b0d2d421d..0ac0e23e3a82e 100644 --- a/src/tools/rbd_nbd/rbd-nbd.cc +++ b/src/tools/rbd_nbd/rbd-nbd.cc @@ -167,6 +167,7 @@ static void usage() << " (default: " << Config().reattach_timeout << ")\n" << " --try-netlink Use the nbd netlink interface\n" << " --show-cookie Show device cookie\n" + << " --cookie Specify device cookie for attach\n" << "\n" << "List options:\n" << " --format plain|json|xml Output format (default: plain)\n" @@ -2093,6 +2094,7 @@ static int parse_args(vector& args, std::ostream *err_msg, cfg->try_netlink = true; } else if (ceph_argparse_flag(args, i, "--show-cookie", (char *)NULL)) { cfg->show_cookie = true; + } else if (ceph_argparse_witharg(args, i, &cfg->cookie, "--cookie", (char *)NULL)) { } else if (ceph_argparse_witharg(args, i, &arg_value, "--encryption-format", (char *)NULL)) { if (arg_value == "luks1") { @@ -2144,6 +2146,16 @@ static int parse_args(vector& args, std::ostream *err_msg, *err_msg << "rbd-nbd: must specify device to attach"; return -EINVAL; } + /* FIXME: Should we allow attach for kernel versions that doesn't + * support NBD_ATTR_BACKEND_IDENTIFIER (danger!) ? + * 1. If we allow attach for lower kernel versions then, we should + * also provide a way to skip cookie check here + * 2. If we do not want to allow, then add kernel version check here + */ + if (cfg->cookie.empty()) { + *err_msg << "rbd-nbd: must specify cookie to attach"; + return -EINVAL; + } [[fallthrough]]; case Map: if (args.begin() == args.end()) { From efeb255d107075516c95f719f9a2ec9375d5bb2f Mon Sep 17 00:00:00 2001 From: Prasanna Kumar Kalever Date: Thu, 13 May 2021 12:39:25 +0530 Subject: [PATCH 0326/2866] rbd-nbd: show per device cookie with list command [root@linux-vm1]# rbd-nbd list-mapped id pool namespace image snap device cookie 8133 rbd-pool image0 - /dev/nbd0 c704cb91-c6cf-466e-a335-0e935c0d5e47 Signed-off-by: Prasanna Kumar Kalever --- src/tools/rbd_nbd/rbd-nbd.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/tools/rbd_nbd/rbd-nbd.cc b/src/tools/rbd_nbd/rbd-nbd.cc index 0ac0e23e3a82e..f8c24397542c2 100644 --- a/src/tools/rbd_nbd/rbd-nbd.cc +++ b/src/tools/rbd_nbd/rbd-nbd.cc @@ -803,6 +803,7 @@ class NBDListIterator { continue; } ifs >> pid; + ifs.close(); // If the rbd-nbd is re-attached the pid may store garbage // here. We are sure this is the case when it is negative or @@ -907,6 +908,7 @@ class NBDListIterator { c.devpath = cfg->devpath; } + c.cookie = get_cookie(cfg->devpath); *cfg = c; return 0; } @@ -1935,6 +1937,7 @@ static int do_list_mapped_devices(const std::string &format, bool pretty_format) tbl.define_column("image", TextTable::LEFT, TextTable::LEFT); tbl.define_column("snap", TextTable::LEFT, TextTable::LEFT); tbl.define_column("device", TextTable::LEFT, TextTable::LEFT); + tbl.define_column("cookie", TextTable::LEFT, TextTable::LEFT); } Config cfg; @@ -1948,6 +1951,7 @@ static int do_list_mapped_devices(const std::string &format, bool pretty_format) f->dump_string("image", cfg.imgname); f->dump_string("snap", cfg.snapname); f->dump_string("device", cfg.devpath); + f->dump_string("cookie", cfg.cookie); f->close_section(); } else { should_print = true; @@ -1955,7 +1959,7 @@ static int do_list_mapped_devices(const std::string &format, bool pretty_format) cfg.snapname = "-"; } tbl << cfg.pid << cfg.poolname << cfg.nsname << cfg.imgname - << cfg.snapname << cfg.devpath << TextTable::endrow; + << cfg.snapname << cfg.devpath << cfg.cookie << TextTable::endrow; } } From 833e71282ad82023237c5eb407b9fa4630280c15 Mon Sep 17 00:00:00 2001 From: Prasanna Kumar Kalever Date: Thu, 17 Jun 2021 19:05:20 +0530 Subject: [PATCH 0327/2866] rbd-nbd: allow attach without --cookie for old kernel versions For backward compatibility allow attach without --cookie option: [root@linux-vm1]# rbd-nbd attach rbd-pool/image0 --device /dev/nbd0 /dev/nbd0 Signed-off-by: Prasanna Kumar Kalever --- src/tools/rbd_nbd/rbd-nbd.cc | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/src/tools/rbd_nbd/rbd-nbd.cc b/src/tools/rbd_nbd/rbd-nbd.cc index f8c24397542c2..5a9492daa3081 100644 --- a/src/tools/rbd_nbd/rbd-nbd.cc +++ b/src/tools/rbd_nbd/rbd-nbd.cc @@ -2144,20 +2144,26 @@ static int parse_args(vector& args, std::ostream *err_msg, return -EINVAL; } + std::string cookie; switch (cmd) { case Attach: if (cfg->devpath.empty()) { *err_msg << "rbd-nbd: must specify device to attach"; return -EINVAL; } - /* FIXME: Should we allow attach for kernel versions that doesn't - * support NBD_ATTR_BACKEND_IDENTIFIER (danger!) ? - * 1. If we allow attach for lower kernel versions then, we should - * also provide a way to skip cookie check here - * 2. If we do not want to allow, then add kernel version check here - */ - if (cfg->cookie.empty()) { - *err_msg << "rbd-nbd: must specify cookie to attach"; + // Allowing attach without --cookie option for kernel without + // NBD_ATTR_BACKEND_IDENTIFIER support for compatibility + cookie = get_cookie(cfg->devpath); + if (!cookie.empty()) { + if (cfg->cookie.empty()) { + *err_msg << "rbd-nbd: must specify cookie to attach"; + return -EINVAL; + } else if (cookie != cfg->cookie) { + *err_msg << "rbd-nbd: cookie mismatch"; + return -EINVAL; + } + } else if (!cfg->cookie.empty()) { + *err_msg << "rbd-nbd: kernel does not have cookie support"; return -EINVAL; } [[fallthrough]]; From 6df06c7a62bdd6e9ce61b26c738dcd1509e68273 Mon Sep 17 00:00:00 2001 From: Prasanna Kumar Kalever Date: Fri, 18 Jun 2021 12:18:35 +0530 Subject: [PATCH 0328/2866] rbd: add --show-cookie/--cookie option for map/attach commands Example: $ rbd device map rbd-pool/image --show-cookie --try-netlink --device-type nbd $ rbd device attach rbd-pool/image --device /dev/nbd0 \ --cookie 6f85d970-10b2-456b-8baf-676aa4d782e4 --device-type nbd older Kernel versions can use --force to skip the cookie validation Signed-off-by: Prasanna Kumar Kalever --- doc/man/8/rbd.rst | 4 ++-- src/test/cli/rbd/help.t | 9 ++++++--- src/tools/rbd/action/Device.cc | 2 ++ src/tools/rbd/action/Nbd.cc | 11 +++++++++-- 4 files changed, 19 insertions(+), 7 deletions(-) diff --git a/doc/man/8/rbd.rst b/doc/man/8/rbd.rst index 7c61d0a5dd7a2..b5d4e3d1ce690 100644 --- a/doc/man/8/rbd.rst +++ b/doc/man/8/rbd.rst @@ -255,7 +255,7 @@ Commands Show the rbd images that are mapped via the rbd kernel module (default) or other supported device. -:command:`device map` [-t | --device-type *device-type*] [--read-only] [--exclusive] [-o | --options *device-options*] *image-spec* | *snap-spec* +:command:`device map` [-t | --device-type *device-type*] [--show-cookie] [--read-only] [--exclusive] [-o | --options *device-options*] *image-spec* | *snap-spec* Map the specified image to a block device via the rbd kernel module (default) or other supported device (*nbd* on Linux or *ggate* on FreeBSD). @@ -270,7 +270,7 @@ Commands The --options argument is a comma separated list of device type specific options (opt1,opt2=val,...). -:command:`device attach` [-t | --device-type *device-type*] --device *device-path* [--read-only] [--exclusive] [--force] [-o | --options *device-options*] *image-spec* | *snap-spec* +:command:`device attach` [-t | --device-type *device-type*] --device *device-path* [--cookie *device-cookie*] [--read-only] [--exclusive] [--force] [-o | --options *device-options*] *image-spec* | *snap-spec* Attach the specified image to the specified block device (currently only `nbd` on Linux). This operation is unsafe and should not be normally used. In particular, specifying the wrong image or the wrong block device may diff --git a/src/test/cli/rbd/help.t b/src/test/cli/rbd/help.t index 22362eb806a1e..7f3622b4fa03e 100644 --- a/src/test/cli/rbd/help.t +++ b/src/test/cli/rbd/help.t @@ -576,8 +576,8 @@ rbd help device attach usage: rbd device attach [--device-type ] [--pool ] [--namespace ] [--image ] - [--snap ] --device [--read-only] - [--force] [--exclusive] [--quiesce] + [--snap ] --device [--cookie ] + [--read-only] [--force] [--exclusive] [--quiesce] [--quiesce-hook ] [--options ] @@ -596,6 +596,7 @@ --image arg image name --snap arg snapshot name --device arg specify device path + --cookie arg specify device cookie --read-only attach read-only --force force attach --exclusive disable automatic exclusive lock transitions @@ -637,7 +638,8 @@ rbd help device map usage: rbd device map [--device-type ] [--pool ] [--namespace ] [--image ] - [--snap ] [--read-only] [--exclusive] [--quiesce] + [--snap ] [--show-cookie] [--read-only] + [--exclusive] [--quiesce] [--quiesce-hook ] [--options ] @@ -655,6 +657,7 @@ --namespace arg namespace name --image arg image name --snap arg snapshot name + --show-cookie show device cookie --read-only map read-only --exclusive disable automatic exclusive lock transitions --quiesce use quiesce hooks diff --git a/src/tools/rbd/action/Device.cc b/src/tools/rbd/action/Device.cc index a9903cfa78c47..350213449551a 100644 --- a/src/tools/rbd/action/Device.cc +++ b/src/tools/rbd/action/Device.cc @@ -175,6 +175,7 @@ void get_map_arguments(po::options_description *positional, at::add_image_or_snap_spec_options(positional, options, at::ARGUMENT_MODIFIER_NONE); options->add_options() + ("show-cookie", po::bool_switch(), "show device cookie") ("read-only", po::bool_switch(), "map read-only") ("exclusive", po::bool_switch(), "disable automatic exclusive lock transitions") ("quiesce", po::bool_switch(), "use quiesce hooks") @@ -212,6 +213,7 @@ void get_attach_arguments(po::options_description *positional, at::ARGUMENT_MODIFIER_NONE); options->add_options() ("device", po::value()->required(), "specify device path") + ("cookie", po::value(), "specify device cookie") ("read-only", po::bool_switch(), "attach read-only") ("force", po::bool_switch(), "force attach") ("exclusive", po::bool_switch(), "disable automatic exclusive lock transitions") diff --git a/src/tools/rbd/action/Nbd.cc b/src/tools/rbd/action/Nbd.cc index 117a2592dd1e8..fe9a737f9603f 100644 --- a/src/tools/rbd/action/Nbd.cc +++ b/src/tools/rbd/action/Nbd.cc @@ -163,10 +163,13 @@ int execute_attach(const po::variables_map &vm, return -EINVAL; } - if (!vm["force"].as()) { + if (vm.count("cookie")) { + args.push_back("--cookie"); + args.push_back(vm["cookie"].as()); + } else if (!vm["force"].as()) { std::cerr << "rbd: could not validate attach request\n"; std::cerr << "rbd: mismatching the image and the device may lead to data corruption\n"; - std::cerr << "rbd: must specify --force to proceed" << std::endl; + std::cerr << "rbd: must specify --cookie or --force to proceed" << std::endl; return -EINVAL; } @@ -259,6 +262,10 @@ int execute_map(const po::variables_map &vm, args.push_back("--quiesce"); } + if (vm["show-cookie"].as()) { + args.push_back("--show-cookie"); + } + if (vm["read-only"].as()) { args.push_back("--read-only"); } From 16404dede832f3908d48fc36c7714891bfcfde20 Mon Sep 17 00:00:00 2001 From: Prasanna Kumar Kalever Date: Wed, 11 Aug 2021 15:32:25 +0530 Subject: [PATCH 0329/2866] rbd-nbd: allow user to specify cookie at map Allow user to specify cookie of choice at the time of map $ rbd device attach rbd-pool/image --device /dev/nbd0 \ --cookie 6f85d970-10b2-456b-8baf-676aa4d782e4 --options try-netlink Signed-off-by: Prasanna Kumar Kalever --- doc/man/8/rbd.rst | 4 ++-- src/test/cli/rbd/help.t | 11 +++++++---- src/tools/rbd/action/Device.cc | 2 ++ src/tools/rbd/action/Nbd.cc | 9 +++++++++ src/tools/rbd_nbd/rbd-nbd.cc | 9 +++++---- 5 files changed, 25 insertions(+), 10 deletions(-) diff --git a/doc/man/8/rbd.rst b/doc/man/8/rbd.rst index b5d4e3d1ce690..93483586caf5a 100644 --- a/doc/man/8/rbd.rst +++ b/doc/man/8/rbd.rst @@ -255,7 +255,7 @@ Commands Show the rbd images that are mapped via the rbd kernel module (default) or other supported device. -:command:`device map` [-t | --device-type *device-type*] [--show-cookie] [--read-only] [--exclusive] [-o | --options *device-options*] *image-spec* | *snap-spec* +:command:`device map` [-t | --device-type *device-type*] [--cookie *device-cookie*] [--show-cookie] [--read-only] [--exclusive] [-o | --options *device-options*] *image-spec* | *snap-spec* Map the specified image to a block device via the rbd kernel module (default) or other supported device (*nbd* on Linux or *ggate* on FreeBSD). @@ -270,7 +270,7 @@ Commands The --options argument is a comma separated list of device type specific options (opt1,opt2=val,...). -:command:`device attach` [-t | --device-type *device-type*] --device *device-path* [--cookie *device-cookie*] [--read-only] [--exclusive] [--force] [-o | --options *device-options*] *image-spec* | *snap-spec* +:command:`device attach` [-t | --device-type *device-type*] --device *device-path* [--cookie *device-cookie*] [--show-cookie] [--read-only] [--exclusive] [--force] [-o | --options *device-options*] *image-spec* | *snap-spec* Attach the specified image to the specified block device (currently only `nbd` on Linux). This operation is unsafe and should not be normally used. In particular, specifying the wrong image or the wrong block device may diff --git a/src/test/cli/rbd/help.t b/src/test/cli/rbd/help.t index 7f3622b4fa03e..045da189c5247 100644 --- a/src/test/cli/rbd/help.t +++ b/src/test/cli/rbd/help.t @@ -576,8 +576,9 @@ rbd help device attach usage: rbd device attach [--device-type ] [--pool ] [--namespace ] [--image ] - [--snap ] --device [--cookie ] - [--read-only] [--force] [--exclusive] [--quiesce] + [--snap ] --device [--show-cookie] + [--cookie ] [--read-only] [--force] + [--exclusive] [--quiesce] [--quiesce-hook ] [--options ] @@ -596,6 +597,7 @@ --image arg image name --snap arg snapshot name --device arg specify device path + --show-cookie show device cookie --cookie arg specify device cookie --read-only attach read-only --force force attach @@ -638,8 +640,8 @@ rbd help device map usage: rbd device map [--device-type ] [--pool ] [--namespace ] [--image ] - [--snap ] [--show-cookie] [--read-only] - [--exclusive] [--quiesce] + [--snap ] [--show-cookie] [--cookie ] + [--read-only] [--exclusive] [--quiesce] [--quiesce-hook ] [--options ] @@ -658,6 +660,7 @@ --image arg image name --snap arg snapshot name --show-cookie show device cookie + --cookie arg specify device cookie --read-only map read-only --exclusive disable automatic exclusive lock transitions --quiesce use quiesce hooks diff --git a/src/tools/rbd/action/Device.cc b/src/tools/rbd/action/Device.cc index 350213449551a..20cff569f9b61 100644 --- a/src/tools/rbd/action/Device.cc +++ b/src/tools/rbd/action/Device.cc @@ -176,6 +176,7 @@ void get_map_arguments(po::options_description *positional, at::ARGUMENT_MODIFIER_NONE); options->add_options() ("show-cookie", po::bool_switch(), "show device cookie") + ("cookie", po::value(), "specify device cookie") ("read-only", po::bool_switch(), "map read-only") ("exclusive", po::bool_switch(), "disable automatic exclusive lock transitions") ("quiesce", po::bool_switch(), "use quiesce hooks") @@ -213,6 +214,7 @@ void get_attach_arguments(po::options_description *positional, at::ARGUMENT_MODIFIER_NONE); options->add_options() ("device", po::value()->required(), "specify device path") + ("show-cookie", po::bool_switch(), "show device cookie") ("cookie", po::value(), "specify device cookie") ("read-only", po::bool_switch(), "attach read-only") ("force", po::bool_switch(), "force attach") diff --git a/src/tools/rbd/action/Nbd.cc b/src/tools/rbd/action/Nbd.cc index fe9a737f9603f..f36537430c83f 100644 --- a/src/tools/rbd/action/Nbd.cc +++ b/src/tools/rbd/action/Nbd.cc @@ -163,6 +163,10 @@ int execute_attach(const po::variables_map &vm, return -EINVAL; } + if (vm["show-cookie"].as()) { + args.push_back("--show-cookie"); + } + if (vm.count("cookie")) { args.push_back("--cookie"); args.push_back(vm["cookie"].as()); @@ -266,6 +270,11 @@ int execute_map(const po::variables_map &vm, args.push_back("--show-cookie"); } + if (vm.count("cookie")) { + args.push_back("--cookie"); + args.push_back(vm["cookie"].as()); + } + if (vm["read-only"].as()) { args.push_back("--read-only"); } diff --git a/src/tools/rbd_nbd/rbd-nbd.cc b/src/tools/rbd_nbd/rbd-nbd.cc index 5a9492daa3081..db964d389374f 100644 --- a/src/tools/rbd_nbd/rbd-nbd.cc +++ b/src/tools/rbd_nbd/rbd-nbd.cc @@ -167,7 +167,7 @@ static void usage() << " (default: " << Config().reattach_timeout << ")\n" << " --try-netlink Use the nbd netlink interface\n" << " --show-cookie Show device cookie\n" - << " --cookie Specify device cookie for attach\n" + << " --cookie Specify device cookie\n" << "\n" << "List options:\n" << " --format plain|json|xml Output format (default: plain)\n" @@ -1743,7 +1743,8 @@ static int do_map(int argc, const char *argv[], Config *cfg, bool reconnect) use_netlink = cfg->try_netlink || reconnect; if (use_netlink) { - if (!reconnect) { + // generate when the cookie is not supplied at CLI + if (!reconnect && cfg->cookie.empty()) { uuid_d uuid_gen; uuid_gen.generate_random(); cfg->cookie = uuid_gen.to_string(); @@ -1791,9 +1792,9 @@ static int do_map(int argc, const char *argv[], Config *cfg, bool reconnect) goto close_nbd; std::string cookie; - if (use_netlink && !cfg->cookie.empty()) { + if (use_netlink) { cookie = get_cookie(cfg->devpath); - ceph_assert(cookie == cfg->cookie); + ceph_assert(cookie == cfg->cookie || cookie.empty()); } if (cfg->show_cookie && !cookie.empty()) { cout << cfg->devpath << " " << cookie << std::endl; From 2adfe55bcce08f8f46989e9ddbf46aa74e681992 Mon Sep 17 00:00:00 2001 From: Prasanna Kumar Kalever Date: Mon, 17 May 2021 14:17:53 +0530 Subject: [PATCH 0330/2866] qa/workunits/rbd: test map/attach with --show-cookie/--cookie options Signed-off-by: Prasanna Kumar Kalever --- qa/workunits/rbd/rbd-nbd.sh | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/qa/workunits/rbd/rbd-nbd.sh b/qa/workunits/rbd/rbd-nbd.sh index 77176ef7a2c85..0fc777168a29f 100755 --- a/qa/workunits/rbd/rbd-nbd.sh +++ b/qa/workunits/rbd/rbd-nbd.sh @@ -365,22 +365,39 @@ cat ${LOG_FILE} expect_false grep 'quiesce failed' ${LOG_FILE} # test detach/attach -DEV=`_sudo rbd device --device-type nbd --options try-netlink map ${POOL}/${IMAGE}` +OUT=`_sudo rbd device --device-type nbd --options try-netlink,show-cookie map ${POOL}/${IMAGE}` +read DEV COOKIE <<< "${OUT}" get_pid ${POOL} _sudo mount ${DEV} ${TEMPDIR}/mnt _sudo rbd device detach ${POOL}/${IMAGE} --device-type nbd expect_false get_pid ${POOL} expect_false _sudo rbd device attach --device ${DEV} ${POOL}/${IMAGE} --device-type nbd -_sudo rbd device attach --device ${DEV} ${POOL}/${IMAGE} --device-type nbd --force +if [ -n "${COOKIE}" ]; then + _sudo rbd device attach --device ${DEV} --cookie ${COOKIE} ${POOL}/${IMAGE} --device-type nbd +else + _sudo rbd device attach --device ${DEV} ${POOL}/${IMAGE} --device-type nbd --force +fi get_pid ${POOL} _sudo rbd device detach ${DEV} --device-type nbd expect_false get_pid ${POOL} -_sudo rbd device attach --device ${DEV} ${POOL}/${IMAGE} --device-type nbd --force +if [ -n "${COOKIE}" ]; then + _sudo rbd device attach --device ${DEV} --cookie ${COOKIE} ${POOL}/${IMAGE} --device-type nbd +else + _sudo rbd device attach --device ${DEV} ${POOL}/${IMAGE} --device-type nbd --force +fi get_pid ${POOL} ls ${TEMPDIR}/mnt/ dd if=${TEMPDIR}/mnt/test of=/dev/null bs=1M count=1 _sudo dd if=${DATA} of=${TEMPDIR}/mnt/test1 bs=1M count=1 oflag=direct _sudo umount ${TEMPDIR}/mnt unmap_device ${DEV} ${PID} +# if kernel supports cookies +if [ -n "${COOKIE}" ]; then + OUT=`_sudo rbd device --device-type nbd --show-cookie --cookie "abc de" --options try-netlink map ${POOL}/${IMAGE}` + read DEV ANOTHER_COOKIE <<< "${OUT}" + get_pid ${POOL} + test "${ANOTHER_COOKIE}" = "abc de" + unmap_device ${DEV} ${PID} +fi echo OK From 7e070cee3ecf0dc8d8c2f5e090b8bc927e02ddad Mon Sep 17 00:00:00 2001 From: Adam King Date: Tue, 26 Oct 2021 09:08:14 -0400 Subject: [PATCH 0331/2866] cephadm: make agent gatherer classes generic Signed-off-by: Adam King --- src/cephadm/cephadm | 82 +++++++++++++-------------------------------- 1 file changed, 23 insertions(+), 59 deletions(-) diff --git a/src/cephadm/cephadm b/src/cephadm/cephadm index 1763d1e1b6383..8bd9bd50961b3 100755 --- a/src/cephadm/cephadm +++ b/src/cephadm/cephadm @@ -3523,18 +3523,12 @@ class CephadmAgent(): self.ack = 1 self.event = Event() self.mgr_listener = MgrListener(self) - self.ls_gatherer = LsGatherer(self) - self.ls_event = Event() - self.volume_gatherer = VolumeGatherer(self) - self.volume_event = Event() + self.ls_gatherer = AgentGatherer(self, self._get_ls, 'Ls') + self.volume_gatherer = AgentGatherer(self, self._ceph_volume, 'Volume') self.device_enhanced_scan = False self.recent_iteration_run_times: List[float] = [0.0, 0.0, 0.0] self.recent_iteration_index: int = 0 self.cached_ls_values: Dict[str, Dict[str, str]] = {} - self.ls: Optional[List[Dict[str, str]]] = None - self.ls_ack = 0 - self.volume: str = '' - self.volume_ack = 0 def deploy_daemon_unit(self, config: Dict[str, str] = {}) -> None: if not config: @@ -3663,10 +3657,12 @@ WantedBy=ceph-{fsid}.target networks_list[key] = {k: list(v)} data = json.dumps({'host': self.host, - 'ls': self.ls if self.ack == self.ls_ack else [], + 'ls': (self.ls_gatherer.data if self.ack == self.ls_gatherer.ack + and self.ls_gatherer.data is not None else []), 'networks': networks_list, 'facts': HostFacts(self.ctx).dump(), - 'volume': self.volume if self.ack == self.volume_ack else '', + 'volume': (self.volume_gatherer.data if self.ack == self.volume_gatherer.ack + and self.volume_gatherer.data is not None else ''), 'ack': str(ack), 'keyring': self.keyring, 'port': self.listener_port}) @@ -3821,50 +3817,18 @@ WantedBy=ceph-{fsid}.target return ls -class LsGatherer(Thread): - def __init__(self, agent: 'CephadmAgent') -> None: - self.agent = agent - self.stop = False - self.recent_iteration_run_times: List[float] = [0.0, 0.0, 0.0] - self.recent_iteration_index: int = 0 - super(LsGatherer, self).__init__(target=self.run) - - def run(self) -> None: - while not self.stop: - try: - start_time = time.monotonic() - - ack = self.agent.ack - self.agent.ls = self.agent._get_ls() - if ack != self.agent.ls_ack: - self.agent.ls_ack = ack - self.agent.wakeup() - - end_time = time.monotonic() - run_time = datetime.timedelta(seconds=(end_time - start_time)) - self.recent_iteration_run_times[self.recent_iteration_index] = run_time.total_seconds() - self.recent_iteration_index = (self.recent_iteration_index + 1) % 3 - run_time_average = sum(self.recent_iteration_run_times, 0.0) / len([t for t in self.recent_iteration_run_times if t]) - - self.agent.ls_event.wait(max(self.agent.loop_interval - int(run_time_average), 0)) - self.agent.ls_event.clear() - except Exception as e: - logger.error(f'Ls Gatherer encountered exception: {e}') - - def shutdown(self) -> None: - self.stop = True - - def wakeup(self) -> None: - self.agent.ls_event.set() - - -class VolumeGatherer(Thread): - def __init__(self, agent: 'CephadmAgent') -> None: +class AgentGatherer(Thread): + def __init__(self, agent: 'CephadmAgent', func: Callable, gatherer_type: str = 'Unnamed', initial_ack: int = 0) -> None: self.agent = agent + self.func = func + self.gatherer_type = gatherer_type + self.ack = initial_ack + self.event = Event() + self.data: Any = None self.stop = False self.recent_iteration_run_times: List[float] = [0.0, 0.0, 0.0] self.recent_iteration_index: int = 0 - super(VolumeGatherer, self).__init__(target=self.run) + super(AgentGatherer, self).__init__(target=self.run) def run(self) -> None: while not self.stop: @@ -3873,12 +3837,12 @@ class VolumeGatherer(Thread): ack = self.agent.ack try: - self.agent.volume = self.agent._ceph_volume(self.agent.device_enhanced_scan) + self.data = self.func() except Exception as e: - logger.error(f'Failed to get ceph-volume metadata: {e}') - self.agent.volume = '' - if ack != self.agent.volume_ack: - self.agent.volume_ack = ack + logger.error(f'{self.gatherer_type} Gatherer encountered exception gathering data: {e}') + self.data = None + if ack != self.ack: + self.ack = ack self.agent.wakeup() end_time = time.monotonic() @@ -3887,16 +3851,16 @@ class VolumeGatherer(Thread): self.recent_iteration_index = (self.recent_iteration_index + 1) % 3 run_time_average = sum(self.recent_iteration_run_times, 0.0) / len([t for t in self.recent_iteration_run_times if t]) - self.agent.volume_event.wait(max(self.agent.loop_interval - int(run_time_average), 0)) - self.agent.volume_event.clear() + self.event.wait(max(self.agent.loop_interval - int(run_time_average), 0)) + self.event.clear() except Exception as e: - logger.error(f'Ls Gatherer encountered exception: {e}') + logger.error(f'{self.gatherer_type} Gatherer encountered exception: {e}') def shutdown(self) -> None: self.stop = True def wakeup(self) -> None: - self.agent.volume_event.set() + self.event.set() def command_agent(ctx: CephadmContext) -> None: From 8d99512e14189b9fc7bd23ecf97fab8d37bb65e2 Mon Sep 17 00:00:00 2001 From: Javier Cacheiro Date: Tue, 26 Oct 2021 17:14:15 +0200 Subject: [PATCH 0332/2866] doc: data must be a byte string Data must be a byte string in python 3. Signed-off-by: Javier Cacheiro --- doc/rbd/api/librbdpy.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/rbd/api/librbdpy.rst b/doc/rbd/api/librbdpy.rst index dc01f2373ae93..7a74b049828cd 100644 --- a/doc/rbd/api/librbdpy.rst +++ b/doc/rbd/api/librbdpy.rst @@ -29,7 +29,7 @@ image:: To perform I/O on the image, you instantiate an :class:rbd.Image object:: image = rbd.Image(ioctx, 'myimage') - data = 'foo' * 200 + data = b'foo' * 200 image.write(data, 0) This writes 'foo' to the first 600 bytes of the image. Note that data @@ -55,7 +55,7 @@ block:: rbd_inst.create(ioctx, 'myimage', size) image = rbd.Image(ioctx, 'myimage') try: - data = 'foo' * 200 + data = b'foo' * 200 image.write(data, 0) finally: image.close() @@ -75,7 +75,7 @@ above example becomes:: size = 4 * 1024**3 # 4 GiB rbd_inst.create(ioctx, 'myimage', size) with rbd.Image(ioctx, 'myimage') as image: - data = 'foo' * 200 + data = b'foo' * 200 image.write(data, 0) API Reference From e650bc6e87a17471a7c31989d560c708af0621e8 Mon Sep 17 00:00:00 2001 From: Rishabh Dave Date: Tue, 26 Oct 2021 15:54:20 +0530 Subject: [PATCH 0333/2866] qa/vstart_runner: add "managers" to LocalContext instances Without this plenty tests become incompatible with vstart_runner.py. Ideally, vstart_runner.py should've been updated in commit 7812cfb6744f. Fixes: https://tracker.ceph.com/issues/53043 Signed-off-by: Rishabh Dave --- qa/tasks/vstart_runner.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/qa/tasks/vstart_runner.py b/qa/tasks/vstart_runner.py index 916256495d454..63976fc9d6e38 100644 --- a/qa/tasks/vstart_runner.py +++ b/qa/tasks/vstart_runner.py @@ -996,10 +996,13 @@ def run(self, *args, **kwargs): class LocalContext(object): def __init__(self): - self.config = {} + self.config = {'cluster': 'ceph'} self.teuthology_config = teuth_config self.cluster = LocalCluster() self.daemons = DaemonGroup() + if not hasattr(self, 'managers'): + self.managers = {} + self.managers[self.config['cluster']] = LocalCephManager() # Shove some LocalDaemons into the ctx.daemons DaemonGroup instance so that any # tests that want to look these up via ctx can do so. From 70c9c3d341f168e491d5aed22c95fade7f0067f4 Mon Sep 17 00:00:00 2001 From: Adam King Date: Tue, 26 Oct 2021 11:47:50 -0400 Subject: [PATCH 0334/2866] mgr/cephadm: unit test for osd preview limit warning Signed-off-by: Adam King --- src/pybind/mgr/cephadm/tests/test_cephadm.py | 22 ++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/pybind/mgr/cephadm/tests/test_cephadm.py b/src/pybind/mgr/cephadm/tests/test_cephadm.py index aeefffd3612ba..2de2b38484bc7 100644 --- a/src/pybind/mgr/cephadm/tests/test_cephadm.py +++ b/src/pybind/mgr/cephadm/tests/test_cephadm.py @@ -699,6 +699,28 @@ def test_create_noncollocated_osd(self, cephadm_module): out = wait(cephadm_module, c) assert out == "Created no osd(s) on host test; already created?" + @mock.patch("cephadm.serve.CephadmServe._run_cephadm", _run_cephadm('{}')) + @mock.patch('cephadm.services.osd.OSDService._run_ceph_volume_command') + @mock.patch('cephadm.services.osd.OSDService.driveselection_to_ceph_volume') + @mock.patch('cephadm.services.osd.OsdIdClaims.refresh', lambda _: None) + @mock.patch('cephadm.services.osd.OsdIdClaims.get', lambda _: {}) + def test_limit_not_reached(self, d_to_cv, _run_cv_cmd, cephadm_module): + with with_host(cephadm_module, 'test'): + dg = DriveGroupSpec(placement=PlacementSpec(host_pattern='test'), + data_devices=DeviceSelection(limit=5, rotational=1), + service_id='not_enough') + + disks_found = [ + '[{"data": "/dev/vdb", "data_size": "50.00 GB", "encryption": "None"}, {"data": "/dev/vdc", "data_size": "50.00 GB", "encryption": "None"}]'] + d_to_cv.return_value = 'foo' + _run_cv_cmd.return_value = (disks_found, '', 0) + preview = cephadm_module.osd_service.generate_previews([dg], 'test') + + for osd in preview: + assert 'notes' in osd + assert osd['notes'] == [ + 'NOTE: Did not find enough disks matching filter on host test to reach data device limit (Found: 2 | Limit: 5)'] + @mock.patch("cephadm.serve.CephadmServe._run_cephadm", _run_cephadm('{}')) def test_prepare_drivegroup(self, cephadm_module): with with_host(cephadm_module, 'test'): From c8810e46e8f28ed204ef2d39f552f96f4a6cfe39 Mon Sep 17 00:00:00 2001 From: Patrick Donnelly Date: Wed, 20 Oct 2021 21:43:54 -0400 Subject: [PATCH 0335/2866] qa: lengthen health warning wait It's just a little too short! Fixes: https://tracker.ceph.com/issues/52995 Signed-off-by: Patrick Donnelly --- qa/tasks/ceph_test_case.py | 1 + qa/tasks/cephfs/filesystem.py | 11 +++++++++++ qa/tasks/cephfs/test_failover.py | 21 ++++++--------------- qa/tasks/cephfs/test_scrub_checks.py | 4 +--- qa/tasks/cephfs/test_snapshots.py | 18 +++++++----------- 5 files changed, 26 insertions(+), 29 deletions(-) diff --git a/qa/tasks/ceph_test_case.py b/qa/tasks/ceph_test_case.py index 0ce9e81f1c138..8f5b27e28a818 100644 --- a/qa/tasks/ceph_test_case.py +++ b/qa/tasks/ceph_test_case.py @@ -156,6 +156,7 @@ def seen_health_warning(): log.debug("Not found expected summary strings yet ({0})".format(summary_strings)) return False + log.info(f"waiting {timeout}s for health warning matching {pattern}") self.wait_until_true(seen_health_warning, timeout) def wait_for_health_clear(self, timeout): diff --git a/qa/tasks/cephfs/filesystem.py b/qa/tasks/cephfs/filesystem.py index 0f846cf1e0e63..9e079c9c80c0d 100644 --- a/qa/tasks/cephfs/filesystem.py +++ b/qa/tasks/cephfs/filesystem.py @@ -478,6 +478,17 @@ def delete_all_filesystems(self): for fs in self.status().get_filesystems(): Filesystem(ctx=self._ctx, fscid=fs['id']).destroy() + @property + def beacon_timeout(self): + """ + Generate an acceptable timeout for the mons to drive some MDSMap change + because of missed beacons from some MDS. This involves looking up the + grace period in use by the mons and adding an acceptable buffer. + """ + + grace = float(self.get_config("mds_beacon_grace", service_type="mon")) + return grace*2+15 + class Filesystem(MDSCluster): """ diff --git a/qa/tasks/cephfs/test_failover.py b/qa/tasks/cephfs/test_failover.py index 7791e090e0c61..7558ede168483 100644 --- a/qa/tasks/cephfs/test_failover.py +++ b/qa/tasks/cephfs/test_failover.py @@ -306,8 +306,6 @@ def test_simple(self): in thrashing tests. """ - grace = float(self.fs.get_config("mds_beacon_grace", service_type="mon")) - (original_active, ) = self.fs.get_active_names() original_standbys = self.mds_cluster.get_standby_daemons() @@ -321,7 +319,7 @@ def promoted(): log.info("Waiting for promotion of one of the original standbys {0}".format( original_standbys)) - self.wait_until_true(promoted, timeout=grace*2) + self.wait_until_true(promoted, timeout=self.fs.beacon_timeout) # Start the original rank 0 daemon up again, see that he becomes a standby self.fs.mds_restart(original_active) @@ -343,8 +341,6 @@ def test_client_abort(self): if not require_active: self.skipTest("fuse_require_active_mds is not set") - grace = float(self.fs.get_config("mds_beacon_grace", service_type="mon")) - # Check it's not laggy to begin with (original_active, ) = self.fs.get_active_names() self.assertNotIn("laggy_since", self.fs.status().get_mds(original_active)) @@ -367,7 +363,7 @@ def laggy(): return True - self.wait_until_true(laggy, grace * 2) + self.wait_until_true(laggy, self.fs.beacon_timeout) with self.assertRaises(CommandFailedError): self.mounts[0].mount_wait() @@ -379,8 +375,6 @@ def test_standby_count_wanted(self): # Need all my standbys up as well as the active daemons self.wait_for_daemon_start() - grace = float(self.fs.get_config("mds_beacon_grace", service_type="mon")) - standbys = self.mds_cluster.get_standby_daemons() self.assertGreaterEqual(len(standbys), 1) self.fs.mon_manager.raw_cluster_cmd('fs', 'set', self.fs.name, 'standby_count_wanted', str(len(standbys))) @@ -388,8 +382,7 @@ def test_standby_count_wanted(self): # Kill a standby and check for warning victim = standbys.pop() self.fs.mds_stop(victim) - log.info("waiting for insufficient standby daemon warning") - self.wait_for_health("MDS_INSUFFICIENT_STANDBY", grace*2) + self.wait_for_health("MDS_INSUFFICIENT_STANDBY", self.fs.beacon_timeout) # restart the standby, see that he becomes a standby, check health clears self.fs.mds_restart(victim) @@ -403,8 +396,7 @@ def test_standby_count_wanted(self): standbys = self.mds_cluster.get_standby_daemons() self.assertGreaterEqual(len(standbys), 1) self.fs.mon_manager.raw_cluster_cmd('fs', 'set', self.fs.name, 'standby_count_wanted', str(len(standbys)+1)) - log.info("waiting for insufficient standby daemon warning") - self.wait_for_health("MDS_INSUFFICIENT_STANDBY", grace*2) + self.wait_for_health("MDS_INSUFFICIENT_STANDBY", self.fs.beacon_timeout) # Set it to 0 self.fs.mon_manager.raw_cluster_cmd('fs', 'set', self.fs.name, 'standby_count_wanted', '0') @@ -420,7 +412,6 @@ def test_discontinuous_mdsmap(self): self.mount_a.umount_wait() - grace = float(self.fs.get_config("mds_beacon_grace", service_type="mon")) monc_timeout = float(self.fs.get_config("mon_client_ping_timeout", service_type="mds")) mds_0 = self.fs.get_rank(rank=0, status=status) @@ -428,7 +419,7 @@ def test_discontinuous_mdsmap(self): self.fs.rank_signal(signal.SIGSTOP, rank=0, status=status) self.wait_until_true( lambda: "laggy_since" in self.fs.get_rank(), - timeout=grace * 2 + timeout=self.fs.beacon_timeout ) self.fs.rank_fail(rank=1) @@ -441,7 +432,7 @@ def test_discontinuous_mdsmap(self): self.fs.rank_signal(signal.SIGCONT, rank=0) self.wait_until_true( lambda: "laggy_since" not in self.fs.get_rank(rank=0), - timeout=grace * 2 + timeout=self.fs.beacon_timeout ) # mds.b will be stuck at 'reconnect' state if snapserver gets confused diff --git a/qa/tasks/cephfs/test_scrub_checks.py b/qa/tasks/cephfs/test_scrub_checks.py index bcfc2fc9a3a1f..ac86b24f0a57f 100644 --- a/qa/tasks/cephfs/test_scrub_checks.py +++ b/qa/tasks/cephfs/test_scrub_checks.py @@ -168,15 +168,13 @@ def test_scrub_task_status_on_mds_failover(self): # Kill the rank 0 self.fs.mds_stop(original_active) - grace = float(self.fs.get_config("mds_beacon_grace", service_type="mon")) - def promoted(): active = self.fs.get_active_names() return active and active[0] in original_standbys log.info("Waiting for promotion of one of the original standbys {0}".format( original_standbys)) - self.wait_until_true(promoted, timeout=grace*2) + self.wait_until_true(promoted, timeout=self.fs.beacon_timeout) self._check_task_status_na() diff --git a/qa/tasks/cephfs/test_snapshots.py b/qa/tasks/cephfs/test_snapshots.py index 30d4e5801e0df..bc3e6a16c8a5a 100644 --- a/qa/tasks/cephfs/test_snapshots.py +++ b/qa/tasks/cephfs/test_snapshots.py @@ -70,8 +70,6 @@ def test_kill_mdstable(self): self.fs.set_max_mds(2) status = self.fs.wait_for_daemons() - grace = float(self.fs.get_config("mds_beacon_grace", service_type="mon")) - # setup subtrees self.mount_a.run_shell(["mkdir", "-p", "d1/dir"]) self.mount_a.setfattr("d1", "ceph.dir.pin", "1") @@ -92,7 +90,7 @@ def test_kill_mdstable(self): self.fs.rank_freeze(True, rank=0) self.fs.rank_asok(['config', 'set', "mds_kill_mdstable_at", "{0}".format(i)], rank=0, status=status) proc = self.mount_a.run_shell(["mkdir", "d1/dir/.snap/s1{0}".format(i)], wait=False) - self.wait_until_true(lambda: "laggy_since" in self.fs.get_rank(rank=0), timeout=grace*2); + self.wait_until_true(lambda: "laggy_since" in self.fs.get_rank(rank=0), timeout=self.fs.beacon_timeout); self.delete_mds_coredump(rank0['name']); self.fs.rank_fail(rank=0) @@ -120,7 +118,7 @@ def test_kill_mdstable(self): self.fs.rank_freeze(True, rank=1) # prevent failover... self.fs.rank_asok(['config', 'set', "mds_kill_mdstable_at", "{0}".format(i)], rank=0, status=status) proc = self.mount_a.run_shell(["mkdir", "d1/dir/.snap/s2{0}".format(i)], wait=False) - self.wait_until_true(lambda: "laggy_since" in self.fs.get_rank(rank=0), timeout=grace*3); + self.wait_until_true(lambda: "laggy_since" in self.fs.get_rank(rank=0), timeout=self.fs.beacon_timeout); self.delete_mds_coredump(rank0['name']); self.fs.rank_signal(signal.SIGKILL, rank=1) @@ -168,7 +166,7 @@ def test_kill_mdstable(self): self.fs.rank_freeze(True, rank=1) # prevent failover... self.fs.rank_asok(['config', 'set', "mds_kill_mdstable_at", "{0}".format(i)], rank=1, status=status) proc = self.mount_a.run_shell(["mkdir", "d1/dir/.snap/s3{0}".format(i)], wait=False) - self.wait_until_true(lambda: "laggy_since" in self.fs.get_rank(rank=1), timeout=grace*2); + self.wait_until_true(lambda: "laggy_since" in self.fs.get_rank(rank=1), timeout=self.fs.beacon_timeout); self.delete_mds_coredump(rank1['name']); self.mount_a.kill() @@ -210,7 +208,7 @@ def test_kill_mdstable(self): self.fs.rank_asok(['config', 'set', "mds_kill_mdstable_at", "8"], rank=0, status=status) self.fs.rank_asok(['config', 'set', "mds_kill_mdstable_at", "3"], rank=1, status=status) proc = self.mount_a.run_shell(["mkdir", "d1/dir/.snap/s4"], wait=False) - self.wait_until_true(lambda: "laggy_since" in self.fs.get_rank(rank=1), timeout=grace*2); + self.wait_until_true(lambda: "laggy_since" in self.fs.get_rank(rank=1), timeout=self.fs.beacon_timeout); self.delete_mds_coredump(rank1['name']); self.mount_a.kill() @@ -223,7 +221,7 @@ def test_kill_mdstable(self): self.wait_for_daemon_start([rank1['name']]) # rollback triggers assertion - self.wait_until_true(lambda: "laggy_since" in self.fs.get_rank(rank=0), timeout=grace*2); + self.wait_until_true(lambda: "laggy_since" in self.fs.get_rank(rank=0), timeout=self.fs.beacon_timeout); self.delete_mds_coredump(rank0['name']); self.fs.rank_fail(rank=0) self.fs.mds_restart(rank0['name']) @@ -244,8 +242,6 @@ def test_snapclient_cache(self): self.fs.set_max_mds(3) status = self.fs.wait_for_daemons() - grace = float(self.fs.get_config("mds_beacon_grace", service_type="mon")) - self.mount_a.run_shell(["mkdir", "-p", "d0/d1/dir"]) self.mount_a.run_shell(["mkdir", "-p", "d0/d2/dir"]) self.mount_a.setfattr("d0", "ceph.dir.pin", "0") @@ -302,7 +298,7 @@ def _check_snapclient_cache(snaps_dump, cache_dump=None, rank=0): self.fs.rank_freeze(True, rank=2) self.fs.rank_asok(['config', 'set', "mds_kill_mdstable_at", "9"], rank=2, status=status) proc = self.mount_a.run_shell(["mkdir", "d0/d1/dir/.snap/s3"], wait=False) - self.wait_until_true(lambda: "laggy_since" in self.fs.get_rank(rank=2), timeout=grace*2); + self.wait_until_true(lambda: "laggy_since" in self.fs.get_rank(rank=2), timeout=self.fs.beacon_timeout); self.delete_mds_coredump(rank2['name']); # mksnap should wait for notify ack from mds.2 @@ -328,7 +324,7 @@ def _check_snapclient_cache(snaps_dump, cache_dump=None, rank=0): self.fs.rank_asok(['config', 'set', "mds_kill_mdstable_at", "4"], rank=2, status=status) last_created = self._get_last_created_snap(rank=0) proc = self.mount_a.run_shell(["mkdir", "d0/d2/dir/.snap/s{0}".format(i)], wait=False) - self.wait_until_true(lambda: "laggy_since" in self.fs.get_rank(rank=2), timeout=grace*2); + self.wait_until_true(lambda: "laggy_since" in self.fs.get_rank(rank=2), timeout=self.fs.beacon_timeout); self.delete_mds_coredump(rank2['name']); self.mount_a.kill() From ed1c91dc6bd4203671d643700e2d2428edb0c3d0 Mon Sep 17 00:00:00 2001 From: Adam King Date: Tue, 26 Oct 2021 15:33:57 -0400 Subject: [PATCH 0336/2866] mgr/cephadm: agent: kick serve loop if number of daemons on host changes In case something needs to be done about it in serve loop e.g. redeploying a daemon that has disappeared. Signed-off-by: Adam King --- src/pybind/mgr/cephadm/agent.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/pybind/mgr/cephadm/agent.py b/src/pybind/mgr/cephadm/agent.py index 1de205b71df08..4ee5912ad4e4b 100644 --- a/src/pybind/mgr/cephadm/agent.py +++ b/src/pybind/mgr/cephadm/agent.py @@ -174,6 +174,7 @@ def handle_metadata(self, data: Dict[str, Any]) -> None: self.mgr.cache.agent_timestamp[host] = datetime_now() error_daemons_old = set([dd.name() for dd in self.mgr.cache.get_error_daemons()]) + daemon_count_old = len(self.mgr.cache.get_daemons_by_host(host)) agents_down = [] for h in self.mgr.cache.get_hosts(): @@ -204,9 +205,12 @@ def handle_metadata(self, data: Dict[str, Any]) -> None: ret = Devices.from_json(json.loads(data['volume'])) self.mgr.cache.update_host_devices(host, ret.devices) - if error_daemons_old != set([dd.name() for dd in self.mgr.cache.get_error_daemons()]): + if ( + error_daemons_old != set([dd.name() for dd in self.mgr.cache.get_error_daemons()]) + or daemon_count_old != len(self.mgr.cache.get_daemons_by_host(host)) + ): self.mgr.log.info( - f'Change detected in daemons in error state from {host} agent metadata. Kicking serve loop') + f'Change detected in state of daemons from {host} agent metadata. Kicking serve loop') self.mgr._kick_serve_loop() if up_to_date and ('ls' in data and data['ls']): From ed8107e8d4f206069fe8cd7ce641ea68861b8976 Mon Sep 17 00:00:00 2001 From: Radoslaw Zarzynski Date: Tue, 26 Oct 2021 18:03:31 +0000 Subject: [PATCH 0337/2866] crimson/net: FixedCPUServerSocket::accept() respects the listening addr's type. For the sake of compliance with the classical OSD where the type of the listening address is being reflected in the address type of an accepted connection: ``` int PosixServerSocketImpl::accept(ConnectedSocket *sock, const SocketOptions &opt, entity_addr_t *out, Worker *w) { // ... sockaddr_storage ss; // ... int sd = accept_cloexec(_fd, (sockaddr*)&ss, &slen); if (sd < 0) { return -ceph_sock_errno(); } // ... out->set_type(addr_type); out->set_sockaddr((sockaddr*)&ss); handler.set_priority(sd, opt.priority, out->get_family()); } ``` In Rook crimson binds explicitly to v2 address: ``` INFO 2021-10-26 17:55:08,116 [shard 0] osd - picked address v2:0.0.0.0:0/0 ERROR 2021-10-26 17:55:08,116 [shard 0] none - Falling back to public interface INFO 2021-10-26 17:55:08,116 [shard 0] osd - picked address v2:0.0.0.0:0/0 ``` Signed-off-by: Radoslaw Zarzynski --- src/crimson/net/Socket.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/crimson/net/Socket.h b/src/crimson/net/Socket.h index f2b9c2713913b..4e33122a43fbf 100644 --- a/src/crimson/net/Socket.h +++ b/src/crimson/net/Socket.h @@ -226,7 +226,7 @@ class FixedCPUServerSocket auto [socket, paddr] = std::move(accept_result); entity_addr_t peer_addr; peer_addr.set_sockaddr(&paddr.as_posix_sockaddr()); - peer_addr.set_type(entity_addr_t::TYPE_ANY); + peer_addr.set_type(ss.addr.get_type()); SocketRef _socket = std::make_unique( std::move(socket), Socket::side_t::acceptor, peer_addr.get_port(), Socket::construct_tag{}); From 37d006321fa1fae5994728212302c410cbfaee13 Mon Sep 17 00:00:00 2001 From: Adam King Date: Tue, 26 Oct 2021 16:34:01 -0400 Subject: [PATCH 0338/2866] cephadm: agent gatherers wake up agent if change detected In this case it's just for ls to faster report added/removed or change in state of daemons we detect. Signed-off-by: Adam King --- src/cephadm/cephadm | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/src/cephadm/cephadm b/src/cephadm/cephadm index 8bd9bd50961b3..1364c51ac9974 100755 --- a/src/cephadm/cephadm +++ b/src/cephadm/cephadm @@ -3684,7 +3684,7 @@ WantedBy=ceph-{fsid}.target self.event.wait(max(self.loop_interval - int(run_time_average), 0)) self.event.clear() - def _ceph_volume(self, enhanced: bool = False) -> str: + def _ceph_volume(self, enhanced: bool = False) -> Tuple[str, bool]: self.ctx.command = 'inventory --format=json'.split() if enhanced: self.ctx.command.append('--with-lsm') @@ -3697,7 +3697,7 @@ WantedBy=ceph-{fsid}.target stdout = stream.getvalue() if stdout: - return stdout + return (stdout, False) else: raise Exception('ceph-volume returned empty value') @@ -3774,15 +3774,17 @@ WantedBy=ceph-{fsid}.target name_id_mapping[name] = id return name_id_mapping - def _get_ls(self) -> List[Dict[str, str]]: + def _get_ls(self) -> Tuple[List[Dict[str, str]], bool]: if not self.cached_ls_values: logger.info('No cached ls output. Running full daemon ls') ls = list_daemons(self.ctx) for d in ls: self.cached_ls_values[d['name']] = d + return (ls, True) else: ls_subset = self._daemon_ls_subset() need_full_ls = False + state_change = False if set(self.cached_ls_values.keys()) != set(ls_subset.keys()): # case for a new daemon in ls or an old daemon no longer appearing. # If that happens we need a full ls @@ -3790,7 +3792,7 @@ WantedBy=ceph-{fsid}.target ls = list_daemons(self.ctx) for d in ls: self.cached_ls_values[d['name']] = d - return ls + return (ls, True) for daemon, info in self.cached_ls_values.items(): if info['style'] == 'legacy': # for legacy containers, ls_subset just grabs all the info @@ -3801,6 +3803,14 @@ WantedBy=ceph-{fsid}.target # info we didn't grab like version and start time could have changed need_full_ls = True break + + # want to know if a daemons state change because in those cases we want + # to report back quicker + if ( + self.cached_ls_values[daemon]['enabled'] != ls_subset[daemon]['enabled'] + or self.cached_ls_values[daemon]['state'] != ls_subset[daemon]['state'] + ): + state_change = True # if we reach here, container id matched. Update the few values we do track # from ls subset: state, enabled, memory_usage. self.cached_ls_values[daemon]['enabled'] = ls_subset[daemon]['enabled'] @@ -3812,9 +3822,10 @@ WantedBy=ceph-{fsid}.target ls = list_daemons(self.ctx) for d in ls: self.cached_ls_values[d['name']] = d + return (ls, True) else: ls = [info for daemon, info in self.cached_ls_values.items()] - return ls + return (ls, state_change) class AgentGatherer(Thread): @@ -3836,12 +3847,13 @@ class AgentGatherer(Thread): start_time = time.monotonic() ack = self.agent.ack + change = False try: - self.data = self.func() + self.data, change = self.func() except Exception as e: logger.error(f'{self.gatherer_type} Gatherer encountered exception gathering data: {e}') self.data = None - if ack != self.ack: + if ack != self.ack or change: self.ack = ack self.agent.wakeup() From 37b82b879396279bfd4ba8967b510fa23413b3d0 Mon Sep 17 00:00:00 2001 From: Paul Cuzner Date: Wed, 27 Oct 2021 09:58:17 +1300 Subject: [PATCH 0339/2866] mgr/prometheus: remove cmake tests Temporary removal of the cmake test integration Signed-off-by: Paul Cuzner --- monitoring/prometheus/tests/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monitoring/prometheus/tests/CMakeLists.txt b/monitoring/prometheus/tests/CMakeLists.txt index 5a6a062a13306..15fce8e1e02b7 100644 --- a/monitoring/prometheus/tests/CMakeLists.txt +++ b/monitoring/prometheus/tests/CMakeLists.txt @@ -1,4 +1,4 @@ if(WITH_TESTS) include(AddCephTest) - add_tox_test(prometheus-alerts ${CMAKE_CURRENT_SOURCE_DIR} TOX_ENVS py3) + # add_tox_test(prometheus-alerts ${CMAKE_CURRENT_SOURCE_DIR} TOX_ENVS py3) endif() From 73873e3ef3705ea500f336865a198d81fea5588e Mon Sep 17 00:00:00 2001 From: Jianpeng Ma Date: Wed, 27 Oct 2021 09:32:59 +0800 Subject: [PATCH 0340/2866] librbd/pwl: don't need print cache_bl contents. It will produce very very larger message. Signed-off-by: Jianpeng Ma --- src/librbd/cache/pwl/LogEntry.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/src/librbd/cache/pwl/LogEntry.cc b/src/librbd/cache/pwl/LogEntry.cc index 63873f748d908..4118a9e3e3301 100644 --- a/src/librbd/cache/pwl/LogEntry.cc +++ b/src/librbd/cache/pwl/LogEntry.cc @@ -94,7 +94,6 @@ std::ostream& WriteLogEntry::format(std::ostream &os) const { os << ", " << "cache_buffer=" << (void*)cache_buffer << ", "; os << "cache_bp=" << cache_bp << ", "; - os << "cache_bl=" << cache_bl << ", "; os << "bl_refs=" << bl_refs; return os; } From d595c95ef6c3dc34b8389ff4270639ff1550d269 Mon Sep 17 00:00:00 2001 From: tancz1 Date: Tue, 26 Oct 2021 17:25:20 +0800 Subject: [PATCH 0341/2866] osd: fix 'ceph osd stop ' doesn't take effect when the osd state is in the non-active state, the osd daemon can be stopped. Fixes: https://tracker.ceph.com/issues/53039 Signed-off-by: tan changzhi <544463199@qq.com> --- src/osd/OSD.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc index 0d80505656366..67664389c879b 100644 --- a/src/osd/OSD.cc +++ b/src/osd/OSD.cc @@ -8341,6 +8341,9 @@ void OSD::_committed_osd_maps(epoch_t first, epoch_t last, MOSDMap *m) reset_heartbeat_peers(true); } } + } else if (osdmap->get_epoch() > 0 && osdmap->is_stop(whoami)) { + derr << "map says i am stopped by admin. shutting down." << dendl; + do_shutdown = true; } map_lock.unlock(); From e1bc6f24ff9f4deeb5f025088d9c649149829799 Mon Sep 17 00:00:00 2001 From: Pere Diaz Bou Date: Thu, 21 Oct 2021 11:14:29 +0200 Subject: [PATCH 0342/2866] monitoring: ethernet bonding filter in Network Load Signed-off-by: Pere Diaz Bou --- .../grafana/dashboards/hosts-overview.json | 4 +- .../jsonnet/grafana_dashboards.jsonnet | 53 ++++++++++++++----- 2 files changed, 43 insertions(+), 14 deletions(-) diff --git a/monitoring/grafana/dashboards/hosts-overview.json b/monitoring/grafana/dashboards/hosts-overview.json index b1e2d857dd142..115c1824974fe 100644 --- a/monitoring/grafana/dashboards/hosts-overview.json +++ b/monitoring/grafana/dashboards/hosts-overview.json @@ -514,7 +514,7 @@ "tableColumn": "", "targets": [ { - "expr": "sum (\n irate(node_network_receive_bytes{instance=~\"($osd_hosts|mon_hosts|mds_hosts|rgw_hosts).*\",device!=\"lo\"}[1m]) or\n irate(node_network_receive_bytes_total{instance=~\"($osd_hosts|mon_hosts|mds_hosts|rgw_hosts).*\",device!=\"lo\"}[1m])\n ) +\nsum (\n irate(node_network_transmit_bytes{instance=~\"($osd_hosts|mon_hosts|mds_hosts|rgw_hosts).*\",device!=\"lo\"}[1m]) or\n irate(node_network_transmit_bytes_total{instance=~\"($osd_hosts|mon_hosts|mds_hosts|rgw_hosts).*\",device!=\"lo\"}[1m])\n )", + "expr": "sum (\n\t(\n\t\tirate(node_network_receive_bytes{instance=~\"($osd_hosts|mon_hosts|mds_hosts|rgw_hosts).*\",device!=\"lo\"}[1m]) or\n\t\tirate(node_network_receive_bytes_total{instance=~\"($osd_hosts|mon_hosts|mds_hosts|rgw_hosts).*\",device!=\"lo\"}[1m])\n\t) unless on (device, instance)\n\tlabel_replace((bonding_slaves > 0), \"device\", \"$1\", \"master\", \"(.+)\")\n) +\nsum (\n\t(\n\t\tirate(node_network_transmit_bytes{instance=~\"($osd_hosts|mon_hosts|mds_hosts|rgw_hosts).*\",device!=\"lo\"}[1m]) or\n\t\tirate(node_network_transmit_bytes_total{instance=~\"($osd_hosts|mon_hosts|mds_hosts|rgw_hosts).*\",device!=\"lo\"}[1m])\n\t) unless on (device, instance)\n\tlabel_replace((bonding_slaves > 0), \"device\", \"$1\", \"master\", \"(.+)\")\n\t)\n", "format": "time_series", "intervalFactor": 1, "legendFormat": "", @@ -661,7 +661,7 @@ "steppedLine": false, "targets": [ { - "expr": "topk(10, (sum by(instance) (\n (\n irate(node_network_receive_bytes{instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\",device!=\"lo\"}[1m]) or\n irate(node_network_receive_bytes_total{instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\",device!=\"lo\"}[1m])\n ) +\n (\n irate(node_network_transmit_bytes{instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\",device!=\"lo\"}[1m]) or\n irate(node_network_transmit_bytes_total{instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\",device!=\"lo\"}[1m])\n ))\n )\n)", + "expr": "topk(10, (sum by(instance) (\n(\n\tirate(node_network_receive_bytes{instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\",device!=\"lo\"}[1m]) or\n\tirate(node_network_receive_bytes_total{instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\",device!=\"lo\"}[1m])\n) +\n(\n\tirate(node_network_transmit_bytes{instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\",device!=\"lo\"}[1m]) or\n\tirate(node_network_transmit_bytes_total{instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\",device!=\"lo\"}[1m])\n) unless on (device, instance)\n\tlabel_replace((bonding_slaves > 0), \"device\", \"$1\", \"master\", \"(.+)\"))\n))\n", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{instance}}", diff --git a/monitoring/grafana/dashboards/jsonnet/grafana_dashboards.jsonnet b/monitoring/grafana/dashboards/jsonnet/grafana_dashboards.jsonnet index b2638c20bea84..b9f5bffaa808a 100644 --- a/monitoring/grafana/dashboards/jsonnet/grafana_dashboards.jsonnet +++ b/monitoring/grafana/dashboards/jsonnet/grafana_dashboards.jsonnet @@ -35,7 +35,7 @@ local addStyle(alias, colorMode, colors, dateFormat, decimals, mappingType, patt local HostsOverviewSingleStatPanel(format, title, description, valueName, expr, targetFormat, x, y, w, h) = addSingelStatSchema('$datasource', format, title, description, valueName, false, 100, false, false, '') .addTarget(addTargetSchema(expr, 1, targetFormat, '')) + {gridPos: {x: x, y: y, w: w, h: h}}; - + local HostsOverviewGraphPanel(title, description, formatY1, expr, legendFormat, x, y, w, h) = graphPanelSchema({}, title, description, 'null', false, formatY1, 'short', null, null, 0, 1, '$datasource') .addTargets( @@ -84,11 +84,40 @@ local addStyle(alias, colorMode, colors, dateFormat, decimals, mappingType, patt HostsOverviewSingleStatPanel( 'percent', 'AVG Disk Utilization', 'Average Disk utilization for all OSD data devices (i.e. excludes journal/WAL)', 'current', 'avg (\n label_replace((irate(node_disk_io_time_ms[5m]) / 10 ) or\n (irate(node_disk_io_time_seconds_total[5m]) * 100), \"instance\", \"$1\", \"instance\", \"([^.:]*).*\"\n ) *\n on(instance, device, ceph_daemon) label_replace(label_replace(ceph_disk_occupation{instance=~\"($osd_hosts).*\"}, \"device\", \"$1\", \"device\", \"/dev/(.*)\"), \"instance\", \"$1\", \"instance\", \"([^.:]*).*\")\n)', 'time_series', 16, 0, 4, 5), HostsOverviewSingleStatPanel( - 'bytes', 'Network Load', 'Total send/receive network load across all hosts in the ceph cluster', 'current', 'sum (\n irate(node_network_receive_bytes{instance=~\"($osd_hosts|mon_hosts|mds_hosts|rgw_hosts).*\",device!=\"lo\"}[1m]) or\n irate(node_network_receive_bytes_total{instance=~\"($osd_hosts|mon_hosts|mds_hosts|rgw_hosts).*\",device!=\"lo\"}[1m])\n ) +\nsum (\n irate(node_network_transmit_bytes{instance=~\"($osd_hosts|mon_hosts|mds_hosts|rgw_hosts).*\",device!=\"lo\"}[1m]) or\n irate(node_network_transmit_bytes_total{instance=~\"($osd_hosts|mon_hosts|mds_hosts|rgw_hosts).*\",device!=\"lo\"}[1m])\n )', 'time_series', 20, 0, 4, 5), + 'bytes', 'Network Load', 'Total send/receive network load across all hosts in the ceph cluster', 'current', ||| + sum ( + ( + irate(node_network_receive_bytes{instance=~"($osd_hosts|mon_hosts|mds_hosts|rgw_hosts).*",device!="lo"}[1m]) or + irate(node_network_receive_bytes_total{instance=~"($osd_hosts|mon_hosts|mds_hosts|rgw_hosts).*",device!="lo"}[1m]) + ) unless on (device, instance) + label_replace((bonding_slaves > 0), "device", "$1", "master", "(.+)") + ) + + sum ( + ( + irate(node_network_transmit_bytes{instance=~"($osd_hosts|mon_hosts|mds_hosts|rgw_hosts).*",device!="lo"}[1m]) or + irate(node_network_transmit_bytes_total{instance=~"($osd_hosts|mon_hosts|mds_hosts|rgw_hosts).*",device!="lo"}[1m]) + ) unless on (device, instance) + label_replace((bonding_slaves > 0), "device", "$1", "master", "(.+)") + ) + ||| + , 'time_series', 20, 0, 4, 5), HostsOverviewGraphPanel( 'CPU Busy - Top 10 Hosts', 'Show the top 10 busiest hosts by cpu', 'percent', 'topk(10,100 * ( 1 - (\n avg by(instance) \n (irate(node_cpu_seconds_total{mode=\'idle\',instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\"}[1m]) or\n irate(node_cpu{mode=\'idle\',instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\"}[1m]))\n )\n )\n)', '{{instance}}', 0, 5, 12, 9), HostsOverviewGraphPanel( - 'Network Load - Top 10 Hosts', 'Top 10 hosts by network load', 'Bps', 'topk(10, (sum by(instance) (\n (\n irate(node_network_receive_bytes{instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\",device!=\"lo\"}[1m]) or\n irate(node_network_receive_bytes_total{instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\",device!=\"lo\"}[1m])\n ) +\n (\n irate(node_network_transmit_bytes{instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\",device!=\"lo\"}[1m]) or\n irate(node_network_transmit_bytes_total{instance=~\"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*\",device!=\"lo\"}[1m])\n ))\n )\n)', '{{instance}}', 12, 5, 12, 9), + 'Network Load - Top 10 Hosts', 'Top 10 hosts by network load', 'Bps', ||| + topk(10, (sum by(instance) ( + ( + irate(node_network_receive_bytes{instance=~"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*",device!="lo"}[1m]) or + irate(node_network_receive_bytes_total{instance=~"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*",device!="lo"}[1m]) + ) + + ( + irate(node_network_transmit_bytes{instance=~"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*",device!="lo"}[1m]) or + irate(node_network_transmit_bytes_total{instance=~"($osd_hosts|$mon_hosts|$mds_hosts|$rgw_hosts).*",device!="lo"}[1m]) + ) unless on (device, instance) + label_replace((bonding_slaves > 0), "device", "$1", "master", "(.+)")) + )) + ||| + , '{{instance}}', 12, 5, 12, 9), ]) } { @@ -96,7 +125,7 @@ local addStyle(alias, colorMode, colors, dateFormat, decimals, mappingType, patt local HostDetailsSingleStatPanel(format, title, description, valueName, expr, targetFormat, x, y, w, h) = addSingelStatSchema('$datasource', format, title, description, valueName, false, 100, false, false, '') .addTarget(addTargetSchema(expr, 1, targetFormat, '')) + {gridPos: {x: x, y: y, w: w, h: h}}; - + local HostDetailsGraphPanel(alias, title, description, nullPointMode, formatY1, labelY1, expr, legendFormat, x, y, w, h) = graphPanelSchema(alias, title, description, nullPointMode, false, formatY1, 'short', labelY1, null, null, 1, '$datasource') .addTargets( @@ -385,7 +414,7 @@ local addStyle(alias, colorMode, colors, dateFormat, decimals, mappingType, patt RbdOverviewPanel( 'Throughput', 'Bps', 'round(sum(irate(ceph_rbd_write_bytes[30s])))','round(sum(irate(ceph_rbd_read_bytes[30s])))', 'Write', 'Read', 8, 0, 8, 7), RbdOverviewPanel( - 'Average Latency', 'ns', 'round(sum(irate(ceph_rbd_write_latency_sum[30s])) / sum(irate(ceph_rbd_write_latency_count[30s])))','round(sum(irate(ceph_rbd_read_latency_sum[30s])) / sum(irate(ceph_rbd_read_latency_count[30s])))', 'Write', 'Read', 16, 0, 8, 7), + 'Average Latency', 'ns', 'round(sum(irate(ceph_rbd_write_latency_sum[30s])) / sum(irate(ceph_rbd_write_latency_count[30s])))','round(sum(irate(ceph_rbd_read_latency_sum[30s])) / sum(irate(ceph_rbd_read_latency_count[30s])))', 'Write', 'Read', 16, 0, 8, 7), addTableSchema( '$datasource', '', {"col": 3,"desc": true}, [RgwOverviewStyle('Pool', 'pool', 'string', 'short'),RgwOverviewStyle('Image', 'image', 'string', 'short'),RgwOverviewStyle('IOPS', 'Value', 'number', 'iops'), RgwOverviewStyle('', '/.*/', 'hidden', 'short')], 'Highest IOPS', 'table' ) @@ -396,13 +425,13 @@ local addStyle(alias, colorMode, colors, dateFormat, decimals, mappingType, patt '$datasource', '', {"col": 3,"desc": true}, [RgwOverviewStyle('Pool', 'pool', 'string', 'short'),RgwOverviewStyle('Image', 'image', 'string', 'short'),RgwOverviewStyle('Throughput', 'Value', 'number', 'Bps'), RgwOverviewStyle('', '/.*/', 'hidden', 'short')], 'Highest Throughput', 'table' ) .addTarget( - addTargetSchema('topk(10, sort(sum(irate(ceph_rbd_read_bytes[30s]) + irate(ceph_rbd_write_bytes[30s])) by (pool, image, namespace)))', 1, 'table', '') + addTargetSchema('topk(10, sort(sum(irate(ceph_rbd_read_bytes[30s]) + irate(ceph_rbd_write_bytes[30s])) by (pool, image, namespace)))', 1, 'table', '') ) + {gridPos: {x: 8, y: 7, w: 8, h: 7}}, addTableSchema( '$datasource', '', {"col": 3,"desc": true}, [RgwOverviewStyle('Pool', 'pool', 'string', 'short'),RgwOverviewStyle('Image', 'image', 'string', 'short'),RgwOverviewStyle('Latency', 'Value', 'number', 'ns'), RgwOverviewStyle('', '/.*/', 'hidden', 'short')], 'Highest Latency', 'table' ) .addTarget( - addTargetSchema('topk(10,\n sum(\n irate(ceph_rbd_write_latency_sum[30s]) / clamp_min(irate(ceph_rbd_write_latency_count[30s]), 1) +\n irate(ceph_rbd_read_latency_sum[30s]) / clamp_min(irate(ceph_rbd_read_latency_count[30s]), 1)\n ) by (pool, image, namespace)\n)', 1, 'table', '') + addTargetSchema('topk(10,\n sum(\n irate(ceph_rbd_write_latency_sum[30s]) / clamp_min(irate(ceph_rbd_write_latency_count[30s]), 1) +\n irate(ceph_rbd_read_latency_sum[30s]) / clamp_min(irate(ceph_rbd_read_latency_count[30s]), 1)\n ) by (pool, image, namespace)\n)', 1, 'table', '') ) + {gridPos: {x: 16, y: 7, w: 8, h: 7}} ]) } @@ -413,8 +442,8 @@ local addStyle(alias, colorMode, colors, dateFormat, decimals, mappingType, patt .addTarget(addTargetSchema(expr, 1, targetFormat, '')) + {gridPos: {x: x, y: y, w: w, h: h}}; local PoolOverviewStyle(alias, pattern, type, unit, colorMode, thresholds, valueMaps) = - addStyle(alias, colorMode, ["rgba(245, 54, 54, 0.9)","rgba(237, 129, 40, 0.89)","rgba(50, 172, 45, 0.97)"], 'YYYY-MM-DD HH:mm:ss', 2, 1, pattern, thresholds, type, unit, valueMaps); - + addStyle(alias, colorMode, ["rgba(245, 54, 54, 0.9)","rgba(237, 129, 40, 0.89)","rgba(50, 172, 45, 0.97)"], 'YYYY-MM-DD HH:mm:ss', 2, 1, pattern, thresholds, type, unit, valueMaps); + local PoolOverviewGraphPanel(title, description, formatY1, labelY1, expr, targetFormat, legendFormat, x, y, w, h) = graphPanelSchema({}, title, description, 'null as zero', false, formatY1, 'short', labelY1, null, 0, 1, '$datasource') .addTargets( @@ -445,7 +474,7 @@ local addStyle(alias, colorMode, colors, dateFormat, decimals, mappingType, patt PoolOverviewSingleStatPanel( 'bytes', 'Logical Stored ', 'Total of client data stored in the cluster', 'current', 'sum(ceph_pool_stored)', '', 12, 0, 3, 3), PoolOverviewSingleStatPanel( - 'bytes', 'Compression Savings', 'A compression saving is determined as the data eligible to be compressed minus the capacity used to store the data after compression', 'current', 'sum(ceph_pool_compress_under_bytes - ceph_pool_compress_bytes_used)', '', 15, 0, 3, 3), + 'bytes', 'Compression Savings', 'A compression saving is determined as the data eligible to be compressed minus the capacity used to store the data after compression', 'current', 'sum(ceph_pool_compress_under_bytes - ceph_pool_compress_bytes_used)', '', 15, 0, 3, 3), PoolOverviewSingleStatPanel( 'percent', 'Compression Eligibility', 'Indicates how suitable the data is within the pools that are/have been enabled for compression - averaged across all pools holding compressed data\n', 'current', '(sum(ceph_pool_compress_under_bytes > 0) / sum(ceph_pool_stored_raw and ceph_pool_compress_under_bytes > 0)) * 100', 'table', 18, 0, 3, 3), PoolOverviewSingleStatPanel( @@ -486,7 +515,7 @@ local addStyle(alias, colorMode, colors, dateFormat, decimals, mappingType, patt local PoolDetailSingleStatPanel(format, title, description, valueName, colorValue, gaugeMaxValue, gaugeShow, sparkLineShow, thresholds, expr, targetFormat, x, y, w, h) = addSingelStatSchema('$datasource', format, title, description, valueName, colorValue, gaugeMaxValue, gaugeShow, sparkLineShow, thresholds) .addTarget(addTargetSchema(expr, 1, targetFormat, '')) + {gridPos: {x: x, y: y, w: w, h: h}}; - + local PoolDetailGraphPanel(alias, title, description, formatY1, labelY1, expr, targetFormat, legendFormat, x, y, w, h) = graphPanelSchema(alias, title, description, 'null as zero', false, formatY1, 'short', labelY1, null, null, 1, '$datasource') .addTargets( @@ -625,7 +654,7 @@ local addStyle(alias, colorMode, colors, dateFormat, decimals, mappingType, patt OsdOverviewGraphPanel( {},'Read/Write Profile', 'Show the read/write workload profile overtime', 'short', null, null, 'round(sum(irate(ceph_pool_rd[30s])))', 'Reads', 0, 17, 24, 8) .addTargets([addTargetSchema('round(sum(irate(ceph_pool_wr[30s])))', 1, 'time_series', 'Writes')]) - ]) + ]) } { "osd-device-details.json": From 0cd5f2b2921877015ea9c9bfdcd348a13e6c81a4 Mon Sep 17 00:00:00 2001 From: wangyingbin Date: Wed, 27 Oct 2021 15:44:19 +0800 Subject: [PATCH 0343/2866] doc: The description of "Host" on the page is inconsistent In the "https://docs.ceph.com/en/latest/radosgw/adminops/#radosgw-admin-ops" page, the description of "Host" is not uniform, so I changed it. Signed-off-by: wangyingbin --- doc/radosgw/adminops.rst | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/doc/radosgw/adminops.rst b/doc/radosgw/adminops.rst index 36719887a0f3a..80497d495b539 100644 --- a/doc/radosgw/adminops.rst +++ b/doc/radosgw/adminops.rst @@ -790,7 +790,7 @@ Syntax :: PUT /{admin}/user?subuser&format=json HTTP/1.1 - Host {fqdn} + Host: {fqdn} Request Parameters @@ -899,7 +899,7 @@ Syntax :: POST /{admin}/user?subuser&format=json HTTP/1.1 - Host {fqdn} + Host: {fqdn} Request Parameters @@ -1004,7 +1004,7 @@ Syntax :: DELETE /{admin}/user?subuser&format=json HTTP/1.1 - Host {fqdn} + Host: {fqdn} Request Parameters @@ -1064,7 +1064,7 @@ Syntax :: PUT /{admin}/user?key&format=json HTTP/1.1 - Host {fqdn} + Host: {fqdn} Request Parameters @@ -1181,7 +1181,7 @@ Syntax :: DELETE /{admin}/user?key&format=json HTTP/1.1 - Host {fqdn} + Host: {fqdn} Request Parameters @@ -1242,7 +1242,7 @@ Syntax :: GET /{admin}/bucket?format=json HTTP/1.1 - Host {fqdn} + Host: {fqdn} Request Parameters @@ -1355,7 +1355,7 @@ Syntax :: GET /{admin}/bucket?index&format=json HTTP/1.1 - Host {fqdn} + Host: {fqdn} Request Parameters @@ -1411,7 +1411,7 @@ Syntax :: DELETE /{admin}/bucket?format=json HTTP/1.1 - Host {fqdn} + Host: {fqdn} @@ -1464,7 +1464,7 @@ Syntax :: POST /{admin}/bucket?format=json HTTP/1.1 - Host {fqdn} + Host: {fqdn} Request Parameters @@ -1511,7 +1511,7 @@ Syntax :: PUT /{admin}/bucket?format=json HTTP/1.1 - Host {fqdn} + Host: {fqdn} Request Parameters @@ -1614,7 +1614,7 @@ Syntax :: DELETE /{admin}/bucket?object&format=json HTTP/1.1 - Host {fqdn} + Host: {fqdn} Request Parameters ~~~~~~~~~~~~~~~~~~ @@ -1666,7 +1666,7 @@ Syntax :: GET /{admin}/bucket?policy&format=json HTTP/1.1 - Host {fqdn} + Host: {fqdn} Request Parameters @@ -1718,7 +1718,7 @@ Syntax :: PUT /{admin}/user?caps&format=json HTTP/1.1 - Host {fqdn} + Host: {fqdn} Request Parameters ~~~~~~~~~~~~~~~~~~ @@ -1794,7 +1794,7 @@ Syntax :: DELETE /{admin}/user?caps&format=json HTTP/1.1 - Host {fqdn} + Host: {fqdn} Request Parameters ~~~~~~~~~~~~~~~~~~ From 1f304ad6da42a1662a94c919fe5a705f846f0c6a Mon Sep 17 00:00:00 2001 From: Yingxin Cheng Date: Tue, 26 Oct 2021 14:19:33 +0800 Subject: [PATCH 0344/2866] crimson/os/seastore/journal: proper close journal when test finishes Signed-off-by: Yingxin Cheng --- src/test/crimson/seastore/test_seastore_journal.cc | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/test/crimson/seastore/test_seastore_journal.cc b/src/test/crimson/seastore/test_seastore_journal.cc index 53679f174a1fa..f264bf429c0a8 100644 --- a/src/test/crimson/seastore/test_seastore_journal.cc +++ b/src/test/crimson/seastore/test_seastore_journal.cc @@ -114,6 +114,15 @@ struct journal_test_t : seastar_test_suite_t, SegmentProvider { })); } + seastar::future<> tear_down_fut() final { + return journal->close( + ).handle_error( + crimson::ct_error::all_same_way([](auto e) { + ASSERT_FALSE("Unable to close"); + }) + ); + } + template auto replay(T &&f) { return journal->close( @@ -209,10 +218,6 @@ struct journal_test_t : seastar_test_suite_t, SegmentProvider { return addr; } - seastar::future<> tear_down_fut() final { - return seastar::now(); - } - extent_t generate_extent(size_t blocks) { std::uniform_int_distribution distribution( std::numeric_limits::min(), From d5d2a34025c6b36b39ae6fcd79f5cfb60ea2a27d Mon Sep 17 00:00:00 2001 From: Yingxin Cheng Date: Wed, 27 Oct 2021 14:23:02 +0800 Subject: [PATCH 0345/2866] crimson/os/seastore/store-nbd: cleanup headers Signed-off-by: Yingxin Cheng --- src/crimson/tools/store_nbd/store-nbd.cc | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/crimson/tools/store_nbd/store-nbd.cc b/src/crimson/tools/store_nbd/store-nbd.cc index 0857f0f48decb..9f80c3b2cdcdc 100644 --- a/src/crimson/tools/store_nbd/store-nbd.cc +++ b/src/crimson/tools/store_nbd/store-nbd.cc @@ -36,15 +36,17 @@ #include #include +#include #include -#include +#include #include +#include #include +#include +#include -#include "crimson/common/log.h" #include "crimson/common/config_proxy.h" - -#include "test/crimson/seastar_runner.h" +#include "crimson/common/log.h" #include "block_driver.h" From 6bb3df896f5c9c6ebdecfaf51ecfa88762f2e749 Mon Sep 17 00:00:00 2001 From: Yingxin Cheng Date: Mon, 25 Oct 2021 16:39:29 +0800 Subject: [PATCH 0346/2866] crimson/os/seastore/cache: get block-size from reader Cache cannot get the correct block size before SegmentManager is mounted. Signed-off-by: Yingxin Cheng --- src/crimson/os/seastore/cache.cc | 8 +++----- src/crimson/os/seastore/cache.h | 4 +--- src/crimson/os/seastore/extent_reader.h | 6 ++++++ src/crimson/os/seastore/seastore.cc | 2 +- src/crimson/tools/store_nbd/tm_driver.cc | 2 +- src/test/crimson/seastore/test_btree_lba_manager.cc | 2 +- src/test/crimson/seastore/test_randomblock_manager.cc | 2 +- src/test/crimson/seastore/test_seastore_cache.cc | 2 +- .../crimson/seastore/transaction_manager_test_state.h | 2 +- 9 files changed, 16 insertions(+), 14 deletions(-) diff --git a/src/crimson/os/seastore/cache.cc b/src/crimson/os/seastore/cache.cc index f0eb8370958c5..22c8af25d739d 100644 --- a/src/crimson/os/seastore/cache.cc +++ b/src/crimson/os/seastore/cache.cc @@ -21,10 +21,8 @@ using std::string_view; namespace crimson::os::seastore { Cache::Cache( - ExtentReader &reader, - segment_off_t block_size) - : reader(reader), - block_size(block_size) + ExtentReader &reader) + : reader(reader) { register_metrics(); } @@ -1018,7 +1016,7 @@ record_t Cache::prepare_record(Transaction &t) record_header_fullness.ool_stats.total_bytes += ool_stats.header_bytes; auto record_size = get_encoded_record_length( - record, block_size); + record, reader.get_block_size()); auto inline_overhead = record_size.mdlength + record_size.dlength - record.get_raw_data_size(); efforts.inline_record_overhead_bytes += inline_overhead; diff --git a/src/crimson/os/seastore/cache.h b/src/crimson/os/seastore/cache.h index 7ad1af4a87622..8d89423cacb75 100644 --- a/src/crimson/os/seastore/cache.h +++ b/src/crimson/os/seastore/cache.h @@ -90,7 +90,7 @@ class Cache { crimson::ct_error::input_output_error>; using base_iertr = trans_iertr; - Cache(ExtentReader &reader, segment_off_t block_size); + Cache(ExtentReader &reader); ~Cache(); /// Creates empty transaction by source @@ -614,8 +614,6 @@ class Cache { private: ExtentReader &reader; ///< ref to extent reader - segment_off_t block_size; ///< block size of the segment - ///< manager holding journal records RootBlockRef root; ///< ref to current root ExtentIndex extents; ///< set of live extents diff --git a/src/crimson/os/seastore/extent_reader.h b/src/crimson/os/seastore/extent_reader.h index 66dd6db2b482b..a3f1dd53a6811 100644 --- a/src/crimson/os/seastore/extent_reader.h +++ b/src/crimson/os/seastore/extent_reader.h @@ -14,6 +14,12 @@ class TransactionManager; class ExtentReader { public: + segment_off_t get_block_size() const { + assert(segment_managers.size()); + // assume all segment managers have the same block size + return segment_managers[0]->get_block_size(); + } + using read_ertr = SegmentManager::read_ertr; ExtentReader() { segment_managers.resize(DEVICE_ID_MAX, nullptr); diff --git a/src/crimson/os/seastore/seastore.cc b/src/crimson/os/seastore/seastore.cc index 3ccf0e099a5a7..e12ac875153f7 100644 --- a/src/crimson/os/seastore/seastore.cc +++ b/src/crimson/os/seastore/seastore.cc @@ -1295,7 +1295,7 @@ std::unique_ptr make_seastore( false /* detailed */); auto journal = std::make_unique(*sm, scanner_ref); - auto cache = std::make_unique(scanner_ref, sm->get_block_size()); + auto cache = std::make_unique(scanner_ref); auto lba_manager = lba_manager::create_lba_manager(*sm, *cache); auto epm = std::make_unique(*cache, *lba_manager); diff --git a/src/crimson/tools/store_nbd/tm_driver.cc b/src/crimson/tools/store_nbd/tm_driver.cc index f83801730a7cf..dad672044289f 100644 --- a/src/crimson/tools/store_nbd/tm_driver.cc +++ b/src/crimson/tools/store_nbd/tm_driver.cc @@ -137,7 +137,7 @@ void TMDriver::init() std::vector sms; segment_cleaner->mount(segment_manager->get_device_id(), sms); auto journal = std::make_unique(*segment_manager, scanner_ref); - auto cache = std::make_unique(scanner_ref, segment_manager->get_block_size()); + auto cache = std::make_unique(scanner_ref); auto lba_manager = lba_manager::create_lba_manager(*segment_manager, *cache); auto epm = std::make_unique(*cache, *lba_manager); diff --git a/src/test/crimson/seastore/test_btree_lba_manager.cc b/src/test/crimson/seastore/test_btree_lba_manager.cc index 34f0cd7c8d3ea..76cba8a752576 100644 --- a/src/test/crimson/seastore/test_btree_lba_manager.cc +++ b/src/test/crimson/seastore/test_btree_lba_manager.cc @@ -42,7 +42,7 @@ struct btree_test_base : : segment_manager(segment_manager::create_test_ephemeral()), scanner(new ExtentReader()), journal(*segment_manager, *scanner), - cache(*scanner, segment_manager->get_block_size()), + cache(*scanner), block_size(segment_manager->get_block_size()), next(segment_manager->get_device_id(), 0) { diff --git a/src/test/crimson/seastore/test_randomblock_manager.cc b/src/test/crimson/seastore/test_randomblock_manager.cc index 223f540293981..a09daa5d06c16 100644 --- a/src/test/crimson/seastore/test_randomblock_manager.cc +++ b/src/test/crimson/seastore/test_randomblock_manager.cc @@ -59,7 +59,7 @@ struct rbm_test_t : public seastar_test_suite_t, rbm_test_t() : segment_manager(segment_manager::create_test_ephemeral()), reader(new ExtentReader()), - cache(*reader, segment_manager->get_block_size()) + cache(*reader) { reader->add_segment_manager(segment_manager.get()); device = new nvme_device::TestMemory(DEFAULT_TEST_SIZE); diff --git a/src/test/crimson/seastore/test_seastore_cache.cc b/src/test/crimson/seastore/test_seastore_cache.cc index 8c8427aea389d..355b0a4682a72 100644 --- a/src/test/crimson/seastore/test_seastore_cache.cc +++ b/src/test/crimson/seastore/test_seastore_cache.cc @@ -29,7 +29,7 @@ struct cache_test_t : public seastar_test_suite_t { cache_test_t() : segment_manager(segment_manager::create_test_ephemeral()), reader(new ExtentReader()), - cache(*reader, segment_manager->get_block_size()), + cache(*reader), current(segment_id_t(segment_manager->get_device_id(), 0), 0) { reader->add_segment_manager(segment_manager.get()); } diff --git a/src/test/crimson/seastore/transaction_manager_test_state.h b/src/test/crimson/seastore/transaction_manager_test_state.h index 4e868e173b7e3..47e2bb0b43542 100644 --- a/src/test/crimson/seastore/transaction_manager_test_state.h +++ b/src/test/crimson/seastore/transaction_manager_test_state.h @@ -78,7 +78,7 @@ auto get_transaction_manager( std::move(scanner), true); auto journal = std::make_unique(segment_manager, scanner_ref); - auto cache = std::make_unique(scanner_ref, segment_manager.get_block_size()); + auto cache = std::make_unique(scanner_ref); auto lba_manager = lba_manager::create_lba_manager(segment_manager, *cache); auto epm = std::make_unique(*cache, *lba_manager); From f160986ebe5fd09fa0788624c3878d39f13ae841 Mon Sep 17 00:00:00 2001 From: Yingxin Cheng Date: Wed, 27 Oct 2021 14:33:41 +0800 Subject: [PATCH 0347/2866] crimson/os/seastore/randomblock-manager: cleanup unit test Signed-off-by: Yingxin Cheng --- .../seastore/test_randomblock_manager.cc | 27 +++++-------------- 1 file changed, 7 insertions(+), 20 deletions(-) diff --git a/src/test/crimson/seastore/test_randomblock_manager.cc b/src/test/crimson/seastore/test_randomblock_manager.cc index a09daa5d06c16..e98e9b9289b57 100644 --- a/src/test/crimson/seastore/test_randomblock_manager.cc +++ b/src/test/crimson/seastore/test_randomblock_manager.cc @@ -9,9 +9,6 @@ #include "crimson/os/seastore/random_block_manager/nvme_manager.h" #include "crimson/os/seastore/random_block_manager/nvmedevice.h" #include "test/crimson/seastore/transaction_manager_test_state.h" -#include "crimson/os/seastore/transaction_manager.h" -#include "crimson/os/seastore/cache.h" -#include "crimson/os/seastore/segment_manager/ephemeral.h" using namespace crimson; using namespace crimson::os; @@ -26,13 +23,10 @@ namespace { constexpr uint64_t DEFAULT_TEST_SIZE = 1 << 20; constexpr uint64_t DEFAULT_BLOCK_SIZE = 4096; -struct rbm_test_t : public seastar_test_suite_t, - TMTestState { - segment_manager::EphemeralSegmentManagerRef segment_manager; // Need to be deleted, just for Cache - ExtentReaderRef reader; - Cache cache; +struct rbm_test_t : + public seastar_test_suite_t, TMTestState { std::unique_ptr rbm_manager; - nvme_device::NVMeBlockDevice *device; + std::unique_ptr device; struct rbm_transaction { void add_rbm_allocated_blocks(rbm_alloc_delta_t &d) { @@ -56,14 +50,9 @@ struct rbm_test_t : public seastar_test_suite_t, RandomBlockManager::mkfs_config_t config; paddr_t current; - rbm_test_t() : - segment_manager(segment_manager::create_test_ephemeral()), - reader(new ExtentReader()), - cache(*reader) - { - reader->add_segment_manager(segment_manager.get()); - device = new nvme_device::TestMemory(DEFAULT_TEST_SIZE); - rbm_manager.reset(new NVMeManager(device, std::string())); + rbm_test_t() { + device.reset(new nvme_device::TestMemory(DEFAULT_TEST_SIZE)); + rbm_manager.reset(new NVMeManager(device.get(), std::string())); config.start = paddr_t {0, 0, 0}; config.end = paddr_t {0, 0, DEFAULT_TEST_SIZE}; config.block_size = DEFAULT_BLOCK_SIZE; @@ -75,9 +64,7 @@ struct rbm_test_t : public seastar_test_suite_t, } seastar::future<> tear_down_fut() final { - if (device) { - delete device; - } + device.reset(); return tm_teardown(); } From fa4d1316c956e86bdde7833eef9f2f4a08a58d5c Mon Sep 17 00:00:00 2001 From: Yingxin Cheng Date: Wed, 27 Oct 2021 14:37:54 +0800 Subject: [PATCH 0348/2866] crimson/os/seastore: cleanup transaction_manager_test_state Signed-off-by: Yingxin Cheng --- src/test/crimson/seastore/transaction_manager_test_state.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/test/crimson/seastore/transaction_manager_test_state.h b/src/test/crimson/seastore/transaction_manager_test_state.h index 47e2bb0b43542..10ce0a0037964 100644 --- a/src/test/crimson/seastore/transaction_manager_test_state.h +++ b/src/test/crimson/seastore/transaction_manager_test_state.h @@ -20,7 +20,7 @@ using namespace crimson::os::seastore; class EphemeralTestState { protected: - std::unique_ptr segment_manager; + segment_manager::EphemeralSegmentManagerRef segment_manager; EphemeralTestState() : segment_manager(segment_manager::create_test_ephemeral()) {} @@ -124,13 +124,13 @@ class TMTestState : public EphemeralTestState { TMTestState() : EphemeralTestState() {} - virtual void _init() { + virtual void _init() override { tm = get_transaction_manager(*segment_manager); segment_cleaner = tm->get_segment_cleaner(); lba_manager = tm->get_lba_manager(); } - virtual void _destroy() { + virtual void _destroy() override { segment_cleaner = nullptr; lba_manager = nullptr; tm.reset(); From f73bc6a3752297c5c8e9d89a0ed234c1754bd213 Mon Sep 17 00:00:00 2001 From: Yingxin Cheng Date: Wed, 27 Oct 2021 14:39:10 +0800 Subject: [PATCH 0349/2866] crimson/os/seastore: initialize tests in seastar reactor testing::Test from gtest is constructed and destructed outside seastar application. It is unexpected for most seastore components, and causing undefined behaviors. For example, when registering metrics during construction, the logic needs to be called inside seastar reactor thread because metrics are maintained in reactor thread-local variables. Signed-off-by: Yingxin Cheng --- .../seastore/onode_tree/test_staged_fltree.cc | 87 ++++++++++--------- .../seastore/test_btree_lba_manager.cc | 85 ++++++++++-------- .../seastore/test_randomblock_manager.cc | 8 +- src/test/crimson/seastore/test_seastore.cc | 1 + .../crimson/seastore/test_seastore_cache.cc | 43 +++++---- .../crimson/seastore/test_seastore_journal.cc | 23 ++--- .../seastore/transaction_manager_test_state.h | 8 +- 7 files changed, 139 insertions(+), 116 deletions(-) diff --git a/src/test/crimson/seastore/onode_tree/test_staged_fltree.cc b/src/test/crimson/seastore/onode_tree/test_staged_fltree.cc index a2ef111c87480..2ba456192f8cb 100644 --- a/src/test/crimson/seastore/onode_tree/test_staged_fltree.cc +++ b/src/test/crimson/seastore/onode_tree/test_staged_fltree.cc @@ -198,27 +198,28 @@ TEST_F(a_basic_test_t, 2_node_sizes) } struct b_dummy_tree_test_t : public seastar_test_suite_t { - NodeExtentManagerURef moved_nm; TransactionRef ref_t; - Transaction& t; - ValueBuilderImpl vb; - context_t c; - UnboundedBtree tree; + std::unique_ptr tree; - b_dummy_tree_test_t() - : moved_nm{NodeExtentManager::create_dummy(IS_DUMMY_SYNC)}, - ref_t{make_test_transaction()}, - t{*ref_t}, - c{*moved_nm, vb, t}, - tree{std::move(moved_nm)} {} + b_dummy_tree_test_t() = default; seastar::future<> set_up_fut() override final { - return INTR(tree.mkfs, t).handle_error( + ref_t = make_test_transaction(); + tree.reset( + new UnboundedBtree(NodeExtentManager::create_dummy(IS_DUMMY_SYNC)) + ); + return INTR(tree->mkfs, *ref_t).handle_error( crimson::ct_error::all_same_way([] { ASSERT_FALSE("Unable to mkfs"); }) ); } + + seastar::future<> tear_down_fut() final { + ref_t.reset(); + tree.reset(); + return seastar::now(); + } }; TEST_F(b_dummy_tree_test_t, 3_random_insert_erase_leaf_node) @@ -228,9 +229,9 @@ TEST_F(b_dummy_tree_test_t, 3_random_insert_erase_leaf_node) "\nrandomized leaf node insert:\n"); auto key_s = ghobject_t(); auto key_e = ghobject_t::get_max(); - ASSERT_TRUE(INTR_R(tree.find, t, key_s).unsafe_get0().is_end()); - ASSERT_TRUE(INTR(tree.begin, t).unsafe_get0().is_end()); - ASSERT_TRUE(INTR(tree.last, t).unsafe_get0().is_end()); + ASSERT_TRUE(INTR_R(tree->find, *ref_t, key_s).unsafe_get0().is_end()); + ASSERT_TRUE(INTR(tree->begin, *ref_t).unsafe_get0().is_end()); + ASSERT_TRUE(INTR(tree->last, *ref_t).unsafe_get0().is_end()); std::map> insert_history; @@ -238,24 +239,24 @@ TEST_F(b_dummy_tree_test_t, 3_random_insert_erase_leaf_node) auto f_validate_insert_new = [this, &insert_history] ( const ghobject_t& key, const test_item_t& value) { auto conf = UnboundedBtree::tree_value_config_t{value.get_payload_size()}; - auto [cursor, success] = INTR_R(tree.insert, - t, key, conf).unsafe_get0(); - initialize_cursor_from_item(t, key, value, cursor, success); + auto [cursor, success] = INTR_R(tree->insert, + *ref_t, key, conf).unsafe_get0(); + initialize_cursor_from_item(*ref_t, key, value, cursor, success); insert_history.emplace(key, std::make_tuple(value, cursor)); - auto cursor_ = INTR_R(tree.find, t, key).unsafe_get0(); - ceph_assert(cursor_ != tree.end()); + auto cursor_ = INTR_R(tree->find, *ref_t, key).unsafe_get0(); + ceph_assert(cursor_ != tree->end()); ceph_assert(cursor_.value() == cursor.value()); validate_cursor_from_item(key, value, cursor_); return cursor.value(); }; auto f_validate_erase = [this, &insert_history] (const ghobject_t& key) { - auto cursor_erase = INTR_R(tree.find, t, key).unsafe_get0(); - auto cursor_next = INTR(cursor_erase.get_next, t).unsafe_get0(); - auto cursor_ret = INTR_R(tree.erase, t, cursor_erase).unsafe_get0(); + auto cursor_erase = INTR_R(tree->find, *ref_t, key).unsafe_get0(); + auto cursor_next = INTR(cursor_erase.get_next, *ref_t).unsafe_get0(); + auto cursor_ret = INTR_R(tree->erase, *ref_t, cursor_erase).unsafe_get0(); ceph_assert(cursor_erase.is_end()); ceph_assert(cursor_ret == cursor_next); - auto cursor_lb = INTR_R(tree.lower_bound, t, key).unsafe_get0(); + auto cursor_lb = INTR_R(tree->lower_bound, *ref_t, key).unsafe_get0(); ceph_assert(cursor_lb == cursor_next); auto it = insert_history.find(key); ceph_assert(std::get<1>(it->second).is_end()); @@ -278,10 +279,10 @@ TEST_F(b_dummy_tree_test_t, 3_random_insert_erase_leaf_node) // validate lookup { - auto cursor1_s = INTR_R(tree.lower_bound, t, key_s).unsafe_get0(); + auto cursor1_s = INTR_R(tree->lower_bound, *ref_t, key_s).unsafe_get0(); ASSERT_EQ(cursor1_s.get_ghobj(), key1); ASSERT_EQ(cursor1_s.value(), test_value1); - auto cursor1_e = INTR_R(tree.lower_bound, t, key_e).unsafe_get0(); + auto cursor1_e = INTR_R(tree->lower_bound, *ref_t, key_e).unsafe_get0(); ASSERT_TRUE(cursor1_e.is_end()); } @@ -289,8 +290,8 @@ TEST_F(b_dummy_tree_test_t, 3_random_insert_erase_leaf_node) { auto value1_dup = values.pick(); auto conf = UnboundedBtree::tree_value_config_t{value1_dup.get_payload_size()}; - auto [cursor1_dup, ret1_dup] = INTR_R(tree.insert, - t, key1, conf).unsafe_get0(); + auto [cursor1_dup, ret1_dup] = INTR_R(tree->insert, + *ref_t, key1, conf).unsafe_get0(); ASSERT_FALSE(ret1_dup); validate_cursor_from_item(key1, value1, cursor1_dup); } @@ -371,30 +372,30 @@ TEST_F(b_dummy_tree_test_t, 3_random_insert_erase_leaf_node) std::for_each(kvs.begin(), kvs.end(), [&f_insert_erase_insert] (auto& kv) { f_insert_erase_insert(kv.first, kv.second); }); - ASSERT_EQ(INTR(tree.height, t).unsafe_get0(), 1); - ASSERT_FALSE(tree.test_is_clean()); + ASSERT_EQ(INTR(tree->height, *ref_t).unsafe_get0(), 1); + ASSERT_FALSE(tree->test_is_clean()); for (auto& [k, val] : insert_history) { auto& [v, c] = val; // validate values in tree keep intact - auto cursor = with_trans_intr(t, [this, &k=k](auto& tr) { - return tree.find(tr, k); + auto cursor = with_trans_intr(*ref_t, [this, &k=k](auto& tr) { + return tree->find(tr, k); }).unsafe_get0(); - EXPECT_NE(cursor, tree.end()); + EXPECT_NE(cursor, tree->end()); validate_cursor_from_item(k, v, cursor); // validate values in cursors keep intact validate_cursor_from_item(k, v, c); } { - auto cursor = INTR_R(tree.lower_bound, t, key_s).unsafe_get0(); + auto cursor = INTR_R(tree->lower_bound, *ref_t, key_s).unsafe_get0(); validate_cursor_from_item(smallest_key, smallest_value, cursor); } { - auto cursor = INTR(tree.begin, t).unsafe_get0(); + auto cursor = INTR(tree->begin, *ref_t).unsafe_get0(); validate_cursor_from_item(smallest_key, smallest_value, cursor); } { - auto cursor = INTR(tree.last, t).unsafe_get0(); + auto cursor = INTR(tree->last, *ref_t).unsafe_get0(); validate_cursor_from_item(largest_key, largest_value, cursor); } @@ -409,30 +410,30 @@ TEST_F(b_dummy_tree_test_t, 3_random_insert_erase_leaf_node) std::sort(kvs.begin(), kvs.end(), [](auto& l, auto& r) { return l.first < r.first; }); - auto cursor = INTR(tree.begin, t).unsafe_get0(); + auto cursor = INTR(tree->begin, *ref_t).unsafe_get0(); for (auto& [k, v] : kvs) { ASSERT_FALSE(cursor.is_end()); validate_cursor_from_item(k, v, cursor); - cursor = INTR(cursor.get_next, t).unsafe_get0(); + cursor = INTR(cursor.get_next, *ref_t).unsafe_get0(); } ASSERT_TRUE(cursor.is_end()); } std::ostringstream oss; - tree.dump(t, oss); + tree->dump(*ref_t, oss); logger().info("\n{}\n", oss.str()); // randomized erase until empty std::random_shuffle(kvs.begin(), kvs.end()); for (auto& [k, v] : kvs) { - auto e_size = with_trans_intr(t, [this, &k=k](auto& tr) { - return tree.erase(tr, k); + auto e_size = with_trans_intr(*ref_t, [this, &k=k](auto& tr) { + return tree->erase(tr, k); }).unsafe_get0(); ASSERT_EQ(e_size, 1); } - auto cursor = INTR(tree.begin, t).unsafe_get0(); + auto cursor = INTR(tree->begin, *ref_t).unsafe_get0(); ASSERT_TRUE(cursor.is_end()); - ASSERT_EQ(INTR(tree.height, t).unsafe_get0(), 1); + ASSERT_EQ(INTR(tree->height, *ref_t).unsafe_get0(), 1); }); } diff --git a/src/test/crimson/seastore/test_btree_lba_manager.cc b/src/test/crimson/seastore/test_btree_lba_manager.cc index 76cba8a752576..4c4bec1478059 100644 --- a/src/test/crimson/seastore/test_btree_lba_manager.cc +++ b/src/test/crimson/seastore/test_btree_lba_manager.cc @@ -29,27 +29,16 @@ struct btree_test_base : segment_manager::EphemeralSegmentManagerRef segment_manager; ExtentReaderRef scanner; - Journal journal; - Cache cache; + JournalRef journal; + CacheRef cache; - const size_t block_size; + size_t block_size; WritePipeline pipeline; segment_id_t next; - btree_test_base() - : segment_manager(segment_manager::create_test_ephemeral()), - scanner(new ExtentReader()), - journal(*segment_manager, *scanner), - cache(*scanner), - block_size(segment_manager->get_block_size()), - next(segment_manager->get_device_id(), 0) - { - scanner->add_segment_manager(segment_manager.get()); - journal.set_segment_provider(this); - journal.set_write_pipeline(&pipeline); - } + btree_test_base() = default; get_segment_ret get_segment(device_id_t id) final { auto ret = next; @@ -67,27 +56,38 @@ struct btree_test_base : virtual void complete_commit(Transaction &t) {} seastar::future<> submit_transaction(TransactionRef t) { - auto record = cache.prepare_record(*t); - return journal.submit_record(std::move(record), t->get_handle()).safe_then( + auto record = cache->prepare_record(*t); + return journal->submit_record(std::move(record), t->get_handle()).safe_then( [this, t=std::move(t)](auto p) mutable { auto [addr, seq] = p; - cache.complete_commit(*t, addr, seq); + cache->complete_commit(*t, addr, seq); complete_commit(*t); }).handle_error(crimson::ct_error::assert_all{}); } virtual LBAManager::mkfs_ret test_structure_setup(Transaction &t) = 0; seastar::future<> set_up_fut() final { + segment_manager = segment_manager::create_test_ephemeral(); + scanner.reset(new ExtentReader()); + journal.reset(new Journal(*segment_manager, *scanner)); + cache.reset(new Cache(*scanner)); + + block_size = segment_manager->get_block_size(); + next = segment_id_t{segment_manager->get_device_id(), 0}; + scanner->add_segment_manager(segment_manager.get()); + journal->set_segment_provider(this); + journal->set_write_pipeline(&pipeline); + return segment_manager->init( ).safe_then([this] { - return journal.open_for_write(); + return journal->open_for_write(); }).safe_then([this](auto addr) { return seastar::do_with( - cache.create_transaction(Transaction::src_t::MUTATE), + cache->create_transaction(Transaction::src_t::MUTATE), [this](auto &ref_t) { return with_trans_intr(*ref_t, [&](auto &t) { - cache.init(); - return cache.mkfs(t + cache->init(); + return cache->mkfs(t ).si_then([this, &t] { return test_structure_setup(t); }); @@ -102,10 +102,17 @@ struct btree_test_base : ); } + virtual void test_structure_reset() {} seastar::future<> tear_down_fut() final { - return cache.close( + return cache->close( ).safe_then([this] { - return journal.close(); + return journal->close(); + }).safe_then([this] { + test_structure_reset(); + segment_manager.reset(); + scanner.reset(); + journal.reset(); + cache.reset(); }).handle_error( crimson::ct_error::all_same_way([] { ASSERT_FALSE("Unable to close"); @@ -118,14 +125,14 @@ struct lba_btree_test : btree_test_base { std::map check; auto get_op_context(Transaction &t) { - return op_context_t{cache, t}; + return op_context_t{*cache, t}; } LBAManager::mkfs_ret test_structure_setup(Transaction &t) final { - return cache.get_root( + return cache->get_root( t ).si_then([this, &t](RootBlockRef croot) { - auto mut_croot = cache.duplicate_for_write( + auto mut_croot = cache->duplicate_for_write( t, croot )->cast(); mut_croot->root.lba_root = LBABtree::mkfs(get_op_context(t)); @@ -134,7 +141,7 @@ struct lba_btree_test : btree_test_base { void update_if_dirty(Transaction &t, LBABtree &btree, RootBlockRef croot) { if (btree.is_root_dirty()) { - auto mut_croot = cache.duplicate_for_write( + auto mut_croot = cache->duplicate_for_write( t, croot )->cast(); mut_croot->root.lba_root = btree.get_root_undirty(); @@ -143,12 +150,12 @@ struct lba_btree_test : btree_test_base { template auto lba_btree_update(F &&f) { - auto tref = cache.create_transaction(Transaction::src_t::MUTATE); + auto tref = cache->create_transaction(Transaction::src_t::MUTATE); auto &t = *tref; with_trans_intr( t, [this, tref=std::move(tref), f=std::forward(f)](auto &t) mutable { - return cache.get_root( + return cache->get_root( t ).si_then([this, f=std::move(f), &t](RootBlockRef croot) { return seastar::do_with( @@ -169,11 +176,11 @@ struct lba_btree_test : btree_test_base { template auto lba_btree_read(F &&f) { - auto t = cache.create_transaction(Transaction::src_t::READ); + auto t = cache->create_transaction(Transaction::src_t::READ); return with_trans_intr( *t, [this, f=std::forward(f)](auto &t) mutable { - return cache.get_root( + return cache->get_root( t ).si_then([f=std::move(f), &t](RootBlockRef croot) mutable { return seastar::do_with( @@ -265,17 +272,21 @@ TEST_F(lba_btree_test, basic) struct btree_lba_manager_test : btree_test_base { BtreeLBAManagerRef lba_manager; - btree_lba_manager_test() - : lba_manager(new BtreeLBAManager(*segment_manager, cache)) {} + btree_lba_manager_test() = default; void complete_commit(Transaction &t) final { lba_manager->complete_transaction(t); } LBAManager::mkfs_ret test_structure_setup(Transaction &t) final { + lba_manager.reset(new BtreeLBAManager(*segment_manager, *cache)); return lba_manager->mkfs(t); } + void test_structure_reset() final { + lba_manager.reset(); + } + struct test_extent_t { paddr_t addr; size_t len = 0; @@ -290,18 +301,18 @@ struct btree_lba_manager_test : btree_test_base { auto create_transaction(bool create_fake_extent=true) { auto t = test_transaction_t{ - cache.create_transaction(Transaction::src_t::MUTATE), + cache->create_transaction(Transaction::src_t::MUTATE), test_lba_mappings }; if (create_fake_extent) { - cache.alloc_new_extent(*t.t, TestBlockPhysical::SIZE); + cache->alloc_new_extent(*t.t, TestBlockPhysical::SIZE); }; return t; } auto create_weak_transaction() { auto t = test_transaction_t{ - cache.create_weak_transaction(Transaction::src_t::READ), + cache->create_weak_transaction(Transaction::src_t::READ), test_lba_mappings }; return t; diff --git a/src/test/crimson/seastore/test_randomblock_manager.cc b/src/test/crimson/seastore/test_randomblock_manager.cc index e98e9b9289b57..1277c245ed954 100644 --- a/src/test/crimson/seastore/test_randomblock_manager.cc +++ b/src/test/crimson/seastore/test_randomblock_manager.cc @@ -50,20 +50,20 @@ struct rbm_test_t : RandomBlockManager::mkfs_config_t config; paddr_t current; - rbm_test_t() { + rbm_test_t() = default; + + seastar::future<> set_up_fut() final { device.reset(new nvme_device::TestMemory(DEFAULT_TEST_SIZE)); rbm_manager.reset(new NVMeManager(device.get(), std::string())); config.start = paddr_t {0, 0, 0}; config.end = paddr_t {0, 0, DEFAULT_TEST_SIZE}; config.block_size = DEFAULT_BLOCK_SIZE; config.total_size = DEFAULT_TEST_SIZE; - } - - seastar::future<> set_up_fut() final { return tm_setup(); } seastar::future<> tear_down_fut() final { + rbm_manager.reset(); device.reset(); return tm_teardown(); } diff --git a/src/test/crimson/seastore/test_seastore.cc b/src/test/crimson/seastore/test_seastore.cc index 51995e448c95c..28f359ae4a3c5 100644 --- a/src/test/crimson/seastore/test_seastore.cc +++ b/src/test/crimson/seastore/test_seastore.cc @@ -49,6 +49,7 @@ struct seastore_test_t : } seastar::future<> tear_down_fut() final { + coll.reset(); return tm_teardown(); } diff --git a/src/test/crimson/seastore/test_seastore_cache.cc b/src/test/crimson/seastore/test_seastore_cache.cc index 355b0a4682a72..29336e6ca0df1 100644 --- a/src/test/crimson/seastore/test_seastore_cache.cc +++ b/src/test/crimson/seastore/test_seastore_cache.cc @@ -22,21 +22,15 @@ namespace { struct cache_test_t : public seastar_test_suite_t { segment_manager::EphemeralSegmentManagerRef segment_manager; ExtentReaderRef reader; - Cache cache; + CacheRef cache; paddr_t current; journal_seq_t seq; - cache_test_t() - : segment_manager(segment_manager::create_test_ephemeral()), - reader(new ExtentReader()), - cache(*reader), - current(segment_id_t(segment_manager->get_device_id(), 0), 0) { - reader->add_segment_manager(segment_manager.get()); - } + cache_test_t() = default; seastar::future submit_transaction( TransactionRef t) { - auto record = cache.prepare_record(*t); + auto record = cache->prepare_record(*t); bufferlist bl; for (auto &&block : record.extents) { @@ -61,7 +55,7 @@ struct cache_test_t : public seastar_test_suite_t { true ).safe_then( [this, prev, t=std::move(t)]() mutable { - cache.complete_commit(*t, prev, seq /* TODO */); + cache->complete_commit(*t, prev, seq /* TODO */); return prev; }, crimson::ct_error::all_same_way([](auto e) { @@ -71,7 +65,7 @@ struct cache_test_t : public seastar_test_suite_t { } auto get_transaction() { - return cache.create_transaction(Transaction::src_t::MUTATE); + return cache->create_transaction(Transaction::src_t::MUTATE); } template @@ -79,20 +73,25 @@ struct cache_test_t : public seastar_test_suite_t { return with_trans_intr( t, [this](auto &&... args) { - return cache.get_extent(args...); + return cache->get_extent(args...); }, std::forward(args)...); } seastar::future<> set_up_fut() final { + segment_manager = segment_manager::create_test_ephemeral(); + reader.reset(new ExtentReader()); + cache.reset(new Cache(*reader)); + current = paddr_t(segment_id_t(segment_manager->get_device_id(), 0), 0); + reader->add_segment_manager(segment_manager.get()); return segment_manager->init( ).safe_then([this] { return seastar::do_with( get_transaction(), [this](auto &ref_t) { - cache.init(); + cache->init(); return with_trans_intr(*ref_t, [&](auto &t) { - return cache.mkfs(t); + return cache->mkfs(t); }).safe_then([this, &ref_t] { return submit_transaction(std::move(ref_t) ).then([](auto p) {}); @@ -106,8 +105,14 @@ struct cache_test_t : public seastar_test_suite_t { } seastar::future<> tear_down_fut() final { - return cache.close().handle_error( - Cache::close_ertr::assert_all{}); + return cache->close( + ).safe_then([this] { + segment_manager.reset(); + reader.reset(); + cache.reset(); + }).handle_error( + Cache::close_ertr::assert_all{} + ); } }; @@ -118,7 +123,7 @@ TEST_F(cache_test_t, test_addr_fixup) int csum = 0; { auto t = get_transaction(); - auto extent = cache.alloc_new_extent( + auto extent = cache->alloc_new_extent( *t, TestBlockPhysical::SIZE); extent->set_contents('c'); @@ -147,7 +152,7 @@ TEST_F(cache_test_t, test_dirty_extent) { // write out initial test block auto t = get_transaction(); - auto extent = cache.alloc_new_extent( + auto extent = cache->alloc_new_extent( *t, TestBlockPhysical::SIZE); extent->set_contents('c'); @@ -192,7 +197,7 @@ TEST_F(cache_test_t, test_dirty_extent) addr, TestBlockPhysical::SIZE).unsafe_get0(); // duplicate and reset contents - extent = cache.duplicate_for_write(*t, extent)->cast(); + extent = cache->duplicate_for_write(*t, extent)->cast(); extent->set_contents('c'); csum2 = extent->get_crc32c(); ASSERT_EQ(extent->get_paddr(), addr); diff --git a/src/test/crimson/seastore/test_seastore_journal.cc b/src/test/crimson/seastore/test_seastore_journal.cc index f264bf429c0a8..bb69c81750259 100644 --- a/src/test/crimson/seastore/test_seastore_journal.cc +++ b/src/test/crimson/seastore/test_seastore_journal.cc @@ -72,20 +72,13 @@ struct journal_test_t : seastar_test_suite_t, SegmentProvider { std::default_random_engine generator; - const segment_off_t block_size; + segment_off_t block_size; ExtentReaderRef scanner; segment_id_t next; - journal_test_t() - : segment_manager(segment_manager::create_test_ephemeral()), - block_size(segment_manager->get_block_size()), - scanner(new ExtentReader()), - next(segment_manager->get_device_id(), 0) - { - scanner->add_segment_manager(segment_manager.get()); - } + journal_test_t() = default; get_segment_ret get_segment(device_id_t id) final { auto ret = next; @@ -101,9 +94,15 @@ struct journal_test_t : seastar_test_suite_t, SegmentProvider { void update_journal_tail_committed(journal_seq_t paddr) final {} seastar::future<> set_up_fut() final { + segment_manager = segment_manager::create_test_ephemeral(); + block_size = segment_manager->get_block_size(); + scanner.reset(new ExtentReader()); + next = segment_id_t(segment_manager->get_device_id(), 0); journal.reset(new Journal(*segment_manager, *scanner)); + journal->set_segment_provider(this); journal->set_write_pipeline(&pipeline); + scanner->add_segment_manager(segment_manager.get()); return segment_manager->init( ).safe_then([this] { return journal->open_for_write(); @@ -116,7 +115,11 @@ struct journal_test_t : seastar_test_suite_t, SegmentProvider { seastar::future<> tear_down_fut() final { return journal->close( - ).handle_error( + ).safe_then([this] { + segment_manager.reset(); + scanner.reset(); + journal.reset(); + }).handle_error( crimson::ct_error::all_same_way([](auto e) { ASSERT_FALSE("Unable to close"); }) diff --git a/src/test/crimson/seastore/transaction_manager_test_state.h b/src/test/crimson/seastore/transaction_manager_test_state.h index 10ce0a0037964..8beb6b7626d8a 100644 --- a/src/test/crimson/seastore/transaction_manager_test_state.h +++ b/src/test/crimson/seastore/transaction_manager_test_state.h @@ -22,8 +22,7 @@ class EphemeralTestState { protected: segment_manager::EphemeralSegmentManagerRef segment_manager; - EphemeralTestState() - : segment_manager(segment_manager::create_test_ephemeral()) {} + EphemeralTestState() = default; virtual void _init() = 0; void init() { @@ -48,6 +47,7 @@ class EphemeralTestState { } seastar::future<> tm_setup() { + segment_manager = segment_manager::create_test_ephemeral(); init(); return segment_manager->init( ).safe_then([this] { @@ -64,7 +64,9 @@ class EphemeralTestState { } seastar::future<> tm_teardown() { - return _teardown(); + return _teardown().then([this] { + segment_manager.reset(); + }); } }; From 8af00e25aa4ab60d0309e31f6c20edd6cd5be1ee Mon Sep 17 00:00:00 2001 From: Guillaume Abrioux Date: Wed, 27 Oct 2021 08:37:07 +0200 Subject: [PATCH 0350/2866] ceph-volume/tests: update setup_mixed_type playbook we need to create a file with a larger size. see https://github.com/ceph/ceph/pull/43300#issuecomment-951961243 Signed-off-by: Guillaume Abrioux --- src/ceph-volume/ceph_volume/tests/functional/Vagrantfile | 2 +- .../tests/functional/batch/playbooks/setup_mixed_type.yml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/ceph-volume/ceph_volume/tests/functional/Vagrantfile b/src/ceph-volume/ceph_volume/tests/functional/Vagrantfile index 8f8b31b7ecfcc..f5425165a8d66 100644 --- a/src/ceph-volume/ceph_volume/tests/functional/Vagrantfile +++ b/src/ceph-volume/ceph_volume/tests/functional/Vagrantfile @@ -390,7 +390,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| # always make /dev/sd{a/b/c/d} so that CI can ensure that # virtualbox and libvirt will have the same devices to use for OSDs (0..3).each do |d| - lv.storage :file, :device => "sd#{driverletters[d]}", :size => '12G' + lv.storage :file, :device => "sd#{driverletters[d]}", :size => '100G' end lv.memory = MEMORY lv.random_hostname = true diff --git a/src/ceph-volume/ceph_volume/tests/functional/batch/playbooks/setup_mixed_type.yml b/src/ceph-volume/ceph_volume/tests/functional/batch/playbooks/setup_mixed_type.yml index 8b155d97d8c3f..5460fdd0db98c 100644 --- a/src/ceph-volume/ceph_volume/tests/functional/batch/playbooks/setup_mixed_type.yml +++ b/src/ceph-volume/ceph_volume/tests/functional/batch/playbooks/setup_mixed_type.yml @@ -1,4 +1,4 @@ - +--- - hosts: osds become: yes tasks: @@ -119,8 +119,8 @@ - /opt/vdd/loop0_nvme0 - /opt/vde/loop1_nvme1 - - name: create 11GB sparse files for NVMe - command: "fallocate -l 11G {{ item }}" + - name: create 20GB sparse files for NVMe + command: "fallocate -l 20G {{ item }}" loop: - /opt/vdd/loop0_nvme0 - /opt/vde/loop1_nvme1 From 73da90ce278e3bf31265aaa091247efcd69f5c10 Mon Sep 17 00:00:00 2001 From: Zac Dover Date: Wed, 27 Oct 2021 04:14:53 +1000 Subject: [PATCH 0351/2866] doc/rados: storage-devices.rst rewrite This commit is a minor improvement to the English syntax of storage-devices.rst. Signed-off-by: Zac Dover --- doc/rados/configuration/storage-devices.rst | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/doc/rados/configuration/storage-devices.rst b/doc/rados/configuration/storage-devices.rst index 559d470cb415d..2663b09a2d4eb 100644 --- a/doc/rados/configuration/storage-devices.rst +++ b/doc/rados/configuration/storage-devices.rst @@ -27,24 +27,23 @@ There are several Ceph daemons in a storage cluster: OSD Backends ============ -There are two ways that OSDs can manage the data they store. Starting -with the Luminous 12.2.z release, the new default (and recommended) backend is -*BlueStore*. Prior to Luminous, the default (and only option) was +There are two ways that OSDs manage the data they store. +As of the Luminous 12.2.z release, the default (and recommended) backend is +*BlueStore*. Prior to the Luminous release, the default (and only option) was *Filestore*. BlueStore --------- BlueStore is a special-purpose storage backend designed specifically for -managing data on disk for Ceph OSD workloads. BlueStore is based on over a -decade of experience supporting and managing OSDs using FileStore. +managing data on disk for Ceph OSD workloads. BlueStore's design is based on +a decade of experience of supporting and managing Filestore OSDs. Key BlueStore features include: -* Direct management of storage devices. BlueStore consumes raw block - devices or partitions. This avoids intervening layers of - abstraction (such as local file systems like XFS) that can limit - performance or add complexity. +* Direct management of storage devices. BlueStore consumes raw block devices or + partitions. This avoids intervening layers of abstraction (such as local file + systems like XFS) that can limit performance or add complexity. * Metadata management with RocksDB. RocksDB's key/value database is embedded in order to manage internal metadata, including the mapping of object names to block locations on disk. @@ -77,7 +76,7 @@ metadata. FileStore is well-tested and widely used in production. However, it suffers from many performance deficiencies due to its overall design -and its reliance on a traditional file system for storing object data. +and its reliance on a traditional file system for object data storage. Although FileStore is capable of functioning on most POSIX-compatible file systems (including btrfs and ext4), we recommend that only the From 708c2fcbb867443c78f9c04a50d1d3f1a1c2ec82 Mon Sep 17 00:00:00 2001 From: Adam King Date: Wed, 27 Oct 2021 10:56:23 -0400 Subject: [PATCH 0352/2866] qa/suites/orch/cephadm: test with agent on/off Signed-off-by: Adam King --- qa/suites/orch/cephadm/orchestrator_cli/agent | 1 + qa/suites/orch/cephadm/smoke/agent/off.yaml | 5 +++++ qa/suites/orch/cephadm/smoke/agent/on.yaml | 5 +++++ qa/suites/orch/cephadm/upgrade/agent | 1 + qa/suites/orch/cephadm/workunits/agent | 1 + 5 files changed, 13 insertions(+) create mode 120000 qa/suites/orch/cephadm/orchestrator_cli/agent create mode 100644 qa/suites/orch/cephadm/smoke/agent/off.yaml create mode 100644 qa/suites/orch/cephadm/smoke/agent/on.yaml create mode 120000 qa/suites/orch/cephadm/upgrade/agent create mode 120000 qa/suites/orch/cephadm/workunits/agent diff --git a/qa/suites/orch/cephadm/orchestrator_cli/agent b/qa/suites/orch/cephadm/orchestrator_cli/agent new file mode 120000 index 0000000000000..1549242094390 --- /dev/null +++ b/qa/suites/orch/cephadm/orchestrator_cli/agent @@ -0,0 +1 @@ +../smoke/agent \ No newline at end of file diff --git a/qa/suites/orch/cephadm/smoke/agent/off.yaml b/qa/suites/orch/cephadm/smoke/agent/off.yaml new file mode 100644 index 0000000000000..f37c651bc954b --- /dev/null +++ b/qa/suites/orch/cephadm/smoke/agent/off.yaml @@ -0,0 +1,5 @@ +overrides: + ceph: + conf: + mgr: + mgr/cephadm/use_agent: false diff --git a/qa/suites/orch/cephadm/smoke/agent/on.yaml b/qa/suites/orch/cephadm/smoke/agent/on.yaml new file mode 100644 index 0000000000000..90ac298b68e4f --- /dev/null +++ b/qa/suites/orch/cephadm/smoke/agent/on.yaml @@ -0,0 +1,5 @@ +overrides: + ceph: + conf: + mgr: + mgr/cephadm/use_agent: true diff --git a/qa/suites/orch/cephadm/upgrade/agent b/qa/suites/orch/cephadm/upgrade/agent new file mode 120000 index 0000000000000..1549242094390 --- /dev/null +++ b/qa/suites/orch/cephadm/upgrade/agent @@ -0,0 +1 @@ +../smoke/agent \ No newline at end of file diff --git a/qa/suites/orch/cephadm/workunits/agent b/qa/suites/orch/cephadm/workunits/agent new file mode 120000 index 0000000000000..1549242094390 --- /dev/null +++ b/qa/suites/orch/cephadm/workunits/agent @@ -0,0 +1 @@ +../smoke/agent \ No newline at end of file From a578cedea38e14add20561af81607f6d63dd9c08 Mon Sep 17 00:00:00 2001 From: Radoslaw Zarzynski Date: Wed, 27 Oct 2021 16:04:11 +0000 Subject: [PATCH 0353/2866] auth: drop unused get_supported_con_modes() from AuthServer. In 2d530938753313a776258ff9fa8208db637c9d96 its last user has been removed. Signed-off-by: Radoslaw Zarzynski --- src/auth/AuthServer.h | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/auth/AuthServer.h b/src/auth/AuthServer.h index d7ca304a6f1a9..a71f6a70a6ccc 100644 --- a/src/auth/AuthServer.h +++ b/src/auth/AuthServer.h @@ -25,14 +25,6 @@ class AuthServer { auth_registry.get_supported_methods(peer_type, methods, modes); } - /// Get support connection modes for the given peer type and auth method - virtual void get_supported_con_modes( - int peer_type, - uint32_t auth_method, - std::vector *modes) { - auth_registry.get_supported_modes(peer_type, auth_method, modes); - } - /// Get support connection modes for the given peer type and auth method virtual uint32_t pick_con_mode( int peer_type, From 9502953d4dfcd794fcc9400bb52ebf229abb5517 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Fri, 15 Oct 2021 10:44:59 -0500 Subject: [PATCH 0354/2866] mgr/orchestrator: log exception from wrapper This lets us see a proper traceback when we are calling across modules. Signed-off-by: Sage Weil --- src/pybind/mgr/orchestrator/_interface.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/pybind/mgr/orchestrator/_interface.py b/src/pybind/mgr/orchestrator/_interface.py index 1497f80aaa67e..c749921083d77 100644 --- a/src/pybind/mgr/orchestrator/_interface.py +++ b/src/pybind/mgr/orchestrator/_interface.py @@ -124,6 +124,7 @@ def wrapper(*args: Any, **kwargs: Any) -> OrchResult[T]: try: return OrchResult(f(*args, **kwargs)) except Exception as e: + logger.exception(e) return OrchResult(None, exception=e) return cast(Callable[..., OrchResult[T]], wrapper) From d7717256644e45ba670165c6af0941461fb884f4 Mon Sep 17 00:00:00 2001 From: Patrick Donnelly Date: Wed, 27 Oct 2021 20:26:55 -0400 Subject: [PATCH 0355/2866] pybind/mgr/cephadm: upgrade MDS if no MDS is "up" The upgrade process can get stuck if an MDS crashes. This should be rare when straddling v16.2.5 where the compatset of the file system inherits the FSMap "default". The MDS from pre-v16.2.5 do not yet share a compatset with the mons so the mons will do no promotions, causing upgrade task to get stuck. Fixes: https://tracker.ceph.com/issues/53074 Signed-off-by: Patrick Donnelly --- src/pybind/mgr/cephadm/upgrade.py | 32 +++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/src/pybind/mgr/cephadm/upgrade.py b/src/pybind/mgr/cephadm/upgrade.py index 0984a5f32c576..1cebe10651515 100644 --- a/src/pybind/mgr/cephadm/upgrade.py +++ b/src/pybind/mgr/cephadm/upgrade.py @@ -441,23 +441,31 @@ def _prepare_for_mds_upgrade( continue_upgrade = False continue - if not (mdsmap['in'] == [0] and len(mdsmap['up']) == 1): + if not (mdsmap['in'] == [0] and len(mdsmap['up']) <= 1): self.mgr.log.info('Upgrade: Waiting for fs %s to scale down to reach 1 MDS' % (fs_name)) time.sleep(10) continue_upgrade = False continue - mdss = list(mdsmap['info'].values()) - assert len(mdss) == 1 - lone_mds = mdss[0] - if lone_mds['state'] != 'up:active': - self.mgr.log.info('Upgrade: Waiting for mds.%s to be up:active (currently %s)' % ( - lone_mds['name'], - lone_mds['state'], - )) - time.sleep(10) - continue_upgrade = False - continue + if len(mdsmap['up']) == 0: + self.mgr.log.warning("Upgrade: No mds is up; continuing upgrade procedure to poke things in the right direction") + # This can happen because the current version MDS have + # incompatible compatsets; the mons will not do any promotions. + # We must upgrade to continue. + elif len(mdsmap['up']) > 0: + mdss = list(mdsmap['info'].values()) + assert len(mdss) == 1 + lone_mds = mdss[0] + if lone_mds['state'] != 'up:active': + self.mgr.log.info('Upgrade: Waiting for mds.%s to be up:active (currently %s)' % ( + lone_mds['name'], + lone_mds['state'], + )) + time.sleep(10) + continue_upgrade = False + continue + else: + assert False return continue_upgrade From 9951868fc33f0469cd4135fadebb5a8d2b4b55dd Mon Sep 17 00:00:00 2001 From: Yin Congmin Date: Wed, 29 Sep 2021 13:47:17 +0800 Subject: [PATCH 0356/2866] librbd/cache/pwl: cancel advance dispatch of external flush request For external flush request, it new syncpoint after passing guardedrequest and before dispatch. Then dispatch bypass deferred queue But the last write request may still in the deferred queue. It don't dispatch and not associated with any syncpoint. The external flush request will bypass the previous write request in deferred queue now. This does not conform to the semantics of external flush requests. External flush request should strictly follow the order of dispath. But for internal flush request, it will be dispatched after all write request which associated with previous syncpoint, persisted in cache. C_gather guarantee it. It is necessary to distinguish between external and internal flush requests. Internal flush can and should be dispatched in advance bypass deferred queue. At the same time, the order of external requests needs to be kept unchanged. So cancel advance dispatch of external flush request. Fixes: https://tracker.ceph.com/issues/52599 Signed-off-by: Yin Congmin --- src/librbd/cache/pwl/AbstractWriteLog.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/librbd/cache/pwl/AbstractWriteLog.cc b/src/librbd/cache/pwl/AbstractWriteLog.cc index c01d2d411f9da..bfc605285f2ba 100644 --- a/src/librbd/cache/pwl/AbstractWriteLog.cc +++ b/src/librbd/cache/pwl/AbstractWriteLog.cc @@ -1440,8 +1440,9 @@ void AbstractWriteLog::alloc_and_dispatch_io_req(C_BlockIORequestT *req) std::lock_guard locker(m_lock); dispatch_here = m_deferred_ios.empty(); // Only flush req's total_bytes is the max uint64 - if ((req->image_extents_summary.total_bytes == - std::numeric_limits::max())) { + if (req->image_extents_summary.total_bytes == + std::numeric_limits::max() && + static_cast(req)->internal == true) { dispatch_here = true; } } From 59dd41027de391caf2d7f1b0ee73a87092b620c2 Mon Sep 17 00:00:00 2001 From: Pere Diaz Bou Date: Wed, 27 Oct 2021 13:20:04 +0200 Subject: [PATCH 0357/2866] cephadm/box: add no-tmpfs flag prepare osd bluestore Signed-off-by: Pere Diaz Bou --- src/ceph-volume/ceph_volume/devices/lvm/common.py | 6 ++++++ src/ceph-volume/ceph_volume/devices/lvm/prepare.py | 7 +++++-- src/cephadm/box/start | 2 +- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/src/ceph-volume/ceph_volume/devices/lvm/common.py b/src/ceph-volume/ceph_volume/devices/lvm/common.py index 752f354f35aac..b7ce6473dd7a8 100644 --- a/src/ceph-volume/ceph_volume/devices/lvm/common.py +++ b/src/ceph-volume/ceph_volume/devices/lvm/common.py @@ -123,6 +123,12 @@ def rollback_osd(args, osd_id=None): 'type': int, 'default': 1, }, + '--no-tmpfs': { + 'action': 'store_true', + 'dest': 'no_tmpfs', + 'help': ('Disable tmpfs osd data directory with bluestore.' + 'Useful if you want to run lvm preprare from cephadm'), + }, } filestore_args = { diff --git a/src/ceph-volume/ceph_volume/devices/lvm/prepare.py b/src/ceph-volume/ceph_volume/devices/lvm/prepare.py index 3121ede3bf51f..6567c78d9f5ec 100644 --- a/src/ceph-volume/ceph_volume/devices/lvm/prepare.py +++ b/src/ceph-volume/ceph_volume/devices/lvm/prepare.py @@ -81,7 +81,7 @@ def prepare_filestore(device, journal, secrets, tags, osd_id, fsid): ) -def prepare_bluestore(block, wal, db, secrets, tags, osd_id, fsid): +def prepare_bluestore(block, wal, db, secrets, tags, osd_id, fsid, tmpfs=True): """ :param block: The name of the logical volume for the bluestore data :param wal: a regular/plain disk or logical volume, to be used for block.wal @@ -104,7 +104,7 @@ def prepare_bluestore(block, wal, db, secrets, tags, osd_id, fsid): db = prepare_dmcrypt(key, db, 'db', tags) # create the directory - prepare_utils.create_osd_path(osd_id, tmpfs=False) + prepare_utils.create_osd_path(osd_id, tmpfs=tmpfs) # symlink the block prepare_utils.link_block(block, osd_id) # get the latest monmap @@ -384,6 +384,8 @@ def prepare(self): tags['ceph.type'] = 'block' block_lv.set_tags(tags) + tmpfs = not self.args.no_tmpfs + prepare_bluestore( block_lv.lv_path, wal_device, @@ -392,6 +394,7 @@ def prepare(self): tags, self.osd_id, osd_fsid, + tmpfs=tmpfs ) def main(self): diff --git a/src/cephadm/box/start b/src/cephadm/box/start index 7d12e41db41ec..b5883eeb932cc 100755 --- a/src/cephadm/box/start +++ b/src/cephadm/box/start @@ -37,7 +37,7 @@ vgchange --refresh for((i=0;i<$NUM_OSDS;i++)); do echo "Creating osd.${i}" # create osd folder - $CEPHADM_PATH ceph-volume --shared_ceph_folder /ceph lvm create --bluestore --data "/dev/vg1/lv${i}" --no-systemd + $CEPHADM_PATH ceph-volume --shared_ceph_folder /ceph lvm create --bluestore --no-tmpfs --data "/dev/vg1/lv${i}" --no-systemd echo "Deploying osd.${i}..." # deploy osd with osd data folder $CEPHADM_PATH deploy --name "osd.${i}" From 99e40a1e87c44705ffdb557d57bd54eaff31cfe3 Mon Sep 17 00:00:00 2001 From: Igor Fedotov Date: Tue, 9 Feb 2021 18:37:41 +0300 Subject: [PATCH 0358/2866] os/bluestore: permit legacy omap naming scheme in mkfs. Primarily for debug purposes... Signed-off-by: Igor Fedotov --- src/common/options/global.yaml.in | 9 +++++++++ src/os/bluestore/BlueStore.cc | 11 ++++++++--- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/src/common/options/global.yaml.in b/src/common/options/global.yaml.in index 38b95313bc96d..ed529f18fe0b2 100644 --- a/src/common/options/global.yaml.in +++ b/src/common/options/global.yaml.in @@ -5020,6 +5020,15 @@ options: desc: inject crc verification errors into bluestore device reads default: 0 with_legacy: true +- name: bluestore_debug_legacy_omap + type: bool + level: dev + desc: Allows mkfs to create OSD in legacy OMAP naming mode (neither per-pool nor per-pg). + This is intended primarily for developers' purposes. The resulting OSD might/would + be transformed to the currrently default 'per-pg' format when BlueStore's quick-fix or + repair are applied. + default: false + with_legacy: true - name: bluestore_fsck_error_on_no_per_pool_stats type: bool level: advanced diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc index 4f59ebde508a5..e5d291b638034 100644 --- a/src/os/bluestore/BlueStore.cc +++ b/src/os/bluestore/BlueStore.cc @@ -6410,9 +6410,10 @@ void BlueStore::_set_per_pool_omap() auto s = bl.to_str(); if (s == stringify(OMAP_PER_POOL)) { per_pool_omap = OMAP_PER_POOL; - } else { - ceph_assert(s == stringify(OMAP_PER_PG)); + } else if (s == stringify(OMAP_PER_PG)) { per_pool_omap = OMAP_PER_PG; + } else { + ceph_assert(s == stringify(OMAP_BULK)); } dout(10) << __func__ << " per_pool_omap = " << per_pool_omap << dendl; } else { @@ -6715,7 +6716,11 @@ int BlueStore::mkfs() } { bufferlist bl; - bl.append(stringify(OMAP_PER_PG)); + if (cct->_conf.get_val("bluestore_debug_legacy_omap")) { + bl.append(stringify(OMAP_BULK)); + } else { + bl.append(stringify(OMAP_PER_PG)); + } t->set(PREFIX_SUPER, "per_pool_omap", bl); } ondisk_format = latest_ondisk_format; From ccb6fdf3b6ba700da1efea597deabe79329a4504 Mon Sep 17 00:00:00 2001 From: Igor Fedotov Date: Wed, 27 Oct 2021 18:58:52 +0300 Subject: [PATCH 0359/2866] test/store_test: add a UT for omap format upgrade. Signed-off-by: Igor Fedotov --- src/test/objectstore/store_test.cc | 72 ++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/src/test/objectstore/store_test.cc b/src/test/objectstore/store_test.cc index d942eebd784d5..6255de952f7d0 100644 --- a/src/test/objectstore/store_test.cc +++ b/src/test/objectstore/store_test.cc @@ -37,6 +37,7 @@ #include "common/ceph_mutex.h" #include "common/Cond.h" #include "common/errno.h" +#include "common/pretty_binary.h" #include "include/stringify.h" #include "include/coredumpctl.h" @@ -9510,6 +9511,77 @@ TEST_P(StoreTestSpecificAUSize, Ticket45195Repro) { ASSERT_EQ(r, 0x10000); } +TEST_P(StoreTestSpecificAUSize, OmapUpgradeTest) { + if (string(GetParam()) != "bluestore") + return; + + SetVal(g_conf(), "bluestore_debug_legacy_omap", "true"); + g_conf().apply_changes(nullptr); + + StartDeferred(0x1000); + int64_t poolid = 11; + coll_t cid(spg_t(pg_t(1, poolid), shard_id_t::NO_SHARD)); + ghobject_t hoid(hobject_t("tesomap", "", CEPH_NOSNAP, 0, poolid, "")); + auto ch = store->create_new_collection(cid); + int r; + { + ObjectStore::Transaction t; + t.create_collection(cid, 0); + r = queue_transaction(store, ch, std::move(t)); + ASSERT_EQ(r, 0); + } + + map attrs; + bufferlist expected_header; + expected_header.append("this is a header"); + { + ObjectStore::Transaction t; + t.touch(cid, hoid); + bufferlist header; + header.append(expected_header); + t.omap_setheader(cid, hoid, header); + map start_set; + bufferlist bl; + bl.append(string("value")); + start_set.emplace(string("key1"), bl); + t.omap_setkeys(cid, hoid, start_set); + r = queue_transaction(store, ch, std::move(t)); + } + { + map res; + bufferlist h; + r = store->omap_get(ch, hoid, &h, &res); + ASSERT_EQ(r, 0); + ASSERT_TRUE(bl_eq(h, expected_header)); + ASSERT_EQ(res.size(), 1); + ASSERT_EQ(res.begin()->first, "key1"); + } + store->umount(); + ASSERT_EQ(store->fsck(false), 0); + SetVal(g_conf(), "bluestore_debug_legacy_omap", "false"); + SetVal(g_conf(), "bluestore_fsck_error_on_no_per_pool_omap", "true"); + g_conf().apply_changes(nullptr); + ASSERT_EQ(store->fsck(false), 2); + ASSERT_EQ(store->quick_fix(), 0); + store->mount(); + ch = store->open_collection(cid); + { + map res; + bufferlist h; + r = store->omap_get(ch, hoid, &h, &res); + ASSERT_EQ(r, 0); + ASSERT_EQ(res.size(), 1); + ASSERT_EQ(res.begin()->first, "key1"); + } + { + ObjectStore::Transaction t; + t.remove(cid, hoid); + t.remove_collection(cid); + r = queue_transaction(store, ch, std::move(t)); + ASSERT_EQ(r, 0); + } +} + #endif // WITH_BLUESTORE int main(int argc, char **argv) { From cbc97018d883333f81ab9a3cfa99d2f68a9874cd Mon Sep 17 00:00:00 2001 From: Igor Fedotov Date: Wed, 27 Oct 2021 13:59:34 +0300 Subject: [PATCH 0360/2866] os/bluestore: fix invalid omap name conversion when upgrading to per-pg. Fixes: https://tracker.ceph.com/issues/53062 Signed-off-by: Igor Fedotov --- src/os/bluestore/BlueStore.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc index e5d291b638034..5926297c98057 100644 --- a/src/os/bluestore/BlueStore.cc +++ b/src/os/bluestore/BlueStore.cc @@ -8072,6 +8072,7 @@ void BlueStore::_fsck_check_object_omap(FSCKDepth depth, string final_key; Onode::calc_omap_key(new_flags, o.get(), string(), &final_key); size_t base_key_len = final_key.size(); + it->next(); while (it->valid() && it->key() < tail) { string user_key; o->decode_omap_key(it->key(), &user_key); @@ -8079,7 +8080,7 @@ void BlueStore::_fsck_check_object_omap(FSCKDepth depth, << " -> " << user_key << dendl; final_key.resize(base_key_len); - final_key += it->key(); + final_key += user_key; auto v = it->value(); txn->set(new_omap_prefix, final_key, v); txn_cost += final_key.length() + v.length(); From 116a8c420886b5dd5d77d9bef1248791d74cd250 Mon Sep 17 00:00:00 2001 From: Sebastian Wagner Date: Thu, 28 Oct 2021 15:01:29 +0200 Subject: [PATCH 0361/2866] qa/suites/rados/mgr: use only one objectstore instead of all I think we have enough coverage. Always testing all objectstores is a bit excessive in my opinion Signed-off-by: Sebastian Wagner --- qa/suites/rados/mgr/{objectstore => random-objectstore$} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename qa/suites/rados/mgr/{objectstore => random-objectstore$} (100%) diff --git a/qa/suites/rados/mgr/objectstore b/qa/suites/rados/mgr/random-objectstore$ similarity index 100% rename from qa/suites/rados/mgr/objectstore rename to qa/suites/rados/mgr/random-objectstore$ From af60ccdac2baa6f7545d3db1adb60cc2109cd91d Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Thu, 28 Oct 2021 10:06:34 -0400 Subject: [PATCH 0362/2866] ceph_test_rados_api_watch_notify: extend Watch3Timeout test Make this test try harder, since osd peering may interfere with a timeout. Fixes: https://tracker.ceph.com/issues/24990 Signed-off-by: Sage Weil --- src/test/librados/watch_notify.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/librados/watch_notify.cc b/src/test/librados/watch_notify.cc index e5ea22744a043..3797e37c58300 100644 --- a/src/test/librados/watch_notify.cc +++ b/src/test/librados/watch_notify.cc @@ -493,7 +493,7 @@ TEST_F(LibRadosWatchNotify, Watch3Timeout) { rados_conf_set(cluster, "objecter_inject_no_watch_ping", "true"); // allow a long time here since an osd peering event will renew our // watch. - int left = 16 * timeout; + int left = 256 * timeout; std::cout << "waiting up to " << left << " for osd to time us out ..." << std::endl; while (notify_err == 0 && --left) { From afb7736e677daa3f65890a9dc2d5e46e9b60d71b Mon Sep 17 00:00:00 2001 From: Daniel Pivonka Date: Tue, 26 Oct 2021 15:39:17 -0400 Subject: [PATCH 0363/2866] mgr/cephadm: unit test for daemon post actions missing in cache Signed-off-by: Daniel Pivonka --- src/pybind/mgr/cephadm/tests/test_cephadm.py | 60 ++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/src/pybind/mgr/cephadm/tests/test_cephadm.py b/src/pybind/mgr/cephadm/tests/test_cephadm.py index 46615d1ad6335..3b37fb2048c30 100644 --- a/src/pybind/mgr/cephadm/tests/test_cephadm.py +++ b/src/pybind/mgr/cephadm/tests/test_cephadm.py @@ -423,6 +423,66 @@ def test_daemon_check_post(self, cephadm_module: CephadmOrchestrator): {'prefix': 'dashboard set-grafana-api-url', 'value': 'https://1.2.3.4:3000'}, None) + @mock.patch("cephadm.serve.CephadmServe._run_cephadm", _run_cephadm('{}')) + @mock.patch("cephadm.module.CephadmOrchestrator.get_mgr_ip", lambda _: '1.2.3.4') + def test_iscsi_post_actions_with_missing_daemon_in_cache(self, cephadm_module: CephadmOrchestrator): + # https://tracker.ceph.com/issues/52866 + with with_host(cephadm_module, 'test1'): + with with_host(cephadm_module, 'test2'): + with with_service(cephadm_module, IscsiServiceSpec(service_id='foobar', pool='pool', placement=PlacementSpec(host_pattern='*')), CephadmOrchestrator.apply_iscsi, 'test'): + + CephadmServe(cephadm_module)._apply_all_services() + assert len(cephadm_module.cache.get_daemons_by_type('iscsi')) == 2 + + # get a deamons from postaction list (ARRGH sets!!) + tempset = cephadm_module.requires_post_actions.copy() + tempdeamon1 = tempset.pop() + tempdeamon2 = tempset.pop() + + # make sure post actions has 2 daemons in it + assert len(cephadm_module.requires_post_actions) == 2 + + # replicate a host cache that is not in sync when check_daemons is called + tempdd1 = cephadm_module.cache.get_daemon(tempdeamon1) + tempdd2 = cephadm_module.cache.get_daemon(tempdeamon2) + host = 'test1' + if 'test1' not in tempdeamon1: + host = 'test2' + cephadm_module.cache.rm_daemon(host, tempdeamon1) + + # Make sure, _check_daemons does a redeploy due to monmap change: + cephadm_module.mock_store_set('_ceph_get', 'mon_map', { + 'modified': datetime_to_str(datetime_now()), + 'fsid': 'foobar', + }) + cephadm_module.notify('mon_map', None) + cephadm_module.mock_store_set('_ceph_get', 'mgr_map', { + 'modules': ['dashboard'] + }) + + with mock.patch("cephadm.module.IscsiService.config_dashboard") as _cfg_db: + CephadmServe(cephadm_module)._check_daemons() + _cfg_db.assert_called_once_with([tempdd2]) + + # post actions still has the other deamon in it and will run next _check_deamons + assert len(cephadm_module.requires_post_actions) == 1 + + # post actions was missed for a daemon + assert tempdeamon1 in cephadm_module.requires_post_actions + + # put the daemon back in the cache + cephadm_module.cache.add_daemon(host, tempdd1) + + _cfg_db.reset_mock() + # replicate serve loop running again + CephadmServe(cephadm_module)._check_daemons() + + # post actions should have been called again + _cfg_db.asset_called() + + # post actions is now empty + assert len(cephadm_module.requires_post_actions) == 0 + @mock.patch("cephadm.serve.CephadmServe._run_cephadm", _run_cephadm('[]')) def test_mon_add(self, cephadm_module): with with_host(cephadm_module, 'test'): From d15a3ce1801a83092eb14a628c606659d59611ec Mon Sep 17 00:00:00 2001 From: Kefu Chai Date: Thu, 28 Oct 2021 23:41:58 +0800 Subject: [PATCH 0364/2866] mgr: do not handle Python2 this change should have been a part of 48c4bc445fcacd7980e84b4f135a3115b88b05f1 Signed-off-by: Kefu Chai --- src/mgr/BaseMgrModule.cc | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/mgr/BaseMgrModule.cc b/src/mgr/BaseMgrModule.cc index 6386265fc7e91..2d851d46b2ac1 100644 --- a/src/mgr/BaseMgrModule.cc +++ b/src/mgr/BaseMgrModule.cc @@ -1263,11 +1263,7 @@ ceph_add_mds_perf_query(BaseMgrModule *self, PyObject *args) } limit->order_by = it->second; } else if (limit_param_name == NAME_LIMIT_MAX_COUNT) { -#if PY_MAJOR_VERSION <= 2 - if (!PyInt_Check(limit_param_val) && !PyLong_Check(limit_param_val)) { -#else if (!PyLong_Check(limit_param_val)) { -#endif derr << __func__ << " " << limit_param_name << " not an int" << dendl; Py_RETURN_NONE; From bdca57e08efabf48625829ceccc8c018e2f3e447 Mon Sep 17 00:00:00 2001 From: Neha Ojha Date: Thu, 28 Oct 2021 16:56:13 +0000 Subject: [PATCH 0365/2866] doc/releases/pacific.rst: add note about data corruption bug Warn users upgrading their clusters from nautilus/octopus to pacific about https://tracker.ceph.com/issues/53062 Signed-off-by: Neha Ojha --- doc/releases/pacific.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/doc/releases/pacific.rst b/doc/releases/pacific.rst index a18695bcf03fa..2502999017880 100644 --- a/doc/releases/pacific.rst +++ b/doc/releases/pacific.rst @@ -763,6 +763,13 @@ Upgrading from Octopus or Nautilus Before starting, make sure your cluster is stable and healthy (no down or recovering OSDs). (This is optional, but recommended.) + +.. note:: + WARNING: Please do not set `bluestore_fsck_quick_fix_on_mount` to true or + run `ceph-bluestore-tool` repair or quick-fix commands in Pacific versions + <= 16.2.6, because this can lead to data corruption, details in + https://tracker.ceph.com/issues/53062. + Upgrading cephadm clusters ~~~~~~~~~~~~~~~~~~~~~~~~~~ From d7acc1686019f8de349e26eae3ae3591b5894746 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Thu, 28 Oct 2021 11:23:21 -0400 Subject: [PATCH 0366/2866] qa/tasks/cephfs/test_nfs: wait for fs to come up before exporting Signed-off-by: Sage Weil --- qa/tasks/cephfs/test_nfs.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/qa/tasks/cephfs/test_nfs.py b/qa/tasks/cephfs/test_nfs.py index 0b690cd662a89..0d5099504c6b8 100644 --- a/qa/tasks/cephfs/test_nfs.py +++ b/qa/tasks/cephfs/test_nfs.py @@ -6,6 +6,7 @@ from io import BytesIO from tasks.mgr.mgr_test_case import MgrTestCase +from teuthology import contextutil from teuthology.exceptions import CommandFailedError log = logging.getLogger(__name__) @@ -153,6 +154,15 @@ def _create_export(self, export_id, create_fs=False, extra_cmd=None): ''' if create_fs: self._cmd('fs', 'volume', 'create', self.fs_name) + with contextutil.safe_while(sleep=5, tries=30) as proceed: + while proceed(): + output = self._cmd( + 'orch', 'ls', '-f', 'json', + '--service-name', f'mds.{self.fs_name}' + ) + j = json.loads(output) + if j[0]['status']['running']: + break export_cmd = ['nfs', 'export', 'create', 'cephfs', self.fs_name, self.cluster_id] if isinstance(extra_cmd, list): export_cmd.extend(extra_cmd) From 454cc8a18c4c3851de5976d3e36e42644dbb1a70 Mon Sep 17 00:00:00 2001 From: Marcus Watts Date: Fri, 17 Sep 2021 05:28:53 -0400 Subject: [PATCH 0367/2866] Fix vault token file access. Put the vault token file in a location that ceph can read. Make it readable only by ceph. On rhel8 (and indeed, any vanilla rhel machine), $HOME is liable to be mode 700. This means the ceph user can't read things in that user's directory. This causes radosgw to emit the confusing message "ERROR: Vault token file ... not found" even though the teuthology log will plainly show it was created and made readable by ceph. Fixes: http://tracker.ceph.com/issues/51539 Signed-off-by: Marcus Watts --- qa/tasks/rgw.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/qa/tasks/rgw.py b/qa/tasks/rgw.py index aa5e1fc3bd127..693d3d4d6d7bc 100644 --- a/qa/tasks/rgw.py +++ b/qa/tasks/rgw.py @@ -112,7 +112,7 @@ def start_rgw(ctx, config, clients): barbican_role = client_config.get('use-barbican-role', None) pykmip_role = client_config.get('use-pykmip-role', None) - token_path = teuthology.get_testdir(ctx) + '/vault-token' + token_path = '/etc/ceph/vault-root-token' if barbican_role is not None: if not hasattr(ctx, 'barbican'): raise ConfigError('rgw must run after the barbican task') @@ -131,11 +131,11 @@ def start_rgw(ctx, config, clients): raise ConfigError('vault: no "root_token" specified') # create token on file ctx.rgw.vault_role = vault_role - ctx.cluster.only(client).run(args=['echo', '-n', ctx.vault.root_token, run.Raw('>'), token_path]) + ctx.cluster.only(client).run(args=['sudo', 'echo', '-n', ctx.vault.root_token, run.Raw('|'), 'sudo', 'tee', token_path]) log.info("Token file content") ctx.cluster.only(client).run(args=['cat', token_path]) log.info("Restrict access to token file") - ctx.cluster.only(client).run(args=['chmod', '600', token_path]) + ctx.cluster.only(client).run(args=['sudo', 'chmod', '600', token_path]) ctx.cluster.only(client).run(args=['sudo', 'chown', 'ceph', token_path]) rgw_cmd.extend([ @@ -206,7 +206,7 @@ def start_rgw(ctx, config, clients): client=client_with_cluster), ], ) - ctx.cluster.only(client).run(args=['rm', '-f', token_path]) + ctx.cluster.only(client).run(args=['sudo', 'rm', '-f', token_path]) def assign_endpoints(ctx, config, default_cert): role_endpoints = {} From ec4f5cd0e15748d9dc7a579a8c731bbfe30a5aef Mon Sep 17 00:00:00 2001 From: Casey Bodley Date: Thu, 28 Oct 2021 15:20:26 -0400 Subject: [PATCH 0368/2866] rgw: fix self-comparison for RGWCopyObj optimization when RGWCopyObj is copying an object onto itself (for example, 's3cmd modify' does this to add/remove metadata), we can avoid the need to increment the reference count of each tail object. this self-comparison check was broken during zipper refactoring, comparing raw pointers instead of the object keys Fixes: https://tracker.ceph.com/issues/53003 Signed-off-by: Casey Bodley --- src/rgw/rgw_rados.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc index cf16234849ac5..e5d452063c369 100644 --- a/src/rgw/rgw_rados.cc +++ b/src/rgw/rgw_rados.cc @@ -4480,7 +4480,7 @@ int RGWRados::copy_obj(RGWObjectCtx& obj_ctx, bufferlist first_chunk; - bool copy_itself = (dest_obj == src_obj); + const bool copy_itself = (dest_obj->get_obj() == src_obj->get_obj()); RGWObjManifest *pmanifest; ldpp_dout(dpp, 20) << "dest_obj=" << dest_obj << " src_obj=" << src_obj << " copy_itself=" << (int)copy_itself << dendl; From 1b808345ee41fcef7345fe2ce18c41e30ab655dd Mon Sep 17 00:00:00 2001 From: Joseph Sawaya Date: Wed, 8 Sep 2021 09:36:26 -0400 Subject: [PATCH 0369/2866] mgr/rook: implement apply rbd-mirror This commit implements `orch apply rbd-mirror` in the rook orchestrator, it creates a CR with a default name if the service_id isn't specified in the spec, else it sets the name of the CR to the service_id in the spec. This commit also adds `orch apply rbd-mirror` to the rook QA. This commit also implements `orch rm rbd-mirror`. Signed-off-by: Joseph Sawaya --- qa/suites/orch/rook/smoke/3-final.yaml | 1 + qa/tasks/rook.py | 2 +- src/pybind/mgr/rook/module.py | 31 +++++++++++++++++++ src/pybind/mgr/rook/rook_cluster.py | 41 ++++++++++++++++++++++++++ 4 files changed, 74 insertions(+), 1 deletion(-) diff --git a/qa/suites/orch/rook/smoke/3-final.yaml b/qa/suites/orch/rook/smoke/3-final.yaml index 91a43f4acc8b5..152d42f2c4351 100644 --- a/qa/suites/orch/rook/smoke/3-final.yaml +++ b/qa/suites/orch/rook/smoke/3-final.yaml @@ -8,3 +8,4 @@ tasks: - ceph orch device ls - ceph orch apply rgw foo - ceph orch apply mds foo + - ceph orch apply rbd-mirror diff --git a/qa/tasks/rook.py b/qa/tasks/rook.py index 2498828db009f..15e2f04518962 100644 --- a/qa/tasks/rook.py +++ b/qa/tasks/rook.py @@ -651,7 +651,7 @@ def task(ctx, config): if ret.exitstatus == 0: r = json.loads(ret.stdout.getvalue().decode('utf-8')) for service in r: - if service['service_type'] in ['rgw', 'mds', 'nfs']: + if service['service_type'] in ['rgw', 'mds', 'nfs', 'rbd-mirror']: _shell(ctx, config, ['ceph', 'orch', 'rm', service['service_name']]) to_remove.append(service['service_name']) with safe_while(sleep=10, tries=90, action="waiting for service removal") as proceed: diff --git a/src/pybind/mgr/rook/module.py b/src/pybind/mgr/rook/module.py index 4fedaec8ae409..2f58032b481dd 100644 --- a/src/pybind/mgr/rook/module.py +++ b/src/pybind/mgr/rook/module.py @@ -355,6 +355,26 @@ def describe_service(self, last_refresh=now, running= sum(osd.status.phase == 'Running' for osd in all_osds) ) + + if service_type == 'rbd-mirror' or service_type is None: + # rbd-mirrors + all_mirrors = self.rook_cluster.get_resource("cephrbdmirrors") + for mirror in all_mirrors: + logging.warn(mirror) + mirror_name = mirror['metadata']['name'] + svc = 'rbd-mirror.' + mirror_name + if svc in spec: + continue + spec[svc] = orchestrator.ServiceDescription( + spec=ServiceSpec( + service_id=mirror_name, + service_type="rbd-mirror", + placement=PlacementSpec(count=1), + ), + size=1, + last_refresh=now, + ) + for dd in self._list_daemons(): if dd.service_name() not in spec: continue @@ -444,6 +464,8 @@ def _get_pool_params(self) -> Tuple[int, str]: @handle_orch_error def remove_service(self, service_name: str) -> str: + if service_name == 'rbd-mirror': + return self.rook_cluster.rm_service('cephrbdmirrors', 'default-rbd-mirror') service_type, service_name = service_name.split('.', 1) if service_type == 'mds': return self.rook_cluster.rm_service('cephfilesystems', service_name) @@ -451,6 +473,8 @@ def remove_service(self, service_name: str) -> str: return self.rook_cluster.rm_service('cephobjectstores', service_name) elif service_type == 'nfs': return self.rook_cluster.rm_service('cephnfses', service_name) + elif service_type == 'rbd-mirror': + return self.rook_cluster.rm_service('cephrbdmirrors', service_name) else: raise orchestrator.OrchestratorError(f'Service type {service_type} not supported') @@ -470,6 +494,13 @@ def apply_mon(self, spec): return self.rook_cluster.update_mon_count(spec.placement.count) + def apply_rbd_mirror(self, spec: ServiceSpec) -> OrchResult[str]: + try: + self.rook_cluster.rbd_mirror(spec) + return OrchResult("Success") + except Exception as e: + return OrchResult(None, e) + @handle_orch_error def apply_mds(self, spec): # type: (ServiceSpec) -> str diff --git a/src/pybind/mgr/rook/rook_cluster.py b/src/pybind/mgr/rook/rook_cluster.py index 8206ecc150669..a4bc0917ef6a8 100644 --- a/src/pybind/mgr/rook/rook_cluster.py +++ b/src/pybind/mgr/rook/rook_cluster.py @@ -43,6 +43,7 @@ class ApiException(Exception): # type: ignore from .rook_client.ceph import cephnfs as cnfs from .rook_client.ceph import cephobjectstore as cos from .rook_client.ceph import cephcluster as ccl +from .rook_client.ceph import cephrbdmirror as crbdm from .rook_client._helper import CrdClass import orchestrator @@ -1212,6 +1213,46 @@ def create_zap_job(self, host: str, path: str) -> None: ) self.batchV1_api.create_namespaced_job('rook-ceph', body) + def rbd_mirror(self, spec: ServiceSpec) -> None: + service_id = spec.service_id or "default-rbd-mirror" + all_hosts = self.get_hosts() + def _create_rbd_mirror() -> crbdm.CephRBDMirror: + return crbdm.CephRBDMirror( + apiVersion=self.rook_env.api_name, + metadata=dict( + name=service_id, + namespace=self.rook_env.namespace, + ), + spec=crbdm.Spec( + count=spec.placement.count or 1, + placement=crbdm.Placement( + nodeAffinity=crbdm.NodeAffinity( + requiredDuringSchedulingIgnoredDuringExecution=crbdm.RequiredDuringSchedulingIgnoredDuringExecution( + nodeSelectorTerms=crbdm.NodeSelectorTermsList( + [ + placement_spec_to_node_selector(spec.placement, all_hosts) + ] + ) + ) + ) + ) + ) + ) + def _update_rbd_mirror(new: crbdm.CephRBDMirror) -> crbdm.CephRBDMirror: + new.spec.count = spec.placement.count or 1 + new.spec.placement = crbdm.Placement( + nodeAffinity=crbdm.NodeAffinity( + requiredDuringSchedulingIgnoredDuringExecution=crbdm.RequiredDuringSchedulingIgnoredDuringExecution( + nodeSelectorTerms=crbdm.NodeSelectorTermsList( + [ + placement_spec_to_node_selector(spec.placement, all_hosts) + ] + ) + ) + ) + ) + return new + self._create_or_patch(crbdm.CephRBDMirror, 'cephrbdmirrors', service_id, _update_rbd_mirror, _create_rbd_mirror) def _patch(self, crd: Type, crd_name: str, cr_name: str, func: Callable[[CrdClassT, CrdClassT], CrdClassT]) -> str: current_json = self.rook_api_get( "{}/{}".format(crd_name, cr_name) From e7cea80450b0f791db2e81e758acbc4549717b1e Mon Sep 17 00:00:00 2001 From: Yuri Weinstein Date: Thu, 28 Oct 2021 15:13:04 -0700 Subject: [PATCH 0370/2866] qa/tests: added back octopus-x run on master Signed-off-by: Yuri Weinstein --- qa/crontab/teuthology-cronjobs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qa/crontab/teuthology-cronjobs b/qa/crontab/teuthology-cronjobs index 9d2fb8efb34e4..c7a63cee28cdb 100644 --- a/qa/crontab/teuthology-cronjobs +++ b/qa/crontab/teuthology-cronjobs @@ -116,5 +116,5 @@ CEPH_QA_EMAIL="ceph-qa@ceph.io" #20 01 * * 4 CEPH_BRANCH=master; MACHINE_NAME=smithi; /home/teuthology/bin/cron_wrapper teuthology-suite -v -c $CEPH_BRANCH -n 100 -m $MACHINE_NAME -s upgrade-clients/client-upgrade-octopus-pacific -k distro -e $CEPH_QA_EMAIL --suite-branch octopus -#22 14 * * 5 CEPH_BRANCH=master; MACHINE_NAME=smithi; SUITE_NAME=upgrade:octopus-x; KERNEL=distro; /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh 10 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 70 --force-priority +22 14 * * 5 CEPH_BRANCH=master; MACHINE_NAME=smithi; SUITE_NAME=upgrade:octopus-x; KERNEL=distro; /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh 10 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 70 --force-priority From 0ec838099064db9e5bbe31f9573445689339c4aa Mon Sep 17 00:00:00 2001 From: Neha Ojha Date: Thu, 28 Oct 2021 22:26:34 +0000 Subject: [PATCH 0371/2866] qa/suites/upgrade/octopus-x: exclude ceph-volume To fix failure like Failure Reason: Command failed on smithi085 with status 1: 'sudo yum -y install ceph-volume' Signed-off-by: Neha Ojha --- qa/suites/upgrade/octopus-x/parallel/1-tasks.yaml | 2 ++ qa/suites/upgrade/octopus-x/stress-split/1-start.yaml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/qa/suites/upgrade/octopus-x/parallel/1-tasks.yaml b/qa/suites/upgrade/octopus-x/parallel/1-tasks.yaml index fbe632b6f3fcd..0be7a0b7b0c05 100644 --- a/qa/suites/upgrade/octopus-x/parallel/1-tasks.yaml +++ b/qa/suites/upgrade/octopus-x/parallel/1-tasks.yaml @@ -1,6 +1,8 @@ tasks: - install: branch: octopus + exclude_packages: + - ceph-volume - print: "**** done install task..." - print: "**** done start installing octopus cephadm ..." - cephadm: diff --git a/qa/suites/upgrade/octopus-x/stress-split/1-start.yaml b/qa/suites/upgrade/octopus-x/stress-split/1-start.yaml index 8e86449f567df..211a369f35205 100644 --- a/qa/suites/upgrade/octopus-x/stress-split/1-start.yaml +++ b/qa/suites/upgrade/octopus-x/stress-split/1-start.yaml @@ -1,6 +1,8 @@ tasks: - install: branch: octopus + exclude_packages: + - ceph-volume - cephadm: image: docker.io/ceph/daemon-base:latest-octopus From ace857f58a9b7068a566eac07d0699d53fdaeeac Mon Sep 17 00:00:00 2001 From: Casey Bodley Date: Thu, 28 Oct 2021 17:55:06 -0400 Subject: [PATCH 0372/2866] rgw/rest: dump_time() takes real_time by value Signed-off-by: Casey Bodley --- src/rgw/rgw_rest.cc | 4 ++-- src/rgw/rgw_rest.h | 2 +- src/rgw/rgw_rest_s3.cc | 16 ++++++++-------- src/rgw/rgw_rest_swift.cc | 2 +- src/rgw/rgw_sync_module_es_rest.cc | 2 +- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/rgw/rgw_rest.cc b/src/rgw/rgw_rest.cc index c3389a43e28de..93380fb2112f1 100644 --- a/src/rgw/rgw_rest.cc +++ b/src/rgw/rgw_rest.cc @@ -479,10 +479,10 @@ void dump_epoch_header(struct req_state *s, const char *name, real_time t) return dump_header(s, name, std::string_view(buf, len)); } -void dump_time(struct req_state *s, const char *name, real_time *t) +void dump_time(struct req_state *s, const char *name, real_time t) { char buf[TIME_BUF_SIZE]; - rgw_to_iso8601(*t, buf, sizeof(buf)); + rgw_to_iso8601(t, buf, sizeof(buf)); s->formatter->dump_string(name, buf); } diff --git a/src/rgw/rgw_rest.h b/src/rgw/rgw_rest.h index c2129e8e74517..a8f01865d6617 100644 --- a/src/rgw/rgw_rest.h +++ b/src/rgw/rgw_rest.h @@ -801,7 +801,7 @@ extern void dump_range(struct req_state* s, uint64_t ofs, uint64_t end, uint64_t total_size); extern void dump_continue(struct req_state *s); extern void list_all_buckets_end(struct req_state *s); -extern void dump_time(struct req_state *s, const char *name, real_time *t); +extern void dump_time(struct req_state *s, const char *name, real_time t); extern std::string dump_time_to_str(const real_time& t); extern void dump_bucket_from_state(struct req_state *s); extern void dump_redirect(struct req_state *s, const std::string& redirect); diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc index 73f3ed5729a3f..4c3b326413b96 100644 --- a/src/rgw/rgw_rest_s3.cc +++ b/src/rgw/rgw_rest_s3.cc @@ -92,7 +92,7 @@ void dump_bucket(struct req_state *s, rgw::sal::Bucket& obj) { s->formatter->open_object_section("Bucket"); s->formatter->dump_string("Name", obj.get_name()); - dump_time(s, "CreationDate", &obj.get_creation_time()); + dump_time(s, "CreationDate", obj.get_creation_time()); s->formatter->close_section(); } @@ -1601,7 +1601,7 @@ void RGWListBucket_ObjStore_S3::send_versioned_response() } s->formatter->dump_string("VersionId", version_id); s->formatter->dump_bool("IsLatest", iter->is_current()); - dump_time(s, "LastModified", &iter->meta.mtime); + dump_time(s, "LastModified", iter->meta.mtime); if (!iter->is_delete_marker()) { s->formatter->dump_format("ETag", "\"%s\"", iter->meta.etag.c_str()); s->formatter->dump_int("Size", iter->meta.accounted_size); @@ -1691,7 +1691,7 @@ void RGWListBucket_ObjStore_S3::send_response() } else { s->formatter->dump_string("Key", key.name); } - dump_time(s, "LastModified", &iter->meta.mtime); + dump_time(s, "LastModified", iter->meta.mtime); s->formatter->dump_format("ETag", "\"%s\"", iter->meta.etag.c_str()); s->formatter->dump_int("Size", iter->meta.accounted_size); auto& storage_class = rgw_placement_rule::get_canonical_storage_class(iter->meta.storage_class); @@ -1768,7 +1768,7 @@ void RGWListBucket_ObjStore_S3v2::send_versioned_response() } s->formatter->dump_string("VersionId", version_id); s->formatter->dump_bool("IsLatest", iter->is_current()); - dump_time(s, "LastModified", &iter->meta.mtime); + dump_time(s, "LastModified", iter->meta.mtime); if (!iter->is_delete_marker()) { s->formatter->dump_format("ETag", "\"%s\"", iter->meta.etag.c_str()); s->formatter->dump_int("Size", iter->meta.accounted_size); @@ -1849,7 +1849,7 @@ void RGWListBucket_ObjStore_S3v2::send_response() else { s->formatter->dump_string("Key", key.name); } - dump_time(s, "LastModified", &iter->meta.mtime); + dump_time(s, "LastModified", iter->meta.mtime); s->formatter->dump_format("ETag", "\"%s\"", iter->meta.etag.c_str()); s->formatter->dump_int("Size", iter->meta.accounted_size); auto& storage_class = rgw_placement_rule::get_canonical_storage_class(iter->meta.storage_class); @@ -3320,7 +3320,7 @@ void RGWCopyObj_ObjStore_S3::send_response() send_partial_response(0); if (op_ret == 0) { - dump_time(s, "LastModified", &mtime); + dump_time(s, "LastModified", mtime); if (!etag.empty()) { s->formatter->dump_string("ETag", std::move(etag)); } @@ -3846,7 +3846,7 @@ void RGWListMultipart_ObjStore_S3::send_response() s->formatter->open_object_section("Part"); - dump_time(s, "LastModified", &part->get_mtime()); + dump_time(s, "LastModified", part->get_mtime()); s->formatter->dump_unsigned("PartNumber", part->get_num()); s->formatter->dump_format("ETag", "\"%s\"", part->get_etag().c_str()); @@ -3904,7 +3904,7 @@ void RGWListBucketMultiparts_ObjStore_S3::send_response() dump_owner(s, s->user->get_id(), s->user->get_display_name(), "Initiator"); dump_owner(s, s->user->get_id(), s->user->get_display_name()); s->formatter->dump_string("StorageClass", "STANDARD"); - dump_time(s, "Initiated", &upload->get_mtime()); + dump_time(s, "Initiated", upload->get_mtime()); s->formatter->close_section(); } if (!common_prefixes.empty()) { diff --git a/src/rgw/rgw_rest_swift.cc b/src/rgw/rgw_rest_swift.cc index 0a651bbb96a20..414f3a31c1a4c 100644 --- a/src/rgw/rgw_rest_swift.cc +++ b/src/rgw/rgw_rest_swift.cc @@ -401,7 +401,7 @@ void RGWListBucket_ObjStore_SWIFT::send_response() } s->formatter->dump_string("content_type", single_content_type); } - dump_time(s, "last_modified", &iter->meta.mtime); + dump_time(s, "last_modified", iter->meta.mtime); s->formatter->close_section(); } diff --git a/src/rgw/rgw_sync_module_es_rest.cc b/src/rgw/rgw_sync_module_es_rest.cc index 7beec151c1276..f21c307ffd0d1 100644 --- a/src/rgw/rgw_sync_module_es_rest.cc +++ b/src/rgw/rgw_sync_module_es_rest.cc @@ -342,7 +342,7 @@ class RGWMetadataSearch_ObjStore_S3 : public RGWMetadataSearchOp { string instance = (!e.key.instance.empty() ? e.key.instance : "null"); s->formatter->dump_string("Instance", instance.c_str()); s->formatter->dump_int("VersionedEpoch", e.versioned_epoch); - dump_time(s, "LastModified", &e.meta.mtime); + dump_time(s, "LastModified", e.meta.mtime); s->formatter->dump_int("Size", e.meta.size); s->formatter->dump_format("ETag", "\"%s\"", e.meta.etag.c_str()); s->formatter->dump_string("ContentType", e.meta.content_type.c_str()); From 200f9a0c40435f1ed2a469c149a10bd8009f25f3 Mon Sep 17 00:00:00 2001 From: Casey Bodley Date: Thu, 28 Oct 2021 18:13:58 -0400 Subject: [PATCH 0373/2866] rgw/rest: dump_owner() takes name by const ref Signed-off-by: Casey Bodley --- src/rgw/rgw_rest.cc | 2 +- src/rgw/rgw_rest.h | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/rgw/rgw_rest.cc b/src/rgw/rgw_rest.cc index 93380fb2112f1..214c9f27b8bba 100644 --- a/src/rgw/rgw_rest.cc +++ b/src/rgw/rgw_rest.cc @@ -487,7 +487,7 @@ void dump_time(struct req_state *s, const char *name, real_time t) s->formatter->dump_string(name, buf); } -void dump_owner(struct req_state *s, const rgw_user& id, string& name, +void dump_owner(struct req_state *s, const rgw_user& id, const string& name, const char *section) { if (!section) diff --git a/src/rgw/rgw_rest.h b/src/rgw/rgw_rest.h index a8f01865d6617..d12c4e80edb93 100644 --- a/src/rgw/rgw_rest.h +++ b/src/rgw/rgw_rest.h @@ -703,8 +703,8 @@ extern void end_header(struct req_state *s, bool force_no_error = false); extern void dump_start(struct req_state *s); extern void list_all_buckets_start(struct req_state *s); -extern void dump_owner(struct req_state *s, const rgw_user& id, std::string& name, - const char *section = NULL); +extern void dump_owner(struct req_state *s, const rgw_user& id, + const std::string& name, const char *section = NULL); extern void dump_header(struct req_state* s, const std::string_view& name, const std::string_view& val); From b05e92c2791a508508cfade1898d343c567a507a Mon Sep 17 00:00:00 2001 From: Casey Bodley Date: Thu, 28 Oct 2021 17:47:03 -0400 Subject: [PATCH 0374/2866] rgw/sal: MultipartUpload::get_mtime() is const, returns by value also added override specifiers in RadosMultipartUpload Signed-off-by: Casey Bodley --- src/rgw/rgw_sal.h | 2 +- src/rgw/rgw_sal_rados.h | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/rgw/rgw_sal.h b/src/rgw/rgw_sal.h index cf4761f77c993..55788a29dd7ef 100644 --- a/src/rgw/rgw_sal.h +++ b/src/rgw/rgw_sal.h @@ -818,7 +818,7 @@ class MultipartUpload { virtual const std::string& get_meta() const = 0; virtual const std::string& get_key() const = 0; virtual const std::string& get_upload_id() const = 0; - virtual ceph::real_time& get_mtime() = 0; + virtual ceph::real_time get_mtime() const = 0; std::map>& get_parts() { return parts; } diff --git a/src/rgw/rgw_sal_rados.h b/src/rgw/rgw_sal_rados.h index 849f1856a66b7..40951108873a7 100644 --- a/src/rgw/rgw_sal_rados.h +++ b/src/rgw/rgw_sal_rados.h @@ -522,10 +522,10 @@ class RadosMultipartUpload : public MultipartUpload { RadosMultipartUpload(RadosStore* _store, Bucket* _bucket, const std::string& oid, std::optional upload_id, ceph::real_time _mtime) : MultipartUpload(_bucket), store(_store), mp_obj(oid, upload_id), mtime(_mtime) {} virtual ~RadosMultipartUpload() = default; - virtual const std::string& get_meta() const { return mp_obj.get_meta(); } - virtual const std::string& get_key() const { return mp_obj.get_key(); } - virtual const std::string& get_upload_id() const { return mp_obj.get_upload_id(); } - virtual ceph::real_time& get_mtime() { return mtime; } + virtual const std::string& get_meta() const override { return mp_obj.get_meta(); } + virtual const std::string& get_key() const override { return mp_obj.get_key(); } + virtual const std::string& get_upload_id() const override { return mp_obj.get_upload_id(); } + virtual ceph::real_time get_mtime() const override { return mtime; } virtual std::unique_ptr get_meta_obj() override; virtual int init(const DoutPrefixProvider* dpp, optional_yield y, RGWObjectCtx* obj_ctx, ACLOwner& owner, rgw_placement_rule& dest_placement, rgw::sal::Attrs& attrs) override; virtual int list_parts(const DoutPrefixProvider* dpp, CephContext* cct, From 9ee191da9c4ccb2e1c87b991f7afde7d5b3130e1 Mon Sep 17 00:00:00 2001 From: Casey Bodley Date: Thu, 28 Oct 2021 18:08:21 -0400 Subject: [PATCH 0375/2866] rgw/sal: add MultipartUpload::get_owner() for ListMultipartUploads the ListMultipartUploads response includes an Owner and Initiator for each multipart upload in the list, but we were just returning the requesting user for every upload instead we get the real owner field out of rgw_bucket_dir_entry when listing the uploads, so need to store that with MultipartUpload for later use in RGWListBucketMultiparts_ObjStore_S3::send_response() Fixes: https://tracker.ceph.com/issues/52716 Signed-off-by: Casey Bodley --- src/rgw/rgw_rest_s3.cc | 5 +++-- src/rgw/rgw_sal.h | 6 +++++- src/rgw/rgw_sal_dbstore.cc | 5 ++++- src/rgw/rgw_sal_dbstore.h | 5 ++++- src/rgw/rgw_sal_rados.cc | 13 ++++++++++--- src/rgw/rgw_sal_rados.h | 13 +++++++++++-- 6 files changed, 37 insertions(+), 10 deletions(-) diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc index 4c3b326413b96..cbe292e511e1b 100644 --- a/src/rgw/rgw_rest_s3.cc +++ b/src/rgw/rgw_rest_s3.cc @@ -3901,8 +3901,9 @@ void RGWListBucketMultiparts_ObjStore_S3::send_response() s->formatter->dump_string("Key", upload->get_key()); } s->formatter->dump_string("UploadId", upload->get_upload_id()); - dump_owner(s, s->user->get_id(), s->user->get_display_name(), "Initiator"); - dump_owner(s, s->user->get_id(), s->user->get_display_name()); + const ACLOwner& owner = upload->get_owner(); + dump_owner(s, owner.get_id(), owner.get_display_name(), "Initiator"); + dump_owner(s, owner.get_id(), owner.get_display_name()); // Owner s->formatter->dump_string("StorageClass", "STANDARD"); dump_time(s, "Initiated", upload->get_mtime()); s->formatter->close_section(); diff --git a/src/rgw/rgw_sal.h b/src/rgw/rgw_sal.h index 55788a29dd7ef..23d2a43aa7ad3 100644 --- a/src/rgw/rgw_sal.h +++ b/src/rgw/rgw_sal.h @@ -262,7 +262,10 @@ class Store { virtual int get_oidc_providers(const DoutPrefixProvider *dpp, const std::string& tenant, std::vector>& providers) = 0; - virtual std::unique_ptr get_multipart_upload(Bucket* bucket, const std::string& oid, std::optional upload_id=std::nullopt, ceph::real_time mtime=real_clock::now()) = 0; + virtual std::unique_ptr + get_multipart_upload(Bucket* bucket, const std::string& oid, + std::optional upload_id=std::nullopt, + ACLOwner owner={}, ceph::real_time mtime=real_clock::now()) = 0; virtual std::unique_ptr get_append_writer(const DoutPrefixProvider *dpp, optional_yield y, std::unique_ptr _head_obj, @@ -818,6 +821,7 @@ class MultipartUpload { virtual const std::string& get_meta() const = 0; virtual const std::string& get_key() const = 0; virtual const std::string& get_upload_id() const = 0; + virtual const ACLOwner& get_owner() const = 0; virtual ceph::real_time get_mtime() const = 0; std::map>& get_parts() { return parts; } diff --git a/src/rgw/rgw_sal_dbstore.cc b/src/rgw/rgw_sal_dbstore.cc index fe019c95e23e2..a4c6bff9f458e 100644 --- a/src/rgw/rgw_sal_dbstore.cc +++ b/src/rgw/rgw_sal_dbstore.cc @@ -947,7 +947,10 @@ namespace rgw::sal { return 0; } - std::unique_ptr DBStore::get_multipart_upload(Bucket* bucket, const std::string& oid, std::optional upload_id, ceph::real_time mtime) { + std::unique_ptr + DBStore::get_multipart_upload(Bucket* bucket, const std::string& oid, + std::optional upload_id, + ACLOwner owner, ceph::real_time mtime) { return nullptr; } diff --git a/src/rgw/rgw_sal_dbstore.h b/src/rgw/rgw_sal_dbstore.h index dd8b8aa05d00a..16e6ce0608b9d 100644 --- a/src/rgw/rgw_sal_dbstore.h +++ b/src/rgw/rgw_sal_dbstore.h @@ -540,7 +540,10 @@ class DBNotification : public Notification { virtual int get_oidc_providers(const DoutPrefixProvider *dpp, const std::string& tenant, vector>& providers) override; - virtual std::unique_ptr get_multipart_upload(Bucket* bucket, const std::string& oid, std::optional upload_id, ceph::real_time mtime) override; + virtual std::unique_ptr + get_multipart_upload(Bucket* bucket, const std::string& oid, + std::optional upload_id=std::nullopt, + ACLOwner owner={}, ceph::real_time mtime=ceph::real_clock::now()) override; virtual std::unique_ptr get_append_writer(const DoutPrefixProvider *dpp, optional_yield y, std::unique_ptr _head_obj, diff --git a/src/rgw/rgw_sal_rados.cc b/src/rgw/rgw_sal_rados.cc index 1e7b352590d65..bec5a57a01ee9 100644 --- a/src/rgw/rgw_sal_rados.cc +++ b/src/rgw/rgw_sal_rados.cc @@ -777,7 +777,10 @@ int RadosBucket::list_multiparts(const DoutPrefixProvider *dpp, if (!results.objs.empty()) { for (const rgw_bucket_dir_entry& dentry : results.objs) { rgw_obj_key key(dentry.key); - uploads.push_back(store->get_multipart_upload(this, key.name)); + ACLOwner owner(rgw_user(dentry.meta.owner)); + owner.set_name(dentry.meta.owner_display_name); + uploads.push_back(store->get_multipart_upload( + this, key.name, std::nullopt, std::move(owner))); } } if (common_prefixes) { @@ -1441,9 +1444,13 @@ int RadosStore::get_oidc_providers(const DoutPrefixProvider *dpp, return 0; } -std::unique_ptr RadosStore::get_multipart_upload(Bucket* bucket, const std::string& oid, std::optional upload_id, ceph::real_time mtime) +std::unique_ptr +RadosStore::get_multipart_upload(Bucket* bucket, const std::string& oid, + std::optional upload_id, + ACLOwner owner, ceph::real_time mtime) { - return std::make_unique(this, bucket, oid, upload_id, mtime); + return std::make_unique(this, bucket, oid, upload_id, + std::move(owner), mtime); } std::unique_ptr RadosStore::get_append_writer(const DoutPrefixProvider *dpp, diff --git a/src/rgw/rgw_sal_rados.h b/src/rgw/rgw_sal_rados.h index 40951108873a7..b40bb02e19b27 100644 --- a/src/rgw/rgw_sal_rados.h +++ b/src/rgw/rgw_sal_rados.h @@ -444,7 +444,10 @@ class RadosStore : public Store { virtual int get_oidc_providers(const DoutPrefixProvider *dpp, const std::string& tenant, std::vector>& providers) override; - virtual std::unique_ptr get_multipart_upload(Bucket* bucket, const std::string& oid, std::optional upload_id=std::nullopt, ceph::real_time mtime=real_clock::now()) override; + virtual std::unique_ptr + get_multipart_upload(Bucket* bucket, const std::string& oid, + std::optional upload_id=std::nullopt, + ACLOwner owner={}, ceph::real_time mtime=real_clock::now()) override; virtual std::unique_ptr get_append_writer(const DoutPrefixProvider *dpp, optional_yield y, std::unique_ptr _head_obj, @@ -514,17 +517,23 @@ class RadosMultipartPart : public MultipartPart { class RadosMultipartUpload : public MultipartUpload { RadosStore* store; RGWMPObj mp_obj; + ACLOwner owner; ceph::real_time mtime; rgw_placement_rule placement; RGWObjManifest manifest; public: - RadosMultipartUpload(RadosStore* _store, Bucket* _bucket, const std::string& oid, std::optional upload_id, ceph::real_time _mtime) : MultipartUpload(_bucket), store(_store), mp_obj(oid, upload_id), mtime(_mtime) {} + RadosMultipartUpload(RadosStore* _store, Bucket* _bucket, const std::string& oid, + std::optional upload_id, ACLOwner owner, + ceph::real_time _mtime) + : MultipartUpload(_bucket), store(_store), mp_obj(oid, upload_id), + owner(owner), mtime(_mtime) {} virtual ~RadosMultipartUpload() = default; virtual const std::string& get_meta() const override { return mp_obj.get_meta(); } virtual const std::string& get_key() const override { return mp_obj.get_key(); } virtual const std::string& get_upload_id() const override { return mp_obj.get_upload_id(); } + virtual const ACLOwner& get_owner() const override { return owner; } virtual ceph::real_time get_mtime() const override { return mtime; } virtual std::unique_ptr get_meta_obj() override; virtual int init(const DoutPrefixProvider* dpp, optional_yield y, RGWObjectCtx* obj_ctx, ACLOwner& owner, rgw_placement_rule& dest_placement, rgw::sal::Attrs& attrs) override; From 69567f16f3272d6dac105057042a649e5baed080 Mon Sep 17 00:00:00 2001 From: Yuri Weinstein Date: Thu, 28 Oct 2021 16:11:50 -0700 Subject: [PATCH 0376/2866] qa/tests: added pacific-x suite to run on master Signed-off-by: Yuri Weinstein --- qa/crontab/teuthology-cronjobs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/qa/crontab/teuthology-cronjobs b/qa/crontab/teuthology-cronjobs index c7a63cee28cdb..2e210fbaf8e37 100644 --- a/qa/crontab/teuthology-cronjobs +++ b/qa/crontab/teuthology-cronjobs @@ -118,3 +118,6 @@ CEPH_QA_EMAIL="ceph-qa@ceph.io" 22 14 * * 5 CEPH_BRANCH=master; MACHINE_NAME=smithi; SUITE_NAME=upgrade:octopus-x; KERNEL=distro; /home/teuthology/bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh 10 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 70 --force-priority + +23 14 * * 5 CEPH_BRANCH=master; MACHINE_NAME=smithi; SUITE_NAME=upgrade:pacific-x; KERNEL=distro; /home/teuthology/ bin/cron_wrapper /home/teuthology/bin/schedule_subset.sh 10 $CEPH_BRANCH $MACHINE_NAME $SUITE_NAME $CEPH_QA_EMAIL $KERNEL -p 70 --force-priority + From 0f9e74ce0f5c9879a11e72a727b7d72a667635f7 Mon Sep 17 00:00:00 2001 From: Yingxin Cheng Date: Thu, 21 Oct 2021 14:51:36 +0800 Subject: [PATCH 0377/2866] crimson/os/seastore/journal: misc cleanup Signed-off-by: Yingxin Cheng --- src/crimson/os/seastore/extent_placement_manager.cc | 2 +- src/crimson/os/seastore/journal.cc | 13 ++++++------- src/crimson/os/seastore/journal.h | 11 +++++++---- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/src/crimson/os/seastore/extent_placement_manager.cc b/src/crimson/os/seastore/extent_placement_manager.cc index 0cd50930573a6..b78ec32eebc69 100644 --- a/src/crimson/os/seastore/extent_placement_manager.cc +++ b/src/crimson/os/seastore/extent_placement_manager.cc @@ -222,7 +222,7 @@ SegmentedAllocator::Writer::init_segment(Segment& segment) { segment_manager.get_block_size())); bp.zero(); auto header =segment_header_t{ - journal.next_journal_segment_seq - 1, // current seg seq = next seg seq - 1 + journal.get_segment_seq(), segment.get_segment_id(), NO_DELTAS, 0, true}; logger().debug("SegmentedAllocator::Writer::init_segment: initting {}, {}", diff --git a/src/crimson/os/seastore/journal.cc b/src/crimson/os/seastore/journal.cc index 3725aeee86741..cdda5c50747ae 100644 --- a/src/crimson/os/seastore/journal.cc +++ b/src/crimson/os/seastore/journal.cc @@ -9,7 +9,6 @@ #include "include/intarith.h" #include "crimson/os/seastore/segment_cleaner.h" -#include "crimson/os/seastore/segment_manager.h" namespace { seastar::logger& logger() { @@ -206,7 +205,7 @@ Journal::prep_replay_segments( std::vector> segments) { logger().debug( - "prep_replay_segments: have {} segments", + "Journal::prep_replay_segments: have {} segments", segments.size()); if (segments.empty()) { return crimson::ct_error::input_output_error::make(); @@ -247,7 +246,7 @@ Journal::prep_replay_segments( }); if (from->second.journal_segment_seq != journal_tail.segment_seq) { logger().error( - "prep_replay_segments: journal_tail {} does not match {}", + "Journal::prep_replay_segments: journal_tail {} does not match {}", journal_tail, from->second); assert(0 == "invalid"); @@ -285,7 +284,7 @@ std::optional> Journal::try_decode_deltas( bliter += ceph::encoded_sizeof_bounded(); bliter += sizeof(checksum_t) /* crc */; bliter += header.extents * ceph::encoded_sizeof_bounded(); - logger().debug("{}: decoding {} deltas", __func__, header.deltas); + logger().debug("Journal::try_decode_deltas: decoding {} deltas", header.deltas); std::vector deltas(header.deltas); for (auto &&i : deltas) { try { @@ -303,7 +302,7 @@ Journal::replay_segment( segment_header_t header, delta_handler_t &handler) { - logger().debug("replay_segment: starting at {}", seq); + logger().debug("Journal::replay_segment: starting at {}", seq); return seastar::do_with( scan_valid_records_cursor(seq.offset), ExtentReader::found_record_handler_t( @@ -316,7 +315,7 @@ Journal::replay_segment( if (!deltas) { // This should be impossible, we did check the crc on the mdbuf logger().error( - "Journal::replay_segment unable to decode deltas for record {}", + "Journal::replay_segment: unable to decode deltas for record {}", base); assert(deltas); } @@ -375,7 +374,7 @@ Journal::replay_ret Journal::replay( (auto &handler, auto &segments) mutable -> replay_ret { return prep_replay_segments(std::move(segment_headers)).safe_then( [this, &handler, &segments](auto replay_segs) mutable { - logger().debug("replay: found {} segments", replay_segs.size()); + logger().debug("Journal::replay: found {} segments", replay_segs.size()); segments = std::move(replay_segs); return crimson::do_for_each(segments, [this, &handler](auto i) mutable { return replay_segment(i.first, i.second, handler); diff --git a/src/crimson/os/seastore/journal.h b/src/crimson/os/seastore/journal.h index a1917f3770f62..7c6eb51074c05 100644 --- a/src/crimson/os/seastore/journal.h +++ b/src/crimson/os/seastore/journal.h @@ -3,8 +3,6 @@ #pragma once -#include "crimson/common/log.h" - #include #include @@ -32,6 +30,13 @@ class Journal { public: Journal(SegmentManager &segment_manager, ExtentReader& scanner); + /** + * Gets the current journal segment sequence. + */ + segment_seq_t get_segment_seq() const { + return next_journal_segment_seq - 1; + } + /** * Sets the SegmentProvider. * @@ -207,7 +212,6 @@ class Journal { const bufferlist &bl); private: - /// replays records starting at start through end of segment replay_ertr::future<> replay_segment( @@ -217,7 +221,6 @@ class Journal { ); extent_len_t max_record_length() const; - friend class crimson::os::seastore::SegmentedAllocator; }; using JournalRef = std::unique_ptr; From a6d91c3f971ce2c564fe8ad82a66ab6451207cd4 Mon Sep 17 00:00:00 2001 From: Yingxin Cheng Date: Thu, 21 Oct 2021 15:31:47 +0800 Subject: [PATCH 0378/2866] crimson/os/seastore/journal: refactor, introduce JournalSegmentManager In preparation for the follow-up batching control. Signed-off-by: Yingxin Cheng --- src/crimson/os/seastore/journal.cc | 310 ++++++++++++++++------------- src/crimson/os/seastore/journal.h | 176 ++++++++++------ 2 files changed, 284 insertions(+), 202 deletions(-) diff --git a/src/crimson/os/seastore/journal.cc b/src/crimson/os/seastore/journal.cc index cdda5c50747ae..f4b7d4ef03276 100644 --- a/src/crimson/os/seastore/journal.cc +++ b/src/crimson/os/seastore/journal.cc @@ -49,55 +49,12 @@ segment_nonce_t generate_nonce( sizeof(meta.seastore_id.uuid)); } -Journal::Journal(SegmentManager &segment_manager, ExtentReader& scanner) - : segment_manager(segment_manager), scanner(scanner) {} - - -Journal::initialize_segment_ertr::future -Journal::initialize_segment(Segment &segment) -{ - auto new_tail = segment_provider->get_journal_tail_target(); - // write out header - ceph_assert(segment.get_write_ptr() == 0); - bufferlist bl; - - segment_seq_t seq = next_journal_segment_seq++; - current_segment_nonce = generate_nonce( - seq, segment_manager.get_meta()); - auto header = segment_header_t{ - seq, - segment.get_segment_id(), - segment_provider->get_journal_tail_target(), - current_segment_nonce, - false}; - logger().debug( - "initialize_segment {} journal_tail_target {}, header {}", - segment.get_segment_id(), - new_tail, - header); - encode(header, bl); - - bufferptr bp( - ceph::buffer::create_page_aligned( - segment_manager.get_block_size())); - bp.zero(); - auto iter = bl.cbegin(); - iter.copy(bl.length(), bp.c_str()); - bl.clear(); - bl.append(bp); - - written_to = segment_manager.get_block_size(); - committed_to = 0; - return segment.write(0, bl).safe_then( - [=] { - segment_provider->update_journal_tail_committed(new_tail); - return seq; - }, - initialize_segment_ertr::pass_further{}, - crimson::ct_error::assert_all{ - "Invalid error in Journal::initialize_segment" - }); -} +Journal::Journal( + SegmentManager& segment_manager, + ExtentReader& scanner) + : journal_segment_manager(segment_manager), + scanner(scanner) +{} Journal::write_record_ret Journal::write_record( record_size_t rsize, @@ -105,98 +62,26 @@ Journal::write_record_ret Journal::write_record( OrderingHandle &handle) { ceph::bufferlist to_write = encode_record( - rsize, std::move(record), segment_manager.get_block_size(), - committed_to, current_segment_nonce); - auto target = written_to; - assert((to_write.length() % segment_manager.get_block_size()) == 0); - written_to += to_write.length(); - logger().debug( - "write_record, mdlength {}, dlength {}, target {}", - rsize.mdlength, - rsize.dlength, - target); - - auto segment_id = current_journal_segment->get_segment_id(); - + rsize, + std::move(record), + journal_segment_manager.get_block_size(), + journal_segment_manager.get_committed_to(), + journal_segment_manager.get_nonce()); // Start write under the current exclusive stage, but wait for it // in the device_submission concurrent stage to permit multiple // overlapping writes. - auto write_fut = current_journal_segment->write(target, to_write); + auto write_fut = journal_segment_manager.write(to_write); return handle.enter(write_pipeline->device_submission ).then([write_fut = std::move(write_fut)]() mutable { - return std::move(write_fut - ).handle_error( - write_record_ertr::pass_further{}, - crimson::ct_error::assert_all{ - "Invalid error in Journal::write_record" - } - ); - }).safe_then([this, &handle] { - return handle.enter(write_pipeline->finalize); - }).safe_then([this, target, segment_id] { - logger().debug( - "write_record: commit target {}", - target); - if (segment_id == current_journal_segment->get_segment_id()) { - assert(committed_to < target); - committed_to = target; - } - return write_record_ret( - write_record_ertr::ready_future_marker{}, - paddr_t{ - segment_id, - target}); - }); -} - -bool Journal::needs_roll(segment_off_t length) const -{ - return length + written_to > - current_journal_segment->get_write_capacity(); -} - -Journal::roll_journal_segment_ertr::future -Journal::roll_journal_segment() -{ - auto old_segment_id = current_journal_segment ? - current_journal_segment->get_segment_id() : - NULL_SEG_ID; - - return (current_journal_segment ? - current_journal_segment->close() : - Segment::close_ertr::now()).safe_then([this] { - return segment_provider->get_segment(segment_manager.get_device_id()); - }).safe_then([this](auto segment) { - return segment_manager.open(segment); - }).safe_then([this](auto sref) { - current_journal_segment = sref; - written_to = 0; - return initialize_segment(*current_journal_segment); - }).safe_then([=](auto seq) { - if (old_segment_id != NULL_SEG_ID) { - segment_provider->close_segment(old_segment_id); - } - segment_provider->set_journal_segment( - current_journal_segment->get_segment_id(), - seq); - return seq; - }).handle_error( - roll_journal_segment_ertr::pass_further{}, - crimson::ct_error::all_same_way([] { ceph_assert(0 == "TODO"); }) - ); -} - -Journal::open_for_write_ret Journal::open_for_write() -{ - return roll_journal_segment().safe_then([this](auto seq) { - return open_for_write_ret( - open_for_write_ertr::ready_future_marker{}, - journal_seq_t{ - seq, - paddr_t{ - current_journal_segment->get_segment_id(), - static_cast(segment_manager.get_block_size())} - }); + return std::move(write_fut); + }).safe_then([this, &handle, rsize](journal_seq_t write_start) { + return handle.enter(write_pipeline->finalize + ).then([this, write_start, rsize] { + auto committed_to = write_start; + committed_to.offset.offset += (rsize.mdlength + rsize.dlength); + journal_segment_manager.mark_committed(committed_to); + return write_start.offset; + }); }); } @@ -218,8 +103,8 @@ Journal::prep_replay_segments( rt.second.journal_segment_seq; }); - next_journal_segment_seq = - segments.rbegin()->second.journal_segment_seq + 1; + journal_segment_manager.set_segment_seq( + segments.rbegin()->second.journal_segment_seq); std::for_each( segments.begin(), segments.end(), @@ -254,7 +139,7 @@ Journal::prep_replay_segments( } else { replay_from = paddr_t{ from->first, - (segment_off_t)segment_manager.get_block_size()}; + (segment_off_t)journal_segment_manager.get_block_size()}; } auto ret = replay_segments_t(segments.end() - from); std::transform( @@ -264,7 +149,7 @@ Journal::prep_replay_segments( p.second.journal_segment_seq, paddr_t{ p.first, - (segment_off_t)segment_manager.get_block_size()}}; + (segment_off_t)journal_segment_manager.get_block_size()}}; logger().debug( "Journal::prep_replay_segments: replaying from {}", ret); @@ -383,4 +268,149 @@ Journal::replay_ret Journal::replay( }); } +Journal::JournalSegmentManager::JournalSegmentManager( + SegmentManager& segment_manager) + : segment_manager{segment_manager} +{ + reset(); +} + +Journal::JournalSegmentManager::close_ertr::future<> +Journal::JournalSegmentManager::close() +{ + return ( + current_journal_segment ? + current_journal_segment->close() : + Segment::close_ertr::now() + ).handle_error( + close_ertr::pass_further{}, + crimson::ct_error::assert_all{ + "Invalid error in JournalSegmentManager::close()" + } + ).finally([this] { + reset(); + }); +} + +Journal::JournalSegmentManager::roll_ertr::future<> +Journal::JournalSegmentManager::roll() +{ + auto old_segment_id = current_journal_segment ? + current_journal_segment->get_segment_id() : + NULL_SEG_ID; + + return ( + current_journal_segment ? + current_journal_segment->close() : + Segment::close_ertr::now() + ).safe_then([this] { + return segment_provider->get_segment(segment_manager.get_device_id()); + }).safe_then([this](auto segment) { + return segment_manager.open(segment); + }).safe_then([this](auto sref) { + current_journal_segment = sref; + return initialize_segment(*current_journal_segment); + }).safe_then([this, old_segment_id] { + if (old_segment_id != NULL_SEG_ID) { + segment_provider->close_segment(old_segment_id); + } + segment_provider->set_journal_segment( + current_journal_segment->get_segment_id(), + get_segment_seq()); + }).handle_error( + roll_ertr::pass_further{}, + crimson::ct_error::assert_all{ + "Invalid error in JournalSegmentManager::roll" + } + ); +} + +Journal::JournalSegmentManager::write_ret +Journal::JournalSegmentManager::write(ceph::bufferlist to_write) +{ + auto write_length = to_write.length(); + auto write_start_seq = get_current_write_seq(); + logger().debug( + "JournalSegmentManager::write: write_start {} => {}, length={}", + write_start_seq, + write_start_seq.offset.offset + write_length, + write_length); + assert((write_length % segment_manager.get_block_size()) == 0); + assert(!needs_roll(write_length)); + + auto write_start_offset = written_to; + written_to += write_length; + return current_journal_segment->write( + write_start_offset, to_write + ).handle_error( + write_ertr::pass_further{}, + crimson::ct_error::assert_all{ + "Invalid error in JournalSegmentManager::write" + } + ).safe_then([write_start_seq] { + return write_start_seq; + }); +} + +void Journal::JournalSegmentManager::mark_committed( + const journal_seq_t& new_committed_to) +{ + logger().debug( + "JournalSegmentManager::mark_committed: committed_to {} => {}", + committed_to, new_committed_to); + assert(new_committed_to.segment_seq <= + get_segment_seq()); + if (new_committed_to.segment_seq == + get_segment_seq()) { + assert(committed_to.offset.offset < new_committed_to.offset.offset); + committed_to = new_committed_to; + } +} + +Journal::JournalSegmentManager::initialize_segment_ertr::future<> +Journal::JournalSegmentManager::initialize_segment(Segment& segment) +{ + auto new_tail = segment_provider->get_journal_tail_target(); + // write out header + ceph_assert(segment.get_write_ptr() == 0); + bufferlist bl; + + segment_seq_t seq = next_journal_segment_seq++; + current_segment_nonce = generate_nonce( + seq, segment_manager.get_meta()); + auto header = segment_header_t{ + seq, + segment.get_segment_id(), + segment_provider->get_journal_tail_target(), + current_segment_nonce, + false}; + logger().debug( + "JournalSegmentManager::initialize_segment: segment_id {} journal_tail_target {}, header {}", + segment.get_segment_id(), + new_tail, + header); + encode(header, bl); + + bufferptr bp( + ceph::buffer::create_page_aligned( + segment_manager.get_block_size())); + bp.zero(); + auto iter = bl.cbegin(); + iter.copy(bl.length(), bp.c_str()); + bl.clear(); + bl.append(bp); + + written_to = 0; + // FIXME: improve committed_to to point to another segment + committed_to = get_current_write_seq(); + return write(bl + ).safe_then([this, new_tail, write_size=bl.length() + ](journal_seq_t write_start_seq) { + auto committed_to = write_start_seq; + committed_to.offset.offset += write_size; + mark_committed(committed_to); + segment_provider->update_journal_tail_committed(new_tail); + }); +} + } diff --git a/src/crimson/os/seastore/journal.h b/src/crimson/os/seastore/journal.h index 7c6eb51074c05..064020dcc1c5b 100644 --- a/src/crimson/os/seastore/journal.h +++ b/src/crimson/os/seastore/journal.h @@ -4,6 +4,7 @@ #pragma once #include +#include #include @@ -34,7 +35,7 @@ class Journal { * Gets the current journal segment sequence. */ segment_seq_t get_segment_seq() const { - return next_journal_segment_seq - 1; + return journal_segment_manager.get_segment_seq(); } /** @@ -48,6 +49,7 @@ class Journal { */ void set_segment_provider(SegmentProvider *provider) { segment_provider = provider; + journal_segment_manager.set_segment_provider(provider); } /** @@ -59,7 +61,9 @@ class Journal { crimson::ct_error::input_output_error >; using open_for_write_ret = open_for_write_ertr::future; - open_for_write_ret open_for_write(); + open_for_write_ret open_for_write() { + return journal_segment_manager.open(); + } /** * close journal @@ -69,19 +73,7 @@ class Journal { using close_ertr = crimson::errorator< crimson::ct_error::input_output_error>; close_ertr::future<> close() { - return ( - current_journal_segment ? - current_journal_segment->close() : - Segment::close_ertr::now() - ).handle_error( - close_ertr::pass_further{}, - crimson::ct_error::assert_all{ - "Error during Journal::close()" - } - ).finally([this] { - current_journal_segment.reset(); - reset_soft_state(); - }); + return journal_segment_manager.close(); } /** @@ -102,23 +94,24 @@ class Journal { ) { assert(write_pipeline); auto rsize = get_encoded_record_length( - record, segment_manager.get_block_size()); + record, journal_segment_manager.get_block_size()); auto total = rsize.mdlength + rsize.dlength; - if (total > max_record_length()) { + auto max_record_length = journal_segment_manager.get_max_write_length(); + if (total > max_record_length) { auto &logger = crimson::get_logger(ceph_subsys_seastore); logger.error( "Journal::submit_record: record size {} exceeds max {}", total, - max_record_length() + max_record_length ); return crimson::ct_error::erange::make(); } - auto roll = needs_roll(total) - ? roll_journal_segment().safe_then([](auto){}) - : roll_journal_segment_ertr::now(); + auto roll = journal_segment_manager.needs_roll(total) + ? journal_segment_manager.roll() + : JournalSegmentManager::roll_ertr::now(); return roll.safe_then( [this, rsize, record=std::move(record), &handle]() mutable { - auto seq = next_journal_segment_seq - 1; + auto seq = journal_segment_manager.get_segment_seq(); return write_record( rsize, std::move(record), handle @@ -151,32 +144,111 @@ class Journal { } private: - SegmentProvider *segment_provider = nullptr; - SegmentManager &segment_manager; + class JournalSegmentManager { + public: + JournalSegmentManager(SegmentManager&); + + using base_ertr = crimson::errorator< + crimson::ct_error::input_output_error>; + extent_len_t get_max_write_length() const { + return segment_manager.get_segment_size() - + p2align(ceph::encoded_sizeof_bounded(), + size_t(segment_manager.get_block_size())); + } - segment_seq_t next_journal_segment_seq = 0; - segment_nonce_t current_segment_nonce = 0; + segment_off_t get_block_size() const { + return segment_manager.get_block_size(); + } - SegmentRef current_journal_segment; - segment_off_t written_to = 0; - segment_off_t committed_to = 0; + segment_nonce_t get_nonce() const { + return current_segment_nonce; + } - ExtentReader& scanner; - WritePipeline *write_pipeline = nullptr; + segment_off_t get_committed_to() const { + assert(committed_to.segment_seq == + get_segment_seq()); + return committed_to.offset.offset; + } - void reset_soft_state() { - next_journal_segment_seq = 0; - current_segment_nonce = 0; - written_to = 0; - committed_to = 0; - } + segment_seq_t get_segment_seq() const { + return next_journal_segment_seq - 1; + } - /// prepare segment for writes, writes out segment header - using initialize_segment_ertr = crimson::errorator< - crimson::ct_error::input_output_error>; - initialize_segment_ertr::future initialize_segment( - Segment &segment); + void set_segment_provider(SegmentProvider* provider) { + segment_provider = provider; + } + + void set_segment_seq(segment_seq_t current_seq) { + next_journal_segment_seq = (current_seq + 1); + } + + using open_ertr = base_ertr; + using open_ret = open_ertr::future; + open_ret open() { + return roll().safe_then([this] { + return get_current_write_seq(); + }); + } + + using close_ertr = base_ertr; + close_ertr::future<> close(); + // returns true iff the current segment has insufficient space + bool needs_roll(std::size_t length) const { + auto write_capacity = current_journal_segment->get_write_capacity(); + return length + written_to > std::size_t(write_capacity); + } + + // close the current segment and initialize next one + using roll_ertr = base_ertr; + roll_ertr::future<> roll(); + + // write the buffer, return the write start + // May be called concurrently, writes may complete in any order. + using write_ertr = base_ertr; + using write_ret = write_ertr::future; + write_ret write(ceph::bufferlist to_write); + + // mark write committed in order + void mark_committed(const journal_seq_t& new_committed_to); + + private: + journal_seq_t get_current_write_seq() const { + assert(current_journal_segment); + return journal_seq_t{ + get_segment_seq(), + {current_journal_segment->get_segment_id(), written_to} + }; + } + + void reset() { + next_journal_segment_seq = 0; + current_segment_nonce = 0; + current_journal_segment.reset(); + written_to = 0; + committed_to = {}; + } + + // prepare segment for writes, writes out segment header + using initialize_segment_ertr = base_ertr; + initialize_segment_ertr::future<> initialize_segment(Segment&); + + SegmentProvider* segment_provider; + SegmentManager& segment_manager; + + segment_seq_t next_journal_segment_seq; + segment_nonce_t current_segment_nonce; + + SegmentRef current_journal_segment; + segment_off_t written_to; + // committed_to may be in a previous journal segment + journal_seq_t committed_to; + }; + + SegmentProvider* segment_provider = nullptr; + JournalSegmentManager journal_segment_manager; + ExtentReader& scanner; + WritePipeline *write_pipeline = nullptr; /// do record write using write_record_ertr = crimson::errorator< @@ -187,14 +259,6 @@ class Journal { record_t &&record, OrderingHandle &handle); - /// close current segment and initialize next one - using roll_journal_segment_ertr = crimson::errorator< - crimson::ct_error::input_output_error>; - roll_journal_segment_ertr::future roll_journal_segment(); - - /// returns true iff current segment has insufficient space - bool needs_roll(segment_off_t length) const; - /// return ordered vector of segments to replay using replay_segments_t = std::vector< std::pair>; @@ -219,19 +283,7 @@ class Journal { segment_header_t header, ///< [in] segment header delta_handler_t &delta_handler ///< [in] processes deltas in order ); - - extent_len_t max_record_length() const; }; using JournalRef = std::unique_ptr; } - -namespace crimson::os::seastore { - -inline extent_len_t Journal::max_record_length() const { - return segment_manager.get_segment_size() - - p2align(ceph::encoded_sizeof_bounded(), - size_t(segment_manager.get_block_size())); -} - -} From 9c2ef6a75cb05ab142c267f710a566c2745f72fa Mon Sep 17 00:00:00 2001 From: Patrick Donnelly Date: Thu, 28 Oct 2021 22:13:50 -0400 Subject: [PATCH 0379/2866] script: update ceph-debug-docker for centos:stream Signed-off-by: Patrick Donnelly --- src/script/ceph-debug-docker.sh | 36 +++++++++++++++++++++++++++------ 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/src/script/ceph-debug-docker.sh b/src/script/ceph-debug-docker.sh index 84b735b12a378..2c2407e64cb44 100755 --- a/src/script/ceph-debug-docker.sh +++ b/src/script/ceph-debug-docker.sh @@ -14,6 +14,8 @@ set -e CACHE="" FLAVOR="default" +SUDO="" +PRIVILEGED="" function run { printf "%s\n" "$*" @@ -21,7 +23,7 @@ function run { } function main { - eval set -- $(getopt --name "$0" --options 'h' --longoptions 'help,no-cache,flavor:' -- "$@") + eval set -- $(getopt --name "$0" --options 'h' --longoptions 'help,no-cache,flavor:,sudo,privileged' -- "$@") while [ "$#" -gt 0 ]; do case "$1" in @@ -37,6 +39,14 @@ function main { FLAVOR=$2 shift 2 ;; + --privileged) + PRIVILEGED=--privileged + shift 1 + ;; + --sudo) + SUDO=sudo + shift 1 + ;; --) shift break @@ -84,7 +94,13 @@ function main { T=$(mktemp -d) pushd "$T" - distro="${env/://}" + case "$env" in + centos:stream) + distro="centos/8" + ;; + *) + distro="${env/://}" + esac api_url="https://shaman.ceph.com/api/search/?status=ready&project=ceph&flavor=${FLAVOR}&distros=${distro}/$(arch)&ref=${branch}&sha1=${sha}" repo_url="$(wget -O - "$api_url" | jq -r '.[0].chacra_url')repo" # validate url: @@ -104,19 +120,27 @@ RUN apt-key add cephdev.asc && \ apt-get update --yes && \ DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get --assume-yes -q --no-install-recommends install -o Dpkg::Options::=--force-confnew --allow-unauthenticated ceph ceph-osd-dbg ceph-mds-dbg ceph-mgr-dbg ceph-mon-dbg ceph-common-dbg ceph-fuse-dbg ceph-test-dbg radosgw-dbg python3-cephfs python3-rados EOF - time run docker build $CACHE --tag "$tag" . + time run $SUDO docker build $CACHE --tag "$tag" . else # try RHEL flavor case "$env" in centos:7) python_bindings="python36-rados python36-cephfs" + base_debuginfo="" ceph_debuginfo="ceph-debuginfo" debuginfo=/etc/yum.repos.d/CentOS-Linux-Debuginfo.repo ;; centos:8) python_bindings="python3-rados python3-cephfs" + base_debuginfo="glibc-debuginfo" ceph_debuginfo="ceph-base-debuginfo" debuginfo=/etc/yum.repos.d/CentOS-Linux-Debuginfo.repo ;; + centos:stream) + python_bindings="python3-rados python3-cephfs" + base_debuginfo="glibc-debuginfo" + ceph_debuginfo="ceph-base-debuginfo" + debuginfo=/etc/yum.repos.d/CentOS-Stream-Debuginfo.repo + ;; esac if [ "${FLAVOR}" = "crimson" ]; then ceph_debuginfo+=" ceph-crimson-osd-debuginfo ceph-crimson-osd" @@ -132,16 +156,16 @@ RUN yum update -y && \ RUN wget -O /etc/yum.repos.d/ceph-dev.repo $repo_url && \ yum clean all && \ yum upgrade -y && \ - yum install -y ceph ${ceph_debuginfo} ${python_bindings} + yum install -y ceph ${base_debuginfo} ${ceph_debuginfo} ${python_bindings} EOF - time run docker build $CACHE --tag "$tag" . + time run $SUDO docker build $CACHE --tag "$tag" . fi popd rm -rf -- "$T" printf "built image %s\n" "$tag" - run docker run -ti -v /ceph:/ceph:ro -v /cephfs:/cephfs:ro -v /teuthology:/teuthology:ro "$tag" + run $SUDO docker run $PRIVILEGED -ti -v /ceph:/ceph:ro -v /cephfs:/cephfs:ro -v /teuthology:/teuthology:ro "$tag" return 0 } From 75c8d578ed106c4584c166d8ccd9153c69667d01 Mon Sep 17 00:00:00 2001 From: Avan Thakkar Date: Thu, 28 Oct 2021 01:00:28 +0530 Subject: [PATCH 0380/2866] mgr/dashboard: improve error handling for gather_facts Fixes: https://tracker.ceph.com/issues/53084 Signed-off-by: Avan Thakkar --- src/pybind/mgr/dashboard/controllers/host.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/pybind/mgr/dashboard/controllers/host.py b/src/pybind/mgr/dashboard/controllers/host.py index 9d63d0a96ef1f..44ecc26afbeaa 100644 --- a/src/pybind/mgr/dashboard/controllers/host.py +++ b/src/pybind/mgr/dashboard/controllers/host.py @@ -287,18 +287,17 @@ def list(self, sources=None, facts=False): hosts = get_hosts(sources) orch = OrchClient.instance() if str_to_bool(facts): - if orch.available(['get_facts']): - try: + if orch.available(): + if not orch.get_missing_features(['get_facts']): hosts_facts = orch.hosts.get_facts() return merge_list_of_dicts_by_key(hosts, hosts_facts, 'hostname') - except Exception: - raise DashboardException( - code='invalid_orchestrator_backend', # pragma: no cover - msg="Please enable the cephadm orchestrator backend " - "(try `ceph orch set backend cephadm`)", - component='orchestrator', - http_status_code=400) + raise DashboardException( + code='invalid_orchestrator_backend', # pragma: no cover + msg="Please enable the cephadm orchestrator backend " + "(try `ceph orch set backend cephadm`)", + component='orchestrator', + http_status_code=400) raise DashboardException(code='orchestrator_status_unavailable', # pragma: no cover msg="Please configure and enable the orchestrator if you " From 9b2a0d92215fdc21192d0f40a8de31190fa1038f Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Mon, 9 Aug 2021 17:53:19 -0400 Subject: [PATCH 0381/2866] os/bluestore: select 'zoned' freelistmanager during mkfs, not mount Signed-off-by: Sage Weil --- src/os/bluestore/BlueStore.cc | 97 +++++++++++++++++++---------------- src/os/bluestore/BlueStore.h | 1 - 2 files changed, 52 insertions(+), 46 deletions(-) diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc index 9baeec27c1b10..030c1f59429f4 100644 --- a/src/os/bluestore/BlueStore.cc +++ b/src/os/bluestore/BlueStore.cc @@ -5328,11 +5328,6 @@ int BlueStore::_open_bdev(bool create) goto fail_close; } -#ifdef HAVE_LIBZBD - if (bdev->is_smr()) { - freelist_type = "zoned"; - } -#endif return 0; fail_close: @@ -5397,9 +5392,20 @@ int BlueStore::_open_fm(KeyValueDB::Transaction t, bool read_only, bool fm_resto uint64_t alloc_size = min_alloc_size; #ifdef HAVE_LIBZBD if (bdev->is_smr()) { + if (freelist_type != "zoned") { + derr << "SMR device but freelist_type = " << freelist_type << " (not zoned)" + << dendl; + return -EINVAL; + } alloc_size = _zoned_piggyback_device_parameters_onto(alloc_size); - } + } else #endif + if (freelist_type == "zoned") { + derr << "non-SMR device (or SMR support not built-in) but freelist_type = zoned" + << dendl; + return -EINVAL; + } + fm->create(bdev->get_size(), alloc_size, t); auto reserved = _get_ondisk_reserved(); if (fm_restore) { @@ -5514,26 +5520,46 @@ int BlueStore::_create_alloc() ceph_assert(bdev->get_size()); uint64_t alloc_size = min_alloc_size; - + + std::string allocator_type = cct->_conf->bluestore_allocator; + #ifdef HAVE_LIBZBD - if (bdev->is_smr()) { - int r = _zoned_check_config_settings(); - if (r < 0) - return r; + if (freelist_type == "zoned") { + allocator_type = "zoned"; + + // At least for now we want to use large min_alloc_size with HM-SMR drives. + // Populating used_blocks bitset on a debug build of ceph-osd takes about 5 + // minutes with a 14 TB HM-SMR drive and 4 KiB min_alloc_size. + if (min_alloc_size < 64 * 1024) { + dout(1) << __func__ << " The drive is HM-SMR but min_alloc_size is " + << min_alloc_size << ". " + << "Please set to at least 64 KiB." << dendl; + return -EINVAL; + } + + // We don't want to defer writes with HM-SMR because it violates sequential + // write requirement. + if (prefer_deferred_size) { + dout(1) << __func__ << " The drive is HM-SMR but prefer_deferred_size is " + << prefer_deferred_size << ". " + << "Please set to 0." << dendl; + return -EINVAL; + } + alloc_size = _zoned_piggyback_device_parameters_onto(alloc_size); } #endif - shared_alloc.set(Allocator::create(cct, cct->_conf->bluestore_allocator, + shared_alloc.set(Allocator::create(cct, allocator_type, bdev->get_size(), alloc_size, "block")); if (!shared_alloc.a) { - lderr(cct) << __func__ << "Failed to create allocator:: " - << cct->_conf->bluestore_allocator - << dendl; + lderr(cct) << __func__ << " failed to create " << allocator_type << " allocator" + << dendl; return -EINVAL; } + return 0; } @@ -6617,8 +6643,6 @@ int BlueStore::mkfs() } } - freelist_type = "bitmap"; - r = _open_path(); if (r < 0) return r; @@ -6672,6 +6696,17 @@ int BlueStore::mkfs() if (r < 0) goto out_close_fsid; + // choose freelist manager +#ifdef HAVE_LIBZBD + if (bdev->is_smr()) { + freelist_type = "zoned"; + } else +#endif + { + freelist_type = "bitmap"; + } + dout(10) << " freelist_type " << freelist_type << dendl; + // choose min_alloc_size if (cct->_conf->bluestore_min_alloc_size) { min_alloc_size = cct->_conf->bluestore_min_alloc_size; @@ -11857,34 +11892,6 @@ uint64_t BlueStore::_zoned_piggyback_device_parameters_onto(uint64_t min_alloc_s return min_alloc_size; } -int BlueStore::_zoned_check_config_settings() { - if (cct->_conf->bluestore_allocator != "zoned") { - dout(1) << __func__ << " The drive is HM-SMR but " - << cct->_conf->bluestore_allocator << " allocator is specified. " - << "Only zoned allocator can be used with HM-SMR drive." << dendl; - return -EINVAL; - } - - // At least for now we want to use large min_alloc_size with HM-SMR drives. - // Populating used_blocks bitset on a debug build of ceph-osd takes about 5 - // minutes with a 14 TB HM-SMR drive and 4 KiB min_alloc_size. - if (min_alloc_size < 64 * 1024) { - dout(1) << __func__ << " The drive is HM-SMR but min_alloc_size is " - << min_alloc_size << ". " - << "Please set to at least 64 KiB." << dendl; - return -EINVAL; - } - - // We don't want to defer writes with HM-SMR because it violates sequential - // write requirement. - if (prefer_deferred_size) { - dout(1) << __func__ << " The drive is HM-SMR but prefer_deferred_size is " - << prefer_deferred_size << ". " - << "Please set to 0." << dendl; - return -EINVAL; - } - return 0; -} #endif void BlueStore::_txc_finalize_kv(TransContext *txc, KeyValueDB::Transaction t) diff --git a/src/os/bluestore/BlueStore.h b/src/os/bluestore/BlueStore.h index f575e524a8f59..4a4171f41b081 100644 --- a/src/os/bluestore/BlueStore.h +++ b/src/os/bluestore/BlueStore.h @@ -2420,7 +2420,6 @@ class BlueStore : public ObjectStore, #ifdef HAVE_LIBZBD // Functions related to zoned storage. uint64_t _zoned_piggyback_device_parameters_onto(uint64_t min_alloc_size); - int _zoned_check_config_settings(); void _zoned_update_cleaning_metadata(TransContext *txc); std::string _zoned_key(uint64_t offset, const ghobject_t *oid); #endif From b185fb2b69f995720d52ada14fc1109672ee04a7 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Wed, 11 Aug 2021 11:48:45 -0500 Subject: [PATCH 0382/2866] os/bluestore: adjust allocator+freelist interfaces for smr params Instead of shoehorning these fields into alloc_size, adjust the inferfaces to explicitly pass in zone_size and first_sequential_zone for Allocator and FreelistManager. Signed-off-by: Sage Weil --- src/os/bluestore/Allocator.cc | 16 +++- src/os/bluestore/Allocator.h | 11 ++- src/os/bluestore/BitmapFreelistManager.cc | 1 + src/os/bluestore/BitmapFreelistManager.h | 1 + src/os/bluestore/BlueFS.cc | 4 +- src/os/bluestore/BlueStore.cc | 77 ++++++++++++++----- src/os/bluestore/BlueStore.h | 5 +- src/os/bluestore/FreelistManager.h | 1 + src/os/bluestore/ZonedAllocator.cc | 13 ++-- src/os/bluestore/ZonedAllocator.h | 4 +- src/os/bluestore/ZonedFreelistManager.cc | 12 ++- src/os/bluestore/ZonedFreelistManager.h | 2 + src/test/objectstore/Allocator_test.cc | 3 +- src/test/objectstore/allocator_replay_test.cc | 2 +- 14 files changed, 107 insertions(+), 45 deletions(-) diff --git a/src/os/bluestore/Allocator.cc b/src/os/bluestore/Allocator.cc index 8e4a08b2587ab..731ae5de73c5b 100644 --- a/src/os/bluestore/Allocator.cc +++ b/src/os/bluestore/Allocator.cc @@ -109,7 +109,8 @@ class Allocator::SocketHook : public AdminSocketHook { Allocator::Allocator(std::string_view name, int64_t _capacity, int64_t _block_size) - : device_size(_capacity), block_size(_block_size) + : device_size(_capacity), + block_size(_block_size) { asok_hook = new SocketHook(this, name); } @@ -124,8 +125,14 @@ const string& Allocator::get_name() const { return asok_hook->name; } -Allocator *Allocator::create(CephContext* cct, std::string_view type, - int64_t size, int64_t block_size, std::string_view name) +Allocator *Allocator::create( + CephContext* cct, + std::string_view type, + int64_t size, + int64_t block_size, + int64_t zone_size, + int64_t first_sequential_zone, + std::string_view name) { Allocator* alloc = nullptr; if (type == "stupid") { @@ -142,7 +149,8 @@ Allocator *Allocator::create(CephContext* cct, std::string_view type, name); #ifdef HAVE_LIBZBD } else if (type == "zoned") { - return new ZonedAllocator(cct, size, block_size, name); + return new ZonedAllocator(cct, size, block_size, zone_size, first_sequential_zone, + name); #endif } if (alloc == nullptr) { diff --git a/src/os/bluestore/Allocator.h b/src/os/bluestore/Allocator.h index 6f6325d57af9f..5503ed213fb58 100644 --- a/src/os/bluestore/Allocator.h +++ b/src/os/bluestore/Allocator.h @@ -66,8 +66,15 @@ class Allocator { virtual double get_fragmentation_score(); virtual void shutdown() = 0; - static Allocator *create(CephContext* cct, std::string_view type, int64_t size, - int64_t block_size, const std::string_view name = ""); + static Allocator *create( + CephContext* cct, + std::string_view type, + int64_t size, + int64_t block_size, + int64_t zone_size = 0, + int64_t firs_sequential_zone = 0, + const std::string_view name = "" + ); const std::string& get_name() const; diff --git a/src/os/bluestore/BitmapFreelistManager.cc b/src/os/bluestore/BitmapFreelistManager.cc index 2c89712962684..e03a6ecacb748 100644 --- a/src/os/bluestore/BitmapFreelistManager.cc +++ b/src/os/bluestore/BitmapFreelistManager.cc @@ -65,6 +65,7 @@ BitmapFreelistManager::BitmapFreelistManager(CephContext* cct, } int BitmapFreelistManager::create(uint64_t new_size, uint64_t granularity, + uint64_t zone_size, uint64_t first_sequential_zone, KeyValueDB::Transaction txn) { bytes_per_block = granularity; diff --git a/src/os/bluestore/BitmapFreelistManager.h b/src/os/bluestore/BitmapFreelistManager.h index 5b04e8fd28cc0..8e4ea8fd385c4 100644 --- a/src/os/bluestore/BitmapFreelistManager.h +++ b/src/os/bluestore/BitmapFreelistManager.h @@ -63,6 +63,7 @@ class BitmapFreelistManager : public FreelistManager { static void setup_merge_operator(KeyValueDB *db, std::string prefix); int create(uint64_t size, uint64_t granularity, + uint64_t zone_size, uint64_t first_sequential_zone, KeyValueDB::Transaction txn) override; int init(KeyValueDB *kvdb, bool db_in_read_only, diff --git a/src/os/bluestore/BlueFS.cc b/src/os/bluestore/BlueFS.cc index f34996be3e000..5b00c2e9a9121 100644 --- a/src/os/bluestore/BlueFS.cc +++ b/src/os/bluestore/BlueFS.cc @@ -590,7 +590,9 @@ void BlueFS::_init_alloc() << std::dec << dendl; alloc[id] = Allocator::create(cct, cct->_conf->bluefs_allocator, bdev[id]->get_size(), - alloc_size[id], name); + alloc_size[id], + 0, 0, + name); alloc[id]->init_add_free( block_reserved[id], _get_total(id)); diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc index 030c1f59429f4..dd4d8f42d596e 100644 --- a/src/os/bluestore/BlueStore.cc +++ b/src/os/bluestore/BlueStore.cc @@ -5397,7 +5397,6 @@ int BlueStore::_open_fm(KeyValueDB::Transaction t, bool read_only, bool fm_resto << dendl; return -EINVAL; } - alloc_size = _zoned_piggyback_device_parameters_onto(alloc_size); } else #endif if (freelist_type == "zoned") { @@ -5406,7 +5405,13 @@ int BlueStore::_open_fm(KeyValueDB::Transaction t, bool read_only, bool fm_resto return -EINVAL; } - fm->create(bdev->get_size(), alloc_size, t); + fm->create(bdev->get_size(), alloc_size, + zone_size, first_sequential_zone, + t); + + // allocate superblock reserved space. note that we do not mark + // bluefs space as allocated in the freelist; we instead rely on + // bluefs doing that itself. auto reserved = _get_ondisk_reserved(); if (fm_restore) { // we need to allocate the full space in restore case @@ -5545,14 +5550,17 @@ int BlueStore::_create_alloc() << "Please set to 0." << dendl; return -EINVAL; } - - alloc_size = _zoned_piggyback_device_parameters_onto(alloc_size); } #endif - shared_alloc.set(Allocator::create(cct, allocator_type, - bdev->get_size(), - alloc_size, "block")); + shared_alloc.set( + Allocator::create( + cct, allocator_type, + bdev->get_size(), + alloc_size, + zone_size, + first_sequential_zone, + "block")); if (!shared_alloc.a) { lderr(cct) << __func__ << " failed to create " << allocator_type << " allocator" @@ -6700,6 +6708,8 @@ int BlueStore::mkfs() #ifdef HAVE_LIBZBD if (bdev->is_smr()) { freelist_type = "zoned"; + zone_size = bdev->get_zone_size(); + first_sequential_zone = bdev->get_conventional_region_size() / zone_size; } else #endif { @@ -6765,6 +6775,22 @@ int BlueStore::mkfs() bl.append(stringify(OMAP_PER_PG)); t->set(PREFIX_SUPER, "per_pool_omap", bl); } + +#ifdef HAVE_LIBZBD + if (bdev->is_smr()) { + { + bufferlist bl; + encode((uint64_t)zone_size, bl); + t->set(PREFIX_SUPER, "zone_size", bl); + } + { + bufferlist bl; + encode((uint64_t)first_sequential_zone, bl); + t->set(PREFIX_SUPER, "first_sequential_zone", bl); + } + } +#endif + ondisk_format = latest_ondisk_format; _prepare_ondisk_format_super(t); db->submit_transaction_sync(t); @@ -11467,6 +11493,27 @@ int BlueStore::_open_super_meta() << std::dec << dendl; } + // smr fields + { + bufferlist bl; + int r = db->get(PREFIX_SUPER, "zone_size", &bl); + if (r >= 0) { + auto p = bl.cbegin(); + decode(zone_size, p); + dout(1) << __func__ << " zone_size 0x" << std::hex << zone_size << std::dec << dendl; + } + } + { + bufferlist bl; + int r = db->get(PREFIX_SUPER, "first_sequential_zone", &bl); + if (r >= 0) { + auto p = bl.cbegin(); + decode(first_sequential_zone, p); + dout(1) << __func__ << " first_sequential_zone 0x" << std::hex + << first_sequential_zone << std::dec << dendl; + } + } + _set_per_pool_omap(); _open_statfs(); @@ -11880,18 +11927,6 @@ std::string BlueStore::_zoned_key(uint64_t offset, const ghobject_t *oid) { return zone_key + object_key; } -// For now, to avoid interface changes we piggyback zone_size (in MiB) and the -// first sequential zone number onto min_alloc_size and pass it to functions -// Allocator::create and FreelistManager::create. -uint64_t BlueStore::_zoned_piggyback_device_parameters_onto(uint64_t min_alloc_size) { - uint64_t zone_size = bdev->get_zone_size(); - uint64_t zone_size_mb = zone_size / (1024 * 1024); - uint64_t first_seq_zone = bdev->get_conventional_region_size() / zone_size; - min_alloc_size |= (zone_size_mb << 32); - min_alloc_size |= (first_seq_zone << 48); - return min_alloc_size; -} - #endif void BlueStore::_txc_finalize_kv(TransContext *txc, KeyValueDB::Transaction t) @@ -17349,7 +17384,9 @@ int BlueStore::store_allocator(Allocator* src_allocator) Allocator* BlueStore::create_bitmap_allocator(uint64_t bdev_size) { // create allocator uint64_t alloc_size = min_alloc_size; - Allocator* alloc = Allocator::create(cct, "bitmap", bdev_size, alloc_size, "recovery"); + Allocator* alloc = Allocator::create(cct, "bitmap", bdev_size, alloc_size, + zone_size, first_sequential_zone, + "recovery"); if (alloc) { return alloc; } else { diff --git a/src/os/bluestore/BlueStore.h b/src/os/bluestore/BlueStore.h index 4a4171f41b081..c932ca36b528d 100644 --- a/src/os/bluestore/BlueStore.h +++ b/src/os/bluestore/BlueStore.h @@ -2149,6 +2149,10 @@ class BlueStore : public ObjectStore, std::numeric_limits::digits, "not enough bits for min_alloc_size"); + // smr-only + uint64_t zone_size = 0; ///< number of SMR zones + uint64_t first_sequential_zone = 0; ///< first SMR zone that is sequential-only + enum { // Please preserve the order since it's DB persistent OMAP_BULK = 0, @@ -2419,7 +2423,6 @@ class BlueStore : public ObjectStore, #ifdef HAVE_LIBZBD // Functions related to zoned storage. - uint64_t _zoned_piggyback_device_parameters_onto(uint64_t min_alloc_size); void _zoned_update_cleaning_metadata(TransContext *txc); std::string _zoned_key(uint64_t offset, const ghobject_t *oid); #endif diff --git a/src/os/bluestore/FreelistManager.h b/src/os/bluestore/FreelistManager.h index 4d375b4309456..54d27f10804be 100644 --- a/src/os/bluestore/FreelistManager.h +++ b/src/os/bluestore/FreelistManager.h @@ -26,6 +26,7 @@ class FreelistManager { static void setup_merge_operators(KeyValueDB *db, const std::string &type); virtual int create(uint64_t size, uint64_t granularity, + uint64_t zone_size, uint64_t first_sequential_zone, KeyValueDB::Transaction txn) = 0; virtual int init(KeyValueDB *kvdb, bool db_in_read_only, diff --git a/src/os/bluestore/ZonedAllocator.cc b/src/os/bluestore/ZonedAllocator.cc index 6144779cb68e0..4a7a1f7ef5d4d 100644 --- a/src/os/bluestore/ZonedAllocator.cc +++ b/src/os/bluestore/ZonedAllocator.cc @@ -21,17 +21,16 @@ ZonedAllocator::ZonedAllocator(CephContext* cct, int64_t size, int64_t blk_size, + int64_t _zone_size, + int64_t _first_sequential_zone, std::string_view name) - : Allocator(name, size, blk_size & 0x00000000ffffffff), + : Allocator(name, size, blk_size), cct(cct), num_free(0), size(size), - // To avoid interface changes, we piggyback zone size and the first - // sequential zone number onto the first 32 bits of 64-bit |blk_size|. - // The last 32 bits of |blk_size| is holding the actual block size. - block_size((blk_size & 0x00000000ffffffff)), - zone_size(((blk_size & 0x0000ffff00000000) >> 32) * 1024 * 1024), - first_seq_zone_num((blk_size >> 48) & 0xffff), + block_size(blk_size), + zone_size(_zone_size), + first_seq_zone_num(_first_sequential_zone), starting_zone_num(first_seq_zone_num), num_zones(size / zone_size), num_zones_to_clean(0) { diff --git a/src/os/bluestore/ZonedAllocator.h b/src/os/bluestore/ZonedAllocator.h index 5deedcae9ab02..585e8943cec67 100644 --- a/src/os/bluestore/ZonedAllocator.h +++ b/src/os/bluestore/ZonedAllocator.h @@ -70,7 +70,9 @@ class ZonedAllocator : public Allocator { public: ZonedAllocator(CephContext* cct, int64_t size, int64_t block_size, - std::string_view name); + int64_t _zone_size, + int64_t _first_sequential_zone, + std::string_view name); ~ZonedAllocator() override; const char *get_type() const override { diff --git a/src/os/bluestore/ZonedFreelistManager.cc b/src/os/bluestore/ZonedFreelistManager.cc index 3b31e202fae68..60899939200e7 100644 --- a/src/os/bluestore/ZonedFreelistManager.cc +++ b/src/os/bluestore/ZonedFreelistManager.cc @@ -81,16 +81,14 @@ ZonedFreelistManager::ZonedFreelistManager( int ZonedFreelistManager::create( uint64_t new_size, uint64_t granularity, + uint64_t new_zone_size, + uint64_t first_sequential_zone, KeyValueDB::Transaction txn) { - // To avoid interface changes, we piggyback zone size and the first sequential - // zone number onto the first 32 bits of 64-bit |granularity|. The last 32 - // bits of |granularity| is holding the actual allocation granularity, which - // is bytes_per_block. size = new_size; - bytes_per_block = granularity & 0x00000000ffffffff; - zone_size = ((granularity & 0x0000ffff00000000) >> 32) * 1024 * 1024; + bytes_per_block = granularity; + zone_size = new_zone_size; num_zones = size / zone_size; - starting_zone_num = (granularity & 0xffff000000000000) >> 48; + starting_zone_num = first_sequential_zone; enumerate_zone_num = ~0UL; ceph_assert(size % zone_size == 0); diff --git a/src/os/bluestore/ZonedFreelistManager.h b/src/os/bluestore/ZonedFreelistManager.h index c6f9be3151e9e..7ad0723369857 100644 --- a/src/os/bluestore/ZonedFreelistManager.h +++ b/src/os/bluestore/ZonedFreelistManager.h @@ -63,6 +63,8 @@ class ZonedFreelistManager : public FreelistManager { int create(uint64_t size, uint64_t granularity, + uint64_t zone_size, + uint64_t first_sequential_zone, KeyValueDB::Transaction txn) override; int init(KeyValueDB *kvdb, diff --git a/src/test/objectstore/Allocator_test.cc b/src/test/objectstore/Allocator_test.cc index 40fbed12bd3a0..210bc6d9d7457 100644 --- a/src/test/objectstore/Allocator_test.cc +++ b/src/test/objectstore/Allocator_test.cc @@ -26,7 +26,8 @@ class AllocTest : public ::testing::TestWithParam { void init_alloc(int64_t size, uint64_t min_alloc_size) { std::cout << "Creating alloc type " << string(GetParam()) << " \n"; alloc.reset(Allocator::create(g_ceph_context, GetParam(), size, - min_alloc_size)); + min_alloc_size, + 256*1048576, 100*256*1048576ull)); } void init_close() { diff --git a/src/test/objectstore/allocator_replay_test.cc b/src/test/objectstore/allocator_replay_test.cc index 7927790c485f3..811cc92cdeaf4 100644 --- a/src/test/objectstore/allocator_replay_test.cc +++ b/src/test/objectstore/allocator_replay_test.cc @@ -274,7 +274,7 @@ int replay_free_dump_and_apply(char* fname, unique_ptr alloc; alloc.reset(Allocator::create(g_ceph_context, alloc_type, - capacity, alloc_unit, alloc_name)); + capacity, alloc_unit, 0, 0, alloc_name)); auto it = o->find_first(); while (!it.end()) { From d67a885d8fff50dbe364eaf132eaaf4bb4854707 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Fri, 3 Sep 2021 15:53:30 -0500 Subject: [PATCH 0383/2866] blk/zoned: print error during init Otherwise it is easy to miss things like EPERM during testing. Signed-off-by: Sage Weil --- src/blk/zoned/HMSMRDevice.cc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/blk/zoned/HMSMRDevice.cc b/src/blk/zoned/HMSMRDevice.cc index 0ff2f70af3d39..6459e80a54f1a 100644 --- a/src/blk/zoned/HMSMRDevice.cc +++ b/src/blk/zoned/HMSMRDevice.cc @@ -101,17 +101,22 @@ bool HMSMRDevice::set_smr_params(const std::string& path) { int dev = zbd_open(path.c_str(), O_RDWR | O_DIRECT | O_LARGEFILE, nullptr); if (dev < 0) { + derr << __func__ << " zbd_open failed on " << path << ": " + << cpp_strerror(errno) << dendl; return false; } auto close_dev = make_scope_guard([dev] { zbd_close(dev); }); unsigned int nr_zones = 0; if (zbd_report_nr_zones(dev, 0, 0, ZBD_RO_NOT_WP, &nr_zones) != 0) { + derr << __func__ << " zbd_report_nr_zones failed on " << path << ": " + << cpp_strerror(errno) << dendl; return false; } std::vector zones(nr_zones); if (zbd_report_zones(dev, 0, 0, ZBD_RO_NOT_WP, zones.data(), &nr_zones) != 0) { + derr << __func__ << " zbd_report_zones failed on " << path << dendl; return false; } From 8be98e24f610c318936c0058ef31f124996525d9 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Fri, 3 Sep 2021 15:53:50 -0500 Subject: [PATCH 0384/2866] blk/zoned: add reset_all_zones() Signed-off-by: Sage Weil --- src/blk/BlockDevice.h | 1 + src/blk/zoned/HMSMRDevice.cc | 5 +++++ src/blk/zoned/HMSMRDevice.h | 1 + 3 files changed, 7 insertions(+) diff --git a/src/blk/BlockDevice.h b/src/blk/BlockDevice.h index bb49a747e1915..d08e6e3e48668 100644 --- a/src/blk/BlockDevice.h +++ b/src/blk/BlockDevice.h @@ -205,6 +205,7 @@ class BlockDevice { ceph_assert(is_smr()); return conventional_region_size; } + virtual void reset_all_zones() {} virtual void reset_zones(const std::set& zones) { ceph_assert(is_smr()); } diff --git a/src/blk/zoned/HMSMRDevice.cc b/src/blk/zoned/HMSMRDevice.cc index 6459e80a54f1a..c94e89588c241 100644 --- a/src/blk/zoned/HMSMRDevice.cc +++ b/src/blk/zoned/HMSMRDevice.cc @@ -419,6 +419,11 @@ void HMSMRDevice::_detect_vdo() return; } +void HMSMRDevice::reset_all_zones() +{ + zbd_reset_zones(zbd_fd, conventional_region_size, 0); +} + void HMSMRDevice::reset_zones(const std::set& zones) { for (auto zone_num : zones) { if (zbd_reset_zones(zbd_fd, zone_num * zone_size, zone_size) != 0) { diff --git a/src/blk/zoned/HMSMRDevice.h b/src/blk/zoned/HMSMRDevice.h index ea0bf31dbde05..356affdc11e73 100644 --- a/src/blk/zoned/HMSMRDevice.h +++ b/src/blk/zoned/HMSMRDevice.h @@ -136,6 +136,7 @@ class HMSMRDevice final : public BlockDevice { bool is_smr() const final { return true; } + void reset_all_zones() override; void reset_zones(const std::set& zones) override; bool get_thin_utilization(uint64_t *total, uint64_t *avail) const final; From 3f1b065454c0175b946762a6d9ffd3f1bfabdb72 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Fri, 3 Sep 2021 16:22:30 -0500 Subject: [PATCH 0385/2866] blk/zoned: remove dead VDO code VDO won't work on an SMR device Signed-off-by: Sage Weil --- src/blk/zoned/HMSMRDevice.cc | 29 +---------------------------- src/blk/zoned/HMSMRDevice.h | 4 ---- 2 files changed, 1 insertion(+), 32 deletions(-) diff --git a/src/blk/zoned/HMSMRDevice.cc b/src/blk/zoned/HMSMRDevice.cc index c94e89588c241..f16ab1be69417 100644 --- a/src/blk/zoned/HMSMRDevice.cc +++ b/src/blk/zoned/HMSMRDevice.cc @@ -249,7 +249,6 @@ int HMSMRDevice::open(const string& p) rotational = blkdev_buffered.is_rotational(); support_discard = blkdev_buffered.support_discard(); this->devname = devname; - _detect_vdo(); } } @@ -306,11 +305,6 @@ void HMSMRDevice::close() _aio_stop(); _discard_stop(); - if (vdo_fd >= 0) { - VOID_TEMP_FAILURE_RETRY(::close(vdo_fd)); - vdo_fd = -1; - } - for (int i = 0; i < WRITE_LIFE_MAX; i++) { assert(fd_directs[i] >= 0); VOID_TEMP_FAILURE_RETRY(::close(fd_directs[i])); @@ -335,12 +329,6 @@ int HMSMRDevice::collect_metadata(const string& prefix, map *pm) } else { (*pm)[prefix + "type"] = "ssd"; } - if (vdo_fd >= 0) { - (*pm)[prefix + "vdo"] = "true"; - uint64_t total, avail; - get_vdo_utilization(vdo_fd, &total, &avail); - (*pm)[prefix + "vdo_physical_size"] = stringify(total); - } { string res_names; @@ -407,18 +395,6 @@ int HMSMRDevice::collect_metadata(const string& prefix, map *pm) return 0; } -void HMSMRDevice::_detect_vdo() -{ - vdo_fd = get_vdo_stats_handle(devname.c_str(), &vdo_name); - if (vdo_fd >= 0) { - dout(1) << __func__ << " VDO volume " << vdo_name - << " maps to " << devname << dendl; - } else { - dout(20) << __func__ << " no VDO volume maps to " << devname << dendl; - } - return; -} - void HMSMRDevice::reset_all_zones() { zbd_reset_zones(zbd_fd, conventional_region_size, 0); @@ -434,10 +410,7 @@ void HMSMRDevice::reset_zones(const std::set& zones) { bool HMSMRDevice::get_thin_utilization(uint64_t *total, uint64_t *avail) const { - if (vdo_fd < 0) { - return false; - } - return get_vdo_utilization(vdo_fd, total, avail); + return false; } int HMSMRDevice::choose_fd(bool buffered, int write_hint) const diff --git a/src/blk/zoned/HMSMRDevice.h b/src/blk/zoned/HMSMRDevice.h index 356affdc11e73..f3cd98f391fc0 100644 --- a/src/blk/zoned/HMSMRDevice.h +++ b/src/blk/zoned/HMSMRDevice.h @@ -37,9 +37,6 @@ class HMSMRDevice final : public BlockDevice { std::string path; bool aio, dio; - int vdo_fd = -1; ///< fd for vdo sysfs directory - std::string vdo_name; - std::string devname; ///< kernel dev name (/sys/block/$devname), if any int zbd_fd = -1; ///< fd for the zoned block device @@ -110,7 +107,6 @@ class HMSMRDevice final : public BlockDevice { void debug_aio_link(aio_t& aio); void debug_aio_unlink(aio_t& aio); - void _detect_vdo(); int choose_fd(bool buffered, int write_hint) const; bool set_smr_params(const std::string& path); From e8f27a6d3d470a06bc4efeac1a3d2792bd5e4407 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Fri, 3 Sep 2021 15:00:03 -0500 Subject: [PATCH 0386/2866] os/bluestore/ZonedAllocator: whitespace Signed-off-by: Sage Weil --- src/os/bluestore/ZonedAllocator.cc | 46 ++++++++++++++++++++---------- 1 file changed, 31 insertions(+), 15 deletions(-) diff --git a/src/os/bluestore/ZonedAllocator.cc b/src/os/bluestore/ZonedAllocator.cc index 4a7a1f7ef5d4d..ed74ff9b5b0e0 100644 --- a/src/os/bluestore/ZonedAllocator.cc +++ b/src/os/bluestore/ZonedAllocator.cc @@ -33,7 +33,8 @@ ZonedAllocator::ZonedAllocator(CephContext* cct, first_seq_zone_num(_first_sequential_zone), starting_zone_num(first_seq_zone_num), num_zones(size / zone_size), - num_zones_to_clean(0) { + num_zones_to_clean(0) +{ ldout(cct, 10) << __func__ << " size 0x" << std::hex << size << " zone size 0x" << zone_size << std::dec << " number of zones " << num_zones @@ -42,14 +43,17 @@ ZonedAllocator::ZonedAllocator(CephContext* cct, ceph_assert(size % zone_size == 0); } -ZonedAllocator::~ZonedAllocator() {} +ZonedAllocator::~ZonedAllocator() +{ +} int64_t ZonedAllocator::allocate( uint64_t want_size, uint64_t alloc_unit, uint64_t max_alloc_size, int64_t hint, - PExtentVector *extents) { + PExtentVector *extents) +{ std::lock_guard l(lock); ceph_assert(want_size % 4096 == 0); @@ -107,7 +111,8 @@ int64_t ZonedAllocator::allocate( return want_size; } -void ZonedAllocator::release(const interval_set& release_set) { +void ZonedAllocator::release(const interval_set& release_set) +{ std::lock_guard l(lock); for (auto p = cbegin(release_set); p != cend(release_set); ++p) { auto offset = p.get_start(); @@ -123,29 +128,34 @@ void ZonedAllocator::release(const interval_set& release_set) { } } -uint64_t ZonedAllocator::get_free() { +uint64_t ZonedAllocator::get_free() +{ return num_free; } -void ZonedAllocator::dump() { +void ZonedAllocator::dump() +{ std::lock_guard l(lock); } void ZonedAllocator::dump(std::function notify) { + uint64_t length)> notify) +{ std::lock_guard l(lock); } // This just increments |num_free|. The actual free space is added by // init_alloc, as it updates the write pointer for each zone. -void ZonedAllocator::init_add_free(uint64_t offset, uint64_t length) { +void ZonedAllocator::init_add_free(uint64_t offset, uint64_t length) +{ ldout(cct, 40) << __func__ << " " << std::hex << offset << "~" << length << dendl; num_free += length; } -void ZonedAllocator::init_rm_free(uint64_t offset, uint64_t length) { +void ZonedAllocator::init_rm_free(uint64_t offset, uint64_t length) +{ std::lock_guard l(lock); ldout(cct, 40) << __func__ << " 0x" << std::hex << offset << "~" << length << dendl; @@ -174,12 +184,14 @@ void ZonedAllocator::init_rm_free(uint64_t offset, uint64_t length) { } } -const std::set *ZonedAllocator::get_zones_to_clean(void) { +const std::set *ZonedAllocator::get_zones_to_clean(void) +{ ldout(cct, 10) << __func__ << dendl; return num_zones_to_clean ? &zones_to_clean : nullptr; } -bool ZonedAllocator::low_on_space(void) { +bool ZonedAllocator::low_on_space(void) +{ ceph_assert(zones_to_clean.empty()); uint64_t conventional_size = first_seq_zone_num * zone_size; @@ -195,7 +207,8 @@ bool ZonedAllocator::low_on_space(void) { return free_ratio <= 0.25; } -void ZonedAllocator::find_zones_to_clean(void) { +void ZonedAllocator::find_zones_to_clean(void) +{ ldout(cct, 40) << __func__ << dendl; if (num_zones_to_clean || !low_on_space()) @@ -239,7 +252,8 @@ void ZonedAllocator::find_zones_to_clean(void) { void ZonedAllocator::init_alloc(std::vector &&_zone_states, ceph::mutex *_cleaner_lock, - ceph::condition_variable *_cleaner_cond) { + ceph::condition_variable *_cleaner_cond) +{ std::lock_guard l(lock); cleaner_lock = _cleaner_lock; cleaner_cond = _cleaner_cond; @@ -247,7 +261,8 @@ void ZonedAllocator::init_alloc(std::vector &&_zone_states, zone_states = std::move(_zone_states); } -void ZonedAllocator::mark_zones_to_clean_free(void) { +void ZonedAllocator::mark_zones_to_clean_free(void) +{ std::lock_guard l(lock); ldout(cct, 10) << __func__ << dendl; for (auto zone_num : zones_to_clean) { @@ -260,6 +275,7 @@ void ZonedAllocator::mark_zones_to_clean_free(void) { num_zones_to_clean = 0; } -void ZonedAllocator::shutdown() { +void ZonedAllocator::shutdown() +{ ldout(cct, 1) << __func__ << dendl; } From 2f4cc1255912d9a9b5cc4f779fe80009a4d406b5 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Fri, 3 Sep 2021 15:52:25 -0500 Subject: [PATCH 0387/2866] os/bluestore/ZonedAllocator: be consistent with hex debug output Signed-off-by: Sage Weil --- src/os/bluestore/ZonedAllocator.cc | 63 ++++++++++++++++-------------- 1 file changed, 34 insertions(+), 29 deletions(-) diff --git a/src/os/bluestore/ZonedAllocator.cc b/src/os/bluestore/ZonedAllocator.cc index ed74ff9b5b0e0..6809f8023ee4f 100644 --- a/src/os/bluestore/ZonedAllocator.cc +++ b/src/os/bluestore/ZonedAllocator.cc @@ -39,6 +39,7 @@ ZonedAllocator::ZonedAllocator(CephContext* cct, << " zone size 0x" << zone_size << std::dec << " number of zones " << num_zones << " first sequential zone " << starting_zone_num + << std::dec << dendl; ceph_assert(size % zone_size == 0); } @@ -58,25 +59,26 @@ int64_t ZonedAllocator::allocate( ceph_assert(want_size % 4096 == 0); - ldout(cct, 10) << __func__ << " trying to allocate " - << std::hex << want_size << dendl; + ldout(cct, 10) << __func__ << " trying to allocate 0x" + << std::hex << want_size << std::dec << dendl; uint64_t zone_num = starting_zone_num; auto p = zones_to_clean.lower_bound(zone_num); for ( ; zone_num < num_zones; ++zone_num) { if (p != zones_to_clean.cend() && *p == zone_num) { - ldout(cct, 10) << __func__ << " skipping zone " << zone_num - << " because it is being cleaned" << dendl; + ldout(cct, 10) << __func__ << " skipping zone 0x" << std::hex << zone_num + << " because it is being cleaned" << std::dec << dendl; ++p; continue; } if (fits(want_size, zone_num)) { break; } - ldout(cct, 10) << __func__ << " skipping zone " << zone_num + ldout(cct, 10) << __func__ << " skipping zone 0x" << std::hex << zone_num << " because there is not enough space: " - << " want_size = " << want_size - << " available = " << get_remaining_space(zone_num) + << " want_size = 0x" << want_size + << " available = 0x" << get_remaining_space(zone_num) + << std::dec << dendl; } @@ -87,9 +89,10 @@ int64_t ZonedAllocator::allocate( uint64_t offset = get_offset(zone_num); - ldout(cct, 10) << __func__ << " incrementing zone " << std::hex - << zone_num << " write pointer from " << offset - << " to " << offset + want_size << dendl; + ldout(cct, 10) << __func__ << " incrementing zone 0x" << std::hex + << zone_num << " write pointer from 0x" << offset + << " to 0x" << offset + want_size + << std::dec << dendl; increment_write_pointer(zone_num, want_size); num_free -= want_size; @@ -97,13 +100,10 @@ int64_t ZonedAllocator::allocate( starting_zone_num = zone_num + 1; } - ldout(cct, 10) << __func__ << std::hex << " zone " << zone_num - << " offset is now " << get_write_pointer(zone_num) << dendl; - - ldout(cct, 10) << __func__ << " allocated " << std::hex << want_size - << " bytes at offset " << offset - << " located at zone " << zone_num - << " and zone offset " << offset % zone_size << dendl; + ldout(cct, 10) << " allocated 0x" << std::hex << offset << "~" << want_size + << " from zone 0x" << zone_num + << " and zone offset 0x" << (offset % zone_size) + << std::dec << dendl; find_zones_to_clean(); @@ -117,7 +117,8 @@ void ZonedAllocator::release(const interval_set& release_set) for (auto p = cbegin(release_set); p != cend(release_set); ++p) { auto offset = p.get_start(); auto length = p.get_len(); - ldout(cct, 10) << __func__ << " 0x" << std::hex << offset << "~" << length << dendl; + ldout(cct, 10) << __func__ << " 0x" << std::hex << offset << "~" << length + << std::dec << dendl; uint64_t zone_num = offset / zone_size; uint64_t num_dead = std::min(zone_size - offset % zone_size, length); for ( ; length; ++zone_num) { @@ -149,7 +150,7 @@ void ZonedAllocator::dump(std::function= 0); @@ -172,7 +173,7 @@ void ZonedAllocator::init_rm_free(uint64_t offset, uint64_t length) increment_write_pointer(zone_num, remaining_space); ldout(cct, 40) << __func__ << " set zone 0x" << std::hex - << zone_num << " write pointer to 0x" << zone_size << dendl; + << zone_num << " write pointer to 0x" << zone_size << std::dec << dendl; length -= remaining_space; ceph_assert(length % zone_size == 0); @@ -180,7 +181,7 @@ void ZonedAllocator::init_rm_free(uint64_t offset, uint64_t length) for ( ; length; length -= zone_size) { increment_write_pointer(++zone_num, zone_size); ldout(cct, 40) << __func__ << " set zone 0x" << std::hex - << zone_num << " write pointer to 0x" << zone_size << dendl; + << zone_num << " write pointer to 0x" << zone_size << std::dec << dendl; } } @@ -199,9 +200,9 @@ bool ZonedAllocator::low_on_space(void) uint64_t sequential_num_free = num_free - conventional_size; double free_ratio = static_cast(sequential_num_free) / sequential_size; - ldout(cct, 10) << __func__ << " free size " << sequential_num_free - << " total size " << sequential_size - << " free ratio is " << free_ratio << dendl; + ldout(cct, 10) << __func__ << " free size 0x" << std::hex << sequential_num_free + << " total size 0x" << sequential_size << std::dec + << ", free ratio is " << free_ratio << dendl; // TODO: make 0.25 tunable return free_ratio <= 0.25; @@ -225,7 +226,8 @@ void ZonedAllocator::find_zones_to_clean(void) if (cct->_conf->subsys.should_gather()) { for (size_t i = 0; i < zone_states.size(); ++i) { - dout(40) << __func__ << " zone " << i << zone_states[i] << dendl; + dout(40) << __func__ << " zone 0x" << std::hex << i << std::dec << " " + << zone_states[i] << dendl; } } @@ -234,8 +236,10 @@ void ZonedAllocator::find_zones_to_clean(void) return zone_states[i1].num_dead_bytes > zone_states[i2].num_dead_bytes; }); - ldout(cct, 10) << __func__ << " the zone that needs cleaning is " - << *idx.begin() << " num_dead_bytes = " << zone_states[*idx.begin()].num_dead_bytes + ldout(cct, 10) << __func__ << " the zone that needs cleaning is 0x" + << std::hex << *idx.begin() << " num_dead_bytes = 0x" + << zone_states[*idx.begin()].num_dead_bytes + << std::dec << dendl; zones_to_clean = {idx.begin(), idx.begin() + num_zones_to_clean_at_once}; @@ -266,7 +270,8 @@ void ZonedAllocator::mark_zones_to_clean_free(void) std::lock_guard l(lock); ldout(cct, 10) << __func__ << dendl; for (auto zone_num : zones_to_clean) { - ldout(cct, 10) << __func__ << " zone " << zone_num << " is now clean" << dendl; + ldout(cct, 10) << __func__ << " zone 0x" << std::hex << zone_num + << " is now clean" << std::dec << dendl; num_free += zone_states[zone_num].write_pointer; zone_states[zone_num].num_dead_bytes = 0; zone_states[zone_num].write_pointer = 0; From 63333a109c03ffdcdbd5421ba9152451d106b03b Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Fri, 3 Sep 2021 15:56:38 -0500 Subject: [PATCH 0388/2866] os/bluestore/ZonedAllocator: simplify debug output prefix Signed-off-by: Sage Weil --- src/os/bluestore/ZonedAllocator.cc | 45 +++++++++++++++--------------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/src/os/bluestore/ZonedAllocator.cc b/src/os/bluestore/ZonedAllocator.cc index 6809f8023ee4f..05c9f3bf59ae7 100644 --- a/src/os/bluestore/ZonedAllocator.cc +++ b/src/os/bluestore/ZonedAllocator.cc @@ -16,7 +16,7 @@ #define dout_context cct #define dout_subsys ceph_subsys_bluestore #undef dout_prefix -#define dout_prefix *_dout << "ZonedAllocator " << this << " " +#define dout_prefix *_dout << "ZonedAllocator(" << this << ") " << __func__ << " " ZonedAllocator::ZonedAllocator(CephContext* cct, int64_t size, @@ -35,7 +35,7 @@ ZonedAllocator::ZonedAllocator(CephContext* cct, num_zones(size / zone_size), num_zones_to_clean(0) { - ldout(cct, 10) << __func__ << " size 0x" << std::hex << size + ldout(cct, 10) << " size 0x" << std::hex << size << " zone size 0x" << zone_size << std::dec << " number of zones " << num_zones << " first sequential zone " << starting_zone_num @@ -59,14 +59,14 @@ int64_t ZonedAllocator::allocate( ceph_assert(want_size % 4096 == 0); - ldout(cct, 10) << __func__ << " trying to allocate 0x" + ldout(cct, 10) << " trying to allocate 0x" << std::hex << want_size << std::dec << dendl; uint64_t zone_num = starting_zone_num; auto p = zones_to_clean.lower_bound(zone_num); for ( ; zone_num < num_zones; ++zone_num) { if (p != zones_to_clean.cend() && *p == zone_num) { - ldout(cct, 10) << __func__ << " skipping zone 0x" << std::hex << zone_num + ldout(cct, 10) << " skipping zone 0x" << std::hex << zone_num << " because it is being cleaned" << std::dec << dendl; ++p; continue; @@ -74,7 +74,7 @@ int64_t ZonedAllocator::allocate( if (fits(want_size, zone_num)) { break; } - ldout(cct, 10) << __func__ << " skipping zone 0x" << std::hex << zone_num + ldout(cct, 10) << " skipping zone 0x" << std::hex << zone_num << " because there is not enough space: " << " want_size = 0x" << want_size << " available = 0x" << get_remaining_space(zone_num) @@ -83,13 +83,13 @@ int64_t ZonedAllocator::allocate( } if (zone_num == num_zones) { - ldout(cct, 10) << __func__ << " failed to allocate" << dendl; + ldout(cct, 10) << " failed to allocate" << dendl; return -ENOSPC; } uint64_t offset = get_offset(zone_num); - ldout(cct, 10) << __func__ << " incrementing zone 0x" << std::hex + ldout(cct, 10) << " incrementing zone 0x" << std::hex << zone_num << " write pointer from 0x" << offset << " to 0x" << offset + want_size << std::dec << dendl; @@ -117,9 +117,9 @@ void ZonedAllocator::release(const interval_set& release_set) for (auto p = cbegin(release_set); p != cend(release_set); ++p) { auto offset = p.get_start(); auto length = p.get_len(); - ldout(cct, 10) << __func__ << " 0x" << std::hex << offset << "~" << length - << std::dec << dendl; uint64_t zone_num = offset / zone_size; + ldout(cct, 10) << " 0x" << std::hex << offset << "~" << length + << " from zone 0x" << zone_num << std::dec << dendl; uint64_t num_dead = std::min(zone_size - offset % zone_size, length); for ( ; length; ++zone_num) { increment_num_dead_bytes(zone_num, num_dead); @@ -149,7 +149,7 @@ void ZonedAllocator::dump(std::function *ZonedAllocator::get_zones_to_clean(void) { - ldout(cct, 10) << __func__ << dendl; + ldout(cct, 10) << dendl; return num_zones_to_clean ? &zones_to_clean : nullptr; } @@ -200,7 +200,7 @@ bool ZonedAllocator::low_on_space(void) uint64_t sequential_num_free = num_free - conventional_size; double free_ratio = static_cast(sequential_num_free) / sequential_size; - ldout(cct, 10) << __func__ << " free size 0x" << std::hex << sequential_num_free + ldout(cct, 10) << " free size 0x" << std::hex << sequential_num_free << " total size 0x" << sequential_size << std::dec << ", free ratio is " << free_ratio << dendl; @@ -210,7 +210,7 @@ bool ZonedAllocator::low_on_space(void) void ZonedAllocator::find_zones_to_clean(void) { - ldout(cct, 40) << __func__ << dendl; + ldout(cct, 40) << dendl; if (num_zones_to_clean || !low_on_space()) return; @@ -226,7 +226,7 @@ void ZonedAllocator::find_zones_to_clean(void) if (cct->_conf->subsys.should_gather()) { for (size_t i = 0; i < zone_states.size(); ++i) { - dout(40) << __func__ << " zone 0x" << std::hex << i << std::dec << " " + dout(40) << " zone 0x" << std::hex << i << std::dec << " " << zone_states[i] << dendl; } } @@ -236,7 +236,7 @@ void ZonedAllocator::find_zones_to_clean(void) return zone_states[i1].num_dead_bytes > zone_states[i2].num_dead_bytes; }); - ldout(cct, 10) << __func__ << " the zone that needs cleaning is 0x" + ldout(cct, 10) << " the zone that needs cleaning is 0x" << std::hex << *idx.begin() << " num_dead_bytes = 0x" << zone_states[*idx.begin()].num_dead_bytes << std::dec @@ -258,19 +258,20 @@ void ZonedAllocator::init_alloc(std::vector &&_zone_states, ceph::mutex *_cleaner_lock, ceph::condition_variable *_cleaner_cond) { + lderr(cct) << dendl; std::lock_guard l(lock); cleaner_lock = _cleaner_lock; cleaner_cond = _cleaner_cond; - ldout(cct, 10) << __func__ << dendl; + ldout(cct, 10) << dendl; zone_states = std::move(_zone_states); } void ZonedAllocator::mark_zones_to_clean_free(void) { std::lock_guard l(lock); - ldout(cct, 10) << __func__ << dendl; + ldout(cct, 10) << dendl; for (auto zone_num : zones_to_clean) { - ldout(cct, 10) << __func__ << " zone 0x" << std::hex << zone_num + ldout(cct, 10) << " zone 0x" << std::hex << zone_num << " is now clean" << std::dec << dendl; num_free += zone_states[zone_num].write_pointer; zone_states[zone_num].num_dead_bytes = 0; @@ -282,5 +283,5 @@ void ZonedAllocator::mark_zones_to_clean_free(void) void ZonedAllocator::shutdown() { - ldout(cct, 1) << __func__ << dendl; + ldout(cct, 1) << dendl; } From 644461a800b4b0fd4ab2c4bb5183fc8bb87f8c67 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Fri, 3 Sep 2021 15:58:47 -0500 Subject: [PATCH 0389/2866] os/bluestore/ZonedAllocator: less verbose Signed-off-by: Sage Weil --- src/os/bluestore/ZonedAllocator.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/os/bluestore/ZonedAllocator.cc b/src/os/bluestore/ZonedAllocator.cc index 05c9f3bf59ae7..499cd72bf2f99 100644 --- a/src/os/bluestore/ZonedAllocator.cc +++ b/src/os/bluestore/ZonedAllocator.cc @@ -89,9 +89,9 @@ int64_t ZonedAllocator::allocate( uint64_t offset = get_offset(zone_num); - ldout(cct, 10) << " incrementing zone 0x" << std::hex + ldout(cct, 10) << " moving zone 0x" << std::hex << zone_num << " write pointer from 0x" << offset - << " to 0x" << offset + want_size + << " -> 0x" << offset + want_size << std::dec << dendl; increment_write_pointer(zone_num, want_size); From c002b24ca54b10743d16df4ac739eab973da2297 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Fri, 3 Sep 2021 11:34:06 -0500 Subject: [PATCH 0390/2866] os/bluestore: drop SMR 64K min_alloc_size restriction used_blocks is only for fsck. Signed-off-by: Sage Weil --- src/os/bluestore/BlueStore.cc | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc index dd4d8f42d596e..06adf8f55a9e3 100644 --- a/src/os/bluestore/BlueStore.cc +++ b/src/os/bluestore/BlueStore.cc @@ -5532,16 +5532,6 @@ int BlueStore::_create_alloc() if (freelist_type == "zoned") { allocator_type = "zoned"; - // At least for now we want to use large min_alloc_size with HM-SMR drives. - // Populating used_blocks bitset on a debug build of ceph-osd takes about 5 - // minutes with a 14 TB HM-SMR drive and 4 KiB min_alloc_size. - if (min_alloc_size < 64 * 1024) { - dout(1) << __func__ << " The drive is HM-SMR but min_alloc_size is " - << min_alloc_size << ". " - << "Please set to at least 64 KiB." << dendl; - return -EINVAL; - } - // We don't want to defer writes with HM-SMR because it violates sequential // write requirement. if (prefer_deferred_size) { From 6e3e53789927393f01d38eb61ce0db9672eb7174 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Fri, 3 Sep 2021 16:41:03 -0500 Subject: [PATCH 0391/2866] os/bluestore: force prefer_deferred_size=0 for smr Signed-off-by: Sage Weil --- src/os/bluestore/BlueStore.cc | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc index 06adf8f55a9e3..bd22830f7277e 100644 --- a/src/os/bluestore/BlueStore.cc +++ b/src/os/bluestore/BlueStore.cc @@ -5266,10 +5266,15 @@ void BlueStore::_set_alloc_sizes(void) { max_alloc_size = cct->_conf->bluestore_max_alloc_size; +#ifdef HAVE_LIBZBD + ceph_assert(bdev); + if (bdev->is_smr()) { + prefer_deferred_size = 0; + } else +#endif if (cct->_conf->bluestore_prefer_deferred_size) { prefer_deferred_size = cct->_conf->bluestore_prefer_deferred_size; } else { - ceph_assert(bdev); if (_use_rotational_settings()) { prefer_deferred_size = cct->_conf->bluestore_prefer_deferred_size_hdd; } else { @@ -5280,7 +5285,6 @@ void BlueStore::_set_alloc_sizes(void) if (cct->_conf->bluestore_deferred_batch_ops) { deferred_batch_ops = cct->_conf->bluestore_deferred_batch_ops; } else { - ceph_assert(bdev); if (_use_rotational_settings()) { deferred_batch_ops = cct->_conf->bluestore_deferred_batch_ops_hdd; } else { @@ -5531,15 +5535,6 @@ int BlueStore::_create_alloc() #ifdef HAVE_LIBZBD if (freelist_type == "zoned") { allocator_type = "zoned"; - - // We don't want to defer writes with HM-SMR because it violates sequential - // write requirement. - if (prefer_deferred_size) { - dout(1) << __func__ << " The drive is HM-SMR but prefer_deferred_size is " - << prefer_deferred_size << ". " - << "Please set to 0." << dendl; - return -EINVAL; - } } #endif From a491e9020c2fc9c2a751bdf022ae7b7d1d8646ba Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Fri, 3 Sep 2021 16:41:27 -0500 Subject: [PATCH 0392/2866] os/bluestore: properly reset zoned allocator on startup Signed-off-by: Sage Weil --- src/os/bluestore/BlueStore.cc | 1 + src/os/bluestore/ZonedAllocator.cc | 3 +++ 2 files changed, 4 insertions(+) diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc index bd22830f7277e..39c2865b34342 100644 --- a/src/os/bluestore/BlueStore.cc +++ b/src/os/bluestore/BlueStore.cc @@ -6695,6 +6695,7 @@ int BlueStore::mkfs() freelist_type = "zoned"; zone_size = bdev->get_zone_size(); first_sequential_zone = bdev->get_conventional_region_size() / zone_size; + bdev->reset_all_zones(); } else #endif { diff --git a/src/os/bluestore/ZonedAllocator.cc b/src/os/bluestore/ZonedAllocator.cc index 499cd72bf2f99..6425919639892 100644 --- a/src/os/bluestore/ZonedAllocator.cc +++ b/src/os/bluestore/ZonedAllocator.cc @@ -42,6 +42,9 @@ ZonedAllocator::ZonedAllocator(CephContext* cct, << std::dec << dendl; ceph_assert(size % zone_size == 0); + + zone_states.resize(num_zones); + num_free = num_zones * zone_size; } ZonedAllocator::~ZonedAllocator() From 3aee16f7a5c07faa3bc778cfe99f103d43f84bfd Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Fri, 3 Sep 2021 16:46:14 -0500 Subject: [PATCH 0393/2866] os/bluestore: fix/simplify zoned_cleaner thread start error handling Signed-off-by: Sage Weil --- src/os/bluestore/BlueStore.cc | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc index 39c2865b34342..b896519910759 100644 --- a/src/os/bluestore/BlueStore.cc +++ b/src/os/bluestore/BlueStore.cc @@ -7266,22 +7266,17 @@ int BlueStore::_mount() } }); + r = _deferred_replay(); + if (r < 0) { + return r; + } + #ifdef HAVE_LIBZBD - using scope_guard_t = scope_guard>; - std::optional stop_zoned_cleaner; if (bdev->is_smr()) { _zoned_cleaner_start(); - stop_zoned_cleaner = scope_guard_t([this] { - _zoned_cleaner_stop(); - }); } #endif - r = _deferred_replay(); - if (r < 0) { - return r; - } - mempool_thread.init(); if ((!per_pool_stat_collection || per_pool_omap != OMAP_PER_PG) && From ca905ab05add952b1de43362186545123d6a2150 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Mon, 6 Sep 2021 15:36:55 -0500 Subject: [PATCH 0394/2866] blk/zones: implement HMSMRDevice has KernelDevice child No need to duplicate so much code when we are just adding a few things. Also, we want to track KernelDevice changes/improvements. We could probably integrate these SMR capabilities directly into KernelDevice too... Signed-off-by: Sage Weil --- src/blk/kernel/KernelDevice.cc | 6 + src/blk/kernel/KernelDevice.h | 7 +- src/blk/zoned/HMSMRDevice.cc | 1172 ++------------------------------ src/blk/zoned/HMSMRDevice.h | 127 +--- 4 files changed, 57 insertions(+), 1255 deletions(-) diff --git a/src/blk/kernel/KernelDevice.cc b/src/blk/kernel/KernelDevice.cc index d8f65baa0aa32..f64f57c9cee36 100644 --- a/src/blk/kernel/KernelDevice.cc +++ b/src/blk/kernel/KernelDevice.cc @@ -238,6 +238,11 @@ int KernelDevice::open(const string& p) } } + r = _post_open(); + if (r < 0) { + goto out_fail; + } + r = _aio_start(); if (r < 0) { goto out_fail; @@ -290,6 +295,7 @@ void KernelDevice::close() dout(1) << __func__ << dendl; _aio_stop(); _discard_stop(); + _pre_close(); if (vdo_fd >= 0) { VOID_TEMP_FAILURE_RETRY(::close(vdo_fd)); diff --git a/src/blk/kernel/KernelDevice.h b/src/blk/kernel/KernelDevice.h index 9999182d0f8b3..050a8f879029a 100644 --- a/src/blk/kernel/KernelDevice.h +++ b/src/blk/kernel/KernelDevice.h @@ -28,9 +28,11 @@ #define RW_IO_MAX (INT_MAX & CEPH_PAGE_MASK) class KernelDevice : public BlockDevice { +protected: + std::string path; +private: std::vector fd_directs, fd_buffereds; bool enable_wrt = true; - std::string path; bool aio, dio; int vdo_fd = -1; ///< fd for vdo sysfs directory @@ -77,6 +79,9 @@ class KernelDevice : public BlockDevice { std::atomic_int injecting_crash; + virtual int _post_open() { return 0; } // hook for child implementations + virtual void _pre_close() { } // hook for child implementations + void _aio_thread(); void _discard_thread(); int queue_discard(interval_set &to_release) override; diff --git a/src/blk/zoned/HMSMRDevice.cc b/src/blk/zoned/HMSMRDevice.cc index f16ab1be69417..518ac41f4e0a8 100644 --- a/src/blk/zoned/HMSMRDevice.cc +++ b/src/blk/zoned/HMSMRDevice.cc @@ -13,32 +13,12 @@ * */ -#include -#include -#include -#include -#include -#include - #include "HMSMRDevice.h" -#include "include/intarith.h" -#include "include/types.h" -#include "include/compat.h" -#include "include/scope_guard.h" -#include "include/stringify.h" -#include "common/blkdev.h" -#include "common/errno.h" -#if defined(__FreeBSD__) -#include "bsm/audit_errno.h" -#endif -#include "common/debug.h" -#include "common/numa.h" - -#include "kernel/io_uring.h" - extern "C" { #include } +#include "common/debug.h" +#include "common/errno.h" #define dout_context cct #define dout_subsys ceph_subsys_bdev @@ -47,37 +27,13 @@ extern "C" { using namespace std; -HMSMRDevice::HMSMRDevice(CephContext* cct, aio_callback_t cb, void *cbpriv, aio_callback_t d_cb, void *d_cbpriv) - : BlockDevice(cct, cb, cbpriv), - aio(false), dio(false), - discard_callback(d_cb), - discard_callback_priv(d_cbpriv), - aio_stop(false), - discard_started(false), - discard_stop(false), - aio_thread(this), - discard_thread(this), - injecting_crash(0) +HMSMRDevice::HMSMRDevice(CephContext* cct, + aio_callback_t cb, + void *cbpriv, + aio_callback_t d_cb, + void *d_cbpriv) + : KernelDevice(cct, cb, cbpriv, d_cb, d_cbpriv) { - fd_directs.resize(WRITE_LIFE_MAX, -1); - fd_buffereds.resize(WRITE_LIFE_MAX, -1); - - bool use_ioring = cct->_conf.get_val("bdev_ioring"); - unsigned int iodepth = cct->_conf->bdev_aio_max_queue_depth; - - if (use_ioring && ioring_queue_t::supported()) { - bool use_ioring_hipri = cct->_conf.get_val("bdev_ioring_hipri"); - bool use_ioring_sqthread_poll = cct->_conf.get_val("bdev_ioring_sqthread_poll"); - io_queue = std::make_unique(iodepth, use_ioring_hipri, use_ioring_sqthread_poll); - } else { - static bool once; - if (use_ioring && !once) { - derr << "WARNING: io_uring API is not supported! Fallback to libaio!" - << dendl; - once = true; - } - io_queue = std::make_unique(iodepth); - } } bool HMSMRDevice::support(const std::string& path) @@ -85,39 +41,33 @@ bool HMSMRDevice::support(const std::string& path) return zbd_device_is_zoned(path.c_str()) == 1; } -int HMSMRDevice::_lock() +int HMSMRDevice::_post_open() { - dout(10) << __func__ << " " << fd_directs[WRITE_LIFE_NOT_SET] << dendl; - int r = ::flock(fd_directs[WRITE_LIFE_NOT_SET], LOCK_EX | LOCK_NB); - if (r < 0) { - derr << __func__ << " flock failed on " << path << dendl; - return -errno; - } - return 0; -} - -bool HMSMRDevice::set_smr_params(const std::string& path) { - dout(10) << __func__ << " opening " << path << dendl; + dout(10) << __func__ << dendl; - int dev = zbd_open(path.c_str(), O_RDWR | O_DIRECT | O_LARGEFILE, nullptr); - if (dev < 0) { + zbd_fd = zbd_open(path.c_str(), O_RDWR | O_DIRECT | O_LARGEFILE, nullptr); + int r; + if (zbd_fd < 0) { + r = errno; derr << __func__ << " zbd_open failed on " << path << ": " - << cpp_strerror(errno) << dendl; - return false; + << cpp_strerror(r) << dendl; + return -r; } - auto close_dev = make_scope_guard([dev] { zbd_close(dev); }); unsigned int nr_zones = 0; - if (zbd_report_nr_zones(dev, 0, 0, ZBD_RO_NOT_WP, &nr_zones) != 0) { + std::vector zones; + if (zbd_report_nr_zones(zbd_fd, 0, 0, ZBD_RO_NOT_WP, &nr_zones) != 0) { + r = -errno; derr << __func__ << " zbd_report_nr_zones failed on " << path << ": " - << cpp_strerror(errno) << dendl; - return false; + << cpp_strerror(r) << dendl; + goto fail; } - std::vector zones(nr_zones); - if (zbd_report_zones(dev, 0, 0, ZBD_RO_NOT_WP, zones.data(), &nr_zones) != 0) { + zones.resize(nr_zones); + if (zbd_report_zones(zbd_fd, 0, 0, ZBD_RO_NOT_WP, zones.data(), &nr_zones) != 0) { + r = -errno; derr << __func__ << " zbd_report_zones failed on " << path << dendl; - return false; + goto fail; } zone_size = zbd_zone_len(&zones[0]); @@ -127,1082 +77,36 @@ bool HMSMRDevice::set_smr_params(const std::string& path) { << " and conventional region size to " << conventional_region_size << dendl; - return true; -} - -int HMSMRDevice::open(const string& p) -{ - path = p; - int r = 0, i = 0; - dout(1) << __func__ << " path " << path << dendl; - - for (i = 0; i < WRITE_LIFE_MAX; i++) { - int fd = ::open(path.c_str(), O_RDWR | O_DIRECT); - if (fd < 0) { - r = -errno; - break; - } - fd_directs[i] = fd; - - fd = ::open(path.c_str(), O_RDWR | O_CLOEXEC); - if (fd < 0) { - r = -errno; - break; - } - fd_buffereds[i] = fd; - } - - if (i != WRITE_LIFE_MAX) { - derr << __func__ << " open got: " << cpp_strerror(r) << dendl; - goto out_fail; - } - - if (!set_smr_params(p)) { - derr << __func__ << " failed to set HM-SMR parameters" << dendl; - goto out_fail; - } - -#if defined(F_SET_FILE_RW_HINT) - for (i = WRITE_LIFE_NONE; i < WRITE_LIFE_MAX; i++) { - if (fcntl(fd_directs[i], F_SET_FILE_RW_HINT, &i) < 0) { - r = -errno; - break; - } - if (fcntl(fd_buffereds[i], F_SET_FILE_RW_HINT, &i) < 0) { - r = -errno; - break; - } - } - if (i != WRITE_LIFE_MAX) { - enable_wrt = false; - dout(0) << "ioctl(F_SET_FILE_RW_HINT) on " << path << " failed: " << cpp_strerror(r) << dendl; - } -#endif - - dio = true; - aio = cct->_conf->bdev_aio; - if (!aio) { - ceph_abort_msg("non-aio not supported"); - } - - // disable readahead as it will wreak havoc on our mix of - // directio/aio and buffered io. - r = posix_fadvise(fd_buffereds[WRITE_LIFE_NOT_SET], 0, 0, POSIX_FADV_RANDOM); - if (r) { - r = -r; - derr << __func__ << " posix_fadvise got: " << cpp_strerror(r) << dendl; - goto out_fail; - } - - if (lock_exclusive) { - r = _lock(); - if (r < 0) { - derr << __func__ << " failed to lock " << path << ": " << cpp_strerror(r) - << dendl; - goto out_fail; - } - } - - struct stat st; - r = ::fstat(fd_directs[WRITE_LIFE_NOT_SET], &st); - if (r < 0) { - r = -errno; - derr << __func__ << " fstat got " << cpp_strerror(r) << dendl; - goto out_fail; - } - - // Operate as though the block size is 4 KB. The backing file - // blksize doesn't strictly matter except that some file systems may - // require a read/modify/write if we write something smaller than - // it. - block_size = cct->_conf->bdev_block_size; - if (block_size != (unsigned)st.st_blksize) { - dout(1) << __func__ << " backing device/file reports st_blksize " - << st.st_blksize << ", using bdev_block_size " - << block_size << " anyway" << dendl; - } - - - { - BlkDev blkdev_direct(fd_directs[WRITE_LIFE_NOT_SET]); - BlkDev blkdev_buffered(fd_buffereds[WRITE_LIFE_NOT_SET]); - - if (S_ISBLK(st.st_mode)) { - int64_t s; - r = blkdev_direct.get_size(&s); - if (r < 0) { - goto out_fail; - } - size = s; - } else { - size = st.st_size; - } - - char partition[PATH_MAX], devname[PATH_MAX]; - if ((r = blkdev_buffered.partition(partition, PATH_MAX)) || - (r = blkdev_buffered.wholedisk(devname, PATH_MAX))) { - derr << "unable to get device name for " << path << ": " - << cpp_strerror(r) << dendl; - rotational = true; - } else { - dout(20) << __func__ << " devname " << devname << dendl; - rotational = blkdev_buffered.is_rotational(); - support_discard = blkdev_buffered.support_discard(); - this->devname = devname; - } - } - - r = _aio_start(); - if (r < 0) { - goto out_fail; - } - _discard_start(); - - // round size down to an even block - size &= ~(block_size - 1); - - dout(1) << __func__ - << " size " << size - << " (0x" << std::hex << size << std::dec << ", " - << byte_u_t(size) << ")" - << " block_size " << block_size - << " (" << byte_u_t(block_size) << ")" - << " " << (rotational ? "rotational" : "non-rotational") - << " discard " << (support_discard ? "supported" : "not supported") - << dendl; return 0; -out_fail: - for (i = 0; i < WRITE_LIFE_MAX; i++) { - if (fd_directs[i] >= 0) { - VOID_TEMP_FAILURE_RETRY(::close(fd_directs[i])); - fd_directs[i] = -1; - } else { - break; - } - if (fd_buffereds[i] >= 0) { - VOID_TEMP_FAILURE_RETRY(::close(fd_buffereds[i])); - fd_buffereds[i] = -1; - } else { - break; - } - } +fail: + zbd_close(zbd_fd); + zbd_fd = -1; return r; } -int HMSMRDevice::get_devices(std::set *ls) const -{ - if (devname.empty()) { - return 0; - } - get_raw_devices(devname, ls); - return 0; -} - -void HMSMRDevice::close() -{ - dout(1) << __func__ << dendl; - _aio_stop(); - _discard_stop(); - - for (int i = 0; i < WRITE_LIFE_MAX; i++) { - assert(fd_directs[i] >= 0); - VOID_TEMP_FAILURE_RETRY(::close(fd_directs[i])); - fd_directs[i] = -1; - assert(fd_buffereds[i] >= 0); - VOID_TEMP_FAILURE_RETRY(::close(fd_buffereds[i])); - fd_buffereds[i] = -1; - } - path.clear(); -} - -int HMSMRDevice::collect_metadata(const string& prefix, map *pm) const +void HMSMRDevice::_pre_close() { - (*pm)[prefix + "support_discard"] = stringify((int)(bool)support_discard); - (*pm)[prefix + "rotational"] = stringify((int)(bool)rotational); - (*pm)[prefix + "size"] = stringify(get_size()); - (*pm)[prefix + "block_size"] = stringify(get_block_size()); - (*pm)[prefix + "driver"] = "HMSMRDevice"; - if (rotational) { - (*pm)[prefix + "type"] = "hdd"; - } else { - (*pm)[prefix + "type"] = "ssd"; - } - - { - string res_names; - std::set devnames; - if (get_devices(&devnames) == 0) { - for (auto& dev : devnames) { - if (!res_names.empty()) { - res_names += ","; - } - res_names += dev; - } - if (res_names.size()) { - (*pm)[prefix + "devices"] = res_names; - } - } - } - - struct stat st; - int r = ::fstat(fd_buffereds[WRITE_LIFE_NOT_SET], &st); - if (r < 0) - return -errno; - if (S_ISBLK(st.st_mode)) { - (*pm)[prefix + "access_mode"] = "blk"; - - char buffer[1024] = {0}; - BlkDev blkdev{fd_buffereds[WRITE_LIFE_NOT_SET]}; - if (r = blkdev.partition(buffer, sizeof(buffer)); r) { - (*pm)[prefix + "partition_path"] = "unknown"; - } else { - (*pm)[prefix + "partition_path"] = buffer; - } - buffer[0] = '\0'; - if (r = blkdev.partition(buffer, sizeof(buffer)); r) { - (*pm)[prefix + "dev_node"] = "unknown"; - } else { - (*pm)[prefix + "dev_node"] = buffer; - } - if (!r) { - return 0; - } - buffer[0] = '\0'; - blkdev.model(buffer, sizeof(buffer)); - (*pm)[prefix + "model"] = buffer; - - buffer[0] = '\0'; - blkdev.dev(buffer, sizeof(buffer)); - (*pm)[prefix + "dev"] = buffer; - - // nvme exposes a serial number - buffer[0] = '\0'; - blkdev.serial(buffer, sizeof(buffer)); - (*pm)[prefix + "serial"] = buffer; - - // numa - int node; - r = blkdev.get_numa_node(&node); - if (r >= 0) { - (*pm)[prefix + "numa_node"] = stringify(node); - } - } else { - (*pm)[prefix + "access_mode"] = "file"; - (*pm)[prefix + "path"] = path; + if (zbd_fd >= 0) { + zbd_close(zbd_fd); + zbd_fd = -1; } - return 0; } void HMSMRDevice::reset_all_zones() { + dout(10) << __func__ << dendl; zbd_reset_zones(zbd_fd, conventional_region_size, 0); } -void HMSMRDevice::reset_zones(const std::set& zones) { +void HMSMRDevice::reset_zones(const std::set& zones) +{ + dout(10) << __func__ << " 0x" << std::hex << zones << std::dec << dendl; for (auto zone_num : zones) { if (zbd_reset_zones(zbd_fd, zone_num * zone_size, zone_size) != 0) { - derr << __func__ << " resetting zone failed for zone " << zone_num << dendl; - } - } -} - -bool HMSMRDevice::get_thin_utilization(uint64_t *total, uint64_t *avail) const -{ - return false; -} - -int HMSMRDevice::choose_fd(bool buffered, int write_hint) const -{ - assert(write_hint >= WRITE_LIFE_NOT_SET && write_hint < WRITE_LIFE_MAX); - if (!enable_wrt) - write_hint = WRITE_LIFE_NOT_SET; - return buffered ? fd_buffereds[write_hint] : fd_directs[write_hint]; -} - -int HMSMRDevice::flush() -{ - // protect flush with a mutex. note that we are not really protecting - // data here. instead, we're ensuring that if any flush() caller - // sees that io_since_flush is true, they block any racing callers - // until the flush is observed. that allows racing threads to be - // calling flush while still ensuring that *any* of them that got an - // aio completion notification will not return before that aio is - // stable on disk: whichever thread sees the flag first will block - // followers until the aio is stable. - std::lock_guard l(flush_mutex); - - bool expect = true; - if (!io_since_flush.compare_exchange_strong(expect, false)) { - dout(10) << __func__ << " no-op (no ios since last flush), flag is " - << (int)io_since_flush.load() << dendl; - return 0; - } - - dout(10) << __func__ << " start" << dendl; - if (cct->_conf->bdev_inject_crash) { - ++injecting_crash; - // sleep for a moment to give other threads a chance to submit or - // wait on io that races with a flush. - derr << __func__ << " injecting crash. first we sleep..." << dendl; - sleep(cct->_conf->bdev_inject_crash_flush_delay); - derr << __func__ << " and now we die" << dendl; - cct->_log->flush(); - _exit(1); - } - utime_t start = ceph_clock_now(); - int r = ::fdatasync(fd_directs[WRITE_LIFE_NOT_SET]); - utime_t end = ceph_clock_now(); - utime_t dur = end - start; - if (r < 0) { - r = -errno; - derr << __func__ << " fdatasync got: " << cpp_strerror(r) << dendl; - ceph_abort(); - } - dout(5) << __func__ << " in " << dur << dendl;; - return r; -} - -int HMSMRDevice::_aio_start() -{ - if (aio) { - dout(10) << __func__ << dendl; - int r = io_queue->init(fd_directs); - if (r < 0) { - if (r == -EAGAIN) { - derr << __func__ << " io_setup(2) failed with EAGAIN; " - << "try increasing /proc/sys/fs/aio-max-nr" << dendl; - } else { - derr << __func__ << " io_setup(2) failed: " << cpp_strerror(r) << dendl; - } - return r; - } - aio_thread.create("bstore_aio"); - } - return 0; -} - -void HMSMRDevice::_aio_stop() -{ - if (aio) { - dout(10) << __func__ << dendl; - aio_stop = true; - aio_thread.join(); - aio_stop = false; - io_queue->shutdown(); - } -} - -int HMSMRDevice::_discard_start() -{ - discard_thread.create("bstore_discard"); - return 0; -} - -void HMSMRDevice::_discard_stop() -{ - dout(10) << __func__ << dendl; - { - std::unique_lock l(discard_lock); - while (!discard_started) { - discard_cond.wait(l); + derr << __func__ << " resetting zone failed for zone 0x" << std::hex + << zone_num << std::dec << dendl; } - discard_stop = true; - discard_cond.notify_all(); - } - discard_thread.join(); - { - std::lock_guard l(discard_lock); - discard_stop = false; } - dout(10) << __func__ << " stopped" << dendl; -} - -void HMSMRDevice::discard_drain() -{ - dout(10) << __func__ << dendl; - std::unique_lock l(discard_lock); - while (!discard_queued.empty() || discard_running) { - discard_cond.wait(l); - } -} - -static bool is_expected_ioerr(const int r) -{ - // https://lxr.missinglinkelectronics.com/linux+v4.15/block/blk-core.c#L135 - return (r == -EOPNOTSUPP || r == -ETIMEDOUT || r == -ENOSPC || - r == -ENOLINK || r == -EREMOTEIO || r == -EAGAIN || r == -EIO || - r == -ENODATA || r == -EILSEQ || r == -ENOMEM || -#if defined(__linux__) - r == -EREMCHG || r == -EBADE -#elif defined(__FreeBSD__) - r == - BSM_ERRNO_EREMCHG || r == -BSM_ERRNO_EBADE -#endif - ); -} - -void HMSMRDevice::_aio_thread() -{ - dout(10) << __func__ << " start" << dendl; - int inject_crash_count = 0; - while (!aio_stop) { - dout(40) << __func__ << " polling" << dendl; - int max = cct->_conf->bdev_aio_reap_max; - aio_t *aio[max]; - int r = io_queue->get_next_completed(cct->_conf->bdev_aio_poll_ms, - aio, max); - if (r < 0) { - derr << __func__ << " got " << cpp_strerror(r) << dendl; - ceph_abort_msg("got unexpected error from io_getevents"); - } - if (r > 0) { - dout(30) << __func__ << " got " << r << " completed aios" << dendl; - for (int i = 0; i < r; ++i) { - IOContext *ioc = static_cast(aio[i]->priv); - _aio_log_finish(ioc, aio[i]->offset, aio[i]->length); - if (aio[i]->queue_item.is_linked()) { - std::lock_guard l(debug_queue_lock); - debug_aio_unlink(*aio[i]); - } - - // set flag indicating new ios have completed. we do this *before* - // any completion or notifications so that any user flush() that - // follows the observed io completion will include this io. Note - // that an earlier, racing flush() could observe and clear this - // flag, but that also ensures that the IO will be stable before the - // later flush() occurs. - io_since_flush.store(true); - - long r = aio[i]->get_return_value(); - if (r < 0) { - derr << __func__ << " got r=" << r << " (" << cpp_strerror(r) << ")" - << dendl; - if (ioc->allow_eio && is_expected_ioerr(r)) { - derr << __func__ << " translating the error to EIO for upper layer" - << dendl; - ioc->set_return_value(-EIO); - } else { - if (is_expected_ioerr(r)) { - note_io_error_event( - devname.c_str(), - path.c_str(), - r, -#if defined(HAVE_POSIXAIO) - aio[i]->aio.aiocb.aio_lio_opcode, -#else - aio[i]->iocb.aio_lio_opcode, -#endif - aio[i]->offset, - aio[i]->length); - ceph_abort_msg( - "Unexpected IO error. " - "This may suggest a hardware issue. " - "Please check your kernel log!"); - } - ceph_abort_msg( - "Unexpected IO error. " - "This may suggest HW issue. Please check your dmesg!"); - } - } else if (aio[i]->length != (uint64_t)r) { - derr << "aio to 0x" << std::hex << aio[i]->offset - << "~" << aio[i]->length << std::dec - << " but returned: " << r << dendl; - ceph_abort_msg("unexpected aio return value: does not match length"); - } - - dout(10) << __func__ << " finished aio " << aio[i] << " r " << r - << " ioc " << ioc - << " with " << (ioc->num_running.load() - 1) - << " aios left" << dendl; - - // NOTE: once num_running and we either call the callback or - // call aio_wake we cannot touch ioc or aio[] as the caller - // may free it. - if (ioc->priv) { - if (--ioc->num_running == 0) { - aio_callback(aio_callback_priv, ioc->priv); - } - } else { - ioc->try_aio_wake(); - } - } - } - if (cct->_conf->bdev_debug_aio) { - utime_t now = ceph_clock_now(); - std::lock_guard l(debug_queue_lock); - if (debug_oldest) { - if (debug_stall_since == utime_t()) { - debug_stall_since = now; - } else { - if (cct->_conf->bdev_debug_aio_suicide_timeout) { - utime_t cutoff = now; - cutoff -= cct->_conf->bdev_debug_aio_suicide_timeout; - if (debug_stall_since < cutoff) { - derr << __func__ << " stalled aio " << debug_oldest - << " since " << debug_stall_since << ", timeout is " - << cct->_conf->bdev_debug_aio_suicide_timeout - << "s, suicide" << dendl; - ceph_abort_msg("stalled aio... buggy kernel or bad device?"); - } - } - } - } - } - if (cct->_conf->bdev_inject_crash) { - ++inject_crash_count; - if (inject_crash_count * cct->_conf->bdev_aio_poll_ms / 1000 > - cct->_conf->bdev_inject_crash + cct->_conf->bdev_inject_crash_flush_delay) { - derr << __func__ << " bdev_inject_crash trigger from aio thread" - << dendl; - cct->_log->flush(); - _exit(1); - } - } - } - dout(10) << __func__ << " end" << dendl; -} - -void HMSMRDevice::_discard_thread() -{ - std::unique_lock l(discard_lock); - ceph_assert(!discard_started); - discard_started = true; - discard_cond.notify_all(); - while (true) { - ceph_assert(discard_finishing.empty()); - if (discard_queued.empty()) { - if (discard_stop) - break; - dout(20) << __func__ << " sleep" << dendl; - discard_cond.notify_all(); // for the thread trying to drain... - discard_cond.wait(l); - dout(20) << __func__ << " wake" << dendl; - } else { - discard_finishing.swap(discard_queued); - discard_running = true; - l.unlock(); - dout(20) << __func__ << " finishing" << dendl; - for (auto p = discard_finishing.begin();p != discard_finishing.end(); ++p) { - discard(p.get_start(), p.get_len()); - } - - discard_callback(discard_callback_priv, static_cast(&discard_finishing)); - discard_finishing.clear(); - l.lock(); - discard_running = false; - } - } - dout(10) << __func__ << " finish" << dendl; - discard_started = false; -} - -int HMSMRDevice::queue_discard(interval_set &to_release) -{ - if (!support_discard) - return -1; - - if (to_release.empty()) - return 0; - - std::lock_guard l(discard_lock); - discard_queued.insert(to_release); - discard_cond.notify_all(); - return 0; -} - -void HMSMRDevice::_aio_log_start( - IOContext *ioc, - uint64_t offset, - uint64_t length) -{ - dout(20) << __func__ << " 0x" << std::hex << offset << "~" << length - << std::dec << dendl; - if (cct->_conf->bdev_debug_inflight_ios) { - std::lock_guard l(debug_lock); - if (debug_inflight.intersects(offset, length)) { - derr << __func__ << " inflight overlap of 0x" - << std::hex - << offset << "~" << length << std::dec - << " with " << debug_inflight << dendl; - ceph_abort(); - } - debug_inflight.insert(offset, length); - } -} - -void HMSMRDevice::debug_aio_link(aio_t& aio) -{ - if (debug_queue.empty()) { - debug_oldest = &aio; - } - debug_queue.push_back(aio); -} - -void HMSMRDevice::debug_aio_unlink(aio_t& aio) -{ - if (aio.queue_item.is_linked()) { - debug_queue.erase(debug_queue.iterator_to(aio)); - if (debug_oldest == &aio) { - auto age = cct->_conf->bdev_debug_aio_log_age; - if (age && debug_stall_since != utime_t()) { - utime_t cutoff = ceph_clock_now(); - cutoff -= age; - if (debug_stall_since < cutoff) { - derr << __func__ << " stalled aio " << debug_oldest - << " since " << debug_stall_since << ", timeout is " - << age - << "s" << dendl; - } - } - - if (debug_queue.empty()) { - debug_oldest = nullptr; - } else { - debug_oldest = &debug_queue.front(); - } - debug_stall_since = utime_t(); - } - } -} - -void HMSMRDevice::_aio_log_finish( - IOContext *ioc, - uint64_t offset, - uint64_t length) -{ - dout(20) << __func__ << " " << aio << " 0x" - << std::hex << offset << "~" << length << std::dec << dendl; - if (cct->_conf->bdev_debug_inflight_ios) { - std::lock_guard l(debug_lock); - debug_inflight.erase(offset, length); - } -} - -void HMSMRDevice::aio_submit(IOContext *ioc) -{ - dout(20) << __func__ << " ioc " << ioc - << " pending " << ioc->num_pending.load() - << " running " << ioc->num_running.load() - << dendl; - - if (ioc->num_pending.load() == 0) { - return; - } - - // move these aside, and get our end iterator position now, as the - // aios might complete as soon as they are submitted and queue more - // wal aio's. - list::iterator e = ioc->running_aios.begin(); - ioc->running_aios.splice(e, ioc->pending_aios); - - int pending = ioc->num_pending.load(); - ioc->num_running += pending; - ioc->num_pending -= pending; - ceph_assert(ioc->num_pending.load() == 0); // we should be only thread doing this - ceph_assert(ioc->pending_aios.size() == 0); - - if (cct->_conf->bdev_debug_aio) { - list::iterator p = ioc->running_aios.begin(); - while (p != e) { - dout(30) << __func__ << " " << *p << dendl; - std::lock_guard l(debug_queue_lock); - debug_aio_link(*p++); - } - } - - void *priv = static_cast(ioc); - int r, retries = 0; - r = io_queue->submit_batch(ioc->running_aios.begin(), e, - pending, priv, &retries); - - if (retries) - derr << __func__ << " retries " << retries << dendl; - if (r < 0) { - derr << " aio submit got " << cpp_strerror(r) << dendl; - ceph_assert(r == 0); - } -} - -int HMSMRDevice::_sync_write(uint64_t off, bufferlist &bl, bool buffered, int write_hint) -{ - uint64_t len = bl.length(); - dout(5) << __func__ << " 0x" << std::hex << off << "~" << len - << std::dec << (buffered ? " (buffered)" : " (direct)") << dendl; - if (cct->_conf->bdev_inject_crash && - rand() % cct->_conf->bdev_inject_crash == 0) { - derr << __func__ << " bdev_inject_crash: dropping io 0x" << std::hex - << off << "~" << len << std::dec << dendl; - ++injecting_crash; - return 0; - } - vector iov; - bl.prepare_iov(&iov); - int r = ::pwritev(choose_fd(buffered, write_hint), - &iov[0], iov.size(), off); - - if (r < 0) { - r = -errno; - derr << __func__ << " pwritev error: " << cpp_strerror(r) << dendl; - return r; - } -#ifdef HAVE_SYNC_FILE_RANGE - if (buffered) { - // initiate IO and wait till it completes - r = ::sync_file_range(fd_buffereds[WRITE_LIFE_NOT_SET], off, len, SYNC_FILE_RANGE_WRITE|SYNC_FILE_RANGE_WAIT_AFTER|SYNC_FILE_RANGE_WAIT_BEFORE); - if (r < 0) { - r = -errno; - derr << __func__ << " sync_file_range error: " << cpp_strerror(r) << dendl; - return r; - } - } -#endif - - io_since_flush.store(true); - - return 0; -} - -int HMSMRDevice::write( - uint64_t off, - bufferlist &bl, - bool buffered, - int write_hint) -{ - uint64_t len = bl.length(); - dout(20) << __func__ << " 0x" << std::hex << off << "~" << len << std::dec - << (buffered ? " (buffered)" : " (direct)") - << dendl; - ceph_assert(is_valid_io(off, len)); - if (cct->_conf->objectstore_blackhole) { - lderr(cct) << __func__ << " objectstore_blackhole=true, throwing out IO" - << dendl; - return 0; - } - - if ((!buffered || bl.get_num_buffers() >= IOV_MAX) && - bl.rebuild_aligned_size_and_memory(block_size, block_size, IOV_MAX)) { - dout(20) << __func__ << " rebuilding buffer to be aligned" << dendl; - } - dout(40) << "data:\n"; - bl.hexdump(*_dout); - *_dout << dendl; - - return _sync_write(off, bl, buffered, write_hint); -} - -int HMSMRDevice::aio_write( - uint64_t off, - bufferlist &bl, - IOContext *ioc, - bool buffered, - int write_hint) -{ - uint64_t len = bl.length(); - dout(20) << __func__ << " 0x" << std::hex << off << "~" << len << std::dec - << (buffered ? " (buffered)" : " (direct)") - << dendl; - ceph_assert(is_valid_io(off, len)); - if (cct->_conf->objectstore_blackhole) { - lderr(cct) << __func__ << " objectstore_blackhole=true, throwing out IO" - << dendl; - return 0; - } - - if ((!buffered || bl.get_num_buffers() >= IOV_MAX) && - bl.rebuild_aligned_size_and_memory(block_size, block_size, IOV_MAX)) { - dout(20) << __func__ << " rebuilding buffer to be aligned" << dendl; - } - dout(40) << "data:\n"; - bl.hexdump(*_dout); - *_dout << dendl; - - _aio_log_start(ioc, off, len); - -#ifdef HAVE_LIBAIO - if (aio && dio && !buffered) { - if (cct->_conf->bdev_inject_crash && - rand() % cct->_conf->bdev_inject_crash == 0) { - derr << __func__ << " bdev_inject_crash: dropping io 0x" << std::hex - << off << "~" << len << std::dec - << dendl; - // generate a real io so that aio_wait behaves properly, but make it - // a read instead of write, and toss the result. - ioc->pending_aios.push_back(aio_t(ioc, choose_fd(false, write_hint))); - ++ioc->num_pending; - auto& aio = ioc->pending_aios.back(); - bufferptr p = buffer::create_small_page_aligned(len); - aio.bl.append(std::move(p)); - aio.bl.prepare_iov(&aio.iov); - aio.preadv(off, len); - ++injecting_crash; - } else { - if (bl.length() <= RW_IO_MAX) { - // fast path (non-huge write) - ioc->pending_aios.push_back(aio_t(ioc, choose_fd(false, write_hint))); - ++ioc->num_pending; - auto& aio = ioc->pending_aios.back(); - bl.prepare_iov(&aio.iov); - aio.bl.claim_append(bl); - aio.pwritev(off, len); - dout(30) << aio << dendl; - dout(5) << __func__ << " 0x" << std::hex << off << "~" << len - << std::dec << " aio " << &aio << dendl; - } else { - // write in RW_IO_MAX-sized chunks - uint64_t prev_len = 0; - while (prev_len < bl.length()) { - bufferlist tmp; - if (prev_len + RW_IO_MAX < bl.length()) { - tmp.substr_of(bl, prev_len, RW_IO_MAX); - } else { - tmp.substr_of(bl, prev_len, bl.length() - prev_len); - } - auto len = tmp.length(); - ioc->pending_aios.push_back(aio_t(ioc, choose_fd(false, write_hint))); - ++ioc->num_pending; - auto& aio = ioc->pending_aios.back(); - tmp.prepare_iov(&aio.iov); - aio.bl.claim_append(tmp); - aio.pwritev(off + prev_len, len); - dout(30) << aio << dendl; - dout(5) << __func__ << " 0x" << std::hex << off + prev_len - << "~" << len - << std::dec << " aio " << &aio << " (piece)" << dendl; - prev_len += len; - } - } - } - } else -#endif - { - int r = _sync_write(off, bl, buffered, write_hint); - _aio_log_finish(ioc, off, len); - if (r < 0) - return r; - } - return 0; -} - -int HMSMRDevice::discard(uint64_t offset, uint64_t len) -{ - int r = 0; - if (cct->_conf->objectstore_blackhole) { - lderr(cct) << __func__ << " objectstore_blackhole=true, throwing out IO" - << dendl; - return 0; - } - if (support_discard) { - dout(10) << __func__ - << " 0x" << std::hex << offset << "~" << len << std::dec - << dendl; - - r = BlkDev{fd_directs[WRITE_LIFE_NOT_SET]}.discard((int64_t)offset, (int64_t)len); - } - return r; -} - -int HMSMRDevice::read(uint64_t off, uint64_t len, bufferlist *pbl, - IOContext *ioc, - bool buffered) -{ - dout(5) << __func__ << " 0x" << std::hex << off << "~" << len << std::dec - << (buffered ? " (buffered)" : " (direct)") - << dendl; - ceph_assert(is_valid_io(off, len)); - - _aio_log_start(ioc, off, len); - - auto start1 = mono_clock::now(); - - auto p = buffer::ptr_node::create(buffer::create_small_page_aligned(len)); - int r = ::pread(buffered ? fd_buffereds[WRITE_LIFE_NOT_SET] : fd_directs[WRITE_LIFE_NOT_SET], - p->c_str(), len, off); - auto age = cct->_conf->bdev_debug_aio_log_age; - if (mono_clock::now() - start1 >= make_timespan(age)) { - derr << __func__ << " stalled read " - << " 0x" << std::hex << off << "~" << len << std::dec - << (buffered ? " (buffered)" : " (direct)") - << " since " << start1 << ", timeout is " - << age - << "s" << dendl; - } - - if (r < 0) { - if (ioc->allow_eio && is_expected_ioerr(r)) { - r = -EIO; - } else { - r = -errno; - } - goto out; - } - ceph_assert((uint64_t)r == len); - pbl->push_back(std::move(p)); - - dout(40) << "data:\n"; - pbl->hexdump(*_dout); - *_dout << dendl; - - out: - _aio_log_finish(ioc, off, len); - return r < 0 ? r : 0; -} - -int HMSMRDevice::aio_read( - uint64_t off, - uint64_t len, - bufferlist *pbl, - IOContext *ioc) -{ - dout(5) << __func__ << " 0x" << std::hex << off << "~" << len << std::dec - << dendl; - - int r = 0; -#ifdef HAVE_LIBAIO - if (aio && dio) { - ceph_assert(is_valid_io(off, len)); - _aio_log_start(ioc, off, len); - ioc->pending_aios.push_back(aio_t(ioc, fd_directs[WRITE_LIFE_NOT_SET])); - ++ioc->num_pending; - aio_t& aio = ioc->pending_aios.back(); - bufferptr p = buffer::create_small_page_aligned(len); - aio.bl.append(std::move(p)); - aio.bl.prepare_iov(&aio.iov); - aio.preadv(off, len); - dout(30) << aio << dendl; - pbl->append(aio.bl); - dout(5) << __func__ << " 0x" << std::hex << off << "~" << len - << std::dec << " aio " << &aio << dendl; - } else -#endif - { - r = read(off, len, pbl, ioc, false); - } - - return r; -} - -int HMSMRDevice::direct_read_unaligned(uint64_t off, uint64_t len, char *buf) -{ - uint64_t aligned_off = p2align(off, block_size); - uint64_t aligned_len = p2roundup(off+len, block_size) - aligned_off; - bufferptr p = buffer::create_small_page_aligned(aligned_len); - int r = 0; - - auto start1 = mono_clock::now(); - r = ::pread(fd_directs[WRITE_LIFE_NOT_SET], p.c_str(), aligned_len, aligned_off); - auto age = cct->_conf->bdev_debug_aio_log_age; - if (mono_clock::now() - start1 >= make_timespan(age)) { - derr << __func__ << " stalled read " - << " 0x" << std::hex << off << "~" << len << std::dec - << " since " << start1 << ", timeout is " - << age - << "s" << dendl; - } - - if (r < 0) { - r = -errno; - derr << __func__ << " 0x" << std::hex << off << "~" << len << std::dec - << " error: " << cpp_strerror(r) << dendl; - goto out; - } - ceph_assert((uint64_t)r == aligned_len); - memcpy(buf, p.c_str() + (off - aligned_off), len); - - dout(40) << __func__ << " data:\n"; - bufferlist bl; - bl.append(buf, len); - bl.hexdump(*_dout); - *_dout << dendl; - - out: - return r < 0 ? r : 0; -} - -int HMSMRDevice::read_random(uint64_t off, uint64_t len, char *buf, - bool buffered) -{ - dout(5) << __func__ << " 0x" << std::hex << off << "~" << len << std::dec - << "buffered " << buffered - << dendl; - ceph_assert(len > 0); - ceph_assert(off < size); - ceph_assert(off + len <= size); - int r = 0; - auto age = cct->_conf->bdev_debug_aio_log_age; - - //if it's direct io and unaligned, we have to use a internal buffer - if (!buffered && ((off % block_size != 0) - || (len % block_size != 0) - || (uintptr_t(buf) % CEPH_PAGE_SIZE != 0))) - return direct_read_unaligned(off, len, buf); - - auto start1 = mono_clock::now(); - if (buffered) { - //buffered read - auto off0 = off; - char *t = buf; - uint64_t left = len; - while (left > 0) { - r = ::pread(fd_buffereds[WRITE_LIFE_NOT_SET], t, left, off); - if (r < 0) { - r = -errno; - derr << __func__ << " 0x" << std::hex << off << "~" << left - << std::dec << " error: " << cpp_strerror(r) << dendl; - goto out; - } - off += r; - t += r; - left -= r; - } - if (mono_clock::now() - start1 >= make_timespan(age)) { - derr << __func__ << " stalled read " - << " 0x" << std::hex << off0 << "~" << len << std::dec - << " (buffered) since " << start1 << ", timeout is " - << age - << "s" << dendl; - } - } else { - //direct and aligned read - r = ::pread(fd_directs[WRITE_LIFE_NOT_SET], buf, len, off); - if (mono_clock::now() - start1 >= make_timespan(age)) { - derr << __func__ << " stalled read " - << " 0x" << std::hex << off << "~" << len << std::dec - << " (direct) since " << start1 << ", timeout is " - << age - << "s" << dendl; - } - if (r < 0) { - r = -errno; - derr << __func__ << " direct_aligned_read" << " 0x" << std::hex - << off << "~" << left << std::dec << " error: " << cpp_strerror(r) - << dendl; - goto out; - } - ceph_assert((uint64_t)r == len); - } - - dout(40) << __func__ << " data:\n"; - bufferlist bl; - bl.append(buf, len); - bl.hexdump(*_dout); - *_dout << dendl; - - out: - return r < 0 ? r : 0; -} - -int HMSMRDevice::invalidate_cache(uint64_t off, uint64_t len) -{ - dout(5) << __func__ << " 0x" << std::hex << off << "~" << len << std::dec - << dendl; - ceph_assert(off % block_size == 0); - ceph_assert(len % block_size == 0); - int r = posix_fadvise(fd_buffereds[WRITE_LIFE_NOT_SET], off, len, POSIX_FADV_DONTNEED); - if (r) { - r = -r; - derr << __func__ << " 0x" << std::hex << off << "~" << len << std::dec - << " error: " << cpp_strerror(r) << dendl; - } - return r; } diff --git a/src/blk/zoned/HMSMRDevice.h b/src/blk/zoned/HMSMRDevice.h index f3cd98f391fc0..af490fc4e6247 100644 --- a/src/blk/zoned/HMSMRDevice.h +++ b/src/blk/zoned/HMSMRDevice.h @@ -13,9 +13,6 @@ * */ -// Copied from KernelDevice with HM-SMR specific functionality added. Will be -// further specialized for HM-SMR. - #ifndef CEPH_BLK_HMSMRDEVICE_H #define CEPH_BLK_HMSMRDEVICE_H @@ -28,136 +25,26 @@ #include "aio/aio.h" #include "BlockDevice.h" +#include "../kernel/KernelDevice.h" -#define RW_IO_MAX (INT_MAX & CEPH_PAGE_MASK) - -class HMSMRDevice final : public BlockDevice { - std::vector fd_directs, fd_buffereds; - bool enable_wrt = true; - std::string path; - bool aio, dio; - std::string devname; ///< kernel dev name (/sys/block/$devname), if any +class HMSMRDevice final : public KernelDevice { int zbd_fd = -1; ///< fd for the zoned block device - ceph::mutex debug_lock = ceph::make_mutex("HMSMRDevice::debug_lock"); - interval_set debug_inflight; - - std::atomic io_since_flush = {false}; - ceph::mutex flush_mutex = ceph::make_mutex("HMSMRDevice::flush_mutex"); - - std::unique_ptr io_queue; - aio_callback_t discard_callback; - void *discard_callback_priv; - bool aio_stop; - bool discard_started; - bool discard_stop; - - ceph::mutex discard_lock = ceph::make_mutex("HMSMRDevice::discard_lock"); - ceph::condition_variable discard_cond; - bool discard_running = false; - interval_set discard_queued; - interval_set discard_finishing; - - struct AioCompletionThread : public Thread { - HMSMRDevice *bdev; - explicit AioCompletionThread(HMSMRDevice *b) : bdev(b) {} - void *entry() override { - bdev->_aio_thread(); - return NULL; - } - } aio_thread; - - struct DiscardThread : public Thread { - HMSMRDevice *bdev; - explicit DiscardThread(HMSMRDevice *b) : bdev(b) {} - void *entry() override { - bdev->_discard_thread(); - return NULL; - } - } discard_thread; - - std::atomic_int injecting_crash; - - void _aio_thread(); - void _discard_thread(); - int queue_discard(interval_set &to_release) final; - - int _aio_start(); - void _aio_stop(); - - int _discard_start(); - void _discard_stop(); - - void _aio_log_start(IOContext *ioc, uint64_t offset, uint64_t length); - void _aio_log_finish(IOContext *ioc, uint64_t offset, uint64_t length); - - int _sync_write(uint64_t off, bufferlist& bl, bool buffered, int write_hint); - - int _lock(); - - int direct_read_unaligned(uint64_t off, uint64_t len, char *buf); - - // stalled aio debugging - aio_list_t debug_queue; - ceph::mutex debug_queue_lock = - ceph::make_mutex("HMSMRDevice::debug_queue_lock"); - aio_t *debug_oldest = nullptr; - utime_t debug_stall_since; - void debug_aio_link(aio_t& aio); - void debug_aio_unlink(aio_t& aio); - - int choose_fd(bool buffered, int write_hint) const; - - bool set_smr_params(const std::string& path); - public: HMSMRDevice(CephContext* cct, aio_callback_t cb, void *cbpriv, aio_callback_t d_cb, void *d_cbpriv); - static bool support(const std::string& path); - void aio_submit(IOContext *ioc) final; - void discard_drain() final; + static bool support(const std::string& path); - int collect_metadata(const std::string& prefix, - std::map *pm) const final; - int get_devname(std::string *s) const final { - if (devname.empty()) { - return -ENOENT; - } - *s = devname; - return 0; - } - int get_devices(std::set *ls) const final; + // open/close hooks for libzbd + int _post_open() override; + void _pre_close() override; + // smr-specific methods bool is_smr() const final { return true; } - void reset_all_zones() override; void reset_zones(const std::set& zones) override; - - bool get_thin_utilization(uint64_t *total, uint64_t *avail) const final; - - int read(uint64_t off, uint64_t len, bufferlist *pbl, - IOContext *ioc, - bool buffered) final; - int aio_read(uint64_t off, uint64_t len, bufferlist *pbl, - IOContext *ioc) final; - int read_random(uint64_t off, uint64_t len, char *buf, - bool buffered) final; - - int write(uint64_t off, bufferlist& bl, bool buffered, - int write_hint = WRITE_LIFE_NOT_SET) final; - int aio_write(uint64_t off, bufferlist& bl, - IOContext *ioc, - bool buffered, - int write_hint = WRITE_LIFE_NOT_SET) final; - int flush() final; - int discard(uint64_t offset, uint64_t len) final; - - // for managing buffered readers/writers - int invalidate_cache(uint64_t off, uint64_t len) final; - int open(const std::string& path) final; - void close() final; }; #endif //CEPH_BLK_HMSMRDEVICE_H From 54470fe14b216f182aa3270e5dfd9528af42a04f Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Mon, 6 Sep 2021 16:00:18 -0500 Subject: [PATCH 0395/2866] os/bluestore: do not use null freelist with SMR Signed-off-by: Sage Weil --- src/os/bluestore/BlueStore.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc index b896519910759..ed36f945fd6ba 100644 --- a/src/os/bluestore/BlueStore.cc +++ b/src/os/bluestore/BlueStore.cc @@ -6084,7 +6084,11 @@ int BlueStore::_open_db_and_around(bool read_only, bool to_repair) } // when function is called in repair mode (to_repair=true) we skip db->open()/create() - if (!read_only && !to_repair && cct->_conf->bluestore_allocation_from_file) { + if (!read_only && !to_repair && cct->_conf->bluestore_allocation_from_file +#ifdef HAVE_LIBZBD + && !bdev->is_smr() +#endif + ) { dout(5) << __func__ << "::NCB::Commit to Null-Manager" << dendl; commit_to_null_manager(); } From 9c5d4349aae8bef3c87955dfb6af03ea923655c0 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Mon, 6 Sep 2021 16:45:23 -0500 Subject: [PATCH 0396/2866] os/bluestore: fix smr allocator init - initialize from device's pointers - cross check against freelist manager's pointers (warn!) - fix init_{add,rm}_free methods to do little/nothing (the other init method does most of what we need) Signed-off-by: Sage Weil --- src/os/bluestore/BlueStore.cc | 18 +++++-- src/os/bluestore/ZonedAllocator.cc | 75 +++++++++--------------------- src/os/bluestore/ZonedAllocator.h | 13 +++--- 3 files changed, 44 insertions(+), 62 deletions(-) diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc index ed36f945fd6ba..fe59e8eb1ba77 100644 --- a/src/os/bluestore/BlueStore.cc +++ b/src/os/bluestore/BlueStore.cc @@ -5570,12 +5570,22 @@ int BlueStore::_init_alloc() ceph_assert(a); auto f = dynamic_cast(fm); ceph_assert(f); - a->init_alloc(f->get_zone_states(db), - &zoned_cleaner_lock, - &zoned_cleaner_cond); + a->init_from_zone_pointers(f->get_zone_states(db), + &zoned_cleaner_lock, + &zoned_cleaner_cond); + dout(1) << __func__ + << " loaded zone pointers: " + << std::hex + << ", allocator type " << shared_alloc.a->get_type() + << ", capacity 0x" << shared_alloc.a->get_capacity() + << ", block size 0x" << shared_alloc.a->get_block_size() + << ", free 0x" << shared_alloc.a->get_free() + << ", fragmentation " << shared_alloc.a->get_fragmentation() + << std::dec << dendl; + return 0; } #endif - + uint64_t num = 0, bytes = 0; utime_t start_time = ceph_clock_now(); if (!fm->is_null_manager()) { diff --git a/src/os/bluestore/ZonedAllocator.cc b/src/os/bluestore/ZonedAllocator.cc index 6425919639892..0ec8000e17b6e 100644 --- a/src/os/bluestore/ZonedAllocator.cc +++ b/src/os/bluestore/ZonedAllocator.cc @@ -28,6 +28,8 @@ ZonedAllocator::ZonedAllocator(CephContext* cct, cct(cct), num_free(0), size(size), + conventional_size(_first_sequential_zone * _zone_size), + sequential_size(size - conventional_size), block_size(blk_size), zone_size(_zone_size), first_seq_zone_num(_first_sequential_zone), @@ -148,44 +150,26 @@ void ZonedAllocator::dump(std::function &&_zone_states, + ceph::mutex *_cleaner_lock, + ceph::condition_variable *_cleaner_cond) { + // this is called once, based on the device's zone pointers std::lock_guard l(lock); - ldout(cct, 40) << " 0x" << std::hex - << offset << "~" << length << std::dec << dendl; - - num_free -= length; - ceph_assert(num_free >= 0); - - uint64_t zone_num = offset / zone_size; - uint64_t write_pointer = offset % zone_size; - uint64_t remaining_space = get_remaining_space(zone_num); - - ceph_assert(get_write_pointer(zone_num) == write_pointer); - ceph_assert(remaining_space <= length); - increment_write_pointer(zone_num, remaining_space); - - ldout(cct, 40) << " set zone 0x" << std::hex - << zone_num << " write pointer to 0x" << zone_size << std::dec << dendl; - - length -= remaining_space; - ceph_assert(length % zone_size == 0); - - for ( ; length; length -= zone_size) { - increment_write_pointer(++zone_num, zone_size); - ldout(cct, 40) << " set zone 0x" << std::hex - << zone_num << " write pointer to 0x" << zone_size << std::dec << dendl; + ldout(cct, 10) << dendl; + cleaner_lock = _cleaner_lock; + cleaner_cond = _cleaner_cond; + zone_states = std::move(_zone_states); + num_free = 0; + for (size_t i = first_seq_zone_num; i < num_zones; ++i) { + num_free += zone_size - (zone_states[i].write_pointer % zone_size); } + uint64_t conventional_size = first_seq_zone_num * zone_size; + uint64_t sequential_size = size - conventional_size; + ldout(cct, 10) << "free 0x" << std::hex << num_free + << " / 0x" << sequential_size << std::dec + << dendl; } const std::set *ZonedAllocator::get_zones_to_clean(void) @@ -198,16 +182,15 @@ bool ZonedAllocator::low_on_space(void) { ceph_assert(zones_to_clean.empty()); - uint64_t conventional_size = first_seq_zone_num * zone_size; - uint64_t sequential_size = size - conventional_size; uint64_t sequential_num_free = num_free - conventional_size; double free_ratio = static_cast(sequential_num_free) / sequential_size; - ldout(cct, 10) << " free size 0x" << std::hex << sequential_num_free - << " total size 0x" << sequential_size << std::dec + ldout(cct, 10) << " free 0x" << std::hex << sequential_num_free + << "/ 0x" << sequential_size << std::dec << ", free ratio is " << free_ratio << dendl; + ceph_assert(sequential_num_free <= sequential_size); - // TODO: make 0.25 tunable + // TODO: make 0.25 tunable return free_ratio <= 0.25; } @@ -257,18 +240,6 @@ void ZonedAllocator::find_zones_to_clean(void) cleaner_lock->unlock(); } -void ZonedAllocator::init_alloc(std::vector &&_zone_states, - ceph::mutex *_cleaner_lock, - ceph::condition_variable *_cleaner_cond) -{ - lderr(cct) << dendl; - std::lock_guard l(lock); - cleaner_lock = _cleaner_lock; - cleaner_cond = _cleaner_cond; - ldout(cct, 10) << dendl; - zone_states = std::move(_zone_states); -} - void ZonedAllocator::mark_zones_to_clean_free(void) { std::lock_guard l(lock); diff --git a/src/os/bluestore/ZonedAllocator.h b/src/os/bluestore/ZonedAllocator.h index 585e8943cec67..73697e4cfd5f0 100644 --- a/src/os/bluestore/ZonedAllocator.h +++ b/src/os/bluestore/ZonedAllocator.h @@ -32,6 +32,7 @@ class ZonedAllocator : public Allocator { std::atomic num_free; ///< total bytes in freelist uint64_t size; + uint64_t conventional_size, sequential_size; uint64_t block_size; uint64_t zone_size; uint64_t first_seq_zone_num; @@ -91,15 +92,15 @@ class ZonedAllocator : public Allocator { void dump(std::function notify) override; - void init_alloc(std::vector &&_zone_states, - ceph::mutex *_cleaner_lock, - ceph::condition_variable *_cleaner_cond); - const std::set *get_zones_to_clean(void); void mark_zones_to_clean_free(void); - void init_add_free(uint64_t offset, uint64_t length) override; - void init_rm_free(uint64_t offset, uint64_t length) override; + void init_from_zone_pointers( + std::vector &&_zone_states, + ceph::mutex *_cleaner_lock, + ceph::condition_variable *_cleaner_cond); + void init_add_free(uint64_t offset, uint64_t length) override {} + void init_rm_free(uint64_t offset, uint64_t length) override {} void shutdown() override; From 22b06298f1d6ff9f1a7e59fb7766167f23ad379e Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Tue, 7 Sep 2021 11:34:23 -0500 Subject: [PATCH 0397/2866] os/bluestore: reimplement zone backrefs Implement a (zone, offset) -> oid reference. The offset is the first offset referenced by this object in this zone; subsequent extents in the same zone will not generate additional refs. The ref will be removed only when *all* refs for that object to that zone are gone. - minimal metadata per zone (one ref to each object involved - (zone, offset) keys are ordered for efficient sequential cleaning Signed-off-by: Sage Weil --- src/os/bluestore/BlueStore.cc | 159 ++++++++++++++++++----------- src/os/bluestore/BlueStore.h | 50 +++------ src/os/bluestore/bluestore_types.h | 7 +- 3 files changed, 117 insertions(+), 99 deletions(-) diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc index fe59e8eb1ba77..2004f7286457f 100644 --- a/src/os/bluestore/BlueStore.cc +++ b/src/os/bluestore/BlueStore.cc @@ -563,6 +563,24 @@ static int get_key_pool_stat(const string& key, uint64_t* pool_id) return 0; } +#ifdef HAVE_LIBZBD +static void get_zone_offset_key(uint32_t zone, uint64_t offset, std::string *key) +{ + key->clear(); + _key_encode_u32(zone, key); + _key_encode_u64(offset, key); +} + +static int get_key_zone_offset(const string& key, uint32_t *zone, uint64_t *offset) +{ + const char *p = key.c_str(); + if (key.length() < sizeof(uint64_t) + sizeof(uint32_t) + ENCODED_KEY_PREFIX_LEN + 1) + return -1; + _key_decode_u32(p, zone); + _key_decode_u64(p, offset); + return 0; +} +#endif template void _dump_extent_map(CephContext *cct, const BlueStore::ExtentMap &em) @@ -611,6 +629,10 @@ void _dump_onode(CephContext *cct, const BlueStore::Onode& o) << ", " << o.extent_map.spanning_blob_map.size() << " spanning blobs" << dendl; + for (auto& [zone, offset] : o.onode.zone_offset_refs) { + dout(LogLevelV) << __func__ << " zone ref 0x" << std::hex << zone + << " offset 0x" << offset << std::dec << dendl; + } for (auto p = o.onode.attrs.begin(); p != o.onode.attrs.end(); ++p) { @@ -11885,45 +11907,6 @@ void BlueStore::BSPerfTracker::update_from_perfcounters( l_bluestore_commit_lat)); } -#ifdef HAVE_LIBZBD -// For every object we maintain tuple in the -// PREFIX_ZONED_CL_INFO namespace. When a new object written to a zone, we -// insert the corresponding tuple to the database. When an object is truncated, -// we remove the corresponding tuple. When an object is overwritten, we remove -// the old tuple and insert a new tuple corresponding to the new location of the -// object. The cleaner can now identify live objects within the zone -// by enumerating all the keys starting with prefix. -void BlueStore::_zoned_update_cleaning_metadata(TransContext *txc) { - for (const auto &[o, offsets] : txc->zoned_onode_to_offset_map) { - for (auto offset : offsets) { - if (offset > 0) { - bufferlist offset_bl; - encode(offset, offset_bl); - txc->t->set(PREFIX_ZONED_CL_INFO, _zoned_key(offset, &o->oid), offset_bl); - } else { - txc->t->rmkey(PREFIX_ZONED_CL_INFO, _zoned_key(-offset, &o->oid)); - } - } - } -} - -// Given an offset and possibly an oid, returns a key of the form zone_num+oid. -std::string BlueStore::_zoned_key(uint64_t offset, const ghobject_t *oid) { - uint64_t zone_num = offset / bdev->get_zone_size(); - std::string zone_key; - _key_encode_u64(zone_num, &zone_key); - - if (!oid) - return zone_key; - - std::string object_key; - get_object_key(cct, *oid, &object_key); - - return zone_key + object_key; -} - -#endif - void BlueStore::_txc_finalize_kv(TransContext *txc, KeyValueDB::Transaction t) { dout(20) << __func__ << " txc " << txc << std::hex @@ -11974,7 +11957,28 @@ void BlueStore::_txc_finalize_kv(TransContext *txc, KeyValueDB::Transaction t) #ifdef HAVE_LIBZBD if (bdev->is_smr()) { - _zoned_update_cleaning_metadata(txc); + for (auto& i : txc->old_zone_offset_refs) { + for (auto& j : i.second) { + dout(20) << __func__ << " rm ref zone 0x" << std::hex << j.first + << " offset 0x" << j.second << std::dec << dendl; + string key; + get_zone_offset_key(j.first, j.second, &key); + txc->t->rmkey(PREFIX_ZONED_CL_INFO, key); + } + } + for (auto& i : txc->new_zone_offset_refs) { + for (auto& j : i.second) { + // (zone, offset) -> oid + dout(20) << __func__ << " add ref zone 0x" << std::hex << j.first + << " offset 0x" << j.second << std::dec + << " -> " << i.first->oid << dendl; + string key; + get_zone_offset_key(j.first, j.second, &key); + bufferlist v; + encode(i.first->oid, v); + txc->t->set(PREFIX_ZONED_CL_INFO, key, v); + } + } } #endif @@ -14650,6 +14654,26 @@ void BlueStore::_wctx_finish( WriteContext *wctx, set *maybe_unshared_blobs) { +#ifdef HAVE_LIBZBD + if (bdev->is_smr()) { + for (auto& w : wctx->writes) { + for (auto& e : w.b->get_blob().get_extents()) { + if (!e.is_valid()) { + continue; + } + uint32_t zone = e.offset / zone_size; + if (!o->onode.zone_offset_refs.count(zone)) { + uint64_t zoff = e.offset % zone_size; + dout(20) << __func__ << " add ref zone 0x" << std::hex << zone + << " offset 0x" << zoff << std::dec << dendl; + txc->note_write_zone_offset(o, zone, zoff); + } + } + } + } + set zones_with_releases; +#endif + auto oep = wctx->old_extents.begin(); while (oep != wctx->old_extents.end()) { auto &lo = *oep; @@ -14677,6 +14701,12 @@ void BlueStore::_wctx_finish( b->shared_blob->put_ref( e.offset, e.length, &final, unshare_ptr); +#ifdef HAVE_LIBZBD + // we also drop zone ref for shared blob extents + if (bdev->is_smr() && e.is_valid()) { + zones_with_releases.insert(e.offset / zone_size); + } +#endif } if (unshare) { ceph_assert(maybe_unshared_blobs); @@ -14703,6 +14733,11 @@ void BlueStore::_wctx_finish( if (blob.is_compressed()) { txc->statfs_delta.compressed_allocated() -= e.length; } +#ifdef HAVE_LIBZBD + if (bdev->is_smr() && e.is_valid()) { + zones_with_releases.insert(e.offset / zone_size); + } +#endif } if (b->is_spanning() && !b->is_referenced() && lo.blob_empty) { @@ -14712,6 +14747,29 @@ void BlueStore::_wctx_finish( } delete &lo; } + +#ifdef HAVE_LIBZBD + if (!zones_with_releases.empty()) { + // we need to fault the entire extent range in here to determinte if we've dropped + // all refs to a zone. + o->extent_map.fault_range(db, 0, OBJECT_MAX_SIZE); + for (auto& b : o->extent_map.extent_map) { + for (auto& e : b.blob->get_blob().get_extents()) { + if (e.is_valid()) { + zones_with_releases.erase(e.offset / zone_size); + } + } + } + for (auto zone : zones_with_releases) { + auto p = o->onode.zone_offset_refs.find(zone); + if (p != o->onode.zone_offset_refs.end()) { + dout(20) << __func__ << " rm ref zone 0x" << std::hex << zone + << " offset 0x" << p->second << std::dec << dendl; + txc->note_release_zone_offset(o, zone, p->second); + } + } + } +#endif } void BlueStore::_do_write_data( @@ -14973,17 +15031,6 @@ int BlueStore::_do_write( min_alloc_size); } -#ifdef HAVE_LIBZBD - if (bdev->is_smr()) { - if (wctx.old_extents.empty()) { - txc->zoned_note_new_object(o); - } else { - int64_t old_ondisk_offset = wctx.old_extents.begin()->r.begin()->offset; - txc->zoned_note_updated_object(o, old_ondisk_offset); - } - } -#endif - // NB: _wctx_finish() will empty old_extents // so we must do gc estimation before that _wctx_finish(txc, c, o, &wctx); @@ -15117,16 +15164,6 @@ void BlueStore::_do_truncate( o->extent_map.punch_hole(c, offset, length, &wctx.old_extents); o->extent_map.dirty_range(offset, length); -#ifdef HAVE_LIBZBD - if (bdev->is_smr()) { - // On zoned devices, we currently support only removing an object or - // truncating it to zero size, both of which fall through this code path. - ceph_assert(offset == 0 && !wctx.old_extents.empty()); - int64_t ondisk_offset = wctx.old_extents.begin()->r.begin()->offset; - txc->zoned_note_truncated_object(o, ondisk_offset); - } -#endif - _wctx_finish(txc, c, o, &wctx, maybe_unshared_blobs); // if we have shards past EOF, ask for a reshard diff --git a/src/os/bluestore/BlueStore.h b/src/os/bluestore/BlueStore.h index c932ca36b528d..587503fd039d7 100644 --- a/src/os/bluestore/BlueStore.h +++ b/src/os/bluestore/BlueStore.h @@ -1603,16 +1603,13 @@ class BlueStore : public ObjectStore, std::set modified_objects; ///< objects we modified (and need a ref) #ifdef HAVE_LIBZBD - // A map from onode to a vector of object offset. For new objects created - // in the transaction we append the new offset to the vector, for - // overwritten objects we append the negative of the previous ondisk offset - // followed by the new offset, and for truncated objects we append the - // negative of the previous ondisk offset. We need to maintain a vector of - // offsets because *within the same transaction* an object may be truncated - // and then written again, or an object may be overwritten multiple times to - // different zones. See _zoned_update_cleaning_metadata function for how - // this map is used. - std::map> zoned_onode_to_offset_map; + // zone refs to add/remove. each zone ref is a (zone, offset) tuple. The offset + // is the first offset in the zone that the onode touched; subsequent writes + // to that zone do not generate additional refs. This is a bit imprecise but + // is sufficient to generate reasonably sequential reads when doing zone + // cleaning with less metadata than a ref for every extent. + std::map> new_zone_offset_refs; + std::map> old_zone_offset_refs; #endif std::set shared_blobs; ///< these need to be updated/written @@ -1688,28 +1685,13 @@ class BlueStore : public ObjectStore, } #ifdef HAVE_LIBZBD - void zoned_note_new_object(OnodeRef &o) { - auto [_, ok] = zoned_onode_to_offset_map.emplace( - std::pair>(o, {o->zoned_get_ondisk_starting_offset()})); - ceph_assert(ok); + void note_write_zone_offset(OnodeRef& o, uint32_t zone, uint64_t offset) { + o->onode.zone_offset_refs[zone] = offset; + new_zone_offset_refs[o][zone] = offset; } - - void zoned_note_updated_object(OnodeRef &o, int64_t prev_offset) { - int64_t new_offset = o->zoned_get_ondisk_starting_offset(); - auto [it, ok] = zoned_onode_to_offset_map.emplace( - std::pair>(o, {-prev_offset, new_offset})); - if (!ok) { - it->second.push_back(-prev_offset); - it->second.push_back(new_offset); - } - } - - void zoned_note_truncated_object(OnodeRef &o, int64_t offset) { - auto [it, ok] = zoned_onode_to_offset_map.emplace( - std::pair>(o, {-offset})); - if (!ok) { - it->second.push_back(-offset); - } + void note_release_zone_offset(OnodeRef& o, uint32_t zone, uint64_t offset) { + old_zone_offset_refs[o][zone] = offset; + o->onode.zone_offset_refs.erase(zone); } #endif @@ -2421,12 +2403,6 @@ class BlueStore : public ObjectStore, int _setup_block_symlink_or_file(std::string name, std::string path, uint64_t size, bool create); -#ifdef HAVE_LIBZBD - // Functions related to zoned storage. - void _zoned_update_cleaning_metadata(TransContext *txc); - std::string _zoned_key(uint64_t offset, const ghobject_t *oid); -#endif - public: utime_t get_deferred_last_submitted() { std::lock_guard l(deferred_lock); diff --git a/src/os/bluestore/bluestore_types.h b/src/os/bluestore/bluestore_types.h index 656449b19e14a..851278e7e062c 100644 --- a/src/os/bluestore/bluestore_types.h +++ b/src/os/bluestore/bluestore_types.h @@ -960,6 +960,8 @@ struct bluestore_onode_t { uint8_t flags = 0; + std::map zone_offset_refs; ///< (zone, offset) refs to this onode + enum { FLAG_OMAP = 1, ///< object may have omap data FLAG_PGMETA_OMAP = 2, ///< omap data is in meta omap prefix @@ -1031,7 +1033,7 @@ struct bluestore_onode_t { } DENC(bluestore_onode_t, v, p) { - DENC_START(1, 1, p); + DENC_START(2, 1, p); denc_varint(v.nid, p); denc_varint(v.size, p); denc(v.attrs, p); @@ -1040,6 +1042,9 @@ struct bluestore_onode_t { denc_varint(v.expected_object_size, p); denc_varint(v.expected_write_size, p); denc_varint(v.alloc_hint_flags, p); + if (struct_v >= 2) { + denc(v.zone_offset_refs, p); + } DENC_FINISH(p); } void dump(ceph::Formatter *f) const; From a826836dcecfdd42faf061cf9263d9e08c257289 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Tue, 7 Sep 2021 12:36:32 -0500 Subject: [PATCH 0398/2866] os/bluestore: use 64 bit values for zone_state_t Simpler and future-proof for devices with zones >= 4GB. Signed-off-by: Sage Weil --- src/crimson/os/alienstore/CMakeLists.txt | 1 - src/os/CMakeLists.txt | 1 - src/os/bluestore/zoned_types.cc | 24 ------------------ src/os/bluestore/zoned_types.h | 31 +++++++++++++++++------- 4 files changed, 22 insertions(+), 35 deletions(-) delete mode 100644 src/os/bluestore/zoned_types.cc diff --git a/src/crimson/os/alienstore/CMakeLists.txt b/src/crimson/os/alienstore/CMakeLists.txt index b685b94b54f1a..c57c1410778bf 100644 --- a/src/crimson/os/alienstore/CMakeLists.txt +++ b/src/crimson/os/alienstore/CMakeLists.txt @@ -62,7 +62,6 @@ set(alien_store_srcs ${PROJECT_SOURCE_DIR}/src/os/memstore/MemStore.cc) if(WITH_ZBD) list(APPEND alien_store_srcs - ${PROJECT_SOURCE_DIR}/src/os/bluestore/zoned_types.cc ${PROJECT_SOURCE_DIR}/src/os/bluestore/ZonedFreelistManager.cc ${PROJECT_SOURCE_DIR}/src/os/bluestore/ZonedAllocator.cc) endif() diff --git a/src/os/CMakeLists.txt b/src/os/CMakeLists.txt index 377360f9af688..387f150d27b76 100644 --- a/src/os/CMakeLists.txt +++ b/src/os/CMakeLists.txt @@ -40,7 +40,6 @@ endif(WITH_BLUESTORE) if(WITH_ZBD) list(APPEND libos_srcs - bluestore/zoned_types.cc bluestore/ZonedFreelistManager.cc bluestore/ZonedAllocator.cc) endif() diff --git a/src/os/bluestore/zoned_types.cc b/src/os/bluestore/zoned_types.cc deleted file mode 100644 index 549a357fb57b9..0000000000000 --- a/src/os/bluestore/zoned_types.cc +++ /dev/null @@ -1,24 +0,0 @@ -#include "zoned_types.h" - -using ceph::decode; -using ceph::encode; - -std::ostream& operator<<(std::ostream& out, - const zone_state_t& zone_state) { - return out << std::hex - << " dead bytes: 0x" << zone_state.get_num_dead_bytes() - << " write pointer: 0x" << zone_state.get_write_pointer() - << " " << std::dec; -} - -void zone_state_t::encode(ceph::buffer::list &bl) const { - uint64_t v = static_cast(num_dead_bytes) << 32 | write_pointer; - ::encode(v, bl); -} - -void zone_state_t::decode(ceph::buffer::list::const_iterator &p) { - uint64_t v; - ::decode(v, p); - num_dead_bytes = v >> 32; - write_pointer = v; // discard left-most 32 bits -} diff --git a/src/os/bluestore/zoned_types.h b/src/os/bluestore/zoned_types.h index 6ff5d5f313e43..c92f2ada09967 100644 --- a/src/os/bluestore/zoned_types.h +++ b/src/os/bluestore/zoned_types.h @@ -8,19 +8,25 @@ #include "os/kv.h" // Tracks two bits of information about the state of a zone: (1) number of dead -// bytes in a zone and (2) the write pointer. We assume that for now 32 bits is -// enough for the zone capacity and represent these as uint32_t, and we store -// them as a single 64-bit value in RocksDB so that we can use the existing +// bytes in a zone and (2) the write pointer. We use the existing // Int64ArrayMergeOperator for merge and avoid the cost of point queries. // // We use the same struct for an on-disk and in-memory representation of the // state. struct zone_state_t { - uint32_t num_dead_bytes = 0; - uint32_t write_pointer = 0; + uint64_t num_dead_bytes = 0; + uint64_t write_pointer = 0; - void encode(ceph::buffer::list &bl) const; - void decode(ceph::buffer::list::const_iterator &p); + void encode(ceph::buffer::list &bl) const { + using ceph::encode; + encode(write_pointer, bl); + encode(num_dead_bytes, bl); + } + void decode(ceph::buffer::list::const_iterator &p) { + using ceph::decode; + decode(write_pointer, p); + decode(num_dead_bytes, p); + } uint64_t get_num_dead_bytes() const { return num_dead_bytes; @@ -37,8 +43,15 @@ struct zone_state_t { void increment_write_pointer(uint64_t num_bytes) { write_pointer += num_bytes; } -}; -std::ostream& operator<<(std::ostream& out, const zone_state_t& zone_state); + friend std::ostream& operator<<( + std::ostream& out, + const zone_state_t& zone_state) { + return out << std::hex + << " dead bytes: 0x" << zone_state.get_num_dead_bytes() + << " write pointer: 0x" << zone_state.get_write_pointer() + << " " << std::dec; + } +}; #endif From 590f826f908f1548fed4b20eb9c093794d5cde14 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Tue, 7 Sep 2021 13:02:16 -0500 Subject: [PATCH 0399/2866] blk/zoned: add get_zones() to fetch write pointers Signed-off-by: Sage Weil --- src/blk/BlockDevice.h | 3 +++ src/blk/zoned/HMSMRDevice.cc | 20 ++++++++++++++++++++ src/blk/zoned/HMSMRDevice.h | 1 + 3 files changed, 24 insertions(+) diff --git a/src/blk/BlockDevice.h b/src/blk/BlockDevice.h index d08e6e3e48668..14efcbea79f59 100644 --- a/src/blk/BlockDevice.h +++ b/src/blk/BlockDevice.h @@ -209,6 +209,9 @@ class BlockDevice { virtual void reset_zones(const std::set& zones) { ceph_assert(is_smr()); } + virtual std::vector get_zones() { + return std::vector(); + } virtual void aio_submit(IOContext *ioc) = 0; diff --git a/src/blk/zoned/HMSMRDevice.cc b/src/blk/zoned/HMSMRDevice.cc index 518ac41f4e0a8..31e197ea8210f 100644 --- a/src/blk/zoned/HMSMRDevice.cc +++ b/src/blk/zoned/HMSMRDevice.cc @@ -110,3 +110,23 @@ void HMSMRDevice::reset_zones(const std::set& zones) } } } + +std::vector HMSMRDevice::get_zones() +{ + std::vector zones; + unsigned int num_zones = size / zone_size; + zones.resize(num_zones); + + int r = zbd_report_zones(zbd_fd, 0, 0, ZBD_RO_ALL, zones.data(), &num_zones); + if (r != 0) { + derr << __func__ << " zbd_report_zones failed on " << path << ": " + << cpp_strerror(errno) << dendl; + ceph_abort("zbd_report_zones failed"); + } + + std::vector wp(num_zones); + for (unsigned i = 0; i < num_zones; ++i) { + wp[i] = zones[i].wp; + } + return wp; +} diff --git a/src/blk/zoned/HMSMRDevice.h b/src/blk/zoned/HMSMRDevice.h index af490fc4e6247..cfa1310bef64a 100644 --- a/src/blk/zoned/HMSMRDevice.h +++ b/src/blk/zoned/HMSMRDevice.h @@ -45,6 +45,7 @@ class HMSMRDevice final : public KernelDevice { bool is_smr() const final { return true; } void reset_all_zones() override; void reset_zones(const std::set& zones) override; + std::vector get_zones() override; }; #endif //CEPH_BLK_HMSMRDEVICE_H From 7f74551b7bdf13afb0f0d31b18bdafff91df3d82 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Tue, 7 Sep 2021 13:53:06 -0500 Subject: [PATCH 0400/2866] os/bluestore: fix startup vs device write pointers Compare freelist write pointers to the device on startup. Signed-off-by: Sage Weil --- src/os/bluestore/BlueStore.cc | 31 +++++++++++++++++++++++++++++- src/os/bluestore/ZonedAllocator.cc | 2 +- src/os/bluestore/ZonedAllocator.h | 2 +- src/os/bluestore/zoned_types.h | 4 ++-- 4 files changed, 34 insertions(+), 5 deletions(-) diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc index 2004f7286457f..459f7070b1bca 100644 --- a/src/os/bluestore/BlueStore.cc +++ b/src/os/bluestore/BlueStore.cc @@ -5592,7 +5592,36 @@ int BlueStore::_init_alloc() ceph_assert(a); auto f = dynamic_cast(fm); ceph_assert(f); - a->init_from_zone_pointers(f->get_zone_states(db), + vector wp = bdev->get_zones(); + vector zones = f->get_zone_states(db); + ceph_assert(wp.size() == zones.size()); + + // reconcile zone state + auto num_zones = bdev->get_size() / zone_size; + for (unsigned i = first_sequential_zone; i < num_zones; ++i) { + ceph_assert(wp[i] >= i * zone_size); + ceph_assert(wp[i] <= (i + 1) * zone_size); // pos might be at start of next zone + uint64_t p = wp[i] - i * zone_size; + if (zones[i].write_pointer > p) { + derr << __func__ << " zone 0x" << std::hex << i + << " bluestore write pointer 0x" << zones[i].write_pointer + << " > device write pointer 0x" << p + << std::dec << dendl; + ceph_abort("bad write pointer"); + } else if (zones[i].write_pointer < p) { + // this is "normal" in that it can happen after any crash (if we have a + // write in flight but did not manage to commit the transaction) + auto delta = p - zones[i].write_pointer; + dout(1) << __func__ << " zone 0x" << std::hex << i + << " device write pointer 0x" << p + << " > bluestore pointer 0x" << zones[i].write_pointer + << ", advancing 0x" << delta << std::dec << dendl; + zones[i].num_dead_bytes += delta; + zones[i].write_pointer = p; + } + } + + a->init_from_zone_pointers(zones, &zoned_cleaner_lock, &zoned_cleaner_cond); dout(1) << __func__ diff --git a/src/os/bluestore/ZonedAllocator.cc b/src/os/bluestore/ZonedAllocator.cc index 0ec8000e17b6e..03326d2e5fc5b 100644 --- a/src/os/bluestore/ZonedAllocator.cc +++ b/src/os/bluestore/ZonedAllocator.cc @@ -151,7 +151,7 @@ void ZonedAllocator::dump(std::function &&_zone_states, + std::vector _zone_states, ceph::mutex *_cleaner_lock, ceph::condition_variable *_cleaner_cond) { diff --git a/src/os/bluestore/ZonedAllocator.h b/src/os/bluestore/ZonedAllocator.h index 73697e4cfd5f0..903b62fd1a09a 100644 --- a/src/os/bluestore/ZonedAllocator.h +++ b/src/os/bluestore/ZonedAllocator.h @@ -96,7 +96,7 @@ class ZonedAllocator : public Allocator { void mark_zones_to_clean_free(void); void init_from_zone_pointers( - std::vector &&_zone_states, + std::vector _zone_states, ceph::mutex *_cleaner_lock, ceph::condition_variable *_cleaner_cond); void init_add_free(uint64_t offset, uint64_t length) override {} diff --git a/src/os/bluestore/zoned_types.h b/src/os/bluestore/zoned_types.h index c92f2ada09967..93d3138399a0c 100644 --- a/src/os/bluestore/zoned_types.h +++ b/src/os/bluestore/zoned_types.h @@ -14,8 +14,8 @@ // We use the same struct for an on-disk and in-memory representation of the // state. struct zone_state_t { - uint64_t num_dead_bytes = 0; - uint64_t write_pointer = 0; + uint64_t num_dead_bytes = 0; ///< dead bytes deallocated (behind the write pointer) + uint64_t write_pointer = 0; ///< relative offset within the zone void encode(ceph::buffer::list &bl) const { using ceph::encode; From 6dc9233203d5869f6d7e9681e5b37b35afbc5e04 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Tue, 7 Sep 2021 16:19:01 -0500 Subject: [PATCH 0401/2866] os/bluestore/ZonedFreelistManager: whitespace Signed-off-by: Sage Weil --- src/os/bluestore/ZonedFreelistManager.cc | 104 ++++++++++++++--------- 1 file changed, 63 insertions(+), 41 deletions(-) diff --git a/src/os/bluestore/ZonedFreelistManager.cc b/src/os/bluestore/ZonedFreelistManager.cc index 60899939200e7..06fecb60d26fd 100644 --- a/src/os/bluestore/ZonedFreelistManager.cc +++ b/src/os/bluestore/ZonedFreelistManager.cc @@ -31,9 +31,10 @@ using ceph::decode; using ceph::encode; void ZonedFreelistManager::write_zone_state_to_db( - uint64_t zone_num, - const zone_state_t &zone_state, - KeyValueDB::Transaction txn) { + uint64_t zone_num, + const zone_state_t &zone_state, + KeyValueDB::Transaction txn) +{ string key; _key_encode_u64(zone_num, &key); bufferlist bl; @@ -42,9 +43,10 @@ void ZonedFreelistManager::write_zone_state_to_db( } void ZonedFreelistManager::load_zone_state_from_db( - uint64_t zone_num, - zone_state_t &zone_state, - KeyValueDB::Iterator& it) const { + uint64_t zone_num, + zone_state_t &zone_state, + KeyValueDB::Iterator& it) const +{ string k = it->key(); uint64_t zone_num_from_db; _key_decode_u64(k.c_str(), &zone_num_from_db); @@ -55,7 +57,8 @@ void ZonedFreelistManager::load_zone_state_from_db( zone_state.decode(p); } -void ZonedFreelistManager::init_zone_states(KeyValueDB::Transaction txn) { +void ZonedFreelistManager::init_zone_states(KeyValueDB::Transaction txn) +{ dout(10) << __func__ << dendl; for (uint64_t zone_num = 0; zone_num < num_zones; ++zone_num) { zone_state_t zone_state; @@ -63,27 +66,31 @@ void ZonedFreelistManager::init_zone_states(KeyValueDB::Transaction txn) { } } -void ZonedFreelistManager::setup_merge_operator(KeyValueDB *db, string prefix) { +void ZonedFreelistManager::setup_merge_operator(KeyValueDB *db, string prefix) +{ std::shared_ptr merge_op( - new Int64ArrayMergeOperator); + new Int64ArrayMergeOperator); db->set_merge_operator(prefix, merge_op); } ZonedFreelistManager::ZonedFreelistManager( - CephContext* cct, - string meta_prefix, - string info_prefix) + CephContext* cct, + string meta_prefix, + string info_prefix) : FreelistManager(cct), meta_prefix(meta_prefix), info_prefix(info_prefix), - enumerate_zone_num(~0UL) {} + enumerate_zone_num(~0UL) +{ +} int ZonedFreelistManager::create( - uint64_t new_size, - uint64_t granularity, - uint64_t new_zone_size, - uint64_t first_sequential_zone, - KeyValueDB::Transaction txn) { + uint64_t new_size, + uint64_t granularity, + uint64_t new_zone_size, + uint64_t first_sequential_zone, + KeyValueDB::Transaction txn) +{ size = new_size; bytes_per_block = granularity; zone_size = new_zone_size; @@ -131,9 +138,10 @@ int ZonedFreelistManager::create( } int ZonedFreelistManager::init( - KeyValueDB *kvdb, - bool db_in_read_only, - cfg_reader_t cfg_reader) { + KeyValueDB *kvdb, + bool db_in_read_only, + cfg_reader_t cfg_reader) +{ dout(1) << __func__ << dendl; int r = _read_cfg(cfg_reader); if (r != 0) { @@ -152,13 +160,17 @@ int ZonedFreelistManager::init( return 0; } -void ZonedFreelistManager::sync(KeyValueDB* kvdb) {} +void ZonedFreelistManager::sync(KeyValueDB* kvdb) +{ +} -void ZonedFreelistManager::shutdown() { +void ZonedFreelistManager::shutdown() +{ dout(1) << __func__ << dendl; } -void ZonedFreelistManager::enumerate_reset() { +void ZonedFreelistManager::enumerate_reset() +{ std::lock_guard l(lock); dout(1) << __func__ << dendl; @@ -174,9 +186,10 @@ void ZonedFreelistManager::enumerate_reset() { // current semantics of the call and appears to work fine with the clients of // this call. bool ZonedFreelistManager::enumerate_next( - KeyValueDB *kvdb, - uint64_t *offset, - uint64_t *length) { + KeyValueDB *kvdb, + uint64_t *offset, + uint64_t *length) +{ std::lock_guard l(lock); // starting case @@ -207,7 +220,8 @@ bool ZonedFreelistManager::enumerate_next( return true; } -void ZonedFreelistManager::dump(KeyValueDB *kvdb) { +void ZonedFreelistManager::dump(KeyValueDB *kvdb) +{ enumerate_reset(); uint64_t offset, length; while (enumerate_next(kvdb, &offset, &length)) { @@ -218,9 +232,10 @@ void ZonedFreelistManager::dump(KeyValueDB *kvdb) { // Advances the write pointer and writes the updated write pointer to database. void ZonedFreelistManager::allocate( - uint64_t offset, - uint64_t length, - KeyValueDB::Transaction txn) { + uint64_t offset, + uint64_t length, + KeyValueDB::Transaction txn) +{ dout(10) << __func__ << " 0x" << std::hex << offset << "~" << length << dendl; uint64_t zone_num = offset / zone_size; zone_state_t zone_state; @@ -235,9 +250,10 @@ void ZonedFreelistManager::allocate( // which zones to clean -- the ones with most dead bytes are good candidates // since they require less I/O. void ZonedFreelistManager::release( - uint64_t offset, - uint64_t length, - KeyValueDB::Transaction txn) { + uint64_t offset, + uint64_t length, + KeyValueDB::Transaction txn) +{ dout(10) << __func__ << " 0x" << std::hex << offset << "~" << length << dendl; uint64_t zone_num = offset / zone_size; zone_state_t zone_state; @@ -246,8 +262,9 @@ void ZonedFreelistManager::release( } void ZonedFreelistManager::get_meta( - uint64_t target_size, - std::vector>* res) const { + uint64_t target_size, + std::vector>* res) const +{ // We do not support expanding devices for now. ceph_assert(target_size == 0); res->emplace_back("zfm_size", stringify(size)); @@ -258,7 +275,8 @@ void ZonedFreelistManager::get_meta( } std::vector ZonedFreelistManager::get_zone_states( - KeyValueDB *kvdb) const { + KeyValueDB *kvdb) const +{ std::vector zone_states; auto p = kvdb->get_iterator(info_prefix); uint64_t zone_num = 0; @@ -272,7 +290,8 @@ std::vector ZonedFreelistManager::get_zone_states( // TODO: The following function is copied almost verbatim from // BitmapFreelistManager. Eliminate duplication. -int ZonedFreelistManager::_read_cfg(cfg_reader_t cfg_reader) { +int ZonedFreelistManager::_read_cfg(cfg_reader_t cfg_reader) +{ dout(1) << __func__ << dendl; string err; @@ -313,7 +332,8 @@ int ZonedFreelistManager::_read_cfg(cfg_reader_t cfg_reader) { } std::set ZonedFreelistManager::get_cleaning_in_progress_zones( - KeyValueDB *kvdb) const { + KeyValueDB *kvdb) const +{ bufferlist bl; std::set zones_to_clean; if (kvdb->get(meta_prefix, CLEANING_IN_PROGRESS_KEY, &bl) == 0) { @@ -323,7 +343,8 @@ std::set ZonedFreelistManager::get_cleaning_in_progress_zones( } void ZonedFreelistManager::mark_zones_to_clean_free( - const std::set& zones_to_clean, KeyValueDB *kvdb) { + const std::set& zones_to_clean, KeyValueDB *kvdb) +{ dout(10) << __func__ << dendl; KeyValueDB::Transaction txn = kvdb->get_transaction(); @@ -342,7 +363,8 @@ void ZonedFreelistManager::mark_zones_to_clean_free( // if there is a key CLEANING_IN_PROGRESS_KEY in the meta_prefix namespace, and // if so, will read the zones and resume cleaning. void ZonedFreelistManager::mark_zones_to_clean_in_progress( - const std::set& zones_to_clean, KeyValueDB *kvdb) { + const std::set& zones_to_clean, KeyValueDB *kvdb) +{ dout(10) << __func__ << dendl; bufferlist bl; From beef6bbdf1552f2955412dbd9b867af7f81de05e Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Tue, 7 Sep 2021 16:36:01 -0500 Subject: [PATCH 0402/2866] os/bluestore: correct zoned freelist when device write pointers are ahead If we find that the device's write pointers are ahead of ours, then we need to advance the allocator *and* update the ondisk freelist so that we don't see the adjustment on the next bluestore restart too. (Actually, we could perhaps not manage the freelist at all and rely entirely on the device's write pointers and a metadata scan on startup w/ the null freelist mode, but it is cheap to track, and I like having some redundancy so that we see if/when the device does something unexpected.) Signed-off-by: Sage Weil --- src/os/bluestore/BlueStore.cc | 34 ++++++++++++++++++++++++++++++++-- src/os/bluestore/BlueStore.h | 3 ++- 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc index 459f7070b1bca..f1269c757b5f1 100644 --- a/src/os/bluestore/BlueStore.cc +++ b/src/os/bluestore/BlueStore.cc @@ -5578,7 +5578,7 @@ int BlueStore::_create_alloc() return 0; } -int BlueStore::_init_alloc() +int BlueStore::_init_alloc(std::map *zone_adjustments) { int r = _create_alloc(); if (r < 0) { @@ -5616,6 +5616,7 @@ int BlueStore::_init_alloc() << " device write pointer 0x" << p << " > bluestore pointer 0x" << zones[i].write_pointer << ", advancing 0x" << delta << std::dec << dendl; + (*zone_adjustments)[zones[i].write_pointer] = delta; zones[i].num_dead_bytes += delta; zones[i].write_pointer = p; } @@ -5633,6 +5634,7 @@ int BlueStore::_init_alloc() << ", free 0x" << shared_alloc.a->get_free() << ", fragmentation " << shared_alloc.a->get_fragmentation() << std::dec << dendl; + return 0; } #endif @@ -5692,6 +5694,25 @@ int BlueStore::_init_alloc() return 0; } +void BlueStore::_post_init_alloc(const std::map& zone_adjustments) +{ +#ifdef HAVE_LIBZBD + assert(bdev->is_smr()); + dout(1) << __func__ << " adjusting freelist based on device write pointers" << dendl; + auto f = dynamic_cast(fm); + ceph_assert(f); + KeyValueDB::Transaction t = db->get_transaction(); + for (auto& i : zone_adjustments) { + // allocate AND release since this gap is now dead space + // note that the offset is imprecise, but only need to select the zone + f->allocate(i.first, i.second, t); + f->release(i.first, i.second, t); + } + int r = db->submit_transaction_sync(t); + ceph_assert(r == 0); +#endif +} + void BlueStore::_close_alloc() { ceph_assert(bdev); @@ -6077,6 +6098,10 @@ int BlueStore::_open_db_and_around(bool read_only, bool to_repair) } } + // SMR devices may require a freelist adjustment, but that can only happen after + // the db is read-write. we'll stash pending changes here. + std::map zone_adjustments; + int r = _open_path(); if (r < 0) return r; @@ -6114,7 +6139,7 @@ int BlueStore::_open_db_and_around(bool read_only, bool to_repair) if (r < 0) goto out_db; - r = _init_alloc(); + r = _init_alloc(&zone_adjustments); if (r < 0) goto out_fm; @@ -6130,6 +6155,11 @@ int BlueStore::_open_db_and_around(bool read_only, bool to_repair) goto out_alloc; } + if (!read_only && !zone_adjustments.empty()) { + // for SMR devices that have freelist mismatch with device write pointers + _post_init_alloc(zone_adjustments); + } + // when function is called in repair mode (to_repair=true) we skip db->open()/create() // we can't change bluestore allocation so no need to invlidate allocation-file if (fm->is_null_manager() && !read_only && !to_repair) { diff --git a/src/os/bluestore/BlueStore.h b/src/os/bluestore/BlueStore.h index 587503fd039d7..110389fd545ec 100644 --- a/src/os/bluestore/BlueStore.h +++ b/src/os/bluestore/BlueStore.h @@ -2394,7 +2394,8 @@ class BlueStore : public ObjectStore, void _close_fm(); int _write_out_fm_meta(uint64_t target_size); int _create_alloc(); - int _init_alloc(); + int _init_alloc(std::map *zone_adjustments); + void _post_init_alloc(const std::map& zone_adjustments); void _close_alloc(); int _open_collections(); void _fsck_collections(int64_t* errors); From 2e6948be1f620cbb9c3313a1d15bfd96dd361e95 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Tue, 7 Sep 2021 16:56:14 -0500 Subject: [PATCH 0403/2866] os/bluestore: duplicate zone refs when cloning When we clone an object, add zone refs pointing to the new object for any relevant extents. This is only half of the solution; we will want to make sure the cleaner is clever enough to keep shared blobs shared when they are relocated. Signed-off-by: Sage Weil --- src/os/bluestore/BlueStore.cc | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc index f1269c757b5f1..3744a3b9a1ea4 100644 --- a/src/os/bluestore/BlueStore.cc +++ b/src/os/bluestore/BlueStore.cc @@ -15781,6 +15781,38 @@ int BlueStore::_do_clone_range( _dump_onode<30>(cct, *newo); oldo->extent_map.dup(this, txc, c, oldo, newo, srcoff, length, dstoff); + +#ifdef HAVE_LIBZBD + if (bdev->is_smr()) { + // duplicate the refs for the shared region. + Extent dummy(dstoff); + for (auto e = newo->extent_map.extent_map.lower_bound(dummy); + e != newo->extent_map.extent_map.end(); + ++e) { + if (e->logical_offset >= dstoff + length) { + break; + } + for (auto& ex : e->blob->get_blob().get_extents()) { + // note that we may introduce a new extent reference that is + // earlier than the first zone ref. we allow this since it is + // a lot of work to avoid and has marginal impact on cleaning + // performance. + if (!ex.is_valid()) { + continue; + } + uint32_t zone = ex.offset / zone_size; + if (!newo->onode.zone_offset_refs.count(zone)) { + uint64_t zoff = ex.offset % zone_size; + dout(20) << __func__ << " add ref zone 0x" << std::hex << zone + << " offset 0x" << zoff << std::dec + << " -> " << newo->oid << dendl; + txc->note_write_zone_offset(newo, zone, zoff); + } + } + } + } +#endif + _dump_onode<30>(cct, *oldo); _dump_onode<30>(cct, *newo); return 0; From 94d4b65ec12d495f2988312b55289ce86722152f Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Wed, 8 Sep 2021 10:30:24 -0500 Subject: [PATCH 0404/2866] os/bluestore: fsck smr allocations (verify num_dead_bytes, alloc past write pointer) Signed-off-by: Sage Weil --- src/os/bluestore/BlueStore.cc | 204 ++++++++++++++++++++---------- src/os/bluestore/ZonedAllocator.h | 4 + 2 files changed, 144 insertions(+), 64 deletions(-) diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc index 3744a3b9a1ea4..169ecc2257235 100644 --- a/src/os/bluestore/BlueStore.cc +++ b/src/os/bluestore/BlueStore.cc @@ -9218,77 +9218,153 @@ int BlueStore::_fsck_on_open(BlueStore::FSCKDepth depth, bool repair) } } - dout(1) << __func__ << " checking freelist vs allocated" << dendl; // skip freelist vs allocated compare when we have Null fm if (!fm->is_null_manager()) { - fm->enumerate_reset(); - uint64_t offset, length; - while (fm->enumerate_next(db, &offset, &length)) { - bool intersects = false; - apply_for_bitset_range( - offset, length, alloc_size, used_blocks, - [&](uint64_t pos, mempool_dynamic_bitset &bs) { - ceph_assert(pos < bs.size()); - if (bs.test(pos) && !bluefs_used_blocks.test(pos)) { - if (offset == SUPER_RESERVED && - length == min_alloc_size - SUPER_RESERVED) { - // this is due to the change just after luminous to min_alloc_size - // granularity allocations, and our baked in assumption at the top - // of _fsck that 0~round_up_to(SUPER_RESERVED,min_alloc_size) is used - // (vs luminous's round_up_to(SUPER_RESERVED,block_size)). harmless, - // since we will never allocate this region below min_alloc_size. - dout(10) << __func__ << " ignoring free extent between SUPER_RESERVED" - << " and min_alloc_size, 0x" << std::hex << offset << "~" - << length << std::dec << dendl; + dout(1) << __func__ << " checking freelist vs allocated" << dendl; +#ifdef HAVE_LIBZBD + if (freelist_type == "zoned") { + // verify per-zone state + // - verify no allocations beyond write pointer + // - verify num_dead_bytes count (neither allocated nor + // free space past the write pointer) + auto a = dynamic_cast(shared_alloc.a); + auto num_zones = bdev->get_size() / zone_size; + + // mark the free space past the write pointer + for (uint32_t zone = first_sequential_zone; zone < num_zones; ++zone) { + auto wp = a->get_write_pointer(zone); + uint64_t offset = zone_size * zone + wp; + uint64_t length = zone_size - wp; + if (!length) { + continue; + } + bool intersects = false; + dout(10) << " marking zone 0x" << std::hex << zone + << " region after wp 0x" << offset << "~" << length + << std::dec << dendl; + apply_for_bitset_range( + offset, length, alloc_size, used_blocks, + [&](uint64_t pos, mempool_dynamic_bitset &bs) { + if (bs.test(pos)) { + derr << "fsck error: zone 0x" << std::hex << zone + << " has used space at 0x" << pos * alloc_size + << " beyond write pointer 0x" << wp + << std::dec << dendl; + intersects = true; } else { - intersects = true; - if (repair) { - repairer.fix_false_free(db, fm, - pos * min_alloc_size, - min_alloc_size); - } + bs.set(pos); } - } else { - bs.set(pos); - } - } - ); - if (intersects) { - derr << "fsck error: free extent 0x" << std::hex << offset - << "~" << length << std::dec - << " intersects allocated blocks" << dendl; - ++errors; - } - } - fm->enumerate_reset(); - size_t count = used_blocks.count(); - if (used_blocks.size() != count) { - ceph_assert(used_blocks.size() > count); - used_blocks.flip(); - size_t start = used_blocks.find_first(); - while (start != decltype(used_blocks)::npos) { - size_t cur = start; - while (true) { - size_t next = used_blocks.find_next(cur); - if (next != cur + 1) { - ++errors; - derr << "fsck error: leaked extent 0x" << std::hex - << ((uint64_t)start * fm->get_alloc_size()) << "~" - << ((cur + 1 - start) * fm->get_alloc_size()) << std::dec - << dendl; - if (repair) { - repairer.fix_leaked(db, - fm, - start * min_alloc_size, - (cur + 1 - start) * min_alloc_size); + } + ); + if (intersects) { + ++errors; + } + } + + used_blocks.flip(); + + // skip conventional zones + uint64_t pos = (first_sequential_zone * zone_size) / min_alloc_size - 1; + pos = used_blocks.find_next(pos); + + uint64_t zone_dead = 0; + for (uint32_t zone = first_sequential_zone; + zone < num_zones; + ++zone, zone_dead = 0) { + while (pos != decltype(used_blocks)::npos && + (pos * min_alloc_size) / zone_size == zone) { + dout(40) << " zone 0x" << std::hex << zone + << " dead 0x" << (pos * min_alloc_size) << "~" << min_alloc_size + << std::dec << dendl; + zone_dead += min_alloc_size; + pos = used_blocks.find_next(pos); + } + dout(20) << " zone 0x" << std::hex << zone << " dead is 0x" << zone_dead + << std::dec << dendl; + // cross-check dead bytes against zone state + if (a->get_dead_bytes(zone) != zone_dead) { + derr << "fsck error: zone 0x" << std::hex << zone << " has 0x" << zone_dead + << " dead bytes but freelist says 0x" << a->get_dead_bytes(zone) + << dendl; + ++errors; + // TODO: repair + } + } + used_blocks.flip(); + } else +#endif + { + fm->enumerate_reset(); + uint64_t offset, length; + while (fm->enumerate_next(db, &offset, &length)) { + bool intersects = false; + apply_for_bitset_range( + offset, length, alloc_size, used_blocks, + [&](uint64_t pos, mempool_dynamic_bitset &bs) { + ceph_assert(pos < bs.size()); + if (bs.test(pos) && !bluefs_used_blocks.test(pos)) { + if (offset == SUPER_RESERVED && + length == min_alloc_size - SUPER_RESERVED) { + // this is due to the change just after luminous to min_alloc_size + // granularity allocations, and our baked in assumption at the top + // of _fsck that 0~round_up_to(SUPER_RESERVED,min_alloc_size) is used + // (vs luminous's round_up_to(SUPER_RESERVED,block_size)). harmless, + // since we will never allocate this region below min_alloc_size. + dout(10) << __func__ << " ignoring free extent between SUPER_RESERVED" + << " and min_alloc_size, 0x" << std::hex << offset << "~" + << length << std::dec << dendl; + } else { + intersects = true; + if (repair) { + repairer.fix_false_free(db, fm, + pos * min_alloc_size, + min_alloc_size); + } + } + } else { + bs.set(pos); } - start = next; - break; } - cur = next; + ); + if (intersects) { + derr << "fsck error: free extent 0x" << std::hex << offset + << "~" << length << std::dec + << " intersects allocated blocks" << dendl; + ++errors; } - } - used_blocks.flip(); + } + fm->enumerate_reset(); + + // check for leaked extents + size_t count = used_blocks.count(); + if (used_blocks.size() != count) { + ceph_assert(used_blocks.size() > count); + used_blocks.flip(); + size_t start = used_blocks.find_first(); + while (start != decltype(used_blocks)::npos) { + size_t cur = start; + while (true) { + size_t next = used_blocks.find_next(cur); + if (next != cur + 1) { + ++errors; + derr << "fsck error: leaked extent 0x" << std::hex + << ((uint64_t)start * fm->get_alloc_size()) << "~" + << ((cur + 1 - start) * fm->get_alloc_size()) << std::dec + << dendl; + if (repair) { + repairer.fix_leaked(db, + fm, + start * min_alloc_size, + (cur + 1 - start) * min_alloc_size); + } + start = next; + break; + } + cur = next; + } + } + used_blocks.flip(); + } } } } diff --git a/src/os/bluestore/ZonedAllocator.h b/src/os/bluestore/ZonedAllocator.h index 903b62fd1a09a..872ee29325aa8 100644 --- a/src/os/bluestore/ZonedAllocator.h +++ b/src/os/bluestore/ZonedAllocator.h @@ -80,6 +80,10 @@ class ZonedAllocator : public Allocator { return "zoned"; } + uint64_t get_dead_bytes(uint32_t zone) { + return zone_states[zone].num_dead_bytes; + } + int64_t allocate( uint64_t want_size, uint64_t alloc_unit, uint64_t max_alloc_size, int64_t hint, PExtentVector *extents) override; From 247ed18ab3af13308d33349cb7616987ef294215 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Wed, 8 Sep 2021 10:45:50 -0500 Subject: [PATCH 0405/2866] ceph_test_objectstore: add trivial fsck test Signed-off-by: Sage Weil --- src/test/objectstore/store_test.cc | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/test/objectstore/store_test.cc b/src/test/objectstore/store_test.cc index d942eebd784d5..c634b22f9bfbe 100644 --- a/src/test/objectstore/store_test.cc +++ b/src/test/objectstore/store_test.cc @@ -276,6 +276,17 @@ TEST_P(StoreTest, TrivialRemount) { ASSERT_EQ(0, r); } +TEST_P(StoreTest, TrivialRemountFsck) { + if(string(GetParam()) != "bluestore") + return; + int r = store->umount(); + ASSERT_EQ(0, r); + r = store->fsck(false); + ASSERT_EQ(0, r); + r = store->mount(); + ASSERT_EQ(0, r); +} + TEST_P(StoreTest, SimpleRemount) { coll_t cid; ghobject_t hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP))); From 59763171c5cd8a89600bfcbfcef1ec6a75d5cd37 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Wed, 8 Sep 2021 11:37:26 -0500 Subject: [PATCH 0406/2866] os/bluestore: simplify zone to clean selection Only pick one zone to clean based on the current. Since the best victim may change (maybe another zone gets a bunch of releases and new dead bytes!) there is no reason (yet) to explicitly avoid the victim zone during allocation. There is also no need to track which zones we are cleaning on disk because we can choose to clean from any zone at any time, and in general want to clean from the best candidate at the time, not the one that looked the best some time in the past. Signed-off-by: Sage Weil --- src/blk/BlockDevice.h | 7 +- src/blk/zoned/HMSMRDevice.cc | 13 ++- src/blk/zoned/HMSMRDevice.h | 2 +- src/os/bluestore/BlueStore.cc | 45 ++++------ src/os/bluestore/ZonedAllocator.cc | 107 ++++++----------------- src/os/bluestore/ZonedAllocator.h | 13 +-- src/os/bluestore/ZonedFreelistManager.cc | 44 ++-------- src/os/bluestore/ZonedFreelistManager.h | 9 +- 8 files changed, 62 insertions(+), 178 deletions(-) diff --git a/src/blk/BlockDevice.h b/src/blk/BlockDevice.h index 14efcbea79f59..01fa0d4451205 100644 --- a/src/blk/BlockDevice.h +++ b/src/blk/BlockDevice.h @@ -205,11 +205,14 @@ class BlockDevice { ceph_assert(is_smr()); return conventional_region_size; } - virtual void reset_all_zones() {} - virtual void reset_zones(const std::set& zones) { + virtual void reset_all_zones() { + ceph_assert(is_smr()); + } + virtual void reset_zone(uint64_t zone) { ceph_assert(is_smr()); } virtual std::vector get_zones() { + ceph_assert(is_smr()); return std::vector(); } diff --git a/src/blk/zoned/HMSMRDevice.cc b/src/blk/zoned/HMSMRDevice.cc index 31e197ea8210f..416eae4e49fce 100644 --- a/src/blk/zoned/HMSMRDevice.cc +++ b/src/blk/zoned/HMSMRDevice.cc @@ -100,14 +100,13 @@ void HMSMRDevice::reset_all_zones() zbd_reset_zones(zbd_fd, conventional_region_size, 0); } -void HMSMRDevice::reset_zones(const std::set& zones) +void HMSMRDevice::reset_zone(uint64_t zone) { - dout(10) << __func__ << " 0x" << std::hex << zones << std::dec << dendl; - for (auto zone_num : zones) { - if (zbd_reset_zones(zbd_fd, zone_num * zone_size, zone_size) != 0) { - derr << __func__ << " resetting zone failed for zone 0x" << std::hex - << zone_num << std::dec << dendl; - } + dout(10) << __func__ << " zone 0x" << std::hex << zone << std::dec << dendl; + if (zbd_reset_zones(zbd_fd, zone * zone_size, zone_size) != 0) { + derr << __func__ << " resetting zone failed for zone 0x" << std::hex + << zone << std::dec << dendl; + ceph_abort("zbd_reset_zones failed"); } } diff --git a/src/blk/zoned/HMSMRDevice.h b/src/blk/zoned/HMSMRDevice.h index cfa1310bef64a..b6df3200c6521 100644 --- a/src/blk/zoned/HMSMRDevice.h +++ b/src/blk/zoned/HMSMRDevice.h @@ -44,7 +44,7 @@ class HMSMRDevice final : public KernelDevice { // smr-specific methods bool is_smr() const final { return true; } void reset_all_zones() override; - void reset_zones(const std::set& zones) override; + void reset_zone(uint64_t zone) override; std::vector get_zones() override; }; diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc index 169ecc2257235..32f86bedc3d62 100644 --- a/src/os/bluestore/BlueStore.cc +++ b/src/os/bluestore/BlueStore.cc @@ -5622,9 +5622,7 @@ int BlueStore::_init_alloc(std::map *zone_adjustments) } } - a->init_from_zone_pointers(zones, - &zoned_cleaner_lock, - &zoned_cleaner_cond); + a->init_from_zone_pointers(zones); dout(1) << __func__ << " loaded zone pointers: " << std::hex @@ -12840,26 +12838,14 @@ void BlueStore::_kv_finalize_thread() } #ifdef HAVE_LIBZBD -void BlueStore::_zoned_cleaner_start() { +void BlueStore::_zoned_cleaner_start() +{ dout(10) << __func__ << dendl; - - auto f = dynamic_cast(fm); - ceph_assert(f); - - auto zones_to_clean = f->get_cleaning_in_progress_zones(db); - if (!zones_to_clean.empty()) { - dout(10) << __func__ << " resuming cleaning after unclean shutdown." << dendl; - for (auto zone_num : zones_to_clean) { - _zoned_clean_zone(zone_num); - } - bdev->reset_zones(zones_to_clean); - f->mark_zones_to_clean_free(zones_to_clean, db); - } - zoned_cleaner_thread.create("bstore_zcleaner"); } -void BlueStore::_zoned_cleaner_stop() { +void BlueStore::_zoned_cleaner_stop() +{ dout(10) << __func__ << dendl; { std::unique_lock l{zoned_cleaner_lock}; @@ -12877,7 +12863,8 @@ void BlueStore::_zoned_cleaner_stop() { dout(10) << __func__ << " done" << dendl; } -void BlueStore::_zoned_cleaner_thread() { +void BlueStore::_zoned_cleaner_thread() +{ dout(10) << __func__ << " start" << dendl; std::unique_lock l{zoned_cleaner_lock}; ceph_assert(!zoned_cleaner_started); @@ -12888,8 +12875,8 @@ void BlueStore::_zoned_cleaner_thread() { auto f = dynamic_cast(fm); ceph_assert(f); while (true) { - const auto *zones_to_clean = a->get_zones_to_clean(); - if (!zones_to_clean) { + auto zone_to_clean = a->pick_zone_to_clean(); + if (zone_to_clean < 0) { if (zoned_cleaner_stop) { break; } @@ -12898,13 +12885,10 @@ void BlueStore::_zoned_cleaner_thread() { dout(20) << __func__ << " wake" << dendl; } else { l.unlock(); - f->mark_zones_to_clean_in_progress(*zones_to_clean, db); - for (auto zone_num : *zones_to_clean) { - _zoned_clean_zone(zone_num); - } - bdev->reset_zones(*zones_to_clean); - f->mark_zones_to_clean_free(*zones_to_clean, db); - a->mark_zones_to_clean_free(); + _zoned_clean_zone(zone_to_clean); + bdev->reset_zone(zone_to_clean); + f->mark_zone_to_clean_free(zone_to_clean, db); + //a->mark_zone_to_clean_free(); l.lock(); } } @@ -12912,7 +12896,8 @@ void BlueStore::_zoned_cleaner_thread() { zoned_cleaner_started = false; } -void BlueStore::_zoned_clean_zone(uint64_t zone_num) { +void BlueStore::_zoned_clean_zone(uint64_t zone_num) +{ dout(10) << __func__ << " cleaning zone " << zone_num << dendl; // TODO: (1) copy live objects from zone_num to a new zone, (2) issue a RESET // ZONE operation to the device for the corresponding zone. diff --git a/src/os/bluestore/ZonedAllocator.cc b/src/os/bluestore/ZonedAllocator.cc index 03326d2e5fc5b..6933dd972d891 100644 --- a/src/os/bluestore/ZonedAllocator.cc +++ b/src/os/bluestore/ZonedAllocator.cc @@ -34,8 +34,7 @@ ZonedAllocator::ZonedAllocator(CephContext* cct, zone_size(_zone_size), first_seq_zone_num(_first_sequential_zone), starting_zone_num(first_seq_zone_num), - num_zones(size / zone_size), - num_zones_to_clean(0) + num_zones(size / zone_size) { ldout(cct, 10) << " size 0x" << std::hex << size << " zone size 0x" << zone_size << std::dec @@ -68,14 +67,7 @@ int64_t ZonedAllocator::allocate( << std::hex << want_size << std::dec << dendl; uint64_t zone_num = starting_zone_num; - auto p = zones_to_clean.lower_bound(zone_num); for ( ; zone_num < num_zones; ++zone_num) { - if (p != zones_to_clean.cend() && *p == zone_num) { - ldout(cct, 10) << " skipping zone 0x" << std::hex << zone_num - << " because it is being cleaned" << std::dec << dendl; - ++p; - continue; - } if (fits(want_size, zone_num)) { break; } @@ -110,8 +102,6 @@ int64_t ZonedAllocator::allocate( << " and zone offset 0x" << (offset % zone_size) << std::dec << dendl; - find_zones_to_clean(); - extents->emplace_back(bluestore_pextent_t(offset, want_size)); return want_size; } @@ -151,15 +141,11 @@ void ZonedAllocator::dump(std::function _zone_states, - ceph::mutex *_cleaner_lock, - ceph::condition_variable *_cleaner_cond) + std::vector _zone_states) { // this is called once, based on the device's zone pointers std::lock_guard l(lock); ldout(cct, 10) << dendl; - cleaner_lock = _cleaner_lock; - cleaner_cond = _cleaner_cond; zone_states = std::move(_zone_states); num_free = 0; for (size_t i = first_seq_zone_num; i < num_zones; ++i) { @@ -172,16 +158,34 @@ void ZonedAllocator::init_from_zone_pointers( << dendl; } -const std::set *ZonedAllocator::get_zones_to_clean(void) +int64_t ZonedAllocator::pick_zone_to_clean(void) { - ldout(cct, 10) << dendl; - return num_zones_to_clean ? &zones_to_clean : nullptr; + int32_t best = -1; + int64_t best_score = 0; + for (size_t i = first_seq_zone_num; i < num_zones; ++i) { + int64_t score = zone_states[i].num_dead_bytes; + // discount by remaining space so we will tend to clean full zones + score -= (zone_size - zone_states[i].write_pointer) / 2; + if (score > 0 && (best < 0 || score > best_score)) { + best = i; + best_score = score; + } + } + if (best >= 0) { + ldout(cct, 10) << " zone 0x" << std::hex << best << " with score 0x" << best_score + << ": 0x" << zone_states[best].num_dead_bytes + << " dead and 0x" + << zone_states[best].write_pointer - zone_states[best].num_dead_bytes + << " live bytes" << std::dec << dendl; + } else { + ldout(cct, 10) << " no zones found that are good cleaning candidates" << dendl; + } + return best; } bool ZonedAllocator::low_on_space(void) { - ceph_assert(zones_to_clean.empty()); - + std::lock_guard l(lock); uint64_t sequential_num_free = num_free - conventional_size; double free_ratio = static_cast(sequential_num_free) / sequential_size; @@ -194,67 +198,6 @@ bool ZonedAllocator::low_on_space(void) return free_ratio <= 0.25; } -void ZonedAllocator::find_zones_to_clean(void) -{ - ldout(cct, 40) << dendl; - - if (num_zones_to_clean || !low_on_space()) - return; - - ceph_assert(zones_to_clean.empty()); - - // TODO: make this tunable; handle the case when there aren't this many zones - // to clean. - const int64_t num_zones_to_clean_at_once = 1; - - std::vector idx(num_zones); - std::iota(idx.begin(), idx.end(), 0); - - if (cct->_conf->subsys.should_gather()) { - for (size_t i = 0; i < zone_states.size(); ++i) { - dout(40) << " zone 0x" << std::hex << i << std::dec << " " - << zone_states[i] << dendl; - } - } - - std::partial_sort(idx.begin(), idx.begin() + num_zones_to_clean_at_once, idx.end(), - [this](uint64_t i1, uint64_t i2) { - return zone_states[i1].num_dead_bytes > zone_states[i2].num_dead_bytes; - }); - - ldout(cct, 10) << " the zone that needs cleaning is 0x" - << std::hex << *idx.begin() << " num_dead_bytes = 0x" - << zone_states[*idx.begin()].num_dead_bytes - << std::dec - << dendl; - - zones_to_clean = {idx.begin(), idx.begin() + num_zones_to_clean_at_once}; - num_zones_to_clean = num_zones_to_clean_at_once; - - // TODO: handle the case of disk being full. - ceph_assert(!zones_to_clean.empty()); - ceph_assert(num_zones_to_clean != 0); - - cleaner_lock->lock(); - cleaner_cond->notify_one(); - cleaner_lock->unlock(); -} - -void ZonedAllocator::mark_zones_to_clean_free(void) -{ - std::lock_guard l(lock); - ldout(cct, 10) << dendl; - for (auto zone_num : zones_to_clean) { - ldout(cct, 10) << " zone 0x" << std::hex << zone_num - << " is now clean" << std::dec << dendl; - num_free += zone_states[zone_num].write_pointer; - zone_states[zone_num].num_dead_bytes = 0; - zone_states[zone_num].write_pointer = 0; - } - zones_to_clean.clear(); - num_zones_to_clean = 0; -} - void ZonedAllocator::shutdown() { ldout(cct, 1) << dendl; diff --git a/src/os/bluestore/ZonedAllocator.h b/src/os/bluestore/ZonedAllocator.h index 872ee29325aa8..4dccb9f38062e 100644 --- a/src/os/bluestore/ZonedAllocator.h +++ b/src/os/bluestore/ZonedAllocator.h @@ -39,11 +39,6 @@ class ZonedAllocator : public Allocator { uint64_t starting_zone_num; uint64_t num_zones; std::vector zone_states; - std::set zones_to_clean; - std::atomic num_zones_to_clean; - - ceph::mutex *cleaner_lock = nullptr; - ceph::condition_variable *cleaner_cond = nullptr; inline uint64_t get_offset(uint64_t zone_num) const { return zone_num * zone_size + get_write_pointer(zone_num); @@ -96,13 +91,10 @@ class ZonedAllocator : public Allocator { void dump(std::function notify) override; - const std::set *get_zones_to_clean(void); - void mark_zones_to_clean_free(void); + int64_t pick_zone_to_clean(void); void init_from_zone_pointers( - std::vector _zone_states, - ceph::mutex *_cleaner_lock, - ceph::condition_variable *_cleaner_cond); + std::vector _zone_states); void init_add_free(uint64_t offset, uint64_t length) override {} void init_rm_free(uint64_t offset, uint64_t length) override {} @@ -110,7 +102,6 @@ class ZonedAllocator : public Allocator { private: bool low_on_space(void); - void find_zones_to_clean(void); }; #endif diff --git a/src/os/bluestore/ZonedFreelistManager.cc b/src/os/bluestore/ZonedFreelistManager.cc index 06fecb60d26fd..60e72c2cd2727 100644 --- a/src/os/bluestore/ZonedFreelistManager.cc +++ b/src/os/bluestore/ZonedFreelistManager.cc @@ -331,46 +331,14 @@ int ZonedFreelistManager::_read_cfg(cfg_reader_t cfg_reader) return 0; } -std::set ZonedFreelistManager::get_cleaning_in_progress_zones( - KeyValueDB *kvdb) const -{ - bufferlist bl; - std::set zones_to_clean; - if (kvdb->get(meta_prefix, CLEANING_IN_PROGRESS_KEY, &bl) == 0) { - decode(zones_to_clean, bl); - } - return zones_to_clean; -} - -void ZonedFreelistManager::mark_zones_to_clean_free( - const std::set& zones_to_clean, KeyValueDB *kvdb) -{ - dout(10) << __func__ << dendl; - - KeyValueDB::Transaction txn = kvdb->get_transaction(); - for (auto zone_num : zones_to_clean) { - ldout(cct, 10) << __func__ << " zone " << zone_num << " is now clean in DB" << dendl; - - zone_state_t zone_state; - write_zone_state_to_db(zone_num, zone_state, txn); - } - txn->rmkey(meta_prefix, CLEANING_IN_PROGRESS_KEY); - kvdb->submit_transaction_sync(txn); -} - -// Marks the zones currently being cleaned in the db. Should be called before -// starting the cleaning. If we crash mid-cleaning, the recovery code will check -// if there is a key CLEANING_IN_PROGRESS_KEY in the meta_prefix namespace, and -// if so, will read the zones and resume cleaning. -void ZonedFreelistManager::mark_zones_to_clean_in_progress( - const std::set& zones_to_clean, KeyValueDB *kvdb) +void ZonedFreelistManager::mark_zone_to_clean_free( + uint64_t zone, + KeyValueDB *kvdb) { - dout(10) << __func__ << dendl; + dout(10) << __func__ << " zone 0x" << std::hex << zone << std::dec << dendl; - bufferlist bl; - encode(zones_to_clean, bl); - KeyValueDB::Transaction txn = kvdb->get_transaction(); - txn->set(meta_prefix, CLEANING_IN_PROGRESS_KEY, bl); + zone_state_t zone_state; + write_zone_state_to_db(zone, zone_state, txn); kvdb->submit_transaction_sync(txn); } diff --git a/src/os/bluestore/ZonedFreelistManager.h b/src/os/bluestore/ZonedFreelistManager.h index 7ad0723369857..e60a40222eb4f 100644 --- a/src/os/bluestore/ZonedFreelistManager.h +++ b/src/os/bluestore/ZonedFreelistManager.h @@ -22,8 +22,6 @@ using cfg_reader_t = std::function; -const std::string CLEANING_IN_PROGRESS_KEY = "cleaning_in_progress"; - class ZonedFreelistManager : public FreelistManager { std::string meta_prefix; ///< device size, zone size, etc. std::string info_prefix; ///< per zone write pointer, dead bytes @@ -104,11 +102,8 @@ class ZonedFreelistManager : public FreelistManager { std::vector>*) const override; std::vector get_zone_states(KeyValueDB *kvdb) const; - std::set get_cleaning_in_progress_zones(KeyValueDB *kvdb) const; - void mark_zones_to_clean_free(const std::set& zones_to_clean, - KeyValueDB *kvdb); - void mark_zones_to_clean_in_progress(const std::set& zones_to_clean, - KeyValueDB *kvdb); + + void mark_zone_to_clean_free(uint64_t zone, KeyValueDB *kvdb); }; #endif From d492fbf775ca02005389fc48158773bd52747dfb Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Thu, 9 Sep 2021 08:06:19 -0500 Subject: [PATCH 0407/2866] os/bluestore: report mismatch write pointer during fsck Signed-off-by: Sage Weil --- src/os/bluestore/BlueStore.cc | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc index 32f86bedc3d62..8899981b85114 100644 --- a/src/os/bluestore/BlueStore.cc +++ b/src/os/bluestore/BlueStore.cc @@ -5606,8 +5606,7 @@ int BlueStore::_init_alloc(std::map *zone_adjustments) derr << __func__ << " zone 0x" << std::hex << i << " bluestore write pointer 0x" << zones[i].write_pointer << " > device write pointer 0x" << p - << std::dec << dendl; - ceph_abort("bad write pointer"); + << std::dec << " -- VERY SUSPICIOUS!" << dendl; } else if (zones[i].write_pointer < p) { // this is "normal" in that it can happen after any crash (if we have a // write in flight but did not manage to commit the transaction) @@ -8720,6 +8719,30 @@ int BlueStore::_fsck_on_open(BlueStore::FSCKDepth depth, bool repair) dout(1) << __func__ << " debug abort" << dendl; goto out_scan; } + +#ifdef HAVE_LIBZBD + if (bdev->is_smr()) { + auto a = dynamic_cast(shared_alloc.a); + ceph_assert(a); + auto f = dynamic_cast(fm); + ceph_assert(f); + vector wp = bdev->get_zones(); + vector zones = f->get_zone_states(db); + ceph_assert(wp.size() == zones.size()); + auto num_zones = bdev->get_size() / zone_size; + for (unsigned i = first_sequential_zone; i < num_zones; ++i) { + uint64_t p = wp[i] == (i + 1) * zone_size ? zone_size : wp[i] % zone_size; + if (zones[i].write_pointer > p) { + derr << "fsck error: zone 0x" << std::hex << i + << " bluestore write pointer 0x" << zones[i].write_pointer + << " > device write pointer 0x" << p + << std::dec << dendl; + ++errors; + } + } + } +#endif + // walk PREFIX_OBJ { dout(1) << __func__ << " walking object keyspace" << dendl; From f04f8cc8c8bb5dc76ca562cffd4727b26696ba4e Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Thu, 9 Sep 2021 09:59:25 -0500 Subject: [PATCH 0408/2866] ceph_test_objectstore: skip failing tests on SMR Signed-off-by: Sage Weil --- src/test/objectstore/store_test.cc | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/test/objectstore/store_test.cc b/src/test/objectstore/store_test.cc index c634b22f9bfbe..ca9b273397862 100644 --- a/src/test/objectstore/store_test.cc +++ b/src/test/objectstore/store_test.cc @@ -51,6 +51,8 @@ typedef boost::mt11213b gen_type; const uint64_t DEF_STORE_TEST_BLOCKDEV_SIZE = 10240000000; #define dout_context g_ceph_context +bool smr = false; + static bool bl_eq(bufferlist& expected, bufferlist& actual) { if (expected.contents_equal(actual)) @@ -1109,6 +1111,10 @@ void StoreTest::doCompressionTest() TEST_P(StoreTest, CompressionTest) { if (string(GetParam()) != "bluestore") return; + if (smr) { + cout << "TODO: need to adjust statfs check for smr" << std::endl; + return; + } SetVal(g_conf(), "bluestore_compression_algorithm", "snappy"); SetVal(g_conf(), "bluestore_compression_mode", "force"); @@ -8362,6 +8368,10 @@ namespace { TEST_P(StoreTestSpecificAUSize, BluestoreRepairTest) { if (string(GetParam()) != "bluestore") return; + if (smr) { + cout << "TODO: repair mismatched write pointer (+ dead bytes mismatch)" << std::endl; + return; + } const size_t offs_base = 65536 / 2; @@ -9530,6 +9540,13 @@ int main(int argc, char **argv) { CINIT_FLAG_NO_DEFAULT_CONFIG_FILE); common_init_finish(g_ceph_context); + for (auto& i : args) { + if (i == "--smr"s) { + derr << "smr" << dendl; + smr = true; + } + } + // make sure we can adjust any config settings g_ceph_context->_conf._clear_safe_to_start_threads(); From ab8ca4368602489149bf3e3960007d21cae665aa Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Thu, 9 Sep 2021 11:15:22 -0500 Subject: [PATCH 0409/2866] os/bluestore: refactor object key helpers a bit We want methods that work on partial keys. Signed-off-by: Sage Weil --- src/os/bluestore/BlueStore.cc | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc index 8899981b85114..8977421111404 100644 --- a/src/os/bluestore/BlueStore.cc +++ b/src/os/bluestore/BlueStore.cc @@ -379,20 +379,12 @@ static const char *_key_decode_prefix(const char *p, ghobject_t *oid) #define ENCODED_KEY_PREFIX_LEN (1 + 8 + 4) -template -static int get_key_object(const S& key, ghobject_t *oid) +static int _get_key_object(const char *p, ghobject_t *oid) { int r; - const char *p = key.c_str(); - - if (key.length() < ENCODED_KEY_PREFIX_LEN) - return -1; p = _key_decode_prefix(p, oid); - if (key.length() == ENCODED_KEY_PREFIX_LEN) - return -2; - r = decode_escaped(p, &oid->hobj.nspace); if (r < 0) return -2; @@ -437,10 +429,19 @@ static int get_key_object(const S& key, ghobject_t *oid) } template -static void get_object_key(CephContext *cct, const ghobject_t& oid, S *key) +static int get_key_object(const S& key, ghobject_t *oid) { - key->clear(); + if (key.length() < ENCODED_KEY_PREFIX_LEN) + return -1; + if (key.length() == ENCODED_KEY_PREFIX_LEN) + return -2; + const char *p = key.c_str(); + return _get_key_object(p, oid); +} +template +static void _get_object_key(const ghobject_t& oid, S *key) +{ size_t max_len = ENCODED_KEY_PREFIX_LEN + (oid.hobj.nspace.length() * 3 + 1) + (oid.hobj.get_key().length() * 3 + 1) + @@ -475,6 +476,13 @@ static void get_object_key(CephContext *cct, const ghobject_t& oid, S *key) _key_encode_u64(oid.generation, key); key->push_back(ONODE_KEY_SUFFIX); +} + +template +static void get_object_key(CephContext *cct, const ghobject_t& oid, S *key) +{ + key->clear(); + _get_object_key(oid, key); // sanity check if (true) { From ffd3ca10fe38b89f6de3a3b2836b171a3347a9f0 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Thu, 9 Sep 2021 11:16:29 -0500 Subject: [PATCH 0410/2866] os/bluestore: include object in zone ref keys We may have multiple objects referencing the same offset in the same zone if the object gets cloned. Signed-off-by: Sage Weil --- src/os/bluestore/BlueStore.cc | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc index 8977421111404..afc2fd77b790d 100644 --- a/src/os/bluestore/BlueStore.cc +++ b/src/os/bluestore/BlueStore.cc @@ -572,20 +572,33 @@ static int get_key_pool_stat(const string& key, uint64_t* pool_id) } #ifdef HAVE_LIBZBD -static void get_zone_offset_key(uint32_t zone, uint64_t offset, std::string *key) +static void get_zone_offset_object_key( + uint32_t zone, + uint64_t offset, + ghobject_t oid, + std::string *key) { key->clear(); _key_encode_u32(zone, key); _key_encode_u64(offset, key); + _get_object_key(oid, key); } -static int get_key_zone_offset(const string& key, uint32_t *zone, uint64_t *offset) +static int get_key_zone_offset_object( + const string& key, + uint32_t *zone, + uint64_t *offset, + ghobject_t *oid) { const char *p = key.c_str(); if (key.length() < sizeof(uint64_t) + sizeof(uint32_t) + ENCODED_KEY_PREFIX_LEN + 1) return -1; - _key_decode_u32(p, zone); - _key_decode_u64(p, offset); + p = _key_decode_u32(p, zone); + p = _key_decode_u64(p, offset); + int r = _get_key_object(p, oid); + if (r < 0) { + return r; + } return 0; } #endif @@ -12124,9 +12137,10 @@ void BlueStore::_txc_finalize_kv(TransContext *txc, KeyValueDB::Transaction t) for (auto& i : txc->old_zone_offset_refs) { for (auto& j : i.second) { dout(20) << __func__ << " rm ref zone 0x" << std::hex << j.first - << " offset 0x" << j.second << std::dec << dendl; + << " offset 0x" << j.second << std::dec + << " -> " << i.first->oid << dendl; string key; - get_zone_offset_key(j.first, j.second, &key); + get_zone_offset_object_key(j.first, j.second, i.first->oid, &key); txc->t->rmkey(PREFIX_ZONED_CL_INFO, key); } } @@ -12137,9 +12151,8 @@ void BlueStore::_txc_finalize_kv(TransContext *txc, KeyValueDB::Transaction t) << " offset 0x" << j.second << std::dec << " -> " << i.first->oid << dendl; string key; - get_zone_offset_key(j.first, j.second, &key); + get_zone_offset_object_key(j.first, j.second, i.first->oid, &key); bufferlist v; - encode(i.first->oid, v); txc->t->set(PREFIX_ZONED_CL_INFO, key, v); } } From c8803dd982c10a16b8028909da85f030800e943c Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Thu, 9 Sep 2021 11:59:48 -0500 Subject: [PATCH 0411/2866] os/bluestore: fsck verify zone refs Signed-off-by: Sage Weil --- src/os/bluestore/BlueStore.cc | 129 +++++++++++++++++++++++++++++++++- src/os/bluestore/BlueStore.h | 5 +- 2 files changed, 131 insertions(+), 3 deletions(-) diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc index afc2fd77b790d..0a9aa4f864be8 100644 --- a/src/os/bluestore/BlueStore.cc +++ b/src/os/bluestore/BlueStore.cc @@ -7699,6 +7699,8 @@ BlueStore::OnodeRef BlueStore::fsck_check_objects_shallow( &ctx.expected_pool_statfs[pool_id] : &ctx.expected_store_statfs; + map zone_first_offsets; // for zoned/smr devices + dout(10) << __func__ << " " << oid << dendl; OnodeRef o; o.reset(Onode::decode(c, oid, key, value)); @@ -7754,6 +7756,22 @@ BlueStore::OnodeRef BlueStore::fsck_check_objects_shallow( ceph_assert(l.blob); const bluestore_blob_t& blob = l.blob->get_blob(); +#ifdef HAVE_LIBZBD + if (bdev->is_smr() && depth != FSCK_SHALLOW) { + for (auto& e : blob.get_extents()) { + if (e.is_valid()) { + uint32_t zone = e.offset / zone_size; + uint64_t offset = e.offset % zone_size; + auto p = zone_first_offsets.find(zone); + if (p == zone_first_offsets.end() || p->second > offset) { + // FIXME: use interator for guided insert? + zone_first_offsets[zone] = offset; + } + } + } + } +#endif + auto& ref = ref_map[l.blob]; if (ref.is_empty()) { uint32_t min_release_size = blob.get_release_size(min_alloc_size); @@ -7872,6 +7890,34 @@ BlueStore::OnodeRef BlueStore::fsck_check_objects_shallow( } } } + +#ifdef HAVE_LIBZBD + if (bdev->is_smr() && depth != FSCK_SHALLOW) { + for (auto& [zone, first_offset] : zone_first_offsets) { + auto p = (*ctx.zone_refs)[zone].find(oid); + if (p != (*ctx.zone_refs)[zone].end()) { + if (first_offset < p->second) { + dout(20) << " slightly wonky zone ref 0x" << std::hex << zone + << " offset 0x" << p->second + << " but first offset is 0x" << first_offset + << "; this can happen due to clone_range" + << dendl; + } else { + dout(20) << " good zone ref 0x" << std::hex << zone << " offset 0x" << p->second + << " <= first offset 0x" << first_offset + << std::dec << dendl; + } + (*ctx.zone_refs)[zone].erase(p); + } else { + derr << "fsck error: " << oid << " references zone 0x" << std::hex << zone + << " but there is no zone ref" << std::dec << dendl; + // FIXME: add repair + ++errors; + } + } + } +#endif + if (broken) { derr << "fsck error: " << oid << " - " << broken << " zombie spanning blob(s) found, the first one: " @@ -8014,6 +8060,7 @@ class ShallowFSCKThreadPool : public ThreadPool batch->num_spanning_blobs, nullptr, // used_blocks nullptr, //used_omap_head + nullptr, sb_info_lock, *sb_info, batch->expected_store_statfs, @@ -8281,7 +8328,8 @@ void BlueStore::_fsck_check_object_omap(FSCKDepth depth, } } -void BlueStore::_fsck_check_objects(FSCKDepth depth, +void BlueStore::_fsck_check_objects( + FSCKDepth depth, BlueStore::FSCK_ObjectCtx& ctx) { auto& errors = ctx.errors; @@ -8316,7 +8364,7 @@ void BlueStore::_fsck_check_objects(FSCKDepth depth, thread_pool.start(); } - //fill global if not overriden below + // fill global if not overriden below CollectionRef c; int64_t pool_id = -1; spg_t pgid; @@ -8646,6 +8694,9 @@ int BlueStore::_fsck_on_open(BlueStore::FSCKDepth depth, bool repair) sb_info_map_t sb_info; + /// map of oid -> (first_)offset for each zone + std::vector> zone_refs; // FIXME: this may be a lot of RAM! + uint64_t num_objects = 0; uint64_t num_extents = 0; uint64_t num_blobs = 0; @@ -8761,6 +8812,44 @@ int BlueStore::_fsck_on_open(BlueStore::FSCKDepth depth, bool repair) ++errors; } } + + if (depth != FSCK_SHALLOW) { + // load zone refs + zone_refs.resize(bdev->get_size() / zone_size); + it = db->get_iterator(PREFIX_ZONED_CL_INFO, KeyValueDB::ITERATOR_NOCACHE); + if (it) { + for (it->lower_bound(string()); + it->valid(); + it->next()) { + uint32_t zone = 0; + uint64_t offset = 0; + ghobject_t oid; + string key = it->key(); + int r = get_key_zone_offset_object(key, &zone, &offset, &oid); + if (r < 0) { + derr << "fsck error: invalid zone ref key " << pretty_binary_string(key) + << dendl; + if (repair) { + repairer.remove_key(db, PREFIX_ZONED_CL_INFO, key); + } + ++errors; + continue; + } + dout(30) << " zone ref 0x" << std::hex << zone << " offset 0x" << offset + << " -> " << std::dec << oid << dendl; + if (zone_refs[zone].count(oid)) { + derr << "fsck error: second zone ref in zone 0x" << std::hex << zone + << " offset 0x" << offset << std::dec << " for " << oid << dendl; + if (repair) { + repairer.remove_key(db, PREFIX_ZONED_CL_INFO, key); + } + ++errors; + continue; + } + zone_refs[zone][oid] = offset; + } + } + } } #endif @@ -8778,6 +8867,7 @@ int BlueStore::_fsck_on_open(BlueStore::FSCKDepth depth, bool repair) num_spanning_blobs, &used_blocks, &used_omap_head, + &zone_refs, //no need for the below lock when in non-shallow mode as // there is no multithreading in this case depth == FSCK_SHALLOW ? &sb_info_lock : nullptr, @@ -8789,6 +8879,20 @@ int BlueStore::_fsck_on_open(BlueStore::FSCKDepth depth, bool repair) _fsck_check_objects(depth, ctx); } +#ifdef HAVE_LIBZBD + if (bdev->is_smr() && depth != FSCK_SHALLOW) { + dout(1) << __func__ << " checking for leaked zone refs" << dendl; + for (uint32_t zone = 0; zone < zone_refs.size(); ++zone) { + for (auto& [oid, offset] : zone_refs[zone]) { + derr << "fsck error: stray zone ref 0x" << std::hex << zone + << " offset 0x" << offset << " -> " << std::dec << oid << dendl; + // FIXME: add repair + ++errors; + } + } + } +#endif + dout(1) << __func__ << " checking shared_blobs" << dendl; it = db->get_iterator(PREFIX_SHARED_BLOB, KeyValueDB::ITERATOR_NOCACHE); if (it) { @@ -16022,6 +16126,27 @@ int BlueStore::_rename(TransContext *txc, // and read newo's metadata via the old name). txc->note_modified_object(oldo); +#ifdef HAVE_LIBZBD + if (bdev->is_smr()) { + // adjust zone refs + for (auto& [zone, offset] : newo->onode.zone_offset_refs) { + dout(20) << __func__ << " rm ref zone 0x" << std::hex << zone + << " offset 0x" << offset << std::dec + << " -> " << oldo->oid << dendl; + string key; + get_zone_offset_object_key(zone, offset, oldo->oid, &key); + txc->t->rmkey(PREFIX_ZONED_CL_INFO, key); + + dout(20) << __func__ << " add ref zone 0x" << std::hex << zone + << " offset 0x" << offset << std::dec + << " -> " << newo->oid << dendl; + get_zone_offset_object_key(zone, offset, newo->oid, &key); + bufferlist v; + txc->t->set(PREFIX_ZONED_CL_INFO, key, v); + } + } +#endif + out: dout(10) << __func__ << " " << c->cid << " " << old_oid << " -> " << new_oid << " = " << r << dendl; diff --git a/src/os/bluestore/BlueStore.h b/src/os/bluestore/BlueStore.h index 110389fd545ec..ed78a25ad1552 100644 --- a/src/os/bluestore/BlueStore.h +++ b/src/os/bluestore/BlueStore.h @@ -3400,6 +3400,7 @@ class BlueStore : public ObjectStore, mempool_dynamic_bitset* used_blocks; uint64_t_btree_t* used_omap_head; + std::vector> *zone_refs; ceph::mutex* sb_info_lock; sb_info_map_t& sb_info; @@ -3417,7 +3418,8 @@ class BlueStore : public ObjectStore, uint64_t& _num_spanning_blobs, mempool_dynamic_bitset* _ub, uint64_t_btree_t* _used_omap_head, - ceph::mutex* _sb_info_lock, + std::vector> *_zone_refs, + ceph::mutex* _sb_info_lock, sb_info_map_t& _sb_info, store_statfs_t& _store_statfs, per_pool_statfs& _pool_statfs, @@ -3431,6 +3433,7 @@ class BlueStore : public ObjectStore, num_spanning_blobs(_num_spanning_blobs), used_blocks(_ub), used_omap_head(_used_omap_head), + zone_refs(_zone_refs), sb_info_lock(_sb_info_lock), sb_info(_sb_info), expected_store_statfs(_store_statfs), From c33e47af58b9e724a4b2f966c8c76cbe749eb6a9 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Thu, 9 Sep 2021 12:08:55 -0500 Subject: [PATCH 0412/2866] os/bluestore: disable cleaner thread until it is implemented Signed-off-by: Sage Weil --- src/os/bluestore/BlueStore.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc index 0a9aa4f864be8..93fc3546c36b1 100644 --- a/src/os/bluestore/BlueStore.cc +++ b/src/os/bluestore/BlueStore.cc @@ -12989,12 +12989,14 @@ void BlueStore::_kv_finalize_thread() void BlueStore::_zoned_cleaner_start() { dout(10) << __func__ << dendl; + return; // temporarily disable cleaner until it actually works zoned_cleaner_thread.create("bstore_zcleaner"); } void BlueStore::_zoned_cleaner_stop() { dout(10) << __func__ << dendl; + return; // temporarily disable cleaner until it actually works { std::unique_lock l{zoned_cleaner_lock}; while (!zoned_cleaner_started) { @@ -13046,7 +13048,7 @@ void BlueStore::_zoned_cleaner_thread() void BlueStore::_zoned_clean_zone(uint64_t zone_num) { - dout(10) << __func__ << " cleaning zone " << zone_num << dendl; + dout(10) << __func__ << " cleaning zone 0x" << std::hex << zone_num << std::dec << dendl; // TODO: (1) copy live objects from zone_num to a new zone, (2) issue a RESET // ZONE operation to the device for the corresponding zone. } From f893d11d1d740b5395356ab91a9645c04373cb5f Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Fri, 10 Sep 2021 13:50:19 -0500 Subject: [PATCH 0413/2866] ceph_test_objectstore: skip tests that don't work on SMR Signed-off-by: Sage Weil --- src/test/objectstore/store_test.cc | 89 +++++++++++++++++++++++++++++- 1 file changed, 87 insertions(+), 2 deletions(-) diff --git a/src/test/objectstore/store_test.cc b/src/test/objectstore/store_test.cc index ca9b273397862..39272aec9fe4e 100644 --- a/src/test/objectstore/store_test.cc +++ b/src/test/objectstore/store_test.cc @@ -1347,7 +1347,11 @@ TEST_P(StoreTest, SimpleObjectTest) { TEST_P(StoreTestSpecificAUSize, ReproBug41901Test) { if(string(GetParam()) != "bluestore") return; - + if (smr) { + cout << "SKIP (smr)" << std::endl; + return; + } + SetVal(g_conf(), "bluestore_max_blob_size", "524288"); SetVal(g_conf(), "bluestore_debug_enforce_settings", "hdd"); g_conf().apply_changes(nullptr); @@ -1442,6 +1446,10 @@ TEST_P(StoreTestSpecificAUSize, ReproBug41901Test) { TEST_P(StoreTestSpecificAUSize, BluestoreStatFSTest) { if(string(GetParam()) != "bluestore") return; + if (smr) { + cout << "TODO: fix this for smr" << std::endl; + return; + } SetVal(g_conf(), "bluestore_block_db_path", ""); StartDeferred(65536); SetVal(g_conf(), "bluestore_compression_mode", "force"); @@ -1972,6 +1980,10 @@ TEST_P(StoreTestSpecificAUSize, BluestoreStatFSTest) { TEST_P(StoreTestSpecificAUSize, BluestoreFragmentedBlobTest) { if(string(GetParam()) != "bluestore") return; + if (smr) { + cout << "TODO: fix this for smr" << std::endl; + return; + } SetVal(g_conf(), "bluestore_block_db_path", ""); StartDeferred(0x10000); @@ -6744,6 +6756,11 @@ void doMany4KWritesTest(ObjectStore* store, TEST_P(StoreTestSpecificAUSize, Many4KWritesTest) { if (string(GetParam()) != "bluestore") return; + if (smr) { + cout << "SKIP: no deferred; assertions around res_stat.allocated don't apply" + << std::endl; + return; + } StartDeferred(0x10000); @@ -6754,6 +6771,11 @@ TEST_P(StoreTestSpecificAUSize, Many4KWritesTest) { TEST_P(StoreTestSpecificAUSize, Many4KWritesNoCSumTest) { if (string(GetParam()) != "bluestore") return; + if (smr) { + cout << "SKIP: no deferred; assertions around res_stat.allocated don't apply" + << std::endl; + return; + } StartDeferred(0x10000); SetVal(g_conf(), "bluestore_csum_type", "none"); g_ceph_context->_conf.apply_changes(nullptr); @@ -6765,6 +6787,11 @@ TEST_P(StoreTestSpecificAUSize, Many4KWritesNoCSumTest) { TEST_P(StoreTestSpecificAUSize, TooManyBlobsTest) { if (string(GetParam()) != "bluestore") return; + if (smr) { + cout << "SKIP: no deferred; assertions around res_stat.allocated don't apply" + << std::endl; + return; + } StartDeferred(0x10000); const unsigned max_object = 4*1024*1024; doMany4KWritesTest(store.get(), 1, 1000, max_object, 4*1024, 0); @@ -7065,6 +7092,10 @@ TEST_P(StoreTestSpecificAUSize, DeferredOnBigOverwrite) { if (string(GetParam()) != "bluestore") return; + if (smr) { + cout << "SKIP: no deferred" << std::endl; + return; + } size_t block_size = 4096; StartDeferred(block_size); @@ -7500,6 +7531,10 @@ TEST_P(StoreTestSpecificAUSize, DeferredOnBigOverwrite2) { if (string(GetParam()) != "bluestore") return; + if (smr) { + cout << "SKIP: no deferred" << std::endl; + return; + } size_t block_size = 4096; StartDeferred(block_size); @@ -7572,6 +7607,10 @@ TEST_P(StoreTestSpecificAUSize, DeferredOnBigOverwrite3) { if (string(GetParam()) != "bluestore") return; + if (smr) { + cout << "SKIP: no deferred" << std::endl; + return; + } size_t block_size = 4096; StartDeferred(block_size); @@ -7644,6 +7683,10 @@ TEST_P(StoreTestSpecificAUSize, DeferredDifferentChunks) { if (string(GetParam()) != "bluestore") return; + if (smr) { + cout << "SKIP: no deferred" << std::endl; + return; + } size_t alloc_size = 4096; size_t large_object_size = 1 * 1024 * 1024; @@ -7743,6 +7786,10 @@ TEST_P(StoreTestSpecificAUSize, BlobReuseOnOverwriteReverse) { if (string(GetParam()) != "bluestore") return; + if (smr) { + cout << "SKIP: no overwrite" << std::endl; + return; + } size_t block_size = 4096; StartDeferred(block_size); @@ -7924,6 +7971,10 @@ TEST_P(StoreTestSpecificAUSize, BlobReuseOnSmallOverwrite) { if (string(GetParam()) != "bluestore") return; + if (smr) { + cout << "SKIP: no overwrite" << std::endl; + return; + } size_t block_size = 4096; StartDeferred(block_size); @@ -8173,6 +8224,10 @@ TEST_P(StoreTestSpecificAUSize, garbageCollection) { int write_offset = buf_len; if (string(GetParam()) != "bluestore") return; + if (smr) { + cout << "SKIP: assertions about allocations need to be adjusted" << std::endl; + return; + } #define WRITE_AT(offset, _length) {\ ObjectStore::Transaction t;\ @@ -8579,7 +8634,10 @@ TEST_P(StoreTestSpecificAUSize, BluestoreRepairTest) { TEST_P(StoreTestSpecificAUSize, BluestoreBrokenZombieRepairTest) { if (string(GetParam()) != "bluestore") return; - + if (smr) { + cout << "SKIP: smr repair is different" << std::endl; + return; + } SetVal(g_conf(), "bluestore_fsck_on_mount", "false"); SetVal(g_conf(), "bluestore_fsck_on_umount", "false"); @@ -8649,6 +8707,10 @@ TEST_P(StoreTestSpecificAUSize, BluestoreBrokenZombieRepairTest) { TEST_P(StoreTestSpecificAUSize, BluestoreBrokenNoSharedBlobRepairTest) { if (string(GetParam()) != "bluestore") return; + if (smr) { + cout << "SKIP: smr repair is different" << std::endl; + return; + } SetVal(g_conf(), "bluestore_fsck_on_mount", "false"); SetVal(g_conf(), "bluestore_fsck_on_umount", "false"); @@ -9233,6 +9295,10 @@ TEST_P(StoreTestSpecificAUSize, ReproNoBlobMultiTest) { if(string(GetParam()) != "bluestore") return; + if (smr) { + cout << "SKIP (FIXME): bluestore gc does not seem to do the trick here" << std::endl; + return; + } SetVal(g_conf(), "bluestore_block_db_create", "true"); SetVal(g_conf(), "bluestore_block_db_size", "4294967296"); @@ -9349,6 +9415,10 @@ void doManySetAttr(ObjectStore* store, TEST_P(StoreTestSpecificAUSize, SpilloverTest) { if (string(GetParam()) != "bluestore") return; + if (smr) { + cout << "SKIP: (FIXME?) adjust me for smr at some point?" << std::endl; + return; + } SetVal(g_conf(), "bluestore_block_db_create", "true"); SetVal(g_conf(), "bluestore_block_db_size", "3221225472"); @@ -9377,6 +9447,10 @@ TEST_P(StoreTestSpecificAUSize, SpilloverTest) { TEST_P(StoreTestSpecificAUSize, SpilloverFixedTest) { if (string(GetParam()) != "bluestore") return; + if (smr) { + cout << "SKIP: (FIXME?) adjust me for smr at some point?" << std::endl; + return; + } SetVal(g_conf(), "bluestore_block_db_create", "true"); SetVal(g_conf(), "bluestore_block_db_size", "3221225472"); @@ -9401,6 +9475,10 @@ TEST_P(StoreTestSpecificAUSize, SpilloverFixedTest) { TEST_P(StoreTestSpecificAUSize, SpilloverFixed2Test) { if (string(GetParam()) != "bluestore") return; + if (smr) { + cout << "SKIP: (FIXME?) adjust me for smr at some point?" << std::endl; + return; + } SetVal(g_conf(), "bluestore_block_db_create", "true"); SetVal(g_conf(), "bluestore_block_db_size", "3221225472"); @@ -9427,6 +9505,10 @@ TEST_P(StoreTestSpecificAUSize, SpilloverFixed2Test) { TEST_P(StoreTestSpecificAUSize, SpilloverFixed3Test) { if (string(GetParam()) != "bluestore") return; + if (smr) { + cout << "SKIP: (FIXME?) adjust me for smr at some point?" << std::endl; + return; + } SetVal(g_conf(), "bluestore_block_db_create", "true"); SetVal(g_conf(), "bluestore_block_db_size", "3221225472"); @@ -9450,6 +9532,9 @@ TEST_P(StoreTestSpecificAUSize, SpilloverFixed3Test) { TEST_P(StoreTestSpecificAUSize, Ticket45195Repro) { if (string(GetParam()) != "bluestore") return; + if (smr) { + return; + } SetVal(g_conf(), "bluestore_default_buffered_write", "true"); SetVal(g_conf(), "bluestore_max_blob_size", "65536"); From e26a6d9a53ed934278efb3a068a3febd82748f49 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Fri, 10 Sep 2021 15:25:36 -0500 Subject: [PATCH 0414/2866] test/objectstore/run_smr_bluestore_test.sh Needs sudo/root to set up. Requires tcmu-runner and targetcli be installed (and a sufficiently new kernel, not sure what the exact version is!). Signed-off-by: Sage Weil --- .../objectstore/run_smr_bluestore_test.sh | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 src/test/objectstore/run_smr_bluestore_test.sh diff --git a/src/test/objectstore/run_smr_bluestore_test.sh b/src/test/objectstore/run_smr_bluestore_test.sh new file mode 100644 index 0000000000000..22017b0c5cb60 --- /dev/null +++ b/src/test/objectstore/run_smr_bluestore_test.sh @@ -0,0 +1,27 @@ +#!/bin/bash -ex + +echo "cd /backstores/user:zbc +create name=zbc0 size=20G cfgstring=model-HM/zsize-256/conv-10@zbc0.raw +cd /loopback +create naa.50014055e5f25aa0 +cd naa.50014055e5f25aa$1/luns +create /backstores/user:zbc/zbc0 0 +" | sudo targetcli + +function cleanup() { + echo "cd /loopback +delete naa.50014055e5f25aa$1 +cd /backstores/user:zbc +delete zbc0" | sudo targetcli + sudo rm -f zbc0.raw +} +trap cleanup EXIT + +DEV=`lsscsi | grep zbc | awk '{print $7}'` + +sudo ceph_test_objectstore \ + --bluestore-block-path $DEV \ + --gtest_filter=*/2 \ + --bluestore-block-db-create \ + --bluestore-block-db-size 1048576000 \ + $* From f9002c08611a00f71de8ca9871a15886129acb1c Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Thu, 16 Sep 2021 11:26:48 -0500 Subject: [PATCH 0415/2866] os/bluestore: separate alloc pointer from shared_alloc.a For the moment, these are always the same, so no change in functionality form this patch. In a future patch these may be different. Signed-off-by: Sage Weil --- src/os/bluestore/BlueStore.cc | 133 ++++++++++++++++++---------------- src/os/bluestore/BlueStore.h | 3 +- 2 files changed, 71 insertions(+), 65 deletions(-) diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc index 93fc3546c36b1..e13af070afa50 100644 --- a/src/os/bluestore/BlueStore.cc +++ b/src/os/bluestore/BlueStore.cc @@ -4546,8 +4546,8 @@ static void discard_cb(void *priv, void *priv2) void BlueStore::handle_discard(interval_set& to_release) { dout(10) << __func__ << dendl; - ceph_assert(shared_alloc.a); - shared_alloc.a->release(to_release); + ceph_assert(alloc); + alloc->release(to_release); } BlueStore::BlueStore(CephContext *cct, const string& path) @@ -5568,6 +5568,7 @@ int BlueStore::_write_out_fm_meta(uint64_t target_size) int BlueStore::_create_alloc() { + ceph_assert(alloc == NULL); ceph_assert(shared_alloc.a == NULL); ceph_assert(bdev->get_size()); @@ -5580,22 +5581,23 @@ int BlueStore::_create_alloc() allocator_type = "zoned"; } #endif - - shared_alloc.set( - Allocator::create( - cct, allocator_type, - bdev->get_size(), - alloc_size, - zone_size, - first_sequential_zone, - "block")); - if (!shared_alloc.a) { + alloc = Allocator::create( + cct, allocator_type, + bdev->get_size(), + alloc_size, + zone_size, + first_sequential_zone, + "block"); + if (!alloc) { lderr(cct) << __func__ << " failed to create " << allocator_type << " allocator" << dendl; return -EINVAL; } + // BlueFS will share the same allocator + shared_alloc.set(alloc); + return 0; } @@ -5605,11 +5607,11 @@ int BlueStore::_init_alloc(std::map *zone_adjustments) if (r < 0) { return r; } - ceph_assert(shared_alloc.a != NULL); + ceph_assert(alloc != NULL); #ifdef HAVE_LIBZBD if (bdev->is_smr()) { - auto a = dynamic_cast(shared_alloc.a); + auto a = dynamic_cast(alloc); ceph_assert(a); auto f = dynamic_cast(fm); ceph_assert(f); @@ -5646,11 +5648,11 @@ int BlueStore::_init_alloc(std::map *zone_adjustments) dout(1) << __func__ << " loaded zone pointers: " << std::hex - << ", allocator type " << shared_alloc.a->get_type() - << ", capacity 0x" << shared_alloc.a->get_capacity() - << ", block size 0x" << shared_alloc.a->get_block_size() - << ", free 0x" << shared_alloc.a->get_free() - << ", fragmentation " << shared_alloc.a->get_fragmentation() + << ", allocator type " << alloc->get_type() + << ", capacity 0x" << alloc->get_capacity() + << ", block size 0x" << alloc->get_block_size() + << ", free 0x" << alloc->get_free() + << ", fragmentation " << alloc->get_fragmentation() << std::dec << dendl; return 0; @@ -5661,12 +5663,12 @@ int BlueStore::_init_alloc(std::map *zone_adjustments) utime_t start_time = ceph_clock_now(); if (!fm->is_null_manager()) { // This is the original path - loading allocation map from RocksDB and feeding into the allocator - dout(5) << __func__ << "::NCB::loading allocation from FM -> shared_alloc" << dendl; + dout(5) << __func__ << "::NCB::loading allocation from FM -> alloc" << dendl; // initialize from freelist fm->enumerate_reset(); uint64_t offset, length; while (fm->enumerate_next(db, &offset, &length)) { - shared_alloc.a->init_add_free(offset, length); + alloc->init_add_free(offset, length); ++num; bytes += length; } @@ -5674,7 +5676,7 @@ int BlueStore::_init_alloc(std::map *zone_adjustments) utime_t duration = ceph_clock_now() - start_time; dout(5) << __func__ << "::num_entries=" << num << " free_size=" << bytes << " alloc_size=" << - shared_alloc.a->get_capacity() - bytes << " time=" << duration << " seconds" << dendl; + alloc->get_capacity() - bytes << " time=" << duration << " seconds" << dendl; } else { // This is the new path reading the allocation map from a flat bluefs file and feeding them into the allocator @@ -5684,8 +5686,8 @@ int BlueStore::_init_alloc(std::map *zone_adjustments) return -ENOTSUP; // Operation not supported } - if (restore_allocator(shared_alloc.a, &num, &bytes) == 0) { - dout(5) << __func__ << "::NCB::restore_allocator() completed successfully shared_alloc.a=" << shared_alloc.a << dendl; + if (restore_allocator(alloc, &num, &bytes) == 0) { + dout(5) << __func__ << "::NCB::restore_allocator() completed successfully alloc=" << alloc << dendl; } else { // This must mean that we had an unplanned shutdown and didn't manage to destage the allocator dout(1) << __func__ << "::NCB::restore_allocator() failed!" << dendl; @@ -5702,11 +5704,11 @@ int BlueStore::_init_alloc(std::map *zone_adjustments) dout(1) << __func__ << " loaded " << byte_u_t(bytes) << " in " << num << " extents" << std::hex - << ", allocator type " << shared_alloc.a->get_type() - << ", capacity 0x" << shared_alloc.a->get_capacity() - << ", block size 0x" << shared_alloc.a->get_block_size() - << ", free 0x" << shared_alloc.a->get_free() - << ", fragmentation " << shared_alloc.a->get_fragmentation() + << ", allocator type " << alloc->get_type() + << ", capacity 0x" << alloc->get_capacity() + << ", block size 0x" << alloc->get_block_size() + << ", free 0x" << alloc->get_free() + << ", fragmentation " << alloc->get_fragmentation() << std::dec << dendl; return 0; @@ -5737,9 +5739,12 @@ void BlueStore::_close_alloc() bdev->discard_drain(); ceph_assert(shared_alloc.a); + ceph_assert(alloc); + ceph_assert(alloc == shared_alloc.a); shared_alloc.a->shutdown(); delete shared_alloc.a; shared_alloc.reset(); + alloc = nullptr; } int BlueStore::_open_fsid(bool create) @@ -6845,7 +6850,7 @@ int BlueStore::mkfs() } reserved = _get_ondisk_reserved(); - shared_alloc.a->init_add_free(reserved, + alloc->init_add_free(reserved, p2align(bdev->get_size(), min_alloc_size) - reserved); r = _open_db(true); @@ -7425,7 +7430,7 @@ int BlueStore::umount() mounted = false; - ceph_assert(shared_alloc.a); + ceph_assert(alloc); if (!_kv_only) { mempool_thread.shutdown(); @@ -7445,7 +7450,7 @@ int BlueStore::umount() // GBH - Vault the allocation state dout(5) << "NCB::BlueStore::umount->store_allocation_state_on_bluestore() " << dendl; if (was_mounted && fm->is_null_manager()) { - int ret = store_allocator(shared_alloc.a); + int ret = store_allocator(alloc); if (ret != 0) { derr << __func__ << "::NCB::store_allocator() failed (continue with bitmapFreelistManager)" << dendl; _close_db_and_around(false); @@ -8794,7 +8799,7 @@ int BlueStore::_fsck_on_open(BlueStore::FSCKDepth depth, bool repair) #ifdef HAVE_LIBZBD if (bdev->is_smr()) { - auto a = dynamic_cast(shared_alloc.a); + auto a = dynamic_cast(alloc); ceph_assert(a); auto f = dynamic_cast(fm); ceph_assert(f); @@ -9074,19 +9079,19 @@ int BlueStore::_fsck_on_open(BlueStore::FSCKDepth depth, bool repair) continue; } PExtentVector exts; - dout(5) << __func__ << "::NCB::(F)shared_alloc.a=" << shared_alloc.a << ", length=" << e->length << dendl; + dout(5) << __func__ << "::NCB::(F)alloc=" << alloc << ", length=" << e->length << dendl; int64_t alloc_len = - shared_alloc.a->allocate(e->length, min_alloc_size, + alloc->allocate(e->length, min_alloc_size, 0, 0, &exts); if (alloc_len < 0 || alloc_len < (int64_t)e->length) { derr << __func__ << " failed to allocate 0x" << std::hex << e->length << " allocated 0x " << (alloc_len < 0 ? 0 : alloc_len) << " min_alloc_size 0x" << min_alloc_size - << " available 0x " << shared_alloc.a->get_free() + << " available 0x " << alloc->get_free() << std::dec << dendl; if (alloc_len > 0) { - shared_alloc.a->release(exts); + alloc->release(exts); } bypass_rest = true; break; @@ -9166,7 +9171,7 @@ int BlueStore::_fsck_on_open(BlueStore::FSCKDepth depth, bool repair) << "~" << it.get_len() << std::dec << dendl; fm->release(it.get_start(), it.get_len(), txn); } - shared_alloc.a->release(to_release); + alloc->release(to_release); to_release.clear(); } // if (it) { } //if (repair && repairer.preprocess_misreference()) { @@ -9373,7 +9378,7 @@ int BlueStore::_fsck_on_open(BlueStore::FSCKDepth depth, bool repair) // - verify no allocations beyond write pointer // - verify num_dead_bytes count (neither allocated nor // free space past the write pointer) - auto a = dynamic_cast(shared_alloc.a); + auto a = dynamic_cast(alloc); auto num_zones = bdev->get_size() / zone_size; // mark the free space past the write pointer @@ -9579,7 +9584,7 @@ void BlueStore::inject_no_shared_blob_key() void BlueStore::inject_leaked(uint64_t len) { PExtentVector exts; - int64_t alloc_len = shared_alloc.a->allocate(len, min_alloc_size, + int64_t alloc_len = alloc->allocate(len, min_alloc_size, min_alloc_size * 256, 0, &exts); if (fm->is_null_manager()) { @@ -9912,7 +9917,7 @@ void BlueStore::_get_statfs_overall(struct store_statfs_t *buf) buf->omap_allocated = db->estimate_prefix_size(prefix, string()); - uint64_t bfree = shared_alloc.a->get_free(); + uint64_t bfree = alloc->get_free(); if (bluefs) { buf->internally_reserved = 0; @@ -12442,7 +12447,7 @@ void BlueStore::_txc_release_alloc(TransContext *txc) } dout(10) << __func__ << "(sync) " << txc << " " << std::hex << txc->released << std::dec << dendl; - shared_alloc.a->release(txc->released); + alloc->release(txc->released); } out: @@ -12971,7 +12976,7 @@ void BlueStore::_kv_finalize_thread() _reap_collections(); logger->set(l_bluestore_fragmentation, - (uint64_t)(shared_alloc.a->get_fragmentation() * 1000)); + (uint64_t)(alloc->get_fragmentation() * 1000)); log_latency("kv_final", l_bluestore_kv_final_lat, @@ -13020,7 +13025,7 @@ void BlueStore::_zoned_cleaner_thread() ceph_assert(!zoned_cleaner_started); zoned_cleaner_started = true; zoned_cleaner_cond.notify_all(); - auto a = dynamic_cast(shared_alloc.a); + auto a = dynamic_cast(alloc); ceph_assert(a); auto f = dynamic_cast(fm); ceph_assert(f); @@ -14746,20 +14751,20 @@ int BlueStore::_do_alloc_write( PExtentVector prealloc; prealloc.reserve(2 * wctx->writes.size());; int64_t prealloc_left = 0; - prealloc_left = shared_alloc.a->allocate( + prealloc_left = alloc->allocate( need, min_alloc_size, need, 0, &prealloc); if (prealloc_left < 0 || prealloc_left < (int64_t)need) { - dout(5) << __func__ << "::NCB::failed allocation of " << need << " bytes!! shared_alloc.a=" << shared_alloc.a << dendl; + dout(5) << __func__ << "::NCB::failed allocation of " << need << " bytes!! alloc=" << alloc << dendl; derr << __func__ << " failed to allocate 0x" << std::hex << need << " allocated 0x " << (prealloc_left < 0 ? 0 : prealloc_left) << " min_alloc_size 0x" << min_alloc_size - << " available 0x " << shared_alloc.a->get_free() + << " available 0x " << alloc->get_free() << std::dec << dendl; if (prealloc.size()) { - shared_alloc.a->release(prealloc); + alloc->release(prealloc); } - dout(5) << __func__ << "::NCB::(2)shared_alloc.a=" << shared_alloc.a << dendl; + dout(5) << __func__ << "::NCB::(2)alloc=" << alloc << dendl; return -ENOSPC; } _collect_allocation_stats(need, min_alloc_size, prealloc); @@ -18234,8 +18239,8 @@ int BlueStore::read_allocation_from_drive_on_startup() } uint64_t num_entries = 0; - dout(5) << " calling copy_allocator(bitmap_allocator -> shared_alloc.a)" << dendl; - copy_allocator(allocator, shared_alloc.a, &num_entries); + dout(5) << " calling copy_allocator(bitmap_allocator -> alloc)" << dendl; + copy_allocator(allocator, alloc, &num_entries); delete allocator; utime_t duration = ceph_clock_now() - start; dout(5) << " <<>> in " << duration << " seconds, num_entries=" << num_entries << dendl; @@ -18364,7 +18369,7 @@ int BlueStore::add_existing_bluefs_allocation(Allocator* allocator, read_alloc_s } for (auto itr = bluefs_extents.begin(); itr != bluefs_extents.end(); extent_count++, itr++) { //dout(5) << "BlueFS[" << extent_count << "] <" << itr.get_start() << "," << itr.get_len() << ">" << dendl; - allocator->init_rm_free(itr.get_start(), itr.get_len()); + shared_alloc.a->init_rm_free(itr.get_start(), itr.get_len()); stats.extent_count++; } } @@ -18422,18 +18427,18 @@ int BlueStore::read_allocation_from_drive_for_bluestore_tool(bool test_store_and dout(5) << "\n" << " <<>> in " << duration << " seconds; extent_count=" << stats.extent_count << dendl; - dout(5) << "calling compare_allocator(shared_alloc.a) insert_count=" << stats.insert_count << dendl; - ret = compare_allocators(allocator, shared_alloc.a, stats.insert_count, memory_target); + dout(5) << "calling compare_allocator(alloc) insert_count=" << stats.insert_count << dendl; + ret = compare_allocators(allocator, alloc, stats.insert_count, memory_target); if (ret == 0) { - dout(5) << "SUCCESS!!! compare(allocator, shared_alloc.a)" << dendl; + dout(5) << "SUCCESS!!! compare(allocator, alloc)" << dendl; } else { - derr << "**** FAILURE compare(allocator, shared_alloc.a)::ret=" << ret << dendl; + derr << "**** FAILURE compare(allocator, alloc)::ret=" << ret << dendl; } if (test_store_and_restore) { _close_db_leave_bluefs(); - dout(5) << "calling store_allocator(shared_alloc.a)" << dendl; - store_allocator(shared_alloc.a); + dout(5) << "calling store_allocator(alloc)" << dendl; + store_allocator(alloc); Allocator* alloc2 = create_bitmap_allocator(bdev_size); if (alloc2) { dout(5) << "bitmap-allocator=" << alloc2 << dendl; @@ -18447,11 +18452,11 @@ int BlueStore::read_allocation_from_drive_for_bluestore_tool(bool test_store_and _close_db_and_around(false); return ret; } // verify that we can store and restore allocator to/from drive - ret = compare_allocators(alloc2, shared_alloc.a, stats.insert_count, memory_target); + ret = compare_allocators(alloc2, alloc, stats.insert_count, memory_target); if (ret == 0) { - dout(5) << "SUCCESS!!! compare(alloc2, shared_alloc.a)" << dendl; + dout(5) << "SUCCESS!!! compare(alloc2, alloc)" << dendl; } else { - derr << "**** FAILURE compare(alloc2, shared_alloc.a)::ret=" << ret << dendl; + derr << "**** FAILURE compare(alloc2, alloc)::ret=" << ret << dendl; } } else { derr << "******Failed restore_allocator******\n" << dendl; @@ -18493,7 +18498,7 @@ Allocator* BlueStore::clone_allocator_without_bluefs(Allocator *src_allocator) } uint64_t num_entries = 0; - dout(5) << "calling copy_allocator(shared_alloc.a -> bitmap_allocator)" << dendl; + dout(5) << "calling copy_allocator(alloc -> bitmap_allocator)" << dendl; copy_allocator(src_allocator, allocator, &num_entries); // BlueFS stores its internal allocation outside RocksDB (FM) so we should not destage them to the allcoator-file @@ -18598,7 +18603,7 @@ int BlueStore::reset_fm_for_restore() // and compare it to the base allocator passed in int BlueStore::verify_rocksdb_allocations(Allocator *allocator) { - dout(5) << "verify that shared_alloc content is identical to FM" << dendl; + dout(5) << "verify that alloc content is identical to FM" << dendl; // initialize from freelist Allocator* temp_allocator = initialize_allocator_from_freelist(fm); if (temp_allocator == nullptr) { @@ -18646,7 +18651,7 @@ int BlueStore::push_allocation_to_rocksdb() } // start by creating a clone copy of the shared-allocator - unique_ptr allocator(clone_allocator_without_bluefs(shared_alloc.a)); + unique_ptr allocator(clone_allocator_without_bluefs(alloc)); if (!allocator) { return db_cleanup(-1); } diff --git a/src/os/bluestore/BlueStore.h b/src/os/bluestore/BlueStore.h index ed78a25ad1552..5ba447ae13aaa 100644 --- a/src/os/bluestore/BlueStore.h +++ b/src/os/bluestore/BlueStore.h @@ -2044,7 +2044,8 @@ class BlueStore : public ObjectStore, std::string freelist_type; FreelistManager *fm = nullptr; - bluefs_shared_alloc_context_t shared_alloc; + Allocator *alloc = nullptr; ///< allocator consumed by BlueStore + bluefs_shared_alloc_context_t shared_alloc; ///< consumed by BlueFS (may be == alloc) uuid_d fsid; int path_fd = -1; ///< open handle to $path From d7b0e77b56ae10d985652b7f1955e2c98f196b9a Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Thu, 16 Sep 2021 12:09:54 -0500 Subject: [PATCH 0416/2866] os/bluestore: give conventional region of SMR to bluefs Let bluefs use the conventional region of an SMR HDD. It won't be able to use the sequential region if it runs out of space, but it does mean that bluefs can function on an SMR drive without a dedicated db/wal device. Signed-off-by: Sage Weil --- src/os/bluestore/BlueStore.cc | 52 ++++++++++++++++--- .../objectstore/run_smr_bluestore_test.sh | 2 - 2 files changed, 46 insertions(+), 8 deletions(-) diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc index e13af070afa50..a3859480fb423 100644 --- a/src/os/bluestore/BlueStore.cc +++ b/src/os/bluestore/BlueStore.cc @@ -5595,8 +5595,27 @@ int BlueStore::_create_alloc() return -EINVAL; } - // BlueFS will share the same allocator - shared_alloc.set(alloc); +#ifdef HAVE_LIBZBD + if (freelist_type == "zoned") { + Allocator *a = Allocator::create( + cct, cct->_conf->bluestore_allocator, + bdev->get_conventional_region_size(), + alloc_size, + 0, 0, + "zoned_block"); + if (!a) { + lderr(cct) << __func__ << " failed to create " << cct->_conf->bluestore_allocator + << " allocator" << dendl; + delete alloc; + return -EINVAL; + } + shared_alloc.set(a); + } else +#endif + { + // BlueFS will share the same allocator + shared_alloc.set(alloc); + } return 0; } @@ -5644,6 +5663,15 @@ int BlueStore::_init_alloc(std::map *zone_adjustments) } } + // start with conventional zone "free" (bluefs may adjust this when it starts up) + auto reserved = _get_ondisk_reserved(); + // for now we require a conventional zone + ceph_assert(bdev->get_conventional_region_size()); + ceph_assert(shared_alloc.a != alloc); // zoned allocator doesn't use conventional region + shared_alloc.a->init_add_free(reserved, + bdev->get_conventional_region_size() - reserved); + + // init sequential zone based on the device's write pointers a->init_from_zone_pointers(zones); dout(1) << __func__ << " loaded zone pointers: " @@ -5738,11 +5766,16 @@ void BlueStore::_close_alloc() ceph_assert(bdev); bdev->discard_drain(); - ceph_assert(shared_alloc.a); ceph_assert(alloc); - ceph_assert(alloc == shared_alloc.a); - shared_alloc.a->shutdown(); - delete shared_alloc.a; + alloc->shutdown(); + delete alloc; + + ceph_assert(shared_alloc.a); + if (alloc != shared_alloc.a) { + shared_alloc.a->shutdown(); + delete shared_alloc.a; + } + shared_alloc.reset(); alloc = nullptr; } @@ -6852,6 +6885,13 @@ int BlueStore::mkfs() reserved = _get_ondisk_reserved(); alloc->init_add_free(reserved, p2align(bdev->get_size(), min_alloc_size) - reserved); +#ifdef HAVE_LIBZBD + if (bdev->is_smr() && alloc != shared_alloc.a) { + shared_alloc.a->init_add_free(reserved, + p2align(bdev->get_conventional_region_size(), + min_alloc_size) - reserved); + } +#endif r = _open_db(true); if (r < 0) diff --git a/src/test/objectstore/run_smr_bluestore_test.sh b/src/test/objectstore/run_smr_bluestore_test.sh index 22017b0c5cb60..5fa8544e0a1fc 100644 --- a/src/test/objectstore/run_smr_bluestore_test.sh +++ b/src/test/objectstore/run_smr_bluestore_test.sh @@ -22,6 +22,4 @@ DEV=`lsscsi | grep zbc | awk '{print $7}'` sudo ceph_test_objectstore \ --bluestore-block-path $DEV \ --gtest_filter=*/2 \ - --bluestore-block-db-create \ - --bluestore-block-db-size 1048576000 \ $* From d9b95ea2916cfce31c9337d86d8bf544d1c3a6d9 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Thu, 16 Sep 2021 12:10:07 -0500 Subject: [PATCH 0417/2866] os/bluestore/HybridAllocator: whitespace in debug output Signed-off-by: Sage Weil --- src/os/bluestore/HybridAllocator.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/os/bluestore/HybridAllocator.cc b/src/os/bluestore/HybridAllocator.cc index 3a2da67379b59..a6fa0bd75ed2e 100644 --- a/src/os/bluestore/HybridAllocator.cc +++ b/src/os/bluestore/HybridAllocator.cc @@ -168,7 +168,7 @@ void HybridAllocator::init_rm_free(uint64_t offset, uint64_t length) if (bmap_alloc) { bmap_alloc->init_rm_free(o, l); } else { - lderr(cct) << "init_rm_free lambda" << std::hex + lderr(cct) << "init_rm_free lambda " << std::hex << "Uexpected extent: " << " 0x" << o << "~" << l << std::dec << dendl; From cf2c533cd838ef78242eeacf051b5d5cd1e723ab Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Fri, 17 Sep 2021 12:38:34 -0500 Subject: [PATCH 0418/2866] os/bluestore: avoid writes to cleaning zone Signed-off-by: Sage Weil --- src/os/bluestore/BlueStore.cc | 5 ++--- src/os/bluestore/ZonedAllocator.cc | 25 +++++++++++++++++-------- src/os/bluestore/ZonedAllocator.h | 7 +++++++ 3 files changed, 26 insertions(+), 11 deletions(-) diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc index a3859480fb423..c0dfcd36ea484 100644 --- a/src/os/bluestore/BlueStore.cc +++ b/src/os/bluestore/BlueStore.cc @@ -13080,10 +13080,9 @@ void BlueStore::_zoned_cleaner_thread() dout(20) << __func__ << " wake" << dendl; } else { l.unlock(); + a->set_cleaning_zone(zone_to_clean); _zoned_clean_zone(zone_to_clean); - bdev->reset_zone(zone_to_clean); - f->mark_zone_to_clean_free(zone_to_clean, db); - //a->mark_zone_to_clean_free(); + a->clear_cleaning_zone(zone_to_clean); l.lock(); } } diff --git a/src/os/bluestore/ZonedAllocator.cc b/src/os/bluestore/ZonedAllocator.cc index 6933dd972d891..8438c65c643cd 100644 --- a/src/os/bluestore/ZonedAllocator.cc +++ b/src/os/bluestore/ZonedAllocator.cc @@ -66,17 +66,26 @@ int64_t ZonedAllocator::allocate( ldout(cct, 10) << " trying to allocate 0x" << std::hex << want_size << std::dec << dendl; + // FIXME: start search at last zone we successfully allocated from, so we + // avoid re-checking full zones so much + uint64_t zone_num = starting_zone_num; for ( ; zone_num < num_zones; ++zone_num) { - if (fits(want_size, zone_num)) { - break; + if (zone_num == cleaning_zone) { + ldout(cct, 10) << " skipping zone 0x" << std::hex << zone_num + << " because we are cleaning it" << std::dec << dendl; + continue; + } + if (!fits(want_size, zone_num)) { + ldout(cct, 10) << " skipping zone 0x" << std::hex << zone_num + << " because there is not enough space: " + << " want_size = 0x" << want_size + << " available = 0x" << get_remaining_space(zone_num) + << std::dec + << dendl; + continue; } - ldout(cct, 10) << " skipping zone 0x" << std::hex << zone_num - << " because there is not enough space: " - << " want_size = 0x" << want_size - << " available = 0x" << get_remaining_space(zone_num) - << std::dec - << dendl; + break; } if (zone_num == num_zones) { diff --git a/src/os/bluestore/ZonedAllocator.h b/src/os/bluestore/ZonedAllocator.h index 4dccb9f38062e..ab4cc27707a5c 100644 --- a/src/os/bluestore/ZonedAllocator.h +++ b/src/os/bluestore/ZonedAllocator.h @@ -38,6 +38,7 @@ class ZonedAllocator : public Allocator { uint64_t first_seq_zone_num; uint64_t starting_zone_num; uint64_t num_zones; + std::atomic cleaning_zone = -1; std::vector zone_states; inline uint64_t get_offset(uint64_t zone_num) const { @@ -92,6 +93,12 @@ class ZonedAllocator : public Allocator { uint64_t length)> notify) override; int64_t pick_zone_to_clean(void); + void set_cleaning_zone(uint32_t zone) { + cleaning_zone = zone; + } + void clear_cleaning_zone(uint32_t zone) { + cleaning_zone = -1; + } void init_from_zone_pointers( std::vector _zone_states); From 8b072af011667d10a1d515141ae38cd71f242f20 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Fri, 17 Sep 2021 13:54:42 -0500 Subject: [PATCH 0419/2866] os/bluestore: be smarter about picking a zone to clean Base our selection on best "value" (benefit / cost). Signed-off-by: Sage Weil --- src/os/bluestore/BlueStore.cc | 2 +- src/os/bluestore/ZonedAllocator.cc | 39 ++++++++++++++++++++++++------ src/os/bluestore/ZonedAllocator.h | 5 +++- src/os/bluestore/zoned_types.h | 4 +++ 4 files changed, 40 insertions(+), 10 deletions(-) diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc index c0dfcd36ea484..db29a98b3390b 100644 --- a/src/os/bluestore/BlueStore.cc +++ b/src/os/bluestore/BlueStore.cc @@ -13070,7 +13070,7 @@ void BlueStore::_zoned_cleaner_thread() auto f = dynamic_cast(fm); ceph_assert(f); while (true) { - auto zone_to_clean = a->pick_zone_to_clean(); + auto zone_to_clean = a->pick_zone_to_clean(.1, zone_size / 16); // FIXME if (zone_to_clean < 0) { if (zoned_cleaner_stop) { break; diff --git a/src/os/bluestore/ZonedAllocator.cc b/src/os/bluestore/ZonedAllocator.cc index 8438c65c643cd..7feb6f3e014a1 100644 --- a/src/os/bluestore/ZonedAllocator.cc +++ b/src/os/bluestore/ZonedAllocator.cc @@ -167,25 +167,48 @@ void ZonedAllocator::init_from_zone_pointers( << dendl; } -int64_t ZonedAllocator::pick_zone_to_clean(void) +int64_t ZonedAllocator::pick_zone_to_clean(float min_score, uint64_t min_saved) { + std::lock_guard l(lock); int32_t best = -1; - int64_t best_score = 0; + float best_score = 0.0; for (size_t i = first_seq_zone_num; i < num_zones; ++i) { - int64_t score = zone_states[i].num_dead_bytes; - // discount by remaining space so we will tend to clean full zones - score -= (zone_size - zone_states[i].write_pointer) / 2; - if (score > 0 && (best < 0 || score > best_score)) { + // value (score) = benefit / cost + // benefit = how much net free space we'll get (dead bytes) + // cost = how many bytes we'll have to rewrite (live bytes) + // avoid divide by zero on a zone with no live bytes + float score = + (float)zone_states[i].num_dead_bytes / + (float)(zone_states[i].get_num_live_bytes() + 1); + if (score > 0) { + ldout(cct, 20) << " zone 0x" << std::hex << i + << " dead 0x" << zone_states[i].num_dead_bytes + << " score " << score + << dendl; + } + if (zone_states[i].num_dead_bytes < min_saved) { + continue; + } + if (best < 0 || score > best_score) { best = i; best_score = score; } } - if (best >= 0) { - ldout(cct, 10) << " zone 0x" << std::hex << best << " with score 0x" << best_score + if (best_score >= min_score) { + ldout(cct, 10) << " zone 0x" << std::hex << best << " with score " << best_score << ": 0x" << zone_states[best].num_dead_bytes << " dead and 0x" << zone_states[best].write_pointer - zone_states[best].num_dead_bytes << " live bytes" << std::dec << dendl; + } else if (best > 0) { + ldout(cct, 10) << " zone 0x" << std::hex << best << " with score " << best_score + << ": 0x" << zone_states[best].num_dead_bytes + << " dead and 0x" + << zone_states[best].write_pointer - zone_states[best].num_dead_bytes + << " live bytes" << std::dec + << " but below min_score " << min_score + << dendl; + best = -1; } else { ldout(cct, 10) << " no zones found that are good cleaning candidates" << dendl; } diff --git a/src/os/bluestore/ZonedAllocator.h b/src/os/bluestore/ZonedAllocator.h index ab4cc27707a5c..9f5b632eb93a5 100644 --- a/src/os/bluestore/ZonedAllocator.h +++ b/src/os/bluestore/ZonedAllocator.h @@ -79,6 +79,9 @@ class ZonedAllocator : public Allocator { uint64_t get_dead_bytes(uint32_t zone) { return zone_states[zone].num_dead_bytes; } + uint64_t get_live_bytes(uint32_t zone) { + return zone_states[zone].write_pointer - zone_states[zone].num_dead_bytes; + } int64_t allocate( uint64_t want_size, uint64_t alloc_unit, uint64_t max_alloc_size, @@ -92,7 +95,7 @@ class ZonedAllocator : public Allocator { void dump(std::function notify) override; - int64_t pick_zone_to_clean(void); + int64_t pick_zone_to_clean(float min_score, uint64_t min_saved); void set_cleaning_zone(uint32_t zone) { cleaning_zone = zone; } diff --git a/src/os/bluestore/zoned_types.h b/src/os/bluestore/zoned_types.h index 93d3138399a0c..5507e64c7c58c 100644 --- a/src/os/bluestore/zoned_types.h +++ b/src/os/bluestore/zoned_types.h @@ -32,6 +32,10 @@ struct zone_state_t { return num_dead_bytes; } + uint64_t get_num_live_bytes() const { + return write_pointer - num_dead_bytes; + } + uint64_t get_write_pointer() const { return write_pointer; } From 6917b2f8db8a9e51f9eb075a427cb9c82512edaf Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Fri, 17 Sep 2021 14:14:51 -0500 Subject: [PATCH 0420/2866] os/bluestore: simple cleaner Rewrite object ranges to move by reusing _do_read and _do_write. Note that this will rewrite shared/cloned extents multiple times, so it is suitable only for workloads/datasets that do not use clone (snapshots). Signed-off-by: Sage Weil --- src/os/bluestore/BlueStore.cc | 168 +++++++++++++++++++++-- src/os/bluestore/BlueStore.h | 2 + src/os/bluestore/ZonedAllocator.cc | 8 +- src/os/bluestore/ZonedAllocator.h | 6 +- src/os/bluestore/ZonedFreelistManager.cc | 48 ++++++- src/os/bluestore/ZonedFreelistManager.h | 4 +- src/os/bluestore/zoned_types.h | 5 + 7 files changed, 220 insertions(+), 21 deletions(-) diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc index db29a98b3390b..c37aff0c15dfc 100644 --- a/src/os/bluestore/BlueStore.cc +++ b/src/os/bluestore/BlueStore.cc @@ -5668,11 +5668,12 @@ int BlueStore::_init_alloc(std::map *zone_adjustments) // for now we require a conventional zone ceph_assert(bdev->get_conventional_region_size()); ceph_assert(shared_alloc.a != alloc); // zoned allocator doesn't use conventional region - shared_alloc.a->init_add_free(reserved, - bdev->get_conventional_region_size() - reserved); + shared_alloc.a->init_add_free( + reserved, + p2align(bdev->get_conventional_region_size(), min_alloc_size) - reserved); // init sequential zone based on the device's write pointers - a->init_from_zone_pointers(zones); + a->init_from_zone_pointers(std::move(zones)); dout(1) << __func__ << " loaded zone pointers: " << std::hex @@ -8849,10 +8850,12 @@ int BlueStore::_fsck_on_open(BlueStore::FSCKDepth depth, bool repair) auto num_zones = bdev->get_size() / zone_size; for (unsigned i = first_sequential_zone; i < num_zones; ++i) { uint64_t p = wp[i] == (i + 1) * zone_size ? zone_size : wp[i] % zone_size; - if (zones[i].write_pointer > p) { + if (zones[i].write_pointer > p && + zones[i].num_dead_bytes < zones[i].write_pointer) { derr << "fsck error: zone 0x" << std::hex << i << " bluestore write pointer 0x" << zones[i].write_pointer << " > device write pointer 0x" << p + << " (with only 0x" << zones[i].num_dead_bytes << " dead bytes)" << std::dec << dendl; ++errors; } @@ -10081,6 +10084,22 @@ BlueStore::CollectionRef BlueStore::_get_collection(const coll_t& cid) return cp->second; } +BlueStore::CollectionRef BlueStore::_get_collection_by_oid(const ghobject_t& oid) +{ + std::shared_lock l(coll_lock); + + // FIXME: we must replace this with something more efficient + + for (auto& i : coll_map) { + spg_t spgid; + if (i.first.is_pg(&spgid) && + i.second->contains(oid)) { + return i.second; + } + } + return CollectionRef(); +} + void BlueStore::_queue_reap_collection(CollectionRef& c) { dout(10) << __func__ << " " << c << " " << c->cid << dendl; @@ -13034,14 +13053,12 @@ void BlueStore::_kv_finalize_thread() void BlueStore::_zoned_cleaner_start() { dout(10) << __func__ << dendl; - return; // temporarily disable cleaner until it actually works zoned_cleaner_thread.create("bstore_zcleaner"); } void BlueStore::_zoned_cleaner_stop() { dout(10) << __func__ << dendl; - return; // temporarily disable cleaner until it actually works { std::unique_lock l{zoned_cleaner_lock}; while (!zoned_cleaner_started) { @@ -13070,7 +13087,11 @@ void BlueStore::_zoned_cleaner_thread() auto f = dynamic_cast(fm); ceph_assert(f); while (true) { - auto zone_to_clean = a->pick_zone_to_clean(.1, zone_size / 16); // FIXME + // thresholds to trigger cleaning + // FIXME + float min_score = .05; // score: bytes saved / bytes moved + uint64_t min_saved = zone_size / 32; // min bytes saved to consider cleaning + auto zone_to_clean = a->pick_zone_to_clean(min_score, min_saved); if (zone_to_clean < 0) { if (zoned_cleaner_stop) { break; @@ -13090,11 +13111,136 @@ void BlueStore::_zoned_cleaner_thread() zoned_cleaner_started = false; } -void BlueStore::_zoned_clean_zone(uint64_t zone_num) +void BlueStore::_zoned_clean_zone(uint64_t zone) +{ + dout(10) << __func__ << " cleaning zone 0x" << std::hex << zone << std::dec << dendl; + auto a = dynamic_cast(alloc); + auto f = dynamic_cast(fm); + + KeyValueDB::Iterator it = db->get_iterator(PREFIX_ZONED_CL_INFO); + std::string zone_start; + get_zone_offset_object_key(zone, 0, ghobject_t(), &zone_start); + for (it->lower_bound(zone_start); it->valid(); it->next()) { + uint32_t z; + uint64_t offset; + ghobject_t oid; + string k = it->key(); + int r = get_key_zone_offset_object(k, &z, &offset, &oid); + if (r < 0) { + derr << __func__ << " failed to decode zone ref " << pretty_binary_string(k) + << dendl; + continue; + } + if (zone != z) { + dout(10) << __func__ << " reached end of zone refs" << dendl; + break; + } + dout(10) << __func__ << " zone 0x" << std::hex << zone << " offset 0x" << offset + << std::dec << " " << oid << dendl; + _clean_some(oid, zone); + } + + if (a->get_live_bytes(zone) > 0) { + derr << "zone 0x" << std::hex << zone << " still has 0x" << a->get_live_bytes(zone) + << " live bytes" << std::dec << dendl; + // should we do something else here to avoid a live-lock in the event of a problem? + return; + } + + // reset the device zone + dout(10) << __func__ << " resetting zone 0x" << std::hex << zone << std::dec << dendl; + bdev->reset_zone(zone); + + // record that we can now write there + f->mark_zone_to_clean_free(zone, a->get_write_pointer(zone), + a->get_dead_bytes(zone), db); + bdev->flush(); + + // then allow ourselves to start allocating there + dout(10) << __func__ << " done cleaning zone 0x" << std::hex << zone << std::dec + << dendl; + a->reset_zone(zone); +} + +void BlueStore::_clean_some(ghobject_t oid, uint32_t zone) { - dout(10) << __func__ << " cleaning zone 0x" << std::hex << zone_num << std::dec << dendl; - // TODO: (1) copy live objects from zone_num to a new zone, (2) issue a RESET - // ZONE operation to the device for the corresponding zone. + dout(10) << __func__ << " " << oid << " from zone 0x" << std::hex << zone << std::dec + << dendl; + + CollectionRef cref = _get_collection_by_oid(oid); + Collection *c = cref.get(); + + // serialize io dispatch vs other transactions + std::lock_guard l(atomic_alloc_and_submit_lock); + std::unique_lock l2(c->lock); + + auto o = c->get_onode(oid, false); + if (!o) { + derr << __func__ << " can't find " << oid << dendl; + return; + } + + o->extent_map.fault_range(db, 0, OBJECT_MAX_SIZE); + _dump_onode<30>(cct, *o); + + // NOTE: This is a naive rewrite strategy. If any blobs are + // shared, they will be duplicated for each object that references + // them. That means any cloned/snapshotted objects will explode + // their utilization. This won't matter for RGW workloads, but + // for RBD and CephFS it is completely unacceptable, and it's + // entirely reasonable to have "archival" data workloads on SMR + // for CephFS and (possibly/probably) RBD. + // + // At some point we need to replace this with something more + // sophisticated that ensures that a shared blob gets moved once + // and all referencing objects get updated to point to the new + // location. + + map to_move; + for (auto& e : o->extent_map.extent_map) { + bool touches_zone = false; + for (auto& be : e.blob->get_blob().get_extents()) { + if (be.is_valid()) { + uint32_t z = be.offset / zone_size; + if (z == zone) { + touches_zone = true; + break; + } + } + } + if (touches_zone) { + to_move[e.logical_offset] = e.length; + } + } + if (to_move.empty()) { + dout(10) << __func__ << " no references to zone 0x" << std::hex << zone + << std::dec << " from " << oid << dendl; + return; + } + + dout(10) << __func__ << " rewriting object extents 0x" << std::hex << to_move + << std::dec << dendl; + OpSequencer *osr = c->osr.get(); + TransContext *txc = _txc_create(c, osr, nullptr); + + spg_t pgid; + if (c->cid.is_pg(&pgid)) { + txc->osd_pool_id = pgid.pool(); + } + + for (auto& [offset, length] : to_move) { + bufferlist bl; + int r = _do_read(c, o, offset, length, bl, 0); + ceph_assert(r == (int)length); + + r = _do_write(txc, cref, o, offset, length, bl, 0); + ceph_assert(r >= 0); + } + txc->write_onode(o); + + _txc_write_nodes(txc, txc->t); + _txc_finalize_kv(txc, txc->t); + _txc_state_proc(txc); } #endif diff --git a/src/os/bluestore/BlueStore.h b/src/os/bluestore/BlueStore.h index 5ba447ae13aaa..f9e1b31d613a1 100644 --- a/src/os/bluestore/BlueStore.h +++ b/src/os/bluestore/BlueStore.h @@ -2428,6 +2428,7 @@ class BlueStore : public ObjectStore, void _dump_alloc_on_failure(); CollectionRef _get_collection(const coll_t& cid); + CollectionRef _get_collection_by_oid(const ghobject_t& oid); void _queue_reap_collection(CollectionRef& c); void _reap_collections(); void _update_cache_logger(); @@ -2479,6 +2480,7 @@ class BlueStore : public ObjectStore, void _zoned_cleaner_stop(); void _zoned_cleaner_thread(); void _zoned_clean_zone(uint64_t zone_num); + void _clean_some(ghobject_t oid, uint32_t zone_num); #endif bluestore_deferred_op_t *_get_deferred_op(TransContext *txc, uint64_t len); diff --git a/src/os/bluestore/ZonedAllocator.cc b/src/os/bluestore/ZonedAllocator.cc index 7feb6f3e014a1..8401eab1858a4 100644 --- a/src/os/bluestore/ZonedAllocator.cc +++ b/src/os/bluestore/ZonedAllocator.cc @@ -150,7 +150,7 @@ void ZonedAllocator::dump(std::function _zone_states) + std::vector &&_zone_states) { // this is called once, based on the device's zone pointers std::lock_guard l(lock); @@ -215,6 +215,12 @@ int64_t ZonedAllocator::pick_zone_to_clean(float min_score, uint64_t min_saved) return best; } +void ZonedAllocator::reset_zone(uint32_t zone) +{ + num_free += zone_states[zone].write_pointer; + zone_states[zone].reset(); +} + bool ZonedAllocator::low_on_space(void) { std::lock_guard l(lock); diff --git a/src/os/bluestore/ZonedAllocator.h b/src/os/bluestore/ZonedAllocator.h index 9f5b632eb93a5..12fa6cddd3ecc 100644 --- a/src/os/bluestore/ZonedAllocator.h +++ b/src/os/bluestore/ZonedAllocator.h @@ -45,10 +45,11 @@ class ZonedAllocator : public Allocator { return zone_num * zone_size + get_write_pointer(zone_num); } +public: inline uint64_t get_write_pointer(uint64_t zone_num) const { return zone_states[zone_num].get_write_pointer(); } - +private: inline uint64_t get_remaining_space(uint64_t zone_num) const { return zone_size - get_write_pointer(zone_num); } @@ -102,9 +103,10 @@ class ZonedAllocator : public Allocator { void clear_cleaning_zone(uint32_t zone) { cleaning_zone = -1; } + void reset_zone(uint32_t zone); void init_from_zone_pointers( - std::vector _zone_states); + std::vector &&_zone_states); void init_add_free(uint64_t offset, uint64_t length) override {} void init_rm_free(uint64_t offset, uint64_t length) override {} diff --git a/src/os/bluestore/ZonedFreelistManager.cc b/src/os/bluestore/ZonedFreelistManager.cc index 60e72c2cd2727..092e8d540aa90 100644 --- a/src/os/bluestore/ZonedFreelistManager.cc +++ b/src/os/bluestore/ZonedFreelistManager.cc @@ -236,8 +236,9 @@ void ZonedFreelistManager::allocate( uint64_t length, KeyValueDB::Transaction txn) { - dout(10) << __func__ << " 0x" << std::hex << offset << "~" << length << dendl; uint64_t zone_num = offset / zone_size; + dout(10) << __func__ << " 0x" << std::hex << offset << "~" << length + << " zone 0x" << zone_num << std::dec << dendl; zone_state_t zone_state; zone_state.increment_write_pointer(length); write_zone_state_to_db(zone_num, zone_state, txn); @@ -254,8 +255,9 @@ void ZonedFreelistManager::release( uint64_t length, KeyValueDB::Transaction txn) { - dout(10) << __func__ << " 0x" << std::hex << offset << "~" << length << dendl; uint64_t zone_num = offset / zone_size; + dout(10) << __func__ << " 0x" << std::hex << offset << "~" << length + << " zone 0x" << zone_num << std::dec << dendl; zone_state_t zone_state; zone_state.increment_num_dead_bytes(length); write_zone_state_to_db(zone_num, zone_state, txn); @@ -333,12 +335,46 @@ int ZonedFreelistManager::_read_cfg(cfg_reader_t cfg_reader) void ZonedFreelistManager::mark_zone_to_clean_free( uint64_t zone, + uint64_t write_pointer, + uint64_t dead, KeyValueDB *kvdb) { - dout(10) << __func__ << " zone 0x" << std::hex << zone << std::dec << dendl; - + dout(10) << __func__ << " zone 0x" << std::hex << zone + << " (dead 0x" << dead << " write pointer 0x" << write_pointer + << ")" << std::dec << dendl; + + if (true) { + string key; + _key_encode_u64(zone, &key); + KeyValueDB::Iterator it = kvdb->get_iterator(info_prefix); + it->lower_bound(key); + zone_state_t zs; + load_zone_state_from_db(zone, zs, it); + dout(20) << __func__ << " before " << zs << dendl; + ceph_assert(zs.num_dead_bytes == dead); + ceph_assert(zs.write_pointer == write_pointer); + } KeyValueDB::Transaction txn = kvdb->get_transaction(); - zone_state_t zone_state; - write_zone_state_to_db(zone, zone_state, txn); + + zone_state_t neg_zone_state; + neg_zone_state.num_dead_bytes = 0ll - (int64_t)dead; + neg_zone_state.write_pointer = 0ll - (int64_t)write_pointer; + write_zone_state_to_db(zone, neg_zone_state, txn); + + // block here until this commits so that we don't end up starting to allocate and + // write to the new zone before this fully commits. kvdb->submit_transaction_sync(txn); + + if (true) { + // read it back to verify it is really zero! + string key; + _key_encode_u64(zone, &key); + KeyValueDB::Iterator it = kvdb->get_iterator(info_prefix); + it->lower_bound(key); + zone_state_t zs; + load_zone_state_from_db(zone, zs, it); + dout(20) << __func__ << " read back " << zs << dendl; + ceph_assert(zs.num_dead_bytes == 0); + ceph_assert(zs.write_pointer == 0); + } } diff --git a/src/os/bluestore/ZonedFreelistManager.h b/src/os/bluestore/ZonedFreelistManager.h index e60a40222eb4f..4e0b040c80cd3 100644 --- a/src/os/bluestore/ZonedFreelistManager.h +++ b/src/os/bluestore/ZonedFreelistManager.h @@ -103,7 +103,9 @@ class ZonedFreelistManager : public FreelistManager { std::vector get_zone_states(KeyValueDB *kvdb) const; - void mark_zone_to_clean_free(uint64_t zone, KeyValueDB *kvdb); + void mark_zone_to_clean_free(uint64_t zone, + uint64_t write_pointer, uint64_t dead, + KeyValueDB *kvdb); }; #endif diff --git a/src/os/bluestore/zoned_types.h b/src/os/bluestore/zoned_types.h index 5507e64c7c58c..d8ca3a0c7c63a 100644 --- a/src/os/bluestore/zoned_types.h +++ b/src/os/bluestore/zoned_types.h @@ -28,6 +28,11 @@ struct zone_state_t { decode(num_dead_bytes, p); } + void reset() { + write_pointer = 0; + num_dead_bytes = 0; + } + uint64_t get_num_dead_bytes() const { return num_dead_bytes; } From 09c2753175466f22aa236fe2afe84179c9899e02 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Thu, 23 Sep 2021 10:23:11 -0500 Subject: [PATCH 0421/2866] os/bluestore/ZonedAllocator: handle alloc/release spanning zones The same txn may allocate the end of one zone and the start of the next. These get combined in an interval_set in bluestore and submitted as one extent to the freelist. Ensure that we spread the allocate (or release) over the appropriate zones. Signed-off-by: Sage Weil --- src/os/bluestore/ZonedFreelistManager.cc | 34 +++++++++++++++--------- 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/src/os/bluestore/ZonedFreelistManager.cc b/src/os/bluestore/ZonedFreelistManager.cc index 092e8d540aa90..8f7f05ab425e4 100644 --- a/src/os/bluestore/ZonedFreelistManager.cc +++ b/src/os/bluestore/ZonedFreelistManager.cc @@ -236,12 +236,17 @@ void ZonedFreelistManager::allocate( uint64_t length, KeyValueDB::Transaction txn) { - uint64_t zone_num = offset / zone_size; - dout(10) << __func__ << " 0x" << std::hex << offset << "~" << length - << " zone 0x" << zone_num << std::dec << dendl; - zone_state_t zone_state; - zone_state.increment_write_pointer(length); - write_zone_state_to_db(zone_num, zone_state, txn); + while (length > 0) { + uint64_t zone_num = offset / zone_size; + uint64_t this_len = std::min(length, zone_size - offset % zone_size); + dout(10) << __func__ << " 0x" << std::hex << offset << "~" << this_len + << " zone 0x" << zone_num << std::dec << dendl; + zone_state_t zone_state; + zone_state.increment_write_pointer(this_len); + write_zone_state_to_db(zone_num, zone_state, txn); + offset += this_len; + length -= this_len; + } } // Increments the number of dead bytes in a zone and writes the updated value to @@ -255,12 +260,17 @@ void ZonedFreelistManager::release( uint64_t length, KeyValueDB::Transaction txn) { - uint64_t zone_num = offset / zone_size; - dout(10) << __func__ << " 0x" << std::hex << offset << "~" << length - << " zone 0x" << zone_num << std::dec << dendl; - zone_state_t zone_state; - zone_state.increment_num_dead_bytes(length); - write_zone_state_to_db(zone_num, zone_state, txn); + while (length > 0) { + uint64_t zone_num = offset / zone_size; + uint64_t this_len = std::min(length, zone_size - offset % zone_size); + dout(10) << __func__ << " 0x" << std::hex << offset << "~" << this_len + << " zone 0x" << zone_num << std::dec << dendl; + zone_state_t zone_state; + zone_state.increment_num_dead_bytes(this_len); + write_zone_state_to_db(zone_num, zone_state, txn); + length -= this_len; + offset += this_len; + } } void ZonedFreelistManager::get_meta( From 389facfd43801c076622eb9894d6ed9825b515dd Mon Sep 17 00:00:00 2001 From: Adam Kupczyk Date: Fri, 17 Sep 2021 12:22:02 +0200 Subject: [PATCH 0422/2866] test/objectstore/run_smr_bluestore_test.sh Added execution permissions. Modified script to account for different lsscsi outputs (7th column was not device path on Ubuntu 20.04). Added comments. Signed-off-by: Adam Kupczyk --- .../objectstore/run_smr_bluestore_test.sh | 31 ++++++++++++++++--- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/src/test/objectstore/run_smr_bluestore_test.sh b/src/test/objectstore/run_smr_bluestore_test.sh index 5fa8544e0a1fc..d689cf2c50118 100644 --- a/src/test/objectstore/run_smr_bluestore_test.sh +++ b/src/test/objectstore/run_smr_bluestore_test.sh @@ -1,24 +1,47 @@ #!/bin/bash -ex +# 1) run_smr_bluestore_test.sh +# Setup smr device, run all tests + +# 2) run_smr_bluestore_test.sh --smr +# Setup smr device but skip tests failing on smr + + +before_creation=$(mktemp) +lsscsi > $before_creation + echo "cd /backstores/user:zbc create name=zbc0 size=20G cfgstring=model-HM/zsize-256/conv-10@zbc0.raw +/loopback create cd /loopback create naa.50014055e5f25aa0 -cd naa.50014055e5f25aa$1/luns +cd naa.50014055e5f25aa0/luns create /backstores/user:zbc/zbc0 0 " | sudo targetcli +sleep 1 #if too fast device does not show up +after_creation=$(mktemp) +lsscsi > $after_creation +if [[ $(diff $before_creation $after_creation | wc -l ) != 2 ]] +then + echo New zbc device not created + false +fi + function cleanup() { echo "cd /loopback -delete naa.50014055e5f25aa$1 +delete naa.50014055e5f25aa0 cd /backstores/user:zbc delete zbc0" | sudo targetcli sudo rm -f zbc0.raw + rm -f $before_creation $after_creation } trap cleanup EXIT -DEV=`lsscsi | grep zbc | awk '{print $7}'` - +DEV=$(diff $before_creation $after_creation |grep zbc |sed "s@.* /@/@") +sudo chmod 666 $DEV +# Need sudo +# https://patchwork.kernel.org/project/linux-block/patch/20210811110505.29649-3-Niklas.Cassel@wdc.com/ sudo ceph_test_objectstore \ --bluestore-block-path $DEV \ --gtest_filter=*/2 \ From 511539b7e0df4093b227530968de5fc5e2892ff9 Mon Sep 17 00:00:00 2001 From: Adam Kupczyk Date: Fri, 17 Sep 2021 12:22:58 +0200 Subject: [PATCH 0423/2866] ceph_test_objectstore: complain if SMR support not compiled in Signed-off-by: Adam Kupczyk --- src/test/objectstore/store_test.cc | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/test/objectstore/store_test.cc b/src/test/objectstore/store_test.cc index 39272aec9fe4e..cc5d3059c00af 100644 --- a/src/test/objectstore/store_test.cc +++ b/src/test/objectstore/store_test.cc @@ -9627,8 +9627,13 @@ int main(int argc, char **argv) { for (auto& i : args) { if (i == "--smr"s) { - derr << "smr" << dendl; +#if defined(HAVE_LIBZBD) + derr << "Adjusting tests for smr mode." << dendl; smr = true; +#else + derr << "smr mode selected, but support not compiled in" << dendl; + return 1; +#endif } } From d723f65938fda4507a256d3a0871bec5b17292fc Mon Sep 17 00:00:00 2001 From: Adam Kupczyk Date: Wed, 22 Sep 2021 15:18:49 +0200 Subject: [PATCH 0424/2866] ceph_test_objectstore: Test for fixing write pointer Test for checking if bluestore mount() can nicely handle unexpected change of SMR sequential zone write pointer. Signed-off-by: Adam Kupczyk --- src/test/objectstore/store_test.cc | 35 ++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/src/test/objectstore/store_test.cc b/src/test/objectstore/store_test.cc index cc5d3059c00af..1c44f99eba596 100644 --- a/src/test/objectstore/store_test.cc +++ b/src/test/objectstore/store_test.cc @@ -9225,6 +9225,41 @@ TEST_P(StoreTest, mergeRegionTest) { } } +TEST_P(StoreTest, FixSMRWritePointer) { + if(string(GetParam()) != "bluestore") + return; + if (!smr) + return; + int r = store->umount(); + ASSERT_EQ(0, r); + + // copied from StoreTestFixture + std::string path = GetParam() + ".test_temp_dir"s; + + std::string p = path + "/block"; + BlockDevice* bdev = BlockDevice::create(g_ceph_context, p, nullptr, nullptr, nullptr, nullptr); + r = bdev->open(p); + ASSERT_EQ(0, r); + ASSERT_EQ(true, bdev->is_smr()); + + std::vector wp = bdev->get_zones(); + uint64_t first_seq_zone = bdev->get_conventional_region_size() / bdev->get_zone_size(); + + IOContext ioc(g_ceph_context, NULL, true); + bufferlist bl; + bl.append(std::string(1024 * 1024, 'x')); + r = bdev->aio_write(wp[first_seq_zone], bl, &ioc, false); + ASSERT_EQ(0, r); + bdev->aio_submit(&ioc); + ioc.aio_wait(); + bdev->close(); + delete bdev; + + r = store->mount(); + ASSERT_EQ(0, r); +} + + TEST_P(StoreTestSpecificAUSize, BluestoreEnforceHWSettingsHdd) { if (string(GetParam()) != "bluestore") return; From fce3388a0bb481b8175029e188623a62a45d30df Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Fri, 1 Oct 2021 09:44:34 -0500 Subject: [PATCH 0425/2866] os/bluestore: expect smr fields IFF device is smr Signed-off-by: Sage Weil --- src/os/bluestore/BlueStore.cc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc index c37aff0c15dfc..0b43c65cab038 100644 --- a/src/os/bluestore/BlueStore.cc +++ b/src/os/bluestore/BlueStore.cc @@ -11863,6 +11863,9 @@ int BlueStore::_open_super_meta() auto p = bl.cbegin(); decode(zone_size, p); dout(1) << __func__ << " zone_size 0x" << std::hex << zone_size << std::dec << dendl; + ceph_assert(bdev->is_smr()); + } else { + ceph_assert(!bdev->is_smr()); } } { @@ -11873,6 +11876,9 @@ int BlueStore::_open_super_meta() decode(first_sequential_zone, p); dout(1) << __func__ << " first_sequential_zone 0x" << std::hex << first_sequential_zone << std::dec << dendl; + ceph_assert(bdev->is_smr()); + } else { + ceph_assert(!bdev->is_smr()); } } From 2dd7f4688a365eea46fc006a05674596810aad02 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Fri, 1 Oct 2021 12:43:23 -0500 Subject: [PATCH 0426/2866] os/bluestore/ZonedAllocator: count sequential only as 'free' Completely ignore the conventional region of the device. Signed-off-by: Sage Weil --- src/os/bluestore/ZonedAllocator.cc | 29 ++++++++++++++--------------- src/os/bluestore/ZonedAllocator.h | 2 +- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/src/os/bluestore/ZonedAllocator.cc b/src/os/bluestore/ZonedAllocator.cc index 8401eab1858a4..71ca51fe30c60 100644 --- a/src/os/bluestore/ZonedAllocator.cc +++ b/src/os/bluestore/ZonedAllocator.cc @@ -26,10 +26,10 @@ ZonedAllocator::ZonedAllocator(CephContext* cct, std::string_view name) : Allocator(name, size, blk_size), cct(cct), - num_free(0), size(size), conventional_size(_first_sequential_zone * _zone_size), sequential_size(size - conventional_size), + num_sequential_free(0), block_size(blk_size), zone_size(_zone_size), first_seq_zone_num(_first_sequential_zone), @@ -37,15 +37,15 @@ ZonedAllocator::ZonedAllocator(CephContext* cct, num_zones(size / zone_size) { ldout(cct, 10) << " size 0x" << std::hex << size - << " zone size 0x" << zone_size << std::dec - << " number of zones " << num_zones - << " first sequential zone " << starting_zone_num + << ", zone size 0x" << zone_size << std::dec + << ", number of zones 0x" << num_zones + << ", first sequential zone 0x" << starting_zone_num + << ", sequential size 0x" << sequential_size << std::dec << dendl; ceph_assert(size % zone_size == 0); zone_states.resize(num_zones); - num_free = num_zones * zone_size; } ZonedAllocator::~ZonedAllocator() @@ -101,7 +101,7 @@ int64_t ZonedAllocator::allocate( << std::dec << dendl; increment_write_pointer(zone_num, want_size); - num_free -= want_size; + num_sequential_free -= want_size; if (get_remaining_space(zone_num) == 0) { starting_zone_num = zone_num + 1; } @@ -135,7 +135,7 @@ void ZonedAllocator::release(const interval_set& release_set) uint64_t ZonedAllocator::get_free() { - return num_free; + return num_sequential_free; } void ZonedAllocator::dump() @@ -156,13 +156,13 @@ void ZonedAllocator::init_from_zone_pointers( std::lock_guard l(lock); ldout(cct, 10) << dendl; zone_states = std::move(_zone_states); - num_free = 0; + num_sequential_free = 0; for (size_t i = first_seq_zone_num; i < num_zones; ++i) { - num_free += zone_size - (zone_states[i].write_pointer % zone_size); + num_sequential_free += zone_size - (zone_states[i].write_pointer % zone_size); } uint64_t conventional_size = first_seq_zone_num * zone_size; uint64_t sequential_size = size - conventional_size; - ldout(cct, 10) << "free 0x" << std::hex << num_free + ldout(cct, 10) << "free 0x" << std::hex << num_sequential_free << " / 0x" << sequential_size << std::dec << dendl; } @@ -217,20 +217,19 @@ int64_t ZonedAllocator::pick_zone_to_clean(float min_score, uint64_t min_saved) void ZonedAllocator::reset_zone(uint32_t zone) { - num_free += zone_states[zone].write_pointer; + num_sequential_free += zone_states[zone].write_pointer; zone_states[zone].reset(); } bool ZonedAllocator::low_on_space(void) { std::lock_guard l(lock); - uint64_t sequential_num_free = num_free - conventional_size; - double free_ratio = static_cast(sequential_num_free) / sequential_size; + double free_ratio = static_cast(num_sequential_free) / sequential_size; - ldout(cct, 10) << " free 0x" << std::hex << sequential_num_free + ldout(cct, 10) << " free 0x" << std::hex << num_sequential_free << "/ 0x" << sequential_size << std::dec << ", free ratio is " << free_ratio << dendl; - ceph_assert(sequential_num_free <= sequential_size); + ceph_assert(num_sequential_free <= (int64_t)sequential_size); // TODO: make 0.25 tunable return free_ratio <= 0.25; diff --git a/src/os/bluestore/ZonedAllocator.h b/src/os/bluestore/ZonedAllocator.h index 12fa6cddd3ecc..9de9c8440bf4c 100644 --- a/src/os/bluestore/ZonedAllocator.h +++ b/src/os/bluestore/ZonedAllocator.h @@ -30,9 +30,9 @@ class ZonedAllocator : public Allocator { // atomic_alloc_and_submit_lock will be removed. ceph::mutex lock = ceph::make_mutex("ZonedAllocator::lock"); - std::atomic num_free; ///< total bytes in freelist uint64_t size; uint64_t conventional_size, sequential_size; + std::atomic num_sequential_free; ///< total bytes in freelist uint64_t block_size; uint64_t zone_size; uint64_t first_seq_zone_num; From 760a7b5b598583d56e01d86214625d6b6b4be6ec Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Fri, 1 Oct 2021 12:50:21 -0500 Subject: [PATCH 0427/2866] blk/zoned: make discard a no-op Discard is meaningless on SMR or ZNS since we are always explicitly managing the reset of entire zones. Signed-off-by: Sage Weil --- src/blk/zoned/HMSMRDevice.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/blk/zoned/HMSMRDevice.h b/src/blk/zoned/HMSMRDevice.h index b6df3200c6521..326f9ab27f4ba 100644 --- a/src/blk/zoned/HMSMRDevice.h +++ b/src/blk/zoned/HMSMRDevice.h @@ -46,6 +46,12 @@ class HMSMRDevice final : public KernelDevice { void reset_all_zones() override; void reset_zone(uint64_t zone) override; std::vector get_zones() override; + + int discard(uint64_t offset, uint64_t len) override { + // discard is a no-op on a zoned device + return 0; + } + }; #endif //CEPH_BLK_HMSMRDEVICE_H From d15d451cf7fb1ec924863ce8f517de7f8de9d0aa Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Fri, 1 Oct 2021 13:02:24 -0500 Subject: [PATCH 0428/2866] os/bluetore: configurable sleep period for cleaner Signed-off-by: Sage Weil --- src/common/options/global.yaml.in | 6 ++++++ src/os/bluestore/BlueStore.cc | 5 +++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/common/options/global.yaml.in b/src/common/options/global.yaml.in index 38b95313bc96d..dbd31ffc4505d 100644 --- a/src/common/options/global.yaml.in +++ b/src/common/options/global.yaml.in @@ -6069,3 +6069,9 @@ options: - spdk - pmem - hm_smr +- name: bluestore_cleaner_sleep_interval + type: float + level: advanced + desc: How long cleaner should sleep before re-checking utilization + default: 5 + with_legacy: true diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc index 0b43c65cab038..3c57055b32108 100644 --- a/src/os/bluestore/BlueStore.cc +++ b/src/os/bluestore/BlueStore.cc @@ -13102,8 +13102,9 @@ void BlueStore::_zoned_cleaner_thread() if (zoned_cleaner_stop) { break; } - dout(20) << __func__ << " sleep" << dendl; - zoned_cleaner_cond.wait(l); + auto period = ceph::make_timespan(cct->_conf->bluestore_cleaner_sleep_interval); + dout(20) << __func__ << " sleep for " << period << dendl; + zoned_cleaner_cond.wait_for(l, period); dout(20) << __func__ << " wake" << dendl; } else { l.unlock(); From 63b8f2f3ff0e46286cc1285e94c9f3c2d59660e8 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Fri, 1 Oct 2021 13:14:26 -0500 Subject: [PATCH 0429/2866] os/bluestore/ZonedFreelistManager: simplify freelist merge update vs zone reset Signed-off-by: Sage Weil --- src/os/bluestore/BlueStore.cc | 3 +-- src/os/bluestore/ZonedFreelistManager.cc | 34 +++++++++++++----------- src/os/bluestore/ZonedFreelistManager.h | 10 ++++--- 3 files changed, 26 insertions(+), 21 deletions(-) diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc index 3c57055b32108..320909d6133a9 100644 --- a/src/os/bluestore/BlueStore.cc +++ b/src/os/bluestore/BlueStore.cc @@ -13159,8 +13159,7 @@ void BlueStore::_zoned_clean_zone(uint64_t zone) bdev->reset_zone(zone); // record that we can now write there - f->mark_zone_to_clean_free(zone, a->get_write_pointer(zone), - a->get_dead_bytes(zone), db); + f->mark_zone_to_clean_free(zone, db); bdev->flush(); // then allow ourselves to start allocating there diff --git a/src/os/bluestore/ZonedFreelistManager.cc b/src/os/bluestore/ZonedFreelistManager.cc index 8f7f05ab425e4..a940005e7a990 100644 --- a/src/os/bluestore/ZonedFreelistManager.cc +++ b/src/os/bluestore/ZonedFreelistManager.cc @@ -30,7 +30,7 @@ using ceph::bufferptr; using ceph::decode; using ceph::encode; -void ZonedFreelistManager::write_zone_state_to_db( +void ZonedFreelistManager::write_zone_state_delta_to_db( uint64_t zone_num, const zone_state_t &zone_state, KeyValueDB::Transaction txn) @@ -42,6 +42,18 @@ void ZonedFreelistManager::write_zone_state_to_db( txn->merge(info_prefix, key, bl); } +void ZonedFreelistManager::write_zone_state_reset_to_db( + uint64_t zone_num, + const zone_state_t &zone_state, + KeyValueDB::Transaction txn) +{ + string key; + _key_encode_u64(zone_num, &key); + bufferlist bl; + zone_state.encode(bl); + txn->set(info_prefix, key, bl); +} + void ZonedFreelistManager::load_zone_state_from_db( uint64_t zone_num, zone_state_t &zone_state, @@ -62,7 +74,7 @@ void ZonedFreelistManager::init_zone_states(KeyValueDB::Transaction txn) dout(10) << __func__ << dendl; for (uint64_t zone_num = 0; zone_num < num_zones; ++zone_num) { zone_state_t zone_state; - write_zone_state_to_db(zone_num, zone_state, txn); + write_zone_state_reset_to_db(zone_num, zone_state, txn); } } @@ -243,7 +255,7 @@ void ZonedFreelistManager::allocate( << " zone 0x" << zone_num << std::dec << dendl; zone_state_t zone_state; zone_state.increment_write_pointer(this_len); - write_zone_state_to_db(zone_num, zone_state, txn); + write_zone_state_delta_to_db(zone_num, zone_state, txn); offset += this_len; length -= this_len; } @@ -267,7 +279,7 @@ void ZonedFreelistManager::release( << " zone 0x" << zone_num << std::dec << dendl; zone_state_t zone_state; zone_state.increment_num_dead_bytes(this_len); - write_zone_state_to_db(zone_num, zone_state, txn); + write_zone_state_delta_to_db(zone_num, zone_state, txn); length -= this_len; offset += this_len; } @@ -345,13 +357,9 @@ int ZonedFreelistManager::_read_cfg(cfg_reader_t cfg_reader) void ZonedFreelistManager::mark_zone_to_clean_free( uint64_t zone, - uint64_t write_pointer, - uint64_t dead, KeyValueDB *kvdb) { - dout(10) << __func__ << " zone 0x" << std::hex << zone - << " (dead 0x" << dead << " write pointer 0x" << write_pointer - << ")" << std::dec << dendl; + dout(10) << __func__ << " zone 0x" << std::hex << zone << std::dec << dendl; if (true) { string key; @@ -361,15 +369,11 @@ void ZonedFreelistManager::mark_zone_to_clean_free( zone_state_t zs; load_zone_state_from_db(zone, zs, it); dout(20) << __func__ << " before " << zs << dendl; - ceph_assert(zs.num_dead_bytes == dead); - ceph_assert(zs.write_pointer == write_pointer); } KeyValueDB::Transaction txn = kvdb->get_transaction(); - zone_state_t neg_zone_state; - neg_zone_state.num_dead_bytes = 0ll - (int64_t)dead; - neg_zone_state.write_pointer = 0ll - (int64_t)write_pointer; - write_zone_state_to_db(zone, neg_zone_state, txn); + zone_state_t empty_zone_state; + write_zone_state_reset_to_db(zone, empty_zone_state, txn); // block here until this commits so that we don't end up starting to allocate and // write to the new zone before this fully commits. diff --git a/src/os/bluestore/ZonedFreelistManager.h b/src/os/bluestore/ZonedFreelistManager.h index 4e0b040c80cd3..378a20f0a7969 100644 --- a/src/os/bluestore/ZonedFreelistManager.h +++ b/src/os/bluestore/ZonedFreelistManager.h @@ -36,9 +36,12 @@ class ZonedFreelistManager : public FreelistManager { KeyValueDB::Iterator enumerate_p; uint64_t enumerate_zone_num; - void write_zone_state_to_db(uint64_t zone_num, - const zone_state_t &zone_state, - KeyValueDB::Transaction txn); + void write_zone_state_delta_to_db(uint64_t zone_num, + const zone_state_t &zone_state, + KeyValueDB::Transaction txn); + void write_zone_state_reset_to_db(uint64_t zone_num, + const zone_state_t &zone_state, + KeyValueDB::Transaction txn); void load_zone_state_from_db(uint64_t zone_num, zone_state_t &zone_state, KeyValueDB::Iterator &it) const; @@ -104,7 +107,6 @@ class ZonedFreelistManager : public FreelistManager { std::vector get_zone_states(KeyValueDB *kvdb) const; void mark_zone_to_clean_free(uint64_t zone, - uint64_t write_pointer, uint64_t dead, KeyValueDB *kvdb); }; From c565b35da1158ffd21274f1477586460621f1561 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Fri, 1 Oct 2021 15:54:04 -0500 Subject: [PATCH 0430/2866] os/bluestore: drain transactions on cleaner zone finish Make sure all open transactions (the ones from the cleaner especially) get drained before we clear the device zone. Signed-off-by: Sage Weil --- src/os/bluestore/BlueStore.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc index 320909d6133a9..16b77bcdf18f9 100644 --- a/src/os/bluestore/BlueStore.cc +++ b/src/os/bluestore/BlueStore.cc @@ -13154,6 +13154,10 @@ void BlueStore::_zoned_clean_zone(uint64_t zone) return; } + // make sure transactions flush/drain/commit (and data is all rewritten + // safely elsewhere) before we blow away the cleaned zone + _osr_drain_all(); + // reset the device zone dout(10) << __func__ << " resetting zone 0x" << std::hex << zone << std::dec << dendl; bdev->reset_zone(zone); From ed10810c8f39b725c937d84f6146acbba08aae48 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Mon, 18 Oct 2021 12:30:29 -0500 Subject: [PATCH 0431/2866] os/bluestore/ZonedAllocator: fix allocate() search Signed-off-by: Sage Weil --- src/os/bluestore/ZonedAllocator.cc | 13 ++++++------- src/os/bluestore/ZonedAllocator.h | 1 + 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/os/bluestore/ZonedAllocator.cc b/src/os/bluestore/ZonedAllocator.cc index 71ca51fe30c60..0a535e361ce78 100644 --- a/src/os/bluestore/ZonedAllocator.cc +++ b/src/os/bluestore/ZonedAllocator.cc @@ -66,11 +66,12 @@ int64_t ZonedAllocator::allocate( ldout(cct, 10) << " trying to allocate 0x" << std::hex << want_size << std::dec << dendl; - // FIXME: start search at last zone we successfully allocated from, so we - // avoid re-checking full zones so much - + uint64_t left = num_zones - first_seq_zone_num; uint64_t zone_num = starting_zone_num; - for ( ; zone_num < num_zones; ++zone_num) { + for ( ; left > 0; ++zone_num, --left) { + if (zone_num == num_zones) { + zone_num = first_seq_zone_num; + } if (zone_num == cleaning_zone) { ldout(cct, 10) << " skipping zone 0x" << std::hex << zone_num << " because we are cleaning it" << std::dec << dendl; @@ -88,7 +89,7 @@ int64_t ZonedAllocator::allocate( break; } - if (zone_num == num_zones) { + if (left == 0) { ldout(cct, 10) << " failed to allocate" << dendl; return -ENOSPC; } @@ -160,8 +161,6 @@ void ZonedAllocator::init_from_zone_pointers( for (size_t i = first_seq_zone_num; i < num_zones; ++i) { num_sequential_free += zone_size - (zone_states[i].write_pointer % zone_size); } - uint64_t conventional_size = first_seq_zone_num * zone_size; - uint64_t sequential_size = size - conventional_size; ldout(cct, 10) << "free 0x" << std::hex << num_sequential_free << " / 0x" << sequential_size << std::dec << dendl; diff --git a/src/os/bluestore/ZonedAllocator.h b/src/os/bluestore/ZonedAllocator.h index 9de9c8440bf4c..685af9228fb14 100644 --- a/src/os/bluestore/ZonedAllocator.h +++ b/src/os/bluestore/ZonedAllocator.h @@ -81,6 +81,7 @@ class ZonedAllocator : public Allocator { return zone_states[zone].num_dead_bytes; } uint64_t get_live_bytes(uint32_t zone) { + std::scoped_lock l(lock); return zone_states[zone].write_pointer - zone_states[zone].num_dead_bytes; } From 9fe9213c7159a1ffc4989d64e2aa242fe6af3ec0 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Mon, 18 Oct 2021 14:10:15 -0500 Subject: [PATCH 0432/2866] os/bluestore/ZonedFreelistManager: remove sanity checks These were there when I was using the merge operator; not really needed since it's just setting zeroed values now. Signed-off-by: Sage Weil --- src/os/bluestore/ZonedFreelistManager.cc | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/src/os/bluestore/ZonedFreelistManager.cc b/src/os/bluestore/ZonedFreelistManager.cc index a940005e7a990..3a5bce66fe5d1 100644 --- a/src/os/bluestore/ZonedFreelistManager.cc +++ b/src/os/bluestore/ZonedFreelistManager.cc @@ -361,15 +361,6 @@ void ZonedFreelistManager::mark_zone_to_clean_free( { dout(10) << __func__ << " zone 0x" << std::hex << zone << std::dec << dendl; - if (true) { - string key; - _key_encode_u64(zone, &key); - KeyValueDB::Iterator it = kvdb->get_iterator(info_prefix); - it->lower_bound(key); - zone_state_t zs; - load_zone_state_from_db(zone, zs, it); - dout(20) << __func__ << " before " << zs << dendl; - } KeyValueDB::Transaction txn = kvdb->get_transaction(); zone_state_t empty_zone_state; @@ -378,17 +369,4 @@ void ZonedFreelistManager::mark_zone_to_clean_free( // block here until this commits so that we don't end up starting to allocate and // write to the new zone before this fully commits. kvdb->submit_transaction_sync(txn); - - if (true) { - // read it back to verify it is really zero! - string key; - _key_encode_u64(zone, &key); - KeyValueDB::Iterator it = kvdb->get_iterator(info_prefix); - it->lower_bound(key); - zone_state_t zs; - load_zone_state_from_db(zone, zs, it); - dout(20) << __func__ << " read back " << zs << dendl; - ceph_assert(zs.num_dead_bytes == 0); - ceph_assert(zs.write_pointer == 0); - } } From ff2fe48548a6a454e68bacec61e217a050d6c7a8 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Mon, 18 Oct 2021 14:23:20 -0500 Subject: [PATCH 0433/2866] os/bluestore: avoid casting zoned implementations again Signed-off-by: Sage Weil --- src/os/bluestore/BlueStore.cc | 10 ++++++---- src/os/bluestore/BlueStore.h | 4 +++- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc index 16b77bcdf18f9..a6293e3abcd58 100644 --- a/src/os/bluestore/BlueStore.cc +++ b/src/os/bluestore/BlueStore.cc @@ -13109,7 +13109,7 @@ void BlueStore::_zoned_cleaner_thread() } else { l.unlock(); a->set_cleaning_zone(zone_to_clean); - _zoned_clean_zone(zone_to_clean); + _zoned_clean_zone(zone_to_clean, a, f); a->clear_cleaning_zone(zone_to_clean); l.lock(); } @@ -13118,11 +13118,13 @@ void BlueStore::_zoned_cleaner_thread() zoned_cleaner_started = false; } -void BlueStore::_zoned_clean_zone(uint64_t zone) +void BlueStore::_zoned_clean_zone( + uint64_t zone, + ZonedAllocator *a, + ZonedFreelistManager *f + ) { dout(10) << __func__ << " cleaning zone 0x" << std::hex << zone << std::dec << dendl; - auto a = dynamic_cast(alloc); - auto f = dynamic_cast(fm); KeyValueDB::Iterator it = db->get_iterator(PREFIX_ZONED_CL_INFO); std::string zone_start; diff --git a/src/os/bluestore/BlueStore.h b/src/os/bluestore/BlueStore.h index f9e1b31d613a1..f3d16d1ace374 100644 --- a/src/os/bluestore/BlueStore.h +++ b/src/os/bluestore/BlueStore.h @@ -2479,7 +2479,9 @@ class BlueStore : public ObjectStore, void _zoned_cleaner_start(); void _zoned_cleaner_stop(); void _zoned_cleaner_thread(); - void _zoned_clean_zone(uint64_t zone_num); + void _zoned_clean_zone(uint64_t zone_num, + class ZonedAllocator *a, + class ZonedFreelistManager *f); void _clean_some(ghobject_t oid, uint32_t zone_num); #endif From 35b3ecb1ad3d07be6bb33114ce21ada600cf9808 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Mon, 18 Oct 2021 14:28:42 -0500 Subject: [PATCH 0434/2866] os/bluestore: use simpler map<> to track (onode, zone) -> offset Signed-off-by: Sage Weil --- src/os/bluestore/BlueStore.cc | 32 ++++++++++++++------------------ src/os/bluestore/BlueStore.h | 8 ++++---- 2 files changed, 18 insertions(+), 22 deletions(-) diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc index a6293e3abcd58..22f2fe32ef3ae 100644 --- a/src/os/bluestore/BlueStore.cc +++ b/src/os/bluestore/BlueStore.cc @@ -12309,26 +12309,22 @@ void BlueStore::_txc_finalize_kv(TransContext *txc, KeyValueDB::Transaction t) #ifdef HAVE_LIBZBD if (bdev->is_smr()) { for (auto& i : txc->old_zone_offset_refs) { - for (auto& j : i.second) { - dout(20) << __func__ << " rm ref zone 0x" << std::hex << j.first - << " offset 0x" << j.second << std::dec - << " -> " << i.first->oid << dendl; - string key; - get_zone_offset_object_key(j.first, j.second, i.first->oid, &key); - txc->t->rmkey(PREFIX_ZONED_CL_INFO, key); - } + dout(20) << __func__ << " rm ref zone 0x" << std::hex << i.first.second + << " offset 0x" << i.second << std::dec + << " -> " << i.first.first->oid << dendl; + string key; + get_zone_offset_object_key(i.first.second, i.second, i.first.first->oid, &key); + txc->t->rmkey(PREFIX_ZONED_CL_INFO, key); } for (auto& i : txc->new_zone_offset_refs) { - for (auto& j : i.second) { - // (zone, offset) -> oid - dout(20) << __func__ << " add ref zone 0x" << std::hex << j.first - << " offset 0x" << j.second << std::dec - << " -> " << i.first->oid << dendl; - string key; - get_zone_offset_object_key(j.first, j.second, i.first->oid, &key); - bufferlist v; - txc->t->set(PREFIX_ZONED_CL_INFO, key, v); - } + // (zone, offset) -> oid + dout(20) << __func__ << " add ref zone 0x" << std::hex << i.first.second + << " offset 0x" << i.second << std::dec + << " -> " << i.first.first->oid << dendl; + string key; + get_zone_offset_object_key(i.first.second, i.second, i.first.first->oid, &key); + bufferlist v; + txc->t->set(PREFIX_ZONED_CL_INFO, key, v); } } #endif diff --git a/src/os/bluestore/BlueStore.h b/src/os/bluestore/BlueStore.h index f3d16d1ace374..0a157edce1160 100644 --- a/src/os/bluestore/BlueStore.h +++ b/src/os/bluestore/BlueStore.h @@ -1608,8 +1608,8 @@ class BlueStore : public ObjectStore, // to that zone do not generate additional refs. This is a bit imprecise but // is sufficient to generate reasonably sequential reads when doing zone // cleaning with less metadata than a ref for every extent. - std::map> new_zone_offset_refs; - std::map> old_zone_offset_refs; + std::map, uint64_t> new_zone_offset_refs; + std::map, uint64_t> old_zone_offset_refs; #endif std::set shared_blobs; ///< these need to be updated/written @@ -1687,10 +1687,10 @@ class BlueStore : public ObjectStore, #ifdef HAVE_LIBZBD void note_write_zone_offset(OnodeRef& o, uint32_t zone, uint64_t offset) { o->onode.zone_offset_refs[zone] = offset; - new_zone_offset_refs[o][zone] = offset; + new_zone_offset_refs[std::make_pair(o, zone)] = offset; } void note_release_zone_offset(OnodeRef& o, uint32_t zone, uint64_t offset) { - old_zone_offset_refs[o][zone] = offset; + old_zone_offset_refs[std::make_pair(o, zone)] = offset; o->onode.zone_offset_refs.erase(zone); } #endif From e6c3022c1f317f6c4f1ef569c32998d9882cab38 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Tue, 19 Oct 2021 10:39:45 -0500 Subject: [PATCH 0435/2866] os/bluestore: add missing ' ' to LruOnodeCacheShare _[un]pin Signed-off-by: Sage Weil --- src/os/bluestore/BlueStore.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc index 22f2fe32ef3ae..92bead276ae05 100644 --- a/src/os/bluestore/BlueStore.cc +++ b/src/os/bluestore/BlueStore.cc @@ -1130,14 +1130,14 @@ struct LruOnodeCacheShard : public BlueStore::OnodeCacheShard { { lru.erase(lru.iterator_to(*o)); ++num_pinned; - dout(20) << __func__ << this << " " << " " << " " << o->oid << " pinned" << dendl; + dout(20) << __func__ << " " << this << " " << " " << " " << o->oid << " pinned" << dendl; } void _unpin(BlueStore::Onode* o) override { lru.push_front(*o); ceph_assert(num_pinned); --num_pinned; - dout(20) << __func__ << this << " " << " " << " " << o->oid << " unpinned" << dendl; + dout(20) << __func__ << " " << this << " " << " " << " " << o->oid << " unpinned" << dendl; } void _unpin_and_rm(BlueStore::Onode* o) override { From 9e4c576aa60c1b42132ff8c9c4e91534a4202b02 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Tue, 19 Oct 2021 15:09:37 -0500 Subject: [PATCH 0436/2866] os/bluestore: fix cleaner race with collection removal Signed-off-by: Sage Weil --- src/os/bluestore/BlueStore.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc index 92bead276ae05..b4d53766889ae 100644 --- a/src/os/bluestore/BlueStore.cc +++ b/src/os/bluestore/BlueStore.cc @@ -13176,6 +13176,10 @@ void BlueStore::_clean_some(ghobject_t oid, uint32_t zone) << dendl; CollectionRef cref = _get_collection_by_oid(oid); + if (!cref) { + dout(10) << __func__ << " can't find collection for " << oid << dendl; + return; + } Collection *c = cref.get(); // serialize io dispatch vs other transactions @@ -13184,7 +13188,7 @@ void BlueStore::_clean_some(ghobject_t oid, uint32_t zone) auto o = c->get_onode(oid, false); if (!o) { - derr << __func__ << " can't find " << oid << dendl; + dout(10) << __func__ << " can't find " << oid << dendl; return; } From c1a67d9af5f387fb9590660bf42ec58d18f5a5f7 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Tue, 19 Oct 2021 16:28:55 -0500 Subject: [PATCH 0437/2866] os/bluestore: _txc_create inside of alloc_and_submit_lock Create the transaction inside of the SMR lock. Otherwise, we may get a deadlock between the cleaner C and a normal write op W: W C _txc_create seq 1 lock alloc_and_submit _txc_create seq 2 ... unlock alloc_and_submit lock alloc_and_submit ... block on flush _txc_finish_io, but blocked by seq 1 The root issue here is the txc's are misordered with respect to the alloc_and_submit lock. Fix by moving the _txc_create inside the lock! Signed-off-by: Sage Weil --- src/os/bluestore/BlueStore.cc | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc index b4d53766889ae..02e51ed1f6f8c 100644 --- a/src/os/bluestore/BlueStore.cc +++ b/src/os/bluestore/BlueStore.cc @@ -13536,10 +13536,6 @@ int BlueStore::queue_transactions( OpSequencer *osr = c->osr.get(); dout(10) << __func__ << " ch " << c << " " << c->cid << dendl; - // prepare - TransContext *txc = _txc_create(static_cast(ch.get()), osr, - &on_commit, op); - // With HM-SMR drives (and ZNS SSDs) we want the I/O allocation and I/O // submission to happen atomically because if I/O submission happens in a // different order than I/O allocation, we end up issuing non-sequential @@ -13549,6 +13545,11 @@ int BlueStore::queue_transactions( if (bdev->is_smr()) { atomic_alloc_and_submit_lock.lock(); } + + // prepare + TransContext *txc = _txc_create(static_cast(ch.get()), osr, + &on_commit, op); + for (vector::iterator p = tls.begin(); p != tls.end(); ++p) { txc->bytes += (*p).get_num_bytes(); _txc_add_transaction(txc, &(*p)); From 3ddb07088e46646dedccbfa2cbe1dcc87fc6db3e Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Tue, 19 Oct 2021 17:04:34 -0500 Subject: [PATCH 0438/2866] os/bluestore: debug ExtentMap::update() I hit a case where the shard key size didn't match, and it looked as though this code somehow didn't get executed. Signed-off-by: Sage Weil --- src/os/bluestore/BlueStore.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc index 02e51ed1f6f8c..d87764c00b73c 100644 --- a/src/os/bluestore/BlueStore.cc +++ b/src/os/bluestore/BlueStore.cc @@ -2641,6 +2641,8 @@ void BlueStore::ExtentMap::update(KeyValueDB::Transaction t, // schedule DB update for dirty shards string key; for (auto& it : encoded_shards) { + dout(20) << __func__ << " encoding key for shard 0x" << std::hex + << it.shard->shard_info->offset << std::dec << dendl; it.shard->dirty = false; it.shard->shard_info->bytes = it.bl.length(); generate_extent_shard_key_and_apply( From e2f7bd70a379dfc142d633db77e18a06d528399b Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Wed, 20 Oct 2021 08:51:20 -0500 Subject: [PATCH 0439/2866] ceph_test_objectstore: skip BlueStoreUnshareBlobTest with SMR non-deterministic Signed-off-by: Sage Weil --- src/test/objectstore/store_test.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/test/objectstore/store_test.cc b/src/test/objectstore/store_test.cc index 1c44f99eba596..5b7e86e373522 100644 --- a/src/test/objectstore/store_test.cc +++ b/src/test/objectstore/store_test.cc @@ -3615,6 +3615,10 @@ TEST_P(StoreTest, SimpleCloneRangeTest) { TEST_P(StoreTest, BlueStoreUnshareBlobTest) { if (string(GetParam()) != "bluestore") return; + if (smr) { + cout << "SKIP: non-deterministic behavior with smr" << std::endl; + return; + } int r; coll_t cid; auto ch = store->create_new_collection(cid); From f68f9dd7c389cbd7846c366cd528769f382c0217 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Fri, 29 Oct 2021 09:49:17 -0500 Subject: [PATCH 0440/2866] cephadm: --config-json overrides --config or --keyring args These args aren't normally combined, but if --fsid and --name are provided, they may be inferred. Signed-off-by: Sage Weil --- src/cephadm/cephadm | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/cephadm/cephadm b/src/cephadm/cephadm index 35ef6e279e6f9..87c69987f5e69 100755 --- a/src/cephadm/cephadm +++ b/src/cephadm/cephadm @@ -2336,6 +2336,8 @@ def get_config_and_keyring(ctx): d = get_parm(ctx.config_json) config = d.get('config') keyring = d.get('keyring') + if config and keyring: + return config, keyring if 'config' in ctx and ctx.config: try: From b9fcbf2c030eeb05421b5a9f0a67d3a2e745499f Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Fri, 29 Oct 2021 09:53:43 -0500 Subject: [PATCH 0441/2866] Revert "Merge pull request #41325 from rhcs-dashboard/wip-cephadm_box-master" This reverts commit 78d6ac0d35e8db037bb0f42935528cc9fecf2d84, reversing changes made to 9a8de97f7af473a6a773323a463dc6603d7f4d39. This broke cephadm (infers config when it should not) Signed-off-by: Sage Weil --- ceph.spec.in | 2 - .../ceph_volume/devices/lvm/common.py | 6 - .../ceph_volume/devices/lvm/prepare.py | 7 +- src/ceph-volume/ceph_volume/process.py | 5 - src/cephadm/box/Dockerfile | 36 ---- src/cephadm/box/bootstrap.sh | 108 ----------- src/cephadm/box/docker-compose.yml | 44 ----- src/cephadm/box/docker/ceph/.bashrc | 0 src/cephadm/box/docker/ceph/Dockerfile | 9 - src/cephadm/box/docker/ceph/locale.conf | 2 - src/cephadm/box/get_ceph_image.sh | 12 -- src/cephadm/box/setup_loop.sh | 63 ------- src/cephadm/box/setup_ssh.sh | 50 ----- src/cephadm/box/start | 45 ----- src/cephadm/cephadm | 171 ++---------------- 15 files changed, 19 insertions(+), 541 deletions(-) delete mode 100644 src/cephadm/box/Dockerfile delete mode 100755 src/cephadm/box/bootstrap.sh delete mode 100644 src/cephadm/box/docker-compose.yml delete mode 100644 src/cephadm/box/docker/ceph/.bashrc delete mode 100644 src/cephadm/box/docker/ceph/Dockerfile delete mode 100644 src/cephadm/box/docker/ceph/locale.conf delete mode 100755 src/cephadm/box/get_ceph_image.sh delete mode 100755 src/cephadm/box/setup_loop.sh delete mode 100755 src/cephadm/box/setup_ssh.sh delete mode 100755 src/cephadm/box/start diff --git a/ceph.spec.in b/ceph.spec.in index 2c61b7e8bc4fd..3579d7e628eaf 100644 --- a/ceph.spec.in +++ b/ceph.spec.in @@ -465,8 +465,6 @@ Summary: Utility to bootstrap Ceph clusters BuildArch: noarch Requires: lvm2 Requires: python%{python3_pkgversion} -Requires: openssh-server -Requires: which %if 0%{?weak_deps} Recommends: podman >= 2.0.2 %endif diff --git a/src/ceph-volume/ceph_volume/devices/lvm/common.py b/src/ceph-volume/ceph_volume/devices/lvm/common.py index b7ce6473dd7a8..752f354f35aac 100644 --- a/src/ceph-volume/ceph_volume/devices/lvm/common.py +++ b/src/ceph-volume/ceph_volume/devices/lvm/common.py @@ -123,12 +123,6 @@ def rollback_osd(args, osd_id=None): 'type': int, 'default': 1, }, - '--no-tmpfs': { - 'action': 'store_true', - 'dest': 'no_tmpfs', - 'help': ('Disable tmpfs osd data directory with bluestore.' - 'Useful if you want to run lvm preprare from cephadm'), - }, } filestore_args = { diff --git a/src/ceph-volume/ceph_volume/devices/lvm/prepare.py b/src/ceph-volume/ceph_volume/devices/lvm/prepare.py index 6567c78d9f5ec..2f715fdba122c 100644 --- a/src/ceph-volume/ceph_volume/devices/lvm/prepare.py +++ b/src/ceph-volume/ceph_volume/devices/lvm/prepare.py @@ -81,7 +81,7 @@ def prepare_filestore(device, journal, secrets, tags, osd_id, fsid): ) -def prepare_bluestore(block, wal, db, secrets, tags, osd_id, fsid, tmpfs=True): +def prepare_bluestore(block, wal, db, secrets, tags, osd_id, fsid): """ :param block: The name of the logical volume for the bluestore data :param wal: a regular/plain disk or logical volume, to be used for block.wal @@ -104,7 +104,7 @@ def prepare_bluestore(block, wal, db, secrets, tags, osd_id, fsid, tmpfs=True): db = prepare_dmcrypt(key, db, 'db', tags) # create the directory - prepare_utils.create_osd_path(osd_id, tmpfs=tmpfs) + prepare_utils.create_osd_path(osd_id, tmpfs=True) # symlink the block prepare_utils.link_block(block, osd_id) # get the latest monmap @@ -384,8 +384,6 @@ def prepare(self): tags['ceph.type'] = 'block' block_lv.set_tags(tags) - tmpfs = not self.args.no_tmpfs - prepare_bluestore( block_lv.lv_path, wal_device, @@ -394,7 +392,6 @@ def prepare(self): tags, self.osd_id, osd_fsid, - tmpfs=tmpfs ) def main(self): diff --git a/src/ceph-volume/ceph_volume/process.py b/src/ceph-volume/ceph_volume/process.py index 400063ae085a3..e70986892b7fd 100644 --- a/src/ceph-volume/ceph_volume/process.py +++ b/src/ceph-volume/ceph_volume/process.py @@ -5,11 +5,6 @@ from ceph_volume import terminal from ceph_volume.util import as_bytes -import locale - -if locale.getpreferredencoding().upper() != 'UTF-8': - locale.setlocale(locale.LC_ALL, 'en_US.UTF-8') - import logging logger = logging.getLogger(__name__) diff --git a/src/cephadm/box/Dockerfile b/src/cephadm/box/Dockerfile deleted file mode 100644 index 3b685a22e9cae..0000000000000 --- a/src/cephadm/box/Dockerfile +++ /dev/null @@ -1,36 +0,0 @@ -# https://developers.redhat.com/blog/2014/05/05/running-systemd-within-docker-container/ -FROM centos:8 as centos-systemd -ENV container docker -ENV CEPHADM_PATH=/usr/local/sbin/cephadm -#RUN (cd /lib/systemd/system/sysinit.target.wants/; for i in *; do [ $i == systemd-tmpfiles-setup.service ] || rm -f $i; done); \ -#rm -f /lib/systemd/system/multi-user.target.wants/*;\ -#rm -f /etc/systemd/system/*.wants/*;\ -#rm -f /lib/systemd/system/local-fs.target.wants/*; \ -#rm -f /lib/systemd/system/sockets.target.wants/*udev*; \ -#rm -f /lib/systemd/system/sockets.target.wants/*initctl*; \ -#rm -f /lib/systemd/system/basic.target.wants/*;\ -#rm -f /lib/systemd/system/anaconda.target.wants/*; -RUN dnf -y install chrony firewalld lvm2 \ - openssh-server openssh-clients python3 \ - yum-utils sudo which && dnf clean all - -RUN systemctl enable chronyd firewalld sshd - - -FROM centos-systemd as centos-systemd-docker -# To cache cephadm images -RUN yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo -RUN dnf -y install docker-ce && \ - dnf clean all && systemctl enable docker - -# ssh utilities -RUN dnf install epel-release -y && dnf makecache && dnf install sshpass -y - -EXPOSE 8443 -EXPOSE 22 - -FROM centos-systemd-docker -WORKDIR /root -COPY start /usr/local/bin - -CMD [ "/usr/sbin/init" ] diff --git a/src/cephadm/box/bootstrap.sh b/src/cephadm/box/bootstrap.sh deleted file mode 100755 index 41ad55429b45f..0000000000000 --- a/src/cephadm/box/bootstrap.sh +++ /dev/null @@ -1,108 +0,0 @@ -#!/bin/bash - -OSDS=1 -HOSTS=0 -SKIP_LOOP=0 -SKIP_BOOTSTRAP=0 - -function print_usage() { - echo "./bootstrap.sh [OPTIONS]" - echo "options:" - echo " --hosts n: number of hosts to add" - echo " --osds n: number of osds to add" - echo " --update-ceph-image: create/update ceph image" - echo " --update-box-image: create/update cephadm box image" - echo " --skip-create-loop: skip creating loopback device" - echo " --skip-bootstrap: skip deploying the containers" - echo " -l | --list-hosts: list available cephad-box hosts/seed" - echo " -h | --help: this help :)" -} - -function docker-ips() { - docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}} %tab% {{.Name}} %tab% {{.Config.Hostname}}' $(docker ps -aq) | sed 's#%tab%#\t#g' | sed 's#/##g' | sort -t . -k 1,1n -k 2,2n -k 3,3n -k 4,4n -} - -while [ $# -ge 1 ]; do -case $1 in - -h | --help) - print_usage - exit - ;; - -l | --list-hosts) # TODO remove when ceph-ci updated - echo -e "IP\t\tName\t\t Hostname" - docker-ips | grep box - exit - ;; - --update-box-image) - echo Updating box image - docker build -t cephadm-box -f Dockerfile . - ;; - --update-ceph-image) # TODO remove when ceph-ci updated - echo Updating ceph image - source ./get_ceph_image.sh - ;; - --hosts) - HOSTS="$2" - echo "number of hosts: $HOSTS" - shift - ;; - --osds) - OSDS="$2" - echo "number of osds: $OSDS" - shift - ;; - --skip-create-loop) - echo Skiping loop creation - SKIP_LOOP=1 - ;; - --skip-bootstrap) - echo Skiping bootstrap of containers - SKIP_BOOTSTRAP=1 - ;; -esac -shift -done - -# TODO: remove when ceph-ci image has required deps -if [[ ! -a docker/ceph/image/quay.ceph.image.tar ]] -then - echo -e "\033[33mWARNING:\033[0m run ./get_ceph_image.sh to get an updated ceph-ci/ceph image with correct deps." - exit -fi - -if [[ $OSDS -eq 0 ]] -then - SKIP_LOOP=1 -fi - -if [[ $SKIP_LOOP -eq 0 ]] -then - source setup_loop.sh - create_loops $OSDS -fi - - -if [[ $SKIP_BOOTSTRAP -eq 0 ]] -then - # loops should be created before starting docker-compose or else docker could - # not find lvs - docker-compose down - docker-compose up --scale hosts=$HOSTS -d - sleep 3 - - IPS=$(docker-ips | grep "box_hosts" | awk '{ print $1 }') - echo "IPS: " - echo $IPS - - sudo sysctl net.ipv4.conf.all.forwarding=1 - sudo iptables -P FORWARD ACCEPT - - for ((i=1;i<=$HOSTS;i++)) - do - docker-compose exec --index=$i hosts /cephadm/box/setup_ssh.sh run-sshd - done - - docker-compose exec -e NUM_OSDS=${OSDS} seed /cephadm/box/start - - docker-compose exec -e HOST_IPS="${IPS}" seed /cephadm/box/setup_ssh.sh copy-cluster-ssh-key -fi diff --git a/src/cephadm/box/docker-compose.yml b/src/cephadm/box/docker-compose.yml deleted file mode 100644 index 79271761f2d5f..0000000000000 --- a/src/cephadm/box/docker-compose.yml +++ /dev/null @@ -1,44 +0,0 @@ -version: "2.4" -services: - cephadm-host-base: - build: - context: . - environment: - - CEPH_BRANCH=master - image: cephadm-box - # probably not needed with rootless Docker and cgroups v2 - privileged: true - cap_add: - - SYS_ADMIN - - NET_ADMIN - - SYS_TIME - - MKNOD - stop_signal: RTMIN+3 - volumes: - - "/sys/fs/cgroup:/sys/fs/cgroup:ro" - - ../../../:/ceph - - ..:/cephadm - networks: - - public - mem_limit: "20g" - devices: - - /dev/loop0:/dev/ttt:rwm - scale: -1 - seed: - extends: - service: cephadm-host-base - ports: - - "3000:3000" - - "8443:8443" - - "9095:9095" - scale: 1 - hosts: - extends: - service: cephadm-host-base - scale: 3 - - -volumes: - var-lib-docker: -networks: - public: diff --git a/src/cephadm/box/docker/ceph/.bashrc b/src/cephadm/box/docker/ceph/.bashrc deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/src/cephadm/box/docker/ceph/Dockerfile b/src/cephadm/box/docker/ceph/Dockerfile deleted file mode 100644 index 354728db2d181..0000000000000 --- a/src/cephadm/box/docker/ceph/Dockerfile +++ /dev/null @@ -1,9 +0,0 @@ -FROM quay.ceph.io/ceph-ci/ceph:master - -RUN dnf install which sudo glibc-all-langpacks langpacks-en -y -RUN yum -y install glibc-locale-source glibc-langpack-en - -RUN localedef -c -f UTF-8 -i en_US en_US.UTF-8 -COPY locale.conf /etc/locale.conf - -EXPOSE 8443 diff --git a/src/cephadm/box/docker/ceph/locale.conf b/src/cephadm/box/docker/ceph/locale.conf deleted file mode 100644 index 00d76c8cdb651..0000000000000 --- a/src/cephadm/box/docker/ceph/locale.conf +++ /dev/null @@ -1,2 +0,0 @@ -LANG="en_US.UTF-8" -LC_ALL="en_US.UTF-8" diff --git a/src/cephadm/box/get_ceph_image.sh b/src/cephadm/box/get_ceph_image.sh deleted file mode 100755 index 3c431c86695a2..0000000000000 --- a/src/cephadm/box/get_ceph_image.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -set -ex - -IMAGE=quay.ceph.io/ceph-ci/ceph:master -docker pull $IMAGE -# update image with deps -docker build -t $IMAGE docker/ceph -# store to later load within docker -mkdir -p docker/ceph/image -rm -f docker/ceph/image/quay.ceph.image.tar -docker save quay.ceph.io/ceph-ci/ceph:master -o docker/ceph/image/quay.ceph.image.tar diff --git a/src/cephadm/box/setup_loop.sh b/src/cephadm/box/setup_loop.sh deleted file mode 100755 index 661715e803149..0000000000000 --- a/src/cephadm/box/setup_loop.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/bin/bash - -set -e - -function create_loops() { - - NUM_OSDS=$1 - if [[ -z $NUM_OSDS ]]; then - echo "Call setup_loop to setup with more osds" - echo "Using default number of osds: 1." - NUM_OSDS=1 - fi - - # minimum 5 GB for each osd - SIZE=$(expr $NUM_OSDS \* 5) - # extra space just in case - SIZE=$(expr $SIZE + 2) - - echo "Using ${SIZE} GB of space" - - # look for an available loop device - avail_loop=$(sudo losetup -f) - loop_name=$(basename -- $avail_loop) - - if [[ ! -e $avail_loop ]] - then - # in case we have to create the loop, find the minor device number. - num_loops=$(lsmod | grep loop | awk '{print $3}') - num_loops=$((num_loops + 1)) - echo creating loop $avail_loop minor: $num_loops - mknod $avail_loop b 7 $num_loops - fi - - if mountpoint -q $avail_loop - then - sudo umount $avail_loop - fi - - if [[ ! -z $(losetup -l | grep $avail_loop) ]] - then - sudo losetup -d $avail_loop - fi - - if [[ ! -e loop-images ]] - then - mkdir -p loop-images - fi - sudo rm -f loop-images/* - sudo dd if=/dev/zero of="loop-images/disk${loop_name}.img" bs=1G count=$SIZE - sudo losetup $avail_loop "loop-images/disk${loop_name}.img" - - if [[ ! -z $(sudo vgs | grep vg1) ]] - then - sudo lvm vgremove -f -y vg1 - fi - sudo pvcreate $avail_loop - sudo vgcreate vg1 $avail_loop - - for ((i=0;i<$NUM_OSDS;i++)); do - sudo vgchange --refresh - sudo lvcreate --size 5G --name "lv${i}" "vg1" - done; -} diff --git a/src/cephadm/box/setup_ssh.sh b/src/cephadm/box/setup_ssh.sh deleted file mode 100755 index 5b81c2cbf6852..0000000000000 --- a/src/cephadm/box/setup_ssh.sh +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env bash - -set -e - -function run-sshd() { - echo "Creating sshd server on $(hostname):$(hostname -i)" - # SSH - if [[ ! -f "/root/.ssh/id_rsa" ]]; then - mkdir -p ~/.ssh - chmod 700 ~/.ssh - ssh-keygen -b 2048 -t rsa -f ~/.ssh/id_rsa -q -N "" - fi - - cat ~/.ssh/id_rsa.pub >> /root/.ssh/authorized_keys - if [[ ! -f "/root/.ssh/known_hosts" ]]; then - ssh-keygen -A - fi - - # change password - echo "root:root" | chpasswd - echo "PermitRootLogin yes" >> /etc/ssh/sshd_config - echo "PasswordAuthentication yes" >> /etc/ssh/sshd_config - - /usr/sbin/sshd - echo "sshd finished" -} - -function copy-cluster-ssh-key() { - echo "Adding cluster ssh key to all hosts: ${HOST_IPS}" - HOST_IPS=$(echo $HOST_IPS) - for ip in $(echo $HOST_IPS) - do - if [[ ! $ip == $(hostname -i) ]] - then - echo $ip - # copy cluster key - sshpass -p "root" ssh-copy-id -f -o StrictHostKeyChecking=no -i /etc/ceph/ceph.pub "root@${ip}" - fi - done - echo "Finished adding keys, you can now add existing hosts containers to the cluster!" -} - -case $1 in - run-sshd) - run-sshd - ;; - copy-cluster-ssh-key) - copy-cluster-ssh-key - ;; -esac diff --git a/src/cephadm/box/start b/src/cephadm/box/start deleted file mode 100755 index b5883eeb932cc..0000000000000 --- a/src/cephadm/box/start +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env bash -set -euxo pipefail - -# link so we can debug cephadm -ln -s -f /cephadm/cephadm $CEPHADM_PATH -chmod +x $CEPHADM_PATH - -tail -f /var/log/ceph/cephadm.log 1>&2 & - -EXTRA_ARGS=() -if [[ -n "${SHARED_CEPH_FOLDER-}" ]]; then - EXTRA_ARGS+=(--shared_ceph_folder "$SHARED_CEPH_FOLDER") -fi - -docker load < /cephadm/box/docker/ceph/image/quay.ceph.image.tar - -# cephadm guid error because it sometimes tries to use quay.ceph.io/ceph-ci/ceph: -# instead of master's tag -export CEPHADM_IMAGE=quay.ceph.io/ceph-ci/ceph:master -echo "export CEPHADM_IMAGE=quay.ceph.io/ceph-ci/ceph:master" >> ~/.bashrc - -if [[ -n "$CEPHADM_IMAGE" ]]; then - EXTRA_ARGS+=--skip-pull -fi - -export CEPH_SOURCE_FOLDER=/ceph -$CEPHADM_PATH --verbose bootstrap \ - --mon-ip "$(hostname -i)" \ - --allow-fqdn-hostname \ - --initial-dashboard-password admin \ - --dashboard-password-noupdate \ - --shared_ceph_folder /ceph \ - "${EXTRA_ARGS[@]}" - -# make sure vg and lvs are visible -vgchange --refresh -for((i=0;i<$NUM_OSDS;i++)); do - echo "Creating osd.${i}" - # create osd folder - $CEPHADM_PATH ceph-volume --shared_ceph_folder /ceph lvm create --bluestore --no-tmpfs --data "/dev/vg1/lv${i}" --no-systemd - echo "Deploying osd.${i}..." - # deploy osd with osd data folder - $CEPHADM_PATH deploy --name "osd.${i}" - echo "osd.${i} deployed!" -done; diff --git a/src/cephadm/cephadm b/src/cephadm/cephadm index ef41c1aea7a20..35ef6e279e6f9 100755 --- a/src/cephadm/cephadm +++ b/src/cephadm/cephadm @@ -1783,6 +1783,7 @@ def default_image(func: FuncT) -> FuncT: ctx.image = _get_default_image(ctx) return func(ctx) + return cast(FuncT, _default_image) @@ -2214,6 +2215,7 @@ def create_daemon_dirs(ctx, fsid, daemon_type, daemon_id, uid, gid, # type: (CephadmContext, str, str, Union[int, str], int, int, Optional[str], Optional[str]) -> None data_dir = make_data_dir(ctx, fsid, daemon_type, daemon_id, uid=uid, gid=gid) make_log_dir(ctx, fsid, uid=uid, gid=gid) + if config: config_path = os.path.join(data_dir, 'config') with open(config_path, 'w') as f: @@ -2612,79 +2614,6 @@ def extract_uid_gid(ctx, img='', file_path='/var/lib/ceph'): raise RuntimeError('uid/gid not found') -def validate_osd_data_dir(data_dir): - # type: (str) -> None - required_files = ['keyring', 'block', 'type', 'config'] - current_files = os.listdir(data_dir) - error_msg = '' - for file in required_files: - if file not in current_files: - error_msg += f'File {file} not found in {data_dir}\n' - if error_msg: - raise RuntimeError(error_msg) - - -def configure_osd_data_dir(ctx, fsid, daemon_id, uid, gid): - # type: (CephadmContext, str, Union[int, str], int, int) -> None - daemon_type = 'osd' - data_dir = get_data_dir(fsid, ctx.data_dir, daemon_type, daemon_id) - - # Ensure user:group is the expected - for f in os.listdir(data_dir): - os.chown(os.path.join(data_dir, f), uid, gid) - - # Create minimal config - touch(os.path.join(data_dir, 'config'), uid, gid) - mounts = get_container_mounts(ctx, fsid, daemon_type, daemon_id, no_config=True) - mounts[data_dir] = '/var/lib/ceph/osd/ceph-%s' % daemon_id - mounts['/etc/ceph/ceph.conf'] = '/etc/ceph/ceph.conf:z' - mounts['/etc/ceph/ceph.client.admin.keyring'] = '/etc/ceph/ceph.keyring:z' - - CephContainer( - ctx, - image=ctx.image, - entrypoint='/usr/bin/ceph', - args=[ - 'config', 'generate-minimal-conf', - '-o', '/var/lib/ceph/osd/ceph-%s/config' % daemon_id - ], - privileged=True, - volume_mounts=mounts - ).run() - - # Create keyring and then import - key = CephContainer( - ctx, - image=ctx.image, - entrypoint='/usr/bin/ceph-authtool', - args=['--gen-print-key'], - ).run().strip() - - keyring = ('[%s.%s]\n' - '\tkey = %s\n' - '\tcaps osd = allow *\n' - '\tcaps mon = allow *\n' - % (daemon_type, daemon_id, key)) - with open(os.path.join(data_dir, 'keyring'), 'w+') as f: - os.fchmod(f.fileno(), 0o600) - os.fchown(f.fileno(), uid, gid) - f.write(keyring) - CephContainer( - ctx, - image=ctx.image, - entrypoint='/usr/bin/ceph', - args=[ - 'auth', 'import', - '-i', '/var/lib/ceph/osd/ceph-%s/keyring' % daemon_id - ], - privileged=True, - volume_mounts=mounts - ).run() - - # Validate we have needed files - validate_osd_data_dir(data_dir) - - def deploy_daemon(ctx, fsid, daemon_type, daemon_id, c, uid, gid, config=None, keyring=None, osd_fsid=None, @@ -2751,9 +2680,6 @@ def deploy_daemon(ctx, fsid, daemon_type, daemon_id, c, uid, gid, uid, gid, config, keyring) - if daemon_type == 'osd': - configure_osd_data_dir(ctx, fsid, daemon_id, uid, gid) - if not reconfig: if daemon_type == CephadmAgent.daemon_type: if ctx.config_json == '-': @@ -2865,14 +2791,6 @@ def deploy_daemon_units( ) -> None: # cmd data_dir = get_data_dir(fsid, ctx.data_dir, daemon_type, daemon_id) - - # if osd then try to read parameters if not provided - if daemon_type == 'osd': - osd_fsid_path = os.path.join(data_dir, 'fsid') - if 'fsid' in os.listdir(data_dir) and not osd_fsid: - with open(osd_fsid_path, 'r') as f: - osd_fsid = f.read() - with open(data_dir + '/unit.run.new', 'w') as f, \ open(data_dir + '/unit.meta.new', 'w') as metaf: f.write('set -e\n') @@ -3002,9 +2920,7 @@ def deploy_daemon_units( verbosity=CallVerbosity.DEBUG) if enable: call_throws(ctx, ['systemctl', 'enable', unit_name]) - if start: - clean_cgroup(ctx, fsid, unit_name) call_throws(ctx, ['systemctl', 'start', unit_name]) @@ -4791,10 +4707,8 @@ def extract_uid_gid_monitoring(ctx, daemon_type): @default_image -@infer_fsid def command_deploy(ctx): # type: (CephadmContext) -> None - assert ctx.fsid daemon_type, daemon_id = ctx.name.split('.', 1) lock = FileLock(ctx, ctx.fsid) @@ -4975,7 +4889,6 @@ def command_shell(ctx): mounts[pathify(ctx.config)] = '/etc/ceph/ceph.conf:z' if ctx.keyring: mounts[pathify(ctx.keyring)] = '/etc/ceph/ceph.keyring:z' - if ctx.mount: for _mount in ctx.mount: split_src_dst = _mount.split(':') @@ -5077,6 +4990,7 @@ def command_ceph_volume(ctx): (uid, gid) = (0, 0) # ceph-volume runs as root mounts = get_container_mounts(ctx, ctx.fsid, 'osd', None) + tmp_config = None tmp_keyring = None @@ -5087,72 +5001,24 @@ def command_ceph_volume(ctx): tmp_config = write_tmp(config, uid, gid) mounts[tmp_config.name] = '/etc/ceph/ceph.conf:z' - # Ceph-volume uses the bootstrap-osd key in order to do its operations. - # This function retrieves the keyring so it can be provided. - - def get_bootstrap_osd_keyring() -> Optional[str]: - if not ctx.keyring and os.path.exists(SHELL_DEFAULT_KEYRING): - ctx.keyring = SHELL_DEFAULT_KEYRING - (config, keyring) = get_config_and_keyring(ctx) - - mounts = get_container_mounts(ctx, ctx.fsid, 'osd', None, - no_config=True if ctx.config else False) - if ctx.config: - mounts[pathify(ctx.config)] = '/etc/ceph/ceph.conf:z' - if ctx.keyring: - mounts[pathify(ctx.keyring)] = '/etc/ceph/ceph.keyring:z' - c = CephContainer( - ctx, - image=ctx.image, - entrypoint='/usr/bin/ceph', - args='auth get client.bootstrap-osd'.split(), - volume_mounts=mounts, - ) - out, err, code = call_throws(ctx, c.run_cmd()) - if not code: - return out - else: - return None - - if not keyring: - keyring = get_bootstrap_osd_keyring() - if keyring: # tmp keyring file tmp_keyring = write_tmp(keyring, uid, gid) mounts[tmp_keyring.name] = '/var/lib/ceph/bootstrap-osd/ceph.keyring:z' - # If ceph-volume creates osd data directories which won't be persisted - # so we use a tmp dir for that. - # FIXME: probably we can use /var/lib/ceph/{fsid}? - with tempfile.TemporaryDirectory() as tmp_osd_dir: - # match /var/lib/ceph/osd/ dir permissions - os.chown(tmp_osd_dir, uid, gid) - os.chmod(tmp_osd_dir, 0o755) - # store newly created osds here - mounts[tmp_osd_dir] = '/var/lib/ceph/osd/:z' - - c = CephContainer( - ctx, - image=ctx.image, - entrypoint='/usr/sbin/ceph-volume', - envs=ctx.env, - args=ctx.command, - privileged=True, - volume_mounts=mounts, - ) + c = CephContainer( + ctx, + image=ctx.image, + entrypoint='/usr/sbin/ceph-volume', + envs=ctx.env, + args=ctx.command, + privileged=True, + volume_mounts=mounts, + ) - out, err, code = call_throws(ctx, c.run_cmd()) - if not code: - print(out) - else: - print(err) - # If osds were created move osd's data directories - for osd_folder_name in os.listdir(tmp_osd_dir): - if 'ceph-' in osd_folder_name[:5]: - osd_id = osd_folder_name[5:] - osd_data_dir = os.path.join(tmp_osd_dir, osd_folder_name) - copy_tree(ctx, [osd_data_dir], f'/var/lib/ceph/{ctx.fsid}/osd.{osd_id}', uid=uid, gid=gid) + out, err, code = call_throws(ctx, c.run_cmd()) + if not code: + print(out) ################################## @@ -7718,7 +7584,7 @@ def _get_parser(): help='cluster FSID') parser_ceph_volume.add_argument( '--config-json', - help='JSON file with config and (client.bootstrap-osd) key') + help='JSON file with config and (client.bootrap-osd) key') parser_ceph_volume.add_argument( '--config', '-c', help='ceph conf file') @@ -7728,10 +7594,6 @@ def _get_parser(): parser_ceph_volume.add_argument( 'command', nargs=argparse.REMAINDER, help='command') - parser_ceph_volume.add_argument( - '--shared_ceph_folder', - metavar='CEPH_SOURCE_FOLDER', - help='Development mode. Several folders in containers are volumes mapped to different sub-folders in the ceph source folder') parser_zap_osds = subparsers.add_parser( 'zap-osds', help='zap all OSDs associated with a particular fsid') @@ -7953,6 +7815,7 @@ def _get_parser(): help='daemon name (type.id)') parser_deploy.add_argument( '--fsid', + required=True, help='cluster FSID') parser_deploy.add_argument( '--config', '-c', From 24d3a01a3019d30fc1d4dc4963a3942665243a48 Mon Sep 17 00:00:00 2001 From: Rishabh Dave Date: Fri, 29 Oct 2021 20:34:12 +0530 Subject: [PATCH 0442/2866] mgr/dashboard: all pyfakefs must be pinned on same version Without this patch execution of install-deps.sh fails with an error. Fixes: https://tracker.ceph.com/issues/53088 Signed-off-by: Rishabh Dave --- src/pybind/mgr/dashboard/requirements-test.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pybind/mgr/dashboard/requirements-test.txt b/src/pybind/mgr/dashboard/requirements-test.txt index 90cd7f2b121f2..625f5c358f1cb 100644 --- a/src/pybind/mgr/dashboard/requirements-test.txt +++ b/src/pybind/mgr/dashboard/requirements-test.txt @@ -1,3 +1,3 @@ pytest-cov pytest-instafail -pyfakefs +pyfakefs==4.5.0 From b605b786a2ec95606a2c2791cc643dda3fbe24cc Mon Sep 17 00:00:00 2001 From: AndrewSharapov <93389903+AndrewSharapov@users.noreply.github.com> Date: Fri, 29 Oct 2021 18:10:20 +0300 Subject: [PATCH 0443/2866] mgr/cephadm: Fixed spawning ip addresses list for public network interface. Eevery call of find_ip_on_host() actually duplicates the list of public ip addresses in self.networks, while it should NOT change it. As the result value of key mgr/cephadm/host. in kv store becomes very large and may cause crash of ceph mgr. Signed-off-by: Andrew Sharapov --- src/pybind/mgr/cephadm/schedule.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pybind/mgr/cephadm/schedule.py b/src/pybind/mgr/cephadm/schedule.py index 8f4f02e5e4b32..5004ca0fb0219 100644 --- a/src/pybind/mgr/cephadm/schedule.py +++ b/src/pybind/mgr/cephadm/schedule.py @@ -361,8 +361,8 @@ def expand_candidates(ls: List[DaemonPlacement], num: int) -> List[DaemonPlaceme def find_ip_on_host(self, hostname: str, subnets: List[str]) -> Optional[str]: for subnet in subnets: ips: List[str] = [] - for iface, ips in self.networks.get(hostname, {}).get(subnet, {}).items(): - ips.extend(ips) + for iface, iface_ips in self.networks.get(hostname, {}).get(subnet, {}).items(): + ips.extend(iface_ips) if ips: return sorted(ips)[0] return None From e24171f8180bc60fb1876b7c7097f4fa45e17325 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Thu, 28 Oct 2021 18:11:38 -0400 Subject: [PATCH 0444/2866] pybind/mgr/mgr_module: add tool_exec() command Run a (ceph) tool with the appropriate keyring and entity name. Signed-off-by: Sage Weil --- src/pybind/mgr/mgr_module.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/pybind/mgr/mgr_module.py b/src/pybind/mgr/mgr_module.py index fa71980ee99dc..73839000cd565 100644 --- a/src/pybind/mgr/mgr_module.py +++ b/src/pybind/mgr/mgr_module.py @@ -1557,6 +1557,34 @@ def send_command( """ self._ceph_send_command(result, svc_type, svc_id, command, tag, inbuf) + def tool_exec( + self, + args: List[str], + timeout: int = 10, + stdin: Optional[bytes] = None + ) -> Tuple[int, str, str]: + try: + tool = args.pop(0) + cmd = [ + tool, + '-k', str(self.get_ceph_option('keyring')), + '-n', f'mgr.{self.get_mgr_id()}', + ] + args + self.log.debug('exec: ' + ' '.join(cmd)) + p = subprocess.run( + cmd, + input=stdin, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + timeout=timeout, + ) + except subprocess.TimeoutExpired as ex: + self.log.error(ex) + return -errno.ETIMEDOUT, '', str(ex) + if p.returncode: + self.log.error(f'Non-zero return from {cmd}: {p.stderr.decode()}') + return p.returncode, p.stdout.decode(), p.stderr.decode() + def set_health_checks(self, checks: HealthChecksT) -> None: """ Set the module's current map of health checks. Argument is a From 8dcdde71512f951e048dff5cd75cca5d0e4b5641 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Fri, 29 Oct 2021 11:27:50 -0400 Subject: [PATCH 0445/2866] mgr/nfs/export: use tool_exec() instead of private _exec() Fixes: https://tracker.ceph.com/issues/53096 Signed-off-by: Sage Weil Signed-off-by: Michael Fritch --- src/pybind/mgr/nfs/export.py | 31 ++++++---------------------- src/pybind/mgr/nfs/tests/test_nfs.py | 4 ++-- 2 files changed, 8 insertions(+), 27 deletions(-) diff --git a/src/pybind/mgr/nfs/export.py b/src/pybind/mgr/nfs/export.py index 7c2666bf19550..b23a95e04632d 100644 --- a/src/pybind/mgr/nfs/export.py +++ b/src/pybind/mgr/nfs/export.py @@ -1,7 +1,6 @@ import errno import json import logging -import subprocess from typing import List, Any, Dict, Tuple, Optional, TYPE_CHECKING, TypeVar, Callable, cast from os.path import normpath @@ -134,26 +133,6 @@ def _check_rados_notify(ioctx: Any, obj: str) -> None: except TimedOut: log.exception("Ganesha timed out") - def _exec(self, args: List[str]) -> Tuple[int, str, str]: - try: - util = args.pop(0) - cmd = [ - util, - '-k', str(self.mgr.get_ceph_option('keyring')), - '-n', f'mgr.{self.mgr.get_mgr_id()}', - ] + args - log.debug('exec: ' + ' '.join(cmd)) - p = subprocess.run( - cmd, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, - timeout=10, - ) - except subprocess.CalledProcessError as ex: - log.error('Error executing <<%s>>: %s', ex.cmd, ex.output) - except subprocess.TimeoutExpired: - log.error('timeout (10s) executing <<%s>>', cmd) - return p.returncode, p.stdout.decode(), p.stderr.decode() - @property def exports(self) -> Dict[str, List[Export]]: if self._exports is None: @@ -226,14 +205,15 @@ def _create_export_user(self, export: Export) -> None: elif isinstance(export.fsal, RGWFSAL): rgwfsal = cast(RGWFSAL, export.fsal) - ret, out, err = self._exec(['radosgw-admin', 'bucket', 'stats', '--bucket', - export.path]) + ret, out, err = self.mgr.tool_exec( + ['radosgw-admin', 'bucket', 'stats', '--bucket', export.path] + ) if ret: raise NFSException(f'Failed to fetch owner for bucket {export.path}') j = json.loads(out) owner = j.get('owner', '') rgwfsal.user_id = owner - ret, out, err = self._exec([ + ret, out, err = self.mgr.tool_exec([ 'radosgw-admin', 'user', 'info', '--uid', owner ]) if ret: @@ -585,7 +565,8 @@ def create_export_from_dict(self, ex_dict["cluster_id"] = cluster_id export = Export.from_dict(ex_id, ex_dict) export.validate(self.mgr) - log.debug("Successfully created %s export-%s from dict for cluster %s", fsal_type, ex_id, cluster_id) + log.debug("Successfully created %s export-%s from dict for cluster %s", + fsal_type, ex_id, cluster_id) return export def create_cephfs_export(self, diff --git a/src/pybind/mgr/nfs/tests/test_nfs.py b/src/pybind/mgr/nfs/tests/test_nfs.py index d6f73ae456e65..6fc4246e95880 100644 --- a/src/pybind/mgr/nfs/tests/test_nfs.py +++ b/src/pybind/mgr/nfs/tests/test_nfs.py @@ -6,7 +6,7 @@ from contextlib import contextmanager from unittest import mock from unittest.mock import MagicMock -from mgr_module import NFS_POOL_NAME +from mgr_module import MgrModule, NFS_POOL_NAME from rados import ObjectNotFound @@ -222,7 +222,7 @@ def mock_list_daemons(cls, *args, **kwargs): return_value=[self.cluster_id]), \ mock.patch('nfs.export.restart_nfs_service'), \ mock.patch('nfs.cluster.restart_nfs_service'), \ - mock.patch('nfs.export.ExportMgr._exec', mock_exec), \ + mock.patch.object(MgrModule, 'tool_exec', mock_exec), \ mock.patch('nfs.export.check_fs', return_value=True), \ mock.patch('nfs.export_utils.check_fs', return_value=True), \ mock.patch('nfs.export.ExportMgr._create_user_key', From 0f0e5993a9c40bd08688c69d04c5bfa1cd179476 Mon Sep 17 00:00:00 2001 From: chunmei-liu Date: Sun, 24 Oct 2021 14:41:32 -0700 Subject: [PATCH 0446/2866] crimson: osd meta data read from and write to files Signed-off-by: chunmei-liu --- src/crimson/os/seastore/seastore.cc | 96 +++++++++++++++++------------ 1 file changed, 58 insertions(+), 38 deletions(-) diff --git a/src/crimson/os/seastore/seastore.cc b/src/crimson/os/seastore/seastore.cc index e12ac875153f7..452bba333d780 100644 --- a/src/crimson/os/seastore/seastore.cc +++ b/src/crimson/os/seastore/seastore.cc @@ -1220,6 +1220,19 @@ boost::intrusive_ptr SeaStore::_get_collection(const coll_t& return new SeastoreCollection{cid}; } +seastar::future<> SeaStore::write_meta_file(const std::string& key, + const std::string& value) +{ + std::string path = fmt::format("{}/{}", root, key); + std::string fvalue = value; + fvalue += "\n"; + ceph::bufferptr bp(fvalue.length()); + ceph::bufferlist bl; + bp.copy_in(0, fvalue.length(), fvalue.c_str()); + bl.push_back(bp); + return crimson::write_file(std::move(bl), path); +} + seastar::future<> SeaStore::write_meta(const std::string& key, const std::string& value) { @@ -1228,50 +1241,57 @@ seastar::future<> SeaStore::write_meta(const std::string& key, return seastar::do_with( key, value, [this, FNAME](auto& key, auto& value) { - return repeat_eagain([this, FNAME, &key, &value] { - return transaction_manager->with_transaction_intr( - Transaction::src_t::MUTATE, - [this, FNAME, &key, &value](auto& t) { - DEBUGT("Have transaction, key: {}; value: {}", t, key, value); - return transaction_manager->update_root_meta( - t, key, value - ).si_then([this, &t] { - return transaction_manager->submit_transaction(t); - }); - }); - }); - }).handle_error( - crimson::ct_error::assert_all{"Invalid error in SeaStore::write_meta"} - ); + return repeat_eagain([this, FNAME, &key, &value] { + return transaction_manager->with_transaction_intr( + Transaction::src_t::MUTATE, + [this, FNAME, &key, &value](auto& t) { + DEBUGT("Have transaction, key: {}; value: {}", t, key, value); + return transaction_manager->update_root_meta( + t, key, value + ).si_then([this, &t] { + return transaction_manager->submit_transaction(t); + }); + }); + }).safe_then([this, &key, &value] { + return write_meta_file(key, value); + }); + }).handle_error( + crimson::ct_error::assert_all{"Invalid error in SeaStore::write_meta"} + ); +} + +seastar::future> +SeaStore::read_meta_file(const std::string& key) +{ + std::string path = fmt::format("{}/{}", root, key); + return seastar::file_exists( + path + ).then([path] (bool exist) { + if (exist) { + return crimson::read_file(path) + .then([] (auto tmp_buf) { + std::string v = {tmp_buf.get(), tmp_buf.size()}; + std::size_t pos = v.find("\n"); + std::string str = v.substr(0, pos); + return seastar::make_ready_future>(str); + }); + } else { + return seastar::make_ready_future>(std::nullopt); + } + }); } seastar::future> SeaStore::read_meta(const std::string& key) { LOG_PREFIX(SeaStore::read_meta); DEBUG("key: {}", key); - return seastar::do_with( - std::tuple(), key, - [this](auto &ret, auto& key) { - return repeat_eagain([this, &ret, &key] { - return transaction_manager->with_transaction_intr( - Transaction::src_t::READ, - [this, &ret, &key](auto& t) { - return transaction_manager->read_root_meta( - t, key - ).si_then([&ret](auto v) { - if (v) { - ret = std::make_tuple(0, std::move(*v)); - } else { - ret = std::make_tuple(-1, std::string("")); - } - }); - }); - }).safe_then([&ret] { - return std::move(ret); - }); - }).handle_error( - crimson::ct_error::assert_all{"Invalid error in SeaStore::read_meta"} - ); + return read_meta_file(key).then([](auto v) { + if (v) { + return std::make_tuple(0, std::move(*v)); + } else { + return std::make_tuple(-1, std::string("")); + } + }); } uuid_d SeaStore::get_fsid() const From 784aa2f4f24c869e3df91d0bab1c0784a52afcac Mon Sep 17 00:00:00 2001 From: chunmei-liu Date: Sun, 24 Oct 2021 15:06:28 -0700 Subject: [PATCH 0447/2866] crimson: write some other osd metas Signed-off-by: chunmei-liu --- src/crimson/os/seastore/seastore.cc | 38 +++++++++++++++++++++++++++++ src/crimson/os/seastore/seastore.h | 1 + src/crimson/osd/osd.cc | 13 +++++++--- src/crimson/osd/osd.h | 2 +- 4 files changed, 49 insertions(+), 5 deletions(-) diff --git a/src/crimson/os/seastore/seastore.cc b/src/crimson/os/seastore/seastore.cc index 452bba333d780..8dbe258692655 100644 --- a/src/crimson/os/seastore/seastore.cc +++ b/src/crimson/os/seastore/seastore.cc @@ -9,9 +9,12 @@ #include #include +#include +#include #include #include "common/safe_io.h" +#include "include/stringify.h" #include "os/Transaction.h" #include "crimson/common/buffer_io.h" @@ -136,6 +139,24 @@ seastar::future<> SeaStore::umount() ); } +seastar::future<> SeaStore::write_fsid(uuid_d new_osd_fsid) +{ + LOG_PREFIX(SeaStore::write_fsid); + return read_meta("fsid").then([this, FNAME, new_osd_fsid] (auto tuple) { + auto [ret, fsid] = tuple; + std::string str_fsid = stringify(new_osd_fsid); + if (ret == -1) { + return write_meta("fsid", stringify(new_osd_fsid)); + } else if (ret == 0 && fsid != str_fsid) { + ERROR("on-disk fsid {} != provided {}", + fsid, stringify(new_osd_fsid)); + throw std::runtime_error("store fsid error"); + } else { + return seastar::now(); + } + }); +} + SeaStore::mkfs_ertr::future<> SeaStore::mkfs(uuid_d new_osd_fsid) { return seastar::do_with( @@ -238,6 +259,23 @@ SeaStore::mkfs_ertr::future<> SeaStore::mkfs(uuid_d new_osd_fsid) }); }); }); + }).safe_then([this, new_osd_fsid] { + return write_fsid(new_osd_fsid); + }).safe_then([this] { + return read_meta("type").then([this] (auto tuple) { + auto [ret, type] = tuple; + if (ret == 0 && type == "seastore") { + return seastar::now(); + } else if (ret == 0 && type != "seastore") { + LOG_PREFIX(SeaStore::mkfs); + ERROR("expected seastore, but type is {}", type); + throw std::runtime_error("store type error"); + } else { + return write_meta("type", "seastore"); + } + }); + }).safe_then([this] { + return write_meta("mkfs_done", "yes"); }).safe_then([this] { return umount(); }).handle_error( diff --git a/src/crimson/os/seastore/seastore.h b/src/crimson/os/seastore/seastore.h index f5dce81235147..83b4625e13461 100644 --- a/src/crimson/os/seastore/seastore.h +++ b/src/crimson/os/seastore/seastore.h @@ -350,6 +350,7 @@ class SeaStore final : public FuturizedStore { } seastar::metrics::metric_group metrics; void register_metrics(); + seastar::future<> write_fsid(uuid_d new_osd_fsid); }; std::unique_ptr make_seastore( diff --git a/src/crimson/osd/osd.cc b/src/crimson/osd/osd.cc index 969b7af809e6c..ac540d2854026 100644 --- a/src/crimson/osd/osd.cc +++ b/src/crimson/osd/osd.cc @@ -176,6 +176,8 @@ seastar::future<> OSD::mkfs(uuid_d osd_uuid, uuid_d cluster_fsid) return _write_superblock(); }).then([cluster_fsid, this] { return store.write_meta("ceph_fsid", cluster_fsid.to_string()); + }).then([this] { + return store.write_meta("magic", CEPH_OSD_ONDISK_MAGIC); }).then([this] { return store.write_meta("whoami", std::to_string(whoami)); }).then([this] { @@ -1139,9 +1141,10 @@ seastar::future<> OSD::committed_osd_maps(version_t first, return seastar::now(); } } - check_osdmap_features(); - // yay! - return consume_map(osdmap->get_epoch()); + return check_osdmap_features().then([this] { + // yay! + return consume_map(osdmap->get_epoch()); + }); }).then([m, this] { if (state.is_active()) { logger().info("osd.{}: now active", whoami); @@ -1367,9 +1370,11 @@ seastar::future<> OSD::handle_peering_op( return seastar::now(); } -void OSD::check_osdmap_features() +seastar::future<> OSD::check_osdmap_features() { heartbeat->set_require_authorizer(true); + return store.write_meta("require_osd_release", + stringify((int)osdmap->require_osd_release)); } seastar::future<> OSD::consume_map(epoch_t epoch) diff --git a/src/crimson/osd/osd.h b/src/crimson/osd/osd.h index f51c89c2be615..318366aba59a7 100644 --- a/src/crimson/osd/osd.h +++ b/src/crimson/osd/osd.h @@ -202,7 +202,7 @@ class OSD final : public crimson::net::Dispatcher, version_t last, Ref m); - void check_osdmap_features(); + seastar::future<> check_osdmap_features(); seastar::future<> handle_command(crimson::net::ConnectionRef conn, Ref m); From c423abaf0f07ace824e1d7171a3765310226a5ff Mon Sep 17 00:00:00 2001 From: chunmei-liu Date: Tue, 26 Oct 2021 20:23:27 -0700 Subject: [PATCH 0448/2866] crimson: check mkfs_done before seastore do mkfs Signed-off-by: chunmei-liu --- src/crimson/os/seastore/seastore.cc | 251 ++++++++++++++-------------- 1 file changed, 129 insertions(+), 122 deletions(-) diff --git a/src/crimson/os/seastore/seastore.cc b/src/crimson/os/seastore/seastore.cc index 8dbe258692655..304b38cbbc7e1 100644 --- a/src/crimson/os/seastore/seastore.cc +++ b/src/crimson/os/seastore/seastore.cc @@ -159,130 +159,137 @@ seastar::future<> SeaStore::write_fsid(uuid_d new_osd_fsid) SeaStore::mkfs_ertr::future<> SeaStore::mkfs(uuid_d new_osd_fsid) { - return seastar::do_with( - secondary_device_set_t(), - [this, new_osd_fsid](auto& sds) { - auto fut = seastar::now(); - LOG_PREFIX(SeaStore::mkfs); - DEBUG("root: {}", root); - if (!root.empty()) { - fut = seastar::open_directory(root).then( - [this, &sds, new_osd_fsid](seastar::file rdir) mutable { - std::unique_ptr root_f = - std::make_unique(std::move(rdir)); - auto sub = root_f->list_directory( - [this, &sds, new_osd_fsid](auto de) mutable - -> seastar::future<> { - LOG_PREFIX(SeaStore::mkfs); - DEBUG("found file: {}", de.name); - if (de.name.find("block.") == 0 - && de.name.length() > 6 /* 6 for "block." */) { - std::string entry_name = de.name; - auto dtype_end = entry_name.find_first_of('.', 6); - device_type_t dtype = - string_to_device_type( - entry_name.substr(6, dtype_end - 6)); - if (!dtype) { - // invalid device type - return seastar::now(); - } - auto id = std::stoi(entry_name.substr(dtype_end + 1)); - auto sm = std::make_unique< - segment_manager::block::BlockSegmentManager - >(root + "/" + entry_name); - magic_t magic = (magic_t)std::rand(); - sds.emplace( - (device_id_t)id, - device_spec_t{ - magic, - dtype, - (device_id_t)id}); - return sm->mkfs( - segment_manager_config_t{ - false, - magic, - dtype, - (device_id_t)id, - seastore_meta_t{new_osd_fsid}, - secondary_device_set_t()} - ).safe_then([this, sm=std::move(sm), id]() mutable { - LOG_PREFIX(SeaStore::mkfs); - DEBUG("mkfs: finished for segment manager {}", id); - secondaries.emplace_back(std::move(sm)); - return seastar::now(); - }).handle_error(crimson::ct_error::assert_all{"not possible"}); - } - return seastar::now(); - }); - return sub.done().then( - [root_f=std::move(root_f)] { - return seastar::now(); - }); - }); - } - return fut.then([this, &sds, new_osd_fsid] { - return segment_manager->mkfs( - segment_manager_config_t{ - true, - (magic_t)std::rand(), - device_type_t::SEGMENTED, - 0, - seastore_meta_t{new_osd_fsid}, - sds} - ); - }).safe_then([this] { - return crimson::do_for_each(secondaries, [this](auto& sec_sm) { - return sec_sm->mount().safe_then([this, &sec_sm] { - transaction_manager->add_segment_manager(sec_sm.get()); - return seastar::now(); - }); - }); - }); - }).safe_then([this] { - return segment_manager->mount(); - }).safe_then([this] { - transaction_manager->add_segment_manager(segment_manager.get()); - return transaction_manager->mkfs(); - }).safe_then([this] { - return transaction_manager->mount(); - }).safe_then([this] { - return repeat_eagain([this] { - return transaction_manager->with_transaction_intr( - Transaction::src_t::MUTATE, [this](auto& t) { - return onode_manager->mkfs(t - ).si_then([this, &t] { - return collection_manager->mkfs(t); - }).si_then([this, &t](auto coll_root) { - transaction_manager->write_collection_root( - t, coll_root); - return transaction_manager->submit_transaction(t); - }); - }); - }); - }).safe_then([this, new_osd_fsid] { - return write_fsid(new_osd_fsid); - }).safe_then([this] { - return read_meta("type").then([this] (auto tuple) { - auto [ret, type] = tuple; - if (ret == 0 && type == "seastore") { - return seastar::now(); - } else if (ret == 0 && type != "seastore") { + return read_meta("mkfs_done").then([this, new_osd_fsid] (auto tuple) { + auto [done, value] = tuple; + if (done == 0) { + return seastar::now(); + } else { + return seastar::do_with( + secondary_device_set_t(), + [this, new_osd_fsid](auto& sds) { + auto fut = seastar::now(); LOG_PREFIX(SeaStore::mkfs); - ERROR("expected seastore, but type is {}", type); - throw std::runtime_error("store type error"); - } else { - return write_meta("type", "seastore"); - } - }); - }).safe_then([this] { - return write_meta("mkfs_done", "yes"); - }).safe_then([this] { - return umount(); - }).handle_error( - crimson::ct_error::assert_all{ - "Invalid error in SeaStore::mkfs" + DEBUG("root: {}", root); + if (!root.empty()) { + fut = seastar::open_directory(root).then( + [this, &sds, new_osd_fsid](seastar::file rdir) mutable { + std::unique_ptr root_f = + std::make_unique(std::move(rdir)); + auto sub = root_f->list_directory( + [this, &sds, new_osd_fsid](auto de) mutable + -> seastar::future<> { + LOG_PREFIX(SeaStore::mkfs); + DEBUG("found file: {}", de.name); + if (de.name.find("block.") == 0 + && de.name.length() > 6 /* 6 for "block." */) { + std::string entry_name = de.name; + auto dtype_end = entry_name.find_first_of('.', 6); + device_type_t dtype = + string_to_device_type( + entry_name.substr(6, dtype_end - 6)); + if (!dtype) { + // invalid device type + return seastar::now(); + } + auto id = std::stoi(entry_name.substr(dtype_end + 1)); + auto sm = std::make_unique< + segment_manager::block::BlockSegmentManager + >(root + "/" + entry_name); + magic_t magic = (magic_t)std::rand(); + sds.emplace( + (device_id_t)id, + device_spec_t{ + magic, + dtype, + (device_id_t)id}); + return sm->mkfs( + segment_manager_config_t{ + false, + magic, + dtype, + (device_id_t)id, + seastore_meta_t{new_osd_fsid}, + secondary_device_set_t()} + ).safe_then([this, sm=std::move(sm), id]() mutable { + LOG_PREFIX(SeaStore::mkfs); + DEBUG("mkfs: finished for segment manager {}", id); + secondaries.emplace_back(std::move(sm)); + return seastar::now(); + }).handle_error(crimson::ct_error::assert_all{"not possible"}); + } + return seastar::now(); + }); + return sub.done().then( + [root_f=std::move(root_f)] { + return seastar::now(); + }); + }); + } + return fut.then([this, &sds, new_osd_fsid] { + return segment_manager->mkfs( + segment_manager_config_t{ + true, + (magic_t)std::rand(), + device_type_t::SEGMENTED, + 0, + seastore_meta_t{new_osd_fsid}, + sds} + ); + }).safe_then([this] { + return crimson::do_for_each(secondaries, [this](auto& sec_sm) { + return sec_sm->mount().safe_then([this, &sec_sm] { + transaction_manager->add_segment_manager(sec_sm.get()); + return seastar::now(); + }); + }); + }); + }).safe_then([this] { + return segment_manager->mount(); + }).safe_then([this] { + transaction_manager->add_segment_manager(segment_manager.get()); + return transaction_manager->mkfs(); + }).safe_then([this] { + return transaction_manager->mount(); + }).safe_then([this] { + return repeat_eagain([this] { + return transaction_manager->with_transaction_intr( + Transaction::src_t::MUTATE, [this](auto& t) { + return onode_manager->mkfs(t + ).si_then([this, &t] { + return collection_manager->mkfs(t); + }).si_then([this, &t](auto coll_root) { + transaction_manager->write_collection_root( + t, coll_root); + return transaction_manager->submit_transaction(t); + }); + }); + }); + }).safe_then([this, new_osd_fsid] { + return write_fsid(new_osd_fsid); + }).safe_then([this] { + return read_meta("type").then([this] (auto tuple) { + auto [ret, type] = tuple; + if (ret == 0 && type == "seastore") { + return seastar::now(); + } else if (ret == 0 && type != "seastore") { + LOG_PREFIX(SeaStore::mkfs); + ERROR("expected seastore, but type is {}", type); + throw std::runtime_error("store type error"); + } else { + return write_meta("type", "seastore"); + } + }); + }).safe_then([this] { + return write_meta("mkfs_done", "yes"); + }).safe_then([this] { + return umount(); + }).handle_error( + crimson::ct_error::assert_all{ + "Invalid error in SeaStore::mkfs" + } + ); } - ); + }); } seastar::future SeaStore::stat() const From da2b40586a08eaa39256bdf8f45759ea6779bb8c Mon Sep 17 00:00:00 2001 From: Samuel Just Date: Thu, 28 Oct 2021 01:37:13 -0700 Subject: [PATCH 0449/2866] crimson/os/seastore/seastore: factor out MDStore interface Signed-off-by: Samuel Just --- src/crimson/os/seastore/seastore.cc | 94 +++++++++++++++++------------ src/crimson/os/seastore/seastore.h | 28 +++++++++ 2 files changed, 85 insertions(+), 37 deletions(-) diff --git a/src/crimson/os/seastore/seastore.cc b/src/crimson/os/seastore/seastore.cc index 304b38cbbc7e1..3ba1795e502fa 100644 --- a/src/crimson/os/seastore/seastore.cc +++ b/src/crimson/os/seastore/seastore.cc @@ -35,13 +35,52 @@ using crimson::common::local_conf; namespace crimson::os::seastore { +class FileMDStore final : public SeaStore::MDStore { + std::string root; +public: + FileMDStore(std::string root) : root(root) {} + + write_meta_ret write_meta( + const std::string& key, const std::string& value) final { + std::string path = fmt::format("{}/{}", root, key); + std::string fvalue = value; + fvalue += "\n"; + ceph::bufferptr bp(fvalue.length()); + ceph::bufferlist bl; + bp.copy_in(0, fvalue.length(), fvalue.c_str()); + bl.push_back(bp); + return crimson::write_file(std::move(bl), path); + } + + read_meta_ret read_meta(const std::string& key) final { + std::string path = fmt::format("{}/{}", root, key); + return seastar::file_exists( + path + ).then([path] (bool exist) { + if (exist) { + return crimson::read_file(path) + .then([] (auto tmp_buf) { + std::string v = {tmp_buf.get(), tmp_buf.size()}; + std::size_t pos = v.find("\n"); + std::string str = v.substr(0, pos); + return seastar::make_ready_future>(str); + }); + } else { + return seastar::make_ready_future>(std::nullopt); + } + }); + } +}; + SeaStore::SeaStore( std::string root, + MDStoreRef mdstore, SegmentManagerRef sm, TransactionManagerRef tm, CollectionManagerRef cm, OnodeManagerRef om) : root(root), + mdstore(std::move(mdstore)), segment_manager(std::move(sm)), transaction_manager(std::move(tm)), collection_manager(std::move(cm)), @@ -50,6 +89,17 @@ SeaStore::SeaStore( register_metrics(); } +SeaStore::SeaStore( + std::string root, + SegmentManagerRef sm, + TransactionManagerRef tm, + CollectionManagerRef cm, + OnodeManagerRef om) + : SeaStore( + root, + std::make_unique(root), + std::move(sm), std::move(tm), std::move(cm), std::move(om)) {} + SeaStore::~SeaStore() = default; void SeaStore::register_metrics() @@ -1265,19 +1315,6 @@ boost::intrusive_ptr SeaStore::_get_collection(const coll_t& return new SeastoreCollection{cid}; } -seastar::future<> SeaStore::write_meta_file(const std::string& key, - const std::string& value) -{ - std::string path = fmt::format("{}/{}", root, key); - std::string fvalue = value; - fvalue += "\n"; - ceph::bufferptr bp(fvalue.length()); - ceph::bufferlist bl; - bp.copy_in(0, fvalue.length(), fvalue.c_str()); - bl.push_back(bp); - return crimson::write_file(std::move(bl), path); -} - seastar::future<> SeaStore::write_meta(const std::string& key, const std::string& value) { @@ -1298,45 +1335,28 @@ seastar::future<> SeaStore::write_meta(const std::string& key, }); }); }).safe_then([this, &key, &value] { - return write_meta_file(key, value); + return mdstore->write_meta(key, value); }); }).handle_error( crimson::ct_error::assert_all{"Invalid error in SeaStore::write_meta"} ); } -seastar::future> -SeaStore::read_meta_file(const std::string& key) -{ - std::string path = fmt::format("{}/{}", root, key); - return seastar::file_exists( - path - ).then([path] (bool exist) { - if (exist) { - return crimson::read_file(path) - .then([] (auto tmp_buf) { - std::string v = {tmp_buf.get(), tmp_buf.size()}; - std::size_t pos = v.find("\n"); - std::string str = v.substr(0, pos); - return seastar::make_ready_future>(str); - }); - } else { - return seastar::make_ready_future>(std::nullopt); - } - }); -} - seastar::future> SeaStore::read_meta(const std::string& key) { LOG_PREFIX(SeaStore::read_meta); DEBUG("key: {}", key); - return read_meta_file(key).then([](auto v) { + return mdstore->read_meta(key).safe_then([](auto v) { if (v) { return std::make_tuple(0, std::move(*v)); } else { return std::make_tuple(-1, std::string("")); } - }); + }).handle_error( + crimson::ct_error::assert_all{ + "Invalid error in SeaStore::read_meta" + } + ); } uuid_d SeaStore::get_fsid() const diff --git a/src/crimson/os/seastore/seastore.h b/src/crimson/os/seastore/seastore.h index 83b4625e13461..25fe0e0158c7d 100644 --- a/src/crimson/os/seastore/seastore.h +++ b/src/crimson/os/seastore/seastore.h @@ -40,7 +40,34 @@ class SeastoreCollection final : public FuturizedCollection { class SeaStore final : public FuturizedStore { public: + class MDStore { + public: + using base_iertr = crimson::errorator< + crimson::ct_error::input_output_error + >; + + using write_meta_ertr = base_iertr; + using write_meta_ret = write_meta_ertr::future<>; + virtual write_meta_ret write_meta( + const std::string &key, + const std::string &val + ) = 0; + + using read_meta_ertr = base_iertr; + using read_meta_ret = write_meta_ertr::future>; + virtual read_meta_ret read_meta(const std::string &key) = 0; + virtual ~MDStore() {} + }; + using MDStoreRef = std::unique_ptr; + + SeaStore( + std::string root, + MDStoreRef mdstore, + SegmentManagerRef sm, + TransactionManagerRef tm, + CollectionManagerRef cm, + OnodeManagerRef om); SeaStore( std::string root, SegmentManagerRef sm, @@ -265,6 +292,7 @@ class SeaStore final : public FuturizedStore { OMapManager::omap_list_config_t config); std::string root; + MDStoreRef mdstore; SegmentManagerRef segment_manager; std::vector secondaries; TransactionManagerRef transaction_manager; From 9d19de54397b8b01f14a4ea3d6000f8183a3ca2d Mon Sep 17 00:00:00 2001 From: Samuel Just Date: Thu, 28 Oct 2021 02:50:16 -0700 Subject: [PATCH 0450/2866] test/crimson/seastore/test_transaction_manager_test_state: add mock MDStore Signed-off-by: Samuel Just --- .../seastore/transaction_manager_test_state.h | 36 ++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/src/test/crimson/seastore/transaction_manager_test_state.h b/src/test/crimson/seastore/transaction_manager_test_state.h index 8beb6b7626d8a..003966e13d427 100644 --- a/src/test/crimson/seastore/transaction_manager_test_state.h +++ b/src/test/crimson/seastore/transaction_manager_test_state.h @@ -106,11 +106,12 @@ auto get_transaction_manager( scanner_ref); } -auto get_seastore(SegmentManagerRef sm) { +auto get_seastore(SeaStore::MDStoreRef mdstore, SegmentManagerRef sm) { auto tm = get_transaction_manager(*sm); auto cm = std::make_unique(*tm); return std::make_unique( "", + std::move(mdstore), std::move(sm), std::move(tm), std::move(cm), @@ -258,6 +259,38 @@ class TestSegmentManagerWrapper final : public SegmentManager { }; class SeaStoreTestState : public EphemeralTestState { + class TestMDStoreState { + std::map md; + public: + class Store final : public SeaStore::MDStore { + TestMDStoreState &parent; + public: + Store(TestMDStoreState &parent) : parent(parent) {} + + write_meta_ret write_meta( + const std::string& key, const std::string& value) final { + parent.md[key] = value; + return seastar::now(); + } + + read_meta_ret read_meta(const std::string& key) final { + auto iter = parent.md.find(key); + if (iter != parent.md.end()) { + return read_meta_ret( + read_meta_ertr::ready_future_marker{}, + iter->second); + } else { + return read_meta_ret( + read_meta_ertr::ready_future_marker{}, + std::nullopt); + } + } + }; + Store get_mdstore() { + return Store(*this); + } + } mdstore_state; + protected: std::unique_ptr seastore; @@ -265,6 +298,7 @@ class SeaStoreTestState : public EphemeralTestState { virtual void _init() final { seastore = get_seastore( + std::make_unique(mdstore_state.get_mdstore()), std::make_unique(*segment_manager)); } From 89d9b739e80b7eb90203547104a8fe1fa0e20c94 Mon Sep 17 00:00:00 2001 From: Kefu Chai Date: Sat, 30 Oct 2021 15:40:40 +0800 Subject: [PATCH 0451/2866] crimson/os/seastore: minor cleanups * pass string by reference. more consistent this way. * set a bl using list::append(), simpler this way. Signed-off-by: Kefu Chai --- src/crimson/os/seastore/seastore.cc | 12 ++++-------- src/crimson/os/seastore/seastore.h | 4 ++-- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/src/crimson/os/seastore/seastore.cc b/src/crimson/os/seastore/seastore.cc index 3ba1795e502fa..8bfb705d1d54f 100644 --- a/src/crimson/os/seastore/seastore.cc +++ b/src/crimson/os/seastore/seastore.cc @@ -38,17 +38,13 @@ namespace crimson::os::seastore { class FileMDStore final : public SeaStore::MDStore { std::string root; public: - FileMDStore(std::string root) : root(root) {} + FileMDStore(const std::string& root) : root(root) {} write_meta_ret write_meta( const std::string& key, const std::string& value) final { std::string path = fmt::format("{}/{}", root, key); - std::string fvalue = value; - fvalue += "\n"; - ceph::bufferptr bp(fvalue.length()); ceph::bufferlist bl; - bp.copy_in(0, fvalue.length(), fvalue.c_str()); - bl.push_back(bp); + bl.append(value + "\n"); return crimson::write_file(std::move(bl), path); } @@ -73,7 +69,7 @@ class FileMDStore final : public SeaStore::MDStore { }; SeaStore::SeaStore( - std::string root, + const std::string& root, MDStoreRef mdstore, SegmentManagerRef sm, TransactionManagerRef tm, @@ -90,7 +86,7 @@ SeaStore::SeaStore( } SeaStore::SeaStore( - std::string root, + const std::string& root, SegmentManagerRef sm, TransactionManagerRef tm, CollectionManagerRef cm, diff --git a/src/crimson/os/seastore/seastore.h b/src/crimson/os/seastore/seastore.h index 25fe0e0158c7d..8e508c6a3f274 100644 --- a/src/crimson/os/seastore/seastore.h +++ b/src/crimson/os/seastore/seastore.h @@ -62,14 +62,14 @@ class SeaStore final : public FuturizedStore { using MDStoreRef = std::unique_ptr; SeaStore( - std::string root, + const std::string& root, MDStoreRef mdstore, SegmentManagerRef sm, TransactionManagerRef tm, CollectionManagerRef cm, OnodeManagerRef om); SeaStore( - std::string root, + const std::string& root, SegmentManagerRef sm, TransactionManagerRef tm, CollectionManagerRef cm, From 73070f69603e7f6442bbf3afe19a84e8b8e98b57 Mon Sep 17 00:00:00 2001 From: Zac Dover Date: Sun, 31 Oct 2021 01:56:44 +1000 Subject: [PATCH 0452/2866] doc/cephadm: add admonition for tracker 53062 NOTE: This commit also adds an admonition to the pacific.rst page, in the /releases directory. This commit warns users not to upgrade to Ceph v16 (Pacific). Upgrading to v16 (Pacific) can cause data corruption. This commit has been made in response to https://tracker.ceph.com/issues/53062 and in response to Stefan Kooman's urgent request that we update the documentation, adding a warning to the documentation to prevent users from upgrading to v16.2.6. Signed-off-by: Zac Dover --- doc/cephadm/upgrade.rst | 23 ++++++++++++++++++ doc/releases/pacific.rst | 52 ++++++++++++++++++++++++++++++++++++++-- 2 files changed, 73 insertions(+), 2 deletions(-) diff --git a/doc/cephadm/upgrade.rst b/doc/cephadm/upgrade.rst index 429f4f2803e04..d2b609ea533f0 100644 --- a/doc/cephadm/upgrade.rst +++ b/doc/cephadm/upgrade.rst @@ -2,6 +2,29 @@ Upgrading Ceph ============== +.. DANGER:: DATE: 01 NOV 2021. + + DO NOT UPGRADE TO CEPH PACIFIC FROM AN OLDER VERSION. + + A recently-discovered bug (https://tracker.ceph.com/issues/53062) can cause + data corruption. This bug occurs during OMAP format conversion for + clusters that are updated to Pacific. New clusters are not affected by this + bug. + + The trigger for this bug is BlueStore's repair/quick-fix functionality. This + bug can be triggered in two known ways: + + (1) manually via the ceph-bluestore-tool, or + (2) automatically, by OSD if ``bluestore_fsck_quick_fix_on_mount`` is set + to true. + + The fix for this bug is expected to be available in Ceph v16.2.7. + + DO NOT set ``bluestore_quick_fix_on_mount`` to true. If it is currently + set to true in your configuration, immediately set it to false. + + DO NOT run ``ceph-bluestore-tool``'s repair/quick-fix commands. + Cephadm can safely upgrade Ceph from one bugfix release to the next. For example, you can upgrade from v15.2.0 (the first Octopus release) to the next point release, v15.2.1. diff --git a/doc/releases/pacific.rst b/doc/releases/pacific.rst index a18695bcf03fa..0b2caf7ff3816 100644 --- a/doc/releases/pacific.rst +++ b/doc/releases/pacific.rst @@ -5,11 +5,59 @@ Pacific Pacific is the 16th stable release of Ceph. It is named after the giant pacific octopus (Enteroctopus dofleini). +.. DANGER:: DATE: 01 NOV 2021. + + DO NOT UPGRADE TO CEPH PACIFIC FROM AN OLDER VERSION. + + A recently-discovered bug (https://tracker.ceph.com/issues/53062) can cause + data corruption. This bug occurs during OMAP format conversion for + clusters that are updated to Pacific. New clusters are not affected by this + bug. + + The trigger for this bug is BlueStore's repair/quick-fix functionality. This + bug can be triggered in two known ways: + + (1) manually via the ceph-bluestore-tool, or + (2) automatically, by OSD if ``bluestore_fsck_quick_fix_on_mount`` is set + to true. + + The fix for this bug is expected to be available in Ceph v16.2.7. + + DO NOT set ``bluestore_quick_fix_on_mount`` to true. If it is currently + set to true in your configuration, immediately set it to false. + + DO NOT run ``ceph-bluestore-tool``'s repair/quick-fix commands. + + v16.2.6 Pacific =============== -This is the sixth backport release in the Pacific series. We recommend all -users update to this release. +.. DANGER:: DATE: 01 NOV 2021. + + DO NOT UPGRADE TO CEPH PACIFIC FROM AN OLDER VERSION. + + A recently-discovered bug (https://tracker.ceph.com/issues/53062) can cause + data corruption. This bug occurs during OMAP format conversion for + clusters that are updated to Pacific. New clusters are not affected by this + bug. + + The trigger for this bug is BlueStore's repair/quick-fix functionality. This + bug can be triggered in two known ways: + + (1) manually via the ceph-bluestore-tool, or + (2) automatically, by OSD if ``bluestore_fsck_quick_fix_on_mount`` is set + to true. + + The fix for this bug is expected to be available in Ceph v16.2.7. + + DO NOT set ``bluestore_quick_fix_on_mount`` to true. If it is currently + set to true in your configuration, immediately set it to false. + + DO NOT run ``ceph-bluestore-tool``'s repair/quick-fix commands. + + +This is the sixth backport release in the Pacific series. + Notable Changes --------------- From 662d29918d49a9b58268ad360d87aa860b2f96e5 Mon Sep 17 00:00:00 2001 From: Casey Bodley Date: Sat, 30 Oct 2021 20:26:21 -0400 Subject: [PATCH 0453/2866] rgw: fix debug build of OpsLogFile can't use ceph::mutex locks with std::condition_variable::wait() in debug builds. use the ceph::condition_variable wrapper instead Signed-off-by: Casey Bodley --- src/rgw/rgw_log.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rgw/rgw_log.h b/src/rgw/rgw_log.h index 5ee411e8cfaec..a087aa1309806 100644 --- a/src/rgw/rgw_log.h +++ b/src/rgw/rgw_log.h @@ -169,7 +169,7 @@ class OpsLogFile : public JsonOpsLogSink, public Thread, public DoutPrefixProvid ceph::mutex flush_mutex = ceph::make_mutex("OpsLogFile_flush"); std::vector log_buffer; std::vector flush_buffer; - std::condition_variable cond_flush; + ceph::condition_variable cond_flush; std::ofstream file; bool stopped; uint64_t data_size; From a2ae83f8aab18933eae77cf3034b740082a39e4f Mon Sep 17 00:00:00 2001 From: Jianpeng Ma Date: Mon, 1 Nov 2021 08:33:23 +0800 Subject: [PATCH 0454/2866] librbd: send FLUSH_SOURCE_INTERNAL when do copy/deep_copy. copy/deep_copy use object_map to judge whether object exist. If w/ librbdo pwl cache, flush can't flush data to osd which change objectmap state. So we should send flush w/ FLUSH_SOURCE_INTERNAL to make data flush to osd. Fixes:https://tracker.ceph.com/issues/53057 Signed-off-by: Jianpeng Ma --- src/librbd/api/Image.cc | 20 +++++++++++++++++++- src/librbd/internal.cc | 17 ++++++++++++++++- 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/src/librbd/api/Image.cc b/src/librbd/api/Image.cc index 25ae76b1319e7..772dd7f058eaf 100644 --- a/src/librbd/api/Image.cc +++ b/src/librbd/api/Image.cc @@ -26,6 +26,8 @@ #include "librbd/image/PreRemoveRequest.h" #include "librbd/io/ImageDispatcherInterface.h" #include "librbd/io/ObjectDispatcherInterface.h" +#include "librbd/io/AioCompletion.h" +#include "librbd/io/ImageDispatchSpec.h" #include #define dout_subsys ceph_subsys_rbd @@ -680,6 +682,22 @@ int Image::deep_copy(I *src, librados::IoCtx& dest_md_ctx, template int Image::deep_copy(I *src, I *dest, bool flatten, ProgressContext &prog_ctx) { + // ensure previous writes are visible to dest + C_SaferCond flush_ctx; + { + std::shared_lock owner_locker{src->owner_lock}; + auto aio_comp = io::AioCompletion::create_and_start(&flush_ctx, src, + io::AIO_TYPE_FLUSH); + auto req = io::ImageDispatchSpec::create_flush( + *src, io::IMAGE_DISPATCH_LAYER_INTERNAL_START, + aio_comp, io::FLUSH_SOURCE_INTERNAL, {}); + req->send(); + } + int r = flush_ctx.wait(); + if (r < 0) { + return r; + } + librados::snap_t snap_id_start = 0; librados::snap_t snap_id_end; { @@ -696,7 +714,7 @@ int Image::deep_copy(I *src, I *dest, bool flatten, src, dest, snap_id_start, snap_id_end, 0U, flatten, boost::none, asio_engine.get_work_queue(), &snap_seqs, &progress_handler, &cond); req->send(); - int r = cond.wait(); + r = cond.wait(); if (r < 0) { return r; } diff --git a/src/librbd/internal.cc b/src/librbd/internal.cc index dba15bdf23a18..42542b22d28b4 100644 --- a/src/librbd/internal.cc +++ b/src/librbd/internal.cc @@ -1281,12 +1281,27 @@ int validate_pool(IoCtx &io_ctx, CephContext *cct) { return -EINVAL; } + // ensure previous writes are visible to dest + C_SaferCond flush_ctx; + { + auto aio_comp = io::AioCompletion::create_and_start(&flush_ctx, src, + io::AIO_TYPE_FLUSH); + auto req = io::ImageDispatchSpec::create_flush( + *src, io::IMAGE_DISPATCH_LAYER_INTERNAL_START, + aio_comp, io::FLUSH_SOURCE_INTERNAL, {}); + req->send(); + } + int r = flush_ctx.wait(); + if (r < 0) { + return r; + } + C_SaferCond ctx; auto req = deep_copy::MetadataCopyRequest<>::create( src, dest, &ctx); req->send(); - int r = ctx.wait(); + r = ctx.wait(); if (r < 0) { lderr(cct) << "failed to copy metadata: " << cpp_strerror(r) << dendl; return r; From 1fc3be248097eb6087560c22374c1f924bfe0735 Mon Sep 17 00:00:00 2001 From: Jianpeng Ma Date: Mon, 1 Nov 2021 09:25:52 +0800 Subject: [PATCH 0455/2866] librbd/cache/pwl: fix reorder when flush cache-data to osd. Consider the following workload: writeA(0, 4096) writeB(0, 512). pwl can makre sure writeA persist to cache before writeB. But when flush to osd, it use async-read to read data from cache and in the callback function they issue write to osd. So although we by order issue aio-read(4096), aio-read(512). But we can't make sure the return order. If aio-read(512) firstly return, the write order to next layer is writeB(0, 512) writeA(0, 4096). This is wrong from the user point. To avoid this occur, we should firstly read all data from cache. And then send write by order. Fiexs: https://tracker.ceph.com/issues/52511 Tested-by: Feng Hualong Signed-off-by: Jianpeng Ma --- src/librbd/cache/pwl/AbstractWriteLog.cc | 11 ++- src/librbd/cache/pwl/AbstractWriteLog.h | 7 +- src/librbd/cache/pwl/rwl/WriteLog.cc | 32 ++++--- src/librbd/cache/pwl/rwl/WriteLog.h | 5 +- src/librbd/cache/pwl/ssd/WriteLog.cc | 114 +++++++++++++++-------- src/librbd/cache/pwl/ssd/WriteLog.h | 5 +- 6 files changed, 112 insertions(+), 62 deletions(-) diff --git a/src/librbd/cache/pwl/AbstractWriteLog.cc b/src/librbd/cache/pwl/AbstractWriteLog.cc index c01d2d411f9da..421d5dd8bc49e 100644 --- a/src/librbd/cache/pwl/AbstractWriteLog.cc +++ b/src/librbd/cache/pwl/AbstractWriteLog.cc @@ -1687,13 +1687,16 @@ void AbstractWriteLog::process_writeback_dirty_entries() { CephContext *cct = m_image_ctx.cct; bool all_clean = false; int flushed = 0; + bool has_write_entry = false; ldout(cct, 20) << "Look for dirty entries" << dendl; { DeferredContexts post_unlock; + GenericLogEntries entries_to_flush; + std::shared_lock entry_reader_locker(m_entry_reader_lock); + std::lock_guard locker(m_lock); while (flushed < IN_FLIGHT_FLUSH_WRITE_LIMIT) { - std::lock_guard locker(m_lock); if (m_shutting_down) { ldout(cct, 5) << "Flush during shutdown supressed" << dendl; /* Do flush complete only when all flush ops are finished */ @@ -1709,14 +1712,18 @@ void AbstractWriteLog::process_writeback_dirty_entries() { auto candidate = m_dirty_log_entries.front(); bool flushable = can_flush_entry(candidate); if (flushable) { - post_unlock.add(construct_flush_entry_ctx(candidate)); + entries_to_flush.push_back(candidate); flushed++; + if (!has_write_entry) + has_write_entry = candidate->is_write_entry(); m_dirty_log_entries.pop_front(); } else { ldout(cct, 20) << "Next dirty entry isn't flushable yet" << dendl; break; } } + + construct_flush_entries(entries_to_flush, post_unlock, has_write_entry); } if (all_clean) { diff --git a/src/librbd/cache/pwl/AbstractWriteLog.h b/src/librbd/cache/pwl/AbstractWriteLog.h index c982d2631fe3f..128c00d99d82a 100644 --- a/src/librbd/cache/pwl/AbstractWriteLog.h +++ b/src/librbd/cache/pwl/AbstractWriteLog.h @@ -388,10 +388,9 @@ class AbstractWriteLog { virtual void persist_last_flushed_sync_gen() {} virtual void reserve_cache(C_BlockIORequestT *req, bool &alloc_succeeds, bool &no_space) {} - virtual Context *construct_flush_entry_ctx( - const std::shared_ptr log_entry) { - return nullptr; - } + virtual void construct_flush_entries(pwl::GenericLogEntries entries_to_flush, + DeferredContexts &post_unlock, + bool has_write_entry) = 0; virtual uint64_t get_max_extent() { return 0; } diff --git a/src/librbd/cache/pwl/rwl/WriteLog.cc b/src/librbd/cache/pwl/rwl/WriteLog.cc index bf4bd1557b031..f635015329a6c 100644 --- a/src/librbd/cache/pwl/rwl/WriteLog.cc +++ b/src/librbd/cache/pwl/rwl/WriteLog.cc @@ -576,23 +576,27 @@ bool WriteLog::retire_entries(const unsigned long int frees_per_tx) { } template -Context* WriteLog::construct_flush_entry_ctx( - std::shared_ptr log_entry) { +void WriteLog::construct_flush_entries(pwl::GenericLogEntries entries_to_flush, + DeferredContexts &post_unlock, + bool has_write_entry) { bool invalidating = this->m_invalidating; // snapshot so we behave consistently - Context *ctx = this->construct_flush_entry(log_entry, invalidating); - if (invalidating) { - return ctx; - } - return new LambdaContext( - [this, log_entry, ctx](int r) { - m_image_ctx.op_work_queue->queue(new LambdaContext( + for (auto &log_entry : entries_to_flush) { + Context *ctx = this->construct_flush_entry(log_entry, invalidating); + + if (!invalidating) { + ctx = new LambdaContext( [this, log_entry, ctx](int r) { - ldout(m_image_ctx.cct, 15) << "flushing:" << log_entry - << " " << *log_entry << dendl; - log_entry->writeback(this->m_image_writeback, ctx); - }), 0); - }); + m_image_ctx.op_work_queue->queue(new LambdaContext( + [this, log_entry, ctx](int r) { + ldout(m_image_ctx.cct, 15) << "flushing:" << log_entry + << " " << *log_entry << dendl; + log_entry->writeback(this->m_image_writeback, ctx); + }), 0); + }); + } + post_unlock.add(ctx); + } } const unsigned long int ops_flushed_together = 4; diff --git a/src/librbd/cache/pwl/rwl/WriteLog.h b/src/librbd/cache/pwl/rwl/WriteLog.h index dabee07d742aa..5083a2568d497 100644 --- a/src/librbd/cache/pwl/rwl/WriteLog.h +++ b/src/librbd/cache/pwl/rwl/WriteLog.h @@ -102,8 +102,9 @@ class WriteLog : public AbstractWriteLog { void setup_schedule_append( pwl::GenericLogOperationsVector &ops, bool do_early_flush, C_BlockIORequestT *req) override; - Context *construct_flush_entry_ctx( - const std::shared_ptr log_entry) override; + void construct_flush_entries(pwl::GenericLogEntries entries_to_flush, + DeferredContexts &post_unlock, + bool has_write_entry) override; bool initialize_pool(Context *on_finish, pwl::DeferredContexts &later) override; void write_data_to_buffer( std::shared_ptr ws_entry, diff --git a/src/librbd/cache/pwl/ssd/WriteLog.cc b/src/librbd/cache/pwl/ssd/WriteLog.cc index 550123924de0f..9d887227e96d6 100644 --- a/src/librbd/cache/pwl/ssd/WriteLog.cc +++ b/src/librbd/cache/pwl/ssd/WriteLog.cc @@ -521,49 +521,87 @@ void WriteLog::alloc_op_log_entries(GenericLogOperations &ops) { } template -Context* WriteLog::construct_flush_entry_ctx( - std::shared_ptr log_entry) { +void WriteLog::construct_flush_entries(pwl::GenericLogEntries entries_to_flush, + DeferredContexts &post_unlock, + bool has_write_entry) { // snapshot so we behave consistently bool invalidating = this->m_invalidating; - Context *ctx = this->construct_flush_entry(log_entry, invalidating); - - if (invalidating) { - return ctx; - } - if (log_entry->is_write_entry()) { - bufferlist *read_bl_ptr = new bufferlist; - ctx = new LambdaContext( - [this, log_entry, read_bl_ptr, ctx](int r) { - bufferlist captured_entry_bl; - captured_entry_bl.claim_append(*read_bl_ptr); - delete read_bl_ptr; - m_image_ctx.op_work_queue->queue(new LambdaContext( - [this, log_entry, entry_bl=move(captured_entry_bl), ctx](int r) { - auto captured_entry_bl = std::move(entry_bl); - ldout(m_image_ctx.cct, 15) << "flushing:" << log_entry - << " " << *log_entry << dendl; - log_entry->writeback_bl(this->m_image_writeback, ctx, - std::move(captured_entry_bl)); - }), 0); - }); - ctx = new LambdaContext( - [this, log_entry, read_bl_ptr, ctx](int r) { - auto write_entry = static_pointer_cast(log_entry); - write_entry->inc_bl_refs(); - aio_read_data_block(std::move(write_entry), read_bl_ptr, ctx); - }); - return ctx; + if (invalidating || !has_write_entry) { + for (auto &log_entry : entries_to_flush) { + Context *ctx = this->construct_flush_entry(log_entry, invalidating); + + if (!invalidating) { + ctx = new LambdaContext( + [this, log_entry, ctx](int r) { + m_image_ctx.op_work_queue->queue(new LambdaContext( + [this, log_entry, ctx](int r) { + ldout(m_image_ctx.cct, 15) << "flushing:" << log_entry + << " " << *log_entry << dendl; + log_entry->writeback(this->m_image_writeback, ctx); + }), 0); + }); + } + post_unlock.add(ctx); + } } else { - return new LambdaContext( - [this, log_entry, ctx](int r) { - m_image_ctx.op_work_queue->queue(new LambdaContext( - [this, log_entry, ctx](int r) { - ldout(m_image_ctx.cct, 15) << "flushing:" << log_entry - << " " << *log_entry << dendl; - log_entry->writeback(this->m_image_writeback, ctx); - }), 0); + int count = entries_to_flush.size(); + std::vector> log_entries; + std::vector read_bls; + std::vector contexts; + + log_entries.reserve(count); + read_bls.reserve(count); + contexts.reserve(count); + + for (auto &log_entry : entries_to_flush) { + // log_entry already removed from m_dirty_log_entries and + // in construct_flush_entry() it will inc(m_flush_ops_in_flight). + // We call this func here to make ops can track. + Context *ctx = this->construct_flush_entry(log_entry, invalidating); + if (log_entry->is_write_entry()) { + bufferlist *bl = new bufferlist; + auto write_entry = static_pointer_cast(log_entry); + write_entry->inc_bl_refs(); + log_entries.push_back(write_entry); + read_bls.push_back(bl); + } + contexts.push_back(ctx); + } + + Context *ctx = new LambdaContext( + [this, entries_to_flush, read_bls, contexts](int r) { + int i = 0, j = 0; + + for (auto &log_entry : entries_to_flush) { + Context *ctx = contexts[j++]; + + if (log_entry->is_write_entry()) { + bufferlist captured_entry_bl; + + captured_entry_bl.claim_append(*read_bls[i]); + delete read_bls[i++]; + + m_image_ctx.op_work_queue->queue(new LambdaContext( + [this, log_entry, entry_bl=std::move(captured_entry_bl), ctx](int r) { + auto captured_entry_bl = std::move(entry_bl); + ldout(m_image_ctx.cct, 15) << "flushing:" << log_entry + << " " << *log_entry << dendl; + log_entry->writeback_bl(this->m_image_writeback, ctx, + std::move(captured_entry_bl)); + }), 0); + } else { + m_image_ctx.op_work_queue->queue(new LambdaContext( + [this, log_entry, ctx](int r) { + ldout(m_image_ctx.cct, 15) << "flushing:" << log_entry + << " " << *log_entry << dendl; + log_entry->writeback(this->m_image_writeback, ctx); + }), 0); + } + } }); + + aio_read_data_blocks(log_entries, read_bls, ctx); } } diff --git a/src/librbd/cache/pwl/ssd/WriteLog.h b/src/librbd/cache/pwl/ssd/WriteLog.h index 8df2057b6d4ee..69cc366628f24 100644 --- a/src/librbd/cache/pwl/ssd/WriteLog.h +++ b/src/librbd/cache/pwl/ssd/WriteLog.h @@ -122,8 +122,9 @@ class WriteLog : public AbstractWriteLog { bool has_sync_point_logs(GenericLogOperations &ops); void append_op_log_entries(GenericLogOperations &ops); void alloc_op_log_entries(GenericLogOperations &ops); - Context* construct_flush_entry_ctx( - std::shared_ptr log_entry); + void construct_flush_entries(pwl::GenericLogEntries entires_to_flush, + DeferredContexts &post_unlock, + bool has_write_entry) override; void append_ops(GenericLogOperations &ops, Context *ctx, uint64_t* new_first_free_entry); void write_log_entries(GenericLogEntriesVector log_entries, From b3ecfc8f8419e801e6ea51f8d8d7fd6b7740cb09 Mon Sep 17 00:00:00 2001 From: Soumya Koduri Date: Mon, 1 Nov 2021 11:51:29 +0530 Subject: [PATCH 0456/2866] rgw/dbstore: No need for explicit LOCK in DBStore The current form of DBStore is not suited for multiple writers scenario. Hence no need for explicit mutex lock in DBStore but rather rely on inherent sqlite locking mechanism. Signed-off-by: Soumya Koduri --- src/rgw/store/dbstore/common/dbstore.cc | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/rgw/store/dbstore/common/dbstore.cc b/src/rgw/store/dbstore/common/dbstore.cc index 7cf8c44413ba2..c771763ff7f04 100644 --- a/src/rgw/store/dbstore/common/dbstore.cc +++ b/src/rgw/store/dbstore/common/dbstore.cc @@ -250,17 +250,14 @@ int DB::ProcessOp(const DoutPrefixProvider *dpp, string Op, struct DBOpParams *p int ret = -1; class DBOp *db_op; - Lock(dpp); db_op = getDBOp(dpp, Op, params); if (!db_op) { ldpp_dout(dpp, 0)<<"No db_op found for Op("<Execute(dpp, params); - Unlock(dpp); if (ret) { ldpp_dout(dpp, 0)<<"In Process op Execute failed for fop(" \ < Date: Thu, 16 Sep 2021 11:05:38 +0000 Subject: [PATCH 0457/2866] librbd/cache/pwl: Check the cache is clean Signed-off-by: Yin Congmin --- src/librbd/cache/pwl/AbstractWriteLog.cc | 16 +++++++++++++++- src/librbd/cache/pwl/AbstractWriteLog.h | 1 + 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/src/librbd/cache/pwl/AbstractWriteLog.cc b/src/librbd/cache/pwl/AbstractWriteLog.cc index c01d2d411f9da..5b7799e664b59 100644 --- a/src/librbd/cache/pwl/AbstractWriteLog.cc +++ b/src/librbd/cache/pwl/AbstractWriteLog.cc @@ -624,7 +624,7 @@ void AbstractWriteLog::shut_down(Context *on_finish) { } { std::lock_guard locker(m_lock); - ceph_assert(m_dirty_log_entries.size() == 0); + check_image_cache_state_clean(); m_wake_up_enabled = false; m_cache_state->clean = true; m_log_entries.clear(); @@ -2078,6 +2078,20 @@ bool AbstractWriteLog::can_retire_entry(std::shared_ptr log_ return log_entry->can_retire(); } +template +void AbstractWriteLog::check_image_cache_state_clean() { + ceph_assert(m_deferred_ios.empty()); + ceph_assert(m_ops_to_append.empty());; + ceph_assert(m_async_flush_ops == 0); + ceph_assert(m_async_append_ops == 0); + ceph_assert(m_dirty_log_entries.empty()); + ceph_assert(m_ops_to_flush.empty()); + ceph_assert(m_flush_ops_in_flight == 0); + ceph_assert(m_flush_bytes_in_flight == 0); + ceph_assert(m_bytes_dirty == 0); + ceph_assert(m_work_queue.empty()); +} + } // namespace pwl } // namespace cache } // namespace librbd diff --git a/src/librbd/cache/pwl/AbstractWriteLog.h b/src/librbd/cache/pwl/AbstractWriteLog.h index c982d2631fe3f..5aeab4c8de4d1 100644 --- a/src/librbd/cache/pwl/AbstractWriteLog.h +++ b/src/librbd/cache/pwl/AbstractWriteLog.h @@ -234,6 +234,7 @@ class AbstractWriteLog { void pwl_init(Context *on_finish, pwl::DeferredContexts &later); void update_image_cache_state(Context *on_finish); + void check_image_cache_state_clean(); void flush_dirty_entries(Context *on_finish); bool can_flush_entry(const std::shared_ptr log_entry); From c531768838e44ed8eb28e8b27594d7e03ca3ffcf Mon Sep 17 00:00:00 2001 From: Yin Congmin Date: Fri, 27 Aug 2021 15:41:49 +0000 Subject: [PATCH 0458/2866] librbd/cache/pwl/ssd: move finish_op() to the end of callback function finish_op() of ssd cache is not in the end of callback function in append_op_log_entries(), and after finish_op(), some operation also need to get m_lock. So, during shutdown, wait_for_ops() thinks all OPs are over, and no thread will acquire the m_lock, In the subsequent operation of shutdown, the m_lock is obtained, and _aio_stop() in bdev->close() waits for all aio_writes() and aio_submit() to end when the m_lock is held, but the callback function of aio_write() is waiting for the m_lock, causing a deadlock. Move finish_op() to the end to fix dead lock. Fixes: https://tracker.ceph.com/issues/52235 Signed-off-by: Yin Congmin --- src/librbd/cache/pwl/ssd/WriteLog.cc | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/librbd/cache/pwl/ssd/WriteLog.cc b/src/librbd/cache/pwl/ssd/WriteLog.cc index 550123924de0f..e6bcd17c35e4a 100644 --- a/src/librbd/cache/pwl/ssd/WriteLog.cc +++ b/src/librbd/cache/pwl/ssd/WriteLog.cc @@ -448,8 +448,6 @@ void WriteLog::append_op_log_entries(GenericLogOperations &ops) { Context *ctx = new LambdaContext([this, ops](int r) { assert(r == 0); ldout(m_image_ctx.cct, 20) << "Finished root update " << dendl; - this->m_async_update_superblock--; - this->m_async_op_tracker.finish_op(); auto captured_ops = std::move(ops); this->complete_op_log_entries(std::move(captured_ops), r); @@ -469,6 +467,8 @@ void WriteLog::append_op_log_entries(GenericLogOperations &ops) { if (need_finisher) { this->enlist_op_appender(); } + this->m_async_update_superblock--; + this->m_async_op_tracker.finish_op(); }); uint64_t *new_first_free_entry = new(uint64_t); Context *append_ctx = new LambdaContext( @@ -481,8 +481,6 @@ void WriteLog::append_op_log_entries(GenericLogOperations &ops) { for (auto &operation : ops) { operation->log_append_comp_time = now; } - this->m_async_append_ops--; - this->m_async_op_tracker.finish_op(); std::lock_guard locker(this->m_log_append_lock); std::lock_guard locker1(m_lock); @@ -494,6 +492,8 @@ void WriteLog::append_op_log_entries(GenericLogOperations &ops) { delete new_first_free_entry; schedule_update_root(new_root, ctx); } + this->m_async_append_ops--; + this->m_async_op_tracker.finish_op(); }); // Append logs and update first_free_update append_ops(ops, append_ctx, new_first_free_entry); @@ -756,12 +756,12 @@ bool WriteLog::retire_entries(const unsigned long int frees_per_tx) { this->m_alloc_failed_since_retire = false; this->wake_up(); - m_async_update_superblock--; - this->m_async_op_tracker.finish_op(); } this->dispatch_deferred_writes(); this->process_writeback_dirty_entries(); + m_async_update_superblock--; + this->m_async_op_tracker.finish_op(); }); std::lock_guard locker(m_lock); From 35f2fe8279361a3582825a62650b58272b5e3052 Mon Sep 17 00:00:00 2001 From: Yaarit Hatuka Date: Mon, 1 Nov 2021 15:19:30 +0000 Subject: [PATCH 0459/2866] doc/dev/release-checklist: add manual review of telemetry reports When gathering metrics, the telemetry module automatically detects and removes sensitive details (e.g. pools and hosts names) from its reports. We wish to manually double-check here that none of these details are accidentally included in the reports. Signed-off-by: Yaarit Hatuka --- doc/dev/release-checklists.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/dev/release-checklists.rst b/doc/dev/release-checklists.rst index 2668ed84469ed..9ba01893e1411 100644 --- a/doc/dev/release-checklists.rst +++ b/doc/dev/release-checklists.rst @@ -98,6 +98,7 @@ First release candidate ======================= - [ ] src/ceph_release: change type to `rc` +- [ ] opt-in to all telemetry channels, generate telemetry reports, and verify no sensitive details (like pools names) are collected First stable release From 447564e4db680157c72a3f090e2c73cc35cfe700 Mon Sep 17 00:00:00 2001 From: "Adam C. Emerson" Date: Wed, 20 Oct 2021 16:39:15 -0400 Subject: [PATCH 0460/2866] build: Remove LevelDB support Whereas new LevelDB packages are build with -fno-rtti and break our attempts to compile against them, and whereas LevelDB has been deprecated for some time, with the Ceph team expending great effort to migrating OSDs off of it, let it be therefore removed. Signed-off-by: Adam C. Emerson --- CMakeLists.txt | 10 - ceph.spec.in | 1 - cmake/modules/Findleveldb.cmake | 18 -- debian/control | 1 - src/include/config-h.in.cmake | 6 - src/kv/CMakeLists.txt | 6 +- src/kv/KeyValueDB.cc | 13 - src/kv/KeyValueDB.h | 4 +- src/kv/LevelDBStore.cc | 455 -------------------------------- src/kv/LevelDBStore.h | 412 ----------------------------- src/test/common/CMakeLists.txt | 1 - src/test/objectstore/test_kv.cc | 2 +- 12 files changed, 4 insertions(+), 925 deletions(-) delete mode 100644 cmake/modules/Findleveldb.cmake delete mode 100644 src/kv/LevelDBStore.cc delete mode 100644 src/kv/LevelDBStore.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 0350131765570..6a0221c431e85 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -297,16 +297,6 @@ if(LINUX) endif() endif() -option(WITH_LEVELDB "LevelDB is here" ON) -if(WITH_LEVELDB) - if(LEVELDB_PREFIX) - include_directories(SYSTEM ${LEVELDB_PREFIX}/include) - link_directories(${LEVELDB_PREFIX}/lib) - endif() - find_package(leveldb REQUIRED) - find_file(HAVE_LEVELDB_FILTER_POLICY leveldb/filter_policy.h PATHS ${LEVELDB_INCLUDE_DIR}) -endif(WITH_LEVELDB) - find_package(snappy REQUIRED) option(WITH_BROTLI "Brotli compression support" OFF) diff --git a/ceph.spec.in b/ceph.spec.in index 3579d7e628eaf..092053a155554 100644 --- a/ceph.spec.in +++ b/ceph.spec.in @@ -200,7 +200,6 @@ BuildRequires: gperftools-devel >= 2.6.1 BuildRequires: gperftools-devel >= 2.4 %endif %endif -BuildRequires: leveldb-devel > 1.2 BuildRequires: libaio-devel BuildRequires: libblkid-devel >= 2.17 BuildRequires: cryptsetup-devel diff --git a/cmake/modules/Findleveldb.cmake b/cmake/modules/Findleveldb.cmake deleted file mode 100644 index 32786e9799451..0000000000000 --- a/cmake/modules/Findleveldb.cmake +++ /dev/null @@ -1,18 +0,0 @@ -# - Find LevelDB -# -# LEVELDB_INCLUDE_DIR - Where to find leveldb/db.h -# LEVELDB_LIBRARIES - List of libraries when using LevelDB. -# LEVELDB_FOUND - True if LevelDB found. - -find_path(LEVELDB_INCLUDE_DIR leveldb/db.h - HINTS $ENV{LEVELDB_ROOT}/include - DOC "Path in which the file leveldb/db.h is located." ) - -find_library(LEVELDB_LIBRARIES leveldb - HINTS $ENV{LEVELDB_ROOT}/lib - DOC "Path to leveldb library." ) - -mark_as_advanced(LEVELDB_INCLUDE_DIR LEVELDB_LIBRARIES) - -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(leveldb DEFAULT_MSG LEVELDB_LIBRARIES LEVELDB_INCLUDE_DIR) diff --git a/debian/control b/debian/control index 1800b000eeebd..2d50b92a0379a 100644 --- a/debian/control +++ b/debian/control @@ -48,7 +48,6 @@ Build-Depends: automake, librdmacm-dev, libkeyutils-dev, libldap2-dev, - libleveldb-dev, liblttng-ust-dev, liblua5.3-dev, liblz4-dev (>= 0.0~r131), diff --git a/src/include/config-h.in.cmake b/src/include/config-h.in.cmake index 01681c494612c..6834c8a1a71dd 100644 --- a/src/include/config-h.in.cmake +++ b/src/include/config-h.in.cmake @@ -117,9 +117,6 @@ /* PMEM_DEVICE (OSD) conditional compilation */ #cmakedefine HAVE_BLUESTORE_PMEM -/* Defined if LevelDB supports bloom filters */ -#cmakedefine HAVE_LEVELDB_FILTER_POLICY - /* Define if you have tcmalloc */ #cmakedefine HAVE_LIBTCMALLOC #cmakedefine LIBTCMALLOC_MISSING_ALIGNED_ALLOC @@ -157,9 +154,6 @@ /* define if radosgw enabled */ #cmakedefine WITH_RADOSGW -/* define if leveldb is enabled */ -#cmakedefine WITH_LEVELDB - /* define if radosgw has openssl support */ #cmakedefine WITH_CURL_OPENSSL diff --git a/src/kv/CMakeLists.txt b/src/kv/CMakeLists.txt index dce4cd2d181f9..f797aea11355c 100644 --- a/src/kv/CMakeLists.txt +++ b/src/kv/CMakeLists.txt @@ -6,13 +6,9 @@ set(kv_srcs rocksdb_cache/ShardedCache.cc rocksdb_cache/BinnedLRUCache.cc) -if (WITH_LEVELDB) - list(APPEND kv_srcs LevelDBStore.cc) -endif (WITH_LEVELDB) - add_library(kv STATIC ${kv_srcs} $) -target_link_libraries(kv ${LEVELDB_LIBRARIES} +target_link_libraries(kv RocksDB::RocksDB heap_profiler) diff --git a/src/kv/KeyValueDB.cc b/src/kv/KeyValueDB.cc index f050040c15622..c42968bbe184a 100644 --- a/src/kv/KeyValueDB.cc +++ b/src/kv/KeyValueDB.cc @@ -2,9 +2,6 @@ // vim: ts=8 sw=2 smarttab #include "KeyValueDB.h" -#ifdef WITH_LEVELDB -#include "LevelDBStore.h" -#endif #include "MemDB.h" #include "RocksDBStore.h" @@ -16,11 +13,6 @@ KeyValueDB *KeyValueDB::create(CephContext *cct, const string& type, map options, void *p) { -#ifdef WITH_LEVELDB - if (type == "leveldb") { - return new LevelDBStore(cct, dir); - } -#endif if (type == "rocksdb") { return new RocksDBStore(cct, dir, options, p); } @@ -33,11 +25,6 @@ KeyValueDB *KeyValueDB::create(CephContext *cct, const string& type, int KeyValueDB::test_init(const string& type, const string& dir) { -#ifdef WITH_LEVELDB - if (type == "leveldb") { - return LevelDBStore::_test_init(dir); - } -#endif if (type == "rocksdb") { return RocksDBStore::_test_init(dir); } diff --git a/src/kv/KeyValueDB.h b/src/kv/KeyValueDB.h index afbfb6ecfd7f3..3dd0894bb1813 100644 --- a/src/kv/KeyValueDB.h +++ b/src/kv/KeyValueDB.h @@ -17,7 +17,7 @@ /** * Defines virtual interface to be implemented by key value store * - * Kyoto Cabinet or LevelDB should implement this + * Kyoto Cabinet should implement this */ class KeyValueDB { public: @@ -153,7 +153,7 @@ class KeyValueDB { virtual void close() { } - /// Try to repair K/V database. leveldb and rocksdb require that database must be not opened. + /// Try to repair K/V database. rocksdb requires that database must be not opened. virtual int repair(std::ostream &out) { return 0; } virtual Transaction get_transaction() = 0; diff --git a/src/kv/LevelDBStore.cc b/src/kv/LevelDBStore.cc deleted file mode 100644 index 6142e7505a6df..0000000000000 --- a/src/kv/LevelDBStore.cc +++ /dev/null @@ -1,455 +0,0 @@ -// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- -// vim: ts=8 sw=2 smarttab -#include "LevelDBStore.h" - -#include -#include -#include -#include - -#include "common/debug.h" -#include "common/perf_counters.h" - -// re-include our assert to clobber the system one; fix dout: -#include "include/ceph_assert.h" - -#define dout_context cct -#define dout_subsys ceph_subsys_leveldb -#undef dout_prefix -#define dout_prefix *_dout << "leveldb: " - -using std::list; -using std::string; -using std::ostream; -using std::pair; -using std::vector; - -using ceph::bufferlist; -using ceph::bufferptr; - -class CephLevelDBLogger : public leveldb::Logger { - CephContext *cct; -public: - explicit CephLevelDBLogger(CephContext *c) : cct(c) { - cct->get(); - } - ~CephLevelDBLogger() override { - cct->put(); - } - - // Write an entry to the log file with the specified format. - void Logv(const char* format, va_list ap) override { - dout(1); - char buf[65536]; - vsnprintf(buf, sizeof(buf), format, ap); - *_dout << buf << dendl; - } -}; - -leveldb::Logger *create_leveldb_ceph_logger() -{ - return new CephLevelDBLogger(g_ceph_context); -} - -int LevelDBStore::init(string option_str) -{ - // init defaults. caller can override these if they want - // prior to calling open. - options.write_buffer_size = g_conf()->leveldb_write_buffer_size; - options.cache_size = g_conf()->leveldb_cache_size; - options.block_size = g_conf()->leveldb_block_size; - options.bloom_size = g_conf()->leveldb_bloom_size; - options.compression_enabled = g_conf()->leveldb_compression; - options.paranoid_checks = g_conf()->leveldb_paranoid; - options.max_open_files = g_conf()->leveldb_max_open_files; - options.log_file = g_conf()->leveldb_log; - return 0; -} - -int LevelDBStore::open(ostream &out, const std::string& cfs) { - if (!cfs.empty()) { - ceph_abort_msg("Not implemented"); - } - return do_open(out, false); -} - -int LevelDBStore::create_and_open(ostream &out, const std::string& cfs) { - if (!cfs.empty()) { - ceph_abort_msg("Not implemented"); - } - return do_open(out, true); -} - -int LevelDBStore::load_leveldb_options(bool create_if_missing, leveldb::Options &ldoptions) -{ - if (options.write_buffer_size) - ldoptions.write_buffer_size = options.write_buffer_size; - if (options.max_open_files) - ldoptions.max_open_files = options.max_open_files; - if (options.cache_size) { - leveldb::Cache *_db_cache = leveldb::NewLRUCache(options.cache_size); - db_cache.reset(_db_cache); - ldoptions.block_cache = db_cache.get(); - } - if (options.block_size) - ldoptions.block_size = options.block_size; - if (options.bloom_size) { -#ifdef HAVE_LEVELDB_FILTER_POLICY - const leveldb::FilterPolicy *_filterpolicy = - leveldb::NewBloomFilterPolicy(options.bloom_size); - filterpolicy.reset(_filterpolicy); - ldoptions.filter_policy = filterpolicy.get(); -#else - ceph_abort_msg("bloom size set but installed leveldb doesn't support bloom filters"); -#endif - } - if (options.compression_enabled) - ldoptions.compression = leveldb::kSnappyCompression; - else - ldoptions.compression = leveldb::kNoCompression; - if (options.block_restart_interval) - ldoptions.block_restart_interval = options.block_restart_interval; - - ldoptions.error_if_exists = options.error_if_exists; - ldoptions.paranoid_checks = options.paranoid_checks; - ldoptions.create_if_missing = create_if_missing; - - if (g_conf()->leveldb_log_to_ceph_log) { - ceph_logger = new CephLevelDBLogger(g_ceph_context); - ldoptions.info_log = ceph_logger; - } - - if (options.log_file.length()) { - leveldb::Env *env = leveldb::Env::Default(); - env->NewLogger(options.log_file, &ldoptions.info_log); - } - return 0; -} - -int LevelDBStore::do_open(ostream &out, bool create_if_missing) -{ - leveldb::Options ldoptions; - int r = load_leveldb_options(create_if_missing, ldoptions); - if (r) { - dout(1) << "load leveldb options failed" << dendl; - return r; - } - - leveldb::DB *_db; - leveldb::Status status = leveldb::DB::Open(ldoptions, path, &_db); - db.reset(_db); - if (!status.ok()) { - out << status.ToString() << std::endl; - return -EINVAL; - } - - PerfCountersBuilder plb(g_ceph_context, "leveldb", l_leveldb_first, l_leveldb_last); - plb.add_u64_counter(l_leveldb_gets, "leveldb_get", "Gets"); - plb.add_u64_counter(l_leveldb_txns, "leveldb_transaction", "Transactions"); - plb.add_time_avg(l_leveldb_get_latency, "leveldb_get_latency", "Get Latency"); - plb.add_time_avg(l_leveldb_submit_latency, "leveldb_submit_latency", "Submit Latency"); - plb.add_time_avg(l_leveldb_submit_sync_latency, "leveldb_submit_sync_latency", "Submit Sync Latency"); - plb.add_u64_counter(l_leveldb_compact, "leveldb_compact", "Compactions"); - plb.add_u64_counter(l_leveldb_compact_range, "leveldb_compact_range", "Compactions by range"); - plb.add_u64_counter(l_leveldb_compact_queue_merge, "leveldb_compact_queue_merge", "Mergings of ranges in compaction queue"); - plb.add_u64(l_leveldb_compact_queue_len, "leveldb_compact_queue_len", "Length of compaction queue"); - logger = plb.create_perf_counters(); - cct->get_perfcounters_collection()->add(logger); - - if (g_conf()->leveldb_compact_on_mount) { - derr << "Compacting leveldb store..." << dendl; - compact(); - derr << "Finished compacting leveldb store" << dendl; - } - return 0; -} - -int LevelDBStore::_test_init(const string& dir) -{ - leveldb::Options options; - options.create_if_missing = true; - leveldb::DB *db; - leveldb::Status status = leveldb::DB::Open(options, dir, &db); - delete db; - return status.ok() ? 0 : -EIO; -} - -LevelDBStore::~LevelDBStore() -{ - close(); -} - -void LevelDBStore::close() -{ - // stop compaction thread - compact_queue_lock.lock(); - if (compact_thread.is_started()) { - compact_queue_stop = true; - compact_queue_cond.notify_all(); - compact_queue_lock.unlock(); - compact_thread.join(); - } else { - compact_queue_lock.unlock(); - } - - if (logger) { - cct->get_perfcounters_collection()->remove(logger); - delete logger; - logger = nullptr; - } - - // Ensure db is destroyed before dependent db_cache and filterpolicy - db.reset(); - delete ceph_logger; -} - -int LevelDBStore::repair(std::ostream &out) -{ - leveldb::Options ldoptions; - int r = load_leveldb_options(false, ldoptions); - if (r) { - dout(1) << "load leveldb options failed" << dendl; - out << "load leveldb options failed" << std::endl; - return r; - } - leveldb::Status status = leveldb::RepairDB(path, ldoptions); - if (status.ok()) { - return 0; - } else { - out << "repair leveldb failed : " << status.ToString() << std::endl; - return 1; - } -} - -int LevelDBStore::submit_transaction(KeyValueDB::Transaction t) -{ - utime_t start = ceph_clock_now(); - LevelDBTransactionImpl * _t = - static_cast(t.get()); - leveldb::Status s = db->Write(leveldb::WriteOptions(), &(_t->bat)); - utime_t lat = ceph_clock_now() - start; - logger->inc(l_leveldb_txns); - logger->tinc(l_leveldb_submit_latency, lat); - return s.ok() ? 0 : -1; -} - -int LevelDBStore::submit_transaction_sync(KeyValueDB::Transaction t) -{ - utime_t start = ceph_clock_now(); - LevelDBTransactionImpl * _t = - static_cast(t.get()); - leveldb::WriteOptions options; - options.sync = true; - leveldb::Status s = db->Write(options, &(_t->bat)); - utime_t lat = ceph_clock_now() - start; - logger->inc(l_leveldb_txns); - logger->tinc(l_leveldb_submit_sync_latency, lat); - return s.ok() ? 0 : -1; -} - -void LevelDBStore::LevelDBTransactionImpl::set( - const string &prefix, - const string &k, - const bufferlist &to_set_bl) -{ - string key = combine_strings(prefix, k); - size_t bllen = to_set_bl.length(); - // bufferlist::c_str() is non-constant, so we can't call c_str() - if (to_set_bl.is_contiguous() && bllen > 0) { - // bufferlist contains just one ptr or they're contiguous - bat.Put(leveldb::Slice(key), leveldb::Slice(to_set_bl.buffers().front().c_str(), bllen)); - } else if ((bllen <= 32 * 1024) && (bllen > 0)) { - // 2+ bufferptrs that are not contiguopus - // allocate buffer on stack and copy bl contents to that buffer - // make sure the buffer isn't too large or we might crash here... - char* slicebuf = (char*) alloca(bllen); - leveldb::Slice newslice(slicebuf, bllen); - for (const auto& node : to_set_bl.buffers()) { - const size_t ptrlen = node.length(); - memcpy(static_cast(slicebuf), node.c_str(), ptrlen); - slicebuf += ptrlen; - } - bat.Put(leveldb::Slice(key), newslice); - } else { - // 2+ bufferptrs that are not contiguous, and enormous in size - bufferlist val = to_set_bl; - bat.Put(leveldb::Slice(key), leveldb::Slice(val.c_str(), val.length())); - } -} - -void LevelDBStore::LevelDBTransactionImpl::rmkey(const string &prefix, - const string &k) -{ - string key = combine_strings(prefix, k); - bat.Delete(leveldb::Slice(key)); -} - -void LevelDBStore::LevelDBTransactionImpl::rmkeys_by_prefix(const string &prefix) -{ - KeyValueDB::Iterator it = db->get_iterator(prefix); - for (it->seek_to_first(); - it->valid(); - it->next()) { - bat.Delete(leveldb::Slice(combine_strings(prefix, it->key()))); - } -} - -void LevelDBStore::LevelDBTransactionImpl::rm_range_keys(const string &prefix, const string &start, const string &end) -{ - KeyValueDB::Iterator it = db->get_iterator(prefix); - it->lower_bound(start); - while (it->valid()) { - if (it->key() >= end) { - break; - } - bat.Delete(combine_strings(prefix, it->key())); - it->next(); - } -} - -int LevelDBStore::get( - const string &prefix, - const std::set &keys, - std::map *out) -{ - utime_t start = ceph_clock_now(); - for (std::set::const_iterator i = keys.begin(); - i != keys.end(); ++i) { - std::string value; - std::string bound = combine_strings(prefix, *i); - auto status = db->Get(leveldb::ReadOptions(), leveldb::Slice(bound), &value); - if (status.ok()) - (*out)[*i].append(value); - } - utime_t lat = ceph_clock_now() - start; - logger->inc(l_leveldb_gets); - logger->tinc(l_leveldb_get_latency, lat); - return 0; -} - -int LevelDBStore::get(const string &prefix, - const string &key, - bufferlist *out) -{ - ceph_assert(out && (out->length() == 0)); - utime_t start = ceph_clock_now(); - int r = 0; - string value, k; - leveldb::Status s; - k = combine_strings(prefix, key); - s = db->Get(leveldb::ReadOptions(), leveldb::Slice(k), &value); - if (s.ok()) { - out->append(value); - } else { - r = -ENOENT; - } - utime_t lat = ceph_clock_now() - start; - logger->inc(l_leveldb_gets); - logger->tinc(l_leveldb_get_latency, lat); - return r; -} - -string LevelDBStore::combine_strings(const string &prefix, const string &value) -{ - string out = prefix; - out.push_back(0); - out.append(value); - return out; -} - -bufferlist LevelDBStore::to_bufferlist(leveldb::Slice in) -{ - bufferlist bl; - bl.append(bufferptr(in.data(), in.size())); - return bl; -} - -int LevelDBStore::split_key(leveldb::Slice in, string *prefix, string *key) -{ - size_t prefix_len = 0; - - // Find separator inside Slice - char* separator = (char*) memchr(in.data(), 0, in.size()); - if (separator == NULL) - return -EINVAL; - prefix_len = size_t(separator - in.data()); - if (prefix_len >= in.size()) - return -EINVAL; - - if (prefix) - *prefix = string(in.data(), prefix_len); - if (key) - *key = string(separator+1, in.size() - prefix_len - 1); - return 0; -} - -void LevelDBStore::compact() -{ - logger->inc(l_leveldb_compact); - db->CompactRange(NULL, NULL); -} - - -void LevelDBStore::compact_thread_entry() -{ - std::unique_lock l{compact_queue_lock}; - while (!compact_queue_stop) { - while (!compact_queue.empty()) { - pair range = compact_queue.front(); - compact_queue.pop_front(); - logger->set(l_leveldb_compact_queue_len, compact_queue.size()); - l.unlock(); - logger->inc(l_leveldb_compact_range); - if (range.first.empty() && range.second.empty()) { - compact(); - } else { - compact_range(range.first, range.second); - } - l.lock(); - continue; - } - if (compact_queue_stop) - break; - compact_queue_cond.wait(l); - } -} - -void LevelDBStore::compact_range_async(const string& start, const string& end) -{ - std::lock_guard l(compact_queue_lock); - - // try to merge adjacent ranges. this is O(n), but the queue should - // be short. note that we do not cover all overlap cases and merge - // opportunities here, but we capture the ones we currently need. - list< pair >::iterator p = compact_queue.begin(); - while (p != compact_queue.end()) { - if (p->first == start && p->second == end) { - // dup; no-op - return; - } - if (p->first <= end && p->first > start) { - // merge with existing range to the right - compact_queue.push_back(make_pair(start, p->second)); - compact_queue.erase(p); - logger->inc(l_leveldb_compact_queue_merge); - break; - } - if (p->second >= start && p->second < end) { - // merge with existing range to the left - compact_queue.push_back(make_pair(p->first, end)); - compact_queue.erase(p); - logger->inc(l_leveldb_compact_queue_merge); - break; - } - ++p; - } - if (p == compact_queue.end()) { - // no merge, new entry. - compact_queue.push_back(make_pair(start, end)); - logger->set(l_leveldb_compact_queue_len, compact_queue.size()); - } - compact_queue_cond.notify_all(); - if (!compact_thread.is_started()) { - compact_thread.create("levdbst_compact"); - } -} diff --git a/src/kv/LevelDBStore.h b/src/kv/LevelDBStore.h deleted file mode 100644 index 085193ee03321..0000000000000 --- a/src/kv/LevelDBStore.h +++ /dev/null @@ -1,412 +0,0 @@ -// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- -// vim: ts=8 sw=2 smarttab -#ifndef LEVEL_DB_STORE_H -#define LEVEL_DB_STORE_H - -#include "include/types.h" -#include "include/buffer_fwd.h" -#include "KeyValueDB.h" -#include -#include -#include -#include -#include "leveldb/db.h" -#include "leveldb/env.h" -#include "leveldb/write_batch.h" -#include "leveldb/slice.h" -#include "leveldb/cache.h" -#ifdef HAVE_LEVELDB_FILTER_POLICY -#include "leveldb/filter_policy.h" -#endif - -#include -#include "common/errno.h" -#include "common/dout.h" -#include "include/ceph_assert.h" -#include "common/Formatter.h" -#include "common/Cond.h" - -#include "common/ceph_context.h" -#include "include/common_fwd.h" - -// reinclude our assert to clobber the system one -# include "include/ceph_assert.h" - -enum { - l_leveldb_first = 34300, - l_leveldb_gets, - l_leveldb_txns, - l_leveldb_get_latency, - l_leveldb_submit_latency, - l_leveldb_submit_sync_latency, - l_leveldb_compact, - l_leveldb_compact_range, - l_leveldb_compact_queue_merge, - l_leveldb_compact_queue_len, - l_leveldb_last, -}; - -extern leveldb::Logger *create_leveldb_ceph_logger(); - -class CephLevelDBLogger; - -/** - * Uses LevelDB to implement the KeyValueDB interface - */ -class LevelDBStore : public KeyValueDB { - CephContext *cct; - PerfCounters *logger; - CephLevelDBLogger *ceph_logger; - std::string path; - boost::scoped_ptr db_cache; -#ifdef HAVE_LEVELDB_FILTER_POLICY - boost::scoped_ptr filterpolicy; -#endif - boost::scoped_ptr db; - - int load_leveldb_options(bool create_if_missing, leveldb::Options &opts); - int do_open(std::ostream &out, bool create_if_missing); - - // manage async compactions - ceph::mutex compact_queue_lock = - ceph::make_mutex("LevelDBStore::compact_thread_lock"); - ceph::condition_variable compact_queue_cond; - std::list> compact_queue; - bool compact_queue_stop; - class CompactThread : public Thread { - LevelDBStore *db; - public: - explicit CompactThread(LevelDBStore *d) : db(d) {} - void *entry() override { - db->compact_thread_entry(); - return NULL; - } - friend class LevelDBStore; - } compact_thread; - - void compact_thread_entry(); - - void compact_range(const std::string& start, const std::string& end) { - leveldb::Slice cstart(start); - leveldb::Slice cend(end); - db->CompactRange(&cstart, &cend); - } - void compact_range_async(const std::string& start, const std::string& end); - -public: - /// compact the underlying leveldb store - void compact() override; - - void compact_async() override { - compact_range_async({}, {}); - } - - /// compact db for all keys with a given prefix - void compact_prefix(const std::string& prefix) override { - compact_range(prefix, past_prefix(prefix)); - } - void compact_prefix_async(const std::string& prefix) override { - compact_range_async(prefix, past_prefix(prefix)); - } - void compact_range(const std::string& prefix, - const std::string& start, const std::string& end) override { - compact_range(combine_strings(prefix, start), combine_strings(prefix, end)); - } - void compact_range_async(const std::string& prefix, - const std::string& start, const std::string& end) override { - compact_range_async(combine_strings(prefix, start), - combine_strings(prefix, end)); - } - - - /** - * options_t: Holds options which are minimally interpreted - * on initialization and then passed through to LevelDB. - * We transform a couple of these into actual LevelDB - * structures, but the rest are simply passed through unchanged. See - * leveldb/options.h for more precise details on each. - * - * Set them after constructing the LevelDBStore, but before calling - * open() or create_and_open(). - */ - struct options_t { - uint64_t write_buffer_size; /// in-memory write buffer size - int max_open_files; /// maximum number of files LevelDB can open at once - uint64_t cache_size; /// size of extra decompressed cache to use - uint64_t block_size; /// user data per block - int bloom_size; /// number of bits per entry to put in a bloom filter - bool compression_enabled; /// whether to use libsnappy compression or not - - // don't change these ones. No, seriously - int block_restart_interval; - bool error_if_exists; - bool paranoid_checks; - - std::string log_file; - - options_t() : - write_buffer_size(0), //< 0 means default - max_open_files(0), //< 0 means default - cache_size(0), //< 0 means no cache (default) - block_size(0), //< 0 means default - bloom_size(0), //< 0 means no bloom filter (default) - compression_enabled(true), //< set to false for no compression - block_restart_interval(0), //< 0 means default - error_if_exists(false), //< set to true if you want to check nonexistence - paranoid_checks(false) //< set to true if you want paranoid checks - {} - } options; - - LevelDBStore(CephContext *c, const std::string &path) : - cct(c), - logger(NULL), - ceph_logger(NULL), - path(path), - db_cache(NULL), -#ifdef HAVE_LEVELDB_FILTER_POLICY - filterpolicy(NULL), -#endif - compact_queue_stop(false), - compact_thread(this), - options() - {} - - ~LevelDBStore() override; - - static int _test_init(const std::string& dir); - int init(std::string option_str="") override; - - /// Opens underlying db - int open(std::ostream &out, const std::string& cfs="") override; - /// Creates underlying db if missing and opens it - int create_and_open(std::ostream &out, const std::string& cfs="") override; - - void close() override; - - PerfCounters *get_perf_counters() override - { - return logger; - } - int repair(std::ostream &out) override; - - class LevelDBTransactionImpl : public KeyValueDB::TransactionImpl { - public: - leveldb::WriteBatch bat; - LevelDBStore *db; - explicit LevelDBTransactionImpl(LevelDBStore *db) : db(db) {} - void set( - const std::string &prefix, - const std::string &k, - const ceph::buffer::list &bl) override; - using KeyValueDB::TransactionImpl::set; - void rmkey( - const std::string &prefix, - const std::string &k) override; - void rmkeys_by_prefix( - const std::string &prefix - ) override; - virtual void rm_range_keys( - const std::string &prefix, - const std::string &start, - const std::string &end) override; - - using KeyValueDB::TransactionImpl::rmkey; - }; - - KeyValueDB::Transaction get_transaction() override { - return std::make_shared(this); - } - - int submit_transaction(KeyValueDB::Transaction t) override; - int submit_transaction_sync(KeyValueDB::Transaction t) override; - int get( - const std::string &prefix, - const std::set &key, - std::map *out - ) override; - - int get(const std::string &prefix, - const std::string &key, - ceph::buffer::list *value) override; - - using KeyValueDB::get; - - class LevelDBWholeSpaceIteratorImpl : - public KeyValueDB::WholeSpaceIteratorImpl { - protected: - boost::scoped_ptr dbiter; - public: - explicit LevelDBWholeSpaceIteratorImpl(leveldb::Iterator *iter) : - dbiter(iter) { } - ~LevelDBWholeSpaceIteratorImpl() override { } - - int seek_to_first() override { - dbiter->SeekToFirst(); - return dbiter->status().ok() ? 0 : -1; - } - int seek_to_first(const std::string &prefix) override { - leveldb::Slice slice_prefix(prefix); - dbiter->Seek(slice_prefix); - return dbiter->status().ok() ? 0 : -1; - } - int seek_to_last() override { - dbiter->SeekToLast(); - return dbiter->status().ok() ? 0 : -1; - } - int seek_to_last(const std::string &prefix) override { - std::string limit = past_prefix(prefix); - leveldb::Slice slice_limit(limit); - dbiter->Seek(slice_limit); - - if (!dbiter->Valid()) { - dbiter->SeekToLast(); - } else { - dbiter->Prev(); - } - return dbiter->status().ok() ? 0 : -1; - } - int upper_bound(const std::string &prefix, const std::string &after) override { - lower_bound(prefix, after); - if (valid()) { - std::pair key = raw_key(); - if (key.first == prefix && key.second == after) - next(); - } - return dbiter->status().ok() ? 0 : -1; - } - int lower_bound(const std::string &prefix, const std::string &to) override { - std::string bound = combine_strings(prefix, to); - leveldb::Slice slice_bound(bound); - dbiter->Seek(slice_bound); - return dbiter->status().ok() ? 0 : -1; - } - bool valid() override { - return dbiter->Valid(); - } - int next() override { - if (valid()) - dbiter->Next(); - return dbiter->status().ok() ? 0 : -1; - } - int prev() override { - if (valid()) - dbiter->Prev(); - return dbiter->status().ok() ? 0 : -1; - } - std::string key() override { - std::string out_key; - split_key(dbiter->key(), 0, &out_key); - return out_key; - } - std::pair raw_key() override { - std::string prefix, key; - split_key(dbiter->key(), &prefix, &key); - return std::make_pair(prefix, key); - } - bool raw_key_is_prefixed(const std::string &prefix) override { - leveldb::Slice key = dbiter->key(); - if ((key.size() > prefix.length()) && (key[prefix.length()] == '\0')) { - return memcmp(key.data(), prefix.c_str(), prefix.length()) == 0; - } else { - return false; - } - } - ceph::buffer::list value() override { - return to_bufferlist(dbiter->value()); - } - - ceph::bufferptr value_as_ptr() override { - leveldb::Slice data = dbiter->value(); - return ceph::bufferptr(data.data(), data.size()); - } - - int status() override { - return dbiter->status().ok() ? 0 : -1; - } - }; - - /// Utility - static std::string combine_strings(const std::string &prefix, const std::string &value); - static int split_key(leveldb::Slice in, std::string *prefix, std::string *key); - static ceph::buffer::list to_bufferlist(leveldb::Slice in); - static std::string past_prefix(const std::string &prefix) { - std::string limit = prefix; - limit.push_back(1); - return limit; - } - - uint64_t get_estimated_size(std::map &extra) override { - DIR *store_dir = opendir(path.c_str()); - if (!store_dir) { - lderr(cct) << __func__ << " something happened opening the store: " - << cpp_strerror(errno) << dendl; - return 0; - } - - uint64_t total_size = 0; - uint64_t sst_size = 0; - uint64_t log_size = 0; - uint64_t misc_size = 0; - - struct dirent *entry = NULL; - while ((entry = readdir(store_dir)) != NULL) { - std::string n(entry->d_name); - - if (n == "." || n == "..") - continue; - - std::string fpath = path + '/' + n; - struct stat s; - int err = stat(fpath.c_str(), &s); - if (err < 0) - err = -errno; - // we may race against leveldb while reading files; this should only - // happen when those files are being updated, data is being shuffled - // and files get removed, in which case there's not much of a problem - // as we'll get to them next time around. - if (err == -ENOENT) { - continue; - } - if (err < 0) { - lderr(cct) << __func__ << " error obtaining stats for " << fpath - << ": " << cpp_strerror(err) << dendl; - goto err; - } - - size_t pos = n.find_last_of('.'); - if (pos == std::string::npos) { - misc_size += s.st_size; - continue; - } - - std::string ext = n.substr(pos+1); - if (ext == "sst") { - sst_size += s.st_size; - } else if (ext == "log") { - log_size += s.st_size; - } else { - misc_size += s.st_size; - } - } - - total_size = sst_size + log_size + misc_size; - - extra["sst"] = sst_size; - extra["log"] = log_size; - extra["misc"] = misc_size; - extra["total"] = total_size; - -err: - closedir(store_dir); - return total_size; - } - - - WholeSpaceIterator get_wholespace_iterator(IteratorOpts opts = 0) override { - return std::make_shared( - db->NewIterator(leveldb::ReadOptions())); - } - -}; - -#endif diff --git a/src/test/common/CMakeLists.txt b/src/test/common/CMakeLists.txt index 6d5780a983935..0e84b3d722815 100644 --- a/src/test/common/CMakeLists.txt +++ b/src/test/common/CMakeLists.txt @@ -8,7 +8,6 @@ add_executable(get_command_descriptions target_link_libraries(get_command_descriptions mon global - ${LEVELDB_LIBRARIES} ${EXTRALIBS} ${BLKID_LIBRARIES} ${CMAKE_DL_LIBS} diff --git a/src/test/objectstore/test_kv.cc b/src/test/objectstore/test_kv.cc index 5917f140b1084..abc43f46d9d38 100644 --- a/src/test/objectstore/test_kv.cc +++ b/src/test/objectstore/test_kv.cc @@ -1278,7 +1278,7 @@ TEST_F(RocksDBResharding, change_reshard) { INSTANTIATE_TEST_SUITE_P( KeyValueDB, KVTest, - ::testing::Values("leveldb", "rocksdb", "memdb")); + ::testing::Values("rocksdb", "memdb")); INSTANTIATE_TEST_SUITE_P( KeyValueDB, From 5d6a6ec0bed11fb7915a7f2eeb3b8338355581d9 Mon Sep 17 00:00:00 2001 From: Adam King Date: Thu, 28 Oct 2021 16:29:16 -0400 Subject: [PATCH 0461/2866] mgr/cephadm: fix port handling for cephadm endpoint check range of ports if port is taken reject ports < 1024 or > 65535 redeploy server with new port is endpoint_port config option changed fix agent dep to use actual port in use rather than config option Signed-off-by: Adam King --- src/pybind/mgr/cephadm/agent.py | 32 ++++++++++++++----- src/pybind/mgr/cephadm/module.py | 11 ++----- .../mgr/cephadm/services/cephadmservice.py | 18 ++++++----- 3 files changed, 37 insertions(+), 24 deletions(-) diff --git a/src/pybind/mgr/cephadm/agent.py b/src/pybind/mgr/cephadm/agent.py index 71e33b2a411f4..6f4a22741b18a 100644 --- a/src/pybind/mgr/cephadm/agent.py +++ b/src/pybind/mgr/cephadm/agent.py @@ -45,13 +45,12 @@ def __init__(self, mgr: "CephadmOrchestrator") -> None: self.mgr = mgr self.cherrypy_shutdown_event = threading.Event() self.ssl_certs = SSLCerts(self.mgr) + self.server_port = 7150 + self.server_addr = self.mgr.get_mgr_ip() super(CherryPyThread, self).__init__(target=self.run) def run(self) -> None: try: - server_addr = self.mgr.get_mgr_ip() - server_port = self.mgr.endpoint_port - self.ssl_certs.generate_root_cert() cert, key = self.ssl_certs.generate_cert() @@ -67,11 +66,9 @@ def run(self) -> None: verify_tls_files(cert_fname, key_fname) - self.mgr.set_uri('https://{0}:{1}/'.format(server_addr, server_port)) - cherrypy.config.update({ - 'server.socket_host': server_addr, - 'server.socket_port': server_port, + 'server.socket_host': self.server_addr, + 'server.socket_port': self.server_port, 'engine.autoreload.on': False, 'server.ssl_module': 'builtin', 'server.ssl_certificate': cert_fname, @@ -81,7 +78,7 @@ def run(self) -> None: 'tools.response_headers.on': True}} cherrypy.tree.mount(Root(self.mgr), '/', root_conf) self.mgr.log.info('Starting cherrypy engine...') - cherrypy.engine.start() + self.start_engine() self.mgr.log.info('Cherrypy engine started.') # wait for the shutdown event self.cherrypy_shutdown_event.wait() @@ -91,6 +88,25 @@ def run(self) -> None: except Exception as e: self.mgr.log.error(f'Failed to run cephadm cherrypy endpoint: {e}') + def start_engine(self) -> None: + port_connect_attempts = 0 + while port_connect_attempts < 150: + try: + cherrypy.engine.start() + self.mgr.log.info(f'Cephadm endpoint connected to port {self.server_port}') + return + except cherrypy.process.wspbus.ChannelFailures as e: + self.mgr.log.info( + f'{e}. Trying next port.') + self.server_port += 1 + cherrypy.server.httpserver = None + cherrypy.config.update({ + 'server.socket_port': self.server_port + }) + port_connect_attempts += 1 + self.mgr.log.error( + 'Cephadm Endpoint could not find free port in range 7150-7300 and failed to start') + def shutdown(self) -> None: self.mgr.log.info('Stopping cherrypy engine...') self.cherrypy_shutdown_event.set() diff --git a/src/pybind/mgr/cephadm/module.py b/src/pybind/mgr/cephadm/module.py index 5b82e625a2955..fc61ac3536e5f 100644 --- a/src/pybind/mgr/cephadm/module.py +++ b/src/pybind/mgr/cephadm/module.py @@ -349,12 +349,6 @@ class CephadmOrchestrator(orchestrator.Orchestrator, MgrModule, default=20, desc='How often agent on each host will try to gather and send metadata' ), - Option( - 'endpoint_port', - type='int', - default=8499, - desc='Which port cephadm http endpoint will listen on' - ), Option( 'agent_starting_port', type='int', @@ -429,7 +423,6 @@ def __init__(self, *args: Any, **kwargs: Any): self.ssh_pub: Optional[str] = None self.use_agent = False self.agent_refresh_rate = 0 - self.endpoint_port = 0 self.agent_starting_port = 0 self.apply_spec_fails: List[Tuple[str, str]] = [] self.max_osd_draining_count = 10 @@ -2219,11 +2212,13 @@ def _calc_daemon_deps(self, deps = [d.name() for d in daemons if d.daemon_type == 'haproxy'] elif daemon_type == 'agent': root_cert = '' + server_port = '' try: + server_port = str(self.cherrypy_thread.server_port) root_cert = self.cherrypy_thread.ssl_certs.get_root_cert() except Exception: pass - deps = sorted([self.get_mgr_ip(), str(self.endpoint_port), root_cert, + deps = sorted([self.get_mgr_ip(), server_port, root_cert, str(self.get_module_option('device_enhanced_scan'))]) elif daemon_type == 'iscsi': deps = [self.get_mgr_ip()] diff --git a/src/pybind/mgr/cephadm/services/cephadmservice.py b/src/pybind/mgr/cephadm/services/cephadmservice.py index 2dbfb67190761..42e7d55522a32 100644 --- a/src/pybind/mgr/cephadm/services/cephadmservice.py +++ b/src/pybind/mgr/cephadm/services/cephadmservice.py @@ -1022,19 +1022,21 @@ def prepare_create(self, daemon_spec: CephadmDaemonDeploySpec) -> CephadmDaemonD return daemon_spec def generate_config(self, daemon_spec: CephadmDaemonDeploySpec) -> Tuple[Dict[str, Any], List[str]]: - cfg = {'target_ip': self.mgr.get_mgr_ip(), - 'target_port': self.mgr.endpoint_port, - 'refresh_period': self.mgr.agent_refresh_rate, - 'listener_port': self.mgr.agent_starting_port, - 'host': daemon_spec.host, - 'device_enhanced_scan': str(self.mgr.get_module_option('device_enhanced_scan'))} - try: assert self.mgr.cherrypy_thread assert self.mgr.cherrypy_thread.ssl_certs.get_root_cert() + assert self.mgr.cherrypy_thread.server_port except Exception: raise OrchestratorError( 'Cannot deploy agent daemons until cephadm endpoint has finished generating certs') + + cfg = {'target_ip': self.mgr.get_mgr_ip(), + 'target_port': self.mgr.cherrypy_thread.server_port, + 'refresh_period': self.mgr.agent_refresh_rate, + 'listener_port': self.mgr.agent_starting_port, + 'host': daemon_spec.host, + 'device_enhanced_scan': str(self.mgr.get_module_option('device_enhanced_scan'))} + listener_cert, listener_key = self.mgr.cherrypy_thread.ssl_certs.generate_cert( self.mgr.inventory.get_addr(daemon_spec.host)) config = { @@ -1045,6 +1047,6 @@ def generate_config(self, daemon_spec: CephadmDaemonDeploySpec) -> Tuple[Dict[st 'listener.key': listener_key, } - return config, sorted([str(self.mgr.get_mgr_ip()), str(self.mgr.endpoint_port), + return config, sorted([str(self.mgr.get_mgr_ip()), str(self.mgr.cherrypy_thread.server_port), self.mgr.cherrypy_thread.ssl_certs.get_root_cert(), str(self.mgr.get_module_option('device_enhanced_scan'))]) From 1903ce28a979beba4d496cf139c2825a6d132e20 Mon Sep 17 00:00:00 2001 From: Greg Farnum Date: Mon, 1 Nov 2021 23:36:47 +0000 Subject: [PATCH 0462/2866] doc: fix up rbd snapshot docs around crash consistency Signed-off-by: Greg Farnum --- doc/rbd/rbd-snapshot.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/rbd/rbd-snapshot.rst b/doc/rbd/rbd-snapshot.rst index 701acec2f3ad1..96dec81c87c32 100644 --- a/doc/rbd/rbd-snapshot.rst +++ b/doc/rbd/rbd-snapshot.rst @@ -15,13 +15,13 @@ VM image) quickly and easily. Ceph block device snapshots are managed using the .. important:: To use RBD snapshots, you must have a running Ceph cluster. .. note:: Because RBD does not know about any filesystem within an image - (volume), snapshots are not `crash-consistent` unless they are + (volume), snapshots are only `crash-consistent` unless they are coordinated within the mounting (attaching) operating system. We therefore recommend that you pause or stop I/O before taking a snapshot. - If the volume contains a filesystem, it must be in an internally - consistent state before taking a snapshot. Snapshots taken at - inconsistent points may need a `fsck` pass before subsequent - mounting. To stop `I/O` you can use `fsfreeze` command. See + If the volume contains a filesystem, it should be in an internally + consistent state before taking a snapshot. Snapshots taken without + write quiescing may need an `fsck` pass before subsequent + mounting. To quiesce `I/O` you can use `fsfreeze` command. See `fsfreeze(8)` man page for more details. For virtual machines, `qemu-guest-agent` can be used to automatically freeze file systems when creating a snapshot. From bbd7f0dbafdc96899d33252443d168e2aecc697b Mon Sep 17 00:00:00 2001 From: Curt Bruns Date: Thu, 21 Oct 2021 17:53:31 -0400 Subject: [PATCH 0463/2866] rgw/lua: Example read/write of StorageClass field Admins may setup different pools for RGW objects and having the StorageClass field mutable allows the steering of RGW objects to the proper pools. This Lua example shows how a user can modify the StorageClass header when it is empty on a PUT request and steer objects to different pools based on size (Request.ContentLength). Signed-off-by: Curt Bruns --- examples/lua/storage_class.lua | 19 +++++++++++++ examples/lua/storage_class.md | 49 ++++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+) create mode 100644 examples/lua/storage_class.lua create mode 100644 examples/lua/storage_class.md diff --git a/examples/lua/storage_class.lua b/examples/lua/storage_class.lua new file mode 100644 index 0000000000000..08d41094c8c95 --- /dev/null +++ b/examples/lua/storage_class.lua @@ -0,0 +1,19 @@ +local function isempty(input) + return input == nil or input == '' +end + +if Request.RGWOp == 'put_obj' then + RGWDebugLog("Put_Obj with StorageClass: " .. Request.HTTP.StorageClass ) + if (isempty(Request.HTTP.StorageClass)) then + if (Request.ContentLength >= 65536) then + RGWDebugLog("No StorageClass for Object and size >= threshold: " .. Request.Object.Name .. " adding QLC StorageClass") + Request.HTTP.StorageClass = "QLC_CLASS" + else + RGWDebugLog("No StorageClass for Object and size < threshold: " .. Request.Object.Name .. " adding STANDARD StorageClass") + Request.HTTP.StorageClass = "STANDARD" + end + else + RGWDebugLog("Storage Class Header Present on Object: " .. Request.Object.Name .. " with StorageClass: " .. Request.HTTP.StorageClass) + end +end + diff --git a/examples/lua/storage_class.md b/examples/lua/storage_class.md new file mode 100644 index 0000000000000..8da92ccc38b46 --- /dev/null +++ b/examples/lua/storage_class.md @@ -0,0 +1,49 @@ +# Introduction + +This directory contains an example `storage_class.lua` on how to +use [Lua Scripting](https://docs.ceph.com/en/latest/radosgw/lua-scripting/) +to read and write the Storage Class field of a put request. + +## Usage - following examples based on vstart environment built in ceph/build and commands invoked from ceph/build + +* Create Zonegroup placement info for a Storage Class (QLC_CLASS in this example) and point class to a data pool (qlc_pool in this example) +NOTE: RGW will need restarted due to the Zonegroup placement info change. +See: https://docs.ceph.com/en/latest/radosgw/placement/#zonegroup-zone-configuration for more information. + +```bash +# Create Storage Class +./bin/radosgw-admin zonegroup placement add --rgw-zonegroup default --placement-id default-placement --storage-class QLC_CLASS +# Steer objects in QLC_CLASS to the qlc_pool data pool +./bin/radosgw-admin zone placement add --rgw-zone default --placement-id default-placement --storage-class QLC_CLASS --data-pool qlc_pool +``` +* Restart radosgw for Zone/ZoneGroup placement changes to take effect. + +* Upload the script: + +```bash +./bin/radosgw-admin script put --infile=storage_class.lua --context=preRequest +``` + +* Create a bucket and put and object with a Storage Class header (no modification will occur): +```bash +aws --profile=ceph --endpoint=http://localhost:8000 s3api create-bucket --bucket test-bucket +aws --profile=ceph --endpoint=http://localhost:8000 s3api put-object --bucket test-bucket --key truv-0 --body ./64KiB_object.bin --storage-class STANDARD +``` + +* Send a request without a Storage Class header (Storage Class will be changed to QLC_CLASS by Lua script): +```bash +aws --profile=ceph --endpoint=http://localhost:8000 s3api put-object --bucket test-bucket --key truv-0 --body ./64KiB_object.bin +``` +NOTE: If you use s3cmd instead of aws command-line, s3cmd adds "STANDARD" StorageClass to any put request so the example Lua script will not modify it. + +* Verify S3 object had its StorageClass header added +```bash +grep Lua ceph/build/out/radosgw.8000.log + +2021-11-01T17:10:14.048-0400 7f9c7f697640 20 Lua INFO: Put_Obj with StorageClass: +2021-11-01T17:10:14.048-0400 7f9c7f697640 20 Lua INFO: No StorageClass for Object and size >= threshold: truv-0 adding QLC StorageClass +``` + +## Requirements +* Lua 5.3 + From 05bb4e3f023570ec9c3479b17fe89e737fa984f4 Mon Sep 17 00:00:00 2001 From: Laura Flores Date: Tue, 2 Nov 2021 00:31:16 +0000 Subject: [PATCH 0464/2866] doc/dev/developer_guide/testing_integration_tests: update "frequently used options" The `subset` option is important in Teuthology runs for reducing the number of tests that are triggered. This option is outlined in another part of the Teuthology documentation, but I think it's important to place here as well. Also, -n (for how many times the job will run) is incorrect; it should be -N. Signed-off-by: Laura Flores --- .../tests-integration-testing-teuthology-workflow.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/dev/developer_guide/testing_integration_tests/tests-integration-testing-teuthology-workflow.rst b/doc/dev/developer_guide/testing_integration_tests/tests-integration-testing-teuthology-workflow.rst index 5e204ace6509b..a10b6970b05cc 100644 --- a/doc/dev/developer_guide/testing_integration_tests/tests-integration-testing-teuthology-workflow.rst +++ b/doc/dev/developer_guide/testing_integration_tests/tests-integration-testing-teuthology-workflow.rst @@ -109,9 +109,9 @@ teuthology. This procedure explains how to run tests using teuthology. Other frequently used/useful options are ``-d`` (or ``--distro``), ``--distroversion``, ``--filter-out``, ``--timeout``, ``flavor``, ``-rerun``, -``-l`` (for limiting number of jobs) , ``-n`` (for how many times the job will -run). Run ``teuthology-suite --help`` to read descriptions of these and other -options. +``-l`` (for limiting number of jobs) , ``-N`` (for how many times the job will +run), and --subset (used to reduce the number of tests that are triggered). Run +``teuthology-suite --help`` to read descriptions of these and other options. .. _teuthology_testing_qa_changes: From 5a254a2701ebdb90aae40cb2c78a6e76739ccd9d Mon Sep 17 00:00:00 2001 From: Xiubo Li Date: Mon, 1 Nov 2021 10:57:16 +0800 Subject: [PATCH 0465/2866] client: remove optional for dirfd parameter All the callers when calling the create_and_open() there will always be with a dirfd. I beleive this was introduced when coding previous patches temporarily, but forgot to remove or fix it when pushing it. Signed-off-by: Xiubo Li --- src/client/Client.cc | 12 +++++------- src/client/Client.h | 6 +++--- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/src/client/Client.cc b/src/client/Client.cc index 32183de7d03f1..94acb461df6f0 100644 --- a/src/client/Client.cc +++ b/src/client/Client.cc @@ -9121,7 +9121,7 @@ int Client::getdir(const char *relpath, list& contents, /****** file i/o **********/ // common parts for open and openat. call with client_lock locked. -int Client::create_and_open(std::optional dirfd, const char *relpath, int flags, +int Client::create_and_open(int dirfd, const char *relpath, int flags, const UserPerm& perms, mode_t mode, int stripe_unit, int stripe_count, int object_size, const char *data_pool, std::string alternate_name) { @@ -9147,14 +9147,12 @@ int Client::create_and_open(std::optional dirfd, const char *relpath, int f int mask = ceph_caps_for_mode(ceph_flags_to_mode(cflags)); InodeRef dirinode = nullptr; - if (dirfd) { - int r = get_fd_inode(*dirfd, &dirinode); - if (r < 0) { - return r; - } + int r = get_fd_inode(dirfd, &dirinode); + if (r < 0) { + return r; } - int r = path_walk(path, &in, perms, followsym, mask, dirinode); + r = path_walk(path, &in, perms, followsym, mask, dirinode); if (r == 0 && (flags & O_CREAT) && (flags & O_EXCL)) return -CEPHFS_EEXIST; diff --git a/src/client/Client.h b/src/client/Client.h index b6a76faca5083..66ec9c7a49191 100644 --- a/src/client/Client.h +++ b/src/client/Client.h @@ -437,9 +437,9 @@ class Client : public Dispatcher, public md_config_obs_t { // file ops int mknod(const char *path, mode_t mode, const UserPerm& perms, dev_t rdev=0); - int create_and_open(std::optional dirfd, const char *relpath, int flags, const UserPerm& perms, - mode_t mode, int stripe_unit, int stripe_count, int object_size, const char *data_pool, - std::string alternate_name); + int create_and_open(int dirfd, const char *relpath, int flags, const UserPerm& perms, + mode_t mode, int stripe_unit, int stripe_count, int object_size, + const char *data_pool, std::string alternate_name); int open(const char *path, int flags, const UserPerm& perms, mode_t mode=0, std::string alternate_name="") { return open(path, flags, perms, mode, 0, 0, 0, NULL, alternate_name); } From 2ec6212f1587aeedb759f313128e012b182834e3 Mon Sep 17 00:00:00 2001 From: Xiubo Li Date: Mon, 1 Nov 2021 14:08:04 +0800 Subject: [PATCH 0466/2866] client: remove usless _openat() The _openat() is never used and I believe this also was introduced when coding previous interim patches and forgot to remove it. Signed-off-by: Xiubo Li --- src/client/Client.cc | 5 ----- src/client/Client.h | 2 -- 2 files changed, 7 deletions(-) diff --git a/src/client/Client.cc b/src/client/Client.cc index 94acb461df6f0..52a18d1199a3a 100644 --- a/src/client/Client.cc +++ b/src/client/Client.cc @@ -9216,11 +9216,6 @@ int Client::open(const char *relpath, int flags, const UserPerm& perms, stripe_count, object_size, data_pool, alternate_name); } -int Client::_openat(int dirfd, const char *relpath, int flags, const UserPerm& perms, - mode_t mode, std::string alternate_name) { - return create_and_open(dirfd, relpath, flags, perms, mode, 0, 0, 0, NULL, alternate_name); -} - int Client::openat(int dirfd, const char *relpath, int flags, const UserPerm& perms, mode_t mode, int stripe_unit, int stripe_count, int object_size, const char *data_pool, std::string alternate_name) { diff --git a/src/client/Client.h b/src/client/Client.h index 66ec9c7a49191..bbdf7b53c8ea9 100644 --- a/src/client/Client.h +++ b/src/client/Client.h @@ -446,8 +446,6 @@ class Client : public Dispatcher, public md_config_obs_t { int open(const char *path, int flags, const UserPerm& perms, mode_t mode, int stripe_unit, int stripe_count, int object_size, const char *data_pool, std::string alternate_name=""); - int _openat(int dirfd, const char *relpath, int flags, const UserPerm& perms, - mode_t mode=0, std::string alternate_name=""); int openat(int dirfd, const char *relpath, int flags, const UserPerm& perms, mode_t mode, int stripe_unit, int stripe_count, int object_size, const char *data_pool, std::string alternate_name); From 2d64232884ad3e1265d245b4f07f60f73693c914 Mon Sep 17 00:00:00 2001 From: wangyunqing Date: Wed, 22 Sep 2021 11:05:40 +0800 Subject: [PATCH 0467/2866] doc/radosgw/s3-notification-compatibility.rst: fix typos Signed-off-by: wangyunqing --- doc/radosgw/s3-notification-compatibility.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/radosgw/s3-notification-compatibility.rst b/doc/radosgw/s3-notification-compatibility.rst index 09054bed39245..bb2cfb31393b3 100644 --- a/doc/radosgw/s3-notification-compatibility.rst +++ b/doc/radosgw/s3-notification-compatibility.rst @@ -13,7 +13,7 @@ Supported Destination --------------------- AWS supports: **SNS**, **SQS** and **Lambda** as possible destinations (AWS internal destinations). -Currently, we support: **HTTP/S**, **Kafka** and **AMQP**. And also support pulling and acking of events stored in Ceph (as an intrenal destination). +Currently, we support: **HTTP/S**, **Kafka** and **AMQP**. And also support pulling and acking of events stored in Ceph (as an internal destination). We are using the **SNS** ARNs to represent the **HTTP/S**, **Kafka** and **AMQP** destinations. @@ -128,7 +128,7 @@ We also have the following extensions to topic configuration: - In ``CreateTopic`` - we allow setting endpoint attributes - - we allow setting opaque data thta will be sent to the endpoint in the notification + - we allow setting opaque data that will be sent to the endpoint in the notification .. _AWS Simple Notification Service API: https://docs.aws.amazon.com/sns/latest/api/API_Operations.html From 94f9873718a82d2def8f268c1581fbf97fee0f49 Mon Sep 17 00:00:00 2001 From: Yin Congmin Date: Wed, 15 Sep 2021 11:23:43 +0000 Subject: [PATCH 0468/2866] librbd/cache/pwl: fix assert in _aio_stop() during shutdown For wait_for_ops(next_ctx). this next_ctx may run in aio_thread. Then the next program runs on the aio thread. remove_pool_file() calls bdev->close(), then calles _aio_stop(), exec aio_thread.join(), cause assert. Thread can't join itself. Fix it by adding close ctx to m_work_queue, so close() can run in work queue thread. At the same time, correct the order of wait_for_ops(). flush_dirty_entries(next_ctx) may call wake_up() and start_op(). so moving wait_for_ops() behind flush_dirty_entries(next_ctx) is more appropriate. Fixes: https://tracker.ceph.com/issues/52566 Signed-off-by: Yin Congmin --- src/librbd/cache/pwl/AbstractWriteLog.cc | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/librbd/cache/pwl/AbstractWriteLog.cc b/src/librbd/cache/pwl/AbstractWriteLog.cc index 5b7799e664b59..9f4e567bb1382 100644 --- a/src/librbd/cache/pwl/AbstractWriteLog.cc +++ b/src/librbd/cache/pwl/AbstractWriteLog.cc @@ -637,6 +637,14 @@ void AbstractWriteLog::shut_down(Context *on_finish) { } update_image_cache_state(next_ctx); }); + ctx = new LambdaContext( + [this, ctx](int r) { + Context *next_ctx = override_ctx(r, ctx); + ldout(m_image_ctx.cct, 6) << "waiting for in flight operations" << dendl; + // Wait for in progress IOs to complete + next_ctx = util::create_async_context_callback(&m_work_queue, next_ctx); + m_async_op_tracker.wait_for_ops(next_ctx); + }); ctx = new LambdaContext( [this, ctx](int r) { Context *next_ctx = override_ctx(r, ctx); @@ -653,14 +661,6 @@ void AbstractWriteLog::shut_down(Context *on_finish) { } flush_dirty_entries(next_ctx); }); - ctx = new LambdaContext( - [this, ctx](int r) { - Context *next_ctx = override_ctx(r, ctx); - ldout(m_image_ctx.cct, 6) << "waiting for in flight operations" << dendl; - // Wait for in progress IOs to complete - next_ctx = util::create_async_context_callback(m_image_ctx, next_ctx); - m_async_op_tracker.wait_for_ops(next_ctx); - }); ctx = new LambdaContext( [this, ctx](int r) { ldout(m_image_ctx.cct, 6) << "Done internal_flush in shutdown" << dendl; From 8795d33185a2aaafd099e497393dc2fcd8154318 Mon Sep 17 00:00:00 2001 From: Xiubo Li Date: Tue, 2 Nov 2021 11:06:11 +0800 Subject: [PATCH 0469/2866] qa: increase the timeout value to wait a litte longer Sometimes the OpenFileTable::commit() will just come after the 30 seconds' waiting. Fixes: https://tracker.ceph.com/issues/52887 Signed-off-by: Xiubo Li --- qa/tasks/cephfs/test_openfiletable.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/qa/tasks/cephfs/test_openfiletable.py b/qa/tasks/cephfs/test_openfiletable.py index 6df29762a2b0a..eff6b50932e4a 100644 --- a/qa/tasks/cephfs/test_openfiletable.py +++ b/qa/tasks/cephfs/test_openfiletable.py @@ -62,7 +62,7 @@ def test_perf_counters(self): # Open the file p = self.mount_a.open_background("omap_counter_test_file") - self.wait_until_true(lambda: self._check_oft_counter('omap_total_updates', 2), timeout=30) + self.wait_until_true(lambda: self._check_oft_counter('omap_total_updates', 2), timeout=120) perf_dump = self.fs.mds_asok(['perf', 'dump']) omap_total_updates_1 = perf_dump['oft']['omap_total_updates'] @@ -73,8 +73,8 @@ def test_perf_counters(self): # Now close the file self.mount_a.kill_background(p) # Ensure that the file does not exist any more - self.wait_until_true(lambda: self._check_oft_counter('omap_total_removes', 1), timeout=30) - self.wait_until_true(lambda: self._check_oft_counter('omap_total_kv_pairs', 1), timeout=30) + self.wait_until_true(lambda: self._check_oft_counter('omap_total_removes', 1), timeout=120) + self.wait_until_true(lambda: self._check_oft_counter('omap_total_kv_pairs', 1), timeout=120) perf_dump = self.fs.mds_asok(['perf', 'dump']) omap_total_removes = perf_dump['oft']['omap_total_removes'] From c091ec3471973f41717b237d500b8e4374af660f Mon Sep 17 00:00:00 2001 From: Yin Congmin Date: Wed, 30 Jun 2021 19:10:30 +0800 Subject: [PATCH 0470/2866] librbd/cache/pwl/ssd: make log entry pointers 64 bit (on-disk format change) Fixes: https://tracker.ceph.com/issues/50675 Signed-off-by: Yin Congmin --- src/librbd/cache/pwl/AbstractWriteLog.cc | 2 +- src/librbd/cache/pwl/AbstractWriteLog.h | 2 +- src/librbd/cache/pwl/LogEntry.h | 2 +- src/librbd/cache/pwl/Types.h | 4 ++-- src/librbd/cache/pwl/ssd/WriteLog.cc | 4 ++-- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/librbd/cache/pwl/AbstractWriteLog.cc b/src/librbd/cache/pwl/AbstractWriteLog.cc index 421d5dd8bc49e..861469575e95d 100644 --- a/src/librbd/cache/pwl/AbstractWriteLog.cc +++ b/src/librbd/cache/pwl/AbstractWriteLog.cc @@ -344,7 +344,7 @@ template void AbstractWriteLog::update_entries(std::shared_ptr *log_entry, WriteLogCacheEntry *cache_entry, std::map &missing_sync_points, std::map> &sync_point_entries, - int entry_index) { + uint64_t entry_index) { bool writer = cache_entry->is_writer(); if (cache_entry->is_sync_point()) { ldout(m_image_ctx.cct, 20) << "Entry " << entry_index diff --git a/src/librbd/cache/pwl/AbstractWriteLog.h b/src/librbd/cache/pwl/AbstractWriteLog.h index 128c00d99d82a..83cbc11d4ed7f 100644 --- a/src/librbd/cache/pwl/AbstractWriteLog.h +++ b/src/librbd/cache/pwl/AbstractWriteLog.h @@ -336,7 +336,7 @@ class AbstractWriteLog { std::map &missing_sync_points, std::map> &sync_point_entries, - int entry_index); + uint64_t entry_index); void update_sync_points( std::map &missing_sync_points, std::map bool WriteLog::retire_entries(const unsigned long int frees_per_tx) { CephContext *cct = m_image_ctx.cct; GenericLogEntriesVector retiring_entries; - uint32_t initial_first_valid_entry; - uint32_t first_valid_entry; + uint64_t initial_first_valid_entry; + uint64_t first_valid_entry; std::lock_guard retire_locker(this->m_log_retire_lock); ldout(cct, 20) << "Look for entries to retire" << dendl; From 3fead055717ec0ec5576caa589ff5aa32a05bfb9 Mon Sep 17 00:00:00 2001 From: Huber-ming Date: Tue, 2 Nov 2021 16:35:51 +0800 Subject: [PATCH 0471/2866] rgw: delete abbreviation for option "--new-uid" Signed-off-by: Huber-ming --- src/rgw/rgw_admin.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc index 295ae5a2430e2..a5ca10547494f 100644 --- a/src/rgw/rgw_admin.cc +++ b/src/rgw/rgw_admin.cc @@ -3391,7 +3391,7 @@ int main(int argc, const char **argv) cerr << "no value for uid" << std::endl; exit(1); } - } else if (ceph_argparse_witharg(args, i, &val, "-i", "--new-uid", (char*)NULL)) { + } else if (ceph_argparse_witharg(args, i, &val, "--new-uid", (char*)NULL)) { new_user_id.from_str(val); } else if (ceph_argparse_witharg(args, i, &val, "--tenant", (char*)NULL)) { tenant = val; From da3f7c30441e9475e7ec3182823e2a820dc0716b Mon Sep 17 00:00:00 2001 From: Huber-ming Date: Tue, 2 Nov 2021 16:51:12 +0800 Subject: [PATCH 0472/2866] rgw: delete the abbreviation of radosgw-admin's option "--infile" Signed-off-by: Huber-ming --- src/rgw/rgw_admin.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc index 295ae5a2430e2..206783b276ee3 100644 --- a/src/rgw/rgw_admin.cc +++ b/src/rgw/rgw_admin.cc @@ -3595,7 +3595,7 @@ int main(int argc, const char **argv) // do nothing } else if (ceph_argparse_witharg(args, i, &val, "--caps", (char*)NULL)) { caps = val; - } else if (ceph_argparse_witharg(args, i, &val, "-i", "--infile", (char*)NULL)) { + } else if (ceph_argparse_witharg(args, i, &val, "--infile", (char*)NULL)) { infile = val; } else if (ceph_argparse_witharg(args, i, &val, "--metadata-key", (char*)NULL)) { metadata_key = val; From fb319b0079ffd7085b784efa61e29f9b9e4a327e Mon Sep 17 00:00:00 2001 From: Igor Fedotov Date: Tue, 28 Sep 2021 19:49:45 +0300 Subject: [PATCH 0473/2866] os/bluestore: cleanup onode/collection cache after NCB recovery. One needs to properly shutdown Onode cache if NCB stuff has performed full recovery. Signed-off-by: Igor Fedotov --- src/os/bluestore/BlueStore.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc index bab1b77e310a3..7a421c59f1bad 100644 --- a/src/os/bluestore/BlueStore.cc +++ b/src/os/bluestore/BlueStore.cc @@ -17825,6 +17825,9 @@ int BlueStore::read_allocation_from_drive_on_startup() if (ret < 0) { return ret; } + auto shutdown_cache = make_scope_guard([&] { + _shutdown_cache(); + }); read_alloc_stats_t stats = {}; utime_t start = ceph_clock_now(); From efb67445c2480fce4b5a7252811a544b8a74772f Mon Sep 17 00:00:00 2001 From: Igor Fedotov Date: Tue, 28 Sep 2021 19:52:37 +0300 Subject: [PATCH 0474/2866] qa/osd-bluefs-volume-ops: retry data writing if spillover hasn't happened. Fixes: https://tracker.ceph.com/issues/52676 Signed-off-by: Igor Fedotov --- qa/standalone/osd/osd-bluefs-volume-ops.sh | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/qa/standalone/osd/osd-bluefs-volume-ops.sh b/qa/standalone/osd/osd-bluefs-volume-ops.sh index 3e92d17dba3ee..c0235ef6772e5 100755 --- a/qa/standalone/osd/osd-bluefs-volume-ops.sh +++ b/qa/standalone/osd/osd-bluefs-volume-ops.sh @@ -365,13 +365,19 @@ function TEST_bluestore2() { sleep 5 create_pool foo 16 - # write some objects - timeout 60 rados bench -p foo 10 write --write-omap --no-cleanup #|| return 1 - - #give RocksDB some time to cooldown and put files to slow level(s) - sleep 10 - - spilled_over=$( ceph tell osd.0 perf dump bluefs | jq ".bluefs.slow_used_bytes" ) + retry = 0 + while [[ $retry -le 5 ]]; do + # write some objects + timeout 60 rados bench -p foo 10 write --write-omap --no-cleanup #|| return 1 + + #give RocksDB some time to cooldown and put files to slow level(s) + sleep 10 + + db_used=$( ceph tell osd.0 perf dump bluefs | jq ".bluefs.db_used_bytes" ) + spilled_over=$( ceph tell osd.0 perf dump bluefs | jq ".bluefs.slow_used_bytes" ) + ((retry+=1)) + test $spilled_over -eq 0 || break + done test $spilled_over -gt 0 || return 1 while kill $osd_pid0; do sleep 1 ; done From 4ac06bc9befb476cbc0350a0e1d40a9cf25af198 Mon Sep 17 00:00:00 2001 From: Sebastian Wagner Date: Mon, 1 Nov 2021 22:37:55 +0100 Subject: [PATCH 0475/2866] doc/cephadm: Calling miscellaneous ceph tools Signed-off-by: Sebastian Wagner --- doc/cephadm/troubleshooting.rst | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/doc/cephadm/troubleshooting.rst b/doc/cephadm/troubleshooting.rst index 951526db83c35..4ce76ee8166af 100644 --- a/doc/cephadm/troubleshooting.rst +++ b/doc/cephadm/troubleshooting.rst @@ -252,6 +252,28 @@ To access the admin socket, first enter the daemon container on the host:: [root@mon1 ~]# cephadm enter --name [ceph: root@mon1 /]# ceph --admin-daemon /var/run/ceph/ceph-.asok config show +Calling miscellaneous ceph tools +-------------------------------- + +To call miscellaneous like ``ceph-objectstore-tool`` or +``ceph-monstore-tool``, you can run them by calling +``cephadm shell --name `` like so:: + + root@myhostname # cephadm unit --name mon.myhostname stop + root@myhostname # cephadm shell --name mon.myhostname + [ceph: root@myhostname /]# ceph-monstore-tool /var/lib/ceph/mon/ceph-myhostname get monmap > monmap + [ceph: root@myhostname /]# monmaptool --print monmap + monmaptool: monmap file monmap + epoch 1 + fsid 28596f44-3b56-11ec-9034-482ae35a5fbb + last_changed 2021-11-01T20:57:19.755111+0000 + created 2021-11-01T20:57:19.755111+0000 + min_mon_release 17 (quincy) + election_strategy: 1 + 0: [v2:127.0.0.1:3300/0,v1:127.0.0.1:6789/0] mon.myhostname + +This command sets up the environment in a way that is suitable +for extended daemon maintenance and running the deamon interactively. .. _cephadm-restore-quorum: From 1ce1b3b8ea8ef6b99abe8c14d69d83a47cbaf762 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Tue, 3 Aug 2021 14:34:54 -0400 Subject: [PATCH 0476/2866] ceph-volume: raw activate: accept --osd-id and/or --osd-uuid instead of device This makes it possible to start raw osds based on their uuid/id instead of device name (which may not be stable). Signed-off-by: Sage Weil --- .../ceph_volume/devices/raw/activate.py | 38 ++++++++++++++----- 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/src/ceph-volume/ceph_volume/devices/raw/activate.py b/src/ceph-volume/ceph_volume/devices/raw/activate.py index 94f54f1a20b31..c824620db4281 100644 --- a/src/ceph-volume/ceph_volume/devices/raw/activate.py +++ b/src/ceph-volume/ceph_volume/devices/raw/activate.py @@ -64,21 +64,32 @@ def __init__(self, argv): self.args = None @decorators.needs_root - def activate(self, devices, tmpfs, systemd, block_wal, block_db): + def activate(self, device, start_osd_id, start_osd_uuid, + tmpfs, systemd, block_wal, block_db): """ :param args: The parsed arguments coming from the CLI """ - assert devices - found = direct_report(devices) - - for osd_id, meta in found.items(): + assert device or start_osd_id or start_osd_uuid + found = direct_report(device) + + activated_any = False + for osd_uuid, meta in found.items(): + osd_id = meta['osd_id'] + if start_osd_id is not None and str(osd_id) != str(start_osd_id): + continue + if start_osd_uuid is not None and osd_uuid != start_osd_uuid: + continue logger.info('Activating osd.%s uuid %s cluster %s' % ( - osd_id, meta['osd_uuid'], meta['ceph_fsid'])) + osd_id, osd_uuid, meta['ceph_fsid'])) activate_bluestore(meta, tmpfs=tmpfs, systemd=systemd, block_wal=block_wal, block_db=block_db) + activated_any = True + + if not activated_any: + raise RuntimeError('did not find any matching OSD to activate') def main(self): sub_command_help = dedent(""" @@ -97,9 +108,16 @@ def main(self): ) parser.add_argument( '--device', - nargs='+', help='The device for the OSD to start' - ) + ) + parser.add_argument( + '--osd-id', + help='OSD ID to activate' + ) + parser.add_argument( + '--osd-uuid', + help='OSD UUID to active' + ) parser.add_argument( '--no-systemd', dest='no_systemd', @@ -130,7 +148,9 @@ def main(self): if not args.no_systemd: terminal.error('systemd support not yet implemented') raise SystemExit(1) - self.activate(args.device, + self.activate(device=args.device, + start_osd_id=args.osd_id, + start_osd_uuid=args.osd_uuid, tmpfs=not args.no_tmpfs, systemd=not self.args.no_systemd, block_wal=self.args.block_wal, From 451feec4b269e2a5816687136adc74082ec8f2f3 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Thu, 5 Aug 2021 12:02:22 -0400 Subject: [PATCH 0477/2866] ceph-volume: lvm activate: infer bluestore or filestore No need to require --filestore and/or --bluestore args since we can tell from the LV tags which one it is. We can't drop the arguments without breaking existing users, though, so redefine them to mean *force* bluesetore or filestore activation (even though this will error out if the tags don't match). Signed-off-by: Sage Weil --- .../ceph_volume/devices/lvm/activate.py | 18 ++++++++++-------- .../tests/devices/lvm/test_activate.py | 2 +- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/src/ceph-volume/ceph_volume/devices/lvm/activate.py b/src/ceph-volume/ceph_volume/devices/lvm/activate.py index 1069839c04eb4..e4b23b62cadad 100644 --- a/src/ceph-volume/ceph_volume/devices/lvm/activate.py +++ b/src/ceph-volume/ceph_volume/devices/lvm/activate.py @@ -291,9 +291,15 @@ def activate(self, args, osd_id=None, osd_fsid=None): 'assuming bluestore') return activate_bluestore(lvs, args.no_systemd) - if args.bluestore: + + # explicit filestore/bluestore flags take precedence + if getattr(args, 'bluestore', False): + activate_bluestore(lvs, args.no_systemd) + elif getattr(args, 'filestore', False): + activate_filestore(lvs, args.no_systemd) + elif any('ceph.block_device' in lv.tags for lv in lvs): activate_bluestore(lvs, args.no_systemd) - elif args.filestore: + elif any('ceph.data_device' in lv.tags for lv in lvs): activate_filestore(lvs, args.no_systemd) def main(self): @@ -338,12 +344,12 @@ def main(self): parser.add_argument( '--bluestore', action='store_true', - help='bluestore objectstore (default)', + help='force bluestore objectstore activation', ) parser.add_argument( '--filestore', action='store_true', - help='filestore objectstore', + help='force filestore objectstore activation', ) parser.add_argument( '--all', @@ -361,10 +367,6 @@ def main(self): print(sub_command_help) return args = parser.parse_args(self.argv) - # Default to bluestore here since defaulting it in add_argument may - # cause both to be True - if not args.bluestore and not args.filestore: - args.bluestore = True if args.activate_all: self.activate_all(args) else: diff --git a/src/ceph-volume/ceph_volume/tests/devices/lvm/test_activate.py b/src/ceph-volume/ceph_volume/tests/devices/lvm/test_activate.py index 9b8fcbebeaef9..feafea71be056 100644 --- a/src/ceph-volume/ceph_volume/tests/devices/lvm/test_activate.py +++ b/src/ceph-volume/ceph_volume/tests/devices/lvm/test_activate.py @@ -315,7 +315,7 @@ def test_default_objectstore(self, capture): activation.main() parsed_args = capture.calls[0]['args'][0] assert parsed_args.filestore is False - assert parsed_args.bluestore is True + assert parsed_args.bluestore is False def test_uses_filestore(self, capture): args = ['--filestore', '0', 'asdf-ljh-asdf'] From 9dc35338754442b0730b974de4fc3cc6ffb172b6 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Thu, 5 Aug 2021 13:23:27 -0400 Subject: [PATCH 0478/2866] ceph-volume: lvm activate: add --no-tmpfs This isn't necessary for cephadm, but having this arg match raw activate makes the interface more consistent. Signed-off-by: Sage Weil --- src/ceph-volume/ceph_volume/devices/lvm/activate.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/ceph-volume/ceph_volume/devices/lvm/activate.py b/src/ceph-volume/ceph_volume/devices/lvm/activate.py index e4b23b62cadad..bd45f32a2e0b9 100644 --- a/src/ceph-volume/ceph_volume/devices/lvm/activate.py +++ b/src/ceph-volume/ceph_volume/devices/lvm/activate.py @@ -140,7 +140,7 @@ def get_osd_device_path(osd_lvs, device_type, dmcrypt_secret=None): raise RuntimeError('could not find %s with uuid %s' % (device_type, device_uuid)) -def activate_bluestore(osd_lvs, no_systemd=False): +def activate_bluestore(osd_lvs, no_systemd=False, no_tmpfs=False): for lv in osd_lvs: if lv.tags.get('ceph.type') == 'block': osd_block_lv = lv @@ -158,7 +158,7 @@ def activate_bluestore(osd_lvs, no_systemd=False): osd_path = '/var/lib/ceph/osd/%s-%s' % (conf.cluster, osd_id) if not system.path_is_mounted(osd_path): # mkdir -p and mount as tmpfs - prepare_utils.create_osd_path(osd_id, tmpfs=True) + prepare_utils.create_osd_path(osd_id, tmpfs=not no_tmpfs) # XXX This needs to be removed once ceph-bluestore-tool can deal with # symlinks that exist in the osd dir for link_name in ['block', 'block.db', 'block.wal']: @@ -294,11 +294,11 @@ def activate(self, args, osd_id=None, osd_fsid=None): # explicit filestore/bluestore flags take precedence if getattr(args, 'bluestore', False): - activate_bluestore(lvs, args.no_systemd) + activate_bluestore(lvs, args.no_systemd, getattr(args, 'no_tmpfs', False)) elif getattr(args, 'filestore', False): activate_filestore(lvs, args.no_systemd) elif any('ceph.block_device' in lv.tags for lv in lvs): - activate_bluestore(lvs, args.no_systemd) + activate_bluestore(lvs, args.no_systemd, getattr(args, 'no_tmpfs', False)) elif any('ceph.data_device' in lv.tags for lv in lvs): activate_filestore(lvs, args.no_systemd) @@ -363,6 +363,11 @@ def main(self): action='store_true', help='Skip creating and enabling systemd units and starting OSD services', ) + parser.add_argument( + '--no-tmpfs', + action='store_true', + help='Do not use a tmpfs mount for OSD data dir' + ) if len(self.argv) == 0: print(sub_command_help) return From 3d7ceec684b0ac5b83fae4c397b134236fac485e Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Thu, 5 Aug 2021 13:29:17 -0400 Subject: [PATCH 0479/2866] ceph-volume: top-level 'activate' command First try raw, then lvm. Signed-off-by: Sage Weil --- .../ceph_volume/activate/__init__.py | 1 + src/ceph-volume/ceph_volume/activate/main.py | 73 +++++++++++++++++++ src/ceph-volume/ceph_volume/main.py | 3 +- 3 files changed, 76 insertions(+), 1 deletion(-) create mode 100644 src/ceph-volume/ceph_volume/activate/__init__.py create mode 100644 src/ceph-volume/ceph_volume/activate/main.py diff --git a/src/ceph-volume/ceph_volume/activate/__init__.py b/src/ceph-volume/ceph_volume/activate/__init__.py new file mode 100644 index 0000000000000..542bf32bdfaf2 --- /dev/null +++ b/src/ceph-volume/ceph_volume/activate/__init__.py @@ -0,0 +1 @@ +from .main import Activate # noqa diff --git a/src/ceph-volume/ceph_volume/activate/main.py b/src/ceph-volume/ceph_volume/activate/main.py new file mode 100644 index 0000000000000..40690c5d97603 --- /dev/null +++ b/src/ceph-volume/ceph_volume/activate/main.py @@ -0,0 +1,73 @@ +# -*- coding: utf-8 -*- + +import argparse + +from ceph_volume import terminal +from ceph_volume.devices.lvm.activate import Activate as LVMActivate +from ceph_volume.devices.raw.activate import Activate as RAWActivate + + +class Activate(object): + + help = "Activate an OSD" + + def __init__(self, argv): + self.argv = argv + + def main(self): + parser = argparse.ArgumentParser( + prog='ceph-volume activate', + formatter_class=argparse.RawDescriptionHelpFormatter, + description=self.help, + ) + parser.add_argument( + '--osd-id', + help='OSD ID to activate' + ) + parser.add_argument( + '--osd-uuid', + help='OSD UUID to active' + ) + parser.add_argument( + '--no-systemd', + dest='no_systemd', + action='store_true', + help='Skip creating and enabling systemd units and starting OSD services' + ) + parser.add_argument( + '--no-tmpfs', + action='store_true', + help='Do not use a tmpfs mount for OSD data dir' + ) + self.args = parser.parse_args(self.argv) + + # first try raw + try: + RAWActivate([]).activate( + device=None, + start_osd_id=self.args.osd_id, + start_osd_uuid=self.args.osd_uuid, + tmpfs=not self.args.no_tmpfs, + systemd=not self.args.no_systemd, + block_wal=None, + block_db=None, + ) + return + except Exception as e: + terminal.info(f'Failed to activate via raw: {e}') + + # then try lvm + try: + LVMActivate([]).activate( + argparse.Namespace( + osd_id=self.args.osd_id, + osd_fsid=self.args.osd_uuid, + no_tmpfs=self.args.no_tmpfs, + no_systemd=self.args.no_systemd, + ) + ) + return + except Exception as e: + terminal.info(f'Failed to activate via lvm: {e}') + + terminal.error('Failed to activate any OSD(s)') diff --git a/src/ceph-volume/ceph_volume/main.py b/src/ceph-volume/ceph_volume/main.py index a66b7bb7df961..652b0f9c87d13 100644 --- a/src/ceph-volume/ceph_volume/main.py +++ b/src/ceph-volume/ceph_volume/main.py @@ -6,7 +6,7 @@ import logging from ceph_volume.decorators import catches -from ceph_volume import log, devices, configuration, conf, exceptions, terminal, inventory, drive_group +from ceph_volume import log, devices, configuration, conf, exceptions, terminal, inventory, drive_group, activate class Volume(object): @@ -29,6 +29,7 @@ def __init__(self, argv=None, parse=True): 'simple': devices.simple.Simple, 'raw': devices.raw.Raw, 'inventory': inventory.Inventory, + 'activate': activate.Activate, 'drive-group': drive_group.Deploy, } self.plugin_help = "No plugins found/loaded" From 4e9c3288720836a5c7dd23269dd9919c3bc2c1cc Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Thu, 5 Aug 2021 13:32:00 -0400 Subject: [PATCH 0480/2866] cephadm: use generic ceph-volume activate This allows us to activate raw or LVM osds. (In fact, LVM osds often activate via the raw method because the LVs are already available.) Signed-off-by: Sage Weil --- src/cephadm/cephadm | 35 +++++++++++++++++++++++++++++++---- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/src/cephadm/cephadm b/src/cephadm/cephadm index 87c69987f5e69..063fc88418d0c 100755 --- a/src/cephadm/cephadm +++ b/src/cephadm/cephadm @@ -2813,15 +2813,42 @@ def deploy_daemon_units( p = os.path.join(data_dir, n) f.write('[ ! -L {p} ] || chown {uid}:{gid} {p}\n'.format(p=p, uid=uid, gid=gid)) else: - prestart = CephContainer( + # if ceph-volume does not support 'ceph-volume activate', we must + # do 'ceph-volume lvm activate'. + test_cv = CephContainer( ctx, image=ctx.image, entrypoint='/usr/sbin/ceph-volume', - args=[ + args=['activate', '--bad-option'], + privileged=True, + volume_mounts=get_container_mounts(ctx, fsid, daemon_type, daemon_id), + bind_mounts=get_container_binds(ctx, fsid, daemon_type, daemon_id), + cname='ceph-%s-%s.%s-activate-test' % (fsid, daemon_type, daemon_id), + ) + out, err, ret = call(ctx, test_cv.run_cmd(), verbosity=CallVerbosity.SILENT) + # bad: ceph-volume: error: unrecognized arguments: activate --bad-option + # good: ceph-volume: error: unrecognized arguments: --bad-option + if 'unrecognized arguments: activate' in err: + # older ceph-volume without top-level activate or --no-tmpfs + cmd = [ 'lvm', 'activate', str(daemon_id), osd_fsid, - '--no-systemd' - ], + '--no-systemd', + ] + else: + cmd = [ + 'activate', + '--osd-id', str(daemon_id), + '--osd-uuid', osd_fsid, + '--no-systemd', + '--no-tmpfs', + ] + + prestart = CephContainer( + ctx, + image=ctx.image, + entrypoint='/usr/sbin/ceph-volume', + args=cmd, privileged=True, volume_mounts=get_container_mounts(ctx, fsid, daemon_type, daemon_id), bind_mounts=get_container_binds(ctx, fsid, daemon_type, daemon_id), From 81e46bc64472a88c493b26f5948ae29f71ebfbed Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Tue, 3 Aug 2021 14:35:27 -0400 Subject: [PATCH 0481/2866] python-common: drivegroup: add 'method' property The DriveGroup method can be none, 'raw', or 'lvm'. Signed-off-by: Sage Weil --- .../ceph/deployment/drive_group.py | 16 +++++++++++++++- src/python-common/ceph/deployment/translate.py | 18 ++++++++++++++---- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/src/python-common/ceph/deployment/drive_group.py b/src/python-common/ceph/deployment/drive_group.py index e063e0d5d82de..0befac1169e68 100644 --- a/src/python-common/ceph/deployment/drive_group.py +++ b/src/python-common/ceph/deployment/drive_group.py @@ -1,3 +1,4 @@ +import enum import yaml from ceph.deployment.inventory import Device @@ -10,6 +11,11 @@ pass +class OSDMethod(str, enum.Enum): + raw = 'raw' + lvm = 'lvm' + + class DeviceSelection(object): """ Used within :class:`ceph.deployment.drive_group.DriveGroupSpec` to specify the devices @@ -144,7 +150,7 @@ class DriveGroupSpec(ServiceSpec): "data_devices", "db_devices", "wal_devices", "journal_devices", "data_directories", "osds_per_device", "objectstore", "osd_id_claims", "journal_size", "unmanaged", "filter_logic", "preview_only", - "data_allocate_fraction" + "data_allocate_fraction", "method" ] def __init__(self, @@ -169,6 +175,7 @@ def __init__(self, filter_logic='AND', # type: str preview_only=False, # type: bool data_allocate_fraction=None, # type: Optional[float] + method=None, # type: Optional[OSDMethod] ): assert service_type is None or service_type == 'osd' super(DriveGroupSpec, self).__init__('osd', service_id=service_id, @@ -230,6 +237,8 @@ def __init__(self, #: Allocate a fraction of the data device (0,1.0] self.data_allocate_fraction = data_allocate_fraction + self.method = method + @classmethod def _from_json_impl(cls, json_drive_group): # type: (dict) -> DriveGroupSpec @@ -316,6 +325,11 @@ def validate(self): if self.filter_logic not in ['AND', 'OR']: raise DriveGroupValidationError('filter_logic must be either or ') + if self.method not in [None, 'lvm', 'raw']: + raise DriveGroupValidationError('method must be one of None, lvm, raw') + if self.method == 'raw' and self.objectstore == 'filestore': + raise DriveGroupValidationError('method raw only supports bluestore') + def __repr__(self) -> str: keys = [ key for key in self._supported_features if getattr(self, key) is not None diff --git a/src/python-common/ceph/deployment/translate.py b/src/python-common/ceph/deployment/translate.py index 144a82f9b4f62..6ca440cff3bf9 100644 --- a/src/python-common/ceph/deployment/translate.py +++ b/src/python-common/ceph/deployment/translate.py @@ -36,7 +36,16 @@ def run(self): return None cmd = "" - if self.spec.objectstore == 'filestore': + if self.spec.method == 'raw': + assert self.spec.objectstore == 'bluestore' + cmd = "raw prepare --bluestore" + cmd += " --data {}".format(" ".join(data_devices)) + if db_devices: + cmd += " --block.db {}".format(" ".join(db_devices)) + if wal_devices: + cmd += " --block.wal {}".format(" ".join(wal_devices)) + + elif self.spec.objectstore == 'filestore': cmd = "lvm batch --no-auto" cmd += " {}".format(" ".join(data_devices)) @@ -50,7 +59,7 @@ def run(self): cmd += " --filestore" - if self.spec.objectstore == 'bluestore': + elif self.spec.objectstore == 'bluestore': cmd = "lvm batch --no-auto {}".format(" ".join(data_devices)) @@ -78,8 +87,9 @@ def run(self): if self.osd_id_claims: cmd += " --osd-ids {}".format(" ".join(self.osd_id_claims)) - cmd += " --yes" - cmd += " --no-systemd" + if self.spec.method != 'raw': + cmd += " --yes" + cmd += " --no-systemd" if self.preview: cmd += " --report" From cef129d90abd73536a141fb375ce35cc7e5081a4 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Tue, 3 Aug 2021 14:36:39 -0400 Subject: [PATCH 0482/2866] mgr/orchestrator: accept --method arg to 'orch daemon add osd' Signed-off-by: Sage Weil --- src/pybind/mgr/orchestrator/module.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/pybind/mgr/orchestrator/module.py b/src/pybind/mgr/orchestrator/module.py index dde3397ce60cd..b9793f1939c23 100644 --- a/src/pybind/mgr/orchestrator/module.py +++ b/src/pybind/mgr/orchestrator/module.py @@ -9,7 +9,7 @@ from prettytable import PrettyTable from ceph.deployment.inventory import Device -from ceph.deployment.drive_group import DriveGroupSpec, DeviceSelection +from ceph.deployment.drive_group import DriveGroupSpec, DeviceSelection, OSDMethod from ceph.deployment.service_spec import PlacementSpec, ServiceSpec from ceph.deployment.hostspec import SpecValidationError from ceph.utils import datetime_now @@ -794,7 +794,9 @@ def _apply_osd(self, return HandleCommandResult(-errno.EINVAL, stderr=usage) @_cli_write_command('orch daemon add osd') - def _daemon_add_osd(self, svc_arg: Optional[str] = None) -> HandleCommandResult: + def _daemon_add_osd(self, + svc_arg: Optional[str] = None, + method: Optional[OSDMethod] = None) -> HandleCommandResult: """Create an OSD service. Either --svc_arg=host:drives""" # Create one or more OSDs""" @@ -808,10 +810,13 @@ def _daemon_add_osd(self, svc_arg: Optional[str] = None) -> HandleCommandResult: host_name, block_device = svc_arg.split(":") block_devices = block_device.split(',') devs = DeviceSelection(paths=block_devices) - drive_group = DriveGroupSpec(placement=PlacementSpec( - host_pattern=host_name), data_devices=devs) - except (TypeError, KeyError, ValueError): - msg = "Invalid host:device spec: '{}'".format(svc_arg) + usage + drive_group = DriveGroupSpec( + placement=PlacementSpec(host_pattern=host_name), + data_devices=devs, + method=method, + ) + except (TypeError, KeyError, ValueError) as e: + msg = f"Invalid host:device spec: '{svc_arg}': {e}" + usage return HandleCommandResult(-errno.EINVAL, stderr=msg) completion = self.create_osds(drive_group) From 11d366d4410938c8588b0d212d05b5ebe23efe4d Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Tue, 3 Aug 2021 14:36:56 -0400 Subject: [PATCH 0483/2866] mgr/cephadm: identify and instantiate raw osds post-create Signed-off-by: Sage Weil --- src/pybind/mgr/cephadm/services/osd.py | 40 +++++++- src/pybind/mgr/cephadm/tests/test_cephadm.py | 97 ++++++++++++++------ 2 files changed, 106 insertions(+), 31 deletions(-) diff --git a/src/pybind/mgr/cephadm/services/osd.py b/src/pybind/mgr/cephadm/services/osd.py index 775d296911ffa..4d41891a9fb20 100644 --- a/src/pybind/mgr/cephadm/services/osd.py +++ b/src/pybind/mgr/cephadm/services/osd.py @@ -98,7 +98,8 @@ def deploy_osd_daemons_for_existing_osds(self, host: str, service_name: str, if replace_osd_ids is None: replace_osd_ids = OsdIdClaims(self.mgr).filtered_by_host(host) assert replace_osd_ids is not None - # check result + + # check result: lvm osds_elems: dict = CephadmServe(self.mgr)._run_cephadm_json( host, 'osd', 'ceph-volume', [ @@ -142,6 +143,43 @@ def deploy_osd_daemons_for_existing_osds(self, host: str, service_name: str, daemon_spec, osd_uuid_map=osd_uuid_map) + # check result: raw + raw_elems: dict = CephadmServe(self.mgr)._run_cephadm_json( + host, 'osd', 'ceph-volume', + [ + '--', + 'raw', 'list', + '--format', 'json', + ]) + for osd_uuid, osd in raw_elems.items(): + if osd.get('ceph_fsid') != fsid: + continue + osd_id = str(osd.get('osd_id', '-1')) + if osd_id in before_osd_uuid_map and osd_id not in replace_osd_ids: + # if it exists but is part of the replacement operation, don't skip + continue + if osd_id not in osd_uuid_map: + logger.debug('osd id {} does not exist in cluster'.format(osd_id)) + continue + if osd_uuid_map.get(osd_id) != osd_uuid: + logger.debug('mismatched osd uuid (cluster has %s, osd ' + 'has %s)' % (osd_uuid_map.get(osd_id), osd_uuid)) + continue + if osd_id in created: + continue + + created.append(osd_id) + daemon_spec = CephadmDaemonDeploySpec( + service_name=service_name, + daemon_id=osd_id, + host=host, + daemon_type='osd', + ) + daemon_spec.final_config, daemon_spec.deps = self.generate_config(daemon_spec) + CephadmServe(self.mgr)._create_daemon( + daemon_spec, + osd_uuid_map=osd_uuid_map) + if created: self.mgr.cache.invalidate_host_devices(host) self.mgr.cache.invalidate_autotune(host) diff --git a/src/pybind/mgr/cephadm/tests/test_cephadm.py b/src/pybind/mgr/cephadm/tests/test_cephadm.py index f93ededbdee6e..40e91daa91c24 100644 --- a/src/pybind/mgr/cephadm/tests/test_cephadm.py +++ b/src/pybind/mgr/cephadm/tests/test_cephadm.py @@ -1,4 +1,5 @@ import json +import logging from contextlib import contextmanager @@ -592,8 +593,10 @@ def test_apply_osd_save(self, _run_cephadm, cephadm_module: CephadmOrchestrator) ['--config-json', '-', '--', 'lvm', 'batch', '--no-auto', '/dev/sdb', '--yes', '--no-systemd'], env_vars=['CEPH_VOLUME_OSDSPEC_AFFINITY=foo'], error_ok=True, stdin='{"config": "", "keyring": ""}') - _run_cephadm.assert_called_with( + _run_cephadm.assert_any_call( 'test', 'osd', 'ceph-volume', ['--', 'lvm', 'list', '--format', 'json'], image='', no_fsid=False) + _run_cephadm.assert_any_call( + 'test', 'osd', 'ceph-volume', ['--', 'raw', 'list', '--format', 'json'], image='', no_fsid=False) @mock.patch("cephadm.serve.CephadmServe._run_cephadm") def test_apply_osd_save_non_collocated(self, _run_cephadm, cephadm_module: CephadmOrchestrator): @@ -632,8 +635,10 @@ def test_apply_osd_save_non_collocated(self, _run_cephadm, cephadm_module: Cepha '--wal-devices', '/dev/sdd', '--yes', '--no-systemd'], env_vars=['CEPH_VOLUME_OSDSPEC_AFFINITY=noncollocated'], error_ok=True, stdin='{"config": "", "keyring": ""}') - _run_cephadm.assert_called_with( + _run_cephadm.assert_any_call( 'test', 'osd', 'ceph-volume', ['--', 'lvm', 'list', '--format', 'json'], image='', no_fsid=False) + _run_cephadm.assert_any_call( + 'test', 'osd', 'ceph-volume', ['--', 'raw', 'list', '--format', 'json'], image='', no_fsid=False) @mock.patch("cephadm.serve.CephadmServe._run_cephadm", _run_cephadm('{}')) @mock.patch("cephadm.module.SpecStore.save") @@ -1410,17 +1415,31 @@ def test_osd_activate_datadevice(self, _run_cephadm, cephadm_module: CephadmOrch ] }) - ceph_volume_lvm_list = { - '1': [{ - 'tags': { - 'ceph.cluster_fsid': cephadm_module._cluster_fsid, - 'ceph.osd_fsid': 'uuid' - }, - 'type': 'data' - }] - } - _run_cephadm.return_value = (json.dumps(ceph_volume_lvm_list), '', 0) - _run_cephadm.reset_mock() + def _ceph_volume_list(s, host, entity, cmd, **kwargs): + logging.info(f'ceph-volume cmd: {cmd}') + if 'raw' in cmd: + return json.dumps({ + "21a4209b-f51b-4225-81dc-d2dca5b8b2f5": { + "ceph_fsid": "64c84f19-fe1d-452a-a731-ab19dc144aa8", + "device": "/dev/loop0", + "osd_id": 21, + "osd_uuid": "21a4209b-f51b-4225-81dc-d2dca5b8b2f5", + "type": "bluestore" + }, + }), '', 0 + if 'lvm' in cmd: + return json.dumps({ + '1': [{ + 'tags': { + 'ceph.cluster_fsid': cephadm_module._cluster_fsid, + 'ceph.osd_fsid': 'uuid' + }, + 'type': 'data' + }] + }), '', 0 + return '{}', '', 0 + _run_cephadm.reset_mock(return_value=True) + _run_cephadm.side_effect = _ceph_volume_list assert cephadm_module._osd_activate( ['test']).stdout == "Created osd(s) 1 on host 'test'" assert _run_cephadm.mock_calls == [ @@ -1430,6 +1449,8 @@ def test_osd_activate_datadevice(self, _run_cephadm, cephadm_module: CephadmOrch ['--name', 'osd.1', '--meta-json', mock.ANY, '--config-json', '-', '--osd-fsid', 'uuid'], stdin=mock.ANY, image=''), + mock.call('test', 'osd', 'ceph-volume', + ['--', 'raw', 'list', '--format', 'json'], no_fsid=False, image=''), ] @mock.patch("cephadm.serve.CephadmServe._run_cephadm") @@ -1486,23 +1507,37 @@ def test_osd_activate_datadevice_dbdevice(self, _run_cephadm, cephadm_module: Ce ] }) - ceph_volume_lvm_list = { - '1': [{ - 'tags': { - 'ceph.cluster_fsid': cephadm_module._cluster_fsid, - 'ceph.osd_fsid': 'uuid' - }, - 'type': 'data' - }, { - 'tags': { - 'ceph.cluster_fsid': cephadm_module._cluster_fsid, - 'ceph.osd_fsid': 'uuid' - }, - 'type': 'db' - }] - } - _run_cephadm.return_value = (json.dumps(ceph_volume_lvm_list), '', 0) - _run_cephadm.reset_mock() + def _ceph_volume_list(s, host, entity, cmd, **kwargs): + logging.info(f'ceph-volume cmd: {cmd}') + if 'raw' in cmd: + return json.dumps({ + "21a4209b-f51b-4225-81dc-d2dca5b8b2f5": { + "ceph_fsid": "64c84f19-fe1d-452a-a731-ab19dc144aa8", + "device": "/dev/loop0", + "osd_id": 21, + "osd_uuid": "21a4209b-f51b-4225-81dc-d2dca5b8b2f5", + "type": "bluestore" + }, + }), '', 0 + if 'lvm' in cmd: + return json.dumps({ + '1': [{ + 'tags': { + 'ceph.cluster_fsid': cephadm_module._cluster_fsid, + 'ceph.osd_fsid': 'uuid' + }, + 'type': 'data' + }, { + 'tags': { + 'ceph.cluster_fsid': cephadm_module._cluster_fsid, + 'ceph.osd_fsid': 'uuid' + }, + 'type': 'db' + }] + }), '', 0 + return '{}', '', 0 + _run_cephadm.reset_mock(return_value=True) + _run_cephadm.side_effect = _ceph_volume_list assert cephadm_module._osd_activate( ['test']).stdout == "Created osd(s) 1 on host 'test'" assert _run_cephadm.mock_calls == [ @@ -1512,4 +1547,6 @@ def test_osd_activate_datadevice_dbdevice(self, _run_cephadm, cephadm_module: Ce ['--name', 'osd.1', '--meta-json', mock.ANY, '--config-json', '-', '--osd-fsid', 'uuid'], stdin=mock.ANY, image=''), + mock.call('test', 'osd', 'ceph-volume', + ['--', 'raw', 'list', '--format', 'json'], no_fsid=False, image=''), ] From 101c810a80eea14ab2a1edc8166dbbe76cd9e87a Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Thu, 12 Aug 2021 11:12:59 -0400 Subject: [PATCH 0484/2866] ceph-volume: activate: try simple mode too This is of dubious value to cephadm since /etc/ceph/osd/* won't be populated inside of a conatiner. However, it makes sense from a purely ceph-volume perspective. Signed-off-by: Sage Weil --- src/ceph-volume/ceph_volume/activate/main.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/ceph-volume/ceph_volume/activate/main.py b/src/ceph-volume/ceph_volume/activate/main.py index 40690c5d97603..1f8144ab2c501 100644 --- a/src/ceph-volume/ceph_volume/activate/main.py +++ b/src/ceph-volume/ceph_volume/activate/main.py @@ -5,6 +5,7 @@ from ceph_volume import terminal from ceph_volume.devices.lvm.activate import Activate as LVMActivate from ceph_volume.devices.raw.activate import Activate as RAWActivate +from ceph_volume.devices.simple.activate import Activate as SimpleActivate class Activate(object): @@ -70,4 +71,17 @@ def main(self): except Exception as e: terminal.info(f'Failed to activate via lvm: {e}') + # then try simple + try: + SimpleActivate([]).activate( + argparse.Namespace( + osd_id=self.args.osd_id, + osd_fsid=self.args.osd_uuid, + no_systemd=self.args.no_systemd, + ) + ) + return + except Exception as e: + terminal.info(f'Failed to activate via simple: {e}') + terminal.error('Failed to activate any OSD(s)') From 1f7771d4a77ebb271b939b3893d7607d964796f0 Mon Sep 17 00:00:00 2001 From: Adam Kupczyk Date: Tue, 2 Nov 2021 16:56:10 +0100 Subject: [PATCH 0485/2866] os/objectstore/test: Add test for data corruption in file truncation Test for https://tracker.ceph.com/issues/53129 Signed-off-by: Adam Kupczyk --- src/test/objectstore/test_bluefs.cc | 85 +++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/src/test/objectstore/test_bluefs.cc b/src/test/objectstore/test_bluefs.cc index d868c876e1ef9..799996da17fc6 100644 --- a/src/test/objectstore/test_bluefs.cc +++ b/src/test/objectstore/test_bluefs.cc @@ -828,6 +828,91 @@ TEST(BlueFS, test_tracker_50965) { fs.umount(); } +TEST(BlueFS, test_truncate_stable_53129) { + + ConfSaver conf(g_ceph_context->_conf); + conf.SetVal("bluefs_min_flush_size", "65536"); + conf.ApplyChanges(); + + uint64_t size_wal = 1048576 * 64; + TempBdev bdev_wal{size_wal}; + uint64_t size_db = 1048576 * 128; + TempBdev bdev_db{size_db}; + uint64_t size_slow = 1048576 * 256; + TempBdev bdev_slow{size_slow}; + + BlueFS fs(g_ceph_context); + ASSERT_EQ(0, fs.add_block_device(BlueFS::BDEV_WAL, "wal", false, 0)); + ASSERT_EQ(0, fs.add_block_device(BlueFS::BDEV_DB, "db", false, 0)); + ASSERT_EQ(0, fs.add_block_device(BlueFS::BDEV_SLOW, "slow", false, 0)); + uuid_d fsid; + ASSERT_EQ(0, fs.mkfs(fsid, { BlueFS::BDEV_DB, true, true })); + ASSERT_EQ(0, fs.mount()); + ASSERT_EQ(0, fs.maybe_verify_layout({ BlueFS::BDEV_DB, true, true })); + + string dir_slow = "dir.slow"; + ASSERT_EQ(0, fs.mkdir(dir_slow)); + string dir_db = "dir_db"; + ASSERT_EQ(0, fs.mkdir(dir_db)); + + string file_slow = "file"; + BlueFS::FileWriter *h_slow; + ASSERT_EQ(0, fs.open_for_write(dir_slow, file_slow, &h_slow, false)); + ASSERT_NE(nullptr, h_slow); + + string file_db = "file"; + BlueFS::FileWriter *h_db; + ASSERT_EQ(0, fs.open_for_write(dir_db, file_db, &h_db, false)); + ASSERT_NE(nullptr, h_db); + + bufferlist bl1; + std::unique_ptr buf1 = gen_buffer(70000); + bufferptr bp1 = buffer::claim_char(70000, buf1.get()); + bl1.push_back(bp1); + // add 70000 bytes + h_slow->append(bl1.c_str(), bl1.length()); + fs.flush(h_slow); + // and truncate to 60000 bytes + fs.truncate(h_slow, 60000); + + // write something to file on DB device + bufferlist bl2; + std::unique_ptr buf2 = gen_buffer(1000); + bufferptr bp2 = buffer::claim_char(1000, buf2.get()); + bl2.push_back(bp2); + h_db->append(bl2.c_str(), bl2.length()); + // and force bluefs log to flush + fs.fsync(h_db); + + // This is the actual test point. + // We completed truncate, and we expect + // - size to be 60000 + // - data to be stable on slow device + // OR + // - size = 0 or file does not exist + // - dev_dirty is irrelevant + bool h_slow_dev_dirty = fs.debug_get_is_dev_dirty(h_slow, BlueFS::BDEV_SLOW); + // Imagine power goes down here. + + fs.close_writer(h_slow); + fs.close_writer(h_db); + + fs.umount(); + + ASSERT_EQ(0, fs.mount()); + ASSERT_EQ(0, fs.maybe_verify_layout({ BlueFS::BDEV_DB, true, true })); + + uint64_t size; + utime_t mtime; + ASSERT_EQ(0, fs.stat("dir.slow", "file", &size, &mtime)); + // check file size 60000 + ASSERT_EQ(size, 60000); + // check that dev_dirty was false (data stable on media) + ASSERT_EQ(h_slow_dev_dirty, false); + + fs.umount(); +} + int main(int argc, char **argv) { auto args = argv_to_vec(argc, argv); map defaults = { From 1b2ed89db785eb52bda403aa751cdac8ef64d4c4 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Tue, 2 Nov 2021 09:41:49 -0400 Subject: [PATCH 0486/2866] pybind/ceph_argparse: handle misordered keyword arguments Signed-off-by: Sage Weil --- src/pybind/ceph_argparse.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/pybind/ceph_argparse.py b/src/pybind/ceph_argparse.py index 6e1ce7b72d31d..d24b8f8daa49a 100644 --- a/src/pybind/ceph_argparse.py +++ b/src/pybind/ceph_argparse.py @@ -1159,8 +1159,12 @@ def validate(args: List[str], # no arg, but not required? Continue consuming mysig # in case there are later required args - if myarg in (None, []) and not desc.req: - break + if myarg in (None, []): + if not desc.req: + break + # did we already get this argument (as a named arg, earlier?) + if desc.name in d: + break # A keyword argument? if myarg: From b3860b366246b14e0844cb5eef38b5e47ec85412 Mon Sep 17 00:00:00 2001 From: Casey Bodley Date: Tue, 2 Nov 2021 14:18:31 -0400 Subject: [PATCH 0487/2866] rgw: fix ListBucketMultiparts response with common prefixes see documentation and examples in https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListMultipartUploads.html that use Prefix directly Fixes: https://tracker.ceph.com/issues/47527 Signed-off-by: Casey Bodley --- src/rgw/rgw_rest_s3.cc | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc index 73f3ed5729a3f..f1786351a692e 100644 --- a/src/rgw/rgw_rest_s3.cc +++ b/src/rgw/rgw_rest_s3.cc @@ -3876,7 +3876,7 @@ void RGWListBucketMultiparts_ObjStore_S3::send_response() s->formatter->dump_string("Tenant", s->bucket_tenant); s->formatter->dump_string("Bucket", s->bucket_name); if (!prefix.empty()) - s->formatter->dump_string("ListMultipartUploadsResult.Prefix", prefix); + s->formatter->dump_string("Prefix", prefix); if (!marker_key.empty()) s->formatter->dump_string("KeyMarker", marker_key); if (!marker_upload_id.empty()) @@ -3911,10 +3911,9 @@ void RGWListBucketMultiparts_ObjStore_S3::send_response() s->formatter->open_array_section("CommonPrefixes"); for (const auto& kv : common_prefixes) { if (encode_url) { - s->formatter->dump_string("CommonPrefixes.Prefix", - url_encode(kv.first, false)); + s->formatter->dump_string("Prefix", url_encode(kv.first, false)); } else { - s->formatter->dump_string("CommonPrefixes.Prefix", kv.first); + s->formatter->dump_string("Prefix", kv.first); } } s->formatter->close_section(); From dfacf162afe55c6f15a8afb8aca18b52814e80ee Mon Sep 17 00:00:00 2001 From: Igor Fedotov Date: Tue, 2 Nov 2021 14:54:55 +0300 Subject: [PATCH 0488/2866] PendingReleaseNotes: document OMAP upgrade bug. Signed-off-by: Igor Fedotov --- PendingReleaseNotes | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/PendingReleaseNotes b/PendingReleaseNotes index ff1b2a1aa10db..33290d8ea1f64 100644 --- a/PendingReleaseNotes +++ b/PendingReleaseNotes @@ -1,5 +1,11 @@ >=17.0.0 +* Critical bug in OMAP format upgrade is fixed. This could cause data corruption + (improperly formatted OMAP keys) after pre-Pacific cluster upgrade if + bluestore-quick-fix-on-mount parameter is set to true or ceph-bluestore-tool's + quick-fix/repair commands are invoked. + Relevant tracker: https://tracker.ceph.com/issues/53062 + * `ceph-mgr-modules-core` debian package does not recommend `ceph-mgr-rook` anymore. As the latter depends on `python3-numpy` which cannot be imported in different Python sub-interpreters multi-times if the version of From fa4ee0f3c6e748a31367a094bb61cb6d9fbfea2c Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Tue, 2 Nov 2021 12:38:05 -0400 Subject: [PATCH 0489/2866] qa/tasks/cephfs/test_nfs: use keyword args Signed-off-by: Sage Weil --- qa/tasks/cephfs/test_nfs.py | 43 +++++++++++++++++++++++-------------- 1 file changed, 27 insertions(+), 16 deletions(-) diff --git a/qa/tasks/cephfs/test_nfs.py b/qa/tasks/cephfs/test_nfs.py index 0d5099504c6b8..97269a32c55bb 100644 --- a/qa/tasks/cephfs/test_nfs.py +++ b/qa/tasks/cephfs/test_nfs.py @@ -163,11 +163,12 @@ def _create_export(self, export_id, create_fs=False, extra_cmd=None): j = json.loads(output) if j[0]['status']['running']: break - export_cmd = ['nfs', 'export', 'create', 'cephfs', self.fs_name, self.cluster_id] + export_cmd = ['nfs', 'export', 'create', 'cephfs', + '--fsname', self.fs_name, '--cluster-id', self.cluster_id] if isinstance(extra_cmd, list): export_cmd.extend(extra_cmd) else: - export_cmd.append(self.pseudo_path) + export_cmd.extend(['--pseudo-path', self.pseudo_path]) # Runs the nfs export create command self._cmd(*export_cmd) # Check if user id for export is created @@ -350,9 +351,10 @@ def test_create_delete_export_idempotency(self): ''' Test idempotency of export create and delete commands. ''' - self._test_idempotency(self._create_default_export, ['nfs', 'export', 'create', 'cephfs', - self.fs_name, self.cluster_id, - self.pseudo_path]) + self._test_idempotency(self._create_default_export, [ + 'nfs', 'export', 'create', 'cephfs', + '--fsname', self.fs_name, '--cluster-id', self.cluster_id, + '--pseudo-path', self.pseudo_path]) self._test_idempotency(self._delete_export, ['nfs', 'export', 'rm', self.cluster_id, self.pseudo_path]) self._test_delete_cluster() @@ -364,14 +366,18 @@ def test_create_multiple_exports(self): # Export-1 with default values (access type = rw and path = '\') self._create_default_export() # Export-2 with r only - self._create_export(export_id='2', extra_cmd=[self.pseudo_path+'1', '--readonly']) + self._create_export(export_id='2', + extra_cmd=['--pseudo-path', self.pseudo_path+'1', '--readonly']) # Export-3 for subvolume with r only self._cmd('fs', 'subvolume', 'create', self.fs_name, 'sub_vol') fs_path = self._cmd('fs', 'subvolume', 'getpath', self.fs_name, 'sub_vol').strip() - self._create_export(export_id='3', extra_cmd=[self.pseudo_path+'2', '--readonly', - '--path', fs_path]) + self._create_export(export_id='3', + extra_cmd=['--pseudo-path', self.pseudo_path+'2', '--readonly', + '--path', fs_path]) # Export-4 for subvolume - self._create_export(export_id='4', extra_cmd=[self.pseudo_path+'3', fs_path]) + self._create_export(export_id='4', + extra_cmd=['--pseudo-path', self.pseudo_path+'3', + '--path', fs_path]) # Check if exports gets listed self._test_list_detailed(fs_path) self._test_delete_cluster() @@ -404,7 +410,9 @@ def test_export_create_with_non_existing_fsname(self): try: fs_name = 'nfs-test' self._test_create_cluster() - self._nfs_cmd('export', 'create', 'cephfs', fs_name, self.cluster_id, self.pseudo_path) + self._nfs_cmd('export', 'create', 'cephfs', + '--fsname', fs_name, '--cluster-id', self.cluster_id, + '--pseudo-path', self.pseudo_path) self.fail(f"Export created with non-existing filesystem {fs_name}") except CommandFailedError as e: # Command should fail for test to pass @@ -419,7 +427,8 @@ def test_export_create_with_non_existing_clusterid(self): ''' try: cluster_id = 'invalidtest' - self._nfs_cmd('export', 'create', 'cephfs', self.fs_name, cluster_id, self.pseudo_path) + self._nfs_cmd('export', 'create', 'cephfs', '--fsname', self.fs_name, + '--cluster-id', cluster_id, '--pseudo-path', self.pseudo_path) self.fail(f"Export created with non-existing cluster id {cluster_id}") except CommandFailedError as e: # Command should fail for test to pass @@ -432,8 +441,9 @@ def test_export_create_with_relative_pseudo_path_and_root_directory(self): ''' def check_pseudo_path(pseudo_path): try: - self._nfs_cmd('export', 'create', 'cephfs', self.fs_name, self.cluster_id, - pseudo_path) + self._nfs_cmd('export', 'create', 'cephfs', '--fsname', self.fs_name, + '--cluster-id', self.cluster_id, + '--pseudo-path', pseudo_path) self.fail(f"Export created for {pseudo_path}") except CommandFailedError as e: # Command should fail for test to pass @@ -453,7 +463,8 @@ def test_write_to_read_only_export(self): Test write to readonly export. ''' self._test_create_cluster() - self._create_export(export_id='1', create_fs=True, extra_cmd=[self.pseudo_path, '--readonly']) + self._create_export(export_id='1', create_fs=True, + extra_cmd=['--pseudo-path', self.pseudo_path, '--readonly']) port, ip = self._get_port_ip_info() self._check_nfs_cluster_status('running', 'NFS Ganesha cluster restart failed') self._write_to_read_only_export(self.pseudo_path, port, ip) @@ -658,8 +669,8 @@ def exec_cmd_invalid(*cmd): exec_cmd_invalid('cluster', 'config', 'set') exec_cmd_invalid('cluster', 'config', 'reset') exec_cmd_invalid('export', 'create', 'cephfs') - exec_cmd_invalid('export', 'create', 'cephfs', 'a_fs') - exec_cmd_invalid('export', 'create', 'cephfs', 'a_fs', 'clusterid') + exec_cmd_invalid('export', 'create', 'cephfs', 'clusterid') + exec_cmd_invalid('export', 'create', 'cephfs', 'clusterid', 'a_fs') exec_cmd_invalid('export', 'ls') exec_cmd_invalid('export', 'delete') exec_cmd_invalid('export', 'delete', 'clusterid') From 9467f1e89e628e36cc751b1cdaafea79cb3102d4 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Wed, 20 Oct 2021 15:35:49 -0400 Subject: [PATCH 0490/2866] mgr/nfs: document and use keyword args for 'nfs export create cephfs' Signed-off-by: Sage Weil --- doc/mgr/nfs.rst | 8 ++++---- qa/suites/orch/cephadm/mgr-nfs-upgrade/2-nfs.yaml | 2 +- .../cephadm/smoke-roleless/2-services/nfs-ingress.yaml | 2 +- .../cephadm/smoke-roleless/2-services/nfs-ingress2.yaml | 2 +- src/vstart.sh | 4 ++-- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/doc/mgr/nfs.rst b/doc/mgr/nfs.rst index 54ca19fdd2e26..52ddf104fd8fc 100644 --- a/doc/mgr/nfs.rst +++ b/doc/mgr/nfs.rst @@ -239,17 +239,17 @@ Create CephFS Export .. code:: bash - $ ceph nfs export create cephfs [--readonly] [--path=/path/in/cephfs] [--client_addr ...] [--squash ] + $ ceph nfs export create cephfs --cluster-id --pseudo-path --fsname [--readonly] [--path=/path/in/cephfs] [--client_addr ...] [--squash ] This creates export RADOS objects containing the export block, where -```` is the name of the FS volume used by the NFS Ganesha cluster -that will serve this export. - ```` is the NFS Ganesha cluster ID. ```` is the export position within the NFS v4 Pseudo Filesystem where the export will be available on the server. It must be an absolute path and unique. +```` is the name of the FS volume used by the NFS Ganesha cluster +that will serve this export. + ```` is the path within cephfs. Valid path should be given and default path is '/'. It need not be unique. Subvolume path can be fetched using: diff --git a/qa/suites/orch/cephadm/mgr-nfs-upgrade/2-nfs.yaml b/qa/suites/orch/cephadm/mgr-nfs-upgrade/2-nfs.yaml index eb8fa2a547807..34680fc8a6b91 100644 --- a/qa/suites/orch/cephadm/mgr-nfs-upgrade/2-nfs.yaml +++ b/qa/suites/orch/cephadm/mgr-nfs-upgrade/2-nfs.yaml @@ -15,7 +15,7 @@ tasks: - cephadm.shell: host.a: - ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2 - - ceph nfs export create cephfs foofs foo --binding /fake || ceph nfs export create cephfs foofs foo --pseudo-path /fake + - ceph nfs export create cephfs --fsname foofs --clusterid foo --binding /fake || ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake # we can't do wait_for_service here because with octopus it's nfs.ganesha-foo not nfs.foo - while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done diff --git a/qa/suites/orch/cephadm/smoke-roleless/2-services/nfs-ingress.yaml b/qa/suites/orch/cephadm/smoke-roleless/2-services/nfs-ingress.yaml index 3d935218a95e4..b4e843df26116 100644 --- a/qa/suites/orch/cephadm/smoke-roleless/2-services/nfs-ingress.yaml +++ b/qa/suites/orch/cephadm/smoke-roleless/2-services/nfs-ingress.yaml @@ -40,7 +40,7 @@ tasks: - cephadm.shell: host.a: - - ceph nfs export create cephfs foofs foo --pseudo-path /fake + - ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake - vip.exec: host.a: diff --git a/qa/suites/orch/cephadm/smoke-roleless/2-services/nfs-ingress2.yaml b/qa/suites/orch/cephadm/smoke-roleless/2-services/nfs-ingress2.yaml index 58f7e13a2499c..a47dd9d7646bd 100644 --- a/qa/suites/orch/cephadm/smoke-roleless/2-services/nfs-ingress2.yaml +++ b/qa/suites/orch/cephadm/smoke-roleless/2-services/nfs-ingress2.yaml @@ -15,7 +15,7 @@ tasks: host.a: - ceph fs volume create foofs - ceph nfs cluster create foo --ingress --virtual-ip {{VIP0}}/{{VIPPREFIXLEN}} --port 2999 - - ceph nfs export create cephfs foofs foo --pseudo-path /fake + - ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake - cephadm.wait_for_service: service: nfs.foo diff --git a/src/vstart.sh b/src/vstart.sh index bdf0f596dec2c..0fa55eba02ed9 100755 --- a/src/vstart.sh +++ b/src/vstart.sh @@ -1224,7 +1224,7 @@ start_ganesha() { ceph_adm orch set backend test_orchestrator ceph_adm test_orchestrator load_data -i $CEPH_ROOT/src/pybind/mgr/test_orchestrator/dummy_data.json prun ceph_adm nfs cluster create $cluster_id - prun ceph_adm nfs export create cephfs "a" $cluster_id "/cephfs" + prun ceph_adm nfs export create cephfs --fsname "a" --cluster-id $cluster_id --pseudo-path "/cephfs" for name in a b c d e f g h i j k l m n o p do @@ -1664,7 +1664,7 @@ if [ $GANESHA_DAEMON_NUM -gt 0 ]; then port="2049" prun ceph_adm nfs cluster create $cluster_id if [ $CEPH_NUM_MDS -gt 0 ]; then - prun ceph_adm nfs export create cephfs "a" $cluster_id $pseudo_path + prun ceph_adm nfs export create cephfs --fsname "a" --cluster-id $cluster_id --pseudo-path $pseudo_path echo "Mount using: mount -t nfs -o port=$port $IP:$pseudo_path mountpoint" fi if [ "$CEPH_NUM_RGW" -gt 0 ]; then From aef952bc4691ba357ab3f5da69817b5bf581f1e2 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Wed, 20 Oct 2021 15:38:27 -0400 Subject: [PATCH 0491/2866] mgr/nfs: use keyword args for 'nfs export create rgw' Signed-off-by: Sage Weil --- doc/mgr/nfs.rst | 12 ++++++------ ...-ingress-rgw.yaml => nfs-ingress-rgw-bucket.yaml} | 2 +- src/vstart.sh | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) rename qa/suites/orch/cephadm/smoke-roleless/2-services/{nfs-ingress-rgw.yaml => nfs-ingress-rgw-bucket.yaml} (95%) diff --git a/doc/mgr/nfs.rst b/doc/mgr/nfs.rst index 52ddf104fd8fc..548f7b62da780 100644 --- a/doc/mgr/nfs.rst +++ b/doc/mgr/nfs.rst @@ -275,24 +275,24 @@ To export a bucket .. code:: - $ ceph nfs export create rgw [--readonly] [--client_addr ...] [--squash ] + $ ceph nfs export create rgw --cluster-id --pseudo-path --bucket [--readonly] [--client_addr ...] [--squash ] For example, to export *mybucket* via NFS cluster *mynfs* at the pseudo-path */bucketdata* to any host in the ``192.168.10.0/24`` network .. code:: - $ ceph nfs export create rgw mybucket mynfs /bucketdata --client_addr 192.168.10.0/24 + $ ceph nfs export create rgw --cluster-id mynfs --pseudo-path /bucketdata --bucket mybucket --client_addr 192.168.10.0/24 .. note:: Export creation is supported only for NFS Ganesha clusters deployed using nfs interface. -```` is the name of the bucket that will be exported. - -.. note:: Currently, if multi-site RGW is enabled, Ceph can only export RGW buckets in the default realm. - ```` is the NFS Ganesha cluster ID. ```` is the export position within the NFS v4 Pseudo Filesystem where the export will be available on the server. It must be an absolute path and unique. +```` is the name of the bucket that will be exported. + +.. note:: Currently, if multi-site RGW is enabled, Ceph can only export RGW buckets in the default realm. + ```` is the list of client address for which these export permissions will be applicable. By default all clients can access the export according to specified export permissions. See the `NFS-Ganesha Export Sample`_ diff --git a/qa/suites/orch/cephadm/smoke-roleless/2-services/nfs-ingress-rgw.yaml b/qa/suites/orch/cephadm/smoke-roleless/2-services/nfs-ingress-rgw-bucket.yaml similarity index 95% rename from qa/suites/orch/cephadm/smoke-roleless/2-services/nfs-ingress-rgw.yaml rename to qa/suites/orch/cephadm/smoke-roleless/2-services/nfs-ingress-rgw-bucket.yaml index 8e18c5906c31c..3f4964978d929 100644 --- a/qa/suites/orch/cephadm/smoke-roleless/2-services/nfs-ingress-rgw.yaml +++ b/qa/suites/orch/cephadm/smoke-roleless/2-services/nfs-ingress-rgw-bucket.yaml @@ -40,7 +40,7 @@ tasks: - cephadm.shell: host.a: - - ceph nfs export create rgw foobucket foo --pseudo-path /foobucket + - ceph nfs export create rgw --bucket foobucket --cluster-id foo --pseudo-path /foobucket - cephadm.wait_for_service: service: nfs.foo diff --git a/src/vstart.sh b/src/vstart.sh index 0fa55eba02ed9..2b6a4017afab7 100755 --- a/src/vstart.sh +++ b/src/vstart.sh @@ -1670,7 +1670,7 @@ if [ $GANESHA_DAEMON_NUM -gt 0 ]; then if [ "$CEPH_NUM_RGW" -gt 0 ]; then pseudo_path="/rgw" do_rgw_create_bucket - prun ceph_adm nfs export create rgw "nfs-bucket" $cluster_id $pseudo_path + prun ceph_adm nfs export create rgw --cluster-id $cluster_id --pseudo-path $pseudo_path --bucket "nfs-bucket" echo "Mount using: mount -t nfs -o port=$port $IP:$pseudo_path mountpoint" fi else From a07ca2444d7e5a800944fc8f75ebd88e26505c7e Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Wed, 20 Oct 2021 15:39:03 -0400 Subject: [PATCH 0492/2866] mgr/nfs: reorder 'nfs export create cephfs' arguments Put fsname after cluster_id + pseudo_path so that it aligns with the change to the rgw command. Signed-off-by: Sage Weil --- src/pybind/mgr/nfs/module.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pybind/mgr/nfs/module.py b/src/pybind/mgr/nfs/module.py index 0bc3ab63e220a..8eef68f2369de 100644 --- a/src/pybind/mgr/nfs/module.py +++ b/src/pybind/mgr/nfs/module.py @@ -27,9 +27,9 @@ def __init__(self, *args: str, **kwargs: Any) -> None: @CLICommand('nfs export create cephfs', perm='rw') def _cmd_nfs_export_create_cephfs( self, - fsname: str, cluster_id: str, pseudo_path: str, + fsname: str, path: Optional[str] = '/', readonly: Optional[bool] = False, client_addr: Optional[List[str]] = None, From 89e73a20c35ec8292182d4b7cee4425714ee2456 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Wed, 20 Oct 2021 15:39:36 -0400 Subject: [PATCH 0493/2866] mgr/nfs: reorder 'nfs export creage rgw' arguments Put bucket name last so that it paves the way for an optional 'user' arg to go along with it. Signed-off-by: Sage Weil --- src/pybind/mgr/nfs/module.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pybind/mgr/nfs/module.py b/src/pybind/mgr/nfs/module.py index 8eef68f2369de..280a61653ec92 100644 --- a/src/pybind/mgr/nfs/module.py +++ b/src/pybind/mgr/nfs/module.py @@ -44,9 +44,9 @@ def _cmd_nfs_export_create_cephfs( @CLICommand('nfs export create rgw', perm='rw') def _cmd_nfs_export_create_rgw( self, - bucket: str, cluster_id: str, pseudo_path: str, + bucket: str, readonly: Optional[bool] = False, client_addr: Optional[List[str]] = None, squash: str = 'none', From 6b4382aac7596685c31e078ffaf404c1df196819 Mon Sep 17 00:00:00 2001 From: Omri Zeneva Date: Fri, 20 Aug 2021 19:19:26 +0300 Subject: [PATCH 0494/2866] common/tracer: tracer implementation was moved to common/ Signed-off-by: Omri Zeneva --- doc/dev/developer_guide/jaegertracing.rst | 17 +--- doc/index.rst | 1 + doc/jaegertracing/index.rst | 80 +++++++++++++++++ doc/jaegertracing/osd_jaeger.png | Bin 0 -> 117123 bytes doc/jaegertracing/rgw_jaeger.png | Bin 0 -> 131398 bytes src/CMakeLists.txt | 10 +-- src/common/options/global.yaml.in | 9 ++ src/common/options/rgw.yaml.in | 8 -- src/common/tracer.cc | 104 ++++++++-------------- src/common/tracer.h | 92 +++++++++++++++---- src/osd/CMakeLists.txt | 3 +- src/osd/ECBackend.cc | 30 +++---- src/osd/OSD.cc | 50 +++++------ src/osd/OpRequest.h | 16 +--- src/osd/PrimaryLogPG.cc | 58 +++++------- src/osd/ReplicatedBackend.cc | 16 ++-- src/osd/osd_tracer.cc | 12 +++ src/osd/osd_tracer.h | 13 +++ src/osd/scheduler/OpSchedulerItem.cc | 9 +- src/rgw/rgw_common.h | 2 +- src/rgw/rgw_op.cc | 8 +- src/rgw/rgw_process.cc | 20 ++--- src/rgw/rgw_tracer.cc | 58 +++--------- src/rgw/rgw_tracer.h | 83 ++--------------- 24 files changed, 339 insertions(+), 360 deletions(-) create mode 100644 doc/jaegertracing/index.rst create mode 100644 doc/jaegertracing/osd_jaeger.png create mode 100644 doc/jaegertracing/rgw_jaeger.png create mode 100644 src/osd/osd_tracer.cc create mode 100644 src/osd/osd_tracer.h diff --git a/doc/dev/developer_guide/jaegertracing.rst b/doc/dev/developer_guide/jaegertracing.rst index d70b72c385b80..73a48ad83c252 100644 --- a/doc/dev/developer_guide/jaegertracing.rst +++ b/doc/dev/developer_guide/jaegertracing.rst @@ -16,19 +16,8 @@ it across the distributed system. BASIC ARCHITECTURE AND TERMINOLOGY ---------------------------------- -* TRACE: A trace shows the data/execution path through a system. -* SPAN: A single unit of a trace, it is a data structure that stores - information like operation name, timestamps, ordering in a trace. -* JAEGER CLIENT: language-specific implementations of the OpenTracing API. -* JAEGER AGENT: a daemon that listens for spans sent over User Datagram Protocol. - The agent is meant to be placed on the same host as the instrumented - application. (acts like a sidecar listener) -* JAEGER COLLECTOR: Jaeger agent sends the spans to this daemon which then - stitches the spans together to form a trace(if enabled, also persists a database - for these traces) -* JAEGER QUERY AND CONSOLE FRONTEND: UI based frontend to checkout the jaeger - traces, navigate to http://localhost:16686 (if using default `all-in-one - docker `_ to access the Jaeger UI. +refer to the `Ceph Tracing documentation <../../../jaegertracing/#basic-architecture-and-terminology>`_ + HOW TO GET STARTED USING TRACING? --------------------------------- @@ -71,4 +60,4 @@ steps needed: $ bin/rados -p test bench 5 write --no-cleanup .. seealso:: - `using-jaeger-cpp-client-for-distributed-tracing-in-ceph `_ + `using-jaeger-cpp-client-for-distributed-tracing-in-ceph ` \ No newline at end of file diff --git a/doc/index.rst b/doc/index.rst index 62619726df782..cca7031cb8db9 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -109,3 +109,4 @@ about Ceph, see our `Architecture`_ section. releases/index security/index Glossary + Tracing diff --git a/doc/jaegertracing/index.rst b/doc/jaegertracing/index.rst new file mode 100644 index 0000000000000..7261337d3738e --- /dev/null +++ b/doc/jaegertracing/index.rst @@ -0,0 +1,80 @@ +JAEGER- DISTRIBUTED TRACING +=========================== + +Jaeger provides ready to use tracing services for distributed +systems and is becoming the widely used standard because of their simplicity and +standardization. + + +BASIC ARCHITECTURE AND TERMINOLOGY +---------------------------------- + +* TRACE: A trace shows the data/execution path through a system. +* SPAN: A single unit of a trace, it is a data structure that stores + information like operation name, timestamps, ordering in a trace. +* JAEGER CLIENT: language-specific implementations of the OpenTracing API. +* JAEGER AGENT: a daemon that listens for spans sent over User Datagram Protocol. + The agent is meant to be placed on the same host as the instrumented + application. (acts like a sidecar listener) +* JAEGER COLLECTOR: Jaeger agent sends the spans to this daemon which then + stitches the spans together to form a trace(if enabled, also persists a database + for these traces) +* JAEGER QUERY AND CONSOLE FRONTEND: UI based frontend to checkout the jaeger + traces, navigate to http://:16686 + + +read more about jaeger tracing:. + + https://medium.com/opentracing/take-opentracing-for-a-hotrod-ride-f6e3141f7941 + + +JAEGER DEPLOYMENT +----------------- + +there are couple of ways to deploy jaeger. +please refer to: + +`jaeger deployment `_ + +`jaeger performance tuning `_ + + +In addition, spans are being sent to local jaeger agent, so the jaeger agent must be running on each host (not in all-in-one mode). +otherwise, spans of hosts without active jaeger agent will be lost. + +HOW TO ENABLE TRACING IN CEPH +----------------------------- + +tracing in Ceph is disabled by default. +it could be enabled globally, or for each entity seperately (e.g. rgw). + + Enable tracing globally:: + + $ ceph config set global jaeger_tracing_enable true + + + Enable tracing for each entity:: + + $ ceph config set jaeger_tracing_enable true + + +TRACES IN RGW +------------- + +traces of RGW can be found under Service `rgw` in Jaeger Frontend. + +every user request is being traced. each trace contains tags for +`Operation name`, `User id`, `Object name` and `Bucket name`. + +there is also `Upload id` tag for Multipart upload operations. + +rgw service in Jaeger Frontend: + +.. image:: ./rgw_jaeger.png + :width: 400 + + +osd service in Jaeger Frontend: + +.. image:: ./osd_jaeger.png + :width: 400 diff --git a/doc/jaegertracing/osd_jaeger.png b/doc/jaegertracing/osd_jaeger.png new file mode 100644 index 0000000000000000000000000000000000000000..b881e1623cf72c2173df0c5f83d4bd5d803b460e GIT binary patch literal 117123 zcmd432T;`M7bV*2sAGaL0Ez?yIVw3LB4C5$&}39{&N(WhCVdnqR)aqcwfX%q@|4t@Wg0t$7~ z6NNgK^XCb8rSM+;7x?Ft)qOQv6pG+0@_QtNk>Dx{bq$5SC!*vOIos!`88WtCy}15} zP$WP0#{A1+u7daDjSdDTiOky}iEroFanrWp_Z*_#k{Ofx0r+KX&G z+r|aLF*n|)!>GV$?WXDx0gCAD;)R*LUT&*TGqyT0bYuM`9GRKH z`A~YOYLz&IgocLZRUR>uR96p=i;rK$V|3^_Iq}^?C`<;=;x>GEx6qwBuZ|8w!_U+e1pu2g*D9Gso45AWWk(s%n`#lPk zd8~ft+qsdvwK;rywS=fj`UeL(uZB~Egq`(a?8tp!>!Fp6PLGYf=1XMf1o)YNQ^ z2FXh1Nk{SF-DWqlifkv;XM1oMR%Y!|)$R2K%-y?Bxh(tB(Lpq+a8t+KElTF{&#aM9 zQd;&~?ryCTo72ifHs_o7aOl@wg6FLHQ(aeA*M5I@J+!+&LxUS`L>)XmEQjN~CLAMg zC&DqxD!-tUajx6Au55jJBSoHXKQr}+9i$)Dk>^(iQ?~x6|0O?Oq8_W9^BntkFYf{@6Ihc@n%Rt zNqvc%g++#(&#t+SwL;Ti`@^HkT)mnL=SVre-;mc}VqlPj|D#(a<#$MTjzN~8?RdSs zre-9C%jEUd`92NHirsvdwb@i1&s}8#g8hgekx@}v2A&hNZ^8l zb1NMl9;QOib?3FcIDa!Ut>Q7pwkdNh{MREwSt!Bv+35Oq@udVTDF3P z5WYlCZvO30%LnJKa&b%$Om}23o;-Qd&+W@*(ANhvK_NGGV_v`h%O0W}LIpEY<*&~{ zN_6pJe0#f+sn2tq9q&j5(VRlH7vv0lmQID~UoB%fA(2%fDYD)$UUXo-Kf@o5=8FnVt_%xvF&_~x`-_};q)5l4ac#||LW*RL^7h@6 zlumd2w$?FaY03@mxl8Mv*&3G0U%tQT=70QmlKmV*vhAse^6S^T`Se~?{rzkT)|$!q z3?telNndJ?l6&`77!$V$2$QjN^|H3dAD6o!tc$&mBjnUy<{xl$Vzm zuXbpxLY6mK=r8m2^=Vq;S%=(X`HG9RlAAr*AV z9dC-s^4!~$Fzw7#ju-b|8S41C{qf<^)^d|@p5uac*~YNn^pB76Dw&$8P%)<>EX#`~Cu9OC1V?9Q(=j^k z${893Dv&W7%$3uW&D93h#W_ER~m+m%M*pPERj^&vD*NDCxroIe2p$ z+#MoH$}CEcjn6wVJ~ctKrsMTt5b6=*4H3!k%P1tI4LRdA6;uI9I6L3KK+L^+e?w+4 zp6~4I)1vg;`A9&<6I)ow1L;>;Mn)!aUJc?E8x+JWDH{kMfw>sB(*OQkd zo{94A#z-Ka+r|d~K=#WG+z~w{?J4iOats+^sq;I`HcL>t>v?z$<@Z>$4S4RGw;=VM z(|Kw1Ou&)QF<3E)Y5Q9n8%FT_x=S3h2q*+nHkT(Dd3b*GYHJ4!eLhL4^x(m(wS|GK zP90AzNR^pMvN0AG3%x}-H*Vap+xYnivN?Kx-E-eC`iHWL%*frNM~|}nAijLrsYN%- zQ6-Gae!b5TNm0K3{`fdCzgng+&FJCJyV#}0c~w=qlP7&}(L_%1 zFZtco*E!8n2A40m4$O7M|81*(Q`@XZ#lNf!uitWmGmKrLM?oPsR!6uirUWyvKA+J* zE0Z!rD4f#SnmDZ<|D597xezKMQvW4;Yk6~7*}Mp09qVLS(}#22qz>RE1~ zxP+X6k|$4|q-mApBLyUYoIl(!SE{x7y(p*N_rLm_C!$P#e0WdHs+|rXp|+((zO~DT z=SyWJ5fPET44o%DVm_yzvEiJiNfd%EbaZt0q2xN~50syN6CR!hMZ}*%uzNTZEh*`1 z&=j39J8RYuBdh~;O!5Bx7e(#>)YS7#LfF{YRsbzeK?Rp9b6IWAF_g_S!Ass0f7R@| zP?j9Rh?$1G$`x8r_@fL0?)2HS!GVE+#SU|6NV;hfSbilQK$d#s*a>85O!t?SxUA2+ zf1bGlSMRa6X_%OhV6-}oCzkajA|{r^&KHgloV$GMj_pLFG8|@Qu3*4qYh}u$J2w@= zE}qwR{QE?2)9d*7_^q`$oQ6e_825l%j(UNG*7VD4SPsUm3Fs}TUP{R_?DC|Z~WhX z{{gE$4Q{ZFogD{lqJpBLJuElmH7H#0dVs@RcVc8@WS#GfoBedVyoX0QzD{XmTGMQ& z;D|+ii(Q(d~=P}*)GNF*RO9uT-FI%s_A|T z)m<8`Sz23D0rY=~m^cYyM>>q{Td&7Q>t!zU?nx+bcwAO_nn6Q2JpcvolY}ZFB2TDl znwn&D44XO-hziLJfwOQIgXhcEwbA+$94vbE?{2$HrzT69d#-gEu1>}WYIktzSf*ol z%J%oR^Q?yMqFa$Q0&7cwa${&{NWZcD93f$-U4F0#RKTUBC6=kk`T2SMwe?mhR;`sn z2w>ycA0K^Rz2cyy*ki7e5S!Ao$jt9^jES2smQv(*R2g{vm+fN=R-DT`%`x9XS-~NH z$=7$Ku_NQOm2I2!M6_?AbIpaaGJQXr(`W3d*cH#8I%VQ)A)a5Or=Uzr>t1Lc5fH^2 z58$IRf}>@p%#|o8GCj9cpe)}yzP!6g(P5&|nU5P2GymiL?3cS4lK16?!`Mxn!YX1j zK12E7;ZcS1J~q#9Hz~)gStu95ZHe3{kL@|`-PKMVV%en(%_0Se09aW_S=dCgqq_^O zQk$FQ$`-4x);2bFq$(zJ*iS3<*>`Gjf34quf(KOxLEZAuih1m=V#-!p zrHEfvyKk=<_vD)w&t#U=_nbI;IabV%xTe2f+rq*Ez=@)~{F@6GFX9Uc);_%TePBQ*Iy!p%o(aGNNWV#K$+9@-(R{`1em z;Zr(CjvTSEw+|^RE7R1}Y)@6B-Q3))X>63y*4DPMwPmOEym;{<(dEl@va+(gJUl7d zWiDxIIY|IhkW+x>8At-r6W$^KYa!yZU#v><@bcDt{ydZx^!-EhQkAD)hFXp|prYE^ z+SY1HPbODaSNr*1H3Yr^{#>cLYDve)CMb4gUkaNe;{fXAp|C?6pHx~Z0KGuN@b%}A`!NV+Xibz-R#HkRb6tl~~JSFkhcHBDcpvgwwBkYtowV5{eho)a*qzNK<%d zAelN&_J4Tf?d5CiEtMuA7 zq4`Rp6QCxjWT+=X>#J2W*553nXE@V8R;OIqr=^vK!Awl!SB=-_C2Jb+H7{qCjuj_ozHv$Hd?=h@M* zv9LNmDZ{$NV5W-bckfbwDnT1cy)FIu<;*moBTX%>hV5VhAt9muodrJo>F==MH5(HW z5T1l#!AcP6HbPmn+SAq8k!BE%f(KkkbpHJNH#a3}jW|s9b~asRKUS{*1WSP;2tm{a z0jvbi1)k;%bPR2ZvVw^~wV1+M>(&`|%~f??m&CHMv)hl?UqQ18f4BLVrH_Wj_jc&F z*9k<7Dw9wYra&C)KbkBR!` zyZ^$$);5P{0im$wmLE6nN0&Ai{bu6Kg@uKe_W*7q?Nm{Q(+zR- zH|pf9jxnlkYxkn|#R05t8z6Z;=cNSr%F)r$1SG9Mdp5p?lw??uuz1tje*jyON}}T? zFCETtJCvGrZYYbt<$zHm^KhTGQ^yTiQscMC+ko}pnfEUB;pUS8f*Wt@u3h(C95ObqLk zr|F*d_s`xZiO9&99335Pm-HGVdAHy;Qb)OG$LhspI5i9`x$SLEghT&Gl?n+uoK4?X z+jTC_ucND}u8wgT{Sty8?Zby=H)-xPT}Y_O?%OMEv3N6`Z&y2g*=CBDC zek7)bSJsm~@M&A+j+XvvB`Rmu=7P|zqXFc~+nUnO=vr3Yh9`c4=E~<~acWZ5n*qZo zDJ5;JtUkO`^VqW|J$u5>e4;>SwmfqGOn+lzf z2jIUg=-ZGttO1(q;@GP04XS!IGFo2abhr-K1dnF z6+2B?0ahPlvKY0FeKq2k4#8-*ApLAwhEw0)jJu*a=}uDGZQuT>25#=|$hkv7{!q{M z0oL#q^wgY>zrK+4dtZ7bhq|Y&tqn?QGQgVJ>gow!!ye+m2rZ+?sU44VC%m0?xVgE3 zNTlQCRfF z>#lMaC{-8LLWL-7drFgBt_vvve2L)evh5PD*vm=NSgprEi1fhSH%9@@(-fD zAJy+}>oOVCKwmcrj||)B@u?pknG3`f5e0>X$LeH@DB#lR&a46I7(*aM;DCAW$RnYF z`0+#aTU8YU?yFgAo%Pm=u`PNs#$yY8{`~nZ=rdWjq$x>Ak|QjNQ`^^|{V-ae@8j0{ zlAWKNTt3yMsNCfGg)S%d&POPy;SmuEHa5X(a0e8gsr_jr!blZ{rU4}J`amE`|Y(kPM~p@ebk&~OMvdFg9j;KtIW4;>%Q0K-ieDA zbbVVSwAf$f3b5RuTy|<|3R!WSM$OkC5SvRWa;P@9w;4G(m4O3l`#DsJ@Hinw7S8(p zI0ACM6qzVK(pwL`z8OwIw~XAs#Z{ImcpK8Deai=)OlNhgaB*=FX=lFW1N8k47!bk) zDjM+wu;Kb)`?u*mZsd*zMR1z57Y^GV`-M+_q#^Vva3fp8eymKAar1p87%1cl(6~Wn zbHPto+uJKUIOKoJ2OPd!-?`MCn1i2$c7-9n#prgGF|eF(+G`pbnCDM3vs>(($Ks?N@CZynIF-CmIooyHT5jwiV&82Xo&0uk6|esy(j0pAb*2XRoP?Va0h7RlwM9Ahrvg|_g~oboZ+iis zXNqZv7SzD5Kn_KawhGYyqmz>QsXJVQeWddz$1qo*|gRz6<5J6j(?jx2y*H=$twWH-)s z2?{opwYR0Er3|au=O~3V0qAz;n@7N+1HS(f$U52sgKlR?Sd^eDul%eeghpBe0k+WH z*UtoMromMm|KpGE)zuPEsUY>f1wn}GnU7B$WJW~IKnf9%FlfVtu|n$+r7Uehe5RJh zLdAYL;4#B_+(xv%fdK3C=d`{lC;bqbvrd}T;*FcmC8 zTUP)u^`*cs*%+klL}=C^T3Kj(QXqpu z*+$w!J+`pm=;%y{2@R^1Iz?606lem*#>VKOnL|_}=qZ7Sc-%h$2#EOQx^8uV8k4Z& zo~vr-*sJ}uX}lEN)7P+yqn(CNjg5~pfGpKhVA+Y#*|m&WponORh=`C1dz2zgZxpYs zIB0gESy@?kMMWESgCm0-_AXnYB?8Fg_xE;Jfq=k47$Ai9+I$}lswqOHKuAotrv_1_ zdhYKb3=@QtQleytO?5*j(5OItq`I!p16pZrlNHWjn4lq(G^{Q>cJwHRLH%W3etuvu zRhK84Iw6p)mDhI=$r8Gnth7MLD537r?1lk%@7_hQHQ;hIB$ZacCiGKgH51SShwzTM z%=Z?le|-Gxskb*Z8n8Jtd!@FLlAd;!-Qw^!gw#b8-v%Hep_4>ty#swuIQdoFfMtoa+VSICE)z0{s^&p)>y9vVZRG=HCGF0h{(%^G zR#x}(6fCl|wC49ETfYnM6;@$%c(ZeIkRDHPxt<+)B59bcOlXiz;o^$N-fGlyPGQnj z-vDS&H#BL&vHPh4;n|Hcgy0Iqv2U=|z&>0Z)i9AR^R-C}HK-xhFeHZxZh8P^Iw2qR6avbEr)Y#ZN6}xK@{jLjw=){FC zNL3uhtyKPGyb31?DcXTfcW=2MHg>3Rs1%iym?GR6aH!D3jSUXUrPVL+FOmtGFMJFgFXYapm6dkLO?n)p0EP{3dpCdn zjBpaQO-%)=S2s-XDe|HKpwufo%0Qe@1{Vcn4P#)Zkne}q0iq|*u!#a_-&$ZBfqV>F zskqROtcpTX4kIhIoP0t`A+zYn;mJulR#w(LQ-Z7$}QLF{r6#0kn zzF_)Ay;;L7XGA%6rxEtIWCBeEC->CF0oXJ-OB zh$>2p(WyJv7(w_5?E!7BNez~I+=sD|@)!$me=&qC(B$ffU|SVHj)U$(14?Qz(0>*d znbDqm&Semn2wDG@RVVXN<#PmA07Iu+G%}8KC`hfl!fx;#sImdKWq*|{ZS%|vR?xQk zU%q`8E~MSINiZ4k7PIPyJAfgq6WygP?w#=T+`pPdb6b-BzjD$L#pzUdAS84u+n;nX z@B_F9=<5Q7Gxfi5B8rHbfg!D3L5ql*hDIEiHhmTi2SG+ACSU@Bp{!vHZfr`L_Z2Gx z1)koV-BkE-7LcS?vE4_9%Y}bJx8&D4g%nOZyIe%M`IeHd3rQE70MV_pbQ??}=^NQi zD65+}1qGqt^?~87EfA;P#r<}QAOn^l*BBzwZ^sv1yv9Ik6-=DmgTvO=2)@TuPvycH3wLr}_@0jt4@ zKX7oUe|anu11TL3tR5)1RDe#ACylfr~{^S2a0}>+|cusBYXmI410Z|v-+S-~```0H0!)txEIYBUM^K;6G1JjarUgu_(;o!Dj{qDwd$|H8Y9}NCYKqD|ZWOBb zb!%H2V*;h=?(Ktjp;CA{B4OGstKGD0cjl*114kdZ+4u!eDKLkTjXRuadtY--<9tEAO>}b?Xn(S;}$Xd zWj&sCY)vdX=&V~Y1DyJ8D!6Zl_PkDh#1uCeKF#C!WGzyLf4k~o;YK5tm)0y8t;lWJ zrZH2Xq-}M0cE@fa_PWe4*8I8Ve#NQczk@C5Ph8Psd$-b}(-ZQ}KIx0@LHhN&N{(0* ztr|+n^5ikWn}XHXyiZHhtZIMA9e#SaLavG+*+BE+qnzd76K+2y>?DIma*DSYtNzg?Mk z5^?kLVgwFE?{+xRTBa&lg@=beh|tb^x|R#4XX+l_n%t$(Vo3kz`i%=87aEeME0|V33VFBeozHPo^{|``QeRC02)a} z8Rt^l))qf}?7-dXb@i`bk5zitX8TTG`HHYFUo=(xRO>h@;sFgb;?tjQ-mLEt5J)o` z{l3k}_~77N3x_@+VO==>wqHoD&2s*R`n$JHYUeAX_>)ht)C8S)eR@zS9NOQ1SD73y?*%fA5isS(!~M z?(a_wkVW&R0;MS-B7)a5T(3y@`R;b0W?kLEsd&vim!|gQj^ugYu0Z zVh5WOzz0N21c0pswh~UueoaK-#iuDVZ%LxjEzpPO+D^zs3Aw9-7<-z4Km|-h?Vuz3 zxyL{k2#9YQbbX-dW5jv@K(2fv(J=z?h~(AwOWJ5T;OoQ8`}_O1pm#?!4p0M(f&0ZK zKz&yNmfQtwM=ZN1j(GW7Ieh4J+;x@W+uo$G9emBUw2V)bclT2(*Do4-8=-F~asJ}L z4WKh%uNlBs2b$i~OR&HxN>JKOQ&AckQtRnYE%%k!ucs+*Syp(MSZOQ#)yWDTJi+bR z9Bcv6`@JM zzZX4et5F?5PDDmCE*EZkT;rM8-fnWRinyTTIKc1G~ae6m^e0BEu zIa1A%{C=Yz6*Mnzc%eP6g97}Y-XdB@k8#2Wju#b$Z8tJ9v?{i=sE572hB#r`rQ$f*VR7ohGt;2 zudqr~=Xf@bPLNEt$yU%Dy155+>I{<7*3#rfU0YjMesAaWW|jC?Zen))PB>nGkn`R zH%C$C(R+h$Rnzf}QJmTGH^KTuN__r3STwhaT^-$f;(ab;XLAU!DNx6UZLJTO%uOW6 zY_GK3lnysi(Tv=49lfvLPZ4neJRdJ!{E1i=K&K_?Wd$cNEOp?yhmNnqEeSz1eBg58o-dJwKxyl~ptEfH8T1!m_?n*_JGlZf!Hhq5KVmCFjFr>S8+URLqYC?Bk%eEZ zzyd)<2evsz@jnhCr9+>c-E#q_qw}(>YwPj4MaKR6uYKa#nzX(g&XKVuJEs*bMf(|rWw?d?4pE#4;?3nht`&3fg5!2OKJg0j%xxwYeVX#OehUxq&!? zHko9u4%{$+Sd5L05q*-XMvpBO8gfMReaoa42eeg9U*9fvgb`?S(2-K-*4Ea>?LArb zz&HBK%K+@f@`C|L%h0=PAv6Q{Uae}ZQC=C#KU3nUrS{@!q#K1s!EmDugh7Zc6z*R? zKUiSBNwyXeGq0$&CHYD|P*PIB?mdufJrlt-74Ny%68Y1Lb7`q{0I0zJht1MQj*O4Y zo;{IA_-C^Yt*j(Aa&B&QW~g#&>eCT&L0j=>;_=P{xjSx~p>)^DRBjX%-sDuy1S`|m zEGo|BwT{rfVl;M`ot+vzCmE@MEscx{c^jki#PLf$&~{VMc6YwME{c?kx&AT@#6SD| z+|~9J`I`u8*`WXFA{wE70T{O*X#|CDQ@F>5ZE=rOXM> zoM~fk60mL2C8Q9b1IpMh?>VS(KqbV2ab3seEb6D_ExXAm-xsRMFJ9QQT|S$p!F})8 ziIsc@)q@-JkdrH9Cxi;>^y(=Bm6*YYXaze3EQMBqOwj(8R8D1Oh_Y(RjUe>7iKYUc z%w|1oU``H|aX+MfSScL`H;Ur1 zzWZcC(t!ks0u1m=d0@vO=PmHCfjNU!Xk?8bUQeGp$Gq$YOfceWXH^X-(kQS1n^t?E z@LmeIn6z6tot&Mu?4pMJtZi+Bz}wO9G~%yd?%=?c18tRN2-qd1EX!82tNdLP>(>Z{ zBi;OO9zblb8Ep7VomMUeN^QCnTg)7PU578ymYA?7B!Vw4-H0Z!vBR z%!Byz8rF4HASLV6Y(0>;n#UtcVnCq_zpeWmQCh*+tU(qL6_xSo)mh~Xb!F@&QqnZA zeIOQG#9=R)(Q_5B4Nyr&y@l2!ufX1b#4{q{@9gd}E-BO@4o2`yeTS?74|#kJ1hU}d zWW&T5N|ePP@ciSbrdB&?fsxB*xT~O$&3X5qtA8r}fcmVJJs!!~Rhe3(6_31C$OY4w z>FL!SMqf~<=ZHsOgH}h2!eX?J9mCi6Xz0ep=PiCc1!&0F7$};Z^@{K^%NVNEyqFjy zEp!B5B5$m3_lw|4Z7llQ1DQ^rg!t_hiGtW`oGI$cO_tpWc`sjfM@FWTaG5JSX-V0d zxPUDz1k^JJqb9(tW57LBzBSDVENTi67r}H3K<}Z=hWmIFx+Y*zG;<{f+2&Phf;VFUNo{GAtF7Z}z6` zcV)2GvtbUf5D@wj-fNp9p3Y-5o=D6_-@Uog;!&nQ?^Zi?>9 zwHjhI=`T&q6Fs~P-0#Z(d)G<(S=C|ZngM~L!rcc(mlYx_E6I9TnBpT@s@YI*m{MHZ1H2^Hxxpq{sPMZ z^_fmf6+hyJl@1Nc+P=PqxJb_5f#>C!%z3JMCFa;7-zSC zOEZ*0Ts7CGp~aV$?C(#Hg{jyo##^`U0L#tiA%Mo_ zT92rJ(zk=%s&8uQTpBySw795JU_p*gtdoXW1v!1Kr&LG-EEDzD?s^68($x~#lLTaJ zz-9gV*(o7w{N9Xfxw!-f` zL34JX$zIG*#<|g*ptIL@ym!J`M#b1ZaA50nMYZjHhSZdHQ;mYDAx_gK>DURBAOw+> zXi<@~%S=WQ11l@2?;nBCNR1W@vj0XDI+YybV7#?b0X!Go?b{-|TUrcgQg(xc@p@++ zhZ3+tw-m2+<77-VKu<_ZPj5fNT1o?6ohxTg_+NFb?CKRnw|@5a(dP)CzT^hw->IYE zAT!2pIz6f+RClG45*r&!;}dIHzvtj1y0Dk4NZ<0NIA1Fw=h9G-t`$$?&p#utf#z?J zQXoZe4Ch!sU+!>)Gxp$DY1~u#Q z-cAJxx`t?(m@5ro(kM=B4R@W%bscVyz8L8zhYWd7B<7o6jLMpGC|O$&zQ)RGNpjfc z4Qlc!X{KuxZDuPgbD~|V%~Z@AYvlt|mL2u(`a&oSmbRBTuB$JwmgOc3ZB4S1i2ZZe zZC-AjZ$v~UnlS?dJY=mg7!TY+Re)I>ag$&?4tbZ+X(;zFmRgB_bP!WK68U-)DXotz zzYw33WMwk?veqc*g4CGh)?j*1!AHKLzyMgGnGW9GSeA}eQ07!F7|`irh{%5DHZTAC zY7Ap`btX)`MV5XJ<{q%6g|GkKa#9Y&_v(>BB9Z>lUsSCObN*KSAOLf?FE@~aQc_b{ znedi*ghTK9n6ICowY|~dao*HvRL#WjuXAX5pYVQgLa{Ax;;6tfP{)xNu+x(?ge^aT zA)ADfh3JKB7p_AuKrH_PS=QIDe?Zw64u1P~e){LZ_eI_;i;vSY`x0Gz!e`W+uc9t_ z<=1(K>~B{P70>10HtEUFvsJgVOBf!}h0#ci%ixm|&C3VV;$F7Ye4k|ZqA3`dvW%W~ zkYUd*Ew7k8&DTs;yW}~tw<(M98RR0^UkuD7ID+!Zflj_x{n-F%h6Zg)Vd2-~=MUn` zDuiATkZ1Qfg_|xY{?XSWSRWBz!WGNnLCskBq82+)`ZURpq9PrS9y_K*ezei6puo|4 z`N72I>woTlyX+w~N~-eLD;HQ$m@JaXxx}7gJClQNnaY4Y6=Xv3Cirm9ndvBu zf`Usx{2{k$G8JT@=!Y?^8?uU&Kk{R7#`7%JP2kluB?+#Dq(D&PQS`-zxDDX@t%_mf zD&f_ube9S6&M%Tl9K={Guao{mVQKBfD6#+@ToK!(p8i80GHm5Fc=Ev_FUPM5^2avu z`mW8sN4}+zQMXESVfwFwhrq!;!sqZh?@Ojcg;p|yPoLU`{pB>jPXxSFSEwXb_s>0_ z{*qNtlhR?3(@)NI|BZ)7X3rCvFW!p>ABTS;*DV&h(J@-wnn$$zA*k@rbR_JDItNoc zv=moXZ(Y7LI+iGOyJf5TgNO;qh1uz!5q}^_>hP!VS^38nKBoyMt?jAM?y!ht?9ypv zUNW1@IM;73q|!@VVV9;Ra1ngDZSH*~#eAbQVvGOoz~H5^Ia@qb%S11){zNPyCp(XG z``uIbQatqy%_ta@r2H#o(relC-NeOTE8^gggAQ@KyF_yfSsov6XA+D>4uu^ib9lqx z3A%sOQz#x+W08Ay_+#NU4D|WoPvH?{=^y_5pZK7FCb<_b9pENp2(UEBG#9 zxZ0_83~?m-`kr2$YMV$0d9ct5@SX8|uNW3F;qxVDX_bh84+dZ+1}vXwjEAe&c#!t* zfycwJU0Yc>9IV}h^gY;HxARAjY$mv;iEK)6dyC?HJ_mA{BjpUj1@gdKttRriHss`#3 zYo9~UW#|G59~T@Zk*9dG=*<$3T^D3z8B8-6;B{dSfR#_j2q)kBP+(hB$&(rk+F+0% z1a$bl-EEkfnpbdfDW*y-EK~)XH$tu>5)lvxZAo_lUOO17KqYFbFd#FmM`!;sb-k-kP!&3G>e+F zCHa@{ZJA2P2x(=NF8P*k&-0^Ob@rA>!1pMLs5W5xNr{Xk6NCxP)p-fqw=X1P@K*RI{Mw!U3az7~pCr%5eN{t1j^{8$}^f#*j$c&8P`u~F{njWLx&uPcXA z94QVG*Wf6u=yk9RD?-RTC#1{-mn$H|DfKd!JO)NaYGs|j_u#G|<}a`|aAmj+jCq1MnAo1lt;H;(;(LdHH4T+YX2s4e9E^kO*Q9H!_2a zu)}b{h~EnFq(PoQW_-b>ngAZ^+ABSaOvHZh_L;tFL}c?LVMXkqkk%$&sbS%D{g0}o zu9uIXcv1%kuNekvqZ#Nv(7}ZcKFncy`$fG?^lm+Nl!>4-<;ok;t&E3myQ`I)*24|J zW#VVMmhv%PNF_txi2{#8{^TPn#~pg88z3qO$D}wp=V&Y%M~Gg<%$|MVWDCm zVRW=H8E@e1TvWbg@(e7j;12fPS}jq6{260Y7JheO(UViqb0*?YsORA&#GJH>OG%+a z3}IkXhPnNZdmGilh=CDcSq~K&=EW6YaA77LqsX%gRxz0~fBq>4)_?GlWxxyuvWX6m zrcPx&c|CKhGTbnN{pX)n1V`K%stLoK<34@5gQ$@L0-EqtJBBKKLSDbNQlC4DvZArE zU4|aoR#{sc$e+jH-BJ{ErI`U!uAGU94y>3uz!#8weAM$_1p$4+u3$S}%39Kw`QwL? zmg}5yavn2eN4SgA&q%Zpl>-JntNcrb%;pcHJ(dEnlq(ESm#uZ1A_HoAToF-iV7vv} z;8jpG!ISuCX??vD<}FN#JlWaUBAfomYyvre!?IrhS;0W(DMH?#X1Yj1LhJV}4iu$l z&s7K+Hz8}#{odmMvLCWh4X{^K=)}ZCSVXogt^&|SLDAtK(8y3h28xb>LI&|p#p~lL zZ{!x-gL3xWZ-5lI9o{#=bx91}L4IPX|~B z6StG#-E89cg>C#PW+1z{&cfcJ$51w_ZMDJ3Xk)M(f| zq;<9{Cj_k201}XmK9KE2Bqb$D&G=!S1iIX2!x;A@#;o-HI6DOeEO-**L$zJd5`h$W z*k2N?#Sxz~Ob8;q^F4m+2gQmM9l>5zyg|uRCH4AYq!_HO6V?r`#+qSSbz|Fm>rc%(lkHsfgDPMqkm>0~HmBa~BqL z3g8!*K1b$h!55w|eLWfM1Zgno!8y%y=&Avqh1C@@TG#t?#^3Af6RM|AAi)ksL}CkA_N_?z5>U6ESM3%%>>$w2L~`%3@zKy36HPo>XX zCq_0UfQ>&~x@V|pX}JR*#q>$TC?>R4G&sCVtv(Ji0eEAfMF{)yFIx2)@E=}K`wJ+4 zsL4h!L6WTL0J#dr0iVIHKaCy_|9t?xDmhN$S^N+PP|{oJLuc@~PTm)Xb{hc2CGivg zelhC*3K)FHA3|7-i5LKGEl>-5WB}clVKV?2iLN)u5deLYoZ#>ZrQ@Kn18F?u&712v zUFpc!TW4n{{`wJA?>oGSGVoc*<~U#=aCYMG%Mv~O7-^!K=3Kri6I%qb*N(M49m?yB z7d(9^-}<*ME6K^l!f+tQ=TI2F6VK^qxTnrU3|k%)^PNR~I!*(&XV^n3^B>*Ii>8Ii zw*qoCjM*ZuT0Mc`q#@qr!y^_T|9$ue-1n+q4cy@i|6esDPtKHYY=6BdYXn9A-*bbb zkd8fN{m)*AAL)g7vdH-C?n8Z|)uI7+0j%~3x*Lb56n+v)$4^&1ef=p=LplNZA-k*v z2VVjP+Z$?toW6ddw$spYL?M98SUef$X9x4sIK!rBYyymT8-ad;=#MajI%>tES;~#_ zVwYD_ONCMHG|&&&NN4&hVa(??(m}vDEX#|`&ANt$PWGnKl;-BYCUS*$jUylH#)tgB z_AsxA4^GpHrm-Zt0tg~7TvxyxMK0{RgxCgPqN7D;e+z?dMK*%~>$5zi+s`M+raMh8TRGI+TjaM=EgaAaGae}^DSv`G)PWy5OdgG&oI95t z0Y4xjRKLp%GZ^*h0N}xv1I@tSCu6Y}05pTtCJK}LOR%w6UZ*zh{7rGt=8Rz^63NLh z`$dZ`hV7nKVZy1zDG-G+{9?qB21T{I$Tk^<3oa26QKQpRQsmUt!$Gemx^Th9SA|7W zU!QApqkXO+A|&J{h!3!7fg~2>zNBYoLO~B?T$zk-GWz5t1S*Dtwstv736t3NI%PYw zD|RFPooV(nf4FbYtwaA1MCA<=QFcA%x{y7s0@e@LAB)+9*5W#hAR;a-z?`<~nwpVh z1RzPV+ruV^Gq80_2Lym!4Rt6S=Lo5Q(+4Vg5RxG`=5_l81_~!H9z&7BII$FL_J!P@ zU-7&htglC_pBur$NoJy|ba{8S%!1F;H| zZCFqME?w1dUq&44!^4j~+~-(vc@GrqWrsj|My3K)Iy2Ewt$6gAHaFk0hm%}*EVN60 zc)DkroiRK9FpmW|`t<43k!7BQWIWfv4D;^YWk9nqVuj3oLMw<&%5cDZ=^QxmLG0@; zb+*k|Lk6*s{5E?U75fIFMOj@v4K{SMpKKPP397+g=X8Dai1`w6$Cu4=(@v_n0na;t zl-_((n2gkxduUCp;V&+l!SzZ!e%W67^TyVU46e#2e>tggIH&vxJjRd zF^7}qUo>8&$l6*88_=MCRe0AqPB>Qv5W|qooS+?`gMoTv6E0{Ha-oM%R#r|#RxX%W zUX-mYSM)<9x+0qbt?3}|H11m+ifO9t06c~#-kRc0Xs|14_WW*kUCXk7c ztpSl{2>BDJ*Hs_k9TC_8K_1d3l4E|l0uS#DO*`MKW${oJ^3^uk5 z7J!5QPf(0c0*5}JP}r~{3)$2MLTRL`&x*_O5HQLSdXIed8H|2kU;iMi!Qb`70N5Si zTmr@Oz{0p-uk&1FE^#rOd6^$C_XTzbzx6g#S!DqL)l8^-KF2l`!mVJ9GD2Nhi9;Spld z9Dc)=@Dg;Qp!WlNg@A}p4Qd%Gz>*_a7lybq3H}Rf-vL(T)pdC_u_TIJQ4mccRuoh^ z2p9{BQ~_yHR77baQUpOtVj(J`UO=P>D7}k-bg+Vm^d=xhkS;}fXV%6LGv7Qj^H1T) zpYM-e?!E8(p0m&1YpuQZOx@(T>C>ln1_Udw_5)IQs_B4Cx5FJGcXrMX>U=F^E(DA|79?t9+F;9KF)g3Z>yuz zG)M9GmookIKy>Jbppf;-$K176Nk7`RAN_@@nio9LX%u8ovd-$l)fS$Ar!4uNNY4*5 zLB95nc1#rleEZ@( zW1rlqQ-8v$ELuITmg(dqg#v^y!-!XATd%zOWXc!($MpVf4!$`LRFTV@Q%5ZC7>cY$;}#01m!so`+pU6dA+b_}6o$12^uy`eor-a6ag6E_8PVVFkz{ENtLy>1jM zMj#u*iJ2m@f{}ONO##pXX>$B*TF&=opcd`xQsgx$Z~G99yZ^o)XexMfgA0ancS`QDO)jX|Ik>7~dnLWgv$*OHZzSCxdVKv11p} zv%K}U-`Ru9qVkl%Em9Z$n8kgw^`k#6Fw^9Ys%Q6qXMU}){ko7j_jUHszx(GHJ*gJA z=PP-ih>P=x$vJViGFQ4H;aq3K+kEcmP=qiKBM#$o9h;#-o{>4*V!{>23YT~~e+>9L zWYrJt3QZ}IGB_AypTnb|>7S`FJendB8RtSIioQVA^__cu`F1B9gBx4g9!0YjHr{Mi zoH>>7S0;}1Oc_Gwii(zk#@ow#YWJFz7iQInvDB9$uPS|f%>&`O(0OYy}_tp_>U@oy8`;R z7cT*4X>d3`NRD!V8jxQ`Ledn9Cdm}|*us*w?P*fR#NcuAb%kUfDO(9N?)g>EzSATh zg*(wh2zh!GY6puZchzzYaTHn-<2_e!4yC21y92<+`3%r<+OI6*axx|FOO}|HpZb1Q z)Y|~XOa<9OHfrrf0Z!EMP2MPfE5e~`Cht0&M|^DpsD+z8U1;CmgJxd?GCcVe%f zNQd~y=eSJYd22YOewa6?T9=WI_c4JBhbm$NLTyD zsMK=p`|N1-*;$_CmRcR!y^-_7kzMPegnF*{52rkShaolFb-LqgC?-*M&jkGqCGr0 zCJS?M9w?Ca?2!S0Idv~ksd)}gPt>RTYzG5#KADE<@a^pxc<=Zgjd|7`-r9JcXdcRfKA&*s(jFo&L(UYXp-Et3aWHs1Av*{e{-scU_~VBMgW(mO`oB}%!Mlk%A4eM zmp(5=u4VwxeG~~a$+KaAqd+F7_&CxMN-3hzl^W$RHC2XC;_^c?6frqFB`7RGHLFLnyv)NB6(d40#7T;Xc!APCV zgYeo$x73(k?27gv4{cm>=RhuxLCE_BdU|@`Xym5)B&^B|<6?_TwUlkR zGp@eL^=M3n$nolDou;+Mza7)5Q%k*gWu0|BH_UrlJ7QGZ@PBZ#>40h`!3tI#yRjZUn1 zcrSxf*S;e*_yCXxtxhycv{T0k`nQSv0;ReaZPF2>sfRnP0YG>1Z-N7Mn{ysEBTP;S z+=2CkG_dzJf}*A-yQj$M5ZLv!w?7JAa=0LX4VIeZDGrkq2 z1cg6NAz%kv03Wz;@RZ9-bSj=ceU)f=Y)Az~#d(#TgCbUWHr5Y-7Y&Jae`aA}fyL_| zoNYkZBmsF==tyWL){0b(T-m|LcR=9yKr830S(C5=yT^&3{T7T|TTUD_KY(nA4Pq{f zXlQuMZ}xfqL~pJ*2o%M8_dcOIS5Z@IjfshKduXQ&TJm_*`WuY)DtzCdz!9Bw%W5Vb?~LRn{&E2!)~4%vsfCkIuRQ_RXi%MosYWCZ(}6z<9DB$fl{;V_67tA zUaPg|(?aMPt=^?AS-(C6@Sxl!8xatL65l8Q# z$w~X0F46Y!B>RNgmMJo=R?xa}BZ{;D+179TJ2mn+0Od~PlFmUOr#BRsvwBDaPt39G$Iq{aJ+p22WU&;&?wRU6|skAcR z-_o0{vtEA-&%)J(*_ZY^WoG&TS9jp3ES++@^y##4kR)J=b$0#d+p7~3&CFcjm3J70 z7Yw-F@TY57Ru>)uCy4{ILeC>ee`@kdRpykVN!r&ckpMPESEzcws|4J;IJfTF6d*`Y226${)Zz$hEc#oLG5B=Y*w{er9E%AH zIGhW~@B*$T$I;;gn->Uu5$0R+m9nOZX>4?YfvLPh1KCf@H?3ab=$7;Z04 z5Vs8+14Nnjd0VhFDEmWfKeT@U_K0s%3TLJkK5PV>GwR##*VUld(6=WN`qd7sK&&97 zfzB}bi0JM88Ed;0jwk@ewn)LC#)ms2={9bOUSjn?SLy;j-K+>N-WEF?%n)b_Rd46v zsX#9d)Y$_88={jQ`(0neu|*1>X|c-110}cpM7T#=9Uxkl)9`Qev>9z%!?ydg891=A zo7Kn<@dRMB6}L&=7gYQptxViRNK?X~{y2i8FcoxG@Ph|iYRqzFfj5=Aa7g&o{P1(q ztXn|}FJ54=w%5(toV!3?LQAaS9u6^NN$I2Q#vtIhG*bGxu*Y4SuX*|W#_z)li5{Gg z$}*#nBy5Fv2>7zb1wkLyg%DJi$qH(P#OfY0Y%A8o^SZl9mMsP=46EB=)tycC>)75c zDD%tHVg~AF_qyG?XIXXWMM=r4kqKYmvHks5NCXsEb?JAeO4O*F!$__Y#aaK~e`xo~ z{Bh14NIbd*ehw)51w{_59rG|t;y4Ckv}D#MNoq6L0>IRvWJ2a39D?2X_S9CxojjbJ z@i66X+$aT<82Dh6CvROLl&--n?BPzQ#niP!bGGu6BKFRnUT0W>d{B1`i!rr$e7*P2 z7bleWI%a0Mr2p=rU8Mw#6D(b{LSUq&9-(4a(W*N)Db>|ZP|$+gVSL~Zygnx8PT!dx z7WQXbL_~^(w|xlfeVJo+gawf^qAZ+SO9cL#CPhsiLnW*_+dVi}vU-G?x6p6|AEcWz zaCH5PFqjJOcU`~VU)XS!n1hi`UsdOzO-qH0;A(sJ(GA?(`6D8`pN>YmK~_SE3{GRX z7n(&0xHxF{eC;?=R!~U>jZiLZJjQy`cA(wzT@s9Jw#QX!Mx0<@za9*F*V@OG+1VlG zm*wRPL|8=)@obQIwYXJB6#yWAxL>Ja4^Pr{xTqToRCgPqQsz@la5`Wvw01We3!1(X zaGcOQhC|5xv5-ij71+N1g5to=Xej5=%jmiS`W@1BlDykutkc0wd18NCXJU&1sDvXj z#v7$9^Ldb+W6p|FW#>hfbw1mqMu##hD3`4DcH>`JHR8@h zVW&nxk^a!Hud&R7HL%_HfTHEP`7A8={QSCOx+fkMu+A-fa=rH@7H0p~Y@OC1k6Hz5 z>jgLpTE>liO|*rqd!97sjVUD72$YXh?%BnoX@5zAf})?@Ga2dom!skJYSS1dFDt(x zPWUkJ>uc8zGT$!P+bC?99#<{z%Zgy@bL|nHM)of!e3+Bgw7;)I-#6-V$jWUl z_S|sEjNk+xOD0HElxkU)9m{W-UC}Qo?!Me1aIN2#4sVCuo1<-0NWB;txgCau@wI1v z0xU4JeC=9_5kB1ue%_vcvIMFx%T?O&Yu0FSaz`_V`C&n+_DWuL#B(V{%RgG&Rcnim zsj?hJQ(}9~Gf-}}-8+=6tsg`upLt+D>MLe|PE5zZ_LC1GCn-U)*0I9DXzEG4Xn}{^ z+Uuh-`2h^&#ZD@`XfSGOIbIqhJ%JS*y$BGF}C=yp{6!;3?*7t zHOd^8 z^JC&_Aa|j_jP;Gee%r<%4#`IFMx-(Qp!UO^Dd!631Has3bEJ?KZsseW? zg4`eF{I6J|~!V5}1J_n;VsK>}u(flAM0VYi0b zrlkYWB*=?vPEIsj1xmYwDohMfyCOzHmS~I=g4XDz=Z|QZv(&hdot=%QpS=)<Jph{)U(|fh4L zvB&zur9<3~S#3D2&~Ck5>Rm1u+!KSVp2_Yp(0-ej=Ya*-oF{=iTOOjiC6&qMiHK7U zB}hLF&2E;OYO8^PYEig=L*-o0Y@SQAgha7$*aeJ>P$5Z$XI!E|Y?n?BC|`PEm_U^x zMw2XW*L04`S%6&z%jJ7kKh!0V$g*EGKEk5n5P9#4dTD z?;J+0TOqBYwoR1uI2jQ#RNEl9Dy4BBFkjp%D5$!4Pmd#wl!8}Zd8G+jumP!oDhwVT zwx$_U+ji`TI<|fA>(?R_T*_){YDP6M%+tF3^^FbI5q*3(7(&AB{C~d!WvN3}_H$za zBT5#7LHJT;@_$nX6V};Wr>*<}VElXr8gn2#+xUo)-RK$K^dV?_xn=z(t+17$iY7!t z-iy_xU@MC2T<%omz7~g2Xs{L4)E1!1O{x8WQwg$(Z|{#%3H=M9$c6NgtM>NnY=cvY zS+se0S`Rq$h|o~A^XdE*$#UJbHT>45hEZvz&}Jv{siodcM%0GKRgF0J*p+H~|??}dYmNaj@kuvxvBFFd3|i0{7*vF(cR4O)-H}mm##2sGpht=j)cqhRgNu8hEsR_x6N+g8rMED`7Ht z#k{q-F85R{*?W$@T0dEMpa}FgUq0aGlz-?WfMOz^6icOCLHvmNyP-C+OwtUBbVLW++YA22yTLpt?ldg zmKDvt)5OyZN%8+nGxmSq`2vbWA>i@jeJG=`j3Kcg0=+&9yau#2oN<)|o2oj^a6o%q z^^s&Ez~U%-3kd}tA$SPOX-fS}f`&WBNW!mR3p(o8t;=iZm}zQqlr&hSn!01vE#e?f zox&Pm)X%Q(=-`476`>2Cgh3+jN%8mX2P}MpXC}T9WTD^008odJ@%?{@0b-DEp$4Kq z)jR=u#yWq`-|z7J+_;3T)w)%B>Dg0L{JaZu9e?qjpX8YI=yB4?Eh=X;JVT22oKd+b zqWA2ftDbF(TV`>;g|W^TZ#|7O>e;)w3Z$a=V8HNk$x2hHIlrGiJ9qVo42R*y$(xRk znOBA{YkE|?`Peb0EqVP?jbA~C)YX5h0^|1oyG1FF@d@nO%n#oW|9)xV!X^LG^X=u` z`@j0dbzR*4vBd7ce=L$XE94ahZ}0v}PWoji=B+%wvrP<^RqNL~XylfBca;PkHtVGh zQqzlzD885!g-G$?`X4yN zFRL+W@(J+VDkt`Z05~cV)16O z=vKft1)ydiBP}I$4?&wfVm2dbKVV-J-^iGVwz>7Dp^KS+1R}Fk>6u^m2?FMu*g2NA zz~R-)mp}(XXzCZVgkJEXM*RT{t450r29$u4@g2|%(cXo=(r-VeTyd!jgtTb%BG|}~ zOvt&>Wo$(G3WOJaG!c#_uv&S{V+du-Y+C-}pB2oM;r7V26kvyjVNZw2HYG*HQq&7^ zRakmw8?ah166%Zkxd0Rk5UQO`xScg@N7Jk6$22o*I6vvhlOE@C(Qb`@t$T*yh8YPT zUgmhlvFCd~;kQLQ{rmJU{kil1B|!3D^UHtz7rhP)cfeGqxLDE3V_JX&g&pK$HF`5| zUbqD>seY;_4K_a|by^>cG0u~dlU>|1(a%w`i1Vcmr0f*1&}FsA=~Yb=HkL3-4w$&W z;+SR@b#ecrK$86QscZJM0tpNofJX%^D@1G*Ts6aHzk5UJ5x#P&y-p*vzA{&{oCpit!Fyrd;Q zIko{|hP%H<9ep6QY#Aeh17F<~=_Tn{T-E@jWqEbbK{W3~G4-gnIiI&iM;Xt5ps!?mxk z&$Y%xqR9;3sgSQfy~u6pKhNBe%|~vt*0_K!-jjtb(_@Um>UI6U|K$ItXT9g67A+P{ zZV%~P51tCx2}L5{fQE)-<|OG?$>-(y^XF{2V4ph3Y7BOSGhu7b}`zTF$g1y$tnPR)OCrSIYnlCs8BI`}9twCo1T=)Zr( z;`H~tL_XgTC*W{xL*EQ`ss|aTkiHp(Wf8vHc03m;MYK8s7LLT&nh5lBkXJh7Y6GaL zTF*O-dHZ5089b&_YV8I(t5p$!`Gd&>8MZaVg~D>q@cpfuHx2t6r^TiI!WvIc-+<5| zkisioO-*o<<1G?K0O8N3<-y`66x6Z9+qxFwi!oSj1x5y=x8zy*IA&uTLa=PpaAPwm z4&$mS;8O$@Ke|tQY&wK|k+h;`2_2mZ1QdbdI-4*Rhy3VQYSuvvvM}Okc}Je*G?0xH zvY!!EFcKB!&N2u%=F+kflfwa+LB!vf0H_43pU<>@uhxY(E1}Rc3B&9`Jck|XPY*!& z3Xt(ARvIw12$&yp{r20-nkFq%3IY0(l-y#og&`ZW0jKx5v2jD2no&K*a46uS(=&z} zsN~UR(sU}CR!!p_&~15J!k%V1fPjrc$hX{FXttRLXXSBea0;p{IF&FJH6MGjODZxk{WXMLIj<8GCOWyRE$<1g19`PT5@SGgGkvg|x z4wVA>s_0~y!e!IQMMzsMu#C$zVt_fE;SDI6T$*G~&w?uZyLbIj_?OLBlHKG=(>yT@ zgC>Lv1oLR6C^IeU?1sv$+xanIRIt`MK>JeDN?0{LJ}Pll0ydf&Y1Xmg6a7kdJul{I zB%1|sr}ry@gg0jfVnI>SGYXp~Ot##B?hxvh!78Tn%IF*q4s5KmXH=H~<%05UdRlR= z&ZJ`Fog>IZ3M8g9{TwEfVX%A*Cox{ofqP%hVfFD5Tx8gCRY1~15e&5d@*~X21fD>X z9$*Gnff6_x;GbaC$n*lJKR|Y#7+?s~4%O8qxZ@(|04SZ$pplBi?!srDL^fR+VA@qKlzBhp;~gssNxK=V|&NNTYD z2rYY;=!GK&?)=Ub2z_XVsK6q0?h%B1ACy}ekd3CsMhyA3CYRAHt8KJ2Fw{I0UF2A^ z?EnN|y@DCmd?R>N~CGPoRlNxWzaY+^hzM5z6W|J3W(@7K&wnS zoApPiYYkzPrnuz*#6lMZ1$d-dQ{;2!C1M#K>j%-6AsTgBkIJ(TT#g>(te+ z4s~Jq=LdxC30K3fkZMM-#=;dNI!#3q&&KT!T*TPRyW8yZVZ!B7zC6t}%)_$t;B4F< zg(rkhUa@rP(X&HbWi?&KwQ0#mjp8+;n1PT6DbTXOB%g4Ib2OP2_0MP3ph2DPFg2rR zud|h*9%H0gG?b+Sx~<6ADCkmYU=RgQWX}{A!eDYp)09Y>0VLh9EzuC{HJ^C9)A;^;k5uxotTBN+}EA?5DYJ_|oe1y#@#?#yk zYlbc4P6^kIMUGR90EpCNmh5amhydU!q*=EOXgC7=_!oH_F^>Yh3U7l`dN6OJ|u zP2ooSe5kyicuK=fF18ipN3t`d+c0+hV%Nv5sWsZVK(ABv5;ShBZAzV2SZt{By#9!2&O=}rLN<$Iy8{-YBEQ4wbZ98YB z3=0!5!OM?2t!a9!4SHP%V1;CeWf8I`xZMk6dNi#cw4R3pV(Z7>9T6E6tHi`wG%zI^ z0OzLE(YLy zN`dA6)&*LGlR+>3`W`K${$v!3mG5j4j}d+}SBUaK{E)CJI&;znk`_5gwn*7aoz!?z zplGH~fQb&BV@S6k>iZoh5ey{#@w6c7#gOlvQQGNMAI?IJTQvFvmI0V$t}H2;wZ^Xk z!3yX-DJ3OW!-L^h#Ax)s4LT=wm;eAd3J(|@UXYE2;ofV5M|i3c1*0!#1haU=4<+pw zLv$A1LwNvwDTPlAUq^qspNcH{rX+hR3yrjD{(6j0q(cD}N21a?r36qXY*Bd-5a@l} z3}Q+FV`brcbsqU6NqqS=cJZLI9ip6L7}tVJpdgA~H_V`YCV9@X#;L6Pg;8{}BqA0-5-j99ZiZ)$i z<49;`6W~>%M#WM+XLwm2H;4ZZVW=GQ=d zJKatdi5#nx^>^iiMz7SaRMk9c6zPXqIc2M@LV!-SIN*uMkL$4e&@-&4TZynmz%i+{v3a&j5t4+KGvtFtduBaEesT5c zRY{R=7KWvHcqNXtX#;KottcBJlMzfiZ-cyv`bWvMnu-NA9&kWe`#+v#yYI{5nnS~$ zXreOIsL1I2Xkt8YAog5RSs(?^dXNsnnHp}f4A8q7gsEf{obfo1Vd4!2QeD>2oht!5 zW69ii$V1Rm*8Vh?n>VF5@7+7ag9oo|l6jzDR09uLoNE>B+(KKo=BNp7-1w$dJW)em zKlE$gjT&BCYwNdd76M6=DF+T5fTxxkDj6#Lu%AN2?M7Gc*jAvrZrwV9A&`#XP*O3n zhrcPrT5>k&P&{*n;pu1fx7T(fjgs%E-Q_7G*?8@`W=G?YdU_q$KdiU`E4l1CuI#$a zfDQNu%^NakCKhTAIN&E`?ar3?aZ%lV_3wnHY+dGlslAX|9W&Mr8FUDsp)W=+$MxfX4~LJPA8ZW4BlL~axMPKRWgQq<0E-uz8ffIK zbvbQ$hgGsBDrbZ&s=a9dB}9}wNE#m^VGhT4Rl$~I3I7BI4DZWsRJp|63w|`cvUEPi zNgzs*1MYxHKt{AKsx6u+iL&=et>si`_hNE>paT(G1#Je0(iac-Ek@v>ML^M_#Rxg3 zrF}(T;~fd#e!9bl!`JLnnbPE#FF;j@^StG3W>NBMof1*wsYDF8Wh---mRrRjyC_YW zcjtT)>~Ab^x~yKzSK6dJbkc6f8krBFs*ph+LJTT&ob?0jLPZrL7o9;r>xrwKaK)%V zoy^I>E;vSBJF)p+tJPsY5J92>B~Cm2~X~mftB`?;?-y!`b#>y_dt)M!(^xK}@G)YaEwBviJ>5zHjm6O5Im^ z9?`Xnq;@y^$yoli=Uf#Z^k%+hBRD|Aa0b#LjCwExi(X1(@W%mo@`L@YcsR2r3m;EB; zyNl4+2}`4_CrY4%bvk{xf+KZ&nJ89bPP@FIRB8>o{Y@Qgv06c0>?tq%vv>9|Tc zQT;tyNMOWl`W0{%FMakfh8pMLj-q8zbP$o~E_f645G!ohB8H*&fOO<9rH2C<|1f_k zTv|~$vGr>yBmmVs_uI)67f0fBU z69_!s-p(Bf4{;3g^|zVgplJoj7E2HqLdcJd(k!GJG@=h?2~Y32uu=Zw=!H|&M=pd0 zSCaKK9ym16g*R7n5sX8VU6~emJ{=J;8Pt{s&~^ZsDQW^ODxW)Qo&f>FLpb5=Bn*3Z zxZ95{7Iwk$(t+$$0qs6fdrRRTKqL=xFE>EADEd%ohan`8RU6`iDD_(7=9?vJzI*>( z8D1Gh=@BSs7oisF{-JX-V4q?_JCIT;f*N7E*d1DeVi=dB1Et>6>P`aB&+B{w_l=Um9*rbmkAv}8(8BdIsZP*H z6q2T0M$8tC5&UF|{)9|==pE;T>QQBRq2;%qwpis^KjL4Vxii?!qOfb?VTOi`fQHr) z;B=iuv~t(OBQe_P@vyR@z>0Ss1(Jc0>F3{LP>L6c zEDlyD;D~~UT{3Pb-~D7jb!4`NwB@0NaSt&i_;Yl{@{$o9Ubz(r6uBnR_+LPBHWe&j z%^@slG8Lj`Gjt9}gAzsw%zIzBa^zr*KH3iV^2Sjk7&^|$%^oQ7rQ*#$jUhJ!E~^iL z3r@B0UFVTjF^Qp?xYxvnJx%4^g9i^ZblcaWj>JOa4?ZUGNws2L6`mRScK{lqOqJq4 zYHYH6!Z{n0F#c8kwl6uv1Mh)wH;(MxNSLjGJ~eLK4PLQt9dmN+$Xm)$U(tX)G}68^ z769P74~uD6t{mGJ zpRjflUHP>hNmbOJ1Rjv!7aV(}jhOs;$AQc;keg1=wF9z9C=%8+f$X9t0G=l2 zFf~?Zbh(>`!8X1aLoH8#2r=?sMU48>6%keFG_P2Ni;h|sC%W_=Wly&!JXJxeqdz%#FIh_6mrNJ99 zprZxg6+zZ`hGd6>$iR2<<&UTegMP30j7?RD*H1UVfHH@XRUyhg=MkjfZph1F86
    @@ -79,7 +79,7 @@
    - +
    @@ -252,6 +252,9 @@ This field is required. + The pseudo is already in use by another export. Pseudo needs to start with a '/' and can't contain any of the following: >, <, |, &, ( or ). diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/nfs/nfs-form/nfs-form.component.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/nfs/nfs-form/nfs-form.component.spec.ts index 4bf34e2c7a0d1..7cf3d61387a43 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/nfs/nfs-form/nfs-form.component.spec.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/nfs/nfs-form/nfs-form.component.spec.ts @@ -74,14 +74,14 @@ describe('NfsFormComponent', () => { expect(component.nfsForm.value).toEqual({ access_type: 'RW', clients: [], - cluster_id: '', - fsal: { fs_name: 'a', name: '' }, + cluster_id: 'mynfs', + fsal: { fs_name: 'a', name: 'CEPH' }, path: '/', protocolNfsv4: true, pseudo: '', sec_label_xattr: 'security.selinux', security_label: false, - squash: '', + squash: 'no_root_squash', transportTCP: true, transportUDP: true }); diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/nfs/nfs-form/nfs-form.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/nfs/nfs-form/nfs-form.component.ts index 46eeeec52cb22..a56c1105e08e4 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/nfs/nfs-form/nfs-form.component.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/nfs/nfs-form/nfs-form.component.ts @@ -24,6 +24,7 @@ import { CdFormGroup } from '~/app/shared/forms/cd-form-group'; import { CdValidators } from '~/app/shared/forms/cd-validators'; import { FinishedTask } from '~/app/shared/models/finished-task'; import { Permission } from '~/app/shared/models/permissions'; +import { CdHttpErrorResponse } from '~/app/shared/services/api-interceptor.service'; import { AuthStorageService } from '~/app/shared/services/auth-storage.service'; import { TaskWrapperService } from '~/app/shared/services/task-wrapper.service'; import { NfsFormClientComponent } from '../nfs-form-client/nfs-form-client.component'; @@ -163,7 +164,7 @@ export class NfsFormComponent extends CdForm implements OnInit { access_type: new FormControl('RW', { validators: [Validators.required] }), - squash: new FormControl('', { + squash: new FormControl(this.nfsSquash[0], { validators: [Validators.required] }), transportUDP: new FormControl(true, { @@ -222,6 +223,9 @@ export class NfsFormComponent extends CdForm implements OnInit { for (const cluster of clusters) { this.allClusters.push({ cluster_id: cluster }); } + if (!this.isEdit && this.allClusters.length > 0) { + this.nfsForm.get('cluster_id').setValue(this.allClusters[0].cluster_id); + } } resolveFsals(res: string[]) { @@ -234,17 +238,18 @@ export class NfsFormComponent extends CdForm implements OnInit { this.allFsals.push(fsalItem); } }); - - if (this.allFsals.length === 1 && _.isUndefined(this.nfsForm.getValue('fsal'))) { + if (!this.isEdit && this.allFsals.length > 0) { this.nfsForm.patchValue({ - fsal: this.allFsals[0] + fsal: { + name: this.allFsals[0].value + } }); } } resolveFilesystems(filesystems: any[]) { this.allFsNames = filesystems; - if (filesystems.length === 1) { + if (!this.isEdit && filesystems.length > 0) { this.nfsForm.patchValue({ fsal: { fs_name: filesystems[0].name @@ -254,6 +259,7 @@ export class NfsFormComponent extends CdForm implements OnInit { } fsalChangeHandler() { + this.setPathValidation(); const fsalValue = this.nfsForm.getValue('name'); const checkAvailability = fsalValue === 'RGW' @@ -276,13 +282,13 @@ export class NfsFormComponent extends CdForm implements OnInit { checkAvailability.subscribe({ next: () => { this.setFsalAvailability(fsalValue, true); - this.nfsForm.patchValue({ - path: fsalValue === 'RGW' ? '' : '/', - pseudo: this.generatePseudo(), - access_type: this.updateAccessType() - }); - - this.setPathValidation(); + if (!this.isEdit) { + this.nfsForm.patchValue({ + path: fsalValue === 'RGW' ? '' : '/', + pseudo: this.generatePseudo(), + access_type: this.updateAccessType() + }); + } this.cdRef.detectChanges(); }, @@ -360,9 +366,11 @@ export class NfsFormComponent extends CdForm implements OnInit { } pathChangeHandler() { - this.nfsForm.patchValue({ - pseudo: this.generatePseudo() - }); + if (!this.isEdit) { + this.nfsForm.patchValue({ + pseudo: this.generatePseudo() + }); + } } private getBucketTypeahead(path: string): Observable { @@ -430,11 +438,23 @@ export class NfsFormComponent extends CdForm implements OnInit { } action.subscribe({ - error: () => this.nfsForm.setErrors({ cdSubmitButton: true }), + error: (errorResponse: CdHttpErrorResponse) => this.setFormErrors(errorResponse), complete: () => this.router.navigate(['/nfs']) }); } + private setFormErrors(errorResponse: CdHttpErrorResponse) { + if ( + errorResponse.error.detail && + errorResponse.error.detail + .toString() + .includes(`Pseudo ${this.nfsForm.getValue('pseudo')} is already in use`) + ) { + this.nfsForm.get('pseudo').setErrors({ pseudoAlreadyExists: true }); + } + this.nfsForm.setErrors({ cdSubmitButton: true }); + } + private buildRequest() { const requestModel: any = _.cloneDeep(this.nfsForm.value); @@ -500,15 +520,14 @@ export class NfsFormComponent extends CdForm implements OnInit { return of({ required: true }); } const fsName = this.nfsForm.getValue('fsal').fs_name; - return this.nfsService - .lsDir(fsName, control.value) - .pipe( - map((directory: Directory) => - directory.paths.includes(control.value) === requiredExistenceResult - ? null - : { pathNameNotAllowed: true } - ) - ); + return this.nfsService.lsDir(fsName, control.value).pipe( + map((directory: Directory) => + directory.paths.includes(control.value) === requiredExistenceResult + ? null + : { pathNameNotAllowed: true } + ), + catchError(() => of({ pathNameNotAllowed: true })) + ); }; } } diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/nfs.service.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/nfs.service.spec.ts index c977f7ec5b514..139fa490bfd82 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/nfs.service.spec.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/nfs.service.spec.ts @@ -1,5 +1,5 @@ import { HttpClientTestingModule, HttpTestingController } from '@angular/common/http/testing'; -import { TestBed } from '@angular/core/testing'; +import { fakeAsync, TestBed, tick } from '@angular/core/testing'; import { configureTestBed } from '~/testing/unit-test-helper'; import { NfsService } from './nfs.service'; @@ -63,4 +63,12 @@ describe('NfsService', () => { const req = httpTesting.expectOne('ui-api/nfs-ganesha/lsdir/a?root_dir=foo_dir'); expect(req.request.method).toBe('GET'); }); + + it('should not call lsDir if volume is not provided', fakeAsync(() => { + service.lsDir('', 'foo_dir').subscribe({ + error: (error: string) => expect(error).toEqual('Please specify a filesystem volume.') + }); + tick(); + httpTesting.expectNone('ui-api/nfs-ganesha/lsdir/?root_dir=foo_dir'); + })); }); diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/nfs.service.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/nfs.service.ts index 88af7a68216f0..636335673978f 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/nfs.service.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/nfs.service.ts @@ -1,7 +1,7 @@ import { HttpClient } from '@angular/common/http'; import { Injectable } from '@angular/core'; -import { Observable } from 'rxjs'; +import { Observable, throwError } from 'rxjs'; import { NfsFSAbstractionLayer } from '~/app/ceph/nfs/models/nfs.fsal'; import { ApiClient } from '~/app/shared/api/api-client'; @@ -87,6 +87,9 @@ export class NfsService extends ApiClient { } lsDir(fs_name: string, root_dir: string): Observable { + if (!fs_name) { + return throwError($localize`Please specify a filesystem volume.`); + } return this.http.get(`${this.uiApiPath}/lsdir/${fs_name}?root_dir=${root_dir}`); } diff --git a/src/pybind/mgr/dashboard/plugins/feature_toggles.py b/src/pybind/mgr/dashboard/plugins/feature_toggles.py index c9e7a472b7291..c71d325476774 100644 --- a/src/pybind/mgr/dashboard/plugins/feature_toggles.py +++ b/src/pybind/mgr/dashboard/plugins/feature_toggles.py @@ -8,7 +8,7 @@ from ..controllers.cephfs import CephFS from ..controllers.iscsi import Iscsi, IscsiTarget -from ..controllers.nfsganesha import NFSGanesha, NFSGaneshaExports +from ..controllers.nfs import NFSGanesha, NFSGaneshaExports from ..controllers.rbd import Rbd, RbdSnapshot, RbdTrash from ..controllers.rbd_mirroring import RbdMirroringPoolMode, \ RbdMirroringPoolPeer, RbdMirroringSummary diff --git a/src/pybind/mgr/dashboard/tests/test_auth.py b/src/pybind/mgr/dashboard/tests/test_auth.py index 698db40e1e5ef..d9755de98e45a 100644 --- a/src/pybind/mgr/dashboard/tests/test_auth.py +++ b/src/pybind/mgr/dashboard/tests/test_auth.py @@ -41,7 +41,7 @@ def test_request_not_authorized(self): @patch('dashboard.controllers.auth.JwtManager.gen_token', Mock(return_value='my-token')) @patch('dashboard.controllers.auth.AuthManager.authenticate', Mock(return_value={ - 'permissions': {'read-only': ['read']}, + 'permissions': {'rgw': ['read']}, 'pwdExpirationDate': 1000000, 'pwdUpdateRequired': False })) @@ -51,7 +51,7 @@ def test_login(self): self.assertJsonBody({ 'token': 'my-token', 'username': 'my-user', - 'permissions': {'read-only': ['read']}, + 'permissions': {'rgw': ['read']}, 'pwdExpirationDate': 1000000, 'sso': False, 'pwdUpdateRequired': False diff --git a/src/pybind/mgr/dashboard/tests/test_ganesha.py b/src/pybind/mgr/dashboard/tests/test_ganesha.py deleted file mode 100644 index f1fe3ecd853e5..0000000000000 --- a/src/pybind/mgr/dashboard/tests/test_ganesha.py +++ /dev/null @@ -1,136 +0,0 @@ -# -*- coding: utf-8 -*- -# pylint: disable=too-many-lines - -from unittest.mock import patch -from urllib.parse import urlencode - -from ..controllers.nfsganesha import NFSGaneshaExports, NFSGaneshaUi -from . import ControllerTestCase # pylint: disable=no-name-in-module - - -class NFSGaneshaExportsTest(ControllerTestCase): - - def test_get_schema_export(self): - export = { - "export_id": 2, - "path": "bk1", - "cluster_id": "myc", - "pseudo": "/bk-ps", - "access_type": "RO", - "squash": "root_id_squash", - "security_label": False, - "protocols": [ - 4 - ], - "transports": [ - "TCP", - "UDP" - ], - "fsal": { - "name": "RGW", - "user_id": "dashboard", - "access_key_id": "UUU5YVVOQ2P5QTOPYNAN", - "secret_access_key": "7z87tMUUsHr67ZWx12pCbWkp9UyOldxhDuPY8tVN" - }, - "clients": [] - } - expected_schema_export = export - del expected_schema_export['fsal']['access_key_id'] - del expected_schema_export['fsal']['secret_access_key'] - self.assertDictEqual( - expected_schema_export, - NFSGaneshaExports._get_schema_export(export)) # pylint: disable=protected-access - - -class NFSGaneshaUiControllerTest(ControllerTestCase): - @classmethod - def setup_server(cls): - # pylint: disable=protected-access - NFSGaneshaUi._cp_config['tools.authenticate.on'] = False - cls.setup_controllers([NFSGaneshaUi]) - - @classmethod - def _create_ls_dir_url(cls, fs_name, query_params): - api_url = '/ui-api/nfs-ganesha/lsdir/{}'.format(fs_name) - if query_params is not None: - return '{}?{}'.format(api_url, urlencode(query_params)) - return api_url - - @patch('dashboard.controllers.nfsganesha.CephFS') - def test_lsdir(self, cephfs_class): - cephfs_class.return_value.ls_dir.return_value = [ - {'path': '/foo'}, - {'path': '/foo/bar'} - ] - mocked_ls_dir = cephfs_class.return_value.ls_dir - - reqs = [ - { - 'params': None, - 'cephfs_ls_dir_args': ['/', 1], - 'path0': '/', - 'status': 200 - }, - { - 'params': {'root_dir': '/', 'depth': '1'}, - 'cephfs_ls_dir_args': ['/', 1], - 'path0': '/', - 'status': 200 - }, - { - 'params': {'root_dir': '', 'depth': '1'}, - 'cephfs_ls_dir_args': ['/', 1], - 'path0': '/', - 'status': 200 - }, - { - 'params': {'root_dir': '/foo', 'depth': '3'}, - 'cephfs_ls_dir_args': ['/foo', 3], - 'path0': '/foo', - 'status': 200 - }, - { - 'params': {'root_dir': 'foo', 'depth': '6'}, - 'cephfs_ls_dir_args': ['/foo', 5], - 'path0': '/foo', - 'status': 200 - }, - { - 'params': {'root_dir': '/', 'depth': '-1'}, - 'status': 400 - }, - { - 'params': {'root_dir': '/', 'depth': 'abc'}, - 'status': 400 - } - ] - - for req in reqs: - self._get(self._create_ls_dir_url('a', req['params'])) - self.assertStatus(req['status']) - - # Returned paths should contain root_dir as first element - if req['status'] == 200: - paths = self.json_body()['paths'] - self.assertEqual(paths[0], req['path0']) - cephfs_class.assert_called_once_with('a') - - # Check the arguments passed to `CephFS.ls_dir`. - if req.get('cephfs_ls_dir_args'): - mocked_ls_dir.assert_called_once_with(*req['cephfs_ls_dir_args']) - else: - mocked_ls_dir.assert_not_called() - mocked_ls_dir.reset_mock() - cephfs_class.reset_mock() - - @patch('dashboard.controllers.nfsganesha.cephfs') - @patch('dashboard.controllers.nfsganesha.CephFS') - def test_lsdir_non_existed_dir(self, cephfs_class, cephfs): - cephfs.ObjectNotFound = Exception - cephfs.PermissionError = Exception - cephfs_class.return_value.ls_dir.side_effect = cephfs.ObjectNotFound() - self._get(self._create_ls_dir_url('a', {'root_dir': '/foo', 'depth': '3'})) - cephfs_class.assert_called_once_with('a') - cephfs_class.return_value.ls_dir.assert_called_once_with('/foo', 3) - self.assertStatus(200) - self.assertJsonBody({'paths': []}) diff --git a/src/pybind/mgr/dashboard/tests/test_nfs.py b/src/pybind/mgr/dashboard/tests/test_nfs.py new file mode 100644 index 0000000000000..087ca18914b95 --- /dev/null +++ b/src/pybind/mgr/dashboard/tests/test_nfs.py @@ -0,0 +1,229 @@ +# -*- coding: utf-8 -*- +# pylint: disable=too-many-lines +from copy import deepcopy +from unittest.mock import Mock, patch +from urllib.parse import urlencode + +from .. import mgr +from ..controllers._version import APIVersion +from ..controllers.nfs import NFSGaneshaExports, NFSGaneshaUi +from ..tests import ControllerTestCase +from ..tools import NotificationQueue, TaskManager + + +class NFSGaneshaExportsTest(ControllerTestCase): + _nfs_module_export = { + "export_id": 1, + "path": "bk1", + "cluster_id": "myc", + "pseudo": "/bk-ps", + "access_type": "RO", + "squash": "root_id_squash", + "security_label": False, + "protocols": [ + 4 + ], + "transports": [ + "TCP", + "UDP" + ], + "fsal": { + "name": "RGW", + "user_id": "dashboard", + "access_key_id": "UUU5YVVOQ2P5QTOPYNAN", + "secret_access_key": "7z87tMUUsHr67ZWx12pCbWkp9UyOldxhDuPY8tVN" + }, + "clients": [] + } + + @classmethod + def setUpClass(cls): + super().setUpClass() + cls._expected_export = deepcopy(cls._nfs_module_export) + del cls._expected_export['fsal']['access_key_id'] + del cls._expected_export['fsal']['secret_access_key'] + + @classmethod + def tearDownClass(cls): + super().tearDownClass() + NotificationQueue.stop() + + @classmethod + def setup_server(cls): + NotificationQueue.start_queue() + TaskManager.init() + cls.setup_controllers([NFSGaneshaExports]) + + def test_list_exports(self): + mgr.remote = Mock(return_value=[self._nfs_module_export]) + + self._get('/api/nfs-ganesha/export') + self.assertStatus(200) + self.assertJsonBody([self._expected_export]) + + def test_get_export(self): + mgr.remote = Mock(return_value=self._nfs_module_export) + + self._get('/api/nfs-ganesha/export/myc/1') + self.assertStatus(200) + self.assertJsonBody(self._expected_export) + + def test_create_export(self): + export_mgr = Mock() + created_nfs_export = deepcopy(self._nfs_module_export) + created_nfs_export['pseudo'] = 'new-pseudo' + created_nfs_export['export_id'] = 2 + export_mgr.get_export_by_pseudo.side_effect = [None, created_nfs_export] + export_mgr.apply_export.return_value = (0, '', '') + mgr.remote.return_value = export_mgr + + export_create_body = deepcopy(self._expected_export) + del export_create_body['export_id'] + export_create_body['pseudo'] = created_nfs_export['pseudo'] + + self._post('/api/nfs-ganesha/export', + export_create_body, + version=APIVersion(2, 0)) + self.assertStatus(201) + expected_body = export_create_body + expected_body['export_id'] = created_nfs_export['export_id'] + self.assertJsonBody(export_create_body) + + def test_create_export_with_existing_pseudo_fails(self): + export_mgr = Mock() + export_mgr.get_export_by_pseudo.return_value = self._nfs_module_export + mgr.remote.return_value = export_mgr + + export_create_body = deepcopy(self._expected_export) + del export_create_body['export_id'] + + self._post('/api/nfs-ganesha/export', + export_create_body, + version=APIVersion(2, 0)) + self.assertStatus(400) + response = self.json_body() + self.assertIn(f'Pseudo {export_create_body["pseudo"]} is already in use', + response['detail']) + + def test_set_export(self): + export_mgr = Mock() + updated_nfs_export = deepcopy(self._nfs_module_export) + updated_nfs_export['pseudo'] = 'updated-pseudo' + export_mgr.get_export_by_pseudo.return_value = updated_nfs_export + export_mgr.apply_export.return_value = (0, '', '') + mgr.remote.return_value = export_mgr + + updated_export_body = deepcopy(self._expected_export) + updated_export_body['pseudo'] = updated_nfs_export['pseudo'] + + self._put('/api/nfs-ganesha/export/myc/2', + updated_export_body, + version=APIVersion(2, 0)) + self.assertStatus(200) + self.assertJsonBody(updated_export_body) + + def test_delete_export(self): + mgr.remote = Mock(side_effect=[self._nfs_module_export, None]) + + self._delete('/api/nfs-ganesha/export/myc/2', + version=APIVersion(2, 0)) + self.assertStatus(204) + + def test_delete_export_not_found(self): + mgr.remote = Mock(return_value=None) + + self._delete('/api/nfs-ganesha/export/myc/3', + version=APIVersion(2, 0)) + self.assertStatus(404) + + +class NFSGaneshaUiControllerTest(ControllerTestCase): + @classmethod + def setup_server(cls): + cls.setup_controllers([NFSGaneshaUi]) + + @classmethod + def _create_ls_dir_url(cls, fs_name, query_params): + api_url = '/ui-api/nfs-ganesha/lsdir/{}'.format(fs_name) + if query_params is not None: + return '{}?{}'.format(api_url, urlencode(query_params)) + return api_url + + @patch('dashboard.controllers.nfs.CephFS') + def test_lsdir(self, cephfs_class): + cephfs_class.return_value.ls_dir.return_value = [ + {'path': '/foo'}, + {'path': '/foo/bar'} + ] + mocked_ls_dir = cephfs_class.return_value.ls_dir + + reqs = [ + { + 'params': None, + 'cephfs_ls_dir_args': ['/', 1], + 'path0': '/', + 'status': 200 + }, + { + 'params': {'root_dir': '/', 'depth': '1'}, + 'cephfs_ls_dir_args': ['/', 1], + 'path0': '/', + 'status': 200 + }, + { + 'params': {'root_dir': '', 'depth': '1'}, + 'cephfs_ls_dir_args': ['/', 1], + 'path0': '/', + 'status': 200 + }, + { + 'params': {'root_dir': '/foo', 'depth': '3'}, + 'cephfs_ls_dir_args': ['/foo', 3], + 'path0': '/foo', + 'status': 200 + }, + { + 'params': {'root_dir': 'foo', 'depth': '6'}, + 'cephfs_ls_dir_args': ['/foo', 5], + 'path0': '/foo', + 'status': 200 + }, + { + 'params': {'root_dir': '/', 'depth': '-1'}, + 'status': 400 + }, + { + 'params': {'root_dir': '/', 'depth': 'abc'}, + 'status': 400 + } + ] + + for req in reqs: + self._get(self._create_ls_dir_url('a', req['params'])) + self.assertStatus(req['status']) + + # Returned paths should contain root_dir as first element + if req['status'] == 200: + paths = self.json_body()['paths'] + self.assertEqual(paths[0], req['path0']) + cephfs_class.assert_called_once_with('a') + + # Check the arguments passed to `CephFS.ls_dir`. + if req.get('cephfs_ls_dir_args'): + mocked_ls_dir.assert_called_once_with(*req['cephfs_ls_dir_args']) + else: + mocked_ls_dir.assert_not_called() + mocked_ls_dir.reset_mock() + cephfs_class.reset_mock() + + @patch('dashboard.controllers.nfs.cephfs') + @patch('dashboard.controllers.nfs.CephFS') + def test_lsdir_non_existed_dir(self, cephfs_class, cephfs): + cephfs.ObjectNotFound = Exception + cephfs.PermissionError = Exception + cephfs_class.return_value.ls_dir.side_effect = cephfs.ObjectNotFound() + self._get(self._create_ls_dir_url('a', {'root_dir': '/foo', 'depth': '3'})) + cephfs_class.assert_called_once_with('a') + cephfs_class.return_value.ls_dir.assert_called_once_with('/foo', 3) + self.assertStatus(200) + self.assertJsonBody({'paths': []}) diff --git a/src/pybind/mgr/nfs/export.py b/src/pybind/mgr/nfs/export.py index b23a95e04632d..eeb57f8b17eb2 100644 --- a/src/pybind/mgr/nfs/export.py +++ b/src/pybind/mgr/nfs/export.py @@ -420,6 +420,14 @@ def get_export_by_id( export = self._fetch_export_id(cluster_id, export_id) return export.to_dict() if export else None + def get_export_by_pseudo( + self, + cluster_id: str, + pseudo_path: str + ) -> Optional[Dict[str, Any]]: + export = self._fetch_export(cluster_id, pseudo_path) + return export.to_dict() if export else None + def apply_export(self, cluster_id: str, export_config: str) -> Tuple[int, str, str]: try: if not export_config: From 816bacba35c9861eedbb49d35dc70c7cbe8a5e8e Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Thu, 4 Nov 2021 10:07:14 -0400 Subject: [PATCH 0516/2866] mgr/orchestrator: improve usage string for 'orch daemon add osd' Signed-off-by: Sage Weil --- src/pybind/mgr/orchestrator/module.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pybind/mgr/orchestrator/module.py b/src/pybind/mgr/orchestrator/module.py index b9793f1939c23..bab4ba275b317 100644 --- a/src/pybind/mgr/orchestrator/module.py +++ b/src/pybind/mgr/orchestrator/module.py @@ -797,7 +797,7 @@ def _apply_osd(self, def _daemon_add_osd(self, svc_arg: Optional[str] = None, method: Optional[OSDMethod] = None) -> HandleCommandResult: - """Create an OSD service. Either --svc_arg=host:drives""" + """Create OSD daemon(s) on specified host and device(s) (e.g., ceph orch daemon add osd myhost:/dev/sdb)""" # Create one or more OSDs""" usage = """ From bd018541dcb042413512489bc394e80e5c1cce6c Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Wed, 20 Oct 2021 17:33:27 -0400 Subject: [PATCH 0517/2866] mgr/nfs: take user-id and/or bucket for 'nfs export create rgw' - move the bucket / user position after the cluster_id and pseudo_path (since they are optional) - require bucket or user or both - if bucket, use the bucket owner - if bucket+user, use that user - if user only, then export at top-level (all users buckets) Fixes: https://tracker.ceph.com/issues/53134 Signed-off-by: Sage Weil --- src/pybind/mgr/nfs/export.py | 45 +++++++----- src/pybind/mgr/nfs/module.py | 4 +- src/pybind/mgr/nfs/tests/test_nfs.py | 101 ++++++++++++++++++++++++++- 3 files changed, 127 insertions(+), 23 deletions(-) diff --git a/src/pybind/mgr/nfs/export.py b/src/pybind/mgr/nfs/export.py index b23a95e04632d..c9aa52fbc96ba 100644 --- a/src/pybind/mgr/nfs/export.py +++ b/src/pybind/mgr/nfs/export.py @@ -205,20 +205,23 @@ def _create_export_user(self, export: Export) -> None: elif isinstance(export.fsal, RGWFSAL): rgwfsal = cast(RGWFSAL, export.fsal) - ret, out, err = self.mgr.tool_exec( - ['radosgw-admin', 'bucket', 'stats', '--bucket', export.path] - ) - if ret: - raise NFSException(f'Failed to fetch owner for bucket {export.path}') - j = json.loads(out) - owner = j.get('owner', '') - rgwfsal.user_id = owner + if not rgwfsal.user_id: + assert export.path + ret, out, err = self.mgr.tool_exec( + ['radosgw-admin', 'bucket', 'stats', '--bucket', export.path] + ) + if ret: + raise NFSException(f'Failed to fetch owner for bucket {export.path}') + j = json.loads(out) + owner = j.get('owner', '') + rgwfsal.user_id = owner + assert rgwfsal.user_id ret, out, err = self.mgr.tool_exec([ - 'radosgw-admin', 'user', 'info', '--uid', owner + 'radosgw-admin', 'user', 'info', '--uid', rgwfsal.user_id ]) if ret: raise NFSException( - f'Failed to fetch key for bucket {export.path} owner {owner}' + f'Failed to fetch key for bucket {export.path} owner {rgwfsal.user_id}' ) j = json.loads(out) @@ -535,18 +538,15 @@ def create_export_from_dict(self, raise NFSInvalidOperation("export must specify pseudo path") path = ex_dict.get("path") - if not path: + if path is None: raise NFSInvalidOperation("export must specify path") path = self.format_path(path) fsal = ex_dict.get("fsal", {}) fsal_type = fsal.get("name") if fsal_type == NFS_GANESHA_SUPPORTED_FSALS[1]: - if '/' in path: - raise NFSInvalidOperation('"/" is not allowed in path (bucket name)') - uid = f'nfs.{cluster_id}.{path}' - if "user_id" in fsal and fsal["user_id"] != uid: - raise NFSInvalidOperation(f"export FSAL user_id must be '{uid}'") + if '/' in path and path != '/': + raise NFSInvalidOperation('"/" is not allowed in path with bucket name') elif fsal_type == NFS_GANESHA_SUPPORTED_FSALS[0]: fs_name = fsal.get("fs_name") if not fs_name: @@ -610,25 +610,32 @@ def create_cephfs_export(self, return 0, "", "Export already exists" def create_rgw_export(self, - bucket: str, cluster_id: str, pseudo_path: str, access_type: str, read_only: bool, squash: str, + bucket: Optional[str] = None, + user_id: Optional[str] = None, clients: list = []) -> Tuple[int, str, str]: pseudo_path = self.format_path(pseudo_path) + if not bucket and not user_id: + return -errno.EINVAL, "", "Must specify either bucket or user_id" + if not self._fetch_export(cluster_id, pseudo_path): export = self.create_export_from_dict( cluster_id, self._gen_export_id(cluster_id), { "pseudo": pseudo_path, - "path": bucket, + "path": bucket or '/', "access_type": access_type, "squash": squash, - "fsal": {"name": NFS_GANESHA_SUPPORTED_FSALS[1]}, + "fsal": { + "name": NFS_GANESHA_SUPPORTED_FSALS[1], + "user_id": user_id, + }, "clients": clients, } ) diff --git a/src/pybind/mgr/nfs/module.py b/src/pybind/mgr/nfs/module.py index 280a61653ec92..403f89f3a9736 100644 --- a/src/pybind/mgr/nfs/module.py +++ b/src/pybind/mgr/nfs/module.py @@ -46,13 +46,15 @@ def _cmd_nfs_export_create_rgw( self, cluster_id: str, pseudo_path: str, - bucket: str, + bucket: Optional[str] = None, + user_id: Optional[str] = None, readonly: Optional[bool] = False, client_addr: Optional[List[str]] = None, squash: str = 'none', ) -> Tuple[int, str, str]: """Create an RGW export""" return self.export_mgr.create_export(fsal_type='rgw', bucket=bucket, + user_id=user_id, cluster_id=cluster_id, pseudo_path=pseudo_path, read_only=readonly, squash=squash, addr=client_addr) diff --git a/src/pybind/mgr/nfs/tests/test_nfs.py b/src/pybind/mgr/nfs/tests/test_nfs.py index 6fc4246e95880..4688679db4832 100644 --- a/src/pybind/mgr/nfs/tests/test_nfs.py +++ b/src/pybind/mgr/nfs/tests/test_nfs.py @@ -163,6 +163,11 @@ def _mock_orchestrator(self, enable: bool) -> Iterator: ] if enable else [] def mock_exec(cls, args): + if args[1:3] == ['bucket', 'stats']: + bucket_info = { + "owner": "bucket_owner_user", + } + return 0, json.dumps(bucket_info), '' u = { "user_id": "abc", "display_name": "foo", @@ -790,10 +795,54 @@ def _do_test_remove_export(self) -> None: assert len(exports) == 1 assert exports[0].export_id == 1 - def test_create_export_rgw(self): - self._do_mock_test(self._do_test_create_export_rgw) + def test_create_export_rgw_bucket(self): + self._do_mock_test(self._do_test_create_export_rgw_bucket) + + def _do_test_create_export_rgw_bucket(self): + nfs_mod = Module('nfs', '', '') + conf = ExportMgr(nfs_mod) + + exports = conf.list_exports(cluster_id=self.cluster_id) + ls = json.loads(exports[1]) + assert len(ls) == 2 + + r = conf.create_export( + fsal_type='rgw', + cluster_id=self.cluster_id, + bucket='bucket', + pseudo_path='/mybucket', + read_only=False, + squash='root', + addr=["192.168.0.0/16"] + ) + assert r[0] == 0 + + exports = conf.list_exports(cluster_id=self.cluster_id) + ls = json.loads(exports[1]) + assert len(ls) == 3 + + export = conf._fetch_export('foo', '/mybucket') + assert export.export_id + assert export.path == "bucket" + assert export.pseudo == "/mybucket" + assert export.access_type == "none" + assert export.squash == "none" + assert export.protocols == [4] + assert export.transports == ["TCP"] + assert export.fsal.name == "RGW" + assert export.fsal.user_id == "bucket_owner_user" + assert export.fsal.access_key_id == "the_access_key" + assert export.fsal.secret_access_key == "the_secret_key" + assert len(export.clients) == 1 + assert export.clients[0].squash == 'root' + assert export.clients[0].access_type == 'rw' + assert export.clients[0].addresses == ["192.168.0.0/16"] + assert export.cluster_id == self.cluster_id + + def test_create_export_rgw_bucket_user(self): + self._do_mock_test(self._do_test_create_export_rgw_bucket_user) - def _do_test_create_export_rgw(self): + def _do_test_create_export_rgw_bucket_user(self): nfs_mod = Module('nfs', '', '') conf = ExportMgr(nfs_mod) @@ -805,6 +854,7 @@ def _do_test_create_export_rgw(self): fsal_type='rgw', cluster_id=self.cluster_id, bucket='bucket', + user_id='other_user', pseudo_path='/mybucket', read_only=False, squash='root', @@ -829,10 +879,55 @@ def _do_test_create_export_rgw(self): assert export.fsal.secret_access_key == "the_secret_key" assert len(export.clients) == 1 assert export.clients[0].squash == 'root' + assert export.fsal.user_id == "other_user" assert export.clients[0].access_type == 'rw' assert export.clients[0].addresses == ["192.168.0.0/16"] assert export.cluster_id == self.cluster_id + + def test_create_export_rgw_user(self): + self._do_mock_test(self._do_test_create_export_rgw_user) + + def _do_test_create_export_rgw_user(self): + nfs_mod = Module('nfs', '', '') + conf = ExportMgr(nfs_mod) + + exports = conf.list_exports(cluster_id=self.cluster_id) + ls = json.loads(exports[1]) + assert len(ls) == 2 + + r = conf.create_export( + fsal_type='rgw', + cluster_id=self.cluster_id, + user_id='some_user', + pseudo_path='/mybucket', + read_only=False, + squash='root', + addr=["192.168.0.0/16"] + ) + assert r[0] == 0 + exports = conf.list_exports(cluster_id=self.cluster_id) + ls = json.loads(exports[1]) + assert len(ls) == 3 + + export = conf._fetch_export('foo', '/mybucket') + assert export.export_id + assert export.path == "/" + assert export.pseudo == "/mybucket" + assert export.access_type == "none" + assert export.squash == "none" + assert export.protocols == [4] + assert export.transports == ["TCP"] + assert export.fsal.name == "RGW" + assert export.fsal.access_key_id == "the_access_key" + assert export.fsal.secret_access_key == "the_secret_key" + assert len(export.clients) == 1 + assert export.clients[0].squash == 'root' + assert export.fsal.user_id == "some_user" + assert export.clients[0].access_type == 'rw' + assert export.clients[0].addresses == ["192.168.0.0/16"] + assert export.cluster_id == self.cluster_id + def test_create_export_cephfs(self): self._do_mock_test(self._do_test_create_export_cephfs) From d354baa82f53561eafd7e7d48904f12551d2049a Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Mon, 25 Oct 2021 10:53:56 -0400 Subject: [PATCH 0518/2866] qa/suites/orch/cephadm/smoke-roleless: add rgw user nfs export case Signed-off-by: Sage Weil --- .../2-services/nfs-ingress-rgw-user.yaml | 90 +++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 qa/suites/orch/cephadm/smoke-roleless/2-services/nfs-ingress-rgw-user.yaml diff --git a/qa/suites/orch/cephadm/smoke-roleless/2-services/nfs-ingress-rgw-user.yaml b/qa/suites/orch/cephadm/smoke-roleless/2-services/nfs-ingress-rgw-user.yaml new file mode 100644 index 0000000000000..721aecfc34e57 --- /dev/null +++ b/qa/suites/orch/cephadm/smoke-roleless/2-services/nfs-ingress-rgw-user.yaml @@ -0,0 +1,90 @@ +tasks: +- vip: + +# make sure cephadm notices the new IP +- cephadm.shell: + host.a: + - ceph orch device ls --refresh + +# stop kernel nfs server, if running +- vip.exec: + all-hosts: + - systemctl stop nfs-server + +- cephadm.shell: + host.a: + - ceph orch apply rgw foorgw --port 8800 + - ceph nfs cluster create foo --ingress --virtual-ip {{VIP0}}/{{VIPPREFIXLEN}} + +- vip.exec: + host.a: + - dnf install -y python3-boto3 || apt install -y python3-boto3 + - /home/ubuntu/cephtest/cephadm shell radosgw-admin user create --uid foouser --display-name foo > /tmp/user.json + +- python: + host.a: | + import boto3 + import json + + with open('/tmp/user.json', 'rt') as f: + info = json.loads(f.read()) + s3 = boto3.resource( + 's3', + aws_access_key_id=info['keys'][0]['access_key'], + aws_secret_access_key=info['keys'][0]['secret_key'], + endpoint_url='http://localhost:8800', + ) + bucket = s3.Bucket('foobucket') + bucket.create() + bucket.put_object(Key='myobject', Body='thebody') + +- cephadm.shell: + host.a: + - ceph nfs export create rgw --cluster-id foo --pseudo-path /foouser --user-id foouser + +- cephadm.wait_for_service: + service: nfs.foo +- cephadm.wait_for_service: + service: ingress.nfs.foo + +## export and mount + +- vip.exec: + host.a: + - mkdir /mnt/foo + - sleep 5 + - mount -t nfs {{VIP0}}:/foouser /mnt/foo + - test -d /mnt/foo/foobucket + - find /mnt/foo -ls + - grep thebody /mnt/foo/foobucket/myobject + - echo test > /mnt/foo/foobucket/newobject + - sync + +- python: + host.a: | + import boto3 + import json + from io import BytesIO + + with open('/tmp/user.json', 'rt') as f: + info = json.loads(f.read()) + s3 = boto3.resource( + 's3', + aws_access_key_id=info['keys'][0]['access_key'], + aws_secret_access_key=info['keys'][0]['secret_key'], + endpoint_url='http://localhost:8800', + ) + bucket = s3.Bucket('foobucket') + data = BytesIO() + bucket.download_fileobj(Fileobj=data, Key='newobject') + print(data.getvalue()) + assert data.getvalue().decode() == 'test\n' + +- vip.exec: + host.a: + - umount /mnt/foo + +- cephadm.shell: + host.a: + - ceph nfs export rm foo /foouser + - ceph nfs cluster rm foo From 48ce4e9568d001d7dcac0a8396283e2da88fc0b9 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Mon, 25 Oct 2021 15:52:28 -0400 Subject: [PATCH 0519/2866] PendingReleaseNotes: add note about nfs CLI change(s) Signed-off-by: Sage Weil --- PendingReleaseNotes | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/PendingReleaseNotes b/PendingReleaseNotes index ff1b2a1aa10db..f90a72684369b 100644 --- a/PendingReleaseNotes +++ b/PendingReleaseNotes @@ -91,6 +91,15 @@ deprecated and will be removed in a future release. Please use ``nfs cluster rm`` and ``nfs export rm`` instead. +* The ``nfs export create`` CLI arguments have changed, with the + *fsname* or *bucket-name* argument position moving to the right of + *the *cluster-id* and *pseudo-path*. Consider transitioning to + *using named arguments instead of positional arguments (e.g., ``ceph + *nfs export create cephfs --cluster-id mycluster --pseudo-path /foo + *--fsname myfs`` instead of ``ceph nfs export create cephfs + *mycluster /foo myfs`` to ensure correct behavior with any + *version. + * mgr-pg_autoscaler: Autoscaler will now start out by scaling each pool to have a full complements of pgs from the start and will only decrease it when other pools need more pgs due to increased usage. From b6d85e3975381ea00c40db0c078ba88e4056eac0 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Thu, 4 Nov 2021 10:41:44 -0400 Subject: [PATCH 0520/2866] doc/mgr/nfs: document rgw user and bucket exports Signed-off-by: Sage Weil --- doc/mgr/nfs.rst | 34 ++++++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/doc/mgr/nfs.rst b/doc/mgr/nfs.rst index 548f7b62da780..79e6c29be9f99 100644 --- a/doc/mgr/nfs.rst +++ b/doc/mgr/nfs.rst @@ -271,11 +271,21 @@ permissible values. Create RGW Export ----------------- -To export a bucket +There are two kinds of RGW exports: + +- a *user* export will export all buckets owned by an + RGW user, where the top-level directory of the export is a list of buckets. +- a *bucket* export will export a single bucket, where the top-level directory contains + the objects in the bucket. + +RGW bucket export +^^^^^^^^^^^^^^^^^ + +To export a *bucket*: .. code:: - $ ceph nfs export create rgw --cluster-id --pseudo-path --bucket [--readonly] [--client_addr ...] [--squash ] + $ ceph nfs export create rgw --cluster-id --pseudo-path --bucket [--user-id ] [--readonly] [--client_addr ...] [--squash ] For example, to export *mybucket* via NFS cluster *mynfs* at the pseudo-path */bucketdata* to any host in the ``192.168.10.0/24`` network @@ -291,6 +301,10 @@ For example, to export *mybucket* via NFS cluster *mynfs* at the pseudo-path */b ```` is the name of the bucket that will be exported. +```` is optional, and specifies which RGW user will be used for read and write +operations to the bucket. If it is not specified, the user who owns the bucket will be +used. + .. note:: Currently, if multi-site RGW is enabled, Ceph can only export RGW buckets in the default realm. ```` is the list of client address for which these export @@ -302,6 +316,22 @@ for permissible values. value is `no_root_squash`. See the `NFS-Ganesha Export Sample`_ for permissible values. +RGW user export +^^^^^^^^^^^^^^^ + +To export an RGW *user*: + +.. code:: + + $ ceph nfs export create rgw --cluster-id --pseudo-path --user-id [--readonly] [--client_addr ...] [--squash ] + +For example, to export *myuser* via NFS cluster *mynfs* at the pseudo-path */myuser* to any host in the ``192.168.10.0/24`` network + +.. code:: + + $ ceph nfs export create rgw --cluster-id mynfs --pseudo-path /bucketdata --user-id myuser --client_addr 192.168.10.0/24 + + Delete Export ------------- From 4b9134c355a85075767d83ef7ec4d577c43294a1 Mon Sep 17 00:00:00 2001 From: Joseph Sawaya Date: Wed, 15 Sep 2021 15:59:29 -0400 Subject: [PATCH 0521/2866] mgr/rook: persist applied drive groups and fix drive_group_loop This commit changes OSD creation to store applied drive groups in a map and persist that map in the mon's KV store in a map. This commit also modifies the rook orchestrator's serve loop to periodically re-apply drive groups, and creates a module option to specify the interval between re-applies. Signed-off-by: Joseph Sawaya --- src/pybind/mgr/rook/module.py | 56 ++++++++++++++++++++++++++++++----- 1 file changed, 48 insertions(+), 8 deletions(-) diff --git a/src/pybind/mgr/rook/module.py b/src/pybind/mgr/rook/module.py index c0c1c32c9f27b..4ba044413a265 100644 --- a/src/pybind/mgr/rook/module.py +++ b/src/pybind/mgr/rook/module.py @@ -9,7 +9,7 @@ from ceph.deployment.service_spec import ServiceSpec, NFSServiceSpec, RGWSpec, PlacementSpec from ceph.utils import datetime_now -from typing import List, Dict, Optional, Callable, Any, TypeVar, Tuple +from typing import List, Dict, Optional, Callable, Any, TypeVar, Tuple, TYPE_CHECKING try: from ceph.deployment.drive_group import DriveGroupSpec @@ -81,6 +81,12 @@ class RookOrchestrator(MgrModule, orchestrator.Orchestrator): default='local', desc='storage class name for LSO-discovered PVs', ), + Option( + 'drive_group_interval', + type='float', + default=300.0, + desc='interval in seconds between re-application of applied drive_groups', + ), ] @staticmethod @@ -115,8 +121,13 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: self._rook_cluster: Optional[RookCluster] = None self._rook_env = RookEnv() self._k8s_AppsV1_api: Optional[client.AppsV1Api] = None - self.storage_class = self.get_module_option('storage_class') + self.config_notify() + if TYPE_CHECKING: + self.storage_class = 'foo' + self.drive_group_interval = 10.0 + + self._load_drive_groups() self._shutdown = threading.Event() def config_notify(self) -> None: @@ -132,7 +143,10 @@ def config_notify(self) -> None: self.log.debug(' mgr option %s = %s', opt['name'], getattr(self, opt['name'])) # type: ignore assert isinstance(self.storage_class, str) - self.rook_cluster.storage_class = self.storage_class + assert isinstance(self.drive_group_interval, float) + + if self._rook_cluster: + self._rook_cluster.storage_class = self.storage_class def shutdown(self) -> None: self._shutdown.set() @@ -194,9 +208,11 @@ def serve(self) -> None: self.storage_class) self._initialized.set() + self.config_notify() while not self._shutdown.is_set(): - self._shutdown.wait(5) + self._apply_drivegroups(list(self._drive_group_map.values())) + self._shutdown.wait(self.drive_group_interval) @handle_orch_error def get_inventory(self, host_filter: Optional[orchestrator.InventoryFilter] = None, refresh: bool = False) -> List[orchestrator.InventoryHost]: @@ -523,10 +539,18 @@ def remove_daemons(self, names: List[str]) -> List[str]: return self.rook_cluster.remove_pods(names) def apply_drivegroups(self, specs: List[DriveGroupSpec]) -> OrchResult[List[str]]: - result_list = [] - all_hosts = raise_if_exception(self.get_hosts()) for drive_group in specs: - matching_hosts = drive_group.placement.filter_matching_hosts(lambda label=None, as_hostspec=None: all_hosts) + self._drive_group_map[str(drive_group.service_id)] = drive_group + self._save_drive_groups() + return OrchResult(self._apply_drivegroups(specs)) + + def _apply_drivegroups(self, ls: List[DriveGroupSpec]) -> List[str]: + all_hosts = raise_if_exception(self.get_hosts()) + result_list: List[str] = [] + for drive_group in ls: + matching_hosts = drive_group.placement.filter_matching_hosts( + lambda label=None, as_hostspec=None: all_hosts + ) if not self.rook_cluster.node_exists(matching_hosts[0]): raise RuntimeError("Node '{0}' is not in the Kubernetes " @@ -538,7 +562,23 @@ def apply_drivegroups(self, specs: List[DriveGroupSpec]) -> OrchResult[List[str] raise RuntimeError("Rook cluster configuration does not " "support OSD creation.") result_list.append(self.rook_cluster.add_osds(drive_group, matching_hosts)) - return OrchResult(result_list) + return result_list + + def _load_drive_groups(self) -> None: + stored_drive_group = self.get_store("drive_group_map") + self._drive_group_map: Dict[str, DriveGroupSpec] = {} + if stored_drive_group: + for name, dg in json.loads(stored_drive_group).items(): + try: + self._drive_group_map[name] = DriveGroupSpec.from_json(dg) + except ValueError as e: + self.log.error(f'Failed to load drive group {name} ({dg}): {e}') + + def _save_drive_groups(self) -> None: + json_drive_group_map = { + name: dg.to_json() for name, dg in self._drive_group_map.items() + } + self.set_store("drive_group_map", json.dumps(json_drive_group_map)) def remove_osds(self, osd_ids: List[str], replace: bool = False, force: bool = False, zap: bool = False) -> OrchResult[str]: assert self._rook_cluster is not None From 4b798d1cda5c8f899faec81034a282a83974cb2d Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Tue, 2 Nov 2021 11:28:10 -0500 Subject: [PATCH 0522/2866] mgr/rook/rook_cluster: do not duplicate StorageClassDeviceSets Signed-off-by: Joseph Sawaya --- src/pybind/mgr/rook/rook_cluster.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/pybind/mgr/rook/rook_cluster.py b/src/pybind/mgr/rook/rook_cluster.py index a4bc0917ef6a8..4a080fa922379 100644 --- a/src/pybind/mgr/rook/rook_cluster.py +++ b/src/pybind/mgr/rook/rook_cluster.py @@ -415,9 +415,14 @@ def _add_osds(current_cluster, new_cluster): if not hasattr(new_cluster.spec.storage, 'storageClassDeviceSets') or not new_cluster.spec.storage.storageClassDeviceSets: new_cluster.spec.storage.storageClassDeviceSets = ccl.StorageClassDeviceSetsList() + existing_scds = [ + scds.name for scds in new_cluster.spec.storage.storageClassDeviceSets + ] for device in to_create: new_scds = self.device_to_device_set(drive_group, device) new_cluster.spec.storage.storageClassDeviceSets.append(new_scds) + if new_scds.name not in existing_scds: + new_cluster.spec.storage.storageClassDeviceSets.append(new_scds) return new_cluster return _add_osds From 7d0de1bc28ebee066217eb953900f2320487d9a2 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Tue, 2 Nov 2021 11:31:44 -0500 Subject: [PATCH 0523/2866] mgr/rook/rook_cluster: remove drivegroup map and loop Signed-off-by: Sage Weil --- src/pybind/mgr/rook/rook_cluster.py | 32 ++++++++++++----------------- 1 file changed, 13 insertions(+), 19 deletions(-) diff --git a/src/pybind/mgr/rook/rook_cluster.py b/src/pybind/mgr/rook/rook_cluster.py index 4a080fa922379..f837c1014d71f 100644 --- a/src/pybind/mgr/rook/rook_cluster.py +++ b/src/pybind/mgr/rook/rook_cluster.py @@ -658,8 +658,6 @@ def __init__(self, coreV1_api: 'client.CoreV1Api', batchV1_api: 'client.BatchV1A label_selector="rook_cluster={0}".format( self.rook_env.namespace)) self.nodes: KubernetesResource[client.V1Node] = KubernetesResource(self.coreV1_api.list_node) - self.drive_group_map: Dict[str, Any] = {} - self.drive_group_lock = threading.Lock() def rook_url(self, path: str) -> str: prefix = "/apis/ceph.rook.io/%s/namespaces/%s/" % ( @@ -1091,24 +1089,20 @@ def add_osds(self, drive_group, matching_hosts): assert drive_group.service_id storage_class = self.get_storage_class() inventory = self.get_discovered_devices() - self.creator: Optional[DefaultCreator] = None - if storage_class.metadata.labels and ('local.storage.openshift.io/owner-name' in storage_class.metadata.labels): - self.creator = LSOCreator(inventory, self.coreV1_api, self.storage_class) + creator: Optional[DefaultCreator] = None + if ( + storage_class.metadata.labels + and 'local.storage.openshift.io/owner-name' in storage_class.metadata.labels + ): + creator = LSOCreator(inventory, self.coreV1_api, self.storage_class) else: - self.creator = DefaultCreator(inventory, self.coreV1_api, self.storage_class) - _add_osds = self.creator.add_osds(self.rook_pods, drive_group, matching_hosts) - with self.drive_group_lock: - self.drive_group_map[drive_group.service_id] = _add_osds - return self._patch(ccl.CephCluster, 'cephclusters', self.rook_env.cluster_name, _add_osds) - - @threaded - def drive_group_loop(self) -> None: - ten_minutes = 10 * 60 - while True: - sleep(ten_minutes) - with self.drive_group_lock: - for _, add_osd in self.drive_group_map.items(): - self._patch(ccl.CephCluster, 'cephclusters', self.rook_env.cluster_name, add_osd) + creator = DefaultCreator(inventory, self.coreV1_api, self.storage_class) + return self._patch( + ccl.CephCluster, + 'cephclusters', + self.rook_env.cluster_name, + creator.add_osds(self.rook_pods, drive_group, matching_hosts) + ) def remove_osds(self, osd_ids: List[str], replace: bool, force: bool, mon_command: Callable) -> str: inventory = self.get_discovered_devices() From 873c2905f0895d04b1abeaef8553921bf2a2abb2 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Wed, 3 Nov 2021 13:49:24 -0500 Subject: [PATCH 0524/2866] mgr/rook/rook_cluster: whitespace Signed-off-by: Sage Weil --- src/pybind/mgr/rook/rook_cluster.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/pybind/mgr/rook/rook_cluster.py b/src/pybind/mgr/rook/rook_cluster.py index f837c1014d71f..48e8bc95d240a 100644 --- a/src/pybind/mgr/rook/rook_cluster.py +++ b/src/pybind/mgr/rook/rook_cluster.py @@ -641,7 +641,16 @@ def remove(self) -> str: class RookCluster(object): # import of client.CoreV1Api must be optional at import time. # Instead allow mgr/rook to be imported anyway. - def __init__(self, coreV1_api: 'client.CoreV1Api', batchV1_api: 'client.BatchV1Api', customObjects_api: 'client.CustomObjectsApi', storageV1_api: 'client.StorageV1Api', appsV1_api: 'client.AppsV1Api', rook_env: 'RookEnv', storage_class: 'str'): + def __init__( + self, + coreV1_api: 'client.CoreV1Api', + batchV1_api: 'client.BatchV1Api', + customObjects_api: 'client.CustomObjectsApi', + storageV1_api: 'client.StorageV1Api', + appsV1_api: 'client.AppsV1Api', + rook_env: 'RookEnv', + storage_class: 'str' + ): self.rook_env = rook_env # type: RookEnv self.coreV1_api = coreV1_api # client.CoreV1Api self.batchV1_api = batchV1_api From 2aeb9263d643b19d59219e8e187e1a0fa0292693 Mon Sep 17 00:00:00 2001 From: Ronen Friedman Date: Sun, 25 Jul 2021 14:58:51 +0300 Subject: [PATCH 0525/2866] osd/scrub: remove reliance of Scrubber objects' logging on the PG Modify the Scrubber's sub-objects to use their own gen_prefix() functions, instead of using PG::gen_prefix(). Fixes: https://tracker.ceph.com/issues/51843 Signed-off-by: Ronen Friedman --- src/osd/PG.h | 2 - src/osd/scrubber/PrimaryLogScrub.cc | 9 +- src/osd/scrubber/pg_scrubber.cc | 123 ++++++++++++++----------- src/osd/scrubber/pg_scrubber.h | 13 ++- src/osd/scrubber/scrub_machine.cc | 25 +++-- src/osd/scrubber/scrub_machine.h | 4 +- src/osd/scrubber/scrub_machine_lstnr.h | 3 + 7 files changed, 105 insertions(+), 74 deletions(-) diff --git a/src/osd/PG.h b/src/osd/PG.h index 63bcb7c11d0c7..de6a0a32bed75 100644 --- a/src/osd/PG.h +++ b/src/osd/PG.h @@ -176,8 +176,6 @@ class PG : public DoutPrefixProvider, public PeeringState::PeeringListener { friend struct NamedState; friend class PeeringState; friend class PgScrubber; - friend class Scrub::LocalReservation; // dout()-only friendship - friend class Scrub::ReservedByRemotePrimary; // dout()-only friendship public: const pg_shard_t pg_whoami; diff --git a/src/osd/scrubber/PrimaryLogScrub.cc b/src/osd/scrubber/PrimaryLogScrub.cc index 2be7b56f61b03..aafec30592320 100644 --- a/src/osd/scrubber/PrimaryLogScrub.cc +++ b/src/osd/scrubber/PrimaryLogScrub.cc @@ -10,16 +10,17 @@ #include "osd/PrimaryLogPG.h" #include "scrub_machine.h" -#define dout_context (m_pg->get_cct()) +#define dout_context (m_osds->cct) #define dout_subsys ceph_subsys_osd #undef dout_prefix -#define dout_prefix _prefix(_dout, this->m_pg) +#define dout_prefix _prefix(_dout, this) using std::vector; -template static ostream& _prefix(std::ostream* _dout, T* t) +template +static ostream& _prefix(std::ostream* _dout, T* t) { - return t->gen_prefix(*_dout) << " PrimaryLog scrubber pg(" << t->pg_id << ") "; + return t->gen_prefix(*_dout); } using namespace Scrub; diff --git a/src/osd/scrubber/pg_scrubber.cc b/src/osd/scrubber/pg_scrubber.cc index df7fe95a41db8..8f08e499be824 100644 --- a/src/osd/scrubber/pg_scrubber.cc +++ b/src/osd/scrubber/pg_scrubber.cc @@ -30,14 +30,15 @@ using namespace std::chrono; using namespace std::chrono_literals; using namespace std::literals; -#define dout_context (m_pg->get_cct()) +#define dout_context (m_osds->cct) #define dout_subsys ceph_subsys_osd #undef dout_prefix -#define dout_prefix _prefix(_dout, this->m_pg) +#define dout_prefix _prefix(_dout, this) -template static ostream& _prefix(std::ostream* _dout, T* t) +template +static ostream& _prefix(std::ostream* _dout, T* t) { - return t->gen_prefix(*_dout) << " scrubber pg(" << t->pg_id << ") "; + return t->gen_prefix(*_dout); } ostream& operator<<(ostream& out, const scrub_flags_t& sf) @@ -182,7 +183,6 @@ void PgScrubber::initiate_regular_scrub(epoch_t epoch_queued) if (check_interval(epoch_queued)) { dout(10) << "scrubber event -->> StartScrub epoch: " << epoch_queued << dendl; reset_epoch(epoch_queued); - m_fsm->my_states(); m_fsm->process_event(StartScrub{}); dout(10) << "scrubber event --<< StartScrub" << dendl; } @@ -195,7 +195,6 @@ void PgScrubber::initiate_scrub_after_repair(epoch_t epoch_queued) if (check_interval(epoch_queued)) { dout(10) << "scrubber event -->> AfterRepairScrub epoch: " << epoch_queued << dendl; reset_epoch(epoch_queued); - m_fsm->my_states(); m_fsm->process_event(AfterRepairScrub{}); dout(10) << "scrubber event --<< AfterRepairScrub" << dendl; } @@ -205,7 +204,6 @@ void PgScrubber::send_scrub_unblock(epoch_t epoch_queued) { dout(10) << "scrubber event -->> " << __func__ << " epoch: " << epoch_queued << dendl; if (is_message_relevant(epoch_queued)) { - m_fsm->my_states(); m_fsm->process_event(Unblocked{}); } dout(10) << "scrubber event --<< " << __func__ << dendl; @@ -215,7 +213,6 @@ void PgScrubber::send_scrub_resched(epoch_t epoch_queued) { dout(10) << "scrubber event -->> " << __func__ << " epoch: " << epoch_queued << dendl; if (is_message_relevant(epoch_queued)) { - m_fsm->my_states(); m_fsm->process_event(InternalSchedScrub{}); } dout(10) << "scrubber event --<< " << __func__ << dendl; @@ -232,7 +229,6 @@ void PgScrubber::send_start_replica(epoch_t epoch_queued, Scrub::act_token_t tok } if (check_interval(epoch_queued) && is_token_current(token)) { - m_fsm->my_states(); // save us some time by not waiting for updates if there are none // to wait for. Affects the transition from NotActive into either // ReplicaWaitUpdates or ActiveReplica. @@ -249,7 +245,6 @@ void PgScrubber::send_sched_replica(epoch_t epoch_queued, Scrub::act_token_t tok dout(10) << "scrubber event -->> " << __func__ << " epoch: " << epoch_queued << " token: " << token << dendl; if (check_interval(epoch_queued) && is_token_current(token)) { - m_fsm->my_states(); m_fsm->process_event(SchedReplica{}); // retest for map availability } dout(10) << "scrubber event --<< " << __func__ << dendl; @@ -260,7 +255,6 @@ void PgScrubber::active_pushes_notification(epoch_t epoch_queued) // note: Primary only dout(10) << "scrubber event -->> " << __func__ << " epoch: " << epoch_queued << dendl; if (is_message_relevant(epoch_queued)) { - m_fsm->my_states(); m_fsm->process_event(ActivePushesUpd{}); } dout(10) << "scrubber event --<< " << __func__ << dendl; @@ -271,7 +265,6 @@ void PgScrubber::update_applied_notification(epoch_t epoch_queued) // note: Primary only dout(10) << "scrubber event -->> " << __func__ << " epoch: " << epoch_queued << dendl; if (is_message_relevant(epoch_queued)) { - m_fsm->my_states(); m_fsm->process_event(UpdatesApplied{}); } dout(10) << "scrubber event --<< " << __func__ << dendl; @@ -282,7 +275,6 @@ void PgScrubber::digest_update_notification(epoch_t epoch_queued) // note: Primary only dout(10) << "scrubber event -->> " << __func__ << " epoch: " << epoch_queued << dendl; if (is_message_relevant(epoch_queued)) { - m_fsm->my_states(); m_fsm->process_event(DigestUpdate{}); } dout(10) << "scrubber event --<< " << __func__ << dendl; @@ -292,7 +284,6 @@ void PgScrubber::send_local_map_done(epoch_t epoch_queued) { dout(10) << "scrubber event -->> " << __func__ << " epoch: " << epoch_queued << dendl; if (is_message_relevant(epoch_queued)) { - m_fsm->my_states(); m_fsm->process_event(Scrub::IntLocalMapDone{}); } dout(10) << "scrubber event --<< " << __func__ << dendl; @@ -302,7 +293,6 @@ void PgScrubber::send_replica_maps_ready(epoch_t epoch_queued) { dout(10) << "scrubber event -->> " << __func__ << " epoch: " << epoch_queued << dendl; if (is_message_relevant(epoch_queued)) { - m_fsm->my_states(); m_fsm->process_event(GotReplicas{}); } dout(10) << "scrubber event --<< " << __func__ << dendl; @@ -312,7 +302,6 @@ void PgScrubber::send_replica_pushes_upd(epoch_t epoch_queued) { dout(10) << "scrubber event -->> " << __func__ << " epoch: " << epoch_queued << dendl; if (check_interval(epoch_queued)) { - m_fsm->my_states(); m_fsm->process_event(ReplicaPushesUpd{}); } dout(10) << "scrubber event --<< " << __func__ << dendl; @@ -323,7 +312,6 @@ void PgScrubber::send_remotes_reserved(epoch_t epoch_queued) dout(10) << "scrubber event -->> " << __func__ << " epoch: " << epoch_queued << dendl; // note: scrub is not active yet if (check_interval(epoch_queued)) { - m_fsm->my_states(); m_fsm->process_event(RemotesReserved{}); } dout(10) << "scrubber event --<< " << __func__ << dendl; @@ -333,7 +321,6 @@ void PgScrubber::send_reservation_failure(epoch_t epoch_queued) { dout(10) << "scrubber event -->> " << __func__ << " epoch: " << epoch_queued << dendl; if (check_interval(epoch_queued)) { // do not check for 'active'! - m_fsm->my_states(); m_fsm->process_event(ReservationFailure{}); } dout(10) << "scrubber event --<< " << __func__ << dendl; @@ -343,7 +330,6 @@ void PgScrubber::send_full_reset(epoch_t epoch_queued) { dout(10) << "scrubber event -->> " << __func__ << " epoch: " << epoch_queued << dendl; - m_fsm->my_states(); m_fsm->process_event(Scrub::FullReset{}); dout(10) << "scrubber event --<< " << __func__ << dendl; @@ -353,7 +339,6 @@ void PgScrubber::send_chunk_free(epoch_t epoch_queued) { dout(10) << "scrubber event -->> " << __func__ << " epoch: " << epoch_queued << dendl; if (check_interval(epoch_queued)) { - m_fsm->my_states(); m_fsm->process_event(Scrub::SelectedChunkFree{}); } dout(10) << "scrubber event --<< " << __func__ << dendl; @@ -363,7 +348,6 @@ void PgScrubber::send_chunk_busy(epoch_t epoch_queued) { dout(10) << "scrubber event -->> " << __func__ << " epoch: " << epoch_queued << dendl; if (check_interval(epoch_queued)) { - m_fsm->my_states(); m_fsm->process_event(Scrub::ChunkIsBusy{}); } dout(10) << "scrubber event --<< " << __func__ << dendl; @@ -373,7 +357,6 @@ void PgScrubber::send_get_next_chunk(epoch_t epoch_queued) { dout(10) << "scrubber event -->> " << __func__ << " epoch: " << epoch_queued << dendl; if (is_message_relevant(epoch_queued)) { - m_fsm->my_states(); m_fsm->process_event(Scrub::NextChunk{}); } dout(10) << "scrubber event --<< " << __func__ << dendl; @@ -385,7 +368,6 @@ void PgScrubber::send_scrub_is_finished(epoch_t epoch_queued) // can't check for "active" - m_fsm->my_states(); m_fsm->process_event(Scrub::ScrubFinished{}); dout(10) << "scrubber event --<< " << __func__ << dendl; @@ -395,7 +377,6 @@ void PgScrubber::send_maps_compared(epoch_t epoch_queued) { dout(10) << "scrubber event -->> " << __func__ << " epoch: " << epoch_queued << dendl; - m_fsm->my_states(); m_fsm->process_event(Scrub::MapsCompared{}); dout(10) << "scrubber event --<< " << __func__ << dendl; @@ -594,13 +575,15 @@ bool PgScrubber::reserve_local() // try to create the reservation object (which translates into asking the // OSD for the local scrub resource). If failing - undo it immediately - m_local_osd_resource.emplace(m_pg, m_osds); - if (!m_local_osd_resource->is_reserved()) { - m_local_osd_resource.reset(); - return false; + m_local_osd_resource.emplace(m_osds); + if (m_local_osd_resource->is_reserved()) { + dout(15) << __func__ << ": local resources reserved" << dendl; + return true; } - return true; + dout(10) << __func__ << ": failed to reserve local scrub resources" << dendl; + m_local_osd_resource.reset(); + return false; } // ---------------------------------------------------------------------------- @@ -1549,7 +1532,7 @@ void PgScrubber::handle_scrub_reserve_request(OpRequestRef op) if (m_remote_osd_resource.has_value() && m_remote_osd_resource->is_stale()) { // we are holding a stale reservation from a past epoch m_remote_osd_resource.reset(); - dout(10) << __func__ << " stale reservation request" << dendl; + dout(10) << __func__ << " cleared existing stale reservation" << dendl; } if (request_ep < m_pg->get_same_interval_since()) { @@ -1574,7 +1557,7 @@ void PgScrubber::handle_scrub_reserve_request(OpRequestRef op) } else if (m_pg->cct->_conf->osd_scrub_during_recovery || !m_osds->is_recovery_active()) { - m_remote_osd_resource.emplace(m_pg, m_osds, request_ep); + m_remote_osd_resource.emplace(this, m_pg, m_osds, request_ep); // OSD resources allocated? granted = m_remote_osd_resource->is_reserved(); if (!granted) { @@ -1650,7 +1633,7 @@ void PgScrubber::message_all_replicas(int32_t opcode, std::string_view op_text) { ceph_assert(m_pg->recovery_state.get_backfill_targets().empty()); - std::vector> messages; + std::vector> messages; messages.reserve(m_pg->get_actingset().size()); epoch_t epch = get_osdmap_epoch(); @@ -2167,6 +2150,16 @@ ostream& operator<<(ostream& out, const PgScrubber& scrubber) return out << scrubber.m_flags; } +std::ostream& PgScrubber::gen_prefix(std::ostream& out) const +{ + const auto fsm_state = m_fsm ? m_fsm->current_states_desc() : "- :"; + if (m_pg) { + return m_pg->gen_prefix(out) << "scrubber " << fsm_state << ": "; + } else { + return out << " scrubber [~] " << fsm_state << ": "; + } +} + ostream& PgScrubber::show(ostream& out) const { return out << " [ " << m_pg_id << ": " << m_flags << " ] "; @@ -2229,6 +2222,13 @@ ReplicaReservations::ReplicaReservations(PG* pg, pg_shard_t whoami, ScrubQueue:: { epoch_t epoch = m_pg->get_osdmap_epoch(); + { + std::stringstream prefix; + prefix << "osd." << m_osds->whoami << " ep: " << epoch + << " scrubber::ReplicaReservations pg[" << pg->pg_id << "]: "; + m_log_msg_prefix = prefix.str(); + } + // handle the special case of no replicas if (m_pending <= 0) { // just signal the scrub state-machine to continue @@ -2243,7 +2243,7 @@ ReplicaReservations::ReplicaReservations(PG* pg, pg_shard_t whoami, ScrubQueue:: MOSDScrubReserve::REQUEST, m_pg->pg_whoami); m_osds->send_message_osd_cluster(p.osd, m, epoch); m_waited_for_peers.push_back(p); - dout(10) << __func__ << " reserve<-> " << p.osd << dendl; + dout(10) << __func__ << ": reserve " << p.osd << dendl; } } } @@ -2261,7 +2261,7 @@ void ReplicaReservations::send_reject() void ReplicaReservations::discard_all() { - dout(10) << __func__ << " " << m_reserved_peers << dendl; + dout(10) << __func__ << ": " << m_reserved_peers << dendl; m_had_rejections = true; // preventing late-coming responses from triggering events m_reserved_peers.clear(); @@ -2297,7 +2297,7 @@ ReplicaReservations::~ReplicaReservations() */ void ReplicaReservations::handle_reserve_grant(OpRequestRef op, pg_shard_t from) { - dout(10) << __func__ << " granted-> " << from << dendl; + dout(10) << __func__ << ": granted by " << from << dendl; op->mark_started(); { @@ -2310,17 +2310,19 @@ void ReplicaReservations::handle_reserve_grant(OpRequestRef op, pg_shard_t from) // are we forced to reject the reservation? if (m_had_rejections) { - dout(10) << " rejecting late-coming reservation from " << from << dendl; + dout(10) << __func__ << ": rejecting late-coming reservation from " + << from << dendl; release_replica(from, m_pg->get_osdmap_epoch()); } else if (std::find(m_reserved_peers.begin(), m_reserved_peers.end(), from) != m_reserved_peers.end()) { - dout(10) << " already had osd." << from << " reserved" << dendl; + dout(10) << __func__ << ": already had osd." << from << " reserved" << dendl; } else { - dout(10) << " osd." << from << " scrub reserve = success" << dendl; + dout(10) << __func__ << ": osd." << from << " scrub reserve = success" + << dendl; m_reserved_peers.push_back(from); if (--m_pending == 0) { send_all_done(); @@ -2330,8 +2332,8 @@ void ReplicaReservations::handle_reserve_grant(OpRequestRef op, pg_shard_t from) void ReplicaReservations::handle_reserve_reject(OpRequestRef op, pg_shard_t from) { - dout(10) << __func__ << " rejected-> " << from << dendl; - dout(10) << __func__ << " " << *op->get_req() << dendl; + dout(10) << __func__ << ": rejected by " << from << dendl; + dout(15) << __func__ << ": " << *op->get_req() << dendl; op->mark_started(); { @@ -2344,36 +2346,37 @@ void ReplicaReservations::handle_reserve_reject(OpRequestRef op, pg_shard_t from if (m_had_rejections) { // our failure was already handled when the first rejection arrived - dout(15) << " ignoring late-coming rejection from " << from << dendl; + dout(15) << __func__ << ": ignoring late-coming rejection from " + << from << dendl; } else if (std::find(m_reserved_peers.begin(), m_reserved_peers.end(), from) != m_reserved_peers.end()) { - dout(10) << " already had osd." << from << " reserved" << dendl; + dout(10) << __func__ << ": already had osd." << from << " reserved" << dendl; } else { - dout(10) << " osd." << from << " scrub reserve = fail" << dendl; + dout(10) << __func__ << ": osd." << from << " scrub reserve = fail" << dendl; m_had_rejections = true; // preventing any additional notifications send_reject(); } } +std::ostream& ReplicaReservations::gen_prefix(std::ostream& out) const +{ + return out << m_log_msg_prefix; +} // ///////////////////// LocalReservation ////////////////////////////////// -LocalReservation::LocalReservation(PG* pg, OSDService* osds) - : m_pg{pg} // holding the "whole PG" for dout() sake - , m_osds{osds} +// note: no dout()s in LocalReservation functions. Client logs interactions. +LocalReservation::LocalReservation(OSDService* osds) + : m_osds{osds} { - if (!m_osds->get_scrub_services().inc_scrubs_local()) { - dout(10) << __func__ << ": failed to reserve locally " << dendl; + if (m_osds->get_scrub_services().inc_scrubs_local()) { // the failure is signalled by not having m_holding_local_reservation set - return; + m_holding_local_reservation = true; } - - dout(20) << __func__ << ": local OSD scrub resources reserved" << dendl; - m_holding_local_reservation = true; } LocalReservation::~LocalReservation() @@ -2384,11 +2387,16 @@ LocalReservation::~LocalReservation() } } - // ///////////////////// ReservedByRemotePrimary /////////////////////////////// -ReservedByRemotePrimary::ReservedByRemotePrimary(PG* pg, OSDService* osds, epoch_t epoch) - : m_pg{pg}, m_osds{osds}, m_reserved_at{epoch} +ReservedByRemotePrimary::ReservedByRemotePrimary(const PgScrubber* scrubber, + PG* pg, + OSDService* osds, + epoch_t epoch) + : m_scrubber{scrubber} + , m_pg{pg} + , m_osds{osds} + , m_reserved_at{epoch} { if (!m_osds->get_scrub_services().inc_scrubs_remote()) { dout(10) << __func__ << ": failed to reserve at Primary request" << dendl; @@ -2413,6 +2421,11 @@ ReservedByRemotePrimary::~ReservedByRemotePrimary() } } +std::ostream& ReservedByRemotePrimary::gen_prefix(std::ostream& out) const +{ + return m_scrubber->gen_prefix(out); +} + // ///////////////////// MapsCollectionStatus //////////////////////////////// auto MapsCollectionStatus::mark_arriving_map(pg_shard_t from) diff --git a/src/osd/scrubber/pg_scrubber.h b/src/osd/scrubber/pg_scrubber.h index a9845efb1dd5c..b99287755be92 100644 --- a/src/osd/scrubber/pg_scrubber.h +++ b/src/osd/scrubber/pg_scrubber.h @@ -57,6 +57,8 @@ class ReplicaReservations { void send_reject(); public: + std::string m_log_msg_prefix; + /** * quietly discard all knowledge about existing reservations. No messages * are sent to peers. @@ -72,18 +74,19 @@ class ReplicaReservations { void handle_reserve_grant(OpRequestRef op, pg_shard_t from); void handle_reserve_reject(OpRequestRef op, pg_shard_t from); + + std::ostream& gen_prefix(std::ostream& out) const; }; /** * wraps the local OSD scrub resource reservation in an RAII wrapper */ class LocalReservation { - PG* m_pg; OSDService* m_osds; bool m_holding_local_reservation{false}; public: - LocalReservation(PG* pg, OSDService* osds); + LocalReservation(OSDService* osds); ~LocalReservation(); bool is_reserved() const { return m_holding_local_reservation; } }; @@ -92,18 +95,21 @@ class LocalReservation { * wraps the OSD resource we are using when reserved as a replica by a scrubbing master. */ class ReservedByRemotePrimary { + const PgScrubber* m_scrubber; ///< we will be using its gen_prefix() PG* m_pg; OSDService* m_osds; bool m_reserved_by_remote_primary{false}; const epoch_t m_reserved_at; public: - ReservedByRemotePrimary(PG* pg, OSDService* osds, epoch_t epoch); + ReservedByRemotePrimary(const PgScrubber* scrubber, PG* pg, OSDService* osds, epoch_t epoch); ~ReservedByRemotePrimary(); [[nodiscard]] bool is_reserved() const { return m_reserved_by_remote_primary; } /// compare the remembered reserved-at epoch to the current interval [[nodiscard]] bool is_stale() const; + + std::ostream& gen_prefix(std::ostream& out) const; }; /** @@ -426,6 +432,7 @@ class PgScrubber : public ScrubPgIF, public ScrubMachineListener { void set_scrub_duration() final; utime_t scrub_begin_stamp; + std::ostream& gen_prefix(std::ostream& out) const final; protected: bool state_test(uint64_t m) const { return m_pg->state_test(m); } diff --git a/src/osd/scrubber/scrub_machine.cc b/src/osd/scrubber/scrub_machine.cc index d59fbe7e7f1c7..5190c8d8510e6 100644 --- a/src/osd/scrubber/scrub_machine.cc +++ b/src/osd/scrubber/scrub_machine.cc @@ -41,12 +41,15 @@ void on_event_discard(std::string_view nm) dout(20) << " event: --^^^^---- " << nm << dendl; } -void ScrubMachine::my_states() const +std::string ScrubMachine::current_states_desc() const { + std::string sts{"<"}; for (auto si = state_begin(); si != state_end(); ++si) { - const auto& siw{*si}; // prevents a warning re side-effects - dout(20) << " state: " << boost::core::demangle(typeid(siw).name()) << dendl; + const auto& siw{ *si }; // prevents a warning re side-effects + // the '7' is the size of the 'scrub::' + sts += boost::core::demangle(typeid(siw).name()).substr(7, std::string::npos) + "/"; } + return sts + ">"; } void ScrubMachine::assert_not_active() const @@ -69,10 +72,17 @@ bool ScrubMachine::is_accepting_updates() const // for the rest of the code in this file - we know what PG we are dealing with: #undef dout_prefix -#define dout_prefix _prefix(_dout, this->context().m_pg) -template static ostream& _prefix(std::ostream* _dout, T* t) +#define dout_prefix _prefix(_dout, this->context()) + +template +static ostream& _prefix(std::ostream* _dout, T& t) +{ + return t.gen_prefix(*_dout); +} + +std::ostream& ScrubMachine::gen_prefix(std::ostream& out) const { - return t->gen_prefix(*_dout) << " scrubberFSM pg(" << t->pg_id << ") "; + return m_scrbr->gen_prefix(out) << "FSM: "; } // ////////////// the actual actions @@ -457,9 +467,8 @@ sc::result WaitDigestUpdate::react(const ScrubFinished&) } ScrubMachine::ScrubMachine(PG* pg, ScrubMachineListener* pg_scrub) - : m_pg{pg}, m_pg_id{pg->pg_id}, m_scrbr{pg_scrub} + : m_pg_id{pg->pg_id}, m_scrbr{pg_scrub} { - dout(15) << "ScrubMachine created " << m_pg_id << dendl; } ScrubMachine::~ScrubMachine() = default; diff --git a/src/osd/scrubber/scrub_machine.h b/src/osd/scrubber/scrub_machine.h index f3c8d2ced603a..75a7e02207ba0 100644 --- a/src/osd/scrubber/scrub_machine.h +++ b/src/osd/scrubber/scrub_machine.h @@ -124,11 +124,11 @@ class ScrubMachine : public sc::state_machine { explicit ScrubMachine(PG* pg, ScrubMachineListener* pg_scrub); ~ScrubMachine(); - PG* m_pg; // only used for dout messages spg_t m_pg_id; ScrubMachineListener* m_scrbr; + std::ostream& gen_prefix(std::ostream& out) const; - void my_states() const; + std::string current_states_desc() const; void assert_not_active() const; [[nodiscard]] bool is_reserving() const; [[nodiscard]] bool is_accepting_updates() const; diff --git a/src/osd/scrubber/scrub_machine_lstnr.h b/src/osd/scrubber/scrub_machine_lstnr.h index 72ff4b7fdd55f..91c20e34946d1 100644 --- a/src/osd/scrubber/scrub_machine_lstnr.h +++ b/src/osd/scrubber/scrub_machine_lstnr.h @@ -174,4 +174,7 @@ struct ScrubMachineListener { /// a log/debug interface virtual std::string dump_awaited_maps() const = 0; + + /// exposed to be used by the scrub_machine logger + virtual std::ostream& gen_prefix(std::ostream& out) const = 0; }; From a25d1a70aa20bad91b4544574f1d52094c6cfd69 Mon Sep 17 00:00:00 2001 From: Casey Bodley Date: Thu, 4 Nov 2021 12:12:04 -0400 Subject: [PATCH 0526/2866] qa/rgw: master branch targets ceph-master branch of java_s3tests Signed-off-by: Casey Bodley --- qa/suites/rgw/verify/tasks/s3tests-java.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qa/suites/rgw/verify/tasks/s3tests-java.yaml b/qa/suites/rgw/verify/tasks/s3tests-java.yaml index 06c134c37544f..9ad89cc6790a7 100644 --- a/qa/suites/rgw/verify/tasks/s3tests-java.yaml +++ b/qa/suites/rgw/verify/tasks/s3tests-java.yaml @@ -1,6 +1,6 @@ tasks: - s3tests-java: client.0: - force-branch: master + force-branch: ceph-master force-repo: https://github.com/ceph/java_s3tests.git From ab5375076102f4991d6579244b0583e93d979787 Mon Sep 17 00:00:00 2001 From: Laura Flores Date: Thu, 4 Nov 2021 17:55:51 +0000 Subject: [PATCH 0527/2866] mgr/telemetry: modify stats_per_pool There is a much easier way to collect stats_per_pool than the current implementation. Fetching 'pg_dump' from the mgr module already provides a field called "pool_stats" that is the same as aggregated pg stats, which was the implementation up until this commit. All in all, this solution should provide the information we want, with a much cleaner implementation. Signed-off-by: Laura Flores Backport Message: In the case that this commit is backported, it is important to note that the commits in PR #42569 should be backported first, as the implementation of "get_stat_sum_per_pool()" in #42569 precedes the removal of it here. --- src/pybind/mgr/telemetry/module.py | 49 ++---------------------------- 1 file changed, 2 insertions(+), 47 deletions(-) diff --git a/src/pybind/mgr/telemetry/module.py b/src/pybind/mgr/telemetry/module.py index e54a4621891ff..15203188727ee 100644 --- a/src/pybind/mgr/telemetry/module.py +++ b/src/pybind/mgr/telemetry/module.py @@ -338,52 +338,6 @@ def get_mempool(self, mode: str = 'separated') -> Dict[str, dict]: return result - def get_stat_sum_per_pool(self) -> List[dict]: - # Initialize 'result' list - result: List[dict] = [] - - # Create a list of pool ids that will later act as a queue, i.e.: - # pool_queue = [1, 2, 3] - osd_map = self.get('osd_map') - pool_queue = [] - for pool in osd_map['pools']: - pool_queue.append(str(pool['pool'])) - - # Populate 'result', i.e.: - # { - # 'pool_id': '1' - # 'stats_sum': { - # 'num_bytes': 36, - # 'num_bytes_hit_set_archive': 0, - # ... - # 'num_write_kb': 0 - # } - # } - # } - while pool_queue: - # Pop the current pool id out of pool_queue - curr_pool_id = pool_queue.pop(0) - - # Initialize a dict that will hold aggregated stats for the current pool - compiled_stats_dict: Dict[str, Any] = defaultdict(lambda: defaultdict(int)) - - # Find out which pgs belong to the current pool and add up - # their stats - pg_dump = self.get('pg_dump') - for pg in pg_dump['pg_stats']: - pool_id = pg['pgid'].split('.')[0] - if pool_id == curr_pool_id: - compiled_stats_dict['pool_id'] = int(pool_id) - for metric in pg['stat_sum']: - compiled_stats_dict['stats_sum'][metric] += pg['stat_sum'][metric] - else: - continue - # 'compiled_stats_dict' now holds all stats pertaining to - # the current pool. Adding it to the list of results. - result.append(compiled_stats_dict) - - return result - def get_osd_histograms(self, mode: str = 'separated') -> List[Dict[str, dict]]: # Initialize result dict result: Dict[str, dict] = defaultdict(lambda: defaultdict( @@ -949,7 +903,8 @@ def compile_report(self, channels: Optional[List[str]] = None) -> Dict[str, Any] report['perf_counters_aggregated'] = self.gather_perf_counters('aggregated') report['perf_counters_separated'] = self.gather_perf_counters('separated') - report['stat_sum_per_pool'] = self.get_stat_sum_per_pool() + report['stats_per_pool'] = self.get('pg_dump')['pool_stats'] + report['io_rate'] = self.get_io_rate() report['osd_perf_histograms_aggregated'] = self.get_osd_histograms('aggregated') From ec3d0fd4d3b1676886565b13da3a640d035caec6 Mon Sep 17 00:00:00 2001 From: Laura Flores Date: Tue, 2 Nov 2021 02:00:58 +0000 Subject: [PATCH 0528/2866] mgr/telemetry: add stats_per_pg We decided that it might be more useful to report data on a pg level in addition to what we were reporting for pool level. Signed-off-by: Laura Flores --- src/pybind/mgr/telemetry/module.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/pybind/mgr/telemetry/module.py b/src/pybind/mgr/telemetry/module.py index 15203188727ee..508e400b9f3e0 100644 --- a/src/pybind/mgr/telemetry/module.py +++ b/src/pybind/mgr/telemetry/module.py @@ -904,6 +904,7 @@ def compile_report(self, channels: Optional[List[str]] = None) -> Dict[str, Any] report['perf_counters_separated'] = self.gather_perf_counters('separated') report['stats_per_pool'] = self.get('pg_dump')['pool_stats'] + report['stats_per_pg'] = self.get('pg_dump')['pg_stats'] report['io_rate'] = self.get_io_rate() From 1be081da02a45ced9c5f2d6b14a7362028fa70f8 Mon Sep 17 00:00:00 2001 From: Neha Ojha Date: Mon, 1 Nov 2021 23:55:22 +0000 Subject: [PATCH 0529/2866] qa/suites/upgrade/octopus-x/stress-split-no-cephadm: exclude ceph-volume To address failures like ``` Command failed on smithi096 with status 100: 'sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install ceph=15.2.15-11-g5f8f263c-1focal ceph-mds=15.2.15-11-g5f8f263c-1focal ceph-mgr=15.2.15-11-g5f8f263c-1focal ceph-common=15.2.15-11-g5f8f263c-1focal ceph-fuse=15.2.15-11-g5f8f263c-1focal ceph-test=15.2.15-11-g5f8f263c-1focal ceph-volume=15.2.15-11-g5f8f263c-1focal radosgw=15.2.15-11-g5f8f263c-1focal python3-rados=15.2.15-11-g5f8f263c-1focal python3-rgw=15.2.15-11-g5f8f263c-1focal python3-cephfs=15.2.15-11-g5f8f263c-1focal python3-rbd=15.2.15-11-g5f8f263c-1focal libcephfs2=15.2.15-11-g5f8f263c-1focal librados2=15.2.15-11-g5f8f263c-1focal librbd1=15.2.15-11-g5f8f263c-1focal rbd-fuse=15.2.15-11-g5f8f263c-1focal' ``` Signed-off-by: Neha Ojha --- .../stress-split-no-cephadm/1-ceph-install/octopus.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/qa/suites/upgrade/octopus-x/stress-split-no-cephadm/1-ceph-install/octopus.yaml b/qa/suites/upgrade/octopus-x/stress-split-no-cephadm/1-ceph-install/octopus.yaml index 3d7a52a499f12..012482fa6c6df 100644 --- a/qa/suites/upgrade/octopus-x/stress-split-no-cephadm/1-ceph-install/octopus.yaml +++ b/qa/suites/upgrade/octopus-x/stress-split-no-cephadm/1-ceph-install/octopus.yaml @@ -6,6 +6,7 @@ tasks: - ceph-mgr-cephadm - cephadm - libcephfs-dev + - ceph-volume branch: octopus - print: "**** done install octopus" - ceph: From 7ffcbd7f7955b443ffd4293a497b2a99180a3ad2 Mon Sep 17 00:00:00 2001 From: Paul Cuzner Date: Wed, 3 Nov 2021 15:24:20 +1300 Subject: [PATCH 0530/2866] mgr/prometheus: Update rule format and enhance SNMP support Rules now adhere to the format defined by Prometheus.io. This changes alert naming and each alert now includes a a summary description to provide a quick one-liner. In addition to reformatting some missing alerts for MDS and cephadm have been added, and corresponding tests added. The MIB has also been refactored, so it now passes standard lint tests and a README included for devs to understand the OID schema. Fixes: https://tracker.ceph.com/issues/53111 Signed-off-by: Paul Cuzner --- .../prometheus/alerts/ceph_default_alerts.yml | 334 +++++-- monitoring/prometheus/tests/test_alerts.yml | 899 +++++++++++------- monitoring/prometheus/tests/validate_rules.py | 152 ++- monitoring/snmp/CEPH-MIB.txt | 337 +++++++ monitoring/snmp/README.md | 66 +- 5 files changed, 1290 insertions(+), 498 deletions(-) create mode 100644 monitoring/snmp/CEPH-MIB.txt diff --git a/monitoring/prometheus/alerts/ceph_default_alerts.yml b/monitoring/prometheus/alerts/ceph_default_alerts.yml index 4b9ea51ebe1d8..d9e6e35637f92 100644 --- a/monitoring/prometheus/alerts/ceph_default_alerts.yml +++ b/monitoring/prometheus/alerts/ceph_default_alerts.yml @@ -1,41 +1,43 @@ groups: - name: cluster health rules: - - alert: health error + - alert: CephHealthError expr: ceph_health_status == 2 for: 5m labels: severity: critical type: ceph_default - oid: 1.3.6.1.4.1.50495.15.1.2.2.1 + oid: 1.3.6.1.4.1.50495.1.2.1.2.1 annotations: + summary: Cluster is in an ERROR state description: > Ceph in HEALTH_ERROR state for more than 5 minutes. Please check "ceph health detail" for more information. - - alert: health warn + - alert: CephHealthWarning expr: ceph_health_status == 1 for: 15m labels: severity: warning type: ceph_default - oid: 1.3.6.1.4.1.50495.15.1.2.2.2 annotations: + summary: Cluster is in a WARNING state description: > Ceph has been in HEALTH_WARN for more than 15 minutes. Please check "ceph health detail" for more information. - name: mon rules: - - alert: Monitor down, quorum is at risk + - alert: CephMonDownQuorumAtRisk expr: ((ceph_health_detail{name="MON_DOWN"} == 1) * on() (count(ceph_mon_quorum_status == 1) == bool (floor(count(ceph_mon_metadata) / 2) + 1))) == 1 for: 30s labels: severity: critical type: ceph_default - oid: 1.3.6.1.4.1.50495.15.1.2.3.1 + oid: 1.3.6.1.4.1.50495.1.2.1.3.1 annotations: documentation: https://docs.ceph.com/en/latest/rados/operations/health-checks#mon-down + summary: Monitor quorum is at risk description: | {{ $min := query "floor(count(ceph_mon_metadata) / 2) +1" | first | value }}Quorum requires a majority of monitors (x {{ $min }}) to be active Without quorum the cluster will become inoperable, affecting all connected clients and services. @@ -44,7 +46,7 @@ groups: {{- range query "(ceph_mon_quorum_status == 0) + on(ceph_daemon) group_left(hostname) (ceph_mon_metadata * 0)" }} - {{ .Labels.ceph_daemon }} on {{ .Labels.hostname }} {{- end }} - - alert: Monitor down + - alert: CephMonDown expr: (count(ceph_mon_quorum_status == 0) <= (count(ceph_mon_metadata) - floor(count(ceph_mon_metadata) / 2) + 1)) for: 30s labels: @@ -52,6 +54,7 @@ groups: type: ceph_default annotations: documentation: https://docs.ceph.com/en/latest/rados/operations/health-checks#mon-down + summary: One of more ceph monitors are down description: | {{ $down := query "count(ceph_mon_quorum_status == 0)" | first | value }}{{ $s := "" }}{{ if gt $down 1.0 }}{{ $s = "s" }}{{ end }}You have {{ $down }} monitor{{ $s }} down. Quorum is still intact, but the loss of further monitors will make your cluster inoperable. @@ -60,14 +63,16 @@ groups: {{- range query "(ceph_mon_quorum_status == 0) + on(ceph_daemon) group_left(hostname) (ceph_mon_metadata * 0)" }} - {{ .Labels.ceph_daemon }} on {{ .Labels.hostname }} {{- end }} - - alert: Ceph mon disk space critically low + - alert: CephMonDiskspaceCritical expr: ceph_health_detail{name="MON_DISK_CRIT"} == 1 for: 1m labels: severity: critical type: ceph_default + oid: 1.3.6.1.4.1.50495.1.2.1.3.2 annotations: documentation: https://docs.ceph.com/en/latest/rados/operations/health-checks#mon-disk-crit + summary: Disk space on at least one monitor is critically low description: | The free space available to a monitor's store is critically low (<5% by default). You should increase the space available to the monitor(s). The @@ -76,7 +81,7 @@ groups: - {{ .Labels.hostname }} {{- end }} - - alert: Ceph mon disk space running low + - alert: CephMonDiskspaceLow expr: ceph_health_detail{name="MON_DISK_LOW"} == 1 for: 5m labels: @@ -84,6 +89,7 @@ groups: type: ceph_default annotations: documentation: https://docs.ceph.com/en/latest/rados/operations/health-checks#mon-disk-low + summary: Disk space on at least one monitor is approaching full description: | The space available to a monitor's store is approaching full (>70% is the default). You should increase the space available to the monitor store. The @@ -92,7 +98,7 @@ groups: - {{ .Labels.hostname }} {{- end }} - - alert: Clock skew detected across Ceph Monitor daemons + - alert: CephMonClockSkew expr: ceph_health_detail{name="MON_CLOCK_SKEW"} == 1 for: 1m labels: @@ -100,6 +106,7 @@ groups: type: ceph_default annotations: documentation: https://docs.ceph.com/en/latest/rados/operations/health-checks#mon-clock-skew + summary: Clock skew across the Monitor hosts detected description: | The ceph monitors rely on a consistent time reference to maintain quorum and cluster consistency. This event indicates that at least @@ -110,41 +117,45 @@ groups: - name: osd rules: - - alert: 10% OSDs down + - alert: CephOSDDownHigh expr: count(ceph_osd_up == 0) / count(ceph_osd_up) * 100 >= 10 labels: severity: critical type: ceph_default - oid: 1.3.6.1.4.1.50495.15.1.2.4.1 + oid: 1.3.6.1.4.1.50495.1.2.1.4.1 annotations: + summary: More than 10% of OSDs are down description: | - {{ $value | humanize }}% or {{ with query "count(ceph_osd_up == 0)" }}{{ . | first | value }}{{ end }} of {{ with query "count(ceph_osd_up)" }}{{ . | first | value }}{{ end }} OSDs are down (≥ 10%). + {{ $value | humanize }}% or {{ with query "count(ceph_osd_up == 0)" }}{{ . | first | value }}{{ end }} of {{ with query "count(ceph_osd_up)" }}{{ . | first | value }}{{ end }} OSDs are down (>= 10%). The following OSDs are down: {{- range query "(ceph_osd_up * on(ceph_daemon) group_left(hostname) ceph_osd_metadata) == 0" }} - {{ .Labels.ceph_daemon }} on {{ .Labels.hostname }} {{- end }} - - alert: OSD Host is down + - alert: CephOSDHostDown expr: ceph_health_detail{name="OSD_HOST_DOWN"} == 1 for: 5m labels: severity: warning type: ceph_default + oid: 1.3.6.1.4.1.50495.1.2.1.4.8 annotations: + summary: An OSD host is offline description: | The following OSDs are down: {{- range query "(ceph_osd_up * on(ceph_daemon) group_left(hostname) ceph_osd_metadata) == 0" }} - {{ .Labels.hostname }} : {{ .Labels.ceph_daemon }} {{- end }} - - alert: OSD down + - alert: CephOSDDown expr: ceph_health_detail{name="OSD_DOWN"} == 1 for: 5m labels: severity: warning type: ceph_default - oid: 1.3.6.1.4.1.50495.15.1.2.4.2 + oid: 1.3.6.1.4.1.50495.1.2.1.4.2 annotations: documentation: https://docs.ceph.com/en/latest/rados/operations/health-checks#osd-down + summary: An OSD has been marked down/unavailable description: | {{ $num := query "count(ceph_osd_up == 0)" | first | value }}{{ $s := "" }}{{ if gt $num 1.0 }}{{ $s = "s" }}{{ end }}{{ $num }} OSD{{ $s }} down for over 5mins. @@ -153,34 +164,37 @@ groups: - {{ .Labels.ceph_daemon }} on {{ .Labels.hostname }} {{- end }} - - alert: OSDs near full + - alert: CephOSDNearFull expr: ceph_health_detail{name="OSD_NEARFULL"} == 1 for: 5m labels: severity: warning type: ceph_default - oid: 1.3.6.1.4.1.50495.15.1.2.4.3 + oid: 1.3.6.1.4.1.50495.1.2.1.4.3 annotations: documentation: https://docs.ceph.com/en/latest/rados/operations/health-checks#osd-nearfull + summary: OSD(s) running low on free space (NEARFULL) description: | One or more OSDs have reached their NEARFULL threshold Use 'ceph health detail' to identify which OSDs have reached this threshold. To resolve, either add capacity to the cluster, or delete unwanted data - - alert: OSD Full + - alert: CephOSDFull expr: ceph_health_detail{name="OSD_FULL"} > 0 for: 1m labels: severity: critical type: ceph_default + oid: 1.3.6.1.4.1.50495.1.2.1.4.6 annotations: documentation: https://docs.ceph.com/en/latest/rados/operations/health-checks#osd-full + summary: OSD(s) is full, writes blocked description: | An OSD has reached it's full threshold. Writes from all pools that share the affected OSD will be blocked. To resolve, either add capacity to the cluster, or delete unwanted data - - alert: OSD unable to perform rebalance + - alert: CephOSDBackfillFull expr: ceph_health_detail{name="OSD_BACKFILLFULL"} > 0 for: 1m labels: @@ -188,12 +202,13 @@ groups: type: ceph_default annotations: documentation: https://docs.ceph.com/en/latest/rados/operations/health-checks#osd-backfillfull + summary: OSD(s) too full for backfill operations description: | An OSD has reached it's BACKFILL FULL threshold. This will prevent rebalance operations completing for some pools. Check the current capacity utilisation with 'ceph df' To resolve, either add capacity to the cluster, or delete unwanted data - - alert: OSD too many read repairs + - alert: CephOSDTooManyRepairs expr: ceph_health_detail{name="OSD_TOO_MANY_REPAIRS"} == 1 for: 30s labels: @@ -201,30 +216,33 @@ groups: type: ceph_default annotations: documentation: https://docs.ceph.com/en/latest/rados/operations/health-checks#osd-too-many-repairs + summary: OSD has hit a high number of read errors description: | Reads from an OSD have used a secondary PG to return data to the client, indicating a potential failing disk. - - alert: OSD hearbeats running slow (frontend) + - alert: CephOSDTimeoutsPublicNetwork expr: ceph_health_detail{name="OSD_SLOW_PING_TIME_FRONT"} == 1 for: 1m labels: severity: warning type: ceph_default annotations: + summary: Network issues delaying OSD heartbeats (public network) description: | OSD heartbeats on the cluster's 'public' network (frontend) are running slow. Investigate the network for any latency issues on this subnet. Use 'ceph health detail' to show the affected OSDs. - - alert: OSD hearbeats running slow (backend) + - alert: CephOSDTimeoutsClusterNetwork expr: ceph_health_detail{name="OSD_SLOW_PING_TIME_BACK"} == 1 for: 1m labels: severity: warning type: ceph_default annotations: + summary: Network issues delaying OSD heartbeats (cluster network) description: | OSD heartbeats on the cluster's 'cluster' network (backend) are running slow. Investigate the network for any latency issues on this subnet. Use 'ceph health detail' to show the affected OSDs. - - alert: OSD disk size mismatch + - alert: CephOSDInternalDiskSizeMismatch expr: ceph_health_detail{name="BLUESTORE_DISK_SIZE_MISMATCH"} == 1 for: 1m labels: @@ -232,10 +250,11 @@ groups: type: ceph_default annotations: documentation: https://docs.ceph.com/en/latest/rados/operations/health-checks#bluestore-disk-size-mismatch + summary: OSD size inconsistency error description: | One or more OSDs have an internal inconsistency between the size of the physical device and it's metadata. This could lead to the OSD(s) crashing in future. You should redeploy the effected OSDs. - - alert: Device failure predicted + - alert: CephDeviceFailurePredicted expr: ceph_health_detail{name="DEVICE_HEALTH"} == 1 for: 1m labels: @@ -243,6 +262,7 @@ groups: type: ceph_default annotations: documentation: https://docs.ceph.com/en/latest/rados/operations/health-checks#id2 + summary: Device(s) have been predicted to fail soon description: | The device health module has determined that one or more devices will fail soon. To review the device states use 'ceph device ls'. To show a specific @@ -250,20 +270,22 @@ groups: Mark the OSD as out (so data may migrate to other OSDs in the cluster). Once the osd is empty remove and replace the OSD. - - alert: Too many devices predicted to fail + - alert: CephDeviceFailurePredictionTooHigh expr: ceph_health_detail{name="DEVICE_HEALTH_TOOMANY"} == 1 for: 1m labels: severity: critical type: ceph_default + oid: 1.3.6.1.4.1.50495.1.2.1.4.7 annotations: documentation: https://docs.ceph.com/en/latest/rados/operations/health-checks#device-health-toomany + summary: Too many devices have been predicted to fail, unable to resolve description: | The device health module has determined that the number of devices predicted to fail can not be remediated automatically, since it would take too many osd's out of the cluster, impacting performance and potentially availabililty. You should add new OSDs to the cluster to allow data to be relocated to avoid the data integrity issues. - - alert: Device failure predicted, but automatic drain is incomplete + - alert: CephDeviceFailureRelocationIncomplete expr: ceph_health_detail{name="DEVICE_HEALTH_IN_USE"} == 1 for: 1m labels: @@ -271,6 +293,7 @@ groups: type: ceph_default annotations: documentation: https://docs.ceph.com/en/latest/rados/operations/health-checks#device-health-in-use + summary: A device failure is predicted, but unable to relocate data description: | The device health module has determined that one or more devices will fail soon, but the normal process of relocating the data on the device to other @@ -280,7 +303,7 @@ groups: more disks to the cluster to allow the data from the failing device to successfully migrate. - - alert: Flapping OSD + - alert: CephOSDFlapping expr: | ( rate(ceph_osd_up[5m]) @@ -289,9 +312,10 @@ groups: labels: severity: warning type: ceph_default - oid: 1.3.6.1.4.1.50495.15.1.2.4.4 + oid: 1.3.6.1.4.1.50495.1.2.1.4.4 annotations: documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds + summary: Network issues are causing OSD's to flap (mark each other out) description: > OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked down and back up at {{ $value | humanize }} times once a @@ -299,7 +323,7 @@ groups: packet drop, disruption) on the clusters "cluster network". Check the network environment on the listed host(s). - - alert: OSD Read errors + - alert: CephOSDReadErrors expr: ceph_health_detail{name="BLUESTORE_SPURIOUS_READ_ERRORS"} == 1 for: 30s labels: @@ -307,11 +331,12 @@ groups: type: ceph_default annotations: documentation: https://docs.ceph.com/en/latest/rados/operations/health-checks#bluestore-spurious-read-errors + summary: Device read errors detected description: > An OSD has encountered read errors, but the OSD has recovered by retrying the reads. This may indicate an issue with the Hardware or Kernel. # alert on high deviation from average PG count - - alert: high pg count deviation + - alert: CephPGImbalance expr: | abs( ( @@ -322,8 +347,9 @@ groups: labels: severity: warning type: ceph_default - oid: 1.3.6.1.4.1.50495.15.1.2.4.5 + oid: 1.3.6.1.4.1.50495.1.2.1.4.5 annotations: + summary: PG allocations are not balanced across devices description: > OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} deviates by more than 30% from average PG count. @@ -331,28 +357,99 @@ groups: - name: mds rules: - - alert: Ceph Filesystem damage detected + - alert: CephFilesystemDamaged expr: ceph_health_detail{name="MDS_DAMAGE"} > 0 for: 1m labels: severity: critical type: ceph_default + oid: 1.3.6.1.4.1.50495.1.2.1.5.1 annotations: documentation: https://docs.ceph.com/en/latest/cephfs/health-messages#cephfs-health-messages + summary: Ceph filesystem is damaged. description: > The filesystems metadata has been corrupted. Data access may be blocked. Either analyse the output from the mds daemon admin socket, or escalate to support - - alert: Ceph Filesystem switched to READ ONLY + - alert: CephFilesystemOffline + expr: ceph_health_detail{name="MDS_ALL_DOWN"} > 0 + for: 1m + labels: + severity: critical + type: ceph_default + oid: 1.3.6.1.4.1.50495.1.2.1.5.3 + annotations: + documentation: https://docs.ceph.com/en/latest/cephfs/health-messages/#mds-all-down + summary: Ceph filesystem is offline + description: > + All MDS ranks are unavailable. The ceph daemons providing the metadata + for the Ceph filesystem are all down, rendering the filesystem offline. + - alert: CephFilesystemDegraded + expr: ceph_health_detail{name="FS_DEGRADED"} > 0 + for: 1m + labels: + severity: critical + type: ceph_default + oid: 1.3.6.1.4.1.50495.1.2.1.5.4 + annotations: + documentation: https://docs.ceph.com/en/latest/cephfs/health-messages/#fs-degraded + summary: Ceph filesystem is degraded + description: > + One or more metdata daemons (MDS ranks) are failed or in a + damaged state. At best the filesystem is partially available, + worst case is the filesystem is completely unusable. + - alert: CephFilesystemMDSRanksLow + expr: ceph_health_detail{name="MDS_UP_LESS_THAN_MAX"} > 0 + for: 1m + labels: + severity: warning + type: ceph_default + annotations: + documentation: https://docs.ceph.com/en/latest/cephfs/health-messages/#mds-up-less-than-max + summary: Ceph MDS daemon count is lower than configured + description: > + The filesystem's "max_mds" setting defined the number of MDS ranks in + the filesystem. The current number of active MDS daemons is less than + this setting. + - alert: CephFilesystemInsufficientStandby + expr: ceph_health_detail{name="MDS_INSUFFICIENT_STANDBY"} > 0 + for: 1m + labels: + severity: warning + type: ceph_default + annotations: + documentation: https://docs.ceph.com/en/latest/cephfs/health-messages/#mds-insufficient-standby + summary: Ceph filesystem standby daemons too low + description: > + The minimum number of standby daemons determined by standby_count_wanted + is less than the actual number of standby daemons. Adjust the standby count + or increase the number of mds daemons within the filesystem. + - alert: CephFilesystemFailureNoStandby + expr: ceph_health_detail{name="FS_WITH_FAILED_MDS"} > 0 + for: 1m + labels: + severity: critical + type: ceph_default + oid: 1.3.6.1.4.1.50495.1.2.1.5.5 + annotations: + documentation: https://docs.ceph.com/en/latest/cephfs/health-messages/#fs-with-failed-mds + summary: Ceph MDS daemon failed, no further standby available + description: > + An MDS daemon has failed, leaving only one active rank without + further standby. Investigate the cause of the failure or add a + standby daemon + - alert: CephFilesystemReadOnly expr: ceph_health_detail{name="MDS_HEALTH_READ_ONLY"} > 0 for: 1m labels: severity: critical type: ceph_default + oid: 1.3.6.1.4.1.50495.1.2.1.5.2 annotations: documentation: https://docs.ceph.com/en/latest/cephfs/health-messages#cephfs-health-messages + summary: Ceph filesystem in read only mode, due to write error(s) description: > The filesystem has switched to READ ONLY due to an unexpected write error, when writing to the metadata pool @@ -362,25 +459,29 @@ groups: - name: mgr rules: - - alert: mgr module failure + - alert: CephMgrModuleCrash expr: ceph_health_detail{name="RECENT_MGR_MODULE_CRASH"} == 1 for: 5m labels: severity: critical type: ceph_default + oid: 1.3.6.1.4.1.50495.1.2.1.6.1 annotations: documentation: https://docs.ceph.com/en/latest/rados/operations/health-checks#recent-mgr-module-crash + summary: A mgr module has recently crashed description: > One or more mgr modules have crashed and are yet to be acknowledged by the administrator. A crashed module may impact functionality within the cluster. Use the 'ceph crash' commands to investigate which module has failed, and archive it to acknowledge the failure. - - alert: mgr prometheus module is not active + - alert: CephMgrPrometheusModuleInactive expr: up{job="ceph"} == 0 for: 1m labels: severity: critical type: ceph_default + oid: 1.3.6.1.4.1.50495.1.2.1.6.2 annotations: + summary: Ceph's mgr/prometheus module is not available description: > The mgr/prometheus module at {{ $labels.instance }} is unreachable. This could mean that the module has been disabled or the mgr itself is down. @@ -393,37 +494,41 @@ groups: - name: pgs rules: - - alert: pgs inactive + - alert: CephPGsInactive expr: ceph_pool_metadata * on(pool_id,instance) group_left() (ceph_pg_total - ceph_pg_active) > 0 for: 5m labels: severity: critical type: ceph_default - oid: 1.3.6.1.4.1.50495.15.1.2.7.1 + oid: 1.3.6.1.4.1.50495.1.2.1.7.1 annotations: + summary: One or more Placement Groups are inactive description: > {{ $value }} PGs have been inactive for more than 5 minutes in pool {{ $labels.name }}. Inactive placement groups aren't able to serve read/write requests. - - alert: pgs unclean + - alert: CephPGsUnclean expr: ceph_pool_metadata * on(pool_id,instance) group_left() (ceph_pg_total - ceph_pg_clean) > 0 for: 15m labels: severity: warning type: ceph_default - oid: 1.3.6.1.4.1.50495.15.1.2.7.2 + oid: 1.3.6.1.4.1.50495.1.2.1.7.2 annotations: + summary: One or more platcment groups are marked unclean description: > {{ $value }} PGs haven't been clean for more than 15 minutes in pool {{ $labels.name }}. Unclean PGs haven't been able to completely recover from a previous failure. - - alert: Placement Group (PG) damaged + - alert: CephPGsDamaged expr: ceph_health_detail{name=~"PG_DAMAGED|OSD_SCRUB_ERRORS"} == 1 for: 5m labels: severity: critical type: ceph_default + oid: 1.3.6.1.4.1.50495.1.2.1.7.4 annotations: documentation: https://docs.ceph.com/en/latest/rados/operations/health-checks#pg-damaged + summary: Placement group damaged, manual intervention needed description: > During data consistency checks (scrub), at least one PG has been flagged as being damaged or inconsistent. @@ -431,41 +536,47 @@ groups: Check to see which PG is affected, and attempt a manual repair if neccessary. To list problematic placement groups, use 'rados list-inconsistent-pg '. To repair PGs use the 'ceph pg repair ' command. - - alert: Recovery at risk, cluster too full + - alert: CephPGRecoveryAtRisk expr: ceph_health_detail{name="PG_RECOVERY_FULL"} == 1 for: 1m labels: severity: critical type: ceph_default + oid: 1.3.6.1.4.1.50495.1.2.1.7.5 annotations: documentation: https://docs.ceph.com/en/latest/rados/operations/health-checks#pg-recovery-full + summary: OSDs are too full for automatic recovery description: > Data redundancy may be reduced, or is at risk, since one or more OSDs are at or above their 'full' threshold. Add more capacity to the cluster, or delete unwanted data. - - alert: I/O blocked to some data + - alert: CephPGUnavilableBlockingIO # PG_AVAILABILITY, but an OSD is not in a DOWN state expr: ((ceph_health_detail{name="PG_AVAILABILITY"} == 1) - scalar(ceph_health_detail{name="OSD_DOWN"})) == 1 for: 1m labels: severity: critical type: ceph_default + oid: 1.3.6.1.4.1.50495.1.2.1.7.3 annotations: documentation: https://docs.ceph.com/en/latest/rados/operations/health-checks#pg-availability + summary: Placement group is unavailable, blocking some I/O description: > Data availability is reduced impacting the clusters abilty to service I/O to some data. One or more placement groups (PGs) are in a state that blocks IO. - - alert: Cluster too full, automatic data recovery impaired + - alert: CephPGBackfillAtRisk expr: ceph_health_detail{name="PG_BACKFILL_FULL"} == 1 for: 1m labels: severity: critical type: ceph_default + oid: 1.3.6.1.4.1.50495.1.2.1.7.6 annotations: documentation: https://docs.ceph.com/en/latest/rados/operations/health-checks#pg-backfill-full + summary: Backfill operations are blocked, due to lack of freespace description: > Data redundancy may be at risk due to lack of free space within the cluster. One or more OSDs have breached their 'backfillfull' threshold. Add more capacity, or delete unwanted data. - - alert: Placement Group(s) have not been scrubbed + - alert: CephPGNotScrubbed expr: ceph_health_detail{name="PG_NOT_SCRUBBED"} == 1 for: 5m labels: @@ -473,6 +584,7 @@ groups: type: ceph_default annotations: documentation: https://docs.ceph.com/en/latest/rados/operations/health-checks#pg-not-scrubbed + summary: Placement group(s) have not been scrubbed description: | One or more PGs have not been scrubbed recently. The scrub process is a data integrity feature, protectng against bit-rot. It checks that objects and their metadata (size and @@ -481,7 +593,23 @@ groups: scrub window. You can manually initiate a scrub with: ceph pg scrub - - alert: Placement Group(s) have not been 'DEEP' scrubbed + - alert: CephPGsHighPerOSD + expr: ceph_health_detail{name="TOO_MANY_PGS"} == 1 + for: 1m + labels: + severity: warning + type: ceph_default + annotations: + documentation: https://docs.ceph.com/en/latest/rados/operations/health-checks/#too-many-pgs + summary: Placement groups per OSD is too high + description: | + The number of placement groups per OSD is too high (exceeds the mon_max_pg_per_osd setting). + + Check that the pg_autoscaler hasn't been disabled for any of the pools, with 'ceph osd pool autoscale-status' + and that the profile selected is appropriate. You may also adjust the target_size_ratio of a pool to guide + the autoscaler based on the expected relative size of the pool + (i.e. 'ceph osd pool set cephfs.cephfs.meta target_size_ratio .1') + - alert: CephPGNotDeepScrubbed expr: ceph_health_detail{name="PG_NOT_DEEP_SCRUBBED"} == 1 for: 5m labels: @@ -489,6 +617,7 @@ groups: type: ceph_default annotations: documentation: https://docs.ceph.com/en/latest/rados/operations/health-checks#pg-not-deep-scrubbed + summary: Placement group(s) have not been deep scrubbed description: | One or more PGs have not been deep scrubbed recently. Deep scrub is a data integrity feature, protectng against bit-rot. It compares the contents of objects and their @@ -500,19 +629,20 @@ groups: - name: nodes rules: - - alert: root volume full + - alert: CephNodeRootFilesystemFull expr: node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"} * 100 < 5 for: 5m labels: severity: critical type: ceph_default - oid: 1.3.6.1.4.1.50495.15.1.2.8.1 + oid: 1.3.6.1.4.1.50495.1.2.1.8.1 annotations: + summary: Root filesystem is dangerously full description: > Root volume (OSD and MON store) is dangerously full: {{ $value | humanize }}% free. # alert on nic packet errors and drops rates > 1% packets/s - - alert: network packets dropped + - alert: CephNodeNetworkPacketDrops expr: | ( increase(node_network_receive_drop_total{device!="lo"}[1m]) + @@ -527,13 +657,14 @@ groups: labels: severity: warning type: ceph_default - oid: 1.3.6.1.4.1.50495.15.1.2.8.2 + oid: 1.3.6.1.4.1.50495.1.2.1.8.2 annotations: + summary: One or more Nics is seeing packet drops description: > Node {{ $labels.instance }} experiences packet drop > 0.01% or > 10 packets/s on interface {{ $labels.device }}. - - alert: network packet errors + - alert: CephNodeNetworkPacketErrors expr: | ( increase(node_network_receive_errs_total{device!="lo"}[1m]) + @@ -548,102 +679,96 @@ groups: labels: severity: warning type: ceph_default - oid: 1.3.6.1.4.1.50495.15.1.2.8.3 + oid: 1.3.6.1.4.1.50495.1.2.1.8.3 annotations: + summary: One or more Nics is seeing packet errors description: > Node {{ $labels.instance }} experiences packet errors > 0.01% or > 10 packets/s on interface {{ $labels.device }}. # Restrict to device names beginning with '/' to skip false alarms from # tmpfs, overlay type filesystems - - alert: storage filling up + - alert: CephNodeDiskspaceWarning expr: | predict_linear(node_filesystem_free_bytes{device=~"/.*"}[2d], 3600 * 24 * 5) * on(instance) group_left(nodename) node_uname_info < 0 labels: severity: warning type: ceph_default - oid: 1.3.6.1.4.1.50495.15.1.2.8.4 + oid: 1.3.6.1.4.1.50495.1.2.1.8.4 annotations: + summary: Host filesystem freespace is getting low description: > Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will be full in less than 5 days assuming the average fill-up rate of the past 48 hours. - - alert: MTU Mismatch + - alert: CephNodeInconsistentMTU expr: node_network_mtu_bytes{device!="lo"} * (node_network_up{device!="lo"} > 0) != on() group_left() (quantile(0.5, node_network_mtu_bytes{device!="lo"})) labels: severity: warning type: ceph_default - oid: 1.3.6.1.4.1.50495.15.1.2.8.5 annotations: + summary: MTU settings across Ceph hosts are inconsistent description: > Node {{ $labels.instance }} has a different MTU size ({{ $value }}) than the median value on device {{ $labels.device }}. - name: pools rules: - - alert: pool full + - alert: CephPoolGrowthWarning expr: | - ceph_pool_stored / (ceph_pool_stored + ceph_pool_max_avail) - * on(pool_id) group_right ceph_pool_metadata * 100 > 90 - labels: - severity: critical - type: ceph_default - oid: 1.3.6.1.4.1.50495.15.1.2.9.1 - annotations: - description: Pool {{ $labels.name }} at {{ $value | humanize }}% capacity. - - - alert: pool filling up (growth forecast) - expr: | - ( - predict_linear(ceph_pool_stored[2d], 3600 * 24 * 5) - >= ceph_pool_stored + ceph_pool_max_avail - ) * on(pool_id) group_left(name) ceph_pool_metadata + (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) + group_right ceph_pool_metadata) >= 95 labels: severity: warning type: ceph_default - oid: 1.3.6.1.4.1.50495.15.1.2.9.2 + oid: 1.3.6.1.4.1.50495.1.2.1.9.2 annotations: + summary: Pool growth rate may soon exceed it's capacity description: > - Pool {{ $labels.name }} will be full in less than 5 days + Pool '{{ $labels.name }}' will be full in less than 5 days assuming the average fill-up rate of the past 48 hours. - - - alert: Ceph pool is too full for recovery/rebalance + - alert: CephPoolBackfillFull expr: ceph_health_detail{name="POOL_BACKFILLFULL"} > 0 labels: severity: warning type: ceph_default annotations: + summary: Freespace in a pool is too low for recovery/rebalance description: > A pool is approaching it's near full threshold, which will prevent rebalance operations from completing. You should consider adding more capacity to the pool. - - alert: Ceph pool is full - writes blocked + - alert: CephPoolFull expr: ceph_health_detail{name="POOL_FULL"} > 0 for: 1m labels: severity: critical type: ceph_default + oid: 1.3.6.1.4.1.50495.1.2.1.9.1 annotations: documentation: https://docs.ceph.com/en/latest/rados/operations/health-checks#pool-full + summary: Pool is full - writes are blocked description: | A pool has reached it's MAX quota, or the OSDs supporting the pool have reached their FULL threshold. Until this is resolved, writes to the pool will be blocked. - - Determine the affected pool with 'ceph df detail', for example looking - at QUOTA BYTES and STORED. Either increase the pools quota, or add - capacity to the cluster first then increase it's quota - (e.g. ceph osd pool set quota max_bytes ) - - alert: Ceph pool is approaching full + Pool Breakdown (top 5) + {{- range query "topk(5, sort_desc(ceph_pool_percent_used * on(pool_id) group_right ceph_pool_metadata))" }} + - {{ .Labels.name }} at {{ .Value }}% + {{- end }} + Either increase the pools quota, or add capacity to the cluster first + then increase it's quota (e.g. ceph osd pool set quota max_bytes ) + - alert: CephPoolNearFull expr: ceph_health_detail{name="POOL_NEAR_FULL"} > 0 for: 5m labels: severity: warning type: ceph_default annotations: + summary: One or more Ceph pools are getting full description: | A pool has exceeeded it warning (percent full) threshold, or the OSDs supporting the pool have reached their NEARFULL thresholds. Writes may @@ -656,7 +781,7 @@ groups: (e.g. ceph osd pool set quota max_bytes ) - name: healthchecks rules: - - alert: Slow OSD Ops + - alert: CephSlowOps expr: ceph_healthcheck_slow_ops > 0 for: 30s labels: @@ -664,35 +789,40 @@ groups: type: ceph_default annotations: documentation: https://docs.ceph.com/en/latest/rados/operations/health-checks#slow-ops + summary: MON/OSD operations are slow to complete description: > {{ $value }} OSD requests are taking too long to process (osd_op_complaint_time exceeded) # cephadm alerts - name: cephadm rules: - - alert: Cluster upgrade has failed + - alert: CephadmUpgradeFailed expr: ceph_health_detail{name="UPGRADE_EXCEPTION"} > 0 for: 30s labels: severity: critical type: ceph_default + oid: 1.3.6.1.4.1.50495.1.2.1.11.2 annotations: + summary: Ceph version upgrade has failed description: > The cephadm cluster upgrade process has failed. The cluster remains in an undetermined state. Please review the cephadm logs, to understand the nature of the issue - - alert: A daemon managed by cephadm is down + - alert: CephadmDaemonFailed expr: ceph_health_detail{name="CEPHADM_FAILED_DAEMON"} > 0 for: 30s labels: severity: critical type: ceph_default + oid: 1.3.6.1.4.1.50495.1.2.1.11.1 annotations: + summary: A ceph daemon manged by cephadm is down description: > A daemon managed by cephadm is no longer active. Determine, which daemon is down with 'ceph health detail'. you may start daemons with the 'ceph orch daemon start ' - - alert: cephadm management has been paused + - alert: CephadmPaused expr: ceph_health_detail{name="CEPHADM_PAUSED"} > 0 for: 1m labels: @@ -700,21 +830,24 @@ groups: type: ceph_default annotations: documentation: https://docs.ceph.com/en/latest/cephadm/operations#cephadm-paused + summary: Orchestration tasks via cephadm are PAUSED description: > Cluster management has been paused manually. This will prevent the orchestrator from service management and reconciliation. If this is not intentional, resume cephadm operations with 'ceph orch resume' # prometheus alerts - - name: prometheus + - name: PrometheusServer rules: - - alert: Scrape job is missing + - alert: PrometheusJobMissing expr: absent(up{job="ceph"}) for: 30s labels: severity: critical type: ceph_default + oid: 1.3.6.1.4.1.50495.1.2.1.12.1 annotations: + summary: The scrape job for Ceph is missing from Prometheus description: | The prometheus job that scrapes from Ceph is no longer defined, this will effectively mean you'll have no metrics or alerts for the cluster. @@ -724,15 +857,34 @@ groups: # Object related events - name: rados rules: - - alert: Data not found/missing + - alert: CephObjectMissing expr: (ceph_health_detail{name="OBJECT_UNFOUND"} == 1) * on() (count(ceph_osd_up == 1) == bool count(ceph_osd_metadata)) == 1 for: 30s labels: severity: critical type: ceph_default + oid: 1.3.6.1.4.1.50495.1.2.1.10.1 annotations: documentation: https://docs.ceph.com/en/latest/rados/operations/health-checks#object-unfound + summary: Object(s) has been marked UNFOUND description: | A version of a RADOS object can not be found, even though all OSDs are up. I/O requests for this object from clients will block (hang). Resolving this issue may - require the object to be rolled back to a prior version manually, and manually verified. \ No newline at end of file + require the object to be rolled back to a prior version manually, and manually verified. +# Generic + - name: generic + rules: + - alert: CephDaemonCrash + expr: ceph_health_detail{name="RECENT_CRASH"} == 1 + for: 1m + labels: + severity: critical + type: ceph_default + oid: 1.3.6.1.4.1.50495.1.2.1.1.2 + annotations: + documentation: https://docs.ceph.com/en/latest/rados/operations/health-checks/#recent-crash + summary: One or more Ceph daemons have crashed, and are pending acknowledgement + description: | + One or more daemons have crashed recently, and need to be acknowledged. This notification + ensures that software crashes don't go unseen. To acknowledge a crash, use the + 'ceph crash archive ' command. \ No newline at end of file diff --git a/monitoring/prometheus/tests/test_alerts.yml b/monitoring/prometheus/tests/test_alerts.yml index ab423c983d0c7..cd980deb39b3c 100644 --- a/monitoring/prometheus/tests/test_alerts.yml +++ b/monitoring/prometheus/tests/test_alerts.yml @@ -15,17 +15,18 @@ tests: value: 2 alert_rule_test: - eval_time: 1m - alertname: health error + alertname: CephHealthError - eval_time: 6m - alertname: health error + alertname: CephHealthError exp_alerts: - exp_labels: instance: ceph:9283 job: ceph - oid: 1.3.6.1.4.1.50495.15.1.2.2.1 + oid: 1.3.6.1.4.1.50495.1.2.1.2.1 type: ceph_default severity: critical exp_annotations: + summary: Cluster is in an ERROR state description: > Ceph in HEALTH_ERROR state for more than 5 minutes. Please check "ceph health detail" for more information. @@ -43,72 +44,21 @@ tests: value: 1 alert_rule_test: - eval_time: 10m - alertname: health warn + alertname: CephHealthWarning - eval_time: 20m - alertname: health warn + alertname: CephHealthWarning exp_alerts: - exp_labels: instance: ceph:9283 job: ceph - oid: 1.3.6.1.4.1.50495.15.1.2.2.2 type: ceph_default severity: warning exp_annotations: + summary: Cluster is in a WARNING state description: > Ceph has been in HEALTH_WARN for more than 15 minutes. Please check "ceph health detail" for more information. - # low monitor quorum count -# - interval: 1m -# input_series: -# - series: 'ceph_mon_quorum_status{ceph_daemon="mon.a",instance="ceph:9283", -# job="ceph"}' -# values: '1 1 1 1 1' -# - series: 'ceph_mon_quorum_status{ceph_daemon="mon.b",instance="ceph:9283", -# job="ceph"}' -# values: '1 1 1 1 1' -# - series: 'ceph_mon_quorum_status{ceph_daemon="mon.c",instance="ceph:9283", -# job="ceph"}' -# values: '0 0 0 0 0' -# - series: 'ceph_mon_metadata{ceph_daemon="mon.a",ceph_version="ceph version -# 17.0.0-189-g3558fd72 (3558fd7291855971aa6481a2ade468ad61fbb346) pacific -# (dev)",hostname="ceph",instance="ceph:9283",job="ceph", -# public_addr="172.20.0.2",rank="0"}' -# values: '1 1 1 1 1' -# - series: 'ceph_mon_metadata{ceph_daemon="mon.b",ceph_version="ceph version -# 17.0.0-189-g3558fd72 (3558fd7291855971aa6481a2ade468ad61fbb346) pacific -# (dev)",hostname="ceph",instance="ceph:9283",job="ceph", -# public_addr="172.20.0.2",rank="1"}' -# values: '1 1 1 1 1' -# - series: 'ceph_mon_metadata{ceph_daemon="mon.c",ceph_version="ceph version -# 17.0.0-189-g3558fd72 (3558fd7291855971aa6481a2ade468ad61fbb346) pacific -# (dev)",hostname="ceph",instance="ceph:9283",job="ceph", -# public_addr="172.20.0.2",rank="2"}' -# values: '1 1 1 1 1' -# promql_expr_test: -# - expr: sum(ceph_mon_quorum_status) < 3 -# eval_time: 1m -# exp_samples: -# - labels: '{}' -# value: 2 -# alert_rule_test: -# - eval_time: 1m -# alertname: low monitor quorum count -# exp_alerts: -# - exp_labels: -# oid: 1.3.6.1.4.1.50495.15.1.2.3.1 -# type: ceph_default -# severity: critical -# exp_annotations: -# description: | -# Monitor count in quorum is below three. - -# Only 2 of 3 monitors are active. - -# The following monitors are down: -# - mon.c on ceph - - # 10% OSDs down - interval: 1m input_series: @@ -147,156 +97,20 @@ tests: value: 3.333333333333333E+01 alert_rule_test: - eval_time: 1m - alertname: 10% OSDs down + alertname: CephOSDDownHigh exp_alerts: - exp_labels: - oid: 1.3.6.1.4.1.50495.15.1.2.4.1 + oid: 1.3.6.1.4.1.50495.1.2.1.4.1 type: ceph_default severity: critical exp_annotations: + summary: More than 10% of OSDs are down description: | - 33.33% or 1 of 3 OSDs are down (≥ 10%). + 33.33% or 1 of 3 OSDs are down (>= 10%). The following OSDs are down: - osd.1 on ceph - # OSD down -# - interval: 1m -# input_series: -# - series: 'ceph_osd_up{ceph_daemon="osd.0",instance="ceph:9283",job="ceph"}' -# values: '1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1' -# - series: 'ceph_osd_up{ceph_daemon="osd.1",instance="ceph:9283",job="ceph"}' -# values: '0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0' -# - series: 'ceph_osd_up{ceph_daemon="osd.2",instance="ceph:9283",job="ceph"}' -# values: '1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1' -# - series: 'ceph_osd_metadata{back_iface="eth0",ceph_daemon="osd.0", -# ceph_version="ceph version 17.0.0-189-g3558fd72 -# (3558fd7291855971aa6481a2ade468ad61fbb346) pacific (dev)", -# cluster_addr="172.20.0.2",device_class="hdd",front_iface="eth0", -# hostname="ceph",instance="ceph:9283",job="ceph",objectstore="bluestore", -# public_addr="172.20.0.2"}' -# values: '1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1' -# - series: 'ceph_osd_metadata{back_iface="eth0",ceph_daemon="osd.1", -# ceph_version="ceph version 17.0.0-189-g3558fd72 -# (3558fd7291855971aa6481a2ade468ad61fbb346) pacific (dev)", -# cluster_addr="172.20.0.2",device_class="hdd",front_iface="eth0", -# hostname="ceph",instance="ceph:9283",job="ceph",objectstore="bluestore", -# public_addr="172.20.0.2"}' -# values: '1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1' -# - series: 'ceph_osd_metadata{back_iface="eth0",ceph_daemon="osd.2", -# ceph_version="ceph version 17.0.0-189-g3558fd72 -# (3558fd7291855971aa6481a2ade468ad61fbb346) pacific (dev)", -# cluster_addr="172.20.0.2",device_class="hdd",front_iface="eth0", -# hostname="ceph",instance="ceph:9283",job="ceph",objectstore="bluestore", -# public_addr="172.20.0.2"}' -# values: '1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1' -# promql_expr_test: -# - expr: count(ceph_osd_up == 0) > 0 -# eval_time: 1m -# exp_samples: -# - labels: '{}' -# value: 1 -# alert_rule_test: -# - eval_time: 15m -# alertname: OSD down -# exp_alerts: -# - exp_labels: -# oid: 1.3.6.1.4.1.50495.15.1.2.4.2 -# type: ceph_default -# severity: warning -# exp_annotations: -# description: | - -# 1 OSD down for more than 15 minutes. - -# 1 of 3 OSDs are down. - -# The following OSD is down: -# - osd.1 on ceph - -# OSDs near full -# - interval: 1m -# input_series: -# - series: 'ceph_osd_stat_bytes_used{ceph_daemon="osd.0",instance="ceph:9283" -# ,job="ceph"}' -# values: '1076310016 1076310016 1076310016 1076310016 1076310016 -# 1076310016' -# - series: 'ceph_osd_stat_bytes_used{ceph_daemon="osd.1",instance="ceph:9283" -# ,job="ceph"}' -# values: '1076310016 1076310016 1076310016 1076310016 1076310016 -# 1076310016' -# - series: 'ceph_osd_stat_bytes_used{ceph_daemon="osd.2",instance="ceph:9283" -# ,job="ceph"}' -# values: '1076310016 1076310016 1076310016 1076310016 1076310016 -# 100856561909.76' -# - series: 'ceph_osd_stat_bytes{ceph_daemon="osd.0",instance="ceph:9283" -# ,job="ceph"}' -# values: '108447916032 108447916032 108447916032 108447916032 108447916032 -# 108447916032' -# - series: 'ceph_osd_stat_bytes{ceph_daemon="osd.1",instance="ceph:9283" -# ,job="ceph"}' -# values: '108447916032 108447916032 108447916032 108447916032 108447916032 -# 108447916032' -# - series: 'ceph_osd_stat_bytes{ceph_daemon="osd.2",instance="ceph:9283" -# ,job="ceph"}' -# values: '108447916032 108447916032 108447916032 108447916032 108447916032 -# 108447916032' -# - series: 'ceph_osd_up{ceph_daemon="osd.0",instance="ceph:9283",job="ceph"}' -# values: '1 1 1 1 1 1' -# - series: 'ceph_osd_up{ceph_daemon="osd.1",instance="ceph:9283",job="ceph"}' -# values: '1 1 1 1 1 1' -# - series: 'ceph_osd_up{ceph_daemon="osd.2",instance="ceph:9283",job="ceph"}' -# values: '1 1 1 1 1 1' -# - series: 'ceph_osd_metadata{back_iface="eth0",ceph_daemon="osd.0", -# ceph_version="ceph version 17.0.0-189-g3558fd72 -# (3558fd7291855971aa6481a2ade468ad61fbb346) pacific (dev)", -# cluster_addr="172.20.0.2",device_class="hdd",front_iface="eth0", -# hostname="ceph",instance="ceph:9283",job="ceph",objectstore="bluestore", -# public_addr="172.20.0.2"}' -# values: '1 1 1 1 1 1' -# - series: 'ceph_osd_metadata{back_iface="eth0",ceph_daemon="osd.1", -# ceph_version="ceph version 17.0.0-189-g3558fd72 -# (3558fd7291855971aa6481a2ade468ad61fbb346) pacific (dev)", -# cluster_addr="172.20.0.2",device_class="hdd",front_iface="eth0", -# hostname="ceph",instance="ceph:9283",job="ceph",objectstore="bluestore", -# public_addr="172.20.0.2"}' -# values: '1 1 1 1 1 1' -# - series: 'ceph_osd_metadata{back_iface="eth0",ceph_daemon="osd.2", -# ceph_version="ceph version 17.0.0-189-g3558fd72 -# (3558fd7291855971aa6481a2ade468ad61fbb346) pacific (dev)", -# cluster_addr="172.20.0.2",device_class="hdd",front_iface="eth0", -# hostname="ceph",instance="ceph:9283",job="ceph",objectstore="bluestore", -# public_addr="172.20.0.2"}' -# values: '1 1 1 1 1 1' -# promql_expr_test: -# - expr: | -# ( -# ((ceph_osd_stat_bytes_used / ceph_osd_stat_bytes) and on(ceph_daemon) -# ceph_osd_up == 1) * on(ceph_daemon) group_left(hostname) -# ceph_osd_metadata -# ) * 100 > 90 - -# eval_time: 5m -# exp_samples: -# - labels: '{ceph_daemon="osd.2",hostname="ceph",instance="ceph:9283", -# job="ceph"}' -# value: 9.3E+01 -# alert_rule_test: -# - eval_time: 10m -# alertname: OSDs near full -# exp_alerts: -# - exp_labels: -# ceph_daemon: osd.2 -# hostname: ceph -# instance: ceph:9283 -# job: ceph -# oid: 1.3.6.1.4.1.50495.15.1.2.4.3 -# type: ceph_default -# severity: critical -# exp_annotations: -# description: > -# OSD osd.2 on ceph is dangerously full: 93% - # flapping OSD - interval: 1s input_series: @@ -340,18 +154,19 @@ tests: value: 1.2200000000000001E+01 alert_rule_test: - eval_time: 5m - alertname: Flapping OSD + alertname: CephOSDFlapping exp_alerts: - exp_labels: ceph_daemon: osd.0 hostname: ceph instance: ceph:9283 job: ceph - oid: 1.3.6.1.4.1.50495.15.1.2.4.4 + oid: 1.3.6.1.4.1.50495.1.2.1.4.4 severity: warning type: ceph_default exp_annotations: documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds + summary: Network issues are causing OSD's to flap (mark each other out) description: > OSD osd.0 on ceph was marked down and back up at 20.1 times once a minute for 5 minutes. @@ -418,17 +233,18 @@ tests: value: 6E-01 alert_rule_test: - eval_time: 10m - alertname: high pg count deviation + alertname: CephPGImbalance exp_alerts: - exp_labels: ceph_daemon: osd.1 hostname: ceph instance: ceph:9283 job: ceph - oid: 1.3.6.1.4.1.50495.15.1.2.4.5 + oid: 1.3.6.1.4.1.50495.1.2.1.4.5 severity: warning type: ceph_default exp_annotations: + summary: PG allocations are not balanced across devices description: > OSD osd.1 on ceph deviates by more than 30% from average PG count. @@ -468,17 +284,18 @@ tests: value: 1 alert_rule_test: - eval_time: 5m - alertname: pgs inactive + alertname: CephPGsInactive exp_alerts: - exp_labels: instance: ceph:9283 job: ceph name: device_health_metrics - oid: 1.3.6.1.4.1.50495.15.1.2.7.1 + oid: 1.3.6.1.4.1.50495.1.2.1.7.1 pool_id: 3 severity: critical type: ceph_default exp_annotations: + summary: One or more Placement Groups are inactive description: > 1 PGs have been inactive for more than 5 minutes in pool device_health_metrics. @@ -523,17 +340,18 @@ tests: value: 1 alert_rule_test: - eval_time: 16m - alertname: pgs unclean + alertname: CephPGsUnclean exp_alerts: - exp_labels: instance: ceph:9283 job: ceph name: device_health_metrics - oid: 1.3.6.1.4.1.50495.15.1.2.7.2 + oid: 1.3.6.1.4.1.50495.1.2.1.7.2 pool_id: 3 severity: warning type: ceph_default exp_annotations: + summary: One or more platcment groups are marked unclean description: > 1 PGs haven't been clean for more than 15 minutes in pool device_health_metrics. @@ -564,7 +382,7 @@ tests: value: 4.8E+00 alert_rule_test: - eval_time: 10m - alertname: root volume full + alertname: CephNodeRootFilesystemFull exp_alerts: - exp_labels: device: /dev/mapper/fedora_localhost --live-home @@ -572,12 +390,13 @@ tests: instance: node-exporter job: node-exporter mountpoint: / - oid: 1.3.6.1.4.1.50495.15.1.2.8.1 + oid: 1.3.6.1.4.1.50495.1.2.1.8.1 severity: critical type: ceph_default exp_annotations: + summary: Root filesystem is dangerously full description: > - Root volume (OSD and MON store) is dangerously full: 4.811% free. + Root volume (OSD and MON store) is dangerously full: 4.811% free. # network packets dropped - interval: 1s @@ -608,19 +427,20 @@ tests: value: 1.2E+02 alert_rule_test: - eval_time: 5m - alertname: network packets dropped + alertname: CephNodeNetworkPacketDrops exp_alerts: - exp_labels: device: eth0 instance: node-exporter job: node-exporter - oid: 1.3.6.1.4.1.50495.15.1.2.8.2 + oid: 1.3.6.1.4.1.50495.1.2.1.8.2 severity: warning type: ceph_default exp_annotations: + summary: One or more Nics is seeing packet drops description: > - Node node-exporter experiences packet drop > 0.01% or > - 10 packets/s on interface eth0. + Node node-exporter experiences packet drop > 0.01% or > + 10 packets/s on interface eth0. # network packets errors - interval: 1s @@ -651,20 +471,63 @@ tests: value: 1.2E+02 alert_rule_test: - eval_time: 5m - alertname: network packet errors + alertname: CephNodeNetworkPacketErrors exp_alerts: - exp_labels: device: eth0 instance: node-exporter job: node-exporter - oid: 1.3.6.1.4.1.50495.15.1.2.8.3 + oid: 1.3.6.1.4.1.50495.1.2.1.8.3 severity: warning type: ceph_default exp_annotations: + summary: One or more Nics is seeing packet errors description: > - Node node-exporter experiences packet errors > 0.01% or > 10 - packets/s on interface eth0. + Node node-exporter experiences packet errors > 0.01% or > 10 + packets/s on interface eth0. +# Node Storage disk space filling up + - interval: 1m + # 20GB = 21474836480, 256MB = 268435456 + input_series: + - series: 'node_filesystem_free_bytes{device="/dev/mapper/vg-root", + fstype="xfs",instance="node-1",mountpoint="/rootfs"}' + values: '21474836480-268435456x48' + - series: 'node_filesystem_free_bytes{device="/dev/mapper/vg-root", + fstype="xfs",instance="node-2",mountpoint="/rootfs"}' + values: '21474836480+0x48' + - series: 'node_uname_info{instance="node-1", nodename="node-1.unittests.com"}' + values: 1+0x48 + - series: 'node_uname_info{instance="node-2", nodename="node-2.unittests.com"}' + values: 1+0x48 + promql_expr_test: + - expr: | + predict_linear(node_filesystem_free_bytes{device=~"/.*"}[2d], 3600 * 24 * 5) * + on(instance) group_left(nodename) node_uname_info < 0 + eval_time: 5m + exp_samples: + - labels: '{device="/dev/mapper/vg-root",instance="node-1",fstype="xfs", + mountpoint="/rootfs",nodename="node-1.unittests.com"}' + value: -1.912602624E+12 + alert_rule_test: + - eval_time: 5m + alertname: CephNodeDiskspaceWarning + exp_alerts: + - exp_labels: + severity: warning + type: ceph_default + oid: 1.3.6.1.4.1.50495.1.2.1.8.4 + device: /dev/mapper/vg-root + fstype: xfs + instance: node-1 + mountpoint: /rootfs + nodename: node-1.unittests.com + exp_annotations: + summary: Host filesystem freespace is getting low + description: > + Mountpoint /rootfs on node-1.unittests.com + will be full in less than 5 days assuming the average fill-up + rate of the past 48 hours. # MTU Mismatch - interval: 1m input_series: @@ -707,72 +570,88 @@ tests: value: 9000 alert_rule_test: - eval_time: 1m - alertname: MTU Mismatch + alertname: CephNodeInconsistentMTU exp_alerts: - exp_labels: device: eth4 instance: node-exporter job: node-exporter - oid: 1.3.6.1.4.1.50495.15.1.2.8.5 severity: warning type: ceph_default exp_annotations: + summary: MTU settings across Ceph hosts are inconsistent description: > Node node-exporter has a different MTU size (9000) than the median value on device eth4. - # pool full + # pool full, data series has 6 but using topk(5) so to ensure the + # results are working as expected - interval: 1m input_series: - - series: 'ceph_pool_stored{instance="ceph:9283",job="ceph",pool_id="1"}' - values: '0 0 0 0 0 0 0 0 0' - - series: 'ceph_pool_stored{instance="ceph:9283",job="ceph",pool_id="2"}' - values: '1850 1850 1850 1850 1850 1850 1850' - - series: 'ceph_pool_stored{instance="ceph:9283",job="ceph",pool_id="3"}' - values: '900 900 23524 23524 23524 23524 23524 23524 - 23524' - - series: 'ceph_pool_max_avail{instance="ceph:9283",job="ceph",pool_id="1"}' - values: '106287063040 106287063040 106287063040 106287063040 106287063040 - 106287063040 106287063040' - - series: 'ceph_pool_max_avail{instance="ceph:9283",job="ceph",pool_id="2"}' - values: '106287063040 106287063040 106287063040 106287063040 106287063040 - 106287063040 106287063040' - - series: 'ceph_pool_max_avail{instance="ceph:9283",job="ceph",pool_id="3"}' - values: '37.5 37.5 37.5 37.5 37.5 37.5 37.5' + - series: 'ceph_health_detail{name="POOL_FULL"}' + values: '0 0 0 1 1 1 1 1 1 1 1' + - series: 'ceph_pool_percent_used{pool_id="1"}' + values: '32+0x10' + - series: 'ceph_pool_percent_used{pool_id="2"}' + values: '96+0x10' + - series: 'ceph_pool_percent_used{pool_id="3"}' + values: '90+0x10' + - series: 'ceph_pool_percent_used{pool_id="4"}' + values: '72+0x10' + - series: 'ceph_pool_percent_used{pool_id="5"}' + values: '19+0x10' + - series: 'ceph_pool_percent_used{pool_id="6"}' + values: '10+0x10' - series: 'ceph_pool_metadata{instance="ceph:9283",job="ceph", - name="device_health_metrics",pool_id="1"}' + name="cephfs_data",pool_id="1"}' + values: '1 1 1 1 1 1 1 1 1' + - series: 'ceph_pool_metadata{instance="ceph:9283",job="ceph", + name="rbd",pool_id="2"}' values: '1 1 1 1 1 1 1 1 1' - series: 'ceph_pool_metadata{instance="ceph:9283",job="ceph", - name=".rgw.root",pool_id="2"}' + name="iscsi",pool_id="3"}' values: '1 1 1 1 1 1 1 1 1' - series: 'ceph_pool_metadata{instance="ceph:9283",job="ceph", - name="default.rgw.log",pool_id="3"}' + name="default.rgw.index",pool_id="4"}' + values: '1 1 1 1 1 1 1 1 1' + - series: 'ceph_pool_metadata{instance="ceph:9283",job="ceph", + name="default.rgw.log",pool_id="5"}' + values: '1 1 1 1 1 1 1 1 1' + - series: 'ceph_pool_metadata{instance="ceph:9283",job="ceph", + name="dummy",pool_id="6"}' values: '1 1 1 1 1 1 1 1 1' promql_expr_test: - - expr: | - ceph_pool_stored / (ceph_pool_stored + ceph_pool_max_avail) - * on(pool_id) group_right ceph_pool_metadata * 100 > 90 - - eval_time: 1m + - expr: ceph_health_detail{name="POOL_FULL"} > 0 + eval_time: 5m exp_samples: - - labels: '{instance="ceph:9283", job="ceph", name="default.rgw.log", - pool_id="3"}' - value: 9.6E+01 + - labels: '{__name__="ceph_health_detail", name="POOL_FULL"}' + value: 1 alert_rule_test: - - eval_time: 2m - alertname: pool full + - eval_time: 1m + alertname: CephPoolFull + - eval_time: 10m + alertname: CephPoolFull exp_alerts: - exp_labels: - instance: ceph:9283 - job: ceph - name: default.rgw.log - oid: 1.3.6.1.4.1.50495.15.1.2.9.1 - pool_id: 3 + name: POOL_FULL severity: critical type: ceph_default + oid: 1.3.6.1.4.1.50495.1.2.1.9.1 exp_annotations: - description: Pool default.rgw.log at 96% capacity. - + documentation: https://docs.ceph.com/en/latest/rados/operations/health-checks#pool-full + summary: Pool is full - writes are blocked + description: | + A pool has reached it's MAX quota, or the OSDs supporting the pool + have reached their FULL threshold. Until this is resolved, writes to + the pool will be blocked. + Pool Breakdown (top 5) + - rbd at 96% + - iscsi at 90% + - default.rgw.index at 72% + - cephfs_data at 32% + - default.rgw.log at 19% + Either increase the pools quota, or add capacity to the cluster first + then increase it's quota (e.g. ceph osd pool set quota max_bytes ) # slow OSD ops - interval : 1m input_series: @@ -787,7 +666,7 @@ tests: value: 1 alert_rule_test: - eval_time: 20m - alertname: Slow OSD Ops + alertname: CephSlowOps exp_alerts: - exp_labels: instance: ceph:9283 @@ -796,6 +675,7 @@ tests: type: ceph_default exp_annotations: documentation: https://docs.ceph.com/en/latest/rados/operations/health-checks#slow-ops + summary: MON/OSD operations are slow to complete description: > 1 OSD requests are taking too long to process (osd_op_complaint_time exceeded) @@ -813,15 +693,17 @@ tests: value: 1 alert_rule_test: - eval_time: 1m - alertname: Cluster upgrade has failed + alertname: CephadmUpgradeFailed - eval_time: 5m - alertname: Cluster upgrade has failed + alertname: CephadmUpgradeFailed exp_alerts: - exp_labels: name: UPGRADE_EXCEPTION severity: critical type: ceph_default + oid: 1.3.6.1.4.1.50495.1.2.1.11.2 exp_annotations: + summary: Ceph version upgrade has failed description: > The cephadm cluster upgrade process has failed. The cluster remains in an undetermined state. @@ -839,15 +721,17 @@ tests: value: 1 alert_rule_test: - eval_time: 1m - alertname: A daemon managed by cephadm is down + alertname: CephadmDaemonFailed - eval_time: 5m - alertname: A daemon managed by cephadm is down + alertname: CephadmDaemonFailed exp_alerts: - exp_labels: name: CEPHADM_FAILED_DAEMON severity: critical type: ceph_default + oid: 1.3.6.1.4.1.50495.1.2.1.11.1 exp_annotations: + summary: A ceph daemon manged by cephadm is down description: > A daemon managed by cephadm is no longer active. Determine, which daemon is down with 'ceph health detail'. you may start daemons with @@ -864,9 +748,9 @@ tests: value: 1 alert_rule_test: - eval_time: 1m - alertname: cephadm management has been paused + alertname: CephadmPaused - eval_time: 5m - alertname: cephadm management has been paused + alertname: CephadmPaused exp_alerts: - exp_labels: name: CEPHADM_PAUSED @@ -874,6 +758,7 @@ tests: type: ceph_default exp_annotations: documentation: https://docs.ceph.com/en/latest/cephadm/operations#cephadm-paused + summary: Orchestration tasks via cephadm are PAUSED description: > Cluster management has been paused manually. This will prevent the orchestrator from service management and reconciliation. If this is @@ -891,16 +776,18 @@ tests: value: 1 alert_rule_test: - eval_time: 1m - alertname: Ceph Filesystem damage detected + alertname: CephFilesystemDamaged - eval_time: 5m - alertname: Ceph Filesystem damage detected + alertname: CephFilesystemDamaged exp_alerts: - exp_labels: name: MDS_DAMAGE severity: critical type: ceph_default + oid: 1.3.6.1.4.1.50495.1.2.1.5.1 exp_annotations: documentation: https://docs.ceph.com/en/latest/cephfs/health-messages#cephfs-health-messages + summary: Ceph filesystem is damaged. description: > The filesystems metadata has been corrupted. Data access may be blocked. @@ -919,22 +806,161 @@ tests: value: 1 alert_rule_test: - eval_time: 1m - alertname: Ceph Filesystem switched to READ ONLY + alertname: CephFilesystemReadOnly - eval_time: 5m - alertname: Ceph Filesystem switched to READ ONLY + alertname: CephFilesystemReadOnly exp_alerts: - exp_labels: name: MDS_HEALTH_READ_ONLY severity: critical type: ceph_default + oid: 1.3.6.1.4.1.50495.1.2.1.5.2 exp_annotations: documentation: https://docs.ceph.com/en/latest/cephfs/health-messages#cephfs-health-messages + summary: Ceph filesystem in read only mode, due to write error(s) description: > The filesystem has switched to READ ONLY due to an unexpected write error, when writing to the metadata pool Either analyse the output from the mds daemon admin socket, or escalate to support + - interval: 1m + input_series: + - series: 'ceph_health_detail{name="MDS_ALL_DOWN"}' + values: '0 0 1 1 1 1 1 1 1 1 1' + promql_expr_test: + - expr: ceph_health_detail{name="MDS_ALL_DOWN"} > 0 + eval_time: 2m + exp_samples: + - labels: '{__name__="ceph_health_detail", name="MDS_ALL_DOWN"}' + value: 1 + alert_rule_test: + - eval_time: 1m + alertname: CephFilesystemOffline + - eval_time: 10m + alertname: CephFilesystemOffline + exp_alerts: + - exp_labels: + name: MDS_ALL_DOWN + severity: critical + type: ceph_default + oid: 1.3.6.1.4.1.50495.1.2.1.5.3 + exp_annotations: + documentation: https://docs.ceph.com/en/latest/cephfs/health-messages/#mds-all-down + summary: Ceph filesystem is offline + description: > + All MDS ranks are unavailable. The ceph daemons providing the metadata + for the Ceph filesystem are all down, rendering the filesystem offline. + - interval: 1m + input_series: + - series: 'ceph_health_detail{name="FS_DEGRADED"}' + values: '0 0 1 1 1 1 1 1 1 1 1' + promql_expr_test: + - expr: ceph_health_detail{name="FS_DEGRADED"} > 0 + eval_time: 2m + exp_samples: + - labels: '{__name__="ceph_health_detail", name="FS_DEGRADED"}' + value: 1 + alert_rule_test: + - eval_time: 1m + alertname: CephFilesystemDegraded + - eval_time: 10m + alertname: CephFilesystemDegraded + exp_alerts: + - exp_labels: + name: FS_DEGRADED + severity: critical + type: ceph_default + oid: 1.3.6.1.4.1.50495.1.2.1.5.4 + exp_annotations: + documentation: https://docs.ceph.com/en/latest/cephfs/health-messages/#fs-degraded + summary: Ceph filesystem is degraded + description: > + One or more metdata daemons (MDS ranks) are failed or in a + damaged state. At best the filesystem is partially available, + worst case is the filesystem is completely unusable. + - interval: 1m + input_series: + - series: 'ceph_health_detail{name="MDS_INSUFFICIENT_STANDBY"}' + values: '0 0 1 1 1 1 1 1 1 1 1' + promql_expr_test: + - expr: ceph_health_detail{name="MDS_INSUFFICIENT_STANDBY"} > 0 + eval_time: 2m + exp_samples: + - labels: '{__name__="ceph_health_detail", name="MDS_INSUFFICIENT_STANDBY"}' + value: 1 + alert_rule_test: + - eval_time: 1m + alertname: CephFilesystemInsufficientStandby + - eval_time: 10m + alertname: CephFilesystemInsufficientStandby + exp_alerts: + - exp_labels: + name: MDS_INSUFFICIENT_STANDBY + severity: warning + type: ceph_default + exp_annotations: + documentation: https://docs.ceph.com/en/latest/cephfs/health-messages/#mds-insufficient-standby + summary: Ceph filesystem standby daemons too low + description: > + The minimum number of standby daemons determined by standby_count_wanted + is less than the actual number of standby daemons. Adjust the standby count + or increase the number of mds daemons within the filesystem. + - interval: 1m + input_series: + - series: 'ceph_health_detail{name="FS_WITH_FAILED_MDS"}' + values: '0 0 1 1 1 1 1 1 1 1 1' + promql_expr_test: + - expr: ceph_health_detail{name="FS_WITH_FAILED_MDS"} > 0 + eval_time: 2m + exp_samples: + - labels: '{__name__="ceph_health_detail", name="FS_WITH_FAILED_MDS"}' + value: 1 + alert_rule_test: + - eval_time: 1m + alertname: CephFilesystemFailureNoStandby + - eval_time: 10m + alertname: CephFilesystemFailureNoStandby + exp_alerts: + - exp_labels: + name: FS_WITH_FAILED_MDS + severity: critical + type: ceph_default + oid: 1.3.6.1.4.1.50495.1.2.1.5.5 + exp_annotations: + documentation: https://docs.ceph.com/en/latest/cephfs/health-messages/#fs-with-failed-mds + summary: Ceph MDS daemon failed, no further standby available + description: > + An MDS daemon has failed, leaving only one active rank without + further standby. Investigate the cause of the failure or add a + standby daemon + - interval: 1m + input_series: + - series: 'ceph_health_detail{name="MDS_UP_LESS_THAN_MAX"}' + values: '0 0 1 1 1 1 1 1 1 1 1' + promql_expr_test: + - expr: ceph_health_detail{name="MDS_UP_LESS_THAN_MAX"} > 0 + eval_time: 2m + exp_samples: + - labels: '{__name__="ceph_health_detail", name="MDS_UP_LESS_THAN_MAX"}' + value: 1 + alert_rule_test: + - eval_time: 1m + alertname: CephFilesystemMDSRanksLow + - eval_time: 10m + alertname: CephFilesystemMDSRanksLow + exp_alerts: + - exp_labels: + name: MDS_UP_LESS_THAN_MAX + severity: warning + type: ceph_default + exp_annotations: + documentation: https://docs.ceph.com/en/latest/cephfs/health-messages/#mds-up-less-than-max + summary: Ceph MDS daemon count is lower than configured + description: > + The filesystem's "max_mds" setting defined the number of MDS ranks in + the filesystem. The current number of active MDS daemons is less than + this setting. # MGR - interval: 1m input_series: @@ -948,16 +974,18 @@ tests: value: 0 alert_rule_test: - eval_time: 1m - alertname: mgr prometheus module is not active + alertname: CephMgrPrometheusModuleInactive - eval_time: 10m - alertname: mgr prometheus module is not active + alertname: CephMgrPrometheusModuleInactive exp_alerts: - exp_labels: instance: ceph-mgr:9283 job: ceph severity: critical type: ceph_default + oid: 1.3.6.1.4.1.50495.1.2.1.6.2 exp_annotations: + summary: Ceph's mgr/prometheus module is not available description: > The mgr/prometheus module at ceph-mgr:9283 is unreachable. This could mean that the module has been disabled or the mgr itself is down. @@ -979,16 +1007,18 @@ tests: value: 1 alert_rule_test: - eval_time: 1m - alertname: mgr module failure + alertname: CephMgrModuleCrash - eval_time: 15m - alertname: mgr module failure + alertname: CephMgrModuleCrash exp_alerts: - exp_labels: name: RECENT_MGR_MODULE_CRASH severity: critical type: ceph_default + oid: 1.3.6.1.4.1.50495.1.2.1.6.1 exp_annotations: documentation: https://docs.ceph.com/en/latest/rados/operations/health-checks#recent-mgr-module-crash + summary: A mgr module has recently crashed description: > One or more mgr modules have crashed and are yet to be acknowledged by the administrator. A crashed module may impact functionality within the cluster. Use the 'ceph crash' commands to @@ -1008,16 +1038,18 @@ tests: value: 1 alert_rule_test: - eval_time: 1m - alertname: Ceph mon disk space critically low + alertname: CephMonDiskspaceCritical - eval_time: 10m - alertname: Ceph mon disk space critically low + alertname: CephMonDiskspaceCritical exp_alerts: - exp_labels: name: "MON_DISK_CRIT" severity: critical type: ceph_default + oid: 1.3.6.1.4.1.50495.1.2.1.3.2 exp_annotations: documentation: https://docs.ceph.com/en/latest/rados/operations/health-checks#mon-disk-crit + summary: Disk space on at least one monitor is critically low description: | The free space available to a monitor's store is critically low (<5% by default). You should increase the space available to the monitor(s). The @@ -1037,9 +1069,9 @@ tests: value: 1 alert_rule_test: - eval_time: 1m - alertname: Ceph mon disk space running low + alertname: CephMonDiskspaceLow - eval_time: 10m - alertname: Ceph mon disk space running low + alertname: CephMonDiskspaceLow exp_alerts: - exp_labels: name: "MON_DISK_LOW" @@ -1047,6 +1079,7 @@ tests: type: ceph_default exp_annotations: documentation: https://docs.ceph.com/en/latest/rados/operations/health-checks#mon-disk-low + summary: Disk space on at least one monitor is approaching full description: | The space available to a monitor's store is approaching full (>70% is the default). You should increase the space available to the monitor store. The @@ -1064,9 +1097,9 @@ tests: value: 1 alert_rule_test: - eval_time: 1m - alertname: Clock skew detected across Ceph Monitor daemons + alertname: CephMonClockSkew - eval_time: 10m - alertname: Clock skew detected across Ceph Monitor daemons + alertname: CephMonClockSkew exp_alerts: - exp_labels: name: "MON_CLOCK_SKEW" @@ -1074,6 +1107,7 @@ tests: type: ceph_default exp_annotations: documentation: https://docs.ceph.com/en/latest/rados/operations/health-checks#mon-clock-skew + summary: Clock skew across the Monitor hosts detected description: | The ceph monitors rely on a consistent time reference to maintain quorum and cluster consistency. This event indicates that at least @@ -1107,17 +1141,18 @@ tests: value: 1 alert_rule_test: - eval_time: 1m - alertname: Monitor down, quorum is at risk + alertname: CephMonDownQuorumAtRisk # shouldn't fire - eval_time: 10m - alertname: Monitor down, quorum is at risk + alertname: CephMonDownQuorumAtRisk exp_alerts: - exp_labels: severity: critical type: ceph_default - oid: 1.3.6.1.4.1.50495.15.1.2.3.1 + oid: 1.3.6.1.4.1.50495.1.2.1.3.1 exp_annotations: documentation: https://docs.ceph.com/en/latest/rados/operations/health-checks#mon-down + summary: Monitor quorum is at risk description: | Quorum requires a majority of monitors (x 2) to be active Without quorum the cluster will become inoperable, affecting all connected clients and services. @@ -1155,15 +1190,16 @@ tests: value: 1 alert_rule_test: - eval_time: 1m - alertname: Monitor down + alertname: CephMonDown - eval_time: 10m - alertname: Monitor down + alertname: CephMonDown exp_alerts: - exp_labels: severity: warning type: ceph_default exp_annotations: documentation: https://docs.ceph.com/en/latest/rados/operations/health-checks#mon-down + summary: One of more ceph monitors are down description: | You have 1 monitor down. Quorum is still intact, but the loss of further monitors will make your cluster inoperable. @@ -1183,9 +1219,9 @@ tests: value: 1 alert_rule_test: - eval_time: 1m - alertname: Device failure predicted + alertname: CephDeviceFailurePredicted - eval_time: 10m - alertname: Device failure predicted + alertname: CephDeviceFailurePredicted exp_alerts: - exp_labels: name: "DEVICE_HEALTH" @@ -1193,6 +1229,7 @@ tests: type: ceph_default exp_annotations: documentation: https://docs.ceph.com/en/latest/rados/operations/health-checks#id2 + summary: Device(s) have been predicted to fail soon description: | The device health module has determined that one or more devices will fail soon. To review the device states use 'ceph device ls'. To show a specific @@ -1212,16 +1249,18 @@ tests: value: 1 alert_rule_test: - eval_time: 1m - alertname: Too many devices predicted to fail + alertname: CephDeviceFailurePredictionTooHigh - eval_time: 10m - alertname: Too many devices predicted to fail + alertname: CephDeviceFailurePredictionTooHigh exp_alerts: - exp_labels: name: "DEVICE_HEALTH_TOOMANY" severity: critical type: ceph_default + oid: 1.3.6.1.4.1.50495.1.2.1.4.7 exp_annotations: documentation: https://docs.ceph.com/en/latest/rados/operations/health-checks#device-health-toomany + summary: Too many devices have been predicted to fail, unable to resolve description: | The device health module has determined that the number of devices predicted to fail can not be remediated automatically, since it would take too many osd's out of @@ -1239,9 +1278,9 @@ tests: value: 1 alert_rule_test: - eval_time: 1m - alertname: Device failure predicted, but automatic drain is incomplete + alertname: CephDeviceFailureRelocationIncomplete - eval_time: 10m - alertname: Device failure predicted, but automatic drain is incomplete + alertname: CephDeviceFailureRelocationIncomplete exp_alerts: - exp_labels: name: "DEVICE_HEALTH_IN_USE" @@ -1249,6 +1288,7 @@ tests: type: ceph_default exp_annotations: documentation: https://docs.ceph.com/en/latest/rados/operations/health-checks#device-health-in-use + summary: A device failure is predicted, but unable to relocate data description: | The device health module has determined that one or more devices will fail soon, but the normal process of relocating the data on the device to other @@ -1274,15 +1314,17 @@ tests: value: 1 alert_rule_test: - eval_time: 1m - alertname: OSD Host is down + alertname: CephOSDHostDown - eval_time: 10m - alertname: OSD Host is down + alertname: CephOSDHostDown exp_alerts: - exp_labels: name: "OSD_HOST_DOWN" severity: warning type: ceph_default + oid: 1.3.6.1.4.1.50495.1.2.1.4.8 exp_annotations: + summary: An OSD host is offline description: | The following OSDs are down: - ceph-osd-1 : osd.0 @@ -1298,15 +1340,16 @@ tests: value: 0 alert_rule_test: - eval_time: 1m - alertname: OSD hearbeats running slow (frontend) + alertname: CephOSDTimeoutsPublicNetwork - eval_time: 10m - alertname: OSD hearbeats running slow (frontend) + alertname: CephOSDTimeoutsPublicNetwork exp_alerts: - exp_labels: name: "OSD_SLOW_PING_TIME_FRONT" severity: warning type: ceph_default exp_annotations: + summary: Network issues delaying OSD heartbeats (public network) description: | OSD heartbeats on the cluster's 'public' network (frontend) are running slow. Investigate the network for any latency issues on this subnet. Use 'ceph health detail' to show the affected OSDs. @@ -1322,15 +1365,16 @@ tests: value: 0 alert_rule_test: - eval_time: 1m - alertname: OSD hearbeats running slow (backend) + alertname: CephOSDTimeoutsClusterNetwork - eval_time: 10m - alertname: OSD hearbeats running slow (backend) + alertname: CephOSDTimeoutsClusterNetwork exp_alerts: - exp_labels: name: "OSD_SLOW_PING_TIME_BACK" severity: warning type: ceph_default exp_annotations: + summary: Network issues delaying OSD heartbeats (cluster network) description: | OSD heartbeats on the cluster's 'cluster' network (backend) are running slow. Investigate the network for any latency issues on this subnet. Use 'ceph health detail' to show the affected OSDs. @@ -1346,9 +1390,9 @@ tests: value: 0 alert_rule_test: - eval_time: 1m - alertname: OSD disk size mismatch + alertname: CephOSDInternalDiskSizeMismatch - eval_time: 10m - alertname: OSD disk size mismatch + alertname: CephOSDInternalDiskSizeMismatch exp_alerts: - exp_labels: name: "BLUESTORE_DISK_SIZE_MISMATCH" @@ -1356,6 +1400,7 @@ tests: type: ceph_default exp_annotations: documentation: https://docs.ceph.com/en/latest/rados/operations/health-checks#bluestore-disk-size-mismatch + summary: OSD size inconsistency error description: | One or more OSDs have an internal inconsistency between the size of the physical device and it's metadata. This could lead to the OSD(s) crashing in future. You should redeploy the effected OSDs. @@ -1371,9 +1416,9 @@ tests: value: 1 alert_rule_test: - eval_time: 1m - alertname: OSD Read errors + alertname: CephOSDReadErrors - eval_time: 10m - alertname: OSD Read errors + alertname: CephOSDReadErrors exp_alerts: - exp_labels: name: "BLUESTORE_SPURIOUS_READ_ERRORS" @@ -1381,6 +1426,7 @@ tests: type: ceph_default exp_annotations: documentation: https://docs.ceph.com/en/latest/rados/operations/health-checks#bluestore-spurious-read-errors + summary: Device read errors detected description: > An OSD has encountered read errors, but the OSD has recovered by retrying the reads. This may indicate an issue with the Hardware or Kernel. @@ -1408,17 +1454,18 @@ tests: value: 1 alert_rule_test: - eval_time: 1m - alertname: OSD down + alertname: CephOSDDown - eval_time: 10m - alertname: OSD down + alertname: CephOSDDown exp_alerts: - exp_labels: name: "OSD_DOWN" severity: warning type: ceph_default - oid: 1.3.6.1.4.1.50495.15.1.2.4.2 + oid: 1.3.6.1.4.1.50495.1.2.1.4.2 exp_annotations: documentation: https://docs.ceph.com/en/latest/rados/operations/health-checks#osd-down + summary: An OSD has been marked down/unavailable description: | 1 OSD down for over 5mins. @@ -1436,23 +1483,53 @@ tests: value: 1 alert_rule_test: - eval_time: 1m - alertname: OSDs near full + alertname: CephOSDNearFull - eval_time: 10m - alertname: OSDs near full + alertname: CephOSDNearFull exp_alerts: - exp_labels: name: "OSD_NEARFULL" severity: warning type: ceph_default - oid: 1.3.6.1.4.1.50495.15.1.2.4.3 + oid: 1.3.6.1.4.1.50495.1.2.1.4.3 exp_annotations: documentation: https://docs.ceph.com/en/latest/rados/operations/health-checks#osd-nearfull + summary: OSD(s) running low on free space (NEARFULL) description: | One or more OSDs have reached their NEARFULL threshold Use 'ceph health detail' to identify which OSDs have reached this threshold. To resolve, either add capacity to the cluster, or delete unwanted data - interval: 1m + input_series: + - series: 'ceph_health_detail{name="OSD_FULL"}' + values: '0+0x2 1+0x10' + promql_expr_test: + - expr: ceph_health_detail{name="OSD_FULL"} == 1 + eval_time: 3m + exp_samples: + - labels: '{__name__="ceph_health_detail", name="OSD_FULL"}' + value: 1 + alert_rule_test: + - eval_time: 1m + alertname: CephOSDFull + - eval_time: 10m + alertname: CephOSDFull + exp_alerts: + - exp_labels: + name: "OSD_FULL" + severity: critical + type: ceph_default + oid: 1.3.6.1.4.1.50495.1.2.1.4.6 + exp_annotations: + documentation: https://docs.ceph.com/en/latest/rados/operations/health-checks#osd-full + summary: OSD(s) is full, writes blocked + description: | + An OSD has reached it's full threshold. Writes from all pools that share the + affected OSD will be blocked. + + To resolve, either add capacity to the cluster, or delete unwanted data + - interval: 1m input_series: - series: 'ceph_health_detail{name="OSD_BACKFILLFULL"}' values: '0+0x2 1+0x10' @@ -1464,9 +1541,9 @@ tests: value: 1 alert_rule_test: - eval_time: 1m - alertname: OSD unable to perform rebalance + alertname: CephOSDBackfillFull - eval_time: 10m - alertname: OSD unable to perform rebalance + alertname: CephOSDBackfillFull exp_alerts: - exp_labels: name: "OSD_BACKFILLFULL" @@ -1474,6 +1551,7 @@ tests: type: ceph_default exp_annotations: documentation: https://docs.ceph.com/en/latest/rados/operations/health-checks#osd-backfillfull + summary: OSD(s) too full for backfill operations description: | An OSD has reached it's BACKFILL FULL threshold. This will prevent rebalance operations completing for some pools. Check the current capacity utilisation with 'ceph df' @@ -1491,9 +1569,9 @@ tests: value: 0 alert_rule_test: - eval_time: 1m - alertname: OSD too many read repairs + alertname: CephOSDTooManyRepairs - eval_time: 10m - alertname: OSD too many read repairs + alertname: CephOSDTooManyRepairs exp_alerts: - exp_labels: name: "OSD_TOO_MANY_REPAIRS" @@ -1501,67 +1579,72 @@ tests: type: ceph_default exp_annotations: documentation: https://docs.ceph.com/en/latest/rados/operations/health-checks#osd-too-many-repairs + summary: OSD has hit a high number of read errors description: | Reads from an OSD have used a secondary PG to return data to the client, indicating a potential failing disk. # Pools - - interval: 1m + # trigger percent full prediction on pools 1 and 2 only + - interval: 12h input_series: - - series: 'ceph_health_detail{name="POOL_BACKFILLFULL"}' - values: '0+0x2 1+0x10' + - series: 'ceph_pool_percent_used{pool_id="1"}' + values: '70 75 80 87 92' + - series: 'ceph_pool_percent_used{pool_id="2"}' + values: '22 22 23 23 24' + - series: 'ceph_pool_metadata{pool_id="1",name="rbd",type="replicated"}' + values: '1 1 1 1 1' + - series: 'ceph_pool_metadata{pool_id="2",name="default.rgw.index",type="replicated"}' + values: '1 1 1 1 1' promql_expr_test: - - expr: ceph_health_detail{name="POOL_BACKFILLFULL"} == 1 - eval_time: 3m + - expr: | + (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) + group_right ceph_pool_metadata) >= 95 + eval_time: 36h exp_samples: - - labels: '{__name__="ceph_health_detail", name="POOL_BACKFILLFULL"}' - value: 1 + - labels: '{name="rbd",pool_id="1",type="replicated"}' + value: 1.424E+02 # 142% alert_rule_test: - - eval_time: 1m - alertname: Ceph pool is too full for recovery/rebalance - - eval_time: 5m - alertname: Ceph pool is too full for recovery/rebalance + - eval_time: 48h + alertname: CephPoolGrowthWarning exp_alerts: - exp_labels: - name: "POOL_BACKFILLFULL" + name: rbd + pool_id: 1 severity: warning type: ceph_default + oid: 1.3.6.1.4.1.50495.1.2.1.9.2 exp_annotations: + summary: Pool growth rate may soon exceed it's capacity description: > - A pool is approaching it's near full threshold, which will - prevent rebalance operations from completing. You should - consider adding more capacity to the pool. - + Pool 'rbd' will be full in less than 5 days + assuming the average fill-up rate of the past 48 hours. - interval: 1m input_series: - - series: 'ceph_health_detail{name="POOL_FULL"}' + - series: 'ceph_health_detail{name="POOL_BACKFILLFULL"}' values: '0+0x2 1+0x10' promql_expr_test: - - expr: ceph_health_detail{name="POOL_FULL"} == 1 + - expr: ceph_health_detail{name="POOL_BACKFILLFULL"} == 1 eval_time: 3m exp_samples: - - labels: '{__name__="ceph_health_detail", name="POOL_FULL"}' + - labels: '{__name__="ceph_health_detail", name="POOL_BACKFILLFULL"}' value: 1 alert_rule_test: - eval_time: 1m - alertname: Ceph pool is full - writes blocked - - eval_time: 10m - alertname: Ceph pool is full - writes blocked + alertname: CephPoolBackfillFull + - eval_time: 5m + alertname: CephPoolBackfillFull exp_alerts: - exp_labels: - name: "POOL_FULL" - severity: critical + name: "POOL_BACKFILLFULL" + severity: warning type: ceph_default exp_annotations: - documentation: https://docs.ceph.com/en/latest/rados/operations/health-checks#pool-full - description: | - A pool has reached it's MAX quota, or the OSDs supporting the pool - have reached their FULL threshold. Until this is resolved, writes to - the pool will be blocked. + summary: Freespace in a pool is too low for recovery/rebalance + description: > + A pool is approaching it's near full threshold, which will + prevent rebalance operations from completing. You should + consider adding more capacity to the pool. - Determine the affected pool with 'ceph df detail', for example looking - at QUOTA BYTES and STORED. Either increase the pools quota, or add - capacity to the cluster first then increase it's quota - (e.g. ceph osd pool set quota max_bytes ) - interval: 1m input_series: - series: 'ceph_health_detail{name="POOL_NEAR_FULL"}' @@ -1574,15 +1657,16 @@ tests: value: 1 alert_rule_test: - eval_time: 1m - alertname: Ceph pool is approaching full + alertname: CephPoolNearFull - eval_time: 10m - alertname: Ceph pool is approaching full + alertname: CephPoolNearFull exp_alerts: - exp_labels: name: "POOL_NEAR_FULL" severity: warning type: ceph_default exp_annotations: + summary: One or more Ceph pools are getting full description: | A pool has exceeeded it warning (percent full) threshold, or the OSDs supporting the pool have reached their NEARFULL thresholds. Writes may @@ -1607,9 +1691,9 @@ tests: value: 1 alert_rule_test: - eval_time: 1m - alertname: Placement Group(s) have not been scrubbed + alertname: CephPGNotScrubbed - eval_time: 10m - alertname: Placement Group(s) have not been scrubbed + alertname: CephPGNotScrubbed exp_alerts: - exp_labels: name: "PG_NOT_SCRUBBED" @@ -1617,6 +1701,7 @@ tests: type: ceph_default exp_annotations: documentation: https://docs.ceph.com/en/latest/rados/operations/health-checks#pg-not-scrubbed + summary: Placement group(s) have not been scrubbed description: | One or more PGs have not been scrubbed recently. The scrub process is a data integrity feature, protectng against bit-rot. It checks that objects and their metadata (size and @@ -1625,6 +1710,67 @@ tests: scrub window. You can manually initiate a scrub with: ceph pg scrub + - interval: 1m + input_series: + - series: 'ceph_health_detail{name="PG_DAMAGED"}' + values: '0+0x4 1+0x20' + promql_expr_test: + - expr: ceph_health_detail{name=~"PG_DAMAGED|OSD_SCRUB_ERRORS"} == 1 + eval_time: 5m + exp_samples: + - labels: '{__name__="ceph_health_detail", name="PG_DAMAGED"}' + value: 1 + alert_rule_test: + - eval_time: 1m + alertname: CephPGsDamaged + - eval_time: 10m + alertname: CephPGsDamaged + exp_alerts: + - exp_labels: + name: "PG_DAMAGED" + severity: critical + type: ceph_default + oid: 1.3.6.1.4.1.50495.1.2.1.7.4 + exp_annotations: + documentation: https://docs.ceph.com/en/latest/rados/operations/health-checks#pg-damaged + summary: Placement group damaged, manual intervention needed + description: > + During data consistency checks (scrub), at least one PG has been flagged as being + damaged or inconsistent. + + Check to see which PG is affected, and attempt a manual repair if neccessary. To list + problematic placement groups, use 'rados list-inconsistent-pg '. To repair PGs use + the 'ceph pg repair ' command. + - interval: 1m + input_series: + - series: 'ceph_health_detail{name="TOO_MANY_PGS"}' + values: '0+0x4 1+0x20' + promql_expr_test: + - expr: ceph_health_detail{name="TOO_MANY_PGS"} == 1 + eval_time: 5m + exp_samples: + - labels: '{__name__="ceph_health_detail", name="TOO_MANY_PGS"}' + value: 1 + alert_rule_test: + - eval_time: 1m + alertname: CephPGsHighPerOSD + - eval_time: 10m + alertname: CephPGsHighPerOSD + exp_alerts: + - exp_labels: + name: "TOO_MANY_PGS" + severity: warning + type: ceph_default + exp_annotations: + documentation: https://docs.ceph.com/en/latest/rados/operations/health-checks/#too-many-pgs + summary: Placement groups per OSD is too high + description: | + The number of placement groups per OSD is too high (exceeds the mon_max_pg_per_osd setting). + + Check that the pg_autoscaler hasn't been disabled for any of the pools, with 'ceph osd pool autoscale-status' + and that the profile selected is appropriate. You may also adjust the target_size_ratio of a pool to guide + the autoscaler based on the expected relative size of the pool + (i.e. 'ceph osd pool set cephfs.cephfs.meta target_size_ratio .1') - interval: 1m input_series: - series: 'ceph_health_detail{name="PG_RECOVERY_FULL"}' @@ -1637,16 +1783,18 @@ tests: value: 0 alert_rule_test: - eval_time: 1m - alertname: Recovery at risk, cluster too full + alertname: CephPGRecoveryAtRisk - eval_time: 10m - alertname: Recovery at risk, cluster too full + alertname: CephPGRecoveryAtRisk exp_alerts: - exp_labels: name: "PG_RECOVERY_FULL" severity: critical type: ceph_default + oid: 1.3.6.1.4.1.50495.1.2.1.7.5 exp_annotations: documentation: https://docs.ceph.com/en/latest/rados/operations/health-checks#pg-recovery-full + summary: OSDs are too full for automatic recovery description: > Data redundancy may be reduced, or is at risk, since one or more OSDs are at or above their 'full' threshold. Add more capacity to the cluster, or delete unwanted data. @@ -1662,16 +1810,18 @@ tests: value: 0 alert_rule_test: - eval_time: 1m - alertname: Cluster too full, automatic data recovery impaired + alertname: CephPGBackfillAtRisk - eval_time: 10m - alertname: Cluster too full, automatic data recovery impaired + alertname: CephPGBackfillAtRisk exp_alerts: - exp_labels: name: "PG_BACKFILL_FULL" severity: critical type: ceph_default + oid: 1.3.6.1.4.1.50495.1.2.1.7.6 exp_annotations: documentation: https://docs.ceph.com/en/latest/rados/operations/health-checks#pg-backfill-full + summary: Backfill operations are blocked, due to lack of freespace description: > Data redundancy may be at risk due to lack of free space within the cluster. One or more OSDs have breached their 'backfillfull' threshold. Add more capacity, or delete unwanted data. @@ -1689,22 +1839,24 @@ tests: alert_rule_test: # PG_AVAILABILITY and OSD_DOWN not firing .. no alert - eval_time: 1m - alertname: I/O blocked to some data + alertname: CephPGUnavilableBlockingIO exp_alerts: # PG_AVAILABILITY firing, but osd_down is active .. no alert - eval_time: 5m - alertname: I/O blocked to some data + alertname: CephPGUnavilableBlockingIO exp_alerts: # PG_AVAILABILITY firing, AND OSD_DOWN is not active...raise the alert - eval_time: 15m - alertname: I/O blocked to some data + alertname: CephPGUnavilableBlockingIO exp_alerts: - exp_labels: name: "PG_AVAILABILITY" severity: critical type: ceph_default + oid: 1.3.6.1.4.1.50495.1.2.1.7.3 exp_annotations: documentation: https://docs.ceph.com/en/latest/rados/operations/health-checks#pg-availability + summary: Placement group is unavailable, blocking some I/O description: > Data availability is reduced impacting the clusters abilty to service I/O to some data. One or more placement groups (PGs) are in a state that blocks IO. @@ -1720,9 +1872,9 @@ tests: value: 1 alert_rule_test: - eval_time: 1m - alertname: Placement Group(s) have not been 'DEEP' scrubbed + alertname: CephPGNotDeepScrubbed - eval_time: 10m - alertname: Placement Group(s) have not been 'DEEP' scrubbed + alertname: CephPGNotDeepScrubbed exp_alerts: - exp_labels: name: "PG_NOT_DEEP_SCRUBBED" @@ -1730,6 +1882,7 @@ tests: type: ceph_default exp_annotations: documentation: https://docs.ceph.com/en/latest/rados/operations/health-checks#pg-not-deep-scrubbed + summary: Placement group(s) have not been deep scrubbed description: | One or more PGs have not been deep scrubbed recently. Deep scrub is a data integrity feature, protectng against bit-rot. It compares the contents of objects and their @@ -1752,13 +1905,15 @@ tests: value: 1 alert_rule_test: - eval_time: 5m - alertname: Scrape job is missing + alertname: PrometheusJobMissing exp_alerts: - exp_labels: job: ceph severity: critical type: ceph_default + oid: 1.3.6.1.4.1.50495.1.2.1.12.1 exp_annotations: + summary: The scrape job for Ceph is missing from Prometheus description: | The prometheus job that scrapes from Ceph is no longer defined, this will effectively mean you'll have no metrics or alerts for the cluster. @@ -1789,18 +1944,50 @@ tests: alert_rule_test: # OBJECT_UNFOUND but osd.2 is down, so don't fire - eval_time: 5m - alertname: Data not found/missing + alertname: CephObjectMissing exp_alerts: # OBJECT_UNFOUND and all osd's are online, so fire - eval_time: 15m - alertname: Data not found/missing + alertname: CephObjectMissing exp_alerts: - exp_labels: severity: critical type: ceph_default + oid: 1.3.6.1.4.1.50495.1.2.1.10.1 exp_annotations: documentation: https://docs.ceph.com/en/latest/rados/operations/health-checks#object-unfound + summary: Object(s) has been marked UNFOUND description: | A version of a RADOS object can not be found, even though all OSDs are up. I/O requests for this object from clients will block (hang). Resolving this issue may - require the object to be rolled back to a prior version manually, and manually verified. \ No newline at end of file + require the object to be rolled back to a prior version manually, and manually verified. +# Generic Alerts + - interval: 1m + input_series: + - series: 'ceph_health_detail{name="RECENT_CRASH"}' + values: '0 0 0 1 1 1 1 1 1 1 1' + promql_expr_test: + - expr: ceph_health_detail{name="RECENT_CRASH"} == 1 + eval_time: 1m + exp_samples: + alert_rule_test: + # not firing + - eval_time: 1m + alertname: CephDaemonCrash + exp_alerts: + # firing + - eval_time: 10m + alertname: CephDaemonCrash + exp_alerts: + - exp_labels: + name: RECENT_CRASH + severity: critical + type: ceph_default + oid: 1.3.6.1.4.1.50495.1.2.1.1.2 + exp_annotations: + documentation: https://docs.ceph.com/en/latest/rados/operations/health-checks/#recent-crash + summary: One or more Ceph daemons have crashed, and are pending acknowledgement + description: | + One or more daemons have crashed recently, and need to be acknowledged. This notification + ensures that software crashes don't go unseen. To acknowledge a crash, use the + 'ceph crash archive ' command. \ No newline at end of file diff --git a/monitoring/prometheus/tests/validate_rules.py b/monitoring/prometheus/tests/validate_rules.py index b424533242903..428779a47de87 100755 --- a/monitoring/prometheus/tests/validate_rules.py +++ b/monitoring/prometheus/tests/validate_rules.py @@ -8,15 +8,20 @@ # 8 .. Missing fields in YAML # 12 .. Invalid YAML - unable to load # 16 .. Missing input files - -# TODO: logging for debug +# +# Externals +# snmptranslate .. used to determine the oid's in the MIB to verify the rule -> MIB is correct +# import re import os import sys import yaml +import shutil +import string from bs4 import BeautifulSoup from typing import List, Any, Dict, Set, Optional, Tuple +import subprocess import urllib.request import urllib.error @@ -25,6 +30,15 @@ DOCLINK_NAME = 'documentation' DEFAULT_RULES_FILENAME = '../alerts/ceph_default_alerts.yml' DEFAULT_TEST_FILENAME = 'test_alerts.yml' +MIB_FILE = '../../snmp/CEPH-MIB.txt' + + +def isascii(s: str) -> bool: + try: + s.encode('ascii') + except UnicodeEncodeError: + return False + return True def read_file(file_name: str) -> Tuple[str, str]: @@ -52,6 +66,14 @@ def load_yaml(file_name: str) -> Tuple[Dict[str, Any], str]: return data, errs +def run_command(command: str): + c = command.split() + completion = subprocess.run(c, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + return (completion.returncode, + completion.stdout.decode('utf-8').split('\n'), + completion.stderr.decode('utf-8').split('\n')) + + class HTMLCache: def __init__(self) -> None: self.cache: Dict[str, Tuple[int, str]] = {} @@ -93,7 +115,6 @@ class PrometheusRule: expected_attrs = [ 'alert', 'expr', - 'for', 'labels', 'annotations' ] @@ -106,11 +127,30 @@ def __init__(self, rule_group, rule_data: Dict[str, Any]): self.rule = rule_data self.errors: List[str] = [] self.warnings: List[str] = [] - self.validate() + @property + def has_oid(self): + return True if self.rule.get('labels', {}).get('oid', '') else False + + @property + def labels(self) -> Dict[str, str]: + return self.rule.get('labels', {}) + + @property + def annotations(self) -> Dict[str, str]: + return self.rule.get('annotations', {}) + def _check_alert_name(self): - pass + # this is simplistic, but works in the context of the alert name + if self.name[0] in string.ascii_uppercase and \ + self.name != self.name.lower() and \ + self.name != self.name.upper() and \ + " " not in self.name and \ + "_" not in self.name: + return + + self.warnings.append("Alert name is not in CamelCase format") def _check_structure(self): rule_attrs = self.rule.keys() @@ -123,17 +163,16 @@ def _check_structure(self): def _check_labels(self): for rqd in ['severity', 'type']: - if rqd not in self.rule.get('labels', ''): + if rqd not in self.labels.keys(): self.errors.append(f"rule is missing {rqd} label definition") def _check_annotations(self): - for rqd in ['description']: - if rqd not in self.rule.get('annotations', ''): + for rqd in ['summary', 'description']: + if rqd not in self.annotations: self.errors.append(f"rule is missing {rqd} annotation definition") def _check_doclink(self): - annotations = self.rule.get('annotations', {}) - doclink = annotations.get(DOCLINK_NAME, '') + doclink = self.annotations.get(DOCLINK_NAME, '') if doclink: url = urlparse(doclink) @@ -148,20 +187,36 @@ def _check_doclink(self): self.errors.append(f"documentation link error: {status} {content}") def _check_snmp(self): - labels = self.rule.get('labels', {}) - oid = labels.get('oid', '') - if labels.get('severity', '') == 'critical' and not oid: + oid = self.labels.get('oid', '') + + if self.labels.get('severity', '') == 'critical' and not oid: self.warnings.append("critical level alert is missing an SNMP oid entry") - if oid and not re.search('^1.3.6.1.4.1.50495.15.1.2.\\d+.\\d+$', oid): - self.errors.append("invalid OID provided") + if oid and not re.search('^1.3.6.1.4.1.50495.1.2.\\d+.\\d+.\\d+$', oid): + self.errors.append("invalid OID format provided") + if self.group.get_oids(): + if oid and oid not in self.group.get_oids(): + self.errors.append(f"rule defines an OID {oid} that is missing from the MIB file({os.path.basename(MIB_FILE)})") + + def _check_ascii(self): + if 'oid' not in self.labels: + return + + desc = self.annotations.get('description', '') + summary = self.annotations.get('summary', '') + if not isascii(desc): + self.errors.append(f"non-ascii characters found in 'description' field will cause issues in associated snmp trap.") + if not isascii(summary): + self.errors.append(f"non-ascii characters found in 'summary' field will cause issues in associated snmp trap.") def validate(self): + self._check_alert_name() self._check_structure() self._check_labels() self._check_annotations() self._check_doclink() self._check_snmp() + self._check_ascii() char = '.' if self.errors: @@ -200,6 +255,9 @@ def update(self, problem_type:str, alert_name:str): def fetch_html_page(self, url): return self.rule_file.fetch_html_page(url) + def get_oids(self): + return self.rule_file.oid_list + @property def error_count(self): return len(self.problems['error']) @@ -214,10 +272,11 @@ def count(self): class RuleFile: - def __init__(self, parent, file_name, rules): + def __init__(self, parent, file_name, rules, oid_list): self.parent = parent self.file_name = file_name self.rules: Dict[str, Any] = rules + self.oid_list = oid_list self.problems: Set[str] = set() self.group: Dict[str, RuleGroup] = {} self.alert_names_seen: Set[str] = set() @@ -242,10 +301,19 @@ def group_count(self): @property def rule_count(self): rule_count = 0 - for group_name, rule_group in self.group.items(): + for _group_name, rule_group in self.group.items(): rule_count += rule_group.count return rule_count + @property + def oid_count(self): + oid_count = 0 + for _group_name, rule_group in self.group.items(): + for _rule_name, rule in rule_group.rules.items(): + if rule.has_oid: + oid_count += 1 + return oid_count + @property def group_names(self): return self.group.keys() @@ -278,23 +346,23 @@ def load_groups(self): # skipped recording rule pass - def error_report(self): - def max_width(item_list: List[str]) -> int: - return max([len(i) for i in item_list]) + def report(self): + def max_width(item_list: Set[str], min_width: int = 0) -> int: + return max([len(i) for i in item_list] + [min_width]) if not self.problems and not self.duplicate_alert_names: - print("\nNo problems detected in rule file") + print("\nNo problems detected in the rule file") return print("\nProblem Report\n") - group_width = max_width(self.problems) + group_width = max_width(self.problems, 5) alert_names = set() for g in self.problems: group = self.group[g] alert_names.update(group.problems.get('error', [])) alert_names.update(group.problems.get('warning', [])) - alert_width = max_width(alert_names) + alert_width = max_width(alert_names, 10) template = " {group:<{group_width}} {severity:<8} {alert_name:<{alert_width}} {description}" @@ -382,7 +450,7 @@ def _check_alert_names(self, alert_names: List[str]): def process(self, defined_alert_names: List[str]): self._check_alert_names(defined_alert_names) - def error_report(self) -> None: + def report(self) -> None: if not self.problems: print("\nNo problems detected in unit tests file") @@ -404,6 +472,26 @@ def __init__(self, rules_filename: str = None, test_filename: str = None): self.warnings = {} self.error_count = 0 self.warning_count = 0 + self.oid_count = 0 + + self.oid_list = self.build_oid_list() + + def build_oid_list(self) -> List[str]: + + cmd = shutil.which('snmptranslate') + if not cmd: + return [] + + rc, stdout, stderr = run_command(f"{cmd} -Pu -Tz -M ../../snmp:/usr/share/snmp/mibs -m CEPH-MIB") + if rc != 0: + return [] + + oid_list: List[str] = [] + for line in stdout[:-1]: + _label, oid = line.replace('"', '').replace('\t', ' ').split() + oid_list.append(oid) + + return oid_list @property def status(self): @@ -448,36 +536,34 @@ def run(self): print(errs) sys.exit(12) - self.rule_file = RuleFile(self, self.rules_filename, rules) + self.rule_file = RuleFile(self, self.rules_filename, rules, self.oid_list) self.summarise_rule_file() self.unit_tests = UnitTests(self.test_filename) self.unit_tests.process(self.rule_file.alert_names_seen) - def error_report(self): + def report(self): print("\n\nSummary\n") print(f"Rule file : {self.rules_filename}") print(f"Unit Test file : {self.test_filename}") print(f"\nRule groups processed : {self.rule_file.group_count:>3}") print(f"Rules processed : {self.rule_file.rule_count:>3}") + print(f"SNMP OIDs declared : {self.rule_file.oid_count:>3} {'(snmptranslate missing, unable to cross check)' if not self.oid_list else ''}") print(f"Rule errors : {self.error_count:>3}") print(f"Rule warnings : {self.warning_count:>3}") print(f"Rule name duplicates : {len(self.rule_file.duplicate_alert_names):>3}") print(f"Unit tests missing : {len(self.unit_tests.problems):>3}") - if self.rule_file_problems: - self.rule_file.error_report() - if self.unit_tests.problems: - self.unit_tests.error_report() + self.rule_file.report() + self.unit_tests.report() def main(): checker = RuleChecker() checker.run() - if checker.status > 0: - checker.error_report() - print() + checker.report() + print() sys.exit(checker.status) diff --git a/monitoring/snmp/CEPH-MIB.txt b/monitoring/snmp/CEPH-MIB.txt new file mode 100644 index 0000000000000..f54cb3610377a --- /dev/null +++ b/monitoring/snmp/CEPH-MIB.txt @@ -0,0 +1,337 @@ +CEPH-MIB DEFINITIONS ::= BEGIN + +IMPORTS + MODULE-IDENTITY, NOTIFICATION-TYPE, enterprises + FROM SNMPv2-SMI + MODULE-COMPLIANCE, NOTIFICATION-GROUP + FROM SNMPv2-CONF +; + +-- Linting information: +-- +-- # smilint -l 6 -i notification-not-reversible ./CEPH-MIB.txt +-- +-- ignore: notification-not-reversible since our SNMP gateway doesn't use SNMPv1 +-- + +ceph MODULE-IDENTITY + LAST-UPDATED + "202111010000Z" -- Nov 01, 2021 + ORGANIZATION + "The Ceph Project + https://ceph.io" + CONTACT-INFO + "Email: + + Send comments to: " + DESCRIPTION + "The MIB module for Ceph. In it's current form it only + supports Notifications, since Ceph itself doesn't provide + any SNMP agent functionality. + + Notifications are provided through a Prometheus/Alertmanager + webhook passing alerts to an external gateway service that is + responsible for formatting, forwarding and authenticating to + the SNMP receiver. + " + REVISION + "202111010000Z" --Nov 01, 2021 + DESCRIPTION + "Latest version including the following updates; + + - MIB restructure to align with linting + - names shortened and simplified (less verbose) + - Simplified structure due to switch to https://github.com/maxwo/snmp_notifier + - objects removed + - notifications updated + - Added module compliance + - Updated to latest prometheus alert rule definitions + " + ::= { enterprises 50495 } + +cephCluster OBJECT IDENTIFIER ::= { ceph 1 } +cephConformance OBJECT IDENTIFIER ::= { ceph 2 } + +-- cephMetadata is a placeholder for possible future expansion via an agent +-- where we could provide an overview of the clusters configuration +cephMetadata OBJECT IDENTIFIER ::= { cephCluster 1 } +cephNotifications OBJECT IDENTIFIER ::= { cephCluster 2 } + +prometheus OBJECT IDENTIFIER ::= { cephNotifications 1 } + +-- +-- Notifications: first we define the notification 'branches' for the +-- different categories of notifications / alerts +promGeneric OBJECT IDENTIFIER ::= { prometheus 1 } +promHealthStatus OBJECT IDENTIFIER ::= { prometheus 2 } +promMon OBJECT IDENTIFIER ::= { prometheus 3 } +promOsd OBJECT IDENTIFIER ::= { prometheus 4 } +promMds OBJECT IDENTIFIER ::= { prometheus 5 } +promMgr OBJECT IDENTIFIER ::= { prometheus 6 } +promPGs OBJECT IDENTIFIER ::= { prometheus 7 } +promNode OBJECT IDENTIFIER ::= { prometheus 8 } +promPool OBJECT IDENTIFIER ::= { prometheus 9 } +promRados OBJECT IDENTIFIER ::= { prometheus 10 } +promCephadm OBJECT IDENTIFIER ::= { prometheus 11 } +promPrometheus OBJECT IDENTIFIER ::= { prometheus 12 } + +promGenericNotification NOTIFICATION-TYPE + STATUS current + DESCRIPTION "Generic alert issued when the Prometheus rule doesn't provide an OID." +::= { promGeneric 1 } + +promGenericDaemonCrash NOTIFICATION-TYPE + STATUS current + DESCRIPTION "One or more daemons have crashed recently, and are yet to be archived" +::= { promGeneric 2 } + +promHealthStatusError NOTIFICATION-TYPE + STATUS current + DESCRIPTION "Ceph in health_error state for too long." +::= { promHealthStatus 1 } + +promHealthStatusWarning NOTIFICATION-TYPE + STATUS current + DESCRIPTION "Ceph in health_warn for too long." +::= { promHealthStatus 2 } + +promMonLowQuorum NOTIFICATION-TYPE + STATUS current + DESCRIPTION "Monitor count in quorum is low." +::= { promMon 1 } + +promMonDiskSpaceCritical NOTIFICATION-TYPE + STATUS current + DESCRIPTION "Monitor diskspace is critically low." +::= { promMon 2 } + +promOsdDownHigh NOTIFICATION-TYPE + STATUS current + DESCRIPTION "A high number of OSDs are down." +::= { promOsd 1 } + +promOsdDown NOTIFICATION-TYPE + STATUS current + DESCRIPTION "One or more Osds down." +::= { promOsd 2 } + +promOsdNearFull NOTIFICATION-TYPE + STATUS current + DESCRIPTION "An OSD is dangerously full." +::= { promOsd 3 } + +promOsdFlapping NOTIFICATION-TYPE + STATUS current + DESCRIPTION "An OSD was marked down at back up at least once a minute for 5 minutes." +::= { promOsd 4 } + +promOsdHighPgDeviation NOTIFICATION-TYPE + STATUS current + DESCRIPTION "An OSD deviates by more then 30% from average PG count." +::= { promOsd 5 } + +promOsdFull NOTIFICATION-TYPE + STATUS current + DESCRIPTION "An OSD has reached its full threshold." +::= { promOsd 6 } + +promOsdHighPredictedFailures NOTIFICATION-TYPE + STATUS current + DESCRIPTION "Normal self healing unable to cope with the number of devices predicted to fail." +::= { promOsd 7 } + +promOsdHostDown NOTIFICATION-TYPE + STATUS current + DESCRIPTION "Ceph OSD host is down." +::= { promOsd 8 } + +promMdsDamaged NOTIFICATION-TYPE + STATUS current + DESCRIPTION "Cephfs filesystem is damaged." +::= { promMds 1 } + +promMdsReadOnly NOTIFICATION-TYPE + STATUS current + DESCRIPTION "Cephfs filesystem marked as READ-ONLY" +::= { promMds 2 } + +promMdsOffline NOTIFICATION-TYPE + STATUS current + DESCRIPTION "Cephfs filesystem is unavailable/offline." +::= { promMds 3 } + +promMdsDegraded NOTIFICATION-TYPE + STATUS current + DESCRIPTION "Cephfs filesystem is in a degraded state." +::= { promMds 4 } + +promMdsNoStandby NOTIFICATION-TYPE + STATUS current + DESCRIPTION "Cephfs MDS daemon failure, no standby available" +::= { promMds 5 } + +promMgrModuleCrash NOTIFICATION-TYPE + STATUS current + DESCRIPTION "Ceph mgr module has crashed recently" +::= { promMgr 1 } + +promMgrPrometheusInactive NOTIFICATION-TYPE + STATUS current + DESCRIPTION "Ceph mgr prometheus module not responding" +::= { promMgr 2 } + +promPGsInactive NOTIFICATION-TYPE + STATUS current + DESCRIPTION "One or more PGs are inactive for more than 5 minutes." +::= { promPGs 1 } + +promPGsUnclean NOTIFICATION-TYPE + STATUS current + DESCRIPTION "One or more PGs are not clean for more than 15 minutes." +::= { promPGs 2 } + +promPGsUnavailable NOTIFICATION-TYPE + STATUS current + DESCRIPTION "One or more PGs is unavailable, blocking I/O to those objects." +::= { promPGs 3 } + +promPGsDamaged NOTIFICATION-TYPE + STATUS current + DESCRIPTION "One or more PGs is damaged." +::= { promPGs 4 } + +promPGsRecoveryFull NOTIFICATION-TYPE + STATUS current + DESCRIPTION "PG recovery is impaired due to full OSDs." +::= { promPGs 5 } + +promPGsBackfillFull NOTIFICATION-TYPE + STATUS current + DESCRIPTION "PG backfill is impaired due to full OSDs." +::= { promPGs 6 } + +promNodeRootVolumeFull NOTIFICATION-TYPE + STATUS current + DESCRIPTION "Root volume (OSD and MON store) is dangerously full (< 5% free)." +::= { promNode 1 } + +promNodeNetworkPacketDrops NOTIFICATION-TYPE + STATUS current + DESCRIPTION "A node experiences packet drop > 1 packet/s on an interface." +::= { promNode 2 } + +promNodeNetworkPacketErrors NOTIFICATION-TYPE + STATUS current + DESCRIPTION "A node experiences packet errors > 1 packet/s on an interface." +::= { promNode 3 } + +promNodeStorageFilling NOTIFICATION-TYPE + STATUS current + DESCRIPTION "A mountpoint will be full in less then 5 days assuming the average fillup rate of the past 48 hours." +::= { promNode 4 } + +promPoolFull NOTIFICATION-TYPE + STATUS current + DESCRIPTION "A pool is at 90% capacity or over." +::= { promPool 1 } + +promPoolFilling NOTIFICATION-TYPE + STATUS current + DESCRIPTION "A pool will be full in less then 5 days assuming the average fillup rate of the past 48 hours." +::= { promPool 2 } + +promRadosUnfound NOTIFICATION-TYPE + STATUS current + DESCRIPTION "A RADOS object can not be found, even though all OSDs are online." +::= { promRados 1 } + +promCephadmDaemonDown NOTIFICATION-TYPE + STATUS current + DESCRIPTION "Cephadm has determined that a daemon is down." +::= { promCephadm 1 } + +promCephadmUpgradeFailure NOTIFICATION-TYPE + STATUS current + DESCRIPTION "Cephadm attempted to upgrade the cluster and encountered a problem." +::= { promCephadm 2 } + +promPrometheusJobMissing NOTIFICATION-TYPE + STATUS current + DESCRIPTION "The prometheus scrape job is not defined." +::= { promPrometheus 1 } +-- ---------------------------------------------------------- -- +-- IEEE 802.1D MIB - Conformance Information +-- ---------------------------------------------------------- -- + +cephAlertGroups OBJECT IDENTIFIER ::= { cephConformance 1 } +cephCompliances OBJECT IDENTIFIER ::= { cephConformance 2 } + +-- ---------------------------------------------------------- -- +-- units of conformance +-- ---------------------------------------------------------- -- + +-- ---------------------------------------------------------- -- +-- The Trap Notification Group +-- ---------------------------------------------------------- -- + +cephNotificationGroup NOTIFICATION-GROUP + NOTIFICATIONS { + promGenericNotification, + promGenericDaemonCrash, + promHealthStatusError, + promHealthStatusWarning, + promMonLowQuorum, + promMonDiskSpaceCritical, + promOsdDownHigh, + promOsdDown, + promOsdNearFull, + promOsdFlapping, + promOsdHighPgDeviation, + promOsdFull, + promOsdHighPredictedFailures, + promOsdHostDown, + promMdsDamaged, + promMdsReadOnly, + promMdsOffline, + promMdsDegraded, + promMdsNoStandby, + promMgrModuleCrash, + promMgrPrometheusInactive, + promPGsInactive, + promPGsUnclean, + promPGsUnavailable, + promPGsDamaged, + promPGsRecoveryFull, + promPGsBackfillFull, + promNodeRootVolumeFull, + promNodeNetworkPacketDrops, + promNodeNetworkPacketErrors, + promNodeStorageFilling, + promPoolFull, + promPoolFilling, + promRadosUnfound, + promCephadmDaemonDown, + promCephadmUpgradeFailure, + promPrometheusJobMissing + } + STATUS current + DESCRIPTION + "A collection of notifications triggered by the Prometheus + rules to convey Ceph cluster state" + ::= { cephAlertGroups 2 } + +-- ---------------------------------------------------------- -- +-- compliance statements +-- ---------------------------------------------------------- -- + +cephCompliance MODULE-COMPLIANCE + STATUS current + DESCRIPTION + "The Compliance statement for the Ceph MIB" + MODULE + MANDATORY-GROUPS { + cephNotificationGroup + } + ::= { cephCompliances 1 } + +END diff --git a/monitoring/snmp/README.md b/monitoring/snmp/README.md index dccef1908f89d..1a5b609556df4 100644 --- a/monitoring/snmp/README.md +++ b/monitoring/snmp/README.md @@ -1,24 +1,54 @@ # SNMP schema +To show the [OID](https://en.wikipedia.org/wiki/Object_identifier)'s supported by the MIB, use the snmptranslate command. Here's an example: +``` +snmptranslate -Pu -Tz -M ~/git/ceph/monitoring/snmp:/usr/share/snmp/mibs -m CEPH-MIB +``` +*The `snmptranslate` command is in the net-snmp-utils package* -## Traps +The MIB provides a NOTIFICATION only implementation since ceph doesn't have an SNMP +agent feature. -| OID | Description | -| :--- | :--- | -| 1.3.6.1.4.1.50495.15.1.2.1 | The default trap. This is used if no OID is specified in the alert labels. | -| 1.3.6.1.4.1.50495.15.1.2.[2...N] | Custom traps. | +## Integration +The SNMP MIB is has been aligned to the Prometheus rules. Any rule that defines a +critical alert should have a corresponding oid in the CEPH-MIB.txt file. To generate +an SNMP notification, you must use an SNMP gateway that the Prometheus Alertmanager +service can forward alerts through to, via it's webhooks feature. -## Objects +  -The following objects are appended as variable binds to an SNMP trap. +## SNMP Gateway +The recommended SNMP gateway is https://github.com/maxwo/snmp_notifier. This is a widely +used and generic SNMP gateway implementation written in go. It's usage (syntax and +parameters) is very similar to Prometheus, AlertManager and even node-exporter. -| OID | Type | Description | -| :--- | :---: | :--- | -| 1.3.6.1.4.1.50495.15.1.1.1 | String | The name of the Prometheus alert. | -| 1.3.6.1.4.1.50495.15.1.1.2 | String | The status of the Prometheus alert. | -| 1.3.6.1.4.1.50495.15.1.1.3 | String | The severity of the Prometheus alert. | -| 1.3.6.1.4.1.50495.15.1.1.4 | String | Unique identifier for the Prometheus instance. | -| 1.3.6.1.4.1.50495.15.1.1.5 | String | The name of the Prometheus job. | -| 1.3.6.1.4.1.50495.15.1.1.6 | String | The Prometheus alert description field. | -| 1.3.6.1.4.1.50495.15.1.1.7 | String | Additional Prometheus alert labels as JSON string. | -| 1.3.6.1.4.1.50495.15.1.1.8 | Unix timestamp | The time when the Prometheus alert occurred. | -| 1.3.6.1.4.1.50495.15.1.1.9 | String | The raw Prometheus alert as JSON string. | \ No newline at end of file +  +## SNMP OIDs +The main components of the Ceph MIB is can be broken down into discrete areas + + +``` +internet private enterprise ceph ceph Notifications Prometheus Notification + org cluster (alerts) source Category +1.3.6.1 .4 .1 .50495 .1 .2 .1 .2 (Ceph Health) + .3 (MON) + .4 (OSD) + .5 (MDS) + .6 (MGR) + .7 (PGs) + .8 (Nodes) + .9 (Pools) + .10 (Rados) + .11 (cephadm) + .12 (prometheus) + +``` +Individual alerts are placed within the appropriate alert category. For example, to add +a notification relating to a MGR issue, you would use the oid 1.3.6.1.4.1.50495.1.2.1.6.x + +The SNMP gateway also adds additional components to the SNMP notification ; + +| Suffix | Description | +|--------|-------------| +| .1 | The oid | +| .2 | Severity of the alert. When an alert is resolved, severity is 'info', and the description is set to Status:OK| +| .3 | Text of the alert(s) | From 4954b31a1f263c5b0389dd21353d35e084864ca7 Mon Sep 17 00:00:00 2001 From: Huber-ming Date: Fri, 5 Nov 2021 11:19:03 +0800 Subject: [PATCH 0531/2866] radosgw-admin: supplement help documents with 'bilog autotrim' Signed-off-by: Huber-ming --- src/rgw/rgw_admin.cc | 3 ++- src/test/cli/radosgw-admin/help.t | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc index 703930045722f..c1bb10e9ae8a8 100644 --- a/src/rgw/rgw_admin.cc +++ b/src/rgw/rgw_admin.cc @@ -238,12 +238,13 @@ void usage() cout << " metadata rm remove metadata info\n"; cout << " metadata list list metadata info\n"; cout << " mdlog list list metadata log\n"; + cout << " mdlog autotrim auto trim metadata log\n"; cout << " mdlog trim trim metadata log (use marker)\n"; cout << " mdlog status read metadata log status\n"; cout << " bilog list list bucket index log\n"; cout << " bilog trim trim bucket index log (use start-marker, end-marker)\n"; - cout << " mdlog autotrim auto trim metadata log\n"; cout << " bilog status read bucket index log status\n"; + cout << " bilog autotrim auto trim bucket index log\n"; cout << " datalog list list data log\n"; cout << " datalog trim trim data log\n"; cout << " datalog status read data log status\n"; diff --git a/src/test/cli/radosgw-admin/help.t b/src/test/cli/radosgw-admin/help.t index 095c46d0de1dd..1fc852247f400 100644 --- a/src/test/cli/radosgw-admin/help.t +++ b/src/test/cli/radosgw-admin/help.t @@ -130,12 +130,13 @@ metadata rm remove metadata info metadata list list metadata info mdlog list list metadata log + mdlog autotrim auto trim metadata log mdlog trim trim metadata log (use marker) mdlog status read metadata log status bilog list list bucket index log bilog trim trim bucket index log (use start-marker, end-marker) - mdlog autotrim auto trim metadata log bilog status read bucket index log status + bilog autotrim auto trim bucket index log datalog list list data log datalog trim trim data log datalog status read data log status From dc566a3cd30d91ecbe87cb049df5e9462a526b6d Mon Sep 17 00:00:00 2001 From: Yin Congmin Date: Tue, 6 Jul 2021 19:35:42 +0800 Subject: [PATCH 0532/2866] librbd/cache/pwl/ssd: add layout version control Signed-off-by: Yin Congmin --- src/librbd/cache/pwl/Types.h | 5 +- src/librbd/cache/pwl/rwl/WriteLog.cc | 6 +-- src/librbd/cache/pwl/ssd/WriteLog.cc | 77 ++++++++++++++++++---------- 3 files changed, 56 insertions(+), 32 deletions(-) diff --git a/src/librbd/cache/pwl/Types.h b/src/librbd/cache/pwl/Types.h index ce6f6ab9da870..863292e161172 100644 --- a/src/librbd/cache/pwl/Types.h +++ b/src/librbd/cache/pwl/Types.h @@ -177,7 +177,8 @@ const uint64_t MIN_POOL_SIZE = DEFAULT_POOL_SIZE; const uint64_t POOL_SIZE_ALIGN = 1 << 20; constexpr double USABLE_SIZE = (7.0 / 10); const uint64_t BLOCK_ALLOC_OVERHEAD_BYTES = 16; -const uint8_t RWL_POOL_VERSION = 1; +const uint8_t RWL_LAYOUT_VERSION = 1; +const uint8_t SSD_LAYOUT_VERSION = 1; const uint64_t MAX_LOG_ENTRIES = (1024 * 1024); const double AGGRESSIVE_RETIRE_HIGH_WATER = 0.75; const double RETIRE_HIGH_WATER = 0.50; @@ -282,7 +283,7 @@ struct WriteLogPoolRoot { #ifdef WITH_RBD_RWL union { struct { - uint8_t layout_version; /* Version of this structure (RWL_POOL_VERSION) */ + uint8_t layout_version; /* Version of this structure (RWL_LAYOUT_VERSION) */ }; uint64_t _u64; } header; diff --git a/src/librbd/cache/pwl/rwl/WriteLog.cc b/src/librbd/cache/pwl/rwl/WriteLog.cc index f635015329a6c..e90f2fa8eba0b 100644 --- a/src/librbd/cache/pwl/rwl/WriteLog.cc +++ b/src/librbd/cache/pwl/rwl/WriteLog.cc @@ -301,7 +301,7 @@ bool WriteLog::initialize_pool(Context *on_finish, pwl::DeferredContexts &lat m_first_valid_entry = 0; TX_BEGIN(m_log_pool) { TX_ADD(pool_root); - D_RW(pool_root)->header.layout_version = RWL_POOL_VERSION; + D_RW(pool_root)->header.layout_version = RWL_LAYOUT_VERSION; D_RW(pool_root)->log_entries = TX_ZALLOC(struct WriteLogCacheEntry, sizeof(struct WriteLogCacheEntry) * num_small_writes); @@ -334,11 +334,11 @@ bool WriteLog::initialize_pool(Context *on_finish, pwl::DeferredContexts &lat return false; } pool_root = POBJ_ROOT(m_log_pool, struct WriteLogPoolRoot); - if (D_RO(pool_root)->header.layout_version != RWL_POOL_VERSION) { + if (D_RO(pool_root)->header.layout_version != RWL_LAYOUT_VERSION) { // TODO: will handle upgrading version in the future lderr(cct) << "Pool layout version is " << D_RO(pool_root)->header.layout_version - << " expected " << RWL_POOL_VERSION << dendl; + << " expected " << RWL_LAYOUT_VERSION << dendl; on_finish->complete(-EINVAL); return false; } diff --git a/src/librbd/cache/pwl/ssd/WriteLog.cc b/src/librbd/cache/pwl/ssd/WriteLog.cc index 6a30c6f422c30..a0cb80d4eb672 100644 --- a/src/librbd/cache/pwl/ssd/WriteLog.cc +++ b/src/librbd/cache/pwl/ssd/WriteLog.cc @@ -133,6 +133,7 @@ template bool WriteLog::initialize_pool(Context *on_finish, pwl::DeferredContexts &later) { int r; + CephContext *cct = m_image_ctx.cct; ceph_assert(ceph_mutex_is_locked_by_me(m_lock)); if (access(this->m_log_pool_name.c_str(), F_OK) != 0) { @@ -177,6 +178,7 @@ bool WriteLog::initialize_pool(Context *on_finish, m_first_valid_entry = DATA_RING_BUFFER_OFFSET; auto new_root = std::make_shared(pool_root); + new_root->layout_version = SSD_LAYOUT_VERSION; new_root->pool_size = this->m_log_pool_size; new_root->flushed_sync_gen = this->m_flushed_sync_gen; new_root->block_size = MIN_WRITE_ALLOC_SSD_SIZE; @@ -187,8 +189,8 @@ bool WriteLog::initialize_pool(Context *on_finish, r = update_pool_root_sync(new_root); if (r != 0) { - lderr(m_image_ctx.cct) << "failed to initialize pool (" - << this->m_log_pool_name << ")" << dendl; + lderr(cct) << "failed to initialize pool (" + << this->m_log_pool_name << ")" << dendl; bdev->close(); delete bdev; on_finish->complete(r); @@ -201,11 +203,57 @@ bool WriteLog::initialize_pool(Context *on_finish, on_finish->complete(r); return false; } + + bufferlist bl; + SuperBlock superblock; + ::IOContext ioctx(cct, nullptr); + r = bdev->read(0, MIN_WRITE_ALLOC_SSD_SIZE, &bl, &ioctx, false); + if (r < 0) { + lderr(cct) << "Read ssd cache superblock failed " << dendl; + goto error_handle; + } + auto p = bl.cbegin(); + decode(superblock, p); + pool_root = superblock.root; + ldout(cct, 1) << "Decoded root: pool_size=" << pool_root.pool_size + << " first_valid_entry=" << pool_root.first_valid_entry + << " first_free_entry=" << pool_root.first_free_entry + << " flushed_sync_gen=" << pool_root.flushed_sync_gen + << dendl; + ceph_assert(is_valid_pool_root(pool_root)); + if (pool_root.layout_version != SSD_LAYOUT_VERSION) { + lderr(cct) << "Pool layout version is " + << pool_root.layout_version + << " expected " << SSD_LAYOUT_VERSION + << dendl; + goto error_handle; + } + if (pool_root.block_size != MIN_WRITE_ALLOC_SSD_SIZE) { + lderr(cct) << "Pool block size is " << pool_root.block_size + << " expected " << MIN_WRITE_ALLOC_SSD_SIZE + << dendl; + goto error_handle; + } + + this->m_log_pool_size = pool_root.pool_size; + this->m_flushed_sync_gen = pool_root.flushed_sync_gen; + this->m_first_valid_entry = pool_root.first_valid_entry; + this->m_first_free_entry = pool_root.first_free_entry; + this->m_bytes_allocated_cap = this->m_log_pool_size - + DATA_RING_BUFFER_OFFSET - + MIN_WRITE_ALLOC_SSD_SIZE; + load_existing_entries(later); m_cache_state->clean = this->m_dirty_log_entries.empty(); m_cache_state->empty = m_log_entries.empty(); } return true; + +error_handle: + bdev->close(); + delete bdev; + on_finish->complete(-EINVAL); + return false; } template @@ -235,33 +283,8 @@ void WriteLog::remove_pool_file() { template void WriteLog::load_existing_entries(pwl::DeferredContexts &later) { - bufferlist bl; CephContext *cct = m_image_ctx.cct; - ::IOContext ioctx(cct, nullptr); - bdev->read(0, MIN_WRITE_ALLOC_SSD_SIZE, &bl, &ioctx, false); - SuperBlock superblock; - - auto p = bl.cbegin(); - decode(superblock, p); - - pool_root = superblock.root; - ldout(cct, 1) << "Decoded root: pool_size=" << pool_root.pool_size - << " first_valid_entry=" << pool_root.first_valid_entry - << " first_free_entry=" << pool_root.first_free_entry - << " flushed_sync_gen=" << pool_root.flushed_sync_gen - << dendl; - ceph_assert(is_valid_pool_root(pool_root)); - - this->m_log_pool_size = pool_root.pool_size; - this->m_flushed_sync_gen = pool_root.flushed_sync_gen; - this->m_first_valid_entry = pool_root.first_valid_entry; - this->m_first_free_entry = pool_root.first_free_entry; - - this->m_bytes_allocated_cap = this->m_log_pool_size - - DATA_RING_BUFFER_OFFSET - MIN_WRITE_ALLOC_SSD_SIZE; - std::map> sync_point_entries; - std::map missing_sync_points; // Iterate through the log_entries and append all the write_bytes From 39132b0e460d2d667f02019335dced50d5db641d Mon Sep 17 00:00:00 2001 From: Igor Fedotov Date: Fri, 5 Nov 2021 12:38:51 +0300 Subject: [PATCH 0533/2866] os/bluestore: do not select absent device in volume selector Fixes: ttps://tracker.ceph.com/issues/53139 Signed-off-by: Igor Fedotov --- src/os/bluestore/BlueFS.cc | 5 +-- src/os/bluestore/BlueStore.cc | 26 +++++++++++- src/os/bluestore/BlueStore.h | 4 ++ src/test/objectstore/store_test.cc | 63 ++++++++++++++++++++++++++++++ 4 files changed, 93 insertions(+), 5 deletions(-) diff --git a/src/os/bluestore/BlueFS.cc b/src/os/bluestore/BlueFS.cc index f34996be3e000..ccc1c154f9540 100644 --- a/src/os/bluestore/BlueFS.cc +++ b/src/os/bluestore/BlueFS.cc @@ -3786,10 +3786,9 @@ void* OriginalVolumeSelector::get_hint_by_dir(std::string_view dirname) const { // match up with bluestore. the slow device is always the second // one (when a dedicated block.db device is present and used at // bdev 0). the wal device is always last. - if (boost::algorithm::ends_with(dirname, ".slow")) { + if (boost::algorithm::ends_with(dirname, ".slow") && slow_total) { res = BlueFS::BDEV_SLOW; - } - else if (boost::algorithm::ends_with(dirname, ".wal")) { + } else if (boost::algorithm::ends_with(dirname, ".wal") && wal_total) { res = BlueFS::BDEV_WAL; } } diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc index bab1b77e310a3..221e92ea1c564 100644 --- a/src/os/bluestore/BlueStore.cc +++ b/src/os/bluestore/BlueStore.cc @@ -9425,6 +9425,23 @@ void BlueStore::inject_zombie_spanning_blob(coll_t cid, ghobject_t oid, db->submit_transaction_sync(txn); } +void BlueStore::inject_bluefs_file(std::string_view dir, std::string_view name, size_t new_size) +{ + ceph_assert(bluefs); + + BlueFS::FileWriter* p_handle = nullptr; + auto ret = bluefs->open_for_write(dir, name, &p_handle, false); + ceph_assert(ret == 0); + + std::string s('0', new_size); + bufferlist bl; + bl.append(s); + p_handle->append(bl); + + bluefs->fsync(p_handle); + bluefs->close_writer(p_handle); +} + void BlueStore::collect_metadata(map *pm) { dout(10) << __func__ << dendl; @@ -16767,8 +16784,13 @@ uint8_t RocksDBBlueFSVolumeSelector::select_prefer_bdev(void* h) { void RocksDBBlueFSVolumeSelector::get_paths(const std::string& base, paths& res) const { - res.emplace_back(base, l_totals[LEVEL_DB - LEVEL_FIRST]); - res.emplace_back(base + ".slow", l_totals[LEVEL_SLOW - LEVEL_FIRST]); + auto db_size = l_totals[LEVEL_DB - LEVEL_FIRST]; + res.emplace_back(base, db_size); + auto slow_size = l_totals[LEVEL_SLOW - LEVEL_FIRST]; + if (slow_size == 0) { + slow_size = db_size; + } + res.emplace_back(base + ".slow", slow_size); } void* RocksDBBlueFSVolumeSelector::get_hint_by_dir(std::string_view dirname) const { diff --git a/src/os/bluestore/BlueStore.h b/src/os/bluestore/BlueStore.h index f575e524a8f59..4ca912d3af663 100644 --- a/src/os/bluestore/BlueStore.h +++ b/src/os/bluestore/BlueStore.h @@ -3022,6 +3022,10 @@ class BlueStore : public ObjectStore, void inject_legacy_omap(coll_t cid, ghobject_t oid); void inject_stray_omap(uint64_t head, const std::string& name); + void inject_bluefs_file(std::string_view dir, + std::string_view name, + size_t new_size); + void compact() override { ceph_assert(db); db->compact(); diff --git a/src/test/objectstore/store_test.cc b/src/test/objectstore/store_test.cc index 6255de952f7d0..df1d13a9f9f84 100644 --- a/src/test/objectstore/store_test.cc +++ b/src/test/objectstore/store_test.cc @@ -9582,6 +9582,69 @@ TEST_P(StoreTestSpecificAUSize, OmapUpgradeTest) { } } +TEST_P(StoreTestSpecificAUSize, BluefsWriteInSingleDiskEnvTest) { + if (string(GetParam()) != "bluestore") + return; + + g_conf().apply_changes(nullptr); + + StartDeferred(0x1000); + + BlueStore* bstore = dynamic_cast (store.get()); + ceph_assert(bstore); + bstore->inject_bluefs_file("db.slow", "store_test_injection_slow", 1 << 20ul); + bstore->inject_bluefs_file("db.wal", "store_test_injection_wal", 1 << 20ul); + bstore->inject_bluefs_file("db", "store_test_injection_wal", 1 << 20ul); + + AdminSocket* admin_socket = g_ceph_context->get_admin_socket(); + ceph_assert(admin_socket); + + ceph::bufferlist in, out; + ostringstream err; + auto r = admin_socket->execute_command( + { "{\"prefix\": \"bluefs stats\"}" }, + in, err, &out); + if (r != 0) { + cerr << "failure querying: " << cpp_strerror(r) << std::endl; + } else { + std::cout << std::string(out.c_str(), out.length()) << std::endl; + } +} + +TEST_P(StoreTestSpecificAUSize, BluefsWriteInNoWalDiskEnvTest) { + if (string(GetParam()) != "bluestore") + return; + + SetVal(g_conf(), "bluestore_block_db_path", "db"); + SetVal(g_conf(), "bluestore_block_db_size", stringify(1ull << 31).c_str()); + SetVal(g_conf(), "bluestore_block_db_create", "true"); + + g_conf().apply_changes(nullptr); + + StartDeferred(0x1000); + + BlueStore* bstore = dynamic_cast (store.get()); + ceph_assert(bstore); + bstore->inject_bluefs_file("db.slow", "store_test_injection_slow", 1 << 20ul); + bstore->inject_bluefs_file("db.wal", "store_test_injection_wal", 1 << 20ul); + bstore->inject_bluefs_file("db", "store_test_injection_wal", 1 << 20ul); + + AdminSocket* admin_socket = g_ceph_context->get_admin_socket(); + ceph_assert(admin_socket); + + ceph::bufferlist in, out; + ostringstream err; + auto r = admin_socket->execute_command( + { "{\"prefix\": \"bluefs stats\"}" }, + in, err, &out); + if (r != 0) { + cerr << "failure querying: " << cpp_strerror(r) << std::endl; + } + else { + std::cout << std::string(out.c_str(), out.length()) << std::endl; + } +} + #endif // WITH_BLUESTORE int main(int argc, char **argv) { From 41a487005c0727d7ace52ac6f26deffece0a55ea Mon Sep 17 00:00:00 2001 From: Sebastian Wagner Date: Thu, 4 Nov 2021 16:49:21 +0100 Subject: [PATCH 0534/2866] doc/cephadm: Recommend osd_memory_target_autotune In case the cluster runs on hardware that is used exclusively for Ceph, let's recommend `osd_memory_target_autotune` Signed-off-by: Sebastian Wagner --- doc/cephadm/install.rst | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/doc/cephadm/install.rst b/doc/cephadm/install.rst index 269bdfbcad532..61cfb4edc5e60 100644 --- a/doc/cephadm/install.rst +++ b/doc/cephadm/install.rst @@ -316,7 +316,16 @@ available and unused device: ceph orch apply osd --all-available-devices -Or See :ref:`cephadm-deploy-osds` for more detailed instructions. +See :ref:`cephadm-deploy-osds` for more detailed instructions. + +In case the cluster runs on hardware that is used exclusively for +Ceph, it is recommended to enable ``osd_memory_target_autotune``. +See :ref:`osd_autotune`. + + .. prompt:: bash # + + ceph config set osd osd_memory_target_autotune true + Using Ceph ========== From 4bf07e05cea0daebdc149ee02bcddae96da5e588 Mon Sep 17 00:00:00 2001 From: Ronen Friedman Date: Tue, 21 Sep 2021 10:53:20 +0000 Subject: [PATCH 0535/2866] osd/scrub: expose scrubbing schedule to operator Add a 'scrub scheduling info' column to pgs dump. Modify the name and behavior of 'last-scrub-duration'. Signed-off-by: Ronen Friedman Co-Authored-By: Aishwarya Mathuria --- src/include/utime_fmt.h | 32 ++++++ src/mon/PGMap.cc | 11 +- src/osd/PG.cc | 19 ++-- src/osd/PeeringState.cc | 7 ++ src/osd/PeeringState.h | 3 + src/osd/PrimaryLogPG.cc | 2 +- src/osd/osd_types.cc | 72 +++++++++++- src/osd/osd_types.h | 34 ++++-- src/osd/scrubber/osd_scrub_sched.cc | 23 +++- src/osd/scrubber/osd_scrub_sched.h | 6 + src/osd/scrubber/pg_scrubber.cc | 167 +++++++++++++++++++++------- src/osd/scrubber/pg_scrubber.h | 8 +- src/osd/scrubber_common.h | 5 +- 13 files changed, 319 insertions(+), 70 deletions(-) create mode 100644 src/include/utime_fmt.h diff --git a/src/include/utime_fmt.h b/src/include/utime_fmt.h new file mode 100644 index 0000000000000..44c9a40cd7b2d --- /dev/null +++ b/src/include/utime_fmt.h @@ -0,0 +1,32 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +#pragma once +/** + * \file fmtlib formatter for utime_t + */ +#include +#include + +#include + +#include "include/utime.h" + +template <> +struct fmt::formatter { + constexpr auto parse(format_parse_context& ctx) { return ctx.begin(); } + + template + auto format(const utime_t& utime, FormatContext& ctx) + { + if (utime.sec() < ((time_t)(60 * 60 * 24 * 365 * 10))) { + // raw seconds. this looks like a relative time. + return fmt::format_to(ctx.out(), "{}.{:06}", (long)utime.sec(), + utime.usec()); + } + + // this looks like an absolute time. + // conform to http://en.wikipedia.org/wiki/ISO_8601 + auto asgmt = fmt::gmtime(utime.sec()); + return fmt::format_to(ctx.out(), "{:%FT%T}.{:06}{:%z}", asgmt, utime.usec(), asgmt); + } +}; diff --git a/src/mon/PGMap.cc b/src/mon/PGMap.cc index cb90535addd22..d773c2fd3861f 100644 --- a/src/mon/PGMap.cc +++ b/src/mon/PGMap.cc @@ -1657,7 +1657,8 @@ void PGMap::dump_pg_stats_plain( tab.define_column("LAST_DEEP_SCRUB", TextTable::LEFT, TextTable::RIGHT); tab.define_column("DEEP_SCRUB_STAMP", TextTable::LEFT, TextTable::RIGHT); tab.define_column("SNAPTRIMQ_LEN", TextTable::LEFT, TextTable::RIGHT); - tab.define_column("SCRUB_DURATION", TextTable::LEFT, TextTable::RIGHT); + tab.define_column("LAST_SCRUB_DURATION", TextTable::LEFT, TextTable::RIGHT); + tab.define_column("SCRUB_SCHEDULING", TextTable::LEFT, TextTable::LEFT); } for (auto i = pg_stats.begin(); @@ -2230,7 +2231,8 @@ void PGMap::dump_filtered_pg_stats(ostream& ss, set& pgs) const tab.define_column("ACTING", TextTable::LEFT, TextTable::RIGHT); tab.define_column("SCRUB_STAMP", TextTable::LEFT, TextTable::RIGHT); tab.define_column("DEEP_SCRUB_STAMP", TextTable::LEFT, TextTable::RIGHT); - tab.define_column("SCRUB_DURATION", TextTable::LEFT, TextTable::RIGHT); + tab.define_column("LAST_SCRUB_DURATION", TextTable::LEFT, TextTable::RIGHT); + tab.define_column("SCRUB_SCHEDULING", TextTable::LEFT, TextTable::LEFT); for (auto i = pgs.begin(); i != pgs.end(); ++i) { const pg_stat_t& st = pg_stat.at(*i); @@ -2258,8 +2260,9 @@ void PGMap::dump_filtered_pg_stats(ostream& ss, set& pgs) const << actingstr.str() << st.last_scrub_stamp << st.last_deep_scrub_stamp - << st.scrub_duration - << TextTable::endrow; + << st.last_scrub_duration + << st.dump_scrub_schedule() + << TextTable::endrow; } ss << tab; diff --git a/src/osd/PG.cc b/src/osd/PG.cc index 28e4ddb6b3071..a68d75ed160d0 100644 --- a/src/osd/PG.cc +++ b/src/osd/PG.cc @@ -818,10 +818,17 @@ void PG::publish_stats_to_osd() if (!is_primary()) return; + if (m_scrubber) { + recovery_state.update_stats_wo_resched( + [scrubber = m_scrubber.get()](pg_history_t& hist, + pg_stat_t& info) mutable -> void { + info.scrub_sched_status = scrubber->get_schedule(); + }); + } + std::lock_guard l{pg_stats_publish_lock}; - auto stats = recovery_state.prepare_stats_for_publish( - pg_stats_publish, - unstable_stats); + auto stats = + recovery_state.prepare_stats_for_publish(pg_stats_publish, unstable_stats); if (stats) { pg_stats_publish = std::move(stats); } @@ -2533,12 +2540,6 @@ void PG::handle_query_state(Formatter *f) dout(10) << "handle_query_state" << dendl; PeeringState::QueryState q(f); recovery_state.handle_event(q, 0); - - // This code has moved to after the close of recovery_state array. - // I don't think that scrub is a recovery state - if (is_primary() && is_active() && m_scrubber && m_scrubber->is_scrub_active()) { - m_scrubber->handle_query_state(f); - } } void PG::init_collection_pool_opts() diff --git a/src/osd/PeeringState.cc b/src/osd/PeeringState.cc index 5a057b279e4e1..ae9344ec4ab49 100644 --- a/src/osd/PeeringState.cc +++ b/src/osd/PeeringState.cc @@ -3984,6 +3984,13 @@ void PeeringState::update_stats( } } +void PeeringState::update_stats_wo_resched( + std::function f) +{ + f(info.history, info.stats); +} + + bool PeeringState::append_log_entries_update_missing( const mempool::osd_pglog::list &entries, ObjectStore::Transaction &t, std::optional trim_to, diff --git a/src/osd/PeeringState.h b/src/osd/PeeringState.h index 2225fcd3f4ab0..5e676613ca55e 100644 --- a/src/osd/PeeringState.h +++ b/src/osd/PeeringState.h @@ -1809,6 +1809,9 @@ class PeeringState : public MissingLoc::MappingInfo { std::function f, ObjectStore::Transaction *t = nullptr); + void update_stats_wo_resched( + std::function f); + /** * adjust_purged_snaps * diff --git a/src/osd/PrimaryLogPG.cc b/src/osd/PrimaryLogPG.cc index 371ff4c088152..6c2343ba34f4f 100644 --- a/src/osd/PrimaryLogPG.cc +++ b/src/osd/PrimaryLogPG.cc @@ -1032,7 +1032,7 @@ void PrimaryLogPG::do_command( f->close_section(); if (is_primary() && is_active() && m_scrubber) { - m_scrubber->dump(f.get()); + m_scrubber->dump_scrubber(f.get(), m_planned_scrub); } f->open_object_section("agent_state"); diff --git a/src/osd/osd_types.cc b/src/osd/osd_types.cc index 3bb4dfbab4e4f..9e260de9426ea 100644 --- a/src/osd/osd_types.cc +++ b/src/osd/osd_types.cc @@ -36,8 +36,10 @@ extern "C" { #include "common/Formatter.h" #include "common/StackStringStream.h" +#include "include/utime_fmt.h" #include "OSDMap.h" #include "osd_types.h" +#include "osd_types_fmt.h" #include "os/Transaction.h" using std::list; @@ -2856,7 +2858,8 @@ void pg_stat_t::dump(Formatter *f) const f->dump_bool("pin_stats_invalid", pin_stats_invalid); f->dump_bool("manifest_stats_invalid", manifest_stats_invalid); f->dump_unsigned("snaptrimq_len", snaptrimq_len); - f->dump_float("scrub_duration", scrub_duration); + f->dump_int("last_scrub_duration", last_scrub_duration); + f->dump_string("scrub_schedule", dump_scrub_schedule()); stats.dump(f); f->open_array_section("up"); for (auto p = up.cbegin(); p != up.cend(); ++p) @@ -2909,6 +2912,47 @@ void pg_stat_t::dump_brief(Formatter *f) const f->dump_int("acting_primary", acting_primary); } +std::string pg_stat_t::dump_scrub_schedule() const +{ + if (scrub_sched_status.m_is_active) { + return fmt::format( + "{}scrubbing for {}s", + ((scrub_sched_status.m_is_deep == scrub_level_t::deep) ? "deep " : ""), + scrub_sched_status.m_duration_seconds); + } + switch (scrub_sched_status.m_sched_status) { + case pg_scrub_sched_status_t::unknown: + // no reported scrub schedule yet + return "--"s; + case pg_scrub_sched_status_t::not_queued: + return "no scrub is scheduled"s; + case pg_scrub_sched_status_t::scheduled: + return fmt::format( + "{} {}scrub scheduled @ {}", + (scrub_sched_status.m_is_periodic ? "periodic" : "user requested"), + ((scrub_sched_status.m_is_deep == scrub_level_t::deep) ? "deep " : ""), + scrub_sched_status.m_scheduled_at); + case pg_scrub_sched_status_t::queued: + return fmt::format( + "queued for {}scrub", + ((scrub_sched_status.m_is_deep == scrub_level_t::deep) ? "deep " : "")); + default: + // a bug! + return "SCRUB STATE MISMATCH!"s; + } +} + +bool operator==(const pg_scrubbing_status_t& l, const pg_scrubbing_status_t& r) +{ + return + l.m_sched_status == r.m_sched_status && + l.m_scheduled_at == r.m_scheduled_at && + l.m_duration_seconds == r.m_duration_seconds && + l.m_is_active == r.m_is_active && + l.m_is_deep == r.m_is_deep && + l.m_is_periodic == r.m_is_periodic; +} + void pg_stat_t::encode(ceph::buffer::list &bl) const { ENCODE_START(27, 22, bl); @@ -2959,7 +3003,13 @@ void pg_stat_t::encode(ceph::buffer::list &bl) const encode(manifest_stats_invalid, bl); encode(avail_no_missing, bl); encode(object_location_counts, bl); - encode(scrub_duration, bl); + encode(last_scrub_duration, bl); + encode(scrub_sched_status.m_scheduled_at, bl); + encode(scrub_sched_status.m_duration_seconds, bl); + encode((__u16)scrub_sched_status.m_sched_status, bl); + encode(scrub_sched_status.m_is_active, bl); + encode((scrub_sched_status.m_is_deep==scrub_level_t::deep), bl); + encode(scrub_sched_status.m_is_periodic, bl); ENCODE_FINISH(bl); } @@ -3035,7 +3085,18 @@ void pg_stat_t::decode(ceph::buffer::list::const_iterator &bl) decode(object_location_counts, bl); } if (struct_v >= 27) { - decode(scrub_duration, bl); + decode(last_scrub_duration, bl); + decode(scrub_sched_status.m_scheduled_at, bl); + decode(scrub_sched_status.m_duration_seconds, bl); + __u16 scrub_sched_as_u16; + decode(scrub_sched_as_u16, bl); + scrub_sched_status.m_sched_status = (pg_scrub_sched_status_t)(scrub_sched_as_u16); + decode(tmp, bl); + scrub_sched_status.m_is_active = tmp; + decode(tmp, bl); + scrub_sched_status.m_is_deep = tmp ? scrub_level_t::deep : scrub_level_t::shallow; + decode(tmp, bl); + scrub_sched_status.m_is_periodic = tmp; } } DECODE_FINISH(bl); @@ -3069,7 +3130,7 @@ void pg_stat_t::generate_test_instances(list& o) a.last_deep_scrub = eversion_t(13, 14); a.last_deep_scrub_stamp = utime_t(15, 16); a.last_clean_scrub_stamp = utime_t(17, 18); - a.scrub_duration = 0.003; + a.last_scrub_duration = 3617; a.snaptrimq_len = 1048576; list l; object_stat_collection_t::generate_test_instances(l); @@ -3144,7 +3205,8 @@ bool operator==(const pg_stat_t& l, const pg_stat_t& r) l.manifest_stats_invalid == r.manifest_stats_invalid && l.purged_snaps == r.purged_snaps && l.snaptrimq_len == r.snaptrimq_len && - l.scrub_duration == r.scrub_duration; + l.last_scrub_duration == r.last_scrub_duration && + l.scrub_sched_status == r.scrub_sched_status; } // -- store_statfs_t -- diff --git a/src/osd/osd_types.h b/src/osd/osd_types.h index 4f65e14d55d9f..1bb384548d6d6 100644 --- a/src/osd/osd_types.h +++ b/src/osd/osd_types.h @@ -2179,6 +2179,28 @@ inline bool operator==(const object_stat_collection_t& l, return l.sum == r.sum; } +enum class scrub_level_t : bool { shallow = false, deep = true }; +enum class scrub_type_t : bool { not_repair = false, do_repair = true }; + +/// is there a scrub in our future? +enum class pg_scrub_sched_status_t : uint16_t { + unknown, ///< status not reported yet + not_queued, ///< not in the OSD's scrub queue. Probably not active. + active, ///< scrubbing + scheduled, ///< scheduled for a scrub at an already determined time + queued ///< queued to be scrubbed +}; + +struct pg_scrubbing_status_t { + utime_t m_scheduled_at{}; + int32_t m_duration_seconds{0}; // relevant when scrubbing + pg_scrub_sched_status_t m_sched_status{pg_scrub_sched_status_t::unknown}; + bool m_is_active{false}; + scrub_level_t m_is_deep{scrub_level_t::shallow}; + bool m_is_periodic{true}; +}; + +bool operator==(const pg_scrubbing_status_t& l, const pg_scrubbing_status_t& r); /** pg_stat * aggregate stats for a single PG. @@ -2213,6 +2235,7 @@ struct pg_stat_t { utime_t last_scrub_stamp; utime_t last_deep_scrub_stamp; utime_t last_clean_scrub_stamp; + int32_t last_scrub_duration{0}; object_stat_collection_t stats; @@ -2239,6 +2262,8 @@ struct pg_stat_t { // absurd already, so cap it to 2^32 and save 4 bytes at the same time uint32_t snaptrimq_len; + pg_scrubbing_status_t scrub_sched_status; + bool stats_invalid:1; /// true if num_objects_dirty is not accurate (because it was not /// maintained starting from pool creation) @@ -2249,8 +2274,6 @@ struct pg_stat_t { bool pin_stats_invalid:1; bool manifest_stats_invalid:1; - double scrub_duration; - pg_stat_t() : reported_seq(0), reported_epoch(0), @@ -2268,8 +2291,7 @@ struct pg_stat_t { hitset_stats_invalid(false), hitset_bytes_stats_invalid(false), pin_stats_invalid(false), - manifest_stats_invalid(false), - scrub_duration(0) + manifest_stats_invalid(false) { } epoch_t get_effective_last_epoch_clean() const { @@ -2329,6 +2351,7 @@ struct pg_stat_t { bool is_acting_osd(int32_t osd, bool primary) const; void dump(ceph::Formatter *f) const; void dump_brief(ceph::Formatter *f) const; + std::string dump_scrub_schedule() const; void encode(ceph::buffer::list &bl) const; void decode(ceph::buffer::list::const_iterator &bl); static void generate_test_instances(std::list& o); @@ -6082,9 +6105,6 @@ struct PushOp { WRITE_CLASS_ENCODER_FEATURES(PushOp) std::ostream& operator<<(std::ostream& out, const PushOp &op); -enum class scrub_level_t : bool { shallow = false, deep = true }; -enum class scrub_type_t : bool { not_repair = false, do_repair = true }; - /* * summarize pg contents for purposes of a scrub */ diff --git a/src/osd/scrubber/osd_scrub_sched.cc b/src/osd/scrubber/osd_scrub_sched.cc index 6cba89094c691..99f73b1316488 100644 --- a/src/osd/scrubber/osd_scrub_sched.cc +++ b/src/osd/scrubber/osd_scrub_sched.cc @@ -2,8 +2,9 @@ // vim: ts=8 sw=2 smarttab #include "./osd_scrub_sched.h" -#include "include/utime.h" +#include "include/utime_fmt.h" #include "osd/OSD.h" +#include "osd/osd_types_fmt.h" #include "pg_scrubber.h" @@ -48,6 +49,26 @@ void ScrubQueue::ScrubJob::update_schedule( << registration_state() << dendl; } +std::string ScrubQueue::ScrubJob::scheduling_state(utime_t now_is, + bool is_deep_expected) const +{ + // if not in the OSD scheduling queues, not a candidate for scrubbing + if (state != qu_state_t::registered) { + return "no scrub is scheduled"; + } + + // if the time has passed, we are surely in the queue + // (note that for now we do not tell client if 'penalized') + if (now_is > schedule.scheduled_at) { + // we are never sure that the next scrub will indeed be shallow: + return fmt::format("queued for {}scrub", (is_deep_expected ? "deep " : "")); + } + + return fmt::format("{}scrub scheduled @ {}", (is_deep_expected ? "deep " : ""), + schedule.scheduled_at); +} + + // ////////////////////////////////////////////////////////////////////////// // // ScrubQueue diff --git a/src/osd/scrubber/osd_scrub_sched.h b/src/osd/scrubber/osd_scrub_sched.h index f68942568f897..fc34be7a2019b 100644 --- a/src/osd/scrubber/osd_scrub_sched.h +++ b/src/osd/scrubber/osd_scrub_sched.h @@ -139,6 +139,12 @@ class ScrubQueue { : " not-queued"; } + /** + * a text description of the "scheduling intentions" of this PG: + * are we already scheduled for a scrub/deep scrub? when? + */ + std::string scheduling_state(utime_t now_is, bool is_deep_expected) const; + friend std::ostream& operator<<(std::ostream& out, const ScrubJob& pg); }; diff --git a/src/osd/scrubber/pg_scrubber.cc b/src/osd/scrubber/pg_scrubber.cc index df7fe95a41db8..9e3256163e901 100644 --- a/src/osd/scrubber/pg_scrubber.cc +++ b/src/osd/scrubber/pg_scrubber.cc @@ -578,6 +578,7 @@ void PgScrubber::scrub_requested(scrub_level_t scrub_level, dout(20) << __func__ << " pg(" << m_pg_id << ") planned:" << req_flags << dendl; update_scrub_job(req_flags); + m_pg->publish_stats_to_osd(); } @@ -978,6 +979,7 @@ void PgScrubber::on_init() m_start = m_pg->info.pgid.pgid.get_hobj_start(); m_active = true; + m_pg->publish_stats_to_osd(); } void PgScrubber::on_replica_init() @@ -1923,42 +1925,127 @@ void PgScrubber::on_digest_updates() * is cleared once scrubbing starts; Some of the values dumped here are * thus transitory. */ -void PgScrubber::dump(ceph::Formatter* f) const +void PgScrubber::dump_scrubber(ceph::Formatter* f, + const requested_scrub_t& request_flags) const { f->open_object_section("scrubber"); + + if (m_active) { // TBD replace with PR#42780's test + f->dump_bool("active", true); + dump_active_scrubber(f, state_test(PG_STATE_DEEP_SCRUB)); + } else { + f->dump_bool("active", false); + f->dump_bool("must_scrub", + (m_pg->m_planned_scrub.must_scrub || m_flags.required)); + f->dump_bool("must_deep_scrub", request_flags.must_deep_scrub); + f->dump_bool("must_repair", request_flags.must_repair); + f->dump_bool("need_auto", request_flags.need_auto); + + f->dump_stream("scrub_reg_stamp") << m_scrub_job->get_sched_time(); + + // note that we are repeating logic that is coded elsewhere (currently PG.cc). + // This is not optimal. + bool deep_expected = (ceph_clock_now() >= m_pg->next_deepscrub_interval()) || + request_flags.must_deep_scrub || request_flags.need_auto; + auto sched_state = + m_scrub_job->scheduling_state(ceph_clock_now(), deep_expected); + f->dump_string("schedule", sched_state); + } + + f->close_section(); +} + +void PgScrubber::dump_active_scrubber(ceph::Formatter* f, bool is_deep) const +{ f->dump_stream("epoch_start") << m_interval_start; - f->dump_bool("active", m_active); - if (m_active) { - f->dump_stream("start") << m_start; - f->dump_stream("end") << m_end; - f->dump_stream("m_max_end") << m_max_end; - f->dump_stream("subset_last_update") << m_subset_last_update; - f->dump_bool("deep", m_is_deep); - f->dump_bool("must_scrub", (m_pg->m_planned_scrub.must_scrub || m_flags.required)); - f->dump_bool("must_deep_scrub", m_pg->m_planned_scrub.must_deep_scrub); - f->dump_bool("must_repair", m_pg->m_planned_scrub.must_repair); - f->dump_bool("need_auto", m_pg->m_planned_scrub.need_auto); - f->dump_bool("req_scrub", m_flags.required); - f->dump_bool("time_for_deep", m_pg->m_planned_scrub.time_for_deep); - f->dump_bool("auto_repair", m_flags.auto_repair); - f->dump_bool("check_repair", m_flags.check_repair); - f->dump_bool("deep_scrub_on_error", m_flags.deep_scrub_on_error); - f->dump_stream("scrub_reg_stamp") << m_scrub_job->get_sched_time(); // utime_t - f->dump_unsigned("priority", m_flags.priority); - f->dump_int("shallow_errors", m_shallow_errors); - f->dump_int("deep_errors", m_deep_errors); - f->dump_int("fixed", m_fixed_count); - { - f->open_array_section("waiting_on_whom"); - for (const auto& p : m_maps_status.get_awaited()) { - f->dump_stream("shard") << p; - } - f->close_section(); + f->dump_stream("start") << m_start; + f->dump_stream("end") << m_end; + f->dump_stream("max_end") << m_max_end; + f->dump_stream("subset_last_update") << m_subset_last_update; + // note that m_is_deep will be set some time after PG_STATE_DEEP_SCRUB is + // asserted. Thus, using the latter. + f->dump_bool("deep", is_deep); + + // dump the scrub-type flags + f->dump_bool("req_scrub", m_flags.required); + f->dump_bool("auto_repair", m_flags.auto_repair); + f->dump_bool("check_repair", m_flags.check_repair); + f->dump_bool("deep_scrub_on_error", m_flags.deep_scrub_on_error); + f->dump_unsigned("priority", m_flags.priority); + + f->dump_int("shallow_errors", m_shallow_errors); + f->dump_int("deep_errors", m_deep_errors); + f->dump_int("fixed", m_fixed_count); + { + f->open_array_section("waiting_on_whom"); + for (const auto& p : m_maps_status.get_awaited()) { + f->dump_stream("shard") << p; } + f->close_section(); } - f->close_section(); + f->dump_string("schedule", "scrubbing"); } +pg_scrubbing_status_t PgScrubber::get_schedule() const +{ + dout(25) << __func__ << dendl; + + if (!m_scrub_job) { + return pg_scrubbing_status_t{}; + } + + auto now_is = ceph_clock_now(); + + if (m_active) { + // report current scrub info, including updated duration + int32_t duration = (utime_t{now_is} - scrub_begin_stamp).sec(); + + return pg_scrubbing_status_t{ + utime_t{}, + duration, + pg_scrub_sched_status_t::active, + true, // active + (m_is_deep ? scrub_level_t::deep : scrub_level_t::shallow), + false /* is periodic? unknown, actually */}; + } + if (m_scrub_job->state != ScrubQueue::qu_state_t::registered) { + return pg_scrubbing_status_t{utime_t{}, + 0, + pg_scrub_sched_status_t::not_queued, + false, + scrub_level_t::shallow, + false}; + } + + // Will next scrub surely be a deep one? note that deep-scrub might be + // selected even if we report a regular scrub here. + bool deep_expected = (now_is >= m_pg->next_deepscrub_interval()) || + m_pg->m_planned_scrub.must_deep_scrub || + m_pg->m_planned_scrub.need_auto; + scrub_level_t expected_level = + deep_expected ? scrub_level_t::deep : scrub_level_t::shallow; + bool periodic = !m_pg->m_planned_scrub.must_scrub && + !m_pg->m_planned_scrub.need_auto && + !m_pg->m_planned_scrub.must_deep_scrub; + + // are we ripe for scrubbing? + if (now_is > m_scrub_job->schedule.scheduled_at) { + // we are waiting for our turn at the OSD. + return pg_scrubbing_status_t{m_scrub_job->schedule.scheduled_at, + 0, + pg_scrub_sched_status_t::queued, + false, + expected_level, + periodic}; + } + + return pg_scrubbing_status_t{m_scrub_job->schedule.scheduled_at, + 0, + pg_scrub_sched_status_t::scheduled, + false, + expected_level, + periodic}; +} void PgScrubber::handle_query_state(ceph::Formatter* f) { @@ -1969,8 +2056,8 @@ void PgScrubber::handle_query_state(ceph::Formatter* f) f->dump_bool("scrubber.active", m_active); f->dump_stream("scrubber.start") << m_start; f->dump_stream("scrubber.end") << m_end; - f->dump_stream("scrubber.m_max_end") << m_max_end; - f->dump_stream("scrubber.m_subset_last_update") << m_subset_last_update; + f->dump_stream("scrubber.max_end") << m_max_end; + f->dump_stream("scrubber.subset_last_update") << m_subset_last_update; f->dump_bool("scrubber.deep", m_is_deep); { f->open_array_section("scrubber.waiting_on_whom"); @@ -2012,14 +2099,14 @@ void PgScrubber::set_scrub_begin_time() { scrub_begin_stamp = ceph_clock_now(); } -void PgScrubber::set_scrub_duration() { - utime_t stamp = ceph_clock_now(); - utime_t duration = stamp - scrub_begin_stamp; - m_pg->recovery_state.update_stats( - [=](auto &history, auto &stats) { - stats.scrub_duration = double(duration); - return true; - }); +void PgScrubber::set_scrub_duration() +{ + utime_t stamp = ceph_clock_now(); + utime_t duration = stamp - scrub_begin_stamp; + m_pg->recovery_state.update_stats([=](auto& history, auto& stats) { + stats.last_scrub_duration = ceill(duration.to_msec()/1000.0); + return true; + }); } void PgScrubber::reserve_replicas() @@ -2091,8 +2178,6 @@ void PgScrubber::replica_handling_done() state_clear(PG_STATE_DEEP_SCRUB); reset_internal_state(); - - m_pg->publish_stats_to_osd(); } /* diff --git a/src/osd/scrubber/pg_scrubber.h b/src/osd/scrubber/pg_scrubber.h index a9845efb1dd5c..895ff1232daa8 100644 --- a/src/osd/scrubber/pg_scrubber.h +++ b/src/osd/scrubber/pg_scrubber.h @@ -286,7 +286,10 @@ class PgScrubber : public ScrubPgIF, public ScrubMachineListener { void handle_query_state(ceph::Formatter* f) final; - void dump(ceph::Formatter* f) const override; + pg_scrubbing_status_t get_schedule() const final; + + void dump_scrubber(ceph::Formatter* f, + const requested_scrub_t& request_flags) const final; // used if we are a replica @@ -481,6 +484,9 @@ class PgScrubber : public ScrubPgIF, public ScrubMachineListener { void run_callbacks(); + // 'query' command data for an active scrub + void dump_active_scrubber(ceph::Formatter* f, bool is_deep) const; + // ----- methods used to verify the relevance of incoming events: /** diff --git a/src/osd/scrubber_common.h b/src/osd/scrubber_common.h index 2f4376a8b7bfd..12302dad2abb2 100644 --- a/src/osd/scrubber_common.h +++ b/src/osd/scrubber_common.h @@ -192,7 +192,10 @@ struct ScrubPgIF { virtual void handle_query_state(ceph::Formatter* f) = 0; - virtual void dump(ceph::Formatter* f) const = 0; + virtual pg_scrubbing_status_t get_schedule() const = 0; + + virtual void dump_scrubber(ceph::Formatter* f, + const requested_scrub_t& request_flags) const = 0; /** * Return true if soid is currently being scrubbed and pending IOs should block. From 60842eb1a96fee96104c89ee801118bff823169b Mon Sep 17 00:00:00 2001 From: Ronen Friedman Date: Mon, 1 Nov 2021 12:08:57 +0000 Subject: [PATCH 0536/2866] osd/scrub: scrubbing schedule - minor related cleanups Signed-off-by: Ronen Friedman --- .gitignore | 1 + src/mon/PGMap.cc | 18 +++++++++--------- src/mon/PGMap.h | 4 ++-- src/osd/OSD.cc | 1 + src/osd/PeeringState.cc | 3 +-- src/osd/PrimaryLogPG.cc | 4 ++-- src/osd/PrimaryLogPG.h | 2 +- src/osd/osd_types.cc | 3 ++- src/osd/osd_types.h | 2 +- src/osd/scrubber/PrimaryLogScrub.cc | 1 - src/osd/scrubber/pg_scrubber.cc | 13 +++++++------ src/osd/scrubber/pg_scrubber.h | 6 +++--- 12 files changed, 30 insertions(+), 28 deletions(-) diff --git a/.gitignore b/.gitignore index 4c9ff3b26deb6..b01aef839bef6 100644 --- a/.gitignore +++ b/.gitignore @@ -82,3 +82,4 @@ GTAGS # Python building things where it shouldn't /src/python-common/build/ +.cache diff --git a/src/mon/PGMap.cc b/src/mon/PGMap.cc index d773c2fd3861f..ea90270a9ef8f 100644 --- a/src/mon/PGMap.cc +++ b/src/mon/PGMap.cc @@ -1608,7 +1608,7 @@ void PGMap::dump_osd_stats(ceph::Formatter *f, bool with_net) const void PGMap::dump_osd_ping_times(ceph::Formatter *f) const { f->open_array_section("osd_ping_times"); - for (auto& [osd, stat] : osd_stat) { + for (const auto& [osd, stat] : osd_stat) { f->open_object_section("osd_ping_time"); f->dump_int("osd", osd); stat.dump_ping_time(f); @@ -1617,10 +1617,11 @@ void PGMap::dump_osd_ping_times(ceph::Formatter *f) const f->close_section(); } +// note: dump_pg_stats_plain() is static void PGMap::dump_pg_stats_plain( ostream& ss, const mempool::pgmap::unordered_map& pg_stats, - bool brief) const + bool brief) { TextTable tab; @@ -1661,11 +1662,9 @@ void PGMap::dump_pg_stats_plain( tab.define_column("SCRUB_SCHEDULING", TextTable::LEFT, TextTable::LEFT); } - for (auto i = pg_stats.begin(); - i != pg_stats.end(); ++i) { - const pg_stat_t &st(i->second); + for (const auto& [pg, st] : pg_stats) { if (brief) { - tab << i->first + tab << pg << pg_state_string(st.state) << st.up << st.up_primary @@ -1676,7 +1675,7 @@ void PGMap::dump_pg_stats_plain( ostringstream reported; reported << st.reported_epoch << ":" << st.reported_seq; - tab << i->first + tab << pg << st.stats.sum.num_objects << st.stats.sum.num_objects_missing_on_primary << st.stats.sum.num_objects_degraded @@ -1700,7 +1699,8 @@ void PGMap::dump_pg_stats_plain( << st.last_deep_scrub << st.last_deep_scrub_stamp << st.snaptrimq_len - << st.scrub_duration + << st.last_scrub_duration + << st.dump_scrub_schedule() << TextTable::endrow; } } @@ -2271,7 +2271,7 @@ void PGMap::dump_filtered_pg_stats(ostream& ss, set& pgs) const void PGMap::dump_pool_stats_and_io_rate(int64_t poolid, const OSDMap &osd_map, ceph::Formatter *f, stringstream *rs) const { - string pool_name = osd_map.get_pool_name(poolid); + const string& pool_name = osd_map.get_pool_name(poolid); if (f) { f->open_object_section("pool"); f->dump_string("pool_name", pool_name.c_str()); diff --git a/src/mon/PGMap.h b/src/mon/PGMap.h index c592f338ecb33..d55d2397c1b98 100644 --- a/src/mon/PGMap.h +++ b/src/mon/PGMap.h @@ -460,10 +460,10 @@ class PGMap : public PGMapDigest { void dump_pool_stats_and_io_rate(int64_t poolid, const OSDMap &osd_map, ceph::Formatter *f, std::stringstream *ss) const; - void dump_pg_stats_plain( + static void dump_pg_stats_plain( std::ostream& ss, const mempool::pgmap::unordered_map& pg_stats, - bool brief) const; + bool brief); void get_stuck_stats( int types, const utime_t cutoff, mempool::pgmap::unordered_map& stuck_pgs) const; diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc index 67664389c879b..ca1dc472b2197 100644 --- a/src/osd/OSD.cc +++ b/src/osd/OSD.cc @@ -7592,6 +7592,7 @@ void OSD::resched_all_scrubs() MPGStats* OSD::collect_pg_stats() { + dout(15) << __func__ << dendl; // This implementation unconditionally sends every is_primary PG's // stats every time we're called. This has equivalent cost to the // previous implementation's worst case where all PGs are busy and diff --git a/src/osd/PeeringState.cc b/src/osd/PeeringState.cc index ae9344ec4ab49..b5e6b69573e4c 100644 --- a/src/osd/PeeringState.cc +++ b/src/osd/PeeringState.cc @@ -3810,7 +3810,7 @@ std::optional PeeringState::prepare_stats_for_publish( if (info.stats.state != state) { info.stats.last_change = now; // Optimistic estimation, if we just find out an inactive PG, - // assumt it is active till now. + // assume it is active till now. if (!(state & PG_STATE_ACTIVE) && (info.stats.state & PG_STATE_ACTIVE)) info.stats.last_active = now; @@ -3990,7 +3990,6 @@ void PeeringState::update_stats_wo_resched( f(info.history, info.stats); } - bool PeeringState::append_log_entries_update_missing( const mempool::osd_pglog::list &entries, ObjectStore::Transaction &t, std::optional trim_to, diff --git a/src/osd/PrimaryLogPG.cc b/src/osd/PrimaryLogPG.cc index 6c2343ba34f4f..298716a1a0c71 100644 --- a/src/osd/PrimaryLogPG.cc +++ b/src/osd/PrimaryLogPG.cc @@ -947,7 +947,7 @@ PrimaryLogPG::get_pgls_filter(bufferlist::const_iterator& iter) if (type.compare("plain") == 0) { filter = std::make_unique(); } else { - std::size_t dot = type.find("."); + std::size_t dot = type.find('.'); if (dot == std::string::npos || dot == 0 || dot == type.size() - 1) { return { -EINVAL, nullptr }; } @@ -1179,7 +1179,7 @@ void PrimaryLogPG::do_command( if (deep) { set_last_deep_scrub_stamp(stamp); } else { - set_last_scrub_stamp(stamp); + set_last_scrub_stamp(stamp); // also for 'deep', as we use this value to order scrubs } f->open_object_section("result"); f->dump_bool("deep", deep); diff --git a/src/osd/PrimaryLogPG.h b/src/osd/PrimaryLogPG.h index 23ecdffda6396..b92c46bf4a926 100644 --- a/src/osd/PrimaryLogPG.h +++ b/src/osd/PrimaryLogPG.h @@ -196,7 +196,7 @@ class PrimaryLogPG : public PG, public PGBackend::Listener { friend struct CopyFromFinisher; friend class PromoteCallback; friend struct PromoteFinisher; - friend class C_gather; + friend struct C_gather; struct ProxyReadOp { OpRequestRef op; diff --git a/src/osd/osd_types.cc b/src/osd/osd_types.cc index 9e260de9426ea..d439cdb2e8d71 100644 --- a/src/osd/osd_types.cc +++ b/src/osd/osd_types.cc @@ -6397,8 +6397,9 @@ void object_info_t::dump(Formatter *f) const f->dump_unsigned("lost", (int)is_lost()); vector sv = get_flag_vector(flags); f->open_array_section("flags"); - for (auto str: sv) + for (const auto& str: sv) { f->dump_string("flags", str); + } f->close_section(); f->dump_unsigned("truncate_seq", truncate_seq); f->dump_unsigned("truncate_size", truncate_size); diff --git a/src/osd/osd_types.h b/src/osd/osd_types.h index 1bb384548d6d6..65c0cf1f4109d 100644 --- a/src/osd/osd_types.h +++ b/src/osd/osd_types.h @@ -2259,7 +2259,7 @@ struct pg_stat_t { int32_t acting_primary; // snaptrimq.size() is 64bit, but let's be serious - anything over 50k is - // absurd already, so cap it to 2^32 and save 4 bytes at the same time + // absurd already, so cap it to 2^32 and save 4 bytes at the same time uint32_t snaptrimq_len; pg_scrubbing_status_t scrub_sched_status; diff --git a/src/osd/scrubber/PrimaryLogScrub.cc b/src/osd/scrubber/PrimaryLogScrub.cc index 2be7b56f61b03..06dce06f5457f 100644 --- a/src/osd/scrubber/PrimaryLogScrub.cc +++ b/src/osd/scrubber/PrimaryLogScrub.cc @@ -23,7 +23,6 @@ template static ostream& _prefix(std::ostream* _dout, T* t) } using namespace Scrub; -using Scrub::ScrubMachine; bool PrimaryLogScrub::get_store_errors(const scrub_ls_arg_t& arg, scrub_ls_result_t& res_inout) const diff --git a/src/osd/scrubber/pg_scrubber.cc b/src/osd/scrubber/pg_scrubber.cc index 9e3256163e901..038d9392c6271 100644 --- a/src/osd/scrubber/pg_scrubber.cc +++ b/src/osd/scrubber/pg_scrubber.cc @@ -3,6 +3,7 @@ #include "./pg_scrubber.h" // the '.' notation used to affect clang-format order +#include #include #include @@ -520,7 +521,7 @@ void PgScrubber::update_scrub_job(const requested_scrub_t& request_flags) } ScrubQueue::sched_params_t -PgScrubber::determine_scrub_time(const requested_scrub_t& request_flags) +PgScrubber::determine_scrub_time(const requested_scrub_t& request_flags) const { ScrubQueue::sched_params_t res; @@ -795,7 +796,7 @@ void PgScrubber::add_delayed_scheduling() if (m_needs_sleep) { double scrub_sleep = 1000.0 * m_osds->get_scrub_services().scrub_sleep_time(m_flags.required); - sleep_time = milliseconds{long(scrub_sleep)}; + sleep_time = milliseconds{int64_t(scrub_sleep)}; } dout(15) << __func__ << " sleep: " << sleep_time.count() << "ms. needed? " << m_needs_sleep << dendl; @@ -1348,7 +1349,7 @@ void PgScrubber::set_op_parameters(requested_scrub_t& request) // not calling update_op_mode_text() yet, as m_is_deep not set yet } - // the publishing here seems to be required for tests synchronization + // the publishing here is required for tests synchronization m_pg->publish_stats_to_osd(); m_flags.deep_scrub_on_error = request.deep_scrub_on_error; } @@ -1919,7 +1920,6 @@ void PgScrubber::on_digest_updates() } } - /* * note that the flags-set fetched from the PG (m_pg->m_planned_scrub) * is cleared once scrubbing starts; Some of the values dumped here are @@ -2049,7 +2049,7 @@ pg_scrubbing_status_t PgScrubber::get_schedule() const void PgScrubber::handle_query_state(ceph::Formatter* f) { - dout(10) << __func__ << dendl; + dout(15) << __func__ << dendl; f->open_object_section("scrub"); f->dump_stream("scrubber.epoch_start") << m_interval_start; @@ -2095,7 +2095,8 @@ PgScrubber::PgScrubber(PG* pg) m_osds->get_nodeid()); } -void PgScrubber::set_scrub_begin_time() { +void PgScrubber::set_scrub_begin_time() +{ scrub_begin_stamp = ceph_clock_now(); } diff --git a/src/osd/scrubber/pg_scrubber.h b/src/osd/scrubber/pg_scrubber.h index 895ff1232daa8..8da41a0235373 100644 --- a/src/osd/scrubber/pg_scrubber.h +++ b/src/osd/scrubber/pg_scrubber.h @@ -671,8 +671,8 @@ class PgScrubber : public ScrubPgIF, public ScrubMachineListener { */ void request_rescrubbing(requested_scrub_t& req_flags); - ScrubQueue::sched_params_t - determine_scrub_time(const requested_scrub_t& request_flags); + ScrubQueue::sched_params_t determine_scrub_time( + const requested_scrub_t& request_flags) const; void unregister_from_osd(); @@ -745,7 +745,7 @@ class PgScrubber : public ScrubPgIF, public ScrubMachineListener { */ class preemption_data_t : public Scrub::preemption_t { public: - preemption_data_t(PG* pg); // the PG access is used for conf access (and logs) + explicit preemption_data_t(PG* pg); // the PG access is used for conf access (and logs) [[nodiscard]] bool is_preemptable() const final { return m_preemptable; } From 10909c3cba62016c9800d44a382e67aaf9a0abd0 Mon Sep 17 00:00:00 2001 From: Ronen Friedman Date: Mon, 25 Oct 2021 16:03:05 +0000 Subject: [PATCH 0537/2866] osd/scrub: update the stand-alone tests to check 'scrub scheduling' entries Analyzing and verifying the relevant entries in 'pg query' and 'pg dump' output. Signed-off-by: Ronen Friedman --- qa/standalone/scrub/osd-scrub-test.sh | 114 ++++++++++++--- qa/standalone/scrub/scrub-helpers.sh | 196 ++++++++++++++++++++++++++ 2 files changed, 287 insertions(+), 23 deletions(-) create mode 100644 qa/standalone/scrub/scrub-helpers.sh diff --git a/qa/standalone/scrub/osd-scrub-test.sh b/qa/standalone/scrub/osd-scrub-test.sh index 9105474e30f21..6a1132ef22555 100755 --- a/qa/standalone/scrub/osd-scrub-test.sh +++ b/qa/standalone/scrub/osd-scrub-test.sh @@ -15,6 +15,7 @@ # GNU Library Public License for more details. # source $CEPH_ROOT/qa/standalone/ceph-helpers.sh +source $CEPH_ROOT/qa/standalone/scrub/scrub-helpers.sh function run() { local dir=$1 @@ -28,7 +29,9 @@ function run() { export -n CEPH_CLI_TEST_DUP_COMMAND local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')} for func in $funcs ; do + setup $dir || return 1 $func $dir || return 1 + teardown $dir || return 1 done } @@ -40,7 +43,6 @@ function TEST_scrub_test() { TESTDATA="testdata.$$" - setup $dir || return 1 run_mon $dir a --osd_pool_default_size=3 || return 1 run_mgr $dir x || return 1 for osd in $(seq 0 $(expr $OSDS - 1)) @@ -103,8 +105,6 @@ function TEST_scrub_test() { test "$(ceph pg $pgid query | jq '.peer_info[0].stats.stat_sum.num_scrub_errors')" = "0" || return 1 test "$(ceph pg $pgid query | jq '.peer_info[1].stats.stat_sum.num_scrub_errors')" = "0" || return 1 ceph pg dump pgs | grep ^${pgid} | grep -vq -- +inconsistent || return 1 - - teardown $dir || return 1 } # Grab year-month-day @@ -138,7 +138,6 @@ function TEST_interval_changes() { TESTDATA="testdata.$$" - setup $dir || return 1 # This min scrub interval results in 30 seconds backoff time run_mon $dir a --osd_pool_default_size=$OSDS || return 1 run_mgr $dir x || return 1 @@ -183,8 +182,6 @@ function TEST_interval_changes() { ceph osd pool set $poolname scrub_max_interval $(expr $week \* 3) sleep $WAIT_FOR_UPDATE check_dump_scrubs $primary "3 days" "3 week" || return 1 - - teardown $dir || return 1 } function TEST_scrub_extended_sleep() { @@ -205,7 +202,6 @@ function TEST_scrub_extended_sleep() { DAY_START=$(expr $DAY + 2) DAY_END=$(expr $DAY + 3) - setup $dir || return 1 run_mon $dir a --osd_pool_default_size=3 || return 1 run_mgr $dir x || return 1 for osd in $(seq 0 $(expr $OSDS - 1)) @@ -274,8 +270,6 @@ function TEST_scrub_extended_sleep() { then return 1 fi - - teardown $dir || return 1 } function _scrub_abort() { @@ -295,8 +289,6 @@ function _scrub_abort() { check="nodeep_scrub" fi - - setup $dir || return 1 run_mon $dir a --osd_pool_default_size=3 || return 1 run_mgr $dir x || return 1 for osd in $(seq 0 $(expr $OSDS - 1)) @@ -393,8 +385,6 @@ function _scrub_abort() { fi TIMEOUT=$(($objects / 2)) wait_for_scrub $pgid "$last_scrub" || return 1 - - teardown $dir || return 1 } function TEST_scrub_abort() { @@ -415,7 +405,6 @@ function TEST_scrub_permit_time() { TESTDATA="testdata.$$" - setup $dir || return 1 run_mon $dir a --osd_pool_default_size=3 || return 1 run_mgr $dir x || return 1 local scrub_begin_hour=$(date -d '2 hour ago' +"%H" | sed 's/^0//') @@ -450,11 +439,10 @@ function TEST_scrub_permit_time() { fi sleep 1 done - - teardown $dir || return 1 } -function TEST_pg_dump_scrub_duration() { + +function TEST_dump_scrub_schedule() { local dir=$1 local poolname=test local OSDS=3 @@ -462,12 +450,21 @@ function TEST_pg_dump_scrub_duration() { TESTDATA="testdata.$$" - setup $dir || return 1 run_mon $dir a --osd_pool_default_size=$OSDS || return 1 run_mgr $dir x || return 1 + + # Set scheduler to "wpq" until there's a reliable way to query scrub states + # with "--osd-scrub-sleep" set to 0. The "mclock_scheduler" overrides the + # scrub sleep to 0 and as a result the checks in the test fail. + local ceph_osd_args="--osd_deep_scrub_randomize_ratio=0 \ + --osd_scrub_interval_randomize_ratio=0 \ + --osd_scrub_backoff_ratio=0.0 \ + --osd_op_queue=wpq \ + --osd_scrub_sleep=0.2" + for osd in $(seq 0 $(expr $OSDS - 1)) do - run_osd $dir $osd || return 1 + run_osd $dir $osd $ceph_osd_args|| return 1 done # Create a pool with a single pg @@ -483,12 +480,83 @@ function TEST_pg_dump_scrub_duration() { rm -f $TESTDATA local pgid="${poolid}.0" - pg_scrub $pgid || return 1 + local now_is=`date -I"ns"` + + # before the scrubbing starts + + # last scrub duration should be 0. The scheduling data should show + # a time in the future: + # e.g. 'periodic scrub scheduled @ 2021-10-12T20:32:43.645168+0000' - ceph pg $pgid query | jq '.info.stats.scrub_duration' - test "$(ceph pg $pgid query | jq '.info.stats.scrub_duration')" '>' "0" || return 1 + declare -A expct_starting=( ['query_active']="false" ['query_is_future']="true" ['query_schedule']="scrub scheduled" ) + declare -A sched_data + extract_published_sch $pgid $now_is "2019-10-12T20:32:43.645168+0000" sched_data + schedule_against_expected sched_data expct_starting "initial" + (( ${sched_data['dmp_last_duration']} == 0)) || return 1 + echo "last-scrub --- " ${sched_data['query_last_scrub']} - teardown $dir || return 1 + # + # step 1: scrub once (mainly to ensure there is no urgency to scrub) + # + + saved_last_stamp=${sched_data['query_last_stamp']} + ceph tell osd.* config set osd_scrub_sleep "0" + ceph pg deep-scrub $pgid + ceph pg scrub $pgid + + # wait for the 'last duration' entries to change. Note that the 'dump' one will need + # up to 5 seconds to sync + + sleep 5 + sched_data=() + declare -A expct_qry_duration=( ['query_last_duration']="0" ['query_last_duration_neg']="not0" ) + wait_any_cond $pgid 10 $saved_last_stamp expct_qry_duration "WaitingAfterScrub " sched_data || return 1 + # verify that 'pg dump' also shows the change in last_scrub_duration + sched_data=() + declare -A expct_dmp_duration=( ['dmp_last_duration']="0" ['dmp_last_duration_neg']="not0" ) + wait_any_cond $pgid 10 $saved_last_stamp expct_dmp_duration "WaitingAfterScrub_dmp " sched_data || return 1 + + sleep 2 + + # + # step 2: set noscrub and request a "periodic scrub". Watch for the change in the 'is the scrub + # scheduled for the future' value + # + + ceph tell osd.* config set osd_scrub_chunk_max "3" || return 1 + ceph tell osd.* config set osd_scrub_sleep "1.0" || return 1 + ceph osd set noscrub || return 1 + sleep 2 + saved_last_stamp=${sched_data['query_last_stamp']} + + ceph pg $pgid scrub + sleep 1 + sched_data=() + declare -A expct_scrub_peri_sched=( ['query_is_future']="false" ) + wait_any_cond $pgid 10 $saved_last_stamp expct_scrub_peri_sched "waitingBeingScheduled" sched_data || return 1 + + # note: the induced change in 'last_scrub_stamp' that we've caused above, is by itself not a publish-stats + # trigger. Thus it might happen that the information in 'pg dump' will not get updated here. Do not expect + # 'dmp_is_future' to follow 'query_is_future' without a good reason + ## declare -A expct_scrub_peri_sched_dmp=( ['dmp_is_future']="false" ) + ## wait_any_cond $pgid 15 $saved_last_stamp expct_scrub_peri_sched_dmp "waitingBeingScheduled" sched_data || echo "must be fixed" + + # + # step 3: allow scrubs. Watch for the conditions during the scrubbing + # + + saved_last_stamp=${sched_data['query_last_stamp']} + ceph osd unset noscrub + + declare -A cond_active=( ['query_active']="true" ) + sched_data=() + wait_any_cond $pgid 10 $saved_last_stamp cond_active "WaitingActive " sched_data || return 1 + + # check for pg-dump to show being active. But if we see 'query_active' being reset - we've just + # missed it. + declare -A cond_active_dmp=( ['dmp_state_has_scrubbing']="true" ['query_active']="false" ) + sched_data=() + wait_any_cond $pgid 10 $saved_last_stamp cond_active_dmp "WaitingActive " sched_data || return 1 } main osd-scrub-test "$@" diff --git a/qa/standalone/scrub/scrub-helpers.sh b/qa/standalone/scrub/scrub-helpers.sh new file mode 100644 index 0000000000000..cf13732c53854 --- /dev/null +++ b/qa/standalone/scrub/scrub-helpers.sh @@ -0,0 +1,196 @@ +#!/usr/bin/env bash +# @file scrub-helpers.sh +# @brief a collection of bash functions useful for scrub standalone tests +# + +# extract_published_sch() +# +# Use the output from both 'ceph pg dump pgs' and 'ceph pg x.x query' commands to determine +# the published scrub scheduling status of a given PG. +# +# $1: pg id +# $2: 'current' time to compare to +# $3: an additional time-point to compare to +# $4: [out] dictionary +# +function extract_published_sch() { + local pgn="$1" + local -n dict=$4 # a ref to the in/out dictionary + local current_time=$2 + local extra_time=$3 + local extr_dbg=1 # note: 3 and above leave some temp files around + + #turn off '-x' (but remember previous state) + local saved_echo_flag=${-//[^x]/} + set +x + + (( extr_dbg >= 3 )) && ceph pg dump pgs -f json-pretty >> /tmp/a_dmp$$ + (( extr_dbg >= 3 )) && ceph pg $1 query -f json-pretty >> /tmp/a_qry$$ + + from_dmp=`ceph pg dump pgs -f json-pretty | jq -r --arg pgn "$pgn" --arg extra_dt "$extra_time" --arg current_dt "$current_time" '[ + [[.pg_stats[]] | group_by(.pg_stats)][0][0] | + [.[] | + select(has("pgid") and .pgid == $pgn) | + + (.dmp_stat_part=(.scrub_schedule | if test(".*@.*") then (split(" @ ")|first) else . end)) | + (.dmp_when_part=(.scrub_schedule | if test(".*@.*") then (split(" @ ")|last) else "0" end)) | + + [ { + dmp_pg_state: .state, + dmp_state_has_scrubbing: (.state | test(".*scrub.*";"i")), + dmp_last_duration:.last_scrub_duration, + dmp_schedule: .dmp_stat_part, + dmp_schedule_at: .dmp_when_part, + dmp_is_future: ( .dmp_when_part > $current_dt ), + dmp_vs_date: ( .dmp_when_part > $extra_dt ), + dmp_reported_epoch: .reported_epoch, + dmp_seq: .reported_seq + }] ]][][][]'` + + (( extr_dbg >= 2 )) && echo "from pg dump pg: $from_dmp" + (( extr_dbg >= 2 )) && echo "query output:" + (( extr_dbg >= 2 )) && ceph pg $1 query -f json-pretty | awk -e '/scrubber/,/agent_state/ {print;}' + + from_qry=`ceph pg $1 query -f json-pretty | jq -r --arg extra_dt "$extra_time" --arg current_dt "$current_time" --arg spt "'" ' + . | + (.q_stat_part=((.scrubber.schedule// "-") | if test(".*@.*") then (split(" @ ")|first) else . end)) | + (.q_when_part=((.scrubber.schedule// "0") | if test(".*@.*") then (split(" @ ")|last) else "0" end)) | + (.q_when_is_future=(.q_when_part > $current_dt)) | + (.q_vs_date=(.q_when_part > $extra_dt)) | + { + query_epoch: .epoch, + query_seq: .info.stats.reported_seq, + query_active: (.scrubber | if has("active") then .active else "bug" end), + query_schedule: .q_stat_part, + query_schedule_at: .q_when_part, + query_last_duration: .info.stats.last_scrub_duration, + query_last_stamp: .info.history.last_scrub_stamp, + query_last_scrub: (.info.history.last_scrub| sub($spt;"x") ), + query_is_future: .q_when_is_future, + query_vs_date: .q_vs_date, + + } + '` + (( extr_dbg >= 1 )) && echo $from_qry " " $from_dmp | jq -s -r 'add | "(",(to_entries | .[] | "["+(.key)+"]="+(.value|@sh)),")"' + + # note that using a ref to an associative array directly is tricky. Instead - we are copying: + local -A dict_src=`echo $from_qry " " $from_dmp | jq -s -r 'add | "(",(to_entries | .[] | "["+(.key)+"]="+(.value|@sh)),")"'` + dict=() + for k in "${!dict_src[@]}"; do dict[$k]=${dict_src[$k]}; done + + if [[ -n "$saved_echo_flag" ]]; then set -x; fi +} + +# query the PG, until any of the conditions in the 'expected' array are met +# +# A condition may be negated by an additional entry in the 'expected' array. Its +# form should be: +# key: the original key, with a "_neg" suffix; +# Value: not checked +# +# $1: pg id +# $2: max retries +# $3: a date to use in comparisons +# $4: set of K/V conditions +# $5: debug message +# $6: [out] the results array +function wait_any_cond() { + local pgid="$1" + local retries=$2 + local cmp_date=$3 + local -n ep=$4 + local -n out_array=$6 + local -A sc_data + local extr_dbg=2 + + #turn off '-x' (but remember previous state) + local saved_echo_flag=${-//[^x]/} + set +x + + local now_is=`date -I"ns"` + (( extr_dbg >= 2 )) && echo "waiting for any condition ($5): pg:$pgid dt:$cmp_date ($retries retries)" + + for i in $(seq 1 $retries) + do + sleep 0.5 + extract_published_sch $pgid $now_is $cmp_date sc_data + (( extr_dbg >= 4 )) && echo "${sc_data['dmp_last_duration']}" + (( extr_dbg >= 4 )) && echo "----> loop: $i ~ ${sc_data['dmp_last_duration']} / " ${sc_data['query_vs_date']} " / ${sc_data['dmp_is_future']}" + (( extr_dbg >= 2 )) && echo "--> loop: $i ~ ${sc_data['query_active']} / ${sc_data['query_seq']} / ${sc_data['dmp_seq']} " \ + "/ ${sc_data['query_is_future']} / ${sc_data['query_last_stamp']} / ${sc_data['query_schedule']} %%% ${!ep[@]}" + + # perform schedule_against_expected(), but with slightly different out-messages behaviour + for k_ref in "${!ep[@]}" + do + (( extr_dbg >= 3 )) && echo "key is $k_ref" + # is this a real key, or just a negation flag for another key?? + [[ $k_ref =~ "_neg" ]] && continue + + local act_val=${sc_data[$k_ref]} + local exp_val=${ep[$k_ref]} + + # possible negation? look for a matching key + local neg_key="${k_ref}_neg" + (( extr_dbg >= 3 )) && echo "neg-key is $neg_key" + if [ -v 'ep[$neg_key]' ]; then + is_neg=1 + else + is_neg=0 + fi + + (( extr_dbg >= 1 )) && echo "key is $k_ref: negation:$is_neg # expected: $exp_val # in actual: $act_val" + is_eq=0 + [[ $exp_val == $act_val ]] && is_eq=1 + if (($is_eq ^ $is_neg)) + then + echo "$5 - '$k_ref' actual value ($act_val) matches expected ($exp_val) (negation: $is_neg)" + for k in "${!sc_data[@]}"; do out_array[$k]=${sc_data[$k]}; done + if [[ -n "$saved_echo_flag" ]]; then set -x; fi + return 0 + fi + done + done + + echo "$5: wait_any_cond(): failure. Note: query-active=${sc_data['query_active']}" + if [[ -n "$saved_echo_flag" ]]; then set -x; fi + return 1 +} + + +# schedule_against_expected() +# +# Compare the scrub scheduling state collected by extract_published_sch() to a set of expected values. +# All values are expected to match. +# +# $1: the published scheduling state +# $2: a set of conditions to verify +# $3: text to be echoed for a failed match +# +function schedule_against_expected() { + local -n dict=$1 # a ref to the published state + local -n ep=$2 # the expected results + local extr_dbg=1 + + #turn off '-x' (but remember previous state) + local saved_echo_flag=${-//[^x]/} + set +x + + (( extr_dbg >= 1 )) && echo "-- - comparing:" + for k_ref in "${!ep[@]}" + do + local act_val=${dict[$k_ref]} + local exp_val=${ep[$k_ref]} + (( extr_dbg >= 1 )) && echo "key is " $k_ref " expected: " $exp_val " in actual: " $act_val + if [[ $exp_val != $act_val ]] + then + echo "$3 - '$k_ref' actual value ($act_val) differs from expected ($exp_val)" + echo '####################################################^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^' + + if [[ -n "$saved_echo_flag" ]]; then set -x; fi + return 1 + fi + done + + if [[ -n "$saved_echo_flag" ]]; then set -x; fi + return 0 +} From 5108647a55b06018d8d638bbd657a2bb2b528d7a Mon Sep 17 00:00:00 2001 From: Ronen Friedman Date: Thu, 4 Nov 2021 08:09:27 +0200 Subject: [PATCH 0538/2866] doc: document new scrub-related 'pg dump' columns Signed-off-by: Ronen Friedman --- PendingReleaseNotes | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/PendingReleaseNotes b/PendingReleaseNotes index c9a86da9f1718..afabe2b162b8a 100644 --- a/PendingReleaseNotes +++ b/PendingReleaseNotes @@ -73,6 +73,11 @@ https://docs.ceph.com/en/latest/rados/operations/placement-groups/ +* The ``ceph pg dump`` command now prints two additional columns: + `LAST_SCRUB_DURATION` shows the duration (in seconds) of the last completed scrub; + `SCRUB_SCHEDULING` conveys whether a PG is scheduled to be scrubbed at a specified + time, queued for scrubbing, or being scrubbed. + >=16.0.0 -------- * mgr/nfs: ``nfs`` module is moved out of volumes plugin. Prior using the From c1f8bdc0f0a4c7aed3bb60bb3c11622f270291dd Mon Sep 17 00:00:00 2001 From: Curt Bruns Date: Fri, 15 Oct 2021 19:42:25 -0400 Subject: [PATCH 0539/2866] os/bluestore: Set min_alloc_size to optimal io size Block devices may report an "optimal_io_size" that is different than the typical 4KiB. To optimize BlueStore for this io size, the allocator needs to set its min_alloc_size to this optimal_io_size. This PR adds the discovery of the optimal_io_size for a block device and an option to use the optimal_io_size as the min_alloc_size for the bluestore allocator. Older devices may report an optimal_io_size of 0 and if that is the case, the default config min_alloc_size is used. Signed-off-by: Curt Bruns --- src/blk/BlockDevice.h | 2 ++ src/blk/kernel/KernelDevice.cc | 2 ++ src/common/blkdev.cc | 5 +++++ src/common/blkdev.h | 1 + src/common/options/global.yaml.in | 10 ++++++++++ src/os/bluestore/BlueStore.cc | 21 ++++++++++++++++++++- src/os/bluestore/BlueStore.h | 1 + 7 files changed, 41 insertions(+), 1 deletion(-) diff --git a/src/blk/BlockDevice.h b/src/blk/BlockDevice.h index 01fa0d4451205..4eef1ef0df4cb 100644 --- a/src/blk/BlockDevice.h +++ b/src/blk/BlockDevice.h @@ -166,6 +166,7 @@ class BlockDevice { protected: uint64_t size = 0; uint64_t block_size = 0; + uint64_t optimal_io_size = 0; bool support_discard = false; bool rotational = true; bool lock_exclusive = true; @@ -224,6 +225,7 @@ class BlockDevice { uint64_t get_size() const { return size; } uint64_t get_block_size() const { return block_size; } + uint64_t get_optimal_io_size() const { return optimal_io_size; } /// hook to provide utilization of thinly-provisioned device virtual bool get_thin_utilization(uint64_t *total, uint64_t *avail) const { diff --git a/src/blk/kernel/KernelDevice.cc b/src/blk/kernel/KernelDevice.cc index f64f57c9cee36..ebd387aa17e9b 100644 --- a/src/blk/kernel/KernelDevice.cc +++ b/src/blk/kernel/KernelDevice.cc @@ -233,6 +233,7 @@ int KernelDevice::open(const string& p) dout(20) << __func__ << " devname " << devname << dendl; rotational = blkdev_buffered.is_rotational(); support_discard = blkdev_buffered.support_discard(); + optimal_io_size = blkdev_buffered.get_optimal_io_size(); this->devname = devname; _detect_vdo(); } @@ -320,6 +321,7 @@ int KernelDevice::collect_metadata(const string& prefix, map *pm) (*pm)[prefix + "rotational"] = stringify((int)(bool)rotational); (*pm)[prefix + "size"] = stringify(get_size()); (*pm)[prefix + "block_size"] = stringify(get_block_size()); + (*pm)[prefix + "optimal_io_size"] = stringify(get_optimal_io_size()); (*pm)[prefix + "driver"] = "KernelDevice"; if (rotational) { (*pm)[prefix + "type"] = "hdd"; diff --git a/src/common/blkdev.cc b/src/common/blkdev.cc index ea81bf0ae1e9e..e9d3cc1c381d7 100644 --- a/src/common/blkdev.cc +++ b/src/common/blkdev.cc @@ -217,6 +217,11 @@ int BlkDev::discard(int64_t offset, int64_t len) const return ioctl(fd, BLKDISCARD, range); } +int BlkDev::get_optimal_io_size() const +{ + return get_int_property("queue/optimal_io_size"); +} + bool BlkDev::is_rotational() const { return get_int_property("queue/rotational") > 0; diff --git a/src/common/blkdev.h b/src/common/blkdev.h index f8b089b0ca7c0..ed9da450e0f2f 100644 --- a/src/common/blkdev.h +++ b/src/common/blkdev.h @@ -56,6 +56,7 @@ class BlkDev { int partition(char* partition, size_t max) const; // from a device (e.g., "sdb") bool support_discard() const; + int get_optimal_io_size() const; bool is_rotational() const; int get_numa_node(int *node) const; int dev(char *dev, size_t max) const; diff --git a/src/common/options/global.yaml.in b/src/common/options/global.yaml.in index a0fcb61b5c314..04552bf2ea6cb 100644 --- a/src/common/options/global.yaml.in +++ b/src/common/options/global.yaml.in @@ -4287,6 +4287,16 @@ options: flags: - create with_legacy: true +- name: bluestore_use_optimal_io_size_for_min_alloc_size + type: bool + level: advanced + desc: Discover media optimal IO Size and use for min_alloc_size + default: false + see_also: + - bluestore_min_alloc_size + flags: + - create + with_legacy: true - name: bluestore_max_alloc_size type: size level: advanced diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc index 0a215eefefd80..045941b280fe8 100644 --- a/src/os/bluestore/BlueStore.cc +++ b/src/os/bluestore/BlueStore.cc @@ -5376,6 +5376,8 @@ int BlueStore::_open_bdev(bool create) if (r < 0) { goto fail_close; } + // get block dev optimal io size + optimal_io_size = bdev->get_optimal_io_size(); return 0; @@ -6859,7 +6861,14 @@ int BlueStore::mkfs() dout(10) << " freelist_type " << freelist_type << dendl; // choose min_alloc_size - if (cct->_conf->bluestore_min_alloc_size) { + dout(5) << __func__ << " optimal_io_size 0x" << std::hex << optimal_io_size + << " block_size: 0x" << block_size << std::dec << dendl; + if ((cct->_conf->bluestore_use_optimal_io_size_for_min_alloc_size) && (optimal_io_size != 0)) { + dout(5) << __func__ << " optimal_io_size 0x" << std::hex << optimal_io_size + << " for min_alloc_size 0x" << min_alloc_size << std::dec << dendl; + min_alloc_size = optimal_io_size; + } + else if (cct->_conf->bluestore_min_alloc_size) { min_alloc_size = cct->_conf->bluestore_min_alloc_size; } else { ceph_assert(bdev); @@ -6881,6 +6890,16 @@ int BlueStore::mkfs() goto out_close_bdev; } + // make sure min_alloc_size is >= and aligned with block size + if (min_alloc_size % block_size != 0) { + derr << __func__ << " min_alloc_size 0x" + << std::hex << min_alloc_size + << " is less or not aligned with block_size: 0x" + << block_size << std::dec << dendl; + r = -EINVAL; + goto out_close_bdev; + } + r = _create_alloc(); if (r < 0) { goto out_close_bdev; diff --git a/src/os/bluestore/BlueStore.h b/src/os/bluestore/BlueStore.h index 0a157edce1160..b2edf00e3620e 100644 --- a/src/os/bluestore/BlueStore.h +++ b/src/os/bluestore/BlueStore.h @@ -2124,6 +2124,7 @@ class BlueStore : public ObjectStore, uint64_t block_size = 0; ///< block size of block device (power of 2) uint64_t block_mask = 0; ///< mask to get just the block offset size_t block_size_order = 0; ///< bits to shift to get block size + uint64_t optimal_io_size = 0;///< best performance io size for block device uint64_t min_alloc_size; ///< minimum allocation unit (power of 2) ///< bits for min_alloc_size From eece84dada04b99ea5debc694e0bb3452bf94093 Mon Sep 17 00:00:00 2001 From: Daniel Pivonka Date: Fri, 5 Nov 2021 13:54:36 -0400 Subject: [PATCH 0540/2866] cephadm: fix upgrade name already in use Signed-off-by: Daniel Pivonka --- src/pybind/mgr/cephadm/schedule.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pybind/mgr/cephadm/schedule.py b/src/pybind/mgr/cephadm/schedule.py index 5004ca0fb0219..9ee0a5e3c970f 100644 --- a/src/pybind/mgr/cephadm/schedule.py +++ b/src/pybind/mgr/cephadm/schedule.py @@ -104,9 +104,9 @@ def matches_daemon(self, dd: DaemonDescription) -> bool: if self.name and self.name != dd.daemon_id: return False if self.ports: - if self.ports != dd.ports: + if self.ports != dd.ports and dd.ports: return False - if self.ip != dd.ip: + if self.ip != dd.ip and dd.ip: return False return True From d507e5479f6be2510e1edcab03b238ced6b41aa6 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Fri, 5 Nov 2021 14:24:47 -0400 Subject: [PATCH 0541/2866] mgr/orchestrator: clean up 'orch osd rm status' Signed-off-by: Sage Weil --- src/pybind/mgr/orchestrator/module.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/pybind/mgr/orchestrator/module.py b/src/pybind/mgr/orchestrator/module.py index dde3397ce60cd..5ee2ba3e82836 100644 --- a/src/pybind/mgr/orchestrator/module.py +++ b/src/pybind/mgr/orchestrator/module.py @@ -850,14 +850,17 @@ def _osd_rm_status(self, format: Format = Format.plain) -> HandleCommandResult: out = to_format(report, format, many=True, cls=None) else: table = PrettyTable( - ['OSD_ID', 'HOST', 'STATE', 'PG_COUNT', 'REPLACE', 'FORCE', 'DRAIN_STARTED_AT'], + ['OSD', 'HOST', 'STATE', 'PGS', 'REPLACE', 'FORCE', 'ZAP', + 'DRAIN STARTED AT'], border=False) table.align = 'l' + table._align['PGS'] = 'r' table.left_padding_width = 0 table.right_padding_width = 2 for osd in sorted(report, key=lambda o: o.osd_id): table.add_row([osd.osd_id, osd.hostname, osd.drain_status_human(), - osd.get_pg_count(), osd.replace, osd.force, osd.drain_started_at]) + osd.get_pg_count(), osd.replace, osd.force, osd.zap, + osd.drain_started_at or '']) out = table.get_string() return HandleCommandResult(stdout=out) From ca767fc3a8d861d35efd8f331a8ddf275f3beb8c Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Fri, 5 Nov 2021 14:36:54 -0400 Subject: [PATCH 0542/2866] mgr/cephadm: allow zapping devices from other clusters This is the 99% of the devices that ever get zapped. Fixes: b7782084ac9657be9b2da6ebd56b5029cf859225 Signed-off-by: Sage Weil --- src/pybind/mgr/cephadm/module.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/pybind/mgr/cephadm/module.py b/src/pybind/mgr/cephadm/module.py index 5b82e625a2955..b3a7da5abc946 100644 --- a/src/pybind/mgr/cephadm/module.py +++ b/src/pybind/mgr/cephadm/module.py @@ -2030,15 +2030,6 @@ def zap_device(self, host: str, path: str) -> str: for dev in host_devices: if dev.path == path: - # match, so look a little deeper - if dev.lvs: - for lv in cast(List[Dict[str, str]], dev.lvs): - if lv.get('osd_id', ''): - lv_fsid = lv.get('cluster_fsid') - if lv_fsid != self._cluster_fsid: - raise OrchestratorError( - f"device {path} has lv's from a different Ceph cluster ({lv_fsid})") - osd_id_list.append(lv.get('osd_id', '')) path_found = True break if not path_found: From 578b43e8d5cc22a47bcb3efd0168544f03cb801c Mon Sep 17 00:00:00 2001 From: Casey Bodley Date: Fri, 5 Nov 2021 16:02:17 -0400 Subject: [PATCH 0543/2866] rgw/auth: perm_mask uses std::optional MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit resolves a compiler warning about an uninitialized variable in boost::optional: In file included from ../src/rgw/rgw_common.h:1508, from ../src/rgw/rgw_main.cc:18: ../src/rgw/rgw_auth.h: In member function ‘virtual rgw::auth::swift::DefaultStrategy::aplptr_t rgw::auth::swift::DefaultStrategy::_ZThn56_NK3rgw4auth5swift15D efaultStrategy15create_apl_turlEPN4ceph6common11CephContextEPK9req_stateRK11RGWUserInfo(ceph::common::CephContext*, const req_state*, const RGWUserInfo&) cons t’: ../src/rgw/rgw_auth.h:652:38: warning: ‘.boost::optional::.boost::optional_detail::tc_optional_base::m_storage’ may be used uninitialized [-Wmaybe-uninitialized] 652 | this->perm_mask = perm_mask.get(); | ~~~~~~~~~~~~~^~ In file included from ../src/rgw/rgw_rest_swift.h:14, from ../src/rgw/rgw_main.cc:24: ../src/rgw/rgw_swift_auth.h:27:73: note: ‘’ declared here 27 | : LocalApplier(cct, user_info, LocalApplier::NO_SUBUSER, boost::none) { | Signed-off-by: Casey Bodley --- src/rgw/rgw_auth.cc | 2 +- src/rgw/rgw_auth.h | 13 +++++-------- src/rgw/rgw_auth_s3.h | 4 ++-- src/rgw/rgw_rest_s3.cc | 3 ++- src/rgw/rgw_swift_auth.cc | 4 ++-- src/rgw/rgw_swift_auth.h | 8 ++++---- 6 files changed, 16 insertions(+), 18 deletions(-) diff --git a/src/rgw/rgw_auth.cc b/src/rgw/rgw_auth.cc index 78d9ee5272b8f..dfb338818d733 100644 --- a/src/rgw/rgw_auth.cc +++ b/src/rgw/rgw_auth.cc @@ -911,7 +911,7 @@ rgw::auth::AnonymousEngine::authenticate(const DoutPrefixProvider* dpp, const re auto apl = \ apl_factory->create_apl_local(cct, s, user_info, rgw::auth::LocalApplier::NO_SUBUSER, - boost::none); + std::nullopt); return result_t::grant(std::move(apl)); } } diff --git a/src/rgw/rgw_auth.h b/src/rgw/rgw_auth.h index c31e734a17bb3..db343649cc3d0 100644 --- a/src/rgw/rgw_auth.h +++ b/src/rgw/rgw_auth.h @@ -6,6 +6,7 @@ #define CEPH_RGW_AUTH_H #include +#include #include #include #include @@ -645,14 +646,10 @@ class LocalApplier : public IdentityApplier { LocalApplier(CephContext* const cct, const RGWUserInfo& user_info, std::string subuser, - const boost::optional& perm_mask) + const std::optional& perm_mask) : user_info(user_info), - subuser(std::move(subuser)) { - if (perm_mask) { - this->perm_mask = perm_mask.get(); - } else { - this->perm_mask = RGW_PERM_INVALID; - } + subuser(std::move(subuser)), + perm_mask(perm_mask.value_or(RGW_PERM_INVALID)) { } @@ -679,7 +676,7 @@ class LocalApplier : public IdentityApplier { const req_state* s, const RGWUserInfo& user_info, const std::string& subuser, - const boost::optional& perm_mask) const = 0; + const std::optional& perm_mask) const = 0; }; }; diff --git a/src/rgw/rgw_auth_s3.h b/src/rgw/rgw_auth_s3.h index ff28a27fb6d33..edec31a3c6b13 100644 --- a/src/rgw/rgw_auth_s3.h +++ b/src/rgw/rgw_auth_s3.h @@ -56,7 +56,7 @@ class STSAuthStrategy : public rgw::auth::Strategy, const req_state* const s, const RGWUserInfo& user_info, const std::string& subuser, - const boost::optional& perm_mask) const override { + const std::optional& perm_mask) const override { auto apl = rgw::auth::add_sysreq(cct, store, s, rgw::auth::LocalApplier(cct, user_info, subuser, perm_mask)); return aplptr_t(new decltype(apl)(std::move(apl))); @@ -174,7 +174,7 @@ class AWSAuthStrategy : public rgw::auth::Strategy, const req_state* const s, const RGWUserInfo& user_info, const std::string& subuser, - const boost::optional& perm_mask) const override { + const std::optional& perm_mask) const override { auto apl = rgw::auth::add_sysreq(cct, store, s, rgw::auth::LocalApplier(cct, user_info, subuser, perm_mask)); /* TODO(rzarzynski): replace with static_ptr. */ diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc index 73f3ed5729a3f..a02ca472ebc48 100644 --- a/src/rgw/rgw_rest_s3.cc +++ b/src/rgw/rgw_rest_s3.cc @@ -5926,7 +5926,8 @@ rgw::auth::s3::LocalEngine::authenticate( return result_t::deny(-ERR_SIGNATURE_NO_MATCH); } - auto apl = apl_factory->create_apl_local(cct, s, user->get_info(), k.subuser, boost::none); + auto apl = apl_factory->create_apl_local(cct, s, user->get_info(), + k.subuser, std::nullopt); return result_t::grant(std::move(apl), completer_factory(k.key)); } diff --git a/src/rgw/rgw_swift_auth.cc b/src/rgw/rgw_swift_auth.cc index 3791e7d9214b7..c34ee221edff3 100644 --- a/src/rgw/rgw_swift_auth.cc +++ b/src/rgw/rgw_swift_auth.cc @@ -461,7 +461,7 @@ ExternalTokenEngine::authenticate(const DoutPrefixProvider* dpp, auto apl = apl_factory->create_apl_local(cct, s, user->get_info(), extract_swift_subuser(swift_user), - boost::none); + std::nullopt); return result_t::grant(std::move(apl)); } @@ -615,7 +615,7 @@ SignedTokenEngine::authenticate(const DoutPrefixProvider* dpp, auto apl = apl_factory->create_apl_local(cct, s, user->get_info(), extract_swift_subuser(swift_user), - boost::none); + std::nullopt); return result_t::grant(std::move(apl)); } diff --git a/src/rgw/rgw_swift_auth.h b/src/rgw/rgw_swift_auth.h index 13dc1b63a569d..166a2ad793e8c 100644 --- a/src/rgw/rgw_swift_auth.h +++ b/src/rgw/rgw_swift_auth.h @@ -24,7 +24,7 @@ class TempURLApplier : public rgw::auth::LocalApplier { public: TempURLApplier(CephContext* const cct, const RGWUserInfo& user_info) - : LocalApplier(cct, user_info, LocalApplier::NO_SUBUSER, boost::none) { + : LocalApplier(cct, user_info, LocalApplier::NO_SUBUSER, std::nullopt) { }; void modify_request_state(const DoutPrefixProvider* dpp, req_state * s) const override; /* in/out */ @@ -153,8 +153,8 @@ class SwiftAnonymousApplier : public rgw::auth::LocalApplier { public: SwiftAnonymousApplier(CephContext* const cct, const RGWUserInfo& user_info) - : LocalApplier(cct, user_info, LocalApplier::NO_SUBUSER, boost::none) { - }; + : LocalApplier(cct, user_info, LocalApplier::NO_SUBUSER, std::nullopt) { + } bool is_admin_of(const rgw_user& uid) const {return false;} bool is_owner_of(const rgw_user& uid) const {return uid.id.compare(RGW_USER_ANON_ID) == 0;} }; @@ -225,7 +225,7 @@ class DefaultStrategy : public rgw::auth::Strategy, const req_state* const s, const RGWUserInfo& user_info, const std::string& subuser, - const boost::optional& perm_mask) const override { + const std::optional& perm_mask) const override { auto apl = \ rgw::auth::add_3rdparty(store, rgw_user(s->account_name), rgw::auth::add_sysreq(cct, store, s, From 56e84f6f2ddc34e19befce1f3f75e01533c842ce Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Thu, 21 Oct 2021 10:20:20 -0500 Subject: [PATCH 0544/2866] .github/pull_request_template: address review comments Signed-off-by: Sage Weil --- .github/pull_request_template.md | 38 +++++++++++++++++++++++++++----- 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 10121703cf498..6529b7dee9409 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,5 +1,7 @@ + + ## Checklist - Tracker (select at least one) - - [ ] Very recent bug; references commit where it was introduced - - [ ] New feature (no ticket) - [ ] References tracker ticket - - [ ] Doc update (no ticket) + - [ ] Very recent bug; references commit where it was introduced + - [ ] New feature (ticket optional) + - [ ] Doc update (no ticket needed) +- Component impact + - [ ] Affects [Dashboard](https://tracker.ceph.com/projects/dashboard/issues/new), opened tracker ticket + - [ ] Affects [Orchestrator](https://tracker.ceph.com/projects/orchestrator/issues/new), opened tracker ticket + - [ ] No impact that needs to be tracked - Documentation (select at least one) - [ ] Updates relevant documentation - [ ] No doc update is appropriate - Tests (select at least one) - - [ ] Includes unit test(s) - - [ ] Includes integration test(s) + - [ ] Includes [unit test(s)](https://docs.ceph.com/en/latest/dev/developer_guide/tests-unit-tests/) + - [ ] Includes [integration test(s)](https://docs.ceph.com/en/latest/dev/developer_guide/testing_integration_tests/) - [ ] Includes bug reproducer - [ ] No tests - Teuthology - [ ] Completed teuthology run - [ ] No teuthology test necessary (e.g., documentation) + +
    +Show available Jenkins commands + +- `jenkins retest this please` +- `jenkins test classic perf` +- `jenkins test crimson perf` +- `jenkins test signed` +- `jenkins test make check` +- `jenkins test make check arm64` +- `jenkins test submodules` +- `jenkins test dashboard` +- `jenkins test dashboard cephadm` +- `jenkins test api` +- `jenkins test docs` +- `jenkins render docs` +- `jenkins test ceph-volume all` +- `jenkins test ceph-volume tox` + +
    From 5426f6b2a4da743aeb3c3a446fbfbb69c71f6870 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Fri, 5 Nov 2021 14:37:47 -0400 Subject: [PATCH 0545/2866] mgr/cephadm/services/osd: do not log ok-to-stop/safe-to-destroy failures These failures are normal and expected; they should not pollute the log. Signed-off-by: Sage Weil --- src/pybind/mgr/cephadm/services/osd.py | 9 +++++---- src/pybind/mgr/cephadm/tests/test_osd_removal.py | 6 ++++-- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/pybind/mgr/cephadm/services/osd.py b/src/pybind/mgr/cephadm/services/osd.py index 775d296911ffa..42e8f391aef4f 100644 --- a/src/pybind/mgr/cephadm/services/osd.py +++ b/src/pybind/mgr/cephadm/services/osd.py @@ -432,7 +432,7 @@ def ok_to_stop(self, osds: List["OSD"]) -> bool: 'prefix': "osd ok-to-stop", 'ids': [str(osd.osd_id) for osd in osds] } - return self._run_mon_cmd(cmd_args) + return self._run_mon_cmd(cmd_args, error_ok=True) def set_osd_flag(self, osds: List["OSD"], flag: str) -> bool: base_cmd = f"osd {flag}" @@ -492,7 +492,7 @@ def safe_to_destroy(self, osd_ids: List[int]) -> bool: """ Queries the safe-to-destroy flag for OSDs """ cmd_args = {'prefix': 'osd safe-to-destroy', 'ids': [str(x) for x in osd_ids]} - return self._run_mon_cmd(cmd_args) + return self._run_mon_cmd(cmd_args, error_ok=True) def destroy_osd(self, osd_id: int) -> bool: """ Destroys an OSD (forcefully) """ @@ -510,14 +510,15 @@ def purge_osd(self, osd_id: int) -> bool: } return self._run_mon_cmd(cmd_args) - def _run_mon_cmd(self, cmd_args: dict) -> bool: + def _run_mon_cmd(self, cmd_args: dict, error_ok: bool = False) -> bool: """ Generic command to run mon_command and evaluate/log the results """ ret, out, err = self.mgr.mon_command(cmd_args) if ret != 0: self.mgr.log.debug(f"ran {cmd_args} with mon_command") - self.mgr.log.error(f"cmd: {cmd_args.get('prefix')} failed with: {err}. (errno:{ret})") + if not error_ok: + self.mgr.log.error(f"cmd: {cmd_args.get('prefix')} failed with: {err}. (errno:{ret})") return False self.mgr.log.debug(f"cmd: {cmd_args.get('prefix')} returns: {out}") return True diff --git a/src/pybind/mgr/cephadm/tests/test_osd_removal.py b/src/pybind/mgr/cephadm/tests/test_osd_removal.py index 9347678b84daa..6685fcb2a6003 100644 --- a/src/pybind/mgr/cephadm/tests/test_osd_removal.py +++ b/src/pybind/mgr/cephadm/tests/test_osd_removal.py @@ -84,11 +84,13 @@ def test_ready_to_drain_osds(self, max_osd_draining_count, draining_osds, idling def test_ok_to_stop(self, rm_util): rm_util.ok_to_stop([MockOSD(1)]) - rm_util._run_mon_cmd.assert_called_with({'prefix': 'osd ok-to-stop', 'ids': ['1']}) + rm_util._run_mon_cmd.assert_called_with({'prefix': 'osd ok-to-stop', 'ids': ['1']}, + error_ok=True) def test_safe_to_destroy(self, rm_util): rm_util.safe_to_destroy([1]) - rm_util._run_mon_cmd.assert_called_with({'prefix': 'osd safe-to-destroy', 'ids': ['1']}) + rm_util._run_mon_cmd.assert_called_with({'prefix': 'osd safe-to-destroy', + 'ids': ['1']}, error_ok=True) def test_destroy_osd(self, rm_util): rm_util.destroy_osd(1) From 8c1e9c05f76c43466269518a9798fe6a3b944a3e Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Fri, 5 Nov 2021 14:37:58 -0400 Subject: [PATCH 0546/2866] mgr/cephadm/services/osd: less noisy Signed-off-by: Sage Weil --- src/pybind/mgr/cephadm/services/osd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pybind/mgr/cephadm/services/osd.py b/src/pybind/mgr/cephadm/services/osd.py index 42e8f391aef4f..3798ebd06da05 100644 --- a/src/pybind/mgr/cephadm/services/osd.py +++ b/src/pybind/mgr/cephadm/services/osd.py @@ -412,7 +412,7 @@ def find_osd_stop_threshold(self, osds: List["OSD"]) -> Optional[List["OSD"]]: while not self.ok_to_stop(osds): if len(osds) <= 1: # can't even stop one OSD, aborting - self.mgr.log.info( + self.mgr.log.debug( "Can't even stop one OSD. Cluster is probably busy. Retrying later..") return [] From 4abe4c6a78a941c8a87f9f3bdf1c731618415993 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Fri, 5 Nov 2021 15:00:10 -0400 Subject: [PATCH 0547/2866] qa/suites/orch/cephadm: add repave-all test case Signed-off-by: Sage Weil --- qa/suites/orch/cephadm/osds/2-ops/repave-all.yaml | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 qa/suites/orch/cephadm/osds/2-ops/repave-all.yaml diff --git a/qa/suites/orch/cephadm/osds/2-ops/repave-all.yaml b/qa/suites/orch/cephadm/osds/2-ops/repave-all.yaml new file mode 100644 index 0000000000000..16413aba8c2cf --- /dev/null +++ b/qa/suites/orch/cephadm/osds/2-ops/repave-all.yaml @@ -0,0 +1,13 @@ +tasks: +- cephadm.shell: + host.a: + - | + set -e + set -x + ceph orch ps + ceph orch device ls + ceph osd tree + for osd in `ceph osd ls` ; do + ceph orch osd rm $osd --force --zap --replace + done + while ceph orch osd rm ls | wc | grep ^1 ; do sleep 10 ; done From 0804104ec01cd6acb57fd5c764bf1cfa47ff9a16 Mon Sep 17 00:00:00 2001 From: yuliyang Date: Fri, 5 Nov 2021 08:40:44 +0100 Subject: [PATCH 0548/2866] rgw: fix UploadPartCopy error code when src object not exist and src bucket not exist fix https://tracker.ceph.com/issues/53181 Signed-off-by: yuliyang --- src/rgw/rgw_op.cc | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc index f295fb92d5761..1f057cefa5d09 100644 --- a/src/rgw/rgw_op.cc +++ b/src/rgw/rgw_op.cc @@ -367,10 +367,10 @@ static int read_obj_policy(const DoutPrefixProvider *dpp, map& bucket_attrs, RGWAccessControlPolicy* acl, string *storage_class, - boost::optional& policy, + boost::optional& policy, rgw::sal::Bucket* bucket, rgw::sal::Object* object, - optional_yield y, + optional_yield y, bool copy_src=false) { string upload_id; @@ -3489,6 +3489,9 @@ int RGWPutObj::init_processing(optional_yield y) { &bucket, y); if (ret < 0) { ldpp_dout(this, 5) << __func__ << "(): get_bucket() returned ret=" << ret << dendl; + if (ret == -ENOENT) { + ret = -ERR_NO_SUCH_BUCKET; + } return ret; } @@ -3556,9 +3559,9 @@ int RGWPutObj::verify_permission(optional_yield y) cs_object->set_prefetch_data(s->obj_ctx); /* check source object permissions */ - if (read_obj_policy(this, store, s, copy_source_bucket_info, cs_attrs, &cs_acl, nullptr, - policy, cs_bucket.get(), cs_object.get(), y, true) < 0) { - return -EACCES; + if (ret = read_obj_policy(this, store, s, copy_source_bucket_info, cs_attrs, &cs_acl, nullptr, + policy, cs_bucket.get(), cs_object.get(), y, true); ret < 0) { + return ret; } /* admin request overrides permission checks */ From 6d0e6288e8036ae0c09a8e0d582b5afa67c5bd29 Mon Sep 17 00:00:00 2001 From: Xuehan Xu Date: Sun, 7 Nov 2021 15:47:02 +0800 Subject: [PATCH 0549/2866] crimson/os/seastore/segment_cleaner: initialize segments' avail_bytes with segments' sizes Currently, we initialize segments' avail_bytes with "segment_size * num_segments". Both segment_size and num_segments are 32 bits long, multiplying them would lead to overflow. Signed-off-by: Xuehan Xu --- src/crimson/os/seastore/segment_cleaner.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/crimson/os/seastore/segment_cleaner.h b/src/crimson/os/seastore/segment_cleaner.h index aaac218395cf9..c65dcd99c2e26 100644 --- a/src/crimson/os/seastore/segment_cleaner.h +++ b/src/crimson/os/seastore/segment_cleaner.h @@ -38,7 +38,7 @@ class segment_info_set_t { block_size(block_size), empty_segments(empty_segments), size(size), - avail_bytes(segment_size * num_segments) + avail_bytes(size) {} device_id_t device_id = 0; From 2badc8987ed66b5839a053a8482012cc6b804016 Mon Sep 17 00:00:00 2001 From: Ronen Friedman Date: Sun, 7 Nov 2021 11:03:59 +0200 Subject: [PATCH 0550/2866] crimson/common: disable arithmetic operators for atomic enums While class enums are 'integral types', they do not support all that an int does. Signed-off-by: Ronen Friedman --- src/common/ceph_atomic.h | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/src/common/ceph_atomic.h b/src/common/ceph_atomic.h index 320963500e7f5..4a96ed97e9c7a 100644 --- a/src/common/ceph_atomic.h +++ b/src/common/ceph_atomic.h @@ -42,6 +42,7 @@ namespace ceph { } T operator=(T desired) noexcept { value = std::move(desired); + return *this; } operator T() const noexcept { return value; @@ -49,23 +50,30 @@ namespace ceph { // We need to differentiate with SFINAE as std::atomic offers beefier // interface for integral types. - std::enable_if_t, T> operator++() { + + template + std::enable_if_t && std::is_integral_v, TT> operator++() { return ++value; } - std::enable_if_t, T> operator++(int) { + template + std::enable_if_t && std::is_integral_v, TT> operator++(int) { return value++; } - std::enable_if_t, T> operator--() { + template + std::enable_if_t && std::is_integral_v, TT> operator--() { return --value; } - std::enable_if_t, T> operator--(int) { + template + std::enable_if_t && std::is_integral_v, TT> operator--(int) { return value--; } - std::enable_if_t, T> operator+=(const dummy_atomic& b) { + template + std::enable_if_t && std::is_integral_v, TT> operator+=(const dummy_atomic& b) { value += b; return value; } - std::enable_if_t, T> operator-=(const dummy_atomic& b) { + template + std::enable_if_t && std::is_integral_v, TT> operator-=(const dummy_atomic& b) { value -= b; return value; } From 76f4d29d92be3f9f45767cb1ac6cc50da528ecec Mon Sep 17 00:00:00 2001 From: Jianpeng Ma Date: Mon, 8 Nov 2021 14:33:28 +0800 Subject: [PATCH 0551/2866] librbd/cache/pwl: fix reorder issue between func process_writeback_dirty_entries In fact, we not only make sure ops in order in func process_writeback_dirty_entries, but also make sure ops in order between func process_writeback_dirty_entries. Signed-off-by: Jianpeng Ma --- src/librbd/cache/pwl/AbstractWriteLog.cc | 6 ++++++ src/librbd/cache/pwl/AbstractWriteLog.h | 1 + src/librbd/cache/pwl/rwl/WriteLog.cc | 1 + src/librbd/cache/pwl/ssd/WriteLog.cc | 3 +++ 4 files changed, 11 insertions(+) diff --git a/src/librbd/cache/pwl/AbstractWriteLog.cc b/src/librbd/cache/pwl/AbstractWriteLog.cc index 695b2f0041c9d..c63df19c400b0 100644 --- a/src/librbd/cache/pwl/AbstractWriteLog.cc +++ b/src/librbd/cache/pwl/AbstractWriteLog.cc @@ -1639,6 +1639,7 @@ Context* AbstractWriteLog::construct_flush_entry(std::shared_ptrram_entry.sync_gen_number; } m_flush_ops_in_flight += 1; + m_flush_ops_will_send += 1; /* For write same this is the bytes affected by the flush op, not the bytes transferred */ m_flush_bytes_in_flight += log_entry->ram_entry.write_bytes; @@ -1710,6 +1711,11 @@ void AbstractWriteLog::process_writeback_dirty_entries() { all_clean = !m_flush_ops_in_flight; break; } + + if (m_flush_ops_will_send) { + ldout(cct, 20) << "Previous flush-ops is still not sent" << dendl; + break; + } auto candidate = m_dirty_log_entries.front(); bool flushable = can_flush_entry(candidate); if (flushable) { diff --git a/src/librbd/cache/pwl/AbstractWriteLog.h b/src/librbd/cache/pwl/AbstractWriteLog.h index 88e6470f12f7b..887d77010bb45 100644 --- a/src/librbd/cache/pwl/AbstractWriteLog.h +++ b/src/librbd/cache/pwl/AbstractWriteLog.h @@ -294,6 +294,7 @@ class AbstractWriteLog { std::atomic m_async_flush_ops = {0}; std::atomic m_async_append_ops = {0}; + std::atomic m_flush_ops_will_send = {0}; /* Acquire locks in order declared here */ mutable ceph::mutex m_log_retire_lock; diff --git a/src/librbd/cache/pwl/rwl/WriteLog.cc b/src/librbd/cache/pwl/rwl/WriteLog.cc index f635015329a6c..55a22407dd8a0 100644 --- a/src/librbd/cache/pwl/rwl/WriteLog.cc +++ b/src/librbd/cache/pwl/rwl/WriteLog.cc @@ -592,6 +592,7 @@ void WriteLog::construct_flush_entries(pwl::GenericLogEntries entries_to_flus ldout(m_image_ctx.cct, 15) << "flushing:" << log_entry << " " << *log_entry << dendl; log_entry->writeback(this->m_image_writeback, ctx); + this->m_flush_ops_will_send -= 1; }), 0); }); } diff --git a/src/librbd/cache/pwl/ssd/WriteLog.cc b/src/librbd/cache/pwl/ssd/WriteLog.cc index fcced9d28742b..5c4be2fde0a49 100644 --- a/src/librbd/cache/pwl/ssd/WriteLog.cc +++ b/src/librbd/cache/pwl/ssd/WriteLog.cc @@ -539,6 +539,7 @@ void WriteLog::construct_flush_entries(pwl::GenericLogEntries entries_to_flus ldout(m_image_ctx.cct, 15) << "flushing:" << log_entry << " " << *log_entry << dendl; log_entry->writeback(this->m_image_writeback, ctx); + this->m_flush_ops_will_send -= 1; }), 0); }); } @@ -589,6 +590,7 @@ void WriteLog::construct_flush_entries(pwl::GenericLogEntries entries_to_flus << " " << *log_entry << dendl; log_entry->writeback_bl(this->m_image_writeback, ctx, std::move(captured_entry_bl)); + this->m_flush_ops_will_send -= 1; }), 0); } else { m_image_ctx.op_work_queue->queue(new LambdaContext( @@ -596,6 +598,7 @@ void WriteLog::construct_flush_entries(pwl::GenericLogEntries entries_to_flus ldout(m_image_ctx.cct, 15) << "flushing:" << log_entry << " " << *log_entry << dendl; log_entry->writeback(this->m_image_writeback, ctx); + this->m_flush_ops_will_send -= 1; }), 0); } } From 43f607e16c9bda55aac7f5b03985c63f3cd18885 Mon Sep 17 00:00:00 2001 From: Jeff Layton Date: Tue, 26 Oct 2021 10:22:35 -0400 Subject: [PATCH 0552/2866] client: flesh out doc comments for struct ceph_client_callback_args Add more documentation comments around the ceph_client_callback_args functions, to give application writers a better sense of how they are used and when they should be set. Fixes: https://tracker.ceph.com/issues/53004 Signed-off-by: Jeff Layton --- src/include/cephfs/ceph_ll_client.h | 64 +++++++++++++++++++++++++---- 1 file changed, 57 insertions(+), 7 deletions(-) diff --git a/src/include/cephfs/ceph_ll_client.h b/src/include/cephfs/ceph_ll_client.h index 17dc2fec6193c..5bc693846b971 100644 --- a/src/include/cephfs/ceph_ll_client.h +++ b/src/include/cephfs/ceph_ll_client.h @@ -110,28 +110,78 @@ struct ceph_statx { #define FALLOC_FL_PUNCH_HOLE 0x02 #endif -/* delegation recalls */ +/** ceph_deleg_cb_t: Delegation recalls + * + * Called when there is an outstanding Delegation and there is conflicting + * access, either locally or via cap activity. + * @fh: open filehandle + * @priv: private info registered when delegation was acquired + */ typedef void (*ceph_deleg_cb_t)(Fh *fh, void *priv); -/* inode data/metadata invalidation */ +/** + * client_ino_callback_t: Inode data/metadata invalidation + * + * Called when the client wants to invalidate the cached data for a range + * in the file. + * @handle: client callback handle + * @ino: vino of inode to be invalidated + * @off: starting offset of content to be invalidated + * @len: length of region to invalidate + */ typedef void (*client_ino_callback_t)(void *handle, vinodeno_t ino, int64_t off, int64_t len); -/* dentry invalidation */ +/** + * client_dentry_callback_t: Dentry invalidation + * + * Called when the client wants to purge a dentry from its cache. + * @handle: client callback handle + * @dirino: vino of directory that contains dentry to be invalidate + * @ino: vino of inode attached to dentry to be invalidated + * @name: name of dentry to be invalidated + * @len: length of @name + */ typedef void (*client_dentry_callback_t)(void *handle, vinodeno_t dirino, vinodeno_t ino, const char *name, size_t len); -/* remount entire fs */ +/** + * client_remount_callback_t: Remount entire fs + * + * Called when the client needs to purge the dentry cache and the application + * doesn't have a way to purge an individual dentry. Mostly used for ceph-fuse + * on older kernels. + * @handle: client callback handle + */ + typedef int (*client_remount_callback_t)(void *handle); -/* lock request interrupted */ +/** + * client_switch_interrupt_callback_t: Lock request interrupted + * + * Called before file lock request to set the interrupt handler while waiting + * After the wait, called with "data" set to NULL pointer. + * @handle: client callback handle + * @data: opaque data passed to interrupt before call, NULL pointer after. + */ typedef void (*client_switch_interrupt_callback_t)(void *handle, void *data); -/* fetch umask of actor */ +/** + * client_umask_callback_t: Fetch umask of actor + * + * Called when the client needs the umask of the requestor. + * @handle: client callback handle + */ typedef mode_t (*client_umask_callback_t)(void *handle); -/* request that application release Inode references */ +/** + * client_ino_release_t: Request that application release Inode references + * + * Called when the MDS wants to trim caps and Inode records. + * @handle: client callback handle + * @ino: vino of Inode being released + */ typedef void (*client_ino_release_t)(void *handle, vinodeno_t ino); /* From 43915774b90caa473307b5e04aeb88bc451d2226 Mon Sep 17 00:00:00 2001 From: Arjun Sharma Date: Tue, 27 Jul 2021 15:04:06 +0530 Subject: [PATCH 0553/2866] rgw/notification: assigning the value of zonegroup to awsRegion Signed-off-by: Arjun Sharma --- src/rgw/rgw_notify.cc | 1 + src/test/rgw/bucket_notification/test_bn.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/src/rgw/rgw_notify.cc b/src/rgw/rgw_notify.cc index 428c7174c01c7..65b7a84d9fecc 100644 --- a/src/rgw/rgw_notify.cc +++ b/src/rgw/rgw_notify.cc @@ -689,6 +689,7 @@ void populate_event_from_request(const reservation_t& res, event.object_size = size; event.object_etag = etag; event.object_versionId = version; + event.awsRegion = res.store->get_zone()->get_zonegroup().api_name; // use timestamp as per key sequence id (hex encoded) const utime_t ts(real_clock::now()); boost::algorithm::hex((const char*)&ts, (const char*)&ts + sizeof(utime_t), diff --git a/src/test/rgw/bucket_notification/test_bn.py b/src/test/rgw/bucket_notification/test_bn.py index ec6b74d661714..7d420aa064ae9 100644 --- a/src/test/rgw/bucket_notification/test_bn.py +++ b/src/test/rgw/bucket_notification/test_bn.py @@ -827,6 +827,7 @@ def test_ps_s3_notification_filter_on_master(): for event in receiver.get_and_reset_events(): notif_id = event['Records'][0]['s3']['configurationId'] key_name = event['Records'][0]['s3']['object']['key'] + awsRegion = event['Records'][0]['awsRegion'] if notif_id == notification_name+'_1': found_in1.append(key_name) elif notif_id == notification_name+'_2': @@ -841,6 +842,7 @@ def test_ps_s3_notification_filter_on_master(): assert_equal(set(found_in1), set(expected_in1)) assert_equal(set(found_in2), set(expected_in2)) assert_equal(set(found_in3), set(expected_in3)) + assert_equal(awsRegion, zonegroup) if not skip_notif4: assert_equal(set(found_in4), set(expected_in4)) From 0bb60469d1c8439eaabd1b89dc494e49e7863b33 Mon Sep 17 00:00:00 2001 From: Casey Bodley Date: Mon, 8 Nov 2021 10:06:24 -0500 Subject: [PATCH 0554/2866] qa/rgw: bump tempest version to resolve dependency issue Fixes: https://tracker.ceph.com/issues/53095 Signed-off-by: Casey Bodley --- qa/suites/rgw/tempest/tasks/rgw_tempest.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qa/suites/rgw/tempest/tasks/rgw_tempest.yaml b/qa/suites/rgw/tempest/tasks/rgw_tempest.yaml index d745d5c186db1..68cb35cc4e032 100644 --- a/qa/suites/rgw/tempest/tasks/rgw_tempest.yaml +++ b/qa/suites/rgw/tempest/tasks/rgw_tempest.yaml @@ -16,7 +16,7 @@ tasks: use-keystone-role: client.0 - tempest: client.0: - sha1: d43223773d75d2e82fb33a1281038e611c41d0f3 + sha1: train-last force-branch: master use-keystone-role: client.0 auth: From cd6aa1329f70f89338757ba295e279ecfdbc2d07 Mon Sep 17 00:00:00 2001 From: Cory Snyder Date: Fri, 24 Sep 2021 11:56:09 -0400 Subject: [PATCH 0555/2866] ceph-volume: fix bug with miscalculation of required db/wal slot size for VGs with multiple PVs Previous logic for calculating db/wal slot sizes made the assumption that there would only be a single PV backing each db/wal VG. This wasn't the case for OSDs deployed prior to v15.2.8, since ceph-volume previously deployed multiple SSDs in the same VG. This fix removes the assumption and does the correct calculation in either case. Fixes: https://tracker.ceph.com/issues/52730 Signed-off-by: Cory Snyder --- .../ceph_volume/devices/lvm/batch.py | 74 ++++++++++++------- src/ceph-volume/ceph_volume/tests/conftest.py | 6 ++ 2 files changed, 53 insertions(+), 27 deletions(-) diff --git a/src/ceph-volume/ceph_volume/devices/lvm/batch.py b/src/ceph-volume/ceph_volume/devices/lvm/batch.py index 0faa88ec0a88e..0dfd809bdf3bd 100644 --- a/src/ceph-volume/ceph_volume/devices/lvm/batch.py +++ b/src/ceph-volume/ceph_volume/devices/lvm/batch.py @@ -112,35 +112,55 @@ def get_physical_fast_allocs(devices, type_, fast_slots_per_device, new_osds, ar requested_size = get_size_fct(lv_format=False) ret = [] - for dev in devices: - if not dev.available_lvm: - continue - # any LV present is considered a taken slot - occupied_slots = len(dev.lvs) - # this only looks at the first vg on device, unsure if there is a better - # way - dev_size = dev.vg_size[0] - abs_size = disk.Size(b=int(dev_size / requested_slots)) - free_size = dev.vg_free[0] - relative_size = int(abs_size) / dev_size - if requested_size: - if requested_size <= abs_size: - abs_size = requested_size - relative_size = int(abs_size) / dev_size - else: - mlogger.error( - '{} was requested for {}, but only {} can be fulfilled'.format( - requested_size, - '{}_size'.format(type_), - abs_size, - )) - exit(1) - while abs_size <= free_size and len(ret) < new_osds and occupied_slots < fast_slots_per_device: - free_size -= abs_size.b - occupied_slots += 1 - ret.append((dev.path, relative_size, abs_size, requested_slots)) + vg_device_map = group_devices_by_vg(devices) + for vg_devices in vg_device_map.values(): + for dev in vg_devices: + if not dev.available_lvm: + continue + # any LV present is considered a taken slot + occupied_slots = len(dev.lvs) + # prior to v15.2.8, db/wal deployments were grouping multiple fast devices into single VGs - we need to + # multiply requested_slots (per device) by the number of devices in the VG in order to ensure that + # abs_size is calculated correctly from vg_size + slots_for_vg = len(vg_devices) * requested_slots + dev_size = dev.vg_size[0] + # this only looks at the first vg on device, unsure if there is a better + # way + abs_size = disk.Size(b=int(dev_size / slots_for_vg)) + free_size = dev.vg_free[0] + relative_size = int(abs_size) / dev_size + if requested_size: + if requested_size <= abs_size: + abs_size = requested_size + relative_size = int(abs_size) / dev_size + else: + mlogger.error( + '{} was requested for {}, but only {} can be fulfilled'.format( + requested_size, + '{}_size'.format(type_), + abs_size, + )) + exit(1) + while abs_size <= free_size and len(ret) < new_osds and occupied_slots < fast_slots_per_device: + free_size -= abs_size.b + occupied_slots += 1 + ret.append((dev.path, relative_size, abs_size, requested_slots)) return ret +def group_devices_by_vg(devices): + result = dict() + result['unused_devices'] = [] + for dev in devices: + if len(dev.vgs) > 0: + # already using assumption that a PV only belongs to single VG in other places + vg_name = dev.vgs[0].name + if vg_name in result: + result[vg_name].append(dev) + else: + result[vg_name] = [dev] + else: + result['unused_devices'].append(dev) + return result def get_lvm_fast_allocs(lvs): return [("{}/{}".format(d.vg_name, d.lv_name), 100.0, diff --git a/src/ceph-volume/ceph_volume/tests/conftest.py b/src/ceph-volume/ceph_volume/tests/conftest.py index 357881a53f58f..c41a460744397 100644 --- a/src/ceph-volume/ceph_volume/tests/conftest.py +++ b/src/ceph-volume/ceph_volume/tests/conftest.py @@ -63,6 +63,9 @@ def mock_lv(): def mock_devices_available(): dev = create_autospec(device.Device) dev.path = '/dev/foo' + dev.vg_name = 'vg_foo' + dev.lv_name = 'lv_foo' + dev.vgs = [lvm.VolumeGroup(vg_name=dev.vg_name, lv_name=dev.lv_name)] dev.available_lvm = True dev.vg_size = [21474836480] dev.vg_free = dev.vg_size @@ -73,6 +76,9 @@ def mock_device_generator(): def mock_device(): dev = create_autospec(device.Device) dev.path = '/dev/foo' + dev.vg_name = 'vg_foo' + dev.lv_name = 'lv_foo' + dev.vgs = [lvm.VolumeGroup(vg_name=dev.vg_name, lv_name=dev.lv_name)] dev.available_lvm = True dev.vg_size = [21474836480] dev.vg_free = dev.vg_size From ccbc38c1a72d0d8392f0d50c8253422a30fb8927 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Wed, 3 Nov 2021 16:35:58 -0500 Subject: [PATCH 0556/2866] mgr/rook: fix service_type filtering vs crash Signed-off-by: Sage Weil --- src/pybind/mgr/rook/module.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/pybind/mgr/rook/module.py b/src/pybind/mgr/rook/module.py index 4ba044413a265..294d1ffbf0039 100644 --- a/src/pybind/mgr/rook/module.py +++ b/src/pybind/mgr/rook/module.py @@ -279,7 +279,11 @@ def describe_service(self, container_image_name=image_name, last_refresh=now, ) - if not cl['spec'].get('crashCollector', {}).get('disable', False): + + if ( + service_type == 'crash' or service_type is None + and not cl['spec'].get('crashCollector', {}).get('disable', False) + ): spec['crash'] = orchestrator.ServiceDescription( spec=ServiceSpec( 'crash', From 473a8e04f5e2486a37c307c246344b1c04907877 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Thu, 4 Nov 2021 09:50:13 -0500 Subject: [PATCH 0557/2866] mgr/rook/rook_cluster: ignore pods with no containers Signed-off-by: Sage Weil --- src/pybind/mgr/rook/rook_cluster.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/pybind/mgr/rook/rook_cluster.py b/src/pybind/mgr/rook/rook_cluster.py index 48e8bc95d240a..bc29b30986830 100644 --- a/src/pybind/mgr/rook/rook_cluster.py +++ b/src/pybind/mgr/rook/rook_cluster.py @@ -802,7 +802,11 @@ def predicate(item): image_name = c['image'] break - image_id = d['status']['container_statuses'][0]['image_id'] + ls = d['status'].get('container_statuses') + if not ls: + # ignore pods with no containers + continue + image_id = ls[0]['image_id'] image_id = image_id.split(prefix)[1] if prefix in image_id else image_id s = { From cd7ccb3d2868ab7c916f87c5417acd9a2bb410b7 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Thu, 4 Nov 2021 10:25:18 -0500 Subject: [PATCH 0558/2866] mgr/rook: include drivegroups in 'orch ls' result Signed-off-by: Sage Weil --- src/pybind/mgr/rook/module.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/pybind/mgr/rook/module.py b/src/pybind/mgr/rook/module.py index 294d1ffbf0039..4cab22e3496c6 100644 --- a/src/pybind/mgr/rook/module.py +++ b/src/pybind/mgr/rook/module.py @@ -364,6 +364,9 @@ def describe_service(self, ) if service_type == 'osd' or service_type is None: # OSDs + # FIXME: map running OSDs back to their respective services... + + # the catch-all unmanaged all_osds = self.rook_cluster.get_osds() svc = 'osd' spec[svc] = orchestrator.ServiceDescription( @@ -373,8 +376,17 @@ def describe_service(self, ), size=len(all_osds), last_refresh=now, - running= sum(osd.status.phase == 'Running' for osd in all_osds) + running= sum(osd.status.phase == 'Running' for osd in all_osds) ) + + # drivegroups + for name, dg in self._drive_group_map.items(): + spec[f'osd.{name}'] = orchestrator.ServiceDescription( + spec=dg, + last_refresh=now, + size=0, + running=0, + ) if service_type == 'rbd-mirror' or service_type is None: # rbd-mirrors From 4408d814b2b14673c0a53bb52aa37d45b64d617c Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Thu, 4 Nov 2021 10:36:54 -0500 Subject: [PATCH 0559/2866] mgr/rook: implement removal of drive groups Signed-off-by: Sage Weil --- src/pybind/mgr/rook/module.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/pybind/mgr/rook/module.py b/src/pybind/mgr/rook/module.py index 4cab22e3496c6..2a8a140487d67 100644 --- a/src/pybind/mgr/rook/module.py +++ b/src/pybind/mgr/rook/module.py @@ -498,15 +498,20 @@ def _get_pool_params(self) -> Tuple[int, str]: def remove_service(self, service_name: str) -> str: if service_name == 'rbd-mirror': return self.rook_cluster.rm_service('cephrbdmirrors', 'default-rbd-mirror') - service_type, service_name = service_name.split('.', 1) + service_type, service_id = service_name.split('.', 1) if service_type == 'mds': - return self.rook_cluster.rm_service('cephfilesystems', service_name) + return self.rook_cluster.rm_service('cephfilesystems', service_id) elif service_type == 'rgw': - return self.rook_cluster.rm_service('cephobjectstores', service_name) + return self.rook_cluster.rm_service('cephobjectstores', service_id) elif service_type == 'nfs': - return self.rook_cluster.rm_service('cephnfses', service_name) + return self.rook_cluster.rm_service('cephnfses', service_id) elif service_type == 'rbd-mirror': - return self.rook_cluster.rm_service('cephrbdmirrors', service_name) + return self.rook_cluster.rm_service('cephrbdmirrors', service_id) + elif service_type == 'osd': + if service_id in self._drive_group_map: + del self._drive_group_map[service_id] + self._save_drive_groups() + return f'Removed {service_name}' else: raise orchestrator.OrchestratorError(f'Service type {service_type} not supported') From 1838770f0cf90069b901c03bb24687eba510ddbb Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Thu, 4 Nov 2021 10:49:07 -0500 Subject: [PATCH 0560/2866] mgr/rook: make 'osd' unmanaged service appear as unmanaged This makes it match the cephadm result, where e.g., the placement shows as '' in 'orch ls'. Signed-off-by: Sage Weil --- src/pybind/mgr/rook/module.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pybind/mgr/rook/module.py b/src/pybind/mgr/rook/module.py index 2a8a140487d67..160c87d7fad5a 100644 --- a/src/pybind/mgr/rook/module.py +++ b/src/pybind/mgr/rook/module.py @@ -371,12 +371,12 @@ def describe_service(self, svc = 'osd' spec[svc] = orchestrator.ServiceDescription( spec=DriveGroupSpec( + unmanaged=True, service_type='osd', - placement=PlacementSpec(count=len(all_osds), hosts=[osd.metadata.labels['topology-location-host'] for osd in all_osds]), ), size=len(all_osds), last_refresh=now, - running= sum(osd.status.phase == 'Running' for osd in all_osds) + running=sum(osd.status.phase == 'Running' for osd in all_osds) ) # drivegroups From 9b8e3187ed491e4bcfdecbc82babb0bca835bad6 Mon Sep 17 00:00:00 2001 From: Patrick Donnelly Date: Mon, 8 Nov 2021 19:37:19 -0500 Subject: [PATCH 0561/2866] mds/FSMap: allow upgrade when no MDS is "in" Fixes: https://tracker.ceph.com/issues/52975 Signed-off-by: Patrick Donnelly --- src/mds/FSMap.cc | 10 +++++----- src/mds/FSMap.h | 3 ++- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/mds/FSMap.cc b/src/mds/FSMap.cc index e258cc139ada4..8c1cdfbc0fc76 100644 --- a/src/mds/FSMap.cc +++ b/src/mds/FSMap.cc @@ -933,6 +933,11 @@ void FSMap::promote( } auto& info = mds_map.mds_info.at(standby_gid); + if (!filesystem.mds_map.compat.writeable(info.compat)) { + ceph_assert(filesystem.is_upgradeable()); + filesystem.mds_map.compat.merge(info.compat); + } + if (mds_map.stopped.erase(assigned_rank)) { // The cluster is being expanded with a stopped rank info.state = MDSMap::STATE_STARTING; @@ -958,11 +963,6 @@ void FSMap::promote( standby_epochs.erase(standby_gid); } - if (!filesystem.mds_map.compat.writeable(info.compat)) { - ceph_assert(filesystem.is_upgradeable()); - filesystem.mds_map.compat.merge(info.compat); - } - // Indicate that Filesystem has been modified mds_map.epoch = epoch; } diff --git a/src/mds/FSMap.h b/src/mds/FSMap.h index 197d5141c17d5..099c1337ceb86 100644 --- a/src/mds/FSMap.h +++ b/src/mds/FSMap.h @@ -189,7 +189,8 @@ class Filesystem void print(std::ostream& out) const; bool is_upgradeable() const { - return !mds_map.allows_standby_replay() && mds_map.get_num_in_mds() <= 1; + return (mds_map.allows_standby_replay() && mds_map.get_num_in_mds() == 0) + || (!mds_map.allows_standby_replay() && mds_map.get_num_in_mds() <= 1); } /** From c6b3b92fcfeaf573c78703d1cc98fa9cae3923cb Mon Sep 17 00:00:00 2001 From: Yingxin Cheng Date: Fri, 5 Nov 2021 16:00:16 +0800 Subject: [PATCH 0562/2866] crimson/os/seastore/journal: mark committed_to at the write boundary Signed-off-by: Yingxin Cheng --- src/crimson/os/seastore/journal.cc | 87 ++++++++++++++++-------------- src/crimson/os/seastore/journal.h | 23 ++++---- 2 files changed, 61 insertions(+), 49 deletions(-) diff --git a/src/crimson/os/seastore/journal.cc b/src/crimson/os/seastore/journal.cc index b2d4aaf43ea08..eb7084418a515 100644 --- a/src/crimson/os/seastore/journal.cc +++ b/src/crimson/os/seastore/journal.cc @@ -355,11 +355,16 @@ Journal::JournalSegmentManager::write(ceph::bufferlist to_write) write_start_seq, write_start_seq.offset.offset + write_length, write_length); + assert(write_length > 0); assert((write_length % segment_manager.get_block_size()) == 0); assert(!needs_roll(write_length)); auto write_start_offset = written_to; written_to += write_length; + auto write_result = write_result_t{ + write_start_seq, + static_cast(write_length) + }; return current_journal_segment->write( write_start_offset, to_write ).handle_error( @@ -367,8 +372,8 @@ Journal::JournalSegmentManager::write(ceph::bufferlist to_write) crimson::ct_error::assert_all{ "Invalid error in JournalSegmentManager::write" } - ).safe_then([write_start_seq] { - return write_start_seq; + ).safe_then([write_result] { + return write_result; }); } @@ -418,8 +423,7 @@ Journal::JournalSegmentManager::initialize_segment(Segment& segment) written_to = 0; return write(bl - ).safe_then([this, new_tail, write_size=bl.length() - ](journal_seq_t write_start_seq) { + ).safe_then([this, new_tail](auto) { segment_provider->update_journal_tail_committed(new_tail); }); } @@ -442,25 +446,35 @@ Journal::RecordBatch::add_pending( auto new_encoded_length = get_encoded_length(rsize); assert(new_encoded_length < MAX_SEG_OFF); encoded_length = new_encoded_length; + bool is_first; if (state == state_t::EMPTY) { assert(!io_promise.has_value()); - io_promise = seastar::shared_promise >(); + io_promise = seastar::shared_promise(); + is_first = true; } else { assert(io_promise.has_value()); + is_first = false; } state = state_t::PENDING; return io_promise->get_shared_future( - ).then([record_start_offset - ](auto batch_write_start) -> add_pending_ret { - if (!batch_write_start.has_value()) { + ).then([record_start_offset, is_first + ](auto maybe_write_result) -> add_pending_ret { + if (!maybe_write_result.has_value()) { return crimson::ct_error::input_output_error::make(); } - auto record_write_start = batch_write_start.value(); - record_write_start.offset.offset += record_start_offset; + auto write_result = maybe_write_result.value(); + if (is_first) { + assert(record_start_offset == 0); + } else { + assert(record_start_offset > 0); + write_result.write_start_seq.offset.offset += record_start_offset; + // only the first record should update JournalSegmentManager::committed_to + write_result.write_length = 0; + } return add_pending_ret( add_pending_ertr::ready_future_marker{}, - record_write_start); + write_result); }); } @@ -495,14 +509,14 @@ ceph::bufferlist Journal::RecordBatch::encode_records( } void Journal::RecordBatch::set_result( - std::optional batch_write_start) + maybe_result_t maybe_write_result) { - if (batch_write_start.has_value()) { + if (maybe_write_result.has_value()) { logger().debug( - "Journal::RecordBatch::set_result: batches={}, write_start {} => {}", + "Journal::RecordBatch::set_result: batches={}, write_start {} + {}", records.size(), - *batch_write_start, - batch_write_start->offset.offset + encoded_length); + maybe_write_result->write_start_seq, + maybe_write_result->write_length); } else { logger().error( "Journal::RecordBatch::set_result: batches={}, write is failed!", @@ -515,7 +529,7 @@ void Journal::RecordBatch::set_result( encoded_length = 0; records.clear(); record_sizes.clear(); - io_promise->set_value(batch_write_start); + io_promise->set_value(maybe_write_result); io_promise.reset(); } @@ -600,10 +614,10 @@ void Journal::RecordSubmitter::update_state() void Journal::RecordSubmitter::finish_submit_batch( RecordBatch* p_batch, - std::optional result) + maybe_result_t maybe_result) { assert(p_batch->is_submitting()); - p_batch->set_result(result); + p_batch->set_result(maybe_result); free_batch_ptrs.push_back(p_batch); decrement_io_with_flush(); } @@ -621,8 +635,8 @@ void Journal::RecordSubmitter::flush_current_batch() journal_segment_manager.get_committed_to(), journal_segment_manager.get_nonce()); std::ignore = journal_segment_manager.write(to_write - ).safe_then([this, p_batch](journal_seq_t write_start) { - finish_submit_batch(p_batch, write_start); + ).safe_then([this, p_batch](auto write_result) { + finish_submit_batch(p_batch, write_result); }).handle_error( crimson::ct_error::all_same_way([this, p_batch](auto e) { logger().error( @@ -638,23 +652,6 @@ void Journal::RecordSubmitter::flush_current_batch() }); } -seastar::future> -Journal::RecordSubmitter::mark_record_committed_in_order( - OrderingHandle& handle, - const journal_seq_t& write_start_seq, - const record_size_t& rsize) -{ - return handle.enter(write_pipeline->finalize - ).then([this, write_start_seq, rsize] { - auto committed_to = write_start_seq; - committed_to.offset.offset += (rsize.mdlength + rsize.dlength); - journal_segment_manager.mark_committed(committed_to); - return std::make_pair( - write_start_seq.offset.add_offset(rsize.mdlength), - write_start_seq); - }); -} - Journal::RecordSubmitter::submit_pending_ret Journal::RecordSubmitter::submit_pending( record_t&& record, @@ -692,8 +689,18 @@ Journal::RecordSubmitter::submit_pending( return handle.enter(write_pipeline->device_submission ).then([write_fut=std::move(write_fut)]() mutable { return std::move(write_fut); - }).safe_then([this, &handle, rsize](journal_seq_t write_start) { - return mark_record_committed_in_order(handle, write_start, rsize); + }).safe_then([this, &handle, rsize](auto write_result) { + return handle.enter(write_pipeline->finalize + ).then([this, write_result, rsize] { + if (write_result.write_length > 0) { + auto committed_to = write_result.write_start_seq; + committed_to.offset.offset += write_result.write_length; + journal_segment_manager.mark_committed(committed_to); + } + return std::make_pair( + write_result.write_start_seq.offset.add_offset(rsize.mdlength), + write_result.write_start_seq); + }); }); } diff --git a/src/crimson/os/seastore/journal.h b/src/crimson/os/seastore/journal.h index be7c49695617a..893968822c20b 100644 --- a/src/crimson/os/seastore/journal.h +++ b/src/crimson/os/seastore/journal.h @@ -179,8 +179,12 @@ class Journal { // write the buffer, return the write start // May be called concurrently, writes may complete in any order. + struct write_result_t { + journal_seq_t write_start_seq; + segment_off_t write_length; + }; using write_ertr = base_ertr; - using write_ret = write_ertr::future; + using write_ret = write_ertr::future; write_ret write(ceph::bufferlist to_write); // mark write committed in order @@ -278,6 +282,10 @@ class Journal { // Add to the batch, the future will be resolved after the batch is // written. + // + // Set write_result_t::write_length to 0 if the record is not the first one + // in the batch. + using add_pending_result_t = JournalSegmentManager::write_result_t; using add_pending_ertr = JournalSegmentManager::write_ertr; using add_pending_ret = JournalSegmentManager::write_ret; add_pending_ret add_pending(record_t&&, const record_size_t&); @@ -289,7 +297,8 @@ class Journal { segment_nonce_t segment_nonce); // Set the write result and reset for reuse - void set_result(std::optional batch_write_start); + using maybe_result_t = std::optional; + void set_result(maybe_result_t maybe_write_result); // The fast path that is equivalent to submit a single record as a batch. // @@ -319,8 +328,7 @@ class Journal { segment_off_t encoded_length = 0; std::vector records; std::vector record_sizes; - std::optional > > io_promise; + std::optional > io_promise; }; class RecordSubmitter { @@ -404,14 +412,11 @@ class Journal { free_batch_ptrs.pop_front(); } - void finish_submit_batch(RecordBatch*, std::optional); + using maybe_result_t = RecordBatch::maybe_result_t; + void finish_submit_batch(RecordBatch*, maybe_result_t); void flush_current_batch(); - seastar::future> - mark_record_committed_in_order( - OrderingHandle&, const journal_seq_t&, const record_size_t&); - using submit_pending_ertr = JournalSegmentManager::write_ertr; using submit_pending_ret = submit_pending_ertr::future< std::pair >; From 9fbc6957af28d0521ac926322e9c96a3024a95bb Mon Sep 17 00:00:00 2001 From: Yingxin Cheng Date: Mon, 8 Nov 2021 11:52:50 +0800 Subject: [PATCH 0563/2866] crimson/os/seastore/journal: fix updates to journal seq head and target * update journal sequences at the write boundary; * update journal head to the sequence of write end; Signed-off-by: Yingxin Cheng --- src/crimson/os/seastore/journal.cc | 59 +++++++++---------- src/crimson/os/seastore/journal.h | 34 ++++++----- src/crimson/os/seastore/seastore_types.h | 4 ++ .../os/seastore/transaction_manager.cc | 32 ++++++---- .../seastore/test_btree_lba_manager.cc | 8 ++- .../crimson/seastore/test_seastore_journal.cc | 4 +- 6 files changed, 82 insertions(+), 59 deletions(-) diff --git a/src/crimson/os/seastore/journal.cc b/src/crimson/os/seastore/journal.cc index eb7084418a515..408acf2774197 100644 --- a/src/crimson/os/seastore/journal.cc +++ b/src/crimson/os/seastore/journal.cc @@ -207,9 +207,15 @@ Journal::replay_segment( seq.segment_seq)) { return replay_ertr::now(); } else { - return handler( - journal_seq_t{seq.segment_seq, base}, + auto offsets = submit_result_t{ base.add_offset(header.mdlength), + write_result_t{ + journal_seq_t{seq.segment_seq, base}, + static_cast(header.mdlength + header.dlength) + } + }; + return handler( + offsets, delta); } }); @@ -440,41 +446,33 @@ Journal::RecordBatch::add_pending( assert(state != state_t::SUBMITTING); assert(can_batch(rsize)); - auto record_start_offset = encoded_length; + auto block_start_offset = encoded_length + rsize.mdlength; records.push_back(std::move(record)); record_sizes.push_back(rsize); auto new_encoded_length = get_encoded_length(rsize); assert(new_encoded_length < MAX_SEG_OFF); encoded_length = new_encoded_length; - bool is_first; if (state == state_t::EMPTY) { assert(!io_promise.has_value()); io_promise = seastar::shared_promise(); - is_first = true; } else { assert(io_promise.has_value()); - is_first = false; } state = state_t::PENDING; return io_promise->get_shared_future( - ).then([record_start_offset, is_first + ).then([block_start_offset ](auto maybe_write_result) -> add_pending_ret { if (!maybe_write_result.has_value()) { return crimson::ct_error::input_output_error::make(); } - auto write_result = maybe_write_result.value(); - if (is_first) { - assert(record_start_offset == 0); - } else { - assert(record_start_offset > 0); - write_result.write_start_seq.offset.offset += record_start_offset; - // only the first record should update JournalSegmentManager::committed_to - write_result.write_length = 0; - } + auto submit_result = submit_result_t{ + maybe_write_result->start_seq.offset.add_offset(block_start_offset), + *maybe_write_result + }; return add_pending_ret( add_pending_ertr::ready_future_marker{}, - write_result); + submit_result); }); } @@ -515,8 +513,9 @@ void Journal::RecordBatch::set_result( logger().debug( "Journal::RecordBatch::set_result: batches={}, write_start {} + {}", records.size(), - maybe_write_result->write_start_seq, - maybe_write_result->write_length); + maybe_write_result->start_seq, + maybe_write_result->length); + assert(maybe_write_result->length == encoded_length); } else { logger().error( "Journal::RecordBatch::set_result: batches={}, write is failed!", @@ -673,7 +672,12 @@ Journal::RecordSubmitter::submit_pending( journal_segment_manager.get_committed_to(), journal_segment_manager.get_nonce()); return journal_segment_manager.write(to_write - ).finally([this] { + ).safe_then([rsize](auto write_result) { + return submit_result_t{ + write_result.start_seq.offset.add_offset(rsize.mdlength), + write_result + }; + }).finally([this] { decrement_io_with_flush(); }); } else { @@ -689,17 +693,12 @@ Journal::RecordSubmitter::submit_pending( return handle.enter(write_pipeline->device_submission ).then([write_fut=std::move(write_fut)]() mutable { return std::move(write_fut); - }).safe_then([this, &handle, rsize](auto write_result) { + }).safe_then([this, &handle](auto submit_result) { return handle.enter(write_pipeline->finalize - ).then([this, write_result, rsize] { - if (write_result.write_length > 0) { - auto committed_to = write_result.write_start_seq; - committed_to.offset.offset += write_result.write_length; - journal_segment_manager.mark_committed(committed_to); - } - return std::make_pair( - write_result.write_start_seq.offset.add_offset(rsize.mdlength), - write_result.write_start_seq); + ).then([this, submit_result] { + journal_segment_manager.mark_committed( + submit_result.write_result.get_end_seq()); + return submit_result; }); }); } diff --git a/src/crimson/os/seastore/journal.h b/src/crimson/os/seastore/journal.h index 893968822c20b..dec7c89ab54f4 100644 --- a/src/crimson/os/seastore/journal.h +++ b/src/crimson/os/seastore/journal.h @@ -83,14 +83,26 @@ class Journal { /** * submit_record * - * @param write record and returns offset of first block and seq + * write record with the ordering handle */ + struct write_result_t { + journal_seq_t start_seq; + segment_off_t length; + + journal_seq_t get_end_seq() const { + return start_seq.add_offset(length); + } + }; + struct submit_result_t { + paddr_t record_block_base; + write_result_t write_result; + }; using submit_record_ertr = crimson::errorator< crimson::ct_error::erange, crimson::ct_error::input_output_error >; using submit_record_ret = submit_record_ertr::future< - std::pair + submit_result_t >; submit_record_ret submit_record( record_t &&record, @@ -108,8 +120,7 @@ class Journal { using replay_ertr = SegmentManager::read_ertr; using replay_ret = replay_ertr::future<>; using delta_handler_t = std::function< - replay_ret(journal_seq_t seq, - paddr_t record_block_base, + replay_ret(const submit_result_t&, const delta_info_t&)>; replay_ret replay( std::vector>&& segment_headers, @@ -177,12 +188,8 @@ class Journal { using roll_ertr = base_ertr; roll_ertr::future<> roll(); - // write the buffer, return the write start + // write the buffer, return the write result // May be called concurrently, writes may complete in any order. - struct write_result_t { - journal_seq_t write_start_seq; - segment_off_t write_length; - }; using write_ertr = base_ertr; using write_ret = write_ertr::future; write_ret write(ceph::bufferlist to_write); @@ -285,9 +292,8 @@ class Journal { // // Set write_result_t::write_length to 0 if the record is not the first one // in the batch. - using add_pending_result_t = JournalSegmentManager::write_result_t; using add_pending_ertr = JournalSegmentManager::write_ertr; - using add_pending_ret = JournalSegmentManager::write_ret; + using add_pending_ret = add_pending_ertr::future; add_pending_ret add_pending(record_t&&, const record_size_t&); // Encode the batched records for write. @@ -297,8 +303,8 @@ class Journal { segment_nonce_t segment_nonce); // Set the write result and reset for reuse - using maybe_result_t = std::optional; - void set_result(maybe_result_t maybe_write_result); + using maybe_result_t = std::optional; + void set_result(maybe_result_t maybe_write_end_seq); // The fast path that is equivalent to submit a single record as a batch. // @@ -419,7 +425,7 @@ class Journal { using submit_pending_ertr = JournalSegmentManager::write_ertr; using submit_pending_ret = submit_pending_ertr::future< - std::pair >; + submit_result_t>; submit_pending_ret submit_pending( record_t&&, const record_size_t&, OrderingHandle &handle, bool flush); diff --git a/src/crimson/os/seastore/seastore_types.h b/src/crimson/os/seastore/seastore_types.h index a7fc29a46ffc3..8ce7e8d77b063 100644 --- a/src/crimson/os/seastore/seastore_types.h +++ b/src/crimson/os/seastore/seastore_types.h @@ -585,6 +585,10 @@ struct journal_seq_t { segment_seq_t segment_seq = 0; paddr_t offset; + journal_seq_t add_offset(segment_off_t o) const { + return {segment_seq, offset.add_offset(o)}; + } + DENC(journal_seq_t, v, p) { DENC_START(1, 1, p); denc(v.segment_seq, p); diff --git a/src/crimson/os/seastore/transaction_manager.cc b/src/crimson/os/seastore/transaction_manager.cc index 5d52fa83d2c17..643becf009b1d 100644 --- a/src/crimson/os/seastore/transaction_manager.cc +++ b/src/crimson/os/seastore/transaction_manager.cc @@ -75,11 +75,14 @@ TransactionManager::mount_ertr::future<> TransactionManager::mount() [this](auto&& segments) { return journal->replay( std::move(segments), - [this](auto seq, auto paddr, const auto &e) { - auto fut = cache->replay_delta(seq, paddr, e); + [this](const auto &offsets, const auto &e) { + auto start_seq = offsets.write_result.start_seq; segment_cleaner->update_journal_tail_target( - cache->get_oldest_dirty_from().value_or(seq)); - return fut; + cache->get_oldest_dirty_from().value_or(start_seq)); + return cache->replay_delta( + start_seq, + offsets.record_block_base, + e); }); }).safe_then([this] { return journal->open_for_write(); @@ -273,14 +276,23 @@ TransactionManager::submit_transaction_direct( DEBUGT("about to submit to journal", tref); return journal->submit_record(std::move(record), tref.get_handle() - ).safe_then([this, FNAME, &tref](auto p) mutable { - auto [addr, journal_seq] = p; - DEBUGT("journal commit to {} seq {}", tref, addr, journal_seq); - segment_cleaner->set_journal_head(journal_seq); - cache->complete_commit(tref, addr, journal_seq, segment_cleaner.get()); + ).safe_then([this, FNAME, &tref](auto submit_result) mutable { + auto start_seq = submit_result.write_result.start_seq; + auto end_seq = submit_result.write_result.get_end_seq(); + DEBUGT("journal commit to record_block_base={}, start_seq={}, end_seq={}", + tref, + submit_result.record_block_base, + start_seq, + end_seq); + segment_cleaner->set_journal_head(end_seq); + cache->complete_commit( + tref, + submit_result.record_block_base, + start_seq, + segment_cleaner.get()); lba_manager->complete_transaction(tref); segment_cleaner->update_journal_tail_target( - cache->get_oldest_dirty_from().value_or(journal_seq)); + cache->get_oldest_dirty_from().value_or(start_seq)); auto to_release = tref.get_segment_to_release(); if (to_release != NULL_SEG_ID) { return segment_manager.release(to_release diff --git a/src/test/crimson/seastore/test_btree_lba_manager.cc b/src/test/crimson/seastore/test_btree_lba_manager.cc index 4c4bec1478059..9bfa40e4f3955 100644 --- a/src/test/crimson/seastore/test_btree_lba_manager.cc +++ b/src/test/crimson/seastore/test_btree_lba_manager.cc @@ -58,9 +58,11 @@ struct btree_test_base : { auto record = cache->prepare_record(*t); return journal->submit_record(std::move(record), t->get_handle()).safe_then( - [this, t=std::move(t)](auto p) mutable { - auto [addr, seq] = p; - cache->complete_commit(*t, addr, seq); + [this, t=std::move(t)](auto submit_result) mutable { + cache->complete_commit( + *t, + submit_result.record_block_base, + submit_result.write_result.start_seq); complete_commit(*t); }).handle_error(crimson::ct_error::assert_all{}); } diff --git a/src/test/crimson/seastore/test_seastore_journal.cc b/src/test/crimson/seastore/test_seastore_journal.cc index bb69c81750259..366f3f3b9b8a1 100644 --- a/src/test/crimson/seastore/test_seastore_journal.cc +++ b/src/test/crimson/seastore/test_seastore_journal.cc @@ -193,11 +193,11 @@ struct journal_test_t : seastar_test_suite_t, SegmentProvider { replay( [&advance, &delta_checker] - (auto seq, auto base, const auto &di) mutable { + (const auto &offsets, const auto &di) mutable { if (!delta_checker) { EXPECT_FALSE("No Deltas Left"); } - if (!(*delta_checker)(base, di)) { + if (!(*delta_checker)(offsets.record_block_base, di)) { delta_checker = std::nullopt; advance(); } From e253f685382efd7fa82849fb5eb4ad803099ca73 Mon Sep 17 00:00:00 2001 From: Venky Shankar Date: Sun, 1 Aug 2021 10:21:46 -0400 Subject: [PATCH 0564/2866] mgr/mirroring: throttle directory reassigment to mirror daemons This is to avoid over-shuffling directories when lots of mirror daemons come and go. Signed-off-by: Venky Shankar --- src/pybind/mgr/mirroring/fs/dir_map/policy.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/pybind/mgr/mirroring/fs/dir_map/policy.py b/src/pybind/mgr/mirroring/fs/dir_map/policy.py index 608a92bf541bf..aef90b55fe5f4 100644 --- a/src/pybind/mgr/mirroring/fs/dir_map/policy.py +++ b/src/pybind/mgr/mirroring/fs/dir_map/policy.py @@ -27,6 +27,10 @@ def __str__(self): f' purging={self.purging}]' class Policy: + # number of seconds after which a directory can be reshuffled + # to other mirror daemon instances. + DIR_SHUFFLE_THROTTLE_INTERVAL = 300 + def __init__(self): self.dir_states = {} self.instance_to_dir_map = {} @@ -54,7 +58,8 @@ def can_shuffle_dir(self, dir_path): """ log.debug(f'can_shuffle_dir: {dir_path}') dir_state = self.dir_states[dir_path] - return StateTransition.is_idle(dir_state.state) + return StateTransition.is_idle(dir_state.state) and \ + (time.time() - dir_state['mapped_time']) > Policy.DIR_SHUFFLE_THROTTLE_INTERVAL def set_state(self, dir_state, state, ignore_current_state=False): if not ignore_current_state and dir_state.state == state: From 087d7aa8ca4de05cf5162be54996aec90724a540 Mon Sep 17 00:00:00 2001 From: Venky Shankar Date: Tue, 10 Aug 2021 03:04:51 -0400 Subject: [PATCH 0565/2866] tasks/cephfs_mirror: optionally run in foreground cephfs mirror damon thrasher needs to send SIGTERM to mirror daemons. The mirror daemon needs to run in foreground for it to receive signal via `daemon.signal`. Signed-off-by: Venky Shankar --- qa/tasks/cephfs_mirror.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/qa/tasks/cephfs_mirror.py b/qa/tasks/cephfs_mirror.py index 42c01aaea0342..9602a5a7f9711 100644 --- a/qa/tasks/cephfs_mirror.py +++ b/qa/tasks/cephfs_mirror.py @@ -54,6 +54,8 @@ def begin(self): '--id', self.client_id, ]) + if 'run_in_foreground' in self.config: + args.extend(['--foreground']) self.ctx.daemons.add_daemon( self.remote, 'cephfs-mirror', self.client, From 3a4dd30a1e385039c01675bd3fa8c785ebc66286 Mon Sep 17 00:00:00 2001 From: Venky Shankar Date: Wed, 28 Jul 2021 08:24:21 -0400 Subject: [PATCH 0566/2866] test: add cephfs_mirror thrasher Signed-off-by: Venky Shankar --- qa/tasks/cephfs_mirror_thrash.py | 219 +++++++++++++++++++++++++++++++ 1 file changed, 219 insertions(+) create mode 100644 qa/tasks/cephfs_mirror_thrash.py diff --git a/qa/tasks/cephfs_mirror_thrash.py b/qa/tasks/cephfs_mirror_thrash.py new file mode 100644 index 0000000000000..91f60ac50137c --- /dev/null +++ b/qa/tasks/cephfs_mirror_thrash.py @@ -0,0 +1,219 @@ +""" +Task for thrashing cephfs-mirror daemons +""" + +import contextlib +import logging +import random +import signal +import socket +import time + +from gevent import sleep +from gevent.greenlet import Greenlet +from gevent.event import Event + +from teuthology.exceptions import CommandFailedError +from teuthology.orchestra import run +from tasks.thrasher import Thrasher + +log = logging.getLogger(__name__) + + +class CephFSMirrorThrasher(Thrasher, Greenlet): + """ + CephFSMirrorThrasher:: + + The CephFSMirrorThrasher thrashes cephfs-mirror daemons during execution of other + tasks (workunits, etc). + + The config is optional. Many of the config parameters are a maximum value + to use when selecting a random value from a range. The config is a dict + containing some or all of: + + cluster: [default: ceph] cluster to thrash + + max_thrash: [default: 1] the maximum number of active cephfs-mirror daemons per + cluster will be thrashed at any given time. + + min_thrash_delay: [default: 60] minimum number of seconds to delay before + thrashing again. + + max_thrash_delay: [default: 120] maximum number of seconds to delay before + thrashing again. + + max_revive_delay: [default: 10] maximum number of seconds to delay before + bringing back a thrashed cephfs-mirror daemon. + + randomize: [default: true] enables randomization and use the max/min values + + seed: [no default] seed the random number generator + + Examples:: + + The following example disables randomization, and uses the max delay + values: + + tasks: + - ceph: + - cephfs_mirror_thrash: + randomize: False + max_thrash_delay: 10 + """ + + def __init__(self, ctx, config, cluster, daemons): + super(CephFSMirrorThrasher, self).__init__() + + self.ctx = ctx + self.config = config + self.cluster = cluster + self.daemons = daemons + + self.logger = log + self.name = 'thrasher.cephfs_mirror.[{cluster}]'.format(cluster = cluster) + self.stopping = Event() + + self.randomize = bool(self.config.get('randomize', True)) + self.max_thrash = int(self.config.get('max_thrash', 1)) + self.min_thrash_delay = float(self.config.get('min_thrash_delay', 5.0)) + self.max_thrash_delay = float(self.config.get('max_thrash_delay', 10)) + self.max_revive_delay = float(self.config.get('max_revive_delay', 15.0)) + + def _run(self): + try: + self.do_thrash() + except Exception as e: + # See _run exception comment for MDSThrasher + self.set_thrasher_exception(e) + self.logger.exception("exception:") + # Allow successful completion so gevent doesn't see an exception. + # The DaemonWatchdog will observe the error and tear down the test. + + def log(self, x): + """Write data to logger assigned to this CephFSMirrorThrasher""" + self.logger.info(x) + + def stop(self): + self.stopping.set() + + def do_thrash(self): + """ + Perform the random thrashing action + """ + + self.log('starting thrash for cluster {cluster}'.format(cluster=self.cluster)) + stats = { + "kill": 0, + } + + while not self.stopping.is_set(): + delay = self.max_thrash_delay + if self.randomize: + delay = random.randrange(self.min_thrash_delay, self.max_thrash_delay) + + if delay > 0.0: + self.log('waiting for {delay} secs before thrashing'.format(delay=delay)) + self.stopping.wait(delay) + if self.stopping.is_set(): + continue + + killed_daemons = [] + + weight = 1.0 / len(self.daemons) + count = 0 + for daemon in self.daemons: + skip = random.uniform(0.0, 1.0) + if weight <= skip: + self.log('skipping daemon {label} with skip ({skip}) > weight ({weight})'.format( + label=daemon.id_, skip=skip, weight=weight)) + continue + + self.log('kill {label}'.format(label=daemon.id_)) + try: + daemon.signal(signal.SIGTERM) + except Exception as e: + self.log(f'exception when stopping mirror daemon: {e}') + else: + killed_daemons.append(daemon) + stats['kill'] += 1 + + # if we've reached max_thrash, we're done + count += 1 + if count >= self.max_thrash: + break + + if killed_daemons: + # wait for a while before restarting + delay = self.max_revive_delay + if self.randomize: + delay = random.randrange(0.0, self.max_revive_delay) + + self.log('waiting for {delay} secs before reviving daemons'.format(delay=delay)) + sleep(delay) + + for daemon in killed_daemons: + self.log('waiting for {label}'.format(label=daemon.id_)) + try: + run.wait([daemon.proc], timeout=600) + except CommandFailedError: + pass + except: + self.log('Failed to stop {label}'.format(label=daemon.id_)) + + try: + # try to capture a core dump + daemon.signal(signal.SIGABRT) + except socket.error: + pass + raise + finally: + daemon.reset() + + for daemon in killed_daemons: + self.log('reviving {label}'.format(label=daemon.id_)) + daemon.start() + + for stat in stats: + self.log("stat['{key}'] = {value}".format(key = stat, value = stats[stat])) + +@contextlib.contextmanager +def task(ctx, config): + """ + Stress test the cephfs-mirror by thrashing while another task/workunit + is running. + + Please refer to CephFSMirrorThrasher class for further information on the + available options. + """ + if config is None: + config = {} + assert isinstance(config, dict), \ + 'cephfs_mirror_thrash task only accepts a dict for configuration' + + cluster = config.get('cluster', 'ceph') + daemons = list(ctx.daemons.iter_daemons_of_role('cephfs-mirror', cluster)) + assert len(daemons) > 0, \ + 'cephfs_mirror_thrash task requires at least 1 cephfs-mirror daemon' + + # choose random seed + if 'seed' in config: + seed = int(config['seed']) + else: + seed = int(time.time()) + log.info('cephfs_mirror_thrash using random seed: {seed}'.format(seed=seed)) + random.seed(seed) + + thrasher = CephFSMirrorThrasher(ctx, config, cluster, daemons) + thrasher.start() + ctx.ceph[cluster].thrashers.append(thrasher) + + try: + log.debug('Yielding') + yield + finally: + log.info('joining cephfs_mirror_thrash') + thrasher.stop() + if thrasher.exception is not None: + raise RuntimeError('error during thrashing') + thrasher.join() + log.info('done joining') From 22b9c4808ebc33816a087f9a549e5f1b5ecff77a Mon Sep 17 00:00:00 2001 From: zhangmengqian_yw Date: Tue, 9 Nov 2021 10:52:03 +0800 Subject: [PATCH 0567/2866] mgr/dashboard:add unittest in test_osd.py Add some unittests in src/pybind/mgr/dashboard/tests/test_osd.py Signed-off-by: Zhang Meng Qian --- src/pybind/mgr/dashboard/tests/test_osd.py | 58 ++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/src/pybind/mgr/dashboard/tests/test_osd.py b/src/pybind/mgr/dashboard/tests/test_osd.py index 790c9b359262c..775c6ca739708 100644 --- a/src/pybind/mgr/dashboard/tests/test_osd.py +++ b/src/pybind/mgr/dashboard/tests/test_osd.py @@ -253,6 +253,13 @@ def test_osd_list_aggregation(self): self.assertEqual(len(self.json_body()), 2, 'It should display two OSDs without failure') self.assertStatus(200) + @mock.patch('dashboard.controllers.osd.CephService') + def test_osd_scrub(self, ceph_service): + self._task_post('/api/osd/1/scrub', {'deep': True}) + ceph_service.send_command.assert_called_once_with('mon', 'osd deep-scrub', who='1') + self.assertStatus(200) + self.assertJsonBody(None) + @mock.patch('dashboard.controllers.osd.CephService') def test_osd_create_bare(self, ceph_service): ceph_service.send_command.return_value = '5' @@ -270,6 +277,21 @@ def test_osd_create_bare(self, ceph_service): self.assertStatus(201) ceph_service.send_command.assert_called() + # unknown method + data['method'] = 'other' + self._task_post('/api/osd', data) + self.assertStatus(400) + res = self.json_body() + self.assertIn('Unknown method', res['detail']) + + # svc_id is not int + data['data']['svc_id'] = "five" + data['method'] = 'bare' + self._task_post('/api/osd', data) + self.assertStatus(400) + res = self.json_body() + self.assertIn(data['data']['svc_id'], res['detail']) + @mock.patch('dashboard.controllers.orchestrator.OrchClient.instance') def test_osd_create_with_drive_groups(self, instance): # without orchestrator service @@ -312,7 +334,43 @@ def test_osd_mark_all_actions(self, instance): fake_client = mock.Mock() instance.return_value = fake_client action_list = ['OUT', 'IN', 'DOWN'] + for action in action_list: data = {'action': action} self._task_put('/api/osd/1/mark', data) self.assertStatus(200) + + # invalid mark + instance.reset_mock() + with self.assertLogs(level='ERROR') as cm: + self._task_put('/api/osd/1/mark', {'action': 'OTHER'}) + instance.send_command.assert_not_called() + self.assertIn('Invalid OSD mark action', cm.output[0]) + self.assertStatus(200) + self.assertJsonBody(None) + + self._task_post('/api/osd/1/purge', {'svc_id': 1}) + instance.send_command.assert_called_once_with('mon', 'osd purge-actual', id=1, + yes_i_really_mean_it=True) + self.assertStatus(200) + self.assertJsonBody(None) + + @mock.patch('dashboard.controllers.osd.CephService') + def test_reweight_osd(self, instance): + instance.send_command.return_value = '5' + uuid1 = str(uuid.uuid1()) + sample_data = { + 'uuid': uuid1, + 'svc_id': 1 + } + data = { + 'method': 'bare', + 'data': sample_data, + 'tracking_id': 'bare-1' + } + self._task_post('/api/osd', data) + self._task_put('/api/osd/1/mark', {'action': 'DOWN'}) + self.assertStatus(200) + self._task_post('/api/osd/1/reweight', {'weight': '1'}) + instance.send_command.assert_called_with('mon', 'osd reweight', id=1, weight=1.0) + self.assertStatus(200) From e7bdf7fb35e57c01e05d24e2f3ec6d6c7fcdcf0a Mon Sep 17 00:00:00 2001 From: Venky Shankar Date: Wed, 16 Jun 2021 01:23:27 -0400 Subject: [PATCH 0568/2866] test: add cephfs-mirror HA active/active workunit and test yamls Fixes: http://tracker.ceph.com/issues/50372 Signed-off-by: Venky Shankar --- qa/suites/fs/mirror-ha/% | 0 qa/suites/fs/mirror-ha/.qa | 1 + qa/suites/fs/mirror-ha/begin.yaml | 1 + .../cephfs-mirror/three-per-cluster.yaml | 12 ++++ qa/suites/fs/mirror-ha/clients/+ | 0 qa/suites/fs/mirror-ha/clients/.qa | 1 + qa/suites/fs/mirror-ha/clients/mirror.yaml | 32 +++++++++ qa/suites/fs/mirror-ha/cluster/+ | 0 qa/suites/fs/mirror-ha/cluster/1-node.yaml | 20 ++++++ qa/suites/fs/mirror-ha/objectstore/.qa | 1 + .../objectstore/bluestore-bitmap.yaml | 1 + qa/suites/fs/mirror-ha/overrides/+ | 0 qa/suites/fs/mirror-ha/overrides/.qa | 1 + .../mirror-ha/overrides/whitelist_health.yaml | 14 ++++ .../fs/mirror-ha/supported-random-distro$ | 1 + qa/suites/fs/mirror-ha/workloads/.qa | 1 + .../workloads/cephfs-mirror-ha-workunit.yaml | 37 ++++++++++ qa/suites/fs/mirror/tasks/mirror.yaml | 2 +- .../fs/valgrind/mirror/tasks/mirror.yaml | 2 +- qa/workunits/fs/cephfs_mirror_ha_gen.sh | 69 +++++++++++++++++++ qa/workunits/fs/cephfs_mirror_ha_verify.sh | 40 +++++++++++ qa/workunits/fs/cephfs_mirror_helpers.sh | 66 ++++++++++++++++++ 22 files changed, 300 insertions(+), 2 deletions(-) create mode 100644 qa/suites/fs/mirror-ha/% create mode 120000 qa/suites/fs/mirror-ha/.qa create mode 120000 qa/suites/fs/mirror-ha/begin.yaml create mode 100644 qa/suites/fs/mirror-ha/cephfs-mirror/three-per-cluster.yaml create mode 100644 qa/suites/fs/mirror-ha/clients/+ create mode 120000 qa/suites/fs/mirror-ha/clients/.qa create mode 100644 qa/suites/fs/mirror-ha/clients/mirror.yaml create mode 100644 qa/suites/fs/mirror-ha/cluster/+ create mode 100644 qa/suites/fs/mirror-ha/cluster/1-node.yaml create mode 120000 qa/suites/fs/mirror-ha/objectstore/.qa create mode 120000 qa/suites/fs/mirror-ha/objectstore/bluestore-bitmap.yaml create mode 100644 qa/suites/fs/mirror-ha/overrides/+ create mode 120000 qa/suites/fs/mirror-ha/overrides/.qa create mode 100644 qa/suites/fs/mirror-ha/overrides/whitelist_health.yaml create mode 120000 qa/suites/fs/mirror-ha/supported-random-distro$ create mode 120000 qa/suites/fs/mirror-ha/workloads/.qa create mode 100644 qa/suites/fs/mirror-ha/workloads/cephfs-mirror-ha-workunit.yaml create mode 100755 qa/workunits/fs/cephfs_mirror_ha_gen.sh create mode 100755 qa/workunits/fs/cephfs_mirror_ha_verify.sh create mode 100644 qa/workunits/fs/cephfs_mirror_helpers.sh diff --git a/qa/suites/fs/mirror-ha/% b/qa/suites/fs/mirror-ha/% new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/qa/suites/fs/mirror-ha/.qa b/qa/suites/fs/mirror-ha/.qa new file mode 120000 index 0000000000000..fea2489fdf6d9 --- /dev/null +++ b/qa/suites/fs/mirror-ha/.qa @@ -0,0 +1 @@ +../.qa \ No newline at end of file diff --git a/qa/suites/fs/mirror-ha/begin.yaml b/qa/suites/fs/mirror-ha/begin.yaml new file mode 120000 index 0000000000000..311d404f7c276 --- /dev/null +++ b/qa/suites/fs/mirror-ha/begin.yaml @@ -0,0 +1 @@ +.qa/cephfs/begin.yaml \ No newline at end of file diff --git a/qa/suites/fs/mirror-ha/cephfs-mirror/three-per-cluster.yaml b/qa/suites/fs/mirror-ha/cephfs-mirror/three-per-cluster.yaml new file mode 100644 index 0000000000000..095f0893ac26a --- /dev/null +++ b/qa/suites/fs/mirror-ha/cephfs-mirror/three-per-cluster.yaml @@ -0,0 +1,12 @@ +meta: +- desc: run one cephfs-mirror daemon on primary cluster +tasks: +- cephfs-mirror: + client: client.mirror1 + run_in_foreground: True +- cephfs-mirror: + client: client.mirror2 + run_in_foreground: True +- cephfs-mirror: + client: client.mirror3 + run_in_foreground: True diff --git a/qa/suites/fs/mirror-ha/clients/+ b/qa/suites/fs/mirror-ha/clients/+ new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/qa/suites/fs/mirror-ha/clients/.qa b/qa/suites/fs/mirror-ha/clients/.qa new file mode 120000 index 0000000000000..fea2489fdf6d9 --- /dev/null +++ b/qa/suites/fs/mirror-ha/clients/.qa @@ -0,0 +1 @@ +../.qa \ No newline at end of file diff --git a/qa/suites/fs/mirror-ha/clients/mirror.yaml b/qa/suites/fs/mirror-ha/clients/mirror.yaml new file mode 100644 index 0000000000000..620c821e1170e --- /dev/null +++ b/qa/suites/fs/mirror-ha/clients/mirror.yaml @@ -0,0 +1,32 @@ +meta: +- desc: configure the permissions for client.mirror +overrides: + ceph: + conf: + client: + debug cephfs_mirror: 20 + log to stderr: false + # make these predictable + client.mirror1: + admin socket: /var/run/ceph/cephfs-mirror1.asok + pid file: /var/run/ceph/cephfs-mirror1.pid + client.mirror2: + admin socket: /var/run/ceph/cephfs-mirror2.asok + pid file: /var/run/ceph/cephfs-mirror2.pid + client.mirror3: + admin socket: /var/run/ceph/cephfs-mirror3.asok + pid file: /var/run/ceph/cephfs-mirror3.pid +tasks: +- exec: + client.mirror1: + - "sudo ceph auth caps client.mirror1 mon 'profile cephfs-mirror' mds 'allow r' osd 'allow rw tag cephfs metadata=*, allow r tag cephfs data=*' mgr 'allow r'" + client.mirror2: + - "sudo ceph auth caps client.mirror2 mon 'profile cephfs-mirror' mds 'allow r' osd 'allow rw tag cephfs metadata=*, allow r tag cephfs data=*' mgr 'allow r'" + client.mirror3: + - "sudo ceph auth caps client.mirror3 mon 'profile cephfs-mirror' mds 'allow r' osd 'allow rw tag cephfs metadata=*, allow r tag cephfs data=*' mgr 'allow r'" + client.mirror_remote: + - "sudo ceph auth caps client.mirror_remote mon 'allow r' mds 'allow rwps' osd 'allow rw tag cephfs *=*' mgr 'allow r'" + client.1: + - "sudo ceph auth caps client.0 mon 'allow r' mds 'allow rwps' osd 'allow rw tag cephfs *=*' mgr 'allow r'" + client.2: + - "sudo ceph auth caps client.1 mon 'allow r' mds 'allow rwps' osd 'allow rw tag cephfs *=*' mgr 'allow r'" diff --git a/qa/suites/fs/mirror-ha/cluster/+ b/qa/suites/fs/mirror-ha/cluster/+ new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/qa/suites/fs/mirror-ha/cluster/1-node.yaml b/qa/suites/fs/mirror-ha/cluster/1-node.yaml new file mode 100644 index 0000000000000..cc70c106d67e4 --- /dev/null +++ b/qa/suites/fs/mirror-ha/cluster/1-node.yaml @@ -0,0 +1,20 @@ +meta: +- desc: 1 ceph cluster with 1 mon, 1 mgr, 3 osds, 5 mdss +roles: +- - mon.a + - mgr.x + - mds.a + - mds.b + - mds.c + - mds.d + - mds.e + - osd.0 + - osd.1 + - osd.2 + - client.0 + - client.1 + - client.2 + - client.mirror1 + - client.mirror2 + - client.mirror3 + - client.mirror_remote \ No newline at end of file diff --git a/qa/suites/fs/mirror-ha/objectstore/.qa b/qa/suites/fs/mirror-ha/objectstore/.qa new file mode 120000 index 0000000000000..fea2489fdf6d9 --- /dev/null +++ b/qa/suites/fs/mirror-ha/objectstore/.qa @@ -0,0 +1 @@ +../.qa \ No newline at end of file diff --git a/qa/suites/fs/mirror-ha/objectstore/bluestore-bitmap.yaml b/qa/suites/fs/mirror-ha/objectstore/bluestore-bitmap.yaml new file mode 120000 index 0000000000000..a59cf5175069a --- /dev/null +++ b/qa/suites/fs/mirror-ha/objectstore/bluestore-bitmap.yaml @@ -0,0 +1 @@ +.qa/objectstore/bluestore-bitmap.yaml \ No newline at end of file diff --git a/qa/suites/fs/mirror-ha/overrides/+ b/qa/suites/fs/mirror-ha/overrides/+ new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/qa/suites/fs/mirror-ha/overrides/.qa b/qa/suites/fs/mirror-ha/overrides/.qa new file mode 120000 index 0000000000000..fea2489fdf6d9 --- /dev/null +++ b/qa/suites/fs/mirror-ha/overrides/.qa @@ -0,0 +1 @@ +../.qa \ No newline at end of file diff --git a/qa/suites/fs/mirror-ha/overrides/whitelist_health.yaml b/qa/suites/fs/mirror-ha/overrides/whitelist_health.yaml new file mode 100644 index 0000000000000..d40fa4cb82ce7 --- /dev/null +++ b/qa/suites/fs/mirror-ha/overrides/whitelist_health.yaml @@ -0,0 +1,14 @@ +overrides: + ceph: + log-ignorelist: + - overall HEALTH_ + - \(FS_DEGRADED\) + - \(MDS_FAILED\) + - \(MDS_DEGRADED\) + - \(FS_WITH_FAILED_MDS\) + - \(MDS_DAMAGE\) + - \(MDS_ALL_DOWN\) + - \(MDS_UP_LESS_THAN_MAX\) + - \(FS_INLINE_DATA_DEPRECATED\) + - Reduced data availability + - Degraded data redundancy diff --git a/qa/suites/fs/mirror-ha/supported-random-distro$ b/qa/suites/fs/mirror-ha/supported-random-distro$ new file mode 120000 index 0000000000000..0862b4457b37a --- /dev/null +++ b/qa/suites/fs/mirror-ha/supported-random-distro$ @@ -0,0 +1 @@ +.qa/distros/supported-random-distro$ \ No newline at end of file diff --git a/qa/suites/fs/mirror-ha/workloads/.qa b/qa/suites/fs/mirror-ha/workloads/.qa new file mode 120000 index 0000000000000..fea2489fdf6d9 --- /dev/null +++ b/qa/suites/fs/mirror-ha/workloads/.qa @@ -0,0 +1 @@ +../.qa \ No newline at end of file diff --git a/qa/suites/fs/mirror-ha/workloads/cephfs-mirror-ha-workunit.yaml b/qa/suites/fs/mirror-ha/workloads/cephfs-mirror-ha-workunit.yaml new file mode 100644 index 0000000000000..18eb46d786ffb --- /dev/null +++ b/qa/suites/fs/mirror-ha/workloads/cephfs-mirror-ha-workunit.yaml @@ -0,0 +1,37 @@ +meta: +- desc: run the cephfs_mirror_ha.sh workunit to test cephfs-mirror daemon in HA active/active mode + +overrides: + ceph: + conf: + mgr: + debug client: 10 + +tasks: + - exec: + client.1: + - "ceph fs volume create dc" + - "ceph fs volume create dc-backup" + - ceph-fuse: + client.1: + cephfs_name: dc + client.2: + cephfs_name: dc-backup + - cephfs_mirror_thrash: + randomize: False + max_thrash_delay: 10 + - workunit: + subdir: mirror + cleanup: False + clients: + client.1: [fs/cephfs_mirror_ha_gen.sh] + timeout: 1h + - exec: + client.2: + - "echo verifying synchronized snapshots..." + - workunit: + subdir: mirror + cleanup: False + clients: + client.2: [fs/cephfs_mirror_ha_verify.sh] + timeout: 3h \ No newline at end of file diff --git a/qa/suites/fs/mirror/tasks/mirror.yaml b/qa/suites/fs/mirror/tasks/mirror.yaml index af60495e53f8b..07c1e24efbf8e 100644 --- a/qa/suites/fs/mirror/tasks/mirror.yaml +++ b/qa/suites/fs/mirror/tasks/mirror.yaml @@ -7,4 +7,4 @@ overrides: tasks: - cephfs_test_runner: modules: - - tasks.cephfs.test_mirroring + - tasks.cephfs.test_mirroring.TestMirroring diff --git a/qa/suites/fs/valgrind/mirror/tasks/mirror.yaml b/qa/suites/fs/valgrind/mirror/tasks/mirror.yaml index af60495e53f8b..07c1e24efbf8e 100644 --- a/qa/suites/fs/valgrind/mirror/tasks/mirror.yaml +++ b/qa/suites/fs/valgrind/mirror/tasks/mirror.yaml @@ -7,4 +7,4 @@ overrides: tasks: - cephfs_test_runner: modules: - - tasks.cephfs.test_mirroring + - tasks.cephfs.test_mirroring.TestMirroring diff --git a/qa/workunits/fs/cephfs_mirror_ha_gen.sh b/qa/workunits/fs/cephfs_mirror_ha_gen.sh new file mode 100755 index 0000000000000..35ee9d4c7ee34 --- /dev/null +++ b/qa/workunits/fs/cephfs_mirror_ha_gen.sh @@ -0,0 +1,69 @@ +#!/bin/bash -ex +# +# cephfs_mirror_ha_gen.sh - generate workload to synchronize +# + +. $(dirname $0)/cephfs_mirror_helpers.sh + +cleanup() +{ + for i in `seq 1 $NR_DIRECTORIES` + do + local repo_name="${REPO_PATH_PFX}_$i" + for j in `seq 1 $NR_SNAPSHOTS` + do + snap_name=$repo_name/.snap/snap_$j + if test -d $snap_name; then + rmdir $snap_name + fi + done + done + exit 1 +} +trap cleanup EXIT + +configure_peer() +{ + ceph mgr module enable mirroring + ceph fs snapshot mirror enable $PRIMARY_FS + ceph fs snapshot mirror peer_add $PRIMARY_FS client.mirror_remote@ceph $BACKUP_FS + + for i in `seq 1 $NR_DIRECTORIES` + do + local repo_name="${REPO_PATH_PFX}_$i" + ceph fs snapshot mirror add $PRIMARY_FS "$MIRROR_SUBDIR/$repo_name" + done +} + +create_snaps() +{ + for i in `seq 1 $NR_DIRECTORIES` + do + local repo_name="${REPO_PATH_PFX}_$i" + for j in `seq 1 $NR_SNAPSHOTS` + do + snap_name=$repo_name/.snap/snap_$j + r=$(( $RANDOM % 100 + 5 )) + arr=($repo_name "reset" "--hard" "HEAD~$r") + exec_git_cmd "${arr[@]}" + mkdir $snap_name + store_checksum $snap_name + done + done +} + +unset CEPH_CLI_TEST_DUP_COMMAND + +echo "running generator on prmary file system..." + +# setup git repos to be used as data set +setup_repos + +# turn on mirroring, add peers... +configure_peer + +# snapshots on primary +create_snaps + +# do not cleanup when exiting on success.. +trap - EXIT diff --git a/qa/workunits/fs/cephfs_mirror_ha_verify.sh b/qa/workunits/fs/cephfs_mirror_ha_verify.sh new file mode 100755 index 0000000000000..8d8b3859cb6ba --- /dev/null +++ b/qa/workunits/fs/cephfs_mirror_ha_verify.sh @@ -0,0 +1,40 @@ +#!/bin/bash -ex +# +# cephfs_mirror_ha_verify.sh - verify synchronized snapshots +# + +. $(dirname $0)/cephfs_mirror_helpers.sh + +echo "running verifier on secondary file system..." + +for i in `seq 1 $NR_DIRECTORIES` +do + repo_name="${REPO_PATH_PFX}_$i" + for j in `seq 1 $NR_SNAPSHOTS` + do + for s in 1 1 2 4 4 4 4 4 8 8 8 8 16 16 32 64 64 128 128 + do + sleep $s + snap_name=$repo_name/.snap/snap_$j + if test -d $repo_name; then + echo "checking snapshot [$snap_name] in $repo_name" + if test -d $snap_name; then + echo "generating hash for $snap_name" + cksum='' + calc_checksum $snap_name cksum + ret=$(compare_checksum $cksum $snap_name) + if [ $ret -ne 0 ]; then + echo "checksum failed $snap_name ($cksum)" + return $ret + else + echo "checksum matched $snap_name ($cksum)" + break + fi + fi + fi + done + echo "couldn't complete verification for: $snap_name" + done +done + +echo "verify done!" diff --git a/qa/workunits/fs/cephfs_mirror_helpers.sh b/qa/workunits/fs/cephfs_mirror_helpers.sh new file mode 100644 index 0000000000000..69f1c6f3d77e0 --- /dev/null +++ b/qa/workunits/fs/cephfs_mirror_helpers.sh @@ -0,0 +1,66 @@ +PRIMARY_FS='dc' +BACKUP_FS='dc-backup' + +REPO=ceph-qa-suite +REPO_DIR=ceph_repo +REPO_PATH_PFX="$REPO_DIR/$REPO" + +NR_DIRECTORIES=4 +NR_SNAPSHOTS=4 +MIRROR_SUBDIR='/mirror' + +calc_checksum() +{ + local path=$1 + local -n ref=$2 + ref=`find -L $path -type f -exec md5sum {} + | awk '{ print $1 }' | md5sum | awk '{ print $1 }'` +} + +store_checksum() +{ + local path=$1 + local cksum='' #something invalid + local fhash=`echo -n $path | md5sum | awk '{ print $1 }'` + calc_checksum $path cksum + echo -n $cksum > "/tmp/primary-$fhash" +} + +compare_checksum() +{ + local ret=0 + local cksum=$1 + local path=$2 + local fhash=`echo -n $path | md5sum | awk '{ print $1 }'` + local cksum_ondisk=`cat /tmp/primary-$fhash` + if [ $cksum != $cksum_ondisk ]; then + echo "$cksum <> $cksum_ondisk" + ret=1 + fi + echo $ret +} + +exec_git_cmd() +{ + local arg=("$@") + local repo_name=${arg[0]} + local cmd=${arg[@]:1} + git --git-dir "$repo_name/.git" $cmd +} + +clone_repo() +{ + local repo_name=$1 + git clone --branch giant "http://github.com/ceph/$REPO" $repo_name +} + +setup_repos() +{ + mkdir "$REPO_DIR" + + for i in `seq 1 $NR_DIRECTORIES` + do + local repo_name="${REPO_PATH_PFX}_$i" + mkdir $repo_name + clone_repo $repo_name + done +} From 5bec0dd75035f6054869f2c7106c9a48e952989a Mon Sep 17 00:00:00 2001 From: Jos Collin Date: Tue, 22 Dec 2020 18:01:30 +0530 Subject: [PATCH 0569/2866] doc: update #cephfs channel Signed-off-by: Jos Collin --- doc/start/get-involved.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/start/get-involved.rst b/doc/start/get-involved.rst index bc83f9899347c..270b390ff2cc1 100644 --- a/doc/start/get-involved.rst +++ b/doc/start/get-involved.rst @@ -27,9 +27,9 @@ These are exciting times in the Ceph community! Get involved! | | IRC channel particularly during daytime hours | - **Channels:** | | | in the US Pacific Standard Time zone. | ``#ceph``, | | | While ``#ceph`` is a good starting point for | ``#ceph-devel``, | -| | cluster operators and users, there is also | ``#ceph-dashboard`` | -| | ``#ceph-devel`` and ``#ceph-dashboard`` | | -| | dedicated for Ceph developers. | | +| | cluster operators and users, there is also | ``#ceph-dashboard``, | +| | ``#ceph-devel``, ``#ceph-dashboard`` and | ``#cephfs`` | +| | ``#cephfs`` dedicated for Ceph developers. | | +----------------------+-------------------------------------------------+-----------------------------------------------+ | **User List** | Ask and answer user-related questions by | | | | subscribing to the email list at | - `User Subscribe`_ | From 722f95e7ffbffb4b9fc9ab281ad695098de11dfa Mon Sep 17 00:00:00 2001 From: Jos Collin Date: Tue, 22 Dec 2020 18:34:24 +0530 Subject: [PATCH 0570/2866] doc: update dev list and kernel client Signed-off-by: Jos Collin --- doc/rados/troubleshooting/troubleshooting-mon.rst | 2 +- doc/start/get-involved.rst | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/doc/rados/troubleshooting/troubleshooting-mon.rst b/doc/rados/troubleshooting/troubleshooting-mon.rst index f35d3d0b22f5f..549291ef02c36 100644 --- a/doc/rados/troubleshooting/troubleshooting-mon.rst +++ b/doc/rados/troubleshooting/troubleshooting-mon.rst @@ -526,7 +526,7 @@ Reaching out for help ---------------------- You can find us on IRC at #ceph and #ceph-devel at OFTC (server irc.oftc.net) -and on ``ceph-devel@vger.kernel.org`` and ``ceph-users@lists.ceph.com``. Make +and on ``dev@ceph.io`` and ``ceph-users@lists.ceph.com``. Make sure you have grabbed your logs and have them ready if someone asks: the faster the interaction and lower the latency in response, the better chances everyone's time is optimized. diff --git a/doc/start/get-involved.rst b/doc/start/get-involved.rst index 270b390ff2cc1..1303bb31fdafa 100644 --- a/doc/start/get-involved.rst +++ b/doc/start/get-involved.rst @@ -42,6 +42,10 @@ These are exciting times in the Ceph community! Get involved! | | You can opt out of the email list at any time by| - `Devel Unsubscribe`_ | | | unsubscribing. A simple email is all it takes! | - `Devel Archives`_ | +----------------------+-------------------------------------------------+-----------------------------------------------+ +| **Kernel Client** | Linux kernel-related traffic, including kernel | - `Kernel Client Subscribe`_ | +| | patches and discussion of implementation details| - `Kernel Client Unsubscribe`_ | +| | for the kernel client code. | - `Kernel Client Archives`_ | ++----------------------+-------------------------------------------------+-----------------------------------------------+ | **Commit List** | Subscribe to ceph-commit@ceph.com to get | | | | commit notifications via email. You can opt out | - `Commit Subscribe`_ | | | of the email list at any time by unsubscribing. | - `Commit Unsubscribe`_ | @@ -74,6 +78,8 @@ These are exciting times in the Ceph community! Get involved! .. _Devel Subscribe: mailto:dev-request@ceph.io?body=subscribe .. _Devel Unsubscribe: mailto:dev-request@ceph.io?body=unsubscribe +.. _Kernel Client Subscribe: mailto:majordomo@vger.kernel.org?body=subscribe+ceph-devel +.. _Kernel Client Unsubscribe: mailto:majordomo@vger.kernel.org?body=unsubscribe+ceph-devel .. _User Subscribe: mailto:ceph-users-request@ceph.io?body=subscribe .. _User Unsubscribe: mailto:ceph-users-request@ceph.io?body=unsubscribe .. _Community Subscribe: mailto:ceph-community-join@lists.ceph.com @@ -84,6 +90,7 @@ These are exciting times in the Ceph community! Get involved! .. _QA Unsubscribe: mailto:ceph-qa-leave@lists.ceph.com .. _Devel Archives: https://lists.ceph.io/hyperkitty/list/dev@ceph.io/ .. _User Archives: https://lists.ceph.io/hyperkitty/list/ceph-users@ceph.io/ +.. _Kernel Client Archives: https://www.spinics.net/lists/ceph-devel/ .. _Mailing list archives: http://lists.ceph.com/ .. _Blog: http://ceph.com/community/blog/ .. _Tracker: http://tracker.ceph.com/ From e25568ec006ceea2585b9002d922f740100aaa2d Mon Sep 17 00:00:00 2001 From: Nikhilkumar Shelke Date: Fri, 29 Oct 2021 12:14:18 +0530 Subject: [PATCH 0571/2866] vstart.sh: print newline character after vstart.sh help text Without this newline character the help text messes up the expected position of the prompt string. Signed-off-by: Nikhilkumar Shelke --- src/vstart.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/vstart.sh b/src/vstart.sh index 2b6a4017afab7..c6ff55e7d33b8 100755 --- a/src/vstart.sh +++ b/src/vstart.sh @@ -199,7 +199,7 @@ read -r -d '' usage <: bind to specific ip @@ -236,7 +236,7 @@ options: --msgr2: use msgr2 only --msgr21: use msgr2 and msgr1 --crimson: use crimson-osd instead of ceph-osd - --crimson-foreground: use crimson-osd, but run it in the foreground + --crimson-foreground: use crimson-osd, but run it in the foreground --osd-args: specify any extra osd specific options --bluestore-devs: comma-separated list of blockdevs to use for bluestore --bluestore-zoned: blockdevs listed by --bluestore-devs are zoned devices (HM-SMR HDD or ZNS SSD) @@ -246,7 +246,8 @@ options: --no-parallel: dont start all OSDs in parallel --jaeger: use jaegertracing for tracing --seastore-devs: comma-separated list of blockdevs to use for seastore - --seastore-secondary-des: comma-separated list of secondary blockdevs to use for seastore + --seastore-secondary-des: comma-separated list of secondary blockdevs to use for seastore +\n EOF usage_exit() { From 0be2c26a25bf3ca6c5207738f303d77f34e67e70 Mon Sep 17 00:00:00 2001 From: Adam Kupczyk Date: Tue, 9 Nov 2021 13:01:23 +0100 Subject: [PATCH 0572/2866] os/bluestore: Protect _clone against sudden omap format changes Added assert to verify that omap prefixes between cloned objects are exactly the same. If they would differ rewrite_omap_key() will possibly overwrite user key potion of data, or move some part of prefix into user key. This is a follow up from https://github.com/ceph/ceph/pull/43687 Signed-off-by: Adam Kupczyk --- src/os/bluestore/BlueStore.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/os/bluestore/BlueStore.cc b/src/os/bluestore/BlueStore.cc index 309a3af3c8e85..371f92220aae2 100644 --- a/src/os/bluestore/BlueStore.cc +++ b/src/os/bluestore/BlueStore.cc @@ -16150,6 +16150,9 @@ int BlueStore::_clone(TransContext *txc, } else { newo->onode.set_omap_flags(per_pool_omap == OMAP_BULK); } + // check if prefix for omap key is exactly the same size for both objects + // otherwise rewrite_omap_key will corrupt data + ceph_assert(oldo->onode.flags == newo->onode.flags); const string& prefix = newo->get_omap_prefix(); KeyValueDB::Iterator it = db->get_iterator(prefix); string head, tail; From fecfdd464c23275ebc0d29bb3713aa6d3e439a27 Mon Sep 17 00:00:00 2001 From: Radoslaw Zarzynski Date: Tue, 9 Nov 2021 11:36:31 +0000 Subject: [PATCH 0573/2866] msg/async: remove the misleading error description on read failures Before this patch we were calling `cpp_strerror` on the error code returned by messenger's low-level methods for reading from socket. Unfortunately, it leaded to misleading debugs like: ``` (...).read_bulk peer close file descriptor 28 (...).read_until read failed (...).handle_read_frame_preamble_main read frame preamble failed r=-1 ((1) Operation not permitted) ``` The problem is that these lower layers simply don't adhere to the POSIX-like, ernno convention for reporting problems. Therefore, `-1` doesn't really mean _Operation not permitted_. The snippets bellow show the actual convention. ``` /* return -1 means `fd` occurs error or closed, it should be closed * return 0 means EAGAIN or EINTR */ ssize_t AsyncConnection::read_bulk(char *buf, unsigned len) { ssize_t nread; again: nread = cs.read(buf, len); if (nread < 0) { if (nread == -EAGAIN) { nread = 0; } else if (nread == -EINTR) { goto again; } else { ldout(async_msgr->cct, 1) << __func__ << " reading from fd=" << cs.fd() << " : "<< nread << " " << strerror(nread) << dendl; return -1; } } else if (nread == 0) { ldout(async_msgr->cct, 1) << __func__ << " peer close file descriptor " << cs.fd() << dendl; return -1; } return nread; } size_t AsyncConnection::read_until(unsigned len, char *p) { // ... recv_end = recv_start = 0; /* nothing left in the prefetch buffer */ if (left > (uint64_t)recv_max_prefetch) { /* this was a large read, we don't prefetch for these */ do { r = read_bulk(p+state_offset, left); ldout(async_msgr->cct, 25) << __func__ << " read_bulk left is " << left << " got " << r << dendl; if (r < 0) { ldout(async_msgr->cct, 1) << __func__ << " read failed" << dendl; return -1; } else if (r == static_cast(left)) { state_offset = 0; return 0; } state_offset += r; left -= r; } while (r > 0); } else { // ... } CtPtr ProtocolV2::read_frame() { if (state == CLOSED) { return nullptr; } ldout(cct, 20) << __func__ << dendl; rx_preamble.clear(); rx_epilogue.clear(); rx_segments_data.clear(); return READ(rx_frame_asm.get_preamble_onwire_len(), handle_read_frame_preamble_main); } CtPtr ProtocolV2::handle_read_frame_preamble_main(rx_buffer_t &&buffer, int r) { ldout(cct, 20) << __func__ << " r=" << r << dendl; if (r < 0) { ldout(cct, 1) << __func__ << " read frame preamble failed r=" << r << " (" << cpp_strerror(r) << ")" << dendl; return _fault(); } // ... } ``` Signed-off-by: Radoslaw Zarzynski --- src/msg/async/ProtocolV2.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/msg/async/ProtocolV2.cc b/src/msg/async/ProtocolV2.cc index a176fc2c808ac..2916f073e6375 100644 --- a/src/msg/async/ProtocolV2.cc +++ b/src/msg/async/ProtocolV2.cc @@ -1083,7 +1083,7 @@ CtPtr ProtocolV2::handle_read_frame_preamble_main(rx_buffer_t &&buffer, int r) { if (r < 0) { ldout(cct, 1) << __func__ << " read frame preamble failed r=" << r - << " (" << cpp_strerror(r) << ")" << dendl; + << dendl; return _fault(); } From c182f7c6c56af2e8fa6812185c62b717acb2fbdc Mon Sep 17 00:00:00 2001 From: Daniel Gryniewicz Date: Thu, 21 Oct 2021 10:54:00 -0400 Subject: [PATCH 0574/2866] RGW Zipper - API Cleanups During the documentation pass for the Zipper API, a number of cleanups were found: APIs that should be slightly different, or that were unused entirely. This is a rollup commit of all those cleanups. - move get_multipart_upload() to Bucket - remove unused defer_gc - move create_bucket() into User - rename get_bucket_info() to load_bucket() to match load_user() - Remove read_bucket_stats() The codepaths using read_bucket_stats() used CLS data types, and the function is confusingly named. Load the ent in load_bucket(), and use an alternative data structure to get size stats for the bucket. - rename get_bucket_stats to read_stats - Remove remove_metadata() from API - remove copy_obj_data() from API - rename get_obj_layout to dump_obj_layout - use SAL range_to_ofs Signed-off-by: Daniel Gryniewicz --- src/rgw/rgw_bucket.cc | 20 +- src/rgw/rgw_lc.cc | 6 +- src/rgw/rgw_op.cc | 29 ++- src/rgw/rgw_op.h | 2 +- src/rgw/rgw_quota.cc | 6 +- src/rgw/rgw_rados.cc | 21 +- src/rgw/rgw_rest.cc | 2 +- src/rgw/rgw_rest_log.cc | 2 +- src/rgw/rgw_rest_s3.cc | 8 +- src/rgw/rgw_sal.h | 59 +++--- src/rgw/rgw_sal_dbstore.cc | 254 ++++++++++------------- src/rgw/rgw_sal_dbstore.h | 53 +++-- src/rgw/rgw_sal_rados.cc | 378 +++++++++++++++-------------------- src/rgw/rgw_sal_rados.h | 56 +++--- src/rgw/rgw_user.cc | 15 +- src/rgw/rgw_user.h | 10 +- src/test/rgw/test_rgw_lua.cc | 4 +- 17 files changed, 427 insertions(+), 498 deletions(-) diff --git a/src/rgw/rgw_bucket.cc b/src/rgw/rgw_bucket.cc index dc0dd63cb9b1e..4aaa4e206a0e6 100644 --- a/src/rgw/rgw_bucket.cc +++ b/src/rgw/rgw_bucket.cc @@ -936,7 +936,19 @@ int RGWBucketAdminOp::link(rgw::sal::Store* store, RGWBucketAdminOpState& op_sta if (*loc_bucket != *old_bucket) { // like RGWRados::delete_bucket -- excepting no bucket_index work. - r = old_bucket->remove_metadata(dpp, &ep_data.ep_objv, null_yield); + r = static_cast(store)->ctl()->bucket->remove_bucket_entrypoint_info( + old_bucket->get_key(), null_yield, dpp, + RGWBucketCtl::Bucket::RemoveParams() + .set_objv_tracker(&ep_data.ep_objv)); + if (r < 0) { + set_err_msg(err, "failed to unlink old bucket " + old_bucket->get_tenant() + "/" + old_bucket->get_name()); + return r; + } + r = static_cast(store)->ctl()->bucket->remove_bucket_instance_info( + old_bucket->get_key(), old_bucket->get_info(), + null_yield, dpp, + RGWBucketCtl::BucketInstance::RemoveParams() + .set_objv_tracker(&ep_data.ep_objv)); if (r < 0) { set_err_msg(err, "failed to unlink old bucket " + old_bucket->get_tenant() + "/" + old_bucket->get_name()); return r; @@ -1053,7 +1065,7 @@ static int bucket_stats(rgw::sal::Store* store, string bucket_ver, master_ver; string max_marker; - ret = bucket->get_bucket_stats(dpp, RGW_NO_SHARD, &bucket_ver, &master_ver, stats, &max_marker); + ret = bucket->read_stats(dpp, RGW_NO_SHARD, &bucket_ver, &master_ver, stats, &max_marker); if (ret < 0) { cerr << "error getting bucket stats bucket=" << bucket->get_name() << " ret=" << ret << std::endl; return ret; @@ -1153,14 +1165,14 @@ int RGWBucketAdminOp::limit_check(rgw::sal::Store* store, * as we may now not reach the end of * the loop body */ - ret = bucket->get_bucket_info(dpp, null_yield); + ret = bucket->load_bucket(dpp, null_yield); if (ret < 0) continue; /* need stats for num_entries */ string bucket_ver, master_ver; std::map stats; - ret = bucket->get_bucket_stats(dpp, RGW_NO_SHARD, &bucket_ver, &master_ver, stats, nullptr); + ret = bucket->read_stats(dpp, RGW_NO_SHARD, &bucket_ver, &master_ver, stats, nullptr); if (ret < 0) continue; diff --git a/src/rgw/rgw_lc.cc b/src/rgw/rgw_lc.cc index a47799f931413..bd6d997ed9e1d 100644 --- a/src/rgw/rgw_lc.cc +++ b/src/rgw/rgw_lc.cc @@ -844,7 +844,7 @@ int RGWLC::handle_multipart_expiration(rgw::sal::Bucket* target, auto& [rule, obj] = wt; if (obj_has_expired(this, cct, obj.meta.mtime, rule.mp_expiration)) { rgw_obj_key key(obj.key); - std::unique_ptr mpu = store->get_multipart_upload(target, key.name); + std::unique_ptr mpu = target->get_multipart_upload(key.name); RGWObjectCtx rctx(store); int ret = mpu->abort(this, cct, &rctx); if (ret == 0) { @@ -1446,9 +1446,9 @@ int RGWLC::bucket_lc_process(string& shard_id, LCWorker* worker, return ret; } - ret = bucket->get_bucket_info(this, null_yield); + ret = bucket->load_bucket(this, null_yield); if (ret < 0) { - ldpp_dout(this, 0) << "LC:get_bucket_info for " << bucket_name + ldpp_dout(this, 0) << "LC:load_bucket for " << bucket_name << " failed" << dendl; return ret; } diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc index f295fb92d5761..14672bd9e3410 100644 --- a/src/rgw/rgw_op.cc +++ b/src/rgw/rgw_op.cc @@ -389,7 +389,7 @@ static int read_obj_policy(const DoutPrefixProvider *dpp, if (!upload_id.empty() && !copy_src) { /* multipart upload */ std::unique_ptr upload; - upload = store->get_multipart_upload(bucket, object->get_name(), upload_id); + upload = bucket->get_multipart_upload(object->get_name(), upload_id); mpobj = upload->get_meta_obj(); mpobj->set_in_extra_data(true); object = mpobj.get(); @@ -1900,7 +1900,7 @@ int RGWGetObj::handle_user_manifest(const char *prefix, optional_yield y) } s->object->set_obj_size(s->obj_size); - r = RGWRados::Object::Read::range_to_ofs(s->obj_size, ofs, end); + r = s->object->range_to_ofs(s->obj_size, ofs, end); if (r < 0) { return r; } @@ -2049,7 +2049,7 @@ int RGWGetObj::handle_slo_manifest(bufferlist& bl, optional_yield y) s->object->set_obj_size(slo_info.total_size); ldpp_dout(this, 20) << "s->obj_size=" << s->obj_size << dendl; - int r = RGWRados::Object::Read::range_to_ofs(total_len, ofs, end); + int r = s->object->range_to_ofs(total_len, ofs, end); if (r < 0) { return r; } @@ -2827,7 +2827,6 @@ void RGWStatBucket::execute(optional_yield y) if (op_ret) { return; } - op_ret = bucket->update_container_stats(s); } int RGWListBucket::verify_permission(optional_yield y) @@ -3276,7 +3275,7 @@ void RGWCreateBucket::execute(optional_yield y) /* We're replacing bucket with the newly created one */ ldpp_dout(this, 10) << "user=" << s->user << " bucket=" << tmp_bucket << dendl; - op_ret = store->create_bucket(this, s->user.get(), tmp_bucket, zonegroup_id, + op_ret = s->user->create_bucket(this, tmp_bucket, zonegroup_id, placement_rule, info.swift_ver_location, pquota_info, policy, attrs, info, ep_objv, @@ -3300,7 +3299,7 @@ void RGWCreateBucket::execute(optional_yield y) do { map battrs; - op_ret = s->bucket->get_bucket_info(this, y); + op_ret = s->bucket->load_bucket(this, y); if (op_ret < 0) { return; } else if (!s->bucket->is_owner(s->user.get())) { @@ -3492,9 +3491,9 @@ int RGWPutObj::init_processing(optional_yield y) { return ret; } - ret = bucket->get_bucket_info(this, y); + ret = bucket->load_bucket(this, y); if (ret < 0) { - ldpp_dout(this, 5) << __func__ << "(): get_bucket_info() returned ret=" << ret << dendl; + ldpp_dout(this, 5) << __func__ << "(): load_bucket() returned ret=" << ret << dendl; return ret; } copy_source_bucket_info = bucket->get_info(); @@ -3926,7 +3925,7 @@ void RGWPutObj::execute(optional_yield y) if (multipart) { s->trace->SetTag(tracing::UPLOAD_ID, multipart_upload_id); std::unique_ptr upload; - upload = store->get_multipart_upload(s->bucket.get(), s->object->get_name(), + upload = s->bucket->get_multipart_upload(s->object->get_name(), multipart_upload_id); op_ret = upload->get_info(this, s->yield, s->obj_ctx, &pdest_placement); if (op_ret < 0) { @@ -6151,7 +6150,7 @@ void RGWInitMultipart::execute(optional_yield y) } std::unique_ptr upload; - upload = store->get_multipart_upload(s->bucket.get(), s->object->get_name(), + upload = s->bucket->get_multipart_upload(s->object->get_name(), upload_id); op_ret = upload->init(this, s->yield, s->obj_ctx, s->owner, s->dest_placement, attrs); @@ -6278,7 +6277,7 @@ void RGWCompleteMultipart::execute(optional_yield y) return; } - upload = store->get_multipart_upload(s->bucket.get(), s->object->get_name(), upload_id); + upload = s->bucket->get_multipart_upload(s->object->get_name(), upload_id); s->trace->SetTag(tracing::UPLOAD_ID, upload_id); @@ -6495,7 +6494,7 @@ void RGWAbortMultipart::execute(optional_yield y) if (upload_id.empty() || rgw::sal::Object::empty(s->object.get())) return; - upload = store->get_multipart_upload(s->bucket.get(), s->object->get_name(), upload_id); + upload = s->bucket->get_multipart_upload(s->object->get_name(), upload_id); RGWObjectCtx *obj_ctx = static_cast(s->obj_ctx); op_ret = upload->abort(this, s->cct, obj_ctx); } @@ -6523,7 +6522,7 @@ void RGWListMultipart::execute(optional_yield y) if (op_ret < 0) return; - upload = store->get_multipart_upload(s->bucket.get(), s->object->get_name(), upload_id); + upload = s->bucket->get_multipart_upload(s->object->get_name(), upload_id); rgw::sal::Attrs attrs; op_ret = upload->get_info(this, s->yield, s->obj_ctx, nullptr, &attrs); @@ -6961,7 +6960,7 @@ bool RGWBulkDelete::Deleter::delete_single(const acct_path_t& path, optional_yie goto binfo_fail; } - ret = bucket->get_bucket_info(dpp, s->yield); + ret = bucket->load_bucket(dpp, s->yield); if (ret < 0) { goto binfo_fail; } @@ -7241,7 +7240,7 @@ int RGWBulkUploadOp::handle_dir(const std::string_view path, optional_yield y) placement_rule.storage_class = s->info.storage_class; forward_req_info(this, s->cct, info, bucket_name); - op_ret = store->create_bucket(this, s->user.get(), new_bucket, + op_ret = s->user->create_bucket(this, new_bucket, store->get_zone()->get_zonegroup().get_id(), placement_rule, swift_ver_location, pquota_info, policy, attrs, diff --git a/src/rgw/rgw_op.h b/src/rgw/rgw_op.h index 7556a55b5d638..fa2c69fab572e 100644 --- a/src/rgw/rgw_op.h +++ b/src/rgw/rgw_op.h @@ -855,7 +855,7 @@ class RGWGetUsage : public RGWOp { std::map categories; std::map usage; std::map summary_map; - std::map buckets_usage; + std::map buckets_usage; cls_user_header header; RGWStorageStats stats; public: diff --git a/src/rgw/rgw_quota.cc b/src/rgw/rgw_quota.cc index bc74b485ad936..1cbb06ae3a6c2 100644 --- a/src/rgw/rgw_quota.cc +++ b/src/rgw/rgw_quota.cc @@ -273,11 +273,11 @@ int BucketAsyncRefreshHandler::init_fetch() ldpp_dout(&dp, 20) << "initiating async quota refresh for bucket=" << bucket << dendl; - r = rbucket->get_bucket_stats_async(&dp, RGW_NO_SHARD, this); + r = rbucket->read_stats_async(&dp, RGW_NO_SHARD, this); if (r < 0) { ldpp_dout(&dp, 0) << "could not get bucket info for bucket=" << bucket.name << dendl; - /* get_bucket_stats_async() dropped our reference already */ + /* read_stats_async() dropped our reference already */ return r; } @@ -345,7 +345,7 @@ int RGWBucketStatsCache::fetch_stats_from_storage(const rgw_user& _u, const rgw_ string master_ver; map bucket_stats; - r = bucket->get_bucket_stats(dpp, RGW_NO_SHARD, &bucket_ver, &master_ver, bucket_stats); + r = bucket->read_stats(dpp, RGW_NO_SHARD, &bucket_ver, &master_ver, bucket_stats); if (r < 0) { ldpp_dout(dpp, 0) << "could not get bucket stats for bucket=" << _b.name << dendl; diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc index e5d452063c369..bfab1e0f1de7b 100644 --- a/src/rgw/rgw_rados.cc +++ b/src/rgw/rgw_rados.cc @@ -3643,8 +3643,27 @@ static void set_copy_attrs(map& src_attrs, int RGWRados::rewrite_obj(rgw::sal::Object* obj, const DoutPrefixProvider *dpp, optional_yield y) { RGWObjectCtx rctx(this->store); + rgw::sal::Attrs attrset; + uint64_t obj_size; + ceph::real_time mtime; + RGWRados::Object op_target(this, obj->get_bucket()->get_info(), rctx, obj->get_obj()); + RGWRados::Object::Read read_op(&op_target); + + read_op.params.attrs = &attrset; + read_op.params.obj_size = &obj_size; + read_op.params.lastmod = &mtime; + + int ret = read_op.prepare(y, dpp); + if (ret < 0) + return ret; + + attrset.erase(RGW_ATTR_ID_TAG); + attrset.erase(RGW_ATTR_TAIL_TAG); - return obj->copy_obj_data(rctx, obj->get_bucket(), obj, 0, NULL, dpp, y); + return store->getRados()->copy_obj_data(rctx, obj->get_bucket(), + obj->get_bucket()->get_info().placement_rule, + read_op, obj_size - 1, obj, NULL, mtime, + attrset, 0, real_time(), NULL, dpp, y); } struct obj_time_weight { diff --git a/src/rgw/rgw_rest.cc b/src/rgw/rgw_rest.cc index 214c9f27b8bba..51e89fd10a48b 100644 --- a/src/rgw/rgw_rest.cc +++ b/src/rgw/rgw_rest.cc @@ -1615,7 +1615,7 @@ int RGWListBucketMultiparts_ObjStore::get_params(optional_yield y) string upload_id_marker = s->info.args.get("upload-id-marker"); if (!key_marker.empty()) { std::unique_ptr upload; - upload = store->get_multipart_upload(s->bucket.get(), key_marker, + upload = s->bucket->get_multipart_upload(key_marker, upload_id_marker); marker_meta = upload->get_meta(); marker_key = upload->get_key(); diff --git a/src/rgw/rgw_rest_log.cc b/src/rgw/rgw_rest_log.cc index 348c3952dbfb1..f2105f85f0f72 100644 --- a/src/rgw/rgw_rest_log.cc +++ b/src/rgw/rgw_rest_log.cc @@ -484,7 +484,7 @@ void RGWOp_BILog_Info::execute(optional_yield y) { } map stats; - int ret = bucket->get_bucket_stats(s, shard_id, &bucket_ver, &master_ver, stats, &max_marker, &syncstopped); + int ret = bucket->read_stats(s, shard_id, &bucket_ver, &master_ver, stats, &max_marker, &syncstopped); if (ret < 0 && ret != -ENOENT) { op_ret = ret; return; diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc index e3b40bc75a64e..baeb0ad6eb4c6 100644 --- a/src/rgw/rgw_rest_s3.cc +++ b/src/rgw/rgw_rest_s3.cc @@ -1328,7 +1328,7 @@ static void dump_usage_categories_info(Formatter *formatter, const rgw_usage_log formatter->close_section(); // Category } -static void dump_usage_bucket_info(Formatter *formatter, const std::string& name, const cls_user_bucket_entry& entry) +static void dump_usage_bucket_info(Formatter *formatter, const std::string& name, const bucket_meta_entry& entry) { formatter->open_object_section("Entry"); encode_json("Bucket", name, formatter); @@ -1441,7 +1441,7 @@ void RGWGetUsage_ObjStore_S3::send_response() formatter->open_object_section("User"); formatter->open_array_section("Buckets"); for (const auto& biter : buckets_usage) { - const cls_user_bucket_entry& entry = biter.second; + const bucket_meta_entry& entry = biter.second; dump_usage_bucket_info(formatter, biter.first, entry); } formatter->close_section(); // Buckets @@ -2581,7 +2581,7 @@ int RGWPutObj_ObjStore_S3::get_encrypt_filter( int res = 0; if (!multipart_upload_id.empty()) { std::unique_ptr upload = - store->get_multipart_upload(s->bucket.get(), s->object->get_name(), + s->bucket->get_multipart_upload(s->object->get_name(), multipart_upload_id); std::unique_ptr obj = upload->get_meta_obj(); obj->set_in_extra_data(true); @@ -4023,7 +4023,7 @@ void RGWGetObjLayout_ObjStore_S3::send_response() } f.open_object_section("result"); - s->object->get_obj_layout(this, s->yield, &f, s->obj_ctx); + s->object->dump_obj_layout(this, s->yield, &f, s->obj_ctx); f.close_section(); rgw_flush_formatter(s, &f); } diff --git a/src/rgw/rgw_sal.h b/src/rgw/rgw_sal.h index 23d2a43aa7ad3..2c1141ae1ec65 100644 --- a/src/rgw/rgw_sal.h +++ b/src/rgw/rgw_sal.h @@ -183,28 +183,10 @@ class Store { virtual int get_bucket(const DoutPrefixProvider* dpp, User* u, const rgw_bucket& b, std::unique_ptr* bucket, optional_yield y) = 0; virtual int get_bucket(User* u, const RGWBucketInfo& i, std::unique_ptr* bucket) = 0; virtual int get_bucket(const DoutPrefixProvider* dpp, User* u, const std::string& tenant, const std::string& name, std::unique_ptr* bucket, optional_yield y) = 0; - virtual int create_bucket(const DoutPrefixProvider* dpp, - User* u, const rgw_bucket& b, - const std::string& zonegroup_id, - rgw_placement_rule& placement_rule, - std::string& swift_ver_location, - const RGWQuotaInfo* pquota_info, - const RGWAccessControlPolicy& policy, - Attrs& attrs, - RGWBucketInfo& info, - obj_version& ep_objv, - bool exclusive, - bool obj_lock_enabled, - bool* existed, - req_info& req_info, - std::unique_ptr* bucket, - optional_yield y) = 0; virtual bool is_meta_master() = 0; virtual int forward_request_to_master(const DoutPrefixProvider *dpp, User* user, obj_version* objv, bufferlist& in_data, JSONParser* jp, req_info& info, optional_yield y) = 0; - virtual int defer_gc(const DoutPrefixProvider* dpp, RGWObjectCtx* rctx, Bucket* bucket, Object* obj, - optional_yield y) = 0; virtual Zone* get_zone() = 0; virtual std::string zone_unique_id(uint64_t unique_num) = 0; virtual std::string zone_unique_trans_id(const uint64_t unique_num) = 0; @@ -262,10 +244,6 @@ class Store { virtual int get_oidc_providers(const DoutPrefixProvider *dpp, const std::string& tenant, std::vector>& providers) = 0; - virtual std::unique_ptr - get_multipart_upload(Bucket* bucket, const std::string& oid, - std::optional upload_id=std::nullopt, - ACLOwner owner={}, ceph::real_time mtime=real_clock::now()) = 0; virtual std::unique_ptr get_append_writer(const DoutPrefixProvider *dpp, optional_yield y, std::unique_ptr _head_obj, @@ -310,7 +288,23 @@ class User { const std::string& marker, const std::string& end_marker, uint64_t max, bool need_stats, BucketList& buckets, optional_yield y) = 0; - virtual Bucket* create_bucket(rgw_bucket& bucket, ceph::real_time creation_time) = 0; + /** Create a new bucket owned by this user. Creates in the backing store, not just the instantiation. */ + virtual int create_bucket(const DoutPrefixProvider* dpp, + const rgw_bucket& b, + const std::string& zonegroup_id, + rgw_placement_rule& placement_rule, + std::string& swift_ver_location, + const RGWQuotaInfo* pquota_info, + const RGWAccessControlPolicy& policy, + Attrs& attrs, + RGWBucketInfo& info, + obj_version& ep_objv, + bool exclusive, + bool obj_lock_enabled, + bool* existed, + req_info& req_info, + std::unique_ptr* bucket, + optional_yield y) = 0; friend class Bucket; virtual std::string& get_display_name() { return info.display_name; } @@ -437,20 +431,18 @@ class Bucket { DoutPrefixProvider *dpp) = 0; virtual RGWAccessControlPolicy& get_acl(void) = 0; virtual int set_acl(const DoutPrefixProvider* dpp, RGWAccessControlPolicy& acl, optional_yield y) = 0; - virtual int get_bucket_info(const DoutPrefixProvider* dpp, optional_yield y) = 0; - virtual int get_bucket_stats(const DoutPrefixProvider *dpp, int shard_id, + virtual int load_bucket(const DoutPrefixProvider* dpp, optional_yield y) = 0; + virtual int read_stats(const DoutPrefixProvider *dpp, int shard_id, std::string* bucket_ver, std::string* master_ver, std::map& stats, std::string* max_marker = nullptr, bool* syncstopped = nullptr) = 0; - virtual int get_bucket_stats_async(const DoutPrefixProvider *dpp, int shard_id, RGWGetBucketStats_CB* ctx) = 0; - virtual int read_bucket_stats(const DoutPrefixProvider* dpp, optional_yield y) = 0; + virtual int read_stats_async(const DoutPrefixProvider *dpp, int shard_id, RGWGetBucketStats_CB* ctx) = 0; virtual int sync_user_stats(const DoutPrefixProvider *dpp, optional_yield y) = 0; virtual int update_container_stats(const DoutPrefixProvider* dpp) = 0; virtual int check_bucket_shards(const DoutPrefixProvider* dpp) = 0; virtual int chown(const DoutPrefixProvider* dpp, User* new_user, User* old_user, optional_yield y, const std::string* marker = nullptr) = 0; virtual int put_info(const DoutPrefixProvider* dpp, bool exclusive, ceph::real_time mtime) = 0; - virtual int remove_metadata(const DoutPrefixProvider* dpp, RGWObjVersionTracker* objv, optional_yield y) = 0; virtual bool is_owner(User* user) = 0; virtual User* get_owner(void) { return owner; }; virtual ACLOwner get_acl_owner(void) { return ACLOwner(info.owner); }; @@ -491,13 +483,13 @@ class Bucket { bool versioned() { return info.versioned(); } bool versioning_enabled() { return info.versioning_enabled(); } - void convert(cls_user_bucket_entry* b) const { - ent.convert(b); - } - static bool empty(Bucket* b) { return (!b || b->empty()); } virtual std::unique_ptr clone() = 0; + virtual std::unique_ptr get_multipart_upload( + const std::string& oid, + std::optional upload_id=std::nullopt, + ACLOwner owner={}, ceph::real_time mtime=real_clock::now()) = 0; virtual int list_multiparts(const DoutPrefixProvider *dpp, const std::string& prefix, std::string& marker, @@ -703,7 +695,6 @@ class Object { virtual int get_obj_attrs(RGWObjectCtx* rctx, optional_yield y, const DoutPrefixProvider* dpp, rgw_obj* target_obj = NULL) = 0; virtual int modify_obj_attrs(RGWObjectCtx* rctx, const char* attr_name, bufferlist& attr_val, optional_yield y, const DoutPrefixProvider* dpp) = 0; virtual int delete_obj_attrs(const DoutPrefixProvider* dpp, RGWObjectCtx* rctx, const char* attr_name, optional_yield y) = 0; - virtual int copy_obj_data(RGWObjectCtx& rctx, Bucket* dest_bucket, Object* dest_obj, uint16_t olh_epoch, std::string* petag, const DoutPrefixProvider* dpp, optional_yield y) = 0; virtual bool is_expired() = 0; virtual void gen_rand_obj_instance_name() = 0; virtual MPSerializer* get_serializer(const DoutPrefixProvider *dpp, const std::string& lock_name) = 0; @@ -715,7 +706,7 @@ class Object { const DoutPrefixProvider* dpp, optional_yield y) = 0; virtual bool placement_rules_match(rgw_placement_rule& r1, rgw_placement_rule& r2) = 0; - virtual int get_obj_layout(const DoutPrefixProvider *dpp, optional_yield y, Formatter* f, RGWObjectCtx* obj_ctx) = 0; + virtual int dump_obj_layout(const DoutPrefixProvider *dpp, optional_yield y, Formatter* f, RGWObjectCtx* obj_ctx) = 0; Attrs& get_attrs(void) { return attrs; } const Attrs& get_attrs(void) const { return attrs; } diff --git a/src/rgw/rgw_sal_dbstore.cc b/src/rgw/rgw_sal_dbstore.cc index a4c6bff9f458e..bc1934c9df746 100644 --- a/src/rgw/rgw_sal_dbstore.cc +++ b/src/rgw/rgw_sal_dbstore.cc @@ -52,10 +52,103 @@ namespace rgw::sal { return 0; } - Bucket* DBUser::create_bucket(rgw_bucket& bucket, - ceph::real_time creation_time) + int DBUser::create_bucket(const DoutPrefixProvider *dpp, + const rgw_bucket& b, + const string& zonegroup_id, + rgw_placement_rule& placement_rule, + string& swift_ver_location, + const RGWQuotaInfo * pquota_info, + const RGWAccessControlPolicy& policy, + Attrs& attrs, + RGWBucketInfo& info, + obj_version& ep_objv, + bool exclusive, + bool obj_lock_enabled, + bool *existed, + req_info& req_info, + std::unique_ptr* bucket_out, + optional_yield y) { - return NULL; + int ret; + bufferlist in_data; + RGWBucketInfo master_info; + rgw_bucket *pmaster_bucket = nullptr; + uint32_t *pmaster_num_shards = nullptr; + real_time creation_time; + std::unique_ptr bucket; + obj_version objv, *pobjv = NULL; + + /* If it exists, look it up; otherwise create it */ + ret = store->get_bucket(dpp, this, b, &bucket, y); + if (ret < 0 && ret != -ENOENT) + return ret; + + if (ret != -ENOENT) { + RGWAccessControlPolicy old_policy(store->ctx()); + *existed = true; + if (swift_ver_location.empty()) { + swift_ver_location = bucket->get_info().swift_ver_location; + } + placement_rule.inherit_from(bucket->get_info().placement_rule); + + // don't allow changes to the acl policy + /* int r = rgw_op_get_bucket_policy_from_attr(dpp, this, this, bucket->get_attrs(), + &old_policy, y); + if (r >= 0 && old_policy != policy) { + bucket_out->swap(bucket); + return -EEXIST; + }*/ + } else { + bucket = std::make_unique(store, b, this); + *existed = false; + bucket->set_attrs(attrs); + // XXX: For now single default zone and STANDARD storage class + // supported. + placement_rule.name = "default"; + placement_rule.storage_class = "STANDARD"; + } + + /* + * XXX: If not master zone, fwd the request to master zone. + * For now DBStore has single zone. + */ + std::string zid = zonegroup_id; + /* if (zid.empty()) { + zid = svc()->zone->get_zonegroup().get_id(); + } */ + + if (*existed) { + rgw_placement_rule selected_placement_rule; + /* XXX: Handle this when zone is implemented + ret = svc()->zone->select_bucket_placement(this.get_info(), + zid, placement_rule, + &selected_placement_rule, nullptr, y); + if (selected_placement_rule != info.placement_rule) { + ret = -EEXIST; + bucket_out->swap(bucket); + return ret; + } */ + } else { + + /* XXX: We may not need to send all these params. Cleanup the unused ones */ + ret = store->getDB()->create_bucket(dpp, this->get_info(), bucket->get_key(), + zid, placement_rule, swift_ver_location, pquota_info, + attrs, info, pobjv, &ep_objv, creation_time, + pmaster_bucket, pmaster_num_shards, y, exclusive); + if (ret == -EEXIST) { + *existed = true; + ret = 0; + } else if (ret != 0) { + return ret; + } + } + + bucket->set_version(ep_objv); + bucket->get_info() = info; + + bucket_out->swap(bucket); + + return ret; } int DBUser::read_attrs(const DoutPrefixProvider* dpp, optional_yield y) @@ -129,7 +222,7 @@ namespace rgw::sal { { int ret; - ret = get_bucket_info(dpp, y); + ret = load_bucket(dpp, y); if (ret < 0) return ret; @@ -147,7 +240,7 @@ namespace rgw::sal { return 0; } - int DBBucket::get_bucket_info(const DoutPrefixProvider *dpp, optional_yield y) + int DBBucket::load_bucket(const DoutPrefixProvider *dpp, optional_yield y) { int ret = 0; @@ -158,7 +251,7 @@ namespace rgw::sal { } /* stats - Not for first pass */ - int DBBucket::get_bucket_stats(const DoutPrefixProvider *dpp, int shard_id, + int DBBucket::read_stats(const DoutPrefixProvider *dpp, int shard_id, std::string *bucket_ver, std::string *master_ver, std::map& stats, std::string *max_marker, bool *syncstopped) @@ -166,12 +259,7 @@ namespace rgw::sal { return 0; } - int DBBucket::get_bucket_stats_async(const DoutPrefixProvider *dpp, int shard_id, RGWGetBucketStats_CB *ctx) - { - return 0; - } - - int DBBucket::read_bucket_stats(const DoutPrefixProvider *dpp, optional_yield y) + int DBBucket::read_stats_async(const DoutPrefixProvider *dpp, int shard_id, RGWGetBucketStats_CB *ctx) { return 0; } @@ -211,16 +299,6 @@ namespace rgw::sal { } - int DBBucket::remove_metadata(const DoutPrefixProvider* dpp, RGWObjVersionTracker* objv, optional_yield y) - { - /* XXX: same as DBBUcket::remove_bucket() but should return error if there are objects - * in that bucket. */ - - int ret = store->getDB()->remove_bucket(dpp, info); - - return ret; - } - /* Make sure to call get_bucket_info() if you need it first */ bool DBBucket::is_owner(User* user) { @@ -364,6 +442,13 @@ namespace rgw::sal { return ret; } + std::unique_ptr DBBucket::get_multipart_upload( + const std::string& oid, + std::optional upload_id, + ACLOwner owner, ceph::real_time mtime) { + return nullptr; + } + int DBBucket::list_multiparts(const DoutPrefixProvider *dpp, const string& prefix, string& marker, @@ -502,16 +587,6 @@ namespace rgw::sal { return set_obj_attrs(dpp, rctx, nullptr, &rmattr, y, &target); } - int DBObject::copy_obj_data(RGWObjectCtx& rctx, Bucket* dest_bucket, - Object* dest_obj, - uint16_t olh_epoch, - std::string* petag, - const DoutPrefixProvider* dpp, - optional_yield y) - { - return 0; - } - /* RGWObjectCtx will be moved out of sal */ /* XXX: Placeholder. Should not be needed later after Dan's patch */ void DBObject::set_atomic(RGWObjectCtx* rctx) const @@ -599,7 +674,7 @@ namespace rgw::sal { return true; } - int DBObject::get_obj_layout(const DoutPrefixProvider *dpp, optional_yield y, Formatter* f, RGWObjectCtx* obj_ctx) + int DBObject::dump_obj_layout(const DoutPrefixProvider *dpp, optional_yield y, Formatter* f, RGWObjectCtx* obj_ctx) { return 0; } @@ -947,13 +1022,6 @@ namespace rgw::sal { return 0; } - std::unique_ptr - DBStore::get_multipart_upload(Bucket* bucket, const std::string& oid, - std::optional upload_id, - ACLOwner owner, ceph::real_time mtime) { - return nullptr; - } - std::unique_ptr DBStore::get_append_writer(const DoutPrefixProvider *dpp, optional_yield y, std::unique_ptr _head_obj, @@ -1048,7 +1116,7 @@ namespace rgw::sal { Bucket* bp; bp = new DBBucket(this, b, u); - ret = bp->get_bucket_info(dpp, y); + ret = bp->load_bucket(dpp, y); if (ret < 0) { delete bp; return ret; @@ -1079,105 +1147,6 @@ namespace rgw::sal { return get_bucket(dpp, u, b, bucket, y); } - int DBStore::create_bucket(const DoutPrefixProvider *dpp, - User* u, const rgw_bucket& b, - const string& zonegroup_id, - rgw_placement_rule& placement_rule, - string& swift_ver_location, - const RGWQuotaInfo * pquota_info, - const RGWAccessControlPolicy& policy, - Attrs& attrs, - RGWBucketInfo& info, - obj_version& ep_objv, - bool exclusive, - bool obj_lock_enabled, - bool *existed, - req_info& req_info, - std::unique_ptr* bucket_out, - optional_yield y) - { - int ret; - bufferlist in_data; - RGWBucketInfo master_info; - rgw_bucket *pmaster_bucket = nullptr; - uint32_t *pmaster_num_shards = nullptr; - real_time creation_time; - std::unique_ptr bucket; - obj_version objv, *pobjv = NULL; - - /* If it exists, look it up; otherwise create it */ - ret = get_bucket(dpp, u, b, &bucket, y); - if (ret < 0 && ret != -ENOENT) - return ret; - - if (ret != -ENOENT) { - RGWAccessControlPolicy old_policy(ctx()); - *existed = true; - if (swift_ver_location.empty()) { - swift_ver_location = bucket->get_info().swift_ver_location; - } - placement_rule.inherit_from(bucket->get_info().placement_rule); - - // don't allow changes to the acl policy - /* int r = rgw_op_get_bucket_policy_from_attr(dpp, this, u, bucket->get_attrs(), - &old_policy, y); - if (r >= 0 && old_policy != policy) { - bucket_out->swap(bucket); - return -EEXIST; - }*/ - } else { - bucket = std::make_unique(this, b, u); - *existed = false; - bucket->set_attrs(attrs); - // XXX: For now single default zone and STANDARD storage class - // supported. - placement_rule.name = "default"; - placement_rule.storage_class = "STANDARD"; - } - - /* - * XXX: If not master zone, fwd the request to master zone. - * For now DBStore has single zone. - */ - std::string zid = zonegroup_id; - /* if (zid.empty()) { - zid = svc()->zone->get_zonegroup().get_id(); - } */ - - if (*existed) { - rgw_placement_rule selected_placement_rule; - /* XXX: Handle this when zone is implemented - ret = svc()->zone->select_bucket_placement(u.get_info(), - zid, placement_rule, - &selected_placement_rule, nullptr, y); - if (selected_placement_rule != info.placement_rule) { - ret = -EEXIST; - bucket_out->swap(bucket); - return ret; - } */ - } else { - - /* XXX: We may not need to send all these params. Cleanup the unused ones */ - ret = getDB()->create_bucket(dpp, u->get_info(), bucket->get_key(), - zid, placement_rule, swift_ver_location, pquota_info, - attrs, info, pobjv, &ep_objv, creation_time, - pmaster_bucket, pmaster_num_shards, y, exclusive); - if (ret == -EEXIST) { - *existed = true; - ret = 0; - } else if (ret != 0) { - return ret; - } - } - - bucket->set_version(ep_objv); - bucket->get_info() = info; - - bucket_out->swap(bucket); - - return ret; - } - bool DBStore::is_meta_master() { return true; @@ -1191,11 +1160,6 @@ namespace rgw::sal { return 0; } - int DBStore::defer_gc(const DoutPrefixProvider *dpp, RGWObjectCtx *rctx, Bucket* bucket, Object* obj, optional_yield y) - { - return 0; - } - std::string DBStore::zone_unique_id(uint64_t unique_num) { return ""; diff --git a/src/rgw/rgw_sal_dbstore.h b/src/rgw/rgw_sal_dbstore.h index 16e6ce0608b9d..bb47485204343 100644 --- a/src/rgw/rgw_sal_dbstore.h +++ b/src/rgw/rgw_sal_dbstore.h @@ -56,7 +56,22 @@ class DBNotification : public Notification { } int list_buckets(const DoutPrefixProvider *dpp, const std::string& marker, const std::string& end_marker, uint64_t max, bool need_stats, BucketList& buckets, optional_yield y) override; - virtual Bucket* create_bucket(rgw_bucket& bucket, ceph::real_time creation_time) override; + virtual int create_bucket(const DoutPrefixProvider* dpp, + const rgw_bucket& b, + const std::string& zonegroup_id, + rgw_placement_rule& placement_rule, + std::string& swift_ver_location, + const RGWQuotaInfo* pquota_info, + const RGWAccessControlPolicy& policy, + Attrs& attrs, + RGWBucketInfo& info, + obj_version& ep_objv, + bool exclusive, + bool obj_lock_enabled, + bool* existed, + req_info& req_info, + std::unique_ptr* bucket, + optional_yield y) override; virtual int read_attrs(const DoutPrefixProvider* dpp, optional_yield y) override; virtual int read_stats(const DoutPrefixProvider *dpp, optional_yield y, RGWStorageStats* stats, @@ -141,20 +156,18 @@ class DBNotification : public Notification { DoutPrefixProvider *dpp) override; virtual RGWAccessControlPolicy& get_acl(void) override { return acls; } virtual int set_acl(const DoutPrefixProvider *dpp, RGWAccessControlPolicy& acl, optional_yield y) override; - virtual int get_bucket_info(const DoutPrefixProvider *dpp, optional_yield y) override; - virtual int get_bucket_stats(const DoutPrefixProvider *dpp, int shard_id, + virtual int load_bucket(const DoutPrefixProvider *dpp, optional_yield y) override; + virtual int read_stats(const DoutPrefixProvider *dpp, int shard_id, std::string *bucket_ver, std::string *master_ver, std::map& stats, std::string *max_marker = nullptr, bool *syncstopped = nullptr) override; - virtual int get_bucket_stats_async(const DoutPrefixProvider *dpp, int shard_id, RGWGetBucketStats_CB* ctx) override; - virtual int read_bucket_stats(const DoutPrefixProvider *dpp, optional_yield y) override; + virtual int read_stats_async(const DoutPrefixProvider *dpp, int shard_id, RGWGetBucketStats_CB* ctx) override; virtual int sync_user_stats(const DoutPrefixProvider *dpp, optional_yield y) override; virtual int update_container_stats(const DoutPrefixProvider *dpp) override; virtual int check_bucket_shards(const DoutPrefixProvider *dpp) override; virtual int chown(const DoutPrefixProvider *dpp, User* new_user, User* old_user, optional_yield y, const std::string* marker = nullptr) override; virtual int put_info(const DoutPrefixProvider *dpp, bool exclusive, ceph::real_time mtime) override; - virtual int remove_metadata(const DoutPrefixProvider* dpp, RGWObjVersionTracker* objv, optional_yield y) override; virtual bool is_owner(User* user) override; virtual int check_empty(const DoutPrefixProvider *dpp, optional_yield y) override; virtual int check_quota(const DoutPrefixProvider *dpp, RGWQuotaInfo& user_quota, RGWQuotaInfo& bucket_quota, uint64_t obj_size, optional_yield y, bool check_size_only = false) override; @@ -172,6 +185,9 @@ class DBNotification : public Notification { virtual std::unique_ptr clone() override { return std::make_unique(*this); } + virtual std::unique_ptr get_multipart_upload( + const std::string& oid, std::optional upload_id, + ACLOwner owner={}, ceph::real_time mtime=ceph::real_clock::now()) override; virtual int list_multiparts(const DoutPrefixProvider *dpp, const string& prefix, string& marker, @@ -344,7 +360,6 @@ class DBNotification : public Notification { virtual int get_obj_attrs(RGWObjectCtx* rctx, optional_yield y, const DoutPrefixProvider* dpp, rgw_obj* target_obj = NULL) override; virtual int modify_obj_attrs(RGWObjectCtx* rctx, const char* attr_name, bufferlist& attr_val, optional_yield y, const DoutPrefixProvider* dpp) override; virtual int delete_obj_attrs(const DoutPrefixProvider* dpp, RGWObjectCtx* rctx, const char* attr_name, optional_yield y) override; - virtual int copy_obj_data(RGWObjectCtx& rctx, Bucket* dest_bucket, Object* dest_obj, uint16_t olh_epoch, std::string* petag, const DoutPrefixProvider* dpp, optional_yield y) override; virtual bool is_expired() override; virtual void gen_rand_obj_instance_name() override; virtual std::unique_ptr clone() override { @@ -359,7 +374,7 @@ class DBNotification : public Notification { const DoutPrefixProvider* dpp, optional_yield y) override; virtual bool placement_rules_match(rgw_placement_rule& r1, rgw_placement_rule& r2) override; - virtual int get_obj_layout(const DoutPrefixProvider *dpp, optional_yield y, Formatter* f, RGWObjectCtx* obj_ctx) override; + virtual int dump_obj_layout(const DoutPrefixProvider *dpp, optional_yield y, Formatter* f, RGWObjectCtx* obj_ctx) override; /* Swift versioning */ virtual int swift_versioning_restore(RGWObjectCtx* obj_ctx, @@ -459,28 +474,10 @@ class DBNotification : public Notification { virtual int get_bucket(const DoutPrefixProvider *dpp, User* u, const rgw_bucket& b, std::unique_ptr* bucket, optional_yield y) override; virtual int get_bucket(User* u, const RGWBucketInfo& i, std::unique_ptr* bucket) override; virtual int get_bucket(const DoutPrefixProvider *dpp, User* u, const std::string& tenant, const std::string&name, std::unique_ptr* bucket, optional_yield y) override; - virtual int create_bucket(const DoutPrefixProvider* dpp, - User* u, const rgw_bucket& b, - const std::string& zonegroup_id, - rgw_placement_rule& placement_rule, - std::string& swift_ver_location, - const RGWQuotaInfo* pquota_info, - const RGWAccessControlPolicy& policy, - Attrs& attrs, - RGWBucketInfo& info, - obj_version& ep_objv, - bool exclusive, - bool obj_lock_enabled, - bool* existed, - req_info& req_info, - std::unique_ptr* bucket, - optional_yield y) override; virtual bool is_meta_master() override; virtual int forward_request_to_master(const DoutPrefixProvider *dpp, User* user, obj_version* objv, bufferlist& in_data, JSONParser *jp, req_info& info, optional_yield y) override; - virtual int defer_gc(const DoutPrefixProvider *dpp, RGWObjectCtx *rctx, Bucket* bucket, Object* obj, - optional_yield y) override; virtual Zone* get_zone() { return &zone; } virtual std::string zone_unique_id(uint64_t unique_num) override; virtual std::string zone_unique_trans_id(const uint64_t unique_num) override; @@ -540,10 +537,6 @@ class DBNotification : public Notification { virtual int get_oidc_providers(const DoutPrefixProvider *dpp, const std::string& tenant, vector>& providers) override; - virtual std::unique_ptr - get_multipart_upload(Bucket* bucket, const std::string& oid, - std::optional upload_id=std::nullopt, - ACLOwner owner={}, ceph::real_time mtime=ceph::real_clock::now()) override; virtual std::unique_ptr get_append_writer(const DoutPrefixProvider *dpp, optional_yield y, std::unique_ptr _head_obj, diff --git a/src/rgw/rgw_sal_rados.cc b/src/rgw/rgw_sal_rados.cc index bec5a57a01ee9..1bf8417d186a5 100644 --- a/src/rgw/rgw_sal_rados.cc +++ b/src/rgw/rgw_sal_rados.cc @@ -156,10 +156,143 @@ int RadosUser::list_buckets(const DoutPrefixProvider* dpp, const std::string& ma return 0; } -Bucket* RadosUser::create_bucket(rgw_bucket& bucket, - ceph::real_time creation_time) +int RadosUser::create_bucket(const DoutPrefixProvider* dpp, + const rgw_bucket& b, + const std::string& zonegroup_id, + rgw_placement_rule& placement_rule, + std::string& swift_ver_location, + const RGWQuotaInfo * pquota_info, + const RGWAccessControlPolicy& policy, + Attrs& attrs, + RGWBucketInfo& info, + obj_version& ep_objv, + bool exclusive, + bool obj_lock_enabled, + bool* existed, + req_info& req_info, + std::unique_ptr* bucket_out, + optional_yield y) { - return NULL; + int ret; + bufferlist in_data; + RGWBucketInfo master_info; + rgw_bucket* pmaster_bucket; + uint32_t* pmaster_num_shards; + real_time creation_time; + std::unique_ptr bucket; + obj_version objv,* pobjv = NULL; + + /* If it exists, look it up; otherwise create it */ + ret = store->get_bucket(dpp, this, b, &bucket, y); + if (ret < 0 && ret != -ENOENT) + return ret; + + if (ret != -ENOENT) { + RGWAccessControlPolicy old_policy(store->ctx()); + *existed = true; + if (swift_ver_location.empty()) { + swift_ver_location = bucket->get_info().swift_ver_location; + } + placement_rule.inherit_from(bucket->get_info().placement_rule); + + // don't allow changes to the acl policy + int r = rgw_op_get_bucket_policy_from_attr(dpp, store, this, bucket->get_attrs(), + &old_policy, y); + if (r >= 0 && old_policy != policy) { + bucket_out->swap(bucket); + return -EEXIST; + } + } else { + bucket = std::unique_ptr(new RadosBucket(store, b, this)); + *existed = false; + bucket->set_attrs(attrs); + } + + if (!store->svc()->zone->is_meta_master()) { + JSONParser jp; + ret = store->forward_request_to_master(dpp, this, NULL, in_data, &jp, req_info, y); + if (ret < 0) { + return ret; + } + + JSONDecoder::decode_json("entry_point_object_ver", ep_objv, &jp); + JSONDecoder::decode_json("object_ver", objv, &jp); + JSONDecoder::decode_json("bucket_info", master_info, &jp); + ldpp_dout(dpp, 20) << "parsed: objv.tag=" << objv.tag << " objv.ver=" << objv.ver << dendl; + std::time_t ctime = ceph::real_clock::to_time_t(master_info.creation_time); + ldpp_dout(dpp, 20) << "got creation time: << " << std::put_time(std::localtime(&ctime), "%F %T") << dendl; + pmaster_bucket= &master_info.bucket; + creation_time = master_info.creation_time; + pmaster_num_shards = &master_info.layout.current_index.layout.normal.num_shards; + pobjv = &objv; + if (master_info.obj_lock_enabled()) { + info.flags = BUCKET_VERSIONED | BUCKET_OBJ_LOCK_ENABLED; + } + } else { + pmaster_bucket = NULL; + pmaster_num_shards = NULL; + if (obj_lock_enabled) + info.flags = BUCKET_VERSIONED | BUCKET_OBJ_LOCK_ENABLED; + } + + std::string zid = zonegroup_id; + if (zid.empty()) { + zid = store->svc()->zone->get_zonegroup().get_id(); + } + + if (*existed) { + rgw_placement_rule selected_placement_rule; + ret = store->svc()->zone->select_bucket_placement(dpp, this->get_info(), + zid, placement_rule, + &selected_placement_rule, nullptr, y); + if (selected_placement_rule != info.placement_rule) { + ret = -EEXIST; + bucket_out->swap(bucket); + return ret; + } + } else { + + ret = store->getRados()->create_bucket(this->get_info(), bucket->get_key(), + zid, placement_rule, swift_ver_location, pquota_info, + attrs, info, pobjv, &ep_objv, creation_time, + pmaster_bucket, pmaster_num_shards, y, dpp, + exclusive); + if (ret == -EEXIST) { + *existed = true; + /* bucket already existed, might have raced with another bucket creation, + * or might be partial bucket creation that never completed. Read existing + * bucket info, verify that the reported bucket owner is the current user. + * If all is ok then update the user's list of buckets. Otherwise inform + * client about a name conflict. + */ + if (info.owner.compare(this->get_id()) != 0) { + return -EEXIST; + } + ret = 0; + } else if (ret != 0) { + return ret; + } + } + + bucket->set_version(ep_objv); + bucket->get_info() = info; + + RadosBucket* rbucket = static_cast(bucket.get()); + ret = rbucket->link(dpp, this, y, false); + if (ret && !*existed && ret != -EEXIST) { + /* if it exists (or previously existed), don't remove it! */ + ret = rbucket->unlink(dpp, this, y); + if (ret < 0) { + ldpp_dout(dpp, 0) << "WARNING: failed to unlink bucket: ret=" << ret + << dendl; + } + } else if (ret == -EEXIST || (ret == 0 && *existed)) { + ret = -ERR_BUCKET_EXISTS; + } + + bucket_out->swap(bucket); + + return ret; } int RadosUser::read_attrs(const DoutPrefixProvider* dpp, optional_yield y) @@ -235,7 +368,7 @@ int RadosBucket::remove_bucket(const DoutPrefixProvider* dpp, int ret; // Refresh info - ret = get_bucket_info(dpp, y); + ret = load_bucket(dpp, y); if (ret < 0) return ret; @@ -335,11 +468,11 @@ int RadosBucket::remove_bucket_bypass_gc(int concurrent_max, bool string bucket_ver, master_ver; - ret = get_bucket_info(dpp, null_yield); + ret = load_bucket(dpp, null_yield); if (ret < 0) return ret; - ret = get_bucket_stats(dpp, RGW_NO_SHARD, &bucket_ver, &master_ver, stats, NULL); + ret = read_stats(dpp, RGW_NO_SHARD, &bucket_ver, &master_ver, stats, NULL); if (ret < 0) return ret; @@ -455,7 +588,7 @@ int RadosBucket::remove_bucket_bypass_gc(int concurrent_max, bool return ret; } -int RadosBucket::get_bucket_info(const DoutPrefixProvider* dpp, optional_yield y) +int RadosBucket::load_bucket(const DoutPrefixProvider* dpp, optional_yield y) { auto obj_ctx = store->svc()->sysobj->init_obj_ctx(); int ret; @@ -476,15 +609,18 @@ int RadosBucket::get_bucket_info(const DoutPrefixProvider* dpp, optional_yield y .set_attrs(&attrs) .set_bectx_params(bectx_params)); } - if (ret == 0) { - bucket_version = ep_ot.read_version; - ent.placement_rule = info.placement_rule; - ent.bucket = info.bucket; // we looked up bucket_id + if (ret != 0) { + return ret; } + + bucket_version = ep_ot.read_version; + + ret = store->ctl()->bucket->read_bucket_stats(info.bucket, &ent, y, dpp); + return ret; } -int RadosBucket::get_bucket_stats(const DoutPrefixProvider *dpp, int shard_id, +int RadosBucket::read_stats(const DoutPrefixProvider *dpp, int shard_id, std::string* bucket_ver, std::string* master_ver, std::map& stats, std::string* max_marker, bool* syncstopped) @@ -492,18 +628,11 @@ int RadosBucket::get_bucket_stats(const DoutPrefixProvider *dpp, int shard_id, return store->getRados()->get_bucket_stats(dpp, info, shard_id, bucket_ver, master_ver, stats, max_marker, syncstopped); } -int RadosBucket::get_bucket_stats_async(const DoutPrefixProvider *dpp, int shard_id, RGWGetBucketStats_CB* ctx) +int RadosBucket::read_stats_async(const DoutPrefixProvider *dpp, int shard_id, RGWGetBucketStats_CB* ctx) { return store->getRados()->get_bucket_stats_async(dpp, get_info(), shard_id, ctx); } -int RadosBucket::read_bucket_stats(const DoutPrefixProvider* dpp, optional_yield y) -{ - int ret = store->ctl()->bucket->read_bucket_stats(info.bucket, &ent, y, dpp); - info.placement_rule = ent.placement_rule; - return ret; -} - int RadosBucket::sync_user_stats(const DoutPrefixProvider *dpp, optional_yield y) { return store->ctl()->bucket->sync_user_stats(dpp, owner->get_id(), info, y); @@ -594,19 +723,6 @@ int RadosBucket::put_info(const DoutPrefixProvider* dpp, bool exclusive, ceph::r return store->getRados()->put_bucket_instance_info(info, exclusive, mtime, &attrs, dpp); } -int RadosBucket::remove_metadata(const DoutPrefixProvider* dpp, RGWObjVersionTracker* objv, optional_yield y) -{ - int r = store->ctl()->bucket->remove_bucket_entrypoint_info(get_key(), y, dpp, - RGWBucketCtl::Bucket::RemoveParams() - .set_objv_tracker(objv)); - if (r < 0) - return r; - - return store->ctl()->bucket->remove_bucket_instance_info(get_key(), info, y, dpp, - RGWBucketCtl::BucketInstance::RemoveParams() - .set_objv_tracker(objv)); -} - /* Make sure to call get_bucket_info() if you need it first */ bool RadosBucket::is_owner(User* user) { @@ -750,6 +866,15 @@ int RadosBucket::list(const DoutPrefixProvider* dpp, ListParams& params, int max return ret; } +std::unique_ptr RadosBucket::get_multipart_upload( + const std::string& oid, + std::optional upload_id, + ACLOwner owner, ceph::real_time mtime) +{ + return std::make_unique(this->store, this, oid, upload_id, + std::move(owner), mtime); +} + int RadosBucket::list_multiparts(const DoutPrefixProvider *dpp, const string& prefix, string& marker, @@ -779,8 +904,8 @@ int RadosBucket::list_multiparts(const DoutPrefixProvider *dpp, rgw_obj_key key(dentry.key); ACLOwner owner(rgw_user(dentry.meta.owner)); owner.set_name(dentry.meta.owner_display_name); - uploads.push_back(store->get_multipart_upload( - this, key.name, std::nullopt, std::move(owner))); + uploads.push_back(this->get_multipart_upload(key.name, + std::nullopt, std::move(owner))); } } if (common_prefixes) { @@ -925,7 +1050,7 @@ int RadosStore::get_bucket(const DoutPrefixProvider* dpp, User* u, const rgw_buc Bucket* bp; bp = new RadosBucket(this, b, u); - ret = bp->get_bucket_info(dpp, y); + ret = bp->load_bucket(dpp, y); if (ret < 0) { delete bp; return ret; @@ -956,145 +1081,6 @@ int RadosStore::get_bucket(const DoutPrefixProvider* dpp, User* u, const std::st return get_bucket(dpp, u, b, bucket, y); } -int RadosStore::create_bucket(const DoutPrefixProvider* dpp, - User* u, const rgw_bucket& b, - const std::string& zonegroup_id, - rgw_placement_rule& placement_rule, - std::string& swift_ver_location, - const RGWQuotaInfo * pquota_info, - const RGWAccessControlPolicy& policy, - Attrs& attrs, - RGWBucketInfo& info, - obj_version& ep_objv, - bool exclusive, - bool obj_lock_enabled, - bool* existed, - req_info& req_info, - std::unique_ptr* bucket_out, - optional_yield y) -{ - int ret; - bufferlist in_data; - RGWBucketInfo master_info; - rgw_bucket* pmaster_bucket; - uint32_t* pmaster_num_shards; - real_time creation_time; - std::unique_ptr bucket; - obj_version objv,* pobjv = NULL; - - /* If it exists, look it up; otherwise create it */ - ret = get_bucket(dpp, u, b, &bucket, y); - if (ret < 0 && ret != -ENOENT) - return ret; - - if (ret != -ENOENT) { - RGWAccessControlPolicy old_policy(ctx()); - *existed = true; - if (swift_ver_location.empty()) { - swift_ver_location = bucket->get_info().swift_ver_location; - } - placement_rule.inherit_from(bucket->get_info().placement_rule); - - // don't allow changes to the acl policy - int r = rgw_op_get_bucket_policy_from_attr(dpp, this, u, bucket->get_attrs(), - &old_policy, y); - if (r >= 0 && old_policy != policy) { - bucket_out->swap(bucket); - return -EEXIST; - } - } else { - bucket = std::unique_ptr(new RadosBucket(this, b, u)); - *existed = false; - bucket->set_attrs(attrs); - } - - if (!svc()->zone->is_meta_master()) { - JSONParser jp; - ret = forward_request_to_master(dpp, u, NULL, in_data, &jp, req_info, y); - if (ret < 0) { - return ret; - } - - JSONDecoder::decode_json("entry_point_object_ver", ep_objv, &jp); - JSONDecoder::decode_json("object_ver", objv, &jp); - JSONDecoder::decode_json("bucket_info", master_info, &jp); - ldpp_dout(dpp, 20) << "parsed: objv.tag=" << objv.tag << " objv.ver=" << objv.ver << dendl; - std::time_t ctime = ceph::real_clock::to_time_t(master_info.creation_time); - ldpp_dout(dpp, 20) << "got creation time: << " << std::put_time(std::localtime(&ctime), "%F %T") << dendl; - pmaster_bucket= &master_info.bucket; - creation_time = master_info.creation_time; - pmaster_num_shards = &master_info.layout.current_index.layout.normal.num_shards; - pobjv = &objv; - if (master_info.obj_lock_enabled()) { - info.flags = BUCKET_VERSIONED | BUCKET_OBJ_LOCK_ENABLED; - } - } else { - pmaster_bucket = NULL; - pmaster_num_shards = NULL; - if (obj_lock_enabled) - info.flags = BUCKET_VERSIONED | BUCKET_OBJ_LOCK_ENABLED; - } - - std::string zid = zonegroup_id; - if (zid.empty()) { - zid = svc()->zone->get_zonegroup().get_id(); - } - - if (*existed) { - rgw_placement_rule selected_placement_rule; - ret = svc()->zone->select_bucket_placement(dpp, u->get_info(), - zid, placement_rule, - &selected_placement_rule, nullptr, y); - if (selected_placement_rule != info.placement_rule) { - ret = -EEXIST; - bucket_out->swap(bucket); - return ret; - } - } else { - - ret = getRados()->create_bucket(u->get_info(), bucket->get_key(), - zid, placement_rule, swift_ver_location, pquota_info, - attrs, info, pobjv, &ep_objv, creation_time, - pmaster_bucket, pmaster_num_shards, y, dpp, - exclusive); - if (ret == -EEXIST) { - *existed = true; - /* bucket already existed, might have raced with another bucket creation, - * or might be partial bucket creation that never completed. Read existing - * bucket info, verify that the reported bucket owner is the current user. - * If all is ok then update the user's list of buckets. Otherwise inform - * client about a name conflict. - */ - if (info.owner.compare(u->get_id()) != 0) { - return -EEXIST; - } - ret = 0; - } else if (ret != 0) { - return ret; - } - } - - bucket->set_version(ep_objv); - bucket->get_info() = info; - - RadosBucket* rbucket = static_cast(bucket.get()); - ret = rbucket->link(dpp, u, y, false); - if (ret && !*existed && ret != -EEXIST) { - /* if it exists (or previously existed), don't remove it! */ - ret = rbucket->unlink(dpp, u, y); - if (ret < 0) { - ldpp_dout(dpp, 0) << "WARNING: failed to unlink bucket: ret=" << ret - << dendl; - } - } else if (ret == -EEXIST || (ret == 0 && *existed)) { - ret = -ERR_BUCKET_EXISTS; - } - - bucket_out->swap(bucket); - - return ret; -} - bool RadosStore::is_meta_master() { return svc()->zone->is_meta_master(); @@ -1133,11 +1119,6 @@ int RadosStore::forward_request_to_master(const DoutPrefixProvider *dpp, User* u return 0; } -int RadosStore::defer_gc(const DoutPrefixProvider* dpp, RGWObjectCtx* rctx, Bucket* bucket, Object* obj, optional_yield y) -{ - return rados->defer_gc(dpp, rctx, bucket->get_info(), obj->get_obj(), y); -} - std::string RadosStore::zone_unique_id(uint64_t unique_num) { return svc()->zone_utils->unique_id(unique_num); @@ -1444,15 +1425,6 @@ int RadosStore::get_oidc_providers(const DoutPrefixProvider *dpp, return 0; } -std::unique_ptr -RadosStore::get_multipart_upload(Bucket* bucket, const std::string& oid, - std::optional upload_id, - ACLOwner owner, ceph::real_time mtime) -{ - return std::make_unique(this, bucket, oid, upload_id, - std::move(owner), mtime); -} - std::unique_ptr RadosStore::get_append_writer(const DoutPrefixProvider *dpp, optional_yield y, std::unique_ptr _head_obj, @@ -1555,32 +1527,6 @@ int RadosObject::delete_obj_attrs(const DoutPrefixProvider* dpp, RGWObjectCtx* r return set_obj_attrs(dpp, rctx, nullptr, &rmattr, y); } -int RadosObject::copy_obj_data(RGWObjectCtx& rctx, Bucket* dest_bucket, - Object* dest_obj, - uint16_t olh_epoch, - std::string* petag, - const DoutPrefixProvider* dpp, - optional_yield y) -{ - Attrs attrset; - RGWRados::Object op_target(store->getRados(), dest_bucket->get_info(), rctx, get_obj()); - RGWRados::Object::Read read_op(&op_target); - - int ret = read_attrs(dpp, read_op, y); - if (ret < 0) - return ret; - - attrset = attrs; - - attrset.erase(RGW_ATTR_ID_TAG); - attrset.erase(RGW_ATTR_TAIL_TAG); - - return store->getRados()->copy_obj_data(rctx, dest_bucket, - dest_bucket->get_info().placement_rule, read_op, - obj_size - 1, dest_obj, NULL, mtime, attrset, 0, - real_time(), NULL, dpp, y); -} - void RadosObject::set_compressed(RGWObjectCtx* rctx) { rgw_obj obj = get_obj(); store->getRados()->set_compressed(rctx, obj); @@ -1737,7 +1683,7 @@ bool RadosObject::placement_rules_match(rgw_placement_rule& r1, rgw_placement_ru return p1 == p2; } -int RadosObject::get_obj_layout(const DoutPrefixProvider *dpp, optional_yield y, Formatter* f, RGWObjectCtx* obj_ctx) +int RadosObject::dump_obj_layout(const DoutPrefixProvider *dpp, optional_yield y, Formatter* f, RGWObjectCtx* obj_ctx) { int ret; RGWObjManifest *manifest{nullptr}; diff --git a/src/rgw/rgw_sal_rados.h b/src/rgw/rgw_sal_rados.h index b40bb02e19b27..9609547120586 100644 --- a/src/rgw/rgw_sal_rados.h +++ b/src/rgw/rgw_sal_rados.h @@ -55,7 +55,22 @@ class RadosUser : public User { int list_buckets(const DoutPrefixProvider* dpp, const std::string& marker, const std::string& end_marker, uint64_t max, bool need_stats, BucketList& buckets, optional_yield y) override; - virtual Bucket* create_bucket(rgw_bucket& bucket, ceph::real_time creation_time) override; + virtual int create_bucket(const DoutPrefixProvider* dpp, + const rgw_bucket& b, + const std::string& zonegroup_id, + rgw_placement_rule& placement_rule, + std::string& swift_ver_location, + const RGWQuotaInfo * pquota_info, + const RGWAccessControlPolicy& policy, + Attrs& attrs, + RGWBucketInfo& info, + obj_version& ep_objv, + bool exclusive, + bool obj_lock_enabled, + bool* existed, + req_info& req_info, + std::unique_ptr* bucket, + optional_yield y) override; virtual int read_attrs(const DoutPrefixProvider* dpp, optional_yield y) override; virtual int read_stats(const DoutPrefixProvider *dpp, optional_yield y, RGWStorageStats* stats, @@ -157,7 +172,6 @@ class RadosObject : public Object { virtual int get_obj_attrs(RGWObjectCtx* rctx, optional_yield y, const DoutPrefixProvider* dpp, rgw_obj* target_obj = NULL) override; virtual int modify_obj_attrs(RGWObjectCtx* rctx, const char* attr_name, bufferlist& attr_val, optional_yield y, const DoutPrefixProvider* dpp) override; virtual int delete_obj_attrs(const DoutPrefixProvider* dpp, RGWObjectCtx* rctx, const char* attr_name, optional_yield y) override; - virtual int copy_obj_data(RGWObjectCtx& rctx, Bucket* dest_bucket, Object* dest_obj, uint16_t olh_epoch, std::string* petag, const DoutPrefixProvider* dpp, optional_yield y) override; virtual bool is_expired() override; virtual void gen_rand_obj_instance_name() override; void get_raw_obj(rgw_raw_obj* raw_obj); @@ -173,7 +187,7 @@ class RadosObject : public Object { const DoutPrefixProvider* dpp, optional_yield y) override; virtual bool placement_rules_match(rgw_placement_rule& r1, rgw_placement_rule& r2) override; - virtual int get_obj_layout(const DoutPrefixProvider *dpp, optional_yield y, Formatter* f, RGWObjectCtx* obj_ctx) override; + virtual int dump_obj_layout(const DoutPrefixProvider *dpp, optional_yield y, Formatter* f, RGWObjectCtx* obj_ctx) override; /* Swift versioning */ virtual int swift_versioning_restore(RGWObjectCtx* obj_ctx, @@ -275,20 +289,18 @@ class RadosBucket : public Bucket { DoutPrefixProvider *dpp) override; virtual RGWAccessControlPolicy& get_acl(void) override { return acls; } virtual int set_acl(const DoutPrefixProvider* dpp, RGWAccessControlPolicy& acl, optional_yield y) override; - virtual int get_bucket_info(const DoutPrefixProvider* dpp, optional_yield y) override; - virtual int get_bucket_stats(const DoutPrefixProvider *dpp, int shard_id, + virtual int load_bucket(const DoutPrefixProvider* dpp, optional_yield y) override; + virtual int read_stats(const DoutPrefixProvider *dpp, int shard_id, std::string* bucket_ver, std::string* master_ver, std::map& stats, std::string* max_marker = nullptr, bool* syncstopped = nullptr) override; - virtual int get_bucket_stats_async(const DoutPrefixProvider *dpp, int shard_id, RGWGetBucketStats_CB* ctx) override; - virtual int read_bucket_stats(const DoutPrefixProvider* dpp, optional_yield y) override; + virtual int read_stats_async(const DoutPrefixProvider *dpp, int shard_id, RGWGetBucketStats_CB* ctx) override; virtual int sync_user_stats(const DoutPrefixProvider *dpp, optional_yield y) override; virtual int update_container_stats(const DoutPrefixProvider* dpp) override; virtual int check_bucket_shards(const DoutPrefixProvider* dpp) override; virtual int chown(const DoutPrefixProvider* dpp, User* new_user, User* old_user, optional_yield y, const std::string* marker = nullptr) override; virtual int put_info(const DoutPrefixProvider* dpp, bool exclusive, ceph::real_time mtime) override; - virtual int remove_metadata(const DoutPrefixProvider* dpp, RGWObjVersionTracker* objv, optional_yield y) override; virtual bool is_owner(User* user) override; virtual int check_empty(const DoutPrefixProvider* dpp, optional_yield y) override; virtual int check_quota(const DoutPrefixProvider *dpp, RGWQuotaInfo& user_quota, RGWQuotaInfo& bucket_quota, uint64_t obj_size, optional_yield y, bool check_size_only = false) override; @@ -306,6 +318,10 @@ class RadosBucket : public Bucket { virtual std::unique_ptr clone() override { return std::make_unique(*this); } + virtual std::unique_ptr get_multipart_upload( + const std::string& oid, + std::optional upload_id=std::nullopt, + ACLOwner owner={}, ceph::real_time mtime=real_clock::now()) override; virtual int list_multiparts(const DoutPrefixProvider *dpp, const std::string& prefix, std::string& marker, @@ -321,7 +337,7 @@ class RadosBucket : public Bucket { private: int link(const DoutPrefixProvider* dpp, User* new_user, optional_yield y, bool update_entrypoint = true, RGWObjVersionTracker* objv = nullptr); int unlink(const DoutPrefixProvider* dpp, User* new_user, optional_yield y, bool update_entrypoint = true); - friend class RadosStore; + friend class RadosUser; }; class RadosZone : public Zone { @@ -366,28 +382,10 @@ class RadosStore : public Store { virtual int get_bucket(const DoutPrefixProvider* dpp, User* u, const rgw_bucket& b, std::unique_ptr* bucket, optional_yield y) override; virtual int get_bucket(User* u, const RGWBucketInfo& i, std::unique_ptr* bucket) override; virtual int get_bucket(const DoutPrefixProvider* dpp, User* u, const std::string& tenant, const std::string&name, std::unique_ptr* bucket, optional_yield y) override; - virtual int create_bucket(const DoutPrefixProvider* dpp, - User* u, const rgw_bucket& b, - const std::string& zonegroup_id, - rgw_placement_rule& placement_rule, - std::string& swift_ver_location, - const RGWQuotaInfo * pquota_info, - const RGWAccessControlPolicy& policy, - Attrs& attrs, - RGWBucketInfo& info, - obj_version& ep_objv, - bool exclusive, - bool obj_lock_enabled, - bool* existed, - req_info& req_info, - std::unique_ptr* bucket, - optional_yield y) override; virtual bool is_meta_master() override; virtual int forward_request_to_master(const DoutPrefixProvider *dpp, User* user, obj_version* objv, bufferlist& in_data, JSONParser* jp, req_info& info, optional_yield y) override; - virtual int defer_gc(const DoutPrefixProvider* dpp, RGWObjectCtx* rctx, Bucket* bucket, Object* obj, - optional_yield y) override; virtual Zone* get_zone() { return &zone; } virtual std::string zone_unique_id(uint64_t unique_num) override; virtual std::string zone_unique_trans_id(const uint64_t unique_num) override; @@ -444,10 +442,6 @@ class RadosStore : public Store { virtual int get_oidc_providers(const DoutPrefixProvider *dpp, const std::string& tenant, std::vector>& providers) override; - virtual std::unique_ptr - get_multipart_upload(Bucket* bucket, const std::string& oid, - std::optional upload_id=std::nullopt, - ACLOwner owner={}, ceph::real_time mtime=real_clock::now()) override; virtual std::unique_ptr get_append_writer(const DoutPrefixProvider *dpp, optional_yield y, std::unique_ptr _head_obj, diff --git a/src/rgw/rgw_user.cc b/src/rgw/rgw_user.cc index 20323eddee4b7..e231f33624a59 100644 --- a/src/rgw/rgw_user.cc +++ b/src/rgw/rgw_user.cc @@ -68,7 +68,7 @@ int rgw_user_sync_all_stats(const DoutPrefixProvider *dpp, rgw::sal::Store* stor auto& bucket = i->second; - ret = bucket->get_bucket_info(dpp, y); + ret = bucket->load_bucket(dpp, y); if (ret < 0) { ldpp_dout(dpp, 0) << "ERROR: could not read bucket info: bucket=" << bucket << " ret=" << ret << dendl; continue; @@ -97,7 +97,7 @@ int rgw_user_sync_all_stats(const DoutPrefixProvider *dpp, rgw::sal::Store* stor int rgw_user_get_all_buckets_stats(const DoutPrefixProvider *dpp, rgw::sal::Store* store, rgw::sal::User* user, - map& buckets_usage_map, + map& buckets_usage_map, optional_yield y) { CephContext *cct = store->ctx(); @@ -118,13 +118,16 @@ int rgw_user_get_all_buckets_stats(const DoutPrefixProvider *dpp, marker = i.first; auto& bucket_ent = i.second; - ret = bucket_ent->read_bucket_stats(dpp, y); + ret = bucket_ent->load_bucket(dpp, y); if (ret < 0) { ldpp_dout(dpp, 0) << "ERROR: could not get bucket stats: ret=" << ret << dendl; return ret; } - cls_user_bucket_entry entry; - bucket_ent->convert(&entry); + bucket_meta_entry entry; + entry.size = bucket_ent->get_size(); + entry.size_rounded = bucket_ent->get_size_rounded(); + entry.creation_time = bucket_ent->get_creation_time(); + entry.count = bucket_ent->get_count(); buckets_usage_map.emplace(bucket_ent->get_name(), entry); } done = (buckets.count() < max_entries); @@ -1733,7 +1736,7 @@ int RGWUser::execute_rename(const DoutPrefixProvider *dpp, RGWUserAdminOpState& auto& bucket = it->second; marker = it->first; - ret = bucket->get_bucket_info(dpp, y); + ret = bucket->load_bucket(dpp, y); if (ret < 0) { set_err_msg(err_msg, "failed to fetch bucket info for bucket=" + bucket->get_name()); return ret; diff --git a/src/rgw/rgw_user.h b/src/rgw/rgw_user.h index acdb7a0afa052..c29224cbeb48e 100644 --- a/src/rgw/rgw_user.h +++ b/src/rgw/rgw_user.h @@ -57,10 +57,18 @@ struct RGWUID }; WRITE_CLASS_ENCODER(RGWUID) +/** Entry for bucket metadata collection */ +struct bucket_meta_entry { + size_t size; + size_t size_rounded; + ceph::real_time creation_time; + uint64_t count; +}; + extern int rgw_user_sync_all_stats(const DoutPrefixProvider *dpp, rgw::sal::Store* store, rgw::sal::User* user, optional_yield y); extern int rgw_user_get_all_buckets_stats(const DoutPrefixProvider *dpp, rgw::sal::Store* store, rgw::sal::User* user, - std::map& buckets_usage_map, optional_yield y); + std::map& buckets_usage_map, optional_yield y); /** * Get the anonymous (ie, unauthenticated) user info. diff --git a/src/test/rgw/test_rgw_lua.cc b/src/test/rgw/test_rgw_lua.cc index bf70e6fbd821f..32ec599985d73 100644 --- a/src/test/rgw/test_rgw_lua.cc +++ b/src/test/rgw/test_rgw_lua.cc @@ -76,8 +76,8 @@ class TestUser : public sal::User { return 0; } - virtual sal::Bucket* create_bucket(rgw_bucket& bucket, ceph::real_time creation_time) override { - return nullptr; + virtual int create_bucket(const DoutPrefixProvider* dpp, const rgw_bucket& b, const std::string& zonegroup_id, rgw_placement_rule& placement_rule, std::string& swift_ver_location, const RGWQuotaInfo* pquota_info, const RGWAccessControlPolicy& policy, sal::Attrs& attrs, RGWBucketInfo& info, obj_version& ep_objv, bool exclusive, bool obj_lock_enabled, bool* existed, req_info& req_info, std::unique_ptr* bucket, optional_yield y) override { + return 0; } virtual int read_attrs(const DoutPrefixProvider *dpp, optional_yield y) override { From 99cfbcbc0ce8f5d2f18c585d7ec354a44bc47003 Mon Sep 17 00:00:00 2001 From: Daniel Gryniewicz Date: Thu, 21 Oct 2021 13:50:45 -0400 Subject: [PATCH 0575/2866] RGW Zipper - Initial round of API docs This is the initial round of API docs for Zipper, in the form of Doxygen comments. It's not detailed document, but it's a start. Signed-off-by: Daniel Gryniewicz --- src/rgw/rgw_sal.h | 482 +++++++++++++++++++++++++++++++++++++--- src/rgw/rgw_sal_rados.h | 2 +- 2 files changed, 450 insertions(+), 34 deletions(-) diff --git a/src/rgw/rgw_sal.h b/src/rgw/rgw_sal.h index 2c1141ae1ec65..6bad56ceb32e2 100644 --- a/src/rgw/rgw_sal.h +++ b/src/rgw/rgw_sal.h @@ -102,9 +102,57 @@ class RGWGetUserStats_CB : public RefCountedObject { } }; +/** + * @defgroup RGWSAL RGW Store Abstraction Layer + * + * The Store Abstraction Layer is an API that separates the top layer of RGW that + * handles client protocols (such as S3 or Swift) from the bottom layer of RGW that + * interacts with a backing store. It allows the creation of multiple backing stores + * that can co-exist with a single RGW instance, and allows the creation of stacking + * layers of translators that can modify operations as they pass down the stack. + * Examples of translators might be a cache layer, a duplication layer that copies + * operations to multiple stores, or a policy layer that sends some operations to one + * store and some to another. + * + * The basic unit of a SAL implementation is the Store. Whether an actual backing store + * or a translator, there will be a Store implementation that represents it. Examples + * are the RadosStore that communicates via RADOS with a Ceph cluster, and the DBStore + * that uses a SQL db (such as SQLite3) as a backing store. There is a singleton + * instance of each Store. + * + * Data within RGW is owned by a User. The User is the unit of authentication and + * access control. + * + * Data within RGW is stored as an Object. Each Object is a single chunk of data, owned + * by a single User, contained within a single Bucket. It has metadata associated with + * it, such as size, owner, and so in, and a set of key-value attributes that can + * contain anything needed by the top half. + * + * Data with RGW is organized into Buckets. Each Bucket is owned by a User, and + * contains Objects. There is a single, flat layer of Buckets, there is no hierarchy, + * and each Object is contained in a single Bucket. + * + * Instantiations of SAL classes are done as unique pointers, using std::unique_ptr. + * Instances of these classes are acquired via getters, and it's up to the caller to + * manage the lifetime. + * + * @note Anything using RGWObjContext is subject to change, as that type will not be + * used in the final API. + * @{ + */ + +/** + * @file rgw_sal.h + * @brief Base abstractions and API for SAL + */ namespace rgw { namespace sal { +/** + * @addtogroup RGWSAL + * @{ + */ + #define RGW_SAL_VERSION 1 class User; @@ -129,23 +177,31 @@ enum AttrsMod { }; // a simple streaming data processing abstraction +/** + * @brief A simple streaming data processing abstraction + */ class DataProcessor { public: virtual ~DataProcessor() {} - // consume a bufferlist in its entirety at the given object offset. an - // empty bufferlist is given to request that any buffered data be flushed, - // though this doesn't wait for completions + /** + * @brief Consume a bufferlist in its entirety at the given object offset. + * + * An empty bufferlist is given to request that any buffered data be flushed, though this doesn't + * wait for completions + */ virtual int process(bufferlist&& data, uint64_t offset) = 0; }; -// a data consumer that writes an object in a bucket +/** + * @brief a data consumer that writes an object in a bucket + */ class ObjectProcessor : public DataProcessor { public: - // prepare to start processing object data + /** prepare to start processing object data */ virtual int prepare(optional_yield y) = 0; - // complete the operation and make its result visible to clients + /** complete the operation and make its result visible to clients */ virtual int complete(size_t accounted_size, const std::string& etag, ceph::real_time *mtime, ceph::real_time set_mtime, std::map& attrs, @@ -156,9 +212,7 @@ class ObjectProcessor : public DataProcessor { optional_yield y) = 0; }; -/** - * Base class for AIO completions - */ +/** Base class for AIO completions */ class Completions { public: Completions() {} @@ -166,84 +220,149 @@ class Completions { virtual int drain() = 0; }; +/** A list of key-value attributes */ using Attrs = std::map; +/** + * @brief Base singleton representing a Store or stacking layer + * + * The Store is the base abstraction of the SAL layer. It represents a base storage + * mechanism, or a intermediate stacking layer. There is a single instance of a given + * Store per RGW, and this Store mediates all access to it's backing. + * + * A store contains, loosely, @a User, @a Bucket, and @a Object entities. The @a Object + * contains data, and it's associated metadata. The @a Bucket contains Objects, and + * metadata about the bucket. Both Buckets and Objects are owned by a @a User, which is + * the basic unit of access control. + * + * A store also has metadata and some global responsibilities. For example, a store is + * responsible for managing the LifeCycle activities for it's data. + */ class Store { public: Store() {} virtual ~Store() = default; - /* This one does not query the cluster for info */ + /** Get a User from a rgw_user. Does not query store for user info, so quick */ virtual std::unique_ptr get_user(const rgw_user& u) = 0; - /* These three do query the cluster for info */ + /** Lookup a User by access key. Queries store for user info. */ virtual int get_user_by_access_key(const DoutPrefixProvider* dpp, const std::string& key, optional_yield y, std::unique_ptr* user) = 0; + /** Lookup a User by email address. Queries store for user info. */ virtual int get_user_by_email(const DoutPrefixProvider* dpp, const std::string& email, optional_yield y, std::unique_ptr* user) = 0; + /** Lookup a User by swift username. Queries store for user info. */ virtual int get_user_by_swift(const DoutPrefixProvider* dpp, const std::string& user_str, optional_yield y, std::unique_ptr* user) = 0; + /** Get a basic Object. This Object is not looked up, and is incomplete, since is + * does not have a bucket. This should only be used when an Object is needed before + * there is a Bucket, otherwise use the get_object() in the Bucket class. */ virtual std::unique_ptr get_object(const rgw_obj_key& k) = 0; - virtual int get_bucket(const DoutPrefixProvider* dpp, User* u, const rgw_bucket& b, std::unique_ptr* bucket, optional_yield y) = 0; + /** Get a Bucket by info. Does not query the store, just uses the give bucket info. */ virtual int get_bucket(User* u, const RGWBucketInfo& i, std::unique_ptr* bucket) = 0; + /** Lookup a Bucket by key. Queries store for bucket info. */ + virtual int get_bucket(const DoutPrefixProvider* dpp, User* u, const rgw_bucket& b, std::unique_ptr* bucket, optional_yield y) = 0; + /** Lookup a Bucket by name. Queries store for bucket info. */ virtual int get_bucket(const DoutPrefixProvider* dpp, User* u, const std::string& tenant, const std::string& name, std::unique_ptr* bucket, optional_yield y) = 0; + /** For multisite, this Store is the zone's master */ virtual bool is_meta_master() = 0; + /** For multisite, forward an OP to the zone's master */ virtual int forward_request_to_master(const DoutPrefixProvider *dpp, User* user, obj_version* objv, bufferlist& in_data, JSONParser* jp, req_info& info, optional_yield y) = 0; + /** Get zone info for this store */ virtual Zone* get_zone() = 0; + /** Get a unique ID specific to this zone. */ virtual std::string zone_unique_id(uint64_t unique_num) = 0; + /** Get a unique Swift transaction ID specific to this zone */ virtual std::string zone_unique_trans_id(const uint64_t unique_num) = 0; + /** Get statistics about the cluster represented by this Store */ virtual int cluster_stat(RGWClusterStat& stats) = 0; + /** Get a @a Lifecycle object. Used to manage/run lifecycle transitions */ virtual std::unique_ptr get_lifecycle(void) = 0; + /** Get a @a Completions object. Used for Async I/O tracking */ virtual std::unique_ptr get_completions(void) = 0; + /** Get a @a Notification object. Used to communicate with non-RGW daemons, such as + * management/tracking software */ virtual std::unique_ptr get_notification(rgw::sal::Object* obj, struct req_state* s, rgw::notify::EventType event_type, const std::string* object_name=nullptr) = 0; + /** Get access to the lifecycle management thread */ virtual RGWLC* get_rgwlc(void) = 0; + /** Get access to the coroutine registry. Used to create new coroutine managers */ virtual RGWCoroutinesManagerRegistry* get_cr_registry() = 0; + /** Log usage data to the store. Usage data is things like bytes sent/received and + * op count */ virtual int log_usage(const DoutPrefixProvider *dpp, std::map& usage_info) = 0; + /** Log OP data to the store. Data is opaque to SAL */ virtual int log_op(const DoutPrefixProvider *dpp, std::string& oid, bufferlist& bl) = 0; + /** Register this Store to the service map. Somewhat Rados specific; may be removed*/ virtual int register_to_service_map(const DoutPrefixProvider *dpp, const std::string& daemon_type, const std::map& meta) = 0; + /** Get default quota info. Used as fallback if a user or bucket has no quota set*/ virtual void get_quota(RGWQuotaInfo& bucket_quota, RGWQuotaInfo& user_quota) = 0; + /** Enable or disable a set of bucket. e.g. if a User is suspended */ virtual int set_buckets_enabled(const DoutPrefixProvider* dpp, std::vector& buckets, bool enabled) = 0; + /** Get a new request ID */ virtual uint64_t get_new_req_id() = 0; + /** Get a handler for bucket sync policy. */ virtual int get_sync_policy_handler(const DoutPrefixProvider* dpp, std::optional zone, std::optional bucket, RGWBucketSyncPolicyHandlerRef* phandler, optional_yield y) = 0; + /** Get a status manager for bucket sync */ virtual RGWDataSyncStatusManager* get_data_sync_manager(const rgw_zone_id& source_zone) = 0; + /** Wake up sync threads for bucket metadata sync */ virtual void wakeup_meta_sync_shards(std::set& shard_ids) = 0; + /** Wake up sync threads for bucket data sync */ virtual void wakeup_data_sync_shards(const DoutPrefixProvider *dpp, const rgw_zone_id& source_zone, std::map >& shard_ids) = 0; + /** Clear all usage statistics globally */ virtual int clear_usage(const DoutPrefixProvider *dpp) = 0; + /** Get usage statistics for all users and buckets */ virtual int read_all_usage(const DoutPrefixProvider *dpp, uint64_t start_epoch, uint64_t end_epoch, uint32_t max_entries, bool* is_truncated, RGWUsageIter& usage_iter, std::map& usage) = 0; + /** Trim usage log for all users and buckets */ virtual int trim_all_usage(const DoutPrefixProvider *dpp, uint64_t start_epoch, uint64_t end_epoch) = 0; + /** Get a configuration value for the given name */ virtual int get_config_key_val(std::string name, bufferlist* bl) = 0; + /** Start a metadata listing of the given section */ virtual int meta_list_keys_init(const DoutPrefixProvider *dpp, const std::string& section, const std::string& marker, void** phandle) = 0; + /** Get the next key from a metadata list */ virtual int meta_list_keys_next(const DoutPrefixProvider *dpp, void* handle, int max, std::list& keys, bool* truncated) = 0; + /** Complete a metadata listing */ virtual void meta_list_keys_complete(void* handle) = 0; + /** Get the marker associated with the current metadata listing */ virtual std::string meta_get_marker(void* handle) = 0; + /** Remove a specific metadata key */ virtual int meta_remove(const DoutPrefixProvider* dpp, std::string& metadata_key, optional_yield y) = 0; + /** Get an instance of the Sync module for bucket sync */ virtual const RGWSyncModuleInstanceRef& get_sync_module() = 0; + /** Get the ID of the current host */ virtual std::string get_host_id() = 0; + /** Get a Lua script manager for running lua scripts */ virtual std::unique_ptr get_lua_script_manager() = 0; + /** Get an IAM Role by name etc. */ virtual std::unique_ptr get_role(std::string name, std::string tenant, std::string path="", std::string trust_policy="", std::string max_session_duration_str="", std::multimap tags={}) = 0; + /** Get an IAM Role by ID */ virtual std::unique_ptr get_role(std::string id) = 0; + /** Get all IAM Roles optionally filtered by path */ virtual int get_roles(const DoutPrefixProvider *dpp, optional_yield y, const std::string& path_prefix, const std::string& tenant, std::vector>& roles) = 0; + /** Get an empty Open ID Connector provider */ virtual std::unique_ptr get_oidc_provider() = 0; + /** Get all Open ID Connector providers, optionally filtered by tenant */ virtual int get_oidc_providers(const DoutPrefixProvider *dpp, const std::string& tenant, std::vector>& providers) = 0; + /** Get a Writer that appends to an object */ virtual std::unique_ptr get_append_writer(const DoutPrefixProvider *dpp, optional_yield y, std::unique_ptr _head_obj, @@ -252,6 +371,7 @@ class Store { const std::string& unique_tag, uint64_t position, uint64_t *cur_accounted_size) = 0; + /** Get a Writer that atomically writes an entire object */ virtual std::unique_ptr get_atomic_writer(const DoutPrefixProvider *dpp, optional_yield y, std::unique_ptr _head_obj, @@ -260,16 +380,27 @@ class Store { uint64_t olh_epoch, const std::string& unique_tag) = 0; + /** Clean up a store for termination */ virtual void finalize(void) = 0; + /** Get the Ceph context associated with this store. May be removed. */ virtual CephContext* ctx(void) = 0; - // get the location of where lua packages are installed + /** Get the location of where lua packages are installed */ virtual const std::string& get_luarocks_path() const = 0; - // set the location of where lua packages are installed + /** Set the location of where lua packages are installed */ virtual void set_luarocks_path(const std::string& path) = 0; }; +/** + * @brief User abstraction + * + * This represents a user. In general, there will be a @a User associated with an OP + * (the user performing the OP), and potentially several others acting as owners. + * Lifetime of a User is a bit tricky , since it must last as long as any Buckets + * associated with it. A User has associated metadata, including a set of key/value + * attributes, and statistics (including usage) about the User. + */ class User { protected: RGWUserInfo info; @@ -283,7 +414,9 @@ class User { User(User& _o) = default; virtual ~User() = default; + /** Clone a copy of this user. Used when modification is necessary of the copy */ virtual std::unique_ptr clone() = 0; + /** List the buckets owned by a user */ virtual int list_buckets(const DoutPrefixProvider* dpp, const std::string& marker, const std::string& end_marker, uint64_t max, bool need_stats, BucketList& buckets, @@ -305,37 +438,61 @@ class User { req_info& req_info, std::unique_ptr* bucket, optional_yield y) = 0; - friend class Bucket; - virtual std::string& get_display_name() { return info.display_name; } + /** Get the display name for this User */ + virtual std::string& get_display_name() { return info.display_name; } + /** Get the tenant name for this User */ const std::string& get_tenant() { return info.user_id.tenant; } + /** Set the tenant name for this User */ void set_tenant(std::string& _t) { info.user_id.tenant = _t; } + /** Get the namespace for this User */ const std::string& get_ns() { return info.user_id.ns; } + /** Set the namespace for this User */ void set_ns(std::string& _ns) { info.user_id.ns = _ns; } + /** Clear the namespace for this User */ void clear_ns() { info.user_id.ns.clear(); } + /** Get the full ID for this User */ const rgw_user& get_id() const { return info.user_id; } + /** Get the type of this User */ uint32_t get_type() const { return info.type; } + /** Get the maximum number of buckets allowed for this User */ int32_t get_max_buckets() const { return info.max_buckets; } + /** Get the capabilities for this User */ const RGWUserCaps& get_caps() const { return info.caps; } + /** Get the version tracker for this User */ + virtual RGWObjVersionTracker& get_version_tracker() { return objv_tracker; } + /** Get the cached attributes for this User */ + virtual Attrs& get_attrs() { return attrs; } + /** Set the cached attributes fro this User */ + virtual void set_attrs(Attrs& _attrs) { attrs = _attrs; } + /** Check if a User pointer is empty */ static bool empty(User* u) { return (!u || u->info.user_id.id.empty()); } + /** Check if a User unique_pointer is empty */ static bool empty(std::unique_ptr& u) { return (!u || u->info.user_id.id.empty()); } + /** Read the User attributes from the backing Store */ virtual int read_attrs(const DoutPrefixProvider* dpp, optional_yield y) = 0; + /** Read the User stats from the backing Store, synchronous */ virtual int read_stats(const DoutPrefixProvider *dpp, optional_yield y, RGWStorageStats* stats, ceph::real_time* last_stats_sync = nullptr, ceph::real_time* last_stats_update = nullptr) = 0; + /** Read the User stats from the backing Store, asynchronous */ virtual int read_stats_async(const DoutPrefixProvider *dpp, RGWGetUserStats_CB* cb) = 0; + /** Flush accumulated stat changes for this User to the backing store */ virtual int complete_flush_stats(const DoutPrefixProvider *dpp, optional_yield y) = 0; - virtual int read_usage(const DoutPrefixProvider *dpp, uint64_t start_epoch, uint64_t end_epoch, uint32_t max_entries, + /** Read detailed usage stats for this User from the backing store */ + virtual int read_usage(const DoutPrefixProvider *dpp, uint64_t start_epoch, + uint64_t end_epoch, uint32_t max_entries, bool* is_truncated, RGWUsageIter& usage_iter, std::map& usage) = 0; + /** Trim User usage stats to the given epoch range */ virtual int trim_usage(const DoutPrefixProvider *dpp, uint64_t start_epoch, uint64_t end_epoch) = 0; - virtual RGWObjVersionTracker& get_version_tracker() { return objv_tracker; } - virtual Attrs& get_attrs() { return attrs; } - virtual void set_attrs(Attrs& _attrs) { attrs = _attrs; } + /** Load this User from the backing store. requires ID to be set, fills all other fields. */ virtual int load_user(const DoutPrefixProvider* dpp, optional_yield y) = 0; + /** Store this User to the backing store */ virtual int store_user(const DoutPrefixProvider* dpp, optional_yield y, bool exclusive, RGWUserInfo* old_info = nullptr) = 0; + /** Remove this User from the backing store */ virtual int remove_user(const DoutPrefixProvider* dpp, optional_yield y) = 0; /* dang temporary; will be removed when User is complete */ @@ -359,8 +516,17 @@ class User { return out; } + friend class Bucket; }; +/** + * @brief Bucket abstraction + * + * This represents a bucket. A bucket is a container for objects. It is owned by a user, and has + * it's own set of metadata, including a set of key/value attributes. A bucket may not contain + * other buckets, only objects. Buckets have Access Control Lists (ACLs) that control what users + * can access the contents of the bucket, and in what ways. + */ class Bucket { protected: RGWBucketEnt ent; @@ -372,6 +538,9 @@ class Bucket { public: + /** + * @brief Parameters for a bucket list operation + */ struct ListParams { std::string prefix; std::string delim; @@ -385,6 +554,9 @@ class Bucket { bool allow_unordered{false}; int shard_id{RGW_NO_SHARD}; }; + /** + * @brief Results from a bucket list operation + */ struct ListResults { std::vector objs; std::map common_prefixes; @@ -420,33 +592,54 @@ class Bucket { } virtual ~Bucket() = default; + /** Get an @a Object belonging to this bucket */ virtual std::unique_ptr get_object(const rgw_obj_key& key) = 0; + /** List the contents of this bucket */ virtual int list(const DoutPrefixProvider* dpp, ListParams&, int, ListResults&, optional_yield y) = 0; + /** Get the cached attributes associated with this bucket */ virtual Attrs& get_attrs(void) { return attrs; } + /** Set the cached attributes on this bucket */ virtual int set_attrs(Attrs a) { attrs = a; return 0; } + /** Remove this bucket from the backing store */ virtual int remove_bucket(const DoutPrefixProvider* dpp, bool delete_children, std::string prefix, std::string delimiter, bool forward_to_master, req_info* req_info, optional_yield y) = 0; + /** Remove this bucket, bypassing garbage collection. May be removed */ virtual int remove_bucket_bypass_gc(int concurrent_max, bool keep_index_consistent, optional_yield y, const DoutPrefixProvider *dpp) = 0; + /** Get then ACL for this bucket */ virtual RGWAccessControlPolicy& get_acl(void) = 0; + /** Set the ACL for this bucket */ virtual int set_acl(const DoutPrefixProvider* dpp, RGWAccessControlPolicy& acl, optional_yield y) = 0; + /** Load this bucket from the backing store. Requires the key to be set, fills other fields */ virtual int load_bucket(const DoutPrefixProvider* dpp, optional_yield y) = 0; + /** Read the bucket stats from the backing Store, synchronous */ virtual int read_stats(const DoutPrefixProvider *dpp, int shard_id, std::string* bucket_ver, std::string* master_ver, std::map& stats, std::string* max_marker = nullptr, bool* syncstopped = nullptr) = 0; + /** Read the bucket stats from the backing Store, asynchronous */ virtual int read_stats_async(const DoutPrefixProvider *dpp, int shard_id, RGWGetBucketStats_CB* ctx) = 0; + /** Sync this bucket's stats to the owning user's stats in the backing store */ virtual int sync_user_stats(const DoutPrefixProvider *dpp, optional_yield y) = 0; + /** Refresh the metadata stats (size, count, and so on) from the backing store */ virtual int update_container_stats(const DoutPrefixProvider* dpp) = 0; + /** Check if this bucket needs resharding, and schedule it if it does */ virtual int check_bucket_shards(const DoutPrefixProvider* dpp) = 0; + /** Change the owner of this bucket in the backing store */ virtual int chown(const DoutPrefixProvider* dpp, User* new_user, User* old_user, optional_yield y, const std::string* marker = nullptr) = 0; + /** Store the cached bucket info into the backing store */ virtual int put_info(const DoutPrefixProvider* dpp, bool exclusive, ceph::real_time mtime) = 0; + /** Check to see if the given user is the owner of this bucket */ virtual bool is_owner(User* user) = 0; + /** Get the owner of this bucket */ virtual User* get_owner(void) { return owner; }; + /** Get the owner of this bucket in the form of an ACLOwner object */ virtual ACLOwner get_acl_owner(void) { return ACLOwner(info.owner); }; + /** Check in the backing store if this bucket is empty */ virtual int check_empty(const DoutPrefixProvider* dpp, optional_yield y) = 0; + /** Chec k if the given size fits within the quota */ virtual int check_quota(const DoutPrefixProvider *dpp, RGWQuotaInfo& user_quota, RGWQuotaInfo& bucket_quota, uint64_t obj_size, optional_yield y, bool check_size_only = false) = 0; /** Set the attributes in attrs, leaving any other existing attrs set, and * write them to the backing store; a merge operation */ @@ -456,40 +649,68 @@ class Bucket { } return 0; } + /** Try to refresh the cached bucket info from the backing store. Used in + * read-modify-update loop. */ virtual int try_refresh_info(const DoutPrefixProvider* dpp, ceph::real_time* pmtime) = 0; + /** Read usage information about this bucket from the backing store */ virtual int read_usage(const DoutPrefixProvider *dpp, uint64_t start_epoch, uint64_t end_epoch, uint32_t max_entries, bool* is_truncated, RGWUsageIter& usage_iter, std::map& usage) = 0; + /** Trim the usage information to the given epoch range */ virtual int trim_usage(const DoutPrefixProvider *dpp, uint64_t start_epoch, uint64_t end_epoch) = 0; + /** Remove objects from the bucket index of this bucket. May be removed from API */ virtual int remove_objs_from_index(const DoutPrefixProvider *dpp, std::list& objs_to_unlink) = 0; + /** Check the state of the bucket index, and get stats from it. May be removed from API */ virtual int check_index(const DoutPrefixProvider *dpp, std::map& existing_stats, std::map& calculated_stats) = 0; + /** Rebuild the bucket index. May be removed from API */ virtual int rebuild_index(const DoutPrefixProvider *dpp) = 0; + /** Set a timeout on the check_index() call. May be removed from API */ virtual int set_tag_timeout(const DoutPrefixProvider *dpp, uint64_t timeout) = 0; + /** Remove this specific bucket instance from the backing store. May be removed from API */ virtual int purge_instance(const DoutPrefixProvider* dpp) = 0; + /** Check if this instantiation is empty */ bool empty() const { return info.bucket.name.empty(); } + /** Get the cached name of this bucket */ const std::string& get_name() const { return info.bucket.name; } + /** Get the cached tenant of this bucket */ const std::string& get_tenant() const { return info.bucket.tenant; } + /** Get the cached marker of this bucket */ const std::string& get_marker() const { return info.bucket.marker; } + /** Get the cached ID of this bucket */ const std::string& get_bucket_id() const { return info.bucket.bucket_id; } + /** Get the cached size of this bucket */ size_t get_size() const { return ent.size; } + /** Get the cached rounded size of this bucket */ size_t get_size_rounded() const { return ent.size_rounded; } + /** Get the cached object count of this bucket */ uint64_t get_count() const { return ent.count; } + /** Get the cached placement rule of this bucket */ rgw_placement_rule& get_placement_rule() { return info.placement_rule; } + /** Get the cached creation time of this bucket */ ceph::real_time& get_creation_time() { return info.creation_time; } + /** Get the cached modification time of this bucket */ ceph::real_time& get_modification_time() { return mtime; } + /** Get the cached version of this bucket */ obj_version& get_version() { return bucket_version; } + /** Set the cached version of this bucket */ void set_version(obj_version &ver) { bucket_version = ver; } + /** Check if this bucket is versioned */ bool versioned() { return info.versioned(); } + /** Check if this bucket has versioning enabled */ bool versioning_enabled() { return info.versioning_enabled(); } + /** Check if a Bucket pointer is empty */ static bool empty(Bucket* b) { return (!b || b->empty()); } + /** Clone a copy of this bucket. Used when modification is necessary of the copy */ virtual std::unique_ptr clone() = 0; + /** Create a multipart upload in this bucket */ virtual std::unique_ptr get_multipart_upload( const std::string& oid, std::optional upload_id=std::nullopt, ACLOwner owner={}, ceph::real_time mtime=real_clock::now()) = 0; + /** List multipart uploads currently in this bucket */ virtual int list_multiparts(const DoutPrefixProvider *dpp, const std::string& prefix, std::string& marker, @@ -498,6 +719,7 @@ class Bucket { std::vector>& uploads, std::map *common_prefixes, bool *is_truncated) = 0; + /** Abort multipart uploads matching the prefix and delimiter */ virtual int abort_multiparts(const DoutPrefixProvider *dpp, CephContext *cct, std::string& prefix, std::string& delim) = 0; @@ -540,7 +762,11 @@ class Bucket { virtual void set_ent(RGWBucketEnt& _ent) { ent = _ent; info.bucket = ent.bucket; info.placement_rule = ent.placement_rule; } }; - +/** + * @brief A list of buckets + * + * This is the result from a bucket listing operation. + */ class BucketList { std::map> buckets; bool truncated; @@ -560,19 +786,35 @@ class BucketList { return *this; }; + /** Get the list of buckets. The list is a map of pairs. */ std::map>& get_buckets() { return buckets; } + /** True if the list is truncated (that is, there are more buckets to list) */ bool is_truncated(void) const { return truncated; } + /** Set the truncated state of the list */ void set_truncated(bool trunc) { truncated = trunc; } + /** Add a bucket to the list. Takes ownership of the bucket */ void add(std::unique_ptr bucket) { buckets.emplace(bucket->info.bucket.name, std::move(bucket)); } + /** The number of buckets in this list */ size_t count() const { return buckets.size(); } + /** Clear the list */ void clear(void) { buckets.clear(); truncated = false; } }; +/** + * @brief Object abstraction + * + * This represents an Object. An Object is the basic unit of data storage. It + * represents a blob of data, a set of metadata (such as size, owner, ACLs, etc.) and + * a set of key/value attributes. Objects may be versioned. If a versioned object + * is written to, a new object with the same name but a different version is created, + * and the old version of the object is still accessible. If an unversioned object + * is written to, it is replaced, and the old data is not accessible. + */ class Object { protected: rgw_obj_key key; @@ -586,6 +828,12 @@ class Object { public: + /** + * @brief Read operation on an Object + * + * This represents a Read operation on an Object. Read operations are optionally + * asynchronous, using the iterate() API. + */ struct ReadOp { struct Params { const ceph::real_time* mod_ptr{nullptr}; @@ -601,12 +849,22 @@ class Object { virtual ~ReadOp() = default; + /** Prepare the Read op. Must be called first */ virtual int prepare(optional_yield y, const DoutPrefixProvider* dpp) = 0; + /** Synchronous read. Read from @a ofs to @a end into @a bl */ virtual int read(int64_t ofs, int64_t end, bufferlist& bl, optional_yield y, const DoutPrefixProvider* dpp) = 0; + /** Asynchronous read. Read from @a ofs to @a end calling @a cb on each read + * chunk. */ virtual int iterate(const DoutPrefixProvider* dpp, int64_t ofs, int64_t end, RGWGetDataCB* cb, optional_yield y) = 0; + /** Get an attribute by name */ virtual int get_attr(const DoutPrefixProvider* dpp, const char* name, bufferlist& dest, optional_yield y) = 0; }; + /** + * @brief Delete operation on an Object + * + * This deletes an Object from the backing store. + */ struct DeleteOp { struct Params { ACLOwner bucket_owner; @@ -632,6 +890,7 @@ class Object { virtual ~DeleteOp() = default; + /** Delete the object */ virtual int delete_obj(const DoutPrefixProvider* dpp, optional_yield y) = 0; }; @@ -660,12 +919,15 @@ class Object { virtual ~Object() = default; + /** Shortcut synchronous delete call for common deletes */ virtual int delete_object(const DoutPrefixProvider* dpp, RGWObjectCtx* obj_ctx, optional_yield y, bool prevent_versioning = false) = 0; + /** Asynchronous delete call */ virtual int delete_obj_aio(const DoutPrefixProvider* dpp, RGWObjState* astate, Completions* aio, bool keep_index_consistent, optional_yield y) = 0; + /** Copy an this object to another object. */ virtual int copy_object(RGWObjectCtx& obj_ctx, User* user, req_info* info, const rgw_zone_id& source_zone, rgw::sal::Object* dest_object, rgw::sal::Bucket* dest_bucket, @@ -681,23 +943,40 @@ class Object { std::string* version_id, std::string* tag, std::string* etag, void (*progress_cb)(off_t, void *), void* progress_data, const DoutPrefixProvider* dpp, optional_yield y) = 0; + /** Get the ACL for this object */ virtual RGWAccessControlPolicy& get_acl(void) = 0; + /** Set the ACL for this object */ virtual int set_acl(const RGWAccessControlPolicy& acl) = 0; + /** Mark further operations on this object as being atomic */ virtual void set_atomic(RGWObjectCtx* rctx) const = 0; + /** Pre-fetch data when reading */ virtual void set_prefetch_data(RGWObjectCtx* rctx) = 0; + /** Mark data as compressed */ virtual void set_compressed(RGWObjectCtx* rctx) = 0; + /** Check to see if this object has an empty key. This means it's uninitialized */ bool empty() const { return key.empty(); } + /** Get the name of this object */ const std::string &get_name() const { return key.name; } + /** Get the object state for this object. Will be removed in the future */ virtual int get_obj_state(const DoutPrefixProvider* dpp, RGWObjectCtx* rctx, RGWObjState **state, optional_yield y, bool follow_olh = true) = 0; + /** Set attributes for this object from the backing store. Attrs can be set or + * deleted. @note the attribute APIs may be revisited in the future. */ virtual int set_obj_attrs(const DoutPrefixProvider* dpp, RGWObjectCtx* rctx, Attrs* setattrs, Attrs* delattrs, optional_yield y, rgw_obj* target_obj = NULL) = 0; + /** Get attributes for this object */ virtual int get_obj_attrs(RGWObjectCtx* rctx, optional_yield y, const DoutPrefixProvider* dpp, rgw_obj* target_obj = NULL) = 0; + /** Modify attributes for this object. */ virtual int modify_obj_attrs(RGWObjectCtx* rctx, const char* attr_name, bufferlist& attr_val, optional_yield y, const DoutPrefixProvider* dpp) = 0; + /** Delete attributes for this object */ virtual int delete_obj_attrs(const DoutPrefixProvider* dpp, RGWObjectCtx* rctx, const char* attr_name, optional_yield y) = 0; + /** Check to see if this object has expired */ virtual bool is_expired() = 0; + /** Create a randomized instance ID for this object */ virtual void gen_rand_obj_instance_name() = 0; + /** Get a multipart serializer for this object */ virtual MPSerializer* get_serializer(const DoutPrefixProvider *dpp, const std::string& lock_name) = 0; + /** Move the data of an object to new placement storage */ virtual int transition(RGWObjectCtx& rctx, Bucket* bucket, const rgw_placement_rule& placement_rule, @@ -705,26 +984,46 @@ class Object { uint64_t olh_epoch, const DoutPrefixProvider* dpp, optional_yield y) = 0; + /** Check to see if two placement rules match */ virtual bool placement_rules_match(rgw_placement_rule& r1, rgw_placement_rule& r2) = 0; + /** Dump store-specific object layout info in JSON */ virtual int dump_obj_layout(const DoutPrefixProvider *dpp, optional_yield y, Formatter* f, RGWObjectCtx* obj_ctx) = 0; + /** Get the cached attributes for this object */ Attrs& get_attrs(void) { return attrs; } + /** Get the (const) cached attributes for this object */ const Attrs& get_attrs(void) const { return attrs; } + /** Set the cached attributes for this object */ virtual int set_attrs(Attrs a) { attrs = a; return 0; } + /** Get the cached modification time for this object */ ceph::real_time get_mtime(void) const { return mtime; } + /** Get the cached size for this object */ uint64_t get_obj_size(void) const { return obj_size; } + /** Get the bucket containing this object */ Bucket* get_bucket(void) const { return bucket; } + /** Set the bucket containing this object */ void set_bucket(Bucket* b) { bucket = b; } + /** Get the sharding hash representation of this object */ std::string get_hash_source(void) { return index_hash_source; } + /** Set the sharding hash representation of this object */ void set_hash_source(std::string s) { index_hash_source = s; } + /** Build an Object Identifier string for this object */ std::string get_oid(void) const { return key.get_oid(); } + /** True if this object is a delete marker (newest version is deleted) */ bool get_delete_marker(void) { return delete_marker; } + /** True if this object is stored in the extra data pool */ bool get_in_extra_data(void) { return in_extra_data; } + /** Set the in_extra_data field */ void set_in_extra_data(bool i) { in_extra_data = i; } + /** Helper to sanitize object size, offset, and end values */ int range_to_ofs(uint64_t obj_size, int64_t &ofs, int64_t &end); + /** Set the cached size of this object */ void set_obj_size(uint64_t s) { obj_size = s; } + /** Set the cached name of this object */ virtual void set_name(const std::string& n) { key = n; } + /** Set the cached key of this object */ virtual void set_key(const rgw_obj_key& k) { key = k; } + /** Get an rgw_obj representing this object */ virtual rgw_obj get_obj(void) const { rgw_obj obj(bucket->get_key(), key); obj.set_in_extra_data(in_extra_data); @@ -732,39 +1031,48 @@ class Object { return obj; } - /* Swift versioning */ + /** Restore the previous swift version of this object */ virtual int swift_versioning_restore(RGWObjectCtx* obj_ctx, bool& restored, /* out */ const DoutPrefixProvider* dpp) = 0; + /** Copy the current version of a swift object to the configured destination bucket*/ virtual int swift_versioning_copy(RGWObjectCtx* obj_ctx, const DoutPrefixProvider* dpp, optional_yield y) = 0; - /* OPs */ + /** Get a new ReadOp for this object */ virtual std::unique_ptr get_read_op(RGWObjectCtx*) = 0; + /** Get a new DeleteOp for this object */ virtual std::unique_ptr get_delete_op(RGWObjectCtx*) = 0; - /* OMAP */ + /** Get @a count OMAP values via listing, starting at @a marker for this object */ virtual int omap_get_vals(const DoutPrefixProvider *dpp, const std::string& marker, uint64_t count, std::map* m, bool* pmore, optional_yield y) = 0; + /** Get all OMAP key/value pairs for this object */ virtual int omap_get_all(const DoutPrefixProvider *dpp, std::map* m, optional_yield y) = 0; + /** Get the OMAP values matching the given set of keys */ virtual int omap_get_vals_by_keys(const DoutPrefixProvider *dpp, const std::string& oid, const std::set& keys, Attrs* vals) = 0; + /** Get a single OMAP value matching the given key */ virtual int omap_set_val_by_key(const DoutPrefixProvider *dpp, const std::string& key, bufferlist& val, bool must_exist, optional_yield y) = 0; + /** Check to see if the give object pointer is uninitialized */ static bool empty(Object* o) { return (!o || o->empty()); } + /** Get a unique copy of this object */ virtual std::unique_ptr clone() = 0; - /* dang - Not sure if we want this, but it simplifies things a lot */ - /* dang - This is temporary, until the API is completed */ + /** Get the key for this object */ rgw_obj_key& get_key() { return key; } + /** Set the instance for this object */ void set_instance(const std::string &i) { key.set_instance(i); } + /** Get the instance for this object */ const std::string &get_instance() const { return key.instance; } + /** Check to see if this object has an instance set */ bool have_instance(void) { return key.have_instance(); } friend inline std::ostream& operator<<(std::ostream& out, const Object& o) { @@ -786,6 +1094,9 @@ class Object { } }; +/** + * @brief Abstraction of a single part of a multipart upload + */ class MultipartPart { protected: std::string oid; @@ -794,12 +1105,25 @@ class MultipartPart { MultipartPart() = default; virtual ~MultipartPart() = default; + /** Get the part number of this part */ virtual uint32_t get_num() = 0; + /** Get the size of this part */ virtual uint64_t get_size() = 0; + /** Get the etag of this part */ virtual const std::string& get_etag() = 0; + /** Get the modification time of this part */ virtual ceph::real_time& get_mtime() = 0; }; +/** + * @brief Abstraction of a multipart upload + * + * This represents a multipart upload. For large objects, it's inefficient to do a + * single, long-lived upload of the object. Instead, protocols such as S3 allow the + * client to start a multipart upload, and then upload object in smaller parts in + * parallel. A MultipartUpload consists of a target bucket, a unique identifier, and a + * set of upload parts. + */ class MultipartUpload { protected: Bucket* bucket; @@ -809,23 +1133,34 @@ class MultipartUpload { MultipartUpload(Bucket* _bucket) : bucket(_bucket) {} virtual ~MultipartUpload() = default; + /** Get the name of the object representing this upload in the backing store */ virtual const std::string& get_meta() const = 0; + /** Get the name of the target object for this upload */ virtual const std::string& get_key() const = 0; + /** Get the unique ID of this upload */ virtual const std::string& get_upload_id() const = 0; + /** Get the owner of this upload */ virtual const ACLOwner& get_owner() const = 0; - virtual ceph::real_time get_mtime() const = 0; + /** Get the modification time of this upload */ + virtual ceph::real_time& get_mtime() = 0; + /** Get all the cached parts that make up this upload */ std::map>& get_parts() { return parts; } + /** Get the Object that represents this upload */ virtual std::unique_ptr get_meta_obj() = 0; + /** Initialize this upload */ virtual int init(const DoutPrefixProvider* dpp, optional_yield y, RGWObjectCtx* obj_ctx, ACLOwner& owner, rgw_placement_rule& dest_placement, rgw::sal::Attrs& attrs) = 0; + /** List all the parts of this upload, filling the parts cache */ virtual int list_parts(const DoutPrefixProvider* dpp, CephContext* cct, int num_parts, int marker, int* next_marker, bool* truncated, bool assume_unsorted = false) = 0; + /** Abort this upload */ virtual int abort(const DoutPrefixProvider* dpp, CephContext* cct, RGWObjectCtx* obj_ctx) = 0; + /** Complete this upload, making it available as a normal object */ virtual int complete(const DoutPrefixProvider* dpp, optional_yield y, CephContext* cct, std::map& part_etags, @@ -837,8 +1172,10 @@ class MultipartUpload { rgw::sal::Object* target_obj, RGWObjectCtx* obj_ctx) = 0; + /** Get placement and/or attribute info for this upload */ virtual int get_info(const DoutPrefixProvider *dpp, optional_yield y, RGWObjectCtx* obj_ctx, rgw_placement_rule** rule, rgw::sal::Attrs* attrs = nullptr) = 0; + /** Get a Writer to write to a part of this upload */ virtual std::unique_ptr get_writer(const DoutPrefixProvider *dpp, optional_yield y, std::unique_ptr _head_obj, @@ -867,14 +1204,21 @@ class MultipartUpload { } }; +/** + * @brief Interface of a lock/serialization + */ struct Serializer { Serializer() = default; virtual ~Serializer() = default; + /** Try to take the lock for the given amount of time. */ virtual int try_lock(const DoutPrefixProvider *dpp, utime_t dur, optional_yield y) = 0; + /** Unlock the lock */ virtual int unlock() = 0; }; +/** @brief Abstraction of a serializer for multipart uploads + */ struct MPSerializer : Serializer { bool locked; std::string oid; @@ -886,13 +1230,23 @@ struct MPSerializer : Serializer { } }; +/** @brief Abstraction of a serializer for Lifecycle + */ struct LCSerializer : Serializer { LCSerializer() {} virtual ~LCSerializer() = default; }; +/** + * @brief Abstraction for lifecycle processing + * + * Lifecycle processing loops over the objects in a bucket, applying per-bucket policy + * to each object. Examples of policy can be deleting after a certain amount of time, + * deleting extra versions, changing the storage class, and so on. + */ class Lifecycle { public: + /** Head of a lifecycle run. Used for tracking parallel lifecycle runs. */ struct LCHead { time_t start_date{0}; std::string marker; @@ -901,6 +1255,8 @@ class Lifecycle { LCHead(time_t _date, std::string& _marker) : start_date(_date), marker(_marker) {} }; + /** Single entry in a lifecycle run. Multiple entries can exist processing different + * buckets. */ struct LCEntry { std::string bucket; uint64_t start_time{0}; @@ -913,18 +1269,32 @@ class Lifecycle { Lifecycle() = default; virtual ~Lifecycle() = default; + /** Get an entry matching the given marker */ virtual int get_entry(const std::string& oid, const std::string& marker, LCEntry& entry) = 0; + /** Get the entry following the given marker */ virtual int get_next_entry(const std::string& oid, std::string& marker, LCEntry& entry) = 0; + /** Store a modified entry in then backing store */ virtual int set_entry(const std::string& oid, const LCEntry& entry) = 0; + /** List all known entries */ virtual int list_entries(const std::string& oid, const std::string& marker, uint32_t max_entries, std::vector& entries) = 0; + /** Remove an entry from the backing store */ virtual int rm_entry(const std::string& oid, const LCEntry& entry) = 0; + /** Get a head */ virtual int get_head(const std::string& oid, LCHead& head) = 0; + /** Store a modified head to the backing store */ virtual int put_head(const std::string& oid, const LCHead& head) = 0; + /** Get a serializer for lifecycle */ virtual LCSerializer* get_serializer(const std::string& lock_name, const std::string& oid, const std::string& cookie) = 0; }; +/** + * @brief Abstraction for a Notification event + * + * RGW can generate notifications for various events, such as object creation or + * deletion. + */ class Notification { protected: Object* obj; @@ -934,11 +1304,20 @@ class Notification { Notification(Object* _obj, rgw::notify::EventType _type) : obj(_obj), event_type(_type) {} virtual ~Notification() = default; + /** Indicate the start of the event associated with this notification */ virtual int publish_reserve(const DoutPrefixProvider *dpp, RGWObjTags* obj_tags = nullptr) = 0; + /** Indicate the successful completion of the event associated with this notification */ virtual int publish_commit(const DoutPrefixProvider* dpp, uint64_t size, const ceph::real_time& mtime, const std::string& etag, const std::string& version) = 0; }; +/** + * @brief Abstraction for an asynchronous writer + * + * Writing is done through a set of filters. This allows chaining filters to do things + * like compression and encryption on async writes. This is the base abstraction for + * those filters. + */ class Writer : public ObjectProcessor { protected: const DoutPrefixProvider* dpp; @@ -947,13 +1326,13 @@ class Writer : public ObjectProcessor { Writer(const DoutPrefixProvider *_dpp, optional_yield y) : dpp(_dpp) {} virtual ~Writer() = default; - // prepare to start processing object data + /** prepare to start processing object data */ virtual int prepare(optional_yield y) = 0; - // Process a bufferlist + /** Process a buffer. Called multiple times to write different buffers. */ virtual int process(bufferlist&& data, uint64_t offset) = 0; - // complete the operation and make its result visible to clients + /** complete the operation and make its result visible to clients */ virtual int complete(size_t accounted_size, const std::string& etag, ceph::real_time *mtime, ceph::real_time set_mtime, std::map& attrs, @@ -964,51 +1343,88 @@ class Writer : public ObjectProcessor { optional_yield y) = 0; }; +/** + * @brief Abstraction of a Zone + * + * This abstraction allows access to information about zones. This can be the zone + * containing the RGW, or another zone. + */ class Zone { public: virtual ~Zone() = default; + /** Get info about the zonegroup containing this zone */ virtual const RGWZoneGroup& get_zonegroup() = 0; + /** Get info about a zonegroup by ID */ virtual int get_zonegroup(const std::string& id, RGWZoneGroup& zonegroup) = 0; + /** Get the parameters of this zone */ virtual const RGWZoneParams& get_params() = 0; + /** Get the ID of this zone */ virtual const rgw_zone_id& get_id() = 0; + /** Get info about the realm containing this zone */ virtual const RGWRealm& get_realm() = 0; + /** Get the name of this zone */ virtual const std::string& get_name() const = 0; + /** True if this zone is writable */ virtual bool is_writeable() = 0; + /** Get the URL for the endpoint for redirecting to this zone */ virtual bool get_redirect_endpoint(std::string* endpoint) = 0; + /** Check to see if the given API is supported in this zone */ virtual bool has_zonegroup_api(const std::string& api) const = 0; + /** Get the current period ID for this zone */ virtual const std::string& get_current_period_id() = 0; }; +/** + * @brief Abstraction of a manager for Lua scripts + * + * RGW can load and process Lua scripts. This will handle loading/storing scripts. + */ class LuaScriptManager { -protected: - std::string script; - public: virtual ~LuaScriptManager() = default; + /** Get a script named with the given key from the backing store */ virtual int get(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key, std::string& script) = 0; + /** Put a script named with the given key to the backing store */ virtual int put(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key, const std::string& script) = 0; + /** Delete a script named with the given key from the backing store */ virtual int del(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key) = 0; }; +/** @} namespace rgw::sal in group RGWSAL */ } } // namespace rgw::sal +/** + * @brief A manager for Stores + * + * This will manage the singleton instances of the various stores. Stores come in two + * varieties: Full and Raw. A full store is suitable for use in a radosgw daemon. It + * has full access to the cluster, if any. A raw store is a stripped down store, used + * for admin commands. + */ class StoreManager { public: StoreManager() {} + /** Get a full store by service name */ static rgw::sal::Store* get_storage(const DoutPrefixProvider* dpp, CephContext* cct, const std::string svc, bool use_gc_thread, bool use_lc_thread, bool quota_threads, bool run_sync_thread, bool run_reshard_thread, bool use_cache = true, bool use_gc = true) { rgw::sal::Store* store = init_storage_provider(dpp, cct, svc, use_gc_thread, use_lc_thread, quota_threads, run_sync_thread, run_reshard_thread, use_cache, use_gc); return store; } + /** Get a stripped down store by service name */ static rgw::sal::Store* get_raw_storage(const DoutPrefixProvider* dpp, CephContext* cct, const std::string svc) { rgw::sal::Store* store = init_raw_storage_provider(dpp, cct, svc); return store; } + /** Initialize a new full Store */ static rgw::sal::Store* init_storage_provider(const DoutPrefixProvider* dpp, CephContext* cct, const std::string svc, bool use_gc_thread, bool use_lc_thread, bool quota_threads, bool run_sync_thread, bool run_reshard_thread, bool use_metadata_cache, bool use_gc); + /** Initialize a new raw Store */ static rgw::sal::Store* init_raw_storage_provider(const DoutPrefixProvider* dpp, CephContext* cct, const std::string svc); + /** Close a Store when it's no longer needed */ static void close_storage(rgw::sal::Store* store); }; + +/** @} */ diff --git a/src/rgw/rgw_sal_rados.h b/src/rgw/rgw_sal_rados.h index 9609547120586..bc19927a4df1c 100644 --- a/src/rgw/rgw_sal_rados.h +++ b/src/rgw/rgw_sal_rados.h @@ -528,7 +528,7 @@ class RadosMultipartUpload : public MultipartUpload { virtual const std::string& get_key() const override { return mp_obj.get_key(); } virtual const std::string& get_upload_id() const override { return mp_obj.get_upload_id(); } virtual const ACLOwner& get_owner() const override { return owner; } - virtual ceph::real_time get_mtime() const override { return mtime; } + virtual ceph::real_time& get_mtime() override { return mtime; } virtual std::unique_ptr get_meta_obj() override; virtual int init(const DoutPrefixProvider* dpp, optional_yield y, RGWObjectCtx* obj_ctx, ACLOwner& owner, rgw_placement_rule& dest_placement, rgw::sal::Attrs& attrs) override; virtual int list_parts(const DoutPrefixProvider* dpp, CephContext* cct, From 39bdd370edf8253c5b77293385c63fbf9cd552c5 Mon Sep 17 00:00:00 2001 From: Radoslaw Zarzynski Date: Tue, 26 Oct 2021 19:27:06 +0000 Subject: [PATCH 0576/2866] crimson/net: drop crimson-specific check for the addr in ClientIdentFrame. In crimson (but not in the classic OSD) we have an extra check that verifies the address sent by our peer in `ClientIdentFrame` matches `AsyncConnection::target_addr` at our side. In the Rook environment this leads to problems with all cluster entities that are lacking the `ms_learn_addr_from_peer=false` setting in their configurations. This is true for `ceph-mgr`: ``` [root@rook-ceph-tools-698545dc56-zxrrx /]# ceph config show mgr.a ms_learn_addr_from_peer true ``` Unfortunately, testing has shown that: * clients in Rook also lack this extra bit of configuration while * removing the extra check in crimson stops requiring any additional configuration at clients. Although this still might look like a workaround for Rook having `ms_learn_addr_from_peer=false` solely for OSDs, I think we should drop the check to preserve both: * consistency of behaviour between OSD implementations, * compatibility with Ceph clients in existing k8s clusters. ``` INFO 2021-10-26 18:53:26,067 [shard 0] ms - [osd.0(client) v2:172.17.0.2:6800/1270141526 >> unknown.? -@59700] ProtocolV2::start_accept(): target_addr=172.17.0.5:59700/0 DEBUG 2021-10-26 18:53:26,067 [shard 0] ms - [osd.0(client) v2:172.17.0.2:6800/1270141526 >> unknown.? -@59700] TRIGGER ACCEPTING, was NONE DEBUG 2021-10-26 18:53:26,067 [shard 0] ms - [osd.0(client) v2:172.17.0.2:6800/1270141526 >> unknown.? -@59700] SEND(26) banner: len_payload=16, supported=1, required=0, banner="ceph v2 " DEBUG 2021-10-26 18:53:26,068 [shard 0] ms - [osd.0(client) v2:172.17.0.2:6800/1270141526 >> unknown.? -@59700] RECV(10) banner: "ceph v2 " DEBUG 2021-10-26 18:53:26,068 [shard 0] ms - [osd.0(client) v2:172.17.0.2:6800/1270141526 >> unknown.? -@59700] GOT banner: payload_len=16 DEBUG 2021-10-26 18:53:26,068 [shard 0] ms - [osd.0(client) v2:172.17.0.2:6800/1270141526 >> unknown.? -@59700] RECV(16) banner features: supported=1 required=0 DEBUG 2021-10-26 18:53:26,068 [shard 0] ms - [osd.0(client) v2:172.17.0.2:6800/1270141526 >> unknown.? -@59700] WRITE HelloFrame: my_type=osd, peer_addr=172.17.0.5:59700/0 DEBUG 2021-10-26 18:53:26,068 [shard 0] ms - [osd.0(client) v2:172.17.0.2:6800/1270141526 >> unknown.? -@59700] GOT HelloFrame: my_type=client peer_addr=v2:172.17.0.2:6800/1270141526 INFO 2021-10-26 18:53:26,068 [shard 0] ms - [osd.0(client) v2:172.17.0.2:6800/1270141526 >> client.? -@59700] UPDATE: peer_type=client, policy(lossy=true server=true standby=false resetcheck=false) WARN 2021-10-26 18:53:26,068 [shard 0] ms - [osd.0(client) v2:172.17.0.2:6800/1270141526 >> client.? -@59700] my_addr_from_peer v2:172.17.0.2:6800/1270141526 port/nonce DOES match myaddr v2:172.17.0.2:6800/1270141526 DEBUG 2021-10-26 18:53:26,068 [shard 0] ms - [osd.0(client) v2:172.17.0.2:6800/1270141526 >> client.? -@59700] GOT AuthRequestFrame: method=2, preferred_modes={1, 2}, payload_len=174 INFO 2021-10-26 18:53:26,068 [shard 0] monc - added challenge on [osd.0(client) v2:172.17.0.2:6800/1270141526 >> client.? -@59700] DEBUG 2021-10-26 18:53:26,068 [shard 0] ms - [osd.0(client) v2:172.17.0.2:6800/1270141526 >> client.? -@59700] WRITE AuthReplyMoreFrame: payload_len=32 DEBUG 2021-10-26 18:53:26,068 [shard 0] ms - [osd.0(client) v2:172.17.0.2:6800/1270141526 >> client.? -@59700] GOT AuthRequestMoreFrame: payload_len=174 DEBUG 2021-10-26 18:53:26,069 [shard 0] ms - [osd.0(client) v2:172.17.0.2:6800/1270141526 >> client.? -@59700] WRITE AuthDoneFrame: gid=14788, con_mode=crc, payload_len=36 DEBUG 2021-10-26 18:53:26,069 [shard 0] ms - [osd.0(client) v2:172.17.0.2:6800/1270141526 >> client.? -@59700] WRITE AuthSignatureFrame: signature=975c5d3ae09036abcb2ca7d4f7704ee681ca13151d9de2ee29394ec8aed9950c DEBUG 2021-10-26 18:53:26,069 [shard 0] ms - [osd.0(client) v2:172.17.0.2:6800/1270141526 >> client.? -@59700] GOT AuthSignatureFrame: signature=6209032314d560a21a3109ec6d7c0623ebd78cf1ea4fc9462411dbabe28b2d8d DEBUG 2021-10-26 18:53:26,069 [shard 0] ms - [osd.0(client) v2:172.17.0.2:6800/1270141526 >> client.? -@59700] GOT ClientIdentFrame: addrs=172.17.0.1:0/1137248631, target=v2:172.17.0.2:6800/1270141526, gid=14788, gs=9, features_supported=4540138297136906239, features_required=576460752303427584, flags=1, cookie=0 WARN 2021-10-26 18:53:26,069 [shard 0] ms - [osd.0(client) v2:172.17.0.2:6800/1270141526 >> client.? -@59700] peer's address 172.17.0.1:0/1137248631 is not v2 or not the same host with 172.17.0.5:59700/0 INFO 2021-10-26 18:53:26,070 [shard 0] ms - [osd.0(client) v2:172.17.0.2:6800/1270141526 >> client.? -@59700] execute_accepting(): fault at ACCEPTING, going to CLOSING -- std::system_error (error crimson::net:2, bad peer address) ``` This connectivity issue has been overcome by appending `--ms_learn_addr_from_peer=false` to the `argv`: ``` [root@rook-ceph-tools-698545dc56-zxrrx /]# bin/rados bench -p test-pool 5 rand --ms_learn_addr_from_peer=false hints = 1 sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 0 0 0 0 0 0 - 0 1 16 1235 1219 4.76106 4.76172 0.020999 0.0129408 2 16 2531 2515 4.91158 5.0625 0.0131776 0.0126796 3 16 3746 3730 4.8563 4.74609 0.0145268 0.0128361 4 16 4951 4935 4.81889 4.70703 0.0154604 0.0129421 5 15 6236 6221 4.85972 5.02344 0.0121689 0.0128415 Total time run: 5.01136 Total reads made: 6236 Read size: 4096 Object size: 4096 Bandwidth (MB/sec): 4.86083 Average IOPS: 1244 Stddev IOPS: 43.1706 Max IOPS: 1296 Min IOPS: 1205 Average Latency(s): 0.01284 Max latency(s): 0.0244048 Min latency(s): 0.00201867 ``` However, on classical OSD and **crimson with this patch applied** there is no need for any configurables at the client-side: ``` [rook@rook-ceph-tools-698545dc56-xkkpf /]$ bin/rados bench -p test-pool 5 rand hints = 1 sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 0 0 0 0 0 0 - 0 1 16 1124 1108 4.32747 4.32812 0.011878 0.0143472 2 16 2323 2307 4.50534 4.68359 0.0117413 0.0138221 3 16 3517 3501 4.55813 4.66406 0.0195142 0.0136663 4 16 4680 4664 4.55425 4.54297 0.0131425 0.0136958 5 16 5725 5709 4.45976 4.08203 0.0143174 0.0139868 Total time run: 5.01332 Total reads made: 5725 Read size: 4096 Object size: 4096 Bandwidth (MB/sec): 4.46077 Average IOPS: 1141 Stddev IOPS: 65.113 Max IOPS: 1199 Min IOPS: 1045 Average Latency(s): 0.0139892 Max latency(s): 0.0361518 Min latency(s): 0.00231195 ``` Signed-off-by: Radoslaw Zarzynski --- src/crimson/net/ProtocolV2.cc | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/src/crimson/net/ProtocolV2.cc b/src/crimson/net/ProtocolV2.cc index b7aef560b682e..fa47da50832c0 100644 --- a/src/crimson/net/ProtocolV2.cc +++ b/src/crimson/net/ProtocolV2.cc @@ -1209,18 +1209,7 @@ ProtocolV2::server_connect() throw std::system_error( make_error_code(crimson::net::error::bad_peer_address)); } - // TODO: change peer_addr to entity_addrvec_t - entity_addr_t paddr = client_ident.addrs().front(); - if ((paddr.is_msgr2() || paddr.is_any()) && - paddr.is_same_host(conn.target_addr)) { - // good - } else { - logger().warn("{} peer's address {} is not v2 or not the same host with {}", - conn, paddr, conn.target_addr); - throw std::system_error( - make_error_code(crimson::net::error::bad_peer_address)); - } - conn.peer_addr = paddr; + conn.peer_addr = client_ident.addrs().front(); logger().debug("{} UPDATE: peer_addr={}", conn, conn.peer_addr); conn.target_addr = conn.peer_addr; if (!conn.policy.lossy && !conn.policy.server && conn.target_addr.get_port() <= 0) { From 97ac43822e1f54eb217701c411f49e483ce92bfd Mon Sep 17 00:00:00 2001 From: Kamoltat Date: Fri, 29 Oct 2021 21:23:52 +0000 Subject: [PATCH 0577/2866] pybind/mgr/pg_autoscaler: typo default option scale-up to scale-down Typo: `scale-up` should be `scale-down` in Module Option. This typo doesn't trigger a bug because we create a key-value of `scale-down` profile in the function `create_initial()` in `src/mon/KVMonitor.cc`. This will override whatever is the default option in pg_autoscaler/module.py when we start the cluster and the monitor gets created. The command: `ceph osd pool set autoscale-profile