example R: add the librpma example

examples/R/.gitignore

@@ -0,0 +1 @@
+build/

examples/R/CMakeLists.txt

@@ -0,0 +1,50 @@
+#
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright 2020-2021, Intel Corporation
+# Copyright 2021, Fujitsu
+#
+
+cmake_minimum_required(VERSION 3.3)
+project(exampleR C)
+
+set(LIBPMEM_REQUIRED_VERSION 1.6)
+
+# append -Wall and -Werror to CMAKE_C_FLAGS
+set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall")
+set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror")
+
+set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${CMAKE_SOURCE_DIR}/cmake)
+
+find_package(PkgConfig QUIET)
+
+if(PKG_CONFIG_FOUND)
+	pkg_check_modules(LIBRPMA librpma)
+	pkg_check_modules(LIBIBVERBS libibverbs)
+	pkg_check_modules(LIBPMEM libpmem>=${LIBPMEM_REQUIRED_VERSION})
+endif()
+if(NOT LIBRPMA_FOUND)
+	find_package(LIBRPMA REQUIRED librpma)
+endif()
+if(NOT LIBIBVERBS_FOUND)
+	find_package(LIBIBVERBS REQUIRED libibverbs)
+endif()
+if(NOT LIBPMEM_FOUND)
+	find_package(LIBPMEM ${LIBPMEM_REQUIRED_VERSION} REQUIRED libpmem)
+endif()
+
+link_directories(${LIBRPMA_LIBRARY_DIRS})
+
+function(add_example name)
+	set(srcs ${ARGN})
+	add_executable(${name} ${srcs})
+	target_include_directories(${name} PRIVATE ${LIBRPMA_INCLUDE_DIRS} ${LIBPMEM_INCLUDE_DIRS})
+	target_link_libraries(${name} rpma ${LIBIBVERBS_LIBRARIES} ${LIBRT_LIBRARIES} ${LIBPMEM_LIBRARIES})
+endfunction()
+
+add_example(server server.c common.c)
+add_example(client client.c common.c)
+add_example(simple_server simple_server.c common.c)
+add_example(simple_client simple_client.c common.c)
+
+add_custom_target(config_softroce
+	COMMAND ${CMAKE_SOURCE_DIR}/tools/config_softroce.sh)

examples/R/README.md

@@ -0,0 +1,32 @@
+Example of performing an RPMA read, write and flush to persistence operations
+===
+
+This example implements both sides of an RDMA connection:
+- The server prepares a local persistent memory and exposes the memory description
+along with other parameters required to perform an RDMA read, write and flush operations.
+After the connection is established, the server waits for the client to disconnect.
+- The client allocates memory from DRAM and registers it as a reading destination
+and writing source. After the connection is established the client receives
+the server's memory regions registered as a reading source and a writing destination.
+The client performs the RDMA read from the remote memory region to the local memory region,
+then it writes new data to the local memory region and performs the RDMA write
+from the local memory region to the remote memory region followed by the RPMA flush.
+
+**Note**: The server requires a unique <user-id> argument in order to use
+a different part of persistent memory which is shared by all the server instances
+running on the same <pmem-path>. <user-id> is also used to pick a unique TCP port.
+
+**Note**: For the sake of this example, the memory region being written to and
+the server's peer configuration are transferred via the connection's private
+data. In general, it can be transferred via an out-of-band or the in-band
+channel.
+
+## Usage
+
+```bash
+[user@server]$ ./server $server_address $port ${pmem-path} ${user-id}
+```
+
+```bash
+[user@client]$ ./client $server_address $port
+```

examples/R/build_main.sh

@@ -0,0 +1,10 @@
+#!/bin/bash -ex
+#
+# The shell commands to build the main example.
+#
+
+mkdir -p build
+cd build
+cmake ..
+make server
+make client

examples/R/build_simple.sh

@@ -0,0 +1,10 @@
+#!/bin/bash -ex
+#
+# The shell commands to build the simple_* example.
+#
+
+mkdir -p build
+cd build
+cmake ..
+make simple_client
+make simple_server

examples/R/client.c

@@ -0,0 +1,275 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/* Copyright 2020-2021, Intel Corporation */
+/* Copyright 2021, Fujitsu */
+
+/*
+ * client.c -- a client of the "read, write and flush to persistent domain" example
+ */
+
+#include <inttypes.h>
+#include <stdlib.h>
+#include <librpma.h>
+#include <unistd.h>
+#include "common.h"
+
+#define USAGE_STR	"usage: %s <server_address> <port>\n"
+#define FLUSH_ID	(void *)0xF01D /* a random identifier */
+
+static const char *hello_str = "Hello world!";
+
+/*
+ * malloc_aligned -- allocate an aligned chunk of memory
+ */
+void *
+malloc_aligned(size_t size)
+{
+	long pagesize = sysconf(_SC_PAGESIZE);
+	if (pagesize < 0) {
+		perror("sysconf");
+		return NULL;
+	}
+
+	/* allocate a page size aligned local memory pool */
+	void *mem;
+	int ret = posix_memalign(&mem, (size_t)pagesize, size);
+	if (ret) {
+		(void) fprintf(stderr, "Client: error: posix_memalign: %s\n",
+				strerror(ret));
+		return NULL;
+	}
+
+	/* zero the allocated memory */
+	memset(mem, 0, size);
+
+	return mem;
+}
+
+int
+main(int argc, char *argv[])
+{
+	/* validate parameters */
+	if (argc < 3) {
+		fprintf(stderr, USAGE_STR, argv[0]);
+		exit(-1);
+	}
+
+	/* configure logging thresholds to see more details */
+	rpma_log_set_threshold(RPMA_LOG_THRESHOLD, RPMA_LOG_LEVEL_INFO);
+	rpma_log_set_threshold(RPMA_LOG_THRESHOLD_AUX, RPMA_LOG_LEVEL_INFO);
+
+	/* read common parameters */
+	char *addr = argv[1];
+	char *port = argv[2];
+	int ret;
+
+	/* resources - memory region */
+	void *local_mr_ptr;
+	size_t local_mr_size;
+	size_t local_offset = 0;
+	struct rpma_mr_remote *remote_mr = NULL;
+	size_t remote_size = 0;
+	size_t remote_offset = 0;
+	struct rpma_mr_local *local_mr = NULL;
+	struct ibv_wc wc;
+
+	/* RPMA resources */
+	struct rpma_peer_cfg *pcfg = NULL;
+	struct rpma_peer *peer = NULL;
+	struct rpma_conn *conn = NULL;
+	bool direct_write_to_pmem = false;
+
+	local_mr_size = KILOBYTE;
+	local_mr_ptr = malloc_aligned(local_mr_size);
+	if (local_mr_ptr == NULL)
+		return -1;
+
+	/*
+	 * lookup an ibv_context via the address and create a new peer using it
+	 */
+	ret = common_client_peer_via_address(addr, &peer);
+	if (ret)
+		goto err_free;
+
+	/* establish a new connection to a server listening at addr:port */
+	ret = common_client_connect(peer, addr, port, NULL, NULL, &conn);
+	if (ret)
+		goto err_peer_delete;
+
+	/* register the memory for RDMA read and write operations */
+	ret = rpma_mr_reg(peer, local_mr_ptr, local_mr_size,
+			(RPMA_MR_USAGE_READ_DST | RPMA_MR_USAGE_WRITE_SRC),
+			&local_mr);
+	if (ret)
+		goto err_conn_disconnect;
+
+	/* obtain the remote side resources description */
+	struct rpma_conn_private_data pdata;
+	ret = rpma_conn_get_private_data(conn, &pdata);
+	if (ret != 0 || pdata.len < sizeof(struct common_data))
+		goto err_mr_dereg;
+
+	/*
+	 * Create a remote peer configuration structure from the received
+	 * descriptor and apply it to the current connection.
+	 */
+	struct common_data *remote_data = pdata.ptr;
+	ret = rpma_peer_cfg_from_descriptor(
+			&remote_data->descriptors[remote_data->mr_desc_size],
+			remote_data->pcfg_desc_size, &pcfg);
+	if (ret)
+		goto err_mr_dereg;
+	ret = rpma_peer_cfg_get_direct_write_to_pmem(pcfg,
+			&direct_write_to_pmem);
+	ret |= rpma_conn_apply_remote_peer_cfg(conn, pcfg);
+	(void) rpma_peer_cfg_delete(&pcfg);
+	/* either get or apply failed */
+	if (ret)
+		goto err_mr_dereg;
+
+	/* verify the Direct Write to PMem support */
+	if (!direct_write_to_pmem) {
+		(void) fprintf(stderr,
+			"Client: error: the server does not support Direct Write to PMem\n");
+		goto err_mr_dereg;
+	}
+
+	/*
+	 * Create a remote memory registration structure from the received
+	 * descriptor.
+	 */
+	ret = rpma_mr_remote_from_descriptor(&remote_data->descriptors[0],
+			remote_data->mr_desc_size, &remote_mr);
+	if (ret)
+		goto err_mr_dereg;
+
+	/* get the remote memory region size */
+	ret = rpma_mr_remote_get_size(remote_mr, &remote_size);
+	if (ret) {
+		goto err_mr_remote_delete;
+	} else if (remote_size < KILOBYTE) {
+		fprintf(stderr,
+			"Client: error: remote memory region size too small for writing the data of the assumed size (%zu < %d)\n",
+			remote_size, KILOBYTE);
+		goto err_mr_remote_delete;
+	}
+
+	/* read the initial value */
+	size_t len = (local_mr_size < remote_size) ? local_mr_size : remote_size;
+	ret = rpma_read(conn, local_mr, local_offset,
+			remote_mr, remote_offset, len,
+			RPMA_F_COMPLETION_ALWAYS, NULL);
+	if (ret)
+		goto err_mr_remote_delete;
+
+	/* get the connection's main CQ */
+	struct rpma_cq *cq = NULL;
+	ret = rpma_conn_get_cq(conn, &cq);
+	if (ret)
+		goto err_mr_remote_delete;
+
+	/* wait for the completion to be ready */
+	ret = rpma_cq_wait(cq);
+	if (ret)
+		goto err_mr_remote_delete;
+
+	/* wait for a completion of the RDMA read */
+	ret = rpma_cq_get_wc(cq, 1, &wc, NULL);
+	if (ret)
+		goto err_mr_remote_delete;
+
+	if (wc.status != IBV_WC_SUCCESS) {
+		ret = -1;
+		(void) fprintf(stderr, "Client: error: rpma_read() failed: %s\n",
+				ibv_wc_status_str(wc.status));
+		goto err_mr_remote_delete;
+	}
+
+	if (wc.opcode != IBV_WC_RDMA_READ) {
+		ret = -1;
+		(void) fprintf(stderr, "Client: error: unexpected wc.opcode value (%d != %d)\n",
+				wc.opcode, IBV_WC_RDMA_READ);
+		goto err_mr_remote_delete;
+	}
+
+	(void) fprintf(stdout, "Client: read the initial content of the remote (server's) persistent memory: %s\n",
+			(char *)local_mr_ptr + local_offset);
+
+	/* write the next value */
+	strncpy(local_mr_ptr, hello_str, KILOBYTE);
+	(void) printf("Client: writing to the remote (server's) persistent memory: %s\n",
+			(char *)local_mr_ptr);
+
+	ret = rpma_write(conn, remote_mr, remote_offset,
+			local_mr, local_offset, KILOBYTE,
+			RPMA_F_COMPLETION_ON_ERROR, NULL);
+	if (ret) {
+		(void) fprintf(stderr, "Client: error: rpma_write() failed: %s\n",
+				rpma_err_2str(ret));
+		goto err_mr_remote_delete;
+	}
+
+	(void) printf("Client: flushing the remote data to the persistent domain...\n");
+
+	ret = rpma_flush(conn, remote_mr, remote_offset, KILOBYTE,
+			RPMA_FLUSH_TYPE_PERSISTENT,
+			RPMA_F_COMPLETION_ALWAYS, FLUSH_ID);
+	if (ret) {
+		(void) fprintf(stderr, "Client: error: rpma_flush() failed: %s\n",
+				rpma_err_2str(ret));
+		goto err_mr_remote_delete;
+	}
+
+	/* get the connection's main CQ */
+	ret = rpma_conn_get_cq(conn, &cq);
+	if (ret)
+		goto err_mr_remote_delete;
+
+	/* wait for the completion to be ready */
+	ret = rpma_cq_wait(cq);
+	if (ret)
+		goto err_mr_remote_delete;
+
+	/* wait for a completion of the RDMA read */
+	ret = rpma_cq_get_wc(cq, 1, &wc, NULL);
+	if (ret)
+		goto err_mr_remote_delete;
+
+	if (wc.wr_id != (uintptr_t)FLUSH_ID) {
+		ret = -1;
+		(void) fprintf(stderr,
+				"Client: error: unexpected wc.wr_id value "
+				"(0x%" PRIXPTR " != 0x%" PRIXPTR ")\n",
+				(uintptr_t)wc.wr_id, (uintptr_t)FLUSH_ID);
+		goto err_mr_remote_delete;
+	}
+	if (wc.status != IBV_WC_SUCCESS) {
+		ret = -1;
+		(void) fprintf(stderr, "Client: error: rpma_flush() failed: %s\n",
+				ibv_wc_status_str(wc.status));
+		goto err_mr_remote_delete;
+	}
+
+err_mr_remote_delete:
+	/* delete the remote memory region's structure */
+	(void) rpma_mr_remote_delete(&remote_mr);
+
+err_mr_dereg:
+	/* deregister the memory region */
+	(void) rpma_mr_dereg(&local_mr);
+
+err_conn_disconnect:
+	/*
+	 * Disconnect, wait for RPMA_CONN_CLOSED
+	 * and delete the connection structure.
+	 */
+	(void) common_disconnect_and_wait_for_conn_close(&conn);
+
+err_peer_delete:
+	/* delete the peer */
+	(void) rpma_peer_delete(&peer);
+
+err_free:
+	free(local_mr_ptr);
+
+	return ret;
+}