example R: add the librpma example

doc/Install-CentOS8

@@ -37,6 +37,18 @@ It used the 2021-02-19 repo.  You can adjust your commands based on running in a
 	$ sudo dnf --disablerepo="*" --enablerepo="elrepo-kernel" list available | grep kernel-ml
 Install the latest mainline kernel
 	$ sudo dnf --enablerepo=elrepo-kernel install kernel-ml
+
+2b. [required only for Example R] Configure Soft-RoCE
+	In order to configure Soft-RoCE follow the manuals:
+	- https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/7/html/networking_guide/sec-configuring_soft-_roce or/and
+	- https://support.mellanox.com/s/article/howto-configure-soft-roce
+
+Add also the following 4 lines to the end of the '/etc/security/limits.conf' file:
+* soft memlock unlimited
+* hard memlock unlimited
+* soft nofile 1048000
+* hard nofile 1048000
+to set the values of maximum locked memory and maximum open files required by Soft-RoCE.
 
 3. Install the basics you care about, something like:
 	$ sudo yum -y install dnf git ipmctl ndctl vim-enhanced firewalld certbot
@@ -121,6 +133,16 @@ Install the latest mainline kernel
 	$ sudo ipmctl create -goal PersistentMemoryType=AppDirect
 	$ sudo systemctl reboot
 
+11a. [Example R]
+
+	$ sudo ndctl create-namespace -f -e namespace0.0 --mode=devdax
+	$ sudo chown -R $(whoami):$(whoami) /dev/dax0.0
+	$ sudo chmod a+rw /dev/dax0.0
+
+Go to bullet 13. (Add a root cron job to expire old webhackathon sessions).
+
+11b. [examples other than Example R]
+
 	$ sudo ndctl create-namespace --mode fsdax --continue
 (note: this will fail if there is already an fsxdax on the memory, go to the next step)
 

docker/Dockerfile

@@ -11,7 +11,15 @@ FROM fedora:30
 
 LABEL maintainer="[email protected]"
 
+ENV RPMA_DEPS "\
+	iproute \
+	iputils \
+	librdmacm-utils \
+	net-tools \
+	rdma-core-devel"
+
 RUN dnf update -y && dnf install -y\
+	$RPMA_DEPS\
 	autoconf\
 	automake\
 	bash-completion\
@@ -103,7 +111,10 @@ RUN /pmemkv-ruby.sh
 COPY memkind.sh /
 RUN /memkind.sh
 
+COPY librpma.sh /
+RUN /librpma.sh
+
 COPY tz.sh /
 RUN /tz.sh
 
-RUN rm /pmdk.sh /valgrind.sh /pmemobj-cpp.sh /pmemkv.sh /setup-maven-settings.sh /pmemkv-java.sh /pmemkv-python.sh /pmemkv-nodejs.sh /pmemkv-ruby.sh /memkind.sh /tz.sh
+RUN rm /pmdk.sh /valgrind.sh /pmemobj-cpp.sh /pmemkv.sh /setup-maven-settings.sh /pmemkv-java.sh /pmemkv-python.sh /pmemkv-nodejs.sh /pmemkv-ruby.sh /memkind.sh /librpma.sh /tz.sh

docker/librpma.sh

@@ -0,0 +1,19 @@
+#!/bin/bash -ex
+
+LIBRPMA_VERSION=1.1.0
+ZIP_FILE=rpma.zip
+
+# install librpma
+wget -O $ZIP_FILE https://github.com/pmem/rpma/archive/${LIBRPMA_VERSION}.zip
+unzip $ZIP_FILE
+mkdir -p rpma-${LIBRPMA_VERSION}/build
+pushd rpma-${LIBRPMA_VERSION}/build
+cmake .. -DCMAKE_BUILD_TYPE=Debug \
+	-DCMAKE_INSTALL_PREFIX=/usr \
+	-DBUILD_DOC=OFF \
+	-DBUILD_EXAMPLES=OFF \
+	-DBUILD_TESTS=OFF
+make -j$(nproc)
+sudo make -j$(nproc) install
+popd
+rm -rf $ZIP_FILE rpma-${LIBRPMA_VERSION}

scripts/config_softroce.sh

@@ -0,0 +1,119 @@
+#!/usr/bin/env bash
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright 2020-2022, Intel Corporation
+
+#
+# config_softroce.sh - configure SoftRoCE
+#
+# This script configures a software-emulated RDMA network interface (SoftRoCE),
+# which requires:
+# - one machine with a regular Ethernet network adapter,
+# - the libibverbs and librdmacm libraries (or the rdma-core package
+#   containing both of them) installed,
+# - the 'rdma_rxe' kernel module loaded.
+#
+# Usage: config_softroce.sh [<network_interface>]
+#
+# Options:
+# <network_interface> - configure SoftRoCE for the given <network_interface>
+#                       or for the first active and up one if no argument
+#                       is given
+#
+
+NET_IF=$1
+
+MODULE="rdma_rxe"
+DIR="/lib/modules/$(uname -r)"
+STATE_OK="state ACTIVE physical_state LINK_UP"
+
+function get_IP4() {
+	NET_IF=$1
+	ip -4 -j -p a show $NET_IF | grep -e local | cut -d'"' -f4
+}
+
+function print_IP4_of() {
+	NET_IF=$1
+	IP=$(get_IP4 $NET_IF)
+	if [ "$IP" != "" ]; then
+		echo -n "$IP"
+	else
+		echo "no IP address assigned"
+		exit 1
+	fi
+}
+
+function print_IP4_all() {
+	NET_IFS=$(rdma link show | grep -e "$STATE_OK" | cut -d' ' -f8)
+	for NET_IF in $NET_IFS; do
+		IP=$(get_IP4 $NET_IF)
+		[ "$IP" != "" ] && echo -n "$IP"
+	done
+}
+
+if [ $(lsmod | grep -e $MODULE | wc -l) -lt 1 ]; then
+	N_MODULES=$(find $DIR -name "$MODULE.ko*" | wc -l)
+	if [ $N_MODULES -lt 1 ]; then
+		echo "Error: cannot find the '$MODULE' module in the '$DIR' directory"
+		exit 1
+	fi
+
+	if ! sudo modprobe $MODULE; then
+		echo "Error: cannot load the '$MODULE' module"
+		sudo modprobe -v $MODULE
+		exit 1
+	fi
+fi
+
+if ! which ip > /dev/null; then
+	echo "Error: cannot find the 'ip' command. Install the 'iproute/iproute2' package"
+	exit 1
+fi
+
+if ! which rdma > /dev/null; then
+	echo "Error: cannot find the 'rdma' command. Install the 'iproute/iproute2' package"
+	exit 1
+fi
+
+if ! rdma link show > /dev/null ; then
+	echo "Error: the 'rdma link show' command failed"
+	exit 1
+fi
+
+if [ "$NET_IF" == "" ]; then
+	RDMA_LINKS=$(rdma link show | grep -e "$STATE_OK" | wc -l)
+	if [ $RDMA_LINKS -gt 0 ]; then
+		print_IP4_all
+		exit 0
+	fi
+
+	# pick up the first 'up' network interface
+	NET_IF=$(ip link | grep -v -e "LOOPBACK" | grep -e "state UP" | head -n1 | cut -d: -f2 | cut -d' ' -f2)
+	if [ "$NET_IF" == "" ]; then
+		#
+		# Look for a USB Ethernet network interfaces,
+		# which may not have 'state UP',
+		# but only 'UP' and 'state UNKNOWN', for example:
+		# ... <BROADCAST,MULTICAST,UP,LOWER_UP> ... state UNKNOWN ...
+		#
+		NET_IF=$(ip link | grep -v -e "LOOPBACK" | grep -e "UP" | grep -e "state UNKNOWN" | head -n1 | cut -d: -f2 | cut -d' ' -f2)
+		if [ "$NET_IF" == "" ]; then
+			echo "Error: cannot find an active and up network interface"
+			exit 1
+		fi
+	fi
+fi
+
+RXE_NAME="rxe_$NET_IF"
+sudo rdma link add $RXE_NAME type rxe netdev $NET_IF
+if [ $? -ne 0 ]; then
+	echo "Error: configuring SoftRoCE failed"
+	exit 1
+fi
+
+RDMA_LINKS=$(rdma link show | grep -e "$STATE_OK" | grep -e "$NET_IF" | wc -l)
+if [ $RDMA_LINKS -lt 1 ]; then
+	echo "Error: configuring SoftRoCE for the '$NET_IF' network interface failed"
+	exit 1
+fi
+
+print_IP4_of $NET_IF

scripts/enable_pmemusers

@@ -85,13 +85,33 @@ die "$H/img directory not found\n" unless -d "$H/img";
 die "$H/js directory not found\n" unless -d "$H/js";
 die "$H/templates directory not found\n" unless -d "$H/templates";
 die "$H/users directory not found\n" unless -d "$H/users";
+die "the jq command is missing\n" unless system("which jq > /dev/null") == 0;
 
 my $passwd = `grep '^pmemuser[0-9][0-9]*:' /etc/passwd`;
 my $nopasswd = $? >> 8;
 die "no pmemuserX accounts in /etc/passwd\n" if $nopasswd;
 
 my @pmem_mounts = ('/pmem0', '/pmem1');
 
+# set name of device DAX
+my $dax_name = "dax0.0";
+
+# check if the device DAX exists
+my $dev_dax = "/dev/$dax_name";
+die "$dev_dax device DAX not found\n" unless -c "$dev_dax";
+
+# look for device DAX in ndctl
+my $exists = `ndctl list | grep -c -e "$dax_name"`;
+chomp($exists);
+die "$dev_dax device DAX not listed by ndctl\n" if ($exists != 1);
+
+# verify if the device DAX has the minimum required size of 800KiB (4KiB per user)
+my $size = `ndctl list | jq '.[] | select(.chardev=="$dax_name")' | jq '.size'`;
+die "size of device DAX is less than 800KiB ($size < 819200)\n" if ($size < 819200);
+
+# add RW permissions for all users to the device DAX
+runy("chmod a+rw $dev_dax");
+
 for (my $id = $Lo; $id <= $Hi; $id++) {
 	my $thisid = $BASEID + $id;
 	my $user = "pmemuser$id";
@@ -105,7 +125,18 @@ for (my $id = $Lo; $id <= $Hi; $id++) {
 		runy("echo '$user:$Pass' | chpasswd");
 	}
 
-	runy("docker run -t -d --name=$user -v $H/users/$user/shadow:/etc/shadow -v $H/users/$user/passwd:/etc/passwd -v $H/users/$user/group:/etc/group -v $H/users/$user/home:/home/$user -v $pmemdir:/pmem -u $thisid:$thisid --hostname=container$id --workdir=/home/$user --cap-add SYS_PTRACE --restart=unless-stopped $Image");
+	runy("echo 'enable_pmemusers: PMEMUSER_ID=$id'");
+	runy("echo 'enable_pmemusers: DEV_DAX=$dev_dax'");
+
+	my $ip = `./config_softroce.sh`;
+	if ($? >> 8) {
+		say("WARNING: config_softroce.sh failed, SoftRoCE is not configured, the Example R will not work!");
+		$ip = "config_softroce.sh_failed";
+		runy("docker run -t -d --name=$user --env RPMA_SOFT_ROCE_IP=$ip --env PMEMUSER_ID=$id --env DEV_DAX=$dev_dax -v $H/users/$user/shadow:/etc/shadow -v $H/users/$user/passwd:/etc/passwd -v $H/users/$user/group:/etc/group -v $H/users/$user/home:/home/$user -v $pmemdir:/pmem -u $thisid:$thisid --hostname=container$id --workdir=/home/$user --cap-add SYS_PTRACE --restart=unless-stopped $Image");
+	} else {
+		runy("echo 'enable_pmemusers: RPMA_SOFT_ROCE_IP=$ip'");
+		runy("docker run -t -d --name=$user --env RPMA_SOFT_ROCE_IP=$ip --env PMEMUSER_ID=$id --env DEV_DAX=$dev_dax --network host --device /dev/infiniband --device $dev_dax -v /sys/class/infiniband:/sys/class/infiniband -v /sys/class/infiniband_verbs:/sys/class/infiniband_verbs -v /sys/class/misc/rdma_cm:/sys/class/misc/rdma_cm -v $H/users/$user/shadow:/etc/shadow -v $H/users/$user/passwd:/etc/passwd -v $H/users/$user/group:/etc/group -v $H/users/$user/home:/home/$user -v $pmemdir:/pmem -u $thisid:$thisid --hostname=container$id --workdir=/home/$user --cap-add SYS_PTRACE --restart=unless-stopped $Image");
+	}
 }
 
 say('done');