Add linux ARM64 wheels block in Semaphore

Author: anchitj

.semaphore/semaphore.yml

@@ -9,9 +9,6 @@ global_job_config:
       value: v2.0.2
   prologue:
     commands:
-      - export HOME=$WORKSPACE
-      - mkdir $WORKSPACE/confluent-kafka-python
-      - cd $WORKSPACE/confluent-kafka-python
       - checkout
 blocks:
   - name: "Wheels: OSX x64"
@@ -54,7 +51,7 @@ blocks:
             - PIP_INSTALL_OPTIONS="--user" tools/wheels/build-wheels.sh "${LIBRDKAFKA_VERSION#v}" wheelhouse
             - tar -czf wheelhouse-macOS-${ARCH}.tgz wheelhouse
             - artifact push workflow wheelhouse-macOS-${ARCH}.tgz
-  
+
   - name: Source package verification with Python 3 (OSX x64) +docs
     dependencies: []
     task:
@@ -83,3 +80,22 @@ blocks:
             # install confluent-kafka
             - python setup.py build && python setup.py install
             - make docs
+  - name: "Wheels: Linux arm64"
+#    run:
+#      when: "tag =~ '.*'"
+    dependencies: []
+    task:
+      agent:
+        machine:
+          type: s1-prod-ubuntu20-04-arm64-1
+      env_vars:
+        - name: OS_NAME
+          value: linux
+        - name: ARCH
+          value: arm64
+      jobs:
+        - name: Build
+          commands:
+            - ./tools/build-manylinux.sh "${LIBRDKAFKA_VERSION#v}"
+            - tar -czf wheelhouse-linux-${ARCH}.tgz wheelhouse
+            - artifact push workflow wheelhouse-linux-${ARCH}.tgz

CHANGELOG.md

@@ -4,6 +4,7 @@
 
 - Added `set_sasl_credentials`. This new method (on the Producer, Consumer, and AdminClient) allows modifying the stored
   SASL PLAIN/SCRAM credentials that will be used for subsequent (new) connections to a broker (#1511).
+- Wheels for Linux / arm64 (#1496).
 
 
 ## v2.0.2

src/confluent_kafka/src/confluent_kafka.c

@@ -2758,7 +2758,11 @@ static struct PyModuleDef cimpl_moduledef = {
 static PyObject *_init_cimpl (void) {
 	PyObject *m;
 
+/* PyEval_InitThreads became deprecated in Python 3.9 and will be removed in Python 3.11.
+ * Prior to Python 3.7, this call was required to initialize the GIL. */
+#if PY_VERSION_HEX < 0x03090000
         PyEval_InitThreads();
+#endif
 
 	if (PyType_Ready(&KafkaErrorType) < 0)
 		return NULL;

tools/bootstrap-librdkafka.sh

@@ -29,7 +29,7 @@ mkdir -p "$BUILDDIR/librdkafka"
 pushd "$BUILDDIR/librdkafka"
 
 test -f configure ||
-curl -q -L "https://github.com/edenhill/librdkafka/archive/${VERSION}.tar.gz" | \
+curl -q -L "https://github.com/confluentinc/librdkafka/archive/refs/tags/${VERSION}.tar.gz" | \
     tar -xz --strip-components=1 -f -
 
 ./configure --clean

tools/build-manylinux.sh

@@ -33,7 +33,13 @@ if [[ ! -f /.dockerenv ]]; then
         exit 1
     fi
 
-    docker run -t -v $(pwd):/io quay.io/pypa/manylinux2010_x86_64:latest  /io/tools/build-manylinux.sh "$LIBRDKAFKA_VERSION"
+    if [[ $ARCH == arm64* ]]; then
+        docker_image=quay.io/pypa/manylinux_2_28_aarch64:latest
+    else
+        docker_image=quay.io/pypa/manylinux_2_28_x86_64:latest
+    fi
+
+    docker run -t -v $(pwd):/io $docker_image  /io/tools/build-manylinux.sh "v${LIBRDKAFKA_VERSION}"
 
     exit $?
 fi
@@ -44,14 +50,14 @@ fi
 #
 
 echo "# Installing basic system dependencies"
-yum install -y zlib-devel gcc-c++
+yum install -y zlib-devel gcc-c++ python3 curl-devel perl-IPC-Cmd perl-Pod-Html
 
 echo "# Building librdkafka ${LIBRDKAFKA_VERSION}"
 $(dirname $0)/bootstrap-librdkafka.sh --require-ssl ${LIBRDKAFKA_VERSION} /usr
 
 # Compile wheels
 echo "# Compile"
-for PYBIN in /opt/python/*/bin; do
+for PYBIN in /opt/python/cp*/bin; do
     echo "## Compiling $PYBIN"
     CFLAGS="-Werror -Wno-strict-aliasing -Wno-parentheses" \
           "${PYBIN}/pip" wheel /io/ -w unrepaired-wheelhouse/
@@ -73,13 +79,13 @@ done
 
 # Install packages and test
 echo "# Installing wheels"
-for PYBIN in /opt/python/*/bin/; do
+for PYBIN in /opt/python/cp*/bin/; do
     echo "## Installing $PYBIN"
     "${PYBIN}/pip" install confluent_kafka -f /io/wheelhouse
     "${PYBIN}/python" -c 'import confluent_kafka; print(confluent_kafka.libversion())'
+    "${PYBIN}/pip" install -r /io/tests/requirements.txt
+    "${PYBIN}/pytest" /io/tests/test_Producer.py
     echo "## Uninstalling $PYBIN"
     "${PYBIN}/pip" uninstall -y confluent_kafka
 done
 
-
-

tools/wheels/install-librdkafka.sh

@@ -29,7 +29,11 @@ if [[ $OSTYPE == linux* ]]; then
     # Linux
 
     # Copy the librdkafka build with least dependencies to librdkafka.so.1
-    cp -v runtimes/linux-$ARCH/native/{centos6-librdkafka.so,librdkafka.so.1}
+    if [[ $ARCH == arm64* ]]; then
+        cp -v runtimes/linux-$ARCH/native/{librdkafka.so,librdkafka.so.1}
+    else
+        cp -v runtimes/linux-$ARCH/native/{centos6-librdkafka.so,librdkafka.so.1}
+    fi
     ldd runtimes/linux-$ARCH/native/librdkafka.so.1
 
 elif [[ $OSTYPE == darwin* ]]; then