Pythoncall: Support arrays of int, real as args

integration_tests/CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.5 FATAL_ERROR)
+cmake_minimum_required(VERSION 3.15 FATAL_ERROR)
 
 project(lpython_tests C)
 
@@ -37,14 +37,21 @@ set_property(TARGET lpython_rtlib PROPERTY INTERFACE_LINK_LIBRARIES
     ${LPYTHON_RTLIB_LIBRARY})
 target_link_libraries(lpython_rtlib INTERFACE m)
 
-find_package(Python COMPONENTS Development)
+find_package(Python COMPONENTS Interpreter Development)
+execute_process(
+  COMMAND "${Python_EXECUTABLE}"
+  -c "import numpy; print(numpy.get_include())"
+  OUTPUT_VARIABLE NUMPY_INCLUDE_DIR
+  OUTPUT_STRIP_TRAILING_WHITESPACE
+)
 message("\n")
 message("System has the Python development artifacts: ${Python_Development_FOUND}")
 message("The Python include directories: ${Python_INCLUDE_DIRS}")
 message("The Python libraries: ${Python_LIBRARIES}")
 message("The Python library directories: ${Python_LIBRARY_DIRS}")
 message("The Python runtime library directories: ${Python_RUNTIME_LIBRARY_DIRS}")
 message("Python version: ${Python_VERSION}")
+message("Numpy Include Directory: ${NUMPY_INCLUDE_DIR}")
 
 enable_testing()
 
@@ -125,6 +132,7 @@ macro(RUN_UTIL RUN_FAIL RUN_NAME RUN_FILE_NAME RUN_IMPORT_PATH RUN_LABELS RUN_EN
             set_target_properties(${name} PROPERTIES LINKER_LANGUAGE C)
             target_link_libraries(${name} lpython_rtlib)
             if (run_enable_cpython)
+                target_include_directories(${name} PRIVATE ${NUMPY_INCLUDE_DIR})
                 target_link_libraries(${name} Python::Python)
                 if (extra_files)
                     file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/${extra_files} DESTINATION ${CMAKE_CURRENT_BINARY_DIR})
@@ -235,7 +243,7 @@ macro(RUN_UTIL RUN_FAIL RUN_NAME RUN_FILE_NAME RUN_IMPORT_PATH RUN_LABELS RUN_EN
 endmacro(RUN_UTIL)
 
 macro(RUN)
-    set(options FAIL NOFAST ENABLE_CPYTHON)
+    set(options FAIL NOFAST ENABLE_CPYTHON ENABLE_CNUMPY)
     set(oneValueArgs NAME IMPORT_PATH)
     set(multiValueArgs LABELS EXTRAFILES)
     cmake_parse_arguments(RUN "${options}" "${oneValueArgs}"
@@ -248,6 +256,10 @@ macro(RUN)
         set(RUN_EXTRA_ARGS ${RUN_EXTRA_ARGS} --enable-cpython)
     endif()
 
+    if (RUN_ENABLE_CNUMPY)
+        set(RUN_EXTRA_ARGS ${RUN_EXTRA_ARGS} --enable-cnumpy)
+    endif()
+
     RUN_UTIL(RUN_FAIL RUN_NAME RUN_FILE_NAME RUN_IMPORT_PATH RUN_LABELS RUN_ENABLE_CPYTHON RUN_EXTRAFILES RUN_EXTRA_ARGS)
 
     if ((NOT DISABLE_FAST) AND (NOT RUN_NOFAST))
@@ -504,6 +516,7 @@ RUN(NAME bindc_05   LABELS llvm c
 RUN(NAME bindc_06   LABELS llvm c
         EXTRAFILES bindc_06b.c)
 RUN(NAME bindpy_01           LABELS cpython c ENABLE_CPYTHON NOFAST EXTRAFILES bindpy_01_module.py)
+RUN(NAME bindpy_02           LABELS cpython c ENABLE_CPYTHON ENABLE_CNUMPY EXTRAFILES bindpy_02_module.py)
 RUN(NAME test_generics_01    LABELS cpython llvm c NOFAST)
 RUN(NAME test_cmath          LABELS cpython llvm c NOFAST)
 RUN(NAME test_complex_01        LABELS cpython llvm c wasm wasm_x64)

integration_tests/bindpy_02.py

@@ -0,0 +1,80 @@
+from lpython import i32, f64, pythoncall, Const
+from numpy import empty
+
+@pythoncall(module = "bindpy_02_module")
+def get_cpython_version() -> str:
+    pass
+
+@pythoncall(module = "bindpy_02_module")
+def get_int_array_sum(a: i32[:]) -> i32:
+    pass
+
+@pythoncall(module = "bindpy_02_module")
+def get_int_array_product(a: i32[:]) -> i32:
+    pass
+
+@pythoncall(module = "bindpy_02_module")
+def get_float_array_sum(a: f64[:]) -> f64:
+    pass
+
+@pythoncall(module = "bindpy_02_module")
+def get_float_array_product(a: f64[:]) -> f64:
+    pass
+
+@pythoncall(module = "bindpy_02_module")
+def show_array_dot_product(a: i32[:], b: f64[:]):
+    pass
+
+# Integers:
+def test_array_ints():
+    n: Const[i32] = 5
+    a: i32[n] = empty([n], dtype=int)
+
+    i: i32
+    for i in range(n):
+        a[i] = i + 10
+
+    assert get_int_array_sum(a) == i32(60)
+    assert get_int_array_product(a) == i32(240240)
+
+# Floats
+def test_array_floats():
+    n: Const[i32] = 3
+    m: Const[i32] = 5
+    b: f64[n, m] = empty([n, m], dtype=float)
+
+    i: i32
+    j: i32
+
+    for i in range(n):
+        for j in range(m):
+            b[i, j] = f64((i + 1) * (j + 1))
+
+    assert abs(get_float_array_sum(b) - (90.000000)) <= 1e-4
+    assert abs(get_float_array_product(b) - (13436928000.000000)) <= 1e-4
+
+def test_array_broadcast():
+    n: Const[i32] = 3
+    m: Const[i32] = 5
+    a: i32[n] = empty([n], dtype=int)
+    b: f64[n, m] = empty([n, m], dtype=float)
+
+    i: i32
+    j: i32
+    for i in range(n):
+        a[i] = i + 10
+
+    for i in range(n):
+        for j in range(m):
+            b[i, j] = f64((i + 1) * (j + 1))
+
+    show_array_dot_product(a, b)
+
+def main0():
+    print("CPython version: ", get_cpython_version())
+
+    test_array_ints()
+    test_array_floats()
+    test_array_broadcast()
+
+main0()

integration_tests/bindpy_02_module.py

@@ -0,0 +1,21 @@
+import numpy as np
+
+def get_cpython_version():
+    import platform
+    return platform.python_version()
+
+def get_int_array_sum(a):
+    return np.sum(a)
+
+def get_int_array_product(a):
+    return np.prod(a)
+
+def get_float_array_sum(a):
+    return np.sum(a)
+
+def get_float_array_product(a):
+    return np.prod(a)
+
+def show_array_dot_product(a, b):
+    print(a, b)
+    print(a @ b)

src/bin/lpython.cpp

@@ -1249,6 +1249,9 @@ int link_executable(const std::vector<std::string> &infiles,
         if (compiler_options.enable_cpython) {
             std::string py_version = "3.10";
             std::string py_flags = R"(-I $CONDA_PREFIX/include/python)" + py_version + R"( -L$CONDA_PREFIX/lib -Wl,-rpath -Wl,$CONDA_PREFIX/lib -lpython)" + py_version + R"()";
+            if (compiler_options.enable_cnumpy) {
+                py_flags += R"( -I$CONDA_PREFIX/lib/python)" + py_version + R"(/site-packages/numpy/core/include)";
+            }
             cmd += " " + py_flags;
         }
         int err = system(cmd.c_str());
@@ -1562,6 +1565,7 @@ int main(int argc, char *argv[])
         app.add_flag("--get-rtl-dir", print_rtl_dir, "Print the path to the runtime library file");
         app.add_flag("--verbose", compiler_options.verbose, "Print debugging statements");
         app.add_flag("--enable-cpython", compiler_options.enable_cpython, "Enable CPython runtime");
+        app.add_flag("--enable-cnumpy", compiler_options.enable_cnumpy, "Enable C-Numpy runtime");
 
         // LSP specific options
         app.add_flag("--show-errors", show_errors, "Show errors when LSP is running in the background");

src/libasr/asr_utils.h

@@ -3363,8 +3363,9 @@ static inline bool is_pass_array_by_data_possible(ASR::Function_t* x, std::vecto
     // BindC interfaces already pass array by data pointer so we don't need to track
     // them and use extra variables for their dimensional information. Only those functions
     // need to be tracked which by default pass arrays by using descriptors.
-    if (ASRUtils::get_FunctionType(x)->m_abi == ASR::abiType::BindC &&
-        ASRUtils::get_FunctionType(x)->m_deftype == ASR::deftypeType::Interface) {
+    if ((ASRUtils::get_FunctionType(x)->m_abi == ASR::abiType::BindC
+         || ASRUtils::get_FunctionType(x)->m_abi == ASR::abiType::BindPython)
+        && ASRUtils::get_FunctionType(x)->m_deftype == ASR::deftypeType::Interface) {
         return false;
     }
 

src/libasr/codegen/asr_to_c.cpp

@@ -670,7 +670,8 @@ class ASRToCVisitor : public BaseCCPPVisitor<ASRToCVisitor>
         c_utils_functions->set_indentation(indentation_level, indentation_spaces);
         c_utils_functions->set_global_scope(global_scope);
         c_ds_api->set_c_utils_functions(c_utils_functions.get());
-
+        bind_py_utils_functions->set_indentation(indentation_level, indentation_spaces);
+        bind_py_utils_functions->set_global_scope(global_scope);
         std::string head =
 R"(
 #include <stdlib.h>
@@ -783,6 +784,9 @@ R"(
             }
         }
         std::string to_include = "";
+        for (auto &s: user_defines) {
+            to_include += "#define " + s + "\n";
+        }
         for (auto &s: headers) {
             to_include += "#include <" + s + ">\n";
         }
@@ -803,6 +807,12 @@ R"(
         if( c_utils_functions->get_generated_code().size() > 0 ) {
             util_funcs_defined =  "\n" + c_utils_functions->get_generated_code() + "\n";
         }
+        if( bind_py_utils_functions->get_util_func_decls().size() > 0 ) {
+            array_types_decls += "\n" + bind_py_utils_functions->get_util_func_decls() + "\n";
+        }
+        if( bind_py_utils_functions->get_generated_code().size() > 0 ) {
+            util_funcs_defined =  "\n" + bind_py_utils_functions->get_generated_code() + "\n";
+        }
         if( is_string_concat_present ) {
             head += strcat_def;
         }
@@ -914,6 +924,7 @@ R"(
 
         std::string body;
         if (compiler_options.enable_cpython) {
+            headers.insert("Python.h");
             body += R"(
     Py_Initialize();
     wchar_t* argv1 = Py_DecodeLocale("", NULL);
@@ -923,6 +934,21 @@ R"(
             body += "\n";
         }
 
+        if (compiler_options.enable_cnumpy) {
+            user_defines.insert("NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION");
+            headers.insert("numpy/arrayobject.h");
+            body +=
+R"(    // Initialise Numpy
+    if (_import_array() < 0) {
+        PyErr_Print();
+        PyErr_SetString(PyExc_ImportError, "numpy.core.multiarray failed to import");
+        fprintf(stderr, "Failed to import numpy Python module(s)\n");
+        return -1;
+    }
+)";
+            body += "\n";
+        }
+
         for (size_t i=0; i<x.n_body; i++) {
             this->visit_stmt(*x.m_body[i]);
             body += src;

src/libasr/codegen/asr_to_c_cpp.h

@@ -112,7 +112,7 @@ class BaseCCPPVisitor : public ASR::BaseVisitor<Struct>
     // Use std::complex<float/double> or float/double complex
     bool gen_stdcomplex;
     bool is_c;
-    std::set<std::string> headers, user_headers;
+    std::set<std::string> headers, user_headers, user_defines;
     std::vector<std::string> tmp_buffer_src;
 
     SymbolTable* global_scope;
@@ -123,6 +123,7 @@ class BaseCCPPVisitor : public ASR::BaseVisitor<Struct>
     std::string from_std_vector_helper;
 
     std::unique_ptr<CCPPDSUtils> c_ds_api;
+    std::unique_ptr<CUtils::BindPyUtilFunctions> bind_py_utils_functions;
     std::string const_name;
     size_t const_vars_count;
     size_t loop_end_count;
@@ -149,6 +150,7 @@ class BaseCCPPVisitor : public ASR::BaseVisitor<Struct>
         gen_stdstring{gen_stdstring}, gen_stdcomplex{gen_stdcomplex},
         is_c{is_c}, global_scope{nullptr}, lower_bound{default_lower_bound},
         template_number{0}, c_ds_api{std::make_unique<CCPPDSUtils>(is_c, platform)},
+        bind_py_utils_functions{std::make_unique<CUtils::BindPyUtilFunctions>()},
         const_name{"constname"},
         const_vars_count{0}, loop_end_count{0}, bracket_open{0},
         is_string_concat_present{false} {
@@ -504,15 +506,23 @@ R"(#include <stdio.h>
     }
 
     std::string get_arg_conv_bind_python(const ASR::Function_t &x) {
-
         std::string arg_conv = R"(
     pArgs = PyTuple_New()" + std::to_string(x.n_args) + R"();
 )";
         for (size_t i = 0; i < x.n_args; ++i) {
             ASR::Variable_t *arg = ASRUtils::EXPR2VAR(x.m_args[i]);
+            std::string arg_name = std::string(arg->m_name);
+            std::string indent = "\n    ";
+            if (ASRUtils::is_array(arg->m_type)) {
+                arg_conv += indent + bind_py_utils_functions->get_conv_dims_to_1D_arr() + "(" + arg_name + "->n_dims, " + arg_name + "->dims, __new_dims);";
+                std::string func_call = CUtils::get_py_obj_type_conv_func_from_ttype_t(arg->m_type);
+                arg_conv += indent + "pValue = " + func_call + "(" + arg_name + "->n_dims, __new_dims, "
+                    + CUtils::get_numpy_c_obj_type_conv_func_from_ttype_t(arg->m_type) + ", " + arg_name + "->data);";
+            } else {
+                arg_conv += indent + "pValue = " + CUtils::get_py_obj_type_conv_func_from_ttype_t(arg->m_type)
+                    + "(" + arg_name + ");";
+            }
             arg_conv += R"(
-    pValue = )" + CUtils::get_py_obj_type_conv_func_from_ttype_t(arg->m_type) + "("
-    + std::string(arg->m_name) + R"();
     if (!pValue) {
         Py_DECREF(pArgs);
         Py_DECREF(pModule);
@@ -534,15 +544,17 @@ R"(#include <stdio.h>
         std::string ret_var_decl = indent + CUtils::get_c_type_from_ttype_t(r_v->m_type) + " " + std::string(r_v->m_name) + ";";
         std::string ret_assign = indent + std::string(r_v->m_name) + " = " + py_val_cnvrt + ";";
         std::string ret_stmt = indent + "return " + std::string(r_v->m_name) + ";";
-        std::string clear_pValue = "";
-        if (!ASRUtils::is_aggregate_type(r_v->m_type)) {
-            clear_pValue = indent + "Py_DECREF(pValue);";
+        std::string clear_pValue = indent + "Py_DECREF(pValue);";
+        std::string copy_result = "";
+        if (ASRUtils::is_aggregate_type(r_v->m_type)) {
+            if (ASRUtils::is_character(*r_v->m_type)) {
+                copy_result = indent + std::string(r_v->m_name) + " = _lfortran_str_copy(" + std::string(r_v->m_name) + ", 1, 0);";
+            }
         }
-        return ret_var_decl + ret_assign + clear_pValue + ret_stmt + "\n";
+        return ret_var_decl + ret_assign + copy_result + clear_pValue + ret_stmt + "\n";
     }
 
     std::string get_func_body_bind_python(const ASR::Function_t &x) {
-        user_headers.insert("Python.h");
         std::string indent(indentation_level*indentation_spaces, ' ');
         std::string var_decls = "PyObject *pName, *pModule, *pFunc; PyObject *pArgs, *pValue;\n";
         std::string func_body = R"(