From b1567d8b71e3d80ca97779e5337cb26355a4217e Mon Sep 17 00:00:00 2001 From: Vipul Cariappa Date: Sat, 6 Jul 2024 16:08:03 +0530 Subject: [PATCH 01/14] Implement Jupyter kernel --- .gitignore | 1 + CMakeLists.txt | 7 +- doc/src/developers_example.ipynb | 67 +++ examples/example_notebook.ipynb | 134 +++++ share/jupyter/kernels/lpython/kernel.json.in | 10 + src/bin/lpython.cpp | 9 +- src/lpython/CMakeLists.txt | 4 +- src/lpython/python_evaluator.cpp | 132 ++++- src/lpython/python_evaluator.h | 28 +- src/lpython/python_kernel.cpp | 540 +++++++++++++++++++ src/lpython/python_kernel.h | 15 + 11 files changed, 937 insertions(+), 10 deletions(-) create mode 100644 doc/src/developers_example.ipynb create mode 100644 examples/example_notebook.ipynb create mode 100644 share/jupyter/kernels/lpython/kernel.json.in create mode 100644 src/lpython/python_kernel.cpp create mode 100644 src/lpython/python_kernel.h diff --git a/.gitignore b/.gitignore index 9cdcc189be..d18b9b6284 100644 --- a/.gitignore +++ b/.gitignore @@ -61,6 +61,7 @@ src/libasr/wasm_visitor.h src/libasr/pass/intrinsic_function_registry_util.h src/libasr/config.h share/jupyter/kernels/fortran/kernel.json +share/jupyter/kernels/lpython/kernel.json src/runtime/*.o.empty.c python_ast.py python_ast.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 4e1e2ea0b9..8919bbee1f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -208,13 +208,14 @@ endif() # XEUS (Fortran kernel) set(WITH_XEUS no CACHE BOOL "Build with XEUS support") if (WITH_XEUS) - find_package(xeus 0.24.1 REQUIRED) + find_package(xeus 5.1.0 REQUIRED) + find_package(xeus-zmq 3.0.0 REQUIRED) set(HAVE_LFORTRAN_XEUS yes) # Generate kernel.json with correct paths configure_file ( - "${CMAKE_CURRENT_SOURCE_DIR}/share/jupyter/kernels/fortran/kernel.json.in" - "${CMAKE_CURRENT_BINARY_DIR}/share/jupyter/kernels/fortran/kernel.json" + "${CMAKE_CURRENT_SOURCE_DIR}/share/jupyter/kernels/lpython/kernel.json.in" + "${CMAKE_CURRENT_BINARY_DIR}/share/jupyter/kernels/lpython/kernel.json" ) # Configuration and data directories for Jupyter and LFortran diff --git a/doc/src/developers_example.ipynb b/doc/src/developers_example.ipynb new file mode 100644 index 0000000000..c63754c8f0 --- /dev/null +++ b/doc/src/developers_example.ipynb @@ -0,0 +1,67 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "c86338ac-53ca-4115-8c5a-8bf8a5c7113e", + "metadata": {}, + "outputs": [], + "source": [ + "%%showast\n", + "def add(x: i32, y: i32) -> i32:\n", + " return x + y" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "23834b08-2f3f-45e7-a1ce-21a9fd4e5117", + "metadata": {}, + "outputs": [], + "source": [ + "%%showasr\n", + "def add(x: i32, y: i32) -> i32:\n", + " return x + y" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ec7426b4-e2e5-416c-bcae-9bb9c8926c9b", + "metadata": {}, + "outputs": [], + "source": [ + "%%showllvm\n", + "def sub(x: i32, y: i32) -> i32:\n", + " return add(x, -y)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "716c56ef-8210-4daf-aa23-96b385801014", + "metadata": {}, + "outputs": [], + "source": [ + "%%showasm\n", + "def mul(x: i32, y: i32) -> i32:\n", + " return x * y" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "LPython", + "language": "python", + "name": "lpython" + }, + "language_info": { + "file_extension": ".f90", + "mimetype": "text/x-python", + "name": "python", + "version": "2018" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/example_notebook.ipynb b/examples/example_notebook.ipynb new file mode 100644 index 0000000000..17ce1ba2db --- /dev/null +++ b/examples/example_notebook.ipynb @@ -0,0 +1,134 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "e87300c2-64ed-4636-8448-591f36faba29", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Hello, LPython\n" + ] + } + ], + "source": [ + "print(\"Hello, LPython\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "dfcac851-7b49-4065-8c64-4a31658249f7", + "metadata": {}, + "outputs": [], + "source": [ + "def add(x: i32, y: i32) -> i32:\n", + " return x + y" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "09213386-84d5-4e7c-83ba-c3b027f765dd", + "metadata": {}, + "outputs": [], + "source": [ + "def sub(x: i32, y: i32) -> i32:\n", + " return x - y" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "a4b49fd3-bf17-4287-9d5e-60f14ebc9a0f", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "5" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "add(2, 3)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "d6f4961f-7f0c-45a6-9bf8-e549e97098b0", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "-1" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "sub(2, 3)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "398fd4be-d7cc-4912-8aa1-880aa58b37ab", + "metadata": {}, + "outputs": [], + "source": [ + "@dataclass\n", + "class MyClass:\n", + " x: i32\n", + " y: f64\n", + " z: str" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "628f0b7d-09a6-49de-a0e6-2f6c664f2ba2", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "12 2.45000000000000000e+01 LPython\n" + ] + } + ], + "source": [ + "x: MyClass = MyClass(12, 24.5, \"LPython\")\n", + "print(x)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "LPython", + "language": "python", + "name": "lpython" + }, + "language_info": { + "file_extension": ".f90", + "mimetype": "text/x-python", + "name": "python", + "version": "2018" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/share/jupyter/kernels/lpython/kernel.json.in b/share/jupyter/kernels/lpython/kernel.json.in new file mode 100644 index 0000000000..e1af020ba4 --- /dev/null +++ b/share/jupyter/kernels/lpython/kernel.json.in @@ -0,0 +1,10 @@ +{ + "display_name": "LPython", + "argv": [ + "@CMAKE_INSTALL_PREFIX@/@CMAKE_INSTALL_BINDIR@/lpython", + "kernel", + "-f", + "{connection_file}" + ], + "language": "python" +} diff --git a/src/bin/lpython.cpp b/src/bin/lpython.cpp index 43649c57b5..25f7ca8119 100644 --- a/src/bin/lpython.cpp +++ b/src/bin/lpython.cpp @@ -29,6 +29,7 @@ #include #include #include +#include #include #include #include @@ -2012,8 +2013,12 @@ int main(int argc, char *argv[]) // } if (kernel) { - std::cerr << "The kernel subcommand is not implemented yet for LPython." << std::endl; - return 1; +#ifdef HAVE_LFORTRAN_XEUS + return LCompilers::LPython::run_kernel(arg_kernel_f); +#else + std::cerr << "The kernel subcommand requires LFortran to be compiled with XEUS support. Recompile with `WITH_XEUS=yes`." << std::endl; + return 1; +#endif } // if (mod) { diff --git a/src/lpython/CMakeLists.txt b/src/lpython/CMakeLists.txt index b8642d43d0..545fb592e9 100644 --- a/src/lpython/CMakeLists.txt +++ b/src/lpython/CMakeLists.txt @@ -22,7 +22,7 @@ endif() if (WITH_XEUS) set(SRC ${SRC} -# fortran_kernel.cpp + python_kernel.cpp ) endif() add_library(lpython_lib ${SRC}) @@ -35,7 +35,7 @@ endif() target_include_directories(lpython_lib BEFORE PUBLIC ${lpython_SOURCE_DIR}/src) target_include_directories(lpython_lib BEFORE PUBLIC ${lpython_BINARY_DIR}/src) if (WITH_XEUS) - target_link_libraries(lpython_lib xeus) + target_link_libraries(lpython_lib xeus xeus-zmq) endif() if (WITH_BFD) target_link_libraries(lpython_lib p::bfd) diff --git a/src/lpython/python_evaluator.cpp b/src/lpython/python_evaluator.cpp index 836ddaad22..0cbc022261 100644 --- a/src/lpython/python_evaluator.cpp +++ b/src/lpython/python_evaluator.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #ifdef HAVE_LFORTRAN_LLVM #include @@ -29,12 +30,12 @@ namespace LCompilers { PythonCompiler::PythonCompiler(CompilerOptions compiler_options) : + compiler_options{compiler_options}, al{1024*1024}, #ifdef HAVE_LFORTRAN_LLVM e{std::make_unique()}, #endif eval_count{1}, - compiler_options{compiler_options}, symbol_table{nullptr} { } @@ -234,6 +235,66 @@ Result PythonCompiler::evaluate( #endif } +Result PythonCompiler::get_ast(const std::string &code, + LocationManager &lm, diag::Diagnostics &diagnostics) +{ + Result ast = get_ast2(code, diagnostics); + if (ast.ok) { + if (compiler_options.po.tree) { + return LCompilers::LPython::pickle_tree_python(*ast.result, compiler_options.use_colors); + } else if (compiler_options.po.json || compiler_options.po.visualize) { + return LCompilers::LPython::pickle_json(*ast.result, lm); + } + return LCompilers::LPython::pickle_python(*ast.result, compiler_options.use_colors, + compiler_options.indent); + } else { + LCOMPILERS_ASSERT(diagnostics.has_error()) + return ast.error; + } +} + +Result PythonCompiler::get_asr(const std::string &code, + LocationManager &lm, diag::Diagnostics &diagnostics) +{ + Result asr = get_asr2(code, lm, diagnostics); + if (asr.ok) { + if (compiler_options.po.tree) { + return LCompilers::pickle_tree(*asr.result, compiler_options.use_colors); + } else if (compiler_options.po.json) { + return LCompilers::pickle_json(*asr.result, lm, compiler_options.po.no_loc, false); + } + return LCompilers::pickle(*asr.result, + compiler_options.use_colors, compiler_options.indent); + } else { + LCOMPILERS_ASSERT(diagnostics.has_error()) + return asr.error; + } +} + +Result PythonCompiler::get_asr2( + const std::string &code_orig, LocationManager &lm, + diag::Diagnostics &diagnostics) +{ + // Src -> AST + Result res = get_ast2(code_orig, diagnostics); + LCompilers::LPython::AST::ast_t* ast; + if (res.ok) { + ast = res.result; + } else { + LCOMPILERS_ASSERT(diagnostics.has_error()) + return res.error; + } + + // AST -> ASR + Result res2 = get_asr3(*ast, diagnostics, lm, true); + if (res2.ok) { + return res2.result; + } else { + LCOMPILERS_ASSERT(diagnostics.has_error()) + return res2.error; + } +} + Result PythonCompiler::get_ast2( const std::string &code_orig, diag::Diagnostics &diagnostics) { @@ -272,6 +333,48 @@ Result PythonCompiler::get_asr3( return asr; } +Result PythonCompiler::get_llvm( + const std::string &code, LocationManager &lm, LCompilers::PassManager& pass_manager, + diag::Diagnostics &diagnostics + ) +{ + Result> res = get_llvm2(code, lm, pass_manager, diagnostics); + if (res.ok) { +#ifdef HAVE_LFORTRAN_LLVM + return res.result->str(); +#else + throw LCompilersException("LLVM is not enabled"); +#endif + } else { + LCOMPILERS_ASSERT(diagnostics.has_error()) + return res.error; + } +} + +Result> PythonCompiler::get_llvm2( + const std::string &code, LocationManager &lm, LCompilers::PassManager& pass_manager, + diag::Diagnostics &diagnostics) +{ + Result asr = get_asr2(code, lm, diagnostics); + if (!asr.ok) { + return asr.error; + } + Result> res = get_llvm3(*asr.result, pass_manager, + diagnostics, lm.files.back().in_filename); + if (res.ok) { +#ifdef HAVE_LFORTRAN_LLVM + std::unique_ptr m = std::move(res.result); + return m; +#else + throw LCompilersException("LLVM is not enabled"); +#endif + } else { + LCOMPILERS_ASSERT(diagnostics.has_error()) + return res.error; + } +} + + Result> PythonCompiler::get_llvm3( #ifdef HAVE_LFORTRAN_LLVM ASR::TranslationUnit_t &asr, LCompilers::PassManager& lpm, @@ -319,4 +422,31 @@ Result> PythonCompiler::get_llvm3( #endif } +Result PythonCompiler::get_asm( +#ifdef HAVE_LFORTRAN_LLVM + const std::string &code, LocationManager &lm, + LCompilers::PassManager& lpm, + diag::Diagnostics &diagnostics +#else + const std::string &/*code*/, + LocationManager &/*lm*/, + LCompilers::PassManager&/*lpm*/, + diag::Diagnostics &/*diagnostics*/ +#endif + ) +{ +#ifdef HAVE_LFORTRAN_LLVM + Result> res = get_llvm2(code, lm, lpm, diagnostics); + if (res.ok) { + return e->get_asm(*res.result->m_m); + } else { + LCOMPILERS_ASSERT(diagnostics.has_error()) + return res.error; + } +#else + throw LCompilersException("LLVM is not enabled"); +#endif +} + + } // namespace LCompilers::LPython diff --git a/src/lpython/python_evaluator.h b/src/lpython/python_evaluator.h index f5c4d538ec..50958fb840 100644 --- a/src/lpython/python_evaluator.h +++ b/src/lpython/python_evaluator.h @@ -32,6 +32,8 @@ class LLVMEvaluator; class PythonCompiler { public: + CompilerOptions compiler_options; + PythonCompiler(CompilerOptions compiler_options); ~PythonCompiler(); @@ -77,24 +79,46 @@ class PythonCompiler Result evaluate2(const std::string &code); + Result get_ast(const std::string &code, + LocationManager &lm, diag::Diagnostics &diagnostics); + Result get_ast2( const std::string &code_orig, diag::Diagnostics &diagnostics); - + + Result get_asr(const std::string &code, + LocationManager &lm, diag::Diagnostics &diagnostics); + + Result get_asr2( + const std::string &code_orig, LocationManager &lm, + diag::Diagnostics &diagnostics); + Result get_asr3( LCompilers::LPython::AST::ast_t &ast, diag::Diagnostics &diagnostics, LocationManager &lm, bool is_interactive=false); + Result get_llvm( + const std::string &code, LocationManager &lm, LCompilers::PassManager& pass_manager, + diag::Diagnostics &diagnostics); + + Result> get_llvm2( + const std::string &code, LocationManager &lm, LCompilers::PassManager& pass_manager, + diag::Diagnostics &diagnostics); + Result> get_llvm3(ASR::TranslationUnit_t &asr, LCompilers::PassManager& lpm, diag::Diagnostics &diagnostics, const std::string &infile); + Result get_asm(const std::string &code, + LocationManager &lm, + LCompilers::PassManager& pass_manager, + diag::Diagnostics &diagnostics); + private: Allocator al; #ifdef HAVE_LFORTRAN_LLVM std::unique_ptr e; #endif int eval_count; - CompilerOptions compiler_options; SymbolTable *symbol_table; std::string run_fn; }; diff --git a/src/lpython/python_kernel.cpp b/src/lpython/python_kernel.cpp new file mode 100644 index 0000000000..171b043af3 --- /dev/null +++ b/src/lpython/python_kernel.cpp @@ -0,0 +1,540 @@ +#include + +#include +#include + +#ifdef _WIN32 +# include +# define fileno _fileno +# define dup _dup +# define dup2 _dup2 +# define close _close +# include +#else +# include +#endif + +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace nl = nlohmann; + +namespace LCompilers::LPython { + + + class RedirectStdout + { + public: + RedirectStdout(std::string &out) : _out{out} { + stdout_fileno = fileno(stdout); + std::cout << std::flush; + fflush(stdout); + saved_stdout = dup(stdout_fileno); +#ifdef _WIN32 + if (_pipe(out_pipe, 65536, O_BINARY) != 0) { +#else + if (pipe(out_pipe) != 0) { +#endif + throw LCompilersException("pipe() failed"); + } + dup2(out_pipe[1], stdout_fileno); + close(out_pipe[1]); + printf("X"); + } + + ~RedirectStdout() { + fflush(stdout); + read(out_pipe[0], buffer, MAX_LEN); + dup2(saved_stdout, stdout_fileno); + _out = std::string(&buffer[1]); + } + private: + std::string &_out; + static const size_t MAX_LEN=1024; + char buffer[MAX_LEN+1] = {0}; + int out_pipe[2]; + int saved_stdout; + int stdout_fileno; + }; + + class custom_interpreter : public xeus::xinterpreter + { + private: + PythonCompiler e; + + public: + custom_interpreter() : e{CompilerOptions()} { + e.compiler_options.interactive = true; + e.compiler_options.po.disable_main = true; + e.compiler_options.emit_debug_line_column = false; + e.compiler_options.generate_object_code = false; + } + virtual ~custom_interpreter() = default; + + private: + + void configure_impl() override; + + void execute_request_impl(send_reply_callback cb, + int execution_counter, + const std::string& code, + //bool silent, + //bool store_history, + xeus::execute_request_config config, + nl::json user_expressions) override; + + nl::json complete_request_impl(const std::string& code, + int cursor_pos) override; + + nl::json inspect_request_impl(const std::string& code, + int cursor_pos, + int detail_level) override; + + nl::json is_complete_request_impl(const std::string& code) override; + + nl::json kernel_info_request_impl() override; + + void shutdown_request_impl() override; + }; + + + void custom_interpreter::execute_request_impl(send_reply_callback cb, + int execution_counter, // Typically the cell number + const std::string& code, // Code to execute + xeus::execute_request_config, //config + nl::json /*user_expressions*/) + { + PythonCompiler::EvalResult r; + std::string std_out; + std::string code0; + CompilerOptions cu; + try { + if (startswith(code, "%%showast")) { + code0 = code.substr(code.find("\n")+1); + LocationManager lm; + { + LocationManager::FileLocations fl; + fl.in_filename = "input"; + std::ofstream out("input"); + out << code0; + lm.files.push_back(fl); + } + diag::Diagnostics diagnostics; + Result + res = e.get_ast(code0, lm, diagnostics); + nl::json result; + if (res.ok) { + publish_stream("stdout", res.result); + result["status"] = "ok"; + result["payload"] = nl::json::array(); + result["user_expressions"] = nl::json::object(); + } else { + std::string msg = diagnostics.render(lm, cu); + publish_stream("stderr", msg); + result["status"] = "error"; + result["ename"] = "CompilerError"; + result["evalue"] = msg; + result["traceback"] = nl::json::array(); + } + cb(result); + return; + } + if (startswith(code, "%%showasr")) { + code0 = code.substr(code.find("\n")+1); + LocationManager lm; + { + LocationManager::FileLocations fl; + fl.in_filename = "input"; + std::ofstream out("input"); + out << code0; + lm.files.push_back(fl); + } + diag::Diagnostics diagnostics; + Result + res = e.get_asr(code0, lm, diagnostics); + nl::json result; + if (res.ok) { + publish_stream("stdout", res.result); + result["status"] = "ok"; + result["payload"] = nl::json::array(); + result["user_expressions"] = nl::json::object(); + } else { + std::string msg = diagnostics.render(lm, cu); + publish_stream("stderr", msg); + result["status"] = "error"; + result["ename"] = "CompilerError"; + result["evalue"] = msg; + result["traceback"] = nl::json::array(); + } + cb(result); + return; + } + if (startswith(code, "%%showllvm")) { + code0 = code.substr(code.find("\n")+1); + LocationManager lm; + { + LCompilers::LocationManager::FileLocations fl; + fl.in_filename = "input"; + std::ofstream out("input"); + out << code0; + lm.files.push_back(fl); + lm.init_simple(code0); + lm.file_ends.push_back(code0.size()); + } + LCompilers::PassManager lpm; + lpm.use_default_passes(); + diag::Diagnostics diagnostics; + Result + res = e.get_llvm(code0, lm, lpm, diagnostics); + nl::json result; + if (res.ok) { + publish_stream("stdout", res.result); + result["status"] = "ok"; + result["payload"] = nl::json::array(); + result["user_expressions"] = nl::json::object(); + } else { + std::string msg = diagnostics.render(lm, cu); + publish_stream("stderr", msg); + result["status"] = "error"; + result["ename"] = "CompilerError"; + result["evalue"] = msg; + result["traceback"] = nl::json::array(); + } + cb(result); + return; + } + if (startswith(code, "%%showasm")) { + code0 = code.substr(code.find("\n")+1); + LocationManager lm; + { + LCompilers::LocationManager::FileLocations fl; + fl.in_filename = "input"; + std::ofstream out("input"); + out << code0; + lm.files.push_back(fl); + lm.init_simple(code0); + lm.file_ends.push_back(code0.size()); + } + LCompilers::PassManager lpm; + lpm.use_default_passes(); + diag::Diagnostics diagnostics; + Result + res = e.get_asm(code0, lm, lpm, diagnostics); + nl::json result; + if (res.ok) { + publish_stream("stdout", res.result); + result["status"] = "ok"; + result["payload"] = nl::json::array(); + result["user_expressions"] = nl::json::object(); + } else { + std::string msg = diagnostics.render(lm, cu); + publish_stream("stderr", msg); + result["status"] = "error"; + result["ename"] = "CompilerError"; + result["evalue"] = msg; + result["traceback"] = nl::json::array(); + } + cb(result); + return; + } + // if (startswith(code, "%%showcpp")) { + // code0 = code.substr(code.find("\n")+1); + // LocationManager lm; + // { + // LocationManager::FileLocations fl; + // fl.in_filename = "input"; + // std::ofstream out("input"); + // out << code0; + // lm.files.push_back(fl); + // } + // diag::Diagnostics diagnostics; + // Result + // res = e.get_cpp(code0, lm, diagnostics, 1); + // nl::json result; + // if (res.ok) { + // publish_stream("stdout", res.result); + // result["status"] = "ok"; + // result["payload"] = nl::json::array(); + // result["user_expressions"] = nl::json::object(); + // } else { + // std::string msg = diagnostics.render(lm, cu); + // publish_stream("stderr", msg); + // result["status"] = "error"; + // result["ename"] = "CompilerError"; + // result["evalue"] = msg; + // result["traceback"] = nl::json::array(); + // } + // cb(result); + // return; + // } + // if (startswith(code, "%%showfmt")) { + // code0 = code.substr(code.find("\n")+1); + // LocationManager lm; + // { + // LocationManager::FileLocations fl; + // fl.in_filename = "input"; + // std::ofstream out("input"); + // out << code0; + // lm.files.push_back(fl); + // } + // diag::Diagnostics diagnostics; + // Result + // res = e.get_fmt(code0, lm, diagnostics); + // nl::json result; + // if (res.ok) { + // publish_stream("stdout", res.result); + // result["status"] = "ok"; + // result["payload"] = nl::json::array(); + // result["user_expressions"] = nl::json::object(); + // } else { + // std::string msg = diagnostics.render(lm, cu); + // publish_stream("stderr", msg); + // result["status"] = "error"; + // result["ename"] = "CompilerError"; + // result["evalue"] = msg; + // result["traceback"] = nl::json::array(); + // } + // cb(result); + // return; + // } + + RedirectStdout s(std_out); + code0 = code; + LocationManager lm; + { + LCompilers::LocationManager::FileLocations fl; + fl.in_filename = "input"; + std::ofstream out("input"); + out << code0; + lm.files.push_back(fl); + lm.init_simple(code0); + lm.file_ends.push_back(code0.size()); + } + LCompilers::PassManager lpm; + lpm.use_default_passes(); + diag::Diagnostics diagnostics; + Result + res = e.evaluate(code0, false, lm, lpm, diagnostics); + if (res.ok) { + r = res.result; + } else { + std::string msg = diagnostics.render(lm, cu); + publish_stream("stderr", msg); + nl::json result; + result["status"] = "error"; + result["ename"] = "CompilerError"; + result["evalue"] = msg; + result["traceback"] = nl::json::array(); + cb(result); + return; + } + } catch (const LCompilersException &e) { + publish_stream("stderr", "LFortran Exception: " + e.msg()); + nl::json result; + result["status"] = "error"; + result["ename"] = "LCompilersException"; + result["evalue"] = e.msg(); + result["traceback"] = nl::json::array(); + cb(result); + return; + } + + if (std_out.size() > 0) { + publish_stream("stdout", std_out); + } + + switch (r.type) { + case (LCompilers::PythonCompiler::EvalResult::integer4) : { + nl::json pub_data; + pub_data["text/plain"] = std::to_string(r.i32); + publish_execution_result(execution_counter, std::move(pub_data), nl::json::object()); + break; + } + case (LCompilers::PythonCompiler::EvalResult::integer8) : { + nl::json pub_data; + pub_data["text/plain"] = std::to_string(r.i64); + publish_execution_result(execution_counter, std::move(pub_data), nl::json::object()); + break; + } + case (LCompilers::PythonCompiler::EvalResult::real4) : { + nl::json pub_data; + pub_data["text/plain"] = std::to_string(r.f32); + publish_execution_result(execution_counter, std::move(pub_data), nl::json::object()); + break; + } + case (LCompilers::PythonCompiler::EvalResult::real8) : { + nl::json pub_data; + pub_data["text/plain"] = std::to_string(r.f64); + publish_execution_result(execution_counter, std::move(pub_data), nl::json::object()); + break; + } + case (LCompilers::PythonCompiler::EvalResult::complex4) : { + nl::json pub_data; + pub_data["text/plain"] = "(" + std::to_string(r.c32.re) + ", " + std::to_string(r.c32.im) + ")"; + publish_execution_result(execution_counter, std::move(pub_data), nl::json::object()); + break; + } + case (LCompilers::PythonCompiler::EvalResult::complex8) : { + nl::json pub_data; + pub_data["text/plain"] = "(" + std::to_string(r.c64.re) + ", " + std::to_string(r.c64.im) + ")"; + publish_execution_result(execution_counter, std::move(pub_data), nl::json::object()); + break; + } + case (LCompilers::PythonCompiler::EvalResult::statement) : { + break; + } + case (LCompilers::PythonCompiler::EvalResult::none) : { + break; + } + default : throw LCompilersException("Return type not supported"); + } + + nl::json result; + result["status"] = "ok"; + result["payload"] = nl::json::array(); + result["user_expressions"] = nl::json::object(); + cb(result); + return; + } + + void custom_interpreter::configure_impl() + { + // Perform some operations + } + + nl::json custom_interpreter::complete_request_impl(const std::string& code, + int cursor_pos) + { + nl::json result; + + // Code starts with 'H', it could be the following completion + if (code[0] == 'H') + { + result["status"] = "ok"; + result["matches"] = {"Hello", "Hey", "Howdy"}; + result["cursor_start"] = 5; + result["cursor_end"] = cursor_pos; + } + // No completion result + else + { + result["status"] = "ok"; + result["matches"] = nl::json::array(); + result["cursor_start"] = cursor_pos; + result["cursor_end"] = cursor_pos; + } + + return result; + } + + nl::json custom_interpreter::inspect_request_impl(const std::string& code, + int /*cursor_pos*/, + int /*detail_level*/) + { + nl::json result; + + if (code.compare("print") == 0) + { + result["found"] = true; + result["text/plain"] = "Print objects to the text stream file, [...]"; + } + else + { + result["found"] = false; + } + + result["status"] = "ok"; + return result; + } + + nl::json custom_interpreter::is_complete_request_impl(const std::string& /*code*/) + { + nl::json result; + + // if (is_complete(code)) + // { + result["status"] = "complete"; + // } + // else + // { + // result["status"] = "incomplete"; + // result["indent"] = 4; + //} + + return result; + } + + nl::json custom_interpreter::kernel_info_request_impl() + { + nl::json result; + std::string version = LFORTRAN_VERSION; + std::string banner = "" + "LFortran " + version + "\n" + "Jupyter kernel for Fortran"; + result["banner"] = banner; + result["implementation"] = "LFortran"; + result["implementation_version"] = version; + result["language_info"]["name"] = "python"; + result["language_info"]["version"] = "2018"; + result["language_info"]["mimetype"] = "text/x-python"; + result["language_info"]["file_extension"] = ".f90"; + return result; + } + + void custom_interpreter::shutdown_request_impl() { + std::cout << "Bye!!" << std::endl; + } + + int run_kernel(const std::string &connection_filename) + { + std::unique_ptr context = xeus::make_zmq_context(); + + // Create interpreter instance + using interpreter_ptr = std::unique_ptr; + interpreter_ptr interpreter = interpreter_ptr(new custom_interpreter()); + + using history_manager_ptr = std::unique_ptr; + history_manager_ptr hist = xeus::make_in_memory_history_manager(); + + nl::json debugger_config; + + // Load configuration file + xeus::xconfiguration config = xeus::load_configuration(connection_filename); + + // Create kernel instance and start it + xeus::xkernel kernel(config, + xeus::get_user_name(), + std::move(context), + std::move(interpreter), + xeus::make_xserver_shell_main, + std::move(hist), + xeus::make_console_logger(xeus::xlogger::msg_type, + xeus::make_file_logger(xeus::xlogger::content, "xeus.log")), + xeus::make_null_debugger, + debugger_config); + + std::cout << + "Starting xeus-lpython kernel...\n\n" + "If you want to connect to this kernel from an other client, you can use" + " the " + connection_filename + " file." + << std::endl; + + kernel.start(); + + return 0; + } + +} // namespace LCompilers::LFortran diff --git a/src/lpython/python_kernel.h b/src/lpython/python_kernel.h new file mode 100644 index 0000000000..bf0a5c5173 --- /dev/null +++ b/src/lpython/python_kernel.h @@ -0,0 +1,15 @@ +#ifndef LFORTRAN_PYTHON_KERNEL_H +#define LFORTRAN_PYTHON_KERNEL_H + +#include +#include + +namespace LCompilers::LPython { + +#ifdef HAVE_LFORTRAN_XEUS + int run_kernel(const std::string &connection_filename); +#endif + +} // namespace LCompilers::LFortran + +#endif // LFORTRAN_PYTHON_KERNEL_H From 8eda48b655e20baf24254fd6f09a5f9e87df3f46 Mon Sep 17 00:00:00 2001 From: Vipul Cariappa Date: Sat, 6 Jul 2024 16:15:50 +0530 Subject: [PATCH 02/14] Test Kernel; presently it only builds, we should add tests later --- .github/workflows/CI.yml | 45 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index ef534c5c6f..1a4fd7d785 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -497,6 +497,51 @@ jobs: cd integration_tests ./run_tests.py -b cpython c_py + build_jupyter_kernel: + name: Build Jupyter Kernel + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - uses: mamba-org/setup-micromamba@v1 + with: + environment-file: ci/environment.yml + create-args: >- + jupyter + nlohmann_json + python=3.10 + bison=3.4 + xeus=5.1.0 + xeus-zmq=3.0.0 + + - uses: hendrikmuhs/ccache-action@main + with: + variant: sccache + key: ${{ github.job }}-${{ matrix.os }} + + - name: Build LPython with Kernel + shell: bash -e -l {0} + run: | + ./build0.sh + export CXXFLAGS="-Werror" + cmake . -GNinja \ + -DCMAKE_BUILD_TYPE=Debug \ + -DWITH_LLVM=yes \ + -DWITH_XEUS=yes \ + -DCMAKE_PREFIX_PATH="$CONDA_PREFIX" \ + -DCMAKE_INSTALL_PREFIX="$CONDA_PREFIX" + + ninja install + ctest --output-on-failure + jupyter kernelspec list --json + + - name: Test Kernel + shell: bash -e -l {0} + run: | + ctest --output-on-failure + upload_tarball: name: Upload Tarball runs-on: ubuntu-latest From 627ba51d15e445e83d402f94874e88fe3122776f Mon Sep 17 00:00:00 2001 From: Vipul Cariappa Date: Sat, 6 Jul 2024 16:40:20 +0530 Subject: [PATCH 03/14] fix CI --- .github/workflows/CI.yml | 1 - ci/environment.yml | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 1a4fd7d785..599a551516 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -513,7 +513,6 @@ jobs: nlohmann_json python=3.10 bison=3.4 - xeus=5.1.0 xeus-zmq=3.0.0 - uses: hendrikmuhs/ccache-action@main diff --git a/ci/environment.yml b/ci/environment.yml index b9b7fd715f..c2e15df4e9 100644 --- a/ci/environment.yml +++ b/ci/environment.yml @@ -7,7 +7,7 @@ dependencies: - toml - pytest - jupyter - - xeus=1.0.1 + - xeus=5.1.0 - xtl - nlohmann_json - cppzmq From 3e85b2ff50cd49a9dba4fb8eadcd217ccc4cc88f Mon Sep 17 00:00:00 2001 From: Vipul Cariappa Date: Sat, 6 Jul 2024 16:45:53 +0530 Subject: [PATCH 04/14] fix CI --- .github/workflows/CI.yml | 1 + ci/environment.yml | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 599a551516..1a4fd7d785 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -513,6 +513,7 @@ jobs: nlohmann_json python=3.10 bison=3.4 + xeus=5.1.0 xeus-zmq=3.0.0 - uses: hendrikmuhs/ccache-action@main diff --git a/ci/environment.yml b/ci/environment.yml index c2e15df4e9..771b38631c 100644 --- a/ci/environment.yml +++ b/ci/environment.yml @@ -7,7 +7,6 @@ dependencies: - toml - pytest - jupyter - - xeus=5.1.0 - xtl - nlohmann_json - cppzmq From 217343aa7043d37d8afd1ba59a14f9d062c3f150 Mon Sep 17 00:00:00 2001 From: Vipul Cariappa Date: Sat, 6 Jul 2024 16:49:23 +0530 Subject: [PATCH 05/14] fix CI --- .github/workflows/CI.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 1a4fd7d785..d9090d22da 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -46,12 +46,12 @@ jobs: - name: Install Windows Conda Packages if: contains(matrix.os, 'windows') shell: bash -e -l {0} - run: conda install m2-bison=3.0.4 cmake=3.21.1 + run: conda install m2-bison=3.0.4 cmake=3.21.1 xeus=1.0.1 - name: Install Linux / macOS Conda Packages if: contains(matrix.os, 'ubuntu') || contains(matrix.os, 'macos') shell: bash -e -l {0} - run: conda install bison=3.4 nodejs=18 + run: conda install bison=3.4 nodejs=18 xeus=1.0.1 - name: Conda info shell: bash -e -l {0} From 732b5edbaa32341df96385d06d7bc44a140f8051 Mon Sep 17 00:00:00 2001 From: Vipul Cariappa Date: Sat, 6 Jul 2024 16:57:31 +0530 Subject: [PATCH 06/14] fix CI --- CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8919bbee1f..61a3694b89 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -208,8 +208,8 @@ endif() # XEUS (Fortran kernel) set(WITH_XEUS no CACHE BOOL "Build with XEUS support") if (WITH_XEUS) - find_package(xeus 5.1.0 REQUIRED) - find_package(xeus-zmq 3.0.0 REQUIRED) + find_package(xeus REQUIRED) + find_package(xeus-zmq REQUIRED) set(HAVE_LFORTRAN_XEUS yes) # Generate kernel.json with correct paths From 66f8fc35689ca8b6f03d0e6a587aba2d3ee60284 Mon Sep 17 00:00:00 2001 From: Vipul Cariappa Date: Sat, 6 Jul 2024 17:01:19 +0530 Subject: [PATCH 07/14] CI fix --- .github/workflows/CI.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index d9090d22da..773e91ee33 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -46,12 +46,12 @@ jobs: - name: Install Windows Conda Packages if: contains(matrix.os, 'windows') shell: bash -e -l {0} - run: conda install m2-bison=3.0.4 cmake=3.21.1 xeus=1.0.1 + run: conda install m2-bison=3.0.4 cmake=3.21.1 xeus xeus-zmq - name: Install Linux / macOS Conda Packages if: contains(matrix.os, 'ubuntu') || contains(matrix.os, 'macos') shell: bash -e -l {0} - run: conda install bison=3.4 nodejs=18 xeus=1.0.1 + run: conda install bison=3.4 nodejs=18 xeus xeus-zmq - name: Conda info shell: bash -e -l {0} From 89ba50861ee014effa4984f56cfe763ac2fdaf23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20=C4=8Cert=C3=ADk?= Date: Sat, 6 Jul 2024 10:30:36 -0600 Subject: [PATCH 08/14] CI: Do not modify other jobs --- .github/workflows/CI.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 773e91ee33..1a4fd7d785 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -46,12 +46,12 @@ jobs: - name: Install Windows Conda Packages if: contains(matrix.os, 'windows') shell: bash -e -l {0} - run: conda install m2-bison=3.0.4 cmake=3.21.1 xeus xeus-zmq + run: conda install m2-bison=3.0.4 cmake=3.21.1 - name: Install Linux / macOS Conda Packages if: contains(matrix.os, 'ubuntu') || contains(matrix.os, 'macos') shell: bash -e -l {0} - run: conda install bison=3.4 nodejs=18 xeus xeus-zmq + run: conda install bison=3.4 nodejs=18 - name: Conda info shell: bash -e -l {0} From 3a55d86d5e190a0d0e2b67a681f3c896243aea20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20=C4=8Cert=C3=ADk?= Date: Sat, 6 Jul 2024 10:32:00 -0600 Subject: [PATCH 09/14] Hide python_kernel behind ifdef --- src/bin/lpython.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/bin/lpython.cpp b/src/bin/lpython.cpp index 25f7ca8119..b14b168d36 100644 --- a/src/bin/lpython.cpp +++ b/src/bin/lpython.cpp @@ -29,7 +29,9 @@ #include #include #include -#include +#ifdef HAVE_LFORTRAN_XEUS +# include +#endif #include #include #include From ec94d152522e8a4f479c1bf04c6cb057b806f010 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20=C4=8Cert=C3=ADk?= Date: Sat, 6 Jul 2024 10:38:56 -0600 Subject: [PATCH 10/14] Keep xeus in ci/environment.yml --- ci/environment.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/environment.yml b/ci/environment.yml index 771b38631c..c2e15df4e9 100644 --- a/ci/environment.yml +++ b/ci/environment.yml @@ -7,6 +7,7 @@ dependencies: - toml - pytest - jupyter + - xeus=5.1.0 - xtl - nlohmann_json - cppzmq From 16f00af66da1912644b6e4eeb2ce8bd88f61cdd3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20=C4=8Cert=C3=ADk?= Date: Sat, 6 Jul 2024 10:56:00 -0600 Subject: [PATCH 11/14] Use micromamba --- .github/workflows/CI.yml | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 1a4fd7d785..5199e78474 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -26,32 +26,30 @@ jobs: with: fetch-depth: 0 - - name: Cache conda - uses: actions/cache@v3 - env: - CACHE_NUMBER: 0 - with: - path: ~/conda_pkgs_dir - key: - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ hashFiles('ci/environment.yml') }} - - - uses: conda-incubator/setup-miniconda@v2 +# - name: Cache conda +# uses: actions/cache@v3 +# env: +# CACHE_NUMBER: 0 +# with: +# path: ~/conda_pkgs_dir +# key: +# ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ hashFiles('ci/environment.yml') }} + + - uses: mamba-org/setup-micromamba@v1.8.0 with: - miniconda-version: "latest" - auto-update-conda: true environment-file: ci/environment.yml - python-version: ${{ matrix.python-version }} - use-only-tar-bz2: true + create-args: >- + python=${{ matrix.python-version }} - name: Install Windows Conda Packages if: contains(matrix.os, 'windows') shell: bash -e -l {0} - run: conda install m2-bison=3.0.4 cmake=3.21.1 + run: micromamba install m2-bison=3.0.4 m2-filesystem cmake=3.21.1 - name: Install Linux / macOS Conda Packages if: contains(matrix.os, 'ubuntu') || contains(matrix.os, 'macos') shell: bash -e -l {0} - run: conda install bison=3.4 nodejs=18 + run: micromamba install bison=3.4 nodejs=18 - name: Conda info shell: bash -e -l {0} From 506a6be14e4afad20d94fe317eb35f4c09652866 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20=C4=8Cert=C3=ADk?= Date: Sat, 6 Jul 2024 11:00:02 -0600 Subject: [PATCH 12/14] Add xeus-zmq --- CMakeLists.txt | 4 ++-- ci/environment.yml | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 61a3694b89..8919bbee1f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -208,8 +208,8 @@ endif() # XEUS (Fortran kernel) set(WITH_XEUS no CACHE BOOL "Build with XEUS support") if (WITH_XEUS) - find_package(xeus REQUIRED) - find_package(xeus-zmq REQUIRED) + find_package(xeus 5.1.0 REQUIRED) + find_package(xeus-zmq 3.0.0 REQUIRED) set(HAVE_LFORTRAN_XEUS yes) # Generate kernel.json with correct paths diff --git a/ci/environment.yml b/ci/environment.yml index c2e15df4e9..7c610f850c 100644 --- a/ci/environment.yml +++ b/ci/environment.yml @@ -8,6 +8,7 @@ dependencies: - pytest - jupyter - xeus=5.1.0 + - xeus-zmq=3.0.0 - xtl - nlohmann_json - cppzmq From 8362c915475669b9d37b6c886cc9fcb770a9aa92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20=C4=8Cert=C3=ADk?= Date: Sat, 6 Jul 2024 11:03:23 -0600 Subject: [PATCH 13/14] Disable conda info --- .github/workflows/CI.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 5199e78474..bf5ae9e55c 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -51,11 +51,11 @@ jobs: shell: bash -e -l {0} run: micromamba install bison=3.4 nodejs=18 - - name: Conda info - shell: bash -e -l {0} - run: | - conda info - conda list +# - name: Conda info +# shell: bash -e -l {0} +# run: | +# conda info +# conda list - name: Setup Platform (Linux) if: contains(matrix.os, 'ubuntu') From 91eb84e5cdbcf3277daffd3226620af0552ea7f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20=C4=8Cert=C3=ADk?= Date: Sat, 6 Jul 2024 11:09:22 -0600 Subject: [PATCH 14/14] CI: update Windows build commands --- .github/workflows/CI.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index bf5ae9e55c..3bc22358fc 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -85,9 +85,9 @@ jobs: if: contains(matrix.os, 'windows') shell: cmd run: | - set CONDA_INSTALL_LOCN=C:\\Miniconda3 - call %CONDA_INSTALL_LOCN%\Scripts\activate.bat - call conda activate test + set MAMBA_INSTALL_LOCN=C:\\Users\runneradmin\micromamba + call %MAMBA_INSTALL_LOCN%\Scripts\activate.bat + call micromamba activate test set LFORTRAN_CMAKE_GENERATOR=Ninja set WIN=1 set MACOS=0