diff --git a/libc/config/gpu/entrypoints.txt b/libc/config/gpu/entrypoints.txt index 4fb87cb9f5a33..b678350e9fcb1 100644 --- a/libc/config/gpu/entrypoints.txt +++ b/libc/config/gpu/entrypoints.txt @@ -211,6 +211,7 @@ set(TARGET_LIBC_ENTRYPOINTS # gpu/rpc.h entrypoints libc.src.gpu.rpc_host_call + libc.src.gpu.rpc_fprintf ) set(TARGET_LIBM_ENTRYPOINTS diff --git a/libc/include/llvm-libc-types/rpc_opcodes_t.h b/libc/include/llvm-libc-types/rpc_opcodes_t.h index 919ea039c18e3..faed7b5f5ff46 100644 --- a/libc/include/llvm-libc-types/rpc_opcodes_t.h +++ b/libc/include/llvm-libc-types/rpc_opcodes_t.h @@ -31,6 +31,9 @@ typedef enum { RPC_FTELL, RPC_FFLUSH, RPC_UNGETC, + RPC_PRINTF_TO_STDOUT, + RPC_PRINTF_TO_STDERR, + RPC_PRINTF_TO_STREAM, RPC_LAST = 0xFFFF, } rpc_opcode_t; diff --git a/libc/spec/gpu_ext.td b/libc/spec/gpu_ext.td index dce81ff778620..5400e0afa7564 100644 --- a/libc/spec/gpu_ext.td +++ b/libc/spec/gpu_ext.td @@ -10,6 +10,14 @@ def GPUExtensions : StandardSpec<"GPUExtensions"> { RetValSpec, [ArgSpec, ArgSpec, ArgSpec] >, + FunctionSpec< + "rpc_fprintf", + RetValSpec, + [ArgSpec, + ArgSpec, + ArgSpec, + ArgSpec] + >, ] >; let Headers = [ diff --git a/libc/src/__support/arg_list.h b/libc/src/__support/arg_list.h index 9de17651142f4..0965e12afd562 100644 --- a/libc/src/__support/arg_list.h +++ b/libc/src/__support/arg_list.h @@ -13,6 +13,7 @@ #include #include +#include namespace LIBC_NAMESPACE { namespace internal { @@ -60,6 +61,43 @@ class MockArgList { size_t read_count() const { return arg_counter; } }; +// Used for the GPU implementation of `printf`. This models a variadic list as a +// simple array of pointers that are built manually by the implementation. +class StructArgList { + void *ptr; + void *end; + +public: + LIBC_INLINE StructArgList(void *ptr, size_t size) + : ptr(ptr), end(reinterpret_cast(ptr) + size) {} + LIBC_INLINE StructArgList(const StructArgList &other) { + ptr = other.ptr; + end = other.end; + } + LIBC_INLINE StructArgList() = default; + LIBC_INLINE ~StructArgList() = default; + + LIBC_INLINE StructArgList &operator=(const StructArgList &rhs) { + ptr = rhs.ptr; + return *this; + } + + LIBC_INLINE void *get_ptr() const { return ptr; } + + template LIBC_INLINE T next_var() { + ptr = reinterpret_cast( + ((reinterpret_cast(ptr) + alignof(T) - 1) / alignof(T)) * + alignof(T)); + + if (ptr >= end) + return T(-1); + + T val = *reinterpret_cast(ptr); + ptr = reinterpret_cast(ptr) + sizeof(T); + return val; + } +}; + } // namespace internal } // namespace LIBC_NAMESPACE diff --git a/libc/src/gpu/CMakeLists.txt b/libc/src/gpu/CMakeLists.txt index e20228516b511..4508abea7a888 100644 --- a/libc/src/gpu/CMakeLists.txt +++ b/libc/src/gpu/CMakeLists.txt @@ -8,3 +8,15 @@ add_entrypoint_object( libc.src.__support.RPC.rpc_client libc.src.__support.GPU.utils ) + +add_entrypoint_object( + rpc_fprintf + SRCS + rpc_fprintf.cpp + HDRS + rpc_fprintf.h + DEPENDS + libc.src.stdio.gpu.gpu_file + libc.src.__support.RPC.rpc_client + libc.src.__support.GPU.utils +) diff --git a/libc/src/gpu/rpc_fprintf.cpp b/libc/src/gpu/rpc_fprintf.cpp new file mode 100644 index 0000000000000..7b0e60b59baf3 --- /dev/null +++ b/libc/src/gpu/rpc_fprintf.cpp @@ -0,0 +1,71 @@ +//===-- GPU implementation of fprintf -------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "rpc_fprintf.h" + +#include "src/__support/CPP/string_view.h" +#include "src/__support/GPU/utils.h" +#include "src/__support/RPC/rpc_client.h" +#include "src/__support/common.h" +#include "src/stdio/gpu/file.h" + +namespace LIBC_NAMESPACE { + +template +int fprintf_impl(::FILE *__restrict file, const char *__restrict format, + size_t format_size, void *args, size_t args_size) { + uint64_t mask = gpu::get_lane_mask(); + rpc::Client::Port port = rpc::client.open(); + + if constexpr (opcode == RPC_PRINTF_TO_STREAM) { + port.send([&](rpc::Buffer *buffer) { + buffer->data[0] = reinterpret_cast(file); + }); + } + + port.send_n(format, format_size); + port.send_n(args, args_size); + + uint32_t ret = 0; + for (;;) { + const char *str = nullptr; + port.recv([&](rpc::Buffer *buffer) { + ret = static_cast(buffer->data[0]); + str = reinterpret_cast(buffer->data[1]); + }); + // If any lanes have a string argument it needs to be copied back. + if (!gpu::ballot(mask, str)) + break; + + uint64_t size = str ? internal::string_length(str) + 1 : 0; + port.send_n(str, size); + } + + port.close(); + return ret; +} + +// TODO: This is a stand-in function that uses a struct pointer and size in +// place of varargs. Once varargs support is added we will use that to +// implement the real version. +LLVM_LIBC_FUNCTION(int, rpc_fprintf, + (::FILE *__restrict stream, const char *__restrict format, + void *args, size_t size)) { + cpp::string_view str(format); + if (stream == stdout) + return fprintf_impl(stream, format, str.size() + 1, + args, size); + else if (stream == stderr) + return fprintf_impl(stream, format, str.size() + 1, + args, size); + else + return fprintf_impl(stream, format, str.size() + 1, + args, size); +} + +} // namespace LIBC_NAMESPACE diff --git a/libc/src/gpu/rpc_fprintf.h b/libc/src/gpu/rpc_fprintf.h new file mode 100644 index 0000000000000..053f7b4f818ae --- /dev/null +++ b/libc/src/gpu/rpc_fprintf.h @@ -0,0 +1,22 @@ +//===-- Implementation header for RPC functions -----------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_GPU_RPC_HOST_CALL_H +#define LLVM_LIBC_SRC_GPU_RPC_HOST_CALL_H + +#include +#include + +namespace LIBC_NAMESPACE { + +int rpc_fprintf(::FILE *__restrict stream, const char *__restrict format, + void *argc, size_t size); + +} // namespace LIBC_NAMESPACE + +#endif // LLVM_LIBC_SRC_GPU_RPC_HOST_CALL_H diff --git a/libc/test/integration/src/stdio/CMakeLists.txt b/libc/test/integration/src/stdio/CMakeLists.txt index 61caa2e5fa17b..51c5ee25a6b26 100644 --- a/libc/test/integration/src/stdio/CMakeLists.txt +++ b/libc/test/integration/src/stdio/CMakeLists.txt @@ -1,3 +1,6 @@ +if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${LIBC_TARGET_OS}) + add_subdirectory(${LIBC_TARGET_OS}) +endif() add_custom_target(stdio-integration-tests) add_dependencies(libc-integration-tests stdio-integration-tests) diff --git a/libc/test/integration/src/stdio/gpu/CMakeLists.txt b/libc/test/integration/src/stdio/gpu/CMakeLists.txt new file mode 100644 index 0000000000000..6327c45e1ea5a --- /dev/null +++ b/libc/test/integration/src/stdio/gpu/CMakeLists.txt @@ -0,0 +1,21 @@ +add_custom_target(stdio-gpu-integration-tests) +add_dependencies(libc-integration-tests stdio-gpu-integration-tests) + +# Create an output directory for any temporary test files. +file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/testdata) + +# These tests are not for correctness testing, but are instead a convenient way +# to generate hermetic binaries for comparitive binary size testing. +add_integration_test( + printf_test + SUITE + stdio-gpu-integration-tests + SRCS + printf.cpp + DEPENDS + libc.src.gpu.rpc_fprintf + libc.src.stdio.fopen + LOADER_ARGS + --threads 32 + --blocks 4 +) diff --git a/libc/test/integration/src/stdio/gpu/printf.cpp b/libc/test/integration/src/stdio/gpu/printf.cpp new file mode 100644 index 0000000000000..97ad4ace1dcac --- /dev/null +++ b/libc/test/integration/src/stdio/gpu/printf.cpp @@ -0,0 +1,88 @@ +//===-- RPC test to check args to printf ----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "test/IntegrationTest/test.h" + +#include "src/__support/GPU/utils.h" +#include "src/gpu/rpc_fprintf.h" +#include "src/stdio/fopen.h" + +using namespace LIBC_NAMESPACE; + +FILE *file = LIBC_NAMESPACE::fopen("testdata/test_data.txt", "w"); + +TEST_MAIN(int argc, char **argv, char **envp) { + ASSERT_TRUE(file && "failed to open file"); + // Check basic printing. + int written = 0; + written = LIBC_NAMESPACE::rpc_fprintf(file, "A simple string\n", nullptr, 0); + ASSERT_EQ(written, 16); + + const char *str = "A simple string\n"; + written = LIBC_NAMESPACE::rpc_fprintf(file, "%s", &str, sizeof(void *)); + ASSERT_EQ(written, 16); + + // Check printing a different value with each thread. + uint64_t thread_id = gpu::get_thread_id(); + written = LIBC_NAMESPACE::rpc_fprintf(file, "%8ld\n", &thread_id, + sizeof(thread_id)); + ASSERT_EQ(written, 9); + + struct { + uint32_t x = 1; + char c = 'c'; + double f = 1.0; + } args1; + written = + LIBC_NAMESPACE::rpc_fprintf(file, "%d%c%.1f\n", &args1, sizeof(args1)); + ASSERT_EQ(written, 6); + + struct { + uint32_t x = 1; + const char *str = "A simple string\n"; + } args2; + written = + LIBC_NAMESPACE::rpc_fprintf(file, "%032b%s\n", &args2, sizeof(args2)); + ASSERT_EQ(written, 49); + + // Check that the server correctly handles divergent numbers of arguments. + const char *format = gpu::get_thread_id() % 2 ? "%s" : "%20ld\n"; + written = LIBC_NAMESPACE::rpc_fprintf(file, format, &str, sizeof(void *)); + ASSERT_EQ(written, gpu::get_thread_id() % 2 ? 16 : 21); + + format = gpu::get_thread_id() % 2 ? "%s" : str; + written = LIBC_NAMESPACE::rpc_fprintf(file, format, &str, sizeof(void *)); + ASSERT_EQ(written, 16); + + // Check that we handle null arguments correctly. + struct { + void *null = nullptr; + } args3; + written = LIBC_NAMESPACE::rpc_fprintf(file, "%p", &args3, sizeof(args3)); + ASSERT_EQ(written, 9); + +#ifndef LIBC_COPT_PRINTF_NO_NULLPTR_CHECKS + written = LIBC_NAMESPACE::rpc_fprintf(file, "%s", &args3, sizeof(args3)); + ASSERT_EQ(written, 6); +#endif // LIBC_COPT_PRINTF_NO_NULLPTR_CHECKS + + // Check for extremely abused variable width arguments + struct { + uint32_t x = 1; + uint32_t y = 2; + double f = 1.0; + } args4; + written = LIBC_NAMESPACE::rpc_fprintf(file, "%**d", &args4, sizeof(args4)); + ASSERT_EQ(written, 4); + written = LIBC_NAMESPACE::rpc_fprintf(file, "%**d%6d", &args4, sizeof(args4)); + ASSERT_EQ(written, 10); + written = LIBC_NAMESPACE::rpc_fprintf(file, "%**.**f", &args4, sizeof(args4)); + ASSERT_EQ(written, 7); + + return 0; +} diff --git a/libc/utils/gpu/server/CMakeLists.txt b/libc/utils/gpu/server/CMakeLists.txt index 6fca72cfae95f..94347ef394974 100644 --- a/libc/utils/gpu/server/CMakeLists.txt +++ b/libc/utils/gpu/server/CMakeLists.txt @@ -1,4 +1,8 @@ -add_library(llvmlibc_rpc_server STATIC rpc_server.cpp) +add_library(llvmlibc_rpc_server STATIC + ${LIBC_SOURCE_DIR}/src/stdio/printf_core/writer.cpp + ${LIBC_SOURCE_DIR}/src/stdio/printf_core/converter.cpp + rpc_server.cpp +) # Include the RPC implemenation from libc. target_include_directories(llvmlibc_rpc_server PRIVATE ${LIBC_SOURCE_DIR}) @@ -9,6 +13,10 @@ target_include_directories(llvmlibc_rpc_server PUBLIC ${CMAKE_CURRENT_SOURCE_DIR target_compile_options(llvmlibc_rpc_server PUBLIC $<$:-Wno-attributes>) target_compile_definitions(llvmlibc_rpc_server PUBLIC + LIBC_COPT_USE_C_ASSERT + LIBC_COPT_ARRAY_ARG_LIST + LIBC_COPT_PRINTF_DISABLE_WRITE_INT + LIBC_COPT_PRINTF_DISABLE_INDEX_MODE LIBC_NAMESPACE=${LIBC_NAMESPACE}) # Install the server and associated header. diff --git a/libc/utils/gpu/server/rpc_server.cpp b/libc/utils/gpu/server/rpc_server.cpp index fd306642fdcc4..095f3fa13ffad 100644 --- a/libc/utils/gpu/server/rpc_server.cpp +++ b/libc/utils/gpu/server/rpc_server.cpp @@ -14,7 +14,13 @@ #include "llvmlibc_rpc_server.h" #include "src/__support/RPC/rpc.h" +#include "src/__support/arg_list.h" +#include "src/stdio/printf_core/converter.h" +#include "src/stdio/printf_core/parser.h" +#include "src/stdio/printf_core/writer.h" + #include "src/stdio/gpu/file.h" +#include #include #include #include @@ -25,6 +31,7 @@ #include using namespace LIBC_NAMESPACE; +using namespace LIBC_NAMESPACE::printf_core; static_assert(sizeof(rpc_buffer_t) == sizeof(rpc::Buffer), "Buffer size mismatch"); @@ -32,6 +39,141 @@ static_assert(sizeof(rpc_buffer_t) == sizeof(rpc::Buffer), static_assert(RPC_MAXIMUM_PORT_COUNT == rpc::MAX_PORT_COUNT, "Incorrect maximum port count"); +template void handle_printf(rpc::Server::Port &port) { + FILE *files[lane_size] = {nullptr}; + // Get the appropriate output stream to use. + if (port.get_opcode() == RPC_PRINTF_TO_STREAM) + port.recv([&](rpc::Buffer *buffer, uint32_t id) { + files[id] = reinterpret_cast(buffer->data[0]); + }); + else if (port.get_opcode() == RPC_PRINTF_TO_STDOUT) + std::fill(files, files + lane_size, stdout); + else + std::fill(files, files + lane_size, stderr); + + uint64_t format_sizes[lane_size] = {0}; + void *format[lane_size] = {nullptr}; + + uint64_t args_sizes[lane_size] = {0}; + void *args[lane_size] = {nullptr}; + + // Recieve the format string and arguments from the client. + port.recv_n(format, format_sizes, + [&](uint64_t size) { return new char[size]; }); + port.recv_n(args, args_sizes, [&](uint64_t size) { return new char[size]; }); + + // Identify any arguments that are actually pointers to strings on the client. + // Additionally we want to determine how much buffer space we need to print. + std::vector strs_to_copy[lane_size]; + int buffer_size[lane_size] = {0}; + for (uint32_t lane = 0; lane < lane_size; ++lane) { + if (!format[lane]) + continue; + + WriteBuffer wb(nullptr, 0); + Writer writer(&wb); + + internal::StructArgList printf_args(args[lane], args_sizes[lane]); + Parser parser( + reinterpret_cast(format[lane]), printf_args); + + for (FormatSection cur_section = parser.get_next_section(); + !cur_section.raw_string.empty(); + cur_section = parser.get_next_section()) { + if (cur_section.has_conv && cur_section.conv_name == 's' && + cur_section.conv_val_ptr) { + strs_to_copy[lane].emplace_back(cur_section.conv_val_ptr); + } else if (cur_section.has_conv) { + // Ignore conversion errors for the first pass. + convert(&writer, cur_section); + } else { + writer.write(cur_section.raw_string); + } + } + buffer_size[lane] = writer.get_chars_written(); + } + + // Recieve any strings from the client and push them into a buffer. + std::vector copied_strs[lane_size]; + while (std::any_of(std::begin(strs_to_copy), std::end(strs_to_copy), + [](const auto &v) { return !v.empty() && v.back(); })) { + port.send([&](rpc::Buffer *buffer, uint32_t id) { + void *ptr = !strs_to_copy[id].empty() ? strs_to_copy[id].back() : nullptr; + buffer->data[1] = reinterpret_cast(ptr); + if (!strs_to_copy[id].empty()) + strs_to_copy[id].pop_back(); + }); + uint64_t str_sizes[lane_size] = {0}; + void *strs[lane_size] = {nullptr}; + port.recv_n(strs, str_sizes, [](uint64_t size) { return new char[size]; }); + for (uint32_t lane = 0; lane < lane_size; ++lane) { + if (!strs[lane]) + continue; + + copied_strs[lane].emplace_back(strs[lane]); + buffer_size[lane] += str_sizes[lane]; + } + } + + // Perform the final formatting and printing using the LLVM C library printf. + int results[lane_size] = {0}; + std::vector to_be_deleted; + for (uint32_t lane = 0; lane < lane_size; ++lane) { + if (!format[lane]) + continue; + + std::unique_ptr buffer(new char[buffer_size[lane]]); + WriteBuffer wb(buffer.get(), buffer_size[lane]); + Writer writer(&wb); + + internal::StructArgList printf_args(args[lane], args_sizes[lane]); + Parser parser( + reinterpret_cast(format[lane]), printf_args); + + // Parse and print the format string using the arguments we copied from + // the client. + int ret = 0; + for (FormatSection cur_section = parser.get_next_section(); + !cur_section.raw_string.empty(); + cur_section = parser.get_next_section()) { + // If this argument was a string we use the memory buffer we copied from + // the client by replacing the raw pointer with the copied one. + if (cur_section.has_conv && cur_section.conv_name == 's') { + if (!copied_strs[lane].empty()) { + cur_section.conv_val_ptr = copied_strs[lane].back(); + to_be_deleted.push_back(copied_strs[lane].back()); + copied_strs[lane].pop_back(); + } else { + cur_section.conv_val_ptr = nullptr; + } + } + if (cur_section.has_conv) { + ret = convert(&writer, cur_section); + if (ret == -1) + break; + } else { + writer.write(cur_section.raw_string); + } + } + + results[lane] = + fwrite(buffer.get(), 1, writer.get_chars_written(), files[lane]); + if (results[lane] != writer.get_chars_written() || ret == -1) + results[lane] = -1; + } + + // Send the final return value and signal completion by setting the string + // argument to null. + port.send([&](rpc::Buffer *buffer, uint32_t id) { + buffer->data[0] = static_cast(results[id]); + buffer->data[1] = reinterpret_cast(nullptr); + delete[] reinterpret_cast(format[id]); + delete[] reinterpret_cast(args[id]); + }); + for (void *ptr : to_be_deleted) + delete[] reinterpret_cast(ptr); +} + template rpc_status_t handle_server_impl( rpc::Server &server, @@ -195,6 +337,12 @@ rpc_status_t handle_server_impl( }); break; } + case RPC_PRINTF_TO_STREAM: + case RPC_PRINTF_TO_STDOUT: + case RPC_PRINTF_TO_STDERR: { + handle_printf(*port); + break; + } case RPC_NOOP: { port->recv([](rpc::Buffer *) {}); break;