diff --git a/libdevice/device.h b/libdevice/device.h index 1258deda031a4..72b2511e63e83 100644 --- a/libdevice/device.h +++ b/libdevice/device.h @@ -9,6 +9,10 @@ #ifndef __LIBDEVICE_DEVICE_H__ #define __LIBDEVICE_DEVICE_H__ +// We need the following header to ensure the definition of all spirv variables +// required by the wrapper libraries. +#include "spirv_vars.hpp" + #ifdef __cplusplus #define EXTERN_C extern "C" #else // __cplusplus diff --git a/libdevice/spirv_vars.hpp b/libdevice/spirv_vars.hpp new file mode 100644 index 0000000000000..c1cd0358f62e1 --- /dev/null +++ b/libdevice/spirv_vars.hpp @@ -0,0 +1,102 @@ +//==---------- spirv_vars.hpp --- SPIRV variables -------------------------==// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +// ===-------------------------------------------------------------------=== // + +#pragma once + +#include +#include + +#ifdef __SYCL_DEVICE_ONLY__ + +#ifdef __SYCL_NVPTX__ + +SYCL_EXTERNAL size_t __spirv_GlobalInvocationId_x(); +SYCL_EXTERNAL size_t __spirv_GlobalInvocationId_y(); +SYCL_EXTERNAL size_t __spirv_GlobalInvocationId_z(); + +SYCL_EXTERNAL size_t __spirv_LocalInvocationId_x(); +SYCL_EXTERNAL size_t __spirv_LocalInvocationId_y(); +SYCL_EXTERNAL size_t __spirv_LocalInvocationId_z(); + +#else // __SYCL_NVPTX__ + +typedef size_t size_t_vec __attribute__((ext_vector_type(3))); +extern "C" const __attribute__((opencl_constant)) +size_t_vec __spirv_BuiltInGlobalInvocationId; +extern "C" const __attribute__((opencl_constant)) +size_t_vec __spirv_BuiltInLocalInvocationId; + +SYCL_EXTERNAL inline size_t __spirv_GlobalInvocationId_x() { + return __spirv_BuiltInGlobalInvocationId.x; +} +SYCL_EXTERNAL inline size_t __spirv_GlobalInvocationId_y() { + return __spirv_BuiltInGlobalInvocationId.y; +} +SYCL_EXTERNAL inline size_t __spirv_GlobalInvocationId_z() { + return __spirv_BuiltInGlobalInvocationId.z; +} + +SYCL_EXTERNAL inline size_t __spirv_LocalInvocationId_x() { + return __spirv_BuiltInLocalInvocationId.x; +} +SYCL_EXTERNAL inline size_t __spirv_LocalInvocationId_y() { + return __spirv_BuiltInLocalInvocationId.y; +} +SYCL_EXTERNAL inline size_t __spirv_LocalInvocationId_z() { + return __spirv_BuiltInLocalInvocationId.z; +} + +#endif // __SYCL_NVPTX__ + +#define DEFINE_FUNC_ID_TO_XYZ_CONVERTER(POSTFIX) \ + template static inline size_t get##POSTFIX(); \ + template <> size_t get##POSTFIX<0>() { return __spirv_##POSTFIX##_x(); } \ + template <> size_t get##POSTFIX<1>() { return __spirv_##POSTFIX##_y(); } \ + template <> size_t get##POSTFIX<2>() { return __spirv_##POSTFIX##_z(); } + +namespace __spirv { + +DEFINE_FUNC_ID_TO_XYZ_CONVERTER(GlobalInvocationId); +DEFINE_FUNC_ID_TO_XYZ_CONVERTER(LocalInvocationId); + +} // namespace __spirv + +#undef DEFINE_FUNC_ID_TO_XYZ_CONVERTER + +#define DEFINE_INIT_SIZES(POSTFIX) \ + \ + template struct InitSizesST##POSTFIX; \ + \ + template struct InitSizesST##POSTFIX<1, DstT> { \ + static DstT initSize() { return {get##POSTFIX<0>()}; } \ + }; \ + \ + template struct InitSizesST##POSTFIX<2, DstT> { \ + static DstT initSize() { return {get##POSTFIX<1>(), get##POSTFIX<0>()}; } \ + }; \ + \ + template struct InitSizesST##POSTFIX<3, DstT> { \ + static DstT initSize() { \ + return {get##POSTFIX<2>(), get##POSTFIX<1>(), get##POSTFIX<0>()}; \ + } \ + }; \ + \ + template static DstT init##POSTFIX() { \ + return InitSizesST##POSTFIX::initSize(); \ + } + +namespace __spirv { + +DEFINE_INIT_SIZES(GlobalInvocationId) +DEFINE_INIT_SIZES(LocalInvocationId) + +} // namespace __spirv + +#undef DEFINE_INIT_SIZES + +#endif // __SYCL_DEVICE_ONLY__ diff --git a/llvm-spirv/lib/SPIRV/OCL20ToSPIRV.cpp b/llvm-spirv/lib/SPIRV/OCL20ToSPIRV.cpp index ee154e3211de6..f2e240e8e24fe 100644 --- a/llvm-spirv/lib/SPIRV/OCL20ToSPIRV.cpp +++ b/llvm-spirv/lib/SPIRV/OCL20ToSPIRV.cpp @@ -273,31 +273,11 @@ class OCL20ToSPIRV : public ModulePass, public InstVisitor { Module *M; LLVMContext *Ctx; unsigned CLVer; /// OpenCL version as major*10+minor - unsigned CLLang; /// OpenCL language, see `spv::SourceLanguage`. std::set ValuesToDelete; ConstantInt *addInt32(int I) { return getInt32(M, I); } ConstantInt *addSizet(uint64_t I) { return getSizet(M, I); } - /// Return the index of the id dimension represented by the demangled built-in name. - /// ie. given `__spirv__GlobalInvocationId_x`, return `0`. - Optional spirvDimensionFromBuiltin(StringRef Name) { - if (!Name.startswith("__spirv_")) { - return {}; - } - - Optional Result = {}; - if (Name.endswith("_x")) { - Result = 0; - } else if (Name.endswith("_y")) { - Result = 1; - } else if (Name.endswith("_z")) { - Result = 2; - } - - return Result; - } - /// Get vector width from OpenCL vload* function name. SPIRVWord getVecLoadWidth(const std::string &DemangledName) { SPIRVWord Width = 0; @@ -347,8 +327,7 @@ bool OCL20ToSPIRV::runOnModule(Module &Module) { M = &Module; Ctx = &M->getContext(); auto Src = getSPIRVSource(&Module); - CLLang = std::get<0>(Src); - if (CLLang != spv::SourceLanguageOpenCL_C && CLLang != spv::SourceLanguageOpenCL_CPP) + if (std::get<0>(Src) != spv::SourceLanguageOpenCL_C) return false; CLVer = std::get<1>(Src); @@ -1245,18 +1224,9 @@ void OCL20ToSPIRV::transWorkItemBuiltinsToVariables() { std::vector WorkList; for (auto &I : *M) { StringRef DemangledName; - auto MangledName = I.getName(); - LLVM_DEBUG(dbgs() << "Function mangled name: " << MangledName << '\n'); - if (!oclIsBuiltin(MangledName, DemangledName)) + if (!oclIsBuiltin(I.getName(), DemangledName)) continue; LLVM_DEBUG(dbgs() << "Function demangled name: " << DemangledName << '\n'); - auto SpirvDimension {spirvDimensionFromBuiltin(DemangledName)}; - auto IsSpirvBuiltinWithDimensions {SpirvDimension.hasValue()}; - if ((!IsSpirvBuiltinWithDimensions && CLLang == spv::SourceLanguageOpenCL_CPP) || - (IsSpirvBuiltinWithDimensions && CLLang == spv::SourceLanguageOpenCL_C)) { - // Only transform `__spirv_` builtins in OpenCL C++. - continue; - } std::string BuiltinVarName; SPIRVBuiltinVariableKind BVKind; if (!SPIRSPIRVBuiltinVariableMap::find(DemangledName.str(), &BVKind)) @@ -1265,15 +1235,11 @@ void OCL20ToSPIRV::transWorkItemBuiltinsToVariables() { std::string(kSPIRVName::Prefix) + SPIRVBuiltInNameMap::map(BVKind); LLVM_DEBUG(dbgs() << "builtin variable name: " << BuiltinVarName << '\n'); bool IsVec = I.getFunctionType()->getNumParams() > 0; - Type *GVType = (IsVec || IsSpirvBuiltinWithDimensions) ? - VectorType::get(I.getReturnType(), 3) : I.getReturnType(); - // Each of the `__spirv__GlobalInvocationId_*` functions all extract an element of - // the same global variable, so ensure that we only create the global once. - auto BV = M->getOrInsertGlobal(BuiltinVarName, GVType, [&] { - return new GlobalVariable( - *M, GVType, true, GlobalValue::ExternalLinkage, nullptr, BuiltinVarName, - 0, GlobalVariable::NotThreadLocal, SPIRAS_Input); - }); + Type *GVType = + IsVec ? VectorType::get(I.getReturnType(), 3) : I.getReturnType(); + auto BV = new GlobalVariable(*M, GVType, true, GlobalValue::ExternalLinkage, + nullptr, BuiltinVarName, 0, + GlobalVariable::NotThreadLocal, SPIRAS_Input); std::vector InstList; for (auto UI = I.user_begin(), UE = I.user_end(); UI != UE; ++UI) { auto CI = dyn_cast(*UI); @@ -1284,10 +1250,6 @@ void OCL20ToSPIRV::transWorkItemBuiltinsToVariables() { NewValue = ExtractElementInst::Create(NewValue, CI->getArgOperand(0), "", CI); LLVM_DEBUG(dbgs() << *NewValue << '\n'); - } else if (IsSpirvBuiltinWithDimensions) { - auto Index = ConstantInt::get(I.getReturnType(), SpirvDimension.getValue(), false); - NewValue = ExtractElementInst::Create(NewValue, Index, "", CI); - LLVM_DEBUG(dbgs() << *NewValue << '\n'); } NewValue->takeName(CI); CI->replaceAllUsesWith(NewValue); diff --git a/llvm-spirv/lib/SPIRV/OCLUtil.h b/llvm-spirv/lib/SPIRV/OCLUtil.h index c5ffb23f9d447..c1fa75da5dc31 100644 --- a/llvm-spirv/lib/SPIRV/OCLUtil.h +++ b/llvm-spirv/lib/SPIRV/OCLUtil.h @@ -595,46 +595,16 @@ template <> inline void SPIRVMap::init() { template <> inline void SPIRVMap::init() { add("get_work_dim", BuiltInWorkDim); - add("__spirv_GlobalSize_x", BuiltInGlobalSize); - add("__spirv_GlobalSize_y", BuiltInGlobalSize); - add("__spirv_GlobalSize_z", BuiltInGlobalSize); add("get_global_size", BuiltInGlobalSize); - add("__spirv_GlobalInvocationId_x", BuiltInGlobalInvocationId); - add("__spirv_GlobalInvocationId_y", BuiltInGlobalInvocationId); - add("__spirv_GlobalInvocationId_z", BuiltInGlobalInvocationId); add("get_global_id", BuiltInGlobalInvocationId); - add("__spirv_GlobalOffset_x", BuiltInGlobalOffset); - add("__spirv_GlobalOffset_y", BuiltInGlobalOffset); - add("__spirv_GlobalOffset_z", BuiltInGlobalOffset); add("get_global_offset", BuiltInGlobalOffset); - add("__spirv_WorkgroupSize_x", BuiltInWorkgroupSize); - add("__spirv_WorkgroupSize_y", BuiltInWorkgroupSize); - add("__spirv_WorkgroupSize_z", BuiltInWorkgroupSize); add("get_local_size", BuiltInWorkgroupSize); - add("__spirv_WorkgroupSize_x", BuiltInWorkgroupSize); - add("__spirv_WorkgroupSize_y", BuiltInWorkgroupSize); - add("__spirv_WorkgroupSize_z", BuiltInWorkgroupSize); add("get_enqueued_local_size", BuiltInEnqueuedWorkgroupSize); - add("__spirv_LocalInvocationId_x", BuiltInLocalInvocationId); - add("__spirv_LocalInvocationId_y", BuiltInLocalInvocationId); - add("__spirv_LocalInvocationId_z", BuiltInLocalInvocationId); add("get_local_id", BuiltInLocalInvocationId); - add("__spirv_NumWorkgroups_x", BuiltInNumWorkgroups); - add("__spirv_NumWorkgroups_y", BuiltInNumWorkgroups); - add("__spirv_NumWorkgroups_z", BuiltInNumWorkgroups); add("get_num_groups", BuiltInNumWorkgroups); - add("__spirv_WorkgroupId_x", BuiltInWorkgroupId); - add("__spirv_WorkgroupId_y", BuiltInWorkgroupId); - add("__spirv_WorkgroupId_z", BuiltInWorkgroupId); add("get_group_id", BuiltInWorkgroupId); - add("__spirv_WorkgroupId_x", BuiltInWorkgroupId); - add("__spirv_WorkgroupId_y", BuiltInWorkgroupId); - add("__spirv_WorkgroupId_z", BuiltInWorkgroupId); add("get_global_linear_id", BuiltInGlobalLinearId); add("get_local_linear_id", BuiltInLocalInvocationIndex); - add("__spirv_LocalInvocationId_x", BuiltInLocalInvocationId); - add("__spirv_LocalInvocationId_y", BuiltInLocalInvocationId); - add("__spirv_LocalInvocationId_z", BuiltInLocalInvocationId); add("get_sub_group_size", BuiltInSubgroupSize); add("get_max_sub_group_size", BuiltInSubgroupMaxSize); add("get_num_sub_groups", BuiltInNumSubgroups); diff --git a/llvm-spirv/test/builtin_vars_to_func.ll b/llvm-spirv/test/builtin_vars_to_func.ll deleted file mode 100644 index eb905ec7591d4..0000000000000 --- a/llvm-spirv/test/builtin_vars_to_func.ll +++ /dev/null @@ -1,41 +0,0 @@ -; RUN: llvm-as < %s | llvm-spirv -spirv-text -o %t -; RUN: FileCheck < %t %s - -; ModuleID = 'test.cl' -target datalayout = "e-p:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024" -target triple = "spir-unknown-unknown" - -; Check that only OpenCL builtins are translated in OpenCL C. - -; Make sure that OCL builtins *are not* translated. -; CHECK: {{[0-9]+}} Name {{[0-9]+}} "__spirv_BuiltInGlobalInvocationId" -declare dso_local spir_func i64 @_Z13get_global_idj() #6 - -; Make sure that `__spirv` builtins *are not* translated. -; CHECK: {{[0-9]+}} Name {{[0-9]+}} "_Z28__spirv_GlobalInvocationId_xv" -declare dso_local spir_func i64 @_Z28__spirv_GlobalInvocationId_xv() #6 - -; Function Attrs: nounwind -define spir_func void @foo() #0 { -entry: - tail call spir_func i64 @_Z28__spirv_GlobalInvocationId_xv() #2 - tail call spir_func i64 @_Z13get_global_idj() #2 - ret void -} - -!opencl.enable.FP_CONTRACT = !{} -!opencl.spir.version = !{!6} -!opencl.ocl.version = !{!7} -!opencl.used.extensions = !{!8} -!opencl.used.optional.core.features = !{!8} -!opencl.compiler.options = !{!8} -!llvm.ident = !{!9} -!spirv.Source = !{!10} -!spirv.String = !{!11} - -!6 = !{i32 1, i32 2} -!7 = !{i32 2, i32 1} -!8 = !{} -!9 = !{!"clang version 3.6.1 "} -!10 = !{i32 3, i32 200000, !11} -!11 = !{!"test.cl"} diff --git a/llvm-spirv/test/builtin_vars_to_func_cpp.ll b/llvm-spirv/test/builtin_vars_to_func_cpp.ll deleted file mode 100644 index 98ccef6888bd9..0000000000000 --- a/llvm-spirv/test/builtin_vars_to_func_cpp.ll +++ /dev/null @@ -1,42 +0,0 @@ -; RUN: llvm-as < %s | llvm-spirv -spirv-text -o %t -; RUN: FileCheck < %t %s - -; ModuleID = 'test.cl' -target datalayout = "e-p:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024" -target triple = "spir-unknown-unknown" - -; Check that only `__spirv` builtins are translated in OpenCL C++. - -; Make sure that `__spirv` builtins *are* translated. -; CHECK: {{[0-9]+}} Name 5 "__spirv_BuiltInGlobalInvocationId" -declare dso_local spir_func i64 @_Z28__spirv_GlobalInvocationId_xv() #6 - -; Make sure that OCL builtins *are not* translated. -; CHECK: {{[0-9]+}} Name 7 "_Z13get_global_idj" -declare dso_local spir_func i64 @_Z13get_global_idj() #6 - -; Function Attrs: nounwind -define spir_func void @foo() #0 { -entry: - tail call spir_func i64 @_Z28__spirv_GlobalInvocationId_xv() #2 - tail call spir_func i64 @_Z13get_global_idj() #2 - ret void -} - -!opencl.enable.FP_CONTRACT = !{} -!opencl.spir.version = !{!6} -!opencl.ocl.version = !{!7} -!opencl.used.extensions = !{!8} -!opencl.used.optional.core.features = !{!8} -!opencl.compiler.options = !{!8} -!llvm.ident = !{!9} -!spirv.Source = !{!10} -!spirv.String = !{!11} - -!6 = !{i32 1, i32 2} -!7 = !{i32 2, i32 1} -!8 = !{} -!9 = !{!"clang version 3.6.1 "} -; 4 = OpenCL C++ -!10 = !{i32 4, i32 200000, !11} -!11 = !{!"test.cl"} diff --git a/sycl/include/CL/__spirv/spirv_vars.hpp b/sycl/include/CL/__spirv/spirv_vars.hpp index ff85b98482c49..8d46b68872672 100644 --- a/sycl/include/CL/__spirv/spirv_vars.hpp +++ b/sycl/include/CL/__spirv/spirv_vars.hpp @@ -8,8 +8,13 @@ #pragma once +#include +#include + #ifdef __SYCL_DEVICE_ONLY__ +#ifdef __SYCL_NVPTX__ + SYCL_EXTERNAL size_t __spirv_GlobalInvocationId_x(); SYCL_EXTERNAL size_t __spirv_GlobalInvocationId_y(); SYCL_EXTERNAL size_t __spirv_GlobalInvocationId_z(); @@ -38,6 +43,96 @@ SYCL_EXTERNAL size_t __spirv_LocalInvocationId_x(); SYCL_EXTERNAL size_t __spirv_LocalInvocationId_y(); SYCL_EXTERNAL size_t __spirv_LocalInvocationId_z(); +#else // __SYCL_NVPTX__ + +typedef size_t size_t_vec __attribute__((ext_vector_type(3))); +extern "C" const __attribute__((opencl_constant)) +size_t_vec __spirv_BuiltInGlobalInvocationId; +extern "C" const __attribute__((opencl_constant)) +size_t_vec __spirv_BuiltInGlobalSize; +extern "C" const __attribute__((opencl_constant)) +size_t_vec __spirv_BuiltInGlobalOffset; +extern "C" const __attribute__((opencl_constant)) +size_t_vec __spirv_BuiltInNumWorkgroups; +extern "C" const __attribute__((opencl_constant)) +size_t_vec __spirv_BuiltInWorkgroupSize; +extern "C" const __attribute__((opencl_constant)) +size_t_vec __spirv_BuiltInWorkgroupId; +extern "C" const __attribute__((opencl_constant)) +size_t_vec __spirv_BuiltInLocalInvocationId; + +SYCL_EXTERNAL inline size_t __spirv_GlobalInvocationId_x() { + return __spirv_BuiltInGlobalInvocationId.x; +} +SYCL_EXTERNAL inline size_t __spirv_GlobalInvocationId_y() { + return __spirv_BuiltInGlobalInvocationId.y; +} +SYCL_EXTERNAL inline size_t __spirv_GlobalInvocationId_z() { + return __spirv_BuiltInGlobalInvocationId.z; +} + +SYCL_EXTERNAL inline size_t __spirv_GlobalSize_x() { + return __spirv_BuiltInGlobalSize.x; +} +SYCL_EXTERNAL inline size_t __spirv_GlobalSize_y() { + return __spirv_BuiltInGlobalSize.y; +} +SYCL_EXTERNAL inline size_t __spirv_GlobalSize_z() { + return __spirv_BuiltInGlobalSize.z; +} + +SYCL_EXTERNAL inline size_t __spirv_GlobalOffset_x() { + return __spirv_BuiltInGlobalOffset.x; +} +SYCL_EXTERNAL inline size_t __spirv_GlobalOffset_y() { + return __spirv_BuiltInGlobalOffset.y; +} +SYCL_EXTERNAL inline size_t __spirv_GlobalOffset_z() { + return __spirv_BuiltInGlobalOffset.z; +} + +SYCL_EXTERNAL inline size_t __spirv_NumWorkgroups_x() { + return __spirv_BuiltInNumWorkgroups.x; +} +SYCL_EXTERNAL inline size_t __spirv_NumWorkgroups_y() { + return __spirv_BuiltInNumWorkgroups.y; +} +SYCL_EXTERNAL inline size_t __spirv_NumWorkgroups_z() { + return __spirv_BuiltInNumWorkgroups.z; +} + +SYCL_EXTERNAL inline size_t __spirv_WorkgroupSize_x() { + return __spirv_BuiltInWorkgroupSize.x; +} +SYCL_EXTERNAL inline size_t __spirv_WorkgroupSize_y() { + return __spirv_BuiltInWorkgroupSize.y; +} +SYCL_EXTERNAL inline size_t __spirv_WorkgroupSize_z() { + return __spirv_BuiltInWorkgroupSize.z; +} + +SYCL_EXTERNAL inline size_t __spirv_WorkgroupId_x() { + return __spirv_BuiltInWorkgroupId.x; +} +SYCL_EXTERNAL inline size_t __spirv_WorkgroupId_y() { + return __spirv_BuiltInWorkgroupId.y; +} +SYCL_EXTERNAL inline size_t __spirv_WorkgroupId_z() { + return __spirv_BuiltInWorkgroupId.z; +} + +SYCL_EXTERNAL inline size_t __spirv_LocalInvocationId_x() { + return __spirv_BuiltInLocalInvocationId.x; +} +SYCL_EXTERNAL inline size_t __spirv_LocalInvocationId_y() { + return __spirv_BuiltInLocalInvocationId.y; +} +SYCL_EXTERNAL inline size_t __spirv_LocalInvocationId_z() { + return __spirv_BuiltInLocalInvocationId.z; +} + +#endif // __SYCL_NVPTX__ + #define DEFINE_FUNC_ID_TO_XYZ_CONVERTER(POSTFIX) \ template static inline size_t get##POSTFIX(); \ template <> size_t get##POSTFIX<0>() { return __spirv_##POSTFIX##_x(); } \ @@ -58,12 +153,18 @@ DEFINE_FUNC_ID_TO_XYZ_CONVERTER(LocalInvocationId); #undef DEFINE_FUNC_ID_TO_XYZ_CONVERTER -extern "C" const __attribute__((opencl_constant)) uint32_t __spirv_BuiltInSubgroupSize; -extern "C" const __attribute__((opencl_constant)) uint32_t __spirv_BuiltInSubgroupMaxSize; -extern "C" const __attribute__((opencl_constant)) uint32_t __spirv_BuiltInNumSubgroups; -extern "C" const __attribute__((opencl_constant)) uint32_t __spirv_BuiltInNumEnqueuedSubgroups; -extern "C" const __attribute__((opencl_constant)) uint32_t __spirv_BuiltInSubgroupId; -extern "C" const __attribute__((opencl_constant)) uint32_t __spirv_BuiltInSubgroupLocalInvocationId; +extern "C" const __attribute__((opencl_constant)) +uint32_t __spirv_BuiltInSubgroupSize; +extern "C" const __attribute__((opencl_constant)) +uint32_t __spirv_BuiltInSubgroupMaxSize; +extern "C" const __attribute__((opencl_constant)) +uint32_t __spirv_BuiltInNumSubgroups; +extern "C" const __attribute__((opencl_constant)) +uint32_t __spirv_BuiltInNumEnqueuedSubgroups; +extern "C" const __attribute__((opencl_constant)) +uint32_t __spirv_BuiltInSubgroupId; +extern "C" const __attribute__((opencl_constant)) +uint32_t __spirv_BuiltInSubgroupLocalInvocationId; #define DEFINE_INIT_SIZES(POSTFIX) \ \