From 84f91a697424503d891259f10b0dcf9832bd8673 Mon Sep 17 00:00:00 2001 From: vam-google Date: Fri, 24 Apr 2020 23:25:40 -0700 Subject: [PATCH 1/7] Make GAPIC Bazel rules production ready This includes: 1) Fix long time initial load time (5+ min). This was caused by python_rules buildling `grpcio` dependency from sources in one core (which was super slow). Switched to using bazel-native `"@com_github_grpc_grpc//src/python/grpcio/grpc:grpcio"` target instead, which is not only much faster, but is also already used in googleapis, so there is no additional cost for reusing it in microgenerator rules. 2) Properly handle `pandoc` dependency (platform-sepcific version of pandoc is properly pulled by bazel itself using toolchains). 3) Add simplistic version of the `py_gapic_assembly_pkg` rule, to make output of microgenerator compatible with `GAPICBazel` class in synthtool. 4) Add `plugin_args` argument for python_gapic_library rule to pass custom argumetns to the plugin (similar to PHP rules). --- BUILD.bazel | 32 +++++++++++-- WORKSPACE | 39 ++++++++++++++-- gapic/cli/generate_with_pandoc.py | 8 ++++ gapic_generator_python.bzl | 62 +++++++++++++++++++++++++ repositories.bzl | 36 +++++++++++++++ requirements.txt | 1 - rules_python_gapic/py_gapic.bzl | 4 +- rules_python_gapic/py_gapic_pkg.bzl | 70 +++++++++++++++++++++++++++++ 8 files changed, 242 insertions(+), 10 deletions(-) create mode 100644 gapic/cli/generate_with_pandoc.py create mode 100644 gapic_generator_python.bzl create mode 100644 rules_python_gapic/py_gapic_pkg.bzl diff --git a/BUILD.bazel b/BUILD.bazel index cf7a87fbf5..759f054b0d 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -1,17 +1,43 @@ +load("//:gapic_generator_python.bzl", "pandoc_binary", "pandoc_toolchain") load("@gapic_generator_python_pip_deps//:requirements.bzl", "requirement") +toolchain_type( + name = "pandoc_toolchain_type", + visibility = ["//visibility:public"], +) + +pandoc_toolchain( + exec_compatible_with = [ + "@bazel_tools//platforms:linux", + "@bazel_tools//platforms:x86_64", + ], + platform = "linux", +) + +pandoc_toolchain( + exec_compatible_with = [ + "@bazel_tools//platforms:osx", + "@bazel_tools//platforms:x86_64", + ], + platform = "macOS", +) + +pandoc_binary( + name = "pandoc_binary" +) + py_binary( name = "gapic_plugin", srcs = glob(["gapic/**/*.py"]), - data = glob(["gapic/**/*.j2"]), - main = "gapic/cli/generate.py", + data = [":pandoc_binary"] + glob(["gapic/**/*.j2"]), + main = "gapic/cli/generate_with_pandoc.py", visibility = ["//visibility:public"], deps = [ "@com_google_protobuf//:protobuf_python", + "@com_github_grpc_grpc//src/python/grpcio/grpc:grpcio", requirement("click"), requirement("google-api-core"), requirement("googleapis-common-protos"), - requirement("grpcio"), requirement("jinja2"), requirement("MarkupSafe"), requirement("pypandoc"), diff --git a/WORKSPACE b/WORKSPACE index 57a357e7b9..e2d97d6695 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -2,9 +2,11 @@ workspace(name = "gapic_generator_python") load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") -# -# Import rules_python -# +http_archive( + name = "bazel_skylib", + urls = ["https://github.com/bazelbuild/bazel-skylib/releases/download/0.9.0/bazel_skylib-0.9.0.tar.gz"], +) + http_archive( name = "rules_python", strip_prefix = "rules_python-748aa53d7701e71101dfd15d800e100f6ff8e5d1", @@ -22,10 +24,15 @@ pip_repositories() # # Import gapic-generator-python specific dependencies # -load("//:repositories.bzl", "gapic_generator_python") +load("//:repositories.bzl", + "gapic_generator_python", + "gapic_generator_register_toolchains" +) gapic_generator_python() +gapic_generator_register_toolchains() + load("@gapic_generator_python_pip_deps//:requirements.bzl", "pip_install") pip_install() @@ -33,3 +40,27 @@ pip_install() load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps") protobuf_deps() + +# +# Import grpc as a native bazel dependency. This avoids duplication and also +# spedds up loading phase a lot (otherwise python_rules will be building grpcio +# from sources in a single-core speed, which takes around 5 minutes on a regular +# workstation) +# +load("@com_github_grpc_grpc//bazel:grpc_deps.bzl", "grpc_deps") + +grpc_deps() + +load("@upb//bazel:repository_defs.bzl", "bazel_version_repository") + +bazel_version_repository( + name = "bazel_version", +) + +load("@build_bazel_rules_apple//apple:repositories.bzl", "apple_rules_dependencies") + +apple_rules_dependencies() + +load("@build_bazel_apple_support//lib:repositories.bzl", "apple_support_dependencies") + +apple_support_dependencies() diff --git a/gapic/cli/generate_with_pandoc.py b/gapic/cli/generate_with_pandoc.py new file mode 100644 index 0000000000..0270af15f7 --- /dev/null +++ b/gapic/cli/generate_with_pandoc.py @@ -0,0 +1,8 @@ +import os + +from gapic.cli import generate + +if __name__ == '__main__': + os.environ['PYPANDOC_PANDOC'] = os.path.join( + os.path.abspath(__file__).rsplit("gapic", 1)[0], "pandoc") + generate.generate() diff --git a/gapic_generator_python.bzl b/gapic_generator_python.bzl new file mode 100644 index 0000000000..c76f51a125 --- /dev/null +++ b/gapic_generator_python.bzl @@ -0,0 +1,62 @@ +def _pandoc_binary_impl(ctx): + toolchain = ctx.toolchains["@gapic_generator_python//:pandoc_toolchain_type"] + output = ctx.actions.declare_file(ctx.attr.binary_name) + + script = """ + cp {input} {output} + chmod +x {output} + """.format( + input = toolchain.pandoc.files.to_list()[0].path, + output = output.path, + ) + ctx.actions.run_shell( + command = script, + inputs = toolchain.pandoc.files, + outputs = [output], + ) + return [DefaultInfo(files = depset(direct = [output]), executable = output)] + +pandoc_binary = rule( + attrs = { + "binary_name": attr.string(default = "pandoc") + }, + executable = True, + toolchains = ["@gapic_generator_python//:pandoc_toolchain_type"], + implementation = _pandoc_binary_impl, +) + +# +# Toolchains +# +def _pandoc_toolchain_info_impl(ctx): + return [ + platform_common.ToolchainInfo( + pandoc = ctx.attr.pandoc, + ), + ] + +_pandoc_toolchain_info = rule( + attrs = { + "pandoc": attr.label( + allow_single_file = True, + cfg = "host", + executable = True, + ), + }, + implementation = _pandoc_toolchain_info_impl, +) + +def pandoc_toolchain(platform, exec_compatible_with): + toolchain_info_name = "pandoc_toolchain_info_%s" % platform + _pandoc_toolchain_info( + name = toolchain_info_name, + pandoc = "@pandoc_%s//:pandoc" % platform, + visibility = ["//visibility:public"], + ) + + native.toolchain( + name = "pandoc_toolchain_%s" % platform, + exec_compatible_with = exec_compatible_with, + toolchain = toolchain_info_name, + toolchain_type = ":pandoc_toolchain_type", + ) diff --git a/repositories.bzl b/repositories.bzl index ed39838d4f..a31c6440cf 100644 --- a/repositories.bzl +++ b/repositories.bzl @@ -1,6 +1,13 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") load("@rules_python//python:pip.bzl", "pip_import") +_PANDOC_BUILD_FILE = """ +filegroup( + name = "pandoc", + srcs = ["bin/pandoc"], + visibility = ["//visibility:public"], +)""" + def gapic_generator_python(): _maybe( pip_import, @@ -25,6 +32,29 @@ def gapic_generator_python(): urls = ["https://github.com/bazelbuild/bazel-skylib/archive/2169ae1c374aab4a09aa90e65efe1a3aad4e279b.tar.gz"], ) + _maybe( + http_archive, + name = "com_github_grpc_grpc", + strip_prefix = "grpc-8347f4753568b5b66e49111c60ae2841278d3f33", # this is 1.25.0 with fixes + urls = ["https://github.com/grpc/grpc/archive/8347f4753568b5b66e49111c60ae2841278d3f33.zip"], + ) + + _maybe( + http_archive, + name = "pandoc_linux", + build_file_content = _PANDOC_BUILD_FILE, + strip_prefix = "pandoc-2.2.1", + url = "https://github.com/jgm/pandoc/releases/download/2.2.1/pandoc-2.2.1-linux.tar.gz", + ) + + _maybe( + http_archive, + name = "pandoc_macOS", + build_file_content = _PANDOC_BUILD_FILE, + strip_prefix = "pandoc-2.2.1", + url = "https://github.com/jgm/pandoc/releases/download/2.2.1/pandoc-2.2.1-macOS.zip", + ) + _maybe( http_archive, name = "com_google_api_codegen", @@ -32,6 +62,12 @@ def gapic_generator_python(): urls = ["https://github.com/googleapis/gapic-generator/archive/b32c73219d617f90de70bfa6ff0ea0b0dd638dfe.zip"], ) +def gapic_generator_register_toolchains(): + native.register_toolchains( + "@gapic_generator_python//:pandoc_toolchain_linux", + "@gapic_generator_python//:pandoc_toolchain_macOS", + ) + def _maybe(repo_rule, name, strip_repo_prefix = "", **kwargs): if not name.startswith(strip_repo_prefix): return diff --git a/requirements.txt b/requirements.txt index 77abdddb2f..e525ec9e55 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,6 @@ click==7.1.1 google-api-core==1.17.0 googleapis-common-protos==1.51.0 -grpcio==1.28.1 jinja2==2.11.2 MarkupSafe==1.1.1 protobuf==3.11.3 diff --git a/rules_python_gapic/py_gapic.bzl b/rules_python_gapic/py_gapic.bzl index 50d792b82b..f132e32dce 100644 --- a/rules_python_gapic/py_gapic.bzl +++ b/rules_python_gapic/py_gapic.bzl @@ -14,7 +14,7 @@ load("@com_google_api_codegen//rules_gapic:gapic.bzl", "proto_custom_library") -def py_gapic_library(name, srcs, **kwargs): +def py_gapic_library(name, srcs, plugin_args = [], **kwargs): # srcjar_target_name = "%s_srcjar" % name srcjar_target_name = name srcjar_output_suffix = ".srcjar" @@ -23,7 +23,7 @@ def py_gapic_library(name, srcs, **kwargs): name = srcjar_target_name, deps = srcs, plugin = Label("@gapic_generator_python//:gapic_plugin"), - plugin_args = [], + plugin_args = plugin_args, plugin_file_args = {}, output_type = "python_gapic", output_suffix = srcjar_output_suffix, diff --git a/rules_python_gapic/py_gapic_pkg.bzl b/rules_python_gapic/py_gapic_pkg.bzl new file mode 100644 index 0000000000..ec80c87a84 --- /dev/null +++ b/rules_python_gapic/py_gapic_pkg.bzl @@ -0,0 +1,70 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("@com_google_api_codegen//rules_gapic:gapic_pkg.bzl", "construct_package_dir_paths") + +def _py_gapic_src_pkg_impl(ctx): + srcjar_srcs = [] + for dep in ctx.attr.deps: + for f in dep.files.to_list(): + if f.extension in ("srcjar", "jar", "zip"): + srcjar_srcs.append(f) + + paths = construct_package_dir_paths(ctx.attr.package_dir, ctx.outputs.pkg, ctx.label.name) + + script = """ + mkdir -p {package_dir_path} + for srcjar_src in {srcjar_srcs}; do + unzip -q -o $srcjar_src -d {package_dir_path} + done + cd {package_dir_path}/.. + tar -zchpf {package_dir}/{package_dir}.tar.gz {package_dir} + cd - + mv {package_dir_path}/{package_dir}.tar.gz {pkg} + rm -rf {package_dir_path} + """.format( + srcjar_srcs = " ".join(["'%s'" % f.path for f in srcjar_srcs]), + package_dir_path = paths.package_dir_path, + package_dir = paths.package_dir, + pkg = ctx.outputs.pkg.path, + package_dir_expr = paths.package_dir_expr, + ) + + ctx.actions.run_shell( + inputs = srcjar_srcs, + command = script, + outputs = [ctx.outputs.pkg], + ) + +_py_gapic_src_pkg = rule( + attrs = { + "deps": attr.label_list(allow_files = True, mandatory = True), + "package_dir": attr.string(mandatory = True), + }, + outputs = {"pkg": "%{name}.tar.gz"}, + implementation = _py_gapic_src_pkg_impl, +) + +def py_gapic_assembly_pkg(name, deps, assembly_name = None, **kwargs): + package_dir = name + if assembly_name: + package_dir = "%s-%s" % (assembly_name, name) + _py_gapic_src_pkg( + name = name, + deps = deps, + package_dir = package_dir, + **kwargs + ) + + From b0b240b560d3d1271e881672985ca6f5a8a5613a Mon Sep 17 00:00:00 2001 From: vam-google Date: Thu, 30 Apr 2020 15:07:11 -0700 Subject: [PATCH 2/7] Make GAPIC Bazel rules production ready This includes: 1) Fix long time initial load time (5+ min). This was caused by python_rules buildling `grpcio` dependency from sources in one core (which was super slow). Switched to using bazel-native `"@com_github_grpc_grpc//src/python/grpcio/grpc:grpcio"` target instead, which is not only much faster, but is also already used in googleapis, so there is no additional cost for reusing it in microgenerator rules. 2) Properly handle `pandoc` dependency (platform-sepcific version of pandoc is properly pulled by bazel itself using toolchains). 3) Add simplistic version of the `py_gapic_assembly_pkg` rule, to make output of microgenerator compatible with `GAPICBazel` class in synthtool. 4) Add `plugin_args` argument for python_gapic_library rule to pass custom argumetns to the plugin (similar to PHP rules). 5) Add python3.6 support via `--define=gapic_gen_python.3.6` command line argument for for `bazel build command`. Otherwise gapic-generator-python can be executed only on systems with python3.7+. --- BUILD.bazel | 14 +++++++++++--- requirements.txt | 1 + 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/BUILD.bazel b/BUILD.bazel index 759f054b0d..0aa3e7f1ec 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -23,7 +23,12 @@ pandoc_toolchain( ) pandoc_binary( - name = "pandoc_binary" + name = "pandoc_binary", +) + +config_setting( + name = "gapic_gen_python_3_6", + values = {"define": "gapic_gen_python=3.6"}, ) py_binary( @@ -31,6 +36,7 @@ py_binary( srcs = glob(["gapic/**/*.py"]), data = [":pandoc_binary"] + glob(["gapic/**/*.j2"]), main = "gapic/cli/generate_with_pandoc.py", + python_version = "PY3", visibility = ["//visibility:public"], deps = [ "@com_google_protobuf//:protobuf_python", @@ -42,6 +48,8 @@ py_binary( requirement("MarkupSafe"), requirement("pypandoc"), requirement("PyYAML"), - ], - python_version = "PY3", + ] + select({ + ":gapic_gen_python_3_6": [requirement("dataclasses")], + "//conditions:default": [], + }), ) diff --git a/requirements.txt b/requirements.txt index e525ec9e55..82cfe21401 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,3 +6,4 @@ MarkupSafe==1.1.1 protobuf==3.11.3 pypandoc==1.5 PyYAML==5.3.1 +dataclasses==0.6 \ No newline at end of file From 1172bba0e94abe220518f74bf381902ba65d83dc Mon Sep 17 00:00:00 2001 From: vam-google Date: Fri, 1 May 2020 17:32:17 -0700 Subject: [PATCH 3/7] set LC_ALL to fix kokoro build --- gapic/cli/generate_with_pandoc.py | 1 + 1 file changed, 1 insertion(+) diff --git a/gapic/cli/generate_with_pandoc.py b/gapic/cli/generate_with_pandoc.py index 0270af15f7..264d1c8b0b 100644 --- a/gapic/cli/generate_with_pandoc.py +++ b/gapic/cli/generate_with_pandoc.py @@ -5,4 +5,5 @@ if __name__ == '__main__': os.environ['PYPANDOC_PANDOC'] = os.path.join( os.path.abspath(__file__).rsplit("gapic", 1)[0], "pandoc") + os.environ['LC_ALL'] = 'C.UTF-8' generate.generate() From bb5e605aff2f0eb3458ad821b9d701243f4df56f Mon Sep 17 00:00:00 2001 From: vam-google Date: Fri, 1 May 2020 17:47:55 -0700 Subject: [PATCH 4/7] Fix typo --- WORKSPACE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/WORKSPACE b/WORKSPACE index e2d97d6695..b0cee19c28 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -43,7 +43,7 @@ protobuf_deps() # # Import grpc as a native bazel dependency. This avoids duplication and also -# spedds up loading phase a lot (otherwise python_rules will be building grpcio +# speeds up loading phase a lot (otherwise python_rules will be building grpcio # from sources in a single-core speed, which takes around 5 minutes on a regular # workstation) # From 80663850f33af8b339c7c9e4cd147f57232b9b31 Mon Sep 17 00:00:00 2001 From: vam-google Date: Fri, 1 May 2020 23:49:39 -0700 Subject: [PATCH 5/7] add pyenv3_toolchain --- BUILD.bazel | 20 +++++++++++++++++++- pyenv3wrapper.sh | 3 +++ 2 files changed, 22 insertions(+), 1 deletion(-) create mode 100755 pyenv3wrapper.sh diff --git a/BUILD.bazel b/BUILD.bazel index 0aa3e7f1ec..cf18c11c15 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -1,5 +1,6 @@ load("//:gapic_generator_python.bzl", "pandoc_binary", "pandoc_toolchain") load("@gapic_generator_python_pip_deps//:requirements.bzl", "requirement") +load("@bazel_tools//tools/python:toolchain.bzl", "py_runtime_pair") toolchain_type( name = "pandoc_toolchain_type", @@ -31,6 +32,23 @@ config_setting( values = {"define": "gapic_gen_python=3.6"}, ) +py_runtime( + name = "pyenv3_runtime", + interpreter = ":pyenv3wrapper.sh", + python_version="PY3", +) + +py_runtime_pair( + name = "pyenv3_runtime_pair", + py3_runtime = ":pyenv3_runtime", +) + +toolchain( + name = "pyenv3_toolchain", + toolchain = ":pyenv3_runtime_pair", + toolchain_type = "@bazel_tools//tools/python:toolchain_type", +) + py_binary( name = "gapic_plugin", srcs = glob(["gapic/**/*.py"]), @@ -52,4 +70,4 @@ py_binary( ":gapic_gen_python_3_6": [requirement("dataclasses")], "//conditions:default": [], }), -) +) \ No newline at end of file diff --git a/pyenv3wrapper.sh b/pyenv3wrapper.sh new file mode 100755 index 0000000000..0550af36db --- /dev/null +++ b/pyenv3wrapper.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +exec ~/.pyenv/shims/python3 "$@" From 6fae027e246d676d02eb80d49adf125c04fb1148 Mon Sep 17 00:00:00 2001 From: vam-google Date: Sat, 2 May 2020 00:21:29 -0700 Subject: [PATCH 6/7] add pyenv3_toolchain --- pyenv3wrapper.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyenv3wrapper.sh b/pyenv3wrapper.sh index 0550af36db..54176219f7 100755 --- a/pyenv3wrapper.sh +++ b/pyenv3wrapper.sh @@ -1,3 +1,4 @@ #!/bin/sh -exec ~/.pyenv/shims/python3 "$@" +HOME_DIR=$(getent passwd "$(whoami)" | cut -d: -f6) +exec "$HOME_DIR/.pyenv/shims/python3" "$@" From b69b8fcb578e9ad79f13650c11d2645987d942e7 Mon Sep 17 00:00:00 2001 From: vam-google Date: Mon, 4 May 2020 14:47:08 -0700 Subject: [PATCH 7/7] add newline character --- BUILD.bazel | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/BUILD.bazel b/BUILD.bazel index cf18c11c15..4d6451aa92 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -70,4 +70,4 @@ py_binary( ":gapic_gen_python_3_6": [requirement("dataclasses")], "//conditions:default": [], }), -) \ No newline at end of file +)