docs(rustdocfx): gcb to upload reference docs to dev staging (#3283)

* docs(rustdocfx): gcb to upload reference docs to dev staging

* docs(rustdocfx): gcb to upload reference docs to dev staging

* fix missing newline

* fix lint

Author: PhongChuong

.gcb/builds/triggers/main.tf

@@ -34,10 +34,15 @@ locals {
   # connection and just record it here.
   gcb_secret_name = "projects/${var.project}/secrets/github-github-oauthtoken-319d75/versions/latest"
 
-  # Add to this list of you want to have more triggers.
-  builds = {
+  # Add to these lists of you want to have more triggers.
+  pr_builds = {
     integration = {}
   }
+
+  pm_builds = {
+    integration     = {}
+    referenceupload = {}
+  }
 }
 
 # This is used to retrieve the project number. The project number is embedded in
@@ -67,7 +72,7 @@ resource "google_cloudbuildv2_repository" "main" {
 }
 
 resource "google_cloudbuild_trigger" "pull-request" {
-  for_each = tomap(local.builds)
+  for_each = tomap(local.pr_builds)
   location = var.region
   name     = "gcb-pr-${each.key}"
   filename = ".gcb/${each.key}.yaml"
@@ -87,7 +92,7 @@ resource "google_cloudbuild_trigger" "pull-request" {
 }
 
 resource "google_cloudbuild_trigger" "post-merge" {
-  for_each = tomap(local.builds)
+  for_each = tomap(local.pm_builds)
   location = var.region
   name     = "gcb-pm-${each.key}"
   filename = ".gcb/${each.key}.yaml"

.gcb/referenceupload.yaml

@@ -0,0 +1,50 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+options:
+  dynamic_substitutions: true
+  substitutionOption: 'ALLOW_LOOSE'
+  logging: CLOUD_LOGGING_ONLY
+  env:
+    - STAGING_BUCKET=docs-staging-v2-dev
+    - SCCACHE_GCS_BUCKET=${PROJECT_ID}-build-cache
+    - SCCACHE_GCS_RW_MODE=READ_WRITE
+    - SCCACHE_GCS_KEY_PREFIX=sccache/referenceupload
+    - RUSTC_WRAPPER=/workspace/.bin/sccache
+serviceAccount: 'projects/${PROJECT_ID}/serviceAccounts/integration-test-runner@${PROJECT_ID}.iam.gserviceaccount.com'
+steps:
+  - name: 'gcr.io/cloud-builders/curl'
+    script: |
+      #!/usr/bin/env bash
+      set -e
+      mkdir -p .bin
+      curl -fsSL https://github.com/mozilla/sccache/releases/download/v0.9.1/sccache-v0.9.1-x86_64-unknown-linux-musl.tar.gz |
+        tar -C /workspace/.bin -zxf - --strip-components=1
+      chmod 755 /workspace/.bin/sccache
+  - name: 'rust:1.85-bookworm'
+    script: |
+      #!/usr/bin/env bash
+      set -e
+      apt-get update
+      yes Y | apt-get install python3-pip
+      yes Y | apt-get install virtualenv python3-venv
+      python3 -m venv .venv
+      source .venv/bin/activate
+      pip install gcp-docuploader
+      wget https://go.dev/dl/go1.25.1.linux-amd64.tar.gz
+      tar -C /usr/local -xzf ./go1.25.1.linux-amd64.tar.gz
+      export PATH=$PATH:/usr/local/go/bin
+      cargo install --locked cargo-workspaces
+      go build ./doc/rustdocfx/
+      ./rustdocfx -project-root ./ -staging-bucket ${STAGING_BUCKET}

doc/rustdocfx/main.go

@@ -47,7 +47,7 @@ var crateDenyList = []string{"gcp-sdk", "google-cloud-base", "google-cloud-gax-i
 func main() {
 	out := flag.String("out", "docfx", "Output directory within project-root (default docfx)")
 	projectRoot := flag.String("project-root", "", "Top level directory of googleapis/google-cloud-rust")
-	upload := flag.Bool("upload", false, "Upload generated docfx using docuploader")
+	upload := flag.String("staging-bucket", "", "Upload the generated docfx to the gcs bucket using docuploader")
 	flag.Parse()
 
 	crates := flag.Args()
@@ -108,10 +108,9 @@ func main() {
 			}
 			fmt.Printf("Generated docfx for crate: %s\n", crate.Name)
 
-			if *upload {
+			if *upload != "" {
 				fmt.Printf("Uploading crate: %s\n", crate.Name)
-				// TODO: Add a flag to specify bucket location.
-				if err := runCmd(nil, "", "docuploader", "upload", "--staging-bucket=docs-staging-v2-dev", fmt.Sprintf("--metadata-file=%s/docs.metadata", crateOutDir), crateOutDir); err != nil {
+				if err := runCmd(nil, "", "docuploader", "upload", fmt.Sprintf("--staging-bucket=%s", *upload), "--destination-prefix=docfx", fmt.Sprintf("--metadata-file=%s/docs.metadata", crateOutDir), crateOutDir); err != nil {
 					fmt.Printf("error uploading files: %v\n", err)
 				}
 			}

doc/rustdocfx/preflight.go

@@ -21,13 +21,7 @@ import (
 )
 
 // preFlightTests() verifies all the required commands are available.
-func preFlightTests(upload bool) error {
-
-	// TODO: Preflight checks for:
-	// cargo workspaces
-	// cargo rustdoc
-	// docuploader
-	// Failfast if not installed.
+func preFlightTests(upload string) error {
 	if err := testExternalCommand("cargo", "--version"); err != nil {
 		return fmt.Errorf("got an error trying to run `cargo --version`, the instructions on https://www.rust-lang.org/learn/get-started may solve this problem: %w", err)
 	}
@@ -40,7 +34,7 @@ func preFlightTests(upload bool) error {
 	if err := testExternalCommand("cargo", "workspaces", "--version"); err != nil {
 		return fmt.Errorf("got an error trying to run `cargo workspaces --version`, run `cargo install --locked cargo-workspaces` to solve this problem: %w", err)
 	}
-	if !upload {
+	if upload == "" {
 		return nil
 	}
 	if err := testExternalCommand("docuploader", "--version"); err != nil {