feat: [datastream] Max concurrent backfill tasks (#4254)

* feat: Max concurrent backfill tasks

You can now set the number of maximum concurrent backfill tasks for a stream using the Datastream API.

PiperOrigin-RevId: 530067890

Source-Link: googleapis/googleapis@b2c290f

Source-Link: googleapis/googleapis-gen@83c5413
Copy-Tag: eyJwIjoicGFja2FnZXMvZ29vZ2xlLWNsb3VkLWRhdGFzdHJlYW0vLk93bEJvdC55YW1sIiwiaCI6IjgzYzU0MTNlNTM1Y2I2ZmE3MTA1NDFkYzZlMTY5YThhNDRiOGE2NWQifQ==

* 🦉 Updates from OwlBot post-processor

See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md

---------

Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com>
Co-authored-by: gcf-merge-on-green[bot] <60162190+gcf-merge-on-green[bot]@users.noreply.github.com>
Co-authored-by: Denis DelGrosso <[email protected]>

Author: 3 people

packages/google-cloud-datastream/README.md

@@ -1,4 +1,4 @@
-// Copyright 2022 Google LLC
+// Copyright 2023 Google LLC
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -103,14 +103,12 @@ message GcsProfile {
 }
 
 // BigQuery warehouse profile.
-message BigQueryProfile {
+message BigQueryProfile {}
 
-}
-
-// Static IP address connectivity.
-message StaticServiceIpConnectivity {
-
-}
+// Static IP address connectivity. Used when the source database is configured
+// to allow incoming connections from the Datastream public IP addresses
+// for the region specified in the connection profile.
+message StaticServiceIpConnectivity {}
 
 // Forward SSH Tunnel connectivity.
 message ForwardSshTunnelConnectivity {
@@ -181,10 +179,12 @@ message PrivateConnection {
   string name = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
 
   // Output only. The create time of the resource.
-  google.protobuf.Timestamp create_time = 2 [(google.api.field_behavior) = OUTPUT_ONLY];
+  google.protobuf.Timestamp create_time = 2
+      [(google.api.field_behavior) = OUTPUT_ONLY];
 
   // Output only. The update time of the resource.
-  google.protobuf.Timestamp update_time = 3 [(google.api.field_behavior) = OUTPUT_ONLY];
+  google.protobuf.Timestamp update_time = 3
+      [(google.api.field_behavior) = OUTPUT_ONLY];
 
   // Labels.
   map<string, string> labels = 4;
@@ -195,7 +195,8 @@ message PrivateConnection {
   // Output only. The state of the Private Connection.
   State state = 6 [(google.api.field_behavior) = OUTPUT_ONLY];
 
-  // Output only. In case of error, the details of the error in a user-friendly format.
+  // Output only. In case of error, the details of the error in a user-friendly
+  // format.
   Error error = 7 [(google.api.field_behavior) = OUTPUT_ONLY];
 
   // VPC Peering Config.
@@ -226,10 +227,12 @@ message Route {
   string name = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
 
   // Output only. The create time of the resource.
-  google.protobuf.Timestamp create_time = 2 [(google.api.field_behavior) = OUTPUT_ONLY];
+  google.protobuf.Timestamp create_time = 2
+      [(google.api.field_behavior) = OUTPUT_ONLY];
 
   // Output only. The update time of the resource.
-  google.protobuf.Timestamp update_time = 3 [(google.api.field_behavior) = OUTPUT_ONLY];
+  google.protobuf.Timestamp update_time = 3
+      [(google.api.field_behavior) = OUTPUT_ONLY];
 
   // Labels.
   map<string, string> labels = 4;
@@ -262,8 +265,8 @@ message MysqlSslConfig {
   // Output only. Indicates whether the client_certificate field is set.
   bool client_certificate_set = 4 [(google.api.field_behavior) = OUTPUT_ONLY];
 
-  // Input only. PEM-encoded certificate of the CA that signed the source database
-  // server's certificate.
+  // Input only. PEM-encoded certificate of the CA that signed the source
+  // database server's certificate.
   string ca_certificate = 5 [(google.api.field_behavior) = INPUT_ONLY];
 
   // Output only. Indicates whether the ca_certificate field is set.
@@ -282,10 +285,12 @@ message ConnectionProfile {
   string name = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
 
   // Output only. The create time of the resource.
-  google.protobuf.Timestamp create_time = 2 [(google.api.field_behavior) = OUTPUT_ONLY];
+  google.protobuf.Timestamp create_time = 2
+      [(google.api.field_behavior) = OUTPUT_ONLY];
 
   // Output only. The update time of the resource.
-  google.protobuf.Timestamp update_time = 3 [(google.api.field_behavior) = OUTPUT_ONLY];
+  google.protobuf.Timestamp update_time = 3
+      [(google.api.field_behavior) = OUTPUT_ONLY];
 
   // Labels.
   map<string, string> labels = 4;
@@ -383,31 +388,31 @@ message OracleRdbms {
 // Oracle data source configuration
 message OracleSourceConfig {
   // Configuration to drop large object values.
-  message DropLargeObjects {
-
-  }
+  message DropLargeObjects {}
 
   // Configuration to stream large object values.
-  message StreamLargeObjects {
-
-  }
+  message StreamLargeObjects {}
 
   // Oracle objects to include in the stream.
   OracleRdbms include_objects = 1;
 
   // Oracle objects to exclude from the stream.
   OracleRdbms exclude_objects = 2;
 
-  // Maximum number of concurrent CDC tasks. The number should be non negative.
-  // If not set (or set to 0), the system's default value will be used.
+  // Maximum number of concurrent CDC tasks. The number should be non-negative.
+  // If not set (or set to 0), the system's default value is used.
   int32 max_concurrent_cdc_tasks = 3;
 
+  // Maximum number of concurrent backfill tasks. The number should be
+  // non-negative. If not set (or set to 0), the system's default value is used.
+  int32 max_concurrent_backfill_tasks = 4;
+
   // The configuration for handle Oracle large objects.
   oneof large_objects_handling {
     // Drop large object values.
     DropLargeObjects drop_large_objects = 100;
 
-    // Stream large object values.
+    // Stream large object values. NOTE: This feature is currently experimental.
     StreamLargeObjects stream_large_objects = 102;
   }
 }
@@ -473,13 +478,21 @@ message PostgresqlSourceConfig {
   // PostgreSQL objects to exclude from the stream.
   PostgresqlRdbms exclude_objects = 2;
 
-  // Required. The name of the logical replication slot that's configured with the
-  // pgoutput plugin.
-  string replication_slot = 3 [(google.api.field_behavior) = REQUIRED];
+  // Required. Immutable. The name of the logical replication slot that's
+  // configured with the pgoutput plugin.
+  string replication_slot = 3 [
+    (google.api.field_behavior) = REQUIRED,
+    (google.api.field_behavior) = IMMUTABLE
+  ];
 
-  // Required. The name of the publication that includes the set of all tables that are
-  // defined in the stream's include_objects.
+  // Required. The name of the publication that includes the set of all tables
+  // that are defined in the stream's include_objects.
   string publication = 4 [(google.api.field_behavior) = REQUIRED];
+
+  // Maximum number of concurrent backfill tasks. The number should be non
+  // negative. If not set (or set to 0), the system's default value will be
+  // used.
+  int32 max_concurrent_backfill_tasks = 5;
 }
 
 // MySQL Column.
@@ -544,6 +557,11 @@ message MysqlSourceConfig {
   // Maximum number of concurrent CDC tasks. The number should be non negative.
   // If not set (or set to 0), the system's default value will be used.
   int32 max_concurrent_cdc_tasks = 3;
+
+  // Maximum number of concurrent backfill tasks. The number should be non
+  // negative. If not set (or set to 0), the system's default value will be
+  // used.
+  int32 max_concurrent_backfill_tasks = 4;
 }
 
 // The configuration of the stream source.
@@ -571,9 +589,7 @@ message SourceConfig {
 }
 
 // AVRO file format configuration.
-message AvroFileFormat {
-
-}
+message AvroFileFormat {}
 
 // JSON file format configuration.
 message JsonFileFormat {
@@ -617,7 +633,8 @@ message GcsDestinationConfig {
   int32 file_rotation_mb = 2;
 
   // The maximum duration for which new events are added before a file is
-  // closed and a new file is created.
+  // closed and a new file is created. Values within the range of 15-60 seconds
+  // are allowed.
   google.protobuf.Duration file_rotation_interval = 3;
 
   // File Format that the data should be written in.
@@ -630,9 +647,11 @@ message GcsDestinationConfig {
   }
 }
 
+// BigQuery destination configuration
 message BigQueryDestinationConfig {
   // A single target dataset to which all data will be streamed.
   message SingleTargetDataset {
+    // The dataset ID of the target dataset.
     string dataset_id = 1;
   }
 
@@ -661,6 +680,7 @@ message BigQueryDestinationConfig {
       string kms_key_name = 3;
     }
 
+    // The dataset template to use for dynamic dataset creation.
     DatasetTemplate dataset_template = 2;
   }
 
@@ -759,18 +779,18 @@ message Stream {
   }
 
   // Backfill strategy to disable automatic backfill for the Stream's objects.
-  message BackfillNoneStrategy {
-
-  }
+  message BackfillNoneStrategy {}
 
   // Output only. The stream's name.
   string name = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
 
   // Output only. The creation time of the stream.
-  google.protobuf.Timestamp create_time = 2 [(google.api.field_behavior) = OUTPUT_ONLY];
+  google.protobuf.Timestamp create_time = 2
+      [(google.api.field_behavior) = OUTPUT_ONLY];
 
   // Output only. The last update time of the stream.
-  google.protobuf.Timestamp update_time = 3 [(google.api.field_behavior) = OUTPUT_ONLY];
+  google.protobuf.Timestamp update_time = 3
+      [(google.api.field_behavior) = OUTPUT_ONLY];
 
   // Labels.
   map<string, string> labels = 4;
@@ -782,7 +802,8 @@ message Stream {
   SourceConfig source_config = 6 [(google.api.field_behavior) = REQUIRED];
 
   // Required. Destination connection profile configuration.
-  DestinationConfig destination_config = 7 [(google.api.field_behavior) = REQUIRED];
+  DestinationConfig destination_config = 7
+      [(google.api.field_behavior) = REQUIRED];
 
   // The state of the stream.
   State state = 8;
@@ -804,7 +825,8 @@ message Stream {
   // If provided, it will be used to encrypt the data.
   // If left blank, data will be encrypted using an internal Stream-specific
   // encryption key provisioned through KMS.
-  optional string customer_managed_encryption_key = 10 [(google.api.field_behavior) = IMMUTABLE];
+  optional string customer_managed_encryption_key = 10
+      [(google.api.field_behavior) = IMMUTABLE];
 }
 
 // A specific stream object (e.g a specific DB table).
@@ -818,10 +840,12 @@ message StreamObject {
   string name = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
 
   // Output only. The creation time of the object.
-  google.protobuf.Timestamp create_time = 2 [(google.api.field_behavior) = OUTPUT_ONLY];
+  google.protobuf.Timestamp create_time = 2
+      [(google.api.field_behavior) = OUTPUT_ONLY];
 
   // Output only. The last update time of the object.
-  google.protobuf.Timestamp update_time = 3 [(google.api.field_behavior) = OUTPUT_ONLY];
+  google.protobuf.Timestamp update_time = 3
+      [(google.api.field_behavior) = OUTPUT_ONLY];
 
   // Required. Display name.
   string display_name = 5 [(google.api.field_behavior) = REQUIRED];
@@ -930,10 +954,12 @@ message BackfillJob {
   Trigger trigger = 2;
 
   // Output only. Backfill job's start time.
-  google.protobuf.Timestamp last_start_time = 3 [(google.api.field_behavior) = OUTPUT_ONLY];
+  google.protobuf.Timestamp last_start_time = 3
+      [(google.api.field_behavior) = OUTPUT_ONLY];
 
   // Output only. Backfill job's end time.
-  google.protobuf.Timestamp last_end_time = 4 [(google.api.field_behavior) = OUTPUT_ONLY];
+  google.protobuf.Timestamp last_end_time = 4
+      [(google.api.field_behavior) = OUTPUT_ONLY];
 
   // Output only. Errors which caused the backfill job to fail.
   repeated Error errors = 5 [(google.api.field_behavior) = OUTPUT_ONLY];