diff --git a/src/main/java/com/google/api/generator/gapic/composer/RetrySettingsComposer.java b/src/main/java/com/google/api/generator/gapic/composer/RetrySettingsComposer.java index 22608b9f0d..10e722eba1 100644 --- a/src/main/java/com/google/api/generator/gapic/composer/RetrySettingsComposer.java +++ b/src/main/java/com/google/api/generator/gapic/composer/RetrySettingsComposer.java @@ -14,6 +14,9 @@ package com.google.api.generator.gapic.composer; +import com.google.api.gax.batching.BatchingSettings; +import com.google.api.gax.batching.FlowControlSettings; +import com.google.api.gax.batching.FlowController; import com.google.api.gax.grpc.ProtoOperationTransformers; import com.google.api.gax.longrunning.OperationSnapshot; import com.google.api.gax.longrunning.OperationTimedPollAlgorithm; @@ -34,6 +37,7 @@ import com.google.api.generator.engine.ast.ValueExpr; import com.google.api.generator.engine.ast.Variable; import com.google.api.generator.engine.ast.VariableExpr; +import com.google.api.generator.gapic.model.GapicBatchingSettings; import com.google.api.generator.gapic.model.GapicRetrySettings; import com.google.api.generator.gapic.model.GapicServiceConfig; import com.google.api.generator.gapic.model.Method; @@ -365,6 +369,108 @@ public static Expr createLroSettingsBuilderExpr( return builderSettingsExpr; } + public static Expr createBatchingBuilderSettingsExpr( + String settingsGetterMethodName, + GapicBatchingSettings batchingSettings, + VariableExpr builderVarExpr) { + + Expr batchingSettingsBuilderExpr = + MethodInvocationExpr.builder() + .setStaticReferenceType(STATIC_TYPES.get("BatchingSettings")) + .setMethodName("newBuilder") + .build(); + + batchingSettingsBuilderExpr = + MethodInvocationExpr.builder() + .setExprReferenceExpr(batchingSettingsBuilderExpr) + .setMethodName("setElementCountThreshold") + .setArguments(toValExpr(batchingSettings.elementCountThreshold())) + .build(); + + batchingSettingsBuilderExpr = + MethodInvocationExpr.builder() + .setExprReferenceExpr(batchingSettingsBuilderExpr) + .setMethodName("setRequestByteThreshold") + .setArguments(toValExpr(batchingSettings.requestByteThreshold())) + .build(); + + batchingSettingsBuilderExpr = + MethodInvocationExpr.builder() + .setExprReferenceExpr(batchingSettingsBuilderExpr) + .setMethodName("setDelayThreshold") + .setArguments( + createDurationOfMillisExpr(toValExpr(batchingSettings.delayThresholdMillis()))) + .build(); + + // FlowControlSettings. + Expr flowControlSettingsExpr = + MethodInvocationExpr.builder() + .setStaticReferenceType(STATIC_TYPES.get("FlowControlSettings")) + .setMethodName("newBuilder") + .build(); + if (batchingSettings.flowControlElementLimit() != null) { + flowControlSettingsExpr = + MethodInvocationExpr.builder() + .setExprReferenceExpr(flowControlSettingsExpr) + .setMethodName("setMaxOutstandingElementCount") + .setArguments(toValExpr(batchingSettings.flowControlElementLimit())) + .build(); + } + if (batchingSettings.flowControlByteLimit() != null) { + flowControlSettingsExpr = + MethodInvocationExpr.builder() + .setExprReferenceExpr(flowControlSettingsExpr) + .setMethodName("setMaxOutstandingRequestBytes") + .setArguments(toValExpr(batchingSettings.flowControlByteLimit())) + .build(); + } + flowControlSettingsExpr = + MethodInvocationExpr.builder() + .setExprReferenceExpr(flowControlSettingsExpr) + .setMethodName("setLimitExceededBehavior") + .setArguments( + EnumRefExpr.builder() + .setType(STATIC_TYPES.get("LimitExceededBehavior")) + .setName( + JavaStyle.toUpperCamelCase( + batchingSettings + .flowControlLimitExceededBehavior() + .name() + .toLowerCase())) + .build()) + .build(); + flowControlSettingsExpr = + MethodInvocationExpr.builder() + .setExprReferenceExpr(flowControlSettingsExpr) + .setMethodName("build") + .build(); + + batchingSettingsBuilderExpr = + MethodInvocationExpr.builder() + .setExprReferenceExpr(batchingSettingsBuilderExpr) + .setMethodName("setFlowControlSettings") + .setArguments(flowControlSettingsExpr) + .build(); + + batchingSettingsBuilderExpr = + MethodInvocationExpr.builder() + .setExprReferenceExpr(batchingSettingsBuilderExpr) + .setMethodName("build") + .build(); + + // Put everything together. + Expr builderExpr = + MethodInvocationExpr.builder() + .setExprReferenceExpr(builderVarExpr) + .setMethodName(settingsGetterMethodName) + .build(); + return MethodInvocationExpr.builder() + .setExprReferenceExpr(builderExpr) + .setMethodName("setBatchingSettings") + .setArguments(batchingSettingsBuilderExpr) + .build(); + } + private static Expr createRetryCodeDefinitionExpr( String codeName, List retryCodes, VariableExpr definitionsVarExpr) { // Construct something like `definitions.put("code_name", @@ -600,7 +706,10 @@ private static MethodInvocationExpr createDurationOfMillisExpr(ValueExpr valExpr private static Map createStaticTypes() { List concreteClazzes = Arrays.asList( + BatchingSettings.class, org.threeten.bp.Duration.class, + FlowControlSettings.class, + FlowController.LimitExceededBehavior.class, ImmutableMap.class, ImmutableSet.class, Lists.class, diff --git a/src/main/java/com/google/api/generator/gapic/composer/ServiceStubSettingsClassComposer.java b/src/main/java/com/google/api/generator/gapic/composer/ServiceStubSettingsClassComposer.java index 33c0845e6d..737d9054b6 100644 --- a/src/main/java/com/google/api/generator/gapic/composer/ServiceStubSettingsClassComposer.java +++ b/src/main/java/com/google/api/generator/gapic/composer/ServiceStubSettingsClassComposer.java @@ -80,6 +80,7 @@ import com.google.api.generator.engine.ast.Variable; import com.google.api.generator.engine.ast.VariableExpr; import com.google.api.generator.gapic.model.Field; +import com.google.api.generator.gapic.model.GapicBatchingSettings; import com.google.api.generator.gapic.model.GapicClass; import com.google.api.generator.gapic.model.GapicServiceConfig; import com.google.api.generator.gapic.model.Message; @@ -101,6 +102,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.function.Function; import java.util.stream.Collectors; import javax.annotation.Generated; @@ -1199,6 +1201,21 @@ private static MethodDefinition createNestedClassInitDefaultsMethod( if (streamKind.equals(Method.Stream.CLIENT) || streamKind.equals(Method.Stream.BIDI)) { continue; } + if (serviceConfig.hasBatchingSetting(service, method)) { + Optional batchingSettingOpt = + serviceConfig.getBatchingSetting(service, method); + Preconditions.checkState( + batchingSettingOpt.isPresent(), + String.format( + "No batching setting found for service %s, method %s", + service.name(), method.name())); + String settingsGetterMethodName = + String.format("%sSettings", JavaStyle.toLowerCamelCase(method.name())); + bodyExprs.add( + RetrySettingsComposer.createBatchingBuilderSettingsExpr( + settingsGetterMethodName, batchingSettingOpt.get(), builderVarExpr)); + } + bodyExprs.add( RetrySettingsComposer.createSimpleBuilderSettingsExpr( service, diff --git a/src/test/java/com/google/api/generator/gapic/composer/RetrySettingsComposerTest.java b/src/test/java/com/google/api/generator/gapic/composer/RetrySettingsComposerTest.java index 3c4a14e5e4..b9c02053a3 100644 --- a/src/test/java/com/google/api/generator/gapic/composer/RetrySettingsComposerTest.java +++ b/src/test/java/com/google/api/generator/gapic/composer/RetrySettingsComposerTest.java @@ -27,21 +27,30 @@ import com.google.api.generator.engine.ast.Variable; import com.google.api.generator.engine.ast.VariableExpr; import com.google.api.generator.engine.writer.JavaWriterVisitor; +import com.google.api.generator.gapic.model.GapicBatchingSettings; import com.google.api.generator.gapic.model.GapicServiceConfig; import com.google.api.generator.gapic.model.Message; import com.google.api.generator.gapic.model.Method; import com.google.api.generator.gapic.model.ResourceName; import com.google.api.generator.gapic.model.Service; +import com.google.api.generator.gapic.protoparser.BatchingSettingsConfigParser; import com.google.api.generator.gapic.protoparser.Parser; import com.google.api.generator.gapic.protoparser.ServiceConfigParser; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import com.google.logging.v2.LogEntryProto; +import com.google.logging.v2.LoggingConfigProto; +import com.google.logging.v2.LoggingMetricsProto; +import com.google.logging.v2.LoggingProto; import com.google.protobuf.Descriptors.FileDescriptor; import com.google.protobuf.Descriptors.ServiceDescriptor; +import com.google.pubsub.v1.PubsubProto; import com.google.showcase.v1beta1.EchoOuterClass; +import google.cloud.CommonResources; import java.nio.file.Path; import java.nio.file.Paths; import java.util.Arrays; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -52,7 +61,7 @@ import org.junit.Test; public class RetrySettingsComposerTest { - private static final String JSON_DIRECTORY = + private static final String TESTFILES_DIRECTORY = "src/test/java/com/google/api/generator/gapic/testdata/"; private static final VariableExpr RETRY_PARAM_DEFINITIONS_VAR_EXPR = createRetryParamDefinitionsVarExpr(); @@ -80,7 +89,7 @@ public void paramDefinitionsBlock_noConfigsFound() { Service service = services.get(0); String jsonFilename = "retrying_grpc_service_config.json"; - Path jsonPath = Paths.get(JSON_DIRECTORY, jsonFilename); + Path jsonPath = Paths.get(TESTFILES_DIRECTORY, jsonFilename); Optional serviceConfigOpt = ServiceConfigParser.parse(jsonPath.toString(), Optional.empty()); assertTrue(serviceConfigOpt.isPresent()); @@ -117,7 +126,7 @@ public void paramDefinitionsBlock_basic() { Service service = services.get(0); String jsonFilename = "showcase_grpc_service_config.json"; - Path jsonPath = Paths.get(JSON_DIRECTORY, jsonFilename); + Path jsonPath = Paths.get(TESTFILES_DIRECTORY, jsonFilename); Optional serviceConfigOpt = ServiceConfigParser.parse(jsonPath.toString(), Optional.empty()); assertTrue(serviceConfigOpt.isPresent()); @@ -158,7 +167,7 @@ public void codesDefinitionsBlock_noConfigsFound() { Service service = services.get(0); String jsonFilename = "retrying_grpc_service_config.json"; - Path jsonPath = Paths.get(JSON_DIRECTORY, jsonFilename); + Path jsonPath = Paths.get(TESTFILES_DIRECTORY, jsonFilename); Optional serviceConfigOpt = ServiceConfigParser.parse(jsonPath.toString(), Optional.empty()); assertTrue(serviceConfigOpt.isPresent()); @@ -195,7 +204,7 @@ public void codesDefinitionsBlock_basic() { Service service = services.get(0); String jsonFilename = "showcase_grpc_service_config.json"; - Path jsonPath = Paths.get(JSON_DIRECTORY, jsonFilename); + Path jsonPath = Paths.get(TESTFILES_DIRECTORY, jsonFilename); Optional serviceConfigOpt = ServiceConfigParser.parse(jsonPath.toString(), Optional.empty()); assertTrue(serviceConfigOpt.isPresent()); @@ -235,7 +244,7 @@ public void simpleBuilderExpr_basic() { Service service = services.get(0); String jsonFilename = "showcase_grpc_service_config.json"; - Path jsonPath = Paths.get(JSON_DIRECTORY, jsonFilename); + Path jsonPath = Paths.get(TESTFILES_DIRECTORY, jsonFilename); Optional serviceConfigOpt = ServiceConfigParser.parse(jsonPath.toString(), Optional.empty()); assertTrue(serviceConfigOpt.isPresent()); @@ -317,7 +326,7 @@ public void lroBuilderExpr() { Service service = services.get(0); String jsonFilename = "showcase_grpc_service_config.json"; - Path jsonPath = Paths.get(JSON_DIRECTORY, jsonFilename); + Path jsonPath = Paths.get(TESTFILES_DIRECTORY, jsonFilename); Optional serviceConfigOpt = ServiceConfigParser.parse(jsonPath.toString(), Optional.empty()); assertTrue(serviceConfigOpt.isPresent()); @@ -356,6 +365,153 @@ public void lroBuilderExpr() { assertEquals(expected, writerVisitor.write()); } + @Test + public void batchingSettings_minimalFlowControlSettings() { + FileDescriptor serviceFileDescriptor = PubsubProto.getDescriptor(); + FileDescriptor commonResourcesFileDescriptor = CommonResources.getDescriptor(); + ServiceDescriptor serviceDescriptor = serviceFileDescriptor.getServices().get(0); + assertEquals("Publisher", serviceDescriptor.getName()); + + Map resourceNames = new HashMap<>(); + resourceNames.putAll(Parser.parseResourceNames(serviceFileDescriptor)); + resourceNames.putAll(Parser.parseResourceNames(commonResourcesFileDescriptor)); + + Map messageTypes = Parser.parseMessages(serviceFileDescriptor); + + Set outputResourceNames = new HashSet<>(); + List services = + Parser.parseService( + serviceFileDescriptor, messageTypes, resourceNames, outputResourceNames); + + String filename = "pubsub_gapic.yaml"; + Path path = Paths.get(TESTFILES_DIRECTORY, filename); + Optional> batchingSettingsOpt = + BatchingSettingsConfigParser.parse(Optional.of(path.toString())); + assertTrue(batchingSettingsOpt.isPresent()); + + String jsonFilename = "pubsub_grpc_service_config.json"; + Path jsonPath = Paths.get(TESTFILES_DIRECTORY, jsonFilename); + Optional configOpt = + ServiceConfigParser.parse(jsonPath.toString(), batchingSettingsOpt); + assertTrue(configOpt.isPresent()); + GapicServiceConfig config = configOpt.get(); + + Service service = services.get(0); + assertEquals("Publisher", service.name()); + + VariableExpr builderVarExpr = createBuilderVarExpr(service); + String methodSettingsName = "publishSettings"; + GapicBatchingSettings batchingSettings = + GapicBatchingSettings.builder() + .setProtoPakkage("com.google.pubsub.v1") + .setServiceName("Publishing") + .setMethodName("Publish") + .setElementCountThreshold(100) + .setRequestByteThreshold(1048576) + .setDelayThresholdMillis(10) + .build(); + + Expr builderExpr = + RetrySettingsComposer.createBatchingBuilderSettingsExpr( + methodSettingsName, batchingSettings, builderVarExpr); + builderExpr.accept(writerVisitor); + String expected = + createLines( + "builder" + + ".publishSettings()" + + ".setBatchingSettings(" + + "BatchingSettings.newBuilder()" + + ".setElementCountThreshold(100L)" + + ".setRequestByteThreshold(1048576L)" + + ".setDelayThreshold(Duration.ofMillis(10L))" + + ".setFlowControlSettings(" + + "FlowControlSettings.newBuilder()" + + ".setLimitExceededBehavior(FlowController.LimitExceededBehavior.Ignore)" + + ".build())" + + ".build())"); + assertEquals(expected, writerVisitor.write()); + } + + @Test + public void batchingSettings_fullFlowControlSettings() { + FileDescriptor serviceFileDescriptor = LoggingProto.getDescriptor(); + ServiceDescriptor serviceDescriptor = serviceFileDescriptor.getServices().get(0); + assertEquals(serviceDescriptor.getName(), "LoggingServiceV2"); + + List protoFiles = + Arrays.asList( + serviceFileDescriptor, + LogEntryProto.getDescriptor(), + LoggingConfigProto.getDescriptor(), + LoggingMetricsProto.getDescriptor()); + + Map resourceNames = new HashMap<>(); + Map messageTypes = new HashMap<>(); + for (FileDescriptor fileDescriptor : protoFiles) { + resourceNames.putAll(Parser.parseResourceNames(fileDescriptor)); + messageTypes.putAll(Parser.parseMessages(fileDescriptor)); + } + + Set outputResourceNames = new HashSet<>(); + List services = + Parser.parseService( + serviceFileDescriptor, messageTypes, resourceNames, outputResourceNames); + + String filename = "logging_gapic.yaml"; + Path path = Paths.get(TESTFILES_DIRECTORY, filename); + Optional> batchingSettingsOpt = + BatchingSettingsConfigParser.parse(Optional.of(path.toString())); + assertTrue(batchingSettingsOpt.isPresent()); + + String jsonFilename = "logging_grpc_service_config.json"; + Path jsonPath = Paths.get(TESTFILES_DIRECTORY, jsonFilename); + Optional configOpt = + ServiceConfigParser.parse(jsonPath.toString(), batchingSettingsOpt); + assertTrue(configOpt.isPresent()); + GapicServiceConfig config = configOpt.get(); + + Service service = services.get(0); + assertEquals("LoggingServiceV2", service.name()); + + VariableExpr builderVarExpr = createBuilderVarExpr(service); + String methodSettingsName = "writeLogEntriesSettings"; + GapicBatchingSettings batchingSettings = + GapicBatchingSettings.builder() + .setProtoPakkage("com.google.logging.v2") + .setServiceName("LoggingServiceV2") + .setMethodName("WriteLogEntries") + .setElementCountThreshold(1000) + .setRequestByteThreshold(1048576) + .setDelayThresholdMillis(50) + .setFlowControlElementLimit(100000) + .setFlowControlByteLimit(10485760) + .setFlowControlLimitExceededBehavior( + GapicBatchingSettings.FlowControlLimitExceededBehavior.THROW_EXCEPTION) + .build(); + + Expr builderExpr = + RetrySettingsComposer.createBatchingBuilderSettingsExpr( + methodSettingsName, batchingSettings, builderVarExpr); + builderExpr.accept(writerVisitor); + String expected = + createLines( + "builder" + + ".writeLogEntriesSettings()" + + ".setBatchingSettings(" + + "BatchingSettings.newBuilder()" + + ".setElementCountThreshold(1000L)" + + ".setRequestByteThreshold(1048576L)" + + ".setDelayThreshold(Duration.ofMillis(50L))" + + ".setFlowControlSettings(" + + "FlowControlSettings.newBuilder()" + + ".setMaxOutstandingElementCount(100000L)" + + ".setMaxOutstandingRequestBytes(10485760L)" + + ".setLimitExceededBehavior(FlowController.LimitExceededBehavior.ThrowException)" + + ".build())" + + ".build())"); + assertEquals(expected, writerVisitor.write()); + } + private static Method findMethod(Service service, String methodName) { for (Method m : service.methods()) { if (m.name().equals(methodName)) { diff --git a/src/test/java/com/google/api/generator/gapic/composer/ServiceStubSettingsClassComposerTest.java b/src/test/java/com/google/api/generator/gapic/composer/ServiceStubSettingsClassComposerTest.java index 37fb1cfb19..4ef13a2350 100644 --- a/src/test/java/com/google/api/generator/gapic/composer/ServiceStubSettingsClassComposerTest.java +++ b/src/test/java/com/google/api/generator/gapic/composer/ServiceStubSettingsClassComposerTest.java @@ -682,6 +682,8 @@ private static List parseServices( + "import com.google.api.core.ApiFuture;\n" + "import com.google.api.core.BetaApi;\n" + "import com.google.api.gax.batching.BatchingSettings;\n" + + "import com.google.api.gax.batching.FlowControlSettings;\n" + + "import com.google.api.gax.batching.FlowController;\n" + "import com.google.api.gax.core.GaxProperties;\n" + "import com.google.api.gax.core.GoogleCredentialsProvider;\n" + "import com.google.api.gax.core.InstantiatingExecutorProvider;\n" @@ -1188,6 +1190,21 @@ private static List parseServices( + " .setRetrySettings(RETRY_PARAM_DEFINITIONS.get(\"retry_policy_1_params\"));\n" + " builder\n" + " .writeLogEntriesSettings()\n" + + " .setBatchingSettings(\n" + + " BatchingSettings.newBuilder()\n" + + " .setElementCountThreshold(1000L)\n" + + " .setRequestByteThreshold(1048576L)\n" + + " .setDelayThreshold(Duration.ofMillis(50L))\n" + + " .setFlowControlSettings(\n" + + " FlowControlSettings.newBuilder()\n" + + " .setMaxOutstandingElementCount(100000L)\n" + + " .setMaxOutstandingRequestBytes(10485760L)\n" + + " .setLimitExceededBehavior(\n" + + " FlowController.LimitExceededBehavior.ThrowException)\n" + + " .build())\n" + + " .build());\n" + + " builder\n" + + " .writeLogEntriesSettings()\n" + " " + " .setRetryableCodes(RETRYABLE_CODE_DEFINITIONS.get(\"retry_policy_1_codes\"))\n" + " " @@ -1275,6 +1292,8 @@ private static List parseServices( + "import com.google.api.core.ApiFuture;\n" + "import com.google.api.core.BetaApi;\n" + "import com.google.api.gax.batching.BatchingSettings;\n" + + "import com.google.api.gax.batching.FlowControlSettings;\n" + + "import com.google.api.gax.batching.FlowController;\n" + "import com.google.api.gax.core.GaxProperties;\n" + "import com.google.api.gax.core.GoogleCredentialsProvider;\n" + "import com.google.api.gax.core.InstantiatingExecutorProvider;\n" @@ -1864,6 +1883,19 @@ private static List parseServices( + " .setRetrySettings(RETRY_PARAM_DEFINITIONS.get(\"retry_policy_0_params\"));\n" + " builder\n" + " .publishSettings()\n" + + " .setBatchingSettings(\n" + + " BatchingSettings.newBuilder()\n" + + " .setElementCountThreshold(100L)\n" + + " .setRequestByteThreshold(1048576L)\n" + + " .setDelayThreshold(Duration.ofMillis(10L))\n" + + " .setFlowControlSettings(\n" + + " FlowControlSettings.newBuilder()\n" + + " " + + " .setLimitExceededBehavior(FlowController.LimitExceededBehavior.Ignore)\n" + + " .build())\n" + + " .build());\n" + + " builder\n" + + " .publishSettings()\n" + " " + " .setRetryableCodes(RETRYABLE_CODE_DEFINITIONS.get(\"retry_policy_1_codes\"))\n" + " "