diff --git a/googleapis/BUILD.bazel b/googleapis/BUILD.bazel index 42632e0f99d..5b62b21cb3a 100644 --- a/googleapis/BUILD.bazel +++ b/googleapis/BUILD.bazel @@ -13,5 +13,6 @@ java_library( "//core:internal", "//xds", artifact("com.google.guava:guava"), + artifact("com.google.errorprone:error_prone_annotations"), ], ) diff --git a/googleapis/src/main/java/io/grpc/googleapis/GoogleCloudToProdNameResolver.java b/googleapis/src/main/java/io/grpc/googleapis/GoogleCloudToProdNameResolver.java index ebc7dd05ea4..0aee17b6b9f 100644 --- a/googleapis/src/main/java/io/grpc/googleapis/GoogleCloudToProdNameResolver.java +++ b/googleapis/src/main/java/io/grpc/googleapis/GoogleCloudToProdNameResolver.java @@ -20,10 +20,11 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; -import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.io.CharStreams; +import com.google.errorprone.annotations.concurrent.GuardedBy; +import io.grpc.MetricRecorder; import io.grpc.NameResolver; import io.grpc.NameResolverRegistry; import io.grpc.Status; @@ -32,6 +33,13 @@ import io.grpc.internal.GrpcUtil; import io.grpc.internal.SharedResourceHolder; import io.grpc.internal.SharedResourceHolder.Resource; +import io.grpc.xds.InternalGrpcBootstrapperImpl; +import io.grpc.xds.InternalSharedXdsClientPoolProvider; +import io.grpc.xds.InternalSharedXdsClientPoolProvider.XdsClientResult; +import io.grpc.xds.XdsNameResolverProvider; +import io.grpc.xds.client.Bootstrapper.BootstrapInfo; +import io.grpc.xds.client.XdsClient; +import io.grpc.xds.client.XdsInitializationException; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; @@ -41,7 +49,6 @@ import java.net.URISyntaxException; import java.net.URL; import java.nio.charset.StandardCharsets; -import java.util.Map; import java.util.Random; import java.util.concurrent.Executor; import java.util.logging.Level; @@ -63,52 +70,54 @@ final class GoogleCloudToProdNameResolver extends NameResolver { static final String C2P_AUTHORITY = "traffic-director-c2p.xds.googleapis.com"; @VisibleForTesting static boolean isOnGcp = InternalCheckGcpEnvironment.isOnGcp(); - @VisibleForTesting - static boolean xdsBootstrapProvided = - System.getenv("GRPC_XDS_BOOTSTRAP") != null - || System.getProperty("io.grpc.xds.bootstrap") != null - || System.getenv("GRPC_XDS_BOOTSTRAP_CONFIG") != null - || System.getProperty("io.grpc.xds.bootstrapConfig") != null; - @VisibleForTesting - static boolean enableFederation = - Strings.isNullOrEmpty(System.getenv("GRPC_EXPERIMENTAL_XDS_FEDERATION")) - || Boolean.parseBoolean(System.getenv("GRPC_EXPERIMENTAL_XDS_FEDERATION")); private static final String serverUriOverride = System.getenv("GRPC_TEST_ONLY_GOOGLE_C2P_RESOLVER_TRAFFIC_DIRECTOR_URI"); - private HttpConnectionProvider httpConnectionProvider = HttpConnectionFactory.INSTANCE; + @GuardedBy("GoogleCloudToProdNameResolver.class") + private static BootstrapInfo bootstrapInfo; + private static HttpConnectionProvider httpConnectionProvider = HttpConnectionFactory.INSTANCE; + private static int c2pId = new Random().nextInt(); + + private static synchronized BootstrapInfo getBootstrapInfo() + throws XdsInitializationException, IOException { + if (bootstrapInfo != null) { + return bootstrapInfo; + } + BootstrapInfo bootstrapInfoTmp = + InternalGrpcBootstrapperImpl.parseBootstrap(generateBootstrap()); + // Avoid setting global when testing + if (httpConnectionProvider == HttpConnectionFactory.INSTANCE) { + bootstrapInfo = bootstrapInfoTmp; + } + return bootstrapInfoTmp; + } + private final String authority; private final SynchronizationContext syncContext; private final Resource executorResource; - private final BootstrapSetter bootstrapSetter; + private final String target; + private final MetricRecorder metricRecorder; private final NameResolver delegate; - private final Random rand; private final boolean usingExecutorResource; - // It's not possible to use both PSM and DirectPath C2P in the same application. - // Delegate to DNS if user-provided bootstrap is found. - private final String schemeOverride = - !isOnGcp - || (xdsBootstrapProvided && !enableFederation) - ? "dns" : "xds"; + private final String schemeOverride = !isOnGcp ? "dns" : "xds"; + private XdsClientResult xdsClientPool; + private XdsClient xdsClient; private Executor executor; private Listener2 listener; private boolean succeeded; private boolean resolving; private boolean shutdown; - GoogleCloudToProdNameResolver(URI targetUri, Args args, Resource executorResource, - BootstrapSetter bootstrapSetter) { - this(targetUri, args, executorResource, new Random(), bootstrapSetter, + GoogleCloudToProdNameResolver(URI targetUri, Args args, Resource executorResource) { + this(targetUri, args, executorResource, NameResolverRegistry.getDefaultRegistry().asFactory()); } @VisibleForTesting GoogleCloudToProdNameResolver(URI targetUri, Args args, Resource executorResource, - Random rand, BootstrapSetter bootstrapSetter, NameResolver.Factory nameResolverFactory) { + NameResolver.Factory nameResolverFactory) { this.executorResource = checkNotNull(executorResource, "executorResource"); - this.bootstrapSetter = checkNotNull(bootstrapSetter, "bootstrapSetter"); - this.rand = checkNotNull(rand, "rand"); String targetPath = checkNotNull(checkNotNull(targetUri, "targetUri").getPath(), "targetPath"); Preconditions.checkArgument( targetPath.startsWith("/"), @@ -118,9 +127,14 @@ final class GoogleCloudToProdNameResolver extends NameResolver { authority = GrpcUtil.checkAuthority(targetPath.substring(1)); syncContext = checkNotNull(args, "args").getSynchronizationContext(); targetUri = overrideUriScheme(targetUri, schemeOverride); - if (schemeOverride.equals("xds") && enableFederation) { + if (schemeOverride.equals("xds")) { targetUri = overrideUriAuthority(targetUri, C2P_AUTHORITY); + args = args.toBuilder() + .setArg(XdsNameResolverProvider.XDS_CLIENT_SUPPLIER, () -> xdsClient) + .build(); } + target = targetUri.toString(); + metricRecorder = args.getMetricRecorder(); delegate = checkNotNull(nameResolverFactory, "nameResolverFactory").newNameResolver( targetUri, args); executor = args.getOffloadExecutor(); @@ -150,7 +164,7 @@ private void resolve() { resolving = true; if (logger.isLoggable(Level.FINE)) { - logger.fine("resolve with schemaOverride = " + schemeOverride); + logger.log(Level.FINE, "start with schemaOverride = {0}", schemeOverride); } if (schemeOverride.equals("dns")) { @@ -168,28 +182,28 @@ private void resolve() { class Resolve implements Runnable { @Override public void run() { - ImmutableMap rawBootstrap = null; + BootstrapInfo bootstrapInfo = null; try { - // User provided bootstrap configs are only supported with federation. If federation is - // not enabled or there is no user provided config, we set a custom bootstrap override. - // Otherwise, we don't set the override, which will allow a user provided bootstrap config - // to take effect. - if (!enableFederation || !xdsBootstrapProvided) { - rawBootstrap = generateBootstrap(queryZoneMetadata(METADATA_URL_ZONE), - queryIpv6SupportMetadata(METADATA_URL_SUPPORT_IPV6)); - } + bootstrapInfo = getBootstrapInfo(); } catch (IOException e) { listener.onError( Status.INTERNAL.withDescription("Unable to get metadata").withCause(e)); + } catch (XdsInitializationException e) { + listener.onError( + Status.INTERNAL.withDescription("Unable to create c2p bootstrap").withCause(e)); + } catch (Throwable t) { + listener.onError( + Status.INTERNAL.withDescription("Unexpected error creating c2p bootstrap") + .withCause(t)); } finally { - final ImmutableMap finalRawBootstrap = rawBootstrap; + final BootstrapInfo finalBootstrapInfo = bootstrapInfo; syncContext.execute(new Runnable() { @Override public void run() { - if (!shutdown) { - if (finalRawBootstrap != null) { - bootstrapSetter.setBootstrap(finalRawBootstrap); - } + if (!shutdown && finalBootstrapInfo != null) { + xdsClientPool = InternalSharedXdsClientPoolProvider.getOrCreate( + target, finalBootstrapInfo, metricRecorder, null); + xdsClient = xdsClientPool.getObject(); delegate.start(listener); succeeded = true; } @@ -203,9 +217,16 @@ public void run() { executor.execute(new Resolve()); } - private ImmutableMap generateBootstrap(String zone, boolean supportIpv6) { + @VisibleForTesting + static ImmutableMap generateBootstrap() throws IOException { + return generateBootstrap( + queryZoneMetadata(METADATA_URL_ZONE), + queryIpv6SupportMetadata(METADATA_URL_SUPPORT_IPV6)); + } + + private static ImmutableMap generateBootstrap(String zone, boolean supportIpv6) { ImmutableMap.Builder nodeBuilder = ImmutableMap.builder(); - nodeBuilder.put("id", "C2P-" + (rand.nextInt() & Integer.MAX_VALUE)); + nodeBuilder.put("id", "C2P-" + (c2pId & Integer.MAX_VALUE)); if (!zone.isEmpty()) { nodeBuilder.put("locality", ImmutableMap.of("zone", zone)); } @@ -250,12 +271,15 @@ public void shutdown() { if (delegate != null) { delegate.shutdown(); } + if (xdsClient != null) { + xdsClient = xdsClientPool.returnObject(xdsClient); + } if (executor != null && usingExecutorResource) { executor = SharedResourceHolder.release(executorResource, executor); } } - private String queryZoneMetadata(String url) throws IOException { + private static String queryZoneMetadata(String url) throws IOException { HttpURLConnection con = null; String respBody; try { @@ -275,7 +299,7 @@ private String queryZoneMetadata(String url) throws IOException { return index == -1 ? "" : respBody.substring(index + 1); } - private boolean queryIpv6SupportMetadata(String url) throws IOException { + private static boolean queryIpv6SupportMetadata(String url) throws IOException { HttpURLConnection con = null; try { con = httpConnectionProvider.createConnection(url); @@ -294,8 +318,17 @@ private boolean queryIpv6SupportMetadata(String url) throws IOException { } @VisibleForTesting - void setHttpConnectionProvider(HttpConnectionProvider httpConnectionProvider) { - this.httpConnectionProvider = httpConnectionProvider; + static void setHttpConnectionProvider(HttpConnectionProvider httpConnectionProvider) { + if (httpConnectionProvider == null) { + GoogleCloudToProdNameResolver.httpConnectionProvider = HttpConnectionFactory.INSTANCE; + } else { + GoogleCloudToProdNameResolver.httpConnectionProvider = httpConnectionProvider; + } + } + + @VisibleForTesting + static void setC2pId(int c2pId) { + GoogleCloudToProdNameResolver.c2pId = c2pId; } private static URI overrideUriScheme(URI uri, String scheme) { @@ -335,8 +368,4 @@ public HttpURLConnection createConnection(String url) throws IOException { interface HttpConnectionProvider { HttpURLConnection createConnection(String url) throws IOException; } - - public interface BootstrapSetter { - void setBootstrap(Map bootstrap); - } } diff --git a/googleapis/src/main/java/io/grpc/googleapis/GoogleCloudToProdNameResolverProvider.java b/googleapis/src/main/java/io/grpc/googleapis/GoogleCloudToProdNameResolverProvider.java index 8ad292a3d98..c02cf53c2d2 100644 --- a/googleapis/src/main/java/io/grpc/googleapis/GoogleCloudToProdNameResolverProvider.java +++ b/googleapis/src/main/java/io/grpc/googleapis/GoogleCloudToProdNameResolverProvider.java @@ -22,13 +22,11 @@ import io.grpc.NameResolver.Args; import io.grpc.NameResolverProvider; import io.grpc.internal.GrpcUtil; -import io.grpc.xds.InternalSharedXdsClientPoolProvider; import java.net.InetSocketAddress; import java.net.SocketAddress; import java.net.URI; import java.util.Collection; import java.util.Collections; -import java.util.Map; /** * A provider for {@link GoogleCloudToProdNameResolver}. @@ -52,8 +50,7 @@ public GoogleCloudToProdNameResolverProvider() { public NameResolver newNameResolver(URI targetUri, Args args) { if (scheme.equals(targetUri.getScheme())) { return new GoogleCloudToProdNameResolver( - targetUri, args, GrpcUtil.SHARED_CHANNEL_EXECUTOR, - new SharedXdsClientPoolProviderBootstrapSetter()); + targetUri, args, GrpcUtil.SHARED_CHANNEL_EXECUTOR); } return null; } @@ -77,12 +74,4 @@ protected int priority() { public Collection> getProducedSocketAddressTypes() { return Collections.singleton(InetSocketAddress.class); } - - private static final class SharedXdsClientPoolProviderBootstrapSetter - implements GoogleCloudToProdNameResolver.BootstrapSetter { - @Override - public void setBootstrap(Map bootstrap) { - InternalSharedXdsClientPoolProvider.setDefaultProviderBootstrapOverride(bootstrap); - } - } } diff --git a/googleapis/src/test/java/io/grpc/googleapis/GoogleCloudToProdNameResolverTest.java b/googleapis/src/test/java/io/grpc/googleapis/GoogleCloudToProdNameResolverTest.java index edb3126d1e3..6a144696447 100644 --- a/googleapis/src/test/java/io/grpc/googleapis/GoogleCloudToProdNameResolverTest.java +++ b/googleapis/src/test/java/io/grpc/googleapis/GoogleCloudToProdNameResolverTest.java @@ -25,6 +25,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterables; import io.grpc.ChannelLogger; +import io.grpc.MetricRecorder; import io.grpc.NameResolver; import io.grpc.NameResolver.Args; import io.grpc.NameResolver.ServiceConfigParser; @@ -47,7 +48,6 @@ import java.util.Map; import java.util.Random; import java.util.concurrent.Executor; -import java.util.concurrent.atomic.AtomicReference; import org.junit.After; import org.junit.Before; import org.junit.Rule; @@ -77,15 +77,16 @@ public void uncaughtException(Thread t, Throwable e) { throw new AssertionError(e); } }); + private final FakeClock fakeExecutor = new FakeClock(); private final NameResolver.Args args = NameResolver.Args.newBuilder() .setDefaultPort(DEFAULT_PORT) .setProxyDetector(GrpcUtil.DEFAULT_PROXY_DETECTOR) .setSynchronizationContext(syncContext) + .setScheduledExecutorService(fakeExecutor.getScheduledExecutorService()) .setServiceConfigParser(mock(ServiceConfigParser.class)) .setChannelLogger(mock(ChannelLogger.class)) + .setMetricRecorder(new MetricRecorder() {}) .build(); - private final FakeClock fakeExecutor = new FakeClock(); - private final FakeBootstrapSetter fakeBootstrapSetter = new FakeBootstrapSetter(); private final Resource fakeExecutorResource = new Resource() { @Override public Executor create() { @@ -101,34 +102,18 @@ public void close(Executor instance) {} @Mock private NameResolver.Listener2 mockListener; - private Random random = new Random(1); @Captor private ArgumentCaptor errorCaptor; private boolean originalIsOnGcp; - private boolean originalXdsBootstrapProvided; private GoogleCloudToProdNameResolver resolver; + private String responseToIpV6 = "1:1:1"; @Before public void setUp() { nsRegistry.register(new FakeNsProvider("dns")); nsRegistry.register(new FakeNsProvider("xds")); originalIsOnGcp = GoogleCloudToProdNameResolver.isOnGcp; - originalXdsBootstrapProvided = GoogleCloudToProdNameResolver.xdsBootstrapProvided; - } - - @After - public void tearDown() { - GoogleCloudToProdNameResolver.isOnGcp = originalIsOnGcp; - GoogleCloudToProdNameResolver.xdsBootstrapProvided = originalXdsBootstrapProvided; - resolver.shutdown(); - verify(Iterables.getOnlyElement(delegatedResolver.values())).shutdown(); - } - - private void createResolver() { - createResolver("1:1:1"); - } - private void createResolver(String responseToIpV6) { HttpConnectionProvider httpConnections = new HttpConnectionProvider() { @Override public HttpURLConnection createConnection(String url) throws IOException { @@ -148,10 +133,24 @@ public HttpURLConnection createConnection(String url) throws IOException { throw new AssertionError("Unknown http query"); } }; + GoogleCloudToProdNameResolver.setHttpConnectionProvider(httpConnections); + + GoogleCloudToProdNameResolver.setC2pId(new Random(1).nextInt()); + } + + @After + public void tearDown() { + GoogleCloudToProdNameResolver.isOnGcp = originalIsOnGcp; + GoogleCloudToProdNameResolver.setHttpConnectionProvider(null); + if (resolver != null) { + resolver.shutdown(); + verify(Iterables.getOnlyElement(delegatedResolver.values())).shutdown(); + } + } + + private void createResolver() { resolver = new GoogleCloudToProdNameResolver( - TARGET_URI, args, fakeExecutorResource, random, fakeBootstrapSetter, - nsRegistry.asFactory()); - resolver.setHttpConnectionProvider(httpConnections); + TARGET_URI, args, fakeExecutorResource, nsRegistry.asFactory()); } @Test @@ -164,27 +163,19 @@ public void notOnGcp_DelegateToDns() { } @Test - public void hasProvidedBootstrap_DelegateToDns() { + public void onGcpAndNoProvidedBootstrap_DelegateToXds() { GoogleCloudToProdNameResolver.isOnGcp = true; - GoogleCloudToProdNameResolver.xdsBootstrapProvided = true; - GoogleCloudToProdNameResolver.enableFederation = false; createResolver(); resolver.start(mockListener); - assertThat(delegatedResolver.keySet()).containsExactly("dns"); + fakeExecutor.runDueTasks(); + assertThat(delegatedResolver.keySet()).containsExactly("xds"); verify(Iterables.getOnlyElement(delegatedResolver.values())).start(mockListener); } @SuppressWarnings("unchecked") @Test - public void onGcpAndNoProvidedBootstrap_DelegateToXds() { - GoogleCloudToProdNameResolver.isOnGcp = true; - GoogleCloudToProdNameResolver.xdsBootstrapProvided = false; - createResolver(); - resolver.start(mockListener); - fakeExecutor.runDueTasks(); - assertThat(delegatedResolver.keySet()).containsExactly("xds"); - verify(Iterables.getOnlyElement(delegatedResolver.values())).start(mockListener); - Map bootstrap = fakeBootstrapSetter.bootstrapRef.get(); + public void generateBootstrap_ipv6() throws IOException { + Map bootstrap = GoogleCloudToProdNameResolver.generateBootstrap(); Map node = (Map) bootstrap.get("node"); assertThat(node).containsExactly( "id", "C2P-991614323", @@ -204,15 +195,9 @@ public void onGcpAndNoProvidedBootstrap_DelegateToXds() { @SuppressWarnings("unchecked") @Test - public void onGcpAndNoProvidedBootstrap_DelegateToXds_noIpV6() { - GoogleCloudToProdNameResolver.isOnGcp = true; - GoogleCloudToProdNameResolver.xdsBootstrapProvided = false; - createResolver(null); - resolver.start(mockListener); - fakeExecutor.runDueTasks(); - assertThat(delegatedResolver.keySet()).containsExactly("xds"); - verify(Iterables.getOnlyElement(delegatedResolver.values())).start(mockListener); - Map bootstrap = fakeBootstrapSetter.bootstrapRef.get(); + public void generateBootstrap_noIpV6() throws IOException { + responseToIpV6 = null; + Map bootstrap = GoogleCloudToProdNameResolver.generateBootstrap(); Map node = (Map) bootstrap.get("node"); assertThat(node).containsExactly( "id", "C2P-991614323", @@ -231,70 +216,18 @@ public void onGcpAndNoProvidedBootstrap_DelegateToXds_noIpV6() { @SuppressWarnings("unchecked") @Test - public void emptyResolverMeetadataValue() { - GoogleCloudToProdNameResolver.isOnGcp = true; - GoogleCloudToProdNameResolver.xdsBootstrapProvided = false; - createResolver(""); - resolver.start(mockListener); - fakeExecutor.runDueTasks(); - assertThat(delegatedResolver.keySet()).containsExactly("xds"); - verify(Iterables.getOnlyElement(delegatedResolver.values())).start(mockListener); - Map bootstrap = fakeBootstrapSetter.bootstrapRef.get(); + public void emptyResolverMeetadataValue() throws IOException { + responseToIpV6 = ""; + Map bootstrap = GoogleCloudToProdNameResolver.generateBootstrap(); Map node = (Map) bootstrap.get("node"); assertThat(node).containsExactly( "id", "C2P-991614323", "locality", ImmutableMap.of("zone", ZONE)); } - @SuppressWarnings("unchecked") - @Test - public void onGcpAndNoProvidedBootstrapAndFederationEnabled_DelegateToXds() { - GoogleCloudToProdNameResolver.isOnGcp = true; - GoogleCloudToProdNameResolver.xdsBootstrapProvided = false; - GoogleCloudToProdNameResolver.enableFederation = true; - createResolver(); - resolver.start(mockListener); - fakeExecutor.runDueTasks(); - assertThat(delegatedResolver.keySet()).containsExactly("xds"); - verify(Iterables.getOnlyElement(delegatedResolver.values())).start(mockListener); - // check bootstrap - Map bootstrap = fakeBootstrapSetter.bootstrapRef.get(); - Map node = (Map) bootstrap.get("node"); - assertThat(node).containsExactly( - "id", "C2P-991614323", - "locality", ImmutableMap.of("zone", ZONE), - "metadata", ImmutableMap.of("TRAFFICDIRECTOR_DIRECTPATH_C2P_IPV6_CAPABLE", true)); - Map server = Iterables.getOnlyElement( - (List>) bootstrap.get("xds_servers")); - assertThat(server).containsExactly( - "server_uri", "directpath-pa.googleapis.com", - "channel_creds", ImmutableList.of(ImmutableMap.of("type", "google_default")), - "server_features", ImmutableList.of("xds_v3", "ignore_resource_deletion")); - Map authorities = (Map) bootstrap.get("authorities"); - assertThat(authorities).containsExactly( - "traffic-director-c2p.xds.googleapis.com", - ImmutableMap.of("xds_servers", ImmutableList.of(server))); - } - - @SuppressWarnings("unchecked") - @Test - public void onGcpAndProvidedBootstrapAndFederationEnabled_DontDelegateToXds() { - GoogleCloudToProdNameResolver.isOnGcp = true; - GoogleCloudToProdNameResolver.xdsBootstrapProvided = true; - GoogleCloudToProdNameResolver.enableFederation = true; - createResolver(); - resolver.start(mockListener); - fakeExecutor.runDueTasks(); - assertThat(delegatedResolver.keySet()).containsExactly("xds"); - verify(Iterables.getOnlyElement(delegatedResolver.values())).start(mockListener); - // Bootstrapper should not have been set, since there was no user provided config. - assertThat(fakeBootstrapSetter.bootstrapRef.get()).isNull(); - } - @Test public void failToQueryMetadata() { GoogleCloudToProdNameResolver.isOnGcp = true; - GoogleCloudToProdNameResolver.xdsBootstrapProvided = false; createResolver(); HttpConnectionProvider httpConnections = new HttpConnectionProvider() { @Override @@ -304,7 +237,7 @@ public HttpURLConnection createConnection(String url) throws IOException { return con; } }; - resolver.setHttpConnectionProvider(httpConnections); + GoogleCloudToProdNameResolver.setHttpConnectionProvider(httpConnections); resolver.start(mockListener); fakeExecutor.runDueTasks(); verify(mockListener).onError(errorCaptor.capture()); @@ -344,14 +277,4 @@ public String getDefaultScheme() { return scheme; } } - - private static final class FakeBootstrapSetter - implements GoogleCloudToProdNameResolver.BootstrapSetter { - private final AtomicReference> bootstrapRef = new AtomicReference<>(); - - @Override - public void setBootstrap(Map bootstrap) { - bootstrapRef.set(bootstrap); - } - } } diff --git a/xds/src/main/java/io/grpc/xds/ClusterImplLoadBalancer.java b/xds/src/main/java/io/grpc/xds/ClusterImplLoadBalancer.java index 00fe736761b..ad5dff3cea1 100644 --- a/xds/src/main/java/io/grpc/xds/ClusterImplLoadBalancer.java +++ b/xds/src/main/java/io/grpc/xds/ClusterImplLoadBalancer.java @@ -37,7 +37,6 @@ import io.grpc.Status; import io.grpc.internal.ForwardingClientStreamTracer; import io.grpc.internal.GrpcUtil; -import io.grpc.internal.ObjectPool; import io.grpc.services.MetricReport; import io.grpc.util.ForwardingLoadBalancerHelper; import io.grpc.util.ForwardingSubchannel; @@ -96,7 +95,6 @@ final class ClusterImplLoadBalancer extends LoadBalancer { private String cluster; @Nullable private String edsServiceName; - private ObjectPool xdsClientPool; private XdsClient xdsClient; private CallCounterProvider callCounterProvider; private ClusterDropStats dropStats; @@ -119,10 +117,8 @@ final class ClusterImplLoadBalancer extends LoadBalancer { public Status acceptResolvedAddresses(ResolvedAddresses resolvedAddresses) { logger.log(XdsLogLevel.DEBUG, "Received resolution result: {0}", resolvedAddresses); Attributes attributes = resolvedAddresses.getAttributes(); - if (xdsClientPool == null) { - xdsClientPool = attributes.get(io.grpc.xds.XdsAttributes.XDS_CLIENT_POOL); - assert xdsClientPool != null; - xdsClient = xdsClientPool.getObject(); + if (xdsClient == null) { + xdsClient = checkNotNull(attributes.get(io.grpc.xds.XdsAttributes.XDS_CLIENT), "xdsClient"); } if (callCounterProvider == null) { callCounterProvider = attributes.get(io.grpc.xds.XdsAttributes.CALL_COUNTER_PROVIDER); @@ -192,9 +188,7 @@ public void shutdown() { childLbHelper = null; } } - if (xdsClient != null) { - xdsClient = xdsClientPool.returnObject(xdsClient); - } + xdsClient = null; } /** diff --git a/xds/src/main/java/io/grpc/xds/GrpcBootstrapperImpl.java b/xds/src/main/java/io/grpc/xds/GrpcBootstrapperImpl.java index f61fab42cae..fdcf3a972b5 100644 --- a/xds/src/main/java/io/grpc/xds/GrpcBootstrapperImpl.java +++ b/xds/src/main/java/io/grpc/xds/GrpcBootstrapperImpl.java @@ -18,6 +18,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableMap; +import com.google.errorprone.annotations.concurrent.GuardedBy; import io.grpc.ChannelCredentials; import io.grpc.internal.JsonUtil; import io.grpc.xds.client.BootstrapperImpl; @@ -48,7 +49,11 @@ class GrpcBootstrapperImpl extends BootstrapperImpl { @Override public BootstrapInfo bootstrap(Map rawData) throws XdsInitializationException { - return super.bootstrap(rawData); + BootstrapInfo info = super.bootstrap(rawData); + if (info.servers().isEmpty()) { + throw new XdsInitializationException("Invalid bootstrap: 'xds_servers' is empty"); + } + return info; } /** @@ -95,6 +100,16 @@ protected Object getImplSpecificConfig(Map serverConfig, String serve return getChannelCredentials(serverConfig, serverUri); } + @GuardedBy("GrpcBootstrapperImpl.class") + private static BootstrapInfo defaultBootstrap; + + static synchronized BootstrapInfo defaultBootstrap() throws XdsInitializationException { + if (defaultBootstrap == null) { + defaultBootstrap = new GrpcBootstrapperImpl().bootstrap(); + } + return defaultBootstrap; + } + private static ChannelCredentials getChannelCredentials(Map serverConfig, String serverUri) throws XdsInitializationException { diff --git a/xds/src/main/java/io/grpc/xds/InternalGrpcBootstrapperImpl.java b/xds/src/main/java/io/grpc/xds/InternalGrpcBootstrapperImpl.java index 929619c11d7..6a852a2fbb4 100644 --- a/xds/src/main/java/io/grpc/xds/InternalGrpcBootstrapperImpl.java +++ b/xds/src/main/java/io/grpc/xds/InternalGrpcBootstrapperImpl.java @@ -17,8 +17,10 @@ package io.grpc.xds; import io.grpc.Internal; +import io.grpc.xds.client.Bootstrapper.BootstrapInfo; import io.grpc.xds.client.XdsInitializationException; import java.io.IOException; +import java.util.Map; /** * Internal accessors for GrpcBootstrapperImpl. @@ -30,4 +32,9 @@ private InternalGrpcBootstrapperImpl() {} // prevent instantiation public static String getJsonContent() throws XdsInitializationException, IOException { return new GrpcBootstrapperImpl().getJsonContent(); } + + public static BootstrapInfo parseBootstrap(Map bootstrap) + throws XdsInitializationException { + return new GrpcBootstrapperImpl().bootstrap(bootstrap); + } } diff --git a/xds/src/main/java/io/grpc/xds/InternalSharedXdsClientPoolProvider.java b/xds/src/main/java/io/grpc/xds/InternalSharedXdsClientPoolProvider.java index 9c98bba93cf..5eb36a498cf 100644 --- a/xds/src/main/java/io/grpc/xds/InternalSharedXdsClientPoolProvider.java +++ b/xds/src/main/java/io/grpc/xds/InternalSharedXdsClientPoolProvider.java @@ -20,6 +20,7 @@ import io.grpc.Internal; import io.grpc.MetricRecorder; import io.grpc.internal.ObjectPool; +import io.grpc.xds.client.Bootstrapper.BootstrapInfo; import io.grpc.xds.client.XdsClient; import io.grpc.xds.client.XdsInitializationException; import java.util.Map; @@ -32,24 +33,79 @@ public final class InternalSharedXdsClientPoolProvider { // Prevent instantiation private InternalSharedXdsClientPoolProvider() {} + /** + * Override the global bootstrap. + * + * @deprecated Use InternalGrpcBootstrapperImpl.parseBootstrap() and pass the result to + * getOrCreate(). + */ + @Deprecated public static void setDefaultProviderBootstrapOverride(Map bootstrap) { SharedXdsClientPoolProvider.getDefaultProvider().setBootstrapOverride(bootstrap); } + /** + * Get an XdsClient pool. + * + * @deprecated Use InternalGrpcBootstrapperImpl.parseBootstrap() and pass the result to the other + * getOrCreate(). + */ + @Deprecated public static ObjectPool getOrCreate(String target) throws XdsInitializationException { return getOrCreate(target, new MetricRecorder() {}); } + /** + * Get an XdsClient pool. + * + * @deprecated Use InternalGrpcBootstrapperImpl.parseBootstrap() and pass the result to the other + * getOrCreate(). + */ + @Deprecated public static ObjectPool getOrCreate(String target, MetricRecorder metricRecorder) throws XdsInitializationException { return getOrCreate(target, metricRecorder, null); } + /** + * Get an XdsClient pool. + * + * @deprecated Use InternalGrpcBootstrapperImpl.parseBootstrap() and pass the result to the other + * getOrCreate(). + */ + @Deprecated public static ObjectPool getOrCreate( String target, MetricRecorder metricRecorder, CallCredentials transportCallCredentials) throws XdsInitializationException { return SharedXdsClientPoolProvider.getDefaultProvider() .getOrCreate(target, metricRecorder, transportCallCredentials); } + + public static XdsClientResult getOrCreate( + String target, BootstrapInfo bootstrapInfo, MetricRecorder metricRecorder, + CallCredentials transportCallCredentials) { + return new XdsClientResult(SharedXdsClientPoolProvider.getDefaultProvider() + .getOrCreate(target, bootstrapInfo, metricRecorder, transportCallCredentials)); + } + + /** + * An ObjectPool, except without exposing io.grpc.internal, which must not be used for + * cross-package APIs. + */ + public static final class XdsClientResult { + private final ObjectPool xdsClientPool; + + XdsClientResult(ObjectPool xdsClientPool) { + this.xdsClientPool = xdsClientPool; + } + + public XdsClient getObject() { + return xdsClientPool.getObject(); + } + + public XdsClient returnObject(XdsClient xdsClient) { + return xdsClientPool.returnObject(xdsClient); + } + } } diff --git a/xds/src/main/java/io/grpc/xds/SharedXdsClientPoolProvider.java b/xds/src/main/java/io/grpc/xds/SharedXdsClientPoolProvider.java index 5302880d48c..29b6870ab97 100644 --- a/xds/src/main/java/io/grpc/xds/SharedXdsClientPoolProvider.java +++ b/xds/src/main/java/io/grpc/xds/SharedXdsClientPoolProvider.java @@ -73,7 +73,7 @@ static SharedXdsClientPoolProvider getDefaultProvider() { return SharedXdsClientPoolProviderHolder.instance; } - @Override + @Deprecated public void setBootstrapOverride(Map bootstrap) { bootstrapOverride.set(bootstrap); } @@ -84,30 +84,36 @@ public ObjectPool get(String target) { return targetToXdsClientMap.get(target); } - @Override - public ObjectPool getOrCreate(String target, MetricRecorder metricRecorder) + @Deprecated + public ObjectPool getOrCreate( + String target, MetricRecorder metricRecorder, CallCredentials transportCallCredentials) throws XdsInitializationException { - return getOrCreate(target, metricRecorder, null); + BootstrapInfo bootstrapInfo; + Map rawBootstrap = bootstrapOverride.get(); + if (rawBootstrap != null) { + bootstrapInfo = bootstrapper.bootstrap(rawBootstrap); + } else { + bootstrapInfo = bootstrapper.bootstrap(); + } + return getOrCreate(target, bootstrapInfo, metricRecorder, transportCallCredentials); } + @Override public ObjectPool getOrCreate( - String target, MetricRecorder metricRecorder, CallCredentials transportCallCredentials) - throws XdsInitializationException { + String target, BootstrapInfo bootstrapInfo, MetricRecorder metricRecorder) { + return getOrCreate(target, bootstrapInfo, metricRecorder, null); + } + + public ObjectPool getOrCreate( + String target, + BootstrapInfo bootstrapInfo, + MetricRecorder metricRecorder, + CallCredentials transportCallCredentials) { ObjectPool ref = targetToXdsClientMap.get(target); if (ref == null) { synchronized (lock) { ref = targetToXdsClientMap.get(target); if (ref == null) { - BootstrapInfo bootstrapInfo; - Map rawBootstrap = bootstrapOverride.get(); - if (rawBootstrap != null) { - bootstrapInfo = bootstrapper.bootstrap(rawBootstrap); - } else { - bootstrapInfo = bootstrapper.bootstrap(); - } - if (bootstrapInfo.servers().isEmpty()) { - throw new XdsInitializationException("No xDS server provided"); - } ref = new RefCountedXdsClientObjectPool( bootstrapInfo, target, metricRecorder, transportCallCredentials); @@ -157,9 +163,9 @@ class RefCountedXdsClientObjectPool implements ObjectPool { String target, MetricRecorder metricRecorder, CallCredentials transportCallCredentials) { - this.bootstrapInfo = checkNotNull(bootstrapInfo); + this.bootstrapInfo = checkNotNull(bootstrapInfo, "bootstrapInfo"); this.target = target; - this.metricRecorder = metricRecorder; + this.metricRecorder = checkNotNull(metricRecorder, "metricRecorder"); this.transportCallCredentials = transportCallCredentials; } diff --git a/xds/src/main/java/io/grpc/xds/XdsAttributes.java b/xds/src/main/java/io/grpc/xds/XdsAttributes.java index 0e770173219..5647b25e418 100644 --- a/xds/src/main/java/io/grpc/xds/XdsAttributes.java +++ b/xds/src/main/java/io/grpc/xds/XdsAttributes.java @@ -20,7 +20,6 @@ import io.grpc.EquivalentAddressGroup; import io.grpc.Grpc; import io.grpc.NameResolver; -import io.grpc.internal.ObjectPool; import io.grpc.xds.XdsNameResolverProvider.CallCounterProvider; import io.grpc.xds.client.Locality; import io.grpc.xds.client.XdsClient; @@ -33,8 +32,8 @@ final class XdsAttributes { * Attribute key for passing around the XdsClient object pool across NameResolver/LoadBalancers. */ @NameResolver.ResolutionResultAttr - static final Attributes.Key> XDS_CLIENT_POOL = - Attributes.Key.create("io.grpc.xds.XdsAttributes.xdsClientPool"); + static final Attributes.Key XDS_CLIENT = + Attributes.Key.create("io.grpc.xds.XdsAttributes.xdsClient"); /** * Attribute key for passing around the latest XdsConfig across NameResolver/LoadBalancers. diff --git a/xds/src/main/java/io/grpc/xds/XdsClientPoolFactory.java b/xds/src/main/java/io/grpc/xds/XdsClientPoolFactory.java index f10d6504d79..6df8d566a7a 100644 --- a/xds/src/main/java/io/grpc/xds/XdsClientPoolFactory.java +++ b/xds/src/main/java/io/grpc/xds/XdsClientPoolFactory.java @@ -18,20 +18,17 @@ import io.grpc.MetricRecorder; import io.grpc.internal.ObjectPool; +import io.grpc.xds.client.Bootstrapper.BootstrapInfo; import io.grpc.xds.client.XdsClient; -import io.grpc.xds.client.XdsInitializationException; import java.util.List; -import java.util.Map; import javax.annotation.Nullable; interface XdsClientPoolFactory { - void setBootstrapOverride(Map bootstrap); - @Nullable ObjectPool get(String target); - ObjectPool getOrCreate(String target, MetricRecorder metricRecorder) - throws XdsInitializationException; + ObjectPool getOrCreate( + String target, BootstrapInfo bootstrapInfo, MetricRecorder metricRecorder); List getTargets(); } diff --git a/xds/src/main/java/io/grpc/xds/XdsNameResolver.java b/xds/src/main/java/io/grpc/xds/XdsNameResolver.java index 20001b6558d..196d51fb5a6 100644 --- a/xds/src/main/java/io/grpc/xds/XdsNameResolver.java +++ b/xds/src/main/java/io/grpc/xds/XdsNameResolver.java @@ -64,6 +64,7 @@ import io.grpc.xds.client.Bootstrapper.AuthorityInfo; import io.grpc.xds.client.Bootstrapper.BootstrapInfo; import io.grpc.xds.client.XdsClient; +import io.grpc.xds.client.XdsInitializationException; import io.grpc.xds.client.XdsLogger; import io.grpc.xds.client.XdsLogger.XdsLogLevel; import java.net.URI; @@ -80,6 +81,7 @@ import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Supplier; import javax.annotation.Nullable; /** @@ -91,7 +93,6 @@ * @see XdsNameResolverProvider */ final class XdsNameResolver extends NameResolver { - static final CallOptions.Key CLUSTER_SELECTION_KEY = CallOptions.Key.create("io.grpc.xds.CLUSTER_SELECTION_KEY"); static final CallOptions.Key XDS_CONFIG_CALL_OPTION_KEY = @@ -118,7 +119,7 @@ final class XdsNameResolver extends NameResolver { private final ServiceConfigParser serviceConfigParser; private final SynchronizationContext syncContext; private final ScheduledExecutorService scheduler; - private final XdsClientPoolFactory xdsClientPoolFactory; + private final XdsClientPool xdsClientPool; private final ThreadSafeRandom random; private final FilterRegistry filterRegistry; private final XxHash64 hashFunc = XxHash64.INSTANCE; @@ -127,7 +128,6 @@ final class XdsNameResolver extends NameResolver { private final ConcurrentMap clusterRefs = new ConcurrentHashMap<>(); private final ConfigSelector configSelector = new ConfigSelector(); private final long randomChannelId; - private final MetricRecorder metricRecorder; private final Args nameResolverArgs; // Must be accessed in syncContext. // Filter instances are unique per channel, and per filter (name+typeUrl). @@ -136,7 +136,6 @@ final class XdsNameResolver extends NameResolver { private volatile RoutingConfig routingConfig; private Listener2 listener; - private ObjectPool xdsClientPool; private XdsClient xdsClient; private CallCounterProvider callCounterProvider; private ResolveState resolveState; @@ -148,7 +147,10 @@ final class XdsNameResolver extends NameResolver { @Nullable Map bootstrapOverride, MetricRecorder metricRecorder, Args nameResolverArgs) { this(targetUri, targetUri.getAuthority(), name, overrideAuthority, serviceConfigParser, - syncContext, scheduler, SharedXdsClientPoolProvider.getDefaultProvider(), + syncContext, scheduler, + bootstrapOverride == null + ? SharedXdsClientPoolProvider.getDefaultProvider() + : new SharedXdsClientPoolProvider(), ThreadSafeRandomImpl.instance, FilterRegistry.getDefaultRegistry(), bootstrapOverride, metricRecorder, nameResolverArgs); } @@ -173,12 +175,17 @@ final class XdsNameResolver extends NameResolver { this.serviceConfigParser = checkNotNull(serviceConfigParser, "serviceConfigParser"); this.syncContext = checkNotNull(syncContext, "syncContext"); this.scheduler = checkNotNull(scheduler, "scheduler"); - this.xdsClientPoolFactory = bootstrapOverride == null ? checkNotNull(xdsClientPoolFactory, - "xdsClientPoolFactory") : new SharedXdsClientPoolProvider(); - this.xdsClientPoolFactory.setBootstrapOverride(bootstrapOverride); + Supplier xdsClientSupplierArg = + nameResolverArgs.getArg(XdsNameResolverProvider.XDS_CLIENT_SUPPLIER); + if (xdsClientSupplierArg != null) { + this.xdsClientPool = new SupplierXdsClientPool(xdsClientSupplierArg); + } else { + checkNotNull(xdsClientPoolFactory, "xdsClientPoolFactory"); + this.xdsClientPool = new BootstrappingXdsClientPool( + xdsClientPoolFactory, target, bootstrapOverride, metricRecorder); + } this.random = checkNotNull(random, "random"); this.filterRegistry = checkNotNull(filterRegistry, "filterRegistry"); - this.metricRecorder = metricRecorder; this.nameResolverArgs = checkNotNull(nameResolverArgs, "nameResolverArgs"); randomChannelId = random.nextLong(); @@ -196,13 +203,12 @@ public String getServiceAuthority() { public void start(Listener2 listener) { this.listener = checkNotNull(listener, "listener"); try { - xdsClientPool = xdsClientPoolFactory.getOrCreate(target, metricRecorder); + xdsClient = xdsClientPool.getObject(); } catch (Exception e) { listener.onError( Status.UNAVAILABLE.withDescription("Failed to initialize xDS").withCause(e)); return; } - xdsClient = xdsClientPool.getObject(); BootstrapInfo bootstrapInfo = xdsClient.getBootstrapInfo(); String listenerNameTemplate; if (targetAuthority == null) { @@ -319,7 +325,7 @@ private void updateResolutionResult(XdsConfig xdsConfig) { ConfigOrError parsedServiceConfig = serviceConfigParser.parseServiceConfig(rawServiceConfig); Attributes attrs = Attributes.newBuilder() - .set(XdsAttributes.XDS_CLIENT_POOL, xdsClientPool) + .set(XdsAttributes.XDS_CLIENT, xdsClient) .set(XdsAttributes.XDS_CONFIG, xdsConfig) .set(XdsAttributes.XDS_CLUSTER_SUBSCRIPT_REGISTRY, resolveState.xdsDependencyManager) .set(XdsAttributes.CALL_COUNTER_PROVIDER, callCounterProvider) @@ -1034,4 +1040,72 @@ static ClusterRefState forRlsPlugin( return new ClusterRefState(refCount, null, rlsPluginConfig, null); } } + + /** An ObjectPool, except it can throw an exception. */ + private interface XdsClientPool { + XdsClient getObject() throws XdsInitializationException; + + XdsClient returnObject(XdsClient xdsClient); + } + + private static final class BootstrappingXdsClientPool implements XdsClientPool { + private final XdsClientPoolFactory xdsClientPoolFactory; + private final String target; + private final @Nullable Map bootstrapOverride; + private final @Nullable MetricRecorder metricRecorder; + private ObjectPool xdsClientPool; + + BootstrappingXdsClientPool( + XdsClientPoolFactory xdsClientPoolFactory, + String target, + @Nullable Map bootstrapOverride, + @Nullable MetricRecorder metricRecorder) { + this.xdsClientPoolFactory = checkNotNull(xdsClientPoolFactory, "xdsClientPoolFactory"); + this.target = checkNotNull(target, "target"); + this.bootstrapOverride = bootstrapOverride; + this.metricRecorder = metricRecorder; + } + + @Override + public XdsClient getObject() throws XdsInitializationException { + if (xdsClientPool == null) { + BootstrapInfo bootstrapInfo; + if (bootstrapOverride == null) { + bootstrapInfo = GrpcBootstrapperImpl.defaultBootstrap(); + } else { + bootstrapInfo = new GrpcBootstrapperImpl().bootstrap(bootstrapOverride); + } + this.xdsClientPool = + xdsClientPoolFactory.getOrCreate(target, bootstrapInfo, metricRecorder); + } + return xdsClientPool.getObject(); + } + + @Override + public XdsClient returnObject(XdsClient xdsClient) { + return xdsClientPool.returnObject(xdsClient); + } + } + + private static final class SupplierXdsClientPool implements XdsClientPool { + private final Supplier xdsClientSupplier; + + SupplierXdsClientPool(Supplier xdsClientSupplier) { + this.xdsClientSupplier = checkNotNull(xdsClientSupplier, "xdsClientSupplier"); + } + + @Override + public XdsClient getObject() throws XdsInitializationException { + XdsClient xdsClient = xdsClientSupplier.get(); + if (xdsClient == null) { + throw new XdsInitializationException("Caller failed to initialize XDS_CLIENT_SUPPLIER"); + } + return xdsClient; + } + + @Override + public XdsClient returnObject(XdsClient xdsClient) { + return null; + } + } } diff --git a/xds/src/main/java/io/grpc/xds/XdsNameResolverProvider.java b/xds/src/main/java/io/grpc/xds/XdsNameResolverProvider.java index eb3887396a0..e3462276b17 100644 --- a/xds/src/main/java/io/grpc/xds/XdsNameResolverProvider.java +++ b/xds/src/main/java/io/grpc/xds/XdsNameResolverProvider.java @@ -22,6 +22,7 @@ import io.grpc.Internal; import io.grpc.NameResolver.Args; import io.grpc.NameResolverProvider; +import io.grpc.xds.client.XdsClient; import java.net.InetSocketAddress; import java.net.SocketAddress; import java.net.URI; @@ -29,6 +30,7 @@ import java.util.Collections; import java.util.Map; import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Supplier; import javax.annotation.Nullable; /** @@ -43,6 +45,13 @@ */ @Internal public final class XdsNameResolverProvider extends NameResolverProvider { + /** + * If provided, the suppler must return non-null when lb.start() is called (which implies not + * throwing), and the XdsClient must remain alive until lb.shutdown() returns. It may only be + * called from the synchronization context. + */ + public static final Args.Key> XDS_CLIENT_SUPPLIER = + Args.Key.create("io.grpc.xds.XdsNameResolverProvider.XDS_CLIENT_SUPPLIER"); private static final String SCHEME = "xds"; private final String scheme; diff --git a/xds/src/main/java/io/grpc/xds/XdsServerBuilder.java b/xds/src/main/java/io/grpc/xds/XdsServerBuilder.java index 928f22c4d5e..4a4fb71aa84 100644 --- a/xds/src/main/java/io/grpc/xds/XdsServerBuilder.java +++ b/xds/src/main/java/io/grpc/xds/XdsServerBuilder.java @@ -55,6 +55,7 @@ public final class XdsServerBuilder extends ForwardingServerBuilder bootstrapOverride; private long drainGraceTime = 10; private TimeUnit drainGraceTimeUnit = TimeUnit.MINUTES; @@ -127,7 +128,7 @@ public Server build() { } InternalNettyServerBuilder.eagAttributes(delegate, builder.build()); return new XdsServerWrapper("0.0.0.0:" + port, delegate, xdsServingStatusListener, - filterChainSelectorManager, xdsClientPoolFactory, filterRegistry); + filterChainSelectorManager, xdsClientPoolFactory, bootstrapOverride, filterRegistry); } @VisibleForTesting @@ -140,11 +141,10 @@ XdsServerBuilder xdsClientPoolFactory(XdsClientPoolFactory xdsClientPoolFactory) * Allows providing bootstrap override, useful for testing. */ public XdsServerBuilder overrideBootstrapForTest(Map bootstrapOverride) { - checkNotNull(bootstrapOverride, "bootstrapOverride"); + this.bootstrapOverride = checkNotNull(bootstrapOverride, "bootstrapOverride"); if (this.xdsClientPoolFactory == SharedXdsClientPoolProvider.getDefaultProvider()) { this.xdsClientPoolFactory = new SharedXdsClientPoolProvider(); } - this.xdsClientPoolFactory.setBootstrapOverride(bootstrapOverride); return this; } diff --git a/xds/src/main/java/io/grpc/xds/XdsServerWrapper.java b/xds/src/main/java/io/grpc/xds/XdsServerWrapper.java index be64b56eef5..f54c4af5cff 100644 --- a/xds/src/main/java/io/grpc/xds/XdsServerWrapper.java +++ b/xds/src/main/java/io/grpc/xds/XdsServerWrapper.java @@ -56,6 +56,7 @@ import io.grpc.xds.XdsListenerResource.LdsUpdate; import io.grpc.xds.XdsRouteConfigureResource.RdsUpdate; import io.grpc.xds.XdsServerBuilder.XdsServingStatusListener; +import io.grpc.xds.client.Bootstrapper.BootstrapInfo; import io.grpc.xds.client.XdsClient; import io.grpc.xds.client.XdsClient.ResourceWatcher; import io.grpc.xds.internal.security.SslContextProviderSupplier; @@ -103,6 +104,7 @@ public void uncaughtException(Thread t, Throwable e) { private final FilterRegistry filterRegistry; private final ThreadSafeRandom random = ThreadSafeRandomImpl.instance; private final XdsClientPoolFactory xdsClientPoolFactory; + private final @Nullable Map bootstrapOverride; private final XdsServingStatusListener listener; private final FilterChainSelectorManager filterChainSelectorManager; private final AtomicBoolean started = new AtomicBoolean(false); @@ -131,9 +133,17 @@ public void uncaughtException(Thread t, Throwable e) { XdsServingStatusListener listener, FilterChainSelectorManager filterChainSelectorManager, XdsClientPoolFactory xdsClientPoolFactory, + @Nullable Map bootstrapOverride, FilterRegistry filterRegistry) { - this(listenerAddress, delegateBuilder, listener, filterChainSelectorManager, - xdsClientPoolFactory, filterRegistry, SharedResourceHolder.get(GrpcUtil.TIMER_SERVICE)); + this( + listenerAddress, + delegateBuilder, + listener, + filterChainSelectorManager, + xdsClientPoolFactory, + bootstrapOverride, + filterRegistry, + SharedResourceHolder.get(GrpcUtil.TIMER_SERVICE)); sharedTimeService = true; } @@ -144,6 +154,7 @@ public void uncaughtException(Thread t, Throwable e) { XdsServingStatusListener listener, FilterChainSelectorManager filterChainSelectorManager, XdsClientPoolFactory xdsClientPoolFactory, + @Nullable Map bootstrapOverride, FilterRegistry filterRegistry, ScheduledExecutorService timeService) { this.listenerAddress = checkNotNull(listenerAddress, "listenerAddress"); @@ -153,6 +164,7 @@ public void uncaughtException(Thread t, Throwable e) { this.filterChainSelectorManager = checkNotNull(filterChainSelectorManager, "filterChainSelectorManager"); this.xdsClientPoolFactory = checkNotNull(xdsClientPoolFactory, "xdsClientPoolFactory"); + this.bootstrapOverride = bootstrapOverride; this.timeService = checkNotNull(timeService, "timeService"); this.filterRegistry = checkNotNull(filterRegistry,"filterRegistry"); this.delegate = delegateBuilder.build(); @@ -182,7 +194,14 @@ public void run() { private void internalStart() { try { - xdsClientPool = xdsClientPoolFactory.getOrCreate("#server", new MetricRecorder() {}); + BootstrapInfo bootstrapInfo; + if (bootstrapOverride == null) { + bootstrapInfo = GrpcBootstrapperImpl.defaultBootstrap(); + } else { + bootstrapInfo = new GrpcBootstrapperImpl().bootstrap(bootstrapOverride); + } + xdsClientPool = xdsClientPoolFactory.getOrCreate( + "#server", bootstrapInfo, new MetricRecorder() {}); } catch (Exception e) { StatusException statusException = Status.UNAVAILABLE.withDescription( "Failed to initialize xDS").withCause(e).asException(); diff --git a/xds/src/test/java/io/grpc/xds/ClusterImplLoadBalancerTest.java b/xds/src/test/java/io/grpc/xds/ClusterImplLoadBalancerTest.java index 50138cb8e5a..af618beda6a 100644 --- a/xds/src/test/java/io/grpc/xds/ClusterImplLoadBalancerTest.java +++ b/xds/src/test/java/io/grpc/xds/ClusterImplLoadBalancerTest.java @@ -55,7 +55,6 @@ import io.grpc.Status.Code; import io.grpc.SynchronizationContext; import io.grpc.internal.FakeClock; -import io.grpc.internal.ObjectPool; import io.grpc.internal.PickFirstLoadBalancerProvider; import io.grpc.internal.PickSubchannelArgsImpl; import io.grpc.protobuf.ProtoUtils; @@ -140,19 +139,6 @@ public void uncaughtException(Thread t, Throwable e) { private final LoadStatsManager2 loadStatsManager = new LoadStatsManager2(fakeClock.getStopwatchSupplier()); private final FakeXdsClient xdsClient = new FakeXdsClient(); - private final ObjectPool xdsClientPool = new ObjectPool() { - @Override - public XdsClient getObject() { - xdsClientRefs++; - return xdsClient; - } - - @Override - public XdsClient returnObject(Object object) { - xdsClientRefs--; - return null; - } - }; private final CallCounterProvider callCounterProvider = new CallCounterProvider() { @Override public AtomicLong getOrCreate(String cluster, @Nullable String edsServiceName) { @@ -199,8 +185,8 @@ public void handleResolvedAddresses_propagateToChildPolicy() { FakeLoadBalancer childBalancer = Iterables.getOnlyElement(downstreamBalancers); assertThat(Iterables.getOnlyElement(childBalancer.addresses)).isEqualTo(endpoint); assertThat(childBalancer.config).isSameInstanceAs(weightedTargetConfig); - assertThat(childBalancer.attributes.get(io.grpc.xds.XdsAttributes.XDS_CLIENT_POOL)) - .isSameInstanceAs(xdsClientPool); + assertThat(childBalancer.attributes.get(io.grpc.xds.XdsAttributes.XDS_CLIENT)) + .isSameInstanceAs(xdsClient); assertThat(childBalancer.attributes.get(NameResolver.ATTR_BACKEND_SERVICE)).isEqualTo(CLUSTER); } @@ -673,7 +659,7 @@ public void dropRpcsWithRespectToLbConfigDropCategories() { .setAddresses(Collections.singletonList(endpoint)) .setAttributes( Attributes.newBuilder() - .set(io.grpc.xds.XdsAttributes.XDS_CLIENT_POOL, xdsClientPool) + .set(io.grpc.xds.XdsAttributes.XDS_CLIENT, xdsClient) .build()) .setLoadBalancingPolicyConfig(config) .build()); @@ -1070,7 +1056,7 @@ private void deliverAddressesAndConfig(List addresses, .setAddresses(addresses) .setAttributes( Attributes.newBuilder() - .set(io.grpc.xds.XdsAttributes.XDS_CLIENT_POOL, xdsClientPool) + .set(io.grpc.xds.XdsAttributes.XDS_CLIENT, xdsClient) .set(io.grpc.xds.XdsAttributes.CALL_COUNTER_PROVIDER, callCounterProvider) .build()) .setLoadBalancingPolicyConfig(config) diff --git a/xds/src/test/java/io/grpc/xds/CsdsServiceTest.java b/xds/src/test/java/io/grpc/xds/CsdsServiceTest.java index 573aef7ca1e..e8bd7461736 100644 --- a/xds/src/test/java/io/grpc/xds/CsdsServiceTest.java +++ b/xds/src/test/java/io/grpc/xds/CsdsServiceTest.java @@ -513,13 +513,8 @@ public List getTargets() { } @Override - public void setBootstrapOverride(Map bootstrap) { - throw new UnsupportedOperationException("Should not be called"); - } - - - @Override - public ObjectPool getOrCreate(String target, MetricRecorder metricRecorder) { + public ObjectPool getOrCreate( + String target, BootstrapInfo bootstrapInfo, MetricRecorder metricRecorder) { throw new UnsupportedOperationException("Should not be called"); } } diff --git a/xds/src/test/java/io/grpc/xds/GrpcBootstrapperImplTest.java b/xds/src/test/java/io/grpc/xds/GrpcBootstrapperImplTest.java index 3f93cc6f191..2b7bd53d5ef 100644 --- a/xds/src/test/java/io/grpc/xds/GrpcBootstrapperImplTest.java +++ b/xds/src/test/java/io/grpc/xds/GrpcBootstrapperImplTest.java @@ -84,6 +84,19 @@ public void restoreEnvironment() { CommonBootstrapperTestUtils.setEnableXdsFallback(originalExperimentalXdsFallbackFlag); } + @Test + public void parseBootstrap_emptyServers_throws() { + String rawData = "{\n" + + " \"xds_servers\": [\n" + + " ]\n" + + "}"; + + bootstrapper.setFileReader(createFileReader(BOOTSTRAP_FILE_PATH, rawData)); + XdsInitializationException e = Assert.assertThrows(XdsInitializationException.class, + bootstrapper::bootstrap); + assertThat(e).hasMessageThat().isEqualTo("Invalid bootstrap: 'xds_servers' is empty"); + } + @Test public void parseBootstrap_singleXdsServer() throws XdsInitializationException { String rawData = "{\n" @@ -352,7 +365,14 @@ public void parseBootstrap_serverWithoutServerUri() throws XdsInitializationExce public void parseBootstrap_certProviderInstances() throws XdsInitializationException { String rawData = "{\n" - + " \"xds_servers\": [],\n" + + " \"xds_servers\": [\n" + + " {\n" + + " \"server_uri\": \"" + SERVER_URI + "\",\n" + + " \"channel_creds\": [\n" + + " {\"type\": \"insecure\"}\n" + + " ]\n" + + " }\n" + + " ],\n" + " \"certificate_providers\": {\n" + " \"gcp_id\": {\n" + " \"plugin_name\": \"meshca\",\n" @@ -389,7 +409,6 @@ public void parseBootstrap_certProviderInstances() throws XdsInitializationExcep bootstrapper.setFileReader(createFileReader(BOOTSTRAP_FILE_PATH, rawData)); BootstrapInfo info = bootstrapper.bootstrap(); - assertThat(info.servers()).isEmpty(); assertThat(info.node()).isEqualTo(getNodeBuilder().build()); Map certProviders = info.certProviders(); assertThat(certProviders).isNotNull(); @@ -556,7 +575,14 @@ public void parseBootstrap_missingPluginName() { @Test public void parseBootstrap_grpcServerResourceId() throws XdsInitializationException { String rawData = "{\n" - + " \"xds_servers\": [],\n" + + " \"xds_servers\": [\n" + + " {\n" + + " \"server_uri\": \"" + SERVER_URI + "\",\n" + + " \"channel_creds\": [\n" + + " {\"type\": \"insecure\"}\n" + + " ]\n" + + " }\n" + + " ],\n" + " \"server_listener_resource_name_template\": \"grpc/serverx=%s\"\n" + "}"; @@ -779,6 +805,12 @@ public void fallbackToConfigFromSysProp() throws XdsInitializationException { public void parseClientDefaultListenerResourceNameTemplate() throws Exception { String rawData = "{\n" + " \"xds_servers\": [\n" + + " {\n" + + " \"server_uri\": \"" + SERVER_URI + "\",\n" + + " \"channel_creds\": [\n" + + " {\"type\": \"insecure\"}\n" + + " ]\n" + + " }\n" + " ]\n" + "}"; bootstrapper.setFileReader(createFileReader(BOOTSTRAP_FILE_PATH, rawData)); @@ -788,6 +820,12 @@ public void parseClientDefaultListenerResourceNameTemplate() throws Exception { rawData = "{\n" + " \"client_default_listener_resource_name_template\": \"xdstp://a.com/faketype/%s\",\n" + " \"xds_servers\": [\n" + + " {\n" + + " \"server_uri\": \"" + SERVER_URI + "\",\n" + + " \"channel_creds\": [\n" + + " {\"type\": \"insecure\"}\n" + + " ]\n" + + " }\n" + " ]\n" + "}"; bootstrapper.setFileReader(createFileReader(BOOTSTRAP_FILE_PATH, rawData)); diff --git a/xds/src/test/java/io/grpc/xds/SharedXdsClientPoolProviderTest.java b/xds/src/test/java/io/grpc/xds/SharedXdsClientPoolProviderTest.java index 24f1750d5a8..29b149f166f 100644 --- a/xds/src/test/java/io/grpc/xds/SharedXdsClientPoolProviderTest.java +++ b/xds/src/test/java/io/grpc/xds/SharedXdsClientPoolProviderTest.java @@ -19,7 +19,7 @@ import static com.google.common.truth.Truth.assertThat; import static io.grpc.Metadata.ASCII_STRING_MARSHALLER; -import static org.junit.Assert.assertThrows; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.when; @@ -74,16 +74,24 @@ public class SharedXdsClientPoolProviderTest { private GrpcBootstrapperImpl bootstrapper; @Mock private ResourceWatcher ldsResourceWatcher; + @Deprecated @Test - public void noServer() throws XdsInitializationException { + public void sharedXdsClientObjectPool_deprecated() throws XdsInitializationException { + ServerInfo server = ServerInfo.create(SERVER_URI, InsecureChannelCredentials.create()); BootstrapInfo bootstrapInfo = - BootstrapInfo.builder().servers(Collections.emptyList()).node(node).build(); + BootstrapInfo.builder().servers(Collections.singletonList(server)).node(node).build(); when(bootstrapper.bootstrap()).thenReturn(bootstrapInfo); + SharedXdsClientPoolProvider provider = new SharedXdsClientPoolProvider(bootstrapper); - XdsInitializationException e = assertThrows(XdsInitializationException.class, - () -> provider.getOrCreate(DUMMY_TARGET, metricRecorder)); - assertThat(e).hasMessageThat().isEqualTo("No xDS server provided"); assertThat(provider.get(DUMMY_TARGET)).isNull(); + ObjectPool xdsClientPool = + provider.getOrCreate(DUMMY_TARGET, metricRecorder, null); + verify(bootstrapper).bootstrap(); + assertThat(provider.getOrCreate(DUMMY_TARGET, bootstrapInfo, metricRecorder)) + .isSameInstanceAs(xdsClientPool); + assertThat(provider.get(DUMMY_TARGET)).isNotNull(); + assertThat(provider.get(DUMMY_TARGET)).isSameInstanceAs(xdsClientPool); + verifyNoMoreInteractions(bootstrapper); } @Test @@ -91,13 +99,14 @@ public void sharedXdsClientObjectPool() throws XdsInitializationException { ServerInfo server = ServerInfo.create(SERVER_URI, InsecureChannelCredentials.create()); BootstrapInfo bootstrapInfo = BootstrapInfo.builder().servers(Collections.singletonList(server)).node(node).build(); - when(bootstrapper.bootstrap()).thenReturn(bootstrapInfo); SharedXdsClientPoolProvider provider = new SharedXdsClientPoolProvider(bootstrapper); assertThat(provider.get(DUMMY_TARGET)).isNull(); - ObjectPool xdsClientPool = provider.getOrCreate(DUMMY_TARGET, metricRecorder); - verify(bootstrapper).bootstrap(); - assertThat(provider.getOrCreate(DUMMY_TARGET, metricRecorder)).isSameInstanceAs(xdsClientPool); + ObjectPool xdsClientPool = + provider.getOrCreate(DUMMY_TARGET, bootstrapInfo, metricRecorder); + verify(bootstrapper, never()).bootstrap(); + assertThat(provider.getOrCreate(DUMMY_TARGET, bootstrapInfo, metricRecorder)) + .isSameInstanceAs(xdsClientPool); assertThat(provider.get(DUMMY_TARGET)).isNotNull(); assertThat(provider.get(DUMMY_TARGET)).isSameInstanceAs(xdsClientPool); verifyNoMoreInteractions(bootstrapper); @@ -108,7 +117,7 @@ public void refCountedXdsClientObjectPool_delayedCreation() { ServerInfo server = ServerInfo.create(SERVER_URI, InsecureChannelCredentials.create()); BootstrapInfo bootstrapInfo = BootstrapInfo.builder().servers(Collections.singletonList(server)).node(node).build(); - SharedXdsClientPoolProvider provider = new SharedXdsClientPoolProvider(bootstrapper); + SharedXdsClientPoolProvider provider = new SharedXdsClientPoolProvider(); RefCountedXdsClientObjectPool xdsClientPool = provider.new RefCountedXdsClientObjectPool(bootstrapInfo, DUMMY_TARGET, metricRecorder); assertThat(xdsClientPool.getXdsClientForTest()).isNull(); @@ -122,7 +131,7 @@ public void refCountedXdsClientObjectPool_refCounted() { ServerInfo server = ServerInfo.create(SERVER_URI, InsecureChannelCredentials.create()); BootstrapInfo bootstrapInfo = BootstrapInfo.builder().servers(Collections.singletonList(server)).node(node).build(); - SharedXdsClientPoolProvider provider = new SharedXdsClientPoolProvider(bootstrapper); + SharedXdsClientPoolProvider provider = new SharedXdsClientPoolProvider(); RefCountedXdsClientObjectPool xdsClientPool = provider.new RefCountedXdsClientObjectPool(bootstrapInfo, DUMMY_TARGET, metricRecorder); // getObject once @@ -143,7 +152,7 @@ public void refCountedXdsClientObjectPool_getObjectCreatesNewInstanceIfAlreadySh ServerInfo server = ServerInfo.create(SERVER_URI, InsecureChannelCredentials.create()); BootstrapInfo bootstrapInfo = BootstrapInfo.builder().servers(Collections.singletonList(server)).node(node).build(); - SharedXdsClientPoolProvider provider = new SharedXdsClientPoolProvider(bootstrapper); + SharedXdsClientPoolProvider provider = new SharedXdsClientPoolProvider(); RefCountedXdsClientObjectPool xdsClientPool = provider.new RefCountedXdsClientObjectPool(bootstrapInfo, DUMMY_TARGET, metricRecorder); XdsClient xdsClient1 = xdsClientPool.getObject(); @@ -189,8 +198,7 @@ public void xdsClient_usesCallCredentials() throws Exception { ServerInfo server = ServerInfo.create(xdsServerUri, InsecureChannelCredentials.create()); BootstrapInfo bootstrapInfo = BootstrapInfo.builder().servers(Collections.singletonList(server)).node(node).build(); - when(bootstrapper.bootstrap()).thenReturn(bootstrapInfo); - SharedXdsClientPoolProvider provider = new SharedXdsClientPoolProvider(bootstrapper); + SharedXdsClientPoolProvider provider = new SharedXdsClientPoolProvider(); // Create custom xDS transport CallCredentials CallCredentials sampleCreds = @@ -199,7 +207,7 @@ public void xdsClient_usesCallCredentials() throws Exception { // Create xDS client that uses the CallCredentials on the transport ObjectPool xdsClientPool = - provider.getOrCreate("target", metricRecorder, sampleCreds); + provider.getOrCreate("target", bootstrapInfo, metricRecorder, sampleCreds); XdsClient xdsClient = xdsClientPool.getObject(); xdsClient.watchXdsResource( XdsListenerResource.getInstance(), "someLDSresource", ldsResourceWatcher); diff --git a/xds/src/test/java/io/grpc/xds/XdsClientFallbackTest.java b/xds/src/test/java/io/grpc/xds/XdsClientFallbackTest.java index 1e7ce6dc2a2..09b4c5d8637 100644 --- a/xds/src/test/java/io/grpc/xds/XdsClientFallbackTest.java +++ b/xds/src/test/java/io/grpc/xds/XdsClientFallbackTest.java @@ -178,8 +178,10 @@ public void setUp() throws XdsInitializationException { setAdsConfig(fallbackServer, FALLBACK_SERVER); SharedXdsClientPoolProvider clientPoolProvider = new SharedXdsClientPoolProvider(); - clientPoolProvider.setBootstrapOverride(defaultBootstrapOverride()); - xdsClientPool = clientPoolProvider.getOrCreate(DUMMY_TARGET, metricRecorder); + xdsClientPool = clientPoolProvider.getOrCreate( + DUMMY_TARGET, + new GrpcBootstrapperImpl().bootstrap(defaultBootstrapOverride()), + metricRecorder); } @After diff --git a/xds/src/test/java/io/grpc/xds/XdsClientFederationTest.java b/xds/src/test/java/io/grpc/xds/XdsClientFederationTest.java index ee32c4490af..24ef60c4685 100644 --- a/xds/src/test/java/io/grpc/xds/XdsClientFederationTest.java +++ b/xds/src/test/java/io/grpc/xds/XdsClientFederationTest.java @@ -79,8 +79,10 @@ public class XdsClientFederationTest { @Before public void setUp() throws XdsInitializationException { SharedXdsClientPoolProvider clientPoolProvider = new SharedXdsClientPoolProvider(); - clientPoolProvider.setBootstrapOverride(defaultBootstrapOverride()); - xdsClientPool = clientPoolProvider.getOrCreate(DUMMY_TARGET, metricRecorder); + xdsClientPool = clientPoolProvider.getOrCreate( + DUMMY_TARGET, + new GrpcBootstrapperImpl().bootstrap(defaultBootstrapOverride()), + metricRecorder); xdsClient = xdsClientPool.getObject(); } diff --git a/xds/src/test/java/io/grpc/xds/XdsClientWrapperForServerSdsTestMisc.java b/xds/src/test/java/io/grpc/xds/XdsClientWrapperForServerSdsTestMisc.java index f997f583898..53dcd15dd3b 100644 --- a/xds/src/test/java/io/grpc/xds/XdsClientWrapperForServerSdsTestMisc.java +++ b/xds/src/test/java/io/grpc/xds/XdsClientWrapperForServerSdsTestMisc.java @@ -120,7 +120,8 @@ public void setUp() { when(mockBuilder.build()).thenReturn(mockServer); when(mockServer.isShutdown()).thenReturn(false); xdsServerWrapper = new XdsServerWrapper("0.0.0.0:" + PORT, mockBuilder, listener, - selectorManager, new FakeXdsClientPoolFactory(xdsClient), FilterRegistry.newRegistry()); + selectorManager, new FakeXdsClientPoolFactory(xdsClient), + XdsServerTestHelper.RAW_BOOTSTRAP, FilterRegistry.newRegistry()); } @Test diff --git a/xds/src/test/java/io/grpc/xds/XdsNameResolverTest.java b/xds/src/test/java/io/grpc/xds/XdsNameResolverTest.java index 6edd98f923e..d090687a072 100644 --- a/xds/src/test/java/io/grpc/xds/XdsNameResolverTest.java +++ b/xds/src/test/java/io/grpc/xds/XdsNameResolverTest.java @@ -101,7 +101,6 @@ import io.grpc.xds.client.Bootstrapper.ServerInfo; import io.grpc.xds.client.EnvoyProtoData.Node; import io.grpc.xds.client.XdsClient; -import io.grpc.xds.client.XdsInitializationException; import io.grpc.xds.client.XdsResourceType; import java.io.IOException; import java.net.URI; @@ -174,6 +173,15 @@ public ConfigOrError parseServiceConfig(Map rawServiceConfig) { private final CallInfo call2 = new CallInfo("GreetService", "bye"); private final TestChannel channel = new TestChannel(); private final MetricRecorder metricRecorder = new MetricRecorder() {}; + private final Map rawBootstrap = ImmutableMap.of( + "xds_servers", ImmutableList.of( + ImmutableMap.of( + "server_uri", "td.googleapis.com", + "channel_creds", ImmutableList.of( + ImmutableMap.of( + "type", "insecure"))) + )); + private BootstrapInfo bootstrapInfo = BootstrapInfo.builder() .servers(ImmutableList.of(ServerInfo.create( "td.googleapis.com", InsecureChannelCredentials.create()))) @@ -230,7 +238,8 @@ public void setUp() { resolver = new XdsNameResolver(targetUri, null, AUTHORITY, null, serviceConfigParser, syncContext, scheduler, - xdsClientPoolFactory, mockRandom, filterRegistry, null, metricRecorder, nameResolverArgs); + xdsClientPoolFactory, mockRandom, filterRegistry, rawBootstrap, metricRecorder, + nameResolverArgs); } @After @@ -250,46 +259,23 @@ public void tearDown() { @Test public void resolving_failToCreateXdsClientPool() { - XdsClientPoolFactory xdsClientPoolFactory = new XdsClientPoolFactory() { - @Override - public void setBootstrapOverride(Map bootstrap) { - } - - @Override - @Nullable - public ObjectPool get(String target) { - throw new UnsupportedOperationException("Should not be called"); - } - - @Override - public ObjectPool getOrCreate(String target, MetricRecorder metricRecorder) - throws XdsInitializationException { - throw new XdsInitializationException("Fail to read bootstrap file"); - } - - @Override - public List getTargets() { - return null; - } - }; - resolver = new XdsNameResolver(targetUri, null, AUTHORITY, null, serviceConfigParser, syncContext, scheduler, - xdsClientPoolFactory, mockRandom, FilterRegistry.getDefaultRegistry(), null, - metricRecorder, nameResolverArgs); + xdsClientPoolFactory, mockRandom, FilterRegistry.getDefaultRegistry(), + Collections.emptyMap(), metricRecorder, nameResolverArgs); resolver.start(mockListener); verify(mockListener).onError(errorCaptor.capture()); Status error = errorCaptor.getValue(); assertThat(error.getCode()).isEqualTo(Code.UNAVAILABLE); assertThat(error.getDescription()).isEqualTo("Failed to initialize xDS"); - assertThat(error.getCause()).hasMessageThat().isEqualTo("Fail to read bootstrap file"); + assertThat(error.getCause()).hasMessageThat().contains("Invalid bootstrap"); } @Test public void resolving_withTargetAuthorityNotFound() { resolver = new XdsNameResolver(targetUri, "notfound.google.com", AUTHORITY, null, serviceConfigParser, syncContext, scheduler, - xdsClientPoolFactory, mockRandom, FilterRegistry.getDefaultRegistry(), null, + xdsClientPoolFactory, mockRandom, FilterRegistry.getDefaultRegistry(), rawBootstrap, metricRecorder, nameResolverArgs); resolver.start(mockListener); verify(mockListener).onError(errorCaptor.capture()); @@ -312,7 +298,8 @@ public void resolving_noTargetAuthority_templateWithoutXdstp() { resolver = new XdsNameResolver( targetUri, null, serviceAuthority, null, serviceConfigParser, syncContext, scheduler, xdsClientPoolFactory, - mockRandom, FilterRegistry.getDefaultRegistry(), null, metricRecorder, nameResolverArgs); + mockRandom, FilterRegistry.getDefaultRegistry(), rawBootstrap, metricRecorder, + nameResolverArgs); resolver.start(mockListener); verify(mockListener, never()).onError(any(Status.class)); } @@ -332,7 +319,7 @@ public void resolving_noTargetAuthority_templateWithXdstp() { + "%5B::FFFF:129.144.52.38%5D:80?id=1"; resolver = new XdsNameResolver( targetUri, null, serviceAuthority, null, serviceConfigParser, syncContext, scheduler, - xdsClientPoolFactory, mockRandom, FilterRegistry.getDefaultRegistry(), null, + xdsClientPoolFactory, mockRandom, FilterRegistry.getDefaultRegistry(), rawBootstrap, metricRecorder, nameResolverArgs); resolver.start(mockListener); verify(mockListener, never()).onError(any(Status.class)); @@ -353,7 +340,7 @@ public void resolving_noTargetAuthority_xdstpWithMultipleSlashes() { + "path/to/service?id=1"; resolver = new XdsNameResolver( targetUri, null, serviceAuthority, null, serviceConfigParser, syncContext, scheduler, - xdsClientPoolFactory, mockRandom, FilterRegistry.getDefaultRegistry(), null, + xdsClientPoolFactory, mockRandom, FilterRegistry.getDefaultRegistry(), rawBootstrap, metricRecorder, nameResolverArgs); @@ -382,7 +369,7 @@ public void resolving_targetAuthorityInAuthoritiesMap() { + "%5B::FFFF:129.144.52.38%5D:80?bar=2&foo=1"; // query param canonified resolver = new XdsNameResolver(targetUri, "xds.authority.com", serviceAuthority, null, serviceConfigParser, syncContext, scheduler, - xdsClientPoolFactory, mockRandom, FilterRegistry.getDefaultRegistry(), null, + xdsClientPoolFactory, mockRandom, FilterRegistry.getDefaultRegistry(), rawBootstrap, metricRecorder, nameResolverArgs); resolver.start(mockListener); verify(mockListener, never()).onError(any(Status.class)); @@ -415,7 +402,7 @@ public void resolving_ldsResourceUpdateRdsName() { .build(); resolver = new XdsNameResolver(targetUri, null, AUTHORITY, null, serviceConfigParser, syncContext, scheduler, - xdsClientPoolFactory, mockRandom, FilterRegistry.getDefaultRegistry(), null, + xdsClientPoolFactory, mockRandom, FilterRegistry.getDefaultRegistry(), rawBootstrap, metricRecorder, nameResolverArgs); // use different ldsResourceName and service authority. The virtualhost lookup should use // service authority. @@ -617,7 +604,7 @@ public void resolving_matchingVirtualHostNotFound_matchingOverrideAuthority() { resolver = new XdsNameResolver(targetUri, null, AUTHORITY, "random", serviceConfigParser, syncContext, scheduler, - xdsClientPoolFactory, mockRandom, FilterRegistry.getDefaultRegistry(), null, + xdsClientPoolFactory, mockRandom, FilterRegistry.getDefaultRegistry(), rawBootstrap, metricRecorder, nameResolverArgs); resolver.start(mockListener); FakeXdsClient xdsClient = (FakeXdsClient) resolver.getXdsClient(); @@ -642,7 +629,7 @@ public void resolving_matchingVirtualHostNotFound_notMatchingOverrideAuthority() resolver = new XdsNameResolver(targetUri, null, AUTHORITY, "random", serviceConfigParser, syncContext, scheduler, - xdsClientPoolFactory, mockRandom, FilterRegistry.getDefaultRegistry(), null, + xdsClientPoolFactory, mockRandom, FilterRegistry.getDefaultRegistry(), rawBootstrap, metricRecorder, nameResolverArgs); resolver.start(mockListener); FakeXdsClient xdsClient = (FakeXdsClient) resolver.getXdsClient(); @@ -655,7 +642,7 @@ public void resolving_matchingVirtualHostNotFound_notMatchingOverrideAuthority() public void resolving_matchingVirtualHostNotFoundForOverrideAuthority() { resolver = new XdsNameResolver(targetUri, null, AUTHORITY, AUTHORITY, serviceConfigParser, syncContext, scheduler, - xdsClientPoolFactory, mockRandom, FilterRegistry.getDefaultRegistry(), null, + xdsClientPoolFactory, mockRandom, FilterRegistry.getDefaultRegistry(), rawBootstrap, metricRecorder, nameResolverArgs); resolver.start(mockListener); FakeXdsClient xdsClient = (FakeXdsClient) resolver.getXdsClient(); @@ -740,8 +727,8 @@ public void retryPolicyInPerMethodConfigGeneratedByResolverIsValid() { ServiceConfigParser realParser = new ScParser( true, 5, 5, new AutoConfiguredLoadBalancerFactory("pick-first")); resolver = new XdsNameResolver(targetUri, null, AUTHORITY, null, realParser, syncContext, - scheduler, xdsClientPoolFactory, mockRandom, FilterRegistry.getDefaultRegistry(), null, - metricRecorder, nameResolverArgs); + scheduler, xdsClientPoolFactory, mockRandom, FilterRegistry.getDefaultRegistry(), + rawBootstrap, metricRecorder, nameResolverArgs); resolver.start(mockListener); FakeXdsClient xdsClient = (FakeXdsClient) resolver.getXdsClient(); RetryPolicy retryPolicy = RetryPolicy.create( @@ -822,7 +809,7 @@ public void resolved_simpleCallFailedToRoute_routeWithNonForwardingAction() { assertServiceConfigForLoadBalancingConfig( Collections.singletonList(cluster2), (Map) result.getServiceConfig().getConfig()); - assertThat(result.getAttributes().get(XdsAttributes.XDS_CLIENT_POOL)).isNotNull(); + assertThat(result.getAttributes().get(XdsAttributes.XDS_CLIENT)).isNotNull(); assertThat(result.getAttributes().get(XdsAttributes.CALL_COUNTER_PROVIDER)).isNotNull(); InternalConfigSelector configSelector = result.getAttributes().get(InternalConfigSelector.KEY); // Simulates making a call1 RPC. @@ -952,7 +939,7 @@ public void resolved_rpcHashingByChannelId() { when(mockRandom.nextLong()).thenReturn(123L); resolver = new XdsNameResolver(targetUri, null, AUTHORITY, null, serviceConfigParser, syncContext, scheduler, - xdsClientPoolFactory, mockRandom, FilterRegistry.getDefaultRegistry(), null, + xdsClientPoolFactory, mockRandom, FilterRegistry.getDefaultRegistry(), rawBootstrap, metricRecorder, nameResolverArgs); resolver.start(mockListener); xdsClient = (FakeXdsClient) resolver.getXdsClient(); @@ -986,7 +973,7 @@ public void resolved_rpcHashingByChannelId() { public void resolved_routeActionHasAutoHostRewrite_emitsCallOptionForTheSame() { resolver = new XdsNameResolver(targetUri, null, AUTHORITY, null, serviceConfigParser, syncContext, scheduler, xdsClientPoolFactory, mockRandom, - FilterRegistry.getDefaultRegistry(), null, metricRecorder, nameResolverArgs); + FilterRegistry.getDefaultRegistry(), rawBootstrap, metricRecorder, nameResolverArgs); resolver.start(mockListener); FakeXdsClient xdsClient = (FakeXdsClient) resolver.getXdsClient(); xdsClient.deliverLdsUpdate( @@ -1017,7 +1004,7 @@ public void resolved_routeActionHasAutoHostRewrite_emitsCallOptionForTheSame() { public void resolved_routeActionNoAutoHostRewrite_doesntEmitCallOptionForTheSame() { resolver = new XdsNameResolver(targetUri, null, AUTHORITY, null, serviceConfigParser, syncContext, scheduler, xdsClientPoolFactory, mockRandom, - FilterRegistry.getDefaultRegistry(), null, metricRecorder, nameResolverArgs); + FilterRegistry.getDefaultRegistry(), rawBootstrap, metricRecorder, nameResolverArgs); resolver.start(mockListener); FakeXdsClient xdsClient = (FakeXdsClient) resolver.getXdsClient(); xdsClient.deliverLdsUpdate( @@ -1235,7 +1222,7 @@ public void resolved_simpleCallSucceeds_routeToWeightedCluster() { assertThat(result.getAddressesOrError().getValue()).isEmpty(); assertServiceConfigForLoadBalancingConfig( Arrays.asList(cluster1, cluster2), (Map) result.getServiceConfig().getConfig()); - assertThat(result.getAttributes().get(XdsAttributes.XDS_CLIENT_POOL)).isNotNull(); + assertThat(result.getAttributes().get(XdsAttributes.XDS_CLIENT)).isNotNull(); InternalConfigSelector configSelector = result.getAttributes().get(InternalConfigSelector.KEY); assertCallSelectClusterResult(call1, configSelector, cluster2, 20.0); assertCallSelectClusterResult(call1, configSelector, cluster1, 20.0); @@ -1300,7 +1287,7 @@ public void resolved_simpleCallSucceeds_routeToRls() { ImmutableList.of(ImmutableMap.of("rls_experimental", expectedRlsLbConfig))))); assertThat(clusterManagerLbConfig).isEqualTo(expectedClusterManagerLbConfig); - assertThat(result.getAttributes().get(XdsAttributes.XDS_CLIENT_POOL)).isNotNull(); + assertThat(result.getAttributes().get(XdsAttributes.XDS_CLIENT)).isNotNull(); InternalConfigSelector configSelector = result.getAttributes().get(InternalConfigSelector.KEY); assertCallSelectRlsPluginResult( call1, configSelector, "rls-plugin-foo", 20.0); @@ -1489,7 +1476,7 @@ public void filterState_specialCase_sameNameDifferentTypeUrl() { FilterRegistry filterRegistry = FilterRegistry.newRegistry() .register(statefulFilterProvider, altStatefulFilterProvider, ROUTER_FILTER_PROVIDER); resolver = new XdsNameResolver(targetUri, null, AUTHORITY, null, serviceConfigParser, - syncContext, scheduler, xdsClientPoolFactory, mockRandom, filterRegistry, null, + syncContext, scheduler, xdsClientPoolFactory, mockRandom, filterRegistry, rawBootstrap, metricRecorder, nameResolverArgs); resolver.start(mockListener); @@ -1627,7 +1614,7 @@ private StatefulFilter.Provider filterStateTestSetupResolver() { FilterRegistry filterRegistry = FilterRegistry.newRegistry() .register(statefulFilterProvider, ROUTER_FILTER_PROVIDER); resolver = new XdsNameResolver(targetUri, null, AUTHORITY, null, serviceConfigParser, - syncContext, scheduler, xdsClientPoolFactory, mockRandom, filterRegistry, null, + syncContext, scheduler, xdsClientPoolFactory, mockRandom, filterRegistry, rawBootstrap, metricRecorder, nameResolverArgs); resolver.start(mockListener); return statefulFilterProvider; @@ -1761,7 +1748,7 @@ private InternalConfigSelector resolveToClusters() { assertThat(result.getAddressesOrError().getValue()).isEmpty(); assertServiceConfigForLoadBalancingConfig( Arrays.asList(cluster1, cluster2), (Map) result.getServiceConfig().getConfig()); - assertThat(result.getAttributes().get(XdsAttributes.XDS_CLIENT_POOL)).isNotNull(); + assertThat(result.getAttributes().get(XdsAttributes.XDS_CLIENT)).isNotNull(); assertThat(result.getAttributes().get(XdsAttributes.CALL_COUNTER_PROVIDER)).isNotNull(); return result.getAttributes().get(InternalConfigSelector.KEY); } @@ -2423,9 +2410,6 @@ private final class FakeXdsClientPoolFactory implements XdsClientPoolFactory { Set targets = new HashSet<>(); XdsClient xdsClient = new FakeXdsClient(); - @Override - public void setBootstrapOverride(Map bootstrap) {} - @Override @Nullable public ObjectPool get(String target) { @@ -2433,8 +2417,8 @@ public ObjectPool get(String target) { } @Override - public ObjectPool getOrCreate(String target, MetricRecorder metricRecorder) - throws XdsInitializationException { + public ObjectPool getOrCreate( + String target, BootstrapInfo bootstrapInfo, MetricRecorder metricRecorder) { targets.add(target); return new ObjectPool() { @Override diff --git a/xds/src/test/java/io/grpc/xds/XdsSecurityClientServerTest.java b/xds/src/test/java/io/grpc/xds/XdsSecurityClientServerTest.java index 6e4d243e904..9141147702f 100644 --- a/xds/src/test/java/io/grpc/xds/XdsSecurityClientServerTest.java +++ b/xds/src/test/java/io/grpc/xds/XdsSecurityClientServerTest.java @@ -753,6 +753,7 @@ private void buildServerWithFallbackServerCredentials( ServerCredentials xdsCredentials = XdsServerCredentials.create(fallbackCredentials); XdsServerBuilder builder = XdsServerBuilder.forPort(0, xdsCredentials) .xdsClientPoolFactory(fakePoolFactory) + .overrideBootstrapForTest(XdsServerTestHelper.RAW_BOOTSTRAP) .addService(new SimpleServiceImpl()); buildServer(builder, downstreamTlsContext); } @@ -872,7 +873,18 @@ public void run() { } } }); - xdsClient.ldsResource.get(8000, TimeUnit.MILLISECONDS); + try { + xdsClient.ldsResource.get(8000, TimeUnit.MILLISECONDS); + } catch (Exception ex) { + // start() probably failed, so throw its exception + if (settableFuture.isDone()) { + Throwable t = settableFuture.get(); + if (t != null) { + throw new Exception(t); + } + } + throw ex; + } return settableFuture; } diff --git a/xds/src/test/java/io/grpc/xds/XdsServerBuilderTest.java b/xds/src/test/java/io/grpc/xds/XdsServerBuilderTest.java index 2c168c65869..40225d3860d 100644 --- a/xds/src/test/java/io/grpc/xds/XdsServerBuilderTest.java +++ b/xds/src/test/java/io/grpc/xds/XdsServerBuilderTest.java @@ -43,7 +43,6 @@ import java.net.InetSocketAddress; import java.net.ServerSocket; import java.net.SocketAddress; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.ExecutionException; @@ -84,6 +83,7 @@ private void buildBuilder(XdsServerBuilder.XdsServingStatusListener xdsServingSt XdsServerBuilder.forPort( port, XdsServerCredentials.create(InsecureServerCredentials.create())); builder.xdsClientPoolFactory(xdsClientPoolFactory); + builder.overrideBootstrapForTest(XdsServerTestHelper.RAW_BOOTSTRAP); if (xdsServingStatusListener != null) { builder.xdsServingStatusListener(xdsServingStatusListener); } @@ -138,7 +138,18 @@ public void run() { } } }); - xdsClient.ldsResource.get(5000, TimeUnit.MILLISECONDS); + try { + xdsClient.ldsResource.get(5000, TimeUnit.MILLISECONDS); + } catch (TimeoutException ex) { + // start() probably failed, so throw its exception + if (settableFuture.isDone()) { + Throwable t = settableFuture.get(); + if (t != null) { + throw new ExecutionException(t); + } + } + throw ex; + } return settableFuture; } @@ -304,9 +315,12 @@ public void drainGraceTime_negativeThrows() throws IOException { @Test public void testOverrideBootstrap() throws Exception { - Map b = new HashMap<>(); + Map b = XdsServerTestHelper.RAW_BOOTSTRAP; buildBuilder(null); builder.overrideBootstrapForTest(b); - assertThat(xdsClientPoolFactory.savedBootstrap).isEqualTo(b); + xdsServer = cleanupRule.register((XdsServerWrapper) builder.build()); + Future unused = startServerAsync(); + assertThat(xdsClientPoolFactory.savedBootstrapInfo.node().getId()) + .isEqualTo(XdsServerTestHelper.BOOTSTRAP_INFO.node().getId()); } } diff --git a/xds/src/test/java/io/grpc/xds/XdsServerTestHelper.java b/xds/src/test/java/io/grpc/xds/XdsServerTestHelper.java index b0472b4729d..a9236ac77ca 100644 --- a/xds/src/test/java/io/grpc/xds/XdsServerTestHelper.java +++ b/xds/src/test/java/io/grpc/xds/XdsServerTestHelper.java @@ -37,7 +37,6 @@ import io.grpc.xds.client.Bootstrapper.BootstrapInfo; import io.grpc.xds.client.EnvoyProtoData; import io.grpc.xds.client.XdsClient; -import io.grpc.xds.client.XdsInitializationException; import io.grpc.xds.client.XdsResourceType; import java.time.Duration; import java.util.ArrayList; @@ -63,6 +62,17 @@ public class XdsServerTestHelper { "projects/42/networks/default/nodes/5c85b298-6f5b-4722-b74a-f7d1f0ccf5ad"; private static final EnvoyProtoData.Node BOOTSTRAP_NODE = EnvoyProtoData.Node.newBuilder().setId(NODE_ID).build(); + static final Map RAW_BOOTSTRAP = ImmutableMap.of( + "node", ImmutableMap.of( + "id", NODE_ID), + "server_listener_resource_name_template", "grpc/server?udpa.resource.listening_address=%s", + "xds_servers", ImmutableList.of( + ImmutableMap.of( + "server_uri", SERVER_URI, + "channel_creds", ImmutableList.of( + ImmutableMap.of( + "type", "insecure"))) + )); static final Bootstrapper.BootstrapInfo BOOTSTRAP_INFO = Bootstrapper.BootstrapInfo.builder() .servers(Arrays.asList( @@ -140,17 +150,12 @@ static final class FakeXdsClientPoolFactory implements XdsClientPoolFactory { private XdsClient xdsClient; - Map savedBootstrap; + BootstrapInfo savedBootstrapInfo; FakeXdsClientPoolFactory(XdsClient xdsClient) { this.xdsClient = xdsClient; } - @Override - public void setBootstrapOverride(Map bootstrap) { - this.savedBootstrap = bootstrap; - } - @Override @Nullable public ObjectPool get(String target) { @@ -158,8 +163,9 @@ public ObjectPool get(String target) { } @Override - public ObjectPool getOrCreate(String target, MetricRecorder metricRecorder) - throws XdsInitializationException { + public ObjectPool getOrCreate( + String target, BootstrapInfo bootstrapInfo, MetricRecorder metricRecorder) { + this.savedBootstrapInfo = bootstrapInfo; return new ObjectPool() { @Override public XdsClient getObject() { diff --git a/xds/src/test/java/io/grpc/xds/XdsServerWrapperTest.java b/xds/src/test/java/io/grpc/xds/XdsServerWrapperTest.java index ce1c6b94a35..99e9907f7d0 100644 --- a/xds/src/test/java/io/grpc/xds/XdsServerWrapperTest.java +++ b/xds/src/test/java/io/grpc/xds/XdsServerWrapperTest.java @@ -135,6 +135,7 @@ public void setup() { when(mockBuilder.build()).thenReturn(mockServer); xdsServerWrapper = new XdsServerWrapper("0.0.0.0:1", mockBuilder, listener, selectorManager, new FakeXdsClientPoolFactory(xdsClient), + XdsServerTestHelper.RAW_BOOTSTRAP, filterRegistry, executor.getScheduledExecutorService()); } @@ -157,7 +158,8 @@ public void testBootstrap() throws Exception { XdsListenerResource listenerResource = XdsListenerResource.getInstance(); when(xdsClient.getBootstrapInfo()).thenReturn(b); xdsServerWrapper = new XdsServerWrapper("[::FFFF:129.144.52.38]:80", mockBuilder, listener, - selectorManager, new FakeXdsClientPoolFactory(xdsClient), filterRegistry); + selectorManager, new FakeXdsClientPoolFactory(xdsClient), + XdsServerTestHelper.RAW_BOOTSTRAP, filterRegistry); Executors.newSingleThreadExecutor().execute(new Runnable() { @Override public void run() { @@ -190,7 +192,8 @@ private void verifyBootstrapFail(Bootstrapper.BootstrapInfo b) throws Exception XdsClient xdsClient = mock(XdsClient.class); when(xdsClient.getBootstrapInfo()).thenReturn(b); xdsServerWrapper = new XdsServerWrapper("0.0.0.0:1", mockBuilder, listener, - selectorManager, new FakeXdsClientPoolFactory(xdsClient), filterRegistry); + selectorManager, new FakeXdsClientPoolFactory(xdsClient), + XdsServerTestHelper.RAW_BOOTSTRAP, filterRegistry); final SettableFuture start = SettableFuture.create(); Executors.newSingleThreadExecutor().execute(new Runnable() { @Override @@ -229,7 +232,8 @@ public void testBootstrap_templateWithXdstp() throws Exception { XdsListenerResource listenerResource = XdsListenerResource.getInstance(); when(xdsClient.getBootstrapInfo()).thenReturn(b); xdsServerWrapper = new XdsServerWrapper("[::FFFF:129.144.52.38]:80", mockBuilder, listener, - selectorManager, new FakeXdsClientPoolFactory(xdsClient), filterRegistry); + selectorManager, new FakeXdsClientPoolFactory(xdsClient), + XdsServerTestHelper.RAW_BOOTSTRAP, filterRegistry); Executors.newSingleThreadExecutor().execute(new Runnable() { @Override public void run() { @@ -546,6 +550,7 @@ public void onChanged_listenerIsNull() throws ExecutionException, InterruptedException, TimeoutException { xdsServerWrapper = new XdsServerWrapper("10.1.2.3:1", mockBuilder, listener, selectorManager, new FakeXdsClientPoolFactory(xdsClient), + XdsServerTestHelper.RAW_BOOTSTRAP, filterRegistry, executor.getScheduledExecutorService()); final SettableFuture start = SettableFuture.create(); Executors.newSingleThreadExecutor().execute(new Runnable() { @@ -575,6 +580,7 @@ public void onChanged_listenerAddressMissingPort() throws ExecutionException, InterruptedException, TimeoutException { xdsServerWrapper = new XdsServerWrapper("10.1.2.3:1", mockBuilder, listener, selectorManager, new FakeXdsClientPoolFactory(xdsClient), + XdsServerTestHelper.RAW_BOOTSTRAP, filterRegistry, executor.getScheduledExecutorService()); final SettableFuture start = SettableFuture.create(); Executors.newSingleThreadExecutor().execute(new Runnable() { @@ -612,6 +618,7 @@ public void onChanged_listenerAddressMismatch() throws ExecutionException, InterruptedException, TimeoutException { xdsServerWrapper = new XdsServerWrapper("10.1.2.3:1", mockBuilder, listener, selectorManager, new FakeXdsClientPoolFactory(xdsClient), + XdsServerTestHelper.RAW_BOOTSTRAP, filterRegistry, executor.getScheduledExecutorService()); final SettableFuture start = SettableFuture.create(); Executors.newSingleThreadExecutor().execute(new Runnable() { @@ -649,6 +656,7 @@ public void onChanged_listenerAddressPortMismatch() throws ExecutionException, InterruptedException, TimeoutException { xdsServerWrapper = new XdsServerWrapper("10.1.2.3:1", mockBuilder, listener, selectorManager, new FakeXdsClientPoolFactory(xdsClient), + XdsServerTestHelper.RAW_BOOTSTRAP, filterRegistry, executor.getScheduledExecutorService()); final SettableFuture start = SettableFuture.create(); Executors.newSingleThreadExecutor().execute(new Runnable() { @@ -1842,7 +1850,8 @@ private FilterRegistry filterStateTestFilterRegistry( private SettableFuture filterStateTestStartServer(FilterRegistry filterRegistry) { xdsServerWrapper = new XdsServerWrapper("0.0.0.0:1", mockBuilder, listener, - selectorManager, new FakeXdsClientPoolFactory(xdsClient), filterRegistry); + selectorManager, new FakeXdsClientPoolFactory(xdsClient), + XdsServerTestHelper.RAW_BOOTSTRAP, filterRegistry); SettableFuture serverStart = SettableFuture.create(); scheduleServerStart(xdsServerWrapper, serverStart); return serverStart;