From 38739db61dbd7d5cb09f532e43bab4f9e977d819 Mon Sep 17 00:00:00 2001 From: Andreas Ahlenstorf Date: Sat, 8 Jan 2022 17:54:01 +0100 Subject: [PATCH] Allow customization of Spring Batch TaskExecutor --- .../batch/BasicBatchConfigurer.java | 25 +++++++++++ .../batch/BatchConfigurerConfiguration.java | 15 ++++--- .../batch/BatchTaskExecutor.java | 39 ++++++++++++++++ .../batch/JpaBatchConfigurer.java | 24 +++++++++- .../batch/BatchAutoConfigurationTests.java | 45 +++++++++++++++++++ .../JobLauncherApplicationRunnerTests.java | 4 +- .../src/docs/asciidoc/howto/batch.adoc | 30 +++++++++++++ .../BatchConfiguration.java | 37 +++++++++++++++ 8 files changed, 211 insertions(+), 8 deletions(-) create mode 100644 spring-boot-project/spring-boot-autoconfigure/src/main/java/org/springframework/boot/autoconfigure/batch/BatchTaskExecutor.java create mode 100644 spring-boot-project/spring-boot-docs/src/main/java/org/springframework/boot/docs/howto/batch/customizingtaskexecutor/BatchConfiguration.java diff --git a/spring-boot-project/spring-boot-autoconfigure/src/main/java/org/springframework/boot/autoconfigure/batch/BasicBatchConfigurer.java b/spring-boot-project/spring-boot-autoconfigure/src/main/java/org/springframework/boot/autoconfigure/batch/BasicBatchConfigurer.java index dac961ecc25b..1f4bba3c90f9 100644 --- a/spring-boot-project/spring-boot-autoconfigure/src/main/java/org/springframework/boot/autoconfigure/batch/BasicBatchConfigurer.java +++ b/spring-boot-project/spring-boot-autoconfigure/src/main/java/org/springframework/boot/autoconfigure/batch/BasicBatchConfigurer.java @@ -29,6 +29,7 @@ import org.springframework.boot.autoconfigure.batch.BatchProperties.Isolation; import org.springframework.boot.autoconfigure.transaction.TransactionManagerCustomizers; import org.springframework.boot.context.properties.PropertyMapper; +import org.springframework.core.task.TaskExecutor; import org.springframework.jdbc.datasource.DataSourceTransactionManager; import org.springframework.transaction.PlatformTransactionManager; @@ -39,6 +40,7 @@ * @author Andy Wilkinson * @author Kazuki Shimizu * @author Stephane Nicoll + * @author Andreas Ahlenstorf * @since 1.0.0 */ public class BasicBatchConfigurer implements BatchConfigurer, InitializingBean { @@ -51,6 +53,8 @@ public class BasicBatchConfigurer implements BatchConfigurer, InitializingBean { private final TransactionManagerCustomizers transactionManagerCustomizers; + private final TaskExecutor taskExecutor; + private JobRepository jobRepository; private JobLauncher jobLauncher; @@ -63,12 +67,30 @@ public class BasicBatchConfigurer implements BatchConfigurer, InitializingBean { * @param dataSource the underlying data source * @param transactionManagerCustomizers transaction manager customizers (or * {@code null}) + * @deprecated since 2.7.0 for removal in 3.0.0 in favor of + * {@link #BasicBatchConfigurer(BatchProperties, DataSource, TransactionManagerCustomizers, TaskExecutor)} */ + @Deprecated protected BasicBatchConfigurer(BatchProperties properties, DataSource dataSource, TransactionManagerCustomizers transactionManagerCustomizers) { + this(properties, dataSource, transactionManagerCustomizers, null); + } + + /** + * Create a new {@link BasicBatchConfigurer} instance. + * @param properties the batch properties + * @param dataSource the underlying data source + * @param transactionManagerCustomizers transaction manager customizers (or + * {@code null}) + * @param taskExecutor the executor to be used by {@link JobLauncher} (or + * {@code null}) + */ + protected BasicBatchConfigurer(BatchProperties properties, DataSource dataSource, + TransactionManagerCustomizers transactionManagerCustomizers, TaskExecutor taskExecutor) { this.properties = properties; this.dataSource = dataSource; this.transactionManagerCustomizers = transactionManagerCustomizers; + this.taskExecutor = taskExecutor; } @Override @@ -120,6 +142,9 @@ protected JobExplorer createJobExplorer() throws Exception { protected JobLauncher createJobLauncher() throws Exception { SimpleJobLauncher jobLauncher = new SimpleJobLauncher(); jobLauncher.setJobRepository(getJobRepository()); + if (this.taskExecutor != null) { + jobLauncher.setTaskExecutor(this.taskExecutor); + } jobLauncher.afterPropertiesSet(); return jobLauncher; } diff --git a/spring-boot-project/spring-boot-autoconfigure/src/main/java/org/springframework/boot/autoconfigure/batch/BatchConfigurerConfiguration.java b/spring-boot-project/spring-boot-autoconfigure/src/main/java/org/springframework/boot/autoconfigure/batch/BatchConfigurerConfiguration.java index 259b47e01368..20b450dfbe94 100644 --- a/spring-boot-project/spring-boot-autoconfigure/src/main/java/org/springframework/boot/autoconfigure/batch/BatchConfigurerConfiguration.java +++ b/spring-boot-project/spring-boot-autoconfigure/src/main/java/org/springframework/boot/autoconfigure/batch/BatchConfigurerConfiguration.java @@ -1,5 +1,5 @@ /* - * Copyright 2012-2019 the original author or authors. + * Copyright 2012-2022 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -27,12 +27,14 @@ import org.springframework.boot.autoconfigure.transaction.TransactionManagerCustomizers; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.core.task.TaskExecutor; import org.springframework.transaction.PlatformTransactionManager; /** * Provide a {@link BatchConfigurer} according to the current environment. * * @author Stephane Nicoll + * @author Andreas Ahlenstorf */ @ConditionalOnClass(PlatformTransactionManager.class) @ConditionalOnBean(DataSource.class) @@ -47,9 +49,10 @@ static class JdbcBatchConfiguration { @Bean BasicBatchConfigurer batchConfigurer(BatchProperties properties, DataSource dataSource, @BatchDataSource ObjectProvider batchDataSource, - ObjectProvider transactionManagerCustomizers) { + ObjectProvider transactionManagerCustomizers, + @BatchTaskExecutor ObjectProvider batchTaskExecutor) { return new BasicBatchConfigurer(properties, batchDataSource.getIfAvailable(() -> dataSource), - transactionManagerCustomizers.getIfAvailable()); + transactionManagerCustomizers.getIfAvailable(), batchTaskExecutor.getIfAvailable()); } } @@ -63,9 +66,11 @@ static class JpaBatchConfiguration { JpaBatchConfigurer batchConfigurer(BatchProperties properties, DataSource dataSource, @BatchDataSource ObjectProvider batchDataSource, ObjectProvider transactionManagerCustomizers, - EntityManagerFactory entityManagerFactory) { + EntityManagerFactory entityManagerFactory, + @BatchTaskExecutor ObjectProvider batchTaskExecutor) { return new JpaBatchConfigurer(properties, batchDataSource.getIfAvailable(() -> dataSource), - transactionManagerCustomizers.getIfAvailable(), entityManagerFactory); + transactionManagerCustomizers.getIfAvailable(), entityManagerFactory, + batchTaskExecutor.getIfAvailable()); } } diff --git a/spring-boot-project/spring-boot-autoconfigure/src/main/java/org/springframework/boot/autoconfigure/batch/BatchTaskExecutor.java b/spring-boot-project/spring-boot-autoconfigure/src/main/java/org/springframework/boot/autoconfigure/batch/BatchTaskExecutor.java new file mode 100644 index 000000000000..ce0c383a8028 --- /dev/null +++ b/spring-boot-project/spring-boot-autoconfigure/src/main/java/org/springframework/boot/autoconfigure/batch/BatchTaskExecutor.java @@ -0,0 +1,39 @@ +/* + * Copyright 2012-2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.boot.autoconfigure.batch; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +import org.springframework.beans.factory.annotation.Qualifier; + +/** + * Qualifier annotation for a TaskExecutor to be injected into Batch auto-configuration. + * + * @author Andreas Ahlenstorf + * @since 2.7.0 + */ +@Target({ ElementType.FIELD, ElementType.METHOD, ElementType.PARAMETER, ElementType.TYPE, ElementType.ANNOTATION_TYPE }) +@Retention(RetentionPolicy.RUNTIME) +@Documented +@Qualifier +public @interface BatchTaskExecutor { + +} diff --git a/spring-boot-project/spring-boot-autoconfigure/src/main/java/org/springframework/boot/autoconfigure/batch/JpaBatchConfigurer.java b/spring-boot-project/spring-boot-autoconfigure/src/main/java/org/springframework/boot/autoconfigure/batch/JpaBatchConfigurer.java index 1d168debe10b..0d613fce774b 100644 --- a/spring-boot-project/spring-boot-autoconfigure/src/main/java/org/springframework/boot/autoconfigure/batch/JpaBatchConfigurer.java +++ b/spring-boot-project/spring-boot-autoconfigure/src/main/java/org/springframework/boot/autoconfigure/batch/JpaBatchConfigurer.java @@ -24,6 +24,7 @@ import org.springframework.boot.autoconfigure.batch.BatchProperties.Isolation; import org.springframework.boot.autoconfigure.transaction.TransactionManagerCustomizers; +import org.springframework.core.task.TaskExecutor; import org.springframework.orm.jpa.JpaTransactionManager; import org.springframework.transaction.PlatformTransactionManager; @@ -31,6 +32,7 @@ * A {@link BasicBatchConfigurer} tailored for JPA. * * @author Stephane Nicoll + * @author Andreas Ahlenstorf * @since 2.0.0 */ public class JpaBatchConfigurer extends BasicBatchConfigurer { @@ -46,10 +48,30 @@ public class JpaBatchConfigurer extends BasicBatchConfigurer { * @param transactionManagerCustomizers transaction manager customizers (or * {@code null}) * @param entityManagerFactory the entity manager factory (or {@code null}) + * @deprecated since 2.7.0 for removal in 3.0.0 in favor of + * {@link #JpaBatchConfigurer(BatchProperties, DataSource, TransactionManagerCustomizers, EntityManagerFactory, TaskExecutor)} */ + @Deprecated protected JpaBatchConfigurer(BatchProperties properties, DataSource dataSource, TransactionManagerCustomizers transactionManagerCustomizers, EntityManagerFactory entityManagerFactory) { - super(properties, dataSource, transactionManagerCustomizers); + super(properties, dataSource, transactionManagerCustomizers, null); + this.entityManagerFactory = entityManagerFactory; + } + + /** + * Create a new {@link BasicBatchConfigurer} instance. + * @param properties the batch properties + * @param dataSource the underlying data source + * @param transactionManagerCustomizers transaction manager customizers (or + * {@code null}) + * @param entityManagerFactory the entity manager factory (or {@code null}) + * @param taskExecutor the executor to be used by + * {@link org.springframework.batch.core.launch.JobLauncher} (or {@code null}) + */ + protected JpaBatchConfigurer(BatchProperties properties, DataSource dataSource, + TransactionManagerCustomizers transactionManagerCustomizers, EntityManagerFactory entityManagerFactory, + TaskExecutor taskExecutor) { + super(properties, dataSource, transactionManagerCustomizers, taskExecutor); this.entityManagerFactory = entityManagerFactory; } diff --git a/spring-boot-project/spring-boot-autoconfigure/src/test/java/org/springframework/boot/autoconfigure/batch/BatchAutoConfigurationTests.java b/spring-boot-project/spring-boot-autoconfigure/src/test/java/org/springframework/boot/autoconfigure/batch/BatchAutoConfigurationTests.java index 502df3940aaf..b58c5e2209e1 100644 --- a/spring-boot-project/spring-boot-autoconfigure/src/test/java/org/springframework/boot/autoconfigure/batch/BatchAutoConfigurationTests.java +++ b/spring-boot-project/spring-boot-autoconfigure/src/test/java/org/springframework/boot/autoconfigure/batch/BatchAutoConfigurationTests.java @@ -63,6 +63,8 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Primary; +import org.springframework.core.task.SyncTaskExecutor; +import org.springframework.core.task.TaskExecutor; import org.springframework.jdbc.BadSqlGrammarException; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.datasource.DataSourceTransactionManager; @@ -80,6 +82,7 @@ * @author Stephane Nicoll * @author Vedran Pavic * @author Kazuki Shimizu + * @author Andreas Ahlenstorf */ @ExtendWith(OutputCaptureExtension.class) class BatchAutoConfigurationTests { @@ -367,6 +370,37 @@ void whenTheUserDefinesTheirOwnDatabaseInitializerThenTheAutoConfiguredBatchInit .hasBean("customInitializer")); } + @Test + void jobLauncherUsesBatchTaskExecutorIfPresent() { + this.contextRunner.withUserConfiguration(TestConfiguration.class, EmbeddedDataSourceConfiguration.class, + BatchTaskExecutorConfiguration.class).run((context) -> { + assertThat(context).hasBean("batchTaskExecutor"); + + TaskExecutor taskExecutor = context.getBean("batchTaskExecutor", TaskExecutor.class); + BatchConfigurer batchConfigurer = context.getBean(BatchConfigurer.class); + + assertThat(batchConfigurer.getJobLauncher()).hasFieldOrPropertyWithValue("taskExecutor", + taskExecutor); + }); + } + + @Test + void jobLauncherConfiguredByJpaBatchConfigurerUsesBatchTaskExecutorIfPresent() { + this.contextRunner.withUserConfiguration(TestConfiguration.class, EmbeddedDataSourceConfiguration.class, + HibernateJpaAutoConfiguration.class, BatchTaskExecutorConfiguration.class).run((context) -> { + PlatformTransactionManager transactionManager = context.getBean(PlatformTransactionManager.class); + assertThat(transactionManager.toString().contains("JpaTransactionManager")).isTrue(); + + assertThat(context).hasBean("batchTaskExecutor"); + + TaskExecutor taskExecutor = context.getBean("batchTaskExecutor", TaskExecutor.class); + BatchConfigurer batchConfigurer = context.getBean(BatchConfigurer.class); + + assertThat(batchConfigurer.getJobLauncher()).hasFieldOrPropertyWithValue("taskExecutor", + taskExecutor); + }); + } + @Configuration(proxyBeanMethods = false) protected static class BatchDataSourceConfiguration { @@ -531,4 +565,15 @@ DataSourceScriptDatabaseInitializer customInitializer(DataSource dataSource) { } + @Configuration(proxyBeanMethods = false) + protected static class BatchTaskExecutorConfiguration { + + @BatchTaskExecutor + @Bean + public TaskExecutor batchTaskExecutor() { + return new SyncTaskExecutor(); + } + + } + } diff --git a/spring-boot-project/spring-boot-autoconfigure/src/test/java/org/springframework/boot/autoconfigure/batch/JobLauncherApplicationRunnerTests.java b/spring-boot-project/spring-boot-autoconfigure/src/test/java/org/springframework/boot/autoconfigure/batch/JobLauncherApplicationRunnerTests.java index b9d0e26ab6fb..11a438aadcfb 100644 --- a/spring-boot-project/spring-boot-autoconfigure/src/test/java/org/springframework/boot/autoconfigure/batch/JobLauncherApplicationRunnerTests.java +++ b/spring-boot-project/spring-boot-autoconfigure/src/test/java/org/springframework/boot/autoconfigure/batch/JobLauncherApplicationRunnerTests.java @@ -1,5 +1,5 @@ /* - * Copyright 2012-2021 the original author or authors. + * Copyright 2012-2022 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -223,7 +223,7 @@ static class BatchConfiguration extends BasicBatchConfigurer { private final DataSource dataSource; protected BatchConfiguration(DataSource dataSource) { - super(new BatchProperties(), dataSource, new TransactionManagerCustomizers(null)); + super(new BatchProperties(), dataSource, new TransactionManagerCustomizers(null), null); this.dataSource = dataSource; } diff --git a/spring-boot-project/spring-boot-docs/src/docs/asciidoc/howto/batch.adoc b/spring-boot-project/spring-boot-docs/src/docs/asciidoc/howto/batch.adoc index 3b492be4626a..9cdaba69c06a 100644 --- a/spring-boot-project/spring-boot-docs/src/docs/asciidoc/howto/batch.adoc +++ b/spring-boot-project/spring-boot-docs/src/docs/asciidoc/howto/batch.adoc @@ -3,6 +3,19 @@ A number of questions often arise when people use Spring Batch from within a Spring Boot application. This section addresses those questions. +[[howto.batch.architectural-considerations]] +=== Architectural Considerations + +The default configuration of Spring Batch is geared towards single-job containerized Spring Boot apps orchestrated by a job scheduler like Kubernetes, optionally using https://dataflow.spring.io/[Spring Cloud Dataflow]: + +* All `Jobs` in the application context are executed once on application startup; see <<#howto.batch.running-jobs-on-startup>>. +* Jobs are run on the same thread they are launched on. + +However, it is also a valid option to run batch jobs as part of other applications, notably web applications running in servlet containers. +Example use cases include reporting, ad-hoc job running, and web application support. +See {spring-batch-docs}job.html#runningJobsFromWebContainer[the Spring Batch documentation] for how to do this. +You might need to <<#howto.batch.customizing-task-executor,customize the task executor for batch jobs>> if you opt for this approach. + [[howto.batch.specifying-a-data-source]] @@ -57,3 +70,20 @@ This provides only one argument to the batch job: `someParameter=someValue`. Spring Batch requires a data store for the `Job` repository. If you use Spring Boot, you must use an actual database. Note that it can be an in-memory database, see {spring-batch-docs}job.html#configuringJobRepository[Configuring a Job Repository]. + +[[howto.batch.customizing-task-executor]] +=== Customizing the TaskExecutor for Batch Jobs + +By default, jobs are run on the same thread they are launched on. +See {spring-batch-api}/core/configuration/annotation/EnableBatchProcessing.html[the Javadoc of `@EnableBatchProcessing`] for details. +Depending on your type of application, this might be undesirable because the starting thread is blocked while the batch job runs. +Furthermore, additional jobs might have to wait for the currently running job to complete. +In such a case, it is necessary to customize the `TaskExecutor` used by Spring Batch. +The auto-configuration of Spring Batch automatically picks up any `TaskExecutor` annotated with `@BatchTaskExecutor`, for example: + +[source,java,indent=0,subs="verbatim"] +---- +include::{docs-java}/howto/batch/customizingtaskexecutor/BatchConfiguration.java[tag=bean] +---- + +The example configures a `TaskExecutor` that dynamically adapts it size and uses up to 4 threads with a maximum queue size of 10. Adjust this configuration to match the needs of your application. \ No newline at end of file diff --git a/spring-boot-project/spring-boot-docs/src/main/java/org/springframework/boot/docs/howto/batch/customizingtaskexecutor/BatchConfiguration.java b/spring-boot-project/spring-boot-docs/src/main/java/org/springframework/boot/docs/howto/batch/customizingtaskexecutor/BatchConfiguration.java new file mode 100644 index 000000000000..1753d09235e3 --- /dev/null +++ b/spring-boot-project/spring-boot-docs/src/main/java/org/springframework/boot/docs/howto/batch/customizingtaskexecutor/BatchConfiguration.java @@ -0,0 +1,37 @@ +/* + * Copyright 2012-2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.boot.docs.howto.batch.customizingtaskexecutor; + +import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; +import org.springframework.boot.autoconfigure.batch.BatchTaskExecutor; +import org.springframework.boot.task.TaskExecutorBuilder; +import org.springframework.context.annotation.Bean; +import org.springframework.core.task.TaskExecutor; + +@EnableBatchProcessing +public class BatchConfiguration { + + // tag::bean[] + @Bean + @BatchTaskExecutor + public TaskExecutor batchTaskExecutor(TaskExecutorBuilder builder) { + return builder.threadNamePrefix("batch-").corePoolSize(0).maxPoolSize(4).queueCapacity(10) + .allowCoreThreadTimeOut(true).awaitTermination(false).build(); + } + // end::bean[] + +}