SHDP-371 Update boot and core

- Boot from 1.1.0 to 1.1.5. Fix Map handling
  in config properties in SpringYarnClientLocalizerProperties.
- Core from 4.1.0.RC1 to 4.1.0.RC2. Fix bean autowiring
  in YarnAppmasterAutoConfiguration by not define a bean
  in a same @configuration class which tries to autowire
  beans by its interface.

Author: jvalkeal

gradle.properties

@@ -57,9 +57,9 @@ defaultPigVersion = 0.10.1
 defaultHbaseVersion = 0.94.11
 
 ## Common libraries
-springVersion = 4.1.0.RC1
+springVersion = 4.1.0.RC2
 springBatchVersion = 3.0.1.RELEASE
-springBootVersion = 1.1.0.RELEASE
+springBootVersion = 1.1.5.RELEASE
 springIntVersion = 4.0.3.RELEASE
 jacksonVersion = 1.9.13
 jackson2Version = 2.4.2

spring-yarn/spring-yarn-batch/src/test/java/org/springframework/yarn/batch/support/YarnJobLauncherTests.java

@@ -15,6 +15,8 @@
  */
 package org.springframework.yarn.batch.support;
 
+import static org.junit.Assert.assertEquals;
+
 import java.util.ArrayList;
 import java.util.Arrays;
 
@@ -41,14 +43,11 @@
 import org.springframework.batch.repeat.RepeatStatus;
 import org.springframework.batch.support.transaction.ResourcelessTransactionManager;
 import org.springframework.context.annotation.AnnotationConfigApplicationContext;
-import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 import org.springframework.core.task.SyncTaskExecutor;
 import org.springframework.transaction.PlatformTransactionManager;
 import org.springframework.yarn.batch.support.YarnBatchProperties.JobProperties;
 
-import static org.junit.Assert.assertEquals;
-
 /**
  * Tests for {@link YarnJobLauncher}.
  *

spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/YarnAppmasterAutoConfiguration.java

@@ -218,6 +218,20 @@ public YarnJobLauncher yarnJobLauncher() {
 
 	}
 
+	@Configuration
+	@ConditionalOnExpression("${spring.yarn.appmaster.containercluster.enabled:false}")
+	public static class ContainerClusterFactoryConfig  {
+
+		@Bean
+		@ConditionalOnMissingBean(name = "defaultGridProjectionFactory")
+		public GridProjectionFactory defaultGridProjectionFactory() {
+			// on its own @Configuration class because we
+			// autowire in ContainerClusterConfig
+			return new DefaultGridProjectionFactory();
+		}
+
+	}
+
 	@Configuration
 	@EnableConfigurationProperties({ SpringYarnAppmasterProperties.class })
 	@ConditionalOnExpression("${spring.yarn.appmaster.containercluster.enabled:false}")
@@ -229,12 +243,6 @@ public static class ContainerClusterConfig  {
 		@Autowired(required = false)
 		private List<GridProjectionFactory> gridProjectionFactories;
 
-		@Bean
-		@ConditionalOnMissingBean(name = "defaultGridProjectionFactory")
-		public GridProjectionFactory defaultGridProjectionFactory() {
-			return new DefaultGridProjectionFactory();
-		}
-
 		@Bean
 		public GridProjectionFactoryLocator gridProjectionFactoryLocator() {
 			GridProjectionFactoryRegistry registry = new GridProjectionFactoryRegistry();

spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/YarnClientAutoConfiguration.java

@@ -15,6 +15,8 @@
  */
 package org.springframework.yarn.boot;
 
+import java.util.HashMap;
+import java.util.Map;
 import java.util.Properties;
 
 import org.springframework.beans.factory.annotation.Autowired;
@@ -160,7 +162,7 @@ public void configure(YarnResourceLocalizerConfigurer localizer) throws Exceptio
 				.stagingDirectory(syp.getStagingDir())
 				.withCopy()
 					.copy(StringUtils.toStringArray(sycp.getFiles()), applicationDir, applicationDir == null)
-					.raw(syclp.getRawFileContents(), applicationDir);
+					.raw(unescapeMapKeys(syclp.getRawFileContents()), applicationDir);
 
 			LocalResourcesHdfsConfigurer withHdfs = localizer.withHdfs();
 			for (Entry e : localResourcesSelector.select(applicationDir != null ? applicationDir : "/")) {
@@ -233,4 +235,15 @@ private static String[] createMasterCommands(SpringYarnClientLaunchContextProper
 		return factory.getObject();
 	}
 
+	private static Map<String, byte[]> unescapeMapKeys(Map<String, byte[]> map) {
+		if (map == null || map.isEmpty()) {
+			return map;
+		}
+		HashMap<String, byte[]> nmap = new HashMap<String, byte[]>();
+		for (String key : map.keySet()) {
+			nmap.put(SpringYarnBootUtils.unescapeConfigKey(key), map.get(key));
+		}
+		return nmap;
+	}
+
 }

spring-yarn/spring-yarn-boot/src/main/java/org/springframework/yarn/boot/support/SpringYarnBootUtils.java

@@ -20,8 +20,8 @@
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.Map;
-import java.util.Properties;
 import java.util.Map.Entry;
+import java.util.Properties;
 
 import org.springframework.boot.builder.SpringApplicationBuilder;
 import org.springframework.util.Assert;
@@ -117,21 +117,27 @@ public static void addConfigFilesContents(SpringApplicationBuilder builder, Map<
 		if (configFilesContents == null) {
 			return;
 		}
-		Map<String, byte[]> content = new HashMap<String, byte[]>();
+		Map<String, Object> defaultProperties = new HashMap<String, Object>();
 		for (Entry<String, Properties> entry : configFilesContents.entrySet()) {
 			try {
 				ByteArrayOutputStream out = new ByteArrayOutputStream();
 				entry.getValue().store(out, null);
-				content.put(entry.getKey(), out.toByteArray());
+				defaultProperties.put(
+						"spring.yarn.client.localizer.rawFileContents."
+								+ SpringYarnBootUtils.escapeConfigKey(entry.getKey()), out.toByteArray());
 			} catch (IOException e) {
 				// suppress because this should not happen
 			}
 		}
-		if (!content.isEmpty()) {
-			Properties p = new Properties();
-			p.put("spring.yarn.client.localizer.rawFileContents", content);
-			builder.properties(p);
-		}
+		builder.properties(defaultProperties);
+	}
+
+	public static String escapeConfigKey(String key) {
+		return StringUtils.replace(key, ".", "%2E");
+	}
+
+	public static String unescapeConfigKey(String key) {
+		return StringUtils.replace(key, "%2E", ".");
 	}
 
 	public static String resolveApplicationdir(SpringYarnProperties syp) {

spring-yarn/spring-yarn-boot/src/test/java/org/springframework/yarn/boot/properties/SpringYarnClientLocalizerPropertiesTests.java

@@ -20,12 +20,15 @@
 import static org.junit.Assert.assertThat;
 
 import java.util.List;
+import java.util.Properties;
 
 import org.junit.Test;
 import org.springframework.boot.SpringApplication;
+import org.springframework.boot.builder.SpringApplicationBuilder;
 import org.springframework.boot.context.properties.EnableConfigurationProperties;
 import org.springframework.context.ConfigurableApplicationContext;
 import org.springframework.context.annotation.Configuration;
+import org.springframework.yarn.boot.support.SpringYarnBootUtils;
 
 public class SpringYarnClientLocalizerPropertiesTests {
 
@@ -61,6 +64,28 @@ public void testAllPropertiesSet() {
 		context.close();
 	}
 
+	@Test
+	public void testRawFileContents() {
+		SpringApplicationBuilder builder = new SpringApplicationBuilder(TestConfiguration.class);
+
+		Properties p = new Properties();
+		p.put("spring.yarn.client.localizer.rawFileContents." + SpringYarnBootUtils.escapeConfigKey("file1"), new byte[1]);
+		p.put("spring.yarn.client.localizer.rawFileContents." + SpringYarnBootUtils.escapeConfigKey("file2"), new byte[2]);
+		p.put("spring.yarn.client.localizer.rawFileContents." + SpringYarnBootUtils.escapeConfigKey("application.properties"), new byte[3]);
+		builder.properties(p);
+
+		SpringApplication app = builder.application();
+		app.setWebEnvironment(false);
+		ConfigurableApplicationContext context = app.run(new String[0]);
+		SpringYarnClientLocalizerProperties properties = context.getBean(SpringYarnClientLocalizerProperties.class);
+		assertThat(properties, notNullValue());
+		assertThat(properties.getRawFileContents(), notNullValue());
+		assertThat(properties.getRawFileContents().size(), is(3));
+		assertThat(properties.getRawFileContents().get(SpringYarnBootUtils.escapeConfigKey("file1")).length, is(1));
+		assertThat(properties.getRawFileContents().get(SpringYarnBootUtils.escapeConfigKey("file2")).length, is(2));
+		assertThat(properties.getRawFileContents().get(SpringYarnBootUtils.escapeConfigKey("application.properties")).length, is(3));
+	}
+
 	@Configuration
 	@EnableConfigurationProperties({SpringYarnClientLocalizerProperties.class})
 	protected static class TestConfiguration {