diff --git a/src/ServiceManagement/HDInsight/Commands.HDInsight.Test/Commands.HDInsight.Test.csproj b/src/ServiceManagement/HDInsight/Commands.HDInsight.Test/Commands.HDInsight.Test.csproj index fad22f253ca0..92b7754eba9a 100644 --- a/src/ServiceManagement/HDInsight/Commands.HDInsight.Test/Commands.HDInsight.Test.csproj +++ b/src/ServiceManagement/HDInsight/Commands.HDInsight.Test/Commands.HDInsight.Test.csproj @@ -50,11 +50,13 @@ ..\..\..\packages\Microsoft.Data.Services.Client.5.6.0\lib\net40\Microsoft.Data.Services.Client.dll - - ..\..\..\packages\Microsoft.Hadoop.Client.1.3.3.2\lib\net40\Microsoft.Hadoop.Client.dll + + False + ..\..\..\packages\Microsoft.Hadoop.Client.1.4.5\lib\net45\Microsoft.Hadoop.Client.dll - - ..\..\..\packages\Microsoft.Hadoop.Client.1.3.3.2\lib\net40\Microsoft.HDInsight.Net.Http.Formatting.dll + + False + ..\..\..\packages\Microsoft.Hadoop.Client.1.4.5\lib\net45\Microsoft.HDInsight.Net.Http.Formatting.dll ..\..\..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.dll @@ -75,18 +77,27 @@ ..\..\..\packages\Microsoft.WindowsAzure.Common.1.4.1\lib\net45\Microsoft.WindowsAzure.Common.NetFramework.dll + False ..\..\..\packages\Microsoft.WindowsAzure.ConfigurationManager.1.8.0.0\lib\net35-full\Microsoft.WindowsAzure.Configuration.dll - - ..\..\..\packages\Microsoft.WindowsAzure.Management.HDInsight.1.3.3.2\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.dll + + ..\..\..\packages\Microsoft.WindowsAzure.Management.HDInsight.1.4.5\lib\net45\Microsoft.WindowsAzure.Management.HDInsight.dll + True - - ..\..\..\packages\Microsoft.Hadoop.Client.1.3.3.2\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.dll + + False + ..\..\..\packages\Microsoft.WindowsAzure.Management.HDInsight.1.4.5\lib\net45\Microsoft.WindowsAzure.Management.HDInsight.Contracts.dll - - ..\..\..\packages\Microsoft.Hadoop.Client.1.3.3.2\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.Core.dll + + False + ..\..\..\packages\Microsoft.Hadoop.Client.1.4.5\lib\net45\Microsoft.WindowsAzure.Management.HDInsight.Framework.dll + + + ..\..\..\packages\Microsoft.Hadoop.Client.1.4.5\lib\net45\Microsoft.WindowsAzure.Management.HDInsight.Framework.Core.dll + True + False ..\..\..\packages\WindowsAzure.Storage.3.0.3.0\lib\net40\Microsoft.WindowsAzure.Storage.dll @@ -206,7 +217,9 @@ - + + Designer + diff --git a/src/ServiceManagement/HDInsight/Commands.HDInsight.Test/HDInsight/CmdLetTests/AddConfigValuesCmdletTests.cs b/src/ServiceManagement/HDInsight/Commands.HDInsight.Test/HDInsight/CmdLetTests/AddConfigValuesCmdletTests.cs index 736463482963..1cfe33f709c2 100644 --- a/src/ServiceManagement/HDInsight/Commands.HDInsight.Test/HDInsight/CmdLetTests/AddConfigValuesCmdletTests.cs +++ b/src/ServiceManagement/HDInsight/Commands.HDInsight.Test/HDInsight/CmdLetTests/AddConfigValuesCmdletTests.cs @@ -83,6 +83,18 @@ public void CanCallTheAddConfigValuesCmdletTestsCmdlet_StormConfig() } + [TestMethod] + [TestCategory("CheckIn")] + public void CanCallTheAddConfigValuesCmdletTestsCmdlet_SparkConfig() + { + using (IRunspace runspace = this.GetPowerShellRunspace()) + { + var sparkConfig = new Hashtable(); + sparkConfig.Add("spark.fakeconfig.value", "12345"); + RunConfigOptionstest(runspace, CmdletConstants.SparkConfig, sparkConfig, c => c.SparkConfiguration); + } + } + [TestMethod] [TestCategory("CheckIn")] public void CanCallTheAddConfigValuesCmdletTestsCmdlet_HiveConfig() @@ -373,7 +385,9 @@ public void CanCallTheAddConfigValuesCmdletTestsCmdlet_PreserveConfig() var coreConfig = new Hashtable(); var yarnConfig = new Hashtable(); var stormConfig = new Hashtable(); + var sparkConfig = new Hashtable(); stormConfig.Add("storm.fakekey", "123"); + sparkConfig.Add("spark.fakekey", "123"); var clusterConfig = new AzureHDInsightConfig { HiveMetastore = @@ -401,12 +415,14 @@ public void CanCallTheAddConfigValuesCmdletTestsCmdlet_PreserveConfig() .WithParameter(CmdletConstants.CoreConfig, coreConfig) .WithParameter(CmdletConstants.YarnConfig, yarnConfig) .WithParameter(CmdletConstants.StormConfig, stormConfig) + .WithParameter(CmdletConstants.SparkConfig, sparkConfig) .Invoke(); AzureHDInsightConfig config = results.Results.ToEnumerable().First(); Assert.AreEqual(config.CoreConfiguration.Count, coreConfig.Count); Assert.AreEqual(config.YarnConfiguration.Count, yarnConfig.Count); Assert.AreEqual(config.StormConfiguration.Count, stormConfig.Count); + Assert.AreEqual(config.SparkConfiguration.Count, sparkConfig.Count); foreach (object entry in coreConfig.Keys) { @@ -432,6 +448,14 @@ public void CanCallTheAddConfigValuesCmdletTestsCmdlet_PreserveConfig() Assert.AreEqual(stormConfig[entry], configUnderTest.Value, "value doesn't match for storm config option with name '{0}'", entry); } + foreach (object entry in sparkConfig.Keys) + { + KeyValuePair configUnderTest = + config.SparkConfiguration.FirstOrDefault(c => string.Equals(c.Key, entry.ToString(), StringComparison.Ordinal)); + Assert.IsNotNull(configUnderTest, "Unable to find spark config option with name '{0}'", entry); + Assert.AreEqual(sparkConfig[entry], configUnderTest.Value, "value doesn't match for spark config option with name '{0}'", entry); + } + Assert.AreEqual(clusterConfig.HiveMetastore.DatabaseName, config.HiveMetastore.DatabaseName); Assert.AreEqual(clusterConfig.HiveMetastore.SqlAzureServerName, config.HiveMetastore.SqlAzureServerName); Assert.AreEqual(clusterConfig.HiveMetastore.Credential.UserName, config.HiveMetastore.Credential.UserName); diff --git a/src/ServiceManagement/HDInsight/Commands.HDInsight.Test/HDInsight/CmdLetTests/CmdletHardCodes.cs b/src/ServiceManagement/HDInsight/Commands.HDInsight.Test/HDInsight/CmdLetTests/CmdletHardCodes.cs index 6c854263e7b3..c8dd8243593e 100644 --- a/src/ServiceManagement/HDInsight/Commands.HDInsight.Test/HDInsight/CmdLetTests/CmdletHardCodes.cs +++ b/src/ServiceManagement/HDInsight/Commands.HDInsight.Test/HDInsight/CmdLetTests/CmdletHardCodes.cs @@ -91,6 +91,7 @@ public static class CmdletConstants public const string StatusFolder = "StatusFolder"; public const string YarnConfig = "Yarn"; public const string StormConfig = "Storm"; + public const string SparkConfig = "Spark"; public const string HBaseConfig = "HBase"; public const string StdErr = "StandardError"; diff --git a/src/ServiceManagement/HDInsight/Commands.HDInsight.Test/HDInsight/CmdLetTests/NewClusterCmdletTests.cs b/src/ServiceManagement/HDInsight/Commands.HDInsight.Test/HDInsight/CmdLetTests/NewClusterCmdletTests.cs index 4f7a09a448ae..3613ccaea6df 100644 --- a/src/ServiceManagement/HDInsight/Commands.HDInsight.Test/HDInsight/CmdLetTests/NewClusterCmdletTests.cs +++ b/src/ServiceManagement/HDInsight/Commands.HDInsight.Test/HDInsight/CmdLetTests/NewClusterCmdletTests.cs @@ -420,7 +420,7 @@ public void ICanCreateAClusterUsingPowerShellAndConfig_New_Set_Add_Hive_Oozie() .WithParameter(CmdletConstants.Credential, GetPSCredential("hadoop", this.GetRandomValidPassword())) .Invoke(); - ClusterCreateParameters request = AzureHDInsightClusterManagementClientSimulator.LastCreateRequest; + ClusterCreateParametersV2 request = AzureHDInsightClusterManagementClientSimulator.LastCreateRequest; Assert.IsNotNull(request.HiveMetastore); Assert.IsNotNull(request.OozieMetastore); @@ -493,7 +493,7 @@ public void ICanCreateAClusterUsingPowerShellAndConfig_New_Set_Add_Hive_Oozie_An .WithParameter(CmdletConstants.Credential, GetPSCredential("hadoop", this.GetRandomValidPassword())) .Invoke(); - ClusterCreateParameters request = AzureHDInsightClusterManagementClientSimulator.LastCreateRequest; + ClusterCreateParametersV2 request = AzureHDInsightClusterManagementClientSimulator.LastCreateRequest; Assert.IsNotNull(request.HiveMetastore); Assert.IsNotNull(request.OozieMetastore); @@ -565,7 +565,7 @@ public void ICanCreateAClusterUsingPowerShellAndConfig_New_Set_Add_ScriptAction( .WithParameter(CmdletConstants.Credential, GetPSCredential("hadoop", this.GetRandomValidPassword())) .Invoke(); - ClusterCreateParameters request = AzureHDInsightClusterManagementClientSimulator.LastCreateRequest; + ClusterCreateParametersV2 request = AzureHDInsightClusterManagementClientSimulator.LastCreateRequest; Assert.IsTrue(request.ConfigActions != null && request.ConfigActions.Count == 2); Assert.IsTrue( request.ConfigActions.ElementAt(0).Name == "test1" && diff --git a/src/ServiceManagement/HDInsight/Commands.HDInsight.Test/Models/Simulators/AzureHDInsightClusterManagementClientSimulator.cs b/src/ServiceManagement/HDInsight/Commands.HDInsight.Test/Models/Simulators/AzureHDInsightClusterManagementClientSimulator.cs index c4c9716a3f24..a32791074f89 100644 --- a/src/ServiceManagement/HDInsight/Commands.HDInsight.Test/Models/Simulators/AzureHDInsightClusterManagementClientSimulator.cs +++ b/src/ServiceManagement/HDInsight/Commands.HDInsight.Test/Models/Simulators/AzureHDInsightClusterManagementClientSimulator.cs @@ -30,7 +30,7 @@ namespace Microsoft.WindowsAzure.Commands.Test.Utilities.HDInsight.Simulators { internal class AzureHDInsightClusterManagementClientSimulator : IHDInsightClient { - internal static ClusterCreateParameters LastCreateRequest; + internal static ClusterCreateParametersV2 LastCreateRequest; private static readonly Collection Clusters = new Collection { @@ -176,9 +176,12 @@ public void Cancel() { } + + public ClusterDetails CreateCluster(ClusterCreateParameters cluster) { - Task createTask = this.CreateClusterAsync(cluster); + Task createTask = this.CreateClusterAsync( + new ClusterCreateParametersV2( cluster)); createTask.Wait(); return createTask.Result; } @@ -188,7 +191,24 @@ public ClusterDetails CreateCluster(ClusterCreateParameters cluster, TimeSpan ti return this.CreateCluster(cluster); } + public ClusterDetails CreateCluster(ClusterCreateParametersV2 cluster) + { + Task createTask = this.CreateClusterAsync(cluster); + createTask.Wait(); + return createTask.Result; + } + + public ClusterDetails CreateCluster(ClusterCreateParametersV2 cluster, TimeSpan timeout) + { + return this.CreateCluster(cluster); + } + public Task CreateClusterAsync(ClusterCreateParameters clusterCreateParameters) + { + return CreateClusterAsync(new ClusterCreateParametersV2(clusterCreateParameters)); + } + + public Task CreateClusterAsync(ClusterCreateParametersV2 clusterCreateParameters) { this.LogMessage("Creating cluster '{0}' in location {1}", clusterCreateParameters.Name, clusterCreateParameters.Location); LastCreateRequest = clusterCreateParameters; @@ -218,6 +238,16 @@ public void DeleteCluster(string dnsName, TimeSpan timeout) this.DeleteClusterAsync(dnsName).Wait(); } + public void DeleteCluster(string dnsName, string location) + { + throw new NotImplementedException(); + } + + public void DeleteCluster(string dnsName, string location, TimeSpan timeout) + { + throw new NotImplementedException(); + } + public async Task DeleteClusterAsync(string name) { ClusterDetails cluster = await this.GetClusterAsync(name); @@ -230,6 +260,11 @@ public async Task DeleteClusterAsync(string name) Clusters.Remove(GetClusterInternal(name)); } + public Task DeleteClusterAsync(string name, string location) + { + throw new NotImplementedException(); + } + public void DisableHttp(string dnsName, string location) { this.DisableHttpAsync(dnsName, location).Wait(); @@ -265,6 +300,13 @@ public ClusterDetails GetCluster(string dnsName) return getTask.Result; } + public ClusterDetails GetCluster(string dnsName, string location) + { + Task getTask = this.GetClusterAsync(dnsName, location); + getTask.Wait(); + return getTask.Result; + } + public async Task GetClusterAsync(string name) { this.LogMessage("Getting hdinsight clusters for subscriptionid : {0}", this.credentials.SubscriptionId.ToString()); @@ -273,6 +315,11 @@ public async Task GetClusterAsync(string name) return cluster; } + public Task GetClusterAsync(string name, string location) + { + throw new NotImplementedException(); + } + public ClusterDetails ChangeClusterSize(string dnsName, string location, int newSize) { return ChangeClusterSizeAsync(dnsName, location, newSize).WaitForResult(); diff --git a/src/ServiceManagement/HDInsight/Commands.HDInsight.Test/packages.config b/src/ServiceManagement/HDInsight/Commands.HDInsight.Test/packages.config index c33660e0c70c..ea5378b3b675 100644 --- a/src/ServiceManagement/HDInsight/Commands.HDInsight.Test/packages.config +++ b/src/ServiceManagement/HDInsight/Commands.HDInsight.Test/packages.config @@ -6,12 +6,12 @@ - + - + diff --git a/src/ServiceManagement/HDInsight/Commands.HDInsight/Cmdlet/AddAzureHDInsightConfigValuesCmdlet.cs b/src/ServiceManagement/HDInsight/Commands.HDInsight/Cmdlet/AddAzureHDInsightConfigValuesCmdlet.cs index 1fb4d0adb1ef..ce1071e69b37 100644 --- a/src/ServiceManagement/HDInsight/Commands.HDInsight/Cmdlet/AddAzureHDInsightConfigValuesCmdlet.cs +++ b/src/ServiceManagement/HDInsight/Commands.HDInsight/Cmdlet/AddAzureHDInsightConfigValuesCmdlet.cs @@ -81,6 +81,7 @@ public AzureHDInsightConfig Config this.command.Config.SubnetName = value.SubnetName; this.command.Config.StormConfiguration.AddRange(value.StormConfiguration); this.command.Config.HBaseConfiguration.ConfigurationCollection.AddRange(value.HBaseConfiguration.ConfigurationCollection); + this.command.Config.SparkConfiguration.AddRange(value.SparkConfiguration); } } @@ -161,6 +162,17 @@ public Hashtable Storm set { this.command.Storm = value; } } + /// + /// Gets or sets a collection of configuration properties to customize the Spark service. + /// + [SuppressMessage("Microsoft.Usage", "CA2227:CollectionPropertiesShouldBeReadOnly", Justification = "Ease of use in Powershell")] + [Parameter(Mandatory = false, HelpMessage = "a collection of configuration properties to customize the Spark service.")] + public Hashtable Spark + { + get { return this.command.Spark; } + set { this.command.Spark = value; } + } + /// /// Gets or sets a collection of configuration properties to customize the HBase service. /// diff --git a/src/ServiceManagement/HDInsight/Commands.HDInsight/Cmdlet/NewAzureHDInsightClusterCmdlet.cs b/src/ServiceManagement/HDInsight/Commands.HDInsight/Cmdlet/NewAzureHDInsightClusterCmdlet.cs index c07abeaed4be..80988691391b 100644 --- a/src/ServiceManagement/HDInsight/Commands.HDInsight/Cmdlet/NewAzureHDInsightClusterCmdlet.cs +++ b/src/ServiceManagement/HDInsight/Commands.HDInsight/Cmdlet/NewAzureHDInsightClusterCmdlet.cs @@ -94,6 +94,8 @@ public AzureHDInsightConfig Config var result = new AzureHDInsightConfig(); result.ClusterSizeInNodes = this.command.ClusterSizeInNodes; result.HeadNodeVMSize = this.command.HeadNodeSize; + result.DataNodeVMSize = this.command.DataNodeSize; + result.ZookeeperNodeVMSize = this.command.ZookeeperNodeSize; result.ClusterType = this.command.ClusterType; result.VirtualNetworkId = this.command.VirtualNetworkId; result.SubnetName = this.command.SubnetName; @@ -112,6 +114,7 @@ public AzureHDInsightConfig Config result.MapReduceConfiguration.CapacitySchedulerConfigurationCollection.AddRange( this.command.MapReduceConfiguration.CapacitySchedulerConfigurationCollection); result.StormConfiguration.AddRange(this.command.StormConfiguration); + result.SparkConfiguration.AddRange(this.command.SparkConfiguration); result.HBaseConfiguration.AdditionalLibraries = this.command.HBaseConfiguration.AdditionalLibraries; result.HBaseConfiguration.ConfigurationCollection.AddRange(this.command.HBaseConfiguration.ConfigurationCollection); return result; @@ -128,6 +131,8 @@ public AzureHDInsightConfig Config this.command.VirtualNetworkId = value.VirtualNetworkId; this.command.SubnetName = value.SubnetName; this.command.HeadNodeSize = value.HeadNodeVMSize; + this.command.DataNodeSize = value.DataNodeVMSize; + this.command.ZookeeperNodeSize = value.ZookeeperNodeVMSize; this.command.DefaultStorageAccountName = value.DefaultStorageAccount.StorageAccountName; this.command.DefaultStorageAccountKey = value.DefaultStorageAccount.StorageAccountKey; this.command.DefaultStorageContainerName = value.DefaultStorageAccount.StorageContainerName; @@ -147,6 +152,7 @@ public AzureHDInsightConfig Config this.command.HiveMetastore = value.HiveMetastore; this.command.OozieMetastore = value.OozieMetastore; this.command.StormConfiguration.AddRange(value.StormConfiguration); + this.command.SparkConfiguration.AddRange(value.SparkConfiguration); this.command.HBaseConfiguration.AdditionalLibraries = value.HBaseConfiguration.AdditionalLibraries; this.command.HBaseConfiguration.ConfigurationCollection.AddRange(value.HBaseConfiguration.ConfigurationCollection); } @@ -257,7 +263,7 @@ public string Version /// [Parameter(Position = 13, Mandatory = false, HelpMessage = "The size of the headnode VM.", ParameterSetName = AzureHdInsightPowerShellConstants.ParameterSetClusterByNameWithSpecificSubscriptionCredentials)] - public NodeVMSize HeadNodeVMSize + public string HeadNodeVMSize { get { return this.command.HeadNodeSize; } set { this.command.HeadNodeSize = value; } @@ -290,6 +296,24 @@ public string SubnetName set { this.command.SubnetName = value; } } + /// + [Parameter(Position = 17, Mandatory = false, HelpMessage = "The size of the datanode VM.", + ParameterSetName = AzureHdInsightPowerShellConstants.ParameterSetClusterByNameWithSpecificSubscriptionCredentials)] + public string DataNodeVMSize + { + get { return this.command.DataNodeSize; } + set { this.command.DataNodeSize = value; } + } + + /// + [Parameter(Position = 18, Mandatory = false, HelpMessage = "The size of the zookeeper VM.", + ParameterSetName = AzureHdInsightPowerShellConstants.ParameterSetClusterByNameWithSpecificSubscriptionCredentials)] + public string ZookeeperNodeVMSize + { + get { return this.command.ZookeeperNodeSize; } + set { this.command.ZookeeperNodeSize = value; } + } + /// protected override void BeginProcessing() { diff --git a/src/ServiceManagement/HDInsight/Commands.HDInsight/Cmdlet/NewAzureHDInsightClusterConfigCmdlet.cs b/src/ServiceManagement/HDInsight/Commands.HDInsight/Cmdlet/NewAzureHDInsightClusterConfigCmdlet.cs index d4a8f64885fb..715edd67618e 100644 --- a/src/ServiceManagement/HDInsight/Commands.HDInsight/Cmdlet/NewAzureHDInsightClusterConfigCmdlet.cs +++ b/src/ServiceManagement/HDInsight/Commands.HDInsight/Cmdlet/NewAzureHDInsightClusterConfigCmdlet.cs @@ -52,7 +52,7 @@ public int ClusterSizeInNodes /// [Parameter(Position = 1, Mandatory = false, HelpMessage = "The size of the head node VMs.", ParameterSetName = AzureHdInsightPowerShellConstants.ParameterSetConfigClusterSizeInNodesOnly)] - public NodeVMSize HeadNodeVMSize + public string HeadNodeVMSize { get { return command.HeadNodeVMSize; } set { command.HeadNodeVMSize = value; } @@ -85,6 +85,23 @@ public string SubnetName set { command.SubnetName = value; } } + /// + [Parameter(Position = 5, Mandatory = false, HelpMessage = "The size of the data node VMs.", + ParameterSetName = AzureHdInsightPowerShellConstants.ParameterSetConfigClusterSizeInNodesOnly)] + public string DataNodeVMSize + { + get { return command.DataNodeVMSize; } + set { command.DataNodeVMSize = value; } + } + + /// + [Parameter(Position = 6, Mandatory = false, HelpMessage = "The size of the zookeper node VMs.", + ParameterSetName = AzureHdInsightPowerShellConstants.ParameterSetConfigClusterSizeInNodesOnly)] + public string ZookeeperNodeVMSize + { + get { return command.ZookeeperNodeVMSize; } + set { command.ZookeeperNodeVMSize = value; } + } /// /// Finishes the execution of the cmdlet by listing the clusters. /// diff --git a/src/ServiceManagement/HDInsight/Commands.HDInsight/Commands.HDInsight.csproj b/src/ServiceManagement/HDInsight/Commands.HDInsight/Commands.HDInsight.csproj index a9ccabd2e293..752c6c3062cf 100644 --- a/src/ServiceManagement/HDInsight/Commands.HDInsight/Commands.HDInsight.csproj +++ b/src/ServiceManagement/HDInsight/Commands.HDInsight/Commands.HDInsight.csproj @@ -56,11 +56,13 @@ ..\..\..\packages\Microsoft.Data.Services.Client.5.6.0\lib\net40\Microsoft.Data.Services.Client.dll - - ..\..\..\packages\Microsoft.Hadoop.Client.1.3.3.2\lib\net40\Microsoft.Hadoop.Client.dll + + False + ..\..\..\packages\Microsoft.Hadoop.Client.1.4.5\lib\net45\Microsoft.Hadoop.Client.dll - - ..\..\..\packages\Microsoft.Hadoop.Client.1.3.3.2\lib\net40\Microsoft.HDInsight.Net.Http.Formatting.dll + + False + ..\..\..\packages\Microsoft.Hadoop.Client.1.4.5\lib\net45\Microsoft.HDInsight.Net.Http.Formatting.dll ..\..\..\packages\Microsoft.Bcl.Async.1.0.168\lib\net40\Microsoft.Threading.Tasks.dll @@ -80,18 +82,27 @@ ..\..\..\packages\Microsoft.WindowsAzure.Common.1.4.1\lib\net45\Microsoft.WindowsAzure.Common.NetFramework.dll + False ..\..\..\packages\Microsoft.WindowsAzure.ConfigurationManager.1.8.0.0\lib\net35-full\Microsoft.WindowsAzure.Configuration.dll - - ..\..\..\packages\Microsoft.WindowsAzure.Management.HDInsight.1.3.3.2\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.dll + + False + ..\..\..\packages\Microsoft.WindowsAzure.Management.HDInsight.1.4.5\lib\net45\Microsoft.WindowsAzure.Management.HDInsight.dll - - ..\..\..\packages\Microsoft.Hadoop.Client.1.3.3.2\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.dll + + False + ..\..\..\packages\Microsoft.WindowsAzure.Management.HDInsight.1.4.5\lib\net45\Microsoft.WindowsAzure.Management.HDInsight.Contracts.dll - - ..\..\..\packages\Microsoft.Hadoop.Client.1.3.3.2\lib\net40\Microsoft.WindowsAzure.Management.HDInsight.Framework.Core.dll + + False + ..\..\..\packages\Microsoft.Hadoop.Client.1.4.5\lib\net45\Microsoft.WindowsAzure.Management.HDInsight.Framework.dll + + + False + ..\..\..\packages\Microsoft.Hadoop.Client.1.4.5\lib\net45\Microsoft.WindowsAzure.Management.HDInsight.Framework.Core.dll + False ..\..\..\packages\WindowsAzure.Storage.3.0.3.0\lib\net40\Microsoft.WindowsAzure.Storage.dll @@ -316,6 +327,7 @@ + Designer diff --git a/src/ServiceManagement/HDInsight/Commands.HDInsight/Model/Commands/BaseCommandInterfaces/IAddAzureHDInsightConfigValuesBase.cs b/src/ServiceManagement/HDInsight/Commands.HDInsight/Model/Commands/BaseCommandInterfaces/IAddAzureHDInsightConfigValuesBase.cs index b67eea38cc33..3ae6cf6f3601 100644 --- a/src/ServiceManagement/HDInsight/Commands.HDInsight/Model/Commands/BaseCommandInterfaces/IAddAzureHDInsightConfigValuesBase.cs +++ b/src/ServiceManagement/HDInsight/Commands.HDInsight/Model/Commands/BaseCommandInterfaces/IAddAzureHDInsightConfigValuesBase.cs @@ -62,6 +62,11 @@ internal interface IAddAzureHDInsightConfigValuesBase /// Hashtable Storm { get; set; } + /// + /// Gets or sets a collection of configuration properties to customize the Spark service. + /// + Hashtable Spark { get; set; } + /// /// Gets or sets a collection of configuration properties to customize the HBase service. /// diff --git a/src/ServiceManagement/HDInsight/Commands.HDInsight/Model/Commands/BaseCommandInterfaces/INewAzureHDInsightClusterConfigBase.cs b/src/ServiceManagement/HDInsight/Commands.HDInsight/Model/Commands/BaseCommandInterfaces/INewAzureHDInsightClusterConfigBase.cs index eec3b73a7fed..fd1e11c9fc1f 100644 --- a/src/ServiceManagement/HDInsight/Commands.HDInsight/Model/Commands/BaseCommandInterfaces/INewAzureHDInsightClusterConfigBase.cs +++ b/src/ServiceManagement/HDInsight/Commands.HDInsight/Model/Commands/BaseCommandInterfaces/INewAzureHDInsightClusterConfigBase.cs @@ -29,7 +29,23 @@ internal interface INewAzureHDInsightClusterConfigBase /// /// The size of the head node VM. /// - NodeVMSize HeadNodeVMSize { get; set; } + string HeadNodeVMSize { get; set; } + + /// + /// Gets or sets the size of the data node VMs. + /// + /// + /// The size of the data node VM. + /// + string DataNodeVMSize { get; set; } + + /// + /// Gets or sets the size of the zookeeper node VMs. + /// + /// + /// The size of the zookeeper node VM. + /// + string ZookeeperNodeVMSize { get; set; } /// /// Gets or sets the type of the cluster. diff --git a/src/ServiceManagement/HDInsight/Commands.HDInsight/Model/Commands/CommandImplementations/AddAzureHDInsightConfigValuesCommand.cs b/src/ServiceManagement/HDInsight/Commands.HDInsight/Model/Commands/CommandImplementations/AddAzureHDInsightConfigValuesCommand.cs index 99e355626660..4f5aab85a432 100644 --- a/src/ServiceManagement/HDInsight/Commands.HDInsight/Model/Commands/CommandImplementations/AddAzureHDInsightConfigValuesCommand.cs +++ b/src/ServiceManagement/HDInsight/Commands.HDInsight/Model/Commands/CommandImplementations/AddAzureHDInsightConfigValuesCommand.cs @@ -33,6 +33,7 @@ public AddAzureHDInsightConfigValuesCommand() this.Hive = new AzureHDInsightHiveConfiguration(); this.Oozie = new AzureHDInsightOozieConfiguration(); this.Storm = new Hashtable(); + this.Spark = new Hashtable(); this.HBase = new AzureHDInsightHBaseConfiguration(); } @@ -52,6 +53,8 @@ public AddAzureHDInsightConfigValuesCommand() public Hashtable Storm { get; set; } + public Hashtable Spark { get; set; } + public AzureHDInsightHBaseConfiguration HBase { get; set; } public override Task EndProcessing() @@ -65,6 +68,7 @@ public override Task EndProcessing() this.Config.HiveConfiguration.ConfigurationCollection.AddRange(this.Hive.Configuration.ToKeyValuePairs()); this.Config.OozieConfiguration.ConfigurationCollection.AddRange(this.Oozie.Configuration.ToKeyValuePairs()); this.Config.StormConfiguration.AddRange(this.Storm.ToKeyValuePairs()); + this.Config.SparkConfiguration.AddRange(this.Spark.ToKeyValuePairs()); this.Config.HBaseConfiguration.ConfigurationCollection.AddRange(this.HBase.Configuration.ToKeyValuePairs()); if (this.Hive.AdditionalLibraries != null) diff --git a/src/ServiceManagement/HDInsight/Commands.HDInsight/Model/Commands/CommandImplementations/NewAzureHDInsightClusterCommand.cs b/src/ServiceManagement/HDInsight/Commands.HDInsight/Model/Commands/CommandImplementations/NewAzureHDInsightClusterCommand.cs index 60b8716a808d..8b2eb5b05984 100644 --- a/src/ServiceManagement/HDInsight/Commands.HDInsight/Model/Commands/CommandImplementations/NewAzureHDInsightClusterCommand.cs +++ b/src/ServiceManagement/HDInsight/Commands.HDInsight/Model/Commands/CommandImplementations/NewAzureHDInsightClusterCommand.cs @@ -37,6 +37,7 @@ public NewAzureHDInsightClusterCommand() this.HiveConfiguration = new HiveConfiguration(); this.OozieConfiguration = new OozieConfiguration(); this.StormConfiguration = new ConfigValuesCollection(); + this.SparkConfiguration = new ConfigValuesCollection(); this.HBaseConfiguration = new HBaseConfiguration(); } @@ -46,7 +47,13 @@ public NewAzureHDInsightClusterCommand() public int ClusterSizeInNodes { get; set; } /// - public NodeVMSize HeadNodeSize { get; set; } + public string HeadNodeSize { get; set; } + + /// + public string DataNodeSize { get; set; } + + /// + public string ZookeeperNodeSize { get; set; } /// public ConfigValuesCollection CoreConfiguration { get; set; } @@ -91,6 +98,9 @@ public NewAzureHDInsightClusterCommand() /// public ConfigValuesCollection StormConfiguration { get; set; } + /// + public ConfigValuesCollection SparkConfiguration { get; set; } + /// public HBaseConfiguration HBaseConfiguration { get; set; } @@ -111,14 +121,15 @@ public override async Task EndProcessing() { IHDInsightClient client = this.GetClient(); client.ClusterProvisioning += this.ClientOnClusterProvisioning; - ClusterCreateParameters createClusterRequest = this.GetClusterCreateParameters(); + ClusterCreateParametersV2 createClusterRequest = this.GetClusterCreateParameters(); var cluster = await client.CreateClusterAsync(createClusterRequest); this.Output.Add(new AzureHDInsightCluster(cluster)); } - internal ClusterCreateParameters GetClusterCreateParameters() + internal ClusterCreateParametersV2 GetClusterCreateParameters() { - var createClusterRequest = new ClusterCreateParameters(); + var createClusterRequest = new ClusterCreateParametersV2(); + createClusterRequest.Name = this.Name; createClusterRequest.Version = this.Version; createClusterRequest.Location = this.Location; @@ -134,9 +145,10 @@ internal ClusterCreateParameters GetClusterCreateParameters() createClusterRequest.OozieConfiguration.AdditionalSharedLibraries = this.OozieConfiguration.AdditionalSharedLibraries; createClusterRequest.OozieConfiguration.AdditionalActionExecutorLibraries = this.OozieConfiguration.AdditionalActionExecutorLibraries; createClusterRequest.StormConfiguration.AddRange(this.StormConfiguration); + createClusterRequest.SparkConfiguration.AddRange(this.SparkConfiguration); createClusterRequest.HBaseConfiguration.AdditionalLibraries = this.HBaseConfiguration.AdditionalLibraries; createClusterRequest.HBaseConfiguration.ConfigurationCollection.AddRange(this.HBaseConfiguration.ConfigurationCollection); - createClusterRequest.HeadNodeSize = this.HeadNodeSize; + createClusterRequest.DefaultStorageAccountName = this.DefaultStorageAccountName; createClusterRequest.DefaultStorageAccountKey = this.DefaultStorageAccountKey; createClusterRequest.DefaultStorageContainer = this.DefaultStorageContainerName; @@ -171,6 +183,22 @@ internal ClusterCreateParameters GetClusterCreateParameters() this.OozieMetastore.Credential.UserName, this.OozieMetastore.Credential.GetCleartextPassword()); } + + if (!string.IsNullOrEmpty(this.HeadNodeSize)) + { + createClusterRequest.HeadNodeSize = this.HeadNodeSize; + } + + if (!string.IsNullOrEmpty(this.DataNodeSize)) + { + createClusterRequest.DataNodeSize = this.DataNodeSize; + } + + if (!string.IsNullOrEmpty(this.ZookeeperNodeSize)) + { + createClusterRequest.ZookeeperNodeSize = this.ZookeeperNodeSize; + } + return createClusterRequest; } diff --git a/src/ServiceManagement/HDInsight/Commands.HDInsight/Model/Commands/CommandImplementations/NewAzureHDInsightClusterConfigCommand.cs b/src/ServiceManagement/HDInsight/Commands.HDInsight/Model/Commands/CommandImplementations/NewAzureHDInsightClusterConfigCommand.cs index 1d61c95d12e4..316ebed82f76 100644 --- a/src/ServiceManagement/HDInsight/Commands.HDInsight/Model/Commands/CommandImplementations/NewAzureHDInsightClusterConfigCommand.cs +++ b/src/ServiceManagement/HDInsight/Commands.HDInsight/Model/Commands/CommandImplementations/NewAzureHDInsightClusterConfigCommand.cs @@ -33,12 +33,24 @@ public int ClusterSizeInNodes set { this.config.ClusterSizeInNodes = value; } } - public NodeVMSize HeadNodeVMSize + public string HeadNodeVMSize { get { return this.config.HeadNodeVMSize; } set { this.config.HeadNodeVMSize = value; } } + public string DataNodeVMSize + { + get { return this.config.DataNodeVMSize; } + set { this.config.DataNodeVMSize = value; } + } + + public string ZookeeperNodeVMSize + { + get { return this.config.ZookeeperNodeVMSize; } + set { this.config.ZookeeperNodeVMSize = value; } + } + public ClusterType ClusterType { get { return this.config.ClusterType; } diff --git a/src/ServiceManagement/HDInsight/Commands.HDInsight/Model/Commands/CommandInterfaces/INewAzureHDInsightClusterCommand.cs b/src/ServiceManagement/HDInsight/Commands.HDInsight/Model/Commands/CommandInterfaces/INewAzureHDInsightClusterCommand.cs index 188412343f03..8fbe5a6d9f8c 100644 --- a/src/ServiceManagement/HDInsight/Commands.HDInsight/Model/Commands/CommandInterfaces/INewAzureHDInsightClusterCommand.cs +++ b/src/ServiceManagement/HDInsight/Commands.HDInsight/Model/Commands/CommandInterfaces/INewAzureHDInsightClusterCommand.cs @@ -49,7 +49,11 @@ internal interface INewAzureHDInsightClusterCommand : IAzureHDInsightCommand /// The size of the head node VM. /// - public NodeVMSize HeadNodeVMSize { get; set; } + public string HeadNodeVMSize { get; set; } + + /// + /// Gets or sets the size of the head node VM. + /// + /// + /// The size of the head node VM. + /// + public string DataNodeVMSize { get; set; } + + /// + /// Gets or sets the size of the head node VM. + /// + /// + /// The size of the head node VM. + /// + public string ZookeeperNodeVMSize { get; set; } /// /// Gets or sets the size of the cluster in data nodes. @@ -130,6 +149,11 @@ public AzureHDInsightConfig() /// public ConfigValuesCollection StormConfiguration { get; private set; } + /// + /// Gets a collection of configuration properties to customize the Spark service. + /// + public ConfigValuesCollection SparkConfiguration { get; private set; } + /// /// Gets a collection of configuration properties to customize the HBase service. /// diff --git a/src/ServiceManagement/HDInsight/Commands.HDInsight/Model/GetAzureHDInsightClusters/AzureHDInsightClusterCommandBase.cs b/src/ServiceManagement/HDInsight/Commands.HDInsight/Model/GetAzureHDInsightClusters/AzureHDInsightClusterCommandBase.cs index a037f725e205..bd2d0cd3654a 100644 --- a/src/ServiceManagement/HDInsight/Commands.HDInsight/Model/GetAzureHDInsightClusters/AzureHDInsightClusterCommandBase.cs +++ b/src/ServiceManagement/HDInsight/Commands.HDInsight/Model/GetAzureHDInsightClusters/AzureHDInsightClusterCommandBase.cs @@ -47,6 +47,12 @@ internal IHDInsightClient GetClient() this.CurrentSubscription, client.GetEnvironmentOrDefault(this.CurrentSubscription.Environment), client.Profile); + + if (this.Endpoint.IsNotNull()) + { + subscriptionCredentials.Endpoint = this.Endpoint; + } + var clientInstance = ServiceLocator.Instance.Locate().Create(subscriptionCredentials); clientInstance.SetCancellationSource(this.tokenSource); if (this.Logger.IsNotNull()) diff --git a/src/ServiceManagement/HDInsight/Commands.HDInsight/app.config b/src/ServiceManagement/HDInsight/Commands.HDInsight/app.config new file mode 100644 index 000000000000..44298137ae7d --- /dev/null +++ b/src/ServiceManagement/HDInsight/Commands.HDInsight/app.config @@ -0,0 +1,11 @@ + + + + + + + + + + + \ No newline at end of file diff --git a/src/ServiceManagement/HDInsight/Commands.HDInsight/packages.config b/src/ServiceManagement/HDInsight/Commands.HDInsight/packages.config index 105f7505faea..a338b46eb816 100644 --- a/src/ServiceManagement/HDInsight/Commands.HDInsight/packages.config +++ b/src/ServiceManagement/HDInsight/Commands.HDInsight/packages.config @@ -6,12 +6,12 @@ - + - +