Merge pull request #215 from glennsarti/modules-4257-modify-instance-and-features

(MODULES-4257) Modify instance and features for SQL Server 2016

Author: hunner

README.md

@@ -80,13 +80,6 @@ This example creates the same MS SQL instance as shown above with additional opt
 
 ### Install SQL Server tools and features not specific to a SQL Server instance
 
-```puppet
-sqlserver_features { 'Generic Features':
-  source   => 'E:/',
-  features => ['Tools'],
-}
-```
-
 ```puppet
 sqlserver_features { 'Generic Features':
   source   => 'E:/',
@@ -203,8 +196,10 @@ Default: 'present'.
 ##### `features`
 
 *Required.*
-  
-Specifies one or more features to manage. Valid options: 'BC', 'Conn', 'SSMS', 'ADV_SSMS', 'SDK', 'IS', 'MDS', and 'Tools' (the Tools feature includes SSMS, ADV_SSMS, and Conn).
+
+Specifies one or more features to manage. Valid options: 'BC', 'Conn', 'SSMS', 'ADV_SSMS', 'SDK', 'IS', 'MDS', 'BOL', 'DREPLAY_CTLR', 'DREPLAY_CLT'.
+
+The 'Tools' feature is deprecated.  Instead specify 'BC', 'SSMS', 'ADV_SSMS', 'Conn', and 'SDK' explicitly.
 
 ##### `install_switches`
 
@@ -294,7 +289,9 @@ Default: 'present'.
 
 ##### `features`
 
-*Required.* Specifies one or more features to manage. The list of top-level features includes 'SQL', 'AS', and 'RS'. The 'SQL' feature includes the Database Engine, Replication, Full-Text, and Data Quality Services (DQS) server. Valid options: an array containing one or more of the strings 'SQL', 'SQLEngine', 'Replication', 'FullText', 'DQ', 'AS', and 'RS'.
+*Required.* Specifies one or more features to manage. The list of top-level features includes 'AS' and 'RS'. Valid options: an array containing one or more of the strings 'SQL', 'SQLEngine', 'Replication', 'FullText', 'DQ', 'AS', 'RS', 'POLYBASE', and 'ADVANCEDANALYTICS'.
+
+The 'SQL' feature is deprecated.  Instead specify 'DQ', 'FullText', 'Replication', and 'SQLEngine' explicitly.
 
 ##### `install_switches`
 

lib/puppet/provider/sqlserver_features/mssql.rb

@@ -1,5 +1,7 @@
 require 'json'
 require File.expand_path(File.join(File.dirname(__FILE__), '..', 'sqlserver'))
+require File.expand_path(File.join(File.dirname(__FILE__), '..', '..', '..', 'puppet_x/sqlserver/server_helper'))
+require File.expand_path(File.join(File.dirname(__FILE__), '..', '..', '..', 'puppet_x/sqlserver/features'))
 
 FEATURE_RESERVED_SWITCHES =
   %w(AGTSVCACCOUNT AGTSVCPASSWORD ASSVCACCOUNT AGTSVCPASSWORD PID
@@ -10,18 +12,25 @@ def self.instances
     instances = []
     result = Facter.value(:sqlserver_features)
     debug "Parsing result #{result}"
-    result = !result[SQL_2014].empty? ? result[SQL_2014] : result[SQL_2012]
-    if !result.empty?
-      existing_instance = {:name => "Generic Features",
-                           :ensure => :present,
-                           :features => result.sort
+
+    ALL_SQL_VERSIONS.each do |sql_version|
+      next if result[sql_version].empty?
+      instance_props = {:name => "Generic Features",
+                        :ensure => :present,
+                        :features => result[sql_version].sort
       }
-      debug "Parsed features = #{existing_instance[:features]}"
+      debug "Parsed features = #{instance_props[:features]}"
 
-      instance = new(existing_instance)
+      instance = new(instance_props)
       debug "Created instance #{instance}"
       instances << instance
+
+      # Due to MODULES-5060 we can only output one feature set.  If we output
+      # multiple then it is not possible to install or uninstall due to multiple
+      # resources with the same name.
+      break
     end
+
     instances
   end
 
@@ -82,8 +91,7 @@ def create_temp_for_install_switch
       config_file = ["[OPTIONS]"]
       @resource[:install_switches].each_pair do |k, v|
         if FEATURE_RESERVED_SWITCHES.include? k
-          warn("Reserved switch [#{k}] found for `install_switches`, please know the provided value
-may be overridden by some command line arguments")
+          warn("Reserved switch [#{k}] found for `install_switches`, please know the provided value may be overridden by some command line arguments")
         end
         if v.is_a?(Numeric) || (v.is_a?(String) && v =~ /^(true|false|1|0)$/i)
           config_file << "#{k}=#{v}"
@@ -111,7 +119,11 @@ def create
       warn "Uninstalling all sql server features not tied into an instance because an empty array was passed, please use ensure absent instead."
       destroy
     else
-      installNet35(@resource[:windows_feature_source])
+      instance_version = PuppetX::Sqlserver::ServerHelper.sql_version_from_install_source(@resource[:source])
+      Puppet.debug("Installation source detected as version #{instance_version}") unless instance_version.nil?
+
+      installNet35(@resource[:windows_feature_source]) unless instance_version == SQL_2016
+
       debug "Installing features #{@resource[:features]}"
       add_features(@resource[:features])
       @property_hash[:features] = @resource[:features]

lib/puppet/provider/sqlserver_instance/mssql.rb

@@ -1,6 +1,7 @@
 require 'json'
 require File.expand_path(File.join(File.dirname(__FILE__), '..', 'sqlserver'))
-
+require File.expand_path(File.join(File.dirname(__FILE__), '..', '..', '..', 'puppet_x/sqlserver/server_helper'))
+require File.expand_path(File.join(File.dirname(__FILE__), '..', '..', '..', 'puppet_x/sqlserver/features'))
 
 INSTANCE_RESERVED_SWITCHES =
   %w(AGTSVCACCOUNT AGTSVCPASSWORD ASSVCACCOUNT AGTSVCPASSWORD PID
@@ -70,19 +71,12 @@ def create
       warn "Uninstalling all features for instance #{@resource[:name]} because an empty array was passed, please use ensure absent instead."
       destroy
     else
-      installNet35(@resource[:windows_feature_source])
+      instance_version = PuppetX::Sqlserver::ServerHelper.sql_version_from_install_source(@resource[:source])
+      Puppet.debug("Installation source detected as version #{instance_version}") unless instance_version.nil?
+
+      installNet35(@resource[:windows_feature_source]) unless instance_version == SQL_2016
+
       add_features(@resource[:features])
-      # cmd_args = build_cmd_args(@resource[:features])
-      # begin
-      #   config_file = create_temp_for_install_switch
-      #   cmd_args << "/ConfigurationFile=\"#{config_file.path}\"" unless config_file.nil?
-      #   try_execute(cmd_args)
-      # ensure
-      #   if config_file
-      #     config_file.close
-      #     config_file.unlink
-      #   end
-      # end
     end
   end
 
@@ -91,8 +85,7 @@ def create_temp_for_install_switch
       config_file = ["[OPTIONS]"]
       @resource[:install_switches].each_pair do |k, v|
         if INSTANCE_RESERVED_SWITCHES.include? k
-          warn("Reserved switch [#{k}] found for `install_switches`, please know the provided value
-may be overridden by some command line arguments")
+          warn("Reserved switch [#{k}] found for `install_switches`, please know the provided value may be overridden by some command line arguments")
         end
         if v.is_a?(Numeric) || (v.is_a?(String) && v =~ /^(true|false|1|0)$/i)
           config_file << "#{k}=#{v}"

lib/puppet/type/sqlserver_features.rb

@@ -36,11 +36,11 @@
 
   newproperty(:features, :array_matching => :all) do
     desc 'Specifies features to install, uninstall, or upgrade. The list of top-level features include
-         Tools, BC, Conn, SSMS, ADV_SSMS, SDK, IS and MDS. The Tools feature will install Management
-          Tools, Books online components, SQL Server Data Tools, and other shared components.'
-    newvalues(:Tools, :BC, :Conn, :SSMS, :ADV_SSMS, :SDK, :IS, :MDS)
+         BC, Conn, SSMS, ADV_SSMS, SDK, IS and MDS.'
+    newvalues(:Tools, :BC, :Conn, :SSMS, :ADV_SSMS, :SDK, :IS, :MDS, :BOL, :DREPLAY_CTLR, :DREPLAY_CLT)
     munge do |value|
       if PuppetX::Sqlserver::ServerHelper.is_super_feature(value)
+        Puppet.deprecation_warning("Using #{value} is deprecated for features in sql_features resources")
         PuppetX::Sqlserver::ServerHelper.get_sub_features(value).collect { |v| v.to_s }
       else
         value

lib/puppet/type/sqlserver_instance.rb

@@ -27,11 +27,11 @@
 
   newproperty(:features, :array_matching => :all) do
     desc 'Specifies features to install, uninstall, or upgrade. The list of top-level features include
-          SQL, SQLEngine, Replication, FullText, DQ AS, and RS. The SQL feature will install the Database Engine,
-          Replication, Full-Text, and Data Quality Services (DQS) server.'
-    newvalues(:SQL, :SQLEngine, :Replication, :FullText, :DQ, :AS, :RS)
+          SQLEngine, Replication, FullText, DQ AS, and RS.'
+    newvalues(:SQL, :SQLEngine, :Replication, :FullText, :DQ, :AS, :RS, :POLYBASE, :ADVANCEDANALYTICS)
     munge do |value|
       if PuppetX::Sqlserver::ServerHelper.is_super_feature(value)
+        Puppet.deprecation_warning("Using #{value} is deprecated for features in sql_instance resources")
         PuppetX::Sqlserver::ServerHelper.get_sub_features(value).collect { |v| v.to_s }
       else
         value

lib/puppet_x/sqlserver/features.rb

@@ -1,15 +1,31 @@
 SQL_2012 ||= 'SQL_2012'
 SQL_2014 ||= 'SQL_2014'
+SQL_2016 ||= 'SQL_2016'
+
+ALL_SQL_VERSIONS = [SQL_2012, SQL_2014, SQL_2016]
 
 module PuppetX
   module Sqlserver
     # https://msdn.microsoft.com/en-us/library/ms143786.aspx basic feature docs
     class Features
       private
 
-      SQL_WMI_PATH ||= {
-        SQL_2012 => 'ComputerManagement11',
-        SQL_2014 => 'ComputerManagement12',
+      SQL_CONFIGURATION ||= {
+        SQL_2012 => {
+          :major_version => 11,
+          :wmi_path => 'ComputerManagement11',
+          :registry_path => '110',
+        },
+        SQL_2014 => {
+          :major_version => 12,
+          :wmi_path => 'ComputerManagement12',
+          :registry_path => '120',
+        },
+        SQL_2016 => {
+          :major_version => 13,
+          :wmi_path => 'ComputerManagement13',
+          :registry_path => '130',
+        }
       }
 
       SQL_REG_ROOT ||= 'Software\Microsoft\Microsoft SQL Server'
@@ -20,11 +36,12 @@ class Features
 
       def self.connect(version)
         require 'win32ole'
-        ver = SQL_WMI_PATH[version]
+        ver = SQL_CONFIGURATION[version][:wmi_path]
         context = WIN32OLE.new('WbemScripting.SWbemNamedValueSet')
         context.Add("__ProviderArchitecture", 64)
         locator = WIN32OLE.new('WbemScripting.SWbemLocator')
-        locator.ConnectServer('', "root/Microsoft/SqlServer/#{ver}", '', '', nil, nil, nil, context)
+        # WMI Path must use backslashes.   Ruby 2.3 can cause crashes with forward slashes
+        locator.ConnectServer(nil, "root\\Microsoft\\SqlServer\\#{ver}", nil, nil, nil, nil, nil, context)
       end
 
       def self.get_parent_path(key_path)
@@ -118,9 +135,11 @@ def self.get_instance_features(reg_root, instance_name)
           # also reg Replication/IsInstalled set to 1
           'SQL_Replication_Core_Inst' => 'Replication', # SQL Server Replication
           # also WMI: SqlService WHERE SQLServiceType = 1 # MSSQLSERVER
-          'SQL_Engine_Core_Inst' => 'SQLEngine', # Database Engine Services
-          'SQL_FullText_Adv' => 'FullText', # Full-Text and Semantic Extractions for Search
-          'SQL_DQ_Full' => 'DQ', # Data Quality Services
+          'SQL_Engine_Core_Inst'      => 'SQLEngine', # Database Engine Services
+          'SQL_FullText_Adv'          => 'FullText', # Full-Text and Semantic Extractions for Search
+          'SQL_DQ_Full'               => 'DQ', # Data Quality Services
+          'sql_inst_mr'               => 'ADVANCEDANALYTICS', # R Services (In-Database)
+          'SQL_Polybase_Core_Inst'    => 'POLYBASE', # PolyBase Query Service for External Data
         }
 
         feat_root = "#{reg_root}\\ConfigurationState"
@@ -144,18 +163,25 @@ def self.get_instance_features(reg_root, instance_name)
 
       def self.get_shared_features(version)
         shared_features = {
-          'Connectivity_Full' => 'Conn', # Client Tools Connectivity
-          'SDK_Full' => 'SDK', # Client Tools SDK
-          'MDSCoreFeature' => 'MDS', # Master Data Services
-          'Tools_Legacy_Full' => 'BC', # Client Tools Backwards Compatibility
-          'SQL_SSMS_Full' => 'ADV_SSMS', # Management Tools - Complete
-          'SQL_SSMS_Adv' => 'SSMS', # Management Tools - Basic
+          'Connectivity_Full'      => 'Conn', # Client Tools Connectivity
+          'SDK_Full'               => 'SDK', # Client Tools SDK
+          'MDSCoreFeature'         => 'MDS', # Master Data Services
+          'Tools_Legacy_Full'      => 'BC', # Client Tools Backwards Compatibility
+          'SQL_SSMS_Full'          => 'ADV_SSMS', # Management Tools - Complete (Does not exist in SQL 2016)
+          'SQL_SSMS_Adv'           => 'SSMS', # Management Tools - Basic  (Does not exist in SQL 2016)
+          'SQL_DQ_CLIENT_Full'     => 'DQC', # Data Quality Client
+          'SQL_BOL_Components'     => 'BOL', # Documentation Components
+          'SQL_DReplay_Controller' => 'DREPLAY_CTLR', # Distributed Replay Controller
+          'SQL_DReplay_Client'     => 'DREPLAY_CLT', # Distributed Replay Client
+          'sql_shared_mr'          => 'SQL_SHARED_MR', # R Server (Standalone)
+
           # also WMI: SqlService WHERE SQLServiceType = 4 # MsDtsServer
-          'SQL_DTS_Full' => 'IS', # Integration Services
+          'SQL_DTS_Full'           => 'IS', # Integration Services
           # currently ignoring Reporting Services Shared
+          # currently ignoring R Server Standalone
         }
 
-        reg_ver = (version == SQL_2014 ? '120' : '110')
+        reg_ver = SQL_CONFIGURATION[version][:registry_path]
         reg_root = "#{SQL_REG_ROOT}\\#{reg_ver}\\ConfigurationState"
 
         get_sql_reg_val_features(reg_root, shared_features)
@@ -185,11 +211,27 @@ def self.get_shared_features(version)
       #   }
       # }
       def self.get_instances
-        version_instance_map = [SQL_2012, SQL_2014]
+        version_instance_map = ALL_SQL_VERSIONS
           .map do |version|
+            major_version = SQL_CONFIGURATION[version][:major_version]
+
             instances = get_instance_names_by_ver(version)
               .map { |name| [ name, get_instance_info(version, name) ] }
 
+            # Instance names are unique on a single host, but not for a particular SQL Server version therefore
+            # it's possible to request information for a valid instance_name but not for version.  In this case
+            # we just reject any instances that have no information
+            instances.reject! { |value| value[1].nil? }
+
+            # Unfortunately later SQL versions can return previous version SQL instances.  We can weed these out
+            # by inspecting the major version of the instance_version
+            instances.reject! do |value|
+              return true if value[1]['version'].nil?
+              ver = Gem::Version.new(value[1]['version'])
+              # Segment 0 is the major version number of the SQL Instance
+              ver.segments[0] != major_version
+            end
+
             [ version, Hash[instances] ]
           end
 
@@ -203,10 +245,9 @@ def self.get_instances
       #   "SQL_2014" => []
       # }
       def self.get_features
-        {
-          SQL_2012 => get_shared_features(SQL_2012),
-          SQL_2014 => get_shared_features(SQL_2014),
-        }
+        features = {}
+        ALL_SQL_VERSIONS.each { |version| features[version] = get_shared_features(version) }
+        features
       end
 
       # returns a hash containing instance details
@@ -225,8 +266,13 @@ def self.get_features
       #     "RS"
       #   ]
       # }
-      def self.get_instance_info(version = SQL_2014, instance_name)
+      def self.get_instance_info(version, instance_name)
+        return nil if version.nil?
         sql_instance = get_wmi_instance_info(version, instance_name)
+        # Instance names are unique on a single host, but not for a particular SQL Server version therefore
+        # it's possible to request information for a valid instance_name but not for version.  In this case
+        # we just return nil.
+        return nil if sql_instance['reg_root'].nil?
         feats = get_instance_features(sql_instance['reg_root'], sql_instance['name'])
         sql_instance.merge({'features' => feats})
       end

lib/puppet_x/sqlserver/server_helper.rb

@@ -21,6 +21,33 @@ def self.is_domain_or_local_user?(user, hostname)
           true
         end
       end
+
+      # Returns either SQL_2016, SQL_2014 or SQL_2012 if it can determine the SQL Version from the install source
+      # Returns nil if it can not be determined
+      def self.sql_version_from_install_source(source_dir)
+        # Attempt to read the Mediainfo.xml file in the root of the install media
+        media_file = File.expand_path("#{source_dir}/MediaInfo.xml")
+        return nil unless File.exist?(media_file)
+        # As we don't have a XML parser easily, just use simple text matching to find the following XML element. This
+        # also means we can just ignore BOM markers etc.
+        #     <Property Id="BaselineVersion" Value="xx.yyy.zz." />
+        content = File.read(media_file)
+        index1 = content.index('"BaselineVersion"')
+        return nil if index1.nil?
+        index2 = content.index('/>',index1) 
+        return nil if index2.nil?
+        content = content.slice(index1 + 18, index2 - index1 - 18)
+        # Extract the version number from the text snippet
+        #     Value="xx.yyy.zz."
+        ver = content.match('"(.+)"')
+        return nil if ver.nil?
+
+        return SQL_2016 if ver[1].start_with?('13.')
+        return SQL_2014 if ver[1].start_with?('12.')
+        return SQL_2012 if ver[1].start_with?('11.')
+
+        nil
+      end
     end
   end
 end