CI improvements (#525)

* Lock Bundler to v2.3.26

This is the last version that supports Ruby 2.4 and 2.5.

* Remove fix to use rake-compiler-dock on Windows for libiconv

The original author states that the code I removed is for using virtualbox on Windows to compile the gem. I read up on this and while rake-compiler-dock still supports using docker-machine to compile gems, the project has been discontinued in favor of Docker Desktop, which nowadays only works with Docker in WSL2.

* Bump openssl to v1.1.1s

1.1.1d gives an error when compiling against i686-w64-mingw32. Upgrading it seems to work.

* Define task to compile native extensions

* Skip dead connection test on Windows

* Extend pipeline to test gem on Windows

* Package gems on CI before Windows tests

* Ensure test container uses Ruby version defined in CI matrix

* Test against Ruby 2.4

* Publish test results to CircleCI

* Rename artifact for gems

* Restore cross-compiled code from final gem file

* Store precompiled gem into workspace instead of cache

this seems to be the more appropriate type of data store according to the CircleCI documentation.

* Fix order for compiling ports

* Disable progress bar for Invoke-WebRequest

* Add `/ALLUSERS` option to RubyInstaller

Looks like a new option that has been introduced in the most recent 2.7 version and explains the timeout we see with the Ruby installation on CircleCI.
https://github.com/oneclick/rubyinstaller2/wiki/FAQ#user-content-install-mode

Older versions seem to ignore the parameter and run the installation as usual.

Author: andyundso

.circleci/config.yml

@@ -2,14 +2,15 @@ version: 2.1
 
 orbs:
   win: circleci/[email protected]
+  ruby: circleci/[email protected]
 
 jobs:
   test_linux:
     parameters:
       ruby_version:
         description: 'version tag for the cimg/ruby container'
         type: string
-    
+
     machine:
       image: ubuntu-2004:current
 
@@ -19,11 +20,13 @@ jobs:
 
       - run:
           name: start docker-compose build environment
-          command: | 
+          command: |
             sudo ./test/bin/setup_volume_permissions.sh
             docker-compose up -d
             echo "Waiting for containers to start..."
             sleep 10
+          environment:
+            RUBY_VERSION: << parameters.ruby_version >>
 
       - run:
           name: install sql prereqs
@@ -58,7 +61,10 @@ jobs:
       - run:
           name: test gem
           command: |
-             docker exec cimg_ruby bash -c 'bundle exec rake test'
+            docker exec cimg_ruby bash -c 'bundle exec rake test'
+
+      - store_test_results:
+          path: test/reports
 
   test_windows:
     parameters:
@@ -84,6 +90,8 @@ jobs:
       - run:
           name: download and install ruby devkit
           command: |
+            $ProgressPreference='SilentlyContinue'
+
             $uri = 'https://api.github.com/repos/oneclick/rubyinstaller2/tags?per_page=200'
             $releases = ((Invoke-WebRequest $uri) | ConvertFrom-Json).name | select-string -Pattern '<< parameters.ruby_version >>'
             $target_release = (($releases | Sort-Object -Descending)[0] | Out-String).Trim()
@@ -93,7 +101,9 @@ jobs:
 
             [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
             Invoke-WebRequest -UseBasicParsing -uri $download_uri -OutFile ruby-setup.exe
-            .\ruby-setup.exe /VERYSILENT /NORESTART /DIR=C:/Ruby<< parameters.ruby_version >>-x64
+            
+            echo "Download finished, starting installation of $target_version"
+            .\ruby-setup.exe /VERYSILENT /NORESTART /ALLUSERS /DIR=C:/Ruby<< parameters.ruby_version >>-x64
 
       - run:
           name: ruby diagnostics
@@ -108,11 +118,10 @@ jobs:
             rm .\ruby-setup.exe
 
       - run:
-          name: update build env
+          name: install bundler
           command: |
             $Env:PATH = "C:\\Ruby<< parameters.ruby_version >>-x64\\bin;$Env:PATH"
-            ridk install 2
-            gem install bundler
+            gem install bundler -v 2.3.26
 
       - checkout
 
@@ -127,57 +136,143 @@ jobs:
           name: bundle install gems
           command: |
             $Env:PATH = "C:\\Ruby<< parameters.ruby_version >>-x64\\bin;$Env:PATH"
-            bundle install
+            bundle install --path vendor/bundle
 
       - save_cache:
           name: save gem cache
           paths:
             - ./vendor/bundle
           key: v1-bundle-<< parameters.ruby_version >>-{{ .Branch }}-{{ checksum "tiny_tds.gemspec" }}
 
+      - attach_workspace:
+          at: artifacts
+
       - run:
-          name: build openssl
-          no_output_timeout: 30m
+          name: install native gem and restore cross-compiled code from gem
           command: |
             $Env:PATH = "C:\\Ruby<< parameters.ruby_version >>-x64\\bin;$Env:PATH"
-            bundle exec rake ports:openssl
+            $rubyArchitecture = (ruby -e 'puts RUBY_PLATFORM').Trim()
+            $gemVersion = (Get-Content VERSION).Trim()
+
+            gem install --local --install-dir=./tmp "artifacts/gems/tiny_tds-$gemVersion-$rubyArchitecture.gem"
+
+            # Restore precompiled code
+            $source = (Resolve-Path ".\tmp\gems\tiny_tds-$gemVersion-$rubyArchitecture\lib\tiny_tds").Path
+            $destination = (Resolve-Path ".\lib\tiny_tds").Path
+            Get-ChildItem $source -Recurse -Exclude "*.rb" | Copy-Item -Destination {Join-Path $destination $_.FullName.Substring($source.length)}
+            
+            # Restore ports
+            Copy-Item -Path ".\tmp\gems\tiny_tds-$gemVersion-$rubyArchitecture\ports" -Destination "." -Recurse
+
+      - restore_cache:
+          name: restore mssql installation file
+          key: downloads-{{ checksum "test/bin/install-mssql.ps1" }}
 
       - run:
-          name: build libiconv
-          no_output_timeout: 30m
+          name: setup mssql
           command: |
-            $Env:PATH = "C:\\Ruby<< parameters.ruby_version >>-x64\\bin;$Env:PATH"
-            bundle exec rake ports:libiconv
+            .\test\bin\install-mssql.ps1
+
+      - save_cache:
+          name: save downloads cache
+          paths:
+            - C:\Downloads
+          key: downloads-{{ checksum "test/bin/install-mssql.ps1" }}
 
       - run:
-          name: build freetds
-          no_output_timeout: 30m
+          name: install toxiproxy-server
           command: |
-            $Env:PATH = "C:\\Ruby<< parameters.ruby_version >>-x64\\bin;$Env:PATH"
-            bundle exec rake ports:freetds
+            choco install toxiproxy-server --version=2.5.0 -y
+            Start-Process toxiproxy-server
 
       - run:
-          name: build gem
-          no_output_timeout: 30m
+          name: test gem
           command: |
             $Env:PATH = "C:\\Ruby<< parameters.ruby_version >>-x64\\bin;$Env:PATH"
-            bundle exec rake ports
+            bundle exec rake test
+          environment:
+            TOXIPROXY_HOST: "localhost"
+
+      - store_test_results:
+          path: test/reports
+
+  cross_compile_gem:
+    machine:
+      image: ubuntu-2004:current
+
+    steps:
+      - ruby/install:
+          version: '2.7'
+      - checkout
+      - restore_cache:
+          name: restore gem cache
+          keys:
+            - v1-bundle-{{ .Branch }}-{{ checksum "tiny_tds.gemspec" }}
+            - v1-bundle-{{ .Branch }}-
+            - v1-bundle-
+
+      - run:
+          name: bundle install gems
+          command: |
+            bundle install --path vendor/bundle
+
+      - save_cache:
+          name: save gem cache
+          paths:
+            - ./vendor/bundle
+          key: v1-bundle-{{ .Branch }}-{{ checksum "tiny_tds.gemspec" }}
+
+      - run:
+          name: Write used versions for ports into file
+          command: |
+            bundle exec rake ports:version_file
+
+      - restore_cache:
+          name: restore ports cache
+          keys:
+            - ports-{{ checksum ".ports_versions" }}
+            - ports-
+
+      - run:
+          name: Build gems
+          command: |
+            bundle exec rake gem
+            bundle exec rake gem:native
+
+      - run:
+          name: Move gems into separate directory before caching
+          command: |
+            mkdir -p artifacts/gems
+            mv pkg/*.gem artifacts/gems
+
+      - store_artifacts:
+          path: artifacts/gems
+
+      - save_cache:
+          name: save ports cache
+          paths:
+            - ./ports
+          key: ports-{{ checksum ".ports_versions" }}
+
+      - persist_to_workspace:
+          name: save gems into workspace
+          root: artifacts
+          paths:
+            - gems
 
 workflows:
   test_supported_ruby_versions:
     jobs:
-      - test_linux:
-          matrix:
+      - cross_compile_gem
+      - test_windows:
+          requires:
+            - cross_compile_gem
+          matrix: &ruby_versions
             parameters:
               ruby_version:
+                - '2.4'
                 - '2.5'
                 - '2.6'
                 - '2.7'
-
-      - test_windows:
-          matrix:
-            parameters:
-              ruby_version:
-               - '2.5'
-               - '2.6'
-               - '2.7'
+      - test_linux:
+          matrix: *ruby_versions

.gitignore

@@ -18,3 +18,5 @@ misc
 /exe/*
 /ports/*
 !/ports/patches/
+test/reports
+.ports_versions

Gemfile

@@ -1,9 +1,2 @@
 source 'https://rubygems.org'
 gemspec
-
-group :development do
-end
-
-group :test do
-  gem 'minitest'
-end

README.md

@@ -407,7 +407,7 @@ For the convenience of Windows users, TinyTDS ships pre-compiled gems for suppor
 Run the following rake task to compile the gems for Windows. This will check the availability of [Docker](https://www.docker.com/) (and boot2docker on Windows or OS-X) and will give some advice for download and installation. When docker is running, it will download the docker image (once-only) and start the build:
 
 ```
-$ rake gem:windows
+$ rake gem:native
 ```
 
 The compiled gems will exist in `./pkg` directory.

Rakefile

@@ -10,6 +10,7 @@ GEM_PLATFORM_HOSTS = {
   'x86-mingw32' => 'i686-w64-mingw32',
   'x64-mingw32' => 'x86_64-w64-mingw32'
 }
+RUBY_CC_VERSION="2.7.0:2.6.0:2.5.0:2.4.0".freeze
 
 # Add our project specific files to clean for a rebuild
 CLEAN.include FileList["{ext,lib}/**/*.{so,#{RbConfig::CONFIG['DLEXT']},o}"],

docker-compose.yml

@@ -21,7 +21,7 @@ services:
     network_mode: "host"
 
   cimgruby:
-    image: cimg/ruby:2.7.0
+    image: "cimg/ruby:${RUBY_VERSION:-2.7}"
     container_name: cimg_ruby
     environment:
       TESTOPTS: '-v'

ext/tiny_tds/extconsts.rb

@@ -2,7 +2,7 @@
 ICONV_VERSION = ENV['TINYTDS_ICONV_VERSION'] || "1.15"
 ICONV_SOURCE_URI = "http://ftp.gnu.org/pub/gnu/libiconv/libiconv-#{ICONV_VERSION}.tar.gz"
 
-OPENSSL_VERSION = ENV['TINYTDS_OPENSSL_VERSION'] || '1.1.1d'
+OPENSSL_VERSION = ENV['TINYTDS_OPENSSL_VERSION'] || '1.1.1s'
 OPENSSL_SOURCE_URI = "https://www.openssl.org/source/openssl-#{OPENSSL_VERSION}.tar.gz"
 
 FREETDS_VERSION = ENV['TINYTDS_FREETDS_VERSION'] || "1.1.24"

tasks/native_gem.rake

@@ -1,14 +1,13 @@
 # encoding: UTF-8
 
-desc 'Build the windows binary gems per rake-compiler-dock'
-task 'gem:windows' => ['ports:cross'] do
+desc 'Build the native binary gems using rake-compiler-dock'
+task 'gem:native' => ['ports:cross'] do
   require 'rake_compiler_dock'
 
   # make sure to install our bundle
-  build = ['bundle']
+  sh "bundle package --all"   # Avoid repeated downloads of gems by using gem files from the host.
 
-  # and finally build the native gem
-  build << 'rake cross native gem RUBY_CC_VERSION=2.7.0:2.6.0:2.5.0:2.4.0 CFLAGS="-Wall" MAKE="make -j`nproc`"'
-
-  RakeCompilerDock.sh build.join(' && ')
+  GEM_PLATFORM_HOSTS.keys.each do |plat|
+    RakeCompilerDock.sh "bundle --local && RUBY_CC_VERSION=#{RUBY_CC_VERSION} rake native:#{plat} gem", platform: plat
+  end
 end