Parallelize cross compilation

Author: andyundso

.circleci/config.yml

@@ -197,35 +197,26 @@ jobs:
           path: test/reports
 
   cross_compile_gem:
-    machine:
-      image: ubuntu-2004:current
+    parameters:
+      platform:
+        description: "Platform to compile the gem resources"
+        type: string
+
+    docker:
+      - image: "ghcr.io/rake-compiler/rake-compiler-dock-image:1.3.0-mri-<< parameters.platform >>"
 
     steps:
-      - ruby/install:
-          version: '2.7'
       - checkout
-      - restore_cache:
-          name: restore gem cache
-          keys:
-            - v1-bundle-{{ .Branch }}-{{ checksum "tiny_tds.gemspec" }}
-            - v1-bundle-{{ .Branch }}-
-            - v1-bundle-
 
       - run:
           name: bundle install gems
           command: |
-            bundle install --path vendor/bundle
-
-      - save_cache:
-          name: save gem cache
-          paths:
-            - ./vendor/bundle
-          key: v1-bundle-{{ .Branch }}-{{ checksum "tiny_tds.gemspec" }}
+            bundle install
 
       - run:
           name: Write used versions for ports into file
           command: |
-            bundle exec rake ports:version_file
+            rake ports:version_file[<< parameters.platform >>]
 
       - restore_cache:
           name: restore ports cache
@@ -234,16 +225,15 @@ jobs:
             - ports-
 
       - run:
-          name: Build gems
+          name: Build gem
           command: |
-            bundle exec rake gem
-            bundle exec rake gem:native
+            rake gem:for_platform[<< parameters.platform >>]
 
       - run:
           name: Move gems into separate directory before caching
           command: |
-            mkdir -p artifacts/gems
-            mv pkg/*.gem artifacts/gems
+            mkdir -p artifacts-<< parameters.platform >>/gems
+            mv pkg/*.gem artifacts-<< parameters.platform >>/gems
 
       - store_artifacts:
           path: artifacts/gems
@@ -256,14 +246,20 @@ jobs:
 
       - persist_to_workspace:
           name: save gems into workspace
-          root: artifacts
+          root: artifacts-<< parameters.platform >>
           paths:
             - gems
 
 workflows:
   test_supported_ruby_versions:
     jobs:
-      - cross_compile_gem
+      - cross_compile_gem:
+          matrix:
+            parameters:
+              platform:
+                - "x86-mingw32"
+                - "x64-mingw32"
+                - "x64-mingw-ucrt"
       - test_windows:
           requires:
             - cross_compile_gem

Rakefile

@@ -11,28 +11,28 @@ ruby_cc_ucrt_versions = "3.2.0:3.1.0".freeze
 ruby_cc_mingw32_versions = "3.0.0:2.7.0:2.6.0:2.5.0:2.4.0".freeze
 
 GEM_PLATFORM_HOSTS = {
+  'x86-mingw32' => {
+    host: 'i686-w64-mingw32',
+    ruby_versions: ruby_cc_mingw32_versions
+  },
   'x64-mingw32' => {
     host: 'x86_64-w64-mingw32',
     ruby_versions: ruby_cc_mingw32_versions
   },
   'x64-mingw-ucrt' => {
-    host: 'mingw-w64-ucrt-x86_64',
+    host: 'x86_64-w64-mingw32',
     ruby_versions: ruby_cc_ucrt_versions
   },
-  'x86-mingw32' => {
-    host: 'i686-w64-mingw32',
-    ruby_versions: ruby_cc_mingw32_versions
-  },
 }
 
 # Add our project specific files to clean for a rebuild
 CLEAN.include FileList["{ext,lib}/**/*.{so,#{RbConfig::CONFIG['DLEXT']},o}"],
-  FileList["exe/*"]
+              FileList["exe/*"]
 
 # Clobber all our temp files and ports files including .install files
 # and archives
 CLOBBER.include FileList["tmp/**/*"],
-  FileList["ports/**/*"].exclude(%r{^ports/archives})
+                FileList["ports/**/*"].exclude(%r{^ports/archives})
 
 Dir['tasks/*.rake'].sort.each { |f| load f }
 

tasks/native_gem.rake

@@ -5,9 +5,19 @@ task 'gem:native' => ['ports:cross'] do
   require 'rake_compiler_dock'
 
   # make sure to install our bundle
-  sh "bundle package --all"   # Avoid repeated downloads of gems by using gem files from the host.
+  sh "bundle package --all" # Avoid repeated downloads of gems by using gem files from the host.
 
   GEM_PLATFORM_HOSTS.each do |plat, meta|
     RakeCompilerDock.sh "bundle --local && RUBY_CC_VERSION=#{meta[:ruby_versions]} rake native:#{plat} gem", platform: plat
   end
 end
+
+# assumes you are in a container provided by Rake compiler
+# if not, use the task above
+task 'gem:for_platform', [:gem_platform] do |_task, args|
+  args.with_defaults(gem_platform: RbConfig::CONFIG["arch"])
+
+  sh "bundle install"
+  Rake::Task["ports:compile"].invoke(GEM_PLATFORM_HOSTS[args.gem_platform][:host], args.gem_platform)
+  sh "RUBY_CC_VERSION=#{GEM_PLATFORM_HOSTS[args.gem_platform][:ruby_versions]} rake native:#{args.gem_platform} gem"
+end

tasks/ports.rake

@@ -18,7 +18,7 @@ namespace :ports do
   CLEAN.include "ports/*.installed"
 
   task :openssl, [:host, :gem_platform] do |_task, args|
-    args.with_defaults(host: RbConfig::CONFIG['host'], gem_platform: RUBY_PLATFORM)
+    args.with_defaults(host: RbConfig::CONFIG['host'], gem_platform: RbConfig::CONFIG["arch"])
 
     libraries_to_compile[:openssl].files = [OPENSSL_SOURCE_URI]
     libraries_to_compile[:openssl].host = args.host
@@ -29,18 +29,17 @@ namespace :ports do
   end
 
   task :libiconv, [:host, :gem_platform] do |_task, args|
-    args.with_defaults(host: RbConfig::CONFIG['host'], gem_platform: RUBY_PLATFORM)
+    args.with_defaults(host: RbConfig::CONFIG['host'], gem_platform: RbConfig::CONFIG["arch"])
 
     libraries_to_compile[:libiconv].files = [ICONV_SOURCE_URI]
     libraries_to_compile[:libiconv].host = args.host
     libraries_to_compile[:libiconv].gem_platform = args.gem_platform
-
     libraries_to_compile[:libiconv].cook
     libraries_to_compile[:libiconv].activate
   end
 
   task :freetds, [:host, :gem_platform] do |_task, args|
-    args.with_defaults(host: RbConfig::CONFIG['host'], gem_platform: RUBY_PLATFORM)
+    args.with_defaults(host: RbConfig::CONFIG['host'], gem_platform: RbConfig::CONFIG["arch"])
 
     libraries_to_compile[:freetds].files = [FREETDS_SOURCE_URI]
     libraries_to_compile[:freetds].host = args.host
@@ -65,7 +64,7 @@ namespace :ports do
   end
 
   task :compile, [:host, :gem_platform] do |_task, args|
-    args.with_defaults(host: RbConfig::CONFIG['host'], gem_platform: RUBY_PLATFORM)
+    args.with_defaults(host: RbConfig::CONFIG['host'], gem_platform: RbConfig::CONFIG["arch"])
 
     puts "Compiling ports for #{args.host} (Ruby platform #{args.gem_platform}) ..."
 
@@ -82,20 +81,22 @@ namespace :ports do
     GEM_PLATFORM_HOSTS.each do |gem_platform, meta|
       # make sure to install our bundle
       build = ['bundle']
-      build << "rake ports:compile[#{meta[:host]},#{gem_platform}] MAKE='make -j`nproc`'"
+      build << "RUBY_CC_VERSION=#{meta[:ruby_versions]} rake ports:compile[#{meta[:host]},#{gem_platform}] MAKE='make -j`nproc`'"
       RakeCompilerDock.sh build.join(' && '), platform: gem_platform
     end
   end
 
   desc "Notes the actual versions for the compiled ports into a file"
-  task "version_file" do
+  task "version_file", [:gem_platform] do |_task, args|
+    args.with_defaults(gem_platform: RbConfig::CONFIG["arch"])
+
     ports_version = {}
 
     libraries_to_compile.each do |library, library_recipe|
       ports_version[library] = library_recipe.version
     end
 
-    ports_version[:platforms] = GEM_PLATFORM_HOSTS.keys
+    ports_version[:platform] = args.gem_platform
 
     File.open(".ports_versions", "w") do |f|
       f.write ports_version

tasks/ports/recipe.rb

@@ -5,19 +5,22 @@
 
 module Ports
   class Recipe < MiniPortile
-    attr_accessor :gem_platform
+    attr_writer :gem_platform
 
     def cook
       checkpoint = "ports/checkpoints/#{name}-#{version}-#{gem_platform}.installed"
 
       unless File.exist? checkpoint
         super
+        FileUtils.mkdir_p("ports/checkpoints")
         FileUtils.touch checkpoint
       end
     end
 
     private
 
+    attr_reader :gem_platform
+
     def port_path
       "#{@target}/#{gem_platform}/#{@name}/#{@version}"
     end
@@ -59,4 +62,3 @@ def get_patches(libname, version)
     end
   end
 end
-