mysablehats
diff --git a/‎.circleci/README.md‎
Lines changed: 467 additions & 1 deletion b/‎.circleci/README.md‎
Lines changed: 467 additions & 1 deletion
diff --git a/‎.circleci/cimodel/data/binary_build_data.py‎
Lines changed: 54 additions & 15 deletions b/‎.circleci/cimodel/data/binary_build_data.py‎
Lines changed: 54 additions & 15 deletions
diff --git a/‎.circleci/cimodel/data/binary_build_definitions.py‎
Lines changed: 56 additions & 94 deletions b/‎.circleci/cimodel/data/binary_build_definitions.py‎
Lines changed: 56 additions & 94 deletions
@@ -23,6 +23,7 @@
     "static",
 ]
 
+
 DEPS_INCLUSION_DIMENSIONS = [
     "with-deps",
     "without-deps",
@@ -33,20 +34,22 @@ def get_processor_arch_name(cuda_version):
     return "cpu" if not cuda_version else "cu" + cuda_version
 
 
+LINUX_PACKAGE_VARIANTS = OrderedDict(
+    manywheel=[
+        "2.7m",
+        "2.7mu",
+        "3.5m",
+        "3.6m",
+        "3.7m",
+    ],
+    conda=dimensions.STANDARD_PYTHON_VERSIONS,
+    libtorch=[
+        "2.7m",
+    ],
+)
+
 CONFIG_TREE_DATA = OrderedDict(
-    linux=(dimensions.CUDA_VERSIONS, OrderedDict(
-        manywheel=[
-            "2.7m",
-            "2.7mu",
-            "3.5m",
-            "3.6m",
-            "3.7m",
-        ],
-        conda=dimensions.STANDARD_PYTHON_VERSIONS,
-        libtorch=[
-            "2.7m",
-        ]
-    )),
+    linux=(dimensions.CUDA_VERSIONS, LINUX_PACKAGE_VARIANTS),
     macos=([None], OrderedDict(
         wheel=dimensions.STANDARD_PYTHON_VERSIONS,
         conda=dimensions.STANDARD_PYTHON_VERSIONS,
@@ -56,6 +59,22 @@ def get_processor_arch_name(cuda_version):
     )),
 )
 
+# GCC config variants:
+#
+# All the nightlies (except libtorch with new gcc ABI) are built with devtoolset7,
+# which can only build with old gcc ABI. It is better than devtoolset3
+# because it understands avx512, which is needed for good fbgemm performance.
+#
+# Libtorch with new gcc ABI is built with gcc 5.4 on Ubuntu 16.04.
+LINUX_GCC_CONFIG_VARIANTS = OrderedDict(
+    manywheel=['devtoolset7'],
+    conda=['devtoolset7'],
+    libtorch=[
+        "devtoolset7",
+        "gcc5.4_cxx11-abi",
+    ],
+)
+
 
 class TopLevelNode(ConfigNode):
     def __init__(self, node_name, config_tree_data, smoke):
@@ -88,7 +107,27 @@ def __init__(self, parent, package_format, python_versions):
         self.props["package_format"] = package_format
 
     def get_children(self):
-        return [ArchConfigNode(self, v) for v in self.find_prop("cuda_versions")]
+        if self.find_prop("os_name") == "linux":
+            return [LinuxGccConfigNode(self, v) for v in LINUX_GCC_CONFIG_VARIANTS[self.find_prop("package_format")]]
+        else:
+            return [ArchConfigNode(self, v) for v in self.find_prop("cuda_versions")]
+
+
+class LinuxGccConfigNode(ConfigNode):
+    def __init__(self, parent, gcc_config_variant):
+        super(LinuxGccConfigNode, self).__init__(parent, "GCC_CONFIG_VARIANT=" + str(gcc_config_variant))
+
+        self.props["gcc_config_variant"] = gcc_config_variant
+
+    def get_children(self):
+        cuda_versions = self.find_prop("cuda_versions")
+
+        # XXX devtoolset7 on CUDA 9.0 is temporarily disabled
+        # see https://github.com/pytorch/pytorch/issues/20066
+        if self.find_prop("gcc_config_variant") == 'devtoolset7':
+            cuda_versions = filter(lambda x: x != "90", cuda_versions)
+
+        return [ArchConfigNode(self, v) for v in cuda_versions]
 
 
 class ArchConfigNode(ConfigNode):
@@ -113,7 +152,7 @@ def get_children(self):
         package_format = self.find_prop("package_format")
         os_name = self.find_prop("os_name")
 
-        has_libtorch_variants = smoke and package_format == "libtorch" and os_name == "linux"
+        has_libtorch_variants = package_format == "libtorch" and os_name == "linux"
         linking_variants = LINKING_DIMENSIONS if has_libtorch_variants else []
 
         return [LinkingVariantConfigNode(self, v) for v in linking_variants]
 
@@ -9,19 +9,25 @@
 
 
 class Conf(object):
-    def __init__(self, os, cuda_version, pydistro, parms, smoke=False, libtorch_variant=None):
+    def __init__(self, os, cuda_version, pydistro, parms, smoke, libtorch_variant, gcc_config_variant):
 
         self.os = os
         self.cuda_version = cuda_version
         self.pydistro = pydistro
         self.parms = parms
         self.smoke = smoke
         self.libtorch_variant = libtorch_variant
+        self.gcc_config_variant = gcc_config_variant
 
     def gen_build_env_parms(self):
-        return [self.pydistro] + self.parms + [binary_build_data.get_processor_arch_name(self.cuda_version)]
+        elems = [self.pydistro] + self.parms + [binary_build_data.get_processor_arch_name(self.cuda_version)]
+        if self.gcc_config_variant is not None:
+            elems.append(str(self.gcc_config_variant))
+        return elems
 
     def gen_docker_image(self):
+        if self.gcc_config_variant == 'gcc5.4_cxx11-abi':
+            return miniutils.quote("soumith/conda-cuda-cxx11-ubuntu1604:latest")
 
         docker_word_substitution = {
             "manywheel": "manylinux",
@@ -30,54 +36,57 @@ def gen_docker_image(self):
 
         docker_distro_prefix = miniutils.override(self.pydistro, docker_word_substitution)
 
-        alt_docker_suffix = self.cuda_version or "80"
+        # The cpu nightlies are built on the soumith/manylinux-cuda100 docker image
+        alt_docker_suffix = self.cuda_version or "100"
         docker_distro_suffix = "" if self.pydistro == "conda" else alt_docker_suffix
         return miniutils.quote("soumith/" + docker_distro_prefix + "-cuda" + docker_distro_suffix)
 
     def get_name_prefix(self):
         return "smoke" if self.smoke else "binary"
 
     def gen_build_name(self, build_or_test):
-        parts = [self.get_name_prefix(), self.os] + self.gen_build_env_parms()
-
-        if self.smoke:
-            if self.libtorch_variant:
-                parts.append(self.libtorch_variant)
-        else:
-            parts.append(build_or_test)
-
-        return "_".join(parts)
-
-    def gen_yaml_tree(self, build_or_test):
 
-        env_tuples = [("BUILD_ENVIRONMENT", miniutils.quote(" ".join(self.gen_build_env_parms())))]
+        parts = [self.get_name_prefix(), self.os] + self.gen_build_env_parms()
 
         if self.libtorch_variant:
-            env_tuples.append(("LIBTORCH_VARIANT", miniutils.quote(self.libtorch_variant)))
+            parts.append(self.libtorch_variant)
 
-        os_name = miniutils.override(self.os, {"macos": "mac"})
-        d = {"<<": "*" + "_".join([self.get_name_prefix(), os_name, build_or_test])}
+        if not self.smoke:
+            parts.append(build_or_test)
 
-        if build_or_test == "test":
+        joined = "_".join(parts)
+        return joined.replace(".", "_")
 
+    def gen_workflow_job(self, phase, upload_phase_dependency=None):
+        job_def = OrderedDict()
+        job_def["name"] = self.gen_build_name(phase)
+        job_def["build_environment"] = miniutils.quote(" ".join(self.gen_build_env_parms()))
+        job_def["requires"] = ["setup"]
+        job_def["filters"] = {"branches": {"only": "nightly"}}
+        if self.libtorch_variant:
+            job_def["libtorch_variant"] = miniutils.quote(self.libtorch_variant)
+        if phase == "test":
+            if not self.smoke:
+                job_def["requires"].append(self.gen_build_name("build"))
             if not (self.smoke and self.os == "macos"):
-                env_tuples.append(("DOCKER_IMAGE", self.gen_docker_image()))
+                job_def["docker_image"] = self.gen_docker_image()
 
             if self.cuda_version:
-                env_tuples.append(("USE_CUDA_DOCKER_RUNTIME", miniutils.quote("1")))
-
+                job_def["use_cuda_docker_runtime"] = miniutils.quote("1")
         else:
-            if self.os == "linux" and build_or_test != "upload":
-                d["docker"] = [{"image": self.gen_docker_image()}]
-
-        d["environment"] = OrderedDict(env_tuples)
+            if self.os == "linux" and phase != "upload":
+                job_def["docker_image"] = self.gen_docker_image()
 
-        if build_or_test == "test":
+        if phase == "test":
             if self.cuda_version:
-                d["resource_class"] = "gpu.medium"
-
-        return d
+                job_def["resource_class"] = "gpu.medium"
+        if phase == "upload":
+            job_def["context"] = "org-member"
+            job_def["requires"] = ["setup", self.gen_build_name(upload_phase_dependency)]
 
+        os_name = miniutils.override(self.os, {"macos": "mac"})
+        job_name = "_".join([self.get_name_prefix(), os_name, phase])
+        return {job_name : job_def}
 
 def get_root(smoke, name):
 
@@ -101,97 +110,50 @@ def gen_build_env_list(smoke):
             c.find_prop("package_format"),
             [c.find_prop("pyver")],
             c.find_prop("smoke"),
-            c.find_prop("libtorch_variant")
+            c.find_prop("libtorch_variant"),
+            c.find_prop("gcc_config_variant"),
         )
         newlist.append(conf)
 
     return newlist
 
 
 def predicate_exclude_nonlinux_and_libtorch(config):
-    return config.os == "linux" and (config.smoke or config.pydistro != "libtorch")
-
-
-def add_build_entries(jobs_dict, phase, smoke, filter_predicate=lambda x: True):
-
-    configs = gen_build_env_list(smoke)
-    for conf_options in filter(filter_predicate, configs):
-        jobs_dict[conf_options.gen_build_name(phase)] = conf_options.gen_yaml_tree(phase)
-
-
-def add_binary_build_specs(jobs_dict):
-    add_build_entries(jobs_dict, "build", False)
-
-
-def add_binary_build_tests(jobs_dict):
-    add_build_entries(jobs_dict, "test", False, predicate_exclude_nonlinux_and_libtorch)
-
-
-def add_binary_build_uploads(jobs_dict):
-    add_build_entries(jobs_dict, "upload", False)
-
-
-def add_smoke_test_specs(jobs_dict):
-    add_build_entries(jobs_dict, "test", True)
-
-
-def get_nightly_tests():
-
-    configs = gen_build_env_list(False)
-    filtered_configs = filter(predicate_exclude_nonlinux_and_libtorch, configs)
-
-    mylist = []
-    for conf_options in filtered_configs:
-        d = {conf_options.gen_build_name("test"): {"requires": [conf_options.gen_build_name("build")]}}
-        mylist.append(d)
-
-    return mylist
+    return config.os == "linux"
 
 
 def get_nightly_uploads():
-
     configs = gen_build_env_list(False)
-
-    def gen_config(conf, phase_dependency):
-        return {
-            conf.gen_build_name("upload"): OrderedDict([
-                ("context", "org-member"),
-                ("requires", [conf.gen_build_name(phase_dependency)]),
-            ]),
-        }
-
     mylist = []
     for conf in configs:
         phase_dependency = "test" if predicate_exclude_nonlinux_and_libtorch(conf) else "build"
-        mylist.append(gen_config(conf, phase_dependency))
+        mylist.append(conf.gen_workflow_job("upload", phase_dependency))
 
     return mylist
 
+def get_nightly_tests():
+
+    configs = gen_build_env_list(False)
+    filtered_configs = filter(predicate_exclude_nonlinux_and_libtorch, configs)
+
+    tests = []
+    for conf_options in filtered_configs:
+        yaml_item = conf_options.gen_workflow_job("test")
+        tests.append(yaml_item)
 
-def gen_schedule_tree(cron_timing):
-    return [{
-        "schedule": {
-            "cron": miniutils.quote(cron_timing),
-            "filters": {
-                "branches": {
-                    "only": ["master"],
-                },
-            },
-        },
-    }]
+    return tests
 
 
 def add_jobs_and_render(jobs_dict, toplevel_key, smoke, cron_schedule):
 
-    jobs_list = []
+    jobs_list = ["setup"]
 
     configs = gen_build_env_list(smoke)
+    phase = "build" if toplevel_key == "binarybuilds" else "test"
     for build_config in configs:
-        build_name = build_config.gen_build_name("build")
-        jobs_list.append(build_name)
+        jobs_list.append(build_config.gen_workflow_job(phase))
 
     jobs_dict[toplevel_key] = OrderedDict(
-        triggers=gen_schedule_tree(cron_schedule),
         jobs=jobs_list,
     )