Update

[ghstack-poisoned]

Author: swolchok

.github/scripts/extract_benchmark_results.py

@@ -360,6 +360,7 @@ def transform(
                     "app_type": app_type,
                     # Just keep a copy of the benchmark config here
                     "benchmark_config": json.dumps(benchmark_config),
+                    "job_conclusion": "SUCCESS",
                 },
             },
             "model": {
@@ -455,7 +456,7 @@ def transform_failure_record(
         },
         "metric": {
             "name": "FAILURE_REPORT",
-            "benchmark_values": 0,
+            "benchmark_values": [0],
             "target_value": 0,
             "extra_info": {
                 "method": "",

backends/apple/coreml/test/test_coreml_quantizer.py

@@ -32,7 +32,9 @@ def quantize_and_compare(
     ) -> None:
         assert quantization_type in {"PTQ", "QAT"}
 
-        pre_autograd_aten_dialect = export_for_training(model, example_inputs).module()
+        pre_autograd_aten_dialect = export_for_training(
+            model, example_inputs, strict=True
+        ).module()
 
         quantization_config = LinearQuantizerConfig.from_dict(
             {

backends/apple/mps/test/test_mps_utils.py

@@ -207,7 +207,7 @@ def lower_module_and_test_output(
         expected_output = model(*sample_inputs)
 
         model = torch.export.export_for_training(
-            model, sample_inputs, dynamic_shapes=dynamic_shapes
+            model, sample_inputs, dynamic_shapes=dynamic_shapes, strict=True
         ).module()
 
         edge_program = export_to_edge(

backends/arm/operators/op_table.py

@@ -41,7 +41,7 @@ def define_node(
 
         if inputs[0].dtype not in (ts.DType.INT8, ts.DType.INT16):
             raise ValueError(
-                f"TOSA.TABLE only supports int8 or int16 inputs, got {ts.DTypeNames[inputs[0]]}"
+                f"TOSA.TABLE only supports int8 or int16 inputs, got {ts.DTypeNames[inputs[0].dtype]}"
             )
 
         table = self._exported_program.state_dict[node.name]  # type: ignore[union-attr]

backends/arm/test/misc/test_debug_feats.py

@@ -192,16 +192,13 @@ def test_collate_tosa_BI_tests(self):
             .to_edge_transform_and_lower()
             .to_executorch()
         )
+
+        test_collate_dir = "test_collate_tosa_tests/tosa-bi/TestCollateTosaTests/test_collate_tosa_BI_tests"
         # test that the output directory is created and contains the expected files
-        assert os.path.exists(
-            "test_collate_tosa_tests/tosa-bi/TestCollateTosaTests/test_collate_tosa_BI_tests"
-        )
-        assert os.path.exists(
-            "test_collate_tosa_tests/tosa-bi/TestCollateTosaTests/test_collate_tosa_BI_tests/output_tag6_TOSA-0.80+BI.tosa"
-        )
-        assert os.path.exists(
-            "test_collate_tosa_tests/tosa-bi/TestCollateTosaTests/test_collate_tosa_BI_tests/desc_tag6_TOSA-0.80+BI.json"
-        )
+        assert os.path.exists(test_collate_dir)
+
+        for file in os.listdir(test_collate_dir):
+            assert file.endswith(("TOSA-0.80+BI.json", "TOSA-0.80+BI.tosa"))
 
         os.environ.pop("TOSA_TESTCASES_BASE_PATH")
         shutil.rmtree("test_collate_tosa_tests", ignore_errors=True)

backends/arm/test/misc/test_partition_decomposed_quantized_ops.py

@@ -145,7 +145,12 @@ def test_linear_residaul_tosa_MI(test_data: input_t1):
     pipeline.run()
 
 
-@common.parametrize("test_data", test_data)
+@common.parametrize(
+    "test_data",
+    test_data,
+    {"3d_rand": "MLETORCH-855: Issue with Quantization folding."},
+    strict=False,
+)
 def test_linear_residual_tosa_BI(test_data: input_t1):
     pipeline = TosaPipelineBI[input_t1](
         LinearResidualModule(),

backends/arm/test/models/test_nn_functional.py

@@ -104,7 +104,15 @@ def test_nn_functional_MI(test_data):
             raise e
 
 
-@parametrize("test_data", module_tests)
+x_fails = {
+    "normalize": "MLETORCH-852: Support aten.index_put.default",
+    "cosine_similarity": "MLETORCH-854: Support aten.linalg_vector_norm.default",
+    "unfold": "Int64 input && MLETORCH-827: Support aten.index.Tensor",
+    "fold": "Int64 input && MLETORCH-827: Support aten.index_put.default",
+}
+
+
+@parametrize("test_data", module_tests, x_fails, strict=False)
 def test_nn_functional_BI(test_data):
     module, inputs = test_data
     pipeline = TosaPipelineBI[input_t](

backends/arm/test/models/test_w2l_arm.py

@@ -132,6 +132,7 @@ def test_w2l_u55_BI(self):
     @pytest.mark.slow
     @pytest.mark.corstone_fvp
     @conftest.expectedFailureOnFVP  # TODO: MLETORCH-761
+    @pytest.mark.skip(reason="Intermittent timeout issue: MLETORCH-856")
     def test_w2l_u85_BI(self):
         tester = self._test_w2l_ethos_BI_pipeline(
             self.w2l,

backends/cadence/aot/compiler.py

@@ -86,7 +86,7 @@ def convert_pt2(
     remove_decompositions(decomp_table, ops_to_keep)
     # Export with dynamo
     model_gm = (
-        torch.export.export_for_training(model, inputs)
+        torch.export.export_for_training(model, inputs, strict=True)
         .run_decompositions(decomp_table)
         .module()
     )

backends/cadence/aot/tests/test_remove_ops_passes.py

@@ -474,7 +474,7 @@ def forward(self, x):
         # Run the standard quant/convert steps, but without fusing
         # this leaves two redundant quant/dequant pairs to test with
         quantizer = CadenceDefaultQuantizer()
-        model_exp = export_for_training(M(), (inp,)).module()
+        model_exp = export_for_training(M(), (inp,), strict=True).module()
         prepared_model = prepare_pt2e(model_exp, quantizer)
         prepared_model(inp)
         converted_model = convert_pt2e(prepared_model)