- | name: quantized_decomposed_quantize_per_tensor_default_8_0 |
+ | name: quantized_decomposed_quantize_per_tensor_default_0 |
| data_type: Qnn_DataType_t.QNN_DATATYPE_UFIXED_POINT_8 |
| tensor_type: Qnn_TensorType_t.QNN_TENSOR_TYPE_NATIVE |
| dims: [1, 32, 28, 28] |
@@ -3247,12 +3221,12 @@ def test_qnn_backend_draw_graph(self):
| dims: [32] |
| quantization_encoding: Qnn_QuantizationEncoding_t.QNN_QUANTIZATION_ENCODING_AXIS_SCALE_OFFSET |
> color=black fillcolor=transparent shape=box style=rounded]
- quantized_decomposed_quantize_per_tensor_default_8_0 -> aten_convolution_default_0
- input_0_x_0 -> quantized_decomposed_quantize_per_tensor_default_8_0
+ quantized_decomposed_quantize_per_tensor_default_0 -> aten_convolution_default_0
+ input_0_x_0 -> quantized_decomposed_quantize_per_tensor_default_0
b__frozen_param0_0 -> aten_convolution_default_0
b__frozen_param1_0 -> aten_convolution_default_0
aten_convolution_default_0 -> aten_relu_default_0
- quantized_decomposed_quantize_per_tensor_default_8_0 -> aten_convolution_default_1_0
+ quantized_decomposed_quantize_per_tensor_default_0 -> aten_convolution_default_1_0
b__frozen_param2_0 -> aten_convolution_default_1_0
b__frozen_param3_0 -> aten_convolution_default_1_0
aten_convolution_default_1_0 -> aten_relu_default_1_0
@@ -3264,25 +3238,15 @@ def test_qnn_backend_draw_graph(self):
module = DrawGraphModel() # noqa: F405
sample_input = (torch.randn(1, 32, 28, 28),)
module = self.get_qdq_module(module, sample_input)
+ # TODO: Figure out how to get the original graph module with to_edge_transform_and_lowering_to_qnn
delegated_program = capture_program(module, sample_input)
"""
This piece of code simulates the behavior of the final preprocessing step to obtain the op wrapper list.
In practice, users need to set a breakpoint in the preprocessing step and use the DrawGraph tool to visualize the graph.
"""
- qnn_compiler_passes = PassManager(
- passes=[
- InsertRequantize(delegated_program.exported_program),
- InsertIOQDQ(delegated_program.exported_program),
- LayoutTransform(
- delegated_program.exported_program, insert_permute=True
- ),
- FuseConsecutiveTranspose(),
- ]
- )
-
- pass_result = qnn_compiler_passes(
- delegated_program.exported_program.graph_module
+ graph_module = QnnPassManager().transform_for_preprocess_pipeline(
+ delegated_program.exported_program
)
nodes_to_wrappers = defaultdict(dict)
node_visitors = get_node_visitors(
@@ -3290,7 +3254,7 @@ def test_qnn_backend_draw_graph(self):
)
py_op_wrapper_list = []
- for node in pass_result.graph_module.graph.nodes:
+ for node in graph_module.graph.nodes:
if node.op == "call_function":
if node.target.__name__ in node_visitors:
py_op_wrapper = node_visitors[node.target.__name__].define_node(
diff --git a/backends/qualcomm/tests/utils.py b/backends/qualcomm/tests/utils.py
index 769f24ba0d8..932a1e2d6fb 100644
--- a/backends/qualcomm/tests/utils.py
+++ b/backends/qualcomm/tests/utils.py
@@ -15,18 +15,19 @@
import torch
from executorch import exir
-from executorch.backends.qualcomm.partition.qnn_partitioner import QnnPartitioner
from executorch.backends.qualcomm.qnn_preprocess import QnnBackend
from executorch.backends.qualcomm.quantizer.quantizer import QuantDtype
from executorch.backends.qualcomm.serialization.qc_schema import QcomChipset
from executorch.backends.qualcomm.utils.constants import (
QCOM_DTYPE,
+ QCOM_PASS_ACTIVATE_KEY,
+ QCOM_PASS_ARGS_KWARGS_DEFAULTS_KEY,
QCOM_SCALE,
QCOM_ZERO_POINT,
)
from executorch.backends.qualcomm.utils.utils import (
- capture_program,
get_soc_to_chipset_map,
+ to_edge_transform_and_lower_to_qnn,
)
from executorch.devtools import generate_etrecord, Inspector
from executorch.examples.qualcomm.utils import (
@@ -36,7 +37,6 @@
SimpleADB,
)
-from executorch.exir.backend.backend_api import to_backend
from executorch.exir.backend.compile_spec_schema import CompileSpec
from executorch.exir.dialects._ops import ops as exir_ops
from executorch.exir.pass_base import ExportPass
@@ -47,6 +47,7 @@
prepare_pt2e,
prepare_qat_pt2e,
)
+from torch.fx.passes.infra.pass_base import PassResult
def generate_context_binary(
@@ -359,19 +360,6 @@ def validate_intermediate_tensor():
cmd.append(f"--{name}_path")
cmd.append(f"{tmp_dir}/{name}.txt")
- dtype_info = {
- "input_type_size": input_encodings,
- "output_type_size": output_encodings,
- }
- for name, encodings in dtype_info.items():
- with open(f"{tmp_dir}/{name}.txt", "w") as f:
- for e in encodings:
- f.write(
- f"{torch.tensor([], dtype=e[QCOM_DTYPE]).element_size()}\n"
- )
- cmd.append(f"--{name}_path")
- cmd.append(f"{tmp_dir}/{name}.txt")
-
env = dict(os.environ)
env["LD_LIBRARY_PATH"] = f"{qnn_sdk}/lib/{target}/:{build_folder}/lib"
proc = subprocess.run(
@@ -423,16 +411,6 @@ def validate_intermediate_tensor():
if check_io_shape
else None
),
- expected_input_dtype=(
- (encoding[QCOM_DTYPE] for encoding in input_encodings)
- if check_io_shape
- else None
- ),
- expected_output_dtype=(
- (encoding[QCOM_DTYPE] for encoding in output_encodings)
- if check_io_shape
- else None
- ),
)
adb.push(inputs=[processed_inputs], input_list=input_list)
adb.execute(method_index=method_index)
@@ -461,19 +439,18 @@ def lower_module_and_test_output(
skip_node_op_set: set = None,
dynamic_shapes: Dict = None,
):
- qnn_partitioner = QnnPartitioner(
- self.compiler_specs, skip_node_id_set, skip_node_op_set
- )
- delegated_program = capture_program(
- module, sample_inputs, dynamic_shapes=dynamic_shapes
+ delegated_program = to_edge_transform_and_lower_to_qnn(
+ module,
+ sample_inputs,
+ self.compiler_specs,
+ dynamic_shapes=dynamic_shapes,
+ skip_node_id_set=skip_node_id_set,
+ skip_node_op_set=skip_node_op_set,
)
# this is needed for the ETRecord as lowering modifies the graph in-place
edge_copy = copy.deepcopy(delegated_program)
- delegated_program.exported_program = to_backend(
- delegated_program.exported_program, qnn_partitioner
- )
exec_prog = delegated_program.to_executorch(
exir.ExecutorchBackendConfig(
# For shared buffer, user must pass the memory address
@@ -489,12 +466,12 @@ def lower_module_and_test_output(
# Assert the backend name is qnn
self.assertEqual(
- len(exec_prog.program.execution_plan[0].delegates),
+ len(exec_prog.executorch_program.execution_plan[0].delegates),
expected_partitions,
)
for i in range(expected_partitions):
self.assertEqual(
- exec_prog.program.execution_plan[0].delegates[i].id,
+ exec_prog.executorch_program.execution_plan[0].delegates[i].id,
QnnBackend.__name__,
)
@@ -599,24 +576,33 @@ def get_converted_sgd_trained_module(
optimizer.step()
return torch.ao.quantization.quantize_pt2e.convert_pt2e(prepared)
- def split_graph(self, graph_module: torch.fx.GraphModule, division: int):
+ def split_graph(self, division: int):
class SplitGraph(ExportPass):
"""
Split graph based on number of nodes.
"""
- def __init__(self, shares):
+ def __init__(self, division):
super().__init__()
- self.shares = shares
+ self.division = division
def _insert_clone(
self, graph_module: torch.fx.GraphModule
) -> torch.fx.GraphModule:
+ # Count the total of nodes in the graph
+ num_graph_nodes = 0
+ for node in graph_module.graph.nodes:
+ num_graph_nodes += 1 if node.op == "call_function" else 0
+
+ # Compute how many nodes in one share
+ shares = num_graph_nodes // self.division
+
+ # Insert clone op to split model based on the shares
num_graph_nodes = 0
for node in graph_module.graph.nodes:
num_graph_nodes += 1 if node.op == "call_function" else 0
- if num_graph_nodes % self.shares != 0 or node.op != "call_function":
+ if num_graph_nodes % shares != 0 or node.op != "call_function":
continue
with graph_module.graph.inserting_after(node):
@@ -635,9 +621,9 @@ def _insert_clone(
def call(self, graph_module: torch.fx.GraphModule):
self._insert_clone(graph_module)
graph_module.recompile()
+ return PassResult(graph_module, True)
- num_graph_nodes = 0
- for node in graph_module.graph.nodes:
- num_graph_nodes += 1 if node.op == "call_function" else 0
-
- SplitGraph(-(num_graph_nodes // -division))(graph_module)
+ return SplitGraph, {
+ QCOM_PASS_ACTIVATE_KEY: True,
+ QCOM_PASS_ARGS_KWARGS_DEFAULTS_KEY: {"division": division},
+ }
diff --git a/backends/qualcomm/utils/constants.py b/backends/qualcomm/utils/constants.py
index 2e364c37119..ce917bf4115 100644
--- a/backends/qualcomm/utils/constants.py
+++ b/backends/qualcomm/utils/constants.py
@@ -25,6 +25,7 @@
QCOM_ORIG_DTYPE = "orig_dtype"
QCOM_QUANTIZED_IO = "q_tensor_io"
QCOM_QUANT_ATTRS = "quant_attrs"
+QCOM_QUANT_ATTRS_MAP = "quant_attrs_map"
QCOM_QUANT_MIN = "quant_min"
QCOM_QUANT_MAX = "quant_max"
QCOM_REQUANTIZE = "requantize"
diff --git a/backends/qualcomm/utils/utils.py b/backends/qualcomm/utils/utils.py
index 7033f30997a..f7b966ee8ea 100644
--- a/backends/qualcomm/utils/utils.py
+++ b/backends/qualcomm/utils/utils.py
@@ -3,43 +3,21 @@
#
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree.
-import inspect
import operator
import re
import time
import warnings
from collections import OrderedDict
-from typing import Any, Callable, Dict, List, Optional, Tuple
+from typing import Any, Callable, Dict, List, Optional, Tuple, Union
import executorch.backends.qualcomm.python.PyQnnManagerAdaptor as PyQnnManagerAdaptor
import executorch.exir as exir
import torch
-from executorch.backends.qualcomm._passes import (
- AnnotateDecomposed,
- AnnotateQuantAttrs,
- ConstantI64toI32,
- ConvertBmmToMatmul,
- ConvertConv1dToConv2d,
- ConvertToLinear,
- DecomposeAny,
- DecomposeExpM1,
- DecomposeLinalgVectorNorm,
- ExpandBroadcastTensorShape,
- FoldQDQ,
- LayoutTransform,
- LiftConstantScalarOperands,
- RecomposePixelUnshuffle,
- RecomposePReLU,
- RecomposeRmsNorm,
- RemoveRedundancy,
- ReplaceIndexPutInput,
-)
-from executorch.backends.qualcomm._passes.tensor_i64_to_i32 import TensorI64toI32
-from executorch.backends.qualcomm._passes.utils import (
- get_passes_dependency_for_capture_program,
-)
+
+from executorch.backends.qualcomm._passes import AnnotateStack
+from executorch.backends.qualcomm._passes.qnn_pass_manager import QnnPassManager
from executorch.backends.qualcomm.builders.node_visitor import (
QNN_QUANT_TYPE_MAP,
@@ -48,6 +26,7 @@
from executorch.backends.qualcomm.builders.qnn_constants import OpContextLoader
from executorch.backends.qualcomm.partition.qnn_partitioner import (
generate_qnn_executorch_option,
+ get_skip_decomp_table,
QnnPartitioner,
)
from executorch.backends.qualcomm.serialization.qc_schema import (
@@ -68,14 +47,9 @@
option_to_flatbuffer,
)
from executorch.backends.qualcomm.utils.constants import (
- QCOM_PASS_ACTIVATE_KEY,
- QCOM_PASS_ARGS_KWARGS_DEFAULTS_KEY,
QCOM_QNN_COMPILE_SPEC,
QCOM_QUANTIZED_IO,
)
-from executorch.backends.transforms.decompose_sdpa import (
- DecomposeScaledDotProductAttention,
-)
from executorch.exir import (
EdgeCompileConfig,
@@ -86,12 +60,13 @@
from executorch.exir.backend.compile_spec_schema import CompileSpec
from executorch.exir.capture import ExecutorchBackendConfig
from executorch.exir.lowered_backend_module import LoweredBackendModule
-from executorch.exir.passes import PassManager
-from executorch.exir.program._program import _get_updated_graph_signature
+from executorch.exir.program._program import (
+ EdgeProgramManager,
+ to_edge_transform_and_lower,
+)
from torch._decomp import core_aten_decompositions, remove_decompositions
from torch.export.exported_program import ExportedProgram
from torch.fx import passes
-from torch.fx.passes.infra.pass_manager import this_before_that_pass_constraint
from torch.fx.passes.operator_support import OperatorSupportBase
from torch.library import Library
@@ -326,160 +301,114 @@ def canonicalize_program(obj):
def get_decomp_table(passes_job) -> Dict[torch._ops.OperatorBase, Callable]:
source_decompositions = core_aten_decompositions()
# The below super ops are supported by QNN
- skip_decompositions = [
- torch.ops.aten.adaptive_avg_pool2d.default,
- torch.ops.aten.elu.default,
- torch.ops.aten.instance_norm.default,
- torch.ops.aten.pixel_shuffle.default,
- torch.ops.aten.pixel_unshuffle.default,
- torch.ops.aten.hardsigmoid.default,
- torch.ops.aten.hardswish.default,
- torch.ops.pt2e_quant.quantize_affine.default,
- torch.ops.pt2e_quant.dequantize_affine.default,
- torch.ops.aten._safe_softmax.default,
- torch.ops.aten.stack.default,
- torch.ops.aten.unbind.int,
- ]
+ skip_decompositions = get_skip_decomp_table()
# If we want to annotate the decomposed ops, then we should decompose the operation.
- if passes_job and passes_job.get(AnnotateDecomposed, False):
+ if passes_job and passes_job.get(AnnotateStack, False):
skip_decompositions = [
skip_decomp_op
for skip_decomp_op in skip_decompositions
- if skip_decomp_op not in AnnotateDecomposed.decomp_ops
+ if skip_decomp_op not in AnnotateStack.decomp_ops
]
remove_decompositions(source_decompositions, skip_decompositions)
return source_decompositions
-def get_capture_program_passes():
+def to_edge_transform_and_lower_to_qnn(
+ module: Union[torch.nn.Module, torch.fx.GraphModule],
+ inputs: Tuple[torch.Tensor],
+ compiler_specs: List[CompileSpec],
+ constant_methods: Optional[Dict[str, Any]] = None,
+ dynamic_shapes: Optional[Dict] = None,
+ dep_table: Optional[Dict] = None,
+ passes_job: Optional[OrderedDict] = None,
+ skip_node_id_set: Optional[set] = None,
+ skip_node_op_set: Optional[set] = None,
+) -> EdgeProgramManager:
"""
- Defines and returns the default ordered passes for the capture program.
- This function creates an OrderedDict containing a series of default passes.
+ Transforms and lowers a given PyTorch module to QNN backend.
+
+ Args:
+ module (Union[torch.nn.Module, torch.fx.GraphModule]): The PyTorch module or fx.GraphModule to be transformed.
+ inputs (Tuple[torch.Tensor]): The input tensors for the module.
+ compiler_specs (List[CompileSpec]): Compiler specs for Qualcomm AI Engine Direct.
+ constant_methods (Optional[Dict[str, Any]]): An optional dictionary of method name to the constant value
+ returned by that method in eager mode. Often used to store config information on
+ Edge models.
+ dynamic_shapes (Optional[Dict]): Information about dynamic shapes.
+ dep_table (Optional[Dict]): Dependency table for the transformation passes.
+ passes_job (Optional[OrderedDict]): Ordered dictionary of transformation passes.
+ skip_node_id_set (Optional[set]): Set of node IDs to skip during partitioning.
+ skip_node_op_set (Optional[set]): Set of node operations to skip during partitioning.
Returns:
- OrderedDict: An ordered dictionary containing all default passes along with their activation status and initialization parameters.
+ EdgeProgramManager: The manager for the edge program after transformation and lowering.
"""
-
- # The second value in each tuple in `default_passes_and_setting` indicates whether the corresponding pass is activated by default.
- # If a pass is activated, it will be executed by default.
- default_passes_and_setting = [
- (AnnotateDecomposed, False),
- (AnnotateQuantAttrs, True),
- (ConstantI64toI32, True),
- (ConvertBmmToMatmul, True),
- (ConvertConv1dToConv2d, True),
- (ConvertToLinear, True),
- (DecomposeAny, True),
- (DecomposeLinalgVectorNorm, True),
- (ExpandBroadcastTensorShape, False),
- (FoldQDQ, True),
- (LayoutTransform, True),
- (RecomposePReLU, True),
- (RecomposePixelUnshuffle, True),
- (RecomposeRmsNorm, True),
- (RemoveRedundancy, True),
- (ReplaceIndexPutInput, True),
- (TensorI64toI32, True),
- ]
-
- passes = OrderedDict()
- for p, act in default_passes_and_setting:
- init_signature = inspect.signature(p.__init__)
-
- args_kwargs_defaults = {
- k: v.default if v.default is not inspect.Parameter.empty else None
- for k, v in init_signature.parameters.items()
- if k != "self"
- }
-
- passes[p] = {
- QCOM_PASS_ACTIVATE_KEY: act,
- QCOM_PASS_ARGS_KWARGS_DEFAULTS_KEY: args_kwargs_defaults,
- }
-
- return passes
-
-
-def _topological_sort_passes(passes: OrderedDict):
- dep_table = get_passes_dependency_for_capture_program()
- pm = PassManager()
- for p in passes:
- pm.add_pass(p)
-
- for that, these in dep_table.items():
- for this in these:
- pm.add_constraint(this_before_that_pass_constraint(this, that))
-
- pm.solve_constraints()
- sorted_passes = OrderedDict()
- for p in pm.passes:
- sorted_passes[p] = passes[p]
- return sorted_passes
-
-
-def _transform(
- edge_program: ExportedProgram, passes_job: OrderedDict = None
-) -> ExportedProgram:
- # TODO: remove this workaround when target could be correclty detected
- from executorch.backends.qualcomm._passes import utils
- from executorch.exir.dialects._ops import ops as exir_ops
-
- utils.q_ops.add(exir_ops.edge.pt2e_quant.quantize_affine.default)
- utils.dq_ops.add(exir_ops.edge.pt2e_quant.dequantize_affine.default)
-
- # currently ExirExportedProgram.transform does not accept
- # changes of input number which was caused by FoldQDQ
- # apply passes one by one here to avoid IR capture failure
- graph_module = edge_program.graph_module
- passes_job = passes_job if passes_job is not None else get_capture_program_passes()
- passes_job = _topological_sort_passes(passes_job)
- for p in passes_job:
- if not passes_job[p][QCOM_PASS_ACTIVATE_KEY]:
- continue
-
- kwargs = passes_job[p][QCOM_PASS_ARGS_KWARGS_DEFAULTS_KEY]
- if "edge_program" in kwargs:
- kwargs["edge_program"] = edge_program
- p(**kwargs)(graph_module)
-
- # Since QDQ nodes are stripped, update graph signature again to validate program
- edge_program._graph_signature = _get_updated_graph_signature(
- edge_program.graph_signature,
- edge_program.graph_module,
+ ep = torch.export.export(module, inputs, dynamic_shapes=dynamic_shapes, strict=True)
+ # This transformation is primarily intended for the LiftConstantScalarOperands pass
+ # to avoid creating temporary tensors in the operation builder.
+ # However, this pass will create a get_attr node, which should be converted
+ # into a lifted tensor constant by the lift_constant_tensor_pass.
+ # If placed in the to_edge_transform_passes, it will be executed
+ # after the lift_constant_tensor_pass, causing the operation builder
+ # to fail to correctly retrieve the parameter by the get_parameter.
+ ep = QnnPassManager().transform_for_export_pipeline(ep)
+ transform_passes = QnnPassManager().get_to_edge_transform_passes(
+ ep, passes_job=passes_job, dep_table=dep_table
)
- edge_program._validate()
- return edge_program
-
-
-# Modify the fx graph at very beginning for floating point model
-# Aim to reduce registration of scalar at graph_module or program
-def _preprocess_module(module: torch.nn.Module, inputs: Tuple[torch.Tensor]):
- if isinstance(module, torch.fx.graph_module.GraphModule):
- return module
- module = torch.export.export(module, inputs, strict=True).module()
- module = DecomposeScaledDotProductAttention()(module).graph_module
- module = DecomposeLinalgVectorNorm(True)(module).graph_module
- module = DecomposeExpM1()(module).graph_module
- module = LiftConstantScalarOperands()(module).graph_module
- return module
+ qnn_partitioner = QnnPartitioner(
+ compiler_specs,
+ skip_node_id_set=skip_node_id_set,
+ skip_node_op_set=skip_node_op_set,
+ )
+ edge_program_manager = to_edge_transform_and_lower(
+ ep,
+ transform_passes=transform_passes,
+ partitioner=[qnn_partitioner],
+ constant_methods=constant_methods,
+ compile_config=qnn_edge_config(),
+ )
+ return edge_program_manager
def capture_program(
- module: torch.nn.Module,
+ module: Union[torch.nn.Module, torch.fx.GraphModule],
inputs: Tuple[torch.Tensor],
passes_job: OrderedDict = None,
dynamic_shapes: Dict = None,
) -> exir.ExirExportedProgram:
- module = _preprocess_module(module, inputs)
+ """
+ TODO: Deprecated capture_program with to_edge_transform_and_lower_to_qnn
+
+ Captures and transforms a PyTorch module into an Exir exported program.
+
+ Args:
+ module (Union[torch.nn.Module, torch.fx.GraphModule]): The PyTorch module or fx.GraphModule to be captured.
+ inputs (Tuple[torch.Tensor]): The input tensors for the module.
+ passes_job (OrderedDict, optional): Ordered dictionary of transformation passes.
+ dynamic_shapes (Dict, optional): Information about dynamic shapes.
+
+ Returns:
+ exir.ExirExportedProgram: The transformed Exir exported program ready for lowering to QNN backend.
+ """
+ warnings.warn(
+ "capture_program is deprecated. Use to_edge_transform_and_lower_to_qnn instead.",
+ DeprecationWarning,
+ stacklevel=1,
+ )
ep = torch.export.export(module, inputs, dynamic_shapes=dynamic_shapes, strict=True)
- # TODO: Handle stack op. If we want to run annotate_decomposed pass for stack op, we need to make stack op decompose, which means we need to find a method to remove it from skip_decomp table
+ ep = QnnPassManager().transform_for_export_pipeline(ep)
+ # TODO: Handle stack op. If we want to run annotate_decomposed pass for stack op,
+ # we need to make stack op decompose, which means we need to find a method to
+ # remove it from skip_decomp table
decomposed_ep = ep.run_decompositions(get_decomp_table(passes_job))
core_ep = ExirExportedProgram(decomposed_ep, False)
- core_ep.transform(TensorI64toI32(edge_program=core_ep))
edge_ep = core_ep.to_edge(qnn_edge_config())
- _transform(edge_ep.exported_program, passes_job)
+ transform_passes = QnnPassManager().get_to_edge_transform_passes(
+ edge_ep.exported_program, passes_job=passes_job
+ )
+ edge_ep.transform(*transform_passes)
return edge_ep
@@ -523,11 +452,9 @@ def _partition_graph_into_submodules(gm, subgm_tag, subgm_cb, ptn):
return gm
-def _canonicalize_graph_with_lowered_module(gm, subgm_tag, ptn):
- from executorch.exir.backend.backend_api import to_backend
-
+def _canonicalize_graph_with_lowered_module(gm, subgm_tag, compiler_specs):
# return lowered program for user to debug
- exported_progs = []
+ edge_prog_mgrs = []
# partition each submodule which went through convert_pt2e
for node in gm.graph.nodes:
if node.op == "call_module" and subgm_tag in node.name:
@@ -536,14 +463,16 @@ def _canonicalize_graph_with_lowered_module(gm, subgm_tag, ptn):
torch.ones(arg.meta["val"].shape, dtype=arg.meta["val"].dtype)
for arg in node.args
]
- # program meets QNN backend requirement
- sub_prog = capture_program(gm.get_submodule(node.name), tuple(subgm_input))
# start lowering with given partitioner
- exported_progs.append(to_backend(sub_prog.exported_program, ptn))
+ edge_prog_mgrs.append(
+ to_edge_transform_and_lower_to_qnn(
+ gm.get_submodule(node.name), tuple(subgm_input), compiler_specs
+ )
+ )
# replace submodule with lowered module
gm.set_submodule(
node.name,
- exported_progs[-1].graph_module,
+ edge_prog_mgrs[-1].exported_program().graph_module,
)
# if node has multiple outputs, getitems will be default generated
if all(n.target != operator.getitem for n in node.users):
@@ -559,13 +488,13 @@ def _canonicalize_graph_with_lowered_module(gm, subgm_tag, ptn):
)
gm.recompile()
- return gm, exported_progs
+ return gm, edge_prog_mgrs
def skip_annotation(
nn_module: torch.nn.Module,
quantizer,
- partitioner,
+ compiler_specs,
sample_input: Tuple[torch.Tensor, ...],
calibration_cb: Callable[[torch.fx.GraphModule], None],
fp_node_id_set: set = None,
@@ -655,7 +584,7 @@ def skip_annotation(
Args:
nn_module (torch.nn.Module): The module to be lowered.
quantizer (QnnQuantizer): Instance of QnnQuantizer.
- partitioner (QnnPartitioner): Instance of QnnPartitioner.
+ compiler_specs (List[CompileSpec]): Compiler specs for Qualcomm AI Engine Direct.
sample_input ((torch.Tensor, ...)): Sample input tensors for graph exporting.
calibration_cb (callable): Callback function for user-defined calibration.
fp_node_id_set ({str, ...}): Set of operator names to be left in fp precision.
@@ -707,37 +636,26 @@ def prepare_subgm(subgm, subgm_name):
node.name, convert_pt2e(graph_module.get_submodule(node.name))
)
# canonicalize graph for lowering again
- graph_module, exported_progs = _canonicalize_graph_with_lowered_module(
+ graph_module, edge_prog_mgrs = _canonicalize_graph_with_lowered_module(
gm=graph_module,
subgm_tag=subgm_tag,
- ptn=partitioner,
+ compiler_specs=compiler_specs,
)
if not fallback_to_cpu:
try:
- from executorch.exir.backend.partitioner import DelegationSpec
-
# change HTP compiler spec for hardware to enable fp16
- qnn_option = generate_qnn_executorch_option(
- partitioner.compiler_specs_snapshot
- )
+ qnn_option = generate_qnn_executorch_option(compiler_specs)
compile_option = flatbuffer_to_option(qnn_option)
htp_options = compile_option.backend_options.htp_options
htp_options.precision = QnnExecuTorchHtpPrecision.kHtpFp16
- partitioner.delegation_spec = DelegationSpec(
- "QnnBackend",
- [
- CompileSpec(
- QCOM_QNN_COMPILE_SPEC, option_to_flatbuffer(compile_option)
- )
- ],
- )
+ compiler_specs[0].value = option_to_flatbuffer(compile_option)
except:
print(
"Failed to change HTP compiler spec with 'use_fp16' as True,"
" skipped operators will fallback to cpu,"
)
- return graph_module, exported_progs
+ return graph_module, edge_prog_mgrs
# try lowering skipped operator into fp16
capability_partitioner = CapabilityBasedPartitioner(
@@ -752,14 +670,14 @@ def prepare_subgm(subgm, subgm_name):
subgm_cb=lambda subgm, _: subgm,
ptn=capability_partitioner,
)
- graph_module, exported_progs_fp = _canonicalize_graph_with_lowered_module(
+ graph_module, edge_prog_mgrs_fp = _canonicalize_graph_with_lowered_module(
gm=graph_module,
subgm_tag=subgm_tag,
- ptn=partitioner,
+ compiler_specs=compiler_specs,
)
- exported_progs.extend(exported_progs_fp)
+ edge_prog_mgrs.extend(edge_prog_mgrs_fp)
- return graph_module, exported_progs
+ return graph_module, edge_prog_mgrs
def from_context_binary( # noqa: C901
diff --git a/examples/models/llama/export_llama_lib.py b/examples/models/llama/export_llama_lib.py
index f1c5c3a73f1..86e30f0ac62 100644
--- a/examples/models/llama/export_llama_lib.py
+++ b/examples/models/llama/export_llama_lib.py
@@ -825,33 +825,32 @@ def _to_edge_and_lower_llama( # noqa: C901
args.use_kv_cache, args.pt2e_quantize, args.num_sharding, args.soc_model
)
)
- # pyre-ignore: Undefined import [21]: Could not find a module corresponding to import `executorch.backends.qualcomm.utils.utils`
- from executorch.backends.qualcomm._passes.annotate_decomposed import (
- AnnotateDecomposed,
+ # pyre-ignore: Undefined import [21]: Could not find a module corresponding to import `executorch.backends.qualcomm._passes`
+ from executorch.backends.qualcomm._passes import (
+ AnnotateStack,
+ FoldQDQ,
+ RecomposeRmsNorm,
+ TagQuantIO,
)
- from executorch.backends.qualcomm.utils.constants import QCOM_PASS_ACTIVATE_KEY
- from executorch.backends.qualcomm.utils.utils import (
- _transform,
+
+ # pyre-ignore: Undefined import [21]: Could not find a module corresponding to import `executorch.backends.qualcomm._passes.qnn_pass_manager`
+ from executorch.backends.qualcomm._passes.qnn_pass_manager import (
get_capture_program_passes,
- tag_quant_io,
+ get_passes_dependency_for_capture_program,
+ QnnPassManager,
)
- passes_job = get_capture_program_passes()
- passes_job[AnnotateDecomposed][QCOM_PASS_ACTIVATE_KEY] = True
- _transform(builder_exported_to_edge.edge_manager.exported_program(), passes_job)
-
- if args.num_sharding > 0:
- model_sharding.split_graph(
- builder_exported_to_edge.edge_manager.exported_program(),
- builder_exported_to_edge.metadata["get_n_layers"],
- shares=args.num_sharding,
- )
-
# pyre-ignore
from executorch.backends.qualcomm.quantizer.custom_annotation import (
get_custom_quant_ios_dtype,
)
+ # pyre-ignore: Undefined import [21]: Could not find a module corresponding to import `executorch.backends.qualcomm.utils.constants`
+ from executorch.backends.qualcomm.utils.constants import (
+ QCOM_PASS_ACTIVATE_KEY,
+ QCOM_PASS_ARGS_KWARGS_DEFAULTS_KEY,
+ )
+
atten = builder_exported_to_edge.model.layers[0].attention
if args.use_qnn_sha:
cache_shape = torch.Size(
@@ -866,9 +865,28 @@ def _to_edge_and_lower_llama( # noqa: C901
atten.head_dim,
)
)
- tag_quant_io(
- builder_exported_to_edge.edge_manager.exported_program().graph_module,
- partial(get_custom_quant_ios_dtype, cache_shape),
+
+ # TODO: Use to_edge_lower_and_transform for QNN
+ passes_job = get_capture_program_passes()
+ dep_table = get_passes_dependency_for_capture_program()
+ passes_job[AnnotateStack][QCOM_PASS_ACTIVATE_KEY] = True
+ passes_job[RecomposeRmsNorm][QCOM_PASS_ACTIVATE_KEY] = True
+ passes_job[TagQuantIO][QCOM_PASS_ACTIVATE_KEY] = True
+ passes_job[TagQuantIO][QCOM_PASS_ARGS_KWARGS_DEFAULTS_KEY][
+ "get_quant_io_dtype_fn"
+ ] = partial(get_custom_quant_ios_dtype, cache_shape)
+ if args.num_sharding > 0:
+ SplitGraph, setting = model_sharding.get_split_graph_pass(
+ builder_exported_to_edge.metadata["get_n_layers"],
+ shares=args.num_sharding,
+ )
+ passes_job[SplitGraph] = setting
+ dep_table[SplitGraph] = [FoldQDQ]
+ dep_table[TagQuantIO] = [SplitGraph]
+ QnnPassManager().transform_for_to_edge_pipeline(
+ builder_exported_to_edge.edge_manager.exported_program(),
+ dep_table=dep_table,
+ passes_job=passes_job,
)
logging.info("Lowering model using following partitioner(s): ")
diff --git a/examples/qualcomm/executor_runner/qnn_executor_runner.cpp b/examples/qualcomm/executor_runner/qnn_executor_runner.cpp
index f3b83700ce2..83478bd8e68 100644
--- a/examples/qualcomm/executor_runner/qnn_executor_runner.cpp
+++ b/examples/qualcomm/executor_runner/qnn_executor_runner.cpp
@@ -78,16 +78,6 @@ DEFINE_string(
"",
"Path to file with output shapes specified (used in dynamic shape scenario).");
-DEFINE_string(
- input_type_size_path,
- "",
- "Path to file with input dtype sizes specified.");
-
-DEFINE_string(
- output_type_size_path,
- "",
- "Path to file with output dtype sizes specified.");
-
DEFINE_int32(
debug_buffer_size,
20000000, // 20MB
@@ -368,23 +358,6 @@ int main(int argc, char** argv) {
expected_output_shapes.emplace_back(std::move(shape));
}
}
- // currently only expected_output_type_sizes is used
- // TODO: remove following when meta could be correctly propagated
- std::vector