Skip to content

Commit dcb3ca0

Browse files
swolchokswolchok
committed
Update
[ghstack-poisoned]
2 parents 3bfdade + aab8021 commit dcb3ca0

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

49 files changed

+2972
-1480
lines changed

‎.Package.swift/executorch_llm/dummy.swift

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+

‎.Package.swift/executorch_llm_debug/dummy.swift

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+

‎.ci/docker/ci_commit_pins/pytorch.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
90f1e7bed15ca5e48c61c5b6dc5ad4810524f82f
1+
ab43fe4bdf5ccd82897f0e982c451a0127bd175e

‎CMakeLists.txt

Lines changed: 63 additions & 85 deletions
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,9 @@
4545
# ~~~
4646
#
4747

48-
cmake_minimum_required(VERSION 3.24)
48+
# TODO Lower to 3.24 when XNNPACK dependency is updated to include
49+
# https://github.com/google/XNNPACK/commit/c690daa67f883e1b627aadf7684c06797e9a0684
50+
cmake_minimum_required(VERSION 3.29)
4951
project(executorch)
5052

5153
include(${PROJECT_SOURCE_DIR}/tools/cmake/common/preset.cmake)
@@ -176,99 +178,74 @@ endif()
176178

177179
if(EXECUTORCH_BUILD_CPUINFO)
178180
# --- cpuinfo
179-
set(CPUINFO_SOURCE_DIR
180-
"${CMAKE_CURRENT_LIST_DIR}/backends/xnnpack/third-party/cpuinfo"
181+
set(ORIGINAL_CMAKE_POSITION_INDEPENDENT_CODE_FLAG
182+
${CMAKE_POSITION_INDEPENDENT_CODE}
181183
)
182-
set(CPUINFO_BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/cpuinfo")
183-
set(CPUINFO_INSTALL_DIR "${CPUINFO_BINARY_DIR}/install")
184-
set(CPUINFO_LIBRARY
185-
"${CPUINFO_INSTALL_DIR}/${CMAKE_INSTALL_LIBDIR}/libcpuinfo.a"
184+
set(CMAKE_POSITION_INDEPENDENT_CODE ON)
185+
set(CPUINFO_SOURCE_DIR "backends/xnnpack/third-party/cpuinfo")
186+
set(CPUINFO_BUILD_TOOLS
187+
OFF
188+
CACHE BOOL ""
186189
)
187-
188-
get_extra_cmake_args_for_external_project(CPUINFO_EXTRA_CMAKE_ARGS)
189-
ExternalProject_Add(
190-
cpuinfoExternalProject
191-
SOURCE_DIR "${CPUINFO_SOURCE_DIR}"
192-
BINARY_DIR "${CPUINFO_BINARY_DIR}"
193-
INSTALL_DIR "${CPUINFO_INSTALL_DIR}"
194-
BUILD_BYPRODUCTS "${CPUINFO_LIBRARY}"
195-
CMAKE_ARGS "${CPUINFO_EXTRA_CMAKE_ARGS}"
196-
-D
197-
CMAKE_POSITION_INDEPENDENT_CODE=ON
198-
-D
199-
CPUINFO_BUILD_TOOLS=OFF
200-
-D
201-
CPUINFO_BUILD_UNIT_TESTS=OFF
202-
-D
203-
CPUINFO_BUILD_MOCK_TESTS=OFF
204-
-D
205-
CPUINFO_BUILD_BENCHMARKS=OFF
206-
-D
207-
CPUINFO_LIBRARY_TYPE=static
208-
-D
209-
CPUINFO_LOG_LEVEL=error
210-
-D
211-
CMAKE_INSTALL_PREFIX=<INSTALL_DIR>
212-
-D
213-
CLOG_SOURCE_DIR="${CPUINFO_SOURCE_DIR}/deps/clog"
190+
set(CPUINFO_BUILD_UNIT_TESTS
191+
OFF
192+
CACHE BOOL ""
193+
)
194+
set(CPUINFO_BUILD_MOCK_TESTS
195+
OFF
196+
CACHE BOOL ""
197+
)
198+
set(CPUINFO_BUILD_BENCHMARKS
199+
OFF
200+
CACHE BOOL ""
201+
)
202+
set(CPUINFO_LIBRARY_TYPE
203+
"static"
204+
CACHE STRING ""
214205
)
215-
add_library(cpuinfo STATIC IMPORTED)
216-
set_property(TARGET cpuinfo PROPERTY IMPORTED_LOCATION "${CPUINFO_LIBRARY}")
217-
add_dependencies(cpuinfo cpuinfoExternalProject)
218-
# Trailing slash matters here! "Move everything from the temporary pthreadpool
219-
# install directory to the proper install directory."
220-
install(DIRECTORY "${CPUINFO_INSTALL_DIR}/"
221-
DESTINATION "${CMAKE_INSTALL_PREFIX}"
206+
set(CPUINFO_LOG_LEVEL
207+
"error"
208+
CACHE STRING ""
209+
)
210+
set(CLOG_SOURCE_DIR "${CPUINFO_SOURCE_DIR}/deps/clog")
211+
add_subdirectory("${CPUINFO_SOURCE_DIR}")
212+
set(CMAKE_POSITION_INDEPENDENT_CODE
213+
${ORIGINAL_CMAKE_POSITION_INDEPENDENT_CODE_FLAG}
222214
)
223215
endif()
224216

225217
if(EXECUTORCH_BUILD_PTHREADPOOL)
226218
# --- pthreadpool
227-
set(PTHREADPOOL_SOURCE_DIR
228-
"${CMAKE_CURRENT_LIST_DIR}/backends/xnnpack/third-party/pthreadpool"
219+
set(ORIGINAL_CMAKE_POSITION_INDEPENDENT_CODE_FLAG
220+
${CMAKE_POSITION_INDEPENDENT_CODE}
229221
)
230-
include(ExternalProject)
231-
include(GNUInstallDirs)
232-
set(PTHREADPOOL_BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/pthreadpool")
233-
set(PTHREADPOOL_INSTALL_DIR "${PTHREADPOOL_BINARY_DIR}/install")
234-
set(PTHREADPOOL_LIBRARY
235-
"${PTHREADPOOL_INSTALL_DIR}/${CMAKE_INSTALL_LIBDIR}/libpthreadpool.a"
222+
set(CMAKE_POSITION_INDEPENDENT_CODE ON)
223+
set(PTHREADPOOL_SOURCE_DIR "backends/xnnpack/third-party/pthreadpool")
224+
set(PTHREADPOOL_BUILD_TESTS
225+
OFF
226+
CACHE BOOL ""
227+
)
228+
set(PTHREADPOOL_BUILD_BENCHMARKS
229+
OFF
230+
CACHE BOOL ""
231+
)
232+
set(PTHREADPOOL_LIBRARY_TYPE
233+
"static"
234+
CACHE STRING ""
235+
)
236+
set(PTHREADPOOL_ALLOW_DEPRECATED_API
237+
ON
238+
CACHE BOOL ""
236239
)
237-
get_extra_cmake_args_for_external_project(PTHREADPOOL_EXTRA_CMAKE_ARGS)
238240
if(APPLE)
239-
list(APPEND PTHREADPOOL_EXTRA_CMAKE_ARGS -D
240-
PTHREADPOOL_SYNC_PRIMITIVE=condvar
241+
set(PTHREADPOOL_SYNC_PRIMITIVE
242+
"condvar"
243+
CACHE STRING ""
241244
)
242245
endif()
243-
ExternalProject_Add(
244-
pthreadpoolExternalProject
245-
SOURCE_DIR "${PTHREADPOOL_SOURCE_DIR}"
246-
BINARY_DIR "${PTHREADPOOL_BINARY_DIR}"
247-
INSTALL_DIR "${PTHREADPOOL_INSTALL_DIR}"
248-
BUILD_BYPRODUCTS "${PTHREADPOOL_LIBRARY}"
249-
CMAKE_ARGS ${PTHREADPOOL_EXTRA_CMAKE_ARGS}
250-
-D
251-
PTHREADPOOL_BUILD_TESTS=OFF
252-
-D
253-
PTHREADPOOL_BUILD_BENCHMARKS=OFF
254-
-D
255-
PTHREADPOOL_LIBRARY_TYPE=static
256-
-D
257-
PTHREADPOOL_ALLOW_DEPRECATED_API=ON
258-
-D
259-
CMAKE_POSITION_INDEPENDENT_CODE=ON
260-
-D
261-
CMAKE_INSTALL_PREFIX=<INSTALL_DIR>
262-
)
263-
add_library(pthreadpool STATIC IMPORTED)
264-
set_property(
265-
TARGET pthreadpool PROPERTY IMPORTED_LOCATION "${PTHREADPOOL_LIBRARY}"
266-
)
267-
add_dependencies(pthreadpool pthreadpoolExternalProject)
268-
# Trailing slash matters here! "Move everything from the temporary pthreadpool
269-
# install directory to the proper install directory."
270-
install(DIRECTORY "${PTHREADPOOL_INSTALL_DIR}/"
271-
DESTINATION "${CMAKE_INSTALL_PREFIX}"
246+
add_subdirectory("${PTHREADPOOL_SOURCE_DIR}")
247+
set(CMAKE_POSITION_INDEPENDENT_CODE
248+
${ORIGINAL_CMAKE_POSITION_INDEPENDENT_CODE_FLAG}
272249
)
273250
endif()
274251

@@ -585,6 +562,10 @@ if(EXECUTORCH_BUILD_EXTENSION_LLM)
585562
add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/extension/llm/tokenizers)
586563
endif()
587564

565+
if(EXECUTORCH_BUILD_EXTENSION_LLM_APPLE)
566+
add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/extension/llm/apple)
567+
endif()
568+
588569
if(EXECUTORCH_BUILD_EXTENSION_LLM_RUNNER)
589570
add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/extension/llm/runner)
590571
endif()
@@ -756,10 +737,7 @@ if(EXECUTORCH_BUILD_EXECUTOR_RUNNER)
756737
endif()
757738

758739
set(CMAKE_EXECUTABLE_SUFFIX ".html")
759-
target_link_options(
760-
executor_runner PUBLIC -sALLOW_MEMORY_GROWTH --embed-file
761-
"${WASM_MODEL_DIR}@/"
762-
)
740+
target_link_options(executor_runner PUBLIC -sALLOW_MEMORY_GROWTH --embed-file "${WASM_MODEL_DIR}@/")
763741
endif()
764742
endif()
765743

‎Package.swift

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,11 @@ let products = deliverables([
6969
"c++",
7070
],
7171
],
72+
"executorch_llm": [
73+
"targets": [
74+
"executorch",
75+
],
76+
],
7277
"kernels_llm": [:],
7378
"kernels_optimized": [
7479
"frameworks": [

‎backends/apple/coreml/compiler/coreml_preprocess.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -365,7 +365,7 @@ def preprocess_model(
365365

366366
match model_type:
367367
case CoreMLBackend.MODEL_TYPE.COMPILED_MODEL:
368-
shutil.rmtree(str(model_path.resolve()))
368+
shutil.rmtree(str(model_path.resolve()), ignore_errors=True)
369369
model_path = model_dir_path / MODEL_PATHS.COMPILED_MODEL.value
370370
compiled_model_path = mlmodel.get_compiled_model_path()
371371
shutil.move(
@@ -396,7 +396,7 @@ def preprocess_model(
396396
for key, value in model_debug_info.debugSymbolToHandles.items()
397397
}
398398

399-
shutil.rmtree(str(dir_path.resolve()))
399+
shutil.rmtree(str(dir_path.resolve()), ignore_errors=True)
400400
return PreprocessResult(
401401
processed_bytes=processed_bytes,
402402
debug_handle_map=debug_handle_map,

‎backends/apple/coreml/partition/coreml_partitioner.py

Lines changed: 92 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -28,12 +28,21 @@
2828

2929
class OperatorsSupportedForCoreMLBackend(OperatorSupportBase):
3030
def __init__(
31-
self, skip_ops_for_coreml_delegation: Optional[List[str]] = None
31+
self,
32+
skip_ops_for_coreml_delegation: Optional[List[str]] = None,
33+
lower_full_graph: bool = False,
3234
) -> None:
3335
if skip_ops_for_coreml_delegation is None:
3436
skip_ops_for_coreml_delegation = []
3537
super().__init__()
3638
self.skip_ops_for_coreml_delegation = skip_ops_for_coreml_delegation
39+
self.lower_full_graph = lower_full_graph
40+
self._logged_msgs = set()
41+
42+
def log_once(self, msg: str) -> None:
43+
if msg not in self._logged_msgs:
44+
logging.info(msg)
45+
self._logged_msgs.add(msg)
3746

3847
def is_node_supported(self, submodules, node: torch.fx.Node) -> bool:
3948
# get_attr node can always be supported on any backend
@@ -44,14 +53,63 @@ def is_node_supported(self, submodules, node: torch.fx.Node) -> bool:
4453
# skip ops if specified by user
4554
node_target_name = getattr(node.target, "__name__", "").lower()
4655
if node_target_name in (self.skip_ops_for_coreml_delegation or []):
56+
self.log_once(
57+
"Skipping op for CoreML delegation because it is in skip_ops_for_coreml_delegation: "
58+
+ node_target_name
59+
)
60+
assert (
61+
not self.lower_full_graph
62+
), "Cannot have skip_ops_for_coreml_delegation when lower_full_graph is True"
4763
return False
64+
65+
# TODO: enable this after bugs in ExecuTorch's partitioner are fixed
66+
# # If lower_full_graph=False, do not partition nodes with symbolic args because it can result in symbolic args
67+
# # in the placeholders due to partitioning, which CoreML does not support
68+
# if not self.lower_full_graph and any(
69+
# isinstance(arg, torch.fx.Node)
70+
# and isinstance(
71+
# arg.meta.get("val", None),
72+
# (torch.SymInt, torch.SymBool, torch.SymFloat),
73+
# )
74+
# for arg in node.args
75+
# ):
76+
# self.log_once(
77+
# "Skipping op for CoreML delegation because it contains symbolic args: "
78+
# + node_target_name
79+
# )
80+
# assert not self.lower_full_graph
81+
# return False
82+
4883
# query coremltools to see if node is supported
49-
return ct.converters.mil.frontend.torch.is_torch_fx_node_supported(node)
84+
is_supported = ct.converters.mil.frontend.torch.is_torch_fx_node_supported(
85+
node
86+
)
87+
if not is_supported:
88+
if self.lower_full_graph:
89+
raise NotImplementedError(
90+
f"""CoreML does not support the op {node_target_name}, but you have set lower_full_graph=True in the CoreMLPartitioner.
91+
92+
Please set lower_full_graph=False in the CoreMLPartitioner to allow running unsupported ops outside of CoreML. Note that setting lower_full_graph=False may affect performance of CoreML and the available features.
93+
As an alternative to setting lower_full_graph=False, you can try rewriting your model to avoid using this op.
94+
95+
Also consider filing an issue with Apple's coremltools repo to request support for the op: https://github.com/apple/coremltools/issues
96+
Do not file an issue with ExecuTorch for op support.
97+
"""
98+
)
99+
self.log_once(
100+
"Skipping op for CoreML delegation because it is not supported by CoreML: "
101+
+ node_target_name
102+
)
103+
return is_supported
50104
# cowardly refuse to support all other types of node:
51105
# 1. placeholder / output nodes should not be tagged
52106
# reference: https://github.com/pytorch/executorch/pull/1398
53107
# 2. call_module / call_method should have been replaced with call_function?
54108
else:
109+
self.log_once(
110+
"Skipping op for CoreML delegation because it is not get_attr or call_function: "
111+
+ node.op
112+
)
55113
return False
56114

57115

@@ -62,6 +120,8 @@ def __init__(
62120
skip_ops_for_coreml_delegation: Optional[List[str]] = None,
63121
compile_specs: Optional[List[CompileSpec]] = None,
64122
take_over_mutable_buffer: Optional[bool] = True,
123+
lower_full_graph: bool = False,
124+
take_over_constant_data: bool = True,
65125
) -> None:
66126
if skip_ops_for_coreml_delegation is None:
67127
skip_ops_for_coreml_delegation = []
@@ -71,6 +131,20 @@ def __init__(
71131
compile_specs=compile_specs if compile_specs is not None else [],
72132
)
73133
self.take_over_mutable_buffer = take_over_mutable_buffer
134+
self.lower_full_graph = lower_full_graph
135+
self.take_over_constant_data = take_over_constant_data
136+
self._logged_msgs = set()
137+
138+
if self.lower_full_graph:
139+
assert (
140+
len(self.skip_ops_for_coreml_delegation) == 0
141+
), "When lower_full_graph=True, you cannot set skip_ops_for_coreml_delegation"
142+
assert (
143+
self.take_over_constant_data
144+
), "When lower_full_graph=True, you must set take_over_constant_data=True"
145+
assert (
146+
self.take_over_mutable_buffer
147+
), "When lower_full_graph=True, you must set take_over_mutable_buffer=True"
74148

75149
def partition(self, exported_program: ExportedProgram) -> PartitionResult:
76150
# Run the CapabilityBasedPartitioner to return the largest possible
@@ -80,7 +154,9 @@ def partition(self, exported_program: ExportedProgram) -> PartitionResult:
80154

81155
capability_partitioner = CapabilityBasedPartitioner(
82156
exported_program.graph_module,
83-
OperatorsSupportedForCoreMLBackend(self.skip_ops_for_coreml_delegation),
157+
OperatorsSupportedForCoreMLBackend(
158+
self.skip_ops_for_coreml_delegation, self.lower_full_graph
159+
),
84160
allows_single_node_partition=True,
85161
)
86162
partition_list = capability_partitioner.propose_partitions()
@@ -90,7 +166,8 @@ def partition(self, exported_program: ExportedProgram) -> PartitionResult:
90166
node.meta["delegation_tag"] = tag
91167
partition_tags[tag] = self.delegation_spec
92168

93-
tag_constant_data(exported_program)
169+
if self.take_over_constant_data:
170+
tag_constant_data(exported_program)
94171
if self.take_over_mutable_buffer:
95172
logger.info(
96173
"Core ML partitioner will take over torch mutable buffer as Core ML state, "
@@ -105,12 +182,18 @@ def partition(self, exported_program: ExportedProgram) -> PartitionResult:
105182
tagged_exported_program=exported_program, partition_tags=partition_tags
106183
)
107184

185+
def log_once(self, msg: str) -> None:
186+
if msg not in self._logged_msgs:
187+
logging.info(msg)
188+
self._logged_msgs.add(msg)
189+
108190
def ops_to_not_decompose(
109191
self, ep: ExportedProgram
110192
) -> Tuple[List[torch._ops.OpOverload], Optional[Callable[[torch.fx.Node], bool]]]:
111193
do_not_decompose = []
112-
op_support = OperatorsSupportedForCoreMLBackend()
113-
_logged_warnings = set()
194+
op_support = OperatorsSupportedForCoreMLBackend(
195+
self.skip_ops_for_coreml_delegation, self.lower_full_graph
196+
)
114197

115198
# CoreML prevents certain ops (like triu) from lowering to CoreML when put in the ExecuTorch op namespace
116199
# TODO: upstream fixes, but pending ET consuming a new published version of coremltools with the
@@ -134,9 +217,7 @@ def ops_to_not_decompose(
134217
except Exception as e:
135218
# CoreML's op_support.is_node_supported will sometimes throw
136219
# for unsupported ops, rather than returning False
137-
warn_str = f"Encountered exception when checking node support: {e}"
138-
if warn_str not in _logged_warnings:
139-
logger.warning(warn_str)
140-
_logged_warnings.add(warn_str)
141-
220+
self.log_once(
221+
f"Encountered exception when checking node support, treating node as unsupported: {e}"
222+
)
142223
return do_not_decompose, None

0 commit comments

Comments
 (0)