Skip to content

Commit cdeea84

Browse files
cymbalrushcymbalrush
committed
Adds CoreML backend
1 parent 620b769 commit cdeea84

File tree

106 files changed

+39172
-0
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

106 files changed

+39172
-0
lines changed

‎CMakeLists.txt‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -295,6 +295,18 @@ if(EXECUTORCH_BUILD_XNNPACK)
295295
add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/backends/xnnpack)
296296
endif()
297297

298+
# Add COREML subdirectory
299+
if(EXECUTORCH_BUILD_COREML_DELGATE)
300+
# CoreML delegate library can only be built with iOS toolchain
301+
if(CMAKE_TOOLCHAIN_FILE MATCHES ".*ios\.toolchain\.cmake$")
302+
add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/backends/coreml)
303+
else()
304+
message(
305+
FATAL_ERROR "executorch: Building CoreML delegate requires iOS toolchain"
306+
)
307+
endif()
308+
endif()
309+
298310
# Add selective build subdirectory
299311
if(BUILD_SELECTIVE_BUILD_TEST)
300312
add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/examples/selective_build)

‎backends/coreml/.clang-format‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
BasedOnStyle: WebKit
2+
BreakBeforeBraces: Attach
3+
AllowShortIfStatementsOnASingleLine: false
4+
BreakBeforeBinaryOperators: None
5+
BreakConstructorInitializers: BeforeColon
6+
IndentCaseLabels: true
7+
SortIncludes: true
8+
SpaceBeforeRangeBasedForLoopColon: false
9+
SpaceBeforeParens: ControlStatements
10+
AlignAfterOpenBracket: Align
11+
NamespaceIndentation: None
12+
MaxEmptyLinesToKeep: 2
13+
#AlignConsecutiveAssignments: true
14+
15+
ColumnLimit: 120
16+
17+
# When breaking up parameter/argument lists across multiple lines,
18+
# put only one per line instead of packing as many as possible.
19+
BinPackArguments: false
20+
BinPackParameters: false
21+
22+
# Control of individual brace wrapping cases.
23+
BreakBeforeBraces: Custom
24+
BraceWrapping:
25+
BeforeCatch: true
26+
27+
# Options for aligning backslashes in escaped newlines.
28+
AlignEscapedNewlines: Left
29+

‎backends/coreml/.gitignore‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
# Mac OS X
2+
*.DS_Store
3+
4+
cmake-ios-out/
5+
6+
7+
# Xcode
8+
*.pbxuser
9+
*.mode1v3
10+
*.mode2v3
11+
*.perspectivev3
12+
*.xcuserstate
13+
project.xcworkspace/
14+
xcuserdata/
15+
16+
17+
runtime/runner/build/
18+
runtime/libraries/
19+
runtime/inmemoryfs/build/
20+
xcode-runner-build/
21+
xcode-test-build/
22+
23+
*.egg-info

‎backends/coreml/CMakeLists.txt‎

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
# Copyright © 2023 Apple Inc. All rights reserved.
2+
3+
cmake_minimum_required(VERSION 3.19)
4+
5+
project(executorch_coreml_backend)
6+
7+
enable_language(CXX)
8+
enable_language(OBJC)
9+
10+
# Source root directory for executorch.
11+
if(NOT EXECUTORCH_ROOT)
12+
set(EXECUTORCH_ROOT ${CMAKE_CURRENT_SOURCE_DIR}/../..)
13+
endif()
14+
15+
# inmemoryfs sources
16+
set(INMEMORYFS_SOURCES
17+
runtime/inmemoryfs/inmemory_filesystem.cpp
18+
runtime/inmemoryfs/memory_buffer.cpp
19+
runtime/inmemoryfs/memory_stream.cpp
20+
runtime/inmemoryfs/reversed_memory_stream.cpp
21+
)
22+
23+
# kvstore sources
24+
set(KVSTORE_SOURCES
25+
runtime/kvstore/database.cpp
26+
runtime/kvstore/sqlite_error.cpp
27+
runtime/kvstore/key_value_store.cpp
28+
runtime/kvstore/statement.cpp
29+
)
30+
31+
# delegate sources
32+
set(DELEGATE_SOURCES
33+
runtime/delegate/asset.mm
34+
runtime/delegate/backend_delegate.mm
35+
runtime/delegate/coreml_backend_delegate.mm
36+
runtime/delegate/ETCoreMLAsset.mm
37+
runtime/delegate/ETCoreMLAssetManager.mm
38+
runtime/delegate/ETCoreMLLogging.mm
39+
runtime/delegate/ETCoreMLModel.mm
40+
runtime/delegate/ETCoreMLModelManager.mm
41+
runtime/delegate/ETCoreMLStrings.mm
42+
runtime/delegate/MLModel_Prewarm.mm
43+
runtime/delegate/MLMultiArray_Copy.mm
44+
runtime/delegate/multiarray.mm
45+
runtime/delegate/serde_json.mm
46+
)
47+
48+
# Define the delegate library
49+
add_library(coremldelegate)
50+
target_sources(
51+
coremldelegate PRIVATE
52+
${INMEMORYFS_SOURCES}
53+
${KVSTORE_SOURCES}
54+
${DELEGATE_SOURCES}
55+
)
56+
57+
target_include_directories(
58+
coremldelegate PRIVATE
59+
${CMAKE_CURRENT_SOURCE_DIR}/runtime/kvstore
60+
)
61+
target_include_directories(
62+
coremldelegate PRIVATE
63+
${CMAKE_CURRENT_SOURCE_DIR}/runtime/inmemoryfs
64+
)
65+
target_include_directories(
66+
coremldelegate PRIVATE
67+
${CMAKE_CURRENT_SOURCE_DIR}/runtime/delegate
68+
)
69+
target_include_directories(
70+
coremldelegate PRIVATE
71+
${CMAKE_CURRENT_SOURCE_DIR}/third-party/nlohmann
72+
)
73+
target_include_directories(
74+
coremldelegate PRIVATE
75+
${EXECUTORCH_ROOT}/..
76+
)
77+
78+
target_link_libraries(
79+
coremldelegate PRIVATE
80+
executorch
81+
gflags
82+
)
83+
84+
target_compile_options(coremldelegate PUBLIC "-fobjc-arc")
85+
86+
set(
87+
TARGET coremldelegate
88+
APPEND_STRING PROPERTY COMPILE_FLAGS "-x objective-c++"
89+
)
90+
91+
set(
92+
TARGET coremldelegate
93+
APPEND_STRING PROPERTY COMPILE_FLAGS "-Wno-null-character"
94+
)
95+
96+
set(
97+
TARGET coremldelegate
98+
APPEND_STRING PROPERTY COMPILE_FLAGS "-Wno-receiver-expr"
99+
)
100+
101+
set_property(
102+
TARGET coremldelegate
103+
PROPERTY CXX_STANDARD 17
104+
)
105+

‎backends/coreml/README.md‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
## CoreML Backend
2+
3+
4+
For setting up the **CoreML** backend please follow the instructions described in `backend/coreml/setup.md`.
5+
6+
## AOT
7+
8+
For delegating the Program to the **CoreML** backend, the client must be responsible for calling `to_backend` with the **CoreMLBackend** tag.
9+
10+
```
11+
from executorch.backends.coreml.compiler import CoreMLBackend
12+
13+
# This will delegate the whole program to the CoreML backend.
14+
lowered_module = to_backend("CoreMLBackend", edge.exported_program, [])
15+
16+
```
17+
18+
Currently, the **CoreML** backend would delegate the whole module to **CoreML**. If a specific op is not supported by the **CoreML** backend then the `to_backend` call would throw an exception. We will be adding a **Partitioner** for **CoreML** soon to resolve the issue.
19+
20+
The `preprocess` code is located at `backends/coreml/coreml_preprocess.py`. The implementation uses `coremltools` to convert the **ExportedProgram** to **MLPackage** which is flattened and returned as bytes to **ExecuTorch**.
21+
22+
### Compute Units
23+
24+
The `CoreML` runtime by default loads the model with compute units set to `MLComputeUnitsAll`. This can be overridden by providing `compute_units` at the preprocessing stage.
25+
26+
**MLComputeUnitsCPUOnly**
27+
28+
```
29+
# cpu is equivalent to MLComputeUnitsCPUOnly.
30+
lowered_module = to_backend("CoreMLBackend", edge.exported_program, [CompileSpec("compute_units", bytes("cpu", 'utf-8'))])
31+
32+
```
33+
34+
**MLComputeUnitsCPUAndGPU**
35+
36+
```
37+
# gpu is equivalent to MLComputeUnitsCPUAndGPU.
38+
lowered_module = to_backend("CoreMLBackend", edge.exported_program, [CompileSpec("compute_units", bytes("gpu", 'utf-8'))])
39+
40+
```
41+
42+
43+
**MLComputeUnitsCPUAndNeuralEngine**
44+
45+
```
46+
# ane is equivalent to MLComputeUnitsCPUAndNeuralEngine.
47+
lowered_module = to_backend("CoreMLBackend", edge.exported_program, [CompileSpec("compute_units", bytes("ane", 'utf-8'))])
48+
49+
```
50+
51+
**MLComputeUnitsAll**
52+
53+
```
54+
# all is equivalent to MLComputeUnitsAll.
55+
lowered_module = to_backend("CoreMLBackend", edge.exported_program, [CompileSpec("compute_units", bytes("all", 'utf-8'))])
56+
57+
```
58+
59+
60+
## Runtime
61+
62+
Directory Structure.
63+
```
64+
65+
executorch
66+
├── backends
67+
├── coreml # CoreML backend implementation.
68+
├── runtime # CoreML runtime implementation.
69+
├── inmemoryfs # In-Memory filesystem implementation.
70+
├── kvstore # Persistent key-value store implementation.
71+
├── delegate # Delegate implementation.
72+
├── libraries # Linked libraries.
73+
├── runner # Runner app implementation.
74+
├── test # Test files and models.
75+
76+
```
77+
78+
## Integration
79+
80+
There are two steps required to integrate the **CoreML** backend.
81+
- Exporting the Program: The client must call `to_backend` with the `CoreMLBackend` tag.
82+
83+
```
84+
from executorch.backends.coreml.compiler import CoreMLBackend
85+
86+
# This will delegate the whole program to the CoreML backend.
87+
lowered_module = to_backend("CoreMLBackend", edge.exported_program, [])
88+
89+
```
90+
- Running the delegated program: The client application must link with the `coremldelegate` library. Please follow the instructions described in the `backends/coreml/setup.md` to link the `coremldelegate` library. Once linked, there are no additional steps required. **ExecuTorch** would automatically call the **CoreML** delegate to execute the **CoreML** delegated part of the exported **Program**.
91+
92+

‎backends/coreml/compiler/__init__.py‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
# Copyright © 2023 Apple Inc. All rights reserved.
2+
#
3+
# Please refer to the license found in the LICENSE file in the root directory of the source tree.
4+
5+
from .coreml_preprocess import CoreMLBackend
6+
7+
__all__ = [
8+
CoreMLBackend,
9+
]

‎backends/coreml/compiler/coreml_preprocess.py‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
# Copyright © 2023 Apple Inc. All rights reserved.
2+
3+
# CoreML backend for delegating a EdgeProgram to CoreML.
4+
5+
import json
6+
import shutil
7+
import uuid
8+
9+
from pathlib import Path
10+
11+
from typing import final, List
12+
13+
import coremltools as ct
14+
import executorchcoreml
15+
16+
from executorch.exir.backend.backend_details import (
17+
BackendDetails,
18+
ExportedProgram,
19+
PreprocessResult,
20+
)
21+
from executorch.exir.backend.compile_spec_schema import CompileSpec
22+
23+
24+
@final
25+
class CoreMLBackend(BackendDetails):
26+
@staticmethod
27+
def to_bytes(mlmodel):
28+
dir_path = Path("tmp")
29+
model_dir_path = dir_path / "lowered_module"
30+
Path(model_dir_path).mkdir(parents=True, exist_ok=True)
31+
model_path = model_dir_path / "model.mlpackage"
32+
mlmodel.save(model_path)
33+
34+
# save model metdata
35+
spec = mlmodel.get_spec()
36+
input_names = [input.name for input in spec.description.input]
37+
output_names = [output.name for output in spec.description.output]
38+
identifier = uuid.uuid4()
39+
40+
model_metadata = {
41+
"inputNames": input_names,
42+
"outputNames": output_names,
43+
"identifier": str(identifier),
44+
}
45+
46+
# store metadata
47+
model_metadata_path = Path(model_dir_path) / "metadata.json"
48+
json_object = json.dumps(model_metadata)
49+
with open(model_metadata_path, "w") as outfile:
50+
outfile.write(json_object)
51+
52+
# flatten directory contents and convert it to bytes
53+
flattened_bytes = executorchcoreml.flatten_directory_contents(
54+
str(model_dir_path.resolve())
55+
)
56+
shutil.rmtree(str(model_dir_path.resolve()))
57+
return flattened_bytes
58+
59+
@classmethod
60+
# pyre-ignore
61+
def preprocess(
62+
cls,
63+
edge_program: ExportedProgram,
64+
module_compile_spec: List[CompileSpec],
65+
) -> PreprocessResult:
66+
mlmodel = ct.convert(
67+
model=edge_program,
68+
source="pytorch",
69+
convert_to="mlprogram",
70+
pass_pipeline=ct.PassPipeline.DEFAULT,
71+
skip_model_load=True,
72+
)
73+
flattened_bytes = CoreMLBackend.to_bytes(mlmodel)
74+
return PreprocessResult(
75+
processed_bytes=flattened_bytes,
76+
)

‎backends/coreml/pyproject.toml‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
[build-system]
2+
requires = ["setuptools", "wheel"]
3+
4+
[tool.setuptools]
5+
py-modules = []
6+
7+
[project]
8+
name = "executorchcoreml"
9+
version = "0.0.1"
10+
11+
# Python dependencies required for development
12+
dependencies=[
13+
"pybind11",
14+
"numpy",
15+
]

0 commit comments

Comments
 (0)