Benchmark-Sweep script upgrades (#830)

* Align row values with column Headers

* Upgrade `benchmark_sweep.py` to a cli script
Add error catching
Make `benchmark_sweep` function more robust
Default `num_cores` to max available cores if `None`, (used to fail before due to a call to `_parse_num_cores` when in `benchmark_model`)
Add docstrings wherever necessary
Add Style

* Address comments from @KSGulin and @corey-nm

* Address comments from @bfineran

* Dynamic Engine Load (#842)

* Add skeleton for loading custom `Engine`

* Add code to dynamically import the engine class

* Add code on benchmark_sweep side to accept engine from user

* Add support for generating random inputs using the engine object (required for dynamically imported engine classes) as the graph file might not be in `ONNX`, so our `generate_random_inputs` function could potentially fail

* Update src/deepsparse/benchmark/benchmark_model.py

Co-authored-by: Benjamin Fineran <[email protected]>

Co-authored-by: Benjamin Fineran <[email protected]>

* Address comments from @bfineran

Co-authored-by: Benjamin Fineran <[email protected]>

Author: rahul-tuli

src/deepsparse/benchmark/benchmark_model.py

@@ -92,6 +92,7 @@
 """
 
 import argparse
+import importlib
 import json
 import logging
 import os
@@ -214,10 +215,12 @@ def parse_args():
         "--engine",
         type=str,
         default=DEEPSPARSE_ENGINE,
-        choices=[DEEPSPARSE_ENGINE, ORT_ENGINE],
         help=(
             "Inference engine backend to run eval on. Choices are 'deepsparse', "
-            "'onnxruntime'. Default is 'deepsparse'"
+            "'onnxruntime'. Default is 'deepsparse'. Can also specify a user "
+            "defined engine class by giving the script and class name in the "
+            "following format <path to python script>:<Engine Class name>. This "
+            "engine class will be dynamically imported during runtime"
         ),
     )
     parser.add_argument(
@@ -308,6 +311,22 @@ def parse_num_streams(num_streams: int, num_cores: int, scenario: str):
             return default_num_streams
 
 
+def load_custom_engine(custom_engine_identifier: str):
+    """
+    import a custom engine based off the specified `custom_engine_identifier`
+    from user specified script
+
+    :param custom_engine_identifier: string in the form of
+           '<path_to_the_python_script>:<custom_engine_class_name>
+    :return: custom engine class object
+    """
+    path, engine_object_name = custom_engine_identifier.split(":")
+    spec = importlib.util.spec_from_file_location("user_defined_custom_engine", path)
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return getattr(module, engine_object_name)
+
+
 def benchmark_model(
     model_path: str,
     batch_size: int = 1,
@@ -352,6 +371,13 @@ def benchmark_model(
             num_cores=num_cores,
             input_shapes=input_shapes,
         )
+    elif ":" in engine:
+        engine = load_custom_engine(custom_engine_identifier=engine)
+        model = engine(
+            model_path=model_path,
+            batch_size=batch_size,
+            num_cores=num_cores,
+        )
     else:
         raise ValueError(f"Invalid engine choice '{engine}'")
     _LOGGER.info(model)
@@ -361,6 +387,8 @@ def benchmark_model(
     if input_shapes:
         with override_onnx_input_shapes(model_path, input_shapes) as model_path:
             input_list = generate_random_inputs(model_path, batch_size)
+    elif hasattr(engine, "generate_random_inputs"):
+        input_list = engine.generate_random_inputs(batch_size=batch_size)
     else:
         input_list = generate_random_inputs(model_path, batch_size)