From 49459470365d4bef2ed54ddb42612d7a527b2756 Mon Sep 17 00:00:00 2001 From: syntron Date: Tue, 24 Jun 2025 16:31:42 +0200 Subject: [PATCH 01/24] [ModelicaSystemDoE] add class --- OMPython/ModelicaSystem.py | 236 ++++++++++++++++++++++++++++++++++++- 1 file changed, 235 insertions(+), 1 deletion(-) diff --git a/OMPython/ModelicaSystem.py b/OMPython/ModelicaSystem.py index ae480dde..04e94b2d 100644 --- a/OMPython/ModelicaSystem.py +++ b/OMPython/ModelicaSystem.py @@ -35,17 +35,21 @@ import ast import csv from dataclasses import dataclass +import itertools import logging import numbers import numpy as np import os +import pandas as pd import pathlib import platform +import queue import re import subprocess import tempfile import textwrap -from typing import Optional, Any +import threading +from typing import Any, Optional import warnings import xml.etree.ElementTree as ET @@ -1637,3 +1641,233 @@ def getLinearOutputs(self) -> list[str]: def getLinearStates(self) -> list[str]: """Get names of state variables of the linearized model.""" return self._linearized_states + + +class ModelicaSystemDoE: + def __init__( + self, + fileName: Optional[str | os.PathLike | pathlib.Path] = None, + modelName: Optional[str] = None, + lmodel: Optional[list[str | tuple[str, str]]] = None, + commandLineOptions: Optional[str] = None, + variableFilter: Optional[str] = None, + customBuildDirectory: Optional[str | os.PathLike | pathlib.Path] = None, + omhome: Optional[str] = None, + + simargs: Optional[dict[str, Optional[str | dict[str, str]]]] = None, + timeout: Optional[int] = None, + + resultpath: Optional[pathlib.Path] = None, + parameters: Optional[dict[str, list[str] | list[int] | list[float]]] = None, + ) -> None: + self._lmodel = lmodel + self._modelName = modelName + self._fileName = fileName + + self._CommandLineOptions = commandLineOptions + self._variableFilter = variableFilter + self._customBuildDirectory = customBuildDirectory + self._omhome = omhome + + # reference for the model; not used for any simulations but to evaluate parameters, etc. + self._mod = ModelicaSystem( + fileName=self._fileName, + modelName=self._modelName, + lmodel=self._lmodel, + commandLineOptions=self._CommandLineOptions, + variableFilter=self._variableFilter, + customBuildDirectory=self._customBuildDirectory, + omhome=self._omhome, + ) + + self._simargs = simargs + self._timeout = timeout + + if isinstance(resultpath, pathlib.Path): + self._resultpath = resultpath + else: + self._resultpath = pathlib.Path('.') + + if isinstance(parameters, dict): + self._parameters = parameters + else: + self._parameters = {} + + self._sim_df: Optional[pd.DataFrame] = None + self._sim_task_query: queue.Queue = queue.Queue() + + def prepare(self) -> int: + + param_structure = {} + param_simple = {} + for param_name in self._parameters.keys(): + changeable = self._mod.isParameterChangeable(name=param_name) + logger.info(f"Parameter {repr(param_name)} is changeable? {changeable}") + + if changeable: + param_simple[param_name] = self._parameters[param_name] + else: + param_structure[param_name] = self._parameters[param_name] + + param_structure_combinations = list(itertools.product(*param_structure.values())) + param_simple_combinations = list(itertools.product(*param_simple.values())) + + df_entries: list[pd.DataFrame] = [] + for idx_pc_structure, pc_structure in enumerate(param_structure_combinations): + mod_structure = ModelicaSystem( + fileName=self._fileName, + modelName=self._modelName, + lmodel=self._lmodel, + commandLineOptions=self._CommandLineOptions, + variableFilter=self._variableFilter, + customBuildDirectory=self._customBuildDirectory, + omhome=self._omhome, + ) + + sim_args_structure = {} + for idx_structure, pk_structure in enumerate(param_structure.keys()): + sim_args_structure[pk_structure] = pc_structure[idx_structure] + + pk_value = pc_structure[idx_structure] + if isinstance(pk_value, str): + expression = f"setParameterValue({self._modelName}, {pk_structure}, $Code(=\"{pk_value}\"))" + elif isinstance(pk_value, bool): + pk_value_bool_str = "true" if pk_value else "false" + expression = f"setParameterValue({self._modelName}, {pk_structure}, $Code(={pk_value_bool_str}));" + else: + expression = f"setParameterValue({self._modelName}, {pk_structure}, {pk_value})" + mod_structure.sendExpression(expression) + + for idx_pc_simple, pc_simple in enumerate(param_simple_combinations): + sim_args_simple = {} + for idx_simple, pk_simple in enumerate(param_simple.keys()): + sim_args_simple[pk_simple] = str(pc_simple[idx_simple]) + + resfilename = f"DOE_{idx_pc_structure:09d}_{idx_pc_simple:09d}.mat" + logger.info(f"use result file {repr(resfilename)} " + f"for structural parameters: {sim_args_structure} " + f"and simple parameters: {sim_args_simple}") + resultfile = self._resultpath / resfilename + + df_data = ( + { + 'ID structure': idx_pc_structure, + 'ID simple': idx_pc_simple, + 'resulfilename': resfilename, + 'structural parameters ID': idx_pc_structure, + } + | sim_args_structure + | { + 'non-structural parameters ID': idx_pc_simple, + } + | sim_args_simple + | { + 'results available': False, + } + ) + + df_entries.append(pd.DataFrame.from_dict(df_data)) + + cmd = mod_structure.simulate_cmd( + resultfile=resultfile.absolute().resolve(), + simargs={"override": sim_args_simple}, + ) + + self._sim_task_query.put(cmd) + + self._sim_df = pd.concat(df_entries, ignore_index=True) + + logger.info(f"Prepared {self._sim_df.shape[0]} simulation definitions for the defined DoE.") + + return self._sim_df.shape[0] + + def get_doe(self) -> Optional[pd.DataFrame]: + return self._sim_df + + def simulate(self, num_workers: int = 3) -> None: + + sim_count_total = self._sim_task_query.qsize() + + def worker(worker_id, task_queue): + while True: + sim_data = {} + try: + # Get the next task from the queue + cmd: ModelicaSystemCmd = task_queue.get(block=False) + except queue.Empty: + logger.info(f"[Worker {worker_id}] No more simulations to run.") + break + + resultfile = cmd.arg_get(key='r') + resultpath = pathlib.Path(resultfile) + + logger.info(f"[Worker {worker_id}] Performing task: {resultpath.name}") + + try: + sim_data['sim'].run() + except ModelicaSystemError as ex: + logger.warning(f"Simulation error for {resultpath.name}: {ex}") + + # Mark the task as done + task_queue.task_done() + + sim_count_done = sim_count_total - self._sim_task_query.qsize() + logger.info(f"[Worker {worker_id}] Task completed: {resultpath.name} " + f"({sim_count_done}/{sim_count_total} = " + f"{sim_count_done / sim_count_total * 100:.2f}% of tasks left)") + + logger.info(f"Start simulations for DoE with {sim_count_total} simulations " + f"using {num_workers} workers ...") + + # Create and start worker threads + threads = [] + for i in range(num_workers): + thread = threading.Thread(target=worker, args=(i, self._sim_task_query)) + thread.start() + threads.append(thread) + + # Wait for all threads to complete + for thread in threads: + thread.join() + + for idx, row in self._sim_df.to_dict('index').items(): + resultfilename = row['resultfilename'] + resultfile = self._resultpath / resultfilename + + if resultfile.exists(): + self._sim_df.loc[idx, 'results available'] = True + + sim_total = self._sim_df.shape[0] + sim_done = self._sim_df['results available'].sum() + logger.info(f"All workers finished ({sim_done} of {sim_total} simulations with a result file).") + + def get_solutions( + self, + var_list: Optional[list] = None, + ) -> Optional[tuple[str] | dict[str, pd.DataFrame | str]]: + if self._sim_df is None: + return None + + if self._sim_df.shape[0] == 0 or self._sim_df['results available'].sum() == 0: + raise ModelicaSystemError("No result files available - all simulations did fail?") + + if var_list is None: + resultfilename = self._sim_df['resultfilename'].values[0] + resultfile = self._resultpath / resultfilename + return self._mod.getSolutions(resultfile=resultfile) + + sol_dict: dict[str, pd.DataFrame | str] = {} + for row in self._sim_df.to_dict('records'): + resultfilename = row['resultfilename'] + resultfile = self._resultpath / resultfilename + + try: + sol = self._mod.getSolutions(varList=var_list, resultfile=resultfile) + sol_data = {var: sol[idx] for idx, var in var_list} + sol_df = pd.DataFrame(sol_data) + sol_dict[resultfilename] = sol_df + except ModelicaSystemError as ex: + logger.warning(f"No solution for {resultfilename}: {ex}") + sol_dict[resultfilename] = str(ex) + + return sol_dict From a4ce2a83af1eadcf29e6395f7e9889f395e8215a Mon Sep 17 00:00:00 2001 From: syntron Date: Tue, 24 Jun 2025 16:32:04 +0200 Subject: [PATCH 02/24] [__init__] add class ModelicaSystemDoE --- OMPython/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/OMPython/__init__.py b/OMPython/__init__.py index 1da0a0a3..8a0584fd 100644 --- a/OMPython/__init__.py +++ b/OMPython/__init__.py @@ -36,7 +36,8 @@ CONDITIONS OF OSMC-PL. """ -from OMPython.ModelicaSystem import LinearizationResult, ModelicaSystem, ModelicaSystemCmd, ModelicaSystemError +from OMPython.ModelicaSystem import (LinearizationResult, ModelicaSystem, ModelicaSystemCmd, ModelicaSystemDoE, + ModelicaSystemError) from OMPython.OMCSession import (OMCSessionCmd, OMCSessionException, OMCSessionZMQ, OMCProcessPort, OMCProcessLocal, OMCProcessDocker, OMCProcessDockerContainer, OMCProcessWSL) @@ -46,6 +47,7 @@ 'LinearizationResult', 'ModelicaSystem', 'ModelicaSystemCmd', + 'ModelicaSystemDoE', 'ModelicaSystemError', 'OMCSessionCmd', From cc363c3fa92a21a59212364aa38e2be61e48c907 Mon Sep 17 00:00:00 2001 From: syntron Date: Tue, 24 Jun 2025 16:32:57 +0200 Subject: [PATCH 03/24] [test_ModelicaSystemDoE] add test --- tests/test_ModelicaSystemDoE.py | 58 +++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 tests/test_ModelicaSystemDoE.py diff --git a/tests/test_ModelicaSystemDoE.py b/tests/test_ModelicaSystemDoE.py new file mode 100644 index 00000000..a49ef111 --- /dev/null +++ b/tests/test_ModelicaSystemDoE.py @@ -0,0 +1,58 @@ +import OMPython +import pandas as pd +import pathlib +import pytest + +@pytest.fixture +def model_doe(tmp_path: pathlib.Path) -> pathlib.Path: + # see: https://trac.openmodelica.org/OpenModelica/ticket/4052 + mod = tmp_path / "M.mo" + mod.write_text(f""" +model M + parameter Integer p=1; + parameter Integer q=1; + parameter Real a = -1; + parameter Real b = -1; + Real x[p]; + Real y[q]; +equation + der(x) = a * fill(1.0, p); + der(y) = b * fill(1.0, q); +end M; +""") + return mod + + +@pytest.fixture +def param_doe() -> dict[str, list]: + param = { + # structural + 'p': [1, 2], + 'q': [3, 4], + # simple + 'a': [5, 6], + 'b': [7, 8], + } + return param + + +def test_ModelicaSystemDoE(tmp_path, model_doe, param_doe): + + tmpdir = tmp_path / 'DoE' + tmpdir.mkdir(exist_ok=True) + + mod_doe = OMPython.ModelicaSystemDoE( + fileName=model_doe.as_posix(), + modelName="M", + parameters=param_doe, + resultpath=tmpdir, + ) + mod_doe.prepare() + df_doe = mod_doe.get_doe() + assert isinstance(df_doe, pd.DataFrame) + assert df_doe.shape[0] == 16 + assert df_doe['results available'].sum() == 16 + + mod_doe.simulate() + sol = mod_doe.get_solutions(var_list=['x[1]', 'y[1]']) + assert len(sol) == 16 From 4f19a3ede587cab393e85ecfc9f1698ad97389d6 Mon Sep 17 00:00:00 2001 From: syntron Date: Wed, 25 Jun 2025 21:01:25 +0200 Subject: [PATCH 04/24] [ModelicaSystemDoE] add docstrings --- OMPython/ModelicaSystem.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/OMPython/ModelicaSystem.py b/OMPython/ModelicaSystem.py index 04e94b2d..c07981b5 100644 --- a/OMPython/ModelicaSystem.py +++ b/OMPython/ModelicaSystem.py @@ -1644,6 +1644,10 @@ def getLinearStates(self) -> list[str]: class ModelicaSystemDoE: + """ + Class to run DoEs based on a (Open)Modelica model using ModelicaSystem + """ + def __init__( self, fileName: Optional[str | os.PathLike | pathlib.Path] = None, @@ -1660,6 +1664,11 @@ def __init__( resultpath: Optional[pathlib.Path] = None, parameters: Optional[dict[str, list[str] | list[int] | list[float]]] = None, ) -> None: + """ + Initialisation of ModelicaSystemDoE. The parameters are based on: ModelicaSystem.__init__() and + ModelicaSystem.simulate(). Additionally, the path to store the result files is needed (= resultpath) as well as + a list of parameters to vary for the Doe (= parameters). All possible combinations are considered. + """ self._lmodel = lmodel self._modelName = modelName self._fileName = fileName @@ -1697,6 +1706,12 @@ def __init__( self._sim_task_query: queue.Queue = queue.Queue() def prepare(self) -> int: + """ + Prepare the DoE by evaluating the parameters. Each structural parameter requires a new instance of + ModelicaSystem while the non-structural parameters can just be set on the executable. + + The return value is the number of simulation defined. + """ param_structure = {} param_simple = {} @@ -1782,9 +1797,17 @@ def prepare(self) -> int: return self._sim_df.shape[0] def get_doe(self) -> Optional[pd.DataFrame]: + """ + Get the defined Doe as a poandas dataframe. + """ return self._sim_df def simulate(self, num_workers: int = 3) -> None: + """ + Simulate the DoE using the defined number of workers. + + Returns True if all simulations were done successfully, else False. + """ sim_count_total = self._sim_task_query.qsize() @@ -1845,6 +1868,15 @@ def get_solutions( self, var_list: Optional[list] = None, ) -> Optional[tuple[str] | dict[str, pd.DataFrame | str]]: + """ + Get all solutions of the DoE run. The following return values are possible: + + * None, if there no simulation was run + + * A list of variables if val_list == None + + * The Solutions as dict[str, pd.DataFrame] if a value list (== val_list) is defined. + """ if self._sim_df is None: return None From f937eb3908492d5065815a40850c9d2bd981e7b5 Mon Sep 17 00:00:00 2001 From: syntron Date: Wed, 25 Jun 2025 21:03:59 +0200 Subject: [PATCH 05/24] [ModelicaSystemDoE] define dict keys as constants --- OMPython/ModelicaSystem.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/OMPython/ModelicaSystem.py b/OMPython/ModelicaSystem.py index c07981b5..6709cac1 100644 --- a/OMPython/ModelicaSystem.py +++ b/OMPython/ModelicaSystem.py @@ -1648,6 +1648,9 @@ class ModelicaSystemDoE: Class to run DoEs based on a (Open)Modelica model using ModelicaSystem """ + DF_COLUMNS_RESULTFILENAME: str = 'resultfilename' + DF_COLUMNS_RESULTS_AVAILABLE: str = 'results available' + def __init__( self, fileName: Optional[str | os.PathLike | pathlib.Path] = None, @@ -1768,7 +1771,7 @@ def prepare(self) -> int: { 'ID structure': idx_pc_structure, 'ID simple': idx_pc_simple, - 'resulfilename': resfilename, + self.DF_COLUMNS_RESULTFILENAME: resfilename, 'structural parameters ID': idx_pc_structure, } | sim_args_structure @@ -1777,7 +1780,7 @@ def prepare(self) -> int: } | sim_args_simple | { - 'results available': False, + self.DF_COLUMNS_RESULTS_AVAILABLE: False, } ) @@ -1854,14 +1857,15 @@ def worker(worker_id, task_queue): thread.join() for idx, row in self._sim_df.to_dict('index').items(): - resultfilename = row['resultfilename'] + resultfilename = row[self.DF_COLUMNS_RESULTFILENAME] resultfile = self._resultpath / resultfilename if resultfile.exists(): - self._sim_df.loc[idx, 'results available'] = True + mask = self._sim_df[self.DF_COLUMNS_RESULTFILENAME] == resultfilename + self._sim_df.loc[mask, self.DF_COLUMNS_RESULTS_AVAILABLE] = True + sim_done = self._sim_df[self.DF_COLUMNS_RESULTS_AVAILABLE].sum() sim_total = self._sim_df.shape[0] - sim_done = self._sim_df['results available'].sum() logger.info(f"All workers finished ({sim_done} of {sim_total} simulations with a result file).") def get_solutions( @@ -1880,17 +1884,17 @@ def get_solutions( if self._sim_df is None: return None - if self._sim_df.shape[0] == 0 or self._sim_df['results available'].sum() == 0: + if self._sim_df.shape[0] == 0 or self._sim_df[self.DF_COLUMNS_RESULTS_AVAILABLE].sum() == 0: raise ModelicaSystemError("No result files available - all simulations did fail?") if var_list is None: - resultfilename = self._sim_df['resultfilename'].values[0] + resultfilename = self._sim_df[self.DF_COLUMNS_RESULTFILENAME].values[0] resultfile = self._resultpath / resultfilename return self._mod.getSolutions(resultfile=resultfile) sol_dict: dict[str, pd.DataFrame | str] = {} for row in self._sim_df.to_dict('records'): - resultfilename = row['resultfilename'] + resultfilename = row[self.DF_COLUMNS_RESULTFILENAME] resultfile = self._resultpath / resultfilename try: From 167f255947b25696b2f59c37d1256125d1c56632 Mon Sep 17 00:00:00 2001 From: syntron Date: Wed, 25 Jun 2025 21:04:34 +0200 Subject: [PATCH 06/24] [ModelicaSystemDoE] build model after all structural parameters are defined --- OMPython/ModelicaSystem.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/OMPython/ModelicaSystem.py b/OMPython/ModelicaSystem.py index 6709cac1..fbd09ea4 100644 --- a/OMPython/ModelicaSystem.py +++ b/OMPython/ModelicaSystem.py @@ -1740,6 +1740,7 @@ def prepare(self) -> int: variableFilter=self._variableFilter, customBuildDirectory=self._customBuildDirectory, omhome=self._omhome, + build=False, ) sim_args_structure = {} @@ -1756,6 +1757,8 @@ def prepare(self) -> int: expression = f"setParameterValue({self._modelName}, {pk_structure}, {pk_value})" mod_structure.sendExpression(expression) + mod_structure.buildModel(variableFilter=self._variableFilter) + for idx_pc_simple, pc_simple in enumerate(param_simple_combinations): sim_args_simple = {} for idx_simple, pk_simple in enumerate(param_simple.keys()): From fb210f0bb1fd904ac298829dd05c2c4f1727f27c Mon Sep 17 00:00:00 2001 From: syntron Date: Wed, 25 Jun 2025 21:05:30 +0200 Subject: [PATCH 07/24] [ModelicaSystemDoE] cleanup prepare() / rename variables --- OMPython/ModelicaSystem.py | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/OMPython/ModelicaSystem.py b/OMPython/ModelicaSystem.py index fbd09ea4..d708914c 100644 --- a/OMPython/ModelicaSystem.py +++ b/OMPython/ModelicaSystem.py @@ -1743,9 +1743,9 @@ def prepare(self) -> int: build=False, ) - sim_args_structure = {} + sim_param_structure = {} for idx_structure, pk_structure in enumerate(param_structure.keys()): - sim_args_structure[pk_structure] = pc_structure[idx_structure] + sim_param_structure[pk_structure] = pc_structure[idx_structure] pk_value = pc_structure[idx_structure] if isinstance(pk_value, str): @@ -1755,19 +1755,22 @@ def prepare(self) -> int: expression = f"setParameterValue({self._modelName}, {pk_structure}, $Code(={pk_value_bool_str}));" else: expression = f"setParameterValue({self._modelName}, {pk_structure}, {pk_value})" - mod_structure.sendExpression(expression) + res = mod_structure.sendExpression(expression) + if not res: + raise ModelicaSystemError(f"Cannot set structural parameter {self._modelName}.{pk_structure} " + f"to {pk_value} using {repr(expression)}") mod_structure.buildModel(variableFilter=self._variableFilter) for idx_pc_simple, pc_simple in enumerate(param_simple_combinations): - sim_args_simple = {} + sim_param_simple = {} for idx_simple, pk_simple in enumerate(param_simple.keys()): - sim_args_simple[pk_simple] = str(pc_simple[idx_simple]) + sim_param_simple[pk_simple] = str(pc_simple[idx_simple]) resfilename = f"DOE_{idx_pc_structure:09d}_{idx_pc_simple:09d}.mat" logger.info(f"use result file {repr(resfilename)} " - f"for structural parameters: {sim_args_structure} " - f"and simple parameters: {sim_args_simple}") + f"for structural parameters: {sim_param_structure} " + f"and simple parameters: {sim_param_simple}") resultfile = self._resultpath / resfilename df_data = ( @@ -1777,24 +1780,27 @@ def prepare(self) -> int: self.DF_COLUMNS_RESULTFILENAME: resfilename, 'structural parameters ID': idx_pc_structure, } - | sim_args_structure + | sim_param_structure | { 'non-structural parameters ID': idx_pc_simple, } - | sim_args_simple + | sim_param_simple | { self.DF_COLUMNS_RESULTS_AVAILABLE: False, } ) - df_entries.append(pd.DataFrame.from_dict(df_data)) + df_entries.append(pd.DataFrame(data=df_data, index=[0])) - cmd = mod_structure.simulate_cmd( + mscmd = mod_structure.simulate_cmd( resultfile=resultfile.absolute().resolve(), - simargs={"override": sim_args_simple}, + timeout=self._timeout, ) + if self._simargs is not None: + mscmd.args_set(args=self._simargs) + mscmd.args_set(args={"override": sim_param_simple}) - self._sim_task_query.put(cmd) + self._sim_task_query.put(mscmd) self._sim_df = pd.concat(df_entries, ignore_index=True) From c503f82eaa6350fbedd3c95e22e236d876a8225e Mon Sep 17 00:00:00 2001 From: syntron Date: Wed, 25 Jun 2025 21:06:26 +0200 Subject: [PATCH 08/24] [ModelicaSystemDoE] cleanup simulate() / rename variables --- OMPython/ModelicaSystem.py | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/OMPython/ModelicaSystem.py b/OMPython/ModelicaSystem.py index d708914c..fffeb25b 100644 --- a/OMPython/ModelicaSystem.py +++ b/OMPython/ModelicaSystem.py @@ -1814,44 +1814,53 @@ def get_doe(self) -> Optional[pd.DataFrame]: """ return self._sim_df - def simulate(self, num_workers: int = 3) -> None: + def simulate( + self, + num_workers: int = 3, + ) -> bool: """ Simulate the DoE using the defined number of workers. Returns True if all simulations were done successfully, else False. """ - sim_count_total = self._sim_task_query.qsize() + sim_query_total = self._sim_task_query.qsize() + if not isinstance(self._sim_df, pd.DataFrame): + raise ModelicaSystemError("Missing Doe Summary!") + sim_df_total = self._sim_df.shape[0] def worker(worker_id, task_queue): while True: - sim_data = {} + mscmd: Optional[ModelicaSystemCmd] = None try: # Get the next task from the queue - cmd: ModelicaSystemCmd = task_queue.get(block=False) + mscmd = task_queue.get(block=False) except queue.Empty: logger.info(f"[Worker {worker_id}] No more simulations to run.") break - resultfile = cmd.arg_get(key='r') + if mscmd is None: + raise ModelicaSystemError("Missing simulation definition!") + + resultfile = mscmd.arg_get(key='r') resultpath = pathlib.Path(resultfile) logger.info(f"[Worker {worker_id}] Performing task: {resultpath.name}") try: - sim_data['sim'].run() + mscmd.run() except ModelicaSystemError as ex: logger.warning(f"Simulation error for {resultpath.name}: {ex}") # Mark the task as done task_queue.task_done() - sim_count_done = sim_count_total - self._sim_task_query.qsize() + sim_query_done = sim_query_total - self._sim_task_query.qsize() logger.info(f"[Worker {worker_id}] Task completed: {resultpath.name} " - f"({sim_count_done}/{sim_count_total} = " - f"{sim_count_done / sim_count_total * 100:.2f}% of tasks left)") + f"({sim_query_done}/{sim_query_total} = " + f"{sim_query_done / sim_query_total * 100:.2f}% of tasks left)") - logger.info(f"Start simulations for DoE with {sim_count_total} simulations " + logger.info(f"Start simulations for DoE with {sim_query_total} simulations " f"using {num_workers} workers ...") # Create and start worker threads From c27f23f6450ff76574952d7eac645a37d857bce4 Mon Sep 17 00:00:00 2001 From: syntron Date: Wed, 25 Jun 2025 21:06:58 +0200 Subject: [PATCH 09/24] [ModelicaSystemDoE] cleanup get_solutions() / rename variables --- OMPython/ModelicaSystem.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/OMPython/ModelicaSystem.py b/OMPython/ModelicaSystem.py index fffeb25b..c8d442b2 100644 --- a/OMPython/ModelicaSystem.py +++ b/OMPython/ModelicaSystem.py @@ -1874,7 +1874,7 @@ def worker(worker_id, task_queue): for thread in threads: thread.join() - for idx, row in self._sim_df.to_dict('index').items(): + for row in self._sim_df.to_dict('records'): resultfilename = row[self.DF_COLUMNS_RESULTFILENAME] resultfile = self._resultpath / resultfilename @@ -1882,9 +1882,10 @@ def worker(worker_id, task_queue): mask = self._sim_df[self.DF_COLUMNS_RESULTFILENAME] == resultfilename self._sim_df.loc[mask, self.DF_COLUMNS_RESULTS_AVAILABLE] = True - sim_done = self._sim_df[self.DF_COLUMNS_RESULTS_AVAILABLE].sum() - sim_total = self._sim_df.shape[0] - logger.info(f"All workers finished ({sim_done} of {sim_total} simulations with a result file).") + sim_df_done = self._sim_df[self.DF_COLUMNS_RESULTS_AVAILABLE].sum() + logger.info(f"All workers finished ({sim_df_done} of {sim_df_total} simulations with a result file).") + + return sim_df_total == sim_df_done def get_solutions( self, From 9ce0a5652ee5a0334fb5be10d438c21dbd9c26cf Mon Sep 17 00:00:00 2001 From: syntron Date: Wed, 25 Jun 2025 21:07:14 +0200 Subject: [PATCH 10/24] [test_ModelicaSystemDoE] update test --- tests/test_ModelicaSystemDoE.py | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/tests/test_ModelicaSystemDoE.py b/tests/test_ModelicaSystemDoE.py index a49ef111..90b36c51 100644 --- a/tests/test_ModelicaSystemDoE.py +++ b/tests/test_ModelicaSystemDoE.py @@ -1,3 +1,4 @@ +import numpy as np import OMPython import pandas as pd import pathlib @@ -37,7 +38,6 @@ def param_doe() -> dict[str, list]: def test_ModelicaSystemDoE(tmp_path, model_doe, param_doe): - tmpdir = tmp_path / 'DoE' tmpdir.mkdir(exist_ok=True) @@ -46,13 +46,32 @@ def test_ModelicaSystemDoE(tmp_path, model_doe, param_doe): modelName="M", parameters=param_doe, resultpath=tmpdir, + simargs={"override": {'stopTime': 1.0}}, ) mod_doe.prepare() df_doe = mod_doe.get_doe() assert isinstance(df_doe, pd.DataFrame) assert df_doe.shape[0] == 16 - assert df_doe['results available'].sum() == 16 + assert df_doe['results available'].sum() == 0 mod_doe.simulate() - sol = mod_doe.get_solutions(var_list=['x[1]', 'y[1]']) - assert len(sol) == 16 + assert df_doe['results available'].sum() == 16 + + for row in df_doe.to_dict('records'): + resultfilename = row[mod_doe.DF_COLUMNS_RESULTFILENAME] + resultfile = mod_doe._resultpath / resultfilename + + var_dict = { + # simple / non-structural parameters + 'a': float(row['a']), + 'b': float(row['b']), + # structural parameters + 'p': float(row['p']), + 'q': float(row['q']), + # variables using the structural parameters + f"x[{row['p']}]": float(row['a']), + f"y[{row['p']}]": float(row['b']), + } + sol = mod_doe._mod.getSolutions(resultfile=resultfile.as_posix(), varList=list(var_dict.keys())) + + assert np.isclose(sol[:, -1], np.array(list(var_dict.values()))).all() From 73b5573e32f08c58457b2658792477b2ccc118bb Mon Sep 17 00:00:00 2001 From: syntron Date: Wed, 25 Jun 2025 21:46:28 +0200 Subject: [PATCH 11/24] [ModelicaSystemDoE] add example to show the usage --- OMPython/ModelicaSystem.py | 56 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/OMPython/ModelicaSystem.py b/OMPython/ModelicaSystem.py index c8d442b2..9f0d339e 100644 --- a/OMPython/ModelicaSystem.py +++ b/OMPython/ModelicaSystem.py @@ -1646,6 +1646,62 @@ def getLinearStates(self) -> list[str]: class ModelicaSystemDoE: """ Class to run DoEs based on a (Open)Modelica model using ModelicaSystem + + Example + ------- + ``` + import OMPython + import pathlib + + + def run_doe(): + mypath = pathlib.Path('.') + + model = mypath / "M.mo" + model.write_text( + " model M\n" + " parameter Integer p=1;\n" + " parameter Integer q=1;\n" + " parameter Real a = -1;\n" + " parameter Real b = -1;\n" + " Real x[p];\n" + " Real y[q];\n" + " equation\n" + " der(x) = a * fill(1.0, p);\n" + " der(y) = b * fill(1.0, q);\n" + " end M;\n" + ) + + param = { + # structural + 'p': [1, 2], + 'q': [3, 4], + # simple + 'a': [5, 6], + 'b': [7, 8], + } + + resdir = mypath / 'DoE' + resdir.mkdir(exist_ok=True) + + mod_doe = OMPython.ModelicaSystemDoE( + fileName=model.as_posix(), + modelName="M", + parameters=param, + resultpath=resdir, + simargs={"override": {'stopTime': 1.0}}, + ) + mod_doe.prepare() + df_doe = mod_doe.get_doe() + mod_doe.simulate() + var_list = mod_doe.get_solutions() + sol_dict = mod_doe.get_solutions(var_list=var_list) + + + if __name__ == "__main__": + run_doe() + ``` + """ DF_COLUMNS_RESULTFILENAME: str = 'resultfilename' From 5b72f11c5937989ab73408cf8e73d9a002f3ac18 Mon Sep 17 00:00:00 2001 From: syntron Date: Wed, 25 Jun 2025 21:49:36 +0200 Subject: [PATCH 12/24] add pandas as new dependency (use in ModelicaSystemDoE) --- .pre-commit-config.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 48f9ac64..d853e371 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -35,5 +35,6 @@ repos: additional_dependencies: - pyparsing - types-psutil + - pandas-stubs - pyzmq - numpy From f5b1c6a0aa1d0b020073c8c620acdda913eb7d37 Mon Sep 17 00:00:00 2001 From: syntron Date: Wed, 25 Jun 2025 22:07:20 +0200 Subject: [PATCH 13/24] [test_ModelicaSystemDoE] fix mypy --- tests/test_ModelicaSystemDoE.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_ModelicaSystemDoE.py b/tests/test_ModelicaSystemDoE.py index 90b36c51..288db522 100644 --- a/tests/test_ModelicaSystemDoE.py +++ b/tests/test_ModelicaSystemDoE.py @@ -4,11 +4,12 @@ import pathlib import pytest + @pytest.fixture def model_doe(tmp_path: pathlib.Path) -> pathlib.Path: # see: https://trac.openmodelica.org/OpenModelica/ticket/4052 mod = tmp_path / "M.mo" - mod.write_text(f""" + mod.write_text(""" model M parameter Integer p=1; parameter Integer q=1; From 6dd63e68244b82204791c6b90ba62b6a4c992756 Mon Sep 17 00:00:00 2001 From: syntron Date: Wed, 25 Jun 2025 22:11:28 +0200 Subject: [PATCH 14/24] add pandas to requirements in pyproject.toml --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 0abafd0c..d529f93b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,6 +17,7 @@ license = "BSD-3-Clause OR LicenseRef-OSMC-PL-1.2 OR GPL-3.0-only" requires-python = ">=3.10" dependencies = [ "numpy", + "pandas", "psutil", "pyparsing", "pyzmq", From 57ff694600f52289d288c2596a02fb56f90a0791 Mon Sep 17 00:00:00 2001 From: syntron Date: Wed, 25 Jun 2025 22:43:07 +0200 Subject: [PATCH 15/24] [ModelicaSystemDoE] rename class constants --- OMPython/ModelicaSystem.py | 22 +++++++++++----------- tests/test_ModelicaSystemDoE.py | 2 +- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/OMPython/ModelicaSystem.py b/OMPython/ModelicaSystem.py index 9f0d339e..6ebe33f4 100644 --- a/OMPython/ModelicaSystem.py +++ b/OMPython/ModelicaSystem.py @@ -1704,8 +1704,8 @@ def run_doe(): """ - DF_COLUMNS_RESULTFILENAME: str = 'resultfilename' - DF_COLUMNS_RESULTS_AVAILABLE: str = 'results available' + DF_COLUMNS_RESULT_FILENAME: str = 'result filename' + DF_COLUMNS_RESULT_AVAILABLE: str = 'result available' def __init__( self, @@ -1833,7 +1833,7 @@ def prepare(self) -> int: { 'ID structure': idx_pc_structure, 'ID simple': idx_pc_simple, - self.DF_COLUMNS_RESULTFILENAME: resfilename, + self.DF_COLUMNS_RESULT_FILENAME: resfilename, 'structural parameters ID': idx_pc_structure, } | sim_param_structure @@ -1842,7 +1842,7 @@ def prepare(self) -> int: } | sim_param_simple | { - self.DF_COLUMNS_RESULTS_AVAILABLE: False, + self.DF_COLUMNS_RESULT_AVAILABLE: False, } ) @@ -1931,14 +1931,14 @@ def worker(worker_id, task_queue): thread.join() for row in self._sim_df.to_dict('records'): - resultfilename = row[self.DF_COLUMNS_RESULTFILENAME] + resultfilename = row[self.DF_COLUMNS_RESULT_FILENAME] resultfile = self._resultpath / resultfilename if resultfile.exists(): - mask = self._sim_df[self.DF_COLUMNS_RESULTFILENAME] == resultfilename - self._sim_df.loc[mask, self.DF_COLUMNS_RESULTS_AVAILABLE] = True + mask = self._sim_df[self.DF_COLUMNS_RESULT_FILENAME] == resultfilename + self._sim_df.loc[mask, self.DF_COLUMNS_RESULT_AVAILABLE] = True - sim_df_done = self._sim_df[self.DF_COLUMNS_RESULTS_AVAILABLE].sum() + sim_df_done = self._sim_df[self.DF_COLUMNS_RESULT_AVAILABLE].sum() logger.info(f"All workers finished ({sim_df_done} of {sim_df_total} simulations with a result file).") return sim_df_total == sim_df_done @@ -1959,17 +1959,17 @@ def get_solutions( if self._sim_df is None: return None - if self._sim_df.shape[0] == 0 or self._sim_df[self.DF_COLUMNS_RESULTS_AVAILABLE].sum() == 0: + if self._sim_df.shape[0] == 0 or self._sim_df[self.DF_COLUMNS_RESULT_AVAILABLE].sum() == 0: raise ModelicaSystemError("No result files available - all simulations did fail?") if var_list is None: - resultfilename = self._sim_df[self.DF_COLUMNS_RESULTFILENAME].values[0] + resultfilename = self._sim_df[self.DF_COLUMNS_RESULT_FILENAME].values[0] resultfile = self._resultpath / resultfilename return self._mod.getSolutions(resultfile=resultfile) sol_dict: dict[str, pd.DataFrame | str] = {} for row in self._sim_df.to_dict('records'): - resultfilename = row[self.DF_COLUMNS_RESULTFILENAME] + resultfilename = row[self.DF_COLUMNS_RESULT_FILENAME] resultfile = self._resultpath / resultfilename try: diff --git a/tests/test_ModelicaSystemDoE.py b/tests/test_ModelicaSystemDoE.py index 288db522..7561dbf1 100644 --- a/tests/test_ModelicaSystemDoE.py +++ b/tests/test_ModelicaSystemDoE.py @@ -59,7 +59,7 @@ def test_ModelicaSystemDoE(tmp_path, model_doe, param_doe): assert df_doe['results available'].sum() == 16 for row in df_doe.to_dict('records'): - resultfilename = row[mod_doe.DF_COLUMNS_RESULTFILENAME] + resultfilename = row[mod_doe.DF_COLUMNS_RESULT_FILENAME] resultfile = mod_doe._resultpath / resultfilename var_dict = { From fbc5f18915ac7a306f60f8985de9debd30d59363 Mon Sep 17 00:00:00 2001 From: syntron Date: Sat, 28 Jun 2025 21:02:01 +0200 Subject: [PATCH 16/24] [ModelicaSystemDoE] remove dependency on pandas * no need to add aditional requirements * hint how to use pandas in the docstrings * update test to match code changes --- .pre-commit-config.yaml | 1 - OMPython/ModelicaSystem.py | 127 ++++++++++++++++++++------------ pyproject.toml | 1 - tests/test_ModelicaSystemDoE.py | 33 +++++---- 4 files changed, 97 insertions(+), 65 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d853e371..48f9ac64 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -35,6 +35,5 @@ repos: additional_dependencies: - pyparsing - types-psutil - - pandas-stubs - pyzmq - numpy diff --git a/OMPython/ModelicaSystem.py b/OMPython/ModelicaSystem.py index 6ebe33f4..dd485780 100644 --- a/OMPython/ModelicaSystem.py +++ b/OMPython/ModelicaSystem.py @@ -40,7 +40,6 @@ import numbers import numpy as np import os -import pandas as pd import pathlib import platform import queue @@ -1684,18 +1683,19 @@ def run_doe(): resdir = mypath / 'DoE' resdir.mkdir(exist_ok=True) - mod_doe = OMPython.ModelicaSystemDoE( + doe_mod = OMPython.ModelicaSystemDoE( fileName=model.as_posix(), modelName="M", parameters=param, resultpath=resdir, simargs={"override": {'stopTime': 1.0}}, ) - mod_doe.prepare() - df_doe = mod_doe.get_doe() - mod_doe.simulate() - var_list = mod_doe.get_solutions() - sol_dict = mod_doe.get_solutions(var_list=var_list) + doe_mod.prepare() + doe_dict = doe_mod.get_doe() + doe_mod.simulate() + doe_sol = doe_mod.get_solutions() + + # ... work with doe_df and doe_sol ... if __name__ == "__main__": @@ -1761,7 +1761,7 @@ def __init__( else: self._parameters = {} - self._sim_df: Optional[pd.DataFrame] = None + self._sim_dict: Optional[dict[str, dict[str, Any]]] = None self._sim_task_query: queue.Queue = queue.Queue() def prepare(self) -> int: @@ -1786,7 +1786,7 @@ def prepare(self) -> int: param_structure_combinations = list(itertools.product(*param_structure.values())) param_simple_combinations = list(itertools.product(*param_simple.values())) - df_entries: list[pd.DataFrame] = [] + self._sim_dict = {} for idx_pc_structure, pc_structure in enumerate(param_structure_combinations): mod_structure = ModelicaSystem( fileName=self._fileName, @@ -1832,13 +1832,10 @@ def prepare(self) -> int: df_data = ( { 'ID structure': idx_pc_structure, - 'ID simple': idx_pc_simple, - self.DF_COLUMNS_RESULT_FILENAME: resfilename, - 'structural parameters ID': idx_pc_structure, } | sim_param_structure | { - 'non-structural parameters ID': idx_pc_simple, + 'ID non-structure': idx_pc_simple, } | sim_param_simple | { @@ -1846,7 +1843,7 @@ def prepare(self) -> int: } ) - df_entries.append(pd.DataFrame(data=df_data, index=[0])) + self._sim_dict[resfilename] = df_data mscmd = mod_structure.simulate_cmd( resultfile=resultfile.absolute().resolve(), @@ -1858,17 +1855,26 @@ def prepare(self) -> int: self._sim_task_query.put(mscmd) - self._sim_df = pd.concat(df_entries, ignore_index=True) - - logger.info(f"Prepared {self._sim_df.shape[0]} simulation definitions for the defined DoE.") + logger.info(f"Prepared {self._sim_task_query.qsize()} simulation definitions for the defined DoE.") - return self._sim_df.shape[0] + return self._sim_task_query.qsize() - def get_doe(self) -> Optional[pd.DataFrame]: + def get_doe(self) -> Optional[dict[str, dict[str, Any]]]: """ - Get the defined Doe as a poandas dataframe. + Get the defined DoE as a dict, where each key is the result filename and the value is a dict of simulation + settings including structural and non-structural parameters. + + The following code snippet can be used to convert the data to a pandas dataframe: + + ``` + import pandas as pd + + doe_dict = doe_mod.get_doe() + doe_df = pd.DataFrame.from_dict(data=doe_dict, orient='index') + ``` + """ - return self._sim_df + return self._sim_dict def simulate( self, @@ -1881,9 +1887,9 @@ def simulate( """ sim_query_total = self._sim_task_query.qsize() - if not isinstance(self._sim_df, pd.DataFrame): + if not isinstance(self._sim_dict, dict) or len(self._sim_dict) == 0: raise ModelicaSystemError("Missing Doe Summary!") - sim_df_total = self._sim_df.shape[0] + sim_dict_total = len(self._sim_dict) def worker(worker_id, task_queue): while True: @@ -1930,55 +1936,78 @@ def worker(worker_id, task_queue): for thread in threads: thread.join() - for row in self._sim_df.to_dict('records'): - resultfilename = row[self.DF_COLUMNS_RESULT_FILENAME] + sim_dict_done = 0 + for resultfilename in self._sim_dict: resultfile = self._resultpath / resultfilename - if resultfile.exists(): - mask = self._sim_df[self.DF_COLUMNS_RESULT_FILENAME] == resultfilename - self._sim_df.loc[mask, self.DF_COLUMNS_RESULT_AVAILABLE] = True + # include check for an empty (=> 0B) result file which indicates a crash of the model executable + # see: https://github.com/OpenModelica/OMPython/issues/261 + # https://github.com/OpenModelica/OpenModelica/issues/13829 + if resultfile.is_file() and resultfile.stat().st_size > 0: + self._sim_dict[resultfilename][self.DF_COLUMNS_RESULTS_AVAILABLE] = True + sim_dict_done += 1 - sim_df_done = self._sim_df[self.DF_COLUMNS_RESULT_AVAILABLE].sum() - logger.info(f"All workers finished ({sim_df_done} of {sim_df_total} simulations with a result file).") + logger.info(f"All workers finished ({sim_dict_done} of {sim_dict_total} simulations with a result file).") - return sim_df_total == sim_df_done + return sim_dict_total == sim_dict_done def get_solutions( self, var_list: Optional[list] = None, - ) -> Optional[tuple[str] | dict[str, pd.DataFrame | str]]: + ) -> Optional[tuple[str] | dict[str, dict[str, np.ndarray]]]: """ Get all solutions of the DoE run. The following return values are possible: - * None, if there no simulation was run - * A list of variables if val_list == None * The Solutions as dict[str, pd.DataFrame] if a value list (== val_list) is defined. + + The following code snippet can be used to convert the solution data for each run to a pandas dataframe: + + ``` + import pandas as pd + + doe_sol = doe_mod.get_solutions() + for key in doe_sol: + data = doe_sol[key]['data'] + if data: + doe_sol[key]['df'] = pd.DataFrame.from_dict(data=data) + else: + doe_sol[key]['df'] = None + ``` + """ - if self._sim_df is None: + if not isinstance(self._sim_dict, dict): return None - if self._sim_df.shape[0] == 0 or self._sim_df[self.DF_COLUMNS_RESULT_AVAILABLE].sum() == 0: + if len(self._sim_dict) == 0: raise ModelicaSystemError("No result files available - all simulations did fail?") - if var_list is None: - resultfilename = self._sim_df[self.DF_COLUMNS_RESULT_FILENAME].values[0] + sol_dict: dict[str, dict[str, Any]] = {} + for resultfilename in self._sim_dict: resultfile = self._resultpath / resultfilename - return self._mod.getSolutions(resultfile=resultfile) - sol_dict: dict[str, pd.DataFrame | str] = {} - for row in self._sim_df.to_dict('records'): - resultfilename = row[self.DF_COLUMNS_RESULT_FILENAME] - resultfile = self._resultpath / resultfilename + sol_dict[resultfilename] = {} + + if self._sim_dict[resultfilename][self.DF_COLUMNS_RESULTS_AVAILABLE] != True: + sol_dict[resultfilename]['msg'] = 'No result file available!' + sol_dict[resultfilename]['data'] = {} + continue + + if var_list is None: + var_list_row = list(self._mod.getSolutions(resultfile=resultfile)) + else: + var_list_row = var_list try: - sol = self._mod.getSolutions(varList=var_list, resultfile=resultfile) - sol_data = {var: sol[idx] for idx, var in var_list} - sol_df = pd.DataFrame(sol_data) - sol_dict[resultfilename] = sol_df + sol = self._mod.getSolutions(varList=var_list_row, resultfile=resultfile) + sol_data = {var: sol[idx] for idx, var in enumerate(var_list_row)} + sol_dict[resultfilename]['msg'] = 'Simulation available' + sol_dict[resultfilename]['data'] = sol_data except ModelicaSystemError as ex: - logger.warning(f"No solution for {resultfilename}: {ex}") - sol_dict[resultfilename] = str(ex) + msg = f"Error reading solution for {resultfilename}: {ex}" + logger.warning(msg) + sol_dict[resultfilename]['msg'] = msg + sol_dict[resultfilename]['data'] = {} return sol_dict diff --git a/pyproject.toml b/pyproject.toml index d529f93b..0abafd0c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,7 +17,6 @@ license = "BSD-3-Clause OR LicenseRef-OSMC-PL-1.2 OR GPL-3.0-only" requires-python = ">=3.10" dependencies = [ "numpy", - "pandas", "psutil", "pyparsing", "pyzmq", diff --git a/tests/test_ModelicaSystemDoE.py b/tests/test_ModelicaSystemDoE.py index 7561dbf1..40fed90d 100644 --- a/tests/test_ModelicaSystemDoE.py +++ b/tests/test_ModelicaSystemDoE.py @@ -1,6 +1,5 @@ import numpy as np import OMPython -import pandas as pd import pathlib import pytest @@ -42,25 +41,30 @@ def test_ModelicaSystemDoE(tmp_path, model_doe, param_doe): tmpdir = tmp_path / 'DoE' tmpdir.mkdir(exist_ok=True) - mod_doe = OMPython.ModelicaSystemDoE( + doe_mod = OMPython.ModelicaSystemDoE( fileName=model_doe.as_posix(), modelName="M", parameters=param_doe, resultpath=tmpdir, simargs={"override": {'stopTime': 1.0}}, ) - mod_doe.prepare() - df_doe = mod_doe.get_doe() - assert isinstance(df_doe, pd.DataFrame) - assert df_doe.shape[0] == 16 - assert df_doe['results available'].sum() == 0 + doe_count = doe_mod.prepare() + assert doe_count == 16 - mod_doe.simulate() - assert df_doe['results available'].sum() == 16 + doe_dict = doe_mod.get_doe() + assert isinstance(doe_dict, dict) + assert len(doe_dict.keys()) == 16 - for row in df_doe.to_dict('records'): - resultfilename = row[mod_doe.DF_COLUMNS_RESULT_FILENAME] - resultfile = mod_doe._resultpath / resultfilename + doe_status = doe_mod.simulate() + assert doe_status is True + + doe_sol = doe_mod.get_solutions() + + for resultfilename in doe_dict: + row = doe_dict[resultfilename] + + assert resultfilename in doe_sol + sol = doe_sol[resultfilename] var_dict = { # simple / non-structural parameters @@ -73,6 +77,7 @@ def test_ModelicaSystemDoE(tmp_path, model_doe, param_doe): f"x[{row['p']}]": float(row['a']), f"y[{row['p']}]": float(row['b']), } - sol = mod_doe._mod.getSolutions(resultfile=resultfile.as_posix(), varList=list(var_dict.keys())) - assert np.isclose(sol[:, -1], np.array(list(var_dict.values()))).all() + for var in var_dict: + assert var in sol['data'] + assert np.isclose(sol['data'][var][-1], var_dict[var]) From c30639bb287b56085935d57fd4a769649e319437 Mon Sep 17 00:00:00 2001 From: syntron Date: Sat, 28 Jun 2025 21:05:04 +0200 Subject: [PATCH 17/24] [ModelicaSystemDoE.simulate] fix percent of tasks left --- OMPython/ModelicaSystem.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/OMPython/ModelicaSystem.py b/OMPython/ModelicaSystem.py index dd485780..1641cec9 100644 --- a/OMPython/ModelicaSystem.py +++ b/OMPython/ModelicaSystem.py @@ -1919,8 +1919,8 @@ def worker(worker_id, task_queue): sim_query_done = sim_query_total - self._sim_task_query.qsize() logger.info(f"[Worker {worker_id}] Task completed: {resultpath.name} " - f"({sim_query_done}/{sim_query_total} = " - f"{sim_query_done / sim_query_total * 100:.2f}% of tasks left)") + f"({sim_query_total - sim_query_done}/{sim_query_total} = " + f"{(sim_query_total - sim_query_done) / sim_query_total * 100:.2f}% of tasks left)") logger.info(f"Start simulations for DoE with {sim_query_total} simulations " f"using {num_workers} workers ...") From ec8043623aefb281523733acaae593058b710d80 Mon Sep 17 00:00:00 2001 From: syntron Date: Sat, 28 Jun 2025 21:05:34 +0200 Subject: [PATCH 18/24] [ModelicaSystemDoE.prepare] do not convert all non-structural parameters to string --- OMPython/ModelicaSystem.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/OMPython/ModelicaSystem.py b/OMPython/ModelicaSystem.py index 1641cec9..e62b44a2 100644 --- a/OMPython/ModelicaSystem.py +++ b/OMPython/ModelicaSystem.py @@ -1821,7 +1821,7 @@ def prepare(self) -> int: for idx_pc_simple, pc_simple in enumerate(param_simple_combinations): sim_param_simple = {} for idx_simple, pk_simple in enumerate(param_simple.keys()): - sim_param_simple[pk_simple] = str(pc_simple[idx_simple]) + sim_param_simple[pk_simple] = pc_simple[idx_simple] resfilename = f"DOE_{idx_pc_structure:09d}_{idx_pc_simple:09d}.mat" logger.info(f"use result file {repr(resfilename)} " From 11ffb6922eae2a527566403fa12cc6b51a7fe877 Mon Sep 17 00:00:00 2001 From: syntron Date: Sat, 28 Jun 2025 21:05:59 +0200 Subject: [PATCH 19/24] [ModelicaSystemDoE] update set parameter expressions for str and bool --- OMPython/ModelicaSystem.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/OMPython/ModelicaSystem.py b/OMPython/ModelicaSystem.py index e62b44a2..003ae291 100644 --- a/OMPython/ModelicaSystem.py +++ b/OMPython/ModelicaSystem.py @@ -1805,10 +1805,10 @@ def prepare(self) -> int: pk_value = pc_structure[idx_structure] if isinstance(pk_value, str): - expression = f"setParameterValue({self._modelName}, {pk_structure}, $Code(=\"{pk_value}\"))" + expression = f"setParameterValue({self._modelName}, {pk_structure}, \"{pk_value}\")" elif isinstance(pk_value, bool): pk_value_bool_str = "true" if pk_value else "false" - expression = f"setParameterValue({self._modelName}, {pk_structure}, $Code(={pk_value_bool_str}));" + expression = f"setParameterValue({self._modelName}, {pk_structure}, {pk_value_bool_str});" else: expression = f"setParameterValue({self._modelName}, {pk_structure}, {pk_value})" res = mod_structure.sendExpression(expression) From aeb0b4c4ca1dc100b2b1ceabb999d2ff671e7f2b Mon Sep 17 00:00:00 2001 From: syntron Date: Sat, 28 Jun 2025 21:10:17 +0200 Subject: [PATCH 20/24] [ModelicaSystemDoE] rename class constants --- OMPython/ModelicaSystem.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/OMPython/ModelicaSystem.py b/OMPython/ModelicaSystem.py index 003ae291..5f2ad35f 100644 --- a/OMPython/ModelicaSystem.py +++ b/OMPython/ModelicaSystem.py @@ -1704,8 +1704,8 @@ def run_doe(): """ - DF_COLUMNS_RESULT_FILENAME: str = 'result filename' - DF_COLUMNS_RESULT_AVAILABLE: str = 'result available' + DICT_RESULT_FILENAME: str = 'result filename' + DICT_RESULT_AVAILABLE: str = 'result available' def __init__( self, @@ -1839,7 +1839,7 @@ def prepare(self) -> int: } | sim_param_simple | { - self.DF_COLUMNS_RESULT_AVAILABLE: False, + self.DICT_RESULT_AVAILABLE: False, } ) @@ -1944,7 +1944,7 @@ def worker(worker_id, task_queue): # see: https://github.com/OpenModelica/OMPython/issues/261 # https://github.com/OpenModelica/OpenModelica/issues/13829 if resultfile.is_file() and resultfile.stat().st_size > 0: - self._sim_dict[resultfilename][self.DF_COLUMNS_RESULTS_AVAILABLE] = True + self._sim_dict[resultfilename][self.DICT_RESULT_AVAILABLE] = True sim_dict_done += 1 logger.info(f"All workers finished ({sim_dict_done} of {sim_dict_total} simulations with a result file).") @@ -1989,7 +1989,7 @@ def get_solutions( sol_dict[resultfilename] = {} - if self._sim_dict[resultfilename][self.DF_COLUMNS_RESULTS_AVAILABLE] != True: + if self._sim_dict[resultfilename][self.DICT_RESULT_AVAILABLE] != True: sol_dict[resultfilename]['msg'] = 'No result file available!' sol_dict[resultfilename]['data'] = {} continue From bb746f7a19651ce1ba05ce53d3b31d77d034bde6 Mon Sep 17 00:00:00 2001 From: syntron Date: Sat, 28 Jun 2025 21:12:10 +0200 Subject: [PATCH 21/24] [ModelicaSystemDoE] fix bool comparison --- OMPython/ModelicaSystem.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/OMPython/ModelicaSystem.py b/OMPython/ModelicaSystem.py index 5f2ad35f..dfc5c804 100644 --- a/OMPython/ModelicaSystem.py +++ b/OMPython/ModelicaSystem.py @@ -1989,7 +1989,7 @@ def get_solutions( sol_dict[resultfilename] = {} - if self._sim_dict[resultfilename][self.DICT_RESULT_AVAILABLE] != True: + if not self._sim_dict[resultfilename][self.DICT_RESULT_AVAILABLE]: sol_dict[resultfilename]['msg'] = 'No result file available!' sol_dict[resultfilename]['data'] = {} continue From 10b540656ff5924d08abe156b57c95318022f1fb Mon Sep 17 00:00:00 2001 From: syntron Date: Sat, 28 Jun 2025 21:13:17 +0200 Subject: [PATCH 22/24] [ModelicaSystemDoE] remove unused code --- OMPython/ModelicaSystem.py | 1 - 1 file changed, 1 deletion(-) diff --git a/OMPython/ModelicaSystem.py b/OMPython/ModelicaSystem.py index dfc5c804..946798ec 100644 --- a/OMPython/ModelicaSystem.py +++ b/OMPython/ModelicaSystem.py @@ -1893,7 +1893,6 @@ def simulate( def worker(worker_id, task_queue): while True: - mscmd: Optional[ModelicaSystemCmd] = None try: # Get the next task from the queue mscmd = task_queue.get(block=False) From 36202ace28414b6a31506a680a57a649ecef7ca7 Mon Sep 17 00:00:00 2001 From: syntron Date: Wed, 2 Jul 2025 22:40:59 +0200 Subject: [PATCH 23/24] [ModelicaSystemCmd] fix mypy warning in ModelicaSystemDoE on usage of args_set() --- OMPython/ModelicaSystem.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/OMPython/ModelicaSystem.py b/OMPython/ModelicaSystem.py index 946798ec..8ceb6ebc 100644 --- a/OMPython/ModelicaSystem.py +++ b/OMPython/ModelicaSystem.py @@ -125,7 +125,7 @@ def __init__(self, runpath: pathlib.Path, modelname: str, timeout: Optional[floa self._args: dict[str, str | None] = {} self._arg_override: dict[str, str] = {} - def arg_set(self, key: str, val: Optional[str | dict] = None) -> None: + def arg_set(self, key: str, val: Optional[str | dict[str, Any]] = None) -> None: """ Set one argument for the executable model. @@ -168,7 +168,7 @@ def arg_get(self, key: str) -> Optional[str | dict]: return None - def args_set(self, args: dict[str, Optional[str | dict[str, str]]]) -> None: + def args_set(self, args: dict[str, Optional[str | dict[str, Any]]]) -> None: """ Define arguments for the model executable. From e351fb6240932ca52497b605fdd701d51b5eef7d Mon Sep 17 00:00:00 2001 From: syntron Date: Wed, 9 Jul 2025 21:26:57 +0200 Subject: [PATCH 24/24] [ModelicaSystemDoE] fix rebase fallout --- OMPython/ModelicaSystem.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/OMPython/ModelicaSystem.py b/OMPython/ModelicaSystem.py index 8ceb6ebc..b47bd468 100644 --- a/OMPython/ModelicaSystem.py +++ b/OMPython/ModelicaSystem.py @@ -1846,7 +1846,7 @@ def prepare(self) -> int: self._sim_dict[resfilename] = df_data mscmd = mod_structure.simulate_cmd( - resultfile=resultfile.absolute().resolve(), + result_file=resultfile.absolute().resolve(), timeout=self._timeout, ) if self._simargs is not None: @@ -1994,12 +1994,12 @@ def get_solutions( continue if var_list is None: - var_list_row = list(self._mod.getSolutions(resultfile=resultfile)) + var_list_row = list(self._mod.getSolutions(resultfile=resultfile.as_posix())) else: var_list_row = var_list try: - sol = self._mod.getSolutions(varList=var_list_row, resultfile=resultfile) + sol = self._mod.getSolutions(varList=var_list_row, resultfile=resultfile.as_posix()) sol_data = {var: sol[idx] for idx, var in enumerate(var_list_row)} sol_dict[resultfilename]['msg'] = 'Simulation available' sol_dict[resultfilename]['data'] = sol_data