diff --git a/core/ems.py b/core/ems.py new file mode 100644 index 0000000..4c9d419 --- /dev/null +++ b/core/ems.py @@ -0,0 +1,458 @@ +"""EMS: The fluid2d Experiment Management System + +The Experiment Management System provides a convenient way to handle big +sets of fluid2d-experiments. It creates a database in which information +about all the performed experiments is stored. This allows to easily +show, compare, search, filter, organise, analyse them, and a lot more. +Furthermore, the EMS offers a simpler way to modify the parameters of +interest in an experiment, including the automation of running an +experiment several times with different parameter values. To prevent +overwriting files, the EMS automatically assigns a unique identifier for +every new run of an experiment. However, if fluid2d runs on multiple +cores, an initial ID has to be specified manually. + +The usage of the EMS is explained in detail in the experiment on waves, +located in the folder `experiments/Waves with EMS` of fluid2d. + +With the EMShell, a command line interface is provided to access the +experiment-database created by the EMS. The EMShell is run via the file +`Experiment-Manager.py` in the folder `ems` of fluid2d. + +Author: Markus Reinert, May/June 2019 +""" + +import os +import datetime +import sqlite3 +from collections import namedtuple +from itertools import product + + +# Columns with the following names are automatically added to every table of the database. +# Therefore they must not be used as parameter names. +RESERVED_COLUMN_NAMES = [ + 'id', + 'datetime', + 'duration', + 'size_total', + 'size_mp4', + 'size_his', + 'size_diag', + 'size_flux', + 'comment', +] + + +class InputMismatch(Exception): + """Incompatibility between experiment file and corresponding table in the database.""" + pass + + +class ExpFileError(Exception): + """Malformed experiment file.""" + pass + + +class DatabaseError(Exception): + """Malformed table in the experiment database.""" + pass + + +class NotInitializedError(Exception): + """Call to `EMS.initialize` required before this action.""" + pass + + +class EMS: + """Experiment Management System""" + + def __init__(self, experiment_file: str): + """Load the parameters from the experiment file.""" + self.connection = None + self.cursor = None + self.exp_class, self.exp_id, self.description, self.param_name_list, param_values_list = parse_experiment_file(experiment_file) + self.param_combinations = product(*param_values_list) + self.setup_next_parameters(increase_id=False) + + def __del__(self): + """Close the connection to the experiment database if any.""" + if self.connection: + print(" Closing database.") + print("-"*50) + self.connection.close() + + def get_expname(self): + if self.exp_id is None: + raise NotInitializedError("ID not set.") + return "{}_{:03}".format(self.exp_class, self.exp_id) + + def setup_next_parameters(self, increase_id=True): + try: + self.parameters = { + name: val for name, val in + zip(self.param_name_list, next(self.param_combinations)) + } + except StopIteration: + self.parameters = None + if increase_id: + self.exp_id += 1 + + def initialize(self, data_directory: str): + if not self.connection: + self.connect(data_directory) + # All of the names in the next two lists should be in RESERVED_COLUMN_NAMES + DBColumn = namedtuple("DBColumn", ["sql_type", "name", "value"]) + static_columns_start = [ + DBColumn("INTEGER", "id", -1), + DBColumn("TEXT", "datetime", datetime.datetime.now().isoformat()), + ] + static_columns_end = [ + DBColumn("REAL", "duration", -1), + DBColumn("REAL", "size_total", -1), + DBColumn("REAL", "size_mp4", -1), + DBColumn("REAL", "size_his", -1), + DBColumn("REAL", "size_diag", -1), + DBColumn("REAL", "size_flux", -1), + DBColumn("TEXT", "comment", self.description), + ] + new_columns = ( + static_columns_start + + [DBColumn(sql_type(self.parameters[name]), name, self.parameters[name]) + for name in self.param_name_list] + + static_columns_end + ) + # If the table exists, check its columns, otherwise create it + if self.table_exists(self.exp_class): + self.cursor.execute('PRAGMA table_info("{}")'.format(self.exp_class)) + columns = self.cursor.fetchall() + # Check static columns + for st_col in static_columns_start: + column = columns.pop(0) + col_index = column[0] + col_name = column[1] + col_type = column[2] + if col_name != st_col.name or col_type != st_col.sql_type: + raise DatabaseError( + "expected column {} in the database to be {} {} but is {} {}." + .format(col_index, st_col.sql_type, st_col.name, col_type, col_name) + ) + for st_col in reversed(static_columns_end): + column = columns.pop() + col_index = column[0] + col_name = column[1] + col_type = column[2] + if col_name != st_col.name or col_type != st_col.sql_type: + raise DatabaseError( + "expected column {} in the database to be {} {} but is {} {}." + .format(col_index, st_col.sql_type, st_col.name, col_type, col_name) + ) + # Check user-defined columns + # TODO: be more flexible here: + # - allow to skip columns which are no longer needed + # - allow to add new columns if needed + # - allow to convert types + for name in self.param_name_list: + column = columns.pop(0) + col_index = column[0] + col_name = column[1] + col_type = column[2] + type_ = sql_type(self.parameters[name]) + if col_name != name or col_type != type_: + raise InputMismatch( + "parameter {} of type {} does not fit into column {} " + "with name {} and type {}." + .format(name, type_, col_index, col_name, col_type) + ) + else: + print(' Creating new table "{}".'.format(self.exp_class)) + column_string = ", ".join( + ['"{}" {}'.format(col.name, col.sql_type) for col in new_columns] + ) + self.cursor.execute( + 'CREATE TABLE "{}" ({})'.format(self.exp_class, column_string) + ) + # Set the experiment ID if it was not defined in the experiment file + if self.exp_id is None: + new_entry = True + # Get the highest index of the table or start with ID 1 if table is empty + self.cursor.execute( + 'SELECT id from "{}" ORDER BY id DESC'.format(self.exp_class) + ) + highest_entry = self.cursor.fetchone() + self.exp_id = highest_entry[0] + 1 if highest_entry else 1 + else: + # If no entry with this ID exists, create a new one + self.cursor.execute( + 'SELECT id from "{}" WHERE id = ?'.format(self.exp_class), + [self.exp_id], + ) + new_entry = self.cursor.fetchone() is None + new_columns[0] = DBColumn("INTEGER", "id", self.exp_id) + if new_entry: + print(' Adding new entry #{} to table "{}".'.format(self.exp_id, self.exp_class)) + # Use the question mark as a placeholder to profit from string formatting of sqlite + value_string = ', '.join(['?'] * len(new_columns)) + self.cursor.execute( + 'INSERT INTO "{}" VALUES ({})'.format(self.exp_class, value_string), + [sql_value(col.value) for col in new_columns], + ) + else: + print(' Overwriting entry #{} of table "{}".'.format(self.exp_id, self.exp_class)) + # Use the question mark as a placeholder to profit from string formatting of sqlite + column_name_string = ", ".join('"{}" = ?'.format(col.name) for col in new_columns) + self.cursor.execute( + 'UPDATE "{}" SET {} WHERE id = ?'.format(self.exp_class, column_name_string), + [sql_value(col.value) for col in new_columns] + [self.exp_id], + ) + # Save the database + self.connection.commit() + + def finalize(self, fluid2d): + """Save information about the completed run in the database. + + This method must be called when the simulation is finished, + that means, after the line `f2d.loop()`. + It writes the integration time and the sizes of the created + output files into the database. If a blow-up was detected, this + is stored in the comment-field. Furthermore, the datetime-field + of the database entry is set to the current time. + """ + if not self.connection: + return + # Divide every size by 1000*1000 = 1e6 to get the value in MB + # Total size + output_dir = os.path.dirname(fluid2d.output.hisfile) + try: + output_files = [os.path.join(output_dir, f) for f in os.listdir(output_dir)] + total_size = sum(os.path.getsize(path) for path in output_files) / 1e6 + except FileNotFoundError as e: + print(" Error getting total size of output:", e) + total_size = -1 + # History file + try: + his_size = os.path.getsize(fluid2d.output.hisfile) / 1e6 + except FileNotFoundError as e: + print(" Error getting size of his-file:", e) + his_size = -1 + # Diagnostics file + try: + diag_size = os.path.getsize(fluid2d.output.diagfile) / 1e6 + except FileNotFoundError: + print(" Error getting size of diag-file:", e) + diag_size = -1 + # MP4 file + if fluid2d.plot_interactive and hasattr(fluid2d.plotting, 'mp4file'): + try: + mp4_size = os.path.getsize(fluid2d.plotting.mp4file) / 1e6 + except FileNotFoundError: + print(" Error getting size of mp4-file:", e) + mp4_size = -1 + else: + mp4_size = 0 + # Flux file + if fluid2d.diag_fluxes: + try: + flux_size = os.path.getsize(fluid2d.output.flxfile) / 1e6 + except FileNotFoundError: + print(" Error getting size of flux-file:", e) + flux_size = -1 + else: + flux_size = 0 + # Check for blow-up + if hasattr(fluid2d, "blow_up") and fluid2d.blow_up: + comment = "Blow-up! " + self.description + else: + comment = self.description + # Update and save the database + self.cursor.execute( + """UPDATE "{}" SET + duration = ?, + datetime = ?, + size_total = ?, + size_mp4 = ?, + size_his = ?, + size_diag = ?, + size_flux = ?, + comment = ? + WHERE id = ?""".format(self.exp_class), + ( + round(fluid2d.t, 2), + datetime.datetime.now().isoformat(), + round(total_size, 3), + round(mp4_size, 3), + round(his_size, 3), + round(diag_size, 3), + round(flux_size, 3), + comment, + self.exp_id, + ), + ) + self.connection.commit() + + def connect(self, data_dir: str): + print("-"*50) + if data_dir.startswith("~"): + data_dir = os.path.expanduser(data_dir) + if not os.path.isdir(data_dir): + print(" Creating directory {}.".format(data_dir)) + os.makedirs(data_dir) + dbpath = os.path.join(data_dir, "experiments.db") + print(" Opening database {}.".format(dbpath)) + self.connection = sqlite3.connect(dbpath) + self.cursor = self.connection.cursor() + + def table_exists(self, name: str): + """Check if table with given name exists in the connected database.""" + # Get all tables + self.cursor.execute("SELECT name FROM sqlite_master WHERE type='table';") + # This gives a list like that: [('table1',), ('table2',)] + for table in self.cursor.fetchall(): + if table[0] == name: + return True + return False + + +def parse_experiment_file(path: str): + """Parse the given experiment file. + + An experiment file + - must provide a name, + - can provide an ID, + - can provide a multi-line description, + - can provide parameters with one or several values each. + + The name is written in a line starting with "Name:". It must be a + valid string to be used as a filename; in particular, it must not + contain a slash (/) and it must not contain a quotation mark ("). + It is advised to use underscores instead of spaces in the name. + + The description begins in or after a line starting with + "Description:" and goes until the beginning of the parameters. + If no parameters are defined below the description, it goes until + the end of the file. + + The parameters follow after a line starting with "Parameters:". + Every parameter is written in its own line. This line begins with + the name of the parameter followed by one value or several values, + separated by one or several spaces or tabs. The name must not + contain any whitespace characters (otherwise it is not recognised as + one name) and must not contain a quotation mark. The name must not + be in the list of reserved column names. The datatype of the value + must be unambiguous, i.e., the values True and False are interpreted + as Boolean variables, numbers like 3 are treated as integers, + numbers like 3.14 or 1e3 (=1000) are treated as floats and all other + values are taken as strings literally. If a string contains a + whitespace, this is considered as multiple values for one parameter + and not as a string of multiple words. + + Everything after a #-symbol is considered a comment and ignored. + + TODO: + - use of quotation marks for multi-word strings in values + """ + with open(path) as f: + experiment_lines = f.readlines() + name = "" + id_ = None + description_lines = [] + param_name_list = [] + param_values_list = [] + reading_params = False + reading_description = False + for line in experiment_lines: + # Remove comments as well as leading and trailing whitespace + line = line.split("#")[0].strip() + if not line: + # Skip empty lines except in the description + if reading_description: + description_lines.append(line) + else: + continue + elif line.lower().startswith("name:"): + if not name: + name = line[5:].strip() + if '"' in name: + raise ExpFileError("Name must not contain a quotation mark.") + else: + raise ExpFileError("Name defined more than once.") + elif line.lower().startswith("id:"): + if id_ is None: + value = line[3:].strip() + if value != "": + try: + id_ = int(value) + except ValueError: + raise ExpFileError("ID is not an integer.") + else: + raise ExpFileError("ID defined more than once.") + elif line.lower().startswith("description:"): + reading_description = True + reading_params = False + description_lines.append(line[12:].lstrip()) + elif line.lower().startswith("parameters:"): + reading_description = False + reading_params = True + elif reading_description: + description_lines.append(line) + elif reading_params: + # Parse the parameter name + param_name = line.split()[0] + if param_name in RESERVED_COLUMN_NAMES: + raise ExpFileError( + "reserved name {} must not be used as a parameter.".format(param_name) + ) + if '"' in param_name: + raise ExpFileError("parameter name must not contain a quotation mark.") + # Parse the value(s) of the parameter + param_val_text = line[len(param_name):].lstrip() + if not param_val_text: + raise ExpFileError("no value given for parameter {}.".format(param_name)) + # TODO: allow multi-word strings containing quotation marks + param_values = [cast_string(val) for val in param_val_text.split()] + param_name_list.append(param_name) + param_values_list.append(param_values) + else: + raise ExpFileError("unexpected line:\n{!r}".format(line)) + if not name: + raise ExpFileError("Name must be defined in every experiment file.") + return name, id_, "\n".join(description_lines).strip(), param_name_list, param_values_list + + +def sql_type(value): + if isinstance(value, bool): + return "TEXT" + elif isinstance(value, int): + return "INTEGER" + elif isinstance(value, float): + return "REAL" + return "TEXT" + + +def sql_value(value): + """Cast into a type suitable for writing the value in a databse. + + This returns for Boolean variables a string representing the given + value and returns the value unchanged otherwise. + """ + if isinstance(value, bool): + return str(value) + return value + + +def cast_string(value: str): + """Cast into the most specialised Python type possible. + + This method can recognize "True", "False", integers and floats, + everything else is treated as a string and returned unchanged. + """ + if value == "True": + return True + if value == "False": + return False + try: + return int(value) + except ValueError: + try: + return float(value) + except ValueError: + return value diff --git a/core/fluid2d.py b/core/fluid2d.py index 6c1d3c7..39bea63 100644 --- a/core/fluid2d.py +++ b/core/fluid2d.py @@ -296,6 +296,7 @@ def signal_handler(signal, frame): # check for blow-up if self.model.diags['maxspeed'] > 1e3: self.stop = True + self.blow_up = True if self.myrank == 0: print() print('max|u| > 1000, blow-up detected, stopping') diff --git a/core/output.py b/core/output.py index d75f285..43c7fa3 100644 --- a/core/output.py +++ b/core/output.py @@ -26,7 +26,6 @@ def __init__(self, param, grid, diag, flxlist=None): self.template = self.expdir+'/%s_his_%03i.nc' if self.nbproc > 1: self.hisfile = self.template % (self.expname, self.myrank) - self.hisfile_joined = '%s/%s_his.nc' % (self.expdir, self.expname) else: self.hisfile = '%s/%s_his.nc' % (self.expdir, self.expname) @@ -43,7 +42,6 @@ def __init__(self, param, grid, diag, flxlist=None): template = self.expdir+'/%s_flx_%03i.nc' if self.nbproc > 1: self.flxfile = template % (self.expname, self.myrank) - self.flxfile_joined = '%s/%s_flx.nc' % (self.expdir, self.expname) else: self.flxfile = '%s/%s_flx.nc' % (self.expdir, self.expname) @@ -154,13 +152,13 @@ def dump_diag(self): def join(self): if self.nbproc > 1: filename = self.hisfile.split('his')[0]+'his' - join(filename) + self.hisfile = join(filename) if self.diag_fluxes: filename = self.flxfile.split('flx')[0]+'flx' - join(filename) - - - + self.flxfile = join(filename) + + + class NcfileIO(object): """Allow to create() and write() a Netcdf file of 'history' type, i.e. a set of model 2D snapshots. Variables were originally the @@ -186,7 +184,11 @@ def create(self): # store all the parameters as NetCDF global attributes dparam = self.param.__dict__ for k in dparam.keys(): - if type(dparam[k]) == type([0, 1]): + if k == "ems": + # The EMS stores itself and does not need to be stored in the netCDF file + pass + + elif type(dparam[k]) == type([0, 1]): # it is not straightforward to store a list in a netcdf file pass @@ -250,7 +252,8 @@ def join(filename): ''' Join history files without having to mpirun Useful when the run has been broken and one wants to join - things from an interactive session ''' + things from an interactive session. + Return the name of the new, joined history file. ''' template = filename+'_%03i.nc' hisfile_joined = filename+'.nc' @@ -390,3 +393,5 @@ def join(filename): os.remove(ncfile) print('-'*50) + + return hisfile_joined diff --git a/core/param.py b/core/param.py index 9dfed68..fa83474 100644 --- a/core/param.py +++ b/core/param.py @@ -3,6 +3,9 @@ import sys import getopt +# Local import +import ems + class Param(object): """class to set up the default parameters value of the model @@ -17,10 +20,16 @@ class Param(object): """ - def __init__(self, defaultfile): - """defaultfile is a sequel, it's no longer used the default file is - systematically the defaults.json located in the fluid2d/core + def __init__(self, defaultfile=None, ems_file=""): + """Load default parameters and optionally experiment parameters. + + The parameter `defaultfile` is no longer used and exists only + for backwards compatibility. The default file is always + the file `defaults.json` located in the core-folder of fluid2d. + The parameter `ems_file` takes optionally the name of an + experiment file. If given, the Experiment Management System + (EMS) is activated and the experiment file is parsed. """ import grid @@ -42,6 +51,11 @@ def __init__(self, defaultfile): else: self.print_param = False + if ems_file: + self.ems = ems.EMS(ems_file) + else: + self.ems = None + def set_parameters(self, namelist): avail = {} doc = {} @@ -76,6 +90,11 @@ def manall(self): self.man(p) def checkall(self): + if self.ems: + # Only create a new database entry once, not by every core + if self.myrank == 0: + self.ems.initialize(self.datadir) + self.expname = self.ems.get_expname() for p, avail in self.avail.items(): if getattr(self, p) in avail: # the parameter 'p' is well set @@ -109,6 +128,36 @@ def copy(self, obj, list_param): missing.append(k) return missing + def get_experiment_parameters(self): + """Return the experiment parameters dictionary loaded by the EMS. + + The EMS must be activated in the constructor of `Param` to use + this method. It returns the dictionary of experiment parameters + and exits. It is advised to use, when possible, the method + `loop_experiment_parameters` instead. + """ + return self.ems.parameters + + def loop_experiment_parameters(self): + """Iterate over the experiment parameters loaded by the EMS. + + In every iteration, this method returns a new dictionary of + experiment parameters containing a combination of the values + specified in the experiment file. This experiment file for the + EMS must be specified in the constructor of `Param`. If only + one value is given for every parameter in the experiment file, + the method `get_experiment_parameters` can be used instead. + The ID of the experiment is increased in every iteration. + """ + while self.ems.parameters: + yield self.ems.parameters + self.ems.setup_next_parameters() + + def finalize(self, fluid2d): + """Invoke the finalize method of the EMS if activated.""" + if self.ems: + self.ems.finalize(fluid2d) + if __name__ == "__main__": param = Param('default.xml') diff --git a/core/plotting.py b/core/plotting.py index c31a15b..eaf2f7e 100644 --- a/core/plotting.py +++ b/core/plotting.py @@ -32,6 +32,15 @@ def __init__(self, param, grid, var, diag): # Define a dummy cax to prevent create_fig from failing self.cax = [None, None] + if param.npx * param.npy > 1 and self.generate_mp4: + if self.myrank == 0: + print( + 'Warning: It is not possible to generate an mp4-file when ' + 'fluid2d runs on multiple cores.\n' + 'The parameter generate_mp4 is automatically changed to False.' + ) + self.generate_mp4 = False + nh = self.nh nx = param.nx ny = param.ny @@ -219,9 +228,10 @@ def update_fig(self, t, dt, kt): self.process.stdin.write(string) def finalize(self): - """ do nothing but close the mp4 thread if any""" + """Close the mp4 thread if any and the current figure.""" if self.generate_mp4: self.process.communicate() + plt.close() def set_cax(self, z): """ set self.cax, the color range""" diff --git a/core/restart.py b/core/restart.py index c012a02..e3541f3 100644 --- a/core/restart.py +++ b/core/restart.py @@ -78,16 +78,10 @@ def __init__(self, param, grid, f2d, launch=True): f2d.output.template = self.expdir +'/%s_%02i_his' % ( self.expname, self.nextrestart)+'_%03i.nc' f2d.output.hisfile = f2d.output.template % (self.myrank) - f2d.output.hisfile_joined = self.expdir + '/%s_%02i_his.nc' % ( - self.expname, self.nextrestart) if self.diag_fluxes: f2d.output.template = self.expdir +'/%s_%02i_flx' % ( self.expname, self.nextrestart)+'_%03i.nc' f2d.output.flxfile = f2d.output.template % (self.myrank) - f2d.output.flxfile_joined = self.expdir + '/%s_%02i_flx.nc' % ( - self.expname, self.nextrestart) - - # split the integration in 'ninterrestart' intervals and # save a restart at the end of each diff --git a/ems/EMShellExtensions.py b/ems/EMShellExtensions.py new file mode 100644 index 0000000..1410af2 --- /dev/null +++ b/ems/EMShellExtensions.py @@ -0,0 +1,42 @@ +"""Extensions for the EMShell of fluid2d + +The EMShell is the command-line interface of the fluid2d Experiment +Management System (EMS). Its functionality for data analysis can be +extended by adding new functions to the dictionary "extra_tools" in the +EMShell. These extensions are defined here, but they can also be +imported from other files. + +Author: Markus REINERT, June 2019 +""" + +import numpy as np +import netCDF4 as nc +from scipy.fftpack import fft, fftshift, fftfreq + + +def get_strongest_wavenumber_y(his_filename: str) -> float: + """Calculate the most intense wavenumber in y-direction. + + This function opens the given history-file, performs a Fourier + transform in y on the masked streamfunction psi and returns the + highest wavenumber which has at some point in time the highest + intensity apart from the wavenumber zero.""" + # Open history file and load the data + dataset_his = nc.Dataset(his_filename) + ny = dataset_his.ny + dy = dataset_his.Ly / ny + # Save the data as a masked numpy array + psi = dataset_his["psi"][:] + psi.mask = 1 - dataset_his["msk"][:] + # Set length of zero-padded signal (use ny for no zero-padding) + fft_ny = ny + # Caculate zero-padded Fourier-transform in y + fft_psi = fftshift(fft(psi, n=fft_ny, axis=1), axes=1) + # Its sampling frequency is dky = 1/dy/fft_ny + # Calculate the corresponding axis in Fourier-space + ky = fftshift(fftfreq(fft_ny, dy)) + # Remove the zero-frequency because it is always very large + fft_psi[:, ky==0] = 0 + # Calculate the frequency of maximal intensity (apart from the zero frequency) + ky_max = np.abs(ky)[np.argmax(np.max(np.abs(fft_psi), axis=2), axis=1)] + return np.max(ky_max) diff --git a/ems/Experiment-Manager.py b/ems/Experiment-Manager.py new file mode 100644 index 0000000..c7028a9 --- /dev/null +++ b/ems/Experiment-Manager.py @@ -0,0 +1,2053 @@ +"""EMShell: Command-line interface for the EMS of Fluid2D + +EMS is the Experiment Management System of Fluid2D, a powerful and +convenient way to handle big sets of experiments, cf. core/ems.py. +The EMShell is a command line interface to access, inspect, modify +and analyse the database and its entries. + +To start this programme, activate Fluid2D, then run this script with +Python (version 3.6 or newer). Alternatively, without the need to +activate Fluid2D, specify the path to the experiment folder as a +command-line argument when launching this script with Python. + +This code uses f-strings, which were introduced in Python 3.6: +https://docs.python.org/3/whatsnew/3.6.html#whatsnew36-pep498 +Unfortunately, they create a SyntaxError in older Python versions. + +It is possible to access the experiment database from multiple +processes at the same time, so one can add new entries to the database +with the EMS of Fluid2D while looking at the database and performing +data analysis in the EMShell; the EMShell shows automatically the +updated version of the database. However, this works less well if the +database is accessed by processes on different computers. In this +case it can be necessary to restart the EMShell to see changes in the +database. It is always necessary to restart the EMShell when a new +class of experiments (a new table) was added to the database. + +Author: Markus Reinert, May/June 2019, June 2020 +""" + +# Standard library imports +import os +import re +import sys +import cmd +import time +import shutil +import sqlite3 +import datetime +import readline +import itertools +import subprocess + +# Optional matplotlib import +try: + import matplotlib + # Before importing pyplot, choose the Qt5Agg backend of matplotlib, + # which allows to modify graphs and labels in a figure manually. + # If this causes problems, comment or change the following line. + matplotlib.use("Qt5Agg") + import matplotlib.pyplot as plt + from matplotlib.colors import LinearSegmentedColormap +except Exception as e: + print("Warning: matplotlib cannot be imported:", e) + print("Some functionality is deactivated.") + print("Install matplotlib to use all the features of the EMShell.") + matplotlib = None + +# Local import +import EMShellExtensions as EMExt + + +### Extensions +# Make new shell extensions available by adding them to this dictionary. +# The key is the name under which the function is called in the shell, +# which must not contain any whitespace. +# The value is the function, which takes as only argument the path to the +# his-file of an experiment and can return the calculated value. +extra_tools = { + "wavenumber": EMExt.get_strongest_wavenumber_y, + "wavelength": lambda hisname: 1/EMExt.get_strongest_wavenumber_y(hisname), +} + +### External programmes called by the EMShell +# Command to open mp4-files +MP4_PLAYER = "mplayer" +# Command to open NetCDF (his or diag) files +NETCDF_VIEWER = "ncview" + +### Settings for the output of a table +# Maximal length of comments (possible values: "AUTO", "FULL", or a positive integer) +# This setting is ignored if display_all is True. +COMMENT_MAX_LENGTH = "AUTO" +# Format-string for real numbers +FLOAT_FORMAT = "{:.4f}" +# Format-string for file sizes (MB) or disk space (GiB) +# Two decimals are used instead of three, to avoid confusion between the dot as +# a decimal separator and the dot as a delimiter after 1000. +SIZE_FORMAT = "{:.2f}" +# Symbol to separate two columns of the table +LIMITER = " " +# Symbol of the linebreak and its replacement in comments +# This is ignored if display_all is True or COMMENT_MAX_LENGTH is "FULL". +LINEBREAK_REPLACE = ("\n", "|") +# Show date and time in ISO-format or easy-to-read-format +# The easy-to-read-format does not include seconds, independent whether seconds +# are hidden or not. This setting is ignored if display_all is True. +ISO_DATETIME = False + +### Settings to print tables in colour +# Set COLOURS to False or None to disable colours (this does not disable highlighting). +# Otherwise, COLOURS must be list, of which each element describes the colours used in one row. +# The colours of every row are described by a list of colour codes, +# cf. https://en.wikipedia.org/wiki/ANSI_escape_code#Colors. +# An arbitrary number of colours can be specified. +# Apart from background colours, also text colours and font styles can be specified. +# Example to fill rows alternately with white and default colour: +# COLOURS = ( +# ("\033[47m",), +# ("\033[49m",), +# ) +# Example to fill columns alternately with white and default colour: +# COLOURS = ( +# ("\033[47m", "\033[49m",), +# ) +# Example for a check-pattern: +# COLOURS = ( +# ("\033[47m", "\033[49m",), +# ("\033[49m", "\033[47m",), +# ) +# Example for alternating background colours in rows and alternating font colours in columns: +# COLOURS = ( +# ("\033[31;47m", "\033[39;47m",), +# ("\033[31;49m", "\033[39;49m",), +# ) +COLOURS = ( + ("\033[107m",), + ("\033[49m",), +) +# Colour-code to highlight columns (used by "compare -h") +COLOURS_HIGHLIGHT = "\033[103m" +# Colour-code to reset to default colour and default font +COLOURS_END = "\033[39;49m" + + +### Language settings (applies currently only for dates in easy-to-read-format) +# Definition of languages +class English: + JUST_NOW = "just now" + AGO_MINUTES = "{} min ago" + AGO_HOURS = "{}:{:02} h ago" + YESTERDAY = "yesterday" + FUTURE = "in the future" + +class French: + JUST_NOW = "maintenant" + AGO_MINUTES = "il y a {} min" + AGO_HOURS = "il y a {}h{:02}" + YESTERDAY = "hier" + FUTURE = "à l'avenir" + +class German: + JUST_NOW = "gerade eben" + AGO_MINUTES = "vor {} Min." + AGO_HOURS = "vor {}:{:02} Std." + YESTERDAY = "gestern" + FUTURE = "in der Zukunft" + +# Selection of a language +LANG = English + + +# Defintion of a new colormap "iridescent" by Paul Tol +# (https://personal.sron.nl/~pault/#fig:scheme_iridescent) +if matplotlib: + iridescent = LinearSegmentedColormap.from_list( + "iridescent", + ['#FEFBE9', '#FCF7D5', '#F5F3C1', '#EAF0B5', '#DDECBF', + '#D0E7CA', '#C2E3D2', '#B5DDD8', '#A8D8DC', '#9BD2E1', + '#8DCBE4', '#81C4E7', '#7BBCE7', '#7EB2E4', '#88A5DD', + '#9398D2', '#9B8AC4', '#9D7DB2', '#9A709E', '#906388', + '#805770', '#684957', '#46353A'] + ) + iridescent.set_bad('#999999') + iridescent_r = iridescent.reversed() + + +### Global variables modifiable during runtime +# Hide the following information in the table +# They can be activated with the command "enable" during runtime. +# More information can be hidden with the command "disable". +hidden_information = { + 'size_diag', + 'size_flux', + 'datetime_seconds', +} + +# Temporarily show all information, including full comments +# This is set to True by the argument "-v" to commands which print tables. +display_all = False + + +class EMDBConnection: + """Experiment Management Database Connection""" + + def __init__(self, dbpath: str): + """Establish a connection to the database in dbpath and initialise all tables.""" + self.connection = None + if not os.path.isfile(dbpath): + raise FileNotFoundError(f"Database file {dbpath} does not exist.") + + # Create a connection to the given database + print("-"*50) + print("Opening database {}.".format(dbpath)) + self.connection = sqlite3.connect(dbpath) + cursor = self.connection.cursor() + + # Get all tables of the database + cursor.execute("SELECT name FROM sqlite_master WHERE type='table';") + # This gives a list like that: [('table1',), ('table2',)] + self.tables = [EMDBTable(cursor, t[0]) for t in cursor.fetchall()] + + def __del__(self): + """Close the connection to the database (if any).""" + if self.connection: + print("Closing database.") + print("-"*50) + self.connection.close() + + def save_database(self): + self.connection.commit() + + def delete_table(self, table_name): + self.connection.execute(f'DROP TABLE "{table_name}"') + + def get_table_overview(self): + """Return a text with metadata about the tables in the database.""" + if self.tables: + text = "Experiments in database:" + for table in self.tables: + n_entries = len(table) + n_columns = len(table.columns) + text += f"\n - {table.name}: " + text += f"{n_entries} experiment{'s' if n_entries != 1 else ''}, " + text += f"{n_columns} columns" + else: + text = "No experiments in database." + return text + + def is_valid_table(self, table_name): + """Check if a table with the given name exists.""" + for table in self.tables: + if table.name == table_name: + return True + return False + + def is_valid_column(self, column_name): + """Check if a column with the given name exists in any table.""" + for table in self.tables: + for n, t in table.columns: + if column_name == n: + return True + return False + + def get_all_column_names(self): + """Return a list of the names of all columns in all tables.""" + return [n for table in self.tables for n, t in table.columns] + + def do(self, function, table_name, *args, **kwargs): + """Call a function on a table specified by its name and return the result. + + This method calls function(table, *args, **kwargs), where table + is the EMDBTable object with the name table_name, and returns + the result of this function call. If no table with the given + name exists, then a warning is printed and None is returned. + """ + for table in self.tables: + if table.name == table_name: + return function(table, *args, **kwargs) + print('No table exists with name "{}".'.format(table_name)) + return None + + +class EMDBTable: + """Experiment Management Database Table""" + + def __init__(self, cursor: sqlite3.Cursor, name: str): + self.name = str(name) + self.c = cursor + # Get columns + self.c.execute('PRAGMA table_info("{}")'.format(self.name)) + self.columns = [(column[1], column[2]) for column in self.c.fetchall()] + # column[0]: index from 0, column[1]: name, column[2]: type + + def __str__(self): + """Return a text representation of all entries formated as a table.""" + self.c.execute('SELECT * from "{}"'.format(self.name)) + return string_format_table(self.name, self.columns, self.c.fetchall()) + + def __len__(self): + self.c.execute('SELECT Count(*) FROM "{}"'.format(self.name)) + return self.c.fetchone()[0] + + def get_data(self, column_name, condition=""): + try: + self.c.execute( + f'SELECT {column_name} FROM "{self.name}" WHERE {condition}' + if condition else + f'SELECT {column_name} FROM "{self.name}"' + ) + except sqlite3.OperationalError as e: + print(f'SQL error for experiment "{self.name}":', e) + return [] + else: + return [e[0] for e in self.c.fetchall()] + + def get_column_names(self): + return [n for n, t in self.columns] + + def get_latest_entry(self): + self.c.execute('SELECT id from "{}" ORDER BY datetime DESC'.format(self.name)) + result = self.c.fetchone() + self.c.fetchall() # otherwise the database stays locked + if result: + return result[0] + else: + print('Table "{}" is empty.'.format(self.name)) + return None + + def entry_exists(self, id_): + self.c.execute('SELECT id FROM "{}" WHERE id = ?'.format(self.name), (id_,)) + if self.c.fetchone() is not None: + return True + return False + + def delete_entry(self, id_): + self.c.execute('DELETE FROM "{}" WHERE id = ?'.format(self.name), (id_,)) + + def print_selection(self, statement): + try: + self.c.execute('SELECT * FROM "{}" WHERE {}'.format(self.name, statement)) + except sqlite3.OperationalError as e: + print('SQL error for experiment "{}":'.format(self.name), e) + else: + print(string_format_table(self.name, self.columns, self.c.fetchall())) + + def print_sorted(self, statement): + try: + self.c.execute('SELECT * FROM "{}" ORDER BY {}'.format(self.name, statement)) + except sqlite3.OperationalError as e: + print('SQL error for experiment "{}":'.format(self.name), e) + else: + print(string_format_table(self.name, self.columns, self.c.fetchall())) + + def print_comparison(self, ids, highlight=False): + try: + self.c.execute( + 'SELECT * FROM "{}" WHERE id IN {}'.format(self.name, tuple(ids)) + if ids else + 'SELECT * FROM "{}"'.format(self.name) + ) + except sqlite3.OperationalError as e: + print('SQL error for experiment "{}":'.format(self.name), e) + return + full_entries = self.c.fetchall() + different_columns = [] + for i, row in enumerate(full_entries): + if i == 0: + continue + for c, value in enumerate(row): + if c not in different_columns and value != full_entries[i-1][c]: + different_columns.append(c) + if not highlight: + # Print only the columns of the table with differences + print(string_format_table( + self.name, + [col for c, col in enumerate(self.columns) if c in different_columns], + [ + [val for c, val in enumerate(row) if c in different_columns] + for row in full_entries + ], + )) + else: + # Print the full table and highlight the columns with differences + print(string_format_table( + self.name, self.columns, full_entries, + [col[0] for c, col in enumerate(self.columns) if c in different_columns], + )) + + def set_comment(self, id_, new_comment): + try: + self.c.execute( + 'UPDATE "{}" SET comment = ? WHERE id = ?'.format(self.name), + (new_comment, id_,) + ) + except sqlite3.OperationalError as e: + print('SQL error for experiment "{}":'.format(self.name), e) + return False + else: + return True + + def add_column(self, column_name, data): + """Add a new column to the table with the given name and data. + + The new column is always of datatype REAL and is added just + before the column "duration", i.e., at the end of the + user-defined columns. + + Adding a new column is a bit tricky in SQLite, because this is + not supported natively. Therefore, it is necessary to create a + new table with the additional column and move all the data there + from the original table. For this, we rename, at first, the + existing table, then we create a new one with the original name, + we migrate all the data from the old to the new table and we + remove the old table. Then, finally, we fill the new table with + the given data. If anything went wrong during this process, + there should be two tables, one with the original name of the + table, and one with the prefix "tmp_", so it should be possible + for the user to recover the data by manually moving it from the + temporary to the new table. + """ + old_column_name_string = ", ".join( + ['"{}"'.format(col[0]) for col in self.columns] + ) + for i, column in enumerate(self.columns): + if column[0] == "duration": + i_new_column = i + break + else: + print( + 'Cannot add a new column before "duration" because ' + 'no column "duration" exists in the table "{}".'.format(self.name) + ) + return False + self.columns = ( + self.columns[:i_new_column] + [(column_name, "REAL")] + self.columns[i_new_column:] + ) + new_column_name_type_string = ", ".join([ + '"{}" {}'.format(column_name, column_type) + for column_name, column_type in self.columns + ]) + tmp_name = "tmp_" + self.name + try: + # Rename the current table + self.c.execute( + 'ALTER TABLE "{}" RENAME TO "{}"'.format(self.name, tmp_name) + ) + # Create a new table with its name and the new column + self.c.execute( + 'CREATE TABLE "{}" ({})'.format(self.name, new_column_name_type_string) + ) + # Copy the data from the old to the new table + self.c.execute( + 'INSERT INTO "{}" ({}) SELECT {} FROM "{}"'.format( + self.name, old_column_name_string, old_column_name_string, tmp_name + ) + ) + # Remove the old table + self.c.execute('DROP TABLE "{}"'.format(tmp_name)) + except sqlite3.OperationalError as e: + print( + 'SQL error for experiment "{}" when adding column "{}": {}' + .format(self.name, column_name, e) + ) + return False + # Get the IDs of the entries in the table + try: + self.c.execute(f'SELECT id FROM "{self.name}"') + except sqlite3.OperationalError as e: + print(f'SQL error for experiment "{self.name}" when fetching IDs:', e) + return False + # Fill every entry + return_status = True + for entry, value in zip(self.c.fetchall(), data): + id_ = entry[0] + try: + # TODO: do this with ?-notation instead of string format + self.c.execute( + f'UPDATE "{self.name}" SET {column_name} = {value} WHERE id = {id_}' + ) + except sqlite3.OperationalError as e: + print( + 'SQL error for experiment "{}" when adding value {} to entry {}:' + .format(self.name, repr(value), id_), e + ) + return_status = False + return return_status + + def remove_column(self, column_name): + """Remove the column with the given name from the table. + + A similar caveat as for add_column applies. + """ + for column in self.columns: + if column[0] == column_name: + self.columns.remove(column) + break + else: + print('Cannot find column "{}" in table "{}".'.format(column_name, self.name)) + return False + new_column_name_string = ", ".join([ + '"{}"'.format(column_name) + for column_name, column_type in self.columns + ]) + new_column_name_type_string = ", ".join([ + '"{}" {}'.format(column_name, column_type) + for column_name, column_type in self.columns + ]) + tmp_name = "tmp_" + self.name + try: + # Rename the current table + self.c.execute( + 'ALTER TABLE "{}" RENAME TO "{}"'.format(self.name, tmp_name) + ) + # Create a new table with its name and without the column + self.c.execute( + 'CREATE TABLE "{}" ({})'.format(self.name, new_column_name_type_string) + ) + # Copy the data from the old to the new table + self.c.execute( + 'INSERT INTO "{}" SELECT {} FROM "{}"'.format( + self.name, new_column_name_string, tmp_name + ) + ) + # Remove the old table + self.c.execute('DROP TABLE "{}"'.format(tmp_name)) + except sqlite3.OperationalError as e: + print( + 'SQL error for experiment "{}" when removing column "{}": {}' + .format(self.name, column_name, e) + ) + return False + return True + + +class EMShell(cmd.Cmd): + """Experiment Management (System) Shell""" + + intro = ( + "-" * 50 + + "\nType help or ? to list available commands." + + "\nType exit or Ctrl+D or Ctrl+C to exit." + + "\nPress Tab-key for auto-completion of commands, table names, etc." + + "\n" + ) + prompt = "(EMS) " + ruler = "-" + + def __init__(self, experiments_dir: str): + """Establish a connection to the database in the given directory. + + This initialises also the command history. + """ + self.initialized = False + super().__init__() + print("-"*50) + print("Fluid2D Experiment Management System (EMS)") + self.exp_dir = experiments_dir + self.con = EMDBConnection(os.path.join(self.exp_dir, "experiments.db")) + self.intro += "\n" + self.con.get_table_overview() + "\n" + self.selected_table = "" + + # Settings for saving the command history + self.command_history_file = os.path.join(self.exp_dir, ".emshell_history") + readline.set_history_length(1000) + # Load previously saved command history + if os.path.exists(self.command_history_file): + readline.read_history_file(self.command_history_file) + self.initialized = True + + def __del__(self): + """Save the command history (not the database).""" + if self.initialized: + print("Saving command history.") + readline.write_history_file(self.command_history_file) + + ### Functionality to MODIFY how the programme acts + def do_enable(self, params): + global hidden_information + if not params: + print("Please specify information to enable.") + elif params == "all": + hidden_information.clear() + else: + for param in params.split(): + if param == "size": + hidden_information.difference_update( + {'size_diag', 'size_flux', 'size_his', 'size_mp4', 'size_total'} + ) + else: + hidden_information.discard(param) + + def complete_enable(self, text, line, begidx, endidx): + parameters = hidden_information.copy() + if len(parameters) > 0: + parameters.add('all') + if not parameters.isdisjoint({'size_diag', 'size_flux', 'size_his', + 'size_mp4', 'size_total'}): + parameters.add('size') + return [p for p in parameters if p.startswith(text)] + + def help_enable(self): + print( + 'Make hidden information in the experiment table visible.\n' + 'See the help of "disable" for further explanations.' + ) + + def do_disable(self, params): + global hidden_information + if not params: + print("Please specify information to disable.") + return + for param in params.split(): + if param == "size": + # Short notation for the union of two sets + hidden_information |= {'size_diag', 'size_flux', 'size_his', + 'size_mp4', 'size_total'} + elif (param == "datetime_year" + or param == "datetime_seconds" + or self.con.is_valid_column(param) + ): + hidden_information.add(param) + else: + print("Unknown argument:", param) + + def complete_disable(self, text, line, begidx, endidx): + parameters = [ + 'datetime_year', + 'datetime_seconds', + ] + # If not all size columns are hidden, add shorthand for all sizes + if not hidden_information.issuperset({'size_diag', 'size_flux', 'size_his', + 'size_mp4', 'size_total'}): + parameters += ['size'] + # Add columns of the selected table (or of all tables if none is + # selected) to the list of auto-completions + if self.selected_table: + parameters += self.con.do(EMDBTable.get_column_names, self.selected_table) + else: + parameters += self.con.get_all_column_names() + return [p for p in parameters if p.startswith(text) and p not in hidden_information] + + def help_disable(self): + print( + 'Hide unneeded information in the experiment table.\n' + 'Every parameter/column can be hidden, for example:\n' + ' - disable comment\n' + 'Furthermore, the year and number of seconds can be hidden from the datetime ' + 'column by using "disable datetime_year" and "disable datetime_seconds".\n' + 'Multiple parameters can be specified at once, for example:\n' + ' - disable size_his size_diag size_flux\n' + 'To hide all file sizes, use "disable size".\n' + 'To show hidden parameters again, use "enable".\n' + 'In addition to the behaviour described here, it also supports the command ' + '"enable all".' + ) + + ### Functionality to SHOW the content of the database + def do_list(self, params): + """List all experiment classes (tables) in the database.""" + print(self.con.get_table_overview()) + + def do_show(self, params): + """Show the content of a table.""" + global display_all + params = set(params.split()) + if "-v" in params: + display_all = True + params.remove('-v') + else: + display_all = False + if len(params) == 0: + # No table name given + if self.selected_table: + print("-"*50) + self.con.do(print, self.selected_table) + else: + for table in self.con.tables: + print("-"*50) + print(table) + else: + for table_name in sorted(params): + print("-"*50) + self.con.do(print, table_name) + print("-"*50) + + def complete_show(self, text, line, begidx, endidx): + return self.table_name_completion(text) + + def help_show(self): + print("""> show [name(s) of experiment class(es)] [-v] + Show all entries in the database for one or more classes of experiments. + Specify as parameter the name of the experiment class to show. Multiple + names may be specified. If no name is specified, then the experiments of + the currently selected class are shown (cf. "select"). If no name name is + specified and no experiment selected, then all entries are shown. + Use the commands "enable" and "disable" to specify which information + (i.e., which column) is displayed. To display all information instead, use + "show" with the parameter "-v". + + See also: filter, sort.""") + + def do_filter(self, params): + # Check the run condition + if not self.selected_table: + print('No experiment selected. Select an experiment to filter it.') + return + global display_all + if params.endswith(" -v"): + params = params[:-3] + display_all = True + else: + display_all = False + if not params: + print('No condition to filter given. Type "help filter" for further information.') + return + print("-"*50) + self.con.do(EMDBTable.print_selection, self.selected_table, params) + print("-"*50) + + def complete_filter(self, text, line, begidx, endidx): + return self.column_name_completion(self.selected_table, text) + + def help_filter(self): + print("""> filter [-v] + Filter experiments of the currently selected class according to the given + condition. Any valid SQLite WHERE-statement can be used as a filter. + Examples: + - filter intensity <= 0.2 + - filter slope = 0.5 + - filter diffusion = "True" + - filter datetime >= "2019-03-21" + - filter perturbation != "gauss" AND duration > 20 + It is necessary to put the value for string, datetime or boolean argument + in quotation marks as shown. The command + - filter Kdiff + shows all entries where Kdiff is non-zero, whereas the command + - filter NOT Kdiff + shows all entries where Kdiff is zero, following the usual Python convention + that zero is interpreted as False. Note that this does not work for boolean + variables in the database, since they are saved as strings. + To sort the filtered experiments, use SQLite syntax, as in the following + Examples: + - filter intensity > 0.1 ORDER BY intensity + - filter perturbation != "gauss" ORDER BY size_total DESC + The see all information about the filtered experiments, add "-v" at the end + of the command. + + See also: sort, show.""") + + def do_sort(self, params): + # Check the run condition + if not self.selected_table: + print("No experiment selected. Select an experiment to sort it.") + return + global display_all + if params.endswith(" -v"): + params = params[:-3] + display_all = True + else: + display_all = False + if not params: + print('No parameter to sort given. Type "help sort" for further information.') + else: + print("-"*50) + self.con.do(EMDBTable.print_sorted, self.selected_table, params) + print("-"*50) + + def complete_sort(self, text, line, begidx, endidx): + return self.column_name_completion(self.selected_table, text) + + def help_sort(self): + print("""> sort [desc] [-v] + Sort the experiments by the value of a given parameter. + To invert the order, add "desc" (descending) to the command. + Examples: + - sort intensity: show experiments with lowest intensity on top of the table. + - sort size_total desc: show experiments with biggest file size on top. + The see all information about the sorted experiments, add "-v" at the end + of the command. + + See also: filter, show.""") + + def do_compare(self, params): + # Check the run condition + if not self.selected_table: + print('No experiment selected. Select an experiment to make a comparison.') + return + # Parse the arguments + global display_all + display_all = False + highlight = False + other_params = [] + for p in params.split(): + if p == "-v": + display_all = True + elif p == "-h": + highlight = True + else: + other_params.append(p) + if other_params: + ids = self.parse_multiple_ids(other_params) + if ids is None: + return + if len(ids) < 2: + print("Please specify at least 2 different IDs.") + return + elif self.con.do(len, self.selected_table) < 2: + print("Selected experiment contains less than 2 entries.") + return + else: + # No IDs means no filtering, i.e., all entries are compared + ids = [] + print("-"*50) + self.con.do(EMDBTable.print_comparison, self.selected_table, ids, highlight) + print("-"*50) + + def help_compare(self): + print("""> compare [IDs] [-h] [-v] + Show the difference between two or more entries of the selected experiment. + This prints a table which includes only the parameters in which the + specified entries differ. Alternatively, add "-h" to show all parameters + and highlight the differences in colour. Disabled columns are not shown by + default. Add the argument "-v" to show disabled columns, the full date-time + and the full comment. Instead of an ID, "last" can be used to compare with + the latest entry. If no ID is specified, all entries of the selected + experiment class are compared.""") + + ### Functionality to OPEN experiment files + def do_open_mp4(self, params): + """Open the mp4-file for an experiment specified by its name and ID.""" + if params.endswith(" -v"): + verbose = True + params = params[:-3].rstrip() + else: + verbose = False + expname_id = self.parse_params_to_experiment(params) + if not expname_id: + return + expname = "{}_{:03}".format(*expname_id) + dir_ = os.path.join(self.exp_dir, expname) + try: + files = os.listdir(dir_) + except FileNotFoundError: + print("Folder does not exist:", dir_) + return + for f in files: + if f.endswith(".mp4") and f.startswith(expname): + break + else: + print("No mp4-file found in folder:", dir_) + return + path = os.path.join(self.exp_dir, expname, f) + self.open_file(MP4_PLAYER, path, verbose) + + def complete_open_mp4(self, text, line, begidx, endidx): + return self.table_name_completion(text) + + def help_open_mp4(self): + print(f"""> open_mp4 [experiment] [-v] + Open the mp4-file of an experiment with {MP4_PLAYER}.""") + self.print_param_parser_help() + print(""" + Add "-v" to the command to see the output of the external programme. + + The programme to open mp4-files with can be configured in the Python script + of the Experiment-Manager with the constant "MP4_PLAYER".""") + + def do_open_his(self, params): + """Open the his-file for an experiment specified by its name and ID.""" + if params.endswith(" -v"): + verbose = True + params = params[:-3].rstrip() + else: + verbose = False + expname_id = self.parse_params_to_experiment(params) + if not expname_id: + return + expname = "{}_{:03}".format(*expname_id) + path = os.path.join(self.exp_dir, expname, expname + "_his.nc") + if not os.path.isfile(path): + print("File does not exist:", path) + return + self.open_file(NETCDF_VIEWER, path, verbose) + + def complete_open_his(self, text, line, begidx, endidx): + return self.table_name_completion(text) + + def help_open_his(self): + print(f"""> open_his [experiment] [-v] + Open the NetCDF history-file of an experiment with {NETCDF_VIEWER}.""") + self.print_param_parser_help() + print(""" + Add "-v" to the command to see the output of the external programme. + + The programme to open NetCDF-files with can be configured in the Python + script of the Experiment-Manager with the constant "NETCDF_VIEWER".""") + + def do_open_diag(self, params): + """Open the diag-file for an experiment specified by its name and ID.""" + if params.endswith(" -v"): + verbose = True + params = params[:-3].rstrip() + else: + verbose = False + expname_id = self.parse_params_to_experiment(params) + if not expname_id: + return + expname = "{}_{:03}".format(*expname_id) + path = os.path.join(self.exp_dir, expname, expname + "_diag.nc") + if not os.path.isfile(path): + print("File does not exist:", path) + return + self.open_file(NETCDF_VIEWER, path, verbose) + + def complete_open_diag(self, text, line, begidx, endidx): + return self.table_name_completion(text) + + def help_open_diag(self): + print(f"""> open_diag [experiment] [-v] + Open the NetCDF diagnostics-file of an experiment with {NETCDF_VIEWER}.""") + self.print_param_parser_help() + print(""" + Add "-v" to the command to see the output of the external programme. + + The programme to open NetCDF-files with can be configured in the Python + script of the Experiment-Manager with the constant "NETCDF_VIEWER".""") + + ### Functionality for Data Analysis + def do_calculate(self, params): + if " " not in params: + print("At least two arguments are needed.") + return + # Parse and check toolname + i = params.find(" ") + toolname = params[:i] + if not extra_tools: + print('There are no tools for calculations loaded. ' + 'Add some tools in the code and restart the programme.') + return + if toolname not in extra_tools: + print(f'Unknown tool: "{toolname}". The available tools are:') + for t in extra_tools: + print(" -", t) + return + # Parse ID and optionally table name + experiment_name_id = self.parse_params_to_experiment(params[i+1:]) + if experiment_name_id is None: + return + table_name, id_ = experiment_name_id + expname = "{}_{:03}".format(table_name, id_) + path = os.path.join(self.exp_dir, expname, expname + '_his.nc') + try: + val = extra_tools[toolname](path) + except Exception as e: + print(f'Tool "{toolname}" did not succeed on experiment {id_}. ' + 'The error message is:') + print(e) + return + if val is not None: + print(f'-> {toolname}({table_name}:{id_}) = {val}') + else: + print(f'-> {toolname}({table_name}:{id_}) succeeded without return value.') + + def complete_calculate(self, text, line, begidx, endidx): + return [p for p in extra_tools if p.startswith(text)] + + def help_calculate(self): + print("""> calculate [experiment] + Call an EMShell-Extension on an experiment and print the result.""") + self.print_param_parser_help() + print(""" + To make a tool available, load it into the Python script of the + Experiment-Manager and add it to the dictionary "extra_tools". + Its name must not contain any whitespace.""") + + def do_plot(self, params): + # Check that the required module is loaded + if not matplotlib: + print("This feature requires matplotlib, which has not been imported successfully.") + print("The error message can be found at the beginning of the programme output.") + return + # Check the run condition + if not self.selected_table: + print('No experiment selected. Select an experiment to plot its data.') + return + # Parse the arguments + parameters = self.parse_plot_parameters(params, 2) + if not parameters: + return + variables = parameters["variables"] + # Get the data belonging to the parameters + datas = self.get_multiple_data(self.selected_table, variables, parameters["condition"]) + if not datas: + return + # Print and plot the data + for i, c in enumerate(["x", "y"]): + print(f'-> {c} ({variables[i]}):', datas[i]) + plot_label = variables[1] + if parameters["condition"]: + plot_label += ' (' + parameters["condition"] + ')' + plt.title(self.selected_table) + plt.xlabel(variables[0]) + try: + plt.plot( + datas[0], datas[1], parameters["format_string"], label=plot_label, + ) + except Exception as e: + print("Plot did not succeed. Error message:") + print(e) + else: + plt.legend() + if parameters["xmin"] is not None or parameters["xmax"] is not None: + plt.xlim(parameters["xmin"], parameters["xmax"]) + if parameters["ymin"] is not None or parameters["ymax"] is not None: + plt.ylim(parameters["ymin"], parameters["ymax"]) + if parameters["grid"]: + plt.grid(True) + if parameters["save_as"]: + plt.savefig(get_unique_save_filename("plot_{}." + parameters["save_as"])) + plt.show() + + def complete_plot(self, text, line, begidx, endidx): + return self.plot_attribute_completion(text, [ + 'f=', 'grid', + 'png', 'pdf', 'svg', + 'xmin=', 'xmax=', 'ymin=', 'ymax=', + ]) + + def help_plot(self): + print("""> plot [condition] [-grid] [-f=] [-{x|y}{min|max}=] [-{png|pdf|svg}] + Make a diagram showing the relation between the two variables for the + selected experiment.""") + self.print_general_plot_help("2d") + print(""" + Example: + - plot size_his duration id <= 10 -f=g.- -grid + This command creates a plot in green with dots and lines on a grid + comparing the integration time of the first ten experiments in the + selected class of experiments with the size of their history-file.""") + + def do_scatter(self, params): + # Check that the required module is loaded + if not matplotlib: + print("This feature requires matplotlib, which has not been imported successfully.") + print("The error message can be found at the beginning of the programme output.") + return + # Check the run condition + if not self.selected_table: + print('No experiment selected. Select an experiment to plot its data.') + return + # Parse the arguments + parameters = self.parse_plot_parameters(params, 3) + if not parameters: + return + variables = parameters["variables"] + # Get the data belonging to the parameters + datas = self.get_multiple_data(self.selected_table, variables, parameters["condition"]) + if not datas: + return + # Print and plot the data + for i, c in enumerate(["x", "y", "z"]): + print('-> {} ({}):'.format(c, variables[i]), datas[i]) + plot_title = self.selected_table + ": " + variables[2] + if parameters["condition"]: + plot_title += ' (' + parameters["condition"] + ')' + plt.title(plot_title) + plt.xlabel(variables[0]) + plt.ylabel(variables[1]) + try: + plt.scatter( + datas[0], datas[1], c=datas[2], + cmap=parameters["cmap"], + vmin=parameters["zmin"], vmax=parameters["zmax"], + ) + except Exception as e: + print("Scatter did not succeed. Error message:") + print(e) + else: + plt.colorbar() + if parameters["xmin"] is not None or parameters["xmax"] is not None: + plt.xlim(parameters["xmin"], parameters["xmax"]) + if parameters["ymin"] is not None or parameters["ymax"] is not None: + plt.ylim(parameters["ymin"], parameters["ymax"]) + if parameters["grid"]: + plt.grid(True) + if parameters["save_as"]: + plt.savefig(get_unique_save_filename("scatter_{}." + parameters["save_as"])) + plt.show() + + def complete_scatter(self, text, line, begidx, endidx): + return self.plot_attribute_completion(text, [ + 'cmap=', 'grid', + 'png', 'pdf', 'svg', + 'xmin=', 'xmax=', 'ymin=', 'ymax=', 'zmin=', 'zmax=', + ]) + + def help_scatter(self): + print("""> scatter [condition] [-grid] [-cmap=] [-{x|y|z}{min|max}=] [-{png|pdf|svg}] + Make a scatter plot showing the values of the third variable in colour in + relation to the other two variables for the selected experiment.""") + self.print_general_plot_help("3d") + print(""" + Example: + - scatter intensity sigma duration diffusion="False" OR Kdiff=0 -zmin=0 + This command creates a scatter plot with the default colourmap of + matplotlib on a colour-axis starting from 0 which shows the integration + time in relation to the intensity and the value of sigma for experiments + of the current class without diffusion.""") + + def do_pcolor(self, params): + # Check that the required module is loaded + if not matplotlib: + print("This feature requires matplotlib, which has not been imported successfully.") + print("The error message can be found at the beginning of the programme output.") + return + # Check the run condition + if not self.selected_table: + print('No experiment selected. Select an experiment to plot its data.') + return + # Parse the arguments + parameters = self.parse_plot_parameters(params, 3) + if not parameters: + return + variables = parameters["variables"] + # Get the data belonging to the parameters + datas = self.get_multiple_data(self.selected_table, variables, parameters["condition"]) + if not datas: + return + x_data = datas[0] + y_data = datas[1] + z_data = datas[2] + # Arrange the data in a grid + xvalues = sorted(set(x_data)) + yvalues = sorted(set(y_data)) + nx = len(xvalues) + ny = len(yvalues) + print(f"There are {nx} unique x-values and {ny} unique y-values.") + data_grid = [[float("nan")] * nx for i in range(ny)] + for i, zval in enumerate(z_data): + data_grid[yvalues.index(y_data[i])][xvalues.index(x_data[i])] = zval + if parameters["shading"] == 'flat': + # If no shading is applied, extend the axes to have each rectangle + # centred at the corresponding (x,y)-value + xaxis = [xvalues[0] - (xvalues[1] - xvalues[0]) / 2] + xaxis += [xvalues[i] + (xvalues[i+1] - xvalues[i]) / 2 for i in range(len(xvalues) - 1)] + xaxis += [xvalues[-1] + (xvalues[-1] - xvalues[-2]) / 2] + yaxis = [yvalues[0] - (yvalues[1] - yvalues[0]) / 2] + yaxis += [yvalues[i] + (yvalues[i+1] - yvalues[i]) / 2 for i in range(len(yvalues) - 1)] + yaxis += [yvalues[-1] + (yvalues[-1] - yvalues[-2]) / 2] + else: + xaxis = xvalues + yaxis = yvalues + # Print and plot the data + print(f"-> x ({variables[0]}):", xvalues) + print(f"-> y ({variables[1]}):", yvalues) + print(f"-> z ({variables[2]}):") + print(data_grid) + plot_title = self.selected_table + ": " + variables[2] + if parameters["condition"]: + plot_title += ' (' + parameters["condition"] + ')' + plt.title(plot_title) + plt.xlabel(variables[0]) + plt.ylabel(variables[1]) + cmap = parameters["cmap"] + if cmap == "iridescent": + cmap = iridescent + if cmap == "iridescent_r": + cmap = iridescent_r + try: + plt.pcolormesh( + xaxis, yaxis, data_grid, + cmap=cmap, shading=parameters["shading"], + vmin=parameters["zmin"], vmax=parameters["zmax"], + ) + except Exception as e: + print("Pcolormesh did not succeed. Error message:") + print(e) + else: + plt.colorbar() + if parameters["xmin"] is not None or parameters["xmax"] is not None: + plt.xlim(parameters["xmin"], parameters["xmax"]) + if parameters["ymin"] is not None or parameters["ymax"] is not None: + plt.ylim(parameters["ymin"], parameters["ymax"]) + if parameters["grid"]: + plt.grid(True) + if parameters["save_as"]: + plt.savefig(get_unique_save_filename("pcolor_{}." + parameters["save_as"])) + plt.show() + + def complete_pcolor(self, text, line, begidx, endidx): + return self.plot_attribute_completion(text, [ + 'cmap=', 'shading', 'grid', + 'png', 'pdf', 'svg', + 'xmin=', 'xmax=', 'ymin=', 'ymax=', 'zmin=', 'zmax=', + ]) + + def help_pcolor(self): + print("""> pcolor [condition] [-grid] [-shading] [-cmap=] [-{x|y|z}{min|max}=] [-{png|pdf|svg}] + Make a pseudo-colour plot showing the values of the third variable in colour + in relation to the two other variables for the selected experiment.""") + self.print_general_plot_help("3d", shading=True) + print(""" + Example: + - pcolor intensity sigma duration diffusion="False" OR Kdiff=0 -zmin=0 + This command creates a pseudo-colour plot with the default colourmap of + matplotlib on a colour-axis starting from 0 which shows the integration + time in relation to the intensity and the value of sigma for experiments + of the current class without diffusion.""") + + def do_save_figure(self, params): + # Check that the required module is loaded + if not matplotlib: + print("This feature requires matplotlib, which has not been imported successfully.") + print("The error message can be found at the beginning of the programme output.") + return + if not params: + params = "png" + if params in ["png", "pdf", "svg"]: + plt.savefig(get_unique_save_filename("figure_{}." + params)) + else: + plt.savefig(params) + + def complete_save_figure(self, text, line, begidx, endidx): + if "." not in text: + return [e for e in ("png", "pdf", "svg") if e.startswith(text)] + stub = text[:text.rfind(".")] + return [n for n in (stub + ".png", stub + ".pdf", stub + ".svg") if n.startswith(text)] + + def help_save_figure(self): + print("""> save_figure [filename or -type] + Save the currently opened figure in a file. One can either specify the full + filename or one of the filetypes png, pdf or svg alone. If only the type is + specified, then the name is automatically set to "figure_#.type", where + "#" is an integer such that the filename is unique and "type" is the given + filetype. If no type and no name is given, the figure is saved as png.""") + + def do_new_figure(self, params): + """Open a window to draw the next plot in a new figure.""" + # Check that the required module is loaded + if not matplotlib: + print("This feature requires matplotlib, which has not been imported successfully.") + print("The error message can be found at the beginning of the programme output.") + return + plt.figure() + + ### Functionality to MODIFY the entries + def do_new_comment(self, params): + # Check and parse parameters + experiment_name_id = self.parse_params_to_experiment(params) + if experiment_name_id is None: + return + table_name, id_ = experiment_name_id + # Print the current entry fully + global display_all + display_all = True + print("-"*50) + self.con.do(EMDBTable.print_selection, table_name, f"id = {id_}") + print("-"*50) + # Ask for user input + print("Write a new comment for this entry (Ctrl+D to finish, Ctrl+C to cancel):") + comment_lines = [] + while True: + try: + line = input() + except EOFError: + break + except KeyboardInterrupt: + print("Cancelling.") + return + comment_lines.append(line) + comment = "\n".join(comment_lines).strip() + # Update the comment + if self.con.do(EMDBTable.set_comment, table_name, id_, comment): + self.con.save_database() + print("New comment was saved.") + + def help_new_comment(self): + print("""> new_comment [experiment] + Ask the user to enter a new comment for an experiment.""") + self.print_param_parser_help() + + def do_add_column(self, params): + # TODO: this adds currently only REAL data (float) before the "duration" column + # Check the run condition + if not self.selected_table: + print('No experiment selected. Select an experiment to add a column.') + return + params = params.split() + if len(params) != 2: + print( + "Exactly two arguments are needed: name of the new column " + "and name of the tool to fill it." + ) + return + column_name, tool_name = params + if column_name in self.con.do(EMDBTable.get_column_names, self.selected_table): + print( + "A column with the name {} exists already in {}." + .format(column_name, self.selected_table) + ) + return + data = self.get_data(self.selected_table, tool_name, condition="") + if not data: + print("Getting data failed.") + return + if self.con.do(EMDBTable.add_column, self.selected_table, column_name, data): + self.con.save_database() + print("Done.") + else: + print("An error occured.") + + def complete_add_column(self, text, line, begidx, endidx): + return self.plot_attribute_completion(text, []) + + def help_add_column(self): + print("""> add_column + Add a new column with the given name to the selected table. The new + column will be added before the "duration" column, that means, at + the end of the user-defined columns. It will be of type REAL + (float) and will be filled with data computed by the specified tool. + For more information about tools, see "calculate". Instead of a + tool, also the name of an already existing column can be given to + copy the data from there. + + Warning: If you want to run more experiments of the selected class + after adding a new column to the table, you must also add a + corresponding parameter in the experiment file with the same + datatype and at the same position.""" + ) + + def do_remove_column(self, params): + # Check the run condition + if not self.selected_table: + print('No experiment selected. Select an experiment to remove a column.') + return + column_name = params + if column_name not in self.con.do(EMDBTable.get_column_names, self.selected_table): + print( + "No column with the name {} exists in {}." + .format(column_name, self.selected_table) + ) + return + print( + 'Do you really want to permanently delete the column {!r} from {!r}?' + .format(column_name, self.selected_table) + ) + print('This cannot be undone.') + answer = input('Continue [yes/no] ? ') + if answer == 'yes': + if self.con.do(EMDBTable.remove_column, self.selected_table, column_name): + self.con.save_database() + print("Done.") + else: + print("An error occured.") + elif answer == "no": + # Do nothing. + pass + else: + print('Answer was not "yes". No data removed.') + + def complete_remove_column(self, text, line, begidx, endidx): + return self.column_name_completion(self.selected_table, text) + + def help_remove_column(self): + print("""> remove_column + Remove the specified column from the selected table. + + The user is asked for confirmation before the command is executed. + + Warning: If you want to run more experiments of the selected class + after removing a column from the table, you must also remove the + corresponding parameter in the experiment file.""" + ) + + ### Functionality to CLEAN up + def do_remove(self, params): + # Check conditions and parameters + if not self.selected_table: + print('No experiment selected. Select an experiment to remove entries from it.') + return + if not params: + print('No IDs given. Specify at least one ID to remove its entry.') + return + # Parse parameters + ids = self.parse_multiple_ids(params.split()) + if ids is None: + print("No data removed.") + return + + # Print full information of selected entries + global display_all + display_all = True + if len(ids) == 1: + statement = "id = {}".format(ids[0]) + else: + statement = "id IN {}".format(tuple(ids)) + print('WARNING: the following entries will be DELETED:') + print("-"*50) + self.con.do(EMDBTable.print_selection, self.selected_table, statement) + print("-"*50) + + # Print full information of related folders + folders = [] + print('WARNING: the following folders and files will be DELETED:') + for id_ in ids: + expname = "{}_{:03}".format(self.selected_table, id_) + folder = os.path.join(self.exp_dir, expname) + try: + files = os.listdir(folder) + except FileNotFoundError: + print(' x Folder does not exist:', folder) + else: + folders.append(folder) + print(" -", folder) + for f in sorted(files): + print(" -", f) + + # Final check + print('Do you really want to permanently delete these files, folders, and entries?') + print('This cannot be undone.') + answer = input('Continue [yes/no] ? ') + if answer == 'yes': + # Remove entries + for id_ in ids: + self.con.do(EMDBTable.delete_entry, self.selected_table, id_) + print('Deleted entry', id_, 'from experiment "{}".'.format(self.selected_table)) + self.con.save_database() + # Remove files + for folder in folders: + try: + shutil.rmtree(folder) + except OSError as e: + print('Error deleting folder {}:'.format(folder), e) + else: + print('Deleted folder {}.'.format(folder)) + elif answer == "no": + # Do nothing. + pass + else: + print('Answer was not "yes". No data removed.') + + def help_remove(self): + print("""> remove + Delete the entry with the given ID from the currently selected class of + experiments and all files associated with it. Multiple IDs can be specified + to remove several entries and their folders at once. Instead of an ID, the + argument "last" can be used to choose the latest entry. + Before the data is deleted, the user is asked to confirm, which must be + answered with "yes". + The remove an empty class of experiments, use "remove_selected_class".""") + + def do_remove_selected_class(self, params): + global display_all + display_all = True + + # Check conditions and parameters + if not self.selected_table: + print('No experiment selected. Select an experiment to remove it.') + return + if params: + print('This command takes no attributes. Cancelling.') + return + + if self.con.do(len, self.selected_table) > 0: + print('Selected experiment class contains experiments. Remove these ' + 'experiments first before removing the class. Cancelling.') + return + + print('Do you really want to permanently delete the experiment class ' + f'"{self.selected_table}"?') + print('This cannot be undone.') + print('The EMShell will exit after the deletion.') + answer = input('Continue [yes/no] ? ') + if answer == 'yes': + self.con.delete_table(self.selected_table) + self.con.save_database() + print(f'Deleted experiment class "{self.selected_table}".') + print(f'EMShell must be restarted to update the list of tables.') + return True + else: + print('Answer was not "yes". No data removed.') + + def help_remove_selected_class(self): + print("""> remove_selected_class + Delete the currently selected class of experiments from the database. + This works only if no entries are associated with this experiment class. + Before the class is removed, the user is asked to confirm, which must be + answered with "yes". To remove entries from the selected class, use the + command "remove".""") + + def do_print_disk_info(self, params): + # Explanation: https://stackoverflow.com/a/12327880/3661532 + statvfs = os.statvfs(self.exp_dir) + available_space = statvfs.f_frsize * statvfs.f_bavail + print( + "Available disk space in the experiment directory:", + SIZE_FORMAT.format(available_space / 1024**3), + "GiB" + ) + print("Experiment directory:", self.exp_dir) + + def help_print_disk_info(self): + print("""> print_disk_info + Display the available disk space in the experiment directory and its path. + The available disk space is printed in Gibibytes (GiB), this means: + 1 GiB = 1024^3 bytes. + This is used instead of the Gigabyte (GB), which is defined as: + 1 GB = 1000^3 bytes, + because the GiB underestimates the free disk space, which is considered + safer than overestimating it. In contrast, the size used by experiments is + displayed in Megabytes (MB), where + 1 MB = 1000^2 bytes, + since the used space should rather be overestimated. + More information about the unit: https://en.wikipedia.org/wiki/Gibibyte .""" + ) + + ### Functionality to SELECT a specific table + def do_select(self, params): + if params == "": + self.prompt = "(EMS) " + self.selected_table = params + elif self.con.is_valid_table(params): + self.prompt = "({}) ".format(params) + self.selected_table = params + else: + print('Unknown experiment: "{}"'.format(params)) + + def complete_select(self, text, line, begidx, endidx): + return self.table_name_completion(text) + + def help_select(self): + print( + 'Select an experiment by specifing its name.\n' + 'When an experiment is selected, every operation is automatically ' + 'executed for this experiment, except if specified differently.\n' + 'To unselect again, use the "select"-command with no argument or press Ctrl+D.' + ) + + ### Functionality to QUIT the program + def do_exit(self, params): + return True + + def do_EOF(self, params): + """This is called when Ctrl+D is pressed. + If an experiment is selected, it will be deselected. + If no experiment is selected, the programme will exit. + """ + if self.selected_table: + print("select") + self.prompt = "(EMS) " + self.selected_table = "" + else: + print("exit") + return True + + ### Behaviour for empty input + def emptyline(self): + pass + + ### Helper functions + def table_name_completion(self, text): + return [table.name for table in self.con.tables if table.name.startswith(text)] + + def plot_attribute_completion(self, text, parameters): + if not self.selected_table: + print("\nError: select an experiment first!") + return [] + parameters += self.con.do(EMDBTable.get_column_names, self.selected_table) + parameters.extend(extra_tools.keys()) + return [p for p in parameters if p.startswith(text)] + + def column_name_completion(self, table_name, text): + columns = self.con.do(EMDBTable.get_column_names, table_name) + return [c for c in columns if c.startswith(text)] + + def parse_params_to_experiment(self, params: str) -> [str, int]: + """Parse and check input of the form "[experiment] ". + + The argument "experiment" is optional, the ID is necessary. + Instead of an ID, "last" can be used to refer to the latest entry. + If no experiment is given, the selected experiment is taken. + Return None and print a message if input is not valid, otherwise + return the experiment name and the ID.""" + if not params: + print("No ID given.") + return None + params = params.split(" ") + # Parse the ID + specifier = params.pop() + if specifier == "last": + id_ = "last" + else: + try: + id_ = int(specifier) + except ValueError: + print(f'Last argument is not a valid ID: "{specifier}".') + return None + # Parse and check the table name + table_name = " ".join(params).strip() + if table_name: + if not self.con.is_valid_table(table_name): + print(f'No experiment class with the name "{table_name}" exists.') + return None + else: + if not self.selected_table: + print("No experiment selected and no experiment name given.") + return None + table_name = self.selected_table + # Check the ID + if id_ == "last": + id_ = self.con.do(EMDBTable.get_latest_entry, table_name) + if id_ is None: + return None + elif not self.con.do(EMDBTable.entry_exists, table_name, id_): + print(f'No entry with ID {id_} exists for the experiment class "{table_name}".') + return None + return table_name, id_ + + @staticmethod + def parse_plot_parameters(params, n_necessary): + parameters = { + "variables": [], + "condition": "", + "save_as": None, + "grid": False, + "xmin": None, + "xmax": None, + "ymin": None, + "ymax": None, + "zmin": None, + "zmax": None, + "cmap": None, + "shading": "flat", + "format_string": "", + } + param_list = params.split(" ") + # Necessary parameters are at the beginning + if len(param_list) < n_necessary: + print(f'Not enough parameters given, {n_necessary} are necessary.') + return None + for i in range(n_necessary): + parameters["variables"].append(param_list.pop(0)) + # The last parameters can be used to modify the plot + for param in reversed(param_list): + if not param: + # Ignore empty strings + pass + elif param in ["-png", "-pdf", "-svg"]: + parameters["save_as"] = param[1:] + elif param == "-grid": + parameters["grid"] = True + elif param == "-shading": + parameters["shading"] = "gouraud" + elif param.startswith("-f="): + parameters["format_string"] = param[3:] + elif param.startswith("-cmap="): + parameters["cmap"] = param[6:] + elif xyzminmax.match(param): + param_name = param[1:5] + param_value = param[6:] + try: + parameters[param_name] = float(param_value) + except ValueError: + print(f'Value for {param_name} cannot be converted to ' + f'float: "{param_value}".') + else: + # End of extra parameters, beginning of the SQL statement + break + param_list.pop() + # Every other parameter is treated as an SQL condition to filter the data + parameters["condition"] = " ".join(param_list).strip() + return parameters + + def parse_multiple_ids(self, param_list): + """Transform the given list of strings into a list of IDs. + + This method also checks that the IDs are valid entries of the + selected database and returns None if an invalid ID is given. + Otherwise, a sorted list of unique values is returned. + It parses "last" as the latest entry.""" + ids = [] + for id_ in param_list: + if id_ == "last": + id_ = self.con.do(EMDBTable.get_latest_entry, self.selected_table) + if id_ is None: + return + if id_ in ids: + continue + else: + try: + id_ = int(id_) + except ValueError: + print('Parameter', id_, 'is not a valid ID.') + return + if id_ in ids: + continue + if not self.con.do(EMDBTable.entry_exists, self.selected_table, id_): + print('No entry with ID', id_, 'exists for the selected experiment.') + return + ids.append(id_) + return sorted(ids) + + def open_file(self, command, path, verbose=False): + if verbose: + print("Opening file {} with {} in verbose-mode.".format(path, command)) + subprocess.Popen( + [command, path], + # Disable standard input via the shell, for example with mplayer. + stdin=subprocess.DEVNULL, + ) + else: + print("Opening file {} with {} silently.".format(path, command)) + subprocess.Popen( + [command, path], + # Disable standard input via the shell, for example with mplayer. + stdin=subprocess.DEVNULL, + # Throw away output and error messages. + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + + def get_multiple_data(self, table_name, variables, condition): + datas = [] + for var in variables: + data = self.get_data(table_name, var, condition) + if not data: + print(f'Get data for parameter "{var}" ', end="") + if condition: + print(f'under condition "{condition}" ', end="") + print('failed.') + return None + datas.append(data) + return datas + + def get_data(self, table_name, parameter, condition): + if parameter in self.con.do(EMDBTable.get_column_names, table_name): + return self.con.do(EMDBTable.get_data, table_name, parameter, condition) + elif parameter in extra_tools: + data = [] + ids = self.con.do(EMDBTable.get_data, table_name, "id", condition) + print(f'Calculating data with tool "{parameter}" for {len(ids)} experiments.') + tstart = time.time() + for id_ in ids: + expname = "{}_{:03}".format(table_name, id_) + path = os.path.join(self.exp_dir, expname, expname + '_his.nc') + try: + val = extra_tools[parameter](path) + except Exception as e: + print(f'Tool "{parameter}" did not succeed on experiment {id_}. ' + 'The error message is:') + print(e) + print('Using value 0 (zero) as fallback.') + val = 0 + data.append(val) + print('Calculation finished in {:.3f} seconds.'.format(time.time() - tstart)) + return data + else: + print(f'Parameter "{parameter}" is neither a parameter of the ' + 'selected experiment nor a loaded extension.') + return [] + + @staticmethod + def print_general_plot_help(plot_type, shading=False): + print(""" + The variables can be read from the database or calculated using shell- + extensions. Type "help calculate" for further information on extensions. + + A condition can be used to filter or sort the data. Type "help filter" for + further information on filtering and sorting. + + To draw a grid on the plot, add "-grid" to the command.""" + ) + if shading: + print(""" + To make a smooth plot instead of rectangles of solid colour, add "-shading" + to the command. This enables the Gouraud-shading of matplotlib.""" + ) + if plot_type == "2d": + print(""" + To specify the type of plot, use "-f=", where is a format + string for matplotlib like "-f=o" to plot with dots instead of a line, or + "-f=rx" to plot with red crosses. For further information on format strings + in matplotlib, see the Notes section on the following website: + https://matplotlib.org/3.1.0/api/_as_gen/matplotlib.pyplot.plot.html + + To specify the range in x- or y-direction, use the following attributes: + -xmin=, -xmax=, -ymin=, -ymax= + with a number as the argument .""" + ) + elif plot_type == "3d": + print(""" + To specify the colour map of the plot, use "-cmap=", where + is the name of a colour map for matplotlib, like "-cmap=bwr" + to plot in blue-white-red or "-cmap=iridescent" to use Paul Tol's + beautiful colour scheme that also works in colour-blind vision. + For further inspiration, consider the following website: + https://matplotlib.org/users/colormaps.html + + To specify the range in x-, y- or z-direction, use the following attributes: + -xmin=, -xmax=, -ymin=, -ymax=, -zmin=, -zmax= + with a number as the argument . The z-axis refers to the colourbar.""" + ) + print(""" + Add either "-png" or "-pdf" or "-svg" to the command to save the figure in + a file of the corresponding filetype. Alternatively, use the command + "save_figure" after creating the plot. + + The order of the qualifiers starting with a dash ("-") does not play a role, + but they must be at the end of the command, i.e., after the variables and + after the filter condition (if any). If a qualifier is given more than + once, only the first occurence is taken into account. + + While the window with the plot is open, the shell can be used as usual. + If another plot command is executed with the figure still open, the new plot + is superimposed onto the previous one. To draw the next plot command in a + new figure instead, while keeping the previous window open, use the command + "new_figure".""" + ) + + @staticmethod + def print_param_parser_help(): + print(""" + If only an ID and no experiment name is given, take the currently selected + class of experiments. Instead of an ID, the value "last" can be used to + choose the latest entry.""" + ) + + +def get_unique_save_filename(template): + id_ = 0 + filename = template.format(id_) + while os.path.exists(filename): + id_ += 1 + filename = template.format(id_) + print(f'Saving as "{filename}".') + return filename + + +def string_format_table(table_name, columns, rows, highlight_columns=None): + # Get current date and time to make datetime easy to read + if not ISO_DATETIME: + dt_now = datetime.datetime.now() + # Convert rows to list of lists (because fetchall() returns a list of tuples + # and to work on a copy of it) + rows = [list(row) for row in rows] + # Remove ignored columns + columns = columns.copy() + indices_to_remove = [] + if not display_all: + for i, (n, t) in enumerate(columns): + if n in hidden_information: + indices_to_remove.append(i) + for i in sorted(indices_to_remove, reverse=True): + columns.pop(i) + for row in rows: + row.pop(i) + # Get width necessary for each column, get total size of the files + # associated with the table and process the entries of the table. + lengths = [len(n) for n, t in columns] + table_size = 0 + for row in rows: + for i, val in enumerate(row): + # Check name of column + if columns[i][0] == "comment": + if display_all or COMMENT_MAX_LENGTH == "FULL": + # No formatting needed + pass + else: + # Replace linebreaks + val = val.replace(*LINEBREAK_REPLACE) + if type(COMMENT_MAX_LENGTH) is int and COMMENT_MAX_LENGTH > 0: + # Cut comments which are too long and end them with an ellipsis + if len(val) > COMMENT_MAX_LENGTH: + val = val[:COMMENT_MAX_LENGTH-1] + "…" + elif COMMENT_MAX_LENGTH == "AUTO": + # Cut comments later + pass + else: + raise ValueError( + 'COMMENT_MAX_LENGTH has to be "AUTO" or "FULL" ' + 'or a positive integer, not "{}".'.format(COMMENT_MAX_LENGTH) + ) + row[i] = val + elif columns[i][0] == "datetime": + if display_all: + # No formatting needed + pass + elif ISO_DATETIME: + # Cut unnecessary parts of the date and time + if "datetime_year" in hidden_information: + val = val[5:] + if "datetime_seconds" in hidden_information: + val = val.split(".")[0][:-3] + val = val.replace('T', ',') + else: + # Create datetime-object from ISO-format + dt_obj = datetime.datetime.strptime(val, "%Y-%m-%dT%H:%M:%S.%f") + val = make_nice_time_string(dt_obj, dt_now) + row[i] = val + elif columns[i][0] == "size_total": + if val > 0: + table_size += val + # Check type of column and adopt + if columns[i][1] == "TEXT" or columns[i][1] == "INTEGER": + lengths[i] = max(lengths[i], len(str(val))) + elif columns[i][0] in ["size_diag", "size_flux", "size_his", + "size_mp4", "size_total"]: + lengths[i] = max(lengths[i], len(SIZE_FORMAT.format(val))) + elif columns[i][1] == "REAL": + lengths[i] = max(lengths[i], len(FLOAT_FORMAT.format(val))) + else: + # This is an unexpected situation, which probably means that + # sqlite3 does not work as it was, when this script was written. + raise Exception( + "unknown type {} of column {} in table {}." + .format(*columns[i], table_name) + ) + if ( + COMMENT_MAX_LENGTH == "AUTO" + and not display_all + and "comment" == columns[-1][0] + ): + line_length = shutil.get_terminal_size().columns + total_length = sum(lengths[:-1]) + len(LIMITER) * len(lengths[:-1]) + comment_length = line_length - total_length % line_length + lengths[-1] = comment_length + for row in rows: + comment = row[-1] + # Cut comments which are too long and end them with an ellipsis + if len(comment) > comment_length: + comment = comment[:comment_length-1] + "…" + row[-1] = comment + # Create top line of the text to be returned + text = "Experiment: " + table_name + if "size_total" not in hidden_information or display_all: + text += " (" + SIZE_FORMAT.format(table_size) + " MB)" + if len(indices_to_remove) == 1: + text += " (1 parameter hidden)" + elif len(indices_to_remove) > 1: + text += " ({} parameters hidden)".format(len(indices_to_remove)) + text += "\n" + column_names_formatted = [] + format_strings = [] + for (n, t), l in zip(columns, lengths): + # Column names centred, except comment, since it is the last column. + cname_form = f"{n:^{l}}" + # Numbers right justified, + # comments left justified, + # other texts centred. + if n in ["size_diag", "size_flux", "size_his", "size_mp4", "size_total"]: + f_string = SIZE_FORMAT.replace(":", ":>"+str(l)) + elif t == "REAL": + f_string = FLOAT_FORMAT.replace(":", ":>"+str(l)) + elif t == "INTEGER": + f_string = "{:>" + str(l) + "}" + elif n == "comment": + f_string = "{:" + str(l) + "}" + cname_form = n + else: + f_string = "{:^" + str(l) + "}" + if highlight_columns is not None and n in highlight_columns: + f_string = COLOURS_HIGHLIGHT + f_string + COLOURS_END + cname_form = COLOURS_HIGHLIGHT + cname_form + COLOURS_END + format_strings.append(f_string) + column_names_formatted.append(cname_form) + text += LIMITER.join(column_names_formatted) + "\n" + if COLOURS: + row_colours = itertools.cycle(COLOURS) + for row in rows: + if COLOURS: + col_colours = itertools.cycle(next(row_colours)) + text_cols = [next(col_colours) + f_str.format(val) if COLOURS + else f_str.format(val) for f_str, val in zip(format_strings, row)] + text += LIMITER.join(text_cols) + text += "\n" + if text.endswith("\n"): + text = text[:-1] + if COLOURS: + text = text + COLOURS_END + return text + + +def make_nice_time_string(datetime_object, datetime_reference): + """Create an easy to read representation of the datetime object.""" + if datetime_object > datetime_reference: + return LANG.FUTURE + elif datetime_object.date() == datetime_reference.date(): + # same day + dt_diff = datetime_reference - datetime_object + dt_diff_minutes = dt_diff.seconds / 60 + if dt_diff_minutes < 1: + return LANG.JUST_NOW + elif dt_diff_minutes < 60: + return LANG.AGO_MINUTES.format(int(dt_diff_minutes)) + else: + return LANG.AGO_HOURS.format(int(dt_diff_minutes / 60), int(dt_diff_minutes % 60)) + elif datetime_object.date() == (datetime_reference - datetime.timedelta(days=1)).date(): + # yesterday + return datetime_object.strftime(LANG.YESTERDAY + ", %H:%M") + elif datetime_object.date() > (datetime_reference - datetime.timedelta(days=7)).date(): + # this week, i.e., within the last six days + return datetime_object.strftime("%a, %H:%M") + else: + format_string = "%b-%d, %H:%M" + if "datetime_year" not in hidden_information: + format_string = "%Y-" + format_string + return datetime_object.strftime(format_string) + + +# Get the directory of the experiments +if len(sys.argv) == 1: + try: + from param import Param + except ModuleNotFoundError: + raise Exception( + "Please activate fluid2d or specify the experiment-folder as " + "argument when starting this programme." + ) + param = Param(None) # it is not necessary to specify a defaultfile for Param + datadir = param.datadir + del param + if datadir.startswith("~"): + datadir = os.path.expanduser(datadir) +elif len(sys.argv) == 2: + datadir = sys.argv[1] +else: + raise Exception("More than one argument given.") + +# Use fancy colours during 6 days of Carnival +try: + from dateutil.easter import easter +except ModuleNotFoundError: + # No fun at carnival possible + pass +else: + date_today = datetime.date.today() + carnival_start = easter(date_today.year)-datetime.timedelta(days=46+6) # Weiberfastnacht + carnival_end = easter(date_today.year)-datetime.timedelta(days=46) # Aschermittwoch + if carnival_start <= date_today < carnival_end or COLOURS == "HAPPY": + print("It's carnival! Let's hope your terminal supports colours!") + # This looks like a rainbow on many terminals, e.g. xterm. + COLOURS = ( + ("\033[41m",), + ("\033[101m",), + ("\033[43m",), + ("\033[103m",), + ("\033[102m",), + ("\033[106m",), + ("\033[104m",), + ("\033[105m",), + ) + +# Activate interactive plotting +if matplotlib: + plt.ion() + +# Compile regular expression to match parameters +# This matches -xmin=, -xmax=, -ymin=, -ymax=, -zmin=, -zmax= +xyzminmax = re.compile("-[xyz]m(in|ax)=") + +# Start the shell +ems_cli = EMShell(datadir) +ems_cli.cmdloop() diff --git a/experiments/Waves with EMS/wave_breaking.exp b/experiments/Waves with EMS/wave_breaking.exp new file mode 100644 index 0000000..b3da53e --- /dev/null +++ b/experiments/Waves with EMS/wave_breaking.exp @@ -0,0 +1,55 @@ +# Fluid2d-experiment with EMS: wave breaking at the coast (experiment file) +# +# Refer to the documentation of `ems.parse_experiment_file` for a +# detailed description of the experiment file format. +# +# Author: Markus Reinert, May/June 2019 + + +# Name of the experiment class. +# If the name is changed, a new table is created in the database. +Name: Breaking_Waves + + +# ID of the experiment within its class. +# Usually, an ID should NOT be specified, so that the EMS chooses automatically a unique ID. +# It is only necessary to specify an ID explicitly if fluid2d runs on multiple cores, for +# example with mpirun. Furthermore, it is possible to overwrite existing entries in the +# database by specifying their ID. +# The ID is automatically increased by 1 for every new combination if multiple values are +# given for one or several parameters. +# Uncomment the following line to set an ID manually: +#ID: 21 + + +# Descriptions are optional. +# They can go over multiple lines and can also contain empty lines. +# Empty lines at the beginning and at the end of the description are ignored. +Description: Simulate waves on the interface of two fluids. +Modify the parameters in this file and observe the creation, propagation and breaking of waves at a sloping coast. + + +# Here are the parameters that we want to modify in this class of experiments. +# Parameters which we intend to keep constant are in the Python file. +# For every parameter, at least one value has to be specified. Multiple values can be +# specified to automatically run one experiment for every combination of the given values. +Parameters: + +### Physics +# activate or deactivate diffusion +diffusion False +# diffusion coefficient (if diffusion is True) +Kdiff 1e-2 + +### Coast +slope 0.5 +height 1.05 +# beginning of the flat coast +x_start 4.0 + +### Initial Condition +# perturbation can be of type "sin" or "gauss" +perturbation gauss +# height and width of the perturbation +intensity 0.5 +sigma 0.4 0.8 diff --git a/experiments/Waves with EMS/wave_breaking.py b/experiments/Waves with EMS/wave_breaking.py new file mode 100644 index 0000000..108568d --- /dev/null +++ b/experiments/Waves with EMS/wave_breaking.py @@ -0,0 +1,158 @@ +"""Fluid2d-experiment with EMS: wave breaking at the coast (Python file) + +This file, together with its corresponding experiment file, aims to be +both, an interesting experiment and a helpful introduction to the +Experiment Management System (EMS) of fluid2d. + +Compared to a simple fluid2d experiment without EMS, three changes are +necessary to profit from the full functionality of the EMS: + + 1. Create Param by specifying the path to the experiment file. + 2. Fetch and use the experiment parameters from Param. + 3. Call the finalize-method of Param at the end. + +Furthermore, the variable `param.expname` should not be set in the +Python file, since the EMS takes care of setting it. In this way, the +loss of old output files by accidentally replacing them is avoided. +Nevertheless, it is possible to explicitly ask the EMS to replace old +experiments. See the experiment file for more details. + +When the Experiment Management System is activated like this, an entry +in the experiment-database is created. The database is stored in the +same directory as the output of the experiments, which is defined in +`param.datadir`. Every entry contains the unique ID of the experiment, +the date and time of the run, the last point in time of the integration, +the sizes of the output files in MB, a comment or description and +-- most importantly -- the parameters that are chosen by the user to +keep track off. These parameters are defined in the experiment file. +After the EMS is set-up, only the experiment file needs to be changed by +the user. Thanks to the EMS, it is not necessary to modify the Python +file more than once at the beginning to set it up. + +Author: Markus Reinert, April/May/June 2019 +""" + +from fluid2d import Fluid2d +from param import Param +from grid import Grid + +import numpy as np + + +### STEP 1 +# Load default parameter values, load specific parameters from the given +# experiment file and set-up the EMS. +param = Param(ems_file="wave_breaking.exp") + +# Do not set `param.expname` because it will be ignored. + +### STEP 2 +# There are two ways to get the dictionary with the experiment parameters. +# (a) The quick and easy way (using "get"): +# To run exactly one experiment with only the values specified at first, use +# EP = param.get_experiment_parameters() +# In this case, it is not necessary to indent the whole file as below. +# (b) The recommended way (using "loop"): +# To run one experiment for every possible combination of all the values +# specified in the experiment file, use +# for EP in param.loop_experiment_parameters(): +# and put the rest of the Python file within this loop by indenting every line. +# The behaviour of both ways is the same if only one value is given for every +# parameter in the experiment file. Therefore it is recommended to use the +# multi-run implementation, since it is more versatile. +# +# In both cases, the dictionary EP is now available. Its keys are the names +# of the parameters in the experiment file. Every key refers to the +# corresponding value. If multiple values are given in the experiment file, +# in every iteration of the loop, another combination of these values is in EP. +# Within the following lines of Python code, the values of the EP are used to +# implement the desired behaviour of the experiment. +for EP in param.loop_experiment_parameters(): + # Set model type, domain type, size and resolution + param.modelname = "boussinesq" + param.geometry = "closed" + param.ny = 2 * 64 + param.nx = 3 * param.ny + param.Ly = 2 + param.Lx = 3 * param.Ly + # Set number of CPU cores used + # Remember to set a fixed (initial) ID in the experiment file, if multiple + # cores are used. + param.npx = 1 + param.npy = 1 + + # Use a fixed time stepping for a constant framerate in the mp4-file + param.tend = 20.0 + param.adaptable_dt = False + param.dt = 0.02 + + # Choose discretization + param.order = 5 + + # Set output settings + param.var_to_save = ["vorticity", "buoyancy", "psi"] + param.list_diag = "all" + param.freq_his = 0.2 + param.freq_diag = 0.1 + + # Set plot settings + param.plot_var = "buoyancy" + param.plot_interactive = True + param.generate_mp4 = True + param.freq_plot = 10 + param.colorscheme = "imposed" + param.cax = [0, 1] + param.cmap = "Blues_r" # reversed blue colour axis + + # Configure physics + param.gravity = 1.0 + param.forcing = False + param.noslip = False + param.diffusion = EP["diffusion"] + param.Kdiff = EP["Kdiff"] * param.Lx / param.nx + + # Initialize geometry + grid = Grid(param) + xr, yr = grid.xr, grid.yr + + LAND = 0 + AQUA = 1 + + # Add linear sloping shore + m = EP["slope"] + t = EP["height"] - EP["x_start"] * m + grid.msk[(yr <= m*xr + t) & (yr < EP["height"])] = LAND + grid.finalize_msk() + + # Create model + f2d = Fluid2d(param, grid) + model = f2d.model + + # Set initial perturbation of the interface + buoy = model.var.get("buoyancy") + buoy[:] = 1 + if EP["perturbation"].lower() == "sin": + # Use a sinusoidal perturbation + buoy[ + (yr < EP["intensity"] * np.sin(2 * np.pi * xr/EP["sigma"]) + 1.0) + & (xr < EP["x_start"]) + ] = 0 + elif EP["perturbation"].lower() == "gauss": + # Use a Gaussian perturbation + buoy[ + (yr < EP["intensity"] * np.exp(-(xr/EP["sigma"])**2) + 1.0) + & (xr < EP["x_start"]) + ] = 0 + else: + raise ValueError("unknown type of perturbation: {}.".format(EP["perturbation"])) + buoy *= grid.msk + + # Start simulation + f2d.loop() + + ### STEP 3 + # Finish off the EMS database entry. + # Without this call to `finalize`, the size of the output files cannot be + # saved in the database. It is important to have this line within the + # for-loop if the multi-run strategy (b) is used. + param.finalize(f2d)