From 76ec95836470fd8fb119301a25208b01abf942a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=B6nke?= Date: Tue, 29 Aug 2017 02:26:18 +0200 Subject: [PATCH 01/16] cli-lib communication draft with generators --- libiocage/cli/start.py | 5 ++++- libiocage/lib/Jail.py | 23 +++++++++++++++++--- libiocage/lib/events.py | 47 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 71 insertions(+), 4 deletions(-) create mode 100644 libiocage/lib/events.py diff --git a/libiocage/cli/start.py b/libiocage/cli/start.py index 2f8a95c8..59b19c73 100644 --- a/libiocage/cli/start.py +++ b/libiocage/cli/start.py @@ -49,8 +49,11 @@ def cli(ctx, rc, jails, log_level): for jail in ioc_jails.list(filters=jails): logger.log(f"Starting {jail.humanreadable_name}") try: - jail.start() + for i in jail.start(yields=True): + print(f"[+] {i.action} OK") + except Exception: + raise exit(1) logger.log(f"{jail.humanreadable_name} running as JID {jail.jid}") diff --git a/libiocage/lib/Jail.py b/libiocage/lib/Jail.py index 716f74ca..34588b02 100644 --- a/libiocage/lib/Jail.py +++ b/libiocage/lib/Jail.py @@ -14,6 +14,7 @@ import libiocage.lib.ZFSBasejailStorage import libiocage.lib.ZFSShareStorage import libiocage.lib.errors +import libiocage.lib.events import libiocage.lib.helpers @@ -133,7 +134,7 @@ def rc_conf(self): ) return self._rc_conf - def start(self): + def start(self, yields=False): """ Start the jail. """ @@ -158,9 +159,14 @@ def start(self): self.config.fstab.save_with_basedirs() self._launch_jail() + if yields is True: + yield libiocage.lib.events.JailStarted(jail=self) + if self.config["vnet"]: self._start_vimage_network() self._configure_routes() + if yields is True: + yield libiocage.lib.events.JailVnetConfigured(jail=self) self._configure_nameserver() @@ -168,6 +174,18 @@ def start(self): libiocage.lib.ZFSShareStorage.ZFSShareStorage.mount_zfs_shares( self.storage ) + if yields is True: + yield libiocage.lib.events.JailZfsSharesMounted(jail=self) + + if self.config["exec_start"] is not None: + self._start_services() + if yields is True: + yield libiocage.lib.events.JailServicesStarted(jail=self) + + def _start_services(self): + command = self.config["exec_start"].strip().split() + self.logger.debug(f"Running exec_start on {self.humanreadable_name}") + self.exec(command) def stop(self, force=False): """ @@ -472,7 +490,6 @@ def _launch_jail(self): f"exec.prestart={self.config['exec_prestart']}", f"exec.poststart={self.config['exec_poststart']}", f"exec.prestop={self.config['exec_prestop']}", - f"exec.start={self.config['exec_start']}", f"exec.stop={self.config['exec_stop']}", f"exec.clean={self.config['exec_clean']}", f"exec.timeout={self.config['exec_timeout']}", @@ -511,7 +528,7 @@ def _launch_jail(self): def _start_vimage_network(self): - self.logger.log("Starting VNET/VIMAGE", jail=self) + self.logger.debug("Starting VNET/VIMAGE", jail=self) nics = self.config["interfaces"] for nic in nics: diff --git a/libiocage/lib/events.py b/libiocage/lib/events.py new file mode 100644 index 00000000..ea783409 --- /dev/null +++ b/libiocage/lib/events.py @@ -0,0 +1,47 @@ +EVENT_STATUS = ( + "pending", + "done", + "failed" +) + + +class Iocage: + """ + IocageEvent + + Base class for all other iocage events + """ + + def __init__(self, action, **kwargs): + self.action = action + self.data = kwargs + + +class Jail(Iocage): + + def __init__(self, action, jail, **kwargs): + super().__init__(action=action, jail=jail, **kwargs) + + +class JailStarted(Jail): + + def __init__(self, jail, **kwargs): + super().__init__("Started", jail, **kwargs) + + +class JailVnetConfigured(Jail): + + def __init__(self, jail, **kwargs): + super().__init__("Configuring VNET", jail, **kwargs) + + +class JailZfsShareMounted(Jail): + + def __init__(self, jail, **kwargs): + super().__init__("Mounting ZFS shares", jail, **kwargs) + + +class JailServicesStarted(Jail): + + def __init__(self, jail, **kwargs): + super().__init__("Starting services", jail, **kwargs) From 7bd5d1e3c7969755d9b23df1daaaffded79ace61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=B6nke?= Date: Mon, 28 Aug 2017 22:13:06 +0200 Subject: [PATCH 02/16] follow generator event notification approach --- libiocage/cli/fetch.py | 43 ++----- libiocage/cli/start.py | 4 +- libiocage/lib/Jail.py | 59 ++++++++-- libiocage/lib/Logger.py | 150 +++++++++++++++++------- libiocage/lib/Release.py | 180 ++++++++++++++++++++++------- libiocage/lib/errors.py | 15 +++ libiocage/lib/events.py | 242 ++++++++++++++++++++++++++++++++++++--- libiocage/lib/helpers.py | 49 ++++++++ 8 files changed, 599 insertions(+), 143 deletions(-) diff --git a/libiocage/cli/fetch.py b/libiocage/cli/fetch.py index 1ee8d34c..caba5fed 100644 --- a/libiocage/cli/fetch.py +++ b/libiocage/cli/fetch.py @@ -25,11 +25,11 @@ import click import libiocage.lib.Host -import libiocage.lib.Logger import libiocage.lib.Prompts import libiocage.lib.Release import libiocage.lib.errors + __rootcmd__ = True @@ -59,8 +59,7 @@ # type=release_choice(), help="The FreeBSD release to fetch.") @click.option("--update/--no-update", "-U/-NU", default=True, - help="Decide whether or not to update the fetch to the latest " - "patch level.") + help="Update the release to the latest patch level.") @click.option("--fetch-updates/--no-fetch-updates", default=True, help="Skip fetching release updates") # Compat @@ -71,18 +70,15 @@ help="Specify the files to fetch from the mirror. " "(Deprecared: renamed to --file)") @click.option("--log-level", "-d", default=None) -# @click.option("--auth", "-a", default=None, help="Authentication method for " -# "HTTP fetching. Valid " -# "values: basic, digest") -# @click.option("--verify/--noverify", "-V/-NV", default=True, -# help="Enable or disable verifying SSL cert for HTTP fetching.") -# def cli(url, files, release, update): def cli(ctx, **kwargs): logger = ctx.parent.logger logger.print_level = kwargs["log_level"] host = libiocage.lib.Host.Host(logger=logger) prompts = libiocage.lib.Prompts.Prompts(host=host, logger=logger) + if kwargs["log_level"] is not None: + logger.print_level = kwargs["log_level"] + release_input = kwargs["release"] if release_input is None: try: @@ -100,9 +96,6 @@ def cli(ctx, **kwargs): logger.error(f"Invalid Release '{release_input}'") exit(1) - if kwargs["log_level"] is not None: - logger.print_level = kwargs["log_level"] - url_or_files_selected = False if is_option_enabled(kwargs, "url"): @@ -117,28 +110,12 @@ def cli(ctx, **kwargs): logger.error(f"The release '{release.name}' is not available") exit(1) - if release.fetched: - msg = f"Release '{release.name}' is already fetched" - if kwargs["update"] is True: - logger.log(f"{msg} - updating only") - else: - logger.log(f"{msg} - skipping download and updates") - exit(0) - else: - logger.log( - f"Fetching release '{release.name}' from '{release.mirror_url}'" - ) - release.fetch(update=False, fetch_updates=False) - - if kwargs["fetch_updates"] is True: - logger.log("Fetching updates") - release.fetch_updates() - - if kwargs["update"] is True: - logger.log("Updating release") - release.update() + fetch_updates = bool(kwargs["fetch_updates"]) + release.fetch( + update=kwargs["update"], + fetch_updates=fetch_updates + ) - logger.log('done') exit(0) diff --git a/libiocage/cli/start.py b/libiocage/cli/start.py index 59b19c73..bc80ee5a 100644 --- a/libiocage/cli/start.py +++ b/libiocage/cli/start.py @@ -49,8 +49,8 @@ def cli(ctx, rc, jails, log_level): for jail in ioc_jails.list(filters=jails): logger.log(f"Starting {jail.humanreadable_name}") try: - for i in jail.start(yields=True): - print(f"[+] {i.action} OK") + for event in jail.start(yields=True): + print(event.name) except Exception: raise diff --git a/libiocage/lib/Jail.py b/libiocage/lib/Jail.py index 34588b02..ec540ab0 100644 --- a/libiocage/lib/Jail.py +++ b/libiocage/lib/Jail.py @@ -18,7 +18,7 @@ import libiocage.lib.helpers -class Jail: +class JailGenerator: """ iocage unit orchestrates a jail's configuration and manages state @@ -134,7 +134,7 @@ def rc_conf(self): ) return self._rc_conf - def start(self, yields=False): + def start(self): """ Start the jail. """ @@ -144,6 +144,14 @@ def start(self, yields=False): release = self.release + events = libiocage.lib.events + jailLaunchEvent = events.JailLaunch(jail=self) + jailVnetConfigurationEvent = events.JailVnetConfiguration(jail=self) + JailZfsShareMount = events.JailZfsShareMount(jail=self) + jailServicesStartEvent = events.JailServicesStart(jail=self) + + # Determine backend + backend = None if self.config["basejail_type"] == "zfs": @@ -155,32 +163,33 @@ def start(self, yields=False): if backend is not None: backend.apply(self.storage, release) + yield jailLaunchEvent.begin() + self.config.fstab.read_file() self.config.fstab.save_with_basedirs() self._launch_jail() - if yields is True: - yield libiocage.lib.events.JailStarted(jail=self) + yield jailLaunchEvent.end() if self.config["vnet"]: + yield jailVnetConfigurationEvent.begin() self._start_vimage_network() self._configure_routes() - if yields is True: - yield libiocage.lib.events.JailVnetConfigured(jail=self) + yield jailVnetConfigurationEvent.end() self._configure_nameserver() if self.config["jail_zfs"] is True: + yield JailZfsShareMount.begin() libiocage.lib.ZFSShareStorage.ZFSShareStorage.mount_zfs_shares( self.storage ) - if yields is True: - yield libiocage.lib.events.JailZfsSharesMounted(jail=self) + yield JailZfsShareMount.end() if self.config["exec_start"] is not None: + yield jailServicesStartEvent.begin() self._start_services() - if yields is True: - yield libiocage.lib.events.JailServicesStarted(jail=self) + yield jailServicesStartEvent.end() def _start_services(self): command = self.config["exec_start"].strip().split() @@ -202,10 +211,25 @@ def stop(self, force=False): self.require_jail_existing() self.require_jail_running() + + events = libiocage.lib.events + jailDestroyEvent = events.JailDestroy(self) + jailNetworkTeardownEvent = events.JailNetworkTeardown(self) + jailMountTeardownEvent = events.JailMountTeardown(self) + + yield jailDestroyEvent.begin() self._destroy_jail() + yield jailDestroyEvent.end() + if self.config["vnet"]: + yield jailNetworkTeardownEvent.begin() self._stop_vimage_network() + yield jailNetworkTeardownEvent.end() + + yield jailMountTeardownEvent.begin() self._teardown_mounts() + yield jailMountTeardownEvent.end() + self.update_jail_state() def destroy(self, force=False): @@ -875,3 +899,18 @@ def __dir__(self): properties.add(prop) return list(properties) + + +class Jail(JailGenerator): + + def start(self, *args, **kwargs): + libiocage.lib.helpers.print_event_generator( + super().start(*args, **kwargs), + logger=self.logger + ) + + def stop(self, *args, **kwargs): + libiocage.lib.helpers.print_event_generator( + super().stop(*args, **kwargs), + logger=self.logger + ) diff --git a/libiocage/lib/Logger.py b/libiocage/lib/Logger.py index 19d3d257..5179614c 100644 --- a/libiocage/lib/Logger.py +++ b/libiocage/lib/Logger.py @@ -1,9 +1,37 @@ import os +import sys import libiocage.lib.errors +class LogEntry: + + def __init__(self, message, level, indent=0, logger=None, **kwargs): + self.message = message + self.level = level + self.indent = indent + self.logger = logger + + for key in kwargs.keys(): + object.__setattr__(self, key, kwargs[key]) + + def edit(self, message=None, indent=None): + + if self.logger is None: + raise libiocage.lib.errors.CannotRedrawLine( + reason="No logger available" + ) + + if message is not None: + self.message = message + + if indent is not None: + self.indent = indent + + self.logger.redraw(self) + class Logger: + COLORS = ( "black", "red", @@ -17,8 +45,10 @@ class Logger: RESET_SEQ = "\033[0m" BOLD_SEQ = "\033[1m" + LINE_UP_SEQ = "\033[F" LOG_LEVEL_SETTINGS = { + "screen" : {"color": None}, "info" : {"color": None}, "notice" : {"color": "magenta"}, "verbose" : {"color": "blue"}, @@ -38,10 +68,13 @@ class Logger: "verbose", "debug", "spam", + "screen" ) INDENT_PREFIX = " " + PRINT_HISTORY = [] + def __init__(self, print_level=None, log_directory="/var/log/iocage"): self._print_level = print_level self._set_log_directory(log_directory) @@ -80,70 +113,95 @@ def log(self, *args, **kwargs): if "level" not in kwargs: kwargs["level"] = "info" - self._print(**kwargs) - # self._write(**kwargs) + log_entry = LogEntry(logger=self, **kwargs) + + if self._should_print_log_entry(log_entry): + self._print_log_entry(log_entry) + self.PRINT_HISTORY.append(log_entry) + + return log_entry + + def verbose(self, message, indent=0, **kwargs): + return self.log(message, level="verbose", indent=indent, **kwargs) + + def error(self, message, indent=0, **kwargs): + return self.log(message, level="error", indent=indent, **kwargs) - def verbose(self, message, jail=None, indent=0): - self.log( - message=message, - level="verbose", - jail=jail, - indent=indent - ) + def warn(self, message, indent=0, **kwargs): + return self.log(message, level="warn", indent=indent, **kwargs) - def error(self, - message, - jail=None, - indent=0): + def debug(self, message, indent=0, **kwargs): + return self.log(message, level="debug", indent=indent, **kwargs) - self.log(message, level="error", jail=jail, indent=indent) + def spam(self, message, indent=0, **kwargs): + return self.log(message, level="spam", indent=indent, **kwargs) - def warn(self, - message, - jail=None, - indent=0): + def screen(self, message, indent=0, **kwargs): + """ + Screen does never get printed to log files + """ + return self.log(message, level="screen", indent=indent, **kwargs) - self.log(message, level="warn", jail=jail, indent=indent) + def redraw(self, log_entry): - def debug(self, - message, - jail=None, - indent=0): + if log_entry not in self.PRINT_HISTORY: + raise libiocage.lib.errors.CannotRedrawLine( + reason="Log entry not found in history" + ) - self.log(message, level="debug", jail=jail, indent=indent) + if log_entry.level != "screen": + raise libiocage.lib.errors.CannotRedrawLine( + reason=( + "Log level 'screen' is required to redraw, " + f"but got '{self.level}'" + ) + ) - def spam(self, - message, - jail=None, - indent=0): + line_number = self.PRINT_HISTORY.index(log_entry) + delta = len(self.PRINT_HISTORY) - line_number - self.log(message, level="spam", jail=jail, indent=indent) + # ToDo: Handle redrawig of multiline entries with different line count + + output = "".join([ + #"\033[s", # SCP - Save Cursor Position + f"\033[{delta}F", # CPL - Cursor Previous Line + "\r", # CR - Carriage Return + self._indent(log_entry.message, log_entry.indent), + "\033[K", # EL - Erease in Line + "\n" * delta + #"\033[s" # RCP - Restore Cursor Position + ]) + + sys.stdout.write(output) + #print(log_entry.message) + + def _should_print_log_entry(self, log_entry): + + if log_entry.level == "screen": + return True - def _print(self, message, level, jail=None, indent=0): if self.print_level is False: - return + return False print_level = Logger.LOG_LEVELS.index(self.print_level) - if Logger.LOG_LEVELS.index(level) > print_level: - return - - try: - color = Logger.LOG_LEVEL_SETTINGS[level]["color"] - except: - color = "none" + return Logger.LOG_LEVELS.index(log_entry.level) <= print_level + def _beautify_message(self, message, level, indent=0): + color = self._get_level_color(level) message = self._indent(message, indent) message = self._colorize(message, color) - print(message) + return message + + def _print(self, message, level, indent=0): + print(self._beautify_message(message, level, indent)) + + def _print_log_entry(self, log_entry): + return self._print(log_entry.message, log_entry.level, log_entry.indent) def _indent(self, message, level): indent = Logger.INDENT_PREFIX * level return "\n".join(map(lambda x: f"{indent}{x}", message.split("\n"))) - # ToDo: support file logging - # def _write(self, message, level, jail=None): - # log_file = self._get_log_file_path(level=level, jail=jail) - def _get_log_file_path(self, level, jail=None): return self.log_directory @@ -157,6 +215,12 @@ def _create_log_directory(self): def _get_color_code(self, color_name): return Logger.COLORS.index(color_name) + 30 + def _get_level_color(self, log_level): + try: + return Logger.LOG_LEVEL_SETTINGS[log_level]["color"] + except KeyError: + return "none" + def _colorize(self, message, color_name=None): try: color_code = self._get_color_code(color_name) diff --git a/libiocage/lib/Release.py b/libiocage/lib/Release.py index 4c309295..f3a426fd 100644 --- a/libiocage/lib/Release.py +++ b/libiocage/lib/Release.py @@ -13,9 +13,10 @@ import libiocage.lib.Jail import libiocage.lib.errors import libiocage.lib.helpers +import libiocage.lib.events -class Release: +class ReleaseGenerator: DEFAULT_RC_CONF_SERVICES = { "netif" : False, "sendmail" : False, @@ -30,8 +31,6 @@ def __init__(self, name=None, zfs=None, logger=None, check_hashes=True, - auto_fetch_updates=True, - auto_update=True, eol=False): libiocage.lib.helpers.init_logger(self, logger) @@ -48,8 +47,6 @@ def __init__(self, name=None, self._root_dataset = None self.dataset = dataset self.check_hashes = check_hashes is True - self.auto_fetch_updates = auto_fetch_updates is True - self.auto_update = auto_update is True self._hbsd_release_branch = None self._assets = ["base"] @@ -263,27 +260,68 @@ def fetch(self, update=None, fetch_updates=None): release_changed = False + events = libiocage.lib.events + fetchReleaseEvent = events.FetchRelease(self) + releasePrepareStorageEvent = events.ReleasePrepareStorage(self) + releaseDownloadEvent = events.ReleaseDownload(self) + releaseExtractionEvent = events.ReleaseExtraction(self) + releaseConfigurationEvent = events.ReleaseConfiguration(self) + if not self.fetched: + yield fetchReleaseEvent.begin() + yield releasePrepareStorageEvent.begin() + self._clean_dataset() self._create_dataset() self._ensure_dataset_mounted() + + yield releasePrepareStorageEvent.end() + yield releaseDownloadEvent.begin() + self._fetch_assets() - self._extract_assets() + + yield releaseDownloadEvent.end() + yield releaseExtractionEvent.begin() + + try: + self._extract_assets() + except Exception as e: + yield releaseExtractionEvent.fail(e) + raise + + yield releaseExtractionEvent.end() + yield releaseConfigurationEvent.begin() + self._create_default_rcconf() + + yield releaseConfigurationEvent.end() + release_changed = True + + yield fetchReleaseEvent.end() + else: - self.logger.warn( + + yield fetchReleaseEvent.skip( + message="already downloaded" + ) + + self.logger.verbose( "Release was already downloaded. Skipping download." ) - fetch_updates_on = self.auto_fetch_updates and fetch_updates - if fetch_updates_on or fetch_updates: - self.fetch_updates() + if fetch_updates is True: + for event in ReleaseGenerator.fetch_updates(self): + yield event - auto_update_on = self.auto_update and update is not False - if auto_update_on or update: - release_changed = self.update() + if update is True: + for event in ReleaseGenerator.update(self): + if isinstance(event, libiocage.lib.events.IocageEvent): + yield event + else: + # the only non-IocageEvent is out return value + release_changed = event if release_changed: self._update_zfs_base() @@ -292,6 +330,10 @@ def fetch(self, update=None, fetch_updates=None): def fetch_updates(self): + events = libiocage.lib.events + releaseUpdateDownloadEvent = events.ReleaseUpdateDownload(self) + yield releaseUpdateDownloadEvent.begin() + release_updates_dir = self.release_updates_dir release_update_download_dir = f"{release_updates_dir}" @@ -335,6 +377,7 @@ def fetch_updates(self): if key == update_script_name: os.chmod(local_path, 0o755) + elif key == update_conf_name: if self.host.distribution.name == "FreeBSD": @@ -346,7 +389,7 @@ def fetch_updates(self): "Components" )) f.truncate() - f.close() + os.chmod(local_path, 0o644) self.logger.debug( @@ -362,33 +405,41 @@ def fetch_updates(self): self.logger.debug( "No pre-fetching of HardenedBSD updates required - skipping" ) - return + yield releaseUpdateDownloadEvent.skip( + message="pre-fetching not supported on HardenedBSD" + ) + + else: + self.logger.verbose(f"Fetching updates for release '{self.name}'") + libiocage.lib.helpers.exec([ + f"{self.release_updates_dir}/{update_script_name}", + "-d", + release_update_download_dir, + "-f", + f"{self.release_updates_dir}/{update_conf_name}", + "--not-running-from-cron", + "fetch" + ], logger=self.logger) - self.logger.verbose(f"Fetching updates for release '{self.name}'") - libiocage.lib.helpers.exec([ - f"{self.release_updates_dir}/{update_script_name}", - "-d", - release_update_download_dir, - "-f", - f"{self.release_updates_dir}/{update_conf_name}", - "--not-running-from-cron", - "fetch" - ], logger=self.logger) + yield releaseUpdateDownloadEvent.end() def update(self): dataset = self.dataset snapshot_name = self._append_datetime(f"{dataset.name}@pre-update") + runReleaseUpdateEvent = libiocage.lib.events.RunReleaseUpdate(self) + yield runReleaseUpdateEvent.begin() + # create snapshot before the changes dataset.snapshot(snapshot_name, recursive=True) - jail = libiocage.lib.Jail.Jail({ - "uuid" : str(uuid.uuid4()), - "basejail" : False, - "allow_mount_nullfs": "1", - "release" : self.name, - "securelevel" : "0" - }, + jail = libiocage.lib.Jail.JailGenerator({ + "uuid" : str(uuid.uuid4()), + "basejail" : False, + "allow_mount_nullfs": "1", + "release" : self.name, + "securelevel" : "0" + }, new=True, logger=self.logger, zfs=self.zfs, @@ -398,25 +449,42 @@ def update(self): jail.dataset_name = self.dataset_name changed = False + try: if self.host.distribution.name == "HardenedBSD": - changed = self._update_hbsd_jail(jail) + for event in self._update_hbsd_jail(jail): + if isinstance(event, libiocage.lib.events.IocageEvent): + yield event + else: + changed = event else: - changed = self._update_freebsd_jail(jail) - except: + for event in self._update_freebsd_jail(jail): + if isinstance(event, libiocage.lib.events.IocageEvent): + yield event + else: + changed = event + yield runReleaseUpdateEvent.end() + except Exception as e: # kill the helper jail and roll back if anything went wrong self.logger.verbose( "There was an error updating the Jail - reverting the changes" ) jail.stop(force=True) self.zfs.get_snapshot(snapshot_name).rollback(force=True) - raise + yield runReleaseUpdateEvent.fail(e) + raise e return changed def _update_hbsd_jail(self, jail): - jail.start() + events = libiocage.lib.events + executeReleaseUpdateEvent = events.ExecuteReleaseUpdate(self) + + for event in jail.start(): + yield event + + yield executeReleaseUpdateEvent.begin() update_script_path = f"{self.release_updates_dir}/hbsd-update" update_conf_path = f"{self.release_updates_dir}/hbsd-update.conf" @@ -448,13 +516,19 @@ def _update_hbsd_jail(self, jail): logger=self.logger ) - jail.stop() + yield executeReleaseUpdateEvent.end() + + for event in jail.stop(): + yield event self.logger.verbose(f"Release '{self.name}' updated") return True # ToDo: return False if nothing was updated def _update_freebsd_jail(self, jail): + events = libiocage.lib.events + executeReleaseUpdateEvent = events.ExecuteReleaseUpdate(self) + local_update_mountpoint = f"{self.root_dir}/var/db/freebsd-update" if not os.path.isdir(local_update_mountpoint): self.logger.spam( @@ -470,8 +544,10 @@ def _update_freebsd_jail(self, jail): ) jail.config.fstab.save() - jail.start() + for event in jail.start(): + yield event + yield executeReleaseUpdateEvent.begin() child, stdout, stderr = jail.exec([ "/var/db/freebsd-update/freebsd-update.sh", "-d", @@ -483,8 +559,12 @@ def _update_freebsd_jail(self, jail): if child.returncode != 0: if "No updates are available to install." in stdout: + yield executeReleaseUpdateEvent.skip( + message="already up to date" + ) self.logger.debug("Already up to date") else: + yield executeReleaseUpdateEvent.failed() raise libiocage.lib.errors.ReleaseUpdateFailure( release_name=self.name, reason=( @@ -494,12 +574,14 @@ def _update_freebsd_jail(self, jail): logger=self.logger ) else: + yield executeReleaseUpdateEvent.end() self.logger.debug(f"Update of release '{self.name}' finished") - jail.stop() + for event in jail.stop(): + yield event self.logger.verbose(f"Release '{self.name}' updated") - return True # ToDo: return False if nothing was updated + yield True # ToDo: return False if nothing was updated def _append_datetime(self, text): now = datetime.datetime.utcnow() @@ -783,3 +865,19 @@ def _check_tar_info(self, tar_info, asset_name): reason=reason, logger=self.logger ) + + +class Release(ReleaseGenerator): + + def fetch(self, *args, **kwargs): + libiocage.lib.helpers.print_event_generator( + super().fetch(*args, **kwargs), + logger=self.logger + ) + + def update(self, *args, **kwargs): + libiocage.lib.helpers.print_event_generator( + super().update(*args, **kwargs), + logger=self.logger + ) + diff --git a/libiocage/lib/errors.py b/libiocage/lib/errors.py index aecee699..a158c57c 100644 --- a/libiocage/lib/errors.py +++ b/libiocage/lib/errors.py @@ -299,6 +299,21 @@ def __init__(self, devfs_rules_file, reason=None, *args, **kwargs): super().__init__(msg, *args, **kwargs) +# Logger + +class LogException(IocageException): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + +class CannotRedrawLine(LogException): + def __init__(self, reason, *args, **kwargs): + msg = "Logger can't redraw line" + if reason is not None: + msg += f": {reason}" + super().__init__(msg, *args, **kwargs) + + # Missing Features diff --git a/libiocage/lib/events.py b/libiocage/lib/events.py index ea783409..c863167f 100644 --- a/libiocage/lib/events.py +++ b/libiocage/lib/events.py @@ -1,3 +1,5 @@ +from timeit import default_timer as timer + EVENT_STATUS = ( "pending", "done", @@ -5,43 +7,255 @@ ) -class Iocage: +class IocageEvent: """ IocageEvent Base class for all other iocage events """ - def __init__(self, action, **kwargs): - self.action = action + HISTORY = [] + + PENDING_COUNT = 0 + + def __init__(self, message=None, **kwargs): + """ + Initializes an IocageEvent + """ + + for event in IocageEvent.HISTORY: + if event.__hash__() == self.__hash__(): + return event + + self.identifier = None + self._started_at = None + self._stopped_at = None + self._pending = False + self.skipped = False + self.done = True + self.error = None + self.data = kwargs + self.number = len(IocageEvent.HISTORY) + 1 + self.parent_count = IocageEvent.PENDING_COUNT + 0 + + self.message = message + + if self not in IocageEvent.HISTORY: + IocageEvent.HISTORY.append(self) + + @property + def state(self): + return self.get_state() + + def get_state_string(self, + error="failed", + skipped="skipped", + done="done", + pending="pending"): + + if self.error is not None: + return error + + if self.skipped is True: + return skipped + + if self.done is True: + return done + + return pending + + @property + def type(self): + """ + The type of event + + The event type is obtained from an IocageEvent's class name + """ + return type(self).__name__ + + @property + def pending(self): + return self._pending + + @pending.setter + def pending(self, state): + current = self._pending + new_state = (state is True) + + if current == new_state: + return + if new_state is True: + if self._started_at is not None: + raise libiocage.lib.errors.EventAlreadyFinished(event=self) + self._started_at = timer() + if new_state is False: + self._stopped_at = timer() -class Jail(Iocage): + self._pending = new_state + IocageEvent.PENDING_COUNT += 1 if (state is True) else -1 - def __init__(self, action, jail, **kwargs): - super().__init__(action=action, jail=jail, **kwargs) + @property + def duration(self): + if (self._started_at is None) or (self._stopped_at is None): + return None + return self._stopped_at - self._started_at + def _update_message(self, **kwargs): + if "message" in kwargs: + self.message = kwargs["message"] -class JailStarted(Jail): + def begin(self, **kwargs): + self._update_message(**kwargs) + self.pending = True + self.done = False + return self + + def end(self, **kwargs): + self._update_message(**kwargs) + self.done = True + self.pending = False + self.done = True + return self + + def step(self, **kwargs): + self._update_message(**kwargs) + return self + + def skip(self, **kwargs): + self._update_message(**kwargs) + self.skipped = True + self.pending = False + return self + + def fail(self, exception=True, **kwargs): + self._update_message(**kwargs) + self.error = exception + self.pending = False + return self + + def __hash__(self): + identifier = "generic" if self.identifier is None else self.identifier + return hash((self.type, identifier)) + + +# Jail + + +class JailEvent(IocageEvent): def __init__(self, jail, **kwargs): - super().__init__("Started", jail, **kwargs) + try: + self.identifier = jail.humanreadable_name + except: + self.identifier = None + + IocageEvent.__init__(self, jail=jail, **kwargs) -class JailVnetConfigured(Jail): +class JailLaunch(JailEvent): def __init__(self, jail, **kwargs): - super().__init__("Configuring VNET", jail, **kwargs) + JailEvent.__init__(self, jail, **kwargs) -class JailZfsShareMounted(Jail): +class JailVnetConfiguration(JailEvent): def __init__(self, jail, **kwargs): - super().__init__("Mounting ZFS shares", jail, **kwargs) + JailEvent.__init__(self, jail, **kwargs) -class JailServicesStarted(Jail): +class JailZfsShareMount(JailEvent): def __init__(self, jail, **kwargs): - super().__init__("Starting services", jail, **kwargs) + JailEvent.__init__(self, jail, **kwargs) + + +class JailServicesStart(JailEvent): + + def __init__(self, jail, **kwargs): + JailEvent.__init__(self, jail, **kwargs) + + +class JailDestroy(JailEvent): + + def __init__(self, jail, **kwargs): + JailEvent.__init__(self, jail, **kwargs) + + +class JailNetworkTeardown(JailEvent): + + def __init__(self, jail, **kwargs): + JailEvent.__init__(self, jail, **kwargs) + + +class JailMountTeardown(JailEvent): + + def __init__(self, jail, **kwargs): + JailEvent.__init__(self, jail, **kwargs) + +# Release + + +class ReleaseEvent(IocageEvent): + + def __init__(self, release, **kwargs): + try: + self.identifier = release.name + except: + self.identifier = None + + IocageEvent.__init__(self, release=release, **kwargs) + + +class FetchRelease(ReleaseEvent): + + def __init__(self, release, **kwargs): + ReleaseEvent.__init__(self, release, **kwargs) + + +class ReleasePrepareStorage(FetchRelease): + + def __init__(self, release, **kwargs): + FetchRelease.__init__(self, release, **kwargs) + + +class ReleaseDownload(FetchRelease): + + def __init__(self, release, **kwargs): + FetchRelease.__init__(self, release, **kwargs) + + +class ReleaseExtraction(FetchRelease): + + def __init__(self, release, **kwargs): + FetchRelease.__init__(self, release, **kwargs) + + +class ReleaseConfiguration(FetchRelease): + + def __init__(self, release, **kwargs): + FetchRelease.__init__(self, release, **kwargs) + + +class ReleaseUpdate(ReleaseEvent): + + def __init__(self, release, **kwargs): + ReleaseEvent.__init__(self, release, **kwargs) + + +class ReleaseUpdateDownload(ReleaseEvent): + + def __init__(self, release, **kwargs): + ReleaseUpdate.__init__(self, release, **kwargs) + + +class RunReleaseUpdate(ReleaseUpdate): + def __init__(self, release, **kwargs): + ReleaseUpdate.__init__(self, release, **kwargs) + + +class ExecuteReleaseUpdate(ReleaseUpdate): + def __init__(self, release, **kwargs): + ReleaseUpdate.__init__(self, release, **kwargs) diff --git a/libiocage/lib/helpers.py b/libiocage/lib/helpers.py index e60e2110..583a20ce 100644 --- a/libiocage/lib/helpers.py +++ b/libiocage/lib/helpers.py @@ -298,6 +298,55 @@ def umount(mountpoint, force=False, ignore_error=False, logger=None): if ignore_error is False: raise libiocage.lib.errors.UnmountFailed(logger=logger) +def print_event_generator(generator, logger): + lines = {} + last_event = None + for event in generator: + + if event.identifier is None: + identifier = "generic" + else: + identifier = event.identifier + + if event.type not in lines: + lines[event.type] = {} + + # output fragments + running_indicator = "+" if (event.done or event.skipped) else "-" + name = event.type + if event.identifier is not None: + name += f"@{event.identifier}" + + output = f"[{running_indicator}] {name}: " + + if event.message is not None: + output += event.message + else: + output += event.get_state_string( + done="OK", + error="failed", + skipped="skipped", + pending="..." + ) + + if event.duration is not None: + output += " [" + str(round(event.duration, 3)) + "s]" + + indent = event.parent_count + # new line or update of previous + if identifier not in lines[event.type]: + # Indent if previous task is not finished + lines[event.type][identifier] = logger.screen( + output, + indent=event.parent_count + ) + else: + lines[event.type][identifier].edit( + output, + indent=event.parent_count + ) + + last_event = event def get_basedir_list(distribution_name="FreeBSD"): basedirs = [ From c9875a3ca06a7371e1492b41f848ef1670be7503 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=B6nke?= Date: Wed, 30 Aug 2017 18:39:17 +0200 Subject: [PATCH 03/16] apply generators to list --- libiocage/cli/__init__.py | 63 +++++++++++- libiocage/cli/fetch.py | 24 +---- libiocage/cli/list.py | 147 ++++++++++++++++++++-------- libiocage/cli/start.py | 12 +-- libiocage/cli/stop.py | 10 +- libiocage/lib/Distribution.py | 11 ++- libiocage/lib/Host.py | 12 ++- libiocage/lib/Jail.py | 35 ++++--- libiocage/lib/JailFilter.py | 132 +++++++++++++++++++++++++ libiocage/lib/Jails.py | 178 ++++++++++++++-------------------- libiocage/lib/Release.py | 13 +-- libiocage/lib/helpers.py | 55 +---------- 12 files changed, 428 insertions(+), 264 deletions(-) create mode 100644 libiocage/lib/JailFilter.py diff --git a/libiocage/cli/__init__.py b/libiocage/cli/__init__.py index 3aa4f370..5cd14de2 100644 --- a/libiocage/cli/__init__.py +++ b/libiocage/cli/__init__.py @@ -60,6 +60,57 @@ exit(1) +def print_events(generator): + lines = {} + last_event = None + for event in generator: + + if event.identifier is None: + identifier = "generic" + else: + identifier = event.identifier + + if event.type not in lines: + lines[event.type] = {} + + # output fragments + running_indicator = "+" if (event.done or event.skipped) else "-" + name = event.type + if event.identifier is not None: + name += f"@{event.identifier}" + + output = f"[{running_indicator}] {name}: " + + if event.message is not None: + output += event.message + else: + output += event.get_state_string( + done="OK", + error="failed", + skipped="skipped", + pending="..." + ) + + if event.duration is not None: + output += " [" + str(round(event.duration, 3)) + "s]" + + indent = event.parent_count + # new line or update of previous + if identifier not in lines[event.type]: + # Indent if previous task is not finished + lines[event.type][identifier] = logger.screen( + output, + indent=event.parent_count + ) + else: + lines[event.type][identifier].edit( + output, + indent=event.parent_count + ) + + last_event = event + + class IOCageCLI(click.MultiCommand): """ Iterates in the 'cli' directory and will load any module's cli definition. @@ -77,7 +128,7 @@ def list_commands(self, ctx): return rv def get_command(self, ctx, name): - ctx.logger = logger + ctx.print_events = print_events try: mod = __import__(f"libiocage.cli.{name}", None, None, ["cli"]) @@ -96,11 +147,15 @@ def get_command(self, ctx, name): pass return mod.cli except (ImportError, AttributeError): + raise return - +@click.option("--log-level", "-d", default=None) @click.command(cls=IOCageCLI) -@click.version_option(version="0.9.10 07/30/2017", prog_name="iocage", +@click.version_option(version="0.2.11 08/29/2017", prog_name="ioc", message="%(version)s") -def cli(): +@click.pass_context +def cli(ctx, log_level): """A jail manager.""" + logger.print_level = log_level + ctx.logger = logger diff --git a/libiocage/cli/fetch.py b/libiocage/cli/fetch.py index caba5fed..7fd6f997 100644 --- a/libiocage/cli/fetch.py +++ b/libiocage/cli/fetch.py @@ -33,19 +33,6 @@ __rootcmd__ = True -# ToDo: remove disabled feature -# def _prettify_release_names(x): -# if x.name == host.release_version: -# return f"\033[1m{x.name}\033[0m" -# else: -# return x.name -# def release_choice(): -# version = -# return click.Choice(list(map( -# _prettify_release_names, -# host.distribution.releases -# ))) - @click.command(context_settings=dict( max_content_width=400, ), name="fetch", help="Fetch a version of FreeBSD for jail usage or a" @@ -69,16 +56,11 @@ @click.option("--files", multiple=True, help="Specify the files to fetch from the mirror. " "(Deprecared: renamed to --file)") -@click.option("--log-level", "-d", default=None) def cli(ctx, **kwargs): logger = ctx.parent.logger - logger.print_level = kwargs["log_level"] host = libiocage.lib.Host.Host(logger=logger) prompts = libiocage.lib.Prompts.Prompts(host=host, logger=logger) - if kwargs["log_level"] is not None: - logger.print_level = kwargs["log_level"] - release_input = kwargs["release"] if release_input is None: try: @@ -87,7 +69,7 @@ def cli(ctx, **kwargs): exit(1) else: try: - release = libiocage.lib.Release.Release( + release = libiocage.lib.Release.ReleaseGenerator( name=release_input, host=host, logger=logger @@ -111,10 +93,10 @@ def cli(ctx, **kwargs): exit(1) fetch_updates = bool(kwargs["fetch_updates"]) - release.fetch( + ctx.parent.print_events(release.fetch( update=kwargs["update"], fetch_updates=fetch_updates - ) + )) exit(0) diff --git a/libiocage/cli/list.py b/libiocage/cli/list.py index 8493997f..ed64114a 100644 --- a/libiocage/cli/list.py +++ b/libiocage/cli/list.py @@ -24,11 +24,15 @@ """list module for the cli.""" import click import texttable +import typing import libiocage.lib.Host import libiocage.lib.Jails +import libiocage.lib.JailFilter import libiocage.lib.Logger +supported_output_formats = ['table', 'csv', 'list'] + @click.command(name="list", help="List a specified dataset type, by default" " lists all jails.") @@ -37,27 +41,26 @@ flag_value="base", help="List all bases.") @click.option("--template", "-t", "dataset_type", flag_value="template", help="List all templates.") -@click.option("--header", "-h", "-H", is_flag=True, default=False, - help="For scripting, use tabs for separators.") @click.option("--long", "-l", "_long", is_flag=True, default=False, help="Show the full uuid and ip4 address.") @click.option("--remote", "-R", is_flag=True, help="Show remote's available " "RELEASEs.") @click.option("--plugins", "-P", is_flag=True, help="Show available plugins.") -@click.option("--sort", "-s", "_sort", default="name", nargs=1, +@click.option("--sort", "-s", "_sort", default=None, nargs=1, help="Sorts the list by the given type") @click.option("--quick", "-q", is_flag=True, default=False, help="Lists all jails with less processing and fields.") -@click.option("--log-level", "-d", default="info") @click.option("--output", "-o", default=None) +@click.option("--output-format", "-f", default="table", + type=click.Choice(supported_output_formats)) +@click.option("--header/--no-header", "-H/-NH", is_flag=True, default=True, + help="Show or hide column name heading.") @click.argument("filters", nargs=-1) def cli(ctx, dataset_type, header, _long, remote, plugins, - _sort, quick, log_level, output, filters): + _sort, quick, output, output_format, filters): logger = ctx.parent.logger - logger.print_level = log_level host = libiocage.lib.Host.Host(logger=logger) - jails = libiocage.lib.Jails.Jails(logger=logger) if remote and not plugins: @@ -68,52 +71,120 @@ def cli(ctx, dataset_type, header, _long, remote, plugins, if plugins and remote: raise Exception("ToDo: Plugins") + + if output is not None and _long is True: + logger.error("--output and --long can't be used together") + exit(1) + + if output_format != "table" and _sort is not None: + # Sorting destroys the ability to stream generators + # ToDo: Figure out if we need to sort other output formats as well + raise Exception("Sorting only allowed for tables") + + jails = libiocage.lib.Jails.JailsGenerator( + logger=logger, + host=host, + filters=filters # ToDo: allow quoted whitespaces from user input + ) + + columns = _list_output_comumns(output, _long) + + if output_format == "list": + _print_list(jails, columns, header, "\t") + elif output_format == "csv": + _print_list(jails, columns, header, ";") else: + _print_table(jails, columns, header, _sort) - if output: - columns = output.strip().split(',') - else: - columns = ["jid", "name"] - if _long: - columns += ["running", - "release", "ip4.addr", "ip6.addr"] - else: - columns += ["running", "release", "ip4.addr"] +def _print_table( + jails:typing.Generator[libiocage.lib.Jails.JailsGenerator, None, None], + columns:list, + show_header:bool, + sort_key:str=None + ) -> None: - table = texttable.Texttable(max_width=0) - table.set_cols_dtype(["t"] * len(columns)) + table = texttable.Texttable(max_width=0) + table.set_cols_dtype(["t"] * len(columns)) - table_head = (list(x.upper() for x in columns)) - table_data = [] + table_head = (list(x.upper() for x in columns)) + table_data = [] - try: - sort_index = columns.index(_sort) - except ValueError: - sort_index = None + try: + sort_index = columns.index(sort_key) + except ValueError: + sort_index = None - for jail in jails.list(filters=filters): - table_data.append( - [_lookup_jail_value(jail, x) for x in columns] - ) + for jail in jails: + table_data.append(_lookup_jail_values(jail, columns)) - if sort_index is not None: - table_data.sort(key=lambda x: x[sort_index]) + if sort_index is not None: + table_data.sort(key=lambda x: x[sort_index]) + if show_header: table.add_rows([table_head] + table_data) + else: + table.add_rows(table_data) + + print(table.draw()) + + +def _print_list( + jails:typing.Generator[libiocage.lib.Jails.JailsGenerator, None, None], + columns:list, + show_header:bool, + separator:str=";" + ) -> None: - if header: - # TODO: This sucks since it's a protected member - for item in table._rows: - print("\t".join(item)) + if show_header is True: + print(separator.join(columns).upper()) + + for jail in jails: + print(separator.join(_lookup_jail_values(jail, columns))) + + +def _list_output_comumns( + user_input:str="", + long_mode:bool=False + ) -> list: + + if user_input: + return user_input.strip().split(',') + else: + columns = ["jid", "name"] + + if long_mode is True: + columns += [ + "running", + "release", + "ip4.addr", + "ip6.addr" + ] else: - print(table.draw()) + columns += [ + "running", + "release", + "ip4.addr" + ] + + return columns + + +def _lookup_jail_values( + jail:libiocage.lib.Jail.JailGenerator, + keys:str + ) -> list: + + return [_lookup_jail_value(jail, x) for x in keys] - return +def _lookup_jail_value( + jail:libiocage.lib.Jail.JailGenerator, + key:str + ) -> str: -def _lookup_jail_value(jail, key): - if key in libiocage.lib.Jails.Jails.JAIL_KEYS: + # ToDo: Move this into lib/Jails ? + if key in libiocage.lib.Jails.JailsGenerator.JAIL_KEYS: return jail.getstring(key) else: return str(jail.config.__getitem__(key)) diff --git a/libiocage/cli/start.py b/libiocage/cli/start.py index bc80ee5a..8b1ee257 100644 --- a/libiocage/cli/start.py +++ b/libiocage/cli/start.py @@ -35,25 +35,21 @@ @click.option("--rc", default=False, is_flag=True, help="Will start all jails with boot=on, in the specified" " order with smaller value for priority starting first.") -@click.option("--log-level", "-d", default=None) @click.argument("jails", nargs=-1) -def cli(ctx, rc, jails, log_level): +def cli(ctx, rc, jails): """ Starts Jails """ logger = ctx.parent.logger - logger.print_level = log_level - ioc_jails = libiocage.lib.Jails.Jails(logger=logger) + ioc_jails = libiocage.lib.Jails.JailsGenerator(logger=logger, filters=jails) - for jail in ioc_jails.list(filters=jails): + for jail in ioc_jails: logger.log(f"Starting {jail.humanreadable_name}") try: - for event in jail.start(yields=True): - print(event.name) + ctx.parent.print_events(jail.start()) except Exception: - raise exit(1) logger.log(f"{jail.humanreadable_name} running as JID {jail.jid}") diff --git a/libiocage/cli/stop.py b/libiocage/cli/stop.py index a9563cc0..4145723b 100644 --- a/libiocage/cli/stop.py +++ b/libiocage/cli/stop.py @@ -45,15 +45,13 @@ def cli(ctx, rc, log_level, force, jails): location to stop_jail. """ logger = ctx.parent.logger - logger.print_level = log_level - ioc_jails = libiocage.lib.Jails.Jails(logger=logger) + ioc_jails = libiocage.lib.Jails.JailsGenerator(logger=logger, filters=jails) - for jail in ioc_jails.list(filters=jails): - logger.log(f"Stopping jail {jail.humanreadable_name}") + for jail in ioc_jails: try: - jail.stop(force=force) + ctx.parent.print_events(jail.stop(force=force)) except: exit(1) - logger.log("done") + logger.log(f"{jail.name} stopped") exit(0) diff --git a/libiocage/lib/Distribution.py b/libiocage/lib/Distribution.py index 0aefa5bb..6babd78c 100644 --- a/libiocage/lib/Distribution.py +++ b/libiocage/lib/Distribution.py @@ -11,7 +11,10 @@ import libiocage.lib.helpers -class Distribution: +class DistributionGenerator: + + _class_release = libiocage.lib.Release.ReleaseGenerator + release_name_blacklist = [ "", ".", @@ -79,7 +82,7 @@ def fetch_releases(self): ) self.available_releases = list(map( - lambda x: libiocage.lib.Release.Release( + lambda x: self._class_release( name=x, host=self.host, zfs=self.zfs, @@ -170,3 +173,7 @@ def _parse_links(self, text): ) return matches + +class Distribution(DistributionGenerator): + + _class_release = libiocage.lib.Release.Release diff --git a/libiocage/lib/Host.py b/libiocage/lib/Host.py index 59715898..7e03bdbc 100644 --- a/libiocage/lib/Host.py +++ b/libiocage/lib/Host.py @@ -7,7 +7,10 @@ import libiocage.lib.helpers -class Host: +class HostGenerator: + + _class_distribution = libiocage.lib.Distribution.DistributionGenerator + def __init__(self, root_dataset=None, zfs=None, logger=None): libiocage.lib.helpers.init_logger(self, logger) @@ -17,7 +20,7 @@ def __init__(self, root_dataset=None, zfs=None, logger=None): logger=self.logger, zfs=self.zfs ) - self.distribution = libiocage.lib.Distribution.Distribution( + self.distribution = self._class_distribution( host=self, logger=self.logger ) @@ -61,3 +64,8 @@ def release_version(self): @property def processor(self): return platform.processor() + + +class Host(HostGenerator): + + class_distribution = libiocage.lib.Distribution.DistributionGenerator diff --git a/libiocage/lib/Jail.py b/libiocage/lib/Jail.py index ec540ab0..09de131f 100644 --- a/libiocage/lib/Jail.py +++ b/libiocage/lib/Jail.py @@ -61,6 +61,9 @@ class JailGenerator: release (stored in `zpool/iocage/base/) """ + _class_host = libiocage.lib.Host.HostGenerator + _class_storage = libiocage.lib.Storage.Storage + def __init__(self, data={}, zfs=None, host=None, logger=None, new=False): """ Initializes a Jail @@ -96,7 +99,7 @@ def __init__(self, data={}, zfs=None, host=None, logger=None, new=False): self.networks = [] - self.storage = libiocage.lib.Storage.Storage( + self.storage = self._class_storage( auto_create=True, safe_mode=False, jail=self, logger=self.logger, zfs=self.zfs) @@ -852,12 +855,6 @@ def __getattr__(self, key): except AttributeError: pass - try: - method = object.__getattribute__(self, f"_get_{key}") - return method() - except: - pass - try: jail_state = object.__getattribute__(self, "jail_state") except: @@ -893,9 +890,7 @@ def __dir__(self): properties = set() for prop in dict.__dir__(self): - if prop.startswith("_get_"): - properties.add(prop[5:]) - elif not prop.startswith("_"): + if not prop.startswith("_"): properties.add(prop) return list(properties) @@ -903,14 +898,18 @@ def __dir__(self): class Jail(JailGenerator): + _class_host = libiocage.lib.Host.HostGenerator + def start(self, *args, **kwargs): - libiocage.lib.helpers.print_event_generator( - super().start(*args, **kwargs), - logger=self.logger - ) + return list(JailGenerator.start(self, *args, **kwargs)) + # libiocage.lib.helpers.print_event_generator( + # JailGenerator.start(self, *args, **kwargs), + # logger=self.logger + # ) def stop(self, *args, **kwargs): - libiocage.lib.helpers.print_event_generator( - super().stop(*args, **kwargs), - logger=self.logger - ) + return list(JailGenerator.start(self, *args, **kwargs)) + # libiocage.lib.helpers.print_event_generator( + # JailGenerator.stop(self, *args, **kwargs), + # logger=self.logger + # ) diff --git a/libiocage/lib/JailFilter.py b/libiocage/lib/JailFilter.py new file mode 100644 index 00000000..23c3e379 --- /dev/null +++ b/libiocage/lib/JailFilter.py @@ -0,0 +1,132 @@ +from typing import List, Union, Set, Iterable +import re +import libiocage.lib.Jail + + +def match_filter(value:str, filter_string:str): + escaped_characters = [".", "$", "^", "(", ")", "?"] + for character in escaped_characters: + filter_string = filter_string.replace(character, f"\\{character}") + filter_string = filter_string.replace("*", ".*") + filter_string = filter_string.replace("+", ".+") + print + pattern = f"^{filter_string}$" + match = re.match(pattern, value) + return match is not None + + +class Term(list): + + def __init__(self, key, values=list()): + self.key = key + + if values is None: + raise TypeError("Values may not be empty") + elif isinstance(values, str): + data = self._split_filter_values(values) + else: + data = [values] + + list.__init__(self, data) + + def matches_jail(self, jail:libiocage.lib.Jail.JailGenerator) -> bool: + jail_value = jail.getstring(self.key) + return self.matches(jail_value) + + def matches(self, value: str) -> bool: + """ + Returns True if the value matches the term + """ + for filter_value in self: + if match_filter(value, filter_value): + return True + return False + + def _split_filter_values(self, user_input:str) -> List[str]: + values = [] + escaped_comma_blocks = map( + lambda block: block.split(","), + user_input.split("\\,") + ) + for block in escaped_comma_blocks: + n = len(values) + if n > 0: + index = n - 1 + values[index] += f",{block[0]}" + else: + values.append(block[0]) + if len(block) > 1: + values += block[1:] + return values + + +class Terms(list): + """ + A group of jail filter terms. + + Each item in this group must match for a jail to pass the filter. + This can be interpreted as logical AND + """ + + def __init__(self, terms:Iterable[Union[Term,str]]=None): + + data = [] + + if terms is not None: + + for term in terms: + if isinstance(term, str): + data += self._parse_term(term) + elif isinstance(term, Term): + data.append(term) + + list.__init__(self, data) + + @property + def only_by_name(self) -> bool: + """ + True if all terms match by name only + """ + return all([x.key == "name" for x in self]) + + def match_jail(self, jail: libiocage.lib.Jail.JailGenerator) -> bool: + """ + Returns True if all Terms match the jail + """ + for term in self: + if term.matches_jail(jail) is False: + return False + + return True + + def match_key(self, key: str, value: str) -> bool: + """ + Check if a value matches for a given key + + Returns True if the given value matches all terms for the specified key + Returns Fals if one of the terms does not match + """ + for term in self: + + if term.key != key: + continue + + if term.matches(value) is False: + return False + + return True + + def _parse_term(self, user_input:Iterable[str]) -> List[Term]: + + terms = [] + + for user_term in user_input: + try: + prop, value = user_input.split("=", maxsplit=1) + except: + prop = "name" + value = user_input + + terms.append(Term(prop, value)) + + return terms diff --git a/libiocage/lib/Jails.py b/libiocage/lib/Jails.py index e2e45980..1e12da04 100644 --- a/libiocage/lib/Jails.py +++ b/libiocage/lib/Jails.py @@ -1,12 +1,14 @@ import re +from typing import Generator, List, Union, Iterable import libzfs import libiocage.lib.Jail +import libiocage.lib.JailFilter import libiocage.lib.helpers -class Jails: +class JailsGenerator(list): # Keys that are stored on the Jail object, not the configuration JAIL_KEYS = [ "jid", @@ -17,6 +19,7 @@ class Jails: ] def __init__(self, + filters=None, host=None, logger=None, zfs=None): @@ -26,122 +29,85 @@ def __init__(self, libiocage.lib.helpers.init_host(self, host) self.zfs = libzfs.ZFS(history=True, history_prefix="") - def list(self, filters=None): - - if len(filters) == 1: - chars = "+*=" - name = filters[0] - if not any(x in name for x in chars): - single_jail = libiocage.lib.Jail.Jail( - { - "name": name - }, - logger=self.logger, - host=self.host, - zfs=self.zfs - ) - return [single_jail] - - jails = self._get_existing_jails() - - if filters is not None: - return self._filter_jails(jails, filters) - else: - return jails + self._filters = None + self.filters = filters + list.__init__(self, []) - def _filter_jails(self, jails, filters): + def __iter__(self): - filtered_jails = [] - jail_filters = {} + for jail_dataset in self.jail_datasets: - filter_terms = list(map(_split_filter_map, filters)) - for key, value in filter_terms: - if key not in jail_filters.keys(): - jail_filters[key] = [value] - else: - jail_filters[key].append(value) + jail_name = self._get_name_from_jail_dataset(jail_dataset) + if self._filters.match_key("name", jail_name) is not True: + # Skip all jails that do not even match the name + continue - for jail in jails: + # ToDo: Do not load jail if filters do not require to + jail = self._load_jail_from_dataset(jail_dataset) + if self._filters.match_jail(jail): + yield jail - jail_matches = True + def _create_jail(self, *args, **kwargs): + return libiocage.lib.Jail.JailGenerator( + *args, + logger=self.logger, + host=self.host, + zfs=self.zfs, + **kwargs + ) - for group in jail_filters.keys(): + @property + def filters(self): + return self._filters + + @filters.setter + def filters( + self, + value:Union[ + str, + Iterable[Union[libiocage.lib.JailFilter.Terms,str]] + ] + ): + + if isinstance(value, libiocage.lib.JailFilter.Terms): + self._filters = value + else: + self._filters = libiocage.lib.JailFilter.Terms(value) + + @property + def jail_datasets(self) -> list: + jails_dataset = self.host.datasets.jails + return list(jails_dataset.children) - # Providing multiple names = OR (e.g. name=foo, name=bar) - jail_matches_group = False + def _load_jail_from_dataset( + self, + dataset: libzfs.ZFSDataset + ) -> Generator[libiocage.lib.Jail.JailGenerator, None, None]: - for current_filter in jail_filters[group]: - if self._jail_matches_filter(jail, group, current_filter): - jail_matches_group = True + return self._create_jail({ + "name": self._get_name_from_jail_dataset(dataset) + } + ) - if jail_matches_group is False: - jail_matches = False - continue + def _get_name_from_jail_dataset( + self, + dataset:libzfs.ZFSDataset + ) -> str: - if jail_matches is True: - filtered_jails.append(jail) + return dataset.name.split("/").pop() - return filtered_jails - def _get_existing_jails(self): - jails_dataset = self.host.datasets.jails - jail_datasets = list(jails_dataset.children) - - return list(map( - lambda x: libiocage.lib.Jail.Jail({ - "name": x.name.split("/").pop() - }, logger=self.logger, host=self.host, zfs=self.zfs), - jail_datasets - )) - - def _jail_matches_filter(self, jail, key, value): - for filter_value in self._split_filter_values(value): - jail_value = self._lookup_jail_value(jail, key) - if not self._matches_filter(filter_value, jail_value): - return False - return True - - def _matches_filter(self, filter_value, value): - escaped_characters = [".", "$", "^", "(", ")"] - for character in escaped_characters: - filter_value = filter_value.replace(character, f"\\{character}") - filter_value = filter_value.replace("$", "\\$") - filter_value = filter_value.replace(".", "\\.") - filter_value = filter_value.replace("*", ".*") - filter_value = filter_value.replace("+", ".+") - pattern = f"^{filter_value}$" - match = re.match(pattern, value) - return match is not None - - def _lookup_jail_value(self, jail, key): - if key in Jails.JAIL_KEYS: - return jail.getstring(key) - else: - return str(jail.config[key]) +class Jails(JailsGenerator): - def _split_filter_values(self, value): - values = [] - escaped_comma_blocks = map( - lambda block: block.split(","), - value.split("\\,") + def _create_jail(self, *args, **kwargs): + return libiocage.lib.Jail.Jail( + *args, + logger=self.logger, + host=self.host, + zfs=self.zfs, + **kwargs ) - for block in escaped_comma_blocks: - n = len(values) - if n > 0: - index = n - 1 - values[index] += f",{block[0]}" - else: - values.append(block[0]) - if len(block) > 1: - values += block[1:] - return values - - -def _split_filter_map(x): - try: - prop, value = x.split("=", maxsplit=1) - except: - prop = "name" - value = x - - return prop, value + + def __iter__(self): + return list(JailsGenerator.__iter__(self)) + diff --git a/libiocage/lib/Release.py b/libiocage/lib/Release.py index f3a426fd..8a4276a8 100644 --- a/libiocage/lib/Release.py +++ b/libiocage/lib/Release.py @@ -866,18 +866,15 @@ def _check_tar_info(self, tar_info, asset_name): logger=self.logger ) + def __str__(self): + return self.name + class Release(ReleaseGenerator): def fetch(self, *args, **kwargs): - libiocage.lib.helpers.print_event_generator( - super().fetch(*args, **kwargs), - logger=self.logger - ) + return list(ReleaseGenerator.fetch(self, *args, **kwargs)) def update(self, *args, **kwargs): - libiocage.lib.helpers.print_event_generator( - super().update(*args, **kwargs), - logger=self.logger - ) + return list(ReleaseGenerator.update(self, *args, **kwargs)) diff --git a/libiocage/lib/helpers.py b/libiocage/lib/helpers.py index 583a20ce..4de3e535 100644 --- a/libiocage/lib/helpers.py +++ b/libiocage/lib/helpers.py @@ -28,8 +28,10 @@ def init_host(self, host=None): except: logger = None - self.host = libiocage.lib.Host.Host(logger=logger) - + try: + self.host = self._class_host(logger=logger) + except: + self.host = libiocage.lib.Host.HostGenerator(logger=logger) def init_datasets(self, datasets=None): if datasets: @@ -298,55 +300,6 @@ def umount(mountpoint, force=False, ignore_error=False, logger=None): if ignore_error is False: raise libiocage.lib.errors.UnmountFailed(logger=logger) -def print_event_generator(generator, logger): - lines = {} - last_event = None - for event in generator: - - if event.identifier is None: - identifier = "generic" - else: - identifier = event.identifier - - if event.type not in lines: - lines[event.type] = {} - - # output fragments - running_indicator = "+" if (event.done or event.skipped) else "-" - name = event.type - if event.identifier is not None: - name += f"@{event.identifier}" - - output = f"[{running_indicator}] {name}: " - - if event.message is not None: - output += event.message - else: - output += event.get_state_string( - done="OK", - error="failed", - skipped="skipped", - pending="..." - ) - - if event.duration is not None: - output += " [" + str(round(event.duration, 3)) + "s]" - - indent = event.parent_count - # new line or update of previous - if identifier not in lines[event.type]: - # Indent if previous task is not finished - lines[event.type][identifier] = logger.screen( - output, - indent=event.parent_count - ) - else: - lines[event.type][identifier].edit( - output, - indent=event.parent_count - ) - - last_event = event def get_basedir_list(distribution_name="FreeBSD"): basedirs = [ From 57f9b744cd3304758091965f88ce4e25fff19b8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=B6nke?= Date: Wed, 30 Aug 2017 18:53:26 +0200 Subject: [PATCH 04/16] address review #pullrequestreview-59313901 --- libiocage/lib/Logger.py | 9 +++------ libiocage/lib/Release.py | 3 +-- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/libiocage/lib/Logger.py b/libiocage/lib/Logger.py index 5179614c..96f6bc02 100644 --- a/libiocage/lib/Logger.py +++ b/libiocage/lib/Logger.py @@ -138,7 +138,7 @@ def spam(self, message, indent=0, **kwargs): def screen(self, message, indent=0, **kwargs): """ - Screen does never get printed to log files + Screen never gets printed to log files """ return self.log(message, level="screen", indent=indent, **kwargs) @@ -160,20 +160,17 @@ def redraw(self, log_entry): line_number = self.PRINT_HISTORY.index(log_entry) delta = len(self.PRINT_HISTORY) - line_number - # ToDo: Handle redrawig of multiline entries with different line count + # ToDo: Handle redrawing of multiline entries with different line count output = "".join([ - #"\033[s", # SCP - Save Cursor Position f"\033[{delta}F", # CPL - Cursor Previous Line "\r", # CR - Carriage Return self._indent(log_entry.message, log_entry.indent), - "\033[K", # EL - Erease in Line + "\033[K", # EL - Erase in Line "\n" * delta - #"\033[s" # RCP - Restore Cursor Position ]) sys.stdout.write(output) - #print(log_entry.message) def _should_print_log_entry(self, log_entry): diff --git a/libiocage/lib/Release.py b/libiocage/lib/Release.py index 8a4276a8..0f30944a 100644 --- a/libiocage/lib/Release.py +++ b/libiocage/lib/Release.py @@ -320,7 +320,7 @@ def fetch(self, update=None, fetch_updates=None): if isinstance(event, libiocage.lib.events.IocageEvent): yield event else: - # the only non-IocageEvent is out return value + # the only non-IocageEvent is our return value release_changed = event if release_changed: @@ -710,7 +710,6 @@ def _create_default_rcconf(self): with open(file, "w") as f: f.write(content) f.truncate() - f.close() def _generate_default_rcconf_line(self, service_name): if Release.DEFAULT_RC_CONF_SERVICES[service_name] is True: From 331679e6ac0a048a7849fe20e562e32ad0033f13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=B6nke?= Date: Wed, 30 Aug 2017 19:03:46 +0200 Subject: [PATCH 05/16] fix code style --- libiocage/cli/__init__.py | 7 ++--- libiocage/cli/list.py | 44 +++++++++++++++--------------- libiocage/cli/start.py | 5 +++- libiocage/cli/stop.py | 5 +++- libiocage/lib/Distribution.py | 1 + libiocage/lib/JailFilter.py | 30 ++++++++++----------- libiocage/lib/Jails.py | 34 +++++++++++------------ libiocage/lib/Logger.py | 25 ++++++++++------- libiocage/lib/Release.py | 25 +++++++++-------- libiocage/lib/errors.py | 51 +++++++++++++++++++++++++++++++++++ libiocage/lib/events.py | 11 +++++--- libiocage/lib/helpers.py | 3 ++- 12 files changed, 151 insertions(+), 90 deletions(-) diff --git a/libiocage/cli/__init__.py b/libiocage/cli/__init__.py index 5cd14de2..65b0c2fc 100644 --- a/libiocage/cli/__init__.py +++ b/libiocage/cli/__init__.py @@ -62,14 +62,13 @@ def print_events(generator): lines = {} - last_event = None for event in generator: if event.identifier is None: identifier = "generic" else: identifier = event.identifier - + if event.type not in lines: lines[event.type] = {} @@ -94,7 +93,6 @@ def print_events(generator): if event.duration is not None: output += " [" + str(round(event.duration, 3)) + "s]" - indent = event.parent_count # new line or update of previous if identifier not in lines[event.type]: # Indent if previous task is not finished @@ -108,8 +106,6 @@ def print_events(generator): indent=event.parent_count ) - last_event = event - class IOCageCLI(click.MultiCommand): """ @@ -150,6 +146,7 @@ def get_command(self, ctx, name): raise return + @click.option("--log-level", "-d", default=None) @click.command(cls=IOCageCLI) @click.version_option(version="0.2.11 08/29/2017", prog_name="ioc", diff --git a/libiocage/cli/list.py b/libiocage/cli/list.py index ed64114a..b42f81b7 100644 --- a/libiocage/cli/list.py +++ b/libiocage/cli/list.py @@ -51,7 +51,7 @@ @click.option("--quick", "-q", is_flag=True, default=False, help="Lists all jails with less processing and fields.") @click.option("--output", "-o", default=None) -@click.option("--output-format", "-f", default="table", +@click.option("--output-format", "-f", default="table", type=click.Choice(supported_output_formats)) @click.option("--header/--no-header", "-H/-NH", is_flag=True, default=True, help="Show or hide column name heading.") @@ -84,7 +84,7 @@ def cli(ctx, dataset_type, header, _long, remote, plugins, jails = libiocage.lib.Jails.JailsGenerator( logger=logger, host=host, - filters=filters # ToDo: allow quoted whitespaces from user input + filters=filters # ToDo: allow quoted whitespaces from user input ) columns = _list_output_comumns(output, _long) @@ -98,11 +98,11 @@ def cli(ctx, dataset_type, header, _long, remote, plugins, def _print_table( - jails:typing.Generator[libiocage.lib.Jails.JailsGenerator, None, None], - columns:list, - show_header:bool, - sort_key:str=None - ) -> None: + jails: typing.Generator[libiocage.lib.Jails.JailsGenerator, None, None], + columns: list, + show_header: bool, + sort_key: str=None +) -> None: table = texttable.Texttable(max_width=0) table.set_cols_dtype(["t"] * len(columns)) @@ -125,16 +125,16 @@ def _print_table( table.add_rows([table_head] + table_data) else: table.add_rows(table_data) - + print(table.draw()) def _print_list( - jails:typing.Generator[libiocage.lib.Jails.JailsGenerator, None, None], - columns:list, - show_header:bool, - separator:str=";" - ) -> None: + jails: typing.Generator[libiocage.lib.Jails.JailsGenerator, None, None], + columns: list, + show_header: bool, + separator: str=";" +) -> None: if show_header is True: print(separator.join(columns).upper()) @@ -144,9 +144,9 @@ def _print_list( def _list_output_comumns( - user_input:str="", - long_mode:bool=False - ) -> list: + user_input: str="", + long_mode: bool=False +) -> list: if user_input: return user_input.strip().split(',') @@ -171,17 +171,17 @@ def _list_output_comumns( def _lookup_jail_values( - jail:libiocage.lib.Jail.JailGenerator, - keys:str - ) -> list: + jail: libiocage.lib.Jail.JailGenerator, + keys: str +) -> list: return [_lookup_jail_value(jail, x) for x in keys] def _lookup_jail_value( - jail:libiocage.lib.Jail.JailGenerator, - key:str - ) -> str: + jail: libiocage.lib.Jail.JailGenerator, + key: str +) -> str: # ToDo: Move this into lib/Jails ? if key in libiocage.lib.Jails.JailsGenerator.JAIL_KEYS: diff --git a/libiocage/cli/start.py b/libiocage/cli/start.py index 8b1ee257..678f93eb 100644 --- a/libiocage/cli/start.py +++ b/libiocage/cli/start.py @@ -42,7 +42,10 @@ def cli(ctx, rc, jails): """ logger = ctx.parent.logger - ioc_jails = libiocage.lib.Jails.JailsGenerator(logger=logger, filters=jails) + ioc_jails = libiocage.lib.Jails.JailsGenerator( + logger=logger, + filters=jails + ) for jail in ioc_jails: logger.log(f"Starting {jail.humanreadable_name}") diff --git a/libiocage/cli/stop.py b/libiocage/cli/stop.py index 4145723b..9d58eb72 100644 --- a/libiocage/cli/stop.py +++ b/libiocage/cli/stop.py @@ -45,7 +45,10 @@ def cli(ctx, rc, log_level, force, jails): location to stop_jail. """ logger = ctx.parent.logger - ioc_jails = libiocage.lib.Jails.JailsGenerator(logger=logger, filters=jails) + ioc_jails = libiocage.lib.Jails.JailsGenerator( + logger=logger, + filters=jails + ) for jail in ioc_jails: try: diff --git a/libiocage/lib/Distribution.py b/libiocage/lib/Distribution.py index 6babd78c..a387b49a 100644 --- a/libiocage/lib/Distribution.py +++ b/libiocage/lib/Distribution.py @@ -174,6 +174,7 @@ def _parse_links(self, text): return matches + class Distribution(DistributionGenerator): _class_release = libiocage.lib.Release.Release diff --git a/libiocage/lib/JailFilter.py b/libiocage/lib/JailFilter.py index 23c3e379..93eac231 100644 --- a/libiocage/lib/JailFilter.py +++ b/libiocage/lib/JailFilter.py @@ -1,18 +1,18 @@ -from typing import List, Union, Set, Iterable +from typing import List, Union, Iterable import re import libiocage.lib.Jail -def match_filter(value:str, filter_string:str): - escaped_characters = [".", "$", "^", "(", ")", "?"] - for character in escaped_characters: - filter_string = filter_string.replace(character, f"\\{character}") - filter_string = filter_string.replace("*", ".*") - filter_string = filter_string.replace("+", ".+") - print - pattern = f"^{filter_string}$" - match = re.match(pattern, value) - return match is not None +def match_filter(value: str, filter_string: str): + escaped_characters = [".", "$", "^", "(", ")", "?"] + for character in escaped_characters: + filter_string = filter_string.replace(character, f"\\{character}") + filter_string = filter_string.replace("*", ".*") + filter_string = filter_string.replace("+", ".+") + print + pattern = f"^{filter_string}$" + match = re.match(pattern, value) + return match is not None class Term(list): @@ -29,7 +29,7 @@ def __init__(self, key, values=list()): list.__init__(self, data) - def matches_jail(self, jail:libiocage.lib.Jail.JailGenerator) -> bool: + def matches_jail(self, jail: libiocage.lib.Jail.JailGenerator) -> bool: jail_value = jail.getstring(self.key) return self.matches(jail_value) @@ -42,7 +42,7 @@ def matches(self, value: str) -> bool: return True return False - def _split_filter_values(self, user_input:str) -> List[str]: + def _split_filter_values(self, user_input: str) -> List[str]: values = [] escaped_comma_blocks = map( lambda block: block.split(","), @@ -68,7 +68,7 @@ class Terms(list): This can be interpreted as logical AND """ - def __init__(self, terms:Iterable[Union[Term,str]]=None): + def __init__(self, terms: Iterable[Union[Term, str]]=None): data = [] @@ -116,7 +116,7 @@ def match_key(self, key: str, value: str) -> bool: return True - def _parse_term(self, user_input:Iterable[str]) -> List[Term]: + def _parse_term(self, user_input: Iterable[str]) -> List[Term]: terms = [] diff --git a/libiocage/lib/Jails.py b/libiocage/lib/Jails.py index 1e12da04..b665eca1 100644 --- a/libiocage/lib/Jails.py +++ b/libiocage/lib/Jails.py @@ -1,5 +1,4 @@ -import re -from typing import Generator, List, Union, Iterable +from typing import Generator, Union, Iterable import libzfs @@ -62,13 +61,13 @@ def filters(self): @filters.setter def filters( - self, - value:Union[ - str, - Iterable[Union[libiocage.lib.JailFilter.Terms,str]] - ] - ): - + self, + value: Union[ + str, + Iterable[Union[libiocage.lib.JailFilter.Terms, str]] + ] + ): + if isinstance(value, libiocage.lib.JailFilter.Terms): self._filters = value else: @@ -80,19 +79,19 @@ def jail_datasets(self) -> list: return list(jails_dataset.children) def _load_jail_from_dataset( - self, - dataset: libzfs.ZFSDataset - ) -> Generator[libiocage.lib.Jail.JailGenerator, None, None]: + self, + dataset: libzfs.ZFSDataset + ) -> Generator[libiocage.lib.Jail.JailGenerator, None, None]: return self._create_jail({ - "name": self._get_name_from_jail_dataset(dataset) - } + "name": self._get_name_from_jail_dataset(dataset) + } ) def _get_name_from_jail_dataset( - self, - dataset:libzfs.ZFSDataset - ) -> str: + self, + dataset: libzfs.ZFSDataset + ) -> str: return dataset.name.split("/").pop() @@ -110,4 +109,3 @@ def _create_jail(self, *args, **kwargs): def __iter__(self): return list(JailsGenerator.__iter__(self)) - diff --git a/libiocage/lib/Logger.py b/libiocage/lib/Logger.py index 96f6bc02..45a11852 100644 --- a/libiocage/lib/Logger.py +++ b/libiocage/lib/Logger.py @@ -30,6 +30,7 @@ def edit(self, message=None, indent=None): self.logger.redraw(self) + class Logger: COLORS = ( @@ -48,15 +49,15 @@ class Logger: LINE_UP_SEQ = "\033[F" LOG_LEVEL_SETTINGS = { - "screen" : {"color": None}, - "info" : {"color": None}, - "notice" : {"color": "magenta"}, - "verbose" : {"color": "blue"}, - "spam" : {"color": "green"}, + "screen": {"color": None}, + "info": {"color": None}, + "notice": {"color": "magenta"}, + "verbose": {"color": "blue"}, + "spam": {"color": "green"}, "critical": {"color": "red", "bold": True}, - "error" : {"color": "red"}, - "debug" : {"color": "green"}, - "warn" : {"color": "yellow"} + "error": {"color": "red"}, + "debug": {"color": "green"}, + "warn": {"color": "yellow"} } LOG_LEVELS = ( @@ -118,7 +119,7 @@ def log(self, *args, **kwargs): if self._should_print_log_entry(log_entry): self._print_log_entry(log_entry) self.PRINT_HISTORY.append(log_entry) - + return log_entry def verbose(self, message, indent=0, **kwargs): @@ -193,7 +194,11 @@ def _print(self, message, level, indent=0): print(self._beautify_message(message, level, indent)) def _print_log_entry(self, log_entry): - return self._print(log_entry.message, log_entry.level, log_entry.indent) + return self._print( + log_entry.message, + log_entry.level, + log_entry.indent + ) def _indent(self, message, level): indent = Logger.INDENT_PREFIX * level diff --git a/libiocage/lib/Release.py b/libiocage/lib/Release.py index 0f30944a..90992e7e 100644 --- a/libiocage/lib/Release.py +++ b/libiocage/lib/Release.py @@ -18,11 +18,11 @@ class ReleaseGenerator: DEFAULT_RC_CONF_SERVICES = { - "netif" : False, - "sendmail" : False, - "sendmail_submit" : False, + "netif": False, + "sendmail": False, + "sendmail_submit": False, "sendmail_msp_queue": False, - "sendmail_outbound" : False + "sendmail_outbound": False } def __init__(self, name=None, @@ -356,7 +356,7 @@ def fetch_updates(self): files = { update_script_name: f"usr.sbin/{update_name}/{update_script_name}", - update_conf_name : f"etc/{update_conf_name}", + update_conf_name: f"etc/{update_conf_name}", } for key in files.keys(): @@ -408,7 +408,7 @@ def fetch_updates(self): yield releaseUpdateDownloadEvent.skip( message="pre-fetching not supported on HardenedBSD" ) - + else: self.logger.verbose(f"Fetching updates for release '{self.name}'") libiocage.lib.helpers.exec([ @@ -434,12 +434,12 @@ def update(self): dataset.snapshot(snapshot_name, recursive=True) jail = libiocage.lib.Jail.JailGenerator({ - "uuid" : str(uuid.uuid4()), - "basejail" : False, - "allow_mount_nullfs": "1", - "release" : self.name, - "securelevel" : "0" - }, + "uuid": str(uuid.uuid4()), + "basejail": False, + "allow_mount_nullfs": "1", + "release": self.name, + "securelevel": "0" + }, new=True, logger=self.logger, zfs=self.zfs, @@ -876,4 +876,3 @@ def fetch(self, *args, **kwargs): def update(self, *args, **kwargs): return list(ReleaseGenerator.update(self, *args, **kwargs)) - diff --git a/libiocage/lib/errors.py b/libiocage/lib/errors.py index a158c57c..c7f0df1d 100644 --- a/libiocage/lib/errors.py +++ b/libiocage/lib/errors.py @@ -1,4 +1,5 @@ class IocageException(Exception): + def __init__(self, message, errors=None, logger=None, level="error", append_warning=False, warning=None): if logger is not None: @@ -15,42 +16,49 @@ def __init__(self, message, errors=None, logger=None, level="error", class JailDoesNotExist(IocageException): + def __init__(self, jail, *args, **kwargs): msg = f"Jail '{jail.humanreadable_name}' does not exist" super().__init__(msg, *args, **kwargs) class JailAlreadyExists(IocageException): + def __init__(self, jail, *args, **kwargs): msg = f"Jail '{jail.humanreadable_name}' already exists" super().__init__(msg, *args, **kwargs) class JailNotRunning(IocageException): + def __init__(self, jail, *args, **kwargs): msg = f"Jail '{jail.humanreadable_name}' is not running" super().__init__(msg, *args, **kwargs) class JailAlreadyRunning(IocageException): + def __init__(self, jail, *args, **kwargs): msg = f"Jail '{jail.humanreadable_name}' is already running" super().__init__(msg, *args, **kwargs) class JailNotFound(IocageException): + def __init__(self, text, *args, **kwargs): msg = f"No jail matching '{text}' was found" super().__init__(msg, *args, **kwargs) class JailNotSupplied(IocageException): + def __init__(self, *args, **kwargs): msg = f"Please supply a jail" super().__init__(msg, *args, **kwargs) class JailUnknownIdentifier(IocageException): + def __init__(self, *args, **kwargs): msg = "The jail has not identifier yet" super().__init__(msg, *args, **kwargs) @@ -64,6 +72,7 @@ class JailConfigError(IocageException): class InvalidJailName(JailConfigError): + def __init__(self, *args, **kwargs): msg = ( "Invalid jail name: " @@ -73,6 +82,7 @@ def __init__(self, *args, **kwargs): class JailConigZFSIsNotAllowed(JailConfigError): + def __init__(self, *args, **kwargs): msg = ( "jail_zfs is disabled" @@ -82,6 +92,7 @@ def __init__(self, *args, **kwargs): class InvalidJailConfigValue(JailConfigError): + def __init__(self, property_name, jail=None, reason=None, **kwargs): msg = f"Invalid value for property '{property_name}'" if jail is not None: @@ -92,6 +103,7 @@ def __init__(self, property_name, jail=None, reason=None, **kwargs): class InvalidJailConfigAddress(InvalidJailConfigValue): + def __init__(self, value, **kwargs): reason = f"expected \"|
\" but got \"{value}\"" super().__init__( @@ -101,6 +113,7 @@ def __init__(self, value, **kwargs): class JailConfigNotFound(IocageException): + def __init__(self, config_type, *args, **kwargs): msg = f"Could not read {config_type} config" # This is a silent error internally used @@ -108,6 +121,7 @@ def __init__(self, config_type, *args, **kwargs): class DefaultConfigNotFound(IocageException, FileNotFoundError): + def __init__(self, config_file_path, *args, **kwargs): msg = f"Default configuration not found at {config_file_path}" IocageException.__init__(self, msg, *args, **kwargs) @@ -117,6 +131,7 @@ def __init__(self, config_file_path, *args, **kwargs): class IocageNotActivated(IocageException): + def __init__(self, *args, **kwargs): msg = ( "iocage is not activated yet - " @@ -126,6 +141,7 @@ def __init__(self, *args, **kwargs): class MustBeRoot(IocageException): + def __init__(self, msg, *args, **kwargs): _msg = ( f"Must be root to {msg}" @@ -134,6 +150,7 @@ def __init__(self, msg, *args, **kwargs): class CommandFailure(IocageException): + def __init__(self, returncode, *args, **kwargs): msg = f"Command exited with {returncode}" super().__init__(msg, *args, **kwargs) @@ -143,6 +160,7 @@ def __init__(self, returncode, *args, **kwargs): class DistributionUnknown(IocageException): + def __init__(self, distribution_name, *args, **kwargs): msg = f"Unknown Distribution: {distribution_name}" super().__init__(msg, *args, **kwargs) @@ -152,30 +170,35 @@ def __init__(self, distribution_name, *args, **kwargs): class UnmountFailed(IocageException): + def __init__(self, mountpoint, *args, **kwargs): msg = f"Failed to unmount {mountpoint}" super().__init__(msg, *args, **kwargs) class MountFailed(IocageException): + def __init__(self, mountpoint, *args, **kwargs): msg = f"Failed to mount {mountpoint}" super().__init__(msg, *args, **kwargs) class DatasetNotMounted(IocageException): + def __init__(self, dataset, *args, **kwargs): msg = f"Dataset '{dataset.name}' is not mounted" super().__init__(msg, *args, **kwargs) class DatasetNotAvailable(IocageException): + def __init__(self, dataset_name, *args, **kwargs): msg = f"Dataset '{dataset_name}' is not available" super().__init__(msg, *args, **kwargs) class DatasetNotJailed(IocageException): + def __init__(self, dataset, *args, **kwargs): name = dataset.name msg = f"Dataset {name} is not jailed." @@ -185,6 +208,7 @@ def __init__(self, dataset, *args, **kwargs): class ZFSPoolInvalid(IocageException, TypeError): + def __init__(self, consequence=None, *args, **kwargs): msg = "Invalid ZFS pool" @@ -195,6 +219,7 @@ def __init__(self, consequence=None, *args, **kwargs): class ZFSPoolUnavailable(IocageException): + def __init__(self, pool_name, *args, **kwargs): msg = f"ZFS pool '{pool_name}' is UNAVAIL" super().__init__(msg, *args, **kwargs) @@ -204,12 +229,14 @@ def __init__(self, pool_name, *args, **kwargs): class VnetBridgeMissing(IocageException): + def __init__(self, *args, **kwargs): msg = "VNET is enabled and requires setting a bridge" super().__init__(msg, *args, **kwargs) class InvalidNetworkBridge(IocageException, ValueError): + def __init__(self, reason=None, *args, **kwargs): msg = "Invalid network bridge argument" if reason is not None: @@ -221,6 +248,7 @@ def __init__(self, reason=None, *args, **kwargs): class UnknownReleasePool(IocageException): + def __init__(self, *args, **kwargs): msg = ( "Cannot determine the ZFS pool without knowing" @@ -230,6 +258,7 @@ def __init__(self, *args, **kwargs): class ReleaseUpdateFailure(IocageException): + def __init__(self, release_name, reason=None, *args, **kwargs): msg = f"Release update of '{release_name}' failed" if reason is not None: @@ -238,24 +267,28 @@ def __init__(self, release_name, reason=None, *args, **kwargs): class InvalidReleaseAssetSignature(ReleaseUpdateFailure): + def __init__(self, release_name, asset_name, *args, **kwargs): msg = f"Asset {asset_name} has an invalid signature" super().__init__(release_name, reason=msg, *args, **kwargs) class IllegalReleaseAssetContent(ReleaseUpdateFailure): + def __init__(self, release_name, asset_name, reason, *args, **kwargs): msg = f"Asset {asset_name} contains illegal files - {reason}" super().__init__(release_name, reason=msg, *args, **kwargs) class ReleaseNotFetched(IocageException): + def __init__(self, name, *args, **kwargs): msg = f"Release '{name}' does not exist or is not fetched locally" super().__init__(msg, *args, **kwargs) class ReleaseUpdateBranchLookup(IocageException): + def __init__(self, release_name, reason=None, *args, **kwargs): msg = f"Update source of release '{release_name}' not found" if reason is not None: @@ -267,6 +300,7 @@ def __init__(self, release_name, reason=None, *args, **kwargs): class DefaultReleaseNotFound(IocageException): + def __init__(self, host_release_name, *args, **kwargs): msg = ( f"Release '{host_release_name}' not found: " @@ -279,11 +313,13 @@ def __init__(self, host_release_name, *args, **kwargs): class DevfsRuleException(IocageException): + def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) class InvalidDevfsRulesSyntax(DevfsRuleException): + def __init__(self, devfs_rules_file, reason=None, *args, **kwargs): msg = f"Invalid devfs rules in {devfs_rules_file}" if reason is not None: @@ -292,6 +328,7 @@ def __init__(self, devfs_rules_file, reason=None, *args, **kwargs): class DuplicateDevfsRuleset(DevfsRuleException): + def __init__(self, devfs_rules_file, reason=None, *args, **kwargs): msg = "Cannot add duplicate ruleset" if reason is not None: @@ -301,12 +338,15 @@ def __init__(self, devfs_rules_file, reason=None, *args, **kwargs): # Logger + class LogException(IocageException): + def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) class CannotRedrawLine(LogException): + def __init__(self, reason, *args, **kwargs): msg = "Logger can't redraw line" if reason is not None: @@ -314,9 +354,20 @@ def __init__(self, reason, *args, **kwargs): super().__init__(msg, *args, **kwargs) +# Events + + +class EventAlreadyFinished(IocageException): + + def __init__(self, event, *args, **kwargs): + msg = "This {event.type} event is already finished" + IocageException.__init__(self, msg, *args, **kwargs) + + # Missing Features class MissingFeature(IocageException, NotImplementedError): + def __init__(self, message, *args, **kwargs): super().__init__(message, *args, **kwargs) diff --git a/libiocage/lib/events.py b/libiocage/lib/events.py index c863167f..b5021621 100644 --- a/libiocage/lib/events.py +++ b/libiocage/lib/events.py @@ -1,4 +1,5 @@ from timeit import default_timer as timer +import libiocage.lib.errors EVENT_STATUS = ( "pending", @@ -49,10 +50,10 @@ def state(self): return self.get_state() def get_state_string(self, - error="failed", - skipped="skipped", - done="done", - pending="pending"): + error="failed", + skipped="skipped", + done="done", + pending="pending"): if self.error is not None: return error @@ -252,10 +253,12 @@ def __init__(self, release, **kwargs): class RunReleaseUpdate(ReleaseUpdate): + def __init__(self, release, **kwargs): ReleaseUpdate.__init__(self, release, **kwargs) class ExecuteReleaseUpdate(ReleaseUpdate): + def __init__(self, release, **kwargs): ReleaseUpdate.__init__(self, release, **kwargs) diff --git a/libiocage/lib/helpers.py b/libiocage/lib/helpers.py index 4de3e535..10f99775 100644 --- a/libiocage/lib/helpers.py +++ b/libiocage/lib/helpers.py @@ -31,7 +31,8 @@ def init_host(self, host=None): try: self.host = self._class_host(logger=logger) except: - self.host = libiocage.lib.Host.HostGenerator(logger=logger) + self.host = libiocage.lib.Host.HostGenerator(logger=logger) + def init_datasets(self, datasets=None): if datasets: From 76415282abaf84dd8b31de842acbca3597c00aed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=B6nke?= Date: Wed, 30 Aug 2017 19:46:27 +0200 Subject: [PATCH 06/16] improve event stack and release events --- libiocage/lib/Release.py | 5 +++++ libiocage/lib/events.py | 13 ++++++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/libiocage/lib/Release.py b/libiocage/lib/Release.py index 90992e7e..ab323b5d 100644 --- a/libiocage/lib/Release.py +++ b/libiocage/lib/Release.py @@ -266,6 +266,7 @@ def fetch(self, update=None, fetch_updates=None): releaseDownloadEvent = events.ReleaseDownload(self) releaseExtractionEvent = events.ReleaseExtraction(self) releaseConfigurationEvent = events.ReleaseConfiguration(self) + releaseCopyBaseEvent = events.ReleaseCopyBase(self) if not self.fetched: @@ -324,7 +325,11 @@ def fetch(self, update=None, fetch_updates=None): release_changed = event if release_changed: + yield releaseCopyBaseEvent.begin() self._update_zfs_base() + yield releaseCopyBaseEvent.end() + else: + yield releaseCopyBaseEvent.skip(message="release unchanged") self._cleanup() diff --git a/libiocage/lib/events.py b/libiocage/lib/events.py index b5021621..f4e3c06d 100644 --- a/libiocage/lib/events.py +++ b/libiocage/lib/events.py @@ -38,7 +38,7 @@ def __init__(self, message=None, **kwargs): self.data = kwargs self.number = len(IocageEvent.HISTORY) + 1 - self.parent_count = IocageEvent.PENDING_COUNT + 0 + self.parent_count = IocageEvent.PENDING_COUNT self.message = message @@ -111,6 +111,7 @@ def begin(self, **kwargs): self._update_message(**kwargs) self.pending = True self.done = False + self.parent_count = IocageEvent.PENDING_COUNT - 1 return self def end(self, **kwargs): @@ -118,22 +119,26 @@ def end(self, **kwargs): self.done = True self.pending = False self.done = True + self.parent_count = IocageEvent.PENDING_COUNT return self def step(self, **kwargs): self._update_message(**kwargs) + self.parent_count = IocageEvent.PENDING_COUNT return self def skip(self, **kwargs): self._update_message(**kwargs) self.skipped = True self.pending = False + self.parent_count = IocageEvent.PENDING_COUNT return self def fail(self, exception=True, **kwargs): self._update_message(**kwargs) self.error = exception self.pending = False + self.parent_count = IocageEvent.PENDING_COUNT return self def __hash__(self): @@ -234,6 +239,12 @@ def __init__(self, release, **kwargs): FetchRelease.__init__(self, release, **kwargs) +class ReleaseCopyBase(FetchRelease): + + def __init__(self, release, **kwargs): + FetchRelease.__init__(self, release, **kwargs) + + class ReleaseConfiguration(FetchRelease): def __init__(self, release, **kwargs): From 23f6a7030353a1288c4e67b899038efbd4710099 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=B6nke?= Date: Wed, 30 Aug 2017 19:59:06 +0200 Subject: [PATCH 07/16] globs for cli/start and cli/stop --- libiocage/cli/start.py | 9 ++++++++- libiocage/cli/stop.py | 10 ++++++++-- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/libiocage/cli/start.py b/libiocage/cli/start.py index 678f93eb..e08cc48d 100644 --- a/libiocage/cli/start.py +++ b/libiocage/cli/start.py @@ -47,13 +47,20 @@ def cli(ctx, rc, jails): filters=jails ) + failed_jails = [] for jail in ioc_jails: logger.log(f"Starting {jail.humanreadable_name}") try: ctx.parent.print_events(jail.start()) except Exception: + failed_jails.append(jail) + continue exit(1) logger.log(f"{jail.humanreadable_name} running as JID {jail.jid}") - exit(0) + + if len(failed_jails) > 0: + exit(1) + + exit(0) diff --git a/libiocage/cli/stop.py b/libiocage/cli/stop.py index 9d58eb72..7c7352f8 100644 --- a/libiocage/cli/stop.py +++ b/libiocage/cli/stop.py @@ -50,11 +50,17 @@ def cli(ctx, rc, log_level, force, jails): filters=jails ) + failed_jails = [] for jail in ioc_jails: try: ctx.parent.print_events(jail.stop(force=force)) except: - exit(1) + failed_jails.append(jail) + continue logger.log(f"{jail.name} stopped") - exit(0) + + if len(failed_jails) > 0: + exit(1) + + exit(0) From c767b5e458f738683ef9c632ff0733d7e3851ec3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=B6nke?= Date: Thu, 31 Aug 2017 14:17:11 +0200 Subject: [PATCH 08/16] properly handle multi-line log entries for dynamic output editing --- libiocage/lib/Logger.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/libiocage/lib/Logger.py b/libiocage/lib/Logger.py index 45a11852..d0bf93c8 100644 --- a/libiocage/lib/Logger.py +++ b/libiocage/lib/Logger.py @@ -30,6 +30,9 @@ def edit(self, message=None, indent=None): self.logger.redraw(self) + def __len__(self): + return len(self.message.split("\n")) + class Logger: @@ -158,17 +161,19 @@ def redraw(self, log_entry): ) ) - line_number = self.PRINT_HISTORY.index(log_entry) - delta = len(self.PRINT_HISTORY) - line_number - - # ToDo: Handle redrawing of multiline entries with different line count + # calculate the delta of messages printed since + i = self.PRINT_HISTORY.index(log_entry) + n = len(self.PRINT_HISTORY) + delta = sum(map(lambda i: len(self.PRINT_HISTORY[i]), range(i,n))) output = "".join([ + "\r", f"\033[{delta}F", # CPL - Cursor Previous Line "\r", # CR - Carriage Return - self._indent(log_entry.message, log_entry.indent), + self._indent(f"{log_entry.message}: {delta}", log_entry.indent), "\033[K", # EL - Erase in Line - "\n" * delta + "\n" * (delta), + "\r" ]) sys.stdout.write(output) From bd635d8a6a3eec7f265c4b0082c24f3f4c4f01cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=B6nke?= Date: Thu, 31 Aug 2017 15:10:13 +0000 Subject: [PATCH 09/16] improve globs and cli/commands --- libiocage/cli/__init__.py | 5 +-- libiocage/cli/create.py | 11 ++---- libiocage/cli/get.py | 2 +- libiocage/cli/list.py | 35 +++++++---------- libiocage/cli/set.py | 45 ++++++++++++++------- libiocage/cli/start.py | 2 - libiocage/cli/stop.py | 2 + libiocage/lib/Jail.py | 38 ++++++++++-------- libiocage/lib/JailConfig.py | 42 +++++++++++++++++++- libiocage/lib/JailConfigJSON.py | 13 +++++-- libiocage/lib/JailFilter.py | 69 ++++++++++++++++++++++++--------- libiocage/lib/Logger.py | 2 +- libiocage/lib/errors.py | 28 ++++++++++++- libiocage/lib/helpers.py | 17 ++++++-- 14 files changed, 215 insertions(+), 96 deletions(-) diff --git a/libiocage/cli/__init__.py b/libiocage/cli/__init__.py index 65b0c2fc..ebc8e702 100644 --- a/libiocage/cli/__init__.py +++ b/libiocage/cli/__init__.py @@ -85,8 +85,8 @@ def print_events(generator): else: output += event.get_state_string( done="OK", - error="failed", - skipped="skipped", + error="FAILED", + skipped="SKIPPED", pending="..." ) @@ -143,7 +143,6 @@ def get_command(self, ctx, name): pass return mod.cli except (ImportError, AttributeError): - raise return diff --git a/libiocage/cli/create.py b/libiocage/cli/create.py index b405c169..7f5176a4 100644 --- a/libiocage/cli/create.py +++ b/libiocage/cli/create.py @@ -49,6 +49,7 @@ def validate_count(ctx, param, value): @click.command(name="create", help="Create a jail.") +@click.pass_context @click.option("--count", "-c", callback=validate_count, default="1", help="Designate a number of jails to create. Jails are" " numbered sequentially.") @@ -76,17 +77,13 @@ def validate_count(ctx, param, value): help="Do not automatically fetch releases") @click.option("--force", "-f", is_flag=True, default=False, help="Skip the interactive question.") -@click.option("--log-level", "-d", default=None) @click.argument("props", nargs=-1) -def cli(release, template, count, props, pkglist, basejail, basejail_type, - empty, name, no_fetch, force, log_level): +def cli(ctx, release, template, count, props, pkglist, basejail, basejail_type, + empty, name, no_fetch, force): zfs = libiocage.lib.helpers.get_zfs() - logger = libiocage.lib.Logger.Logger() + logger = ctx.parent.logger host = libiocage.lib.Host.Host(logger=logger, zfs=zfs) - if log_level is not None: - logger.print_level = log_level - jail_data = {} if release is None: diff --git a/libiocage/cli/get.py b/libiocage/cli/get.py index 3a14037c..5009f0fc 100644 --- a/libiocage/cli/get.py +++ b/libiocage/cli/get.py @@ -64,7 +64,7 @@ def cli(ctx, prop, _all, _pool, jail, log_level): ) if not jail.exists: - logger.error("Jail '{jail}' does not exist") + logger.error(f"Jail '{jail.name}' does not exist") exit(1) if _all is True: diff --git a/libiocage/cli/list.py b/libiocage/cli/list.py index b42f81b7..38e11608 100644 --- a/libiocage/cli/list.py +++ b/libiocage/cli/list.py @@ -66,11 +66,11 @@ def cli(ctx, dataset_type, header, _long, remote, plugins, available_releases = host.distribution.releases for available_release in available_releases: - print(available_release.name) + logger.screen(available_release.name) return if plugins and remote: - raise Exception("ToDo: Plugins") + raise libiocage.lib.errors.MissingFeature("Plugins", plural=True) if output is not None and _long is True: logger.error("--output and --long can't be used together") @@ -81,6 +81,10 @@ def cli(ctx, dataset_type, header, _long, remote, plugins, # ToDo: Figure out if we need to sort other output formats as well raise Exception("Sorting only allowed for tables") + # empty filters will match all jails + if len(filters) == 0: + filters += ("*",) + jails = libiocage.lib.Jails.JailsGenerator( logger=logger, host=host, @@ -143,6 +147,13 @@ def _print_list( print(separator.join(_lookup_jail_values(jail, columns))) +def _lookup_jail_values(jail, columns) -> typing.List[str]: + return list(map( + lambda column: jail.getstring(column), + columns + )) + + def _list_output_comumns( user_input: str="", long_mode: bool=False @@ -168,23 +179,3 @@ def _list_output_comumns( ] return columns - - -def _lookup_jail_values( - jail: libiocage.lib.Jail.JailGenerator, - keys: str -) -> list: - - return [_lookup_jail_value(jail, x) for x in keys] - - -def _lookup_jail_value( - jail: libiocage.lib.Jail.JailGenerator, - key: str -) -> str: - - # ToDo: Move this into lib/Jails ? - if key in libiocage.lib.Jails.JailsGenerator.JAIL_KEYS: - return jail.getstring(key) - else: - return str(jail.config.__getitem__(key)) diff --git a/libiocage/cli/set.py b/libiocage/cli/set.py index 8caa1dd0..d16667c4 100644 --- a/libiocage/cli/set.py +++ b/libiocage/cli/set.py @@ -24,8 +24,9 @@ """set module for the cli.""" import click -import libiocage.lib.Jail +import libiocage.lib.Jails import libiocage.lib.Logger +import libiocage.lib.helpers __rootcmd__ = True @@ -34,25 +35,41 @@ max_content_width=400, ), name="set", help="Sets the specified property.") @click.pass_context @click.argument("props", nargs=-1) -@click.argument("jail", nargs=1) -@click.option("--log-level", "-d", default=None) -def cli(ctx, props, jail, log_level): +@click.argument("jail", nargs=1, required=True) +def cli(ctx, props, jail): """Get a list of jails and print the property.""" logger = ctx.parent.logger - logger.print_level = log_level - jail = libiocage.lib.Jail.Jail(jail, logger=logger) - for prop in props: + filters = (f"name={jail}",) + ioc_jails = libiocage.lib.Jails.JailsGenerator( + filters, + logger=logger + ) - if _is_setter_property(prop): - key, value = prop.split("=", maxsplit=1) - jail.config[key] = value - else: - key = prop - del jail.config[key] + for jail in ioc_jails: + + updated_properties = set() + + for prop in props: - jail.config.save() + if _is_setter_property(prop): + key, value = prop.split("=", maxsplit=1) + changed = jail.config.set(key, value) + if changed: + updated_properties.add(key) + else: + key = prop + del jail.config[key] + + if len(updated_properties) == 0: + logger.screen(f"Jail '{jail.humanreadable_name}' unchanged") + else: + logger.screen( + f"Jail '{jail.humanreadable_name}' updated: " + + ", ".join(updated_properties) + ) + jail.config.save() def _is_setter_property(property_string): diff --git a/libiocage/cli/start.py b/libiocage/cli/start.py index e08cc48d..34391638 100644 --- a/libiocage/cli/start.py +++ b/libiocage/cli/start.py @@ -49,14 +49,12 @@ def cli(ctx, rc, jails): failed_jails = [] for jail in ioc_jails: - logger.log(f"Starting {jail.humanreadable_name}") try: ctx.parent.print_events(jail.start()) except Exception: failed_jails.append(jail) continue - exit(1) logger.log(f"{jail.humanreadable_name} running as JID {jail.jid}") diff --git a/libiocage/cli/stop.py b/libiocage/cli/stop.py index 7c7352f8..f5031c98 100644 --- a/libiocage/cli/stop.py +++ b/libiocage/cli/stop.py @@ -50,6 +50,8 @@ def cli(ctx, rc, log_level, force, jails): filters=jails ) + print(ioc_jails) + failed_jails = [] for jail in ioc_jails: try: diff --git a/libiocage/lib/Jail.py b/libiocage/lib/Jail.py index 09de131f..2feff7e6 100644 --- a/libiocage/lib/Jail.py +++ b/libiocage/lib/Jail.py @@ -89,7 +89,9 @@ def __init__(self, data={}, zfs=None, host=None, logger=None, new=False): libiocage.lib.helpers.init_host(self, host) if isinstance(data, str): - data = {"id": self._resolve_name(data)} + data = { + "id": self._resolve_name(data) + } self.config = libiocage.lib.JailConfig.JailConfig( data=data, @@ -100,8 +102,12 @@ def __init__(self, data={}, zfs=None, host=None, logger=None, new=False): self.networks = [] self.storage = self._class_storage( - auto_create=True, safe_mode=False, - jail=self, logger=self.logger, zfs=self.zfs) + auto_create=True, + safe_mode=False, + jail=self, + logger=self.logger, + zfs=self.zfs + ) self.jail_state = None self._dataset_name = None @@ -741,17 +747,11 @@ def humanreadable_name(self): """ try: - uuid.UUID(self.name) - return str(self.name)[:8] - except (TypeError, ValueError): - pass - - try: - return self.name - except AttributeError: - pass - - raise libiocage.lib.errors.JailUnknownIdentifier(logger=self.logger) + return libiocage.lib.helpers.to_humanreadable_name(self.name) + except: + raise libiocage.lib.errors.JailUnknownIdentifier( + logger=self.logger + ) @property def stopped(self): @@ -877,11 +877,15 @@ def getstring(self, key): key (string): Name of the jail property to return """ + + try: + return libiocage.helpers.to_string(self.config[key]) + except: + pass + try: - if key == "jid" and self.__getattr__(key) is None: - return "-" + return libiocage.lib.helpers.to_string(self.__getattr__(key)) - return str(self.__getattr__(key)) except AttributeError: return "-" diff --git a/libiocage/lib/JailConfig.py b/libiocage/lib/JailConfig.py index b8ce4be8..3a9f4fca 100644 --- a/libiocage/lib/JailConfig.py +++ b/libiocage/lib/JailConfig.py @@ -551,15 +551,53 @@ def __setitem__(self, key, value, **kwargs): # except: # pass + parsed_value = libiocage.lib.helpers.parse_user_input(value) + setter_method = None try: setter_method = self.__getattribute__(f"_set_{key}") except: - self.data[key] = value + self.data[key] = parsed_value pass if setter_method is not None: - return setter_method(value, **kwargs) + return setter_method(parsed_value, **kwargs) + + def set(self, key: str, value, **kwargs) -> bool: + """ + Set a JailConfig property + + Args: + + key: + The jail config property name + + value: + Value to set the property to + + **kwargs: + Arguments from **kwargs are passed to setter functions + + Returns: + + bool: True if the JailConfig was changed + """ + + try: + hash_before = self.__getitem__(key).__hash__() + except KeyError: + hash_before = None + pass + + self.__setitem__(key, value, **kwargs) + + try: + hash_after = self.__getitem__(key).__hash__() + except KeyError: + hash_after = None + pass + + return (hash_before != hash_after) def __str__(self): return libiocage.lib.JailConfigJSON.JailConfigJSON.toJSON(self) diff --git a/libiocage/lib/JailConfigJSON.py b/libiocage/lib/JailConfigJSON.py index bef4eed6..a14d45a3 100644 --- a/libiocage/lib/JailConfigJSON.py +++ b/libiocage/lib/JailConfigJSON.py @@ -7,10 +7,15 @@ class JailConfigJSON: def toJSON(self): data = self.data - for key in data.keys(): - if data[key] is None: - data[key] = "none" - return json.dumps(data, sort_keys=True, indent=4) + output_data = {} + for key, value in data.items(): + output_data[key] = libiocage.lib.helpers.to_string( + value, + true="yes", + false="no", + none="none" + ) + return json.dumps(output_data, sort_keys=True, indent=4) def save(self): config_file_path = JailConfigJSON.__get_config_json_path(self) diff --git a/libiocage/lib/JailFilter.py b/libiocage/lib/JailFilter.py index 93eac231..4de10aa4 100644 --- a/libiocage/lib/JailFilter.py +++ b/libiocage/lib/JailFilter.py @@ -1,6 +1,7 @@ from typing import List, Union, Iterable import re import libiocage.lib.Jail +import libiocage.lib.errors def match_filter(value: str, filter_string: str): @@ -9,7 +10,6 @@ def match_filter(value: str, filter_string: str): filter_string = filter_string.replace(character, f"\\{character}") filter_string = filter_string.replace("*", ".*") filter_string = filter_string.replace("+", ".+") - print pattern = f"^{filter_string}$" match = re.match(pattern, value) return match is not None @@ -17,6 +17,8 @@ def match_filter(value: str, filter_string: str): class Term(list): + glob_characters = ["*", "+"] + def __init__(self, key, values=list()): self.key = key @@ -30,16 +32,38 @@ def __init__(self, key, values=list()): list.__init__(self, data) def matches_jail(self, jail: libiocage.lib.Jail.JailGenerator) -> bool: - jail_value = jail.getstring(self.key) - return self.matches(jail_value) + return self.matches(jail.getstring(self.key)) def matches(self, value: str) -> bool: """ Returns True if the value matches the term """ for filter_value in self: + + if self.key == "name": + if self._validate_name_filter_string(filter_value) is False: + raise libiocage.lib.errors.JailFilterInvalidName( + filter_value, + logger=self.logger + ) + if match_filter(value, filter_value): return True + + # match against humanreadable names as well + has_humanreadble_length = (len(filter_value) == 8) + has_no_globs = not self._filter_string_has_globs(filter_value) + if (has_humanreadble_length and has_no_globs) is True: + shortname = libiocage.lib.helpers.to_humanreadable_name(value) + if match_filter(shortname, filter_value): + return True + + return False + + def _filter_string_has_globs(self, filter_string: str) -> bool: + for glob in self.glob_characters: + if glob in filter_string: + return True return False def _split_filter_values(self, user_input: str) -> List[str]: @@ -59,6 +83,22 @@ def _split_filter_values(self, user_input: str) -> List[str]: values += block[1:] return values + def _validate_name_filter_string(self, filter_string: str) -> bool: + + globs = self.glob_characters + + # Allow glob only filters + if (len(filter_string) == 1) and (filter_string in globs): + return True + + # replace all glob charaters in user input + filter_string_without_globs = "" + for i, char in enumerate(filter_string): + if char not in globs: + filter_string_without_globs += char + + return libiocage.lib.helpers.validate_name(filter_string_without_globs) + class Terms(list): """ @@ -82,17 +122,11 @@ def __init__(self, terms: Iterable[Union[Term, str]]=None): list.__init__(self, data) - @property - def only_by_name(self) -> bool: - """ - True if all terms match by name only - """ - return all([x.key == "name" for x in self]) - def match_jail(self, jail: libiocage.lib.Jail.JailGenerator) -> bool: """ Returns True if all Terms match the jail """ + for term in self: if term.matches_jail(jail) is False: return False @@ -116,17 +150,16 @@ def match_key(self, key: str, value: str) -> bool: return True - def _parse_term(self, user_input: Iterable[str]) -> List[Term]: + def _parse_term(self, user_input: str) -> List[Term]: terms = [] - for user_term in user_input: - try: - prop, value = user_input.split("=", maxsplit=1) - except: - prop = "name" - value = user_input + try: + prop, value = user_input.split("=", maxsplit=1) + except: + prop = "name" + value = user_input - terms.append(Term(prop, value)) + terms.append(Term(prop, value)) return terms diff --git a/libiocage/lib/Logger.py b/libiocage/lib/Logger.py index d0bf93c8..c1198af5 100644 --- a/libiocage/lib/Logger.py +++ b/libiocage/lib/Logger.py @@ -164,7 +164,7 @@ def redraw(self, log_entry): # calculate the delta of messages printed since i = self.PRINT_HISTORY.index(log_entry) n = len(self.PRINT_HISTORY) - delta = sum(map(lambda i: len(self.PRINT_HISTORY[i]), range(i,n))) + delta = sum(map(lambda i: len(self.PRINT_HISTORY[i]), range(i, n))) output = "".join([ "\r", diff --git a/libiocage/lib/errors.py b/libiocage/lib/errors.py index c7f0df1d..89b3d2aa 100644 --- a/libiocage/lib/errors.py +++ b/libiocage/lib/errors.py @@ -364,10 +364,34 @@ def __init__(self, event, *args, **kwargs): IocageException.__init__(self, msg, *args, **kwargs) +# Jail Filter + + +class JailFilterException(IocageException): + + def __init__(self, *args, **kwargs): + IocageException.__init__(self, *args, **kwargs) + + +class JailFilterInvalidName(JailFilterException): + + def __init__(self, *args, **kwargs): + msg = ( + "Invalid jail selector: " + "Cannot select jail with illegal name" + ) + JailFilterException.__init__(self, msg, *args, **kwargs) + + # Missing Features class MissingFeature(IocageException, NotImplementedError): - def __init__(self, message, *args, **kwargs): - super().__init__(message, *args, **kwargs) + def __init__(self, feature_name: str, plural: bool=False, *args, **kwargs): + message = ( + f"Missing Feature: '{feature_name}' " + "are" if plural is True else "is" + " not implemented yet" + ) + IocageException.__init__(self, message, *args, **kwargs) diff --git a/libiocage/lib/helpers.py b/libiocage/lib/helpers.py index 10f99775..6e4b026e 100644 --- a/libiocage/lib/helpers.py +++ b/libiocage/lib/helpers.py @@ -1,5 +1,6 @@ import re import subprocess +import uuid import libzfs @@ -99,10 +100,20 @@ def _prettify_output(output): )) +def to_humanreadable_name(name: str) -> str: + try: + uuid.UUID(name) + return str(name)[:8] + except (TypeError, ValueError): + return name + + # helper function to validate names -def validate_name(name): - validate = re.compile(r'[a-z0-9][a-z0-9\.\-_]{0,31}', re.I) - return bool(validate.fullmatch(name)) +_validate_name = re.compile(r'[a-z0-9][a-z0-9\.\-_]{0,31}', re.I) + + +def validate_name(name: str): + return bool(_validate_name.fullmatch(name)) def _parse_none(data): From 4852eacc51fae59f419b66f9f503afde2e3e428b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=B6nke?= Date: Thu, 31 Aug 2017 17:13:27 +0000 Subject: [PATCH 10/16] cli/list can generate json output --- libiocage/cli/list.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/libiocage/cli/list.py b/libiocage/cli/list.py index 38e11608..0b494df9 100644 --- a/libiocage/cli/list.py +++ b/libiocage/cli/list.py @@ -23,6 +23,7 @@ # POSSIBILITY OF SUCH DAMAGE. """list module for the cli.""" import click +import json import texttable import typing @@ -31,7 +32,7 @@ import libiocage.lib.JailFilter import libiocage.lib.Logger -supported_output_formats = ['table', 'csv', 'list'] +supported_output_formats = ['table', 'csv', 'list', 'json'] @click.command(name="list", help="List a specified dataset type, by default" @@ -97,6 +98,8 @@ def cli(ctx, dataset_type, header, _long, remote, plugins, _print_list(jails, columns, header, "\t") elif output_format == "csv": _print_list(jails, columns, header, ";") + elif output_format == "json": + _print_json(jails, columns) else: _print_table(jails, columns, header, _sort) @@ -147,6 +150,26 @@ def _print_list( print(separator.join(_lookup_jail_values(jail, columns))) +def _print_json( + jails: typing.Generator[libiocage.lib.Jails.JailsGenerator, None, None], + columns: list, + **json_dumps_args +): + + if "indent" not in json_dumps_args.keys(): + json_dumps_args["indent"] = 2 + + if "sort_keys" not in json_dumps_args.keys(): + json_dumps_args["sort_keys"] = True + + output = [] + + for jail in jails: + output.append(dict(zip(columns, _lookup_jail_values(jail, columns)))) + + print(json.dumps(output, **json_dumps_args)) + + def _lookup_jail_values(jail, columns) -> typing.List[str]: return list(map( lambda column: jail.getstring(column), From 107bc48786b1cbd24a33b2fc6c685200ba3aa6c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=B6nke?= Date: Thu, 31 Aug 2017 18:02:15 +0000 Subject: [PATCH 11/16] utilize helpers.validatate_name in JailConfig --- libiocage/lib/JailConfig.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/libiocage/lib/JailConfig.py b/libiocage/lib/JailConfig.py index 3a9f4fca..180b171e 100644 --- a/libiocage/lib/JailConfig.py +++ b/libiocage/lib/JailConfig.py @@ -205,8 +205,7 @@ def _set_name(self, name, **kwargs): ) self.logger.error(msg) - name_pattern = f"^[A-z0-9]([A-z0-9\\._\\-]+[A-z0-9])*$" - if not re.match(name_pattern, name): + if libiocage.lib.helpers.validate_name(name) is False: raise libiocage.lib.errors.InvalidJailName(logger=self.logger) self.id = name From fb4c6d81a1edc421b87baf5c9ddf22a13fa36d52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=B6nke?= Date: Thu, 31 Aug 2017 19:11:52 +0000 Subject: [PATCH 12/16] improve id/name handling --- libiocage/lib/Jail.py | 33 +++++++++-------------- libiocage/lib/JailConfig.py | 41 +++++++++++++++++------------ libiocage/lib/JailConfigDefaults.py | 1 + libiocage/lib/Jails.py | 22 ++++++---------- libiocage/lib/Logger.py | 2 +- 5 files changed, 46 insertions(+), 53 deletions(-) diff --git a/libiocage/lib/Jail.py b/libiocage/lib/Jail.py index 2feff7e6..bb0683cb 100644 --- a/libiocage/lib/Jail.py +++ b/libiocage/lib/Jail.py @@ -711,22 +711,21 @@ def _teardown_mounts(self): ) def _resolve_name(self, text): + + if (text is None) or (len(text)==0): + raise libiocage.lib.errors.JailNotSupplied(logger=self.logger) + jails_dataset = self.host.datasets.jails - best_guess = "" + for dataset in list(jails_dataset.children): + dataset_name = dataset.name[(len(jails_dataset.name) + 1):] - if text == dataset_name: - # Exact match, immediately return - return dataset_name - elif dataset_name.startswith(text) and len(text) > len(best_guess): - best_guess = text - - if len(best_guess) > 0: - self.logger.debug(f"Resolved {text} to uuid {dataset_name}") - return best_guess + humanreadable_name = libiocage.lib.helpers.to_humanreadable_name( + dataset_name + ) - if not best_guess: - raise libiocage.lib.errors.JailNotSupplied(logger=self.logger) + if text in [dataset_name, humanreadable_name]: + return dataset_name raise libiocage.lib.errors.JailNotFound(text, logger=self.logger) @@ -906,14 +905,6 @@ class Jail(JailGenerator): def start(self, *args, **kwargs): return list(JailGenerator.start(self, *args, **kwargs)) - # libiocage.lib.helpers.print_event_generator( - # JailGenerator.start(self, *args, **kwargs), - # logger=self.logger - # ) def stop(self, *args, **kwargs): - return list(JailGenerator.start(self, *args, **kwargs)) - # libiocage.lib.helpers.print_event_generator( - # JailGenerator.stop(self, *args, **kwargs), - # logger=self.logger - # ) + return list(JailGenerator.stop(self, *args, **kwargs)) diff --git a/libiocage/lib/JailConfig.py b/libiocage/lib/JailConfig.py index 180b171e..9e74bef3 100644 --- a/libiocage/lib/JailConfig.py +++ b/libiocage/lib/JailConfig.py @@ -1,4 +1,5 @@ import re +import uuid import libiocage.lib.JailConfigAddresses import libiocage.lib.JailConfigDefaults @@ -73,15 +74,12 @@ def __init__(self, self.jail = None self.fstab = None - data_keys = data.keys() - - # the UUID is used in many other variables and needs to be set first - if "name" in data_keys: - self["name"] = data["name"] - elif "uuid" in data_keys: - self["name"] = data["uuid"] - else: - self["id"] = None + # the name is used in many other variables and needs to be set first + self["id"] = None + for key in ["id", "name", "uuid"]: + if key in data.keys(): + self["name"] = data[key] + break # be aware of iocage-legacy jails for migration try: @@ -129,8 +127,13 @@ def clone(self, data, skip_on_error=False): Passed to __setitem__ """ - for key in data: - self.__setitem__(key, data[key], skip_on_error=skip_on_error) + current_id = self["id"] + for key, value in data.items(): + + if (key in ["id", "name", "uuid"]) and (current_id is not None): + value = current_id + + self.__setitem__(key, value, skip_on_error=skip_on_error) def read(self): @@ -205,15 +208,19 @@ def _set_name(self, name, **kwargs): ) self.logger.error(msg) - if libiocage.lib.helpers.validate_name(name) is False: - raise libiocage.lib.errors.InvalidJailName(logger=self.logger) - - self.id = name + is_valid_name = libiocage.lib.helpers.validate_name(name) + if is_valid_name is True: + self["id"] = name + else: + try: + self["id"] = str(uuid.UUID(name)) # legacy support + except: + raise libiocage.lib.errors.InvalidJailName(logger=self.logger) try: - self.host_hostname + self["host_hostname"] except: - self.host_hostname = name + self["host_hostname"] = name self.logger.spam( f"Set jail name to {name}", diff --git a/libiocage/lib/JailConfigDefaults.py b/libiocage/lib/JailConfigDefaults.py index 0ac1adae..8df712de 100644 --- a/libiocage/lib/JailConfigDefaults.py +++ b/libiocage/lib/JailConfigDefaults.py @@ -7,6 +7,7 @@ class JailConfigDefaults(dict): DEFAULTS = { + "id": None, "basejail": False, "defaultrouter": None, "defaultrouter6": None, diff --git a/libiocage/lib/Jails.py b/libiocage/lib/Jails.py index b665eca1..1d4a0b30 100644 --- a/libiocage/lib/Jails.py +++ b/libiocage/lib/Jails.py @@ -47,13 +47,10 @@ def __iter__(self): yield jail def _create_jail(self, *args, **kwargs): - return libiocage.lib.Jail.JailGenerator( - *args, - logger=self.logger, - host=self.host, - zfs=self.zfs, - **kwargs - ) + kwargs["logger"] = self.logger + kwargs["host"] = self.host + kwargs["zfs"] = self.zfs + return libiocage.lib.Jail.Jail(*args,**kwargs) @property def filters(self): @@ -99,13 +96,10 @@ def _get_name_from_jail_dataset( class Jails(JailsGenerator): def _create_jail(self, *args, **kwargs): - return libiocage.lib.Jail.Jail( - *args, - logger=self.logger, - host=self.host, - zfs=self.zfs, - **kwargs - ) + kwargs["logger"] = self.logger + kwargs["host"] = self.host + kwargs["zfs"] = self.zfs + return libiocage.lib.Jail.Jail(*args,**kwargs) def __iter__(self): return list(JailsGenerator.__iter__(self)) diff --git a/libiocage/lib/Logger.py b/libiocage/lib/Logger.py index c1198af5..d3d34321 100644 --- a/libiocage/lib/Logger.py +++ b/libiocage/lib/Logger.py @@ -85,7 +85,7 @@ def __init__(self, print_level=None, log_directory="/var/log/iocage"): @property def default_print_level(self): - return "info" + return "spam" @property def print_level(self): From 18e4db1824df077f84abd98a9d05e131a0cc5a97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=B6nke?= Date: Thu, 31 Aug 2017 19:12:23 +0000 Subject: [PATCH 13/16] refactor cli/start exit --- libiocage/cli/start.py | 5 +---- libiocage/lib/Jail.py | 4 ++-- libiocage/lib/JailConfig.py | 2 +- libiocage/lib/Jails.py | 4 ++-- libiocage/lib/Logger.py | 2 +- 5 files changed, 7 insertions(+), 10 deletions(-) diff --git a/libiocage/cli/start.py b/libiocage/cli/start.py index 34391638..25c7d4d2 100644 --- a/libiocage/cli/start.py +++ b/libiocage/cli/start.py @@ -58,7 +58,4 @@ def cli(ctx, rc, jails): logger.log(f"{jail.humanreadable_name} running as JID {jail.jid}") - if len(failed_jails) > 0: - exit(1) - - exit(0) + exit(1) if len(failed_jails) > 0 else exit(0) diff --git a/libiocage/lib/Jail.py b/libiocage/lib/Jail.py index bb0683cb..aa5ab85d 100644 --- a/libiocage/lib/Jail.py +++ b/libiocage/lib/Jail.py @@ -712,13 +712,13 @@ def _teardown_mounts(self): def _resolve_name(self, text): - if (text is None) or (len(text)==0): + if (text is None) or (len(text) == 0): raise libiocage.lib.errors.JailNotSupplied(logger=self.logger) jails_dataset = self.host.datasets.jails for dataset in list(jails_dataset.children): - + dataset_name = dataset.name[(len(jails_dataset.name) + 1):] humanreadable_name = libiocage.lib.helpers.to_humanreadable_name( dataset_name diff --git a/libiocage/lib/JailConfig.py b/libiocage/lib/JailConfig.py index 9e74bef3..b90a3205 100644 --- a/libiocage/lib/JailConfig.py +++ b/libiocage/lib/JailConfig.py @@ -213,7 +213,7 @@ def _set_name(self, name, **kwargs): self["id"] = name else: try: - self["id"] = str(uuid.UUID(name)) # legacy support + self["id"] = str(uuid.UUID(name)) # legacy support except: raise libiocage.lib.errors.InvalidJailName(logger=self.logger) diff --git a/libiocage/lib/Jails.py b/libiocage/lib/Jails.py index 1d4a0b30..34cfb864 100644 --- a/libiocage/lib/Jails.py +++ b/libiocage/lib/Jails.py @@ -50,7 +50,7 @@ def _create_jail(self, *args, **kwargs): kwargs["logger"] = self.logger kwargs["host"] = self.host kwargs["zfs"] = self.zfs - return libiocage.lib.Jail.Jail(*args,**kwargs) + return libiocage.lib.Jail.Jail(*args, **kwargs) @property def filters(self): @@ -99,7 +99,7 @@ def _create_jail(self, *args, **kwargs): kwargs["logger"] = self.logger kwargs["host"] = self.host kwargs["zfs"] = self.zfs - return libiocage.lib.Jail.Jail(*args,**kwargs) + return libiocage.lib.Jail.Jail(*args, **kwargs) def __iter__(self): return list(JailsGenerator.__iter__(self)) diff --git a/libiocage/lib/Logger.py b/libiocage/lib/Logger.py index d3d34321..c1198af5 100644 --- a/libiocage/lib/Logger.py +++ b/libiocage/lib/Logger.py @@ -85,7 +85,7 @@ def __init__(self, print_level=None, log_directory="/var/log/iocage"): @property def default_print_level(self): - return "spam" + return "info" @property def print_level(self): From 58be9260d64016d7615f5cc48d4e330133fbc1a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=B6nke?= Date: Thu, 31 Aug 2017 22:54:39 +0000 Subject: [PATCH 14/16] fix typo in errors.JailUnknownIdentifier --- libiocage/lib/errors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libiocage/lib/errors.py b/libiocage/lib/errors.py index 89b3d2aa..fda38a19 100644 --- a/libiocage/lib/errors.py +++ b/libiocage/lib/errors.py @@ -60,7 +60,7 @@ def __init__(self, *args, **kwargs): class JailUnknownIdentifier(IocageException): def __init__(self, *args, **kwargs): - msg = "The jail has not identifier yet" + msg = "The jail has no identifier yet" super().__init__(msg, *args, **kwargs) From 6d92b92b2804cfa76cc885ad6466784bbb1609a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=B6nke?= Date: Thu, 31 Aug 2017 23:10:31 +0000 Subject: [PATCH 15/16] fix an error showing up in the logs when no custom hostname was set --- libiocage/lib/Jail.py | 1 - libiocage/lib/JailConfig.py | 17 +---------------- 2 files changed, 1 insertion(+), 17 deletions(-) diff --git a/libiocage/lib/Jail.py b/libiocage/lib/Jail.py index aa5ab85d..f5c16648 100644 --- a/libiocage/lib/Jail.py +++ b/libiocage/lib/Jail.py @@ -744,7 +744,6 @@ def humanreadable_name(self): Whenever a Jail is found to have a UUID as identifier, a shortened string of the first 8 characters is returned """ - try: return libiocage.lib.helpers.to_humanreadable_name(self.name) except: diff --git a/libiocage/lib/JailConfig.py b/libiocage/lib/JailConfig.py index b90a3205..8897d505 100644 --- a/libiocage/lib/JailConfig.py +++ b/libiocage/lib/JailConfig.py @@ -217,11 +217,6 @@ def _set_name(self, name, **kwargs): except: raise libiocage.lib.errors.InvalidJailName(logger=self.logger) - try: - self["host_hostname"] - except: - self["host_hostname"] = name - self.logger.spam( f"Set jail name to {name}", jail=self.jail @@ -638,17 +633,7 @@ def all_properties(self): return list(properties) def stringify(self, value, enabled=True): - - if not enabled: - return value - elif value is None: - return "-" - elif value is True: - return "on" - elif value is False: - return "off" - else: - return str(value) + return libiocage.helpers.to_string if (enabled is True) else value class JailConfigList(list): From 70a8a4eb949b815127a65bcf46420b064214ce42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=B6nke?= Date: Thu, 31 Aug 2017 23:46:02 +0000 Subject: [PATCH 16/16] refactoring generators branch --- libiocage/cli/stop.py | 7 +------ libiocage/lib/Jail.py | 28 ++++++++++++++++------------ libiocage/lib/Jails.py | 3 +-- 3 files changed, 18 insertions(+), 20 deletions(-) diff --git a/libiocage/cli/stop.py b/libiocage/cli/stop.py index f5031c98..8fcc7078 100644 --- a/libiocage/cli/stop.py +++ b/libiocage/cli/stop.py @@ -50,8 +50,6 @@ def cli(ctx, rc, log_level, force, jails): filters=jails ) - print(ioc_jails) - failed_jails = [] for jail in ioc_jails: try: @@ -62,7 +60,4 @@ def cli(ctx, rc, log_level, force, jails): logger.log(f"{jail.name} stopped") - if len(failed_jails) > 0: - exit(1) - - exit(0) + exit(1) if len(failed_jails) > 0 else exit(0) diff --git a/libiocage/lib/Jail.py b/libiocage/lib/Jail.py index f5c16648..a719fdf1 100644 --- a/libiocage/lib/Jail.py +++ b/libiocage/lib/Jail.py @@ -159,18 +159,8 @@ def start(self): JailZfsShareMount = events.JailZfsShareMount(jail=self) jailServicesStartEvent = events.JailServicesStart(jail=self) - # Determine backend - - backend = None - - if self.config["basejail_type"] == "zfs": - backend = libiocage.lib.ZFSBasejailStorage.ZFSBasejailStorage - - if self.config["basejail_type"] == "nullfs": - backend = libiocage.lib.NullFSBasejailStorage.NullFSBasejailStorage - - if backend is not None: - backend.apply(self.storage, release) + if self.basejail_backend is not None: + self.basejail_backend.apply(self.storage, release) yield jailLaunchEvent.begin() @@ -200,6 +190,20 @@ def start(self): self._start_services() yield jailServicesStartEvent.end() + @property + def basejail_backend(self): + + if self.config["basejail"] is False: + return None + + if self.config["basejail_type"] == "nullfs": + return libiocage.lib.NullFSBasejailStorage.NullFSBasejailStorage + + if self.config["basejail_type"] == "zfs": + return libiocage.lib.ZFSBasejailStorage.ZFSBasejailStorage + + return None + def _start_services(self): command = self.config["exec_start"].strip().split() self.logger.debug(f"Running exec_start on {self.humanreadable_name}") diff --git a/libiocage/lib/Jails.py b/libiocage/lib/Jails.py index 34cfb864..4351258a 100644 --- a/libiocage/lib/Jails.py +++ b/libiocage/lib/Jails.py @@ -82,8 +82,7 @@ def _load_jail_from_dataset( return self._create_jail({ "name": self._get_name_from_jail_dataset(dataset) - } - ) + }) def _get_name_from_jail_dataset( self,