diff --git a/libiocage/cli/__init__.py b/libiocage/cli/__init__.py index 3aa4f370..ebc8e702 100644 --- a/libiocage/cli/__init__.py +++ b/libiocage/cli/__init__.py @@ -60,6 +60,53 @@ exit(1) +def print_events(generator): + lines = {} + for event in generator: + + if event.identifier is None: + identifier = "generic" + else: + identifier = event.identifier + + if event.type not in lines: + lines[event.type] = {} + + # output fragments + running_indicator = "+" if (event.done or event.skipped) else "-" + name = event.type + if event.identifier is not None: + name += f"@{event.identifier}" + + output = f"[{running_indicator}] {name}: " + + if event.message is not None: + output += event.message + else: + output += event.get_state_string( + done="OK", + error="FAILED", + skipped="SKIPPED", + pending="..." + ) + + if event.duration is not None: + output += " [" + str(round(event.duration, 3)) + "s]" + + # new line or update of previous + if identifier not in lines[event.type]: + # Indent if previous task is not finished + lines[event.type][identifier] = logger.screen( + output, + indent=event.parent_count + ) + else: + lines[event.type][identifier].edit( + output, + indent=event.parent_count + ) + + class IOCageCLI(click.MultiCommand): """ Iterates in the 'cli' directory and will load any module's cli definition. @@ -77,7 +124,7 @@ def list_commands(self, ctx): return rv def get_command(self, ctx, name): - ctx.logger = logger + ctx.print_events = print_events try: mod = __import__(f"libiocage.cli.{name}", None, None, ["cli"]) @@ -99,8 +146,12 @@ def get_command(self, ctx, name): return +@click.option("--log-level", "-d", default=None) @click.command(cls=IOCageCLI) -@click.version_option(version="0.9.10 07/30/2017", prog_name="iocage", +@click.version_option(version="0.2.11 08/29/2017", prog_name="ioc", message="%(version)s") -def cli(): +@click.pass_context +def cli(ctx, log_level): """A jail manager.""" + logger.print_level = log_level + ctx.logger = logger diff --git a/libiocage/cli/create.py b/libiocage/cli/create.py index b405c169..7f5176a4 100644 --- a/libiocage/cli/create.py +++ b/libiocage/cli/create.py @@ -49,6 +49,7 @@ def validate_count(ctx, param, value): @click.command(name="create", help="Create a jail.") +@click.pass_context @click.option("--count", "-c", callback=validate_count, default="1", help="Designate a number of jails to create. Jails are" " numbered sequentially.") @@ -76,17 +77,13 @@ def validate_count(ctx, param, value): help="Do not automatically fetch releases") @click.option("--force", "-f", is_flag=True, default=False, help="Skip the interactive question.") -@click.option("--log-level", "-d", default=None) @click.argument("props", nargs=-1) -def cli(release, template, count, props, pkglist, basejail, basejail_type, - empty, name, no_fetch, force, log_level): +def cli(ctx, release, template, count, props, pkglist, basejail, basejail_type, + empty, name, no_fetch, force): zfs = libiocage.lib.helpers.get_zfs() - logger = libiocage.lib.Logger.Logger() + logger = ctx.parent.logger host = libiocage.lib.Host.Host(logger=logger, zfs=zfs) - if log_level is not None: - logger.print_level = log_level - jail_data = {} if release is None: diff --git a/libiocage/cli/fetch.py b/libiocage/cli/fetch.py index 1ee8d34c..7fd6f997 100644 --- a/libiocage/cli/fetch.py +++ b/libiocage/cli/fetch.py @@ -25,26 +25,13 @@ import click import libiocage.lib.Host -import libiocage.lib.Logger import libiocage.lib.Prompts import libiocage.lib.Release import libiocage.lib.errors -__rootcmd__ = True +__rootcmd__ = True -# ToDo: remove disabled feature -# def _prettify_release_names(x): -# if x.name == host.release_version: -# return f"\033[1m{x.name}\033[0m" -# else: -# return x.name -# def release_choice(): -# version = -# return click.Choice(list(map( -# _prettify_release_names, -# host.distribution.releases -# ))) @click.command(context_settings=dict( max_content_width=400, ), @@ -59,8 +46,7 @@ # type=release_choice(), help="The FreeBSD release to fetch.") @click.option("--update/--no-update", "-U/-NU", default=True, - help="Decide whether or not to update the fetch to the latest " - "patch level.") + help="Update the release to the latest patch level.") @click.option("--fetch-updates/--no-fetch-updates", default=True, help="Skip fetching release updates") # Compat @@ -70,16 +56,8 @@ @click.option("--files", multiple=True, help="Specify the files to fetch from the mirror. " "(Deprecared: renamed to --file)") -@click.option("--log-level", "-d", default=None) -# @click.option("--auth", "-a", default=None, help="Authentication method for " -# "HTTP fetching. Valid " -# "values: basic, digest") -# @click.option("--verify/--noverify", "-V/-NV", default=True, -# help="Enable or disable verifying SSL cert for HTTP fetching.") -# def cli(url, files, release, update): def cli(ctx, **kwargs): logger = ctx.parent.logger - logger.print_level = kwargs["log_level"] host = libiocage.lib.Host.Host(logger=logger) prompts = libiocage.lib.Prompts.Prompts(host=host, logger=logger) @@ -91,7 +69,7 @@ def cli(ctx, **kwargs): exit(1) else: try: - release = libiocage.lib.Release.Release( + release = libiocage.lib.Release.ReleaseGenerator( name=release_input, host=host, logger=logger @@ -100,9 +78,6 @@ def cli(ctx, **kwargs): logger.error(f"Invalid Release '{release_input}'") exit(1) - if kwargs["log_level"] is not None: - logger.print_level = kwargs["log_level"] - url_or_files_selected = False if is_option_enabled(kwargs, "url"): @@ -117,28 +92,12 @@ def cli(ctx, **kwargs): logger.error(f"The release '{release.name}' is not available") exit(1) - if release.fetched: - msg = f"Release '{release.name}' is already fetched" - if kwargs["update"] is True: - logger.log(f"{msg} - updating only") - else: - logger.log(f"{msg} - skipping download and updates") - exit(0) - else: - logger.log( - f"Fetching release '{release.name}' from '{release.mirror_url}'" - ) - release.fetch(update=False, fetch_updates=False) - - if kwargs["fetch_updates"] is True: - logger.log("Fetching updates") - release.fetch_updates() - - if kwargs["update"] is True: - logger.log("Updating release") - release.update() + fetch_updates = bool(kwargs["fetch_updates"]) + ctx.parent.print_events(release.fetch( + update=kwargs["update"], + fetch_updates=fetch_updates + )) - logger.log('done') exit(0) diff --git a/libiocage/cli/get.py b/libiocage/cli/get.py index 3a14037c..5009f0fc 100644 --- a/libiocage/cli/get.py +++ b/libiocage/cli/get.py @@ -64,7 +64,7 @@ def cli(ctx, prop, _all, _pool, jail, log_level): ) if not jail.exists: - logger.error("Jail '{jail}' does not exist") + logger.error(f"Jail '{jail.name}' does not exist") exit(1) if _all is True: diff --git a/libiocage/cli/list.py b/libiocage/cli/list.py index 8493997f..0b494df9 100644 --- a/libiocage/cli/list.py +++ b/libiocage/cli/list.py @@ -23,12 +23,17 @@ # POSSIBILITY OF SUCH DAMAGE. """list module for the cli.""" import click +import json import texttable +import typing import libiocage.lib.Host import libiocage.lib.Jails +import libiocage.lib.JailFilter import libiocage.lib.Logger +supported_output_formats = ['table', 'csv', 'list', 'json'] + @click.command(name="list", help="List a specified dataset type, by default" " lists all jails.") @@ -37,83 +42,163 @@ flag_value="base", help="List all bases.") @click.option("--template", "-t", "dataset_type", flag_value="template", help="List all templates.") -@click.option("--header", "-h", "-H", is_flag=True, default=False, - help="For scripting, use tabs for separators.") @click.option("--long", "-l", "_long", is_flag=True, default=False, help="Show the full uuid and ip4 address.") @click.option("--remote", "-R", is_flag=True, help="Show remote's available " "RELEASEs.") @click.option("--plugins", "-P", is_flag=True, help="Show available plugins.") -@click.option("--sort", "-s", "_sort", default="name", nargs=1, +@click.option("--sort", "-s", "_sort", default=None, nargs=1, help="Sorts the list by the given type") @click.option("--quick", "-q", is_flag=True, default=False, help="Lists all jails with less processing and fields.") -@click.option("--log-level", "-d", default="info") @click.option("--output", "-o", default=None) +@click.option("--output-format", "-f", default="table", + type=click.Choice(supported_output_formats)) +@click.option("--header/--no-header", "-H/-NH", is_flag=True, default=True, + help="Show or hide column name heading.") @click.argument("filters", nargs=-1) def cli(ctx, dataset_type, header, _long, remote, plugins, - _sort, quick, log_level, output, filters): + _sort, quick, output, output_format, filters): logger = ctx.parent.logger - logger.print_level = log_level host = libiocage.lib.Host.Host(logger=logger) - jails = libiocage.lib.Jails.Jails(logger=logger) if remote and not plugins: available_releases = host.distribution.releases for available_release in available_releases: - print(available_release.name) + logger.screen(available_release.name) return if plugins and remote: - raise Exception("ToDo: Plugins") + raise libiocage.lib.errors.MissingFeature("Plugins", plural=True) + + if output is not None and _long is True: + logger.error("--output and --long can't be used together") + exit(1) + + if output_format != "table" and _sort is not None: + # Sorting destroys the ability to stream generators + # ToDo: Figure out if we need to sort other output formats as well + raise Exception("Sorting only allowed for tables") + + # empty filters will match all jails + if len(filters) == 0: + filters += ("*",) + + jails = libiocage.lib.Jails.JailsGenerator( + logger=logger, + host=host, + filters=filters # ToDo: allow quoted whitespaces from user input + ) + + columns = _list_output_comumns(output, _long) + + if output_format == "list": + _print_list(jails, columns, header, "\t") + elif output_format == "csv": + _print_list(jails, columns, header, ";") + elif output_format == "json": + _print_json(jails, columns) else: + _print_table(jails, columns, header, _sort) - if output: - columns = output.strip().split(',') - else: - columns = ["jid", "name"] - if _long: - columns += ["running", - "release", "ip4.addr", "ip6.addr"] - else: - columns += ["running", "release", "ip4.addr"] +def _print_table( + jails: typing.Generator[libiocage.lib.Jails.JailsGenerator, None, None], + columns: list, + show_header: bool, + sort_key: str=None +) -> None: - table = texttable.Texttable(max_width=0) - table.set_cols_dtype(["t"] * len(columns)) + table = texttable.Texttable(max_width=0) + table.set_cols_dtype(["t"] * len(columns)) - table_head = (list(x.upper() for x in columns)) - table_data = [] + table_head = (list(x.upper() for x in columns)) + table_data = [] - try: - sort_index = columns.index(_sort) - except ValueError: - sort_index = None + try: + sort_index = columns.index(sort_key) + except ValueError: + sort_index = None - for jail in jails.list(filters=filters): - table_data.append( - [_lookup_jail_value(jail, x) for x in columns] - ) + for jail in jails: + table_data.append(_lookup_jail_values(jail, columns)) - if sort_index is not None: - table_data.sort(key=lambda x: x[sort_index]) + if sort_index is not None: + table_data.sort(key=lambda x: x[sort_index]) + if show_header: table.add_rows([table_head] + table_data) + else: + table.add_rows(table_data) - if header: - # TODO: This sucks since it's a protected member - for item in table._rows: - print("\t".join(item)) - else: - print(table.draw()) + print(table.draw()) + + +def _print_list( + jails: typing.Generator[libiocage.lib.Jails.JailsGenerator, None, None], + columns: list, + show_header: bool, + separator: str=";" +) -> None: + + if show_header is True: + print(separator.join(columns).upper()) + + for jail in jails: + print(separator.join(_lookup_jail_values(jail, columns))) + + +def _print_json( + jails: typing.Generator[libiocage.lib.Jails.JailsGenerator, None, None], + columns: list, + **json_dumps_args +): - return + if "indent" not in json_dumps_args.keys(): + json_dumps_args["indent"] = 2 + if "sort_keys" not in json_dumps_args.keys(): + json_dumps_args["sort_keys"] = True -def _lookup_jail_value(jail, key): - if key in libiocage.lib.Jails.Jails.JAIL_KEYS: - return jail.getstring(key) + output = [] + + for jail in jails: + output.append(dict(zip(columns, _lookup_jail_values(jail, columns)))) + + print(json.dumps(output, **json_dumps_args)) + + +def _lookup_jail_values(jail, columns) -> typing.List[str]: + return list(map( + lambda column: jail.getstring(column), + columns + )) + + +def _list_output_comumns( + user_input: str="", + long_mode: bool=False +) -> list: + + if user_input: + return user_input.strip().split(',') else: - return str(jail.config.__getitem__(key)) + columns = ["jid", "name"] + + if long_mode is True: + columns += [ + "running", + "release", + "ip4.addr", + "ip6.addr" + ] + else: + columns += [ + "running", + "release", + "ip4.addr" + ] + + return columns diff --git a/libiocage/cli/set.py b/libiocage/cli/set.py index 8caa1dd0..d16667c4 100644 --- a/libiocage/cli/set.py +++ b/libiocage/cli/set.py @@ -24,8 +24,9 @@ """set module for the cli.""" import click -import libiocage.lib.Jail +import libiocage.lib.Jails import libiocage.lib.Logger +import libiocage.lib.helpers __rootcmd__ = True @@ -34,25 +35,41 @@ max_content_width=400, ), name="set", help="Sets the specified property.") @click.pass_context @click.argument("props", nargs=-1) -@click.argument("jail", nargs=1) -@click.option("--log-level", "-d", default=None) -def cli(ctx, props, jail, log_level): +@click.argument("jail", nargs=1, required=True) +def cli(ctx, props, jail): """Get a list of jails and print the property.""" logger = ctx.parent.logger - logger.print_level = log_level - jail = libiocage.lib.Jail.Jail(jail, logger=logger) - for prop in props: + filters = (f"name={jail}",) + ioc_jails = libiocage.lib.Jails.JailsGenerator( + filters, + logger=logger + ) - if _is_setter_property(prop): - key, value = prop.split("=", maxsplit=1) - jail.config[key] = value - else: - key = prop - del jail.config[key] + for jail in ioc_jails: + + updated_properties = set() + + for prop in props: - jail.config.save() + if _is_setter_property(prop): + key, value = prop.split("=", maxsplit=1) + changed = jail.config.set(key, value) + if changed: + updated_properties.add(key) + else: + key = prop + del jail.config[key] + + if len(updated_properties) == 0: + logger.screen(f"Jail '{jail.humanreadable_name}' unchanged") + else: + logger.screen( + f"Jail '{jail.humanreadable_name}' updated: " + + ", ".join(updated_properties) + ) + jail.config.save() def _is_setter_property(property_string): diff --git a/libiocage/cli/start.py b/libiocage/cli/start.py index 2f8a95c8..25c7d4d2 100644 --- a/libiocage/cli/start.py +++ b/libiocage/cli/start.py @@ -35,23 +35,27 @@ @click.option("--rc", default=False, is_flag=True, help="Will start all jails with boot=on, in the specified" " order with smaller value for priority starting first.") -@click.option("--log-level", "-d", default=None) @click.argument("jails", nargs=-1) -def cli(ctx, rc, jails, log_level): +def cli(ctx, rc, jails): """ Starts Jails """ logger = ctx.parent.logger - logger.print_level = log_level - ioc_jails = libiocage.lib.Jails.Jails(logger=logger) + ioc_jails = libiocage.lib.Jails.JailsGenerator( + logger=logger, + filters=jails + ) - for jail in ioc_jails.list(filters=jails): - logger.log(f"Starting {jail.humanreadable_name}") + failed_jails = [] + for jail in ioc_jails: try: - jail.start() + ctx.parent.print_events(jail.start()) + except Exception: - exit(1) + failed_jails.append(jail) + continue logger.log(f"{jail.humanreadable_name} running as JID {jail.jid}") - exit(0) + + exit(1) if len(failed_jails) > 0 else exit(0) diff --git a/libiocage/cli/stop.py b/libiocage/cli/stop.py index a9563cc0..8fcc7078 100644 --- a/libiocage/cli/stop.py +++ b/libiocage/cli/stop.py @@ -45,15 +45,19 @@ def cli(ctx, rc, log_level, force, jails): location to stop_jail. """ logger = ctx.parent.logger - logger.print_level = log_level - ioc_jails = libiocage.lib.Jails.Jails(logger=logger) + ioc_jails = libiocage.lib.Jails.JailsGenerator( + logger=logger, + filters=jails + ) - for jail in ioc_jails.list(filters=jails): - logger.log(f"Stopping jail {jail.humanreadable_name}") + failed_jails = [] + for jail in ioc_jails: try: - jail.stop(force=force) + ctx.parent.print_events(jail.stop(force=force)) except: - exit(1) + failed_jails.append(jail) + continue - logger.log("done") - exit(0) + logger.log(f"{jail.name} stopped") + + exit(1) if len(failed_jails) > 0 else exit(0) diff --git a/libiocage/lib/Distribution.py b/libiocage/lib/Distribution.py index 0aefa5bb..a387b49a 100644 --- a/libiocage/lib/Distribution.py +++ b/libiocage/lib/Distribution.py @@ -11,7 +11,10 @@ import libiocage.lib.helpers -class Distribution: +class DistributionGenerator: + + _class_release = libiocage.lib.Release.ReleaseGenerator + release_name_blacklist = [ "", ".", @@ -79,7 +82,7 @@ def fetch_releases(self): ) self.available_releases = list(map( - lambda x: libiocage.lib.Release.Release( + lambda x: self._class_release( name=x, host=self.host, zfs=self.zfs, @@ -170,3 +173,8 @@ def _parse_links(self, text): ) return matches + + +class Distribution(DistributionGenerator): + + _class_release = libiocage.lib.Release.Release diff --git a/libiocage/lib/Host.py b/libiocage/lib/Host.py index 59715898..7e03bdbc 100644 --- a/libiocage/lib/Host.py +++ b/libiocage/lib/Host.py @@ -7,7 +7,10 @@ import libiocage.lib.helpers -class Host: +class HostGenerator: + + _class_distribution = libiocage.lib.Distribution.DistributionGenerator + def __init__(self, root_dataset=None, zfs=None, logger=None): libiocage.lib.helpers.init_logger(self, logger) @@ -17,7 +20,7 @@ def __init__(self, root_dataset=None, zfs=None, logger=None): logger=self.logger, zfs=self.zfs ) - self.distribution = libiocage.lib.Distribution.Distribution( + self.distribution = self._class_distribution( host=self, logger=self.logger ) @@ -61,3 +64,8 @@ def release_version(self): @property def processor(self): return platform.processor() + + +class Host(HostGenerator): + + class_distribution = libiocage.lib.Distribution.DistributionGenerator diff --git a/libiocage/lib/Jail.py b/libiocage/lib/Jail.py index 716f74ca..a719fdf1 100644 --- a/libiocage/lib/Jail.py +++ b/libiocage/lib/Jail.py @@ -14,10 +14,11 @@ import libiocage.lib.ZFSBasejailStorage import libiocage.lib.ZFSShareStorage import libiocage.lib.errors +import libiocage.lib.events import libiocage.lib.helpers -class Jail: +class JailGenerator: """ iocage unit orchestrates a jail's configuration and manages state @@ -60,6 +61,9 @@ class Jail: release (stored in `zpool/iocage/base/) """ + _class_host = libiocage.lib.Host.HostGenerator + _class_storage = libiocage.lib.Storage.Storage + def __init__(self, data={}, zfs=None, host=None, logger=None, new=False): """ Initializes a Jail @@ -85,7 +89,9 @@ def __init__(self, data={}, zfs=None, host=None, logger=None, new=False): libiocage.lib.helpers.init_host(self, host) if isinstance(data, str): - data = {"id": self._resolve_name(data)} + data = { + "id": self._resolve_name(data) + } self.config = libiocage.lib.JailConfig.JailConfig( data=data, @@ -95,9 +101,13 @@ def __init__(self, data={}, zfs=None, host=None, logger=None, new=False): self.networks = [] - self.storage = libiocage.lib.Storage.Storage( - auto_create=True, safe_mode=False, - jail=self, logger=self.logger, zfs=self.zfs) + self.storage = self._class_storage( + auto_create=True, + safe_mode=False, + jail=self, + logger=self.logger, + zfs=self.zfs + ) self.jail_state = None self._dataset_name = None @@ -143,31 +153,61 @@ def start(self): release = self.release - backend = None - - if self.config["basejail_type"] == "zfs": - backend = libiocage.lib.ZFSBasejailStorage.ZFSBasejailStorage + events = libiocage.lib.events + jailLaunchEvent = events.JailLaunch(jail=self) + jailVnetConfigurationEvent = events.JailVnetConfiguration(jail=self) + JailZfsShareMount = events.JailZfsShareMount(jail=self) + jailServicesStartEvent = events.JailServicesStart(jail=self) - if self.config["basejail_type"] == "nullfs": - backend = libiocage.lib.NullFSBasejailStorage.NullFSBasejailStorage + if self.basejail_backend is not None: + self.basejail_backend.apply(self.storage, release) - if backend is not None: - backend.apply(self.storage, release) + yield jailLaunchEvent.begin() self.config.fstab.read_file() self.config.fstab.save_with_basedirs() self._launch_jail() + yield jailLaunchEvent.end() + if self.config["vnet"]: + yield jailVnetConfigurationEvent.begin() self._start_vimage_network() self._configure_routes() + yield jailVnetConfigurationEvent.end() self._configure_nameserver() if self.config["jail_zfs"] is True: + yield JailZfsShareMount.begin() libiocage.lib.ZFSShareStorage.ZFSShareStorage.mount_zfs_shares( self.storage ) + yield JailZfsShareMount.end() + + if self.config["exec_start"] is not None: + yield jailServicesStartEvent.begin() + self._start_services() + yield jailServicesStartEvent.end() + + @property + def basejail_backend(self): + + if self.config["basejail"] is False: + return None + + if self.config["basejail_type"] == "nullfs": + return libiocage.lib.NullFSBasejailStorage.NullFSBasejailStorage + + if self.config["basejail_type"] == "zfs": + return libiocage.lib.ZFSBasejailStorage.ZFSBasejailStorage + + return None + + def _start_services(self): + command = self.config["exec_start"].strip().split() + self.logger.debug(f"Running exec_start on {self.humanreadable_name}") + self.exec(command) def stop(self, force=False): """ @@ -184,10 +224,25 @@ def stop(self, force=False): self.require_jail_existing() self.require_jail_running() + + events = libiocage.lib.events + jailDestroyEvent = events.JailDestroy(self) + jailNetworkTeardownEvent = events.JailNetworkTeardown(self) + jailMountTeardownEvent = events.JailMountTeardown(self) + + yield jailDestroyEvent.begin() self._destroy_jail() + yield jailDestroyEvent.end() + if self.config["vnet"]: + yield jailNetworkTeardownEvent.begin() self._stop_vimage_network() + yield jailNetworkTeardownEvent.end() + + yield jailMountTeardownEvent.begin() self._teardown_mounts() + yield jailMountTeardownEvent.end() + self.update_jail_state() def destroy(self, force=False): @@ -472,7 +527,6 @@ def _launch_jail(self): f"exec.prestart={self.config['exec_prestart']}", f"exec.poststart={self.config['exec_poststart']}", f"exec.prestop={self.config['exec_prestop']}", - f"exec.start={self.config['exec_start']}", f"exec.stop={self.config['exec_stop']}", f"exec.clean={self.config['exec_clean']}", f"exec.timeout={self.config['exec_timeout']}", @@ -511,7 +565,7 @@ def _launch_jail(self): def _start_vimage_network(self): - self.logger.log("Starting VNET/VIMAGE", jail=self) + self.logger.debug("Starting VNET/VIMAGE", jail=self) nics = self.config["interfaces"] for nic in nics: @@ -661,22 +715,21 @@ def _teardown_mounts(self): ) def _resolve_name(self, text): + + if (text is None) or (len(text) == 0): + raise libiocage.lib.errors.JailNotSupplied(logger=self.logger) + jails_dataset = self.host.datasets.jails - best_guess = "" + for dataset in list(jails_dataset.children): - dataset_name = dataset.name[(len(jails_dataset.name) + 1):] - if text == dataset_name: - # Exact match, immediately return - return dataset_name - elif dataset_name.startswith(text) and len(text) > len(best_guess): - best_guess = text - if len(best_guess) > 0: - self.logger.debug(f"Resolved {text} to uuid {dataset_name}") - return best_guess + dataset_name = dataset.name[(len(jails_dataset.name) + 1):] + humanreadable_name = libiocage.lib.helpers.to_humanreadable_name( + dataset_name + ) - if not best_guess: - raise libiocage.lib.errors.JailNotSupplied(logger=self.logger) + if text in [dataset_name, humanreadable_name]: + return dataset_name raise libiocage.lib.errors.JailNotFound(text, logger=self.logger) @@ -695,19 +748,12 @@ def humanreadable_name(self): Whenever a Jail is found to have a UUID as identifier, a shortened string of the first 8 characters is returned """ - - try: - uuid.UUID(self.name) - return str(self.name)[:8] - except (TypeError, ValueError): - pass - try: - return self.name - except AttributeError: - pass - - raise libiocage.lib.errors.JailUnknownIdentifier(logger=self.logger) + return libiocage.lib.helpers.to_humanreadable_name(self.name) + except: + raise libiocage.lib.errors.JailUnknownIdentifier( + logger=self.logger + ) @property def stopped(self): @@ -811,12 +857,6 @@ def __getattr__(self, key): except AttributeError: pass - try: - method = object.__getattribute__(self, f"_get_{key}") - return method() - except: - pass - try: jail_state = object.__getattribute__(self, "jail_state") except: @@ -839,11 +879,15 @@ def getstring(self, key): key (string): Name of the jail property to return """ + + try: + return libiocage.helpers.to_string(self.config[key]) + except: + pass + try: - if key == "jid" and self.__getattr__(key) is None: - return "-" + return libiocage.lib.helpers.to_string(self.__getattr__(key)) - return str(self.__getattr__(key)) except AttributeError: return "-" @@ -852,9 +896,18 @@ def __dir__(self): properties = set() for prop in dict.__dir__(self): - if prop.startswith("_get_"): - properties.add(prop[5:]) - elif not prop.startswith("_"): + if not prop.startswith("_"): properties.add(prop) return list(properties) + + +class Jail(JailGenerator): + + _class_host = libiocage.lib.Host.HostGenerator + + def start(self, *args, **kwargs): + return list(JailGenerator.start(self, *args, **kwargs)) + + def stop(self, *args, **kwargs): + return list(JailGenerator.stop(self, *args, **kwargs)) diff --git a/libiocage/lib/JailConfig.py b/libiocage/lib/JailConfig.py index b8ce4be8..8897d505 100644 --- a/libiocage/lib/JailConfig.py +++ b/libiocage/lib/JailConfig.py @@ -1,4 +1,5 @@ import re +import uuid import libiocage.lib.JailConfigAddresses import libiocage.lib.JailConfigDefaults @@ -73,15 +74,12 @@ def __init__(self, self.jail = None self.fstab = None - data_keys = data.keys() - - # the UUID is used in many other variables and needs to be set first - if "name" in data_keys: - self["name"] = data["name"] - elif "uuid" in data_keys: - self["name"] = data["uuid"] - else: - self["id"] = None + # the name is used in many other variables and needs to be set first + self["id"] = None + for key in ["id", "name", "uuid"]: + if key in data.keys(): + self["name"] = data[key] + break # be aware of iocage-legacy jails for migration try: @@ -129,8 +127,13 @@ def clone(self, data, skip_on_error=False): Passed to __setitem__ """ - for key in data: - self.__setitem__(key, data[key], skip_on_error=skip_on_error) + current_id = self["id"] + for key, value in data.items(): + + if (key in ["id", "name", "uuid"]) and (current_id is not None): + value = current_id + + self.__setitem__(key, value, skip_on_error=skip_on_error) def read(self): @@ -205,16 +208,14 @@ def _set_name(self, name, **kwargs): ) self.logger.error(msg) - name_pattern = f"^[A-z0-9]([A-z0-9\\._\\-]+[A-z0-9])*$" - if not re.match(name_pattern, name): - raise libiocage.lib.errors.InvalidJailName(logger=self.logger) - - self.id = name - - try: - self.host_hostname - except: - self.host_hostname = name + is_valid_name = libiocage.lib.helpers.validate_name(name) + if is_valid_name is True: + self["id"] = name + else: + try: + self["id"] = str(uuid.UUID(name)) # legacy support + except: + raise libiocage.lib.errors.InvalidJailName(logger=self.logger) self.logger.spam( f"Set jail name to {name}", @@ -551,15 +552,53 @@ def __setitem__(self, key, value, **kwargs): # except: # pass + parsed_value = libiocage.lib.helpers.parse_user_input(value) + setter_method = None try: setter_method = self.__getattribute__(f"_set_{key}") except: - self.data[key] = value + self.data[key] = parsed_value pass if setter_method is not None: - return setter_method(value, **kwargs) + return setter_method(parsed_value, **kwargs) + + def set(self, key: str, value, **kwargs) -> bool: + """ + Set a JailConfig property + + Args: + + key: + The jail config property name + + value: + Value to set the property to + + **kwargs: + Arguments from **kwargs are passed to setter functions + + Returns: + + bool: True if the JailConfig was changed + """ + + try: + hash_before = self.__getitem__(key).__hash__() + except KeyError: + hash_before = None + pass + + self.__setitem__(key, value, **kwargs) + + try: + hash_after = self.__getitem__(key).__hash__() + except KeyError: + hash_after = None + pass + + return (hash_before != hash_after) def __str__(self): return libiocage.lib.JailConfigJSON.JailConfigJSON.toJSON(self) @@ -594,17 +633,7 @@ def all_properties(self): return list(properties) def stringify(self, value, enabled=True): - - if not enabled: - return value - elif value is None: - return "-" - elif value is True: - return "on" - elif value is False: - return "off" - else: - return str(value) + return libiocage.helpers.to_string if (enabled is True) else value class JailConfigList(list): diff --git a/libiocage/lib/JailConfigDefaults.py b/libiocage/lib/JailConfigDefaults.py index 0ac1adae..8df712de 100644 --- a/libiocage/lib/JailConfigDefaults.py +++ b/libiocage/lib/JailConfigDefaults.py @@ -7,6 +7,7 @@ class JailConfigDefaults(dict): DEFAULTS = { + "id": None, "basejail": False, "defaultrouter": None, "defaultrouter6": None, diff --git a/libiocage/lib/JailConfigJSON.py b/libiocage/lib/JailConfigJSON.py index bef4eed6..a14d45a3 100644 --- a/libiocage/lib/JailConfigJSON.py +++ b/libiocage/lib/JailConfigJSON.py @@ -7,10 +7,15 @@ class JailConfigJSON: def toJSON(self): data = self.data - for key in data.keys(): - if data[key] is None: - data[key] = "none" - return json.dumps(data, sort_keys=True, indent=4) + output_data = {} + for key, value in data.items(): + output_data[key] = libiocage.lib.helpers.to_string( + value, + true="yes", + false="no", + none="none" + ) + return json.dumps(output_data, sort_keys=True, indent=4) def save(self): config_file_path = JailConfigJSON.__get_config_json_path(self) diff --git a/libiocage/lib/JailFilter.py b/libiocage/lib/JailFilter.py new file mode 100644 index 00000000..4de10aa4 --- /dev/null +++ b/libiocage/lib/JailFilter.py @@ -0,0 +1,165 @@ +from typing import List, Union, Iterable +import re +import libiocage.lib.Jail +import libiocage.lib.errors + + +def match_filter(value: str, filter_string: str): + escaped_characters = [".", "$", "^", "(", ")", "?"] + for character in escaped_characters: + filter_string = filter_string.replace(character, f"\\{character}") + filter_string = filter_string.replace("*", ".*") + filter_string = filter_string.replace("+", ".+") + pattern = f"^{filter_string}$" + match = re.match(pattern, value) + return match is not None + + +class Term(list): + + glob_characters = ["*", "+"] + + def __init__(self, key, values=list()): + self.key = key + + if values is None: + raise TypeError("Values may not be empty") + elif isinstance(values, str): + data = self._split_filter_values(values) + else: + data = [values] + + list.__init__(self, data) + + def matches_jail(self, jail: libiocage.lib.Jail.JailGenerator) -> bool: + return self.matches(jail.getstring(self.key)) + + def matches(self, value: str) -> bool: + """ + Returns True if the value matches the term + """ + for filter_value in self: + + if self.key == "name": + if self._validate_name_filter_string(filter_value) is False: + raise libiocage.lib.errors.JailFilterInvalidName( + filter_value, + logger=self.logger + ) + + if match_filter(value, filter_value): + return True + + # match against humanreadable names as well + has_humanreadble_length = (len(filter_value) == 8) + has_no_globs = not self._filter_string_has_globs(filter_value) + if (has_humanreadble_length and has_no_globs) is True: + shortname = libiocage.lib.helpers.to_humanreadable_name(value) + if match_filter(shortname, filter_value): + return True + + return False + + def _filter_string_has_globs(self, filter_string: str) -> bool: + for glob in self.glob_characters: + if glob in filter_string: + return True + return False + + def _split_filter_values(self, user_input: str) -> List[str]: + values = [] + escaped_comma_blocks = map( + lambda block: block.split(","), + user_input.split("\\,") + ) + for block in escaped_comma_blocks: + n = len(values) + if n > 0: + index = n - 1 + values[index] += f",{block[0]}" + else: + values.append(block[0]) + if len(block) > 1: + values += block[1:] + return values + + def _validate_name_filter_string(self, filter_string: str) -> bool: + + globs = self.glob_characters + + # Allow glob only filters + if (len(filter_string) == 1) and (filter_string in globs): + return True + + # replace all glob charaters in user input + filter_string_without_globs = "" + for i, char in enumerate(filter_string): + if char not in globs: + filter_string_without_globs += char + + return libiocage.lib.helpers.validate_name(filter_string_without_globs) + + +class Terms(list): + """ + A group of jail filter terms. + + Each item in this group must match for a jail to pass the filter. + This can be interpreted as logical AND + """ + + def __init__(self, terms: Iterable[Union[Term, str]]=None): + + data = [] + + if terms is not None: + + for term in terms: + if isinstance(term, str): + data += self._parse_term(term) + elif isinstance(term, Term): + data.append(term) + + list.__init__(self, data) + + def match_jail(self, jail: libiocage.lib.Jail.JailGenerator) -> bool: + """ + Returns True if all Terms match the jail + """ + + for term in self: + if term.matches_jail(jail) is False: + return False + + return True + + def match_key(self, key: str, value: str) -> bool: + """ + Check if a value matches for a given key + + Returns True if the given value matches all terms for the specified key + Returns Fals if one of the terms does not match + """ + for term in self: + + if term.key != key: + continue + + if term.matches(value) is False: + return False + + return True + + def _parse_term(self, user_input: str) -> List[Term]: + + terms = [] + + try: + prop, value = user_input.split("=", maxsplit=1) + except: + prop = "name" + value = user_input + + terms.append(Term(prop, value)) + + return terms diff --git a/libiocage/lib/Jails.py b/libiocage/lib/Jails.py index e2e45980..4351258a 100644 --- a/libiocage/lib/Jails.py +++ b/libiocage/lib/Jails.py @@ -1,12 +1,13 @@ -import re +from typing import Generator, Union, Iterable import libzfs import libiocage.lib.Jail +import libiocage.lib.JailFilter import libiocage.lib.helpers -class Jails: +class JailsGenerator(list): # Keys that are stored on the Jail object, not the configuration JAIL_KEYS = [ "jid", @@ -17,6 +18,7 @@ class Jails: ] def __init__(self, + filters=None, host=None, logger=None, zfs=None): @@ -26,122 +28,77 @@ def __init__(self, libiocage.lib.helpers.init_host(self, host) self.zfs = libzfs.ZFS(history=True, history_prefix="") - def list(self, filters=None): - - if len(filters) == 1: - chars = "+*=" - name = filters[0] - if not any(x in name for x in chars): - single_jail = libiocage.lib.Jail.Jail( - { - "name": name - }, - logger=self.logger, - host=self.host, - zfs=self.zfs - ) - return [single_jail] - - jails = self._get_existing_jails() - - if filters is not None: - return self._filter_jails(jails, filters) + self._filters = None + self.filters = filters + list.__init__(self, []) + + def __iter__(self): + + for jail_dataset in self.jail_datasets: + + jail_name = self._get_name_from_jail_dataset(jail_dataset) + if self._filters.match_key("name", jail_name) is not True: + # Skip all jails that do not even match the name + continue + + # ToDo: Do not load jail if filters do not require to + jail = self._load_jail_from_dataset(jail_dataset) + if self._filters.match_jail(jail): + yield jail + + def _create_jail(self, *args, **kwargs): + kwargs["logger"] = self.logger + kwargs["host"] = self.host + kwargs["zfs"] = self.zfs + return libiocage.lib.Jail.Jail(*args, **kwargs) + + @property + def filters(self): + return self._filters + + @filters.setter + def filters( + self, + value: Union[ + str, + Iterable[Union[libiocage.lib.JailFilter.Terms, str]] + ] + ): + + if isinstance(value, libiocage.lib.JailFilter.Terms): + self._filters = value else: - return jails + self._filters = libiocage.lib.JailFilter.Terms(value) - def _filter_jails(self, jails, filters): - - filtered_jails = [] - jail_filters = {} - - filter_terms = list(map(_split_filter_map, filters)) - for key, value in filter_terms: - if key not in jail_filters.keys(): - jail_filters[key] = [value] - else: - jail_filters[key].append(value) - - for jail in jails: + @property + def jail_datasets(self) -> list: + jails_dataset = self.host.datasets.jails + return list(jails_dataset.children) - jail_matches = True + def _load_jail_from_dataset( + self, + dataset: libzfs.ZFSDataset + ) -> Generator[libiocage.lib.Jail.JailGenerator, None, None]: - for group in jail_filters.keys(): + return self._create_jail({ + "name": self._get_name_from_jail_dataset(dataset) + }) - # Providing multiple names = OR (e.g. name=foo, name=bar) - jail_matches_group = False + def _get_name_from_jail_dataset( + self, + dataset: libzfs.ZFSDataset + ) -> str: - for current_filter in jail_filters[group]: - if self._jail_matches_filter(jail, group, current_filter): - jail_matches_group = True + return dataset.name.split("/").pop() - if jail_matches_group is False: - jail_matches = False - continue - if jail_matches is True: - filtered_jails.append(jail) +class Jails(JailsGenerator): - return filtered_jails + def _create_jail(self, *args, **kwargs): + kwargs["logger"] = self.logger + kwargs["host"] = self.host + kwargs["zfs"] = self.zfs + return libiocage.lib.Jail.Jail(*args, **kwargs) - def _get_existing_jails(self): - jails_dataset = self.host.datasets.jails - jail_datasets = list(jails_dataset.children) - - return list(map( - lambda x: libiocage.lib.Jail.Jail({ - "name": x.name.split("/").pop() - }, logger=self.logger, host=self.host, zfs=self.zfs), - jail_datasets - )) - - def _jail_matches_filter(self, jail, key, value): - for filter_value in self._split_filter_values(value): - jail_value = self._lookup_jail_value(jail, key) - if not self._matches_filter(filter_value, jail_value): - return False - return True - - def _matches_filter(self, filter_value, value): - escaped_characters = [".", "$", "^", "(", ")"] - for character in escaped_characters: - filter_value = filter_value.replace(character, f"\\{character}") - filter_value = filter_value.replace("$", "\\$") - filter_value = filter_value.replace(".", "\\.") - filter_value = filter_value.replace("*", ".*") - filter_value = filter_value.replace("+", ".+") - pattern = f"^{filter_value}$" - match = re.match(pattern, value) - return match is not None - - def _lookup_jail_value(self, jail, key): - if key in Jails.JAIL_KEYS: - return jail.getstring(key) - else: - return str(jail.config[key]) - - def _split_filter_values(self, value): - values = [] - escaped_comma_blocks = map( - lambda block: block.split(","), - value.split("\\,") - ) - for block in escaped_comma_blocks: - n = len(values) - if n > 0: - index = n - 1 - values[index] += f",{block[0]}" - else: - values.append(block[0]) - if len(block) > 1: - values += block[1:] - return values - - -def _split_filter_map(x): - try: - prop, value = x.split("=", maxsplit=1) - except: - prop = "name" - value = x - - return prop, value + def __iter__(self): + return list(JailsGenerator.__iter__(self)) diff --git a/libiocage/lib/Logger.py b/libiocage/lib/Logger.py index 19d3d257..c1198af5 100644 --- a/libiocage/lib/Logger.py +++ b/libiocage/lib/Logger.py @@ -1,9 +1,41 @@ import os +import sys import libiocage.lib.errors +class LogEntry: + + def __init__(self, message, level, indent=0, logger=None, **kwargs): + self.message = message + self.level = level + self.indent = indent + self.logger = logger + + for key in kwargs.keys(): + object.__setattr__(self, key, kwargs[key]) + + def edit(self, message=None, indent=None): + + if self.logger is None: + raise libiocage.lib.errors.CannotRedrawLine( + reason="No logger available" + ) + + if message is not None: + self.message = message + + if indent is not None: + self.indent = indent + + self.logger.redraw(self) + + def __len__(self): + return len(self.message.split("\n")) + + class Logger: + COLORS = ( "black", "red", @@ -17,16 +49,18 @@ class Logger: RESET_SEQ = "\033[0m" BOLD_SEQ = "\033[1m" + LINE_UP_SEQ = "\033[F" LOG_LEVEL_SETTINGS = { - "info" : {"color": None}, - "notice" : {"color": "magenta"}, - "verbose" : {"color": "blue"}, - "spam" : {"color": "green"}, + "screen": {"color": None}, + "info": {"color": None}, + "notice": {"color": "magenta"}, + "verbose": {"color": "blue"}, + "spam": {"color": "green"}, "critical": {"color": "red", "bold": True}, - "error" : {"color": "red"}, - "debug" : {"color": "green"}, - "warn" : {"color": "yellow"} + "error": {"color": "red"}, + "debug": {"color": "green"}, + "warn": {"color": "yellow"} } LOG_LEVELS = ( @@ -38,10 +72,13 @@ class Logger: "verbose", "debug", "spam", + "screen" ) INDENT_PREFIX = " " + PRINT_HISTORY = [] + def __init__(self, print_level=None, log_directory="/var/log/iocage"): self._print_level = print_level self._set_log_directory(log_directory) @@ -80,70 +117,98 @@ def log(self, *args, **kwargs): if "level" not in kwargs: kwargs["level"] = "info" - self._print(**kwargs) - # self._write(**kwargs) + log_entry = LogEntry(logger=self, **kwargs) - def verbose(self, message, jail=None, indent=0): - self.log( - message=message, - level="verbose", - jail=jail, - indent=indent - ) + if self._should_print_log_entry(log_entry): + self._print_log_entry(log_entry) + self.PRINT_HISTORY.append(log_entry) + + return log_entry + + def verbose(self, message, indent=0, **kwargs): + return self.log(message, level="verbose", indent=indent, **kwargs) - def error(self, - message, - jail=None, - indent=0): + def error(self, message, indent=0, **kwargs): + return self.log(message, level="error", indent=indent, **kwargs) - self.log(message, level="error", jail=jail, indent=indent) + def warn(self, message, indent=0, **kwargs): + return self.log(message, level="warn", indent=indent, **kwargs) - def warn(self, - message, - jail=None, - indent=0): + def debug(self, message, indent=0, **kwargs): + return self.log(message, level="debug", indent=indent, **kwargs) - self.log(message, level="warn", jail=jail, indent=indent) + def spam(self, message, indent=0, **kwargs): + return self.log(message, level="spam", indent=indent, **kwargs) - def debug(self, - message, - jail=None, - indent=0): + def screen(self, message, indent=0, **kwargs): + """ + Screen never gets printed to log files + """ + return self.log(message, level="screen", indent=indent, **kwargs) - self.log(message, level="debug", jail=jail, indent=indent) + def redraw(self, log_entry): - def spam(self, - message, - jail=None, - indent=0): + if log_entry not in self.PRINT_HISTORY: + raise libiocage.lib.errors.CannotRedrawLine( + reason="Log entry not found in history" + ) - self.log(message, level="spam", jail=jail, indent=indent) + if log_entry.level != "screen": + raise libiocage.lib.errors.CannotRedrawLine( + reason=( + "Log level 'screen' is required to redraw, " + f"but got '{self.level}'" + ) + ) + + # calculate the delta of messages printed since + i = self.PRINT_HISTORY.index(log_entry) + n = len(self.PRINT_HISTORY) + delta = sum(map(lambda i: len(self.PRINT_HISTORY[i]), range(i, n))) + + output = "".join([ + "\r", + f"\033[{delta}F", # CPL - Cursor Previous Line + "\r", # CR - Carriage Return + self._indent(f"{log_entry.message}: {delta}", log_entry.indent), + "\033[K", # EL - Erase in Line + "\n" * (delta), + "\r" + ]) + + sys.stdout.write(output) + + def _should_print_log_entry(self, log_entry): + + if log_entry.level == "screen": + return True - def _print(self, message, level, jail=None, indent=0): if self.print_level is False: - return + return False print_level = Logger.LOG_LEVELS.index(self.print_level) - if Logger.LOG_LEVELS.index(level) > print_level: - return - - try: - color = Logger.LOG_LEVEL_SETTINGS[level]["color"] - except: - color = "none" + return Logger.LOG_LEVELS.index(log_entry.level) <= print_level + def _beautify_message(self, message, level, indent=0): + color = self._get_level_color(level) message = self._indent(message, indent) message = self._colorize(message, color) - print(message) + return message + + def _print(self, message, level, indent=0): + print(self._beautify_message(message, level, indent)) + + def _print_log_entry(self, log_entry): + return self._print( + log_entry.message, + log_entry.level, + log_entry.indent + ) def _indent(self, message, level): indent = Logger.INDENT_PREFIX * level return "\n".join(map(lambda x: f"{indent}{x}", message.split("\n"))) - # ToDo: support file logging - # def _write(self, message, level, jail=None): - # log_file = self._get_log_file_path(level=level, jail=jail) - def _get_log_file_path(self, level, jail=None): return self.log_directory @@ -157,6 +222,12 @@ def _create_log_directory(self): def _get_color_code(self, color_name): return Logger.COLORS.index(color_name) + 30 + def _get_level_color(self, log_level): + try: + return Logger.LOG_LEVEL_SETTINGS[log_level]["color"] + except KeyError: + return "none" + def _colorize(self, message, color_name=None): try: color_code = self._get_color_code(color_name) diff --git a/libiocage/lib/Release.py b/libiocage/lib/Release.py index 4c309295..ab323b5d 100644 --- a/libiocage/lib/Release.py +++ b/libiocage/lib/Release.py @@ -13,15 +13,16 @@ import libiocage.lib.Jail import libiocage.lib.errors import libiocage.lib.helpers +import libiocage.lib.events -class Release: +class ReleaseGenerator: DEFAULT_RC_CONF_SERVICES = { - "netif" : False, - "sendmail" : False, - "sendmail_submit" : False, + "netif": False, + "sendmail": False, + "sendmail_submit": False, "sendmail_msp_queue": False, - "sendmail_outbound" : False + "sendmail_outbound": False } def __init__(self, name=None, @@ -30,8 +31,6 @@ def __init__(self, name=None, zfs=None, logger=None, check_hashes=True, - auto_fetch_updates=True, - auto_update=True, eol=False): libiocage.lib.helpers.init_logger(self, logger) @@ -48,8 +47,6 @@ def __init__(self, name=None, self._root_dataset = None self.dataset = dataset self.check_hashes = check_hashes is True - self.auto_fetch_updates = auto_fetch_updates is True - self.auto_update = auto_update is True self._hbsd_release_branch = None self._assets = ["base"] @@ -263,35 +260,85 @@ def fetch(self, update=None, fetch_updates=None): release_changed = False + events = libiocage.lib.events + fetchReleaseEvent = events.FetchRelease(self) + releasePrepareStorageEvent = events.ReleasePrepareStorage(self) + releaseDownloadEvent = events.ReleaseDownload(self) + releaseExtractionEvent = events.ReleaseExtraction(self) + releaseConfigurationEvent = events.ReleaseConfiguration(self) + releaseCopyBaseEvent = events.ReleaseCopyBase(self) + if not self.fetched: + yield fetchReleaseEvent.begin() + yield releasePrepareStorageEvent.begin() + self._clean_dataset() self._create_dataset() self._ensure_dataset_mounted() + + yield releasePrepareStorageEvent.end() + yield releaseDownloadEvent.begin() + self._fetch_assets() - self._extract_assets() + + yield releaseDownloadEvent.end() + yield releaseExtractionEvent.begin() + + try: + self._extract_assets() + except Exception as e: + yield releaseExtractionEvent.fail(e) + raise + + yield releaseExtractionEvent.end() + yield releaseConfigurationEvent.begin() + self._create_default_rcconf() + + yield releaseConfigurationEvent.end() + release_changed = True + + yield fetchReleaseEvent.end() + else: - self.logger.warn( + + yield fetchReleaseEvent.skip( + message="already downloaded" + ) + + self.logger.verbose( "Release was already downloaded. Skipping download." ) - fetch_updates_on = self.auto_fetch_updates and fetch_updates - if fetch_updates_on or fetch_updates: - self.fetch_updates() + if fetch_updates is True: + for event in ReleaseGenerator.fetch_updates(self): + yield event - auto_update_on = self.auto_update and update is not False - if auto_update_on or update: - release_changed = self.update() + if update is True: + for event in ReleaseGenerator.update(self): + if isinstance(event, libiocage.lib.events.IocageEvent): + yield event + else: + # the only non-IocageEvent is our return value + release_changed = event if release_changed: + yield releaseCopyBaseEvent.begin() self._update_zfs_base() + yield releaseCopyBaseEvent.end() + else: + yield releaseCopyBaseEvent.skip(message="release unchanged") self._cleanup() def fetch_updates(self): + events = libiocage.lib.events + releaseUpdateDownloadEvent = events.ReleaseUpdateDownload(self) + yield releaseUpdateDownloadEvent.begin() + release_updates_dir = self.release_updates_dir release_update_download_dir = f"{release_updates_dir}" @@ -314,7 +361,7 @@ def fetch_updates(self): files = { update_script_name: f"usr.sbin/{update_name}/{update_script_name}", - update_conf_name : f"etc/{update_conf_name}", + update_conf_name: f"etc/{update_conf_name}", } for key in files.keys(): @@ -335,6 +382,7 @@ def fetch_updates(self): if key == update_script_name: os.chmod(local_path, 0o755) + elif key == update_conf_name: if self.host.distribution.name == "FreeBSD": @@ -346,7 +394,7 @@ def fetch_updates(self): "Components" )) f.truncate() - f.close() + os.chmod(local_path, 0o644) self.logger.debug( @@ -362,32 +410,40 @@ def fetch_updates(self): self.logger.debug( "No pre-fetching of HardenedBSD updates required - skipping" ) - return + yield releaseUpdateDownloadEvent.skip( + message="pre-fetching not supported on HardenedBSD" + ) - self.logger.verbose(f"Fetching updates for release '{self.name}'") - libiocage.lib.helpers.exec([ - f"{self.release_updates_dir}/{update_script_name}", - "-d", - release_update_download_dir, - "-f", - f"{self.release_updates_dir}/{update_conf_name}", - "--not-running-from-cron", - "fetch" - ], logger=self.logger) + else: + self.logger.verbose(f"Fetching updates for release '{self.name}'") + libiocage.lib.helpers.exec([ + f"{self.release_updates_dir}/{update_script_name}", + "-d", + release_update_download_dir, + "-f", + f"{self.release_updates_dir}/{update_conf_name}", + "--not-running-from-cron", + "fetch" + ], logger=self.logger) + + yield releaseUpdateDownloadEvent.end() def update(self): dataset = self.dataset snapshot_name = self._append_datetime(f"{dataset.name}@pre-update") + runReleaseUpdateEvent = libiocage.lib.events.RunReleaseUpdate(self) + yield runReleaseUpdateEvent.begin() + # create snapshot before the changes dataset.snapshot(snapshot_name, recursive=True) - jail = libiocage.lib.Jail.Jail({ - "uuid" : str(uuid.uuid4()), - "basejail" : False, + jail = libiocage.lib.Jail.JailGenerator({ + "uuid": str(uuid.uuid4()), + "basejail": False, "allow_mount_nullfs": "1", - "release" : self.name, - "securelevel" : "0" + "release": self.name, + "securelevel": "0" }, new=True, logger=self.logger, @@ -398,25 +454,42 @@ def update(self): jail.dataset_name = self.dataset_name changed = False + try: if self.host.distribution.name == "HardenedBSD": - changed = self._update_hbsd_jail(jail) + for event in self._update_hbsd_jail(jail): + if isinstance(event, libiocage.lib.events.IocageEvent): + yield event + else: + changed = event else: - changed = self._update_freebsd_jail(jail) - except: + for event in self._update_freebsd_jail(jail): + if isinstance(event, libiocage.lib.events.IocageEvent): + yield event + else: + changed = event + yield runReleaseUpdateEvent.end() + except Exception as e: # kill the helper jail and roll back if anything went wrong self.logger.verbose( "There was an error updating the Jail - reverting the changes" ) jail.stop(force=True) self.zfs.get_snapshot(snapshot_name).rollback(force=True) - raise + yield runReleaseUpdateEvent.fail(e) + raise e return changed def _update_hbsd_jail(self, jail): - jail.start() + events = libiocage.lib.events + executeReleaseUpdateEvent = events.ExecuteReleaseUpdate(self) + + for event in jail.start(): + yield event + + yield executeReleaseUpdateEvent.begin() update_script_path = f"{self.release_updates_dir}/hbsd-update" update_conf_path = f"{self.release_updates_dir}/hbsd-update.conf" @@ -448,13 +521,19 @@ def _update_hbsd_jail(self, jail): logger=self.logger ) - jail.stop() + yield executeReleaseUpdateEvent.end() + + for event in jail.stop(): + yield event self.logger.verbose(f"Release '{self.name}' updated") return True # ToDo: return False if nothing was updated def _update_freebsd_jail(self, jail): + events = libiocage.lib.events + executeReleaseUpdateEvent = events.ExecuteReleaseUpdate(self) + local_update_mountpoint = f"{self.root_dir}/var/db/freebsd-update" if not os.path.isdir(local_update_mountpoint): self.logger.spam( @@ -470,8 +549,10 @@ def _update_freebsd_jail(self, jail): ) jail.config.fstab.save() - jail.start() + for event in jail.start(): + yield event + yield executeReleaseUpdateEvent.begin() child, stdout, stderr = jail.exec([ "/var/db/freebsd-update/freebsd-update.sh", "-d", @@ -483,8 +564,12 @@ def _update_freebsd_jail(self, jail): if child.returncode != 0: if "No updates are available to install." in stdout: + yield executeReleaseUpdateEvent.skip( + message="already up to date" + ) self.logger.debug("Already up to date") else: + yield executeReleaseUpdateEvent.failed() raise libiocage.lib.errors.ReleaseUpdateFailure( release_name=self.name, reason=( @@ -494,12 +579,14 @@ def _update_freebsd_jail(self, jail): logger=self.logger ) else: + yield executeReleaseUpdateEvent.end() self.logger.debug(f"Update of release '{self.name}' finished") - jail.stop() + for event in jail.stop(): + yield event self.logger.verbose(f"Release '{self.name}' updated") - return True # ToDo: return False if nothing was updated + yield True # ToDo: return False if nothing was updated def _append_datetime(self, text): now = datetime.datetime.utcnow() @@ -628,7 +715,6 @@ def _create_default_rcconf(self): with open(file, "w") as f: f.write(content) f.truncate() - f.close() def _generate_default_rcconf_line(self, service_name): if Release.DEFAULT_RC_CONF_SERVICES[service_name] is True: @@ -783,3 +869,15 @@ def _check_tar_info(self, tar_info, asset_name): reason=reason, logger=self.logger ) + + def __str__(self): + return self.name + + +class Release(ReleaseGenerator): + + def fetch(self, *args, **kwargs): + return list(ReleaseGenerator.fetch(self, *args, **kwargs)) + + def update(self, *args, **kwargs): + return list(ReleaseGenerator.update(self, *args, **kwargs)) diff --git a/libiocage/lib/errors.py b/libiocage/lib/errors.py index aecee699..fda38a19 100644 --- a/libiocage/lib/errors.py +++ b/libiocage/lib/errors.py @@ -1,4 +1,5 @@ class IocageException(Exception): + def __init__(self, message, errors=None, logger=None, level="error", append_warning=False, warning=None): if logger is not None: @@ -15,44 +16,51 @@ def __init__(self, message, errors=None, logger=None, level="error", class JailDoesNotExist(IocageException): + def __init__(self, jail, *args, **kwargs): msg = f"Jail '{jail.humanreadable_name}' does not exist" super().__init__(msg, *args, **kwargs) class JailAlreadyExists(IocageException): + def __init__(self, jail, *args, **kwargs): msg = f"Jail '{jail.humanreadable_name}' already exists" super().__init__(msg, *args, **kwargs) class JailNotRunning(IocageException): + def __init__(self, jail, *args, **kwargs): msg = f"Jail '{jail.humanreadable_name}' is not running" super().__init__(msg, *args, **kwargs) class JailAlreadyRunning(IocageException): + def __init__(self, jail, *args, **kwargs): msg = f"Jail '{jail.humanreadable_name}' is already running" super().__init__(msg, *args, **kwargs) class JailNotFound(IocageException): + def __init__(self, text, *args, **kwargs): msg = f"No jail matching '{text}' was found" super().__init__(msg, *args, **kwargs) class JailNotSupplied(IocageException): + def __init__(self, *args, **kwargs): msg = f"Please supply a jail" super().__init__(msg, *args, **kwargs) class JailUnknownIdentifier(IocageException): + def __init__(self, *args, **kwargs): - msg = "The jail has not identifier yet" + msg = "The jail has no identifier yet" super().__init__(msg, *args, **kwargs) @@ -64,6 +72,7 @@ class JailConfigError(IocageException): class InvalidJailName(JailConfigError): + def __init__(self, *args, **kwargs): msg = ( "Invalid jail name: " @@ -73,6 +82,7 @@ def __init__(self, *args, **kwargs): class JailConigZFSIsNotAllowed(JailConfigError): + def __init__(self, *args, **kwargs): msg = ( "jail_zfs is disabled" @@ -82,6 +92,7 @@ def __init__(self, *args, **kwargs): class InvalidJailConfigValue(JailConfigError): + def __init__(self, property_name, jail=None, reason=None, **kwargs): msg = f"Invalid value for property '{property_name}'" if jail is not None: @@ -92,6 +103,7 @@ def __init__(self, property_name, jail=None, reason=None, **kwargs): class InvalidJailConfigAddress(InvalidJailConfigValue): + def __init__(self, value, **kwargs): reason = f"expected \"|
\" but got \"{value}\"" super().__init__( @@ -101,6 +113,7 @@ def __init__(self, value, **kwargs): class JailConfigNotFound(IocageException): + def __init__(self, config_type, *args, **kwargs): msg = f"Could not read {config_type} config" # This is a silent error internally used @@ -108,6 +121,7 @@ def __init__(self, config_type, *args, **kwargs): class DefaultConfigNotFound(IocageException, FileNotFoundError): + def __init__(self, config_file_path, *args, **kwargs): msg = f"Default configuration not found at {config_file_path}" IocageException.__init__(self, msg, *args, **kwargs) @@ -117,6 +131,7 @@ def __init__(self, config_file_path, *args, **kwargs): class IocageNotActivated(IocageException): + def __init__(self, *args, **kwargs): msg = ( "iocage is not activated yet - " @@ -126,6 +141,7 @@ def __init__(self, *args, **kwargs): class MustBeRoot(IocageException): + def __init__(self, msg, *args, **kwargs): _msg = ( f"Must be root to {msg}" @@ -134,6 +150,7 @@ def __init__(self, msg, *args, **kwargs): class CommandFailure(IocageException): + def __init__(self, returncode, *args, **kwargs): msg = f"Command exited with {returncode}" super().__init__(msg, *args, **kwargs) @@ -143,6 +160,7 @@ def __init__(self, returncode, *args, **kwargs): class DistributionUnknown(IocageException): + def __init__(self, distribution_name, *args, **kwargs): msg = f"Unknown Distribution: {distribution_name}" super().__init__(msg, *args, **kwargs) @@ -152,30 +170,35 @@ def __init__(self, distribution_name, *args, **kwargs): class UnmountFailed(IocageException): + def __init__(self, mountpoint, *args, **kwargs): msg = f"Failed to unmount {mountpoint}" super().__init__(msg, *args, **kwargs) class MountFailed(IocageException): + def __init__(self, mountpoint, *args, **kwargs): msg = f"Failed to mount {mountpoint}" super().__init__(msg, *args, **kwargs) class DatasetNotMounted(IocageException): + def __init__(self, dataset, *args, **kwargs): msg = f"Dataset '{dataset.name}' is not mounted" super().__init__(msg, *args, **kwargs) class DatasetNotAvailable(IocageException): + def __init__(self, dataset_name, *args, **kwargs): msg = f"Dataset '{dataset_name}' is not available" super().__init__(msg, *args, **kwargs) class DatasetNotJailed(IocageException): + def __init__(self, dataset, *args, **kwargs): name = dataset.name msg = f"Dataset {name} is not jailed." @@ -185,6 +208,7 @@ def __init__(self, dataset, *args, **kwargs): class ZFSPoolInvalid(IocageException, TypeError): + def __init__(self, consequence=None, *args, **kwargs): msg = "Invalid ZFS pool" @@ -195,6 +219,7 @@ def __init__(self, consequence=None, *args, **kwargs): class ZFSPoolUnavailable(IocageException): + def __init__(self, pool_name, *args, **kwargs): msg = f"ZFS pool '{pool_name}' is UNAVAIL" super().__init__(msg, *args, **kwargs) @@ -204,12 +229,14 @@ def __init__(self, pool_name, *args, **kwargs): class VnetBridgeMissing(IocageException): + def __init__(self, *args, **kwargs): msg = "VNET is enabled and requires setting a bridge" super().__init__(msg, *args, **kwargs) class InvalidNetworkBridge(IocageException, ValueError): + def __init__(self, reason=None, *args, **kwargs): msg = "Invalid network bridge argument" if reason is not None: @@ -221,6 +248,7 @@ def __init__(self, reason=None, *args, **kwargs): class UnknownReleasePool(IocageException): + def __init__(self, *args, **kwargs): msg = ( "Cannot determine the ZFS pool without knowing" @@ -230,6 +258,7 @@ def __init__(self, *args, **kwargs): class ReleaseUpdateFailure(IocageException): + def __init__(self, release_name, reason=None, *args, **kwargs): msg = f"Release update of '{release_name}' failed" if reason is not None: @@ -238,24 +267,28 @@ def __init__(self, release_name, reason=None, *args, **kwargs): class InvalidReleaseAssetSignature(ReleaseUpdateFailure): + def __init__(self, release_name, asset_name, *args, **kwargs): msg = f"Asset {asset_name} has an invalid signature" super().__init__(release_name, reason=msg, *args, **kwargs) class IllegalReleaseAssetContent(ReleaseUpdateFailure): + def __init__(self, release_name, asset_name, reason, *args, **kwargs): msg = f"Asset {asset_name} contains illegal files - {reason}" super().__init__(release_name, reason=msg, *args, **kwargs) class ReleaseNotFetched(IocageException): + def __init__(self, name, *args, **kwargs): msg = f"Release '{name}' does not exist or is not fetched locally" super().__init__(msg, *args, **kwargs) class ReleaseUpdateBranchLookup(IocageException): + def __init__(self, release_name, reason=None, *args, **kwargs): msg = f"Update source of release '{release_name}' not found" if reason is not None: @@ -267,6 +300,7 @@ def __init__(self, release_name, reason=None, *args, **kwargs): class DefaultReleaseNotFound(IocageException): + def __init__(self, host_release_name, *args, **kwargs): msg = ( f"Release '{host_release_name}' not found: " @@ -279,11 +313,13 @@ def __init__(self, host_release_name, *args, **kwargs): class DevfsRuleException(IocageException): + def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) class InvalidDevfsRulesSyntax(DevfsRuleException): + def __init__(self, devfs_rules_file, reason=None, *args, **kwargs): msg = f"Invalid devfs rules in {devfs_rules_file}" if reason is not None: @@ -292,6 +328,7 @@ def __init__(self, devfs_rules_file, reason=None, *args, **kwargs): class DuplicateDevfsRuleset(DevfsRuleException): + def __init__(self, devfs_rules_file, reason=None, *args, **kwargs): msg = "Cannot add duplicate ruleset" if reason is not None: @@ -299,9 +336,62 @@ def __init__(self, devfs_rules_file, reason=None, *args, **kwargs): super().__init__(msg, *args, **kwargs) +# Logger + + +class LogException(IocageException): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + +class CannotRedrawLine(LogException): + + def __init__(self, reason, *args, **kwargs): + msg = "Logger can't redraw line" + if reason is not None: + msg += f": {reason}" + super().__init__(msg, *args, **kwargs) + + +# Events + + +class EventAlreadyFinished(IocageException): + + def __init__(self, event, *args, **kwargs): + msg = "This {event.type} event is already finished" + IocageException.__init__(self, msg, *args, **kwargs) + + +# Jail Filter + + +class JailFilterException(IocageException): + + def __init__(self, *args, **kwargs): + IocageException.__init__(self, *args, **kwargs) + + +class JailFilterInvalidName(JailFilterException): + + def __init__(self, *args, **kwargs): + msg = ( + "Invalid jail selector: " + "Cannot select jail with illegal name" + ) + JailFilterException.__init__(self, msg, *args, **kwargs) + + # Missing Features class MissingFeature(IocageException, NotImplementedError): - def __init__(self, message, *args, **kwargs): - super().__init__(message, *args, **kwargs) + + def __init__(self, feature_name: str, plural: bool=False, *args, **kwargs): + message = ( + f"Missing Feature: '{feature_name}' " + "are" if plural is True else "is" + " not implemented yet" + ) + IocageException.__init__(self, message, *args, **kwargs) diff --git a/libiocage/lib/events.py b/libiocage/lib/events.py new file mode 100644 index 00000000..f4e3c06d --- /dev/null +++ b/libiocage/lib/events.py @@ -0,0 +1,275 @@ +from timeit import default_timer as timer +import libiocage.lib.errors + +EVENT_STATUS = ( + "pending", + "done", + "failed" +) + + +class IocageEvent: + """ + IocageEvent + + Base class for all other iocage events + """ + + HISTORY = [] + + PENDING_COUNT = 0 + + def __init__(self, message=None, **kwargs): + """ + Initializes an IocageEvent + """ + + for event in IocageEvent.HISTORY: + if event.__hash__() == self.__hash__(): + return event + + self.identifier = None + self._started_at = None + self._stopped_at = None + self._pending = False + self.skipped = False + self.done = True + self.error = None + + self.data = kwargs + self.number = len(IocageEvent.HISTORY) + 1 + self.parent_count = IocageEvent.PENDING_COUNT + + self.message = message + + if self not in IocageEvent.HISTORY: + IocageEvent.HISTORY.append(self) + + @property + def state(self): + return self.get_state() + + def get_state_string(self, + error="failed", + skipped="skipped", + done="done", + pending="pending"): + + if self.error is not None: + return error + + if self.skipped is True: + return skipped + + if self.done is True: + return done + + return pending + + @property + def type(self): + """ + The type of event + + The event type is obtained from an IocageEvent's class name + """ + return type(self).__name__ + + @property + def pending(self): + return self._pending + + @pending.setter + def pending(self, state): + current = self._pending + new_state = (state is True) + + if current == new_state: + return + + if new_state is True: + if self._started_at is not None: + raise libiocage.lib.errors.EventAlreadyFinished(event=self) + self._started_at = timer() + if new_state is False: + self._stopped_at = timer() + + self._pending = new_state + IocageEvent.PENDING_COUNT += 1 if (state is True) else -1 + + @property + def duration(self): + if (self._started_at is None) or (self._stopped_at is None): + return None + return self._stopped_at - self._started_at + + def _update_message(self, **kwargs): + if "message" in kwargs: + self.message = kwargs["message"] + + def begin(self, **kwargs): + self._update_message(**kwargs) + self.pending = True + self.done = False + self.parent_count = IocageEvent.PENDING_COUNT - 1 + return self + + def end(self, **kwargs): + self._update_message(**kwargs) + self.done = True + self.pending = False + self.done = True + self.parent_count = IocageEvent.PENDING_COUNT + return self + + def step(self, **kwargs): + self._update_message(**kwargs) + self.parent_count = IocageEvent.PENDING_COUNT + return self + + def skip(self, **kwargs): + self._update_message(**kwargs) + self.skipped = True + self.pending = False + self.parent_count = IocageEvent.PENDING_COUNT + return self + + def fail(self, exception=True, **kwargs): + self._update_message(**kwargs) + self.error = exception + self.pending = False + self.parent_count = IocageEvent.PENDING_COUNT + return self + + def __hash__(self): + identifier = "generic" if self.identifier is None else self.identifier + return hash((self.type, identifier)) + + +# Jail + + +class JailEvent(IocageEvent): + + def __init__(self, jail, **kwargs): + try: + self.identifier = jail.humanreadable_name + except: + self.identifier = None + + IocageEvent.__init__(self, jail=jail, **kwargs) + + +class JailLaunch(JailEvent): + + def __init__(self, jail, **kwargs): + JailEvent.__init__(self, jail, **kwargs) + + +class JailVnetConfiguration(JailEvent): + + def __init__(self, jail, **kwargs): + JailEvent.__init__(self, jail, **kwargs) + + +class JailZfsShareMount(JailEvent): + + def __init__(self, jail, **kwargs): + JailEvent.__init__(self, jail, **kwargs) + + +class JailServicesStart(JailEvent): + + def __init__(self, jail, **kwargs): + JailEvent.__init__(self, jail, **kwargs) + + +class JailDestroy(JailEvent): + + def __init__(self, jail, **kwargs): + JailEvent.__init__(self, jail, **kwargs) + + +class JailNetworkTeardown(JailEvent): + + def __init__(self, jail, **kwargs): + JailEvent.__init__(self, jail, **kwargs) + + +class JailMountTeardown(JailEvent): + + def __init__(self, jail, **kwargs): + JailEvent.__init__(self, jail, **kwargs) + +# Release + + +class ReleaseEvent(IocageEvent): + + def __init__(self, release, **kwargs): + try: + self.identifier = release.name + except: + self.identifier = None + + IocageEvent.__init__(self, release=release, **kwargs) + + +class FetchRelease(ReleaseEvent): + + def __init__(self, release, **kwargs): + ReleaseEvent.__init__(self, release, **kwargs) + + +class ReleasePrepareStorage(FetchRelease): + + def __init__(self, release, **kwargs): + FetchRelease.__init__(self, release, **kwargs) + + +class ReleaseDownload(FetchRelease): + + def __init__(self, release, **kwargs): + FetchRelease.__init__(self, release, **kwargs) + + +class ReleaseExtraction(FetchRelease): + + def __init__(self, release, **kwargs): + FetchRelease.__init__(self, release, **kwargs) + + +class ReleaseCopyBase(FetchRelease): + + def __init__(self, release, **kwargs): + FetchRelease.__init__(self, release, **kwargs) + + +class ReleaseConfiguration(FetchRelease): + + def __init__(self, release, **kwargs): + FetchRelease.__init__(self, release, **kwargs) + + +class ReleaseUpdate(ReleaseEvent): + + def __init__(self, release, **kwargs): + ReleaseEvent.__init__(self, release, **kwargs) + + +class ReleaseUpdateDownload(ReleaseEvent): + + def __init__(self, release, **kwargs): + ReleaseUpdate.__init__(self, release, **kwargs) + + +class RunReleaseUpdate(ReleaseUpdate): + + def __init__(self, release, **kwargs): + ReleaseUpdate.__init__(self, release, **kwargs) + + +class ExecuteReleaseUpdate(ReleaseUpdate): + + def __init__(self, release, **kwargs): + ReleaseUpdate.__init__(self, release, **kwargs) diff --git a/libiocage/lib/helpers.py b/libiocage/lib/helpers.py index e60e2110..6e4b026e 100644 --- a/libiocage/lib/helpers.py +++ b/libiocage/lib/helpers.py @@ -1,5 +1,6 @@ import re import subprocess +import uuid import libzfs @@ -28,7 +29,10 @@ def init_host(self, host=None): except: logger = None - self.host = libiocage.lib.Host.Host(logger=logger) + try: + self.host = self._class_host(logger=logger) + except: + self.host = libiocage.lib.Host.HostGenerator(logger=logger) def init_datasets(self, datasets=None): @@ -96,10 +100,20 @@ def _prettify_output(output): )) +def to_humanreadable_name(name: str) -> str: + try: + uuid.UUID(name) + return str(name)[:8] + except (TypeError, ValueError): + return name + + # helper function to validate names -def validate_name(name): - validate = re.compile(r'[a-z0-9][a-z0-9\.\-_]{0,31}', re.I) - return bool(validate.fullmatch(name)) +_validate_name = re.compile(r'[a-z0-9][a-z0-9\.\-_]{0,31}', re.I) + + +def validate_name(name: str): + return bool(_validate_name.fullmatch(name)) def _parse_none(data):