diff --git a/ignite/contrib/handlers/base_logger.py b/ignite/contrib/handlers/base_logger.py index 2db416575bb5..1d983ee59415 100644 --- a/ignite/contrib/handlers/base_logger.py +++ b/ignite/contrib/handlers/base_logger.py @@ -1,10 +1,10 @@ import numbers import warnings from abc import ABCMeta, abstractmethod -from collections.abc import Sequence -from typing import Any, Mapping +from typing import Any, Callable, List, Optional, Sequence, Union import torch +import torch.nn as nn from torch.optim import Optimizer from ignite.engine import Engine, State @@ -21,7 +21,7 @@ class BaseOptimizerParamsHandler(BaseHandler): Base handler for logging optimizer parameters """ - def __init__(self, optimizer, param_name="lr", tag=None): + def __init__(self, optimizer: Optimizer, param_name: str = "lr", tag: Optional[str] = None): if not ( isinstance(optimizer, Optimizer) or (hasattr(optimizer, "param_groups") and isinstance(optimizer.param_groups, Sequence)) @@ -41,7 +41,13 @@ class BaseOutputHandler(BaseHandler): Helper handler to log engine's output and/or metrics """ - def __init__(self, tag, metric_names=None, output_transform=None, global_step_transform=None): + def __init__( + self, + tag: str, + metric_names: Optional[Union[str, List[str]]] = None, + output_transform: Optional[Callable] = None, + global_step_transform: Optional[Callable] = None, + ): if metric_names is not None: if not (isinstance(metric_names, list) or (isinstance(metric_names, str) and metric_names == "all")): @@ -70,7 +76,7 @@ def global_step_transform(engine, event_name): self.output_transform = output_transform self.global_step_transform = global_step_transform - def _setup_output_metrics(self, engine): + def _setup_output_metrics(self, engine: Engine): """Helper method to setup metrics to log """ metrics = {} @@ -102,14 +108,14 @@ class BaseWeightsScalarHandler(BaseHandler): Helper handler to log model's weights as scalars. """ - def __init__(self, model, reduction=torch.norm, tag=None): + def __init__(self, model: nn.Module, reduction: Callable = torch.norm, tag: Optional[str] = None): if not isinstance(model, torch.nn.Module): raise TypeError("Argument model should be of type torch.nn.Module, " "but given {}".format(type(model))) if not callable(reduction): raise TypeError("Argument reduction should be callable, " "but given {}".format(type(reduction))) - def _is_0D_tensor(t): + def _is_0D_tensor(t: torch.Tensor): return isinstance(t, torch.Tensor) and t.ndimension() == 0 # Test reduction function on a tensor @@ -127,7 +133,7 @@ class BaseWeightsHistHandler(BaseHandler): Helper handler to log model's weights as histograms. """ - def __init__(self, model, tag=None): + def __init__(self, model: nn.Module, tag: Optional[str] = None): if not isinstance(model, torch.nn.Module): raise TypeError("Argument model should be of type torch.nn.Module, " "but given {}".format(type(model))) @@ -141,7 +147,7 @@ class BaseLogger(metaclass=ABCMeta): """ - def attach(self, engine, log_handler, event_name): + def attach(self, engine: Engine, log_handler: Callable, event_name: Any): """Attach the logger to the engine and execute `log_handler` function at `event_name` events. Args: @@ -161,7 +167,7 @@ def attach(self, engine, log_handler, event_name): return engine.add_event_handler(event_name, log_handler, self, name) - def attach_output_handler(self, engine: Engine, event_name: Any, *args: Any, **kwargs: Mapping): + def attach_output_handler(self, engine: Engine, event_name: Any, *args: Any, **kwargs: Any): """Shortcut method to attach `OutputHandler` to the logger. Args: @@ -177,7 +183,7 @@ def attach_output_handler(self, engine: Engine, event_name: Any, *args: Any, **k """ return self.attach(engine, self._create_output_handler(*args, **kwargs), event_name=event_name) - def attach_opt_params_handler(self, engine: Engine, event_name: Any, *args: Any, **kwargs: Mapping): + def attach_opt_params_handler(self, engine: Engine, event_name: Any, *args: Any, **kwargs: Any): """Shortcut method to attach `OptimizerParamsHandler` to the logger. Args: @@ -194,11 +200,11 @@ def attach_opt_params_handler(self, engine: Engine, event_name: Any, *args: Any, self.attach(engine, self._create_opt_params_handler(*args, **kwargs), event_name=event_name) @abstractmethod - def _create_output_handler(self, engine, *args, **kwargs): + def _create_output_handler(self, engine: Engine, *args: Any, **kwargs: Any): pass @abstractmethod - def _create_opt_params_handler(self, *args, **kwargs): + def _create_opt_params_handler(self, *args: Any, **kwargs: Any): pass def __enter__(self): diff --git a/ignite/contrib/handlers/lr_finder.py b/ignite/contrib/handlers/lr_finder.py index 18f0328562eb..4dd58c3b4f0f 100644 --- a/ignite/contrib/handlers/lr_finder.py +++ b/ignite/contrib/handlers/lr_finder.py @@ -3,10 +3,11 @@ import logging import tempfile import warnings -from collections.abc import Mapping from pathlib import Path +from typing import Callable, Mapping, Optional import torch +from torch.optim import Optimizer from torch.optim.lr_scheduler import _LRScheduler from ignite.contrib.handlers.param_scheduler import LRScheduler, PiecewiseLinear @@ -77,7 +78,17 @@ def __init__(self): self._lr_schedule = None self.logger = logging.getLogger(__name__) - def _run(self, trainer, optimizer, output_transform, num_iter, end_lr, step_mode, smooth_f, diverge_th): + def _run( + self, + trainer: Engine, + optimizer: Optimizer, + output_transform: Callable, + num_iter: int, + end_lr: float, + step_mode: str, + smooth_f: float, + diverge_th: float, + ): self._history = {"lr": [], "loss": []} self._best_loss = None @@ -116,13 +127,13 @@ def _run(self, trainer, optimizer, output_transform, num_iter, end_lr, step_mode if not trainer.has_event_handler(self._lr_schedule): trainer.add_event_handler(Events.ITERATION_COMPLETED, self._lr_schedule, num_iter) - def _reset(self, trainer): + def _reset(self, trainer: Engine): self.logger.debug("Completed LR finder run") trainer.remove_event_handler(self._lr_schedule, Events.ITERATION_COMPLETED) trainer.remove_event_handler(self._log_lr_and_loss, Events.ITERATION_COMPLETED) trainer.remove_event_handler(self._reached_num_iterations, Events.ITERATION_COMPLETED) - def _log_lr_and_loss(self, trainer, output_transform, smooth_f, diverge_th): + def _log_lr_and_loss(self, trainer: Engine, output_transform: Callable, smooth_f: float, diverge_th: float): output = trainer.state.output loss = output_transform(output) lr = self._lr_schedule.get_param() @@ -142,7 +153,7 @@ def _log_lr_and_loss(self, trainer, output_transform, smooth_f, diverge_th): self.logger.info("Stopping early, the loss has diverged") trainer.terminate() - def _reached_num_iterations(self, trainer, num_iter): + def _reached_num_iterations(self, trainer: Engine, num_iter: int): if trainer.state.iteration > num_iter: trainer.terminate() @@ -154,7 +165,7 @@ def _warning(self, _): UserWarning, ) - def _detach(self, trainer): + def _detach(self, trainer: Engine): """ Detaches lr_finder from trainer. @@ -175,7 +186,7 @@ def get_results(self): """ return self._history - def plot(self, skip_start=10, skip_end=5, log_lr=True): + def plot(self, skip_start: int = 10, skip_end: int = 5, log_lr: bool = True): """Plots the learning rate range test. This method requires `matplotlib` package to be installed: @@ -242,14 +253,14 @@ def lr_suggestion(self): @contextlib.contextmanager def attach( self, - trainer, - to_save, - output_transform=lambda output: output, - num_iter=None, - end_lr=10.0, - step_mode="exp", - smooth_f=0.05, - diverge_th=5.0, + trainer: Engine, + to_save: Mapping, + output_transform: Callable = lambda output: output, + num_iter: Optional[int] = None, + end_lr: float = 10.0, + step_mode: str = "exp", + smooth_f: float = 0.05, + diverge_th: float = 5.0, ): """Attaches lr_finder to a given trainer. It also resets model and optimizer at the end of the run. @@ -361,7 +372,7 @@ class _ExponentialLR(_LRScheduler): """ - def __init__(self, optimizer, end_lr, num_iter, last_epoch=-1): + def __init__(self, optimizer: Optimizer, end_lr: float, num_iter: int, last_epoch: int = -1): self.end_lr = end_lr self.num_iter = num_iter super(_ExponentialLR, self).__init__(optimizer, last_epoch) diff --git a/ignite/contrib/handlers/mlflow_logger.py b/ignite/contrib/handlers/mlflow_logger.py index 20e56c4b2128..bc8478f0d17a 100644 --- a/ignite/contrib/handlers/mlflow_logger.py +++ b/ignite/contrib/handlers/mlflow_logger.py @@ -1,14 +1,125 @@ import numbers import warnings +from typing import Any, Callable, List, Optional, Union import torch +from torch.optim import Optimizer from ignite.contrib.handlers.base_logger import BaseLogger, BaseOptimizerParamsHandler, BaseOutputHandler +from ignite.engine import Engine, EventEnum from ignite.handlers import global_step_from_engine __all__ = ["MLflowLogger", "OutputHandler", "OptimizerParamsHandler", "global_step_from_engine"] +class MLflowLogger(BaseLogger): + """ + `MLflow `_ tracking client handler to log parameters and metrics during the training + and validation. + + This class requires `mlflow package `_ to be installed: + + .. code-block:: bash + + pip install mlflow + + Args: + tracking_uri (str): MLflow tracking uri. See MLflow docs for more details + + Examples: + + .. code-block:: python + + from ignite.contrib.handlers.mlflow_logger import * + + # Create a logger + mlflow_logger = MLflowLogger() + + # Log experiment parameters: + mlflow_logger.log_params({ + "seed": seed, + "batch_size": batch_size, + "model": model.__class__.__name__, + + "pytorch version": torch.__version__, + "ignite version": ignite.__version__, + "cuda version": torch.version.cuda, + "device name": torch.cuda.get_device_name(0) + }) + + # Attach the logger to the trainer to log training loss at each iteration + mlflow_logger.attach_output_handler( + trainer, + event_name=Events.ITERATION_COMPLETED, + tag="training", + output_transform=lambda loss: {'loss': loss} + ) + + # Attach the logger to the evaluator on the training dataset and log NLL, Accuracy metrics after each epoch + # We setup `global_step_transform=global_step_from_engine(trainer)` to take the epoch + # of the `trainer` instead of `train_evaluator`. + mlflow_logger.attach_output_handler( + train_evaluator, + event_name=Events.EPOCH_COMPLETED, + tag="training", + metric_names=["nll", "accuracy"], + global_step_transform=global_step_from_engine(trainer), + ) + + # Attach the logger to the evaluator on the validation dataset and log NLL, Accuracy metrics after + # each epoch. We setup `global_step_transform=global_step_from_engine(trainer)` to take the epoch of the + # `trainer` instead of `evaluator`. + mlflow_logger.attach_output_handler( + evaluator, + event_name=Events.EPOCH_COMPLETED, + tag="validation", + metric_names=["nll", "accuracy"], + global_step_transform=global_step_from_engine(trainer)), + ) + + # Attach the logger to the trainer to log optimizer's parameters, e.g. learning rate at each iteration + mlflow_logger.attach_opt_params_handler( + trainer, + event_name=Events.ITERATION_STARTED, + optimizer=optimizer, + param_name='lr' # optional + ) + """ + + def __init__(self, tracking_uri: Optional[str] = None): + try: + import mlflow + except ImportError: + raise RuntimeError( + "This contrib module requires mlflow to be installed. " + "Please install it with command: \n pip install mlflow" + ) + + if tracking_uri is not None: + mlflow.set_tracking_uri(tracking_uri) + + self.active_run = mlflow.active_run() + if self.active_run is None: + self.active_run = mlflow.start_run() + + def __getattr__(self, attr: Any): + + import mlflow + + return getattr(mlflow, attr) + + def close(self): + import mlflow + + mlflow.end_run() + + def _create_output_handler(self, *args: Any, **kwargs: Any): + return OutputHandler(*args, **kwargs) + + def _create_opt_params_handler(self, *args: Any, **kwargs: Any): + return OptimizerParamsHandler(*args, **kwargs) + + class OutputHandler(BaseOutputHandler): """Helper handler to log engine's output and/or metrics. @@ -95,10 +206,16 @@ def global_step_transform(engine, event_name): """ - def __init__(self, tag, metric_names=None, output_transform=None, global_step_transform=None): + def __init__( + self, + tag: str, + metric_names: Optional[Union[str, List[str]]] = None, + output_transform: Optional[Callable] = None, + global_step_transform: Optional[Callable] = None, + ): super(OutputHandler, self).__init__(tag, metric_names, output_transform, global_step_transform) - def __call__(self, engine, logger, event_name): + def __call__(self, engine: Engine, logger: MLflowLogger, event_name: Union[str, EventEnum]): if not isinstance(logger, MLflowLogger): raise TypeError("Handler 'OutputHandler' works only with MLflowLogger") @@ -173,10 +290,10 @@ class OptimizerParamsHandler(BaseOptimizerParamsHandler): tag (str, optional): common title for all produced plots. For example, 'generator' """ - def __init__(self, optimizer, param_name="lr", tag=None): + def __init__(self, optimizer: Optimizer, param_name: str = "lr", tag: Optional[str] = None): super(OptimizerParamsHandler, self).__init__(optimizer, param_name, tag) - def __call__(self, engine, logger, event_name): + def __call__(self, engine: Engine, logger: MLflowLogger, event_name: Union[str, EventEnum]): if not isinstance(logger, MLflowLogger): raise TypeError("Handler OptimizerParamsHandler works only with MLflowLogger") @@ -188,111 +305,3 @@ def __call__(self, engine, logger, event_name): } logger.log_metrics(params, step=global_step) - - -class MLflowLogger(BaseLogger): - """ - `MLflow `_ tracking client handler to log parameters and metrics during the training - and validation. - - This class requires `mlflow package `_ to be installed: - - .. code-block:: bash - - pip install mlflow - - Args: - tracking_uri (str): MLflow tracking uri. See MLflow docs for more details - - Examples: - - .. code-block:: python - - from ignite.contrib.handlers.mlflow_logger import * - - # Create a logger - mlflow_logger = MLflowLogger() - - # Log experiment parameters: - mlflow_logger.log_params({ - "seed": seed, - "batch_size": batch_size, - "model": model.__class__.__name__, - - "pytorch version": torch.__version__, - "ignite version": ignite.__version__, - "cuda version": torch.version.cuda, - "device name": torch.cuda.get_device_name(0) - }) - - # Attach the logger to the trainer to log training loss at each iteration - mlflow_logger.attach_output_handler( - trainer, - event_name=Events.ITERATION_COMPLETED, - tag="training", - output_transform=lambda loss: {'loss': loss} - ) - - # Attach the logger to the evaluator on the training dataset and log NLL, Accuracy metrics after each epoch - # We setup `global_step_transform=global_step_from_engine(trainer)` to take the epoch - # of the `trainer` instead of `train_evaluator`. - mlflow_logger.attach_output_handler( - train_evaluator, - event_name=Events.EPOCH_COMPLETED, - tag="training", - metric_names=["nll", "accuracy"], - global_step_transform=global_step_from_engine(trainer), - ) - - # Attach the logger to the evaluator on the validation dataset and log NLL, Accuracy metrics after - # each epoch. We setup `global_step_transform=global_step_from_engine(trainer)` to take the epoch of the - # `trainer` instead of `evaluator`. - mlflow_logger.attach_output_handler( - evaluator, - event_name=Events.EPOCH_COMPLETED, - tag="validation", - metric_names=["nll", "accuracy"], - global_step_transform=global_step_from_engine(trainer)), - ) - - # Attach the logger to the trainer to log optimizer's parameters, e.g. learning rate at each iteration - mlflow_logger.attach_opt_params_handler( - trainer, - event_name=Events.ITERATION_STARTED, - optimizer=optimizer, - param_name='lr' # optional - ) - """ - - def __init__(self, tracking_uri=None): - try: - import mlflow - except ImportError: - raise RuntimeError( - "This contrib module requires mlflow to be installed. " - "Please install it with command: \n pip install mlflow" - ) - - if tracking_uri is not None: - mlflow.set_tracking_uri(tracking_uri) - - self.active_run = mlflow.active_run() - if self.active_run is None: - self.active_run = mlflow.start_run() - - def __getattr__(self, attr): - - import mlflow - - return getattr(mlflow, attr) - - def close(self): - import mlflow - - mlflow.end_run() - - def _create_output_handler(self, *args, **kwargs): - return OutputHandler(*args, **kwargs) - - def _create_opt_params_handler(self, *args, **kwargs): - return OptimizerParamsHandler(*args, **kwargs) diff --git a/ignite/contrib/handlers/neptune_logger.py b/ignite/contrib/handlers/neptune_logger.py index 0536202563e8..18cf523ee62d 100644 --- a/ignite/contrib/handlers/neptune_logger.py +++ b/ignite/contrib/handlers/neptune_logger.py @@ -1,9 +1,11 @@ import numbers import tempfile import warnings -from typing import Mapping, Optional +from typing import Any, Callable, List, Mapping, Optional, Union import torch +import torch.nn as nn +from torch.optim import Optimizer import ignite import ignite.distributed as idist @@ -13,6 +15,7 @@ BaseOutputHandler, BaseWeightsScalarHandler, ) +from ignite.engine import Engine, EventEnum from ignite.handlers import global_step_from_engine from ignite.handlers.checkpoint import BaseSaveHandler @@ -27,6 +30,191 @@ ] +class NeptuneLogger(BaseLogger): + """ + `Neptune `_ handler to log metrics, model/optimizer parameters, gradients during the training + and validation. It can also log model checkpoints to Neptune server. + + .. code-block:: bash + + pip install neptune-client + + Args: + api_token (str | None): Required in online mode. Neputne API token, found on https://neptune.ai. + Read how to get your API key + https://docs.neptune.ai/python-api/tutorials/get-started.html#copy-api-token. + project_name (str): Required in online mode. Qualified name of a project in a form of + "namespace/project_name" for example "tom/minst-classification". + If None, the value of NEPTUNE_PROJECT environment variable will be taken. + You need to create the project in https://neptune.ai first. + offline_mode (bool): Optional default False. If offline_mode=True no logs will be send to neptune. + Usually used for debug purposes. + experiment_name (str, optional): Optional. Editable name of the experiment. + Name is displayed in the experiment’s Details (Metadata section) and in experiments view as a column. + upload_source_files (list, optional): Optional. List of source files to be uploaded. + Must be list of str or single str. Uploaded sources are displayed in the experiment’s Source code tab. + If None is passed, Python file from which experiment was created will be uploaded. + Pass empty list (`[]`) to upload no files. Unix style pathname pattern expansion is supported. + For example, you can pass `*.py` to upload all python source files from the current directory. + For recursion lookup use `**/*.py` (for Python 3.5 and later). For more information see glob library. + params (dict, optional): Optional. Parameters of the experiment. After experiment creation params are read-only. + Parameters are displayed in the experiment’s Parameters section and each key-value pair can be + viewed in experiments view as a column. + properties (dict, optional): Optional default is `{}`. Properties of the experiment. + They are editable after experiment is created. Properties are displayed in the experiment’s Details and + each key-value pair can be viewed in experiments view as a column. + tags (list, optional): Optional default `[]`. Must be list of str. Tags of the experiment. + Tags are displayed in the experiment’s Details and can be viewed in experiments view as a column. + + Examples: + + .. code-block:: python + + from ignite.contrib.handlers.neptune_logger import * + + # Create a logger + # We are using the api_token for the anonymous user neptuner but you can use your own. + + npt_logger = NeptuneLogger( + api_token="ANONYMOUS", + project_name="shared/pytorch-ignite-integration", + experiment_name="cnn-mnist", # Optional, + params={"max_epochs": 10}, # Optional, + tags=["pytorch-ignite","minst"] # Optional + ) + + # Attach the logger to the trainer to log training loss at each iteration + npt_logger.attach_output_handler( + trainer, + event_name=Events.ITERATION_COMPLETED, + tag="training", + output_transform=lambda loss: {'loss': loss} + ) + + # Attach the logger to the evaluator on the training dataset and log NLL, Accuracy metrics after each epoch + # We setup `global_step_transform=global_step_from_engine(trainer)` to take the epoch + # of the `trainer` instead of `train_evaluator`. + npt_logger.attach_output_handler( + train_evaluator, + event_name=Events.EPOCH_COMPLETED, + tag="training", + metric_names=["nll", "accuracy"], + global_step_transform=global_step_from_engine(trainer), + ) + + # Attach the logger to the evaluator on the validation dataset and log NLL, Accuracy metrics after + # each epoch. We setup `global_step_transform=global_step_from_engine(trainer)` to take the epoch of the + # `trainer` instead of `evaluator`. + npt_logger.attach_output_handler( + evaluator, + event_name=Events.EPOCH_COMPLETED, + tag="validation", + metric_names=["nll", "accuracy"], + global_step_transform=global_step_from_engine(trainer)), + ) + + # Attach the logger to the trainer to log optimizer's parameters, e.g. learning rate at each iteration + npt_logger.attach_opt_params_handler( + trainer, + event_name=Events.ITERATION_STARTED, + optimizer=optimizer, + param_name='lr' # optional + ) + + # Attach the logger to the trainer to log model's weights norm after each iteration + npt_logger.attach( + trainer, + event_name=Events.ITERATION_COMPLETED, + log_handler=WeightsScalarHandler(model) + ) + + Explore an experiment with neptune tracking here: + https://ui.neptune.ai/o/shared/org/pytorch-ignite-integration/e/PYTOR1-18/charts + You can save model checkpoints to a Neptune server: + + .. code-block:: python + + from ignite.handlers import Checkpoint + + def score_function(engine): + return engine.state.metrics["accuracy"] + + to_save = {"model": model} + handler = Checkpoint( + to_save, + NeptuneSaver(npt_logger), n_saved=2, + filename_prefix="best", + score_function=score_function, + score_name="validation_accuracy", + global_step_transform=global_step_from_engine(trainer) + ) + validation_evaluator.add_event_handler(Events.COMPLETED, handler) + + It is also possible to use the logger as context manager: + + .. code-block:: python + + from ignite.contrib.handlers.neptune_logger import * + + # We are using the api_token for the anonymous user neptuner but you can use your own. + + with NeptuneLogger(api_token="ANONYMOUS", + project_name="shared/pytorch-ignite-integration", + experiment_name="cnn-mnist", # Optional, + params={"max_epochs": 10}, # Optional, + tags=["pytorch-ignite","minst"] # Optional + ) as npt_logger: + + trainer = Engine(update_fn) + # Attach the logger to the trainer to log training loss at each iteration + npt_logger.attach_output_handler( + trainer, + event_name=Events.ITERATION_COMPLETED, + tag="training", + output_transform=lambda loss: {"loss": loss} + ) + + """ + + def __getattr__(self, attr: Any): + + import neptune + + return getattr(neptune, attr) + + def __init__(self, *args: Any, **kwargs: Any): + try: + import neptune + except ImportError: + raise RuntimeError( + "This contrib module requires neptune-client to be installed. " + "You may install neptune with command: \n pip install neptune-client \n" + ) + + if kwargs.get("offline_mode", False): + self.mode = "offline" + neptune.init(project_qualified_name="dry-run/project", backend=neptune.OfflineBackend()) + else: + self.mode = "online" + neptune.init(api_token=kwargs.get("api_token"), project_qualified_name=kwargs.get("project_name")) + + kwargs["name"] = kwargs.pop("experiment_name", None) + self._experiment_kwargs = { + k: v for k, v in kwargs.items() if k not in ["api_token", "project_name", "offline_mode"] + } + + self.experiment = neptune.create_experiment(**self._experiment_kwargs) + + def close(self): + self.stop() + + def _create_output_handler(self, *args: Any, **kwargs: Any): + return OutputHandler(*args, **kwargs) + + def _create_opt_params_handler(self, *args: Any, **kwargs: Any): + return OptimizerParamsHandler(*args, **kwargs) + + class OutputHandler(BaseOutputHandler): """Helper handler to log engine's output and/or metrics @@ -129,10 +317,16 @@ def global_step_transform(engine, event_name): """ - def __init__(self, tag, metric_names=None, output_transform=None, global_step_transform=None): + def __init__( + self, + tag: str, + metric_names: Optional[Union[str, List[str]]] = None, + output_transform: Optional[Callable] = None, + global_step_transform: Optional[Callable] = None, + ): super(OutputHandler, self).__init__(tag, metric_names, output_transform, global_step_transform) - def __call__(self, engine, logger, event_name): + def __call__(self, engine: Engine, logger: NeptuneLogger, event_name: Union[str, EventEnum]): if not isinstance(logger, NeptuneLogger): raise TypeError("Handler OutputHandler works only with NeptuneLogger") @@ -197,10 +391,10 @@ class OptimizerParamsHandler(BaseOptimizerParamsHandler): tag (str, optional): common title for all produced plots. For example, "generator" """ - def __init__(self, optimizer, param_name="lr", tag=None): + def __init__(self, optimizer: Optimizer, param_name: str = "lr", tag: Optional[str] = None): super(OptimizerParamsHandler, self).__init__(optimizer, param_name, tag) - def __call__(self, engine, logger, event_name): + def __call__(self, engine: Engine, logger: NeptuneLogger, event_name: Union[str, EventEnum]): if not isinstance(logger, NeptuneLogger): raise TypeError("Handler OptimizerParamsHandler works only with NeptuneLogger") @@ -251,10 +445,10 @@ class WeightsScalarHandler(BaseWeightsScalarHandler): """ - def __init__(self, model, reduction=torch.norm, tag=None): + def __init__(self, model: nn.Module, reduction: Callable = torch.norm, tag: Optional[str] = None): super(WeightsScalarHandler, self).__init__(model, reduction, tag=tag) - def __call__(self, engine, logger, event_name): + def __call__(self, engine: Engine, logger: NeptuneLogger, event_name: Union[str, EventEnum]): if not isinstance(logger, NeptuneLogger): raise TypeError("Handler WeightsScalarHandler works only with NeptuneLogger") @@ -309,10 +503,10 @@ class GradsScalarHandler(BaseWeightsScalarHandler): """ - def __init__(self, model, reduction=torch.norm, tag=None): + def __init__(self, model: nn.Module, reduction: Callable = torch.norm, tag: Optional[str] = None): super(GradsScalarHandler, self).__init__(model, reduction, tag=tag) - def __call__(self, engine, logger, event_name): + def __call__(self, engine: Engine, logger: NeptuneLogger, event_name: Any): if not isinstance(logger, NeptuneLogger): raise TypeError("Handler GradsScalarHandler works only with NeptuneLogger") @@ -330,191 +524,6 @@ def __call__(self, engine, logger, event_name): ) -class NeptuneLogger(BaseLogger): - """ - `Neptune `_ handler to log metrics, model/optimizer parameters, gradients during the training - and validation. It can also log model checkpoints to Neptune server. - - .. code-block:: bash - - pip install neptune-client - - Args: - api_token (str | None): Required in online mode. Neputne API token, found on https://neptune.ai. - Read how to get your API key - https://docs.neptune.ai/python-api/tutorials/get-started.html#copy-api-token. - project_name (str): Required in online mode. Qualified name of a project in a form of - "namespace/project_name" for example "tom/minst-classification". - If None, the value of NEPTUNE_PROJECT environment variable will be taken. - You need to create the project in https://neptune.ai first. - offline_mode (bool): Optional default False. If offline_mode=True no logs will be send to neptune. - Usually used for debug purposes. - experiment_name (str, optional): Optional. Editable name of the experiment. - Name is displayed in the experiment’s Details (Metadata section) and in experiments view as a column. - upload_source_files (list, optional): Optional. List of source files to be uploaded. - Must be list of str or single str. Uploaded sources are displayed in the experiment’s Source code tab. - If None is passed, Python file from which experiment was created will be uploaded. - Pass empty list (`[]`) to upload no files. Unix style pathname pattern expansion is supported. - For example, you can pass `*.py` to upload all python source files from the current directory. - For recursion lookup use `**/*.py` (for Python 3.5 and later). For more information see glob library. - params (dict, optional): Optional. Parameters of the experiment. After experiment creation params are read-only. - Parameters are displayed in the experiment’s Parameters section and each key-value pair can be - viewed in experiments view as a column. - properties (dict, optional): Optional default is `{}`. Properties of the experiment. - They are editable after experiment is created. Properties are displayed in the experiment’s Details and - each key-value pair can be viewed in experiments view as a column. - tags (list, optional): Optional default `[]`. Must be list of str. Tags of the experiment. - Tags are displayed in the experiment’s Details and can be viewed in experiments view as a column. - - Examples: - - .. code-block:: python - - from ignite.contrib.handlers.neptune_logger import * - - # Create a logger - # We are using the api_token for the anonymous user neptuner but you can use your own. - - npt_logger = NeptuneLogger( - api_token="ANONYMOUS", - project_name="shared/pytorch-ignite-integration", - experiment_name="cnn-mnist", # Optional, - params={"max_epochs": 10}, # Optional, - tags=["pytorch-ignite","minst"] # Optional - ) - - # Attach the logger to the trainer to log training loss at each iteration - npt_logger.attach_output_handler( - trainer, - event_name=Events.ITERATION_COMPLETED, - tag="training", - output_transform=lambda loss: {'loss': loss} - ) - - # Attach the logger to the evaluator on the training dataset and log NLL, Accuracy metrics after each epoch - # We setup `global_step_transform=global_step_from_engine(trainer)` to take the epoch - # of the `trainer` instead of `train_evaluator`. - npt_logger.attach_output_handler( - train_evaluator, - event_name=Events.EPOCH_COMPLETED, - tag="training", - metric_names=["nll", "accuracy"], - global_step_transform=global_step_from_engine(trainer), - ) - - # Attach the logger to the evaluator on the validation dataset and log NLL, Accuracy metrics after - # each epoch. We setup `global_step_transform=global_step_from_engine(trainer)` to take the epoch of the - # `trainer` instead of `evaluator`. - npt_logger.attach_output_handler( - evaluator, - event_name=Events.EPOCH_COMPLETED, - tag="validation", - metric_names=["nll", "accuracy"], - global_step_transform=global_step_from_engine(trainer)), - ) - - # Attach the logger to the trainer to log optimizer's parameters, e.g. learning rate at each iteration - npt_logger.attach_opt_params_handler( - trainer, - event_name=Events.ITERATION_STARTED, - optimizer=optimizer, - param_name='lr' # optional - ) - - # Attach the logger to the trainer to log model's weights norm after each iteration - npt_logger.attach( - trainer, - event_name=Events.ITERATION_COMPLETED, - log_handler=WeightsScalarHandler(model) - ) - - Explore an experiment with neptune tracking here: - https://ui.neptune.ai/o/shared/org/pytorch-ignite-integration/e/PYTOR1-18/charts - You can save model checkpoints to a Neptune server: - - .. code-block:: python - - from ignite.handlers import Checkpoint - - def score_function(engine): - return engine.state.metrics["accuracy"] - - to_save = {"model": model} - handler = Checkpoint( - to_save, - NeptuneSaver(npt_logger), n_saved=2, - filename_prefix="best", - score_function=score_function, - score_name="validation_accuracy", - global_step_transform=global_step_from_engine(trainer) - ) - validation_evaluator.add_event_handler(Events.COMPLETED, handler) - - It is also possible to use the logger as context manager: - - .. code-block:: python - - from ignite.contrib.handlers.neptune_logger import * - - # We are using the api_token for the anonymous user neptuner but you can use your own. - - with NeptuneLogger(api_token="ANONYMOUS", - project_name="shared/pytorch-ignite-integration", - experiment_name="cnn-mnist", # Optional, - params={"max_epochs": 10}, # Optional, - tags=["pytorch-ignite","minst"] # Optional - ) as npt_logger: - - trainer = Engine(update_fn) - # Attach the logger to the trainer to log training loss at each iteration - npt_logger.attach_output_handler( - trainer, - event_name=Events.ITERATION_COMPLETED, - tag="training", - output_transform=lambda loss: {"loss": loss} - ) - - """ - - def __getattr__(self, attr): - - import neptune - - return getattr(neptune, attr) - - def __init__(self, *args, **kwargs): - try: - import neptune - except ImportError: - raise RuntimeError( - "This contrib module requires neptune-client to be installed. " - "You may install neptune with command: \n pip install neptune-client \n" - ) - - if kwargs.get("offline_mode", False): - self.mode = "offline" - neptune.init(project_qualified_name="dry-run/project", backend=neptune.OfflineBackend()) - else: - self.mode = "online" - neptune.init(api_token=kwargs.get("api_token"), project_qualified_name=kwargs.get("project_name")) - - kwargs["name"] = kwargs.pop("experiment_name", None) - self._experiment_kwargs = { - k: v for k, v in kwargs.items() if k not in ["api_token", "project_name", "offline_mode"] - } - - self.experiment = neptune.create_experiment(**self._experiment_kwargs) - - def close(self): - self.stop() - - def _create_output_handler(self, *args, **kwargs): - return OutputHandler(*args, **kwargs) - - def _create_opt_params_handler(self, *args, **kwargs): - return OptimizerParamsHandler(*args, **kwargs) - - class NeptuneSaver(BaseSaveHandler): """Handler that saves input checkpoint to the Neptune server. diff --git a/ignite/contrib/handlers/param_scheduler.py b/ignite/contrib/handlers/param_scheduler.py index 3a86ec260f00..024d477c82ae 100644 --- a/ignite/contrib/handlers/param_scheduler.py +++ b/ignite/contrib/handlers/param_scheduler.py @@ -4,15 +4,16 @@ import tempfile from abc import ABCMeta, abstractmethod from collections import OrderedDict -from collections.abc import Mapping, Sequence from copy import copy from pathlib import Path -from typing import List, Optional, Union +from typing import Any, List, Mapping, Optional, Sequence, Tuple, Union import torch from torch.optim.lr_scheduler import _LRScheduler from torch.optim.optimizer import Optimizer +from ignite.engine import Engine + class ParamScheduler(metaclass=ABCMeta): """An abstract class for updating an optimizer's parameter value during @@ -33,7 +34,13 @@ class ParamScheduler(metaclass=ABCMeta): """ - def __init__(self, optimizer, param_name, save_history=False, param_group_index=None): + def __init__( + self, + optimizer: Optimizer, + param_name: str, + save_history: bool = False, + param_group_index: Optional[int] = None, + ): if not ( isinstance(optimizer, Optimizer) @@ -51,7 +58,7 @@ def __init__(self, optimizer, param_name, save_history=False, param_group_index= self.event_index = 0 self._state_attrs = ["event_index", "param_name", "save_history", "param_group_index"] - def __call__(self, engine, name=None): + def __call__(self, engine: Engine, name: Optional[str] = None): value = self.get_param() @@ -84,9 +91,7 @@ def __call__(self, engine, name=None): def optimizer_param_groups(self): if self.param_group_index is None: return self.optimizer.param_groups - return [ - self.optimizer.param_groups[self.param_group_index], - ] + return [self.optimizer.param_groups[self.param_group_index]] def state_dict(self): """Returns a dictionary containing a whole state of ParamScheduler. @@ -104,7 +109,7 @@ def state_dict(self): destination[name] = copy(val) return destination - def load_state_dict(self, state_dict): + def load_state_dict(self, state_dict: Mapping): """Copies parameters from :attr:`state_dict` into this ParamScheduler. Args: @@ -137,7 +142,7 @@ def get_param(self) -> Union[List[float], float]: pass @classmethod - def simulate_values(cls, num_events, **scheduler_kwargs): + def simulate_values(cls, num_events: int, **scheduler_kwargs: Any): """Method to simulate scheduled values during `num_events` events. Args: @@ -173,7 +178,7 @@ def simulate_values(cls, num_events, **scheduler_kwargs): return values @classmethod - def plot_values(cls, num_events, **scheduler_kwargs): + def plot_values(cls, num_events: int, **scheduler_kwargs: Mapping): """Method to plot simulated scheduled values during `num_events` events. This class requires `matplotlib package `_ to be installed: @@ -243,16 +248,16 @@ class CyclicalScheduler(ParamScheduler): def __init__( self, - optimizer, - param_name, - start_value, - end_value, - cycle_size, - cycle_mult=1.0, - start_value_mult=1.0, - end_value_mult=1.0, - save_history=False, - param_group_index=None, + optimizer: Optimizer, + param_name: str, + start_value: float, + end_value: float, + cycle_size: int, + cycle_mult: float = 1.0, + start_value_mult: float = 1.0, + end_value_mult: float = 1.0, + save_history: bool = False, + param_group_index: Optional[int] = None, ): super(CyclicalScheduler, self).__init__( optimizer, param_name, save_history=save_history, param_group_index=param_group_index @@ -280,7 +285,7 @@ def __init__( "end_value_mult", ] - def __call__(self, engine, name=None): + def __call__(self, engine: Engine, name: Optional[str] = None): if self.event_index != 0 and self.event_index % self.cycle_size == 0: self.event_index = 0 self.cycle_size *= self.cycle_mult @@ -436,7 +441,7 @@ class ConcatScheduler(ParamScheduler): """ - def __init__(self, schedulers, durations, save_history=False): + def __init__(self, schedulers: List[ParamScheduler], durations: List[int], save_history: bool = False): if not isinstance(schedulers, Sequence): raise TypeError("Argument schedulers should be a sequence, but given {}".format(schedulers)) @@ -513,7 +518,7 @@ def state_dict(self): state_dict["schedulers"].append(s.state_dict()) return state_dict - def load_state_dict(self, state_dict): + def load_state_dict(self, state_dict: Mapping): """Copies parameters from :attr:`state_dict` into this ConcatScheduler. Args: @@ -546,7 +551,7 @@ def _setup_scheduler(self): self.durations[self._scheduler_index] if self._scheduler_index < len(self.durations) else -1 ) - def __call__(self, engine, name=None): + def __call__(self, engine: Engine, name: Optional[str] = None): if self._current_duration == 0: self._scheduler_index += 1 self._setup_scheduler() @@ -564,7 +569,7 @@ def save_history(self): return self._current_scheduler.save_history @save_history.setter - def save_history(self, value): + def save_history(self, value: bool): for s in self.schedulers: s.save_history = value @@ -572,7 +577,14 @@ def get_param(self): return self._current_scheduler.get_param() @classmethod - def simulate_values(cls, num_events, schedulers, durations, param_names=None, **kwargs): + def simulate_values( + cls, + num_events: int, + schedulers: List[ParamScheduler], + durations: List[int], + param_names: Optional[Union[List[str], Tuple[str]]] = None, + **kwargs: Any + ): """Method to simulate scheduled values during num_events events. Args: @@ -663,7 +675,7 @@ class LRScheduler(ParamScheduler): trainer.add_event_handler(Events.ITERATION_COMPLETED, scheduler) """ - def __init__(self, lr_scheduler, save_history=False): + def __init__(self, lr_scheduler: _LRScheduler, save_history=False): if not isinstance(lr_scheduler, _LRScheduler): raise TypeError( @@ -675,11 +687,9 @@ def __init__(self, lr_scheduler, save_history=False): super(LRScheduler, self).__init__( optimizer=self.lr_scheduler.optimizer, param_name="lr", save_history=save_history ) - self._state_attrs += [ - "lr_scheduler", - ] + self._state_attrs += ["lr_scheduler"] - def __call__(self, engine, name=None): + def __call__(self, engine: Engine, name: Optional[str] = None): self.lr_scheduler.last_epoch += 1 super(LRScheduler, self).__call__(engine, name) @@ -696,7 +706,7 @@ def get_param(self) -> Union[float, List[float]]: return lr_list @classmethod - def simulate_values(cls, num_events, lr_scheduler, **kwargs): + def simulate_values(cls, num_events: int, lr_scheduler: _LRScheduler, **kwargs: Any): """Method to simulate scheduled values during num_events events. Args: @@ -719,10 +729,7 @@ def simulate_values(cls, num_events, lr_scheduler, **kwargs): # not perturb original scheduler. with tempfile.TemporaryDirectory() as tmpdirname: cache_filepath = Path(tmpdirname) / "ignite_lr_scheduler_cache.pt" - obj = { - "lr_scheduler": lr_scheduler.state_dict(), - "optimizer": lr_scheduler.optimizer.state_dict(), - } + obj = {"lr_scheduler": lr_scheduler.state_dict(), "optimizer": lr_scheduler.optimizer.state_dict()} torch.save(obj, cache_filepath.as_posix()) values = [] @@ -740,12 +747,12 @@ def simulate_values(cls, num_events, lr_scheduler, **kwargs): def create_lr_scheduler_with_warmup( - lr_scheduler, - warmup_start_value, - warmup_duration, - warmup_end_value=None, - save_history=False, - output_simulated_values=None, + lr_scheduler: Union[ParamScheduler, _LRScheduler], + warmup_start_value: float, + warmup_duration: int, + warmup_end_value: Optional[float] = None, + save_history: bool = False, + output_simulated_values: Optional[List] = None, ): """ Helper method to create a learning rate scheduler with a linear warm-up. @@ -755,7 +762,7 @@ def create_lr_scheduler_with_warmup( after the warm-up. warmup_start_value (float): learning rate start value of the warm-up phase. warmup_duration (int): warm-up phase duration, number of events. - warmup_end_value (float): learning rate end value of the warm-up phase, (default=None). If None, + warmup_end_value (float, optional): learning rate end value of the warm-up phase, (default=None). If None, warmup_end_value is set to optimizer initial lr. save_history (bool, optional): whether to log the parameter values to `engine.state.param_history`, (default=False). @@ -842,9 +849,7 @@ def create_lr_scheduler_with_warmup( warmup_scheduler = ParamGroupScheduler(warmup_schedulers, save_history=save_history) schedulers = [warmup_scheduler, lr_scheduler] - durations = [ - milestones_values[-1][0] + 1, - ] + durations = [milestones_values[-1][0] + 1] combined_scheduler = ConcatScheduler(schedulers, durations=durations, save_history=save_history) if output_simulated_values is not None: @@ -891,7 +896,14 @@ class PiecewiseLinear(ParamScheduler): # """ - def __init__(self, optimizer, param_name, milestones_values, save_history=False, param_group_index=None): + def __init__( + self, + optimizer: Optimizer, + param_name: str, + milestones_values: List[Tuple[int, float]], + save_history: bool = False, + param_group_index: Optional[int] = None, + ): super(PiecewiseLinear, self).__init__(optimizer, param_name, save_history, param_group_index=param_group_index) if not isinstance(milestones_values, Sequence): @@ -928,12 +940,7 @@ def _get_start_end(self): if self.milestones[0] > self.event_index: return self.event_index - 1, self.event_index, self.values[0], self.values[0] elif self.milestones[-1] <= self.event_index: - return ( - self.event_index, - self.event_index + 1, - self.values[-1], - self.values[-1], - ) + return (self.event_index, self.event_index + 1, self.values[-1], self.values[-1]) elif self.milestones[self._index] <= self.event_index < self.milestones[self._index + 1]: return ( self.milestones[self._index], @@ -978,7 +985,7 @@ class ParamGroupScheduler: """ - def __init__(self, schedulers: List[ParamScheduler], names: Optional[List[str]] = None, save_history=False): + def __init__(self, schedulers: List[ParamScheduler], names: Optional[List[str]] = None, save_history: bool = False): if not isinstance(schedulers, Sequence): raise TypeError("Argument schedulers should be a list/tuple, but given {}".format(schedulers)) @@ -1011,7 +1018,7 @@ def __init__(self, schedulers: List[ParamScheduler], names: Optional[List[str]] self.optimizer = [s.optimizer for s in self.schedulers] self.param_name = [s.param_name for s in self.schedulers] - def __call__(self, engine, name=None): + def __call__(self, engine: Engine, name: Optional[str] = None): for scheduler, name in zip(self.schedulers, self.names): scheduler(engine, name) @@ -1024,7 +1031,7 @@ def save_history(self): return self.schedulers[0].save_history @save_history.setter - def save_history(self, value): + def save_history(self, value: bool): for s in self.schedulers: s.save_history = value @@ -1041,7 +1048,7 @@ def state_dict(self): state_dict["schedulers"].append((n, s.state_dict())) return state_dict - def load_state_dict(self, state_dict): + def load_state_dict(self, state_dict: Mapping): """Copies parameters from :attr:`state_dict` into this ParamScheduler. Args: @@ -1072,12 +1079,12 @@ def load_state_dict(self, state_dict): s.load_state_dict(sd) @classmethod - def simulate_values(cls, num_events, schedulers, **kwargs): + def simulate_values(cls, num_events: int, schedulers: _LRScheduler, **kwargs: Any): """Method to simulate scheduled values during num_events events. Args: num_events (int): number of events during the simulation. - lr_schedulers (subclass of `torch.optim.lr_scheduler._LRScheduler`): lr_scheduler object to wrap. + schedulers (subclass of `torch.optim.lr_scheduler._LRScheduler`): lr_scheduler object to wrap. Returns: list of pairs: [event_index, value] @@ -1110,7 +1117,7 @@ def simulate_values(cls, num_events, schedulers, **kwargs): return values -def _get_fake_optimizer(optimizer_cls=None, **kwargs): +def _get_fake_optimizer(optimizer_cls: Optional[Optimizer] = None, **kwargs: Any): t = torch.zeros([1], requires_grad=True) if optimizer_cls is None: optimizer_cls = torch.optim.SGD diff --git a/ignite/contrib/handlers/polyaxon_logger.py b/ignite/contrib/handlers/polyaxon_logger.py index e675cbee07e7..9d75dff12481 100644 --- a/ignite/contrib/handlers/polyaxon_logger.py +++ b/ignite/contrib/handlers/polyaxon_logger.py @@ -1,14 +1,117 @@ import numbers import warnings +from typing import Any, Callable, List, Optional, Union import torch +from torch.optim import Optimizer from ignite.contrib.handlers.base_logger import BaseLogger, BaseOptimizerParamsHandler, BaseOutputHandler +from ignite.engine import Engine, EventEnum from ignite.handlers import global_step_from_engine __all__ = ["PolyaxonLogger", "OutputHandler", "OptimizerParamsHandler", "global_step_from_engine"] +class PolyaxonLogger(BaseLogger): + """ + `Polyaxon `_ tracking client handler to log parameters and metrics during the training + and validation. + + This class requires `polyaxon-client `_ package to be installed: + + .. code-block:: bash + + pip install polyaxon-client + + + Examples: + + .. code-block:: python + + from ignite.contrib.handlers.polyaxon_logger import * + + # Create a logger + plx_logger = PolyaxonLogger() + + # Log experiment parameters: + plx_logger.log_params(**{ + "seed": seed, + "batch_size": batch_size, + "model": model.__class__.__name__, + + "pytorch version": torch.__version__, + "ignite version": ignite.__version__, + "cuda version": torch.version.cuda, + "device name": torch.cuda.get_device_name(0) + }) + + # Attach the logger to the trainer to log training loss at each iteration + plx_logger.attach_output_handler( + trainer, + event_name=Events.ITERATION_COMPLETED, + tag="training", + output_transform=lambda loss: {"loss": loss} + ) + + # Attach the logger to the evaluator on the training dataset and log NLL, Accuracy metrics after each epoch + # We setup `global_step_transform=global_step_from_engine(trainer)` to take the epoch + # of the `trainer` instead of `train_evaluator`. + plx_logger.attach_output_handler( + train_evaluator, + event_name=Events.EPOCH_COMPLETED, + tag="training", + metric_names=["nll", "accuracy"], + global_step_transform=global_step_from_engine(trainer), + ) + + # Attach the logger to the evaluator on the validation dataset and log NLL, Accuracy metrics after + # each epoch. We setup `global_step_transform=global_step_from_engine(trainer)` to take the epoch of the + # `trainer` instead of `evaluator`. + plx_logger.attach_output_handler( + evaluator, + event_name=Events.EPOCH_COMPLETED, + tag="validation", + metric_names=["nll", "accuracy"], + global_step_transform=global_step_from_engine(trainer)), + ) + + # Attach the logger to the trainer to log optimizer's parameters, e.g. learning rate at each iteration + plx_logger.attach_opt_params_handler( + trainer, + event_name=Events.ITERATION_STARTED, + optimizer=optimizer, + param_name='lr' # optional + ) + + Args: + *args: Positional arguments accepted from + `Experiment `_. + **kwargs: Keyword arguments accepted from + `Experiment `_. + + """ + + def __init__(self, *args: Any, **kwargs: Any): + try: + from polyaxon_client.tracking import Experiment + except ImportError: + raise RuntimeError( + "This contrib module requires polyaxon-client to be installed. " + "Please install it with command: \n pip install polyaxon-client" + ) + + self.experiment = Experiment(*args, **kwargs) + + def __getattr__(self, attr: Any): + return getattr(self.experiment, attr) + + def _create_output_handler(self, *args: Any, **kwargs: Any): + return OutputHandler(*args, **kwargs) + + def _create_opt_params_handler(self, *args: Any, **kwargs: Any): + return OptimizerParamsHandler(*args, **kwargs) + + class OutputHandler(BaseOutputHandler): """Helper handler to log engine's output and/or metrics. @@ -95,10 +198,16 @@ def global_step_transform(engine, event_name): """ - def __init__(self, tag, metric_names=None, output_transform=None, global_step_transform=None): + def __init__( + self, + tag: str, + metric_names: Optional[List[str]] = None, + output_transform: Optional[Callable] = None, + global_step_transform: Optional[Callable] = None, + ): super(OutputHandler, self).__init__(tag, metric_names, output_transform, global_step_transform) - def __call__(self, engine, logger, event_name): + def __call__(self, engine: Engine, logger: PolyaxonLogger, event_name: Union[str, EventEnum]): if not isinstance(logger, PolyaxonLogger): raise RuntimeError("Handler 'OutputHandler' works only with PolyaxonLogger") @@ -160,10 +269,10 @@ class OptimizerParamsHandler(BaseOptimizerParamsHandler): tag (str, optional): common title for all produced plots. For example, "generator" """ - def __init__(self, optimizer, param_name="lr", tag=None): + def __init__(self, optimizer: Optimizer, param_name: str = "lr", tag: Optional[str] = None): super(OptimizerParamsHandler, self).__init__(optimizer, param_name, tag) - def __call__(self, engine, logger, event_name): + def __call__(self, engine: Engine, logger: PolyaxonLogger, event_name: Union[str, EventEnum]): if not isinstance(logger, PolyaxonLogger): raise RuntimeError("Handler OptimizerParamsHandler works only with PolyaxonLogger") @@ -175,103 +284,3 @@ def __call__(self, engine, logger, event_name): } params["step"] = global_step logger.log_metrics(**params) - - -class PolyaxonLogger(BaseLogger): - """ - `Polyaxon `_ tracking client handler to log parameters and metrics during the training - and validation. - - This class requires `polyaxon-client `_ package to be installed: - - .. code-block:: bash - - pip install polyaxon-client - - - Examples: - - .. code-block:: python - - from ignite.contrib.handlers.polyaxon_logger import * - - # Create a logger - plx_logger = PolyaxonLogger() - - # Log experiment parameters: - plx_logger.log_params(**{ - "seed": seed, - "batch_size": batch_size, - "model": model.__class__.__name__, - - "pytorch version": torch.__version__, - "ignite version": ignite.__version__, - "cuda version": torch.version.cuda, - "device name": torch.cuda.get_device_name(0) - }) - - # Attach the logger to the trainer to log training loss at each iteration - plx_logger.attach_output_handler( - trainer, - event_name=Events.ITERATION_COMPLETED, - tag="training", - output_transform=lambda loss: {"loss": loss} - ) - - # Attach the logger to the evaluator on the training dataset and log NLL, Accuracy metrics after each epoch - # We setup `global_step_transform=global_step_from_engine(trainer)` to take the epoch - # of the `trainer` instead of `train_evaluator`. - plx_logger.attach_output_handler( - train_evaluator, - event_name=Events.EPOCH_COMPLETED, - tag="training", - metric_names=["nll", "accuracy"], - global_step_transform=global_step_from_engine(trainer), - ) - - # Attach the logger to the evaluator on the validation dataset and log NLL, Accuracy metrics after - # each epoch. We setup `global_step_transform=global_step_from_engine(trainer)` to take the epoch of the - # `trainer` instead of `evaluator`. - plx_logger.attach_output_handler( - evaluator, - event_name=Events.EPOCH_COMPLETED, - tag="validation", - metric_names=["nll", "accuracy"], - global_step_transform=global_step_from_engine(trainer)), - ) - - # Attach the logger to the trainer to log optimizer's parameters, e.g. learning rate at each iteration - plx_logger.attach_opt_params_handler( - trainer, - event_name=Events.ITERATION_STARTED, - optimizer=optimizer, - param_name='lr' # optional - ) - - Args: - *args: Positional arguments accepted from - `Experiment `_. - **kwargs: Keyword arguments accepted from - `Experiment `_. - - """ - - def __init__(self, *args, **kwargs): - try: - from polyaxon_client.tracking import Experiment - except ImportError: - raise RuntimeError( - "This contrib module requires polyaxon-client to be installed. " - "Please install it with command: \n pip install polyaxon-client" - ) - - self.experiment = Experiment(*args, **kwargs) - - def __getattr__(self, attr): - return getattr(self.experiment, attr) - - def _create_output_handler(self, *args, **kwargs): - return OutputHandler(*args, **kwargs) - - def _create_opt_params_handler(self, *args, **kwargs): - return OptimizerParamsHandler(*args, **kwargs) diff --git a/ignite/contrib/handlers/stores.py b/ignite/contrib/handlers/stores.py index b5514a76d4ed..6bd2b5167bdb 100644 --- a/ignite/contrib/handlers/stores.py +++ b/ignite/contrib/handlers/stores.py @@ -1,4 +1,6 @@ -from ignite.engine import Events +from typing import Callable, Optional + +from ignite.engine import Engine, Events class EpochOutputStore: @@ -31,17 +33,17 @@ def log_training_results(engine): """ - def __init__(self, output_transform=lambda x: x): + def __init__(self, output_transform: Callable = lambda x: x): self.data = None self.output_transform = output_transform def reset(self): self.data = [] - def update(self, engine): + def update(self, engine: Engine): output = self.output_transform(engine.state.output) self.data.append(output) - def attach(self, engine): + def attach(self, engine: Engine): engine.add_event_handler(Events.EPOCH_STARTED, self.reset) engine.add_event_handler(Events.ITERATION_COMPLETED, self.update) diff --git a/ignite/contrib/handlers/tensorboard_logger.py b/ignite/contrib/handlers/tensorboard_logger.py index 0bf51adf53ff..798e11769f56 100644 --- a/ignite/contrib/handlers/tensorboard_logger.py +++ b/ignite/contrib/handlers/tensorboard_logger.py @@ -1,7 +1,10 @@ import numbers import warnings +from typing import Any, Callable, List, Optional, Union import torch +import torch.nn as nn +from torch.optim import Optimizer from ignite.contrib.handlers.base_logger import ( BaseLogger, @@ -10,6 +13,7 @@ BaseWeightsHistHandler, BaseWeightsScalarHandler, ) +from ignite.engine import Engine, EventEnum from ignite.handlers import global_step_from_engine __all__ = [ @@ -24,6 +28,152 @@ ] +class TensorboardLogger(BaseLogger): + """ + TensorBoard handler to log metrics, model/optimizer parameters, gradients during the training and validation. + + By default, this class favors `tensorboardX `_ package if installed: + + .. code-block:: bash + + pip install tensorboardX + + otherwise, it falls back to using + `PyTorch's SummaryWriter + `_ + (>=v1.2.0). + + Args: + *args: Positional arguments accepted from + `SummaryWriter + `_. + **kwargs: Keyword arguments accepted from + `SummaryWriter + `_. + For example, `log_dir` to setup path to the directory where to log. + + Examples: + + .. code-block:: python + + from ignite.contrib.handlers.tensorboard_logger import * + + # Create a logger + tb_logger = TensorboardLogger(log_dir="experiments/tb_logs") + + # Attach the logger to the trainer to log training loss at each iteration + tb_logger.attach_output_handler( + trainer, + event_name=Events.ITERATION_COMPLETED, + tag="training", + output_transform=lambda loss: {"loss": loss} + ) + + # Attach the logger to the evaluator on the training dataset and log NLL, Accuracy metrics after each epoch + # We setup `global_step_transform=global_step_from_engine(trainer)` to take the epoch + # of the `trainer` instead of `train_evaluator`. + tb_logger.attach_output_handler( + train_evaluator, + event_name=Events.EPOCH_COMPLETED, + tag="training", + metric_names=["nll", "accuracy"], + global_step_transform=global_step_from_engine(trainer), + ) + + # Attach the logger to the evaluator on the validation dataset and log NLL, Accuracy metrics after + # each epoch. We setup `global_step_transform=global_step_from_engine(trainer)` to take the epoch of the + # `trainer` instead of `evaluator`. + tb_logger.attach_output_handler( + evaluator, + event_name=Events.EPOCH_COMPLETED, + tag="validation", + metric_names=["nll", "accuracy"], + global_step_transform=global_step_from_engine(trainer)), + ) + + # Attach the logger to the trainer to log optimizer's parameters, e.g. learning rate at each iteration + tb_logger.attach_opt_params_handler( + trainer, + event_name=Events.ITERATION_STARTED, + optimizer=optimizer, + param_name='lr' # optional + ) + + # Attach the logger to the trainer to log model's weights norm after each iteration + tb_logger.attach( + trainer, + event_name=Events.ITERATION_COMPLETED, + log_handler=WeightsScalarHandler(model) + ) + + # Attach the logger to the trainer to log model's weights as a histogram after each epoch + tb_logger.attach( + trainer, + event_name=Events.EPOCH_COMPLETED, + log_handler=WeightsHistHandler(model) + ) + + # Attach the logger to the trainer to log model's gradients norm after each iteration + tb_logger.attach( + trainer, + event_name=Events.ITERATION_COMPLETED, + log_handler=GradsScalarHandler(model) + ) + + # Attach the logger to the trainer to log model's gradients as a histogram after each epoch + tb_logger.attach( + trainer, + event_name=Events.EPOCH_COMPLETED, + log_handler=GradsHistHandler(model) + ) + + # We need to close the logger with we are done + tb_logger.close() + + It is also possible to use the logger as context manager: + + .. code-block:: python + + from ignite.contrib.handlers.tensorboard_logger import * + + with TensorboardLogger(log_dir="experiments/tb_logs") as tb_logger: + + trainer = Engine(update_fn) + # Attach the logger to the trainer to log training loss at each iteration + tb_logger.attach_output_handler( + trainer, + event_name=Events.ITERATION_COMPLETED, + tag="training", + output_transform=lambda loss: {"loss": loss} + ) + + """ + + def __init__(self, *args: Any, **kwargs: Any): + try: + from tensorboardX import SummaryWriter + except ImportError: + try: + from torch.utils.tensorboard import SummaryWriter + except ImportError: + raise RuntimeError( + "This contrib module requires either tensorboardX or torch >= 1.2.0. " + "You may install tensorboardX with command: \n pip install tensorboardX \n" + "or upgrade PyTorch using your package manager of choice (pip or conda)." + ) + + self.writer = SummaryWriter(*args, **kwargs) + + def close(self): + self.writer.close() + + def _create_output_handler(self, *args: Any, **kwargs: Any): + return OutputHandler(*args, **kwargs) + + def _create_opt_params_handler(self, *args: Any, **kwargs: Any): + return OptimizerParamsHandler(*args, **kwargs) + + class OutputHandler(BaseOutputHandler): """Helper handler to log engine's output and/or metrics @@ -110,10 +260,16 @@ def global_step_transform(engine, event_name): """ - def __init__(self, tag, metric_names=None, output_transform=None, global_step_transform=None): + def __init__( + self, + tag: str, + metric_names: Optional[List[str]] = None, + output_transform: Optional[Callable] = None, + global_step_transform: Optional[Callable] = None, + ): super(OutputHandler, self).__init__(tag, metric_names, output_transform, global_step_transform) - def __call__(self, engine, logger, event_name): + def __call__(self, engine: Engine, logger: TensorboardLogger, event_name: Union[str, EventEnum]): if not isinstance(logger, TensorboardLogger): raise RuntimeError("Handler 'OutputHandler' works only with TensorboardLogger") @@ -169,10 +325,10 @@ class OptimizerParamsHandler(BaseOptimizerParamsHandler): tag (str, optional): common title for all produced plots. For example, "generator" """ - def __init__(self, optimizer, param_name="lr", tag=None): + def __init__(self, optimizer: Optimizer, param_name: str = "lr", tag: Optional[str] = None): super(OptimizerParamsHandler, self).__init__(optimizer, param_name, tag) - def __call__(self, engine, logger, event_name): + def __call__(self, engine: Engine, logger: TensorboardLogger, event_name: Union[str, EventEnum]): if not isinstance(logger, TensorboardLogger): raise RuntimeError("Handler OptimizerParamsHandler works only with TensorboardLogger") @@ -215,10 +371,10 @@ class WeightsScalarHandler(BaseWeightsScalarHandler): """ - def __init__(self, model, reduction=torch.norm, tag=None): + def __init__(self, model: nn.Module, reduction: Callable = torch.norm, tag: Optional[str] = None): super(WeightsScalarHandler, self).__init__(model, reduction, tag=tag) - def __call__(self, engine, logger, event_name): + def __call__(self, engine: Engine, logger: TensorboardLogger, event_name: Union[str, EventEnum]): if not isinstance(logger, TensorboardLogger): raise RuntimeError("Handler 'WeightsScalarHandler' works only with TensorboardLogger") @@ -260,10 +416,10 @@ class WeightsHistHandler(BaseWeightsHistHandler): """ - def __init__(self, model, tag=None): + def __init__(self, model: nn.Module, tag: Optional[str] = None): super(WeightsHistHandler, self).__init__(model, tag=tag) - def __call__(self, engine, logger, event_name): + def __call__(self, engine: Engine, logger: TensorboardLogger, event_name: Union[str, EventEnum]): if not isinstance(logger, TensorboardLogger): raise RuntimeError("Handler 'WeightsHistHandler' works only with TensorboardLogger") @@ -309,10 +465,10 @@ class GradsScalarHandler(BaseWeightsScalarHandler): """ - def __init__(self, model, reduction=torch.norm, tag=None): + def __init__(self, model: nn.Module, reduction: Callable = torch.norm, tag: Optional[str] = None): super(GradsScalarHandler, self).__init__(model, reduction, tag=tag) - def __call__(self, engine, logger, event_name): + def __call__(self, engine: Engine, logger: TensorboardLogger, event_name: Union[str, EventEnum]): if not isinstance(logger, TensorboardLogger): raise RuntimeError("Handler 'GradsScalarHandler' works only with TensorboardLogger") @@ -353,10 +509,10 @@ class GradsHistHandler(BaseWeightsHistHandler): """ - def __init__(self, model, tag=None): + def __init__(self, model: nn.Module, tag: Optional[str] = None): super(GradsHistHandler, self).__init__(model, tag=tag) - def __call__(self, engine, logger, event_name): + def __call__(self, engine: Engine, logger: TensorboardLogger, event_name: Union[str, EventEnum]): if not isinstance(logger, TensorboardLogger): raise RuntimeError("Handler 'GradsHistHandler' works only with TensorboardLogger") @@ -370,149 +526,3 @@ def __call__(self, engine, logger, event_name): logger.writer.add_histogram( tag="{}grads/{}".format(tag_prefix, name), values=p.grad.detach().cpu().numpy(), global_step=global_step ) - - -class TensorboardLogger(BaseLogger): - """ - TensorBoard handler to log metrics, model/optimizer parameters, gradients during the training and validation. - - By default, this class favors `tensorboardX `_ package if installed: - - .. code-block:: bash - - pip install tensorboardX - - otherwise, it falls back to using - `PyTorch's SummaryWriter - `_ - (>=v1.2.0). - - Args: - *args: Positional arguments accepted from - `SummaryWriter - `_. - **kwargs: Keyword arguments accepted from - `SummaryWriter - `_. - For example, `log_dir` to setup path to the directory where to log. - - Examples: - - .. code-block:: python - - from ignite.contrib.handlers.tensorboard_logger import * - - # Create a logger - tb_logger = TensorboardLogger(log_dir="experiments/tb_logs") - - # Attach the logger to the trainer to log training loss at each iteration - tb_logger.attach_output_handler( - trainer, - event_name=Events.ITERATION_COMPLETED, - tag="training", - output_transform=lambda loss: {"loss": loss} - ) - - # Attach the logger to the evaluator on the training dataset and log NLL, Accuracy metrics after each epoch - # We setup `global_step_transform=global_step_from_engine(trainer)` to take the epoch - # of the `trainer` instead of `train_evaluator`. - tb_logger.attach_output_handler( - train_evaluator, - event_name=Events.EPOCH_COMPLETED, - tag="training", - metric_names=["nll", "accuracy"], - global_step_transform=global_step_from_engine(trainer), - ) - - # Attach the logger to the evaluator on the validation dataset and log NLL, Accuracy metrics after - # each epoch. We setup `global_step_transform=global_step_from_engine(trainer)` to take the epoch of the - # `trainer` instead of `evaluator`. - tb_logger.attach_output_handler( - evaluator, - event_name=Events.EPOCH_COMPLETED, - tag="validation", - metric_names=["nll", "accuracy"], - global_step_transform=global_step_from_engine(trainer)), - ) - - # Attach the logger to the trainer to log optimizer's parameters, e.g. learning rate at each iteration - tb_logger.attach_opt_params_handler( - trainer, - event_name=Events.ITERATION_STARTED, - optimizer=optimizer, - param_name='lr' # optional - ) - - # Attach the logger to the trainer to log model's weights norm after each iteration - tb_logger.attach( - trainer, - event_name=Events.ITERATION_COMPLETED, - log_handler=WeightsScalarHandler(model) - ) - - # Attach the logger to the trainer to log model's weights as a histogram after each epoch - tb_logger.attach( - trainer, - event_name=Events.EPOCH_COMPLETED, - log_handler=WeightsHistHandler(model) - ) - - # Attach the logger to the trainer to log model's gradients norm after each iteration - tb_logger.attach( - trainer, - event_name=Events.ITERATION_COMPLETED, - log_handler=GradsScalarHandler(model) - ) - - # Attach the logger to the trainer to log model's gradients as a histogram after each epoch - tb_logger.attach( - trainer, - event_name=Events.EPOCH_COMPLETED, - log_handler=GradsHistHandler(model) - ) - - # We need to close the logger with we are done - tb_logger.close() - - It is also possible to use the logger as context manager: - - .. code-block:: python - - from ignite.contrib.handlers.tensorboard_logger import * - - with TensorboardLogger(log_dir="experiments/tb_logs") as tb_logger: - - trainer = Engine(update_fn) - # Attach the logger to the trainer to log training loss at each iteration - tb_logger.attach_output_handler( - trainer, - event_name=Events.ITERATION_COMPLETED, - tag="training", - output_transform=lambda loss: {"loss": loss} - ) - - """ - - def __init__(self, *args, **kwargs): - try: - from tensorboardX import SummaryWriter - except ImportError: - try: - from torch.utils.tensorboard import SummaryWriter - except ImportError: - raise RuntimeError( - "This contrib module requires either tensorboardX or torch >= 1.2.0. " - "You may install tensorboardX with command: \n pip install tensorboardX \n" - "or upgrade PyTorch using your package manager of choice (pip or conda)." - ) - - self.writer = SummaryWriter(*args, **kwargs) - - def close(self): - self.writer.close() - - def _create_output_handler(self, *args, **kwargs): - return OutputHandler(*args, **kwargs) - - def _create_opt_params_handler(self, *args, **kwargs): - return OptimizerParamsHandler(*args, **kwargs) diff --git a/ignite/contrib/handlers/time_profilers.py b/ignite/contrib/handlers/time_profilers.py index 178216bcea70..64c8fe23b89d 100644 --- a/ignite/contrib/handlers/time_profilers.py +++ b/ignite/contrib/handlers/time_profilers.py @@ -1,4 +1,5 @@ from collections import OrderedDict +from typing import Dict, Iterable, Mapping, Sequence, Union import torch @@ -77,7 +78,7 @@ def __init__(self): self._as_last_completed, ] - def _reset(self, num_epochs, total_num_iters): + def _reset(self, num_epochs: int, total_num_iters: int): self.dataflow_times = torch.zeros(total_num_iters) self.processing_times = torch.zeros(total_num_iters) self.event_handlers_times = { @@ -91,7 +92,7 @@ def _reset(self, num_epochs, total_num_iters): Events.GET_BATCH_STARTED: torch.zeros(total_num_iters), } - def _as_first_started(self, engine): + def _as_first_started(self, engine: Engine): if hasattr(engine.state.dataloader, "__len__"): num_iters_per_epoch = len(engine.state.dataloader) else: @@ -123,30 +124,30 @@ def _as_first_started(self, engine): # Let's go self._event_handlers_timer.reset() - def _as_last_started(self, engine): + def _as_last_started(self, engine: Engine): self.event_handlers_times[Events.STARTED][0] = self._event_handlers_timer.value() - def _as_first_epoch_started(self, engine): + def _as_first_epoch_started(self, engine: Engine): self._event_handlers_timer.reset() - def _as_last_epoch_started(self, engine): + def _as_last_epoch_started(self, engine: Engine): t = self._event_handlers_timer.value() e = engine.state.epoch - 1 self.event_handlers_times[Events.EPOCH_STARTED][e] = t - def _as_first_get_batch_started(self, engine): + def _as_first_get_batch_started(self, engine: Engine): self._event_handlers_timer.reset() self._dataflow_timer.reset() - def _as_last_get_batch_started(self, engine): + def _as_last_get_batch_started(self, engine: Engine): t = self._event_handlers_timer.value() i = engine.state.iteration - 1 self.event_handlers_times[Events.GET_BATCH_STARTED][i] = t - def _as_first_get_batch_completed(self, engine): + def _as_first_get_batch_completed(self, engine: Engine): self._event_handlers_timer.reset() - def _as_last_get_batch_completed(self, engine): + def _as_last_get_batch_completed(self, engine: Engine): t = self._event_handlers_timer.value() i = engine.state.iteration - 1 self.event_handlers_times[Events.GET_BATCH_COMPLETED][i] = t @@ -156,40 +157,40 @@ def _as_last_get_batch_completed(self, engine): self._dataflow_timer.reset() - def _as_first_iter_started(self, engine): + def _as_first_iter_started(self, engine: Engine): self._event_handlers_timer.reset() - def _as_last_iter_started(self, engine): + def _as_last_iter_started(self, engine: Engine): t = self._event_handlers_timer.value() i = engine.state.iteration - 1 self.event_handlers_times[Events.ITERATION_STARTED][i] = t self._processing_timer.reset() - def _as_first_iter_completed(self, engine): + def _as_first_iter_completed(self, engine: Engine): t = self._processing_timer.value() i = engine.state.iteration - 1 self.processing_times[i] = t self._event_handlers_timer.reset() - def _as_last_iter_completed(self, engine): + def _as_last_iter_completed(self, engine: Engine): t = self._event_handlers_timer.value() i = engine.state.iteration - 1 self.event_handlers_times[Events.ITERATION_COMPLETED][i] = t - def _as_first_epoch_completed(self, engine): + def _as_first_epoch_completed(self, engine: Engine): self._event_handlers_timer.reset() - def _as_last_epoch_completed(self, engine): + def _as_last_epoch_completed(self, engine: Engine): t = self._event_handlers_timer.value() e = engine.state.epoch - 1 self.event_handlers_times[Events.EPOCH_COMPLETED][e] = t - def _as_first_completed(self, engine): + def _as_first_completed(self, engine: Engine): self._event_handlers_timer.reset() - def _as_last_completed(self, engine): + def _as_last_completed(self, engine: Engine): self.event_handlers_times[Events.COMPLETED][0] = self._event_handlers_timer.value() # Remove added handlers: @@ -201,7 +202,7 @@ def _as_last_completed(self, engine): for e, m in zip(self._events, self._lmethods): engine.remove_event_handler(m, e) - def attach(self, engine): + def attach(self, engine: Engine): if not isinstance(engine, Engine): raise TypeError("Argument engine should be ignite.engine.Engine, " "but given {}".format(type(engine))) @@ -209,7 +210,7 @@ def attach(self, engine): engine._event_handlers[Events.STARTED].insert(0, (self._as_first_started, (engine,), {})) @staticmethod - def _compute_basic_stats(data): + def _compute_basic_stats(data: Sequence): # compute on non-zero data: data = data[data > 0] out = [("total", torch.sum(data).item() if len(data) > 0 else "not yet triggered")] @@ -255,7 +256,7 @@ def get_results(self): ] ) - def write_results(self, output_path): + def write_results(self, output_path: str): """ Method to store the unaggregated profiling results to a csv file @@ -334,7 +335,7 @@ def write_results(self, output_path): results_df.to_csv(output_path, index=False) @staticmethod - def print_results(results): + def print_results(results: Union[Dict, Iterable]): """ Method to print the aggregated results from the profiler @@ -380,14 +381,14 @@ def print_results(results): """ - def to_str(v): + def to_str(v: Union[str, tuple]): if isinstance(v, str): return v elif isinstance(v, tuple): return "{:.5f}/{}".format(v[0], v[1]) return "{:.5f}".format(v) - def odict_to_str(d): + def odict_to_str(d: Mapping): out = " | ".join([to_str(v) for v in d.values()]) return out diff --git a/ignite/contrib/handlers/tqdm_logger.py b/ignite/contrib/handlers/tqdm_logger.py index f1a4d58d3c03..979986079a06 100644 --- a/ignite/contrib/handlers/tqdm_logger.py +++ b/ignite/contrib/handlers/tqdm_logger.py @@ -1,87 +1,13 @@ # -*- coding: utf-8 -*- import warnings -from typing import Any, Mapping +from enum import Enum +from typing import Any, Callable, Optional, Union import torch from ignite.contrib.handlers.base_logger import BaseLogger, BaseOutputHandler from ignite.engine import Engine, Events -from ignite.engine.events import CallableEventWithFilter - - -class _OutputHandler(BaseOutputHandler): - """Helper handler to log engine's output and/or metrics - - Args: - description (str): progress bar description. - metric_names (list of str, optional): list of metric names to plot or a string "all" to plot all available - metrics. - output_transform (callable, optional): output transform function to prepare `engine.state.output` as a number. - For example, `output_transform = lambda output: output` - This function can also return a dictionary, e.g `{'loss': loss1, 'another_loss': loss2}` to label the plot - with corresponding keys. - closing_event_name: event's name on which the progress bar is closed. Valid events are from - :class:`~ignite.engine.events.Events` or any `event_name` added by - :meth:`~ignite.engine.engine.Engine.register_events`. - - """ - - def __init__( - self, description, metric_names=None, output_transform=None, closing_event_name=Events.EPOCH_COMPLETED - ): - if metric_names is None and output_transform is None: - # This helps to avoid 'Either metric_names or output_transform should be defined' of BaseOutputHandler - metric_names = [] - super(_OutputHandler, self).__init__(description, metric_names, output_transform, global_step_transform=None) - self.closing_event_name = closing_event_name - - @staticmethod - def get_max_number_events(event_name, engine): - if event_name in (Events.ITERATION_STARTED, Events.ITERATION_COMPLETED): - return engine.state.epoch_length - if event_name in (Events.EPOCH_STARTED, Events.EPOCH_COMPLETED): - return engine.state.max_epochs - return 1 - - def __call__(self, engine, logger, event_name): - - pbar_total = self.get_max_number_events(event_name, engine) - if logger.pbar is None: - logger._reset(pbar_total=pbar_total) - - max_epochs = engine.state.max_epochs - default_desc = "Iteration" if max_epochs == 1 else "Epoch" - - desc = self.tag or default_desc - max_num_of_closing_events = self.get_max_number_events(self.closing_event_name, engine) - if max_num_of_closing_events > 1: - global_step = engine.state.get_event_attrib_value(self.closing_event_name) - desc += " [{}/{}]".format(global_step, max_num_of_closing_events) - logger.pbar.set_description(desc) - - metrics = self._setup_output_metrics(engine) - - rendered_metrics = {} - for key, value in metrics.items(): - if isinstance(value, torch.Tensor): - if value.ndimension() == 0: - rendered_metrics[key] = value.item() - elif value.ndimension() == 1: - for i, v in enumerate(value): - k = "{}_{}".format(key, i) - rendered_metrics[k] = v.item() - else: - warnings.warn("ProgressBar can not log " "tensor with {} dimensions".format(value.ndimension())) - else: - rendered_metrics[key] = value - - if rendered_metrics: - logger.pbar.set_postfix(**rendered_metrics) - - global_step = engine.state.get_event_attrib_value(event_name) - if pbar_total is not None: - global_step = (global_step - 1) % pbar_total + 1 - logger.pbar.update(global_step - logger.pbar.n) +from ignite.engine.events import CallableEventWithFilter, EventEnum class ProgressBar(BaseLogger): @@ -177,9 +103,9 @@ class ProgressBar(BaseLogger): def __init__( self, - persist=False, - bar_format="{desc}[{n_fmt}/{total_fmt}] {percentage:3.0f}%|{bar}{postfix} [{elapsed}<{remaining}]", - **tqdm_kwargs + persist: bool = False, + bar_format: str = "{desc}[{n_fmt}/{total_fmt}] {percentage:3.0f}%|{bar}{postfix} [{elapsed}<{remaining}]", + **tqdm_kwargs: Any ): try: @@ -196,12 +122,12 @@ def __init__( self.bar_format = bar_format self.tqdm_kwargs = tqdm_kwargs - def _reset(self, pbar_total): + def _reset(self, pbar_total: int): self.pbar = self.pbar_cls( total=pbar_total, leave=self.persist, bar_format=self.bar_format, initial=1, **self.tqdm_kwargs ) - def _close(self, engine): + def _close(self, engine: Engine): if self.pbar is not None: # https://github.com/tqdm/notebook.py#L240-L250 # issue #1115 : notebook backend of tqdm checks if n < total (error or KeyboardInterrupt) @@ -212,12 +138,12 @@ def _close(self, engine): self.pbar = None @staticmethod - def _compare_lt(event1, event2): + def _compare_lt(event1: EventEnum, event2: EventEnum): i1 = ProgressBar._events_order.index(event1) i2 = ProgressBar._events_order.index(event2) return i1 < i2 - def log_message(self, message): + def log_message(self, message: str): """ Logs a message, preserving the progress bar correct output format. @@ -230,11 +156,11 @@ def log_message(self, message): def attach( self, - engine, - metric_names=None, - output_transform=None, - event_name=Events.ITERATION_COMPLETED, - closing_event_name=Events.EPOCH_COMPLETED, + engine: Engine, + metric_names: Optional[str] = None, + output_transform: Optional[Callable] = None, + event_name: Events = Events.ITERATION_COMPLETED, + closing_event_name: Events = Events.EPOCH_COMPLETED, ): """ Attaches the progress bar to an engine object. @@ -273,13 +199,92 @@ def attach( super(ProgressBar, self).attach(engine, log_handler, event_name) engine.add_event_handler(closing_event_name, self._close) - def attach_opt_params_handler(self, engine: Engine, event_name: str, *args: Any, **kwargs: Mapping): + def attach_opt_params_handler(self, engine: Engine, event_name: Union[str, EventEnum], *args: Any, **kwargs: Any): """Intentionally empty""" pass - def _create_output_handler(self, *args, **kwargs): + def _create_output_handler(self, *args: Any, **kwargs: Any): return _OutputHandler(*args, **kwargs) - def _create_opt_params_handler(self, *args, **kwargs): + def _create_opt_params_handler(self, *args: Any, **kwargs: Any): """Intentionally empty""" pass + + +class _OutputHandler(BaseOutputHandler): + """Helper handler to log engine's output and/or metrics + + Args: + description (str): progress bar description. + metric_names (list of str, optional): list of metric names to plot or a string "all" to plot all available + metrics. + output_transform (callable, optional): output transform function to prepare `engine.state.output` as a number. + For example, `output_transform = lambda output: output` + This function can also return a dictionary, e.g `{'loss': loss1, 'another_loss': loss2}` to label the plot + with corresponding keys. + closing_event_name: event's name on which the progress bar is closed. Valid events are from + :class:`~ignite.engine.events.Events` or any `event_name` added by + :meth:`~ignite.engine.engine.Engine.register_events`. + + """ + + def __init__( + self, + description: str, + metric_names: Optional[str] = None, + output_transform: Optional[Callable] = None, + closing_event_name: EventEnum = Events.EPOCH_COMPLETED, + ): + if metric_names is None and output_transform is None: + # This helps to avoid 'Either metric_names or output_transform should be defined' of BaseOutputHandler + metric_names = [] + super(_OutputHandler, self).__init__(description, metric_names, output_transform, global_step_transform=None) + self.closing_event_name = closing_event_name + + @staticmethod + def get_max_number_events(event_name: Union[CallableEventWithFilter, Enum], engine: Engine): + if event_name in (Events.ITERATION_STARTED, Events.ITERATION_COMPLETED): + return engine.state.epoch_length + if event_name in (Events.EPOCH_STARTED, Events.EPOCH_COMPLETED): + return engine.state.max_epochs + return 1 + + def __call__(self, engine: Engine, logger: ProgressBar, event_name: Union[CallableEventWithFilter, Enum]): + + pbar_total = self.get_max_number_events(event_name, engine) + if logger.pbar is None: + logger._reset(pbar_total=pbar_total) + + max_epochs = engine.state.max_epochs + default_desc = "Iteration" if max_epochs == 1 else "Epoch" + + desc = self.tag or default_desc + max_num_of_closing_events = self.get_max_number_events(self.closing_event_name, engine) + if max_num_of_closing_events > 1: + global_step = engine.state.get_event_attrib_value(self.closing_event_name) + desc += " [{}/{}]".format(global_step, max_num_of_closing_events) + logger.pbar.set_description(desc) + + metrics = self._setup_output_metrics(engine) + + rendered_metrics = {} + for key, value in metrics.items(): + if isinstance(value, torch.Tensor): + if value.ndimension() == 0: + rendered_metrics[key] = value.item() + elif value.ndimension() == 1: + for i, v in enumerate(value): + k = "{}_{}".format(key, i) + rendered_metrics[k] = v.item() + else: + warnings.warn("ProgressBar can not log " "tensor with {} dimensions".format(value.ndimension())) + else: + rendered_metrics[key] = value + + if rendered_metrics: + logger.pbar.set_postfix(**rendered_metrics) + + global_step = engine.state.get_event_attrib_value(event_name) + if pbar_total is not None: + global_step = (global_step - 1) % pbar_total + 1 + logger.pbar.update(global_step - logger.pbar.n) diff --git a/ignite/contrib/handlers/trains_logger.py b/ignite/contrib/handlers/trains_logger.py index 61f16530d981..6c8c2f885b57 100644 --- a/ignite/contrib/handlers/trains_logger.py +++ b/ignite/contrib/handlers/trains_logger.py @@ -5,9 +5,11 @@ from collections import defaultdict from datetime import datetime from enum import Enum -from typing import Any, List, Mapping, Optional, Type +from typing import Any, Callable, List, Mapping, Optional, Type import torch +from torch.nn import Module +from torch.optim import Optimizer import ignite.distributed as idist from ignite.contrib.handlers.base_logger import ( @@ -17,6 +19,7 @@ BaseWeightsHistHandler, BaseWeightsScalarHandler, ) +from ignite.engine import Engine, EventEnum from ignite.handlers import global_step_from_engine from ignite.handlers.checkpoint import DiskSaver @@ -33,6 +36,161 @@ ] +class TrainsLogger(BaseLogger): + """ + `Trains `_ handler to log metrics, text, model/optimizer parameters, + plots during training and validation. + Also supports model checkpoints logging and upload to the storage solution of your choice (i.e. Trains File server, + S3 bucket etc.) + + .. code-block:: bash + + pip install trains + trains-init + + Args: + project_name (str): The name of the project in which the experiment will be created. If the project + does not exist, it is created. If ``project_name`` is ``None``, the repository name is used. (Optional) + task_name (str): The name of Task (experiment). If ``task_name`` is ``None``, the Python experiment + script's file name is used. (Optional) + task_type (str): Optional. The task type. Valid values are: + - ``TaskTypes.training`` (Default) + - ``TaskTypes.train`` + - ``TaskTypes.testing`` + - ``TaskTypes.inference`` + + Examples: + + .. code-block:: python + + from ignite.contrib.handlers.trains_logger import * + + # Create a logger + + trains_logger = TrainsLogger( + project_name="pytorch-ignite-integration", + task_name="cnn-mnist" + ) + + # Attach the logger to the trainer to log training loss at each iteration + trains_logger.attach_output_handler( + trainer, + event_name=Events.ITERATION_COMPLETED, + tag="training", + output_transform=lambda loss: {"loss": loss} + ) + + # Attach the logger to the evaluator on the training dataset and log NLL, Accuracy metrics after each epoch + # We setup `global_step_transform=global_step_from_engine(trainer)` to take the epoch + # of the `trainer` instead of `train_evaluator`. + trains_logger.attach_output_handler( + train_evaluator, + event_name=Events.EPOCH_COMPLETED, + tag="training", + metric_names=["nll", "accuracy"], + global_step_transform=global_step_from_engine(trainer), + ) + + # Attach the logger to the evaluator on the validation dataset and log NLL, Accuracy metrics after + # each epoch. We setup `global_step_transform=global_step_from_engine(trainer)` to take the epoch of the + # `trainer` instead of `evaluator`. + trains_logger.attach_output_handler( + evaluator, + event_name=Events.EPOCH_COMPLETED, + tag="validation", + metric_names=["nll", "accuracy"], + global_step_transform=global_step_from_engine(trainer)), + ) + + # Attach the logger to the trainer to log optimizer's parameters, e.g. learning rate at each iteration + trains_logger.attach_opt_params_handler( + trainer, + event_name=Events.ITERATION_STARTED, + optimizer=optimizer, + param_name='lr' # optional + ) + + # Attach the logger to the trainer to log model's weights norm after each iteration + trains_logger.attach( + trainer, + event_name=Events.ITERATION_COMPLETED, + log_handler=WeightsScalarHandler(model) + ) + + """ + + def __init__(self, *_, **kwargs: Any): + try: + from trains import Task + from trains.binding.frameworks.tensorflow_bind import WeightsGradientHistHelper + except ImportError: + raise RuntimeError( + "This contrib module requires trains to be installed. " + "You may install trains using: \n pip install trains \n" + ) + + experiment_kwargs = {k: v for k, v in kwargs.items() if k not in ("project_name", "task_name", "task_type")} + + if self.bypass_mode(): + warnings.warn("TrainsSaver: running in bypass mode") + + class _Stub(object): + def __call__(self, *_, **__): + return self + + def __getattr__(self, attr): + if attr in ("name", "id"): + return "" + return self + + def __setattr__(self, attr, val): + pass + + self._task = _Stub() + else: + self._task = Task.init( + project_name=kwargs.get("project_name"), + task_name=kwargs.get("task_name"), + task_type=kwargs.get("task_type", Task.TaskTypes.training), + **experiment_kwargs, + ) + + self.trains_logger = self._task.get_logger() + + self.grad_helper = WeightsGradientHistHelper(logger=self.trains_logger) + + @classmethod + def set_bypass_mode(cls, bypass: bool) -> None: + """ + Will bypass all outside communication, and will drop all logs. + Should only be used in "standalone mode", when there is no access to the *trains-server*. + Args: + bypass: If ``True``, all outside communication is skipped. + """ + setattr(cls, "_bypass", bypass) + + @classmethod + def bypass_mode(cls) -> bool: + """ + Returns the bypass mode state. + Note: + `GITHUB_ACTIONS` env will automatically set bypass_mode to ``True`` + unless overridden specifically with ``TrainsLogger.set_bypass_mode(False)``. + Return: + If True, all outside communication is skipped. + """ + return getattr(cls, "_bypass", bool(os.environ.get("CI"))) + + def close(self): + self.trains_logger.flush() + + def _create_output_handler(self, *args: Any, **kwargs: Any): + return OutputHandler(*args, **kwargs) + + def _create_opt_params_handler(self, *args: Any, **kwargs: Any): + return OptimizerParamsHandler(*args, **kwargs) + + class OutputHandler(BaseOutputHandler): """Helper handler to log engine's output and/or metrics @@ -128,10 +286,16 @@ def global_step_transform(engine, event_name): """ - def __init__(self, tag, metric_names=None, output_transform=None, global_step_transform=None): + def __init__( + self, + tag: str, + metric_names: Optional[List[str]] = None, + output_transform: Optional[Callable] = None, + global_step_transform: Optional[Callable] = None, + ): super(OutputHandler, self).__init__(tag, metric_names, output_transform, global_step_transform) - def __call__(self, engine, logger, event_name): + def __call__(self, engine: Engine, logger: TrainsLogger, event_name: EventEnum): if not isinstance(logger, TrainsLogger): raise RuntimeError("Handler OutputHandler works only with TrainsLogger") @@ -194,10 +358,10 @@ class OptimizerParamsHandler(BaseOptimizerParamsHandler): tag (str, optional): common title for all produced plots. For example, "generator" """ - def __init__(self, optimizer, param_name="lr", tag=None): + def __init__(self, optimizer: Optimizer, param_name: str = "lr", tag: Optional[str] = None): super(OptimizerParamsHandler, self).__init__(optimizer, param_name, tag) - def __call__(self, engine, logger, event_name): + def __call__(self, engine: Engine, logger: TrainsLogger, event_name: EventEnum): if not isinstance(logger, TrainsLogger): raise RuntimeError("Handler OptimizerParamsHandler works only with TrainsLogger") @@ -245,10 +409,10 @@ class WeightsScalarHandler(BaseWeightsScalarHandler): """ - def __init__(self, model, reduction=torch.norm, tag=None): + def __init__(self, model: Module, reduction: Callable = torch.norm, tag: Optional[str] = None): super(WeightsScalarHandler, self).__init__(model, reduction, tag=tag) - def __call__(self, engine, logger, event_name): + def __call__(self, engine: Engine, logger: TrainsLogger, event_name: EventEnum): if not isinstance(logger, TrainsLogger): raise RuntimeError("Handler WeightsScalarHandler works only with TrainsLogger") @@ -297,10 +461,10 @@ class WeightsHistHandler(BaseWeightsHistHandler): """ - def __init__(self, model, tag=None): + def __init__(self, model: Module, tag: Optional[str] = None): super(WeightsHistHandler, self).__init__(model, tag=tag) - def __call__(self, engine, logger, event_name): + def __call__(self, engine: Engine, logger: TrainsLogger, event_name: EventEnum): if not isinstance(logger, TrainsLogger): raise RuntimeError("Handler 'WeightsHistHandler' works only with TrainsLogger") @@ -352,10 +516,10 @@ class GradsScalarHandler(BaseWeightsScalarHandler): """ - def __init__(self, model, reduction=torch.norm, tag=None): + def __init__(self, model: Module, reduction: Callable = torch.norm, tag: Optional[str] = None): super(GradsScalarHandler, self).__init__(model, reduction, tag=tag) - def __call__(self, engine, logger, event_name): + def __call__(self, engine: Engine, logger: TrainsLogger, event_name: EventEnum): if not isinstance(logger, TrainsLogger): raise RuntimeError("Handler GradsScalarHandler works only with TrainsLogger") @@ -403,10 +567,10 @@ class GradsHistHandler(BaseWeightsHistHandler): """ - def __init__(self, model, tag=None): + def __init__(self, model: Module, tag: Optional[str] = None): super(GradsHistHandler, self).__init__(model, tag=tag) - def __call__(self, engine, logger, event_name): + def __call__(self, engine: Engine, logger: TrainsLogger, event_name: EventEnum): if not isinstance(logger, TrainsLogger): raise RuntimeError("Handler 'GradsHistHandler' works only with TrainsLogger") @@ -426,161 +590,6 @@ def __call__(self, engine, logger, event_name): ) -class TrainsLogger(BaseLogger): - """ - `Trains `_ handler to log metrics, text, model/optimizer parameters, - plots during training and validation. - Also supports model checkpoints logging and upload to the storage solution of your choice (i.e. Trains File server, - S3 bucket etc.) - - .. code-block:: bash - - pip install trains - trains-init - - Args: - project_name (str): The name of the project in which the experiment will be created. If the project - does not exist, it is created. If ``project_name`` is ``None``, the repository name is used. (Optional) - task_name (str): The name of Task (experiment). If ``task_name`` is ``None``, the Python experiment - script's file name is used. (Optional) - task_type (str): Optional. The task type. Valid values are: - - ``TaskTypes.training`` (Default) - - ``TaskTypes.train`` - - ``TaskTypes.testing`` - - ``TaskTypes.inference`` - - Examples: - - .. code-block:: python - - from ignite.contrib.handlers.trains_logger import * - - # Create a logger - - trains_logger = TrainsLogger( - project_name="pytorch-ignite-integration", - task_name="cnn-mnist" - ) - - # Attach the logger to the trainer to log training loss at each iteration - trains_logger.attach_output_handler( - trainer, - event_name=Events.ITERATION_COMPLETED, - tag="training", - output_transform=lambda loss: {"loss": loss} - ) - - # Attach the logger to the evaluator on the training dataset and log NLL, Accuracy metrics after each epoch - # We setup `global_step_transform=global_step_from_engine(trainer)` to take the epoch - # of the `trainer` instead of `train_evaluator`. - trains_logger.attach_output_handler( - train_evaluator, - event_name=Events.EPOCH_COMPLETED, - tag="training", - metric_names=["nll", "accuracy"], - global_step_transform=global_step_from_engine(trainer), - ) - - # Attach the logger to the evaluator on the validation dataset and log NLL, Accuracy metrics after - # each epoch. We setup `global_step_transform=global_step_from_engine(trainer)` to take the epoch of the - # `trainer` instead of `evaluator`. - trains_logger.attach_output_handler( - evaluator, - event_name=Events.EPOCH_COMPLETED, - tag="validation", - metric_names=["nll", "accuracy"], - global_step_transform=global_step_from_engine(trainer)), - ) - - # Attach the logger to the trainer to log optimizer's parameters, e.g. learning rate at each iteration - trains_logger.attach_opt_params_handler( - trainer, - event_name=Events.ITERATION_STARTED, - optimizer=optimizer, - param_name='lr' # optional - ) - - # Attach the logger to the trainer to log model's weights norm after each iteration - trains_logger.attach( - trainer, - event_name=Events.ITERATION_COMPLETED, - log_handler=WeightsScalarHandler(model) - ) - - """ - - def __init__(self, *_, **kwargs): - try: - from trains import Task - from trains.binding.frameworks.tensorflow_bind import WeightsGradientHistHelper - except ImportError: - raise RuntimeError( - "This contrib module requires trains to be installed. " - "You may install trains using: \n pip install trains \n" - ) - - experiment_kwargs = {k: v for k, v in kwargs.items() if k not in ("project_name", "task_name", "task_type",)} - - if self.bypass_mode(): - warnings.warn("TrainsSaver: running in bypass mode") - - class _Stub(object): - def __call__(self, *_, **__): - return self - - def __getattr__(self, attr): - if attr in ("name", "id"): - return "" - return self - - def __setattr__(self, attr, val): - pass - - self._task = _Stub() - else: - self._task = Task.init( - project_name=kwargs.get("project_name"), - task_name=kwargs.get("task_name"), - task_type=kwargs.get("task_type", Task.TaskTypes.training), - **experiment_kwargs, - ) - - self.trains_logger = self._task.get_logger() - - self.grad_helper = WeightsGradientHistHelper(logger=self.trains_logger,) - - @classmethod - def set_bypass_mode(cls, bypass: bool) -> None: - """ - Will bypass all outside communication, and will drop all logs. - Should only be used in "standalone mode", when there is no access to the *trains-server*. - Args: - bypass: If ``True``, all outside communication is skipped. - """ - setattr(cls, "_bypass", bypass) - - @classmethod - def bypass_mode(cls) -> bool: - """ - Returns the bypass mode state. - Note: - `GITHUB_ACTIONS` env will automatically set bypass_mode to ``True`` - unless overridden specifically with ``TrainsLogger.set_bypass_mode(False)``. - Return: - If True, all outside communication is skipped. - """ - return getattr(cls, "_bypass", bool(os.environ.get("CI"))) - - def close(self): - self.trains_logger.flush() - - def _create_output_handler(self, *args, **kwargs): - return OutputHandler(*args, **kwargs) - - def _create_opt_params_handler(self, *args, **kwargs): - return OptimizerParamsHandler(*args, **kwargs) - - class TrainsSaver(DiskSaver): """ Handler that saves input checkpoint as Trains artifacts @@ -622,7 +631,9 @@ class TrainsSaver(DiskSaver): """ - def __init__(self, logger: TrainsLogger = None, output_uri: str = None, dirname: str = None, *args, **kwargs): + def __init__( + self, logger: TrainsLogger = None, output_uri: str = None, dirname: str = None, *args: Any, **kwargs: Any + ): self._setup_check_trains(logger, output_uri) @@ -647,7 +658,7 @@ def __init__(self, logger: TrainsLogger = None, output_uri: str = None, dirname: super(TrainsSaver, self).__init__(dirname=dirname, *args, **kwargs) @idist.one_rank_only() - def _setup_check_trains(self, logger, output_uri): + def _setup_check_trains(self, logger: TrainsLogger, output_uri: str): try: from trains import Task except ImportError: @@ -708,8 +719,7 @@ def post_callback(self, action: str, model_info: Any): model_info.model.name = "{}: {}".format(model_info.task.name, self._filename) prefix = "Checkpoint Metadata: " metadata = "{}{}".format( - prefix, - ", ".join("{}={}".format(k, v) for k, v in self._metadata.items()) if self._metadata else "none", + prefix, ", ".join("{}={}".format(k, v) for k, v in self._metadata.items()) if self._metadata else "none" ) comment = "\n".join( metadata if line.startswith(prefix) else line for line in (model_info.model.comment or "").split("\n") diff --git a/ignite/contrib/handlers/visdom_logger.py b/ignite/contrib/handlers/visdom_logger.py index 01e6d98c44fc..010ab3b05493 100644 --- a/ignite/contrib/handlers/visdom_logger.py +++ b/ignite/contrib/handlers/visdom_logger.py @@ -1,8 +1,11 @@ import numbers import os import warnings +from typing import Any, Callable, Optional, Union import torch +import torch.nn as nn +from torch.optim import Optimizer from ignite.contrib.handlers.base_logger import ( BaseLogger, @@ -10,6 +13,7 @@ BaseOutputHandler, BaseWeightsScalarHandler, ) +from ignite.engine import Engine from ignite.handlers import global_step_from_engine __all__ = [ @@ -22,12 +26,196 @@ ] +class VisdomLogger(BaseLogger): + """ + VisdomLogger handler to log metrics, model/optimizer parameters, gradients during the training and validation. + + This class requires `visdom `_ package to be installed: + + .. code-block:: bash + + + pip install git+https://github.com/facebookresearch/visdom.git + + Args: + server (str, optional): visdom server URL. It can be also specified by environment variable `VISDOM_SERVER_URL` + port (int, optional): visdom server's port. It can be also specified by environment variable `VISDOM_PORT` + num_workers (int, optional): number of workers to use in `concurrent.futures.ThreadPoolExecutor` to post data to + visdom server. Default, `num_workers=1`. If `num_workers=0` and logger uses the main thread. If using + Python 2.7 and `num_workers>0` the package `futures` should be installed: `pip install futures` + **kwargs: kwargs to pass into + `visdom.Visdom `_. + + Note: + We can also specify username/password using environment variables: VISDOM_USERNAME, VISDOM_PASSWORD + + + .. warning:: + + Frequent logging, e.g. when logger is attached to `Events.ITERATION_COMPLETED`, can slow down the run if the + main thread is used to send the data to visdom server (`num_workers=0`). To avoid this situation we can either + log less frequently or set `num_workers=1`. + + Examples: + + .. code-block:: python + + from ignite.contrib.handlers.visdom_logger import * + + # Create a logger + vd_logger = VisdomLogger() + + # Attach the logger to the trainer to log training loss at each iteration + vd_logger.attach_output_handler( + trainer, + event_name=Events.ITERATION_COMPLETED, + tag="training", + output_transform=lambda loss: {"loss": loss} + ) + + # Attach the logger to the evaluator on the training dataset and log NLL, Accuracy metrics after each epoch + # We setup `global_step_transform=global_step_from_engine(trainer)` to take the epoch + # of the `trainer` instead of `train_evaluator`. + vd_logger.attach_output_handler( + train_evaluator, + event_name=Events.EPOCH_COMPLETED, + tag="training", + metric_names=["nll", "accuracy"], + global_step_transform=global_step_from_engine(trainer), + ) + + # Attach the logger to the evaluator on the validation dataset and log NLL, Accuracy metrics after + # each epoch. We setup `global_step_transform=global_step_from_engine(trainer)` to take the epoch of the + # `trainer` instead of `evaluator`. + vd_logger.attach_output_handler( + evaluator, + event_name=Events.EPOCH_COMPLETED, + tag="validation", + metric_names=["nll", "accuracy"], + global_step_transform=global_step_from_engine(trainer)), + ) + + # Attach the logger to the trainer to log optimizer's parameters, e.g. learning rate at each iteration + vd_logger.attach_opt_params_handler( + trainer, + event_name=Events.ITERATION_STARTED, + optimizer=optimizer, + param_name='lr' # optional + ) + + # Attach the logger to the trainer to log model's weights norm after each iteration + vd_logger.attach( + trainer, + event_name=Events.ITERATION_COMPLETED, + log_handler=WeightsScalarHandler(model) + ) + + # Attach the logger to the trainer to log model's gradients norm after each iteration + vd_logger.attach( + trainer, + event_name=Events.ITERATION_COMPLETED, + log_handler=GradsScalarHandler(model) + ) + + # We need to close the logger with we are done + vd_logger.close() + + It is also possible to use the logger as context manager: + + .. code-block:: python + + from ignite.contrib.handlers.visdom_logger import * + + with VisdomLogger() as vd_logger: + + trainer = Engine(update_fn) + # Attach the logger to the trainer to log training loss at each iteration + vd_logger.attach_output_handler( + trainer, + event_name=Events.ITERATION_COMPLETED, + tag="training", + output_transform=lambda loss: {"loss": loss} + ) + + """ + + def __init__( + self, + server: Optional[str] = None, + port: Optional[int] = None, + num_workers: int = 1, + raise_exceptions: bool = True, + **kwargs: Any + ): + try: + import visdom + except ImportError: + raise RuntimeError( + "This contrib module requires visdom package. " + "Please install it with command:\n" + "pip install git+https://github.com/facebookresearch/visdom.git" + ) + + if num_workers > 0: + # If visdom is installed, one of its dependencies `tornado` + # requires also `futures` to be installed. + # Let's check anyway if we can import it. + try: + import concurrent.futures + except ImportError: + raise RuntimeError( + "This contrib module requires concurrent.futures module" + "Please install it with command:\n" + "pip install futures" + ) + + if server is None: + server = os.environ.get("VISDOM_SERVER_URL", "localhost") + + if port is None: + port = int(os.environ.get("VISDOM_PORT", 8097)) + + if "username" not in kwargs: + username = os.environ.get("VISDOM_USERNAME", None) + kwargs["username"] = username + + if "password" not in kwargs: + password = os.environ.get("VISDOM_PASSWORD", None) + kwargs["password"] = password + + self.vis = visdom.Visdom(server=server, port=port, raise_exceptions=raise_exceptions, **kwargs) + + if not self.vis.offline and not self.vis.check_connection(): + raise RuntimeError( + "Failed to connect to Visdom server at {}. Did you run python -m visdom.server ?".format(server) + ) + + self.executor = _DummyExecutor() + if num_workers > 0: + from concurrent.futures import ThreadPoolExecutor + + self.executor = ThreadPoolExecutor(max_workers=num_workers) + + def _save(self): + self.vis.save([self.vis.env]) + + def close(self): + self.executor.shutdown() + self.vis = None + + def _create_output_handler(self, *args: Any, **kwargs: Any): + return OutputHandler(*args, **kwargs) + + def _create_opt_params_handler(self, *args: Any, **kwargs: Any): + return OptimizerParamsHandler(*args, **kwargs) + + class _BaseVisDrawer: - def __init__(self, show_legend=False): + def __init__(self, show_legend: bool = False): self.windows = {} self.show_legend = show_legend - def add_scalar(self, logger, k, v, event_name, global_step): + def add_scalar(self, logger: VisdomLogger, k: str, v: Union[str, float], event_name: Any, global_step: int): """ Helper method to log a scalar with VisdomLogger. @@ -50,8 +238,8 @@ def add_scalar(self, logger, k, v, event_name, global_step): update = None if self.windows[k]["win"] is None else "append" kwargs = { - "X": [global_step,], - "Y": [v,], + "X": [global_step], + "Y": [v], "env": logger.vis.env, "win": self.windows[k]["win"], "update": update, @@ -152,12 +340,17 @@ def global_step_transform(engine, event_name): """ def __init__( - self, tag, metric_names=None, output_transform=None, global_step_transform=None, show_legend=False, + self, + tag: str, + metric_names: Optional[str] = None, + output_transform: Optional[Callable] = None, + global_step_transform: Optional[Callable] = None, + show_legend: bool = False, ): super(OutputHandler, self).__init__(tag, metric_names, output_transform, global_step_transform) _BaseVisDrawer.__init__(self, show_legend=show_legend) - def __call__(self, engine, logger, event_name): + def __call__(self, engine: Engine, logger: VisdomLogger, event_name: Any): if not isinstance(logger, VisdomLogger): raise RuntimeError("Handler 'OutputHandler' works only with VisdomLogger") @@ -225,11 +418,13 @@ class OptimizerParamsHandler(BaseOptimizerParamsHandler, _BaseVisDrawer): show_legend (bool, optional): flag to show legend in the window """ - def __init__(self, optimizer, param_name="lr", tag=None, show_legend=False): + def __init__( + self, optimizer: Optimizer, param_name: str = "lr", tag: Optional[str] = None, show_legend: bool = False, + ): super(OptimizerParamsHandler, self).__init__(optimizer, param_name, tag) _BaseVisDrawer.__init__(self, show_legend=show_legend) - def __call__(self, engine, logger, event_name): + def __call__(self, engine: Engine, logger: VisdomLogger, event_name: Any): if not isinstance(logger, VisdomLogger): raise RuntimeError("Handler OptimizerParamsHandler works only with VisdomLogger") @@ -274,11 +469,13 @@ class WeightsScalarHandler(BaseWeightsScalarHandler, _BaseVisDrawer): show_legend (bool, optional): flag to show legend in the window """ - def __init__(self, model, reduction=torch.norm, tag=None, show_legend=False): + def __init__( + self, model: nn.Module, reduction: Callable = torch.norm, tag: Optional[str] = None, show_legend: bool = False, + ): super(WeightsScalarHandler, self).__init__(model, reduction, tag=tag) _BaseVisDrawer.__init__(self, show_legend=show_legend) - def __call__(self, engine, logger, event_name): + def __call__(self, engine: Engine, logger: VisdomLogger, event_name: Any): if not isinstance(logger, VisdomLogger): raise RuntimeError("Handler 'WeightsScalarHandler' works only with VisdomLogger") @@ -323,11 +520,13 @@ class GradsScalarHandler(BaseWeightsScalarHandler, _BaseVisDrawer): """ - def __init__(self, model, reduction=torch.norm, tag=None, show_legend=False): + def __init__( + self, model: nn.Module, reduction: Callable = torch.norm, tag: Optional[str] = None, show_legend: bool = False, + ): super(GradsScalarHandler, self).__init__(model, reduction, tag) _BaseVisDrawer.__init__(self, show_legend=show_legend) - def __call__(self, engine, logger, event_name): + def __call__(self, engine: Engine, logger: VisdomLogger, event_name: Any): if not isinstance(logger, VisdomLogger): raise RuntimeError("Handler 'GradsScalarHandler' works only with VisdomLogger") @@ -342,195 +541,19 @@ def __call__(self, engine, logger, event_name): logger._save() -class VisdomLogger(BaseLogger): - """ - VisdomLogger handler to log metrics, model/optimizer parameters, gradients during the training and validation. - - This class requires `visdom `_ package to be installed: - - .. code-block:: bash - - pip install git+https://github.com/facebookresearch/visdom.git - - Args: - server (str, optional): visdom server URL. It can be also specified by environment variable `VISDOM_SERVER_URL` - port (int, optional): visdom server's port. It can be also specified by environment variable `VISDOM_PORT` - num_workers (int, optional): number of workers to use in `concurrent.futures.ThreadPoolExecutor` to post data to - visdom server. Default, `num_workers=1`. If `num_workers=0` and logger uses the main thread. If using - Python 2.7 and `num_workers>0` the package `futures` should be installed: `pip install futures` - **kwargs: kwargs to pass into - `visdom.Visdom `_. - - Note: - We can also specify username/password using environment variables: VISDOM_USERNAME, VISDOM_PASSWORD - - - .. warning:: - - Frequent logging, e.g. when logger is attached to `Events.ITERATION_COMPLETED`, can slow down the run if the - main thread is used to send the data to visdom server (`num_workers=0`). To avoid this situation we can either - log less frequently or set `num_workers=1`. - - Examples: - - .. code-block:: python - - from ignite.contrib.handlers.visdom_logger import * - - # Create a logger - vd_logger = VisdomLogger() - - # Attach the logger to the trainer to log training loss at each iteration - vd_logger.attach_output_handler( - trainer, - event_name=Events.ITERATION_COMPLETED, - tag="training", - output_transform=lambda loss: {"loss": loss} - ) - - # Attach the logger to the evaluator on the training dataset and log NLL, Accuracy metrics after each epoch - # We setup `global_step_transform=global_step_from_engine(trainer)` to take the epoch - # of the `trainer` instead of `train_evaluator`. - vd_logger.attach_output_handler( - train_evaluator, - event_name=Events.EPOCH_COMPLETED, - tag="training", - metric_names=["nll", "accuracy"], - global_step_transform=global_step_from_engine(trainer), - ) - - # Attach the logger to the evaluator on the validation dataset and log NLL, Accuracy metrics after - # each epoch. We setup `global_step_transform=global_step_from_engine(trainer)` to take the epoch of the - # `trainer` instead of `evaluator`. - vd_logger.attach_output_handler( - evaluator, - event_name=Events.EPOCH_COMPLETED, - tag="validation", - metric_names=["nll", "accuracy"], - global_step_transform=global_step_from_engine(trainer)), - ) - - # Attach the logger to the trainer to log optimizer's parameters, e.g. learning rate at each iteration - vd_logger.attach_opt_params_handler( - trainer, - event_name=Events.ITERATION_STARTED, - optimizer=optimizer, - param_name='lr' # optional - ) - - # Attach the logger to the trainer to log model's weights norm after each iteration - vd_logger.attach( - trainer, - event_name=Events.ITERATION_COMPLETED, - log_handler=WeightsScalarHandler(model) - ) - - # Attach the logger to the trainer to log model's gradients norm after each iteration - vd_logger.attach( - trainer, - event_name=Events.ITERATION_COMPLETED, - log_handler=GradsScalarHandler(model) - ) - - # We need to close the logger with we are done - vd_logger.close() - - It is also possible to use the logger as context manager: - - .. code-block:: python - - from ignite.contrib.handlers.visdom_logger import * - - with VisdomLogger() as vd_logger: - - trainer = Engine(update_fn) - # Attach the logger to the trainer to log training loss at each iteration - vd_logger.attach_output_handler( - trainer, - event_name=Events.ITERATION_COMPLETED, - tag="training", - output_transform=lambda loss: {"loss": loss} - ) - - """ - - def __init__(self, server=None, port=None, num_workers=1, raise_exceptions=True, **kwargs): - try: - import visdom - except ImportError: - raise RuntimeError( - "This contrib module requires visdom package. " - "Please install it with command:\n" - "pip install git+https://github.com/facebookresearch/visdom.git" - ) - - if num_workers > 0: - # If visdom is installed, one of its dependencies `tornado` - # requires also `futures` to be installed. - # Let's check anyway if we can import it. - try: - import concurrent.futures - except ImportError: - raise RuntimeError( - "This contrib module requires concurrent.futures module" - "Please install it with command:\n" - "pip install futures" - ) - - if server is None: - server = os.environ.get("VISDOM_SERVER_URL", "localhost") - - if port is None: - port = int(os.environ.get("VISDOM_PORT", 8097)) - - if "username" not in kwargs: - username = os.environ.get("VISDOM_USERNAME", None) - kwargs["username"] = username - - if "password" not in kwargs: - password = os.environ.get("VISDOM_PASSWORD", None) - kwargs["password"] = password - - self.vis = visdom.Visdom(server=server, port=port, raise_exceptions=raise_exceptions, **kwargs) - - if not self.vis.offline and not self.vis.check_connection(): - raise RuntimeError( - "Failed to connect to Visdom server at {}. Did you run python -m visdom.server ?".format(server) - ) - - self.executor = _DummyExecutor() - if num_workers > 0: - from concurrent.futures import ThreadPoolExecutor - - self.executor = ThreadPoolExecutor(max_workers=num_workers) - - def _save(self): - self.vis.save([self.vis.env]) - - def close(self): - self.executor.shutdown() - self.vis = None - - def _create_output_handler(self, *args, **kwargs): - return OutputHandler(*args, **kwargs) - - def _create_opt_params_handler(self, *args, **kwargs): - return OptimizerParamsHandler(*args, **kwargs) - - class _DummyExecutor: class _DummyFuture: - def __init__(self, result): + def __init__(self, result: Any): self._output = result def result(self): return self._output - def __init__(self, *args, **kwargs): + def __init__(self, *args: Any, **kwargs: Any): pass - def submit(self, fn, **kwargs): + def submit(self, fn: Callable, **kwargs: Any): return _DummyExecutor._DummyFuture(fn(**kwargs)) - def shutdown(self, *args, **kwargs): + def shutdown(self, *args: Any, **kwargs: Any): pass diff --git a/ignite/contrib/handlers/wandb_logger.py b/ignite/contrib/handlers/wandb_logger.py index dd4945780b22..de1ac2e1a4a0 100644 --- a/ignite/contrib/handlers/wandb_logger.py +++ b/ignite/contrib/handlers/wandb_logger.py @@ -1,9 +1,145 @@ +from enum import Enum +from typing import Any, Callable, List, Optional, Union + +from torch.optim import Optimizer + from ignite.contrib.handlers.base_logger import BaseLogger, BaseOptimizerParamsHandler, BaseOutputHandler +from ignite.engine import CallableEventWithFilter, Engine from ignite.handlers import global_step_from_engine __all__ = ["WandBLogger", "OutputHandler", "OptimizerParamsHandler", "global_step_from_engine"] +class WandBLogger(BaseLogger): + """`Weights & Biases `_ handler to log metrics, model/optimizer parameters, gradients + during training and validation. It can also be used to log model checkpoints to the Weights & Biases cloud. + + .. code-block:: bash + + pip install wandb + + This class is also a wrapper for the wandb module. This means that you can call any wandb function using + this wrapper. See examples on how to save model parameters and gradients. + + Args: + *args: Positional arguments accepted by `wandb.init`. + **kwargs: Keyword arguments accepted by `wandb.init`. + Please see `wandb.init `_ for documentation of possible parameters. + + Examples: + + .. code-block:: python + + from ignite.contrib.handlers.wandb_logger import * + + # Create a logger. All parameters are optional. See documentation + # on wandb.init for details. + + wandb_logger = WandBLogger( + entity="shared", + project="pytorch-ignite-integration", + name="cnn-mnist", + config={"max_epochs": 10}, + tags=["pytorch-ignite", "minst"] + ) + + # Attach the logger to the trainer to log training loss at each iteration + wandb_logger.attach_output_handler( + trainer, + event_name=Events.ITERATION_COMPLETED, + tag="training", + output_transform=lambda loss: {"loss": loss} + ) + + # Attach the logger to the evaluator on the training dataset and log NLL, Accuracy metrics after each epoch + # We setup `global_step_transform=lambda *_: trainer.state.iteration` to take iteration value + # of the `trainer`: + wandb_logger.attach_output_handler( + train_evaluator, + event_name=Events.EPOCH_COMPLETED, + tag="training", + metric_names=["nll", "accuracy"], + global_step_transform=lambda *_: trainer.state.iteration, + ) + + # Attach the logger to the evaluator on the validation dataset and log NLL, Accuracy metrics after + # each epoch. We setup `global_step_transform=lambda *_: trainer.state.iteration` to take iteration value + # of the `trainer` instead of `evaluator`. + wandb_logger.attach_output_handler( + evaluator, + event_name=Events.EPOCH_COMPLETED, + tag="validation", + metric_names=["nll", "accuracy"], + global_step_transform=lambda *_: trainer.state.iteration, + ) + + # Attach the logger to the trainer to log optimizer's parameters, e.g. learning rate at each iteration + wandb_logger.attach_opt_params_handler( + trainer, + event_name=Events.ITERATION_STARTED, + optimizer=optimizer, + param_name='lr' # optional + ) + + If you want to log model gradients, the model call graph, etc., use the logger as wrapper of wandb. Refer + to the documentation of wandb.watch for details: + + .. code-block:: python + + wandb_logger = WandBLogger( + entity="shared", + project="pytorch-ignite-integration", + name="cnn-mnist", + config={"max_epochs": 10}, + tags=["pytorch-ignite", "minst"] + ) + + model = torch.nn.Sequential(...) + wandb_logger.watch(model) + + For model checkpointing, Weights & Biases creates a local run dir, and automatically synchronizes all + files saved there at the end of the run. You can just use the `wandb_logger.run.dir` as path for the + `ModelCheckpoint`: + + .. code-block:: python + + from ignite.handlers import ModelCheckpoint + + def score_function(engine): + return engine.state.metrics['accuracy'] + + model_checkpoint = ModelCheckpoint( + wandb_logger.run.dir, n_saved=2, filename_prefix='best', + require_empty=False, score_function=score_function, + score_name="validation_accuracy", + global_step_transform=global_step_from_engine(trainer) + ) + evaluator.add_event_handler(Events.COMPLETED, model_checkpoint, {'model': model}) + """ + + def __init__(self, *args: Any, **kwargs: Any): + try: + import wandb + + self._wandb = wandb + except ImportError: + raise RuntimeError( + "This contrib module requires wandb to be installed. " + "You man install wandb with the command:\n pip install wandb\n" + ) + if kwargs.get("init", True): + wandb.init(*args, **kwargs) + + def __getattr__(self, attr: Any): + return getattr(self._wandb, attr) + + def _create_output_handler(self, *args: Any, **kwargs: Any): + return OutputHandler(*args, **kwargs) + + def _create_opt_params_handler(self, *args: Any, **kwargs: Any): + return OptimizerParamsHandler(*args, **kwargs) + + class OutputHandler(BaseOutputHandler): """Helper handler to log engine's output and/or metrics @@ -109,11 +245,18 @@ def global_step_transform(engine, event_name): """ - def __init__(self, tag, metric_names=None, output_transform=None, global_step_transform=None, sync=None): + def __init__( + self, + tag: str, + metric_names: Optional[List[str]] = None, + output_transform: Optional[Callable] = None, + global_step_transform: Optional[Callable] = None, + sync: Optional[bool] = None, + ): super().__init__(tag, metric_names, output_transform, global_step_transform) self.sync = sync - def __call__(self, engine, logger, event_name): + def __call__(self, engine: Engine, logger: WandBLogger, event_name: Union[CallableEventWithFilter, Enum]): if not isinstance(logger, WandBLogger): raise RuntimeError("Handler '{}' works only with WandBLogger.".format(self.__class__.__name__)) @@ -174,11 +317,13 @@ class OptimizerParamsHandler(BaseOptimizerParamsHandler): the default value of wandb.log. """ - def __init__(self, optimizer, param_name="lr", tag=None, sync=None): + def __init__( + self, optimizer: Optimizer, param_name: str = "lr", tag: Optional[str] = None, sync: Optional[bool] = None, + ): super(OptimizerParamsHandler, self).__init__(optimizer, param_name, tag) self.sync = sync - def __call__(self, engine, logger, event_name): + def __call__(self, engine: Engine, logger: WandBLogger, event_name: Union[CallableEventWithFilter, Enum]): if not isinstance(logger, WandBLogger): raise RuntimeError("Handler OptimizerParamsHandler works only with WandBLogger") @@ -189,133 +334,3 @@ def __call__(self, engine, logger, event_name): for i, param_group in enumerate(self.optimizer.param_groups) } logger.log(params, step=global_step, sync=self.sync) - - -class WandBLogger(BaseLogger): - """`Weights & Biases `_ handler to log metrics, model/optimizer parameters, gradients - during training and validation. It can also be used to log model checkpoints to the Weights & Biases cloud. - - .. code-block:: bash - - pip install wandb - - This class is also a wrapper for the wandb module. This means that you can call any wandb function using - this wrapper. See examples on how to save model parameters and gradients. - - Args: - *args: Positional arguments accepted by `wandb.init`. - **kwargs: Keyword arguments accepted by `wandb.init`. - Please see `wandb.init `_ for documentation of possible parameters. - - Examples: - - .. code-block:: python - - from ignite.contrib.handlers.wandb_logger import * - - # Create a logger. All parameters are optional. See documentation - # on wandb.init for details. - - wandb_logger = WandBLogger( - entity="shared", - project="pytorch-ignite-integration", - name="cnn-mnist", - config={"max_epochs": 10}, - tags=["pytorch-ignite", "minst"] - ) - - # Attach the logger to the trainer to log training loss at each iteration - wandb_logger.attach_output_handler( - trainer, - event_name=Events.ITERATION_COMPLETED, - tag="training", - output_transform=lambda loss: {"loss": loss} - ) - - # Attach the logger to the evaluator on the training dataset and log NLL, Accuracy metrics after each epoch - # We setup `global_step_transform=lambda *_: trainer.state.iteration` to take iteration value - # of the `trainer`: - wandb_logger.attach_output_handler( - train_evaluator, - event_name=Events.EPOCH_COMPLETED, - tag="training", - metric_names=["nll", "accuracy"], - global_step_transform=lambda *_: trainer.state.iteration, - ) - - # Attach the logger to the evaluator on the validation dataset and log NLL, Accuracy metrics after - # each epoch. We setup `global_step_transform=lambda *_: trainer.state.iteration` to take iteration value - # of the `trainer` instead of `evaluator`. - wandb_logger.attach_output_handler( - evaluator, - event_name=Events.EPOCH_COMPLETED, - tag="validation", - metric_names=["nll", "accuracy"], - global_step_transform=lambda *_: trainer.state.iteration, - ) - - # Attach the logger to the trainer to log optimizer's parameters, e.g. learning rate at each iteration - wandb_logger.attach_opt_params_handler( - trainer, - event_name=Events.ITERATION_STARTED, - optimizer=optimizer, - param_name='lr' # optional - ) - - If you want to log model gradients, the model call graph, etc., use the logger as wrapper of wandb. Refer - to the documentation of wandb.watch for details: - - .. code-block:: python - - wandb_logger = WandBLogger( - entity="shared", - project="pytorch-ignite-integration", - name="cnn-mnist", - config={"max_epochs": 10}, - tags=["pytorch-ignite", "minst"] - ) - - model = torch.nn.Sequential(...) - wandb_logger.watch(model) - - For model checkpointing, Weights & Biases creates a local run dir, and automatically synchronizes all - files saved there at the end of the run. You can just use the `wandb_logger.run.dir` as path for the - `ModelCheckpoint`: - - .. code-block:: python - - from ignite.handlers import ModelCheckpoint - - def score_function(engine): - return engine.state.metrics['accuracy'] - - model_checkpoint = ModelCheckpoint( - wandb_logger.run.dir, n_saved=2, filename_prefix='best', - require_empty=False, score_function=score_function, - score_name="validation_accuracy", - global_step_transform=global_step_from_engine(trainer) - ) - evaluator.add_event_handler(Events.COMPLETED, model_checkpoint, {'model': model}) - """ - - def __init__(self, *args, **kwargs): - try: - import wandb - - self._wandb = wandb - except ImportError: - raise RuntimeError( - "This contrib module requires wandb to be installed. " - "You man install wandb with the command:\n pip install wandb\n" - ) - if kwargs.get("init", True): - wandb.init(*args, **kwargs) - - def __getattr__(self, attr): - return getattr(self._wandb, attr) - - def _create_output_handler(self, *args, **kwargs): - return OutputHandler(*args, **kwargs) - - def _create_opt_params_handler(self, *args, **kwargs): - return OptimizerParamsHandler(*args, **kwargs)