diff --git a/setup.py b/setup.py index 85b9192f4f..9862deb896 100644 --- a/setup.py +++ b/setup.py @@ -131,7 +131,6 @@ def get_version() -> str: extras_require=extras, entry_points={ "console_scripts": [ - "huggingface-cli=huggingface_hub.commands.huggingface_cli:main", "hf=huggingface_hub.cli.hf:main", "tiny-agents=huggingface_hub.inference._mcp.cli:app", ], diff --git a/src/huggingface_hub/_login.py b/src/huggingface_hub/_login.py index 847af281a3..4a358293c2 100644 --- a/src/huggingface_hub/_login.py +++ b/src/huggingface_hub/_login.py @@ -20,7 +20,7 @@ from typing import Optional from . import constants -from .commands._cli_utils import ANSI +from .cli._cli_utils import ANSI from .utils import ( capture_output, get_token, @@ -244,7 +244,7 @@ def interpreter_login(*, skip_if_logged_in: bool = False) -> None: logger.info("User is already logged in.") return - from .commands.delete_cache import _ask_for_confirmation_no_tui + from .cli.cache import _ask_for_confirmation_no_tui print(_HF_LOGO_ASCII) if get_token() is not None: diff --git a/src/huggingface_hub/cli/_cli_utils.py b/src/huggingface_hub/cli/_cli_utils.py index 2cd08d3416..9181dd5569 100644 --- a/src/huggingface_hub/cli/_cli_utils.py +++ b/src/huggingface_hub/cli/_cli_utils.py @@ -15,13 +15,23 @@ import os from enum import Enum -from typing import Annotated, Optional, Union +from typing import TYPE_CHECKING, Annotated, Optional, Union import click import typer from huggingface_hub import __version__ -from huggingface_hub.hf_api import HfApi + + +if TYPE_CHECKING: + from huggingface_hub.hf_api import HfApi + + +def get_hf_api(token: Optional[str] = None) -> "HfApi": + # Import here to avoid circular import + from huggingface_hub.hf_api import HfApi + + return HfApi(token=token, library_name="hf", library_version=__version__) class ANSI: @@ -140,7 +150,3 @@ class RepoType(str, Enum): help="Git revision id which can be a branch name, a tag, or a commit hash.", ), ] - - -def get_hf_api(token: Optional[str] = None) -> HfApi: - return HfApi(token=token, library_name="hf", library_version=__version__) diff --git a/src/huggingface_hub/commands/__init__.py b/src/huggingface_hub/commands/__init__.py deleted file mode 100644 index 49d0882145..0000000000 --- a/src/huggingface_hub/commands/__init__.py +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright 2020 The HuggingFace Team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from abc import ABC, abstractmethod -from argparse import _SubParsersAction - - -class BaseHuggingfaceCLICommand(ABC): - @staticmethod - @abstractmethod - def register_subcommand(parser: _SubParsersAction): - raise NotImplementedError() - - @abstractmethod - def run(self): - raise NotImplementedError() diff --git a/src/huggingface_hub/commands/_cli_utils.py b/src/huggingface_hub/commands/_cli_utils.py deleted file mode 100644 index 3f4819c26c..0000000000 --- a/src/huggingface_hub/commands/_cli_utils.py +++ /dev/null @@ -1,74 +0,0 @@ -# Copyright 2022 The HuggingFace Team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Contains a utility for good-looking prints.""" - -import os -from typing import Union - - -class ANSI: - """ - Helper for en.wikipedia.org/wiki/ANSI_escape_code - """ - - _bold = "\u001b[1m" - _gray = "\u001b[90m" - _red = "\u001b[31m" - _reset = "\u001b[0m" - _yellow = "\u001b[33m" - - @classmethod - def bold(cls, s: str) -> str: - return cls._format(s, cls._bold) - - @classmethod - def gray(cls, s: str) -> str: - return cls._format(s, cls._gray) - - @classmethod - def red(cls, s: str) -> str: - return cls._format(s, cls._bold + cls._red) - - @classmethod - def yellow(cls, s: str) -> str: - return cls._format(s, cls._yellow) - - @classmethod - def _format(cls, s: str, code: str) -> str: - if os.environ.get("NO_COLOR"): - # See https://no-color.org/ - return s - return f"{code}{s}{cls._reset}" - - -def tabulate(rows: list[list[Union[str, int]]], headers: list[str]) -> str: - """ - Inspired by: - - - stackoverflow.com/a/8356620/593036 - - stackoverflow.com/questions/9535954/printing-lists-as-tabular-data - """ - col_widths = [max(len(str(x)) for x in col) for col in zip(*rows, headers)] - row_format = ("{{:{}}} " * len(headers)).format(*col_widths) - lines = [] - lines.append(row_format.format(*headers)) - lines.append(row_format.format(*["-" * w for w in col_widths])) - for row in rows: - lines.append(row_format.format(*row)) - return "\n".join(lines) - - -def show_deprecation_warning(old_command: str, new_command: str): - """Show a yellow warning about deprecated CLI command.""" - print(ANSI.yellow(f"⚠️ Warning: '{old_command}' is deprecated. Use '{new_command}' instead.")) diff --git a/src/huggingface_hub/commands/delete_cache.py b/src/huggingface_hub/commands/delete_cache.py deleted file mode 100644 index 2983ab80b0..0000000000 --- a/src/huggingface_hub/commands/delete_cache.py +++ /dev/null @@ -1,476 +0,0 @@ -# coding=utf-8 -# Copyright 2022-present, the HuggingFace Inc. team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Contains command to delete some revisions from the HF cache directory. - -Usage: - huggingface-cli delete-cache - huggingface-cli delete-cache --disable-tui - huggingface-cli delete-cache --dir ~/.cache/huggingface/hub - huggingface-cli delete-cache --sort=size - -NOTE: - This command is based on `InquirerPy` to build the multiselect menu in the terminal. - This dependency has to be installed with `pip install "huggingface_hub[cli]"`. Since - we want to avoid as much as possible cross-platform issues, I chose a library that - is built on top of `python-prompt-toolkit` which seems to be a reference in terminal - GUI (actively maintained on both Unix and Windows, 7.9k stars). - - For the moment, the TUI feature is in beta. - - See: - - https://github.com/kazhala/InquirerPy - - https://inquirerpy.readthedocs.io/en/latest/ - - https://github.com/prompt-toolkit/python-prompt-toolkit - - Other solutions could have been: - - `simple_term_menu`: would be good as well for our use case but some issues suggest - that Windows is less supported. - See: https://github.com/IngoMeyer441/simple-term-menu - - `PyInquirer`: very similar to `InquirerPy` but older and not maintained anymore. - In particular, no support of Python3.10. - See: https://github.com/CITGuru/PyInquirer - - `pick` (or `pickpack`): easy to use and flexible but built on top of Python's - standard library `curses` that is specific to Unix (not implemented on Windows). - See https://github.com/wong2/pick and https://github.com/anafvana/pickpack. - - `inquirer`: lot of traction (700 stars) but explicitly states "experimental - support of Windows". Not built on top of `python-prompt-toolkit`. - See https://github.com/magmax/python-inquirer - -TODO: add support for `huggingface-cli delete-cache aaaaaa bbbbbb cccccc (...)` ? -TODO: add "--keep-last" arg to delete revisions that are not on `main` ref -TODO: add "--filter" arg to filter repositories by name ? -TODO: add "--limit" arg to limit to X repos ? -TODO: add "-y" arg for immediate deletion ? -See discussions in https://github.com/huggingface/huggingface_hub/issues/1025. -""" - -import os -from argparse import Namespace, _SubParsersAction -from functools import wraps -from tempfile import mkstemp -from typing import Any, Callable, Iterable, Literal, Optional, Union - -from ..utils import CachedRepoInfo, CachedRevisionInfo, HFCacheInfo, scan_cache_dir -from . import BaseHuggingfaceCLICommand -from ._cli_utils import ANSI, show_deprecation_warning - - -try: - from InquirerPy import inquirer - from InquirerPy.base.control import Choice - from InquirerPy.separator import Separator - - _inquirer_py_available = True -except ImportError: - _inquirer_py_available = False - -SortingOption_T = Literal["alphabetical", "lastUpdated", "lastUsed", "size"] - - -def require_inquirer_py(fn: Callable) -> Callable: - """Decorator to flag methods that require `InquirerPy`.""" - - # TODO: refactor this + imports in a unified pattern across codebase - @wraps(fn) - def _inner(*args, **kwargs): - if not _inquirer_py_available: - raise ImportError( - "The `delete-cache` command requires extra dependencies to work with" - ' the TUI.\nPlease run `pip install "huggingface_hub[cli]"` to install' - " them.\nOtherwise, disable TUI using the `--disable-tui` flag." - ) - - return fn(*args, **kwargs) - - return _inner - - -# Possibility for the user to cancel deletion -_CANCEL_DELETION_STR = "CANCEL_DELETION" - - -class DeleteCacheCommand(BaseHuggingfaceCLICommand): - @staticmethod - def register_subcommand(parser: _SubParsersAction): - delete_cache_parser = parser.add_parser("delete-cache", help="Delete revisions from the cache directory.") - - delete_cache_parser.add_argument( - "--dir", - type=str, - default=None, - help="cache directory (optional). Default to the default HuggingFace cache.", - ) - - delete_cache_parser.add_argument( - "--disable-tui", - action="store_true", - help=( - "Disable Terminal User Interface (TUI) mode. Useful if your" - " platform/terminal doesn't support the multiselect menu." - ), - ) - - delete_cache_parser.add_argument( - "--sort", - nargs="?", - choices=["alphabetical", "lastUpdated", "lastUsed", "size"], - help=( - "Sort repositories by the specified criteria. Options: " - "'alphabetical' (A-Z), " - "'lastUpdated' (newest first), " - "'lastUsed' (most recent first), " - "'size' (largest first)." - ), - ) - - delete_cache_parser.set_defaults(func=DeleteCacheCommand) - - def __init__(self, args: Namespace) -> None: - self.cache_dir: Optional[str] = args.dir - self.disable_tui: bool = args.disable_tui - self.sort_by: Optional[SortingOption_T] = args.sort - - def run(self): - """Run `delete-cache` command with or without TUI.""" - show_deprecation_warning("huggingface-cli delete-cache", "hf cache delete") - - # Scan cache directory - hf_cache_info = scan_cache_dir(self.cache_dir) - - # Manual review from the user - if self.disable_tui: - selected_hashes = _manual_review_no_tui(hf_cache_info, preselected=[], sort_by=self.sort_by) - else: - selected_hashes = _manual_review_tui(hf_cache_info, preselected=[], sort_by=self.sort_by) - - # If deletion is not cancelled - if len(selected_hashes) > 0 and _CANCEL_DELETION_STR not in selected_hashes: - confirm_message = _get_expectations_str(hf_cache_info, selected_hashes) + " Confirm deletion ?" - - # Confirm deletion - if self.disable_tui: - confirmed = _ask_for_confirmation_no_tui(confirm_message) - else: - confirmed = _ask_for_confirmation_tui(confirm_message) - - # Deletion is confirmed - if confirmed: - strategy = hf_cache_info.delete_revisions(*selected_hashes) - print("Start deletion.") - strategy.execute() - print( - f"Done. Deleted {len(strategy.repos)} repo(s) and" - f" {len(strategy.snapshots)} revision(s) for a total of" - f" {strategy.expected_freed_size_str}." - ) - return - - # Deletion is cancelled - print("Deletion is cancelled. Do nothing.") - - -def _get_repo_sorting_key(repo: CachedRepoInfo, sort_by: Optional[SortingOption_T] = None): - if sort_by == "alphabetical": - return (repo.repo_type, repo.repo_id.lower()) # by type then name - elif sort_by == "lastUpdated": - return -max(rev.last_modified for rev in repo.revisions) # newest first - elif sort_by == "lastUsed": - return -repo.last_accessed # most recently used first - elif sort_by == "size": - return -repo.size_on_disk # largest first - else: - return (repo.repo_type, repo.repo_id) # default stable order - - -@require_inquirer_py -def _manual_review_tui( - hf_cache_info: HFCacheInfo, - preselected: list[str], - sort_by: Optional[SortingOption_T] = None, -) -> list[str]: - """Ask the user for a manual review of the revisions to delete. - - Displays a multi-select menu in the terminal (TUI). - """ - # Define multiselect list - choices = _get_tui_choices_from_scan( - repos=hf_cache_info.repos, - preselected=preselected, - sort_by=sort_by, - ) - checkbox = inquirer.checkbox( - message="Select revisions to delete:", - choices=choices, # List of revisions with some pre-selection - cycle=False, # No loop between top and bottom - height=100, # Large list if possible - # We use the instruction to display to the user the expected effect of the - # deletion. - instruction=_get_expectations_str( - hf_cache_info, - selected_hashes=[c.value for c in choices if isinstance(c, Choice) and c.enabled], - ), - # We use the long instruction to should keybindings instructions to the user - long_instruction="Press to select, to validate and to quit without modification.", - # Message that is displayed once the user validates its selection. - transformer=lambda result: f"{len(result)} revision(s) selected.", - ) - - # Add a callback to update the information line when a revision is - # selected/unselected - def _update_expectations(_) -> None: - # Hacky way to dynamically set an instruction message to the checkbox when - # a revision hash is selected/unselected. - checkbox._instruction = _get_expectations_str( - hf_cache_info, - selected_hashes=[choice["value"] for choice in checkbox.content_control.choices if choice["enabled"]], - ) - - checkbox.kb_func_lookup["toggle"].append({"func": _update_expectations}) - - # Finally display the form to the user. - try: - return checkbox.execute() - except KeyboardInterrupt: - return [] # Quit without deletion - - -@require_inquirer_py -def _ask_for_confirmation_tui(message: str, default: bool = True) -> bool: - """Ask for confirmation using Inquirer.""" - return inquirer.confirm(message, default=default).execute() - - -def _get_tui_choices_from_scan( - repos: Iterable[CachedRepoInfo], - preselected: list[str], - sort_by: Optional[SortingOption_T] = None, -) -> list: - """Build a list of choices from the scanned repos. - - Args: - repos (*Iterable[`CachedRepoInfo`]*): - List of scanned repos on which we want to delete revisions. - preselected (*list[`str`]*): - List of revision hashes that will be preselected. - sort_by (*Optional[SortingOption_T]*): - Sorting direction. Choices: "alphabetical", "lastUpdated", "lastUsed", "size". - - Return: - The list of choices to pass to `inquirer.checkbox`. - """ - choices: list[Union[Choice, Separator]] = [] - - # First choice is to cancel the deletion - choices.append( - Choice( - _CANCEL_DELETION_STR, - name="None of the following (if selected, nothing will be deleted).", - enabled=False, - ) - ) - - # Sort repos based on specified criteria - sorted_repos = sorted(repos, key=lambda repo: _get_repo_sorting_key(repo, sort_by)) - - for repo in sorted_repos: - # Repo as separator - choices.append( - Separator( - f"\n{repo.repo_type.capitalize()} {repo.repo_id} ({repo.size_on_disk_str}," - f" used {repo.last_accessed_str})" - ) - ) - for revision in sorted(repo.revisions, key=_revision_sorting_order): - # Revision as choice - choices.append( - Choice( - revision.commit_hash, - name=( - f"{revision.commit_hash[:8]}:" - f" {', '.join(sorted(revision.refs)) or '(detached)'} #" - f" modified {revision.last_modified_str}" - ), - enabled=revision.commit_hash in preselected, - ) - ) - - # Return choices - return choices - - -def _manual_review_no_tui( - hf_cache_info: HFCacheInfo, - preselected: list[str], - sort_by: Optional[SortingOption_T] = None, -) -> list[str]: - """Ask the user for a manual review of the revisions to delete. - - Used when TUI is disabled. Manual review happens in a separate tmp file that the - user can manually edit. - """ - # 1. Generate temporary file with delete commands. - fd, tmp_path = mkstemp(suffix=".txt") # suffix to make it easier to find by editors - os.close(fd) - - lines = [] - - sorted_repos = sorted(hf_cache_info.repos, key=lambda repo: _get_repo_sorting_key(repo, sort_by)) - - for repo in sorted_repos: - lines.append( - f"\n# {repo.repo_type.capitalize()} {repo.repo_id} ({repo.size_on_disk_str}," - f" used {repo.last_accessed_str})" - ) - for revision in sorted(repo.revisions, key=_revision_sorting_order): - lines.append( - # Deselect by prepending a '#' - f"{'' if revision.commit_hash in preselected else '#'} " - f" {revision.commit_hash} # Refs:" - # Print `refs` as comment on same line - f" {', '.join(sorted(revision.refs)) or '(detached)'} # modified" - # Print `last_modified` as comment on same line - f" {revision.last_modified_str}" - ) - - with open(tmp_path, "w") as f: - f.write(_MANUAL_REVIEW_NO_TUI_INSTRUCTIONS) - f.write("\n".join(lines)) - - # 2. Prompt instructions to user. - instructions = f""" - TUI is disabled. In order to select which revisions you want to delete, please edit - the following file using the text editor of your choice. Instructions for manual - editing are located at the beginning of the file. Edit the file, save it and confirm - to continue. - File to edit: {ANSI.bold(tmp_path)} - """ - print("\n".join(line.strip() for line in instructions.strip().split("\n"))) - - # 3. Wait for user confirmation. - while True: - selected_hashes = _read_manual_review_tmp_file(tmp_path) - if _ask_for_confirmation_no_tui( - _get_expectations_str(hf_cache_info, selected_hashes) + " Continue ?", - default=False, - ): - break - - # 4. Return selected_hashes sorted to maintain stable order - os.remove(tmp_path) - return sorted(selected_hashes) # Sort to maintain stable order - - -def _ask_for_confirmation_no_tui(message: str, default: bool = True) -> bool: - """Ask for confirmation using pure-python.""" - YES = ("y", "yes", "1") - NO = ("n", "no", "0") - DEFAULT = "" - ALL = YES + NO + (DEFAULT,) - full_message = message + (" (Y/n) " if default else " (y/N) ") - while True: - answer = input(full_message).lower() - if answer == DEFAULT: - return default - if answer in YES: - return True - if answer in NO: - return False - print(f"Invalid input. Must be one of {ALL}") - - -def _get_expectations_str(hf_cache_info: HFCacheInfo, selected_hashes: list[str]) -> str: - """Format a string to display to the user how much space would be saved. - - Example: - ``` - >>> _get_expectations_str(hf_cache_info, selected_hashes) - '7 revisions selected counting for 4.3G.' - ``` - """ - if _CANCEL_DELETION_STR in selected_hashes: - return "Nothing will be deleted." - strategy = hf_cache_info.delete_revisions(*selected_hashes) - return f"{len(selected_hashes)} revisions selected counting for {strategy.expected_freed_size_str}." - - -def _read_manual_review_tmp_file(tmp_path: str) -> list[str]: - """Read the manually reviewed instruction file and return a list of revision hash. - - Example: - ```txt - # This is the tmp file content - ### - - # Commented out line - 123456789 # revision hash - - # Something else - # a_newer_hash # 2 days ago - an_older_hash # 3 days ago - ``` - - ```py - >>> _read_manual_review_tmp_file(tmp_path) - ['123456789', 'an_older_hash'] - ``` - """ - with open(tmp_path) as f: - content = f.read() - - # Split lines - lines = [line.strip() for line in content.split("\n")] - - # Filter commented lines - selected_lines = [line for line in lines if not line.startswith("#")] - - # Select only before comment - selected_hashes = [line.split("#")[0].strip() for line in selected_lines] - - # Return revision hashes - return [hash for hash in selected_hashes if len(hash) > 0] - - -_MANUAL_REVIEW_NO_TUI_INSTRUCTIONS = f""" -# INSTRUCTIONS -# ------------ -# This is a temporary file created by running `huggingface-cli delete-cache` with the -# `--disable-tui` option. It contains a set of revisions that can be deleted from your -# local cache directory. -# -# Please manually review the revisions you want to delete: -# - Revision hashes can be commented out with '#'. -# - Only non-commented revisions in this file will be deleted. -# - Revision hashes that are removed from this file are ignored as well. -# - If `{_CANCEL_DELETION_STR}` line is uncommented, the all cache deletion is cancelled and -# no changes will be applied. -# -# Once you've manually reviewed this file, please confirm deletion in the terminal. This -# file will be automatically removed once done. -# ------------ - -# KILL SWITCH -# ------------ -# Un-comment following line to completely cancel the deletion process -# {_CANCEL_DELETION_STR} -# ------------ - -# REVISIONS -# ------------ -""".strip() - - -def _revision_sorting_order(revision: CachedRevisionInfo) -> Any: - # Sort by last modified (oldest first) - return revision.last_modified diff --git a/src/huggingface_hub/commands/download.py b/src/huggingface_hub/commands/download.py deleted file mode 100644 index 06ce8905c1..0000000000 --- a/src/huggingface_hub/commands/download.py +++ /dev/null @@ -1,195 +0,0 @@ -# coding=utf-8 -# Copyright 2023-present, the HuggingFace Inc. team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Contains command to download files from the Hub with the CLI. - -Usage: - huggingface-cli download --help - - # Download file - huggingface-cli download gpt2 config.json - - # Download entire repo - huggingface-cli download fffiloni/zeroscope --repo-type=space --revision=refs/pr/78 - - # Download repo with filters - huggingface-cli download gpt2 --include="*.safetensors" - - # Download with token - huggingface-cli download Wauplin/private-model --token=hf_*** - - # Download quietly (no progress bar, no warnings, only the returned path) - huggingface-cli download gpt2 config.json --quiet - - # Download to local dir - huggingface-cli download gpt2 --local-dir=./models/gpt2 -""" - -import warnings -from argparse import Namespace, _SubParsersAction -from typing import Optional - -from huggingface_hub import logging -from huggingface_hub._snapshot_download import snapshot_download -from huggingface_hub.commands import BaseHuggingfaceCLICommand -from huggingface_hub.file_download import hf_hub_download -from huggingface_hub.utils import disable_progress_bars, enable_progress_bars - -from ._cli_utils import show_deprecation_warning - - -logger = logging.get_logger(__name__) - - -class DownloadCommand(BaseHuggingfaceCLICommand): - @staticmethod - def register_subcommand(parser: _SubParsersAction): - download_parser = parser.add_parser("download", help="Download files from the Hub") - download_parser.add_argument( - "repo_id", type=str, help="ID of the repo to download from (e.g. `username/repo-name`)." - ) - download_parser.add_argument( - "filenames", type=str, nargs="*", help="Files to download (e.g. `config.json`, `data/metadata.jsonl`)." - ) - download_parser.add_argument( - "--repo-type", - choices=["model", "dataset", "space"], - default="model", - help="Type of repo to download from (defaults to 'model').", - ) - download_parser.add_argument( - "--revision", - type=str, - help="An optional Git revision id which can be a branch name, a tag, or a commit hash.", - ) - download_parser.add_argument( - "--include", nargs="*", type=str, help="Glob patterns to match files to download." - ) - download_parser.add_argument( - "--exclude", nargs="*", type=str, help="Glob patterns to exclude from files to download." - ) - download_parser.add_argument( - "--cache-dir", type=str, help="Path to the directory where to save the downloaded files." - ) - download_parser.add_argument( - "--local-dir", - type=str, - help=( - "If set, the downloaded file will be placed under this directory. Check out" - " https://huggingface.co/docs/huggingface_hub/guides/download#download-files-to-local-folder for more" - " details." - ), - ) - download_parser.add_argument( - "--local-dir-use-symlinks", - choices=["auto", "True", "False"], - help=("Deprecated and ignored. Downloading to a local directory does not use symlinks anymore."), - ) - download_parser.add_argument( - "--force-download", - action="store_true", - help="If True, the files will be downloaded even if they are already cached.", - ) - download_parser.add_argument( - "--resume-download", - action="store_true", - help="Deprecated and ignored. Downloading a file to local dir always attempts to resume previously interrupted downloads (unless hf-transfer is enabled).", - ) - download_parser.add_argument( - "--token", type=str, help="A User Access Token generated from https://huggingface.co/settings/tokens" - ) - download_parser.add_argument( - "--quiet", - action="store_true", - help="If True, progress bars are disabled and only the path to the download files is printed.", - ) - download_parser.add_argument( - "--max-workers", - type=int, - default=8, - help="Maximum number of workers to use for downloading files. Default is 8.", - ) - download_parser.set_defaults(func=DownloadCommand) - - def __init__(self, args: Namespace) -> None: - self.token = args.token - self.repo_id: str = args.repo_id - self.filenames: list[str] = args.filenames - self.repo_type: str = args.repo_type - self.revision: Optional[str] = args.revision - self.include: Optional[list[str]] = args.include - self.exclude: Optional[list[str]] = args.exclude - self.cache_dir: Optional[str] = args.cache_dir - self.local_dir: Optional[str] = args.local_dir - self.force_download: bool = args.force_download - self.quiet: bool = args.quiet - self.max_workers: int = args.max_workers - - def run(self) -> None: - show_deprecation_warning("huggingface-cli download", "hf download") - - if self.quiet: - disable_progress_bars() - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - print(self._download()) # Print path to downloaded files - enable_progress_bars() - else: - logging.set_verbosity_info() - print(self._download()) # Print path to downloaded files - logging.set_verbosity_warning() - - def _download(self) -> str: - # Warn user if patterns are ignored - if len(self.filenames) > 0: - if self.include is not None and len(self.include) > 0: - warnings.warn("Ignoring `--include` since filenames have being explicitly set.") - if self.exclude is not None and len(self.exclude) > 0: - warnings.warn("Ignoring `--exclude` since filenames have being explicitly set.") - - # Single file to download: use `hf_hub_download` - if len(self.filenames) == 1: - return hf_hub_download( - repo_id=self.repo_id, - repo_type=self.repo_type, - revision=self.revision, - filename=self.filenames[0], - cache_dir=self.cache_dir, - force_download=self.force_download, - token=self.token, - local_dir=self.local_dir, - library_name="huggingface-cli", - ) - - # Otherwise: use `snapshot_download` to ensure all files comes from same revision - elif len(self.filenames) == 0: - allow_patterns = self.include - ignore_patterns = self.exclude - else: - allow_patterns = self.filenames - ignore_patterns = None - - return snapshot_download( - repo_id=self.repo_id, - repo_type=self.repo_type, - revision=self.revision, - allow_patterns=allow_patterns, - ignore_patterns=ignore_patterns, - force_download=self.force_download, - cache_dir=self.cache_dir, - token=self.token, - local_dir=self.local_dir, - library_name="huggingface-cli", - max_workers=self.max_workers, - ) diff --git a/src/huggingface_hub/commands/env.py b/src/huggingface_hub/commands/env.py deleted file mode 100644 index ad674738b2..0000000000 --- a/src/huggingface_hub/commands/env.py +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright 2022 The HuggingFace Team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Contains command to print information about the environment. - -Usage: - huggingface-cli env -""" - -from argparse import _SubParsersAction - -from ..utils import dump_environment_info -from . import BaseHuggingfaceCLICommand -from ._cli_utils import show_deprecation_warning - - -class EnvironmentCommand(BaseHuggingfaceCLICommand): - def __init__(self, args): - self.args = args - - @staticmethod - def register_subcommand(parser: _SubParsersAction): - env_parser = parser.add_parser("env", help="Print information about the environment.") - env_parser.set_defaults(func=EnvironmentCommand) - - def run(self) -> None: - show_deprecation_warning("huggingface-cli env", "hf env") - - dump_environment_info() diff --git a/src/huggingface_hub/commands/huggingface_cli.py b/src/huggingface_hub/commands/huggingface_cli.py deleted file mode 100644 index 697c85d1e3..0000000000 --- a/src/huggingface_hub/commands/huggingface_cli.py +++ /dev/null @@ -1,65 +0,0 @@ -# Copyright 2020 The HuggingFace Team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from argparse import ArgumentParser - -from huggingface_hub.commands._cli_utils import show_deprecation_warning -from huggingface_hub.commands.delete_cache import DeleteCacheCommand -from huggingface_hub.commands.download import DownloadCommand -from huggingface_hub.commands.env import EnvironmentCommand -from huggingface_hub.commands.lfs import LfsCommands -from huggingface_hub.commands.repo import RepoCommands -from huggingface_hub.commands.repo_files import RepoFilesCommand -from huggingface_hub.commands.scan_cache import ScanCacheCommand -from huggingface_hub.commands.tag import TagCommands -from huggingface_hub.commands.upload import UploadCommand -from huggingface_hub.commands.upload_large_folder import UploadLargeFolderCommand -from huggingface_hub.commands.user import UserCommands -from huggingface_hub.commands.version import VersionCommand - - -def main(): - parser = ArgumentParser("huggingface-cli", usage="huggingface-cli []") - commands_parser = parser.add_subparsers(help="huggingface-cli command helpers") - - # Register commands - DownloadCommand.register_subcommand(commands_parser) - UploadCommand.register_subcommand(commands_parser) - RepoFilesCommand.register_subcommand(commands_parser) - EnvironmentCommand.register_subcommand(commands_parser) - UserCommands.register_subcommand(commands_parser) - RepoCommands.register_subcommand(commands_parser) - LfsCommands.register_subcommand(commands_parser) - ScanCacheCommand.register_subcommand(commands_parser) - DeleteCacheCommand.register_subcommand(commands_parser) - TagCommands.register_subcommand(commands_parser) - VersionCommand.register_subcommand(commands_parser) - - # Experimental - UploadLargeFolderCommand.register_subcommand(commands_parser) - - # Let's go - args = parser.parse_args() - if not hasattr(args, "func"): - show_deprecation_warning("huggingface-cli", "hf") - parser.print_help() - exit(1) - - # Run - service = args.func(args) - service.run() - - -if __name__ == "__main__": - main() diff --git a/src/huggingface_hub/commands/lfs.py b/src/huggingface_hub/commands/lfs.py deleted file mode 100644 index 2133bf1f00..0000000000 --- a/src/huggingface_hub/commands/lfs.py +++ /dev/null @@ -1,200 +0,0 @@ -""" -Implementation of a custom transfer agent for the transfer type "multipart" for -git-lfs. - -Inspired by: -github.com/cbartz/git-lfs-swift-transfer-agent/blob/master/git_lfs_swift_transfer.py - -Spec is: github.com/git-lfs/git-lfs/blob/master/docs/custom-transfers.md - - -To launch debugger while developing: - -``` [lfs "customtransfer.multipart"] -path = /path/to/huggingface_hub/.env/bin/python args = -m debugpy --listen 5678 ---wait-for-client -/path/to/huggingface_hub/src/huggingface_hub/commands/huggingface_cli.py -lfs-multipart-upload ```""" - -import json -import os -import subprocess -import sys -from argparse import _SubParsersAction -from typing import Optional - -from huggingface_hub.commands import BaseHuggingfaceCLICommand -from huggingface_hub.lfs import LFS_MULTIPART_UPLOAD_COMMAND - -from ..utils import get_session, hf_raise_for_status, logging -from ..utils._lfs import SliceFileObj - - -logger = logging.get_logger(__name__) - - -class LfsCommands(BaseHuggingfaceCLICommand): - """ - Implementation of a custom transfer agent for the transfer type "multipart" - for git-lfs. This lets users upload large files >5GB 🔥. Spec for LFS custom - transfer agent is: - https://github.com/git-lfs/git-lfs/blob/master/docs/custom-transfers.md - - This introduces two commands to the CLI: - - 1. $ huggingface-cli lfs-enable-largefiles - - This should be executed once for each model repo that contains a model file - >5GB. It's documented in the error message you get if you just try to git - push a 5GB file without having enabled it before. - - 2. $ huggingface-cli lfs-multipart-upload - - This command is called by lfs directly and is not meant to be called by the - user. - """ - - @staticmethod - def register_subcommand(parser: _SubParsersAction): - enable_parser = parser.add_parser( - "lfs-enable-largefiles", help="Configure your repository to enable upload of files > 5GB." - ) - enable_parser.add_argument("path", type=str, help="Local path to repository you want to configure.") - enable_parser.set_defaults(func=lambda args: LfsEnableCommand(args)) - - # Command will get called by git-lfs, do not call it directly. - upload_parser = parser.add_parser(LFS_MULTIPART_UPLOAD_COMMAND, add_help=False) - upload_parser.set_defaults(func=lambda args: LfsUploadCommand(args)) - - -class LfsEnableCommand: - def __init__(self, args): - self.args = args - - def run(self): - local_path = os.path.abspath(self.args.path) - if not os.path.isdir(local_path): - print("This does not look like a valid git repo.") - exit(1) - subprocess.run( - "git config lfs.customtransfer.multipart.path huggingface-cli".split(), - check=True, - cwd=local_path, - ) - subprocess.run( - f"git config lfs.customtransfer.multipart.args {LFS_MULTIPART_UPLOAD_COMMAND}".split(), - check=True, - cwd=local_path, - ) - print("Local repo set up for largefiles") - - -def write_msg(msg: dict): - """Write out the message in Line delimited JSON.""" - msg_str = json.dumps(msg) + "\n" - sys.stdout.write(msg_str) - sys.stdout.flush() - - -def read_msg() -> Optional[dict]: - """Read Line delimited JSON from stdin.""" - msg = json.loads(sys.stdin.readline().strip()) - - if "terminate" in (msg.get("type"), msg.get("event")): - # terminate message received - return None - - if msg.get("event") not in ("download", "upload"): - logger.critical("Received unexpected message") - sys.exit(1) - - return msg - - -class LfsUploadCommand: - def __init__(self, args) -> None: - self.args = args - - def run(self) -> None: - # Immediately after invoking a custom transfer process, git-lfs - # sends initiation data to the process over stdin. - # This tells the process useful information about the configuration. - init_msg = json.loads(sys.stdin.readline().strip()) - if not (init_msg.get("event") == "init" and init_msg.get("operation") == "upload"): - write_msg({"error": {"code": 32, "message": "Wrong lfs init operation"}}) - sys.exit(1) - - # The transfer process should use the information it needs from the - # initiation structure, and also perform any one-off setup tasks it - # needs to do. It should then respond on stdout with a simple empty - # confirmation structure, as follows: - write_msg({}) - - # After the initiation exchange, git-lfs will send any number of - # transfer requests to the stdin of the transfer process, in a serial sequence. - while True: - msg = read_msg() - if msg is None: - # When all transfers have been processed, git-lfs will send - # a terminate event to the stdin of the transfer process. - # On receiving this message the transfer process should - # clean up and terminate. No response is expected. - sys.exit(0) - - oid = msg["oid"] - filepath = msg["path"] - completion_url = msg["action"]["href"] - header = msg["action"]["header"] - chunk_size = int(header.pop("chunk_size")) - presigned_urls: list[str] = list(header.values()) - - # Send a "started" progress event to allow other workers to start. - # Otherwise they're delayed until first "progress" event is reported, - # i.e. after the first 5GB by default (!) - write_msg( - { - "event": "progress", - "oid": oid, - "bytesSoFar": 1, - "bytesSinceLast": 0, - } - ) - - parts = [] - with open(filepath, "rb") as file: - for i, presigned_url in enumerate(presigned_urls): - with SliceFileObj( - file, - seek_from=i * chunk_size, - read_limit=chunk_size, - ) as data: - r = get_session().put(presigned_url, data=data) - hf_raise_for_status(r) - parts.append( - { - "etag": r.headers.get("etag"), - "partNumber": i + 1, - } - ) - # In order to support progress reporting while data is uploading / downloading, - # the transfer process should post messages to stdout - write_msg( - { - "event": "progress", - "oid": oid, - "bytesSoFar": (i + 1) * chunk_size, - "bytesSinceLast": chunk_size, - } - ) - # Not precise but that's ok. - - r = get_session().post( - completion_url, - json={ - "oid": oid, - "parts": parts, - }, - ) - hf_raise_for_status(r) - - write_msg({"event": "complete", "oid": oid}) diff --git a/src/huggingface_hub/commands/repo.py b/src/huggingface_hub/commands/repo.py deleted file mode 100644 index fe75349d67..0000000000 --- a/src/huggingface_hub/commands/repo.py +++ /dev/null @@ -1,151 +0,0 @@ -# Copyright 2025 The HuggingFace Team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Contains commands to interact with repositories on the Hugging Face Hub. - -Usage: - # create a new dataset repo on the Hub - huggingface-cli repo create my-cool-dataset --repo-type=dataset - - # create a private model repo on the Hub - huggingface-cli repo create my-cool-model --private -""" - -import argparse -from argparse import _SubParsersAction -from typing import Optional - -from huggingface_hub.commands import BaseHuggingfaceCLICommand -from huggingface_hub.commands._cli_utils import ANSI -from huggingface_hub.constants import SPACES_SDK_TYPES -from huggingface_hub.hf_api import HfApi -from huggingface_hub.utils import logging - -from ._cli_utils import show_deprecation_warning - - -logger = logging.get_logger(__name__) - - -class RepoCommands(BaseHuggingfaceCLICommand): - @staticmethod - def register_subcommand(parser: _SubParsersAction): - repo_parser = parser.add_parser("repo", help="{create} Commands to interact with your huggingface.co repos.") - repo_subparsers = repo_parser.add_subparsers(help="huggingface.co repos related commands") - repo_create_parser = repo_subparsers.add_parser("create", help="Create a new repo on huggingface.co") - repo_create_parser.add_argument( - "repo_id", - type=str, - help="The ID of the repo to create to (e.g. `username/repo-name`). The username is optional and will be set to your username if not provided.", - ) - repo_create_parser.add_argument( - "--repo-type", - type=str, - help='Optional: set to "dataset" or "space" if creating a dataset or space, default is model.', - ) - repo_create_parser.add_argument( - "--space_sdk", - type=str, - help='Optional: Hugging Face Spaces SDK type. Required when --type is set to "space".', - choices=SPACES_SDK_TYPES, - ) - repo_create_parser.add_argument( - "--private", - action="store_true", - help="Whether to create a private repository. Defaults to public unless the organization's default is private.", - ) - repo_create_parser.add_argument( - "--token", - type=str, - help="Hugging Face token. Will default to the locally saved token if not provided.", - ) - repo_create_parser.add_argument( - "--exist-ok", - action="store_true", - help="Do not raise an error if repo already exists.", - ) - repo_create_parser.add_argument( - "--resource-group-id", - type=str, - help="Resource group in which to create the repo. Resource groups is only available for Enterprise Hub organizations.", - ) - repo_create_parser.add_argument( - "--type", - type=str, - help="[Deprecated]: use --repo-type instead.", - ) - repo_create_parser.add_argument( - "-y", - "--yes", - action="store_true", - help="[Deprecated] no effect.", - ) - repo_create_parser.add_argument( - "--organization", type=str, help="[Deprecated] Pass the organization namespace directly in the repo_id." - ) - repo_create_parser.set_defaults(func=lambda args: RepoCreateCommand(args)) - - -class RepoCreateCommand: - def __init__(self, args: argparse.Namespace): - self.repo_id: str = args.repo_id - self.repo_type: Optional[str] = args.repo_type or args.type - self.space_sdk: Optional[str] = args.space_sdk - self.organization: Optional[str] = args.organization - self.yes: bool = args.yes - self.private: bool = args.private - self.token: Optional[str] = args.token - self.exist_ok: bool = args.exist_ok - self.resource_group_id: Optional[str] = args.resource_group_id - - if args.type is not None: - print( - ANSI.yellow( - "The --type argument is deprecated and will be removed in a future version. Use --repo-type instead." - ) - ) - if self.organization is not None: - print( - ANSI.yellow( - "The --organization argument is deprecated and will be removed in a future version. Pass the organization namespace directly in the repo_id." - ) - ) - if self.yes: - print( - ANSI.yellow( - "The --yes argument is deprecated and will be removed in a future version. It does not have any effect." - ) - ) - - self._api = HfApi() - - def run(self): - show_deprecation_warning("huggingface-cli repo", "hf repo") - - if self.organization is not None: - if "/" in self.repo_id: - print(ANSI.red("You cannot pass both --organization and a repo_id with a namespace.")) - exit(1) - self.repo_id = f"{self.organization}/{self.repo_id}" - - repo_url = self._api.create_repo( - repo_id=self.repo_id, - repo_type=self.repo_type, - private=self.private, - token=self.token, - exist_ok=self.exist_ok, - resource_group_id=self.resource_group_id, - space_sdk=self.space_sdk, - ) - print(f"Successfully created {ANSI.bold(repo_url.repo_id)} on the Hub.") - print(f"Your repo is now available at {ANSI.bold(repo_url)}") diff --git a/src/huggingface_hub/commands/repo_files.py b/src/huggingface_hub/commands/repo_files.py deleted file mode 100644 index b914a2dc92..0000000000 --- a/src/huggingface_hub/commands/repo_files.py +++ /dev/null @@ -1,132 +0,0 @@ -# coding=utf-8 -# Copyright 2023-present, the HuggingFace Inc. team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Contains command to update or delete files in a repository using the CLI. - -Usage: - # delete all - huggingface-cli repo-files delete "*" - - # delete single file - huggingface-cli repo-files delete file.txt - - # delete single folder - huggingface-cli repo-files delete folder/ - - # delete multiple - huggingface-cli repo-files delete file.txt folder/ file2.txt - - # delete multiple patterns - huggingface-cli repo-files delete file.txt "*.json" "folder/*.parquet" - - # delete from different revision / repo-type - huggingface-cli repo-files delete file.txt --revision=refs/pr/1 --repo-type=dataset -""" - -from argparse import _SubParsersAction -from typing import Optional - -from huggingface_hub import logging -from huggingface_hub.commands import BaseHuggingfaceCLICommand -from huggingface_hub.hf_api import HfApi - -from ._cli_utils import show_deprecation_warning - - -logger = logging.get_logger(__name__) - - -class DeleteFilesSubCommand: - def __init__(self, args) -> None: - self.args = args - self.repo_id: str = args.repo_id - self.repo_type: Optional[str] = args.repo_type - self.revision: Optional[str] = args.revision - self.api: HfApi = HfApi(token=args.token, library_name="huggingface-cli") - self.patterns: list[str] = args.patterns - self.commit_message: Optional[str] = args.commit_message - self.commit_description: Optional[str] = args.commit_description - self.create_pr: bool = args.create_pr - self.token: Optional[str] = args.token - - def run(self) -> None: - show_deprecation_warning("huggingface-cli repo-files", "hf repo-files") - - logging.set_verbosity_info() - url = self.api.delete_files( - delete_patterns=self.patterns, - repo_id=self.repo_id, - repo_type=self.repo_type, - revision=self.revision, - commit_message=self.commit_message, - commit_description=self.commit_description, - create_pr=self.create_pr, - ) - print(f"Files correctly deleted from repo. Commit: {url}.") - logging.set_verbosity_warning() - - -class RepoFilesCommand(BaseHuggingfaceCLICommand): - @staticmethod - def register_subcommand(parser: _SubParsersAction): - repo_files_parser = parser.add_parser("repo-files", help="Manage files in a repo on the Hub") - repo_files_parser.add_argument( - "repo_id", type=str, help="The ID of the repo to manage (e.g. `username/repo-name`)." - ) - repo_files_subparsers = repo_files_parser.add_subparsers( - help="Action to execute against the files.", - required=True, - ) - delete_subparser = repo_files_subparsers.add_parser( - "delete", - help="Delete files from a repo on the Hub", - ) - delete_subparser.set_defaults(func=lambda args: DeleteFilesSubCommand(args)) - delete_subparser.add_argument( - "patterns", - nargs="+", - type=str, - help="Glob patterns to match files to delete.", - ) - delete_subparser.add_argument( - "--repo-type", - choices=["model", "dataset", "space"], - default="model", - help="Type of the repo to upload to (e.g. `dataset`).", - ) - delete_subparser.add_argument( - "--revision", - type=str, - help=( - "An optional Git revision to push to. It can be a branch name " - "or a PR reference. If revision does not" - " exist and `--create-pr` is not set, a branch will be automatically created." - ), - ) - delete_subparser.add_argument( - "--commit-message", type=str, help="The summary / title / first line of the generated commit." - ) - delete_subparser.add_argument( - "--commit-description", type=str, help="The description of the generated commit." - ) - delete_subparser.add_argument( - "--create-pr", action="store_true", help="Whether to create a new Pull Request for these changes." - ) - repo_files_parser.add_argument( - "--token", - type=str, - help="A User Access Token generated from https://huggingface.co/settings/tokens", - ) - - repo_files_parser.set_defaults(func=RepoFilesCommand) diff --git a/src/huggingface_hub/commands/scan_cache.py b/src/huggingface_hub/commands/scan_cache.py deleted file mode 100644 index 711a5d09cc..0000000000 --- a/src/huggingface_hub/commands/scan_cache.py +++ /dev/null @@ -1,183 +0,0 @@ -# coding=utf-8 -# Copyright 2022-present, the HuggingFace Inc. team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Contains command to scan the HF cache directory. - -Usage: - huggingface-cli scan-cache - huggingface-cli scan-cache -v - huggingface-cli scan-cache -vvv - huggingface-cli scan-cache --dir ~/.cache/huggingface/hub -""" - -import time -from argparse import Namespace, _SubParsersAction -from typing import Optional - -from ..utils import CacheNotFound, HFCacheInfo, scan_cache_dir -from . import BaseHuggingfaceCLICommand -from ._cli_utils import ANSI, show_deprecation_warning, tabulate - - -class ScanCacheCommand(BaseHuggingfaceCLICommand): - @staticmethod - def register_subcommand(parser: _SubParsersAction): - scan_cache_parser = parser.add_parser("scan-cache", help="Scan cache directory.") - - scan_cache_parser.add_argument( - "--dir", - type=str, - default=None, - help="cache directory to scan (optional). Default to the default HuggingFace cache.", - ) - scan_cache_parser.add_argument( - "-v", - "--verbose", - action="count", - default=0, - help="show a more verbose output", - ) - scan_cache_parser.set_defaults(func=ScanCacheCommand) - - def __init__(self, args: Namespace) -> None: - self.verbosity: int = args.verbose - self.cache_dir: Optional[str] = args.dir - - def run(self): - show_deprecation_warning("huggingface-cli scan-cache", "hf cache scan") - - try: - t0 = time.time() - hf_cache_info = scan_cache_dir(self.cache_dir) - t1 = time.time() - except CacheNotFound as exc: - cache_dir = exc.cache_dir - print(f"Cache directory not found: {cache_dir}") - return - - self._print_hf_cache_info_as_table(hf_cache_info) - - print( - f"\nDone in {round(t1 - t0, 1)}s. Scanned {len(hf_cache_info.repos)} repo(s)" - f" for a total of {ANSI.red(hf_cache_info.size_on_disk_str)}." - ) - if len(hf_cache_info.warnings) > 0: - message = f"Got {len(hf_cache_info.warnings)} warning(s) while scanning." - if self.verbosity >= 3: - print(ANSI.gray(message)) - for warning in hf_cache_info.warnings: - print(ANSI.gray(str(warning))) - else: - print(ANSI.gray(message + " Use -vvv to print details.")) - - def _print_hf_cache_info_as_table(self, hf_cache_info: HFCacheInfo) -> None: - print(get_table(hf_cache_info, verbosity=self.verbosity)) - - -def get_table(hf_cache_info: HFCacheInfo, *, verbosity: int = 0) -> str: - """Generate a table from the [`HFCacheInfo`] object. - - Pass `verbosity=0` to get a table with a single row per repo, with columns - "repo_id", "repo_type", "size_on_disk", "nb_files", "last_accessed", "last_modified", "refs", "local_path". - - Pass `verbosity=1` to get a table with a row per repo and revision (thus multiple rows can appear for a single repo), with columns - "repo_id", "repo_type", "revision", "size_on_disk", "nb_files", "last_modified", "refs", "local_path". - - Example: - ```py - >>> from huggingface_hub.utils import scan_cache_dir - >>> from huggingface_hub.commands.scan_cache import get_table - - >>> hf_cache_info = scan_cache_dir() - HFCacheInfo(...) - - >>> print(get_table(hf_cache_info, verbosity=0)) - REPO ID REPO TYPE SIZE ON DISK NB FILES LAST_ACCESSED LAST_MODIFIED REFS LOCAL PATH - --------------------------------------------------- --------- ------------ -------- ------------- ------------- ---- -------------------------------------------------------------------------------------------------- - roberta-base model 2.7M 5 1 day ago 1 week ago main C:\\Users\\admin\\.cache\\huggingface\\hub\\models--roberta-base - suno/bark model 8.8K 1 1 week ago 1 week ago main C:\\Users\\admin\\.cache\\huggingface\\hub\\models--suno--bark - t5-base model 893.8M 4 4 days ago 7 months ago main C:\\Users\\admin\\.cache\\huggingface\\hub\\models--t5-base - t5-large model 3.0G 4 5 weeks ago 5 months ago main C:\\Users\\admin\\.cache\\huggingface\\hub\\models--t5-large - - >>> print(get_table(hf_cache_info, verbosity=1)) - REPO ID REPO TYPE REVISION SIZE ON DISK NB FILES LAST_MODIFIED REFS LOCAL PATH - --------------------------------------------------- --------- ---------------------------------------- ------------ -------- ------------- ---- ----------------------------------------------------------------------------------------------------------------------------------------------------- - roberta-base model e2da8e2f811d1448a5b465c236feacd80ffbac7b 2.7M 5 1 week ago main C:\\Users\\admin\\.cache\\huggingface\\hub\\models--roberta-base\\snapshots\\e2da8e2f811d1448a5b465c236feacd80ffbac7b - suno/bark model 70a8a7d34168586dc5d028fa9666aceade177992 8.8K 1 1 week ago main C:\\Users\\admin\\.cache\\huggingface\\hub\\models--suno--bark\\snapshots\\70a8a7d34168586dc5d028fa9666aceade177992 - t5-base model a9723ea7f1b39c1eae772870f3b547bf6ef7e6c1 893.8M 4 7 months ago main C:\\Users\\admin\\.cache\\huggingface\\hub\\models--t5-base\\snapshots\\a9723ea7f1b39c1eae772870f3b547bf6ef7e6c1 - t5-large model 150ebc2c4b72291e770f58e6057481c8d2ed331a 3.0G 4 5 months ago main C:\\Users\\admin\\.cache\\huggingface\\hub\\models--t5-large\\snapshots\\150ebc2c4b72291e770f58e6057481c8d2ed331a ``` - ``` - - Args: - hf_cache_info ([`HFCacheInfo`]): - The HFCacheInfo object to print. - verbosity (`int`, *optional*): - The verbosity level. Defaults to 0. - - Returns: - `str`: The table as a string. - """ - if verbosity == 0: - return tabulate( - rows=[ - [ - repo.repo_id, - repo.repo_type, - "{:>12}".format(repo.size_on_disk_str), - repo.nb_files, - repo.last_accessed_str, - repo.last_modified_str, - ", ".join(sorted(repo.refs)), - str(repo.repo_path), - ] - for repo in sorted(hf_cache_info.repos, key=lambda repo: repo.repo_path) - ], - headers=[ - "REPO ID", - "REPO TYPE", - "SIZE ON DISK", - "NB FILES", - "LAST_ACCESSED", - "LAST_MODIFIED", - "REFS", - "LOCAL PATH", - ], - ) - else: - return tabulate( - rows=[ - [ - repo.repo_id, - repo.repo_type, - revision.commit_hash, - "{:>12}".format(revision.size_on_disk_str), - revision.nb_files, - revision.last_modified_str, - ", ".join(sorted(revision.refs)), - str(revision.snapshot_path), - ] - for repo in sorted(hf_cache_info.repos, key=lambda repo: repo.repo_path) - for revision in sorted(repo.revisions, key=lambda revision: revision.commit_hash) - ], - headers=[ - "REPO ID", - "REPO TYPE", - "REVISION", - "SIZE ON DISK", - "NB FILES", - "LAST_MODIFIED", - "REFS", - "LOCAL PATH", - ], - ) diff --git a/src/huggingface_hub/commands/tag.py b/src/huggingface_hub/commands/tag.py deleted file mode 100644 index a961791155..0000000000 --- a/src/huggingface_hub/commands/tag.py +++ /dev/null @@ -1,159 +0,0 @@ -# coding=utf-8 -# Copyright 2024-present, the HuggingFace Inc. team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Contains commands to perform tag management with the CLI. - -Usage Examples: - - Create a tag: - $ huggingface-cli tag user/my-model 1.0 --message "First release" - $ huggingface-cli tag user/my-model 1.0 -m "First release" --revision develop - $ huggingface-cli tag user/my-dataset 1.0 -m "First release" --repo-type dataset - $ huggingface-cli tag user/my-space 1.0 - - List all tags: - $ huggingface-cli tag -l user/my-model - $ huggingface-cli tag --list user/my-dataset --repo-type dataset - - Delete a tag: - $ huggingface-cli tag -d user/my-model 1.0 - $ huggingface-cli tag --delete user/my-dataset 1.0 --repo-type dataset - $ huggingface-cli tag -d user/my-space 1.0 -y -""" - -from argparse import Namespace, _SubParsersAction - -from huggingface_hub.commands import BaseHuggingfaceCLICommand -from huggingface_hub.constants import ( - REPO_TYPES, -) -from huggingface_hub.hf_api import HfApi - -from ..errors import HfHubHTTPError, RepositoryNotFoundError, RevisionNotFoundError -from ._cli_utils import ANSI, show_deprecation_warning - - -class TagCommands(BaseHuggingfaceCLICommand): - @staticmethod - def register_subcommand(parser: _SubParsersAction): - tag_parser = parser.add_parser("tag", help="(create, list, delete) tags for a repo in the hub") - - tag_parser.add_argument("repo_id", type=str, help="The ID of the repo to tag (e.g. `username/repo-name`).") - tag_parser.add_argument("tag", nargs="?", type=str, help="The name of the tag for creation or deletion.") - tag_parser.add_argument("-m", "--message", type=str, help="The description of the tag to create.") - tag_parser.add_argument("--revision", type=str, help="The git revision to tag.") - tag_parser.add_argument( - "--token", type=str, help="A User Access Token generated from https://huggingface.co/settings/tokens." - ) - tag_parser.add_argument( - "--repo-type", - choices=["model", "dataset", "space"], - default="model", - help="Set the type of repository (model, dataset, or space).", - ) - tag_parser.add_argument("-y", "--yes", action="store_true", help="Answer Yes to prompts automatically.") - - tag_parser.add_argument("-l", "--list", action="store_true", help="List tags for a repository.") - tag_parser.add_argument("-d", "--delete", action="store_true", help="Delete a tag for a repository.") - - tag_parser.set_defaults(func=lambda args: handle_commands(args)) - - -def handle_commands(args: Namespace): - show_deprecation_warning("huggingface-cli tag", "hf repo tag") - - if args.list: - return TagListCommand(args) - elif args.delete: - return TagDeleteCommand(args) - else: - return TagCreateCommand(args) - - -class TagCommand: - def __init__(self, args: Namespace): - self.args = args - self.api = HfApi(token=self.args.token) - self.repo_id = self.args.repo_id - self.repo_type = self.args.repo_type - if self.repo_type not in REPO_TYPES: - print("Invalid repo --repo-type") - exit(1) - - -class TagCreateCommand(TagCommand): - def run(self): - print(f"You are about to create tag {ANSI.bold(self.args.tag)} on {self.repo_type} {ANSI.bold(self.repo_id)}") - - try: - self.api.create_tag( - repo_id=self.repo_id, - tag=self.args.tag, - tag_message=self.args.message, - revision=self.args.revision, - repo_type=self.repo_type, - ) - except RepositoryNotFoundError: - print(f"{self.repo_type.capitalize()} {ANSI.bold(self.repo_id)} not found.") - exit(1) - except RevisionNotFoundError: - print(f"Revision {ANSI.bold(self.args.revision)} not found.") - exit(1) - except HfHubHTTPError as e: - if e.response.status_code == 409: - print(f"Tag {ANSI.bold(self.args.tag)} already exists on {ANSI.bold(self.repo_id)}") - exit(1) - raise e - - print(f"Tag {ANSI.bold(self.args.tag)} created on {ANSI.bold(self.repo_id)}") - - -class TagListCommand(TagCommand): - def run(self): - try: - refs = self.api.list_repo_refs( - repo_id=self.repo_id, - repo_type=self.repo_type, - ) - except RepositoryNotFoundError: - print(f"{self.repo_type.capitalize()} {ANSI.bold(self.repo_id)} not found.") - exit(1) - except HfHubHTTPError as e: - print(e) - print(ANSI.red(e.response.text)) - exit(1) - if len(refs.tags) == 0: - print("No tags found") - exit(0) - print(f"Tags for {self.repo_type} {ANSI.bold(self.repo_id)}:") - for tag in refs.tags: - print(tag.name) - - -class TagDeleteCommand(TagCommand): - def run(self): - print(f"You are about to delete tag {ANSI.bold(self.args.tag)} on {self.repo_type} {ANSI.bold(self.repo_id)}") - - if not self.args.yes: - choice = input("Proceed? [Y/n] ").lower() - if choice not in ("", "y", "yes"): - print("Abort") - exit() - try: - self.api.delete_tag(repo_id=self.repo_id, tag=self.args.tag, repo_type=self.repo_type) - except RepositoryNotFoundError: - print(f"{self.repo_type.capitalize()} {ANSI.bold(self.repo_id)} not found.") - exit(1) - except RevisionNotFoundError: - print(f"Tag {ANSI.bold(self.args.tag)} not found on {ANSI.bold(self.repo_id)}") - exit(1) - print(f"Tag {ANSI.bold(self.args.tag)} deleted on {ANSI.bold(self.repo_id)}") diff --git a/src/huggingface_hub/commands/upload.py b/src/huggingface_hub/commands/upload.py deleted file mode 100644 index 180f8ef58b..0000000000 --- a/src/huggingface_hub/commands/upload.py +++ /dev/null @@ -1,318 +0,0 @@ -# coding=utf-8 -# Copyright 2023-present, the HuggingFace Inc. team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Contains command to upload a repo or file with the CLI. - -Usage: - # Upload file (implicit) - huggingface-cli upload my-cool-model ./my-cool-model.safetensors - - # Upload file (explicit) - huggingface-cli upload my-cool-model ./my-cool-model.safetensors model.safetensors - - # Upload directory (implicit). If `my-cool-model/` is a directory it will be uploaded, otherwise an exception is raised. - huggingface-cli upload my-cool-model - - # Upload directory (explicit) - huggingface-cli upload my-cool-model ./models/my-cool-model . - - # Upload filtered directory (example: tensorboard logs except for the last run) - huggingface-cli upload my-cool-model ./model/training /logs --include "*.tfevents.*" --exclude "*20230905*" - - # Upload with wildcard - huggingface-cli upload my-cool-model "./model/training/*.safetensors" - - # Upload private dataset - huggingface-cli upload Wauplin/my-cool-dataset ./data . --repo-type=dataset --private - - # Upload with token - huggingface-cli upload Wauplin/my-cool-model --token=hf_**** - - # Sync local Space with Hub (upload new files, delete removed files) - huggingface-cli upload Wauplin/space-example --repo-type=space --exclude="/logs/*" --delete="*" --commit-message="Sync local Space with Hub" - - # Schedule commits every 30 minutes - huggingface-cli upload Wauplin/my-cool-model --every=30 -""" - -import os -import time -import warnings -from argparse import Namespace, _SubParsersAction -from typing import Optional - -from huggingface_hub import logging -from huggingface_hub._commit_scheduler import CommitScheduler -from huggingface_hub.commands import BaseHuggingfaceCLICommand -from huggingface_hub.constants import HF_HUB_ENABLE_HF_TRANSFER -from huggingface_hub.errors import RevisionNotFoundError -from huggingface_hub.hf_api import HfApi -from huggingface_hub.utils import disable_progress_bars, enable_progress_bars -from huggingface_hub.utils._runtime import is_xet_available - -from ._cli_utils import show_deprecation_warning - - -logger = logging.get_logger(__name__) - - -class UploadCommand(BaseHuggingfaceCLICommand): - @staticmethod - def register_subcommand(parser: _SubParsersAction): - upload_parser = parser.add_parser("upload", help="Upload a file or a folder to a repo on the Hub") - upload_parser.add_argument( - "repo_id", type=str, help="The ID of the repo to upload to (e.g. `username/repo-name`)." - ) - upload_parser.add_argument( - "local_path", - nargs="?", - help="Local path to the file or folder to upload. Wildcard patterns are supported. Defaults to current directory.", - ) - upload_parser.add_argument( - "path_in_repo", - nargs="?", - help="Path of the file or folder in the repo. Defaults to the relative path of the file or folder.", - ) - upload_parser.add_argument( - "--repo-type", - choices=["model", "dataset", "space"], - default="model", - help="Type of the repo to upload to (e.g. `dataset`).", - ) - upload_parser.add_argument( - "--revision", - type=str, - help=( - "An optional Git revision to push to. It can be a branch name or a PR reference. If revision does not" - " exist and `--create-pr` is not set, a branch will be automatically created." - ), - ) - upload_parser.add_argument( - "--private", - action="store_true", - help=( - "Whether to create a private repo if repo doesn't exist on the Hub. Ignored if the repo already" - " exists." - ), - ) - upload_parser.add_argument("--include", nargs="*", type=str, help="Glob patterns to match files to upload.") - upload_parser.add_argument( - "--exclude", nargs="*", type=str, help="Glob patterns to exclude from files to upload." - ) - upload_parser.add_argument( - "--delete", - nargs="*", - type=str, - help="Glob patterns for file to be deleted from the repo while committing.", - ) - upload_parser.add_argument( - "--commit-message", type=str, help="The summary / title / first line of the generated commit." - ) - upload_parser.add_argument("--commit-description", type=str, help="The description of the generated commit.") - upload_parser.add_argument( - "--create-pr", action="store_true", help="Whether to upload content as a new Pull Request." - ) - upload_parser.add_argument( - "--every", - type=float, - help="If set, a background job is scheduled to create commits every `every` minutes.", - ) - upload_parser.add_argument( - "--token", type=str, help="A User Access Token generated from https://huggingface.co/settings/tokens" - ) - upload_parser.add_argument( - "--quiet", - action="store_true", - help="If True, progress bars are disabled and only the path to the uploaded files is printed.", - ) - upload_parser.set_defaults(func=UploadCommand) - - def __init__(self, args: Namespace) -> None: - self.repo_id: str = args.repo_id - self.repo_type: Optional[str] = args.repo_type - self.revision: Optional[str] = args.revision - self.private: bool = args.private - - self.include: Optional[list[str]] = args.include - self.exclude: Optional[list[str]] = args.exclude - self.delete: Optional[list[str]] = args.delete - - self.commit_message: Optional[str] = args.commit_message - self.commit_description: Optional[str] = args.commit_description - self.create_pr: bool = args.create_pr - self.api: HfApi = HfApi(token=args.token, library_name="huggingface-cli") - self.quiet: bool = args.quiet # disable warnings and progress bars - - # Check `--every` is valid - if args.every is not None and args.every <= 0: - raise ValueError(f"`every` must be a positive value (got '{args.every}')") - self.every: Optional[float] = args.every - - # Resolve `local_path` and `path_in_repo` - repo_name: str = args.repo_id.split("/")[-1] # e.g. "Wauplin/my-cool-model" => "my-cool-model" - self.local_path: str - self.path_in_repo: str - - if args.local_path is not None and any(c in args.local_path for c in ["*", "?", "["]): - if args.include is not None: - raise ValueError("Cannot set `--include` when passing a `local_path` containing a wildcard.") - if args.path_in_repo is not None and args.path_in_repo != ".": - raise ValueError("Cannot set `path_in_repo` when passing a `local_path` containing a wildcard.") - self.local_path = "." - self.include = args.local_path - self.path_in_repo = "." - elif args.local_path is None and os.path.isfile(repo_name): - # Implicit case 1: user provided only a repo_id which happen to be a local file as well => upload it with same name - self.local_path = repo_name - self.path_in_repo = repo_name - elif args.local_path is None and os.path.isdir(repo_name): - # Implicit case 2: user provided only a repo_id which happen to be a local folder as well => upload it at root - self.local_path = repo_name - self.path_in_repo = "." - elif args.local_path is None: - # Implicit case 3: user provided only a repo_id that does not match a local file or folder - # => the user must explicitly provide a local_path => raise exception - raise ValueError(f"'{repo_name}' is not a local file or folder. Please set `local_path` explicitly.") - elif args.path_in_repo is None and os.path.isfile(args.local_path): - # Explicit local path to file, no path in repo => upload it at root with same name - self.local_path = args.local_path - self.path_in_repo = os.path.basename(args.local_path) - elif args.path_in_repo is None: - # Explicit local path to folder, no path in repo => upload at root - self.local_path = args.local_path - self.path_in_repo = "." - else: - # Finally, if both paths are explicit - self.local_path = args.local_path - self.path_in_repo = args.path_in_repo - - def run(self) -> None: - show_deprecation_warning("huggingface-cli upload", "hf upload") - - if self.quiet: - disable_progress_bars() - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - print(self._upload()) - enable_progress_bars() - else: - logging.set_verbosity_info() - print(self._upload()) - logging.set_verbosity_warning() - - def _upload(self) -> str: - if os.path.isfile(self.local_path): - if self.include is not None and len(self.include) > 0: - warnings.warn("Ignoring `--include` since a single file is uploaded.") - if self.exclude is not None and len(self.exclude) > 0: - warnings.warn("Ignoring `--exclude` since a single file is uploaded.") - if self.delete is not None and len(self.delete) > 0: - warnings.warn("Ignoring `--delete` since a single file is uploaded.") - - if not is_xet_available() and not HF_HUB_ENABLE_HF_TRANSFER: - logger.info( - "Consider using `hf_transfer` for faster uploads. This solution comes with some limitations. See" - " https://huggingface.co/docs/huggingface_hub/hf_transfer for more details." - ) - - # Schedule commits if `every` is set - if self.every is not None: - if os.path.isfile(self.local_path): - # If file => watch entire folder + use allow_patterns - folder_path = os.path.dirname(self.local_path) - path_in_repo = ( - self.path_in_repo[: -len(self.local_path)] # remove filename from path_in_repo - if self.path_in_repo.endswith(self.local_path) - else self.path_in_repo - ) - allow_patterns = [self.local_path] - ignore_patterns = [] - else: - folder_path = self.local_path - path_in_repo = self.path_in_repo - allow_patterns = self.include or [] - ignore_patterns = self.exclude or [] - if self.delete is not None and len(self.delete) > 0: - warnings.warn("Ignoring `--delete` when uploading with scheduled commits.") - - scheduler = CommitScheduler( - folder_path=folder_path, - repo_id=self.repo_id, - repo_type=self.repo_type, - revision=self.revision, - allow_patterns=allow_patterns, - ignore_patterns=ignore_patterns, - path_in_repo=path_in_repo, - private=self.private, - every=self.every, - hf_api=self.api, - ) - print(f"Scheduling commits every {self.every} minutes to {scheduler.repo_id}.") - try: # Block main thread until KeyboardInterrupt - while True: - time.sleep(100) - except KeyboardInterrupt: - scheduler.stop() - return "Stopped scheduled commits." - - # Otherwise, create repo and proceed with the upload - if not os.path.isfile(self.local_path) and not os.path.isdir(self.local_path): - raise FileNotFoundError(f"No such file or directory: '{self.local_path}'.") - repo_id = self.api.create_repo( - repo_id=self.repo_id, - repo_type=self.repo_type, - exist_ok=True, - private=self.private, - space_sdk="gradio" if self.repo_type == "space" else None, - # ^ We don't want it to fail when uploading to a Space => let's set Gradio by default. - # ^ I'd rather not add CLI args to set it explicitly as we already have `huggingface-cli repo create` for that. - ).repo_id - - # Check if branch already exists and if not, create it - if self.revision is not None and not self.create_pr: - try: - self.api.repo_info(repo_id=repo_id, repo_type=self.repo_type, revision=self.revision) - except RevisionNotFoundError: - logger.info(f"Branch '{self.revision}' not found. Creating it...") - self.api.create_branch(repo_id=repo_id, repo_type=self.repo_type, branch=self.revision, exist_ok=True) - # ^ `exist_ok=True` to avoid race concurrency issues - - # File-based upload - if os.path.isfile(self.local_path): - return self.api.upload_file( - path_or_fileobj=self.local_path, - path_in_repo=self.path_in_repo, - repo_id=repo_id, - repo_type=self.repo_type, - revision=self.revision, - commit_message=self.commit_message, - commit_description=self.commit_description, - create_pr=self.create_pr, - ) - - # Folder-based upload - else: - return self.api.upload_folder( - folder_path=self.local_path, - path_in_repo=self.path_in_repo, - repo_id=repo_id, - repo_type=self.repo_type, - revision=self.revision, - commit_message=self.commit_message, - commit_description=self.commit_description, - create_pr=self.create_pr, - allow_patterns=self.include, - ignore_patterns=self.exclude, - delete_patterns=self.delete, - ) diff --git a/src/huggingface_hub/commands/upload_large_folder.py b/src/huggingface_hub/commands/upload_large_folder.py deleted file mode 100644 index b0597868ea..0000000000 --- a/src/huggingface_hub/commands/upload_large_folder.py +++ /dev/null @@ -1,131 +0,0 @@ -# coding=utf-8 -# Copyright 2023-present, the HuggingFace Inc. team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Contains command to upload a large folder with the CLI.""" - -import os -from argparse import Namespace, _SubParsersAction -from typing import Optional - -from huggingface_hub import logging -from huggingface_hub.commands import BaseHuggingfaceCLICommand -from huggingface_hub.hf_api import HfApi -from huggingface_hub.utils import disable_progress_bars - -from ._cli_utils import ANSI, show_deprecation_warning - - -logger = logging.get_logger(__name__) - - -class UploadLargeFolderCommand(BaseHuggingfaceCLICommand): - @staticmethod - def register_subcommand(parser: _SubParsersAction): - subparser = parser.add_parser("upload-large-folder", help="Upload a large folder to a repo on the Hub") - subparser.add_argument( - "repo_id", type=str, help="The ID of the repo to upload to (e.g. `username/repo-name`)." - ) - subparser.add_argument("local_path", type=str, help="Local path to the file or folder to upload.") - subparser.add_argument( - "--repo-type", - choices=["model", "dataset", "space"], - help="Type of the repo to upload to (e.g. `dataset`).", - ) - subparser.add_argument( - "--revision", - type=str, - help=("An optional Git revision to push to. It can be a branch name or a PR reference."), - ) - subparser.add_argument( - "--private", - action="store_true", - help=( - "Whether to create a private repo if repo doesn't exist on the Hub. Ignored if the repo already exists." - ), - ) - subparser.add_argument("--include", nargs="*", type=str, help="Glob patterns to match files to upload.") - subparser.add_argument("--exclude", nargs="*", type=str, help="Glob patterns to exclude from files to upload.") - subparser.add_argument( - "--token", type=str, help="A User Access Token generated from https://huggingface.co/settings/tokens" - ) - subparser.add_argument( - "--num-workers", type=int, help="Number of workers to use to hash, upload and commit files." - ) - subparser.add_argument("--no-report", action="store_true", help="Whether to disable regular status report.") - subparser.add_argument("--no-bars", action="store_true", help="Whether to disable progress bars.") - subparser.set_defaults(func=UploadLargeFolderCommand) - - def __init__(self, args: Namespace) -> None: - self.repo_id: str = args.repo_id - self.local_path: str = args.local_path - self.repo_type: str = args.repo_type - self.revision: Optional[str] = args.revision - self.private: bool = args.private - - self.include: Optional[list[str]] = args.include - self.exclude: Optional[list[str]] = args.exclude - - self.api: HfApi = HfApi(token=args.token, library_name="huggingface-cli") - - self.num_workers: Optional[int] = args.num_workers - self.no_report: bool = args.no_report - self.no_bars: bool = args.no_bars - - if not os.path.isdir(self.local_path): - raise ValueError("Large upload is only supported for folders.") - - def run(self) -> None: - show_deprecation_warning("huggingface-cli upload-large-folder", "hf upload-large-folder") - - logging.set_verbosity_info() - - print( - ANSI.yellow( - "You are about to upload a large folder to the Hub using `huggingface-cli upload-large-folder`. " - "This is a new feature so feedback is very welcome!\n" - "\n" - "A few things to keep in mind:\n" - " - Repository limits still apply: https://huggingface.co/docs/hub/repositories-recommendations\n" - " - Do not start several processes in parallel.\n" - " - You can interrupt and resume the process at any time. " - "The script will pick up where it left off except for partially uploaded files that would have to be entirely reuploaded.\n" - " - Do not upload the same folder to several repositories. If you need to do so, you must delete the `./.cache/huggingface/` folder first.\n" - "\n" - f"Some temporary metadata will be stored under `{self.local_path}/.cache/huggingface`.\n" - " - You must not modify those files manually.\n" - " - You must not delete the `./.cache/huggingface/` folder while a process is running.\n" - " - You can delete the `./.cache/huggingface/` folder to reinitialize the upload state when process is not running. Files will have to be hashed and preuploaded again, except for already committed files.\n" - "\n" - "If the process output is too verbose, you can disable the progress bars with `--no-bars`. " - "You can also entirely disable the status report with `--no-report`.\n" - "\n" - "For more details, run `huggingface-cli upload-large-folder --help` or check the documentation at " - "https://huggingface.co/docs/huggingface_hub/guides/upload#upload-a-large-folder." - ) - ) - - if self.no_bars: - disable_progress_bars() - - self.api.upload_large_folder( - repo_id=self.repo_id, - folder_path=self.local_path, - repo_type=self.repo_type, - revision=self.revision, - private=self.private, - allow_patterns=self.include, - ignore_patterns=self.exclude, - num_workers=self.num_workers, - print_report=not self.no_report, - ) diff --git a/src/huggingface_hub/commands/user.py b/src/huggingface_hub/commands/user.py deleted file mode 100644 index b187876328..0000000000 --- a/src/huggingface_hub/commands/user.py +++ /dev/null @@ -1,207 +0,0 @@ -# Copyright 2020 The HuggingFace Team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Contains commands to authenticate to the Hugging Face Hub and interact with your repositories. - -Usage: - # login and save token locally. - huggingface-cli login --token=hf_*** --add-to-git-credential - - # switch between tokens - huggingface-cli auth switch - - # list all tokens - huggingface-cli auth list - - # logout from a specific token, if no token-name is provided, all tokens will be deleted from your machine. - huggingface-cli logout --token-name=your_token_name - - # find out which huggingface.co account you are logged in as - huggingface-cli whoami -""" - -from argparse import _SubParsersAction -from typing import Optional - -from huggingface_hub.commands import BaseHuggingfaceCLICommand -from huggingface_hub.constants import ENDPOINT -from huggingface_hub.errors import HfHubHTTPError -from huggingface_hub.hf_api import HfApi - -from .._login import auth_list, auth_switch, login, logout -from ..utils import get_stored_tokens, get_token, logging -from ._cli_utils import ANSI, show_deprecation_warning - - -logger = logging.get_logger(__name__) - -try: - from InquirerPy import inquirer - from InquirerPy.base.control import Choice - - _inquirer_py_available = True -except ImportError: - _inquirer_py_available = False - - -class UserCommands(BaseHuggingfaceCLICommand): - @staticmethod - def register_subcommand(parser: _SubParsersAction): - login_parser = parser.add_parser("login", help="Log in using a token from huggingface.co/settings/tokens") - login_parser.add_argument( - "--token", - type=str, - help="Token generated from https://huggingface.co/settings/tokens", - ) - login_parser.add_argument( - "--add-to-git-credential", - action="store_true", - help="Optional: Save token to git credential helper.", - ) - login_parser.set_defaults(func=lambda args: LoginCommand(args)) - whoami_parser = parser.add_parser("whoami", help="Find out which huggingface.co account you are logged in as.") - whoami_parser.set_defaults(func=lambda args: WhoamiCommand(args)) - - logout_parser = parser.add_parser("logout", help="Log out") - logout_parser.add_argument( - "--token-name", - type=str, - help="Optional: Name of the access token to log out from.", - ) - logout_parser.set_defaults(func=lambda args: LogoutCommand(args)) - - auth_parser = parser.add_parser("auth", help="Other authentication related commands") - auth_subparsers = auth_parser.add_subparsers(help="Authentication subcommands") - auth_switch_parser = auth_subparsers.add_parser("switch", help="Switch between access tokens") - auth_switch_parser.add_argument( - "--token-name", - type=str, - help="Optional: Name of the access token to switch to.", - ) - auth_switch_parser.add_argument( - "--add-to-git-credential", - action="store_true", - help="Optional: Save token to git credential helper.", - ) - auth_switch_parser.set_defaults(func=lambda args: AuthSwitchCommand(args)) - auth_list_parser = auth_subparsers.add_parser("list", help="List all stored access tokens") - auth_list_parser.set_defaults(func=lambda args: AuthListCommand(args)) - - -class BaseUserCommand: - def __init__(self, args): - self.args = args - self._api = HfApi() - - -class LoginCommand(BaseUserCommand): - def run(self): - show_deprecation_warning("huggingface-cli login", "hf auth login") - - logging.set_verbosity_info() - login( - token=self.args.token, - add_to_git_credential=self.args.add_to_git_credential, - ) - - -class LogoutCommand(BaseUserCommand): - def run(self): - show_deprecation_warning("huggingface-cli logout", "hf auth logout") - - logging.set_verbosity_info() - logout(token_name=self.args.token_name) - - -class AuthSwitchCommand(BaseUserCommand): - def run(self): - show_deprecation_warning("huggingface-cli auth switch", "hf auth switch") - - logging.set_verbosity_info() - token_name = self.args.token_name - if token_name is None: - token_name = self._select_token_name() - - if token_name is None: - print("No token name provided. Aborting.") - exit() - auth_switch(token_name, add_to_git_credential=self.args.add_to_git_credential) - - def _select_token_name(self) -> Optional[str]: - token_names = list(get_stored_tokens().keys()) - - if not token_names: - logger.error("No stored tokens found. Please login first.") - return None - - if _inquirer_py_available: - return self._select_token_name_tui(token_names) - # if inquirer is not available, use a simpler terminal UI - print("Available stored tokens:") - for i, token_name in enumerate(token_names, 1): - print(f"{i}. {token_name}") - while True: - try: - choice = input("Enter the number of the token to switch to (or 'q' to quit): ") - if choice.lower() == "q": - return None - index = int(choice) - 1 - if 0 <= index < len(token_names): - return token_names[index] - else: - print("Invalid selection. Please try again.") - except ValueError: - print("Invalid input. Please enter a number or 'q' to quit.") - - def _select_token_name_tui(self, token_names: list[str]) -> Optional[str]: - choices = [Choice(token_name, name=token_name) for token_name in token_names] - try: - return inquirer.select( - message="Select a token to switch to:", - choices=choices, - default=None, - ).execute() - except KeyboardInterrupt: - logger.info("Token selection cancelled.") - return None - - -class AuthListCommand(BaseUserCommand): - def run(self): - show_deprecation_warning("huggingface-cli auth list", "hf auth list") - - logging.set_verbosity_info() - auth_list() - - -class WhoamiCommand(BaseUserCommand): - def run(self): - show_deprecation_warning("huggingface-cli whoami", "hf auth whoami") - - token = get_token() - if token is None: - print("Not logged in") - exit() - try: - info = self._api.whoami(token) - print(info["name"]) - orgs = [org["name"] for org in info["orgs"]] - if orgs: - print(ANSI.bold("orgs: "), ",".join(orgs)) - - if ENDPOINT != "https://huggingface.co": - print(f"Authenticated through private endpoint: {ENDPOINT}") - except HfHubHTTPError as e: - print(e) - print(ANSI.red(e.response.text)) - exit(1) diff --git a/src/huggingface_hub/commands/version.py b/src/huggingface_hub/commands/version.py deleted file mode 100644 index 10d341bcdb..0000000000 --- a/src/huggingface_hub/commands/version.py +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright 2022 The HuggingFace Team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Contains command to print information about the version. - -Usage: - huggingface-cli version -""" - -from argparse import _SubParsersAction - -from huggingface_hub import __version__ - -from . import BaseHuggingfaceCLICommand -from ._cli_utils import show_deprecation_warning - - -class VersionCommand(BaseHuggingfaceCLICommand): - def __init__(self, args): - self.args = args - - @staticmethod - def register_subcommand(parser: _SubParsersAction): - version_parser = parser.add_parser("version", help="Print information about the huggingface-cli version.") - version_parser.set_defaults(func=VersionCommand) - - def run(self) -> None: - show_deprecation_warning("huggingface-cli version", "hf version") - - print(f"huggingface_hub version: {__version__}") diff --git a/src/huggingface_hub/utils/_cache_manager.py b/src/huggingface_hub/utils/_cache_manager.py index 8db7b2f0fc..2d927cf278 100644 --- a/src/huggingface_hub/utils/_cache_manager.py +++ b/src/huggingface_hub/utils/_cache_manager.py @@ -24,7 +24,7 @@ from huggingface_hub.errors import CacheNotFound, CorruptedCacheException -from ..commands._cli_utils import tabulate +from ..cli._cli_utils import tabulate from ..constants import HF_HUB_CACHE from . import logging diff --git a/tests/test_auth_cli.py b/tests/test_auth_cli.py deleted file mode 100644 index 5d4dd96111..0000000000 --- a/tests/test_auth_cli.py +++ /dev/null @@ -1,158 +0,0 @@ -import logging -import os -import tempfile -from unittest.mock import patch - -import pytest -from pytest import CaptureFixture, LogCaptureFixture - -from huggingface_hub import constants -from huggingface_hub.commands.user import AuthListCommand, AuthSwitchCommand, LoginCommand, LogoutCommand - -from .testing_constants import ENDPOINT_STAGING -from .testing_utils import assert_in_logs - - -# fixtures & constants - -MOCK_TOKEN = "hf_1234" - - -@pytest.fixture(autouse=True) -def use_tmp_file_paths(): - """ - Fixture to temporarily override HF_TOKEN_PATH, HF_STORED_TOKENS_PATH, and ENDPOINT. - """ - with tempfile.TemporaryDirectory() as tmp_hf_home: - hf_token_path = os.path.join(tmp_hf_home, "token") - hf_stored_tokens_path = os.path.join(tmp_hf_home, "stored_tokens") - with patch.multiple( - constants, - HF_TOKEN_PATH=hf_token_path, - HF_STORED_TOKENS_PATH=hf_stored_tokens_path, - ENDPOINT=ENDPOINT_STAGING, - ): - yield - - -@pytest.fixture -def mock_whoami_api_call(): - MOCK_WHOAMI_RESPONSE = { - "auth": { - "accessToken": { - "displayName": "test_token", - "role": "write", - "createdAt": "2024-01-01T00:00:00.000Z", - } - } - } - with patch("huggingface_hub.hf_api.whoami", return_value=MOCK_WHOAMI_RESPONSE): - yield - - -@pytest.fixture -def mock_stored_tokens(): - """Mock stored tokens.""" - stored_tokens = { - "token1": "hf_1234", - "token2": "hf_5678", - "active_token": "hf_9012", - } - with patch("huggingface_hub._login.get_stored_tokens", return_value=stored_tokens): - with patch("huggingface_hub.utils._auth.get_stored_tokens", return_value=stored_tokens): - yield stored_tokens - - -def test_login_command_basic(mock_whoami_api_call, caplog: LogCaptureFixture): - """Test basic login command execution.""" - caplog.set_level(logging.INFO) - - args = type("Args", (), {"token": MOCK_TOKEN, "add_to_git_credential": False})() - cmd = LoginCommand(args) - cmd.run() - - assert_in_logs(caplog, "Login successful") - assert_in_logs(caplog, "Token is valid") - assert_in_logs(caplog, "The current active token is: `test_token`") - - -def test_login_command_with_git(mock_whoami_api_call, caplog: LogCaptureFixture): - """Test login command with git credential option.""" - caplog.set_level(logging.INFO) - - args = type("Args", (), {"token": MOCK_TOKEN, "add_to_git_credential": True})() - cmd = LoginCommand(args) - - with patch("huggingface_hub._login._is_git_credential_helper_configured", return_value=True): - with patch("huggingface_hub.utils.set_git_credential"): - cmd.run() - - assert_in_logs(caplog, "Login successful") - assert_in_logs(caplog, "Your token has been saved in your configured git credential helpers") - - -def test_logout_specific_token(mock_stored_tokens, caplog: LogCaptureFixture): - """Test logout command for a specific token.""" - caplog.set_level(logging.INFO) - - args = type("Args", (), {"token_name": "token1"})() - cmd = LogoutCommand(args) - cmd.run() - - assert_in_logs(caplog, "Successfully logged out from access token: token1") - - -def test_logout_active_token(mock_stored_tokens, caplog: LogCaptureFixture): - """Test logout command for active token.""" - caplog.set_level(logging.INFO) - - with patch("huggingface_hub._login._get_token_from_file", return_value="hf_9012"): - args = type("Args", (), {"token_name": "active_token"})() - cmd = LogoutCommand(args) - cmd.run() - - assert_in_logs(caplog, "Successfully logged out from access token: active_token") - assert_in_logs(caplog, "Active token 'active_token' has been deleted") - - -def test_logout_all_tokens(mock_stored_tokens, caplog: LogCaptureFixture): - """Test logout command for all tokens.""" - caplog.set_level(logging.INFO) - - args = type("Args", (), {"token_name": None})() - cmd = LogoutCommand(args) - cmd.run() - - assert_in_logs(caplog, "Successfully logged out from all access tokens") - - -def test_switch_token(mock_stored_tokens, caplog: LogCaptureFixture): - """Test switching between tokens.""" - caplog.set_level(logging.INFO) - - args = type("Args", (), {"token_name": "token1", "add_to_git_credential": False})() - cmd = AuthSwitchCommand(args) - cmd.run() - - assert_in_logs(caplog, "The current active token is: token1") - - -def test_switch_nonexistent_token(mock_stored_tokens): - """Test switching to a non-existent token.""" - args = type("Args", (), {"token_name": "nonexistent", "add_to_git_credential": False})() - cmd = AuthSwitchCommand(args) - - with pytest.raises(ValueError, match="Access token nonexistent not found"): - cmd.run() - - -def test_list_tokens(mock_stored_tokens, capsys: CaptureFixture): - """Test listing tokens command.""" - args = type("Args", (), {})() - cmd = AuthListCommand(args) - cmd.run() - - captured = capsys.readouterr() - assert "token1" in captured.out - assert "hf_****1234" in captured.out - assert "token2" in captured.out diff --git a/tests/test_command_delete_cache.py b/tests/test_command_delete_cache.py index 1d82b50ad8..49041ea6ec 100644 --- a/tests/test_command_delete_cache.py +++ b/tests/test_command_delete_cache.py @@ -7,9 +7,8 @@ from InquirerPy.base.control import Choice from InquirerPy.separator import Separator -from huggingface_hub.commands.delete_cache import ( +from huggingface_hub.cli.cache import ( _CANCEL_DELETION_STR, - DeleteCacheCommand, _ask_for_confirmation_no_tui, _get_expectations_str, _get_tui_choices_from_scan, @@ -18,17 +17,15 @@ ) from huggingface_hub.utils import SoftTemporaryDirectory, capture_output -from .testing_utils import handle_injection - class TestDeleteCacheHelpers(unittest.TestCase): def test_get_tui_choices_from_scan_empty(self) -> None: choices = _get_tui_choices_from_scan(repos={}, preselected=[], sort_by=None) - self.assertEqual(len(choices), 1) - self.assertIsInstance(choices[0], Choice) - self.assertEqual(choices[0].value, _CANCEL_DELETION_STR) - self.assertTrue(len(choices[0].name) != 0) # Something displayed to the user - self.assertFalse(choices[0].enabled) + assert len(choices) == 1 + assert isinstance(choices[0], Choice) + assert choices[0].value == _CANCEL_DELETION_STR + assert len(choices[0].name) != 0 # Something displayed to the user + assert not choices[0].enabled def test_get_tui_choices_from_scan_with_preselection(self) -> None: choices = _get_tui_choices_from_scan( @@ -40,76 +37,72 @@ def test_get_tui_choices_from_scan_with_preselection(self) -> None: ], sort_by=None, # Don't sort to maintain original order ) - self.assertEqual(len(choices), 8) + assert len(choices) == 8 # Item to cancel everything - self.assertIsInstance(choices[0], Choice) - self.assertEqual(choices[0].value, _CANCEL_DELETION_STR) - self.assertTrue(len(choices[0].name) != 0) - self.assertFalse(choices[0].enabled) + assert isinstance(choices[0], Choice) + assert choices[0].value == _CANCEL_DELETION_STR + assert len(choices[0].name) != 0 + assert not choices[0].enabled # Dataset repo separator - self.assertIsInstance(choices[1], Separator) - self.assertEqual(choices[1]._line, "\nDataset dummy_dataset (8M, used 2 weeks ago)") + assert isinstance(choices[1], Separator) + assert choices[1]._line == "\nDataset dummy_dataset (8M, used 2 weeks ago)" # Only revision of `dummy_dataset` - self.assertIsInstance(choices[2], Choice) - self.assertEqual(choices[2].value, "dataset_revision_hash_id") - self.assertEqual( - choices[2].name, - # truncated hash id + detached + last modified - "dataset_: (detached) # modified 1 day ago", - ) - self.assertTrue(choices[2].enabled) # preselected + assert isinstance(choices[2], Choice) + assert choices[2].value == "dataset_revision_hash_id" + assert choices[2].name == "dataset_: (detached) # modified 1 day ago" + assert choices[2].enabled # preselected # Model `dummy_model` separator - self.assertIsInstance(choices[3], Separator) - self.assertEqual(choices[3]._line, "\nModel dummy_model (1.4K, used 2 years ago)") + assert isinstance(choices[3], Separator) + assert choices[3]._line == "\nModel dummy_model (1.4K, used 2 years ago)" # Recent revision of `dummy_model` (appears first due to sorting by last_modified) - self.assertIsInstance(choices[4], Choice) - self.assertEqual(choices[4].value, "recent_hash_id") - self.assertEqual(choices[4].name, "recent_h: main # modified 2 years ago") - self.assertFalse(choices[4].enabled) + assert isinstance(choices[4], Choice) + assert choices[4].value == "recent_hash_id" + assert choices[4].name == "recent_h: main # modified 2 years ago" + assert not choices[4].enabled # Oldest revision of `dummy_model` - self.assertIsInstance(choices[5], Choice) - self.assertEqual(choices[5].value, "older_hash_id") - self.assertEqual(choices[5].name, "older_ha: (detached) # modified 3 years ago") - self.assertTrue(choices[5].enabled) # preselected + assert isinstance(choices[5], Choice) + assert choices[5].value == "older_hash_id" + assert choices[5].name == "older_ha: (detached) # modified 3 years ago" + assert choices[5].enabled # preselected # Model `gpt2` separator - self.assertIsInstance(choices[6], Separator) - self.assertEqual(choices[6]._line, "\nModel gpt2 (3.6G, used 2 hours ago)") + assert isinstance(choices[6], Separator) + assert choices[6]._line == "\nModel gpt2 (3.6G, used 2 hours ago)" # Only revision of `gpt2` - self.assertIsInstance(choices[7], Choice) - self.assertEqual(choices[7].value, "abcdef123456789") - self.assertEqual(choices[7].name, "abcdef12: main, refs/pr/1 # modified 2 years ago") - self.assertFalse(choices[7].enabled) + assert isinstance(choices[7], Choice) + assert choices[7].value == "abcdef123456789" + assert choices[7].name == "abcdef12: main, refs/pr/1 # modified 2 years ago" + assert not choices[7].enabled def test_get_tui_choices_from_scan_with_sort_size(self) -> None: """Test sorting by size.""" choices = _get_tui_choices_from_scan(repos=_get_cache_mock().repos, preselected=[], sort_by="size") # Verify repo order: gpt2 (3.6G) -> dummy_dataset (8M) -> dummy_model (1.4K) - self.assertIsInstance(choices[1], Separator) - self.assertIn("gpt2", choices[1]._line) + assert isinstance(choices[1], Separator) + assert "gpt2" in choices[1]._line - self.assertIsInstance(choices[3], Separator) - self.assertIn("dummy_dataset", choices[3]._line) + assert isinstance(choices[3], Separator) + assert "dummy_dataset" in choices[3]._line - self.assertIsInstance(choices[5], Separator) - self.assertIn("dummy_model", choices[5]._line) + assert isinstance(choices[5], Separator) + assert "dummy_model" in choices[5]._line def test_get_expectations_str_on_no_deletion_item(self) -> None: """Test `_get_instructions` when `_CANCEL_DELETION_STR` is passed.""" - self.assertEqual( + assert ( _get_expectations_str( hf_cache_info=Mock(), selected_hashes=["hash_1", _CANCEL_DELETION_STR, "hash_2"], - ), - "Nothing will be deleted.", + ) + == "Nothing will be deleted." ) def test_get_expectations_str_with_selection(self) -> None: @@ -120,12 +113,12 @@ def test_get_expectations_str_with_selection(self) -> None: cache_mock = Mock() cache_mock.delete_revisions.return_value = strategy_mock - self.assertEqual( + assert ( _get_expectations_str( hf_cache_info=cache_mock, selected_hashes=["hash_1", "hash_2"], - ), - "2 revisions selected counting for 5.1M.", + ) + == "2 revisions selected counting for 5.1M." ) cache_mock.delete_revisions.assert_called_once_with("hash_1", "hash_2") @@ -154,19 +147,16 @@ def test_read_manual_review_tmp_file(self) -> None: # Only non-commented lines are returned # Order is kept and lines are not de-duplicated - self.assertListEqual( - _read_manual_review_tmp_file(tmp_path), - [ - "a_revision_hash", - "a_revision_hash_with_a_comment", - "a_revision_hash_after_spaces", - "a_revision_hash_with_a_comment_after_spaces", - "a_revision_hash", - ], - ) - - @patch("huggingface_hub.commands.delete_cache.input") - @patch("huggingface_hub.commands.delete_cache.mkstemp") + assert _read_manual_review_tmp_file(tmp_path) == [ + "a_revision_hash", + "a_revision_hash_with_a_comment", + "a_revision_hash_after_spaces", + "a_revision_hash_with_a_comment_after_spaces", + "a_revision_hash", + ] + + @patch("huggingface_hub.cli.cache.input") + @patch("huggingface_hub.cli.cache.mkstemp") def test_manual_review_no_tui(self, mock_mkstemp: Mock, mock_input: Mock) -> None: # Mock file creation so that we know the file location in test fd, tmp_path = mkstemp() @@ -183,16 +173,16 @@ def _input_answers(): self.assertTrue(content.startswith("# INSTRUCTIONS")) # older_hash_id is not commented - self.assertIn("\n older_hash_id # Refs: (detached)", content) + self.assertIn("\n older_hash_id # Refs: (detached)", content) # same for abcdef123456789 - self.assertIn("\n abcdef123456789 # Refs: main, refs/pr/1", content) + self.assertIn("\n abcdef123456789 # Refs: main, refs/pr/1", content) # dataset revision is not preselected - self.assertIn("# dataset_revision_hash_id", content) + self.assertIn("# dataset_revision_hash_id", content) # same for recent_hash_id - self.assertIn("# recent_hash_id", content) + self.assertIn("# recent_hash_id", content) # Select dataset revision - content = content.replace("# dataset_revision_hash_id", "dataset_revision_hash_id") + content = content.replace("# dataset_revision_hash_id", "dataset_revision_hash_id") # Deselect abcdef123456789 content = content.replace("abcdef123456789", "# abcdef123456789") with open(tmp_path, "w") as f: @@ -206,49 +196,48 @@ def _input_answers(): # Run manual review with capture_output() as output: selected_hashes = _manual_review_no_tui( - hf_cache_info=cache_mock, preselected=["abcdef123456789", "older_hash_id"], sort_by=None + hf_cache_info=cache_mock, + preselected=["abcdef123456789", "older_hash_id"], + sort_by=None, ) # Tmp file has been created but is now deleted mock_mkstemp.assert_called_once_with(suffix=".txt") - self.assertFalse(os.path.isfile(tmp_path)) # now deleted + assert not os.path.isfile(tmp_path) # now deleted # User changed the selection - self.assertListEqual(selected_hashes, ["dataset_revision_hash_id", "older_hash_id"]) + assert selected_hashes == ["dataset_revision_hash_id", "older_hash_id"] # Check printed instructions printed = output.getvalue() - self.assertTrue(printed.startswith("TUI is disabled. In order to")) # ... - self.assertIn(tmp_path, printed) + assert printed.startswith("TUI is disabled. In order to") + assert str(tmp_path) in printed # Check input called twice - self.assertEqual(mock_input.call_count, 2) + assert mock_input.call_count == 2 - @patch("huggingface_hub.commands.delete_cache.input") + @patch("huggingface_hub.cli.cache.input") def test_ask_for_confirmation_no_tui(self, mock_input: Mock) -> None: """Test `_ask_for_confirmation_no_tui`.""" # Answer yes mock_input.side_effect = ("y",) value = _ask_for_confirmation_no_tui("custom message 1", default=True) mock_input.assert_called_with("custom message 1 (Y/n) ") - self.assertTrue(value) + assert value # Answer no mock_input.side_effect = ("NO",) value = _ask_for_confirmation_no_tui("custom message 2", default=True) mock_input.assert_called_with("custom message 2 (Y/n) ") - self.assertFalse(value) + assert not value # Answer invalid, then default mock_input.side_effect = ("foo", "") with capture_output() as output: value = _ask_for_confirmation_no_tui("custom message 3", default=False) mock_input.assert_called_with("custom message 3 (y/N) ") - self.assertFalse(value) - self.assertEqual( - output.getvalue(), - "Invalid input. Must be one of ('y', 'yes', '1', 'n', 'no', '0', '')\n", - ) + assert not value + assert output.getvalue() == "Invalid input. Must be one of ('y', 'yes', '1', 'n', 'no', '0', '')\n" def test_get_tui_choices_from_scan_with_different_sorts(self) -> None: """Test different sorting modes.""" @@ -257,185 +246,42 @@ def test_get_tui_choices_from_scan_with_different_sorts(self) -> None: # Test size sorting (largest first) - order: gpt2 (3.6G) -> dummy_dataset (8M) -> dummy_model (1.4K) size_choices = _get_tui_choices_from_scan(cache_mock.repos, [], sort_by="size") # Separators at positions 1, 3, 5 - self.assertIsInstance(size_choices[1], Separator) - self.assertIn("gpt2", size_choices[1]._line) - self.assertIsInstance(size_choices[3], Separator) - self.assertIn("dummy_dataset", size_choices[3]._line) - self.assertIsInstance(size_choices[5], Separator) - self.assertIn("dummy_model", size_choices[5]._line) + assert isinstance(size_choices[1], Separator) + assert "gpt2" in size_choices[1]._line + assert isinstance(size_choices[3], Separator) + assert "dummy_dataset" in size_choices[3]._line + assert isinstance(size_choices[5], Separator) + assert "dummy_model" in size_choices[5]._line # Test alphabetical sorting - order: dummy_dataset -> dummy_model -> gpt2 alpha_choices = _get_tui_choices_from_scan(cache_mock.repos, [], sort_by="alphabetical") # Separators at positions 1, 3, 6 (dummy_model has 2 revisions) - self.assertIsInstance(alpha_choices[1], Separator) - self.assertIn("dummy_dataset", alpha_choices[1]._line) - self.assertIsInstance(alpha_choices[3], Separator) - self.assertIn("dummy_model", alpha_choices[3]._line) - self.assertIsInstance(alpha_choices[6], Separator) - self.assertIn("gpt2", alpha_choices[6]._line) + assert isinstance(alpha_choices[1], Separator) + assert "dummy_dataset" in alpha_choices[1]._line + assert isinstance(alpha_choices[3], Separator) + assert "dummy_model" in alpha_choices[3]._line + assert isinstance(alpha_choices[6], Separator) + assert "gpt2" in alpha_choices[6]._line # Test lastUpdated sorting - order: dummy_dataset (1 day) -> gpt2 (2 years) -> dummy_model (3 years) updated_choices = _get_tui_choices_from_scan(cache_mock.repos, [], sort_by="lastUpdated") # Separators at positions 1, 3, 5 - self.assertIsInstance(updated_choices[1], Separator) - self.assertIn("dummy_dataset", updated_choices[1]._line) - self.assertIsInstance(updated_choices[3], Separator) - self.assertIn("gpt2", updated_choices[3]._line) - self.assertIsInstance(updated_choices[5], Separator) - self.assertIn("dummy_model", updated_choices[5]._line) + assert isinstance(updated_choices[1], Separator) + assert "dummy_dataset" in updated_choices[1]._line + assert isinstance(updated_choices[3], Separator) + assert "gpt2" in updated_choices[3]._line + assert isinstance(updated_choices[5], Separator) + assert "dummy_model" in updated_choices[5]._line # Test lastUsed sorting - order: gpt2 (2h) -> dummy_dataset (2w) -> dummy_model (2y) used_choices = _get_tui_choices_from_scan(cache_mock.repos, [], sort_by="lastUsed") # Separators at positions 1, 3, 5 - self.assertIsInstance(used_choices[1], Separator) - self.assertIn("gpt2", used_choices[1]._line) - self.assertIsInstance(used_choices[3], Separator) - self.assertIn("dummy_dataset", used_choices[3]._line) - self.assertIsInstance(used_choices[5], Separator) - self.assertIn("dummy_model", used_choices[5]._line) - - -@patch("huggingface_hub.commands.delete_cache._ask_for_confirmation_no_tui") -@patch("huggingface_hub.commands.delete_cache._get_expectations_str") -@patch("huggingface_hub.commands.delete_cache.inquirer.confirm") -@patch("huggingface_hub.commands.delete_cache._manual_review_tui") -@patch("huggingface_hub.commands.delete_cache._manual_review_no_tui") -@patch("huggingface_hub.commands.delete_cache.scan_cache_dir") -@handle_injection -class TestMockedDeleteCacheCommand(unittest.TestCase): - """Test case with a patched `DeleteCacheCommand` to test `.run()` without testing - the manual review. - """ - - args: Mock - command: DeleteCacheCommand - - def setUp(self) -> None: - self.args = Mock() - self.args.sort = None - self.command = DeleteCacheCommand(self.args) - - def test_run_and_delete_with_tui( - self, - mock_scan_cache_dir: Mock, - mock__manual_review_tui: Mock, - mock__get_expectations_str: Mock, - mock_confirm: Mock, - ) -> None: - """Test command run with a mocked manual review step.""" - # Mock return values - mock__manual_review_tui.return_value = ["hash_1", "hash_2"] - mock__get_expectations_str.return_value = "Will delete A and B." - mock_confirm.return_value.execute.return_value = True - mock_scan_cache_dir.return_value = _get_cache_mock() - - # Run - self.command.disable_tui = False - with capture_output() as output: - self.command.run() - - # Step 1: scan - mock_scan_cache_dir.assert_called_once_with(self.args.dir) - cache_mock = mock_scan_cache_dir.return_value - - # Step 2: manual review - mock__manual_review_tui.assert_called_once_with(cache_mock, preselected=[], sort_by=None) - - # Step 3: ask confirmation - mock__get_expectations_str.assert_called_once_with(cache_mock, ["hash_1", "hash_2"]) - mock_confirm.assert_called_once_with("Will delete A and B. Confirm deletion ?", default=True) - mock_confirm().execute.assert_called_once_with() - - # Step 4: delete - cache_mock.delete_revisions.assert_called_once_with("hash_1", "hash_2") - strategy_mock = cache_mock.delete_revisions.return_value - strategy_mock.execute.assert_called_once_with() - - # Check output - assert "Start deletion.\nDone. Deleted 0 repo(s) and 0 revision(s) for a total of 5.1M.\n" in output.getvalue() - - def test_run_nothing_selected_with_tui(self, mock__manual_review_tui: Mock) -> None: - """Test command run but nothing is selected in manual review.""" - # Mock return value - mock__manual_review_tui.return_value = [] - - # Run - self.command.disable_tui = False - with capture_output() as output: - self.command.run() - - # Check output - assert "Deletion is cancelled. Do nothing.\n" in output.getvalue() - - def test_run_stuff_selected_but_cancel_item_as_well_with_tui(self, mock__manual_review_tui: Mock) -> None: - """Test command run when some are selected but "cancel item" as well.""" - # Mock return value - mock__manual_review_tui.return_value = [ - "hash_1", - "hash_2", - _CANCEL_DELETION_STR, - ] - - # Run - self.command.disable_tui = False - with capture_output() as output: - self.command.run() - - # Check output - assert "Deletion is cancelled. Do nothing.\n" in output.getvalue() - - def test_run_and_delete_no_tui( - self, - mock_scan_cache_dir: Mock, - mock__manual_review_no_tui: Mock, - mock__get_expectations_str: Mock, - mock__ask_for_confirmation_no_tui: Mock, - ) -> None: - """Test command run with a mocked manual review step.""" - # Mock return values - mock__manual_review_no_tui.return_value = ["hash_1", "hash_2"] - mock__get_expectations_str.return_value = "Will delete A and B." - mock__ask_for_confirmation_no_tui.return_value.return_value = True - mock_scan_cache_dir.return_value = _get_cache_mock() - - # Run - self.command.disable_tui = True - with capture_output() as output: - self.command.run() - - # Step 1: scan - mock_scan_cache_dir.assert_called_once_with(self.args.dir) - cache_mock = mock_scan_cache_dir.return_value - - # Step 2: manual review - mock__manual_review_no_tui.assert_called_once_with(cache_mock, preselected=[], sort_by=None) - - # Step 3: ask confirmation - mock__get_expectations_str.assert_called_once_with(cache_mock, ["hash_1", "hash_2"]) - mock__ask_for_confirmation_no_tui.assert_called_once_with("Will delete A and B. Confirm deletion ?") - - # Step 4: delete - cache_mock.delete_revisions.assert_called_once_with("hash_1", "hash_2") - strategy_mock = cache_mock.delete_revisions.return_value - strategy_mock.execute.assert_called_once_with() - - # Check output - assert "Start deletion.\nDone. Deleted 0 repo(s) and 0 revision(s) for a total of 5.1M.\n" in output.getvalue() - - def test_run_with_sorting(self): - """Test command run with sorting enabled.""" - self.args.sort = "size" - self.command = DeleteCacheCommand(self.args) - - mock_scan_cache_dir = Mock() - mock_scan_cache_dir.return_value = _get_cache_mock() - - with patch("huggingface_hub.commands.delete_cache.scan_cache_dir", mock_scan_cache_dir): - with patch("huggingface_hub.commands.delete_cache._manual_review_tui") as mock_review: - self.command.disable_tui = False - self.command.run() - - mock_review.assert_called_once_with(mock_scan_cache_dir.return_value, preselected=[], sort_by="size") + assert isinstance(used_choices[1], Separator) + assert "gpt2" in used_choices[1]._line + assert isinstance(used_choices[3], Separator) + assert "dummy_dataset" in used_choices[3]._line + assert isinstance(used_choices[5], Separator) + assert "dummy_model" in used_choices[5]._line def _get_cache_mock() -> Mock: diff --git a/tests/test_utils_cache.py b/tests/test_utils_cache.py index e3b29262fd..e7d5d350a3 100644 --- a/tests/test_utils_cache.py +++ b/tests/test_utils_cache.py @@ -9,7 +9,7 @@ import pytest from huggingface_hub._snapshot_download import snapshot_download -from huggingface_hub.commands.scan_cache import ScanCacheCommand +from huggingface_hub.cli.cache import cache_scan from huggingface_hub.utils import DeleteCacheStrategy, HFCacheInfo, _format_size, capture_output, scan_cache_dir from huggingface_hub.utils._cache_manager import CacheNotFound, _format_timesince, _try_delete_path @@ -236,12 +236,8 @@ def test_cli_scan_cache_quiet(self) -> None: End-to-end test just to see if output is in expected format. """ - args = Mock() - args.verbose = 0 - args.dir = self.cache_dir - with capture_output() as output: - ScanCacheCommand(args).run() + cache_scan(dir=self.cache_dir, verbose=0) expected_output = f""" REPO ID REPO TYPE SIZE ON DISK NB FILES LAST_ACCESSED LAST_MODIFIED REFS LOCAL PATH @@ -263,12 +259,8 @@ def test_cli_scan_cache_verbose(self) -> None: End-to-end test just to see if output is in expected format. """ - args = Mock() - args.verbose = 1 - args.dir = self.cache_dir - with capture_output() as output: - ScanCacheCommand(args).run() + cache_scan(dir=self.cache_dir, verbose=1) expected_output = f""" REPO ID REPO TYPE REVISION SIZE ON DISK NB FILES LAST_MODIFIED REFS LOCAL PATH @@ -294,12 +286,8 @@ def test_cli_scan_missing_cache(self) -> None: tmp_dir = tempfile.mkdtemp() os.rmdir(tmp_dir) - args = Mock() - args.verbose = 0 - args.dir = tmp_dir - with capture_output() as output: - ScanCacheCommand(args).run() + cache_scan(dir=tmp_dir, verbose=0) expected_output = f""" Cache directory not found: {Path(tmp_dir).resolve()} diff --git a/tests/test_utils_cli.py b/tests/test_utils_cli.py index e1d68118da..5e6cdedeab 100644 --- a/tests/test_utils_cli.py +++ b/tests/test_utils_cli.py @@ -2,7 +2,7 @@ import unittest from unittest import mock -from huggingface_hub.commands._cli_utils import ANSI, tabulate +from huggingface_hub.cli._cli_utils import ANSI, tabulate class TestCLIUtils(unittest.TestCase):