diff --git a/.github/tools/fix_file_headers.py b/.github/tools/fix_file_headers.py new file mode 100755 index 000000000..69ff7970b --- /dev/null +++ b/.github/tools/fix_file_headers.py @@ -0,0 +1,966 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +"""A script to check and enforce standardized license and authorship headers. + +Location: ./.github/tools/fix_file_headers.py +Copyright 2025 +SPDX-License-Identifier: Apache-2.0 +Authors: Arnav Bhattacharya, Mihai Criveti + +This script scans Python files to ensure they contain a standard header +with copyright, license, and author information. By default, it runs in +check mode (dry run) and requires explicit flags to modify files. + +Operating modes: +- Check (default): Reports files with missing or incorrect headers without modifying +- Fix: Modifies headers for specific files/directories (requires --fix and --path) +- Fix-All: Automatically corrects headers of all python files (requires --fix-all) +- Interactive: Prompts for confirmation before fixing each file (requires --interactive) + +The script is designed to be run from the command line, either directly +or via the provided Makefile targets. It uses Python's AST module for safe +parsing and modification of Python source files. + +Attributes: + PROJECT_ROOT (Path): The root directory of the project. + INCLUDE_DIRS (List[str]): Directories to include in the scan. + EXCLUDE_DIRS (Set[str]): Directories to exclude from the scan. + COPYRIGHT_YEAR (int): The current year for copyright notices. + AUTHORS (str): Default author name(s) for headers. + LICENSE (str): The project's license identifier. + +Examples: + Check all files (default behavior - dry run): + >>> # python3 .github/tools/fix_file_headers.py + + Check with diff preview: + >>> # python3 .github/tools/fix_file_headers.py --show-diff + + Fix all files (requires explicit flag): + >>> # python3 .github/tools/fix_file_headers.py --fix-all + + Fix a specific file or directory: + >>> # python3 .github/tools/fix_file_headers.py --fix --path ./mcpgateway/main.py + + Fix with specific authors: + >>> # python3 .github/tools/fix_file_headers.py --fix --path ./mcpgateway/main.py --authors "John Doe, Jane Smith" + + Interactive mode: + >>> # python3 .github/tools/fix_file_headers.py --interactive + +Note: + This script will NOT modify files unless explicitly told to with --fix, --fix-all, + or --interactive flags. Always commit your changes before running in fix mode to + allow easy rollback if needed. + +Testing: + Run doctests with: python -m doctest .github/tools/fix_file_headers.py -v +""" + +# Standard +import argparse +import ast +from datetime import datetime +import difflib +import os +from pathlib import Path +import re +import sys +from typing import Any, Dict, Generator, List, Optional, Set, Tuple + +# Configuration constants +PROJECT_ROOT: Path = Path(__file__).parent.parent.parent.resolve() +INCLUDE_DIRS: List[str] = ["mcpgateway", "tests"] +EXCLUDE_DIRS: Set[str] = {".git", ".venv", "venv", "__pycache__", "build", "dist", ".idea", ".vscode", "node_modules", ".tox", ".pytest_cache", ".mypy_cache", ".ruff_cache"} +COPYRIGHT_YEAR: int = datetime.now().year +AUTHORS: str = "Mihai Criveti" +LICENSE: str = "Apache-2.0" + +# Constants for header validation +SHEBANG_LINE: str = "#!/usr/bin/env python3" +ENCODING_LINE: str = "# -*- coding: utf-8 -*-" +HEADER_FIELDS: List[str] = ["Location", "Copyright", "SPDX-License-Identifier", "Authors"] + + +def is_executable(file_path: Path) -> bool: + """Check if a file has executable permissions. + + Args: + file_path: The path to check. + + Returns: + bool: True if the file is executable, False otherwise. + + Examples: + >>> from tempfile import NamedTemporaryFile + >>> import os + >>> with NamedTemporaryFile(mode='w', delete=False) as tmp: + ... tmp_path = Path(tmp.name) + >>> is_executable(tmp_path) + False + >>> os.chmod(tmp_path, 0o755) + >>> is_executable(tmp_path) + True + >>> tmp_path.unlink() + """ + return os.access(file_path, os.X_OK) + + +def validate_authors(authors: str) -> bool: + """Validate that the authors string is properly formatted. + + Args: + authors: A string containing author names, typically comma-separated. + + Returns: + bool: True if the authors string is valid, False otherwise. + + Examples: + >>> validate_authors("John Doe") + True + >>> validate_authors("John Doe, Jane Smith") + True + >>> validate_authors("") + False + >>> validate_authors(" ") + False + >>> validate_authors("John@Doe") + True + """ + return bool(authors and authors.strip()) + + +def validate_path(path: Path, require_in_project: bool = True) -> Tuple[bool, Optional[str]]: + """Validate that a path is safe to process. + + Args: + path: The path to validate. + require_in_project: Whether to require the path be within PROJECT_ROOT. + + Returns: + Tuple[bool, Optional[str]]: A tuple of (is_valid, error_message). + If is_valid is True, error_message is None. + + Examples: + >>> # Test with a file that exists (this script itself) + >>> p = Path(__file__) + >>> valid, msg = validate_path(p) + >>> valid + True + >>> msg is None + True + + >>> # Test with non-existent file + >>> p = PROJECT_ROOT / "nonexistent_test_file_12345.py" + >>> valid, msg = validate_path(p) + >>> valid + False + >>> "does not exist" in msg + True + + >>> p = Path("/etc/passwd") + >>> valid, msg = validate_path(p) + >>> valid + False + >>> "outside project root" in msg + True + """ + if not path.exists(): + return False, f"Path does not exist: {path}" + + if require_in_project: + try: + path.relative_to(PROJECT_ROOT) + except ValueError: + return False, f"Path is outside project root: {path}" + + return True, None + + +def get_header_template(relative_path: str, authors: str = AUTHORS, include_shebang: bool = True, include_encoding: bool = True) -> str: + """Generate the full, standardized header text. + + Args: + relative_path: The relative path from project root to the file. + authors: The author name(s) to include in the header. + include_shebang: Whether to include the shebang line. + include_encoding: Whether to include the encoding line. + + Returns: + str: The complete header template with proper formatting. + + Examples: + >>> header = get_header_template("test/example.py", "John Doe") + >>> "#!/usr/bin/env python3" in header + True + >>> "Location: ./test/example.py" in header + True + >>> "Authors: John Doe" in header + True + >>> f"Copyright {COPYRIGHT_YEAR}" in header + True + + >>> header_no_shebang = get_header_template("test/example.py", "John Doe", include_shebang=False) + >>> "#!/usr/bin/env python3" in header_no_shebang + False + >>> "# -*- coding: utf-8 -*-" in header_no_shebang + True + """ + lines = [] + + if include_shebang: + lines.append(SHEBANG_LINE) + if include_encoding: + lines.append(ENCODING_LINE) + + lines.append(f'''"""Module Description. +Location: ./{relative_path} +Copyright {COPYRIGHT_YEAR} +SPDX-License-Identifier: {LICENSE} +Authors: {authors} + +Module documentation... +"""''') + + return '\n'.join(lines) + + +def _write_file(file_path: Path, content: str) -> None: + """Write content to a file with proper encoding and error handling. + + Args: + file_path: The path to the file to write. + content: The content to write to the file. + + Raises: + IOError: If the file cannot be written. + + Examples: + >>> from tempfile import NamedTemporaryFile + >>> with NamedTemporaryFile(mode='w', delete=False) as tmp: + ... tmp_path = Path(tmp.name) + >>> _write_file(tmp_path, "test content") + >>> tmp_path.read_text() + 'test content' + >>> tmp_path.unlink() + """ + try: + file_path.write_text(content, encoding="utf-8") + except Exception as e: + raise IOError(f"Failed to write file {file_path}: {e}") + + +def find_python_files(base_path: Optional[Path] = None) -> Generator[Path, None, None]: + """Yield all Python files in the project, respecting include/exclude rules. + + Args: + base_path: Optional specific path to search. If None, searches INCLUDE_DIRS. + + Yields: + Path: Paths to Python files found in the search directories. + + Examples: + >>> # Find files in a test directory + >>> test_dir = PROJECT_ROOT / "test_dir" + >>> test_dir.mkdir(exist_ok=True) + >>> (test_dir / "test.py").write_text("# test") + 6 + >>> (test_dir / "test.txt").write_text("not python") + 10 + >>> files = list(find_python_files(test_dir)) + >>> len(files) == 1 + True + >>> files[0].name == "test.py" + True + >>> # Cleanup + >>> (test_dir / "test.py").unlink() + >>> (test_dir / "test.txt").unlink() + >>> test_dir.rmdir() + """ + search_paths: List[Path] = [base_path] if base_path else [PROJECT_ROOT / d for d in INCLUDE_DIRS] + + for search_dir in search_paths: + if not search_dir.exists(): + continue + + if search_dir.is_file() and search_dir.suffix == ".py": + yield search_dir + continue + + if not search_dir.is_dir(): + continue + + for file_path in search_dir.rglob("*.py"): + try: + relative_to_project = file_path.relative_to(PROJECT_ROOT) + # Check if any part of the path is in EXCLUDE_DIRS + if not any(ex_dir in relative_to_project.parts for ex_dir in EXCLUDE_DIRS): + yield file_path + except ValueError: + # File is outside PROJECT_ROOT, skip it + continue + + +def extract_header_info(source_code: str, docstring: str) -> Dict[str, Optional[str]]: + """Extract existing header information from a docstring. + + Args: + source_code: The complete source code of the file. + docstring: The module docstring to parse. + + Returns: + Dict[str, Optional[str]]: A dictionary mapping header field names to their values. + + Examples: + >>> docstring = '''Module description. + ... Location: ./test/file.py + ... Copyright 2025 + ... SPDX-License-Identifier: Apache-2.0 + ... Authors: John Doe + ... + ... More documentation.''' + >>> info = extract_header_info("", docstring) + >>> info["Location"] + 'Location: ./test/file.py' + >>> info["Authors"] + 'Authors: John Doe' + >>> "Copyright" in info["Copyright"] + True + """ + # source_code parameter is kept for API compatibility but not used in current implementation + _ = source_code + + header_info: Dict[str, Optional[str]] = {"Location": None, "Copyright": None, "SPDX-License-Identifier": None, "Authors": None} + + for line in docstring.splitlines(): + line = line.strip() + if line.startswith("Location:"): + header_info["Location"] = line + elif line.startswith("Copyright"): + header_info["Copyright"] = line + elif line.startswith("SPDX-License-Identifier:"): + header_info["SPDX-License-Identifier"] = line + elif line.startswith("Authors:"): + header_info["Authors"] = line + + return header_info + + +def generate_diff(original: str, modified: str, filename: str) -> str: + r"""Generate a unified diff between original and modified content. + + Args: + original: The original file content. + modified: The modified file content. + filename: The name of the file for the diff header. + + Returns: + str: A unified diff string. + + Examples: + >>> original = "line1\nline2\n" + >>> modified = "line1\nline2 modified\n" + >>> diff = generate_diff(original, modified, "test.py") + >>> "@@" in diff + True + >>> "+line2 modified" in diff + True + """ + original_lines = original.splitlines(keepends=True) + modified_lines = modified.splitlines(keepends=True) + + diff = difflib.unified_diff(original_lines, modified_lines, fromfile=f"a/{filename}", tofile=f"b/{filename}", lineterm="") + + return "\n".join(diff) + + +def show_file_lines(file_path: Path, num_lines: int = 10) -> str: + """Show the first few lines of a file for debugging. + + Args: + file_path: The path to the file. + num_lines: Number of lines to show. + + Returns: + str: A formatted string showing the first lines of the file. + """ + try: + lines = file_path.read_text(encoding="utf-8").splitlines() + result = [] + for i, line in enumerate(lines[:num_lines], 1): + result.append(f"{i:3d}: {repr(line)}") + if len(lines) > num_lines: + result.append(f" ... ({len(lines) - num_lines} more lines)") + return "\n".join(result) + except Exception as e: + return f"Error reading file: {e}" + + +def process_file(file_path: Path, mode: str, authors: str, show_diff: bool = False, debug: bool = False, + require_shebang: Optional[bool] = None, require_encoding: bool = True) -> Optional[Dict[str, Any]]: + """Check a single file and optionally fix its header. + + Args: + file_path: The path to the Python file to process. + mode: The processing mode ("check", "fix-all", "fix", or "interactive"). + authors: The author name(s) to use in headers. + show_diff: Whether to show a diff preview in check mode. + debug: Whether to show debug information about file contents. + require_shebang: Whether to require shebang line. If None, only required for executable files. + require_encoding: Whether to require encoding line. + + Returns: + Optional[Dict[str, Any]]: A dictionary containing: + - 'file': The relative path to the file + - 'issues': List of header issues found + - 'fixed': Whether the file was fixed (optional) + - 'skipped': Whether the fix was skipped in interactive mode (optional) + - 'diff': The diff preview if show_diff is True (optional) + - 'debug': Debug information if debug is True (optional) + Returns None if no issues were found. + + Examples: + >>> from tempfile import NamedTemporaryFile + >>> with NamedTemporaryFile(mode='w', suffix='.py', delete=False) as tmp: + ... tmp.write('print("test")') + ... tmp_path = Path(tmp.name) + 13 + >>> result = process_file(tmp_path, "check", "Test Author") + >>> result is not None + True + >>> "Missing encoding line" in result['issues'] + True + >>> tmp_path.unlink() + """ + try: + relative_path_str = str(file_path.relative_to(PROJECT_ROOT)).replace("\\", "/") + except ValueError: + relative_path_str = str(file_path) + + try: + source_code = file_path.read_text(encoding="utf-8") + tree = ast.parse(source_code) + except SyntaxError as e: + return {"file": relative_path_str, "issues": [f"Syntax error: {e}"]} + except Exception as e: + return {"file": relative_path_str, "issues": [f"Error reading/parsing file: {e}"]} + + issues: List[str] = [] + lines = source_code.splitlines() + + # Determine if shebang is required + file_is_executable = is_executable(file_path) + shebang_required = require_shebang if require_shebang is not None else file_is_executable + + # Check for shebang and encoding + has_shebang = bool(lines and lines[0].strip() == SHEBANG_LINE) + has_encoding = len(lines) > 1 and lines[1].strip() == ENCODING_LINE + + # Handle encoding on first line if no shebang + if not has_shebang and lines and lines[0].strip() == ENCODING_LINE: + has_encoding = True + + if shebang_required and not has_shebang: + issues.append("Missing shebang line (file is executable)" if file_is_executable else "Missing shebang line") + + if require_encoding and not has_encoding: + issues.append("Missing encoding line") + + # Get module docstring + docstring_node = ast.get_docstring(tree, clean=False) + module_body = tree.body + new_source_code = None + + if docstring_node is not None: + # Check for required header fields + location_match = re.search(r"^Location: \./(.*)$", docstring_node, re.MULTILINE) + if not location_match: + issues.append("Missing 'Location' line") + + if f"Copyright {COPYRIGHT_YEAR}" not in docstring_node: + issues.append("Missing 'Copyright' line") + + if f"SPDX-License-Identifier: {LICENSE}" not in docstring_node: + issues.append("Missing 'SPDX-License-Identifier' line") + + if not re.search(r"^Authors: ", docstring_node, re.MULTILINE): + issues.append("Missing 'Authors' line") + + if not issues: + return None + + if mode in ["fix-all", "fix", "interactive"]: + # Extract the raw docstring from source + if module_body and isinstance(module_body[0], ast.Expr): + docstring_expr_node = module_body[0] + raw_docstring = ast.get_source_segment(source_code, docstring_expr_node) + + if raw_docstring: + # Determine quote style + quotes = '"""' if raw_docstring.startswith('"""') else "'''" + inner_content = raw_docstring.strip(quotes) + + # Extract existing header fields and body + existing_header_fields = extract_header_info(source_code, inner_content) + + # Find where the header ends and body begins + docstring_lines = inner_content.strip().splitlines() + header_end_idx = 0 + + for i, line in enumerate(docstring_lines): + if any(line.strip().startswith(field + ":") for field in HEADER_FIELDS): + header_end_idx = i + 1 + elif header_end_idx > 0 and line.strip(): + # Found first non-header content + break + + # Extract body content + docstring_body_lines = docstring_lines[header_end_idx:] + if docstring_body_lines and not docstring_body_lines[0].strip(): + docstring_body_lines = docstring_body_lines[1:] + + # Build new header + new_header_lines = [] + new_header_lines.append(existing_header_fields.get("Location") or f"Location: ./{relative_path_str}") + new_header_lines.append(existing_header_fields.get("Copyright") or f"Copyright {COPYRIGHT_YEAR}") + new_header_lines.append(existing_header_fields.get("SPDX-License-Identifier") or f"SPDX-License-Identifier: {LICENSE}") + new_header_lines.append(f"Authors: {authors}") + + # Reconstruct docstring + new_inner_content = "\n".join(new_header_lines) + if docstring_body_lines: + new_inner_content += "\n\n" + "\n".join(docstring_body_lines).strip() + + new_docstring = f"{quotes}{new_inner_content.strip()}{quotes}" + + # Prepare source with appropriate headers + header_lines = [] + if shebang_required: + header_lines.append(SHEBANG_LINE) + if require_encoding: + header_lines.append(ENCODING_LINE) + + if header_lines: + shebang_lines = "\n".join(header_lines) + "\n" + else: + shebang_lines = "" + + # Remove existing shebang/encoding if present + start_line = 0 + if has_shebang: + start_line += 1 + if has_encoding and len(lines) > start_line and lines[start_line].strip() == ENCODING_LINE: + start_line += 1 + + source_without_headers = "\n".join(lines[start_line:]) if start_line < len(lines) else "" + + # Replace the docstring + new_source_code = source_without_headers.replace(raw_docstring, new_docstring, 1) + new_source_code = shebang_lines + new_source_code + + else: + # No docstring found + issues.append("No docstring found") + + if mode in ["fix-all", "fix", "interactive"]: + # Create new header + new_header = get_header_template( + relative_path_str, + authors=authors, + include_shebang=shebang_required, + include_encoding=require_encoding + ) + + # Remove existing shebang/encoding if present + start_line = 0 + if has_shebang: + start_line += 1 + if has_encoding and len(lines) > start_line and lines[start_line].strip() == ENCODING_LINE: + start_line += 1 + + remaining_content = "\n".join(lines[start_line:]) if start_line < len(lines) else source_code + new_source_code = new_header + "\n" + remaining_content + + # Handle the result + result: Dict[str, Any] = {"file": relative_path_str, "issues": issues} + + if debug: + result["debug"] = { + "executable": file_is_executable, + "has_shebang": has_shebang, + "has_encoding": has_encoding, + "first_lines": show_file_lines(file_path, 5) + } + + if show_diff and new_source_code and new_source_code != source_code: + result["diff"] = generate_diff(source_code, new_source_code, relative_path_str) + + if new_source_code and new_source_code != source_code and mode != "check": + if mode == "interactive": + print(f"\n๐Ÿ“„ File: {relative_path_str}") + print(f" Issues: {', '.join(issues)}") + if debug: + print(f" Executable: {file_is_executable}") + print(" First lines:") + print(" " + "\n ".join(show_file_lines(file_path, 5).split("\n"))) + if show_diff: + print("\n--- Proposed changes ---") + print(result.get("diff", "")) + confirm = input("\n Apply changes? (y/n): ").lower().strip() + if confirm != "y": + result["fixed"] = False + result["skipped"] = True + return result + + try: + _write_file(file_path, new_source_code) + result["fixed"] = True + except IOError as e: + result["issues"].append(f"Failed to write file: {e}") + result["fixed"] = False + else: + result["fixed"] = False + + return result if issues else None + + +def parse_arguments(argv: Optional[List[str]] = None) -> argparse.Namespace: + """Parse command line arguments. + + Args: + argv: Optional list of arguments. If None, uses sys.argv[1:]. + + Returns: + argparse.Namespace: Parsed command line arguments. + + Examples: + >>> args = parse_arguments(["--check"]) + >>> args.check + True + >>> args.fix_all + False + + >>> args = parse_arguments(["--fix", "--path", "test.py", "--authors", "John Doe"]) + >>> args.fix + True + >>> args.path + 'test.py' + >>> args.authors + 'John Doe' + + >>> # Default behavior with no args + >>> args = parse_arguments([]) + >>> args.check + False + >>> args.fix + False + >>> args.fix_all + False + """ + parser = argparse.ArgumentParser( + description="Check and fix file headers in Python source files. " "By default, runs in check mode (dry run).", + epilog="Examples:\n" + " %(prog)s # Check all files (default)\n" + " %(prog)s --fix-all # Fix all files\n" + " %(prog)s --fix --path file.py # Fix specific file\n" + " %(prog)s --interactive # Fix with prompts", + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + + parser.add_argument("files", nargs="*", help="Files to process (usually passed by pre-commit).") + + mode_group = parser.add_mutually_exclusive_group() + mode_group.add_argument("--check", action="store_true", help="Dry run: check files but do not make changes (default behavior).") + mode_group.add_argument("--fix", action="store_true", help="Fix headers in files specified by --path. Requires --path.") + mode_group.add_argument("--fix-all", action="store_true", help="Automatically fix all incorrect headers in the project.") + mode_group.add_argument("--interactive", action="store_true", help="Interactively review and apply fixes.") + + parser.add_argument("--path", type=str, help="Specify a file or directory to process. Required with --fix.") + parser.add_argument("--authors", type=str, default=AUTHORS, help=f"Specify the author name(s) for new headers. Default: {AUTHORS}") + parser.add_argument("--show-diff", action="store_true", help="Show diff preview of changes in check mode.") + parser.add_argument("--debug", action="store_true", help="Show debug information about file contents.") + + # Header configuration options + header_group = parser.add_argument_group("header configuration") + header_group.add_argument("--require-shebang", choices=["always", "never", "auto"], default="auto", + help="Require shebang line: 'always', 'never', or 'auto' (only for executable files). Default: auto") + header_group.add_argument("--require-encoding", action="store_true", default=True, + help="Require encoding line. Default: True") + header_group.add_argument("--no-encoding", action="store_false", dest="require_encoding", + help="Don't require encoding line.") + header_group.add_argument("--copyright-year", type=int, default=COPYRIGHT_YEAR, + help=f"Copyright year to use. Default: {COPYRIGHT_YEAR}") + header_group.add_argument("--license", type=str, default=LICENSE, + help=f"License identifier to use. Default: {LICENSE}") + + return parser.parse_args(argv) + + +def determine_mode(args: argparse.Namespace) -> str: + """Determine the operating mode from parsed arguments. + + Args: + args: Parsed command line arguments. + + Returns: + str: The mode to operate in ("check", "fix-all", "fix", or "interactive"). + + Examples: + >>> from argparse import Namespace + >>> args = Namespace(files=[], check=True, fix_all=False, interactive=False, path=None, fix=False) + >>> determine_mode(args) + 'check' + + >>> args = Namespace(files=[], check=False, fix_all=True, interactive=False, path=None, fix=False) + >>> determine_mode(args) + 'fix-all' + + >>> args = Namespace(files=[], check=False, fix_all=False, interactive=False, path="test.py", fix=True) + >>> determine_mode(args) + 'fix' + + >>> # Default behavior with no flags + >>> args = Namespace(files=[], check=False, fix_all=False, interactive=False, path=None, fix=False) + >>> determine_mode(args) + 'check' + """ + # Check if any modification mode is explicitly requested + if args.fix_all: + return "fix-all" + if args.interactive: + return "interactive" + if args.fix and args.path: + return "fix" + if args.check: + return "check" + # Default to check mode if no flags specified + return "check" + + +def collect_files_to_process(args: argparse.Namespace) -> List[Path]: + """Collect all files that need to be processed based on arguments. + + Args: + args: Parsed command line arguments. + + Returns: + List[Path]: List of file paths to process. + + Raises: + SystemExit: If an invalid path is specified. + + Examples: + >>> from argparse import Namespace + >>> args = Namespace(files=[], path=None) + >>> files = collect_files_to_process(args) + >>> isinstance(files, list) + True + """ + files_to_process: List[Path] = [] + + if args.files: + files_to_process = [Path(f) for f in args.files] + elif args.path: + target_path = Path(args.path) + + # Convert to absolute path if relative + if not target_path.is_absolute(): + target_path = PROJECT_ROOT / target_path + + # Validate the path + valid, error_msg = validate_path(target_path) + if not valid: + print(f"Error: {error_msg}", file=sys.stderr) + sys.exit(1) + + if target_path.is_file() and target_path.suffix == ".py": + files_to_process = [target_path] + elif target_path.is_dir(): + files_to_process = list(find_python_files(target_path)) + else: + print(f"Error: Path '{args.path}' is not a valid Python file or directory.", file=sys.stderr) + sys.exit(1) + else: + files_to_process = list(find_python_files()) + + return files_to_process + + +def print_results(issues_found: List[Dict[str, Any]], mode: str, modified_count: int) -> None: + """Print the results of the header checking/fixing process. + + Args: + issues_found: List of dictionaries containing file issues and status. + mode: The mode that was used ("check", "fix-all", etc.). + modified_count: Number of files that were modified. + + Examples: + >>> issues = [{"file": "test.py", "issues": ["Missing header"], "fixed": True}] + >>> import sys + >>> from io import StringIO + >>> old_stderr = sys.stderr + >>> sys.stderr = StringIO() + >>> try: + ... print_results(issues, "fix-all", 1) + ... output = sys.stderr.getvalue() + ... "โœ… Fixed: test.py" in output + ... finally: + ... sys.stderr = old_stderr + True + """ + if not issues_found: + print("All Python file headers are correct. โœจ", file=sys.stdout) + return + + print("\n--- Header Issues Found ---", file=sys.stderr) + + for issue_info in issues_found: + file_name = issue_info["file"] + issues_list = issue_info["issues"] + fixed_status = issue_info.get("fixed", False) + skipped_status = issue_info.get("skipped", False) + + if fixed_status: + print(f"โœ… Fixed: {file_name} (Issues: {', '.join(issues_list)})", file=sys.stderr) + elif skipped_status: + print(f"โš ๏ธ Skipped: {file_name} (Issues: {', '.join(issues_list)})", file=sys.stderr) + else: + print(f"โŒ Needs Fix: {file_name} (Issues: {', '.join(issues_list)})", file=sys.stderr) + + # Show debug info if available + if "debug" in issue_info: + debug = issue_info["debug"] + print(f" Debug info:", file=sys.stderr) + print(f" Executable: {debug['executable']}", file=sys.stderr) + print(f" Has shebang: {debug['has_shebang']}", file=sys.stderr) + print(f" Has encoding: {debug['has_encoding']}", file=sys.stderr) + + # Show diff if available + if "diff" in issue_info and mode == "check": + print(f"\n--- Diff preview for {file_name} ---", file=sys.stderr) + print(issue_info["diff"], file=sys.stderr) + + # Print helpful messages based on mode + if mode == "check": + print("\nTo fix these headers, run: make fix-all-headers", file=sys.stderr) + print("Or add to your pre-commit config with '--fix-all' argument.", file=sys.stderr) + elif mode == "interactive": + print("\nSome files may have been skipped in interactive mode.", file=sys.stderr) + print("To fix all remaining headers, run: make fix-all-headers", file=sys.stderr) + elif modified_count > 0: + print(f"\nSuccessfully fixed {modified_count} file(s). " f"Please re-stage and commit.", file=sys.stderr) + + +def main(argv: Optional[List[str]] = None) -> None: + """Parse arguments and run the script. + + Args: + argv: Optional list of command line arguments. If None, uses sys.argv[1:]. + + Raises: + SystemExit: With code 0 on success, 1 if issues were found. + + Examples: + >>> # Test with no arguments (check mode by default) + >>> import sys + >>> from io import StringIO + >>> old_stdout = sys.stdout + >>> sys.stdout = StringIO() + >>> try: + ... main([]) # Should run in check mode + ... except SystemExit as e: + ... sys.stdout = old_stdout + ... e.code in (0, 1) + True + + >>> # Test with explicit fix mode on non-existent file + >>> try: + ... main(["--fix", "--path", "nonexistent.py"]) + ... except SystemExit as e: + ... e.code == 1 + True + """ + global COPYRIGHT_YEAR, LICENSE + + args = parse_arguments(argv) + + # Update global config from arguments + COPYRIGHT_YEAR = args.copyright_year + LICENSE = args.license + + # Validate --fix requires --path + if args.fix and not args.path: + print("Error: --fix requires --path to specify which file or directory to fix.", file=sys.stderr) + print("Usage: fix_file_headers.py --fix --path ", file=sys.stderr) + sys.exit(1) + + mode = determine_mode(args) + + # Validate authors + if not validate_authors(args.authors): + print("Error: Invalid authors string. Authors cannot be empty.", file=sys.stderr) + sys.exit(1) + + # Collect files to process + files_to_process = collect_files_to_process(args) + + if not files_to_process: + print("No Python files found to process.", file=sys.stdout) + sys.exit(0) + + # Show mode information + if mode == "check": + print("๐Ÿ” Running in CHECK mode (dry run). No files will be modified.") + if args.show_diff: + print(" Diff preview enabled.") + if args.debug: + print(" Debug mode enabled.") + elif mode == "fix": + print(f"๐Ÿ”ง Running in FIX mode for: {args.path}") + print(" Files WILL be modified!") + elif mode == "fix-all": + print("๐Ÿ”ง Running in FIX-ALL mode.") + print(" ALL files with incorrect headers WILL be modified!") + elif mode == "interactive": + print("๐Ÿ’ฌ Running in INTERACTIVE mode.") + print(" You will be prompted before each change.") + + # Determine shebang requirement + require_shebang = None + if args.require_shebang == "always": + require_shebang = True + elif args.require_shebang == "never": + require_shebang = False + # else: auto mode, require_shebang remains None + + # Process files + issues_found_in_files: List[Dict[str, Any]] = [] + modified_files_count = 0 + + for file_path in files_to_process: + result = process_file( + file_path, + mode, + args.authors, + show_diff=args.show_diff, + debug=args.debug, + require_shebang=require_shebang, + require_encoding=args.require_encoding + ) + if result: + issues_found_in_files.append(result) + if result.get("fixed", False): + modified_files_count += 1 + + # Print results + print_results(issues_found_in_files, mode, modified_files_count) + + # Exit with appropriate code + if issues_found_in_files: + sys.exit(1) + else: + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/check-headers.yml.inactive b/.github/workflows/check-headers.yml.inactive new file mode 100644 index 000000000..8cbbf1e18 --- /dev/null +++ b/.github/workflows/check-headers.yml.inactive @@ -0,0 +1,226 @@ +# =============================================================== +# ๐Ÿ” MCP Gateway โ–ธ Python File Header Validation +# =============================================================== +# +# This workflow: +# - Checks all Python files for proper headers ๐Ÿ” +# - Validates copyright, license, and author information ๐Ÿ“‹ +# - Shows diff preview of what needs to be fixed ๐Ÿ“ +# - Fails if any files have incorrect headers โŒ +# +# --------------------------------------------------------------- +# When it runs: +# --------------------------------------------------------------- +# - On every pull request (to catch issues early) +# - On pushes to main/master (to ensure compliance) +# - Manual trigger available (workflow_dispatch) +# +# --------------------------------------------------------------- +# What it checks: +# --------------------------------------------------------------- +# โœ“ Shebang line (for executable files) +# โœ“ Encoding declaration +# โœ“ Module docstring with: +# - Location path +# - Copyright year +# - SPDX license identifier +# - Authors field +# +# --------------------------------------------------------------- + +name: ๐Ÿ” Check Python Headers + +on: + pull_request: + paths: + - '**.py' + - '.github/workflows/check-headers.yml' + - '.github/tools/fix_file_headers.py' + + push: + branches: + - main + - master + paths: + - '**.py' + + workflow_dispatch: + inputs: + debug_mode: + description: 'Enable debug mode' + required: false + type: boolean + default: false + show_diff: + description: 'Show diff preview' + required: false + type: boolean + default: true + +# ----------------------------------------------------------------- +# Minimal permissions (Principle of Least Privilege) +# ----------------------------------------------------------------- +permissions: + contents: read + pull-requests: write # For PR comments + +# ----------------------------------------------------------------- +# Cancel in-progress runs when new commits are pushed +# ----------------------------------------------------------------- +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + check-headers: + name: ๐Ÿ” Validate Python Headers + runs-on: ubuntu-latest + + steps: + # ----------------------------------------------------------- + # 0๏ธโƒฃ Checkout repository + # ----------------------------------------------------------- + - name: โฌ‡๏ธ Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 # Full history for better path resolution + + # ----------------------------------------------------------- + # 1๏ธโƒฃ Set up Python + # ----------------------------------------------------------- + - name: ๐Ÿ Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + cache: 'pip' + + # ----------------------------------------------------------- + # 2๏ธโƒฃ Display Python version & path info + # ----------------------------------------------------------- + - name: ๐Ÿ“ Display Python info + run: | + echo "๐Ÿ Python version:" + python --version + echo "๐Ÿ“‚ Python path:" + which python + echo "๐Ÿ“ Working directory:" + pwd + echo "๐Ÿ“Š Python files to check:" + find . -name "*.py" -not -path "./.venv/*" -not -path "./.git/*" | wc -l + + # ----------------------------------------------------------- + # 3๏ธโƒฃ Run header check (with optional debug/diff) + # ----------------------------------------------------------- + - name: ๐Ÿ” Check Python file headers + id: check + run: | + echo "๐Ÿ” Checking Python file headers..." + + # Build command based on inputs + CHECK_CMD="python3 .github/tools/fix_file_headers.py" + + # Add flags based on workflow inputs + if [[ "${{ inputs.show_diff }}" == "true" ]] || [[ "${{ github.event_name }}" == "pull_request" ]]; then + CHECK_CMD="$CHECK_CMD --show-diff" + fi + + if [[ "${{ inputs.debug_mode }}" == "true" ]]; then + CHECK_CMD="$CHECK_CMD --debug" + fi + + echo "๐Ÿƒ Running: $CHECK_CMD" + + # Run check and capture output + if $CHECK_CMD > header-check-output.txt 2>&1; then + echo "โœ… All Python file headers are correct!" + echo "check_passed=true" >> $GITHUB_OUTPUT + else + echo "โŒ Some files have incorrect headers" + echo "check_passed=false" >> $GITHUB_OUTPUT + + # Show the output + cat header-check-output.txt + + # Save summary for PR comment + echo '```' > header-check-summary.md + cat header-check-output.txt >> header-check-summary.md + echo '```' >> header-check-summary.md + fi + + # ----------------------------------------------------------- + # 4๏ธโƒฃ Comment on PR (if applicable) + # ----------------------------------------------------------- + - name: ๐Ÿ’ฌ Comment on PR + if: github.event_name == 'pull_request' && steps.check.outputs.check_passed == 'false' + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + const summary = fs.readFileSync('header-check-summary.md', 'utf8'); + + const body = `## โŒ Python Header Check Failed + + Some Python files have incorrect or missing headers. Please fix them before merging. + + ### ๐Ÿ”ง How to fix: + + 1. **Fix all files automatically:** + \`\`\`bash + make fix-all-headers + \`\`\` + + 2. **Fix specific files:** + \`\`\`bash + make fix-header path=path/to/file.py + \`\`\` + + 3. **Review changes interactively:** + \`\`\`bash + make interactive-fix-headers + \`\`\` + + ### ๐Ÿ“‹ Check Results: + + ${summary} + + --- + ๐Ÿค– *This check ensures all Python files have proper copyright, license, and author information.*`; + + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: body + }); + + # ----------------------------------------------------------- + # 5๏ธโƒฃ Upload check results as artifact + # ----------------------------------------------------------- + - name: ๐Ÿ“ค Upload check results + if: failure() + uses: actions/upload-artifact@v4 + with: + name: header-check-results + path: | + header-check-output.txt + header-check-summary.md + retention-days: 7 + + # ----------------------------------------------------------- + # 6๏ธโƒฃ Fail the workflow if headers are incorrect + # ----------------------------------------------------------- + - name: ๐Ÿšจ Fail if headers incorrect + if: steps.check.outputs.check_passed == 'false' + run: | + echo "โŒ Header check failed!" + echo "Please run 'make fix-all-headers' locally and commit the changes." + exit 1 + + # ----------------------------------------------------------- + # 7๏ธโƒฃ Success message + # ----------------------------------------------------------- + - name: โœ… Success + if: steps.check.outputs.check_passed == 'true' + run: | + echo "โœ… All Python file headers are properly formatted!" + echo "๐ŸŽ‰ No action needed." \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a180b2168..589c7cc82 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -82,3 +82,24 @@ before submitting. ## Coding style guidelines **FIXME** Optional, but recommended: please share any specific style guidelines you might have for your project. + +### Python File Headers + +All Python source files (`.py`) must begin with the following standardized header. This ensures consistency and proper licensing across the codebase. + +The header format is as follows: + +```python +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +"""Module Description. +Location: ./path/to/your/file.py +Copyright 2025 +SPDX-License-Identifier: Apache-2.0 +Authors: "Author One, Author Two" + +Your detailed module documentation begins here... +""" +``` + +You can automatically check and fix file headers using the provided `make` targets. For detailed usage and examples, please see the [File Header Management section](../docs/docs/development/module-documentation.md) in our development documentation. diff --git a/Makefile b/Makefile index 0a39ead4d..de753f57e 100644 --- a/Makefile +++ b/Makefile @@ -41,7 +41,8 @@ FILES_TO_CLEAN := .coverage coverage.xml mcp.prof mcp.pstats \ *.db *.sqlite *.sqlite3 mcp.db-journal *.py,cover \ .depsorter_cache.json .depupdate.* \ grype-results.sarif devskim-results.sarif \ - *.tar.gz *.tar.bz2 *.tar.xz *.zip *.deb + *.tar.gz *.tar.bz2 *.tar.xz *.zip *.deb \ + *.log mcpgateway.sbom.xml COVERAGE_DIR ?= $(DOCS_DIR)/docs/coverage LICENSES_MD ?= $(DOCS_DIR)/docs/test/licenses.md @@ -408,6 +409,14 @@ images: # ๐Ÿ” LINTING & STATIC ANALYSIS # ============================================================================= # help: ๐Ÿ” LINTING & STATIC ANALYSIS +# help: TARGET= - Override default target (mcpgateway) +# help: Usage Examples: +# help: make lint - Run all linters on default targets (mcpgateway) +# help: make lint TARGET=myfile.py - Run file-aware linters on specific file +# help: make lint myfile.py - Run file-aware linters on a file (shortcut) +# help: make lint-quick myfile.py - Fast linters only (ruff, black, isort) +# help: make lint-fix myfile.py - Auto-fix formatting issues +# help: make lint-changed - Lint only git-changed files # help: lint - Run the full linting suite (see targets below) # help: black - Reformat code with black # help: autoflake - Remove unused imports / variables with autoflake @@ -442,59 +451,239 @@ images: # help: unimport - Unused import detection # help: vulture - Dead code detection -# List of individual lint targets; lint loops over these +# Allow specific file/directory targeting +DEFAULT_TARGETS := mcpgateway +TARGET ?= $(DEFAULT_TARGETS) + +# Add dummy targets for file arguments passed to lint commands only +# This prevents make from trying to build file targets when they're used as arguments +ifneq ($(filter lint lint-quick lint-fix lint-smart,$(MAKECMDGOALS)),) + # Get all arguments after the first goal + LINT_FILE_ARGS := $(wordlist 2,$(words $(MAKECMDGOALS)),$(MAKECMDGOALS)) + # Create dummy targets for each file argument + $(LINT_FILE_ARGS): + @: +endif + +# List of individual lint targets LINTERS := isort flake8 pylint mypy bandit pydocstyle pycodestyle pre-commit \ - ruff pyright radon pyroma pyrefly spellcheck importchecker \ - pytype check-manifest markdownlint vulture unimport + ruff ty pyright radon pyroma pyrefly spellcheck importchecker \ + pytype check-manifest markdownlint vulture unimport + +# Linters that work well with individual files/directories +FILE_AWARE_LINTERS := isort black flake8 pylint mypy bandit pydocstyle \ + pycodestyle ruff pyright vulture unimport markdownlint -.PHONY: lint $(LINTERS) black fawltydeps wily depend snakeviz pstats \ - spellcheck-sort tox pytype sbom +.PHONY: lint $(LINTERS) black autoflake lint-py lint-yaml lint-json lint-md lint-strict \ + lint-count-errors lint-report lint-changed lint-staged lint-commit \ + lint-pre-commit lint-pre-push lint-parallel lint-cache-clear lint-stats \ + lint-complexity lint-watch lint-watch-quick \ + lint-install-hooks lint-quick lint-fix lint-smart lint-target lint-all ## --------------------------------------------------------------------------- ## -## Master target +## Main target with smart file/directory detection ## --------------------------------------------------------------------------- ## lint: - @echo "๐Ÿ” Running full lint suite..." + @# Handle multiple file arguments + @file_args="$(wordlist 2,$(words $(MAKECMDGOALS)),$(MAKECMDGOALS))"; \ + if [ -n "$$file_args" ]; then \ + echo "๐ŸŽฏ Running linters on specified files: $$file_args"; \ + for file in $$file_args; do \ + if [ ! -e "$$file" ]; then \ + echo "โŒ File/directory not found: $$file"; \ + exit 1; \ + fi; \ + echo "๐Ÿ” Linting: $$file"; \ + $(MAKE) --no-print-directory lint-smart "$$file"; \ + done; \ + else \ + echo "๐Ÿ” Running full lint suite on: $(TARGET)"; \ + $(MAKE) --no-print-directory lint-all TARGET="$(TARGET)"; \ + fi + + +.PHONY: lint-target +lint-target: + @# Check if target exists + @if [ ! -e "$(TARGET)" ]; then \ + echo "โŒ File/directory not found: $(TARGET)"; \ + exit 1; \ + fi + @# Run only file-aware linters + @echo "๐Ÿ” Running file-aware linters on: $(TARGET)" + @set -e; for t in $(FILE_AWARE_LINTERS); do \ + echo "โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”"; \ + echo "- $$t on $(TARGET)"; \ + $(MAKE) --no-print-directory $$t TARGET="$(TARGET)" || true; \ + done + +.PHONY: lint-all +lint-all: @set -e; for t in $(LINTERS); do \ - echo "โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”"; \ - echo "- $$t"; \ - $(MAKE) $$t || true; \ + echo "โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”"; \ + echo "- $$t"; \ + $(MAKE) --no-print-directory $$t TARGET="$(TARGET)" || true; \ done ## --------------------------------------------------------------------------- ## -## Individual targets (alphabetical) +## Convenience targets +## --------------------------------------------------------------------------- ## + +# Quick lint - only fast linters (ruff, black, isort) +.PHONY: lint-quick +lint-quick: + @# Handle file arguments + @target_file="$(word 2,$(MAKECMDGOALS))"; \ + if [ -n "$$target_file" ] && [ "$$target_file" != "" ]; then \ + actual_target="$$target_file"; \ + else \ + actual_target="$(TARGET)"; \ + fi; \ + echo "โšก Quick lint of $$actual_target (ruff + black + isort)..."; \ + $(MAKE) --no-print-directory ruff-check TARGET="$$actual_target"; \ + $(MAKE) --no-print-directory black-check TARGET="$$actual_target"; \ + $(MAKE) --no-print-directory isort-check TARGET="$$actual_target" + +# Fix formatting issues +.PHONY: lint-fix +lint-fix: + @# Handle file arguments + @target_file="$(word 2,$(MAKECMDGOALS))"; \ + if [ -n "$$target_file" ] && [ "$$target_file" != "" ]; then \ + actual_target="$$target_file"; \ + else \ + actual_target="$(TARGET)"; \ + fi; \ + for target in $$(echo $$actual_target); do \ + if [ ! -e "$$target" ]; then \ + echo "โŒ File/directory not found: $$target"; \ + exit 1; \ + fi; \ + done; \ + echo "๐Ÿ”ง Fixing lint issues in $$actual_target..."; \ + $(MAKE) --no-print-directory black TARGET="$$actual_target"; \ + $(MAKE) --no-print-directory isort TARGET="$$actual_target"; \ + $(MAKE) --no-print-directory ruff-fix TARGET="$$actual_target" + +# Smart linting based on file extension +.PHONY: lint-smart +lint-smart: + @# Handle arguments passed to this target - FIXED VERSION + @target_file="$(word 2,$(MAKECMDGOALS))"; \ + if [ -n "$$target_file" ] && [ "$$target_file" != "" ]; then \ + actual_target="$$target_file"; \ + else \ + actual_target="mcpgateway"; \ + fi; \ + if [ ! -e "$$actual_target" ]; then \ + echo "โŒ File/directory not found: $$actual_target"; \ + exit 1; \ + fi; \ + case "$$actual_target" in \ + *.py) \ + echo "๐Ÿ Python file detected: $$actual_target"; \ + $(MAKE) --no-print-directory lint-target TARGET="$$actual_target" ;; \ + *.yaml|*.yml) \ + echo "๐Ÿ“„ YAML file detected: $$actual_target"; \ + $(MAKE) --no-print-directory yamllint TARGET="$$actual_target" ;; \ + *.json) \ + echo "๐Ÿ“„ JSON file detected: $$actual_target"; \ + $(MAKE) --no-print-directory jsonlint TARGET="$$actual_target" ;; \ + *.md) \ + echo "๐Ÿ“ Markdown file detected: $$actual_target"; \ + $(MAKE) --no-print-directory markdownlint TARGET="$$actual_target" ;; \ + *.toml) \ + echo "๐Ÿ“„ TOML file detected: $$actual_target"; \ + $(MAKE) --no-print-directory tomllint TARGET="$$actual_target" ;; \ + *.sh) \ + echo "๐Ÿš Shell script detected: $$actual_target"; \ + $(MAKE) --no-print-directory shell-lint TARGET="$$actual_target" ;; \ + Makefile|*.mk) \ + echo "๐Ÿ”จ Makefile detected: $$actual_target"; \ + echo "โ„น๏ธ Makefile linting not supported, skipping Python linters"; \ + echo "๐Ÿ’ก Consider using shellcheck for shell portions if needed" ;; \ + *) \ + if [ -d "$$actual_target" ]; then \ + echo "๐Ÿ“ Directory detected: $$actual_target"; \ + $(MAKE) --no-print-directory lint-target TARGET="$$actual_target"; \ + else \ + echo "โ“ Unknown file type, running Python linters"; \ + $(MAKE) --no-print-directory lint-target TARGET="$$actual_target"; \ + fi ;; \ + esac + + fi + +## --------------------------------------------------------------------------- ## +## Individual targets (alphabetical, updated to use TARGET) ## --------------------------------------------------------------------------- ## autoflake: ## ๐Ÿงน Strip unused imports / vars + @echo "๐Ÿงน autoflake $(TARGET)..." @$(VENV_DIR)/bin/autoflake --in-place --remove-all-unused-imports \ - --remove-unused-variables -r mcpgateway tests + --remove-unused-variables -r $(TARGET) black: ## ๐ŸŽจ Reformat code with black - @echo "๐ŸŽจ black ..." && $(VENV_DIR)/bin/black -l 200 mcpgateway tests + @echo "๐ŸŽจ black $(TARGET)..." && $(VENV_DIR)/bin/black -l 200 $(TARGET) + +# Black check mode (separate target) +black-check: + @echo "๐ŸŽจ black --check $(TARGET)..." && $(VENV_DIR)/bin/black -l 200 --check --diff $(TARGET) isort: ## ๐Ÿ”€ Sort imports - @echo "๐Ÿ”€ isort ..." && $(VENV_DIR)/bin/isort . + @echo "๐Ÿ”€ isort $(TARGET)..." && $(VENV_DIR)/bin/isort $(TARGET) + +# Isort check mode (separate target) +isort-check: + @echo "๐Ÿ”€ isort --check $(TARGET)..." && $(VENV_DIR)/bin/isort --check-only --diff $(TARGET) flake8: ## ๐Ÿ flake8 checks - @$(VENV_DIR)/bin/flake8 mcpgateway + @echo "๐Ÿ flake8 $(TARGET)..." && $(VENV_DIR)/bin/flake8 $(TARGET) pylint: ## ๐Ÿ› pylint checks - @$(VENV_DIR)/bin/pylint mcpgateway + @echo "๐Ÿ› pylint $(TARGET)..." && $(VENV_DIR)/bin/pylint $(TARGET) markdownlint: ## ๐Ÿ“– Markdown linting - @$(VENV_DIR)/bin/markdownlint -c .markdownlint.json . + @# Install markdownlint-cli2 if not present + @if ! command -v markdownlint-cli2 >/dev/null 2>&1; then \ + echo "๐Ÿ“ฆ Installing markdownlint-cli2..."; \ + if command -v npm >/dev/null 2>&1; then \ + npm install -g markdownlint-cli2; \ + else \ + echo "โŒ npm not found. Please install Node.js/npm first."; \ + echo "๐Ÿ’ก Install with:"; \ + echo " โ€ข macOS: brew install node"; \ + echo " โ€ข Linux: sudo apt-get install nodejs npm"; \ + exit 1; \ + fi; \ + fi + @if [ -f "$(TARGET)" ] && echo "$(TARGET)" | grep -qE '\.(md|markdown)$$'; then \ + echo "๐Ÿ“– markdownlint $(TARGET)..."; \ + markdownlint-cli2 "$(TARGET)" || true; \ + elif [ -d "$(TARGET)" ]; then \ + echo "๐Ÿ“– markdownlint $(TARGET)..."; \ + markdownlint-cli2 "$(TARGET)/**/*.md" || true; \ + else \ + echo "๐Ÿ“– markdownlint (default)..."; \ + markdownlint-cli2 "**/*.md" || true; \ + fi mypy: ## ๐Ÿท๏ธ mypy type-checking - @$(VENV_DIR)/bin/mypy mcpgateway + @echo "๐Ÿท๏ธ mypy $(TARGET)..." && $(VENV_DIR)/bin/mypy $(TARGET) bandit: ## ๐Ÿ›ก๏ธ bandit security scan - @$(VENV_DIR)/bin/bandit -r mcpgateway + @echo "๐Ÿ›ก๏ธ bandit $(TARGET)..." + @if [ -d "$(TARGET)" ]; then \ + $(VENV_DIR)/bin/bandit -r $(TARGET); \ + else \ + $(VENV_DIR)/bin/bandit $(TARGET); \ + fi pydocstyle: ## ๐Ÿ“š Docstring style - @$(VENV_DIR)/bin/pydocstyle mcpgateway + @echo "๐Ÿ“š pydocstyle $(TARGET)..." && $(VENV_DIR)/bin/pydocstyle $(TARGET) pycodestyle: ## ๐Ÿ“ Simple PEP-8 checker - @$(VENV_DIR)/bin/pycodestyle mcpgateway --max-line-length=200 + @echo "๐Ÿ“ pycodestyle $(TARGET)..." && $(VENV_DIR)/bin/pycodestyle $(TARGET) --max-line-length=200 pre-commit: ## ๐Ÿช„ Run pre-commit hooks @echo "๐Ÿช„ Running pre-commit hooks..." @@ -506,19 +695,29 @@ pre-commit: ## ๐Ÿช„ Run pre-commit hooks @/bin/bash -c "source $(VENV_DIR)/bin/activate && pre-commit run --all-files --show-diff-on-failure" ruff: ## โšก Ruff lint + format - @$(VENV_DIR)/bin/ruff check mcpgateway && $(VENV_DIR)/bin/ruff format mcpgateway tests + @echo "โšก ruff $(TARGET)..." && $(VENV_DIR)/bin/ruff check $(TARGET) && $(VENV_DIR)/bin/ruff format $(TARGET) + +# Separate ruff targets for different modes +ruff-check: + @echo "โšก ruff check $(TARGET)..." && $(VENV_DIR)/bin/ruff check $(TARGET) + +ruff-fix: + @echo "โšก ruff check --fix $(TARGET)..." && $(VENV_DIR)/bin/ruff check --fix $(TARGET) + +ruff-format: + @echo "โšก ruff format $(TARGET)..." && $(VENV_DIR)/bin/ruff format $(TARGET) ty: ## โšก Ty type checker - @$(VENV_DIR)/bin/ty check mcpgateway tests + @echo "โšก ty $(TARGET)..." && $(VENV_DIR)/bin/ty check $(TARGET) pyright: ## ๐Ÿท๏ธ Pyright type-checking - @$(VENV_DIR)/bin/pyright mcpgateway tests + @echo "๐Ÿท๏ธ pyright $(TARGET)..." && $(VENV_DIR)/bin/pyright $(TARGET) radon: ## ๐Ÿ“ˆ Complexity / MI metrics - @$(VENV_DIR)/bin/radon mi -s mcpgateway tests && \ - $(VENV_DIR)/bin/radon cc -s mcpgateway tests && \ - $(VENV_DIR)/bin/radon hal mcpgateway tests && \ - $(VENV_DIR)/bin/radon raw -s mcpgateway tests + @$(VENV_DIR)/bin/radon mi -s $(TARGET) && \ + $(VENV_DIR)/bin/radon cc -s $(TARGET) && \ + $(VENV_DIR)/bin/radon hal $(TARGET) && \ + $(VENV_DIR)/bin/radon raw -s $(TARGET) pyroma: ## ๐Ÿ“ฆ Packaging metadata check @$(VENV_DIR)/bin/pyroma -d . @@ -546,7 +745,7 @@ pyre: ## ๐Ÿง  Facebook Pyre analysis @$(VENV_DIR)/bin/pyre pyrefly: ## ๐Ÿง  Facebook Pyrefly analysis (faster, rust) - @$(VENV_DIR)/bin/pyrefly check mcpgateway + @echo "๐Ÿง  pyrefly $(TARGET)..." && $(VENV_DIR)/bin/pyrefly check $(TARGET) depend: ## ๐Ÿ“ฆ List dependencies @echo "๐Ÿ“ฆ List dependencies" @@ -620,17 +819,388 @@ sbom: ## ๐Ÿ›ก๏ธ Generate SBOM & security report pytype: ## ๐Ÿง  Pytype static type analysis @echo "๐Ÿง  Pytype analysis..." - @$(VENV_DIR)/bin/pytype -V 3.12 -j auto mcpgateway tests + @$(VENV_DIR)/bin/pytype -V 3.12 -j auto $(TARGET) check-manifest: ## ๐Ÿ“ฆ Verify MANIFEST.in completeness @echo "๐Ÿ“ฆ Verifying MANIFEST.in completeness..." @$(VENV_DIR)/bin/check-manifest unimport: ## ๐Ÿ“ฆ Unused import detection - @echo "๐Ÿ“ฆ unimport โ€ฆ" && $(VENV_DIR)/bin/unimport --check --diff mcpgateway + @echo "๐Ÿ“ฆ unimport $(TARGET)โ€ฆ" && $(VENV_DIR)/bin/unimport --check --diff $(TARGET) vulture: ## ๐Ÿงน Dead code detection - @echo "๐Ÿงน vulture โ€ฆ" && $(VENV_DIR)/bin/vulture mcpgateway --min-confidence 80 + @echo "๐Ÿงน vulture $(TARGET) โ€ฆ" && $(VENV_DIR)/bin/vulture $(TARGET) --min-confidence 80 + +# Shell script linting for individual files +shell-lint-file: ## ๐Ÿš Lint shell script + @if [ -f "$(TARGET)" ]; then \ + echo "๐Ÿš Linting shell script: $(TARGET)"; \ + if command -v shellcheck >/dev/null 2>&1; then \ + shellcheck "$(TARGET)" || true; \ + else \ + echo "โš ๏ธ shellcheck not installed - skipping"; \ + fi; \ + if command -v shfmt >/dev/null 2>&1; then \ + shfmt -d -i 4 -ci "$(TARGET)" || true; \ + elif [ -f "$(HOME)/go/bin/shfmt" ]; then \ + $(HOME)/go/bin/shfmt -d -i 4 -ci "$(TARGET)" || true; \ + else \ + echo "โš ๏ธ shfmt not installed - skipping"; \ + fi; \ + else \ + echo "โŒ $(TARGET) is not a file"; \ + fi + +# ----------------------------------------------------------------------------- +# ๐Ÿ” LINT CHANGED FILES (GIT INTEGRATION) +# ----------------------------------------------------------------------------- +# help: lint-changed - Lint only git-changed files +# help: lint-staged - Lint only git-staged files +# help: lint-commit - Lint files in specific commit (use COMMIT=hash) +.PHONY: lint-changed lint-staged lint-commit + +lint-changed: ## ๐Ÿ” Lint only changed files (git) + @echo "๐Ÿ” Linting changed files..." + @changed_files=$$(git diff --name-only --diff-filter=ACM HEAD 2>/dev/null || true); \ + if [ -z "$$changed_files" ]; then \ + echo "โ„น๏ธ No changed files to lint"; \ + else \ + echo "Changed files:"; \ + echo "$$changed_files" | sed 's/^/ - /'; \ + echo ""; \ + for file in $$changed_files; do \ + if [ -e "$$file" ]; then \ + echo "๐ŸŽฏ Linting: $$file"; \ + $(MAKE) --no-print-directory lint-smart "$$file"; \ + fi; \ + done; \ + fi + +lint-staged: ## ๐Ÿ” Lint only staged files (git) + @echo "๐Ÿ” Linting staged files..." + @staged_files=$$(git diff --name-only --cached --diff-filter=ACM 2>/dev/null || true); \ + if [ -z "$$staged_files" ]; then \ + echo "โ„น๏ธ No staged files to lint"; \ + else \ + echo "Staged files:"; \ + echo "$$staged_files" | sed 's/^/ - /'; \ + echo ""; \ + for file in $$staged_files; do \ + if [ -e "$$file" ]; then \ + echo "๐ŸŽฏ Linting: $$file"; \ + $(MAKE) --no-print-directory lint-smart "$$file"; \ + fi; \ + done; \ + fi + +# Lint files in specific commit (use COMMIT=hash) +COMMIT ?= HEAD +lint-commit: ## ๐Ÿ” Lint files changed in commit + @echo "๐Ÿ” Linting files changed in commit $(COMMIT)..." + @commit_files=$$(git diff-tree --no-commit-id --name-only -r $(COMMIT) 2>/dev/null || true); \ + if [ -z "$$commit_files" ]; then \ + echo "โ„น๏ธ No files found in commit $(COMMIT)"; \ + else \ + echo "Files in commit $(COMMIT):"; \ + echo "$$commit_files" | sed 's/^/ - /'; \ + echo ""; \ + for file in $$commit_files; do \ + if [ -e "$$file" ]; then \ + echo "๐ŸŽฏ Linting: $$file"; \ + $(MAKE) --no-print-directory lint-smart "$$file"; \ + fi; \ + done; \ + fi + +# ----------------------------------------------------------------------------- +# ๐Ÿ‘๏ธ WATCH MODE - LINT ON FILE CHANGES +# ----------------------------------------------------------------------------- +# help: lint-watch - Watch files for changes and auto-lint +# help: lint-watch-quick - Watch files with quick linting only +.PHONY: lint-watch lint-watch-quick install-watchdog + +install-watchdog: ## ๐Ÿ“ฆ Install watchdog for file watching + @echo "๐Ÿ“ฆ Installing watchdog for file watching..." + @test -d "$(VENV_DIR)" || $(MAKE) venv + @/bin/bash -c "source $(VENV_DIR)/bin/activate && \ + python3 -m pip install -q watchdog" + +# Watch mode - lint on file changes +lint-watch: install-watchdog ## ๐Ÿ‘๏ธ Watch for changes and auto-lint + @echo "๐Ÿ‘๏ธ Watching $(TARGET) for changes (Ctrl+C to stop)..." + @echo "๐Ÿ’ก Will run 'make lint-smart' on changed Python files" + @test -d "$(VENV_DIR)" || $(MAKE) venv + @/bin/bash -c "source $(VENV_DIR)/bin/activate && \ + $(VENV_DIR)/bin/watchmedo shell-command \ + --patterns='*.py;*.yaml;*.yml;*.json;*.md;*.toml' \ + --recursive \ + --command='echo \"๐Ÿ“ File changed: \$${watch_src_path}\" && make --no-print-directory lint-smart \"\$${watch_src_path}\"' \ + $(TARGET)" + +# Watch mode with quick linting only +lint-watch-quick: install-watchdog ## ๐Ÿ‘๏ธ Watch for changes and quick-lint + @echo "๐Ÿ‘๏ธ Quick-watching $(TARGET) for changes (Ctrl+C to stop)..." + @echo "๐Ÿ’ก Will run 'make lint-quick' on changed Python files" + @test -d "$(VENV_DIR)" || $(MAKE) venv + @/bin/bash -c "source $(VENV_DIR)/bin/activate && \ + $(VENV_DIR)/bin/watchmedo shell-command \ + --patterns='*.py' \ + --recursive \ + --command='echo \"โšก File changed: \$${watch_src_path}\" && make --no-print-directory lint-quick \"\$${watch_src_path}\"' \ + $(TARGET)" + +# ----------------------------------------------------------------------------- +# ๐Ÿšจ STRICT LINTING WITH ERROR THRESHOLDS +# ----------------------------------------------------------------------------- +# help: lint-strict - Lint with error threshold (fail on errors) +# help: lint-count-errors - Count and report linting errors +# help: lint-report - Generate detailed linting report +.PHONY: lint-strict lint-count-errors lint-report + +# Lint with error threshold +lint-strict: ## ๐Ÿšจ Lint with strict error checking + @echo "๐Ÿšจ Running strict linting on $(TARGET)..." + @mkdir -p $(DOCS_DIR)/reports + @$(MAKE) lint TARGET="$(TARGET)" 2>&1 | tee $(DOCS_DIR)/reports/lint-report.txt + @errors=$$(grep -ic "error\|failed\|โŒ" $(DOCS_DIR)/reports/lint-report.txt 2>/dev/null || echo 0); \ + warnings=$$(grep -ic "warning\|warn\|โš ๏ธ" $(DOCS_DIR)/reports/lint-report.txt 2>/dev/null || echo 0); \ + echo ""; \ + echo "๐Ÿ“Š Linting Summary:"; \ + echo " โŒ Errors: $$errors"; \ + echo " โš ๏ธ Warnings: $$warnings"; \ + if [ $$errors -gt 0 ]; then \ + echo ""; \ + echo "โŒ Linting failed with $$errors errors"; \ + echo "๐Ÿ“„ Full report: $(DOCS_DIR)/reports/lint-report.txt"; \ + exit 1; \ + else \ + echo "โœ… All linting checks passed!"; \ + fi + +# Count errors from different linters +lint-count-errors: ## ๐Ÿ“Š Count linting errors by tool + @echo "๐Ÿ“Š Counting linting errors by tool..." + @mkdir -p $(DOCS_DIR)/reports + @echo "# Linting Error Report - $$(date)" > $(DOCS_DIR)/reports/error-count.md + @echo "" >> $(DOCS_DIR)/reports/error-count.md + @echo "| Tool | Errors | Warnings |" >> $(DOCS_DIR)/reports/error-count.md + @echo "|------|--------|----------|" >> $(DOCS_DIR)/reports/error-count.md + @for tool in flake8 pylint mypy bandit ruff; do \ + echo "๐Ÿ” Checking $$tool errors..."; \ + errors=0; warnings=0; \ + if $(MAKE) --no-print-directory $$tool TARGET="$(TARGET)" 2>&1 | tee /tmp/$$tool.log >/dev/null; then \ + errors=$$(grep -c "error:" /tmp/$$tool.log 2>/dev/null || echo 0); \ + warnings=$$(grep -c "warning:" /tmp/$$tool.log 2>/dev/null || echo 0); \ + fi; \ + echo "| $$tool | $$errors | $$warnings |" >> $(DOCS_DIR)/reports/error-count.md; \ + rm -f /tmp/$$tool.log; \ + done + @echo "" >> $(DOCS_DIR)/reports/error-count.md + @echo "Generated: $$(date)" >> $(DOCS_DIR)/reports/error-count.md + @cat $(DOCS_DIR)/reports/error-count.md + @echo "๐Ÿ“„ Report saved: $(DOCS_DIR)/reports/error-count.md" + +# Generate comprehensive linting report +lint-report: ## ๐Ÿ“‹ Generate comprehensive linting report + @echo "๐Ÿ“‹ Generating comprehensive linting report..." + @mkdir -p $(DOCS_DIR)/reports + @echo "# Comprehensive Linting Report" > $(DOCS_DIR)/reports/full-lint-report.md + @echo "Generated: $$(date)" >> $(DOCS_DIR)/reports/full-lint-report.md + @echo "" >> $(DOCS_DIR)/reports/full-lint-report.md + @echo "## Target: $(TARGET)" >> $(DOCS_DIR)/reports/full-lint-report.md + @echo "" >> $(DOCS_DIR)/reports/full-lint-report.md + @echo "## Quick Summary" >> $(DOCS_DIR)/reports/full-lint-report.md + @$(MAKE) --no-print-directory lint-quick TARGET="$(TARGET)" >> $(DOCS_DIR)/reports/full-lint-report.md 2>&1 || true + @echo "" >> $(DOCS_DIR)/reports/full-lint-report.md + @echo "## Detailed Analysis" >> $(DOCS_DIR)/reports/full-lint-report.md + @$(MAKE) --no-print-directory lint TARGET="$(TARGET)" >> $(DOCS_DIR)/reports/full-lint-report.md 2>&1 || true + @echo "" >> $(DOCS_DIR)/reports/full-lint-report.md + @echo "## Error Count by Tool" >> $(DOCS_DIR)/reports/full-lint-report.md + @$(MAKE) --no-print-directory lint-count-errors TARGET="$(TARGET)" >> $(DOCS_DIR)/reports/full-lint-report.md 2>&1 || true + @echo "๐Ÿ“„ Report generated: $(DOCS_DIR)/reports/full-lint-report.md" + +# ----------------------------------------------------------------------------- +# ๐Ÿ”ง PRE-COMMIT INTEGRATION +# ----------------------------------------------------------------------------- +# help: lint-install-hooks - Install git pre-commit hooks for linting +# help: lint-pre-commit - Run linting as pre-commit check +# help: lint-pre-push - Run linting as pre-push check +.PHONY: lint-install-hooks lint-pre-commit lint-pre-push + +# Install git hooks for linting +lint-install-hooks: ## ๐Ÿ”ง Install git hooks for auto-linting + @echo "๐Ÿ”ง Installing git pre-commit hooks for linting..." + @if [ ! -d ".git" ]; then \ + echo "โŒ Not a git repository"; \ + exit 1; \ + fi + @echo '#!/bin/bash' > .git/hooks/pre-commit + @echo '# Auto-generated pre-commit hook for linting' >> .git/hooks/pre-commit + @echo 'echo "๐Ÿ” Running pre-commit linting..."' >> .git/hooks/pre-commit + @echo 'make lint-pre-commit' >> .git/hooks/pre-commit + @chmod +x .git/hooks/pre-commit + @echo '#!/bin/bash' > .git/hooks/pre-push + @echo '# Auto-generated pre-push hook for linting' >> .git/hooks/pre-push + @echo 'echo "๐Ÿ” Running pre-push linting..."' >> .git/hooks/pre-push + @echo 'make lint-pre-push' >> .git/hooks/pre-push + @chmod +x .git/hooks/pre-push + @echo "โœ… Git hooks installed:" + @echo " ๐Ÿ“ pre-commit: .git/hooks/pre-commit" + @echo " ๐Ÿ“ค pre-push: .git/hooks/pre-push" + @echo "๐Ÿ’ก To disable: rm .git/hooks/pre-commit .git/hooks/pre-push" + +# Pre-commit hook (lint staged files) +lint-pre-commit: ## ๐Ÿ” Pre-commit linting check + @echo "๐Ÿ” Pre-commit linting check..." + @$(MAKE) --no-print-directory lint-staged + @echo "โœ… Pre-commit linting passed!" + +# Pre-push hook (lint all changed files) +lint-pre-push: ## ๐Ÿ” Pre-push linting check + @echo "๐Ÿ” Pre-push linting check..." + @$(MAKE) --no-print-directory lint-changed + @echo "โœ… Pre-push linting passed!" + +# ----------------------------------------------------------------------------- +# ๐ŸŽฏ FILE TYPE SPECIFIC LINTING +# ----------------------------------------------------------------------------- +# Lint only Python files in target +lint-py: ## ๐Ÿ Lint only Python files + @echo "๐Ÿ Linting Python files in $(TARGET)..." + @for target in $(DEFAULT_TARGETS); do \ + if [ -f "$$target" ] && echo "$$target" | grep -qE '\.py$$'; then \ + echo "๐ŸŽฏ Linting Python file: $$target"; \ + $(MAKE) --no-print-directory lint-target TARGET="$$target"; \ + elif [ -d "$$target" ]; then \ + echo "๐Ÿ” Finding Python files in: $$target"; \ + find "$$target" -name "*.py" -type f | while read f; do \ + echo "๐ŸŽฏ Linting: $$f"; \ + $(MAKE) --no-print-directory lint-target TARGET="$$f"; \ + done; \ + else \ + echo "โš ๏ธ Skipping non-existent target: $$target"; \ + fi; \ + done + echo "โš ๏ธ Skipping non-existent target: $$target"; \ + fi; \ + done + exit 1; \ + fi + +# Lint only YAML files +lint-yaml: ## ๐Ÿ“„ Lint only YAML files + @echo "๐Ÿ“„ Linting YAML files in $(TARGET)..." + @for target in $(DEFAULT_TARGETS); do \ + if [ -f "$$target" ] && echo "$$target" | grep -qE '\.(yaml|yml)$$'; then \ + $(MAKE) --no-print-directory yamllint TARGET="$$target"; \ + elif [ -d "$$target" ]; then \ + find "$$target" -name "*.yaml" -o -name "*.yml" | while read f; do \ + echo "๐ŸŽฏ Linting: $$f"; \ + $(MAKE) --no-print-directory yamllint TARGET="$$f"; \ + done; \ + else \ + echo "โš ๏ธ Skipping non-existent target: $$target"; \ + fi; \ + done + fi + +# Lint only JSON files +lint-json: ## ๐Ÿ“„ Lint only JSON files + @echo "๐Ÿ“„ Linting JSON files in $(TARGET)..." + @for target in $(DEFAULT_TARGETS); do \ + if [ -f "$$target" ] && echo "$$target" | grep -qE '\.json$$'; then \ + $(MAKE) --no-print-directory jsonlint TARGET="$$target"; \ + elif [ -d "$$target" ]; then \ + find "$$target" -name "*.json" | while read f; do \ + echo "๐ŸŽฏ Linting: $$f"; \ + $(MAKE) --no-print-directory jsonlint TARGET="$$f"; \ + done; \ + else \ + echo "โš ๏ธ Skipping non-existent target: $$target"; \ + fi; \ + done + fi + +# Lint only Markdown files +lint-md: ## ๐Ÿ“ Lint only Markdown files + @echo "๐Ÿ“ Linting Markdown files in $(TARGET)..." + @for target in $(DEFAULT_TARGETS); do \ + if [ -f "$$target" ] && echo "$$target" | grep -qE '\.(md|markdown)$$'; then \ + $(MAKE) --no-print-directory markdownlint TARGET="$$target"; \ + elif [ -d "$$target" ]; then \ + find "$$target" -name "*.md" -o -name "*.markdown" | while read f; do \ + echo "๐ŸŽฏ Linting: $$f"; \ + $(MAKE) --no-print-directory markdownlint TARGET="$$f"; \ + done; \ + else \ + echo "โš ๏ธ Skipping non-existent target: $$target"; \ + fi; \ + done + fi + +# ----------------------------------------------------------------------------- +# ๐Ÿš€ PERFORMANCE OPTIMIZATION +# ----------------------------------------------------------------------------- +# help: lint-parallel - Run linters in parallel for speed +# help: lint-cache-clear - Clear linting caches +.PHONY: lint-parallel lint-cache-clear + +# Parallel linting for better performance +lint-parallel: ## ๐Ÿš€ Run linters in parallel + @echo "๐Ÿš€ Running linters in parallel on $(TARGET)..." + @test -d "$(VENV_DIR)" || $(MAKE) venv + @/bin/bash -c "source $(VENV_DIR)/bin/activate && \ + python3 -m pip install -q pytest-xdist" + @# Run fast linters in parallel + @$(MAKE) --no-print-directory ruff-check TARGET="$(TARGET)" & \ + $(MAKE) --no-print-directory black-check TARGET="$(TARGET)" & \ + $(MAKE) --no-print-directory isort-check TARGET="$(TARGET)" & \ + wait + @echo "โœ… Parallel linting completed!" + +# Clear linting caches +lint-cache-clear: ## ๐Ÿงน Clear linting caches + @echo "๐Ÿงน Clearing linting caches..." + @rm -rf .mypy_cache .ruff_cache .pytest_cache __pycache__ + @find . -name "*.pyc" -delete + @find . -name "__pycache__" -type d -exec rm -rf {} + 2>/dev/null || true + @echo "โœ… Linting caches cleared!" + +# ----------------------------------------------------------------------------- +# ๐Ÿ“Š LINTING STATISTICS AND METRICS +# ----------------------------------------------------------------------------- +# help: lint-stats - Show linting statistics +# help: lint-complexity - Analyze code complexity +.PHONY: lint-stats lint-complexity + +# Show linting statistics +lint-stats: ## ๐Ÿ“Š Show linting statistics + @echo "๐Ÿ“Š Linting statistics for $(TARGET)..." + @echo "" + @echo "๐Ÿ“ File counts:" + @if [ -d "$(TARGET)" ]; then \ + echo " ๐Ÿ Python files: $$(find $(TARGET) -name '*.py' | wc -l)"; \ + echo " ๐Ÿ“„ YAML files: $$(find $(TARGET) -name '*.yaml' -o -name '*.yml' | wc -l)"; \ + echo " ๐Ÿ“„ JSON files: $$(find $(TARGET) -name '*.json' | wc -l)"; \ + echo " ๐Ÿ“ Markdown files: $$(find $(TARGET) -name '*.md' | wc -l)"; \ + elif [ -f "$(TARGET)" ]; then \ + echo " ๐Ÿ“„ Single file: $(TARGET)"; \ + fi + @echo "" + @echo "๐Ÿ” Running quick analysis..." + @$(MAKE) --no-print-directory lint-count-errors TARGET="$(TARGET)" 2>/dev/null || true + +# Analyze code complexity +lint-complexity: ## ๐Ÿ“ˆ Analyze code complexity + @echo "๐Ÿ“ˆ Analyzing code complexity for $(TARGET)..." + @test -d "$(VENV_DIR)" || $(MAKE) venv + @/bin/bash -c "source $(VENV_DIR)/bin/activate && \ + python3 -m pip install -q radon && \ + echo '๐Ÿ“Š Cyclomatic Complexity:' && \ + $(VENV_DIR)/bin/radon cc $(TARGET) -s && \ + echo '' && \ + echo '๐Ÿ“Š Maintainability Index:' && \ + $(VENV_DIR)/bin/radon mi $(TARGET) -s" # ----------------------------------------------------------------------------- # ๐Ÿ“‘ GRYPE SECURITY/VULNERABILITY SCANNING @@ -843,8 +1413,8 @@ osv-scan: osv-scan-source osv-scan-image # help: sonar-deps-docker - Install docker-compose + supporting tools # help: sonar-up-podman - Launch SonarQube with podman-compose # help: sonar-up-docker - Launch SonarQube with docker-compose -# help: sonar-submit-docker - Run containerised Sonar Scanner CLI with Docker -# help: sonar-submit-podman - Run containerised Sonar Scanner CLI with Podman +# help: sonar-submit-docker - Run containerized Sonar Scanner CLI with Docker +# help: sonar-submit-podman - Run containerized Sonar Scanner CLI with Podman # help: pysonar-scanner - Run scan with Python wrapper (pysonar-scanner) # help: sonar-info - How to create a token & which env vars to export @@ -890,9 +1460,9 @@ sonar-up-docker: @sleep 30 && $(COMPOSE_CMD) ps | grep sonarqube || \ echo "โš ๏ธ Server may still be starting." -## โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ Containerised Scanner CLI (Docker / Podman) โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +## โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ Containerized Scanner CLI (Docker / Podman) โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ sonar-submit-docker: - @echo "๐Ÿ“ก Scanning code with containerised Sonar Scanner CLI (Docker) ..." + @echo "๐Ÿ“ก Scanning code with containerized Sonar Scanner CLI (Docker) ..." docker run --rm \ -e SONAR_HOST_URL="$(SONAR_HOST_URL)" \ $(if $(SONAR_TOKEN),-e SONAR_TOKEN="$(SONAR_TOKEN)",) \ @@ -901,7 +1471,7 @@ sonar-submit-docker: -Dproject.settings=$(SONAR_PROPS) sonar-submit-podman: - @echo "๐Ÿ“ก Scanning code with containerised Sonar Scanner CLI (Podman) ..." + @echo "๐Ÿ“ก Scanning code with containerized Sonar Scanner CLI (Podman) ..." podman run --rm \ --network $(SONAR_NETWORK) \ -e SONAR_HOST_URL="$(SONAR_HOST_URL)" \ @@ -2035,7 +2605,7 @@ IMAGE ?= $(IMG):$(TAG) # or IMAGE=ghcr.io/ibm/mcp-context-forge:$(TA # help: minikube-port-forward - Run kubectl port-forward -n mcp-private svc/mcp-stack-mcpgateway 8080:80 # help: minikube-dashboard - Print & (best-effort) open the Kubernetes dashboard URL # help: minikube-image-load - Load $(IMAGE) into Minikube container runtime -# help: minikube-k8s-apply - Apply manifests from k8s/ - access with `kubectl port-forward svc/mcp-context-forge 8080:80` +# help: minikube-k8s-apply - Apply manifests from deployment/k8s/ - access with `kubectl port-forward svc/mcp-context-forge 8080:80` # help: minikube-status - Cluster + addon health overview # help: minikube-context - Switch kubectl context to Minikube # help: minikube-ssh - SSH into the Minikube VM @@ -2126,7 +2696,7 @@ minikube-image-load: minikube-k8s-apply: @echo "๐Ÿงฉ Applying k8s manifests in ./k8s ..." - @kubectl apply -f k8s/ --recursive + @kubectl apply -f deployment/k8s/ --recursive # ----------------------------------------------------------------------------- # ๐Ÿ” Utility: print the current registry URL (host-port) - works after cluster @@ -3090,14 +3660,14 @@ semgrep: ## ๐Ÿ” Security patterns & anti-patterns @test -d "$(VENV_DIR)" || $(MAKE) venv @/bin/bash -c "source $(VENV_DIR)/bin/activate && \ python3 -m pip install -q semgrep && \ - $(VENV_DIR)/bin/semgrep --config=auto mcpgateway tests --exclude-rule python.lang.compatibility.python37.python37-compatibility-importlib2 || true" + $(VENV_DIR)/bin/semgrep --config=auto $(TARGET) --exclude-rule python.lang.compatibility.python37.python37-compatibility-importlib2 || true" dodgy: ## ๐Ÿ” Suspicious code patterns @echo "๐Ÿ” dodgy - scanning for hardcoded secrets..." @test -d "$(VENV_DIR)" || $(MAKE) venv @/bin/bash -c "source $(VENV_DIR)/bin/activate && \ python3 -m pip install -q dodgy && \ - $(VENV_DIR)/bin/dodgy mcpgateway tests || true" + $(VENV_DIR)/bin/dodgy $(TARGET) || true" dlint: ## ๐Ÿ“ Python best practices @echo "๐Ÿ“ dlint - checking Python best practices..." @@ -3111,8 +3681,8 @@ pyupgrade: ## โฌ†๏ธ Upgrade Python syntax @test -d "$(VENV_DIR)" || $(MAKE) venv @/bin/bash -c "source $(VENV_DIR)/bin/activate && \ python3 -m pip install -q pyupgrade && \ - find mcpgateway tests -name '*.py' -exec $(VENV_DIR)/bin/pyupgrade --py312-plus --diff {} + || true" - @echo "๐Ÿ’ก To apply changes, run: find mcpgateway tests -name '*.py' -exec $(VENV_DIR)/bin/pyupgrade --py312-plus {} +" + find $(TARGET) -name '*.py' -exec $(VENV_DIR)/bin/pyupgrade --py312-plus --diff {} + || true" + @echo "๐Ÿ’ก To apply changes, run: find $(TARGET) -name '*.py' -exec $(VENV_DIR)/bin/pyupgrade --py312-plus {} +" interrogate: ## ๐Ÿ“ Docstring coverage @echo "๐Ÿ“ interrogate - checking docstring coverage..." @@ -3234,12 +3804,12 @@ security-report: ## ๐Ÿ“Š Generate comprehensive security repo @echo "## Code Security Patterns (semgrep)" >> $(DOCS_DIR)/docs/security/report.md @/bin/bash -c "source $(VENV_DIR)/bin/activate && \ python3 -m pip install -q semgrep && \ - $(VENV_DIR)/bin/semgrep --config=auto mcpgateway tests --quiet || true" >> $(DOCS_DIR)/docs/security/report.md 2>&1 + $(VENV_DIR)/bin/semgrep --config=auto $(TARGET) --quiet || true" >> $(DOCS_DIR)/docs/security/report.md 2>&1 @echo "" >> $(DOCS_DIR)/docs/security/report.md @echo "## Suspicious Code Patterns (dodgy)" >> $(DOCS_DIR)/docs/security/report.md @/bin/bash -c "source $(VENV_DIR)/bin/activate && \ python3 -m pip install -q dodgy && \ - $(VENV_DIR)/bin/dodgy mcpgateway tests || true" >> $(DOCS_DIR)/docs/security/report.md 2>&1 + $(VENV_DIR)/bin/dodgy $(TARGET) || true" >> $(DOCS_DIR)/docs/security/report.md 2>&1 @echo "" >> $(DOCS_DIR)/docs/security/report.md @echo "## DevSkim Security Anti-patterns" >> $(DOCS_DIR)/docs/security/report.md @if command -v devskim >/dev/null 2>&1 || [ -f "$$HOME/.dotnet/tools/devskim" ]; then \ @@ -3255,7 +3825,7 @@ security-fix: ## ๐Ÿ”ง Auto-fix security issues where possi @echo "โžค Upgrading Python syntax with pyupgrade..." @/bin/bash -c "source $(VENV_DIR)/bin/activate && \ python3 -m pip install -q pyupgrade && \ - find mcpgateway tests -name '*.py' -exec $(VENV_DIR)/bin/pyupgrade --py312-plus {} +" + find $(TARGET) -name '*.py' -exec $(VENV_DIR)/bin/pyupgrade --py312-plus {} +" @echo "โžค Updating dependencies to latest secure versions..." @/bin/bash -c "source $(VENV_DIR)/bin/activate && \ python3 -m pip install --upgrade pip setuptools && \ @@ -3502,3 +4072,118 @@ snyk-helm-test: ## โŽˆ Test Helm charts for security issues else \ echo "โš ๏ธ No Helm charts found in charts/mcp-stack/"; \ fi + +# ============================================================================== +# ๐Ÿ” HEADER MANAGEMENT - Check and fix Python file headers +# ============================================================================== +# help: ๐Ÿ” HEADER MANAGEMENT - Check and fix Python file headers +# help: check-headers - Check all Python file headers (dry run - default) +# help: check-headers-diff - Check headers and show diff preview +# help: check-headers-debug - Check headers with debug information +# help: check-header - Check specific file/directory (use: path=...) +# help: fix-all-headers - Fix ALL files with incorrect headers (modifies files!) +# help: fix-all-headers-no-encoding - Fix headers without encoding line requirement +# help: fix-all-headers-custom - Fix with custom config (year=YYYY license=... shebang=...) +# help: interactive-fix-headers - Fix headers with prompts before each change +# help: fix-header - Fix specific file/directory (use: path=... authors=...) +# help: pre-commit-check-headers - Check headers for pre-commit hooks +# help: pre-commit-fix-headers - Fix headers for pre-commit hooks + +.PHONY: check-headers fix-all-headers interactive-fix-headers fix-header check-headers-diff check-header \ + check-headers-debug fix-all-headers-no-encoding fix-all-headers-custom \ + pre-commit-check-headers pre-commit-fix-headers + +## --------------------------------------------------------------------------- ## +## Check modes (no modifications) +## --------------------------------------------------------------------------- ## +check-headers: ## ๐Ÿ” Check all Python file headers (dry run - default) + @echo "๐Ÿ” Checking Python file headers (dry run - no files will be modified)..." + @python3 .github/tools/fix_file_headers.py + +check-headers-diff: ## ๐Ÿ” Check headers and show diff preview + @echo "๐Ÿ” Checking Python file headers with diff preview..." + @python3 .github/tools/fix_file_headers.py --show-diff + +check-headers-debug: ## ๐Ÿ” Check headers with debug information + @echo "๐Ÿ” Checking Python file headers with debug info..." + @python3 .github/tools/fix_file_headers.py --debug + +check-header: ## ๐Ÿ” Check specific file/directory (use: path=... debug=1 diff=1) + @if [ -z "$(path)" ]; then \ + echo "โŒ Error: 'path' parameter is required"; \ + echo "๐Ÿ’ก Usage: make check-header path= [debug=1] [diff=1]"; \ + exit 1; \ + fi + @echo "๐Ÿ” Checking headers in $(path) (dry run)..." + @extra_args=""; \ + if [ "$(debug)" = "1" ]; then \ + extra_args="$$extra_args --debug"; \ + fi; \ + if [ "$(diff)" = "1" ]; then \ + extra_args="$$extra_args --show-diff"; \ + fi; \ + python3 .github/tools/fix_file_headers.py --check --path "$(path)" $$extra_args + +## --------------------------------------------------------------------------- ## +## Fix modes (will modify files) +## --------------------------------------------------------------------------- ## +fix-all-headers: ## ๐Ÿ”ง Fix ALL files with incorrect headers (โš ๏ธ modifies files!) + @echo "โš ๏ธ WARNING: This will modify all Python files with incorrect headers!" + @echo "๐Ÿ”ง Automatically fixing all Python file headers..." + @python3 .github/tools/fix_file_headers.py --fix-all + +fix-all-headers-no-encoding: ## ๐Ÿ”ง Fix headers without encoding line requirement + @echo "๐Ÿ”ง Fixing headers without encoding line requirement..." + @python3 .github/tools/fix_file_headers.py --fix-all --no-encoding + +fix-all-headers-custom: ## ๐Ÿ”ง Fix with custom config (year=YYYY license=... shebang=...) + @echo "๐Ÿ”ง Fixing headers with custom configuration..." + @if [ -n "$(year)" ]; then \ + extra_args="$$extra_args --copyright-year $(year)"; \ + fi; \ + if [ -n "$(license)" ]; then \ + extra_args="$$extra_args --license $(license)"; \ + fi; \ + if [ -n "$(shebang)" ]; then \ + extra_args="$$extra_args --require-shebang $(shebang)"; \ + fi; \ + python3 .github/tools/fix_file_headers.py --fix-all $$extra_args + +interactive-fix-headers: ## ๐Ÿ’ฌ Fix headers with prompts before each change + @echo "๐Ÿ’ฌ Interactively fixing Python file headers..." + @echo "You will be prompted before each change." + @python3 .github/tools/fix_file_headers.py --interactive + +fix-header: ## ๐Ÿ”ง Fix specific file/directory (use: path=... authors=... shebang=... encoding=no) + @if [ -z "$(path)" ]; then \ + echo "โŒ Error: 'path' parameter is required"; \ + echo "๐Ÿ’ก Usage: make fix-header path= [authors=\"Name1, Name2\"] [shebang=auto|always|never] [encoding=no]"; \ + exit 1; \ + fi + @echo "๐Ÿ”ง Fixing headers in $(path)" + @echo "โš ๏ธ This will modify the file(s)!" + @extra_args=""; \ + if [ -n "$(authors)" ]; then \ + echo " Authors: $(authors)"; \ + extra_args="$$extra_args --authors \"$(authors)\""; \ + fi; \ + if [ -n "$(shebang)" ]; then \ + echo " Shebang requirement: $(shebang)"; \ + extra_args="$$extra_args --require-shebang $(shebang)"; \ + fi; \ + if [ "$(encoding)" = "no" ]; then \ + echo " Encoding line: not required"; \ + extra_args="$$extra_args --no-encoding"; \ + fi; \ + eval python3 .github/tools/fix_file_headers.py --fix --path "$(path)" $$extra_args + +## --------------------------------------------------------------------------- ## +## Pre-commit integration +## --------------------------------------------------------------------------- ## +pre-commit-check-headers: ## ๐Ÿช Check headers for pre-commit hooks + @echo "๐Ÿช Checking headers for pre-commit..." + @python3 .github/tools/fix_file_headers.py --check + +pre-commit-fix-headers: ## ๐Ÿช Fix headers for pre-commit hooks + @echo "๐Ÿช Fixing headers for pre-commit..." + @python3 .github/tools/fix_file_headers.py --fix-all diff --git a/docs/docs/development/.pages b/docs/docs/development/.pages index b3d5ea545..d111543d8 100644 --- a/docs/docs/development/.pages +++ b/docs/docs/development/.pages @@ -8,3 +8,4 @@ nav: - packaging.md - developer-workstation.md - doctest-coverage.md + - module-documentation.md diff --git a/docs/docs/development/module-documentation.md b/docs/docs/development/module-documentation.md new file mode 100644 index 000000000..854842d42 --- /dev/null +++ b/docs/docs/development/module-documentation.md @@ -0,0 +1,160 @@ +# Module documentation + +## โœ๏ธ File Header Management + +To ensure consistency, all Python source files must include a standardized header containing metadata like copyright, license, and authors. We use a script to automate the checking and fixing of these headers. + +**By default, the script runs in check mode (dry run) and will NOT modify any files unless explicitly told to do so with fix flags.** + +### ๐Ÿ” Checking Headers (No Modifications) + +These commands only check files and report issues without making any changes: + +* **`make check-headers`**: + Scans all Python files in `mcpgateway/` and `tests/` and reports any files with missing or incorrect headers. This is the default behavior. + + ```bash + make check-headers + ``` + +* **`make check-headers-diff`**: + Same as `check-headers` but also shows a diff preview of what would be changed. + + ```bash + make check-headers-diff + ``` + +* **`make check-headers-debug`**: + Checks headers with additional debug information (file permissions, shebang status, etc.). + + ```bash + make check-headers-debug + ``` + +* **`make check-header`**: + Check a specific file or directory without modifying it. + + ```bash + # Check a single file + make check-header path="mcpgateway/main.py" + + # Check with debug info and diff preview + make check-header path="tests/" debug=1 diff=1 + ``` + +### ๐Ÿ”ง Fixing Headers (Will Modify Files) + +**โš ๏ธ WARNING**: These commands WILL modify your files. Always commit your changes before running fix commands. + +* **`make fix-all-headers`**: + Automatically fixes all Python files with incorrect headers across the entire project. + + ```bash + make fix-all-headers + ``` + +* **`make fix-all-headers-no-encoding`**: + Fix all headers but don't require the encoding line (`# -*- coding: utf-8 -*-`). + + ```bash + make fix-all-headers-no-encoding + ``` + +* **`make fix-all-headers-custom`**: + Fix all headers with custom configuration options. + + ```bash + # Custom copyright year + make fix-all-headers-custom year=2024 + + # Custom license + make fix-all-headers-custom license=MIT + + # Custom shebang requirement + make fix-all-headers-custom shebang=always + + # Combine multiple options + make fix-all-headers-custom year=2024 license=MIT shebang=never + ``` + +* **`make interactive-fix-headers`**: + Scans all files and prompts for confirmation before applying each fix. This gives you full control over which files are modified. + + ```bash + make interactive-fix-headers + ``` + +* **`make fix-header`**: + Fix headers for a specific file or directory with various options. + + ```bash + # Fix a single file + make fix-header path="mcpgateway/main.py" + + # Fix all files in a directory + make fix-header path="tests/unit/" + + # Fix with specific authors + make fix-header path="mcpgateway/models.py" authors="John Doe, Jane Smith" + + # Fix with custom shebang requirement + make fix-header path="scripts/" shebang=always + + # Fix without encoding line + make fix-header path="lib/helper.py" encoding=no + + # Combine multiple options + make fix-header path="mcpgateway/" authors="Team Alpha" shebang=auto encoding=no + ``` + +### ๐Ÿ“‹ Header Format + +The standardized header format is: + +```python +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +"""Module Description. +Location: ./relative/path/to/file.py +Copyright 2025 +SPDX-License-Identifier: Apache-2.0 +Authors: Author Name(s) + +Your module documentation continues here... +""" +``` + +### โš™๏ธ Configuration Options + +* **`authors`**: Specify author name(s) for the header +* **`shebang`**: Control shebang requirement + - `auto` (default): Only required for executable files + - `always`: Always require shebang line + - `never`: Never require shebang line +* **`encoding`**: Set to `no` to skip encoding line requirement +* **`year`**: Override copyright year (for `fix-all-headers-custom`) +* **`license`**: Override license identifier (for `fix-all-headers-custom`) +* **`debug`**: Set to `1` to show debug information (for check commands) +* **`diff`**: Set to `1` to show diff preview (for check commands) + +### ๐Ÿช Pre-commit Integration + +For use with pre-commit hooks: + +```bash +# Check only (recommended for pre-commit) +make pre-commit-check-headers + +# Fix mode (use with caution) +make pre-commit-fix-headers +``` + +### ๐Ÿ’ก Best Practices + +1. **Always run `check-headers` first** to see what needs to be fixed +2. **Commit your code before running fix commands** to allow easy rollback +3. **Use `interactive-fix-headers`** when you want to review each change +4. **Use `check-headers-diff`** to preview changes before applying them +5. **Executable scripts** should have shebang lines - the script detects this automatically in `auto` mode + +---