From d7f05458878bae094d18c3087a4e16efadbe70b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gonzalo=20Pe=C3=B1a-Castellanos?= Date: Wed, 7 May 2025 20:42:44 -0500 Subject: [PATCH 1/9] Add script to import translations --- .github/workflows/docbuild-and-upload.yml | 3 + .gitignore | 8 ++ web/pandas_translations.py | 89 +++++++++++++++++++++++ 3 files changed, 100 insertions(+) create mode 100644 web/pandas_translations.py diff --git a/.github/workflows/docbuild-and-upload.yml b/.github/workflows/docbuild-and-upload.yml index 294334ca1d54b..7bef8653d3b09 100644 --- a/.github/workflows/docbuild-and-upload.yml +++ b/.github/workflows/docbuild-and-upload.yml @@ -50,6 +50,9 @@ jobs: # https://pytest-qt.readthedocs.io/en/latest/troubleshooting.html#github-actions-azure-pipelines-travis-ci-and-gitlab-ci-cd run: sudo apt-get update && sudo apt-get install -y libegl1 libopengl0 + - name: Download and update translations + run: python web/pandas_translations.py + - name: Test website run: python -m pytest web/ diff --git a/.gitignore b/.gitignore index d951f3fb9cbad..6c0a4b578c599 100644 --- a/.gitignore +++ b/.gitignore @@ -141,3 +141,11 @@ doc/source/savefig/ # Pyodide/WASM related files # ############################## /.pyodide-xbuildenv-* + + +# Web & Translations # +############################## +web/pandas-translations.tar.gz +web/translations/ +web/pandas/pt/ +web/pandas/es/ diff --git a/web/pandas_translations.py b/web/pandas_translations.py new file mode 100644 index 0000000000000..38448fbbb51b1 --- /dev/null +++ b/web/pandas_translations.py @@ -0,0 +1,89 @@ +import os +from pathlib import Path +from subprocess import ( + PIPE, + Popen, +) +import tarfile + +import requests +import yaml + + +def download_translations(url, fname): + """ + Download the translations from the GitHub repository. + """ + response = requests.get(url) + if response.status_code == 200: + with open(fname, "wb") as f: + f.write(response.content) + else: + raise Exception(f"Failed to download translations: {response.status_code}") + + +def extract_translations(fpath, dir_name): + """ + Extract the translations from the tar file. + """ + with tarfile.open(fpath, "r:gz") as tar: + tar.extractall(dir_name) + print(f"Translations extracted to '{dir_name}' directory.") + + +def load_translations_config(path): + """ + Load the translations configuration from the YAML file. + """ + with open(path) as f: + config = yaml.safe_load(f) + return config + + +def load_status_config(path): + """ + Load the translations configuration from the YAML file. + """ + with open(path) as f: + config = yaml.safe_load(f) + return config + + +def copy_translations(data, translation_percentage, dir_name, dest_dir): + """ + Copy the translations to the appropriate directory. + """ + for lang, value in data.items(): + if value["progress"] >= translation_percentage: + language_code = lang[:2] + src_path = ( + f"{dir_name}/pandas-translations-main/web/pandas/{language_code}/" + ) + dest_path = f"{dest_dir}/{language_code}/" + cmds = ["rsync", "-av", "--delete", src_path, dest_path] + p = Popen(cmds, stdout=PIPE, stderr=PIPE) + stdout, stderr = p.communicate() + print(stdout.decode()) + print(stderr.decode()) + + +if __name__ == "__main__": + os.chdir(Path(__file__).parent.parent) + url = "https://github.com/Scientific-Python-Translations/pandas-translations/archive/refs/heads/main.tar.gz" + fpath = "web/pandas-translations.tar.gz" + dir_name = "web/translations" + dest_dir = "web/pandas" + config_path = ( + f"{dir_name}/pandas-translations-main/.github/workflows/sync_translations.yml" + ) + status_path = f"{dir_name}/pandas-translations-main/status.yml" + + download_translations(url, fpath) + extract_translations(fpath, dir_name) + config = load_translations_config(config_path) + variables = config["jobs"]["sync_translations"]["steps"][0]["with"] + translation_percentage = int(variables["translation-percentage"]) + status = load_status_config(status_path) + copy_translations( + status, translation_percentage, dir_name=dir_name, dest_dir=dest_dir + ) From 17063a772bc9fdc0505ceec470b9ee1e08b6053c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gonzalo=20Pe=C3=B1a-Castellanos?= Date: Sun, 11 May 2025 18:03:38 -0500 Subject: [PATCH 2/9] Update scripts to handle tranlsations --- .github/workflows/docbuild-and-upload.yml | 3 - .gitignore | 1 - web/pandas/_templates/layout.html | 6 +- web/pandas/config.yml | 6 + web/pandas/static/js/language_switcher.js | 60 +++++++ web/pandas_translations.py | 89 ---------- web/pandas_web.py | 197 +++++++++++++++++----- 7 files changed, 226 insertions(+), 136 deletions(-) create mode 100644 web/pandas/static/js/language_switcher.js delete mode 100644 web/pandas_translations.py diff --git a/.github/workflows/docbuild-and-upload.yml b/.github/workflows/docbuild-and-upload.yml index 7bef8653d3b09..294334ca1d54b 100644 --- a/.github/workflows/docbuild-and-upload.yml +++ b/.github/workflows/docbuild-and-upload.yml @@ -50,9 +50,6 @@ jobs: # https://pytest-qt.readthedocs.io/en/latest/troubleshooting.html#github-actions-azure-pipelines-travis-ci-and-gitlab-ci-cd run: sudo apt-get update && sudo apt-get install -y libegl1 libopengl0 - - name: Download and update translations - run: python web/pandas_translations.py - - name: Test website run: python -m pytest web/ diff --git a/.gitignore b/.gitignore index 6c0a4b578c599..1d1739424c42e 100644 --- a/.gitignore +++ b/.gitignore @@ -145,7 +145,6 @@ doc/source/savefig/ # Web & Translations # ############################## -web/pandas-translations.tar.gz web/translations/ web/pandas/pt/ web/pandas/es/ diff --git a/web/pandas/_templates/layout.html b/web/pandas/_templates/layout.html index c26b093b0c4ba..82ca9c4c589b9 100644 --- a/web/pandas/_templates/layout.html +++ b/web/pandas/_templates/layout.html @@ -1,5 +1,5 @@ - + pandas - Python Data Analysis Library @@ -15,6 +15,8 @@ href="{{ base_url }}{{ stylesheet }}"> {% endfor %} + +
@@ -50,6 +52,8 @@ {% endif %} {% endfor %} + +
diff --git a/web/pandas/config.yml b/web/pandas/config.yml index cb5447591dab6..90d9b7012e6e7 100644 --- a/web/pandas/config.yml +++ b/web/pandas/config.yml @@ -204,3 +204,9 @@ sponsors: kind: partner roadmap: pdeps_path: pdeps +translations: + url: https://github.com/Scientific-Python-Translations/pandas-translations/archive/refs/heads/main.tar.gz + folder: translations + default_language: 'en' + ignore: + - docs/ diff --git a/web/pandas/static/js/language_switcher.js b/web/pandas/static/js/language_switcher.js new file mode 100644 index 0000000000000..da38aee4f5417 --- /dev/null +++ b/web/pandas/static/js/language_switcher.js @@ -0,0 +1,60 @@ +window.addEventListener("DOMContentLoaded", function() { + var BASE_URL = location.protocol + "//" + location.hostname + ":" + location.port + var CURRENT_LANGUAGE = document.documentElement.lang; + var PATHNAME = location.pathname.replace('/' + CURRENT_LANGUAGE + '/', '') + var languages = JSON.parse(document.getElementById("languages").getAttribute('data-lang').replace(/'/g, '"')); + const language_names = { + 'en': 'English', + 'es': 'Español', + 'fr': 'Français', + 'pt': 'Português' + } + + // Create dropdown menu + function makeDropdown(options) { + var dropdown = document.createElement("li"); + dropdown.classList.add("nav-item"); + dropdown.classList.add("dropdown"); + + var link = document.createElement("a"); + link.classList.add("nav-link"); + link.classList.add("dropdown-toggle"); + link.setAttribute("data-bs-toggle", "dropdown"); + link.setAttribute("href", "#"); + link.setAttribute("role", "button"); + link.setAttribute("aria-haspopup", "true"); + link.setAttribute("aria-expanded", "false"); + link.textContent = language_names[CURRENT_LANGUAGE]; + + var dropdownMenu = document.createElement("div"); + dropdownMenu.classList.add("dropdown-menu"); + + options.forEach(function(i) { + var dropdownItem = document.createElement("a"); + dropdownItem.classList.add("dropdown-item"); + dropdownItem.textContent = language_names[i] || i.toUpperCase(); + dropdownItem.setAttribute("href", "#"); + dropdownItem.addEventListener("click", function() { + if (i == 'en') { + URL_LANGUAGE = ''; + } else { + URL_LANGUAGE = '/' + i; + } + var PATHNAME = location.pathname.replace('/' + CURRENT_LANGUAGE + '/', '/') + var newUrl = BASE_URL + URL_LANGUAGE + PATHNAME + window.location.href = newUrl; + }); + dropdownMenu.appendChild(dropdownItem); + }); + + dropdown.appendChild(link); + dropdown.appendChild(dropdownMenu); + return dropdown; + } + + var container = document.getElementById("language-switcher-container"); + if (container) { + var dropdown = makeDropdown(languages); + container.appendChild(dropdown); + } +}); diff --git a/web/pandas_translations.py b/web/pandas_translations.py deleted file mode 100644 index 38448fbbb51b1..0000000000000 --- a/web/pandas_translations.py +++ /dev/null @@ -1,89 +0,0 @@ -import os -from pathlib import Path -from subprocess import ( - PIPE, - Popen, -) -import tarfile - -import requests -import yaml - - -def download_translations(url, fname): - """ - Download the translations from the GitHub repository. - """ - response = requests.get(url) - if response.status_code == 200: - with open(fname, "wb") as f: - f.write(response.content) - else: - raise Exception(f"Failed to download translations: {response.status_code}") - - -def extract_translations(fpath, dir_name): - """ - Extract the translations from the tar file. - """ - with tarfile.open(fpath, "r:gz") as tar: - tar.extractall(dir_name) - print(f"Translations extracted to '{dir_name}' directory.") - - -def load_translations_config(path): - """ - Load the translations configuration from the YAML file. - """ - with open(path) as f: - config = yaml.safe_load(f) - return config - - -def load_status_config(path): - """ - Load the translations configuration from the YAML file. - """ - with open(path) as f: - config = yaml.safe_load(f) - return config - - -def copy_translations(data, translation_percentage, dir_name, dest_dir): - """ - Copy the translations to the appropriate directory. - """ - for lang, value in data.items(): - if value["progress"] >= translation_percentage: - language_code = lang[:2] - src_path = ( - f"{dir_name}/pandas-translations-main/web/pandas/{language_code}/" - ) - dest_path = f"{dest_dir}/{language_code}/" - cmds = ["rsync", "-av", "--delete", src_path, dest_path] - p = Popen(cmds, stdout=PIPE, stderr=PIPE) - stdout, stderr = p.communicate() - print(stdout.decode()) - print(stderr.decode()) - - -if __name__ == "__main__": - os.chdir(Path(__file__).parent.parent) - url = "https://github.com/Scientific-Python-Translations/pandas-translations/archive/refs/heads/main.tar.gz" - fpath = "web/pandas-translations.tar.gz" - dir_name = "web/translations" - dest_dir = "web/pandas" - config_path = ( - f"{dir_name}/pandas-translations-main/.github/workflows/sync_translations.yml" - ) - status_path = f"{dir_name}/pandas-translations-main/status.yml" - - download_translations(url, fpath) - extract_translations(fpath, dir_name) - config = load_translations_config(config_path) - variables = config["jobs"]["sync_translations"]["steps"][0]["with"] - translation_percentage = int(variables["translation-percentage"]) - status = load_status_config(status_path) - copy_translations( - status, translation_percentage, dir_name=dir_name, dest_dir=dest_dir - ) diff --git a/web/pandas_web.py b/web/pandas_web.py index b3872b829c73a..45a136b288366 100755 --- a/web/pandas_web.py +++ b/web/pandas_web.py @@ -28,6 +28,7 @@ import collections import datetime import importlib +import io import itertools import json import operator @@ -35,7 +36,12 @@ import pathlib import re import shutil +from subprocess import ( + PIPE, + Popen, +) import sys +import tarfile import time import typing @@ -81,11 +87,15 @@ def navbar_add_info(context): ``has_subitems`` that tells which one of them every element is. It also adds a ``slug`` field to be used as a CSS id. """ + ignore = context["translations"]["ignore"] for i, item in enumerate(context["navbar"]): + if item["target"] in ignore: + item["target"] = "/" + item["target"] + context["navbar"][i] = dict( item, has_subitems=isinstance(item["target"], list), - slug=(item["name"].replace(" ", "-").lower()), + slug=item["name"].replace(" ", "-").lower(), ) return context @@ -386,16 +396,25 @@ def get_callable(obj_as_str: str) -> object: return obj -def get_context(config_fname: str, **kwargs): +def get_config(config_fname: str) -> dict: """ - Load the config yaml as the base context, and enrich it with the - information added by the context preprocessors defined in the file. + Load the config yaml file and return it as a dictionary. """ with open(config_fname, encoding="utf-8") as f: context = yaml.safe_load(f) + return context + +def get_context(config_fname: str, **kwargs): + """ + Load the config yaml as the base context, and enrich it with the + information added by the context preprocessors defined in the file. + """ + context = get_config(config_fname) context["source_path"] = os.path.dirname(config_fname) context.update(kwargs) + context["languages"] = context.get("languages", ["en"]) + context["selected_language"] = context.get("language", "en") preprocessors = ( get_callable(context_prep) @@ -409,14 +428,27 @@ def get_context(config_fname: str, **kwargs): return context -def get_source_files(source_path: str) -> typing.Generator[str, None, None]: +def get_source_files( + source_path: str, language, languages +) -> typing.Generator[str, None, None]: """ Generate the list of files present in the source directory. """ + paths = [] + all_languages = languages[:] + all_languages.remove(language) for root, dirs, fnames in os.walk(source_path): root_rel_path = os.path.relpath(root, source_path) for fname in fnames: - yield os.path.join(root_rel_path, fname) + path = os.path.join(root_rel_path, fname) + for language in all_languages: + if path.startswith(language + "/"): + break + else: + paths.append(path) + + for path in paths: + yield path def extend_base_template(content: str, base_template: str) -> str: @@ -431,6 +463,51 @@ def extend_base_template(content: str, base_template: str) -> str: return result +def download_and_extract_translations(url: str, dir_name: str): + """ + Download the translations from the GitHub repository. + """ + response = requests.get(url) + if response.status_code == 200: + doc = io.BytesIO(response.content) + with tarfile.open(None, "r:gz", doc) as tar: + tar.extractall(dir_name) + else: + raise Exception(f"Failed to download translations: {response.status_code}") + + +def get_languages(source_path: str): + """ + Get the list of languages available in the translations directory. + """ + languages_path = f"{source_path}/pandas-translations-main/web/pandas/" + en_path = f"{languages_path}/en/" + if os.path.exists(en_path): + shutil.rmtree(en_path) + + paths = os.listdir(languages_path) + return [path for path in paths if os.path.isdir(f"{languages_path}/{path}")] + + +def copy_translations(source_path: str, target_path: str, languages: list[str]): + """ + Copy the translations to the appropriate directory. + """ + languages_path = f"{source_path}/pandas-translations-main/web/pandas/" + for lang in languages: + cmds = [ + "rsync", + "-av", + "--delete", + f"{languages_path}/{lang}/", + f"{target_path}/{lang}/", + ] + p = Popen(cmds, stdout=PIPE, stderr=PIPE) + stdout, stderr = p.communicate() + sys.stderr.write(stdout.decode()) + sys.stderr.write(stderr.decode()) + + def main( source_path: str, target_path: str, @@ -441,49 +518,85 @@ def main( For ``.md`` and ``.html`` files, render them with the context before copying them. ``.md`` files are transformed to HTML. """ - config_fname = os.path.join(source_path, "config.yml") + base_folder = os.path.dirname(__file__) + base_source_path = source_path + base_target_path = target_path + base_config_fname = os.path.join(source_path, "config.yml") + + config = get_config(base_config_fname) + translations_path = os.path.join(base_folder, f"{config['translations']['folder']}") + + sys.stderr.write("Downloading and extracting translations...\n") + download_and_extract_translations(config["translations"]["url"], translations_path) - shutil.rmtree(target_path, ignore_errors=True) - os.makedirs(target_path, exist_ok=True) + translated_languages = get_languages(translations_path) + default_language = config["translations"]["default_language"] + languages = [default_language] + translated_languages - sys.stderr.write("Generating context...\n") - context = get_context(config_fname, target_path=target_path) - sys.stderr.write("Context generated\n") + sys.stderr.write("Copying translations...\n") + copy_translations(translations_path, source_path, translated_languages) - templates_path = os.path.join(source_path, context["main"]["templates_path"]) - jinja_env = jinja2.Environment(loader=jinja2.FileSystemLoader(templates_path)) + for language in languages: + sys.stderr.write(f"\nProcessing language: {language}...\n\n") + if language != default_language: + target_path = os.path.join(base_target_path, language) + source_path = os.path.join(base_source_path, language) - for fname in get_source_files(source_path): - if os.path.normpath(fname) in context["main"]["ignore"]: - continue + shutil.rmtree(target_path, ignore_errors=True) + os.makedirs(target_path, exist_ok=True) - sys.stderr.write(f"Processing {fname}\n") - dirname = os.path.dirname(fname) - os.makedirs(os.path.join(target_path, dirname), exist_ok=True) + config_fname = os.path.join(source_path, "config.yml") + sys.stderr.write("Generating context...\n") + + context = get_context( + config_fname, + target_path=target_path, + language=language, + languages=languages, + ) + sys.stderr.write("Context generated\n") - extension = os.path.splitext(fname)[-1] - if extension in (".html", ".md"): - with open(os.path.join(source_path, fname), encoding="utf-8") as f: - content = f.read() - if extension == ".md": - body = markdown.markdown( - content, extensions=context["main"]["markdown_extensions"] + templates_path = os.path.join(source_path, context["main"]["templates_path"]) + jinja_env = jinja2.Environment(loader=jinja2.FileSystemLoader(templates_path)) + + for fname in get_source_files(source_path, language, languages): + if os.path.normpath(fname) in context["main"]["ignore"]: + continue + + sys.stderr.write(f"Processing {fname}\n") + dirname = os.path.dirname(fname) + os.makedirs(os.path.join(target_path, dirname), exist_ok=True) + + extension = os.path.splitext(fname)[-1] + if extension in (".html", ".md"): + with open(os.path.join(source_path, fname), encoding="utf-8") as f: + content = f.read() + if extension == ".md": + body = markdown.markdown( + content, extensions=context["main"]["markdown_extensions"] + ) + # Apply Bootstrap's table formatting manually + # Python-Markdown doesn't let us config table attributes by hand + body = body.replace( + "", '
' + ) + content = extend_base_template( + body, context["main"]["base_template"] + ) + context["base_url"] = "".join( + ["../"] * os.path.normpath(fname).count("/") ) - # Apply Bootstrap's table formatting manually - # Python-Markdown doesn't let us config table attributes by hand - body = body.replace("
", '
') - content = extend_base_template(body, context["main"]["base_template"]) - context["base_url"] = "".join(["../"] * os.path.normpath(fname).count("/")) - content = jinja_env.from_string(content).render(**context) - fname_html = os.path.splitext(fname)[0] + ".html" - with open( - os.path.join(target_path, fname_html), "w", encoding="utf-8" - ) as f: - f.write(content) - else: - shutil.copy( - os.path.join(source_path, fname), os.path.join(target_path, dirname) - ) + content = jinja_env.from_string(content).render(**context) + fname_html = os.path.splitext(fname)[0] + ".html" + with open( + os.path.join(target_path, fname_html), "w", encoding="utf-8" + ) as f: + f.write(content) + else: + shutil.copy( + os.path.join(source_path, fname), os.path.join(target_path, dirname) + ) + return 0 if __name__ == "__main__": From 7666eb3878007edea779722423e1181c9c710ea9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gonzalo=20Pe=C3=B1a-Castellanos?= Date: Sun, 11 May 2025 23:02:08 -0500 Subject: [PATCH 3/9] Move translations source path to config --- .gitignore | 3 +- web/pandas/config.yml | 1 + web/pandas/static/js/language_switcher.js | 25 +++++++++-------- web/pandas_web.py | 34 ++++++++++++++--------- 4 files changed, 37 insertions(+), 26 deletions(-) diff --git a/.gitignore b/.gitignore index 1d1739424c42e..9d0d6d1b1d8d1 100644 --- a/.gitignore +++ b/.gitignore @@ -146,5 +146,6 @@ doc/source/savefig/ # Web & Translations # ############################## web/translations/ -web/pandas/pt/ web/pandas/es/ +web/pandas/pt/ +web/pandas/fr/ diff --git a/web/pandas/config.yml b/web/pandas/config.yml index 90d9b7012e6e7..4c253fd7cfa68 100644 --- a/web/pandas/config.yml +++ b/web/pandas/config.yml @@ -207,6 +207,7 @@ roadmap: translations: url: https://github.com/Scientific-Python-Translations/pandas-translations/archive/refs/heads/main.tar.gz folder: translations + source_path: pandas-translations-main/web/pandas/ default_language: 'en' ignore: - docs/ diff --git a/web/pandas/static/js/language_switcher.js b/web/pandas/static/js/language_switcher.js index da38aee4f5417..aad09d351f007 100644 --- a/web/pandas/static/js/language_switcher.js +++ b/web/pandas/static/js/language_switcher.js @@ -1,9 +1,11 @@ window.addEventListener("DOMContentLoaded", function() { - var BASE_URL = location.protocol + "//" + location.hostname + ":" + location.port - var CURRENT_LANGUAGE = document.documentElement.lang; - var PATHNAME = location.pathname.replace('/' + CURRENT_LANGUAGE + '/', '') + var baseUrl = location.protocol + "//" + location.hostname + if (location.port) { + baseUrl = baseUrl + ":" + location.port + } + var currentLanguage = document.documentElement.lang; var languages = JSON.parse(document.getElementById("languages").getAttribute('data-lang').replace(/'/g, '"')); - const language_names = { + const languageNames = { 'en': 'English', 'es': 'Español', 'fr': 'Français', @@ -24,7 +26,7 @@ window.addEventListener("DOMContentLoaded", function() { link.setAttribute("role", "button"); link.setAttribute("aria-haspopup", "true"); link.setAttribute("aria-expanded", "false"); - link.textContent = language_names[CURRENT_LANGUAGE]; + link.textContent = languageNames[currentLanguage]; var dropdownMenu = document.createElement("div"); dropdownMenu.classList.add("dropdown-menu"); @@ -32,16 +34,15 @@ window.addEventListener("DOMContentLoaded", function() { options.forEach(function(i) { var dropdownItem = document.createElement("a"); dropdownItem.classList.add("dropdown-item"); - dropdownItem.textContent = language_names[i] || i.toUpperCase(); + dropdownItem.textContent = languageNames[i] || i.toUpperCase(); dropdownItem.setAttribute("href", "#"); dropdownItem.addEventListener("click", function() { - if (i == 'en') { - URL_LANGUAGE = ''; - } else { - URL_LANGUAGE = '/' + i; + var urlLanguage = ''; + if (i !== 'en') { + urlLanguage = '/' + i; } - var PATHNAME = location.pathname.replace('/' + CURRENT_LANGUAGE + '/', '/') - var newUrl = BASE_URL + URL_LANGUAGE + PATHNAME + var pathName = location.pathname.replace('/' + currentLanguage + '/', '/') + var newUrl = baseUrl + urlLanguage + pathName window.location.href = newUrl; }); dropdownMenu.appendChild(dropdownItem); diff --git a/web/pandas_web.py b/web/pandas_web.py index 45a136b288366..ea9bbcc7a38c5 100755 --- a/web/pandas_web.py +++ b/web/pandas_web.py @@ -467,6 +467,7 @@ def download_and_extract_translations(url: str, dir_name: str): """ Download the translations from the GitHub repository. """ + shutil.rmtree(dir_name, ignore_errors=True) response = requests.get(url) if response.status_code == 200: doc = io.BytesIO(response.content) @@ -480,30 +481,31 @@ def get_languages(source_path: str): """ Get the list of languages available in the translations directory. """ - languages_path = f"{source_path}/pandas-translations-main/web/pandas/" - en_path = f"{languages_path}/en/" + en_path = f"{source_path}/en/" if os.path.exists(en_path): shutil.rmtree(en_path) - paths = os.listdir(languages_path) - return [path for path in paths if os.path.isdir(f"{languages_path}/{path}")] + paths = os.listdir(source_path) + return [path for path in paths if os.path.isdir(f"{source_path}/{path}")] def copy_translations(source_path: str, target_path: str, languages: list[str]): """ Copy the translations to the appropriate directory. """ - languages_path = f"{source_path}/pandas-translations-main/web/pandas/" for lang in languages: + dest = f"{target_path}/{lang}/" + shutil.rmtree(dest, ignore_errors=True) cmds = [ "rsync", "-av", "--delete", - f"{languages_path}/{lang}/", - f"{target_path}/{lang}/", + f"{source_path}/{lang}/", + dest, ] p = Popen(cmds, stdout=PIPE, stderr=PIPE) stdout, stderr = p.communicate() + sys.stderr.write(f"\nCopying: {lang}...\n\n") sys.stderr.write(stdout.decode()) sys.stderr.write(stderr.decode()) @@ -526,15 +528,22 @@ def main( config = get_config(base_config_fname) translations_path = os.path.join(base_folder, f"{config['translations']['folder']}") - sys.stderr.write("Downloading and extracting translations...\n") - download_and_extract_translations(config["translations"]["url"], translations_path) + sys.stderr.write("\nDownloading and extracting translations...\n\n") + translations_extract_path = translations_path + translations_source_path = os.path.join( + translations_path, config["translations"]["source_path"] + ) + + download_and_extract_translations( + config["translations"]["url"], translations_extract_path + ) - translated_languages = get_languages(translations_path) + translated_languages = get_languages(translations_source_path) default_language = config["translations"]["default_language"] languages = [default_language] + translated_languages - sys.stderr.write("Copying translations...\n") - copy_translations(translations_path, source_path, translated_languages) + sys.stderr.write("\nCopying translations...\n") + copy_translations(translations_source_path, source_path, translated_languages) for language in languages: sys.stderr.write(f"\nProcessing language: {language}...\n\n") @@ -596,7 +605,6 @@ def main( shutil.copy( os.path.join(source_path, fname), os.path.join(target_path, dirname) ) - return 0 if __name__ == "__main__": From 30efbbafb05ef5d0c98b408bb4a834b1b3797037 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gonzalo=20Pe=C3=B1a-Castellanos?= Date: Mon, 12 May 2025 13:00:17 -0500 Subject: [PATCH 4/9] Fix language switcher to handle previews --- .gitignore | 1 + web/pandas/static/js/language_switcher.js | 14 ++++++++++++-- web/pandas_web.py | 5 ++++- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 9d0d6d1b1d8d1..0f854f61ae0e5 100644 --- a/.gitignore +++ b/.gitignore @@ -145,6 +145,7 @@ doc/source/savefig/ # Web & Translations # ############################## +web/preview/ web/translations/ web/pandas/es/ web/pandas/pt/ diff --git a/web/pandas/static/js/language_switcher.js b/web/pandas/static/js/language_switcher.js index aad09d351f007..eca361d30e20f 100644 --- a/web/pandas/static/js/language_switcher.js +++ b/web/pandas/static/js/language_switcher.js @@ -1,4 +1,5 @@ window.addEventListener("DOMContentLoaded", function() { + var absBaseUrl = document.baseURI; var baseUrl = location.protocol + "//" + location.hostname if (location.port) { baseUrl = baseUrl + ":" + location.port @@ -12,6 +13,15 @@ window.addEventListener("DOMContentLoaded", function() { 'pt': 'Português' } + // Handle preview URLs on github + // If preview URL changes, this regex will need to be updated + const re = /preview\/pandas-dev\/pandas\/(?[0-9]*)\//g; + var previewUrl = ''; + for (const match of absBaseUrl.matchAll(re)) { + previewUrl = `/preview/pandas-dev/pandas/${match.groups.pr}`; + } + var pathName = location.pathname.replace(previewUrl, '') + // Create dropdown menu function makeDropdown(options) { var dropdown = document.createElement("li"); @@ -41,8 +51,8 @@ window.addEventListener("DOMContentLoaded", function() { if (i !== 'en') { urlLanguage = '/' + i; } - var pathName = location.pathname.replace('/' + currentLanguage + '/', '/') - var newUrl = baseUrl + urlLanguage + pathName + pathName = pathName.replace('/' + currentLanguage + '/', '/') + var newUrl = baseUrl + previewUrl + urlLanguage + pathName window.location.href = newUrl; }); dropdownMenu.appendChild(dropdownItem); diff --git a/web/pandas_web.py b/web/pandas_web.py index ea9bbcc7a38c5..4360f3db54722 100755 --- a/web/pandas_web.py +++ b/web/pandas_web.py @@ -87,10 +87,13 @@ def navbar_add_info(context): ``has_subitems`` that tells which one of them every element is. It also adds a ``slug`` field to be used as a CSS id. """ + lang = context["selected_language"] ignore = context["translations"]["ignore"] + default_language = context["translations"]["default_language"] for i, item in enumerate(context["navbar"]): if item["target"] in ignore: - item["target"] = "/" + item["target"] + if lang != default_language: + item["target"] = "../" + item["target"] context["navbar"][i] = dict( item, From f8e6a020e1ef3a19c1e1a3ab3a7b4635dad05b90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gonzalo=20Pe=C3=B1a-Castellanos?= Date: Tue, 13 May 2025 12:25:44 -0500 Subject: [PATCH 5/9] Simplify scripts and logic --- .github/workflows/docbuild-and-upload.yml | 2 +- web/pandas/config.yml | 33 ----- web/pandas/navbar.yml | 33 +++++ web/pandas_translations.py | 109 +++++++++++++++ web/pandas_web.py | 159 +++++++--------------- 5 files changed, 195 insertions(+), 141 deletions(-) create mode 100644 web/pandas/navbar.yml create mode 100755 web/pandas_translations.py diff --git a/.github/workflows/docbuild-and-upload.yml b/.github/workflows/docbuild-and-upload.yml index 294334ca1d54b..8fbde58b0a462 100644 --- a/.github/workflows/docbuild-and-upload.yml +++ b/.github/workflows/docbuild-and-upload.yml @@ -54,7 +54,7 @@ jobs: run: python -m pytest web/ - name: Build website - run: python web/pandas_web.py web/pandas --target-path=web/build + run: python web/pandas_web.py web/pandas --target-path=web/build --translations - name: Build documentation run: doc/make.py --warnings-are-errors diff --git a/web/pandas/config.yml b/web/pandas/config.yml index 4c253fd7cfa68..35e6ebf429df8 100644 --- a/web/pandas/config.yml +++ b/web/pandas/config.yml @@ -25,39 +25,6 @@ static: css: - static/css/pandas.css - static/css/codehilite.css -navbar: - - name: "About us" - target: - - name: "About pandas" - target: about/ - - name: "Project roadmap" - target: about/roadmap.html - - name: "Governance" - target: about/governance.html - - name: "Team" - target: about/team.html - - name: "Sponsors" - target: about/sponsors.html - - name: "Citing and logo" - target: about/citing.html - - name: "Getting started" - target: getting_started.html - - name: "Documentation" - target: docs/ - - name: "Community" - target: - - name: "Blog" - target: community/blog/ - - name: "Ask a question (StackOverflow)" - target: https://stackoverflow.com/questions/tagged/pandas - - name: "Code of conduct" - target: community/coc.html - - name: "Ecosystem" - target: community/ecosystem.html - - name: "Benchmarks" - target: community/benchmarks.html - - name: "Contribute" - target: contribute.html blog: num_posts: 50 posts_path: community/blog diff --git a/web/pandas/navbar.yml b/web/pandas/navbar.yml new file mode 100644 index 0000000000000..bcc2d062fc3f5 --- /dev/null +++ b/web/pandas/navbar.yml @@ -0,0 +1,33 @@ +navbar: + - name: "About us" + target: + - name: "About pandas" + target: about/ + - name: "Project roadmap" + target: about/roadmap.html + - name: "Governance" + target: about/governance.html + - name: "Team" + target: about/team.html + - name: "Sponsors" + target: about/sponsors.html + - name: "Citing and logo" + target: about/citing.html + - name: "Getting started" + target: getting_started.html + - name: "Documentation" + target: docs/ + - name: "Community" + target: + - name: "Blog" + target: community/blog/ + - name: "Ask a question (StackOverflow)" + target: https://stackoverflow.com/questions/tagged/pandas + - name: "Code of conduct" + target: community/coc.html + - name: "Ecosystem" + target: community/ecosystem.html + - name: "Benchmarks" + target: community/benchmarks.html + - name: "Contribute" + target: contribute.html diff --git a/web/pandas_translations.py b/web/pandas_translations.py new file mode 100755 index 0000000000000..4809e9aecea27 --- /dev/null +++ b/web/pandas_translations.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 +""" +Utilities to download and extract translations from the GitHub repository. +""" + +import io +import os +import shutil +from subprocess import ( + PIPE, + Popen, +) +import sys +import tarfile + +import requests +import yaml + + +def get_config(config_fname: str) -> dict: + """ + Load the config yaml file and return it as a dictionary. + """ + with open(config_fname, encoding="utf-8") as f: + context = yaml.safe_load(f) + return context + + +def download_and_extract_translations(url: str, dir_name: str) -> None: + """ + Download the translations from the GitHub repository. + """ + shutil.rmtree(dir_name, ignore_errors=True) + response = requests.get(url) + if response.status_code == 200: + doc = io.BytesIO(response.content) + with tarfile.open(None, "r:gz", doc) as tar: + tar.extractall(dir_name) + else: + raise Exception(f"Failed to download translations: {response.status_code}") + + +def get_languages(source_path: str) -> list[str]: + """ + Get the list of languages available in the translations directory. + """ + en_path = f"{source_path}/en/" + if os.path.exists(en_path): + shutil.rmtree(en_path) + + paths = os.listdir(source_path) + return [path for path in paths if os.path.isdir(f"{source_path}/{path}")] + + +def remove_translations(source_path: str, languages: list[str]) -> None: + """ + Remove the translations from the source path. + """ + for language in languages: + shutil.rmtree(os.path.join(source_path, language), ignore_errors=True) + + +def copy_translations(source_path: str, target_path: str, languages: list[str]) -> None: + """ + Copy the translations to the appropriate directory. + """ + for lang in languages: + dest = f"{target_path}/{lang}/" + shutil.rmtree(dest, ignore_errors=True) + cmds = [ + "rsync", + "-av", + "--delete", + f"{source_path}/{lang}/", + dest, + ] + p = Popen(cmds, stdout=PIPE, stderr=PIPE) + stdout, stderr = p.communicate() + sys.stderr.write(f"\nCopying: {lang}...\n\n") + sys.stderr.write(stdout.decode()) + sys.stderr.write(stderr.decode()) + + +def process_translations( + config_fname: str, source_path: str, process_translations: bool +) -> tuple[list[str], list[str]]: + """ + Process the translations by downloading and extracting them from + the GitHub repository. + """ + base_folder = os.path.dirname(__file__) + config = get_config(os.path.join(source_path, config_fname)) + translations_path = os.path.join(base_folder, f"{config['translations']['folder']}") + translations_source_path = os.path.join( + translations_path, config["translations"]["source_path"] + ) + default_language = config["translations"]["default_language"] + sys.stderr.write("\nDownloading and extracting translations...\n\n") + download_and_extract_translations(config["translations"]["url"], translations_path) + languages = [default_language] + translated_languages = get_languages(translations_source_path) + remove_translations(source_path, translated_languages) + if process_translations: + languages = [default_language] + translated_languages + + sys.stderr.write("\nCopying translations...\n") + copy_translations(translations_source_path, source_path, translated_languages) + + return translated_languages, languages diff --git a/web/pandas_web.py b/web/pandas_web.py index 4360f3db54722..cc938808b12a3 100755 --- a/web/pandas_web.py +++ b/web/pandas_web.py @@ -28,7 +28,6 @@ import collections import datetime import importlib -import io import itertools import json import operator @@ -36,12 +35,7 @@ import pathlib import re import shutil -from subprocess import ( - PIPE, - Popen, -) import sys -import tarfile import time import typing @@ -71,7 +65,7 @@ class Preprocessors: """ @staticmethod - def current_year(context): + def current_year(context: dict) -> dict: """ Add the current year to the context, so it can be used for the copyright note, or other places where it is needed. @@ -80,13 +74,16 @@ def current_year(context): return context @staticmethod - def navbar_add_info(context): + def navbar_add_info(context: dict, skip: bool = True) -> dict: """ Items in the main navigation bar can be direct links, or dropdowns with subitems. This context preprocessor adds a boolean field ``has_subitems`` that tells which one of them every element is. It also adds a ``slug`` field to be used as a CSS id. """ + if skip: + return context + lang = context["selected_language"] ignore = context["translations"]["ignore"] default_language = context["translations"]["default_language"] @@ -103,7 +100,7 @@ def navbar_add_info(context): return context @staticmethod - def blog_add_posts(context): + def blog_add_posts(context: dict) -> dict: """ Given the blog feed defined in the configuration yaml, this context preprocessor fetches the posts in the feeds, and returns the relevant @@ -175,7 +172,7 @@ def blog_add_posts(context): return context @staticmethod - def maintainers_add_info(context): + def maintainers_add_info(context: dict) -> dict: """ Given the active maintainers defined in the yaml file, it fetches the GitHub user information for them. @@ -225,7 +222,7 @@ def maintainers_add_info(context): return context @staticmethod - def home_add_releases(context): + def home_add_releases(context: dict) -> dict: context["releases"] = [] github_repo_url = context["main"]["github_repo_url"] @@ -285,7 +282,7 @@ def home_add_releases(context): return context @staticmethod - def roadmap_pdeps(context): + def roadmap_pdeps(context: dict) -> dict: """ PDEP's (pandas enhancement proposals) are not part of the bar navigation. They are included as lists in the "Roadmap" page @@ -399,26 +396,16 @@ def get_callable(obj_as_str: str) -> object: return obj -def get_config(config_fname: str) -> dict: +def get_context(config_fname: str, **kwargs: dict) -> dict: """ - Load the config yaml file and return it as a dictionary. + Load the config yaml as the base context, and enrich it with the + information added by the context preprocessors defined in the file. """ with open(config_fname, encoding="utf-8") as f: context = yaml.safe_load(f) - return context - -def get_context(config_fname: str, **kwargs): - """ - Load the config yaml as the base context, and enrich it with the - information added by the context preprocessors defined in the file. - """ - context = get_config(config_fname) context["source_path"] = os.path.dirname(config_fname) context.update(kwargs) - context["languages"] = context.get("languages", ["en"]) - context["selected_language"] = context.get("language", "en") - preprocessors = ( get_callable(context_prep) for context_prep in context["main"]["context_preprocessors"] @@ -431,8 +418,24 @@ def get_context(config_fname: str, **kwargs): return context +def update_navbar_context(context: dict) -> dict: + """ + Update the context with the navbar information for each language. + """ + language = context["selected_language"] + lang_prefix = ( + language if language != context["translations"]["default_language"] else "" + ) + navbar_path = os.path.join(context["source_path"], lang_prefix, "navbar.yml") + with open(navbar_path) as f: + navbar = yaml.safe_load(f) + + context.update(navbar) + return Preprocessors.navbar_add_info(context, skip=False) + + def get_source_files( - source_path: str, language, languages + source_path: str, language: str, languages: list[str] ) -> typing.Generator[str, None, None]: """ Generate the list of files present in the source directory. @@ -466,56 +469,10 @@ def extend_base_template(content: str, base_template: str) -> str: return result -def download_and_extract_translations(url: str, dir_name: str): - """ - Download the translations from the GitHub repository. - """ - shutil.rmtree(dir_name, ignore_errors=True) - response = requests.get(url) - if response.status_code == 200: - doc = io.BytesIO(response.content) - with tarfile.open(None, "r:gz", doc) as tar: - tar.extractall(dir_name) - else: - raise Exception(f"Failed to download translations: {response.status_code}") - - -def get_languages(source_path: str): - """ - Get the list of languages available in the translations directory. - """ - en_path = f"{source_path}/en/" - if os.path.exists(en_path): - shutil.rmtree(en_path) - - paths = os.listdir(source_path) - return [path for path in paths if os.path.isdir(f"{source_path}/{path}")] - - -def copy_translations(source_path: str, target_path: str, languages: list[str]): - """ - Copy the translations to the appropriate directory. - """ - for lang in languages: - dest = f"{target_path}/{lang}/" - shutil.rmtree(dest, ignore_errors=True) - cmds = [ - "rsync", - "-av", - "--delete", - f"{source_path}/{lang}/", - dest, - ] - p = Popen(cmds, stdout=PIPE, stderr=PIPE) - stdout, stderr = p.communicate() - sys.stderr.write(f"\nCopying: {lang}...\n\n") - sys.stderr.write(stdout.decode()) - sys.stderr.write(stderr.decode()) - - def main( source_path: str, target_path: str, + process_translations: bool = False, ) -> int: """ Copy every file in the source directory to the target directory. @@ -526,51 +483,37 @@ def main( base_folder = os.path.dirname(__file__) base_source_path = source_path base_target_path = target_path - base_config_fname = os.path.join(source_path, "config.yml") - config = get_config(base_config_fname) - translations_path = os.path.join(base_folder, f"{config['translations']['folder']}") + shutil.rmtree(target_path, ignore_errors=True) + os.makedirs(target_path, exist_ok=True) - sys.stderr.write("\nDownloading and extracting translations...\n\n") - translations_extract_path = translations_path - translations_source_path = os.path.join( - translations_path, config["translations"]["source_path"] + # Handle translations + sys.path.append(base_folder) + trans = importlib.import_module("pandas_translations") + translated_languages, languages = trans.process_translations( + "config.yml", source_path, process_translations ) - download_and_extract_translations( - config["translations"]["url"], translations_extract_path + sys.stderr.write("Generating context...\n") + context = get_context( + os.path.join(source_path, "config.yml"), + target_path=target_path, + languages=languages, ) + sys.stderr.write("Context generated\n") - translated_languages = get_languages(translations_source_path) - default_language = config["translations"]["default_language"] - languages = [default_language] + translated_languages - - sys.stderr.write("\nCopying translations...\n") - copy_translations(translations_source_path, source_path, translated_languages) + templates_path = os.path.join(source_path, context["main"]["templates_path"]) + jinja_env = jinja2.Environment(loader=jinja2.FileSystemLoader(templates_path)) for language in languages: + context["selected_language"] = language + context = update_navbar_context(context) sys.stderr.write(f"\nProcessing language: {language}...\n\n") - if language != default_language: + + if language != context["translations"]["default_language"]: target_path = os.path.join(base_target_path, language) source_path = os.path.join(base_source_path, language) - shutil.rmtree(target_path, ignore_errors=True) - os.makedirs(target_path, exist_ok=True) - - config_fname = os.path.join(source_path, "config.yml") - sys.stderr.write("Generating context...\n") - - context = get_context( - config_fname, - target_path=target_path, - language=language, - languages=languages, - ) - sys.stderr.write("Context generated\n") - - templates_path = os.path.join(source_path, context["main"]["templates_path"]) - jinja_env = jinja2.Environment(loader=jinja2.FileSystemLoader(templates_path)) - for fname in get_source_files(source_path, language, languages): if os.path.normpath(fname) in context["main"]["ignore"]: continue @@ -608,6 +551,7 @@ def main( shutil.copy( os.path.join(source_path, fname), os.path.join(target_path, dirname) ) + return 0 if __name__ == "__main__": @@ -618,5 +562,6 @@ def main( parser.add_argument( "--target-path", default="build", help="directory where to write the output" ) + parser.add_argument("-t", "--translations", action="store_true") args = parser.parse_args() - sys.exit(main(args.source_path, args.target_path)) + sys.exit(main(args.source_path, args.target_path, args.translations)) From de291cd03fc684645a4a1f243b2a52c14c078771 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gonzalo=20Pe=C3=B1a-Castellanos?= Date: Wed, 14 May 2025 11:31:49 -0500 Subject: [PATCH 6/9] Process all languages in a single step --- .github/workflows/docbuild-and-upload.yml | 2 +- web/pandas/_templates/layout.html | 2 +- web/pandas/config.yml | 1 + web/pandas_translations.py | 12 +- web/pandas_web.py | 170 ++++++++++------------ 5 files changed, 84 insertions(+), 103 deletions(-) diff --git a/.github/workflows/docbuild-and-upload.yml b/.github/workflows/docbuild-and-upload.yml index 8fbde58b0a462..294334ca1d54b 100644 --- a/.github/workflows/docbuild-and-upload.yml +++ b/.github/workflows/docbuild-and-upload.yml @@ -54,7 +54,7 @@ jobs: run: python -m pytest web/ - name: Build website - run: python web/pandas_web.py web/pandas --target-path=web/build --translations + run: python web/pandas_web.py web/pandas --target-path=web/build - name: Build documentation run: doc/make.py --warnings-are-errors diff --git a/web/pandas/_templates/layout.html b/web/pandas/_templates/layout.html index 82ca9c4c589b9..500e8fce2156a 100644 --- a/web/pandas/_templates/layout.html +++ b/web/pandas/_templates/layout.html @@ -30,7 +30,7 @@
    - {% for item in navbar %} + {% for item in navbar[selected_language] %} {% if not item.has_subitems %}
  • {{ item.name }} diff --git a/web/pandas/config.yml b/web/pandas/config.yml index 35e6ebf429df8..28a2c3b8fffcc 100644 --- a/web/pandas/config.yml +++ b/web/pandas/config.yml @@ -176,5 +176,6 @@ translations: folder: translations source_path: pandas-translations-main/web/pandas/ default_language: 'en' + default_prefix: '' ignore: - docs/ diff --git a/web/pandas_translations.py b/web/pandas_translations.py index 4809e9aecea27..c023605cff340 100755 --- a/web/pandas_translations.py +++ b/web/pandas_translations.py @@ -82,7 +82,7 @@ def copy_translations(source_path: str, target_path: str, languages: list[str]) def process_translations( - config_fname: str, source_path: str, process_translations: bool + config_fname: str, source_path: str ) -> tuple[list[str], list[str]]: """ Process the translations by downloading and extracting them from @@ -95,15 +95,15 @@ def process_translations( translations_path, config["translations"]["source_path"] ) default_language = config["translations"]["default_language"] + sys.stderr.write("\nDownloading and extracting translations...\n\n") download_and_extract_translations(config["translations"]["url"], translations_path) - languages = [default_language] + translated_languages = get_languages(translations_source_path) remove_translations(source_path, translated_languages) - if process_translations: - languages = [default_language] + translated_languages - sys.stderr.write("\nCopying translations...\n") - copy_translations(translations_source_path, source_path, translated_languages) + languages = [default_language] + translated_languages + sys.stderr.write("\nCopying translations...\n") + copy_translations(translations_source_path, source_path, translated_languages) return translated_languages, languages diff --git a/web/pandas_web.py b/web/pandas_web.py index cc938808b12a3..58424c876f5aa 100755 --- a/web/pandas_web.py +++ b/web/pandas_web.py @@ -81,22 +81,17 @@ def navbar_add_info(context: dict, skip: bool = True) -> dict: ``has_subitems`` that tells which one of them every element is. It also adds a ``slug`` field to be used as a CSS id. """ - if skip: - return context - - lang = context["selected_language"] ignore = context["translations"]["ignore"] - default_language = context["translations"]["default_language"] - for i, item in enumerate(context["navbar"]): - if item["target"] in ignore: - if lang != default_language: - item["target"] = "../" + item["target"] - - context["navbar"][i] = dict( - item, - has_subitems=isinstance(item["target"], list), - slug=item["name"].replace(" ", "-").lower(), - ) + for language in context["languages"]: + for i, item in enumerate(context["navbar"][language]): + if item["target"] in ignore: + item["target"] = f"../{item['target']}" + + context["navbar"][language][i] = dict( + item, + has_subitems=isinstance(item["target"], list), + slug=(item["name"].replace(" ", "-").lower()), + ) return context @staticmethod @@ -396,7 +391,9 @@ def get_callable(obj_as_str: str) -> object: return obj -def get_context(config_fname: str, **kwargs: dict) -> dict: +def get_context( + config_fname: str, navbar_fname: str, languages: list[str], **kwargs: dict +) -> dict: """ Load the config yaml as the base context, and enrich it with the information added by the context preprocessors defined in the file. @@ -405,6 +402,22 @@ def get_context(config_fname: str, **kwargs: dict) -> dict: context = yaml.safe_load(f) context["source_path"] = os.path.dirname(config_fname) + + navbar = {} + context["languages"] = languages + default_language = context["translations"]["default_language"] + default_prefix = context["translations"]["default_prefix"] + for language in languages: + prefix = default_prefix if language == default_language else language + navbar_path = os.path.join(context["source_path"], prefix, navbar_fname) + + with open(navbar_path, encoding="utf-8") as f: + navbar_lang = yaml.safe_load(f) + + navbar[language] = navbar_lang["navbar"] + + context["navbar"] = navbar + context.update(kwargs) preprocessors = ( get_callable(context_prep) @@ -418,43 +431,14 @@ def get_context(config_fname: str, **kwargs: dict) -> dict: return context -def update_navbar_context(context: dict) -> dict: - """ - Update the context with the navbar information for each language. - """ - language = context["selected_language"] - lang_prefix = ( - language if language != context["translations"]["default_language"] else "" - ) - navbar_path = os.path.join(context["source_path"], lang_prefix, "navbar.yml") - with open(navbar_path) as f: - navbar = yaml.safe_load(f) - - context.update(navbar) - return Preprocessors.navbar_add_info(context, skip=False) - - -def get_source_files( - source_path: str, language: str, languages: list[str] -) -> typing.Generator[str, None, None]: +def get_source_files(source_path: str) -> typing.Generator[str, None, None]: """ Generate the list of files present in the source directory. """ - paths = [] - all_languages = languages[:] - all_languages.remove(language) for root, dirs, fnames in os.walk(source_path): root_rel_path = os.path.relpath(root, source_path) for fname in fnames: - path = os.path.join(root_rel_path, fname) - for language in all_languages: - if path.startswith(language + "/"): - break - else: - paths.append(path) - - for path in paths: - yield path + yield os.path.join(root_rel_path, fname) def extend_base_template(content: str, base_template: str) -> str: @@ -472,7 +456,6 @@ def extend_base_template(content: str, base_template: str) -> str: def main( source_path: str, target_path: str, - process_translations: bool = False, ) -> int: """ Copy every file in the source directory to the target directory. @@ -481,8 +464,6 @@ def main( before copying them. ``.md`` files are transformed to HTML. """ base_folder = os.path.dirname(__file__) - base_source_path = source_path - base_target_path = target_path shutil.rmtree(target_path, ignore_errors=True) os.makedirs(target_path, exist_ok=True) @@ -491,12 +472,13 @@ def main( sys.path.append(base_folder) trans = importlib.import_module("pandas_translations") translated_languages, languages = trans.process_translations( - "config.yml", source_path, process_translations + "config.yml", source_path ) sys.stderr.write("Generating context...\n") context = get_context( os.path.join(source_path, "config.yml"), + navbar_fname="navbar.yml", target_path=target_path, languages=languages, ) @@ -505,52 +487,51 @@ def main( templates_path = os.path.join(source_path, context["main"]["templates_path"]) jinja_env = jinja2.Environment(loader=jinja2.FileSystemLoader(templates_path)) - for language in languages: - context["selected_language"] = language - context = update_navbar_context(context) - sys.stderr.write(f"\nProcessing language: {language}...\n\n") - - if language != context["translations"]["default_language"]: - target_path = os.path.join(base_target_path, language) - source_path = os.path.join(base_source_path, language) + default_language = context["translations"]["default_language"] + for fname in get_source_files(source_path): + selected_language = context["translations"]["default_language"] + for language in translated_languages: + if fname.startswith(language + "/"): + selected_language = language + break - for fname in get_source_files(source_path, language, languages): - if os.path.normpath(fname) in context["main"]["ignore"]: - continue + context["selected_language"] = selected_language + if os.path.normpath(fname) in context["main"]["ignore"]: + continue + + sys.stderr.write(f"Processing {fname}\n") + dirname = os.path.dirname(fname) + os.makedirs(os.path.join(target_path, dirname), exist_ok=True) + + extension = os.path.splitext(fname)[-1] + if extension in (".html", ".md"): + with open(os.path.join(source_path, fname), encoding="utf-8") as f: + content = f.read() + if extension == ".md": + body = markdown.markdown( + content, extensions=context["main"]["markdown_extensions"] + ) + # Apply Bootstrap's table formatting manually + # Python-Markdown doesn't let us config table attributes by hand + body = body.replace("
", '
') + content = extend_base_template(body, context["main"]["base_template"]) - sys.stderr.write(f"Processing {fname}\n") - dirname = os.path.dirname(fname) - os.makedirs(os.path.join(target_path, dirname), exist_ok=True) - - extension = os.path.splitext(fname)[-1] - if extension in (".html", ".md"): - with open(os.path.join(source_path, fname), encoding="utf-8") as f: - content = f.read() - if extension == ".md": - body = markdown.markdown( - content, extensions=context["main"]["markdown_extensions"] - ) - # Apply Bootstrap's table formatting manually - # Python-Markdown doesn't let us config table attributes by hand - body = body.replace( - "
", '
' - ) - content = extend_base_template( - body, context["main"]["base_template"] - ) + context["base_url"] = "".join(["../"] * os.path.normpath(fname).count("/")) + if selected_language != default_language: context["base_url"] = "".join( - ["../"] * os.path.normpath(fname).count("/") - ) - content = jinja_env.from_string(content).render(**context) - fname_html = os.path.splitext(fname)[0] + ".html" - with open( - os.path.join(target_path, fname_html), "w", encoding="utf-8" - ) as f: - f.write(content) - else: - shutil.copy( - os.path.join(source_path, fname), os.path.join(target_path, dirname) + ["../"] * (os.path.normpath(fname).count("/") - 1) ) + + content = jinja_env.from_string(content).render(**context) + fname_html = os.path.splitext(fname)[0] + ".html" + with open( + os.path.join(target_path, fname_html), "w", encoding="utf-8" + ) as f: + f.write(content) + else: + shutil.copy( + os.path.join(source_path, fname), os.path.join(target_path, dirname) + ) return 0 @@ -562,6 +543,5 @@ def main( parser.add_argument( "--target-path", default="build", help="directory where to write the output" ) - parser.add_argument("-t", "--translations", action="store_true") args = parser.parse_args() - sys.exit(main(args.source_path, args.target_path, args.translations)) + sys.exit(main(args.source_path, args.target_path)) From 0d2bfff0378b1fe199bc114dbb4dbd7f6f535001 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gonzalo=20Pe=C3=B1a-Castellanos?= Date: Mon, 19 May 2025 18:15:54 -0500 Subject: [PATCH 7/9] Simplify code and move download and extract to pandas_web --- web/pandas/_templates/layout.html | 78 +++++++++++++++- web/pandas/config.yml | 7 +- web/pandas/navbar.yml | 1 + web/pandas/static/js/language_switcher.js | 71 -------------- web/pandas_translations.py | 109 ---------------------- web/pandas_web.py | 66 +++++++------ 6 files changed, 120 insertions(+), 212 deletions(-) delete mode 100644 web/pandas/static/js/language_switcher.js delete mode 100755 web/pandas_translations.py diff --git a/web/pandas/_templates/layout.html b/web/pandas/_templates/layout.html index 500e8fce2156a..41af9184ba211 100644 --- a/web/pandas/_templates/layout.html +++ b/web/pandas/_templates/layout.html @@ -15,8 +15,82 @@ href="{{ base_url }}{{ stylesheet }}"> {% endfor %} - - +
diff --git a/web/pandas/config.yml b/web/pandas/config.yml index 28a2c3b8fffcc..827f7060371c4 100644 --- a/web/pandas/config.yml +++ b/web/pandas/config.yml @@ -177,5 +177,8 @@ translations: source_path: pandas-translations-main/web/pandas/ default_language: 'en' default_prefix: '' - ignore: - - docs/ + languages: + en: English + es: Español + pt: Português + # fr: Français diff --git a/web/pandas/navbar.yml b/web/pandas/navbar.yml index bcc2d062fc3f5..07ff0a64314a5 100644 --- a/web/pandas/navbar.yml +++ b/web/pandas/navbar.yml @@ -17,6 +17,7 @@ navbar: target: getting_started.html - name: "Documentation" target: docs/ + translated: false - name: "Community" target: - name: "Blog" diff --git a/web/pandas/static/js/language_switcher.js b/web/pandas/static/js/language_switcher.js deleted file mode 100644 index eca361d30e20f..0000000000000 --- a/web/pandas/static/js/language_switcher.js +++ /dev/null @@ -1,71 +0,0 @@ -window.addEventListener("DOMContentLoaded", function() { - var absBaseUrl = document.baseURI; - var baseUrl = location.protocol + "//" + location.hostname - if (location.port) { - baseUrl = baseUrl + ":" + location.port - } - var currentLanguage = document.documentElement.lang; - var languages = JSON.parse(document.getElementById("languages").getAttribute('data-lang').replace(/'/g, '"')); - const languageNames = { - 'en': 'English', - 'es': 'Español', - 'fr': 'Français', - 'pt': 'Português' - } - - // Handle preview URLs on github - // If preview URL changes, this regex will need to be updated - const re = /preview\/pandas-dev\/pandas\/(?[0-9]*)\//g; - var previewUrl = ''; - for (const match of absBaseUrl.matchAll(re)) { - previewUrl = `/preview/pandas-dev/pandas/${match.groups.pr}`; - } - var pathName = location.pathname.replace(previewUrl, '') - - // Create dropdown menu - function makeDropdown(options) { - var dropdown = document.createElement("li"); - dropdown.classList.add("nav-item"); - dropdown.classList.add("dropdown"); - - var link = document.createElement("a"); - link.classList.add("nav-link"); - link.classList.add("dropdown-toggle"); - link.setAttribute("data-bs-toggle", "dropdown"); - link.setAttribute("href", "#"); - link.setAttribute("role", "button"); - link.setAttribute("aria-haspopup", "true"); - link.setAttribute("aria-expanded", "false"); - link.textContent = languageNames[currentLanguage]; - - var dropdownMenu = document.createElement("div"); - dropdownMenu.classList.add("dropdown-menu"); - - options.forEach(function(i) { - var dropdownItem = document.createElement("a"); - dropdownItem.classList.add("dropdown-item"); - dropdownItem.textContent = languageNames[i] || i.toUpperCase(); - dropdownItem.setAttribute("href", "#"); - dropdownItem.addEventListener("click", function() { - var urlLanguage = ''; - if (i !== 'en') { - urlLanguage = '/' + i; - } - pathName = pathName.replace('/' + currentLanguage + '/', '/') - var newUrl = baseUrl + previewUrl + urlLanguage + pathName - window.location.href = newUrl; - }); - dropdownMenu.appendChild(dropdownItem); - }); - - dropdown.appendChild(link); - dropdown.appendChild(dropdownMenu); - return dropdown; - } - - var container = document.getElementById("language-switcher-container"); - if (container) { - var dropdown = makeDropdown(languages); - container.appendChild(dropdown); - } -}); diff --git a/web/pandas_translations.py b/web/pandas_translations.py deleted file mode 100755 index c023605cff340..0000000000000 --- a/web/pandas_translations.py +++ /dev/null @@ -1,109 +0,0 @@ -#!/usr/bin/env python3 -""" -Utilities to download and extract translations from the GitHub repository. -""" - -import io -import os -import shutil -from subprocess import ( - PIPE, - Popen, -) -import sys -import tarfile - -import requests -import yaml - - -def get_config(config_fname: str) -> dict: - """ - Load the config yaml file and return it as a dictionary. - """ - with open(config_fname, encoding="utf-8") as f: - context = yaml.safe_load(f) - return context - - -def download_and_extract_translations(url: str, dir_name: str) -> None: - """ - Download the translations from the GitHub repository. - """ - shutil.rmtree(dir_name, ignore_errors=True) - response = requests.get(url) - if response.status_code == 200: - doc = io.BytesIO(response.content) - with tarfile.open(None, "r:gz", doc) as tar: - tar.extractall(dir_name) - else: - raise Exception(f"Failed to download translations: {response.status_code}") - - -def get_languages(source_path: str) -> list[str]: - """ - Get the list of languages available in the translations directory. - """ - en_path = f"{source_path}/en/" - if os.path.exists(en_path): - shutil.rmtree(en_path) - - paths = os.listdir(source_path) - return [path for path in paths if os.path.isdir(f"{source_path}/{path}")] - - -def remove_translations(source_path: str, languages: list[str]) -> None: - """ - Remove the translations from the source path. - """ - for language in languages: - shutil.rmtree(os.path.join(source_path, language), ignore_errors=True) - - -def copy_translations(source_path: str, target_path: str, languages: list[str]) -> None: - """ - Copy the translations to the appropriate directory. - """ - for lang in languages: - dest = f"{target_path}/{lang}/" - shutil.rmtree(dest, ignore_errors=True) - cmds = [ - "rsync", - "-av", - "--delete", - f"{source_path}/{lang}/", - dest, - ] - p = Popen(cmds, stdout=PIPE, stderr=PIPE) - stdout, stderr = p.communicate() - sys.stderr.write(f"\nCopying: {lang}...\n\n") - sys.stderr.write(stdout.decode()) - sys.stderr.write(stderr.decode()) - - -def process_translations( - config_fname: str, source_path: str -) -> tuple[list[str], list[str]]: - """ - Process the translations by downloading and extracting them from - the GitHub repository. - """ - base_folder = os.path.dirname(__file__) - config = get_config(os.path.join(source_path, config_fname)) - translations_path = os.path.join(base_folder, f"{config['translations']['folder']}") - translations_source_path = os.path.join( - translations_path, config["translations"]["source_path"] - ) - default_language = config["translations"]["default_language"] - - sys.stderr.write("\nDownloading and extracting translations...\n\n") - download_and_extract_translations(config["translations"]["url"], translations_path) - - translated_languages = get_languages(translations_source_path) - remove_translations(source_path, translated_languages) - - languages = [default_language] + translated_languages - sys.stderr.write("\nCopying translations...\n") - copy_translations(translations_source_path, source_path, translated_languages) - - return translated_languages, languages diff --git a/web/pandas_web.py b/web/pandas_web.py index 58424c876f5aa..69c893a117c02 100755 --- a/web/pandas_web.py +++ b/web/pandas_web.py @@ -28,6 +28,7 @@ import collections import datetime import importlib +import io import itertools import json import operator @@ -36,6 +37,7 @@ import re import shutil import sys +import tarfile import time import typing @@ -74,17 +76,16 @@ def current_year(context: dict) -> dict: return context @staticmethod - def navbar_add_info(context: dict, skip: bool = True) -> dict: + def navbar_add_info(context: dict) -> dict: """ Items in the main navigation bar can be direct links, or dropdowns with subitems. This context preprocessor adds a boolean field ``has_subitems`` that tells which one of them every element is. It also adds a ``slug`` field to be used as a CSS id. """ - ignore = context["translations"]["ignore"] - for language in context["languages"]: + for language in context["translations"]["languages"]: for i, item in enumerate(context["navbar"][language]): - if item["target"] in ignore: + if not item.get("translated", True): item["target"] = f"../{item['target']}" context["navbar"][language][i] = dict( @@ -391,9 +392,7 @@ def get_callable(obj_as_str: str) -> object: return obj -def get_context( - config_fname: str, navbar_fname: str, languages: list[str], **kwargs: dict -) -> dict: +def get_context(config_fname: str, navbar_fname: str, **kwargs: dict) -> dict: """ Load the config yaml as the base context, and enrich it with the information added by the context preprocessors defined in the file. @@ -403,19 +402,20 @@ def get_context( context["source_path"] = os.path.dirname(config_fname) - navbar = {} - context["languages"] = languages default_language = context["translations"]["default_language"] default_prefix = context["translations"]["default_prefix"] - for language in languages: + translated_languages = context["translations"]["languages"].copy() + translated_languages.pop(default_language) + context["translated_languages"] = translated_languages + download_and_extract_translations(context) + navbar = {} + for language in context["translations"]["languages"]: prefix = default_prefix if language == default_language else language - navbar_path = os.path.join(context["source_path"], prefix, navbar_fname) - - with open(navbar_path, encoding="utf-8") as f: + with open( + os.path.join(context["source_path"], prefix, navbar_fname), encoding="utf-8" + ) as f: navbar_lang = yaml.safe_load(f) - navbar[language] = navbar_lang["navbar"] - context["navbar"] = navbar context.update(kwargs) @@ -453,6 +453,27 @@ def extend_base_template(content: str, base_template: str) -> str: return result +def download_and_extract_translations(context: dict) -> None: + """ + Download the translations from the GitHub repository and extract them. + """ + base_folder = os.path.dirname(__file__) + extract_path = os.path.join(base_folder, context["translations"]["folder"]) + shutil.rmtree(extract_path, ignore_errors=True) + response = requests.get(context["translations"]["url"]) + if response.status_code == 200: + with tarfile.open(None, "r:gz", io.BytesIO(response.content)) as tar: + tar.extractall(os.path.join(base_folder, context["translations"]["folder"])) + else: + raise Exception(f"Failed to download translations: {response.status_code}") + for lang in context["translated_languages"]: + shutil.rmtree(os.path.join(base_folder, "pandas", lang), ignore_errors=True) + shutil.move( + os.path.join(extract_path, context["translations"]["source_path"], lang), + os.path.join(base_folder, "pandas", lang), + ) + + def main( source_path: str, target_path: str, @@ -463,34 +484,23 @@ def main( For ``.md`` and ``.html`` files, render them with the context before copying them. ``.md`` files are transformed to HTML. """ - base_folder = os.path.dirname(__file__) - shutil.rmtree(target_path, ignore_errors=True) os.makedirs(target_path, exist_ok=True) - # Handle translations - sys.path.append(base_folder) - trans = importlib.import_module("pandas_translations") - translated_languages, languages = trans.process_translations( - "config.yml", source_path - ) - sys.stderr.write("Generating context...\n") context = get_context( os.path.join(source_path, "config.yml"), navbar_fname="navbar.yml", target_path=target_path, - languages=languages, ) sys.stderr.write("Context generated\n") templates_path = os.path.join(source_path, context["main"]["templates_path"]) jinja_env = jinja2.Environment(loader=jinja2.FileSystemLoader(templates_path)) - default_language = context["translations"]["default_language"] for fname in get_source_files(source_path): selected_language = context["translations"]["default_language"] - for language in translated_languages: + for language in context["translated_languages"]: if fname.startswith(language + "/"): selected_language = language break @@ -517,7 +527,7 @@ def main( content = extend_base_template(body, context["main"]["base_template"]) context["base_url"] = "".join(["../"] * os.path.normpath(fname).count("/")) - if selected_language != default_language: + if selected_language != context["translations"]["default_language"]: context["base_url"] = "".join( ["../"] * (os.path.normpath(fname).count("/") - 1) ) From ac0b8f03655f189c43c0a0a862033851459ec93b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gonzalo=20Pe=C3=B1a-Castellanos?= Date: Wed, 21 May 2025 03:54:22 -0500 Subject: [PATCH 8/9] Create preprocessor and fix review comments --- .gitignore | 2 +- web/pandas/_templates/layout.html | 118 +++++++++++------------------- web/pandas/config.yml | 6 +- web/pandas_web.py | 116 ++++++++++++++--------------- 4 files changed, 103 insertions(+), 139 deletions(-) diff --git a/.gitignore b/.gitignore index 0f854f61ae0e5..e78fca6f30fc0 100644 --- a/.gitignore +++ b/.gitignore @@ -146,7 +146,7 @@ doc/source/savefig/ # Web & Translations # ############################## web/preview/ -web/translations/ +web/pandas-translations-main/ web/pandas/es/ web/pandas/pt/ web/pandas/fr/ diff --git a/web/pandas/_templates/layout.html b/web/pandas/_templates/layout.html index 41af9184ba211..eeefc715b1853 100644 --- a/web/pandas/_templates/layout.html +++ b/web/pandas/_templates/layout.html @@ -1,5 +1,5 @@ - + pandas - Python Data Analysis Library @@ -16,80 +16,36 @@ {% endfor %} @@ -104,7 +60,7 @@
    - {% for item in navbar[selected_language] %} + {% for item in navbar[lang] %} {% if not item.has_subitems %}
  • {{ item.name }} @@ -127,7 +83,21 @@ {% endif %} {% endfor %} -
    +
  • + +
    + {% for language, name in translations["languages"].items() -%} + {{ name }} + {% endfor -%} +
    +
diff --git a/web/pandas/config.yml b/web/pandas/config.yml index 827f7060371c4..a2f339df16a9e 100644 --- a/web/pandas/config.yml +++ b/web/pandas/config.yml @@ -1,12 +1,14 @@ main: templates_path: _templates base_template: "layout.html" + navbar_fname: "navbar.yml" production_url: "https://pandas.pydata.org/" ignore: - _templates/layout.html - config.yml github_repo_url: pandas-dev/pandas context_preprocessors: + - pandas_web.Preprocessors.process_translations - pandas_web.Preprocessors.current_year - pandas_web.Preprocessors.navbar_add_info - pandas_web.Preprocessors.blog_add_posts @@ -173,12 +175,8 @@ roadmap: pdeps_path: pdeps translations: url: https://github.com/Scientific-Python-Translations/pandas-translations/archive/refs/heads/main.tar.gz - folder: translations source_path: pandas-translations-main/web/pandas/ - default_language: 'en' - default_prefix: '' languages: en: English es: Español pt: Português - # fr: Français diff --git a/web/pandas_web.py b/web/pandas_web.py index 69c893a117c02..e19532dd921ca 100755 --- a/web/pandas_web.py +++ b/web/pandas_web.py @@ -66,6 +66,29 @@ class Preprocessors: anything else needed just be added with context preprocessors. """ + @staticmethod + def process_translations(context: dict) -> dict: + """ + Download the translations from the GitHub repository and extract them. + """ + base_folder = os.path.dirname(__file__) + extract_path = os.path.join(base_folder, context["translations"]["source_path"]) + shutil.rmtree(extract_path, ignore_errors=True) + response = requests.get(context["translations"]["url"]) + if response.status_code == 200: + with tarfile.open(None, "r:gz", io.BytesIO(response.content)) as tar: + tar.extractall(base_folder) + else: + raise Exception(f"Failed to download translations: {response.status_code}") + + for lang in list(context["translations"]["languages"].keys())[1:]: + shutil.rmtree(os.path.join(base_folder, "pandas", lang), ignore_errors=True) + shutil.move( + os.path.join(extract_path, lang), + os.path.join(base_folder, "pandas", lang), + ) + return context + @staticmethod def current_year(context: dict) -> dict: """ @@ -83,14 +106,38 @@ def navbar_add_info(context: dict) -> dict: ``has_subitems`` that tells which one of them every element is. It also adds a ``slug`` field to be used as a CSS id. """ - for language in context["translations"]["languages"]: - for i, item in enumerate(context["navbar"][language]): - if not item.get("translated", True): - item["target"] = f"../{item['target']}" - context["navbar"][language][i] = dict( + def update_target(item: dict, url_prefix: str) -> None: + if item.get("translated", True): + item["target"] = f"{url_prefix}{item['target']}" + else: + item["target"] = f"../{item['target']}" + + context["navbar"] = {} + for lang in context["translations"]["languages"]: + prefix = "" if lang == "en" else lang + url_prefix = "" if lang == "en" else lang + "/" + with open( + os.path.join( + context["source_path"], prefix, context["main"]["navbar_fname"] + ), + encoding="utf-8", + ) as f: + navbar_lang = yaml.safe_load(f) + + context["navbar"][lang] = navbar_lang["navbar"] + for i, item in enumerate(navbar_lang["navbar"]): + has_subitems = isinstance(item["target"], list) + if lang != "en": + if has_subitems: + for sub_item in item["target"]: + update_target(sub_item, url_prefix) + else: + update_target(item, url_prefix) + + context["navbar"][lang][i] = dict( item, - has_subitems=isinstance(item["target"], list), + has_subitems=has_subitems, slug=(item["name"].replace(" ", "-").lower()), ) return context @@ -392,7 +439,7 @@ def get_callable(obj_as_str: str) -> object: return obj -def get_context(config_fname: str, navbar_fname: str, **kwargs: dict) -> dict: +def get_context(config_fname: str, **kwargs: dict) -> dict: """ Load the config yaml as the base context, and enrich it with the information added by the context preprocessors defined in the file. @@ -401,24 +448,8 @@ def get_context(config_fname: str, navbar_fname: str, **kwargs: dict) -> dict: context = yaml.safe_load(f) context["source_path"] = os.path.dirname(config_fname) - - default_language = context["translations"]["default_language"] - default_prefix = context["translations"]["default_prefix"] - translated_languages = context["translations"]["languages"].copy() - translated_languages.pop(default_language) - context["translated_languages"] = translated_languages - download_and_extract_translations(context) - navbar = {} - for language in context["translations"]["languages"]: - prefix = default_prefix if language == default_language else language - with open( - os.path.join(context["source_path"], prefix, navbar_fname), encoding="utf-8" - ) as f: - navbar_lang = yaml.safe_load(f) - navbar[language] = navbar_lang["navbar"] - context["navbar"] = navbar - context.update(kwargs) + preprocessors = ( get_callable(context_prep) for context_prep in context["main"]["context_preprocessors"] @@ -453,27 +484,6 @@ def extend_base_template(content: str, base_template: str) -> str: return result -def download_and_extract_translations(context: dict) -> None: - """ - Download the translations from the GitHub repository and extract them. - """ - base_folder = os.path.dirname(__file__) - extract_path = os.path.join(base_folder, context["translations"]["folder"]) - shutil.rmtree(extract_path, ignore_errors=True) - response = requests.get(context["translations"]["url"]) - if response.status_code == 200: - with tarfile.open(None, "r:gz", io.BytesIO(response.content)) as tar: - tar.extractall(os.path.join(base_folder, context["translations"]["folder"])) - else: - raise Exception(f"Failed to download translations: {response.status_code}") - for lang in context["translated_languages"]: - shutil.rmtree(os.path.join(base_folder, "pandas", lang), ignore_errors=True) - shutil.move( - os.path.join(extract_path, context["translations"]["source_path"], lang), - os.path.join(base_folder, "pandas", lang), - ) - - def main( source_path: str, target_path: str, @@ -490,22 +500,14 @@ def main( sys.stderr.write("Generating context...\n") context = get_context( os.path.join(source_path, "config.yml"), - navbar_fname="navbar.yml", target_path=target_path, ) sys.stderr.write("Context generated\n") templates_path = os.path.join(source_path, context["main"]["templates_path"]) jinja_env = jinja2.Environment(loader=jinja2.FileSystemLoader(templates_path)) - for fname in get_source_files(source_path): - selected_language = context["translations"]["default_language"] - for language in context["translated_languages"]: - if fname.startswith(language + "/"): - selected_language = language - break - - context["selected_language"] = selected_language + context["lang"] = fname[0:2] if fname[2] == "/" else "en" if os.path.normpath(fname) in context["main"]["ignore"]: continue @@ -525,13 +527,7 @@ def main( # Python-Markdown doesn't let us config table attributes by hand body = body.replace("
", '
') content = extend_base_template(body, context["main"]["base_template"]) - context["base_url"] = "".join(["../"] * os.path.normpath(fname).count("/")) - if selected_language != context["translations"]["default_language"]: - context["base_url"] = "".join( - ["../"] * (os.path.normpath(fname).count("/") - 1) - ) - content = jinja_env.from_string(content).render(**context) fname_html = os.path.splitext(fname)[0] + ".html" with open( From 99e9635e8ca036596670aa022de59533a7bfb350 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gonzalo=20Pe=C3=B1a-Castellanos?= Date: Wed, 21 May 2025 07:45:36 -0500 Subject: [PATCH 9/9] Split preprocessor logic and more code review changes --- .gitignore | 9 ----- web/pandas/config.yml | 4 +- web/pandas_web.py | 92 ++++++++++++++++++++++++++++++------------- 3 files changed, 67 insertions(+), 38 deletions(-) diff --git a/.gitignore b/.gitignore index e78fca6f30fc0..d951f3fb9cbad 100644 --- a/.gitignore +++ b/.gitignore @@ -141,12 +141,3 @@ doc/source/savefig/ # Pyodide/WASM related files # ############################## /.pyodide-xbuildenv-* - - -# Web & Translations # -############################## -web/preview/ -web/pandas-translations-main/ -web/pandas/es/ -web/pandas/pt/ -web/pandas/fr/ diff --git a/web/pandas/config.yml b/web/pandas/config.yml index a2f339df16a9e..b0a4bc8f5cf7e 100644 --- a/web/pandas/config.yml +++ b/web/pandas/config.yml @@ -8,9 +8,11 @@ main: - config.yml github_repo_url: pandas-dev/pandas context_preprocessors: - - pandas_web.Preprocessors.process_translations - pandas_web.Preprocessors.current_year + - pandas_web.Preprocessors.download_translated_content + - pandas_web.Preprocessors.add_navbar_content - pandas_web.Preprocessors.navbar_add_info + - pandas_web.Preprocessors.navbar_add_translated_info - pandas_web.Preprocessors.blog_add_posts - pandas_web.Preprocessors.maintainers_add_info - pandas_web.Preprocessors.home_add_releases diff --git a/web/pandas_web.py b/web/pandas_web.py index e19532dd921ca..77da821cc98c5 100755 --- a/web/pandas_web.py +++ b/web/pandas_web.py @@ -67,9 +67,25 @@ class Preprocessors: """ @staticmethod - def process_translations(context: dict) -> dict: + def current_year(context: dict) -> dict: """ - Download the translations from the GitHub repository and extract them. + Add the current year to the context, so it can be used for the copyright + note, or other places where it is needed. + """ + context["current_year"] = datetime.datetime.now().year + return context + + @staticmethod + def download_translated_content(context: dict) -> dict: + """ + Download the translations from the mirror translations repository. + https://github.com/Scientific-Python-Translations/pandas-translations + + All translated languages are downloaded, extracted and place inside the + ``pandas`` folder in a separate folder for each language (e.g. ``pandas/es``). + + The extracted folder and the translations folders are deleted before + downloading the information, so the translations are always up to date. """ base_folder = os.path.dirname(__file__) extract_path = os.path.join(base_folder, context["translations"]["source_path"]) @@ -90,50 +106,70 @@ def process_translations(context: dict) -> dict: return context @staticmethod - def current_year(context: dict) -> dict: + def add_navbar_content(context: dict) -> dict: """ - Add the current year to the context, so it can be used for the copyright - note, or other places where it is needed. + Add the navbar content to the context. + + The navbar content is loaded for all available languages. """ - context["current_year"] = datetime.datetime.now().year + context["navbar"] = {} + for lang in context["translations"]["languages"]: + path = os.path.join( + context["source_path"], + "" if lang == "en" else f"{lang}", + context["main"]["navbar_fname"], + ) + if os.path.exists(path): + with open( + path, + encoding="utf-8", + ) as f: + navbar_lang = yaml.safe_load(f) + context["navbar"][lang] = navbar_lang["navbar"] + return context @staticmethod def navbar_add_info(context: dict) -> dict: """ + Items in the main navigation bar can be direct links, or dropdowns with + subitems. This context preprocessor adds a boolean field + ``has_subitems`` that tells which one of them every element is. It + also adds a ``slug`` field to be used as a CSS id. + """ + for i, item in enumerate(context["navbar"]["en"]): + context["navbar"]["en"][i] = dict( + item, + has_subitems=isinstance(item["target"], list), + slug=(item["name"].replace(" ", "-").lower()), + ) + return context + + @staticmethod + def navbar_add_translated_info(context: dict) -> dict: + """ + Prepare the translated navbar information for the template. + Items in the main navigation bar can be direct links, or dropdowns with subitems. This context preprocessor adds a boolean field ``has_subitems`` that tells which one of them every element is. It also adds a ``slug`` field to be used as a CSS id. """ - def update_target(item: dict, url_prefix: str) -> None: + def update_target(item: dict, prefix: str) -> None: if item.get("translated", True): - item["target"] = f"{url_prefix}{item['target']}" + item["target"] = f"{prefix}/{item['target']}" else: item["target"] = f"../{item['target']}" - context["navbar"] = {} - for lang in context["translations"]["languages"]: - prefix = "" if lang == "en" else lang - url_prefix = "" if lang == "en" else lang + "/" - with open( - os.path.join( - context["source_path"], prefix, context["main"]["navbar_fname"] - ), - encoding="utf-8", - ) as f: - navbar_lang = yaml.safe_load(f) - - context["navbar"][lang] = navbar_lang["navbar"] - for i, item in enumerate(navbar_lang["navbar"]): + for lang in list(context["translations"]["languages"].keys())[1:]: + for i, item in enumerate(context["navbar"][lang]): has_subitems = isinstance(item["target"], list) - if lang != "en": - if has_subitems: - for sub_item in item["target"]: - update_target(sub_item, url_prefix) - else: - update_target(item, url_prefix) + if has_subitems: + for sub_item in item["target"]: + update_target(sub_item, lang) + else: + update_target(item, lang) context["navbar"][lang][i] = dict( item,