Synced webpages from sitemap to Document model for search.

Author: sarahboyce

docs/management/commands/update_docs.py

@@ -132,6 +132,8 @@ def build_doc_release(self, release, force=False, interactive=False):
         if self.verbosity >= 1:
             self.stdout.write(f"Starting update for {release} at {datetime.now()}...")
 
+        release.sync_from_sitemap()
+
         # checkout_dir is shared for all languages.
         checkout_dir = settings.DOCS_BUILD_ROOT.joinpath("sources", release.version)
         parent_build_dir = settings.DOCS_BUILD_ROOT.joinpath(

docs/models.py

@@ -5,6 +5,7 @@
 from functools import partial, reduce
 from pathlib import Path
 
+import requests
 from django.conf import settings
 from django.contrib.postgres.indexes import GinIndex
 from django.contrib.postgres.search import (
@@ -34,8 +35,16 @@
     START_SEL,
     STOP_SEL,
     TSEARCH_CONFIG_LANGUAGES,
+    DocumentationCategory,
+    fetch_html,
     get_document_search_vector,
 )
+from .utils import extract_inner_html
+
+
+def get_search_config(lang):
+    """Determine the PostgreSQL search language"""
+    return TSEARCH_CONFIG_LANGUAGES.get(lang[:2], DEFAULT_TEXT_SEARCH_CONFIG)
 
 
 class DocumentReleaseQuerySet(models.QuerySet):
@@ -175,7 +184,7 @@ def sync_to_db(self, decoded_documents):
         the database. Deletes all the release's documents first then
         reinserts them as needed.
         """
-        self.documents.all().delete()
+        self.documents.exclude(metadata__parents=DocumentationCategory.WEBSITE).delete()
 
         # Read excluded paths from robots.docs.txt.
         robots_path = settings.BASE_DIR.joinpath(
@@ -206,16 +215,54 @@ def sync_to_db(self, decoded_documents):
                 path=document_path,
                 title=html.unescape(strip_tags(document["title"])),
                 metadata=document,
-                config=TSEARCH_CONFIG_LANGUAGES.get(
-                    self.lang[:2], DEFAULT_TEXT_SEARCH_CONFIG
-                ),
+                config=get_search_config(self.lang),
             )
-        for document in self.documents.all():
+        for document in self.documents.exclude(
+            metadata__parents=DocumentationCategory.WEBSITE
+        ):
             document.metadata["breadcrumbs"] = list(
                 Document.objects.breadcrumbs(document).values("title", "path")
             )
             document.save(update_fields=("metadata",))
 
+    def sync_from_sitemap(self):
+        from djangoproject.urls.www import sitemaps
+
+        if self.lang != "en" or self.release:
+            return
+
+        for sitemap in sitemaps.values():
+            for url in sitemap().get_urls():
+                try:
+                    page_html = fetch_html(url["location"])
+                except requests.RequestException:
+                    continue
+                try:
+                    main_html = extract_inner_html(page_html, tag="main")
+                    title = extract_inner_html(page_html, tag="h1")
+                except ValueError:
+                    continue
+                Document.objects.update_or_create(
+                    release=self,
+                    path=url["location"],
+                    defaults={
+                        "title": title,
+                        "metadata": {
+                            "body": main_html,
+                            "breadcrumbs": [
+                                {
+                                    "path": DocumentationCategory.WEBSITE,
+                                    "title": "Website",
+                                },
+                            ],
+                            "parents": DocumentationCategory.WEBSITE,
+                            "title": title,
+                            "toc": "",
+                        },
+                        "config": get_search_config(self.lang),
+                    },
+                )
+
 
 def _clean_document_path(path):
     # We have to be a bit careful to reverse-engineer the correct
@@ -228,7 +275,9 @@ def _clean_document_path(path):
 
 
 def document_url(doc):
-    if doc.path:
+    if doc.metadata.get("parents") == DocumentationCategory.WEBSITE:
+        return doc.path
+    elif doc.path:
         kwargs = {
             "lang": doc.release.lang,
             "version": doc.release.version,
@@ -273,6 +322,14 @@ def search(self, query_text, release, document_category=None):
                 config=models.F("config"),
             )
             base_filter = Q(release_id=release.id)
+            if release.lang == "en" and release.version != "dev":
+                dev_release = DocumentRelease.objects.get_by_version_and_lang(
+                    "dev", "en"
+                )
+                base_filter |= Q(
+                    release_id=dev_release.id,
+                    metadata__parents=DocumentationCategory.WEBSITE,
+                )
             if document_category:
                 base_filter &= Q(metadata__parents__startswith=document_category)
             base_qs = (

docs/search.py

@@ -1,3 +1,4 @@
+import requests
 from django.contrib.postgres.search import SearchVector
 from django.db.models import TextChoices
 from django.db.models.fields.json import KeyTextTransform
@@ -67,10 +68,41 @@ class DocumentationCategory(TextChoices):
     TOPICS = "topics", _("Using Django")
     HOWTO = "howto", _("How-to guides")
     RELEASE_NOTES = "releases", _("Release notes")
+    WEBSITE = "website", _("Django Website")
 
     @classmethod
     def parse(cls, value, default=None):
         try:
             return cls(value)
         except ValueError:
             return None
+
+
+def fetch_html(url, timeout=10):
+    """
+    Fetch the HTML of a page if status code is 200.
+    Simulates a human browser and accepts only text/html.
+    """
+
+    headers = {
+        "User-Agent": (
+            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+            "AppleWebKit/537.36 (KHTML, like Gecko) "
+            "Chrome/123.0.0.0 Safari/537.36 Edg/123.0.0.0"
+        ),
+        "Accept": "text/html",
+        "Accept-Language": "en-US,en;q=0.9",
+    }
+
+    response = requests.get(url, headers=headers, timeout=timeout)
+
+    if response.status_code == 200:
+        content_type = response.headers.get("Content-Type", "")
+        if "text/html" in content_type:
+            return response.text
+        else:
+            raise requests.RequestException(f"Unexpected Content-Type: {content_type}")
+    else:
+        raise requests.RequestException(
+            f"Failed to fetch {url}, status code: {response.status_code}"
+        )

docs/templates/docs/search_results.html

@@ -43,11 +43,11 @@ <h2>{% translate "No search query given" %}</h2>
         {% for result in page.object_list %}
           <dt>
             <h2 class="result-title">
-              <a href="{% url 'document-detail' lang=result.release.lang version=result.release.version url=result.path host 'docs' %}{% if not start_sel in result.headline %}{{ result.highlight|fragment }}{% endif %}">{{ result.headline|safe }}</a>
+              <a href="{{ result.get_absolute_url }}{% if not start_sel in result.headline %}{{ result.highlight|fragment }}{% endif %}">{{ result.headline|safe }}</a>
             </h2>
             <span class="meta breadcrumbs">
               {% for breadcrumb in result.breadcrumbs %}
-                <a href="{% url 'document-detail' lang=result.release.lang version=result.release.version url=breadcrumb.path host 'docs' %}">{{ breadcrumb.title }}</a>{% if not forloop.last %} <span class="arrow">»</span>{% endif %}
+                <a href="{{ result.get_absolute_url }}">{{ breadcrumb.title }}</a>{% if not forloop.last %} <span class="arrow">»</span>{% endif %}
               {% endfor %}
             </span>
           </dt>
@@ -60,7 +60,7 @@ <h2 class="result-title">
               <ul class="code-links">
                 {% for name, value in result_code_links.items %}
                   <li>
-                    <a href="{% url 'document-detail' lang=result.release.lang version=result.release.version url=result.path host 'docs' %}#{{ value.full_path }}">
+                    <a href="{{ result.get_absolute_url }}#{{ value.full_path }}">
                       <div>
                         <code>{{ name }}</code>
                         {% if value.module_path %}<div class="meta">{{ value.module_path }}</div>{% endif %}

docs/tests/test_models.py

@@ -1,13 +1,17 @@
 import datetime
 from operator import attrgetter
 
+import requests_mock
 from django.conf import settings
 from django.db import connection
 from django.test import TestCase
+from django.utils import timezone
 
+from blog.models import ContentFormat, Entry
 from releases.models import Release
 
 from ..models import Document, DocumentRelease
+from ..search import DocumentationCategory
 
 
 class ModelsTests(TestCase):
@@ -173,6 +177,7 @@ def test_get_available_languages_by_version(self):
 class DocumentManagerTest(TestCase):
     @classmethod
     def setUpTestData(cls):
+        cls.dev_release = DocumentRelease.objects.create(lang="en")
         cls.release = DocumentRelease.objects.create(
             release=Release.objects.create(version="1.2.3"),
         )
@@ -347,6 +352,20 @@ def setUpTestData(cls):
                 "release": cls.release_fr,
                 "title": "Notes de publication de Django 1.9.4",
             },
+            {
+                "metadata": {
+                    "body": "Main 1",
+                    "breadcrumbs": [
+                        {"path": DocumentationCategory.WEBSITE, "title": "Website"}
+                    ],
+                    "parents": DocumentationCategory.WEBSITE,
+                    "title": "Title 1",
+                    "toc": "",
+                },
+                "path": "example",
+                "release": cls.dev_release,
+                "title": "Blog post",
+            },
         ]
         Document.objects.bulk_create(Document(**doc) for doc in documents)
 
@@ -446,6 +465,16 @@ def test_search_title(self):
                 ),
             )
 
+    def test_website_document_items_included_english(self):
+        self.assertQuerySetEqual(
+            Document.objects.search("Main", self.release),
+            ["Blog post"],
+            transform=attrgetter("title"),
+        )
+
+    def test_website_document_items_excluded_non_english(self):
+        self.assertEqual(Document.objects.search("Main", self.release_fr).count(), 0)
+
 
 class UpdateDocTests(TestCase):
     @classmethod
@@ -547,3 +576,77 @@ def test_excluded_documents(self):
         )
         document = release.documents.get()
         self.assertEqual(document.path, "nonexcluded/bar")
+
+    def test_sync_to_db_not_delete_website_docs(self):
+        Document.objects.create(
+            release=self.release,
+            path="example_path",
+            title="Title 1",
+            metadata={
+                "body": "Main 1",
+                "breadcrumbs": [
+                    {"path": DocumentationCategory.WEBSITE, "title": "Website"}
+                ],
+                "parents": DocumentationCategory.WEBSITE,
+                "title": "Title 1",
+                "toc": "",
+            },
+        )
+        self.release.sync_to_db([])
+        self.assertEqual(Document.objects.filter(release=self.release).count(), 1)
+
+    def test_sync_from_sitemap_skip_non_en_dev_release(self):
+        release = Release.objects.create(version="5.2")
+        Entry.objects.create(
+            pub_date=timezone.now() - datetime.timedelta(days=2),
+            slug="a",
+            body="<strong>test</strong>",
+            content_format=ContentFormat.HTML,
+            is_active=True,
+        )
+        for lang, release_obj in [("fr", None), ("fr", release), ("en", release)]:
+            doc_release = DocumentRelease.objects.create(
+                lang=lang,
+                release=release_obj,
+            )
+            with self.subTest(lang=lang, release=release_obj):
+                doc_release.sync_from_sitemap()
+                self.assertFalse(Document.objects.exists())
+
+    @requests_mock.mock()
+    def test_sync_from_sitemap(self, mocker):
+        blog_entry = Entry.objects.create(
+            pub_date=timezone.now() - datetime.timedelta(days=2),
+            slug="a",
+            body="<strong>test</strong>",
+            content_format=ContentFormat.HTML,
+            is_active=True,
+        )
+        mocker.get(
+            blog_entry.get_absolute_url(),
+            text="<html><main>Main 1</main><h1>Title 1</h1></html>",
+            headers={"Content-Type": "text/html"},
+        )
+        self.release.sync_from_sitemap()
+
+        document = Document.objects.get(release=self.release)
+        self.assertEqual(
+            document.path,
+            blog_entry.get_absolute_url(),
+        )
+        self.assertEqual(
+            document.title,
+            "Title 1",
+        )
+        self.assertEqual(
+            document.metadata,
+            {
+                "body": "Main 1",
+                "breadcrumbs": [
+                    {"path": DocumentationCategory.WEBSITE, "title": "Website"}
+                ],
+                "parents": DocumentationCategory.WEBSITE,
+                "title": "Title 1",
+                "toc": "",
+            },
+        )

docs/tests/test_utils.py

@@ -3,7 +3,7 @@
 
 from django.test import SimpleTestCase
 
-from ..utils import get_doc_path, sanitize_for_trigram
+from ..utils import extract_inner_html, get_doc_path, sanitize_for_trigram
 
 
 class TestUtils(SimpleTestCase):
@@ -38,3 +38,39 @@ def test_sanitize_for_trigram(self):
         ]:
             with self.subTest(query=query):
                 self.assertEqual(sanitize_for_trigram(query), sanitized_query)
+
+    def test_extract_inner_html(self):
+        for html, expected_output in [
+            ("<main><p>Hello</p></main>", "<p>Hello</p>"),
+            (
+                '<header>Test</header><main id="app" class="container">'
+                "<h1>Title</h1></main>",
+                "<h1>Title</h1>",
+            ),
+            ("<main>&amp; &lt; &gt; &#169;</main>", "& < > ©"),
+            ("<main></main>", ""),
+            ("<main>Hello world</main>", "Hello world"),
+            ("<main><h1>Hi</h1>Text<p>Bye</p></main>", "<h1>Hi</h1>Text<p>Bye</p>"),
+        ]:
+            with self.subTest(html=html):
+                self.assertEqual(extract_inner_html(html, tag="main"), expected_output)
+
+    def test_extract_inner_html_multiple_same_tags_raises(self):
+        with self.assertRaisesMessage(
+            ValueError, "<main> occurs more than once in HTML."
+        ):
+            extract_inner_html(
+                "<main>One main</main><main id='dupe'>Two main</main>", tag="main"
+            )
+
+    def test_extract_inner_html_multiple_same_tags_nested_raises(self):
+        with self.assertRaisesMessage(
+            ValueError, "Nested <main> tags are not allowed."
+        ):
+            extract_inner_html(
+                "<main>One main<main id='dupe'>Two main</main></main>", tag="main"
+            )
+
+    def test_extract_inner_html_tag_not_found_raises(self):
+        with self.assertRaisesMessage(ValueError, "<main> not found in HTML."):
+            extract_inner_html("<p>Test</p>", tag="main")