Create utility to render and store transactional snapshot of simple detail page for a project (#8586)

* helper function to render and hash simple detail for a specific project

* use the existing Jinja2 environment

* file storage

* format with black

* store hashed project indexes at `/simple/<HASH>.<PROJECT_NAME>` as discussed

also store unhashed index as normal

* add 'simple.backend' config for tests

* add a jinja renderer to pyramid_request fixture

* restore tests for existing functionality

* test new SimpleStorage services

* license

* test render_simple_index utility

* Update warehouse/packaging/utils.py

* Update warehouse/packaging/utils.py

* fix tests, store last serial information on metadata of simple files

* reformat/lint

* Remove print statement

* Add flush

* Fix tests

* Simpflify duplication in storage services

Co-authored-by: Dustin Ingram <[email protected]>

Author: ewdurbin

dev/environment

@@ -19,6 +19,7 @@ CAMO_KEY=insecurecamokey
 DOCS_URL="https://pythonhosted.org/{project}/"
 
 FILES_BACKEND=warehouse.packaging.services.LocalFileStorage path=/var/opt/warehouse/packages/ url=http://localhost:9001/packages/{path}
+SIMPLE_BACKEND=warehouse.packaging.services.LocalSimpleStorage path=/var/opt/warehouse/simple/ url=http://localhost:9001/simple/{path}
 DOCS_BACKEND=warehouse.packaging.services.LocalDocsStorage path=/var/opt/warehouse/docs/
 SPONSORLOGOS_BACKEND=warehouse.admin.services.LocalSponsorLogoStorage path=/var/opt/warehouse/sponsorlogos/
 

docker-compose.yml

@@ -1,6 +1,7 @@
 version: '3'
 
 volumes:
+  simple:
   packages:
   sponsorlogos:
   vault:
@@ -86,6 +87,7 @@ services:
       - .coveragerc:/opt/warehouse/src/.coveragerc:z
       - packages:/var/opt/warehouse/packages
       - sponsorlogos:/var/opt/warehouse/sponsorlogos
+      - simple:/var/opt/warehouse/simple
       - ./bin:/opt/warehouse/src/bin:z
     ports:
       - "80:8000"
@@ -98,6 +100,7 @@ services:
     volumes:
       - packages:/var/opt/warehouse/packages
       - sponsorlogos:/var/opt/warehouse/sponsorlogos
+      - simple:/var/opt/warehouse/simple
     ports:
       - "9001:9001"
 
@@ -113,6 +116,7 @@ services:
     environment:
       C_FORCE_ROOT: "1"
       FILES_BACKEND: "warehouse.packaging.services.LocalFileStorage path=/var/opt/warehouse/packages/ url=http://files:9001/packages/{path}"
+      SIMPLE_BACKEND: "warehouse.packaging.services.LocalSimpleStorage path=/var/opt/warehouse/simple/ url=http://files:9001/simple/{path}"
 
   static:
     build:

tests/conftest.py

@@ -25,13 +25,17 @@
 import pytest
 import webtest as _webtest
 
+from jinja2 import Environment, FileSystemLoader
 from psycopg2.errors import InvalidCatalogName
 from pyramid.i18n import TranslationString
 from pyramid.static import ManifestCacheBuster
+from pyramid_jinja2 import IJinja2Environment
 from pytest_postgresql.config import get_config
 from pytest_postgresql.janitor import DatabaseJanitor
 from sqlalchemy import event
 
+import warehouse
+
 from warehouse import admin, config, static
 from warehouse.accounts import services as account_services
 from warehouse.macaroons import services as macaroon_services
@@ -76,6 +80,22 @@ def metrics():
     )
 
 
+@pytest.fixture
+def jinja():
+    dir_name = os.path.join(os.path.dirname(warehouse.__file__))
+
+    env = Environment(
+        loader=FileSystemLoader(dir_name),
+        extensions=[
+            "jinja2.ext.i18n",
+            "warehouse.utils.html.ClientSideIncludeExtension",
+        ],
+        cache_size=0,
+    )
+
+    return env
+
+
 class _Services:
     def __init__(self):
         self._services = defaultdict(lambda: defaultdict(dict))
@@ -98,11 +118,13 @@ def pyramid_services(metrics):
 
 
 @pytest.fixture
-def pyramid_request(pyramid_services):
+def pyramid_request(pyramid_services, jinja):
     dummy_request = pyramid.testing.DummyRequest()
     dummy_request.find_service = pyramid_services.find_service
     dummy_request.remote_addr = "1.2.3.4"
 
+    dummy_request.registry.registerUtility(jinja, IJinja2Environment, name=".jinja2")
+
     def localize(message, **kwargs):
         ts = TranslationString(message, **kwargs)
         return ts.interpolate()
@@ -184,7 +206,8 @@ def app_config(database):
         "ratelimit.url": "memory://",
         "elasticsearch.url": "https://localhost/warehouse",
         "files.backend": "warehouse.packaging.services.LocalFileStorage",
-        "docs.backend": "warehouse.packaging.services.LocalFileStorage",
+        "simple.backend": "warehouse.packaging.services.LocalSimpleStorage",
+        "docs.backend": "warehouse.packaging.services.LocalDocsStorage",
         "sponsorlogos.backend": "warehouse.admin.services.LocalSponsorLogoStorage",
         "mail.backend": "warehouse.email.services.SMTPEmailSender",
         "malware_check.backend": (

tests/unit/packaging/test_init.py

@@ -18,7 +18,7 @@
 from warehouse import packaging
 from warehouse.accounts.models import Email, User
 from warehouse.manage.tasks import update_role_invitation_status
-from warehouse.packaging.interfaces import IDocsStorage, IFileStorage
+from warehouse.packaging.interfaces import IDocsStorage, IFileStorage, ISimpleStorage
 from warehouse.packaging.models import File, Project, Release, Role
 from warehouse.packaging.tasks import (  # sync_bigquery_release_files,
     compute_trending,
@@ -51,7 +51,11 @@ def key_factory(keystring, iterate_on=None):
             lambda factory, iface, name=None: None
         ),
         registry=pretend.stub(
-            settings={"files.backend": "foo.bar", "docs.backend": "wu.tang"}
+            settings={
+                "files.backend": "foo.bar",
+                "simple.backend": "bread.butter",
+                "docs.backend": "wu.tang",
+            }
         ),
         register_origin_cache_keys=pretend.call_recorder(lambda c, **kw: None),
         get_settings=lambda: settings,
@@ -62,6 +66,7 @@ def key_factory(keystring, iterate_on=None):
 
     assert config.register_service_factory.calls == [
         pretend.call(storage_class.create_service, IFileStorage),
+        pretend.call(storage_class.create_service, ISimpleStorage),
         pretend.call(storage_class.create_service, IDocsStorage),
     ]
     assert config.register_origin_cache_keys.calls == [

tests/unit/packaging/test_services.py

@@ -22,11 +22,14 @@
 
 import warehouse.packaging.services
 
-from warehouse.packaging.interfaces import IDocsStorage, IFileStorage
+from warehouse.packaging.interfaces import IDocsStorage, IFileStorage, ISimpleStorage
 from warehouse.packaging.services import (
     GCSFileStorage,
+    GCSSimpleStorage,
+    GenericLocalBlobStorage,
     LocalDocsStorage,
     LocalFileStorage,
+    LocalSimpleStorage,
     S3DocsStorage,
     S3FileStorage,
 )
@@ -137,6 +140,67 @@ def test_delete_already_gone(self, tmpdir):
         assert response is None
 
 
+class TestLocalSimpleStorage:
+    def test_verify_service(self):
+        assert verifyClass(ISimpleStorage, LocalSimpleStorage)
+
+    def test_basic_init(self):
+        storage = LocalSimpleStorage("/foo/bar/")
+        assert storage.base == "/foo/bar/"
+
+    def test_create_service(self):
+        request = pretend.stub(
+            registry=pretend.stub(settings={"simple.path": "/simple/one/two/"})
+        )
+        storage = LocalSimpleStorage.create_service(None, request)
+        assert storage.base == "/simple/one/two/"
+
+    def test_gets_file(self, tmpdir):
+        with open(str(tmpdir.join("file.txt")), "wb") as fp:
+            fp.write(b"my test file contents")
+
+        storage = LocalSimpleStorage(str(tmpdir))
+        file_object = storage.get("file.txt")
+        assert file_object.read() == b"my test file contents"
+
+    def test_raises_when_file_non_existent(self, tmpdir):
+        storage = LocalSimpleStorage(str(tmpdir))
+        with pytest.raises(FileNotFoundError):
+            storage.get("file.txt")
+
+    def test_stores_file(self, tmpdir):
+        filename = str(tmpdir.join("testfile.txt"))
+        with open(filename, "wb") as fp:
+            fp.write(b"Test File!")
+
+        storage_dir = str(tmpdir.join("storage"))
+        storage = LocalSimpleStorage(storage_dir)
+        storage.store("foo/bar.txt", filename)
+
+        with open(os.path.join(storage_dir, "foo/bar.txt"), "rb") as fp:
+            assert fp.read() == b"Test File!"
+
+    def test_stores_two_files(self, tmpdir):
+        filename1 = str(tmpdir.join("testfile1.txt"))
+        with open(filename1, "wb") as fp:
+            fp.write(b"First Test File!")
+
+        filename2 = str(tmpdir.join("testfile2.txt"))
+        with open(filename2, "wb") as fp:
+            fp.write(b"Second Test File!")
+
+        storage_dir = str(tmpdir.join("storage"))
+        storage = LocalSimpleStorage(storage_dir)
+        storage.store("foo/first.txt", filename1)
+        storage.store("foo/second.txt", filename2)
+
+        with open(os.path.join(storage_dir, "foo/first.txt"), "rb") as fp:
+            assert fp.read() == b"First Test File!"
+
+        with open(os.path.join(storage_dir, "foo/second.txt"), "rb") as fp:
+            assert fp.read() == b"Second Test File!"
+
+
 class TestS3FileStorage:
     def test_verify_service(self):
         assert verifyClass(IFileStorage, S3FileStorage)
@@ -479,3 +543,98 @@ def test_delete_by_prefix_with_storage_prefix(self):
                 },
             ),
         ]
+
+
+class TestGCSSimpleStorage:
+    def test_verify_service(self):
+        assert verifyClass(ISimpleStorage, GCSSimpleStorage)
+
+    def test_basic_init(self):
+        bucket = pretend.stub()
+        storage = GCSSimpleStorage(bucket)
+        assert storage.bucket is bucket
+
+    def test_create_service(self):
+        service = pretend.stub(
+            get_bucket=pretend.call_recorder(lambda bucket_name: pretend.stub())
+        )
+        request = pretend.stub(
+            find_service=pretend.call_recorder(lambda name: service),
+            registry=pretend.stub(settings={"simple.bucket": "froblob"}),
+        )
+        GCSSimpleStorage.create_service(None, request)
+
+        assert request.find_service.calls == [pretend.call(name="gcloud.gcs")]
+        assert service.get_bucket.calls == [pretend.call("froblob")]
+
+    def test_gets_file_raises(self):
+        storage = GCSSimpleStorage(pretend.stub())
+
+        with pytest.raises(NotImplementedError):
+            storage.get("file.txt")
+
+    def test_stores_file(self, tmpdir):
+        filename = str(tmpdir.join("testfile.txt"))
+        with open(filename, "wb") as fp:
+            fp.write(b"Test File!")
+
+        blob = pretend.stub(
+            upload_from_filename=pretend.call_recorder(lambda file_path: None),
+            exists=lambda: False,
+        )
+        bucket = pretend.stub(blob=pretend.call_recorder(lambda path: blob))
+        storage = GCSSimpleStorage(bucket)
+        storage.store("foo/bar.txt", filename)
+
+        assert bucket.blob.calls == [pretend.call("foo/bar.txt")]
+        assert blob.upload_from_filename.calls == [pretend.call(filename)]
+
+    def test_stores_two_files(self, tmpdir):
+        filename1 = str(tmpdir.join("testfile1.txt"))
+        with open(filename1, "wb") as fp:
+            fp.write(b"First Test File!")
+
+        filename2 = str(tmpdir.join("testfile2.txt"))
+        with open(filename2, "wb") as fp:
+            fp.write(b"Second Test File!")
+
+        blob = pretend.stub(
+            upload_from_filename=pretend.call_recorder(lambda file_path: None),
+            exists=lambda: False,
+        )
+        bucket = pretend.stub(blob=pretend.call_recorder(lambda path: blob))
+        storage = GCSSimpleStorage(bucket)
+        storage.store("foo/first.txt", filename1)
+        storage.store("foo/second.txt", filename2)
+
+        assert bucket.blob.calls == [
+            pretend.call("foo/first.txt"),
+            pretend.call("foo/second.txt"),
+        ]
+        assert blob.upload_from_filename.calls == [
+            pretend.call(filename1),
+            pretend.call(filename2),
+        ]
+
+    def test_stores_metadata(self, tmpdir):
+        filename = str(tmpdir.join("testfile.txt"))
+        with open(filename, "wb") as fp:
+            fp.write(b"Test File!")
+
+        blob = pretend.stub(
+            upload_from_filename=pretend.call_recorder(lambda file_path: None),
+            patch=pretend.call_recorder(lambda: None),
+            exists=lambda: False,
+        )
+        bucket = pretend.stub(blob=pretend.call_recorder(lambda path: blob))
+        storage = GCSSimpleStorage(bucket)
+        meta = {"foo": "bar"}
+        storage.store("foo/bar.txt", filename, meta=meta)
+
+        assert blob.metadata == meta
+
+
+class TestGenericLocalBlobStorage:
+    def test_notimplementederror(self):
+        with pytest.raises(NotImplementedError):
+            GenericLocalBlobStorage.create_service(pretend.stub(), pretend.stub())