Support generic sorting

PBENCH-1126

With pagination, a client (e.g., the dashboard) can't rely on client-side
column sorting. Instead, add generalize sorting to `GET /datasets`, allowing
the returned datasets to be sorted by any column or metadata value, either
ascending (default) or descending.

`GET /api/v1/datasets?sort=user.dashboard.favorite:desc,dataset.uploaded` will
return all accessible datasets, sorted first by whether the authenticated user
has marked the dataset "favorite" and second by the upload timestamp. (All
"favorited" datasets will appear first, in upload order, followed by all "non-
favorited" datasets in upload order.)

Author: dbutenhof

docs/API/V1/list.md

@@ -6,7 +6,9 @@ can only list datasets with access `public`.)
 
 The collection of datasets may be filtered using any combination of a number
 of query parameters, including `owner`, `access`, `name` substring, date range,
-and arbitrary metadata filter expressions.
+and arbitrary metadata filter expressions. The selected datasets may be sorted
+by any metadata key value in either ascending or descending order. Multiple
+sort parameters will be processed in order.
 
 Large collections can be paginated for efficiency using the `limit` and `offset`
 query parameters.
@@ -90,6 +92,24 @@ with a paginated display or to limit data transfer requirements.
 Select only datasets owned by the specified username. Unless the username
 matches the authenticated user, only "public" datasets can be selected.
 
+`sort` sort expression \
+Sort the returned datasets by one or more sort expressions. You can separate
+multiple expressions using comma lists, or across separate `sort` query
+parameters, which will be processed in order. Any Metadata namespace key can
+be specified.
+
+Specify a sort order using the keywords `asc` (ascending) or `desc`
+(descending), separated from the key name with a colon (`:`). For example,
+`dataset.name:asc` or `dataset.metalog.pbench.script:desc`. The default is
+"ascending" if no order is specified. If no sort expressions are specified,
+datasets are returned sorted by `dataset.resource_id`.
+
+For example, `GET /api/v1/datasets?sort=global.dashboard.seen:desc,dataset.name`
+will return selected datasets sorted first in descending order based on whether
+the dataset has been marked "seen" by the dashboard, and secondly sorted by the
+dataset name. The Pbench Dashboard stores `global.dashboard.seen` as a `boolean`
+value, so in this case `true` values will appear before `false` values.
+
 `start` date/time \
 Select only datasets created on or after the specified time. Time should be
 specified in ISO standard format, as `YYYY-MM-DDThh:mm:ss.ffffff[+|-]HH:MM`.

lib/pbench/server/api/resources/datasets_list.py

@@ -5,7 +5,7 @@
 from flask import current_app
 from flask.json import jsonify
 from flask.wrappers import Request, Response
-from sqlalchemy import and_, cast, or_, String
+from sqlalchemy import and_, asc, cast, desc, or_, String
 from sqlalchemy.exc import ProgrammingError, StatementError
 from sqlalchemy.orm import aliased, Query
 from sqlalchemy.sql.expression import Alias
@@ -79,6 +79,12 @@ def __init__(self, config: PbenchServerConfig):
                         string_list=",",
                         metalog_ok=True,
                     ),
+                    Parameter(
+                        "sort",
+                        ParamType.LIST,
+                        element_type=ParamType.STRING,
+                        string_list=",",
+                    ),
                 ),
                 authorization=ApiAuthorizationType.USER_ACCESS,
             ),
@@ -104,7 +110,7 @@ def get_paginated_obj(
             start to narrow down the result.
         """
         paginated_result = {}
-        query = query.order_by(Dataset.resource_id).distinct()
+        query = query.distinct()
         total_count = query.count()
 
         # Shift the query search by user specified offset value,
@@ -221,7 +227,7 @@ def filter_query(
                 k, v = kw.split(":", maxsplit=1)
             except ValueError:
                 raise APIAbort(
-                    HTTPStatus.BAD_REQUEST, f"filter {kw!r} must have the form 'k=v'"
+                    HTTPStatus.BAD_REQUEST, f"filter {kw!r} must have the form 'k:v'"
                 )
             if k.startswith("^"):
                 combine_or = True
@@ -348,20 +354,77 @@ def keyspace(self, query: Query) -> JSONOBJECT:
                     self.accumulate(aggregate, m.key, m.value)
         return aggregate
 
-    def datasets(self, request: Request, json: JSONOBJECT, query: Query) -> JSONOBJECT:
+    def datasets(
+        self, request: Request, aliases: dict[str, Any], json: JSONOBJECT, query: Query
+    ) -> JSONOBJECT:
         """Gather and paginate the selected datasets
 
         Run the query we've compiled, with pagination limits applied; collect
         results into a list of JSON objects including selected metadata keys.
 
         Args:
             request: The HTTP Request object
+            aliases: Map of join column aliases for each Metadata namespace
             json: The JSON query parameters
             query: The basic filtered SQLAlchemy query object
 
         Returns:
             The paginated dataset listing
         """
+
+        # Process a possible list of sort terms. By default, we sort by the
+        # dataset resource_id.
+        sorters = []
+        for sort in json.get("sort", ["dataset.resource_id"]):
+            if ":" not in sort:
+                k = sort
+                order = asc
+            else:
+                try:
+                    k, o = sort.split(":", maxsplit=1)
+                except ValueError:
+                    raise APIAbort(
+                        HTTPStatus.BAD_REQUEST,
+                        f"sort {sort!r} must have the form 'k[:o]'",
+                    )
+                if o.lower() == "asc":
+                    order = asc
+                elif o.lower() == "desc":
+                    order = desc
+                else:
+                    raise APIAbort(
+                        HTTPStatus.BAD_REQUEST,
+                        f"sort order in {sort!r} must be 'asc' or 'desc'",
+                    )
+
+            if not Metadata.is_key_path(k, Metadata.METADATA_KEYS, metalog_key_ok=True):
+                raise APIAbort(HTTPStatus.BAD_REQUEST, str(MetadataBadKey(k)))
+            keys = k.split(".")
+            native_key = keys.pop(0).lower()
+            sorter = None
+            if native_key == Metadata.DATASET:
+                second = keys[0].lower()
+                # The dataset namespace requires special handling because
+                # "dataset.metalog" is really a special native key space
+                # named "metalog", while other "dataset" sub-keys are primary
+                # columns in the Dataset table.
+                if second == Metadata.METALOG:
+                    native_key = keys.pop(0).lower()
+                else:
+                    try:
+                        c = getattr(Dataset, second)
+                    except AttributeError as e:
+                        raise APIAbort(
+                            HTTPStatus.BAD_REQUEST, str(MetadataBadKey(k))
+                        ) from e
+                    sorter = order(c)
+            if sorter is None:
+                sorter = order(aliases[native_key].value[keys])
+            sorters.append(sorter)
+
+        # Apply our list of sort terms
+        query = query.order_by(*sorters)
+
         try:
             datasets, paginated_result = self.get_paginated_obj(
                 query=query, json=json, url=request.url
@@ -498,4 +561,4 @@ def _get(
         if json.get("keysummary"):
             return jsonify(self.keyspace(query))
         else:
-            return jsonify(self.datasets(request, json, query))
+            return jsonify(self.datasets(request, aliases, json, query))

lib/pbench/test/unit/server/conftest.py

@@ -386,10 +386,10 @@ def more_datasets(
         test        20 private  1970-01-01:00:42
         fio_1        3 public   1978-06-26:08:00
         fio_2       20 public   2022-01-01:00:00
-        uperf_1     20 private  1978-06-26:08:00
-        uperf_2     20 private  1978-06-26:08:00
-        uperf_3     20 private  1978-06-26:08:00
-        uperf_4     20 private  1978-06-26:08:00
+        uperf_1     20 private  1978-06-26:08:10
+        uperf_2     20 private  1978-06-26:09:01
+        uperf_3     20 private  1978-06-26:09:30
+        uperf_4     20 private  1978-06-26:10:00
 
     Args:
         client: Provide a Flask API client
@@ -399,44 +399,48 @@ def more_datasets(
         attach_dataset: Provide some datasets
         create_user: Create the "test" user
     """
-    with freeze_time("1978-06-26 08:00:00"):
-        Dataset(
-            owner=create_drb_user,
-            name="fio_1",
-            access="public",
-            resource_id="random_md5_string3",
-        ).add()
-        Dataset(
-            owner=create_user,
-            uploaded=datetime.datetime(2022, 1, 1),
-            name="fio_2",
-            access="public",
-            resource_id="random_md5_string4",
-        ).add()
-        Dataset(
-            owner=create_user,
-            name="uperf_1",
-            access="private",
-            resource_id="random_md5_string5",
-        ).add()
-        Dataset(
-            owner=create_user,
-            name="uperf_2",
-            access="private",
-            resource_id="random_md5_string6",
-        ).add()
-        Dataset(
-            owner=create_user,
-            name="uperf_3",
-            access="private",
-            resource_id="random_md5_string7",
-        ).add()
-        Dataset(
-            owner=create_user,
-            name="uperf_4",
-            access="private",
-            resource_id="random_md5_string8",
-        ).add()
+    Dataset(
+        owner=create_drb_user,
+        uploaded=datetime.datetime(1978, 6, 26, 8, 0, 0, 0),
+        name="fio_1",
+        access="public",
+        resource_id="random_md5_string3",
+    ).add()
+    Dataset(
+        owner=create_user,
+        uploaded=datetime.datetime(2022, 1, 1),
+        name="fio_2",
+        access="public",
+        resource_id="random_md5_string4",
+    ).add()
+    Dataset(
+        owner=create_user,
+        uploaded=datetime.datetime(1978, 6, 26, 8, 1, 0, 0),
+        name="uperf_1",
+        access="private",
+        resource_id="random_md5_string5",
+    ).add()
+    Dataset(
+        owner=create_user,
+        uploaded=datetime.datetime(1978, 6, 26, 9, 0, 0, 0),
+        name="uperf_2",
+        access="private",
+        resource_id="random_md5_string6",
+    ).add()
+    Dataset(
+        owner=create_user,
+        uploaded=datetime.datetime(1978, 6, 26, 9, 30, 0, 0),
+        name="uperf_3",
+        access="private",
+        resource_id="random_md5_string7",
+    ).add()
+    Dataset(
+        owner=create_user,
+        uploaded=datetime.datetime(1978, 6, 26, 10, 0, 0, 0),
+        name="uperf_4",
+        access="private",
+        resource_id="random_md5_string8",
+    ).add()
 
 
 @pytest.fixture()

lib/pbench/test/unit/server/test_datasets_list.py

@@ -5,7 +5,7 @@
 
 import pytest
 import requests
-from sqlalchemy import and_
+from sqlalchemy import and_, desc
 from sqlalchemy.exc import ProgrammingError
 from sqlalchemy.orm import aliased, Query
 
@@ -14,6 +14,7 @@
 from pbench.server.api.resources.datasets_list import DatasetsList, urlencode_json
 from pbench.server.database.database import Database
 from pbench.server.database.models.datasets import Dataset, Metadata
+from pbench.server.database.models.users import User
 from pbench.test.unit.server import DRB_USER_ID
 
 FLATTEN = re.compile(r"[\n\s]+")
@@ -147,7 +148,7 @@ def get_results(self, name_list: list[str], query: JSON, server_config) -> JSON:
             paginated_name_list = name_list[offset:]
             next_url = ""
 
-        for name in sorted(paginated_name_list):
+        for name in paginated_name_list:
             dataset = Dataset.query(name=name)
             results.append(
                 {
@@ -175,8 +176,8 @@ def compare_results(
         expected = self.get_results(name_list, query, server_config)
         for k, v in result.items():
             if k == "results":
-                assert sorted(v, key=lambda d: d["resource_id"]) == sorted(
-                    expected[k], key=lambda d: d["resource_id"]
+                assert (
+                    v == expected[k]
                 ), f"Actual {k}={v} doesn't match expected {expected[k]}"
             else:
                 assert (
@@ -698,3 +699,76 @@ def test_key_summary(self, query_as):
                 "origin": None,
             },
         }
+
+    @pytest.mark.parametrize(
+        "sort,results",
+        [
+            (
+                "dataset.name",
+                ["fio_1", "fio_2", "test", "uperf_1", "uperf_2", "uperf_3", "uperf_4"],
+            ),
+            (
+                "dataset.name:asc",
+                ["fio_1", "fio_2", "test", "uperf_1", "uperf_2", "uperf_3", "uperf_4"],
+            ),
+            (
+                "dataset.name:desc",
+                ["uperf_4", "uperf_3", "uperf_2", "uperf_1", "test", "fio_2", "fio_1"],
+            ),
+            (
+                "dataset.uploaded",
+                ["test", "fio_1", "uperf_1", "uperf_2", "uperf_3", "uperf_4", "fio_2"],
+            ),
+            (
+                "dataset.uploaded:desc",
+                ["fio_2", "uperf_4", "uperf_3", "uperf_2", "uperf_1", "fio_1", "test"],
+            ),
+            (
+                "dataset.metalog.run.controller",
+                ["test", "fio_1", "fio_2", "uperf_1", "uperf_2", "uperf_3", "uperf_4"],
+            ),
+            (
+                "global.test.sequence:desc",
+                ["fio_1", "fio_2", "test", "uperf_1", "uperf_2", "uperf_3", "uperf_4"],
+            ),
+            (
+                "global.test.sequence",
+                ["uperf_4", "uperf_3", "uperf_2", "uperf_1", "test", "fio_2", "fio_1"],
+            ),
+            (
+                "user.test.odd,global.test.sequence:desc",
+                ["fio_1", "test", "uperf_2", "uperf_4", "fio_2", "uperf_1", "uperf_3"],
+            ),
+            (
+                "user.test.odd:desc,dataset.name:desc",
+                ["uperf_3", "uperf_1", "fio_2", "uperf_4", "uperf_2", "test", "fio_1"],
+            ),
+        ],
+    )
+    def test_dataset_sort(self, server_config, query_as, sort, results):
+        """Test `datasets/list?sort`
+
+        We want a couple of consistent values sequences to play with. We can
+        use the dataset.name and dataset.resource_id fields, but we want to
+        cross Metadata namespaces, so add "global" and "user" keys we can
+        order.
+
+        Args:
+            server_config: The PbenchServerConfig object
+            query_as: A fixture to provide a helper that executes the API call
+            login: The username as which to perform a query
+            query: A JSON representation of the query parameters (these will be
+                automatically supplemented with a metadata request term)
+            results: A list of the dataset names we expect to be returned
+        """
+
+        # Assign "sequence numbers" in the inverse order of name
+        test = User.query(username="test")
+        all = Database.db_session.query(Dataset).order_by(desc(Dataset.name)).all()
+        for i, d in enumerate(all):
+            odd = i & 1
+            Metadata.setvalue(d, "global.test.sequence", i)
+            Metadata.setvalue(d, "user.test.odd", odd, user=test)
+        query = {"sort": sort, "metadata": ["dataset.uploaded"]}
+        result = query_as(query, "test", HTTPStatus.OK)
+        self.compare_results(result.json, results, query, server_config)