Merge remote-tracking branch 'origin' into Statnett-255

Author: nelly-hateva

CHANGELOG.md

@@ -3,6 +3,7 @@ Next release
 
 * [#255](https://github.com/statnett/Talk2PowerSystem_PM/issues/255): OBO auth flow for Cognite
 * [#255](https://github.com/statnett/Talk2PowerSystem_PM/issues/255): Update the version of `cognite-sdk` from `7.86.0` to `7.89.0`
+* [#256](https://github.com/statnett/Talk2PowerSystem_PM/issues/256): Update the queries for the ontologies and datasets information served from the `__about` endpoint
 * [#251](https://github.com/statnett/Talk2PowerSystem_PM/issues/251): Change N-Shot tool configuration to default to the base GraphDB
 * [#251](https://github.com/statnett/Talk2PowerSystem_PM/issues/251): Change N-Shot tool configuration SPARQL query template
 * [#254](https://github.com/statnett/Talk2PowerSystem_PM/issues/254): Update the version of `ttyg` from `1.9.3` to `1.10.0`, so that the chat bot can run without admin access to GraphDB

src/talk2powersystemllm/agent.py

@@ -1,6 +1,7 @@
 from base64 import b64encode
 from enum import Enum
 from pathlib import Path
+from typing import Any
 
 import yaml
 from langchain_core.language_models import BaseChatModel
@@ -81,17 +82,11 @@ class CogniteSettings(BaseModel):
 
     @model_validator(mode="after")
     def check_credentials(self) -> "CogniteSettings":
-        if self.client_secret:
-            if self.token_file_path:
-                raise ValueError("Both token_file_path and client_secret for Cognite are provided. "
-                                 "Set only one of them!")
-            elif self.interactive_client_id:
-                raise ValueError("Both interactive_client_id and client_secret for Cognite are provided. "
-                                 "Set only one of them!")
-        elif self.token_file_path:
-            if self.interactive_client_id:
-                raise ValueError("Both token_file_path and interactive_client_id for Cognite are provided. "
-                                 "Set only one of them!")
+        def exactly_one_is_not_none(*args: Any) -> bool:
+            return sum(a is not None for a in args) == 1
+
+        if not exactly_one_is_not_none(self.client_secret, self.token_file_path, self.interactive_client_id):
+            raise ValueError("Pass exactly one of 'client_secret', 'token_file_path' or 'interactive_client_id'!")
 
         if self.interactive_client_id and not self.tenant_id:
             raise ValueError("Tenant id is required!")

src/talk2powersystemllm/app/server/main.py

@@ -410,17 +410,22 @@ async def get_auth_config(
         authority=settings.security.authority,
         client_credential=agent_factory.settings.tools.cognite.client_secret,
     )
+    COGNITE_SCOPES = [f"{agent_factory.settings.tools.cognite.base_url}/.default", "offline_access"]
 
 
     def exchange_obo_for_cognite(user_access_token: str) -> dict:
-        # Try cache first (MSAL keeps an in-memory cache by default)
+        result = confidential_app.acquire_token_silent(COGNITE_SCOPES, account=None)
+        if result:
+            logging.debug("Cognite token acquired.")
+            return result["access_token"]
+
+        logging.debug("Acquiring token for Cognite using OBO.")
         result = confidential_app.acquire_token_on_behalf_of(
             user_assertion=user_access_token,
-            scopes=[f"{agent_factory.settings.tools.cognite.base_url}/.default", "offline_access"],
+            scopes=COGNITE_SCOPES,
         )
         if "access_token" not in result:
-            error_message = (f"Failed to obtain OBO token for Cognite, error: {result.get("error")}, "
-                             f"description: {result.get("error_description")}")
+            error_message = f"Failed to obtain OBO token for Cognite: {result}"
             logging.error(error_message)
             raise HTTPException(status_code=401, detail=error_message)
         return result

src/talk2powersystemllm/app/server/queries/about_datasets_query.rq

@@ -1,23 +1,21 @@
 PREFIX dcat: <http://www.w3.org/ns/dcat#>
-PREFIX onto: <http://www.ontotext.com/>
 PREFIX dct: <http://purl.org/dc/terms/>
 
-SELECT DISTINCT ?uri ?name ?date
-FROM onto:explicit {
+SELECT ?uri ?name ?date {
     ?uri a dcat:Dataset
+    OPTIONAL {
+        ?uri dct:description ?name.
+        FILTER(lang(?name) != "no")
+    }
+    OPTIONAL {
+        ?uri dct:title ?name.
+    }
     OPTIONAL {
         SELECT ?uri (SAMPLE(SUBSTR(STR(?dateTime), 1, 10)) AS ?date) {
             ?uri a dcat:Dataset;
             dct:issued ?dateTime
         }
         GROUP BY ?uri
     }
-    OPTIONAL {
-        ?uri dct:title ?title
-    }
-    OPTIONAL {
-        ?uri dct:description ?descr
-    }
-    BIND(COALESCE(?title, ?descr) AS ?name)
 }
 ORDER BY ?name

src/talk2powersystemllm/app/server/queries/about_ontologies_query.rq

@@ -1,23 +1,20 @@
-PREFIX onto: <http://www.ontotext.com/>
 PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
 PREFIX dct: <http://purl.org/dc/terms/>
 PREFIX owl: <http://www.w3.org/2002/07/owl#>
 
-SELECT ?uri ?name ?version ?date
-FROM onto:explicit {
+SELECT ?uri ?name ?date ?version {
     ?uri a owl:Ontology.
     OPTIONAL {
-        ?uri dct:title ?title
+        ?uri rdfs:label ?name
     }
     OPTIONAL {
-        ?uri rdfs:label ?label
+        ?uri dct:title ?name
     }
     OPTIONAL {
         ?uri owl:versionInfo ?version
     }
     OPTIONAL {
         ?uri dct:modified ?date
     }
-    BIND(COALESCE(?title, ?label) AS ?name)
 }
 ORDER BY ?name

src/talk2powersystemllm/app/trouble.md

@@ -536,17 +536,6 @@ Sample Response Body:
             "description": "Checks if Redis can be queried.",
             "message": "Redis can be queried."
         },
-        {
-            "status": "OK",
-            "severity": "HIGH",
-            "id": "http://talk2powersystem.no/talk2powersystem-api/cognite-healthcheck",
-            "name": "Cognite Health Check",
-            "type": "cognite",
-            "impact": "Chat bot won't be able to query Cognite or tools may not function as expected.",
-            "troubleshooting": "http://localhost:8000/__trouble#cognite-health-check-status-is-not-ok",
-            "description": "Checks if Cognite can be queried by listing the time series with limit of 1.",
-            "message": "Cognite can be queried."
-        },
         {
             "status": "OK",
             "severity": "HIGH",
@@ -1408,53 +1397,48 @@ Sample Response Body:
 The `ontologies`, `datasets` and `graphdb` sections are updated on a scheduled basis (30 seconds by default). The `agent` and `backend` sections are static and initialized at the start of the application, since the data don't change at run time.
 The SPARQL query, which fetches the `ontologies` data is
 ```
-PREFIX onto: <http://www.ontotext.com/>
 PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
 PREFIX dct: <http://purl.org/dc/terms/>
 PREFIX owl: <http://www.w3.org/2002/07/owl#>
 
-SELECT ?uri ?name ?version ?date
-FROM onto:explicit {
+SELECT ?uri ?name ?date ?version {
     ?uri a owl:Ontology.
     OPTIONAL {
-        ?uri dct:title ?title
+        ?uri rdfs:label ?name
     }
     OPTIONAL {
-        ?uri rdfs:label ?label
+        ?uri dct:title ?name
     }
     OPTIONAL {
         ?uri owl:versionInfo ?version
     }
     OPTIONAL {
         ?uri dct:modified ?date
     }
-    BIND(COALESCE(?title, ?label) AS ?name)
 }
 ORDER BY ?name
 ```
 The SPARQL query, which fetches the `datasets` data is
 ```
 PREFIX dcat: <http://www.w3.org/ns/dcat#>
-PREFIX onto: <http://www.ontotext.com/>
 PREFIX dct: <http://purl.org/dc/terms/>
 
-SELECT DISTINCT ?uri ?name ?date
-FROM onto:explicit {
+SELECT ?uri ?name ?date {
     ?uri a dcat:Dataset
+    OPTIONAL {
+        ?uri dct:description ?name.
+        FILTER(lang(?name) != "no")
+    }
+    OPTIONAL {
+        ?uri dct:title ?name.
+    }
     OPTIONAL {
         SELECT ?uri (SAMPLE(SUBSTR(STR(?dateTime), 1, 10)) AS ?date) {
             ?uri a dcat:Dataset;
             dct:issued ?dateTime
         }
         GROUP BY ?uri
     }
-    OPTIONAL {
-        ?uri dct:title ?title
-    }
-    OPTIONAL {
-        ?uri dct:description ?descr
-    }
-    BIND(COALESCE(?title, ?descr) AS ?name)
 }
 ORDER BY ?name
 ```
@@ -1497,6 +1481,7 @@ experienced with the following:
 * GraphDB
 * Cognite
 * Azure OpenAI
+* OpenID, Microsoft Entra ID
 * Redis
 
 ## Resolving Known Issues
@@ -1511,10 +1496,6 @@ Most probable cause: ["GraphDB can't be queried or is mis-configured"](#graphdb-
 
 Most probable cause: ["GraphDB can't be queried or is mis-configured"](#graphdb-cant-be-queried-or-is-mis-configured)
 
-#### Calls made by the LLM agent to the tools `sparql_query`, `autocomplete_search`, `retrieval_search` are failing
-
-Most probable cause: ["Cognite can't be queried or is mis-configured"](#cognite-cant-be-queried-or-is-mis-configured)
-
 #### Calls made by the LLM agent to the tools `retrieve_time_series`, `retrieve_data_points` are failing
 
 Most probable cause: ["Cognite can't be queried or is mis-configured"](#cognite-cant-be-queried-or-is-mis-configured)
@@ -1551,12 +1532,21 @@ This section lists the causes of known issues and provides solutions.
 ##### Solution
 
 - Make sure Cognite is reachable from the app host.
-- Make sure Cognite credentials are correct.
+- Make sure the application security is enabled, otherwise Cognite won't be accessible.
+- Make sure that in the application configuration in Azure `user impersonation` for Cognite is set under API permissions and is approved.
+- Make sure the property `tools.cognite.client_secret` is set and has a correct value. To create / obtain it you (or your Azure admin) must:
+    1. Go to the Azure Portal → Microsoft Entra ID → App registrations → Your FastAPI Backend App
+    2. In the left menu, choose Certificates & secrets
+    3. Under Client secrets, click ➕ New client secret
+    4. Give it a description and choose an expiry (6 months, 12 months, custom)
+    5. Click Add
+    Azure will show you:
+        - Value – this is your actual secret (copy it now → you can’t view it later)
+        - Secret ID – an internal reference (not needed in code)
 
 ##### Verification
 
-- Check the response status code of the `__gtg` endpoint, it must be `200`.
-- Check the response body of the `__health` endpoint, Cognite health check must have status `OK`.
+Users no longer report that the calls made by the LLM agent to the tools `retrieve_time_series`, `retrieve_data_points` are failing.
 
 #### Redis can't be queried or is mis-configured
 

src/talk2powersystemllm/tools/cognite/base.py

@@ -43,11 +43,7 @@ def __init__(
         """
 
         if obo_token:
-            credentials = OAuthInteractive(
-                authority_url=f"https://login.microsoftonline.com/{tenant_id}",
-                client_id=interactive_client_id,
-                scopes=[f"{base_url}/.default"],
-            )
+            credentials = CredentialProvider.load({"token": obo_token})
         elif token_file_path:
             self._token_file_path = token_file_path
             credentials = self._refresh()

tests/acceptance_tests/docker-compose/statements.ttl

@@ -1,7 +1,8 @@
 @prefix dcat: <http://www.w3.org/ns/dcat#> .
-@prefix dct: <http://purl.org/dc/terms/> .
+@prefix dct:  <http://purl.org/dc/terms/> .
 @prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
-@prefix owl: <http://www.w3.org/2002/07/owl#> .
+@prefix owl:  <http://www.w3.org/2002/07/owl#> .
+@prefix xsd:  <http://www.w3.org/2001/XMLSchema#> .
 
 <urn:uuid:f176961e-9aeb-11e5-91da-b8763fd99c5f> a <https://cim.ucaiug.io/ns#Substation>;
   <https://cim.ucaiug.io/ns#IdentifiedObject.mRID> "f176961e-9aeb-11e5-91da-b8763fd99c5f";
@@ -11,22 +12,32 @@
 
 <urn:uuid:5b6a8b13-4c20-4147-8ed6-7249e303e647> a dcat:Dataset;
   dct:description "State Variable (SV) part of the Nordic 44-bus synthetic test model developed by Statnett SF of the Nordic region.";
-  dct:issued "2025-02-14"^^<http://www.w3.org/2001/XMLSchema#date> , "2025-06-14"^^<http://www.w3.org/2001/XMLSchema#date> .
+  dct:issued "2025-02-14"^^xsd:date .
 
 <urn:uuid:5ad50f29-f3e5-4cf9-8519-cef17d71f8de> a dcat:Dataset;
   dct:description "DIGIN10 CGMES v3.0 Geographical Region Reference model"@en;
-  dct:issued "2022-04-01T08:55:55"^^<http://www.w3.org/2001/XMLSchema#date>;
+  dct:issued "2022-04-01T08:55:55"^^xsd:date;
   dct:title "DIGIN10-30-GeographicalRegion_RD" .
 
 <urn:uuid:f1d9a88d-0ff5-4e4b-9d6a-c353fe8232c3> a dcat:Dataset .
 
+<urn:uuid:f4c70c71-77e2-410e-9903-cbd85305cdc4> a dcat:Dataset;
+  dct:description "DIGIN10 CGMES v3.0 Base Voltage Reference model"@en, "DIGIN10 CGMES v3.0 Medium Voltage 1 (MV1) Low Voltage 1 (LV1) Boundary Model"@en;
+  dct:issued "2022-04-01T08:55:55"^^xsd:date, "2022-04-06T11:32:55"^^xsd:date;
+  dct:title "DIGIN10-30-BaseVoltage_RD", "DIGIN10-30-MV1-LV1_BM" .
+
+<urn:uuid:971c4254-5365-4aaf-8fa6-02658b3f8e05> a dcat:Dataset;
+  dct:description "Geospartial GridCapacity MAS1"@en, "Stedsdata Nettkapasitet MAS1"@no;
+  dct:issued "2023-02-21T08:08:00.000Z"^^xsd:dateTime;
+  dct:title "DIGIN10-30-WattApp-GL" .
+
 <https://ap-voc.cim4.eu/AssessedElement#Ontology> a owl:Ontology;
-  dct:modified "2024-09-07"^^<http://www.w3.org/2001/XMLSchema#date>;
+  dct:modified "2024-09-07"^^xsd:date;
   dct:title "Assessed Element Vocabulary"@en;
   owl:versionInfo "2.3.1" .
 
 <https://cim.ucaiug.io/rules#> a owl:Ontology;
-  dct:modified "2025-08-13"^^<http://www.w3.org/2001/XMLSchema#date>;
+  dct:modified "2025-08-13"^^xsd:date;
   rdfs:label "CIM Inferred Extension Ontology";
   owl:versionInfo "1.1" .