Merge pull request #14 from agile-learning-institute/add-load-and-validate-error-handling

Add load and validate error handling

Author: FlatBallFlyer

CONTRIBUTING.md

@@ -44,17 +44,30 @@ This project follows the [Stage0 development standards](https://github.com/agile
 
 - Stage0 [Developers Edition](https://github.com/agile-learning-institute/stage0/blob/main/developer_edition/README.md)
 
-### Installation
+### Quick Start
 
 ```bash
 # Clone the repository
-git clone <repository-url>
+git clone [email protected]:agile-learning-institute/stage0_mongodb_api.git
 cd stage0_mongodb_api
+pipenv run service
+# Open http://localhost:8082/
 ```
 
 ### Developer Commands
 
 ```bash
+# Configure environment for testing
+export MONGO_DB_NAME=test_database
+export MONGO_CONNECTION_STRING=mongodb://localhost:27017/?replicaSet=rs0
+
+# Select a test_case for the server
+export INPUT_FOLDER=./tests/test_cases/small_sample
+export INPUT_FOLDER=./tests/test_cases/large_sample
+
+# Set Debug Mode if needed
+export LOGGING_LEVEL=DEBUG
+
 # Install dependencies
 pipenv install --dev
 
@@ -64,64 +77,55 @@ pipenv run test
 # Run a backing mongo database
 pipenv run database
 
-# Change the testing database name
-export MONGO_DB_NAME=test_database
-# NOTE: Must be test_database for db-compare to work
-
-# Select a test_case for the server
-export INPUT_FOLDER=./tests/test_cases/small_sample
-export INPUT_FOLDER=./tests/test_cases/large_sample
-
-# Set Debug Mode
-export LOGGING_LEVEL=DEBUG
-
-# Start API server if database is already running
+## All run locally commands assume the database is running
+# Start server locally**
 pipenv run local
 
-# Start with debugging
-pipenv run debug
+# Start locally with debugging 
+pipenv run debug 
 
-# Run in Batch mode (process and shut down)
+# Run locally in Batch mode (process and shut down)
 pipenv run batch
 
-#####################
-# Black Box Testing #
+# Build container after code changes
+pipenv run build
 
-# Silent drop MONGO_DB_NAME for automation
-pipenv run db-drop-silent
+# Start Containerized Stack (Database, API, and SPA)
+pipenv run service
 
-# Compare with INPUT_FOLDER/MONGO_DB_NAME
-pipenv run db-compare
+# Stop the testing containers
+pipenv run down
 
-# Harvest current state to INPUT_FOLDER/MONGO_DB_NAME
-pipenv run db-harvest
+#####################
+# Black Box Testing #
 
-# Combine DB actions with Batch testing 
-pipenv run db-drop-silent 
-pipenv run batch 
-pipenv run db-compare
+# MongoDB Utilities
+pipenv run db-drop-silent   # drop the testing database
+pipenv run db-compare       # Compare the database to a know set of data
+pipenv run db-harvest       # Update the set of known data from the database
 
 # Run StepCI black box testing 
 pipenv run stepci-observability
 pipenv run stepci-small
 pipenv run stepci-large
 
-# Combine DB actions with StepCI testing 
-export INPUT_FOLDER=./tests/test_cases/large_sample
-pipenv run local
-# Then in a different window
+# Combine DB actions with Batch testing 
 pipenv run db-drop-silent 
-pipenv run stepci-large
-pipenv run db-compare
-
-# Build the API Docker Image
-pipenv run build
+pipenv run db-compare                   # Should fail
+pipenv run batch 
+pipenv run db-compare                   # Should pass
 
-# Run API Docker Image + a MongoDB Image
-pipenv run container
+# Combine DB actions, containerized runtime, and StepCI testing 
+pipenv run service
+pipenv run db-compare                   # Should fail
+pipenv run stepci-large
+pipenv run db-compare                   # Should pass
 
-# Stop the API and DB containers
-pipenv run down
+# Use the SPA to find errors and test configuration
+pipenv run service      # if it's not already running
+pipenv run db-compare   # Should fail
+# visit http://localhost:8082 and "process all"
+pipenv run db-compare   # Should pass
 
 ```
 

Pipfile

@@ -16,9 +16,9 @@ db-drop-silent = "python tests/db_util.py drop --passphrase DROP_DROWSSAP_YEK"
 db-compare = "python tests/db_util.py compare"
 db-harvest = "python tests/db_util.py harvest"
 build = "docker build --tag ghcr.io/agile-learning-institute/stage0_mongodb_api:latest ."
-database = "docker compose --profile mongodb up --detach"
-container = "docker compose --profile all up --detach"
-down = "docker compose down stage0_mongodb_api mongodb"
+service = "sh -c 'pipenv run down && docker compose --profile mongodb up --detach && echo `Visit http://localhost:8082/`'"
+database = "sh -c 'pipenv run down && docker compose --profile mongodb-only up --detach'"
+down = "docker compose down mongodb_spa mongodb_api mongodb"
 
 [packages]
 python-dotenv = "*"

docker-compose.yaml

@@ -1,6 +1,9 @@
 # docker-compose.yaml for stage0_mongodb_api testing
 
 services:
+  ##################################
+  # MongoDB Backing Service
+  ##################################
   mongodb:
     image: mongo:7.0.5
     ports:
@@ -15,23 +18,42 @@ services:
       retries: 30
     command: ["--replSet", "rs0", "--bind_ip_all", "--port", "27017"]
     profiles:
+      - mongodb-only
+      - mongodb-api
       - mongodb
-      - all
 
-  stage0_mongodb_api:
-    image: ghcr.io/agile-learning-institute/stage0_mongodb_api:latest
+  ##################################
+  # MongoDB configuration Service
+  ##################################
+  mongodb_api:
+    image: ghcr.io/agile-learning-institute/stage0_mongo_api:latest
     restart: no
     ports:
-      - "8081:8081"
-    volumes:
-    - ./tests/test_cases/large_sample:/input
+      - 8081:8081
     environment:
       MONGO_CONNECTION_STRING: mongodb://mongodb:27017/?replicaSet=rs0
-      MONGO_DB_NAME: test_database
-      AUTO_PROCESS: True
+      AUTO_PROCESS: False
       LOAD_TEST_DATA: True
+      MONGO_DB_NAME: test_database
+    volumes:
+      - ${INPUT_FOLDER:-./tests/test_cases/large_sample}:/input
     depends_on:
       mongodb:
         condition: service_healthy
     profiles:
-      - all
+      - mongodb-api
+      - mongodb
+
+  mongodb_spa:
+    image: ghcr.io/agile-learning-institute/stage0_mongodb_spa:latest
+    restart: no
+    environment:
+      MONGODB_API_HOST: stage0_mongodb_api
+      MONGODB_API_PORT: 8081
+      MONGODB_SPA_PORT: 8082
+    ports:
+      - 8082:8082
+    depends_on:
+      - mongodb_api
+    profiles:
+      - mongodb

docs/openapi.yaml

@@ -9,7 +9,7 @@ info:
     name: Apache 2.0
     url: http://www.apache.org/licenses/LICENSE-2.0.html
 paths:
-  /api/collections:
+  /api/collections/:
     get:
       summary: List all Configured Collections 
       operationId: list_collections
@@ -46,7 +46,7 @@ paths:
                 type: array
                 items:
                   type: object
-  /api/collections/{name}:
+  /api/collections/{name}/:
     get:
       summary: Get a collection configuration
       operationId: get_collection
@@ -108,7 +108,7 @@ paths:
                 items:
                   type: object
 
-  /api/render/json_schema/{schema_name}:
+  /api/render/json_schema/{schema_name}/:
     get:
       summary: Get a Json Schema rendered for a schema
       description: Json Schema for a specific schema version (e.g., "collection.1.0.0.1")
@@ -142,7 +142,7 @@ paths:
                 items:
                   type: object
 
-  /api/render/bson_schema/{schema_name}:
+  /api/render/bson_schema/{schema_name}/:
     get:
       summary: Get a Bson Schema rendered for a schema
       description: Bson Schema for a specific schema version (e.g., "collection.1.0.0.1")
@@ -176,7 +176,7 @@ paths:
                 items:
                   type: object
 
-  /api/render/openapi/{schema_name}:
+  /api/render/openapi/{schema_name}/:
     get:
       summary: Get a OpenAPI rendered for a schema
       description: OpenAPI Specification for a specific schema version (e.g., "collection.1.0.0.1")
@@ -248,6 +248,9 @@ components:
             type: string
           version:
             type: string
+          targeted_version:
+            type: string
+            description: The target version (last version) for this collection
     collection:
       description: SchemaManager Collection Configuration
       type: object

stage0_mongodb_api/routes/render_routes.py

@@ -10,7 +10,7 @@
 def create_render_routes():
     blueprint = Blueprint('renders', __name__)
 
-    @blueprint.route('json_schema/<schema_name>', methods=['GET'])
+    @blueprint.route('json_schema/<schema_name>/', methods=['GET'])
     def render_json_schema(schema_name):
         """Render Json Schema for a schema"""
         token = create_flask_token()
@@ -34,7 +34,7 @@ def render_json_schema(schema_name):
                 "message": str(e)
             }]), 500
 
-    @blueprint.route('bson_schema/<schema_name>', methods=['GET'])
+    @blueprint.route('bson_schema/<schema_name>/', methods=['GET'])
     def render_bson_schema(schema_name):
         """Render Bson Schema for a schema"""
         token = create_flask_token()
@@ -58,7 +58,7 @@ def render_bson_schema(schema_name):
                 "message": str(e)
             }]), 500
 
-    @blueprint.route('openapi/<schema_name>', methods=['GET'])
+    @blueprint.route('openapi/<schema_name>/', methods=['GET'])
     def render_openapi(schema_name):
         """Render OpenAPI for a schema"""
         token = create_flask_token()

stage0_mongodb_api/services/collection_service.py

@@ -33,7 +33,7 @@ def list_collections(token: Dict = None) -> List[Dict]:
             token: Authentication token for RBAC enforcement
             
         Returns:
-            List of dictionaries with collection_name and version
+            List of dictionaries with collection_name, version, and targeted_version
             
         Raises:
             CollectionProcessingError: If there are load or validation errors
@@ -52,9 +52,17 @@ def list_collections(token: Dict = None) -> List[Dict]:
         # Create a list of collection objects matching the OpenAPI schema
         collections = []
         for collection_name, collection in config_manager.collection_configs.items():
+            # Get current version
+            current_version = VersionManager.get_current_version(collection_name)
+            
+            # Get targeted version (last version in the versions array)
+            versions = collection.get("versions", [])
+            targeted_version = f"{collection_name}.{versions[-1]['version']}" if versions else None
+            
             collections.append({
                 "collection_name": collection_name,
-                "version": VersionManager.get_current_version(collection_name)
+                "version": current_version,
+                "targeted_version": targeted_version
             })
         return collections