Merge pull request #6 from agile-learning-institute/improve-logging

Improved logging, Better container testing, Updates related to collec…

Author: FlatBallFlyer

CONTRIBUTING.md

@@ -61,6 +61,9 @@ pipenv install --dev
 # Run Unit Tests and generate coverage report
 pipenv run test
 
+# Run a backing mongo database
+pipenv run database
+
 # Change the testing database name
 export MONGO_DB_NAME=test_database
 # NOTE: Must be test_database for db-compare to work
@@ -105,8 +108,10 @@ pipenv run stepci-large
 
 # Combine DB actions with StepCI testing 
 export INPUT_FOLDER=./tests/test_cases/large_sample
+pipenv run local
+# Then in a different window
 pipenv run db-drop-silent 
-pipenv run stepci_large
+pipenv run stepci-large
 pipenv run db-compare
 
 # Build the API Docker Image
@@ -160,31 +165,31 @@ If you need a new set of test data to validate features you are adding, feel fre
 
 ```bash
 # Get Configuration
-curl -X GET http://localhost:8582/api/config
+curl -X GET http://localhost:8081/api/config
 
 # Health Check
-curl -X GET http://localhost:8582/health
+curl -X GET http://localhost:8081/health
 
 # List Collections
-curl -X GET http://localhost:8582/api/collections/
+curl -X GET http://localhost:8081/api/collections/
 
 # Get a Collection Config
-curl -X GET http://localhost:8582/api/collections/{collection_name}
+curl -X GET http://localhost:8081/api/collections/{collection_name}
 
 # Process All Collections
-curl -X POST http://localhost:8582/api/collections/
+curl -X POST http://localhost:8081/api/collections/
 
 # Process Specific Collection
-curl -X POST http://localhost:8582/api/collections/{collection_name}
+curl -X POST http://localhost:8081/api/collections/{collection_name}
 
 # Render BSON Schema
-curl -X GET http://localhost:8582/api/render/bson_schema/{version_name}
+curl -X GET http://localhost:8081/api/render/bson_schema/{version_name}
 
 # Render JSON Schema
-curl -X GET http://localhost:8582/api/render/json_schema/{version_name}
+curl -X GET http://localhost:8081/api/render/json_schema/{version_name}
 
 # Render OpenAPI Specification
-curl -X GET http://localhost:8582/api/render/openapi/{version_name}
+curl -X GET http://localhost:8081/api/render/openapi/{version_name}
 
 ```
 

Dockerfile

@@ -31,11 +31,10 @@ RUN pip install -e .
 RUN pip install gunicorn
 
 # Expose the port the app will run on
-EXPOSE 8582
+EXPOSE 8081
 
 # Set Environment Variables
 ENV PYTHONPATH=/opt/stage0_mongodb_api/stage0_mongodb_api
 
 # Command to run the application using Gunicorn 
-CMD exec gunicorn --bind 0.0.0.0:8582 --timeout 120 --preload stage0_mongodb_api.server:app
-#CMD exec gunicorn --bind 0.0.0.0:8582 --workers 1 --timeout 120 --preload stage0_mongodb_api.server:app
+CMD exec gunicorn --bind 0.0.0.0:8081 --timeout 120 --preload stage0_mongodb_api.server:app

Pipfile

@@ -16,8 +16,9 @@ db-drop-silent = "python tests/db_util.py drop --passphrase DROP_DROWSSAP_YEK"
 db-compare = "python tests/db_util.py compare"
 db-harvest = "python tests/db_util.py harvest"
 build = "docker build --tag ghcr.io/agile-learning-institute/stage0_mongodb_api:latest ."
-container = "docker compose up --detach"
-down = "docker compose down"
+database = "docker compose --profile mongodb up --detach"
+container = "docker compose --profile all up --detach"
+down = "docker compose down stage0_mongodb_api mongodb"
 
 [packages]
 python-dotenv = "*"

Pipfile.lock

@@ -76,7 +76,7 @@ The API is configured through environment variables.
 
 | Variable | Description | Default |
 |----------|-------------|---------|
-| `MONGODB_API_PORT` | API Port number | `8582` |
+| `MONGODB_API_PORT` | API Port number | `8081` |
 | `MONGO_DB_NAME` | MongoDB database name | `stage0` |
 | `MONGO_CONNECTION_STRING` | MongoDB connection string | `mongodb://root:example@localhost:27017/?tls=false&directConnection=true` |
 | `VERSION_COLLECTION_NAME`| MongoDB Version Collection name | `CollectionVersions` |

REFERENCE.md

@@ -1,32 +1,38 @@
 # docker-compose.yaml for stage0_mongodb_api testing
 
 services:
-  mongodb-server:
+  mongodb:
     image: mongo:7.0.5
-    environment:
-      MONGO_INITDB_ROOT_USERNAME: root
-      MONGO_INITDB_ROOT_PASSWORD: example
     ports:
-      - "27017:27017"
+      - 27017:27017
+    extra_hosts:
+      - "mongodb:127.0.0.1"
     healthcheck:
-      test: ["CMD", "mongosh", "-u", "root", "-p", "example", "--authenticationDatabase", "admin", "--eval", "db.adminCommand('ping')"]
-      interval: 2s
-      timeout: 2s
-      retries: 15
+      test: echo "try { rs.status() } catch (err) { rs.initiate({_id:'rs0',members:[{_id:0,host:'mongodb:27017'}]}) }" | mongosh --port 27017 --quiet
+      interval: 5s
+      timeout: 30s
+      start_period: 0s
+      retries: 30
+    command: ["--replSet", "rs0", "--bind_ip_all", "--port", "27017"]
+    profiles:
+      - mongodb
+      - all
 
   stage0_mongodb_api:
     image: ghcr.io/agile-learning-institute/stage0_mongodb_api:latest
     restart: no
     ports:
-      - "8582:8582"
+      - "8081:8081"
     volumes:
     - ./tests/test_cases/large_sample:/input
     environment:
-      MONGO_CONNECTION_STRING: mongodb://root:example@mongodb-server:27017
+      MONGO_CONNECTION_STRING: mongodb://mongodb:27017/?replicaSet=rs0
       MONGO_DB_NAME: test_database
-      MONGODB_API_PORT: 8582
+      MONGODB_API_PORT: 8081
       AUTO_PROCESS: True
       LOAD_TEST_DATA: True
     depends_on:
-      mongodb-server:
+      mongodb:
         condition: service_healthy
+    profiles:
+      - all

docker-compose.yaml

@@ -30,6 +30,10 @@ def handle_exit(signum, frame):
 signal.signal(signal.SIGTERM, handle_exit)
 signal.signal(signal.SIGINT, handle_exit)
 
+# Initialize Flask App
+app = Flask(__name__)
+app.json = MongoJSONEncoder(app)
+
 # Auto-processing logic - runs when module is imported (including by Gunicorn)
 if config.AUTO_PROCESS:
     logger.info(f"============= Auto Processing is Enabled ===============")
@@ -48,17 +52,13 @@ def handle_exit(signum, frame):
 
     # Process all collections
     processing_output = config_manager.process_all_collections()
-    logger.info(f"Processing Output: {json.dumps(processing_output, indent=4)}")
+    logger.info(f"Processing Output: {app.json.dumps(processing_output)}")
     logger.info(f"============= Auto Processing is Completed ===============")
 
 if config.EXIT_AFTER_PROCESSING:
     logger.info(f"============= Exiting After Processing ===============")
     sys.exit(0)
 
-# Initialize Flask App
-app = Flask(__name__)
-app.json = MongoJSONEncoder(app)
-
 # Apply Prometheus monitoring middleware
 metrics = PrometheusMetrics(app, path='/api/health')
 metrics.info('app_info', 'Application info', version=config.BUILT_AT)

stage0_mongodb_api/server.py

@@ -192,9 +192,20 @@ def _compare_collection_with_file(self, collection_name: str, json_file: Path) -
             # Convert ObjectIds to strings for comparison
             actual_data = json.loads(json_util.dumps(actual_data))
 
+            # Remove _id fields from both expected and actual data
+            expected_data_no_id = []
+            for doc in expected_data:
+                doc_copy = {k: v for k, v in doc.items() if k != '_id'}
+                expected_data_no_id.append(doc_copy)
+            
+            actual_data_no_id = []
+            for doc in actual_data:
+                doc_copy = {k: v for k, v in doc.items() if k != '_id'}
+                actual_data_no_id.append(doc_copy)
+            
             # Compare document counts
-            expected_count = len(expected_data)
-            actual_count = len(actual_data)
+            expected_count = len(expected_data_no_id)
+            actual_count = len(actual_data_no_id)
 
             if expected_count != actual_count:
                 return {
@@ -210,8 +221,8 @@ def _compare_collection_with_file(self, collection_name: str, json_file: Path) -
             matches = 0
             mismatches = []
 
-            for i, (expected, actual) in enumerate(zip(expected_data, actual_data)):
-                if self._documents_match(expected, actual):
+            for i, (expected, actual) in enumerate(zip(expected_data_no_id, actual_data_no_id)):
+                if expected == actual:
                     matches += 1
                 else:
                     mismatches.append({
@@ -249,22 +260,6 @@ def _compare_collection_with_file(self, collection_name: str, json_file: Path) -
                 "message": f"Error comparing collection: {str(e)}"
             }
 
-    def _documents_match(self, expected: Dict, actual: Dict) -> bool:
-        """Compare two documents for equality.
-        
-        Args:
-            expected: Expected document
-            actual: Actual document
-            
-        Returns:
-            True if documents match, False otherwise
-        """
-        # Remove _id field for comparison (it's auto-generated)
-        expected_copy = {k: v for k, v in expected.items() if k != '_id'}
-        actual_copy = {k: v for k, v in actual.items() if k != '_id'}
-        
-        return expected_copy == actual_copy
-    
     def harvest_database(self, output_path: Optional[str] = None) -> Dict[str, Any]:
         """Harvest all database contents to JSON files.
         

tests/db_util.py

@@ -1,7 +1,7 @@
 name: mongodb_api_large_sample
 version: "1.0"
 env:
-  host: localhost:8582
+  host: localhost:8081
 
 tests:
   collections:

tests/stepci/large_sample.yaml

@@ -1,7 +1,7 @@
 name: mongodb_api_observability
 version: "1.0"
 env:
-  host: localhost:8582
+  host: localhost:8081
 
 tests:
   observable: