Merge pull request #19 from agile-learning-institute/feature/add-enumerators-processing

Feature/add enumerators processing

Author: FlatBallFlyer

Pipfile.lock

@@ -70,6 +70,55 @@ The API processes collection configurations to manage MongoDB schemas, indexes,
 - Apply new Schema Validation
 - Load Test Data (optional)
 
+## Enumerators Processing
+
+The API processes enumerators from the `data/enumerators.json` file during the "process all" operation. This processing:
+
+- Loads enumerator definitions from `INPUT_FOLDER/data/enumerators.json`
+- Validates the structure of each enumerator version
+- Up-serts enumerator documents into the database using the `ENUMERATORS_COLLECTION_NAME` collection
+- Uses the `version` field as the key for upsert operations
+
+### Enumerators File Format
+
+The `enumerators.json` file contains an array of enumerator version objects:
+
+```json
+[
+    {
+        "version": 0,
+        "name": "Enumerations",
+        "status": "Deprecated",
+        "enumerators": {}
+    },
+    {
+        "version": 1,
+        "name": "Enumerations", 
+        "status": "Active",
+        "enumerators": {
+            "default_status": {
+                "active": "Not Deleted",
+                "archived": "Soft Delete Indicator"
+            },
+            "media_type": {
+                "movie": "A motion picture",
+                "tv_show": "A television series"
+            }
+        }
+    }
+]
+```
+
+### Enumerators Processing Workflow
+
+When processing all collections, the API:
+
+1. **Processes Enumerators First**: Loads and up-serts all enumerator versions from `enumerators.json`
+2. **Processes Collections**: Then processes all configured collections as usual
+3. **Reports Results**: Includes enumerators processing results in the overall operation results
+
+The enumerators processing is included in the "process all" operation results under the `"enumerators"` key, with the same operation result format as collection processing.
+
 ## Configuration Reference
 
 The API is configured through environment variables.
@@ -80,6 +129,7 @@ The API is configured through environment variables.
 | `MONGO_DB_NAME` | MongoDB database name | `stage0` |
 | `MONGO_CONNECTION_STRING` | MongoDB connection string | `mongodb://root:example@localhost:27017/?tls=false&directConnection=true` |
 | `VERSION_COLLECTION_NAME`| MongoDB Version Collection name | `CollectionVersions` |
+| `ENUMERATORS_COLLECTION_NAME` | MongoDB Enumerators Collection name | `Enumerators` |
 | `INPUT_FOLDER` | Directory containing configurations | `/input` |
 | `LOAD_TEST_DATA` | Load Test data during processing | `false` |
 | `AUTO_PROCESS` | Process configurations on startup | `false` |

REFERENCE.md

@@ -178,6 +178,15 @@ def process_all_collections(self) -> Dict[str, List[Dict]]:
         results = {}
         any_collection_failed = False
 
+        # Process enumerators first
+        enumerators_result = self._process_enumerators()
+        results["enumerators"] = [enumerators_result]
+        
+        # Check if enumerators processing failed
+        if enumerators_result.get("status") == "error":
+            any_collection_failed = True
+        
+        # Process all collections
         for collection_name in self.collection_configs.keys():
             try:
                 results[collection_name] = self.process_collection_versions(collection_name)
@@ -479,6 +488,60 @@ def _load_test_data(self, collection_name: str, test_data_file: str) -> Dict:
                 "status": "error"
             }
 
+    def _process_enumerators(self) -> Dict:
+        """Process enumerators from the enumerators.json file.
+        
+        Returns:
+            Dict containing operation result in consistent format
+        """
+        try:
+            # Use the already-loaded enumerators from schema_manager
+            enumerators = self.schema_manager.enumerators
+            
+            # Process each enumerator version
+            processed_count = 0
+            
+            for document in enumerators:
+                version = document.get("version")
+                    
+                # Upsert the document using version as the key
+                result = self.mongo_io.upsert_document(
+                    self.config.ENUMERATORS_COLLECTION_NAME,
+                    {"version": version},
+                    document
+                )
+                
+                # upsert_document returns the document itself, so if we get a result, it succeeded
+                if result and isinstance(result, dict):
+                    processed_count += 1
+                else:
+                    raise Exception(f"Failed to upsert version {version}")
+            
+            return {
+                "operation": "process_enumerators",
+                "collection": self.config.ENUMERATORS_COLLECTION_NAME,
+                "message": f"Successfully processed {processed_count} enumerator versions",
+                "details_type": "success",
+                "details": {
+                    "processed_count": processed_count,
+                    "total_count": len(enumerators)
+                },
+                "status": "success"
+            }
+                
+        except Exception as e:
+            return {
+                "operation": "process_enumerators",
+                "collection": self.config.ENUMERATORS_COLLECTION_NAME,
+                "message": f"Error processing enumerators: {str(e)}",
+                "details_type": "error",
+                "details": {
+                    "error": str(e),
+                    "error_type": type(e).__name__
+                },
+                "status": "error"
+            }
+
     def _assert_no_errors(self, operations: List[Dict]) -> None:
         """Check the last operation for errors and raise an exception if found.
         

stage0_mongodb_api/managers/config_manager.py

@@ -106,7 +106,7 @@ def process_collections(token: Dict = None) -> List[Dict]:
             token: Authentication token for RBAC enforcement
         
         Returns:
-            List of processing results for each collection
+            List of processing results for each collection and enumerators
             
         Raises:
             CollectionProcessingError: If there are load or validation errors
@@ -122,22 +122,24 @@ def process_collections(token: Dict = None) -> List[Dict]:
         if validation_errors:
             raise CollectionProcessingError("collections", validation_errors)
 
+        # Use process_all_collections to include enumerators processing
+        all_results = config_manager.process_all_collections()
+        
+        # Convert the dictionary format to list format for API consistency
         results = []
-        for collection_name, collection in config_manager.collection_configs.items():
-            try:
-                result = CollectionService.process_collection(collection_name, token)
-                results.append(result)
-            except CollectionNotFoundError:
-                # Skip collections that don't exist
-                continue
-            except Exception as e:
-                logger.error(f"Error processing collection {collection_name}: {str(e)}")
-                results.append({
-                    "status": "error",
-                    "collection": collection_name,
-                    "message": str(e),
-                    "error": str(e)
-                })
+        for collection_name, operations in all_results.items():
+            # Check if any operations have an error status
+            has_errors = any(
+                isinstance(op, dict) and op.get("status") == "error" 
+                for op in operations
+            )
+            
+            results.append({
+                "collection": collection_name,
+                "operations": operations,
+                "status": "error" if has_errors else "success"
+            })
+        
         return results
 
     @staticmethod

stage0_mongodb_api/services/collection_service.py

@@ -213,5 +213,100 @@ def test_process_all_collections_structure(self):
                         # Status should be the last property
                         self.assertEqual(list(operation.keys())[-1], "status")
 
+    def test_process_enumerators_success(self):
+        """Test that _process_enumerators successfully processes enumerators."""
+        test_case_dir = os.path.join(self.test_cases_dir, "small_sample")
+        self.config.INPUT_FOLDER = test_case_dir
+        self.config.ENUMERATORS_COLLECTION_NAME = "Enumerators"
+        
+        # Mock schema_manager to return test enumerators
+        mock_enumerators = [
+            {"version": 0, "enumerators": {}},
+            {"version": 1, "enumerators": {"default_status": ["active", "archived"]}}
+        ]
+        
+        # Reset and mock MongoIO upsert_document to return the input document
+        self.mock_mongoio_get_instance.return_value.upsert_document.reset_mock()
+        def upsert_side_effect(collection, filter, document):
+            return document
+        self.mock_mongoio_get_instance.return_value.upsert_document.side_effect = upsert_side_effect
+        
+        with patch('stage0_mongodb_api.managers.config_manager.SchemaManager') as mock_schema_manager_class:
+            # Create a mock schema manager instance
+            mock_schema_manager = MagicMock()
+            mock_schema_manager.enumerators = mock_enumerators
+            mock_schema_manager_class.return_value = mock_schema_manager
+            
+            config_manager = ConfigManager()
+            result = config_manager._process_enumerators()
+            
+            # Test that we get the expected success structure
+            self.assertEqual(result["operation"], "process_enumerators")
+            self.assertEqual(result["collection"], "Enumerators")
+            self.assertEqual(result["status"], "success")
+            self.assertEqual(result["details_type"], "success")
+            self.assertEqual(result["details"]["processed_count"], 2)
+            self.assertEqual(result["details"]["total_count"], 2)
+            
+            # Verify upsert_document was called for each enumerator
+            self.assertEqual(
+                self.mock_mongoio_get_instance.return_value.upsert_document.call_count, 2
+            )
+
+    def test_process_all_collections_includes_enumerators(self):
+        """Test that process_all_collections includes enumerators processing."""
+        test_case_dir = os.path.join(self.test_cases_dir, "small_sample")
+        self.config.INPUT_FOLDER = test_case_dir
+        self.config.ENUMERATORS_COLLECTION_NAME = "Enumerators"
+        
+        # Mock schema_manager to return test enumerators
+        mock_enumerators = [
+            {"version": 0, "enumerators": {}},
+            {"version": 1, "enumerators": {"default_status": ["active", "archived"]}}
+        ]
+        
+        # Reset and mock MongoIO upsert_document to return the input document
+        self.mock_mongoio_get_instance.return_value.upsert_document.reset_mock()
+        def upsert_side_effect(collection, filter, document):
+            return document
+        self.mock_mongoio_get_instance.return_value.upsert_document.side_effect = upsert_side_effect
+        
+        # Mock VersionManager.get_current_version to return a version that will be processed
+        with patch('stage0_mongodb_api.managers.config_manager.VersionManager.get_current_version') as mock_get_version:
+            mock_get_version.return_value = "simple.0.0.0.0"
+            
+            # Mock all the manager operations to return success
+            with patch('stage0_mongodb_api.managers.config_manager.SchemaManager') as mock_schema_manager, \
+                 patch('stage0_mongodb_api.managers.config_manager.IndexManager') as mock_index_manager, \
+                 patch('stage0_mongodb_api.managers.config_manager.MigrationManager') as mock_migration_manager, \
+                 patch('stage0_mongodb_api.managers.config_manager.VersionManager') as mock_version_manager:
+                
+                # Set up mock schema manager with enumerators
+                mock_schema_manager.return_value.enumerators = mock_enumerators
+                mock_schema_manager.return_value.remove_schema.return_value = {
+                    "operation": "remove_schema", "collection": "simple", "status": "success"
+                }
+                mock_schema_manager.return_value.apply_schema.return_value = {
+                    "operation": "apply_schema", "collection": "simple", "schema": {}, "status": "success"
+                }
+                mock_version_manager.return_value.update_version.return_value = {
+                    "operation": "version_update", "collection": "simple", "version": "simple.1.0.0.1", "status": "success"
+                }
+                
+                config_manager = ConfigManager()
+                result = config_manager.process_all_collections()
+                
+                # Test that we get a dict with enumerators and collections
+                self.assertIsInstance(result, dict)
+                self.assertIn("enumerators", result)
+                self.assertIn("simple", result)
+                
+                # Test that enumerators processing is included
+                enumerators_result = result["enumerators"]
+                self.assertIsInstance(enumerators_result, list)
+                self.assertEqual(len(enumerators_result), 2)  # enumerators result + overall_status
+                self.assertEqual(enumerators_result[0]["operation"], "process_enumerators")
+                self.assertEqual(enumerators_result[0]["status"], "success")
+
 if __name__ == '__main__':
     unittest.main()