From c4be4eceac82d51a8da8335d7de790f7667576b6 Mon Sep 17 00:00:00 2001 From: Mike Storey Date: Tue, 24 Jun 2025 21:38:40 -0400 Subject: [PATCH] Improved logging, Better container testing, Updates related to collection version name --- CONTRIBUTING.md | 25 +- Dockerfile | 5 +- Pipfile | 5 +- Pipfile.lock | 216 +++++++++--------- REFERENCE.md | 2 +- docker-compose.yaml | 32 +-- stage0_mongodb_api/server.py | 10 +- tests/db_util.py | 35 ++- tests/stepci/large_sample.yaml | 2 +- tests/stepci/observability.yaml | 2 +- tests/stepci/small_sample.yaml | 2 +- ...tVersions.json => CollectionVersions.json} | 0 ...tVersions.json => CollectionVersions.json} | 0 tests/test_server.py | 2 +- 14 files changed, 166 insertions(+), 172 deletions(-) rename tests/test_cases/large_sample/test_database/{msmCurrentVersions.json => CollectionVersions.json} (100%) rename tests/test_cases/small_sample/test_database/{msmCurrentVersions.json => CollectionVersions.json} (100%) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c1f89c1..20b4314 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -61,6 +61,9 @@ pipenv install --dev # Run Unit Tests and generate coverage report pipenv run test +# Run a backing mongo database +pipenv run database + # Change the testing database name export MONGO_DB_NAME=test_database # NOTE: Must be test_database for db-compare to work @@ -105,8 +108,10 @@ pipenv run stepci-large # Combine DB actions with StepCI testing export INPUT_FOLDER=./tests/test_cases/large_sample +pipenv run local +# Then in a different window pipenv run db-drop-silent -pipenv run stepci_large +pipenv run stepci-large pipenv run db-compare # Build the API Docker Image @@ -160,31 +165,31 @@ If you need a new set of test data to validate features you are adding, feel fre ```bash # Get Configuration -curl -X GET http://localhost:8582/api/config +curl -X GET http://localhost:8081/api/config # Health Check -curl -X GET http://localhost:8582/health +curl -X GET http://localhost:8081/health # List Collections -curl -X GET http://localhost:8582/api/collections/ +curl -X GET http://localhost:8081/api/collections/ # Get a Collection Config -curl -X GET http://localhost:8582/api/collections/{collection_name} +curl -X GET http://localhost:8081/api/collections/{collection_name} # Process All Collections -curl -X POST http://localhost:8582/api/collections/ +curl -X POST http://localhost:8081/api/collections/ # Process Specific Collection -curl -X POST http://localhost:8582/api/collections/{collection_name} +curl -X POST http://localhost:8081/api/collections/{collection_name} # Render BSON Schema -curl -X GET http://localhost:8582/api/render/bson_schema/{version_name} +curl -X GET http://localhost:8081/api/render/bson_schema/{version_name} # Render JSON Schema -curl -X GET http://localhost:8582/api/render/json_schema/{version_name} +curl -X GET http://localhost:8081/api/render/json_schema/{version_name} # Render OpenAPI Specification -curl -X GET http://localhost:8582/api/render/openapi/{version_name} +curl -X GET http://localhost:8081/api/render/openapi/{version_name} ``` diff --git a/Dockerfile b/Dockerfile index 979b4e6..f2d0d1b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -31,11 +31,10 @@ RUN pip install -e . RUN pip install gunicorn # Expose the port the app will run on -EXPOSE 8582 +EXPOSE 8081 # Set Environment Variables ENV PYTHONPATH=/opt/stage0_mongodb_api/stage0_mongodb_api # Command to run the application using Gunicorn -CMD exec gunicorn --bind 0.0.0.0:8582 --timeout 120 --preload stage0_mongodb_api.server:app -#CMD exec gunicorn --bind 0.0.0.0:8582 --workers 1 --timeout 120 --preload stage0_mongodb_api.server:app \ No newline at end of file +CMD exec gunicorn --bind 0.0.0.0:8081 --timeout 120 --preload stage0_mongodb_api.server:app diff --git a/Pipfile b/Pipfile index e680ff8..fb6ba92 100644 --- a/Pipfile +++ b/Pipfile @@ -16,8 +16,9 @@ db-drop-silent = "python tests/db_util.py drop --passphrase DROP_DROWSSAP_YEK" db-compare = "python tests/db_util.py compare" db-harvest = "python tests/db_util.py harvest" build = "docker build --tag ghcr.io/agile-learning-institute/stage0_mongodb_api:latest ." -container = "docker compose up --detach" -down = "docker compose down" +database = "docker compose --profile mongodb up --detach" +container = "docker compose --profile all up --detach" +down = "docker compose down stage0_mongodb_api mongodb" [packages] python-dotenv = "*" diff --git a/Pipfile.lock b/Pipfile.lock index da6094b..2950ce5 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -431,119 +431,107 @@ }, "multidict": { "hashes": [ - "sha256:03c0923da300120830fc467e23805d63bbb4e98b94032bd863bc7797ea5fa653", - "sha256:046a7540cfbb4d5dc846a1fd9843f3ba980c6523f2e0c5b8622b4a5c94138ae6", - "sha256:08db204213d0375a91a381cae0677ab95dd8c67a465eb370549daf6dbbf8ba10", - "sha256:0ad73a60e11aa92f1f2c9330efdeaac4531b719fc568eb8d312fd4112f34cc18", - "sha256:0b5ac6ebaf5d9814b15f399337ebc6d3a7f4ce9331edd404e76c49a01620b68d", - "sha256:0e5b19f8cd67235fab3e195ca389490415d9fef5a315b1fa6f332925dc924262", - "sha256:0ec1c3fbbb0b655a6540bce408f48b9a7474fd94ed657dcd2e890671fefa7743", - "sha256:0f32a1777465a35c35ddbbd7fc1293077938a69402fcc59e40b2846d04a120dd", - "sha256:177b081e4dec67c3320b16b3aa0babc178bbf758553085669382c7ec711e1ec8", - "sha256:17f78a52c214481d30550ec18208e287dfc4736f0c0148208334b105fd9e0887", - "sha256:1bb986c8ea9d49947bc325c51eced1ada6d8d9b4c5b15fd3fcdc3c93edef5a74", - "sha256:20d30c9410ac3908abbaa52ee5967a754c62142043cf2ba091e39681bd51d21a", - "sha256:220c74009507e847a3a6fc5375875f2a2e05bd9ce28cf607be0e8c94600f4472", - "sha256:2261b538145723ca776e55208640fffd7ee78184d223f37c2b40b9edfe0e818a", - "sha256:2540395b63723da748f850568357a39cd8d8d4403ca9439f9fcdad6dd423c780", - "sha256:2966d0099cb2e2039f9b0e73e7fd5eb9c85805681aa2a7f867f9d95b35356921", - "sha256:2d24a00d34808b22c1f15902899b9d82d0faeca9f56281641c791d8605eacd35", - "sha256:2e118a202904623b1d2606d1c8614e14c9444b59d64454b0c355044058066469", - "sha256:300da0fa4f8457d9c4bd579695496116563409e676ac79b5e4dca18e49d1c308", - "sha256:3233f21abdcd180b2624eb6988a1e1287210e99bca986d8320afca5005d85844", - "sha256:3e86eb90015c6f21658dbd257bb8e6aa18bdb365b92dd1fba27ec04e58cdc31b", - "sha256:3f805b8b951d1fadc5bc18c3c93e509608ac5a883045ee33bc22e28806847c20", - "sha256:3fe9fada8bc0839466b09fa3f6894f003137942984843ec0c3848846329a36ae", - "sha256:40ff26f58323795f5cd2855e2718a1720a1123fb90df4553426f0efd76135462", - "sha256:42bdee30424c1f4dcda96e07ac60e2a4ede8a89f8ae2f48b5e4ccc060f294c52", - "sha256:44cb5c53fb2d4cbcee70a768d796052b75d89b827643788a75ea68189f0980a1", - "sha256:46bb05d50219655c42a4b8fcda9c7ee658a09adbb719c48e65a20284e36328ea", - "sha256:4c78d5ec00fdd35c91680ab5cf58368faad4bd1a8721f87127326270248de9bc", - "sha256:4d30a2cc106a7d116b52ee046207614db42380b62e6b1dd2a50eba47c5ca5eb1", - "sha256:4e990cbcb6382f9eae4ec720bcac6a1351509e6fc4a5bb70e4984b27973934e6", - "sha256:51d33fafa82640c0217391d4ce895d32b7e84a832b8aee0dcc1b04d8981ec7f4", - "sha256:529b03600466480ecc502000d62e54f185a884ed4570dee90d9a273ee80e37b5", - "sha256:53d92df1752df67a928fa7f884aa51edae6f1cf00eeb38cbcf318cf841c17456", - "sha256:54f524d73f4d54e87e03c98f6af601af4777e4668a52b1bd2ae0a4d6fc7b392b", - "sha256:5634b35f225977605385f56153bd95a7133faffc0ffe12ad26e10517537e8dfc", - "sha256:58b2ded1a7982cf7b8322b0645713a0086b2b3cf5bb9f7c01edfc1a9f98d20dc", - "sha256:5b02e1ca495d71e07e652e4cef91adae3bf7ae4493507a263f56e617de65dafc", - "sha256:5cc7968b7d1bf8b973c307d38aa3a2f2c783f149bcac855944804252f1df5105", - "sha256:60c3f8f13d443426c55f88cf3172547bbc600a86d57fd565458b9259239a6737", - "sha256:63b3b24fadc7067282c88fae5b2f366d5b3a7c15c021c2838de8c65a50eeefb4", - "sha256:64306121171d988af77d74be0d8c73ee1a69cf6f96aea7fa6030c88f32a152dd", - "sha256:67c4a640952371c9ca65b6a710598be246ef3be5ca83ed38c16a7660d3980877", - "sha256:680210de2c38eef17ce46b8df8bf2c1ece489261a14a6e43c997d49843a27c99", - "sha256:6994bad9d471ef2156f2b6850b51e20ee409c6b9deebc0e57be096be9faffdce", - "sha256:69ad681ad7c93a41ee7005cc83a144b5b34a3838bcf7261e2b5356057b0f78de", - "sha256:6bb5f65ff91daf19ce97f48f63585e51595539a8a523258b34f7cef2ec7e0617", - "sha256:6c65068cc026f217e815fa519d8e959a7188e94ec163ffa029c94ca3ef9d4a73", - "sha256:6cb9bcedd9391b313e5ec2fb3aa07c03e050550e7b9e4646c076d5c24ba01532", - "sha256:6dcee5e7e92060b4bb9bb6f01efcbb78c13d0e17d9bc6eec71660dd71dc7b0c2", - "sha256:70b599f70ae6536e5976364d3c3cf36f40334708bd6cebdd1e2438395d5e7676", - "sha256:7673ee4f63879ecd526488deb1989041abcb101b2d30a9165e1e90c489f3f7fb", - "sha256:76803a29fd71869a8b59c2118c9dcfb3b8f9c8723e2cce6baeb20705459505cf", - "sha256:7f78caf409914f108f4212b53a9033abfdc2cbab0647e9ac3a25bb0f21ab43d2", - "sha256:7fe92a62326eef351668eec4e2dfc494927764a0840a1895cff16707fceffcd3", - "sha256:80d696fa38d738fcebfd53eec4d2e3aeb86a67679fd5e53c325756682f152826", - "sha256:828bab777aa8d29d59700018178061854e3a47727e0611cb9bec579d3882de3b", - "sha256:82d0cf0ea49bae43d9e8c3851e21954eff716259ff42da401b668744d1760bcb", - "sha256:84a51e3baa77ded07be4766a9e41d977987b97e49884d4c94f6d30ab6acaee14", - "sha256:84ca75ad8a39ed75f079a8931435a5b51ee4c45d9b32e1740f99969a5d1cc2ee", - "sha256:86fb42ed5ed1971c642cc52acc82491af97567534a8e381a8d50c02169c4e684", - "sha256:8b2d61afbafc679b7eaf08e9de4fa5d38bd5dc7a9c0a577c9f9588fb49f02dbb", - "sha256:8b4bf6bb15a05796a07a248084e3e46e032860c899c7a9b981030e61368dba95", - "sha256:8de67f79314d24179e9b1869ed15e88d6ba5452a73fc9891ac142e0ee018b5d6", - "sha256:9232a117341e7e979d210e41c04e18f1dc3a1d251268df6c818f5334301274e1", - "sha256:942bd8002492ba819426a8d7aefde3189c1b87099cdf18aaaefefcf7f3f7b6d2", - "sha256:95750a9a9741cd1855d1b6cb4c6031ae01c01ad38d280217b64bfae986d39d56", - "sha256:96d109e663d032280ef8ef62b50924b2e887d5ddf19e301844a6cb7e91a172a6", - "sha256:9a19bd108c35877b57393243d392d024cfbfdefe759fd137abb98f6fc910b64c", - "sha256:9cc1e10c14ce8112d1e6d8971fe3cdbe13e314f68bea0e727429249d4a6ce164", - "sha256:a05b5604c5a75df14a63eeeca598d11b2c3745b9008539b70826ea044063a572", - "sha256:a10227168a24420c158747fc201d4279aa9af1671f287371597e2b4f2ff21879", - "sha256:a133e7ddc9bc7fb053733d0ff697ce78c7bf39b5aec4ac12857b6116324c8d75", - "sha256:a42995bdcaff4e22cb1280ae7752c3ed3fbb398090c6991a2797a4a0e5ed16a9", - "sha256:a72933bc308d7a64de37f0d51795dbeaceebdfb75454f89035cdfc6a74cfd129", - "sha256:a7d130ed7a112e25ab47309962ecafae07d073316f9d158bc7b3936b52b80121", - "sha256:a9695fc1462f17b131c111cf0856a22ff154b0480f86f539d24b2778571ff94d", - "sha256:aadc3cb78be90a887f8f6b73945b840da44b4a483d1c9750459ae69687940c97", - "sha256:b15f817276c96cde9060569023808eec966bd8da56a97e6aa8116f34ddab6534", - "sha256:b4ac1dd5eb0ecf6f7351d5a9137f30a83f7182209c5d37f61614dfdce5714853", - "sha256:b4bf507c991db535a935b2127cf057a58dbc688c9f309c72080795c63e796f58", - "sha256:b4e47ef51237841d1087e1e1548071a6ef22e27ed0400c272174fa585277c4b4", - "sha256:b555329c9894332401f03b9a87016f0b707b6fccd4706793ec43b4a639e75869", - "sha256:b8a09aec921b34bd8b9f842f0bcfd76c6a8c033dc5773511e15f2d517e7e1068", - "sha256:bab4a8337235365f4111a7011a1f028826ca683834ebd12de4b85e2844359c36", - "sha256:be4c08f3a2a6cc42b414496017928d95898964fed84b1b2dace0c9ee763061f9", - "sha256:bee5c0b79fca78fd2ab644ca4dc831ecf793eb6830b9f542ee5ed2c91bc35a0e", - "sha256:c0078358470da8dc90c37456f4a9cde9f86200949a048d53682b9cd21e5bbf2b", - "sha256:c96aedff25f4e47b6697ba048b2c278f7caa6df82c7c3f02e077bcc8d47b4b76", - "sha256:cbbc88abea2388fde41dd574159dec2cda005cb61aa84950828610cb5010f21a", - "sha256:d1c185fc1069781e3fc8b622c4331fb3b433979850392daa5efbb97f7f9959bb", - "sha256:d245973d4ecc04eea0a8e5ebec7882cf515480036e1b48e65dffcfbdf86d00be", - "sha256:d8646b4259450c59b9286db280dd57745897897284f6308edbdf437166d93855", - "sha256:d98f4ac9c1ede7e9d04076e2e6d967e15df0079a6381b297270f6bcab661195e", - "sha256:d99a59d64bb1f7f2117bec837d9e534c5aeb5dcedf4c2b16b9753ed28fdc20a3", - "sha256:df7ecbc65a53a2ce1b3a0c82e6ad1a43dcfe7c6137733f9176a92516b9f5b851", - "sha256:e053a4d690f4352ce46583080fefade9a903ce0fa9d820db1be80bdb9304fa2f", - "sha256:e279259bcb936732bfa1a8eec82b5d2352b3df69d2fa90d25808cfc403cee90a", - "sha256:e2977ef8b7ce27723ee8c610d1bd1765da4f3fbe5a64f9bf1fd3b4770e31fbc0", - "sha256:e355ac668a8c3e49c2ca8daa4c92f0ad5b705d26da3d5af6f7d971e46c096da7", - "sha256:e3b1425fe54ccfde66b8cfb25d02be34d5dfd2261a71561ffd887ef4088b4b69", - "sha256:e80de5ad995de210fd02a65c2350649b8321d09bd2e44717eaefb0f5814503e8", - "sha256:e8ef15cc97c9890212e1caf90f0d63f6560e1e101cf83aeaf63a57556689fb34", - "sha256:e95c5e07a06594bdc288117ca90e89156aee8cb2d7c330b920d9c3dd19c05414", - "sha256:f34a90fbd9959d0f857323bd3c52b3e6011ed48f78d7d7b9e04980b8a41da3af", - "sha256:f94c6ea6405fcf81baef1e459b209a78cda5442e61b5b7a57ede39d99b5204a0", - "sha256:fa097ae2a29f573de7e2d86620cbdda5676d27772d4ed2669cfa9961a0d73955", - "sha256:fcb2aa79ac6aef8d5b709bbfc2fdb1d75210ba43038d70fbb595b35af470ce06", - "sha256:fdeae096ca36c12d8aca2640b8407a9d94e961372c68435bef14e31cce726138", - "sha256:ff07b504c23b67f2044533244c230808a1258b3493aaf3ea2a0785f70b7be461", - "sha256:ffa58e3e215af8f6536dc837a990e456129857bb6fd546b3991be470abd9597a" + "sha256:0120ed5cff2082c7a0ed62a8f80f4f6ac266010c722381816462f279bfa19487", + "sha256:0499cbc67c1b02ba333781798560c5b1e7cd03e9273b678c92c6de1b1657fac9", + "sha256:059fb556c3e6ce1a168496f92ef139ad839a47f898eaa512b1d43e5e05d78c6b", + "sha256:05dcaed3e5e54f0d0f99a39762b0195274b75016cbf246f600900305581cf1a2", + "sha256:071b962f4cc87469cda90c7cc1c077b76496878b39851d7417a3d994e27fe2c6", + "sha256:090e0b37fde199b58ea050c472c21dc8a3fbf285f42b862fe1ff02aab8942239", + "sha256:0ce8f0ea49e8f54203f7d80e083a7aa017dbcb6f2d76d674273e25144c8aa3d7", + "sha256:11f5ecf3e741a18c578d118ad257c5588ca33cc7c46d51c0487d7ae76f072c32", + "sha256:153d7ff738d9b67b94418b112dc5a662d89d2fc26846a9e942f039089048c804", + "sha256:189f0c2bd1c0ae5509e453707d0e187e030c9e873a0116d1f32d1c870d0fc347", + "sha256:195882f2f6272dacc88194ecd4de3608ad0ee29b161e541403b781a5f5dd346f", + "sha256:199a0a9b3de8bbeb6881460d32b857dc7abec94448aeb6d607c336628c53580a", + "sha256:1d784c0a1974f00d87f632d0fb6b1078baf7e15d2d2d1408af92f54d120f136e", + "sha256:228533a5f99f1248cd79f6470779c424d63bc3e10d47c82511c65cc294458445", + "sha256:22b47f7e76ebea0b802df9ed08165b1e6ab52b140c7180c3e740e6205b3781b3", + "sha256:253e5c41fcc02e2956ab276b5a702f3972db1e87c080be55e87ca31a2f4f8012", + "sha256:279a37cb9d04097bf1c6308d7495cb4dfbd8fb538301bfe464266b045dfeb1cd", + "sha256:29eff1c9a905e298e9cd29f856f77485e58e59355f0ee323ac748203e002bbd3", + "sha256:2bf0068fe9abb0ebed1436a4e415117386951cf598eb8146ded4baf8e1ff6d1e", + "sha256:2d7def2fc47695c46a427b8f298fb5ace03d635c1fb17f30d6192c9a8fb69e70", + "sha256:2ec3aa63f0c668f591d43195f8e555f803826dee34208c29ade9d63355f9e095", + "sha256:2fb5dde79a7f6d98ac5e26a4c9de77ccd2c5224a7ce89aeac6d99df7bbe06464", + "sha256:36b138c6ec3aedaa975653ea90099efb22042bab31727dd4cd2921a64de46b25", + "sha256:38755bcba18720cb2338bea23a5afcff234445ee75fa11518f6130e22f2ab970", + "sha256:3c78fc6bc1dd7a139dab7ee9046f79a2082dce9360e3899b762615d564e2e857", + "sha256:3d749b10cc6acb2c0814df881910ffd8d8ab1ec54493585579b4a75f89fe86d6", + "sha256:3dea06ba27401c4b54317aa04791182dc9295e7aa623732dd459071a0e0f65db", + "sha256:48f95fe064f63d9601ef7a3dce2fc2a437d5fcc11bca960bc8be720330b13b6a", + "sha256:4b299a2ffed33ad0733a9d47805b538d59465f8439bfea44df542cfb285c4db2", + "sha256:5125a9faed98738d7d6e23650bd8af70abb95628d011f57f70a4d8f349e6d073", + "sha256:527076fdf5854901b1246c589af9a8a18b4a308375acb0020b585f696a10c794", + "sha256:562d4714fa43f6ebc043a657535e4575e7d6141a818c9b3055f0868d29a1a41b", + "sha256:576a1887a5c5becbe4fb484d0bdf6ed8ec89e9c11770f8f3214fd127ba137b8b", + "sha256:5776f9d2c3a1053f022f744af5f467c2f65b40d4cc00082bcf70e8c462c7dbad", + "sha256:5891e3327e6a426ddd443c87339b967c84feb8c022dd425e0c025fa0fcd71e68", + "sha256:5e8fefd7c062b0657af2480d789dcb347450d17c7bd20b02303c25f1f59a33a7", + "sha256:6037beca8cb481307fb586ee0b73fae976a3e00d8f6ad7eb8af94a878a4893f0", + "sha256:627ba4b7ce7c0115981f0fd91921f5d101dfb9972622178aeef84ccce1c2bbf3", + "sha256:62eb76be8c20d9017a82b74965db93ddcf472b929b6b2b78c56972c73bacf2e4", + "sha256:6eb3bf26cd94eb306e4bc776d0964cc67a7967e4ad9299309f0ff5beec3c62be", + "sha256:70c0e51d55f9bc5e97de950c3b3e88f501b3ca2b3894f231f3957dd3985b4d54", + "sha256:70c742357dd6207be30922207f8d59c91e2776ddbefa23830c55c09020e59f8a", + "sha256:719af50a44ce9cf9ab15d829bf8cf146de486b4816284c17c3c9b9c9735abb8f", + "sha256:7710c716243525cc05cd038c6e09f1807ee0fef2510a6e484450712c389c8d7f", + "sha256:77bc8ab5c6bfe696eff564824e73a451fdeca22f3b960261750836cee02bcbfa", + "sha256:79101d58094419b6e8d07e24946eba440136b9095590271cd6ccc4a90674a57d", + "sha256:79d13e06d5241f9c8479dfeaf0f7cce8f453a4a302c9a0b1fa9b1a6869ff7757", + "sha256:7a365a579fb3e067943d0278474e14c2244c252f460b401ccbf49f962e7b70fa", + "sha256:7b7b6e1ce9b61f721417c68eeeb37599b769f3b631e6b25c21f50f8f619420b9", + "sha256:7b7d75cb5b90fa55700edbbdca12cd31f6b19c919e98712933c7a1c3c6c71b73", + "sha256:7cbf407313236a79ce9b8af11808c29756cfb9c9a49a7f24bb1324537eec174b", + "sha256:83eb172b4856ffff2814bdcf9c7792c0439302faab1b31376817b067b26cd8f5", + "sha256:84292145303f354a35558e601c665cdf87059d87b12777417e2e57ba3eb98903", + "sha256:895354f4a38f53a1df2cc3fa2223fa714cff2b079a9f018a76cad35e7f0f044c", + "sha256:8a0d22e8b07cf620e9aeb1582340d00f0031e6a1f3e39d9c2dcbefa8691443b4", + "sha256:8a266373c604e49552d295d9f8ec4fd59bd364f2dd73eb18e7d36d5533b88f45", + "sha256:8a6605dc74cd333be279e1fcb568ea24f7bdf1cf09f83a77360ce4dd32d67f14", + "sha256:8b83b055889bda09fc866c0a652cdb6c36eeeafc2858259c9a7171fe82df5773", + "sha256:8d64e30ae9ba66ce303a567548a06d64455d97c5dff7052fe428d154274d7174", + "sha256:8e63ac6adc668cfe52e29c00afe33c3b8dbec8e37b529aa83bf31ba4bad0c509", + "sha256:93b21be44f3cfee3be68ed5cd8848a3c0420d76dbd12d74f7776bde6b29e5f33", + "sha256:98011312f36d1e496f15454a95578d1212bc2ffc25650a8484752b06d304fd9b", + "sha256:9a17a17bad5c22f43e6a6b285dd9c16b1e8f8428202cd9bc22adaac68d0bbfed", + "sha256:9eec51891d3c210948ead894ec1483d48748abec08db5ce9af52cc13fef37aee", + "sha256:a0f1890f9a05d038720a7c3b5d82467534495bcb6bbda929f6f0914977cc56d1", + "sha256:a6523258f2eb24c91995ae64172c19cd73bacd5a7f2b0733676966c527ab08f8", + "sha256:a835ea8103f4723915d7d621529c80ef48db48ae0c818afcabe0f95aa1febc3a", + "sha256:a9e15dfe441aec31e0fa78f497aca83f0ad992ca58782cbaba8220e5a87608fc", + "sha256:aa6dcf25ced31cdce10f004506dbc26129f28a911b32ed10e54453a0842a6173", + "sha256:ab94923ae54385ed480e4ab19f10269ee60f3eabd0b35e2a5d1ba6dbf3b0cc27", + "sha256:ad32e43e028276612bf5bab762677e7d131d2df00106b53de2efb2b8a28d5bce", + "sha256:b3bd88c1bc1f749db6a1e1f01696c3498bc25596136eceebb45766d24a320b27", + "sha256:b632c1e4a2ff0bb4c1367d6c23871aa95dbd616bf4a847034732a142bb6eea94", + "sha256:b7bd4d655dc460c7aebb73b58ed1c074e85f7286105b012556cf0f25c6d1dba3", + "sha256:b948eb625411c20b15088fca862c51a39140b9cf7875b5fb47a72bb249fa2f42", + "sha256:bae589fb902b47bd94e6f539b34eefe55a1736099f616f614ec1544a43f95b05", + "sha256:c07d5f38b39acb4f8f61a7aa4166d140ed628245ff0441630df15340532e3b3c", + "sha256:c5c18f8646a520cc34d00f65f9f6f77782b8a8c59fd8de10713e0de7f470b5d0", + "sha256:cc80c7e8f297484c4511e887c244adec9a7ed3a76826cb8dbc6183b717a37d1f", + "sha256:d3468f0db187aca59eb56e0aa9f7c8c5427bcb844ad1c86557b4886aeb4484d8", + "sha256:dc62c8ac1b73ec704ed1a05be0267358fd5c99d1952f30448db1637336635cf8", + "sha256:de2b253a3a90e1fa55eef5f9b3146bb5c722bd3400747112c9963404a2f5b9cf", + "sha256:dedf667cded1cdac5bfd3f3c2ff30010f484faccae4e871cc8a9316d2dc27363", + "sha256:e09d7852100bcc3e466e63478ee18c68cc4d2ca2a978f29b90d5e2ea814f7b3e", + "sha256:e5e1a5a99c72d1531501406fcc06b6bf699ebd079dacd6807bb43fc0ff260e5c", + "sha256:e615032b684a1d6faffe41d64cd896801bd3f2c1b642355e9b5d11fd8d40223e", + "sha256:e81f23b4b6f2a588f15d5cb554b2d8b482bb6044223d64b86bc7079cae9ebaad", + "sha256:eb27128141474a1d545f0531b496c7c2f1c4beff50cb5a828f36eb62fef16c67", + "sha256:ed98ac527278372251fbc8f5c6c41bdf64ded1db0e6e86f9b9622744306060f6", + "sha256:ee2d333380f22d35a56c6461f4579cfe186e143cd0b010b9524ac027de2a34cd", + "sha256:efd1951edab4a6cb65108d411867811f2b283f4b972337fb4269e40142f7f6a6", + "sha256:f369d6619b24da4df4a02455fea8641fe8324fc0100a3e0dcebc5bf55fa903f3", + "sha256:f42fef9bcba3c32fd4e4a23c5757fc807d218b249573aaffa8634879f95feb73", + "sha256:f8316e58db799a1972afbc46770dfaaf20b0847003ab80de6fcb9861194faa3f", + "sha256:f97680c839dd9fa208e9584b1c2a5f1224bd01d31961f7f7d94984408c4a6b9e", + "sha256:fc993a96dfc8300befd03d03df46efdb1d8d5a46911b014e956a4443035f470d", + "sha256:fcdaa72261bff25fad93e7cb9bd7112bd4bac209148e698e380426489d8ed8a9", + "sha256:fe09318a28b00c6f43180d0d889df1535e98fb2d93d25955d46945f8d5410d87" ], "markers": "python_version >= '3.9'", - "version": "==6.5.0" + "version": "==6.5.1" }, "ollama": { "hashes": [ @@ -933,12 +921,12 @@ }, "stage0-py-utils": { "hashes": [ - "sha256:085a6fa0a6b1eb14990fec22f66ccd9bbb705543b247a68c8f7d4b15aa3338bd", - "sha256:42054976b17923bfb5bd7ab44bbbc04a9bd2606b077f915586915f3cfab5d799" + "sha256:3a6f22538217c4c216cd002558666e405551f8d51db647b439a58ef9546fb712", + "sha256:601cb3228da2187d4a8f6c0f2b66762bd7784eb6975dbe3a897a0a02b78ea646" ], "index": "pypi", "markers": "python_version >= '3.8'", - "version": "==0.2.3" + "version": "==0.2.4" }, "typing-extensions": { "hashes": [ diff --git a/REFERENCE.md b/REFERENCE.md index 1a1dc5b..1a63a84 100644 --- a/REFERENCE.md +++ b/REFERENCE.md @@ -76,7 +76,7 @@ The API is configured through environment variables. | Variable | Description | Default | |----------|-------------|---------| -| `MONGODB_API_PORT` | API Port number | `8582` | +| `MONGODB_API_PORT` | API Port number | `8081` | | `MONGO_DB_NAME` | MongoDB database name | `stage0` | | `MONGO_CONNECTION_STRING` | MongoDB connection string | `mongodb://root:example@localhost:27017/?tls=false&directConnection=true` | | `VERSION_COLLECTION_NAME`| MongoDB Version Collection name | `CollectionVersions` | diff --git a/docker-compose.yaml b/docker-compose.yaml index 2a3d959..452c3e2 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,32 +1,38 @@ # docker-compose.yaml for stage0_mongodb_api testing services: - mongodb-server: + mongodb: image: mongo:7.0.5 - environment: - MONGO_INITDB_ROOT_USERNAME: root - MONGO_INITDB_ROOT_PASSWORD: example ports: - - "27017:27017" + - 27017:27017 + extra_hosts: + - "mongodb:127.0.0.1" healthcheck: - test: ["CMD", "mongosh", "-u", "root", "-p", "example", "--authenticationDatabase", "admin", "--eval", "db.adminCommand('ping')"] - interval: 2s - timeout: 2s - retries: 15 + test: echo "try { rs.status() } catch (err) { rs.initiate({_id:'rs0',members:[{_id:0,host:'mongodb:27017'}]}) }" | mongosh --port 27017 --quiet + interval: 5s + timeout: 30s + start_period: 0s + retries: 30 + command: ["--replSet", "rs0", "--bind_ip_all", "--port", "27017"] + profiles: + - mongodb + - all stage0_mongodb_api: image: ghcr.io/agile-learning-institute/stage0_mongodb_api:latest restart: no ports: - - "8582:8582" + - "8081:8081" volumes: - ./tests/test_cases/large_sample:/input environment: - MONGO_CONNECTION_STRING: mongodb://root:example@mongodb-server:27017 + MONGO_CONNECTION_STRING: mongodb://mongodb:27017/?replicaSet=rs0 MONGO_DB_NAME: test_database - MONGODB_API_PORT: 8582 + MONGODB_API_PORT: 8081 AUTO_PROCESS: True LOAD_TEST_DATA: True depends_on: - mongodb-server: + mongodb: condition: service_healthy + profiles: + - all diff --git a/stage0_mongodb_api/server.py b/stage0_mongodb_api/server.py index 5ef6c41..964404a 100644 --- a/stage0_mongodb_api/server.py +++ b/stage0_mongodb_api/server.py @@ -30,6 +30,10 @@ def handle_exit(signum, frame): signal.signal(signal.SIGTERM, handle_exit) signal.signal(signal.SIGINT, handle_exit) +# Initialize Flask App +app = Flask(__name__) +app.json = MongoJSONEncoder(app) + # Auto-processing logic - runs when module is imported (including by Gunicorn) if config.AUTO_PROCESS: logger.info(f"============= Auto Processing is Enabled ===============") @@ -48,17 +52,13 @@ def handle_exit(signum, frame): # Process all collections processing_output = config_manager.process_all_collections() - logger.info(f"Processing Output: {json.dumps(processing_output, indent=4)}") + logger.info(f"Processing Output: {app.json.dumps(processing_output)}") logger.info(f"============= Auto Processing is Completed ===============") if config.EXIT_AFTER_PROCESSING: logger.info(f"============= Exiting After Processing ===============") sys.exit(0) -# Initialize Flask App -app = Flask(__name__) -app.json = MongoJSONEncoder(app) - # Apply Prometheus monitoring middleware metrics = PrometheusMetrics(app, path='/api/health') metrics.info('app_info', 'Application info', version=config.BUILT_AT) diff --git a/tests/db_util.py b/tests/db_util.py index 7524ff3..3f65f00 100644 --- a/tests/db_util.py +++ b/tests/db_util.py @@ -192,9 +192,20 @@ def _compare_collection_with_file(self, collection_name: str, json_file: Path) - # Convert ObjectIds to strings for comparison actual_data = json.loads(json_util.dumps(actual_data)) + # Remove _id fields from both expected and actual data + expected_data_no_id = [] + for doc in expected_data: + doc_copy = {k: v for k, v in doc.items() if k != '_id'} + expected_data_no_id.append(doc_copy) + + actual_data_no_id = [] + for doc in actual_data: + doc_copy = {k: v for k, v in doc.items() if k != '_id'} + actual_data_no_id.append(doc_copy) + # Compare document counts - expected_count = len(expected_data) - actual_count = len(actual_data) + expected_count = len(expected_data_no_id) + actual_count = len(actual_data_no_id) if expected_count != actual_count: return { @@ -210,8 +221,8 @@ def _compare_collection_with_file(self, collection_name: str, json_file: Path) - matches = 0 mismatches = [] - for i, (expected, actual) in enumerate(zip(expected_data, actual_data)): - if self._documents_match(expected, actual): + for i, (expected, actual) in enumerate(zip(expected_data_no_id, actual_data_no_id)): + if expected == actual: matches += 1 else: mismatches.append({ @@ -249,22 +260,6 @@ def _compare_collection_with_file(self, collection_name: str, json_file: Path) - "message": f"Error comparing collection: {str(e)}" } - def _documents_match(self, expected: Dict, actual: Dict) -> bool: - """Compare two documents for equality. - - Args: - expected: Expected document - actual: Actual document - - Returns: - True if documents match, False otherwise - """ - # Remove _id field for comparison (it's auto-generated) - expected_copy = {k: v for k, v in expected.items() if k != '_id'} - actual_copy = {k: v for k, v in actual.items() if k != '_id'} - - return expected_copy == actual_copy - def harvest_database(self, output_path: Optional[str] = None) -> Dict[str, Any]: """Harvest all database contents to JSON files. diff --git a/tests/stepci/large_sample.yaml b/tests/stepci/large_sample.yaml index 0a8a885..bff61b2 100644 --- a/tests/stepci/large_sample.yaml +++ b/tests/stepci/large_sample.yaml @@ -1,7 +1,7 @@ name: mongodb_api_large_sample version: "1.0" env: - host: localhost:8582 + host: localhost:8081 tests: collections: diff --git a/tests/stepci/observability.yaml b/tests/stepci/observability.yaml index 0d47009..8689bf1 100644 --- a/tests/stepci/observability.yaml +++ b/tests/stepci/observability.yaml @@ -1,7 +1,7 @@ name: mongodb_api_observability version: "1.0" env: - host: localhost:8582 + host: localhost:8081 tests: observable: diff --git a/tests/stepci/small_sample.yaml b/tests/stepci/small_sample.yaml index 47fa54c..4513204 100644 --- a/tests/stepci/small_sample.yaml +++ b/tests/stepci/small_sample.yaml @@ -1,7 +1,7 @@ name: mongodb_api_small_sample version: "1.0" env: - host: localhost:8582 + host: localhost:8081 tests: collections: diff --git a/tests/test_cases/large_sample/test_database/msmCurrentVersions.json b/tests/test_cases/large_sample/test_database/CollectionVersions.json similarity index 100% rename from tests/test_cases/large_sample/test_database/msmCurrentVersions.json rename to tests/test_cases/large_sample/test_database/CollectionVersions.json diff --git a/tests/test_cases/small_sample/test_database/msmCurrentVersions.json b/tests/test_cases/small_sample/test_database/CollectionVersions.json similarity index 100% rename from tests/test_cases/small_sample/test_database/msmCurrentVersions.json rename to tests/test_cases/small_sample/test_database/CollectionVersions.json diff --git a/tests/test_server.py b/tests/test_server.py index a264aa2..747a20e 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -7,7 +7,7 @@ mock_config = MagicMock() mock_config.AUTO_PROCESS = False mock_config.EXIT_AFTER_PROCESSING = False -mock_config.MONGODB_API_PORT = 8582 +mock_config.MONGODB_API_PORT = 8081 mock_config.BUILT_AT = "test" # Patch both Config and MongoIO before importing server