Added SCIM /Bulk API endpoint (#1985)

Author: rhysyngsun

main/factories.py

@@ -4,7 +4,7 @@
 
 import ulid
 from django.conf import settings
-from factory import LazyFunction, RelatedFactory, SubFactory, Trait
+from factory import Faker, LazyFunction, RelatedFactory, SubFactory, Trait
 from factory.django import DjangoModelFactory
 from factory.fuzzy import FuzzyText
 from social_django.models import UserSocialAuth
@@ -15,8 +15,8 @@ class UserFactory(DjangoModelFactory):
 
     username = LazyFunction(lambda: ulid.new().str)
     email = FuzzyText(suffix="@example.com")
-    first_name = FuzzyText()
-    last_name = FuzzyText()
+    first_name = Faker("first_name")
+    last_name = Faker("last_name")
 
     profile = RelatedFactory("profiles.factories.ProfileFactory", "user")
 

main/settings.py

@@ -122,6 +122,7 @@
     "data_fixtures",
     "vector_search",
     "ai_chat",
+    "scim",
 )
 
 if not get_bool("RUN_DATA_MIGRATIONS", default=False):
@@ -141,9 +142,11 @@
             "documentationUri": "",
         },
     ],
-    "USER_ADAPTER": "profiles.scim.adapters.LearnSCIMUser",
-    "USER_MODEL_GETTER": "profiles.scim.adapters.get_user_model_for_scim",
-    "USER_FILTER_PARSER": "profiles.scim.filters.LearnUserFilterQuery",
+    "SERVICE_PROVIDER_CONFIG_MODEL": "scim.config.LearnSCIMServiceProviderConfig",
+    "USER_ADAPTER": "scim.adapters.LearnSCIMUser",
+    "USER_MODEL_GETTER": "scim.adapters.get_user_model_for_scim",
+    "USER_FILTER_PARSER": "scim.filters.LearnUserFilterQuery",
+    "GET_IS_AUTHENTICATED_PREDICATE": "scim.utils.is_authenticated_predicate",
 }
 
 

main/settings_celery.py

@@ -131,6 +131,12 @@
         "schedule": crontab(minute=30, hour=18),  # 2:30pm EST
         "kwargs": {"period": "daily", "subscription_type": "channel_subscription_type"},
     },
+    "daily_embed_new_learning_resources": {
+        "task": "vector_search.tasks.embed_new_learning_resources",
+        "schedule": get_int(
+            "EMBED_NEW_RESOURCES_SCHEDULE_SECONDS", 60 * 30
+        ),  # default is every 30 minutes
+    },
     "send-search-subscription-emails-every-1-days": {
         "task": "learning_resources_search.tasks.send_subscription_emails",
         "schedule": crontab(minute=0, hour=19),  # 3:00pm EST

main/urls.py

@@ -17,7 +17,7 @@
 from django.conf import settings
 from django.conf.urls.static import static
 from django.contrib import admin
-from django.urls import include, path, re_path
+from django.urls import include, re_path
 from django.views.generic.base import RedirectView
 from rest_framework.routers import DefaultRouter
 
@@ -41,7 +41,6 @@
 
 urlpatterns = (
     [  # noqa: RUF005
-        path("scim/v2/", include("django_scim.urls")),
         re_path(r"^o/", include("oauth2_provider.urls", namespace="oauth2_provider")),
         re_path(r"^admin/", admin.site.urls),
         re_path(r"", include("authentication.urls")),
@@ -58,6 +57,7 @@
         re_path(r"", include("articles.urls")),
         re_path(r"", include("testimonials.urls")),
         re_path(r"", include("news_events.urls")),
+        re_path(r"", include("scim.urls")),
         re_path(r"", include(features_router.urls)),
         re_path(r"^app", RedirectView.as_view(url=settings.APP_BASE_URL)),
         # Hijack

poetry.lock

@@ -1,6 +1,8 @@
 """Factories for making test data"""
 
-from factory import Faker, Sequence, SubFactory
+import uuid
+
+from factory import Faker, LazyFunction, SelfAttribute, Sequence, SubFactory
 from factory.django import DjangoModelFactory
 from factory.fuzzy import FuzzyChoice
 from faker.providers import BaseProvider
@@ -49,6 +51,9 @@ class ProfileFactory(DjangoModelFactory):
         [Profile.CertificateDesired.YES.value, Profile.CertificateDesired.NO.value]
     )
 
+    scim_external_id = LazyFunction(uuid.uuid4)
+    scim_username = SelfAttribute("user.email")
+
     class Meta:
         model = Profile
 

profiles/factories.py

@@ -91,6 +91,7 @@ llama-index-llms-openai = "^0.3.12"
 llama-index-agent-openai = "^0.4.1"
 langchain-experimental = "^0.3.4"
 langchain-openai = "^0.3.2"
+deepmerge = "^2.0"
 
 
 [tool.poetry.group.dev.dependencies]

profiles/scim/views_test.py

@@ -0,0 +1,36 @@
+## SCIM
+
+## Prerequisites
+
+- You need the following a local [Keycloak](https://www.keycloak.org/) instance running. Note which major version you are running (should be at least 26.x).
+  - You should have custom user profile fields setup on your `olapps` realm:
+    - `fullName`: required, otherwise defaults
+    - `emailOptIn`: defaults
+
+## Install the scim-for-keycloak plugin
+
+Sign up for an account on https://scim-for-keycloak.de and follow the instructions here: https://scim-for-keycloak.de/documentation/installation/install
+
+## Configure SCIM
+
+In the SCIM admin console, do the following:
+
+### Configure Remote SCIM Provider
+
+- In django-admin, go to OAuth Toolkit and create a new access token
+- Go to Remote SCIM Provider
+- Click the `+` button
+- Specify a base URL for your learn API backend: `http://<IP_OR_HOSTNAME>:8063/scim/v2/`
+- At the bottom of the page, click "Use default configuration"
+- Add a new authentication method:
+  - Type: Long Life Bearer Token
+  - Bearer Token: the access token you created above
+- On the Schemas tab, edit the User schema and add these custom attributes:
+  - Add a `fullName` attribute and set the Custom Attribute Name to `fullName`
+  - Add an attribute named `emailOptIn` with the following settings:
+    - Type: integer
+    - Custom Attribute Name: `emailOptIn`
+- On the Realm Assignments tab, assign to the `olapps` realm
+- Go to the Synchronization tab and perform one:
+  - Identifier attribute: email
+  - Synchronization strategy: Search and Bulk

pyproject.toml

@@ -44,6 +44,7 @@ class LearnSCIMUser(SCIMUser):
         ("active", None, None): "is_active",
         ("name", "givenName", None): "first_name",
         ("name", "familyName", None): "last_name",
+        ("userName", None, None): "username",
     }
 
     IGNORED_PATHS = {
@@ -158,7 +159,7 @@ def delete(self):
         """
         self.obj.is_active = False
         self.obj.save()
-        logger.info("Deactivated user id %i", self.obj.user.id)
+        logger.info("Deactivated user id %i", self.obj.id)
 
     def handle_add(
         self,
@@ -193,7 +194,7 @@ def parse_scim_for_keycloak_payload(self, payload: str) -> dict:
 
             if isinstance(value, dict):
                 for nested_key, nested_value in value.items():
-                    result[f"{key}.{nested_key}"] = nested_value
+                    result[self.split_path(f"{key}.{nested_key}")] = nested_value
             else:
                 result[key] = value
 
@@ -202,11 +203,32 @@ def parse_scim_for_keycloak_payload(self, payload: str) -> dict:
     def parse_path_and_values(
         self, path: Optional[str], value: Union[str, list, dict]
     ) -> list:
-        if not path and isinstance(value, str):
+        """Parse the incoming value(s)"""
+        if isinstance(value, str):
             # scim-for-keycloak sends this as a noncompliant JSON-encoded string
-            value = self.parse_scim_for_keycloak_payload(value)
+            if path is None:
+                val = json.loads(value)
+            else:
+                msg = "Called with a non-null path and a str value"
+                raise ValueError(msg)
+        else:
+            val = value
+
+        results = []
+
+        for attr_path, attr_value in val.items():
+            if isinstance(attr_value, dict):
+                # nested object, we want to recursively flatten it to `first.second`
+                results.extend(self.parse_path_and_values(attr_path, attr_value))
+            else:
+                flattened_path = (
+                    f"{path}.{attr_path}" if path is not None else attr_path
+                )
+                new_path = self.split_path(flattened_path)
+                new_value = attr_value
+                results.append((new_path, new_value))
 
-        return super().parse_path_and_values(path, value)
+        return results
 
     def handle_replace(
         self,
@@ -219,22 +241,20 @@ def handle_replace(
 
         All operations happen within an atomic transaction.
         """
+
         if not isinstance(value, dict):
             # Restructure for use in loop below.
             value = {path: value}
 
         for nested_path, nested_value in (value or {}).items():
             if nested_path.first_path in self.ATTR_MAP:
-                setattr(
-                    self.obj, self.ATTR_MAP.get(nested_path.first_path), nested_value
-                )
-
+                setattr(self.obj, self.ATTR_MAP[nested_path.first_path], nested_value)
             elif nested_path.first_path == ("fullName", None, None):
                 self.obj.profile.name = nested_value
             elif nested_path.first_path == ("emailOptIn", None, None):
                 self.obj.profile.email_optin = nested_value == 1
             elif nested_path.first_path == ("emails", None, None):
-                self.parse_emails(value)
+                self.parse_emails(nested_value)
             elif nested_path.first_path not in self.IGNORED_PATHS:
                 logger.debug(
                     "Ignoring SCIM update for path: %s", nested_path.first_path

scim/README.md

@@ -0,0 +1,5 @@
+from django.apps import AppConfig
+
+
+class ScimConfig(AppConfig):
+    name = "scim"

profiles/scim/__init__.py renamed to scim/__init__.py

@@ -0,0 +1,13 @@
+from django_scim.models import SCIMServiceProviderConfig
+
+
+class LearnSCIMServiceProviderConfig(SCIMServiceProviderConfig):
+    """Custom provider config"""
+
+    def to_dict(self):
+        result = super().to_dict()
+
+        result["bulk"]["supported"] = True
+        result["filter"]["supported"] = True
+
+        return result