feat(github): add support for custom GitHub domains in API service

Author: thesujai

backend/analytics_server/mhq/exapi/github.py

@@ -22,11 +22,20 @@ class GithubRateLimitExceeded(Exception):
 
 
 class GithubApiService:
-    def __init__(self, access_token: str):
+    def __init__(self, access_token: str, domain = "https://api.github.com"):
         self._token = access_token
-        self._g = Github(self._token, per_page=PAGE_SIZE)
-        self.base_url = "https://api.github.com"
+        self.base_url = self._get_api_url(domain)
+        self._g = Github(
+            self._token, 
+            base_url=self.base_url,
+            per_page=PAGE_SIZE
+        )
         self.headers = {"Authorization": f"Bearer {self._token}"}
+    def _get_api_url(self, domain: str) -> str:
+        if domain == "https://api.github.com":
+            return domain
+        else:
+            return f"{domain}/api/v3"
 
     @contextlib.contextmanager
     def temp_config(self, per_page: int = 30):

backend/analytics_server/mhq/service/code/sync/etl_github_handler.py

@@ -358,6 +358,26 @@ def _dt_from_github_dt_string(dt_string: str) -> datetime:
 
 
 def get_github_etl_handler(org_id: str) -> GithubETLHandler:
+    def _get_custom_github_domain() -> Optional[str]:
+        DEFAULT_DOMAIN = "https://api.github.com"
+        core_repo_service = CoreRepoService()
+        integrations = core_repo_service.get_org_integrations_for_names(
+            org_id, [UserIdentityProvider.GITHUB.value]
+        )
+
+        github_domain = (
+            integrations[0].provider_meta.get("custom_domain")
+            if integrations[0].provider_meta
+            else None
+        )
+
+        if not github_domain:
+            LOG.warn(
+                f"Custom domain not found for intergration for org {org_id} and provider {UserIdentityProvider.GITLAB.value}"
+            )
+            return DEFAULT_DOMAIN
+
+        return github_domain
     def _get_access_token():
         core_repo_service = CoreRepoService()
         access_token = core_repo_service.get_access_token(
@@ -371,7 +391,7 @@ def _get_access_token():
 
     return GithubETLHandler(
         org_id,
-        GithubApiService(_get_access_token()),
+        GithubApiService(_get_access_token(), _get_custom_github_domain()),
         CodeRepoService(),
         CodeETLAnalyticsService(),
         get_revert_prs_github_sync_handler(),

backend/analytics_server/mhq/service/external_integrations_service.py

@@ -26,29 +26,29 @@ def __init__(
         self.custom_domain = custom_domain
 
     def get_github_organizations(self):
-        github_api_service = GithubApiService(self.access_token)
+        github_api_service = GithubApiService(self.access_token, self.custom_domain)
         try:
             orgs: [GithubOrganization] = github_api_service.get_org_list()
         except GithubException as e:
             raise e
         return orgs
 
     def get_github_org_repos(self, org_login: str, page_size: int, page: int):
-        github_api_service = GithubApiService(self.access_token)
+        github_api_service = GithubApiService(self.access_token, self.custom_domain)
         try:
             return github_api_service.get_repos_raw(org_login, page_size, page)
         except GithubException as e:
             raise e
 
     def get_github_personal_repos(self, page_size: int, page: int):
-        github_api_service = GithubApiService(self.access_token)
+        github_api_service = GithubApiService(self.access_token, self.custom_domain)
         try:
             return github_api_service.get_user_repos_raw(page_size, page)
         except GithubException as e:
             raise e
 
     def get_repo_workflows(self, gh_org_name: str, gh_org_repo_name: str):
-        github_api_service = GithubApiService(self.access_token)
+        github_api_service = GithubApiService(self.access_token, self.custom_domain)
         try:
             workflows = github_api_service.get_repo_workflows(
                 gh_org_name, gh_org_repo_name

backend/analytics_server/mhq/service/workflows/sync/etl_github_actions_handler.py

@@ -169,6 +169,26 @@ def _get_datetime_from_gh_datetime(datetime_str: str) -> datetime:
 
 
 def get_github_actions_etl_handler(org_id):
+    def _get_custom_github_domain() -> Optional[str]:
+        DEFAULT_DOMAIN = "https://api.github.com"
+        core_repo_service = CoreRepoService()
+        integrations = core_repo_service.get_org_integrations_for_names(
+            org_id, [UserIdentityProvider.GITHUB.value]
+        )
+
+        github_domain = (
+            integrations[0].provider_meta.get("custom_domain")
+            if integrations[0].provider_meta
+            else None
+        )
+
+        if not github_domain:
+            LOG.warn(
+                f"Custom domain not found for intergration for org {org_id} and provider {UserIdentityProvider.GITLAB.value}"
+            )
+            return DEFAULT_DOMAIN
+
+        return github_domain
     def _get_access_token():
         core_repo_service = CoreRepoService()
         access_token = core_repo_service.get_access_token(
@@ -181,5 +201,5 @@ def _get_access_token():
         return access_token
 
     return GithubActionsETLHandler(
-        org_id, GithubApiService(_get_access_token()), WorkflowRepoService()
+        org_id, GithubApiService(_get_access_token(), _get_custom_github_domain()), WorkflowRepoService()
     )

web-server/pages/api/internal/[org_id]/utils.ts

@@ -53,7 +53,7 @@ export const searchGithubRepos = async (
 };
 
 const searchRepoWithURL = async (searchString: string) => {
-  const apiUrl = `https://api.github.com/repos/${searchString}`;
+  const apiUrl = await getGitHubRestApiUrl(`repos/${searchString}`);
   const response = await axios.get<GithubRepo>(apiUrl);
   const repo = response.data;
   return [
@@ -104,7 +104,9 @@ export const searchGithubReposWithNames = async (
 
   const queryString = `${searchString} in:name fork:true`;
 
-  const response = await fetch(GITHUB_API_URL, {
+  const githubApiUrl = await getGitHubGraphQLUrl();
+
+  const response = await fetch(githubApiUrl, {
     method: 'POST',
     headers: {
       'Content-Type': 'application/json',
@@ -304,3 +306,23 @@ const replaceURL = async (url: string): Promise<string> => {
 
   return url;
 };
+
+const getGitHubCustomDomain = async (): Promise<string | null> => {
+  const provider_meta = await db('Integration')
+    .where('name', Integration.GITHUB)
+    .then((r: Row<'Integration'>[]) => r.map((item) => item.provider_meta));
+
+  return head(provider_meta || [])?.custom_domain || null;
+};
+
+const getGitHubRestApiUrl = async (path: string): Promise<string> => {
+  const customDomain = await getGitHubCustomDomain();
+  const baseUrl = customDomain ? `https://${customDomain}/api/v3` : GITHUB_API_URL;
+  return `${baseUrl}/${path}`.replace(/\/+/g, '/');
+};
+
+const getGitHubGraphQLUrl = async (): Promise<string> => {
+  const customDomain = await getGitHubCustomDomain();
+  return customDomain ? `https://${customDomain}/api/graphql` : GITHUB_API_URL;
+};
+