From b7e6d0cfa971588c6f3af8faeeeb3885ca0658ee Mon Sep 17 00:00:00 2001 From: Yeganathan S Date: Thu, 12 Dec 2024 19:46:36 +0530 Subject: [PATCH 1/2] fix mergeWithLLM workflow for members and organizations when it fails to find data --- .../src/activities.ts | 4 ++++ .../src/activities/memberMergeSuggestions.ts | 8 +++++++ .../organizationMergeSuggestions.ts | 8 +++++++ .../src/workflows/mergeMembersWithLLM.ts | 1 + .../workflows/mergeOrganizationsWithLLM.ts | 1 + .../memberMergeSuggestions.repo.ts | 22 +++++++++++++++++++ .../organizationMergeSuggestions.repo.ts | 22 +++++++++++++++++++ 7 files changed, 66 insertions(+) diff --git a/services/apps/merge_suggestions_worker/src/activities.ts b/services/apps/merge_suggestions_worker/src/activities.ts index fe3cde744d..ecd9698c99 100644 --- a/services/apps/merge_suggestions_worker/src/activities.ts +++ b/services/apps/merge_suggestions_worker/src/activities.ts @@ -12,6 +12,7 @@ import { getMembers, getMembersForLLMConsumption, getRawMemberMergeSuggestions, + removeRawMemberMergeSuggestions, updateMemberMergeSuggestionsLastGeneratedAt, } from './activities/memberMergeSuggestions' import { @@ -21,6 +22,7 @@ import { getOrganizations, getOrganizationsForLLMConsumption, getRawOrganizationMergeSuggestions, + removeRawOrganizationMergeSuggestions, updateOrganizationMergeSuggestionsLastGeneratedAt, } from './activities/organizationMergeSuggestions' @@ -40,7 +42,9 @@ export { getMembersForLLMConsumption, getOrganizationsForLLMConsumption, getRawOrganizationMergeSuggestions, + removeRawOrganizationMergeSuggestions, getRawMemberMergeSuggestions, + removeRawMemberMergeSuggestions, saveLLMVerdict, mergeMembers, mergeOrganizations, diff --git a/services/apps/merge_suggestions_worker/src/activities/memberMergeSuggestions.ts b/services/apps/merge_suggestions_worker/src/activities/memberMergeSuggestions.ts index 48e448732c..f5962b4931 100644 --- a/services/apps/merge_suggestions_worker/src/activities/memberMergeSuggestions.ts +++ b/services/apps/merge_suggestions_worker/src/activities/memberMergeSuggestions.ts @@ -350,3 +350,11 @@ export async function getRawMemberMergeSuggestions( ) return memberMergeSuggestionsRepo.getRawMemberSuggestions(similarityFilter, limit) } + +export async function removeRawMemberMergeSuggestions(suggestion: string[]): Promise { + const memberMergeSuggestionsRepo = new MemberMergeSuggestionsRepository( + svc.postgres.writer.connection(), + svc.log, + ) + await memberMergeSuggestionsRepo.removeRawMemberSuggestions(suggestion) +} diff --git a/services/apps/merge_suggestions_worker/src/activities/organizationMergeSuggestions.ts b/services/apps/merge_suggestions_worker/src/activities/organizationMergeSuggestions.ts index bec8f29746..a058ca644b 100644 --- a/services/apps/merge_suggestions_worker/src/activities/organizationMergeSuggestions.ts +++ b/services/apps/merge_suggestions_worker/src/activities/organizationMergeSuggestions.ts @@ -427,3 +427,11 @@ export async function getRawOrganizationMergeSuggestions( return suggestions } + +export async function removeRawOrganizationMergeSuggestions(suggestion: string[]): Promise { + const organizationMergeSuggestionsRepo = new OrganizationMergeSuggestionsRepository( + svc.postgres.writer.connection(), + svc.log, + ) + await organizationMergeSuggestionsRepo.removeRawOrganizationMergeSuggestions(suggestion) +} diff --git a/services/apps/merge_suggestions_worker/src/workflows/mergeMembersWithLLM.ts b/services/apps/merge_suggestions_worker/src/workflows/mergeMembersWithLLM.ts index 79ffcb1e53..f2d0c3719a 100644 --- a/services/apps/merge_suggestions_worker/src/workflows/mergeMembersWithLLM.ts +++ b/services/apps/merge_suggestions_worker/src/workflows/mergeMembersWithLLM.ts @@ -63,6 +63,7 @@ export async function mergeMembersWithLLM( if (members.length !== 2) { console.log(`Failed getting members data in suggestion. Skipping suggestion: ${suggestion}`) + await memberActivitiesProxy.removeRawMemberMergeSuggestions(suggestion) continue } diff --git a/services/apps/merge_suggestions_worker/src/workflows/mergeOrganizationsWithLLM.ts b/services/apps/merge_suggestions_worker/src/workflows/mergeOrganizationsWithLLM.ts index e0288c5a61..9267674276 100644 --- a/services/apps/merge_suggestions_worker/src/workflows/mergeOrganizationsWithLLM.ts +++ b/services/apps/merge_suggestions_worker/src/workflows/mergeOrganizationsWithLLM.ts @@ -51,6 +51,7 @@ export async function mergeOrganizationsWithLLM( console.log( `Failed getting organization data in suggestion. Skipping suggestion: ${suggestion}`, ) + await organizationActivitiesProxy.removeRawOrganizationMergeSuggestions(suggestion) continue } diff --git a/services/libs/data-access-layer/src/old/apps/merge_suggestions_worker/memberMergeSuggestions.repo.ts b/services/libs/data-access-layer/src/old/apps/merge_suggestions_worker/memberMergeSuggestions.repo.ts index fa7976f74f..5b5ad201cd 100644 --- a/services/libs/data-access-layer/src/old/apps/merge_suggestions_worker/memberMergeSuggestions.repo.ts +++ b/services/libs/data-access-layer/src/old/apps/merge_suggestions_worker/memberMergeSuggestions.repo.ts @@ -283,6 +283,28 @@ class MemberMergeSuggestionsRepository { return results.map((r) => [r.memberId, r.toMergeId]) } + + async removeRawMemberSuggestions(suggestion: string[]): Promise { + const query = ` + delete from "memberToMergeRaw" + where + ("memberId" = $(memberId) and "toMergeId" = $(toMergeId)) + or + ("memberId" = $(toMergeId) and "toMergeId" = $(memberId)) + ` + + const replacements = { + memberId: suggestion[0], + toMergeId: suggestion[1], + } + + try { + await this.connection.result(query, replacements) + } catch (error) { + this.log.error('Error removing raw member suggestions', error) + throw error + } + } } export default MemberMergeSuggestionsRepository diff --git a/services/libs/data-access-layer/src/old/apps/merge_suggestions_worker/organizationMergeSuggestions.repo.ts b/services/libs/data-access-layer/src/old/apps/merge_suggestions_worker/organizationMergeSuggestions.repo.ts index 310f210470..1ad8f138e4 100644 --- a/services/libs/data-access-layer/src/old/apps/merge_suggestions_worker/organizationMergeSuggestions.repo.ts +++ b/services/libs/data-access-layer/src/old/apps/merge_suggestions_worker/organizationMergeSuggestions.repo.ts @@ -300,6 +300,28 @@ class OrganizationMergeSuggestionsRepository { return results.map((r) => [r.organizationId, r.toMergeId]) } + + async removeRawOrganizationMergeSuggestions(suggestion: string[]): Promise { + const query = ` + delete from "organizationToMergeRaw" + where + ("organizationId" = $(organizationId) and "toMergeId" = $(toMergeId)) + or + ("organizationId" = $(toMergeId) and "toMergeId" = $(organizationId)) + ` + + const replacements = { + organizationId: suggestion[0], + toMergeId: suggestion[1], + } + + try { + await this.connection.result(query, replacements) + } catch (error) { + this.log.error('Error removing raw organization suggestions', error) + throw error + } + } } export default OrganizationMergeSuggestionsRepository From 06f3fb8ea65d12aaa16683320bfa4a1ff9c299c1 Mon Sep 17 00:00:00 2001 From: Yeganathan S Date: Fri, 13 Dec 2024 20:51:17 +0530 Subject: [PATCH 2/2] fix --- .../merge_suggestions_worker/memberMergeSuggestions.repo.ts | 2 +- .../organizationMergeSuggestions.repo.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/services/libs/data-access-layer/src/old/apps/merge_suggestions_worker/memberMergeSuggestions.repo.ts b/services/libs/data-access-layer/src/old/apps/merge_suggestions_worker/memberMergeSuggestions.repo.ts index 5b5ad201cd..eb9b4c9a79 100644 --- a/services/libs/data-access-layer/src/old/apps/merge_suggestions_worker/memberMergeSuggestions.repo.ts +++ b/services/libs/data-access-layer/src/old/apps/merge_suggestions_worker/memberMergeSuggestions.repo.ts @@ -299,7 +299,7 @@ class MemberMergeSuggestionsRepository { } try { - await this.connection.result(query, replacements) + await this.connection.none(query, replacements) } catch (error) { this.log.error('Error removing raw member suggestions', error) throw error diff --git a/services/libs/data-access-layer/src/old/apps/merge_suggestions_worker/organizationMergeSuggestions.repo.ts b/services/libs/data-access-layer/src/old/apps/merge_suggestions_worker/organizationMergeSuggestions.repo.ts index 1ad8f138e4..c25b858486 100644 --- a/services/libs/data-access-layer/src/old/apps/merge_suggestions_worker/organizationMergeSuggestions.repo.ts +++ b/services/libs/data-access-layer/src/old/apps/merge_suggestions_worker/organizationMergeSuggestions.repo.ts @@ -316,7 +316,7 @@ class OrganizationMergeSuggestionsRepository { } try { - await this.connection.result(query, replacements) + await this.connection.none(query, replacements) } catch (error) { this.log.error('Error removing raw organization suggestions', error) throw error