diff --git a/cla-backend-go/auth/auth0.go b/cla-backend-go/auth/auth0.go index 2d0dc717f..dcaa3ef98 100644 --- a/cla-backend-go/auth/auth0.go +++ b/cla-backend-go/auth/auth0.go @@ -24,7 +24,7 @@ type Validator struct { } // NewAuthValidator creates a new auth0 validator based on the specified parameters -func NewAuthValidator(domain, clientID, usernameClaim, algorithm string) (Validator, error) { // nolint +func NewAuthValidator(domain, clientID, usernameClaim, nameClaim, emailClaim, algorithm string) (Validator, error) { // nolint if domain == "" { return Validator{}, errors.New("missing Domain") } @@ -43,8 +43,8 @@ func NewAuthValidator(domain, clientID, usernameClaim, algorithm string) (Valida usernameClaim: usernameClaim, algorithm: algorithm, wellKnownURL: "https://" + path.Join(domain, ".well-known/jwks.json"), - nameClaim: "name", - emailClaim: "email", + nameClaim: nameClaim, + emailClaim: emailClaim, } return validator, nil diff --git a/cla-backend-go/cmd/response_metrics.go b/cla-backend-go/cmd/response_metrics.go index e3241a290..98eee4e7c 100644 --- a/cla-backend-go/cmd/response_metrics.go +++ b/cla-backend-go/cmd/response_metrics.go @@ -4,6 +4,7 @@ package cmd import ( + "sync" "time" "github.com/linuxfoundation/easycla/cla-backend-go/utils" @@ -18,32 +19,36 @@ type responseMetrics struct { expire time.Time } -var reqMap = make(map[string]*responseMetrics, 5) +var reqMap sync.Map // requestStart holds the request ID, method and timing information in a small structure func requestStart(reqID, method string) { now, _ := utils.CurrentTime() - reqMap[reqID] = &responseMetrics{ + rm := &responseMetrics{ reqID: reqID, method: method, start: now, elapsed: 0, expire: now.Add(time.Minute * 5), } + reqMap.Store(reqID, rm) } // getRequestMetrics returns the response metrics based on the request id value func getRequestMetrics(reqID string) *responseMetrics { - if x, found := reqMap[reqID]; found { + if val, found := reqMap.Load(reqID); found { + rm, ok := val.(*responseMetrics) + if !ok { + return nil + } now, _ := utils.CurrentTime() - x.elapsed = now.Sub(x.start) - return x + rm.elapsed = now.Sub(rm.start) + return rm } - return nil } // clearRequestMetrics removes the request from the map func clearRequestMetrics(reqID string) { - delete(reqMap, reqID) + reqMap.Delete(reqID) } diff --git a/cla-backend-go/cmd/server.go b/cla-backend-go/cmd/server.go index 10fd7eae3..c6def26ca 100644 --- a/cla-backend-go/cmd/server.go +++ b/cla-backend-go/cmd/server.go @@ -236,11 +236,24 @@ func server(localMode bool) http.Handler { } // LG: to test with manual tokens - // configFile.Auth0.UsernameClaim = "http://lfx.dev/claims/username" + customClaimUsername := os.Getenv("AUTH0_USERNAME_CLAIM_CLI") + if customClaimUsername != "" { + configFile.Auth0.UsernameClaim = customClaimUsername + } + nameClaimName := os.Getenv("AUTH0_NAME_CLAIM_CLI") + if nameClaimName == "" { + nameClaimName = "name" + } + emailClaimName := os.Getenv("AUTH0_EMAIL_CLAIM_CLI") + if emailClaimName == "" { + emailClaimName = "email" + } authValidator, err := auth.NewAuthValidator( configFile.Auth0.Domain, configFile.Auth0.ClientID, configFile.Auth0.UsernameClaim, + nameClaimName, + emailClaimName, configFile.Auth0.Algorithm) if err != nil { logrus.Panic(err) @@ -336,7 +349,7 @@ func server(localMode bool) http.Handler { // Setup our API handlers users.Configure(api, usersService, eventsService) project.Configure(api, v1ProjectService, eventsService, gerritService, v1RepositoriesService, v1SignaturesService) - v2Project.Configure(v2API, v1ProjectService, v2ProjectService, eventsService) + v2Project.Configure(v2API, v1ProjectService, v2ProjectService, eventsService, v1ProjectClaGroupService, v2RepositoriesService, gerritService) health.Configure(api, healthService) v2Health.Configure(v2API, healthService) template.Configure(api, templateService, eventsService) diff --git a/cla-backend-go/project/mocks/mock_repo.go b/cla-backend-go/project/mocks/mock_repo.go index e5b2a39be..684a2c2e3 100644 --- a/cla-backend-go/project/mocks/mock_repo.go +++ b/cla-backend-go/project/mocks/mock_repo.go @@ -83,6 +83,21 @@ func (mr *MockProjectRepositoryMockRecorder) GetCLAGroupByID(ctx, claGroupID, lo return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCLAGroupByID", reflect.TypeOf((*MockProjectRepository)(nil).GetCLAGroupByID), ctx, claGroupID, loadRepoDetails) } +// GetCLAGroupByIDCompat mocks base method. +func (m *MockProjectRepository) GetCLAGroupByIDCompat(ctx context.Context, claGroupID string, loadRepoDetails bool) (*models.ClaGroup, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetCLAGroupByIDCompat", ctx, claGroupID, loadRepoDetails) + ret0, _ := ret[0].(*models.ClaGroup) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetCLAGroupByIDCompat indicates an expected call of GetCLAGroupByIDCompat. +func (mr *MockProjectRepositoryMockRecorder) GetCLAGroupByIDCompat(ctx, claGroupID, loadRepoDetails interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCLAGroupByIDCompat", reflect.TypeOf((*MockProjectRepository)(nil).GetCLAGroupByIDCompat), ctx, claGroupID, loadRepoDetails) +} + // GetCLAGroupByName mocks base method. func (m *MockProjectRepository) GetCLAGroupByName(ctx context.Context, claGroupName string) (*models.ClaGroup, error) { m.ctrl.T.Helper() diff --git a/cla-backend-go/project/mocks/mock_service.go b/cla-backend-go/project/mocks/mock_service.go index d534036a2..eab7941ce 100644 --- a/cla-backend-go/project/mocks/mock_service.go +++ b/cla-backend-go/project/mocks/mock_service.go @@ -83,6 +83,21 @@ func (mr *MockServiceMockRecorder) GetCLAGroupByID(ctx, claGroupID interface{}) return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCLAGroupByID", reflect.TypeOf((*MockService)(nil).GetCLAGroupByID), ctx, claGroupID) } +// GetCLAGroupByIDCompat mocks base method. +func (m *MockService) GetCLAGroupByIDCompat(ctx context.Context, claGroupID string) (*models.ClaGroup, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetCLAGroupByIDCompat", ctx, claGroupID) + ret0, _ := ret[0].(*models.ClaGroup) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetCLAGroupByIDCompat indicates an expected call of GetCLAGroupByIDCompat. +func (mr *MockServiceMockRecorder) GetCLAGroupByIDCompat(ctx, claGroupID interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCLAGroupByIDCompat", reflect.TypeOf((*MockService)(nil).GetCLAGroupByIDCompat), ctx, claGroupID) +} + // GetCLAGroupByName mocks base method. func (m *MockService) GetCLAGroupByName(ctx context.Context, projectName string) (*models.ClaGroup, error) { m.ctrl.T.Helper() diff --git a/cla-backend-go/project/repository/repository.go b/cla-backend-go/project/repository/repository.go index a14441a34..093d27db5 100644 --- a/cla-backend-go/project/repository/repository.go +++ b/cla-backend-go/project/repository/repository.go @@ -50,6 +50,7 @@ const ( type ProjectRepository interface { //nolint CreateCLAGroup(ctx context.Context, claGroupModel *models.ClaGroup) (*models.ClaGroup, error) GetCLAGroupByID(ctx context.Context, claGroupID string, loadRepoDetails bool) (*models.ClaGroup, error) + GetCLAGroupByIDCompat(ctx context.Context, claGroupID string, loadRepoDetails bool) (*models.ClaGroup, error) GetCLAGroupsByExternalID(ctx context.Context, params *project.GetProjectsByExternalIDParams, loadRepoDetails bool) (*models.ClaGroups, error) GetCLAGroupByName(ctx context.Context, claGroupName string) (*models.ClaGroup, error) GetExternalCLAGroup(ctx context.Context, claGroupExternalID string) (*models.ClaGroup, error) @@ -149,7 +150,7 @@ func (repo *repo) CreateCLAGroup(ctx context.Context, claGroupModel *models.ClaG return claGroupModel, nil } -func (repo *repo) getCLAGroupByID(ctx context.Context, claGroupID string, loadCLAGroupDetails bool) (*models.ClaGroup, error) { +func (repo *repo) getCLAGroupByID(ctx context.Context, claGroupID string, loadCLAGroupDetails bool, claEnabledDefaultIsTrue bool) (*models.ClaGroup, error) { f := logrus.Fields{ "functionName": "project.repository.getCLAGroupByID", utils.XREQUESTID: ctx.Value(utils.XREQUESTID), @@ -188,11 +189,21 @@ func (repo *repo) getCLAGroupByID(ctx context.Context, claGroupID string, loadCL return nil, &utils.CLAGroupNotFound{CLAGroupID: claGroupID} } var dbModel models2.DBProjectModel - err = dynamodbattribute.UnmarshalMap(results.Items[0], &dbModel) + rawItem := results.Items[0] + err = dynamodbattribute.UnmarshalMap(rawItem, &dbModel) if err != nil { log.WithFields(f).Warnf("error unmarshalling db cla group model, error: %+v", err) return nil, err } + if claEnabledDefaultIsTrue { + // If missing, assume true like Pynamo default=True + if _, ok := rawItem["project_icla_enabled"]; !ok { + dbModel.ProjectIclaEnabled = true + } + if _, ok := rawItem["project_ccla_enabled"]; !ok { + dbModel.ProjectCclaEnabled = true + } + } // Convert the database model to an API response model return repo.buildCLAGroupModel(ctx, dbModel, loadCLAGroupDetails), nil @@ -200,7 +211,14 @@ func (repo *repo) getCLAGroupByID(ctx context.Context, claGroupID string, loadCL // GetCLAGroupByID returns the cla group model associated for the specified claGroupID func (repo *repo) GetCLAGroupByID(ctx context.Context, claGroupID string, loadRepoDetails bool) (*models.ClaGroup, error) { - return repo.getCLAGroupByID(ctx, claGroupID, loadRepoDetails) + return repo.getCLAGroupByID(ctx, claGroupID, loadRepoDetails, false) +} + +// GetCLAGroupByIDCompat returns the cla group model associated for the specified claGroupID +func (repo *repo) GetCLAGroupByIDCompat(ctx context.Context, claGroupID string, loadRepoDetails bool) (*models.ClaGroup, error) { + // Uses compatible mode (with python v2): claEnabledDefaultIsTrue - means if project_ccla_enabled or project_icla_enabled + // aren't set on dynamoDB item - they will default to true as in Py V2 API + return repo.getCLAGroupByID(ctx, claGroupID, loadRepoDetails, true) } // GetCLAGroupsByExternalID queries the database and returns a list of the cla groups @@ -383,7 +401,7 @@ func (repo *repo) GetClaGroupByProjectSFID(ctx context.Context, projectSFID stri log.WithFields(f).Debugf("found CLA Group ID: %s for project SFID: %s", claGroupProject.ClaGroupID, projectSFID) - return repo.getCLAGroupByID(ctx, claGroupProject.ClaGroupID, loadRepoDetails) + return repo.getCLAGroupByID(ctx, claGroupProject.ClaGroupID, loadRepoDetails, false) } // GetCLAGroupByName returns the project model associated for the specified project name diff --git a/cla-backend-go/project/service/service.go b/cla-backend-go/project/service/service.go index c9f246ea5..6c6e5c108 100644 --- a/cla-backend-go/project/service/service.go +++ b/cla-backend-go/project/service/service.go @@ -30,6 +30,7 @@ type Service interface { CreateCLAGroup(ctx context.Context, project *models.ClaGroup) (*models.ClaGroup, error) GetCLAGroups(ctx context.Context, params *project.GetProjectsParams) (*models.ClaGroups, error) GetCLAGroupByID(ctx context.Context, claGroupID string) (*models.ClaGroup, error) + GetCLAGroupByIDCompat(ctx context.Context, claGroupID string) (*models.ClaGroup, error) GetCLAGroupsByExternalSFID(ctx context.Context, projectSFID string) (*models.ClaGroups, error) GetCLAGroupsByExternalID(ctx context.Context, params *project.GetProjectsByExternalIDParams) (*models.ClaGroups, error) GetCLAGroupByName(ctx context.Context, projectName string) (*models.ClaGroup, error) @@ -75,6 +76,16 @@ func (s ProjectService) GetCLAGroups(ctx context.Context, params *project.GetPro // GetCLAGroupByID service method func (s ProjectService) GetCLAGroupByID(ctx context.Context, claGroupID string) (*models.ClaGroup, error) { + return s.getCLAGroupByID(ctx, claGroupID, false) +} + +// GetCLAGroupByIDCompat service method +func (s ProjectService) GetCLAGroupByIDCompat(ctx context.Context, claGroupID string) (*models.ClaGroup, error) { + return s.getCLAGroupByID(ctx, claGroupID, true) +} + +// getCLAGroupByID service method +func (s ProjectService) getCLAGroupByID(ctx context.Context, claGroupID string, claEnabledDefaultIsTrue bool) (*models.ClaGroup, error) { f := logrus.Fields{ "functionName": "GetCLAGroupByID", utils.XREQUESTID: ctx.Value(utils.XREQUESTID), @@ -83,7 +94,15 @@ func (s ProjectService) GetCLAGroupByID(ctx context.Context, claGroupID string) } log.WithFields(f).Debug("locating CLA Group by ID...") - project, err := s.repo.GetCLAGroupByID(ctx, claGroupID, repository.LoadRepoDetails) + var ( + project *models.ClaGroup + err error + ) + if claEnabledDefaultIsTrue { + project, err = s.repo.GetCLAGroupByIDCompat(ctx, claGroupID, repository.LoadRepoDetails) + } else { + project, err = s.repo.GetCLAGroupByID(ctx, claGroupID, repository.LoadRepoDetails) + } if err != nil { return nil, err } diff --git a/cla-backend-go/swagger/cla.v2.yaml b/cla-backend-go/swagger/cla.v2.yaml index 672b96e6f..a267a8851 100644 --- a/cla-backend-go/swagger/cla.v2.yaml +++ b/cla-backend-go/swagger/cla.v2.yaml @@ -889,6 +889,38 @@ paths: tags: - project + /project-compat/{projectID}: + parameters: + - $ref: "#/parameters/x-request-id" + - name: projectID + in: path + type: string + required: true + pattern: '^[a-fA-F0-9]{8}-?[a-fA-F0-9]{4}-?4[a-fA-F0-9]{3}-?[89ab][a-fA-F0-9]{3}-?[a-fA-F0-9]{12}$' # uuidv4 + get: + summary: Get project by ID (returns data in the same format as Py V2 API) + security: [ ] + operationId: getProjectCompat + responses: + '200': + description: 'Success' + headers: + x-request-id: + type: string + description: The unique request ID value - assigned/set by the API Gateway based on the session + schema: + $ref: '#/definitions/project-compat' + '400': + $ref: '#/responses/invalid-request' + '401': + $ref: '#/responses/unauthorized' + '403': + $ref: '#/responses/forbidden' + '404': + $ref: '#/responses/not-found' + tags: + - project + /events/recent: get: summary: List recent events - requires Admin-level access @@ -4956,6 +4988,9 @@ definitions: sf-project-summary: $ref: './common/sf-project-summary.yaml' + project-compat: + $ref: './common/project-compat.yaml' + # --------------------------------------------------------------------------- # CLA Template Definitions # --------------------------------------------------------------------------- diff --git a/cla-backend-go/swagger/common/project-compat.yaml b/cla-backend-go/swagger/common/project-compat.yaml new file mode 100644 index 000000000..7abb5aa21 --- /dev/null +++ b/cla-backend-go/swagger/common/project-compat.yaml @@ -0,0 +1,128 @@ +# Copyright The Linux Foundation and each contributor to CommunityBridge. +# SPDX-License-Identifier: MIT + +type: object +x-nullable: false +title: Project Model in Py V2 format +description: Project Model - in Py V2 - minimal fields needed by FE +properties: + project_id: + description: Project's UUID + example: '88ee12de-122b-4c46-9046-19422054ed8d' + type: string + x-omitempty: false + project_name: + description: Project name + example: 'Cloud Native Computing Foundation' + type: string + x-omitempty: false + foundation_sfid: + description: The salesforce foundation ID + example: 'a09410000182dD2AAI' + type: string + x-omitempty: false + project_ccla_enabled: + description: Is CCLA enabled? + example: true + type: boolean + x-omitempty: false + project_icla_enabled: + description: Is ICLA enabled? + example: true + type: boolean + x-omitempty: false + project_ccla_requires_icla_signature: + description: CCLA requires ICLA signature? + example: true + type: boolean + x-omitempty: false + signed_at_foundation_level: + description: Is signed at the foundation level? + example: true + type: boolean + x-omitempty: false + project_individual_documents: + type: array + items: + type: object + properties: + document_major_version: + description: Document major version + example: '2' + type: string + x-omitempty: false + document_minor_version: + description: Document minor version + example: '0' + type: string + x-omitempty: false + project_corporate_documents: + type: array + items: + type: object + properties: + document_major_version: + description: Document major version + example: '2' + type: string + x-omitempty: false + document_minor_version: + description: Document minor version + example: '0' + type: string + x-omitempty: false + projects: + type: array + items: + type: object + properties: + cla_group_id: + description: Project's UUID + example: '88ee12de-122b-4c46-9046-19422054ed8d' + type: string + x-omitempty: false + foundation_sfid: + description: The salesforce foundation ID + example: 'a09410000182dD2AAI' + type: string + x-omitempty: false + project_sfid: + description: The salesforce project ID + example: 'a09410000182dD2AAI' + type: string + x-omitempty: false + project_name: + description: Project name + example: 'Kubernetes' + type: string + x-omitempty: false + github_repos: + type: array + items: + type: object + properties: + repository_name: + description: Repository name + example: 'cncf/devstats' + type: string + x-omitempty: false + gitlab_repos: + type: array + items: + type: object + properties: + repository_name: + description: Repository name + example: 'cncf/devstats' + type: string + x-omitempty: false + gerrit_repos: + type: array + items: + type: object + properties: + gerrit_url: + description: Repository URL + example: 'cncf/devstats' + type: string + x-omitempty: false diff --git a/cla-backend-go/v2/project/handlers.go b/cla-backend-go/v2/project/handlers.go index e248ca195..ce5ad3164 100644 --- a/cla-backend-go/v2/project/handlers.go +++ b/cla-backend-go/v2/project/handlers.go @@ -6,8 +6,12 @@ package project import ( "context" "fmt" + "sort" + "github.com/linuxfoundation/easycla/cla-backend-go/gerrits" v1Project "github.com/linuxfoundation/easycla/cla-backend-go/project/service" + "github.com/linuxfoundation/easycla/cla-backend-go/projects_cla_groups" + v2Repositories "github.com/linuxfoundation/easycla/cla-backend-go/v2/repositories" projectService "github.com/linuxfoundation/easycla/cla-backend-go/v2/project-service" v2ProjectServiceModels "github.com/linuxfoundation/easycla/cla-backend-go/v2/project-service/models" @@ -22,6 +26,7 @@ import ( "github.com/go-openapi/runtime/middleware" "github.com/linuxfoundation/easycla/cla-backend-go/events" + v1Models "github.com/linuxfoundation/easycla/cla-backend-go/gen/v1/models" v1ProjectOps "github.com/linuxfoundation/easycla/cla-backend-go/gen/v1/restapi/operations/project" "github.com/linuxfoundation/easycla/cla-backend-go/gen/v2/models" "github.com/linuxfoundation/easycla/cla-backend-go/gen/v2/restapi/operations" @@ -30,7 +35,9 @@ import ( ) // Configure establishes the middleware handlers for the project service -func Configure(api *operations.EasyclaAPI, service v1Project.Service, v2Service Service, eventsService events.Service) { //nolint +func Configure(api *operations.EasyclaAPI, service v1Project.Service, v2Service Service, eventsService events.Service, projectsClaGroupsService projects_cla_groups.Service, v2RepositoriesService v2Repositories.ServiceInterface, gerritService gerrits.Service) { //nolint + + const projectDoesNotExist = "project does not exist" // Get Projects api.ProjectGetProjectsHandler = project.GetProjectsHandlerFunc(func(params project.GetProjectsParams, authUser *auth.User) middleware.Responder { reqID := utils.GetRequestID(params.XREQUESTID) @@ -74,7 +81,7 @@ func Configure(api *operations.EasyclaAPI, service v1Project.Service, v2Service claGroupModel, err := service.GetCLAGroupByID(ctx, params.ProjectSfdcID) if err != nil { - if err.Error() == "project does not exist" { + if err.Error() == projectDoesNotExist { return project.NewGetProjectByIDNotFound().WithXRequestID(reqID).WithPayload(errorResponse(reqID, err)) } return project.NewGetProjectByIDBadRequest().WithXRequestID(reqID).WithPayload(errorResponse(reqID, err)) @@ -332,6 +339,127 @@ func Configure(api *operations.EasyclaAPI, service v1Project.Service, v2Service summary := buildSFProjectSummary(sfProject, parentName) return project.NewGetSFProjectInfoByIDOK().WithXRequestID(reqID).WithPayload(summary) }) + + api.ProjectGetProjectCompatHandler = project.GetProjectCompatHandlerFunc(func(params project.GetProjectCompatParams) middleware.Responder { + reqID := utils.GetRequestID(params.XREQUESTID) + ctx := context.WithValue(context.Background(), utils.XREQUESTID, reqID) // nolint + f := logrus.Fields{ + "functionName": "v2.project.handlers.ProjectGetProjectCompatHandler", + utils.XREQUESTID: ctx.Value(utils.XREQUESTID), + "projectID": params.ProjectID, + } + + proj, err := service.GetCLAGroupByIDCompat(ctx, params.ProjectID) + if err != nil { + if err.Error() == projectDoesNotExist { + return project.NewGetProjectCompatNotFound().WithXRequestID(reqID).WithPayload(errorResponse(reqID, err)) + } + log.WithFields(f).WithError(err).Warnf("unable to load compat project by ID: %s: %+v", params.ProjectID, err) + return project.NewGetProjectCompatBadRequest().WithXRequestID(reqID).WithPayload(errorResponse(reqID, err)) + } + if proj == nil { + return project.NewGetProjectCompatNotFound().WithXRequestID(reqID) + } + projectsClaGroups, err := projectsClaGroupsService.GetProjectsIdsForClaGroup(ctx, params.ProjectID) + if err != nil { + return project.NewGetProjectCompatBadRequest().WithXRequestID(reqID).WithPayload(errorResponse(reqID, err)) + } + sfidReposMap := make(map[string][][2]string) + for _, prjClaGrp := range projectsClaGroups { + sfid := prjClaGrp.ProjectSFID + if sfid == "" { + continue + } + _, ok := sfidReposMap[sfid] + if ok { + continue + } + repos, reposErr := v2RepositoriesService.GetRepositoriesByProjectSFID(ctx, sfid) + if reposErr != nil { + log.WithFields(f).WithError(reposErr).Warnf("unable to get github/gitlab repos list for SFID: %s: %+v", sfid, reposErr) + } + gerrits, gerritsErr := gerritService.GetGerritsByProjectSFID(ctx, sfid) + if gerritsErr != nil { + log.WithFields(f).WithError(gerritsErr).Warnf("unable to get gerrit repos list for SFID: %s: %+v", sfid, gerritsErr) + } + entry := [][2]string{} + if reposErr == nil { + for _, repo := range repos { + entry = append(entry, [2]string{repo.RepositoryType, repo.RepositoryName}) + } + } + if gerritsErr == nil { + for _, repo := range gerrits.List { + entry = append(entry, [2]string{"gerrit", string(repo.GerritURL)}) + } + } + sort.Slice(entry, func(i, j int) bool { + return entry[i][1] < entry[j][1] + }) + sfidReposMap[sfid] = entry + } + compatProject := buildCompatProject(proj, projectsClaGroups, sfidReposMap) + return project.NewGetProjectCompatOK().WithXRequestID(reqID).WithPayload(compatProject) + }) +} + +func buildCompatProject(project *v1Models.ClaGroup, projectClaGroups []*projects_cla_groups.ProjectClaGroup, sfidReposMap map[string][][2]string) *models.ProjectCompat { + projectCorporateDocuments := []*models.ProjectCompatProjectCorporateDocumentsItems0{} + for _, doc := range project.ProjectCorporateDocuments { + projectCorporateDocuments = append(projectCorporateDocuments, &models.ProjectCompatProjectCorporateDocumentsItems0{ + DocumentMajorVersion: doc.DocumentMajorVersion, + DocumentMinorVersion: doc.DocumentMinorVersion, + }) + } + projectIndividualDocuments := []*models.ProjectCompatProjectIndividualDocumentsItems0{} + for _, doc := range project.ProjectIndividualDocuments { + projectIndividualDocuments = append(projectIndividualDocuments, &models.ProjectCompatProjectIndividualDocumentsItems0{ + DocumentMajorVersion: doc.DocumentMajorVersion, + DocumentMinorVersion: doc.DocumentMinorVersion, + }) + } + projects := []*models.ProjectCompatProjectsItems0{} + for _, prjClaGrp := range projectClaGroups { + gerritRepos := []*models.ProjectCompatProjectsItems0GerritReposItems0{} + githubRepos := []*models.ProjectCompatProjectsItems0GithubReposItems0{} + gitlabRepos := []*models.ProjectCompatProjectsItems0GitlabReposItems0{} + sfid := prjClaGrp.ProjectSFID + if sfid != "" { + repos, ok := sfidReposMap[sfid] + if ok { + for _, repo := range repos { + if repo[0] == "github" { + githubRepos = append(githubRepos, &models.ProjectCompatProjectsItems0GithubReposItems0{RepositoryName: repo[1]}) + } else if repo[0] == "gitlab" { + gitlabRepos = append(gitlabRepos, &models.ProjectCompatProjectsItems0GitlabReposItems0{RepositoryName: repo[1]}) + } else if repo[0] == "gerrit" { + gerritRepos = append(gerritRepos, &models.ProjectCompatProjectsItems0GerritReposItems0{GerritURL: repo[1]}) + } + } + } + } + projects = append(projects, &models.ProjectCompatProjectsItems0{ + ClaGroupID: prjClaGrp.ClaGroupID, + FoundationSfid: prjClaGrp.FoundationSFID, + ProjectName: prjClaGrp.ProjectName, + ProjectSfid: prjClaGrp.ProjectSFID, + GerritRepos: gerritRepos, + GithubRepos: githubRepos, + GitlabRepos: gitlabRepos, + }) + } + return &models.ProjectCompat{ + FoundationSfid: project.FoundationSFID, + ProjectName: project.ProjectName, + ProjectCclaEnabled: project.ProjectCCLAEnabled, + ProjectCclaRequiresIclaSignature: project.ProjectCCLARequiresICLA, + ProjectIclaEnabled: project.ProjectICLAEnabled, + ProjectID: project.ProjectID, + SignedAtFoundationLevel: project.FoundationLevelCLA, + ProjectCorporateDocuments: projectCorporateDocuments, + ProjectIndividualDocuments: projectIndividualDocuments, + Projects: projects, + } } func buildSFProjectSummary(sfProject *v2ProjectServiceModels.ProjectOutputDetailed, parentName string) *models.SfProjectSummary { diff --git a/cla-backend-go/v2/repositories/repository.go b/cla-backend-go/v2/repositories/repository.go index 7496ed75b..c31a94858 100644 --- a/cla-backend-go/v2/repositories/repository.go +++ b/cla-backend-go/v2/repositories/repository.go @@ -6,6 +6,7 @@ package repositories import ( "context" "fmt" + "reflect" "strconv" "strings" @@ -28,6 +29,7 @@ type RepositoryInterface interface { GitHubGetRepositoriesByCLAGroupDisabled(ctx context.Context, claGroupID string) ([]*repoModels.RepositoryDBModel, error) GitHubGetRepositoriesByProjectSFID(ctx context.Context, projectSFID string) ([]*repoModels.RepositoryDBModel, error) GitHubGetRepositoriesByOrganizationName(ctx context.Context, orgName string) ([]*repoModels.RepositoryDBModel, error) + GetRepositoriesByProjectSFID(ctx context.Context, projectSFID string) ([]*repoModels.RepositoryDBModel, error) GitLabGetRepository(ctx context.Context, repositoryID string) (*repoModels.RepositoryDBModel, error) GitLabGetRepositoryByName(ctx context.Context, repositoryName string) (*repoModels.RepositoryDBModel, error) @@ -244,6 +246,16 @@ func (r *Repository) GitHubGetRepositoriesByCLAGroupDisabled(ctx context.Context return records, nil } +// GetRepositoriesByProjectSFID returns a list of repositories associated with the specified project +func (r *Repository) GetRepositoriesByProjectSFID(ctx context.Context, projectSFID string) ([]*repoModels.RepositoryDBModel, error) { + condition := expression.Key(repoModels.RepositoryProjectIDColumn).Equal(expression.Value(projectSFID)) + records, err := r.getRepositoriesWithConditionFilter(ctx, condition, expression.ConditionBuilder{}, repoModels.RepositoryProjectSFIDIndex) + if err != nil { + return nil, err + } + return records, nil +} + // GitHubGetRepositoriesByProjectSFID returns a list of repositories associated with the specified project func (r *Repository) GitHubGetRepositoriesByProjectSFID(ctx context.Context, projectSFID string) ([]*repoModels.RepositoryDBModel, error) { condition := expression.Key(repoModels.RepositoryProjectIDColumn).Equal(expression.Value(projectSFID)) @@ -548,6 +560,10 @@ func (r *Repository) getRepositoryWithConditionFilter(ctx context.Context, condi return repositories[0], nil } +func isZeroCondition(filter expression.ConditionBuilder) bool { + return reflect.ValueOf(filter).IsZero() +} + // getRepositoriesWithConditionFilter fetches the repository entry based on the specified condition and filter criteria // using the provided index func (r *Repository) getRepositoriesWithConditionFilter(ctx context.Context, condition expression.KeyConditionBuilder, filter expression.ConditionBuilder, indexName string) ([]*repoModels.RepositoryDBModel, error) { @@ -557,7 +573,11 @@ func (r *Repository) getRepositoriesWithConditionFilter(ctx context.Context, con "indexName": indexName, } - expr, err := expression.NewBuilder().WithKeyCondition(condition).WithFilter(filter).Build() + builder := expression.NewBuilder().WithKeyCondition(condition) + if !isZeroCondition(filter) { + builder = builder.WithFilter(filter) + } + expr, err := builder.Build() if err != nil { log.WithFields(f).WithError(err).Warn("problem creating builder") return nil, err diff --git a/cla-backend-go/v2/repositories/service.go b/cla-backend-go/v2/repositories/service.go index 7615e4682..85c19895a 100644 --- a/cla-backend-go/v2/repositories/service.go +++ b/cla-backend-go/v2/repositories/service.go @@ -69,6 +69,9 @@ type ServiceInterface interface { GitLabEnrollCLAGroupRepositories(ctx context.Context, claGroupID string, enrollValue bool) error GitLabDeleteRepositories(ctx context.Context, gitLabGroupPath string) error GitLabDeleteRepositoryByExternalID(ctx context.Context, gitLabExternalID int64) error + + // All + GetRepositoriesByProjectSFID(ctx context.Context, projectSFID string) ([]*v1Repositories.RepositoryDBModel, error) } // GitLabOrgRepo redefine the interface here to avoid circular dependency issues @@ -664,3 +667,12 @@ func (s *Service) GitHubDisableCLAGroupRepositories(ctx context.Context, claGrou } return nil } + +// GetRepositoriesByProjectSFID service function +func (s *Service) GetRepositoriesByProjectSFID(ctx context.Context, projectSFID string) ([]*v1Repositories.RepositoryDBModel, error) { + data, err := s.gitV2Repository.GetRepositoriesByProjectSFID(ctx, projectSFID) + if err != nil { + return nil, err + } + return data, nil +} diff --git a/docs/Python_APIs.md b/docs/Python_APIs.md index 1e1d30328..b075c33f4 100644 --- a/docs/Python_APIs.md +++ b/docs/Python_APIs.md @@ -13,7 +13,6 @@ - `/v2/check-prepare-employee-signature`. - `/v2/request-employee-signature`. - `/v2/user//project//last-signature`. -- `/v2/project/`. 3. EasyCLA corporate console [here](https://github.com/LF-Engineering/lfx-corp-cla-console/blob/main/backend/src/data/cla-api.ts): diff --git a/tests/py2go/Makefile b/tests/py2go/Makefile new file mode 100644 index 000000000..4841a492e --- /dev/null +++ b/tests/py2go/Makefile @@ -0,0 +1,5 @@ +.phony: fmt test +test: fmt + go test -v +fmt: + go fmt *.go diff --git a/tests/py2go/README.md b/tests/py2go/README.md new file mode 100644 index 000000000..6c5b0e6b2 --- /dev/null +++ b/tests/py2go/README.md @@ -0,0 +1,23 @@ +# Testing porting APIs from Python to golang + +1) Start `python` API backend: +- `` source setenv.sh; cd cla-backend; source .venv/bin/activate; yarn serve:ext ``. + +2) Start `golang` API backend: +- `` source setenv.sh; cd cla-backend-go; make swagger; make build-linux ``. +- `` PORT=5001 AUTH0_USERNAME_CLAIM_CLI='http://lfx.dev/claims/username' AUTH0_EMAIL_CLAIM_CLI='http://lfx.dev/claims/email' AUTH0_NAME_CLAIM_CLI='http://lfx.dev/claims/username' ./bin/cla ``. +- Or: `` ../utils/run_go_api_server.sh ``. + +3) Get `auth0` token from browser session (login using `LFID`): +- `` ./get_oauth_token.sh ``. Copy the token value. + +3) Exacute API tests: +- `` export TOKEN='' ``. +- `` export XACL="$(cat ../../x-acl.secret)" ``. +- `` make ``. +- `` DEBUG=1 PROJECT_UUID=88ee12de-122b-4c46-9046-19422054ed8d PY_API_URL=https://api.lfcla.dev.platform.linuxfoundation.org GO_API_URL=https://api-gw.dev.platform.linuxfoundation.org/cla-service make ``. +- `` MAX_PARALLEL=8 PY_API_URL=https://api.lfcla.dev.platform.linuxfoundation.org go test -v -run '^TestAllProjectsCompatAPI$' ``. +- To run a specific test case(s): `` DEBUG=1 PROJECT_UUID=88ee12de-122b-4c46-9046-19422054ed8d PY_API_URL=https://api.lfcla.dev.platform.linuxfoundation.org go test -v -run '^TestProjectCompatAPI$' ``. +- Manually via `cURL`: `` curl -s -XGET http://127.0.0.1:5001/v4/project-compat/01af041c-fa69-4052-a23c-fb8c1d3bef24 | jq . ``. +- To manually see given project values if APIs differ (to dewbug): `` aws --region us-east-1 --profile lfproduct-dev dynamodb get-item --table-name cla-dev-projects --key '{"project_id": {"S": "4a855799-0aea-4e01-98b7-ef3da09df478"}}' | jq '.Item' ``. +- And `` aws --region us-east-1 --profile lfproduct-dev dynamodb query --table-name cla-dev-projects-cla-groups --index-name cla-group-id-index --key-condition-expression "cla_group_id = :project_id" --expression-attribute-values '{":project_id":{"S":"4a855799-0aea-4e01-98b7-ef3da09df478"}}' | jq '.Items' ``. diff --git a/tests/py2go/api_test.go b/tests/py2go/api_test.go new file mode 100644 index 000000000..bde05c18d --- /dev/null +++ b/tests/py2go/api_test.go @@ -0,0 +1,486 @@ +package main + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "sort" + "strconv" + "strings" + "sync" + "testing" + "time" + + "github.com/google/uuid" + "github.com/stretchr/testify/assert" +) + +var ( + TOKEN string + XACL string + PY_API_URL string + GO_API_URL string + DEBUG bool + MAX_PARALLEL int + PROJECT_UUID string + ProjectAPIPath = [3]string{"/v2/project/%s", "/v4/project/%s", "/v4/project-compat/%s"} + ProjectAPIKeyMapping = map[string]string{ + "date_created": "dateCreated", + "date_modified": "dateModified", + "project_name": "projectName", + "foundation_sfid": "foundationSFID", + "project_acl": "projectACL", + "project_ccla_enabled": "projectCCLAEnabled", + "project_ccla_requires_icla_signature": "projectCCLARequiresICLA", + "project_icla_enabled": "projectICLAEnabled", + "project_id": "projectID", + "project_live": "projectLive", + "version": "version", + // "project_individual_documents": "projectIndividualDocuments", + // "project_corporate_documents": "projectCorporateDocuments", + // "project_member_documents": "projectMemberDocuments", + // "signed_at_foundation_level": "foundationLevelCLA", + // "root_project_repositories_count": "rootProjectRepositoriesCount", + } + ProjectCompatAPIKeyMapping = map[string]interface{}{ + "foundation_sfid": nil, + "project_ccla_enabled": nil, + "project_ccla_requires_icla_signature": nil, + "project_icla_enabled": nil, + "project_id": nil, + "project_name": nil, + "project_individual_documents": []interface{}{map[string]interface{}{ + "document_major_version": nil, + "document_minor_version": nil, + }}, + "project_corporate_documents": []interface{}{map[string]interface{}{ + "document_major_version": nil, + "document_minor_version": nil, + }}, + "projects": []interface{}{map[string]interface{}{ + "cla_group_id": nil, + "foundation_sfid": nil, + "project_name": nil, + "project_sfid": nil, + "github_repos": []interface{}{map[string]interface{}{"repository_name": nil}}, + "gitlab_repos": []interface{}{map[string]interface{}{"repository_name": nil}}, + "gerrit_repos": []interface{}{map[string]interface{}{"gerrit_url": nil}}, + }}, + // "signed_at_foundation_level": nil, + } + ProjectCompatAPISortMap = map[string]string{ + "github_repos": "repository_name", + "gitlab_repos": "repository_name", + "gerrit_repos": "gerrit_url", + } +) + +func init() { + TOKEN = os.Getenv("TOKEN") + XACL = os.Getenv("XACL") + PY_API_URL = os.Getenv("PY_API_URL") + if PY_API_URL == "" { + PY_API_URL = "http://127.0.0.1:5000" + } + GO_API_URL = os.Getenv("GO_API_URL") + if GO_API_URL == "" { + GO_API_URL = "http://127.0.0.1:5001" + } + dbg := os.Getenv("DEBUG") + if dbg != "" { + DEBUG = true + } + MAX_PARALLEL = 1 + par := os.Getenv("MAX_PARALLEL") + if par != "" { + iPar, err := strconv.Atoi(par) + if err != nil { + fmt.Printf("MAX_PARALLEL environment value should be integer >= 1\n") + } else if iPar > 0 { + MAX_PARALLEL = iPar + } + } + PROJECT_UUID = os.Getenv("PROJECT_UUID") +} + +func tryParseTime(val interface{}) (time.Time, bool) { + str, ok := val.(string) + if !ok { + return time.Time{}, false + } + layouts := []string{ + time.RFC3339, + "2006-01-02T15:04:05.000000Z0700", + "2006-01-02T15:04:05Z07:00", + "2006-01-02T15:04:05", + "2006-01-02 15:04:05", + } + for _, layout := range layouts { + if t, err := time.Parse(layout, str); err == nil { + return t.UTC(), true + } + } + return time.Time{}, false +} + +func compareMappedFields(t *testing.T, pyData, goData map[string]interface{}, keyMapping map[string]string) { + for pyKey, goKey := range keyMapping { + Debugf("checking %s - %s\n", pyKey, goKey) + + pyVal, pyOk := pyData[pyKey] + goVal, goOk := goData[goKey] + + if !pyOk { + t.Errorf("Missing key in Python response: %s", pyKey) + continue + } + if !goOk { + t.Errorf("Missing key in Go response: %s", goKey) + continue + } + + pyTime, okPyTime := tryParseTime(pyVal) + goTime, okGoTime := tryParseTime(goVal) + + if okPyTime && okGoTime { + if !pyTime.Equal(goTime) { + t.Errorf("Datetime mismatch for key '%s' (Go: '%s'): py:%s != go:%s", pyKey, goKey, pyTime, goTime) + } + continue + } + + if (pyVal == nil && goVal == "") || (goVal == nil && pyVal == "") { + continue + } + + if fmt.Sprint(pyVal) != fmt.Sprint(goVal) { + t.Errorf("Mismatch for key '%s' (Go: '%s'): py:%+v != go:%+v", pyKey, goKey, pyVal, goVal) + } + } +} + +func sortByKey(arr []interface{}, key string) { + sort.Slice(arr, func(i, j int) bool { + m1, _ := arr[i].(map[string]interface{}) + m2, _ := arr[j].(map[string]interface{}) + s1 := fmt.Sprint(m1[key]) + s2 := fmt.Sprint(m2[key]) + return s1 < s2 + }) +} + +func compareNestedFields(t *testing.T, pyData, goData, keyMapping map[string]interface{}, sortMap map[string]string) { + for k, v := range keyMapping { + if v == nil { + Debugf("checking values of '%s'\n", k) + } + + pyVal, pyOk := pyData[k] + goVal, goOk := goData[k] + if !pyOk { + t.Errorf("Missing key in Python response: %s", k) + continue + } + if !goOk { + t.Errorf("Missing key in Go response: %s", k) + continue + } + + nestedMapping, nested := v.(map[string]interface{}) + if nested { + Debugf("checking nested object '%s'\n", k) + pyNestedVal, pyOk := pyVal.(map[string]interface{}) + goNestedVal, goOk := goVal.(map[string]interface{}) + if !pyOk { + t.Errorf("%s value in Python response is not a nested object: %+v", k, pyVal) + continue + } + if !goOk { + t.Errorf("%s value in Go response is not a nested object: %+v", k, goVal) + continue + } + compareNestedFields(t, pyNestedVal, goNestedVal, nestedMapping, sortMap) + continue + } + + arrayMapping, array := v.([]interface{}) + if array { + Debugf("checking nested array '%s'\n", k) + if len(arrayMapping) < 1 { + t.Errorf("%s value in key mapping should be array of single object: %+v", k, v) + continue + } + nestedMapping, nested := arrayMapping[0].(map[string]interface{}) + if !nested { + t.Errorf("%s value in key mapping should be array of single object: %+v", k, v) + continue + } + pyArrayVal, pyOk := pyVal.([]interface{}) + goArrayVal, goOk := goVal.([]interface{}) + if !pyOk { + t.Errorf("%s value in Python response is not an array: %+v", k, pyVal) + continue + } + if !goOk { + t.Errorf("%s value in Go response is not an array: %+v", k, goVal) + continue + } + lenPyArrayVal := len(pyArrayVal) + lenGoArrayVal := len(goArrayVal) + if lenPyArrayVal != lenGoArrayVal { + t.Errorf("%s arrays length mismatch: %d != %d", k, lenPyArrayVal, lenGoArrayVal) + continue + } + sortKey, needSort := sortMap[k] + if needSort { + Debugf("sorting '%s' key values by %s\n", k, sortKey) + sortByKey(pyArrayVal, sortKey) + sortByKey(goArrayVal, sortKey) + } + for idx := range pyArrayVal { + pyNestedVal, pyOk := pyArrayVal[idx].(map[string]interface{}) + goNestedVal, goOk := goArrayVal[idx].(map[string]interface{}) + if !pyOk { + t.Errorf("%s:%d value in Python response is not a nested object: %+v", k, idx, pyArrayVal[idx]) + continue + } + if !goOk { + t.Errorf("%s:%d value in Go response is not a nested object: %+v", k, idx, goArrayVal[idx]) + continue + } + compareNestedFields(t, pyNestedVal, goNestedVal, nestedMapping, sortMap) + } + continue + } + + pyTime, okPyTime := tryParseTime(pyVal) + goTime, okGoTime := tryParseTime(goVal) + + if okPyTime && okGoTime { + if !pyTime.Equal(goTime) { + t.Errorf("Datetime mismatch for key '%s': py:%s != go:%s", k, pyTime, goTime) + } + continue + } + + if (pyVal == nil && goVal == "") || (goVal == nil && pyVal == "") { + continue + } + + if fmt.Sprint(pyVal) != fmt.Sprint(goVal) { + t.Errorf("Mismatch for key '%s': py:%+v != go:%+v", k, pyVal, goVal) + } + } +} + +func runProjectCompatAPIForProject(t *testing.T, projectId string) { + apiURL := PY_API_URL + fmt.Sprintf(ProjectAPIPath[0], projectId) + Debugf("Py API call: %s\n", apiURL) + oldResp, err := http.Get(apiURL) + if err != nil { + t.Fatalf("Failed to call API: %v", err) + } + assert.Equal(t, http.StatusOK, oldResp.StatusCode, "Expected 200 from PY API") + defer oldResp.Body.Close() + oldBody, _ := io.ReadAll(oldResp.Body) + var oldJSON interface{} + err = json.Unmarshal(oldBody, &oldJSON) + assert.NoError(t, err) + Debugf("Py raw response: %+v\n", string(oldBody)) + Debugf("Py response: %+v\n", oldJSON) + + apiURL = GO_API_URL + fmt.Sprintf(ProjectAPIPath[2], projectId) + Debugf("Go API call: %s\n", apiURL) + newResp, err := http.Get(apiURL) + if err != nil { + t.Fatalf("Failed to call API: %v", err) + } + assert.Equal(t, http.StatusOK, newResp.StatusCode, "Expected 200 from GO API") + defer newResp.Body.Close() + newBody, _ := io.ReadAll(newResp.Body) + var newJSON interface{} + err = json.Unmarshal(newBody, &newJSON) + assert.NoError(t, err) + Debugf("Go raw Response: %+v\n", string(newBody)) + Debugf("Go response: %+v\n", newJSON) + + oldMap, ok1 := oldJSON.(map[string]interface{}) + newMap, ok2 := newJSON.(map[string]interface{}) + + if !ok1 || !ok2 { + t.Fatalf("Expected both responses to be JSON objects") + } + compareNestedFields(t, oldMap, newMap, ProjectCompatAPIKeyMapping, ProjectCompatAPISortMap) + + if DEBUG { + oky := []string{} + for k, _ := range oldMap { + oky = append(oky, k) + } + sort.Strings(oky) + nky := []string{} + for k, _ := range newMap { + nky = append(nky, k) + } + sort.Strings(nky) + Debugf("old keys: %+v\n", oky) + Debugf("new keys: %+v\n", nky) + } +} + +func TestProjectCompatAPI(t *testing.T) { + projectId := PROJECT_UUID + if projectId == "" { + projectId = uuid.New().String() + putTestItem("projects", "project_id", projectId, "S", map[string]interface{}{ + "project_name": "CNCF", + "project_icla_enabled": true, + "project_ccla_enabled": true, + "project_ccla_requires_icla_signature": true, + "date_created": "2022-11-21T10:31:31Z", + "date_modified": "2023-02-23T13:14:48Z", + "foundation_sfid": "a09410000182dD2AAI", + "version": "2", + }, DEBUG) + defer deleteTestItem("projects", "project_id", projectId, "S", DEBUG) + } + + runProjectCompatAPIForProject(t, projectId) +} + +func TestAllProjectsCompatAPI(t *testing.T) { + allProjects := getAllPrimaryKeys("projects", "project_id", "S") + + var failedProjects []string + var mtx sync.Mutex + sem := make(chan struct{}, MAX_PARALLEL) + var wg sync.WaitGroup + + for _, projectID := range allProjects { + projID, ok := projectID.(string) + if !ok { + t.Errorf("Expected string project_id, got: %T", projectID) + continue + } + + wg.Add(1) + sem <- struct{}{} + + go func(projID string) { + defer wg.Done() + defer func() { <-sem }() + + // Use t.Run in a thread-safe wrapper with a dummy parent test + t.Run(fmt.Sprintf("ProjectId=%s", projID), func(t *testing.T) { + runProjectCompatAPIForProject(t, projID) + if t.Failed() { + mtx.Lock() + failedProjects = append(failedProjects, projID) + mtx.Unlock() + } + }) + }(projID) + } + + wg.Wait() + + if len(failedProjects) > 0 { + fmt.Fprintf(os.Stderr, "\nFailed Project IDs (%d):\n%s\n\n", + len(failedProjects), + strings.Join(failedProjects, "\n"), + ) + t.Fail() // Mark test as failed + } else { + fmt.Println("\nAll projects passed.") + } +} + +func TestProjectAPI(t *testing.T) { + if TOKEN == "" || XACL == "" { + t.Fatalf("TOKEN and XACL environment variables must be set") + } + projectId := PROJECT_UUID + if projectId == "" { + projectId = uuid.New().String() + putTestItem("projects", "project_id", projectId, "S", map[string]interface{}{ + "project_name": "CNCF", + "project_icla_enabled": true, + "project_ccla_enabled": true, + "project_ccla_requires_icla_signature": true, + "date_created": "2022-11-21T10:31:31Z", + "date_modified": "2023-02-23T13:14:48Z", + "foundation_sfid": "a09410000182dD2AAI", + "version": "2", + }, DEBUG) + defer deleteTestItem("projects", "project_id", projectId, "S", DEBUG) + } + + apiURL := PY_API_URL + fmt.Sprintf(ProjectAPIPath[0], projectId) + Debugf("Py API call: %s\n", apiURL) + oldResp, err := http.Get(apiURL) + if err != nil { + t.Fatalf("Failed to call API: %v", err) + } + assert.Equal(t, http.StatusOK, oldResp.StatusCode, "Expected 200 from PY API") + defer oldResp.Body.Close() + oldBody, _ := io.ReadAll(oldResp.Body) + var oldJSON interface{} + err = json.Unmarshal(oldBody, &oldJSON) + assert.NoError(t, err) + Debugf("Py raw response: %+v\n", string(oldBody)) + Debugf("Py response: %+v\n", oldJSON) + + apiURL = GO_API_URL + fmt.Sprintf(ProjectAPIPath[1], projectId) + Debugf("Go API call: %s\n", apiURL) + // newResp, err := http.Get(apiURL) + req, err := http.NewRequest("GET", apiURL, nil) + if err != nil { + t.Fatalf("Failed to create request: %v", err) + } + + req.Header.Set("Authorization", "Bearer "+TOKEN) + req.Header.Set("X-ACL", XACL) + + newResp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("Failed to call API: %v", err) + } + assert.Equal(t, http.StatusOK, newResp.StatusCode, "Expected 200 from GO API") + defer newResp.Body.Close() + newBody, _ := io.ReadAll(newResp.Body) + var newJSON interface{} + err = json.Unmarshal(newBody, &newJSON) + assert.NoError(t, err) + Debugf("Go raw Response: %+v\n", string(newBody)) + Debugf("Go response: %+v\n", newJSON) + + // For full equality + // Strict + // assert.Equal(t, oldJSON, newJSON) + // Smart - ignore keys order + // assert.JSONEq(t, string(oldBody), string(newBody)) + oldMap, ok1 := oldJSON.(map[string]interface{}) + newMap, ok2 := newJSON.(map[string]interface{}) + + if !ok1 || !ok2 { + t.Fatalf("Expected both responses to be JSON objects") + } + compareMappedFields(t, oldMap, newMap, ProjectAPIKeyMapping) + + if DEBUG { + oky := []string{} + for k, _ := range oldMap { + oky = append(oky, k) + } + sort.Strings(oky) + nky := []string{} + for k, _ := range newMap { + nky = append(nky, k) + } + sort.Strings(nky) + Debugf("old keys: %+v\n", oky) + Debugf("new keys: %+v\n", nky) + } +} diff --git a/tests/py2go/dynamo.go b/tests/py2go/dynamo.go new file mode 100644 index 000000000..c4bc8c86b --- /dev/null +++ b/tests/py2go/dynamo.go @@ -0,0 +1,188 @@ +package main + +import ( + "context" + "fmt" + "log" + "os" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/dynamodb" + "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" +) + +var ( + STAGE string + PROFILE string + REGION string +) + +func init() { + REGION = os.Getenv("AWS_REGION") + if REGION == "" { + REGION = "us-east-1" + } + STAGE = os.Getenv("STAGE") + if STAGE == "" { + STAGE = "dev" + } + PROFILE = os.Getenv("AWS_PROFILE") + if PROFILE == "" { + PROFILE = "lfproduct-" + STAGE + } +} + +func putTestItem(tableName, keyName string, keyValue interface{}, keyType string, extraFields map[string]interface{}, dbg bool) { + cfg, err := config.LoadDefaultConfig( + context.TODO(), + config.WithRegion(REGION), + config.WithSharedConfigProfile(PROFILE), + ) + if err != nil { + log.Fatalf("unable to load SDK config, %v", err) + } + client := dynamodb.NewFromConfig(cfg) + + item := make(map[string]types.AttributeValue) + + switch keyType { + case "S": + item[keyName] = &types.AttributeValueMemberS{Value: fmt.Sprint(keyValue)} + case "N": + item[keyName] = &types.AttributeValueMemberN{Value: fmt.Sprint(keyValue)} + default: + log.Fatalf("Unsupported key type: %s", keyType) + } + + for k, v := range extraFields { + switch val := v.(type) { + case string: + item[k] = &types.AttributeValueMemberS{Value: val} + case int, int64, float64: + item[k] = &types.AttributeValueMemberN{Value: fmt.Sprint(val)} + case bool: + item[k] = &types.AttributeValueMemberBOOL{Value: val} + case []string: + item[k] = &types.AttributeValueMemberSS{Value: val} + case []interface{}: + Debugf("Skipping field %s: generic list not supported directly", k) + default: + Debugf("Unsupported type for field %s: %T", k, v) + } + } + + tName := "cla-" + STAGE + "-" + tableName + _, err = client.PutItem(context.TODO(), &dynamodb.PutItemInput{ + TableName: aws.String(tName), + Item: item, + }) + if err != nil { + log.Fatalf("PutItem error: %v", err) + } + Debugf("created entry in %s: %s=%s, %+v\n", tName, keyName, keyValue, extraFields) +} + +func deleteTestItem(tableName, keyName string, keyValue interface{}, keyType string, dbg bool) { + cfg, err := config.LoadDefaultConfig( + context.TODO(), + config.WithRegion(REGION), + config.WithSharedConfigProfile(PROFILE), + ) + if err != nil { + log.Fatalf("unable to load SDK config, %v", err) + } + client := dynamodb.NewFromConfig(cfg) + + var key types.AttributeValue + + switch keyType { + case "S": + key = &types.AttributeValueMemberS{Value: fmt.Sprint(keyValue)} + case "N": + key = &types.AttributeValueMemberN{Value: fmt.Sprint(keyValue)} + case "BOOL": + b, ok := keyValue.(bool) + if !ok { + log.Fatalf("Key value must be boolean for BOOL type") + } + key = &types.AttributeValueMemberBOOL{Value: b} + default: + log.Fatalf("Unsupported key type: %s", keyType) + } + + tName := "cla-" + STAGE + "-" + tableName + _, err = client.DeleteItem(context.TODO(), &dynamodb.DeleteItemInput{ + TableName: aws.String(tName), + Key: map[string]types.AttributeValue{ + keyName: key, + }, + }) + if err != nil { + log.Fatalf("DeleteItem error: %v", err) + } + Debugf("deleted entry in %s: %s=%s\n", tName, keyName, keyValue) +} + +func getAllPrimaryKeys(tableName, keyName, keyType string) []interface{} { + cfg, err := config.LoadDefaultConfig( + context.TODO(), + config.WithRegion(REGION), + config.WithSharedConfigProfile(PROFILE), + ) + if err != nil { + log.Fatalf("unable to load SDK config, %v", err) + } + client := dynamodb.NewFromConfig(cfg) + + tName := "cla-" + STAGE + "-" + tableName + Debugf("getting all keys form %s\n", tName) + var results []interface{} + var lastEvaluatedKey map[string]types.AttributeValue + + for { + input := &dynamodb.ScanInput{ + TableName: aws.String(tName), + ProjectionExpression: aws.String(keyName), + ExclusiveStartKey: lastEvaluatedKey, + } + + output, err := client.Scan(context.TODO(), input) + if err != nil { + log.Fatalf("Scan error on table %s: %v", tName, err) + } + + for _, item := range output.Items { + attr, ok := item[keyName] + if !ok { + Debugf("Key %s not found in item: %+v", keyName, item) + continue + } + + switch keyType { + case "S": + if v, ok := attr.(*types.AttributeValueMemberS); ok { + results = append(results, v.Value) + } + case "N": + if v, ok := attr.(*types.AttributeValueMemberN); ok { + results = append(results, v.Value) + } + case "BOOL": + if v, ok := attr.(*types.AttributeValueMemberBOOL); ok { + results = append(results, v.Value) + } + default: + log.Fatalf("Unsupported key type: %s", keyType) + } + } + + if output.LastEvaluatedKey == nil || len(output.LastEvaluatedKey) == 0 { + break + } + lastEvaluatedKey = output.LastEvaluatedKey + } + + Debugf("got keys: %+v\n", results) + return results +} diff --git a/tests/py2go/go.mod b/tests/py2go/go.mod new file mode 100644 index 000000000..9af2fb2a0 --- /dev/null +++ b/tests/py2go/go.mod @@ -0,0 +1,29 @@ +module github.com/linuxfoundation/easycla/tests/py2go + +go 1.24.4 + +require ( + github.com/aws/aws-sdk-go-v2 v1.36.5 + github.com/aws/aws-sdk-go-v2/config v1.29.17 + github.com/aws/aws-sdk-go-v2/service/dynamodb v1.44.0 + github.com/google/uuid v1.6.0 + github.com/stretchr/testify v1.10.0 +) + +require ( + github.com/aws/aws-sdk-go-v2/credentials v1.17.70 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.32 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.36 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.36 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.4 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.10.17 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.17 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.25.5 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.3 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.34.0 // indirect + github.com/aws/smithy-go v1.22.4 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/tests/py2go/go.sum b/tests/py2go/go.sum new file mode 100644 index 000000000..54cfa5aec --- /dev/null +++ b/tests/py2go/go.sum @@ -0,0 +1,42 @@ +github.com/aws/aws-sdk-go-v2 v1.36.5 h1:0OF9RiEMEdDdZEMqF9MRjevyxAQcf6gY+E7vwBILFj0= +github.com/aws/aws-sdk-go-v2 v1.36.5/go.mod h1:EYrzvCCN9CMUTa5+6lf6MM4tq3Zjp8UhSGR/cBsjai0= +github.com/aws/aws-sdk-go-v2/config v1.29.17 h1:jSuiQ5jEe4SAMH6lLRMY9OVC+TqJLP5655pBGjmnjr0= +github.com/aws/aws-sdk-go-v2/config v1.29.17/go.mod h1:9P4wwACpbeXs9Pm9w1QTh6BwWwJjwYvJ1iCt5QbCXh8= +github.com/aws/aws-sdk-go-v2/credentials v1.17.70 h1:ONnH5CM16RTXRkS8Z1qg7/s2eDOhHhaXVd72mmyv4/0= +github.com/aws/aws-sdk-go-v2/credentials v1.17.70/go.mod h1:M+lWhhmomVGgtuPOhO85u4pEa3SmssPTdcYpP/5J/xc= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.32 h1:KAXP9JSHO1vKGCr5f4O6WmlVKLFFXgWYAGoJosorxzU= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.32/go.mod h1:h4Sg6FQdexC1yYG9RDnOvLbW1a/P986++/Y/a+GyEM8= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.36 h1:SsytQyTMHMDPspp+spo7XwXTP44aJZZAC7fBV2C5+5s= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.36/go.mod h1:Q1lnJArKRXkenyog6+Y+zr7WDpk4e6XlR6gs20bbeNo= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.36 h1:i2vNHQiXUvKhs3quBR6aqlgJaiaexz/aNvdCktW/kAM= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.36/go.mod h1:UdyGa7Q91id/sdyHPwth+043HhmP6yP9MBHgbZM0xo8= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo= +github.com/aws/aws-sdk-go-v2/service/dynamodb v1.44.0 h1:A99gjqZDbdhjtjJVZrmVzVKO2+p3MSg35bDWtbMQVxw= +github.com/aws/aws-sdk-go-v2/service/dynamodb v1.44.0/go.mod h1:mWB0GE1bqcVSvpW7OtFA0sKuHk52+IqtnsYU2jUfYAs= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.4 h1:CXV68E2dNqhuynZJPB80bhPQwAKqBWVer887figW6Jc= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.4/go.mod h1:/xFi9KtvBXP97ppCz1TAEvU1Uf66qvid89rbem3wCzQ= +github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.10.17 h1:x187MqiHwBGjMGAed8Y8K1VGuCtFvQvXb24r+bwmSdo= +github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.10.17/go.mod h1:mC9qMbA6e1pwEq6X3zDGtZRXMG2YaElJkbJlMVHLs5I= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.17 h1:t0E6FzREdtCsiLIoLCWsYliNsRBgyGD/MCK571qk4MI= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.17/go.mod h1:ygpklyoaypuyDvOM5ujWGrYWpAK3h7ugnmKCU/76Ys4= +github.com/aws/aws-sdk-go-v2/service/sso v1.25.5 h1:AIRJ3lfb2w/1/8wOOSqYb9fUKGwQbtysJ2H1MofRUPg= +github.com/aws/aws-sdk-go-v2/service/sso v1.25.5/go.mod h1:b7SiVprpU+iGazDUqvRSLf5XmCdn+JtT1on7uNL6Ipc= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.3 h1:BpOxT3yhLwSJ77qIY3DoHAQjZsc4HEGfMCE4NGy3uFg= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.3/go.mod h1:vq/GQR1gOFLquZMSrxUK/cpvKCNVYibNyJ1m7JrU88E= +github.com/aws/aws-sdk-go-v2/service/sts v1.34.0 h1:NFOJ/NXEGV4Rq//71Hs1jC/NvPs1ezajK+yQmkwnPV0= +github.com/aws/aws-sdk-go-v2/service/sts v1.34.0/go.mod h1:7ph2tGpfQvwzgistp2+zga9f+bCjlQJPkPUmMgDSD7w= +github.com/aws/smithy-go v1.22.4 h1:uqXzVZNuNexwc/xrh6Tb56u89WDlJY6HS+KC0S4QSjw= +github.com/aws/smithy-go v1.22.4/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/tests/py2go/log.go b/tests/py2go/log.go new file mode 100644 index 000000000..a39fe0267 --- /dev/null +++ b/tests/py2go/log.go @@ -0,0 +1,11 @@ +package main + +import ( + "fmt" +) + +func Debugf(format string, args ...interface{}) { + if DEBUG { + fmt.Printf(format, args...) + } +} diff --git a/utils/run_go_api_server.sh b/utils/run_go_api_server.sh new file mode 100755 index 000000000..bb725cf72 --- /dev/null +++ b/utils/run_go_api_server.sh @@ -0,0 +1,2 @@ +#!/bin/bash +make build-linux && PORT=5001 AUTH0_USERNAME_CLAIM_CLI='http://lfx.dev/claims/username' AUTH0_EMAIL_CLAIM_CLI='http://lfx.dev/claims/email' AUTH0_NAME_CLAIM_CLI='http://lfx.dev/claims/username' ./bin/cla