From 6f59aed4f5a0de941d018a981e246c93dc47a676 Mon Sep 17 00:00:00 2001 From: "alessandro.pinna" Date: Tue, 5 Sep 2023 11:49:27 +0200 Subject: [PATCH] gateway: add cursor pagination update list APIs update list cmd --- cmd/agola/cmd/orgmemberlist.go | 17 +- cmd/agola/cmd/projectlist.go | 16 +- cmd/agola/cmd/projectsecretlist.go | 32 +- cmd/agola/cmd/projectvariablelist.go | 32 +- cmd/agola/cmd/remotesourcelist.go | 22 +- cmd/agola/cmd/runlist.go | 46 +- cmd/agola/cmd/userlist.go | 22 +- internal/services/configstore/action/org.go | 6 +- .../configstore/action/projectgroup.go | 4 +- .../services/configstore/action/secret.go | 6 +- .../services/configstore/action/variable.go | 6 +- internal/services/configstore/api/org.go | 60 +- .../services/configstore/api/projectgroup.go | 33 +- .../services/configstore/api/remotesource.go | 19 +- internal/services/configstore/api/secret.go | 40 +- internal/services/configstore/api/user.go | 18 +- internal/services/configstore/api/variable.go | 40 +- .../services/configstore/configstore_test.go | 12 +- internal/services/configstore/db/db.go | 80 +- internal/services/gateway/action/auth.go | 2 +- internal/services/gateway/action/org.go | 12 +- .../services/gateway/action/projectgroup.go | 4 +- .../services/gateway/action/remotesource.go | 2 +- internal/services/gateway/action/run.go | 10 +- internal/services/gateway/action/secret.go | 22 +- internal/services/gateway/action/user.go | 18 +- internal/services/gateway/action/variable.go | 46 +- internal/services/gateway/api/org.go | 162 +++- internal/services/gateway/api/projectgroup.go | 78 +- internal/services/gateway/api/remotesource.go | 82 +- internal/services/gateway/api/run.go | 104 ++- internal/services/gateway/api/secret.go | 90 +- internal/services/gateway/api/user.go | 91 +- internal/services/gateway/api/variable.go | 90 +- internal/services/runservice/api/api.go | 14 +- services/configstore/api/types/org.go | 22 +- services/configstore/api/types/project.go | 5 + .../configstore/api/types/remotesource.go | 11 +- services/configstore/api/types/secret.go | 5 + services/configstore/api/types/user.go | 17 +- services/configstore/api/types/variable.go | 5 + services/configstore/client/client.go | 121 ++- services/gateway/api/types/org.go | 8 +- services/gateway/api/types/project.go | 5 + services/gateway/api/types/remotesource.go | 5 + services/gateway/api/types/run.go | 7 +- services/gateway/api/types/secret.go | 5 + services/gateway/api/types/user.go | 5 + services/gateway/api/types/variable.go | 5 + services/gateway/client/client.go | 153 +++- services/runservice/api/types/run.go | 1 + tests/setup_test.go | 864 +++++++++++++++--- 52 files changed, 2182 insertions(+), 400 deletions(-) diff --git a/cmd/agola/cmd/orgmemberlist.go b/cmd/agola/cmd/orgmemberlist.go index a01855a5b..ac5e5e80b 100644 --- a/cmd/agola/cmd/orgmemberlist.go +++ b/cmd/agola/cmd/orgmemberlist.go @@ -23,6 +23,7 @@ import ( "github.com/sorintlab/errors" "github.com/spf13/cobra" + gwapitypes "agola.io/agola/services/gateway/api/types" gwclient "agola.io/agola/services/gateway/client" ) @@ -56,13 +57,21 @@ func init() { func orgMemberList(cmd *cobra.Command, args []string) error { gwclient := gwclient.NewClient(gatewayURL, token) + var orgMembersAll []*gwapitypes.OrgMemberResponse - orgMembers, _, err := gwclient.GetOrgMembers(context.TODO(), orgMemberListOpts.orgname) - if err != nil { - return errors.Wrapf(err, "failed to get organization member") + hasMoreData := true + var cursor string + for hasMoreData { + orgMembers, _, err := gwclient.GetOrgMembers(context.TODO(), orgMemberListOpts.orgname, false, 0, "") + if err != nil { + return errors.Wrapf(err, "failed to get organization member") + } + orgMembersAll = append(orgMembersAll, orgMembers.OrgMembers...) + cursor = orgMembers.Cursor + hasMoreData = cursor != "" } - out, err := json.MarshalIndent(orgMembers, "", "\t") + out, err := json.MarshalIndent(orgMembersAll, "", "\t") if err != nil { return errors.WithStack(err) } diff --git a/cmd/agola/cmd/projectlist.go b/cmd/agola/cmd/projectlist.go index d2fb1d5c0..968b6d8cc 100644 --- a/cmd/agola/cmd/projectlist.go +++ b/cmd/agola/cmd/projectlist.go @@ -18,8 +18,8 @@ import ( "context" "fmt" + "github.com/pkg/errors" "github.com/rs/zerolog/log" - "github.com/sorintlab/errors" "github.com/spf13/cobra" gwapitypes "agola.io/agola/services/gateway/api/types" @@ -63,12 +63,18 @@ func printProjects(projects []*gwapitypes.ProjectResponse) { func projectList(cmd *cobra.Command, args []string) error { gwclient := gwclient.NewClient(gatewayURL, token) - projects, _, err := gwclient.GetProjectGroupProjects(context.TODO(), projectListOpts.parentPath) - if err != nil { - return errors.WithStack(err) + var projectsAll []*gwapitypes.ProjectResponse + hasMoreData := true + var cursor string + for hasMoreData { + projectsResp, _, err := gwclient.GetProjectGroupProjects(context.TODO(), projectListOpts.parentPath, 0, cursor) + projectsAll = append(projectsAll, projectsResp.Projects...) + if err != nil { + return errors.WithStack(err) + } } - printProjects(projects) + printProjects(projectsAll) return nil } diff --git a/cmd/agola/cmd/projectsecretlist.go b/cmd/agola/cmd/projectsecretlist.go index b58f44e84..9035a284b 100644 --- a/cmd/agola/cmd/projectsecretlist.go +++ b/cmd/agola/cmd/projectsecretlist.go @@ -67,21 +67,37 @@ func secretList(cmd *cobra.Command, ownertype string, args []string) error { func printSecrets(ownertype, description string, tree, removeoverridden bool) error { - var err error - var secrets []*gwapitypes.SecretResponse + var secretsAll []*gwapitypes.SecretResponse gwclient := gwclient.NewClient(gatewayURL, token) switch ownertype { case "project": - secrets, _, err = gwclient.GetProjectSecrets(context.TODO(), secretListOpts.parentRef, tree, removeoverridden) + hasMoreData := true + var cursor string + for hasMoreData { + secrets, _, err := gwclient.GetProjectSecrets(context.TODO(), secretListOpts.parentRef, tree, removeoverridden, "", false, 0, cursor) + if err != nil { + return errors.Wrapf(err, "failed to list %s secrets", ownertype) + } + secretsAll = append(secretsAll, secrets.Secrets...) + cursor = secrets.Cursor + hasMoreData = cursor != "" + } case "projectgroup": - secrets, _, err = gwclient.GetProjectGroupSecrets(context.TODO(), secretListOpts.parentRef, tree, removeoverridden) - } - if err != nil { - return errors.Wrapf(err, "failed to list %s secrets", ownertype) + hasMoreData := true + var cursor string + for hasMoreData { + secrets, _, err := gwclient.GetProjectGroupSecrets(context.TODO(), secretListOpts.parentRef, tree, removeoverridden, "", false, 0, cursor) + if err != nil { + return errors.Wrapf(err, "failed to list %s secrets", ownertype) + } + secretsAll = append(secretsAll, secrets.Secrets...) + cursor = secrets.Cursor + hasMoreData = cursor != "" + } } - prettyJSON, err := json.MarshalIndent(secrets, "", "\t") + prettyJSON, err := json.MarshalIndent(secretsAll, "", "\t") if err != nil { return errors.Wrapf(err, "failed to convert %s secrets to json", ownertype) } diff --git a/cmd/agola/cmd/projectvariablelist.go b/cmd/agola/cmd/projectvariablelist.go index ef4fbd751..51fa8761c 100644 --- a/cmd/agola/cmd/projectvariablelist.go +++ b/cmd/agola/cmd/projectvariablelist.go @@ -67,21 +67,37 @@ func variableList(cmd *cobra.Command, ownertype string, args []string) error { func printVariables(ownertype, description string, tree, removeoverridden bool) error { - var err error - var variables []*gwapitypes.VariableResponse + var variablesAll []*gwapitypes.VariableResponse gwclient := gwclient.NewClient(gatewayURL, token) switch ownertype { case "project": - variables, _, err = gwclient.GetProjectVariables(context.TODO(), variableListOpts.parentRef, tree, removeoverridden) + hasMoreData := true + var cursor string + for hasMoreData { + variables, _, err := gwclient.GetProjectVariables(context.TODO(), variableListOpts.parentRef, tree, removeoverridden, "", false, 0, cursor) + if err != nil { + return errors.Wrapf(err, "failed to list %s variables", ownertype) + } + variablesAll = append(variablesAll, variables.Variables...) + cursor = variables.Cursor + hasMoreData = cursor != "" + } case "projectgroup": - variables, _, err = gwclient.GetProjectGroupVariables(context.TODO(), variableListOpts.parentRef, tree, removeoverridden) - } - if err != nil { - return errors.Wrapf(err, "failed to list %s variables", ownertype) + hasMoreData := true + var cursor string + for hasMoreData { + variables, _, err := gwclient.GetProjectGroupVariables(context.TODO(), variableListOpts.parentRef, tree, removeoverridden, "", false, 0, cursor) + if err != nil { + return errors.Wrapf(err, "failed to list %s variables", ownertype) + } + variablesAll = append(variablesAll, variables.Variables...) + cursor = variables.Cursor + hasMoreData = cursor != "" + } } - prettyJSON, err := json.MarshalIndent(variables, "", "\t") + prettyJSON, err := json.MarshalIndent(variablesAll, "", "\t") if err != nil { return errors.Wrapf(err, "failed to convert %s variables to json", ownertype) } diff --git a/cmd/agola/cmd/remotesourcelist.go b/cmd/agola/cmd/remotesourcelist.go index ded0a16dc..f9b99c831 100644 --- a/cmd/agola/cmd/remotesourcelist.go +++ b/cmd/agola/cmd/remotesourcelist.go @@ -61,12 +61,26 @@ func printRemoteSources(remoteSources []*gwapitypes.RemoteSourceResponse) { func remoteSourceList(cmd *cobra.Command, args []string) error { gwclient := gwclient.NewClient(gatewayURL, token) - remouteSources, _, err := gwclient.GetRemoteSources(context.TODO(), remoteSourceListOpts.start, remoteSourceListOpts.limit, false) - if err != nil { - return errors.WithStack(err) + var remouteSourcesAll []*gwapitypes.RemoteSourceResponse + if runListOpts.limit == 0 { + hasMoreData := true + var cursor string + for hasMoreData { + remouteSourcesResp, _, err := gwclient.GetRemoteSources(context.TODO(), remoteSourceListOpts.start, 0, false, cursor) + if err != nil { + return errors.WithStack(err) + } + remouteSourcesAll = append(remouteSourcesAll, remouteSourcesResp.RemoteSources...) + } + } else { + remouteSourcesResp, _, err := gwclient.GetRemoteSources(context.TODO(), remoteSourceListOpts.start, remoteSourceListOpts.limit, false, "") + if err != nil { + return errors.WithStack(err) + } + remouteSourcesAll = remouteSourcesResp.RemoteSources } - printRemoteSources(remouteSources) + printRemoteSources(remouteSourcesAll) return nil } diff --git a/cmd/agola/cmd/runlist.go b/cmd/agola/cmd/runlist.go index 817964749..e649e7d8b 100644 --- a/cmd/agola/cmd/runlist.go +++ b/cmd/agola/cmd/runlist.go @@ -105,19 +105,47 @@ func runList(cmd *cobra.Command, args []string) error { isProject := !flags.Changed("username") - var runsResp []*gwapitypes.RunsResponse - var err error + var runsAll []*gwapitypes.Runs if isProject { - runsResp, _, err = gwclient.GetProjectRuns(context.TODO(), runListOpts.projectRef, runListOpts.phaseFilter, nil, runListOpts.start, runListOpts.limit, false) + if runListOpts.limit == 0 { + hasMoreData := true + var cursor string + for hasMoreData { + runsResp, _, err := gwclient.GetProjectRuns(context.TODO(), runListOpts.projectRef, runListOpts.phaseFilter, nil, runListOpts.start, 0, false, cursor) + if err != nil { + return errors.WithStack(err) + } + runsAll = append(runsAll, runsResp.Runs...) + } + } else { + runsResp, _, err := gwclient.GetProjectRuns(context.TODO(), runListOpts.projectRef, runListOpts.phaseFilter, nil, runListOpts.start, runListOpts.limit, false, "") + if err != nil { + return errors.WithStack(err) + } + runsAll = runsResp.Runs + } } else { - runsResp, _, err = gwclient.GetUserRuns(context.TODO(), runListOpts.username, runListOpts.phaseFilter, nil, runListOpts.start, runListOpts.limit, false) - } - if err != nil { - return errors.WithStack(err) + if runListOpts.limit == 0 { + hasMoreData := true + var cursor string + for hasMoreData { + runsResp, _, err := gwclient.GetUserRuns(context.TODO(), runListOpts.username, runListOpts.phaseFilter, nil, runListOpts.start, 0, false, cursor) + if err != nil { + return errors.WithStack(err) + } + runsAll = append(runsAll, runsResp.Runs...) + } + } else { + runsResp, _, err := gwclient.GetUserRuns(context.TODO(), runListOpts.username, runListOpts.phaseFilter, nil, runListOpts.start, runListOpts.limit, false, "") + if err != nil { + return errors.WithStack(err) + } + runsAll = runsResp.Runs + } } - runs := make([]*runDetails, len(runsResp)) - for i, runResponse := range runsResp { + runs := make([]*runDetails, len(runsAll)) + for i, runResponse := range runsAll { var err error var run *gwapitypes.RunResponse if isProject { diff --git a/cmd/agola/cmd/userlist.go b/cmd/agola/cmd/userlist.go index 04c1aa85c..20465396e 100644 --- a/cmd/agola/cmd/userlist.go +++ b/cmd/agola/cmd/userlist.go @@ -61,12 +61,26 @@ func printUsers(users []*gwapitypes.PrivateUserResponse) { func userList(cmd *cobra.Command, args []string) error { gwclient := gwclient.NewClient(gatewayURL, token) - users, _, err := gwclient.GetUsers(context.TODO(), userListOpts.start, userListOpts.limit, false) - if err != nil { - return errors.WithStack(err) + var usersAll []*gwapitypes.PrivateUserResponse + if runListOpts.limit == 0 { + hasMoreData := true + var cursor string + for hasMoreData { + usersResp, _, err := gwclient.GetUsers(context.TODO(), userListOpts.start, 0, false, cursor) + if err != nil { + return errors.WithStack(err) + } + usersAll = append(usersAll, usersResp.Users...) + } + } else { + usersResp, _, err := gwclient.GetUsers(context.TODO(), userListOpts.start, userListOpts.limit, false, "") + if err != nil { + return errors.WithStack(err) + } + usersAll = usersResp.Users } - printUsers(users) + printUsers(usersAll) return nil } diff --git a/internal/services/configstore/action/org.go b/internal/services/configstore/action/org.go index b7d0310f3..98f41d6a0 100644 --- a/internal/services/configstore/action/org.go +++ b/internal/services/configstore/action/org.go @@ -38,7 +38,7 @@ func orgMemberResponse(orgUser *db.OrgUser) *OrgMemberResponse { } } -func (h *ActionHandler) GetOrgMembers(ctx context.Context, orgRef string) ([]*OrgMemberResponse, error) { +func (h *ActionHandler) GetOrgMembers(ctx context.Context, orgRef string, start string, limit int, asc bool) ([]*OrgMemberResponse, error) { var orgUsers []*db.OrgUser err := h.d.Do(ctx, func(tx *sql.Tx) error { var err error @@ -50,7 +50,7 @@ func (h *ActionHandler) GetOrgMembers(ctx context.Context, orgRef string) ([]*Or return util.NewAPIError(util.ErrNotExist, errors.Errorf("org %q doesn't exist", orgRef)) } - orgUsers, err = h.d.GetOrgUsers(tx, org.ID) + orgUsers, err = h.d.GetOrgUsers(tx, org.ID, start, limit, asc) return errors.WithStack(err) }) if err != nil { @@ -211,7 +211,7 @@ func (h *ActionHandler) DeleteOrg(ctx context.Context, orgRef string) error { return errors.WithStack(err) } for _, subgroup := range subgroups { - projects, err := h.d.GetProjectGroupProjects(tx, subgroup.ID) + projects, err := h.d.GetProjectGroupProjects(tx, subgroup.ID, "", 0) if err != nil { return errors.WithStack(err) } diff --git a/internal/services/configstore/action/projectgroup.go b/internal/services/configstore/action/projectgroup.go index 2b7dcc53a..e5ea06f7f 100644 --- a/internal/services/configstore/action/projectgroup.go +++ b/internal/services/configstore/action/projectgroup.go @@ -67,7 +67,7 @@ func (h *ActionHandler) GetProjectGroupSubgroups(ctx context.Context, projectGro return projectGroups, nil } -func (h *ActionHandler) GetProjectGroupProjects(ctx context.Context, projectGroupRef string) ([]*types.Project, error) { +func (h *ActionHandler) GetProjectGroupProjects(ctx context.Context, projectGroupRef string, start string, limit int) ([]*types.Project, error) { var projects []*types.Project err := h.d.Do(ctx, func(tx *sql.Tx) error { var err error @@ -80,7 +80,7 @@ func (h *ActionHandler) GetProjectGroupProjects(ctx context.Context, projectGrou return util.NewAPIError(util.ErrNotExist, errors.Errorf("project group %q doesn't exist", projectGroupRef)) } - projects, err = h.d.GetProjectGroupProjects(tx, projectGroup.ID) + projects, err = h.d.GetProjectGroupProjects(tx, projectGroup.ID, start, limit) return errors.WithStack(err) }) if err != nil { diff --git a/internal/services/configstore/action/secret.go b/internal/services/configstore/action/secret.go index bb6962e8a..554984427 100644 --- a/internal/services/configstore/action/secret.go +++ b/internal/services/configstore/action/secret.go @@ -42,7 +42,7 @@ func (h *ActionHandler) GetSecret(ctx context.Context, secretID string) (*types. return secret, nil } -func (h *ActionHandler) GetSecrets(ctx context.Context, parentKind types.ObjectKind, parentRef string, tree bool) ([]*types.Secret, error) { +func (h *ActionHandler) GetSecrets(ctx context.Context, parentKind types.ObjectKind, parentRef string, tree bool, startSecretName string, asc bool, limit int) ([]*types.Secret, error) { var secrets []*types.Secret err := h.d.Do(ctx, func(tx *sql.Tx) error { parentID, err := h.ResolveObjectID(tx, parentKind, parentRef) @@ -50,9 +50,9 @@ func (h *ActionHandler) GetSecrets(ctx context.Context, parentKind types.ObjectK return errors.WithStack(err) } if tree { - secrets, err = h.d.GetSecretsTree(tx, parentKind, parentID) + secrets, err = h.d.GetSecretsTree(tx, parentKind, parentID, startSecretName, asc, limit) } else { - secrets, err = h.d.GetSecrets(tx, parentID) + secrets, err = h.d.GetSecrets(tx, parentID, startSecretName, asc, limit) } return errors.WithStack(err) }) diff --git a/internal/services/configstore/action/variable.go b/internal/services/configstore/action/variable.go index 0cb17092d..c7956df28 100644 --- a/internal/services/configstore/action/variable.go +++ b/internal/services/configstore/action/variable.go @@ -24,7 +24,7 @@ import ( "agola.io/agola/services/configstore/types" ) -func (h *ActionHandler) GetVariables(ctx context.Context, parentKind types.ObjectKind, parentRef string, tree bool) ([]*types.Variable, error) { +func (h *ActionHandler) GetVariables(ctx context.Context, parentKind types.ObjectKind, parentRef string, tree bool, startVariableName string, asc bool, limit int) ([]*types.Variable, error) { var variables []*types.Variable err := h.d.Do(ctx, func(tx *sql.Tx) error { parentID, err := h.ResolveObjectID(tx, parentKind, parentRef) @@ -32,9 +32,9 @@ func (h *ActionHandler) GetVariables(ctx context.Context, parentKind types.Objec return errors.WithStack(err) } if tree { - variables, err = h.d.GetVariablesTree(tx, parentKind, parentID) + variables, err = h.d.GetVariablesTree(tx, parentKind, parentID, startVariableName, asc, limit) } else { - variables, err = h.d.GetVariables(tx, parentID) + variables, err = h.d.GetVariables(tx, parentID, startVariableName, asc, limit) } return errors.WithStack(err) }) diff --git a/internal/services/configstore/api/org.go b/internal/services/configstore/api/org.go index dd8239f32..34efbbd99 100644 --- a/internal/services/configstore/api/org.go +++ b/internal/services/configstore/api/org.go @@ -166,7 +166,7 @@ func (h *DeleteOrgHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { const ( DefaultOrgsLimit = 10 - MaxOrgsLimit = 20 + MaxOrgsLimit = 21 ) type OrgsHandler struct { @@ -206,19 +206,34 @@ func (h *OrgsHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { start := query.Get("start") + var hasMoreData bool + qLimit := limit + if qLimit != 0 { + qLimit++ + } + var orgs []*types.Organization err := h.d.Do(ctx, func(tx *sql.Tx) error { var err error - orgs, err = h.d.GetOrgs(tx, start, limit, asc) + orgs, err = h.d.GetOrgs(tx, start, qLimit, asc) return errors.WithStack(err) }) + if limit != 0 && len(orgs) > limit { + hasMoreData = true + orgs = orgs[:limit] + } + if err != nil { h.log.Err(err).Send() util.HTTPError(w, err) return } - if err := util.HTTPResponse(w, http.StatusOK, orgs); err != nil { + response := csapitypes.OrgsResponse{ + Orgs: orgs, + HasMoreData: hasMoreData, + } + if err := util.HTTPResponse(w, http.StatusOK, response); err != nil { h.log.Err(err).Send() } } @@ -303,20 +318,51 @@ func NewOrgMembersHandler(log zerolog.Logger, ah *action.ActionHandler) *OrgMemb func (h *OrgMembersHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { ctx := r.Context() vars := mux.Vars(r) + query := r.URL.Query() orgRef := vars["orgref"] - orgUsers, err := h.ah.GetOrgMembers(ctx, orgRef) + var limit int + limitS := query.Get("limit") + if limitS != "" { + var err error + limit, err = strconv.Atoi(limitS) + if err != nil { + util.HTTPError(w, util.NewAPIError(util.ErrBadRequest, errors.Wrapf(err, "cannot parse limit"))) + return + } + } + asc := false + if _, ok := query["asc"]; ok { + asc = true + } + start := query.Get("start") + + var hasMoreData bool + aLimit := limit + if aLimit != 0 { + aLimit++ + } + orgUsers, err := h.ah.GetOrgMembers(ctx, orgRef, start, aLimit, asc) if util.HTTPError(w, err) { h.log.Err(err).Send() return } - res := make([]*csapitypes.OrgMemberResponse, len(orgUsers)) + if limit != 0 && len(orgUsers) > limit { + hasMoreData = true + orgUsers = orgUsers[:limit] + } + + orgMembers := make([]*csapitypes.OrgMemberResponse, len(orgUsers)) for i, orgUser := range orgUsers { - res[i] = orgMemberResponse(orgUser) + orgMembers[i] = orgMemberResponse(orgUser) } - if err := util.HTTPResponse(w, http.StatusOK, res); err != nil { + response := csapitypes.OrgMembersResponse{ + OrgMembers: orgMembers, + HasMoreData: hasMoreData, + } + if err := util.HTTPResponse(w, http.StatusOK, response); err != nil { h.log.Err(err).Send() } } diff --git a/internal/services/configstore/api/projectgroup.go b/internal/services/configstore/api/projectgroup.go index 68aa51a58..d8ffa9521 100644 --- a/internal/services/configstore/api/projectgroup.go +++ b/internal/services/configstore/api/projectgroup.go @@ -20,6 +20,7 @@ import ( "net/http" "net/url" "path" + "strconv" "github.com/gorilla/mux" "github.com/rs/zerolog" @@ -130,6 +131,19 @@ func NewProjectGroupProjectsHandler(log zerolog.Logger, ah *action.ActionHandler func (h *ProjectGroupProjectsHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { ctx := r.Context() vars := mux.Vars(r) + query := r.URL.Query() + + start := query.Get("start") + var limit int + limitS := query.Get("limit") + if limitS != "" { + var err error + limit, err = strconv.Atoi(limitS) + if err != nil { + util.HTTPError(w, util.NewAPIError(util.ErrBadRequest, errors.Wrapf(err, "cannot parse limit"))) + return + } + } projectGroupRef, err := url.PathUnescape(vars["projectgroupref"]) if err != nil { @@ -137,19 +151,34 @@ func (h *ProjectGroupProjectsHandler) ServeHTTP(w http.ResponseWriter, r *http.R return } - projects, err := h.ah.GetProjectGroupProjects(ctx, projectGroupRef) + var hasMoreData bool + aLimit := limit + if aLimit != 0 { + aLimit++ + } + + projects, err := h.ah.GetProjectGroupProjects(ctx, projectGroupRef, start, aLimit) if util.HTTPError(w, err) { h.log.Err(err).Send() return } + if limit != 0 && len(projects) > limit { + hasMoreData = true + projects = projects[:limit] + } + resProjects, err := projectsResponse(ctx, h.readDB, projects) if util.HTTPError(w, err) { h.log.Err(err).Send() return } - if err := util.HTTPResponse(w, http.StatusOK, resProjects); err != nil { + response := csapitypes.ProjectsResponse{ + Projects: resProjects, + HasMoreData: hasMoreData, + } + if err := util.HTTPResponse(w, http.StatusOK, response); err != nil { h.log.Err(err).Send() } } diff --git a/internal/services/configstore/api/remotesource.go b/internal/services/configstore/api/remotesource.go index bd6686740..5473efcff 100644 --- a/internal/services/configstore/api/remotesource.go +++ b/internal/services/configstore/api/remotesource.go @@ -224,10 +224,16 @@ func (h *RemoteSourcesHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) start := query.Get("start") + var hasMoreData bool + qLimit := limit + if qLimit != 0 { + qLimit++ + } + var remoteSources []*types.RemoteSource err := h.d.Do(ctx, func(tx *sql.Tx) error { var err error - remoteSources, err = h.d.GetRemoteSources(tx, start, limit, asc) + remoteSources, err = h.d.GetRemoteSources(tx, start, qLimit, asc) return errors.WithStack(err) }) if err != nil { @@ -236,7 +242,16 @@ func (h *RemoteSourcesHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) return } - if err := util.HTTPResponse(w, http.StatusOK, remoteSources); err != nil { + if limit != 0 && len(remoteSources) > limit { + hasMoreData = true + remoteSources = remoteSources[:limit] + } + + response := csapitypes.RemoteSourcesReponse{ + RemoteSources: remoteSources, + HasMoreData: hasMoreData, + } + if err := util.HTTPResponse(w, http.StatusOK, response); err != nil { h.log.Err(err).Send() } } diff --git a/internal/services/configstore/api/secret.go b/internal/services/configstore/api/secret.go index 4581f0ec9..2670e0d25 100644 --- a/internal/services/configstore/api/secret.go +++ b/internal/services/configstore/api/secret.go @@ -17,6 +17,7 @@ package api import ( "encoding/json" "net/http" + "strconv" "github.com/gorilla/mux" "github.com/rs/zerolog" @@ -71,18 +72,49 @@ func (h *SecretsHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { query := r.URL.Query() _, tree := query["tree"] + var limit int + limitS := query.Get("limit") + if limitS != "" { + var err error + limit, err = strconv.Atoi(limitS) + if err != nil { + util.HTTPError(w, util.NewAPIError(util.ErrBadRequest, errors.Wrapf(err, "cannot parse limit"))) + return + } + } + if limit < 0 { + util.HTTPError(w, util.NewAPIError(util.ErrBadRequest, errors.Errorf("limit must be greater or equal than 0"))) + return + } + asc := false + if _, ok := query["asc"]; ok { + asc = true + } + start := query.Get("start") + parentKind, parentRef, err := GetObjectKindRef(r) if util.HTTPError(w, err) { h.log.Err(err).Send() return } - secrets, err := h.ah.GetSecrets(ctx, parentKind, parentRef, tree) + var hasMoreData bool + aLimit := limit + if aLimit != 0 { + aLimit++ + } + + secrets, err := h.ah.GetSecrets(ctx, parentKind, parentRef, tree, start, asc, aLimit) if util.HTTPError(w, err) { h.log.Err(err).Send() return } + if limit != 0 && len(secrets) > limit { + hasMoreData = true + secrets = secrets[:limit] + } + resSecrets := make([]*csapitypes.Secret, len(secrets)) for i, s := range secrets { resSecrets[i] = &csapitypes.Secret{Secret: s} @@ -105,7 +137,11 @@ func (h *SecretsHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } - if err := util.HTTPResponse(w, http.StatusOK, resSecrets); err != nil { + response := &csapitypes.SecretsResponse{ + Secrets: resSecrets, + HasMoreData: hasMoreData, + } + if err := util.HTTPResponse(w, http.StatusOK, response); err != nil { h.log.Err(err).Send() } } diff --git a/internal/services/configstore/api/user.go b/internal/services/configstore/api/user.go index 3950f402e..ef78b1a06 100644 --- a/internal/services/configstore/api/user.go +++ b/internal/services/configstore/api/user.go @@ -220,6 +220,7 @@ func (h *UsersHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { queryType := query.Get("query_type") var users []*types.User + var hasMoreData bool err := h.d.Do(ctx, func(tx *sql.Tx) error { switch queryType { case "bytoken": @@ -264,7 +265,11 @@ func (h *UsersHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { default: // default query var err error - users, err = h.d.GetUsers(tx, start, limit, asc) + qLimit := limit + if qLimit != 0 { + qLimit++ + } + users, err = h.d.GetUsers(tx, start, qLimit, asc) return errors.WithStack(err) } @@ -276,7 +281,16 @@ func (h *UsersHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } - if err := util.HTTPResponse(w, http.StatusOK, users); err != nil { + if limit != 0 && len(users) > limit { + hasMoreData = true + users = users[:limit] + } + + response := csapitypes.PrivateUsersResponse{ + Users: users, + HasMoreData: hasMoreData, + } + if err := util.HTTPResponse(w, http.StatusOK, response); err != nil { h.log.Err(err).Send() } } diff --git a/internal/services/configstore/api/variable.go b/internal/services/configstore/api/variable.go index f5cfdfbc6..880461814 100644 --- a/internal/services/configstore/api/variable.go +++ b/internal/services/configstore/api/variable.go @@ -17,6 +17,7 @@ package api import ( "encoding/json" "net/http" + "strconv" "github.com/gorilla/mux" "github.com/rs/zerolog" @@ -45,18 +46,49 @@ func (h *VariablesHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { query := r.URL.Query() _, tree := query["tree"] + var limit int + limitS := query.Get("limit") + if limitS != "" { + var err error + limit, err = strconv.Atoi(limitS) + if err != nil { + util.HTTPError(w, util.NewAPIError(util.ErrBadRequest, errors.Wrapf(err, "cannot parse limit"))) + return + } + } + if limit < 0 { + util.HTTPError(w, util.NewAPIError(util.ErrBadRequest, errors.Errorf("limit must be greater or equal than 0"))) + return + } + asc := false + if _, ok := query["asc"]; ok { + asc = true + } + start := query.Get("start") + parentKind, parentRef, err := GetObjectKindRef(r) if util.HTTPError(w, err) { h.log.Err(err).Send() return } - variables, err := h.ah.GetVariables(ctx, parentKind, parentRef, tree) + var hasMoreData bool + aLimit := limit + if aLimit != 0 { + aLimit++ + } + + variables, err := h.ah.GetVariables(ctx, parentKind, parentRef, tree, start, asc, aLimit) if util.HTTPError(w, err) { h.log.Err(err).Send() return } + if limit != 0 && len(variables) > limit { + hasMoreData = true + variables = variables[:limit] + } + resVariables := make([]*csapitypes.Variable, len(variables)) for i, v := range variables { resVariables[i] = &csapitypes.Variable{Variable: v} @@ -78,7 +110,11 @@ func (h *VariablesHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } - if err := util.HTTPResponse(w, http.StatusOK, resVariables); err != nil { + response := &csapitypes.VariablesResponse{ + Variables: resVariables, + HasMoreData: hasMoreData, + } + if err := util.HTTPResponse(w, http.StatusOK, response); err != nil { h.log.Err(err).Send() } } diff --git a/internal/services/configstore/configstore_test.go b/internal/services/configstore/configstore_test.go index 52887bdd0..76d585c1d 100644 --- a/internal/services/configstore/configstore_test.go +++ b/internal/services/configstore/configstore_test.go @@ -977,7 +977,7 @@ func TestProjectGroupDeleteDontSeeOldChildObjects(t *testing.T) { } // Get by projectgroup id - projects, err := cs.ah.GetProjectGroupProjects(ctx, spg01.ID) + projects, err := cs.ah.GetProjectGroupProjects(ctx, spg01.ID, "", 0) if err != nil { t.Fatalf("unexpected err: %v", err) } @@ -986,7 +986,7 @@ func TestProjectGroupDeleteDontSeeOldChildObjects(t *testing.T) { } // Get by projectgroup path - projects, err = cs.ah.GetProjectGroupProjects(ctx, path.Join("org", org.Name, pg01.Name, spg01.Name)) + projects, err = cs.ah.GetProjectGroupProjects(ctx, path.Join("org", org.Name, pg01.Name, spg01.Name), "", 0) if err != nil { t.Fatalf("unexpected err: %v", err) } @@ -994,7 +994,7 @@ func TestProjectGroupDeleteDontSeeOldChildObjects(t *testing.T) { t.Fatalf("mismatch (-want +got):\n%s", diff) } - secrets, err := cs.ah.GetSecrets(ctx, types.ObjectKindProject, project.ID, false) + secrets, err := cs.ah.GetSecrets(ctx, types.ObjectKindProject, project.ID, false, "", false, 0) if err != nil { t.Fatalf("unexpected err: %v", err) } @@ -1002,7 +1002,7 @@ func TestProjectGroupDeleteDontSeeOldChildObjects(t *testing.T) { t.Fatalf("mismatch (-want +got):\n%s", diff) } - secrets, err = cs.ah.GetSecrets(ctx, types.ObjectKindProject, path.Join("org", org.Name, pg01.Name, spg01.Name, project.Name), false) + secrets, err = cs.ah.GetSecrets(ctx, types.ObjectKindProject, path.Join("org", org.Name, pg01.Name, spg01.Name, project.Name), false, "", false, 0) if err != nil { t.Fatalf("unexpected err: %v", err) } @@ -1010,7 +1010,7 @@ func TestProjectGroupDeleteDontSeeOldChildObjects(t *testing.T) { t.Fatalf("mismatch (-want +got):\n%s", diff) } - variables, err := cs.ah.GetVariables(ctx, types.ObjectKindProject, project.ID, false) + variables, err := cs.ah.GetVariables(ctx, types.ObjectKindProject, project.ID, false, "", false, 0) if err != nil { t.Fatalf("unexpected err: %v", err) } @@ -1018,7 +1018,7 @@ func TestProjectGroupDeleteDontSeeOldChildObjects(t *testing.T) { t.Fatalf("mismatch (-want +got):\n%s", diff) } - variables, err = cs.ah.GetVariables(ctx, types.ObjectKindProject, path.Join("org", org.Name, pg01.Name, spg01.Name, project.Name), false) + variables, err = cs.ah.GetVariables(ctx, types.ObjectKindProject, path.Join("org", org.Name, pg01.Name, spg01.Name, project.Name), false, "", false, 0) if err != nil { t.Fatalf("unexpected err: %v", err) } diff --git a/internal/services/configstore/db/db.go b/internal/services/configstore/db/db.go index bd84a5b03..9ba663708 100644 --- a/internal/services/configstore/db/db.go +++ b/internal/services/configstore/db/db.go @@ -535,14 +535,28 @@ type OrgUser struct { } // TODO(sgotti) implement cursor fetching -func (d *DB) GetOrgUsers(tx *sql.Tx, orgID string) ([]*OrgUser, error) { +func (d *DB) GetOrgUsers(tx *sql.Tx, orgID string, start string, limit int, asc bool) ([]*OrgUser, error) { cols := organizationMemberSelectColumns() cols = append(cols, userSelectColumns()...) q := sq.Select(cols...).From("orgmember") q = q.Join("user_t", "user_t.id = orgmember.user_id") q = q.Where(q.E("orgmember.organization_id", orgID)) - q = q.OrderBy("user_t.name") + if start != "" { + if asc { + q = q.Where(q.G("user_t.name", start)) + } else { + q = q.Where(q.L("user_t.name", start)) + } + } + if asc { + q = q.OrderBy("name").Asc() + } else { + q = q.OrderBy("name").Desc() + } + if limit > 0 { + q = q.Limit(limit) + } rows, err := d.query(tx, q) if err != nil { @@ -904,9 +918,16 @@ func (d *DB) GetProjectByPath(tx *sql.Tx, projectPath string) (*types.Project, e return project, nil } -func (d *DB) GetProjectGroupProjects(tx *sql.Tx, parentID string) ([]*types.Project, error) { +func (d *DB) GetProjectGroupProjects(tx *sql.Tx, parentID string, start string, limit int) ([]*types.Project, error) { q := projectSelect() q.Where(q.E("parent_id", parentID)) + if start != "" { + q.Where(q.G("name", start)) + } + q = q.OrderBy("name") + if limit > 0 { + q.Limit(limit) + } projects, _, err := d.fetchProjects(tx, q) return projects, errors.WithStack(err) @@ -936,9 +957,24 @@ func (d *DB) GetSecretByName(tx *sql.Tx, parentID, name string) (*types.Secret, return out, errors.WithStack(err) } -func (d *DB) GetSecrets(tx *sql.Tx, parentID string) ([]*types.Secret, error) { +func (d *DB) GetSecrets(tx *sql.Tx, parentID string, startSecretName string, asc bool, limit int) ([]*types.Secret, error) { q := secretSelect() q.Where(q.E("parent_id", parentID)) + if asc { + q.OrderBy("name").Asc() + } else { + q.OrderBy("name").Desc() + } + if startSecretName != "" { + if asc { + q.Where(q.G("name", startSecretName)) + } else { + q.Where(q.L("name", startSecretName)) + } + } + if limit > 0 { + q.Limit(limit) + } secrets, _, err := d.fetchSecrets(tx, q) return secrets, errors.WithStack(err) } @@ -980,16 +1016,21 @@ func (d *DB) GetSecretTree(tx *sql.Tx, parentKind types.ObjectKind, parentID, na return nil, nil } -func (d *DB) GetSecretsTree(tx *sql.Tx, parentKind types.ObjectKind, parentID string) ([]*types.Secret, error) { +func (d *DB) GetSecretsTree(tx *sql.Tx, parentKind types.ObjectKind, parentID string, startSecretName string, asc bool, limit int) ([]*types.Secret, error) { allSecrets := []*types.Secret{} for parentKind == types.ObjectKindProjectGroup || parentKind == types.ObjectKindProject { - secrets, err := d.GetSecrets(tx, parentID) + secrets, err := d.GetSecrets(tx, parentID, startSecretName, asc, limit) if err != nil { return nil, errors.Wrapf(err, "failed to get secrets for %s %q", parentKind, parentID) } allSecrets = append(allSecrets, secrets...) + if limit > 0 && len(allSecrets) > limit { + allSecrets = allSecrets[:limit] + return allSecrets, nil + } + switch parentKind { case types.ObjectKindProjectGroup: projectGroup, err := d.GetProjectGroup(tx, parentID) @@ -1041,23 +1082,44 @@ func (d *DB) GetVariableByName(tx *sql.Tx, parentID, name string) (*types.Variab return out, errors.WithStack(err) } -func (d *DB) GetVariables(tx *sql.Tx, parentID string) ([]*types.Variable, error) { +func (d *DB) GetVariables(tx *sql.Tx, parentID string, startVariableName string, asc bool, limit int) ([]*types.Variable, error) { q := variableSelect() q.Where(q.E("parent_id", parentID)) + q.Where(q.E("parent_id", parentID)) + if asc { + q.OrderBy("name").Asc() + } else { + q.OrderBy("name").Desc() + } + if startVariableName != "" { + if asc { + q.Where(q.G("name", startVariableName)) + } else { + q.Where(q.L("name", startVariableName)) + } + } + if limit > 0 { + q.Limit(limit) + } variables, _, err := d.fetchVariables(tx, q) return variables, errors.WithStack(err) } -func (d *DB) GetVariablesTree(tx *sql.Tx, parentKind types.ObjectKind, parentID string) ([]*types.Variable, error) { +func (d *DB) GetVariablesTree(tx *sql.Tx, parentKind types.ObjectKind, parentID string, startVariableName string, asc bool, limit int) ([]*types.Variable, error) { allVariables := []*types.Variable{} for parentKind == types.ObjectKindProjectGroup || parentKind == types.ObjectKindProject { - vars, err := d.GetVariables(tx, parentID) + vars, err := d.GetVariables(tx, parentID, startVariableName, asc, limit) if err != nil { return nil, errors.Wrapf(err, "failed to get variables for %s %q", parentKind, parentID) } allVariables = append(allVariables, vars...) + if limit > 0 && len(allVariables) > limit { + allVariables = allVariables[:limit] + return allVariables, nil + } + switch parentKind { case types.ObjectKindProjectGroup: projectGroup, err := d.GetProjectGroup(tx, parentID) diff --git a/internal/services/gateway/action/auth.go b/internal/services/gateway/action/auth.go index 8d5f99103..84eb83236 100644 --- a/internal/services/gateway/action/auth.go +++ b/internal/services/gateway/action/auth.go @@ -228,7 +228,7 @@ func (h *ActionHandler) IsOrgMember(ctx context.Context, userRef, orgRef string) return false, errors.Wrapf(err, "failed to get user %s:", userRef) } - orgMembers, err := h.GetOrgMembers(ctx, orgRef) + orgMembers, err := h.GetOrgMembers(ctx, orgRef, "", 0, false) if err != nil { return false, errors.Wrapf(err, "failed to get org %s members:", orgRef) } diff --git a/internal/services/gateway/action/org.go b/internal/services/gateway/action/org.go index 5f92efb84..9022911cf 100644 --- a/internal/services/gateway/action/org.go +++ b/internal/services/gateway/action/org.go @@ -39,7 +39,7 @@ type GetOrgsRequest struct { Asc bool } -func (h *ActionHandler) GetOrgs(ctx context.Context, req *GetOrgsRequest) ([]*cstypes.Organization, error) { +func (h *ActionHandler) GetOrgs(ctx context.Context, req *GetOrgsRequest) (*csapitypes.OrgsResponse, error) { orgs, _, err := h.configstoreClient.GetOrgs(ctx, req.Start, req.Limit, req.Asc) if err != nil { return nil, util.NewAPIError(util.KindFromRemoteError(err), err) @@ -50,6 +50,7 @@ func (h *ActionHandler) GetOrgs(ctx context.Context, req *GetOrgsRequest) ([]*cs type OrgMembersResponse struct { Organization *cstypes.Organization Members []*OrgMemberResponse + HasMoreData bool } type OrgMemberResponse struct { @@ -57,22 +58,23 @@ type OrgMemberResponse struct { Role cstypes.MemberRole } -func (h *ActionHandler) GetOrgMembers(ctx context.Context, orgRef string) (*OrgMembersResponse, error) { +func (h *ActionHandler) GetOrgMembers(ctx context.Context, orgRef string, start string, limit int, asc bool) (*OrgMembersResponse, error) { org, _, err := h.configstoreClient.GetOrg(ctx, orgRef) if err != nil { return nil, util.NewAPIError(util.KindFromRemoteError(err), err) } - orgMembers, _, err := h.configstoreClient.GetOrgMembers(ctx, orgRef) + orgMembers, _, err := h.configstoreClient.GetOrgMembers(ctx, orgRef, start, limit, asc) if err != nil { return nil, util.NewAPIError(util.KindFromRemoteError(err), err) } res := &OrgMembersResponse{ Organization: org, - Members: make([]*OrgMemberResponse, len(orgMembers)), + Members: make([]*OrgMemberResponse, len(orgMembers.OrgMembers)), + HasMoreData: orgMembers.HasMoreData, } - for i, orgMember := range orgMembers { + for i, orgMember := range orgMembers.OrgMembers { res.Members[i] = &OrgMemberResponse{ User: orgMember.User, Role: orgMember.Role, diff --git a/internal/services/gateway/action/projectgroup.go b/internal/services/gateway/action/projectgroup.go index 941e2d810..4e6b4cb39 100644 --- a/internal/services/gateway/action/projectgroup.go +++ b/internal/services/gateway/action/projectgroup.go @@ -41,8 +41,8 @@ func (h *ActionHandler) GetProjectGroupSubgroups(ctx context.Context, projectGro return projectGroups, nil } -func (h *ActionHandler) GetProjectGroupProjects(ctx context.Context, projectGroupRef string) ([]*csapitypes.Project, error) { - projects, _, err := h.configstoreClient.GetProjectGroupProjects(ctx, projectGroupRef) +func (h *ActionHandler) GetProjectGroupProjects(ctx context.Context, projectGroupRef string, start string, limit int) (*csapitypes.ProjectsResponse, error) { + projects, _, err := h.configstoreClient.GetProjectGroupProjects(ctx, projectGroupRef, start, limit) if err != nil { return nil, util.NewAPIError(util.KindFromRemoteError(err), err) } diff --git a/internal/services/gateway/action/remotesource.go b/internal/services/gateway/action/remotesource.go index 55d52b679..50304d24b 100644 --- a/internal/services/gateway/action/remotesource.go +++ b/internal/services/gateway/action/remotesource.go @@ -39,7 +39,7 @@ type GetRemoteSourcesRequest struct { Asc bool } -func (h *ActionHandler) GetRemoteSources(ctx context.Context, req *GetRemoteSourcesRequest) ([]*cstypes.RemoteSource, error) { +func (h *ActionHandler) GetRemoteSources(ctx context.Context, req *GetRemoteSourcesRequest) (*csapitypes.RemoteSourcesReponse, error) { remoteSources, _, err := h.configstoreClient.GetRemoteSources(ctx, req.Start, req.Limit, req.Asc) if err != nil { return nil, errors.WithStack(err) diff --git a/internal/services/gateway/action/run.go b/internal/services/gateway/action/run.go index 23c2b5cdf..2a176cb52 100644 --- a/internal/services/gateway/action/run.go +++ b/internal/services/gateway/action/run.go @@ -615,20 +615,20 @@ func (h *ActionHandler) genRunVariables(ctx context.Context, req *CreateRunReque variables := map[string]string{} // get project variables - pvars, _, err := h.configstoreClient.GetProjectVariables(ctx, req.Project.ID, true) + pvars, _, err := h.configstoreClient.GetProjectVariables(ctx, req.Project.ID, true, "", false, 0) if err != nil { return nil, errors.Wrapf(err, "failed to get project variables") } // remove overriden variables - pvars = scommon.FilterOverriddenVariables(pvars) + pvars.Variables = scommon.FilterOverriddenVariables(pvars.Variables) // get project secrets - secrets, _, err := h.configstoreClient.GetProjectSecrets(ctx, req.Project.ID, true) + secrets, _, err := h.configstoreClient.GetProjectSecrets(ctx, req.Project.ID, true, "", false, 0) if err != nil { return nil, errors.Wrapf(err, "failed to get project secrets") } - for _, pvar := range pvars { + for _, pvar := range pvars.Variables { // find the value match var varval cstypes.VariableValue for _, varval = range pvar.Values { @@ -637,7 +637,7 @@ func (h *ActionHandler) genRunVariables(ctx context.Context, req *CreateRunReque continue } // get the secret value referenced by the variable, it must be a secret at the same level or a lower level - secret := scommon.GetVarValueMatchingSecret(varval, pvar.ParentPath, secrets) + secret := scommon.GetVarValueMatchingSecret(varval, pvar.ParentPath, secrets.Secrets) if secret != nil { varValue, ok := secret.Data[varval.SecretVar] if ok { diff --git a/internal/services/gateway/action/secret.go b/internal/services/gateway/action/secret.go index 435f926e9..9bac482b8 100644 --- a/internal/services/gateway/action/secret.go +++ b/internal/services/gateway/action/secret.go @@ -31,16 +31,24 @@ type GetSecretsRequest struct { Tree bool RemoveOverridden bool + Start string + Asc bool + Limit int } -func (h *ActionHandler) GetSecrets(ctx context.Context, req *GetSecretsRequest) ([]*csapitypes.Secret, error) { - var cssecrets []*csapitypes.Secret +func (h *ActionHandler) GetSecrets(ctx context.Context, req *GetSecretsRequest) (*csapitypes.SecretsResponse, error) { + limit := req.Limit + if req.RemoveOverridden { + limit = 0 + } + + var cssecrets *csapitypes.SecretsResponse var err error switch req.ParentType { case cstypes.ObjectKindProjectGroup: - cssecrets, _, err = h.configstoreClient.GetProjectGroupSecrets(ctx, req.ParentRef, req.Tree) + cssecrets, _, err = h.configstoreClient.GetProjectGroupSecrets(ctx, req.ParentRef, req.Tree, req.Start, req.Asc, limit) case cstypes.ObjectKindProject: - cssecrets, _, err = h.configstoreClient.GetProjectSecrets(ctx, req.ParentRef, req.Tree) + cssecrets, _, err = h.configstoreClient.GetProjectSecrets(ctx, req.ParentRef, req.Tree, req.Start, req.Asc, limit) } if err != nil { return nil, util.NewAPIError(util.KindFromRemoteError(err), err) @@ -48,7 +56,11 @@ func (h *ActionHandler) GetSecrets(ctx context.Context, req *GetSecretsRequest) if req.RemoveOverridden { // remove overriden secrets - cssecrets = common.FilterOverriddenSecrets(cssecrets) + cssecrets.Secrets = common.FilterOverriddenSecrets(cssecrets.Secrets) + } + + if limit > 0 && len(cssecrets.Secrets) > limit { + cssecrets.Secrets = cssecrets.Secrets[:limit] } return cssecrets, nil diff --git a/internal/services/gateway/action/user.go b/internal/services/gateway/action/user.go index 1c043e9eb..7c78fdc02 100644 --- a/internal/services/gateway/action/user.go +++ b/internal/services/gateway/action/user.go @@ -53,6 +53,11 @@ type PrivateUserResponse struct { LinkedAccounts []*cstypes.LinkedAccount } +type PrivateUsersResponse struct { + Users []*PrivateUserResponse + HasMoreData bool +} + func (h *ActionHandler) GetCurrentUser(ctx context.Context, userRef string) (*PrivateUserResponse, error) { if !common.IsUserLoggedOrAdmin(ctx) { return nil, errors.Errorf("user not logged in") @@ -106,7 +111,7 @@ type GetUsersRequest struct { Asc bool } -func (h *ActionHandler) GetUsers(ctx context.Context, req *GetUsersRequest) ([]*PrivateUserResponse, error) { +func (h *ActionHandler) GetUsers(ctx context.Context, req *GetUsersRequest) (*PrivateUsersResponse, error) { if !common.IsUserAdmin(ctx) { return nil, util.NewAPIError(util.ErrUnauthorized, errors.Errorf("user not admin")) } @@ -116,8 +121,8 @@ func (h *ActionHandler) GetUsers(ctx context.Context, req *GetUsersRequest) ([]* return nil, util.NewAPIError(util.KindFromRemoteError(err), err) } - users := make([]*PrivateUserResponse, len(csusers)) - for i, user := range csusers { + users := make([]*PrivateUserResponse, len(csusers.Users)) + for i, user := range csusers.Users { tokens, _, err := h.configstoreClient.GetUserTokens(ctx, user.ID) if err != nil { return nil, util.NewAPIError(util.KindFromRemoteError(err), errors.Wrapf(err, "failed to get user %q tokens", user.ID)) @@ -131,7 +136,12 @@ func (h *ActionHandler) GetUsers(ctx context.Context, req *GetUsersRequest) ([]* users[i] = &PrivateUserResponse{User: user, Tokens: tokens, LinkedAccounts: linkedAccounts} } - return users, nil + res := &PrivateUsersResponse{ + Users: users, + HasMoreData: csusers.HasMoreData, + } + + return res, nil } type CreateUserRequest struct { diff --git a/internal/services/gateway/action/variable.go b/internal/services/gateway/action/variable.go index ff293f82f..cdb3e33ec 100644 --- a/internal/services/gateway/action/variable.go +++ b/internal/services/gateway/action/variable.go @@ -31,30 +31,38 @@ type GetVariablesRequest struct { Tree bool RemoveOverridden bool + Start string + Asc bool + Limit int } -func (h *ActionHandler) GetVariables(ctx context.Context, req *GetVariablesRequest) ([]*csapitypes.Variable, []*csapitypes.Secret, error) { - var csvars []*csapitypes.Variable - var cssecrets []*csapitypes.Secret +func (h *ActionHandler) GetVariables(ctx context.Context, req *GetVariablesRequest) (*csapitypes.VariablesResponse, []*csapitypes.Secret, error) { + limit := req.Limit + if req.RemoveOverridden { + limit = 0 + } + + var csvars *csapitypes.VariablesResponse + var cssecrets *csapitypes.SecretsResponse switch req.ParentType { case cstypes.ObjectKindProjectGroup: var err error - csvars, _, err = h.configstoreClient.GetProjectGroupVariables(ctx, req.ParentRef, req.Tree) + csvars, _, err = h.configstoreClient.GetProjectGroupVariables(ctx, req.ParentRef, req.Tree, req.Start, req.Asc, limit) if err != nil { return nil, nil, util.NewAPIError(util.KindFromRemoteError(err), err) } - cssecrets, _, err = h.configstoreClient.GetProjectGroupSecrets(ctx, req.ParentRef, true) + cssecrets, _, err = h.configstoreClient.GetProjectGroupSecrets(ctx, req.ParentRef, true, "", false, 0) if err != nil { return nil, nil, util.NewAPIError(util.KindFromRemoteError(err), err) } case cstypes.ObjectKindProject: var err error - csvars, _, err = h.configstoreClient.GetProjectVariables(ctx, req.ParentRef, req.Tree) + csvars, _, err = h.configstoreClient.GetProjectVariables(ctx, req.ParentRef, req.Tree, req.Start, req.Asc, limit) if err != nil { return nil, nil, util.NewAPIError(util.KindFromRemoteError(err), err) } - cssecrets, _, err = h.configstoreClient.GetProjectSecrets(ctx, req.ParentRef, true) + cssecrets, _, err = h.configstoreClient.GetProjectSecrets(ctx, req.ParentRef, true, "", false, 0) if err != nil { return nil, nil, util.NewAPIError(util.KindFromRemoteError(err), err) } @@ -62,10 +70,14 @@ func (h *ActionHandler) GetVariables(ctx context.Context, req *GetVariablesReque if req.RemoveOverridden { // remove overriden variables - csvars = common.FilterOverriddenVariables(csvars) + csvars.Variables = common.FilterOverriddenVariables(csvars.Variables) + } + + if limit > 0 && len(csvars.Variables) > limit { + csvars.Variables = csvars.Variables[:limit] } - return csvars, cssecrets, nil + return csvars, cssecrets.Secrets, nil } type CreateVariableRequest struct { @@ -99,13 +111,13 @@ func (h *ActionHandler) CreateVariable(ctx context.Context, req *CreateVariableR Values: req.Values, } - var cssecrets []*csapitypes.Secret + var cssecrets *csapitypes.SecretsResponse var rv *csapitypes.Variable switch req.ParentType { case cstypes.ObjectKindProjectGroup: var err error - cssecrets, _, err = h.configstoreClient.GetProjectGroupSecrets(ctx, req.ParentRef, true) + cssecrets, _, err = h.configstoreClient.GetProjectGroupSecrets(ctx, req.ParentRef, true, "", false, 0) if err != nil { return nil, nil, util.NewAPIError(util.KindFromRemoteError(err), errors.Wrapf(err, "failed to get project group %q secrets", req.ParentRef)) } @@ -117,7 +129,7 @@ func (h *ActionHandler) CreateVariable(ctx context.Context, req *CreateVariableR } case cstypes.ObjectKindProject: var err error - cssecrets, _, err = h.configstoreClient.GetProjectSecrets(ctx, req.ParentRef, true) + cssecrets, _, err = h.configstoreClient.GetProjectSecrets(ctx, req.ParentRef, true, "", false, 0) if err != nil { return nil, nil, util.NewAPIError(util.KindFromRemoteError(err), errors.Wrapf(err, "failed to get project %q secrets", req.ParentRef)) } @@ -130,7 +142,7 @@ func (h *ActionHandler) CreateVariable(ctx context.Context, req *CreateVariableR } h.log.Info().Msgf("variable %s created, ID: %s", rv.Name, rv.ID) - return rv, cssecrets, nil + return rv, cssecrets.Secrets, nil } type UpdateVariableRequest struct { @@ -166,13 +178,13 @@ func (h *ActionHandler) UpdateVariable(ctx context.Context, req *UpdateVariableR Values: req.Values, } - var cssecrets []*csapitypes.Secret + var cssecrets *csapitypes.SecretsResponse var rv *csapitypes.Variable switch req.ParentType { case cstypes.ObjectKindProjectGroup: var err error - cssecrets, _, err = h.configstoreClient.GetProjectGroupSecrets(ctx, req.ParentRef, true) + cssecrets, _, err = h.configstoreClient.GetProjectGroupSecrets(ctx, req.ParentRef, true, "", false, 0) if err != nil { return nil, nil, util.NewAPIError(util.KindFromRemoteError(err), errors.Wrapf(err, "failed to get project group %q secrets", req.ParentRef)) } @@ -184,7 +196,7 @@ func (h *ActionHandler) UpdateVariable(ctx context.Context, req *UpdateVariableR } case cstypes.ObjectKindProject: var err error - cssecrets, _, err = h.configstoreClient.GetProjectSecrets(ctx, req.ParentRef, true) + cssecrets, _, err = h.configstoreClient.GetProjectSecrets(ctx, req.ParentRef, true, "", false, 0) if err != nil { return nil, nil, util.NewAPIError(util.KindFromRemoteError(err), errors.Wrapf(err, "failed to get project %q secrets", req.ParentRef)) } @@ -197,7 +209,7 @@ func (h *ActionHandler) UpdateVariable(ctx context.Context, req *UpdateVariableR } h.log.Info().Msgf("variable %s created, ID: %s", rv.Name, rv.ID) - return rv, cssecrets, nil + return rv, cssecrets.Secrets, nil } func (h *ActionHandler) DeleteVariable(ctx context.Context, parentType cstypes.ObjectKind, parentRef, name string) error { diff --git a/internal/services/gateway/api/org.go b/internal/services/gateway/api/org.go index f7bc5ab8b..e0213eb43 100644 --- a/internal/services/gateway/api/org.go +++ b/internal/services/gateway/api/org.go @@ -15,6 +15,7 @@ package api import ( + "encoding/base64" "encoding/json" "net/http" "strconv" @@ -171,6 +172,16 @@ func createOrgResponse(o *cstypes.Organization) *gwapitypes.OrgResponse { return org } +type OrgsCursor struct { + LastOrgID string + Asc bool +} + +const ( + DefaultOrgsLimit = 10 + MaxOrgsLimit = 20 +) + type OrgsHandler struct { log zerolog.Logger ah *action.ActionHandler @@ -184,8 +195,45 @@ func (h *OrgsHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { ctx := r.Context() query := r.URL.Query() + cursorS := query.Get("cursor") + var start string + var asc bool + + if cursorS != "" { + decodedCursor, err := base64.StdEncoding.DecodeString(cursorS) + if err != nil { + util.HTTPError(w, util.NewAPIError(util.ErrBadRequest, errors.Wrapf(err, "cannot decode cursor"))) + return + } + + var cursor OrgsCursor + if err := json.Unmarshal(decodedCursor, &cursor); err != nil { + util.HTTPError(w, util.NewAPIError(util.ErrBadRequest, errors.Wrapf(err, "cannot unmarshal cursor"))) + return + } + + asc = cursor.Asc + + org, err := h.ah.GetOrg(ctx, cursor.LastOrgID) + if util.HTTPError(w, err) { + h.log.Err(err).Send() + return + } + if org == nil { + util.HTTPError(w, util.NewAPIError(util.ErrBadRequest, errors.Wrapf(err, "cursor not valid"))) + return + } + start = org.Name + } else { + if _, ok := query["asc"]; ok { + asc = true + } + + start = query.Get("start") + } + + limit := DefaultOrgsLimit limitS := query.Get("limit") - limit := DefaultRunsLimit if limitS != "" { var err error limit, err = strconv.Atoi(limitS) @@ -198,15 +246,9 @@ func (h *OrgsHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { util.HTTPError(w, util.NewAPIError(util.ErrBadRequest, errors.Errorf("limit must be greater or equal than 0"))) return } - if limit > MaxRunsLimit { - limit = MaxRunsLimit + if limit > MaxOrgsLimit { + limit = MaxOrgsLimit } - asc := false - if _, ok := query["asc"]; ok { - asc = true - } - - start := query.Get("start") areq := &action.GetOrgsRequest{ Start: start, @@ -219,11 +261,29 @@ func (h *OrgsHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } - orgs := make([]*gwapitypes.OrgResponse, len(csorgs)) - for i, p := range csorgs { + cursorS = "" + if csorgs.HasMoreData { + cursor := OrgsCursor{ + LastOrgID: csorgs.Orgs[limit-1].ID, + Asc: asc, + } + serializedCursor, err := json.Marshal(&cursor) + if err != nil { + util.HTTPError(w, util.NewAPIError(util.ErrInternal, errors.Wrapf(err, "cannot marshal cursor"))) + return + } + cursorS = base64.StdEncoding.EncodeToString(serializedCursor) + } + + orgs := make([]*gwapitypes.OrgResponse, len(csorgs.Orgs)) + for i, p := range csorgs.Orgs { orgs[i] = createOrgResponse(p) } - if err := util.HTTPResponse(w, http.StatusOK, orgs); err != nil { + response := &gwapitypes.OrgsResponse{ + Orgs: orgs, + Cursor: cursorS, + } + if err := util.HTTPResponse(w, http.StatusOK, response); err != nil { h.log.Err(err).Send() } } @@ -235,6 +295,11 @@ func createOrgMemberResponse(user *cstypes.User, role cstypes.MemberRole) *gwapi } } +type OrgMembersCursor struct { + LastOrgUserID string + Asc bool +} + type OrgMembersHandler struct { log zerolog.Logger ah *action.ActionHandler @@ -246,22 +311,89 @@ func NewOrgMembersHandler(log zerolog.Logger, ah *action.ActionHandler) *OrgMemb func (h *OrgMembersHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { ctx := r.Context() + query := r.URL.Query() vars := mux.Vars(r) orgRef := vars["orgref"] - ares, err := h.ah.GetOrgMembers(ctx, orgRef) + cursorS := query.Get("cursor") + var asc bool + var start string + + if cursorS != "" { + decodedCursor, err := base64.StdEncoding.DecodeString(cursorS) + if err != nil { + util.HTTPError(w, util.NewAPIError(util.ErrBadRequest, errors.Wrapf(err, "cannot decode cursor"))) + return + } + + var cursor OrgMembersCursor + if err := json.Unmarshal(decodedCursor, &cursor); err != nil { + util.HTTPError(w, util.NewAPIError(util.ErrBadRequest, errors.Wrapf(err, "cannot unmarshal cursor"))) + return + } + + user, err := h.ah.GetUser(ctx, cursor.LastOrgUserID) + if util.HTTPError(w, err) { + h.log.Err(err).Send() + return + } + if user == nil { + util.HTTPError(w, util.NewAPIError(util.ErrBadRequest, errors.Wrapf(err, "cursor not valid"))) + return + } + asc = cursor.Asc + start = user.Name + } else { + if _, ok := query["asc"]; ok { + asc = true + } + + start = query.Get("start") + } + + var limit int + limitS := query.Get("limit") + if limitS != "" { + var err error + limit, err = strconv.Atoi(limitS) + if err != nil { + util.HTTPError(w, util.NewAPIError(util.ErrBadRequest, errors.Wrapf(err, "cannot parse limit"))) + return + } + } + if limit < 0 { + util.HTTPError(w, util.NewAPIError(util.ErrBadRequest, errors.Errorf("limit must be greater or equal than 0"))) + return + } + + ares, err := h.ah.GetOrgMembers(ctx, orgRef, start, limit, asc) if util.HTTPError(w, err) { h.log.Err(err).Send() return } + cursorS = "" + if ares.HasMoreData { + cursor := OrgMembersCursor{ + LastOrgUserID: ares.Members[limit-1].User.ID, + Asc: asc, + } + serializedCursor, err := json.Marshal(&cursor) + if err != nil { + util.HTTPError(w, util.NewAPIError(util.ErrInternal, errors.Wrapf(err, "cannot marshal cursor"))) + return + } + cursorS = base64.StdEncoding.EncodeToString(serializedCursor) + } + res := &gwapitypes.OrgMembersResponse{ Organization: createOrgResponse(ares.Organization), - Members: make([]*gwapitypes.OrgMemberResponse, len(ares.Members)), + OrgMembers: make([]*gwapitypes.OrgMemberResponse, len(ares.Members)), + Cursor: cursorS, } for i, m := range ares.Members { - res.Members[i] = createOrgMemberResponse(m.User, m.Role) + res.OrgMembers[i] = createOrgMemberResponse(m.User, m.Role) } if err := util.HTTPResponse(w, http.StatusOK, res); err != nil { h.log.Err(err).Send() diff --git a/internal/services/gateway/api/projectgroup.go b/internal/services/gateway/api/projectgroup.go index bc5d92776..ef6c5ffc7 100644 --- a/internal/services/gateway/api/projectgroup.go +++ b/internal/services/gateway/api/projectgroup.go @@ -15,9 +15,11 @@ package api import ( + "encoding/base64" "encoding/json" "net/http" "net/url" + "strconv" "github.com/gorilla/mux" "github.com/rs/zerolog" @@ -182,6 +184,10 @@ func (h *ProjectGroupHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) } } +type ProjectGroupProjectsCursor struct { + LastProjectID string +} + type ProjectGroupProjectsHandler struct { log zerolog.Logger ah *action.ActionHandler @@ -194,24 +200,88 @@ func NewProjectGroupProjectsHandler(log zerolog.Logger, ah *action.ActionHandler func (h *ProjectGroupProjectsHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { ctx := r.Context() vars := mux.Vars(r) + query := r.URL.Query() projectGroupRef, err := url.PathUnescape(vars["projectgroupref"]) if err != nil { util.HTTPError(w, util.NewAPIError(util.ErrBadRequest, err)) return } - csprojects, err := h.ah.GetProjectGroupProjects(ctx, projectGroupRef) + var start string + + cursorS := query.Get("cursor") + if cursorS != "" { + decodedCursor, err := base64.StdEncoding.DecodeString(cursorS) + if err != nil { + util.HTTPError(w, util.NewAPIError(util.ErrBadRequest, errors.Wrapf(err, "cannot decode cursor"))) + return + } + + var cursor ProjectGroupProjectsCursor + if err := json.Unmarshal(decodedCursor, &cursor); err != nil { + util.HTTPError(w, util.NewAPIError(util.ErrBadRequest, errors.Wrapf(err, "cannot unmarshal cursor"))) + return + } + + project, err := h.ah.GetProject(ctx, cursor.LastProjectID) + if util.HTTPError(w, err) { + h.log.Err(err).Send() + return + } + if project == nil { + util.HTTPError(w, util.NewAPIError(util.ErrBadRequest, errors.Wrapf(err, "cursor not valid"))) + return + } + start = project.Name + } else { + start = query.Get("start") + } + + var limit int + limitS := query.Get("limit") + if limitS != "" { + var err error + limit, err = strconv.Atoi(limitS) + if err != nil { + util.HTTPError(w, util.NewAPIError(util.ErrBadRequest, errors.Wrapf(err, "cannot parse limit"))) + return + } + } + + if limit < 0 { + util.HTTPError(w, util.NewAPIError(util.ErrBadRequest, errors.Errorf("limit must be greater or equal than 0"))) + return + } + + csprojects, err := h.ah.GetProjectGroupProjects(ctx, projectGroupRef, start, limit) if util.HTTPError(w, err) { h.log.Err(err).Send() return } - projects := make([]*gwapitypes.ProjectResponse, len(csprojects)) - for i, p := range csprojects { + cursorS = "" + if csprojects.HasMoreData { + cursor := ProjectGroupProjectsCursor{ + LastProjectID: csprojects.Projects[limit-1].ID, + } + serializedCursor, err := json.Marshal(&cursor) + if err != nil { + util.HTTPError(w, util.NewAPIError(util.ErrInternal, errors.Wrapf(err, "cannot marshal cursor"))) + return + } + cursorS = base64.StdEncoding.EncodeToString(serializedCursor) + } + + projects := make([]*gwapitypes.ProjectResponse, len(csprojects.Projects)) + for i, p := range csprojects.Projects { projects[i] = createProjectResponse(p) } - if err := util.HTTPResponse(w, http.StatusOK, projects); err != nil { + response := gwapitypes.ProjectsResponse{ + Projects: projects, + Cursor: cursorS, + } + if err := util.HTTPResponse(w, http.StatusOK, response); err != nil { h.log.Err(err).Send() } } diff --git a/internal/services/gateway/api/remotesource.go b/internal/services/gateway/api/remotesource.go index 8a620ea1d..3e2064d6e 100644 --- a/internal/services/gateway/api/remotesource.go +++ b/internal/services/gateway/api/remotesource.go @@ -15,6 +15,7 @@ package api import ( + "encoding/base64" "encoding/json" "net/http" "strconv" @@ -156,6 +157,16 @@ func (h *RemoteSourceHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) } } +type RemoteSourcesCursor struct { + LastRemoteSourceID string + Asc bool +} + +const ( + DefaultRemoteSourcesLimit = 25 + MaxRemoteSourcesLimit = 40 +) + type RemoteSourcesHandler struct { log zerolog.Logger ah *action.ActionHandler @@ -169,8 +180,43 @@ func (h *RemoteSourcesHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) ctx := r.Context() query := r.URL.Query() + cursorS := query.Get("cursor") + var start string + var asc bool + + if cursorS != "" { + decodedCursor, err := base64.StdEncoding.DecodeString(cursorS) + if err != nil { + util.HTTPError(w, util.NewAPIError(util.ErrBadRequest, errors.Wrapf(err, "cannot decode cursor"))) + return + } + + var cursor RemoteSourcesCursor + if err := json.Unmarshal(decodedCursor, &cursor); err != nil { + util.HTTPError(w, util.NewAPIError(util.ErrBadRequest, errors.Wrapf(err, "cannot unmarshal cursor"))) + return + } + + remoteSource, err := h.ah.GetRemoteSource(ctx, cursor.LastRemoteSourceID) + if util.HTTPError(w, err) { + h.log.Err(err).Send() + return + } + if remoteSource == nil { + util.HTTPError(w, util.NewAPIError(util.ErrBadRequest, errors.Wrapf(err, "cursor not valid"))) + return + } + start = remoteSource.Name + asc = cursor.Asc + } else { + if _, ok := query["asc"]; ok { + asc = true + } + start = query.Get("start") + } + + limit := DefaultRemoteSourcesLimit limitS := query.Get("limit") - limit := DefaultRunsLimit if limitS != "" { var err error limit, err = strconv.Atoi(limitS) @@ -183,16 +229,10 @@ func (h *RemoteSourcesHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) util.HTTPError(w, util.NewAPIError(util.ErrBadRequest, errors.Errorf("limit must be greater or equal than 0"))) return } - if limit > MaxRunsLimit { - limit = MaxRunsLimit - } - asc := false - if _, ok := query["asc"]; ok { - asc = true + if limit > MaxRemoteSourcesLimit { + limit = MaxRemoteSourcesLimit } - start := query.Get("start") - areq := &action.GetRemoteSourcesRequest{ Start: start, Limit: limit, @@ -204,12 +244,30 @@ func (h *RemoteSourcesHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) return } - remoteSources := make([]*gwapitypes.RemoteSourceResponse, len(csRemoteSources)) - for i, rs := range csRemoteSources { + cursorS = "" + if csRemoteSources.HasMoreData { + cursor := RemoteSourcesCursor{ + LastRemoteSourceID: csRemoteSources.RemoteSources[limit-1].ID, + Asc: asc, + } + serializedCursor, err := json.Marshal(&cursor) + if err != nil { + util.HTTPError(w, util.NewAPIError(util.ErrInternal, errors.Wrapf(err, "cannot marshal cursor"))) + return + } + cursorS = base64.StdEncoding.EncodeToString(serializedCursor) + } + + remoteSources := make([]*gwapitypes.RemoteSourceResponse, len(csRemoteSources.RemoteSources)) + for i, rs := range csRemoteSources.RemoteSources { remoteSources[i] = createRemoteSourceResponse(rs) } - if err := util.HTTPResponse(w, http.StatusOK, remoteSources); err != nil { + response := &gwapitypes.RemoteSourcesResponse{ + RemoteSources: remoteSources, + Cursor: cursorS, + } + if err := util.HTTPResponse(w, http.StatusOK, response); err != nil { h.log.Err(err).Send() } } diff --git a/internal/services/gateway/api/run.go b/internal/services/gateway/api/run.go index 3a7e7593b..b21f2ddbb 100644 --- a/internal/services/gateway/api/run.go +++ b/internal/services/gateway/api/run.go @@ -15,6 +15,7 @@ package api import ( + "encoding/base64" "encoding/json" "io" "net/http" @@ -286,8 +287,8 @@ const ( MaxRunsLimit = 40 ) -func createRunsResponse(r *rstypes.Run) *gwapitypes.RunsResponse { - run := &gwapitypes.RunsResponse{ +func createRunsResponse(r *rstypes.Run) *gwapitypes.Runs { + run := &gwapitypes.Runs{ Number: r.Counter, Name: r.Name, Annotations: r.Annotations, @@ -304,6 +305,14 @@ func createRunsResponse(r *rstypes.Run) *gwapitypes.RunsResponse { return run } +type RunsCursor struct { + LastRunNumber uint64 + Asc bool + SubGroup string + PhaseFilter []string + ResultFilter []string +} + type RunsHandler struct { log zerolog.Logger ah *action.ActionHandler @@ -332,12 +341,54 @@ func (h *RunsHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { ref = vars["userref"] } - subGroup := q.Get("subgroup") - phaseFilter := q["phase"] - resultFilter := q["result"] + cursorS := q.Get("cursor") + var startRunNumber uint64 + var asc bool + var subGroup string + var phaseFilter []string + var resultFilter []string + + if cursorS != "" { + decodedCursor, err := base64.StdEncoding.DecodeString(cursorS) + if err != nil { + util.HTTPError(w, util.NewAPIError(util.ErrBadRequest, errors.Wrapf(err, "cannot decode cursor"))) + return + } + + var cursor RunsCursor + if err := json.Unmarshal(decodedCursor, &cursor); err != nil { + util.HTTPError(w, util.NewAPIError(util.ErrBadRequest, errors.Wrapf(err, "cannot unmarshal cursor"))) + return + } + + startRunNumber = cursor.LastRunNumber + asc = cursor.Asc + subGroup = cursor.SubGroup + phaseFilter = cursor.PhaseFilter + resultFilter = cursor.ResultFilter + } else { + subGroup = q.Get("subgroup") + phaseFilter = q["phase"] + resultFilter = q["result"] + + if _, ok := q["asc"]; ok { + asc = true + } + + startRunNumberStr := q.Get("start") + + if startRunNumberStr != "" { + var err error + startRunNumber, err = strconv.ParseUint(startRunNumberStr, 10, 64) + if err != nil { + util.HTTPError(w, util.NewAPIError(util.ErrBadRequest, errors.Wrapf(err, "cannot parse run number"))) + return + } + } + } - limitS := q.Get("limit") limit := DefaultRunsLimit + limitS := q.Get("limit") if limitS != "" { var err error limit, err = strconv.Atoi(limitS) @@ -353,22 +404,6 @@ func (h *RunsHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { if limit > MaxRunsLimit { limit = MaxRunsLimit } - asc := false - if _, ok := q["asc"]; ok { - asc = true - } - - startRunNumberStr := q.Get("start") - - var startRunNumber uint64 - if startRunNumberStr != "" { - var err error - startRunNumber, err = strconv.ParseUint(startRunNumberStr, 10, 64) - if err != nil { - util.HTTPError(w, util.NewAPIError(util.ErrBadRequest, errors.Wrapf(err, "cannot parse run number"))) - return - } - } areq := &action.GetRunsRequest{ GroupType: h.groupType, @@ -386,11 +421,32 @@ func (h *RunsHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } - runs := make([]*gwapitypes.RunsResponse, len(runsResp.Runs)) + cursorS = "" + if runsResp.HasMoreData { + cursor := RunsCursor{ + LastRunNumber: runsResp.Runs[limit-1].Counter, + Asc: asc, + SubGroup: subGroup, + PhaseFilter: phaseFilter, + ResultFilter: resultFilter, + } + serializedCursor, err := json.Marshal(&cursor) + if err != nil { + util.HTTPError(w, util.NewAPIError(util.ErrInternal, errors.Wrapf(err, "cannot marshal cursor"))) + return + } + cursorS = base64.StdEncoding.EncodeToString(serializedCursor) + } + + runs := make([]*gwapitypes.Runs, len(runsResp.Runs)) for i, r := range runsResp.Runs { runs[i] = createRunsResponse(r) } - if err := util.HTTPResponse(w, http.StatusOK, runs); err != nil { + response := &gwapitypes.RunsResponse{ + Runs: runs, + Cursor: cursorS, + } + if err := util.HTTPResponse(w, http.StatusOK, response); err != nil { h.log.Err(err).Send() } } diff --git a/internal/services/gateway/api/secret.go b/internal/services/gateway/api/secret.go index 65af6107d..ded950819 100644 --- a/internal/services/gateway/api/secret.go +++ b/internal/services/gateway/api/secret.go @@ -15,11 +15,14 @@ package api import ( + "encoding/base64" "encoding/json" "net/http" + "strconv" "github.com/gorilla/mux" "github.com/rs/zerolog" + "github.com/sorintlab/errors" "agola.io/agola/internal/services/gateway/action" "agola.io/agola/internal/util" @@ -36,6 +39,13 @@ func createSecretResponse(s *csapitypes.Secret) *gwapitypes.SecretResponse { } } +type SecretsCursor struct { + LastSecretName string + Asc bool + Tree bool + RemoveOverridden bool +} + type SecretHandler struct { log zerolog.Logger ah *action.ActionHandler @@ -48,8 +58,55 @@ func NewSecretHandler(log zerolog.Logger, ah *action.ActionHandler) *SecretHandl func (h *SecretHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { ctx := r.Context() query := r.URL.Query() - _, tree := query["tree"] - _, removeoverridden := query["removeoverridden"] + + cursorS := query.Get("cursor") + var start string + var asc bool + var tree bool + var removeoverridden bool + + if cursorS != "" { + decodedCursor, err := base64.StdEncoding.DecodeString(cursorS) + if err != nil { + util.HTTPError(w, util.NewAPIError(util.ErrBadRequest, errors.Wrapf(err, "cannot decode cursor"))) + return + } + + var cursor SecretsCursor + if err := json.Unmarshal(decodedCursor, &cursor); err != nil { + util.HTTPError(w, util.NewAPIError(util.ErrBadRequest, errors.Wrapf(err, "cannot unmarshal cursor"))) + return + } + + start = cursor.LastSecretName + asc = cursor.Asc + tree = cursor.Tree + removeoverridden = cursor.RemoveOverridden + } else { + if _, ok := query["asc"]; ok { + asc = true + } + + start = query.Get("start") + + _, tree = query["tree"] + _, removeoverridden = query["removeoverridden"] + } + + var limit int + limitS := query.Get("limit") + if limitS != "" { + var err error + limit, err = strconv.Atoi(limitS) + if err != nil { + util.HTTPError(w, util.NewAPIError(util.ErrBadRequest, errors.Wrapf(err, "cannot parse limit"))) + return + } + } + if limit < 0 { + util.HTTPError(w, util.NewAPIError(util.ErrBadRequest, errors.Errorf("limit must be greater or equal than 0"))) + return + } parentType, parentRef, err := GetConfigTypeRef(r) if util.HTTPError(w, err) { @@ -60,6 +117,9 @@ func (h *SecretHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { areq := &action.GetSecretsRequest{ ParentType: parentType, ParentRef: parentRef, + Start: start, + Asc: asc, + Limit: limit, Tree: tree, RemoveOverridden: removeoverridden, } @@ -69,12 +129,32 @@ func (h *SecretHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } - secrets := make([]*gwapitypes.SecretResponse, len(cssecrets)) - for i, s := range cssecrets { + cursorS = "" + if cssecrets.HasMoreData { + cursor := SecretsCursor{ + LastSecretName: cssecrets.Secrets[limit-1].Name, + Asc: asc, + Tree: tree, + RemoveOverridden: removeoverridden, + } + serializedCursor, err := json.Marshal(&cursor) + if err != nil { + util.HTTPError(w, util.NewAPIError(util.ErrInternal, errors.Wrapf(err, "cannot marshal cursor"))) + return + } + cursorS = base64.StdEncoding.EncodeToString(serializedCursor) + } + + secrets := make([]*gwapitypes.SecretResponse, len(cssecrets.Secrets)) + for i, s := range cssecrets.Secrets { secrets[i] = createSecretResponse(s) } - if err := util.HTTPResponse(w, http.StatusOK, secrets); err != nil { + response := &gwapitypes.SecretsResponse{ + Secrets: secrets, + Cursor: cursorS, + } + if err := util.HTTPResponse(w, http.StatusOK, response); err != nil { h.log.Err(err).Send() } } diff --git a/internal/services/gateway/api/user.go b/internal/services/gateway/api/user.go index e1102c00d..7d3ff02f4 100644 --- a/internal/services/gateway/api/user.go +++ b/internal/services/gateway/api/user.go @@ -16,6 +16,7 @@ package api import ( "context" + "encoding/base64" "encoding/json" "net/http" "sort" @@ -182,6 +183,17 @@ func createUserResponse(u *cstypes.User) *gwapitypes.UserResponse { return user } +type UsersCursor struct { + LastUserID string + Asc bool + QueryType string +} + +const ( + DefaultUsersLimit = 25 + MaxUsersLimit = 40 +) + type UsersHandler struct { log zerolog.Logger ah *action.ActionHandler @@ -196,8 +208,47 @@ func (h *UsersHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { query := r.URL.Query() + cursorS := query.Get("cursor") + var start string + var asc bool + var queryType string + + if cursorS != "" { + decodedCursor, err := base64.StdEncoding.DecodeString(cursorS) + if err != nil { + util.HTTPError(w, util.NewAPIError(util.ErrBadRequest, errors.Wrapf(err, "cannot decode cursor"))) + return + } + + var cursor UsersCursor + if err := json.Unmarshal(decodedCursor, &cursor); err != nil { + util.HTTPError(w, util.NewAPIError(util.ErrBadRequest, errors.Wrapf(err, "cannot unmarshal cursor"))) + return + } + + user, err := h.ah.GetUser(ctx, cursor.LastUserID) + if util.HTTPError(w, err) { + h.log.Err(err).Send() + return + } + if user == nil { + util.HTTPError(w, util.NewAPIError(util.ErrBadRequest, errors.Wrapf(err, "cursor not valid"))) + return + } + start = user.Name + asc = cursor.Asc + queryType = cursor.QueryType + } else { + if _, ok := query["asc"]; ok { + asc = true + } + + start = query.Get("start") + queryType = query.Get("query_type") + } + + limit := DefaultUsersLimit limitS := query.Get("limit") - limit := DefaultRunsLimit if limitS != "" { var err error limit, err = strconv.Atoi(limitS) @@ -210,19 +261,12 @@ func (h *UsersHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { util.HTTPError(w, util.NewAPIError(util.ErrBadRequest, errors.Errorf("limit must be greater or equal than 0"))) return } - if limit > MaxRunsLimit { - limit = MaxRunsLimit + if limit > MaxUsersLimit { + limit = MaxUsersLimit } - asc := false - if _, ok := query["asc"]; ok { - asc = true - } - - start := query.Get("start") - queryType := query.Get("query_type") var ausers []*action.PrivateUserResponse - var err error + var hasMoreData bool switch queryType { case "byremoteuser": remoteUserID := query.Get("remoteuserid") @@ -240,22 +284,43 @@ func (h *UsersHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { Limit: limit, Asc: asc, } - ausers, err = h.ah.GetUsers(ctx, areq) + apusers, err := h.ah.GetUsers(ctx, areq) if util.HTTPError(w, err) { h.log.Err(err).Send() return } + ausers = apusers.Users + hasMoreData = apusers.HasMoreData default: util.HTTPError(w, util.NewAPIError(util.ErrBadRequest, errors.Errorf("unknown query_type: %q", queryType))) return } + cursorS = "" + if hasMoreData { + cursor := UsersCursor{ + LastUserID: ausers[limit-1].User.ID, + Asc: asc, + QueryType: queryType, + } + serializedCursor, err := json.Marshal(&cursor) + if err != nil { + util.HTTPError(w, util.NewAPIError(util.ErrInternal, errors.Wrapf(err, "cannot marshal cursor"))) + return + } + cursorS = base64.StdEncoding.EncodeToString(serializedCursor) + } + users := make([]*gwapitypes.PrivateUserResponse, len(ausers)) for i, p := range ausers { users[i] = createPrivateUserResponse(p.User, p.Tokens, p.LinkedAccounts) } - if err := util.HTTPResponse(w, http.StatusOK, users); err != nil { + response := &gwapitypes.PrivateUsersResponse{ + Users: users, + Cursor: cursorS, + } + if err := util.HTTPResponse(w, http.StatusOK, response); err != nil { h.log.Err(err).Send() } } diff --git a/internal/services/gateway/api/variable.go b/internal/services/gateway/api/variable.go index 5883ac94d..74c69470b 100644 --- a/internal/services/gateway/api/variable.go +++ b/internal/services/gateway/api/variable.go @@ -15,11 +15,14 @@ package api import ( + "encoding/base64" "encoding/json" "net/http" + "strconv" "github.com/gorilla/mux" "github.com/rs/zerolog" + "github.com/sorintlab/errors" "agola.io/agola/internal/services/common" "agola.io/agola/internal/services/gateway/action" @@ -53,6 +56,13 @@ func createVariableResponse(v *csapitypes.Variable, secrets []*csapitypes.Secret return nv } +type VariablesCursor struct { + LastVariableName string + Asc bool + Tree bool + RemoveOverridden bool +} + type VariableHandler struct { log zerolog.Logger ah *action.ActionHandler @@ -65,8 +75,55 @@ func NewVariableHandler(log zerolog.Logger, ah *action.ActionHandler) *VariableH func (h *VariableHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { ctx := r.Context() query := r.URL.Query() - _, tree := query["tree"] - _, removeoverridden := query["removeoverridden"] + + cursorS := query.Get("cursor") + var start string + var asc bool + var tree bool + var removeoverridden bool + + if cursorS != "" { + decodedCursor, err := base64.StdEncoding.DecodeString(cursorS) + if err != nil { + util.HTTPError(w, util.NewAPIError(util.ErrBadRequest, errors.Wrapf(err, "cannot decode cursor"))) + return + } + + var cursor VariablesCursor + if err := json.Unmarshal(decodedCursor, &cursor); err != nil { + util.HTTPError(w, util.NewAPIError(util.ErrBadRequest, errors.Wrapf(err, "cannot unmarshal cursor"))) + return + } + + start = cursor.LastVariableName + asc = cursor.Asc + tree = cursor.Tree + removeoverridden = cursor.RemoveOverridden + } else { + if _, ok := query["asc"]; ok { + asc = true + } + + start = query.Get("start") + + _, tree = query["tree"] + _, removeoverridden = query["removeoverridden"] + } + + var limit int + limitS := query.Get("limit") + if limitS != "" { + var err error + limit, err = strconv.Atoi(limitS) + if err != nil { + util.HTTPError(w, util.NewAPIError(util.ErrBadRequest, errors.Wrapf(err, "cannot parse limit"))) + return + } + } + if limit < 0 { + util.HTTPError(w, util.NewAPIError(util.ErrBadRequest, errors.Errorf("limit must be greater or equal than 0"))) + return + } parentType, parentRef, err := GetConfigTypeRef(r) if util.HTTPError(w, err) { @@ -77,6 +134,9 @@ func (h *VariableHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { areq := &action.GetVariablesRequest{ ParentType: parentType, ParentRef: parentRef, + Start: start, + Asc: asc, + Limit: limit, Tree: tree, RemoveOverridden: removeoverridden, } @@ -86,12 +146,32 @@ func (h *VariableHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } - variables := make([]*gwapitypes.VariableResponse, len(csvars)) - for i, v := range csvars { + cursorS = "" + if csvars.HasMoreData { + cursor := VariablesCursor{ + LastVariableName: csvars.Variables[limit-1].Name, + Asc: asc, + Tree: tree, + RemoveOverridden: removeoverridden, + } + serializedCursor, err := json.Marshal(&cursor) + if err != nil { + util.HTTPError(w, util.NewAPIError(util.ErrInternal, errors.Wrapf(err, "cannot marshal cursor"))) + return + } + cursorS = base64.StdEncoding.EncodeToString(serializedCursor) + } + + variables := make([]*gwapitypes.VariableResponse, len(csvars.Variables)) + for i, v := range csvars.Variables { variables[i] = createVariableResponse(v, cssecrets) } - if err := util.HTTPResponse(w, http.StatusOK, variables); err != nil { + response := &gwapitypes.VariablesResponse{ + Variables: variables, + Cursor: cursorS, + } + if err := util.HTTPResponse(w, http.StatusOK, response); err != nil { h.log.Err(err).Send() } } diff --git a/internal/services/runservice/api/api.go b/internal/services/runservice/api/api.go index 1ac725325..b2655535d 100644 --- a/internal/services/runservice/api/api.go +++ b/internal/services/runservice/api/api.go @@ -734,9 +734,15 @@ func (h *RunsByGroupHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { var runs []*types.Run var cgt *types.ChangeGroupsUpdateToken + var hasMoreData bool + aLimit := limit + if aLimit != 0 { + aLimit++ + } + err = h.d.Do(ctx, func(tx *sql.Tx) error { var err error - runs, err = h.d.GetGroupRuns(tx, group, phaseFilter, resultFilter, startRunCounter, limit, sortOrder) + runs, err = h.d.GetGroupRuns(tx, group, phaseFilter, resultFilter, startRunCounter, aLimit, sortOrder) if err != nil { h.log.Err(err).Send() return errors.WithStack(err) @@ -750,6 +756,11 @@ func (h *RunsByGroupHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } + if limit != 0 && len(runs) > limit { + hasMoreData = true + runs = runs[:limit] + } + cgts, err := types.MarshalChangeGroupsUpdateToken(cgt) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) @@ -759,6 +770,7 @@ func (h *RunsByGroupHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { res := &rsapitypes.GetRunsResponse{ Runs: runs, ChangeGroupsUpdateToken: cgts, + HasMoreData: hasMoreData, } if err := util.HTTPResponse(w, http.StatusOK, res); err != nil { h.log.Err(err).Send() diff --git a/services/configstore/api/types/org.go b/services/configstore/api/types/org.go index 7de2ce0ec..a8cad2ec1 100644 --- a/services/configstore/api/types/org.go +++ b/services/configstore/api/types/org.go @@ -15,24 +15,34 @@ package types import ( - cstypes "agola.io/agola/services/configstore/types" + "agola.io/agola/services/configstore/types" ) type CreateOrgRequest struct { Name string - Visibility cstypes.Visibility + Visibility types.Visibility CreatorUserID string } type AddOrgMemberRequest struct { - Role cstypes.MemberRole + Role types.MemberRole +} + +type OrgsResponse struct { + Orgs []*types.Organization + HasMoreData bool } type OrgMemberResponse struct { - User *cstypes.User - Role cstypes.MemberRole + User *types.User + Role types.MemberRole +} + +type OrgMembersResponse struct { + OrgMembers []*OrgMemberResponse + HasMoreData bool } type UpdateOrgRequest struct { - Visibility cstypes.Visibility + Visibility types.Visibility } diff --git a/services/configstore/api/types/project.go b/services/configstore/api/types/project.go index e04ce33a7..2b9d2df71 100644 --- a/services/configstore/api/types/project.go +++ b/services/configstore/api/types/project.go @@ -44,3 +44,8 @@ type Project struct { ParentPath string GlobalVisibility cstypes.Visibility } + +type ProjectsResponse struct { + Projects []*Project + HasMoreData bool +} diff --git a/services/configstore/api/types/remotesource.go b/services/configstore/api/types/remotesource.go index 6742176d6..628aadfba 100644 --- a/services/configstore/api/types/remotesource.go +++ b/services/configstore/api/types/remotesource.go @@ -15,15 +15,15 @@ package types import ( - cstypes "agola.io/agola/services/configstore/types" + "agola.io/agola/services/configstore/types" ) type CreateUpdateRemoteSourceRequest struct { Name string APIURL string SkipVerify bool - Type cstypes.RemoteSourceType - AuthType cstypes.RemoteSourceAuthType + Type types.RemoteSourceType + AuthType types.RemoteSourceAuthType Oauth2ClientID string Oauth2ClientSecret string SSHHostKey string @@ -31,3 +31,8 @@ type CreateUpdateRemoteSourceRequest struct { RegistrationEnabled bool LoginEnabled bool } + +type RemoteSourcesReponse struct { + RemoteSources []*types.RemoteSource + HasMoreData bool +} diff --git a/services/configstore/api/types/secret.go b/services/configstore/api/types/secret.go index 570ad713b..f9e96986f 100644 --- a/services/configstore/api/types/secret.go +++ b/services/configstore/api/types/secret.go @@ -33,3 +33,8 @@ type Secret struct { // dynamic data ParentPath string } + +type SecretsResponse struct { + Secrets []*Secret + HasMoreData bool +} diff --git a/services/configstore/api/types/user.go b/services/configstore/api/types/user.go index c8633650f..add864339 100644 --- a/services/configstore/api/types/user.go +++ b/services/configstore/api/types/user.go @@ -17,7 +17,7 @@ package types import ( "time" - cstypes "agola.io/agola/services/configstore/types" + "agola.io/agola/services/configstore/types" ) type CreateUserRequest struct { @@ -59,15 +59,20 @@ type CreateUserTokenResponse struct { } type UserOrgsResponse struct { - Organization *cstypes.Organization - Role cstypes.MemberRole + Organization *types.Organization + Role types.MemberRole } // User augments cstypes.User with the user tokens and linked accouts type User struct { - *cstypes.User + *types.User // dynamic data - Tokens []*cstypes.UserToken - LinkedAccounts []*cstypes.LinkedAccount + Tokens []*types.UserToken + LinkedAccounts []*types.LinkedAccount +} + +type PrivateUsersResponse struct { + Users []*types.User + HasMoreData bool } diff --git a/services/configstore/api/types/variable.go b/services/configstore/api/types/variable.go index c27a3e966..660b477b6 100644 --- a/services/configstore/api/types/variable.go +++ b/services/configstore/api/types/variable.go @@ -30,3 +30,8 @@ type Variable struct { // dynamic data ParentPath string } + +type VariablesResponse struct { + Variables []*Variable + HasMoreData bool +} diff --git a/services/configstore/client/client.go b/services/configstore/client/client.go index d0ffb9e87..de2988833 100644 --- a/services/configstore/client/client.go +++ b/services/configstore/client/client.go @@ -53,9 +53,17 @@ func (c *Client) GetProjectGroupSubgroups(ctx context.Context, projectGroupRef s return projectGroups, resp, errors.WithStack(err) } -func (c *Client) GetProjectGroupProjects(ctx context.Context, projectGroupRef string) ([]*csapitypes.Project, *http.Response, error) { - projects := []*csapitypes.Project{} - resp, err := c.GetParsedResponse(ctx, "GET", fmt.Sprintf("/projectgroups/%s/projects", url.PathEscape(projectGroupRef)), nil, common.JSONContent, nil, &projects) +func (c *Client) GetProjectGroupProjects(ctx context.Context, projectGroupRef string, start string, limit int) (*csapitypes.ProjectsResponse, *http.Response, error) { + q := url.Values{} + if start != "" { + q.Add("start", start) + } + if limit > 0 { + q.Add("limit", strconv.Itoa(limit)) + } + + projects := new(csapitypes.ProjectsResponse) + resp, err := c.GetParsedResponse(ctx, "GET", fmt.Sprintf("/projectgroups/%s/projects", url.PathEscape(projectGroupRef)), q, common.JSONContent, nil, projects) return projects, resp, errors.WithStack(err) } @@ -119,25 +127,43 @@ func (c *Client) DeleteProject(ctx context.Context, projectRef string) (*http.Re return resp, errors.WithStack(err) } -func (c *Client) GetProjectGroupSecrets(ctx context.Context, projectGroupRef string, tree bool) ([]*csapitypes.Secret, *http.Response, error) { +func (c *Client) GetProjectGroupSecrets(ctx context.Context, projectGroupRef string, tree bool, start string, asc bool, limit int) (*csapitypes.SecretsResponse, *http.Response, error) { q := url.Values{} if tree { q.Add("tree", "") } + if start != "" { + q.Add("start", start) + } + if limit > 0 { + q.Add("limit", strconv.Itoa(limit)) + } + if asc { + q.Add("asc", "") + } - secrets := []*csapitypes.Secret{} - resp, err := c.GetParsedResponse(ctx, "GET", fmt.Sprintf("/projectgroups/%s/secrets", url.PathEscape(projectGroupRef)), q, common.JSONContent, nil, &secrets) + secrets := new(csapitypes.SecretsResponse) + resp, err := c.GetParsedResponse(ctx, "GET", fmt.Sprintf("/projectgroups/%s/secrets", url.PathEscape(projectGroupRef)), q, common.JSONContent, nil, secrets) return secrets, resp, errors.WithStack(err) } -func (c *Client) GetProjectSecrets(ctx context.Context, projectRef string, tree bool) ([]*csapitypes.Secret, *http.Response, error) { +func (c *Client) GetProjectSecrets(ctx context.Context, projectRef string, tree bool, start string, asc bool, limit int) (*csapitypes.SecretsResponse, *http.Response, error) { q := url.Values{} if tree { q.Add("tree", "") } + if start != "" { + q.Add("start", start) + } + if limit > 0 { + q.Add("limit", strconv.Itoa(limit)) + } + if asc { + q.Add("asc", "") + } - secrets := []*csapitypes.Secret{} - resp, err := c.GetParsedResponse(ctx, "GET", fmt.Sprintf("/projects/%s/secrets", url.PathEscape(projectRef)), q, common.JSONContent, nil, &secrets) + secrets := new(csapitypes.SecretsResponse) + resp, err := c.GetParsedResponse(ctx, "GET", fmt.Sprintf("/projects/%s/secrets", url.PathEscape(projectRef)), q, common.JSONContent, nil, secrets) return secrets, resp, errors.WithStack(err) } @@ -195,25 +221,43 @@ func (c *Client) DeleteProjectSecret(ctx context.Context, projectRef, secretName return resp, errors.WithStack(err) } -func (c *Client) GetProjectGroupVariables(ctx context.Context, projectGroupRef string, tree bool) ([]*csapitypes.Variable, *http.Response, error) { +func (c *Client) GetProjectGroupVariables(ctx context.Context, projectGroupRef string, tree bool, start string, asc bool, limit int) (*csapitypes.VariablesResponse, *http.Response, error) { q := url.Values{} if tree { q.Add("tree", "") } + if start != "" { + q.Add("start", start) + } + if limit > 0 { + q.Add("limit", strconv.Itoa(limit)) + } + if asc { + q.Add("asc", "") + } - variables := []*csapitypes.Variable{} - resp, err := c.GetParsedResponse(ctx, "GET", fmt.Sprintf("/projectgroups/%s/variables", url.PathEscape(projectGroupRef)), q, common.JSONContent, nil, &variables) + variables := new(csapitypes.VariablesResponse) + resp, err := c.GetParsedResponse(ctx, "GET", fmt.Sprintf("/projectgroups/%s/variables", url.PathEscape(projectGroupRef)), q, common.JSONContent, nil, variables) return variables, resp, errors.WithStack(err) } -func (c *Client) GetProjectVariables(ctx context.Context, projectRef string, tree bool) ([]*csapitypes.Variable, *http.Response, error) { +func (c *Client) GetProjectVariables(ctx context.Context, projectRef string, tree bool, start string, asc bool, limit int) (*csapitypes.VariablesResponse, *http.Response, error) { q := url.Values{} if tree { q.Add("tree", "") } + if start != "" { + q.Add("start", start) + } + if limit > 0 { + q.Add("limit", strconv.Itoa(limit)) + } + if asc { + q.Add("asc", "") + } - variables := []*csapitypes.Variable{} - resp, err := c.GetParsedResponse(ctx, "GET", fmt.Sprintf("/projects/%s/variables", url.PathEscape(projectRef)), q, common.JSONContent, nil, &variables) + variables := new(csapitypes.VariablesResponse) + resp, err := c.GetParsedResponse(ctx, "GET", fmt.Sprintf("/projects/%s/variables", url.PathEscape(projectRef)), q, common.JSONContent, nil, variables) return variables, resp, errors.WithStack(err) } @@ -282,12 +326,12 @@ func (c *Client) GetUserByToken(ctx context.Context, token string) (*cstypes.Use q.Add("query_type", "bytoken") q.Add("token", token) - users := []*cstypes.User{} + users := new(csapitypes.PrivateUsersResponse) resp, err := c.GetParsedResponse(ctx, "GET", "/users", q, common.JSONContent, nil, &users) if err != nil { return nil, resp, errors.WithStack(err) } - return users[0], resp, errors.WithStack(err) + return users.Users[0], resp, errors.WithStack(err) } func (c *Client) GetUserByLinkedAccountRemoteUserAndSource(ctx context.Context, remoteUserID, remoteSourceID string) (*cstypes.User, *http.Response, error) { @@ -296,12 +340,12 @@ func (c *Client) GetUserByLinkedAccountRemoteUserAndSource(ctx context.Context, q.Add("remoteuserid", remoteUserID) q.Add("remotesourceid", remoteSourceID) - users := []*cstypes.User{} + users := new(csapitypes.PrivateUsersResponse) resp, err := c.GetParsedResponse(ctx, "GET", "/users", q, common.JSONContent, nil, &users) if err != nil { return nil, resp, errors.WithStack(err) } - return users[0], resp, errors.WithStack(err) + return users.Users[0], resp, errors.WithStack(err) } func (c *Client) GetUserByLinkedAccount(ctx context.Context, linkedAccountID string) (*cstypes.User, *http.Response, error) { @@ -309,12 +353,12 @@ func (c *Client) GetUserByLinkedAccount(ctx context.Context, linkedAccountID str q.Add("query_type", "bylinkedaccount") q.Add("linkedaccountid", linkedAccountID) - users := []*cstypes.User{} + users := new(csapitypes.PrivateUsersResponse) resp, err := c.GetParsedResponse(ctx, "GET", "/users", q, common.JSONContent, nil, &users) if err != nil { return nil, resp, errors.WithStack(err) } - return users[0], resp, errors.WithStack(err) + return users.Users[0], resp, errors.WithStack(err) } func (c *Client) CreateUser(ctx context.Context, req *csapitypes.CreateUserRequest) (*cstypes.User, *http.Response, error) { @@ -344,7 +388,7 @@ func (c *Client) DeleteUser(ctx context.Context, userRef string) (*http.Response return resp, errors.WithStack(err) } -func (c *Client) GetUsers(ctx context.Context, start string, limit int, asc bool) ([]*cstypes.User, *http.Response, error) { +func (c *Client) GetUsers(ctx context.Context, start string, limit int, asc bool) (*csapitypes.PrivateUsersResponse, *http.Response, error) { q := url.Values{} if start != "" { q.Add("start", start) @@ -356,8 +400,8 @@ func (c *Client) GetUsers(ctx context.Context, start string, limit int, asc bool q.Add("asc", "") } - users := []*cstypes.User{} - resp, err := c.GetParsedResponse(ctx, "GET", "/users", q, common.JSONContent, nil, &users) + users := new(csapitypes.PrivateUsersResponse) + resp, err := c.GetParsedResponse(ctx, "GET", "/users", q, common.JSONContent, nil, users) return users, resp, errors.WithStack(err) } @@ -428,7 +472,7 @@ func (c *Client) GetRemoteSource(ctx context.Context, rsRef string) (*cstypes.Re return rs, resp, errors.WithStack(err) } -func (c *Client) GetRemoteSources(ctx context.Context, start string, limit int, asc bool) ([]*cstypes.RemoteSource, *http.Response, error) { +func (c *Client) GetRemoteSources(ctx context.Context, start string, limit int, asc bool) (*csapitypes.RemoteSourcesReponse, *http.Response, error) { q := url.Values{} if start != "" { q.Add("start", start) @@ -440,8 +484,8 @@ func (c *Client) GetRemoteSources(ctx context.Context, start string, limit int, q.Add("asc", "") } - rss := []*cstypes.RemoteSource{} - resp, err := c.GetParsedResponse(ctx, "GET", "/remotesources", q, common.JSONContent, nil, &rss) + rss := new(csapitypes.RemoteSourcesReponse) + resp, err := c.GetParsedResponse(ctx, "GET", "/remotesources", q, common.JSONContent, nil, rss) return rss, resp, errors.WithStack(err) } @@ -532,7 +576,7 @@ func (c *Client) RemoveOrgMember(ctx context.Context, orgRef, userRef string) (* return resp, errors.WithStack(err) } -func (c *Client) GetOrgs(ctx context.Context, start string, limit int, asc bool) ([]*cstypes.Organization, *http.Response, error) { +func (c *Client) GetOrgs(ctx context.Context, start string, limit int, asc bool) (*csapitypes.OrgsResponse, *http.Response, error) { q := url.Values{} if start != "" { q.Add("start", start) @@ -544,8 +588,8 @@ func (c *Client) GetOrgs(ctx context.Context, start string, limit int, asc bool) q.Add("asc", "") } - orgs := []*cstypes.Organization{} - resp, err := c.GetParsedResponse(ctx, "GET", "/orgs", q, common.JSONContent, nil, &orgs) + orgs := new(csapitypes.OrgsResponse) + resp, err := c.GetParsedResponse(ctx, "GET", "/orgs", q, common.JSONContent, nil, orgs) return orgs, resp, errors.WithStack(err) } @@ -555,9 +599,20 @@ func (c *Client) GetOrg(ctx context.Context, orgRef string) (*cstypes.Organizati return org, resp, errors.WithStack(err) } -func (c *Client) GetOrgMembers(ctx context.Context, orgRef string) ([]*csapitypes.OrgMemberResponse, *http.Response, error) { - orgMembers := []*csapitypes.OrgMemberResponse{} - resp, err := c.GetParsedResponse(ctx, "GET", fmt.Sprintf("/orgs/%s/members", orgRef), nil, common.JSONContent, nil, &orgMembers) +func (c *Client) GetOrgMembers(ctx context.Context, orgRef string, start string, limit int, asc bool) (*csapitypes.OrgMembersResponse, *http.Response, error) { + q := url.Values{} + if start != "" { + q.Add("start", start) + } + if limit > 0 { + q.Add("limit", strconv.Itoa(limit)) + } + if asc { + q.Add("asc", "") + } + + orgMembers := new(csapitypes.OrgMembersResponse) + resp, err := c.GetParsedResponse(ctx, "GET", fmt.Sprintf("/orgs/%s/members", orgRef), q, common.JSONContent, nil, orgMembers) return orgMembers, resp, errors.WithStack(err) } diff --git a/services/gateway/api/types/org.go b/services/gateway/api/types/org.go index 06b63aaec..c3b9c12b1 100644 --- a/services/gateway/api/types/org.go +++ b/services/gateway/api/types/org.go @@ -32,13 +32,19 @@ type OrgResponse struct { Visibility Visibility `json:"visibility,omitempty"` } +type OrgsResponse struct { + Orgs []*OrgResponse `json:"orgs"` + Cursor string `json:"cursor"` +} + type UpdateOrgRequest struct { Visibility *Visibility `json:"visibility"` } type OrgMembersResponse struct { Organization *OrgResponse `json:"organization"` - Members []*OrgMemberResponse `json:"members"` + OrgMembers []*OrgMemberResponse `json:"org_members"` + Cursor string `json:"cursor"` } type OrgMemberResponse struct { diff --git a/services/gateway/api/types/project.go b/services/gateway/api/types/project.go index 9ab26e94b..a80be2ee8 100644 --- a/services/gateway/api/types/project.go +++ b/services/gateway/api/types/project.go @@ -42,6 +42,11 @@ type ProjectResponse struct { DefaultBranch string `json:"default_branch,omitempty"` } +type ProjectsResponse struct { + Projects []*ProjectResponse `json:"projects,omitempty"` + Cursor string `json:"cursor,omitempty"` +} + type ProjectCreateRunRequest struct { Branch string `json:"branch,omitempty"` Tag string `json:"tag,omitempty"` diff --git a/services/gateway/api/types/remotesource.go b/services/gateway/api/types/remotesource.go index 82fffffcd..06eb19ebd 100644 --- a/services/gateway/api/types/remotesource.go +++ b/services/gateway/api/types/remotesource.go @@ -47,3 +47,8 @@ type RemoteSourceResponse struct { RegistrationEnabled bool `json:"registration_enabled"` LoginEnabled bool `json:"login_enabled"` } + +type RemoteSourcesResponse struct { + RemoteSources []*RemoteSourceResponse `json:"remote_sources"` + Cursor string `json:"cursor"` +} diff --git a/services/gateway/api/types/run.go b/services/gateway/api/types/run.go index 84dd98681..fd0199c33 100644 --- a/services/gateway/api/types/run.go +++ b/services/gateway/api/types/run.go @@ -26,7 +26,7 @@ import ( // calculated considering also the related run service id (or use a global // sequential id generator). -type RunsResponse struct { +type Runs struct { Number uint64 `json:"number"` Name string `json:"name"` Annotations map[string]string `json:"annotations"` @@ -40,6 +40,11 @@ type RunsResponse struct { EndTime *time.Time `json:"end_time"` } +type RunsResponse struct { + Runs []*Runs + Cursor string +} + type RunResponse struct { Number uint64 `json:"number"` Name string `json:"name"` diff --git a/services/gateway/api/types/secret.go b/services/gateway/api/types/secret.go index bca21a2c3..f6c98b489 100644 --- a/services/gateway/api/types/secret.go +++ b/services/gateway/api/types/secret.go @@ -27,6 +27,11 @@ type SecretResponse struct { ParentPath string `json:"parent_path"` } +type SecretsResponse struct { + Secrets []*SecretResponse `json:"secrets"` + Cursor string `json:"cursor"` +} + type CreateSecretRequest struct { Name string `json:"name,omitempty"` diff --git a/services/gateway/api/types/user.go b/services/gateway/api/types/user.go index be22e1e3c..ee3c2214a 100644 --- a/services/gateway/api/types/user.go +++ b/services/gateway/api/types/user.go @@ -35,6 +35,11 @@ type PrivateUserResponse struct { LinkedAccounts []*LinkedAccountResponse `json:"linked_accounts"` } +type PrivateUsersResponse struct { + Users []*PrivateUserResponse `json:"users"` + Cursor string `json:"cursor"` +} + type UserResponse struct { ID string `json:"id"` UserName string `json:"username"` diff --git a/services/gateway/api/types/variable.go b/services/gateway/api/types/variable.go index aeef7a8ae..d9a1114b6 100644 --- a/services/gateway/api/types/variable.go +++ b/services/gateway/api/types/variable.go @@ -40,6 +40,11 @@ type VariableResponse struct { ParentPath string `json:"parent_path"` } +type VariablesResponse struct { + Variables []*VariableResponse `json:"variables"` + Cursor string `json:"cursor"` +} + type CreateVariableRequest struct { Name string `json:"name,omitempty"` diff --git a/services/gateway/client/client.go b/services/gateway/client/client.go index 8262145ac..1353ed007 100644 --- a/services/gateway/client/client.go +++ b/services/gateway/client/client.go @@ -113,9 +113,17 @@ func (c *Client) GetProjectGroupSubgroups(ctx context.Context, projectGroupRef s return projectGroups, resp, errors.WithStack(err) } -func (c *Client) GetProjectGroupProjects(ctx context.Context, projectGroupRef string) ([]*gwapitypes.ProjectResponse, *http.Response, error) { - projects := []*gwapitypes.ProjectResponse{} - resp, err := c.getParsedResponse(ctx, "GET", fmt.Sprintf("/projectgroups/%s/projects", url.PathEscape(projectGroupRef)), nil, jsonContent, nil, &projects) +func (c *Client) GetProjectGroupProjects(ctx context.Context, projectGroupRef string, limit int, cursor string) (*gwapitypes.ProjectsResponse, *http.Response, error) { + q := url.Values{} + if cursor != "" { + q.Add("cursor", cursor) + } + if limit > 0 { + q.Add("limit", strconv.Itoa(limit)) + } + + projects := new(gwapitypes.ProjectsResponse) + resp, err := c.getParsedResponse(ctx, "GET", fmt.Sprintf("/projectgroups/%s/projects", url.PathEscape(projectGroupRef)), q, jsonContent, nil, projects) return projects, resp, errors.WithStack(err) } @@ -199,8 +207,7 @@ func (c *Client) DeleteProjectGroupSecret(ctx context.Context, projectGroupRef, return c.getResponse(ctx, "DELETE", path.Join("/projectgroups", url.PathEscape(projectGroupRef), "secrets", secretName), nil, jsonContent, nil) } -func (c *Client) GetProjectGroupSecrets(ctx context.Context, projectRef string, tree, removeoverridden bool) ([]*gwapitypes.SecretResponse, *http.Response, error) { - secrets := []*gwapitypes.SecretResponse{} +func (c *Client) GetProjectGroupSecrets(ctx context.Context, projectRef string, tree, removeoverridden bool, start string, asc bool, limit int, cursor string) (*gwapitypes.SecretsResponse, *http.Response, error) { q := url.Values{} if tree { q.Add("tree", "") @@ -208,7 +215,21 @@ func (c *Client) GetProjectGroupSecrets(ctx context.Context, projectRef string, if removeoverridden { q.Add("removeoverridden", "") } - resp, err := c.getParsedResponse(ctx, "GET", fmt.Sprintf("/projectgroups/%s/secrets", url.PathEscape(projectRef)), q, jsonContent, nil, &secrets) + if start != "" { + q.Add("start", start) + } + if limit > 0 { + q.Add("limit", strconv.Itoa(limit)) + } + if asc { + q.Add("asc", "") + } + if cursor != "" { + q.Add("cursor", cursor) + } + + secrets := new(gwapitypes.SecretsResponse) + resp, err := c.getParsedResponse(ctx, "GET", fmt.Sprintf("/projectgroups/%s/secrets", url.PathEscape(projectRef)), q, jsonContent, nil, secrets) return secrets, resp, errors.WithStack(err) } @@ -238,8 +259,7 @@ func (c *Client) DeleteProjectSecret(ctx context.Context, projectRef, secretName return c.getResponse(ctx, "DELETE", path.Join("/projects", url.PathEscape(projectRef), "secrets", secretName), nil, jsonContent, nil) } -func (c *Client) GetProjectSecrets(ctx context.Context, projectRef string, tree, removeoverridden bool) ([]*gwapitypes.SecretResponse, *http.Response, error) { - secrets := []*gwapitypes.SecretResponse{} +func (c *Client) GetProjectSecrets(ctx context.Context, projectRef string, tree, removeoverridden bool, start string, asc bool, limit int, cursor string) (*gwapitypes.SecretsResponse, *http.Response, error) { q := url.Values{} if tree { q.Add("tree", "") @@ -247,6 +267,20 @@ func (c *Client) GetProjectSecrets(ctx context.Context, projectRef string, tree, if removeoverridden { q.Add("removeoverridden", "") } + if start != "" { + q.Add("start", start) + } + if limit > 0 { + q.Add("limit", strconv.Itoa(limit)) + } + if asc { + q.Add("asc", "") + } + if cursor != "" { + q.Add("cursor", cursor) + } + + secrets := new(gwapitypes.SecretsResponse) resp, err := c.getParsedResponse(ctx, "GET", fmt.Sprintf("/projects/%s/secrets", url.PathEscape(projectRef)), q, jsonContent, nil, &secrets) return secrets, resp, errors.WithStack(err) } @@ -277,8 +311,7 @@ func (c *Client) DeleteProjectGroupVariable(ctx context.Context, projectGroupRef return c.getResponse(ctx, "DELETE", path.Join("/projectgroups", url.PathEscape(projectGroupRef), "variables", variableName), nil, jsonContent, nil) } -func (c *Client) GetProjectGroupVariables(ctx context.Context, projectRef string, tree, removeoverridden bool) ([]*gwapitypes.VariableResponse, *http.Response, error) { - variables := []*gwapitypes.VariableResponse{} +func (c *Client) GetProjectGroupVariables(ctx context.Context, projectRef string, tree, removeoverridden bool, start string, asc bool, limit int, cursor string) (*gwapitypes.VariablesResponse, *http.Response, error) { q := url.Values{} if tree { q.Add("tree", "") @@ -286,7 +319,21 @@ func (c *Client) GetProjectGroupVariables(ctx context.Context, projectRef string if removeoverridden { q.Add("removeoverridden", "") } - resp, err := c.getParsedResponse(ctx, "GET", fmt.Sprintf("/projectgroups/%s/variables", url.PathEscape(projectRef)), q, jsonContent, nil, &variables) + if start != "" { + q.Add("start", start) + } + if limit > 0 { + q.Add("limit", strconv.Itoa(limit)) + } + if asc { + q.Add("asc", "") + } + if cursor != "" { + q.Add("cursor", cursor) + } + + variables := new(gwapitypes.VariablesResponse) + resp, err := c.getParsedResponse(ctx, "GET", fmt.Sprintf("/projectgroups/%s/variables", url.PathEscape(projectRef)), q, jsonContent, nil, variables) return variables, resp, errors.WithStack(err) } @@ -316,8 +363,7 @@ func (c *Client) DeleteProjectVariable(ctx context.Context, projectRef, variable return c.getResponse(ctx, "DELETE", path.Join("/projects", url.PathEscape(projectRef), "variables", variableName), nil, jsonContent, nil) } -func (c *Client) GetProjectVariables(ctx context.Context, projectRef string, tree, removeoverridden bool) ([]*gwapitypes.VariableResponse, *http.Response, error) { - variables := []*gwapitypes.VariableResponse{} +func (c *Client) GetProjectVariables(ctx context.Context, projectRef string, tree, removeoverridden bool, start string, asc bool, limit int, cursor string) (*gwapitypes.VariablesResponse, *http.Response, error) { q := url.Values{} if tree { q.Add("tree", "") @@ -325,7 +371,21 @@ func (c *Client) GetProjectVariables(ctx context.Context, projectRef string, tre if removeoverridden { q.Add("removeoverridden", "") } - resp, err := c.getParsedResponse(ctx, "GET", fmt.Sprintf("/projects/%s/variables", url.PathEscape(projectRef)), q, jsonContent, nil, &variables) + if start != "" { + q.Add("start", start) + } + if limit > 0 { + q.Add("limit", strconv.Itoa(limit)) + } + if asc { + q.Add("asc", "") + } + if cursor != "" { + q.Add("cursor", cursor) + } + + variables := new(gwapitypes.VariablesResponse) + resp, err := c.getParsedResponse(ctx, "GET", fmt.Sprintf("/projects/%s/variables", url.PathEscape(projectRef)), q, jsonContent, nil, variables) return variables, resp, errors.WithStack(err) } @@ -358,7 +418,7 @@ func (c *Client) GetUser(ctx context.Context, userRef string) (*gwapitypes.UserR return user, resp, errors.WithStack(err) } -func (c *Client) GetUsers(ctx context.Context, start string, limit int, asc bool) ([]*gwapitypes.PrivateUserResponse, *http.Response, error) { +func (c *Client) GetUsers(ctx context.Context, start string, limit int, asc bool, cursor string) (*gwapitypes.PrivateUsersResponse, *http.Response, error) { q := url.Values{} if start != "" { q.Add("start", start) @@ -369,9 +429,12 @@ func (c *Client) GetUsers(ctx context.Context, start string, limit int, asc bool if asc { q.Add("asc", "") } + if cursor != "" { + q.Add("cursor", cursor) + } - users := []*gwapitypes.PrivateUserResponse{} - resp, err := c.getParsedResponse(ctx, "GET", "/users", q, jsonContent, nil, &users) + users := new(gwapitypes.PrivateUsersResponse) + resp, err := c.getParsedResponse(ctx, "GET", "/users", q, jsonContent, nil, users) return users, resp, errors.WithStack(err) } @@ -381,12 +444,12 @@ func (c *Client) GetUserByLinkedAccountRemoteUserAndSource(ctx context.Context, q.Add("remoteuserid", remoteUserID) q.Add("remotesourceref", remoteSourceRef) - users := []*gwapitypes.PrivateUserResponse{} - resp, err := c.getParsedResponse(ctx, "GET", "/users", q, jsonContent, nil, &users) + users := new(gwapitypes.PrivateUsersResponse) + resp, err := c.getParsedResponse(ctx, "GET", "/users", q, jsonContent, nil, users) if err != nil { return nil, resp, errors.WithStack(err) } - return users[0], resp, errors.WithStack(err) + return users.Users[0], resp, errors.WithStack(err) } func (c *Client) CreateUser(ctx context.Context, req *gwapitypes.CreateUserRequest) (*gwapitypes.UserResponse, *http.Response, error) { @@ -493,15 +556,15 @@ func (c *Client) getRunTask(ctx context.Context, groupType, groupRef string, run return task, resp, errors.WithStack(err) } -func (c *Client) GetProjectRuns(ctx context.Context, projectRef string, phaseFilter, resultFilter []string, start uint64, limit int, asc bool) ([]*gwapitypes.RunsResponse, *http.Response, error) { - return c.getRuns(ctx, "projects", projectRef, phaseFilter, resultFilter, start, limit, asc) +func (c *Client) GetProjectRuns(ctx context.Context, projectRef string, phaseFilter, resultFilter []string, start uint64, limit int, asc bool, cursor string) (*gwapitypes.RunsResponse, *http.Response, error) { + return c.getRuns(ctx, "projects", projectRef, phaseFilter, resultFilter, start, limit, asc, cursor) } -func (c *Client) GetUserRuns(ctx context.Context, userRef string, phaseFilter, resultFilter []string, start uint64, limit int, asc bool) ([]*gwapitypes.RunsResponse, *http.Response, error) { - return c.getRuns(ctx, "users", userRef, phaseFilter, resultFilter, start, limit, asc) +func (c *Client) GetUserRuns(ctx context.Context, userRef string, phaseFilter, resultFilter []string, start uint64, limit int, asc bool, cursor string) (*gwapitypes.RunsResponse, *http.Response, error) { + return c.getRuns(ctx, "users", userRef, phaseFilter, resultFilter, start, limit, asc, cursor) } -func (c *Client) getRuns(ctx context.Context, groupType, groupRef string, phaseFilter, resultFilter []string, start uint64, limit int, asc bool) ([]*gwapitypes.RunsResponse, *http.Response, error) { +func (c *Client) getRuns(ctx context.Context, groupType, groupRef string, phaseFilter, resultFilter []string, start uint64, limit int, asc bool, cursor string) (*gwapitypes.RunsResponse, *http.Response, error) { q := url.Values{} for _, phase := range phaseFilter { q.Add("phase", phase) @@ -518,9 +581,12 @@ func (c *Client) getRuns(ctx context.Context, groupType, groupRef string, phaseF if asc { q.Add("asc", "") } + if cursor != "" { + q.Add("cursor", cursor) + } - getRunsResponse := []*gwapitypes.RunsResponse{} - resp, err := c.getParsedResponse(ctx, "GET", fmt.Sprintf("/%s/%s/runs", groupType, url.PathEscape(groupRef)), q, jsonContent, nil, &getRunsResponse) + getRunsResponse := new(gwapitypes.RunsResponse) + resp, err := c.getParsedResponse(ctx, "GET", fmt.Sprintf("/%s/%s/runs", groupType, url.PathEscape(groupRef)), q, jsonContent, nil, getRunsResponse) return getRunsResponse, resp, errors.WithStack(err) } @@ -570,7 +636,7 @@ func (c *Client) GetRemoteSource(ctx context.Context, rsRef string) (*gwapitypes return rs, resp, errors.WithStack(err) } -func (c *Client) GetRemoteSources(ctx context.Context, start string, limit int, asc bool) ([]*gwapitypes.RemoteSourceResponse, *http.Response, error) { +func (c *Client) GetRemoteSources(ctx context.Context, start string, limit int, asc bool, cursor string) (*gwapitypes.RemoteSourcesResponse, *http.Response, error) { q := url.Values{} if start != "" { q.Add("start", start) @@ -581,9 +647,12 @@ func (c *Client) GetRemoteSources(ctx context.Context, start string, limit int, if asc { q.Add("asc", "") } + if cursor != "" { + q.Add("cursor", cursor) + } - rss := []*gwapitypes.RemoteSourceResponse{} - resp, err := c.getParsedResponse(ctx, "GET", "/remotesources", q, jsonContent, nil, &rss) + rss := new(gwapitypes.RemoteSourcesResponse) + resp, err := c.getParsedResponse(ctx, "GET", "/remotesources", q, jsonContent, nil, rss) return rss, resp, errors.WithStack(err) } @@ -619,8 +688,11 @@ func (c *Client) GetOrg(ctx context.Context, orgRef string) (*gwapitypes.OrgResp return res, resp, errors.WithStack(err) } -func (c *Client) GetOrgs(ctx context.Context, start string, limit int, asc bool) ([]*gwapitypes.OrgResponse, *http.Response, error) { +func (c *Client) GetOrgs(ctx context.Context, start string, limit int, asc bool, cursor string) (*gwapitypes.OrgsResponse, *http.Response, error) { q := url.Values{} + if cursor != "" { + q.Add("cursor", cursor) + } if start != "" { q.Add("start", start) } @@ -631,8 +703,8 @@ func (c *Client) GetOrgs(ctx context.Context, start string, limit int, asc bool) q.Add("asc", "") } - orgs := []*gwapitypes.OrgResponse{} - resp, err := c.getParsedResponse(ctx, "GET", "/orgs", q, jsonContent, nil, &orgs) + orgs := new(gwapitypes.OrgsResponse) + resp, err := c.getParsedResponse(ctx, "GET", "/orgs", q, jsonContent, nil, orgs) return orgs, resp, errors.WithStack(err) } @@ -680,9 +752,20 @@ func (c *Client) RemoveOrgMember(ctx context.Context, orgRef, userRef string) (* return c.getResponse(ctx, "DELETE", fmt.Sprintf("/orgs/%s/members/%s", orgRef, userRef), nil, jsonContent, nil) } -func (c *Client) GetOrgMembers(ctx context.Context, orgRef string) (*gwapitypes.OrgMembersResponse, *http.Response, error) { +func (c *Client) GetOrgMembers(ctx context.Context, orgRef string, asc bool, limit int, cursor string) (*gwapitypes.OrgMembersResponse, *http.Response, error) { + q := url.Values{} + if cursor != "" { + q.Add("cursor", cursor) + } + if limit > 0 { + q.Add("limit", strconv.Itoa(limit)) + } + if asc { + q.Add("asc", "") + } + res := &gwapitypes.OrgMembersResponse{} - resp, err := c.getParsedResponse(ctx, "GET", fmt.Sprintf("/orgs/%s/members", orgRef), nil, jsonContent, nil, &res) + resp, err := c.getParsedResponse(ctx, "GET", fmt.Sprintf("/orgs/%s/members", orgRef), q, jsonContent, nil, &res) return res, resp, errors.WithStack(err) } diff --git a/services/runservice/api/types/run.go b/services/runservice/api/types/run.go index 90e09d145..2b78f05d4 100644 --- a/services/runservice/api/types/run.go +++ b/services/runservice/api/types/run.go @@ -27,6 +27,7 @@ type RunResponse struct { type GetRunsResponse struct { Runs []*rstypes.Run `json:"runs"` ChangeGroupsUpdateToken string `json:"change_groups_update_tokens"` + HasMoreData bool `json:"has_more_data"` } type RunCreateRequest struct { diff --git a/tests/setup_test.go b/tests/setup_test.go index 10799f394..32fd7a9ba 100644 --- a/tests/setup_test.go +++ b/tests/setup_test.go @@ -19,6 +19,7 @@ import ( "context" "crypto/hmac" "crypto/sha256" + "encoding/base64" "encoding/hex" "encoding/json" "fmt" @@ -52,6 +53,7 @@ import ( "agola.io/agola/internal/services/configstore" "agola.io/agola/internal/services/executor" "agola.io/agola/internal/services/gateway" + gwapi "agola.io/agola/internal/services/gateway/api" "agola.io/agola/internal/services/gateway/common" "agola.io/agola/internal/services/gitserver" "agola.io/agola/internal/services/notification" @@ -80,6 +82,9 @@ const ( agolaOrg02 = "org02" agolaOrg03 = "org03" + agolaProject01 = "project01" + agolaProject02 = "project02" + configstoreService = "configstore" runserviceService = "runservice" ) @@ -1148,15 +1153,6 @@ func push(t *testing.T, config, cloneURL, remoteToken, message string, pushNewBr } t.Logf("sshurl: %s", cloneURL) - if err := r.Push(&git.PushOptions{ - RemoteName: "origin", - Auth: &githttp.BasicAuth{ - Username: giteaUser01, - Password: remoteToken, - }, - }); err != nil { - t.Fatalf("unexpected err: %v", err) - } if pushNewBranch { // change worktree and push to a new branch @@ -1203,6 +1199,16 @@ func push(t *testing.T, config, cloneURL, remoteToken, message string, pushNewBr }); err != nil { t.Fatalf("unexpected err: %v", err) } + } else { + if err := r.Push(&git.PushOptions{ + RemoteName: "origin", + Auth: &githttp.BasicAuth{ + Username: giteaUser01, + Password: remoteToken, + }, + }); err != nil { + t.Fatalf("unexpected err: %v", err) + } } } @@ -1375,15 +1381,15 @@ func TestPush(t *testing.T) { push(t, tt.config, giteaRepo.CloneURL, giteaToken, tt.message, false) _ = testutil.Wait(30*time.Second, func() (bool, error) { - runs, _, err := gwClient.GetProjectRuns(ctx, project.ID, nil, nil, 0, 0, false) + runs, _, err := gwClient.GetProjectRuns(ctx, project.ID, nil, nil, 0, 0, false, "") if err != nil { return false, nil } - if len(runs) == 0 { + if len(runs.Runs) == 0 { return false, nil } - run := runs[0] + run := runs.Runs[0] if run.Phase != rstypes.RunPhaseFinished { return false, nil } @@ -1391,19 +1397,19 @@ func TestPush(t *testing.T) { return true, nil }) - runs, _, err := gwClient.GetProjectRuns(ctx, project.ID, nil, nil, 0, 0, false) + runs, _, err := gwClient.GetProjectRuns(ctx, project.ID, nil, nil, 0, 0, false, "") if err != nil { t.Fatalf("unexpected err: %v", err) } - t.Logf("runs: %s", util.Dump(runs)) + t.Logf("runs: %s", util.Dump(runs.Runs)) - if len(runs) != tt.num { - t.Fatalf("expected %d run got: %d", tt.num, len(runs)) + if len(runs.Runs) != tt.num { + t.Fatalf("expected %d run got: %d", tt.num, len(runs.Runs)) } - if len(runs) > 0 { - run := runs[0] + if len(runs.Runs) > 0 { + run := runs.Runs[0] if run.Phase != rstypes.RunPhaseFinished { t.Fatalf("expected run phase %q, got %q", rstypes.RunPhaseFinished, run.Phase) } @@ -1595,16 +1601,16 @@ func testDirectRun(t *testing.T, internalServicesAuth bool) { directRun(t, dir, config, ConfigFormatJsonnet, sc.config.Gateway.APIExposedURL, token, tt.args...) _ = testutil.Wait(30*time.Second, func() (bool, error) { - runs, _, err := gwClient.GetUserRuns(ctx, user.ID, nil, nil, 0, 0, false) + runs, _, err := gwClient.GetUserRuns(ctx, user.ID, nil, nil, 0, 0, false, "") if err != nil { return false, nil } - if len(runs) != 1 { + if len(runs.Runs) != 1 { return false, nil } - run := runs[0] + run := runs.Runs[0] if run.Phase != rstypes.RunPhaseFinished { return false, nil } @@ -1612,18 +1618,18 @@ func testDirectRun(t *testing.T, internalServicesAuth bool) { return true, nil }) - runs, _, err := gwClient.GetUserRuns(ctx, user.ID, nil, nil, 0, 0, false) + runs, _, err := gwClient.GetUserRuns(ctx, user.ID, nil, nil, 0, 0, false, "") if err != nil { t.Fatalf("unexpected err: %v", err) } - t.Logf("runs: %s", util.Dump(runs)) + t.Logf("runs: %s", util.Dump(runs.Runs)) - if len(runs) != 1 { - t.Fatalf("expected 1 run got: %d", len(runs)) + if len(runs.Runs) != 1 { + t.Fatalf("expected 1 run got: %d", len(runs.Runs)) } - run := runs[0] + run := runs.Runs[0] if run.Phase != rstypes.RunPhaseFinished { t.Fatalf("expected run phase %q, got %q", rstypes.RunPhaseFinished, run.Phase) } @@ -1749,16 +1755,16 @@ func TestDirectRunVariables(t *testing.T) { // TODO(sgotti) add an util to wait for a run phase _ = testutil.Wait(30*time.Second, func() (bool, error) { - runs, _, err := gwClient.GetUserRuns(ctx, user.ID, nil, nil, 0, 0, false) + runs, _, err := gwClient.GetUserRuns(ctx, user.ID, nil, nil, 0, 0, false, "") if err != nil { return false, nil } - if len(runs) != 1 { + if len(runs.Runs) != 1 { return false, nil } - run := runs[0] + run := runs.Runs[0] if run.Phase != rstypes.RunPhaseFinished { return false, nil } @@ -1766,18 +1772,18 @@ func TestDirectRunVariables(t *testing.T) { return true, nil }) - runs, _, err := gwClient.GetUserRuns(ctx, user.ID, nil, nil, 0, 0, false) + runs, _, err := gwClient.GetUserRuns(ctx, user.ID, nil, nil, 0, 0, false, "") if err != nil { t.Fatalf("unexpected err: %v", err) } - t.Logf("runs: %s", util.Dump(runs)) + t.Logf("runs: %s", util.Dump(runs.Runs)) - if len(runs) != 1 { - t.Fatalf("expected 1 run got: %d", len(runs)) + if len(runs.Runs) != 1 { + t.Fatalf("expected 1 run got: %d", len(runs.Runs)) } - run, _, err := gwClient.GetUserRun(ctx, user.ID, runs[0].Number) + run, _, err := gwClient.GetUserRun(ctx, user.ID, runs.Runs[0].Number) if err != nil { t.Fatalf("unexpected err: %v", err) } @@ -1919,16 +1925,16 @@ func TestDirectRunLogs(t *testing.T) { directRun(t, dir, config, ConfigFormatJsonnet, sc.config.Gateway.APIExposedURL, token) _ = testutil.Wait(30*time.Second, func() (bool, error) { - runs, _, err := gwClient.GetUserRuns(ctx, user.ID, nil, nil, 0, 0, false) + runs, _, err := gwClient.GetUserRuns(ctx, user.ID, nil, nil, 0, 0, false, "") if err != nil { return false, nil } - if len(runs) != 1 { + if len(runs.Runs) != 1 { return false, nil } - run := runs[0] + run := runs.Runs[0] if run.Phase != rstypes.RunPhaseFinished { return false, nil } @@ -1936,18 +1942,18 @@ func TestDirectRunLogs(t *testing.T) { return true, nil }) - runs, _, err := gwClient.GetUserRuns(ctx, user.ID, nil, nil, 0, 0, false) + runs, _, err := gwClient.GetUserRuns(ctx, user.ID, nil, nil, 0, 0, false, "") if err != nil { t.Fatalf("unexpected err: %v", err) } - t.Logf("runs: %s", util.Dump(runs)) + t.Logf("runs: %s", util.Dump(runs.Runs)) - if len(runs) != 1 { - t.Fatalf("expected 1 run got: %d", len(runs)) + if len(runs.Runs) != 1 { + t.Fatalf("expected 1 run got: %d", len(runs.Runs)) } - run, _, err := gwClient.GetUserRun(ctx, user.ID, runs[0].Number) + run, _, err := gwClient.GetUserRun(ctx, user.ID, runs.Runs[0].Number) if err != nil { t.Fatalf("unexpected err: %v", err) } @@ -1968,7 +1974,7 @@ func TestDirectRunLogs(t *testing.T) { } _ = testutil.Wait(30*time.Second, func() (bool, error) { - t, _, err := gwClient.GetUserRunTask(ctx, user.ID, runs[0].Number, task.ID) + t, _, err := gwClient.GetUserRunTask(ctx, user.ID, runs.Runs[0].Number, task.ID) if err != nil { return false, nil } @@ -2246,15 +2252,15 @@ func TestPullRequest(t *testing.T) { } } _ = testutil.Wait(30*time.Second, func() (bool, error) { - runs, _, err := gwClient.GetProjectRuns(ctx, project.ID, nil, nil, 0, 0, false) + runs, _, err := gwClient.GetProjectRuns(ctx, project.ID, nil, nil, 0, 0, false, "") if err != nil { return false, nil } - if len(runs) == 0 { + if len(runs.Runs) == 0 { return false, nil } - run := runs[0] + run := runs.Runs[0] if run.Phase != rstypes.RunPhaseFinished { return false, nil } @@ -2262,14 +2268,14 @@ func TestPullRequest(t *testing.T) { return true, nil }) - runs, _, err := gwClient.GetProjectRuns(ctx, project.ID, nil, nil, 0, 0, false) + runs, _, err := gwClient.GetProjectRuns(ctx, project.ID, nil, nil, 0, 0, false, "") if err != nil { t.Fatalf("unexpected err: %v", err) } - t.Logf("runs: %s", util.Dump(runs)) + t.Logf("runs: %s", util.Dump(runs.Runs)) - run, _, err := gwClient.GetProjectRun(ctx, project.ID, runs[0].Number) + run, _, err := gwClient.GetProjectRun(ctx, project.ID, runs.Runs[0].Number) if err != nil { t.Fatalf("unexpected err: %v", err) } @@ -2282,8 +2288,8 @@ func TestPullRequest(t *testing.T) { } } - if len(runs) > 0 { - run := runs[0] + if len(runs.Runs) > 0 { + run := runs.Runs[0] if run.Phase != rstypes.RunPhaseFinished { t.Fatalf("expected run phase %q, got %q", rstypes.RunPhaseFinished, run.Phase) } @@ -2461,16 +2467,16 @@ def main(ctx): // TODO(sgotti) add an util to wait for a run phase _ = testutil.Wait(30*time.Second, func() (bool, error) { - runs, _, err := gwClient.GetUserRuns(ctx, user.ID, nil, nil, 0, 0, false) + runs, _, err := gwClient.GetUserRuns(ctx, user.ID, nil, nil, 0, 0, false, "") if err != nil { return false, nil } - if len(runs) != 1 { + if len(runs.Runs) != 1 { return false, nil } - run := runs[0] + run := runs.Runs[0] if run.Phase != rstypes.RunPhaseFinished { return false, nil } @@ -2478,18 +2484,18 @@ def main(ctx): return true, nil }) - runs, _, err := gwClient.GetUserRuns(ctx, user.ID, nil, nil, 0, 0, false) + runs, _, err := gwClient.GetUserRuns(ctx, user.ID, nil, nil, 0, 0, false, "") if err != nil { t.Fatalf("unexpected err: %v", err) } - t.Logf("runs: %s", util.Dump(runs)) + t.Logf("runs: %s", util.Dump(runs.Runs)) - if len(runs) != 1 { - t.Fatalf("expected 1 run got: %d", len(runs)) + if len(runs.Runs) != 1 { + t.Fatalf("expected 1 run got: %d", len(runs.Runs)) } - run, _, err := gwClient.GetUserRun(ctx, user.ID, runs[0].Number) + run, _, err := gwClient.GetUserRun(ctx, user.ID, runs.Runs[0].Number) if err != nil { t.Fatalf("unexpected err: %v", err) } @@ -2966,17 +2972,17 @@ func TestAddUpdateOrgUserMembers(t *testing.T) { Role: gwapitypes.MemberRoleMember, } - orgMembers, _, err := gwClient.GetOrgMembers(ctx, agolaOrg01) + orgMembers, _, err := gwClient.GetOrgMembers(ctx, agolaOrg01, false, 0, "") if err != nil { t.Fatalf("unexpected err: %v", err) } if orgMembers == nil { t.Fatal("unexpected members nil") } - if len(orgMembers.Members) != 1 { - t.Fatalf("expected Members len 1, got: %d", len(orgMembers.Members)) + if len(orgMembers.OrgMembers) != 1 { + t.Fatalf("expected Members len 1, got: %d", len(orgMembers.OrgMembers)) } - if diff := cmp.Diff(*orgMembers.Members[0], expectedOrgMember); diff != "" { + if diff := cmp.Diff(*orgMembers.OrgMembers[0], expectedOrgMember); diff != "" { t.Fatalf("org member mismatch (-expected +got):\n%s", diff) } @@ -2988,17 +2994,17 @@ func TestAddUpdateOrgUserMembers(t *testing.T) { expectedOrgMember.Role = gwapitypes.MemberRoleOwner - orgMembers, _, err = gwClient.GetOrgMembers(ctx, agolaOrg01) + orgMembers, _, err = gwClient.GetOrgMembers(ctx, agolaOrg01, false, 0, "") if err != nil { t.Fatalf("unexpected err: %v", err) } if orgMembers == nil { t.Fatal("unexpected members nil") } - if len(orgMembers.Members) != 1 { - t.Fatalf("expected Members len 1, got: %d", len(orgMembers.Members)) + if len(orgMembers.OrgMembers) != 1 { + t.Fatalf("expected Members len 1, got: %d", len(orgMembers.OrgMembers)) } - if diff := cmp.Diff(*orgMembers.Members[0], expectedOrgMember); diff != "" { + if diff := cmp.Diff(*orgMembers.OrgMembers[0], expectedOrgMember); diff != "" { t.Fatalf("org member mismatch (-expected +got):\n%s", diff) } } @@ -3254,12 +3260,12 @@ func TestOrgInvitation(t *testing.T) { t.Fatalf("expected err %v, got err: %v", expectedErr, err) } - org01Members, _, err := tc.gwClientUser01.GetOrgMembers(ctx, agolaOrg01) + org01Members, _, err := tc.gwClientUser01.GetOrgMembers(ctx, agolaOrg01, false, 0, "") if err != nil { t.Fatalf("unexpected err: %v", err) } - if len(org01Members.Members) != 2 { - t.Fatalf("expected 2 members got: %d", len(org01Members.Members)) + if len(org01Members.OrgMembers) != 2 { + t.Fatalf("expected 2 members got: %d", len(org01Members.OrgMembers)) } }, }, @@ -3395,12 +3401,12 @@ func TestOrgInvitation(t *testing.T) { t.Fatalf("expected err %v, got err: %v", expectedErr, err) } - orgMembers, _, err := gwClientUser01.GetOrgMembers(ctx, agolaOrg01) + orgMembers, _, err := gwClientUser01.GetOrgMembers(ctx, agolaOrg01, false, 0, "") if err != nil { t.Fatalf("unexpected err: %v", err) } - if len(orgMembers.Members) != 2 { - t.Fatalf("expected 2 members got: %d", len(orgMembers.Members)) + if len(orgMembers.OrgMembers) != 2 { + t.Fatalf("expected 2 members got: %d", len(orgMembers.OrgMembers)) } }, }, @@ -3648,11 +3654,11 @@ func TestGetUsers(t *testing.T) { {ID: user01.ID, UserName: user01.UserName, Tokens: []string{}, LinkedAccounts: []*gwapitypes.LinkedAccountResponse{}}, {ID: user02.ID, UserName: user02.UserName, Tokens: []string{}, LinkedAccounts: []*gwapitypes.LinkedAccountResponse{}}, } - users, _, err := gwClient.GetUsers(ctx, "", 0, true) + users, _, err := gwClient.GetUsers(ctx, "", 0, true, "") if err != nil { t.Fatalf("unexpected err: %v", err) } - if diff := cmp.Diff(expectedUsers, users); diff != "" { + if diff := cmp.Diff(expectedUsers, users.Users); diff != "" { t.Fatalf("users mismatch (-want +got):\n%s", diff) } }, @@ -3664,7 +3670,7 @@ func TestGetUsers(t *testing.T) { gwClient := gwclient.NewClient(sc.config.Gateway.APIExposedURL, user01Token) - _, _, err := gwClient.GetUsers(ctx, "", 0, true) + _, _, err := gwClient.GetUsers(ctx, "", 0, true, "") expectedErr := "remote error unauthorized" if err == nil { t.Fatalf("expected error %v, got nil err", expectedErr) @@ -4007,15 +4013,15 @@ func TestExportImport(t *testing.T) { push(t, config, giteaRepo.CloneURL, giteaToken, "commit", false) _ = testutil.Wait(30*time.Second, func() (bool, error) { - runs, _, err := gwClient.GetProjectRuns(ctx, project.ID, nil, nil, 0, 0, false) + runs, _, err := gwClient.GetProjectRuns(ctx, project.ID, nil, nil, 0, 0, false, "") if err != nil { return false, nil } - if len(runs) == 0 { + if len(runs.Runs) == 0 { return false, nil } - run := runs[0] + run := runs.Runs[0] if run.Phase != rstypes.RunPhaseFinished { return false, nil } @@ -4023,17 +4029,17 @@ func TestExportImport(t *testing.T) { return true, nil }) - runs, _, err := gwClient.GetProjectRuns(ctx, project.ID, nil, nil, 0, 0, false) + runs, _, err := gwClient.GetProjectRuns(ctx, project.ID, nil, nil, 0, 0, false, "") if err != nil { t.Fatalf("unexpected err: %v", err) } - if len(runs) != 1 { - t.Fatalf("expected %d run got: %d", 1, len(runs)) + if len(runs.Runs) != 1 { + t.Fatalf("expected %d run got: %d", 1, len(runs.Runs)) } gwClient = gwclient.NewClient(sc.config.Gateway.APIExposedURL, sc.config.Gateway.AdminToken) - users, _, err := gwClient.GetUsers(ctx, "", 0, false) + users, _, err := gwClient.GetUsers(ctx, "", 0, false, "") if err != nil { t.Fatalf("unexpected err: %v", err) } @@ -4043,12 +4049,12 @@ func TestExportImport(t *testing.T) { t.Fatalf("unexpected err: %v", err) } - remotesources, _, err := gwClient.GetRemoteSources(ctx, "", 0, false) + remotesources, _, err := gwClient.GetRemoteSources(ctx, "", 0, false, "") if err != nil { t.Fatalf("unexpected err: %v", err) } - user01Projects, _, err := gwClient.GetProjectGroupProjects(ctx, "user/user01") + user01Projects, _, err := gwClient.GetProjectGroupProjects(ctx, "user/user01", 0, "") if err != nil { t.Fatalf("unexpected err: %v", err) } @@ -4175,7 +4181,7 @@ func TestExportImport(t *testing.T) { return true, nil }) - impUsers, _, err := gwClient.GetUsers(ctx, "", 0, false) + impUsers, _, err := gwClient.GetUsers(ctx, "", 0, false, "") if err != nil { t.Fatalf("unexpected err: %v", err) } @@ -4191,7 +4197,7 @@ func TestExportImport(t *testing.T) { t.Fatalf("projectgroup mismatch (-want +got):\n%s", diff) } - impRemotesources, _, err := gwClient.GetRemoteSources(ctx, "", 0, false) + impRemotesources, _, err := gwClient.GetRemoteSources(ctx, "", 0, false, "") if err != nil { t.Fatalf("unexpected err: %v", err) } @@ -4199,7 +4205,7 @@ func TestExportImport(t *testing.T) { t.Fatalf("remotesources mismatch (-want +got):\n%s", diff) } - impUser01Projects, _, err := gwClient.GetProjectGroupProjects(ctx, "user/user01") + impUser01Projects, _, err := gwClient.GetProjectGroupProjects(ctx, "user/user01", 0, "") if err != nil { t.Fatalf("unexpected err: %v", err) } @@ -4207,7 +4213,7 @@ func TestExportImport(t *testing.T) { t.Fatalf("user01 projects mismatch (-want +got):\n%s", diff) } - impRuns, _, err := gwClient.GetProjectRuns(ctx, project.ID, nil, nil, 0, 0, false) + impRuns, _, err := gwClient.GetProjectRuns(ctx, project.ID, nil, nil, 0, 0, false, "") if err != nil { t.Fatalf("unexpected err: %v", err) } @@ -4215,12 +4221,12 @@ func TestExportImport(t *testing.T) { t.Fatalf("runs mismatch (-want +got):\n%s", diff) } - orgs, _, err := gwClient.GetOrgs(ctx, "", 0, false) + orgs, _, err := gwClient.GetOrgs(ctx, "", 0, false, "") if err != nil { t.Fatalf("unexpected err: %v", err) } - if len(orgs) != 0 { - t.Fatalf("expected 0 orgs got: %d", len(orgs)) + if len(orgs.Orgs) != 0 { + t.Fatalf("expected 0 orgs got: %d", len(orgs.Orgs)) } } @@ -4316,15 +4322,15 @@ func TestGetProjectRuns(t *testing.T) { push(t, config, giteaRepo.CloneURL, giteaToken, "commit", false) _ = testutil.Wait(30*time.Second, func() (bool, error) { - runs, _, err := gwClient.GetProjectRuns(ctx, project.ID, nil, nil, 0, 0, false) + runs, _, err := gwClient.GetProjectRuns(ctx, project.ID, nil, nil, 0, 0, false, "") if err != nil { return false, nil } - if len(runs) == 0 { + if len(runs.Runs) == 0 { return false, nil } - run := runs[0] + run := runs.Runs[0] if run.Phase != rstypes.RunPhaseFinished { return false, nil } @@ -4332,19 +4338,19 @@ func TestGetProjectRuns(t *testing.T) { return true, nil }) - runs, _, err := gwClient.GetProjectRuns(ctx, project.ID, tt.phaseFilter, tt.resultFilter, 0, 0, false) + runs, _, err := gwClient.GetProjectRuns(ctx, project.ID, tt.phaseFilter, tt.resultFilter, 0, 0, false, "") if err != nil { t.Fatalf("unexpected err: %v", err) } t.Logf("runs: %s", util.Dump(runs)) - if len(runs) != tt.num { - t.Fatalf("expected %d run got: %d", tt.num, len(runs)) + if len(runs.Runs) != tt.num { + t.Fatalf("expected %d run got: %d", tt.num, len(runs.Runs)) } - if len(runs) > 0 { - run := runs[0] + if len(runs.Runs) > 0 { + run := runs.Runs[0] if run.Phase != rstypes.RunPhaseFinished { t.Fatalf("expected run phase %q, got %q", rstypes.RunPhaseFinished, run.Phase) } @@ -4474,37 +4480,37 @@ func TestRunEventsNotification(t *testing.T) { push(t, tt.config, giteaRepo.CloneURL, giteaToken, "commit", false) _ = testutil.Wait(30*time.Second, func() (bool, error) { - runs, _, err := gwClient.GetProjectRuns(ctx, project.ID, nil, nil, 0, 0, false) + runs, _, err := gwClient.GetProjectRuns(ctx, project.ID, nil, nil, 0, 0, false, "") if err != nil { return false, nil } - if len(runs) == 0 { + if len(runs.Runs) == 0 { return false, nil } - if runs[0].Phase != tt.expectedRunPhase { + if runs.Runs[0].Phase != tt.expectedRunPhase { return false, nil } return true, nil }) - runs, _, err := gwClient.GetProjectRuns(ctx, project.ID, nil, nil, 0, 0, false) + runs, _, err := gwClient.GetProjectRuns(ctx, project.ID, nil, nil, 0, 0, false, "") if err != nil { t.Fatalf("unexpected err: %v", err) } - if len(runs) != 1 { - t.Fatalf("expected %d run got: %d", 1, len(runs)) + if len(runs.Runs) != 1 { + t.Fatalf("expected %d run got: %d", 1, len(runs.Runs)) } - if runs[0].Phase != tt.expectedRunPhase { - t.Fatalf("expected run phase %q, got %q", tt.expectedRunPhase, runs[0].Phase) + if runs.Runs[0].Phase != tt.expectedRunPhase { + t.Fatalf("expected run phase %q, got %q", tt.expectedRunPhase, runs.Runs[0].Phase) } - if runs[0].Result != tt.expectedRunResult { - t.Fatalf("expected run result %q, got %q", tt.expectedRunResult, runs[0].Result) + if runs.Runs[0].Result != tt.expectedRunResult { + t.Fatalf("expected run result %q, got %q", tt.expectedRunResult, runs.Runs[0].Result) } - run, _, err := gwClient.GetProjectRun(ctx, project.ID, runs[0].Number) + run, _, err := gwClient.GetProjectRun(ctx, project.ID, runs.Runs[0].Number) if err != nil { t.Fatalf("unexpected err: %v", err) } @@ -4595,3 +4601,639 @@ func TestRunEventsNotification(t *testing.T) { }) } } + +func TestOrgsCursor(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + sc := setup(ctx, t, dir) + defer sc.stop() + + gwClient := gwclient.NewClient(sc.config.Gateway.APIExposedURL, "admintoken") + + org01, _, err := gwClient.CreateOrg(ctx, &gwapitypes.CreateOrgRequest{Name: agolaOrg01, Visibility: gwapitypes.VisibilityPublic}) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + org02, _, err := gwClient.CreateOrg(ctx, &gwapitypes.CreateOrgRequest{Name: agolaOrg02, Visibility: gwapitypes.VisibilityPublic}) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + + cursor := gwapi.OrgsCursor{ + LastOrgID: org01.ID, + Asc: true, + } + serializedCursor, err := json.Marshal(&cursor) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + expectedCursorS := base64.StdEncoding.EncodeToString(serializedCursor) + + orgsResponse, _, err := gwClient.GetOrgs(ctx, "", 1, true, "") + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + if len(orgsResponse.Orgs) != 1 { + t.Fatalf("expected %d org got: %d", 1, len(orgsResponse.Orgs)) + } + if diff := cmp.Diff(orgsResponse.Orgs[0], org01); diff != "" { + t.Fatalf("org mismatch (-expected +got):\n%s", diff) + } + if orgsResponse.Cursor != expectedCursorS { + t.Fatalf("expected cursor %s, got %s", expectedCursorS, orgsResponse.Cursor) + } + + orgsResponse, _, err = gwClient.GetOrgs(ctx, "", 1, true, orgsResponse.Cursor) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + if diff := cmp.Diff(orgsResponse.Orgs[0], org02); diff != "" { + t.Fatalf("org mismatch (-expected +got):\n%s", diff) + } + if orgsResponse.Cursor != "" { + t.Fatalf("expected cursor empty, got %s", orgsResponse.Cursor) + } +} + +func TestOrgMembersCursor(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + sc := setup(ctx, t, dir) + defer sc.stop() + + gwClient := gwclient.NewClient(sc.config.Gateway.APIExposedURL, "admintoken") + + _, _, err := gwClient.CreateUser(ctx, &gwapitypes.CreateUserRequest{UserName: agolaUser01}) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + _, _, err = gwClient.CreateUser(ctx, &gwapitypes.CreateUserRequest{UserName: agolaUser02}) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + _, _, err = gwClient.CreateOrg(ctx, &gwapitypes.CreateOrgRequest{Name: agolaOrg01, Visibility: gwapitypes.VisibilityPublic}) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + orgMember01, _, err := gwClient.AddOrgMember(ctx, agolaOrg01, agolaUser01, gwapitypes.MemberRoleMember) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + orgMember02, _, err := gwClient.AddOrgMember(ctx, agolaOrg01, agolaUser02, gwapitypes.MemberRoleMember) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + + cursor := gwapi.OrgMembersCursor{ + LastOrgUserID: orgMember01.User.ID, + Asc: true, + } + serializedCursor, err := json.Marshal(&cursor) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + expectedCursorS := base64.StdEncoding.EncodeToString(serializedCursor) + + orgMembers, _, err := gwClient.GetOrgMembers(ctx, agolaOrg01, true, 1, "") + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + if len(orgMembers.OrgMembers) != 1 { + t.Fatalf("expected %d org got: %d", 1, len(orgMembers.OrgMembers)) + } + if diff := cmp.Diff(*orgMembers.OrgMembers[0], orgMember01.OrgMemberResponse); diff != "" { + t.Fatalf("org mismatch (-expected +got):\n%s", diff) + } + if orgMembers.Cursor != expectedCursorS { + t.Fatalf("expected cursor %s, got %s", expectedCursorS, orgMembers.Cursor) + } + + orgMembers, _, err = gwClient.GetOrgMembers(ctx, agolaOrg01, true, 1, orgMembers.Cursor) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + if len(orgMembers.OrgMembers) != 1 { + t.Fatalf("expected %d org got: %d", 1, len(orgMembers.OrgMembers)) + } + if diff := cmp.Diff(*orgMembers.OrgMembers[0], orgMember02.OrgMemberResponse); diff != "" { + t.Fatalf("org mismatch (-expected +got):\n%s", diff) + } + if orgMembers.Cursor != "" { + t.Fatalf("expected cursor empty, got %s", orgMembers.Cursor) + } +} + +func TestRunsCursor(t *testing.T) { + t.Parallel() + + config := ` + { + runs: [ + { + name: 'run01', + tasks: [ + { + name: 'task01', + runtime: { + containers: [ + { + image: 'alpine/git', + }, + ], + }, + steps: [ + { type: 'clone' }, + { type: 'run', command: 'env' }, + ], + }, + ], + }, + ], + } + ` + + dir := t.TempDir() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + sc := setup(ctx, t, dir, withGitea(true)) + defer sc.stop() + + giteaAPIURL := fmt.Sprintf("http://%s:%s", sc.gitea.HTTPListenAddress, sc.gitea.HTTPPort) + + giteaToken, token := createLinkedAccount(ctx, t, sc.gitea, sc.config) + + giteaClient, err := gitea.NewClient(giteaAPIURL, gitea.SetToken(giteaToken)) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + + gwClient := gwclient.NewClient(sc.config.Gateway.APIExposedURL, token) + + giteaRepo, project := createProject(ctx, t, giteaClient, gwClient) + + push(t, config, giteaRepo.CloneURL, giteaToken, "commit", false) + + push(t, config, giteaRepo.CloneURL, giteaToken, "commit", true) + + _ = testutil.Wait(30*time.Second, func() (bool, error) { + runs, _, err := gwClient.GetProjectRuns(ctx, project.ID, nil, nil, 0, 0, false, "") + if err != nil { + return false, nil + } + + if len(runs.Runs) != 2 { + return false, nil + } + + return true, nil + }) + + cursor := gwapi.RunsCursor{ + LastRunNumber: 1, + Asc: true, + } + serializedCursor, err := json.Marshal(&cursor) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + expectedCursorS := base64.StdEncoding.EncodeToString(serializedCursor) + + runs, _, err := gwClient.GetProjectRuns(ctx, project.ID, nil, nil, 0, 1, true, "") + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + if len(runs.Runs) != 1 { + t.Fatalf("expected %d run got: %d", 1, len(runs.Runs)) + } + if runs.Cursor != expectedCursorS { + t.Fatalf("expected cursor %s, got %s", expectedCursorS, runs.Cursor) + } + + runs, _, err = gwClient.GetProjectRuns(ctx, project.ID, nil, nil, 0, 1, true, runs.Cursor) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + if len(runs.Runs) != 1 { + t.Fatalf("expected %d run got: %d", 1, len(runs.Runs)) + } + if runs.Cursor != "" { + t.Fatalf("expected cursor empy, got %s", runs.Cursor) + } +} + +func TestProjectgroupProjectsCursor(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + sc := setup(ctx, t, dir, withGitea(true)) + defer sc.stop() + + giteaAPIURL := fmt.Sprintf("http://%s:%s", sc.gitea.HTTPListenAddress, sc.gitea.HTTPPort) + + giteaToken, token := createLinkedAccount(ctx, t, sc.gitea, sc.config) + + giteaClient, err := gitea.NewClient(giteaAPIURL, gitea.SetToken(giteaToken)) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + + gwClient := gwclient.NewClient(sc.config.Gateway.APIExposedURL, token) + + _, _, err = gwClient.CreateOrg(ctx, &gwapitypes.CreateOrgRequest{Name: agolaOrg01, Visibility: gwapitypes.VisibilityPublic}) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + + _, _, err = giteaClient.CreateRepo(gitea.CreateRepoOption{ + Name: "repo01", + Private: false, + }) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + + project01, _, err := gwClient.CreateProject(ctx, &gwapitypes.CreateProjectRequest{ + Name: agolaProject01, + ParentRef: path.Join("org", agolaOrg01), + RemoteSourceName: "gitea", + RepoPath: path.Join(giteaUser01, "repo01"), + Visibility: gwapitypes.VisibilityPublic, + }) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + + project02, _, err := gwClient.CreateProject(ctx, &gwapitypes.CreateProjectRequest{ + Name: agolaProject02, + ParentRef: path.Join("org", agolaOrg01), + RemoteSourceName: "gitea", + RepoPath: path.Join(giteaUser01, "repo01"), + Visibility: gwapitypes.VisibilityPublic, + }) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + + cursor := gwapi.ProjectGroupProjectsCursor{ + LastProjectID: project01.ID, + } + serializedCursor, err := json.Marshal(&cursor) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + expectedCursorS := base64.StdEncoding.EncodeToString(serializedCursor) + + projects, _, err := gwClient.GetProjectGroupProjects(ctx, path.Join("org", agolaOrg01), 1, "") + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + if len(projects.Projects) != 1 { + t.Fatalf("expected %d run got: %d", 1, len(projects.Projects)) + } + if diff := cmp.Diff(projects.Projects[0], project01); diff != "" { + t.Fatalf("project mismatch (-expected +got):\n%s", diff) + } + if projects.Cursor != expectedCursorS { + t.Fatalf("expected cursor %s, got %s", expectedCursorS, projects.Cursor) + } + + projects, _, err = gwClient.GetProjectGroupProjects(ctx, path.Join("org", agolaOrg01), 1, projects.Cursor) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + if len(projects.Projects) != 1 { + t.Fatalf("expected %d run got: %d", 1, len(projects.Projects)) + } + if diff := cmp.Diff(projects.Projects[0], project02); diff != "" { + t.Fatalf("project mismatch (-expected +got):\n%s", diff) + } + if projects.Cursor != "" { + t.Fatalf("expected cursor empty, got %s", projects.Cursor) + } +} + +func TestUsersCursor(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + sc := setup(ctx, t, dir) + defer sc.stop() + + gwClient := gwclient.NewClient(sc.config.Gateway.APIExposedURL, "admintoken") + + user01, _, err := gwClient.CreateUser(ctx, &gwapitypes.CreateUserRequest{UserName: agolaUser01}) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + expectedUser01 := &gwapitypes.PrivateUserResponse{ + ID: user01.ID, + UserName: user01.UserName, + Tokens: []string{}, + LinkedAccounts: []*gwapitypes.LinkedAccountResponse{}, + } + + user02, _, err := gwClient.CreateUser(ctx, &gwapitypes.CreateUserRequest{UserName: agolaUser02}) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + expectedUser02 := &gwapitypes.PrivateUserResponse{ + ID: user02.ID, + UserName: user02.UserName, + Tokens: []string{}, + LinkedAccounts: []*gwapitypes.LinkedAccountResponse{}, + } + + cursor := gwapi.UsersCursor{ + LastUserID: user01.ID, + Asc: true, + } + serializedCursor, err := json.Marshal(&cursor) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + expectedCursorS := base64.StdEncoding.EncodeToString(serializedCursor) + + users, _, err := gwClient.GetUsers(ctx, "", 1, true, "") + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + if len(users.Users) != 1 { + t.Fatalf("expected %d run got: %d", 1, len(users.Users)) + } + if diff := cmp.Diff(users.Users[0], expectedUser01); diff != "" { + t.Fatalf("user mismatch (-expected +got):\n%s", diff) + } + if users.Cursor != expectedCursorS { + t.Fatalf("expected cursor %s, got %s", expectedCursorS, users.Cursor) + } + + users, _, err = gwClient.GetUsers(ctx, "", 1, true, users.Cursor) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + if len(users.Users) != 1 { + t.Fatalf("expected %d run got: %d", 1, len(users.Users)) + } + if diff := cmp.Diff(users.Users[0], expectedUser02); diff != "" { + t.Fatalf("user mismatch (-expected +got):\n%s", diff) + } + if users.Cursor != "" { + t.Fatalf("expected cursor empty, got %s", users.Cursor) + } +} + +func TestRemoteSourcesCursor(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + sc := setup(ctx, t, dir) + defer sc.stop() + + gwClient := gwclient.NewClient(sc.config.Gateway.APIExposedURL, "admintoken") + + remoteSource01, _, err := gwClient.CreateRemoteSource(ctx, &gwapitypes.CreateRemoteSourceRequest{ + Name: "gitea01", + APIURL: "http://test", + Type: "gitea", + AuthType: "password", + SkipSSHHostKeyCheck: true, + }) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + + remoteSource02, _, err := gwClient.CreateRemoteSource(ctx, &gwapitypes.CreateRemoteSourceRequest{ + Name: "gitea02", + APIURL: "http://test", + Type: "gitea", + AuthType: "password", + SkipSSHHostKeyCheck: true, + }) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + + cursor := gwapi.RemoteSourcesCursor{ + LastRemoteSourceID: remoteSource01.ID, + Asc: true, + } + serializedCursor, err := json.Marshal(&cursor) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + expectedCursorS := base64.StdEncoding.EncodeToString(serializedCursor) + + remoteSources, _, err := gwClient.GetRemoteSources(ctx, "", 1, true, "") + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + if len(remoteSources.RemoteSources) != 1 { + t.Fatalf("expected %d remoteSource got: %d", 1, len(remoteSources.RemoteSources)) + } + if diff := cmp.Diff(remoteSources.RemoteSources[0], remoteSource01); diff != "" { + t.Fatalf("user mismatch (-expected +got):\n%s", diff) + } + if remoteSources.Cursor != expectedCursorS { + t.Fatalf("expected cursor %s, got %s", expectedCursorS, remoteSources.Cursor) + } + + remoteSources, _, err = gwClient.GetRemoteSources(ctx, "", 1, true, remoteSources.Cursor) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + if len(remoteSources.RemoteSources) != 1 { + t.Fatalf("expected %d run got: %d", 1, len(remoteSources.RemoteSources)) + } + if diff := cmp.Diff(remoteSources.RemoteSources[0], remoteSource02); diff != "" { + t.Fatalf("user mismatch (-expected +got):\n%s", diff) + } + if remoteSources.Cursor != "" { + t.Fatalf("expected cursor empty, got %s", remoteSources.Cursor) + } +} + +func TestSecretsCursor(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + sc := setup(ctx, t, dir) + defer sc.stop() + + gwClient := gwclient.NewClient(sc.config.Gateway.APIExposedURL, "admintoken") + + _, _, err := gwClient.CreateOrg(ctx, &gwapitypes.CreateOrgRequest{Name: agolaOrg01, Visibility: gwapitypes.VisibilityPublic}) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + + secret01, _, err := gwClient.CreateProjectGroupSecret(ctx, path.Join("org", agolaOrg01), &gwapitypes.CreateSecretRequest{ + Name: "secret01", + Type: gwapitypes.SecretTypeInternal, + Data: map[string]string{"key": "value"}, + }) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + secret01.ParentPath = path.Join("org", agolaOrg01) + + secret02, _, err := gwClient.CreateProjectGroupSecret(ctx, path.Join("org", agolaOrg01), &gwapitypes.CreateSecretRequest{ + Name: "secret02", + Type: gwapitypes.SecretTypeInternal, + Data: map[string]string{"key": "value"}, + }) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + secret02.ParentPath = path.Join("org", agolaOrg01) + + cursor := gwapi.SecretsCursor{ + LastSecretName: secret01.Name, + Asc: true, + } + serializedCursor, err := json.Marshal(&cursor) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + expectedCursorS := base64.StdEncoding.EncodeToString(serializedCursor) + + secrets, _, err := gwClient.GetProjectGroupSecrets(ctx, path.Join("org", agolaOrg01), false, false, "", true, 1, "") + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + if len(secrets.Secrets) != 1 { + t.Fatalf("expected %d secret got: %d", 1, len(secrets.Secrets)) + } + if diff := cmp.Diff(secrets.Secrets[0], secret01); diff != "" { + t.Fatalf("user mismatch (-expected +got):\n%s", diff) + } + if secrets.Cursor != expectedCursorS { + t.Fatalf("expected cursor %s, got %s", expectedCursorS, secrets.Cursor) + } + + secrets, _, err = gwClient.GetProjectGroupSecrets(ctx, path.Join("org", agolaOrg01), false, false, "", true, 1, secrets.Cursor) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + if len(secrets.Secrets) != 1 { + t.Fatalf("expected %d run got: %d", 1, len(secrets.Secrets)) + } + if diff := cmp.Diff(secrets.Secrets[0], secret02); diff != "" { + t.Fatalf("user mismatch (-expected +got):\n%s", diff) + } + if secrets.Cursor != "" { + t.Fatalf("expected cursor empty, got %s", secrets.Cursor) + } +} + +func TestVariablesCursor(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + sc := setup(ctx, t, dir) + defer sc.stop() + + gwClient := gwclient.NewClient(sc.config.Gateway.APIExposedURL, "admintoken") + + _, _, err := gwClient.CreateOrg(ctx, &gwapitypes.CreateOrgRequest{Name: agolaOrg01, Visibility: gwapitypes.VisibilityPublic}) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + + variable01, _, err := gwClient.CreateProjectGroupVariable(ctx, path.Join("org", agolaOrg01), &gwapitypes.CreateVariableRequest{ + Name: "variable01", + Values: []gwapitypes.VariableValueRequest{ + { + SecretName: "secret01", + SecretVar: "secretvar", + }, + }, + }) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + variable01.ParentPath = path.Join("org", agolaOrg01) + + variable02, _, err := gwClient.CreateProjectGroupVariable(ctx, path.Join("org", agolaOrg01), &gwapitypes.CreateVariableRequest{ + Name: "variable02", + Values: []gwapitypes.VariableValueRequest{ + { + SecretName: "secret01", + SecretVar: "secretvar", + }, + }, + }) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + variable02.ParentPath = path.Join("org", agolaOrg01) + + cursor := gwapi.VariablesCursor{ + LastVariableName: variable01.Name, + Asc: true, + } + serializedCursor, err := json.Marshal(&cursor) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + expectedCursorS := base64.StdEncoding.EncodeToString(serializedCursor) + + variables, _, err := gwClient.GetProjectGroupVariables(ctx, path.Join("org", agolaOrg01), false, false, "", true, 1, "") + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + if len(variables.Variables) != 1 { + t.Fatalf("expected %d secret got: %d", 1, len(variables.Variables)) + } + if diff := cmp.Diff(variables.Variables[0], variable01); diff != "" { + t.Fatalf("user mismatch (-expected +got):\n%s", diff) + } + if variables.Cursor != expectedCursorS { + t.Fatalf("expected cursor %s, got %s", expectedCursorS, variables.Cursor) + } + + variables, _, err = gwClient.GetProjectGroupVariables(ctx, path.Join("org", agolaOrg01), false, false, "", true, 1, variables.Cursor) + if err != nil { + t.Fatalf("unexpected err: %v", err) + } + if len(variables.Variables) != 1 { + t.Fatalf("expected %d run got: %d", 1, len(variables.Variables)) + } + if diff := cmp.Diff(variables.Variables[0], variable02); diff != "" { + t.Fatalf("user mismatch (-expected +got):\n%s", diff) + } + if variables.Cursor != "" { + t.Fatalf("expected cursor empty, got %s", variables.Cursor) + } +}