Feature - Pagination for git tree API (go-gitea#5838)

* Feature - Pagination for git tree API

* Handles case when page is negative

* Does a for loop over the start and end rather than all entries

* Removed redundent logic

* Adds per_page as a query parameter

* Adds DEFAULT_GIT_TREES_PER_PAGE for settings, ran make fmt

* Fix typo in cheat-sheet en

* Makes page start at 1, generated swagger

* Use updates to SDK

* Updates to use latest sdk

* Updates swagger for tree api

* Adds test for GetTreeBySHA

* Updates per PR reviews

* Updates per PR reviews

* Remove file

* Formatting

* Fix to swagger file

* Fix to swagger

* Update v1_json.tmpl

* Fix to swagger file

Author: richmahn

Gopkg.lock

@@ -610,6 +610,10 @@ MIN_INTERVAL = 10m
 ENABLE_SWAGGER = true
 ; Max number of items in a page
 MAX_RESPONSE_ITEMS = 50
+; Default paging number of api
+DEFAULT_PAGING_NUM = 30
+; Default and maximum number of items per page for git trees api
+DEFAULT_GIT_TREES_PER_PAGE = 1000
 
 [i18n]
 LANGS = en-US,zh-CN,zh-HK,zh-TW,de-DE,fr-FR,nl-NL,lv-LV,ru-RU,uk-UA,ja-JP,es-ES,pt-BR,pl-PL,bg-BG,it-IT,fi-FI,tr-TR,cs-CZ,sr-SP,sv-SE,ko-KR

custom/conf/app.ini.sample

@@ -317,6 +317,8 @@ Values containing `#` or `;` must be quoted using `` ` `` or `"""`.
 
 - `ENABLE_SWAGGER_ENDPOINT`: **true**: Enables /api/swagger, /api/v1/swagger etc. endpoints. True or false; default is true.
 - `MAX_RESPONSE_ITEMS`: **50**: Max number of items in a page.
+- `DEFAULT_PAGING_NUM`: **30**: Default paging number of api.
+- `DEFAULT_GIT_TREES_PER_PAGE`: **1000**: Default and maximum number of items per page for git trees api.
 
 ## i18n (`i18n`)
 

docs/content/doc/advanced/config-cheat-sheet.en-us.md

@@ -194,7 +194,14 @@ menu:
 - `PULL`: **300**: 内部仓库间拉取的超时时间，单位秒
 - `GC`: **60**: git仓库GC的超时时间，单位秒
 
-## markup (`markup`)
+## API (`api`)
+
+- `ENABLE_SWAGGER`: **true**: 是否启用swagger路由 /api/swagger, /api/v1/swagger etc. endpoints. True 或 false; 默认是  true.
+- `MAX_RESPONSE_ITEMS`: **50**: 一个页面最大的项目数。
+- `DEFAULT_PAGING_NUM`: **30**: API中默认分页条数。
+- `DEFAULT_GIT_TREES_PER_PAGE`: **1000**: GIT TREES API每页的默认和最大项数.
+
+## Markup (`markup`)
 
 外部渲染工具支持，你可以用你熟悉的文档渲染工具. 比如一下将新增一个名字为 `asciidoc` 的渲染工具which is followed `markup.` ini section. And there are some config items below.
 

docs/content/doc/advanced/config-cheat-sheet.zh-cn.md

@@ -549,11 +549,15 @@ var (
 
 	// API settings
 	API = struct {
-		EnableSwagger    bool
-		MaxResponseItems int
+		EnableSwagger          bool
+		MaxResponseItems       int
+		DefaultPagingNum       int
+		DefaultGitTreesPerPage int
 	}{
-		EnableSwagger:    true,
-		MaxResponseItems: 50,
+		EnableSwagger:          true,
+		MaxResponseItems:       50,
+		DefaultPagingNum:       30,
+		DefaultGitTreesPerPage: 1000,
 	}
 
 	U2F = struct {

modules/setting/setting.go

@@ -37,19 +37,34 @@ func GetTree(ctx *context.APIContext) {
 	//   description: sha of the commit
 	//   type: string
 	//   required: true
+	// - name: recursive
+	//   in: query
+	//   description: show all directories and files
+	//   required: false
+	//   type: boolean
+	// - name: page
+	//   in: query
+	//   description: page number; the 'truncated' field in the response will be true if there are still more items after this page, false if the last page
+	//   required: false
+	//   type: integer
+	// - name: per_page
+	//   in: query
+	//   description: number of items per page; default is 1000 or what is set in app.ini as DEFAULT_GIT_TREES_PER_PAGE
+	//   required: false
+	//   type: integer
 	// responses:
 	//   "200":
 	//     "$ref": "#/responses/GitTreeResponse"
 	sha := ctx.Params("sha")
 	if len(sha) == 0 {
-		ctx.Error(400, "sha not provided", nil)
+		ctx.Error(400, "", "sha not provided")
 		return
 	}
 	tree := GetTreeBySHA(ctx, sha)
 	if tree != nil {
 		ctx.JSON(200, tree)
 	} else {
-		ctx.Error(400, "sha invalid", nil)
+		ctx.Error(400, "", "sha invalid")
 	}
 }
 
@@ -87,29 +102,44 @@ func GetTreeBySHA(ctx *context.APIContext, sha string) *gitea.GitTreeResponse {
 	// 40 is the size of the sha1 hash in hexadecimal format.
 	copyPos := len(treeURL) - 40
 
-	if len(entries) > 1000 {
-		tree.Entries = make([]gitea.GitEntry, 1000)
+	page := ctx.QueryInt("page")
+	perPage := ctx.QueryInt("per_page")
+	if perPage <= 0 || perPage > setting.API.DefaultGitTreesPerPage {
+		perPage = setting.API.DefaultGitTreesPerPage
+	}
+	if page <= 0 {
+		page = 1
+	}
+	tree.Page = page
+	tree.TotalCount = len(entries)
+	rangeStart := perPage * (page - 1)
+	if rangeStart >= len(entries) {
+		return tree
+	}
+	var rangeEnd int
+	if len(entries) > perPage {
+		tree.Truncated = true
+	}
+	if rangeStart+perPage < len(entries) {
+		rangeEnd = rangeStart + perPage
 	} else {
-		tree.Entries = make([]gitea.GitEntry, len(entries))
+		rangeEnd = len(entries)
 	}
-	for e := range entries {
-		if e > 1000 {
-			tree.Truncated = true
-			break
-		}
-
-		tree.Entries[e].Path = entries[e].Name()
-		tree.Entries[e].Mode = fmt.Sprintf("%06x", entries[e].Mode())
-		tree.Entries[e].Type = string(entries[e].Type)
-		tree.Entries[e].Size = entries[e].Size()
-		tree.Entries[e].SHA = entries[e].ID.String()
+	tree.Entries = make([]gitea.GitEntry, rangeEnd-rangeStart)
+	for e := rangeStart; e < rangeEnd; e++ {
+		i := e - rangeStart
+		tree.Entries[i].Path = entries[e].Name()
+		tree.Entries[i].Mode = fmt.Sprintf("%06x", entries[e].Mode())
+		tree.Entries[i].Type = string(entries[e].Type)
+		tree.Entries[i].Size = entries[e].Size()
+		tree.Entries[i].SHA = entries[e].ID.String()
 
 		if entries[e].IsDir() {
 			copy(treeURL[copyPos:], entries[e].ID.String())
-			tree.Entries[e].URL = string(treeURL[:])
+			tree.Entries[i].URL = string(treeURL[:])
 		} else {
 			copy(blobURL[copyPos:], entries[e].ID.String())
-			tree.Entries[e].URL = string(blobURL[:])
+			tree.Entries[i].URL = string(blobURL[:])
 		}
 	}
 	return tree

routers/api/v1/repo/tree.go

@@ -0,0 +1,48 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package repo
+
+import (
+	"github.com/stretchr/testify/assert"
+	"testing"
+
+	"code.gitea.io/gitea/models"
+	"code.gitea.io/gitea/modules/context"
+	"code.gitea.io/gitea/modules/test"
+	"code.gitea.io/sdk/gitea"
+)
+
+func TestGetTreeBySHA(t *testing.T) {
+	models.PrepareTestEnv(t)
+	sha := "master"
+	ctx := test.MockContext(t, "user2/repo1")
+	ctx.SetParams(":id", "1")
+	ctx.SetParams(":sha", sha)
+	test.LoadRepo(t, ctx, 1)
+	test.LoadRepoCommit(t, ctx)
+	test.LoadUser(t, ctx, 2)
+	test.LoadGitRepo(t, ctx)
+
+	tree := GetTreeBySHA(&context.APIContext{Context: ctx, Org: nil}, ctx.Params("sha"))
+	expectedTree := &gitea.GitTreeResponse{
+		SHA: "65f1bf27bc3bf70f64657658635e66094edbcb4d",
+		URL: "https://try.gitea.io/api/v1/repos/user2/repo1/git/trees/65f1bf27bc3bf70f64657658635e66094edbcb4d",
+		Entries: []gitea.GitEntry{
+			{
+				Path: "README.md",
+				Mode: "100644",
+				Type: "blob",
+				Size: 30,
+				SHA:  "4b4851ad51df6a7d9f25c979345979eaeb5b349f",
+				URL:  "https://try.gitea.io/api/v1/repos/user2/repo1/git/blobs/4b4851ad51df6a7d9f25c979345979eaeb5b349f",
+			},
+		},
+		Truncated:  false,
+		Page:       1,
+		TotalCount: 1,
+	}
+
+	assert.EqualValues(t, tree, expectedTree)
+}

routers/api/v1/repo/tree_test.go

@@ -1694,6 +1694,24 @@
             "name": "sha",
             "in": "path",
             "required": true
+          },
+          {
+            "type": "boolean",
+            "description": "show all directories and files",
+            "name": "recursive",
+            "in": "query"
+          },
+          {
+            "type": "integer",
+            "description": "page number; the 'truncated' field in the response will be true if there are still more items after this page, false if the last page",
+            "name": "page",
+            "in": "query"
+          },
+          {
+            "type": "integer",
+            "description": "number of items per page; default is 1000 or what is set in app.ini as DEFAULT_GIT_TREES_PER_PAGE",
+            "name": "per_page",
+            "in": "query"
           }
         ],
         "responses": {
@@ -7135,10 +7153,20 @@
       "description": "GitTreeResponse returns a git tree",
       "type": "object",
       "properties": {
+        "page": {
+          "type": "integer",
+          "format": "int64",
+          "x-go-name": "Page"
+        },
         "sha": {
           "type": "string",
           "x-go-name": "SHA"
         },
+        "total_count": {
+          "type": "integer",
+          "format": "int64",
+          "x-go-name": "TotalCount"
+        },
         "tree": {
           "type": "array",
           "items": {