Skip to content

Commit 95383b7

Browse files
danogdanog
authored
Add sitemap support (#18407)
1 parent 97bfabc commit 95383b7

File tree

9 files changed

+257
-2
lines changed

9 files changed

+257
-2
lines changed

‎custom/conf/app.example.ini

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1097,6 +1097,9 @@ PATH =
10971097
;; Number of items that are displayed in home feed
10981098
;FEED_PAGING_NUM = 20
10991099
;;
1100+
;; Number of items that are displayed in a single subsitemap
1101+
;SITEMAP_PAGING_NUM = 20
1102+
;;
11001103
;; Number of maximum commits displayed in commit graph.
11011104
;GRAPH_MAX_COMMIT_NUM = 100
11021105
;;

‎docs/content/doc/advanced/config-cheat-sheet.en-us.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -174,6 +174,7 @@ The following configuration set `Content-Type: application/vnd.android.package-a
174174
- `MEMBERS_PAGING_NUM`: **20**: Number of members that are shown in organization members.
175175
- `FEED_MAX_COMMIT_NUM`: **5**: Number of maximum commits shown in one activity feed.
176176
- `FEED_PAGING_NUM`: **20**: Number of items that are displayed in home feed.
177+
- `SITEMAP_PAGING_NUM`: **20**: Number of items that are displayed in a single subsitemap.
177178
- `GRAPH_MAX_COMMIT_NUM`: **100**: Number of maximum commits shown in the commit graph.
178179
- `CODE_COMMENT_LINES`: **4**: Number of line of codes shown for a code comment.
179180
- `DEFAULT_THEME`: **auto**: \[auto, gitea, arc-green\]: Set the default theme for the Gitea install.

‎modules/setting/setting.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -207,6 +207,7 @@ var (
207207
// UI settings
208208
UI = struct {
209209
ExplorePagingNum int
210+
SitemapPagingNum int
210211
IssuePagingNum int
211212
RepoSearchPagingNum int
212213
MembersPagingNum int
@@ -260,6 +261,7 @@ var (
260261
} `ini:"ui.meta"`
261262
}{
262263
ExplorePagingNum: 20,
264+
SitemapPagingNum: 20,
263265
IssuePagingNum: 10,
264266
RepoSearchPagingNum: 10,
265267
MembersPagingNum: 20,

‎modules/sitemap/sitemap.go

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
// Copyright 2022 The Gitea Authors. All rights reserved.
2+
// Use of this source code is governed by a MIT-style
3+
// license that can be found in the LICENSE file.
4+
5+
package sitemap
6+
7+
import (
8+
"bytes"
9+
"encoding/xml"
10+
"fmt"
11+
"io"
12+
"time"
13+
)
14+
15+
// sitemapFileLimit contains the maximum size of a sitemap file
16+
const sitemapFileLimit = 50 * 1024 * 1024
17+
18+
// Url represents a single sitemap entry
19+
type URL struct {
20+
URL string `xml:"loc"`
21+
LastMod *time.Time `xml:"lastmod,omitempty"`
22+
}
23+
24+
// SitemapUrl represents a sitemap
25+
type Sitemap struct {
26+
XMLName xml.Name
27+
Namespace string `xml:"xmlns,attr"`
28+
29+
URLs []URL `xml:"url"`
30+
}
31+
32+
// NewSitemap creates a sitemap
33+
func NewSitemap() *Sitemap {
34+
return &Sitemap{
35+
XMLName: xml.Name{Local: "urlset"},
36+
Namespace: "http://www.sitemaps.org/schemas/sitemap/0.9",
37+
}
38+
}
39+
40+
// NewSitemap creates a sitemap index.
41+
func NewSitemapIndex() *Sitemap {
42+
return &Sitemap{
43+
XMLName: xml.Name{Local: "sitemapindex"},
44+
Namespace: "http://www.sitemaps.org/schemas/sitemap/0.9",
45+
}
46+
}
47+
48+
// Add adds a URL to the sitemap
49+
func (s *Sitemap) Add(u URL) {
50+
s.URLs = append(s.URLs, u)
51+
}
52+
53+
// Write writes the sitemap to a response
54+
func (s *Sitemap) WriteTo(w io.Writer) (int64, error) {
55+
if len(s.URLs) > 50000 {
56+
return 0, fmt.Errorf("The sitemap contains too many URLs: %d", len(s.URLs))
57+
}
58+
buf := bytes.NewBufferString(xml.Header)
59+
if err := xml.NewEncoder(buf).Encode(s); err != nil {
60+
return 0, err
61+
}
62+
if err := buf.WriteByte('\n'); err != nil {
63+
return 0, err
64+
}
65+
if buf.Len() > sitemapFileLimit {
66+
return 0, fmt.Errorf("The sitemap is too big: %d", buf.Len())
67+
}
68+
return buf.WriteTo(w)
69+
}

‎modules/sitemap/sitemap_test.go

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
// Copyright 2022 The Gitea Authors. All rights reserved.
2+
// Use of this source code is governed by a MIT-style
3+
// license that can be found in the LICENSE file.
4+
5+
package sitemap
6+
7+
import (
8+
"bytes"
9+
"encoding/xml"
10+
"fmt"
11+
"strings"
12+
"testing"
13+
"time"
14+
15+
"github.com/stretchr/testify/assert"
16+
)
17+
18+
func TestOk(t *testing.T) {
19+
testReal := func(s *Sitemap, name string, urls []URL, expected string) {
20+
for _, url := range urls {
21+
s.Add(url)
22+
}
23+
buf := &bytes.Buffer{}
24+
_, err := s.WriteTo(buf)
25+
assert.NoError(t, nil, err)
26+
assert.Equal(t, xml.Header+"<"+name+" xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">"+expected+"</"+name+">\n", buf.String())
27+
}
28+
test := func(urls []URL, expected string) {
29+
testReal(NewSitemap(), "urlset", urls, expected)
30+
testReal(NewSitemapIndex(), "sitemapindex", urls, expected)
31+
}
32+
33+
ts := time.Unix(1651322008, 0).UTC()
34+
35+
test(
36+
[]URL{},
37+
"",
38+
)
39+
test(
40+
[]URL{
41+
{URL: "https://gitea.io/test1", LastMod: &ts},
42+
},
43+
"<url><loc>https://gitea.io/test1</loc><lastmod>2022-04-30T12:33:28Z</lastmod></url>",
44+
)
45+
test(
46+
[]URL{
47+
{URL: "https://gitea.io/test2", LastMod: nil},
48+
},
49+
"<url><loc>https://gitea.io/test2</loc></url>",
50+
)
51+
test(
52+
[]URL{
53+
{URL: "https://gitea.io/test1", LastMod: &ts},
54+
{URL: "https://gitea.io/test2", LastMod: nil},
55+
},
56+
"<url><loc>https://gitea.io/test1</loc><lastmod>2022-04-30T12:33:28Z</lastmod></url>"+
57+
"<url><loc>https://gitea.io/test2</loc></url>",
58+
)
59+
}
60+
61+
func TestTooManyURLs(t *testing.T) {
62+
s := NewSitemap()
63+
for i := 0; i < 50001; i++ {
64+
s.Add(URL{URL: fmt.Sprintf("https://gitea.io/test%d", i)})
65+
}
66+
buf := &bytes.Buffer{}
67+
_, err := s.WriteTo(buf)
68+
assert.EqualError(t, err, "The sitemap contains too many URLs: 50001")
69+
}
70+
71+
func TestSitemapTooBig(t *testing.T) {
72+
s := NewSitemap()
73+
s.Add(URL{URL: strings.Repeat("b", sitemapFileLimit)})
74+
buf := &bytes.Buffer{}
75+
_, err := s.WriteTo(buf)
76+
assert.EqualError(t, err, "The sitemap is too big: 52428931")
77+
}

‎routers/web/explore/repo.go

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,9 @@ import (
1111
repo_model "code.gitea.io/gitea/models/repo"
1212
"code.gitea.io/gitea/modules/base"
1313
"code.gitea.io/gitea/modules/context"
14+
"code.gitea.io/gitea/modules/log"
1415
"code.gitea.io/gitea/modules/setting"
16+
"code.gitea.io/gitea/modules/sitemap"
1517
)
1618

1719
const (
@@ -30,11 +32,21 @@ type RepoSearchOptions struct {
3032

3133
// RenderRepoSearch render repositories search page
3234
func RenderRepoSearch(ctx *context.Context, opts *RepoSearchOptions) {
33-
page := ctx.FormInt("page")
35+
// Sitemap index for sitemap paths
36+
page := int(ctx.ParamsInt64("idx"))
37+
isSitemap := ctx.Params("idx") != ""
38+
if page <= 1 {
39+
page = ctx.FormInt("page")
40+
}
41+
3442
if page <= 0 {
3543
page = 1
3644
}
3745

46+
if isSitemap {
47+
opts.PageSize = setting.UI.SitemapPagingNum
48+
}
49+
3850
var (
3951
repos []*repo_model.Repository
4052
count int64
@@ -100,6 +112,18 @@ func RenderRepoSearch(ctx *context.Context, opts *RepoSearchOptions) {
100112
ctx.ServerError("SearchRepository", err)
101113
return
102114
}
115+
if isSitemap {
116+
m := sitemap.NewSitemap()
117+
for _, item := range repos {
118+
m.Add(sitemap.URL{URL: item.HTMLURL(), LastMod: item.UpdatedUnix.AsTimePtr()})
119+
}
120+
ctx.Resp.Header().Set("Content-Type", "text/xml")
121+
if _, err := m.WriteTo(ctx.Resp); err != nil {
122+
log.Error("Failed writing sitemap: %v", err)
123+
}
124+
return
125+
}
126+
103127
ctx.Data["Keyword"] = keyword
104128
ctx.Data["Total"] = count
105129
ctx.Data["Repos"] = repos

‎routers/web/explore/user.go

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,9 @@ import (
1212
user_model "code.gitea.io/gitea/models/user"
1313
"code.gitea.io/gitea/modules/base"
1414
"code.gitea.io/gitea/modules/context"
15+
"code.gitea.io/gitea/modules/log"
1516
"code.gitea.io/gitea/modules/setting"
17+
"code.gitea.io/gitea/modules/sitemap"
1618
"code.gitea.io/gitea/modules/structs"
1719
"code.gitea.io/gitea/modules/util"
1820
)
@@ -33,11 +35,20 @@ func isKeywordValid(keyword string) bool {
3335

3436
// RenderUserSearch render user search page
3537
func RenderUserSearch(ctx *context.Context, opts *user_model.SearchUserOptions, tplName base.TplName) {
36-
opts.Page = ctx.FormInt("page")
38+
// Sitemap index for sitemap paths
39+
opts.Page = int(ctx.ParamsInt64("idx"))
40+
isSitemap := ctx.Params("idx") != ""
41+
if opts.Page <= 1 {
42+
opts.Page = ctx.FormInt("page")
43+
}
3744
if opts.Page <= 1 {
3845
opts.Page = 1
3946
}
4047

48+
if isSitemap {
49+
opts.PageSize = setting.UI.SitemapPagingNum
50+
}
51+
4152
var (
4253
users []*user_model.User
4354
count int64
@@ -73,6 +84,18 @@ func RenderUserSearch(ctx *context.Context, opts *user_model.SearchUserOptions,
7384
return
7485
}
7586
}
87+
if isSitemap {
88+
m := sitemap.NewSitemap()
89+
for _, item := range users {
90+
m.Add(sitemap.URL{URL: item.HTMLURL(), LastMod: item.UpdatedUnix.AsTimePtr()})
91+
}
92+
ctx.Resp.Header().Set("Content-Type", "text/xml")
93+
if _, err := m.WriteTo(ctx.Resp); err != nil {
94+
log.Error("Failed writing sitemap: %v", err)
95+
}
96+
return
97+
}
98+
7699
ctx.Data["Keyword"] = opts.Keyword
77100
ctx.Data["Total"] = count
78101
ctx.Data["Users"] = users

‎routers/web/home.go

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,11 +7,18 @@ package web
77

88
import (
99
"net/http"
10+
"strconv"
1011

12+
"code.gitea.io/gitea/models/db"
13+
repo_model "code.gitea.io/gitea/models/repo"
14+
user_model "code.gitea.io/gitea/models/user"
1115
"code.gitea.io/gitea/modules/base"
1216
"code.gitea.io/gitea/modules/context"
1317
"code.gitea.io/gitea/modules/log"
1418
"code.gitea.io/gitea/modules/setting"
19+
"code.gitea.io/gitea/modules/sitemap"
20+
"code.gitea.io/gitea/modules/structs"
21+
"code.gitea.io/gitea/modules/util"
1522
"code.gitea.io/gitea/modules/web/middleware"
1623
"code.gitea.io/gitea/routers/web/auth"
1724
"code.gitea.io/gitea/routers/web/user"
@@ -59,6 +66,52 @@ func Home(ctx *context.Context) {
5966
ctx.HTML(http.StatusOK, tplHome)
6067
}
6168

69+
// HomeSitemap renders the main sitemap
70+
func HomeSitemap(ctx *context.Context) {
71+
m := sitemap.NewSitemapIndex()
72+
if !setting.Service.Explore.DisableUsersPage {
73+
_, cnt, err := user_model.SearchUsers(&user_model.SearchUserOptions{
74+
Type: user_model.UserTypeIndividual,
75+
ListOptions: db.ListOptions{PageSize: 1},
76+
IsActive: util.OptionalBoolTrue,
77+
Visible: []structs.VisibleType{structs.VisibleTypePublic},
78+
})
79+
if err != nil {
80+
ctx.ServerError("SearchUsers", err)
81+
return
82+
}
83+
count := int(cnt)
84+
idx := 1
85+
for i := 0; i < count; i += setting.UI.SitemapPagingNum {
86+
m.Add(sitemap.URL{URL: setting.AppURL + "explore/users/sitemap-" + strconv.Itoa(idx) + ".xml"})
87+
idx++
88+
}
89+
}
90+
91+
_, cnt, err := repo_model.SearchRepository(&repo_model.SearchRepoOptions{
92+
ListOptions: db.ListOptions{
93+
PageSize: 1,
94+
},
95+
Actor: ctx.Doer,
96+
AllPublic: true,
97+
})
98+
if err != nil {
99+
ctx.ServerError("SearchRepository", err)
100+
return
101+
}
102+
count := int(cnt)
103+
idx := 1
104+
for i := 0; i < count; i += setting.UI.SitemapPagingNum {
105+
m.Add(sitemap.URL{URL: setting.AppURL + "explore/repos/sitemap-" + strconv.Itoa(idx) + ".xml"})
106+
idx++
107+
}
108+
109+
ctx.Resp.Header().Set("Content-Type", "text/xml")
110+
if _, err := m.WriteTo(ctx.Resp); err != nil {
111+
log.Error("Failed writing sitemap: %v", err)
112+
}
113+
}
114+
62115
// NotFound render 404 page
63116
func NotFound(ctx *context.Context) {
64117
ctx.Data["Title"] = "Page Not Found"

‎routers/web/web.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -294,6 +294,7 @@ func RegisterRoutes(m *web.Route) {
294294
// Routers.
295295
// for health check
296296
m.Get("/", Home)
297+
m.Get("/sitemap.xml", ignExploreSignIn, HomeSitemap)
297298
m.Group("/.well-known", func() {
298299
m.Get("/openid-configuration", auth.OIDCWellKnown)
299300
m.Group("", func() {
@@ -310,7 +311,9 @@ func RegisterRoutes(m *web.Route) {
310311
ctx.Redirect(setting.AppSubURL + "/explore/repos")
311312
})
312313
m.Get("/repos", explore.Repos)
314+
m.Get("/repos/sitemap-{idx}.xml", explore.Repos)
313315
m.Get("/users", explore.Users)
316+
m.Get("/users/sitemap-{idx}.xml", explore.Users)
314317
m.Get("/organizations", explore.Organizations)
315318
m.Get("/code", explore.Code)
316319
m.Get("/topics/search", explore.TopicSearch)

0 commit comments

Comments
 (0)