Skip to content

Add avif image file support #32508

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Nov 15, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion custom/conf/app.example.ini
Original file line number Diff line number Diff line change
Expand Up @@ -1912,7 +1912,7 @@ LEVEL = Info
;ENABLED = true
;;
;; Comma-separated list of allowed file extensions (`.zip`), mime types (`text/plain`) or wildcard type (`image/*`, `audio/*`, `video/*`). Empty value or `*/*` allows all types.
;ALLOWED_TYPES = .csv,.docx,.fodg,.fodp,.fods,.fodt,.gif,.gz,.jpeg,.jpg,.log,.md,.mov,.mp4,.odf,.odg,.odp,.ods,.odt,.patch,.pdf,.png,.pptx,.svg,.tgz,.txt,.webm,.xls,.xlsx,.zip
;ALLOWED_TYPES = .avif,.cpuprofile,.csv,.dmp,.docx,.fodg,.fodp,.fods,.fodt,.gif,.gz,.jpeg,.jpg,.json,.jsonc,.log,.md,.mov,.mp4,.odf,.odg,.odp,.ods,.odt,.patch,.pdf,.png,.pptx,.svg,.tgz,.txt,.webm,.webp,.xls,.xlsx,.zip
;;
;; Max size of each file. Defaults to 2048MB
;MAX_SIZE = 2048
Expand Down
4 changes: 2 additions & 2 deletions modules/httplib/serve.go
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ func ServeSetHeaders(w http.ResponseWriter, opts *ServeHeaderOptions) {
w.Header().Add(gzhttp.HeaderNoCompression, "1")
}

contentType := typesniffer.ApplicationOctetStream
contentType := typesniffer.MimeTypeApplicationOctetStream
if opts.ContentType != "" {
if opts.ContentTypeCharset != "" {
contentType = opts.ContentType + "; charset=" + strings.ToLower(opts.ContentTypeCharset)
Expand Down Expand Up @@ -107,7 +107,7 @@ func setServeHeadersByFile(r *http.Request, w http.ResponseWriter, filePath stri
} else if isPlain {
opts.ContentType = "text/plain"
} else {
opts.ContentType = typesniffer.ApplicationOctetStream
opts.ContentType = typesniffer.MimeTypeApplicationOctetStream
}
}

Expand Down
26 changes: 13 additions & 13 deletions modules/setting/attachment.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,33 +3,33 @@

package setting

// Attachment settings
var Attachment = struct {
type AttachmentSettingType struct {
Storage *Storage
AllowedTypes string
MaxSize int64
MaxFiles int
Enabled bool
}{
Storage: &Storage{},
AllowedTypes: ".cpuprofile,.csv,.dmp,.docx,.fodg,.fodp,.fods,.fodt,.gif,.gz,.jpeg,.jpg,.json,.jsonc,.log,.md,.mov,.mp4,.odf,.odg,.odp,.ods,.odt,.patch,.pdf,.png,.pptx,.svg,.tgz,.txt,.webm,.xls,.xlsx,.zip",
MaxSize: 2048,
MaxFiles: 5,
Enabled: true,
}

var Attachment AttachmentSettingType

func loadAttachmentFrom(rootCfg ConfigProvider) (err error) {
Attachment = AttachmentSettingType{
AllowedTypes: ".avif,.cpuprofile,.csv,.dmp,.docx,.fodg,.fodp,.fods,.fodt,.gif,.gz,.jpeg,.jpg,.json,.jsonc,.log,.md,.mov,.mp4,.odf,.odg,.odp,.ods,.odt,.patch,.pdf,.png,.pptx,.svg,.tgz,.txt,.webm,.webp,.xls,.xlsx,.zip",
MaxSize: 2048,
MaxFiles: 5,
Enabled: true,
}
sec, _ := rootCfg.GetSection("attachment")
if sec == nil {
Attachment.Storage, err = getStorage(rootCfg, "attachments", "", nil)
return err
}

Attachment.AllowedTypes = sec.Key("ALLOWED_TYPES").MustString(".cpuprofile,.csv,.dmp,.docx,.fodg,.fodp,.fods,.fodt,.gif,.gz,.jpeg,.jpg,.json,.jsonc,.log,.md,.mov,.mp4,.odf,.odg,.odp,.ods,.odt,.patch,.pdf,.png,.pptx,.svg,.tgz,.txt,.webm,.xls,.xlsx,.zip")
Attachment.MaxSize = sec.Key("MAX_SIZE").MustInt64(2048)
Attachment.MaxFiles = sec.Key("MAX_FILES").MustInt(5)
Attachment.Enabled = sec.Key("ENABLED").MustBool(true)

Attachment.AllowedTypes = sec.Key("ALLOWED_TYPES").MustString(Attachment.AllowedTypes)
Attachment.MaxSize = sec.Key("MAX_SIZE").MustInt64(Attachment.MaxSize)
Attachment.MaxFiles = sec.Key("MAX_FILES").MustInt(Attachment.MaxFiles)
Attachment.Enabled = sec.Key("ENABLED").MustBool(Attachment.Enabled)
Attachment.Storage, err = getStorage(rootCfg, "attachments", "", sec)
return err
}
40 changes: 33 additions & 7 deletions modules/typesniffer/typesniffer.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,12 @@ package typesniffer

import (
"bytes"
"encoding/binary"
"fmt"
"io"
"net/http"
"regexp"
"slices"
"strings"

"code.gitea.io/gitea/modules/util"
Expand All @@ -18,10 +20,10 @@ import (
const sniffLen = 1024

const (
// SvgMimeType MIME type of SVG images.
SvgMimeType = "image/svg+xml"
// ApplicationOctetStream MIME type of binary files.
ApplicationOctetStream = "application/octet-stream"
MimeTypeImageSvg = "image/svg+xml"
MimeTypeImageAvif = "image/avif"

MimeTypeApplicationOctetStream = "application/octet-stream"
)

var (
Expand All @@ -47,7 +49,7 @@ func (ct SniffedType) IsImage() bool {

// IsSvgImage detects if data is an SVG image format
func (ct SniffedType) IsSvgImage() bool {
return strings.Contains(ct.contentType, SvgMimeType)
return strings.Contains(ct.contentType, MimeTypeImageSvg)
}

// IsPDF detects if data is a PDF format
Expand Down Expand Up @@ -81,6 +83,26 @@ func (ct SniffedType) GetMimeType() string {
return strings.SplitN(ct.contentType, ";", 2)[0]
}

// https://en.wikipedia.org/wiki/ISO_base_media_file_format#File_type_box
func detectFileTypeBox(data []byte) (brands []string, found bool) {
if len(data) < 12 {
return nil, false
}
boxSize := int(binary.BigEndian.Uint32(data[:4]))
if boxSize < 12 || boxSize > len(data) {
return nil, false
}
tag := string(data[4:8])
if tag != "ftyp" {
return nil, false
}
brands = append(brands, string(data[8:12]))
for i := 16; i+4 <= boxSize; i += 4 {
brands = append(brands, string(data[i:i+4]))
}
return brands, true
}

// DetectContentType extends http.DetectContentType with more content types. Defaults to text/unknown if input is empty.
func DetectContentType(data []byte) SniffedType {
if len(data) == 0 {
Expand All @@ -94,15 +116,14 @@ func DetectContentType(data []byte) SniffedType {
}

// SVG is unsupported by http.DetectContentType, https://github.com/golang/go/issues/15888

detectByHTML := strings.Contains(ct, "text/plain") || strings.Contains(ct, "text/html")
detectByXML := strings.Contains(ct, "text/xml")
if detectByHTML || detectByXML {
dataProcessed := svgComment.ReplaceAll(data, nil)
dataProcessed = bytes.TrimSpace(dataProcessed)
if detectByHTML && svgTagRegex.Match(dataProcessed) ||
detectByXML && svgTagInXMLRegex.Match(dataProcessed) {
ct = SvgMimeType
ct = MimeTypeImageSvg
}
}

Expand All @@ -116,6 +137,11 @@ func DetectContentType(data []byte) SniffedType {
}
}

fileTypeBrands, found := detectFileTypeBox(data)
if found && slices.Contains(fileTypeBrands, "avif") {
ct = MimeTypeImageAvif
}

if ct == "application/ogg" {
dataHead := data
if len(dataHead) > 256 {
Expand Down
30 changes: 30 additions & 0 deletions modules/typesniffer/typesniffer_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -134,3 +134,33 @@ func TestDetectContentTypeOgg(t *testing.T) {
assert.NoError(t, err)
assert.True(t, st.IsVideo())
}

func TestDetectFileTypeBox(t *testing.T) {
_, found := detectFileTypeBox([]byte("\x00\x00\xff\xffftypAAAA...."))
assert.False(t, found)

brands, found := detectFileTypeBox([]byte("\x00\x00\x00\x0cftypAAAA"))
assert.True(t, found)
assert.Equal(t, []string{"AAAA"}, brands)

brands, found = detectFileTypeBox([]byte("\x00\x00\x00\x10ftypAAAA....BBBB"))
assert.True(t, found)
assert.Equal(t, []string{"AAAA"}, brands)

brands, found = detectFileTypeBox([]byte("\x00\x00\x00\x14ftypAAAA....BBBB"))
assert.True(t, found)
assert.Equal(t, []string{"AAAA", "BBBB"}, brands)

_, found = detectFileTypeBox([]byte("\x00\x00\x00\x14ftypAAAA....BBB"))
assert.False(t, found)

brands, found = detectFileTypeBox([]byte("\x00\x00\x00\x13ftypAAAA....BBB"))
assert.True(t, found)
assert.Equal(t, []string{"AAAA"}, brands)
}

func TestDetectContentTypeAvif(t *testing.T) {
buf := []byte("\x00\x00\x00\x20ftypavif.......................")
st := DetectContentType(buf)
assert.Equal(t, MimeTypeImageAvif, st.contentType)
}
2 changes: 1 addition & 1 deletion web_src/js/utils.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,7 @@ test('encodeURLEncodedBase64, decodeURLEncodedBase64', () => {
});

test('file detection', () => {
for (const name of ['a.jpg', '/a.jpeg', '.file.png', '.webp', 'file.svg']) {
for (const name of ['a.avif', 'a.jpg', '/a.jpeg', '.file.png', '.webp', 'file.svg']) {
expect(isImageFile({name})).toBeTruthy();
}
for (const name of ['', 'a.jpg.x', '/path.png/x', 'webp']) {
Expand Down
2 changes: 1 addition & 1 deletion web_src/js/utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -165,7 +165,7 @@ export function sleep(ms: number): Promise<void> {
}

export function isImageFile({name, type}: {name: string, type?: string}): boolean {
return /\.(jpe?g|png|gif|webp|svg|heic)$/i.test(name || '') || type?.startsWith('image/');
return /\.(avif|jpe?g|png|gif|webp|svg|heic)$/i.test(name || '') || type?.startsWith('image/');
}

export function isVideoFile({name, type}: {name: string, type?: string}): boolean {
Expand Down
Loading