|
| 1 | +// Copyright 2014 The Gogs Authors. All rights reserved. |
| 2 | +// Use of this source code is governed by a MIT-style |
| 3 | +// license that can be found in the LICENSE file. |
| 4 | + |
| 5 | +package charset |
| 6 | + |
| 7 | +import ( |
| 8 | + "bytes" |
| 9 | + "fmt" |
| 10 | + "unicode/utf8" |
| 11 | + |
| 12 | + "code.gitea.io/gitea/modules/log" |
| 13 | + "code.gitea.io/gitea/modules/setting" |
| 14 | + |
| 15 | + "github.com/gogits/chardet" |
| 16 | + "golang.org/x/net/html/charset" |
| 17 | + "golang.org/x/text/transform" |
| 18 | +) |
| 19 | + |
| 20 | +// UTF8BOM is the utf-8 byte-order marker |
| 21 | +var UTF8BOM = []byte{'\xef', '\xbb', '\xbf'} |
| 22 | + |
| 23 | +// ToUTF8WithErr converts content to UTF8 encoding |
| 24 | +func ToUTF8WithErr(content []byte) (string, error) { |
| 25 | + charsetLabel, err := DetectEncoding(content) |
| 26 | + if err != nil { |
| 27 | + return "", err |
| 28 | + } else if charsetLabel == "UTF-8" { |
| 29 | + return string(RemoveBOMIfPresent(content)), nil |
| 30 | + } |
| 31 | + |
| 32 | + encoding, _ := charset.Lookup(charsetLabel) |
| 33 | + if encoding == nil { |
| 34 | + return string(content), fmt.Errorf("Unknown encoding: %s", charsetLabel) |
| 35 | + } |
| 36 | + |
| 37 | + // If there is an error, we concatenate the nicely decoded part and the |
| 38 | + // original left over. This way we won't lose data. |
| 39 | + result, n, err := transform.Bytes(encoding.NewDecoder(), content) |
| 40 | + if err != nil { |
| 41 | + result = append(result, content[n:]...) |
| 42 | + } |
| 43 | + |
| 44 | + result = RemoveBOMIfPresent(result) |
| 45 | + |
| 46 | + return string(result), err |
| 47 | +} |
| 48 | + |
| 49 | +// ToUTF8WithFallback detects the encoding of content and coverts to UTF-8 if possible |
| 50 | +func ToUTF8WithFallback(content []byte) []byte { |
| 51 | + charsetLabel, err := DetectEncoding(content) |
| 52 | + if err != nil || charsetLabel == "UTF-8" { |
| 53 | + return RemoveBOMIfPresent(content) |
| 54 | + } |
| 55 | + |
| 56 | + encoding, _ := charset.Lookup(charsetLabel) |
| 57 | + if encoding == nil { |
| 58 | + return content |
| 59 | + } |
| 60 | + |
| 61 | + // If there is an error, we concatenate the nicely decoded part and the |
| 62 | + // original left over. This way we won't lose data. |
| 63 | + result, n, err := transform.Bytes(encoding.NewDecoder(), content) |
| 64 | + if err != nil { |
| 65 | + return append(result, content[n:]...) |
| 66 | + } |
| 67 | + |
| 68 | + return RemoveBOMIfPresent(result) |
| 69 | +} |
| 70 | + |
| 71 | +// ToUTF8 converts content to UTF8 encoding and ignore error |
| 72 | +func ToUTF8(content string) string { |
| 73 | + res, _ := ToUTF8WithErr([]byte(content)) |
| 74 | + return res |
| 75 | +} |
| 76 | + |
| 77 | +// ToUTF8DropErrors makes sure the return string is valid utf-8; attempts conversion if possible |
| 78 | +func ToUTF8DropErrors(content []byte) []byte { |
| 79 | + charsetLabel, err := DetectEncoding(content) |
| 80 | + if err != nil || charsetLabel == "UTF-8" { |
| 81 | + return RemoveBOMIfPresent(content) |
| 82 | + } |
| 83 | + |
| 84 | + encoding, _ := charset.Lookup(charsetLabel) |
| 85 | + if encoding == nil { |
| 86 | + return content |
| 87 | + } |
| 88 | + |
| 89 | + // We ignore any non-decodable parts from the file. |
| 90 | + // Some parts might be lost |
| 91 | + var decoded []byte |
| 92 | + decoder := encoding.NewDecoder() |
| 93 | + idx := 0 |
| 94 | + for { |
| 95 | + result, n, err := transform.Bytes(decoder, content[idx:]) |
| 96 | + decoded = append(decoded, result...) |
| 97 | + if err == nil { |
| 98 | + break |
| 99 | + } |
| 100 | + decoded = append(decoded, ' ') |
| 101 | + idx = idx + n + 1 |
| 102 | + if idx >= len(content) { |
| 103 | + break |
| 104 | + } |
| 105 | + } |
| 106 | + |
| 107 | + return RemoveBOMIfPresent(decoded) |
| 108 | +} |
| 109 | + |
| 110 | +// RemoveBOMIfPresent removes a UTF-8 BOM from a []byte |
| 111 | +func RemoveBOMIfPresent(content []byte) []byte { |
| 112 | + if len(content) > 2 && bytes.Equal(content[0:3], UTF8BOM) { |
| 113 | + return content[3:] |
| 114 | + } |
| 115 | + return content |
| 116 | +} |
| 117 | + |
| 118 | +// DetectEncoding detect the encoding of content |
| 119 | +func DetectEncoding(content []byte) (string, error) { |
| 120 | + if utf8.Valid(content) { |
| 121 | + log.Debug("Detected encoding: utf-8 (fast)") |
| 122 | + return "UTF-8", nil |
| 123 | + } |
| 124 | + |
| 125 | + textDetector := chardet.NewTextDetector() |
| 126 | + var detectContent []byte |
| 127 | + if len(content) < 1024 { |
| 128 | + // Check if original content is valid |
| 129 | + if _, err := textDetector.DetectBest(content); err != nil { |
| 130 | + return "", err |
| 131 | + } |
| 132 | + times := 1024 / len(content) |
| 133 | + detectContent = make([]byte, 0, times*len(content)) |
| 134 | + for i := 0; i < times; i++ { |
| 135 | + detectContent = append(detectContent, content...) |
| 136 | + } |
| 137 | + } else { |
| 138 | + detectContent = content |
| 139 | + } |
| 140 | + result, err := textDetector.DetectBest(detectContent) |
| 141 | + if err != nil { |
| 142 | + return "", err |
| 143 | + } |
| 144 | + // FIXME: to properly decouple this function the fallback ANSI charset should be passed as an argument |
| 145 | + if result.Charset != "UTF-8" && len(setting.Repository.AnsiCharset) > 0 { |
| 146 | + log.Debug("Using default AnsiCharset: %s", setting.Repository.AnsiCharset) |
| 147 | + return setting.Repository.AnsiCharset, err |
| 148 | + } |
| 149 | + |
| 150 | + log.Debug("Detected encoding: %s", result.Charset) |
| 151 | + return result.Charset, err |
| 152 | +} |
0 commit comments