Allow markdown files to read from the LFS (#5787)

This PR makes it possible for the markdown renderer to render images and media straight from the LFS.

Fix #5746

Signed-off-by: Andrew Thornton [[email protected]](mailto:[email protected])

Author: zeripath

integrations/download_test.go

@@ -22,3 +22,15 @@ func TestDownloadByID(t *testing.T) {
 
 	assert.Equal(t, "# repo1\n\nDescription for repo1", resp.Body.String())
 }
+
+func TestDownloadByIDMedia(t *testing.T) {
+	prepareTestEnv(t)
+
+	session := loginUser(t, "user2")
+
+	// Request raw blob
+	req := NewRequest(t, "GET", "/user2/repo1/media/blob/4b4851ad51df6a7d9f25c979345979eaeb5b349f")
+	resp := session.MakeRequest(t, req, http.StatusOK)
+
+	assert.Equal(t, "# repo1\n\nDescription for repo1", resp.Body.String())
+}

integrations/git_test.go

@@ -8,13 +8,16 @@ import (
 	"crypto/rand"
 	"fmt"
 	"io/ioutil"
+	"net/http"
 	"net/url"
 	"os"
+	"path"
 	"path/filepath"
 	"testing"
 	"time"
 
 	"code.gitea.io/git"
+	"code.gitea.io/gitea/models"
 
 	"github.com/stretchr/testify/assert"
 )
@@ -39,6 +42,8 @@ func testGit(t *testing.T, u *url.URL) {
 		httpContext.Reponame = "repo-tmp-17"
 
 		dstPath, err := ioutil.TempDir("", httpContext.Reponame)
+		var little, big, littleLFS, bigLFS string
+
 		assert.NoError(t, err)
 		defer os.RemoveAll(dstPath)
 		t.Run("Standard", func(t *testing.T) {
@@ -53,10 +58,10 @@ func testGit(t *testing.T, u *url.URL) {
 
 			t.Run("PushCommit", func(t *testing.T) {
 				t.Run("Little", func(t *testing.T) {
-					commitAndPush(t, littleSize, dstPath)
+					little = commitAndPush(t, littleSize, dstPath)
 				})
 				t.Run("Big", func(t *testing.T) {
-					commitAndPush(t, bigSize, dstPath)
+					big = commitAndPush(t, bigSize, dstPath)
 				})
 			})
 		})
@@ -71,16 +76,60 @@ func testGit(t *testing.T, u *url.URL) {
 				assert.NoError(t, err)
 
 				t.Run("Little", func(t *testing.T) {
-					commitAndPush(t, littleSize, dstPath)
+					littleLFS = commitAndPush(t, littleSize, dstPath)
 				})
 				t.Run("Big", func(t *testing.T) {
-					commitAndPush(t, bigSize, dstPath)
+					bigLFS = commitAndPush(t, bigSize, dstPath)
 				})
 			})
 			t.Run("Locks", func(t *testing.T) {
 				lockTest(t, u.String(), dstPath)
 			})
 		})
+		t.Run("Raw", func(t *testing.T) {
+			session := loginUser(t, "user2")
+
+			// Request raw paths
+			req := NewRequest(t, "GET", path.Join("/user2/repo-tmp-17/raw/branch/master/", little))
+			resp := session.MakeRequest(t, req, http.StatusOK)
+			assert.Equal(t, littleSize, resp.Body.Len())
+
+			req = NewRequest(t, "GET", path.Join("/user2/repo-tmp-17/raw/branch/master/", big))
+			nilResp := session.MakeRequestNilResponseRecorder(t, req, http.StatusOK)
+			assert.Equal(t, bigSize, nilResp.Length)
+
+			req = NewRequest(t, "GET", path.Join("/user2/repo-tmp-17/raw/branch/master/", littleLFS))
+			resp = session.MakeRequest(t, req, http.StatusOK)
+			assert.NotEqual(t, littleSize, resp.Body.Len())
+			assert.Contains(t, resp.Body.String(), models.LFSMetaFileIdentifier)
+
+			req = NewRequest(t, "GET", path.Join("/user2/repo-tmp-17/raw/branch/master/", bigLFS))
+			resp = session.MakeRequest(t, req, http.StatusOK)
+			assert.NotEqual(t, bigSize, resp.Body.Len())
+			assert.Contains(t, resp.Body.String(), models.LFSMetaFileIdentifier)
+
+		})
+		t.Run("Media", func(t *testing.T) {
+			session := loginUser(t, "user2")
+
+			// Request media paths
+			req := NewRequest(t, "GET", path.Join("/user2/repo-tmp-17/media/branch/master/", little))
+			resp := session.MakeRequestNilResponseRecorder(t, req, http.StatusOK)
+			assert.Equal(t, littleSize, resp.Length)
+
+			req = NewRequest(t, "GET", path.Join("/user2/repo-tmp-17/media/branch/master/", big))
+			resp = session.MakeRequestNilResponseRecorder(t, req, http.StatusOK)
+			assert.Equal(t, bigSize, resp.Length)
+
+			req = NewRequest(t, "GET", path.Join("/user2/repo-tmp-17/media/branch/master/", littleLFS))
+			resp = session.MakeRequestNilResponseRecorder(t, req, http.StatusOK)
+			assert.Equal(t, littleSize, resp.Length)
+
+			req = NewRequest(t, "GET", path.Join("/user2/repo-tmp-17/media/branch/master/", bigLFS))
+			resp = session.MakeRequestNilResponseRecorder(t, req, http.StatusOK)
+			assert.Equal(t, bigSize, resp.Length)
+		})
+
 	})
 	t.Run("SSH", func(t *testing.T) {
 		sshContext := baseAPITestContext
@@ -97,6 +146,7 @@ func testGit(t *testing.T, u *url.URL) {
 			dstPath, err := ioutil.TempDir("", sshContext.Reponame)
 			assert.NoError(t, err)
 			defer os.RemoveAll(dstPath)
+			var little, big, littleLFS, bigLFS string
 
 			t.Run("Standard", func(t *testing.T) {
 				t.Run("CreateRepo", doAPICreateRepository(sshContext, false))
@@ -107,10 +157,10 @@ func testGit(t *testing.T, u *url.URL) {
 				//time.Sleep(5 * time.Minute)
 				t.Run("PushCommit", func(t *testing.T) {
 					t.Run("Little", func(t *testing.T) {
-						commitAndPush(t, littleSize, dstPath)
+						little = commitAndPush(t, littleSize, dstPath)
 					})
 					t.Run("Big", func(t *testing.T) {
-						commitAndPush(t, bigSize, dstPath)
+						big = commitAndPush(t, bigSize, dstPath)
 					})
 				})
 			})
@@ -125,16 +175,59 @@ func testGit(t *testing.T, u *url.URL) {
 					assert.NoError(t, err)
 
 					t.Run("Little", func(t *testing.T) {
-						commitAndPush(t, littleSize, dstPath)
+						littleLFS = commitAndPush(t, littleSize, dstPath)
 					})
 					t.Run("Big", func(t *testing.T) {
-						commitAndPush(t, bigSize, dstPath)
+						bigLFS = commitAndPush(t, bigSize, dstPath)
 					})
 				})
 				t.Run("Locks", func(t *testing.T) {
 					lockTest(t, u.String(), dstPath)
 				})
 			})
+			t.Run("Raw", func(t *testing.T) {
+				session := loginUser(t, "user2")
+
+				// Request raw paths
+				req := NewRequest(t, "GET", path.Join("/user2/repo-tmp-18/raw/branch/master/", little))
+				resp := session.MakeRequest(t, req, http.StatusOK)
+				assert.Equal(t, littleSize, resp.Body.Len())
+
+				req = NewRequest(t, "GET", path.Join("/user2/repo-tmp-18/raw/branch/master/", big))
+				resp = session.MakeRequest(t, req, http.StatusOK)
+				assert.Equal(t, bigSize, resp.Body.Len())
+
+				req = NewRequest(t, "GET", path.Join("/user2/repo-tmp-18/raw/branch/master/", littleLFS))
+				resp = session.MakeRequest(t, req, http.StatusOK)
+				assert.NotEqual(t, littleSize, resp.Body.Len())
+				assert.Contains(t, resp.Body.String(), models.LFSMetaFileIdentifier)
+
+				req = NewRequest(t, "GET", path.Join("/user2/repo-tmp-18/raw/branch/master/", bigLFS))
+				resp = session.MakeRequest(t, req, http.StatusOK)
+				assert.NotEqual(t, bigSize, resp.Body.Len())
+				assert.Contains(t, resp.Body.String(), models.LFSMetaFileIdentifier)
+
+			})
+			t.Run("Media", func(t *testing.T) {
+				session := loginUser(t, "user2")
+
+				// Request media paths
+				req := NewRequest(t, "GET", path.Join("/user2/repo-tmp-18/media/branch/master/", little))
+				resp := session.MakeRequest(t, req, http.StatusOK)
+				assert.Equal(t, littleSize, resp.Body.Len())
+
+				req = NewRequest(t, "GET", path.Join("/user2/repo-tmp-18/media/branch/master/", big))
+				resp = session.MakeRequest(t, req, http.StatusOK)
+				assert.Equal(t, bigSize, resp.Body.Len())
+
+				req = NewRequest(t, "GET", path.Join("/user2/repo-tmp-18/media/branch/master/", littleLFS))
+				resp = session.MakeRequest(t, req, http.StatusOK)
+				assert.Equal(t, littleSize, resp.Body.Len())
+
+				req = NewRequest(t, "GET", path.Join("/user2/repo-tmp-18/media/branch/master/", bigLFS))
+				resp = session.MakeRequest(t, req, http.StatusOK)
+				assert.Equal(t, bigSize, resp.Body.Len())
+			})
 
 		})
 
@@ -162,34 +255,35 @@ func lockTest(t *testing.T, remote, repoPath string) {
 	assert.NoError(t, err)
 }
 
-func commitAndPush(t *testing.T, size int, repoPath string) {
-	err := generateCommitWithNewData(size, repoPath, "[email protected]", "User Two")
+func commitAndPush(t *testing.T, size int, repoPath string) string {
+	name, err := generateCommitWithNewData(size, repoPath, "[email protected]", "User Two")
 	assert.NoError(t, err)
 	_, err = git.NewCommand("push").RunInDir(repoPath) //Push
 	assert.NoError(t, err)
+	return name
 }
 
-func generateCommitWithNewData(size int, repoPath, email, fullName string) error {
+func generateCommitWithNewData(size int, repoPath, email, fullName string) (string, error) {
 	//Generate random file
 	data := make([]byte, size)
 	_, err := rand.Read(data)
 	if err != nil {
-		return err
+		return "", err
 	}
 	tmpFile, err := ioutil.TempFile(repoPath, "data-file-")
 	if err != nil {
-		return err
+		return "", err
 	}
 	defer tmpFile.Close()
 	_, err = tmpFile.Write(data)
 	if err != nil {
-		return err
+		return "", err
 	}
 
 	//Commit
 	err = git.AddChanges(repoPath, false, filepath.Base(tmpFile.Name()))
 	if err != nil {
-		return err
+		return "", err
 	}
 	err = git.CommitChanges(repoPath, git.CommitChangesOptions{
 		Committer: &git.Signature{
@@ -204,5 +298,5 @@ func generateCommitWithNewData(size int, repoPath, email, fullName string) error
 		},
 		Message: fmt.Sprintf("Testing commit @ %v", time.Now()),
 	})
-	return err
+	return filepath.Base(tmpFile.Name()), err
 }

integrations/integration_test.go

@@ -35,6 +35,23 @@ import (
 
 var mac *macaron.Macaron
 
+type NilResponseRecorder struct {
+	httptest.ResponseRecorder
+	Length int
+}
+
+func (n *NilResponseRecorder) Write(b []byte) (int, error) {
+	n.Length = n.Length + len(b)
+	return len(b), nil
+}
+
+// NewRecorder returns an initialized ResponseRecorder.
+func NewNilResponseRecorder() *NilResponseRecorder {
+	return &NilResponseRecorder{
+		ResponseRecorder: *httptest.NewRecorder(),
+	}
+}
+
 func TestMain(m *testing.M) {
 	initIntegrationTest()
 	mac = routes.NewMacaron()
@@ -192,6 +209,22 @@ func (s *TestSession) MakeRequest(t testing.TB, req *http.Request, expectedStatu
 	return resp
 }
 
+func (s *TestSession) MakeRequestNilResponseRecorder(t testing.TB, req *http.Request, expectedStatus int) *NilResponseRecorder {
+	baseURL, err := url.Parse(setting.AppURL)
+	assert.NoError(t, err)
+	for _, c := range s.jar.Cookies(baseURL) {
+		req.AddCookie(c)
+	}
+	resp := MakeRequestNilResponseRecorder(t, req, expectedStatus)
+
+	ch := http.Header{}
+	ch.Add("Cookie", strings.Join(resp.HeaderMap["Set-Cookie"], ";"))
+	cr := http.Request{Header: ch}
+	s.jar.SetCookies(baseURL, cr.Cookies())
+
+	return resp
+}
+
 const userPassword = "password"
 
 var loginSessionCache = make(map[string]*TestSession, 10)
@@ -305,6 +338,18 @@ func MakeRequest(t testing.TB, req *http.Request, expectedStatus int) *httptest.
 	return recorder
 }
 
+func MakeRequestNilResponseRecorder(t testing.TB, req *http.Request, expectedStatus int) *NilResponseRecorder {
+	recorder := NewNilResponseRecorder()
+	mac.ServeHTTP(recorder, req)
+	if expectedStatus != NoExpectedStatus {
+		if !assert.EqualValues(t, expectedStatus, recorder.Code,
+			"Request: %s %s", req.Method, req.URL.String()) {
+			logUnexpectedResponse(t, &recorder.ResponseRecorder)
+		}
+	}
+	return recorder
+}
+
 // logUnexpectedResponse logs the contents of an unexpected response.
 func logUnexpectedResponse(t testing.TB, recorder *httptest.ResponseRecorder) {
 	respBytes := recorder.Body.Bytes()

modules/lfs/pointers.go

@@ -0,0 +1,69 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package lfs
+
+import (
+	"io"
+	"strconv"
+	"strings"
+
+	"code.gitea.io/gitea/models"
+	"code.gitea.io/gitea/modules/base"
+	"code.gitea.io/gitea/modules/setting"
+)
+
+// ReadPointerFile will return a partially filled LFSMetaObject if the provided reader is a pointer file
+func ReadPointerFile(reader io.Reader) (*models.LFSMetaObject, *[]byte) {
+	if !setting.LFS.StartServer {
+		return nil, nil
+	}
+
+	buf := make([]byte, 1024)
+	n, _ := reader.Read(buf)
+	buf = buf[:n]
+
+	if isTextFile := base.IsTextFile(buf); !isTextFile {
+		return nil, nil
+	}
+
+	return IsPointerFile(&buf), &buf
+}
+
+// IsPointerFile will return a partially filled LFSMetaObject if the provided byte slice is a pointer file
+func IsPointerFile(buf *[]byte) *models.LFSMetaObject {
+	if !setting.LFS.StartServer {
+		return nil
+	}
+
+	headString := string(*buf)
+	if !strings.HasPrefix(headString, models.LFSMetaFileIdentifier) {
+		return nil
+	}
+
+	splitLines := strings.Split(headString, "\n")
+	if len(splitLines) < 3 {
+		return nil
+	}
+
+	oid := strings.TrimPrefix(splitLines[1], models.LFSMetaFileOidPrefix)
+	size, err := strconv.ParseInt(strings.TrimPrefix(splitLines[2], "size "), 10, 64)
+	if len(oid) != 64 || err != nil {
+		return nil
+	}
+
+	contentStore := &ContentStore{BasePath: setting.LFS.ContentPath}
+	meta := &models.LFSMetaObject{Oid: oid, Size: size}
+	if !contentStore.Exists(meta) {
+		return nil
+	}
+
+	return meta
+}
+
+// ReadMetaObject will read a models.LFSMetaObject and return a reader
+func ReadMetaObject(meta *models.LFSMetaObject) (io.ReadCloser, error) {
+	contentStore := &ContentStore{BasePath: setting.LFS.ContentPath}
+	return contentStore.Get(meta, 0)
+}

modules/markup/markdown/markdown.go

@@ -106,7 +106,7 @@ func (r *Renderer) Image(out *bytes.Buffer, link []byte, title []byte, alt []byt
 	if r.IsWiki {
 		prefix = util.URLJoin(prefix, "wiki", "raw")
 	}
-	prefix = strings.Replace(prefix, "/src/", "/raw/", 1)
+	prefix = strings.Replace(prefix, "/src/", "/media/", 1)
 	if len(link) > 0 && !markup.IsLink(link) {
 		lnk := string(link)
 		lnk = util.URLJoin(prefix, lnk)