mirror of
https://codeberg.org/forgejo/forgejo
synced 2024-08-21 23:21:11 +02:00
This PR adds basic repository LFS management UI including the ability to find all possible pointers within the repository. Locks are not managed at present but would be addable through some simple additions. * Add basic repository lfs management * add auto-associate function * Add functionality to find commits with this lfs file * Add link to find commits on the lfs file view * Adjust commit view to state the likely branch causing the commit * Only read Oid from database
604 lines
15 KiB
Go
604 lines
15 KiB
Go
package lfs
|
|
|
|
import (
|
|
"encoding/base64"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"path"
|
|
"regexp"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"code.gitea.io/gitea/models"
|
|
"code.gitea.io/gitea/modules/context"
|
|
"code.gitea.io/gitea/modules/log"
|
|
"code.gitea.io/gitea/modules/setting"
|
|
|
|
"gitea.com/macaron/macaron"
|
|
"github.com/dgrijalva/jwt-go"
|
|
)
|
|
|
|
const (
|
|
metaMediaType = "application/vnd.git-lfs+json"
|
|
)
|
|
|
|
// RequestVars contain variables from the HTTP request. Variables from routing, json body decoding, and
|
|
// some headers are stored.
|
|
type RequestVars struct {
|
|
Oid string
|
|
Size int64
|
|
User string
|
|
Password string
|
|
Repo string
|
|
Authorization string
|
|
}
|
|
|
|
// BatchVars contains multiple RequestVars processed in one batch operation.
|
|
// https://github.com/git-lfs/git-lfs/blob/master/docs/api/batch.md
|
|
type BatchVars struct {
|
|
Transfers []string `json:"transfers,omitempty"`
|
|
Operation string `json:"operation"`
|
|
Objects []*RequestVars `json:"objects"`
|
|
}
|
|
|
|
// BatchResponse contains multiple object metadata Representation structures
|
|
// for use with the batch API.
|
|
type BatchResponse struct {
|
|
Transfer string `json:"transfer,omitempty"`
|
|
Objects []*Representation `json:"objects"`
|
|
}
|
|
|
|
// Representation is object metadata as seen by clients of the lfs server.
|
|
type Representation struct {
|
|
Oid string `json:"oid"`
|
|
Size int64 `json:"size"`
|
|
Actions map[string]*link `json:"actions"`
|
|
Error *ObjectError `json:"error,omitempty"`
|
|
}
|
|
|
|
// ObjectError defines the JSON structure returned to the client in case of an error
|
|
type ObjectError struct {
|
|
Code int `json:"code"`
|
|
Message string `json:"message"`
|
|
}
|
|
|
|
// ObjectLink builds a URL linking to the object.
|
|
func (v *RequestVars) ObjectLink() string {
|
|
return setting.AppURL + path.Join(v.User, v.Repo+".git", "info/lfs/objects", v.Oid)
|
|
}
|
|
|
|
// VerifyLink builds a URL for verifying the object.
|
|
func (v *RequestVars) VerifyLink() string {
|
|
return setting.AppURL + path.Join(v.User, v.Repo+".git", "info/lfs/verify")
|
|
}
|
|
|
|
// link provides a structure used to build a hypermedia representation of an HTTP link.
|
|
type link struct {
|
|
Href string `json:"href"`
|
|
Header map[string]string `json:"header,omitempty"`
|
|
ExpiresAt time.Time `json:"expires_at,omitempty"`
|
|
}
|
|
|
|
var oidRegExp = regexp.MustCompile(`^[A-Fa-f0-9]+$`)
|
|
|
|
func isOidValid(oid string) bool {
|
|
return oidRegExp.MatchString(oid)
|
|
}
|
|
|
|
// ObjectOidHandler is the main request routing entry point into LFS server functions
|
|
func ObjectOidHandler(ctx *context.Context) {
|
|
if !setting.LFS.StartServer {
|
|
writeStatus(ctx, 404)
|
|
return
|
|
}
|
|
|
|
if ctx.Req.Method == "GET" || ctx.Req.Method == "HEAD" {
|
|
if MetaMatcher(ctx.Req) {
|
|
getMetaHandler(ctx)
|
|
return
|
|
}
|
|
|
|
getContentHandler(ctx)
|
|
return
|
|
} else if ctx.Req.Method == "PUT" {
|
|
PutHandler(ctx)
|
|
return
|
|
}
|
|
|
|
}
|
|
|
|
func getAuthenticatedRepoAndMeta(ctx *context.Context, rv *RequestVars, requireWrite bool) (*models.LFSMetaObject, *models.Repository) {
|
|
if !isOidValid(rv.Oid) {
|
|
writeStatus(ctx, 404)
|
|
return nil, nil
|
|
}
|
|
|
|
repository, err := models.GetRepositoryByOwnerAndName(rv.User, rv.Repo)
|
|
if err != nil {
|
|
log.Debug("Could not find repository: %s/%s - %s", rv.User, rv.Repo, err)
|
|
writeStatus(ctx, 404)
|
|
return nil, nil
|
|
}
|
|
|
|
if !authenticate(ctx, repository, rv.Authorization, requireWrite) {
|
|
requireAuth(ctx)
|
|
return nil, nil
|
|
}
|
|
|
|
meta, err := repository.GetLFSMetaObjectByOid(rv.Oid)
|
|
if err != nil {
|
|
writeStatus(ctx, 404)
|
|
return nil, nil
|
|
}
|
|
|
|
return meta, repository
|
|
}
|
|
|
|
// getContentHandler gets the content from the content store
|
|
func getContentHandler(ctx *context.Context) {
|
|
rv := unpack(ctx)
|
|
|
|
meta, _ := getAuthenticatedRepoAndMeta(ctx, rv, false)
|
|
if meta == nil {
|
|
return
|
|
}
|
|
|
|
// Support resume download using Range header
|
|
var fromByte int64
|
|
statusCode := 200
|
|
if rangeHdr := ctx.Req.Header.Get("Range"); rangeHdr != "" {
|
|
regex := regexp.MustCompile(`bytes=(\d+)\-.*`)
|
|
match := regex.FindStringSubmatch(rangeHdr)
|
|
if len(match) > 1 {
|
|
statusCode = 206
|
|
fromByte, _ = strconv.ParseInt(match[1], 10, 32)
|
|
ctx.Resp.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", fromByte, meta.Size-1, meta.Size-fromByte))
|
|
}
|
|
}
|
|
|
|
contentStore := &ContentStore{BasePath: setting.LFS.ContentPath}
|
|
content, err := contentStore.Get(meta, fromByte)
|
|
if err != nil {
|
|
writeStatus(ctx, 404)
|
|
return
|
|
}
|
|
|
|
ctx.Resp.Header().Set("Content-Length", strconv.FormatInt(meta.Size-fromByte, 10))
|
|
ctx.Resp.Header().Set("Content-Type", "application/octet-stream")
|
|
|
|
filename := ctx.Params("filename")
|
|
if len(filename) > 0 {
|
|
decodedFilename, err := base64.RawURLEncoding.DecodeString(filename)
|
|
if err == nil {
|
|
ctx.Resp.Header().Set("Content-Disposition", "attachment; filename=\""+string(decodedFilename)+"\"")
|
|
}
|
|
}
|
|
|
|
ctx.Resp.WriteHeader(statusCode)
|
|
_, _ = io.Copy(ctx.Resp, content)
|
|
_ = content.Close()
|
|
logRequest(ctx.Req, statusCode)
|
|
}
|
|
|
|
// getMetaHandler retrieves metadata about the object
|
|
func getMetaHandler(ctx *context.Context) {
|
|
rv := unpack(ctx)
|
|
|
|
meta, _ := getAuthenticatedRepoAndMeta(ctx, rv, false)
|
|
if meta == nil {
|
|
return
|
|
}
|
|
|
|
ctx.Resp.Header().Set("Content-Type", metaMediaType)
|
|
|
|
if ctx.Req.Method == "GET" {
|
|
enc := json.NewEncoder(ctx.Resp)
|
|
_ = enc.Encode(Represent(rv, meta, true, false))
|
|
}
|
|
|
|
logRequest(ctx.Req, 200)
|
|
}
|
|
|
|
// PostHandler instructs the client how to upload data
|
|
func PostHandler(ctx *context.Context) {
|
|
if !setting.LFS.StartServer {
|
|
writeStatus(ctx, 404)
|
|
return
|
|
}
|
|
|
|
if !MetaMatcher(ctx.Req) {
|
|
writeStatus(ctx, 400)
|
|
return
|
|
}
|
|
|
|
rv := unpack(ctx)
|
|
|
|
repository, err := models.GetRepositoryByOwnerAndName(rv.User, rv.Repo)
|
|
if err != nil {
|
|
log.Debug("Could not find repository: %s/%s - %s", rv.User, rv.Repo, err)
|
|
writeStatus(ctx, 404)
|
|
return
|
|
}
|
|
|
|
if !authenticate(ctx, repository, rv.Authorization, true) {
|
|
requireAuth(ctx)
|
|
return
|
|
}
|
|
|
|
if !isOidValid(rv.Oid) {
|
|
writeStatus(ctx, 404)
|
|
return
|
|
}
|
|
|
|
meta, err := models.NewLFSMetaObject(&models.LFSMetaObject{Oid: rv.Oid, Size: rv.Size, RepositoryID: repository.ID})
|
|
if err != nil {
|
|
writeStatus(ctx, 404)
|
|
return
|
|
}
|
|
|
|
ctx.Resp.Header().Set("Content-Type", metaMediaType)
|
|
|
|
sentStatus := 202
|
|
contentStore := &ContentStore{BasePath: setting.LFS.ContentPath}
|
|
if meta.Existing && contentStore.Exists(meta) {
|
|
sentStatus = 200
|
|
}
|
|
ctx.Resp.WriteHeader(sentStatus)
|
|
|
|
enc := json.NewEncoder(ctx.Resp)
|
|
_ = enc.Encode(Represent(rv, meta, meta.Existing, true))
|
|
logRequest(ctx.Req, sentStatus)
|
|
}
|
|
|
|
// BatchHandler provides the batch api
|
|
func BatchHandler(ctx *context.Context) {
|
|
if !setting.LFS.StartServer {
|
|
writeStatus(ctx, 404)
|
|
return
|
|
}
|
|
|
|
if !MetaMatcher(ctx.Req) {
|
|
writeStatus(ctx, 400)
|
|
return
|
|
}
|
|
|
|
bv := unpackbatch(ctx)
|
|
|
|
var responseObjects []*Representation
|
|
|
|
// Create a response object
|
|
for _, object := range bv.Objects {
|
|
if !isOidValid(object.Oid) {
|
|
continue
|
|
}
|
|
|
|
repository, err := models.GetRepositoryByOwnerAndName(object.User, object.Repo)
|
|
|
|
if err != nil {
|
|
log.Debug("Could not find repository: %s/%s - %s", object.User, object.Repo, err)
|
|
writeStatus(ctx, 404)
|
|
return
|
|
}
|
|
|
|
requireWrite := false
|
|
if bv.Operation == "upload" {
|
|
requireWrite = true
|
|
}
|
|
|
|
if !authenticate(ctx, repository, object.Authorization, requireWrite) {
|
|
requireAuth(ctx)
|
|
return
|
|
}
|
|
|
|
contentStore := &ContentStore{BasePath: setting.LFS.ContentPath}
|
|
|
|
meta, err := repository.GetLFSMetaObjectByOid(object.Oid)
|
|
if err == nil && contentStore.Exists(meta) { // Object is found and exists
|
|
responseObjects = append(responseObjects, Represent(object, meta, true, false))
|
|
continue
|
|
}
|
|
|
|
// Object is not found
|
|
meta, err = models.NewLFSMetaObject(&models.LFSMetaObject{Oid: object.Oid, Size: object.Size, RepositoryID: repository.ID})
|
|
if err == nil {
|
|
responseObjects = append(responseObjects, Represent(object, meta, meta.Existing, !contentStore.Exists(meta)))
|
|
}
|
|
}
|
|
|
|
ctx.Resp.Header().Set("Content-Type", metaMediaType)
|
|
|
|
respobj := &BatchResponse{Objects: responseObjects}
|
|
|
|
enc := json.NewEncoder(ctx.Resp)
|
|
_ = enc.Encode(respobj)
|
|
logRequest(ctx.Req, 200)
|
|
}
|
|
|
|
// PutHandler receives data from the client and puts it into the content store
|
|
func PutHandler(ctx *context.Context) {
|
|
rv := unpack(ctx)
|
|
|
|
meta, repository := getAuthenticatedRepoAndMeta(ctx, rv, true)
|
|
if meta == nil {
|
|
return
|
|
}
|
|
|
|
contentStore := &ContentStore{BasePath: setting.LFS.ContentPath}
|
|
bodyReader := ctx.Req.Body().ReadCloser()
|
|
defer bodyReader.Close()
|
|
if err := contentStore.Put(meta, bodyReader); err != nil {
|
|
ctx.Resp.WriteHeader(500)
|
|
fmt.Fprintf(ctx.Resp, `{"message":"%s"}`, err)
|
|
if _, err = repository.RemoveLFSMetaObjectByOid(rv.Oid); err != nil {
|
|
log.Error("RemoveLFSMetaObjectByOid: %v", err)
|
|
}
|
|
return
|
|
}
|
|
|
|
logRequest(ctx.Req, 200)
|
|
}
|
|
|
|
// VerifyHandler verify oid and its size from the content store
|
|
func VerifyHandler(ctx *context.Context) {
|
|
if !setting.LFS.StartServer {
|
|
writeStatus(ctx, 404)
|
|
return
|
|
}
|
|
|
|
if !MetaMatcher(ctx.Req) {
|
|
writeStatus(ctx, 400)
|
|
return
|
|
}
|
|
|
|
rv := unpack(ctx)
|
|
|
|
meta, _ := getAuthenticatedRepoAndMeta(ctx, rv, true)
|
|
if meta == nil {
|
|
return
|
|
}
|
|
|
|
contentStore := &ContentStore{BasePath: setting.LFS.ContentPath}
|
|
ok, err := contentStore.Verify(meta)
|
|
if err != nil {
|
|
ctx.Resp.WriteHeader(500)
|
|
fmt.Fprintf(ctx.Resp, `{"message":"%s"}`, err)
|
|
return
|
|
}
|
|
if !ok {
|
|
writeStatus(ctx, 422)
|
|
return
|
|
}
|
|
|
|
logRequest(ctx.Req, 200)
|
|
}
|
|
|
|
// Represent takes a RequestVars and Meta and turns it into a Representation suitable
|
|
// for json encoding
|
|
func Represent(rv *RequestVars, meta *models.LFSMetaObject, download, upload bool) *Representation {
|
|
rep := &Representation{
|
|
Oid: meta.Oid,
|
|
Size: meta.Size,
|
|
Actions: make(map[string]*link),
|
|
}
|
|
|
|
header := make(map[string]string)
|
|
|
|
if rv.Authorization == "" {
|
|
//https://github.com/github/git-lfs/issues/1088
|
|
header["Authorization"] = "Authorization: Basic dummy"
|
|
} else {
|
|
header["Authorization"] = rv.Authorization
|
|
}
|
|
|
|
if download {
|
|
rep.Actions["download"] = &link{Href: rv.ObjectLink(), Header: header}
|
|
}
|
|
|
|
if upload {
|
|
rep.Actions["upload"] = &link{Href: rv.ObjectLink(), Header: header}
|
|
}
|
|
|
|
if upload && !download {
|
|
// Force client side verify action while gitea lacks proper server side verification
|
|
verifyHeader := make(map[string]string)
|
|
for k, v := range header {
|
|
verifyHeader[k] = v
|
|
}
|
|
|
|
// This is only needed to workaround https://github.com/git-lfs/git-lfs/issues/3662
|
|
verifyHeader["Accept"] = metaMediaType
|
|
|
|
rep.Actions["verify"] = &link{Href: rv.VerifyLink(), Header: verifyHeader}
|
|
}
|
|
|
|
return rep
|
|
}
|
|
|
|
// MetaMatcher provides a mux.MatcherFunc that only allows requests that contain
|
|
// an Accept header with the metaMediaType
|
|
func MetaMatcher(r macaron.Request) bool {
|
|
mediaParts := strings.Split(r.Header.Get("Accept"), ";")
|
|
mt := mediaParts[0]
|
|
return mt == metaMediaType
|
|
}
|
|
|
|
func unpack(ctx *context.Context) *RequestVars {
|
|
r := ctx.Req
|
|
rv := &RequestVars{
|
|
User: ctx.Params("username"),
|
|
Repo: strings.TrimSuffix(ctx.Params("reponame"), ".git"),
|
|
Oid: ctx.Params("oid"),
|
|
Authorization: r.Header.Get("Authorization"),
|
|
}
|
|
|
|
if r.Method == "POST" { // Maybe also check if +json
|
|
var p RequestVars
|
|
bodyReader := r.Body().ReadCloser()
|
|
defer bodyReader.Close()
|
|
dec := json.NewDecoder(bodyReader)
|
|
err := dec.Decode(&p)
|
|
if err != nil {
|
|
return rv
|
|
}
|
|
|
|
rv.Oid = p.Oid
|
|
rv.Size = p.Size
|
|
}
|
|
|
|
return rv
|
|
}
|
|
|
|
// TODO cheap hack, unify with unpack
|
|
func unpackbatch(ctx *context.Context) *BatchVars {
|
|
|
|
r := ctx.Req
|
|
var bv BatchVars
|
|
|
|
bodyReader := r.Body().ReadCloser()
|
|
defer bodyReader.Close()
|
|
dec := json.NewDecoder(bodyReader)
|
|
err := dec.Decode(&bv)
|
|
if err != nil {
|
|
return &bv
|
|
}
|
|
|
|
for i := 0; i < len(bv.Objects); i++ {
|
|
bv.Objects[i].User = ctx.Params("username")
|
|
bv.Objects[i].Repo = strings.TrimSuffix(ctx.Params("reponame"), ".git")
|
|
bv.Objects[i].Authorization = r.Header.Get("Authorization")
|
|
}
|
|
|
|
return &bv
|
|
}
|
|
|
|
func writeStatus(ctx *context.Context, status int) {
|
|
message := http.StatusText(status)
|
|
|
|
mediaParts := strings.Split(ctx.Req.Header.Get("Accept"), ";")
|
|
mt := mediaParts[0]
|
|
if strings.HasSuffix(mt, "+json") {
|
|
message = `{"message":"` + message + `"}`
|
|
}
|
|
|
|
ctx.Resp.WriteHeader(status)
|
|
fmt.Fprint(ctx.Resp, message)
|
|
logRequest(ctx.Req, status)
|
|
}
|
|
|
|
func logRequest(r macaron.Request, status int) {
|
|
log.Debug("LFS request - Method: %s, URL: %s, Status %d", r.Method, r.URL, status)
|
|
}
|
|
|
|
// authenticate uses the authorization string to determine whether
|
|
// or not to proceed. This server assumes an HTTP Basic auth format.
|
|
func authenticate(ctx *context.Context, repository *models.Repository, authorization string, requireWrite bool) bool {
|
|
accessMode := models.AccessModeRead
|
|
if requireWrite {
|
|
accessMode = models.AccessModeWrite
|
|
}
|
|
|
|
// ctx.IsSigned is unnecessary here, this will be checked in perm.CanAccess
|
|
perm, err := models.GetUserRepoPermission(repository, ctx.User)
|
|
if err != nil {
|
|
return false
|
|
}
|
|
|
|
canRead := perm.CanAccess(accessMode, models.UnitTypeCode)
|
|
if canRead {
|
|
return true
|
|
}
|
|
|
|
user, repo, opStr, err := parseToken(authorization)
|
|
if err != nil {
|
|
return false
|
|
}
|
|
ctx.User = user
|
|
if opStr == "basic" {
|
|
perm, err = models.GetUserRepoPermission(repository, ctx.User)
|
|
if err != nil {
|
|
return false
|
|
}
|
|
return perm.CanAccess(accessMode, models.UnitTypeCode)
|
|
}
|
|
if repository.ID == repo.ID {
|
|
if requireWrite && opStr != "upload" {
|
|
return false
|
|
}
|
|
return true
|
|
}
|
|
return false
|
|
}
|
|
|
|
func parseToken(authorization string) (*models.User, *models.Repository, string, error) {
|
|
if authorization == "" {
|
|
return nil, nil, "unknown", fmt.Errorf("No token")
|
|
}
|
|
if strings.HasPrefix(authorization, "Bearer ") {
|
|
token, err := jwt.Parse(authorization[7:], func(t *jwt.Token) (interface{}, error) {
|
|
if _, ok := t.Method.(*jwt.SigningMethodHMAC); !ok {
|
|
return nil, fmt.Errorf("unexpected signing method: %v", t.Header["alg"])
|
|
}
|
|
return setting.LFS.JWTSecretBytes, nil
|
|
})
|
|
if err != nil {
|
|
return nil, nil, "unknown", err
|
|
}
|
|
claims, claimsOk := token.Claims.(jwt.MapClaims)
|
|
if !token.Valid || !claimsOk {
|
|
return nil, nil, "unknown", fmt.Errorf("Token claim invalid")
|
|
}
|
|
opStr, ok := claims["op"].(string)
|
|
if !ok {
|
|
return nil, nil, "unknown", fmt.Errorf("Token operation invalid")
|
|
}
|
|
repoID, ok := claims["repo"].(float64)
|
|
if !ok {
|
|
return nil, nil, opStr, fmt.Errorf("Token repository id invalid")
|
|
}
|
|
r, err := models.GetRepositoryByID(int64(repoID))
|
|
if err != nil {
|
|
return nil, nil, opStr, err
|
|
}
|
|
userID, ok := claims["user"].(float64)
|
|
if !ok {
|
|
return nil, r, opStr, fmt.Errorf("Token user id invalid")
|
|
}
|
|
u, err := models.GetUserByID(int64(userID))
|
|
if err != nil {
|
|
return nil, r, opStr, err
|
|
}
|
|
return u, r, opStr, nil
|
|
}
|
|
|
|
if strings.HasPrefix(authorization, "Basic ") {
|
|
c, err := base64.StdEncoding.DecodeString(strings.TrimPrefix(authorization, "Basic "))
|
|
if err != nil {
|
|
return nil, nil, "basic", err
|
|
}
|
|
cs := string(c)
|
|
i := strings.IndexByte(cs, ':')
|
|
if i < 0 {
|
|
return nil, nil, "basic", fmt.Errorf("Basic auth invalid")
|
|
}
|
|
user, password := cs[:i], cs[i+1:]
|
|
u, err := models.GetUserByName(user)
|
|
if err != nil {
|
|
return nil, nil, "basic", err
|
|
}
|
|
if !u.IsPasswordSet() || !u.ValidatePassword(password) {
|
|
return nil, nil, "basic", fmt.Errorf("Basic auth failed")
|
|
}
|
|
return u, nil, "basic", nil
|
|
}
|
|
|
|
return nil, nil, "unknown", fmt.Errorf("Token not found")
|
|
}
|
|
|
|
func requireAuth(ctx *context.Context) {
|
|
ctx.Resp.Header().Set("WWW-Authenticate", "Basic realm=gitea-lfs")
|
|
writeStatus(ctx, 401)
|
|
}
|