mirror of
https://codeberg.org/forgejo/forgejo
synced 2024-08-20 06:31:35 +02:00
- In Go 1.21 the crypto/sha256 [got a massive improvement](https://go.dev/doc/go1.21#crypto/sha256) by utilizing the SHA instructions for AMD64 CPUs, which sha256-simd already was doing. The performance is now on par and I think it's preferable to use the standard library rather than a package when possible. ``` cpu: AMD Ryzen 5 3600X 6-Core Processor │ simd.txt │ go.txt │ │ sec/op │ sec/op vs base │ Hash/8Bytes-12 63.25n ± 1% 73.38n ± 1% +16.02% (p=0.002 n=6) Hash/64Bytes-12 98.73n ± 1% 105.30n ± 1% +6.65% (p=0.002 n=6) Hash/1K-12 567.2n ± 1% 572.8n ± 1% +0.99% (p=0.002 n=6) Hash/8K-12 4.062µ ± 1% 4.062µ ± 1% ~ (p=0.396 n=6) Hash/1M-12 512.1µ ± 0% 510.6µ ± 1% ~ (p=0.485 n=6) Hash/5M-12 2.556m ± 1% 2.564m ± 0% ~ (p=0.093 n=6) Hash/10M-12 5.112m ± 0% 5.127m ± 0% ~ (p=0.093 n=6) geomean 13.82µ 14.27µ +3.28% │ simd.txt │ go.txt │ │ B/s │ B/s vs base │ Hash/8Bytes-12 120.6Mi ± 1% 104.0Mi ± 1% -13.81% (p=0.002 n=6) Hash/64Bytes-12 618.2Mi ± 1% 579.8Mi ± 1% -6.22% (p=0.002 n=6) Hash/1K-12 1.682Gi ± 1% 1.665Gi ± 1% -0.98% (p=0.002 n=6) Hash/8K-12 1.878Gi ± 1% 1.878Gi ± 1% ~ (p=0.310 n=6) Hash/1M-12 1.907Gi ± 0% 1.913Gi ± 1% ~ (p=0.485 n=6) Hash/5M-12 1.911Gi ± 1% 1.904Gi ± 0% ~ (p=0.093 n=6) Hash/10M-12 1.910Gi ± 0% 1.905Gi ± 0% ~ (p=0.093 n=6) geomean 1.066Gi 1.032Gi -3.18% ``` (cherry picked from commitabd94ff5b5
) (cherry picked from commit15e81637ab
) Conflicts: go.mod https://codeberg.org/forgejo/forgejo/pulls/1581 (cherry picked from commit325d92917f
) Conflicts: modules/context/context_cookie.go https://codeberg.org/forgejo/forgejo/pulls/1617 (cherry picked from commit358819e895
) (cherry picked from commit362fd7aae1
) (cherry picked from commit4f64ee294e
) (cherry picked from commit4bde77f7b1
) (cherry picked from commit1311e30a81
) (cherry picked from commit57b69e334c
) (cherry picked from commit52dc892fad
) (cherry picked from commit77f54f4187
) (cherry picked from commit0d0392f3a5
) Conflicts: go.mod https://codeberg.org/forgejo/forgejo/pulls/2034 (cherry picked from commit92798364e8
) (cherry picked from commit43d2181277
) (cherry picked from commit45c88b86a3
) (cherry picked from commita1cd6f4e3a
) (cherry picked from commit01191dc2ad
) (cherry picked from commit151e07f37e
)
130 lines
3.2 KiB
Go
130 lines
3.2 KiB
Go
// Copyright 2021 The Gitea Authors. All rights reserved.
|
|
// SPDX-License-Identifier: MIT
|
|
|
|
package lfs
|
|
|
|
import (
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"path"
|
|
"regexp"
|
|
"strconv"
|
|
"strings"
|
|
)
|
|
|
|
const (
|
|
blobSizeCutoff = 1024
|
|
|
|
// MetaFileIdentifier is the string appearing at the first line of LFS pointer files.
|
|
// https://github.com/git-lfs/git-lfs/blob/master/docs/spec.md
|
|
MetaFileIdentifier = "version https://git-lfs.github.com/spec/v1"
|
|
|
|
// MetaFileOidPrefix appears in LFS pointer files on a line before the sha256 hash.
|
|
MetaFileOidPrefix = "oid sha256:"
|
|
)
|
|
|
|
var (
|
|
// ErrMissingPrefix occurs if the content lacks the LFS prefix
|
|
ErrMissingPrefix = errors.New("content lacks the LFS prefix")
|
|
|
|
// ErrInvalidStructure occurs if the content has an invalid structure
|
|
ErrInvalidStructure = errors.New("content has an invalid structure")
|
|
|
|
// ErrInvalidOIDFormat occurs if the oid has an invalid format
|
|
ErrInvalidOIDFormat = errors.New("OID has an invalid format")
|
|
)
|
|
|
|
// ReadPointer tries to read LFS pointer data from the reader
|
|
func ReadPointer(reader io.Reader) (Pointer, error) {
|
|
buf := make([]byte, blobSizeCutoff)
|
|
n, err := io.ReadFull(reader, buf)
|
|
if err != nil && err != io.ErrUnexpectedEOF {
|
|
return Pointer{}, err
|
|
}
|
|
buf = buf[:n]
|
|
|
|
return ReadPointerFromBuffer(buf)
|
|
}
|
|
|
|
var oidPattern = regexp.MustCompile(`^[a-f\d]{64}$`)
|
|
|
|
// ReadPointerFromBuffer will return a pointer if the provided byte slice is a pointer file or an error otherwise.
|
|
func ReadPointerFromBuffer(buf []byte) (Pointer, error) {
|
|
var p Pointer
|
|
|
|
headString := string(buf)
|
|
if !strings.HasPrefix(headString, MetaFileIdentifier) {
|
|
return p, ErrMissingPrefix
|
|
}
|
|
|
|
splitLines := strings.Split(headString, "\n")
|
|
if len(splitLines) < 3 {
|
|
return p, ErrInvalidStructure
|
|
}
|
|
|
|
oid := strings.TrimPrefix(splitLines[1], MetaFileOidPrefix)
|
|
if len(oid) != 64 || !oidPattern.MatchString(oid) {
|
|
return p, ErrInvalidOIDFormat
|
|
}
|
|
size, err := strconv.ParseInt(strings.TrimPrefix(splitLines[2], "size "), 10, 64)
|
|
if err != nil {
|
|
return p, err
|
|
}
|
|
|
|
p.Oid = oid
|
|
p.Size = size
|
|
|
|
return p, nil
|
|
}
|
|
|
|
// IsValid checks if the pointer has a valid structure.
|
|
// It doesn't check if the pointed-to-content exists.
|
|
func (p Pointer) IsValid() bool {
|
|
if len(p.Oid) != 64 {
|
|
return false
|
|
}
|
|
if !oidPattern.MatchString(p.Oid) {
|
|
return false
|
|
}
|
|
if p.Size < 0 {
|
|
return false
|
|
}
|
|
return true
|
|
}
|
|
|
|
// StringContent returns the string representation of the pointer
|
|
// https://github.com/git-lfs/git-lfs/blob/main/docs/spec.md#the-pointer
|
|
func (p Pointer) StringContent() string {
|
|
return fmt.Sprintf("%s\n%s%s\nsize %d\n", MetaFileIdentifier, MetaFileOidPrefix, p.Oid, p.Size)
|
|
}
|
|
|
|
// RelativePath returns the relative storage path of the pointer
|
|
func (p Pointer) RelativePath() string {
|
|
if len(p.Oid) < 5 {
|
|
return p.Oid
|
|
}
|
|
|
|
return path.Join(p.Oid[0:2], p.Oid[2:4], p.Oid[4:])
|
|
}
|
|
|
|
func (p Pointer) LogString() string {
|
|
if p.Oid == "" && p.Size == 0 {
|
|
return "<LFSPointer empty>"
|
|
}
|
|
return fmt.Sprintf("<LFSPointer %s:%d>", p.Oid, p.Size)
|
|
}
|
|
|
|
// GeneratePointer generates a pointer for arbitrary content
|
|
func GeneratePointer(content io.Reader) (Pointer, error) {
|
|
h := sha256.New()
|
|
c, err := io.Copy(h, content)
|
|
if err != nil {
|
|
return Pointer{}, err
|
|
}
|
|
sum := h.Sum(nil)
|
|
return Pointer{Oid: hex.EncodeToString(sum), Size: c}, nil
|
|
}
|