mirror of
https://codeberg.org/forgejo/forgejo
synced 2024-08-22 07:31:52 +02:00
- Remove options that currently aren't set
on `GarbageCollectLFSMetaObjectsOptions` and
`IterateLFSMetaObjectsForRepoOptions`.
- Simplify `IterateRepositoryIDsWithLFSMetaObjects` and
`IterateLFSMetaObjectsForRepo`.
- `IterateLFSMetaObjectsForRepo` was previously able to get in a
loop (`gc-lfs` doctor check was able to reproduce this) because the code
expected that the records would be updated to not match the SQL query,
but that wasn't the case. Simply enforce that only records higher than
the latest `id` from the previous iteration are allowed.
- For `gc-lfs` doctor check this was because `UpdatedLessRecentlyThan`
option was not set, which caused that records just marked as active in
the iteration weren't being filtered.
- Add unit tests
- Most likely a regression from 2cc3a6381c
.
- The bug with `gc-lfs` was found on Codeberg.
124 lines
4.2 KiB
Go
124 lines
4.2 KiB
Go
// Copyright 2022 The Gitea Authors. All rights reserved.
|
|
// SPDX-License-Identifier: MIT
|
|
|
|
package repository
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"time"
|
|
|
|
git_model "code.gitea.io/gitea/models/git"
|
|
repo_model "code.gitea.io/gitea/models/repo"
|
|
"code.gitea.io/gitea/modules/git"
|
|
"code.gitea.io/gitea/modules/gitrepo"
|
|
"code.gitea.io/gitea/modules/lfs"
|
|
"code.gitea.io/gitea/modules/log"
|
|
"code.gitea.io/gitea/modules/setting"
|
|
"code.gitea.io/gitea/modules/timeutil"
|
|
)
|
|
|
|
// GarbageCollectLFSMetaObjectsOptions provides options for GarbageCollectLFSMetaObjects function
|
|
type GarbageCollectLFSMetaObjectsOptions struct {
|
|
LogDetail func(format string, v ...any)
|
|
AutoFix bool
|
|
OlderThan time.Time
|
|
UpdatedLessRecentlyThan time.Time
|
|
}
|
|
|
|
// GarbageCollectLFSMetaObjects garbage collects LFS objects for all repositories
|
|
func GarbageCollectLFSMetaObjects(ctx context.Context, opts GarbageCollectLFSMetaObjectsOptions) error {
|
|
log.Trace("Doing: GarbageCollectLFSMetaObjects")
|
|
defer log.Trace("Finished: GarbageCollectLFSMetaObjects")
|
|
|
|
if opts.LogDetail == nil {
|
|
opts.LogDetail = log.Debug
|
|
}
|
|
|
|
if !setting.LFS.StartServer {
|
|
opts.LogDetail("LFS support is disabled")
|
|
return nil
|
|
}
|
|
|
|
return git_model.IterateRepositoryIDsWithLFSMetaObjects(ctx, func(ctx context.Context, repoID, count int64) error {
|
|
repo, err := repo_model.GetRepositoryByID(ctx, repoID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
return GarbageCollectLFSMetaObjectsForRepo(ctx, repo, opts)
|
|
})
|
|
}
|
|
|
|
// GarbageCollectLFSMetaObjectsForRepo garbage collects LFS objects for a specific repository
|
|
func GarbageCollectLFSMetaObjectsForRepo(ctx context.Context, repo *repo_model.Repository, opts GarbageCollectLFSMetaObjectsOptions) error {
|
|
opts.LogDetail("Checking %-v", repo)
|
|
total, orphaned, collected, deleted := int64(0), 0, 0, 0
|
|
defer func() {
|
|
if orphaned == 0 {
|
|
opts.LogDetail("Found %d total LFSMetaObjects in %-v", total, repo)
|
|
} else if !opts.AutoFix {
|
|
opts.LogDetail("Found %d/%d orphaned LFSMetaObjects in %-v", orphaned, total, repo)
|
|
} else {
|
|
opts.LogDetail("Collected %d/%d orphaned/%d total LFSMetaObjects in %-v. %d removed from storage.", collected, orphaned, total, repo, deleted)
|
|
}
|
|
}()
|
|
|
|
gitRepo, err := gitrepo.OpenRepository(ctx, repo)
|
|
if err != nil {
|
|
log.Error("Unable to open git repository %-v: %v", repo, err)
|
|
return err
|
|
}
|
|
defer gitRepo.Close()
|
|
|
|
store := lfs.NewContentStore()
|
|
objectFormat := git.ObjectFormatFromName(repo.ObjectFormatName)
|
|
|
|
err = git_model.IterateLFSMetaObjectsForRepo(ctx, repo.ID, func(ctx context.Context, metaObject *git_model.LFSMetaObject) error {
|
|
total++
|
|
pointerSha := git.ComputeBlobHash(objectFormat, []byte(metaObject.Pointer.StringContent()))
|
|
|
|
if gitRepo.IsObjectExist(pointerSha.String()) {
|
|
return git_model.MarkLFSMetaObject(ctx, metaObject.ID)
|
|
}
|
|
orphaned++
|
|
|
|
if !opts.AutoFix {
|
|
return nil
|
|
}
|
|
// Non-existent pointer file
|
|
_, err = git_model.RemoveLFSMetaObjectByOidFn(ctx, repo.ID, metaObject.Oid, func(count int64) error {
|
|
if count > 0 {
|
|
return nil
|
|
}
|
|
|
|
if err := store.Delete(metaObject.RelativePath()); err != nil {
|
|
log.Error("Unable to remove lfs metaobject %s from store: %v", metaObject.Oid, err)
|
|
}
|
|
deleted++
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
return fmt.Errorf("unable to remove meta-object %s in %s: %w", metaObject.Oid, repo.FullName(), err)
|
|
}
|
|
collected++
|
|
|
|
return nil
|
|
}, &git_model.IterateLFSMetaObjectsForRepoOptions{
|
|
// Only attempt to garbage collect lfs meta objects older than a week as the order of git lfs upload
|
|
// and git object upload is not necessarily guaranteed. It's possible to imagine a situation whereby
|
|
// an LFS object is uploaded but the git branch is not uploaded immediately, or there are some rapid
|
|
// changes in new branches that might lead to lfs objects becoming temporarily unassociated with git
|
|
// objects.
|
|
//
|
|
// It is likely that a week is potentially excessive but it should definitely be enough that any
|
|
// unassociated LFS object is genuinely unassociated.
|
|
OlderThan: timeutil.TimeStamp(opts.OlderThan.Unix()),
|
|
UpdatedLessRecentlyThan: timeutil.TimeStamp(opts.UpdatedLessRecentlyThan.Unix()),
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return nil
|
|
}
|