mirror of
https://codeberg.org/forgejo/forgejo
synced 2024-08-27 18:11:09 +02:00
[FEAT] add Forgejo Git Service (squash) register a Forgejo factory If the Forgejo factory for the Forgejo service is not registered, newDownloader will fallback to a git service and not migrate issues etc. Refs: https://codeberg.org/forgejo/forgejo/issues/1678 (cherry picked from commit51938cd161
) [FEAT] add Forgero Git Service Signed-off-by: cassiozareck <cassiomilczareck@gmail.com> (cherry picked from commita878adfe62
) Adding description and Forgejo SVG (cherry picked from commit13738c0380
) Undo reordering and tmpl redirection (cherry picked from commit9ae51c46f4
) (cherry picked from commit70fffdc61d
) (cherry picked from commitc0ebfa9da3
) (cherry picked from commit9922c92787
) (cherry picked from commit00c0effbc7
) (cherry picked from commite4c9525b13
) (cherry picked from commit09d7b83211
) (cherry picked from commitbbcd5975c9
) (cherry picked from commit55c70a0e18
) (cherry picked from commit76596410c0
) (cherry picked from commit1308043931
) (cherry picked from commit919d6aedfe
) [FEAT] add Forgero Git Service (squash) more tests Previously only Gitea service was being tested under self-hosted migrations. Since Forgejo is also self-hosted and in fact use the same downloader/migrator we can add to this suite another test that will do the same, migrating the same repository under the same local instance but for the Forgejo service (represented by 9) Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/1709 Co-authored-by: zareck <cassiomilczareck@gmail.com> Co-committed-by: zareck <cassiomilczareck@gmail.com> (cherry picked from commit40a4b8f1a8
) (cherry picked from commit3198b4a642
) (cherry picked from commit4edda1f389
) (cherry picked from commit4d91b77d29
) (cherry picked from commitafe85c52e3
) (cherry picked from commit5ea7df79ad
) (cherry picked from commita667182542
) (cherry picked from commita9bebb1e71
) (cherry picked from commit4831a89e46
) (cherry picked from commite02a74651f
) (cherry picked from commit05dcef59aa
) (cherry picked from commitc8bac187f9
) (cherry picked from commitc87903a0cc
)
516 lines
15 KiB
Go
516 lines
15 KiB
Go
// Copyright 2019 The Gitea Authors. All rights reserved.
|
|
// Copyright 2018 Jonas Franz. All rights reserved.
|
|
// SPDX-License-Identifier: MIT
|
|
|
|
package migrations
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"net"
|
|
"net/url"
|
|
"path/filepath"
|
|
"strings"
|
|
|
|
"code.gitea.io/gitea/models"
|
|
repo_model "code.gitea.io/gitea/models/repo"
|
|
system_model "code.gitea.io/gitea/models/system"
|
|
user_model "code.gitea.io/gitea/models/user"
|
|
"code.gitea.io/gitea/modules/hostmatcher"
|
|
"code.gitea.io/gitea/modules/log"
|
|
base "code.gitea.io/gitea/modules/migration"
|
|
"code.gitea.io/gitea/modules/setting"
|
|
"code.gitea.io/gitea/modules/structs"
|
|
"code.gitea.io/gitea/modules/util"
|
|
)
|
|
|
|
// MigrateOptions is equal to base.MigrateOptions
|
|
type MigrateOptions = base.MigrateOptions
|
|
|
|
var (
|
|
factories []base.DownloaderFactory
|
|
|
|
allowList *hostmatcher.HostMatchList
|
|
blockList *hostmatcher.HostMatchList
|
|
)
|
|
|
|
// RegisterDownloaderFactory registers a downloader factory
|
|
func RegisterDownloaderFactory(factory base.DownloaderFactory) {
|
|
factories = append(factories, factory)
|
|
}
|
|
|
|
// IsMigrateURLAllowed checks if an URL is allowed to be migrated from
|
|
func IsMigrateURLAllowed(remoteURL string, doer *user_model.User) error {
|
|
// Remote address can be HTTP/HTTPS/Git URL or local path.
|
|
u, err := url.Parse(remoteURL)
|
|
if err != nil {
|
|
return &models.ErrInvalidCloneAddr{IsURLError: true, Host: remoteURL}
|
|
}
|
|
|
|
if u.Scheme == "file" || u.Scheme == "" {
|
|
if !doer.CanImportLocal() {
|
|
return &models.ErrInvalidCloneAddr{Host: "<LOCAL_FILESYSTEM>", IsPermissionDenied: true, LocalPath: true}
|
|
}
|
|
isAbs := filepath.IsAbs(u.Host + u.Path)
|
|
if !isAbs {
|
|
return &models.ErrInvalidCloneAddr{Host: "<LOCAL_FILESYSTEM>", IsInvalidPath: true, LocalPath: true}
|
|
}
|
|
isDir, err := util.IsDir(u.Host + u.Path)
|
|
if err != nil {
|
|
log.Error("Unable to check if %s is a directory: %v", u.Host+u.Path, err)
|
|
return err
|
|
}
|
|
if !isDir {
|
|
return &models.ErrInvalidCloneAddr{Host: "<LOCAL_FILESYSTEM>", IsInvalidPath: true, LocalPath: true}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
if u.Scheme == "git" && u.Port() != "" && (strings.Contains(remoteURL, "%0d") || strings.Contains(remoteURL, "%0a")) {
|
|
return &models.ErrInvalidCloneAddr{Host: u.Host, IsURLError: true}
|
|
}
|
|
|
|
if u.Opaque != "" || u.Scheme != "" && u.Scheme != "http" && u.Scheme != "https" && u.Scheme != "git" {
|
|
return &models.ErrInvalidCloneAddr{Host: u.Host, IsProtocolInvalid: true, IsPermissionDenied: true, IsURLError: true}
|
|
}
|
|
|
|
hostName, _, err := net.SplitHostPort(u.Host)
|
|
if err != nil {
|
|
// u.Host can be "host" or "host:port"
|
|
err = nil //nolint
|
|
hostName = u.Host
|
|
}
|
|
|
|
// some users only use proxy, there is no DNS resolver. it's safe to ignore the LookupIP error
|
|
addrList, _ := net.LookupIP(hostName)
|
|
return checkByAllowBlockList(hostName, addrList)
|
|
}
|
|
|
|
func checkByAllowBlockList(hostName string, addrList []net.IP) error {
|
|
var ipAllowed bool
|
|
var ipBlocked bool
|
|
for _, addr := range addrList {
|
|
ipAllowed = ipAllowed || allowList.MatchIPAddr(addr)
|
|
ipBlocked = ipBlocked || blockList.MatchIPAddr(addr)
|
|
}
|
|
var blockedError error
|
|
if blockList.MatchHostName(hostName) || ipBlocked {
|
|
blockedError = &models.ErrInvalidCloneAddr{Host: hostName, IsPermissionDenied: true}
|
|
}
|
|
// if we have an allow-list, check the allow-list before return to get the more accurate error
|
|
if !allowList.IsEmpty() {
|
|
if !allowList.MatchHostName(hostName) && !ipAllowed {
|
|
return &models.ErrInvalidCloneAddr{Host: hostName, IsPermissionDenied: true}
|
|
}
|
|
}
|
|
// otherwise, we always follow the blocked list
|
|
return blockedError
|
|
}
|
|
|
|
// MigrateRepository migrate repository according MigrateOptions
|
|
func MigrateRepository(ctx context.Context, doer *user_model.User, ownerName string, opts base.MigrateOptions, messenger base.Messenger) (*repo_model.Repository, error) {
|
|
err := IsMigrateURLAllowed(opts.CloneAddr, doer)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if opts.LFS && len(opts.LFSEndpoint) > 0 {
|
|
err := IsMigrateURLAllowed(opts.LFSEndpoint, doer)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
downloader, err := newDownloader(ctx, ownerName, opts)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
uploader := NewGiteaLocalUploader(ctx, doer, ownerName, opts.RepoName)
|
|
uploader.gitServiceType = opts.GitServiceType
|
|
|
|
if err := migrateRepository(ctx, doer, downloader, uploader, opts, messenger); err != nil {
|
|
if err1 := uploader.Rollback(); err1 != nil {
|
|
log.Error("rollback failed: %v", err1)
|
|
}
|
|
if err2 := system_model.CreateRepositoryNotice(fmt.Sprintf("Migrate repository from %s failed: %v", opts.OriginalURL, err)); err2 != nil {
|
|
log.Error("create respotiry notice failed: ", err2)
|
|
}
|
|
return nil, err
|
|
}
|
|
return uploader.repo, nil
|
|
}
|
|
|
|
func getFactoryFromServiceType(serviceType structs.GitServiceType) base.DownloaderFactory {
|
|
for _, factory := range factories {
|
|
if factory.GitServiceType() == serviceType {
|
|
return factory
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func newDownloader(ctx context.Context, ownerName string, opts base.MigrateOptions) (base.Downloader, error) {
|
|
var (
|
|
downloader base.Downloader
|
|
err error
|
|
)
|
|
|
|
if factory := getFactoryFromServiceType(opts.GitServiceType); factory != nil {
|
|
downloader, err = factory.New(ctx, opts)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
|
|
if downloader == nil {
|
|
opts.Wiki = true
|
|
opts.Milestones = false
|
|
opts.Labels = false
|
|
opts.Releases = false
|
|
opts.Comments = false
|
|
opts.Issues = false
|
|
opts.PullRequests = false
|
|
downloader = NewPlainGitDownloader(ownerName, opts.RepoName, opts.CloneAddr)
|
|
log.Trace("Will migrate from git: %s", opts.OriginalURL)
|
|
}
|
|
|
|
if setting.Migrations.MaxAttempts > 1 {
|
|
downloader = base.NewRetryDownloader(ctx, downloader, setting.Migrations.MaxAttempts, setting.Migrations.RetryBackoff)
|
|
}
|
|
return downloader, nil
|
|
}
|
|
|
|
// migrateRepository will download information and then upload it to Uploader, this is a simple
|
|
// process for small repository. For a big repository, save all the data to disk
|
|
// before upload is better
|
|
func migrateRepository(ctx context.Context, doer *user_model.User, downloader base.Downloader, uploader base.Uploader, opts base.MigrateOptions, messenger base.Messenger) error {
|
|
if messenger == nil {
|
|
messenger = base.NilMessenger
|
|
}
|
|
|
|
repo, err := downloader.GetRepoInfo()
|
|
if err != nil {
|
|
if !base.IsErrNotSupported(err) {
|
|
return err
|
|
}
|
|
log.Info("migrating repo infos is not supported, ignored")
|
|
}
|
|
repo.IsPrivate = opts.Private
|
|
repo.IsMirror = opts.Mirror
|
|
if opts.Description != "" {
|
|
repo.Description = opts.Description
|
|
}
|
|
if repo.CloneURL, err = downloader.FormatCloneURL(opts, repo.CloneURL); err != nil {
|
|
return err
|
|
}
|
|
|
|
// SECURITY: If the downloader is not a RepositoryRestorer then we need to recheck the CloneURL
|
|
if _, ok := downloader.(*RepositoryRestorer); !ok {
|
|
// Now the clone URL can be rewritten by the downloader so we must recheck
|
|
if err := IsMigrateURLAllowed(repo.CloneURL, doer); err != nil {
|
|
return err
|
|
}
|
|
|
|
// SECURITY: Ensure that we haven't been redirected from an external to a local filesystem
|
|
// Now we know all of these must parse
|
|
cloneAddrURL, _ := url.Parse(opts.CloneAddr)
|
|
cloneURL, _ := url.Parse(repo.CloneURL)
|
|
|
|
if cloneURL.Scheme == "file" || cloneURL.Scheme == "" {
|
|
if cloneAddrURL.Scheme != "file" && cloneAddrURL.Scheme != "" {
|
|
return fmt.Errorf("repo info has changed from external to local filesystem")
|
|
}
|
|
}
|
|
|
|
// We don't actually need to check the OriginalURL as it isn't used anywhere
|
|
}
|
|
|
|
log.Trace("migrating git data from %s", repo.CloneURL)
|
|
messenger("repo.migrate.migrating_git")
|
|
if err = uploader.CreateRepo(repo, opts); err != nil {
|
|
return err
|
|
}
|
|
defer uploader.Close()
|
|
|
|
log.Trace("migrating topics")
|
|
messenger("repo.migrate.migrating_topics")
|
|
topics, err := downloader.GetTopics()
|
|
if err != nil {
|
|
if !base.IsErrNotSupported(err) {
|
|
return err
|
|
}
|
|
log.Warn("migrating topics is not supported, ignored")
|
|
}
|
|
if len(topics) != 0 {
|
|
if err = uploader.CreateTopics(topics...); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
if opts.Milestones {
|
|
log.Trace("migrating milestones")
|
|
messenger("repo.migrate.migrating_milestones")
|
|
milestones, err := downloader.GetMilestones()
|
|
if err != nil {
|
|
if !base.IsErrNotSupported(err) {
|
|
return err
|
|
}
|
|
log.Warn("migrating milestones is not supported, ignored")
|
|
}
|
|
|
|
msBatchSize := uploader.MaxBatchInsertSize("milestone")
|
|
for len(milestones) > 0 {
|
|
if len(milestones) < msBatchSize {
|
|
msBatchSize = len(milestones)
|
|
}
|
|
|
|
if err := uploader.CreateMilestones(milestones...); err != nil {
|
|
return err
|
|
}
|
|
milestones = milestones[msBatchSize:]
|
|
}
|
|
}
|
|
|
|
if opts.Labels {
|
|
log.Trace("migrating labels")
|
|
messenger("repo.migrate.migrating_labels")
|
|
labels, err := downloader.GetLabels()
|
|
if err != nil {
|
|
if !base.IsErrNotSupported(err) {
|
|
return err
|
|
}
|
|
log.Warn("migrating labels is not supported, ignored")
|
|
}
|
|
|
|
lbBatchSize := uploader.MaxBatchInsertSize("label")
|
|
for len(labels) > 0 {
|
|
if len(labels) < lbBatchSize {
|
|
lbBatchSize = len(labels)
|
|
}
|
|
|
|
if err := uploader.CreateLabels(labels[:lbBatchSize]...); err != nil {
|
|
return err
|
|
}
|
|
labels = labels[lbBatchSize:]
|
|
}
|
|
}
|
|
|
|
if opts.Releases {
|
|
log.Trace("migrating releases")
|
|
messenger("repo.migrate.migrating_releases")
|
|
releases, err := downloader.GetReleases()
|
|
if err != nil {
|
|
if !base.IsErrNotSupported(err) {
|
|
return err
|
|
}
|
|
log.Warn("migrating releases is not supported, ignored")
|
|
}
|
|
|
|
relBatchSize := uploader.MaxBatchInsertSize("release")
|
|
for len(releases) > 0 {
|
|
if len(releases) < relBatchSize {
|
|
relBatchSize = len(releases)
|
|
}
|
|
|
|
if err = uploader.CreateReleases(releases[:relBatchSize]...); err != nil {
|
|
return err
|
|
}
|
|
releases = releases[relBatchSize:]
|
|
}
|
|
|
|
// Once all releases (if any) are inserted, sync any remaining non-release tags
|
|
if err = uploader.SyncTags(); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
var (
|
|
commentBatchSize = uploader.MaxBatchInsertSize("comment")
|
|
reviewBatchSize = uploader.MaxBatchInsertSize("review")
|
|
)
|
|
|
|
supportAllComments := downloader.SupportGetRepoComments()
|
|
|
|
if opts.Issues {
|
|
log.Trace("migrating issues and comments")
|
|
messenger("repo.migrate.migrating_issues")
|
|
issueBatchSize := uploader.MaxBatchInsertSize("issue")
|
|
|
|
for i := 1; ; i++ {
|
|
issues, isEnd, err := downloader.GetIssues(i, issueBatchSize)
|
|
if err != nil {
|
|
if !base.IsErrNotSupported(err) {
|
|
return err
|
|
}
|
|
log.Warn("migrating issues is not supported, ignored")
|
|
break
|
|
}
|
|
|
|
if err := uploader.CreateIssues(issues...); err != nil {
|
|
return err
|
|
}
|
|
|
|
if opts.Comments && !supportAllComments {
|
|
allComments := make([]*base.Comment, 0, commentBatchSize)
|
|
for _, issue := range issues {
|
|
log.Trace("migrating issue %d's comments", issue.Number)
|
|
comments, _, err := downloader.GetComments(issue)
|
|
if err != nil {
|
|
if !base.IsErrNotSupported(err) {
|
|
return err
|
|
}
|
|
log.Warn("migrating comments is not supported, ignored")
|
|
}
|
|
|
|
allComments = append(allComments, comments...)
|
|
|
|
if len(allComments) >= commentBatchSize {
|
|
if err = uploader.CreateComments(allComments[:commentBatchSize]...); err != nil {
|
|
return err
|
|
}
|
|
|
|
allComments = allComments[commentBatchSize:]
|
|
}
|
|
}
|
|
|
|
if len(allComments) > 0 {
|
|
if err = uploader.CreateComments(allComments...); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
|
|
if isEnd {
|
|
break
|
|
}
|
|
}
|
|
}
|
|
|
|
if opts.PullRequests {
|
|
log.Trace("migrating pull requests and comments")
|
|
messenger("repo.migrate.migrating_pulls")
|
|
prBatchSize := uploader.MaxBatchInsertSize("pullrequest")
|
|
for i := 1; ; i++ {
|
|
prs, isEnd, err := downloader.GetPullRequests(i, prBatchSize)
|
|
if err != nil {
|
|
if !base.IsErrNotSupported(err) {
|
|
return err
|
|
}
|
|
log.Warn("migrating pull requests is not supported, ignored")
|
|
break
|
|
}
|
|
|
|
if err := uploader.CreatePullRequests(prs...); err != nil {
|
|
return err
|
|
}
|
|
|
|
if opts.Comments {
|
|
if !supportAllComments {
|
|
// plain comments
|
|
allComments := make([]*base.Comment, 0, commentBatchSize)
|
|
for _, pr := range prs {
|
|
log.Trace("migrating pull request %d's comments", pr.Number)
|
|
comments, _, err := downloader.GetComments(pr)
|
|
if err != nil {
|
|
if !base.IsErrNotSupported(err) {
|
|
return err
|
|
}
|
|
log.Warn("migrating comments is not supported, ignored")
|
|
}
|
|
|
|
allComments = append(allComments, comments...)
|
|
|
|
if len(allComments) >= commentBatchSize {
|
|
if err = uploader.CreateComments(allComments[:commentBatchSize]...); err != nil {
|
|
return err
|
|
}
|
|
allComments = allComments[commentBatchSize:]
|
|
}
|
|
}
|
|
if len(allComments) > 0 {
|
|
if err = uploader.CreateComments(allComments...); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
|
|
// migrate reviews
|
|
allReviews := make([]*base.Review, 0, reviewBatchSize)
|
|
for _, pr := range prs {
|
|
reviews, err := downloader.GetReviews(pr)
|
|
if err != nil {
|
|
if !base.IsErrNotSupported(err) {
|
|
return err
|
|
}
|
|
log.Warn("migrating reviews is not supported, ignored")
|
|
break
|
|
}
|
|
|
|
allReviews = append(allReviews, reviews...)
|
|
|
|
if len(allReviews) >= reviewBatchSize {
|
|
if err = uploader.CreateReviews(allReviews[:reviewBatchSize]...); err != nil {
|
|
return err
|
|
}
|
|
allReviews = allReviews[reviewBatchSize:]
|
|
}
|
|
}
|
|
if len(allReviews) > 0 {
|
|
if err = uploader.CreateReviews(allReviews...); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
|
|
if isEnd {
|
|
break
|
|
}
|
|
}
|
|
}
|
|
|
|
if opts.Comments && supportAllComments {
|
|
log.Trace("migrating comments")
|
|
for i := 1; ; i++ {
|
|
comments, isEnd, err := downloader.GetAllComments(i, commentBatchSize)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if err := uploader.CreateComments(comments...); err != nil {
|
|
return err
|
|
}
|
|
|
|
if isEnd {
|
|
break
|
|
}
|
|
}
|
|
}
|
|
|
|
return uploader.Finish()
|
|
}
|
|
|
|
// Init migrations service
|
|
func Init() error {
|
|
// TODO: maybe we can deprecate these legacy ALLOWED_DOMAINS/ALLOW_LOCALNETWORKS/BLOCKED_DOMAINS, use ALLOWED_HOST_LIST/BLOCKED_HOST_LIST instead
|
|
|
|
blockList = hostmatcher.ParseSimpleMatchList("migrations.BLOCKED_DOMAINS", setting.Migrations.BlockedDomains)
|
|
|
|
allowList = hostmatcher.ParseSimpleMatchList("migrations.ALLOWED_DOMAINS/ALLOW_LOCALNETWORKS", setting.Migrations.AllowedDomains)
|
|
if allowList.IsEmpty() {
|
|
// the default policy is that migration module can access external hosts
|
|
allowList.AppendBuiltin(hostmatcher.MatchBuiltinExternal)
|
|
}
|
|
if setting.Migrations.AllowLocalNetworks {
|
|
allowList.AppendBuiltin(hostmatcher.MatchBuiltinPrivate)
|
|
allowList.AppendBuiltin(hostmatcher.MatchBuiltinLoopback)
|
|
}
|
|
// TODO: at the moment, if ALLOW_LOCALNETWORKS=false, ALLOWED_DOMAINS=domain.com, and domain.com has IP 127.0.0.1, then it's still allowed.
|
|
// if we want to block such case, the private&loopback should be added to the blockList when ALLOW_LOCALNETWORKS=false
|
|
|
|
if setting.Proxy.Enabled && setting.Proxy.ProxyURLFixed != nil {
|
|
allowList.AppendPattern(setting.Proxy.ProxyURLFixed.Host)
|
|
}
|
|
|
|
return nil
|
|
}
|