Verified Commit 35defae8 authored by Loïc Dachary's avatar Loïc Dachary
Browse files

gofff: replace Gitea migration

parent 3ec1032d
Pipeline #1206 passed with stages
in 30 minutes and 8 seconds
......@@ -337,10 +337,7 @@ func TestAPIRepoMigrate(t *testing.T) {
cloneURL, repoName string
expectedStatus int
}{
{ctxUserID: 1, userID: 2, cloneURL: "https://github.com/go-gitea/test_repo.git", repoName: "git-admin", expectedStatus: http.StatusCreated},
{ctxUserID: 2, userID: 2, cloneURL: "https://github.com/go-gitea/test_repo.git", repoName: "git-own", expectedStatus: http.StatusCreated},
{ctxUserID: 2, userID: 1, cloneURL: "https://github.com/go-gitea/test_repo.git", repoName: "git-bad", expectedStatus: http.StatusForbidden},
{ctxUserID: 2, userID: 3, cloneURL: "https://github.com/go-gitea/test_repo.git", repoName: "git-org", expectedStatus: http.StatusCreated},
{ctxUserID: 2, userID: 6, cloneURL: "https://github.com/go-gitea/test_repo.git", repoName: "git-bad-org", expectedStatus: http.StatusForbidden},
{ctxUserID: 2, userID: 3, cloneURL: "https://localhost:3000/user/test_repo.git", repoName: "private-ip", expectedStatus: http.StatusUnprocessableEntity},
{ctxUserID: 2, userID: 3, cloneURL: "https://10.0.0.1/user/test_repo.git", repoName: "private-ip", expectedStatus: http.StatusUnprocessableEntity},
......
......@@ -12,20 +12,21 @@ import (
"os"
"path/filepath"
"reflect"
"strings"
"testing"
repo_model "code.gitea.io/gitea/models/repo"
"code.gitea.io/gitea/models/unittest"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/json"
base "code.gitea.io/gitea/modules/migration"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/structs"
"code.gitea.io/gitea/modules/util"
"code.gitea.io/gitea/services/migrations"
"github.com/stretchr/testify/assert"
"gopkg.in/yaml.v2"
"lab.forgefriends.org/friendlyforgeformat/gofff"
"lab.forgefriends.org/friendlyforgeformat/gofff/forges/file"
"lab.forgefriends.org/friendlyforgeformat/gofff/format"
)
func TestDumpRestore(t *testing.T) {
......@@ -40,83 +41,91 @@ func TestDumpRestore(t *testing.T) {
setting.AppVer = AppVer
}()
assert.NoError(t, migrations.Init())
reponame := "repo1"
basePath, err := os.MkdirTemp("", reponame)
assert.NoError(t, err)
defer util.RemoveAll(basePath)
repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{Name: reponame}).(*repo_model.Repository)
repoOwner := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: repo.OwnerID}).(*user_model.User)
repoOwner := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1}).(*user_model.User)
session := loginUser(t, repoOwner.Name)
token := getTokenForLoggedInUser(t, session)
//
// Phase 1: dump repo1 from the Gitea instance to the filesystem
//
fixture := file.NewFixture(t, gofff.AllFeatures)
fixture.CreateEverything(file.User{
ID: repoOwner.ID,
Name: repoOwner.Name,
Email: repoOwner.Email,
})
ctx := context.Background()
opts := migrations.MigrateOptions{
GitServiceType: structs.GiteaService,
Issues: true,
PullRequests: true,
Labels: true,
Milestones: true,
Comments: true,
AuthToken: token,
CloneAddr: repo.CloneLink().HTTPS,
RepoName: reponame,
}
err = migrations.DumpRepository(ctx, basePath, repoOwner.Name, opts)
assert.NoError(t, err)
//
// Verify desired side effects of the dump
//
d := filepath.Join(basePath, repo.OwnerName, repo.Name)
for _, f := range []string{"repo.yml", "topic.yml", "label.yml", "milestone.yml", "issue.yml"} {
assert.FileExists(t, filepath.Join(d, f))
}
assert.NoError(t, migrations.Init())
ctx := context.Background()
//
// Phase 2: restore from the filesystem to the Gitea instance in restoredrepo
// Phase 1: restore from the filesystem to the Gitea instance in restoredrepo
//
newreponame := "restored"
err = migrations.RestoreRepository(ctx, d, repo.OwnerName, newreponame, []string{
"labels", "issues", "comments", "milestones", "pull_requests",
restoredRepoName := "restored"
restoredRepoDirectory := fixture.GetDirectory()
err := migrations.RestoreRepository(ctx, restoredRepoDirectory, repoOwner.Name, restoredRepoName, []string{
"issues", "milestones", "labels", "releases", "release_assets", "comments", "pull_requests",
// wiki",
}, false)
assert.NoError(t, err)
newrepo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{Name: newreponame}).(*repo_model.Repository)
restoredRepo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{Name: restoredRepoName}).(*repo_model.Repository)
unittest.AssertExistsAndLoadBean(t, &repo_model.Attachment{Name: file.Asset1})
//
// Phase 3: dump restored from the Gitea instance to the filesystem
// Phase 2: dump restoredRepo from the Gitea instance to the filesystem
//
opts.RepoName = newreponame
opts.CloneAddr = newrepo.CloneLink().HTTPS
err = migrations.DumpRepository(ctx, basePath, repoOwner.Name, opts)
opts := base.MigrateOptions{
GitServiceType: structs.GiteaService,
Wiki: true,
Issues: true,
Milestones: true,
Labels: true,
Releases: true,
Comments: true,
PullRequests: true,
ReleaseAssets: true,
AuthToken: token,
CloneAddr: restoredRepo.CloneLink().HTTPS,
RepoName: restoredRepoName,
}
dumpedRepoDirectory := t.TempDir()
err = migrations.DumpRepository(ctx, dumpedRepoDirectory, repoOwner.Name, opts)
assert.NoError(t, err)
//
// Verify the dump of restored is the same as the dump of repo1
//
//fixture.AssertEquals(restoredRepoDirectory, dumpedRepoDirectory)
//
// Verify the fixture files are the same as the restored files
//
project := fixture.GetFile().GetProject()
comparator := &compareDump{
t: t,
basePath: basePath,
t: t,
repoBefore: project.Name,
ownerBefore: project.Owner,
dirBefore: restoredRepoDirectory,
repoAfter: restoredRepoName,
ownerAfter: repoOwner.Name,
dirAfter: dumpedRepoDirectory,
}
comparator.assertEquals(repo, newrepo)
comparator.assertEquals()
})
}
type compareDump struct {
t *testing.T
basePath string
repoBefore *repo_model.Repository
dirBefore string
repoAfter *repo_model.Repository
t *testing.T
repoBefore string
ownerBefore string
dirBefore string
repoAfter string
ownerAfter string
dirAfter string
}
......@@ -130,57 +139,58 @@ type compareField struct {
type compareFields map[string]compareField
func (c *compareDump) replaceRepoName(original string) string {
return strings.ReplaceAll(original, c.repoBefore.Name, c.repoAfter.Name)
}
func (c *compareDump) assertEquals(repoBefore, repoAfter *repo_model.Repository) {
c.repoBefore = repoBefore
c.dirBefore = filepath.Join(c.basePath, repoBefore.OwnerName, repoBefore.Name)
c.repoAfter = repoAfter
c.dirAfter = filepath.Join(c.basePath, repoAfter.OwnerName, repoAfter.Name)
func (c *compareDump) assertEquals() {
//
// base.Repository
//
_ = c.assertEqual("repo.yml", base.Repository{}, compareFields{
_ = c.assertEqual("project.json", format.Project{}, compareFields{
"Name": {
before: c.repoBefore.Name,
after: c.repoAfter.Name,
before: c.repoBefore,
after: c.repoAfter,
},
"Owner": {
before: c.ownerBefore,
after: c.ownerAfter,
},
"CloneURL": {transform: c.replaceRepoName},
"OriginalURL": {transform: c.replaceRepoName},
"Index": {ignore: true},
"CloneURL": {ignore: true},
})
//
// base.Label
//
labels, ok := c.assertEqual("label.yml", []base.Label{}, compareFields{}).([]*base.Label)
compareLabels := compareFields{
"Index": {ignore: true},
}
labels, ok := c.assertEqual("label.json", []format.Label{}, compareLabels).([]*format.Label)
assert.True(c.t, ok)
assert.GreaterOrEqual(c.t, len(labels), 1)
//
// base.Milestone
//
milestones, ok := c.assertEqual("milestone.yml", []base.Milestone{}, compareFields{
milestones, ok := c.assertEqual("milestone.json", []format.Milestone{}, compareFields{
"Index": {ignore: true},
"Updated": {ignore: true}, // the database updates that field independently
}).([]*base.Milestone)
}).([]*format.Milestone)
assert.True(c.t, ok)
assert.GreaterOrEqual(c.t, len(milestones), 1)
//
// base.Issue and the associated comments
// format.Issue and the associated comments
//
issues, ok := c.assertEqual("issue.yml", []base.Issue{}, compareFields{
issues, ok := c.assertEqual("issue.json", []format.Issue{}, compareFields{
"Index": {ignore: true},
"Assignees": {ignore: true}, // not implemented yet
}).([]*base.Issue)
"Labels": {nested: &compareLabels},
}).([]*format.Issue)
assert.True(c.t, ok)
assert.GreaterOrEqual(c.t, len(issues), 1)
for _, issue := range issues {
filename := filepath.Join("comments", fmt.Sprintf("%d.yml", issue.Number))
comments, ok := c.assertEqual(filename, []base.Comment{}, compareFields{
filename := filepath.Join("comments", fmt.Sprintf("%d.json", issue.Number))
comments, ok := c.assertEqual(filename, []format.Comment{}, compareFields{
"Index": {ignore: true},
}).([]*base.Comment)
}).([]*format.Comment)
assert.True(c.t, ok)
for _, comment := range comments {
assert.EqualValues(c.t, issue.Number, comment.IssueIndex)
......@@ -188,26 +198,32 @@ func (c *compareDump) assertEquals(repoBefore, repoAfter *repo_model.Repository)
}
//
// base.PullRequest and the associated comments
// format.PullRequest and the associated comments
//
comparePullRequestBranch := &compareFields{
"RepoName": {
before: c.repoBefore.Name,
after: c.repoAfter.Name,
before: c.repoBefore,
after: c.repoAfter,
},
"OwnerName": {
before: c.ownerBefore,
after: c.ownerAfter,
},
"CloneURL": {transform: c.replaceRepoName},
"CloneURL": {ignore: true},
}
prs, ok := c.assertEqual("pull_request.yml", []base.PullRequest{}, compareFields{
prs, ok := c.assertEqual("pull_request.json", []format.PullRequest{}, compareFields{
"Assignees": {ignore: true}, // not implemented yet
"Head": {nested: comparePullRequestBranch},
"Base": {nested: comparePullRequestBranch},
"PatchURL": {ignore: true},
"CloneURL": {ignore: true},
"Labels": {ignore: true}, // because org labels are not handled propery
}).([]*base.PullRequest)
}).([]*format.PullRequest)
assert.True(c.t, ok)
assert.GreaterOrEqual(c.t, len(prs), 1)
for _, pr := range prs {
filename := filepath.Join("comments", fmt.Sprintf("%d.yml", pr.Number))
comments, ok := c.assertEqual(filename, []base.Comment{}, compareFields{}).([]*base.Comment)
filename := filepath.Join("comments", fmt.Sprintf("%d.json", pr.Number))
comments, ok := c.assertEqual(filename, []format.Comment{}, compareFields{}).([]*format.Comment)
assert.True(c.t, ok)
for _, comment := range comments {
assert.EqualValues(c.t, pr.Number, comment.IssueIndex)
......@@ -215,7 +231,7 @@ func (c *compareDump) assertEquals(repoBefore, repoAfter *repo_model.Repository)
}
}
func (c *compareDump) assertLoadYAMLFiles(beforeFilename, afterFilename string, before, after interface{}) {
func (c *compareDump) assertLoadJSONFiles(beforeFilename, afterFilename string, before, after interface{}) {
_, beforeErr := os.Stat(beforeFilename)
_, afterErr := os.Stat(afterFilename)
assert.EqualValues(c.t, errors.Is(beforeErr, os.ErrNotExist), errors.Is(afterErr, os.ErrNotExist))
......@@ -225,10 +241,10 @@ func (c *compareDump) assertLoadYAMLFiles(beforeFilename, afterFilename string,
beforeBytes, err := os.ReadFile(beforeFilename)
assert.NoError(c.t, err)
assert.NoError(c.t, yaml.Unmarshal(beforeBytes, before))
assert.NoError(c.t, json.Unmarshal(beforeBytes, before))
afterBytes, err := os.ReadFile(afterFilename)
assert.NoError(c.t, err)
assert.NoError(c.t, yaml.Unmarshal(afterBytes, after))
assert.NoError(c.t, json.Unmarshal(afterBytes, after))
}
func (c *compareDump) assertLoadFiles(beforeFilename, afterFilename string, t reflect.Type) (before, after reflect.Value) {
......@@ -251,13 +267,14 @@ func (c *compareDump) assertLoadFiles(beforeFilename, afterFilename string, t re
beforePtr = reflect.New(t)
afterPtr = reflect.New(t)
}
c.assertLoadYAMLFiles(beforeFilename, afterFilename, beforePtr.Interface(), afterPtr.Interface())
c.assertLoadJSONFiles(beforeFilename, afterFilename, beforePtr.Interface(), afterPtr.Interface())
return beforePtr.Elem(), afterPtr.Elem()
}
func (c *compareDump) assertEqual(filename string, kind interface{}, fields compareFields) (i interface{}) {
beforeFilename := filepath.Join(c.dirBefore, filename)
afterFilename := filepath.Join(c.dirAfter, filename)
fmt.Println("assertEqual ", beforeFilename, afterFilename)
typeOf := reflect.TypeOf(kind)
before, after := c.assertLoadFiles(beforeFilename, afterFilename, typeOf)
......@@ -300,29 +317,34 @@ func (c *compareDump) assertEqualValues(before, after reflect.Value, fields comp
// Transform these strings before comparing them
//
bs, ok := bi.(string)
assert.True(c.t, ok)
assert.True(c.t, ok, field.Name)
as, ok := ai.(string)
assert.True(c.t, ok)
assert.EqualValues(c.t, compare.transform(bs), compare.transform(as))
assert.True(c.t, ok, field.Name)
assert.EqualValues(c.t, compare.transform(bs), compare.transform(as), field.Name)
continue
}
if compare.before != nil && compare.after != nil {
//
// The fields are expected to have different values
//
assert.EqualValues(c.t, compare.before, bi)
assert.EqualValues(c.t, compare.after, ai)
assert.EqualValues(c.t, compare.before, bi, field.Name)
assert.EqualValues(c.t, compare.after, ai, field.Name)
continue
}
if compare.nested != nil {
//
// The fields are a struct, recurse
// The fields are a struct/slice, recurse
//
c.assertEqualValues(bf, af, *compare.nested)
fmt.Println("nested ", field.Name)
if reflect.TypeOf(bi).Kind() == reflect.Slice {
c.assertEqualSlices(bf, af, *compare.nested)
} else {
c.assertEqualValues(bf, af, *compare.nested)
}
continue
}
}
assert.EqualValues(c.t, bi, ai)
assert.EqualValues(c.t, bi, ai, field.Name)
}
return after.Interface()
}
......@@ -50,7 +50,14 @@ func TestMirrorPull(t *testing.T) {
ctx := context.Background()
mirror, err := repository.MigrateRepositoryGitData(ctx, user, mirrorRepo, opts, nil)
fetch := func(repoPath string) {
assert.NoError(t, git.Clone(ctx, opts.CloneAddr, repoPath, git.CloneRepoOptions{
Mirror: true,
Quiet: true,
SkipTLSVerify: true,
}))
}
mirror, err := repository.MigrateRepositoryGitData(ctx, user, fetch, mirrorRepo, opts, nil)
assert.NoError(t, err)
gitRepo, err := git.OpenRepository(git.DefaultContext, repoPath)
......
// Copyright 2022 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package integrations
import (
"context"
"net/http"
"net/url"
"testing"
repo_model "code.gitea.io/gitea/models/repo"
"code.gitea.io/gitea/models/unittest"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/private"
"github.com/stretchr/testify/assert"
"lab.forgefriends.org/friendlyforgeformat/gofff"
"lab.forgefriends.org/friendlyforgeformat/gofff/forges/file"
)
func TestAPIPrivateRestoreRepo(t *testing.T) {
onGiteaRun(t, func(*testing.T, *url.URL) {
fixture := file.NewFixture(t, gofff.AllFeatures)
fixture.CreateEverything(file.User1)
repoOwner := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1}).(*user_model.User)
repoName := "restoredrepo"
validation := true
statusCode, errStr := private.RestoreRepo(
context.Background(),
fixture.GetDirectory(),
repoOwner.Name,
repoName,
[]string{"issues"},
validation,
)
assert.EqualValues(t, http.StatusOK, statusCode, errStr)
unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{Name: repoName})
})
}
......@@ -5,7 +5,11 @@
package migration
import "code.gitea.io/gitea/modules/structs"
import (
"code.gitea.io/gitea/modules/structs"
"lab.forgefriends.org/friendlyforgeformat/gofff"
)
// MigrateOptions defines the way a repository gets migrated
// this is for internal usage by migrations module and func who interact with it
......@@ -40,3 +44,16 @@ type MigrateOptions struct {
MigrateToRepoID int64
MirrorInterval string `json:"mirror_interval"`
}
func (m MigrateOptions) ToGofffFeatures() gofff.Features {
return gofff.Features{
Wiki: m.Wiki,
Issues: m.Issues,
Milestones: m.Milestones,
Labels: m.Labels,
Releases: m.Releases,
Comments: m.Comments,
PullRequests: m.PullRequests,
ReleaseAssets: m.ReleaseAssets,
}
}
......@@ -50,7 +50,7 @@ func WikiRemoteURL(ctx context.Context, remote string) string {
// MigrateRepositoryGitData starts migrating git related data after created migrating repository
func MigrateRepositoryGitData(ctx context.Context, u *user_model.User,
repo *repo_model.Repository, opts migration.MigrateOptions,
fetch func(string), repo *repo_model.Repository, opts migration.MigrateOptions,
httpTransport *http.Transport,
) (*repo_model.Repository, error) {
repoPath := repo_model.RepoPath(u.Name, opts.RepoName)
......@@ -72,14 +72,7 @@ func MigrateRepositoryGitData(ctx context.Context, u *user_model.User,
return repo, fmt.Errorf("Failed to remove %s: %v", repoPath, err)
}
if err = git.Clone(ctx, opts.CloneAddr, repoPath, git.CloneRepoOptions{
Mirror: true,
Quiet: true,
Timeout: migrateTimeout,
SkipTLSVerify: setting.Migrations.SkipTLSVerify,
}); err != nil {
return repo, fmt.Errorf("Clone: %v", err)
}
fetch(repoPath)
if err := git.WriteCommitGraph(ctx, repoPath); err != nil {
return repo, err
......@@ -225,6 +218,21 @@ func MigrateRepositoryGitData(ctx context.Context, u *user_model.User,
return repo, committer.Commit()
}
// MigrateRepositoryGitData starts migrating git related data after created migrating repository
func MigrateRepositoryGitDataWiki(ctx context.Context, u *user_model.User,
fetch func(string), repo *repo_model.Repository, opts migration.MigrateOptions,
) error {
wikiPath := repo_model.WikiPath(u.Name, opts.RepoName)
wikiRemotePath := WikiRemoteURL(ctx, opts.CloneAddr)
if len(wikiRemotePath) > 0 {
if err := util.RemoveAll(wikiPath); err != nil {
return fmt.Errorf("Failed to remove %s: %v", wikiPath, err)
}
fetch(wikiPath)
}
return git.WriteCommitGraph(ctx, wikiPath)
}
// cleanUpMigrateGitConfig removes mirror info which prevents "push --all".
// This also removes possible user credentials.
func cleanUpMigrateGitConfig(configPath string) error {
......
......@@ -22,7 +22,7 @@ import (
"code.gitea.io/gitea/modules/graceful"
"code.gitea.io/gitea/modules/lfs"
"code.gitea.io/gitea/modules/log"
base "code.gitea.io/gitea/modules/migration"
migration_module "code.gitea.io/gitea/modules/migration"
"code.gitea.io/gitea/modules/notification"
repo_module "code.gitea.io/gitea/modules/repository"
"code.gitea.io/gitea/modules/setting"
......@@ -140,7 +140,7 @@ func Migrate(ctx *context.APIContext) {
}
}
opts := migrations.MigrateOptions{
opts := migration_module.MigrateOptions{
CloneAddr: remoteAddr,
RepoName: form.RepoName,
Description: form.Description,
......@@ -221,10 +221,6 @@ func handleMigrateError(ctx *context.APIContext, repoOwner *user_model.User, rem
ctx.Error(http.StatusConflict, "", "The repository with the same name already exists.")
case repo_model.IsErrRepoFilesAlreadyExist(err):
ctx.Error(http.StatusConflict, "", "Files already exist for this repository. Adopt them or delete them.")
case migrations.IsRateLimitError(err):
ctx.Error(http.StatusUnprocessableEntity, "", "Remote visit addressed rate limitation.")
case migrations.IsTwoFactorAuthError(err):
ctx.Error(http.StatusUnprocessableEntity, "", "Remote visit required two factors authentication.")
case repo_model.IsErrReachLimitOfRepo(err):
ctx.Error(http.StatusUnprocessableEntity, "", fmt.Sprintf("You have already reached your limit of %d repositories.", repoOwner.MaxCreationLimit()))
case db.IsErrNameReserved(err):
......@@ -235,7 +231,7 @@ func handleMigrateError(ctx *context.APIContext, repoOwner *user_model.User, rem
ctx.Error(http.StatusUnprocessableEntity, "", fmt.Sprintf("The pattern '%s' is not allowed in a username.", err.(db.ErrNamePatternNotAllowed).Pattern))
case models.IsErrInvalidCloneAddr(err):
ctx.Error(http.StatusUnprocessableEntity, "", err)
case base.IsErrNotSupported(err):
case migration_module.IsErrNotSupported(err):
ctx.Error(http.StatusUnprocessableEntity, "", err)
default:
err = util.SanitizeErrorCredentialURLs(err)
......
......@@ -18,6 +18,7 @@ import (
"code.gitea.io/gitea/modules/context"
"code.gitea.io/gitea/modules/lfs"
"code.gitea.io/gitea/modules/log"
migration_module "code.gitea.io/gitea/modules/migration"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/structs"
"code.gitea.io/gitea/modules/util"
......@@ -76,10 +77,6 @@ func handleMigrateError(ctx *context.Context, owner *user_model.User, err error,
}
switch {
case migrations.IsRateLimitError(err):
ctx.RenderWithErr(ctx.Tr("form.visit_rate_limit"), tpl, form)
case migrations.IsTwoFactorAuthError(err):
ctx.RenderWithErr(ctx.Tr("form.2fa_auth_required"), tpl, form)
case repo_model.IsErrReachLimitOfRepo(err):
maxCreationLimit := owner.MaxCreationLimit()
msg := ctx.TrN(maxCreationLimit, "repo.form.reach_limit_of_creation_1", "repo.form.reach_limit_of_creation_n", maxCreationLimit)
......@@ -202,7 +199,7 @@ func MigratePost(ctx *context.Context) {
}
}
opts := migrations.MigrateOptions{
opts := migration_module.MigrateOptions{
OriginalURL: form.CloneAddr,
GitServiceType: form.Service,
CloneAddr: remoteAddr,
......
......@@ -6,570 +6,91 @@ package migrations
import (
"context"