Some repository refactors (#17950)

* some repository refactors

* remove unnecessary code

* Fix test

* Remove unnecessary banner

services/cron/tasks_extended.go

@@ -118,7 +118,7 @@ func registerRemoveRandomAvatars() {
 		RunAtStart: false,
 		Schedule:   "@every 72h",
 	}, func(ctx context.Context, _ *user_model.User, _ Config) error {
-		return models.RemoveRandomAvatars(ctx)
+		return repo_service.RemoveRandomAvatars(ctx)
 	})
 }
 

services/mailer/mail_issue.go

@@ -8,6 +8,8 @@ import (
 	"fmt"
 
 	"code.gitea.io/gitea/models"
+	"code.gitea.io/gitea/models/db"
+	repo_model "code.gitea.io/gitea/models/repo"
 	"code.gitea.io/gitea/models/unit"
 	user_model "code.gitea.io/gitea/models/user"
 	"code.gitea.io/gitea/modules/log"
@@ -78,7 +80,7 @@ func mailIssueCommentToParticipants(ctx *mailCommentContext, mentions []*user_mo
 	// =========== Repo watchers ===========
 	// Make repo watchers last, since it's likely the list with the most users
 	if !(ctx.Issue.IsPull && ctx.Issue.PullRequest.IsWorkInProgress() && ctx.ActionType != models.ActionCreatePullRequest) {
-		ids, err = models.GetRepoWatchersIDs(ctx.Issue.RepoID)
+		ids, err = repo_model.GetRepoWatchersIDs(db.DefaultContext, ctx.Issue.RepoID)
 		if err != nil {
 			return fmt.Errorf("GetRepoWatchersIDs(%d): %v", ctx.Issue.RepoID, err)
 		}

services/mailer/mail_release.go

@@ -8,6 +8,8 @@ import (
 	"bytes"
 
 	"code.gitea.io/gitea/models"
+	"code.gitea.io/gitea/models/db"
+	repo_model "code.gitea.io/gitea/models/repo"
 	user_model "code.gitea.io/gitea/models/user"
 	"code.gitea.io/gitea/modules/base"
 	"code.gitea.io/gitea/modules/log"
@@ -29,7 +31,7 @@ func MailNewRelease(rel *models.Release) {
 		return
 	}
 
-	watcherIDList, err := models.GetRepoWatchersIDs(rel.RepoID)
+	watcherIDList, err := repo_model.GetRepoWatchersIDs(db.DefaultContext, rel.RepoID)
 	if err != nil {
 		log.Error("GetRepoWatchersIDs(%d): %v", rel.RepoID, err)
 		return

services/migrations/gitea_uploader.go

@@ -154,7 +154,7 @@ func (g *GiteaLocalUploader) CreateTopics(topics ...string) error {
 		}
 	}
 	topics = topics[:c]
-	return models.SaveTopics(g.repo.ID, topics...)
+	return repo_model.SaveTopics(g.repo.ID, topics...)
 }
 
 // CreateMilestones creates milestones
@@ -980,5 +980,5 @@ func (g *GiteaLocalUploader) Finish() error {
 	}
 
 	g.repo.Status = repo_model.RepositoryReady
-	return models.UpdateRepositoryCols(g.repo, "status")
+	return repo_model.UpdateRepositoryCols(g.repo, "status")
 }

services/mirror/mirror_pull.go

@@ -60,7 +60,7 @@ func UpdateAddress(m *repo_model.Mirror, addr string) error {
 	}
 
 	m.Repo.OriginalURL = addr
-	return models.UpdateRepositoryCols(m.Repo, "original_url")
+	return repo_model.UpdateRepositoryCols(m.Repo, "original_url")
 }
 
 // mirrorSyncResult contains information of a updated reference.
@@ -476,7 +476,7 @@ func SyncPullMirror(ctx context.Context, repoID int64) bool {
 		return false
 	}
 
-	if err = models.UpdateRepositoryUpdatedTime(m.RepoID, commitDate); err != nil {
+	if err = repo_model.UpdateRepositoryUpdatedTime(m.RepoID, commitDate); err != nil {
 		log.Error("Update repository 'updated_unix' [%d]: %v", m.RepoID, err)
 		return false
 	}
@@ -539,7 +539,7 @@ func checkAndUpdateEmptyRepository(m *repo_model.Mirror, gitRepo *git.Repository
 		}
 		m.Repo.IsEmpty = false
 		// Update the is empty and default_branch columns
-		if err := models.UpdateRepositoryCols(m.Repo, "default_branch", "is_empty"); err != nil {
+		if err := repo_model.UpdateRepositoryCols(m.Repo, "default_branch", "is_empty"); err != nil {
 			log.Error("Failed to update default branch of repository %-v. Error: %v", m.Repo, err)
 			desc := fmt.Sprintf("Failed to uupdate default branch of repository '%s': %v", m.Repo.RepoPath(), err)
 			if err = admin_model.CreateRepositoryNotice(desc); err != nil {

services/repository/adopt.go

@@ -28,7 +28,7 @@ import (
 // AdoptRepository adopts pre-existing repository files for the user/organization.
 func AdoptRepository(doer, u *user_model.User, opts models.CreateRepoOptions) (*repo_model.Repository, error) {
 	if !doer.IsAdmin && !u.CanCreateRepo() {
-		return nil, models.ErrReachLimitOfRepo{
+		return nil, repo_model.ErrReachLimitOfRepo{
 			Limit: u.MaxRepoCreation,
 		}
 	}
@@ -188,7 +188,7 @@ func adoptRepository(ctx context.Context, repoPath string, u *user_model.User, r
 
 // DeleteUnadoptedRepository deletes unadopted repository files from the filesystem
 func DeleteUnadoptedRepository(doer, u *user_model.User, repoName string) error {
-	if err := models.IsUsableRepoName(repoName); err != nil {
+	if err := repo_model.IsUsableRepoName(repoName); err != nil {
 		return err
 	}
 
@@ -208,7 +208,7 @@ func DeleteUnadoptedRepository(doer, u *user_model.User, repoName string) error
 	if exist, err := repo_model.IsRepositoryExist(u, repoName); err != nil {
 		return err
 	} else if exist {
-		return models.ErrRepoAlreadyExist{
+		return repo_model.ErrRepoAlreadyExist{
 			Uname: u.Name,
 			Name:  repoName,
 		}
@@ -312,7 +312,7 @@ func ListUnadoptedRepositories(query string, opts *db.ListOptions) ([]string, in
 			return filepath.SkipDir
 		}
 		name = name[:len(name)-4]
-		if models.IsUsableRepoName(name) != nil || strings.ToLower(name) != name || !globRepo.Match(name) {
+		if repo_model.IsUsableRepoName(name) != nil || strings.ToLower(name) != name || !globRepo.Match(name) {
 			return filepath.SkipDir
 		}
 		if count < end {

services/repository/avatar.go

@@ -0,0 +1,121 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package repository
+
+import (
+	"context"
+	"crypto/md5"
+	"fmt"
+	"image/png"
+	"io"
+	"strconv"
+	"strings"
+
+	"code.gitea.io/gitea/models/db"
+	repo_model "code.gitea.io/gitea/models/repo"
+	"code.gitea.io/gitea/modules/avatar"
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/storage"
+)
+
+// UploadAvatar saves custom avatar for repository.
+// FIXME: split uploads to different subdirs in case we have massive number of repos.
+func UploadAvatar(repo *repo_model.Repository, data []byte) error {
+	m, err := avatar.Prepare(data)
+	if err != nil {
+		return err
+	}
+
+	newAvatar := fmt.Sprintf("%d-%x", repo.ID, md5.Sum(data))
+	if repo.Avatar == newAvatar { // upload the same picture
+		return nil
+	}
+
+	ctx, committer, err := db.TxContext()
+	if err != nil {
+		return err
+	}
+	defer committer.Close()
+
+	oldAvatarPath := repo.CustomAvatarRelativePath()
+
+	// Users can upload the same image to other repo - prefix it with ID
+	// Then repo will be removed - only it avatar file will be removed
+	repo.Avatar = newAvatar
+	if err := repo_model.UpdateRepositoryColsCtx(ctx, repo, "avatar"); err != nil {
+		return fmt.Errorf("UploadAvatar: Update repository avatar: %v", err)
+	}
+
+	if err := storage.SaveFrom(storage.RepoAvatars, repo.CustomAvatarRelativePath(), func(w io.Writer) error {
+		if err := png.Encode(w, *m); err != nil {
+			log.Error("Encode: %v", err)
+		}
+		return err
+	}); err != nil {
+		return fmt.Errorf("UploadAvatar %s failed: Failed to remove old repo avatar %s: %v", repo.RepoPath(), newAvatar, err)
+	}
+
+	if len(oldAvatarPath) > 0 {
+		if err := storage.RepoAvatars.Delete(oldAvatarPath); err != nil {
+			return fmt.Errorf("UploadAvatar: Failed to remove old repo avatar %s: %v", oldAvatarPath, err)
+		}
+	}
+
+	return committer.Commit()
+}
+
+// DeleteAvatar deletes the repos's custom avatar.
+func DeleteAvatar(repo *repo_model.Repository) error {
+	// Avatar not exists
+	if len(repo.Avatar) == 0 {
+		return nil
+	}
+
+	avatarPath := repo.CustomAvatarRelativePath()
+	log.Trace("DeleteAvatar[%d]: %s", repo.ID, avatarPath)
+
+	ctx, committer, err := db.TxContext()
+	if err != nil {
+		return err
+	}
+	defer committer.Close()
+
+	repo.Avatar = ""
+	if err := repo_model.UpdateRepositoryColsCtx(ctx, repo, "avatar"); err != nil {
+		return fmt.Errorf("DeleteAvatar: Update repository avatar: %v", err)
+	}
+
+	if err := storage.RepoAvatars.Delete(avatarPath); err != nil {
+		return fmt.Errorf("DeleteAvatar: Failed to remove %s: %v", avatarPath, err)
+	}
+
+	return committer.Commit()
+}
+
+// RemoveRandomAvatars removes the randomly generated avatars that were created for repositories
+func RemoveRandomAvatars(ctx context.Context) error {
+	return repo_model.IterateRepository(func(repository *repo_model.Repository) error {
+		select {
+		case <-ctx.Done():
+			return db.ErrCancelledf("before random avatars removed for %s", repository.FullName())
+		default:
+		}
+		stringifiedID := strconv.FormatInt(repository.ID, 10)
+		if repository.Avatar == stringifiedID {
+			return DeleteAvatar(repository)
+		}
+		return nil
+	})
+}
+
+// generateAvatar generates the avatar from a template repository
+func generateAvatar(ctx context.Context, templateRepo, generateRepo *repo_model.Repository) error {
+	generateRepo.Avatar = strings.Replace(templateRepo.Avatar, strconv.FormatInt(templateRepo.ID, 10), strconv.FormatInt(generateRepo.ID, 10), 1)
+	if _, err := storage.Copy(storage.RepoAvatars, generateRepo.CustomAvatarRelativePath(), storage.RepoAvatars, templateRepo.CustomAvatarRelativePath()); err != nil {
+		return err
+	}
+
+	return repo_model.UpdateRepositoryColsCtx(ctx, generateRepo, "avatar")
+}

services/repository/avatar_test.go

@@ -0,0 +1,64 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package repository
+
+import (
+	"bytes"
+	"crypto/md5"
+	"fmt"
+	"image"
+	"image/png"
+	"testing"
+
+	repo_model "code.gitea.io/gitea/models/repo"
+	"code.gitea.io/gitea/models/unittest"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestUploadAvatar(t *testing.T) {
+	// Generate image
+	myImage := image.NewRGBA(image.Rect(0, 0, 1, 1))
+	var buff bytes.Buffer
+	png.Encode(&buff, myImage)
+
+	assert.NoError(t, unittest.PrepareTestDatabase())
+	repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 10}).(*repo_model.Repository)
+
+	err := UploadAvatar(repo, buff.Bytes())
+	assert.NoError(t, err)
+	assert.Equal(t, fmt.Sprintf("%d-%x", 10, md5.Sum(buff.Bytes())), repo.Avatar)
+}
+
+func TestUploadBigAvatar(t *testing.T) {
+	// Generate BIG image
+	myImage := image.NewRGBA(image.Rect(0, 0, 5000, 1))
+	var buff bytes.Buffer
+	png.Encode(&buff, myImage)
+
+	assert.NoError(t, unittest.PrepareTestDatabase())
+	repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 10}).(*repo_model.Repository)
+
+	err := UploadAvatar(repo, buff.Bytes())
+	assert.Error(t, err)
+}
+
+func TestDeleteAvatar(t *testing.T) {
+	// Generate image
+	myImage := image.NewRGBA(image.Rect(0, 0, 1, 1))
+	var buff bytes.Buffer
+	png.Encode(&buff, myImage)
+
+	assert.NoError(t, unittest.PrepareTestDatabase())
+	repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 10}).(*repo_model.Repository)
+
+	err := UploadAvatar(repo, buff.Bytes())
+	assert.NoError(t, err)
+
+	err = DeleteAvatar(repo)
+	assert.NoError(t, err)
+
+	assert.Equal(t, "", repo.Avatar)
+}

services/repository/fork.go

@@ -22,9 +22,16 @@ import (
 	"code.gitea.io/gitea/modules/util"
 )
 
+// ForkRepoOptions contains the fork repository options
+type ForkRepoOptions struct {
+	BaseRepo    *repo_model.Repository
+	Name        string
+	Description string
+}
+
 // ForkRepository forks a repository
-func ForkRepository(doer, owner *user_model.User, opts models.ForkRepoOptions) (_ *repo_model.Repository, err error) {
-	forkedRepo, err := models.GetUserFork(opts.BaseRepo.ID, owner.ID)
+func ForkRepository(doer, owner *user_model.User, opts ForkRepoOptions) (_ *repo_model.Repository, err error) {
+	forkedRepo, err := repo_model.GetUserFork(opts.BaseRepo.ID, owner.ID)
 	if err != nil {
 		return nil, err
 	}

services/repository/fork_test.go

@@ -22,7 +22,7 @@ func TestForkRepository(t *testing.T) {
 	user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 13}).(*user_model.User)
 	repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 10}).(*repo_model.Repository)
 
-	fork, err := ForkRepository(user, user, models.ForkRepoOptions{
+	fork, err := ForkRepository(user, user, ForkRepoOptions{
 		BaseRepo:    repo,
 		Name:        "test",
 		Description: "test",

services/repository/hooks.go

@@ -10,6 +10,7 @@ import (
 
 	"code.gitea.io/gitea/models/db"
 	repo_model "code.gitea.io/gitea/models/repo"
+	"code.gitea.io/gitea/modules/git"
 	"code.gitea.io/gitea/modules/log"
 	repo_module "code.gitea.io/gitea/modules/repository"
 
@@ -50,3 +51,36 @@ func SyncRepositoryHooks(ctx context.Context) error {
 	log.Trace("Finished: SyncRepositoryHooks")
 	return nil
 }
+
+// GenerateGitHooks generates git hooks from a template repository
+func GenerateGitHooks(ctx context.Context, templateRepo, generateRepo *repo_model.Repository) error {
+	generateGitRepo, err := git.OpenRepository(generateRepo.RepoPath())
+	if err != nil {
+		return err
+	}
+	defer generateGitRepo.Close()
+
+	templateGitRepo, err := git.OpenRepository(templateRepo.RepoPath())
+	if err != nil {
+		return err
+	}
+	defer templateGitRepo.Close()
+
+	templateHooks, err := templateGitRepo.Hooks()
+	if err != nil {
+		return err
+	}
+
+	for _, templateHook := range templateHooks {
+		generateHook, err := generateGitRepo.GetHook(templateHook.Name())
+		if err != nil {
+			return err
+		}
+
+		generateHook.Content = templateHook.Content
+		if err := generateHook.Update(); err != nil {
+			return err
+		}
+	}
+	return nil
+}

services/repository/push.go

@@ -166,7 +166,7 @@ func pushUpdates(optsList []*repo_module.PushUpdateOptions) error {
 							}
 						}
 						// Update the is empty and default_branch columns
-						if err := models.UpdateRepositoryCols(repo, "default_branch", "is_empty"); err != nil {
+						if err := repo_model.UpdateRepositoryCols(repo, "default_branch", "is_empty"); err != nil {
 							return fmt.Errorf("UpdateRepositoryCols: %v", err)
 						}
 					}
@@ -227,7 +227,7 @@ func pushUpdates(optsList []*repo_module.PushUpdateOptions) error {
 			}
 
 			// Even if user delete a branch on a repository which he didn't watch, he will be watch that.
-			if err = models.WatchIfAuto(opts.PusherID, repo.ID, true); err != nil {
+			if err = repo_model.WatchIfAuto(opts.PusherID, repo.ID, true); err != nil {
 				log.Warn("Fail to perform auto watch on user %v for repo %v: %v", opts.PusherID, repo.ID, err)
 			}
 		} else {
@@ -239,7 +239,7 @@ func pushUpdates(optsList []*repo_module.PushUpdateOptions) error {
 	}
 
 	// Change repository last updated time.
-	if err := models.UpdateRepositoryUpdatedTime(repo.ID, time.Now()); err != nil {
+	if err := repo_model.UpdateRepositoryUpdatedTime(repo.ID, time.Now()); err != nil {
 		return fmt.Errorf("UpdateRepositoryUpdatedTime: %v", err)
 	}
 

services/repository/template.go

@@ -19,7 +19,7 @@ import (
 // GenerateRepository generates a repository from a template
 func GenerateRepository(doer, owner *user_model.User, templateRepo *repo_model.Repository, opts models.GenerateRepoOptions) (_ *repo_model.Repository, err error) {
 	if !doer.IsAdmin && !owner.CanCreateRepo() {
-		return nil, models.ErrReachLimitOfRepo{
+		return nil, repo_model.ErrReachLimitOfRepo{
 			Limit: owner.MaxRepoCreation,
 		}
 	}
@@ -40,14 +40,14 @@ func GenerateRepository(doer, owner *user_model.User, templateRepo *repo_model.R
 
 		// Topics
 		if opts.Topics {
-			if err = models.GenerateTopics(ctx, templateRepo, generateRepo); err != nil {
+			if err = repo_model.GenerateTopics(ctx, templateRepo, generateRepo); err != nil {
 				return err
 			}
 		}
 
 		// Git Hooks
 		if opts.GitHooks {
-			if err = models.GenerateGitHooks(ctx, templateRepo, generateRepo); err != nil {
+			if err = GenerateGitHooks(ctx, templateRepo, generateRepo); err != nil {
 				return err
 			}
 		}
@@ -61,7 +61,7 @@ func GenerateRepository(doer, owner *user_model.User, templateRepo *repo_model.R
 
 		// Avatar
 		if opts.Avatar && len(templateRepo.Avatar) > 0 {
-			if err = models.GenerateAvatar(ctx, templateRepo, generateRepo); err != nil {
+			if err = generateAvatar(ctx, templateRepo, generateRepo); err != nil {
 				return err
 			}
 		}

services/repository/transfer.go

@@ -64,7 +64,7 @@ func ChangeRepositoryName(doer *user_model.User, repo *repo_model.Repository, ne
 	// local copy's origin accordingly.
 
 	repoWorkingPool.CheckIn(fmt.Sprint(repo.ID))
-	if err := models.ChangeRepositoryName(doer, repo, newRepoName); err != nil {
+	if err := repo_model.ChangeRepositoryName(doer, repo, newRepoName); err != nil {
 		repoWorkingPool.CheckOut(fmt.Sprint(repo.ID))
 		return err
 	}

services/task/migrate.go

@@ -27,9 +27,9 @@ import (
 
 func handleCreateError(owner *user_model.User, err error) error {
 	switch {
-	case models.IsErrReachLimitOfRepo(err):
+	case repo_model.IsErrReachLimitOfRepo(err):
 		return fmt.Errorf("You have already reached your limit of %d repositories", owner.MaxCreationLimit())
-	case models.IsErrRepoAlreadyExist(err):
+	case repo_model.IsErrRepoAlreadyExist(err):
 		return errors.New("The repository name is already used")
 	case db.IsErrNameReserved(err):
 		return fmt.Errorf("The repository name '%s' is reserved", err.(db.ErrNameReserved).Name)
@@ -123,7 +123,7 @@ func runMigrateTask(t *models.Task) (err error) {
 		return
 	}
 
-	if models.IsErrRepoAlreadyExist(err) {
+	if repo_model.IsErrRepoAlreadyExist(err) {
 		err = errors.New("The repository name is already used")
 		return
 	}

services/wiki/wiki.go

@@ -375,7 +375,7 @@ func DeleteWikiPage(doer *user_model.User, repo *repo_model.Repository, wikiName
 
 // DeleteWiki removes the actual and local copy of repository wiki.
 func DeleteWiki(repo *repo_model.Repository) error {
-	if err := models.UpdateRepositoryUnits(repo, nil, []unit.Type{unit.TypeWiki}); err != nil {
+	if err := repo_model.UpdateRepositoryUnits(repo, nil, []unit.Type{unit.TypeWiki}); err != nil {
 		return err
 	}