Move restore repo to internal router and invoke from command to avoid open the same db file or queues files (#15790)

* Move restore repo to internal router and invoke from command to avoid open the same db file or queues files

* Follow @zeripath's review

* set no timeout for resotre repo private request

* make restore repo cancelable

cmd/restore_repo.go

@@ -5,15 +5,12 @@
 package cmd
 
 import (
-	"context"
-	"strings"
+	"errors"
+	"net/http"
 
 	"code.gitea.io/gitea/modules/log"
-	"code.gitea.io/gitea/modules/migrations"
-	"code.gitea.io/gitea/modules/migrations/base"
+	"code.gitea.io/gitea/modules/private"
 	"code.gitea.io/gitea/modules/setting"
-	"code.gitea.io/gitea/modules/storage"
-	pull_service "code.gitea.io/gitea/services/pull"
 
 	"github.com/urfave/cli"
 )
@@ -50,70 +47,18 @@ wiki, issues, labels, releases, release_assets, milestones, pull_requests, comme
 }
 
 func runRestoreRepository(ctx *cli.Context) error {
-	if err := initDB(); err != nil {
-		return err
-	}
+	setting.NewContext()
 
-	log.Trace("AppPath: %s", setting.AppPath)
-	log.Trace("AppWorkPath: %s", setting.AppWorkPath)
-	log.Trace("Custom path: %s", setting.CustomPath)
-	log.Trace("Log path: %s", setting.LogRootPath)
-	setting.InitDBConfig()
-
-	if err := storage.Init(); err != nil {
-		return err
-	}
-
-	if err := pull_service.Init(); err != nil {
-		return err
-	}
-
-	var opts = base.MigrateOptions{
-		RepoName: ctx.String("repo_name"),
-	}
-
-	if len(ctx.String("units")) == 0 {
-		opts.Wiki = true
-		opts.Issues = true
-		opts.Milestones = true
-		opts.Labels = true
-		opts.Releases = true
-		opts.Comments = true
-		opts.PullRequests = true
-		opts.ReleaseAssets = true
-	} else {
-		units := strings.Split(ctx.String("units"), ",")
-		for _, unit := range units {
-			switch strings.ToLower(unit) {
-			case "wiki":
-				opts.Wiki = true
-			case "issues":
-				opts.Issues = true
-			case "milestones":
-				opts.Milestones = true
-			case "labels":
-				opts.Labels = true
-			case "releases":
-				opts.Releases = true
-			case "release_assets":
-				opts.ReleaseAssets = true
-			case "comments":
-				opts.Comments = true
-			case "pull_requests":
-				opts.PullRequests = true
-			}
-		}
-	}
-
-	if err := migrations.RestoreRepository(
-		context.Background(),
+	statusCode, errStr := private.RestoreRepo(
 		ctx.String("repo_dir"),
 		ctx.String("owner_name"),
 		ctx.String("repo_name"),
-	); err != nil {
-		log.Fatal("Failed to restore repository: %v", err)
-		return err
+		ctx.StringSlice("units"),
+	)
+	if statusCode == http.StatusOK {
+		return nil
 	}
 
-	return nil
+	log.Fatal("Failed to restore repository: %v", errStr)
+	return errors.New(errStr)
 }

modules/migrations/dump.go

@@ -13,6 +13,7 @@ import (
 	"os"
 	"path/filepath"
 	"strconv"
+	"strings"
 	"time"
 
 	"code.gitea.io/gitea/models"
@@ -563,8 +564,42 @@ func DumpRepository(ctx context.Context, baseDir, ownerName string, opts base.Mi
 	return nil
 }
 
+func updateOptionsUnits(opts *base.MigrateOptions, units []string) {
+	if len(units) == 0 {
+		opts.Wiki = true
+		opts.Issues = true
+		opts.Milestones = true
+		opts.Labels = true
+		opts.Releases = true
+		opts.Comments = true
+		opts.PullRequests = true
+		opts.ReleaseAssets = true
+	} else {
+		for _, unit := range units {
+			switch strings.ToLower(unit) {
+			case "wiki":
+				opts.Wiki = true
+			case "issues":
+				opts.Issues = true
+			case "milestones":
+				opts.Milestones = true
+			case "labels":
+				opts.Labels = true
+			case "releases":
+				opts.Releases = true
+			case "release_assets":
+				opts.ReleaseAssets = true
+			case "comments":
+				opts.Comments = true
+			case "pull_requests":
+				opts.PullRequests = true
+			}
+		}
+	}
+}
+
 // RestoreRepository restore a repository from the disk directory
-func RestoreRepository(ctx context.Context, baseDir string, ownerName, repoName string) error {
+func RestoreRepository(ctx context.Context, baseDir string, ownerName, repoName string, units []string) error {
 	doer, err := models.GetAdminUser()
 	if err != nil {
 		return err
@@ -580,17 +615,12 @@ func RestoreRepository(ctx context.Context, baseDir string, ownerName, repoName
 	}
 	tp, _ := strconv.Atoi(opts["service_type"])
 
-	if err = migrateRepository(downloader, uploader, base.MigrateOptions{
-		Wiki:           true,
-		Issues:         true,
-		Milestones:     true,
-		Labels:         true,
-		Releases:       true,
-		Comments:       true,
-		PullRequests:   true,
-		ReleaseAssets:  true,
+	var migrateOpts = base.MigrateOptions{
 		GitServiceType: structs.GitServiceType(tp),
-	}); err != nil {
+	}
+	updateOptionsUnits(&migrateOpts, units)
+
+	if err = migrateRepository(downloader, uploader, migrateOpts); err != nil {
 		if err1 := uploader.Rollback(); err1 != nil {
 			log.Error("rollback failed: %v", err1)
 		}

modules/private/restore_repo.go

@@ -0,0 +1,60 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package private
+
+import (
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"time"
+
+	"code.gitea.io/gitea/modules/setting"
+	jsoniter "github.com/json-iterator/go"
+)
+
+// RestoreParams structure holds a data for restore repository
+type RestoreParams struct {
+	RepoDir   string
+	OwnerName string
+	RepoName  string
+	Units     []string
+}
+
+// RestoreRepo calls the internal RestoreRepo function
+func RestoreRepo(repoDir, ownerName, repoName string, units []string) (int, string) {
+	reqURL := setting.LocalURL + "api/internal/restore_repo"
+
+	req := newInternalRequest(reqURL, "POST")
+	req.SetTimeout(3*time.Second, 0) // since the request will spend much time, don't timeout
+	req = req.Header("Content-Type", "application/json")
+	json := jsoniter.ConfigCompatibleWithStandardLibrary
+	jsonBytes, _ := json.Marshal(RestoreParams{
+		RepoDir:   repoDir,
+		OwnerName: ownerName,
+		RepoName:  repoName,
+		Units:     units,
+	})
+	req.Body(jsonBytes)
+	resp, err := req.Response()
+	if err != nil {
+		return http.StatusInternalServerError, fmt.Sprintf("Unable to contact gitea: %v, could you confirm it's running?", err.Error())
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != 200 {
+		var ret = struct {
+			Err string `json:"err"`
+		}{}
+		body, err := ioutil.ReadAll(resp.Body)
+		if err != nil {
+			return http.StatusInternalServerError, fmt.Sprintf("Response body error: %v", err.Error())
+		}
+		if err := json.Unmarshal(body, &ret); err != nil {
+			return http.StatusInternalServerError, fmt.Sprintf("Response body Unmarshal error: %v", err.Error())
+		}
+	}
+
+	return http.StatusOK, fmt.Sprintf("Restore repo %s/%s successfully", ownerName, repoName)
+}

routers/private/internal.go

@@ -69,6 +69,7 @@ func Routes() *web.Route {
 	r.Post("/manager/add-logger", bind(private.LoggerOptions{}), AddLogger)
 	r.Post("/manager/remove-logger/{group}/{name}", RemoveLogger)
 	r.Post("/mail/send", SendEmail)
+	r.Post("/restore_repo", RestoreRepo)
 
 	return r
 }

routers/private/restore_repo.go

@@ -0,0 +1,51 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package private
+
+import (
+	"io/ioutil"
+
+	myCtx "code.gitea.io/gitea/modules/context"
+	"code.gitea.io/gitea/modules/migrations"
+	jsoniter "github.com/json-iterator/go"
+)
+
+// RestoreRepo restore a repository from data
+func RestoreRepo(ctx *myCtx.PrivateContext) {
+	json := jsoniter.ConfigCompatibleWithStandardLibrary
+	bs, err := ioutil.ReadAll(ctx.Req.Body)
+	if err != nil {
+		ctx.JSON(500, map[string]string{
+			"err": err.Error(),
+		})
+		return
+	}
+	var params = struct {
+		RepoDir   string
+		OwnerName string
+		RepoName  string
+		Units     []string
+	}{}
+	if err = json.Unmarshal(bs, &params); err != nil {
+		ctx.JSON(500, map[string]string{
+			"err": err.Error(),
+		})
+		return
+	}
+
+	if err := migrations.RestoreRepository(
+		ctx.Req.Context(),
+		params.RepoDir,
+		params.OwnerName,
+		params.RepoName,
+		params.Units,
+	); err != nil {
+		ctx.JSON(500, map[string]string{
+			"err": err.Error(),
+		})
+	} else {
+		ctx.Status(200)
+	}
+}