2022-02-09 20:28:55 +00:00
|
|
|
// Copyright 2021 The Gitea Authors. All rights reserved.
|
2022-11-27 18:20:29 +00:00
|
|
|
// SPDX-License-Identifier: MIT
|
2022-02-09 20:28:55 +00:00
|
|
|
|
|
|
|
package files
|
|
|
|
|
|
|
|
import (
|
|
|
|
"context"
|
|
|
|
"fmt"
|
|
|
|
"strings"
|
|
|
|
|
|
|
|
"code.gitea.io/gitea/models"
|
2022-06-12 15:51:54 +00:00
|
|
|
git_model "code.gitea.io/gitea/models/git"
|
2022-02-09 20:28:55 +00:00
|
|
|
repo_model "code.gitea.io/gitea/models/repo"
|
|
|
|
user_model "code.gitea.io/gitea/models/user"
|
|
|
|
"code.gitea.io/gitea/modules/git"
|
Simplify how git repositories are opened (#28937)
## Purpose
This is a refactor toward building an abstraction over managing git
repositories.
Afterwards, it does not matter anymore if they are stored on the local
disk or somewhere remote.
## What this PR changes
We used `git.OpenRepository` everywhere previously.
Now, we should split them into two distinct functions:
Firstly, there are temporary repositories which do not change:
```go
git.OpenRepository(ctx, diskPath)
```
Gitea managed repositories having a record in the database in the
`repository` table are moved into the new package `gitrepo`:
```go
gitrepo.OpenRepository(ctx, repo_model.Repo)
```
Why is `repo_model.Repository` the second parameter instead of file
path?
Because then we can easily adapt our repository storage strategy.
The repositories can be stored locally, however, they could just as well
be stored on a remote server.
## Further changes in other PRs
- A Git Command wrapper on package `gitrepo` could be created. i.e.
`NewCommand(ctx, repo_model.Repository, commands...)`. `git.RunOpts{Dir:
repo.RepoPath()}`, the directory should be empty before invoking this
method and it can be filled in the function only. #28940
- Remove the `RepoPath()`/`WikiPath()` functions to reduce the
possibility of mistakes.
---------
Co-authored-by: delvh <dev.lh@web.de>
2024-01-27 20:09:51 +00:00
|
|
|
"code.gitea.io/gitea/modules/gitrepo"
|
2022-02-09 20:28:55 +00:00
|
|
|
"code.gitea.io/gitea/modules/log"
|
|
|
|
"code.gitea.io/gitea/modules/structs"
|
|
|
|
asymkey_service "code.gitea.io/gitea/services/asymkey"
|
|
|
|
)
|
|
|
|
|
|
|
|
// ApplyDiffPatchOptions holds the repository diff patch update options
|
|
|
|
type ApplyDiffPatchOptions struct {
|
|
|
|
LastCommitID string
|
|
|
|
OldBranch string
|
|
|
|
NewBranch string
|
|
|
|
Message string
|
|
|
|
Content string
|
|
|
|
SHA string
|
|
|
|
Author *IdentityOptions
|
|
|
|
Committer *IdentityOptions
|
|
|
|
Dates *CommitDateOptions
|
|
|
|
Signoff bool
|
|
|
|
}
|
|
|
|
|
|
|
|
// Validate validates the provided options
|
|
|
|
func (opts *ApplyDiffPatchOptions) Validate(ctx context.Context, repo *repo_model.Repository, doer *user_model.User) error {
|
|
|
|
// If no branch name is set, assume master
|
|
|
|
if opts.OldBranch == "" {
|
|
|
|
opts.OldBranch = repo.DefaultBranch
|
|
|
|
}
|
|
|
|
if opts.NewBranch == "" {
|
|
|
|
opts.NewBranch = opts.OldBranch
|
|
|
|
}
|
|
|
|
|
Simplify how git repositories are opened (#28937)
## Purpose
This is a refactor toward building an abstraction over managing git
repositories.
Afterwards, it does not matter anymore if they are stored on the local
disk or somewhere remote.
## What this PR changes
We used `git.OpenRepository` everywhere previously.
Now, we should split them into two distinct functions:
Firstly, there are temporary repositories which do not change:
```go
git.OpenRepository(ctx, diskPath)
```
Gitea managed repositories having a record in the database in the
`repository` table are moved into the new package `gitrepo`:
```go
gitrepo.OpenRepository(ctx, repo_model.Repo)
```
Why is `repo_model.Repository` the second parameter instead of file
path?
Because then we can easily adapt our repository storage strategy.
The repositories can be stored locally, however, they could just as well
be stored on a remote server.
## Further changes in other PRs
- A Git Command wrapper on package `gitrepo` could be created. i.e.
`NewCommand(ctx, repo_model.Repository, commands...)`. `git.RunOpts{Dir:
repo.RepoPath()}`, the directory should be empty before invoking this
method and it can be filled in the function only. #28940
- Remove the `RepoPath()`/`WikiPath()` functions to reduce the
possibility of mistakes.
---------
Co-authored-by: delvh <dev.lh@web.de>
2024-01-27 20:09:51 +00:00
|
|
|
gitRepo, closer, err := gitrepo.RepositoryFromContextOrOpen(ctx, repo)
|
2022-02-09 20:28:55 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
defer closer.Close()
|
|
|
|
|
|
|
|
// oldBranch must exist for this operation
|
|
|
|
if _, err := gitRepo.GetBranch(opts.OldBranch); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
// A NewBranch can be specified for the patch to be applied to.
|
|
|
|
// Check to make sure the branch does not already exist, otherwise we can't proceed.
|
|
|
|
// If we aren't branching to a new branch, make sure user can commit to the given branch
|
|
|
|
if opts.NewBranch != opts.OldBranch {
|
|
|
|
existingBranch, err := gitRepo.GetBranch(opts.NewBranch)
|
|
|
|
if existingBranch != nil {
|
2023-06-29 10:03:20 +00:00
|
|
|
return git_model.ErrBranchAlreadyExists{
|
2022-02-09 20:28:55 +00:00
|
|
|
BranchName: opts.NewBranch,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if err != nil && !git.IsErrBranchNotExist(err) {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
} else {
|
2023-01-16 08:00:22 +00:00
|
|
|
protectedBranch, err := git_model.GetFirstMatchProtectedBranchRule(ctx, repo.ID, opts.OldBranch)
|
2022-02-09 20:28:55 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2023-01-16 08:00:22 +00:00
|
|
|
if protectedBranch != nil {
|
|
|
|
protectedBranch.Repo = repo
|
|
|
|
if !protectedBranch.CanUserPush(ctx, doer) {
|
|
|
|
return models.ErrUserCannotCommit{
|
|
|
|
UserName: doer.LowerName,
|
|
|
|
}
|
2022-02-09 20:28:55 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
if protectedBranch != nil && protectedBranch.RequireSignedCommits {
|
|
|
|
_, _, _, err := asymkey_service.SignCRUDAction(ctx, repo.RepoPath(), doer, repo.RepoPath(), opts.OldBranch)
|
|
|
|
if err != nil {
|
|
|
|
if !asymkey_service.IsErrWontSign(err) {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
return models.ErrUserCannotCommit{
|
|
|
|
UserName: doer.LowerName,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// ApplyDiffPatch applies a patch to the given repository
|
|
|
|
func ApplyDiffPatch(ctx context.Context, repo *repo_model.Repository, doer *user_model.User, opts *ApplyDiffPatchOptions) (*structs.FileResponse, error) {
|
2023-09-21 23:43:29 +00:00
|
|
|
err := repo.MustNotBeArchived()
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
2022-02-09 20:28:55 +00:00
|
|
|
if err := opts.Validate(ctx, repo, doer); err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
message := strings.TrimSpace(opts.Message)
|
|
|
|
|
|
|
|
author, committer := GetAuthorAndCommitterUsers(opts.Author, opts.Committer, doer)
|
|
|
|
|
|
|
|
t, err := NewTemporaryUploadRepository(ctx, repo)
|
|
|
|
if err != nil {
|
2024-04-07 11:17:06 +00:00
|
|
|
log.Error("NewTemporaryUploadRepository failed: %v", err)
|
2022-02-09 20:28:55 +00:00
|
|
|
}
|
|
|
|
defer t.Close()
|
2024-01-16 15:06:51 +00:00
|
|
|
if err := t.Clone(opts.OldBranch, true); err != nil {
|
2022-02-09 20:28:55 +00:00
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
if err := t.SetDefaultIndex(); err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
// Get the commit of the original branch
|
|
|
|
commit, err := t.GetBranchCommit(opts.OldBranch)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err // Couldn't get a commit for the branch
|
|
|
|
}
|
|
|
|
|
|
|
|
// Assigned LastCommitID in opts if it hasn't been set
|
|
|
|
if opts.LastCommitID == "" {
|
|
|
|
opts.LastCommitID = commit.ID.String()
|
|
|
|
} else {
|
2023-12-13 21:02:00 +00:00
|
|
|
lastCommitID, err := t.gitRepo.ConvertToGitID(opts.LastCommitID)
|
2022-02-09 20:28:55 +00:00
|
|
|
if err != nil {
|
2022-10-24 19:29:17 +00:00
|
|
|
return nil, fmt.Errorf("ApplyPatch: Invalid last commit ID: %w", err)
|
2022-02-09 20:28:55 +00:00
|
|
|
}
|
|
|
|
opts.LastCommitID = lastCommitID.String()
|
|
|
|
if commit.ID.String() != opts.LastCommitID {
|
|
|
|
return nil, models.ErrCommitIDDoesNotMatch{
|
|
|
|
GivenCommitID: opts.LastCommitID,
|
|
|
|
CurrentCommitID: opts.LastCommitID,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
stdout := &strings.Builder{}
|
|
|
|
stderr := &strings.Builder{}
|
|
|
|
|
Refactor git command package to improve security and maintainability (#22678)
This PR follows #21535 (and replace #22592)
## Review without space diff
https://github.com/go-gitea/gitea/pull/22678/files?diff=split&w=1
## Purpose of this PR
1. Make git module command completely safe (risky user inputs won't be
passed as argument option anymore)
2. Avoid low-level mistakes like
https://github.com/go-gitea/gitea/pull/22098#discussion_r1045234918
3. Remove deprecated and dirty `CmdArgCheck` function, hide the `CmdArg`
type
4. Simplify code when using git command
## The main idea of this PR
* Move the `git.CmdArg` to the `internal` package, then no other package
except `git` could use it. Then developers could never do
`AddArguments(git.CmdArg(userInput))` any more.
* Introduce `git.ToTrustedCmdArgs`, it's for user-provided and already
trusted arguments. It's only used in a few cases, for example: use git
arguments from config file, help unit test with some arguments.
* Introduce `AddOptionValues` and `AddOptionFormat`, they make code more
clear and simple:
* Before: `AddArguments("-m").AddDynamicArguments(message)`
* After: `AddOptionValues("-m", message)`
* -
* Before: `AddArguments(git.CmdArg(fmt.Sprintf("--author='%s <%s>'",
sig.Name, sig.Email)))`
* After: `AddOptionFormat("--author='%s <%s>'", sig.Name, sig.Email)`
## FAQ
### Why these changes were not done in #21535 ?
#21535 is mainly a search&replace, it did its best to not change too
much logic.
Making the framework better needs a lot of changes, so this separate PR
is needed as the second step.
### The naming of `AddOptionXxx`
According to git's manual, the `--xxx` part is called `option`.
### How can it guarantee that `internal.CmdArg` won't be not misused?
Go's specification guarantees that. Trying to access other package's
internal package causes compilation error.
And, `golangci-lint` also denies the git/internal package. Only the
`git/command.go` can use it carefully.
### There is still a `ToTrustedCmdArgs`, will it still allow developers
to make mistakes and pass untrusted arguments?
Generally speaking, no. Because when using `ToTrustedCmdArgs`, the code
will be very complex (see the changes for examples). Then developers and
reviewers can know that something might be unreasonable.
### Why there was a `CmdArgCheck` and why it's removed?
At the moment of #21535, to reduce unnecessary changes, `CmdArgCheck`
was introduced as a hacky patch. Now, almost all code could be written
as `cmd := NewCommand(); cmd.AddXxx(...)`, then there is no need for
`CmdArgCheck` anymore.
### Why many codes for `signArg == ""` is deleted?
Because in the old code, `signArg` could never be empty string, it's
either `-S[key-id]` or `--no-gpg-sign`. So the `signArg == ""` is just
dead code.
---------
Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>
2023-02-04 02:30:43 +00:00
|
|
|
cmdApply := git.NewCommand(ctx, "apply", "--index", "--recount", "--cached", "--ignore-whitespace", "--whitespace=fix", "--binary")
|
2022-02-09 20:28:55 +00:00
|
|
|
if git.CheckGitVersionAtLeast("2.32") == nil {
|
Refactor git command package to improve security and maintainability (#22678)
This PR follows #21535 (and replace #22592)
## Review without space diff
https://github.com/go-gitea/gitea/pull/22678/files?diff=split&w=1
## Purpose of this PR
1. Make git module command completely safe (risky user inputs won't be
passed as argument option anymore)
2. Avoid low-level mistakes like
https://github.com/go-gitea/gitea/pull/22098#discussion_r1045234918
3. Remove deprecated and dirty `CmdArgCheck` function, hide the `CmdArg`
type
4. Simplify code when using git command
## The main idea of this PR
* Move the `git.CmdArg` to the `internal` package, then no other package
except `git` could use it. Then developers could never do
`AddArguments(git.CmdArg(userInput))` any more.
* Introduce `git.ToTrustedCmdArgs`, it's for user-provided and already
trusted arguments. It's only used in a few cases, for example: use git
arguments from config file, help unit test with some arguments.
* Introduce `AddOptionValues` and `AddOptionFormat`, they make code more
clear and simple:
* Before: `AddArguments("-m").AddDynamicArguments(message)`
* After: `AddOptionValues("-m", message)`
* -
* Before: `AddArguments(git.CmdArg(fmt.Sprintf("--author='%s <%s>'",
sig.Name, sig.Email)))`
* After: `AddOptionFormat("--author='%s <%s>'", sig.Name, sig.Email)`
## FAQ
### Why these changes were not done in #21535 ?
#21535 is mainly a search&replace, it did its best to not change too
much logic.
Making the framework better needs a lot of changes, so this separate PR
is needed as the second step.
### The naming of `AddOptionXxx`
According to git's manual, the `--xxx` part is called `option`.
### How can it guarantee that `internal.CmdArg` won't be not misused?
Go's specification guarantees that. Trying to access other package's
internal package causes compilation error.
And, `golangci-lint` also denies the git/internal package. Only the
`git/command.go` can use it carefully.
### There is still a `ToTrustedCmdArgs`, will it still allow developers
to make mistakes and pass untrusted arguments?
Generally speaking, no. Because when using `ToTrustedCmdArgs`, the code
will be very complex (see the changes for examples). Then developers and
reviewers can know that something might be unreasonable.
### Why there was a `CmdArgCheck` and why it's removed?
At the moment of #21535, to reduce unnecessary changes, `CmdArgCheck`
was introduced as a hacky patch. Now, almost all code could be written
as `cmd := NewCommand(); cmd.AddXxx(...)`, then there is no need for
`CmdArgCheck` anymore.
### Why many codes for `signArg == ""` is deleted?
Because in the old code, `signArg` could never be empty string, it's
either `-S[key-id]` or `--no-gpg-sign`. So the `signArg == ""` is just
dead code.
---------
Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>
2023-02-04 02:30:43 +00:00
|
|
|
cmdApply.AddArguments("-3")
|
2022-02-09 20:28:55 +00:00
|
|
|
}
|
|
|
|
|
Refactor git command package to improve security and maintainability (#22678)
This PR follows #21535 (and replace #22592)
## Review without space diff
https://github.com/go-gitea/gitea/pull/22678/files?diff=split&w=1
## Purpose of this PR
1. Make git module command completely safe (risky user inputs won't be
passed as argument option anymore)
2. Avoid low-level mistakes like
https://github.com/go-gitea/gitea/pull/22098#discussion_r1045234918
3. Remove deprecated and dirty `CmdArgCheck` function, hide the `CmdArg`
type
4. Simplify code when using git command
## The main idea of this PR
* Move the `git.CmdArg` to the `internal` package, then no other package
except `git` could use it. Then developers could never do
`AddArguments(git.CmdArg(userInput))` any more.
* Introduce `git.ToTrustedCmdArgs`, it's for user-provided and already
trusted arguments. It's only used in a few cases, for example: use git
arguments from config file, help unit test with some arguments.
* Introduce `AddOptionValues` and `AddOptionFormat`, they make code more
clear and simple:
* Before: `AddArguments("-m").AddDynamicArguments(message)`
* After: `AddOptionValues("-m", message)`
* -
* Before: `AddArguments(git.CmdArg(fmt.Sprintf("--author='%s <%s>'",
sig.Name, sig.Email)))`
* After: `AddOptionFormat("--author='%s <%s>'", sig.Name, sig.Email)`
## FAQ
### Why these changes were not done in #21535 ?
#21535 is mainly a search&replace, it did its best to not change too
much logic.
Making the framework better needs a lot of changes, so this separate PR
is needed as the second step.
### The naming of `AddOptionXxx`
According to git's manual, the `--xxx` part is called `option`.
### How can it guarantee that `internal.CmdArg` won't be not misused?
Go's specification guarantees that. Trying to access other package's
internal package causes compilation error.
And, `golangci-lint` also denies the git/internal package. Only the
`git/command.go` can use it carefully.
### There is still a `ToTrustedCmdArgs`, will it still allow developers
to make mistakes and pass untrusted arguments?
Generally speaking, no. Because when using `ToTrustedCmdArgs`, the code
will be very complex (see the changes for examples). Then developers and
reviewers can know that something might be unreasonable.
### Why there was a `CmdArgCheck` and why it's removed?
At the moment of #21535, to reduce unnecessary changes, `CmdArgCheck`
was introduced as a hacky patch. Now, almost all code could be written
as `cmd := NewCommand(); cmd.AddXxx(...)`, then there is no need for
`CmdArgCheck` anymore.
### Why many codes for `signArg == ""` is deleted?
Because in the old code, `signArg` could never be empty string, it's
either `-S[key-id]` or `--no-gpg-sign`. So the `signArg == ""` is just
dead code.
---------
Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>
2023-02-04 02:30:43 +00:00
|
|
|
if err := cmdApply.Run(&git.RunOpts{
|
2022-04-01 02:55:30 +00:00
|
|
|
Dir: t.basePath,
|
|
|
|
Stdout: stdout,
|
|
|
|
Stderr: stderr,
|
|
|
|
Stdin: strings.NewReader(opts.Content),
|
2022-02-09 20:28:55 +00:00
|
|
|
}); err != nil {
|
2022-10-24 19:29:17 +00:00
|
|
|
return nil, fmt.Errorf("Error: Stdout: %s\nStderr: %s\nErr: %w", stdout.String(), stderr.String(), err)
|
2022-02-09 20:28:55 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Now write the tree
|
|
|
|
treeHash, err := t.WriteTree()
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
// Now commit the tree
|
|
|
|
var commitHash string
|
|
|
|
if opts.Dates != nil {
|
2022-03-28 19:48:41 +00:00
|
|
|
commitHash, err = t.CommitTreeWithDate("HEAD", author, committer, treeHash, message, opts.Signoff, opts.Dates.Author, opts.Dates.Committer)
|
2022-02-09 20:28:55 +00:00
|
|
|
} else {
|
2022-03-28 19:48:41 +00:00
|
|
|
commitHash, err = t.CommitTree("HEAD", author, committer, treeHash, message, opts.Signoff)
|
2022-02-09 20:28:55 +00:00
|
|
|
}
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
// Then push this tree to NewBranch
|
|
|
|
if err := t.Push(doer, commitHash, opts.NewBranch); err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
commit, err = t.GetCommit(commitHash)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
fileCommitResponse, _ := GetFileCommitResponse(repo, commit) // ok if fails, then will be nil
|
Add context cache as a request level cache (#22294)
To avoid duplicated load of the same data in an HTTP request, we can set
a context cache to do that. i.e. Some pages may load a user from a
database with the same id in different areas on the same page. But the
code is hidden in two different deep logic. How should we share the
user? As a result of this PR, now if both entry functions accept
`context.Context` as the first parameter and we just need to refactor
`GetUserByID` to reuse the user from the context cache. Then it will not
be loaded twice on an HTTP request.
But of course, sometimes we would like to reload an object from the
database, that's why `RemoveContextData` is also exposed.
The core context cache is here. It defines a new context
```go
type cacheContext struct {
ctx context.Context
data map[any]map[any]any
lock sync.RWMutex
}
var cacheContextKey = struct{}{}
func WithCacheContext(ctx context.Context) context.Context {
return context.WithValue(ctx, cacheContextKey, &cacheContext{
ctx: ctx,
data: make(map[any]map[any]any),
})
}
```
Then you can use the below 4 methods to read/write/del the data within
the same context.
```go
func GetContextData(ctx context.Context, tp, key any) any
func SetContextData(ctx context.Context, tp, key, value any)
func RemoveContextData(ctx context.Context, tp, key any)
func GetWithContextCache[T any](ctx context.Context, cacheGroupKey string, cacheTargetID any, f func() (T, error)) (T, error)
```
Then let's take a look at how `system.GetString` implement it.
```go
func GetSetting(ctx context.Context, key string) (string, error) {
return cache.GetWithContextCache(ctx, contextCacheKey, key, func() (string, error) {
return cache.GetString(genSettingCacheKey(key), func() (string, error) {
res, err := GetSettingNoCache(ctx, key)
if err != nil {
return "", err
}
return res.SettingValue, nil
})
})
}
```
First, it will check if context data include the setting object with the
key. If not, it will query from the global cache which may be memory or
a Redis cache. If not, it will get the object from the database. In the
end, if the object gets from the global cache or database, it will be
set into the context cache.
An object stored in the context cache will only be destroyed after the
context disappeared.
2023-02-15 13:37:34 +00:00
|
|
|
verification := GetPayloadCommitVerification(ctx, commit)
|
2022-02-09 20:28:55 +00:00
|
|
|
fileResponse := &structs.FileResponse{
|
|
|
|
Commit: fileCommitResponse,
|
|
|
|
Verification: verification,
|
|
|
|
}
|
|
|
|
|
|
|
|
return fileResponse, nil
|
|
|
|
}
|