2020-09-11 22:14:48 +08:00
|
|
|
// Copyright 2020 The Gitea Authors. All rights reserved.
|
2022-11-28 02:20:29 +08:00
|
|
|
// SPDX-License-Identifier: MIT
|
2020-09-11 22:14:48 +08:00
|
|
|
|
|
|
|
package repository
|
|
|
|
|
|
|
|
import (
|
2021-11-17 23:17:31 +08:00
|
|
|
"context"
|
2021-10-18 03:47:12 +08:00
|
|
|
"errors"
|
2020-09-11 22:14:48 +08:00
|
|
|
"fmt"
|
2021-11-17 23:17:31 +08:00
|
|
|
"strings"
|
2020-09-21 01:55:15 +08:00
|
|
|
"time"
|
2020-09-11 22:14:48 +08:00
|
|
|
|
2021-09-19 19:49:59 +08:00
|
|
|
"code.gitea.io/gitea/models/db"
|
2022-06-12 23:51:54 +08:00
|
|
|
git_model "code.gitea.io/gitea/models/git"
|
2021-12-10 09:27:50 +08:00
|
|
|
repo_model "code.gitea.io/gitea/models/repo"
|
2021-11-24 17:49:20 +08:00
|
|
|
user_model "code.gitea.io/gitea/models/user"
|
2020-09-11 22:14:48 +08:00
|
|
|
"code.gitea.io/gitea/modules/cache"
|
|
|
|
"code.gitea.io/gitea/modules/git"
|
Simplify how git repositories are opened (#28937)
## Purpose
This is a refactor toward building an abstraction over managing git
repositories.
Afterwards, it does not matter anymore if they are stored on the local
disk or somewhere remote.
## What this PR changes
We used `git.OpenRepository` everywhere previously.
Now, we should split them into two distinct functions:
Firstly, there are temporary repositories which do not change:
```go
git.OpenRepository(ctx, diskPath)
```
Gitea managed repositories having a record in the database in the
`repository` table are moved into the new package `gitrepo`:
```go
gitrepo.OpenRepository(ctx, repo_model.Repo)
```
Why is `repo_model.Repository` the second parameter instead of file
path?
Because then we can easily adapt our repository storage strategy.
The repositories can be stored locally, however, they could just as well
be stored on a remote server.
## Further changes in other PRs
- A Git Command wrapper on package `gitrepo` could be created. i.e.
`NewCommand(ctx, repo_model.Repository, commands...)`. `git.RunOpts{Dir:
repo.RepoPath()}`, the directory should be empty before invoking this
method and it can be filled in the function only. #28940
- Remove the `RepoPath()`/`WikiPath()` functions to reduce the
possibility of mistakes.
---------
Co-authored-by: delvh <dev.lh@web.de>
2024-01-28 04:09:51 +08:00
|
|
|
"code.gitea.io/gitea/modules/gitrepo"
|
2020-09-11 22:14:48 +08:00
|
|
|
"code.gitea.io/gitea/modules/graceful"
|
|
|
|
"code.gitea.io/gitea/modules/log"
|
2022-01-20 07:26:57 +08:00
|
|
|
"code.gitea.io/gitea/modules/process"
|
2020-09-11 22:14:48 +08:00
|
|
|
"code.gitea.io/gitea/modules/queue"
|
|
|
|
repo_module "code.gitea.io/gitea/modules/repository"
|
|
|
|
"code.gitea.io/gitea/modules/setting"
|
2021-11-17 23:17:31 +08:00
|
|
|
"code.gitea.io/gitea/modules/timeutil"
|
2021-11-21 17:11:48 +08:00
|
|
|
issue_service "code.gitea.io/gitea/services/issue"
|
2023-09-06 02:37:47 +08:00
|
|
|
notify_service "code.gitea.io/gitea/services/notify"
|
2020-09-11 22:14:48 +08:00
|
|
|
pull_service "code.gitea.io/gitea/services/pull"
|
|
|
|
)
|
|
|
|
|
|
|
|
// pushQueue represents a queue to handle update pull request tests
|
Rewrite queue (#24505)
# ⚠️ Breaking
Many deprecated queue config options are removed (actually, they should
have been removed in 1.18/1.19).
If you see the fatal message when starting Gitea: "Please update your
app.ini to remove deprecated config options", please follow the error
messages to remove these options from your app.ini.
Example:
```
2023/05/06 19:39:22 [E] Removed queue option: `[indexer].ISSUE_INDEXER_QUEUE_TYPE`. Use new options in `[queue.issue_indexer]`
2023/05/06 19:39:22 [E] Removed queue option: `[indexer].UPDATE_BUFFER_LEN`. Use new options in `[queue.issue_indexer]`
2023/05/06 19:39:22 [F] Please update your app.ini to remove deprecated config options
```
Many options in `[queue]` are are dropped, including:
`WRAP_IF_NECESSARY`, `MAX_ATTEMPTS`, `TIMEOUT`, `WORKERS`,
`BLOCK_TIMEOUT`, `BOOST_TIMEOUT`, `BOOST_WORKERS`, they can be removed
from app.ini.
# The problem
The old queue package has some legacy problems:
* complexity: I doubt few people could tell how it works.
* maintainability: Too many channels and mutex/cond are mixed together,
too many different structs/interfaces depends each other.
* stability: due to the complexity & maintainability, sometimes there
are strange bugs and difficult to debug, and some code doesn't have test
(indeed some code is difficult to test because a lot of things are mixed
together).
* general applicability: although it is called "queue", its behavior is
not a well-known queue.
* scalability: it doesn't seem easy to make it work with a cluster
without breaking its behaviors.
It came from some very old code to "avoid breaking", however, its
technical debt is too heavy now. It's a good time to introduce a better
"queue" package.
# The new queue package
It keeps using old config and concept as much as possible.
* It only contains two major kinds of concepts:
* The "base queue": channel, levelqueue, redis
* They have the same abstraction, the same interface, and they are
tested by the same testing code.
* The "WokerPoolQueue", it uses the "base queue" to provide "worker
pool" function, calls the "handler" to process the data in the base
queue.
* The new code doesn't do "PushBack"
* Think about a queue with many workers, the "PushBack" can't guarantee
the order for re-queued unhandled items, so in new code it just does
"normal push"
* The new code doesn't do "pause/resume"
* The "pause/resume" was designed to handle some handler's failure: eg:
document indexer (elasticsearch) is down
* If a queue is paused for long time, either the producers blocks or the
new items are dropped.
* The new code doesn't do such "pause/resume" trick, it's not a common
queue's behavior and it doesn't help much.
* If there are unhandled items, the "push" function just blocks for a
few seconds and then re-queue them and retry.
* The new code doesn't do "worker booster"
* Gitea's queue's handlers are light functions, the cost is only the
go-routine, so it doesn't make sense to "boost" them.
* The new code only use "max worker number" to limit the concurrent
workers.
* The new "Push" never blocks forever
* Instead of creating more and more blocking goroutines, return an error
is more friendly to the server and to the end user.
There are more details in code comments: eg: the "Flush" problem, the
strange "code.index" hanging problem, the "immediate" queue problem.
Almost ready for review.
TODO:
* [x] add some necessary comments during review
* [x] add some more tests if necessary
* [x] update documents and config options
* [x] test max worker / active worker
* [x] re-run the CI tasks to see whether any test is flaky
* [x] improve the `handleOldLengthConfiguration` to provide more
friendly messages
* [x] fine tune default config values (eg: length?)
## Code coverage:
![image](https://user-images.githubusercontent.com/2114189/236620635-55576955-f95d-4810-b12f-879026a3afdf.png)
2023-05-08 19:49:59 +08:00
|
|
|
var pushQueue *queue.WorkerPoolQueue[[]*repo_module.PushUpdateOptions]
|
2020-09-11 22:14:48 +08:00
|
|
|
|
|
|
|
// handle passed PR IDs and test the PRs
|
Rewrite queue (#24505)
# ⚠️ Breaking
Many deprecated queue config options are removed (actually, they should
have been removed in 1.18/1.19).
If you see the fatal message when starting Gitea: "Please update your
app.ini to remove deprecated config options", please follow the error
messages to remove these options from your app.ini.
Example:
```
2023/05/06 19:39:22 [E] Removed queue option: `[indexer].ISSUE_INDEXER_QUEUE_TYPE`. Use new options in `[queue.issue_indexer]`
2023/05/06 19:39:22 [E] Removed queue option: `[indexer].UPDATE_BUFFER_LEN`. Use new options in `[queue.issue_indexer]`
2023/05/06 19:39:22 [F] Please update your app.ini to remove deprecated config options
```
Many options in `[queue]` are are dropped, including:
`WRAP_IF_NECESSARY`, `MAX_ATTEMPTS`, `TIMEOUT`, `WORKERS`,
`BLOCK_TIMEOUT`, `BOOST_TIMEOUT`, `BOOST_WORKERS`, they can be removed
from app.ini.
# The problem
The old queue package has some legacy problems:
* complexity: I doubt few people could tell how it works.
* maintainability: Too many channels and mutex/cond are mixed together,
too many different structs/interfaces depends each other.
* stability: due to the complexity & maintainability, sometimes there
are strange bugs and difficult to debug, and some code doesn't have test
(indeed some code is difficult to test because a lot of things are mixed
together).
* general applicability: although it is called "queue", its behavior is
not a well-known queue.
* scalability: it doesn't seem easy to make it work with a cluster
without breaking its behaviors.
It came from some very old code to "avoid breaking", however, its
technical debt is too heavy now. It's a good time to introduce a better
"queue" package.
# The new queue package
It keeps using old config and concept as much as possible.
* It only contains two major kinds of concepts:
* The "base queue": channel, levelqueue, redis
* They have the same abstraction, the same interface, and they are
tested by the same testing code.
* The "WokerPoolQueue", it uses the "base queue" to provide "worker
pool" function, calls the "handler" to process the data in the base
queue.
* The new code doesn't do "PushBack"
* Think about a queue with many workers, the "PushBack" can't guarantee
the order for re-queued unhandled items, so in new code it just does
"normal push"
* The new code doesn't do "pause/resume"
* The "pause/resume" was designed to handle some handler's failure: eg:
document indexer (elasticsearch) is down
* If a queue is paused for long time, either the producers blocks or the
new items are dropped.
* The new code doesn't do such "pause/resume" trick, it's not a common
queue's behavior and it doesn't help much.
* If there are unhandled items, the "push" function just blocks for a
few seconds and then re-queue them and retry.
* The new code doesn't do "worker booster"
* Gitea's queue's handlers are light functions, the cost is only the
go-routine, so it doesn't make sense to "boost" them.
* The new code only use "max worker number" to limit the concurrent
workers.
* The new "Push" never blocks forever
* Instead of creating more and more blocking goroutines, return an error
is more friendly to the server and to the end user.
There are more details in code comments: eg: the "Flush" problem, the
strange "code.index" hanging problem, the "immediate" queue problem.
Almost ready for review.
TODO:
* [x] add some necessary comments during review
* [x] add some more tests if necessary
* [x] update documents and config options
* [x] test max worker / active worker
* [x] re-run the CI tasks to see whether any test is flaky
* [x] improve the `handleOldLengthConfiguration` to provide more
friendly messages
* [x] fine tune default config values (eg: length?)
## Code coverage:
![image](https://user-images.githubusercontent.com/2114189/236620635-55576955-f95d-4810-b12f-879026a3afdf.png)
2023-05-08 19:49:59 +08:00
|
|
|
func handler(items ...[]*repo_module.PushUpdateOptions) [][]*repo_module.PushUpdateOptions {
|
|
|
|
for _, opts := range items {
|
2020-09-11 22:14:48 +08:00
|
|
|
if err := pushUpdates(opts); err != nil {
|
2023-11-30 08:46:51 +08:00
|
|
|
// Username and repository stays the same between items in opts.
|
|
|
|
pushUpdate := opts[0]
|
|
|
|
log.Error("pushUpdate[%s/%s] failed: %v", pushUpdate.RepoUserName, pushUpdate.RepoName, err)
|
2020-09-11 22:14:48 +08:00
|
|
|
}
|
|
|
|
}
|
2022-01-23 05:22:14 +08:00
|
|
|
return nil
|
2020-09-11 22:14:48 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
func initPushQueue() error {
|
Improve queue and logger context (#24924)
Before there was a "graceful function": RunWithShutdownFns, it's mainly
for some modules which doesn't support context.
The old queue system doesn't work well with context, so the old queues
need it.
After the queue refactoring, the new queue works with context well, so,
use Golang context as much as possible, the `RunWithShutdownFns` could
be removed (replaced by RunWithCancel for context cancel mechanism), the
related code could be simplified.
This PR also fixes some legacy queue-init problems, eg:
* typo : archiver: "unable to create codes indexer queue" => "unable to
create repo-archive queue"
* no nil check for failed queues, which causes unfriendly panic
After this PR, many goroutines could have better display name:
![image](https://github.com/go-gitea/gitea/assets/2114189/701b2a9b-8065-4137-aeaa-0bda2b34604a)
![image](https://github.com/go-gitea/gitea/assets/2114189/f1d5f50f-0534-40f0-b0be-f2c9daa5fe92)
2023-05-26 15:31:55 +08:00
|
|
|
pushQueue = queue.CreateSimpleQueue(graceful.GetManager().ShutdownContext(), "push_update", handler)
|
2020-09-11 22:14:48 +08:00
|
|
|
if pushQueue == nil {
|
Improve queue and logger context (#24924)
Before there was a "graceful function": RunWithShutdownFns, it's mainly
for some modules which doesn't support context.
The old queue system doesn't work well with context, so the old queues
need it.
After the queue refactoring, the new queue works with context well, so,
use Golang context as much as possible, the `RunWithShutdownFns` could
be removed (replaced by RunWithCancel for context cancel mechanism), the
related code could be simplified.
This PR also fixes some legacy queue-init problems, eg:
* typo : archiver: "unable to create codes indexer queue" => "unable to
create repo-archive queue"
* no nil check for failed queues, which causes unfriendly panic
After this PR, many goroutines could have better display name:
![image](https://github.com/go-gitea/gitea/assets/2114189/701b2a9b-8065-4137-aeaa-0bda2b34604a)
![image](https://github.com/go-gitea/gitea/assets/2114189/f1d5f50f-0534-40f0-b0be-f2c9daa5fe92)
2023-05-26 15:31:55 +08:00
|
|
|
return errors.New("unable to create push_update queue")
|
2020-09-11 22:14:48 +08:00
|
|
|
}
|
Improve queue and logger context (#24924)
Before there was a "graceful function": RunWithShutdownFns, it's mainly
for some modules which doesn't support context.
The old queue system doesn't work well with context, so the old queues
need it.
After the queue refactoring, the new queue works with context well, so,
use Golang context as much as possible, the `RunWithShutdownFns` could
be removed (replaced by RunWithCancel for context cancel mechanism), the
related code could be simplified.
This PR also fixes some legacy queue-init problems, eg:
* typo : archiver: "unable to create codes indexer queue" => "unable to
create repo-archive queue"
* no nil check for failed queues, which causes unfriendly panic
After this PR, many goroutines could have better display name:
![image](https://github.com/go-gitea/gitea/assets/2114189/701b2a9b-8065-4137-aeaa-0bda2b34604a)
![image](https://github.com/go-gitea/gitea/assets/2114189/f1d5f50f-0534-40f0-b0be-f2c9daa5fe92)
2023-05-26 15:31:55 +08:00
|
|
|
go graceful.GetManager().RunWithCancel(pushQueue)
|
2020-09-11 22:14:48 +08:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// PushUpdate is an alias of PushUpdates for single push update options
|
2020-10-31 05:59:02 +08:00
|
|
|
func PushUpdate(opts *repo_module.PushUpdateOptions) error {
|
|
|
|
return PushUpdates([]*repo_module.PushUpdateOptions{opts})
|
2020-09-11 22:14:48 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
// PushUpdates adds a push update to push queue
|
2020-10-31 05:59:02 +08:00
|
|
|
func PushUpdates(opts []*repo_module.PushUpdateOptions) error {
|
2020-09-11 22:14:48 +08:00
|
|
|
if len(opts) == 0 {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
for _, opt := range opts {
|
|
|
|
if opt.IsNewRef() && opt.IsDelRef() {
|
2023-12-14 05:02:00 +08:00
|
|
|
return fmt.Errorf("Old and new revisions are both NULL")
|
2020-09-11 22:14:48 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return pushQueue.Push(opts)
|
|
|
|
}
|
|
|
|
|
|
|
|
// pushUpdates generates push action history feeds for push updating multiple refs
|
2020-10-31 05:59:02 +08:00
|
|
|
func pushUpdates(optsList []*repo_module.PushUpdateOptions) error {
|
2020-09-11 22:14:48 +08:00
|
|
|
if len(optsList) == 0 {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2022-01-20 07:26:57 +08:00
|
|
|
ctx, _, finished := process.GetManager().AddContext(graceful.GetManager().HammerContext(), fmt.Sprintf("PushUpdates: %s/%s", optsList[0].RepoUserName, optsList[0].RepoName))
|
|
|
|
defer finished()
|
|
|
|
|
2022-12-03 10:48:26 +08:00
|
|
|
repo, err := repo_model.GetRepositoryByOwnerAndName(ctx, optsList[0].RepoUserName, optsList[0].RepoName)
|
2020-09-11 22:14:48 +08:00
|
|
|
if err != nil {
|
2022-10-25 03:29:17 +08:00
|
|
|
return fmt.Errorf("GetRepositoryByOwnerAndName failed: %w", err)
|
2020-09-11 22:14:48 +08:00
|
|
|
}
|
|
|
|
|
Simplify how git repositories are opened (#28937)
## Purpose
This is a refactor toward building an abstraction over managing git
repositories.
Afterwards, it does not matter anymore if they are stored on the local
disk or somewhere remote.
## What this PR changes
We used `git.OpenRepository` everywhere previously.
Now, we should split them into two distinct functions:
Firstly, there are temporary repositories which do not change:
```go
git.OpenRepository(ctx, diskPath)
```
Gitea managed repositories having a record in the database in the
`repository` table are moved into the new package `gitrepo`:
```go
gitrepo.OpenRepository(ctx, repo_model.Repo)
```
Why is `repo_model.Repository` the second parameter instead of file
path?
Because then we can easily adapt our repository storage strategy.
The repositories can be stored locally, however, they could just as well
be stored on a remote server.
## Further changes in other PRs
- A Git Command wrapper on package `gitrepo` could be created. i.e.
`NewCommand(ctx, repo_model.Repository, commands...)`. `git.RunOpts{Dir:
repo.RepoPath()}`, the directory should be empty before invoking this
method and it can be filled in the function only. #28940
- Remove the `RepoPath()`/`WikiPath()` functions to reduce the
possibility of mistakes.
---------
Co-authored-by: delvh <dev.lh@web.de>
2024-01-28 04:09:51 +08:00
|
|
|
gitRepo, err := gitrepo.OpenRepository(ctx, repo)
|
2020-09-11 22:14:48 +08:00
|
|
|
if err != nil {
|
Simplify how git repositories are opened (#28937)
## Purpose
This is a refactor toward building an abstraction over managing git
repositories.
Afterwards, it does not matter anymore if they are stored on the local
disk or somewhere remote.
## What this PR changes
We used `git.OpenRepository` everywhere previously.
Now, we should split them into two distinct functions:
Firstly, there are temporary repositories which do not change:
```go
git.OpenRepository(ctx, diskPath)
```
Gitea managed repositories having a record in the database in the
`repository` table are moved into the new package `gitrepo`:
```go
gitrepo.OpenRepository(ctx, repo_model.Repo)
```
Why is `repo_model.Repository` the second parameter instead of file
path?
Because then we can easily adapt our repository storage strategy.
The repositories can be stored locally, however, they could just as well
be stored on a remote server.
## Further changes in other PRs
- A Git Command wrapper on package `gitrepo` could be created. i.e.
`NewCommand(ctx, repo_model.Repository, commands...)`. `git.RunOpts{Dir:
repo.RepoPath()}`, the directory should be empty before invoking this
method and it can be filled in the function only. #28940
- Remove the `RepoPath()`/`WikiPath()` functions to reduce the
possibility of mistakes.
---------
Co-authored-by: delvh <dev.lh@web.de>
2024-01-28 04:09:51 +08:00
|
|
|
return fmt.Errorf("OpenRepository[%s]: %w", repo.FullName(), err)
|
2020-09-11 22:14:48 +08:00
|
|
|
}
|
|
|
|
defer gitRepo.Close()
|
|
|
|
|
2023-12-14 05:02:00 +08:00
|
|
|
objectFormat, err := gitRepo.GetObjectFormat()
|
|
|
|
if err != nil {
|
Simplify how git repositories are opened (#28937)
## Purpose
This is a refactor toward building an abstraction over managing git
repositories.
Afterwards, it does not matter anymore if they are stored on the local
disk or somewhere remote.
## What this PR changes
We used `git.OpenRepository` everywhere previously.
Now, we should split them into two distinct functions:
Firstly, there are temporary repositories which do not change:
```go
git.OpenRepository(ctx, diskPath)
```
Gitea managed repositories having a record in the database in the
`repository` table are moved into the new package `gitrepo`:
```go
gitrepo.OpenRepository(ctx, repo_model.Repo)
```
Why is `repo_model.Repository` the second parameter instead of file
path?
Because then we can easily adapt our repository storage strategy.
The repositories can be stored locally, however, they could just as well
be stored on a remote server.
## Further changes in other PRs
- A Git Command wrapper on package `gitrepo` could be created. i.e.
`NewCommand(ctx, repo_model.Repository, commands...)`. `git.RunOpts{Dir:
repo.RepoPath()}`, the directory should be empty before invoking this
method and it can be filled in the function only. #28940
- Remove the `RepoPath()`/`WikiPath()` functions to reduce the
possibility of mistakes.
---------
Co-authored-by: delvh <dev.lh@web.de>
2024-01-28 04:09:51 +08:00
|
|
|
return fmt.Errorf("unknown repository ObjectFormat [%s]: %w", repo.FullName(), err)
|
2023-12-14 05:02:00 +08:00
|
|
|
}
|
|
|
|
|
2022-06-06 16:01:49 +08:00
|
|
|
if err = repo_module.UpdateRepoSize(ctx, repo); err != nil {
|
2023-06-29 18:03:20 +08:00
|
|
|
return fmt.Errorf("Failed to update size for repository: %v", err)
|
2020-09-11 22:14:48 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
addTags := make([]string, 0, len(optsList))
|
|
|
|
delTags := make([]string, 0, len(optsList))
|
2021-11-24 17:49:20 +08:00
|
|
|
var pusher *user_model.User
|
2020-09-11 22:14:48 +08:00
|
|
|
|
|
|
|
for _, opts := range optsList {
|
2023-01-20 06:31:44 +08:00
|
|
|
log.Trace("pushUpdates: %-v %s %s %s", repo, opts.OldCommitID, opts.NewCommitID, opts.RefFullName)
|
|
|
|
|
2020-09-11 22:14:48 +08:00
|
|
|
if opts.IsNewRef() && opts.IsDelRef() {
|
2023-12-17 19:56:08 +08:00
|
|
|
return fmt.Errorf("old and new revisions are both %s", objectFormat.EmptyObjectID())
|
2020-09-11 22:14:48 +08:00
|
|
|
}
|
2023-05-26 09:04:48 +08:00
|
|
|
if opts.RefFullName.IsTag() {
|
2020-12-12 01:27:00 +08:00
|
|
|
if pusher == nil || pusher.ID != opts.PusherID {
|
Implement actions (#21937)
Close #13539.
Co-authored by: @lunny @appleboy @fuxiaohei and others.
Related projects:
- https://gitea.com/gitea/actions-proto-def
- https://gitea.com/gitea/actions-proto-go
- https://gitea.com/gitea/act
- https://gitea.com/gitea/act_runner
### Summary
The target of this PR is to bring a basic implementation of "Actions",
an internal CI/CD system of Gitea. That means even though it has been
merged, the state of the feature is **EXPERIMENTAL**, and please note
that:
- It is disabled by default;
- It shouldn't be used in a production environment currently;
- It shouldn't be used in a public Gitea instance currently;
- Breaking changes may be made before it's stable.
**Please comment on #13539 if you have any different product design
ideas**, all decisions reached there will be adopted here. But in this
PR, we don't talk about **naming, feature-creep or alternatives**.
### ⚠️ Breaking
`gitea-actions` will become a reserved user name. If a user with the
name already exists in the database, it is recommended to rename it.
### Some important reviews
- What is `DEFAULT_ACTIONS_URL` in `app.ini` for?
- https://github.com/go-gitea/gitea/pull/21937#discussion_r1055954954
- Why the api for runners is not under the normal `/api/v1` prefix?
- https://github.com/go-gitea/gitea/pull/21937#discussion_r1061173592
- Why DBFS?
- https://github.com/go-gitea/gitea/pull/21937#discussion_r1061301178
- Why ignore events triggered by `gitea-actions` bot?
- https://github.com/go-gitea/gitea/pull/21937#discussion_r1063254103
- Why there's no permission control for actions?
- https://github.com/go-gitea/gitea/pull/21937#discussion_r1090229868
### What it looks like
<details>
#### Manage runners
<img width="1792" alt="image"
src="https://user-images.githubusercontent.com/9418365/205870657-c72f590e-2e08-4cd4-be7f-2e0abb299bbf.png">
#### List runs
<img width="1792" alt="image"
src="https://user-images.githubusercontent.com/9418365/205872794-50fde990-2b45-48c1-a178-908e4ec5b627.png">
#### View logs
<img width="1792" alt="image"
src="https://user-images.githubusercontent.com/9418365/205872501-9b7b9000-9542-4991-8f55-18ccdada77c3.png">
</details>
### How to try it
<details>
#### 1. Start Gitea
Clone this branch and [install from
source](https://docs.gitea.io/en-us/install-from-source).
Add additional configurations in `app.ini` to enable Actions:
```ini
[actions]
ENABLED = true
```
Start it.
If all is well, you'll see the management page of runners:
<img width="1792" alt="image"
src="https://user-images.githubusercontent.com/9418365/205877365-8e30a780-9b10-4154-b3e8-ee6c3cb35a59.png">
#### 2. Start runner
Clone the [act_runner](https://gitea.com/gitea/act_runner), and follow
the
[README](https://gitea.com/gitea/act_runner/src/branch/main/README.md)
to start it.
If all is well, you'll see a new runner has been added:
<img width="1792" alt="image"
src="https://user-images.githubusercontent.com/9418365/205878000-216f5937-e696-470d-b66c-8473987d91c3.png">
#### 3. Enable actions for a repo
Create a new repo or open an existing one, check the `Actions` checkbox
in settings and submit.
<img width="1792" alt="image"
src="https://user-images.githubusercontent.com/9418365/205879705-53e09208-73c0-4b3e-a123-2dcf9aba4b9c.png">
<img width="1792" alt="image"
src="https://user-images.githubusercontent.com/9418365/205879383-23f3d08f-1a85-41dd-a8b3-54e2ee6453e8.png">
If all is well, you'll see a new tab "Actions":
<img width="1792" alt="image"
src="https://user-images.githubusercontent.com/9418365/205881648-a8072d8c-5803-4d76-b8a8-9b2fb49516c1.png">
#### 4. Upload workflow files
Upload some workflow files to `.gitea/workflows/xxx.yaml`, you can
follow the [quickstart](https://docs.github.com/en/actions/quickstart)
of GitHub Actions. Yes, Gitea Actions is compatible with GitHub Actions
in most cases, you can use the same demo:
```yaml
name: GitHub Actions Demo
run-name: ${{ github.actor }} is testing out GitHub Actions 🚀
on: [push]
jobs:
Explore-GitHub-Actions:
runs-on: ubuntu-latest
steps:
- run: echo "🎉 The job was automatically triggered by a ${{ github.event_name }} event."
- run: echo "🐧 This job is now running on a ${{ runner.os }} server hosted by GitHub!"
- run: echo "🔎 The name of your branch is ${{ github.ref }} and your repository is ${{ github.repository }}."
- name: Check out repository code
uses: actions/checkout@v3
- run: echo "💡 The ${{ github.repository }} repository has been cloned to the runner."
- run: echo "🖥️ The workflow is now ready to test your code on the runner."
- name: List files in the repository
run: |
ls ${{ github.workspace }}
- run: echo "🍏 This job's status is ${{ job.status }}."
```
If all is well, you'll see a new run in `Actions` tab:
<img width="1792" alt="image"
src="https://user-images.githubusercontent.com/9418365/205884473-79a874bc-171b-4aaf-acd5-0241a45c3b53.png">
#### 5. Check the logs of jobs
Click a run and you'll see the logs:
<img width="1792" alt="image"
src="https://user-images.githubusercontent.com/9418365/205884800-994b0374-67f7-48ff-be9a-4c53f3141547.png">
#### 6. Go on
You can try more examples in [the
documents](https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions)
of GitHub Actions, then you might find a lot of bugs.
Come on, PRs are welcome.
</details>
See also: [Feature Preview: Gitea
Actions](https://blog.gitea.io/2022/12/feature-preview-gitea-actions/)
---------
Co-authored-by: a1012112796 <1012112796@qq.com>
Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>
Co-authored-by: delvh <dev.lh@web.de>
Co-authored-by: ChristopherHX <christopher.homberger@web.de>
Co-authored-by: John Olheiser <john.olheiser@gmail.com>
2023-01-31 09:45:19 +08:00
|
|
|
if opts.PusherID == user_model.ActionsUserID {
|
|
|
|
pusher = user_model.NewActionsUser()
|
|
|
|
} else {
|
|
|
|
var err error
|
|
|
|
if pusher, err = user_model.GetUserByID(ctx, opts.PusherID); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2020-12-12 01:27:00 +08:00
|
|
|
}
|
|
|
|
}
|
2023-05-26 09:04:48 +08:00
|
|
|
tagName := opts.RefFullName.TagName()
|
2020-09-11 22:14:48 +08:00
|
|
|
if opts.IsDelRef() {
|
2023-09-06 02:37:47 +08:00
|
|
|
notify_service.PushCommits(
|
2023-03-01 06:17:51 +08:00
|
|
|
ctx, pusher, repo,
|
2020-12-16 20:41:21 +08:00
|
|
|
&repo_module.PushUpdateOptions{
|
2023-05-26 09:04:48 +08:00
|
|
|
RefFullName: git.RefNameFromTag(tagName),
|
2020-12-16 20:41:21 +08:00
|
|
|
OldCommitID: opts.OldCommitID,
|
2023-12-17 19:56:08 +08:00
|
|
|
NewCommitID: objectFormat.EmptyObjectID().String(),
|
2020-12-16 20:41:21 +08:00
|
|
|
}, repo_module.NewPushCommits())
|
|
|
|
|
2020-09-11 22:14:48 +08:00
|
|
|
delTags = append(delTags, tagName)
|
2023-09-06 02:37:47 +08:00
|
|
|
notify_service.DeleteRef(ctx, pusher, repo, opts.RefFullName)
|
2020-09-11 22:14:48 +08:00
|
|
|
} else { // is new tag
|
2021-12-29 19:40:57 +08:00
|
|
|
newCommit, err := gitRepo.GetCommit(opts.NewCommitID)
|
|
|
|
if err != nil {
|
2023-01-20 06:31:44 +08:00
|
|
|
return fmt.Errorf("gitRepo.GetCommit(%s) in %s/%s[%d]: %w", opts.NewCommitID, repo.OwnerName, repo.Name, repo.ID, err)
|
2021-12-29 19:40:57 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
commits := repo_module.NewPushCommits()
|
|
|
|
commits.HeadCommit = repo_module.CommitToPushCommit(newCommit)
|
2023-12-17 19:56:08 +08:00
|
|
|
commits.CompareURL = repo.ComposeCompareURL(objectFormat.EmptyObjectID().String(), opts.NewCommitID)
|
2021-12-29 19:40:57 +08:00
|
|
|
|
2023-09-06 02:37:47 +08:00
|
|
|
notify_service.PushCommits(
|
2023-03-01 06:17:51 +08:00
|
|
|
ctx, pusher, repo,
|
2020-12-16 20:41:21 +08:00
|
|
|
&repo_module.PushUpdateOptions{
|
2023-05-26 09:04:48 +08:00
|
|
|
RefFullName: opts.RefFullName,
|
2023-12-17 19:56:08 +08:00
|
|
|
OldCommitID: objectFormat.EmptyObjectID().String(),
|
2020-12-16 20:41:21 +08:00
|
|
|
NewCommitID: opts.NewCommitID,
|
2021-12-29 19:40:57 +08:00
|
|
|
}, commits)
|
2020-12-16 20:41:21 +08:00
|
|
|
|
2020-09-11 22:14:48 +08:00
|
|
|
addTags = append(addTags, tagName)
|
2023-09-06 02:37:47 +08:00
|
|
|
notify_service.CreateRef(ctx, pusher, repo, opts.RefFullName, opts.NewCommitID)
|
2020-09-11 22:14:48 +08:00
|
|
|
}
|
2023-05-26 09:04:48 +08:00
|
|
|
} else if opts.RefFullName.IsBranch() {
|
2020-09-11 22:14:48 +08:00
|
|
|
if pusher == nil || pusher.ID != opts.PusherID {
|
Implement actions (#21937)
Close #13539.
Co-authored by: @lunny @appleboy @fuxiaohei and others.
Related projects:
- https://gitea.com/gitea/actions-proto-def
- https://gitea.com/gitea/actions-proto-go
- https://gitea.com/gitea/act
- https://gitea.com/gitea/act_runner
### Summary
The target of this PR is to bring a basic implementation of "Actions",
an internal CI/CD system of Gitea. That means even though it has been
merged, the state of the feature is **EXPERIMENTAL**, and please note
that:
- It is disabled by default;
- It shouldn't be used in a production environment currently;
- It shouldn't be used in a public Gitea instance currently;
- Breaking changes may be made before it's stable.
**Please comment on #13539 if you have any different product design
ideas**, all decisions reached there will be adopted here. But in this
PR, we don't talk about **naming, feature-creep or alternatives**.
### ⚠️ Breaking
`gitea-actions` will become a reserved user name. If a user with the
name already exists in the database, it is recommended to rename it.
### Some important reviews
- What is `DEFAULT_ACTIONS_URL` in `app.ini` for?
- https://github.com/go-gitea/gitea/pull/21937#discussion_r1055954954
- Why the api for runners is not under the normal `/api/v1` prefix?
- https://github.com/go-gitea/gitea/pull/21937#discussion_r1061173592
- Why DBFS?
- https://github.com/go-gitea/gitea/pull/21937#discussion_r1061301178
- Why ignore events triggered by `gitea-actions` bot?
- https://github.com/go-gitea/gitea/pull/21937#discussion_r1063254103
- Why there's no permission control for actions?
- https://github.com/go-gitea/gitea/pull/21937#discussion_r1090229868
### What it looks like
<details>
#### Manage runners
<img width="1792" alt="image"
src="https://user-images.githubusercontent.com/9418365/205870657-c72f590e-2e08-4cd4-be7f-2e0abb299bbf.png">
#### List runs
<img width="1792" alt="image"
src="https://user-images.githubusercontent.com/9418365/205872794-50fde990-2b45-48c1-a178-908e4ec5b627.png">
#### View logs
<img width="1792" alt="image"
src="https://user-images.githubusercontent.com/9418365/205872501-9b7b9000-9542-4991-8f55-18ccdada77c3.png">
</details>
### How to try it
<details>
#### 1. Start Gitea
Clone this branch and [install from
source](https://docs.gitea.io/en-us/install-from-source).
Add additional configurations in `app.ini` to enable Actions:
```ini
[actions]
ENABLED = true
```
Start it.
If all is well, you'll see the management page of runners:
<img width="1792" alt="image"
src="https://user-images.githubusercontent.com/9418365/205877365-8e30a780-9b10-4154-b3e8-ee6c3cb35a59.png">
#### 2. Start runner
Clone the [act_runner](https://gitea.com/gitea/act_runner), and follow
the
[README](https://gitea.com/gitea/act_runner/src/branch/main/README.md)
to start it.
If all is well, you'll see a new runner has been added:
<img width="1792" alt="image"
src="https://user-images.githubusercontent.com/9418365/205878000-216f5937-e696-470d-b66c-8473987d91c3.png">
#### 3. Enable actions for a repo
Create a new repo or open an existing one, check the `Actions` checkbox
in settings and submit.
<img width="1792" alt="image"
src="https://user-images.githubusercontent.com/9418365/205879705-53e09208-73c0-4b3e-a123-2dcf9aba4b9c.png">
<img width="1792" alt="image"
src="https://user-images.githubusercontent.com/9418365/205879383-23f3d08f-1a85-41dd-a8b3-54e2ee6453e8.png">
If all is well, you'll see a new tab "Actions":
<img width="1792" alt="image"
src="https://user-images.githubusercontent.com/9418365/205881648-a8072d8c-5803-4d76-b8a8-9b2fb49516c1.png">
#### 4. Upload workflow files
Upload some workflow files to `.gitea/workflows/xxx.yaml`, you can
follow the [quickstart](https://docs.github.com/en/actions/quickstart)
of GitHub Actions. Yes, Gitea Actions is compatible with GitHub Actions
in most cases, you can use the same demo:
```yaml
name: GitHub Actions Demo
run-name: ${{ github.actor }} is testing out GitHub Actions 🚀
on: [push]
jobs:
Explore-GitHub-Actions:
runs-on: ubuntu-latest
steps:
- run: echo "🎉 The job was automatically triggered by a ${{ github.event_name }} event."
- run: echo "🐧 This job is now running on a ${{ runner.os }} server hosted by GitHub!"
- run: echo "🔎 The name of your branch is ${{ github.ref }} and your repository is ${{ github.repository }}."
- name: Check out repository code
uses: actions/checkout@v3
- run: echo "💡 The ${{ github.repository }} repository has been cloned to the runner."
- run: echo "🖥️ The workflow is now ready to test your code on the runner."
- name: List files in the repository
run: |
ls ${{ github.workspace }}
- run: echo "🍏 This job's status is ${{ job.status }}."
```
If all is well, you'll see a new run in `Actions` tab:
<img width="1792" alt="image"
src="https://user-images.githubusercontent.com/9418365/205884473-79a874bc-171b-4aaf-acd5-0241a45c3b53.png">
#### 5. Check the logs of jobs
Click a run and you'll see the logs:
<img width="1792" alt="image"
src="https://user-images.githubusercontent.com/9418365/205884800-994b0374-67f7-48ff-be9a-4c53f3141547.png">
#### 6. Go on
You can try more examples in [the
documents](https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions)
of GitHub Actions, then you might find a lot of bugs.
Come on, PRs are welcome.
</details>
See also: [Feature Preview: Gitea
Actions](https://blog.gitea.io/2022/12/feature-preview-gitea-actions/)
---------
Co-authored-by: a1012112796 <1012112796@qq.com>
Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>
Co-authored-by: delvh <dev.lh@web.de>
Co-authored-by: ChristopherHX <christopher.homberger@web.de>
Co-authored-by: John Olheiser <john.olheiser@gmail.com>
2023-01-31 09:45:19 +08:00
|
|
|
if opts.PusherID == user_model.ActionsUserID {
|
|
|
|
pusher = user_model.NewActionsUser()
|
|
|
|
} else {
|
|
|
|
var err error
|
|
|
|
if pusher, err = user_model.GetUserByID(ctx, opts.PusherID); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2020-09-11 22:14:48 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-05-26 09:04:48 +08:00
|
|
|
branch := opts.RefFullName.BranchName()
|
2020-09-11 22:14:48 +08:00
|
|
|
if !opts.IsDelRef() {
|
2020-10-09 06:17:23 +08:00
|
|
|
log.Trace("TriggerTask '%s/%s' by %s", repo.Name, branch, pusher.Name)
|
|
|
|
go pull_service.AddTestPullRequestTask(pusher, repo.ID, branch, true, opts.OldCommitID, opts.NewCommitID)
|
2020-09-11 22:14:48 +08:00
|
|
|
|
|
|
|
newCommit, err := gitRepo.GetCommit(opts.NewCommitID)
|
|
|
|
if err != nil {
|
2023-01-20 06:31:44 +08:00
|
|
|
return fmt.Errorf("gitRepo.GetCommit(%s) in %s/%s[%d]: %w", opts.NewCommitID, repo.OwnerName, repo.Name, repo.ID, err)
|
2020-09-11 22:14:48 +08:00
|
|
|
}
|
|
|
|
|
2020-12-08 10:23:18 +08:00
|
|
|
refName := opts.RefName()
|
|
|
|
|
2020-09-11 22:14:48 +08:00
|
|
|
// Push new branch.
|
2021-08-10 02:08:51 +08:00
|
|
|
var l []*git.Commit
|
2020-09-11 22:14:48 +08:00
|
|
|
if opts.IsNewRef() {
|
2020-12-08 10:23:18 +08:00
|
|
|
if repo.IsEmpty { // Change default branch and empty status only if pushed ref is non-empty branch.
|
|
|
|
repo.DefaultBranch = refName
|
|
|
|
repo.IsEmpty = false
|
|
|
|
if repo.DefaultBranch != setting.Repository.DefaultBranch {
|
|
|
|
if err := gitRepo.SetDefaultBranch(repo.DefaultBranch); err != nil {
|
|
|
|
if !git.IsErrUnsupportedVersion(err) {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
// Update the is empty and default_branch columns
|
2023-03-01 06:17:51 +08:00
|
|
|
if err := repo_model.UpdateRepositoryCols(ctx, repo, "default_branch", "is_empty"); err != nil {
|
2022-10-25 03:29:17 +08:00
|
|
|
return fmt.Errorf("UpdateRepositoryCols: %w", err)
|
2020-12-08 10:23:18 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-09-11 22:14:48 +08:00
|
|
|
l, err = newCommit.CommitsBeforeLimit(10)
|
|
|
|
if err != nil {
|
2022-10-25 03:29:17 +08:00
|
|
|
return fmt.Errorf("newCommit.CommitsBeforeLimit: %w", err)
|
2020-09-11 22:14:48 +08:00
|
|
|
}
|
2023-09-06 02:37:47 +08:00
|
|
|
notify_service.CreateRef(ctx, pusher, repo, opts.RefFullName, opts.NewCommitID)
|
2020-09-11 22:14:48 +08:00
|
|
|
} else {
|
|
|
|
l, err = newCommit.CommitsBeforeUntil(opts.OldCommitID)
|
|
|
|
if err != nil {
|
2022-10-25 03:29:17 +08:00
|
|
|
return fmt.Errorf("newCommit.CommitsBeforeUntil: %w", err)
|
2020-09-11 22:14:48 +08:00
|
|
|
}
|
2020-10-09 06:17:23 +08:00
|
|
|
|
2023-03-10 02:14:22 +08:00
|
|
|
isForcePush, err := newCommit.IsForcePush(opts.OldCommitID)
|
2020-10-09 06:17:23 +08:00
|
|
|
if err != nil {
|
2023-03-10 02:14:22 +08:00
|
|
|
log.Error("IsForcePush %s:%s failed: %v", repo.FullName(), branch, err)
|
2020-10-09 06:17:23 +08:00
|
|
|
}
|
|
|
|
|
2023-03-10 02:14:22 +08:00
|
|
|
if isForcePush {
|
2020-10-09 06:17:23 +08:00
|
|
|
log.Trace("Push %s is a force push", opts.NewCommitID)
|
|
|
|
|
|
|
|
cache.Remove(repo.GetCommitsCountCacheKey(opts.RefName(), true))
|
|
|
|
} else {
|
|
|
|
// TODO: increment update the commit count cache but not remove
|
|
|
|
cache.Remove(repo.GetCommitsCountCacheKey(opts.RefName(), true))
|
|
|
|
}
|
2020-09-11 22:14:48 +08:00
|
|
|
}
|
|
|
|
|
2021-08-10 02:08:51 +08:00
|
|
|
commits := repo_module.GitToPushCommits(l)
|
2021-06-29 21:34:03 +08:00
|
|
|
commits.HeadCommit = repo_module.CommitToPushCommit(newCommit)
|
2021-06-26 17:13:51 +08:00
|
|
|
|
2023-07-22 22:14:27 +08:00
|
|
|
if err := issue_service.UpdateIssuesCommit(ctx, pusher, repo, commits.Commits, refName); err != nil {
|
2021-06-26 17:13:51 +08:00
|
|
|
log.Error("updateIssuesCommit: %v", err)
|
|
|
|
}
|
|
|
|
|
2022-03-23 21:40:12 +08:00
|
|
|
oldCommitID := opts.OldCommitID
|
2023-12-17 19:56:08 +08:00
|
|
|
if oldCommitID == objectFormat.EmptyObjectID().String() && len(commits.Commits) > 0 {
|
2022-03-23 21:40:12 +08:00
|
|
|
oldCommit, err := gitRepo.GetCommit(commits.Commits[len(commits.Commits)-1].Sha1)
|
|
|
|
if err != nil && !git.IsErrNotExist(err) {
|
|
|
|
log.Error("unable to GetCommit %s from %-v: %v", oldCommitID, repo, err)
|
|
|
|
}
|
|
|
|
if oldCommit != nil {
|
|
|
|
for i := 0; i < oldCommit.ParentCount(); i++ {
|
|
|
|
commitID, _ := oldCommit.ParentID(i)
|
|
|
|
if !commitID.IsZero() {
|
|
|
|
oldCommitID = commitID.String()
|
|
|
|
break
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-12-17 19:56:08 +08:00
|
|
|
if oldCommitID == objectFormat.EmptyObjectID().String() && repo.DefaultBranch != branch {
|
2022-03-23 21:40:12 +08:00
|
|
|
oldCommitID = repo.DefaultBranch
|
|
|
|
}
|
|
|
|
|
2023-12-17 19:56:08 +08:00
|
|
|
if oldCommitID != objectFormat.EmptyObjectID().String() {
|
2022-03-23 21:40:12 +08:00
|
|
|
commits.CompareURL = repo.ComposeCompareURL(oldCommitID, opts.NewCommitID)
|
|
|
|
} else {
|
|
|
|
commits.CompareURL = ""
|
|
|
|
}
|
|
|
|
|
2022-10-17 12:58:20 +08:00
|
|
|
if len(commits.Commits) > setting.UI.FeedMaxCommitNum {
|
|
|
|
commits.Commits = commits.Commits[:setting.UI.FeedMaxCommitNum]
|
|
|
|
}
|
|
|
|
|
Also sync DB branches on push if necessary (#28361)
Fix #28056
This PR will check whether the repo has zero branch when pushing a
branch. If that, it means this repository hasn't been synced.
The reason caused that is after user upgrade from v1.20 -> v1.21, he
just push branches without visit the repository user interface. Because
all repositories routers will check whether a branches sync is necessary
but push has not such check.
For every repository, it has two states, synced or not synced. If there
is zero branch for a repository, then it will be assumed as non-sync
state. Otherwise, it's synced state. So if we think it's synced, we just
need to update branch/insert new branch. Otherwise do a full sync. So
that, for every push, there will be almost no extra load added. It's
high performance than yours.
For the implementation, we in fact will try to update the branch first,
if updated success with affect records > 0, then all are done. Because
that means the branch has been in the database. If no record is
affected, that means the branch does not exist in database. So there are
two possibilities. One is this is a new branch, then we just need to
insert the record. Another is the branches haven't been synced, then we
need to sync all the branches into database.
2023-12-09 21:30:56 +08:00
|
|
|
if err = syncBranchToDB(ctx, repo.ID, opts.PusherID, branch, newCommit); err != nil {
|
2023-06-29 18:03:20 +08:00
|
|
|
return fmt.Errorf("git_model.UpdateBranch %s:%s failed: %v", repo.FullName(), branch, err)
|
2020-09-11 22:14:48 +08:00
|
|
|
}
|
|
|
|
|
2023-09-06 02:37:47 +08:00
|
|
|
notify_service.PushCommits(ctx, pusher, repo, opts, commits)
|
2023-07-20 15:18:52 +08:00
|
|
|
|
2020-10-09 06:17:23 +08:00
|
|
|
// Cache for big repository
|
2021-11-17 23:17:31 +08:00
|
|
|
if err := CacheRef(graceful.GetManager().HammerContext(), repo, gitRepo, opts.RefFullName); err != nil {
|
2020-10-09 06:17:23 +08:00
|
|
|
log.Error("repo_module.CacheRef %s/%s failed: %v", repo.ID, branch, err)
|
|
|
|
}
|
2020-12-08 10:23:18 +08:00
|
|
|
} else {
|
2023-09-06 02:37:47 +08:00
|
|
|
notify_service.DeleteRef(ctx, pusher, repo, opts.RefFullName)
|
2023-07-22 22:14:27 +08:00
|
|
|
if err = pull_service.CloseBranchPulls(ctx, pusher, repo.ID, branch); err != nil {
|
2020-12-08 10:23:18 +08:00
|
|
|
// close all related pulls
|
|
|
|
log.Error("close related pull request failed: %v", err)
|
|
|
|
}
|
2023-06-29 18:03:20 +08:00
|
|
|
|
2023-07-22 22:14:27 +08:00
|
|
|
if err := git_model.AddDeletedBranch(ctx, repo.ID, branch, pusher.ID); err != nil {
|
2023-06-29 18:03:20 +08:00
|
|
|
return fmt.Errorf("AddDeletedBranch %s:%s failed: %v", repo.FullName(), branch, err)
|
2023-06-08 18:10:59 +08:00
|
|
|
}
|
2020-09-11 22:14:48 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
// Even if user delete a branch on a repository which he didn't watch, he will be watch that.
|
2023-03-01 06:17:51 +08:00
|
|
|
if err = repo_model.WatchIfAuto(ctx, opts.PusherID, repo.ID, true); err != nil {
|
2020-09-11 22:14:48 +08:00
|
|
|
log.Warn("Fail to perform auto watch on user %v for repo %v: %v", opts.PusherID, repo.ID, err)
|
|
|
|
}
|
2020-12-08 10:23:18 +08:00
|
|
|
} else {
|
|
|
|
log.Trace("Non-tag and non-branch commits pushed.")
|
2020-09-11 22:14:48 +08:00
|
|
|
}
|
|
|
|
}
|
2023-03-01 06:17:51 +08:00
|
|
|
if err := PushUpdateAddDeleteTags(ctx, repo, gitRepo, addTags, delTags); err != nil {
|
2022-10-25 03:29:17 +08:00
|
|
|
return fmt.Errorf("PushUpdateAddDeleteTags: %w", err)
|
2020-09-11 22:14:48 +08:00
|
|
|
}
|
|
|
|
|
2020-09-21 01:55:15 +08:00
|
|
|
// Change repository last updated time.
|
2023-09-16 22:39:12 +08:00
|
|
|
if err := repo_model.UpdateRepositoryUpdatedTime(ctx, repo.ID, time.Now()); err != nil {
|
2022-10-25 03:29:17 +08:00
|
|
|
return fmt.Errorf("UpdateRepositoryUpdatedTime: %w", err)
|
2020-09-11 22:14:48 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
2021-11-17 23:17:31 +08:00
|
|
|
|
|
|
|
// PushUpdateAddDeleteTags updates a number of added and delete tags
|
2023-03-01 06:17:51 +08:00
|
|
|
func PushUpdateAddDeleteTags(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository, addTags, delTags []string) error {
|
|
|
|
return db.WithTx(ctx, func(ctx context.Context) error {
|
2022-08-25 10:31:57 +08:00
|
|
|
if err := repo_model.PushUpdateDeleteTagsContext(ctx, repo, delTags); err != nil {
|
2021-11-17 23:17:31 +08:00
|
|
|
return err
|
|
|
|
}
|
|
|
|
return pushUpdateAddTags(ctx, repo, gitRepo, addTags)
|
|
|
|
})
|
|
|
|
}
|
|
|
|
|
|
|
|
// pushUpdateAddTags updates a number of add tags
|
2021-12-10 09:27:50 +08:00
|
|
|
func pushUpdateAddTags(ctx context.Context, repo *repo_model.Repository, gitRepo *git.Repository, tags []string) error {
|
2021-11-17 23:17:31 +08:00
|
|
|
if len(tags) == 0 {
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2024-01-15 10:19:25 +08:00
|
|
|
releases, err := db.Find[repo_model.Release](ctx, repo_model.FindReleasesOptions{
|
|
|
|
RepoID: repo.ID,
|
2024-02-19 17:31:36 +08:00
|
|
|
TagNames: tags,
|
2024-01-15 10:19:25 +08:00
|
|
|
})
|
2021-11-17 23:17:31 +08:00
|
|
|
if err != nil {
|
2024-01-15 10:19:25 +08:00
|
|
|
return fmt.Errorf("db.Find[repo_model.Release]: %w", err)
|
2021-11-17 23:17:31 +08:00
|
|
|
}
|
2022-08-25 10:31:57 +08:00
|
|
|
relMap := make(map[string]*repo_model.Release)
|
2021-11-17 23:17:31 +08:00
|
|
|
for _, rel := range releases {
|
|
|
|
relMap[rel.LowerTagName] = rel
|
|
|
|
}
|
|
|
|
|
2024-02-19 17:31:36 +08:00
|
|
|
lowerTags := make([]string, 0, len(tags))
|
|
|
|
for _, tag := range tags {
|
|
|
|
lowerTags = append(lowerTags, strings.ToLower(tag))
|
|
|
|
}
|
|
|
|
|
2022-08-25 10:31:57 +08:00
|
|
|
newReleases := make([]*repo_model.Release, 0, len(lowerTags)-len(relMap))
|
2021-11-17 23:17:31 +08:00
|
|
|
|
2021-11-24 17:49:20 +08:00
|
|
|
emailToUser := make(map[string]*user_model.User)
|
2021-11-17 23:17:31 +08:00
|
|
|
|
|
|
|
for i, lowerTag := range lowerTags {
|
|
|
|
tag, err := gitRepo.GetTag(tags[i])
|
|
|
|
if err != nil {
|
2022-10-25 03:29:17 +08:00
|
|
|
return fmt.Errorf("GetTag: %w", err)
|
2021-11-17 23:17:31 +08:00
|
|
|
}
|
2022-01-13 04:37:46 +08:00
|
|
|
commit, err := tag.Commit(gitRepo)
|
2021-11-17 23:17:31 +08:00
|
|
|
if err != nil {
|
2022-10-25 03:29:17 +08:00
|
|
|
return fmt.Errorf("Commit: %w", err)
|
2021-11-17 23:17:31 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
sig := tag.Tagger
|
|
|
|
if sig == nil {
|
|
|
|
sig = commit.Author
|
|
|
|
}
|
|
|
|
if sig == nil {
|
|
|
|
sig = commit.Committer
|
|
|
|
}
|
2021-11-24 17:49:20 +08:00
|
|
|
var author *user_model.User
|
2022-01-21 01:46:10 +08:00
|
|
|
createdAt := time.Unix(1, 0)
|
2021-11-17 23:17:31 +08:00
|
|
|
|
|
|
|
if sig != nil {
|
|
|
|
var ok bool
|
|
|
|
author, ok = emailToUser[sig.Email]
|
|
|
|
if !ok {
|
Add context cache as a request level cache (#22294)
To avoid duplicated load of the same data in an HTTP request, we can set
a context cache to do that. i.e. Some pages may load a user from a
database with the same id in different areas on the same page. But the
code is hidden in two different deep logic. How should we share the
user? As a result of this PR, now if both entry functions accept
`context.Context` as the first parameter and we just need to refactor
`GetUserByID` to reuse the user from the context cache. Then it will not
be loaded twice on an HTTP request.
But of course, sometimes we would like to reload an object from the
database, that's why `RemoveContextData` is also exposed.
The core context cache is here. It defines a new context
```go
type cacheContext struct {
ctx context.Context
data map[any]map[any]any
lock sync.RWMutex
}
var cacheContextKey = struct{}{}
func WithCacheContext(ctx context.Context) context.Context {
return context.WithValue(ctx, cacheContextKey, &cacheContext{
ctx: ctx,
data: make(map[any]map[any]any),
})
}
```
Then you can use the below 4 methods to read/write/del the data within
the same context.
```go
func GetContextData(ctx context.Context, tp, key any) any
func SetContextData(ctx context.Context, tp, key, value any)
func RemoveContextData(ctx context.Context, tp, key any)
func GetWithContextCache[T any](ctx context.Context, cacheGroupKey string, cacheTargetID any, f func() (T, error)) (T, error)
```
Then let's take a look at how `system.GetString` implement it.
```go
func GetSetting(ctx context.Context, key string) (string, error) {
return cache.GetWithContextCache(ctx, contextCacheKey, key, func() (string, error) {
return cache.GetString(genSettingCacheKey(key), func() (string, error) {
res, err := GetSettingNoCache(ctx, key)
if err != nil {
return "", err
}
return res.SettingValue, nil
})
})
}
```
First, it will check if context data include the setting object with the
key. If not, it will query from the global cache which may be memory or
a Redis cache. If not, it will get the object from the database. In the
end, if the object gets from the global cache or database, it will be
set into the context cache.
An object stored in the context cache will only be destroyed after the
context disappeared.
2023-02-15 21:37:34 +08:00
|
|
|
author, err = user_model.GetUserByEmail(ctx, sig.Email)
|
2021-11-24 17:49:20 +08:00
|
|
|
if err != nil && !user_model.IsErrUserNotExist(err) {
|
2022-10-25 03:29:17 +08:00
|
|
|
return fmt.Errorf("GetUserByEmail: %w", err)
|
2021-11-17 23:17:31 +08:00
|
|
|
}
|
|
|
|
if author != nil {
|
|
|
|
emailToUser[sig.Email] = author
|
|
|
|
}
|
|
|
|
}
|
|
|
|
createdAt = sig.When
|
|
|
|
}
|
|
|
|
|
|
|
|
commitsCount, err := commit.CommitsCount()
|
|
|
|
if err != nil {
|
2022-10-25 03:29:17 +08:00
|
|
|
return fmt.Errorf("CommitsCount: %w", err)
|
2021-11-17 23:17:31 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
rel, has := relMap[lowerTag]
|
|
|
|
|
|
|
|
if !has {
|
2023-03-27 21:41:33 +08:00
|
|
|
parts := strings.SplitN(tag.Message, "\n", 2)
|
|
|
|
note := ""
|
|
|
|
if len(parts) > 1 {
|
|
|
|
note = parts[1]
|
|
|
|
}
|
2022-08-25 10:31:57 +08:00
|
|
|
rel = &repo_model.Release{
|
2021-11-17 23:17:31 +08:00
|
|
|
RepoID: repo.ID,
|
2023-03-27 21:41:33 +08:00
|
|
|
Title: parts[0],
|
2021-11-17 23:17:31 +08:00
|
|
|
TagName: tags[i],
|
|
|
|
LowerTagName: lowerTag,
|
|
|
|
Target: "",
|
|
|
|
Sha1: commit.ID.String(),
|
|
|
|
NumCommits: commitsCount,
|
2023-03-27 21:41:33 +08:00
|
|
|
Note: note,
|
2021-11-17 23:17:31 +08:00
|
|
|
IsDraft: false,
|
|
|
|
IsPrerelease: false,
|
|
|
|
IsTag: true,
|
|
|
|
CreatedUnix: timeutil.TimeStamp(createdAt.Unix()),
|
|
|
|
}
|
|
|
|
if author != nil {
|
|
|
|
rel.PublisherID = author.ID
|
|
|
|
}
|
|
|
|
|
|
|
|
newReleases = append(newReleases, rel)
|
|
|
|
} else {
|
|
|
|
rel.Sha1 = commit.ID.String()
|
|
|
|
rel.CreatedUnix = timeutil.TimeStamp(createdAt.Unix())
|
|
|
|
rel.NumCommits = commitsCount
|
|
|
|
rel.IsDraft = false
|
|
|
|
if rel.IsTag && author != nil {
|
|
|
|
rel.PublisherID = author.ID
|
|
|
|
}
|
2022-08-25 10:31:57 +08:00
|
|
|
if err = repo_model.UpdateRelease(ctx, rel); err != nil {
|
2022-10-25 03:29:17 +08:00
|
|
|
return fmt.Errorf("Update: %w", err)
|
2021-11-17 23:17:31 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if len(newReleases) > 0 {
|
2022-06-03 14:13:58 +08:00
|
|
|
if err = db.Insert(ctx, newReleases); err != nil {
|
2022-10-25 03:29:17 +08:00
|
|
|
return fmt.Errorf("Insert: %w", err)
|
2021-11-17 23:17:31 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|