mirror of https://codeberg.org/forgejo/forgejo
Rewrite queue (#24505)
# ⚠️ Breaking Many deprecated queue config options are removed (actually, they should have been removed in 1.18/1.19). If you see the fatal message when starting Gitea: "Please update your app.ini to remove deprecated config options", please follow the error messages to remove these options from your app.ini. Example: ``` 2023/05/06 19:39:22 [E] Removed queue option: `[indexer].ISSUE_INDEXER_QUEUE_TYPE`. Use new options in `[queue.issue_indexer]` 2023/05/06 19:39:22 [E] Removed queue option: `[indexer].UPDATE_BUFFER_LEN`. Use new options in `[queue.issue_indexer]` 2023/05/06 19:39:22 [F] Please update your app.ini to remove deprecated config options ``` Many options in `[queue]` are are dropped, including: `WRAP_IF_NECESSARY`, `MAX_ATTEMPTS`, `TIMEOUT`, `WORKERS`, `BLOCK_TIMEOUT`, `BOOST_TIMEOUT`, `BOOST_WORKERS`, they can be removed from app.ini. # The problem The old queue package has some legacy problems: * complexity: I doubt few people could tell how it works. * maintainability: Too many channels and mutex/cond are mixed together, too many different structs/interfaces depends each other. * stability: due to the complexity & maintainability, sometimes there are strange bugs and difficult to debug, and some code doesn't have test (indeed some code is difficult to test because a lot of things are mixed together). * general applicability: although it is called "queue", its behavior is not a well-known queue. * scalability: it doesn't seem easy to make it work with a cluster without breaking its behaviors. It came from some very old code to "avoid breaking", however, its technical debt is too heavy now. It's a good time to introduce a better "queue" package. # The new queue package It keeps using old config and concept as much as possible. * It only contains two major kinds of concepts: * The "base queue": channel, levelqueue, redis * They have the same abstraction, the same interface, and they are tested by the same testing code. * The "WokerPoolQueue", it uses the "base queue" to provide "worker pool" function, calls the "handler" to process the data in the base queue. * The new code doesn't do "PushBack" * Think about a queue with many workers, the "PushBack" can't guarantee the order for re-queued unhandled items, so in new code it just does "normal push" * The new code doesn't do "pause/resume" * The "pause/resume" was designed to handle some handler's failure: eg: document indexer (elasticsearch) is down * If a queue is paused for long time, either the producers blocks or the new items are dropped. * The new code doesn't do such "pause/resume" trick, it's not a common queue's behavior and it doesn't help much. * If there are unhandled items, the "push" function just blocks for a few seconds and then re-queue them and retry. * The new code doesn't do "worker booster" * Gitea's queue's handlers are light functions, the cost is only the go-routine, so it doesn't make sense to "boost" them. * The new code only use "max worker number" to limit the concurrent workers. * The new "Push" never blocks forever * Instead of creating more and more blocking goroutines, return an error is more friendly to the server and to the end user. There are more details in code comments: eg: the "Flush" problem, the strange "code.index" hanging problem, the "immediate" queue problem. Almost ready for review. TODO: * [x] add some necessary comments during review * [x] add some more tests if necessary * [x] update documents and config options * [x] test max worker / active worker * [x] re-run the CI tasks to see whether any test is flaky * [x] improve the `handleOldLengthConfiguration` to provide more friendly messages * [x] fine tune default config values (eg: length?) ## Code coverage: soft-fork/2023-05-16/main
wxiaoguang
GitHub
parent
cb700aedd1
commit
6f9c278559
|
|
@ -18,7 +18,6 @@ import (
|
|||
"code.gitea.io/gitea/modules/private"
|
||||
repo_module "code.gitea.io/gitea/modules/repository"
|
||||
"code.gitea.io/gitea/modules/setting"
|
||||
"code.gitea.io/gitea/modules/util"
|
||||
|
||||
"github.com/urfave/cli"
|
||||
)
|
||||
|
|
@ -141,7 +140,7 @@ func (d *delayWriter) Close() error {
|
|||
if d == nil {
|
||||
return nil
|
||||
}
|
||||
stopped := util.StopTimer(d.timer)
|
||||
stopped := d.timer.Stop()
|
||||
if stopped || d.buf == nil {
|
||||
return nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -926,12 +926,6 @@ ROUTER = console
|
|||
;; Global limit of repositories per user, applied at creation time. -1 means no limit
|
||||
;MAX_CREATION_LIMIT = -1
|
||||
;;
|
||||
;; Mirror sync queue length, increase if mirror syncing starts hanging (DEPRECATED: please use [queue.mirror] LENGTH instead)
|
||||
;MIRROR_QUEUE_LENGTH = 1000
|
||||
;;
|
||||
;; Patch test queue length, increase if pull request patch testing starts hanging (DEPRECATED: please use [queue.pr_patch_checker] LENGTH instead)
|
||||
;PULL_REQUEST_QUEUE_LENGTH = 1000
|
||||
;;
|
||||
;; Preferred Licenses to place at the top of the List
|
||||
;; The name here must match the filename in options/license or custom/options/license
|
||||
;PREFERRED_LICENSES = Apache License 2.0,MIT License
|
||||
|
|
@ -1376,22 +1370,6 @@ ROUTER = console
|
|||
;; Set to -1 to disable timeout.
|
||||
;STARTUP_TIMEOUT = 30s
|
||||
;;
|
||||
;; Issue indexer queue, currently support: channel, levelqueue or redis, default is levelqueue (deprecated - use [queue.issue_indexer])
|
||||
;ISSUE_INDEXER_QUEUE_TYPE = levelqueue; **DEPRECATED** use settings in `[queue.issue_indexer]`.
|
||||
;;
|
||||
;; When ISSUE_INDEXER_QUEUE_TYPE is levelqueue, this will be the path where the queue will be saved.
|
||||
;; This can be overridden by `ISSUE_INDEXER_QUEUE_CONN_STR`.
|
||||
;; default is queues/common
|
||||
;ISSUE_INDEXER_QUEUE_DIR = queues/common; **DEPRECATED** use settings in `[queue.issue_indexer]`. Relative paths will be made absolute against `%(APP_DATA_PATH)s`.
|
||||
;;
|
||||
;; When `ISSUE_INDEXER_QUEUE_TYPE` is `redis`, this will store the redis connection string.
|
||||
;; When `ISSUE_INDEXER_QUEUE_TYPE` is `levelqueue`, this is a directory or additional options of
|
||||
;; the form `leveldb://path/to/db?option=value&....`, and overrides `ISSUE_INDEXER_QUEUE_DIR`.
|
||||
;ISSUE_INDEXER_QUEUE_CONN_STR = "addrs=127.0.0.1:6379 db=0"; **DEPRECATED** use settings in `[queue.issue_indexer]`.
|
||||
;;
|
||||
;; Batch queue number, default is 20
|
||||
;ISSUE_INDEXER_QUEUE_BATCH_NUMBER = 20; **DEPRECATED** use settings in `[queue.issue_indexer]`.
|
||||
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
;; Repository Indexer settings
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
|
|
@ -1418,8 +1396,6 @@ ROUTER = console
|
|||
;; A comma separated list of glob patterns to exclude from the index; ; default is empty
|
||||
;REPO_INDEXER_EXCLUDE =
|
||||
;;
|
||||
;;
|
||||
;UPDATE_BUFFER_LEN = 20; **DEPRECATED** use settings in `[queue.issue_indexer]`.
|
||||
;MAX_FILE_SIZE = 1048576
|
||||
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
|
|
@ -1441,7 +1417,7 @@ ROUTER = console
|
|||
;DATADIR = queues/ ; Relative paths will be made absolute against `%(APP_DATA_PATH)s`.
|
||||
;;
|
||||
;; Default queue length before a channel queue will block
|
||||
;LENGTH = 20
|
||||
;LENGTH = 100
|
||||
;;
|
||||
;; Batch size to send for batched queues
|
||||
;BATCH_LENGTH = 20
|
||||
|
|
@ -1449,7 +1425,7 @@ ROUTER = console
|
|||
;; Connection string for redis queues this will store the redis connection string.
|
||||
;; When `TYPE` is `persistable-channel`, this provides a directory for the underlying leveldb
|
||||
;; or additional options of the form `leveldb://path/to/db?option=value&....`, and will override `DATADIR`.
|
||||
;CONN_STR = "addrs=127.0.0.1:6379 db=0"
|
||||
;CONN_STR = "redis://127.0.0.1:6379/0"
|
||||
;;
|
||||
;; Provides the suffix of the default redis/disk queue name - specific queues can be overridden within in their [queue.name] sections.
|
||||
;QUEUE_NAME = "_queue"
|
||||
|
|
@ -1457,29 +1433,8 @@ ROUTER = console
|
|||
;; Provides the suffix of the default redis/disk unique queue set name - specific queues can be overridden within in their [queue.name] sections.
|
||||
;SET_NAME = "_unique"
|
||||
;;
|
||||
;; If the queue cannot be created at startup - level queues may need a timeout at startup - wrap the queue:
|
||||
;WRAP_IF_NECESSARY = true
|
||||
;;
|
||||
;; Attempt to create the wrapped queue at max
|
||||
;MAX_ATTEMPTS = 10
|
||||
;;
|
||||
;; Timeout queue creation
|
||||
;TIMEOUT = 15m30s
|
||||
;;
|
||||
;; Create a pool with this many workers
|
||||
;WORKERS = 0
|
||||
;;
|
||||
;; Dynamically scale the worker pool to at this many workers
|
||||
;MAX_WORKERS = 10
|
||||
;;
|
||||
;; Add boost workers when the queue blocks for BLOCK_TIMEOUT
|
||||
;BLOCK_TIMEOUT = 1s
|
||||
;;
|
||||
;; Remove the boost workers after BOOST_TIMEOUT
|
||||
;BOOST_TIMEOUT = 5m
|
||||
;;
|
||||
;; During a boost add BOOST_WORKERS
|
||||
;BOOST_WORKERS = 1
|
||||
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
|
|
|
|||
|
|
@ -89,10 +89,6 @@ In addition there is _`StaticRootPath`_ which can be set as a built-in at build
|
|||
- `DEFAULT_PUSH_CREATE_PRIVATE`: **true**: Default private when creating a new repository with push-to-create.
|
||||
- `MAX_CREATION_LIMIT`: **-1**: Global maximum creation limit of repositories per user,
|
||||
`-1` means no limit.
|
||||
- `PULL_REQUEST_QUEUE_LENGTH`: **1000**: Length of pull request patch test queue, make it. **DEPRECATED** use `LENGTH` in `[queue.pr_patch_checker]`.
|
||||
as large as possible. Use caution when editing this value.
|
||||
- `MIRROR_QUEUE_LENGTH`: **1000**: Patch test queue length, increase if pull request patch
|
||||
testing starts hanging. **DEPRECATED** use `LENGTH` in `[queue.mirror]`.
|
||||
- `PREFERRED_LICENSES`: **Apache License 2.0,MIT License**: Preferred Licenses to place at
|
||||
the top of the list. Name must match file name in options/license or custom/options/license.
|
||||
- `DISABLE_HTTP_GIT`: **false**: Disable the ability to interact with repositories over the
|
||||
|
|
@ -465,11 +461,6 @@ relation to port exhaustion.
|
|||
- `ISSUE_INDEXER_CONN_STR`: ****: Issue indexer connection string, available when ISSUE_INDEXER_TYPE is elasticsearch, or meilisearch. i.e. http://elastic:changeme@localhost:9200
|
||||
- `ISSUE_INDEXER_NAME`: **gitea_issues**: Issue indexer name, available when ISSUE_INDEXER_TYPE is elasticsearch
|
||||
- `ISSUE_INDEXER_PATH`: **indexers/issues.bleve**: Index file used for issue search; available when ISSUE_INDEXER_TYPE is bleve and elasticsearch. Relative paths will be made absolute against _`AppWorkPath`_.
|
||||
- The next 4 configuration values are deprecated and should be set in `queue.issue_indexer` however are kept for backwards compatibility:
|
||||
- `ISSUE_INDEXER_QUEUE_TYPE`: **levelqueue**: Issue indexer queue, currently supports:`channel`, `levelqueue`, `redis`. **DEPRECATED** use settings in `[queue.issue_indexer]`.
|
||||
- `ISSUE_INDEXER_QUEUE_DIR`: **queues/common**: When `ISSUE_INDEXER_QUEUE_TYPE` is `levelqueue`, this will be the path where the queue will be saved. **DEPRECATED** use settings in `[queue.issue_indexer]`. Relative paths will be made absolute against `%(APP_DATA_PATH)s`.
|
||||
- `ISSUE_INDEXER_QUEUE_CONN_STR`: **addrs=127.0.0.1:6379 db=0**: When `ISSUE_INDEXER_QUEUE_TYPE` is `redis`, this will store the redis connection string. When `ISSUE_INDEXER_QUEUE_TYPE` is `levelqueue`, this is a directory or additional options of the form `leveldb://path/to/db?option=value&....`, and overrides `ISSUE_INDEXER_QUEUE_DIR`. **DEPRECATED** use settings in `[queue.issue_indexer]`.
|
||||
- `ISSUE_INDEXER_QUEUE_BATCH_NUMBER`: **20**: Batch queue number. **DEPRECATED** use settings in `[queue.issue_indexer]`.
|
||||
|
||||
- `REPO_INDEXER_ENABLED`: **false**: Enables code search (uses a lot of disk space, about 6 times more than the repository size).
|
||||
- `REPO_INDEXER_TYPE`: **bleve**: Code search engine type, could be `bleve` or `elasticsearch`.
|
||||
|
|
@ -480,7 +471,6 @@ relation to port exhaustion.
|
|||
- `REPO_INDEXER_INCLUDE`: **empty**: A comma separated list of glob patterns (see https://github.com/gobwas/glob) to **include** in the index. Use `**.txt` to match any files with .txt extension. An empty list means include all files.
|
||||
- `REPO_INDEXER_EXCLUDE`: **empty**: A comma separated list of glob patterns (see https://github.com/gobwas/glob) to **exclude** from the index. Files that match this list will not be indexed, even if they match in `REPO_INDEXER_INCLUDE`.
|
||||
- `REPO_INDEXER_EXCLUDE_VENDORED`: **true**: Exclude vendored files from index.
|
||||
- `UPDATE_BUFFER_LEN`: **20**: Buffer length of index request. **DEPRECATED** use settings in `[queue.issue_indexer]`.
|
||||
- `MAX_FILE_SIZE`: **1048576**: Maximum size in bytes of files to be indexed.
|
||||
- `STARTUP_TIMEOUT`: **30s**: If the indexer takes longer than this timeout to start - fail. (This timeout will be added to the hammer time above for child processes - as bleve will not start until the previous parent is shutdown.) Set to -1 to never timeout.
|
||||
|
||||
|
|
@ -488,23 +478,14 @@ relation to port exhaustion.
|
|||
|
||||
Configuration at `[queue]` will set defaults for queues with overrides for individual queues at `[queue.*]`. (However see below.)
|
||||
|
||||
- `TYPE`: **persistable-channel**: General queue type, currently support: `persistable-channel` (uses a LevelDB internally), `channel`, `level`, `redis`, `dummy`
|
||||
- `DATADIR`: **queues/**: Base DataDir for storing persistent and level queues. `DATADIR` for individual queues can be set in `queue.name` sections but will default to `DATADIR/`**`common`**. (Previously each queue would default to `DATADIR/`**`name`**.) Relative paths will be made absolute against `%(APP_DATA_PATH)s`.
|
||||
- `LENGTH`: **20**: Maximal queue size before channel queues block
|
||||
- `TYPE`: **level**: General queue type, currently support: `level` (uses a LevelDB internally), `channel`, `redis`, `dummy`. Invalid types are treated as `level`.
|
||||
- `DATADIR`: **queues/common**: Base DataDir for storing level queues. `DATADIR` for individual queues can be set in `queue.name` sections. Relative paths will be made absolute against `%(APP_DATA_PATH)s`.
|
||||
- `LENGTH`: **100**: Maximal queue size before channel queues block
|
||||
- `BATCH_LENGTH`: **20**: Batch data before passing to the handler
|
||||
- `CONN_STR`: **redis://127.0.0.1:6379/0**: Connection string for the redis queue type. Options can be set using query params. Similarly LevelDB options can also be set using: **leveldb://relative/path?option=value** or **leveldb:///absolute/path?option=value**, and will override `DATADIR`
|
||||
- `CONN_STR`: **redis://127.0.0.1:6379/0**: Connection string for the redis queue type. Options can be set using query params. Similarly, LevelDB options can also be set using: **leveldb://relative/path?option=value** or **leveldb:///absolute/path?option=value**, and will override `DATADIR`
|
||||
- `QUEUE_NAME`: **_queue**: The suffix for default redis and disk queue name. Individual queues will default to **`name`**`QUEUE_NAME` but can be overridden in the specific `queue.name` section.
|
||||
- `SET_NAME`: **_unique**: The suffix that will be added to the default redis and disk queue `set` name for unique queues. Individual queues will default to
|
||||
**`name`**`QUEUE_NAME`_`SET_NAME`_ but can be overridden in the specific `queue.name` section.
|
||||
- `WRAP_IF_NECESSARY`: **true**: Will wrap queues with a timeoutable queue if the selected queue is not ready to be created - (Only relevant for the level queue.)
|
||||
- `MAX_ATTEMPTS`: **10**: Maximum number of attempts to create the wrapped queue
|
||||
- `TIMEOUT`: **GRACEFUL_HAMMER_TIME + 30s**: Timeout the creation of the wrapped queue if it takes longer than this to create.
|
||||
- Queues by default come with a dynamically scaling worker pool. The following settings configure this:
|
||||
- `WORKERS`: **0**: Number of initial workers for the queue.
|
||||
- `SET_NAME`: **_unique**: The suffix that will be added to the default redis and disk queue `set` name for unique queues. Individual queues will default to **`name`**`QUEUE_NAME`_`SET_NAME`_ but can be overridden in the specific `queue.name` section.
|
||||
- `MAX_WORKERS`: **10**: Maximum number of worker go-routines for the queue.
|
||||
- `BLOCK_TIMEOUT`: **1s**: If the queue blocks for this time, boost the number of workers - the `BLOCK_TIMEOUT` will then be doubled before boosting again whilst the boost is ongoing.
|
||||
- `BOOST_TIMEOUT`: **5m**: Boost workers will timeout after this long.
|
||||
- `BOOST_WORKERS`: **1**: This many workers will be added to the worker pool if there is a boost.
|
||||
|
||||
Gitea creates the following non-unique queues:
|
||||
|
||||
|
|
@ -522,21 +503,6 @@ And the following unique queues:
|
|||
- `mirror`
|
||||
- `pr_patch_checker`
|
||||
|
||||
Certain queues have defaults that override the defaults set in `[queue]` (this occurs mostly to support older configuration):
|
||||
|
||||
- `[queue.issue_indexer]`
|
||||
- `TYPE` this will default to `[queue]` `TYPE` if it is set but if not it will appropriately convert `[indexer]` `ISSUE_INDEXER_QUEUE_TYPE` if that is set.
|
||||
- `LENGTH` will default to `[indexer]` `UPDATE_BUFFER_LEN` if that is set.
|
||||
- `BATCH_LENGTH` will default to `[indexer]` `ISSUE_INDEXER_QUEUE_BATCH_NUMBER` if that is set.
|
||||
- `DATADIR` will default to `[indexer]` `ISSUE_INDEXER_QUEUE_DIR` if that is set.
|
||||
- `CONN_STR` will default to `[indexer]` `ISSUE_INDEXER_QUEUE_CONN_STR` if that is set.
|
||||
- `[queue.mailer]`
|
||||
- `LENGTH` will default to **100** or whatever `[mailer]` `SEND_BUFFER_LEN` is.
|
||||
- `[queue.pr_patch_checker]`
|
||||
- `LENGTH` will default to **1000** or whatever `[repository]` `PULL_REQUEST_QUEUE_LENGTH` is.
|
||||
- `[queue.mirror]`
|
||||
- `LENGTH` will default to **1000** or whatever `[repository]` `MIRROR_QUEUE_LENGTH` is.
|
||||
|
||||
## Admin (`admin`)
|
||||
|
||||
- `DEFAULT_EMAIL_NOTIFICATIONS`: **enabled**: Default configuration for email notifications for users (user configurable). Options: enabled, onmention, disabled
|
||||
|
|
|
|||
|
|
@ -43,7 +43,6 @@ menu:
|
|||
- `DEFAULT_PRIVATE`: 默认创建的git工程为私有。 可以是`last`, `private` 或 `public`。默认值是 `last`表示用户最后创建的Repo的选择。
|
||||
- `DEFAULT_PUSH_CREATE_PRIVATE`: **true**: 通过 ``push-to-create`` 方式创建的仓库是否默认为私有仓库.
|
||||
- `MAX_CREATION_LIMIT`: 全局最大每个用户创建的git工程数目， `-1` 表示没限制。
|
||||
- `PULL_REQUEST_QUEUE_LENGTH`: 小心：合并请求测试队列的长度，尽量放大。
|
||||
|
||||
### Repository - Release (`repository.release`)
|
||||
|
||||
|
|
@ -111,10 +110,6 @@ menu:
|
|||
- `ISSUE_INDEXER_CONN_STR`: ****: 工单索引连接字符串，仅当 ISSUE_INDEXER_TYPE 为 `elasticsearch` 时有效。例如: http://elastic:changeme@localhost:9200
|
||||
- `ISSUE_INDEXER_NAME`: **gitea_issues**: 工单索引名称，仅当 ISSUE_INDEXER_TYPE 为 `elasticsearch` 时有效。
|
||||
- `ISSUE_INDEXER_PATH`: **indexers/issues.bleve**: 工单索引文件存放路径，当索引类型为 `bleve` 时有效。
|
||||
- `ISSUE_INDEXER_QUEUE_TYPE`: **levelqueue**: 工单索引队列类型，当前支持 `channel`， `levelqueue` 或 `redis`。
|
||||
- `ISSUE_INDEXER_QUEUE_DIR`: **indexers/issues.queue**: 当 `ISSUE_INDEXER_QUEUE_TYPE` 为 `levelqueue` 时，保存索引队列的磁盘路径。
|
||||
- `ISSUE_INDEXER_QUEUE_CONN_STR`: **addrs=127.0.0.1:6379 db=0**: 当 `ISSUE_INDEXER_QUEUE_TYPE` 为 `redis` 时，保存Redis队列的连接字符串。
|
||||
- `ISSUE_INDEXER_QUEUE_BATCH_NUMBER`: **20**: 队列处理中批量提交数量。
|
||||
|
||||
- `REPO_INDEXER_ENABLED`: **false**: 是否启用代码搜索（启用后会占用比较大的磁盘空间，如果是bleve可能需要占用约6倍存储空间）。
|
||||
- `REPO_INDEXER_TYPE`: **bleve**: 代码搜索引擎类型，可以为 `bleve` 或者 `elasticsearch`。
|
||||
|
|
@ -122,7 +117,6 @@ menu:
|
|||
- `REPO_INDEXER_CONN_STR`: ****: 代码搜索引擎连接字符串，当 `REPO_INDEXER_TYPE` 为 `elasticsearch` 时有效。例如： http://elastic:changeme@localhost:9200
|
||||
- `REPO_INDEXER_NAME`: **gitea_codes**: 代码搜索引擎的名字，当 `REPO_INDEXER_TYPE` 为 `elasticsearch` 时有效。
|
||||
|
||||
- `UPDATE_BUFFER_LEN`: **20**: 代码索引请求的缓冲区长度。
|
||||
- `MAX_FILE_SIZE`: **1048576**: 进行解析的源代码文件的最大长度，小于该值时才会索引。
|
||||
|
||||
## Security (`security`)
|
||||
|
|
|
|||
|
|
@ -30,7 +30,6 @@ Gitea can search through the files of the repositories by enabling this function
|
|||
; ...
|
||||
REPO_INDEXER_ENABLED = true
|
||||
REPO_INDEXER_PATH = indexers/repos.bleve
|
||||
UPDATE_BUFFER_LEN = 20
|
||||
MAX_FILE_SIZE = 1048576
|
||||
REPO_INDEXER_INCLUDE =
|
||||
REPO_INDEXER_EXCLUDE = resources/bin/**
|
||||
|
|
|
|||
|
|
@ -1,180 +0,0 @@
|
|||
// Copyright 2019 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package base
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"code.gitea.io/gitea/modules/json"
|
||||
"code.gitea.io/gitea/modules/log"
|
||||
"code.gitea.io/gitea/modules/queue"
|
||||
)
|
||||
|
||||
var (
|
||||
prefix string
|
||||
slowTest = 10 * time.Second
|
||||
slowFlush = 5 * time.Second
|
||||
)
|
||||
|
||||
// TestLogger is a logger which will write to the testing log
|
||||
type TestLogger struct {
|
||||
log.WriterLogger
|
||||
}
|
||||
|
||||
var writerCloser = &testLoggerWriterCloser{}
|
||||
|
||||
type testLoggerWriterCloser struct {
|
||||
sync.RWMutex
|
||||
t []*testing.TB
|
||||
}
|
||||
|
||||
func (w *testLoggerWriterCloser) setT(t *testing.TB) {
|
||||
w.Lock()
|
||||
w.t = append(w.t, t)
|
||||
w.Unlock()
|
||||
}
|
||||
|
||||
func (w *testLoggerWriterCloser) Write(p []byte) (int, error) {
|
||||
w.RLock()
|
||||
var t *testing.TB
|
||||
if len(w.t) > 0 {
|
||||
t = w.t[len(w.t)-1]
|
||||
}
|
||||
w.RUnlock()
|
||||
if t != nil && *t != nil {
|
||||
if len(p) > 0 && p[len(p)-1] == '\n' {
|
||||
p = p[:len(p)-1]
|
||||
}
|
||||
|
||||
defer func() {
|
||||
err := recover()
|
||||
if err == nil {
|
||||
return
|
||||
}
|
||||
var errString string
|
||||
errErr, ok := err.(error)
|
||||
if ok {
|
||||
errString = errErr.Error()
|
||||
} else {
|
||||
errString, ok = err.(string)
|
||||
}
|
||||
if !ok {
|
||||
panic(err)
|
||||
}
|
||||
if !strings.HasPrefix(errString, "Log in goroutine after ") {
|
||||
panic(err)
|
||||
}
|
||||
}()
|
||||
|
||||
(*t).Log(string(p))
|
||||
return len(p), nil
|
||||
}
|
||||
return len(p), nil
|
||||
}
|
||||
|
||||
func (w *testLoggerWriterCloser) Close() error {
|
||||
w.Lock()
|
||||
if len(w.t) > 0 {
|
||||
w.t = w.t[:len(w.t)-1]
|
||||
}
|
||||
w.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
// PrintCurrentTest prints the current test to os.Stdout
|
||||
func PrintCurrentTest(t testing.TB, skip ...int) func() {
|
||||
start := time.Now()
|
||||
actualSkip := 1
|
||||
if len(skip) > 0 {
|
||||
actualSkip = skip[0]
|
||||
}
|
||||
_, filename, line, _ := runtime.Caller(actualSkip)
|
||||
|
||||
if log.CanColorStdout {
|
||||
fmt.Fprintf(os.Stdout, "=== %s (%s:%d)\n", fmt.Formatter(log.NewColoredValue(t.Name())), strings.TrimPrefix(filename, prefix), line)
|
||||
} else {
|
||||
fmt.Fprintf(os.Stdout, "=== %s (%s:%d)\n", t.Name(), strings.TrimPrefix(filename, prefix), line)
|
||||
}
|
||||
writerCloser.setT(&t)
|
||||
return func() {
|
||||
took := time.Since(start)
|
||||
if took > slowTest {
|
||||
if log.CanColorStdout {
|
||||
fmt.Fprintf(os.Stdout, "+++ %s is a slow test (took %v)\n", fmt.Formatter(log.NewColoredValue(t.Name(), log.Bold, log.FgYellow)), fmt.Formatter(log.NewColoredValue(took, log.Bold, log.FgYellow)))
|
||||
} else {
|
||||
fmt.Fprintf(os.Stdout, "+++ %s is a slow test (took %v)\n", t.Name(), took)
|
||||
}
|
||||
}
|
||||
timer := time.AfterFunc(slowFlush, func() {
|
||||
if log.CanColorStdout {
|
||||
fmt.Fprintf(os.Stdout, "+++ %s ... still flushing after %v ...\n", fmt.Formatter(log.NewColoredValue(t.Name(), log.Bold, log.FgRed)), slowFlush)
|
||||
} else {
|
||||
fmt.Fprintf(os.Stdout, "+++ %s ... still flushing after %v ...\n", t.Name(), slowFlush)
|
||||
}
|
||||
})
|
||||
if err := queue.GetManager().FlushAll(context.Background(), -1); err != nil {
|
||||
t.Errorf("Flushing queues failed with error %v", err)
|
||||
}
|
||||
timer.Stop()
|
||||
flushTook := time.Since(start) - took
|
||||
if flushTook > slowFlush {
|
||||
if log.CanColorStdout {
|
||||
fmt.Fprintf(os.Stdout, "+++ %s had a slow clean-up flush (took %v)\n", fmt.Formatter(log.NewColoredValue(t.Name(), log.Bold, log.FgRed)), fmt.Formatter(log.NewColoredValue(flushTook, log.Bold, log.FgRed)))
|
||||
} else {
|
||||
fmt.Fprintf(os.Stdout, "+++ %s had a slow clean-up flush (took %v)\n", t.Name(), flushTook)
|
||||
}
|
||||
}
|
||||
_ = writerCloser.Close()
|
||||
}
|
||||
}
|
||||
|
||||
// Printf takes a format and args and prints the string to os.Stdout
|
||||
func Printf(format string, args ...interface{}) {
|
||||
if log.CanColorStdout {
|
||||
for i := 0; i < len(args); i++ {
|
||||
args[i] = log.NewColoredValue(args[i])
|
||||
}
|
||||
}
|
||||
fmt.Fprintf(os.Stdout, "\t"+format, args...)
|
||||
}
|
||||
|
||||
// NewTestLogger creates a TestLogger as a log.LoggerProvider
|
||||
func NewTestLogger() log.LoggerProvider {
|
||||
logger := &TestLogger{}
|
||||
logger.Colorize = log.CanColorStdout
|
||||
logger.Level = log.TRACE
|
||||
return logger
|
||||
}
|
||||
|
||||
// Init inits connection writer with json config.
|
||||
// json config only need key "level".
|
||||
func (log *TestLogger) Init(config string) error {
|
||||
err := json.Unmarshal([]byte(config), log)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
log.NewWriterLogger(writerCloser)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Flush when log should be flushed
|
||||
func (log *TestLogger) Flush() {
|
||||
}
|
||||
|
||||
// ReleaseReopen does nothing
|
||||
func (log *TestLogger) ReleaseReopen() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetName returns the default name for this implementation
|
||||
func (log *TestLogger) GetName() string {
|
||||
return "test"
|
||||
}
|
||||
|
|
@ -11,7 +11,6 @@ import (
|
|||
"path"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"code.gitea.io/gitea/models/unittest"
|
||||
|
|
@ -19,6 +18,7 @@ import (
|
|||
"code.gitea.io/gitea/modules/git"
|
||||
"code.gitea.io/gitea/modules/log"
|
||||
"code.gitea.io/gitea/modules/setting"
|
||||
"code.gitea.io/gitea/modules/testlogger"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"xorm.io/xorm"
|
||||
|
|
@ -32,7 +32,7 @@ func PrepareTestEnv(t *testing.T, skip int, syncModels ...interface{}) (*xorm.En
|
|||
t.Helper()
|
||||
ourSkip := 2
|
||||
ourSkip += skip
|
||||
deferFn := PrintCurrentTest(t, ourSkip)
|
||||
deferFn := testlogger.PrintCurrentTest(t, ourSkip)
|
||||
assert.NoError(t, os.RemoveAll(setting.RepoRootPath))
|
||||
assert.NoError(t, unittest.CopyDir(path.Join(filepath.Dir(setting.AppPath), "tests/gitea-repositories-meta"), setting.RepoRootPath))
|
||||
ownerDirs, err := os.ReadDir(setting.RepoRootPath)
|
||||
|
|
@ -110,9 +110,7 @@ func PrepareTestEnv(t *testing.T, skip int, syncModels ...interface{}) (*xorm.En
|
|||
}
|
||||
|
||||
func MainTest(m *testing.M) {
|
||||
log.Register("test", NewTestLogger)
|
||||
_, filename, _, _ := runtime.Caller(0)
|
||||
prefix = strings.TrimSuffix(filename, "tests/testlogger.go")
|
||||
log.Register("test", testlogger.NewTestLogger)
|
||||
|
||||
giteaRoot := base.SetupGiteaRoot()
|
||||
if giteaRoot == "" {
|
||||
|
|
|
|||
|
|
@ -202,6 +202,9 @@ type FixturesOptions struct {
|
|||
func CreateTestEngine(opts FixturesOptions) error {
|
||||
x, err := xorm.NewEngine("sqlite3", "file::memory:?cache=shared&_txlock=immediate")
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "unknown driver") {
|
||||
return fmt.Errorf(`sqlite3 requires: import _ "github.com/mattn/go-sqlite3" or -tags sqlite,sqlite_unlock_notify%s%w`, "\n", err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
x.SetMapper(names.GonicMapper{})
|
||||
|
|
|
|||
|
|
@ -273,10 +273,6 @@ func (b *BleveIndexer) Close() {
|
|||
log.Info("PID: %d Repository Indexer closed", os.Getpid())
|
||||
}
|
||||
|
||||
// SetAvailabilityChangeCallback does nothing
|
||||
func (b *BleveIndexer) SetAvailabilityChangeCallback(callback func(bool)) {
|
||||
}
|
||||
|
||||
// Ping does nothing
|
||||
func (b *BleveIndexer) Ping() bool {
|
||||
return true
|
||||
|
|
|
|||
|
|
@ -42,12 +42,11 @@ var _ Indexer = &ElasticSearchIndexer{}
|
|||
|
||||
// ElasticSearchIndexer implements Indexer interface
|
||||
type ElasticSearchIndexer struct {
|
||||
client *elastic.Client
|
||||
indexerAliasName string
|
||||
available bool
|
||||
availabilityCallback func(bool)
|
||||
stopTimer chan struct{}
|
||||
lock sync.RWMutex
|
||||
client *elastic.Client
|
||||
indexerAliasName string
|
||||
available bool
|
||||
stopTimer chan struct{}
|
||||
lock sync.RWMutex
|
||||
}
|
||||
|
||||
type elasticLogger struct {
|
||||
|
|
@ -198,13 +197,6 @@ func (b *ElasticSearchIndexer) init() (bool, error) {
|
|||
return exists, nil
|
||||
}
|
||||
|
||||
// SetAvailabilityChangeCallback sets callback that will be triggered when availability changes
|
||||
func (b *ElasticSearchIndexer) SetAvailabilityChangeCallback(callback func(bool)) {
|
||||
b.lock.Lock()
|
||||
defer b.lock.Unlock()
|
||||
b.availabilityCallback = callback
|
||||
}
|
||||
|
||||
// Ping checks if elastic is available
|
||||
func (b *ElasticSearchIndexer) Ping() bool {
|
||||
b.lock.RLock()
|
||||
|
|
@ -529,8 +521,4 @@ func (b *ElasticSearchIndexer) setAvailability(available bool) {
|
|||
}
|
||||
|
||||
b.available = available
|
||||
if b.availabilityCallback != nil {
|
||||
// Call the callback from within the lock to ensure that the ordering remains correct
|
||||
b.availabilityCallback(b.available)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -44,7 +44,6 @@ type SearchResultLanguages struct {
|
|||
// Indexer defines an interface to index and search code contents
|
||||
type Indexer interface {
|
||||
Ping() bool
|
||||
SetAvailabilityChangeCallback(callback func(bool))
|
||||
Index(ctx context.Context, repo *repo_model.Repository, sha string, changes *repoChanges) error
|
||||
Delete(repoID int64) error
|
||||
Search(ctx context.Context, repoIDs []int64, language, keyword string, page, pageSize int, isMatch bool) (int64, []*SearchResult, []*SearchResultLanguages, error)
|
||||
|
|
@ -81,7 +80,7 @@ type IndexerData struct {
|
|||
RepoID int64
|
||||
}
|
||||
|
||||
var indexerQueue queue.UniqueQueue
|
||||
var indexerQueue *queue.WorkerPoolQueue[*IndexerData]
|
||||
|
||||
func index(ctx context.Context, indexer Indexer, repoID int64) error {
|
||||
repo, err := repo_model.GetRepositoryByID(ctx, repoID)
|
||||
|
|
@ -137,37 +136,45 @@ func Init() {
|
|||
// Create the Queue
|
||||
switch setting.Indexer.RepoType {
|
||||
case "bleve", "elasticsearch":
|
||||
handler := func(data ...queue.Data) []queue.Data {
|
||||
handler := func(items ...*IndexerData) (unhandled []*IndexerData) {
|
||||
idx, err := indexer.get()
|
||||
if idx == nil || err != nil {
|
||||
log.Error("Codes indexer handler: unable to get indexer!")
|
||||
return data
|
||||
return items
|
||||
}
|
||||
|
||||
unhandled := make([]queue.Data, 0, len(data))
|
||||
for _, datum := range data {
|
||||
indexerData, ok := datum.(*IndexerData)
|
||||
if !ok {
|
||||
log.Error("Unable to process provided datum: %v - not possible to cast to IndexerData", datum)
|
||||
continue
|
||||
}
|
||||
for _, indexerData := range items {
|
||||
log.Trace("IndexerData Process Repo: %d", indexerData.RepoID)
|
||||
|
||||
// FIXME: it seems there is a bug in `CatFileBatch` or `nio.Pipe`, which will cause the process to hang forever in rare cases
|
||||
/*
|
||||
sync.(*Cond).Wait(cond.go:70)
|
||||
github.com/djherbis/nio/v3.(*PipeReader).Read(sync.go:106)
|
||||
bufio.(*Reader).fill(bufio.go:106)
|
||||
bufio.(*Reader).ReadSlice(bufio.go:372)
|
||||
bufio.(*Reader).collectFragments(bufio.go:447)
|
||||
bufio.(*Reader).ReadString(bufio.go:494)
|
||||
code.gitea.io/gitea/modules/git.ReadBatchLine(batch_reader.go:149)
|
||||
code.gitea.io/gitea/modules/indexer/code.(*BleveIndexer).addUpdate(bleve.go:214)
|
||||
code.gitea.io/gitea/modules/indexer/code.(*BleveIndexer).Index(bleve.go:296)
|
||||
code.gitea.io/gitea/modules/indexer/code.(*wrappedIndexer).Index(wrapped.go:74)
|
||||
code.gitea.io/gitea/modules/indexer/code.index(indexer.go:105)
|
||||
*/
|
||||
if err := index(ctx, indexer, indexerData.RepoID); err != nil {
|
||||
if !setting.IsInTesting {
|
||||
log.Error("indexer index error for repo %v: %v", indexerData.RepoID, err)
|
||||
}
|
||||
if indexer.Ping() {
|
||||
if !idx.Ping() {
|
||||
log.Error("Code indexer handler: indexer is unavailable.")
|
||||
unhandled = append(unhandled, indexerData)
|
||||
continue
|
||||
}
|
||||
// Add back to queue
|
||||
unhandled = append(unhandled, datum)
|
||||
if !setting.IsInTesting {
|
||||
log.Error("Codes indexer handler: index error for repo %v: %v", indexerData.RepoID, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return unhandled
|
||||
}
|
||||
|
||||
indexerQueue = queue.CreateUniqueQueue("code_indexer", handler, &IndexerData{})
|
||||
indexerQueue = queue.CreateUniqueQueue("code_indexer", handler)
|
||||
if indexerQueue == nil {
|
||||
log.Fatal("Unable to create codes indexer queue")
|
||||
}
|
||||
|
|
@ -224,18 +231,6 @@ func Init() {
|
|||
|
||||
indexer.set(rIndexer)
|
||||
|
||||
if queue, ok := indexerQueue.(queue.Pausable); ok {
|
||||
rIndexer.SetAvailabilityChangeCallback(func(available bool) {
|
||||
if !available {
|
||||
log.Info("Code index queue paused")
|
||||
queue.Pause()
|
||||
} else {
|
||||
log.Info("Code index queue resumed")
|
||||
queue.Resume()
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// Start processing the queue
|
||||
go graceful.GetManager().RunWithShutdownFns(indexerQueue.Run)
|
||||
|
||||
|
|
|
|||
|
|
@ -56,16 +56,6 @@ func (w *wrappedIndexer) get() (Indexer, error) {
|
|||
return w.internal, nil
|
||||
}
|
||||
|
||||
// SetAvailabilityChangeCallback sets callback that will be triggered when availability changes
|
||||
func (w *wrappedIndexer) SetAvailabilityChangeCallback(callback func(bool)) {
|
||||
indexer, err := w.get()
|
||||
if err != nil {
|
||||
log.Error("Failed to get indexer: %v", err)
|
||||
return
|
||||
}
|
||||
indexer.SetAvailabilityChangeCallback(callback)
|
||||
}
|
||||
|
||||
// Ping checks if elastic is available
|
||||
func (w *wrappedIndexer) Ping() bool {
|
||||
indexer, err := w.get()
|
||||
|
|
|
|||
|
|
@ -187,10 +187,6 @@ func (b *BleveIndexer) Init() (bool, error) {
|
|||
return false, err
|
||||
}
|
||||
|
||||
// SetAvailabilityChangeCallback does nothing
|
||||
func (b *BleveIndexer) SetAvailabilityChangeCallback(callback func(bool)) {
|
||||
}
|
||||
|
||||
// Ping does nothing
|
||||
func (b *BleveIndexer) Ping() bool {
|
||||
return true
|
||||
|
|
|
|||
|
|
@ -18,10 +18,6 @@ func (i *DBIndexer) Init() (bool, error) {
|
|||
return false, nil
|
||||
}
|
||||
|
||||
// SetAvailabilityChangeCallback dummy function
|
||||
func (i *DBIndexer) SetAvailabilityChangeCallback(callback func(bool)) {
|
||||
}
|
||||
|
||||
// Ping checks if database is available
|
||||
func (i *DBIndexer) Ping() bool {
|
||||
return db.GetEngine(db.DefaultContext).Ping() != nil
|
||||
|
|
|
|||
|
|
@ -22,12 +22,11 @@ var _ Indexer = &ElasticSearchIndexer{}
|
|||
|
||||
// ElasticSearchIndexer implements Indexer interface
|
||||
type ElasticSearchIndexer struct {
|
||||
client *elastic.Client
|
||||
indexerName string
|
||||
available bool
|
||||
availabilityCallback func(bool)
|
||||
stopTimer chan struct{}
|
||||
lock sync.RWMutex
|
||||
client *elastic.Client
|
||||
indexerName string
|
||||
available bool
|
||||
stopTimer chan struct{}
|
||||
lock sync.RWMutex
|
||||
}
|
||||
|
||||
type elasticLogger struct {
|
||||
|
|
@ -138,13 +137,6 @@ func (b *ElasticSearchIndexer) Init() (bool, error) {
|
|||
return true, nil
|
||||
}
|
||||
|
||||
// SetAvailabilityChangeCallback sets callback that will be triggered when availability changes
|
||||
func (b *ElasticSearchIndexer) SetAvailabilityChangeCallback(callback func(bool)) {
|
||||
b.lock.Lock()
|
||||
defer b.lock.Unlock()
|
||||
b.availabilityCallback = callback
|
||||
}
|
||||
|
||||
// Ping checks if elastic is available
|
||||
func (b *ElasticSearchIndexer) Ping() bool {
|
||||
b.lock.RLock()
|
||||
|
|
@ -305,8 +297,4 @@ func (b *ElasticSearchIndexer) setAvailability(available bool) {
|
|||
}
|
||||
|
||||
b.available = available
|
||||
if b.availabilityCallback != nil {
|
||||
// Call the callback from within the lock to ensure that the ordering remains correct
|
||||
b.availabilityCallback(b.available)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -49,7 +49,6 @@ type SearchResult struct {
|
|||
type Indexer interface {
|
||||
Init() (bool, error)
|
||||
Ping() bool
|
||||
SetAvailabilityChangeCallback(callback func(bool))
|
||||
Index(issue []*IndexerData) error
|
||||
Delete(ids ...int64) error
|
||||
Search(ctx context.Context, kw string, repoIDs []int64, limit, start int) (*SearchResult, error)
|
||||
|
|
@ -94,7 +93,7 @@ func (h *indexerHolder) get() Indexer {
|
|||
|
||||
var (
|
||||
// issueIndexerQueue queue of issue ids to be updated
|
||||
issueIndexerQueue queue.Queue
|
||||
issueIndexerQueue *queue.WorkerPoolQueue[*IndexerData]
|
||||
holder = newIndexerHolder()
|
||||
)
|
||||
|
||||
|
|
@ -108,62 +107,44 @@ func InitIssueIndexer(syncReindex bool) {
|
|||
// Create the Queue
|
||||
switch setting.Indexer.IssueType {
|
||||
case "bleve", "elasticsearch", "meilisearch":
|
||||
handler := func(data ...queue.Data) []queue.Data {
|
||||
handler := func(items ...*IndexerData) (unhandled []*IndexerData) {
|
||||
indexer := holder.get()
|
||||
if indexer == nil {
|
||||
log.Error("Issue indexer handler: unable to get indexer!")
|
||||
return data
|
||||
log.Error("Issue indexer handler: unable to get indexer.")
|
||||
return items
|
||||
}
|
||||
|
||||
iData := make([]*IndexerData, 0, len(data))
|
||||
unhandled := make([]queue.Data, 0, len(data))
|
||||
for _, datum := range data {
|
||||
indexerData, ok := datum.(*IndexerData)
|
||||
if !ok {
|
||||
log.Error("Unable to process provided datum: %v - not possible to cast to IndexerData", datum)
|
||||
continue
|
||||
}
|
||||
toIndex := make([]*IndexerData, 0, len(items))
|
||||
for _, indexerData := range items {
|
||||
log.Trace("IndexerData Process: %d %v %t", indexerData.ID, indexerData.IDs, indexerData.IsDelete)
|
||||
if indexerData.IsDelete {
|
||||
if err := indexer.Delete(indexerData.IDs...); err != nil {
|
||||
log.Error("Error whilst deleting from index: %v Error: %v", indexerData.IDs, err)
|
||||
if indexer.Ping() {
|
||||
continue
|
||||
log.Error("Issue indexer handler: failed to from index: %v Error: %v", indexerData.IDs, err)
|
||||
if !indexer.Ping() {
|
||||
log.Error("Issue indexer handler: indexer is unavailable when deleting")
|
||||
unhandled = append(unhandled, indexerData)
|
||||
}
|
||||
// Add back to queue
|
||||
unhandled = append(unhandled, datum)
|
||||
}
|
||||
continue
|
||||
}
|
||||
iData = append(iData, indexerData)
|
||||
toIndex = append(toIndex, indexerData)
|
||||
}
|
||||
if len(unhandled) > 0 {
|
||||
for _, indexerData := range iData {
|
||||
unhandled = append(unhandled, indexerData)
|
||||
if err := indexer.Index(toIndex); err != nil {
|
||||
log.Error("Error whilst indexing: %v Error: %v", toIndex, err)
|
||||
if !indexer.Ping() {
|
||||
log.Error("Issue indexer handler: indexer is unavailable when indexing")
|
||||
unhandled = append(unhandled, toIndex...)
|
||||
}
|
||||
return unhandled
|
||||
}
|
||||
if err := indexer.Index(iData); err != nil {
|
||||
log.Error("Error whilst indexing: %v Error: %v", iData, err)
|
||||
if indexer.Ping() {
|
||||
return nil
|
||||
}
|
||||
// Add back to queue
|
||||
for _, indexerData := range iData {
|
||||
unhandled = append(unhandled, indexerData)
|
||||
}
|
||||
return unhandled
|
||||
}
|
||||
return nil
|
||||
return unhandled
|
||||
}
|
||||
|
||||
issueIndexerQueue = queue.CreateQueue("issue_indexer", handler, &IndexerData{})
|
||||
issueIndexerQueue = queue.CreateSimpleQueue("issue_indexer", handler)
|
||||
|
||||
if issueIndexerQueue == nil {
|
||||
log.Fatal("Unable to create issue indexer queue")
|
||||
}
|
||||
default:
|
||||
issueIndexerQueue = &queue.DummyQueue{}
|
||||
issueIndexerQueue = queue.CreateSimpleQueue[*IndexerData]("issue_indexer", nil)
|
||||
}
|
||||
|
||||
// Create the Indexer
|
||||
|
|
@ -240,18 +221,6 @@ func InitIssueIndexer(syncReindex bool) {
|
|||
log.Fatal("Unknown issue indexer type: %s", setting.Indexer.IssueType)
|
||||
}
|
||||
|
||||
if queue, ok := issueIndexerQueue.(queue.Pausable); ok {
|
||||
holder.get().SetAvailabilityChangeCallback(func(available bool) {
|
||||
if !available {
|
||||
log.Info("Issue index queue paused")
|
||||
queue.Pause()
|
||||
} else {
|
||||
log.Info("Issue index queue resumed")
|
||||
queue.Resume()
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// Start processing the queue
|
||||
go graceful.GetManager().RunWithShutdownFns(issueIndexerQueue.Run)
|
||||
|
||||
|
|
@ -285,9 +254,7 @@ func InitIssueIndexer(syncReindex bool) {
|
|||
case <-graceful.GetManager().IsShutdown():
|
||||
log.Warn("Shutdown occurred before issue index initialisation was complete")
|
||||
case <-time.After(timeout):
|
||||
if shutdownable, ok := issueIndexerQueue.(queue.Shutdownable); ok {
|
||||
shutdownable.Terminate()
|
||||
}
|
||||
issueIndexerQueue.ShutdownWait(5 * time.Second)
|
||||
log.Fatal("Issue Indexer Initialization timed-out after: %v", timeout)
|
||||
}
|
||||
}()
|
||||
|
|
|
|||
|
|
@ -17,12 +17,11 @@ var _ Indexer = &MeilisearchIndexer{}
|
|||
|
||||
// MeilisearchIndexer implements Indexer interface
|
||||
type MeilisearchIndexer struct {
|
||||
client *meilisearch.Client
|
||||
indexerName string
|
||||
available bool
|
||||
availabilityCallback func(bool)
|
||||
stopTimer chan struct{}
|
||||
lock sync.RWMutex
|
||||
client *meilisearch.Client
|
||||
indexerName string
|
||||
available bool
|
||||
stopTimer chan struct{}
|
||||
lock sync.RWMutex
|
||||
}
|
||||
|
||||
// MeilisearchIndexer creates a new meilisearch indexer
|
||||
|
|
@ -73,13 +72,6 @@ func (b *MeilisearchIndexer) Init() (bool, error) {
|
|||
return false, b.checkError(err)
|
||||
}
|
||||
|
||||
// SetAvailabilityChangeCallback sets callback that will be triggered when availability changes
|
||||
func (b *MeilisearchIndexer) SetAvailabilityChangeCallback(callback func(bool)) {
|
||||
b.lock.Lock()
|
||||
defer b.lock.Unlock()
|
||||
b.availabilityCallback = callback
|
||||
}
|
||||
|
||||
// Ping checks if meilisearch is available
|
||||
func (b *MeilisearchIndexer) Ping() bool {
|
||||
b.lock.RLock()
|
||||
|
|
@ -178,8 +170,4 @@ func (b *MeilisearchIndexer) setAvailability(available bool) {
|
|||
}
|
||||
|
||||
b.available = available
|
||||
if b.availabilityCallback != nil {
|
||||
// Call the callback from within the lock to ensure that the ordering remains correct
|
||||
b.availabilityCallback(b.available)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -41,7 +41,7 @@ func TestRepoStatsIndex(t *testing.T) {
|
|||
err = UpdateRepoIndexer(repo)
|
||||
assert.NoError(t, err)
|
||||
|
||||
queue.GetManager().FlushAll(context.Background(), 5*time.Second)
|
||||
assert.NoError(t, queue.GetManager().FlushAll(context.Background(), 5*time.Second))
|
||||
|
||||
status, err := repo_model.GetIndexerStatus(db.DefaultContext, repo, repo_model.RepoIndexerTypeStats)
|
||||
assert.NoError(t, err)
|
||||
|
|
|
|||
|
|
@ -14,12 +14,11 @@ import (
|
|||
)
|
||||
|
||||
// statsQueue represents a queue to handle repository stats updates
|
||||
var statsQueue queue.UniqueQueue
|
||||
var statsQueue *queue.WorkerPoolQueue[int64]
|
||||
|
||||
// handle passed PR IDs and test the PRs
|
||||
func handle(data ...queue.Data) []queue.Data {
|
||||
for _, datum := range data {
|
||||
opts := datum.(int64)
|
||||
func handler(items ...int64) []int64 {
|
||||
for _, opts := range items {
|
||||
if err := indexer.Index(opts); err != nil {
|
||||
if !setting.IsInTesting {
|
||||
log.Error("stats queue indexer.Index(%d) failed: %v", opts, err)
|
||||
|
|
@ -30,7 +29,7 @@ func handle(data ...queue.Data) []queue.Data {
|
|||
}
|
||||
|
||||
func initStatsQueue() error {
|
||||
statsQueue = queue.CreateUniqueQueue("repo_stats_update", handle, int64(0))
|
||||
statsQueue = queue.CreateUniqueQueue("repo_stats_update", handler)
|
||||
if statsQueue == nil {
|
||||
return fmt.Errorf("Unable to create repo_stats_update Queue")
|
||||
}
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ import (
|
|||
"code.gitea.io/gitea/modules/setting"
|
||||
)
|
||||
|
||||
var mirrorQueue queue.UniqueQueue
|
||||
var mirrorQueue *queue.WorkerPoolQueue[*SyncRequest]
|
||||
|
||||
// SyncType type of sync request
|
||||
type SyncType int
|
||||
|
|
@ -29,11 +29,11 @@ type SyncRequest struct {
|
|||
}
|
||||
|
||||
// StartSyncMirrors starts a go routine to sync the mirrors
|
||||
func StartSyncMirrors(queueHandle func(data ...queue.Data) []queue.Data) {
|
||||
func StartSyncMirrors(queueHandle func(data ...*SyncRequest) []*SyncRequest) {
|
||||
if !setting.Mirror.Enabled {
|
||||
return
|
||||
}
|
||||
mirrorQueue = queue.CreateUniqueQueue("mirror", queueHandle, new(SyncRequest))
|
||||
mirrorQueue = queue.CreateUniqueQueue("mirror", queueHandle)
|
||||
|
||||
go graceful.GetManager().RunWithShutdownFns(mirrorQueue.Run)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ import (
|
|||
type (
|
||||
notificationService struct {
|
||||
base.NullNotifier
|
||||
issueQueue queue.Queue
|
||||
issueQueue *queue.WorkerPoolQueue[issueNotificationOpts]
|
||||
}
|
||||
|
||||
issueNotificationOpts struct {
|
||||
|
|
@ -37,13 +37,12 @@ var _ base.Notifier = ¬ificationService{}
|
|||
// NewNotifier create a new notificationService notifier
|
||||
func NewNotifier() base.Notifier {
|
||||
ns := ¬ificationService{}
|
||||
ns.issueQueue = queue.CreateQueue("notification-service", ns.handle, issueNotificationOpts{})
|
||||
ns.issueQueue = queue.CreateSimpleQueue("notification-service", handler)
|
||||
return ns
|
||||
}
|
||||
|
||||
func (ns *notificationService) handle(data ...queue.Data) []queue.Data {
|
||||
for _, datum := range data {
|
||||
opts := datum.(issueNotificationOpts)
|
||||
func handler(items ...issueNotificationOpts) []issueNotificationOpts {
|
||||
for _, opts := range items {
|
||||
if err := activities_model.CreateOrUpdateIssueNotifications(opts.IssueID, opts.CommentID, opts.NotificationAuthorID, opts.ReceiverID); err != nil {
|
||||
log.Error("Was unable to create issue notification: %v", err)
|
||||
}
|
||||
|
|
@ -52,7 +51,7 @@ func (ns *notificationService) handle(data ...queue.Data) []queue.Data {
|
|||
}
|
||||
|
||||
func (ns *notificationService) Run() {
|
||||
graceful.GetManager().RunWithShutdownFns(ns.issueQueue.Run)
|
||||
go graceful.GetManager().RunWithShutdownFns(ns.issueQueue.Run)
|
||||
}
|
||||
|
||||
func (ns *notificationService) NotifyCreateIssueComment(ctx context.Context, doer *user_model.User, repo *repo_model.Repository,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,63 @@
|
|||
// Copyright 2023 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package queue
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
backoffBegin = 50 * time.Millisecond
|
||||
backoffUpper = 2 * time.Second
|
||||
)
|
||||
|
||||
type (
|
||||
backoffFuncRetErr[T any] func() (retry bool, ret T, err error)
|
||||
backoffFuncErr func() (retry bool, err error)
|
||||
)
|
||||
|
||||
func backoffRetErr[T any](ctx context.Context, begin, upper time.Duration, end <-chan time.Time, fn backoffFuncRetErr[T]) (ret T, err error) {
|
||||
d := begin
|
||||
for {
|
||||
// check whether the context has been cancelled or has reached the deadline, return early
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ret, ctx.Err()
|
||||
case <-end:
|
||||
return ret, context.DeadlineExceeded
|
||||
default:
|
||||
}
|
||||
|
||||
// call the target function
|
||||
retry, ret, err := fn()
|
||||
if err != nil {
|
||||
return ret, err
|
||||
}
|
||||
if !retry {
|
||||
return ret, nil
|
||||
}
|
||||
|
||||
// wait for a while before retrying, and also respect the context & deadline
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ret, ctx.Err()
|
||||
case <-time.After(d):
|
||||
d *= 2
|
||||
if d > upper {
|
||||
d = upper
|
||||
}
|
||||
case <-end:
|
||||
return ret, context.DeadlineExceeded
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func backoffErr(ctx context.Context, begin, upper time.Duration, end <-chan time.Time, fn backoffFuncErr) error {
|
||||
_, err := backoffRetErr(ctx, begin, upper, end, func() (retry bool, ret any, err error) {
|
||||
retry, err = fn()
|
||||
return retry, nil, err
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
// Copyright 2023 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package queue
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
)
|
||||
|
||||
var pushBlockTime = 5 * time.Second
|
||||
|
||||
type baseQueue interface {
|
||||
PushItem(ctx context.Context, data []byte) error
|
||||
PopItem(ctx context.Context) ([]byte, error)
|
||||
HasItem(ctx context.Context, data []byte) (bool, error)
|
||||
Len(ctx context.Context) (int, error)
|
||||
Close() error
|
||||
RemoveAll(ctx context.Context) error
|
||||
}
|
||||
|
||||
func popItemByChan(ctx context.Context, popItemFn func(ctx context.Context) ([]byte, error)) (chanItem chan []byte, chanErr chan error) {
|
||||
chanItem = make(chan []byte)
|
||||
chanErr = make(chan error)
|
||||
go func() {
|
||||
for {
|
||||
it, err := popItemFn(ctx)
|
||||
if err != nil {
|
||||
close(chanItem)
|
||||
chanErr <- err
|
||||
return
|
||||
}
|
||||
if it == nil {
|
||||
close(chanItem)
|
||||
close(chanErr)
|
||||
return
|
||||
}
|
||||
chanItem <- it
|
||||
}
|
||||
}()
|
||||
return chanItem, chanErr
|
||||
}
|
||||
|
|
@ -0,0 +1,123 @@
|
|||
// Copyright 2023 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package queue
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"code.gitea.io/gitea/modules/container"
|
||||
)
|
||||
|
||||
var errChannelClosed = errors.New("channel is closed")
|
||||
|
||||
type baseChannel struct {
|
||||
c chan []byte
|
||||
set container.Set[string]
|
||||
mu sync.Mutex
|
||||
|
||||
isUnique bool
|
||||
}
|
||||
|
||||
var _ baseQueue = (*baseChannel)(nil)
|
||||
|
||||
func newBaseChannelGeneric(cfg *BaseConfig, unique bool) (baseQueue, error) {
|
||||
q := &baseChannel{c: make(chan []byte, cfg.Length), isUnique: unique}
|
||||
if unique {
|
||||
q.set = container.Set[string]{}
|
||||
}
|
||||
return q, nil
|
||||
}
|
||||
|
||||
func newBaseChannelSimple(cfg *BaseConfig) (baseQueue, error) {
|
||||
return newBaseChannelGeneric(cfg, false)
|
||||
}
|
||||
|
||||
func newBaseChannelUnique(cfg *BaseConfig) (baseQueue, error) {
|
||||
return newBaseChannelGeneric(cfg, true)
|
||||
}
|
||||
|
||||
func (q *baseChannel) PushItem(ctx context.Context, data []byte) error {
|
||||
if q.c == nil {
|
||||
return errChannelClosed
|
||||
}
|
||||
|
||||
if q.isUnique {
|
||||
q.mu.Lock()
|
||||
has := q.set.Contains(string(data))
|
||||
q.mu.Unlock()
|
||||
if has {
|
||||
return ErrAlreadyInQueue
|
||||
}
|
||||
}
|
||||
|
||||
select {
|
||||
case q.c <- data:
|
||||
if q.isUnique {
|
||||
q.mu.Lock()
|
||||
q.set.Add(string(data))
|
||||
q.mu.Unlock()
|
||||
}
|
||||
return nil
|
||||
case <-time.After(pushBlockTime):
|
||||
return context.DeadlineExceeded
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
func (q *baseChannel) PopItem(ctx context.Context) ([]byte, error) {
|
||||
select {
|
||||
case data, ok := <-q.c:
|
||||
if !ok {
|
||||
return nil, errChannelClosed
|
||||
}
|
||||
q.mu.Lock()
|
||||
q.set.Remove(string(data))
|
||||
q.mu.Unlock()
|
||||
return data, nil
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
func (q *baseChannel) HasItem(ctx context.Context, data []byte) (bool, error) {
|
||||
q.mu.Lock()
|
||||
defer q.mu.Unlock()
|
||||
|
||||
return q.set.Contains(string(data)), nil
|
||||
}
|
||||
|
||||
func (q *baseChannel) Len(ctx context.Context) (int, error) {
|
||||
q.mu.Lock()
|
||||
defer q.mu.Unlock()
|
||||
|
||||
if q.c == nil {
|
||||
return 0, errChannelClosed
|
||||
}
|
||||
|
||||
return len(q.c), nil
|
||||
}
|
||||
|
||||
func (q *baseChannel) Close() error {
|
||||
q.mu.Lock()
|
||||
defer q.mu.Unlock()
|
||||
|
||||
close(q.c)
|
||||
q.set = container.Set[string]{}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (q *baseChannel) RemoveAll(ctx context.Context) error {
|
||||
q.mu.Lock()
|
||||
defer q.mu.Unlock()
|
||||
|
||||
for q.c != nil && len(q.c) > 0 {
|
||||
<-q.c
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
// Copyright 2023 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package queue
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestBaseChannel(t *testing.T) {
|
||||
testQueueBasic(t, newBaseChannelSimple, &BaseConfig{ManagedName: "baseChannel", Length: 10}, false)
|
||||
testQueueBasic(t, newBaseChannelUnique, &BaseConfig{ManagedName: "baseChannel", Length: 10}, true)
|
||||
}
|
||||
|
|
@ -0,0 +1,38 @@
|
|||
// Copyright 2023 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package queue
|
||||
|
||||
import "context"
|
||||
|
||||
type baseDummy struct{}
|
||||
|
||||
var _ baseQueue = (*baseDummy)(nil)
|
||||
|
||||
func newBaseDummy(cfg *BaseConfig, unique bool) (baseQueue, error) {
|
||||
return &baseDummy{}, nil
|
||||
}
|
||||
|
||||
func (q *baseDummy) PushItem(ctx context.Context, data []byte) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (q *baseDummy) PopItem(ctx context.Context) ([]byte, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (q *baseDummy) Len(ctx context.Context) (int, error) {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
func (q *baseDummy) HasItem(ctx context.Context, data []byte) (bool, error) {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
func (q *baseDummy) Close() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (q *baseDummy) RemoveAll(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
|
@ -0,0 +1,72 @@
|
|||
// Copyright 2023 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package queue
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"code.gitea.io/gitea/modules/nosql"
|
||||
|
||||
"gitea.com/lunny/levelqueue"
|
||||
)
|
||||
|
||||
type baseLevelQueue struct {
|
||||
internal *levelqueue.Queue
|
||||
conn string
|
||||
cfg *BaseConfig
|
||||
}
|
||||