diff --git a/.forgejo/upgrades/default-app.ini b/.forgejo/upgrades/default-app.ini index 8ae71431ea..a51290a8b2 100644 --- a/.forgejo/upgrades/default-app.ini +++ b/.forgejo/upgrades/default-app.ini @@ -13,7 +13,7 @@ PATH = ${WORK_PATH}/forgejo.db [log] MODE = file -LEVEL = debug +LEVEL = trace ROUTER = file [log.file] @@ -22,5 +22,9 @@ FILE_NAME = forgejo.log [security] INSTALL_LOCK = true +[repository] +ENABLE_PUSH_CREATE_USER = true +DEFAULT_PUSH_CREATE_PRIVATE = false + [actions] ENABLED = true diff --git a/.forgejo/upgrades/fixtures.sh b/.forgejo/upgrades/fixtures.sh new file mode 100644 index 0000000000..1b8cb44cac --- /dev/null +++ b/.forgejo/upgrades/fixtures.sh @@ -0,0 +1,187 @@ +#!/bin/bash +# SPDX-License-Identifier: MIT + +#ONEPIXEL="iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==" +# +# one pixel scaled to 290x290 because that's what versions lower or equal to v1.19.4-0 want +# by default and any other size will be transformed which make it difficult to compare. +# +ONEPIXEL="iVBORw0KGgoAAAANSUhEUgAAASIAAAEiCAYAAABdvt+2AAADrElEQVR4nOzUMRHAMADEsL9eeQd6AsOLhMCT/7udAYS+OgDAiICcEQE5IwJyRgTkjAjIGRGQMyIgZ0RAzoiAnBEBOSMCckYE5IwIyBkRkDMiIGdEQM6IgJwRATkjAnJGBOSMCMgZEZAzIiBnREDOiICcEQE5IwJyRgTkjAjIGRGQMyIgZ0RAzoiAnBEBOSMCckYE5IwIyBkRkDMiIGdEQM6IgJwRATkjAnJGBOSMCMgZEZAzIiBnREDOiICcEQE5IwJyRgTkjAjIGRGQMyIgZ0RAzoiAnBEBOSMCckYE5IwIyBkRkDMiIGdEQM6IgJwRATkjAnJGBOSMCMgZEZAzIiBnREDOiICcEQE5IwJyRgTkjAjIGRGQMyIgZ0RAzoiAnBEBOSMCckYE5IwIyBkRkDMiIGdEQM6IgJwRATkjAnJGBOSMCMgZEZAzIiBnREDOiICcEQE5IwJyRgTkjAjIGRGQMyIgZ0RAzoiAnBEBOSMCckYE5IwIyBkRkDMiIGdEQM6IgJwRATkjAnJGBOSMCMgZEZAzIiBnREDOiICcEQE5IwJyRgTkjAjIGRGQMyIgZ0RAzoiAnBEBOSMCckYE5IwIyBkRkDMiIGdEQM6IgJwRATkjAnJGBOSMCMgZEZAzIiBnREDOiICcEQE5IwJyRgTkjAjIGRGQMyIgZ0RAzoiAnBEBOSMCckYE5IwIyBkRkDMiIGdEQM6IgJwRATkjAnJGBOSMCMgZEZAzIiBnREDOiICcEQE5IwJyRgTkjAjIGRGQMyIgZ0RAzoiAnBEBOSMCckYE5IwIyBkRkDMiIGdEQM6IgJwRATkjAnJGBOSMCMgZEZAzIiBnREDOiICcEQE5IwJyRgTkjAjIGRGQMyIgZ0RAzoiAnBEBOSMCckYE5IwIyBkRkDMiIGdEQM6IgJwRATkjAnJGBOSMCMgZEZAzIiBnREDOiICcEQE5IwJyRgTkjAjIGRGQMyIgZ0RAzoiAnBEBOSMCckYE5IwIyBkRkDMiIGdEQM6IgJwRATkjAnJGBOSMCMgZEZAzIiBnREDOiICcEQE5IwJyRgTkjAjIGRGQMyIgZ0RAzoiAnBEBOSMCckYE5IwIyBkRkDMiIGdEQM6IgJwRATkjAnJGBOSMCMgZEZAzIiBnREDOiICcEQE5IwJyRgTkjAjIGRGQMyIgZ0RAzoiAnBEBOSMCckYE5IwIyBkRkDMiIGdEQM6IgJwRATkjAnJGBOSMCMgZEZAzIiBnREDuBQAA//+4jAPFe1H1tgAAAABJRU5ErkJggg==" + +function fixture_get_paths_s3() { + local path=$1 + + ( + echo -n $path/ + mc ls --quiet --recursive testS3/$path | sed -e 's/.* //' + ) > $DIR/path +} + +function fixture_get_paths_local() { + local path=$1 + local work_path=$DIR/forgejo-work-path + + ( cd $work_path ; find $path -type f) > $DIR/path +} + +function fixture_get_one_path() { + local storage=$1 + local path=$2 + + fixture_get_paths_$storage $path + + if test $(wc -l < $DIR/path) != 1 ; then + echo expected one path but got + cat $DIR/path + return 1 + fi + cat $DIR/path +} + +function fixture_repo_archive_create() { + retry curl -f -sS http://${HOST_PORT}/root/fixture/archive/main.zip -o /dev/null +} + +function fixture_repo_archive_assert_s3() { + mc ls --recursive testS3/forgejo/repo-archive | grep --quiet '.zip$' +} + +function fixture_repo_archive_assert_local() { + local path=$1 + local work_path=$DIR/forgejo-work-path + + find $work_path/$path | grep --quiet '.zip$' +} + +function fixture_lfs_create() { + ( + cd $DIR/fixture + git lfs track "*.txt" + echo CONTENT > file.txt + git add . + git commit -m 'lfs files' + git push + ) +} + +function fixture_lfs_assert_s3() { + local content=$(mc cat testS3/forgejo/lfs/d6/1e/5fa787e50330288923bd0c9866b44643925965144262288447cf52f9f9b7) + test "$content" = CONTENT +} + +function fixture_lfs_assert_local() { + local path=$1 + local work_path=$DIR/forgejo-work-path + + local content=$(mc cat $work_path/$path/d6/1e/5fa787e50330288923bd0c9866b44643925965144262288447cf52f9f9b7) + test "$content" = CONTENT +} + +function fixture_packages_create() { + echo PACKAGE_CONTENT > $DIR/fixture/package + $work_path/forgejo-api -X DELETE http://${HOST_PORT}/api/packages/${FORGEJO_USER}/generic/test_package/1.0.0/file.txt || true + $work_path/forgejo-api --upload-file $DIR/fixture/package http://${HOST_PORT}/api/packages/${FORGEJO_USER}/generic/test_package/1.0.0/file.txt +} + +function fixture_packages_assert_s3() { + local content=$(mc cat testS3/forgejo/packages/aa/cf/aacf02e660d813e95d2854e27926ba1ad5c87299dc5f7661d5f08f076c6bbc17) + test "$content" = PACKAGE_CONTENT +} + +function fixture_packages_assert_local() { + local path=$1 + + local content=$(cat $work_path/$path/aa/cf/aacf02e660d813e95d2854e27926ba1ad5c87299dc5f7661d5f08f076c6bbc17) + test "$content" = PACKAGE_CONTENT +} + +function fixture_avatars_create() { + echo -n $ONEPIXEL | base64 --decode > $DIR/avatar.png + $work_path/forgejo-client --form avatar=@$DIR/avatar.png http://${HOST_PORT}/user/settings/avatar +} + +function fixture_avatars_assert_s3() { + local filename=$(fixture_get_one_path s3 forgejo/avatars) + local content=$(mc cat testS3/$filename | base64 -w0) + test "$content" = "$ONEPIXEL" +} + +function fixture_avatars_assert_local() { + local path=$1 + + local filename=$(fixture_get_one_path local $path) + local content=$(cat $work_path/$filename | base64 -w0) + test "$content" = "$ONEPIXEL" +} + +function fixture_repo_avatars_create() { + echo -n $ONEPIXEL | base64 --decode > $DIR/repo-avatar.png + $work_path/forgejo-client --form avatar=@$DIR/repo-avatar.png http://${HOST_PORT}/${FORGEJO_USER}/${FORGEJO_REPO}/settings/avatar + # v1.21 only + #$work_path/forgejo-api -X POST --data-raw '{"body":"'$avatar'"}' http://${HOST_PORT}/api/v1/repos/${FORGEJO_USER}/${FORGEJO_REPO}/avatar +} + +function fixture_repo_avatars_assert_s3() { + local filename=$(fixture_get_one_path s3 forgejo/repo-avatars) + local content=$(mc cat testS3/$filename | base64 -w0) + test "$content" = "$ONEPIXEL" +} + +function fixture_repo_avatars_assert_local() { + local path=$1 + + local filename=$(fixture_get_one_path local $path) + local content=$(cat $work_path/$filename | base64 -w0) + test "$content" = "$ONEPIXEL" +} + +function fixture_attachments_create_1_18() { + echo -n $ONEPIXEL | base64 --decode > $DIR/attachment.png + $work_path/forgejo-client --trace-ascii - --form file=@$DIR/attachment.png http://${HOST_PORT}/${FORGEJO_USER}/${FORGEJO_REPO}/issues/attachments +} + +function fixture_attachments_create() { + if $work_path/forgejo-api http://${HOST_PORT}/api/v1/version | grep --quiet --fixed-strings 1.18. ; then + fixture_attachments_create_1_18 + return + fi + id=$($work_path/forgejo-api --data-raw '{"title":"TITLE"}' http://${HOST_PORT}/api/v1/repos/${FORGEJO_USER}/${FORGEJO_REPO}/issues | jq .id) + echo -n $ONEPIXEL | base64 --decode > $DIR/attachment.png + $work_path/forgejo-client -H @$DIR/forgejo-work-path/forgejo-header --form name=attachment.png --form attachment=@$DIR/attachment.png http://${HOST_PORT}/api/v1/repos/${FORGEJO_USER}/${FORGEJO_REPO}/issues/$id/assets +} + +function fixture_attachments_assert_s3() { + local filename=$(fixture_get_one_path s3 forgejo/attachments) + local content=$(mc cat testS3/$filename | base64 -w0) + test "$content" = "$ONEPIXEL" +} + +function fixture_attachments_assert_local() { + local path=$1 + + local filename=$(fixture_get_one_path local $path) + local content=$(cat $work_path/$filename | base64 -w0) + test "$content" = "$ONEPIXEL" +} + +function fixture_create() { + local work_path=$DIR/forgejo-work-path + + rm -fr $DIR/fixture + mkdir -p $DIR/fixture + ( + cd $DIR/fixture + git init + git checkout -b main + git remote add origin http://${FORGEJO_USER}:${FORGEJO_PASSWORD}@${HOST_PORT}/${FORGEJO_USER}/${FORGEJO_REPO} + git config user.email root@example.com + git config user.name username + echo SOMETHING > README + git add README + git commit -m 'initial commit' + git push --set-upstream --force origin main + ) + for fun in ${STORAGE_FUN} ; do + fixture_${fun}_create + done +} diff --git a/.forgejo/upgrades/legagy-relative-app.ini b/.forgejo/upgrades/legagy-relative-app.ini index 078b0da671..130294a4aa 100644 --- a/.forgejo/upgrades/legagy-relative-app.ini +++ b/.forgejo/upgrades/legagy-relative-app.ini @@ -23,6 +23,10 @@ FILE_NAME = forgejo.log [security] INSTALL_LOCK = true +[repository] +ENABLE_PUSH_CREATE_USER = true +DEFAULT_PUSH_CREATE_PRIVATE = false + [picture] AVATAR_UPLOAD_PATH = relative-avatars REPOSITORY_AVATAR_UPLOAD_PATH = relative-repo-avatars diff --git a/.forgejo/upgrades/merged-app.ini b/.forgejo/upgrades/merged-app.ini index b812f1f042..0b7150c077 100644 --- a/.forgejo/upgrades/merged-app.ini +++ b/.forgejo/upgrades/merged-app.ini @@ -21,6 +21,10 @@ FILE_NAME = forgejo.log [security] INSTALL_LOCK = true +[repository] +ENABLE_PUSH_CREATE_USER = true +DEFAULT_PUSH_CREATE_PRIVATE = false + [actions] ENABLED = true diff --git a/.forgejo/upgrades/misplace-app.ini b/.forgejo/upgrades/misplace-app.ini index 42ebaab0d4..0aeff45319 100644 --- a/.forgejo/upgrades/misplace-app.ini +++ b/.forgejo/upgrades/misplace-app.ini @@ -21,6 +21,10 @@ FILE_NAME = forgejo.log [security] INSTALL_LOCK = true +[repository] +ENABLE_PUSH_CREATE_USER = true +DEFAULT_PUSH_CREATE_PRIVATE = false + [actions] ENABLED = true diff --git a/.forgejo/upgrades/misplace-s3-app.ini b/.forgejo/upgrades/misplace-s3-app.ini new file mode 100644 index 0000000000..d9243dd250 --- /dev/null +++ b/.forgejo/upgrades/misplace-s3-app.ini @@ -0,0 +1,89 @@ +RUN_MODE = prod +WORK_PATH = ${WORK_PATH} + +[server] +APP_DATA_PATH = ${WORK_PATH}/elsewhere +HTTP_PORT = 3000 +SSH_LISTEN_PORT = 2222 +LFS_START_SERVER = true + +[database] +DB_TYPE = sqlite3 + +[log] +MODE = file +LEVEL = debug +ROUTER = file + +[log.file] +FILE_NAME = forgejo.log + +[security] +INSTALL_LOCK = true + +[repository] +ENABLE_PUSH_CREATE_USER = true +DEFAULT_PUSH_CREATE_PRIVATE = false + +[actions] +ENABLED = true + +[attachment] +STORAGE_TYPE = minio +SERVE_DIRECT = false +MINIO_ENDPOINT = 127.0.0.1:9000 +MINIO_ACCESS_KEY_ID = 123456 +MINIO_SECRET_ACCESS_KEY = 12345678 +MINIO_BUCKET = forgejo +MINIO_LOCATION = us-east-1 +MINIO_USE_SSL = false + +[lfs] +STORAGE_TYPE = minio +SERVE_DIRECT = false +MINIO_ENDPOINT = 127.0.0.1:9000 +MINIO_ACCESS_KEY_ID = 123456 +MINIO_SECRET_ACCESS_KEY = 12345678 +MINIO_BUCKET = forgejo +MINIO_LOCATION = us-east-1 +MINIO_USE_SSL = false + +[repo-avatar] +STORAGE_TYPE = minio +SERVE_DIRECT = false +MINIO_ENDPOINT = 127.0.0.1:9000 +MINIO_ACCESS_KEY_ID = 123456 +MINIO_SECRET_ACCESS_KEY = 12345678 +MINIO_BUCKET = forgejo +MINIO_LOCATION = us-east-1 +MINIO_USE_SSL = false + +[avatar] +STORAGE_TYPE = minio +SERVE_DIRECT = false +MINIO_ENDPOINT = 127.0.0.1:9000 +MINIO_ACCESS_KEY_ID = 123456 +MINIO_SECRET_ACCESS_KEY = 12345678 +MINIO_BUCKET = forgejo +MINIO_LOCATION = us-east-1 +MINIO_USE_SSL = false + +[repo-archive] +STORAGE_TYPE = minio +SERVE_DIRECT = false +MINIO_ENDPOINT = 127.0.0.1:9000 +MINIO_ACCESS_KEY_ID = 123456 +MINIO_SECRET_ACCESS_KEY = 12345678 +MINIO_BUCKET = forgejo +MINIO_LOCATION = us-east-1 +MINIO_USE_SSL = false + +[packages] +STORAGE_TYPE = minio +SERVE_DIRECT = false +MINIO_ENDPOINT = 127.0.0.1:9000 +MINIO_ACCESS_KEY_ID = 123456 +MINIO_SECRET_ACCESS_KEY = 12345678 +MINIO_BUCKET = forgejo +MINIO_LOCATION = us-east-1 +MINIO_USE_SSL = false diff --git a/.forgejo/upgrades/relative-app.ini b/.forgejo/upgrades/relative-app.ini index b587500dcf..d53c291780 100644 --- a/.forgejo/upgrades/relative-app.ini +++ b/.forgejo/upgrades/relative-app.ini @@ -21,6 +21,10 @@ FILE_NAME = forgejo.log [security] INSTALL_LOCK = true +[repository] +ENABLE_PUSH_CREATE_USER = true +DEFAULT_PUSH_CREATE_PRIVATE = false + [attachment] PATH = relative-attachments diff --git a/.forgejo/upgrades/specific-app.ini b/.forgejo/upgrades/specific-app.ini index c4e36e1e99..d7a0badc13 100644 --- a/.forgejo/upgrades/specific-app.ini +++ b/.forgejo/upgrades/specific-app.ini @@ -21,6 +21,10 @@ FILE_NAME = forgejo.log [security] INSTALL_LOCK = true +[repository] +ENABLE_PUSH_CREATE_USER = true +DEFAULT_PUSH_CREATE_PRIVATE = false + [actions] ENABLED = true diff --git a/.forgejo/upgrades/stable-s3-app.ini b/.forgejo/upgrades/stable-s3-app.ini new file mode 100644 index 0000000000..e8c48ae2e1 --- /dev/null +++ b/.forgejo/upgrades/stable-s3-app.ini @@ -0,0 +1,39 @@ +RUN_MODE = prod +WORK_PATH = ${WORK_PATH} + +[server] +APP_DATA_PATH = ${WORK_PATH}/elsewhere +HTTP_PORT = 3000 +SSH_LISTEN_PORT = 2222 +LFS_START_SERVER = true + +[database] +DB_TYPE = sqlite3 + +[log] +MODE = file +LEVEL = debug +ROUTER = file + +[log.file] +FILE_NAME = forgejo.log + +[security] +INSTALL_LOCK = true + +[repository] +ENABLE_PUSH_CREATE_USER = true +DEFAULT_PUSH_CREATE_PRIVATE = false + +[actions] +ENABLED = true + +[storage] +STORAGE_TYPE = minio +SERVE_DIRECT = false +MINIO_ENDPOINT = 127.0.0.1:9000 +MINIO_ACCESS_KEY_ID = 123456 +MINIO_SECRET_ACCESS_KEY = 12345678 +MINIO_BUCKET = forgejo +MINIO_LOCATION = us-east-1 +MINIO_USE_SSL = false diff --git a/.forgejo/upgrades/storage-relative-app.ini b/.forgejo/upgrades/storage-relative-app.ini index c413811424..eba5232dd0 100644 --- a/.forgejo/upgrades/storage-relative-app.ini +++ b/.forgejo/upgrades/storage-relative-app.ini @@ -21,6 +21,10 @@ FILE_NAME = forgejo.log [security] INSTALL_LOCK = true +[repository] +ENABLE_PUSH_CREATE_USER = true +DEFAULT_PUSH_CREATE_PRIVATE = false + [storage.attachments] PATH = relative-attachments diff --git a/.forgejo/upgrades/test-upgrade.sh b/.forgejo/upgrades/test-upgrade.sh index 3630907b85..53d0cf7fb9 100755 --- a/.forgejo/upgrades/test-upgrade.sh +++ b/.forgejo/upgrades/test-upgrade.sh @@ -6,29 +6,53 @@ # # ./.forgejo/upgrades/test-upgrade.sh dependencies # ./.forgejo/upgrades/test-upgrade.sh build_all -# ./.forgejo/upgrades/test-upgrade.sh test_downgrade_1.20.2_fails +# VERBOSE=true ./.forgejo/upgrades/test-upgrade.sh test_downgrade_1.20.2_fails # # Everything happens in /tmp/forgejo-upgrades # -set -ex - +PREFIX=============== HOST_PORT=0.0.0.0:3000 STORAGE_PATHS="attachments avatars lfs packages repo-archive repo-avatars" +STORAGE_FUN="attachments avatars lfs packages repo_archive repo_avatars" DIR=/tmp/forgejo-upgrades +if ${VERBOSE:-false} ; then + set -ex + PS4='${BASH_SOURCE[0]}:$LINENO: ${FUNCNAME[0]}: ' +else + set -e +fi SELF_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -PS4='${BASH_SOURCE[0]}:$LINENO: ${FUNCNAME[0]}: ' +: ${FORGEJO_USER:=root} +: ${FORGEJO_REPO:=fixture} +: ${FORGEJO_PASSWORD:=admin1234} + +source $SELF_DIR/fixtures.sh function maybe_sudo() { if test $(id -u) != 0 ; then - SUDO=sudo + SUDO=sudo fi } +function log_info() { + echo "$PREFIX $@" +} + function dependencies() { - if ! which curl daemon > /dev/null ; then - maybe_sudo - $SUDO apt-get install -y -qq curl daemon + maybe_sudo + if ! which curl daemon jq git-lfs > /dev/null ; then + $SUDO apt-get install -y -qq curl daemon git-lfs jq + fi + if ! which minio mc > /dev/null ; then + $SUDO curl -sS https://dl.min.io/client/mc/release/linux-amd64/mc -o /usr/local/bin/mc + $SUDO chmod +x /usr/local/bin/mc + $SUDO curl -sS https://dl.min.io/server/minio/release/linux-amd64/archive/minio.RELEASE.2023-08-23T10-07-06Z -o /usr/local/bin/minio + $SUDO chmod +x /usr/local/bin/minio + fi + if ! which garage > /dev/null ; then + $SUDO curl -sS https://garagehq.deuxfleurs.fr/_releases/v0.8.2/x86_64-unknown-linux-musl/garage -o /usr/local/bin/garage + $SUDO chmod +x /usr/local/bin/garage fi } @@ -37,9 +61,9 @@ function build() { local semver=$2 if ! test -f $DIR/forgejo-$version ; then - mkdir -p $DIR - make VERSION=v$version GITEA_VERSION=v$version FORGEJO_VERSION=$semver TAGS='bindata sqlite sqlite_unlock_notify' generate gitea - mv gitea $DIR/forgejo-$version + mkdir -p $DIR + make VERSION=v$version GITEA_VERSION=v$version FORGEJO_VERSION=$semver TAGS='bindata sqlite sqlite_unlock_notify' generate gitea + mv gitea $DIR/forgejo-$version fi } @@ -48,21 +72,21 @@ function build_all() { build 1.21.0-0 6.0.0+0-gitea-1.21.0 } -function wait_for() { - rm -f $DIR/setup-forgejo.out +function retry() { + rm -f $DIR/wait-for.out success=false for delay in 1 1 5 5 15 ; do - if "$@" >> $DIR/setup-forgejo.out 2>&1 ; then - success=true - break - fi - cat $DIR/setup-forgejo.out - echo waiting $delay - sleep $delay + if "$@" >> $DIR/wait-for.out 2>&1 ; then + success=true + break + fi + cat $DIR/wait-for.out + echo waiting $delay + sleep $delay done if test $success = false ; then - cat $DIR/setup-forgejo.out - return 1 + cat $DIR/wait-for.out + return 1 fi } @@ -70,9 +94,9 @@ function download() { local version=$1 if ! test -f $DIR/forgejo-$version ; then - mkdir -p $DIR - wget -O $DIR/forgejo-$version --quiet https://codeberg.org/forgejo/forgejo/releases/download/v$version/forgejo-$version-linux-amd64 - chmod +x $DIR/forgejo-$version + mkdir -p $DIR + wget -O $DIR/forgejo-$version --quiet https://codeberg.org/forgejo/forgejo/releases/download/v$version/forgejo-$version-linux-amd64 + chmod +x $DIR/forgejo-$version fi } @@ -83,19 +107,92 @@ function cleanup_logs() { rm -f $work_path/log/*.log } -function start() { +function clobber() { + rm -fr /tmp/forgejo-upgrades +} + +function start_forgejo() { local version=$1 download $version local work_path=$DIR/forgejo-work-path daemon --chdir=$DIR --unsafe --env="TERM=$TERM" --env="HOME=$HOME" --env="PATH=$PATH" --pidfile=$DIR/forgejo-pid --errlog=$DIR/forgejo-err.log --output=$DIR/forgejo-out.log -- $DIR/forgejo-$version --config $work_path/app.ini --work-path $work_path - if ! wait_for grep 'Starting server on' $work_path/log/forgejo.log ; then - cat $DIR/*.log - cat $work_path/log/*.log - return 1 + if ! retry grep 'Starting server on' $work_path/log/forgejo.log ; then + cat $DIR/*.log + cat $work_path/log/*.log + return 1 fi create_user $version - $work_path/forgejo-api http://${HOST_PORT}/api/v1/version +} + +function start_minio() { + mkdir -p $DIR/minio + daemon --chdir=$DIR --unsafe \ + --env="PATH=$PATH" \ + --env=MINIO_ROOT_USER=123456 \ + --env=MINIO_ROOT_PASSWORD=12345678 \ + --env=MINIO_VOLUMES=$DIR/minio \ + --pidfile=$DIR/minio-pid --errlog=$DIR/minio-err.log --output=$DIR/minio-out.log -- /usr/local/bin/minio server + retry mc alias set testS3 http://127.0.0.1:9000 123456 12345678 +} + +function start_garage() { + mkdir -p $DIR/garage/{data,meta} + cat > $DIR/garage/garage.toml < $work_path/forgejo-token + + # + # forgejo-cli is to use with api/v1 enpoints + # + # tail -1 is because there are logs creating noise in the output in v1.19.4-0 + # + $cli admin user generate-access-token -u $FORGEJO_USER --raw $scopes | tail -1 > $work_path/forgejo-token ( echo -n 'Authorization: token ' ; cat $work_path/forgejo-token ) > $work_path/forgejo-header - ( echo "#!/bin/sh" ; echo 'curl -sS -H "Content-Type: application/json" -H @'$work_path/forgejo-header' "$@"' ) > $work_path/forgejo-api && chmod +x $work_path/forgejo-api + ( echo "#!/bin/sh" ; echo 'curl -f -sS -H "Content-Type: application/json" -H @'$work_path/forgejo-header' "$@"' ) > $work_path/forgejo-api && chmod +x $work_path/forgejo-api + $work_path/forgejo-api http://${HOST_PORT}/api/v1/version + + # + # forgejo-client is to use with web endpoints + # + # + # login and obtain a CSRF, all stored in the cookie file + # + ( echo "#!/bin/sh" ; echo 'curl --cookie-jar '$DIR/cookies' --cookie '$DIR/cookies' -f -sS "$@"' ) > $work_path/forgejo-client-update-cookies && chmod +x $work_path/forgejo-client-update-cookies + $work_path/forgejo-client-update-cookies http://${HOST_PORT}/user/login -o /dev/null + $work_path/forgejo-client-update-cookies --verbose -X POST --data user_name=${FORGEJO_USER} --data password=${FORGEJO_PASSWORD} http://${HOST_PORT}/user/login >& $DIR/login.html + $work_path/forgejo-client-update-cookies http://${HOST_PORT}/user/login -o /dev/null + local csrf=$(sed -n -e '/csrf/s/.*csrf\t//p' $DIR/cookies) + # + # use the cookie file but do not modify it + # + ( echo "#!/bin/sh" ; echo 'curl --cookie '$DIR/cookies' -H "X-Csrf-Token: '$csrf'" -f -sS "$@"' ) > $work_path/forgejo-client && chmod +x $work_path/forgejo-client +} + +function stop_daemon() { + local daemon=$1 + + if test -f $DIR/$daemon-pid ; then + local pid=$(cat $DIR/$daemon-pid) + kill -TERM $pid + pidwait $pid || true + for delay in 1 1 2 2 5 5 ; do + if ! test -f $DIR/$daemon-pid ; then + break + fi + sleep $delay + done + ! test -f $DIR/$daemon-pid + fi } function stop() { - if test -f $DIR/forgejo-pid ; then - local pid=$(cat $DIR/forgejo-pid) - kill -TERM $pid - pidwait $pid || true - for delay in 1 1 2 2 5 5 ; do - if ! test -f $DIR/forgejo-pid ; then - break - fi - sleep $delay - done - ! test -f $DIR/forgejo-pid - fi + stop_daemon forgejo + stop_daemon minio + stop_daemon garage + cleanup_logs } -function reset() { +function reset_forgejo() { local config=$1 local work_path=$DIR/forgejo-work-path rm -fr $work_path @@ -144,11 +271,26 @@ function reset() { WORK_PATH=$work_path envsubst < $SELF_DIR/$config-app.ini > $work_path/app.ini } +function reset_minio() { + rm -fr $DIR/minio +} + +function reset_garage() { + rm -fr $DIR/garage +} + +function reset() { + local config=$1 + reset_forgejo $config + reset_minio + reset_garage +} + function verify_storage() { local work_path=$DIR/forgejo-work-path for path in ${STORAGE_PATHS} ; do - test -d $work_path/data/$path + test -d $work_path/data/$path done } @@ -156,16 +298,16 @@ function cleanup_storage() { local work_path=$DIR/forgejo-work-path for path in ${STORAGE_PATHS} ; do - rm -fr $work_path/data/$path + rm -fr $work_path/data/$path done } function test_downgrade_1.20.2_fails() { local work_path=$DIR/forgejo-work-path - echo "================ See also https://codeberg.org/forgejo/forgejo/pulls/1225" + log_info "See also https://codeberg.org/forgejo/forgejo/pulls/1225" - echo "================ downgrading from 1.20.3-0 to 1.20.2-0 fails" + log_info "downgrading from 1.20.3-0 to 1.20.2-0 fails" stop reset default start 1.20.3-0 @@ -173,192 +315,270 @@ function test_downgrade_1.20.2_fails() { download 1.20.2-0 timeout 60 $DIR/forgejo-1.20.2-0 --config $work_path/app.ini --work-path $work_path || true if ! grep --fixed-strings --quiet 'use the newer database' $work_path/log/forgejo.log ; then - cat $work_path/log/forgejo.log - return 1 + cat $work_path/log/forgejo.log + return 1 fi } function test_bug_storage_merged() { local work_path=$DIR/forgejo-work-path - echo "================ See also https://codeberg.org/forgejo/forgejo/pulls/1225" + log_info "See also https://codeberg.org/forgejo/forgejo/pulls/1225" - echo "================ using < 1.20.3-0 and [storage].PATH merge all storage" + log_info "using < 1.20.3-0 and [storage].PATH merge all storage" for version in 1.18.5-0 1.19.4-0 1.20.2-0 ; do - stop - reset merged - start $version - for path in ${STORAGE_PATHS} ; do - ! test -d $work_path/data/$path - done - for path in ${STORAGE_PATHS} ; do - ! test -d $work_path/merged/$path - done - test -d $work_path/merged + stop + reset merged + start $version + for path in ${STORAGE_PATHS} ; do + ! test -d $work_path/data/$path + done + for path in ${STORAGE_PATHS} ; do + ! test -d $work_path/merged/$path + done + test -d $work_path/merged done stop - echo "================ upgrading from 1.20.2-0 with [storage].PATH fails" + log_info "upgrading from 1.20.2-0 with [storage].PATH fails" download 1.20.3-0 timeout 60 $DIR/forgejo-1.20.3-0 --config $work_path/app.ini --work-path $work_path || true if ! grep --fixed-strings --quiet '[storage].PATH is set and may create storage issues' $work_path/log/forgejo.log ; then - cat $work_path/log/forgejo.log - return 1 + cat $work_path/log/forgejo.log + return 1 fi } function test_bug_storage_relative_path() { local work_path=$DIR/forgejo-work-path - echo "================ using < 1.20.3-0 legacy [server].XXXX and [picture].XXXX are relative to WORK_PATH" + log_info "using < 1.20.3-0 legacy [server].XXXX and [picture].XXXX are relative to WORK_PATH" for version in 1.18.5-0 1.19.4-0 1.20.2-0 ; do - stop - reset legagy-relative - start $version - test -d $work_path/relative-lfs - test -d $work_path/relative-avatars - test -d $work_path/relative-repo-avatars + stop + reset legagy-relative + start $version + test -d $work_path/relative-lfs + test -d $work_path/relative-avatars + test -d $work_path/relative-repo-avatars done - echo "================ using >= 1.20.3-0 legacy [server].XXXX and [picture].XXXX are relative to APP_DATA_PATH" + log_info "using >= 1.20.3-0 legacy [server].XXXX and [picture].XXXX are relative to APP_DATA_PATH" for version in 1.20.3-0 1.21.0-0 ; do - stop - reset legagy-relative - start $version - test -d $work_path/data/relative-lfs - test -d $work_path/data/relative-avatars - test -d $work_path/data/relative-repo-avatars + stop + reset legagy-relative + start $version + test -d $work_path/data/relative-lfs + test -d $work_path/data/relative-avatars + test -d $work_path/data/relative-repo-avatars done - echo "================ using >= 1.20.3-0 relative [storage.XXXX].PATHS are relative to APP_DATA_PATH" + log_info "using >= 1.20.3-0 relative [storage.XXXX].PATHS are relative to APP_DATA_PATH" for version in 1.20.3-0 1.21.0-0 ; do - stop - reset storage-relative - start $version - for path in ${STORAGE_PATHS} ; do - test -d $work_path/data/relative-$path - done + stop + reset storage-relative + start $version + for path in ${STORAGE_PATHS} ; do + test -d $work_path/data/relative-$path + done done - echo "================ using 1.20.[12]-0 relative [storage.XXXX].PATHS are inconsistent" + log_info "using 1.20.[12]-0 relative [storage.XXXX].PATHS are inconsistent" for version in 1.20.2-0 ; do - stop - reset storage-relative - start $version - test -d $work_path/data/packages - test -d $work_path/relative-repo-archive - test -d $work_path/relative-attachments - test -d $work_path/relative-lfs - test -d $work_path/data/avatars - test -d $work_path/data/repo-avatars + stop + reset storage-relative + start $version + test -d $work_path/data/packages + test -d $work_path/relative-repo-archive + test -d $work_path/relative-attachments + test -d $work_path/relative-lfs + test -d $work_path/data/avatars + test -d $work_path/data/repo-avatars done - echo "================ using < 1.20 relative [storage.XXXX].PATHS are inconsistent" + log_info "using < 1.20 relative [storage.XXXX].PATHS are inconsistent" for version in 1.18.5-0 1.19.4-0 ; do - stop - reset storage-relative - start $version - test -d $work_path/relative-packages - test -d $work_path/relative-repo-archive - test -d $work_path/relative-attachments - test -d $work_path/data/lfs - test -d $work_path/data/avatars - test -d $work_path/data/repo-avatars + stop + reset storage-relative + start $version + test -d $work_path/relative-packages + test -d $work_path/relative-repo-archive + test -d $work_path/relative-attachments + test -d $work_path/data/lfs + test -d $work_path/data/avatars + test -d $work_path/data/repo-avatars done - echo "================ using < 1.20.3-0 relative [XXXX].PATHS are relative to WORK_PATH" + log_info "using < 1.20.3-0 relative [XXXX].PATHS are relative to WORK_PATH" for version in 1.18.5-0 1.19.4-0 1.20.2-0 ; do - stop - reset relative - start $version - for path in ${STORAGE_PATHS} ; do - test -d $work_path/relative-$path - done + stop + reset relative + start $version + for path in ${STORAGE_PATHS} ; do + test -d $work_path/relative-$path + done done - echo "================ using >= 1.20.3-0 relative [XXXX].PATHS are relative to APP_DATA_PATH" + log_info "using >= 1.20.3-0 relative [XXXX].PATHS are relative to APP_DATA_PATH" for version in 1.20.3-0 1.21.0-0 ; do - stop - reset relative - start $version - for path in ${STORAGE_PATHS} ; do - test -d $work_path/data/relative-$path - done + stop + reset relative + start $version + for path in ${STORAGE_PATHS} ; do + test -d $work_path/data/relative-$path + done done stop } +function test_bug_storage_s3_misplace() { + local work_path=$DIR/forgejo-work-path + local s3_backend=${2:-minio} + + log_info "See also https://codeberg.org/forgejo/forgejo/issues/1338" + + for version in 1.20.2-0 1.20.3-0 ; do + log_info "Forgejo $version & $s3_backend" + stop + reset misplace-s3 + start $version $s3_backend + fixture_create + for fun in ${STORAGE_FUN} ; do + fixture_${fun}_assert_s3 + done + done + + for version in 1.18.5-0 1.19.4-0 ; do + log_info "Forgejo $version & $s3_backend" + stop + reset misplace-s3 + start $version $s3_backend + fixture_create + # + # some storage are in S3 + # + fixture_attachments_assert_s3 + fixture_lfs_assert_s3 + # + # others are in local + # + fixture_repo_archive_assert_local elsewhere/repo-archive + fixture_avatars_assert_local elsewhere/avatars + fixture_packages_assert_local elsewhere/packages + fixture_repo_avatars_assert_local elsewhere/repo-avatars + done +} + +function test_storage_stable_s3() { + local work_path=$DIR/forgejo-work-path + local s3_backend=${1:-minio} + + log_info "See also https://codeberg.org/forgejo/forgejo/issues/1338" + + for version in 1.18.5-0 1.19.4-0 1.20.2-0 1.20.3-0 ; do + log_info "Forgejo $version & $s3_backend" + stop + reset stable-s3 + start $version $s3_backend + fixture_create + for fun in ${STORAGE_FUN} ; do + fixture_${fun}_assert_s3 + done + done +} + function test_bug_storage_misplace() { local work_path=$DIR/forgejo-work-path - echo "================ See also https://codeberg.org/forgejo/forgejo/pulls/1225" + log_info "See also https://codeberg.org/forgejo/forgejo/pulls/1225" - echo "================ using < 1.20 and conflicting sections misplace storage" + log_info "using < 1.20 and conflicting sections misplace storage" for version in 1.18.5-0 1.19.4-0 ; do - stop - reset misplace - start $version - # - # some storage are where they should be - # - test -d $work_path/data/packages - test -d $work_path/data/repo-archive - test -d $work_path/data/attachments - # - # others are under APP_DATA_PATH - # - test -d $work_path/elsewhere/lfs - test -d $work_path/elsewhere/avatars - test -d $work_path/elsewhere/repo-avatars + stop + reset misplace + start $version + # + # some storage are where they should be + # + test -d $work_path/data/packages + test -d $work_path/data/repo-archive + test -d $work_path/data/attachments + # + # others are under APP_DATA_PATH + # + test -d $work_path/elsewhere/lfs + test -d $work_path/elsewhere/avatars + test -d $work_path/elsewhere/repo-avatars done - echo "================ using < 1.20.[12]-0 and conflicting sections ignores [storage.*]" + log_info "using < 1.20.[12]-0 and conflicting sections ignores [storage.*]" for version in 1.20.2-0 ; do - stop - reset misplace - start $version - for path in ${STORAGE_PATHS} ; do - test -d $work_path/elsewhere/$path - done + stop + reset misplace + start $version + for path in ${STORAGE_PATHS} ; do + test -d $work_path/elsewhere/$path + done done stop - echo "================ upgrading from 1.20.2-0 with conflicting sections fails" + log_info "upgrading from 1.20.2-0 with conflicting sections fails" download 1.20.3-0 timeout 60 $DIR/forgejo-1.20.3-0 --config $work_path/app.ini --work-path $work_path || true for path in ${STORAGE_PATHS} ; do - if ! grep --fixed-strings --quiet "[storage.$path] may conflict" $work_path/log/forgejo.log ; then - cat $work_path/log/forgejo.log - return 1 - fi + if ! grep --fixed-strings --quiet "[storage.$path] may conflict" $work_path/log/forgejo.log ; then + cat $work_path/log/forgejo.log + return 1 + fi done } function test_successful_upgrades() { for config in default specific ; do - echo "================ using $config app.ini" - reset $config + log_info "using $config app.ini" + reset $config - for version in 1.18.5-0 1.19.4-0 1.20.2-0 1.20.3-0 1.21.0-0 ; do - echo "================ run $version" - cleanup_storage - start $version - verify_storage - stop - done + for version in 1.18.5-0 1.19.4-0 1.20.2-0 1.20.3-0 1.21.0-0 ; do + log_info "run $version" + cleanup_storage + start $version + verify_storage + stop + done done } +function run() { + local fun=$1 + shift + + echo Start running $fun + mkdir -p $DIR + > $DIR/$fun.out + tail --follow $DIR/$fun.out | sed --unbuffered -n -e "/^$PREFIX/s/^$PREFIX //p" & + pid=$! + if ! VERBOSE=true ${BASH_SOURCE[0]} $fun "$@" >& $DIR/$fun.out ; then + kill $pid + cat $DIR/$fun.out + echo Failure running $fun + return 1 + fi + kill $pid + echo Success running $fun +} + function test_upgrades() { - stop - dependencies - build_all - test_successful_upgrades - test_bug_storage_misplace - test_bug_storage_merged - test_downgrade_1.20.2_fails + run stop + run dependencies + run build_all + run test_successful_upgrades + run test_bug_storage_misplace + run test_bug_storage_merged + run test_downgrade_1.20.2_fails + run test_bug_storage_s3_misplace + run test_storage_stable_s3 minio + run test_storage_stable_s3 garage } "$@" diff --git a/.forgejo/workflows/testing.yml b/.forgejo/workflows/testing.yml index f56d589d53..9d2d4b5292 100644 --- a/.forgejo/workflows/testing.yml +++ b/.forgejo/workflows/testing.yml @@ -180,18 +180,9 @@ jobs: chown -R gitea:gitea . /go - run: | su gitea -c 'make deps-backend' - - run: | - su gitea -c 'make backend' - env: - TAGS: bindata sqlite sqlite_unlock_notify - - run: | - su gitea -c 'make gitea' - cp -a gitea /tmp/forgejo-development - timeout-minutes: 50 - env: - TAGS: bindata sqlite sqlite_unlock_notify - run: | script=$(pwd)/.forgejo/upgrades/test-upgrade.sh - $script dependencies + $script run dependencies + $script clobber su gitea -c "$script test_upgrades"