From c6e27d5f03ad85feb762872dd037f3bb82498fbb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Mar 2024 09:10:21 -0800 Subject: [PATCH 01/11] chore(deps): bump docker/setup-buildx-action from 3.0.0 to 3.1.0 (#5359) Bumps [docker/setup-buildx-action](https://github.com/docker/setup-buildx-action) from 3.0.0 to 3.1.0. - [Release notes](https://github.com/docker/setup-buildx-action/releases) - [Commits](https://github.com/docker/setup-buildx-action/compare/f95db51fddba0c2d1ec667646a06c2ce06100226...0d103c3126aa41d772a8362f6aa67afac040f80c) --- updated-dependencies: - dependency-name: docker/setup-buildx-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/container_dev.yml | 2 +- .github/workflows/container_latest.yml | 2 +- .github/workflows/container_release1.yml | 2 +- .github/workflows/container_release2.yml | 2 +- .github/workflows/container_release3.yml | 2 +- .github/workflows/container_release4.yml | 2 +- .github/workflows/container_release5.yml | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/container_dev.yml b/.github/workflows/container_dev.yml index 1e999fcc8..0b1280480 100644 --- a/.github/workflows/container_dev.yml +++ b/.github/workflows/container_dev.yml @@ -36,7 +36,7 @@ jobs: uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v1 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v1 + uses: docker/setup-buildx-action@0d103c3126aa41d772a8362f6aa67afac040f80c # v1 with: buildkitd-flags: "--debug" - diff --git a/.github/workflows/container_latest.yml b/.github/workflows/container_latest.yml index 4d6d1feef..c5aee45dd 100644 --- a/.github/workflows/container_latest.yml +++ b/.github/workflows/container_latest.yml @@ -37,7 +37,7 @@ jobs: uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v1 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v1 + uses: docker/setup-buildx-action@0d103c3126aa41d772a8362f6aa67afac040f80c # v1 with: buildkitd-flags: "--debug" - diff --git a/.github/workflows/container_release1.yml b/.github/workflows/container_release1.yml index 570d5a87e..c3b257797 100644 --- a/.github/workflows/container_release1.yml +++ b/.github/workflows/container_release1.yml @@ -37,7 +37,7 @@ jobs: uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v1 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v1 + uses: docker/setup-buildx-action@0d103c3126aa41d772a8362f6aa67afac040f80c # v1 - name: Login to Docker Hub if: github.event_name != 'pull_request' diff --git a/.github/workflows/container_release2.yml b/.github/workflows/container_release2.yml index f20ca7d93..260873fa8 100644 --- a/.github/workflows/container_release2.yml +++ b/.github/workflows/container_release2.yml @@ -38,7 +38,7 @@ jobs: uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v1 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v1 + uses: docker/setup-buildx-action@0d103c3126aa41d772a8362f6aa67afac040f80c # v1 - name: Login to Docker Hub if: github.event_name != 'pull_request' diff --git a/.github/workflows/container_release3.yml b/.github/workflows/container_release3.yml index 8413d570c..7a407a3ee 100644 --- a/.github/workflows/container_release3.yml +++ b/.github/workflows/container_release3.yml @@ -38,7 +38,7 @@ jobs: uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v1 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v1 + uses: docker/setup-buildx-action@0d103c3126aa41d772a8362f6aa67afac040f80c # v1 - name: Login to Docker Hub if: github.event_name != 'pull_request' diff --git a/.github/workflows/container_release4.yml b/.github/workflows/container_release4.yml index bb874eedf..eeaf93675 100644 --- a/.github/workflows/container_release4.yml +++ b/.github/workflows/container_release4.yml @@ -37,7 +37,7 @@ jobs: uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v1 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v1 + uses: docker/setup-buildx-action@0d103c3126aa41d772a8362f6aa67afac040f80c # v1 - name: Login to Docker Hub if: github.event_name != 'pull_request' diff --git a/.github/workflows/container_release5.yml b/.github/workflows/container_release5.yml index a5454e052..370eb241a 100644 --- a/.github/workflows/container_release5.yml +++ b/.github/workflows/container_release5.yml @@ -37,7 +37,7 @@ jobs: uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v1 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v1 + uses: docker/setup-buildx-action@0d103c3126aa41d772a8362f6aa67afac040f80c # v1 - name: Login to Docker Hub if: github.event_name != 'pull_request' From a39397f4e5df5cb76c0fe91dd447e93354357cd9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Mar 2024 09:10:28 -0800 Subject: [PATCH 02/11] chore(deps): bump azure/setup-helm from 3 to 4 (#5358) Bumps [azure/setup-helm](https://github.com/azure/setup-helm) from 3 to 4. - [Release notes](https://github.com/azure/setup-helm/releases) - [Changelog](https://github.com/Azure/setup-helm/blob/main/CHANGELOG.md) - [Commits](https://github.com/azure/setup-helm/compare/v3...v4) --- updated-dependencies: - dependency-name: azure/setup-helm dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/helm_ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/helm_ci.yml b/.github/workflows/helm_ci.yml index 97bd488a7..57b8358ff 100644 --- a/.github/workflows/helm_ci.yml +++ b/.github/workflows/helm_ci.yml @@ -21,7 +21,7 @@ jobs: fetch-depth: 0 - name: Set up Helm - uses: azure/setup-helm@v3 + uses: azure/setup-helm@v4 with: version: v3.10.0 From 54650e048a08320e83bf94fbb0fc15ee6d296a1f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Mar 2024 09:10:48 -0800 Subject: [PATCH 03/11] chore(deps): bump github.com/rdleal/intervalst from 1.2.2 to 1.3.0 (#5353) Bumps [github.com/rdleal/intervalst](https://github.com/rdleal/intervalst) from 1.2.2 to 1.3.0. - [Commits](https://github.com/rdleal/intervalst/compare/v1.2.2...v1.3.0) --- updated-dependencies: - dependency-name: github.com/rdleal/intervalst dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index c8fc9c71a..b5dbea174 100644 --- a/go.mod +++ b/go.mod @@ -154,7 +154,7 @@ require ( github.com/puzpuzpuz/xsync/v2 v2.5.1 github.com/rabbitmq/amqp091-go v1.9.0 github.com/rclone/rclone v1.65.2 - github.com/rdleal/intervalst v1.2.2 + github.com/rdleal/intervalst v1.3.0 github.com/schollz/progressbar/v3 v3.14.1 github.com/shirou/gopsutil/v3 v3.24.1 github.com/tikv/client-go/v2 v2.0.7 diff --git a/go.sum b/go.sum index 8fec6c554..6545d59b9 100644 --- a/go.sum +++ b/go.sum @@ -805,8 +805,8 @@ github.com/rclone/rclone v1.65.2 h1:ZBMzSKqGwimbDWkBqSx6MneRAE7Z5UI0lFxyC8HCGMQ= github.com/rclone/rclone v1.65.2/go.mod h1:nSMqGk0ovcw70cnEJ4xkRXZSL2h2GP8DojLeA+uu1rI= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= -github.com/rdleal/intervalst v1.2.2 h1:cw93gelL8qIAPuOwT+oHFewumFdrdW/QMFqgRhLUrlY= -github.com/rdleal/intervalst v1.2.2/go.mod h1:xO89Z6BC+LQDH+IPQQw/OESt5UADgFD41tYMUINGpxQ= +github.com/rdleal/intervalst v1.3.0 h1:Ldf/5uohv4VGsrI7KQgMQ63g2EaSf9GOAhwedIoBhvc= +github.com/rdleal/intervalst v1.3.0/go.mod h1:xO89Z6BC+LQDH+IPQQw/OESt5UADgFD41tYMUINGpxQ= github.com/redis/go-redis/v9 v9.0.2 h1:BA426Zqe/7r56kCcvxYLWe1mkaz71LKF77GwgFzSxfE= github.com/redis/go-redis/v9 v9.0.2/go.mod h1:/xDTe9EF1LM61hek62Poq2nzQSGj0xSrEtEHbBQevps= github.com/redis/rueidis v1.0.19 h1:s65oWtotzlIFN8eMPhyYwxlwLR1lUdhza2KtWprKYSo= From caa9f34ff0759cc5f093c61a3ccb236c8cc30b2e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Mar 2024 09:10:56 -0800 Subject: [PATCH 04/11] chore(deps): bump golang.org/x/sys from 0.17.0 to 0.18.0 (#5354) Bumps [golang.org/x/sys](https://github.com/golang/sys) from 0.17.0 to 0.18.0. - [Commits](https://github.com/golang/sys/compare/v0.17.0...v0.18.0) --- updated-dependencies: - dependency-name: golang.org/x/sys dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index b5dbea174..787d0e6e5 100644 --- a/go.mod +++ b/go.mod @@ -114,7 +114,7 @@ require ( golang.org/x/image v0.15.0 golang.org/x/net v0.21.0 golang.org/x/oauth2 v0.17.0 // indirect - golang.org/x/sys v0.17.0 + golang.org/x/sys v0.18.0 golang.org/x/text v0.14.0 // indirect golang.org/x/tools v0.16.1 golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect diff --git a/go.sum b/go.sum index 6545d59b9..f27f3d793 100644 --- a/go.sum +++ b/go.sum @@ -1254,8 +1254,8 @@ golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= -golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= From fad64a6af70eabe04cda76a4c746a2d647965f44 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Mar 2024 09:11:04 -0800 Subject: [PATCH 05/11] chore(deps): bump github.com/stretchr/testify from 1.8.4 to 1.9.0 (#5355) Bumps [github.com/stretchr/testify](https://github.com/stretchr/testify) from 1.8.4 to 1.9.0. - [Release notes](https://github.com/stretchr/testify/releases) - [Commits](https://github.com/stretchr/testify/compare/v1.8.4...v1.9.0) --- updated-dependencies: - dependency-name: github.com/stretchr/testify dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 787d0e6e5..acbf259e2 100644 --- a/go.mod +++ b/go.mod @@ -87,7 +87,7 @@ require ( github.com/spf13/afero v1.11.0 // indirect github.com/spf13/cast v1.6.0 // indirect github.com/spf13/viper v1.18.2 - github.com/stretchr/testify v1.8.4 + github.com/stretchr/testify v1.9.0 github.com/stvp/tempredis v0.0.0-20181119212430-b82af8480203 github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965 github.com/tidwall/gjson v1.17.0 diff --git a/go.sum b/go.sum index f27f3d793..46ca274f3 100644 --- a/go.sum +++ b/go.sum @@ -877,8 +877,9 @@ github.com/spf13/viper v1.18.2/go.mod h1:EKmWIqdnk5lOcmR72yw6hS+8OPYcwD0jteitLMV github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.3.1-0.20190311161405-34c6fa2dc709/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -890,8 +891,9 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/stvp/tempredis v0.0.0-20181119212430-b82af8480203 h1:QVqDTf3h2WHt08YuiTGPZLls0Wq99X9bWd0Q5ZSBesM= github.com/stvp/tempredis v0.0.0-20181119212430-b82af8480203/go.mod h1:oqN97ltKNihBbwlX8dLpwxCl3+HnXKV/R0e+sRLd9C8= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= From 332cf579b5a205c3d77bf2f3343d0ee9a2058314 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Mar 2024 09:11:12 -0800 Subject: [PATCH 06/11] chore(deps): bump cloud.google.com/go/pubsub from 1.36.1 to 1.36.2 (#5356) Bumps [cloud.google.com/go/pubsub](https://github.com/googleapis/google-cloud-go) from 1.36.1 to 1.36.2. - [Release notes](https://github.com/googleapis/google-cloud-go/releases) - [Changelog](https://github.com/googleapis/google-cloud-go/blob/main/CHANGES.md) - [Commits](https://github.com/googleapis/google-cloud-go/compare/pubsub/v1.36.1...pubsub/v1.36.2) --- updated-dependencies: - dependency-name: cloud.google.com/go/pubsub dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 8 ++++---- go.sum | 20 ++++++++++---------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/go.mod b/go.mod index acbf259e2..e6f0a204b 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.22.0 require ( cloud.google.com/go v0.112.0 // indirect - cloud.google.com/go/pubsub v1.36.1 + cloud.google.com/go/pubsub v1.36.2 cloud.google.com/go/storage v1.36.0 github.com/Azure/azure-pipeline-go v0.2.3 github.com/Azure/azure-storage-blob-go v0.15.0 @@ -120,7 +120,7 @@ require ( golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect google.golang.org/api v0.167.0 google.golang.org/appengine v1.6.8 // indirect - google.golang.org/genproto v0.0.0-20240205150955-31a09d347014 // indirect + google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 // indirect google.golang.org/grpc v1.61.1 google.golang.org/protobuf v1.32.0 gopkg.in/inf.v0 v0.9.1 // indirect @@ -165,7 +165,7 @@ require ( ) require ( - cloud.google.com/go/compute v1.23.4 // indirect + cloud.google.com/go/compute v1.24.0 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect cloud.google.com/go/iam v1.1.6 // indirect github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.0 // indirect @@ -335,7 +335,7 @@ require ( golang.org/x/sync v0.6.0 // indirect golang.org/x/term v0.17.0 // indirect golang.org/x/time v0.5.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240205150955-31a09d347014 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240221002015-b0ce06bbee7c // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240213162025-012b6fc9bca9 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect diff --git a/go.sum b/go.sum index 46ca274f3..001506261 100644 --- a/go.sum +++ b/go.sum @@ -22,22 +22,22 @@ cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvf cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/compute v1.23.4 h1:EBT9Nw4q3zyE7G45Wvv3MzolIrCJEuHys5muLY0wvAw= -cloud.google.com/go/compute v1.23.4/go.mod h1:/EJMj55asU6kAFnuZET8zqgwgJ9FvXWXOkkfQZa4ioI= +cloud.google.com/go/compute v1.24.0 h1:phWcR2eWzRJaL/kOiJwfFsPs4BaKq1j6vnpZrc1YlVg= +cloud.google.com/go/compute v1.24.0/go.mod h1:kw1/T+h/+tK2LJK0wiPPx1intgdAM3j/g3hFDlscY40= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/iam v1.1.6 h1:bEa06k05IO4f4uJonbB5iAgKTPpABy1ayxaIZV/GHVc= cloud.google.com/go/iam v1.1.6/go.mod h1:O0zxdPeGBoFdWW3HWmBxJsk0pfvNM/p/qa82rWOGTwI= -cloud.google.com/go/kms v1.15.6 h1:ktpEMQmsOAYj3VZwH020FcQlm23BVYg8T8O1woG2GcE= -cloud.google.com/go/kms v1.15.6/go.mod h1:yF75jttnIdHfGBoE51AKsD/Yqf+/jICzB9v1s1acsms= +cloud.google.com/go/kms v1.15.7 h1:7caV9K3yIxvlQPAcaFffhlT7d1qpxjB1wHBtjWa13SM= +cloud.google.com/go/kms v1.15.7/go.mod h1:ub54lbsa6tDkUwnu4W7Yt1aAIFLnspgh0kPGToDukeI= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= -cloud.google.com/go/pubsub v1.36.1 h1:dfEPuGCHGbWUhaMCTHUFjfroILEkx55iUmKBZTP5f+Y= -cloud.google.com/go/pubsub v1.36.1/go.mod h1:iYjCa9EzWOoBiTdd4ps7QoMtMln5NwaZQpK1hbRfBDE= +cloud.google.com/go/pubsub v1.36.2 h1:nAUD4aiWHZFYyINhRag1qOnHUk0/7QiWEa04XWnqACA= +cloud.google.com/go/pubsub v1.36.2/go.mod h1:mHCFLNG8abCrPzhuOnpBcr9DUy+l3/LWWn0qoJdbh1w= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= @@ -1415,10 +1415,10 @@ google.golang.org/genproto v0.0.0-20200806141610-86f49bd18e98/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= google.golang.org/genproto v0.0.0-20220421151946-72621c1f0bd3/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= -google.golang.org/genproto v0.0.0-20240205150955-31a09d347014 h1:g/4bk7P6TPMkAUbUhquq98xey1slwvuVJPosdBqYJlU= -google.golang.org/genproto v0.0.0-20240205150955-31a09d347014/go.mod h1:xEgQu1e4stdSSsxPDK8Azkrk/ECl5HvdPf6nbZrTS5M= -google.golang.org/genproto/googleapis/api v0.0.0-20240205150955-31a09d347014 h1:x9PwdEgd11LgK+orcck69WVRo7DezSO4VUMPI4xpc8A= -google.golang.org/genproto/googleapis/api v0.0.0-20240205150955-31a09d347014/go.mod h1:rbHMSEDyoYX62nRVLOCc4Qt1HbsdytAYoVwgjiOhF3I= +google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 h1:9+tzLLstTlPTRyJTh+ah5wIMsBW5c4tQwGTN3thOW9Y= +google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9/go.mod h1:mqHbVIp48Muh7Ywss/AD6I5kNVKZMmAa/QEW58Gxp2s= +google.golang.org/genproto/googleapis/api v0.0.0-20240221002015-b0ce06bbee7c h1:9g7erC9qu44ks7UK4gDNlnk4kOxZG707xKm4jVniy6o= +google.golang.org/genproto/googleapis/api v0.0.0-20240221002015-b0ce06bbee7c/go.mod h1:5iCWqnniDlqZHrd3neWVTOwvh/v6s3232omMecelax8= google.golang.org/genproto/googleapis/rpc v0.0.0-20240213162025-012b6fc9bca9 h1:hZB7eLIaYlW9qXRfCq/qDaPdbeY3757uARz5Vvfv+cY= google.golang.org/genproto/googleapis/rpc v0.0.0-20240213162025-012b6fc9bca9/go.mod h1:YUWgXUFRPfoYK1IHMuxH5K6nPEXSCzIMljnQ59lLRCk= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= From 18d06aaf6547d6740bf04b7897753a3ccc066511 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Mar 2024 09:11:21 -0800 Subject: [PATCH 07/11] chore(deps): bump github.com/tidwall/gjson from 1.17.0 to 1.17.1 (#5357) Bumps [github.com/tidwall/gjson](https://github.com/tidwall/gjson) from 1.17.0 to 1.17.1. - [Commits](https://github.com/tidwall/gjson/compare/v1.17.0...v1.17.1) --- updated-dependencies: - dependency-name: github.com/tidwall/gjson dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index e6f0a204b..87910a3d4 100644 --- a/go.mod +++ b/go.mod @@ -90,7 +90,7 @@ require ( github.com/stretchr/testify v1.9.0 github.com/stvp/tempredis v0.0.0-20181119212430-b82af8480203 github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965 - github.com/tidwall/gjson v1.17.0 + github.com/tidwall/gjson v1.17.1 github.com/tidwall/match v1.1.1 github.com/tidwall/pretty v1.2.0 // indirect github.com/tsuna/gohbase v0.0.0-20201125011725-348991136365 diff --git a/go.sum b/go.sum index 001506261..ba5fa2310 100644 --- a/go.sum +++ b/go.sum @@ -904,8 +904,8 @@ github.com/t3rm1n4l/go-mega v0.0.0-20230228171823-a01a2cda13ca h1:I9rVnNXdIkij4U github.com/t3rm1n4l/go-mega v0.0.0-20230228171823-a01a2cda13ca/go.mod h1:suDIky6yrK07NnaBadCB4sS0CqFOvUK91lH7CR+JlDA= github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a h1:J/YdBZ46WKpXsxsW93SG+q0F8KI+yFrcIDT4c/RNoc4= github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a/go.mod h1:h4xBhSNtOeEosLJ4P7JyKXX7Cabg7AVkWCK5gV2vOrM= -github.com/tidwall/gjson v1.17.0 h1:/Jocvlh98kcTfpN2+JzGQWQcqrPQwDrVEMApx/M5ZwM= -github.com/tidwall/gjson v1.17.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.17.1 h1:wlYEnwqAHgzmhNUFfw7Xalt2JzQvsMx2Se4PcoFCT/U= +github.com/tidwall/gjson v1.17.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= From 872b5161e660daaac9a842b2ec77b5122f47a7db Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.com> Date: Tue, 5 Mar 2024 22:09:08 +0500 Subject: [PATCH 08/11] [shell] fs log purge (#5344) * fix: install cronie * chore: shell delete old logs * Revert "fix: install cronie" This reverts commit 920832b04b89a0a0c9e12b3d51fe710fc622dad6. * fix: dayAgo * fix: Nit: dayAgo => daysAgo --- weed/shell/command_fs_log.go | 53 ++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 weed/shell/command_fs_log.go diff --git a/weed/shell/command_fs_log.go b/weed/shell/command_fs_log.go new file mode 100644 index 000000000..5567f76e6 --- /dev/null +++ b/weed/shell/command_fs_log.go @@ -0,0 +1,53 @@ +package shell + +import ( + "flag" + "fmt" + "github.com/seaweedfs/seaweedfs/weed/filer" + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "io" + "time" +) + +func init() { + Commands = append(Commands, &commandFsLogPurge{}) +} + +type commandFsLogPurge struct { +} + +func (c *commandFsLogPurge) Name() string { + return "fs.log.purge" +} + +func (c *commandFsLogPurge) Help() string { + return `purge filer logs + + fs.log.purge [-v] [-modifyDayAgo 365] +` +} + +func (c *commandFsLogPurge) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) { + fsLogPurgeCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError) + daysAgo := fsLogPurgeCommand.Uint("daysAgo", 365, "purge logs older than N days") + verbose := fsLogPurgeCommand.Bool("v", false, "verbose mode") + + if err = fsLogPurgeCommand.Parse(args); err != nil { + return err + } + + modificationTimeAgo := time.Now().Add(-time.Hour * 24 * time.Duration(*daysAgo)).Unix() + err = filer_pb.ReadDirAllEntries(commandEnv, filer.SystemLogDir, "", func(entry *filer_pb.Entry, isLast bool) error { + if entry.Attributes.Mtime > modificationTimeAgo { + return nil + } + if errDel := filer_pb.Remove(commandEnv, filer.SystemLogDir, entry.Name, true, true, true, false, nil); errDel != nil { + return errDel + } + if *verbose { + fmt.Fprintf(writer, "delete %s\n", entry.Name) + } + return nil + }) + return err +} From 3e3e0fa2402ee29aabf523dc20aa5cfd5b22d569 Mon Sep 17 00:00:00 2001 From: adasauce <60991921+adasauce@users.noreply.github.com> Date: Wed, 6 Mar 2024 11:24:44 -0400 Subject: [PATCH 09/11] feat: add support for S3 ListObjects and ListObjectsV2 (#5350) --- weed/s3api/s3api_objects_list_handlers.go | 67 +++++++++++++++-------- 1 file changed, 44 insertions(+), 23 deletions(-) diff --git a/weed/s3api/s3api_objects_list_handlers.go b/weed/s3api/s3api_objects_list_handlers.go index 78b77a044..b2ad915b9 100644 --- a/weed/s3api/s3api_objects_list_handlers.go +++ b/weed/s3api/s3api_objects_list_handlers.go @@ -47,10 +47,6 @@ func (s3a *S3ApiServer) ListObjectsV2Handler(w http.ResponseWriter, r *http.Requ s3err.WriteErrorResponse(w, r, s3err.ErrInvalidMaxKeys) return } - if delimiter != "" && delimiter != "/" { - s3err.WriteErrorResponse(w, r, s3err.ErrNotImplemented) - return - } marker := continuationToken if continuationToken == "" { @@ -103,10 +99,6 @@ func (s3a *S3ApiServer) ListObjectsV1Handler(w http.ResponseWriter, r *http.Requ s3err.WriteErrorResponse(w, r, s3err.ErrInvalidMaxKeys) return } - if delimiter != "" && delimiter != "/" { - s3err.WriteErrorResponse(w, r, s3err.ErrNotImplemented) - return - } response, err := s3a.listFilerEntries(bucket, originalPrefix, maxKeys, marker, delimiter) @@ -171,22 +163,51 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m cursor.maxKeys-- } } else { - storageClass := "STANDARD" - if v, ok := entry.Extended[s3_constants.AmzStorageClass]; ok { - storageClass = string(v) + var delimiterFound bool + if delimiter != "" { + // keys that contain the same string between the prefix and the first occurrence of the delimiter are grouped together as a commonPrefix. + // extract the string between the prefix and the delimiter and add it to the commonPrefixes if it's unique. + fullPath := fmt.Sprintf("%s/%s", dir, entry.Name)[len(bucketPrefix):] + delimitedPath := strings.SplitN(fullPath, delimiter, 2) + if len(delimitedPath) == 2 { + + // S3 clients expect the delimited prefix to contain the delimiter. + delimitedPrefix := delimitedPath[0] + delimiter + + for i := range commonPrefixes { + if commonPrefixes[i].Prefix == delimitedPrefix { + delimiterFound = true + break + } + } + + if !delimiterFound { + commonPrefixes = append(commonPrefixes, PrefixEntry{ + Prefix: delimitedPrefix, + }) + cursor.maxKeys-- + delimiterFound = true + } + } + } + if !delimiterFound { + storageClass := "STANDARD" + if v, ok := entry.Extended[s3_constants.AmzStorageClass]; ok { + storageClass = string(v) + } + contents = append(contents, ListEntry{ + Key: fmt.Sprintf("%s/%s", dir, entry.Name)[len(bucketPrefix):], + LastModified: time.Unix(entry.Attributes.Mtime, 0).UTC(), + ETag: "\"" + filer.ETag(entry) + "\"", + Size: int64(filer.FileSize(entry)), + Owner: CanonicalUser{ + ID: fmt.Sprintf("%x", entry.Attributes.Uid), + DisplayName: entry.Attributes.UserName, + }, + StorageClass: StorageClass(storageClass), + }) + cursor.maxKeys-- } - contents = append(contents, ListEntry{ - Key: fmt.Sprintf("%s/%s", dir, entry.Name)[len(bucketPrefix):], - LastModified: time.Unix(entry.Attributes.Mtime, 0).UTC(), - ETag: "\"" + filer.ETag(entry) + "\"", - Size: int64(filer.FileSize(entry)), - Owner: CanonicalUser{ - ID: fmt.Sprintf("%x", entry.Attributes.Uid), - DisplayName: entry.Attributes.UserName, - }, - StorageClass: StorageClass(storageClass), - }) - cursor.maxKeys-- } }) if doErr != nil { From ecc154fa9e9bd11f2a664d2d11bd15c76d8b6c49 Mon Sep 17 00:00:00 2001 From: Seyed Mahdi Sadegh Shobeiri <36403983+SmsS4@users.noreply.github.com> Date: Wed, 6 Mar 2024 18:58:46 +0330 Subject: [PATCH 10/11] Add deleted bytes to total_disk_size (#5161) --- weed/storage/store.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/weed/storage/store.go b/weed/storage/store.go index d290909f1..782eb5b79 100644 --- a/weed/storage/store.go +++ b/weed/storage/store.go @@ -251,6 +251,7 @@ func (s *Store) CollectHeartbeat() *master_pb.Heartbeat { maxVolumeCounts := make(map[string]uint32) var maxFileKey NeedleId collectionVolumeSize := make(map[string]int64) + collectionVolumeDeletedBytes := make(map[string]int64) collectionVolumeReadOnlyCount := make(map[string]map[string]uint8) for _, location := range s.Locations { var deleteVids []needle.VolumeId @@ -283,9 +284,11 @@ func (s *Store) CollectHeartbeat() *master_pb.Heartbeat { if _, exist := collectionVolumeSize[v.Collection]; !exist { collectionVolumeSize[v.Collection] = 0 + collectionVolumeDeletedBytes[v.Collection] = 0 } if !shouldDeleteVolume { collectionVolumeSize[v.Collection] += int64(volumeMessage.Size) + collectionVolumeDeletedBytes[v.Collection] += int64(volumeMessage.DeletedByteCount) } else { collectionVolumeSize[v.Collection] -= int64(volumeMessage.Size) if collectionVolumeSize[v.Collection] <= 0 { @@ -342,6 +345,10 @@ func (s *Store) CollectHeartbeat() *master_pb.Heartbeat { stats.VolumeServerDiskSizeGauge.WithLabelValues(col, "normal").Set(float64(size)) } + for col, deletedBytes := range collectionVolumeDeletedBytes{ + stats.VolumeServerDiskSizeGauge.WithLabelValues(col, "deleted_bytes").Set(float64(deletedBytes)) + } + for col, types := range collectionVolumeReadOnlyCount { for t, count := range types { stats.VolumeServerReadOnlyVolumeGauge.WithLabelValues(col, t).Set(float64(count)) From 170b63d6f89284b804b153343380537fd09e0025 Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.com> Date: Thu, 7 Mar 2024 21:35:51 +0500 Subject: [PATCH 11/11] [filer.backup] add param uploader_part_size for S3sink (#5352) * fix: install cronie * chore: refactor configure S3Sink * chore: refactor cinfig * add filer-backup compose file * fix: X-Amz-Meta-Mtime and resolve with comments * fix: attr mtime * fix: MaxUploadPartst is reduced to the maximum allowable * fix: env and force set max MaxUploadParts * fix: env WEED_SINK_S3_UPLOADER_PART_SIZE_MB --- docker/Makefile | 3 + docker/compose/local-filer-backup-compose.yml | 54 ++++++++ weed/command/filer_backup.go | 3 +- weed/replication/sink/s3sink/s3_sink.go | 121 ++++++++++++------ weed/s3api/s3_constants/header.go | 1 + 5 files changed, 140 insertions(+), 42 deletions(-) create mode 100644 docker/compose/local-filer-backup-compose.yml diff --git a/docker/Makefile b/docker/Makefile index 89a896f77..6d6c88190 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -81,6 +81,9 @@ cluster: build 2mount: build docker compose -f compose/local-sync-mount-compose.yml -p seaweedfs up +filer_backup: build + docker compose -f compose/local-filer-backup-compose.yml -p seaweedfs up + hashicorp_raft: build docker compose -f compose/local-hashicorp-raft-compose.yml -p seaweedfs up diff --git a/docker/compose/local-filer-backup-compose.yml b/docker/compose/local-filer-backup-compose.yml new file mode 100644 index 000000000..3e4baf5fa --- /dev/null +++ b/docker/compose/local-filer-backup-compose.yml @@ -0,0 +1,54 @@ +version: '3.9' + +services: + server-left: + image: chrislusf/seaweedfs:local + command: "-v=0 server -ip=server-left -filer -filer.maxMB 5 -s3 -s3.config=/etc/seaweedfs/s3.json -volume.max=0 -master.volumeSizeLimitMB=1024 -volume.preStopSeconds=1" + volumes: + - ./s3.json:/etc/seaweedfs/s3.json + healthcheck: + test: [ "CMD", "curl", "--fail", "-I", "http://localhost:9333/cluster/healthz" ] + interval: 3s + start_period: 15s + timeout: 30s + server-right: + image: chrislusf/seaweedfs:local + command: "-v=0 server -ip=server-right -filer -filer.maxMB 64 -s3 -s3.config=/etc/seaweedfs/s3.json -volume.max=0 -master.volumeSizeLimitMB=1024 -volume.preStopSeconds=1" + volumes: + - ./s3.json:/etc/seaweedfs/s3.json + healthcheck: + test: [ "CMD", "curl", "--fail", "-I", "http://localhost:9333/cluster/healthz" ] + interval: 3s + start_period: 15s + timeout: 30s + filer-backup: + image: chrislusf/seaweedfs:local + command: "-v=0 filer.backup -debug -doDeleteFiles=False -filer server-left:8888" + volumes: + - ./replication.toml:/etc/seaweedfs/replication.toml + environment: + WEED_SINK_LOCAL_INCREMENTAL_ENABLED: "false" + WEED_SINK_S3_ENABLED: "true" + WEED_SINK_S3_BUCKET: "backup" + WEED_SINK_S3_ENDPOINT: "http://server-right:8333" + WEED_SINK_S3_DIRECTORY: "/" + WEED_SINK_S3_AWS_ACCESS_KEY_ID: "some_access_key1" + WEED_SINK_S3_AWS_SECRET_ACCESS_KEY: "some_secret_key1" + WEED_SINK_S3_S3_DISABLE_CONTENT_MD5_VALIDATION: "false" + WEED_SINK_S3_UPLOADER_PART_SIZE_MB: "5" + WEED_SINK_S3_KEEP_PART_SIZE: "false" + depends_on: + server-left: + condition: service_healthy + server-right: + condition: service_healthy + minio-warp: + image: minio/warp + command: 'mixed --duration 5s --obj.size=6mb --md5 --objects 10 --concurrent 2' + restart: on-failure + environment: + WARP_HOST: "server-left:8333" + WARP_ACCESS_KEY: "some_access_key1" + WARP_SECRET_KEY: "some_secret_key1" + depends_on: + - filer-backup \ No newline at end of file diff --git a/weed/command/filer_backup.go b/weed/command/filer_backup.go index 691b1c0b5..4aeab60f2 100644 --- a/weed/command/filer_backup.go +++ b/weed/command/filer_backup.go @@ -85,8 +85,7 @@ const ( func doFilerBackup(grpcDialOption grpc.DialOption, backupOption *FilerBackupOptions, clientId int32, clientEpoch int32) error { // find data sink - config := util.GetViper() - dataSink := findSink(config) + dataSink := findSink(util.GetViper()) if dataSink == nil { return fmt.Errorf("no data sink configured in replication.toml") } diff --git a/weed/replication/sink/s3sink/s3_sink.go b/weed/replication/sink/s3sink/s3_sink.go index a032b58e8..276ea30d6 100644 --- a/weed/replication/sink/s3sink/s3_sink.go +++ b/weed/replication/sink/s3sink/s3_sink.go @@ -8,6 +8,8 @@ import ( "github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go/service/s3/s3iface" "github.com/aws/aws-sdk-go/service/s3/s3manager" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" + "strconv" "strings" "github.com/seaweedfs/seaweedfs/weed/filer" @@ -19,14 +21,20 @@ import ( ) type S3Sink struct { - conn s3iface.S3API - region string - bucket string - dir string - endpoint string - acl string - filerSource *source.FilerSource - isIncremental bool + conn s3iface.S3API + filerSource *source.FilerSource + isIncremental bool + keepPartSize bool + s3DisableContentMD5Validation bool + s3ForcePathStyle bool + uploaderConcurrency int + uploaderMaxUploadParts int + uploaderPartSizeMb int + region string + bucket string + dir string + endpoint string + acl string } func init() { @@ -46,21 +54,49 @@ func (s3sink *S3Sink) IsIncremental() bool { } func (s3sink *S3Sink) Initialize(configuration util.Configuration, prefix string) error { - glog.V(0).Infof("sink.s3.region: %v", configuration.GetString(prefix+"region")) - glog.V(0).Infof("sink.s3.bucket: %v", configuration.GetString(prefix+"bucket")) - glog.V(0).Infof("sink.s3.directory: %v", configuration.GetString(prefix+"directory")) - glog.V(0).Infof("sink.s3.endpoint: %v", configuration.GetString(prefix+"endpoint")) - glog.V(0).Infof("sink.s3.acl: %v", configuration.GetString(prefix+"acl")) - glog.V(0).Infof("sink.s3.is_incremental: %v", configuration.GetString(prefix+"is_incremental")) + configuration.SetDefault(prefix+"region", "us-east-2") + configuration.SetDefault(prefix+"directory", "/") + configuration.SetDefault(prefix+"keep_part_size", true) + configuration.SetDefault(prefix+"uploader_max_upload_parts", 1000) + configuration.SetDefault(prefix+"uploader_part_size_mb", 8) + configuration.SetDefault(prefix+"uploader_concurrency", 8) + configuration.SetDefault(prefix+"s3_disable_content_md5_validation", true) + configuration.SetDefault(prefix+"s3_force_path_style", true) + s3sink.region = configuration.GetString(prefix + "region") + s3sink.bucket = configuration.GetString(prefix + "bucket") + s3sink.dir = configuration.GetString(prefix + "directory") + s3sink.endpoint = configuration.GetString(prefix + "endpoint") + s3sink.acl = configuration.GetString(prefix + "acl") s3sink.isIncremental = configuration.GetBool(prefix + "is_incremental") + s3sink.keepPartSize = configuration.GetBool(prefix + "keep_part_size") + s3sink.s3DisableContentMD5Validation = configuration.GetBool(prefix + "s3_disable_content_md5_validation") + s3sink.s3ForcePathStyle = configuration.GetBool(prefix + "s3_force_path_style") + s3sink.uploaderMaxUploadParts = configuration.GetInt(prefix + "uploader_max_upload_parts") + s3sink.uploaderPartSizeMb = configuration.GetInt(prefix + "uploader_part_size") + s3sink.uploaderConcurrency = configuration.GetInt(prefix + "uploader_concurrency") + + glog.V(0).Infof("sink.s3.region: %v", s3sink.region) + glog.V(0).Infof("sink.s3.bucket: %v", s3sink.bucket) + glog.V(0).Infof("sink.s3.directory: %v", s3sink.dir) + glog.V(0).Infof("sink.s3.endpoint: %v", s3sink.endpoint) + glog.V(0).Infof("sink.s3.acl: %v", s3sink.acl) + glog.V(0).Infof("sink.s3.is_incremental: %v", s3sink.isIncremental) + glog.V(0).Infof("sink.s3.s3_disable_content_md5_validation: %v", s3sink.s3DisableContentMD5Validation) + glog.V(0).Infof("sink.s3.s3_force_path_style: %v", s3sink.s3ForcePathStyle) + glog.V(0).Infof("sink.s3.keep_part_size: %v", s3sink.keepPartSize) + if s3sink.uploaderMaxUploadParts > s3manager.MaxUploadParts { + s3sink.uploaderMaxUploadParts = s3manager.MaxUploadParts + glog.Warningf("uploader_max_upload_parts is greater than the maximum number of parts allowed when uploading multiple parts to Amazon S3") + glog.V(0).Infof("sink.s3.uploader_max_upload_parts: %v => %v", s3sink.uploaderMaxUploadParts, s3manager.MaxUploadParts) + } else { + glog.V(0).Infof("sink.s3.uploader_max_upload_parts: %v", s3sink.uploaderMaxUploadParts) + } + glog.V(0).Infof("sink.s3.uploader_part_size_mb: %v", s3sink.uploaderPartSizeMb) + glog.V(0).Infof("sink.s3.uploader_concurrency: %v", s3sink.uploaderConcurrency) + return s3sink.initialize( configuration.GetString(prefix+"aws_access_key_id"), configuration.GetString(prefix+"aws_secret_access_key"), - configuration.GetString(prefix+"region"), - configuration.GetString(prefix+"bucket"), - configuration.GetString(prefix+"directory"), - configuration.GetString(prefix+"endpoint"), - configuration.GetString(prefix+"acl"), ) } @@ -68,18 +104,12 @@ func (s3sink *S3Sink) SetSourceFiler(s *source.FilerSource) { s3sink.filerSource = s } -func (s3sink *S3Sink) initialize(awsAccessKeyId, awsSecretAccessKey, region, bucket, dir, endpoint, acl string) error { - s3sink.region = region - s3sink.bucket = bucket - s3sink.dir = dir - s3sink.endpoint = endpoint - s3sink.acl = acl - +func (s3sink *S3Sink) initialize(awsAccessKeyId, awsSecretAccessKey string) error { config := &aws.Config{ Region: aws.String(s3sink.region), Endpoint: aws.String(s3sink.endpoint), - S3ForcePathStyle: aws.Bool(true), - S3DisableContentMD5Validation: aws.Bool(true), + S3DisableContentMD5Validation: aws.Bool(s3sink.s3DisableContentMD5Validation), + S3ForcePathStyle: aws.Bool(s3sink.s3ForcePathStyle), } if awsAccessKeyId != "" && awsSecretAccessKey != "" { config.Credentials = credentials.NewStaticCredentials(awsAccessKeyId, awsSecretAccessKey, "") @@ -128,19 +158,26 @@ func (s3sink *S3Sink) CreateEntry(key string, entry *filer_pb.Entry, signatures reader := filer.NewFileReader(s3sink.filerSource, entry) - fileSize := int64(filer.FileSize(entry)) - - partSize := int64(8 * 1024 * 1024) // The minimum/default allowed part size is 5MB - for partSize*1000 < fileSize { - partSize *= 4 - } - // Create an uploader with the session and custom options uploader := s3manager.NewUploaderWithClient(s3sink.conn, func(u *s3manager.Uploader) { - u.PartSize = partSize - u.Concurrency = 8 + u.PartSize = int64(s3sink.uploaderPartSizeMb * 1024 * 1024) + u.Concurrency = s3sink.uploaderConcurrency + u.MaxUploadParts = s3sink.uploaderMaxUploadParts }) + if s3sink.keepPartSize { + switch chunkCount := len(entry.Chunks); { + case chunkCount > 1: + if firstChunkSize := int64(entry.Chunks[0].Size); firstChunkSize > s3manager.MinUploadPartSize { + uploader.PartSize = firstChunkSize + } + default: + uploader.PartSize = 0 + } + } + if _, ok := entry.Extended[s3_constants.AmzUserMetaMtime]; !ok { + entry.Extended[s3_constants.AmzUserMetaMtime] = []byte(strconv.FormatInt(entry.Attributes.Mtime, 10)) + } // process tagging tags := "" if true { @@ -153,14 +190,18 @@ func (s3sink *S3Sink) CreateEntry(key string, entry *filer_pb.Entry, signatures } // Upload the file to S3. - _, err = uploader.Upload(&s3manager.UploadInput{ + uploadInput := s3manager.UploadInput{ Bucket: aws.String(s3sink.bucket), Key: aws.String(key), Body: reader, Tagging: aws.String(tags), - }) + } + if len(entry.Attributes.Md5) > 0 { + uploadInput.ContentMD5 = aws.String(fmt.Sprintf("%x", entry.Attributes.Md5)) + } + _, err = uploader.Upload(&uploadInput) - return + return err } diff --git a/weed/s3api/s3_constants/header.go b/weed/s3api/s3_constants/header.go index 5037f4691..30a878ccb 100644 --- a/weed/s3api/s3_constants/header.go +++ b/weed/s3api/s3_constants/header.go @@ -30,6 +30,7 @@ const ( // S3 user-defined metadata AmzUserMetaPrefix = "X-Amz-Meta-" AmzUserMetaDirective = "X-Amz-Metadata-Directive" + AmzUserMetaMtime = "X-Amz-Meta-Mtime" // S3 object tagging AmzObjectTagging = "X-Amz-Tagging"