mirror of
https://github.com/chrislusf/seaweedfs
synced 2024-05-19 01:40:15 +02:00
Merge branch 'master' into mq-subscribe
This commit is contained in:
chrislu
commit
8f79bb3987
2
.github/workflows/container_dev.yml
vendored
2
.github/workflows/container_dev.yml
vendored
|
|
@ -36,7 +36,7 @@ jobs:
|
|||
uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v1
|
||||
-
|
||||
name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v1
|
||||
uses: docker/setup-buildx-action@0d103c3126aa41d772a8362f6aa67afac040f80c # v1
|
||||
with:
|
||||
buildkitd-flags: "--debug"
|
||||
-
|
||||
|
|
|
|||
2
.github/workflows/container_latest.yml
vendored
2
.github/workflows/container_latest.yml
vendored
|
|
@ -37,7 +37,7 @@ jobs:
|
|||
uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v1
|
||||
-
|
||||
name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v1
|
||||
uses: docker/setup-buildx-action@0d103c3126aa41d772a8362f6aa67afac040f80c # v1
|
||||
with:
|
||||
buildkitd-flags: "--debug"
|
||||
-
|
||||
|
|
|
|||
2
.github/workflows/container_release1.yml
vendored
2
.github/workflows/container_release1.yml
vendored
|
|
@ -37,7 +37,7 @@ jobs:
|
|||
uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v1
|
||||
-
|
||||
name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v1
|
||||
uses: docker/setup-buildx-action@0d103c3126aa41d772a8362f6aa67afac040f80c # v1
|
||||
-
|
||||
name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
|
|
|
|||
2
.github/workflows/container_release2.yml
vendored
2
.github/workflows/container_release2.yml
vendored
|
|
@ -38,7 +38,7 @@ jobs:
|
|||
uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v1
|
||||
-
|
||||
name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v1
|
||||
uses: docker/setup-buildx-action@0d103c3126aa41d772a8362f6aa67afac040f80c # v1
|
||||
-
|
||||
name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
|
|
|
|||
2
.github/workflows/container_release3.yml
vendored
2
.github/workflows/container_release3.yml
vendored
|
|
@ -38,7 +38,7 @@ jobs:
|
|||
uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v1
|
||||
-
|
||||
name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v1
|
||||
uses: docker/setup-buildx-action@0d103c3126aa41d772a8362f6aa67afac040f80c # v1
|
||||
-
|
||||
name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
|
|
|
|||
2
.github/workflows/container_release4.yml
vendored
2
.github/workflows/container_release4.yml
vendored
|
|
@ -37,7 +37,7 @@ jobs:
|
|||
uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v1
|
||||
-
|
||||
name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v1
|
||||
uses: docker/setup-buildx-action@0d103c3126aa41d772a8362f6aa67afac040f80c # v1
|
||||
-
|
||||
name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
|
|
|
|||
2
.github/workflows/container_release5.yml
vendored
2
.github/workflows/container_release5.yml
vendored
|
|
@ -37,7 +37,7 @@ jobs:
|
|||
uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v1
|
||||
-
|
||||
name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v1
|
||||
uses: docker/setup-buildx-action@0d103c3126aa41d772a8362f6aa67afac040f80c # v1
|
||||
-
|
||||
name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
|
|
|
|||
2
.github/workflows/helm_ci.yml
vendored
2
.github/workflows/helm_ci.yml
vendored
|
|
@ -21,7 +21,7 @@ jobs:
|
|||
fetch-depth: 0
|
||||
|
||||
- name: Set up Helm
|
||||
uses: azure/setup-helm@v3
|
||||
uses: azure/setup-helm@v4
|
||||
with:
|
||||
version: v3.10.0
|
||||
|
||||
|
|
|
|||
|
|
@ -83,6 +83,9 @@ cluster: build
|
|||
2mount: build
|
||||
docker compose -f compose/local-sync-mount-compose.yml -p seaweedfs up
|
||||
|
||||
filer_backup: build
|
||||
docker compose -f compose/local-filer-backup-compose.yml -p seaweedfs up
|
||||
|
||||
hashicorp_raft: build
|
||||
docker compose -f compose/local-hashicorp-raft-compose.yml -p seaweedfs up
|
||||
|
||||
|
|
|
|||
54
docker/compose/local-filer-backup-compose.yml
Normal file
54
docker/compose/local-filer-backup-compose.yml
Normal file
|
|
@ -0,0 +1,54 @@
|
|||
version: '3.9'
|
||||
|
||||
services:
|
||||
server-left:
|
||||
image: chrislusf/seaweedfs:local
|
||||
command: "-v=0 server -ip=server-left -filer -filer.maxMB 5 -s3 -s3.config=/etc/seaweedfs/s3.json -volume.max=0 -master.volumeSizeLimitMB=1024 -volume.preStopSeconds=1"
|
||||
volumes:
|
||||
- ./s3.json:/etc/seaweedfs/s3.json
|
||||
healthcheck:
|
||||
test: [ "CMD", "curl", "--fail", "-I", "http://localhost:9333/cluster/healthz" ]
|
||||
interval: 3s
|
||||
start_period: 15s
|
||||
timeout: 30s
|
||||
server-right:
|
||||
image: chrislusf/seaweedfs:local
|
||||
command: "-v=0 server -ip=server-right -filer -filer.maxMB 64 -s3 -s3.config=/etc/seaweedfs/s3.json -volume.max=0 -master.volumeSizeLimitMB=1024 -volume.preStopSeconds=1"
|
||||
volumes:
|
||||
- ./s3.json:/etc/seaweedfs/s3.json
|
||||
healthcheck:
|
||||
test: [ "CMD", "curl", "--fail", "-I", "http://localhost:9333/cluster/healthz" ]
|
||||
interval: 3s
|
||||
start_period: 15s
|
||||
timeout: 30s
|
||||
filer-backup:
|
||||
image: chrislusf/seaweedfs:local
|
||||
command: "-v=0 filer.backup -debug -doDeleteFiles=False -filer server-left:8888"
|
||||
volumes:
|
||||
- ./replication.toml:/etc/seaweedfs/replication.toml
|
||||
environment:
|
||||
WEED_SINK_LOCAL_INCREMENTAL_ENABLED: "false"
|
||||
WEED_SINK_S3_ENABLED: "true"
|
||||
WEED_SINK_S3_BUCKET: "backup"
|
||||
WEED_SINK_S3_ENDPOINT: "http://server-right:8333"
|
||||
WEED_SINK_S3_DIRECTORY: "/"
|
||||
WEED_SINK_S3_AWS_ACCESS_KEY_ID: "some_access_key1"
|
||||
WEED_SINK_S3_AWS_SECRET_ACCESS_KEY: "some_secret_key1"
|
||||
WEED_SINK_S3_S3_DISABLE_CONTENT_MD5_VALIDATION: "false"
|
||||
WEED_SINK_S3_UPLOADER_PART_SIZE_MB: "5"
|
||||
WEED_SINK_S3_KEEP_PART_SIZE: "false"
|
||||
depends_on:
|
||||
server-left:
|
||||
condition: service_healthy
|
||||
server-right:
|
||||
condition: service_healthy
|
||||
minio-warp:
|
||||
image: minio/warp
|
||||
command: 'mixed --duration 5s --obj.size=6mb --md5 --objects 10 --concurrent 2'
|
||||
restart: on-failure
|
||||
environment:
|
||||
WARP_HOST: "server-left:8333"
|
||||
WARP_ACCESS_KEY: "some_access_key1"
|
||||
WARP_SECRET_KEY: "some_secret_key1"
|
||||
depends_on:
|
||||
- filer-backup
|
||||
16
go.mod
16
go.mod
|
|
@ -4,7 +4,7 @@ go 1.22.0
|
|||
|
||||
require (
|
||||
cloud.google.com/go v0.112.0 // indirect
|
||||
cloud.google.com/go/pubsub v1.36.1
|
||||
cloud.google.com/go/pubsub v1.36.2
|
||||
cloud.google.com/go/storage v1.36.0
|
||||
github.com/Azure/azure-pipeline-go v0.2.3
|
||||
github.com/Azure/azure-storage-blob-go v0.15.0
|
||||
|
|
@ -87,10 +87,10 @@ require (
|
|||
github.com/spf13/afero v1.11.0 // indirect
|
||||
github.com/spf13/cast v1.6.0 // indirect
|
||||
github.com/spf13/viper v1.18.2
|
||||
github.com/stretchr/testify v1.8.4
|
||||
github.com/stretchr/testify v1.9.0
|
||||
github.com/stvp/tempredis v0.0.0-20181119212430-b82af8480203
|
||||
github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965
|
||||
github.com/tidwall/gjson v1.17.0
|
||||
github.com/tidwall/gjson v1.17.1
|
||||
github.com/tidwall/match v1.1.1
|
||||
github.com/tidwall/pretty v1.2.0 // indirect
|
||||
github.com/tsuna/gohbase v0.0.0-20201125011725-348991136365
|
||||
|
|
@ -114,13 +114,13 @@ require (
|
|||
golang.org/x/image v0.15.0
|
||||
golang.org/x/net v0.21.0
|
||||
golang.org/x/oauth2 v0.17.0 // indirect
|
||||
golang.org/x/sys v0.17.0
|
||||
golang.org/x/sys v0.18.0
|
||||
golang.org/x/text v0.14.0 // indirect
|
||||
golang.org/x/tools v0.16.1
|
||||
golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
|
||||
google.golang.org/api v0.167.0
|
||||
google.golang.org/appengine v1.6.8 // indirect
|
||||
google.golang.org/genproto v0.0.0-20240205150955-31a09d347014 // indirect
|
||||
google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 // indirect
|
||||
google.golang.org/grpc v1.61.1
|
||||
google.golang.org/protobuf v1.32.0
|
||||
gopkg.in/inf.v0 v0.9.1 // indirect
|
||||
|
|
@ -154,7 +154,7 @@ require (
|
|||
github.com/puzpuzpuz/xsync/v2 v2.5.1
|
||||
github.com/rabbitmq/amqp091-go v1.9.0
|
||||
github.com/rclone/rclone v1.65.2
|
||||
github.com/rdleal/intervalst v1.2.2
|
||||
github.com/rdleal/intervalst v1.3.0
|
||||
github.com/schollz/progressbar/v3 v3.14.1
|
||||
github.com/shirou/gopsutil/v3 v3.24.1
|
||||
github.com/tikv/client-go/v2 v2.0.7
|
||||
|
|
@ -165,7 +165,7 @@ require (
|
|||
)
|
||||
|
||||
require (
|
||||
cloud.google.com/go/compute v1.23.4 // indirect
|
||||
cloud.google.com/go/compute v1.24.0 // indirect
|
||||
cloud.google.com/go/compute/metadata v0.2.3 // indirect
|
||||
cloud.google.com/go/iam v1.1.6 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.0 // indirect
|
||||
|
|
@ -335,7 +335,7 @@ require (
|
|||
golang.org/x/sync v0.6.0 // indirect
|
||||
golang.org/x/term v0.17.0 // indirect
|
||||
golang.org/x/time v0.5.0 // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20240205150955-31a09d347014 // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20240221002015-b0ce06bbee7c // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20240213162025-012b6fc9bca9 // indirect
|
||||
gopkg.in/ini.v1 v1.67.0 // indirect
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
|
||||
|
|
|
|||
38
go.sum
38
go.sum
|
|
@ -22,22 +22,22 @@ cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvf
|
|||
cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg=
|
||||
cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=
|
||||
cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
|
||||
cloud.google.com/go/compute v1.23.4 h1:EBT9Nw4q3zyE7G45Wvv3MzolIrCJEuHys5muLY0wvAw=
|
||||
cloud.google.com/go/compute v1.23.4/go.mod h1:/EJMj55asU6kAFnuZET8zqgwgJ9FvXWXOkkfQZa4ioI=
|
||||
cloud.google.com/go/compute v1.24.0 h1:phWcR2eWzRJaL/kOiJwfFsPs4BaKq1j6vnpZrc1YlVg=
|
||||
cloud.google.com/go/compute v1.24.0/go.mod h1:kw1/T+h/+tK2LJK0wiPPx1intgdAM3j/g3hFDlscY40=
|
||||
cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY=
|
||||
cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA=
|
||||
cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
|
||||
cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
|
||||
cloud.google.com/go/iam v1.1.6 h1:bEa06k05IO4f4uJonbB5iAgKTPpABy1ayxaIZV/GHVc=
|
||||
cloud.google.com/go/iam v1.1.6/go.mod h1:O0zxdPeGBoFdWW3HWmBxJsk0pfvNM/p/qa82rWOGTwI=
|
||||
cloud.google.com/go/kms v1.15.6 h1:ktpEMQmsOAYj3VZwH020FcQlm23BVYg8T8O1woG2GcE=
|
||||
cloud.google.com/go/kms v1.15.6/go.mod h1:yF75jttnIdHfGBoE51AKsD/Yqf+/jICzB9v1s1acsms=
|
||||
cloud.google.com/go/kms v1.15.7 h1:7caV9K3yIxvlQPAcaFffhlT7d1qpxjB1wHBtjWa13SM=
|
||||
cloud.google.com/go/kms v1.15.7/go.mod h1:ub54lbsa6tDkUwnu4W7Yt1aAIFLnspgh0kPGToDukeI=
|
||||
cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
|
||||
cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
|
||||
cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
|
||||
cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU=
|
||||
cloud.google.com/go/pubsub v1.36.1 h1:dfEPuGCHGbWUhaMCTHUFjfroILEkx55iUmKBZTP5f+Y=
|
||||
cloud.google.com/go/pubsub v1.36.1/go.mod h1:iYjCa9EzWOoBiTdd4ps7QoMtMln5NwaZQpK1hbRfBDE=
|
||||
cloud.google.com/go/pubsub v1.36.2 h1:nAUD4aiWHZFYyINhRag1qOnHUk0/7QiWEa04XWnqACA=
|
||||
cloud.google.com/go/pubsub v1.36.2/go.mod h1:mHCFLNG8abCrPzhuOnpBcr9DUy+l3/LWWn0qoJdbh1w=
|
||||
cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
|
||||
cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
|
||||
cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
|
||||
|
|
@ -805,8 +805,8 @@ github.com/rclone/rclone v1.65.2 h1:ZBMzSKqGwimbDWkBqSx6MneRAE7Z5UI0lFxyC8HCGMQ=
|
|||
github.com/rclone/rclone v1.65.2/go.mod h1:nSMqGk0ovcw70cnEJ4xkRXZSL2h2GP8DojLeA+uu1rI=
|
||||
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM=
|
||||
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
|
||||
github.com/rdleal/intervalst v1.2.2 h1:cw93gelL8qIAPuOwT+oHFewumFdrdW/QMFqgRhLUrlY=
|
||||
github.com/rdleal/intervalst v1.2.2/go.mod h1:xO89Z6BC+LQDH+IPQQw/OESt5UADgFD41tYMUINGpxQ=
|
||||
github.com/rdleal/intervalst v1.3.0 h1:Ldf/5uohv4VGsrI7KQgMQ63g2EaSf9GOAhwedIoBhvc=
|
||||
github.com/rdleal/intervalst v1.3.0/go.mod h1:xO89Z6BC+LQDH+IPQQw/OESt5UADgFD41tYMUINGpxQ=
|
||||
github.com/redis/go-redis/v9 v9.0.2 h1:BA426Zqe/7r56kCcvxYLWe1mkaz71LKF77GwgFzSxfE=
|
||||
github.com/redis/go-redis/v9 v9.0.2/go.mod h1:/xDTe9EF1LM61hek62Poq2nzQSGj0xSrEtEHbBQevps=
|
||||
github.com/redis/rueidis v1.0.19 h1:s65oWtotzlIFN8eMPhyYwxlwLR1lUdhza2KtWprKYSo=
|
||||
|
|
@ -877,8 +877,9 @@ github.com/spf13/viper v1.18.2/go.mod h1:EKmWIqdnk5lOcmR72yw6hS+8OPYcwD0jteitLMV
|
|||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
|
||||
github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
|
||||
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
|
||||
github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
|
||||
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
|
||||
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.3.1-0.20190311161405-34c6fa2dc709/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
|
|
@ -890,8 +891,9 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
|
|||
github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals=
|
||||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
|
||||
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
|
||||
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
|
||||
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
||||
github.com/stvp/tempredis v0.0.0-20181119212430-b82af8480203 h1:QVqDTf3h2WHt08YuiTGPZLls0Wq99X9bWd0Q5ZSBesM=
|
||||
github.com/stvp/tempredis v0.0.0-20181119212430-b82af8480203/go.mod h1:oqN97ltKNihBbwlX8dLpwxCl3+HnXKV/R0e+sRLd9C8=
|
||||
github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
|
||||
|
|
@ -902,8 +904,8 @@ github.com/t3rm1n4l/go-mega v0.0.0-20230228171823-a01a2cda13ca h1:I9rVnNXdIkij4U
|
|||
github.com/t3rm1n4l/go-mega v0.0.0-20230228171823-a01a2cda13ca/go.mod h1:suDIky6yrK07NnaBadCB4sS0CqFOvUK91lH7CR+JlDA=
|
||||
github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a h1:J/YdBZ46WKpXsxsW93SG+q0F8KI+yFrcIDT4c/RNoc4=
|
||||
github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a/go.mod h1:h4xBhSNtOeEosLJ4P7JyKXX7Cabg7AVkWCK5gV2vOrM=
|
||||
github.com/tidwall/gjson v1.17.0 h1:/Jocvlh98kcTfpN2+JzGQWQcqrPQwDrVEMApx/M5ZwM=
|
||||
github.com/tidwall/gjson v1.17.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
|
||||
github.com/tidwall/gjson v1.17.1 h1:wlYEnwqAHgzmhNUFfw7Xalt2JzQvsMx2Se4PcoFCT/U=
|
||||
github.com/tidwall/gjson v1.17.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
|
||||
github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
|
||||
github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
|
||||
github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
|
||||
|
|
@ -1254,8 +1256,8 @@ golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
|||
golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y=
|
||||
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
|
||||
golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
|
|
@ -1413,10 +1415,10 @@ google.golang.org/genproto v0.0.0-20200806141610-86f49bd18e98/go.mod h1:FWY/as6D
|
|||
google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||
google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
|
||||
google.golang.org/genproto v0.0.0-20220421151946-72621c1f0bd3/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo=
|
||||
google.golang.org/genproto v0.0.0-20240205150955-31a09d347014 h1:g/4bk7P6TPMkAUbUhquq98xey1slwvuVJPosdBqYJlU=
|
||||
google.golang.org/genproto v0.0.0-20240205150955-31a09d347014/go.mod h1:xEgQu1e4stdSSsxPDK8Azkrk/ECl5HvdPf6nbZrTS5M=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20240205150955-31a09d347014 h1:x9PwdEgd11LgK+orcck69WVRo7DezSO4VUMPI4xpc8A=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20240205150955-31a09d347014/go.mod h1:rbHMSEDyoYX62nRVLOCc4Qt1HbsdytAYoVwgjiOhF3I=
|
||||
google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 h1:9+tzLLstTlPTRyJTh+ah5wIMsBW5c4tQwGTN3thOW9Y=
|
||||
google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9/go.mod h1:mqHbVIp48Muh7Ywss/AD6I5kNVKZMmAa/QEW58Gxp2s=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20240221002015-b0ce06bbee7c h1:9g7erC9qu44ks7UK4gDNlnk4kOxZG707xKm4jVniy6o=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20240221002015-b0ce06bbee7c/go.mod h1:5iCWqnniDlqZHrd3neWVTOwvh/v6s3232omMecelax8=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20240213162025-012b6fc9bca9 h1:hZB7eLIaYlW9qXRfCq/qDaPdbeY3757uARz5Vvfv+cY=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20240213162025-012b6fc9bca9/go.mod h1:YUWgXUFRPfoYK1IHMuxH5K6nPEXSCzIMljnQ59lLRCk=
|
||||
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
|
||||
|
|
|
|||
|
|
@ -85,8 +85,7 @@ const (
|
|||
func doFilerBackup(grpcDialOption grpc.DialOption, backupOption *FilerBackupOptions, clientId int32, clientEpoch int32) error {
|
||||
|
||||
// find data sink
|
||||
config := util.GetViper()
|
||||
dataSink := findSink(config)
|
||||
dataSink := findSink(util.GetViper())
|
||||
if dataSink == nil {
|
||||
return fmt.Errorf("no data sink configured in replication.toml")
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,6 +8,8 @@ import (
|
|||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
"github.com/aws/aws-sdk-go/service/s3/s3iface"
|
||||
"github.com/aws/aws-sdk-go/service/s3/s3manager"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/filer"
|
||||
|
|
@ -19,14 +21,20 @@ import (
|
|||
)
|
||||
|
||||
type S3Sink struct {
|
||||
conn s3iface.S3API
|
||||
region string
|
||||
bucket string
|
||||
dir string
|
||||
endpoint string
|
||||
acl string
|
||||
filerSource *source.FilerSource
|
||||
isIncremental bool
|
||||
conn s3iface.S3API
|
||||
filerSource *source.FilerSource
|
||||
isIncremental bool
|
||||
keepPartSize bool
|
||||
s3DisableContentMD5Validation bool
|
||||
s3ForcePathStyle bool
|
||||
uploaderConcurrency int
|
||||
uploaderMaxUploadParts int
|
||||
uploaderPartSizeMb int
|
||||
region string
|
||||
bucket string
|
||||
dir string
|
||||
endpoint string
|
||||
acl string
|
||||
}
|
||||
|
||||
func init() {
|
||||
|
|
@ -46,21 +54,49 @@ func (s3sink *S3Sink) IsIncremental() bool {
|
|||
}
|
||||
|
||||
func (s3sink *S3Sink) Initialize(configuration util.Configuration, prefix string) error {
|
||||
glog.V(0).Infof("sink.s3.region: %v", configuration.GetString(prefix+"region"))
|
||||
glog.V(0).Infof("sink.s3.bucket: %v", configuration.GetString(prefix+"bucket"))
|
||||
glog.V(0).Infof("sink.s3.directory: %v", configuration.GetString(prefix+"directory"))
|
||||
glog.V(0).Infof("sink.s3.endpoint: %v", configuration.GetString(prefix+"endpoint"))
|
||||
glog.V(0).Infof("sink.s3.acl: %v", configuration.GetString(prefix+"acl"))
|
||||
glog.V(0).Infof("sink.s3.is_incremental: %v", configuration.GetString(prefix+"is_incremental"))
|
||||
configuration.SetDefault(prefix+"region", "us-east-2")
|
||||
configuration.SetDefault(prefix+"directory", "/")
|
||||
configuration.SetDefault(prefix+"keep_part_size", true)
|
||||
configuration.SetDefault(prefix+"uploader_max_upload_parts", 1000)
|
||||
configuration.SetDefault(prefix+"uploader_part_size_mb", 8)
|
||||
configuration.SetDefault(prefix+"uploader_concurrency", 8)
|
||||
configuration.SetDefault(prefix+"s3_disable_content_md5_validation", true)
|
||||
configuration.SetDefault(prefix+"s3_force_path_style", true)
|
||||
s3sink.region = configuration.GetString(prefix + "region")
|
||||
s3sink.bucket = configuration.GetString(prefix + "bucket")
|
||||
s3sink.dir = configuration.GetString(prefix + "directory")
|
||||
s3sink.endpoint = configuration.GetString(prefix + "endpoint")
|
||||
s3sink.acl = configuration.GetString(prefix + "acl")
|
||||
s3sink.isIncremental = configuration.GetBool(prefix + "is_incremental")
|
||||
s3sink.keepPartSize = configuration.GetBool(prefix + "keep_part_size")
|
||||
s3sink.s3DisableContentMD5Validation = configuration.GetBool(prefix + "s3_disable_content_md5_validation")
|
||||
s3sink.s3ForcePathStyle = configuration.GetBool(prefix + "s3_force_path_style")
|
||||
s3sink.uploaderMaxUploadParts = configuration.GetInt(prefix + "uploader_max_upload_parts")
|
||||
s3sink.uploaderPartSizeMb = configuration.GetInt(prefix + "uploader_part_size")
|
||||
s3sink.uploaderConcurrency = configuration.GetInt(prefix + "uploader_concurrency")
|
||||
|
||||
glog.V(0).Infof("sink.s3.region: %v", s3sink.region)
|
||||
glog.V(0).Infof("sink.s3.bucket: %v", s3sink.bucket)
|
||||
glog.V(0).Infof("sink.s3.directory: %v", s3sink.dir)
|
||||
glog.V(0).Infof("sink.s3.endpoint: %v", s3sink.endpoint)
|
||||
glog.V(0).Infof("sink.s3.acl: %v", s3sink.acl)
|
||||
glog.V(0).Infof("sink.s3.is_incremental: %v", s3sink.isIncremental)
|
||||
glog.V(0).Infof("sink.s3.s3_disable_content_md5_validation: %v", s3sink.s3DisableContentMD5Validation)
|
||||
glog.V(0).Infof("sink.s3.s3_force_path_style: %v", s3sink.s3ForcePathStyle)
|
||||
glog.V(0).Infof("sink.s3.keep_part_size: %v", s3sink.keepPartSize)
|
||||
if s3sink.uploaderMaxUploadParts > s3manager.MaxUploadParts {
|
||||
s3sink.uploaderMaxUploadParts = s3manager.MaxUploadParts
|
||||
glog.Warningf("uploader_max_upload_parts is greater than the maximum number of parts allowed when uploading multiple parts to Amazon S3")
|
||||
glog.V(0).Infof("sink.s3.uploader_max_upload_parts: %v => %v", s3sink.uploaderMaxUploadParts, s3manager.MaxUploadParts)
|
||||
} else {
|
||||
glog.V(0).Infof("sink.s3.uploader_max_upload_parts: %v", s3sink.uploaderMaxUploadParts)
|
||||
}
|
||||
glog.V(0).Infof("sink.s3.uploader_part_size_mb: %v", s3sink.uploaderPartSizeMb)
|
||||
glog.V(0).Infof("sink.s3.uploader_concurrency: %v", s3sink.uploaderConcurrency)
|
||||
|
||||
return s3sink.initialize(
|
||||
configuration.GetString(prefix+"aws_access_key_id"),
|
||||
configuration.GetString(prefix+"aws_secret_access_key"),
|
||||
configuration.GetString(prefix+"region"),
|
||||
configuration.GetString(prefix+"bucket"),
|
||||
configuration.GetString(prefix+"directory"),
|
||||
configuration.GetString(prefix+"endpoint"),
|
||||
configuration.GetString(prefix+"acl"),
|
||||
)
|
||||
}
|
||||
|
||||
|
|
@ -68,18 +104,12 @@ func (s3sink *S3Sink) SetSourceFiler(s *source.FilerSource) {
|
|||
s3sink.filerSource = s
|
||||
}
|
||||
|
||||
func (s3sink *S3Sink) initialize(awsAccessKeyId, awsSecretAccessKey, region, bucket, dir, endpoint, acl string) error {
|
||||
s3sink.region = region
|
||||
s3sink.bucket = bucket
|
||||
s3sink.dir = dir
|
||||
s3sink.endpoint = endpoint
|
||||
s3sink.acl = acl
|
||||
|
||||
func (s3sink *S3Sink) initialize(awsAccessKeyId, awsSecretAccessKey string) error {
|
||||
config := &aws.Config{
|
||||
Region: aws.String(s3sink.region),
|
||||
Endpoint: aws.String(s3sink.endpoint),
|
||||
S3ForcePathStyle: aws.Bool(true),
|
||||
S3DisableContentMD5Validation: aws.Bool(true),
|
||||
S3DisableContentMD5Validation: aws.Bool(s3sink.s3DisableContentMD5Validation),
|
||||
S3ForcePathStyle: aws.Bool(s3sink.s3ForcePathStyle),
|
||||
}
|
||||
if awsAccessKeyId != "" && awsSecretAccessKey != "" {
|
||||
config.Credentials = credentials.NewStaticCredentials(awsAccessKeyId, awsSecretAccessKey, "")
|
||||
|
|
@ -128,19 +158,26 @@ func (s3sink *S3Sink) CreateEntry(key string, entry *filer_pb.Entry, signatures
|
|||
|
||||
reader := filer.NewFileReader(s3sink.filerSource, entry)
|
||||
|
||||
fileSize := int64(filer.FileSize(entry))
|
||||
|
||||
partSize := int64(8 * 1024 * 1024) // The minimum/default allowed part size is 5MB
|
||||
for partSize*1000 < fileSize {
|
||||
partSize *= 4
|
||||
}
|
||||
|
||||
// Create an uploader with the session and custom options
|
||||
uploader := s3manager.NewUploaderWithClient(s3sink.conn, func(u *s3manager.Uploader) {
|
||||
u.PartSize = partSize
|
||||
u.Concurrency = 8
|
||||
u.PartSize = int64(s3sink.uploaderPartSizeMb * 1024 * 1024)
|
||||
u.Concurrency = s3sink.uploaderConcurrency
|
||||
u.MaxUploadParts = s3sink.uploaderMaxUploadParts
|
||||
})
|
||||
|
||||
if s3sink.keepPartSize {
|
||||
switch chunkCount := len(entry.Chunks); {
|
||||
case chunkCount > 1:
|
||||
if firstChunkSize := int64(entry.Chunks[0].Size); firstChunkSize > s3manager.MinUploadPartSize {
|
||||
uploader.PartSize = firstChunkSize
|
||||
}
|
||||
default:
|
||||
uploader.PartSize = 0
|
||||
}
|
||||
}
|
||||
if _, ok := entry.Extended[s3_constants.AmzUserMetaMtime]; !ok {
|
||||
entry.Extended[s3_constants.AmzUserMetaMtime] = []byte(strconv.FormatInt(entry.Attributes.Mtime, 10))
|
||||
}
|
||||
// process tagging
|
||||
tags := ""
|
||||
if true {
|
||||
|
|
@ -153,14 +190,18 @@ func (s3sink *S3Sink) CreateEntry(key string, entry *filer_pb.Entry, signatures
|
|||
}
|
||||
|
||||
// Upload the file to S3.
|
||||
_, err = uploader.Upload(&s3manager.UploadInput{
|
||||
uploadInput := s3manager.UploadInput{
|
||||
Bucket: aws.String(s3sink.bucket),
|
||||
Key: aws.String(key),
|
||||
Body: reader,
|
||||
Tagging: aws.String(tags),
|
||||
})
|
||||
}
|
||||
if len(entry.Attributes.Md5) > 0 {
|
||||
uploadInput.ContentMD5 = aws.String(fmt.Sprintf("%x", entry.Attributes.Md5))
|
||||
}
|
||||
_, err = uploader.Upload(&uploadInput)
|
||||
|
||||
return
|
||||
return err
|
||||
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -30,6 +30,7 @@ const (
|
|||
// S3 user-defined metadata
|
||||
AmzUserMetaPrefix = "X-Amz-Meta-"
|
||||
AmzUserMetaDirective = "X-Amz-Metadata-Directive"
|
||||
AmzUserMetaMtime = "X-Amz-Meta-Mtime"
|
||||
|
||||
// S3 object tagging
|
||||
AmzObjectTagging = "X-Amz-Tagging"
|
||||
|
|
|
|||
|
|
@ -47,10 +47,6 @@ func (s3a *S3ApiServer) ListObjectsV2Handler(w http.ResponseWriter, r *http.Requ
|
|||
s3err.WriteErrorResponse(w, r, s3err.ErrInvalidMaxKeys)
|
||||
return
|
||||
}
|
||||
if delimiter != "" && delimiter != "/" {
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrNotImplemented)
|
||||
return
|
||||
}
|
||||
|
||||
marker := continuationToken
|
||||
if continuationToken == "" {
|
||||
|
|
@ -103,10 +99,6 @@ func (s3a *S3ApiServer) ListObjectsV1Handler(w http.ResponseWriter, r *http.Requ
|
|||
s3err.WriteErrorResponse(w, r, s3err.ErrInvalidMaxKeys)
|
||||
return
|
||||
}
|
||||
if delimiter != "" && delimiter != "/" {
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrNotImplemented)
|
||||
return
|
||||
}
|
||||
|
||||
response, err := s3a.listFilerEntries(bucket, originalPrefix, maxKeys, marker, delimiter)
|
||||
|
||||
|
|
@ -171,22 +163,51 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m
|
|||
cursor.maxKeys--
|
||||
}
|
||||
} else {
|
||||
storageClass := "STANDARD"
|
||||
if v, ok := entry.Extended[s3_constants.AmzStorageClass]; ok {
|
||||
storageClass = string(v)
|
||||
var delimiterFound bool
|
||||
if delimiter != "" {
|
||||
// keys that contain the same string between the prefix and the first occurrence of the delimiter are grouped together as a commonPrefix.
|
||||
// extract the string between the prefix and the delimiter and add it to the commonPrefixes if it's unique.
|
||||
fullPath := fmt.Sprintf("%s/%s", dir, entry.Name)[len(bucketPrefix):]
|
||||
delimitedPath := strings.SplitN(fullPath, delimiter, 2)
|
||||
if len(delimitedPath) == 2 {
|
||||
|
||||
// S3 clients expect the delimited prefix to contain the delimiter.
|
||||
delimitedPrefix := delimitedPath[0] + delimiter
|
||||
|
||||
for i := range commonPrefixes {
|
||||
if commonPrefixes[i].Prefix == delimitedPrefix {
|
||||
delimiterFound = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if !delimiterFound {
|
||||
commonPrefixes = append(commonPrefixes, PrefixEntry{
|
||||
Prefix: delimitedPrefix,
|
||||
})
|
||||
cursor.maxKeys--
|
||||
delimiterFound = true
|
||||
}
|
||||
}
|
||||
}
|
||||
if !delimiterFound {
|
||||
storageClass := "STANDARD"
|
||||
if v, ok := entry.Extended[s3_constants.AmzStorageClass]; ok {
|
||||
storageClass = string(v)
|
||||
}
|
||||
contents = append(contents, ListEntry{
|
||||
Key: fmt.Sprintf("%s/%s", dir, entry.Name)[len(bucketPrefix):],
|
||||
LastModified: time.Unix(entry.Attributes.Mtime, 0).UTC(),
|
||||
ETag: "\"" + filer.ETag(entry) + "\"",
|
||||
Size: int64(filer.FileSize(entry)),
|
||||
Owner: CanonicalUser{
|
||||
ID: fmt.Sprintf("%x", entry.Attributes.Uid),
|
||||
DisplayName: entry.Attributes.UserName,
|
||||
},
|
||||
StorageClass: StorageClass(storageClass),
|
||||
})
|
||||
cursor.maxKeys--
|
||||
}
|
||||
contents = append(contents, ListEntry{
|
||||
Key: fmt.Sprintf("%s/%s", dir, entry.Name)[len(bucketPrefix):],
|
||||
LastModified: time.Unix(entry.Attributes.Mtime, 0).UTC(),
|
||||
ETag: "\"" + filer.ETag(entry) + "\"",
|
||||
Size: int64(filer.FileSize(entry)),
|
||||
Owner: CanonicalUser{
|
||||
ID: fmt.Sprintf("%x", entry.Attributes.Uid),
|
||||
DisplayName: entry.Attributes.UserName,
|
||||
},
|
||||
StorageClass: StorageClass(storageClass),
|
||||
})
|
||||
cursor.maxKeys--
|
||||
}
|
||||
})
|
||||
if doErr != nil {
|
||||
|
|
|
|||
|
|
@ -84,8 +84,6 @@ func (fs *FilerServer) Ping(ctx context.Context, req *filer_pb.PingRequest) (res
|
|||
|
||||
func (fs *FilerServer) GetFilerConfiguration(ctx context.Context, req *filer_pb.GetFilerConfigurationRequest) (resp *filer_pb.GetFilerConfigurationResponse, err error) {
|
||||
|
||||
clusterId, _ := fs.filer.Store.KvGet(context.Background(), []byte("clusterId"))
|
||||
|
||||
t := &filer_pb.GetFilerConfigurationResponse{
|
||||
Masters: fs.option.Masters.GetInstancesAsStrings(),
|
||||
Collection: fs.option.Collection,
|
||||
|
|
@ -97,7 +95,6 @@ func (fs *FilerServer) GetFilerConfiguration(ctx context.Context, req *filer_pb.
|
|||
MetricsAddress: fs.metricsAddress,
|
||||
MetricsIntervalSec: int32(fs.metricsIntervalSec),
|
||||
Version: util.Version(),
|
||||
ClusterId: string(clusterId),
|
||||
FilerGroup: fs.option.FilerGroup,
|
||||
}
|
||||
|
||||
|
|
|
|||
53
weed/shell/command_fs_log.go
Normal file
53
weed/shell/command_fs_log.go
Normal file
|
|
@ -0,0 +1,53 @@
|
|||
package shell
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"github.com/seaweedfs/seaweedfs/weed/filer"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"io"
|
||||
"time"
|
||||
)
|
||||
|
||||
func init() {
|
||||
Commands = append(Commands, &commandFsLogPurge{})
|
||||
}
|
||||
|
||||
type commandFsLogPurge struct {
|
||||
}
|
||||
|
||||
func (c *commandFsLogPurge) Name() string {
|
||||
return "fs.log.purge"
|
||||
}
|
||||
|
||||
func (c *commandFsLogPurge) Help() string {
|
||||
return `purge filer logs
|
||||
|
||||
fs.log.purge [-v] [-modifyDayAgo 365]
|
||||
`
|
||||
}
|
||||
|
||||
func (c *commandFsLogPurge) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
|
||||
fsLogPurgeCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
|
||||
daysAgo := fsLogPurgeCommand.Uint("daysAgo", 365, "purge logs older than N days")
|
||||
verbose := fsLogPurgeCommand.Bool("v", false, "verbose mode")
|
||||
|
||||
if err = fsLogPurgeCommand.Parse(args); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
modificationTimeAgo := time.Now().Add(-time.Hour * 24 * time.Duration(*daysAgo)).Unix()
|
||||
err = filer_pb.ReadDirAllEntries(commandEnv, filer.SystemLogDir, "", func(entry *filer_pb.Entry, isLast bool) error {
|
||||
if entry.Attributes.Mtime > modificationTimeAgo {
|
||||
return nil
|
||||
}
|
||||
if errDel := filer_pb.Remove(commandEnv, filer.SystemLogDir, entry.Name, true, true, true, false, nil); errDel != nil {
|
||||
return errDel
|
||||
}
|
||||
if *verbose {
|
||||
fmt.Fprintf(writer, "delete %s\n", entry.Name)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
|
@ -5,7 +5,6 @@ import (
|
|||
"fmt"
|
||||
"github.com/seaweedfs/seaweedfs/weed/cluster"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util/grace"
|
||||
|
|
@ -74,24 +73,6 @@ func RunShell(options ShellOptions) {
|
|||
fmt.Println()
|
||||
}
|
||||
|
||||
if commandEnv.option.FilerAddress != "" {
|
||||
commandEnv.WithFilerClient(false, func(filerClient filer_pb.SeaweedFilerClient) error {
|
||||
resp, err := filerClient.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if resp.ClusterId != "" {
|
||||
fmt.Printf(`
|
||||
---
|
||||
Free Monitoring Data URL:
|
||||
https://cloud.seaweedfs.com/ui/%s
|
||||
---
|
||||
`, resp.ClusterId)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
for {
|
||||
cmd, err := line.Prompt("> ")
|
||||
if err != nil {
|
||||
|
|
|
|||
|
|
@ -251,6 +251,7 @@ func (s *Store) CollectHeartbeat() *master_pb.Heartbeat {
|
|||
maxVolumeCounts := make(map[string]uint32)
|
||||
var maxFileKey NeedleId
|
||||
collectionVolumeSize := make(map[string]int64)
|
||||
collectionVolumeDeletedBytes := make(map[string]int64)
|
||||
collectionVolumeReadOnlyCount := make(map[string]map[string]uint8)
|
||||
for _, location := range s.Locations {
|
||||
var deleteVids []needle.VolumeId
|
||||
|
|
@ -283,9 +284,11 @@ func (s *Store) CollectHeartbeat() *master_pb.Heartbeat {
|
|||
|
||||
if _, exist := collectionVolumeSize[v.Collection]; !exist {
|
||||
collectionVolumeSize[v.Collection] = 0
|
||||
collectionVolumeDeletedBytes[v.Collection] = 0
|
||||
}
|
||||
if !shouldDeleteVolume {
|
||||
collectionVolumeSize[v.Collection] += int64(volumeMessage.Size)
|
||||
collectionVolumeDeletedBytes[v.Collection] += int64(volumeMessage.DeletedByteCount)
|
||||
} else {
|
||||
collectionVolumeSize[v.Collection] -= int64(volumeMessage.Size)
|
||||
if collectionVolumeSize[v.Collection] <= 0 {
|
||||
|
|
@ -342,6 +345,10 @@ func (s *Store) CollectHeartbeat() *master_pb.Heartbeat {
|
|||
stats.VolumeServerDiskSizeGauge.WithLabelValues(col, "normal").Set(float64(size))
|
||||
}
|
||||
|
||||
for col, deletedBytes := range collectionVolumeDeletedBytes{
|
||||
stats.VolumeServerDiskSizeGauge.WithLabelValues(col, "deleted_bytes").Set(float64(deletedBytes))
|
||||
}
|
||||
|
||||
for col, types := range collectionVolumeReadOnlyCount {
|
||||
for t, count := range types {
|
||||
stats.VolumeServerReadOnlyVolumeGauge.WithLabelValues(col, t).Set(float64(count))
|
||||
|
|
|
|||
Loading…
Reference in a new issue