mirror of
https://github.com/chrislusf/seaweedfs
synced 2025-08-17 01:22:47 +02:00
* initial design * added simulation as tests * reorganized the codebase to move the simulation framework and tests into their own dedicated package * integration test. ec worker task * remove "enhanced" reference * start master, volume servers, filer Current Status ✅ Master: Healthy and running (port 9333) ✅ Filer: Healthy and running (port 8888) ✅ Volume Servers: All 6 servers running (ports 8080-8085) 🔄 Admin/Workers: Will start when dependencies are ready * generate write load * tasks are assigned * admin start wtih grpc port. worker has its own working directory * Update .gitignore * working worker and admin. Task detection is not working yet. * compiles, detection uses volumeSizeLimitMB from master * compiles * worker retries connecting to admin * build and restart * rendering pending tasks * skip task ID column * sticky worker id * test canScheduleTaskNow * worker reconnect to admin * clean up logs * worker register itself first * worker can run ec work and report status but: 1. one volume should not be repeatedly worked on. 2. ec shards needs to be distributed and source data should be deleted. * move ec task logic * listing ec shards * local copy, ec. Need to distribute. * ec is mostly working now * distribution of ec shards needs improvement * need configuration to enable ec * show ec volumes * interval field UI component * rename * integration test with vauuming * garbage percentage threshold * fix warning * display ec shard sizes * fix ec volumes list * Update ui.go * show default values * ensure correct default value * MaintenanceConfig use ConfigField * use schema defined defaults * config * reduce duplication * refactor to use BaseUIProvider * each task register its schema * checkECEncodingCandidate use ecDetector * use vacuumDetector * use volumeSizeLimitMB * remove remove * remove unused * refactor * use new framework * remove v2 reference * refactor * left menu can scroll now * The maintenance manager was not being initialized when no data directory was configured for persistent storage. * saving config * Update task_config_schema_templ.go * enable/disable tasks * protobuf encoded task configurations * fix system settings * use ui component * remove logs * interface{} Reduction * reduce interface{} * reduce interface{} * avoid from/to map * reduce interface{} * refactor * keep it DRY * added logging * debug messages * debug level * debug * show the log caller line * use configured task policy * log level * handle admin heartbeat response * Update worker.go * fix EC rack and dc count * Report task status to admin server * fix task logging, simplify interface checking, use erasure_coding constants * factor in empty volume server during task planning * volume.list adds disk id * track disk id also * fix locking scheduled and manual scanning * add active topology * simplify task detector * ec task completed, but shards are not showing up * implement ec in ec_typed.go * adjust log level * dedup * implementing ec copying shards and only ecx files * use disk id when distributing ec shards 🎯 Planning: ActiveTopology creates DestinationPlan with specific TargetDisk 📦 Task Creation: maintenance_integration.go creates ECDestination with DiskId 🚀 Task Execution: EC task passes DiskId in VolumeEcShardsCopyRequest 💾 Volume Server: Receives disk_id and stores shards on specific disk (vs.store.Locations[req.DiskId]) 📂 File System: EC shards and metadata land in the exact disk directory planned * Delete original volume from all locations * clean up existing shard locations * local encoding and distributing * Update docker/admin_integration/EC-TESTING-README.md Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> * check volume id range * simplify * fix tests * fix types * clean up logs and tests --------- Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
124 lines
3.8 KiB
Go
124 lines
3.8 KiB
Go
package maintenance
|
|
|
|
import (
|
|
"fmt"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
|
|
)
|
|
|
|
// VerifyProtobufConfig demonstrates that the protobuf configuration system is working
|
|
func VerifyProtobufConfig() error {
|
|
// Create configuration manager
|
|
configManager := NewMaintenanceConfigManager()
|
|
config := configManager.GetConfig()
|
|
|
|
// Verify basic configuration
|
|
if !config.Enabled {
|
|
return fmt.Errorf("expected config to be enabled by default")
|
|
}
|
|
|
|
if config.ScanIntervalSeconds != 30*60 {
|
|
return fmt.Errorf("expected scan interval to be 1800 seconds, got %d", config.ScanIntervalSeconds)
|
|
}
|
|
|
|
// Verify policy configuration
|
|
if config.Policy == nil {
|
|
return fmt.Errorf("expected policy to be configured")
|
|
}
|
|
|
|
if config.Policy.GlobalMaxConcurrent != 4 {
|
|
return fmt.Errorf("expected global max concurrent to be 4, got %d", config.Policy.GlobalMaxConcurrent)
|
|
}
|
|
|
|
// Verify task policies
|
|
vacuumPolicy := config.Policy.TaskPolicies["vacuum"]
|
|
if vacuumPolicy == nil {
|
|
return fmt.Errorf("expected vacuum policy to be configured")
|
|
}
|
|
|
|
if !vacuumPolicy.Enabled {
|
|
return fmt.Errorf("expected vacuum policy to be enabled")
|
|
}
|
|
|
|
// Verify typed configuration access
|
|
vacuumConfig := vacuumPolicy.GetVacuumConfig()
|
|
if vacuumConfig == nil {
|
|
return fmt.Errorf("expected vacuum config to be accessible")
|
|
}
|
|
|
|
if vacuumConfig.GarbageThreshold != 0.3 {
|
|
return fmt.Errorf("expected garbage threshold to be 0.3, got %f", vacuumConfig.GarbageThreshold)
|
|
}
|
|
|
|
// Verify helper functions work
|
|
if !IsTaskEnabled(config.Policy, "vacuum") {
|
|
return fmt.Errorf("expected vacuum task to be enabled via helper function")
|
|
}
|
|
|
|
maxConcurrent := GetMaxConcurrent(config.Policy, "vacuum")
|
|
if maxConcurrent != 2 {
|
|
return fmt.Errorf("expected vacuum max concurrent to be 2, got %d", maxConcurrent)
|
|
}
|
|
|
|
// Verify erasure coding configuration
|
|
ecPolicy := config.Policy.TaskPolicies["erasure_coding"]
|
|
if ecPolicy == nil {
|
|
return fmt.Errorf("expected EC policy to be configured")
|
|
}
|
|
|
|
ecConfig := ecPolicy.GetErasureCodingConfig()
|
|
if ecConfig == nil {
|
|
return fmt.Errorf("expected EC config to be accessible")
|
|
}
|
|
|
|
// Verify configurable EC fields only
|
|
if ecConfig.FullnessRatio <= 0 || ecConfig.FullnessRatio > 1 {
|
|
return fmt.Errorf("expected EC config to have valid fullness ratio (0-1), got %f", ecConfig.FullnessRatio)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// GetProtobufConfigSummary returns a summary of the current protobuf configuration
|
|
func GetProtobufConfigSummary() string {
|
|
configManager := NewMaintenanceConfigManager()
|
|
config := configManager.GetConfig()
|
|
|
|
summary := fmt.Sprintf("SeaweedFS Protobuf Maintenance Configuration:\n")
|
|
summary += fmt.Sprintf(" Enabled: %v\n", config.Enabled)
|
|
summary += fmt.Sprintf(" Scan Interval: %d seconds\n", config.ScanIntervalSeconds)
|
|
summary += fmt.Sprintf(" Max Retries: %d\n", config.MaxRetries)
|
|
summary += fmt.Sprintf(" Global Max Concurrent: %d\n", config.Policy.GlobalMaxConcurrent)
|
|
summary += fmt.Sprintf(" Task Policies: %d configured\n", len(config.Policy.TaskPolicies))
|
|
|
|
for taskType, policy := range config.Policy.TaskPolicies {
|
|
summary += fmt.Sprintf(" %s: enabled=%v, max_concurrent=%d\n",
|
|
taskType, policy.Enabled, policy.MaxConcurrent)
|
|
}
|
|
|
|
return summary
|
|
}
|
|
|
|
// CreateCustomConfig demonstrates creating a custom protobuf configuration
|
|
func CreateCustomConfig() *worker_pb.MaintenanceConfig {
|
|
return &worker_pb.MaintenanceConfig{
|
|
Enabled: true,
|
|
ScanIntervalSeconds: 60 * 60, // 1 hour
|
|
MaxRetries: 5,
|
|
Policy: &worker_pb.MaintenancePolicy{
|
|
GlobalMaxConcurrent: 8,
|
|
TaskPolicies: map[string]*worker_pb.TaskPolicy{
|
|
"custom_vacuum": {
|
|
Enabled: true,
|
|
MaxConcurrent: 4,
|
|
TaskConfig: &worker_pb.TaskPolicy_VacuumConfig{
|
|
VacuumConfig: &worker_pb.VacuumTaskConfig{
|
|
GarbageThreshold: 0.5,
|
|
MinVolumeAgeHours: 48,
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
}
|