1
0
Fork 0
mirror of https://github.com/chrislusf/seaweedfs synced 2025-08-17 01:22:47 +02:00
seaweedfs/weed/admin/config/schema_test.go
Chris Lu 891a2fb6eb
Admin: misc improvements on admin server and workers. EC now works. (#7055)
* initial design

* added simulation as tests

* reorganized the codebase to move the simulation framework and tests into their own dedicated package

* integration test. ec worker task

* remove "enhanced" reference

* start master, volume servers, filer

Current Status
 Master: Healthy and running (port 9333)
 Filer: Healthy and running (port 8888)
 Volume Servers: All 6 servers running (ports 8080-8085)
🔄 Admin/Workers: Will start when dependencies are ready

* generate write load

* tasks are assigned

* admin start wtih grpc port. worker has its own working directory

* Update .gitignore

* working worker and admin. Task detection is not working yet.

* compiles, detection uses volumeSizeLimitMB from master

* compiles

* worker retries connecting to admin

* build and restart

* rendering pending tasks

* skip task ID column

* sticky worker id

* test canScheduleTaskNow

* worker reconnect to admin

* clean up logs

* worker register itself first

* worker can run ec work and report status

but:
1. one volume should not be repeatedly worked on.
2. ec shards needs to be distributed and source data should be deleted.

* move ec task logic

* listing ec shards

* local copy, ec. Need to distribute.

* ec is mostly working now

* distribution of ec shards needs improvement
* need configuration to enable ec

* show ec volumes

* interval field UI component

* rename

* integration test with vauuming

* garbage percentage threshold

* fix warning

* display ec shard sizes

* fix ec volumes list

* Update ui.go

* show default values

* ensure correct default value

* MaintenanceConfig use ConfigField

* use schema defined defaults

* config

* reduce duplication

* refactor to use BaseUIProvider

* each task register its schema

* checkECEncodingCandidate use ecDetector

* use vacuumDetector

* use volumeSizeLimitMB

* remove

remove

* remove unused

* refactor

* use new framework

* remove v2 reference

* refactor

* left menu can scroll now

* The maintenance manager was not being initialized when no data directory was configured for persistent storage.

* saving config

* Update task_config_schema_templ.go

* enable/disable tasks

* protobuf encoded task configurations

* fix system settings

* use ui component

* remove logs

* interface{} Reduction

* reduce interface{}

* reduce interface{}

* avoid from/to map

* reduce interface{}

* refactor

* keep it DRY

* added logging

* debug messages

* debug level

* debug

* show the log caller line

* use configured task policy

* log level

* handle admin heartbeat response

* Update worker.go

* fix EC rack and dc count

* Report task status to admin server

* fix task logging, simplify interface checking, use erasure_coding constants

* factor in empty volume server during task planning

* volume.list adds disk id

* track disk id also

* fix locking scheduled and manual scanning

* add active topology

* simplify task detector

* ec task completed, but shards are not showing up

* implement ec in ec_typed.go

* adjust log level

* dedup

* implementing ec copying shards and only ecx files

* use disk id when distributing ec shards

🎯 Planning: ActiveTopology creates DestinationPlan with specific TargetDisk
📦 Task Creation: maintenance_integration.go creates ECDestination with DiskId
🚀 Task Execution: EC task passes DiskId in VolumeEcShardsCopyRequest
💾 Volume Server: Receives disk_id and stores shards on specific disk (vs.store.Locations[req.DiskId])
📂 File System: EC shards and metadata land in the exact disk directory planned

* Delete original volume from all locations

* clean up existing shard locations

* local encoding and distributing

* Update docker/admin_integration/EC-TESTING-README.md

Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>

* check volume id range

* simplify

* fix tests

* fix types

* clean up logs and tests

---------

Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
2025-07-30 12:38:03 -07:00

226 lines
6.3 KiB
Go

package config
import (
"testing"
)
// Test structs that mirror the actual configuration structure
type TestBaseConfigForSchema struct {
Enabled bool `json:"enabled"`
ScanIntervalSeconds int `json:"scan_interval_seconds"`
MaxConcurrent int `json:"max_concurrent"`
}
// ApplySchemaDefaults implements ConfigWithDefaults for test struct
func (c *TestBaseConfigForSchema) ApplySchemaDefaults(schema *Schema) error {
return schema.ApplyDefaultsToProtobuf(c)
}
// Validate implements ConfigWithDefaults for test struct
func (c *TestBaseConfigForSchema) Validate() error {
return nil
}
type TestTaskConfigForSchema struct {
TestBaseConfigForSchema
TaskSpecificField float64 `json:"task_specific_field"`
AnotherSpecificField string `json:"another_specific_field"`
}
// ApplySchemaDefaults implements ConfigWithDefaults for test struct
func (c *TestTaskConfigForSchema) ApplySchemaDefaults(schema *Schema) error {
return schema.ApplyDefaultsToProtobuf(c)
}
// Validate implements ConfigWithDefaults for test struct
func (c *TestTaskConfigForSchema) Validate() error {
return nil
}
func createTestSchema() *Schema {
return &Schema{
Fields: []*Field{
{
Name: "enabled",
JSONName: "enabled",
Type: FieldTypeBool,
DefaultValue: true,
},
{
Name: "scan_interval_seconds",
JSONName: "scan_interval_seconds",
Type: FieldTypeInt,
DefaultValue: 1800,
},
{
Name: "max_concurrent",
JSONName: "max_concurrent",
Type: FieldTypeInt,
DefaultValue: 3,
},
{
Name: "task_specific_field",
JSONName: "task_specific_field",
Type: FieldTypeFloat,
DefaultValue: 0.25,
},
{
Name: "another_specific_field",
JSONName: "another_specific_field",
Type: FieldTypeString,
DefaultValue: "default_value",
},
},
}
}
func TestApplyDefaults_WithEmbeddedStruct(t *testing.T) {
schema := createTestSchema()
// Start with zero values
config := &TestTaskConfigForSchema{}
err := schema.ApplyDefaultsToConfig(config)
if err != nil {
t.Fatalf("ApplyDefaultsToConfig failed: %v", err)
}
// Verify embedded struct fields got default values
if config.Enabled != true {
t.Errorf("Expected Enabled=true (default), got %v", config.Enabled)
}
if config.ScanIntervalSeconds != 1800 {
t.Errorf("Expected ScanIntervalSeconds=1800 (default), got %v", config.ScanIntervalSeconds)
}
if config.MaxConcurrent != 3 {
t.Errorf("Expected MaxConcurrent=3 (default), got %v", config.MaxConcurrent)
}
// Verify task-specific fields got default values
if config.TaskSpecificField != 0.25 {
t.Errorf("Expected TaskSpecificField=0.25 (default), got %v", config.TaskSpecificField)
}
if config.AnotherSpecificField != "default_value" {
t.Errorf("Expected AnotherSpecificField='default_value' (default), got %v", config.AnotherSpecificField)
}
}
func TestApplyDefaults_PartiallySet(t *testing.T) {
schema := createTestSchema()
// Start with some pre-set values
config := &TestTaskConfigForSchema{
TestBaseConfigForSchema: TestBaseConfigForSchema{
Enabled: true, // Non-zero value, should not be overridden
ScanIntervalSeconds: 0, // Should get default
MaxConcurrent: 5, // Non-zero value, should not be overridden
},
TaskSpecificField: 0.0, // Should get default
AnotherSpecificField: "custom", // Non-zero value, should not be overridden
}
err := schema.ApplyDefaultsToConfig(config)
if err != nil {
t.Fatalf("ApplyDefaultsToConfig failed: %v", err)
}
// Verify already-set values are preserved
if config.Enabled != true {
t.Errorf("Expected Enabled=true (pre-set), got %v", config.Enabled)
}
if config.MaxConcurrent != 5 {
t.Errorf("Expected MaxConcurrent=5 (pre-set), got %v", config.MaxConcurrent)
}
if config.AnotherSpecificField != "custom" {
t.Errorf("Expected AnotherSpecificField='custom' (pre-set), got %v", config.AnotherSpecificField)
}
// Verify zero values got defaults
if config.ScanIntervalSeconds != 1800 {
t.Errorf("Expected ScanIntervalSeconds=1800 (default), got %v", config.ScanIntervalSeconds)
}
if config.TaskSpecificField != 0.25 {
t.Errorf("Expected TaskSpecificField=0.25 (default), got %v", config.TaskSpecificField)
}
}
func TestApplyDefaults_NonPointer(t *testing.T) {
schema := createTestSchema()
config := TestTaskConfigForSchema{}
// This should fail since we need a pointer to modify the struct
err := schema.ApplyDefaultsToProtobuf(config)
if err == nil {
t.Fatal("Expected error for non-pointer config, but got nil")
}
}
func TestApplyDefaults_NonStruct(t *testing.T) {
schema := createTestSchema()
var config interface{} = "not a struct"
err := schema.ApplyDefaultsToProtobuf(config)
if err == nil {
t.Fatal("Expected error for non-struct config, but got nil")
}
}
func TestApplyDefaults_EmptySchema(t *testing.T) {
schema := &Schema{Fields: []*Field{}}
config := &TestTaskConfigForSchema{}
err := schema.ApplyDefaultsToConfig(config)
if err != nil {
t.Fatalf("ApplyDefaultsToConfig failed for empty schema: %v", err)
}
// All fields should remain at zero values since no defaults are defined
if config.Enabled != false {
t.Errorf("Expected Enabled=false (zero value), got %v", config.Enabled)
}
}
func TestApplyDefaults_MissingSchemaField(t *testing.T) {
// Schema with fewer fields than the struct
schema := &Schema{
Fields: []*Field{
{
Name: "enabled",
JSONName: "enabled",
Type: FieldTypeBool,
DefaultValue: true,
},
// Note: missing scan_interval_seconds and other fields
},
}
config := &TestTaskConfigForSchema{}
err := schema.ApplyDefaultsToConfig(config)
if err != nil {
t.Fatalf("ApplyDefaultsToConfig failed: %v", err)
}
// Only the field with a schema definition should get a default
if config.Enabled != true {
t.Errorf("Expected Enabled=true (has schema), got %v", config.Enabled)
}
// Fields without schema should remain at zero values
if config.ScanIntervalSeconds != 0 {
t.Errorf("Expected ScanIntervalSeconds=0 (no schema), got %v", config.ScanIntervalSeconds)
}
}
func BenchmarkApplyDefaults(b *testing.B) {
schema := createTestSchema()
config := &TestTaskConfigForSchema{}
b.ResetTimer()
for i := 0; i < b.N; i++ {
_ = schema.ApplyDefaultsToConfig(config)
}
}