1
0
Fork 0
mirror of https://github.com/chrislusf/seaweedfs synced 2025-09-20 02:00:23 +02:00
seaweedfs/test/kafka/scripts/wait-for-services.sh
chrislu 00a672d12e Add comprehensive Docker Compose setup for Kafka integration tests
MAJOR ENHANCEMENT: Complete Docker-based integration testing infrastructure

## New Docker Compose Infrastructure:
- docker-compose.yml: Complete multi-service setup with health checks
  - Apache Kafka + Zookeeper
  - Confluent Schema Registry
  - SeaweedFS full stack (Master, Volume, Filer, MQ Broker, MQ Agent)
  - Kafka Gateway service
  - Test setup and utility services

## Docker Services:
- Dockerfile.kafka-gateway: Custom Kafka Gateway container
- Dockerfile.test-setup: Schema registration and test data setup
- kafka-gateway-start.sh: Service startup script with dependency waiting
- wait-for-services.sh: Comprehensive service readiness verification

## Test Setup Utility:
- cmd/setup/main.go: Automated schema registration utility
- Registers User, UserEvent, and LogEntry Avro schemas
- Handles service discovery and health checking

## Integration Tests:
- docker_integration_test.go: Comprehensive Docker-based integration tests
  - Kafka connectivity and topic operations
  - Schema Registry integration
  - Kafka Gateway functionality
  - Sarama and kafka-go client compatibility
  - Cross-client message compatibility
  - Performance benchmarking

## Build and Test Infrastructure:
- Makefile: 30+ targets for development and testing
  - setup, test-unit, test-integration, test-e2e
  - Performance testing and benchmarking
  - Individual service management
  - Debugging and monitoring tools
  - CI/CD integration targets

## Documentation:
- README.md: Comprehensive documentation
  - Architecture overview and service descriptions
  - Quick start guide and development workflow
  - Troubleshooting and performance tuning
  - CI/CD integration examples

## Key Features:
 Complete service orchestration with health checks
 Automated schema registration and test data setup
 Multi-client compatibility testing (Sarama, kafka-go)
 Performance benchmarking and monitoring
 Development-friendly debugging tools
 CI/CD ready with proper cleanup
 Comprehensive documentation and examples

## Usage:
make setup-schemas  # Start all services and register schemas
make test-e2e      # Run end-to-end integration tests
make clean         # Clean up environment

This provides a production-ready testing infrastructure that ensures
Kafka Gateway compatibility with real Kafka ecosystems and validates
schema registry integration in realistic deployment scenarios.
2025-09-12 08:46:03 -07:00

135 lines
4.2 KiB
Bash
Executable file

#!/bin/bash
# Wait for Services Script for Kafka Integration Tests
set -e
echo "Waiting for services to be ready..."
# Configuration
KAFKA_HOST=${KAFKA_HOST:-localhost}
KAFKA_PORT=${KAFKA_PORT:-9092}
SCHEMA_REGISTRY_URL=${SCHEMA_REGISTRY_URL:-http://localhost:8081}
KAFKA_GATEWAY_HOST=${KAFKA_GATEWAY_HOST:-localhost}
KAFKA_GATEWAY_PORT=${KAFKA_GATEWAY_PORT:-9093}
SEAWEEDFS_MASTER_URL=${SEAWEEDFS_MASTER_URL:-http://localhost:9333}
MAX_WAIT=${MAX_WAIT:-300} # 5 minutes
# Colors
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color
# Helper function to wait for a service
wait_for_service() {
local service_name=$1
local check_command=$2
local timeout=${3:-60}
echo -e "${BLUE}Waiting for ${service_name}...${NC}"
local count=0
while [ $count -lt $timeout ]; do
if eval "$check_command" > /dev/null 2>&1; then
echo -e "${GREEN}${service_name} is ready${NC}"
return 0
fi
if [ $((count % 10)) -eq 0 ]; then
echo -e "${YELLOW}Still waiting for ${service_name}... (${count}s)${NC}"
fi
sleep 1
count=$((count + 1))
done
echo -e "${RED}${service_name} failed to start within ${timeout} seconds${NC}"
return 1
}
# Wait for Zookeeper
echo "=== Checking Zookeeper ==="
wait_for_service "Zookeeper" "nc -z localhost 2181" 30
# Wait for Kafka
echo "=== Checking Kafka ==="
wait_for_service "Kafka" "nc -z ${KAFKA_HOST} ${KAFKA_PORT}" 60
# Test Kafka broker API
echo "=== Testing Kafka API ==="
wait_for_service "Kafka API" "timeout 5 kafka-broker-api-versions --bootstrap-server ${KAFKA_HOST}:${KAFKA_PORT}" 30
# Wait for Schema Registry
echo "=== Checking Schema Registry ==="
wait_for_service "Schema Registry" "curl -f ${SCHEMA_REGISTRY_URL}/subjects" 60
# Wait for SeaweedFS Master
echo "=== Checking SeaweedFS Master ==="
wait_for_service "SeaweedFS Master" "curl -f ${SEAWEEDFS_MASTER_URL}/cluster/status" 30
# Wait for SeaweedFS Volume
echo "=== Checking SeaweedFS Volume ==="
wait_for_service "SeaweedFS Volume" "curl -f http://localhost:8080/status" 30
# Wait for SeaweedFS Filer
echo "=== Checking SeaweedFS Filer ==="
wait_for_service "SeaweedFS Filer" "curl -f http://localhost:8888/" 30
# Wait for SeaweedFS MQ Broker
echo "=== Checking SeaweedFS MQ Broker ==="
wait_for_service "SeaweedFS MQ Broker" "nc -z localhost 17777" 30
# Wait for SeaweedFS MQ Agent
echo "=== Checking SeaweedFS MQ Agent ==="
wait_for_service "SeaweedFS MQ Agent" "nc -z localhost 16777" 30
# Wait for Kafka Gateway
echo "=== Checking Kafka Gateway ==="
wait_for_service "Kafka Gateway" "nc -z ${KAFKA_GATEWAY_HOST} ${KAFKA_GATEWAY_PORT}" 60
# Final verification
echo "=== Final Verification ==="
# Test Kafka topic creation
echo "Testing Kafka topic operations..."
TEST_TOPIC="health-check-$(date +%s)"
if kafka-topics --create --topic "$TEST_TOPIC" --bootstrap-server "${KAFKA_HOST}:${KAFKA_PORT}" --partitions 1 --replication-factor 1 > /dev/null 2>&1; then
echo -e "${GREEN}✅ Kafka topic creation works${NC}"
kafka-topics --delete --topic "$TEST_TOPIC" --bootstrap-server "${KAFKA_HOST}:${KAFKA_PORT}" > /dev/null 2>&1 || true
else
echo -e "${RED}❌ Kafka topic creation failed${NC}"
exit 1
fi
# Test Schema Registry
echo "Testing Schema Registry..."
if curl -f "${SCHEMA_REGISTRY_URL}/subjects" > /dev/null 2>&1; then
echo -e "${GREEN}✅ Schema Registry is accessible${NC}"
else
echo -e "${RED}❌ Schema Registry is not accessible${NC}"
exit 1
fi
# Test Kafka Gateway connectivity
echo "Testing Kafka Gateway..."
if nc -z "${KAFKA_GATEWAY_HOST}" "${KAFKA_GATEWAY_PORT}"; then
echo -e "${GREEN}✅ Kafka Gateway is accessible${NC}"
else
echo -e "${RED}❌ Kafka Gateway is not accessible${NC}"
exit 1
fi
echo -e "${GREEN}🎉 All services are ready!${NC}"
echo ""
echo "Service endpoints:"
echo " Kafka: ${KAFKA_HOST}:${KAFKA_PORT}"
echo " Schema Registry: ${SCHEMA_REGISTRY_URL}"
echo " Kafka Gateway: ${KAFKA_GATEWAY_HOST}:${KAFKA_GATEWAY_PORT}"
echo " SeaweedFS Master: ${SEAWEEDFS_MASTER_URL}"
echo " SeaweedFS Filer: http://localhost:8888"
echo " SeaweedFS MQ Broker: localhost:17777"
echo " SeaweedFS MQ Agent: localhost:16777"
echo ""
echo "Ready to run integration tests!"