Configuration and Deployment

# config.yaml
server:
  host: "0.0.0.0"
  port: 6379
  max_connections: 10000

database:
  memtable_size: 67108864  # 64MB
  shard_count: 16
  wal_sync_interval: 100

replication:
  mode: "semi-sync"
  min_replicas: 1
  timeout_ms: 5000

cluster:
  nodes:
    - "node1:6379"
    - "node2:6379"
    - "node3:6379"
  
monitoring:
  metrics_port: 9090
  health_port: 8080

# config.toml
[server]
host = "0.0.0.0"
port = 6379
max_connections = 10000

[database]
memtable_size = 67108864
shard_count = 16
wal_sync_interval = 100

[replication]
mode = "semi-sync"
min_replicas = 1
timeout_ms = 5000

[[cluster.nodes]]
address = "node1:6379"

[[cluster.nodes]]
address = "node2:6379"

package config

import (
    "fmt"
    "os"
    "github.com/spf13/viper"
)

type Config struct {
    Server struct {
        Host           string
        Port           int
        MaxConnections int
    }
    
    Database struct {
        MemtableSize   int64
        ShardCount     int
        WalSyncInterval int
    }
    
    Replication struct {
        Mode        string
        MinReplicas int
        TimeoutMs   int
    }
    
    Cluster struct {
        Nodes []string
    }
    
    Monitoring struct {
        MetricsPort int
        HealthPort  int
    }
}

// LoadConfig loads configuration from file
func LoadConfig(filepath string) (*Config, error) {
    viper.SetConfigFile(filepath)
    viper.SetConfigType("yaml")
    
    // Read config file
    if err := viper.ReadInConfig(); err != nil {
        return nil, fmt.Errorf("error reading config: %w", err)
    }
    
    var cfg Config
    if err := viper.Unmarshal(&cfg); err != nil {
        return nil, fmt.Errorf("error unmarshaling config: %w", err)
    }
    
    return &cfg, nil
}

// LoadConfigWithDefaults loads with defaults
func LoadConfigWithDefaults() *Config {
    viper.SetDefault("server.host", "0.0.0.0")
    viper.SetDefault("server.port", 6379)
    viper.SetDefault("server.max_connections", 10000)
    viper.SetDefault("database.memtable_size", 67108864)
    viper.SetDefault("database.shard_count", 16)
    
    cfg, _ := LoadConfig("config.yaml")
    return cfg
}

// Override config with environment variables
func LoadConfigFromEnv() *Config {
    viper.AutomaticEnv()
    viper.SetEnvPrefix("KVDB")
    
    // Usage: KVDB_SERVER_PORT=7000
    
    cfg, _ := LoadConfig("config.yaml")
    return cfg
}

// Example environment variables:
// KVDB_SERVER_HOST=0.0.0.0
// KVDB_SERVER_PORT=6379
// KVDB_DATABASE_SHARD_COUNT=32
// KVDB_REPLICATION_MODE=sync

// ValidateConfig checks configuration for errors
func (c *Config) Validate() error {
    if c.Server.Port <= 0 || c.Server.Port > 65535 {
        return fmt.Errorf("invalid port: %d", c.Server.Port)
    }
    
    if c.Database.MemtableSize <= 0 {
        return fmt.Errorf("invalid memtable size: %d", c.Database.MemtableSize)
    }
    
    if c.Database.ShardCount <= 0 || c.Database.ShardCount > 256 {
        return fmt.Errorf("invalid shard count: %d", c.Database.ShardCount)
    }
    
    if c.Replication.Mode != "async" && c.Replication.Mode != "sync" && c.Replication.Mode != "semi-sync" {
        return fmt.Errorf("invalid replication mode: %s", c.Replication.Mode)
    }
    
    if len(c.Cluster.Nodes) == 0 {
        return fmt.Errorf("no cluster nodes configured")
    }
    
    return nil
}

# Multi-stage build for small image
FROM golang:1.21-alpine AS builder

WORKDIR /build

# Copy go mod files
COPY go.mod go.sum ./

# Download dependencies
RUN go mod download

# Copy source code
COPY . .

# Build binary
RUN CGO_ENABLED=0 GOOS=linux go build -o kvdb ./cmd/server

# Final stage
FROM alpine:3.18

# Install runtime dependencies
RUN apk add --no-cache ca-certificates tzdata

# Create app user
RUN addgroup -g 1000 app && adduser -D -u 1000 -G app app

WORKDIR /app

# Copy binary from builder
COPY --from=builder /build/kvdb .

# Copy config
COPY config.yaml .

# Change ownership
RUN chown -R app:app /app

# Switch to non-root user
USER app

# Expose ports
EXPOSE 6379 9090 8080

# Health check
HEALTHCHECK --interval=30s --timeout=3s --start-period=10s --retries=3 \
    CMD wget --quiet --tries=1 --spider http://localhost:8080/healthz || exit 1

# Run server
CMD ["./kvdb", "-config", "config.yaml"]

# docker-compose.yml
version: '3.8'

services:
  kvdb-1:
    build: .
    container_name: kvdb-1
    ports:
      - "6379:6379"
      - "9090:9090"
      - "8080:8080"
    environment:
      - KVDB_SERVER_HOST=0.0.0.0
      - KVDB_SERVER_PORT=6379
      - KVDB_CLUSTER_NODES=kvdb-1:6379,kvdb-2:6379,kvdb-3:6379
    volumes:
      - kvdb-1-data:/data
      - ./config.yaml:/app/config.yaml
    networks:
      - kvdb-network
    healthcheck:
      test: ["CMD", "wget", "--quiet", "--tries=1", "--spider", "http://localhost:8080/healthz"]
      interval: 10s
      timeout: 3s
      retries: 3
      start_period: 10s

  kvdb-2:
    build: .
    container_name: kvdb-2
    ports:
      - "6380:6379"
      - "9091:9090"
      - "8081:8080"
    environment:
      - KVDB_SERVER_HOST=0.0.0.0
      - KVDB_CLUSTER_NODES=kvdb-1:6379,kvdb-2:6379,kvdb-3:6379
    volumes:
      - kvdb-2-data:/data
      - ./config.yaml:/app/config.yaml
    networks:
      - kvdb-network
    depends_on:
      - kvdb-1

  kvdb-3:
    build: .
    container_name: kvdb-3
    ports:
      - "6381:6379"
      - "9092:9090"
      - "8082:8080"
    environment:
      - KVDB_SERVER_HOST=0.0.0.0
      - KVDB_CLUSTER_NODES=kvdb-1:6379,kvdb-2:6379,kvdb-3:6379
    volumes:
      - kvdb-3-data:/data
      - ./config.yaml:/app/config.yaml
    networks:
      - kvdb-network
    depends_on:
      - kvdb-1

volumes:
  kvdb-1-data:
  kvdb-2-data:
  kvdb-3-data:

networks:
  kvdb-network:
    driver: bridge

# Build Docker image
docker build -t kvdb:latest .

# Run container
docker run -p 6379:6379 -p 9090:9090 -p 8080:8080 kvdb:latest

# Run with custom config
docker run -p 6379:6379 \
  -v $(pwd)/config.yaml:/app/config.yaml \
  kvdb:latest ./kvdb -config /app/config.yaml

# Local cluster with Docker Compose
docker-compose up -d

# View logs
docker-compose logs -f kvdb-1

# Stop cluster
docker-compose down

# deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: kvdb
  namespace: default
spec:
  replicas: 3
  selector:
    matchLabels:
      app: kvdb
  template:
    metadata:
      labels:
        app: kvdb
    spec:
      containers:
      - name: kvdb
        image: kvdb:latest
        imagePullPolicy: IfNotPresent
        ports:
        - containerPort: 6379
          name: server
        - containerPort: 9090
          name: metrics
        - containerPort: 8080
          name: health
        
        env:
        - name: KVDB_SERVER_HOST
          value: "0.0.0.0"
        - name: KVDB_SERVER_PORT
          value: "6379"
        - name: KVDB_POD_NAME
          valueFrom:
            fieldRef:
              fieldPath: metadata.name
        - name: KVDB_POD_NAMESPACE
          valueFrom:
            fieldRef:
              fieldPath: metadata.namespace
        
        livenessProbe:
          httpGet:
            path: /healthz
            port: health
          initialDelaySeconds: 10
          periodSeconds: 10
          timeoutSeconds: 3
          failureThreshold: 3
        
        readinessProbe:
          httpGet:
            path: /readyz
            port: health
          initialDelaySeconds: 5
          periodSeconds: 5
          timeoutSeconds: 3
          failureThreshold: 2
        
        resources:
          requests:
            cpu: 500m
            memory: 512Mi
          limits:
            cpu: 2000m
            memory: 2Gi
        
        volumeMounts:
        - name: data
          mountPath: /data
        - name: config
          mountPath: /app/config.yaml
          subPath: config.yaml
      
      volumes:
      - name: data
        emptyDir: 
      - name: config
        configMap:
          name: kvdb-config

# service.yaml
apiVersion: v1
kind: Service
metadata:
  name: kvdb
  namespace: default
spec:
  type: ClusterIP
  ports:
  - port: 6379
    targetPort: 6379
    protocol: TCP
    name: server
  - port: 9090
    targetPort: 9090
    protocol: TCP
    name: metrics
  selector:
    app: kvdb
---
apiVersion: v1
kind: Service
metadata:
  name: kvdb-headless
  namespace: default
spec:
  type: ClusterIP
  clusterIP: None
  ports:
  - port: 6379
    targetPort: 6379
  selector:
    app: kvdb

# configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: kvdb-config
data:
  config.yaml: |
    server:
      host: "0.0.0.0"
      port: 6379
      max_connections: 10000
    
    database:
      memtable_size: 67108864
      shard_count: 16
      wal_sync_interval: 100
    
    replication:
      mode: "semi-sync"
      min_replicas: 1
      timeout_ms: 5000
    
    cluster:
      nodes:
        - "kvdb-0:6379"
        - "kvdb-1:6379"
        - "kvdb-2:6379"

# Create namespace
kubectl create namespace kvdb

# Apply ConfigMap
kubectl apply -f configmap.yaml

# Apply Deployment
kubectl apply -f deployment.yaml

# Apply Service
kubectl apply -f service.yaml

# Check deployment status
kubectl get deployment kvdb

# View pods
kubectl get pods -l app=kvdb

# View logs
kubectl logs deployment/kvdb -f

# Port forward for testing
kubectl port-forward svc/kvdb 6379:6379

# Scale replicas
kubectl scale deployment kvdb --replicas=5

# Delete deployment
kubectl delete deployment kvdb

// BackupManager handles database backups
type BackupManager struct {
    backupDir  string
    store      Store
    logger     *zap.Logger
}

// CreateBackup creates point-in-time backup
func (bm *BackupManager) CreateBackup(name string) error {
    backupPath := filepath.Join(bm.backupDir, name)
    
    if err := os.MkdirAll(backupPath, 0755); err != nil {
        return err
    }
    
    // Get snapshot from store
    snapshot, err := bm.store.GetSnapshot()
    if err != nil {
        return err
    }
    defer snapshot.Release()
    
    // Write snapshot to backup directory
    iter := snapshot.Iterator()
    defer iter.Close()
    
    backupFile := filepath.Join(backupPath, "data.backup")
    f, err := os.Create(backupFile)
    if err != nil {
        return err
    }
    defer f.Close()
    
    encoder := json.NewEncoder(f)
    
    for iter.Next() {
        entry := map[string]interface{
            "key":   string(iter.Key()),
            "value": string(iter.Value()),
        }
        encoder.Encode(entry)
    }
    
    bm.logger.Info("backup created", zap.String("path", backupPath))
    return nil
}

// RestoreBackup restores database from backup
func (bm *BackupManager) RestoreBackup(name string) error {
    backupPath := filepath.Join(bm.backupDir, name)
    backupFile := filepath.Join(backupPath, "data.backup")
    
    f, err := os.Open(backupFile)
    if err != nil {
        return err
    }
    defer f.Close()
    
    decoder := json.NewDecoder(f)
    
    for {
        var entry map[string]interface
        if err := decoder.Decode(&entry); err != nil {
            if err == io.EOF {
                break
            }
            return err
        }
        
        key := []byte(entry["key"].(string))
        value := []byte(entry["value"].(string))
        
        bm.store.Put(context.Background(), key, value)
    }
    
    bm.logger.Info("backup restored", zap.String("path", backupPath))
    return nil
}

// ListBackups lists all available backups
func (bm *BackupManager) ListBackups() ([]string, error) {
    entries, err := os.ReadDir(bm.backupDir)
    if err != nil {
        return nil, err
    }
    
    var backups []string
    for _, entry := range entries {
        if entry.IsDir() {
            backups = append(backups, entry.Name())
        }
    }
    
    return backups, nil
}

// DeleteBackup deletes a backup
func (bm *BackupManager) DeleteBackup(name string) error {
    backupPath := filepath.Join(bm.backupDir, name)
    return os.RemoveAll(backupPath)
}

// ScheduledBackupManager handles periodic backups
type ScheduledBackupManager struct {
    backupMgr   *BackupManager
    interval    time.Duration
    maxBackups  int
    ticker      *time.Ticker
    done        chan struct
}

// Start begins scheduled backups
func (sbm *ScheduledBackupManager) Start() {
    sbm.ticker = time.NewTicker(sbm.interval)
    
    go func() {
        for {
            select {
            case <-sbm.ticker.C:
                // Create backup with timestamp
                name := fmt.Sprintf("backup-%d", time.Now().Unix())
                if err := sbm.backupMgr.CreateBackup(name); err != nil {
                    log.Printf("backup failed: %v", err)
                    continue
                }
                
                // Clean old backups
                sbm.cleanOldBackups()
                
            case <-sbm.done:
                return
            }
        }
    }()
}

// cleanOldBackups removes old backups
func (sbm *ScheduledBackupManager) cleanOldBackups() {
    backups, _ := sbm.backupMgr.ListBackups()
    
    if len(backups) > sbm.maxBackups {
        // Sort by name (timestamp)
        sort.Strings(backups)
        
        // Delete oldest
        for i := 0; i < len(backups)-sbm.maxBackups; i++ {
            sbm.backupMgr.DeleteBackup(backups[i])
        }
    }
}

// Stop stops scheduled backups
func (sbm *ScheduledBackupManager) Stop() {
    sbm.ticker.Stop()
    close(sbm.done)
}

// TODO: Implement configuration management
func LoadConfig(filepath string) (*Config, error) {
    // Load YAML/TOML configuration
    // Validate configuration
    // Return parsed config
}

// TODO: Implement Docker health checks
func (s *Server) HealthCheck() bool {
    // Check database connectivity
    // Check replication status
    // Check resource usage
    return true
}

// TODO: Implement backup system
func (s *Server) CreateBackup() error {
    // Create point-in-time backup
    // Compress backup data
    // Store backup metadata
    return nil
}

// TODO: Implement graceful shutdown
func (s *Server) GracefulShutdown(timeout time.Duration) error {
    // Stop accepting new connections
    // Wait for active connections to finish
    // Clean up resources
    return nil
}

func TestConfiguration(t *testing.T) {
    cfg, err := LoadConfig("test-config.yaml")
    assert.NoError(t, err)
    assert.NotNil(t, cfg)
    
    // Test configuration validation
    err = cfg.Validate()
    assert.NoError(t, err)
    
    // Test environment variable override
    os.Setenv("KVDB_SERVER_PORT", "7000")
    cfg, _ = LoadConfigFromEnv()
    assert.Equal(t, 7000, cfg.Server.Port)
}

func TestDockerHealthCheck(t *testing.T) {
    server := NewServer()
    
    // Test health check
    healthy := server.HealthCheck()
    assert.True(t, healthy)
}

func TestBackupRestore(t *testing.T) {
    store := NewStore()
    backupMgr := NewBackupManager(store)
    
    // Create test data
    store.Put([]byte("key1"), []byte("value1"))
    store.Put([]byte("key2"), []byte("value2"))
    
    // Create backup
    err := backupMgr.CreateBackup("test-backup")
    assert.NoError(t, err)
    
    // Clear store
    store.Clear()
    
    // Restore backup
    err = backupMgr.RestoreBackup("test-backup")
    assert.NoError(t, err)
    
    // Verify data restored
    value, _ := store.Get([]byte("key1"))
    assert.Equal(t, "value1", string(value))
}

Key Skills Mastered:

• ✅ Manage configuration with multiple formats
• ✅ Containerize applications with Docker
• ✅ Deploy to Kubernetes clusters
• ✅ Implement backup and recovery strategies
• ✅ Design production deployment pipelines
• ✅ Handle graceful shutdowns and upgrades