Security and Production Hardening

Learning Objectives

• Implement TLS/SSL encryption for secure communication
• Build authentication systems with passwords and tokens
• Design authorization and access control mechanisms
• Implement rate limiting and DDoS protection
• Apply security hardening best practices
• Handle graceful shutdowns and rolling upgrades

Lesson 14.1: TLS/SSL Encryption

Generate TLS Certificates

# Generate private key
openssl genrsa -out server.key 2048

# Generate certificate signing request
openssl req -new -key server.key -out server.csr

# Self-signed certificate (for testing)
openssl x509 -req -days 365 -in server.csr -signkey server.key -out server.crt

# For production: use CA-signed certificate

TLS Server Implementation

import "crypto/tls"

// Create TLS listener
func (s *Server) StartTLS(certFile, keyFile string) error {
    cert, err := tls.LoadX509KeyPair(certFile, keyFile)
    if err != nil {
        return err
    }
    
    tlsConfig := &tls.Config{
        Certificates: []tls.Certificate{cert},
        MinVersion:   tls.VersionTLS13,
        CipherSuites: []uint16{
            tls.TLS_AES_256_GCM_SHA384,
            tls.TLS_CHACHA20_POLY1305_SHA256,
            tls.TLS_AES_128_GCM_SHA256,
        },
    }
    
    listener, err := tls.Listen("tcp", s.addr, tlsConfig)
    if err != nil {
        return err
    }
    
    return s.serve(listener)
}

// TLS client connection
func (c *Client) ConnectTLS(addr, certFile string) error {
    cert, err := tls.LoadX509KeyPair(certFile, "")
    if err != nil {
        return err
    }
    
    tlsConfig := &tls.Config{
        Certificates: []tls.Certificate{cert},
        InsecureSkipVerify: false, // Set to true for self-signed certs
    }
    
    conn, err := tls.Dial("tcp", addr, tlsConfig)
    if err != nil {
        return err
    }
    
    c.conn = conn
    return nil
}

Lesson 14.2: Authentication

Password-based Authentication

import "golang.org/x/crypto/bcrypt"

// User represents a database user
type User struct {
    Username     string
    PasswordHash []byte
    Permissions  []string
    CreatedAt    time.Time
    LastLogin    time.Time
}

// Authenticate user with password
func (s *Server) Authenticate(username, password string) (*User, error) {
    // Lookup user
    user, err := s.getUserByUsername(username)
    if err != nil {
        return nil, err
    }
    
    // Verify password hash
    if err := bcrypt.CompareHashAndPassword(user.PasswordHash, []byte(password)); err != nil {
        return nil, fmt.Errorf("invalid credentials")
    }
    
    // Update last login
    user.LastLogin = time.Now()
    s.updateUser(user)
    
    return user, nil
}

// Hash password for storage
func hashPassword(password string) ([]byte, error) {
    return bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost)
}

// CreateUser creates a new user
func (s *Server) CreateUser(username, password string, permissions []string) error {
    hash, err := hashPassword(password)
    if err != nil {
        return err
    }
    
    user := &User{
        Username:     username,
        PasswordHash: hash,
        Permissions:  permissions,
        CreatedAt:    time.Now(),
    }
    
    return s.saveUser(user)
}

Token-based Authentication

import "github.com/golang-jwt/jwt/v5"

// JWTClaims represents JWT token claims
type JWTClaims struct {
    Username   string   `json:"username"`
    Permissions []string `json:"permissions"`
    jwt.RegisteredClaims
}

// GenerateToken creates JWT token
func (s *Server) GenerateToken(user *User) (string, error) {
    claims := JWTClaims{
        Username:   user.Username,
        Permissions: user.Permissions,
        RegisteredClaims: jwt.RegisteredClaims{
            ExpiresAt: jwt.NewNumericDate(time.Now().Add(24 * time.Hour)),
            IssuedAt:  jwt.NewNumericDate(time.Now()),
            NotBefore: jwt.NewNumericDate(time.Now()),
        },
    }
    
    token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims)
    return token.SignedString([]byte(s.jwtSecret))
}

// ValidateToken checks JWT token
func (s *Server) ValidateToken(tokenString string) (*JWTClaims, error) {
    token, err := jwt.ParseWithClaims(tokenString, &JWTClaims{}, func(token *jwt.Token) (interface, error) {
        if _, ok := token.Method.(*jwt.SigningMethodHMAC); !ok {
            return nil, fmt.Errorf("unexpected signing method: %v", token.Header["alg"])
        }
        return []byte(s.jwtSecret), nil
    })
    
    if err != nil {
        return nil, err
    }
    
    if claims, ok := token.Claims.(*JWTClaims); ok && token.Valid {
        return claims, nil
    }
    
    return nil, fmt.Errorf("invalid token")
}

Lesson 14.3: Authorization

Role-Based Access Control (RBAC)

// Permission represents a specific permission
type Permission struct {
    Resource string // e.g., "database", "backup", "config"
    Action   string // e.g., "read", "write", "delete", "admin"
}

// Role represents a user role
type Role struct {
    Name        string
    Permissions []Permission
    Description string
}

// Predefined roles
var (
    AdminRole = Role{
        Name: "admin",
        Permissions: []Permission{
            {Resource: "database", Action: "read"},
            {Resource: "database", Action: "write"},
            {Resource: "database", Action: "delete"},
            {Resource: "backup", Action: "read"},
            {Resource: "backup", Action: "write"},
            {Resource: "config", Action: "read"},
            {Resource: "config", Action: "write"},
        },
        Description: "Full access to all resources",
    }
    
    ReadOnlyRole = Role{
        Name: "readonly",
        Permissions: []Permission{
            {Resource: "database", Action: "read"},
        },
        Description: "Read-only access to database",
    }
    
    BackupRole = Role{
        Name: "backup",
        Permissions: []Permission{
            {Resource: "database", Action: "read"},
            {Resource: "backup", Action: "read"},
            {Resource: "backup", Action: "write"},
        },
        Description: "Database read and backup access",
    }
)

// CheckPermission checks if user has permission
func (s *Server) CheckPermission(user *User, resource, action string) bool {
    for _, roleName := range user.Roles {
        role, exists := s.roles[roleName]
        if !exists {
            continue
        }
        
        for _, perm := range role.Permissions {
            if perm.Resource == resource && perm.Action == action {
                return true
            }
        }
    }
    
    return false
}

// AuthorizeRequest checks if request is authorized
func (s *Server) AuthorizeRequest(user *User, operation string) error {
    var resource, action string
    
    switch operation {
    case "GET", "SCAN":
        resource, action = "database", "read"
    case "PUT", "POST":
        resource, action = "database", "write"
    case "DELETE":
        resource, action = "database", "delete"
    case "BACKUP":
        resource, action = "backup", "write"
    case "RESTORE":
        resource, action = "backup", "read"
    case "CONFIG":
        resource, action = "config", "read"
    default:
        return fmt.Errorf("unknown operation: %s", operation)
    }
    
    if !s.CheckPermission(user, resource, action) {
        return fmt.Errorf("access denied: insufficient permissions for %s on %s", action, resource)
    }
    
    return nil
}

Lesson 14.4: Rate Limiting

Token Bucket Rate Limiter

import "golang.org/x/time/rate"

// RateLimiter controls request rate
type RateLimiter struct {
    limiters map[string]*rate.Limiter
    mu       sync.RWMutex
    burst    int
}

// NewRateLimiter creates a new rate limiter
func NewRateLimiter() *RateLimiter {
    return &RateLimiter{
        limiters: make(map[string]*rate.Limiter),
        burst:    10,
    }
}

// AllowRequest checks if request allowed
func (rl *RateLimiter) AllowRequest(clientIP string, rps float64) bool {
    rl.mu.RLock()
    limiter, exists := rl.limiters[clientIP]
    rl.mu.RUnlock()
    
    if !exists {
        limiter = rate.NewLimiter(rate.Limit(rps), rl.burst)
        rl.mu.Lock()
        rl.limiters[clientIP] = limiter
        rl.mu.Unlock()
    }
    
    return limiter.Allow()
}

// WaitForToken waits for token to become available
func (rl *RateLimiter) WaitForToken(clientIP string, rps float64) error {
    rl.mu.RLock()
    limiter, exists := rl.limiters[clientIP]
    rl.mu.RUnlock()
    
    if !exists {
        limiter = rate.NewLimiter(rate.Limit(rps), rl.burst)
        rl.mu.Lock()
        rl.limiters[clientIP] = limiter
        rl.mu.Unlock()
    }
    
    ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
    defer cancel()
    
    return limiter.Wait(ctx)
}

Sliding Window Rate Limiter

// SlidingWindowLimiter implements sliding window rate limiting
type SlidingWindowLimiter struct {
    windows map[string]*SlidingWindow
    mu      sync.RWMutex
    limit   int
    window  time.Duration
}

// SlidingWindow represents a sliding window for a client
type SlidingWindow struct {
    requests []time.Time
    limit    int
    window   time.Duration
    mu       sync.Mutex
}

// NewSlidingWindowLimiter creates a new sliding window limiter
func NewSlidingWindowLimiter(limit int, window time.Duration) *SlidingWindowLimiter {
    return &SlidingWindowLimiter{
        windows: make(map[string]*SlidingWindow),
        limit:   limit,
        window:  window,
    }
}

// AllowRequest checks if request allowed in sliding window
func (swl *SlidingWindowLimiter) AllowRequest(clientIP string) bool {
    swl.mu.RLock()
    window, exists := swl.windows[clientIP]
    swl.mu.RUnlock()
    
    if !exists {
        window = &SlidingWindow{
            limit:  swl.limit,
            window: swl.window,
        }
        swl.mu.Lock()
        swl.windows[clientIP] = window
        swl.mu.Unlock()
    }
    
    return window.AllowRequest()
}

// AllowRequest checks if request allowed in this window
func (sw *SlidingWindow) AllowRequest() bool {
    sw.mu.Lock()
    defer sw.mu.Unlock()
    
    now := time.Now()
    cutoff := now.Add(-sw.window)
    
    // Remove old requests
    var validRequests []time.Time
    for _, reqTime := range sw.requests {
        if reqTime.After(cutoff) {
            validRequests = append(validRequests, reqTime)
        }
    }
    sw.requests = validRequests
    
    // Check if under limit
    if len(sw.requests) < sw.limit {
        sw.requests = append(sw.requests, now)
        return true
    }
    
    return false
}

Lesson 14.5: Graceful Shutdown and Upgrades

Graceful Shutdown

// GracefulShutdown waits for connections to finish
func (s *Server) GracefulShutdown(timeout time.Duration) error {
    s.logger.Info("starting graceful shutdown")
    
    // Stop accepting new connections
    if err := s.listener.Close(); err != nil {
        s.logger.Error("error closing listener", zap.Error(err))
    }
    
    // Wait for active connections to finish
    deadline := time.Now().Add(timeout)
    
    for time.Now().Before(deadline) {
        activeCount := s.getActiveConnectionCount()
        if activeCount == 0 {
            s.logger.Info("all connections closed")
            return nil
        }
        
        s.logger.Info("waiting for connections to close", 
            zap.Int("active", activeCount),
            zap.Duration("remaining", deadline.Sub(time.Now())),
        )
        
        time.Sleep(100 * time.Millisecond)
    }
    
    return fmt.Errorf("timeout waiting for connections to close")
}

// ShutdownWithSignal handles shutdown signals
func (s *Server) ShutdownWithSignal() {
    c := make(chan os.Signal, 1)
    signal.Notify(c, os.Interrupt, syscall.SIGTERM)
    
    <-c
    s.logger.Info("received shutdown signal")
    
    if err := s.GracefulShutdown(30 * time.Second); err != nil {
        s.logger.Error("graceful shutdown failed", zap.Error(err))
        os.Exit(1)
    }
    
    s.logger.Info("server shutdown complete")
    os.Exit(0)
}

Rolling Upgrades

// RollingUpgrade upgrades nodes one at a time
func (cluster *Cluster) RollingUpgrade(newImage string) error {
    cluster.logger.Info("starting rolling upgrade", zap.String("image", newImage))
    
    for i, node := range cluster.nodes {
        cluster.logger.Info("upgrading node", 
            zap.Int("index", i),
            zap.String("node", node.Address()),
        )
        
        // Remove node from load balancer
        if err := cluster.deregisterNode(node); err != nil {
            return fmt.Errorf("failed to deregister node %s: %w", node.Address(), err)
        }
        
        // Wait for in-flight requests to complete
        cluster.logger.Info("waiting for in-flight requests", zap.String("node", node.Address()))
        time.Sleep(10 * time.Second)
        
        // Perform upgrade
        if err := node.Upgrade(newImage); err != nil {
            // Try to re-register node on failure
            cluster.registerNode(node)
            return fmt.Errorf("failed to upgrade node %s: %w", node.Address(), err)
        }
        
        // Wait for health check
        cluster.logger.Info("waiting for node to become healthy", zap.String("node", node.Address()))
        if err := node.WaitHealthy(30 * time.Second); err != nil {
            // Try to re-register node on failure
            cluster.registerNode(node)
            return fmt.Errorf("node %s failed health check: %w", node.Address(), err)
        }
        
        // Add back to load balancer
        if err := cluster.registerNode(node); err != nil {
            return fmt.Errorf("failed to re-register node %s: %w", node.Address(), err)
        }
        
        cluster.logger.Info("node upgrade complete", 
            zap.Int("index", i),
            zap.String("node", node.Address()),
        )
    }
    
    cluster.logger.Info("rolling upgrade complete")
    return nil
}

// CanaryUpgrade performs canary deployment
func (cluster *Cluster) CanaryUpgrade(newImage string, percentage int) error {
    if percentage < 1 || percentage > 100 {
        return fmt.Errorf("invalid percentage: %d", percentage)
    }
    
    canaryCount := len(cluster.nodes) * percentage / 100
    if canaryCount == 0 {
        canaryCount = 1
    }
    
    cluster.logger.Info("starting canary upgrade", 
        zap.String("image", newImage),
        zap.Int("canary_count", canaryCount),
    )
    
    // Upgrade canary nodes
    for i := 0; i < canaryCount; i++ {
        node := cluster.nodes[i]
        if err := cluster.upgradeNode(node, newImage); err != nil {
            return fmt.Errorf("canary upgrade failed for node %s: %w", node.Address(), err)
        }
    }
    
    // Monitor canary nodes
    cluster.logger.Info("monitoring canary nodes", zap.Duration("duration", 5*time.Minute))
    time.Sleep(5 * time.Minute)
    
    // Check canary health
    for i := 0; i < canaryCount; i++ {
        node := cluster.nodes[i]
        if !node.IsHealthy() {
            return fmt.Errorf("canary node %s is unhealthy", node.Address())
        }
    }
    
    cluster.logger.Info("canary upgrade successful, proceeding with full rollout")
    return cluster.RollingUpgrade(newImage)
}

Lab 14.1: Security Implementation

Objective

Build a comprehensive security system with TLS encryption, authentication, authorization, rate limiting, and security hardening.

Requirements

• TLS/SSL Encryption: End-to-end encrypted communication
• Authentication: Password and JWT token authentication
• Authorization: Role-based access control (RBAC)
• Rate Limiting: Token bucket and sliding window limiters
• Security Hardening: Input validation, secure defaults
• Audit Logging: Security event logging and monitoring

Starter Code

// TODO: Implement TLS server
func (s *Server) StartTLS(certFile, keyFile string) error {
    // Load TLS certificates
    // Configure TLS settings
    // Start TLS listener
    return nil
}

// TODO: Implement authentication
func (s *Server) Authenticate(username, password string) (*User, error) {
    // Validate credentials
    // Return user if valid
    return nil, nil
}

// TODO: Implement authorization
func (s *Server) AuthorizeRequest(user *User, operation string) error {
    // Check user permissions
    // Return error if not authorized
    return nil
}

// TODO: Implement rate limiting
func (s *Server) CheckRateLimit(clientIP string) bool {
    // Check if request is within rate limit
    // Return true if allowed
    return true
}

// TODO: Implement security middleware
func (s *Server) SecurityMiddleware(next http.Handler) http.Handler {
    return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
        // Rate limiting
        // Authentication
        // Authorization
        // Input validation
        next.ServeHTTP(w, r)
    })
}

Test Template

func TestTLSConnection(t *testing.T) {
    server := NewServer()
    err := server.StartTLS("server.crt", "server.key")
    assert.NoError(t, err)
    
    // Test TLS client connection
    client := NewClient()
    err = client.ConnectTLS("localhost:6379", "server.crt")
    assert.NoError(t, err)
}

func TestAuthentication(t *testing.T) {
    server := NewServer()
    
    // Create test user
    err := server.CreateUser("testuser", "password123", []string{"read"})
    assert.NoError(t, err)
    
    // Test valid authentication
    user, err := server.Authenticate("testuser", "password123")
    assert.NoError(t, err)
    assert.Equal(t, "testuser", user.Username)
    
    // Test invalid authentication
    _, err = server.Authenticate("testuser", "wrongpassword")
    assert.Error(t, err)
}

func TestAuthorization(t *testing.T) {
    server := NewServer()
    
    // Create user with read-only permissions
    user := &User{Username: "readonly", Roles: []string{"readonly"}}
    
    // Test authorized operation
    err := server.AuthorizeRequest(user, "GET")
    assert.NoError(t, err)
    
    // Test unauthorized operation
    err = server.AuthorizeRequest(user, "PUT")
    assert.Error(t, err)
}

func TestRateLimiting(t *testing.T) {
    limiter := NewRateLimiter()
    
    // Test within rate limit
    for i := 0; i < 5; i++ {
        allowed := limiter.AllowRequest("127.0.0.1", 10.0)
        assert.True(t, allowed)
    }
    
    // Test rate limit exceeded
    time.Sleep(100 * time.Millisecond)
    allowed := limiter.AllowRequest("127.0.0.1", 0.1) // Very low rate
    assert.False(t, allowed)
}

Acceptance Criteria

✅ TLS encryption works for all connections
✅ Password authentication with bcrypt hashing
✅ JWT token authentication and validation
✅ Role-based access control (RBAC)
✅ Rate limiting prevents abuse
✅ Input validation prevents injection attacks
✅ Security audit logging implemented
✅ Graceful shutdown handles connections
✅ > 90% code coverage
✅ All tests pass

Summary: Week 14 Complete

By completing Week 14, you've learned and implemented:

1. TLS/SSL Encryption

• Certificate generation and management
• TLS server and client implementation
• Secure cipher suite configuration
• Certificate validation

2. Authentication

• Password-based authentication
• JWT token authentication
• Password hashing with bcrypt
• User session management

3. Authorization

• Role-based access control (RBAC)
• Permission-based authorization
• Resource and action mapping
• Access control enforcement

4. Rate Limiting

• Token bucket rate limiting
• Sliding window rate limiting
• Per-client rate limiting
• DDoS protection

Key Skills Mastered:

✅ Implement end-to-end encryption with TLS
✅ Build secure authentication systems
✅ Design authorization and access control
✅ Implement rate limiting and DDoS protection
✅ Apply security hardening best practices
✅ Handle graceful shutdowns and upgrades

Ready for the Capstone Project?

Next week we'll build a complete production-ready key-value database from scratch, incorporating all the concepts we've learned throughout the course.

Continue to Capstone Project →

Course Navigation

All Lessons