Debugging

This guide covers techniques for debugging TinySystems components and flows, from development to production issues.

Development Debugging

Local Development

Run your module locally with verbose logging:

# Enable debug logging
export LOG_LEVEL=debug

# Run module
go run ./cmd/main.go

IDE Debugging

VS Code launch.json:

{
    "version": "0.2.0",
    "configurations": [
        {
            "name": "Debug Module",
            "type": "go",
            "request": "launch",
            "mode": "auto",
            "program": "${workspaceFolder}/cmd/main.go",
            "env": {
                "LOG_LEVEL": "debug",
                "KUBECONFIG": "${env:HOME}/.kube/config"
            }
        }
    ]
}

GoLand:

1. Run → Edit Configurations
2. Add Go Build
3. Set environment variables
4. Add breakpoints

Print Debugging

Quick debugging with structured logging:

func (c *Component) Handle(ctx context.Context, output module.Handler, port string, msg any) error {
    log := log.FromContext(ctx)

    log.Info(">>> HANDLE CALLED",
        "port", port,
        "msgType", fmt.Sprintf("%T", msg),
        "msg", fmt.Sprintf("%+v", msg),
    )

    result, err := c.process(msg)

    log.Info("<<< HANDLE RESULT",
        "result", fmt.Sprintf("%+v", result),
        "error", err,
    )

    return err
}

Component Debugging

Debug Port

Add a debug port for introspection:

type DebugInfo struct {
    Settings   any            `json:"settings"`
    State      string         `json:"state"`
    IsLeader   bool           `json:"isLeader"`
    NodeName   string         `json:"nodeName"`
    Metadata   map[string]any `json:"metadata"`
}

func (c *Component) Ports() []module.Port {
    return []module.Port{
        // ... other ports
        {
            Name:     "debug",
            Label:    "Debug",
            Source:   true,
            Position: module.PositionTop,
            Schema:   schema.FromGo(struct{}{}),
        },
        {
            Name:     "debug_out",
            Label:    "Debug Info",
            Source:   false,
            Position: module.PositionBottom,
            Schema:   schema.FromGo(DebugInfo{}),
        },
    }
}

func (c *Component) Handle(ctx context.Context, output module.Handler, port string, msg any) error {
    if port == "debug" {
        return output(ctx, "debug_out", DebugInfo{
            Settings: c.settings,
            State:    c.state,
            IsLeader: utils.IsLeader(ctx),
            NodeName: c.nodeName,
            Metadata: c.metadata,
        })
    }
    // ...
}

Control Port Debugging

Add debug buttons:

type Control struct {
    Start      bool `json:"start,omitempty" title:"Start" format:"button"`
    Stop       bool `json:"stop,omitempty" title:"Stop" format:"button"`
    DumpState  bool `json:"dumpState,omitempty" title:"Dump State" format:"button"`
    Reset      bool `json:"reset,omitempty" title:"Reset" format:"button"`
}

func (c *Component) Handle(ctx context.Context, output module.Handler, port string, msg any) error {
    if port == v1alpha1.ControlPort {
        control := msg.(Control)
        if control.DumpState {
            c.dumpStateToLog()
            return nil
        }
        // ...
    }
}

func (c *Component) dumpStateToLog() {
    log.Info("COMPONENT STATE DUMP",
        "settings", fmt.Sprintf("%+v", c.settings),
        "isRunning", c.isRunning,
        "currentPort", c.currentPort,
        "connections", len(c.connections),
    )
}

Flow Debugging

Message Tracing

Track message flow:

func (c *Component) Handle(ctx context.Context, output module.Handler, port string, msg any) error {
    traceID := trace.SpanFromContext(ctx).SpanContext().TraceID().String()

    log.Info("MESSAGE TRACE",
        "traceID", traceID,
        "component", c.GetInfo().Name,
        "port", port,
        "msgPreview", preview(msg, 100),
    )

    // Wrap output to trace outgoing
    tracedOutput := func(ctx context.Context, port string, msg any) error {
        log.Info("MESSAGE OUT",
            "traceID", traceID,
            "toPort", port,
            "msgPreview", preview(msg, 100),
        )
        return output(ctx, port, msg)
    }

    return c.process(ctx, tracedOutput, port, msg)
}

func preview(v any, maxLen int) string {
    s := fmt.Sprintf("%+v", v)
    if len(s) > maxLen {
        return s[:maxLen] + "..."
    }
    return s
}

Debug Component

Create a debug component that logs everything:

type Debug struct {
    settings Settings
}

type Settings struct {
    Label   string `json:"label" title:"Label" default:"debug"`
    Verbose bool   `json:"verbose" title:"Verbose Output"`
}

func (d *Debug) Handle(ctx context.Context, output module.Handler, port string, msg any) error {
    if port == "input" {
        log.Info("DEBUG: "+d.settings.Label,
            "type", fmt.Sprintf("%T", msg),
            "value", func() any {
                if d.settings.Verbose {
                    return msg
                }
                return preview(msg, 200)
            }(),
        )
        return output(ctx, "output", msg)
    }
    return nil
}

Kubernetes Debugging

Check Pod Status

# List pods
kubectl get pods -n tinysystems -l app=my-module

# Describe pod for events
kubectl describe pod my-module-abc123 -n tinysystems

# Check logs
kubectl logs my-module-abc123 -n tinysystems
kubectl logs my-module-abc123 -n tinysystems --previous  # Crashed pod

# Follow logs
kubectl logs -f my-module-abc123 -n tinysystems

Check CRDs

# List TinyNodes
kubectl get tinynodes -n tinysystems

# Describe specific node
kubectl describe tinynode my-flow-abc123 -n tinysystems

# Get TinyNode YAML
kubectl get tinynode my-flow-abc123 -n tinysystems -o yaml

# Check TinySignals
kubectl get tinysignals -n tinysystems

# Check TinyModules
kubectl get tinymodules -n tinysystems

Debug with Shell

# Exec into pod
kubectl exec -it my-module-abc123 -n tinysystems -- sh

# Port forward for local access
kubectl port-forward my-module-abc123 8080:8080 -n tinysystems

# Check environment
kubectl exec my-module-abc123 -n tinysystems -- env

Network Debugging

# Check services
kubectl get svc -n tinysystems

# Test DNS resolution
kubectl run debug --rm -it --image=busybox -- nslookup my-module-v1.tinysystems

# Test connectivity
kubectl run debug --rm -it --image=curlimages/curl -- \
  curl http://my-module-v1.tinysystems:50051

Common Issues

Component Not Receiving Messages

Symptoms: Handle() never called

Check:

1. TinyNode exists and is Ready
2. Edges are properly configured
3. Source component is sending

# Check node status
kubectl get tinynode my-node -n tinysystems -o yaml

# Check edges
kubectl get tinynode my-node -n tinysystems -o jsonpath='{.spec.edges}'

Messages Lost Between Modules

Symptoms: Cross-module messages not arriving

Check:

1. Both modules discovered each other
2. gRPC connection healthy
3. No network policies blocking

# Check TinyModules
kubectl get tinymodules -n tinysystems

# Check module addresses
kubectl get tinymodule my-module-v1 -n tinysystems -o jsonpath='{.status.addr}'

Leader Not Processing Controls

Symptoms: Control buttons don't work

Check:

1. Leader election working
2. Component checking IsLeader

# Check leases
kubectl get leases -n tinysystems

# Check which pod is leader
kubectl get lease my-module-leader -n tinysystems -o jsonpath='{.spec.holderIdentity}'

Settings Not Applied

Symptoms: Component ignores settings

Check:

1. Settings port handler implemented
2. TinyNode has _settings edge
3. Settings schema matches

# Check TinyNode edges for _settings
edges:
  - from: _settings
    data:
      timeout: 5000

Port Not Exposed

Symptoms: HTTP endpoint not accessible

Check:

1. Service created
2. Ingress created
3. DNS resolving
4. Certificate ready

kubectl get svc,ingress -n tinysystems -l tinysystems.io/node=my-node
kubectl get certificate -n tinysystems

Debugging Checklist

Flow Not Working

• [ ] TinyNode exists and status is Ready
• [ ] All referenced components available
• [ ] Edges properly configured
• [ ] Expression syntax correct
• [ ] Port schemas match data

Component Not Starting

• [ ] Settings delivered before other messages
• [ ] Required settings have values
• [ ] No initialization errors in logs
• [ ] Context not cancelled prematurely

Cross-Module Issues

• [ ] Both TinyModules have addresses
• [ ] gRPC port accessible
• [ ] No network policies blocking
• [ ] Client pool has connection

Multi-Replica Issues

• [ ] Leader election working
• [ ] Only leader writes to CRs
• [ ] All pods watching CR changes
• [ ] Metadata propagating correctly

Debug Tools Summary

Tool	Use Case
kubectl logs	View component logs
kubectl describe	Check events and status
kubectl get -o yaml	View full resource
kubectl exec	Interactive debugging
kubectl port-forward	Local access
IDE debugger	Step through code
Debug component	Inspect flow data
Control buttons	Trigger debug actions

Next Steps

• Testing Components - Test before debug
• Observability - Production monitoring
• FAQ - Common problems and solutions