HTTP Client Component
A complete example of an HTTP client component for making external API calls.
Overview
This component makes HTTP requests to external services. It demonstrates:
- HTTP client configuration
- Request/response handling
- Error handling
- Retry logic
Complete Implementation
go
package httpclient
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"strings"
"time"
"github.com/tiny-systems/module/api/v1alpha1"
"github.com/tiny-systems/module/pkg/module"
)
const ComponentName = "http_client"
type Component struct {
settings Settings
client *http.Client
}
// Settings configuration
type Settings struct {
BaseURL string `json:"baseUrl,omitempty" title:"Base URL"
description:"Base URL for all requests (optional)"`
DefaultHeaders map[string]string `json:"defaultHeaders,omitempty" title:"Default Headers"
description:"Headers added to all requests"`
Timeout string `json:"timeout" title:"Timeout" default:"30s"
description:"Request timeout duration"`
RetryCount int `json:"retryCount" title:"Retry Count" default:"0" minimum:"0" maximum:"5"
description:"Number of retry attempts on failure"`
RetryDelay string `json:"retryDelay" title:"Retry Delay" default:"1s"
description:"Delay between retry attempts"`
}
// RequestInput for making requests
type RequestInput struct {
Method string `json:"method" title:"Method" default:"GET"
enum:"GET,POST,PUT,PATCH,DELETE,HEAD,OPTIONS" configurable:"true"
description:"HTTP method"`
URL string `json:"url" title:"URL" required:"true" configurable:"true"
description:"Request URL (absolute or relative to base URL)"`
Headers map[string]string `json:"headers,omitempty" title:"Headers" configurable:"true"
description:"Request headers"`
QueryParams map[string]string `json:"queryParams,omitempty" title:"Query Parameters" configurable:"true"
description:"URL query parameters"`
Body any `json:"body,omitempty" title:"Body" configurable:"true"
description:"Request body (will be JSON encoded if object)"`
ContentType string `json:"contentType,omitempty" title:"Content Type" default:"application/json"
description:"Content-Type header"`
}
// ResponseOutput for successful responses
type ResponseOutput struct {
StatusCode int `json:"statusCode" title:"Status Code"`
Status string `json:"status" title:"Status"`
Headers map[string]string `json:"headers" title:"Response Headers"`
Body string `json:"body" title:"Response Body"`
BodyJSON any `json:"bodyJson,omitempty" title:"Parsed JSON Body"`
Duration int64 `json:"duration" title:"Duration (ms)"`
URL string `json:"url" title:"Request URL"`
Method string `json:"method" title:"Request Method"`
RetryAttempt int `json:"retryAttempt,omitempty" title:"Retry Attempt"`
}
// ErrorOutput for failed requests
type ErrorOutput struct {
Error string `json:"error" title:"Error Message"`
StatusCode int `json:"statusCode,omitempty" title:"Status Code"`
URL string `json:"url" title:"Request URL"`
Method string `json:"method" title:"Request Method"`
RetryAttempt int `json:"retryAttempt,omitempty" title:"Retry Attempt"`
Duration int64 `json:"duration" title:"Duration (ms)"`
}
var _ module.Component = (*Component)(nil)
func (c *Component) GetInfo() module.ComponentInfo {
return module.ComponentInfo{
Name: ComponentName,
Title: "HTTP Client",
Description: "Makes HTTP requests to external services",
Category: "HTTP",
Tags: []string{"http", "client", "api", "request"},
}
}
func (c *Component) Ports() []module.Port {
return []module.Port{
{
Name: v1alpha1.SettingsPort,
Label: "Settings",
Position: module.PositionTop,
Source: true,
Configuration: Settings{},
},
{
Name: "request",
Label: "Request",
Position: module.PositionLeft,
Source: true,
Configuration: RequestInput{},
},
{
Name: "response",
Label: "Response",
Position: module.PositionRight,
Source: false,
Configuration: ResponseOutput{},
},
{
Name: "error",
Label: "Error",
Position: module.PositionBottom,
Source: false,
Configuration: ErrorOutput{},
},
}
}
func (c *Component) Handle(
ctx context.Context,
output module.Handler,
port string,
msg any,
) error {
switch port {
case v1alpha1.SettingsPort:
c.settings = msg.(Settings)
c.initClient()
return nil
case "request":
return c.handleRequest(ctx, output, msg.(RequestInput))
default:
return fmt.Errorf("unknown port: %s", port)
}
}
func (c *Component) initClient() {
timeout, err := time.ParseDuration(c.settings.Timeout)
if err != nil {
timeout = 30 * time.Second
}
c.client = &http.Client{
Timeout: timeout,
Transport: &http.Transport{
MaxIdleConns: 100,
MaxIdleConnsPerHost: 10,
IdleConnTimeout: 90 * time.Second,
},
}
}
func (c *Component) handleRequest(
ctx context.Context,
output module.Handler,
input RequestInput,
) error {
if c.client == nil {
c.initClient()
}
retryDelay, _ := time.ParseDuration(c.settings.RetryDelay)
if retryDelay == 0 {
retryDelay = time.Second
}
var lastErr error
for attempt := 0; attempt <= c.settings.RetryCount; attempt++ {
if attempt > 0 {
time.Sleep(retryDelay)
}
resp, err := c.doRequest(ctx, input, attempt)
if err != nil {
lastErr = err
continue
}
// Success - check if it's an error status
if resp.StatusCode >= 400 {
// Retry on server errors
if resp.StatusCode >= 500 && attempt < c.settings.RetryCount {
lastErr = fmt.Errorf("server error: %d", resp.StatusCode)
continue
}
}
return output(ctx, "response", *resp)
}
// All retries failed
return output(ctx, "error", ErrorOutput{
Error: lastErr.Error(),
URL: c.buildURL(input.URL),
Method: input.Method,
RetryAttempt: c.settings.RetryCount,
})
}
func (c *Component) doRequest(
ctx context.Context,
input RequestInput,
attempt int,
) (*ResponseOutput, error) {
startTime := time.Now()
// Build URL
url := c.buildURL(input.URL)
// Add query parameters
if len(input.QueryParams) > 0 {
params := make([]string, 0, len(input.QueryParams))
for k, v := range input.QueryParams {
params = append(params, fmt.Sprintf("%s=%s", k, v))
}
separator := "?"
if strings.Contains(url, "?") {
separator = "&"
}
url = url + separator + strings.Join(params, "&")
}
// Prepare body
var bodyReader io.Reader
if input.Body != nil {
switch b := input.Body.(type) {
case string:
bodyReader = strings.NewReader(b)
case []byte:
bodyReader = bytes.NewReader(b)
default:
jsonBody, err := json.Marshal(b)
if err != nil {
return nil, fmt.Errorf("failed to marshal body: %w", err)
}
bodyReader = bytes.NewReader(jsonBody)
}
}
// Create request
method := input.Method
if method == "" {
method = "GET"
}
req, err := http.NewRequestWithContext(ctx, method, url, bodyReader)
if err != nil {
return nil, fmt.Errorf("failed to create request: %w", err)
}
// Set default headers
for k, v := range c.settings.DefaultHeaders {
req.Header.Set(k, v)
}
// Set request headers (override defaults)
for k, v := range input.Headers {
req.Header.Set(k, v)
}
// Set content type if body present
if input.Body != nil && input.ContentType != "" {
req.Header.Set("Content-Type", input.ContentType)
}
// Execute request
resp, err := c.client.Do(req)
if err != nil {
return nil, fmt.Errorf("request failed: %w", err)
}
defer resp.Body.Close()
// Read response body
body, err := io.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("failed to read response: %w", err)
}
duration := time.Since(startTime).Milliseconds()
// Extract response headers
headers := make(map[string]string)
for k, v := range resp.Header {
if len(v) > 0 {
headers[k] = v[0]
}
}
// Try to parse JSON body
var bodyJSON any
if strings.Contains(resp.Header.Get("Content-Type"), "application/json") {
json.Unmarshal(body, &bodyJSON)
}
return &ResponseOutput{
StatusCode: resp.StatusCode,
Status: resp.Status,
Headers: headers,
Body: string(body),
BodyJSON: bodyJSON,
Duration: duration,
URL: url,
Method: method,
RetryAttempt: attempt,
}, nil
}
func (c *Component) buildURL(path string) string {
if strings.HasPrefix(path, "http://") || strings.HasPrefix(path, "https://") {
return path
}
if c.settings.BaseURL != "" {
base := strings.TrimSuffix(c.settings.BaseURL, "/")
if !strings.HasPrefix(path, "/") {
path = "/" + path
}
return base + path
}
return path
}
func (c *Component) Instance() module.Component {
return &Component{}
}Usage Examples
Basic GET Request
yaml
edges:
- port: _settings
data:
timeout: "30s"
- port: request
data:
method: "GET"
url: "https://api.example.com/users/{{$.userId}}"POST with JSON Body
yaml
edges:
- port: request
data:
method: "POST"
url: "https://api.example.com/users"
contentType: "application/json"
body:
name: "{{$.name}}"
email: "{{$.email}}"With Base URL and Default Headers
yaml
edges:
- port: _settings
data:
baseUrl: "https://api.example.com/v1"
defaultHeaders:
Authorization: "Bearer {{$.secrets.apiKey}}"
X-API-Version: "2024-01"
timeout: "60s"
retryCount: 3
retryDelay: "2s"
- port: request
data:
method: "GET"
url: "/users" # Becomes https://api.example.com/v1/users
queryParams:
page: "{{$.page}}"
limit: "100"Response Handling
Success Response
json
{
"statusCode": 200,
"status": "200 OK",
"headers": {
"Content-Type": "application/json",
"X-Request-Id": "abc123"
},
"body": "{\"users\":[{\"id\":1,\"name\":\"John\"}]}",
"bodyJson": {
"users": [{"id": 1, "name": "John"}]
},
"duration": 245,
"url": "https://api.example.com/v1/users",
"method": "GET",
"retryAttempt": 0
}Error Response
json
{
"error": "request failed: connection refused",
"url": "https://api.example.com/v1/users",
"method": "GET",
"retryAttempt": 3,
"duration": 5234
}Key Patterns Demonstrated
1. Client Reuse
Initialize once, reuse for all requests:
go
func (c *Component) initClient() {
c.client = &http.Client{
Timeout: timeout,
Transport: &http.Transport{
MaxIdleConns: 100,
MaxIdleConnsPerHost: 10,
},
}
}2. Retry Logic
Automatic retries with configurable delay:
go
for attempt := 0; attempt <= c.settings.RetryCount; attempt++ {
if attempt > 0 {
time.Sleep(retryDelay)
}
resp, err := c.doRequest(ctx, input, attempt)
// ...
}3. Base URL Handling
Support both absolute and relative URLs:
go
func (c *Component) buildURL(path string) string {
if strings.HasPrefix(path, "http://") {
return path // Absolute URL
}
return c.settings.BaseURL + path // Relative to base
}4. JSON Auto-Parsing
Parse JSON responses automatically:
go
if strings.Contains(resp.Header.Get("Content-Type"), "application/json") {
json.Unmarshal(body, &bodyJSON)
}Visual Flow
┌─────────────────────────────────────────────────────────────┐
│ HTTP Client │
│ │
│ Request Input │
│ ┌────────────────┐ │
│ │ method: POST │ │
│ │ url: /users │ │
│ │ body: {...} │ │
│ └───────┬────────┘ │
│ │ │
│ ▼ │
│ ┌────────────────────────────────────────────────────┐ │
│ │ Request Pipeline │ │
│ │ │ │
│ │ Build URL ──► Set Headers ──► Encode Body │ │
│ │ │ │
│ └────────────────────────┬───────────────────────────┘ │
│ │ │
│ ▼ │
│ ┌────────────────────────────────────────────────────┐ │
│ │ HTTP Transport │ │
│ │ │ │
│ │ Retry Loop: attempt 0 ──► 1 ──► 2 ──► 3 │ │
│ │ │ │
│ └────────────────────────┬───────────────────────────┘ │
│ │ │
│ ┌───────────────┴───────────────┐ │
│ │ │ │
│ ▼ ▼ │
│ ┌─────────────────┐ ┌─────────────────┐ │
│ │ Response │ │ Error │ │
│ │ Output │ │ Output │ │
│ └─────────────────┘ └─────────────────┘ │
└─────────────────────────────────────────────────────────────┘Common Use Cases
1. API Integration
yaml
# Fetch data from external API
request:
method: "GET"
url: "https://api.github.com/repos/{{$.owner}}/{{$.repo}}"
headers:
Accept: "application/vnd.github.v3+json"2. Webhook Delivery
yaml
# POST webhook payload
request:
method: "POST"
url: "{{$.webhook.url}}"
body: "{{$.payload}}"
headers:
X-Webhook-Secret: "{{$.secrets.webhookSecret}}"3. Form Submission
yaml
request:
method: "POST"
url: "/api/submit"
contentType: "application/x-www-form-urlencoded"
body: "name={{$.name}}&email={{$.email}}"Extension Ideas
- OAuth Support: Built-in OAuth2 authentication
- Certificate Pinning: Custom TLS configuration
- Request Signing: AWS Signature, HMAC signing
- Response Caching: Cache responses with TTL
- Circuit Breaker: Fail fast on repeated errors