feat: add agents management and enhance inspector with typed forms

- Introduced a new "Agents" section in the app sidebar for better navigation. - Enhanced the Inspector component to support typed forms for various node types, improving user experience when configuring nodes. - Created a new NodeForm component to handle specific forms for different node types, including Trigger, Build, Test, Eval, Policy, Approval, Deploy, Promote, and Rollback. - Updated the API layer to manage agents, including listing, creating, deleting, and testing agent authentication. - Modified the node catalog to include new node types and their respective configurations. - Adjusted the Next.js configuration and Nginx settings to reflect changes in API endpoint ports.
2026-08-03 07:21:04 +02:00 · 2026-05-13 22:15:08 +05:30
parent 64a857d1b7
commit 0a1874e736
21 changed files with 3175 additions and 117 deletions
@@ -0,0 +1,521 @@
+package api
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"log/slog"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/lyzrai/flow/pkg/engine"
+	"github.com/lyzrai/flow/pkg/github"
+	"github.com/lyzrai/flow/pkg/models"
+	"github.com/lyzrai/flow/pkg/orchestrator"
+	"github.com/lyzrai/flow/pkg/storage"
+)
+
+// Agent is the wire shape of an agent record. PAT and webhook secret are
+// scrubbed; HasPAT and webhook fields surface only the safe parts.
+type Agent struct {
+	ID                 string             `json:"id"`
+	Name               string             `json:"name"`
+	RepoURL            string             `json:"repoUrl"`
+	Ref                string             `json:"ref,omitempty"`
+	HasPAT             bool               `json:"hasPat"`
+	WebhookID          int64              `json:"webhookId,omitempty"`
+	WebhookURL         string             `json:"webhookUrl,omitempty"`
+	WebhookInstalled   bool               `json:"webhookInstalled"`
+	WebhookInstalledAt *time.Time         `json:"webhookInstalledAt,omitempty"`
+	AuthStatus         storage.AuthStatus `json:"authStatus,omitempty"`
+	AuthCheckedAt      *time.Time         `json:"authCheckedAt,omitempty"`
+	AttachedPipelines  []string           `json:"attachedPipelines,omitempty"`
+	CreatedAt          time.Time          `json:"createdAt"`
+	UpdatedAt          time.Time          `json:"updatedAt"`
+}
+
+func (s *Server) publicAgent(a *storage.Agent) Agent {
+	return Agent{
+		ID:                 a.ID,
+		Name:               a.Name,
+		RepoURL:            a.RepoURL,
+		Ref:                a.Ref,
+		HasPAT:             a.PAT != "",
+		WebhookID:          a.WebhookID,
+		WebhookURL:         s.webhookURLFor(a.ID),
+		WebhookInstalled:   a.WebhookID != 0,
+		WebhookInstalledAt: a.WebhookInstalledAt,
+		AuthStatus:         a.AuthStatus,
+		AuthCheckedAt:      a.AuthCheckedAt,
+		AttachedPipelines:  a.AttachedPipelines,
+		CreatedAt:          a.CreatedAt,
+		UpdatedAt:          a.UpdatedAt,
+	}
+}
+
+func (s *Server) webhookURLFor(agentID string) string {
+	if s.publicURL == "" {
+		return ""
+	}
+	return s.publicURL + "/webhooks/github/" + agentID
+}
+
+// --- CRUD -----------------------------------------------------------------
+
+func (s *Server) handleListAgents(w http.ResponseWriter, r *http.Request) {
+	agents, err := s.agents.List(r.Context())
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, err)
+		return
+	}
+	out := make([]Agent, 0, len(agents))
+	for _, a := range agents {
+		out = append(out, s.publicAgent(a))
+	}
+	writeJSON(w, http.StatusOK, out)
+}
+
+func (s *Server) handleCreateAgent(w http.ResponseWriter, r *http.Request) {
+	var body struct {
+		RepoURL string `json:"repoUrl"`
+		PAT     string `json:"pat"`
+		Ref     string `json:"ref,omitempty"`
+		Name    string `json:"name,omitempty"`
+	}
+	if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
+		writeError(w, http.StatusBadRequest, err)
+		return
+	}
+	repo := strings.TrimSpace(body.RepoURL)
+	if repo == "" {
+		writeError(w, http.StatusBadRequest, errors.New("repoUrl is required"))
+		return
+	}
+	if _, err := url.Parse(repo); err != nil {
+		writeError(w, http.StatusBadRequest, fmt.Errorf("invalid repoUrl: %w", err))
+		return
+	}
+
+	now := time.Now().UTC()
+	id := newID()
+	name := strings.TrimSpace(body.Name)
+	if name == "" {
+		name = deriveAgentName(repo)
+	}
+	ref := strings.TrimSpace(body.Ref)
+	if ref == "" {
+		ref = "main"
+	}
+
+	a := &storage.Agent{
+		ID:         id,
+		Name:       name,
+		RepoURL:    repo,
+		Ref:        ref,
+		PAT:        strings.TrimSpace(body.PAT),
+		AuthStatus: storage.AuthUntested,
+		CreatedAt:  now,
+		UpdatedAt:  now,
+	}
+	if err := s.agents.Create(r.Context(), a); err != nil {
+		writeError(w, http.StatusInternalServerError, err)
+		return
+	}
+	writeJSON(w, http.StatusCreated, s.publicAgent(a))
+}
+
+func (s *Server) handleGetAgent(w http.ResponseWriter, r *http.Request) {
+	id := r.PathValue("id")
+	a, err := s.agents.Get(r.Context(), id)
+	if err != nil {
+		writeStorageErr(w, err, "agent not found")
+		return
+	}
+	writeJSON(w, http.StatusOK, s.publicAgent(a))
+}
+
+func (s *Server) handleDeleteAgent(w http.ResponseWriter, r *http.Request) {
+	id := r.PathValue("id")
+	// Best-effort uninstall webhook before deleting, so we don't leak
+	// dangling hooks pointing at a dead agent ID.
+	if a, err := s.agents.Get(r.Context(), id); err == nil && a.WebhookID != 0 {
+		if repo, perr := github.ParseRepo(a.RepoURL); perr == nil {
+			ctx, cancel := context.WithTimeout(r.Context(), 10*time.Second)
+			_ = github.NewClient(a.PAT).UninstallWebhook(ctx, repo, a.WebhookID)
+			cancel()
+		}
+	}
+	if err := s.agents.Delete(r.Context(), id); err != nil {
+		writeStorageErr(w, err, "agent not found")
+		return
+	}
+	w.WriteHeader(http.StatusNoContent)
+}
+
+// --- auth probe ----------------------------------------------------------
+
+func (s *Server) handleTestAgentAuth(w http.ResponseWriter, r *http.Request) {
+	id := r.PathValue("id")
+	a, err := s.agents.Get(r.Context(), id)
+	if err != nil {
+		writeStorageErr(w, err, "agent not found")
+		return
+	}
+	repo, err := github.ParseRepo(a.RepoURL)
+	if err != nil {
+		writeError(w, http.StatusBadRequest, err)
+		return
+	}
+	ctx, cancel := context.WithTimeout(r.Context(), 15*time.Second)
+	defer cancel()
+	authErr := github.NewClient(a.PAT).TestAuth(ctx, repo)
+
+	now := time.Now().UTC()
+	a.AuthCheckedAt = &now
+	if authErr == nil {
+		a.AuthStatus = storage.AuthOK
+	} else {
+		a.AuthStatus = storage.AuthFailed
+	}
+	a.UpdatedAt = now
+	if uerr := s.agents.Update(r.Context(), a); uerr != nil {
+		writeError(w, http.StatusInternalServerError, uerr)
+		return
+	}
+	resp := map[string]any{
+		"authStatus":    a.AuthStatus,
+		"authCheckedAt": a.AuthCheckedAt,
+	}
+	if authErr != nil {
+		resp["error"] = authErr.Error()
+	}
+	writeJSON(w, http.StatusOK, resp)
+}
+
+// --- webhook install / uninstall -----------------------------------------
+
+func (s *Server) handleInstallWebhook(w http.ResponseWriter, r *http.Request) {
+	if s.publicURL == "" {
+		writeError(w, http.StatusServiceUnavailable,
+			errors.New("FLOW_PUBLIC_URL is not configured"))
+		return
+	}
+	id := r.PathValue("id")
+	a, err := s.agents.Get(r.Context(), id)
+	if err != nil {
+		writeStorageErr(w, err, "agent not found")
+		return
+	}
+	if a.PAT == "" {
+		writeError(w, http.StatusBadRequest, errors.New("agent has no PAT — re-create with one"))
+		return
+	}
+	repo, err := github.ParseRepo(a.RepoURL)
+	if err != nil {
+		writeError(w, http.StatusBadRequest, err)
+		return
+	}
+
+	// Fresh secret per install so rotating is just "uninstall + install".
+	secret, err := github.GenerateSecret()
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, err)
+		return
+	}
+	callback := s.webhookURLFor(id)
+
+	ctx, cancel := context.WithTimeout(r.Context(), 20*time.Second)
+	defer cancel()
+	hookID, err := github.NewClient(a.PAT).InstallWebhook(ctx, repo, callback, secret)
+	if err != nil {
+		writeError(w, http.StatusBadGateway, err)
+		return
+	}
+
+	now := time.Now().UTC()
+	a.WebhookID = hookID
+	a.WebhookSecret = secret
+	a.WebhookInstalledAt = &now
+	a.UpdatedAt = now
+	if err := s.agents.Update(r.Context(), a); err != nil {
+		// Try to roll back the hook so we don't leak it.
+		_ = github.NewClient(a.PAT).UninstallWebhook(ctx, repo, hookID)
+		writeError(w, http.StatusInternalServerError, err)
+		return
+	}
+	writeJSON(w, http.StatusOK, s.publicAgent(a))
+}
+
+func (s *Server) handleUninstallWebhook(w http.ResponseWriter, r *http.Request) {
+	id := r.PathValue("id")
+	a, err := s.agents.Get(r.Context(), id)
+	if err != nil {
+		writeStorageErr(w, err, "agent not found")
+		return
+	}
+	if a.WebhookID == 0 {
+		writeError(w, http.StatusBadRequest, errors.New("no webhook installed"))
+		return
+	}
+	repo, err := github.ParseRepo(a.RepoURL)
+	if err != nil {
+		writeError(w, http.StatusBadRequest, err)
+		return
+	}
+	ctx, cancel := context.WithTimeout(r.Context(), 15*time.Second)
+	defer cancel()
+	if err := github.NewClient(a.PAT).UninstallWebhook(ctx, repo, a.WebhookID); err != nil {
+		writeError(w, http.StatusBadGateway, err)
+		return
+	}
+
+	a.WebhookID = 0
+	a.WebhookSecret = ""
+	a.WebhookInstalledAt = nil
+	a.UpdatedAt = time.Now().UTC()
+	if err := s.agents.Update(r.Context(), a); err != nil {
+		writeError(w, http.StatusInternalServerError, err)
+		return
+	}
+	writeJSON(w, http.StatusOK, s.publicAgent(a))
+}
+
+// --- pipeline attachments ------------------------------------------------
+
+func (s *Server) handleAttachPipeline(w http.ResponseWriter, r *http.Request) {
+	id := r.PathValue("id")
+	pipelineID := r.PathValue("pipelineId")
+	a, err := s.agents.Get(r.Context(), id)
+	if err != nil {
+		writeStorageErr(w, err, "agent not found")
+		return
+	}
+	if _, err := s.pipelines.Get(r.Context(), pipelineID); err != nil {
+		writeStorageErr(w, err, "pipeline not found")
+		return
+	}
+	for _, existing := range a.AttachedPipelines {
+		if existing == pipelineID {
+			writeJSON(w, http.StatusOK, s.publicAgent(a)) // already attached
+			return
+		}
+	}
+	a.AttachedPipelines = append(a.AttachedPipelines, pipelineID)
+	a.UpdatedAt = time.Now().UTC()
+	if err := s.agents.Update(r.Context(), a); err != nil {
+		writeError(w, http.StatusInternalServerError, err)
+		return
+	}
+	writeJSON(w, http.StatusOK, s.publicAgent(a))
+}
+
+func (s *Server) handleDetachPipeline(w http.ResponseWriter, r *http.Request) {
+	id := r.PathValue("id")
+	pipelineID := r.PathValue("pipelineId")
+	a, err := s.agents.Get(r.Context(), id)
+	if err != nil {
+		writeStorageErr(w, err, "agent not found")
+		return
+	}
+	out := a.AttachedPipelines[:0]
+	for _, p := range a.AttachedPipelines {
+		if p != pipelineID {
+			out = append(out, p)
+		}
+	}
+	a.AttachedPipelines = out
+	a.UpdatedAt = time.Now().UTC()
+	if err := s.agents.Update(r.Context(), a); err != nil {
+		writeError(w, http.StatusInternalServerError, err)
+		return
+	}
+	w.WriteHeader(http.StatusNoContent)
+}
+
+// --- manual trigger ------------------------------------------------------
+
+// handleTriggerAgent dispatches a run on each attached pipeline. Trigger
+// data describes who/what triggered the run (manual / webhook / etc.).
+func (s *Server) handleTriggerAgent(w http.ResponseWriter, r *http.Request) {
+	if s.orch == nil {
+		writeError(w, http.StatusServiceUnavailable, errors.New("orchestrator not configured"))
+		return
+	}
+	id := r.PathValue("id")
+	a, err := s.agents.Get(r.Context(), id)
+	if err != nil {
+		writeStorageErr(w, err, "agent not found")
+		return
+	}
+	if len(a.AttachedPipelines) == 0 {
+		writeError(w, http.StatusBadRequest, errors.New("agent has no attached pipelines"))
+		return
+	}
+	triggerData := []map[string]any{{
+		"source":    "manual",
+		"agentId":   a.ID,
+		"agentName": a.Name,
+		"repoUrl":   a.RepoURL,
+		"ref":       a.Ref,
+	}}
+	execIDs, err := s.dispatchAgent(r.Context(), a, triggerData)
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, err)
+		return
+	}
+	writeJSON(w, http.StatusAccepted, map[string]any{
+		"executionIds": execIDs,
+	})
+}
+
+// dispatchAgent runs each attached pipeline asynchronously. Returns the
+// execution IDs collected. Failures on individual pipelines are logged
+// but don't abort the rest.
+func (s *Server) dispatchAgent(ctx context.Context, a *storage.Agent, trigger any) ([]string, error) {
+	triggerJSON, _ := json.Marshal(trigger)
+	var triggerItems []models.Item
+	if items, ok := trigger.([]map[string]any); ok {
+		for _, m := range items {
+			triggerItems = append(triggerItems, models.Item(m))
+		}
+	}
+
+	out := make([]string, 0, len(a.AttachedPipelines))
+	for _, pid := range a.AttachedPipelines {
+		p, err := s.pipelines.Get(ctx, pid)
+		if err != nil {
+			slog.WarnContext(ctx, "agent_dispatch_pipeline_missing",
+				slog.String("agent_id", a.ID),
+				slog.String("pipeline_id", pid),
+			)
+			continue
+		}
+		wf, err := engine.ParseWorkflow(p.Definition)
+		if err != nil {
+			slog.WarnContext(ctx, "agent_dispatch_parse_failed",
+				slog.String("pipeline_id", pid),
+				slog.Any("error", err),
+			)
+			continue
+		}
+		runCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
+		execID, err := s.orch.RunAsync(runCtx, &orchestrator.RunRequest{
+			RequestMeta: orchestrator.RequestMeta{WorkflowID: pid},
+			Workflow:    wf,
+			TriggerData: triggerItems,
+		})
+		cancel()
+		if err != nil {
+			slog.WarnContext(ctx, "agent_dispatch_failed",
+				slog.String("pipeline_id", pid),
+				slog.Any("error", err),
+			)
+			continue
+		}
+		if s.runs != nil {
+			_ = s.runs.Insert(ctx, &storage.Run{
+				ID:           execID,
+				PipelineID:   pid,
+				PipelineName: p.Name,
+				Status:       "running",
+				StartedAt:    time.Now().UTC(),
+				TriggerData:  triggerJSON,
+			})
+		}
+		out = append(out, execID)
+	}
+	return out, nil
+}
+
+// --- webhook receiver ----------------------------------------------------
+
+func (s *Server) handleGitHubWebhook(w http.ResponseWriter, r *http.Request) {
+	id := r.PathValue("id")
+	a, err := s.agents.Get(r.Context(), id)
+	if err != nil {
+		writeStorageErr(w, err, "agent not found")
+		return
+	}
+	body, err := io.ReadAll(r.Body)
+	if err != nil {
+		writeError(w, http.StatusBadRequest, err)
+		return
+	}
+	if err := github.VerifySignature(r.Header.Get("X-Hub-Signature-256"), a.WebhookSecret, body); err != nil {
+		slog.WarnContext(r.Context(), "github_webhook_signature_invalid",
+			slog.String("agent_id", id),
+			slog.Any("error", err),
+		)
+		writeError(w, http.StatusUnauthorized, errors.New("signature invalid"))
+		return
+	}
+
+	event := r.Header.Get("X-GitHub-Event")
+	switch event {
+	case "ping":
+		writeJSON(w, http.StatusOK, map[string]string{"message": "pong"})
+		return
+	case "push":
+		// fall through
+	default:
+		// Acknowledge unknown events; nothing to dispatch.
+		writeJSON(w, http.StatusOK, map[string]string{"message": "ignored"})
+		return
+	}
+
+	push, err := github.ParsePushEvent(body)
+	if err != nil {
+		writeError(w, http.StatusBadRequest, err)
+		return
+	}
+
+	// Per the agent's configured ref: only dispatch if the branch matches.
+	if a.Ref != "" && a.Ref != "*" && github.BranchFromRef(push.Ref) != a.Ref {
+		slog.InfoContext(r.Context(), "github_webhook_ref_mismatch",
+			slog.String("agent_id", id),
+			slog.String("event_ref", push.Ref),
+			slog.String("agent_ref", a.Ref),
+		)
+		writeJSON(w, http.StatusOK, map[string]string{"message": "ref filtered"})
+		return
+	}
+
+	if s.orch == nil {
+		writeError(w, http.StatusServiceUnavailable, errors.New("orchestrator not configured"))
+		return
+	}
+
+	trigger := []map[string]any{{
+		"source":    "github_push",
+		"agentId":   a.ID,
+		"agentName": a.Name,
+		"repoUrl":   a.RepoURL,
+		"ref":       push.Ref,
+		"branch":    github.BranchFromRef(push.Ref),
+		"commit":    push.After,
+		"pusher":    push.Pusher.Name,
+	}}
+	execIDs, err := s.dispatchAgent(r.Context(), a, trigger)
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, err)
+		return
+	}
+	writeJSON(w, http.StatusAccepted, map[string]any{
+		"executionIds": execIDs,
+	})
+}
+
+// --- helpers -------------------------------------------------------------
+
+// deriveAgentName extracts an "org/repo" name from a git URL. Falls back to
+// the URL itself if parsing fails.
+func deriveAgentName(raw string) string {
+	if r, err := github.ParseRepo(raw); err == nil {
+		return r.String()
+	}
+	return strings.TrimSpace(raw)
+}
@@ -16,13 +16,14 @@ import (
 	"io"
 	"log/slog"
 	"net/http"
+	"strconv"
 	"strings"
-	"sync"
 	"time"

 	"github.com/lyzrai/flow/pkg/engine"
 	"github.com/lyzrai/flow/pkg/models"
 	"github.com/lyzrai/flow/pkg/orchestrator"
+	"github.com/lyzrai/flow/pkg/storage"
 )

 // FlowSummary is the list-shape returned to the dashboard.
@@ -38,11 +39,19 @@ type FlowSummary struct {
 // ServerDeps groups the construction-time dependencies of the HTTP server.
 // Orchestrator drives durable execution; RestateIngressURL is used to
 // resolve Awakeables from the resume handler. CORSOrigins lists allowed
-// browser origins (use "*" to allow any — fine for dev).
+// browser origins (use "*" to allow any — fine for dev). Pipelines / Runs /
+// Agents are required — the API has no in-memory fallback.
 type ServerDeps struct {
 	Orchestrator      orchestrator.Orchestrator
 	RestateIngressURL string
 	CORSOrigins       []string
+	// PublicURL is the externally-reachable base URL for this API (e.g.
+	// "https://abcd.trycloudflare.com"). Used to render webhook callback
+	// URLs that GitHub can hit. Empty means webhook install is disabled.
+	PublicURL string
+	Pipelines storage.PipelineStore
+	Runs      storage.RunStore
+	Agents    storage.AgentStore
 }

 // Server is a thin JSON API server. It does not serve a frontend.
@@ -51,30 +60,26 @@ type Server struct {
 	orch          orchestrator.Orchestrator
 	restateIngres string
 	corsOrigins   []string
+	publicURL     string

-	mu    sync.Mutex
-	flows map[string]storedFlow // in-memory placeholder; storage layer lands next
-}
-
-// Definition is held as raw n8n-format JSON so we don't lose connection
-// shape on round-trip. We parse on read for validation + node count.
-type storedFlow struct {
-	ID         string          `json:"id"`
-	Name       string          `json:"name"`
-	Definition json.RawMessage `json:"definition"`
-	UpdatedAt  time.Time       `json:"updatedAt"`
-	NodeCount  int             `json:"nodeCount"`
+	pipelines storage.PipelineStore
+	runs      storage.RunStore
+	agents    storage.AgentStore
 }

 // NewServer constructs an API-only Server. deps.Orchestrator may be nil —
-// execute routes will then return 503.
+// execute routes will then return 503. Stores must be non-nil; CRUD routes
+// will panic without them — the API has no in-memory fallback.
 func NewServer(deps ServerDeps) *Server {
 	s := &Server{
 		mux:           http.NewServeMux(),
 		orch:          deps.Orchestrator,
 		restateIngres: deps.RestateIngressURL,
 		corsOrigins:   deps.CORSOrigins,
-		flows:         make(map[string]storedFlow),
+		publicURL:     strings.TrimRight(deps.PublicURL, "/"),
+		pipelines:     deps.Pipelines,
+		runs:          deps.Runs,
+		agents:        deps.Agents,
 	}
 	s.routes()
 	return s
@@ -93,15 +98,33 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {

 func (s *Server) routes() {
 	s.mux.HandleFunc("GET /api/health", s.handleHealth)
+	s.mux.HandleFunc("GET /api/config", s.handleConfig)
 	s.mux.HandleFunc("GET /api/workflows", s.handleListFlows)
 	s.mux.HandleFunc("POST /api/workflows", s.handleCreateFlow)
 	s.mux.HandleFunc("GET /api/workflows/{id}", s.handleGetFlow)
 	s.mux.HandleFunc("PUT /api/workflows/{id}", s.handleUpdateFlow)
 	s.mux.HandleFunc("DELETE /api/workflows/{id}", s.handleDeleteFlow)
 	s.mux.HandleFunc("POST /api/workflows/execute", s.handleExecuteWorkflow)
+	s.mux.HandleFunc("GET /api/executions", s.handleListExecutions)
 	s.mux.HandleFunc("GET /api/executions/{id}", s.handleGetExecution)
 	s.mux.HandleFunc("POST /api/executions/{id}/resume", s.handleResumeExecution)

+	// Agents — Langship-style agent registry (git URL + PAT)
+	s.mux.HandleFunc("GET /api/agents", s.handleListAgents)
+	s.mux.HandleFunc("POST /api/agents", s.handleCreateAgent)
+	s.mux.HandleFunc("GET /api/agents/{id}", s.handleGetAgent)
+	s.mux.HandleFunc("DELETE /api/agents/{id}", s.handleDeleteAgent)
+	s.mux.HandleFunc("POST /api/agents/{id}/test-auth", s.handleTestAgentAuth)
+	s.mux.HandleFunc("POST /api/agents/{id}/webhook", s.handleInstallWebhook)
+	s.mux.HandleFunc("DELETE /api/agents/{id}/webhook", s.handleUninstallWebhook)
+	s.mux.HandleFunc("POST /api/agents/{id}/pipelines/{pipelineId}", s.handleAttachPipeline)
+	s.mux.HandleFunc("DELETE /api/agents/{id}/pipelines/{pipelineId}", s.handleDetachPipeline)
+	s.mux.HandleFunc("POST /api/agents/{id}/trigger", s.handleTriggerAgent)
+
+	// Public webhook receiver. GitHub posts here; HMAC signature is the
+	// authentication. Must NOT require CORS / API auth.
+	s.mux.HandleFunc("POST /webhooks/github/{id}", s.handleGitHubWebhook)
+
 	// Anything not under /api/ is not our concern — the UI server handles it.
 	s.mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
 		writeError(w, http.StatusNotFound, fmt.Errorf("no handler for %s", r.URL.Path))
@@ -139,17 +162,30 @@ func (s *Server) handleHealth(w http.ResponseWriter, _ *http.Request) {
 	writeJSON(w, http.StatusOK, map[string]string{"status": "ok"})
 }

-func (s *Server) handleListFlows(w http.ResponseWriter, _ *http.Request) {
-	s.mu.Lock()
-	defer s.mu.Unlock()
-	out := make([]FlowSummary, 0, len(s.flows))
-	for _, f := range s.flows {
+// handleConfig exposes a few server-side config values to the UI so it can
+// render webhook URLs / decide whether to disable buttons.
+func (s *Server) handleConfig(w http.ResponseWriter, _ *http.Request) {
+	writeJSON(w, http.StatusOK, map[string]any{
+		"publicUrl":           s.publicURL,
+		"webhooksAvailable":   s.publicURL != "",
+		"orchestratorEnabled": s.orch != nil,
+	})
+}
+
+func (s *Server) handleListFlows(w http.ResponseWriter, r *http.Request) {
+	pipes, err := s.pipelines.List(r.Context())
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, err)
+		return
+	}
+	out := make([]FlowSummary, 0, len(pipes))
+	for _, p := range pipes {
 		out = append(out, FlowSummary{
-			ID:        f.ID,
-			Name:      f.Name,
-			UpdatedAt: f.UpdatedAt,
-			NodeCount: f.NodeCount,
-			Status:    "draft",
+			ID:        p.ID,
+			Name:      p.Name,
+			UpdatedAt: p.UpdatedAt,
+			NodeCount: p.NodeCount,
+			Status:    firstNonEmpty(p.Status, "draft"),
 		})
 	}
 	writeJSON(w, http.StatusOK, out)
@@ -169,30 +205,31 @@ func (s *Server) handleCreateFlow(w http.ResponseWriter, r *http.Request) {
 		writeError(w, http.StatusBadRequest, err)
 		return
 	}
+	now := time.Now().UTC()
 	id := newID()
-	f := storedFlow{
+	p := &storage.Pipeline{
 		ID:         id,
-		Name:       firstNonEmpty(body.Name, parsedName, "Untitled flow"),
+		Name:       firstNonEmpty(body.Name, parsedName, "Untitled pipeline"),
 		Definition: body.Definition,
-		UpdatedAt:  time.Now().UTC(),
 		NodeCount:  count,
+		CreatedAt:  now,
+		UpdatedAt:  now,
+	}
+	if err := s.pipelines.Create(r.Context(), p); err != nil {
+		writeError(w, http.StatusInternalServerError, err)
+		return
 	}
-	s.mu.Lock()
-	s.flows[id] = f
-	s.mu.Unlock()
 	writeJSON(w, http.StatusCreated, map[string]string{"id": id})
 }

 func (s *Server) handleGetFlow(w http.ResponseWriter, r *http.Request) {
 	id := r.PathValue("id")
-	s.mu.Lock()
-	f, ok := s.flows[id]
-	s.mu.Unlock()
-	if !ok {
-		writeError(w, http.StatusNotFound, errors.New("flow not found"))
+	p, err := s.pipelines.Get(r.Context(), id)
+	if err != nil {
+		writeStorageErr(w, err, "pipeline not found")
 		return
 	}
-	writeJSON(w, http.StatusOK, f)
+	writeJSON(w, http.StatusOK, p)
 }

 func (s *Server) handleUpdateFlow(w http.ResponseWriter, r *http.Request) {
@@ -205,15 +242,15 @@ func (s *Server) handleUpdateFlow(w http.ResponseWriter, r *http.Request) {
 		writeError(w, http.StatusBadRequest, err)
 		return
 	}
-	s.mu.Lock()
-	defer s.mu.Unlock()
-	f, ok := s.flows[id]
-	if !ok {
-		writeError(w, http.StatusNotFound, errors.New("flow not found"))
+
+	p, err := s.pipelines.Get(r.Context(), id)
+	if err != nil {
+		writeStorageErr(w, err, "pipeline not found")
 		return
 	}
+
 	if body.Name != nil {
-		f.Name = *body.Name
+		p.Name = *body.Name
 	}
 	if len(body.Definition) > 0 {
 		count, _, err := analyzeDefinition(body.Definition)
@@ -221,11 +258,14 @@ func (s *Server) handleUpdateFlow(w http.ResponseWriter, r *http.Request) {
 			writeError(w, http.StatusBadRequest, err)
 			return
 		}
-		f.Definition = body.Definition
-		f.NodeCount = count
+		p.Definition = body.Definition
+		p.NodeCount = count
+	}
+	p.UpdatedAt = time.Now().UTC()
+	if err := s.pipelines.Update(r.Context(), p); err != nil {
+		writeError(w, http.StatusInternalServerError, err)
+		return
 	}
-	f.UpdatedAt = time.Now().UTC()
-	s.flows[id] = f
 	w.WriteHeader(http.StatusNoContent)
 }

@@ -253,9 +293,10 @@ func analyzeDefinition(raw json.RawMessage) (int, string, error) {

 func (s *Server) handleDeleteFlow(w http.ResponseWriter, r *http.Request) {
 	id := r.PathValue("id")
-	s.mu.Lock()
-	delete(s.flows, id)
-	s.mu.Unlock()
+	if err := s.pipelines.Delete(r.Context(), id); err != nil && !errors.Is(err, storage.ErrNotFound) {
+		writeError(w, http.StatusInternalServerError, err)
+		return
+	}
 	w.WriteHeader(http.StatusNoContent)
 }

@@ -289,12 +330,16 @@ func (s *Server) handleExecuteWorkflow(w http.ResponseWriter, r *http.Request) {
 		return
 	}

+	// Pull the originating pipeline ID/name out of the body again so we can
+	// stamp the run record (parseExecuteRequest doesn't expose it).
+	pipelineID, pipelineName := pipelineRefFromBody(rawBody, wf.Name)
+
 	apiKey := r.Header.Get("X-API-Key")
 	ctx, cancel := context.WithTimeout(r.Context(), 10*time.Second)
 	defer cancel()

 	execID, err := s.orch.RunAsync(ctx, &orchestrator.RunRequest{
-		RequestMeta: orchestrator.RequestMeta{APIKey: apiKey},
+		RequestMeta: orchestrator.RequestMeta{APIKey: apiKey, WorkflowID: pipelineID},
 		Workflow:    wf,
 		TriggerData: input,
 	})
@@ -303,12 +348,45 @@ func (s *Server) handleExecuteWorkflow(w http.ResponseWriter, r *http.Request) {
 		return
 	}

+	// Best-effort run record. A failure here shouldn't block the response —
+	// the orchestrator already accepted the workflow.
+	if s.runs != nil {
+		triggerJSON, _ := json.Marshal(input)
+		if err := s.runs.Insert(r.Context(), &storage.Run{
+			ID:           execID,
+			PipelineID:   pipelineID,
+			PipelineName: pipelineName,
+			Status:       "running",
+			StartedAt:    time.Now().UTC(),
+			TriggerData:  triggerJSON,
+		}); err != nil {
+			slog.WarnContext(r.Context(), "run_insert_failed",
+				slog.String("execution_id", execID),
+				slog.Any("error", err),
+			)
+		}
+	}
+
 	writeJSON(w, http.StatusAccepted, map[string]string{
 		"execution_id": execID,
 		"status":       "running",
 	})
 }

+// pipelineRefFromBody peeks at the execute request body to recover the
+// pipeline ID + name for the run record. Best-effort; missing fields are OK.
+func pipelineRefFromBody(body []byte, fallbackName string) (id string, name string) {
+	var probe struct {
+		WorkflowID string `json:"workflow_id"`
+		Workflow   struct {
+			Name string `json:"name"`
+		} `json:"workflow"`
+	}
+	_ = json.Unmarshal(body, &probe)
+	name = firstNonEmpty(probe.Workflow.Name, fallbackName)
+	return probe.WorkflowID, name
+}
+
 // handleResumeExecution resolves a Restate Awakeable so a paused workflow
 // continues. Body: { "awakeable_id": "...", "data": <any> }
 //
@@ -386,6 +464,35 @@ func (s *Server) handleGetExecution(w http.ResponseWriter, r *http.Request) {
 	writeJSON(w, http.StatusOK, status)
 }

+// handleListExecutions returns recent runs from storage. Optional
+// ?pipeline_id= filters by source pipeline; ?limit= caps the page size.
+func (s *Server) handleListExecutions(w http.ResponseWriter, r *http.Request) {
+	if s.runs == nil {
+		writeError(w, http.StatusServiceUnavailable, errors.New("run store not configured"))
+		return
+	}
+	limit := 50
+	if v := r.URL.Query().Get("limit"); v != "" {
+		if n, err := strconv.Atoi(v); err == nil && n > 0 && n <= 500 {
+			limit = n
+		}
+	}
+	var (
+		runs []*storage.Run
+		err  error
+	)
+	if pid := r.URL.Query().Get("pipeline_id"); pid != "" {
+		runs, err = s.runs.ListByPipeline(r.Context(), pid, limit)
+	} else {
+		runs, err = s.runs.List(r.Context(), limit)
+	}
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, err)
+		return
+	}
+	writeJSON(w, http.StatusOK, runs)
+}
+
 // parseExecuteRequest accepts the structured ExecuteRequest shape:
 //   { "workflow": <n8n-JSON object>, "input": [{...}, ...] }
 //   { "workflow_id": "<id>", "input": [...] }
@@ -406,13 +513,13 @@ func (s *Server) parseExecuteRequest(body []byte) (json.RawMessage, []models.Ite
 	}

 	if probe.WorkflowID != "" {
-		s.mu.Lock()
-		f, ok := s.flows[probe.WorkflowID]
-		s.mu.Unlock()
-		if !ok {
+		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+		defer cancel()
+		p, err := s.pipelines.Get(ctx, probe.WorkflowID)
+		if err != nil {
 			return nil, nil, fmt.Errorf("workflow %q not found", probe.WorkflowID)
 		}
-		return f.Definition, input, nil
+		return p.Definition, input, nil
 	}
 	if len(probe.Workflow) > 0 && string(probe.Workflow) != "null" {
 		return probe.Workflow, input, nil
@@ -434,6 +541,16 @@ func writeError(w http.ResponseWriter, status int, err error) {
 	writeJSON(w, status, map[string]string{"error": err.Error()})
 }

+// writeStorageErr maps storage.ErrNotFound to 404 and other errors to 500.
+// notFoundMsg is the user-facing message on 404.
+func writeStorageErr(w http.ResponseWriter, err error, notFoundMsg string) {
+	if errors.Is(err, storage.ErrNotFound) {
+		writeError(w, http.StatusNotFound, errors.New(notFoundMsg))
+		return
+	}
+	writeError(w, http.StatusInternalServerError, err)
+}
+
 func firstNonEmpty(ss ...string) string {
 	for _, s := range ss {
 		if strings.TrimSpace(s) != "" {
@@ -13,10 +13,27 @@ import (

 	"github.com/lyzrai/flow/pkg/models"
 	"github.com/lyzrai/flow/pkg/orchestrator"
+	"github.com/lyzrai/flow/pkg/storage"
 )

+// newTestServer returns a Server with in-memory stores wired up. Tests pass
+// extra fields (Orchestrator, RestateIngressURL) that override the defaults.
+func newTestServer(deps ServerDeps) *Server {
+	mem := storage.NewMemory()
+	if deps.Pipelines == nil {
+		deps.Pipelines = mem.Pipelines()
+	}
+	if deps.Runs == nil {
+		deps.Runs = mem.Runs()
+	}
+	if deps.Agents == nil {
+		deps.Agents = mem.Agents()
+	}
+	return NewServer(deps)
+}
+
 func TestHealth_returns200(t *testing.T) {
-	srv := NewServer(ServerDeps{})
+	srv := newTestServer(ServerDeps{})
 	r := httptest.NewRequest(http.MethodGet, "/api/health", nil)
 	w := httptest.NewRecorder()
 	srv.ServeHTTP(w, r)
@@ -34,7 +51,7 @@ func TestHealth_returns200(t *testing.T) {
 }

 func TestWorkflowCRUD_roundTrip(t *testing.T) {
-	srv := NewServer(ServerDeps{})
+	srv := newTestServer(ServerDeps{})

 	// Create
 	wf := map[string]any{
@@ -132,7 +149,7 @@ func TestWorkflowCRUD_roundTrip(t *testing.T) {
 }

 func TestExecute_requiresOrchestrator(t *testing.T) {
-	srv := NewServer(ServerDeps{})
+	srv := newTestServer(ServerDeps{})
 	body, _ := json.Marshal(map[string]any{
 		"workflow": map[string]any{
 			"name":        "x",
@@ -176,7 +193,7 @@ func (s *stubOrch) GetExecution(_ context.Context, _ string) (*orchestrator.Exec

 func TestExecute_inlineWorkflow_callsOrchestrator(t *testing.T) {
 	orch := &stubOrch{execID: "exec-123"}
-	srv := NewServer(ServerDeps{Orchestrator: orch})
+	srv := newTestServer(ServerDeps{Orchestrator: orch})

 	body := []byte(`{"workflow":{"name":"x","nodes":[{"id":"1","name":"T","type":"n8n-nodes-base.manualTrigger","parameters":{},"position":[0,0]}],"connections":{}},"input":[{"k":"v"}]}`)
 	r := httptest.NewRequest(http.MethodPost, "/api/workflows/execute", bytes.NewReader(body))
@@ -204,7 +221,7 @@ func TestExecute_inlineWorkflow_callsOrchestrator(t *testing.T) {

 func TestExecute_byWorkflowID(t *testing.T) {
 	orch := &stubOrch{execID: "exec-7"}
-	srv := NewServer(ServerDeps{Orchestrator: orch})
+	srv := newTestServer(ServerDeps{Orchestrator: orch})

 	// Pre-create a workflow.
 	body, _ := json.Marshal(map[string]any{
@@ -234,7 +251,7 @@ func TestExecute_byWorkflowID(t *testing.T) {
 }

 func TestExecute_missing_workflow_400(t *testing.T) {
-	srv := NewServer(ServerDeps{Orchestrator: &stubOrch{}})
+	srv := newTestServer(ServerDeps{Orchestrator: &stubOrch{}})
 	body := []byte(`{}`)
 	r := httptest.NewRequest(http.MethodPost, "/api/workflows/execute", bytes.NewReader(body))
 	w := httptest.NewRecorder()
@@ -246,7 +263,7 @@ func TestExecute_missing_workflow_400(t *testing.T) {

 func TestGetExecution_returnsOrchStatus(t *testing.T) {
 	want := &orchestrator.ExecutionStatus{ExecutionID: "abc", Status: "success"}
-	srv := NewServer(ServerDeps{Orchestrator: &stubOrch{statusOut: want}})
+	srv := newTestServer(ServerDeps{Orchestrator: &stubOrch{statusOut: want}})

 	r := httptest.NewRequest(http.MethodGet, "/api/executions/abc", nil)
 	w := httptest.NewRecorder()
@@ -262,7 +279,7 @@ func TestGetExecution_returnsOrchStatus(t *testing.T) {
 }

 func TestGetExecution_notFound(t *testing.T) {
-	srv := NewServer(ServerDeps{Orchestrator: &stubOrch{statusErr: errors.New("nope")}})
+	srv := newTestServer(ServerDeps{Orchestrator: &stubOrch{statusErr: errors.New("nope")}})
 	r := httptest.NewRequest(http.MethodGet, "/api/executions/abc", nil)
 	w := httptest.NewRecorder()
 	srv.ServeHTTP(w, r)
@@ -285,7 +302,7 @@ func TestResume_proxiesToRestate(t *testing.T) {
 	}))
 	defer stub.Close()

-	srv := NewServer(ServerDeps{RestateIngressURL: stub.URL})
+	srv := newTestServer(ServerDeps{RestateIngressURL: stub.URL})
 	body, _ := json.Marshal(map[string]any{
 		"awakeable_id": "sign_xyz",
 		"data":         map[string]any{"approved": true},
@@ -309,7 +326,7 @@ func TestResume_proxiesToRestate(t *testing.T) {
 }

 func TestResume_requiresAwakeableID(t *testing.T) {
-	srv := NewServer(ServerDeps{RestateIngressURL: "http://localhost"})
+	srv := newTestServer(ServerDeps{RestateIngressURL: "http://localhost"})
 	body, _ := json.Marshal(map[string]any{"awakeable_id": "", "data": map[string]any{}})
 	r := httptest.NewRequest(http.MethodPost, "/api/executions/exec-1/resume", bytes.NewReader(body))
 	w := httptest.NewRecorder()
@@ -320,7 +337,7 @@ func TestResume_requiresAwakeableID(t *testing.T) {
 }

 func TestResume_unwiredIngress503(t *testing.T) {
-	srv := NewServer(ServerDeps{}) // no RestateIngressURL
+	srv := newTestServer(ServerDeps{}) // no RestateIngressURL
 	body, _ := json.Marshal(map[string]any{"awakeable_id": "sign_x", "data": map[string]any{}})
 	r := httptest.NewRequest(http.MethodPost, "/api/executions/exec-1/resume", bytes.NewReader(body))
 	w := httptest.NewRecorder()
@@ -332,7 +349,7 @@ func TestResume_unwiredIngress503(t *testing.T) {

 func TestNonAPI_returns404(t *testing.T) {
 	// API server no longer hosts the SPA — the UI is a separate process.
-	srv := NewServer(ServerDeps{})
+	srv := newTestServer(ServerDeps{})
 	for _, path := range []string{"/", "/flows/abc", "/runs/123"} {
 		r := httptest.NewRequest(http.MethodGet, path, nil)
 		w := httptest.NewRecorder()
@@ -344,7 +361,7 @@ func TestNonAPI_returns404(t *testing.T) {
 }

 func TestUnknownAPIRoute_404(t *testing.T) {
-	srv := NewServer(ServerDeps{})
+	srv := newTestServer(ServerDeps{})
 	r := httptest.NewRequest(http.MethodGet, "/api/does-not-exist", nil)
 	w := httptest.NewRecorder()
 	srv.ServeHTTP(w, r)