mirror of
https://github.com/open-gitagent/langship.sh.git
synced 2026-08-03 07:21:04 +02:00
feat: add Gateway page and Runs page with real-time updates
- Implemented a new Gateway page that provides a uniform ingress for deployed agents. - Created a Runs page that lists recent pipeline executions with auto-refresh and error handling. - Added a RunsListener component to handle real-time notifications for new runs via SSE. - Updated the AppSidebar to include links to the new Gateway and Runs pages. - Enhanced FlowNode component to improve status display and styling. - Introduced EmptySection component for consistent empty state presentation. - Updated API utility to include a new endpoint for the runs stream.
This commit is contained in:
patel-lyzr
@@ -434,6 +434,28 @@ func (s *Server) dispatchAgent(ctx context.Context, a *storage.Agent, trigger an
|
||||
})
|
||||
continue
|
||||
}
|
||||
// Hook the log archiver onto the new execution so per-node lines
|
||||
// land in MinIO when each node finishes.
|
||||
startLogArchiver(context.Background(), s.logs, s.events, execID)
|
||||
|
||||
// Broadcast so /runs etc. light up without polling. We extract
|
||||
// `source` from the trigger payload (manual / github_push).
|
||||
source := ""
|
||||
if items, ok := trigger.([]map[string]any); ok && len(items) > 0 {
|
||||
if s, _ := items[0]["source"].(string); s != "" {
|
||||
source = s
|
||||
}
|
||||
}
|
||||
s.runsBus.Publish(RunCreatedEvent{
|
||||
Type: "run_created",
|
||||
ExecutionID: execID,
|
||||
PipelineID: pid,
|
||||
PipelineName: p.Name,
|
||||
AgentID: a.ID,
|
||||
Source: source,
|
||||
StartedAt: time.Now().UTC(),
|
||||
})
|
||||
|
||||
if s.runs != nil {
|
||||
_ = s.runs.Insert(ctx, &storage.Run{
|
||||
ID: execID,
|
||||
|
||||
@@ -0,0 +1,140 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/lyzrai/flow/pkg/engine"
|
||||
"github.com/lyzrai/flow/pkg/logstore"
|
||||
)
|
||||
|
||||
// logArchiver subscribes to the in-memory event bus for a single execution,
|
||||
// buffers `node_log` lines per node, and flushes each node's buffer to the
|
||||
// log store when the node finishes (node_completed / node_error). On the
|
||||
// terminal `done` event it flushes any leftovers and stops.
|
||||
//
|
||||
// One archiver per active execution. They drop themselves from the
|
||||
// archive map when finished.
|
||||
type logArchiver struct {
|
||||
execID string
|
||||
store logstore.Store
|
||||
bus EventSubscriber
|
||||
|
||||
// in-memory buffer per node — the live SSE listener (the UI) reads from
|
||||
// the bus directly; this struct is purely for archive-on-finish.
|
||||
mu sync.Mutex
|
||||
buffers map[string]*bytes.Buffer
|
||||
}
|
||||
|
||||
// startLogArchiver spins up a goroutine that drains the per-exec event bus
|
||||
// into MinIO. Safe to call once per execution; idempotent if the bus is nil.
|
||||
func startLogArchiver(parentCtx context.Context, store logstore.Store, bus EventSubscriber, execID string) {
|
||||
if store == nil || bus == nil || execID == "" {
|
||||
return
|
||||
}
|
||||
ar := &logArchiver{
|
||||
execID: execID,
|
||||
store: store,
|
||||
bus: bus,
|
||||
buffers: map[string]*bytes.Buffer{},
|
||||
}
|
||||
go ar.run(parentCtx)
|
||||
}
|
||||
|
||||
func (a *logArchiver) run(parentCtx context.Context) {
|
||||
ch, cancel := a.bus.Subscribe(a.execID)
|
||||
defer cancel()
|
||||
|
||||
// Detach from the request context that triggered the run; once submitted
|
||||
// we want to keep archiving even if the caller disconnects. Cap with a
|
||||
// per-run deadline so a stuck workflow doesn't leak this goroutine
|
||||
// forever.
|
||||
ctx, cancelCtx := context.WithTimeout(context.Background(), 1*time.Hour)
|
||||
defer cancelCtx()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
a.flushAll(ctx)
|
||||
return
|
||||
case <-parentCtx.Done():
|
||||
// Process is shutting down.
|
||||
a.flushAll(context.Background())
|
||||
return
|
||||
case ev, ok := <-ch:
|
||||
if !ok {
|
||||
a.flushAll(ctx)
|
||||
return
|
||||
}
|
||||
a.handle(ctx, ev)
|
||||
if ev.Type == engine.EventDone {
|
||||
a.flushAll(ctx)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (a *logArchiver) handle(ctx context.Context, ev engine.ExecutionEvent) {
|
||||
switch ev.Type {
|
||||
case engine.EventNodeLog:
|
||||
if ev.Node == "" || ev.Content == "" {
|
||||
return
|
||||
}
|
||||
a.mu.Lock()
|
||||
buf := a.buffers[ev.Node]
|
||||
if buf == nil {
|
||||
buf = &bytes.Buffer{}
|
||||
a.buffers[ev.Node] = buf
|
||||
}
|
||||
// One line per Log event; force a trailing newline so the archived
|
||||
// file is line-oriented and easy to tail.
|
||||
buf.WriteString(strings.TrimRight(ev.Content, "\r\n"))
|
||||
buf.WriteByte('\n')
|
||||
a.mu.Unlock()
|
||||
|
||||
case engine.EventNodeCompleted, engine.EventNodeError:
|
||||
if ev.Node == "" {
|
||||
return
|
||||
}
|
||||
a.flushNode(ctx, ev.Node)
|
||||
}
|
||||
}
|
||||
|
||||
func (a *logArchiver) flushNode(ctx context.Context, node string) {
|
||||
a.mu.Lock()
|
||||
buf := a.buffers[node]
|
||||
if buf == nil || buf.Len() == 0 {
|
||||
a.mu.Unlock()
|
||||
return
|
||||
}
|
||||
data := append([]byte(nil), buf.Bytes()...)
|
||||
delete(a.buffers, node)
|
||||
a.mu.Unlock()
|
||||
|
||||
putCtx, cancel := context.WithTimeout(ctx, 15*time.Second)
|
||||
defer cancel()
|
||||
if err := a.store.Put(putCtx, a.execID, node, data); err != nil {
|
||||
slog.WarnContext(ctx, "log_archive_failed",
|
||||
slog.String("execution_id", a.execID),
|
||||
slog.String("node", node),
|
||||
slog.Any("error", err),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
func (a *logArchiver) flushAll(ctx context.Context) {
|
||||
a.mu.Lock()
|
||||
nodes := make([]string, 0, len(a.buffers))
|
||||
for n := range a.buffers {
|
||||
nodes = append(nodes, n)
|
||||
}
|
||||
a.mu.Unlock()
|
||||
for _, n := range nodes {
|
||||
a.flushNode(ctx, n)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// RunCreatedEvent is the payload broadcast on /api/runs/stream whenever a
|
||||
// new run is dispatched (manual API call or GitHub webhook).
|
||||
type RunCreatedEvent struct {
|
||||
Type string `json:"type"` // "run_created"
|
||||
ExecutionID string `json:"executionId"`
|
||||
PipelineID string `json:"pipelineId,omitempty"`
|
||||
PipelineName string `json:"pipelineName,omitempty"`
|
||||
AgentID string `json:"agentId,omitempty"`
|
||||
Source string `json:"source,omitempty"` // "manual" | "github_push"
|
||||
StartedAt time.Time `json:"startedAt"`
|
||||
}
|
||||
|
||||
// runsBus is a tiny broadcast bus for cross-execution UI events. The SSE
|
||||
// stream handler subscribes; dispatchers publish. All in-memory; one bus
|
||||
// per process.
|
||||
type runsBus struct {
|
||||
mu sync.RWMutex
|
||||
subscribers map[chan RunCreatedEvent]struct{}
|
||||
}
|
||||
|
||||
func newRunsBus() *runsBus {
|
||||
return &runsBus{subscribers: map[chan RunCreatedEvent]struct{}{}}
|
||||
}
|
||||
|
||||
// Publish fans an event out to every active subscriber. Non-blocking — slow
|
||||
// subscribers drop the event so dispatchers never stall.
|
||||
func (b *runsBus) Publish(ev RunCreatedEvent) {
|
||||
b.mu.RLock()
|
||||
subs := make([]chan RunCreatedEvent, 0, len(b.subscribers))
|
||||
for c := range b.subscribers {
|
||||
subs = append(subs, c)
|
||||
}
|
||||
b.mu.RUnlock()
|
||||
for _, c := range subs {
|
||||
select {
|
||||
case c <- ev:
|
||||
default:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Subscribe registers a buffered channel and returns it + an unsubscribe
|
||||
// func that closes the channel exactly once.
|
||||
func (b *runsBus) Subscribe() (<-chan RunCreatedEvent, func()) {
|
||||
ch := make(chan RunCreatedEvent, 16)
|
||||
b.mu.Lock()
|
||||
b.subscribers[ch] = struct{}{}
|
||||
b.mu.Unlock()
|
||||
var once sync.Once
|
||||
cancel := func() {
|
||||
once.Do(func() {
|
||||
b.mu.Lock()
|
||||
delete(b.subscribers, ch)
|
||||
b.mu.Unlock()
|
||||
close(ch)
|
||||
})
|
||||
}
|
||||
return ch, cancel
|
||||
}
|
||||
|
||||
// marshal is a small helper so handlers don't import encoding/json just for
|
||||
// the event payload format.
|
||||
func (e RunCreatedEvent) marshal() ([]byte, error) { return json.Marshal(e) }
|
||||
@@ -21,6 +21,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/lyzrai/flow/pkg/engine"
|
||||
"github.com/lyzrai/flow/pkg/logstore"
|
||||
"github.com/lyzrai/flow/pkg/models"
|
||||
"github.com/lyzrai/flow/pkg/orchestrator"
|
||||
"github.com/lyzrai/flow/pkg/storage"
|
||||
@@ -56,6 +57,11 @@ type ServerDeps struct {
|
||||
// per-node lifecycle events to. The SSE handler subscribes per
|
||||
// execution ID. Nil disables /api/executions/{id}/stream.
|
||||
Events EventSubscriber
|
||||
|
||||
// Logs is the archive backend (MinIO/S3). When set, every dispatched
|
||||
// run starts a background archiver that flushes per-node log buffers
|
||||
// to object storage. Nil disables archiving (live SSE still works).
|
||||
Logs logstore.Store
|
||||
}
|
||||
|
||||
// EventSubscriber is the slice of execevents.MemoryBus the API needs.
|
||||
@@ -77,6 +83,12 @@ type Server struct {
|
||||
runs storage.RunStore
|
||||
agents storage.AgentStore
|
||||
events EventSubscriber
|
||||
logs logstore.Store
|
||||
|
||||
// runsBus broadcasts run_created events to every UI tab subscribed to
|
||||
// /api/runs/stream. Used so a webhook-triggered run shows up live in
|
||||
// the dashboard / runs list / agent detail page without polling.
|
||||
runsBus *runsBus
|
||||
}
|
||||
|
||||
// NewServer constructs an API-only Server. deps.Orchestrator may be nil —
|
||||
@@ -93,6 +105,8 @@ func NewServer(deps ServerDeps) *Server {
|
||||
runs: deps.Runs,
|
||||
agents: deps.Agents,
|
||||
events: deps.Events,
|
||||
logs: deps.Logs,
|
||||
runsBus: newRunsBus(),
|
||||
}
|
||||
s.routes()
|
||||
return s
|
||||
@@ -119,8 +133,10 @@ func (s *Server) routes() {
|
||||
s.mux.HandleFunc("DELETE /api/workflows/{id}", s.handleDeleteFlow)
|
||||
s.mux.HandleFunc("POST /api/workflows/execute", s.handleExecuteWorkflow)
|
||||
s.mux.HandleFunc("GET /api/executions", s.handleListExecutions)
|
||||
s.mux.HandleFunc("GET /api/runs/stream", s.handleRunsStream)
|
||||
s.mux.HandleFunc("GET /api/executions/{id}", s.handleGetExecution)
|
||||
s.mux.HandleFunc("GET /api/executions/{id}/stream", s.handleStreamExecution)
|
||||
s.mux.HandleFunc("GET /api/executions/{id}/logs/{node}", s.handleNodeLog)
|
||||
s.mux.HandleFunc("POST /api/executions/{id}/resume", s.handleResumeExecution)
|
||||
|
||||
// Agents — Langship-style agent registry (git URL + PAT)
|
||||
@@ -362,6 +378,21 @@ func (s *Server) handleExecuteWorkflow(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
// Subscribe the log archiver to this execution. Drains node_log events
|
||||
// off the event bus and flushes per-node buffers to MinIO when each
|
||||
// node completes.
|
||||
startLogArchiver(context.Background(), s.logs, s.events, execID)
|
||||
|
||||
// Broadcast so /runs and other tabs flip to live without polling.
|
||||
s.runsBus.Publish(RunCreatedEvent{
|
||||
Type: "run_created",
|
||||
ExecutionID: execID,
|
||||
PipelineID: pipelineID,
|
||||
PipelineName: pipelineName,
|
||||
Source: "manual",
|
||||
StartedAt: time.Now().UTC(),
|
||||
})
|
||||
|
||||
// Best-effort run record. A failure here shouldn't block the response —
|
||||
// the orchestrator already accepted the workflow.
|
||||
if s.runs != nil {
|
||||
@@ -545,6 +576,83 @@ func (s *Server) handleStreamExecution(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
}
|
||||
|
||||
// handleRunsStream is a global SSE feed of run_created events. UI tabs
|
||||
// subscribe once and react to runs from any source (manual trigger, agent
|
||||
// trigger, GitHub push). Heartbeats every 15s.
|
||||
func (s *Server) handleRunsStream(w http.ResponseWriter, r *http.Request) {
|
||||
flusher, ok := w.(http.Flusher)
|
||||
if !ok {
|
||||
writeError(w, http.StatusInternalServerError, errors.New("streaming not supported"))
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "text/event-stream")
|
||||
w.Header().Set("Cache-Control", "no-cache")
|
||||
w.Header().Set("Connection", "keep-alive")
|
||||
w.Header().Set("X-Accel-Buffering", "no")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
|
||||
ch, cancel := s.runsBus.Subscribe()
|
||||
defer cancel()
|
||||
|
||||
_, _ = fmt.Fprint(w, "event: open\ndata: {}\n\n")
|
||||
flusher.Flush()
|
||||
|
||||
heartbeat := time.NewTicker(15 * time.Second)
|
||||
defer heartbeat.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-r.Context().Done():
|
||||
return
|
||||
case <-heartbeat.C:
|
||||
_, _ = fmt.Fprint(w, ": heartbeat\n\n")
|
||||
flusher.Flush()
|
||||
case ev, ok := <-ch:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
payload, err := ev.marshal()
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
_, _ = fmt.Fprintf(w, "data: %s\n\n", payload)
|
||||
flusher.Flush()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// handleNodeLog streams the archived log for one node of an execution.
|
||||
// Reads through the configured logstore (MinIO/S3 in prod) so the browser
|
||||
// never talks to the object store directly. text/plain.
|
||||
func (s *Server) handleNodeLog(w http.ResponseWriter, r *http.Request) {
|
||||
if s.logs == nil {
|
||||
writeError(w, http.StatusServiceUnavailable, errors.New("log archive not configured"))
|
||||
return
|
||||
}
|
||||
id := r.PathValue("id")
|
||||
node := r.PathValue("node")
|
||||
if id == "" || node == "" {
|
||||
writeError(w, http.StatusBadRequest, errors.New("execution id and node required"))
|
||||
return
|
||||
}
|
||||
rc, err := s.logs.Get(r.Context(), id, node)
|
||||
if err != nil {
|
||||
if errors.Is(err, logstore.ErrNotFound) {
|
||||
writeError(w, http.StatusNotFound, errors.New("log not found (run may still be in progress)"))
|
||||
return
|
||||
}
|
||||
writeError(w, http.StatusBadGateway, fmt.Errorf("log fetch: %w", err))
|
||||
return
|
||||
}
|
||||
defer rc.Close()
|
||||
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
|
||||
w.Header().Set("Cache-Control", "no-cache")
|
||||
if _, err := io.Copy(w, rc); err != nil {
|
||||
// Connection may have dropped; nothing to do.
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// handleListExecutions returns recent runs from storage. Optional
|
||||
// ?pipeline_id= filters by source pipeline; ?limit= caps the page size.
|
||||
func (s *Server) handleListExecutions(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
Reference in New Issue
Block a user