mirror of
https://github.com/open-gitagent/langship.sh.git
synced 2026-08-03 07:21:04 +02:00
patel-lyzr
6b1a13ecdc
- Added environment creation and editing pages with forms for name and description. - Integrated environment listing with options to edit and delete environments. - Updated agent detail page to manage environments followed by agents. - Enhanced API to support environment operations including listing, creating, updating, and deleting environments. - Refactored related components and state management for improved clarity and functionality.
597 lines
16 KiB
Go
597 lines
16 KiB
Go
package storage
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"time"
|
|
|
|
"go.mongodb.org/mongo-driver/v2/bson"
|
|
"go.mongodb.org/mongo-driver/v2/mongo"
|
|
"go.mongodb.org/mongo-driver/v2/mongo/options"
|
|
)
|
|
|
|
// Mongo holds the connected client + database handle. Construct via NewMongo
|
|
// and Close on shutdown.
|
|
type Mongo struct {
|
|
client *mongo.Client
|
|
db *mongo.Database
|
|
}
|
|
|
|
// NewMongo dials Mongo with a 10s connect+ping timeout and ensures indexes
|
|
// exist. Returns an error if the URI is unreachable.
|
|
func NewMongo(ctx context.Context, uri, dbName string) (*Mongo, error) {
|
|
if uri == "" {
|
|
return nil, errors.New("MONGO_URI is required")
|
|
}
|
|
if dbName == "" {
|
|
dbName = "flow"
|
|
}
|
|
|
|
dialCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
|
defer cancel()
|
|
|
|
client, err := mongo.Connect(options.Client().ApplyURI(uri))
|
|
if err != nil {
|
|
return nil, fmt.Errorf("mongo connect: %w", err)
|
|
}
|
|
if err := client.Ping(dialCtx, nil); err != nil {
|
|
_ = client.Disconnect(context.Background())
|
|
return nil, fmt.Errorf("mongo ping %q: %w", uri, err)
|
|
}
|
|
|
|
m := &Mongo{client: client, db: client.Database(dbName)}
|
|
if err := m.ensureIndexes(ctx); err != nil {
|
|
return nil, fmt.Errorf("ensure indexes: %w", err)
|
|
}
|
|
return m, nil
|
|
}
|
|
|
|
// Close disconnects the client. Safe to call multiple times.
|
|
func (m *Mongo) Close(ctx context.Context) error {
|
|
if m == nil || m.client == nil {
|
|
return nil
|
|
}
|
|
return m.client.Disconnect(ctx)
|
|
}
|
|
|
|
// Pipelines returns the PipelineStore backed by this Mongo connection.
|
|
func (m *Mongo) Pipelines() PipelineStore { return &mongoPipelines{coll: m.db.Collection("pipelines")} }
|
|
|
|
// Runs returns the RunStore backed by this Mongo connection.
|
|
func (m *Mongo) Runs() RunStore { return &mongoRuns{coll: m.db.Collection("runs")} }
|
|
|
|
// Agents returns the AgentStore backed by this Mongo connection.
|
|
func (m *Mongo) Agents() AgentStore { return &mongoAgents{coll: m.db.Collection("agents")} }
|
|
|
|
// Credentials returns the global CredentialStore backed by this Mongo
|
|
// connection. Per-agent overrides live on agent.Credentials and are not
|
|
// persisted here; this collection is the org-wide pool.
|
|
func (m *Mongo) Credentials() CredentialStore {
|
|
return &mongoCredentials{coll: m.db.Collection("credentials")}
|
|
}
|
|
|
|
// Environments returns the global EnvironmentStore backed by this Mongo
|
|
// connection.
|
|
func (m *Mongo) Environments() EnvironmentStore {
|
|
return &mongoEnvironments{coll: m.db.Collection("environments")}
|
|
}
|
|
|
|
func (m *Mongo) ensureIndexes(ctx context.Context) error {
|
|
if _, err := m.db.Collection("pipelines").Indexes().CreateMany(ctx, []mongo.IndexModel{
|
|
{Keys: bson.D{{Key: "updated_at", Value: -1}}},
|
|
}); err != nil {
|
|
return fmt.Errorf("pipelines indexes: %w", err)
|
|
}
|
|
if _, err := m.db.Collection("runs").Indexes().CreateMany(ctx, []mongo.IndexModel{
|
|
{Keys: bson.D{{Key: "pipeline_id", Value: 1}, {Key: "started_at", Value: -1}}},
|
|
{Keys: bson.D{{Key: "started_at", Value: -1}}},
|
|
}); err != nil {
|
|
return fmt.Errorf("runs indexes: %w", err)
|
|
}
|
|
if _, err := m.db.Collection("agents").Indexes().CreateMany(ctx, []mongo.IndexModel{
|
|
{Keys: bson.D{{Key: "updated_at", Value: -1}}},
|
|
}); err != nil {
|
|
return fmt.Errorf("agents indexes: %w", err)
|
|
}
|
|
if _, err := m.db.Collection("credentials").Indexes().CreateMany(ctx, []mongo.IndexModel{
|
|
{Keys: bson.D{{Key: "name", Value: 1}}, Options: options.Index().SetUnique(true)},
|
|
{Keys: bson.D{{Key: "updated_at", Value: -1}}},
|
|
}); err != nil {
|
|
return fmt.Errorf("credentials indexes: %w", err)
|
|
}
|
|
if _, err := m.db.Collection("environments").Indexes().CreateMany(ctx, []mongo.IndexModel{
|
|
{Keys: bson.D{{Key: "name", Value: 1}}, Options: options.Index().SetUnique(true)},
|
|
{Keys: bson.D{{Key: "updated_at", Value: -1}}},
|
|
}); err != nil {
|
|
return fmt.Errorf("environments indexes: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// --- pipelines ------------------------------------------------------------
|
|
|
|
type mongoPipelines struct{ coll *mongo.Collection }
|
|
|
|
// pipelineDoc swaps json.RawMessage for bson.Raw so the n8n definition stays
|
|
// queryable and round-trips cleanly through Mongo.
|
|
type pipelineDoc struct {
|
|
ID string `bson:"_id"`
|
|
Name string `bson:"name"`
|
|
Definition bson.Raw `bson:"definition"`
|
|
NodeCount int `bson:"node_count"`
|
|
Status string `bson:"status,omitempty"`
|
|
CreatedAt time.Time `bson:"created_at"`
|
|
UpdatedAt time.Time `bson:"updated_at"`
|
|
}
|
|
|
|
func (s *mongoPipelines) Create(ctx context.Context, p *Pipeline) error {
|
|
def, err := jsonToBSON(p.Definition)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
_, err = s.coll.InsertOne(ctx, pipelineDoc{
|
|
ID: p.ID,
|
|
Name: p.Name,
|
|
Definition: def,
|
|
NodeCount: p.NodeCount,
|
|
Status: p.Status,
|
|
CreatedAt: p.CreatedAt,
|
|
UpdatedAt: p.UpdatedAt,
|
|
})
|
|
return err
|
|
}
|
|
|
|
func (s *mongoPipelines) Get(ctx context.Context, id string) (*Pipeline, error) {
|
|
var doc pipelineDoc
|
|
if err := s.coll.FindOne(ctx, bson.M{"_id": id}).Decode(&doc); err != nil {
|
|
if errors.Is(err, mongo.ErrNoDocuments) {
|
|
return nil, ErrNotFound
|
|
}
|
|
return nil, err
|
|
}
|
|
return docToPipeline(&doc)
|
|
}
|
|
|
|
func (s *mongoPipelines) Update(ctx context.Context, p *Pipeline) error {
|
|
set := bson.M{
|
|
"name": p.Name,
|
|
"node_count": p.NodeCount,
|
|
"updated_at": p.UpdatedAt,
|
|
}
|
|
if p.Status != "" {
|
|
set["status"] = p.Status
|
|
}
|
|
if len(p.Definition) > 0 {
|
|
def, err := jsonToBSON(p.Definition)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
set["definition"] = def
|
|
}
|
|
res, err := s.coll.UpdateByID(ctx, p.ID, bson.M{"$set": set})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if res.MatchedCount == 0 {
|
|
return ErrNotFound
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s *mongoPipelines) Delete(ctx context.Context, id string) error {
|
|
res, err := s.coll.DeleteOne(ctx, bson.M{"_id": id})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if res.DeletedCount == 0 {
|
|
return ErrNotFound
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s *mongoPipelines) List(ctx context.Context) ([]*Pipeline, error) {
|
|
cur, err := s.coll.Find(ctx, bson.M{}, options.Find().SetSort(bson.D{{Key: "updated_at", Value: -1}}))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer cur.Close(ctx)
|
|
|
|
var docs []pipelineDoc
|
|
if err := cur.All(ctx, &docs); err != nil {
|
|
return nil, err
|
|
}
|
|
out := make([]*Pipeline, 0, len(docs))
|
|
for i := range docs {
|
|
p, err := docToPipeline(&docs[i])
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
out = append(out, p)
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
func docToPipeline(d *pipelineDoc) (*Pipeline, error) {
|
|
def, err := bsonToJSON(d.Definition)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return &Pipeline{
|
|
ID: d.ID,
|
|
Name: d.Name,
|
|
Definition: def,
|
|
NodeCount: d.NodeCount,
|
|
Status: d.Status,
|
|
CreatedAt: d.CreatedAt,
|
|
UpdatedAt: d.UpdatedAt,
|
|
}, nil
|
|
}
|
|
|
|
// --- runs ----------------------------------------------------------------
|
|
|
|
type mongoRuns struct{ coll *mongo.Collection }
|
|
|
|
type runDoc struct {
|
|
ID string `bson:"_id"`
|
|
PipelineID string `bson:"pipeline_id"`
|
|
PipelineName string `bson:"pipeline_name,omitempty"`
|
|
Status string `bson:"status"`
|
|
StartedAt time.Time `bson:"started_at"`
|
|
FinishedAt *time.Time `bson:"finished_at,omitempty"`
|
|
TriggerData bson.Raw `bson:"trigger_data,omitempty"`
|
|
Outputs bson.Raw `bson:"outputs,omitempty"`
|
|
NodeOutputs bson.Raw `bson:"node_outputs,omitempty"`
|
|
Errors []string `bson:"errors,omitempty"`
|
|
}
|
|
|
|
func (s *mongoRuns) Insert(ctx context.Context, r *Run) error {
|
|
td, err := jsonToBSON(r.TriggerData)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
_, err = s.coll.InsertOne(ctx, runDoc{
|
|
ID: r.ID,
|
|
PipelineID: r.PipelineID,
|
|
PipelineName: r.PipelineName,
|
|
Status: r.Status,
|
|
StartedAt: r.StartedAt,
|
|
TriggerData: td,
|
|
})
|
|
return err
|
|
}
|
|
|
|
func (s *mongoRuns) UpdateStatus(ctx context.Context, id, status string) error {
|
|
_, err := s.coll.UpdateByID(ctx, id, bson.M{"$set": bson.M{"status": status}})
|
|
return err
|
|
}
|
|
|
|
func (s *mongoRuns) Complete(ctx context.Context, id, status string, outputs, nodeOutputs json.RawMessage, errMsg string) error {
|
|
out, err := jsonToBSON(outputs)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
nodeOut, err := jsonToBSON(nodeOutputs)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
now := time.Now().UTC()
|
|
set := bson.M{
|
|
"status": status,
|
|
"finished_at": now,
|
|
}
|
|
if out != nil {
|
|
set["outputs"] = out
|
|
}
|
|
if nodeOut != nil {
|
|
set["node_outputs"] = nodeOut
|
|
}
|
|
if errMsg != "" {
|
|
set["errors"] = []string{errMsg}
|
|
}
|
|
_, err = s.coll.UpdateByID(ctx, id, bson.M{"$set": set})
|
|
return err
|
|
}
|
|
|
|
func (s *mongoRuns) Get(ctx context.Context, id string) (*Run, error) {
|
|
var d runDoc
|
|
if err := s.coll.FindOne(ctx, bson.M{"_id": id}).Decode(&d); err != nil {
|
|
if errors.Is(err, mongo.ErrNoDocuments) {
|
|
return nil, ErrNotFound
|
|
}
|
|
return nil, err
|
|
}
|
|
return docToRun(&d)
|
|
}
|
|
|
|
func (s *mongoRuns) ListByPipeline(ctx context.Context, pipelineID string, limit int) ([]*Run, error) {
|
|
if limit <= 0 {
|
|
limit = 50
|
|
}
|
|
cur, err := s.coll.Find(ctx, bson.M{"pipeline_id": pipelineID},
|
|
options.Find().SetSort(bson.D{{Key: "started_at", Value: -1}}).SetLimit(int64(limit)))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer cur.Close(ctx)
|
|
return collectRuns(ctx, cur)
|
|
}
|
|
|
|
func (s *mongoRuns) List(ctx context.Context, limit int) ([]*Run, error) {
|
|
if limit <= 0 {
|
|
limit = 50
|
|
}
|
|
cur, err := s.coll.Find(ctx, bson.M{},
|
|
options.Find().SetSort(bson.D{{Key: "started_at", Value: -1}}).SetLimit(int64(limit)))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer cur.Close(ctx)
|
|
return collectRuns(ctx, cur)
|
|
}
|
|
|
|
func collectRuns(ctx context.Context, cur *mongo.Cursor) ([]*Run, error) {
|
|
var docs []runDoc
|
|
if err := cur.All(ctx, &docs); err != nil {
|
|
return nil, err
|
|
}
|
|
out := make([]*Run, 0, len(docs))
|
|
for i := range docs {
|
|
r, err := docToRun(&docs[i])
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
out = append(out, r)
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
func docToRun(d *runDoc) (*Run, error) {
|
|
td, err := bsonToJSON(d.TriggerData)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
out, err := bsonToJSON(d.Outputs)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
nodeOut, err := bsonToJSON(d.NodeOutputs)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return &Run{
|
|
ID: d.ID,
|
|
PipelineID: d.PipelineID,
|
|
PipelineName: d.PipelineName,
|
|
Status: d.Status,
|
|
StartedAt: d.StartedAt,
|
|
FinishedAt: d.FinishedAt,
|
|
TriggerData: td,
|
|
Outputs: out,
|
|
NodeOutputs: nodeOut,
|
|
Errors: d.Errors,
|
|
}, nil
|
|
}
|
|
|
|
// --- json/bson conversions -----------------------------------------------
|
|
|
|
// jsonToBSON converts arbitrary JSON to a bson.Raw document so Mongo stores
|
|
// it as a real subdocument (queryable, indexable) rather than an opaque string.
|
|
// Returns nil for empty/null input.
|
|
func jsonToBSON(raw json.RawMessage) (bson.Raw, error) {
|
|
if len(raw) == 0 || string(raw) == "null" {
|
|
return nil, nil
|
|
}
|
|
var v any
|
|
if err := json.Unmarshal(raw, &v); err != nil {
|
|
return nil, fmt.Errorf("jsonToBSON: %w", err)
|
|
}
|
|
// Mongo top-level docs must be objects; wrap scalars/arrays under a key.
|
|
if _, isObj := v.(map[string]any); !isObj {
|
|
v = map[string]any{"_value": v}
|
|
}
|
|
b, err := bson.Marshal(v)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("bson marshal: %w", err)
|
|
}
|
|
return b, nil
|
|
}
|
|
|
|
func bsonToJSON(r bson.Raw) (json.RawMessage, error) {
|
|
if len(r) == 0 {
|
|
return nil, nil
|
|
}
|
|
var v any
|
|
if err := bson.Unmarshal(r, &v); err != nil {
|
|
return nil, fmt.Errorf("bson unmarshal: %w", err)
|
|
}
|
|
if m, ok := v.(map[string]any); ok {
|
|
if inner, hasWrap := m["_value"]; hasWrap && len(m) == 1 {
|
|
v = inner
|
|
}
|
|
}
|
|
b, err := json.Marshal(v)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("json marshal: %w", err)
|
|
}
|
|
return b, nil
|
|
}
|
|
|
|
// --- agents ---------------------------------------------------------------
|
|
|
|
type mongoAgents struct{ coll *mongo.Collection }
|
|
|
|
func (s *mongoAgents) Create(ctx context.Context, a *Agent) error {
|
|
_, err := s.coll.InsertOne(ctx, a)
|
|
return err
|
|
}
|
|
|
|
func (s *mongoAgents) Update(ctx context.Context, a *Agent) error {
|
|
res, err := s.coll.ReplaceOne(ctx, bson.M{"_id": a.ID}, a)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if res.MatchedCount == 0 {
|
|
return ErrNotFound
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s *mongoAgents) Get(ctx context.Context, id string) (*Agent, error) {
|
|
var a Agent
|
|
if err := s.coll.FindOne(ctx, bson.M{"_id": id}).Decode(&a); err != nil {
|
|
if errors.Is(err, mongo.ErrNoDocuments) {
|
|
return nil, ErrNotFound
|
|
}
|
|
return nil, err
|
|
}
|
|
return &a, nil
|
|
}
|
|
|
|
func (s *mongoAgents) Delete(ctx context.Context, id string) error {
|
|
res, err := s.coll.DeleteOne(ctx, bson.M{"_id": id})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if res.DeletedCount == 0 {
|
|
return ErrNotFound
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s *mongoAgents) List(ctx context.Context) ([]*Agent, error) {
|
|
cur, err := s.coll.Find(ctx, bson.M{}, options.Find().SetSort(bson.D{{Key: "updated_at", Value: -1}}))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer cur.Close(ctx)
|
|
var out []*Agent
|
|
if err := cur.All(ctx, &out); err != nil {
|
|
return nil, err
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// --- credentials (global pool) -------------------------------------------
|
|
|
|
type mongoCredentials struct{ coll *mongo.Collection }
|
|
|
|
func (s *mongoCredentials) Create(ctx context.Context, c *Credential) error {
|
|
if _, err := s.coll.InsertOne(ctx, c); err != nil {
|
|
// Surface the duplicate-name index violation as a typed error so
|
|
// the API layer can return 409 instead of a generic 500.
|
|
if mongo.IsDuplicateKeyError(err) {
|
|
return ErrAlreadyExists
|
|
}
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s *mongoCredentials) GetByName(ctx context.Context, name string) (*Credential, error) {
|
|
var c Credential
|
|
if err := s.coll.FindOne(ctx, bson.M{"name": name}).Decode(&c); err != nil {
|
|
if errors.Is(err, mongo.ErrNoDocuments) {
|
|
return nil, ErrNotFound
|
|
}
|
|
return nil, err
|
|
}
|
|
return &c, nil
|
|
}
|
|
|
|
func (s *mongoCredentials) Update(ctx context.Context, c *Credential) error {
|
|
res, err := s.coll.ReplaceOne(ctx, bson.M{"name": c.Name}, c)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if res.MatchedCount == 0 {
|
|
return ErrNotFound
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s *mongoCredentials) Delete(ctx context.Context, name string) error {
|
|
res, err := s.coll.DeleteOne(ctx, bson.M{"name": name})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if res.DeletedCount == 0 {
|
|
return ErrNotFound
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s *mongoCredentials) List(ctx context.Context) ([]*Credential, error) {
|
|
cur, err := s.coll.Find(ctx, bson.M{}, options.Find().SetSort(bson.D{{Key: "updated_at", Value: -1}}))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer cur.Close(ctx)
|
|
var out []*Credential
|
|
if err := cur.All(ctx, &out); err != nil {
|
|
return nil, err
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// --- environments (global) -----------------------------------------------
|
|
|
|
type mongoEnvironments struct{ coll *mongo.Collection }
|
|
|
|
func (s *mongoEnvironments) Create(ctx context.Context, e *Environment) error {
|
|
if _, err := s.coll.InsertOne(ctx, e); err != nil {
|
|
if mongo.IsDuplicateKeyError(err) {
|
|
return ErrAlreadyExists
|
|
}
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s *mongoEnvironments) GetByName(ctx context.Context, name string) (*Environment, error) {
|
|
var e Environment
|
|
if err := s.coll.FindOne(ctx, bson.M{"name": name}).Decode(&e); err != nil {
|
|
if errors.Is(err, mongo.ErrNoDocuments) {
|
|
return nil, ErrNotFound
|
|
}
|
|
return nil, err
|
|
}
|
|
return &e, nil
|
|
}
|
|
|
|
func (s *mongoEnvironments) Update(ctx context.Context, e *Environment) error {
|
|
res, err := s.coll.ReplaceOne(ctx, bson.M{"name": e.Name}, e)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if res.MatchedCount == 0 {
|
|
return ErrNotFound
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s *mongoEnvironments) Delete(ctx context.Context, name string) error {
|
|
res, err := s.coll.DeleteOne(ctx, bson.M{"name": name})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if res.DeletedCount == 0 {
|
|
return ErrNotFound
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s *mongoEnvironments) List(ctx context.Context) ([]*Environment, error) {
|
|
cur, err := s.coll.Find(ctx, bson.M{}, options.Find().SetSort(bson.D{{Key: "name", Value: 1}}))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer cur.Close(ctx)
|
|
var out []*Environment
|
|
if err := cur.All(ctx, &out); err != nil {
|
|
return nil, err
|
|
}
|
|
return out, nil
|
|
}
|