mirror of
https://github.com/bleak-ai/gcontext.git
synced 2026-08-11 13:19:23 +02:00
Add workflow lifecycle: search, install, check, update (v0.7.0)
Extract registry pipeline into registry.py with .template.yaml manifest (per-file SHA256 hashes) for three-way check/update. New workflow MCP tool with 4 actions (search, install, check, update). CLI commands: gcontext update and gcontext search. Hide .template.yaml from list_dir/grep/walk. Re-register module commands on update. Build script for registry.json. 113 tests (24 new in test_workflow_tool.py). Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.6
parent
9128beebd8
commit
c5b34103d3
@@ -166,6 +166,8 @@ Developing the dashboard itself needs node: `make web-dev` runs a Vite dev serve
|
||||
| `gcontext status [dir]` | Server state, connected clients, state overview |
|
||||
| `gcontext connect [client]` | Connection steps for claude, desktop, codex, cursor |
|
||||
| `gcontext add <id>` | Install a workflow from the registry repo ([github.com/bleak-ai/workflows](https://github.com/bleak-ai/workflows)) or from any public GitHub repo folder via `gcontext add <github-url>` |
|
||||
| `gcontext update <id>` | Update an installed workflow from the registry (three-way merge: keeps your local changes, writes `.new` files on conflicts) |
|
||||
| `gcontext search [query]` | Search the workflow registry by name, description, or tags |
|
||||
| `gcontext share <path>` | Validate a workflow template folder and print the steps to submit it as a pull request to the registry |
|
||||
| `gcontext context [dir]` | Print the context ledger |
|
||||
|
||||
|
||||
+3
-1
@@ -130,7 +130,9 @@ Never ships, generated locally at setup and use:
|
||||
- the user's own run folders in `runs/`
|
||||
- every personalized file: configs, credentials references, scripts bound to the user's systems, playbooks learned from the user's own work
|
||||
|
||||
Installs are snapshots. The user's copy is theirs: personalized, growing, never overwritten by an update. `gcontext add` on an existing module warns and stops instead of overwriting.
|
||||
On install, `gcontext add` writes a `.template.yaml` file inside the module. It records per-file SHA256 hashes of every shipped file, the registry source, and the install ref. This manifest is hidden from `list_dir`, `grep`, and resource listings (same policy as `.git`), but stays readable by explicit path. `gcontext update <id>` (or the `workflow` tool's update action) uses it to pull upstream changes without touching personalized files: unchanged-locally files get the upstream version, locally-modified files are kept, files changed on both sides get the upstream version written as `<file>.new` for the agent to merge. Your runs, insights, and personal state are never in the manifest and are never touched.
|
||||
|
||||
`gcontext add` on an existing module warns and stops instead of overwriting.
|
||||
|
||||
## The example run
|
||||
|
||||
|
||||
+58
-205
@@ -1,21 +1,20 @@
|
||||
"""gcontext CLI. One server you start, clients connect to its URL. State is files."""
|
||||
|
||||
import argparse
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import socket
|
||||
import sys
|
||||
import tarfile
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from pathlib import Path, PurePosixPath
|
||||
from pathlib import Path
|
||||
|
||||
from . import __version__
|
||||
from . import exec as exec_mod
|
||||
from . import ledger as ledger_mod
|
||||
from . import registry as registry_mod
|
||||
from . import secrets as secrets_mod
|
||||
from . import server
|
||||
from . import state
|
||||
@@ -28,11 +27,6 @@ RESET = "\033[0m"
|
||||
|
||||
DEFAULT_PORT = 4242
|
||||
|
||||
# GitHub registry: "owner/repo@ref" or a full "https://..." URL to a .tar.gz.
|
||||
# The env var GCONTEXT_REGISTRY accepts both forms. A full URL is useful for
|
||||
# tests: serve a local tarball over HTTP and point the env var at it.
|
||||
DEFAULT_REGISTRY = "bleak-ai/workflows@main"
|
||||
|
||||
STATUS_COLOR = {
|
||||
"loaded": GREEN,
|
||||
"on demand": DIM,
|
||||
@@ -415,210 +409,23 @@ def cmd_context(args):
|
||||
ID_RE = re.compile(r"^[a-z0-9][a-z0-9-]*$")
|
||||
|
||||
|
||||
def _parse_registry() -> str:
|
||||
"""Return the tarball URL for the configured registry.
|
||||
|
||||
GCONTEXT_REGISTRY accepts two forms:
|
||||
- "owner/repo@ref" -> fetches from GitHub codeload
|
||||
- "https://..." -> used as-is (for tests serving a local tarball)
|
||||
"""
|
||||
reg = os.environ.get("GCONTEXT_REGISTRY", DEFAULT_REGISTRY)
|
||||
if reg.startswith("http://") or reg.startswith("https://"):
|
||||
return reg
|
||||
return _codeload_url(reg)
|
||||
|
||||
|
||||
def _codeload_url(spec: str) -> str:
|
||||
"""Build https://codeload.github.com/<owner>/<repo>/tar.gz/refs/heads/<ref>."""
|
||||
if "@" in spec:
|
||||
repo_part, ref = spec.rsplit("@", 1)
|
||||
else:
|
||||
repo_part, ref = spec, "main"
|
||||
return f"https://codeload.github.com/{repo_part}/tar.gz/refs/heads/{ref}"
|
||||
|
||||
|
||||
def _download_tarball(url: str) -> tarfile.TarFile:
|
||||
"""Download a tarball into memory and return an open TarFile. Exits on failure."""
|
||||
try:
|
||||
with urllib.request.urlopen(url, timeout=30) as resp:
|
||||
data = resp.read()
|
||||
except (urllib.error.HTTPError, urllib.error.URLError, OSError):
|
||||
print("Error: could not reach GitHub.", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
return tarfile.open(fileobj=io.BytesIO(data), mode="r:gz")
|
||||
|
||||
|
||||
def _extract_files(tf: tarfile.TarFile, subpath: str = "") -> list[dict]:
|
||||
"""Extract regular files from the tarball into [{path, content}].
|
||||
|
||||
The first path component (the repo-ref prefix GitHub adds) is stripped
|
||||
generically. If subpath is given, only members under that prefix are
|
||||
returned, with the prefix removed. Symlinks and non-regular files are
|
||||
skipped. Non-UTF-8 files emit a warning to stderr and are skipped.
|
||||
"""
|
||||
files = []
|
||||
for member in tf.getmembers():
|
||||
if not member.isfile():
|
||||
continue
|
||||
if member.issym() or member.islnk():
|
||||
continue
|
||||
parts = PurePosixPath(member.name).parts
|
||||
if len(parts) < 2:
|
||||
continue
|
||||
# Strip the first component (e.g. "workflows-main/")
|
||||
rel = str(PurePosixPath(*parts[1:]))
|
||||
if subpath:
|
||||
norm = subpath.rstrip("/") + "/"
|
||||
if not (rel + "/").startswith(norm) and rel != subpath.rstrip("/"):
|
||||
continue
|
||||
rel = rel[len(norm):] if rel.startswith(norm) else ""
|
||||
if not rel:
|
||||
continue
|
||||
try:
|
||||
raw = tf.extractfile(member)
|
||||
if raw is None:
|
||||
continue
|
||||
content = raw.read().decode("utf-8")
|
||||
except (UnicodeDecodeError, ValueError):
|
||||
print(f"Skipping {rel}: not a text file.", file=sys.stderr)
|
||||
continue
|
||||
files.append({"path": rel, "content": content})
|
||||
return files
|
||||
|
||||
|
||||
def _parse_github_url(url: str) -> tuple[str, str, str]:
|
||||
"""Parse a GitHub URL into (owner/repo, ref, subpath).
|
||||
|
||||
Accepted forms:
|
||||
https://github.com/owner/repo
|
||||
https://github.com/owner/repo/tree/ref
|
||||
https://github.com/owner/repo/tree/ref/sub/path
|
||||
github.com/owner/repo (no scheme)
|
||||
"""
|
||||
cleaned = url
|
||||
if cleaned.startswith("github.com/"):
|
||||
cleaned = "https://" + cleaned
|
||||
# Remove scheme + host
|
||||
path = cleaned.split("github.com/", 1)[1] if "github.com/" in cleaned else ""
|
||||
segments = path.strip("/").split("/")
|
||||
if len(segments) < 2:
|
||||
print(f"Error: cannot parse GitHub URL: {url}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
owner_repo = f"{segments[0]}/{segments[1]}"
|
||||
ref = "main"
|
||||
subpath = ""
|
||||
if len(segments) > 3 and segments[2] == "tree":
|
||||
ref = segments[3]
|
||||
if len(segments) > 4:
|
||||
subpath = "/".join(segments[4:])
|
||||
return owner_repo, ref, subpath
|
||||
|
||||
|
||||
def fetch_workflow_by_id(workflow_id: str) -> list[dict]:
|
||||
"""Fetch a workflow by id from the configured registry. Returns [{path, content}]."""
|
||||
url = _parse_registry()
|
||||
tf = _download_tarball(url)
|
||||
all_files = _extract_files(tf)
|
||||
# Find files under the top-level folder matching the id
|
||||
prefix = workflow_id + "/"
|
||||
matched = []
|
||||
for f in all_files:
|
||||
if f["path"].startswith(prefix):
|
||||
matched.append({"path": f["path"][len(prefix):], "content": f["content"]})
|
||||
elif f["path"] == workflow_id:
|
||||
# single file at top level (unlikely but handle it)
|
||||
matched.append({"path": f["path"], "content": f["content"]})
|
||||
if not matched:
|
||||
print(f"Error: no workflow '{workflow_id}' found in the registry.", file=sys.stderr)
|
||||
print("Browse available workflows:", file=sys.stderr)
|
||||
print(" https://github.com/bleak-ai/workflows", file=sys.stderr)
|
||||
print(" https://gcontext.ai/workflows/", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
return matched
|
||||
|
||||
|
||||
def fetch_workflow_by_url(url: str) -> list[dict]:
|
||||
"""Fetch a workflow from a GitHub repo URL. Returns [{path, content}].
|
||||
|
||||
Accepts https://github.com/<owner>/<repo>[/tree/<ref>[/<subpath>]] or
|
||||
a direct http(s):// URL to a .tar.gz (useful for testing).
|
||||
"""
|
||||
if "github.com/" in url or url.startswith("github.com/"):
|
||||
owner_repo, ref, subpath = _parse_github_url(url)
|
||||
tarball_url = _codeload_url(f"{owner_repo}@{ref}")
|
||||
else:
|
||||
# Direct tarball URL (e.g. local test server)
|
||||
tarball_url = url
|
||||
subpath = ""
|
||||
tf = _download_tarball(tarball_url)
|
||||
files = _extract_files(tf, subpath=subpath)
|
||||
if not files:
|
||||
print(f"Error: no files found at {url}.", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
return files
|
||||
|
||||
|
||||
def validate_bundle(files) -> dict:
|
||||
"""Check paths and the index.md manifest; return the parsed frontmatter.
|
||||
|
||||
Raises ValueError on any problem. Runs entirely in memory so a bad
|
||||
bundle never leaves files behind.
|
||||
"""
|
||||
from .commands import parse_command
|
||||
|
||||
if not isinstance(files, list) or not files:
|
||||
raise ValueError("the bundle has no files")
|
||||
for f in files:
|
||||
path = f.get("path") or ""
|
||||
parts = PurePosixPath(path).parts
|
||||
if not path or path.startswith("/") or "\\" in path or ".." in parts:
|
||||
raise ValueError(f"unsafe file path in bundle: {path!r}")
|
||||
index = next((f for f in files if f["path"] == "index.md"), None)
|
||||
if index is None:
|
||||
raise ValueError("the bundle has no index.md")
|
||||
try:
|
||||
meta, _ = parse_command(index["content"])
|
||||
except ValueError as e:
|
||||
raise ValueError(f"index.md frontmatter: {e}")
|
||||
for field in ("id", "name", "description"):
|
||||
if not meta.get(field):
|
||||
raise ValueError(f"index.md frontmatter is missing '{field}'")
|
||||
return meta
|
||||
|
||||
|
||||
def _is_url_source(source: str) -> bool:
|
||||
"""Return True when the source looks like a URL rather than a plain id."""
|
||||
return "://" in source or source.startswith("github.com/")
|
||||
|
||||
|
||||
def cmd_add(args):
|
||||
project_dir = find_project_dir(args.project)
|
||||
source = args.source
|
||||
|
||||
if _is_url_source(source):
|
||||
files = fetch_workflow_by_url(source)
|
||||
else:
|
||||
files = fetch_workflow_by_id(source)
|
||||
|
||||
try:
|
||||
meta = validate_bundle(files)
|
||||
except ValueError as e:
|
||||
print(f"Error: invalid workflow bundle: {e}", file=sys.stderr)
|
||||
result = registry_mod.install_workflow(project_dir, source)
|
||||
except (registry_mod.RegistryError, ValueError) as e:
|
||||
msg = str(e)
|
||||
print(f"Error: {msg}", file=sys.stderr)
|
||||
if "no workflow" in msg:
|
||||
print("Browse available workflows:", file=sys.stderr)
|
||||
print(" https://github.com/bleak-ai/workflows", file=sys.stderr)
|
||||
print(" https://gcontext.ai/workflows/", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
module_dir = project_dir / "modules" / meta["id"]
|
||||
if module_dir.exists():
|
||||
print(f"Error: module '{meta['id']}' already exists at {module_dir}.", file=sys.stderr)
|
||||
print("Installs are snapshots: your copy is personalized and is never overwritten.", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
for f in files:
|
||||
dest = module_dir / f["path"]
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
dest.write_text(f["content"])
|
||||
|
||||
rel = f"modules/{meta['id']}"
|
||||
print(f"{BOLD}gcontext{RESET} {DIM}-{RESET} installed {meta['name']} ({len(files)} files) at {rel}/")
|
||||
rel = result["path"]
|
||||
print(f"{BOLD}gcontext{RESET} {DIM}-{RESET} installed {result['name']} ({result['count']} files) at {rel}/")
|
||||
print()
|
||||
print("Next step: personalize it. (Re)start the server and tell your agent:")
|
||||
print(f" \"Run the setup in {rel}/commands/setup.md\"")
|
||||
@@ -718,6 +525,43 @@ def cmd_share(args):
|
||||
print(f' gh pr create --title "Add {wid}" --body "New workflow: {meta["name"]}"')
|
||||
|
||||
|
||||
def cmd_update(args):
|
||||
project_dir = find_project_dir(args.project)
|
||||
try:
|
||||
report = registry_mod.update_workflow(project_dir, args.id)
|
||||
except (registry_mod.RegistryError, ValueError) as e:
|
||||
print(f"Error: {e}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
print(registry_mod.format_update_report(report))
|
||||
if report.get("conflicts"):
|
||||
print()
|
||||
print("Resolve each conflict: merge <file>.new into <file>, then delete the .new file.")
|
||||
if report.get("commands_changed"):
|
||||
print()
|
||||
print("Commands changed: restart the server to re-register them (stop, gcontext up, reconnect the client).")
|
||||
|
||||
|
||||
def cmd_search(args):
|
||||
try:
|
||||
entries = registry_mod.search_catalog(args.query or "")
|
||||
except (registry_mod.RegistryError, ValueError) as e:
|
||||
print(f"Error: {e}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
if not entries:
|
||||
print(f"No workflows match '{args.query}'.")
|
||||
return
|
||||
|
||||
for e in entries:
|
||||
tags = ", ".join(e.get("tags", []))
|
||||
print(f" {e['id']} {e['name']} [{tags}]")
|
||||
if e.get("description"):
|
||||
print(f" {DIM}{e['description']}{RESET}")
|
||||
print()
|
||||
print(f"Install: gcontext add <id>")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
prog="gcontext",
|
||||
@@ -759,6 +603,13 @@ def main():
|
||||
share_parser = subparsers.add_parser("share", help="Validate a workflow template and show how to submit it via PR")
|
||||
share_parser.add_argument("module_path", help="Path to the template folder")
|
||||
|
||||
update_parser = subparsers.add_parser("update", help="Update an installed workflow from the registry")
|
||||
update_parser.add_argument("id", help="Workflow id (the modules/ folder name)")
|
||||
update_parser.add_argument("project", nargs="?", help="Path to gcontext project directory")
|
||||
|
||||
search_parser = subparsers.add_parser("search", help="Search the workflow registry")
|
||||
search_parser.add_argument("query", nargs="?", default="", help="Substring to match against id, name, description, tags")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
commands = {
|
||||
@@ -769,6 +620,8 @@ def main():
|
||||
"context": cmd_context,
|
||||
"add": cmd_add,
|
||||
"share": cmd_share,
|
||||
"update": cmd_update,
|
||||
"search": cmd_search,
|
||||
}
|
||||
if args.command in commands:
|
||||
commands[args.command](args)
|
||||
|
||||
+42
-20
@@ -158,29 +158,51 @@ def register_framework_prompts(mcp) -> int:
|
||||
return count
|
||||
|
||||
|
||||
def _register_one(mcp, root: Path, path: Path) -> bool:
|
||||
"""Register a single command file as a prompt. Returns True on success."""
|
||||
from fastmcp.prompts.prompt import Prompt
|
||||
|
||||
owner = path.parent.parent.name
|
||||
name = f"{owner}__{path.stem}"
|
||||
try:
|
||||
text = path.read_text(encoding="utf-8")
|
||||
if path.suffix == ".md":
|
||||
meta, body = parse_command(text)
|
||||
else:
|
||||
meta = parse_script_command(text)
|
||||
body = _script_prompt_body(str(path.relative_to(root)), meta)
|
||||
fn = _render_fn(body, meta.get("parameters") or [])
|
||||
fn.__name__ = name
|
||||
mcp.add_prompt(
|
||||
Prompt.from_function(fn, name=name, description=meta.get("description", ""))
|
||||
)
|
||||
except (ValueError, KeyError, yaml.YAMLError) as e:
|
||||
print(f" ! skipping command {path}: {e}", file=sys.stderr)
|
||||
return False
|
||||
except Exception as e:
|
||||
print(f" ! could not register prompt {name}: {e}", file=sys.stderr)
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def register_commands(mcp, root: Path) -> int:
|
||||
"""Scan connection and module `commands/` folders and register each file
|
||||
as a prompt named `<owner>__<command>`."""
|
||||
from fastmcp.prompts.prompt import Prompt
|
||||
|
||||
count = 0
|
||||
for path in discover(root):
|
||||
owner = path.parent.parent.name
|
||||
name = f"{owner}__{path.stem}"
|
||||
try:
|
||||
text = path.read_text(encoding="utf-8")
|
||||
if path.suffix == ".md":
|
||||
meta, body = parse_command(text)
|
||||
else:
|
||||
meta = parse_script_command(text)
|
||||
body = _script_prompt_body(str(path.relative_to(root)), meta)
|
||||
fn = _render_fn(body, meta.get("parameters") or [])
|
||||
fn.__name__ = name
|
||||
mcp.add_prompt(
|
||||
Prompt.from_function(fn, name=name, description=meta.get("description", ""))
|
||||
)
|
||||
except (ValueError, KeyError, yaml.YAMLError) as e:
|
||||
print(f" ! skipping command {path}: {e}", file=sys.stderr)
|
||||
continue
|
||||
count += 1
|
||||
if _register_one(mcp, root, path):
|
||||
count += 1
|
||||
return count
|
||||
|
||||
|
||||
def register_module_commands(mcp, root: Path, module_name: str) -> int:
|
||||
"""Register commands for a single module (e.g. after install or update)."""
|
||||
commands_dir = root / "modules" / module_name / "commands"
|
||||
if not commands_dir.is_dir():
|
||||
return 0
|
||||
count = 0
|
||||
for path in sorted(commands_dir.glob("*")):
|
||||
if path.suffix in (".md", ".py"):
|
||||
if _register_one(mcp, root, path):
|
||||
count += 1
|
||||
return count
|
||||
|
||||
+6
-3
@@ -14,6 +14,7 @@ from pathlib import Path
|
||||
# Machine folders: never served to the dashboard browser, skipped by
|
||||
# list_dir and grep.
|
||||
SKIP_DIRS = {".venv", ".git", "__pycache__", "node_modules"}
|
||||
SKIP_FILES = {".template.yaml"}
|
||||
BROWSER_BLOCKED = SKIP_DIRS
|
||||
|
||||
GREP_MAX_MATCHES = 100
|
||||
@@ -62,7 +63,7 @@ def walk_files(root: Path) -> list[str]:
|
||||
parts = f.relative_to(resolved).parts
|
||||
if (SKIP_DIRS | {"archive"}) & set(parts):
|
||||
continue
|
||||
if f.name == "secrets.env":
|
||||
if f.name == "secrets.env" or f.name in SKIP_FILES:
|
||||
continue
|
||||
out.append("/".join(parts))
|
||||
return out
|
||||
@@ -88,7 +89,7 @@ def _index_siblings(folder: Path) -> list[str]:
|
||||
"""
|
||||
names = []
|
||||
for entry in sorted(folder.iterdir(), key=lambda e: e.name):
|
||||
if entry.name in SKIP_DIRS | {"index.md", "secrets.env", "archive"}:
|
||||
if entry.name in SKIP_DIRS | SKIP_FILES | {"index.md", "secrets.env", "archive"}:
|
||||
continue
|
||||
names.append(entry.name)
|
||||
return names
|
||||
@@ -208,6 +209,8 @@ def list_dir(root: Path, path: str = ".") -> str:
|
||||
for entry in sorted(target.iterdir(), key=lambda e: e.name):
|
||||
if entry.name in SKIP_DIRS:
|
||||
continue
|
||||
if entry.is_file() and entry.name in SKIP_FILES:
|
||||
continue
|
||||
if entry.is_dir():
|
||||
dirs.append(f"{entry.name}/")
|
||||
else:
|
||||
@@ -240,7 +243,7 @@ def grep(root: Path, pattern: str, path: str = ".", glob: str = "") -> str:
|
||||
rel_parts = f.relative_to(resolved_root).parts
|
||||
if SKIP_DIRS & set(rel_parts):
|
||||
continue
|
||||
if f.name == "secrets.env":
|
||||
if f.name == "secrets.env" or f.name in SKIP_FILES:
|
||||
continue
|
||||
if glob and not fnmatch.fnmatch(f.name, glob):
|
||||
continue
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
Manage workflows from the registry: search, install, check for updates, and update.
|
||||
|
||||
Four actions:
|
||||
|
||||
- **search**: find workflows in the registry. Pass a query to filter by id,
|
||||
name, description, or tags; omit it to list all available workflows.
|
||||
- **install**: install a workflow by id into modules/. Refuses if the module
|
||||
already exists. After install, run the setup command to personalize it.
|
||||
- **check**: compare installed workflows against the registry. Reports which
|
||||
files changed upstream, which you modified locally, and which changed on
|
||||
both sides. Pass an id to check one, or omit it to check all.
|
||||
- **update**: pull upstream changes into an installed workflow. Files you
|
||||
modified locally are kept. Files changed on both sides get the upstream
|
||||
version written as `<file>.new` next to your version; merge the two and
|
||||
delete the `.new` file. Your runs, insights, and personal state are
|
||||
never touched.
|
||||
|
||||
Args:
|
||||
action: one of "search", "install", "check", "update"
|
||||
id: workflow id (required for install and update, optional for check)
|
||||
query: substring filter (used only by search)
|
||||
@@ -0,0 +1,528 @@
|
||||
"""Workflow registry: fetch, install, check, and update workflows from a GitHub registry.
|
||||
|
||||
The registry is a GitHub repo with one folder per workflow template.
|
||||
The env var GCONTEXT_REGISTRY accepts "owner/repo@ref" or a direct URL to
|
||||
a .tar.gz (useful for tests). All failures raise RegistryError; callers
|
||||
handle the presentation (CLI prints and exits, server tool returns a string).
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import tarfile
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from pathlib import Path, PurePosixPath
|
||||
|
||||
import yaml
|
||||
|
||||
|
||||
class RegistryError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
DEFAULT_REGISTRY = "bleak-ai/workflows@main"
|
||||
MANIFEST_NAME = ".template.yaml"
|
||||
|
||||
|
||||
def registry_spec() -> str:
|
||||
return os.environ.get("GCONTEXT_REGISTRY", DEFAULT_REGISTRY)
|
||||
|
||||
|
||||
def registry_name() -> str:
|
||||
spec = registry_spec()
|
||||
if spec.startswith("http://") or spec.startswith("https://"):
|
||||
return spec
|
||||
return spec.rsplit("@", 1)[0] if "@" in spec else spec
|
||||
|
||||
|
||||
def codeload_url(spec: str) -> str:
|
||||
if "@" in spec:
|
||||
repo_part, ref = spec.rsplit("@", 1)
|
||||
else:
|
||||
repo_part, ref = spec, "main"
|
||||
return f"https://codeload.github.com/{repo_part}/tar.gz/refs/heads/{ref}"
|
||||
|
||||
|
||||
def parse_registry() -> str:
|
||||
spec = registry_spec()
|
||||
if spec.startswith("http://") or spec.startswith("https://"):
|
||||
return spec
|
||||
return codeload_url(spec)
|
||||
|
||||
|
||||
def download_tarball(url: str) -> tarfile.TarFile:
|
||||
try:
|
||||
with urllib.request.urlopen(url, timeout=30) as resp:
|
||||
data = resp.read()
|
||||
except (urllib.error.HTTPError, urllib.error.URLError, OSError):
|
||||
raise RegistryError(f"could not reach the registry at {url}")
|
||||
try:
|
||||
return tarfile.open(fileobj=io.BytesIO(data), mode="r:gz")
|
||||
except tarfile.TarError:
|
||||
raise RegistryError("the registry did not return a valid tarball")
|
||||
|
||||
|
||||
def tarball_ref(tf: tarfile.TarFile) -> str:
|
||||
return (getattr(tf, "pax_headers", None) or {}).get("comment") or "unknown"
|
||||
|
||||
|
||||
def extract_files(tf: tarfile.TarFile, subpath: str = "") -> list[dict]:
|
||||
files = []
|
||||
for member in tf.getmembers():
|
||||
if not member.isfile():
|
||||
continue
|
||||
if member.issym() or member.islnk():
|
||||
continue
|
||||
parts = PurePosixPath(member.name).parts
|
||||
if len(parts) < 2:
|
||||
continue
|
||||
rel = str(PurePosixPath(*parts[1:]))
|
||||
if subpath:
|
||||
norm = subpath.rstrip("/") + "/"
|
||||
if not (rel + "/").startswith(norm) and rel != subpath.rstrip("/"):
|
||||
continue
|
||||
rel = rel[len(norm):] if rel.startswith(norm) else ""
|
||||
if not rel:
|
||||
continue
|
||||
try:
|
||||
raw = tf.extractfile(member)
|
||||
if raw is None:
|
||||
continue
|
||||
content = raw.read().decode("utf-8")
|
||||
except (UnicodeDecodeError, ValueError):
|
||||
continue
|
||||
files.append({"path": rel, "content": content})
|
||||
return files
|
||||
|
||||
|
||||
def parse_github_url(url: str) -> tuple[str, str, str]:
|
||||
cleaned = url
|
||||
if cleaned.startswith("github.com/"):
|
||||
cleaned = "https://" + cleaned
|
||||
path = cleaned.split("github.com/", 1)[1] if "github.com/" in cleaned else ""
|
||||
segments = path.strip("/").split("/")
|
||||
if len(segments) < 2:
|
||||
raise RegistryError(f"cannot parse GitHub URL: {url}")
|
||||
owner_repo = f"{segments[0]}/{segments[1]}"
|
||||
ref = "main"
|
||||
subpath = ""
|
||||
if len(segments) > 3 and segments[2] == "tree":
|
||||
ref = segments[3]
|
||||
if len(segments) > 4:
|
||||
subpath = "/".join(segments[4:])
|
||||
return owner_repo, ref, subpath
|
||||
|
||||
|
||||
def fetch_workflow_by_id(workflow_id: str) -> tuple[list[dict], str]:
|
||||
url = parse_registry()
|
||||
tf = download_tarball(url)
|
||||
ref = tarball_ref(tf)
|
||||
all_files = extract_files(tf)
|
||||
prefix = workflow_id + "/"
|
||||
matched = []
|
||||
for f in all_files:
|
||||
if f["path"].startswith(prefix):
|
||||
matched.append({"path": f["path"][len(prefix):], "content": f["content"]})
|
||||
elif f["path"] == workflow_id:
|
||||
matched.append({"path": f["path"], "content": f["content"]})
|
||||
if not matched:
|
||||
raise RegistryError(f"no workflow '{workflow_id}' found in the registry")
|
||||
return matched, ref
|
||||
|
||||
|
||||
def fetch_workflow_by_url(url: str) -> tuple[list[dict], str]:
|
||||
if "github.com/" in url or url.startswith("github.com/"):
|
||||
owner_repo, ref, subpath = parse_github_url(url)
|
||||
tarball_url = codeload_url(f"{owner_repo}@{ref}")
|
||||
else:
|
||||
tarball_url = url
|
||||
subpath = ""
|
||||
tf = download_tarball(tarball_url)
|
||||
ref = tarball_ref(tf)
|
||||
files = extract_files(tf, subpath=subpath)
|
||||
if not files:
|
||||
raise RegistryError(f"no files found at {url}")
|
||||
return files, ref
|
||||
|
||||
|
||||
def validate_bundle(files) -> dict:
|
||||
from .commands import parse_command
|
||||
|
||||
if not isinstance(files, list) or not files:
|
||||
raise ValueError("the bundle has no files")
|
||||
for f in files:
|
||||
path = f.get("path") or ""
|
||||
parts = PurePosixPath(path).parts
|
||||
if not path or path.startswith("/") or "\\" in path or ".." in parts:
|
||||
raise ValueError(f"unsafe file path in bundle: {path!r}")
|
||||
index = next((f for f in files if f["path"] == "index.md"), None)
|
||||
if index is None:
|
||||
raise ValueError("the bundle has no index.md")
|
||||
try:
|
||||
meta, _ = parse_command(index["content"])
|
||||
except ValueError as e:
|
||||
raise ValueError(f"index.md frontmatter: {e}")
|
||||
for field in ("id", "name", "description"):
|
||||
if not meta.get(field):
|
||||
raise ValueError(f"index.md frontmatter is missing '{field}'")
|
||||
return meta
|
||||
|
||||
|
||||
def file_hash(content: str) -> str:
|
||||
return "sha256:" + hashlib.sha256(content.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def write_manifest(module_dir: Path, workflow_id: str, ref: str, files: list[dict]):
|
||||
data = {
|
||||
"template": workflow_id,
|
||||
"registry": registry_name(),
|
||||
"installed_ref": ref,
|
||||
"files": {f["path"]: file_hash(f["content"]) for f in sorted(files, key=lambda f: f["path"])},
|
||||
}
|
||||
(module_dir / MANIFEST_NAME).write_text(yaml.safe_dump(data, sort_keys=False))
|
||||
|
||||
|
||||
def read_manifest(module_dir: Path) -> dict | None:
|
||||
path = module_dir / MANIFEST_NAME
|
||||
if not path.exists():
|
||||
return None
|
||||
try:
|
||||
return yaml.safe_load(path.read_text())
|
||||
except (yaml.YAMLError, OSError):
|
||||
return None
|
||||
|
||||
|
||||
def install_workflow(project_dir: Path, source: str) -> dict:
|
||||
if "://" in source or source.startswith("github.com/"):
|
||||
files, ref = fetch_workflow_by_url(source)
|
||||
else:
|
||||
files, ref = fetch_workflow_by_id(source)
|
||||
|
||||
try:
|
||||
meta = validate_bundle(files)
|
||||
except ValueError as e:
|
||||
raise RegistryError(f"invalid workflow bundle: {e}")
|
||||
|
||||
module_dir = project_dir / "modules" / meta["id"]
|
||||
if module_dir.exists():
|
||||
raise RegistryError(
|
||||
f"module '{meta['id']}' already exists at modules/{meta['id']}. "
|
||||
"Installs are snapshots: your copy is personalized and is never overwritten."
|
||||
)
|
||||
|
||||
for f in files:
|
||||
dest = module_dir / f["path"]
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
dest.write_text(f["content"])
|
||||
|
||||
write_manifest(module_dir, meta["id"], ref, files)
|
||||
|
||||
return {"id": meta["id"], "name": meta["name"], "count": len(files), "path": f"modules/{meta['id']}"}
|
||||
|
||||
|
||||
# --- Catalog (search) ---
|
||||
|
||||
def load_catalog() -> list[dict]:
|
||||
url = parse_registry()
|
||||
tf = download_tarball(url)
|
||||
all_files = extract_files(tf)
|
||||
catalog_file = next((f for f in all_files if f["path"] == "registry.json"), None)
|
||||
if catalog_file is None:
|
||||
raise RegistryError("the registry has no registry.json catalog")
|
||||
try:
|
||||
catalog = json.loads(catalog_file["content"])
|
||||
return catalog["workflows"]
|
||||
except (json.JSONDecodeError, KeyError, TypeError):
|
||||
raise RegistryError("the registry has no registry.json catalog")
|
||||
|
||||
|
||||
def search_catalog(query: str = "") -> list[dict]:
|
||||
entries = load_catalog()
|
||||
if not query:
|
||||
return entries
|
||||
q = query.lower()
|
||||
return [
|
||||
e for e in entries
|
||||
if q in e.get("id", "").lower()
|
||||
or q in e.get("name", "").lower()
|
||||
or q in e.get("description", "").lower()
|
||||
or any(q in t.lower() for t in e.get("tags", []))
|
||||
]
|
||||
|
||||
|
||||
# --- Check and update ---
|
||||
|
||||
def check_workflow(project_dir: Path, module_name: str, upstream: dict | None = None) -> dict:
|
||||
module_dir = project_dir / "modules" / module_name
|
||||
manifest = read_manifest(module_dir)
|
||||
if manifest is None:
|
||||
return {"id": module_name, "status": "untracked"}
|
||||
|
||||
template_id = manifest.get("template", module_name)
|
||||
if upstream is None:
|
||||
try:
|
||||
files, ref = fetch_workflow_by_id(template_id)
|
||||
except RegistryError:
|
||||
return {"id": module_name, "status": "missing-upstream"}
|
||||
upstream_map = {f["path"]: f["content"] for f in files}
|
||||
else:
|
||||
upstream_map, ref = upstream
|
||||
|
||||
base_hashes = manifest.get("files", {})
|
||||
all_paths = set(base_hashes) | set(upstream_map)
|
||||
|
||||
changed = {}
|
||||
for path in sorted(all_paths):
|
||||
base = base_hashes.get(path)
|
||||
upstream_content = upstream_map.get(path)
|
||||
upstream_h = file_hash(upstream_content) if upstream_content is not None else None
|
||||
|
||||
local_file = module_dir / path
|
||||
try:
|
||||
local_content = local_file.read_text() if local_file.exists() else None
|
||||
except (OSError, UnicodeDecodeError):
|
||||
local_content = None
|
||||
local_h = file_hash(local_content) if local_content is not None else None
|
||||
|
||||
if base is None:
|
||||
if upstream_h is not None and local_h is None:
|
||||
changed[path] = "new-upstream"
|
||||
elif upstream_h is not None and local_h is not None:
|
||||
if upstream_h == local_h:
|
||||
continue
|
||||
changed[path] = "both-changed"
|
||||
continue
|
||||
|
||||
if upstream_h is None:
|
||||
if local_h == base:
|
||||
changed[path] = "deleted-upstream"
|
||||
elif local_h is not None:
|
||||
changed[path] = "deleted-upstream-modified-locally"
|
||||
else:
|
||||
continue
|
||||
continue
|
||||
|
||||
if local_h is None:
|
||||
changed[path] = "deleted-locally"
|
||||
continue
|
||||
|
||||
if local_h == upstream_h:
|
||||
continue
|
||||
if local_h == base and upstream_h != base:
|
||||
changed[path] = "upstream-changed"
|
||||
elif local_h != base and upstream_h == base:
|
||||
changed[path] = "local-changed"
|
||||
elif local_h != base and upstream_h != base:
|
||||
changed[path] = "both-changed"
|
||||
|
||||
status = "up-to-date" if not changed else "changes"
|
||||
return {"id": module_name, "status": status, "ref": ref, "files": changed}
|
||||
|
||||
|
||||
def check_all(project_dir: Path) -> list[dict]:
|
||||
modules_dir = project_dir / "modules"
|
||||
if not modules_dir.is_dir():
|
||||
return []
|
||||
|
||||
tracked = {}
|
||||
for d in sorted(modules_dir.iterdir()):
|
||||
if not d.is_dir():
|
||||
continue
|
||||
manifest = read_manifest(d)
|
||||
if manifest is None:
|
||||
continue
|
||||
tracked[d.name] = manifest.get("template", d.name)
|
||||
|
||||
if not tracked:
|
||||
return []
|
||||
|
||||
url = parse_registry()
|
||||
tf = download_tarball(url)
|
||||
ref = tarball_ref(tf)
|
||||
all_files = extract_files(tf)
|
||||
|
||||
upstream_by_template = {}
|
||||
for template_id in set(tracked.values()):
|
||||
prefix = template_id + "/"
|
||||
matched = {}
|
||||
for f in all_files:
|
||||
if f["path"].startswith(prefix):
|
||||
matched[f["path"][len(prefix):]] = f["content"]
|
||||
upstream_by_template[template_id] = matched
|
||||
|
||||
results = []
|
||||
for module_name, template_id in tracked.items():
|
||||
upstream_map = upstream_by_template.get(template_id, {})
|
||||
report = check_workflow(project_dir, module_name, upstream=(upstream_map, ref))
|
||||
results.append(report)
|
||||
return results
|
||||
|
||||
|
||||
def update_workflow(project_dir: Path, module_name: str) -> dict:
|
||||
module_dir = project_dir / "modules" / module_name
|
||||
manifest = read_manifest(module_dir)
|
||||
if manifest is None:
|
||||
raise RegistryError(
|
||||
f"modules/{module_name} has no {MANIFEST_NAME}; "
|
||||
"it was not installed from the registry"
|
||||
)
|
||||
|
||||
template_id = manifest.get("template", module_name)
|
||||
files, ref = fetch_workflow_by_id(template_id)
|
||||
upstream_map = {f["path"]: f["content"] for f in files}
|
||||
base_hashes = dict(manifest.get("files", {}))
|
||||
new_hashes = dict(base_hashes)
|
||||
|
||||
all_paths = set(base_hashes) | set(upstream_map)
|
||||
|
||||
report = {
|
||||
"id": module_name,
|
||||
"ref": ref,
|
||||
"updated": [],
|
||||
"kept_local": [],
|
||||
"conflicts": [],
|
||||
"added": [],
|
||||
"deleted": [],
|
||||
"kept_deleted_upstream": [],
|
||||
"commands_changed": False,
|
||||
}
|
||||
|
||||
for path in sorted(all_paths):
|
||||
base = base_hashes.get(path)
|
||||
upstream_content = upstream_map.get(path)
|
||||
upstream_h = file_hash(upstream_content) if upstream_content is not None else None
|
||||
|
||||
local_file = module_dir / path
|
||||
try:
|
||||
local_content = local_file.read_text() if local_file.exists() else None
|
||||
except (OSError, UnicodeDecodeError):
|
||||
local_content = None
|
||||
local_h = file_hash(local_content) if local_content is not None else None
|
||||
|
||||
if base is None:
|
||||
if upstream_h is not None and local_h is None:
|
||||
dest = module_dir / path
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
dest.write_text(upstream_content)
|
||||
new_hashes[path] = upstream_h
|
||||
report["added"].append(path)
|
||||
elif upstream_h is not None and local_h is not None:
|
||||
if upstream_h == local_h:
|
||||
new_hashes[path] = upstream_h
|
||||
else:
|
||||
new_path = module_dir / (path + ".new")
|
||||
new_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
new_path.write_text(upstream_content)
|
||||
new_hashes[path] = upstream_h
|
||||
report["conflicts"].append(path)
|
||||
continue
|
||||
|
||||
if upstream_h is None:
|
||||
if local_h == base:
|
||||
if local_file.exists():
|
||||
local_file.unlink()
|
||||
new_hashes.pop(path, None)
|
||||
report["deleted"].append(path)
|
||||
elif local_h is not None:
|
||||
new_hashes.pop(path, None)
|
||||
report["kept_deleted_upstream"].append(path)
|
||||
else:
|
||||
new_hashes.pop(path, None)
|
||||
continue
|
||||
|
||||
if local_h is None:
|
||||
report["kept_local"].append(path + " (deleted locally)")
|
||||
continue
|
||||
|
||||
if local_h == upstream_h:
|
||||
new_hashes[path] = upstream_h
|
||||
continue
|
||||
|
||||
if local_h == base and upstream_h != base:
|
||||
local_file.write_text(upstream_content)
|
||||
new_hashes[path] = upstream_h
|
||||
report["updated"].append(path)
|
||||
elif local_h != base and upstream_h == base:
|
||||
report["kept_local"].append(path)
|
||||
elif local_h != base and upstream_h != base:
|
||||
new_path = module_dir / (path + ".new")
|
||||
new_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
new_path.write_text(upstream_content)
|
||||
new_hashes[path] = upstream_h
|
||||
report["conflicts"].append(path)
|
||||
|
||||
commands_paths = report["updated"] + report["added"] + report["conflicts"] + report["deleted"]
|
||||
report["commands_changed"] = any(p.startswith("commands/") for p in commands_paths)
|
||||
|
||||
manifest_data = {
|
||||
"template": template_id,
|
||||
"registry": manifest.get("registry", registry_name()),
|
||||
"installed_ref": ref,
|
||||
"files": dict(sorted(new_hashes.items())),
|
||||
}
|
||||
(module_dir / MANIFEST_NAME).write_text(yaml.safe_dump(manifest_data, sort_keys=False))
|
||||
|
||||
return report
|
||||
|
||||
|
||||
# --- Report formatting (shared by CLI and server tool) ---
|
||||
|
||||
def format_check_report(report: dict) -> str:
|
||||
if report["status"] == "untracked":
|
||||
return f"{report['id']}: not tracked (no {MANIFEST_NAME})"
|
||||
if report["status"] == "missing-upstream":
|
||||
return f"{report['id']}: template not found in the registry"
|
||||
if report["status"] == "up-to-date":
|
||||
return f"{report['id']}: up to date (ref {report.get('ref', 'unknown')})"
|
||||
lines = [f"{report['id']}: changes available (ref {report.get('ref', 'unknown')})"]
|
||||
labels = {
|
||||
"upstream-changed": "upstream changed",
|
||||
"local-changed": "locally modified",
|
||||
"both-changed": "both changed",
|
||||
"new-upstream": "new upstream file",
|
||||
"deleted-upstream": "deleted upstream",
|
||||
"deleted-upstream-modified-locally": "deleted upstream, modified locally",
|
||||
"deleted-locally": "deleted locally",
|
||||
}
|
||||
for path, classification in sorted(report.get("files", {}).items()):
|
||||
lines.append(f" {path}: {labels.get(classification, classification)}")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def format_update_report(report: dict) -> str:
|
||||
lines = []
|
||||
|
||||
if not any(report.get(k) for k in ("updated", "added", "deleted", "conflicts", "kept_local", "kept_deleted_upstream")):
|
||||
lines.append(f"{report['id']}: up to date")
|
||||
else:
|
||||
lines.append(f"{report['id']}: updated")
|
||||
|
||||
if report.get("updated"):
|
||||
lines.append("Updated:")
|
||||
for p in report["updated"]:
|
||||
lines.append(f" {p}")
|
||||
if report.get("added"):
|
||||
lines.append("Added:")
|
||||
for p in report["added"]:
|
||||
lines.append(f" {p}")
|
||||
if report.get("deleted"):
|
||||
lines.append("Deleted:")
|
||||
for p in report["deleted"]:
|
||||
lines.append(f" {p}")
|
||||
if report.get("conflicts"):
|
||||
lines.append("Conflicts (merge <file>.new into <file>, then delete the .new file):")
|
||||
for p in report["conflicts"]:
|
||||
lines.append(f" {p}")
|
||||
if report.get("kept_local"):
|
||||
lines.append("Kept local changes:")
|
||||
for p in report["kept_local"]:
|
||||
lines.append(f" {p}")
|
||||
if report.get("kept_deleted_upstream"):
|
||||
lines.append("Kept (upstream deleted, you modified):")
|
||||
for p in report["kept_deleted_upstream"]:
|
||||
lines.append(f" {p}")
|
||||
|
||||
lines.append(f"Manifest updated to ref {report.get('ref', 'unknown')}.")
|
||||
return "\n".join(lines)
|
||||
@@ -32,6 +32,7 @@ from starlette.responses import JSONResponse
|
||||
from . import commands as commands_mod
|
||||
from . import exec as exec_mod
|
||||
from . import fs
|
||||
from . import registry as registry_mod
|
||||
from . import secrets as secrets_mod
|
||||
from . import state
|
||||
|
||||
@@ -436,4 +437,88 @@ def run_adhoc_script(
|
||||
|
||||
|
||||
|
||||
def _register_module_commands(module_id: str):
|
||||
commands_mod.register_module_commands(mcp, PROJECT_DIR, module_id)
|
||||
|
||||
|
||||
def _notify_prompts_changed():
|
||||
try:
|
||||
import asyncio
|
||||
from fastmcp.server.dependencies import get_context
|
||||
ctx = get_context()
|
||||
session = getattr(ctx, "session", None)
|
||||
if session and hasattr(session, "send_prompt_list_changed"):
|
||||
loop = asyncio.get_running_loop()
|
||||
loop.create_task(session.send_prompt_list_changed())
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
@mcp.tool(description=_tool_doc("workflow"), output_schema=None)
|
||||
def workflow(action: str, id: str = "", query: str = "") -> str:
|
||||
if action == "search":
|
||||
try:
|
||||
entries = registry_mod.search_catalog(query)
|
||||
except (registry_mod.RegistryError, ValueError) as e:
|
||||
return f"Error: {e}."
|
||||
if not entries:
|
||||
return f"No workflows match '{query}'."
|
||||
lines = []
|
||||
for e in entries:
|
||||
tags = ", ".join(e.get("tags", []))
|
||||
lines.append(f"{e['id']}: {e['name']}")
|
||||
if e.get("description"):
|
||||
lines.append(f" {e['description']}")
|
||||
if tags:
|
||||
lines.append(f" tags: {tags}")
|
||||
lines.append("")
|
||||
lines.append('Install one with workflow(action="install", id="<id>")')
|
||||
return "\n".join(lines)
|
||||
|
||||
elif action == "install":
|
||||
if not id:
|
||||
return "Error: install needs an id."
|
||||
try:
|
||||
result = registry_mod.install_workflow(PROJECT_DIR, id)
|
||||
except (registry_mod.RegistryError, ValueError) as e:
|
||||
return f"Error: {e}."
|
||||
_register_module_commands(result["id"])
|
||||
_notify_prompts_changed()
|
||||
snapshot_startup_files()
|
||||
return (
|
||||
f"Installed {result['name']} ({result['count']} files) at {result['path']}/.\n"
|
||||
f"Next step: run the setup in {result['path']}/commands/setup.md"
|
||||
)
|
||||
|
||||
elif action == "check":
|
||||
try:
|
||||
if id:
|
||||
module_dir = PROJECT_DIR / "modules" / id
|
||||
if not module_dir.is_dir():
|
||||
return f"Error: modules/{id} does not exist."
|
||||
reports = [registry_mod.check_workflow(PROJECT_DIR, id)]
|
||||
else:
|
||||
reports = registry_mod.check_all(PROJECT_DIR)
|
||||
except (registry_mod.RegistryError, ValueError) as e:
|
||||
return f"Error: {e}."
|
||||
if not reports:
|
||||
return f"No installed workflows track a template (no {registry_mod.MANIFEST_NAME} files found)."
|
||||
return "\n".join(registry_mod.format_check_report(r) for r in reports)
|
||||
|
||||
elif action == "update":
|
||||
if not id:
|
||||
return "Error: update needs an id."
|
||||
try:
|
||||
report = registry_mod.update_workflow(PROJECT_DIR, id)
|
||||
except (registry_mod.RegistryError, ValueError) as e:
|
||||
return f"Error: {e}."
|
||||
if report.get("commands_changed"):
|
||||
_register_module_commands(report["id"])
|
||||
_notify_prompts_changed()
|
||||
snapshot_startup_files()
|
||||
return registry_mod.format_update_report(report)
|
||||
|
||||
return f"Error: unknown action '{action}'. Use search, install, check, or update."
|
||||
|
||||
|
||||
from . import dashboard # noqa: E402,F401 registers /api/* and the static catch-all
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[project]
|
||||
name = "gcontext-ai"
|
||||
version = "0.6.0"
|
||||
version = "0.7.0"
|
||||
description = "The framework for building stateful agents. Your agent is a folder of state, served over MCP, used from any runtime."
|
||||
readme = "README.md"
|
||||
license = { text = "MIT" }
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
"""Build registry.json from a workflows repo checkout.
|
||||
|
||||
Usage: uv run scripts/build_registry.py <path-to-workflows-checkout>
|
||||
|
||||
Scans each top-level directory for an index.md with valid frontmatter,
|
||||
collects id/name/description/tags and the file list, and writes
|
||||
registry.json at the checkout root.
|
||||
"""
|
||||
|
||||
import json
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def build(checkout: Path) -> dict:
|
||||
from gcontext.commands import parse_command
|
||||
|
||||
workflows = []
|
||||
for d in sorted(checkout.iterdir()):
|
||||
if not d.is_dir() or d.name.startswith("."):
|
||||
continue
|
||||
index = d / "index.md"
|
||||
if not index.exists():
|
||||
continue
|
||||
try:
|
||||
meta, _ = parse_command(index.read_text(encoding="utf-8"))
|
||||
except (ValueError, OSError):
|
||||
continue
|
||||
if not meta.get("id"):
|
||||
continue
|
||||
|
||||
files = []
|
||||
for f in sorted(d.rglob("*")):
|
||||
if not f.is_file():
|
||||
continue
|
||||
rel_parts = f.relative_to(d).parts
|
||||
if any(p.startswith(".") or p.startswith("__") for p in rel_parts):
|
||||
continue
|
||||
try:
|
||||
f.read_text(encoding="utf-8")
|
||||
except (UnicodeDecodeError, OSError):
|
||||
continue
|
||||
files.append(str(f.relative_to(d)))
|
||||
|
||||
workflows.append({
|
||||
"id": meta["id"],
|
||||
"name": meta.get("name", meta["id"]),
|
||||
"description": meta.get("description", ""),
|
||||
"tags": meta.get("tags", []),
|
||||
"files": files,
|
||||
})
|
||||
|
||||
workflows.sort(key=lambda w: w["id"])
|
||||
return {
|
||||
"generated": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
|
||||
"workflows": workflows,
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) < 2:
|
||||
print("Usage: uv run scripts/build_registry.py <path-to-workflows-checkout>")
|
||||
sys.exit(1)
|
||||
|
||||
checkout = Path(sys.argv[1]).resolve()
|
||||
if not checkout.is_dir():
|
||||
print(f"Error: {checkout} is not a directory.")
|
||||
sys.exit(1)
|
||||
|
||||
catalog = build(checkout)
|
||||
out = checkout / "registry.json"
|
||||
out.write_text(json.dumps(catalog, indent=2) + "\n")
|
||||
|
||||
for w in catalog["workflows"]:
|
||||
print(f" {w['id']} ({len(w['files'])} files)")
|
||||
print(f"\n{len(catalog['workflows'])} workflows written to {out}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,391 @@
|
||||
"""Tests for the workflow MCP tool: search, install, check, update."""
|
||||
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import tarfile
|
||||
import threading
|
||||
from http.server import BaseHTTPRequestHandler, HTTPServer
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
|
||||
from gcontext import registry as registry_mod, server, fs
|
||||
|
||||
INDEX_MD = """---
|
||||
id: demo-flow
|
||||
name: Demo Flow
|
||||
description: A tiny demo workflow for tests.
|
||||
tags: [demo]
|
||||
---
|
||||
|
||||
Objective paragraph.
|
||||
"""
|
||||
|
||||
SETUP_MD = """---
|
||||
description: Set up the demo workflow
|
||||
---
|
||||
|
||||
Interview the user.
|
||||
"""
|
||||
|
||||
BUNDLE_FILES = [
|
||||
{"path": "index.md", "content": INDEX_MD},
|
||||
{"path": "steps/index.md", "content": "1-sync.md: sync things\n"},
|
||||
{"path": "steps/1-sync.md", "content": "# Step 1\n\nSync.\n"},
|
||||
{"path": "commands/setup.md", "content": SETUP_MD},
|
||||
{"path": "runs/example/index.md", "content": "# Example run\n"},
|
||||
]
|
||||
|
||||
CATALOG = {
|
||||
"generated": "2026-08-10T12:00:00Z",
|
||||
"workflows": [
|
||||
{
|
||||
"id": "demo-flow",
|
||||
"name": "Demo Flow",
|
||||
"description": "A tiny demo workflow for tests.",
|
||||
"tags": ["demo"],
|
||||
"files": [f["path"] for f in BUNDLE_FILES],
|
||||
},
|
||||
{
|
||||
"id": "ops-flow",
|
||||
"name": "Ops Flow",
|
||||
"description": "An operations workflow.",
|
||||
"tags": ["ops", "infra"],
|
||||
"files": ["index.md"],
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _build_tarball(files, prefix="workflows-main"):
|
||||
buf = io.BytesIO()
|
||||
with tarfile.open(fileobj=buf, mode="w:gz") as tf:
|
||||
for f in files:
|
||||
member_path = f"{prefix}/{f['path']}"
|
||||
data = f["content"].encode("utf-8")
|
||||
info = tarfile.TarInfo(name=member_path)
|
||||
info.size = len(data)
|
||||
tf.addfile(info, io.BytesIO(data))
|
||||
buf.seek(0)
|
||||
return buf.read()
|
||||
|
||||
|
||||
def _registry_files(workflow_id="demo-flow"):
|
||||
return [{"path": f"{workflow_id}/{f['path']}", "content": f["content"]} for f in BUNDLE_FILES]
|
||||
|
||||
|
||||
def _file_hash(content):
|
||||
return "sha256:" + hashlib.sha256(content.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def registry(monkeypatch):
|
||||
tarball_data = [None]
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
def do_GET(self):
|
||||
if tarball_data[0] is not None:
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "application/gzip")
|
||||
self.end_headers()
|
||||
self.wfile.write(tarball_data[0])
|
||||
else:
|
||||
self.send_response(404)
|
||||
self.end_headers()
|
||||
|
||||
def log_message(self, *args):
|
||||
pass
|
||||
|
||||
srv = HTTPServer(("127.0.0.1", 0), Handler)
|
||||
thread = threading.Thread(target=srv.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
url = f"http://127.0.0.1:{srv.server_port}/registry.tar.gz"
|
||||
monkeypatch.setenv("GCONTEXT_REGISTRY", url)
|
||||
yield tarball_data
|
||||
srv.shutdown()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def project(tmp_path, monkeypatch):
|
||||
p = tmp_path / "agent"
|
||||
p.mkdir()
|
||||
(p / "gcontext.yaml").write_text("name: test-agent\n")
|
||||
(p / "modules").mkdir()
|
||||
(p / "connections").mkdir()
|
||||
monkeypatch.setattr(server, "PROJECT_DIR", p)
|
||||
return p
|
||||
|
||||
|
||||
def _tarball_with_catalog(extra_files=None):
|
||||
files = _registry_files()
|
||||
catalog_content = json.dumps(CATALOG)
|
||||
files.append({"path": "registry.json", "content": catalog_content})
|
||||
if extra_files:
|
||||
files.extend(extra_files)
|
||||
return _build_tarball(files)
|
||||
|
||||
|
||||
# --- Search tests ---
|
||||
|
||||
def test_search_returns_all(registry, project):
|
||||
registry[0] = _tarball_with_catalog()
|
||||
result = server.workflow(action="search")
|
||||
assert "demo-flow" in result
|
||||
assert "ops-flow" in result
|
||||
|
||||
|
||||
def test_search_filters_by_query(registry, project):
|
||||
registry[0] = _tarball_with_catalog()
|
||||
result = server.workflow(action="search", query="demo")
|
||||
assert "demo-flow" in result
|
||||
assert "ops-flow" not in result
|
||||
|
||||
|
||||
def test_search_case_insensitive(registry, project):
|
||||
registry[0] = _tarball_with_catalog()
|
||||
result = server.workflow(action="search", query="DEMO")
|
||||
assert "demo-flow" in result
|
||||
|
||||
|
||||
def test_search_no_match(registry, project):
|
||||
registry[0] = _tarball_with_catalog()
|
||||
result = server.workflow(action="search", query="nomatch")
|
||||
assert "No workflows match" in result
|
||||
|
||||
|
||||
def test_search_without_catalog_errors(registry, project):
|
||||
registry[0] = _build_tarball(_registry_files())
|
||||
result = server.workflow(action="search")
|
||||
assert result.startswith("Error:")
|
||||
assert "registry.json" in result
|
||||
|
||||
|
||||
# --- Install tests ---
|
||||
|
||||
def test_install_creates_module_and_manifest(registry, project):
|
||||
registry[0] = _tarball_with_catalog()
|
||||
result = server.workflow(action="install", id="demo-flow")
|
||||
assert "Demo Flow" in result
|
||||
assert "commands/setup.md" in result
|
||||
|
||||
module = project / "modules" / "demo-flow"
|
||||
assert (module / "index.md").exists()
|
||||
assert (module / "steps" / "1-sync.md").exists()
|
||||
|
||||
manifest = yaml.safe_load((module / ".template.yaml").read_text())
|
||||
assert manifest["template"] == "demo-flow"
|
||||
assert manifest["installed_ref"] == "unknown"
|
||||
for f in BUNDLE_FILES:
|
||||
assert manifest["files"][f["path"]] == _file_hash(f["content"])
|
||||
|
||||
|
||||
def test_install_existing_module_refuses(registry, project):
|
||||
marker = project / "modules" / "demo-flow" / "personal.md"
|
||||
marker.parent.mkdir(parents=True)
|
||||
marker.write_text("mine")
|
||||
registry[0] = _tarball_with_catalog()
|
||||
result = server.workflow(action="install", id="demo-flow")
|
||||
assert result.startswith("Error:")
|
||||
assert "already exists" in result
|
||||
assert marker.read_text() == "mine"
|
||||
|
||||
|
||||
def test_install_missing_id(registry, project):
|
||||
result = server.workflow(action="install")
|
||||
assert result.startswith("Error:")
|
||||
assert "needs an id" in result
|
||||
|
||||
|
||||
def test_unknown_action(registry, project):
|
||||
result = server.workflow(action="frobnicate")
|
||||
assert result.startswith("Error:")
|
||||
assert "unknown action" in result
|
||||
|
||||
|
||||
# --- Hidden manifest tests ---
|
||||
|
||||
def test_template_manifest_hidden_from_list_dir(project):
|
||||
mod = project / "modules" / "demo"
|
||||
mod.mkdir(parents=True)
|
||||
(mod / ".template.yaml").write_text("template: demo\n")
|
||||
(mod / "index.md").write_text("# demo\n")
|
||||
result = fs.list_dir(project, "modules/demo")
|
||||
assert "index.md" in result
|
||||
assert ".template.yaml" not in result
|
||||
|
||||
|
||||
def test_template_manifest_hidden_from_grep(project):
|
||||
mod = project / "modules" / "demo"
|
||||
mod.mkdir(parents=True)
|
||||
(mod / ".template.yaml").write_text("template: demo\n")
|
||||
(mod / "index.md").write_text("# demo\n")
|
||||
result = fs.grep(project, "template", "modules")
|
||||
assert ".template.yaml" not in result
|
||||
|
||||
|
||||
def test_template_manifest_hidden_from_walk(project):
|
||||
mod = project / "modules" / "demo"
|
||||
mod.mkdir(parents=True)
|
||||
(mod / ".template.yaml").write_text("template: demo\n")
|
||||
(mod / "index.md").write_text("# demo\n")
|
||||
walked = fs.walk_files(project)
|
||||
assert not any(".template.yaml" in p for p in walked)
|
||||
|
||||
|
||||
def test_template_manifest_still_readable(project):
|
||||
mod = project / "modules" / "demo"
|
||||
mod.mkdir(parents=True)
|
||||
(mod / ".template.yaml").write_text("template: demo\n")
|
||||
result = fs.read_file(project, "modules/demo/.template.yaml")
|
||||
assert "template: demo" in result
|
||||
|
||||
|
||||
def test_index_warning_ignores_template_manifest(project):
|
||||
mod = project / "modules" / "demo"
|
||||
mod.mkdir(parents=True)
|
||||
(mod / ".template.yaml").write_text("template: demo\n")
|
||||
(mod / "steps").mkdir()
|
||||
result = fs.write_file(project, "modules/demo/index.md", "# demo\n\n- [steps](steps/)\n")
|
||||
assert ".template.yaml" not in result
|
||||
|
||||
|
||||
# --- Check tests ---
|
||||
|
||||
def test_check_up_to_date(registry, project):
|
||||
registry[0] = _tarball_with_catalog()
|
||||
server.workflow(action="install", id="demo-flow")
|
||||
result = server.workflow(action="check", id="demo-flow")
|
||||
assert "up to date" in result
|
||||
|
||||
|
||||
def test_check_detects_changes(registry, project):
|
||||
registry[0] = _tarball_with_catalog()
|
||||
server.workflow(action="install", id="demo-flow")
|
||||
|
||||
modified_step = "# Step 1 MODIFIED\n\nNew sync.\n"
|
||||
modified_files = []
|
||||
for f in BUNDLE_FILES:
|
||||
if f["path"] == "steps/1-sync.md":
|
||||
modified_files.append({"path": f["path"], "content": modified_step})
|
||||
else:
|
||||
modified_files.append(f)
|
||||
new_registry = _registry_files_from(modified_files)
|
||||
new_registry.append({"path": "registry.json", "content": json.dumps(CATALOG)})
|
||||
registry[0] = _build_tarball(new_registry)
|
||||
|
||||
(project / "modules" / "demo-flow" / "commands" / "setup.md").write_text("local edit")
|
||||
|
||||
result = server.workflow(action="check", id="demo-flow")
|
||||
assert "upstream changed" in result
|
||||
assert "locally modified" in result
|
||||
|
||||
|
||||
def test_check_nonexistent_module(registry, project):
|
||||
result = server.workflow(action="check", id="nope")
|
||||
assert result.startswith("Error:")
|
||||
assert "does not exist" in result
|
||||
|
||||
|
||||
def test_check_all_no_tracked(registry, project):
|
||||
result = server.workflow(action="check")
|
||||
assert "No installed workflows" in result
|
||||
|
||||
|
||||
# --- Update tests ---
|
||||
|
||||
def test_update_applies_three_way(registry, project):
|
||||
registry[0] = _tarball_with_catalog()
|
||||
server.workflow(action="install", id="demo-flow")
|
||||
|
||||
(project / "modules" / "demo-flow" / "commands" / "setup.md").write_text("local edit")
|
||||
|
||||
new_step = "# Step 1 UPDATED\n"
|
||||
new_index = INDEX_MD.replace("Objective paragraph.", "Updated objective.")
|
||||
modified_files = []
|
||||
for f in BUNDLE_FILES:
|
||||
if f["path"] == "steps/1-sync.md":
|
||||
modified_files.append({"path": f["path"], "content": new_step})
|
||||
elif f["path"] == "index.md":
|
||||
modified_files.append({"path": f["path"], "content": new_index})
|
||||
else:
|
||||
modified_files.append(f)
|
||||
modified_files.append({"path": "steps/2-verify.md", "content": "# Verify\n"})
|
||||
|
||||
new_registry = _registry_files_from(modified_files)
|
||||
new_registry.append({"path": "registry.json", "content": json.dumps(CATALOG)})
|
||||
registry[0] = _build_tarball(new_registry)
|
||||
|
||||
(project / "modules" / "demo-flow" / "index.md").write_text(
|
||||
INDEX_MD.replace("Objective paragraph.", "My local objective.")
|
||||
)
|
||||
|
||||
result = server.workflow(action="update", id="demo-flow")
|
||||
|
||||
assert (project / "modules" / "demo-flow" / "steps" / "1-sync.md").read_text() == new_step
|
||||
assert (project / "modules" / "demo-flow" / "commands" / "setup.md").read_text() == "local edit"
|
||||
assert (project / "modules" / "demo-flow" / "index.md.new").exists()
|
||||
assert (project / "modules" / "demo-flow" / "steps" / "2-verify.md").exists()
|
||||
assert "Conflicts" in result or "conflicts" in result.lower()
|
||||
|
||||
|
||||
def test_update_without_manifest_errors(registry, project):
|
||||
mod = project / "modules" / "handmade"
|
||||
mod.mkdir(parents=True)
|
||||
(mod / "index.md").write_text("# handmade\n")
|
||||
result = server.workflow(action="update", id="handmade")
|
||||
assert result.startswith("Error:")
|
||||
assert ".template.yaml" in result
|
||||
|
||||
|
||||
def test_update_missing_id(registry, project):
|
||||
result = server.workflow(action="update")
|
||||
assert result.startswith("Error:")
|
||||
assert "needs an id" in result
|
||||
|
||||
|
||||
# --- CLI wrapper tests ---
|
||||
|
||||
def _run_cli(*args, cwd, env=None):
|
||||
return subprocess.run(
|
||||
[sys.executable, "-m", "gcontext.cli", *args],
|
||||
capture_output=True, text=True, cwd=cwd, env=env,
|
||||
)
|
||||
|
||||
|
||||
def test_cli_search(registry, tmp_path):
|
||||
registry[0] = _tarball_with_catalog()
|
||||
agent = tmp_path / "a"
|
||||
_run_cli("init", "a", cwd=tmp_path)
|
||||
result = _run_cli("search", "demo", cwd=agent)
|
||||
assert result.returncode == 0
|
||||
assert "demo-flow" in result.stdout
|
||||
|
||||
|
||||
def test_cli_update_up_to_date(registry, tmp_path):
|
||||
registry[0] = _tarball_with_catalog()
|
||||
agent = tmp_path / "a"
|
||||
_run_cli("init", "a", cwd=tmp_path)
|
||||
_run_cli("add", "demo-flow", cwd=agent)
|
||||
result = _run_cli("update", "demo-flow", cwd=agent)
|
||||
assert result.returncode == 0
|
||||
assert "up to date" in result.stdout
|
||||
|
||||
|
||||
def test_cli_update_unknown_module(registry, tmp_path):
|
||||
registry[0] = _tarball_with_catalog()
|
||||
agent = tmp_path / "a"
|
||||
_run_cli("init", "a", cwd=tmp_path)
|
||||
result = _run_cli("update", "nope", cwd=agent)
|
||||
assert result.returncode == 1
|
||||
assert "Error:" in result.stderr
|
||||
|
||||
|
||||
# --- Helpers ---
|
||||
|
||||
def _registry_files_from(bundle_files, workflow_id="demo-flow"):
|
||||
return [{"path": f"{workflow_id}/{f['path']}", "content": f["content"]} for f in bundle_files]
|
||||
Reference in New Issue
Block a user