commit baf1e740bed3add8c885bc00e8bf4bea39a29856 Author: Lukasz Raczylo Date: Fri Jun 12 02:48:30 2026 +0100 initial commit diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..b63d250 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,5 @@ +bin/ +.git/ +*.md +Dockerfile +.dockerignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..17217fd --- /dev/null +++ b/.gitignore @@ -0,0 +1,11 @@ +bin/ +sessions/ +.DS_Store +*.test +*.out + +# Go +vendor/ + +# Local session data (defensive: keep developers' auth out of git) +.mcp-readitall/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..0028b50 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,19 @@ +# Build stage +FROM golang:1.26-alpine AS build +WORKDIR /src +COPY go.mod go.sum ./ +RUN go mod download +COPY . . +RUN CGO_ENABLED=0 go build -trimpath -ldflags="-s -w" -o /out/readitall ./cmd/readitall + +# Runtime: chromedp/headless-shell is distroless-based and ships only +# headless Chrome at /headless-shell/headless-shell (on PATH, where +# chromedp's exec allocator finds it). +FROM chromedp/headless-shell:latest +LABEL org.opencontainers.image.source="https://github.com/lukaszraczylo/mcp-readitall" +COPY --from=build /out/readitall /usr/local/bin/readitall +ENV READITALL_SESSIONS_DIR=/data \ + READITALL_ADDR=:8080 +VOLUME ["/data"] +EXPOSE 8080 +ENTRYPOINT ["/usr/local/bin/readitall"] diff --git a/README.md b/README.md new file mode 100644 index 0000000..e6a23ab --- /dev/null +++ b/README.md @@ -0,0 +1,208 @@ +# mcp-readitall + +A Go-based [Model Context Protocol](https://modelcontextprotocol.io) (MCP) +server that fetches any website — including modern SPAs that need a real +browser to render — and converts the page into clean Markdown. It also +supports per-host authenticated sessions so you can read content that lives +behind a login wall. + +The server speaks MCP over **stdio** and is designed to be wired into any +MCP-aware client (Claude Desktop, Claude Code, Cursor, etc.). + +## Features + +- **Real browser rendering** via [chromedp] (a pure-Go Chrome DevTools + Protocol client) + system Chromium — handles SPAs, JS hydration, + lazy-loaded content, client-side routing. No Node.js bridge. +- **Markdown conversion** with the [html-to-markdown] plugin pipeline: + tables, code blocks (with language detection), strikethrough, commonmark + semantics, relative-URL resolution. +- **Per-host authenticated sessions** — the `login` tool opens a visible + browser window for you to authenticate, then persists cookies + + localStorage so future reads reuse the session automatically. +- **Session lifecycle tools** — `list_sessions` and `clear_session` for + inspection and revocation. +- **Content extraction** — `extract` and the `selector` parameter on + `read_url` let you pull just the part of a page you care about. +- **Cross-process safe** — session state is persisted to disk (0600) so the + server can restart without losing auth. + +[html-to-markdown]: https://github.com/JohannesKaufmann/html-to-markdown +[chromedp]: https://github.com/chromedp/chromedp + +## Tools + +| Tool | Purpose | +| --- | --- | +| `read_url` | Fetch a URL (with optional CSS selector), render JS, return Markdown. | +| `login` | Open a browser, let the operator log in, persist the session. | +| `list_sessions` | Show every host for which a session is saved. | +| `clear_session` | Delete a saved session. | +| `extract` | Read a URL and return only the part under a CSS selector. | + +### `read_url` arguments + +| Field | Type | Description | +| --- | --- | --- | +| `url` | string (required) | Absolute http(s) URL. | +| `selector` | string | Optional CSS selector to extract a region. | +| `wait_for` | string | Optional CSS selector to wait for (useful for SPAs that hydrate after `networkidle`). | +| `timeout_seconds` | int | Overall timeout. Default 30. | +| `max_chars` | int | Truncate the returned Markdown. Default ~200k. | + +### `login` arguments + +| Field | Type | Description | +| --- | --- | --- | +| `url` | string (required) | The URL to open (typically the login page). | +| `success_url` | string | Substring of the post-login URL (e.g. `/dashboard`). Polled every second. | +| `success_selector` | string | CSS selector that becomes visible once auth succeeds (e.g. an avatar menu). | +| `timeout_seconds` | int | How long to wait for the operator to complete login. Default 300. | + +`success_url` **or** `success_selector` is required so the tool knows when +to stop waiting and save the session. + +## Build & install + +Requirements: + +- Go 1.22+ +- A Chromium-family browser on `$PATH` or installed at the standard + location — `google-chrome`, `chromium`, `chromium-browser`, or Edge. + macOS: `brew install --cask chromium` (or the standard Chrome install). + Linux: `apt install chromium` / `dnf install chromium`. + +```bash +# Build the server binary +go build -o bin/mcp-readitall ./cmd/readitall +``` + +> If the project is not under git you may need `-buildvcs=false`: +> `go build -buildvcs=false -o bin/mcp-readitall ./cmd/readitall` + +chromedp will auto-discover Chromium via [`exec.LookPath`] and the usual +Chromium env vars (`CHROME_BIN`, `CHROMIUM_BIN`, etc.). If you have a +non-standard install, point at the binary directly: + +```bash +CHROME_BIN=/Applications/Chromium.app/Contents/MacOS/Chromium ./bin/mcp-readitall +``` + +[`exec.LookPath`]: https://pkg.go.dev/os/exec#LookPath + +## Wire into an MCP client + +### Claude Desktop (`~/Library/Application Support/Claude/claude_desktop_config.json`) + +```json +{ + "mcpServers": { + "readitall": { + "command": "/absolute/path/to/mcp-readitall/bin/mcp-readitall" + } + } +} +``` + +### Claude Code (`.mcp.json` in your project) + +```json +{ + "mcpServers": { + "readitall": { + "command": "/absolute/path/to/mcp-readitall/bin/mcp-readitall" + } + } +} +``` + +Restart the client after editing the config. + +## Usage flow + +### 1. Read a public page + +Ask your MCP client: + +> "Read https://news.ycombinator.com and summarize the top 5 stories" + +Under the hood the model calls `read_url`. The server launches a +headless Chromium, navigates, waits for the body to be ready (plus a +short grace period for hydration, or a `wait_for` selector if supplied), +converts the HTML to Markdown, and returns it. + +### 2. Read an authenticated page + +> "Log me into GitHub, then read my notifications" + +The model first calls: + +```json +{ + "name": "login", + "arguments": { + "url": "https://github.com/login", + "success_url": "/", + "timeout_seconds": 300 + } +} +``` + +A real browser window pops up on your machine. Log in normally. When the +URL changes to contain `/`, the tool saves the session and closes the +window. Subsequent calls to `read_url` for `github.com` will reuse the +saved cookies + localStorage automatically. + +To invalidate a session: + +> "Clear my session for github.com" + +The model calls `clear_session`. + +## Session storage + +Saved sessions live in: + +- `~/.mcp-readitall/sessions//state.json` by default +- Override with `READITALL_SESSIONS_DIR=/some/path` + +State files are written with mode `0600` (owner read/write only) and +use Playwright's standard storage state shape (`{cookies:[…], origins:[{origin, +localStorage:[…]}]}`) — sessions captured by Playwright can be loaded by +us, and vice versa. Treat them as secrets — they grant access to your +authenticated sessions. + +## Tests + +```bash +go test -race ./... +``` + +The suite is hermetic — no browser is launched. It covers: + +- Hostname normalization (good / bad / edge). +- Storage-state validation. +- Session-store CRUD and concurrency safety. +- HTML→Markdown conversion (headings, lists, tables, code, entities, URL + resolution, security-sensitive stripping of `

also visible

`, + want: []string{"visible", "also visible"}, + notWant: []string{ + "alert(1)", + "p{}", + }, + }, + { + name: "html comments are stripped", + html: `

before

after

`, + want: []string{"before", "after"}, + notWant: []string{ + "a comment", + }, + }, + { + name: "nested tags are flattened correctly", + html: `

outer with nested emphasis

`, + want: []string{"outer **with _nested_ emphasis**"}, + }, + + // ===== EDGE CASES ===== + { + name: "empty string", + html: "", + want: []string{}, + }, + { + name: "whitespace only", + html: " \n\t ", + want: []string{}, + }, + { + name: "plain text without tags", + html: `just plain text`, + want: []string{"just plain text"}, + }, + { + name: "html entities are decoded in text", + html: `

5 < 10 & 10 > 5

`, + // In EscapeModeSmart the library decodes & to & but + // preserves </> as entities so the output cannot be + // interpreted as raw HTML/Markdown by downstream renderers. + want: []string{"5 < 10 & 10 > 5"}, + }, + { + name: "deeply nested lists", + html: ``, + want: []string{"- one", "nested", "- two"}, + }, + { + name: "self-closing void elements do not break parser", + html: `

line one
line two

line four

`, + want: []string{"line one", "line two", "line four"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + got, err := reader.HTMLToMarkdown(tt.html, tt.domain) + require.NoError(t, err) + + // Trim trailing whitespace from each want so test data is + // easier to read; markdown output may add trailing newlines. + for _, w := range tt.want { + assert.Contains(t, got, w, "expected substring %q in output:\n%s", w, got) + } + for _, nw := range tt.notWant { + assert.NotContains(t, got, nw, "unexpected substring %q in output:\n%s", nw, got) + } + }) + } +} + +func TestHTMLToMarkdown_RealisticArticle(t *testing.T) { + t.Parallel() + html := ` + + +Sample + + +
+

How to Test

+

Testing is important. See the docs.

+

Setup

+
    +
  1. Install Go
    2. +
  2. Run go test ./...
    3. +
+

Example

+ 
package main
+import "fmt"
+func main() { fmt.Println("hi") }
+
+ + + +
InputOutput
1one
+
+
Copyright 2026
+ +` + + md, err := reader.HTMLToMarkdown(html, "https://example.test/blog") + require.NoError(t, err) + + // Spot-check the key transformations + mustContain := []string{ + "# How to Test", + "Testing is **important**", + "[the docs](https://example.test/docs)", + "## Setup", + "1. Install Go", + "2. Run `go test ./...`", + "## Example", + "```", + "package main", + "| Input | Output |", + "| 1 | one |", + } + for _, s := range mustContain { + assert.Contains(t, md, s, "missing %q in:\n%s", s, md) + } +} + +func TestHTMLToMarkdown_DoesNotLeakScriptOrStyle(t *testing.T) { + t.Parallel() + html := `

safe

+ + +

also safe

` + + md, err := reader.HTMLToMarkdown(html, "") + require.NoError(t, err) + + // Output should not contain the malicious payload + assert.NotContains(t, md, "ABCDEFG") + assert.NotContains(t, md, "evil.test") + assert.NotContains(t, md, "document.cookie") + + // But it should still contain the visible text + assert.Contains(t, strings.ToLower(md), "safe") + assert.Contains(t, strings.ToLower(md), "also safe") +} diff --git a/internal/reader/reader.go b/internal/reader/reader.go new file mode 100644 index 0000000..8358bc5 --- /dev/null +++ b/internal/reader/reader.go @@ -0,0 +1,166 @@ +// Package reader fetches pages through a headless browser, waits for the +// page to settle (which is what makes SPA support work), and converts the +// rendered HTML to markdown. It reuses per-host session state when +// available so authenticated pages render as the logged-in user. +package reader + +import ( + "context" + "errors" + "fmt" + "log" + "strings" + "time" + + "github.com/chromedp/chromedp" + "github.com/lukaszraczylo/mcp-readitall/internal/browser" +) + +// Default values exposed so tool handlers can reference them. +const ( + DefaultTimeout = 30 * time.Second + MaxMarkdown = 200_000 // hard ceiling on returned markdown size +) + +// Reader is a thin facade over the Browser wrapper and session store. +type Reader struct { + pw *browser.Browser + sess *browser.SessionStore + log *log.Logger +} + +// New constructs a Reader. The store is consulted on every read to decide +// whether to attach a saved storage state to the new browser context. +func New(pw *browser.Browser, sess *browser.SessionStore, logger *log.Logger) *Reader { + if logger == nil { + logger = log.Default() + } + return &Reader{pw: pw, sess: sess, log: logger} +} + +// ReadResult is the normalised output of a read operation. +type ReadResult struct { + URL string `json:"url"` + FinalURL string `json:"final_url"` + Title string `json:"title"` + Markdown string `json:"markdown"` + UsedSession bool `json:"used_session"` + Truncated bool `json:"truncated"` + Bytes int `json:"bytes"` +} + +// ReadOptions controls a single Read call. +type ReadOptions struct { + URL string // page to fetch + Selector string // optional CSS selector; if set, content is extracted from the matching element(s) only + WaitSelector string // optional CSS selector to wait for before extracting (useful for SPA hydration) + Timeout time.Duration // overall timeout for navigation + waits + MaxChars int // truncate markdown to roughly this many characters (0 = no truncation) +} + +// Read fetches the URL through Chromium, waits for it to settle, optionally +// narrows the content with a CSS selector, and converts the resulting HTML +// to markdown. +func (r *Reader) Read(parentCtx context.Context, opts ReadOptions) (*ReadResult, error) { + if opts.URL == "" { + return nil, errors.New("url is required") + } + if opts.Timeout == 0 { + opts.Timeout = DefaultTimeout + } + if opts.MaxChars == 0 { + opts.MaxChars = MaxMarkdown + } + + timeoutCtx, cancel := context.WithTimeout(parentCtx, opts.Timeout) + defer cancel() + + usedSession, statePath, err := r.resolveSession(opts.URL) + if err != nil { + return nil, err + } + + cdpCtx, releaseBrowser, err := r.pw.NewContext(timeoutCtx, statePath) + if err != nil { + return nil, err + } + defer releaseBrowser() + + r.log.Printf("read %s (session=%v)", opts.URL, usedSession) + + var title, finalURL string + var html string + + // Build the action chain. Cookies were already applied by NewContext + // (if a session exists). localStorage is applied post-navigation + // because it is origin-scoped. + actions := []chromedp.Action{ + chromedp.Navigate(opts.URL), + chromedp.WaitReady("body"), + } + if statePath != "" { + actions = append(actions, chromedp.ActionFunc(func(ctx context.Context) error { + return browser.ApplyLocalStorageOnCurrentOrigin(ctx, statePath) + })) + } + // Small grace period so post-load JS (lazy components, hydration) has + // a chance to settle before we snapshot the HTML. + actions = append(actions, + chromedp.Sleep(500*time.Millisecond), + ) + if opts.WaitSelector != "" { + actions = append(actions, chromedp.WaitVisible(opts.WaitSelector, chromedp.ByQuery)) + } + actions = append(actions, + chromedp.Title(&title), + chromedp.Location(&finalURL), + ) + + if err := chromedp.Run(cdpCtx, actions...); err != nil { + return nil, fmt.Errorf("navigate: %w", err) + } + + html, err = r.extractHTML(cdpCtx, opts.Selector) + if err != nil { + return nil, err + } + + md, err := HTMLToMarkdown(html, finalURL) + if err != nil { + return nil, fmt.Errorf("convert to markdown: %w", err) + } + + truncated := false + if len(md) > opts.MaxChars { + md = md[:opts.MaxChars] + "\n\n…[truncated]" + truncated = true + } + + return &ReadResult{ + URL: opts.URL, + FinalURL: finalURL, + Title: strings.TrimSpace(title), + Markdown: md, + UsedSession: usedSession, + Truncated: truncated, + Bytes: len(md), + }, nil +} + +// resolveSession checks whether a saved session exists for the URL's host +// and returns its path. An empty path means "no session, use a fresh +// context". +func (r *Reader) resolveSession(rawURL string) (bool, string, error) { + exists, err := r.sess.Exists(rawURL) + if err != nil { + return false, "", err + } + if !exists { + return false, "", nil + } + path, err := r.sess.StatePath(rawURL) + if err != nil { + return false, "", err + } + return true, path, nil +} diff --git a/internal/server/integration_test.go b/internal/server/integration_test.go new file mode 100644 index 0000000..687a0aa --- /dev/null +++ b/internal/server/integration_test.go @@ -0,0 +1,14 @@ +//go:build integration + +// Package integration_test exercises the MCP server end-to-end through a +// real chromedp-managed Chromium. It is gated by the `integration` build +// tag so the default `go test ./...` run stays hermetic. +// +// Run with: +// +// go test -tags=integration -race -v ./internal/server/... +// +// Make sure a Chromium or Chrome binary is on the system (chromedp will +// find it via the default Chrome discovery rules; override with +// chromedp.ExecPath if needed). +package server diff --git a/internal/server/server.go b/internal/server/server.go new file mode 100644 index 0000000..1ac9a7c --- /dev/null +++ b/internal/server/server.go @@ -0,0 +1,130 @@ +package server + +import ( + "context" + "fmt" + "log" + "net/http" + "sync" + "time" + + "github.com/lukaszraczylo/mcp-readitall/internal/browser" + "github.com/lukaszraczylo/mcp-readitall/internal/reader" + "github.com/lukaszraczylo/mcp-readitall/internal/tools" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +// Server wires the MCP server, browser lifecycle, session store, and reader +// together, and exposes the tool surface to MCP clients. +type Server struct { + mcp *mcp.Server + br *browser.Browser + sess *browser.SessionStore + readr *reader.Reader + log *log.Logger + + closeOnce sync.Once + closeErr error +} + +const ( + serverName = "mcp-readitall" + serverVersion = "0.1.0" +) + +// New constructs the MCP server, initialises the browser manager and the +// session store, and registers every tool. It does not start serving: call +// RunHTTP for that. +func New(ctx context.Context, logger *log.Logger) (*Server, error) { + if logger == nil { + logger = log.Default() + } + + sess, err := browser.NewSessionStore() + if err != nil { + return nil, fmt.Errorf("session store: %w", err) + } + + br, err := browser.New(ctx, logger) + if err != nil { + _ = sess.Close() + return nil, fmt.Errorf("browser: %w", err) + } + + readr := reader.New(br, sess, logger) + + mcpServer := mcp.NewServer( + &mcp.Implementation{Name: serverName, Version: serverVersion}, + nil, + ) + + s := &Server{ + mcp: mcpServer, + br: br, + sess: sess, + readr: readr, + log: logger, + } + + tools.RegisterRead(mcpServer, readr) + tools.RegisterLogin(mcpServer, br, sess, logger) + tools.RegisterListSessions(mcpServer, sess) + tools.RegisterClearSession(mcpServer, sess) + tools.RegisterExtract(mcpServer, readr) + + return s, nil +} + +// RunHTTP blocks and serves MCP over HTTP until ctx is cancelled. It exposes +// the streamable HTTP transport at /mcp, the legacy SSE transport at /sse, +// and a liveness probe at /healthz. +func (s *Server) RunHTTP(ctx context.Context, addr string) error { + getServer := func(*http.Request) *mcp.Server { return s.mcp } + + mux := http.NewServeMux() + mux.Handle("/mcp", mcp.NewStreamableHTTPHandler(getServer, nil)) + mux.Handle("/sse", mcp.NewSSEHandler(getServer, nil)) + mux.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("ok")) + }) + + httpSrv := &http.Server{ + Addr: addr, + Handler: mux, + ReadHeaderTimeout: 10 * time.Second, + } + + errCh := make(chan error, 1) + go func() { errCh <- httpSrv.ListenAndServe() }() + + select { + case <-ctx.Done(): + shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + return httpSrv.Shutdown(shutdownCtx) + case err := <-errCh: + return err + } +} + +// Close releases the browser and the session store. Safe to call multiple times. +func (s *Server) Close() error { + s.closeOnce.Do(func() { + var firstErr error + if s.br != nil { + if err := s.br.Close(); err != nil { + firstErr = err + s.log.Printf("browser close: %v", err) + } + } + if s.sess != nil { + if err := s.sess.Close(); err != nil && firstErr == nil { + firstErr = err + s.log.Printf("session store close: %v", err) + } + } + s.closeErr = firstErr + }) + return s.closeErr +} diff --git a/internal/tools/extract.go b/internal/tools/extract.go new file mode 100644 index 0000000..6f0aa0b --- /dev/null +++ b/internal/tools/extract.go @@ -0,0 +1,101 @@ +package tools + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/lukaszraczylo/mcp-readitall/internal/reader" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +// ExtractInput is the schema for the extract tool. It is essentially +// read_url with a default selector set to
when omitted, so common CMS +// layouts work without ceremony. +type ExtractInput struct { + URL string `json:"url" jsonschema:"Absolute http(s) URL to read"` + Selector string `json:"selector,omitempty" jsonschema:"CSS selector to extract. Defaults to 'main, article, [role=main]' which works for most content sites."` + WaitFor string `json:"wait_for,omitempty" jsonschema:"Optional CSS selector to wait for before extracting (useful for SPAs)"` + TimeoutSeconds int `json:"timeout_seconds,omitempty" jsonschema:"Overall timeout in seconds (1-300, default 30)"` + MaxChars int `json:"max_chars,omitempty" jsonschema:"Truncate returned markdown to roughly this many characters (100-500000)"` +} + +// ExtractOutput is the structured result. +type ExtractOutput struct { + URL string `json:"url"` + FinalURL string `json:"final_url"` + Title string `json:"title"` + Markdown string `json:"markdown"` + Selector string `json:"selector"` + UsedSession bool `json:"used_session"` + Truncated bool `json:"truncated"` + Bytes int `json:"bytes"` +} + +// RegisterExtract wires the extract tool to the server. +func RegisterExtract(s *mcp.Server, r *reader.Reader) { + mcp.AddTool(s, &mcp.Tool{ + Name: "extract", + Description: "Fetch a URL and return only the content under one (or more, comma-separated) CSS selectors, converted to Markdown. This is the right tool when you want the page's main body without nav, headers, footers, and sidebars. Works on SPA and SSR sites.", + }, func(ctx context.Context, _ *mcp.CallToolRequest, in ExtractInput) (*mcp.CallToolResult, ExtractOutput, error) { + if in.URL == "" { + return nil, ExtractOutput{}, fmt.Errorf("url is required") + } + selector := in.Selector + if selector == "" { + selector = "main, article, [role=main]" + } + timeout := time.Duration(in.TimeoutSeconds) * time.Second + + res, err := r.Read(ctx, reader.ReadOptions{ + URL: in.URL, + Selector: firstSelector(selector), + WaitSelector: in.WaitFor, + Timeout: timeout, + MaxChars: in.MaxChars, + }) + if err != nil { + return nil, ExtractOutput{}, err + } + out := ExtractOutput{ + URL: res.URL, + FinalURL: res.FinalURL, + Title: res.Title, + Markdown: res.Markdown, + Selector: selector, + UsedSession: res.UsedSession, + Truncated: res.Truncated, + Bytes: res.Bytes, + } + var b strings.Builder + b.WriteString("# ") + b.WriteString(out.Title) + if out.Title == "" { + b.WriteString(out.URL) + } + b.WriteString("\n\n") + b.WriteString(out.Markdown) + if out.Truncated { + b.WriteString("\n\n[truncated]") + } + return &mcp.CallToolResult{ + Content: []mcp.Content{&mcp.TextContent{Text: b.String()}}, + }, out, nil + }) +} + +// firstSelector picks the first non-empty selector from a comma-separated +// list. The reader is intentionally simple and only supports a single +// selector today; we surface the first to keep the schema natural for +// humans ("main, article, [role=main]") without pretending to support +// union selectors internally. Empty entries are skipped so leading or +// consecutive commas (" , main") do not produce a blank selector. +func firstSelector(selectorList string) string { + for _, p := range strings.Split(selectorList, ",") { + if s := strings.TrimSpace(p); s != "" { + return s + } + } + return "" +} diff --git a/internal/tools/helpers_test.go b/internal/tools/helpers_test.go new file mode 100644 index 0000000..b5e6351 --- /dev/null +++ b/internal/tools/helpers_test.go @@ -0,0 +1,273 @@ +// White-box tests for unexported helpers in the tools package. We use +// `package tools` (not `tools_test`) so we can reach readText, firstSelector, +// and hostOf directly. These helpers are the bits of logic that don't +// require a browser or MCP plumbing, so they're cheap to cover. +package tools + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +// ============================================================================= +// readText +// ============================================================================= + +func TestReadText(t *testing.T) { + tests := []struct { + name string + in ReadOutput + mustHave []string + mustNot []string + }{ + // ===== GOOD CASES ===== + { + name: "title and body", + in: ReadOutput{ + URL: "https://example.test/", + Title: "Example Domain", + Markdown: "Hello **world**.", + }, + mustHave: []string{ + "# Example Domain", + "Hello **world**.", + }, + }, + { + name: "falls back to URL when title empty", + in: ReadOutput{ + URL: "https://example.test/", + Markdown: "body", + }, + mustHave: []string{ + "# https://example.test/", + "body", + }, + }, + { + name: "truncated flag is surfaced", + in: ReadOutput{ + URL: "https://x.test/", + Title: "T", + Markdown: "cut", + Truncated: true, + }, + mustHave: []string{ + "# T", + "cut", + "[truncated]", + }, + }, + { + name: "redirect is shown only when final URL differs", + in: ReadOutput{ + URL: "https://x.test/old", + FinalURL: "https://x.test/new", + Title: "T", + Markdown: "body", + }, + mustHave: []string{ + "_Resolved to: https://x.test/new_", + }, + }, + { + name: "no redirect note when URLs match", + in: ReadOutput{ + URL: "https://x.test/", + FinalURL: "https://x.test/", + Title: "T", + Markdown: "body", + }, + mustNot: []string{ + "Resolved to", + }, + }, + { + name: "session-used flag is surfaced", + in: ReadOutput{ + URL: "https://x.test/", + Title: "T", + Markdown: "body", + UsedSession: true, + }, + mustHave: []string{ + "_Used saved session._", + }, + }, + + // ===== EDGE CASES ===== + { + name: "empty body still renders header", + in: ReadOutput{ + URL: "https://x.test/", + Title: "T", + }, + mustHave: []string{ + "# T", + }, + }, + { + name: "redirect + session flags combined", + in: ReadOutput{ + URL: "https://x.test/old", + FinalURL: "https://x.test/new", + Title: "T", + Markdown: "body", + UsedSession: true, + Truncated: true, + }, + mustHave: []string{ + "_Resolved to: https://x.test/new_", + "_Used saved session._", + "[truncated]", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + got := readText(tt.in) + for _, s := range tt.mustHave { + assert.Contains(t, got, s, "missing %q in:\n%s", s, got) + } + for _, s := range tt.mustNot { + assert.NotContains(t, got, s, "unexpected %q in:\n%s", s, got) + } + }) + } +} + +// ============================================================================= +// firstSelector +// ============================================================================= + +func TestFirstSelector(t *testing.T) { + tests := []struct { + name string + in string + want string + }{ + // ===== GOOD CASES ===== + { + name: "single selector", + in: "main", + want: "main", + }, + { + name: "comma list returns first", + in: "main, article, [role=main]", + want: "main", + }, + { + name: "whitespace around first entry is trimmed", + in: " #content , aside", + want: "#content", + }, + + // ===== EDGE CASES ===== + { + name: "trailing comma", + in: "main,", + want: "main", + }, + { + name: "leading comma", + in: ",main", + want: "main", + }, + { + name: "multiple consecutive commas", + in: ",,main,,article", + want: "main", + }, + { + name: "empty string", + in: "", + want: "", + }, + { + name: "only commas", + in: ",,,", + want: "", + }, + { + name: "selector with internal spaces is preserved", + in: "div[data-role=main content], aside", + want: "div[data-role=main content]", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + got := firstSelector(tt.in) + assert.Equal(t, tt.want, got) + }) + } +} + +// ============================================================================= +// hostOf +// ============================================================================= + +func TestHostOf(t *testing.T) { + tests := []struct { + name string + in string + want string + }{ + // ===== GOOD CASES ===== + { + name: "https with path", + in: "https://example.com/some/path", + want: "example.com", + }, + { + name: "subdomain", + in: "https://app.example.co.uk", + want: "app.example.co.uk", + }, + { + name: "with port", + in: "http://localhost:8080/x", + want: "localhost:8080", + }, + { + name: "uppercase host is preserved (url.Parse does not lowercase)", + in: "https://Example.COM/x", + want: "Example.COM", + }, + + // ===== BAD / EDGE CASES ===== + { + name: "empty string returns unknown", + in: "", + want: "unknown", + }, + { + name: "garbage returns unknown (does not panic)", + in: "::::", + want: "unknown", + }, + { + name: "scheme-only returns unknown", + in: "https://", + want: "unknown", + }, + { + name: "path-only returns unknown", + in: "/just/a/path", + want: "unknown", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + got := hostOf(tt.in) + assert.Equal(t, tt.want, got) + }) + } +} diff --git a/internal/tools/login.go b/internal/tools/login.go new file mode 100644 index 0000000..f4ca4f4 --- /dev/null +++ b/internal/tools/login.go @@ -0,0 +1,225 @@ +package tools + +import ( + "context" + "fmt" + "log" + "os" + "strings" + "time" + + "github.com/chromedp/cdproto/cdp" + "github.com/chromedp/chromedp" + "github.com/lukaszraczylo/mcp-readitall/internal/browser" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +// LoginInput is the schema for the login tool. The flow is: +// +// 1. A visible Chromium window opens and navigates to URL. +// 2. The human operator (you) authenticates by hand in that window. +// 3. The tool watches for SuccessURL (substring match against the +// address bar) or SuccessSelector (a CSS selector that becomes +// visible, e.g. an avatar dropdown) and, when matched, persists the +// browser's storage state to the canonical session path for the +// host. +// 4. Future read_url calls against this host will reuse the saved +// state. +// +// On a headless host (e.g. a remote MCP server) the visible-window +// approach will not work; the operator must run the MCP server on a +// machine with a display. We surface a clear error in that case so the +// caller knows the limitation is environmental, not a bug. +type LoginInput struct { + URL string `json:"url" jsonschema:"Absolute http(s) URL to open (typically the login page)"` + SuccessURL string `json:"success_url,omitempty" jsonschema:"Substring of the URL bar that indicates login succeeded (e.g. /dashboard). Polled every second."` + SuccessSelector string `json:"success_selector,omitempty" jsonschema:"CSS selector that becomes visible once login is complete (e.g. an avatar menu). Polled every second."` + TimeoutSeconds int `json:"timeout_seconds,omitempty" jsonschema:"How long to wait for a successful login in seconds (5-3600, default 300)"` +} + +// LoginOutput reports what happened. +type LoginOutput struct { + Domain string `json:"domain"` + Saved bool `json:"saved"` + Path string `json:"path,omitempty"` + FinalURL string `json:"final_url"` + Message string `json:"message"` +} + +// RegisterLogin wires the login tool to the server. +func RegisterLogin(s *mcp.Server, pw *browser.Browser, sess *browser.SessionStore, logger *log.Logger) { + mcp.AddTool(s, &mcp.Tool{ + Name: "login", + Description: "Open a visible browser to a URL, wait for the human operator to authenticate, then persist the resulting browser storage state (cookies + localStorage) for the host. Subsequent read_url calls against this host will use the saved state automatically. Useful for sites that require login to expose their content.", + }, func(parentCtx context.Context, _ *mcp.CallToolRequest, in LoginInput) (*mcp.CallToolResult, LoginOutput, error) { + if in.URL == "" { + return nil, LoginOutput{}, fmt.Errorf("url is required") + } + if in.SuccessURL == "" && in.SuccessSelector == "" { + return nil, LoginOutput{}, fmt.Errorf("success_url or success_selector is required so the tool knows when login is complete") + } + timeout := time.Duration(in.TimeoutSeconds) * time.Second + if timeout == 0 { + timeout = 5 * time.Minute + } + + ctx, cancel := context.WithTimeout(parentCtx, timeout) + defer cancel() + + browserCtx, releaseBrowser, err := pw.NewHeadedContext() + if err != nil { + return nil, LoginOutput{}, fmt.Errorf("launch headed browser: %w", err) + } + defer releaseBrowser() + + // Start from a clean cookie jar so a stale session from a + // previous attempt cannot mask the new login's success. + if err := browser.ClearAllCookies(browserCtx); err != nil { + return nil, LoginOutput{}, fmt.Errorf("clear cookies: %w", err) + } + + var title string + if err := chromedp.Run(browserCtx, + chromedp.Navigate(in.URL), + chromedp.WaitReady("body"), + chromedp.Title(&title), + ); err != nil { + return nil, LoginOutput{}, fmt.Errorf("navigate: %w", err) + } + + logger.Printf("login flow started for %s, waiting for success condition", in.URL) + + if err := waitForLogin(ctx, browserCtx, in, timeout); err != nil { + return nil, LoginOutput{ + Domain: hostOf(in.URL), + FinalURL: currentLocation(browserCtx), + Message: err.Error(), + }, fmt.Errorf("login did not complete: %w", err) + } + + finalURL := currentLocation(browserCtx) + + statePath, err := sess.StatePath(in.URL) + if err != nil { + return nil, LoginOutput{}, err + } + if err := browser.CaptureStorageState(browserCtx, statePath); err != nil { + return nil, LoginOutput{ + Domain: hostOf(in.URL), + FinalURL: finalURL, + Message: "success condition matched but capturing storage state failed", + }, err + } + + // Validate the captured state has at least one cookie or origin + // entry, otherwise we would happily save an "authenticated" file + // even when the success condition matched an unrelated selector. + if err := validateCapturedState(statePath); err != nil { + return nil, LoginOutput{ + Domain: hostOf(in.URL), + FinalURL: finalURL, + Message: "success condition matched but storage state is empty; auth likely failed", + }, err + } + + msg := fmt.Sprintf("session saved for %s", hostOf(in.URL)) + return &mcp.CallToolResult{ + Content: []mcp.Content{&mcp.TextContent{Text: msg}}, + }, LoginOutput{ + Domain: hostOf(in.URL), + Saved: true, + Path: statePath, + FinalURL: finalURL, + Message: msg, + }, nil + }) +} + +// waitForLogin polls the page once per second until the success +// condition is met or the timeout fires. It uses chromedp.Location and +// chromedp.Nodes (with a visibility check) for the two supported +// condition types. +func waitForLogin(ctx context.Context, pageCtx context.Context, in LoginInput, timeout time.Duration) error { + deadline := time.Now().Add(timeout) + ticker := time.NewTicker(time.Second) + defer ticker.Stop() + + check := func() (bool, error) { + if in.SuccessURL != "" { + var current string + if err := chromedp.Run(pageCtx, chromedp.Location(¤t)); err != nil { + return false, err + } + if strings.Contains(current, in.SuccessURL) { + return true, nil + } + } + if in.SuccessSelector != "" { + var nodes []*cdp.Node + if err := chromedp.Run(pageCtx, chromedp.Nodes(in.SuccessSelector, &nodes, chromedp.ByQuery)); err != nil { + return false, err + } + if len(nodes) > 0 { + var visible bool + if err := chromedp.Run(pageCtx, chromedp.Evaluate( + `(()=>{const e=document.querySelector(`+jsQuote(in.SuccessSelector)+`);return !!e && e.getBoundingClientRect().height>0;})()`, + &visible, + )); err != nil { + return false, err + } + if visible { + return true, nil + } + } + } + return false, nil + } + + // Immediate check so an already-authenticated session is captured + // without a 1-second wait. + if ok, err := check(); err != nil { + return err + } else if ok { + return nil + } + + for { + select { + case <-ctx.Done(): + return ctx.Err() + case <-ticker.C: + if time.Now().After(deadline) { + return fmt.Errorf("timed out after %s waiting for login", timeout) + } + ok, err := check() + if err != nil { + return err + } + if ok { + return nil + } + } + } +} + +// currentLocation returns the page's current URL or "" if the query +// fails (e.g. the browser has already been torn down). +func currentLocation(ctx context.Context) string { + var s string + if err := chromedp.Run(ctx, chromedp.Location(&s)); err != nil { + return "" + } + return s +} + +// validateCapturedState reads the just-saved storage state file and +// ensures it has at least one cookie or one localStorage entry. This +// guards against "login success" false positives (e.g. selector matches +// a generic header element that happens to be visible). +func validateCapturedState(path string) error { + raw, err := os.ReadFile(path) + if err != nil { + return fmt.Errorf("read state: %w", err) + } + return browser.ValidateStorageState(raw) +} diff --git a/internal/tools/read.go b/internal/tools/read.go new file mode 100644 index 0000000..0276d4e --- /dev/null +++ b/internal/tools/read.go @@ -0,0 +1,96 @@ +// Package tools registers each MCP tool with the server. One file per tool +// keeps the schemas, handlers, and tool registration together so changes are +// easy to audit. +package tools + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/lukaszraczylo/mcp-readitall/internal/reader" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +// ReadInput is the JSON schema for the read_url tool. +type ReadInput struct { + URL string `json:"url" jsonschema:"Absolute http(s) URL to read"` + Selector string `json:"selector,omitempty" jsonschema:"Optional CSS selector to extract only a region of the page"` + WaitFor string `json:"wait_for,omitempty" jsonschema:"Optional CSS selector to wait for before extracting (useful for SPAs that hydrate after networkidle)"` + TimeoutSeconds int `json:"timeout_seconds,omitempty" jsonschema:"Overall timeout in seconds (1-300, default 30)"` + MaxChars int `json:"max_chars,omitempty" jsonschema:"Truncate returned markdown to roughly this many characters (100-500000)"` +} + +// ReadOutput is returned to the MCP client. The Markdown field is the primary +// payload; other fields give the model enough context to decide what to do +// next. +type ReadOutput struct { + URL string `json:"url"` + FinalURL string `json:"final_url"` + Title string `json:"title"` + Markdown string `json:"markdown"` + UsedSession bool `json:"used_session"` + Truncated bool `json:"truncated"` + Bytes int `json:"bytes"` +} + +// ReadResult is the text-side companion of ReadOutput, surfaced via the +// CallToolResult's Content for clients that ignore structured content. +func readText(o ReadOutput) string { + var b strings.Builder + b.WriteString("# ") + b.WriteString(o.Title) + if o.Title == "" { + b.WriteString(o.URL) + } + b.WriteString("\n\n") + b.WriteString(o.Markdown) + if o.Truncated { + b.WriteString("\n\n[truncated]") + } + if o.FinalURL != "" && o.FinalURL != o.URL { + b.WriteString("\n\n_Resolved to: ") + b.WriteString(o.FinalURL) + b.WriteString("_") + } + if o.UsedSession { + b.WriteString("\n\n_Used saved session._") + } + return b.String() +} + +// RegisterRead wires the read_url tool to the server. +func RegisterRead(s *mcp.Server, r *reader.Reader) { + mcp.AddTool(s, &mcp.Tool{ + Name: "read_url", + Description: "Fetch a URL through a headless browser (full SPA support), render the page, and return its content as Markdown. If a session has previously been saved for the URL's host via the login tool, it is used automatically so authenticated pages render as the logged-in user.", + }, func(ctx context.Context, _ *mcp.CallToolRequest, in ReadInput) (*mcp.CallToolResult, ReadOutput, error) { + if in.URL == "" { + return nil, ReadOutput{}, fmt.Errorf("url is required") + } + timeout := time.Duration(in.TimeoutSeconds) * time.Second + res, err := r.Read(ctx, reader.ReadOptions{ + URL: in.URL, + Selector: in.Selector, + WaitSelector: in.WaitFor, + Timeout: timeout, + MaxChars: in.MaxChars, + }) + if err != nil { + return nil, ReadOutput{}, err + } + out := ReadOutput{ + URL: res.URL, + FinalURL: res.FinalURL, + Title: res.Title, + Markdown: res.Markdown, + UsedSession: res.UsedSession, + Truncated: res.Truncated, + Bytes: res.Bytes, + } + return &mcp.CallToolResult{ + Content: []mcp.Content{&mcp.TextContent{Text: readText(out)}}, + }, out, nil + }) +} diff --git a/internal/tools/sessions.go b/internal/tools/sessions.go new file mode 100644 index 0000000..2eda807 --- /dev/null +++ b/internal/tools/sessions.go @@ -0,0 +1,84 @@ +package tools + +import ( + "context" + "fmt" + "strings" + + "github.com/lukaszraczylo/mcp-readitall/internal/browser" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +// ListSessionsInput is empty: the tool has no parameters. +type ListSessionsInput struct{} + +// ListSessionsOutput lists every saved session. +type ListSessionsOutput struct { + Sessions []browser.SessionInfo `json:"sessions"` + Count int `json:"count"` +} + +// RegisterListSessions wires the list_sessions tool to the server. +func RegisterListSessions(s *mcp.Server, store *browser.SessionStore) { + mcp.AddTool(s, &mcp.Tool{ + Name: "list_sessions", + Description: "List every host for which an authenticated browser session is currently saved. Each entry can be passed to clear_session to invalidate it.", + }, func(ctx context.Context, _ *mcp.CallToolRequest, _ ListSessionsInput) (*mcp.CallToolResult, ListSessionsOutput, error) { + sessions, err := store.List() + if err != nil { + return nil, ListSessionsOutput{}, fmt.Errorf("list sessions: %w", err) + } + var b strings.Builder + b.WriteString(fmt.Sprintf("%d saved session(s):\n", len(sessions))) + for _, s := range sessions { + b.WriteString(" - ") + b.WriteString(s.PrettyPrint()) + b.WriteString("\n") + } + if len(sessions) == 0 { + b.WriteString(" (none)\n") + } + return &mcp.CallToolResult{ + Content: []mcp.Content{&mcp.TextContent{Text: strings.TrimRight(b.String(), "\n")}}, + }, ListSessionsOutput{Sessions: sessions, Count: len(sessions)}, nil + }) +} + +// ClearSessionInput is the schema for clear_session. +type ClearSessionInput struct { + URL string `json:"url" jsonschema:"URL whose host's saved session should be deleted"` +} + +// ClearSessionOutput reports what was deleted. +type ClearSessionOutput struct { + URL string `json:"url"` + Domain string `json:"domain"` + Cleared bool `json:"cleared"` + Message string `json:"message"` +} + +// RegisterClearSession wires the clear_session tool to the server. +func RegisterClearSession(s *mcp.Server, store *browser.SessionStore) { + mcp.AddTool(s, &mcp.Tool{ + Name: "clear_session", + Description: "Delete the saved authenticated session for the URL's host. Use this when a session has expired or is causing issues.", + }, func(ctx context.Context, _ *mcp.CallToolRequest, in ClearSessionInput) (*mcp.CallToolResult, ClearSessionOutput, error) { + if in.URL == "" { + return nil, ClearSessionOutput{}, fmt.Errorf("url is required") + } + existed, err := store.Clear(in.URL) + if err != nil { + return nil, ClearSessionOutput{}, err + } + key, _ := browser.HostKey(in.URL) + msg := fmt.Sprintf("cleared session for %s (existed=%v)", key, existed) + return &mcp.CallToolResult{ + Content: []mcp.Content{&mcp.TextContent{Text: msg}}, + }, ClearSessionOutput{ + URL: in.URL, + Domain: key, + Cleared: existed, + Message: msg, + }, nil + }) +} diff --git a/internal/tools/util.go b/internal/tools/util.go new file mode 100644 index 0000000..df05333 --- /dev/null +++ b/internal/tools/util.go @@ -0,0 +1,27 @@ +package tools + +import ( + "encoding/json" + "net/url" +) + +// jsQuote produces a JS-safe string literal from a CSS selector. We +// assume selectors do not contain backslashes or newlines (which would +// be invalid CSS) and just JSON-encode the value, which is a valid JS +// double-quoted string. +func jsQuote(s string) string { + b, _ := json.Marshal(s) + return string(b) +} + +// hostOf extracts the scheme-less host (with port) from a URL string. +// It returns "unknown" for inputs that do not parse to a URL with a +// host, rather than panicking. Case is preserved — url.Parse does not +// normalise the host — so "Example.COM" stays "Example.COM". +func hostOf(rawURL string) string { + u, err := url.Parse(rawURL) + if err != nil || u.Host == "" { + return "unknown" + } + return u.Host +}