initial commit

This commit is contained in:
2026-06-12 02:48:30 +01:00
commit baf1e740be
25 changed files with 3203 additions and 0 deletions
+101
View File
@@ -0,0 +1,101 @@
package tools
import (
"context"
"fmt"
"strings"
"time"
"github.com/lukaszraczylo/mcp-readitall/internal/reader"
"github.com/modelcontextprotocol/go-sdk/mcp"
)
// ExtractInput is the schema for the extract tool. It is essentially
// read_url with a default selector set to <main> when omitted, so common CMS
// layouts work without ceremony.
type ExtractInput struct {
URL string `json:"url" jsonschema:"Absolute http(s) URL to read"`
Selector string `json:"selector,omitempty" jsonschema:"CSS selector to extract. Defaults to 'main, article, [role=main]' which works for most content sites."`
WaitFor string `json:"wait_for,omitempty" jsonschema:"Optional CSS selector to wait for before extracting (useful for SPAs)"`
TimeoutSeconds int `json:"timeout_seconds,omitempty" jsonschema:"Overall timeout in seconds (1-300, default 30)"`
MaxChars int `json:"max_chars,omitempty" jsonschema:"Truncate returned markdown to roughly this many characters (100-500000)"`
}
// ExtractOutput is the structured result.
type ExtractOutput struct {
URL string `json:"url"`
FinalURL string `json:"final_url"`
Title string `json:"title"`
Markdown string `json:"markdown"`
Selector string `json:"selector"`
UsedSession bool `json:"used_session"`
Truncated bool `json:"truncated"`
Bytes int `json:"bytes"`
}
// RegisterExtract wires the extract tool to the server.
func RegisterExtract(s *mcp.Server, r *reader.Reader) {
mcp.AddTool(s, &mcp.Tool{
Name: "extract",
Description: "Fetch a URL and return only the content under one (or more, comma-separated) CSS selectors, converted to Markdown. This is the right tool when you want the page's main body without nav, headers, footers, and sidebars. Works on SPA and SSR sites.",
}, func(ctx context.Context, _ *mcp.CallToolRequest, in ExtractInput) (*mcp.CallToolResult, ExtractOutput, error) {
if in.URL == "" {
return nil, ExtractOutput{}, fmt.Errorf("url is required")
}
selector := in.Selector
if selector == "" {
selector = "main, article, [role=main]"
}
timeout := time.Duration(in.TimeoutSeconds) * time.Second
res, err := r.Read(ctx, reader.ReadOptions{
URL: in.URL,
Selector: firstSelector(selector),
WaitSelector: in.WaitFor,
Timeout: timeout,
MaxChars: in.MaxChars,
})
if err != nil {
return nil, ExtractOutput{}, err
}
out := ExtractOutput{
URL: res.URL,
FinalURL: res.FinalURL,
Title: res.Title,
Markdown: res.Markdown,
Selector: selector,
UsedSession: res.UsedSession,
Truncated: res.Truncated,
Bytes: res.Bytes,
}
var b strings.Builder
b.WriteString("# ")
b.WriteString(out.Title)
if out.Title == "" {
b.WriteString(out.URL)
}
b.WriteString("\n\n")
b.WriteString(out.Markdown)
if out.Truncated {
b.WriteString("\n\n[truncated]")
}
return &mcp.CallToolResult{
Content: []mcp.Content{&mcp.TextContent{Text: b.String()}},
}, out, nil
})
}
// firstSelector picks the first non-empty selector from a comma-separated
// list. The reader is intentionally simple and only supports a single
// selector today; we surface the first to keep the schema natural for
// humans ("main, article, [role=main]") without pretending to support
// union selectors internally. Empty entries are skipped so leading or
// consecutive commas (" , main") do not produce a blank selector.
func firstSelector(selectorList string) string {
for _, p := range strings.Split(selectorList, ",") {
if s := strings.TrimSpace(p); s != "" {
return s
}
}
return ""
}
+273
View File
@@ -0,0 +1,273 @@
// White-box tests for unexported helpers in the tools package. We use
// `package tools` (not `tools_test`) so we can reach readText, firstSelector,
// and hostOf directly. These helpers are the bits of logic that don't
// require a browser or MCP plumbing, so they're cheap to cover.
package tools
import (
"testing"
"github.com/stretchr/testify/assert"
)
// =============================================================================
// readText
// =============================================================================
func TestReadText(t *testing.T) {
tests := []struct {
name string
in ReadOutput
mustHave []string
mustNot []string
}{
// ===== GOOD CASES =====
{
name: "title and body",
in: ReadOutput{
URL: "https://example.test/",
Title: "Example Domain",
Markdown: "Hello **world**.",
},
mustHave: []string{
"# Example Domain",
"Hello **world**.",
},
},
{
name: "falls back to URL when title empty",
in: ReadOutput{
URL: "https://example.test/",
Markdown: "body",
},
mustHave: []string{
"# https://example.test/",
"body",
},
},
{
name: "truncated flag is surfaced",
in: ReadOutput{
URL: "https://x.test/",
Title: "T",
Markdown: "cut",
Truncated: true,
},
mustHave: []string{
"# T",
"cut",
"[truncated]",
},
},
{
name: "redirect is shown only when final URL differs",
in: ReadOutput{
URL: "https://x.test/old",
FinalURL: "https://x.test/new",
Title: "T",
Markdown: "body",
},
mustHave: []string{
"_Resolved to: https://x.test/new_",
},
},
{
name: "no redirect note when URLs match",
in: ReadOutput{
URL: "https://x.test/",
FinalURL: "https://x.test/",
Title: "T",
Markdown: "body",
},
mustNot: []string{
"Resolved to",
},
},
{
name: "session-used flag is surfaced",
in: ReadOutput{
URL: "https://x.test/",
Title: "T",
Markdown: "body",
UsedSession: true,
},
mustHave: []string{
"_Used saved session._",
},
},
// ===== EDGE CASES =====
{
name: "empty body still renders header",
in: ReadOutput{
URL: "https://x.test/",
Title: "T",
},
mustHave: []string{
"# T",
},
},
{
name: "redirect + session flags combined",
in: ReadOutput{
URL: "https://x.test/old",
FinalURL: "https://x.test/new",
Title: "T",
Markdown: "body",
UsedSession: true,
Truncated: true,
},
mustHave: []string{
"_Resolved to: https://x.test/new_",
"_Used saved session._",
"[truncated]",
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
got := readText(tt.in)
for _, s := range tt.mustHave {
assert.Contains(t, got, s, "missing %q in:\n%s", s, got)
}
for _, s := range tt.mustNot {
assert.NotContains(t, got, s, "unexpected %q in:\n%s", s, got)
}
})
}
}
// =============================================================================
// firstSelector
// =============================================================================
func TestFirstSelector(t *testing.T) {
tests := []struct {
name string
in string
want string
}{
// ===== GOOD CASES =====
{
name: "single selector",
in: "main",
want: "main",
},
{
name: "comma list returns first",
in: "main, article, [role=main]",
want: "main",
},
{
name: "whitespace around first entry is trimmed",
in: " #content , aside",
want: "#content",
},
// ===== EDGE CASES =====
{
name: "trailing comma",
in: "main,",
want: "main",
},
{
name: "leading comma",
in: ",main",
want: "main",
},
{
name: "multiple consecutive commas",
in: ",,main,,article",
want: "main",
},
{
name: "empty string",
in: "",
want: "",
},
{
name: "only commas",
in: ",,,",
want: "",
},
{
name: "selector with internal spaces is preserved",
in: "div[data-role=main content], aside",
want: "div[data-role=main content]",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
got := firstSelector(tt.in)
assert.Equal(t, tt.want, got)
})
}
}
// =============================================================================
// hostOf
// =============================================================================
func TestHostOf(t *testing.T) {
tests := []struct {
name string
in string
want string
}{
// ===== GOOD CASES =====
{
name: "https with path",
in: "https://example.com/some/path",
want: "example.com",
},
{
name: "subdomain",
in: "https://app.example.co.uk",
want: "app.example.co.uk",
},
{
name: "with port",
in: "http://localhost:8080/x",
want: "localhost:8080",
},
{
name: "uppercase host is preserved (url.Parse does not lowercase)",
in: "https://Example.COM/x",
want: "Example.COM",
},
// ===== BAD / EDGE CASES =====
{
name: "empty string returns unknown",
in: "",
want: "unknown",
},
{
name: "garbage returns unknown (does not panic)",
in: "::::",
want: "unknown",
},
{
name: "scheme-only returns unknown",
in: "https://",
want: "unknown",
},
{
name: "path-only returns unknown",
in: "/just/a/path",
want: "unknown",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
got := hostOf(tt.in)
assert.Equal(t, tt.want, got)
})
}
}
+225
View File
@@ -0,0 +1,225 @@
package tools
import (
"context"
"fmt"
"log"
"os"
"strings"
"time"
"github.com/chromedp/cdproto/cdp"
"github.com/chromedp/chromedp"
"github.com/lukaszraczylo/mcp-readitall/internal/browser"
"github.com/modelcontextprotocol/go-sdk/mcp"
)
// LoginInput is the schema for the login tool. The flow is:
//
// 1. A visible Chromium window opens and navigates to URL.
// 2. The human operator (you) authenticates by hand in that window.
// 3. The tool watches for SuccessURL (substring match against the
// address bar) or SuccessSelector (a CSS selector that becomes
// visible, e.g. an avatar dropdown) and, when matched, persists the
// browser's storage state to the canonical session path for the
// host.
// 4. Future read_url calls against this host will reuse the saved
// state.
//
// On a headless host (e.g. a remote MCP server) the visible-window
// approach will not work; the operator must run the MCP server on a
// machine with a display. We surface a clear error in that case so the
// caller knows the limitation is environmental, not a bug.
type LoginInput struct {
URL string `json:"url" jsonschema:"Absolute http(s) URL to open (typically the login page)"`
SuccessURL string `json:"success_url,omitempty" jsonschema:"Substring of the URL bar that indicates login succeeded (e.g. /dashboard). Polled every second."`
SuccessSelector string `json:"success_selector,omitempty" jsonschema:"CSS selector that becomes visible once login is complete (e.g. an avatar menu). Polled every second."`
TimeoutSeconds int `json:"timeout_seconds,omitempty" jsonschema:"How long to wait for a successful login in seconds (5-3600, default 300)"`
}
// LoginOutput reports what happened.
type LoginOutput struct {
Domain string `json:"domain"`
Saved bool `json:"saved"`
Path string `json:"path,omitempty"`
FinalURL string `json:"final_url"`
Message string `json:"message"`
}
// RegisterLogin wires the login tool to the server.
func RegisterLogin(s *mcp.Server, pw *browser.Browser, sess *browser.SessionStore, logger *log.Logger) {
mcp.AddTool(s, &mcp.Tool{
Name: "login",
Description: "Open a visible browser to a URL, wait for the human operator to authenticate, then persist the resulting browser storage state (cookies + localStorage) for the host. Subsequent read_url calls against this host will use the saved state automatically. Useful for sites that require login to expose their content.",
}, func(parentCtx context.Context, _ *mcp.CallToolRequest, in LoginInput) (*mcp.CallToolResult, LoginOutput, error) {
if in.URL == "" {
return nil, LoginOutput{}, fmt.Errorf("url is required")
}
if in.SuccessURL == "" && in.SuccessSelector == "" {
return nil, LoginOutput{}, fmt.Errorf("success_url or success_selector is required so the tool knows when login is complete")
}
timeout := time.Duration(in.TimeoutSeconds) * time.Second
if timeout == 0 {
timeout = 5 * time.Minute
}
ctx, cancel := context.WithTimeout(parentCtx, timeout)
defer cancel()
browserCtx, releaseBrowser, err := pw.NewHeadedContext()
if err != nil {
return nil, LoginOutput{}, fmt.Errorf("launch headed browser: %w", err)
}
defer releaseBrowser()
// Start from a clean cookie jar so a stale session from a
// previous attempt cannot mask the new login's success.
if err := browser.ClearAllCookies(browserCtx); err != nil {
return nil, LoginOutput{}, fmt.Errorf("clear cookies: %w", err)
}
var title string
if err := chromedp.Run(browserCtx,
chromedp.Navigate(in.URL),
chromedp.WaitReady("body"),
chromedp.Title(&title),
); err != nil {
return nil, LoginOutput{}, fmt.Errorf("navigate: %w", err)
}
logger.Printf("login flow started for %s, waiting for success condition", in.URL)
if err := waitForLogin(ctx, browserCtx, in, timeout); err != nil {
return nil, LoginOutput{
Domain: hostOf(in.URL),
FinalURL: currentLocation(browserCtx),
Message: err.Error(),
}, fmt.Errorf("login did not complete: %w", err)
}
finalURL := currentLocation(browserCtx)
statePath, err := sess.StatePath(in.URL)
if err != nil {
return nil, LoginOutput{}, err
}
if err := browser.CaptureStorageState(browserCtx, statePath); err != nil {
return nil, LoginOutput{
Domain: hostOf(in.URL),
FinalURL: finalURL,
Message: "success condition matched but capturing storage state failed",
}, err
}
// Validate the captured state has at least one cookie or origin
// entry, otherwise we would happily save an "authenticated" file
// even when the success condition matched an unrelated selector.
if err := validateCapturedState(statePath); err != nil {
return nil, LoginOutput{
Domain: hostOf(in.URL),
FinalURL: finalURL,
Message: "success condition matched but storage state is empty; auth likely failed",
}, err
}
msg := fmt.Sprintf("session saved for %s", hostOf(in.URL))
return &mcp.CallToolResult{
Content: []mcp.Content{&mcp.TextContent{Text: msg}},
}, LoginOutput{
Domain: hostOf(in.URL),
Saved: true,
Path: statePath,
FinalURL: finalURL,
Message: msg,
}, nil
})
}
// waitForLogin polls the page once per second until the success
// condition is met or the timeout fires. It uses chromedp.Location and
// chromedp.Nodes (with a visibility check) for the two supported
// condition types.
func waitForLogin(ctx context.Context, pageCtx context.Context, in LoginInput, timeout time.Duration) error {
deadline := time.Now().Add(timeout)
ticker := time.NewTicker(time.Second)
defer ticker.Stop()
check := func() (bool, error) {
if in.SuccessURL != "" {
var current string
if err := chromedp.Run(pageCtx, chromedp.Location(&current)); err != nil {
return false, err
}
if strings.Contains(current, in.SuccessURL) {
return true, nil
}
}
if in.SuccessSelector != "" {
var nodes []*cdp.Node
if err := chromedp.Run(pageCtx, chromedp.Nodes(in.SuccessSelector, &nodes, chromedp.ByQuery)); err != nil {
return false, err
}
if len(nodes) > 0 {
var visible bool
if err := chromedp.Run(pageCtx, chromedp.Evaluate(
`(()=>{const e=document.querySelector(`+jsQuote(in.SuccessSelector)+`);return !!e && e.getBoundingClientRect().height>0;})()`,
&visible,
)); err != nil {
return false, err
}
if visible {
return true, nil
}
}
}
return false, nil
}
// Immediate check so an already-authenticated session is captured
// without a 1-second wait.
if ok, err := check(); err != nil {
return err
} else if ok {
return nil
}
for {
select {
case <-ctx.Done():
return ctx.Err()
case <-ticker.C:
if time.Now().After(deadline) {
return fmt.Errorf("timed out after %s waiting for login", timeout)
}
ok, err := check()
if err != nil {
return err
}
if ok {
return nil
}
}
}
}
// currentLocation returns the page's current URL or "" if the query
// fails (e.g. the browser has already been torn down).
func currentLocation(ctx context.Context) string {
var s string
if err := chromedp.Run(ctx, chromedp.Location(&s)); err != nil {
return ""
}
return s
}
// validateCapturedState reads the just-saved storage state file and
// ensures it has at least one cookie or one localStorage entry. This
// guards against "login success" false positives (e.g. selector matches
// a generic header element that happens to be visible).
func validateCapturedState(path string) error {
raw, err := os.ReadFile(path)
if err != nil {
return fmt.Errorf("read state: %w", err)
}
return browser.ValidateStorageState(raw)
}
+96
View File
@@ -0,0 +1,96 @@
// Package tools registers each MCP tool with the server. One file per tool
// keeps the schemas, handlers, and tool registration together so changes are
// easy to audit.
package tools
import (
"context"
"fmt"
"strings"
"time"
"github.com/lukaszraczylo/mcp-readitall/internal/reader"
"github.com/modelcontextprotocol/go-sdk/mcp"
)
// ReadInput is the JSON schema for the read_url tool.
type ReadInput struct {
URL string `json:"url" jsonschema:"Absolute http(s) URL to read"`
Selector string `json:"selector,omitempty" jsonschema:"Optional CSS selector to extract only a region of the page"`
WaitFor string `json:"wait_for,omitempty" jsonschema:"Optional CSS selector to wait for before extracting (useful for SPAs that hydrate after networkidle)"`
TimeoutSeconds int `json:"timeout_seconds,omitempty" jsonschema:"Overall timeout in seconds (1-300, default 30)"`
MaxChars int `json:"max_chars,omitempty" jsonschema:"Truncate returned markdown to roughly this many characters (100-500000)"`
}
// ReadOutput is returned to the MCP client. The Markdown field is the primary
// payload; other fields give the model enough context to decide what to do
// next.
type ReadOutput struct {
URL string `json:"url"`
FinalURL string `json:"final_url"`
Title string `json:"title"`
Markdown string `json:"markdown"`
UsedSession bool `json:"used_session"`
Truncated bool `json:"truncated"`
Bytes int `json:"bytes"`
}
// ReadResult is the text-side companion of ReadOutput, surfaced via the
// CallToolResult's Content for clients that ignore structured content.
func readText(o ReadOutput) string {
var b strings.Builder
b.WriteString("# ")
b.WriteString(o.Title)
if o.Title == "" {
b.WriteString(o.URL)
}
b.WriteString("\n\n")
b.WriteString(o.Markdown)
if o.Truncated {
b.WriteString("\n\n[truncated]")
}
if o.FinalURL != "" && o.FinalURL != o.URL {
b.WriteString("\n\n_Resolved to: ")
b.WriteString(o.FinalURL)
b.WriteString("_")
}
if o.UsedSession {
b.WriteString("\n\n_Used saved session._")
}
return b.String()
}
// RegisterRead wires the read_url tool to the server.
func RegisterRead(s *mcp.Server, r *reader.Reader) {
mcp.AddTool(s, &mcp.Tool{
Name: "read_url",
Description: "Fetch a URL through a headless browser (full SPA support), render the page, and return its content as Markdown. If a session has previously been saved for the URL's host via the login tool, it is used automatically so authenticated pages render as the logged-in user.",
}, func(ctx context.Context, _ *mcp.CallToolRequest, in ReadInput) (*mcp.CallToolResult, ReadOutput, error) {
if in.URL == "" {
return nil, ReadOutput{}, fmt.Errorf("url is required")
}
timeout := time.Duration(in.TimeoutSeconds) * time.Second
res, err := r.Read(ctx, reader.ReadOptions{
URL: in.URL,
Selector: in.Selector,
WaitSelector: in.WaitFor,
Timeout: timeout,
MaxChars: in.MaxChars,
})
if err != nil {
return nil, ReadOutput{}, err
}
out := ReadOutput{
URL: res.URL,
FinalURL: res.FinalURL,
Title: res.Title,
Markdown: res.Markdown,
UsedSession: res.UsedSession,
Truncated: res.Truncated,
Bytes: res.Bytes,
}
return &mcp.CallToolResult{
Content: []mcp.Content{&mcp.TextContent{Text: readText(out)}},
}, out, nil
})
}
+84
View File
@@ -0,0 +1,84 @@
package tools
import (
"context"
"fmt"
"strings"
"github.com/lukaszraczylo/mcp-readitall/internal/browser"
"github.com/modelcontextprotocol/go-sdk/mcp"
)
// ListSessionsInput is empty: the tool has no parameters.
type ListSessionsInput struct{}
// ListSessionsOutput lists every saved session.
type ListSessionsOutput struct {
Sessions []browser.SessionInfo `json:"sessions"`
Count int `json:"count"`
}
// RegisterListSessions wires the list_sessions tool to the server.
func RegisterListSessions(s *mcp.Server, store *browser.SessionStore) {
mcp.AddTool(s, &mcp.Tool{
Name: "list_sessions",
Description: "List every host for which an authenticated browser session is currently saved. Each entry can be passed to clear_session to invalidate it.",
}, func(ctx context.Context, _ *mcp.CallToolRequest, _ ListSessionsInput) (*mcp.CallToolResult, ListSessionsOutput, error) {
sessions, err := store.List()
if err != nil {
return nil, ListSessionsOutput{}, fmt.Errorf("list sessions: %w", err)
}
var b strings.Builder
b.WriteString(fmt.Sprintf("%d saved session(s):\n", len(sessions)))
for _, s := range sessions {
b.WriteString(" - ")
b.WriteString(s.PrettyPrint())
b.WriteString("\n")
}
if len(sessions) == 0 {
b.WriteString(" (none)\n")
}
return &mcp.CallToolResult{
Content: []mcp.Content{&mcp.TextContent{Text: strings.TrimRight(b.String(), "\n")}},
}, ListSessionsOutput{Sessions: sessions, Count: len(sessions)}, nil
})
}
// ClearSessionInput is the schema for clear_session.
type ClearSessionInput struct {
URL string `json:"url" jsonschema:"URL whose host's saved session should be deleted"`
}
// ClearSessionOutput reports what was deleted.
type ClearSessionOutput struct {
URL string `json:"url"`
Domain string `json:"domain"`
Cleared bool `json:"cleared"`
Message string `json:"message"`
}
// RegisterClearSession wires the clear_session tool to the server.
func RegisterClearSession(s *mcp.Server, store *browser.SessionStore) {
mcp.AddTool(s, &mcp.Tool{
Name: "clear_session",
Description: "Delete the saved authenticated session for the URL's host. Use this when a session has expired or is causing issues.",
}, func(ctx context.Context, _ *mcp.CallToolRequest, in ClearSessionInput) (*mcp.CallToolResult, ClearSessionOutput, error) {
if in.URL == "" {
return nil, ClearSessionOutput{}, fmt.Errorf("url is required")
}
existed, err := store.Clear(in.URL)
if err != nil {
return nil, ClearSessionOutput{}, err
}
key, _ := browser.HostKey(in.URL)
msg := fmt.Sprintf("cleared session for %s (existed=%v)", key, existed)
return &mcp.CallToolResult{
Content: []mcp.Content{&mcp.TextContent{Text: msg}},
}, ClearSessionOutput{
URL: in.URL,
Domain: key,
Cleared: existed,
Message: msg,
}, nil
})
}
+27
View File
@@ -0,0 +1,27 @@
package tools
import (
"encoding/json"
"net/url"
)
// jsQuote produces a JS-safe string literal from a CSS selector. We
// assume selectors do not contain backslashes or newlines (which would
// be invalid CSS) and just JSON-encode the value, which is a valid JS
// double-quoted string.
func jsQuote(s string) string {
b, _ := json.Marshal(s)
return string(b)
}
// hostOf extracts the scheme-less host (with port) from a URL string.
// It returns "unknown" for inputs that do not parse to a URL with a
// host, rather than panicking. Case is preserved — url.Parse does not
// normalise the host — so "Example.COM" stays "Example.COM".
func hostOf(rawURL string) string {
u, err := url.Parse(rawURL)
if err != nil || u.Host == "" {
return "unknown"
}
return u.Host
}