mirror of
https://github.com/lukaszraczylo/mcp-readitall.git
synced 2026-10-15 01:07:36 +00:00
initial commit
This commit is contained in:
@@ -0,0 +1,101 @@
|
||||
package tools
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/lukaszraczylo/mcp-readitall/internal/reader"
|
||||
"github.com/modelcontextprotocol/go-sdk/mcp"
|
||||
)
|
||||
|
||||
// ExtractInput is the schema for the extract tool. It is essentially
|
||||
// read_url with a default selector set to <main> when omitted, so common CMS
|
||||
// layouts work without ceremony.
|
||||
type ExtractInput struct {
|
||||
URL string `json:"url" jsonschema:"Absolute http(s) URL to read"`
|
||||
Selector string `json:"selector,omitempty" jsonschema:"CSS selector to extract. Defaults to 'main, article, [role=main]' which works for most content sites."`
|
||||
WaitFor string `json:"wait_for,omitempty" jsonschema:"Optional CSS selector to wait for before extracting (useful for SPAs)"`
|
||||
TimeoutSeconds int `json:"timeout_seconds,omitempty" jsonschema:"Overall timeout in seconds (1-300, default 30)"`
|
||||
MaxChars int `json:"max_chars,omitempty" jsonschema:"Truncate returned markdown to roughly this many characters (100-500000)"`
|
||||
}
|
||||
|
||||
// ExtractOutput is the structured result.
|
||||
type ExtractOutput struct {
|
||||
URL string `json:"url"`
|
||||
FinalURL string `json:"final_url"`
|
||||
Title string `json:"title"`
|
||||
Markdown string `json:"markdown"`
|
||||
Selector string `json:"selector"`
|
||||
UsedSession bool `json:"used_session"`
|
||||
Truncated bool `json:"truncated"`
|
||||
Bytes int `json:"bytes"`
|
||||
}
|
||||
|
||||
// RegisterExtract wires the extract tool to the server.
|
||||
func RegisterExtract(s *mcp.Server, r *reader.Reader) {
|
||||
mcp.AddTool(s, &mcp.Tool{
|
||||
Name: "extract",
|
||||
Description: "Fetch a URL and return only the content under one (or more, comma-separated) CSS selectors, converted to Markdown. This is the right tool when you want the page's main body without nav, headers, footers, and sidebars. Works on SPA and SSR sites.",
|
||||
}, func(ctx context.Context, _ *mcp.CallToolRequest, in ExtractInput) (*mcp.CallToolResult, ExtractOutput, error) {
|
||||
if in.URL == "" {
|
||||
return nil, ExtractOutput{}, fmt.Errorf("url is required")
|
||||
}
|
||||
selector := in.Selector
|
||||
if selector == "" {
|
||||
selector = "main, article, [role=main]"
|
||||
}
|
||||
timeout := time.Duration(in.TimeoutSeconds) * time.Second
|
||||
|
||||
res, err := r.Read(ctx, reader.ReadOptions{
|
||||
URL: in.URL,
|
||||
Selector: firstSelector(selector),
|
||||
WaitSelector: in.WaitFor,
|
||||
Timeout: timeout,
|
||||
MaxChars: in.MaxChars,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, ExtractOutput{}, err
|
||||
}
|
||||
out := ExtractOutput{
|
||||
URL: res.URL,
|
||||
FinalURL: res.FinalURL,
|
||||
Title: res.Title,
|
||||
Markdown: res.Markdown,
|
||||
Selector: selector,
|
||||
UsedSession: res.UsedSession,
|
||||
Truncated: res.Truncated,
|
||||
Bytes: res.Bytes,
|
||||
}
|
||||
var b strings.Builder
|
||||
b.WriteString("# ")
|
||||
b.WriteString(out.Title)
|
||||
if out.Title == "" {
|
||||
b.WriteString(out.URL)
|
||||
}
|
||||
b.WriteString("\n\n")
|
||||
b.WriteString(out.Markdown)
|
||||
if out.Truncated {
|
||||
b.WriteString("\n\n[truncated]")
|
||||
}
|
||||
return &mcp.CallToolResult{
|
||||
Content: []mcp.Content{&mcp.TextContent{Text: b.String()}},
|
||||
}, out, nil
|
||||
})
|
||||
}
|
||||
|
||||
// firstSelector picks the first non-empty selector from a comma-separated
|
||||
// list. The reader is intentionally simple and only supports a single
|
||||
// selector today; we surface the first to keep the schema natural for
|
||||
// humans ("main, article, [role=main]") without pretending to support
|
||||
// union selectors internally. Empty entries are skipped so leading or
|
||||
// consecutive commas (" , main") do not produce a blank selector.
|
||||
func firstSelector(selectorList string) string {
|
||||
for _, p := range strings.Split(selectorList, ",") {
|
||||
if s := strings.TrimSpace(p); s != "" {
|
||||
return s
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -0,0 +1,273 @@
|
||||
// White-box tests for unexported helpers in the tools package. We use
|
||||
// `package tools` (not `tools_test`) so we can reach readText, firstSelector,
|
||||
// and hostOf directly. These helpers are the bits of logic that don't
|
||||
// require a browser or MCP plumbing, so they're cheap to cover.
|
||||
package tools
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
// =============================================================================
|
||||
// readText
|
||||
// =============================================================================
|
||||
|
||||
func TestReadText(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
in ReadOutput
|
||||
mustHave []string
|
||||
mustNot []string
|
||||
}{
|
||||
// ===== GOOD CASES =====
|
||||
{
|
||||
name: "title and body",
|
||||
in: ReadOutput{
|
||||
URL: "https://example.test/",
|
||||
Title: "Example Domain",
|
||||
Markdown: "Hello **world**.",
|
||||
},
|
||||
mustHave: []string{
|
||||
"# Example Domain",
|
||||
"Hello **world**.",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "falls back to URL when title empty",
|
||||
in: ReadOutput{
|
||||
URL: "https://example.test/",
|
||||
Markdown: "body",
|
||||
},
|
||||
mustHave: []string{
|
||||
"# https://example.test/",
|
||||
"body",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "truncated flag is surfaced",
|
||||
in: ReadOutput{
|
||||
URL: "https://x.test/",
|
||||
Title: "T",
|
||||
Markdown: "cut",
|
||||
Truncated: true,
|
||||
},
|
||||
mustHave: []string{
|
||||
"# T",
|
||||
"cut",
|
||||
"[truncated]",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "redirect is shown only when final URL differs",
|
||||
in: ReadOutput{
|
||||
URL: "https://x.test/old",
|
||||
FinalURL: "https://x.test/new",
|
||||
Title: "T",
|
||||
Markdown: "body",
|
||||
},
|
||||
mustHave: []string{
|
||||
"_Resolved to: https://x.test/new_",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "no redirect note when URLs match",
|
||||
in: ReadOutput{
|
||||
URL: "https://x.test/",
|
||||
FinalURL: "https://x.test/",
|
||||
Title: "T",
|
||||
Markdown: "body",
|
||||
},
|
||||
mustNot: []string{
|
||||
"Resolved to",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "session-used flag is surfaced",
|
||||
in: ReadOutput{
|
||||
URL: "https://x.test/",
|
||||
Title: "T",
|
||||
Markdown: "body",
|
||||
UsedSession: true,
|
||||
},
|
||||
mustHave: []string{
|
||||
"_Used saved session._",
|
||||
},
|
||||
},
|
||||
|
||||
// ===== EDGE CASES =====
|
||||
{
|
||||
name: "empty body still renders header",
|
||||
in: ReadOutput{
|
||||
URL: "https://x.test/",
|
||||
Title: "T",
|
||||
},
|
||||
mustHave: []string{
|
||||
"# T",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "redirect + session flags combined",
|
||||
in: ReadOutput{
|
||||
URL: "https://x.test/old",
|
||||
FinalURL: "https://x.test/new",
|
||||
Title: "T",
|
||||
Markdown: "body",
|
||||
UsedSession: true,
|
||||
Truncated: true,
|
||||
},
|
||||
mustHave: []string{
|
||||
"_Resolved to: https://x.test/new_",
|
||||
"_Used saved session._",
|
||||
"[truncated]",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
got := readText(tt.in)
|
||||
for _, s := range tt.mustHave {
|
||||
assert.Contains(t, got, s, "missing %q in:\n%s", s, got)
|
||||
}
|
||||
for _, s := range tt.mustNot {
|
||||
assert.NotContains(t, got, s, "unexpected %q in:\n%s", s, got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// firstSelector
|
||||
// =============================================================================
|
||||
|
||||
func TestFirstSelector(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
in string
|
||||
want string
|
||||
}{
|
||||
// ===== GOOD CASES =====
|
||||
{
|
||||
name: "single selector",
|
||||
in: "main",
|
||||
want: "main",
|
||||
},
|
||||
{
|
||||
name: "comma list returns first",
|
||||
in: "main, article, [role=main]",
|
||||
want: "main",
|
||||
},
|
||||
{
|
||||
name: "whitespace around first entry is trimmed",
|
||||
in: " #content , aside",
|
||||
want: "#content",
|
||||
},
|
||||
|
||||
// ===== EDGE CASES =====
|
||||
{
|
||||
name: "trailing comma",
|
||||
in: "main,",
|
||||
want: "main",
|
||||
},
|
||||
{
|
||||
name: "leading comma",
|
||||
in: ",main",
|
||||
want: "main",
|
||||
},
|
||||
{
|
||||
name: "multiple consecutive commas",
|
||||
in: ",,main,,article",
|
||||
want: "main",
|
||||
},
|
||||
{
|
||||
name: "empty string",
|
||||
in: "",
|
||||
want: "",
|
||||
},
|
||||
{
|
||||
name: "only commas",
|
||||
in: ",,,",
|
||||
want: "",
|
||||
},
|
||||
{
|
||||
name: "selector with internal spaces is preserved",
|
||||
in: "div[data-role=main content], aside",
|
||||
want: "div[data-role=main content]",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
got := firstSelector(tt.in)
|
||||
assert.Equal(t, tt.want, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// hostOf
|
||||
// =============================================================================
|
||||
|
||||
func TestHostOf(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
in string
|
||||
want string
|
||||
}{
|
||||
// ===== GOOD CASES =====
|
||||
{
|
||||
name: "https with path",
|
||||
in: "https://example.com/some/path",
|
||||
want: "example.com",
|
||||
},
|
||||
{
|
||||
name: "subdomain",
|
||||
in: "https://app.example.co.uk",
|
||||
want: "app.example.co.uk",
|
||||
},
|
||||
{
|
||||
name: "with port",
|
||||
in: "http://localhost:8080/x",
|
||||
want: "localhost:8080",
|
||||
},
|
||||
{
|
||||
name: "uppercase host is preserved (url.Parse does not lowercase)",
|
||||
in: "https://Example.COM/x",
|
||||
want: "Example.COM",
|
||||
},
|
||||
|
||||
// ===== BAD / EDGE CASES =====
|
||||
{
|
||||
name: "empty string returns unknown",
|
||||
in: "",
|
||||
want: "unknown",
|
||||
},
|
||||
{
|
||||
name: "garbage returns unknown (does not panic)",
|
||||
in: "::::",
|
||||
want: "unknown",
|
||||
},
|
||||
{
|
||||
name: "scheme-only returns unknown",
|
||||
in: "https://",
|
||||
want: "unknown",
|
||||
},
|
||||
{
|
||||
name: "path-only returns unknown",
|
||||
in: "/just/a/path",
|
||||
want: "unknown",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
got := hostOf(tt.in)
|
||||
assert.Equal(t, tt.want, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,225 @@
|
||||
package tools
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/chromedp/cdproto/cdp"
|
||||
"github.com/chromedp/chromedp"
|
||||
"github.com/lukaszraczylo/mcp-readitall/internal/browser"
|
||||
"github.com/modelcontextprotocol/go-sdk/mcp"
|
||||
)
|
||||
|
||||
// LoginInput is the schema for the login tool. The flow is:
|
||||
//
|
||||
// 1. A visible Chromium window opens and navigates to URL.
|
||||
// 2. The human operator (you) authenticates by hand in that window.
|
||||
// 3. The tool watches for SuccessURL (substring match against the
|
||||
// address bar) or SuccessSelector (a CSS selector that becomes
|
||||
// visible, e.g. an avatar dropdown) and, when matched, persists the
|
||||
// browser's storage state to the canonical session path for the
|
||||
// host.
|
||||
// 4. Future read_url calls against this host will reuse the saved
|
||||
// state.
|
||||
//
|
||||
// On a headless host (e.g. a remote MCP server) the visible-window
|
||||
// approach will not work; the operator must run the MCP server on a
|
||||
// machine with a display. We surface a clear error in that case so the
|
||||
// caller knows the limitation is environmental, not a bug.
|
||||
type LoginInput struct {
|
||||
URL string `json:"url" jsonschema:"Absolute http(s) URL to open (typically the login page)"`
|
||||
SuccessURL string `json:"success_url,omitempty" jsonschema:"Substring of the URL bar that indicates login succeeded (e.g. /dashboard). Polled every second."`
|
||||
SuccessSelector string `json:"success_selector,omitempty" jsonschema:"CSS selector that becomes visible once login is complete (e.g. an avatar menu). Polled every second."`
|
||||
TimeoutSeconds int `json:"timeout_seconds,omitempty" jsonschema:"How long to wait for a successful login in seconds (5-3600, default 300)"`
|
||||
}
|
||||
|
||||
// LoginOutput reports what happened.
|
||||
type LoginOutput struct {
|
||||
Domain string `json:"domain"`
|
||||
Saved bool `json:"saved"`
|
||||
Path string `json:"path,omitempty"`
|
||||
FinalURL string `json:"final_url"`
|
||||
Message string `json:"message"`
|
||||
}
|
||||
|
||||
// RegisterLogin wires the login tool to the server.
|
||||
func RegisterLogin(s *mcp.Server, pw *browser.Browser, sess *browser.SessionStore, logger *log.Logger) {
|
||||
mcp.AddTool(s, &mcp.Tool{
|
||||
Name: "login",
|
||||
Description: "Open a visible browser to a URL, wait for the human operator to authenticate, then persist the resulting browser storage state (cookies + localStorage) for the host. Subsequent read_url calls against this host will use the saved state automatically. Useful for sites that require login to expose their content.",
|
||||
}, func(parentCtx context.Context, _ *mcp.CallToolRequest, in LoginInput) (*mcp.CallToolResult, LoginOutput, error) {
|
||||
if in.URL == "" {
|
||||
return nil, LoginOutput{}, fmt.Errorf("url is required")
|
||||
}
|
||||
if in.SuccessURL == "" && in.SuccessSelector == "" {
|
||||
return nil, LoginOutput{}, fmt.Errorf("success_url or success_selector is required so the tool knows when login is complete")
|
||||
}
|
||||
timeout := time.Duration(in.TimeoutSeconds) * time.Second
|
||||
if timeout == 0 {
|
||||
timeout = 5 * time.Minute
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(parentCtx, timeout)
|
||||
defer cancel()
|
||||
|
||||
browserCtx, releaseBrowser, err := pw.NewHeadedContext()
|
||||
if err != nil {
|
||||
return nil, LoginOutput{}, fmt.Errorf("launch headed browser: %w", err)
|
||||
}
|
||||
defer releaseBrowser()
|
||||
|
||||
// Start from a clean cookie jar so a stale session from a
|
||||
// previous attempt cannot mask the new login's success.
|
||||
if err := browser.ClearAllCookies(browserCtx); err != nil {
|
||||
return nil, LoginOutput{}, fmt.Errorf("clear cookies: %w", err)
|
||||
}
|
||||
|
||||
var title string
|
||||
if err := chromedp.Run(browserCtx,
|
||||
chromedp.Navigate(in.URL),
|
||||
chromedp.WaitReady("body"),
|
||||
chromedp.Title(&title),
|
||||
); err != nil {
|
||||
return nil, LoginOutput{}, fmt.Errorf("navigate: %w", err)
|
||||
}
|
||||
|
||||
logger.Printf("login flow started for %s, waiting for success condition", in.URL)
|
||||
|
||||
if err := waitForLogin(ctx, browserCtx, in, timeout); err != nil {
|
||||
return nil, LoginOutput{
|
||||
Domain: hostOf(in.URL),
|
||||
FinalURL: currentLocation(browserCtx),
|
||||
Message: err.Error(),
|
||||
}, fmt.Errorf("login did not complete: %w", err)
|
||||
}
|
||||
|
||||
finalURL := currentLocation(browserCtx)
|
||||
|
||||
statePath, err := sess.StatePath(in.URL)
|
||||
if err != nil {
|
||||
return nil, LoginOutput{}, err
|
||||
}
|
||||
if err := browser.CaptureStorageState(browserCtx, statePath); err != nil {
|
||||
return nil, LoginOutput{
|
||||
Domain: hostOf(in.URL),
|
||||
FinalURL: finalURL,
|
||||
Message: "success condition matched but capturing storage state failed",
|
||||
}, err
|
||||
}
|
||||
|
||||
// Validate the captured state has at least one cookie or origin
|
||||
// entry, otherwise we would happily save an "authenticated" file
|
||||
// even when the success condition matched an unrelated selector.
|
||||
if err := validateCapturedState(statePath); err != nil {
|
||||
return nil, LoginOutput{
|
||||
Domain: hostOf(in.URL),
|
||||
FinalURL: finalURL,
|
||||
Message: "success condition matched but storage state is empty; auth likely failed",
|
||||
}, err
|
||||
}
|
||||
|
||||
msg := fmt.Sprintf("session saved for %s", hostOf(in.URL))
|
||||
return &mcp.CallToolResult{
|
||||
Content: []mcp.Content{&mcp.TextContent{Text: msg}},
|
||||
}, LoginOutput{
|
||||
Domain: hostOf(in.URL),
|
||||
Saved: true,
|
||||
Path: statePath,
|
||||
FinalURL: finalURL,
|
||||
Message: msg,
|
||||
}, nil
|
||||
})
|
||||
}
|
||||
|
||||
// waitForLogin polls the page once per second until the success
|
||||
// condition is met or the timeout fires. It uses chromedp.Location and
|
||||
// chromedp.Nodes (with a visibility check) for the two supported
|
||||
// condition types.
|
||||
func waitForLogin(ctx context.Context, pageCtx context.Context, in LoginInput, timeout time.Duration) error {
|
||||
deadline := time.Now().Add(timeout)
|
||||
ticker := time.NewTicker(time.Second)
|
||||
defer ticker.Stop()
|
||||
|
||||
check := func() (bool, error) {
|
||||
if in.SuccessURL != "" {
|
||||
var current string
|
||||
if err := chromedp.Run(pageCtx, chromedp.Location(¤t)); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if strings.Contains(current, in.SuccessURL) {
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
if in.SuccessSelector != "" {
|
||||
var nodes []*cdp.Node
|
||||
if err := chromedp.Run(pageCtx, chromedp.Nodes(in.SuccessSelector, &nodes, chromedp.ByQuery)); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if len(nodes) > 0 {
|
||||
var visible bool
|
||||
if err := chromedp.Run(pageCtx, chromedp.Evaluate(
|
||||
`(()=>{const e=document.querySelector(`+jsQuote(in.SuccessSelector)+`);return !!e && e.getBoundingClientRect().height>0;})()`,
|
||||
&visible,
|
||||
)); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if visible {
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// Immediate check so an already-authenticated session is captured
|
||||
// without a 1-second wait.
|
||||
if ok, err := check(); err != nil {
|
||||
return err
|
||||
} else if ok {
|
||||
return nil
|
||||
}
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-ticker.C:
|
||||
if time.Now().After(deadline) {
|
||||
return fmt.Errorf("timed out after %s waiting for login", timeout)
|
||||
}
|
||||
ok, err := check()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if ok {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// currentLocation returns the page's current URL or "" if the query
|
||||
// fails (e.g. the browser has already been torn down).
|
||||
func currentLocation(ctx context.Context) string {
|
||||
var s string
|
||||
if err := chromedp.Run(ctx, chromedp.Location(&s)); err != nil {
|
||||
return ""
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// validateCapturedState reads the just-saved storage state file and
|
||||
// ensures it has at least one cookie or one localStorage entry. This
|
||||
// guards against "login success" false positives (e.g. selector matches
|
||||
// a generic header element that happens to be visible).
|
||||
func validateCapturedState(path string) error {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read state: %w", err)
|
||||
}
|
||||
return browser.ValidateStorageState(raw)
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
// Package tools registers each MCP tool with the server. One file per tool
|
||||
// keeps the schemas, handlers, and tool registration together so changes are
|
||||
// easy to audit.
|
||||
package tools
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/lukaszraczylo/mcp-readitall/internal/reader"
|
||||
"github.com/modelcontextprotocol/go-sdk/mcp"
|
||||
)
|
||||
|
||||
// ReadInput is the JSON schema for the read_url tool.
|
||||
type ReadInput struct {
|
||||
URL string `json:"url" jsonschema:"Absolute http(s) URL to read"`
|
||||
Selector string `json:"selector,omitempty" jsonschema:"Optional CSS selector to extract only a region of the page"`
|
||||
WaitFor string `json:"wait_for,omitempty" jsonschema:"Optional CSS selector to wait for before extracting (useful for SPAs that hydrate after networkidle)"`
|
||||
TimeoutSeconds int `json:"timeout_seconds,omitempty" jsonschema:"Overall timeout in seconds (1-300, default 30)"`
|
||||
MaxChars int `json:"max_chars,omitempty" jsonschema:"Truncate returned markdown to roughly this many characters (100-500000)"`
|
||||
}
|
||||
|
||||
// ReadOutput is returned to the MCP client. The Markdown field is the primary
|
||||
// payload; other fields give the model enough context to decide what to do
|
||||
// next.
|
||||
type ReadOutput struct {
|
||||
URL string `json:"url"`
|
||||
FinalURL string `json:"final_url"`
|
||||
Title string `json:"title"`
|
||||
Markdown string `json:"markdown"`
|
||||
UsedSession bool `json:"used_session"`
|
||||
Truncated bool `json:"truncated"`
|
||||
Bytes int `json:"bytes"`
|
||||
}
|
||||
|
||||
// ReadResult is the text-side companion of ReadOutput, surfaced via the
|
||||
// CallToolResult's Content for clients that ignore structured content.
|
||||
func readText(o ReadOutput) string {
|
||||
var b strings.Builder
|
||||
b.WriteString("# ")
|
||||
b.WriteString(o.Title)
|
||||
if o.Title == "" {
|
||||
b.WriteString(o.URL)
|
||||
}
|
||||
b.WriteString("\n\n")
|
||||
b.WriteString(o.Markdown)
|
||||
if o.Truncated {
|
||||
b.WriteString("\n\n[truncated]")
|
||||
}
|
||||
if o.FinalURL != "" && o.FinalURL != o.URL {
|
||||
b.WriteString("\n\n_Resolved to: ")
|
||||
b.WriteString(o.FinalURL)
|
||||
b.WriteString("_")
|
||||
}
|
||||
if o.UsedSession {
|
||||
b.WriteString("\n\n_Used saved session._")
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// RegisterRead wires the read_url tool to the server.
|
||||
func RegisterRead(s *mcp.Server, r *reader.Reader) {
|
||||
mcp.AddTool(s, &mcp.Tool{
|
||||
Name: "read_url",
|
||||
Description: "Fetch a URL through a headless browser (full SPA support), render the page, and return its content as Markdown. If a session has previously been saved for the URL's host via the login tool, it is used automatically so authenticated pages render as the logged-in user.",
|
||||
}, func(ctx context.Context, _ *mcp.CallToolRequest, in ReadInput) (*mcp.CallToolResult, ReadOutput, error) {
|
||||
if in.URL == "" {
|
||||
return nil, ReadOutput{}, fmt.Errorf("url is required")
|
||||
}
|
||||
timeout := time.Duration(in.TimeoutSeconds) * time.Second
|
||||
res, err := r.Read(ctx, reader.ReadOptions{
|
||||
URL: in.URL,
|
||||
Selector: in.Selector,
|
||||
WaitSelector: in.WaitFor,
|
||||
Timeout: timeout,
|
||||
MaxChars: in.MaxChars,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, ReadOutput{}, err
|
||||
}
|
||||
out := ReadOutput{
|
||||
URL: res.URL,
|
||||
FinalURL: res.FinalURL,
|
||||
Title: res.Title,
|
||||
Markdown: res.Markdown,
|
||||
UsedSession: res.UsedSession,
|
||||
Truncated: res.Truncated,
|
||||
Bytes: res.Bytes,
|
||||
}
|
||||
return &mcp.CallToolResult{
|
||||
Content: []mcp.Content{&mcp.TextContent{Text: readText(out)}},
|
||||
}, out, nil
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
package tools
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/lukaszraczylo/mcp-readitall/internal/browser"
|
||||
"github.com/modelcontextprotocol/go-sdk/mcp"
|
||||
)
|
||||
|
||||
// ListSessionsInput is empty: the tool has no parameters.
|
||||
type ListSessionsInput struct{}
|
||||
|
||||
// ListSessionsOutput lists every saved session.
|
||||
type ListSessionsOutput struct {
|
||||
Sessions []browser.SessionInfo `json:"sessions"`
|
||||
Count int `json:"count"`
|
||||
}
|
||||
|
||||
// RegisterListSessions wires the list_sessions tool to the server.
|
||||
func RegisterListSessions(s *mcp.Server, store *browser.SessionStore) {
|
||||
mcp.AddTool(s, &mcp.Tool{
|
||||
Name: "list_sessions",
|
||||
Description: "List every host for which an authenticated browser session is currently saved. Each entry can be passed to clear_session to invalidate it.",
|
||||
}, func(ctx context.Context, _ *mcp.CallToolRequest, _ ListSessionsInput) (*mcp.CallToolResult, ListSessionsOutput, error) {
|
||||
sessions, err := store.List()
|
||||
if err != nil {
|
||||
return nil, ListSessionsOutput{}, fmt.Errorf("list sessions: %w", err)
|
||||
}
|
||||
var b strings.Builder
|
||||
b.WriteString(fmt.Sprintf("%d saved session(s):\n", len(sessions)))
|
||||
for _, s := range sessions {
|
||||
b.WriteString(" - ")
|
||||
b.WriteString(s.PrettyPrint())
|
||||
b.WriteString("\n")
|
||||
}
|
||||
if len(sessions) == 0 {
|
||||
b.WriteString(" (none)\n")
|
||||
}
|
||||
return &mcp.CallToolResult{
|
||||
Content: []mcp.Content{&mcp.TextContent{Text: strings.TrimRight(b.String(), "\n")}},
|
||||
}, ListSessionsOutput{Sessions: sessions, Count: len(sessions)}, nil
|
||||
})
|
||||
}
|
||||
|
||||
// ClearSessionInput is the schema for clear_session.
|
||||
type ClearSessionInput struct {
|
||||
URL string `json:"url" jsonschema:"URL whose host's saved session should be deleted"`
|
||||
}
|
||||
|
||||
// ClearSessionOutput reports what was deleted.
|
||||
type ClearSessionOutput struct {
|
||||
URL string `json:"url"`
|
||||
Domain string `json:"domain"`
|
||||
Cleared bool `json:"cleared"`
|
||||
Message string `json:"message"`
|
||||
}
|
||||
|
||||
// RegisterClearSession wires the clear_session tool to the server.
|
||||
func RegisterClearSession(s *mcp.Server, store *browser.SessionStore) {
|
||||
mcp.AddTool(s, &mcp.Tool{
|
||||
Name: "clear_session",
|
||||
Description: "Delete the saved authenticated session for the URL's host. Use this when a session has expired or is causing issues.",
|
||||
}, func(ctx context.Context, _ *mcp.CallToolRequest, in ClearSessionInput) (*mcp.CallToolResult, ClearSessionOutput, error) {
|
||||
if in.URL == "" {
|
||||
return nil, ClearSessionOutput{}, fmt.Errorf("url is required")
|
||||
}
|
||||
existed, err := store.Clear(in.URL)
|
||||
if err != nil {
|
||||
return nil, ClearSessionOutput{}, err
|
||||
}
|
||||
key, _ := browser.HostKey(in.URL)
|
||||
msg := fmt.Sprintf("cleared session for %s (existed=%v)", key, existed)
|
||||
return &mcp.CallToolResult{
|
||||
Content: []mcp.Content{&mcp.TextContent{Text: msg}},
|
||||
}, ClearSessionOutput{
|
||||
URL: in.URL,
|
||||
Domain: key,
|
||||
Cleared: existed,
|
||||
Message: msg,
|
||||
}, nil
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
package tools
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/url"
|
||||
)
|
||||
|
||||
// jsQuote produces a JS-safe string literal from a CSS selector. We
|
||||
// assume selectors do not contain backslashes or newlines (which would
|
||||
// be invalid CSS) and just JSON-encode the value, which is a valid JS
|
||||
// double-quoted string.
|
||||
func jsQuote(s string) string {
|
||||
b, _ := json.Marshal(s)
|
||||
return string(b)
|
||||
}
|
||||
|
||||
// hostOf extracts the scheme-less host (with port) from a URL string.
|
||||
// It returns "unknown" for inputs that do not parse to a URL with a
|
||||
// host, rather than panicking. Case is preserved — url.Parse does not
|
||||
// normalise the host — so "Example.COM" stays "Example.COM".
|
||||
func hostOf(rawURL string) string {
|
||||
u, err := url.Parse(rawURL)
|
||||
if err != nil || u.Host == "" {
|
||||
return "unknown"
|
||||
}
|
||||
return u.Host
|
||||
}
|
||||
Reference in New Issue
Block a user