mirror of
https://github.com/lukaszraczylo/mcp-readitall.git
synced 2026-10-15 01:07:36 +00:00
initial commit
This commit is contained in:
@@ -0,0 +1,187 @@
|
||||
// Package browser manages the chromedp driver lifecycle used to fetch
|
||||
// pages, render SPAs, and capture storage state. The browser process is
|
||||
// started lazily on first use and shared across all subsequent reads.
|
||||
//
|
||||
// chromedp is a pure-Go Chrome DevTools Protocol client: no Node.js bridge,
|
||||
// no Playwright driver. It talks to a local Chromium (or any other CDP-
|
||||
// compatible browser) over a websocket.
|
||||
package browser
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"sync"
|
||||
|
||||
"github.com/chromedp/cdproto/cdp"
|
||||
"github.com/chromedp/cdproto/network"
|
||||
"github.com/chromedp/chromedp"
|
||||
)
|
||||
|
||||
// Browser is a thin wrapper around a long-lived chromedp ExecAllocator and
|
||||
// its single Chromium process. Each read (or login) creates a fresh
|
||||
// incognito-like browser context on top, so state never leaks between
|
||||
// calls.
|
||||
type Browser struct {
|
||||
log *log.Logger
|
||||
|
||||
startOnce sync.Once
|
||||
startErr error
|
||||
|
||||
allocCtx context.Context
|
||||
allocCancel context.CancelFunc
|
||||
browserCtx context.Context // base context that owns the browser allocation
|
||||
}
|
||||
|
||||
// New returns a Browser manager. The underlying Chromium is not started
|
||||
// until the first call that needs it; this keeps server startup cheap and
|
||||
// gives a clear error path if Chrome is missing.
|
||||
func New(_ context.Context, logger *log.Logger) (*Browser, error) {
|
||||
if logger == nil {
|
||||
logger = log.Default()
|
||||
}
|
||||
return &Browser{log: logger}, nil
|
||||
}
|
||||
|
||||
// ensureAllocator starts a headless Chromium the first time it is needed
|
||||
// and reuses it for every subsequent call. The flags mirror the old
|
||||
// Playwright defaults: no sandbox (needed inside many CI / container
|
||||
// environments), no default browser check, GPU disabled for headless.
|
||||
//
|
||||
// chromedp allocates the browser lazily on the first action; we eagerly
|
||||
// run a no-op Navigate("about:blank") on a base context so the
|
||||
// allocation is done once, up front, and the base context is ready to
|
||||
// host WithNewBrowserContext children. Without this warmup,
|
||||
// chromedp.NewContext(alloc, WithNewBrowserContext) panics because
|
||||
// WithNewBrowserContext requires a *Browser to already be set on the
|
||||
// parent chromedp context.
|
||||
func (b *Browser) ensureAllocator() (context.Context, error) {
|
||||
b.startOnce.Do(func() {
|
||||
opts := append(chromedp.DefaultExecAllocatorOptions[:],
|
||||
chromedp.NoSandbox,
|
||||
chromedp.NoDefaultBrowserCheck,
|
||||
chromedp.DisableGPU,
|
||||
chromedp.Flag("disable-dev-shm-usage", true),
|
||||
)
|
||||
b.allocCtx, b.allocCancel = chromedp.NewExecAllocator(context.Background(), opts...)
|
||||
|
||||
browserCtx, cancel := chromedp.NewContext(b.allocCtx)
|
||||
if err := chromedp.Run(browserCtx, chromedp.Navigate("about:blank")); err != nil {
|
||||
b.startErr = fmt.Errorf("warm up browser: %w", err)
|
||||
cancel()
|
||||
return
|
||||
}
|
||||
// The base context must stay alive for the lifetime of the
|
||||
// browser. We intentionally do not call cancel() here; the
|
||||
// alloc's cancel will cascade when the Browser is closed.
|
||||
b.browserCtx = browserCtx
|
||||
})
|
||||
return b.allocCtx, b.startErr
|
||||
}
|
||||
|
||||
// NewContext returns a fresh chromedp context ready for a read. The
|
||||
// first call triggers the browser process startup; subsequent calls
|
||||
// share the same Chromium process. The caller's cancel() tears down the
|
||||
// chromedp context (closing its tab) but leaves the singleton browser
|
||||
// alive. Cancellation or deadline on parentCtx is propagated to the
|
||||
// chromedp context, so a timeout applied by the caller is honoured by
|
||||
// the browser actions.
|
||||
//
|
||||
// State isolation: the previous read's cookies are wiped with
|
||||
// network.ClearBrowserCookies before this call returns. This is the
|
||||
// manual equivalent of chromedp.WithNewBrowserContext; we use it
|
||||
// because WithNewBrowserContext is broken in chromedp v0.15.1 (the
|
||||
// first child context fails with "no browser is open"). localStorage
|
||||
// is origin-scoped so it is naturally wiped when we navigate to a new
|
||||
// URL, except in the rare case where two reads target the same origin
|
||||
// — that is handled by the reader's post-navigation re-apply step.
|
||||
//
|
||||
// If statePath points to a valid storage state file, its cookies are
|
||||
// then applied to the (now empty) cookie jar via CDP before navigation.
|
||||
// (The caller is responsible for applying localStorage post-navigation;
|
||||
// see reader.applyLocalStorage.)
|
||||
func (b *Browser) NewContext(parentCtx context.Context, statePath string) (context.Context, context.CancelFunc, error) {
|
||||
if _, err := b.ensureAllocator(); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
ctx, cancel := chromedp.NewContext(b.browserCtx)
|
||||
|
||||
// Wipe cookies from any previous call before applying the new
|
||||
// state. See the doc comment above for why we do this manually
|
||||
// instead of using WithNewBrowserContext.
|
||||
if err := ClearAllCookies(ctx); err != nil {
|
||||
cancel()
|
||||
return nil, nil, fmt.Errorf("clear cookies: %w", err)
|
||||
}
|
||||
|
||||
// Forward parentCtx cancellation into the chromedp context. The
|
||||
// goroutine exits as soon as either side is done, so it does not
|
||||
// leak even on the happy path.
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
select {
|
||||
case <-parentCtx.Done():
|
||||
cancel()
|
||||
case <-done:
|
||||
}
|
||||
}()
|
||||
|
||||
if statePath != "" {
|
||||
if err := ApplyStorageStateCookies(ctx, statePath); err != nil {
|
||||
cancel()
|
||||
close(done)
|
||||
return nil, nil, fmt.Errorf("apply storage state: %w", err)
|
||||
}
|
||||
}
|
||||
return ctx, func() {
|
||||
cancel()
|
||||
close(done)
|
||||
}, nil
|
||||
}
|
||||
|
||||
// ClearAllCookies wipes every cookie in the current browser context via
|
||||
// CDP. Exposed for the NewHeadedContext login flow, which uses it as
|
||||
// the first step of a fresh login (otherwise stale session cookies from
|
||||
// a previous attempt can mask the new login's success).
|
||||
func ClearAllCookies(ctx context.Context) error {
|
||||
return chromedp.Run(ctx, chromedp.ActionFunc(func(ctx context.Context) error {
|
||||
target := chromedp.FromContext(ctx).Target
|
||||
return network.ClearBrowserCookies().Do(cdp.WithExecutor(ctx, target))
|
||||
}))
|
||||
}
|
||||
|
||||
// NewHeadedContext spawns a *separate* Chromium process with a visible
|
||||
// window. It is used only by the login tool. Headed mode is intentionally
|
||||
// isolated from the singleton headless allocator because chromedp cannot
|
||||
// toggle the headless flag on a running browser.
|
||||
//
|
||||
// On a server without a display (no $DISPLAY on Linux, no logged-in user
|
||||
// on macOS), Chrome will exit immediately. Callers should surface a clear
|
||||
// error to the operator.
|
||||
func (b *Browser) NewHeadedContext() (context.Context, context.CancelFunc, error) {
|
||||
opts := append(chromedp.DefaultExecAllocatorOptions[:],
|
||||
chromedp.NoSandbox,
|
||||
chromedp.NoDefaultBrowserCheck,
|
||||
chromedp.Flag("headless", false),
|
||||
)
|
||||
allocCtx, allocCancel := chromedp.NewExecAllocator(context.Background(), opts...)
|
||||
ctx, cancel := chromedp.NewContext(allocCtx)
|
||||
// Tie the alloc lifetime to the returned cancel so the headed browser
|
||||
// is shut down when the caller is done.
|
||||
wrapped := context.AfterFunc(ctx, allocCancel)
|
||||
origCancel := cancel
|
||||
cancel = func() {
|
||||
origCancel()
|
||||
wrapped()
|
||||
}
|
||||
return ctx, cancel, nil
|
||||
}
|
||||
|
||||
// Close shuts down the singleton browser. Safe to call once.
|
||||
func (b *Browser) Close() error {
|
||||
if b.allocCancel != nil {
|
||||
b.allocCancel()
|
||||
b.allocCancel = nil
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,250 @@
|
||||
package browser
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// SessionStore persists browser storage state per host. State files live
|
||||
// under a configurable root directory (default ~/.mcp-readitall/sessions/<host>/
|
||||
// state.json) and contain cookies + localStorage entries that authenticate
|
||||
// subsequent browser contexts against the same host. The on-disk JSON shape
|
||||
// is the same as Playwright's storage state so sessions are interchangeable.
|
||||
type SessionStore struct {
|
||||
root string
|
||||
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
// SessionInfo describes a persisted session, returned to the MCP client by
|
||||
// list_sessions.
|
||||
type SessionInfo struct {
|
||||
Domain string `json:"domain"`
|
||||
Path string `json:"path"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
}
|
||||
|
||||
// NewSessionStore creates a store rooted at the default location
|
||||
// ($HOME/.mcp-readitall/sessions) unless overridden via READITALL_SESSIONS_DIR.
|
||||
func NewSessionStore() (*SessionStore, error) {
|
||||
root := os.Getenv("READITALL_SESSIONS_DIR")
|
||||
if root == "" {
|
||||
home, err := os.UserHomeDir()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("locate home dir: %w", err)
|
||||
}
|
||||
root = filepath.Join(home, ".mcp-readitall", "sessions")
|
||||
}
|
||||
if err := os.MkdirAll(root, 0o700); err != nil {
|
||||
return nil, fmt.Errorf("create sessions root %q: %w", root, err)
|
||||
}
|
||||
return &SessionStore{root: root}, nil
|
||||
}
|
||||
|
||||
// Root returns the directory where session files are stored.
|
||||
func (s *SessionStore) Root() string { return s.root }
|
||||
|
||||
// HostKey normalises a URL to a safe directory name. It lower-cases the host
|
||||
// and replaces any non-alphanumeric characters (except . and -) with _.
|
||||
func HostKey(rawURL string) (string, error) {
|
||||
u, err := url.Parse(rawURL)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("parse url: %w", err)
|
||||
}
|
||||
host := strings.ToLower(u.Host)
|
||||
if host == "" {
|
||||
return "", fmt.Errorf("url has no host: %q", rawURL)
|
||||
}
|
||||
// Strip credentials if present.
|
||||
if i := strings.Index(host, "@"); i >= 0 {
|
||||
host = host[i+1:]
|
||||
}
|
||||
// Use a separate var to satisfy go vet's "loop variable captured" check
|
||||
// even though we only iterate once.
|
||||
safe := make([]rune, 0, len(host))
|
||||
for _, r := range host {
|
||||
switch {
|
||||
case r >= 'a' && r <= 'z',
|
||||
r >= '0' && r <= '9',
|
||||
r == '.', r == '-':
|
||||
safe = append(safe, r)
|
||||
default:
|
||||
safe = append(safe, '_')
|
||||
}
|
||||
}
|
||||
return string(safe), nil
|
||||
}
|
||||
|
||||
// StatePath returns the path where storage state for the given URL's host is
|
||||
// (or will be) stored. The file may not exist yet.
|
||||
func (s *SessionStore) StatePath(rawURL string) (string, error) {
|
||||
key, err := HostKey(rawURL)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return filepath.Join(s.root, key, "state.json"), nil
|
||||
}
|
||||
|
||||
// Exists reports whether a session is stored for the given URL's host.
|
||||
func (s *SessionStore) Exists(rawURL string) (bool, error) {
|
||||
path, err := s.StatePath(rawURL)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
_, err = os.Stat(path)
|
||||
switch {
|
||||
case err == nil:
|
||||
return true, nil
|
||||
case errors.Is(err, os.ErrNotExist):
|
||||
return false, nil
|
||||
default:
|
||||
return false, err
|
||||
}
|
||||
}
|
||||
|
||||
// Save writes the JSON-encoded storage state (cookies + origins) to the
|
||||
// canonical path for the URL's host. The state value is expected to be the
|
||||
// raw JSON object matching the standard {cookies, origins} storage state
|
||||
// shape used by Playwright and chromedp.
|
||||
func (s *SessionStore) Save(rawURL string, stateJSON []byte) (string, error) {
|
||||
key, err := HostKey(rawURL)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
dir := filepath.Join(s.root, key)
|
||||
if err := os.MkdirAll(dir, 0o700); err != nil {
|
||||
return "", fmt.Errorf("create session dir: %w", err)
|
||||
}
|
||||
path := filepath.Join(dir, "state.json")
|
||||
if err := os.WriteFile(path, stateJSON, 0o600); err != nil {
|
||||
return "", fmt.Errorf("write state: %w", err)
|
||||
}
|
||||
return path, nil
|
||||
}
|
||||
|
||||
// List returns metadata for every stored session, sorted by host.
|
||||
func (s *SessionStore) List() ([]SessionInfo, error) {
|
||||
entries, err := os.ReadDir(s.root)
|
||||
if err != nil {
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
var out []SessionInfo
|
||||
for _, e := range entries {
|
||||
if !e.IsDir() {
|
||||
continue
|
||||
}
|
||||
path := filepath.Join(s.root, e.Name(), "state.json")
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
out = append(out, SessionInfo{
|
||||
Domain: e.Name(),
|
||||
Path: path,
|
||||
UpdatedAt: info.ModTime(),
|
||||
SizeBytes: info.Size(),
|
||||
})
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].Domain < out[j].Domain })
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Clear removes the saved session for the given host. It returns true when a
|
||||
// file was deleted, false when nothing existed.
|
||||
func (s *SessionStore) Clear(rawURL string) (bool, error) {
|
||||
key, err := HostKey(rawURL)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
dir := filepath.Join(s.root, key)
|
||||
// Stat first so we can report whether anything was actually there, then
|
||||
// remove. We cannot rely on os.RemoveAll's return value to distinguish
|
||||
// "missing" from "removed" — both succeed.
|
||||
existed := false
|
||||
if _, statErr := os.Stat(dir); statErr == nil {
|
||||
existed = true
|
||||
} else if !errors.Is(statErr, os.ErrNotExist) {
|
||||
return false, fmt.Errorf("stat session: %w", statErr)
|
||||
}
|
||||
if !existed {
|
||||
return false, nil
|
||||
}
|
||||
if err := os.RemoveAll(dir); err != nil {
|
||||
return false, fmt.Errorf("remove session: %w", err)
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// Close releases any resources held by the store. Currently a no-op kept
|
||||
// for symmetry with the Browser wrapper.
|
||||
func (s *SessionStore) Close() error { return nil }
|
||||
|
||||
// PrettyPrint returns a human-readable summary of a session, used in tool
|
||||
// responses.
|
||||
func (s SessionInfo) PrettyPrint() string {
|
||||
age := time.Since(s.UpdatedAt).Truncate(time.Second)
|
||||
return fmt.Sprintf("%s (%d bytes, updated %s ago)", s.Domain, s.SizeBytes, age)
|
||||
}
|
||||
|
||||
// storageStateShape is used only to validate that a value looks like a
|
||||
// Playwright storage state blob before we persist it.
|
||||
type storageStateShape struct {
|
||||
Cookies []json.RawMessage `json:"cookies"`
|
||||
Origins []json.RawMessage `json:"origins"`
|
||||
}
|
||||
|
||||
// ValidateStorageState does a cheap structural check on the JSON so we do
|
||||
// not silently store an empty or malformed blob.
|
||||
func ValidateStorageState(stateJSON []byte) error {
|
||||
var s storageStateShape
|
||||
if err := json.Unmarshal(stateJSON, &s); err != nil {
|
||||
return fmt.Errorf("storage state is not valid JSON: %w", err)
|
||||
}
|
||||
if len(s.Cookies) == 0 && len(s.Origins) == 0 {
|
||||
return fmt.Errorf("storage state has no cookies or origins; auth likely failed")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ValidateStorageStateStruct is the equivalent of ValidateStorageState for
|
||||
// callers that already hold a parsed value (e.g. a slice of CDP
|
||||
// *network.Cookie or a Playwright *StorageState). It round-trips through
|
||||
// JSON and only inspects the cookies / origins fields, so the exact
|
||||
// concrete type does not matter as long as the JSON shape is correct.
|
||||
func ValidateStorageStateStruct(state any) error {
|
||||
if state == nil {
|
||||
return fmt.Errorf("storage state is nil; auth likely failed")
|
||||
}
|
||||
// We type-switch on the concrete shape because the input may be a
|
||||
// third-party type that we cannot import directly; we only care about
|
||||
// cookies and origins here.
|
||||
type shaped struct {
|
||||
Cookies []any `json:"cookies"`
|
||||
Origins []any `json:"origins"`
|
||||
}
|
||||
raw, err := json.Marshal(state)
|
||||
if err != nil {
|
||||
return fmt.Errorf("encode storage state: %w", err)
|
||||
}
|
||||
var s shaped
|
||||
if err := json.Unmarshal(raw, &s); err != nil {
|
||||
return fmt.Errorf("storage state has unexpected shape: %w", err)
|
||||
}
|
||||
if len(s.Cookies) == 0 && len(s.Origins) == 0 {
|
||||
return fmt.Errorf("storage state has no cookies or origins; auth likely failed")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,400 @@
|
||||
package browser_test
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/lukaszraczylo/mcp-readitall/internal/browser"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// =============================================================================
|
||||
// TEST HELPERS
|
||||
// =============================================================================
|
||||
|
||||
// newStoreInTemp constructs a SessionStore rooted at a fresh temp directory.
|
||||
// It is automatically cleaned up by t.TempDir() and the env override is
|
||||
// reverted by t.Setenv() at the end of the test.
|
||||
func newStoreInTemp(t *testing.T) *browser.SessionStore {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
t.Setenv("READITALL_SESSIONS_DIR", dir)
|
||||
s, err := browser.NewSessionStore()
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = s.Close() })
|
||||
return s
|
||||
}
|
||||
|
||||
// validStateJSON returns a minimal but well-formed Playwright storage state
|
||||
// blob containing one cookie and one origin entry. Tests can pass it through
|
||||
// ValidateStorageState or Save without modification.
|
||||
func validStateJSON(t *testing.T) []byte {
|
||||
t.Helper()
|
||||
state := map[string]any{
|
||||
"cookies": []map[string]any{
|
||||
{"name": "session", "value": "abc123", "domain": "example.com", "path": "/"},
|
||||
},
|
||||
"origins": []map[string]any{
|
||||
{
|
||||
"origin": "https://example.com",
|
||||
"localStorage": []map[string]any{
|
||||
{"name": "token", "value": "xyz"},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
b, err := json.Marshal(state)
|
||||
require.NoError(t, err)
|
||||
return b
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// HostKey
|
||||
// =============================================================================
|
||||
|
||||
func TestHostKey(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
url string
|
||||
want string
|
||||
// optional: when set, expect an error containing this substring
|
||||
wantErr string
|
||||
}{
|
||||
// ===== GOOD CASES =====
|
||||
{
|
||||
name: "simple http",
|
||||
url: "http://example.com/path",
|
||||
want: "example.com",
|
||||
},
|
||||
{
|
||||
name: "https with deep path and query",
|
||||
url: "https://example.com/some/deep/path?x=1&y=2",
|
||||
want: "example.com",
|
||||
},
|
||||
{
|
||||
name: "subdomain preserved",
|
||||
url: "https://app.acme.example.com",
|
||||
want: "app.acme.example.com",
|
||||
},
|
||||
{
|
||||
name: "host with port",
|
||||
url: "http://localhost:8443/admin",
|
||||
want: "localhost_8443",
|
||||
},
|
||||
{
|
||||
name: "uppercase host is lowercased",
|
||||
url: "HTTPS://Example.COM/X",
|
||||
want: "example.com",
|
||||
},
|
||||
{
|
||||
name: "host with hyphens",
|
||||
url: "https://my-app.example-site.io",
|
||||
want: "my-app.example-site.io",
|
||||
},
|
||||
{
|
||||
name: "credentials in URL are stripped",
|
||||
url: "https://user:[email protected]/",
|
||||
want: "example.com",
|
||||
},
|
||||
|
||||
// ===== BAD CASES =====
|
||||
{
|
||||
name: "empty url",
|
||||
url: "",
|
||||
// url.Parse("") returns a URL with an empty host; HostKey
|
||||
// catches that and reports the "no host" condition.
|
||||
wantErr: "no host",
|
||||
},
|
||||
{
|
||||
name: "url with no host",
|
||||
url: "/just/a/path",
|
||||
wantErr: "no host",
|
||||
},
|
||||
{
|
||||
name: "garbage url",
|
||||
url: "::::::",
|
||||
// Garbage fails net/url parsing before HostKey can look at the
|
||||
// host. The "parse url" wrapper surfaces that.
|
||||
wantErr: "parse url",
|
||||
},
|
||||
|
||||
// ===== EDGE CASES =====
|
||||
{
|
||||
name: "weird chars in host get underscored",
|
||||
url: "https://xn--bcher-kva.example/", // punycode begins with "xn--"
|
||||
want: "xn--bcher-kva.example",
|
||||
},
|
||||
{
|
||||
name: "ipv4 host preserved",
|
||||
url: "http://127.0.0.1:8080/",
|
||||
want: "127.0.0.1_8080",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
got, err := browser.HostKey(tt.url)
|
||||
if tt.wantErr != "" {
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), tt.wantErr)
|
||||
return
|
||||
}
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tt.want, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// ValidateStorageState
|
||||
// =============================================================================
|
||||
|
||||
func TestValidateStorageState(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
state string
|
||||
wantErr string
|
||||
}{
|
||||
// ===== GOOD CASES =====
|
||||
{
|
||||
name: "valid state with cookies only",
|
||||
state: `{"cookies":[{"name":"a","value":"b"}]}`,
|
||||
},
|
||||
{
|
||||
name: "valid state with origins only",
|
||||
state: `{"origins":[{"origin":"https://x.test"}]}`,
|
||||
},
|
||||
{
|
||||
name: "valid state with both",
|
||||
state: `{"cookies":[{"name":"a"}],"origins":[{"origin":"https://x.test"}]}`,
|
||||
},
|
||||
|
||||
// ===== BAD CASES =====
|
||||
{
|
||||
name: "not json",
|
||||
state: `<html>not json</html>`,
|
||||
wantErr: "not valid JSON",
|
||||
},
|
||||
{
|
||||
name: "empty json object",
|
||||
state: `{}`,
|
||||
wantErr: "no cookies or origins",
|
||||
},
|
||||
{
|
||||
name: "empty arrays",
|
||||
state: `{"cookies":[],"origins":[]}`,
|
||||
wantErr: "no cookies or origins",
|
||||
},
|
||||
{
|
||||
name: "completely empty string",
|
||||
state: ``,
|
||||
wantErr: "not valid JSON",
|
||||
},
|
||||
|
||||
// ===== EDGE CASES =====
|
||||
{
|
||||
name: "whitespace-only json returns parse error",
|
||||
state: ` `,
|
||||
// json.Unmarshal treats whitespace as truncated input and
|
||||
// returns a parse error before we ever look at fields.
|
||||
wantErr: "not valid JSON",
|
||||
},
|
||||
{
|
||||
name: "extra unknown fields are tolerated",
|
||||
state: `{"cookies":[{"name":"a"}],"origins":[],"extra":{"foo":"bar"}}`,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
err := browser.ValidateStorageState([]byte(tt.state))
|
||||
if tt.wantErr != "" {
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), tt.wantErr)
|
||||
return
|
||||
}
|
||||
require.NoError(t, err)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// Save / Exists / StatePath / List / Clear
|
||||
// =============================================================================
|
||||
|
||||
func TestSessionStore_SaveAndExists(t *testing.T) {
|
||||
// Uses t.Setenv via newStoreInTemp, so cannot be t.Parallel().
|
||||
store := newStoreInTemp(t)
|
||||
|
||||
url := "https://example.com/dashboard"
|
||||
state := validStateJSON(t)
|
||||
|
||||
// Exists should be false before save
|
||||
exists, err := store.Exists(url)
|
||||
require.NoError(t, err)
|
||||
assert.False(t, exists)
|
||||
|
||||
path, err := store.Save(url, state)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, store.Root(), filepath.Dir(filepath.Dir(path)))
|
||||
assert.FileExists(t, path)
|
||||
|
||||
// Permissions should be 0600 — credentials must not leak
|
||||
info, err := os.Stat(path)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, os.FileMode(0o600), info.Mode().Perm(), "state file must be owner-only")
|
||||
|
||||
exists, err = store.Exists(url)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, exists)
|
||||
}
|
||||
|
||||
func TestSessionStore_ExistsAndStatePath_ForUnknownURL(t *testing.T) {
|
||||
// Uses t.Setenv via newStoreInTemp, so cannot be t.Parallel().
|
||||
store := newStoreInTemp(t)
|
||||
|
||||
exists, err := store.Exists("https://never-saved.test/")
|
||||
require.NoError(t, err)
|
||||
assert.False(t, exists)
|
||||
|
||||
path, err := store.StatePath("https://never-saved.test/")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, filepath.Join(store.Root(), "never-saved.test", "state.json"), path)
|
||||
_, statErr := os.Stat(path)
|
||||
assert.True(t, os.IsNotExist(statErr), "expected no file at %s", path)
|
||||
}
|
||||
|
||||
func TestSessionStore_RejectsEmptyState(t *testing.T) {
|
||||
// Uses t.Setenv via newStoreInTemp, so cannot be t.Parallel().
|
||||
store := newStoreInTemp(t)
|
||||
|
||||
// Validation must happen before persistence; otherwise we'd write an
|
||||
// empty file and look authenticated on subsequent reads.
|
||||
empty := []byte(`{"cookies":[],"origins":[]}`)
|
||||
_, err := store.Save("https://broken.test/", empty)
|
||||
// Save does not itself validate — that's a separate concern. Verify
|
||||
// ValidateStorageState is the gate.
|
||||
require.NoError(t, err, "Save is a low-level write; validation belongs to callers")
|
||||
|
||||
// Caller-side validation catches it.
|
||||
require.Error(t, browser.ValidateStorageState(empty))
|
||||
}
|
||||
|
||||
func TestSessionStore_List(t *testing.T) {
|
||||
store := newStoreInTemp(t)
|
||||
state := validStateJSON(t)
|
||||
|
||||
// Empty store: no entries
|
||||
infos, err := store.List()
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, infos)
|
||||
|
||||
// Save a few
|
||||
hosts := []string{
|
||||
"https://a.test/x",
|
||||
"https://b.test/y",
|
||||
"https://sub.c.test/z",
|
||||
}
|
||||
for _, u := range hosts {
|
||||
_, err := store.Save(u, state)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
infos, err = store.List()
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, infos, 3)
|
||||
|
||||
got := make([]string, 0, len(infos))
|
||||
for _, i := range infos {
|
||||
got = append(got, i.Domain)
|
||||
}
|
||||
sort.Strings(got)
|
||||
assert.Equal(t, []string{"a.test", "b.test", "sub.c.test"}, got)
|
||||
|
||||
for _, i := range infos {
|
||||
assert.Greater(t, i.SizeBytes, int64(0))
|
||||
assert.False(t, i.UpdatedAt.IsZero())
|
||||
}
|
||||
}
|
||||
|
||||
func TestSessionStore_Clear(t *testing.T) {
|
||||
store := newStoreInTemp(t)
|
||||
state := validStateJSON(t)
|
||||
url := "https://clear-me.test/page"
|
||||
|
||||
_, err := store.Save(url, state)
|
||||
require.NoError(t, err)
|
||||
|
||||
// First Clear returns existed=true
|
||||
cleared, err := store.Clear(url)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, cleared)
|
||||
|
||||
exists, err := store.Exists(url)
|
||||
require.NoError(t, err)
|
||||
assert.False(t, exists)
|
||||
|
||||
// Second Clear returns existed=false (idempotent)
|
||||
cleared, err = store.Clear(url)
|
||||
require.NoError(t, err)
|
||||
assert.False(t, cleared)
|
||||
}
|
||||
|
||||
func TestSessionStore_ClearUnknownHost(t *testing.T) {
|
||||
// Uses t.Setenv via newStoreInTemp, so cannot be t.Parallel().
|
||||
store := newStoreInTemp(t)
|
||||
|
||||
cleared, err := store.Clear("https://never-saved.test/")
|
||||
require.NoError(t, err)
|
||||
assert.False(t, cleared)
|
||||
}
|
||||
|
||||
func TestSessionStore_UpdatedAtAdvances(t *testing.T) {
|
||||
// Not parallel: relies on sequential time advancement.
|
||||
store := newStoreInTemp(t)
|
||||
state := validStateJSON(t)
|
||||
url := "https://time.test/"
|
||||
|
||||
_, err := store.Save(url, state)
|
||||
require.NoError(t, err)
|
||||
first, err := store.List()
|
||||
require.NoError(t, err)
|
||||
require.Len(t, first, 1)
|
||||
firstStamp := first[0].UpdatedAt
|
||||
|
||||
// Sleep just long enough that mtime can differ (filesystem mtime
|
||||
// resolution varies, especially on macOS APFS where it's 1ns).
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
|
||||
_, err = store.Save(url, state)
|
||||
require.NoError(t, err)
|
||||
second, err := store.List()
|
||||
require.NoError(t, err)
|
||||
require.Len(t, second, 1)
|
||||
|
||||
assert.True(t, second[0].UpdatedAt.After(firstStamp),
|
||||
"expected updated_at to advance: first=%s second=%s", firstStamp, second[0].UpdatedAt)
|
||||
}
|
||||
|
||||
func TestSessionStore_RoundTripPreservesStateBytes(t *testing.T) {
|
||||
// Uses t.Setenv via newStoreInTemp, so cannot be t.Parallel().
|
||||
// We do not parse the saved file; we just ensure the bytes we put in
|
||||
// are the bytes we get out.
|
||||
store := newStoreInTemp(t)
|
||||
url := "https://roundtrip.test/"
|
||||
state := validStateJSON(t)
|
||||
|
||||
path, err := store.Save(url, state)
|
||||
require.NoError(t, err)
|
||||
|
||||
got, err := os.ReadFile(path)
|
||||
require.NoError(t, err)
|
||||
assert.JSONEq(t, string(state), string(got))
|
||||
}
|
||||
@@ -0,0 +1,327 @@
|
||||
package browser
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/chromedp/cdproto/cdp"
|
||||
"github.com/chromedp/cdproto/network"
|
||||
"github.com/chromedp/chromedp"
|
||||
)
|
||||
|
||||
// storageState is the on-disk JSON shape for a saved session. It is
|
||||
// intentionally identical to Playwright's storage state so sessions
|
||||
// captured by either tool can be loaded by the other.
|
||||
//
|
||||
// Cookies and localStorage entries are stored as their raw JSON-encoded
|
||||
// representation, not as strongly-typed fields, so the file remains
|
||||
// tolerant of optional / future fields in the upstream format.
|
||||
type storageState struct {
|
||||
Cookies []json.RawMessage `json:"cookies"`
|
||||
Origins []json.RawMessage `json:"origins"`
|
||||
}
|
||||
|
||||
// localStorageEntry is the {name, value} pair shape Playwright (and we)
|
||||
// use to persist per-origin localStorage. Defined at package level so
|
||||
// buildLocalStorageScript can take a named slice type.
|
||||
type localStorageEntry struct {
|
||||
Name string `json:"name"`
|
||||
Value string `json:"value"`
|
||||
}
|
||||
|
||||
// localStorageOrigin groups localStorage entries by origin.
|
||||
type localStorageOrigin struct {
|
||||
Origin string `json:"origin"`
|
||||
LocalStorage []localStorageEntry `json:"localStorage"`
|
||||
}
|
||||
|
||||
// readStorageState parses a storage state file into the in-memory shape.
|
||||
func readStorageState(path string) (*storageState, error) {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read: %w", err)
|
||||
}
|
||||
var s storageState
|
||||
if err := json.Unmarshal(raw, &s); err != nil {
|
||||
return nil, fmt.Errorf("parse: %w", err)
|
||||
}
|
||||
return &s, nil
|
||||
}
|
||||
|
||||
// ApplyStorageStateCookies sets every cookie in the storage state on the
|
||||
// chromedp context. Cookies are domain-bound and can be set without being
|
||||
// on the target origin, so we apply them pre-navigation.
|
||||
//
|
||||
// Exported so the Browser wrapper can call it from NewContext; not part
|
||||
// of the public API for tool authors.
|
||||
func ApplyStorageStateCookies(ctx context.Context, statePath string) error {
|
||||
state, err := readStorageState(statePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(state.Cookies) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
cookies, err := decodeCookies(state.Cookies)
|
||||
if err != nil {
|
||||
return fmt.Errorf("decode cookies: %w", err)
|
||||
}
|
||||
if len(cookies) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
return chromedp.Run(ctx, chromedp.ActionFunc(func(ctx context.Context) error {
|
||||
target := chromedp.FromContext(ctx).Target
|
||||
return network.SetCookies(cookies).Do(cdp.WithExecutor(ctx, target))
|
||||
}))
|
||||
}
|
||||
|
||||
// ApplyLocalStorageOnCurrentOrigin sets every localStorage entry that
|
||||
// belongs to the page's current origin. Must be called after the page has
|
||||
// navigated, because localStorage is origin-scoped.
|
||||
//
|
||||
// Entries belonging to other origins are silently skipped: they cannot be
|
||||
// set from the wrong origin anyway.
|
||||
//
|
||||
// Exported so the reader can call it after navigation.
|
||||
func ApplyLocalStorageOnCurrentOrigin(ctx context.Context, statePath string) error {
|
||||
state, err := readStorageState(statePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(state.Origins) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
var currentURL string
|
||||
if err := chromedp.Run(ctx, chromedp.Location(¤tURL)); err != nil {
|
||||
return fmt.Errorf("read current origin: %w", err)
|
||||
}
|
||||
currentOrigin := normalizeOrigin(currentURL)
|
||||
if currentOrigin == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
var entries []localStorageEntry
|
||||
for _, raw := range state.Origins {
|
||||
var o localStorageOrigin
|
||||
if err := json.Unmarshal(raw, &o); err != nil {
|
||||
continue
|
||||
}
|
||||
if normalizeOrigin(o.Origin) != currentOrigin {
|
||||
continue
|
||||
}
|
||||
entries = append(entries, o.LocalStorage...)
|
||||
}
|
||||
if len(entries) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
return chromedp.Run(ctx, chromedp.Evaluate(buildLocalStorageScript(entries), nil))
|
||||
}
|
||||
|
||||
// CaptureStorageState reads every cookie set on the current target and
|
||||
// every localStorage entry for the current origin, then marshals them to
|
||||
// the on-disk JSON shape. It is intended to be called by the login tool
|
||||
// after a successful interactive authentication.
|
||||
func CaptureStorageState(ctx context.Context, statePath string) error {
|
||||
// 1) Cookies
|
||||
cookies, err := captureCDPCookies(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("get cookies: %w", err)
|
||||
}
|
||||
|
||||
// 2) Current origin + localStorage
|
||||
var currentURL string
|
||||
if err := chromedp.Run(ctx, chromedp.Location(¤tURL)); err != nil {
|
||||
return fmt.Errorf("read location: %w", err)
|
||||
}
|
||||
origin := normalizeOrigin(currentURL)
|
||||
|
||||
lsRaw, err := captureLocalStorage(ctx, origin)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read localStorage: %w", err)
|
||||
}
|
||||
|
||||
// 3) Build the on-disk shape.
|
||||
cookieMsgs := make([]json.RawMessage, 0, len(cookies))
|
||||
for _, c := range cookies {
|
||||
raw, err := json.Marshal(c)
|
||||
if err != nil {
|
||||
return fmt.Errorf("encode cookie: %w", err)
|
||||
}
|
||||
cookieMsgs = append(cookieMsgs, raw)
|
||||
}
|
||||
|
||||
out := storageState{Cookies: cookieMsgs}
|
||||
if lsRaw != nil {
|
||||
out.Origins = []json.RawMessage{lsRaw}
|
||||
}
|
||||
|
||||
// 4) Write atomically: tmp + rename, mode 0600.
|
||||
body, err := json.MarshalIndent(out, "", " ")
|
||||
if err != nil {
|
||||
return fmt.Errorf("encode state: %w", err)
|
||||
}
|
||||
tmp := statePath + ".tmp"
|
||||
if err := os.WriteFile(tmp, body, 0o600); err != nil {
|
||||
return fmt.Errorf("write tmp: %w", err)
|
||||
}
|
||||
if err := os.Rename(tmp, statePath); err != nil {
|
||||
return fmt.Errorf("rename: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// captureCDPCookies asks the current target for its full cookie list via
|
||||
// the network domain.
|
||||
func captureCDPCookies(ctx context.Context) ([]*network.Cookie, error) {
|
||||
var out []*network.Cookie
|
||||
err := chromedp.Run(ctx, chromedp.ActionFunc(func(ctx context.Context) error {
|
||||
target := chromedp.FromContext(ctx).Target
|
||||
var err error
|
||||
out, err = network.GetCookies().Do(cdp.WithExecutor(ctx, target))
|
||||
return err
|
||||
}))
|
||||
return out, err
|
||||
}
|
||||
|
||||
// captureLocalStorage serialises the current page's localStorage to a
|
||||
// JSON-encoded localStorageOrigin, or returns nil if there is no
|
||||
// scannable origin / no entries.
|
||||
func captureLocalStorage(ctx context.Context, origin string) (json.RawMessage, error) {
|
||||
if origin == "" {
|
||||
return nil, nil
|
||||
}
|
||||
const script = `JSON.stringify(Object.fromEntries(Object.keys(localStorage).map(function(k){return [k, localStorage.getItem(k)];})))`
|
||||
var lsJSON string
|
||||
if err := chromedp.Run(ctx, chromedp.Evaluate(script, &lsJSON)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if lsJSON == "" || lsJSON == "[]" || lsJSON == "{}" {
|
||||
return nil, nil
|
||||
}
|
||||
var kv [][2]string
|
||||
if err := json.Unmarshal([]byte(lsJSON), &kv); err != nil {
|
||||
return nil, fmt.Errorf("decode localStorage: %w", err)
|
||||
}
|
||||
if len(kv) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
entries := make([]localStorageEntry, 0, len(kv))
|
||||
for _, pair := range kv {
|
||||
entries = append(entries, localStorageEntry{Name: pair[0], Value: pair[1]})
|
||||
}
|
||||
o := localStorageOrigin{Origin: origin, LocalStorage: entries}
|
||||
raw, err := json.Marshal(o)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return raw, nil
|
||||
}
|
||||
|
||||
// decodeCookies turns the on-disk json.RawMessage cookie list into CDP
|
||||
// *network.CookieParam values ready for network.SetCookies. Session
|
||||
// cookies (negative or zero Expires) are left with a nil Expires field,
|
||||
// which CDP interprets as "session cookie".
|
||||
func decodeCookies(raws []json.RawMessage) ([]*network.CookieParam, error) {
|
||||
out := make([]*network.CookieParam, 0, len(raws))
|
||||
for _, raw := range raws {
|
||||
var c onDiskCookie
|
||||
if err := json.Unmarshal(raw, &c); err != nil {
|
||||
return nil, fmt.Errorf("cookie %s: %w", string(raw), err)
|
||||
}
|
||||
out = append(out, onDiskToCookieParam(c))
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// onDiskCookie is the subset of cookie fields we care about when
|
||||
// reading a stored state file. Anything else (priority, source scheme,
|
||||
// partition keys…) is ignored.
|
||||
type onDiskCookie struct {
|
||||
Name string `json:"name"`
|
||||
Value string `json:"value"`
|
||||
Domain string `json:"domain"`
|
||||
Path string `json:"path"`
|
||||
Expires float64 `json:"expires"`
|
||||
HTTPOnly bool `json:"httpOnly"`
|
||||
Secure bool `json:"secure"`
|
||||
SameSite string `json:"sameSite"`
|
||||
}
|
||||
|
||||
// onDiskToCookieParam converts our on-disk shape into a CDP
|
||||
// network.CookieParam. Session cookies (Expires <= 0) get a nil Expires
|
||||
// pointer; persistent cookies get a *cdp.TimeSinceEpoch built from the
|
||||
// Unix timestamp.
|
||||
func onDiskToCookieParam(c onDiskCookie) *network.CookieParam {
|
||||
p := &network.CookieParam{
|
||||
Name: c.Name,
|
||||
Value: c.Value,
|
||||
Domain: c.Domain,
|
||||
Path: c.Path,
|
||||
Secure: c.Secure,
|
||||
HTTPOnly: c.HTTPOnly,
|
||||
SameSite: parseSameSite(c.SameSite),
|
||||
}
|
||||
if c.Expires > 0 {
|
||||
t := cdp.TimeSinceEpoch(time.Unix(int64(c.Expires), 0))
|
||||
p.Expires = &t
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
// parseSameSite translates a Playwright-style SameSite string to a CDP
|
||||
// enum. Unknown / empty values become the safe default (Lax) rather
|
||||
// than None, to match the previous Playwright behaviour.
|
||||
func parseSameSite(s string) network.CookieSameSite {
|
||||
switch strings.ToLower(s) {
|
||||
case "strict":
|
||||
return network.CookieSameSiteStrict
|
||||
case "lax":
|
||||
return network.CookieSameSiteLax
|
||||
case "none":
|
||||
return network.CookieSameSiteNone
|
||||
default:
|
||||
return network.CookieSameSiteLax
|
||||
}
|
||||
}
|
||||
|
||||
// buildLocalStorageScript returns a JS snippet that, when run inside the
|
||||
// page, sets every (k, v) pair in the page's localStorage.
|
||||
func buildLocalStorageScript(entries []localStorageEntry) string {
|
||||
var b strings.Builder
|
||||
b.WriteString("(function(){")
|
||||
for _, e := range entries {
|
||||
b.WriteString("localStorage.setItem(")
|
||||
b.WriteString(jsonString(e.Name))
|
||||
b.WriteString(",")
|
||||
b.WriteString(jsonString(e.Value))
|
||||
b.WriteString(");")
|
||||
}
|
||||
b.WriteString("})();")
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// jsonString produces a JS-safe double-quoted string literal.
|
||||
func jsonString(s string) string {
|
||||
b, _ := json.Marshal(s)
|
||||
return string(b)
|
||||
}
|
||||
|
||||
// normalizeOrigin returns the scheme+host[:port] of a URL, or "" if
|
||||
// parsing fails. Used to match storage state origins against the page's
|
||||
// current location.
|
||||
func normalizeOrigin(raw string) string {
|
||||
u, err := url.Parse(raw)
|
||||
if err != nil || u.Host == "" {
|
||||
return ""
|
||||
}
|
||||
return u.Scheme + "://" + u.Host
|
||||
}
|
||||
Reference in New Issue
Block a user