From 6e496eeac887acb7c18a3aa1ecc059dc403c936b Mon Sep 17 00:00:00 2001
From: Lukasz Raczylo <lukasz@raczylo.com>
Date: Sun, 7 Jun 2026 13:16:54 +0100
Subject: [PATCH] feat(captcha): image-grid tile solver with centroid mapping

---
 internal/captcha/grid.go      | 75 +++++++++++++++++++++++++++++++++++
 internal/captcha/grid_test.go | 27 +++++++++++++
 2 files changed, 102 insertions(+)
 create mode 100644 internal/captcha/grid.go
 create mode 100644 internal/captcha/grid_test.go

diff --git a/internal/captcha/grid.go b/internal/captcha/grid.go
new file mode 100644
index 0000000..79695d2
--- /dev/null
+++ b/internal/captcha/grid.go
@@ -0,0 +1,75 @@
+package captcha
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+
+	"github.com/lukaszraczylo/captcha-solver-mcp/internal/imageutil"
+	"github.com/lukaszraczylo/captcha-solver-mcp/internal/llm"
+)
+
+// GridOptions describes a reCAPTCHA v2 / hCaptcha tile grid challenge.
+type GridOptions struct {
+	Instruction string
+	CaptchaType string // recaptcha | hcaptcha (informational)
+	Rows        int
+	Cols        int
+	ImageWidth  int
+	ImageHeight int
+}
+
+// GridResult lists tiles to click (row-major 0-based) plus optional pixel
+// centroids when grid dimensions are known.
+type GridResult struct {
+	Tiles              []int             `json:"tiles"`
+	Centroids          []imageutil.Point `json:"centroids,omitempty"`
+	Confidence         float64           `json:"confidence"`
+	RecheckRecommended bool              `json:"recheck_recommended"`
+}
+
+func gridPrompt(o GridOptions, strict bool) string {
+	p := fmt.Sprintf(
+		"You are solving an image-grid captcha. The image is a %dx%d grid of tiles, "+
+			"numbered row-major starting at 0 (top-left). Instruction: %q. "+
+			"Return the 0-based indices of every tile that matches the instruction. "+
+			"If after selection more matching tiles would likely appear (dynamic grid), set recheck_recommended true. "+
+			`Respond ONLY with compact JSON: {"tiles":[..],"confidence":<0..1>,"recheck_recommended":<bool>}.`,
+		o.Rows, o.Cols, o.Instruction)
+	if strict {
+		p += " You MUST output valid JSON only. No prose."
+	}
+	return p
+}
+
+// SolveGrid asks the vision model which tiles to click and (when grid geometry
+// is supplied) maps them to pixel centroids for humanized clicking.
+func SolveGrid(ctx context.Context, c llm.Client, img []byte, mediaType string, o GridOptions) (GridResult, error) {
+	images := []llm.Image{{MediaType: mediaType, Data: img}}
+	for attempt := 0; attempt < 2; attempt++ {
+		raw, err := c.Vision(ctx, gridPrompt(o, attempt > 0), images, llm.Options{})
+		if err != nil {
+			return GridResult{}, err
+		}
+		var res GridResult
+		if err := json.Unmarshal([]byte(extractJSON(raw)), &res); err == nil && res.Tiles != nil {
+			res.Centroids = mapCentroids(res.Tiles, o)
+			return res, nil
+		}
+	}
+	return GridResult{}, fmt.Errorf("model did not return parseable tile selection")
+}
+
+func mapCentroids(tiles []int, o GridOptions) []imageutil.Point {
+	if o.Rows <= 0 || o.Cols <= 0 || o.ImageWidth <= 0 || o.ImageHeight <= 0 {
+		return nil
+	}
+	all := imageutil.GridCentroids(o.ImageWidth, o.ImageHeight, o.Rows, o.Cols)
+	out := make([]imageutil.Point, 0, len(tiles))
+	for _, idx := range tiles {
+		if idx >= 0 && idx < len(all) {
+			out = append(out, all[idx])
+		}
+	}
+	return out
+}
diff --git a/internal/captcha/grid_test.go b/internal/captcha/grid_test.go
new file mode 100644
index 0000000..d8693a5
--- /dev/null
+++ b/internal/captcha/grid_test.go
@@ -0,0 +1,27 @@
+package captcha
+
+import (
+	"context"
+	"testing"
+
+	"github.com/lukaszraczylo/captcha-solver-mcp/internal/llm"
+)
+
+func TestSolveGrid(t *testing.T) {
+	f := &llm.Fake{VisionResp: `{"tiles":[0,4,8],"confidence":0.8,"recheck_recommended":true}`}
+	res, err := SolveGrid(context.Background(), f, []byte{1}, "image/png", GridOptions{
+		Instruction: "select all buses", Rows: 3, Cols: 3, ImageWidth: 300, ImageHeight: 300,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(res.Tiles) != 3 || res.Tiles[0] != 0 {
+		t.Fatalf("tiles %+v", res.Tiles)
+	}
+	if len(res.Centroids) != 3 || res.Centroids[1].X != 150 { // tile 4 centre
+		t.Fatalf("centroids %+v", res.Centroids)
+	}
+	if !res.RecheckRecommended {
+		t.Fatal("recheck flag lost")
+	}
+}