ci: add goreleaser and semver-driven release workflow

The workflow calculates the version with semver-generator, tags the
commit when the version is new, and runs goreleaser. goreleaser ships
two distribution paths: tar.gz binaries for darwin and linux on amd64
and arm64, and a multi-arch ghcr.io container that carries the go,
python3, and C toolchains so the exec evaluators run in-container.
The new llmtest version subcommand reports the stamped build version.

.github/workflows/release.yml

@@ -0,0 +1,80 @@
+name: Release
+
+on:
+  push:
+    branches: [main]
+  workflow_dispatch:
+
+permissions:
+  contents: write
+  packages: write
+
+jobs:
+  release:
+    name: Version, tag, and release
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Calculate semantic version
+        id: semver
+        uses: lukaszraczylo/semver-generator@v1
+        with:
+          config_file: semver.yaml
+          repository_local: true
+          existing: true
+
+      - name: Create tag if new
+        id: tag
+        run: |
+          VERSION="v${{ steps.semver.outputs.semantic_version }}"
+          echo "version=${VERSION}" >> "$GITHUB_OUTPUT"
+          if git rev-parse "${VERSION}" >/dev/null 2>&1; then
+            echo "Tag ${VERSION} already exists; skipping release."
+            echo "new=false" >> "$GITHUB_OUTPUT"
+          else
+            git config user.name "github-actions[bot]"
+            git config user.email "github-actions[bot]@users.noreply.github.com"
+            git tag -a "${VERSION}" -m "Release ${VERSION}"
+            git push origin "${VERSION}"
+            echo "new=true" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Set up Go
+        if: steps.tag.outputs.new == 'true'
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+
+      - name: Run tests
+        if: steps.tag.outputs.new == 'true'
+        run: go test -race ./...
+
+      - name: Set up QEMU
+        if: steps.tag.outputs.new == 'true'
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        if: steps.tag.outputs.new == 'true'
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to GitHub Container Registry
+        if: steps.tag.outputs.new == 'true'
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Run goreleaser
+        if: steps.tag.outputs.new == 'true'
+        uses: goreleaser/goreleaser-action@v6
+        with:
+          distribution: goreleaser
+          version: "~> v2"
+          args: release --clean
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.goreleaser.yaml

@@ -0,0 +1,78 @@
+version: 2
+
+project_name: llm-testbench
+
+before:
+  hooks:
+    # Read-only verification only - release tooling must not mutate the tree.
+    - go mod verify
+
+builds:
+  - id: llmtest
+    main: ./cmd/llmtest
+    binary: llmtest
+    env:
+      - CGO_ENABLED=0
+    goos:
+      - linux
+      - darwin
+    goarch:
+      - amd64
+      - arm64
+    ldflags:
+      - -s -w -X main.version=v{{ .Version }}
+
+archives:
+  - id: llmtest
+    ids: [llmtest]
+    formats: [tar.gz]
+    name_template: "{{ .ProjectName }}_{{ .Version }}_{{ .Os }}_{{ .Arch }}"
+
+checksum:
+  name_template: checksums.txt
+
+dockers:
+  - id: llmtest-amd64
+    ids: [llmtest]
+    goarch: amd64
+    use: buildx
+    image_templates:
+      - ghcr.io/lukaszraczylo/llm-testbench:v{{ .Version }}-amd64
+    build_flag_templates:
+      - --platform=linux/amd64
+      - --label=org.opencontainers.image.title={{ .ProjectName }}
+      - --label=org.opencontainers.image.version=v{{ .Version }}
+      - --label=org.opencontainers.image.revision={{ .FullCommit }}
+      - --label=org.opencontainers.image.source=https://github.com/lukaszraczylo/llm-testbench
+  - id: llmtest-arm64
+    ids: [llmtest]
+    goarch: arm64
+    use: buildx
+    image_templates:
+      - ghcr.io/lukaszraczylo/llm-testbench:v{{ .Version }}-arm64
+    build_flag_templates:
+      - --platform=linux/arm64
+      - --label=org.opencontainers.image.title={{ .ProjectName }}
+      - --label=org.opencontainers.image.version=v{{ .Version }}
+      - --label=org.opencontainers.image.revision={{ .FullCommit }}
+      - --label=org.opencontainers.image.source=https://github.com/lukaszraczylo/llm-testbench
+
+docker_manifests:
+  - name_template: ghcr.io/lukaszraczylo/llm-testbench:v{{ .Version }}
+    image_templates:
+      - ghcr.io/lukaszraczylo/llm-testbench:v{{ .Version }}-amd64
+      - ghcr.io/lukaszraczylo/llm-testbench:v{{ .Version }}-arm64
+  - name_template: ghcr.io/lukaszraczylo/llm-testbench:latest
+    image_templates:
+      - ghcr.io/lukaszraczylo/llm-testbench:v{{ .Version }}-amd64
+      - ghcr.io/lukaszraczylo/llm-testbench:v{{ .Version }}-arm64
+
+changelog:
+  use: github-native
+
+release:
+  github:
+    owner: lukaszraczylo
+    name: llm-testbench
+  draft: false
+  prerelease: auto

Dockerfile

@@ -0,0 +1,29 @@
+# Runtime image for the llm-testbench test runner. goreleaser copies the
+# prebuilt, statically linked linux binary into the build context, so this
+# file only assembles the runtime environment - it does not compile Go.
+#
+# The image carries the go, python3, and C toolchains so the exec
+# evaluators (GoRun/PyRun/CRun) run inside the container instead of being
+# skipped. That is the point of the image; a slim variant would silently
+# skip a third of the programming category.
+FROM golang:1.26-alpine
+
+RUN apk add --no-cache python3 gcc musl-dev \
+    && adduser -D -u 10001 llmtest \
+    && mkdir -p /work \
+    && chown llmtest:llmtest /work
+
+COPY llmtest /usr/local/bin/llmtest
+
+USER llmtest
+WORKDIR /work
+
+# Toolchain caches for the exec evaluators live in the user's home.
+ENV GOCACHE=/home/llmtest/.cache/go-build \
+    GOMODCACHE=/home/llmtest/go/pkg/mod
+
+# Mount a config at /work/config.yaml (or pass --config) and run:
+#   docker run --rm -v $PWD/config.yaml:/work/config.yaml \
+#     ghcr.io/lukaszraczylo/llm-testbench run --format table
+ENTRYPOINT ["/usr/local/bin/llmtest"]
+CMD ["--help"]

cmd/llmtest/main.go

@@ -21,6 +21,10 @@ import (
 // it is not exposed as a flag.
 const requestTemperature = 0.0
 
+// version is stamped by the release build via
+// -ldflags "-X main.version=vX.Y.Z"; a source build reports "dev".
+var version = "dev"
+
 func main() {
 	if len(os.Args) < 2 {
 		usage()
@@ -33,6 +37,9 @@ func main() {
 		err = runCommand(os.Args[2:])
 	case "list":
 		err = listCommand(os.Args[2:])
+	case "version", "-v", "--version":
+		fmt.Println("llmtest " + version)
+		return
 	case "-h", "--help", "help":
 		usage()
 		return
@@ -54,6 +61,7 @@ func usage() {
 Usage:
   llmtest run  [flags]   Run the test catalog against configured models.
   llmtest list [flags]   List the test catalog.
+  llmtest version        Print the build version.
 
 Run "llmtest run -h" or "llmtest list -h" for flag details.
 `)

semver.yaml

@@ -0,0 +1,20 @@
+version: 1
+force:
+  existing: true
+blacklist:
+  - "Merge branch"
+  - "Merge pull request"
+wording:
+  patch:
+    - fix
+    - chore
+    - docs
+    - style
+    - refactor
+    - perf
+    - ci
+  minor:
+    - feat
+  major:
+    - "breaking change"
+    - "major overhaul"