diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..a047374 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,80 @@ +name: Release + +on: + push: + branches: [main] + workflow_dispatch: + +permissions: + contents: write + packages: write + +jobs: + release: + name: Version, tag, and release + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Calculate semantic version + id: semver + uses: lukaszraczylo/semver-generator@v1 + with: + config_file: semver.yaml + repository_local: true + existing: true + + - name: Create tag if new + id: tag + run: | + VERSION="v${{ steps.semver.outputs.semantic_version }}" + echo "version=${VERSION}" >> "$GITHUB_OUTPUT" + if git rev-parse "${VERSION}" >/dev/null 2>&1; then + echo "Tag ${VERSION} already exists; skipping release." + echo "new=false" >> "$GITHUB_OUTPUT" + else + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git tag -a "${VERSION}" -m "Release ${VERSION}" + git push origin "${VERSION}" + echo "new=true" >> "$GITHUB_OUTPUT" + fi + + - name: Set up Go + if: steps.tag.outputs.new == 'true' + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + + - name: Run tests + if: steps.tag.outputs.new == 'true' + run: go test -race ./... + + - name: Set up QEMU + if: steps.tag.outputs.new == 'true' + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + if: steps.tag.outputs.new == 'true' + uses: docker/setup-buildx-action@v3 + + - name: Log in to GitHub Container Registry + if: steps.tag.outputs.new == 'true' + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Run goreleaser + if: steps.tag.outputs.new == 'true' + uses: goreleaser/goreleaser-action@v6 + with: + distribution: goreleaser + version: "~> v2" + args: release --clean + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.goreleaser.yaml b/.goreleaser.yaml new file mode 100644 index 0000000..5db25d2 --- /dev/null +++ b/.goreleaser.yaml @@ -0,0 +1,78 @@ +version: 2 + +project_name: llm-testbench + +before: + hooks: + # Read-only verification only - release tooling must not mutate the tree. + - go mod verify + +builds: + - id: llmtest + main: ./cmd/llmtest + binary: llmtest + env: + - CGO_ENABLED=0 + goos: + - linux + - darwin + goarch: + - amd64 + - arm64 + ldflags: + - -s -w -X main.version=v{{ .Version }} + +archives: + - id: llmtest + ids: [llmtest] + formats: [tar.gz] + name_template: "{{ .ProjectName }}_{{ .Version }}_{{ .Os }}_{{ .Arch }}" + +checksum: + name_template: checksums.txt + +dockers: + - id: llmtest-amd64 + ids: [llmtest] + goarch: amd64 + use: buildx + image_templates: + - ghcr.io/lukaszraczylo/llm-testbench:v{{ .Version }}-amd64 + build_flag_templates: + - --platform=linux/amd64 + - --label=org.opencontainers.image.title={{ .ProjectName }} + - --label=org.opencontainers.image.version=v{{ .Version }} + - --label=org.opencontainers.image.revision={{ .FullCommit }} + - --label=org.opencontainers.image.source=https://github.com/lukaszraczylo/llm-testbench + - id: llmtest-arm64 + ids: [llmtest] + goarch: arm64 + use: buildx + image_templates: + - ghcr.io/lukaszraczylo/llm-testbench:v{{ .Version }}-arm64 + build_flag_templates: + - --platform=linux/arm64 + - --label=org.opencontainers.image.title={{ .ProjectName }} + - --label=org.opencontainers.image.version=v{{ .Version }} + - --label=org.opencontainers.image.revision={{ .FullCommit }} + - --label=org.opencontainers.image.source=https://github.com/lukaszraczylo/llm-testbench + +docker_manifests: + - name_template: ghcr.io/lukaszraczylo/llm-testbench:v{{ .Version }} + image_templates: + - ghcr.io/lukaszraczylo/llm-testbench:v{{ .Version }}-amd64 + - ghcr.io/lukaszraczylo/llm-testbench:v{{ .Version }}-arm64 + - name_template: ghcr.io/lukaszraczylo/llm-testbench:latest + image_templates: + - ghcr.io/lukaszraczylo/llm-testbench:v{{ .Version }}-amd64 + - ghcr.io/lukaszraczylo/llm-testbench:v{{ .Version }}-arm64 + +changelog: + use: github-native + +release: + github: + owner: lukaszraczylo + name: llm-testbench + draft: false + prerelease: auto diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..f230951 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,29 @@ +# Runtime image for the llm-testbench test runner. goreleaser copies the +# prebuilt, statically linked linux binary into the build context, so this +# file only assembles the runtime environment - it does not compile Go. +# +# The image carries the go, python3, and C toolchains so the exec +# evaluators (GoRun/PyRun/CRun) run inside the container instead of being +# skipped. That is the point of the image; a slim variant would silently +# skip a third of the programming category. +FROM golang:1.26-alpine + +RUN apk add --no-cache python3 gcc musl-dev \ + && adduser -D -u 10001 llmtest \ + && mkdir -p /work \ + && chown llmtest:llmtest /work + +COPY llmtest /usr/local/bin/llmtest + +USER llmtest +WORKDIR /work + +# Toolchain caches for the exec evaluators live in the user's home. +ENV GOCACHE=/home/llmtest/.cache/go-build \ + GOMODCACHE=/home/llmtest/go/pkg/mod + +# Mount a config at /work/config.yaml (or pass --config) and run: +# docker run --rm -v $PWD/config.yaml:/work/config.yaml \ +# ghcr.io/lukaszraczylo/llm-testbench run --format table +ENTRYPOINT ["/usr/local/bin/llmtest"] +CMD ["--help"] diff --git a/cmd/llmtest/main.go b/cmd/llmtest/main.go index ba10536..4970dd0 100644 --- a/cmd/llmtest/main.go +++ b/cmd/llmtest/main.go @@ -21,6 +21,10 @@ import ( // it is not exposed as a flag. const requestTemperature = 0.0 +// version is stamped by the release build via +// -ldflags "-X main.version=vX.Y.Z"; a source build reports "dev". +var version = "dev" + func main() { if len(os.Args) < 2 { usage() @@ -33,6 +37,9 @@ func main() { err = runCommand(os.Args[2:]) case "list": err = listCommand(os.Args[2:]) + case "version", "-v", "--version": + fmt.Println("llmtest " + version) + return case "-h", "--help", "help": usage() return @@ -54,6 +61,7 @@ func usage() { Usage: llmtest run [flags] Run the test catalog against configured models. llmtest list [flags] List the test catalog. + llmtest version Print the build version. Run "llmtest run -h" or "llmtest list -h" for flag details. `) diff --git a/semver.yaml b/semver.yaml new file mode 100644 index 0000000..7c57eb7 --- /dev/null +++ b/semver.yaml @@ -0,0 +1,20 @@ +version: 1 +force: + existing: true +blacklist: + - "Merge branch" + - "Merge pull request" +wording: + patch: + - fix + - chore + - docs + - style + - refactor + - perf + - ci + minor: + - feat + major: + - "breaking change" + - "major overhaul"