lukaszraczylo/claude-mnemonic
1
0
Fork 0
mirror of https://github.com/lukaszraczylo/claude-mnemonic.git synced 2026-10-12 04:55:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor: replace Git LFS with runtime model download from Hugging Face

Browse Source 
Remove ~170MB of model files from the repository (LFS + committed).
Models are now downloaded at runtime from Hugging Face on first use
and cached to the OS cache directory with progress reporting and retries.

- Add internal/models/download.go: runtime downloader with retry, progress bar, checksums
- Remove go:embed for ONNX models (keep tokenizers embedded)
- Use file-based ONNX session loading instead of byte-slice
- Add scripts/download-models.sh for dev/CI model setup
- Update Makefile with setup-models target
- Update workflow-prepare.sh to download models in CI
- Set lfs: false in all CI workflows
- SHA256: bge=828e14..., cross-encoder=5d3e70...
This commit is contained in:
lukaszraczylo
2026-05-26 17:53:30 +01:00
parent c8b462aaec
commit 1a4fea5c17
15 changed files with 431 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files
workflow-prepare.sh
+48 -29
View File
@@ -10,46 +10,65 @@ HOST_OS=$(uname -s | tr '[:upper:]' '[:lower:]')
# Determine target platform for ONNX library download
# Use TARGET_GOOS/TARGET_GOARCH from CI matrix if available, otherwise auto-detect
if [ -n "$TARGET_GOOS" ] && [ -n "$TARGET_GOARCH" ]; then
ONNX_PLATFORM="${TARGET_GOOS}-${TARGET_GOARCH}"
echo "Target platform from CI matrix: $ONNX_PLATFORM"
ONNX_PLATFORM="${TARGET_GOOS}-${TARGET_GOARCH}"
echo "Target platform from CI matrix: $ONNX_PLATFORM"
else
ONNX_PLATFORM="auto"
ONNX_PLATFORM="auto"
fi
# On Windows, install SQLite development headers for CGO
if [[ "$HOST_OS" == mingw* ]] || [[ "$HOST_OS" == msys* ]] || [[ "$HOST_OS" == cygwin* ]]; then
echo "Installing SQLite for Windows..."
echo "Installing SQLite for Windows..."
# Download SQLite amalgamation and set up for CGO
SQLITE_VERSION="3470200"
SQLITE_YEAR="2024"
SQLITE_DIR="/c/sqlite"
SQLITE_URL="https://www.sqlite.org/${SQLITE_YEAR}/sqlite-amalgamation-${SQLITE_VERSION}.zip"
# Download SQLite amalgamation and set up for CGO
SQLITE_VERSION="3470200"
SQLITE_YEAR="2024"
SQLITE_DIR="/c/sqlite"
SQLITE_URL="https://www.sqlite.org/${SQLITE_YEAR}/sqlite-amalgamation-${SQLITE_VERSION}.zip"
mkdir -p "$SQLITE_DIR"
curl -sSL "$SQLITE_URL" -o /tmp/sqlite.zip
unzip -q /tmp/sqlite.zip -d /tmp/
cp /tmp/sqlite-amalgamation-${SQLITE_VERSION}/* "$SQLITE_DIR/"
rm -rf /tmp/sqlite.zip /tmp/sqlite-amalgamation-${SQLITE_VERSION}
mkdir -p "$SQLITE_DIR"
curl -sSL "$SQLITE_URL" -o /tmp/sqlite.zip
unzip -q /tmp/sqlite.zip -d /tmp/
cp /tmp/sqlite-amalgamation-${SQLITE_VERSION}/* "$SQLITE_DIR/"
rm -rf /tmp/sqlite.zip /tmp/sqlite-amalgamation-${SQLITE_VERSION}
# Download Go modules first so we can patch sqlite-vec
echo "Downloading Go modules..."
go mod download
# Download Go modules first so we can patch sqlite-vec
echo "Downloading Go modules..."
go mod download
# Find the sqlite-vec module and copy sqlite3.h there
SQLITE_VEC_PATH=$(go list -m -f '{{.Dir}}' github.com/asg017/sqlite-vec-go-bindings 2>/dev/null || true)
if [ -n "$SQLITE_VEC_PATH" ] && [ -d "$SQLITE_VEC_PATH/cgo" ]; then
# Make module writable (it's read-only by default)
chmod -R u+w "$SQLITE_VEC_PATH"
cp "$SQLITE_DIR/sqlite3.h" "$SQLITE_VEC_PATH/cgo/"
cp "$SQLITE_DIR/sqlite3.c" "$SQLITE_VEC_PATH/cgo/"
echo "SQLite headers copied to $SQLITE_VEC_PATH/cgo/"
fi
# Find the sqlite-vec module and copy sqlite3.h there
SQLITE_VEC_PATH=$(go list -m -f '{{.Dir}}' github.com/asg017/sqlite-vec-go-bindings 2>/dev/null || true)
if [ -n "$SQLITE_VEC_PATH" ] && [ -d "$SQLITE_VEC_PATH/cgo" ]; then
# Make module writable (it's read-only by default)
chmod -R u+w "$SQLITE_VEC_PATH"
cp "$SQLITE_DIR/sqlite3.h" "$SQLITE_VEC_PATH/cgo/"
cp "$SQLITE_DIR/sqlite3.c" "$SQLITE_VEC_PATH/cgo/"
echo "SQLite headers copied to $SQLITE_VEC_PATH/cgo/"
fi
# Tell linker to allow multiple definitions (both go-sqlite3 and sqlite-vec embed SQLite)
echo "CGO_LDFLAGS=-Wl,--allow-multiple-definition" >> "$GITHUB_ENV"
echo "SQLite setup complete"
# Tell linker to allow multiple definitions (both go-sqlite3 and sqlite-vec embed SQLite)
echo "CGO_LDFLAGS=-Wl,--allow-multiple-definition" >>"$GITHUB_ENV"
echo "SQLite setup complete"
fi
# Download ONNX runtime libraries for target platform
./scripts/download-onnx-libs.sh "$ONNX_PLATFORM"
# Download ONNX models from Hugging Face and stage for tests
# Non-fatal: if download fails, tests will attempt runtime download themselves
set +e
./scripts/download-models.sh
DOWNLOAD_EXIT=$?
set -e
if [ $DOWNLOAD_EXIT -eq 0 ]; then
# Export model directory so tests can find models without network access
if [ -n "$GITHUB_ENV" ]; then
echo "CLAUDE_MNEMONIC_MODEL_DIR=$GITHUB_WORKSPACE/testdata/models" >> "$GITHUB_ENV"
fi
else
echo "Warning: Model download failed — tests will try to download models at runtime"
fi
else
echo "Warning: Model download failed — tests will try to download models at runtime"
fi