Quick Start#

mkdir my-llm-app && cd my-llm-app
go mod init my-llm-app
go get github.com/zerfoo/zerfoo@latest

package main

import (
	"fmt"
	"log"

	"github.com/zerfoo/zerfoo"
)

func main() {
	m, err := zerfoo.Load("google/gemma-3-4b")
	if err != nil {
		log.Fatal(err)
	}
	defer m.Close()

	reply, err := m.Chat("Explain quicksort in one sentence.")
	if err != nil {
		log.Fatal(err)
	}
	fmt.Println(reply)
}

go run main.go

google/gemma-3-4b/Q8_0

package main

import (
	"context"
	"fmt"
	"log"

	"github.com/zerfoo/zerfoo/inference"
)

func main() {
	mdl, err := inference.Load("gemma-3-1b-q4")
	if err != nil {
		log.Fatal(err)
	}
	defer mdl.Close()

	resp, err := mdl.Chat(context.Background(), []inference.Message{
		{Role: "system", Content: "You are a helpful assistant."},
		{Role: "user", Content: "What is the capital of France?"},
	},
		inference.WithTemperature(0.5),
		inference.WithMaxTokens(64),
	)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Println(resp.Content)
	fmt.Printf("Tokens used: %d (prompt: %d, completion: %d)\n",
		resp.TokensUsed, resp.PromptTokens, resp.CompletionTokens)
}

zerfoo run gemma-3-1b-q4

Model loaded. Type your message (Ctrl-D to quit).

> What is the capital of France?
The capital of France is Paris.
>

package main

import (
	"context"
	"fmt"
	"log"

	"github.com/zerfoo/zerfoo"
)

func main() {
	m, err := zerfoo.Load("google/gemma-3-4b")
	if err != nil {
		log.Fatal(err)
	}
	defer m.Close()

	stream, err := m.ChatStream(context.Background(), "Write a haiku about Go.")
	if err != nil {
		log.Fatal(err)
	}
	for tok := range stream {
		if tok.Done {
			break
		}
		fmt.Print(tok.Text)
	}
	fmt.Println()
}

err = mdl.GenerateStream(ctx, "Tell me a joke.",
	generate.TokenStreamFunc(func(token string, done bool) error {
		if !done {
			fmt.Print(token)
		}
		return nil
	}),
	inference.WithMaxTokens(128),
)

zerfoo predict --model gemma-3-1b-q4 --prompt "Write a haiku about Go."

zerfoo serve gemma-3-1b-q4 --port 8080

curl http://localhost:8080/v1/embeddings \
  -H "Content-Type: application/json" \
  -d '{
    "model": "gemma-3-1b-q4",
    "input": "Zerfoo is an ML framework for Go."
  }'

// Using the standard net/http package
package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"log"
	"net/http"
)

func main() {
	body, _ := json.Marshal(map[string]any{
		"model": "gemma-3-1b-q4",
		"input": "Zerfoo is an ML framework for Go.",
	})

	resp, err := http.Post(
		"http://localhost:8080/v1/embeddings",
		"application/json",
		bytes.NewReader(body),
	)
	if err != nil {
		log.Fatal(err)
	}
	defer resp.Body.Close()

	var result map[string]any
	json.NewDecoder(resp.Body).Decode(&result)
	fmt.Println(result)
}

package main

import (
	"context"
	"fmt"
	"log"

	"github.com/zerfoo/zerfoo/inference"
)

func main() {
	mdl, err := inference.Load("gemma-3-1b-q4")
	if err != nil {
		log.Fatal(err)
	}
	defer mdl.Close()

	result, err := mdl.Generate(
		context.Background(),
		`Return a JSON object with the fields "name", "capital", and "population" for France. Output only valid JSON, no other text.`,
		inference.WithMaxTokens(128),
		inference.WithTemperature(0.0),
	)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Println(result)
}

zerfoo predict \
  --model gemma-3-1b-q4 \
  --prompt 'Return a JSON object with the fields "name", "capital", and "population" for France. Output only valid JSON, no other text.' \
  --temperature 0 \
  --max-tokens 128

result, err := m.Generate(context.Background(), "Tell me a joke.",
	zerfoo.WithGenTemperature(0.7),
	zerfoo.WithGenMaxTokens(128),
	zerfoo.WithGenTopP(0.9),
)
if err != nil {
	log.Fatal(err)
}
fmt.Println(result.Text)
fmt.Printf("Tokens: %d, Duration: %s\n", result.TokenCount, result.Duration)