Model bench

1nika: v12workflow:3  id: model-bench4  description: "One question → three local models → a measured comparison table"5​6model: ollama/qwen3.5:4b # the house default · contenders below pick their own seats7​8const:9  question:10    # A TYPED constant is `{ type, value }` — both keys, exactly those two.11    # `{ type, default }` is not a declaration at all (01-envelope.md §const:12    # an object missing either key is a bare literal object), so the prompt13    # below would receive the whole `{type, default}` map instead of the14    # question. Measured: the model gets a JSON blob and the bench is junk.15    type: string16    value: "Explain, in five lines, why a workflow should be audited before it runs."17⛨permits:18  ⛨tools: ["nika:jq", "nika:write"]19  ⛨fs: { write: ["out/model-bench.md"] }20​21tasks:22  ask_incumbent:23    ◇infer:24      model: ollama/qwen2.5:14b # the quality bar · the seat to beat25      prompt: "${{ const.question }}"26      max_tokens: 60027​28  ask_challenger:29    ◇infer:30      model: ollama/llama3.2:3b # another family · same size class31      prompt: "${{ const.question }}"32      max_tokens: 60033​34  ask_tiny:35    ◇infer:36      model: ollama/qwen2.5:0.5b # the "is small enough?" probe37      prompt: "${{ const.question }}"38      max_tokens: 60039​40  tabulate:41    with:42      ask_incumbent_duration_ms: ${{ tasks.ask_incumbent.duration_ms }}43      ask_incumbent: ${{ tasks.ask_incumbent.output }}44      ask_challenger_duration_ms: ${{ tasks.ask_challenger.duration_ms }}45      ask_challenger: ${{ tasks.ask_challenger.output }}46      ask_tiny_duration_ms: ${{ tasks.ask_tiny.duration_ms }}47      ask_tiny: ${{ tasks.ask_tiny.output }}48    ◆invoke:49      tool: "nika:jq"50      args:51        input:52          - model: "ollama/qwen2.5:14b"53            ms: "${{ with.ask_incumbent_duration_ms }}"54            answer: "${{ with.ask_incumbent }}"55          - model: "ollama/llama3.2:3b"56            ms: "${{ with.ask_challenger_duration_ms }}"57            answer: "${{ with.ask_challenger }}"58          - model: "ollama/qwen2.5:0.5b"59            ms: "${{ with.ask_tiny_duration_ms }}"60            answer: "${{ with.ask_tiny }}"61        expression: >-62          "| model | latency | chars | answer (first 160) |\n|---|---|---|---|\n"63          + (map("| " + .model + " | "64               + (if .ms == null then "cached" else (.ms | tostring) + "ms" end) + " | "65               + (.answer | length | tostring) + " | "66               + (.answer | gsub("\n"; " ") | .[0:160]) + " |")67             | join("\n"))68​69  persist:70    with:71      tabulate: ${{ tasks.tabulate.output }}72    ◆invoke:73      tool: "nika:write"74      args:75        path: out/model-bench.md76        create_dirs: true77        content: |78          # Model bench — same question, measured79​80          ${{ with.tabulate }}81​82          Facts only: latency from the run's clock, length in characters.83          The answers are side by side — the quality call is yours.84​85outputs:86  table:87    value: ${{ tasks.tabulate.output }}88    description: "The measured comparison table · latency + length per seat"89​