{
  "title": "Follow the math path",
  "footnote": "Each measurement is one model on one card, at one prompt size for LLMs. Time-to-first-token is left out because it mirrors prefill. Green paths gained 30% or more, gold paths 5 to 30%, gray paths under 5%.",
  "height": 560,
  "kind": "sankey",
  "options": {
    "nodes": [
      {
        "id": "LLM prefill",
        "label": "LLM prefill",
        "_color": "muted"
      },
      {
        "id": "LLM decode",
        "label": "LLM decode",
        "_color": "muted"
      },
      {
        "id": "Image",
        "label": "Image",
        "_color": "muted"
      },
      {
        "id": "Video",
        "label": "Video",
        "_color": "muted"
      },
      {
        "id": "Embeddings",
        "label": "Embeddings",
        "_color": "muted"
      },
      {
        "id": "Reranking",
        "label": "Reranking",
        "_color": "muted"
      },
      {
        "id": "Speech",
        "label": "Speech",
        "_color": "muted"
      },
      {
        "id": "Vision",
        "label": "Vision",
        "_color": "muted"
      },
      {
        "id": "Decision",
        "label": "Decision",
        "_color": "muted"
      },
      {
        "id": "16-bit float (BF16, FP16)",
        "label": "16-bit float (BF16, FP16)",
        "_color": 2
      },
      {
        "id": "INT4 on vLLM",
        "label": "INT4 on vLLM",
        "_color": 2
      },
      {
        "id": "BF16 GGUF on llama.cpp",
        "label": "BF16 GGUF on llama.cpp",
        "_color": 2
      },
      {
        "id": "FP8 weights in ComfyUI",
        "label": "FP8 weights in ComfyUI",
        "_color": 2
      },
      {
        "id": "INT8 in ComfyUI",
        "label": "INT8 in ComfyUI",
        "_color": 2
      },
      {
        "id": "Q4 or Q8 GGUF",
        "label": "Q4 or Q8 GGUF",
        "_color": 2
      },
      {
        "id": "+30% or more",
        "label": "+30% or more",
        "_color": 1
      },
      {
        "id": "+5 to 30%",
        "label": "+5 to 30%",
        "_color": 6
      },
      {
        "id": "Under 5%",
        "label": "Under 5%",
        "_color": "muted"
      }
    ],
    "links": [
      {
        "source": "Speech",
        "target": "16-bit float (BF16, FP16)",
        "value": 4,
        "_color": "muted"
      },
      {
        "source": "Embeddings",
        "target": "16-bit float (BF16, FP16)",
        "value": 4,
        "_color": "muted"
      },
      {
        "source": "Image",
        "target": "16-bit float (BF16, FP16)",
        "value": 13,
        "_color": "muted"
      },
      {
        "source": "Image",
        "target": "INT8 in ComfyUI",
        "value": 2,
        "_color": "muted"
      },
      {
        "source": "LLM decode",
        "target": "Q4 or Q8 GGUF",
        "value": 18,
        "_color": "muted"
      },
      {
        "source": "LLM prefill",
        "target": "Q4 or Q8 GGUF",
        "value": 18,
        "_color": "muted"
      },
      {
        "source": "LLM decode",
        "target": "INT4 on vLLM",
        "value": 3,
        "_color": "muted"
      },
      {
        "source": "LLM prefill",
        "target": "INT4 on vLLM",
        "value": 3,
        "_color": "muted"
      },
      {
        "source": "LLM decode",
        "target": "16-bit float (BF16, FP16)",
        "value": 18,
        "_color": "muted"
      },
      {
        "source": "LLM prefill",
        "target": "16-bit float (BF16, FP16)",
        "value": 18,
        "_color": "muted"
      },
      {
        "source": "Reranking",
        "target": "16-bit float (BF16, FP16)",
        "value": 2,
        "_color": "muted"
      },
      {
        "source": "Speech",
        "target": "Q4 or Q8 GGUF",
        "value": 2,
        "_color": "muted"
      },
      {
        "source": "Vision",
        "target": "Q4 or Q8 GGUF",
        "value": 1,
        "_color": "muted"
      },
      {
        "source": "Image",
        "target": "FP8 weights in ComfyUI",
        "value": 1,
        "_color": "muted"
      },
      {
        "source": "Video",
        "target": "16-bit float (BF16, FP16)",
        "value": 1,
        "_color": "muted"
      },
      {
        "source": "Decision",
        "target": "Q4 or Q8 GGUF",
        "value": 1,
        "_color": "muted"
      },
      {
        "source": "LLM decode",
        "target": "BF16 GGUF on llama.cpp",
        "value": 3,
        "_color": "muted"
      },
      {
        "source": "LLM prefill",
        "target": "BF16 GGUF on llama.cpp",
        "value": 3,
        "_color": "muted"
      },
      {
        "source": "Video",
        "target": "INT8 in ComfyUI",
        "value": 2,
        "_color": "muted"
      },
      {
        "source": "16-bit float (BF16, FP16)",
        "target": "Under 5%",
        "value": 17,
        "_color": "muted"
      },
      {
        "source": "16-bit float (BF16, FP16)",
        "target": "+30% or more",
        "value": 30,
        "_color": 1
      },
      {
        "source": "INT8 in ComfyUI",
        "target": "Under 5%",
        "value": 1,
        "_color": "muted"
      },
      {
        "source": "Q4 or Q8 GGUF",
        "target": "Under 5%",
        "value": 40,
        "_color": "muted"
      },
      {
        "source": "INT4 on vLLM",
        "target": "+5 to 30%",
        "value": 2,
        "_color": 6
      },
      {
        "source": "INT4 on vLLM",
        "target": "Under 5%",
        "value": 1,
        "_color": "muted"
      },
      {
        "source": "INT4 on vLLM",
        "target": "+30% or more",
        "value": 3,
        "_color": 1
      },
      {
        "source": "16-bit float (BF16, FP16)",
        "target": "+5 to 30%",
        "value": 13,
        "_color": 6
      },
      {
        "source": "FP8 weights in ComfyUI",
        "target": "+30% or more",
        "value": 1,
        "_color": 1
      },
      {
        "source": "INT8 in ComfyUI",
        "target": "+5 to 30%",
        "value": 3,
        "_color": 6
      },
      {
        "source": "BF16 GGUF on llama.cpp",
        "target": "Under 5%",
        "value": 3,
        "_color": "muted"
      },
      {
        "source": "BF16 GGUF on llama.cpp",
        "target": "+30% or more",
        "value": 2,
        "_color": 1
      },
      {
        "source": "BF16 GGUF on llama.cpp",
        "target": "+5 to 30%",
        "value": 1,
        "_color": 6
      }
    ],
    "nodePadding": 10,
    "labelMaxWidth": 150
  },
  "rows": [
    {
      "label": "Decision via Q4 or Q8 GGUF",
      "count": 1
    },
    {
      "label": "Embeddings via 16-bit float (BF16, FP16)",
      "count": 4
    },
    {
      "label": "Image via 16-bit float (BF16, FP16)",
      "count": 13
    },
    {
      "label": "Image via FP8 weights in ComfyUI",
      "count": 1
    },
    {
      "label": "Image via INT8 in ComfyUI",
      "count": 2
    },
    {
      "label": "LLM decode via 16-bit float (BF16, FP16)",
      "count": 18
    },
    {
      "label": "LLM decode via BF16 GGUF on llama.cpp",
      "count": 3
    },
    {
      "label": "LLM decode via INT4 on vLLM",
      "count": 3
    },
    {
      "label": "LLM decode via Q4 or Q8 GGUF",
      "count": 18
    },
    {
      "label": "LLM prefill via 16-bit float (BF16, FP16)",
      "count": 18
    },
    {
      "label": "LLM prefill via BF16 GGUF on llama.cpp",
      "count": 3
    },
    {
      "label": "LLM prefill via INT4 on vLLM",
      "count": 3
    },
    {
      "label": "LLM prefill via Q4 or Q8 GGUF",
      "count": 18
    },
    {
      "label": "Reranking via 16-bit float (BF16, FP16)",
      "count": 2
    },
    {
      "label": "Speech via 16-bit float (BF16, FP16)",
      "count": 4
    },
    {
      "label": "Speech via Q4 or Q8 GGUF",
      "count": 2
    },
    {
      "label": "Video via 16-bit float (BF16, FP16)",
      "count": 1
    },
    {
      "label": "Video via INT8 in ComfyUI",
      "count": 2
    },
    {
      "label": "Vision via Q4 or Q8 GGUF",
      "count": 1
    },
    {
      "label": "16-bit float (BF16, FP16) to +30% or more",
      "count": 30
    },
    {
      "label": "16-bit float (BF16, FP16) to +5 to 30%",
      "count": 13
    },
    {
      "label": "16-bit float (BF16, FP16) to Under 5%",
      "count": 17
    },
    {
      "label": "BF16 GGUF on llama.cpp to +30% or more",
      "count": 2
    },
    {
      "label": "BF16 GGUF on llama.cpp to +5 to 30%",
      "count": 1
    },
    {
      "label": "BF16 GGUF on llama.cpp to Under 5%",
      "count": 3
    },
    {
      "label": "FP8 weights in ComfyUI to +30% or more",
      "count": 1
    },
    {
      "label": "INT4 on vLLM to +30% or more",
      "count": 3
    },
    {
      "label": "INT4 on vLLM to +5 to 30%",
      "count": 2
    },
    {
      "label": "INT4 on vLLM to Under 5%",
      "count": 1
    },
    {
      "label": "INT8 in ComfyUI to +5 to 30%",
      "count": 3
    },
    {
      "label": "INT8 in ComfyUI to Under 5%",
      "count": 1
    },
    {
      "label": "Q4 or Q8 GGUF to Under 5%",
      "count": 40
    }
  ],
  "series": [
    {
      "key": "count",
      "label": "Measurements"
    }
  ],
  "subtitle": "117 before and after measurements across every workload. Left to right: what the job is, how it computes, what it gained.",
  "__hash__": "tpAqFJEsP0iMP1k72VMUgPHQleCZngtHyorvZcrsDDk"
}