{
  "formatVersion": 1,
  "scope": "Curated public research extract; not the full raw run.",
  "canonical": "https://embiro.ai/research/kinyarwanda-speech-diagnostic",
  "fieldNote": 13,
  "slug": "kinyarwanda-speech-diagnostic",
  "title": "Kinyarwanda: promising is not ready.",
  "description": "Forty recordings show why language coverage on a model card needs a local evaluation before a pilot release.",
  "category": "Speech",
  "art": "speech-wave",
  "thumbnail": {
    "src": "/research/thumbnails/kinyarwanda-speech-diagnostic.webp",
    "alt": "A mint magnifying lens examining a ceramic speech waveform."
  },
  "date": "2026-10-12",
  "published": "2026-10-12",
  "updated": "2026-10-12",
  "author": "Embiro Labs Research",
  "status": "Exploratory benchmark",
  "headline": "A language token is a starting point.",
  "metric": "35.83%",
  "metricLabel": "word error rate · 40 recordings",
  "method": [
    "Cape Town T4; production Sunflower weights converted to CTranslate2 INT8_FLOAT16, beam five, voice-activity detection and waveform normalisation.",
    "First twenty test rows from each of kin_commonvoice and kin_makbenchmark in the gated Sunbird speech benchmark, revision 4747e3bd3a1915de1cdf5fb032ec12e0cb68c19d. Forty recordings total 195.37 seconds. Access was granted before evaluation.",
    "WER/CER use NFKC, lowercasing, punctuation-to-space and collapsed whitespace, weighted by reference words/characters. Warm latency excludes model loading."
  ],
  "sections": [
    {
      "title": "Fast enough is only half the answer",
      "paragraphs": [
        "Warm inference processed 195.37 seconds of audio in 25.81 seconds, with a median clip time of 0.594 seconds. Yet combined word error rate was 35.83%. The difference between Common Voice and MAK also shows how dataset choice changes the picture."
      ]
    },
    {
      "title": "What we decided",
      "paragraphs": [
        "Keep Kinyarwanda as an evaluation candidate. Review representative errors with fluent speakers and test independent real recordings before enabling a general transcription offering. Do not compare this WER directly with Swahili or Luganda measured on FLEURS.",
        "Orpheus also generated two short Kinyarwanda voice samples. Valid WAV files establish technical execution, not correct pronunciation or naturalness; fluent listening review remains necessary."
      ]
    }
  ],
  "limitations": [
    "Small publisher-provided benchmark; independence from training data was not established. No gated recordings or transcripts are redistributed here.",
    "Two short TTS samples are insufficient to endorse a voice. Kinyarwanda was not enabled by this diagnostic."
  ],
  "tables": [
    {
      "title": "Sunflower Kinyarwanda",
      "columns": [
        "Subset",
        "Clips",
        "WER ↓",
        "CER ↓",
        "Audio (s)",
        "Inference (s)"
      ],
      "rows": [
        [
          "Common Voice",
          "20",
          "42.16%",
          "19.36%",
          "81.55",
          "12.37"
        ],
        [
          "MAK",
          "20",
          "31.58%",
          "12.68%",
          "113.82",
          "13.44"
        ],
        [
          "Combined",
          "40",
          "35.83%",
          "15.28%",
          "195.37",
          "25.81"
        ]
      ],
      "note": "Combined error rates are weighted by reference counts, not averages of the two percentages."
    }
  ],
  "references": [
    {
      "label": "SunflowerASR · model card",
      "url": "https://huggingface.co/Sunbird/SunflowerASR-51-african-languages"
    },
    {
      "label": "Sunbird speech benchmark · access-gated source",
      "url": "https://huggingface.co/datasets/Sunbird/speech-benchmark"
    }
  ],
  "audio": [],
  "evidence": [
    {
      "file": "README.md",
      "sha256": "8987348ef6dbd613f1573fa8570d2fd7ae1aa5bb1d5a52c8d06750438561cade"
    },
    {
      "file": "asr-results.json",
      "sha256": "0f3cd7e4b51c3ee3372d3ebc072756868862a0b27f18658de15e0b583228ed7f"
    }
  ]
}