{
 "kind": "tool",
 "slug": "evaluate",
 "path": "tools/evaluate/",
 "name": "Evaluate",
 "maker": null,
 "density": "basic",
 "crumbs": [
  [
   "Tools",
   "tools/"
  ],
  [
   "Prompts, structured output and evals",
   "tools/llm-tooling/"
  ]
 ],
 "lead": "HuggingFace library for evaluating machine learning models with standardized metrics.",
 "chips": [
  {
   "label": "Prompts, structured output and evals",
   "path": "tools/llm-tooling/"
  },
  {
   "label": "PYPI"
  }
 ],
 "primary": {
  "label": "Copy install command",
  "copy": "pip install evaluate"
 },
 "links": [
  {
   "label": "GitHub",
   "href": "https://github.com/huggingface/evaluate"
  },
  {
   "label": "Website",
   "href": "https://github.com/huggingface/evaluate"
  }
 ],
 "track": "New versions, deprecation and license changes on your dashboard",
 "verdicts": [
  {
   "key": "license",
   "label": "License",
   "value": "Apache-2.0",
   "status": "yes",
   "sub": "Use it in paid apps",
   "quote": null,
   "note": "License from PyPI classifier.",
   "src": "https://github.com/huggingface/evaluate"
  },
  {
   "key": "maint",
   "label": "Maintained",
   "value": "Maintained",
   "status": "yes",
   "sub": "Last change Nov 16, 2026",
   "note": "Latest GitHub release Sep 18, 2025. ",
   "src": "https://github.com/huggingface/evaluate"
  },
  {
   "key": "pop",
   "label": "Used by agents' projects",
   "value": "4 projects",
   "status": "info",
   "sub": "#25 of 78 in its category",
   "note": "Dependency in 4 public GitHub projects set up for AI coding agents that we scanned (0.1%).",
   "src": null
  },
  {
   "key": "agent",
   "label": "Agent access",
   "value": "Community skills",
   "status": "info",
   "sub": "3 skills by others",
   "note": "Agent skills for this library in the skills catalog; community skills are written by others, read them before you install.",
   "src": null
  }
 ],
 "vfoot": "From the package registry and GitHub, checked daily.",
 "blocks": [
  {
   "id": "overview",
   "nav": "Overview",
   "h2": "What Evaluate does",
   "parts": [
    {
     "t": "prose",
     "text": "HuggingFace library for evaluating machine learning models with standardized metrics."
    }
   ]
  },
  {
   "id": "popularity",
   "nav": "Popularity",
   "h2": "Evaluate popularity",
   "parts": [
    {
     "t": "stats",
     "items": [
      {
       "n": "48",
       "of": "/ 100",
       "l": "Gid Score, #2041 overall"
      },
      {
       "n": "1.1M",
       "l": "downloads a week, pypi"
      },
      {
       "n": "2,497",
       "l": "GitHub stars"
      },
      {
       "n": "4",
       "l": "agent projects use it"
      }
     ]
    },
    {
     "t": "small",
     "text": "Gid Score measures public popularity and activity, not quality.",
     "link": "How the Gid Score is calculated",
     "path": "tools/top/"
    }
   ]
  },
  {
   "id": "usage",
   "nav": "How it is used",
   "h2": "How Evaluate is used",
   "parts": [
    {
     "t": "list",
     "items": [
      "Evaluate is a dependency in 4 of the 6,254 public GitHub projects set up for AI coding agents that we scanned, too few to describe it from them alone. Our scan also samples projects by dependency (Solana 3,063, Ethereum and EVM 1,777, 3D and games 3,120, agent frameworks and SDKs 13,629); counting those, it appears in 63 projects, and what it is used with below comes from all of them.",
      "The most used alternative in our scan is Ragas (3 projects).",
      "Projects that use it most often also use Hugging Face Datasets (88.9%), NumPy (81.0%), pandas (81.0%)."
     ]
    },
    {
     "t": "h3",
     "text": "Often used with"
    },
    {
     "t": "rows",
     "rows": [
      {
       "l": "In 88.9% of its projects",
       "v": "Hugging Face Datasets",
       "path": "tools/datasets/"
      },
      {
       "l": "In 81.0% of its projects",
       "v": "NumPy",
       "path": "tools/numpy/"
      },
      {
       "l": "In 81.0% of its projects",
       "v": "pandas",
       "path": "tools/pandas/"
      },
      {
       "l": "In 79.4% of its projects",
       "v": "Accelerate",
       "path": "tools/accelerate/"
      },
      {
       "l": "In 73.0% of its projects",
       "v": "OpenAI Python Library",
       "path": "tools/openai-pypi/"
      },
      {
       "l": "In 68.3% of its projects",
       "v": "Hugging Face Transformers",
       "path": "tools/transformers/"
      },
      {
       "l": "In 66.7% of its projects",
       "v": "Hugging Face Hub",
       "path": "tools/huggingface-hub/"
      },
      {
       "l": "In 66.7% of its projects",
       "v": "LangChain",
       "path": "tools/langchain-pypi/"
      }
     ]
    }
   ]
  },
  {
   "id": "more",
   "nav": null,
   "spy_as": "usage",
   "h2": "More about Evaluate",
   "parts": [
    {
     "t": "more",
     "items": [
      {
       "title": "Agent Skills for it",
       "path": "skills/",
       "items": [
        "evaluate: Use when the user wants proof that finished work meets its acceptance criteria, not just that tests pass: 'verify it is actually done', 'che",
        "evaluate-presets: Evaluate and audit Ralph's built-in presets: run the evaluation scripts, check hat routing, review a preset's design. Use after changing a p",
        "evaluate-rag: Guides evaluation of RAG pipeline retrieval and generation quality. Use when evaluating a retrieval-augmented generation system, measuring r"
       ]
      },
      {
       "title": "More in Prompts, structured output and evals",
       "path": "tools/llm-tooling/",
       "items": [
        {
         "text": "Ultracite",
         "path": "tools/ultracite/"
        },
        {
         "text": "tiktoken (Python)",
         "path": "tools/tiktoken-pypi/"
        },
        {
         "text": "tokenizers",
         "path": "tools/tokenizers/"
        },
        {
         "text": "LangSmith",
         "path": "tools/langsmith-pypi/"
        },
        {
         "text": "GPT Tokenizer",
         "path": "tools/gpt-tokenizer/"
        },
        {
         "text": "Langfuse (Python)",
         "path": "tools/langfuse-pypi/"
        },
        {
         "text": "TokenLens",
         "path": "tools/tokenlens/"
        },
        {
         "text": "@graphql-codegen/typescript-operations",
         "path": "tools/graphql-codegen-typescript-operations/"
        }
       ]
      }
     ]
    }
   ]
  },
  {
   "id": "alternatives",
   "nav": "Alternatives",
   "h2": "Alternatives",
   "parts": [
    {
     "t": "alts",
     "items": [
      {
       "name": "Evaluate",
       "me": true,
       "facts": [
        "Apache-2.0",
        "Gid 48",
        "1.1M/wk"
       ]
      },
      {
       "name": "Ragas",
       "why": "Evaluate RAG and LLM applications with automated metrics and scoring.",
       "path": "tools/ragas/",
       "facts": [
        "Apache-2.0",
        "Gid 52",
        "283K/wk"
       ]
      }
     ],
     "more": "Compare Evaluate with another tool",
     "more_path": "tools/compare/"
    }
   ]
  },
  {
   "id": "start",
   "nav": "Get started",
   "h2": "Get started",
   "parts": [
    {
     "t": "install",
     "intro": "Install it.",
     "code": "pip install evaluate"
    },
    {
     "t": "gap",
     "text": "No step-by-step start for Evaluate yet: we have not written its hand-checked card."
    }
   ]
  }
 ],
 "glance": [
  [
   "Package",
   "pypi evaluate"
  ],
  [
   "Category",
   "Prompts, structured output and evals"
  ],
  [
   "License",
   "Apache-2.0"
  ],
  [
   "Gid Score",
   "48.4"
  ],
  [
   "Rank in category",
   "#25 of 78"
  ],
  [
   "GitHub",
   "huggingface/evaluate"
  ],
  [
   "Last change",
   "Nov 16, 2026"
  ]
 ],
 "good": [],
 "checked_line": "From the package registry, GitHub and our scan of public agent projects; numbers update daily.",
 "logo": "https://agentgid.com/media/avatars/huggingface.png"
}