[
  {
    "id": "citarium-core-validator",
    "title": "Citarium content validator source (@citarium/core v0.1.0, compiler/validate.ts)",
    "publisher": "Citarium framework",
    "tier": "primary",
    "reliability": 0.95,
    "accessed": "2026-08-08",
    "notes": "Proprietary framework code, verified locally in this instance's node_modules. Contains the 15 incident-driven check families (VALIDATOR_CHECKS, \"the counter CI tracks\"), the tier-overclaim check, and the locale-contract check born from the pt-BR incident. Header states every check exists because an incident paid for it."
  },
  {
    "id": "citarium-cli-audit",
    "title": "Citarium audit command source (@citarium/cli v0.1.0, commands.ts)",
    "publisher": "Citarium framework",
    "tier": "primary",
    "reliability": 0.95,
    "accessed": "2026-08-08",
    "notes": "Proprietary framework code, verified locally. Implements the honesty figures a maintainer must watch instead of a single score: evidence-tier mix vs quota, unused sources (\"inventory is not evidence\"), moat count, oldest verification date."
  },
  {
    "id": "citarium-methodology-brief",
    "title": "agent-reliability editorial brief and blueprint (Gate 1 approved, 2026-08-08)",
    "publisher": "Citarium / Santiago Santa Maria Morales",
    "tier": "primary",
    "reliability": 0.85,
    "accessed": "2026-08-08",
    "notes": "In-repo documents (docs/BRIEF.md, citarium.yaml). First-hand account of the falsified-guardian methodology (plant, red, restore, green), the check-counter regression alarm and the CAS-I/CAS-E Goodhart separation, described as living case studies with real incident history in production."
  },
  {
    "id": "santismm-eureka",
    "title": "EUREKA — applied AI, agents and robotics knowledge platform",
    "url": "https://santismm.com",
    "publisher": "Santiago Santa Maria Morales",
    "tier": "primary",
    "reliability": 0.9,
    "accessed": "2026-08-08",
    "notes": "The declared expert's production knowledge platform. Maintains a benchmark tracker and a public catalog of 69 agentic benchmarks; frames the field as the move from measuring capability to measuring agency."
  },
  {
    "id": "anthropic-eval-challenges",
    "title": "Challenges in evaluating AI systems",
    "url": "https://www.anthropic.com/news/evaluating-ai-systems",
    "publisher": "Anthropic",
    "tier": "primary",
    "reliability": 0.9,
    "accessed": "2026-08-08",
    "notes": "First-party account (2023-10-04) of why robust evaluations are extremely difficult to develop and implement; formatting inconsistencies alone moved MMLU scores by about 5%."
  },
  {
    "id": "openai-evals",
    "title": "openai/evals — framework for evaluating LLMs and LLM systems",
    "url": "https://github.com/openai/evals",
    "publisher": "OpenAI",
    "tier": "primary",
    "reliability": 0.85,
    "license": "MIT",
    "accessed": "2026-08-08",
    "notes": "Eval registry plus templates for basic and model-graded evals; supports private evals built on your own data without public exposure."
  },
  {
    "id": "nist-ai-rmf",
    "title": "NIST AI Risk Management Framework (AI RMF 1.0, NIST AI 100-1)",
    "url": "https://www.nist.gov/itl/ai-risk-management-framework",
    "publisher": "NIST",
    "tier": "primary",
    "reliability": 0.95,
    "accessed": "2026-08-08",
    "notes": "Released 2023-01-26. Govern / Map / Measure / Manage functions for trustworthiness across the AI lifecycle, from design through deployment and evaluation; generative-AI profile added July 2024."
  },
  {
    "id": "owasp-llm-top10",
    "title": "OWASP Top 10 for LLM Applications and Generative AI (2025)",
    "url": "https://genai.owasp.org/llm-top-10/",
    "publisher": "OWASP Gen AI Security Project",
    "tier": "primary",
    "reliability": 0.9,
    "accessed": "2026-08-08",
    "notes": "The standard document itself. 2025 edition ranks Prompt Injection as LLM01 and Excessive Agency as LLM06; risk categories and mitigations span development, deployment and management phases."
  },
  {
    "id": "inspect-ai",
    "title": "Inspect — evaluation framework for large language models",
    "url": "https://inspect.aisi.org.uk/",
    "publisher": "UK AI Security Institute",
    "tier": "primary",
    "reliability": 0.9,
    "accessed": "2026-08-08",
    "notes": "First-party documentation. Open-source (GitHub UKGovernmentBEIS/inspect_ai): solvers, scorers, model-graded and text-based grading, sandboxing via Docker/Kubernetes/Modal and others, ReAct and multi-agent primitives, ability to drive external agents (Claude Code, Codex CLI, Gemini CLI), 200+ prebuilt evals."
  },
  {
    "id": "anthropic-model-deprecations",
    "title": "Anthropic model deprecations (Claude API documentation)",
    "url": "https://platform.claude.com/docs/en/docs/about-claude/model-deprecations",
    "publisher": "Anthropic",
    "tier": "primary",
    "reliability": 0.95,
    "accessed": "2026-08-08",
    "notes": "First-party lifecycle policy: Active / Legacy / Deprecated / Retired states; at least 60 days' notice before retirement of public models; requests to retired models fail; vendor recommends testing replacements well before retirement and auditing usage by API key and model. Deprecation history 2024-2026 with recommended replacements per model."
  },
  {
    "id": "otel-genai-semconv",
    "title": "OpenTelemetry semantic conventions for generative AI",
    "url": "https://github.com/open-telemetry/semantic-conventions-genai",
    "publisher": "OpenTelemetry (CNCF)",
    "tier": "primary",
    "reliability": 0.8,
    "license": "Apache-2.0",
    "accessed": "2026-08-08",
    "notes": "The standard's own repository: spans, metrics and events for GenAI clients, MCP and provider-specific conventions, covering agent spans, tool execution and token usage. In active development — schema URL still marked TODO — so pin versions when adopting."
  },
  {
    "id": "anthropic-building-agents",
    "title": "Building effective agents",
    "url": "https://www.anthropic.com/research/building-effective-agents",
    "publisher": "Anthropic",
    "tier": "primary",
    "reliability": 0.9,
    "accessed": "2026-08-08",
    "notes": "First-party engineering guidance (2024-12-19): agents should pause for human feedback at checkpoints or blockers; autonomous operation and compounding errors demand extensive sandboxed testing plus guardrails; add complexity only when it demonstrably improves outcomes; simplicity, transparency and tool documentation as core principles."
  },
  {
    "id": "anthropic-red-teaming",
    "title": "Challenges in red teaming AI systems",
    "url": "https://www.anthropic.com/news/challenges-in-red-teaming-ai-systems",
    "publisher": "Anthropic",
    "tier": "primary",
    "reliability": 0.9,
    "accessed": "2026-08-08",
    "notes": "First-party account (2024-06-12) of red-teaming practice: domain expert, frontier-threats (CBRN, cyber, autonomous AI), multilingual, automated model-vs-model, multimodal, crowdsourced and community methods, with policy recommendations including standards and third-party access."
  },
  {
    "id": "swebench-verified-dataset",
    "title": "SWE-bench Verified dataset card (princeton-nlp/SWE-bench_Verified)",
    "url": "https://huggingface.co/datasets/princeton-nlp/SWE-bench_Verified",
    "publisher": "Princeton NLP (Hugging Face)",
    "tier": "primary",
    "reliability": 0.85,
    "accessed": "2026-08-08",
    "notes": "The dataset's own card: 500 samples from the SWE-bench test set, human-validated for quality; grading via unit-test verification using post-PR behavior as the reference solution."
  },
  {
    "id": "mcp-spec",
    "title": "Model Context Protocol specification (2025-06-18)",
    "url": "https://modelcontextprotocol.io/specification/2025-06-18",
    "publisher": "Model Context Protocol project",
    "tier": "primary",
    "reliability": 0.9,
    "accessed": "2026-08-08",
    "notes": "The authoritative spec: JSON-RPC 2.0 between hosts, clients and servers with capability negotiation; servers expose tools, resources and prompts; clients may offer sampling, roots, elicitation. Security section: tools are arbitrary code execution, tool descriptions/annotations are untrusted unless from a trusted server, explicit user consent required before invoking any tool, sampling requires user approval with limited server visibility."
  },
  {
    "id": "mt-bench-llm-judge",
    "title": "Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena",
    "url": "https://arxiv.org/abs/2306.05685",
    "publisher": "arXiv (Zheng et al.)",
    "tier": "secondary",
    "reliability": 0.9,
    "accessed": "2026-08-08",
    "notes": "Strong LLM judges reach over 80% agreement with human preferences; documents position, verbosity and self-enhancement biases plus limited reasoning ability in judges."
  },
  {
    "id": "goodhart-variants",
    "title": "Categorizing Variants of Goodhart's Law",
    "url": "https://arxiv.org/abs/1803.04585",
    "publisher": "arXiv (Manheim & Garrabrant)",
    "tier": "secondary",
    "reliability": 0.85,
    "accessed": "2026-08-08",
    "notes": "Identifies at least four distinct mechanisms by which optimizing a proxy metric degrades what the metric was meant to measure."
  },
  {
    "id": "swe-bench",
    "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?",
    "url": "https://arxiv.org/abs/2310.06770",
    "publisher": "arXiv (Jimenez et al.)",
    "tier": "secondary",
    "reliability": 0.9,
    "accessed": "2026-08-08",
    "notes": "2,294 real GitHub issues from Python repositories, graded by execution; requires coordinating multi-file edits and interacting with execution environments. Best 2023 baseline resolved 1.96%."
  },
  {
    "id": "tau-bench",
    "title": "tau-bench: A Benchmark for Tool-Agent-User Interaction in Real-World Domains",
    "url": "https://arxiv.org/abs/2406.12045",
    "publisher": "arXiv (Yao et al.)",
    "tier": "secondary",
    "reliability": 0.9,
    "accessed": "2026-08-08",
    "notes": "Grades agents by comparing the final database state against an annotated goal state. Introduces pass^k for reliability over repeated trials; state-of-the-art function-calling agents succeeded on under 50% of tasks, with pass^8 under 25% in retail."
  },
  {
    "id": "codex-humaneval",
    "title": "Evaluating Large Language Models Trained on Code",
    "url": "https://arxiv.org/abs/2107.03374",
    "publisher": "arXiv (Chen et al., OpenAI)",
    "tier": "secondary",
    "reliability": 0.9,
    "accessed": "2026-08-09",
    "notes": "Introduces Codex and releases HumanEval, an evaluation set measuring functional correctness of programs synthesized from docstrings. The canonical any-of-k result: the model solves 28.8% of problems with a single sample and 70.2% with 100 samples per problem, establishing repeated sampling as a capability-raising strategy — the opposite direction to reliability metrics that require every trial to succeed."
  },
  {
    "id": "deepmind-dangerous-capabilities",
    "title": "Evaluating Frontier Models for Dangerous Capabilities",
    "url": "https://arxiv.org/abs/2403.13793",
    "publisher": "arXiv (Phuong et al., Google DeepMind)",
    "tier": "secondary",
    "reliability": 0.9,
    "accessed": "2026-08-08",
    "notes": "Evaluation framework piloted on Gemini 1.0 models across four domains: persuasion and deception, cybersecurity, self-proliferation, and self-reasoning."
  },
  {
    "id": "metr",
    "title": "METR — Model Evaluation & Threat Research",
    "url": "https://metr.org/",
    "publisher": "METR",
    "tier": "secondary",
    "reliability": 0.85,
    "accessed": "2026-08-08",
    "notes": "Autonomy evaluations of frontier models; the time-horizon metric (length of tasks agents can complete, rising exponentially); Hawk, an open-source large-scale agent-evaluation platform built on Inspect; research on evaluation integrity threats and monitoring."
  },
  {
    "id": "gaia-benchmark-paper",
    "title": "GAIA: a benchmark for General AI Assistants",
    "url": "https://arxiv.org/abs/2311.12983",
    "publisher": "arXiv (Mialon et al.)",
    "tier": "secondary",
    "reliability": 0.9,
    "accessed": "2026-08-08",
    "notes": "466 real-world questions testing reasoning, multi-modality, web browsing and tool use; humans score 92% vs 15% for GPT-4 with plugins."
  },
  {
    "id": "agentbench-paper",
    "title": "AgentBench: Evaluating LLMs as Agents",
    "url": "https://arxiv.org/abs/2308.03688",
    "publisher": "arXiv (Liu et al.)",
    "tier": "secondary",
    "reliability": 0.9,
    "accessed": "2026-08-08",
    "notes": "Eight distinct environments for LLM-as-agent evaluation; top commercial models act well as agents while sub-70B open models lag markedly; failures concentrate in long-term reasoning, decision-making and instruction following."
  },
  {
    "id": "webarena-paper",
    "title": "WebArena: A Realistic Web Environment for Building Autonomous Agents",
    "url": "https://arxiv.org/abs/2307.13854",
    "publisher": "arXiv (Zhou et al.)",
    "tier": "secondary",
    "reliability": 0.9,
    "accessed": "2026-08-08",
    "notes": "Self-hosted realistic sites across e-commerce, forums, collaborative development and content management; grades functional correctness of long-horizon tasks; best GPT-4 agent 14.41% vs human 78.24%."
  },
  {
    "id": "helm-stanford",
    "title": "HELM — Holistic Evaluation of Language Models",
    "url": "https://crfm.stanford.edu/helm/",
    "publisher": "Stanford CRFM",
    "tier": "secondary",
    "reliability": 0.9,
    "accessed": "2026-08-08",
    "notes": "Public, transparent, reproducible benchmark suites and leaderboards evaluating models across many scenarios and metrics; the reference project for standardized open evaluation practice."
  },
  {
    "id": "osworld-paper",
    "title": "OSWorld: Benchmarking Multimodal Agents for Open-Ended Tasks in Real Computer Environments",
    "url": "https://arxiv.org/abs/2404.07972",
    "publisher": "arXiv (Xie et al.)",
    "tier": "secondary",
    "reliability": 0.9,
    "accessed": "2026-08-08",
    "notes": "369 real computer tasks across web and desktop apps, OS file I/O and multi-app workflows on Ubuntu/Windows/macOS; each task ships an initial-state setup and an execution-based evaluation script. Humans 72.36% vs best model 12.24%; failures concentrate in GUI grounding and operational knowledge."
  },
  {
    "id": "error-bars-evals",
    "title": "Adding Error Bars to Evals: A Statistical Approach to Language Model Evaluations",
    "url": "https://arxiv.org/abs/2411.00640",
    "publisher": "arXiv (Evan Miller)",
    "tier": "secondary",
    "reliability": 0.85,
    "accessed": "2026-08-08",
    "notes": "Conceptualizes eval questions as draws from an unseen super-population; gives formulas and reporting recommendations that minimize statistical noise and maximize informativeness of model comparisons."
  },
  {
    "id": "ragas-paper",
    "title": "Ragas: Automated Evaluation of Retrieval Augmented Generation",
    "url": "https://arxiv.org/abs/2309.15217",
    "publisher": "arXiv (Es et al.)",
    "tier": "secondary",
    "reliability": 0.85,
    "accessed": "2026-08-08",
    "notes": "Reference-free evaluation of RAG pipelines without ground-truth annotations: retrieval's ability to find relevant, focused context; the model's faithful use of retrieved passages; and generation quality."
  },
  {
    "id": "verify-step-by-step",
    "title": "Let's Verify Step by Step",
    "url": "https://arxiv.org/abs/2305.20050",
    "publisher": "arXiv (Lightman et al., OpenAI)",
    "tier": "secondary",
    "reliability": 0.9,
    "accessed": "2026-08-08",
    "notes": "Process supervision (feedback on each intermediate step) significantly outperforms outcome supervision on MATH; the process-supervised reward model reaches 78% on a representative test subset. Releases PRM800K, 800k step-level human labels."
  },
  {
    "id": "principles-chaos",
    "title": "Principles of Chaos Engineering",
    "url": "https://principlesofchaos.org/",
    "publisher": "principlesofchaos.org community",
    "tier": "secondary",
    "reliability": 0.8,
    "accessed": "2026-08-08",
    "notes": "Steady-state hypothesis (measurable output indicating normal behavior), inject real-world events, minimize blast radius; four-step experimental method with control and test groups."
  }
]