{"$schema":"https://botlandscape.com/claims.schema.json","schema_version":1,"projection":"filtered-compare-index","source_schema_version":3,"compare_index_url":"https://botlandscape.com/compare-index.json","entity_id":"benchmark--vlabench","convention":{"class":"exact data-class","metric_id":"exact row id","state":"reported or missing","evidence_grade":"manual, filing, research, press, or analyst"},"counts":{"tables":1,"rows":16,"claims":16,"sources":2},"tables":[{"key":"topics/benchmarks.html::vla-manipulation--vla-manipulation-eval","page_path":"topics/benchmarks.html","id":"vla-manipulation--vla-manipulation-eval","class":"vla-manipulation-eval","label":"Robotics benchmarks · Manipulation evals","caption":"Sources: VLABench paper ; VLABench project ; RoboTwin 2.0 paper ; RoboTwin 2.0 project . Identity and protocol only; method scores remain in the papers and official leaderboards.","subjects":[{"id":"vlabench","label":"VLABench"}],"rows":[{"id":"vla-manipulation--kind","metric":"Kind","url":"https://botlandscape.com/benchmarks#vla-manipulation--kind","cells":[{"id":"topics/benchmarks.html::vla-manipulation--vla-manipulation-eval::vla-manipulation--kind::vlabench","column":1,"subject_ids":["vlabench"],"state":"reported","text":"language-conditioned manipulation benchmark for VLAs, foundation-model workflows, and VLMs","evidence":["benchmarks-r32"],"links":[]}]},{"id":"vla-manipulation--first-public-date","metric":"First public date","url":"https://botlandscape.com/benchmarks#vla-manipulation--first-public-date","cells":[{"id":"topics/benchmarks.html::vla-manipulation--vla-manipulation-eval::vla-manipulation--first-public-date::vlabench","column":1,"subject_ids":["vlabench"],"state":"reported","text":"arXiv 2024-12-24","evidence":["benchmarks-r32"],"links":[]}]},{"id":"vla-manipulation--paper","metric":"Paper","url":"https://botlandscape.com/benchmarks#vla-manipulation--paper","cells":[{"id":"topics/benchmarks.html::vla-manipulation--vla-manipulation-eval::vla-manipulation--paper::vlabench","column":1,"subject_ids":["vlabench"],"state":"reported","text":"arXiv:2412.18194; ICCV 2025","evidence":["benchmarks-r32","benchmarks-r33"],"links":[]}]},{"id":"vla-manipulation--task-structure","metric":"Task structure","url":"https://botlandscape.com/benchmarks#vla-manipulation--task-structure","cells":[{"id":"topics/benchmarks.html::vla-manipulation--vla-manipulation-eval::vla-manipulation--task-structure::vlabench","column":1,"subject_ids":["vlabench"],"state":"reported","text":"100 task categories: 60 primitive + 40 composite","evidence":["benchmarks-r32"],"links":[]}]},{"id":"vla-manipulation--asset-library","metric":"Asset library","url":"https://botlandscape.com/benchmarks#vla-manipulation--asset-library","cells":[{"id":"topics/benchmarks.html::vla-manipulation--vla-manipulation-eval::vla-manipulation--asset-library::vlabench","column":1,"subject_ids":["vlabench"],"state":"reported","text":"2,164 objects across 163 categories","evidence":["benchmarks-r32"],"links":[]}]},{"id":"vla-manipulation--released-trajectories","metric":"Released trajectories","url":"https://botlandscape.com/benchmarks#vla-manipulation--released-trajectories","cells":[{"id":"topics/benchmarks.html::vla-manipulation--vla-manipulation-eval::vla-manipulation--released-trajectories::vlabench","column":1,"subject_ids":["vlabench"],"state":"reported","text":"standard evaluation episodes and primitive-task fine-tuning data are public; no one suite-wide trajectory count is stated","evidence":["benchmarks-r33"],"links":[]}]},{"id":"vla-manipulation--evaluation-scope","metric":"Evaluation scope","url":"https://botlandscape.com/benchmarks#vla-manipulation--evaluation-scope","cells":[{"id":"topics/benchmarks.html::vla-manipulation--vla-manipulation-eval::vla-manipulation--evaluation-scope::vlabench","column":1,"subject_ids":["vlabench"],"state":"reported","text":"6 capability dimensions: mesh / texture, spatial, common sense / knowledge, semantic instruction, physical laws, and long-horizon reasoning","evidence":["benchmarks-r32"],"links":[]}]},{"id":"vla-manipulation--medium","metric":"Medium","url":"https://botlandscape.com/benchmarks#vla-manipulation--medium","cells":[{"id":"topics/benchmarks.html::vla-manipulation--vla-manipulation-eval::vla-manipulation--medium::vlabench","column":1,"subject_ids":["vlabench"],"state":"reported","text":"simulation","evidence":["benchmarks-r32"],"links":[]}]},{"id":"vla-manipulation--embodiment-in-the-source","metric":"Embodiment in the source","url":"https://botlandscape.com/benchmarks#vla-manipulation--embodiment-in-the-source","cells":[{"id":"topics/benchmarks.html::vla-manipulation--vla-manipulation-eval::vla-manipulation--embodiment-in-the-source::vlabench","column":1,"subject_ids":["vlabench"],"state":"reported","text":"standard evaluation: 7-DoF Franka Emika Panda with parallel gripper; framework also supports other forms","evidence":["benchmarks-r32"],"links":[]}]},{"id":"vla-manipulation--engine-renderer","metric":"Engine / renderer","url":"https://botlandscape.com/benchmarks#vla-manipulation--engine-renderer","cells":[{"id":"topics/benchmarks.html::vla-manipulation--vla-manipulation-eval::vla-manipulation--engine-renderer::vlabench","column":1,"subject_ids":["vlabench"],"state":"reported","text":"MuJoCo + dm_control","evidence":["benchmarks-r32"],"links":[]}]},{"id":"vla-manipulation--primary-metric-family","metric":"Primary metric family","url":"https://botlandscape.com/benchmarks#vla-manipulation--primary-metric-family","cells":[{"id":"topics/benchmarks.html::vla-manipulation--vla-manipulation-eval::vla-manipulation--primary-metric-family::vlabench","column":1,"subject_ids":["vlabench"],"state":"reported","text":"interactive task success + graduated Progress Score; non-interactive VLM action-sequence matching","evidence":["benchmarks-r32"],"links":[]}]},{"id":"vla-manipulation--language-conditioned","metric":"Language-conditioned","url":"https://botlandscape.com/benchmarks#vla-manipulation--language-conditioned","cells":[{"id":"topics/benchmarks.html::vla-manipulation--vla-manipulation-eval::vla-manipulation--language-conditioned::vlabench","column":1,"subject_ids":["vlabench"],"state":"reported","text":"yes — natural instructions include implicit intent, common sense, and multi-step reasoning","evidence":["benchmarks-r32"],"links":[]}]},{"id":"vla-manipulation--public-code","metric":"Public code","url":"https://botlandscape.com/benchmarks#vla-manipulation--public-code","cells":[{"id":"topics/benchmarks.html::vla-manipulation--vla-manipulation-eval::vla-manipulation--public-code::vlabench","column":1,"subject_ids":["vlabench"],"state":"reported","text":"yes — MIT-licensed official repository","evidence":["benchmarks-r33"],"links":[]}]},{"id":"vla-manipulation--public-data","metric":"Public data","url":"https://botlandscape.com/benchmarks#vla-manipulation--public-data","cells":[{"id":"topics/benchmarks.html::vla-manipulation--vla-manipulation-eval::vla-manipulation--public-data::vlabench","column":1,"subject_ids":["vlabench"],"state":"reported","text":"yes — evaluation episodes and fine-tuning datasets are linked from the official project","evidence":["benchmarks-r33"],"links":[]}]},{"id":"vla-manipulation--public-leaderboard","metric":"Public leaderboard","url":"https://botlandscape.com/benchmarks#vla-manipulation--public-leaderboard","cells":[{"id":"topics/benchmarks.html::vla-manipulation--vla-manipulation-eval::vla-manipulation--public-leaderboard::vlabench","column":1,"subject_ids":["vlabench"],"state":"reported","text":"incomplete — the official repository still lists standard VLA / VLM leaderboard integration as open","evidence":["benchmarks-r33"],"links":[]}]},{"id":"vla-manipulation--claim-boundary","metric":"Claim boundary","url":"https://botlandscape.com/benchmarks#vla-manipulation--claim-boundary","cells":[{"id":"topics/benchmarks.html::vla-manipulation--vla-manipulation-eval::vla-manipulation--claim-boundary::vlabench","column":1,"subject_ids":["vlabench"],"state":"reported","text":"100 task categories are not RoboTwin’s bimanual tasks; performance tables stay in the paper","evidence":["benchmarks-r32"],"links":[]}]}]}],"sources":[{"id":"benchmarks-r32","marker":"R32","grade":"research","citation":"VLABench — “VLABench: A Large-Scale Benchmark for Language-Conditioned Robotics Manipulation with Long-Horizon Reasoning Tasks”","access_status":"direct","registry_url":"https://botlandscape.com/sources.html#benchmarks-r32","direct_url":"https://arxiv.org/abs/2412.18194"},{"id":"benchmarks-r33","marker":"R33","grade":"research","citation":"VLABench — official project site and linked MIT-licensed repository","access_status":"direct","registry_url":"https://botlandscape.com/sources.html#benchmarks-r33","direct_url":"https://vlabench.github.io/"}]}