{"$schema":"https://botlandscape.com/entity.schema.json","schema_version":1,"data_release":"b9842fdc0c6511f51b2ca90210842e439f40db0ca516d7d5c5db6365bd06b7e6","data_audited":"2026-09-04","entity":{"id":"benchmark--vlabench","kind":"benchmark","name":"VLABench","summary":"Language-conditioned simulation benchmark for VLA manipulation.","url":"https://arxiv.org/abs/2412.18194","dossier_url":"https://botlandscape.com/entities/benchmark--vlabench.html","aliases":[],"category":null,"publication_state":"released","deployment_stage":"research","bindings":[{"page_path":"topics/benchmarks.html","table_id":"vla-manipulation--vla-manipulation-eval","subject_id":"vlabench"}],"facets":[],"claim_ids":["topics/benchmarks.html::vla-manipulation--vla-manipulation-eval::vla-manipulation--kind::vlabench","topics/benchmarks.html::vla-manipulation--vla-manipulation-eval::vla-manipulation--first-public-date::vlabench","topics/benchmarks.html::vla-manipulation--vla-manipulation-eval::vla-manipulation--paper::vlabench","topics/benchmarks.html::vla-manipulation--vla-manipulation-eval::vla-manipulation--task-structure::vlabench","topics/benchmarks.html::vla-manipulation--vla-manipulation-eval::vla-manipulation--asset-library::vlabench","topics/benchmarks.html::vla-manipulation--vla-manipulation-eval::vla-manipulation--released-trajectories::vlabench","topics/benchmarks.html::vla-manipulation--vla-manipulation-eval::vla-manipulation--evaluation-scope::vlabench","topics/benchmarks.html::vla-manipulation--vla-manipulation-eval::vla-manipulation--medium::vlabench","topics/benchmarks.html::vla-manipulation--vla-manipulation-eval::vla-manipulation--embodiment-in-the-source::vlabench","topics/benchmarks.html::vla-manipulation--vla-manipulation-eval::vla-manipulation--engine-renderer::vlabench","topics/benchmarks.html::vla-manipulation--vla-manipulation-eval::vla-manipulation--primary-metric-family::vlabench","topics/benchmarks.html::vla-manipulation--vla-manipulation-eval::vla-manipulation--language-conditioned::vlabench","topics/benchmarks.html::vla-manipulation--vla-manipulation-eval::vla-manipulation--public-code::vlabench","topics/benchmarks.html::vla-manipulation--vla-manipulation-eval::vla-manipulation--public-data::vlabench","topics/benchmarks.html::vla-manipulation--vla-manipulation-eval::vla-manipulation--public-leaderboard::vlabench","topics/benchmarks.html::vla-manipulation--vla-manipulation-eval::vla-manipulation--claim-boundary::vlabench"],"claim_counts":{"total":16,"reported":16,"missing":0},"evidence_source_ids":["benchmarks-r32","benchmarks-r33"],"evidence_grades":{"research":2},"last_reviewed":"2026-09-04"},"relations":[{"id":"relation--vlabench-uses-mujoco","from":"benchmark--vlabench","predicate":"uses-simulator","label":"uses simulator","to":"simulator--mujoco","status":"verified","scope":"benchmark execution","note":"The benchmark states MuJoCo plus dm_control.","claim_ids":["topics/benchmarks.html::vla-manipulation--vla-manipulation-eval::vla-manipulation--engine-renderer::vlabench"],"source_ids":["benchmarks-r32"],"valid_from":null,"valid_to":null,"evidence_grades":{"research":1},"first_observed":"2026-09-04","last_reviewed":"2026-09-04","url":"https://botlandscape.com/ecosystem.html?entity=benchmark--vlabench&edge=relation--vlabench-uses-mujoco#explorer","direction":"outbound","related_entity":{"id":"simulator--mujoco","name":"MuJoCo","kind":"simulator","url":"https://github.com/google-deepmind/mujoco","dossier_url":"https://botlandscape.com/entities/simulator--mujoco.html"}}],"claims":[{"id":"topics/benchmarks.html::vla-manipulation--vla-manipulation-eval::vla-manipulation--asset-library::vlabench","url":"https://botlandscape.com/topics/benchmarks.html#vla-manipulation--asset-library","page_path":"topics/benchmarks.html","section_id":"manipulation","table_id":"vla-manipulation--vla-manipulation-eval","class":"vla-manipulation-eval","row_id":"vla-manipulation--asset-library","metric":"Asset library","subject_ids":["vlabench"],"subjects":[{"id":"vlabench","label":"VLABench"}],"value":{"state":"reported","text":"2,164 objects across 163 categories"},"evidence":["benchmarks-r32"],"links":[]},{"id":"topics/benchmarks.html::vla-manipulation--vla-manipulation-eval::vla-manipulation--claim-boundary::vlabench","url":"https://botlandscape.com/topics/benchmarks.html#vla-manipulation--claim-boundary","page_path":"topics/benchmarks.html","section_id":"manipulation","table_id":"vla-manipulation--vla-manipulation-eval","class":"vla-manipulation-eval","row_id":"vla-manipulation--claim-boundary","metric":"Claim boundary","subject_ids":["vlabench"],"subjects":[{"id":"vlabench","label":"VLABench"}],"value":{"state":"reported","text":"100 task categories are not RoboTwin’s bimanual tasks; performance tables stay in the paper"},"evidence":["benchmarks-r32"],"links":[]},{"id":"topics/benchmarks.html::vla-manipulation--vla-manipulation-eval::vla-manipulation--embodiment-in-the-source::vlabench","url":"https://botlandscape.com/topics/benchmarks.html#vla-manipulation--embodiment-in-the-source","page_path":"topics/benchmarks.html","section_id":"manipulation","table_id":"vla-manipulation--vla-manipulation-eval","class":"vla-manipulation-eval","row_id":"vla-manipulation--embodiment-in-the-source","metric":"Embodiment in the source","subject_ids":["vlabench"],"subjects":[{"id":"vlabench","label":"VLABench"}],"value":{"state":"reported","text":"standard evaluation: 7-DoF Franka Emika Panda with parallel gripper; framework also supports other forms"},"evidence":["benchmarks-r32"],"links":[]},{"id":"topics/benchmarks.html::vla-manipulation--vla-manipulation-eval::vla-manipulation--engine-renderer::vlabench","url":"https://botlandscape.com/topics/benchmarks.html#vla-manipulation--engine-renderer","page_path":"topics/benchmarks.html","section_id":"manipulation","table_id":"vla-manipulation--vla-manipulation-eval","class":"vla-manipulation-eval","row_id":"vla-manipulation--engine-renderer","metric":"Engine / renderer","subject_ids":["vlabench"],"subjects":[{"id":"vlabench","label":"VLABench"}],"value":{"state":"reported","text":"MuJoCo + dm_control"},"evidence":["benchmarks-r32"],"links":[]},{"id":"topics/benchmarks.html::vla-manipulation--vla-manipulation-eval::vla-manipulation--evaluation-scope::vlabench","url":"https://botlandscape.com/topics/benchmarks.html#vla-manipulation--evaluation-scope","page_path":"topics/benchmarks.html","section_id":"manipulation","table_id":"vla-manipulation--vla-manipulation-eval","class":"vla-manipulation-eval","row_id":"vla-manipulation--evaluation-scope","metric":"Evaluation scope","subject_ids":["vlabench"],"subjects":[{"id":"vlabench","label":"VLABench"}],"value":{"state":"reported","text":"6 capability dimensions: mesh / texture, spatial, common sense / knowledge, semantic instruction, physical laws, and long-horizon reasoning"},"evidence":["benchmarks-r32"],"links":[]},{"id":"topics/benchmarks.html::vla-manipulation--vla-manipulation-eval::vla-manipulation--first-public-date::vlabench","url":"https://botlandscape.com/topics/benchmarks.html#vla-manipulation--first-public-date","page_path":"topics/benchmarks.html","section_id":"manipulation","table_id":"vla-manipulation--vla-manipulation-eval","class":"vla-manipulation-eval","row_id":"vla-manipulation--first-public-date","metric":"First public date","subject_ids":["vlabench"],"subjects":[{"id":"vlabench","label":"VLABench"}],"value":{"state":"reported","text":"arXiv 2024-12-24"},"evidence":["benchmarks-r32"],"links":[]},{"id":"topics/benchmarks.html::vla-manipulation--vla-manipulation-eval::vla-manipulation--kind::vlabench","url":"https://botlandscape.com/topics/benchmarks.html#vla-manipulation--kind","page_path":"topics/benchmarks.html","section_id":"manipulation","table_id":"vla-manipulation--vla-manipulation-eval","class":"vla-manipulation-eval","row_id":"vla-manipulation--kind","metric":"Kind","subject_ids":["vlabench"],"subjects":[{"id":"vlabench","label":"VLABench"}],"value":{"state":"reported","text":"language-conditioned manipulation benchmark for VLAs, foundation-model workflows, and VLMs"},"evidence":["benchmarks-r32"],"links":[]},{"id":"topics/benchmarks.html::vla-manipulation--vla-manipulation-eval::vla-manipulation--language-conditioned::vlabench","url":"https://botlandscape.com/topics/benchmarks.html#vla-manipulation--language-conditioned","page_path":"topics/benchmarks.html","section_id":"manipulation","table_id":"vla-manipulation--vla-manipulation-eval","class":"vla-manipulation-eval","row_id":"vla-manipulation--language-conditioned","metric":"Language-conditioned","subject_ids":["vlabench"],"subjects":[{"id":"vlabench","label":"VLABench"}],"value":{"state":"reported","text":"yes — natural instructions include implicit intent, common sense, and multi-step reasoning"},"evidence":["benchmarks-r32"],"links":[]},{"id":"topics/benchmarks.html::vla-manipulation--vla-manipulation-eval::vla-manipulation--medium::vlabench","url":"https://botlandscape.com/topics/benchmarks.html#vla-manipulation--medium","page_path":"topics/benchmarks.html","section_id":"manipulation","table_id":"vla-manipulation--vla-manipulation-eval","class":"vla-manipulation-eval","row_id":"vla-manipulation--medium","metric":"Medium","subject_ids":["vlabench"],"subjects":[{"id":"vlabench","label":"VLABench"}],"value":{"state":"reported","text":"simulation"},"evidence":["benchmarks-r32"],"links":[]},{"id":"topics/benchmarks.html::vla-manipulation--vla-manipulation-eval::vla-manipulation--paper::vlabench","url":"https://botlandscape.com/topics/benchmarks.html#vla-manipulation--paper","page_path":"topics/benchmarks.html","section_id":"manipulation","table_id":"vla-manipulation--vla-manipulation-eval","class":"vla-manipulation-eval","row_id":"vla-manipulation--paper","metric":"Paper","subject_ids":["vlabench"],"subjects":[{"id":"vlabench","label":"VLABench"}],"value":{"state":"reported","text":"arXiv:2412.18194; ICCV 2025"},"evidence":["benchmarks-r32","benchmarks-r33"],"links":[]},{"id":"topics/benchmarks.html::vla-manipulation--vla-manipulation-eval::vla-manipulation--primary-metric-family::vlabench","url":"https://botlandscape.com/topics/benchmarks.html#vla-manipulation--primary-metric-family","page_path":"topics/benchmarks.html","section_id":"manipulation","table_id":"vla-manipulation--vla-manipulation-eval","class":"vla-manipulation-eval","row_id":"vla-manipulation--primary-metric-family","metric":"Primary metric family","subject_ids":["vlabench"],"subjects":[{"id":"vlabench","label":"VLABench"}],"value":{"state":"reported","text":"interactive task success + graduated Progress Score; non-interactive VLM action-sequence matching"},"evidence":["benchmarks-r32"],"links":[]},{"id":"topics/benchmarks.html::vla-manipulation--vla-manipulation-eval::vla-manipulation--public-code::vlabench","url":"https://botlandscape.com/topics/benchmarks.html#vla-manipulation--public-code","page_path":"topics/benchmarks.html","section_id":"manipulation","table_id":"vla-manipulation--vla-manipulation-eval","class":"vla-manipulation-eval","row_id":"vla-manipulation--public-code","metric":"Public code","subject_ids":["vlabench"],"subjects":[{"id":"vlabench","label":"VLABench"}],"value":{"state":"reported","text":"yes — MIT-licensed official repository"},"evidence":["benchmarks-r33"],"links":[]},{"id":"topics/benchmarks.html::vla-manipulation--vla-manipulation-eval::vla-manipulation--public-data::vlabench","url":"https://botlandscape.com/topics/benchmarks.html#vla-manipulation--public-data","page_path":"topics/benchmarks.html","section_id":"manipulation","table_id":"vla-manipulation--vla-manipulation-eval","class":"vla-manipulation-eval","row_id":"vla-manipulation--public-data","metric":"Public data","subject_ids":["vlabench"],"subjects":[{"id":"vlabench","label":"VLABench"}],"value":{"state":"reported","text":"yes — evaluation episodes and fine-tuning datasets are linked from the official project"},"evidence":["benchmarks-r33"],"links":[]},{"id":"topics/benchmarks.html::vla-manipulation--vla-manipulation-eval::vla-manipulation--public-leaderboard::vlabench","url":"https://botlandscape.com/topics/benchmarks.html#vla-manipulation--public-leaderboard","page_path":"topics/benchmarks.html","section_id":"manipulation","table_id":"vla-manipulation--vla-manipulation-eval","class":"vla-manipulation-eval","row_id":"vla-manipulation--public-leaderboard","metric":"Public leaderboard","subject_ids":["vlabench"],"subjects":[{"id":"vlabench","label":"VLABench"}],"value":{"state":"reported","text":"incomplete — the official repository still lists standard VLA / VLM leaderboard integration as open"},"evidence":["benchmarks-r33"],"links":[]},{"id":"topics/benchmarks.html::vla-manipulation--vla-manipulation-eval::vla-manipulation--released-trajectories::vlabench","url":"https://botlandscape.com/topics/benchmarks.html#vla-manipulation--released-trajectories","page_path":"topics/benchmarks.html","section_id":"manipulation","table_id":"vla-manipulation--vla-manipulation-eval","class":"vla-manipulation-eval","row_id":"vla-manipulation--released-trajectories","metric":"Released trajectories","subject_ids":["vlabench"],"subjects":[{"id":"vlabench","label":"VLABench"}],"value":{"state":"reported","text":"standard evaluation episodes and primitive-task fine-tuning data are public; no one suite-wide trajectory count is stated"},"evidence":["benchmarks-r33"],"links":[]},{"id":"topics/benchmarks.html::vla-manipulation--vla-manipulation-eval::vla-manipulation--task-structure::vlabench","url":"https://botlandscape.com/topics/benchmarks.html#vla-manipulation--task-structure","page_path":"topics/benchmarks.html","section_id":"manipulation","table_id":"vla-manipulation--vla-manipulation-eval","class":"vla-manipulation-eval","row_id":"vla-manipulation--task-structure","metric":"Task structure","subject_ids":["vlabench"],"subjects":[{"id":"vlabench","label":"VLABench"}],"value":{"state":"reported","text":"100 task categories: 60 primitive + 40 composite"},"evidence":["benchmarks-r32"],"links":[]}],"sources":[{"id":"benchmarks-r32","marker":"R32","grade":"research","citation":"VLABench — “VLABench: A Large-Scale Benchmark for Language-Conditioned Robotics Manipulation with Long-Horizon Reasoning Tasks”. Type: research paper. ID: arXiv:2412.18194; ICCV 2025. Retrieved: 2026-09-04.","retrieved":"2026-09-04","registry_url":"https://botlandscape.com/sources.html#benchmarks-r32","direct_url":"https://arxiv.org/abs/2412.18194"},{"id":"benchmarks-r33","marker":"R33","grade":"research","citation":"VLABench — official project site and linked MIT-licensed repository. Type: research project page. Retrieved: 2026-09-04.","retrieved":"2026-09-04","registry_url":"https://botlandscape.com/sources.html#benchmarks-r33","direct_url":"https://vlabench.github.io/"}]}