{
  "$schema": "https://json-schema.org/draft/2020-12/schema",
  "$id": "https://botlandscape.com/benchmark-history.schema.json",
  "title": "Bot Landscape benchmark history projection",
  "description": "Per-suite dated progress frontier, perturbation studies and published real-robot agreement, generated by scripts/benchmark_history.py from catalogue/benchmark-history-ledger.json. Every point is a cited number. Sets no badges; see benchmark-health.schema.json.",
  "type": "object",
  "additionalProperties": false,
  "required": [
    "$schema",
    "schema_version",
    "method",
    "suites",
    "proxies"
  ],
  "properties": {
    "$schema": {
      "const": "https://botlandscape.com/benchmark-history.schema.json"
    },
    "schema_version": {
      "const": 1
    },
    "method": {
      "type": "object",
      "additionalProperties": false,
      "required": [
        "version",
        "frontier",
        "robustness",
        "fidelity",
        "scope",
        "dates"
      ],
      "properties": {
        "version": {
          "type": "string",
          "pattern": "^benchmark-history/[0-9]+$"
        },
        "frontier": {
          "type": "string",
          "minLength": 1
        },
        "robustness": {
          "type": "string",
          "minLength": 1
        },
        "fidelity": {
          "type": "string",
          "minLength": 1
        },
        "scope": {
          "type": "string",
          "minLength": 1
        },
        "dates": {
          "type": "string",
          "minLength": 1
        }
      }
    },
    "suites": {
      "type": "array",
      "minItems": 1,
      "items": {
        "type": "object",
        "additionalProperties": false,
        "required": [
          "id",
          "name",
          "metric",
          "unit",
          "scale_max",
          "medium",
          "catalogue_href",
          "note",
          "best_percent",
          "headroom_percent",
          "first_date",
          "latest_date",
          "median_retained",
          "worst_retained",
          "frontier",
          "progress",
          "robustness_rows",
          "fidelity_rows",
          "ceiling",
          "released"
        ],
        "properties": {
          "id": {
            "type": "string",
            "pattern": "^[a-z0-9]+(?:-{1,2}[a-z0-9]+)*$"
          },
          "name": {
            "type": "string",
            "minLength": 1
          },
          "metric": {
            "type": "string",
            "minLength": 1
          },
          "unit": {
            "type": "string",
            "minLength": 1
          },
          "scale_max": {
            "type": "number",
            "minimum": 0
          },
          "medium": {
            "enum": [
              "sim",
              "real",
              "paired"
            ]
          },
          "catalogue_href": {
            "type": "string",
            "pattern": "^/topics/benchmarks\\.html#"
          },
          "note": {
            "type": "string"
          },
          "best_percent": {
            "type": [
              "number",
              "null"
            ],
            "minimum": 0
          },
          "headroom_percent": {
            "type": [
              "number",
              "null"
            ]
          },
          "first_date": {
            "type": [
              "string",
              "null"
            ]
          },
          "latest_date": {
            "type": [
              "string",
              "null"
            ]
          },
          "median_retained": {
            "type": [
              "number",
              "null"
            ],
            "minimum": 0
          },
          "worst_retained": {
            "type": [
              "number",
              "null"
            ],
            "minimum": 0
          },
          "frontier": {
            "type": "array",
            "items": {
              "type": "object",
              "additionalProperties": false,
              "required": [
                "date",
                "percent",
                "policy",
                "id"
              ],
              "properties": {
                "id": {
                  "type": "string",
                  "pattern": "^[a-z0-9]+(?:-[a-z0-9]+)*$"
                },
                "date": {
                  "type": "string"
                },
                "percent": {
                  "type": "number",
                  "minimum": 0
                },
                "policy": {
                  "type": "string",
                  "minLength": 1
                }
              }
            }
          },
          "progress": {
            "type": "array",
            "items": {
              "type": "object",
              "additionalProperties": false,
              "required": [
                "id",
                "policy",
                "date",
                "value",
                "percent",
                "self_reported",
                "source_url",
                "locator",
                "quote",
                "note",
                "method_date",
                "method_url",
                "plot_date"
              ],
              "properties": {
                "id": {
                  "type": "string",
                  "pattern": "^[a-z0-9]+(?:-[a-z0-9]+)*$"
                },
                "policy": {
                  "type": "string",
                  "minLength": 1
                },
                "date": {
                  "type": "string",
                  "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$"
                },
                "value": {
                  "type": "number",
                  "minimum": 0
                },
                "percent": {
                  "type": "number",
                  "minimum": 0
                },
                "self_reported": {
                  "type": "boolean"
                },
                "source_url": {
                  "type": "string",
                  "pattern": "^https://"
                },
                "locator": {
                  "type": "string",
                  "minLength": 1
                },
                "quote": {
                  "type": "string",
                  "minLength": 1
                },
                "note": {
                  "type": "string"
                },
                "method_date": {
                  "type": [
                    "string",
                    "null"
                  ],
                  "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$"
                },
                "method_url": {
                  "type": [
                    "string",
                    "null"
                  ],
                  "pattern": "^https://"
                },
                "plot_date": {
                  "type": "string",
                  "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$"
                }
              }
            }
          },
          "robustness_rows": {
            "type": "array",
            "items": {
              "type": "object",
              "additionalProperties": false,
              "required": [
                "id",
                "perturbation_suite",
                "perturbation",
                "policy",
                "clean_value",
                "perturbed_value",
                "clean_percent",
                "perturbed_percent",
                "drop_points",
                "retained",
                "source_url",
                "locator",
                "quote",
                "note"
              ],
              "properties": {
                "id": {
                  "type": "string",
                  "pattern": "^[a-z0-9]+(?:-[a-z0-9]+)*$"
                },
                "perturbation_suite": {
                  "type": "string",
                  "minLength": 1
                },
                "perturbation": {
                  "type": "string",
                  "minLength": 1
                },
                "policy": {
                  "type": "string",
                  "minLength": 1
                },
                "clean_value": {
                  "type": "number",
                  "minimum": 0
                },
                "perturbed_value": {
                  "type": "number",
                  "minimum": 0
                },
                "clean_percent": {
                  "type": "number",
                  "minimum": 0
                },
                "perturbed_percent": {
                  "type": "number",
                  "minimum": 0
                },
                "drop_points": {
                  "type": "number"
                },
                "retained": {
                  "type": "number",
                  "minimum": 0
                },
                "source_url": {
                  "type": "string",
                  "pattern": "^https://"
                },
                "locator": {
                  "type": "string",
                  "minLength": 1
                },
                "quote": {
                  "type": "string",
                  "minLength": 1
                },
                "note": {
                  "type": "string"
                }
              }
            }
          },
          "fidelity_rows": {
            "type": "array",
            "items": {
              "type": "object",
              "additionalProperties": false,
              "required": [
                "statistic",
                "value",
                "n_policies",
                "real_reference",
                "source_url",
                "locator",
                "quote",
                "note"
              ],
              "properties": {
                "statistic": {
                  "enum": [
                    "pearson_r",
                    "spearman",
                    "kendall_tau",
                    "r_squared",
                    "mmrv",
                    "statement"
                  ]
                },
                "value": {
                  "type": [
                    "number",
                    "null"
                  ]
                },
                "n_policies": {
                  "type": [
                    "integer",
                    "null"
                  ],
                  "minimum": 2
                },
                "real_reference": {
                  "type": "string",
                  "minLength": 1
                },
                "source_url": {
                  "type": "string",
                  "pattern": "^https://"
                },
                "locator": {
                  "type": "string",
                  "minLength": 1
                },
                "quote": {
                  "type": "string",
                  "minLength": 1
                },
                "note": {
                  "type": "string"
                }
              }
            }
          },
          "ceiling": {
            "type": [
              "object",
              "null"
            ],
            "additionalProperties": false,
            "required": [
              "label",
              "percent",
              "source_url",
              "locator",
              "quote"
            ],
            "properties": {
              "label": {
                "type": "string",
                "minLength": 1
              },
              "percent": {
                "type": "number",
                "minimum": 0
              },
              "source_url": {
                "type": "string",
                "pattern": "^https://"
              },
              "locator": {
                "type": "string",
                "minLength": 1
              },
              "quote": {
                "type": "string",
                "minLength": 1
              }
            }
          },
          "released": {
            "type": [
              "string",
              "null"
            ],
            "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$"
          }
        }
      }
    },
    "proxies": {
      "type": "array",
      "items": {
        "type": "object",
        "additionalProperties": false,
        "required": [
          "suite_name",
          "statistic",
          "value",
          "n_policies",
          "real_reference",
          "source_url",
          "locator",
          "quote",
          "note"
        ],
        "properties": {
          "suite_name": {
            "type": "string",
            "minLength": 1
          },
          "statistic": {
            "enum": [
              "pearson_r",
              "spearman",
              "kendall_tau",
              "r_squared",
              "mmrv",
              "statement"
            ]
          },
          "value": {
            "type": [
              "number",
              "null"
            ]
          },
          "n_policies": {
            "type": [
              "integer",
              "null"
            ],
            "minimum": 2
          },
          "real_reference": {
            "type": "string",
            "minLength": 1
          },
          "source_url": {
            "type": "string",
            "pattern": "^https://"
          },
          "locator": {
            "type": "string",
            "minLength": 1
          },
          "quote": {
            "type": "string",
            "minLength": 1
          },
          "note": {
            "type": "string"
          }
        }
      }
    }
  }
}