{
  "$schema": "https://json-schema.org/draft/2020-12/schema",
  "title": "Bot Landscape benchmark health cards",
  "type": "object",
  "additionalProperties": false,
  "required": ["$schema", "schema_version", "source", "min_verdict_policies", "method", "badges", "cards"],
  "properties": {
    "$schema": {"const": "https://botlandscape.com/benchmark-health.schema.json"},
    "schema_version": {"const": 1},
    "source": {"type": "string", "minLength": 1},
    "min_verdict_policies": {"type": "integer", "minimum": 5},
    "method": {"type": "object", "additionalProperties": {"type": "string", "minLength": 1}},
    "badges": {
      "type": "array",
      "items": {
        "type": "object", "additionalProperties": false, "required": ["id", "label"],
        "properties": {"id": {"$ref": "#/$defs/badge"}, "label": {"type": "string", "minLength": 1}}
      }
    },
    "cards": {
      "type": "array",
      "items": {
        "type": "object",
        "additionalProperties": false,
        "required": ["benchmark", "status", "badges", "catalogue", "boards", "robustness", "sim_real", "evidence"],
        "properties": {
          "benchmark": {"type": "string", "minLength": 1},
          "status": {"$ref": "#/$defs/badge"},
          "badges": {"type": "array", "items": {"$ref": "#/$defs/badge"}},
          "catalogue": {
            "type": ["object", "null"], "additionalProperties": false,
            "required": ["id", "name", "first_public", "href"],
            "properties": {"id": {"type": "string"}, "name": {"type": "string"}, "first_public": {"type": "string"}, "href": {"type": "string"}}
          },
          "boards": {
            "type": "array", "minItems": 1,
            "items": {
              "type": "object", "additionalProperties": false,
              "required": ["id", "split", "condition", "metric", "medium", "rows", "rows_with_n", "best", "headroom", "leaders"],
              "properties": {
                "id": {"type": "string"}, "split": {"type": "string"}, "condition": {"type": "string"}, "metric": {"type": "string"},
                "medium": {"type": "array", "items": {"enum": ["sim", "real"]}},
                "rows": {"type": "integer", "minimum": 1}, "rows_with_n": {"type": "integer", "minimum": 0},
                "best": {
                  "type": "object", "additionalProperties": false,
                  "required": ["policy", "value_percent", "interval_low", "interval_high", "n_trials", "row_id"],
                  "properties": {
                    "policy": {"type": "string"}, "value_percent": {"type": "number", "minimum": 0, "maximum": 100},
                    "interval_low": {"type": ["number", "null"]}, "interval_high": {"type": ["number", "null"]},
                    "n_trials": {"type": ["integer", "null"]}, "row_id": {"type": "string"}
                  }
                },
                "headroom": {"type": "number", "minimum": 0, "maximum": 100},
                "leaders": {"type": ["array", "null"], "items": {"type": "string"}}
              }
            }
          },
          "robustness": {
            "type": "array",
            "items": {
              "type": "object", "additionalProperties": false,
              "required": ["condition", "board", "mean_drop", "verdict", "worst", "policies"],
              "properties": {
                "condition": {"type": "string"}, "board": {"type": "string"}, "mean_drop": {"type": "number"},
                "verdict": {"enum": ["collapses", "holds", "too-few-policies"]},
                "worst": {"$ref": "#/$defs/drop"}, "policies": {"type": "array", "items": {"$ref": "#/$defs/drop"}}
              }
            }
          },
          "sim_real": {
            "type": "array",
            "items": {
              "type": "object", "additionalProperties": false,
              "required": ["split", "sim_board", "real_board", "policies", "spearman", "mmrv", "mean_abs_gap", "real_separates", "verdict"],
              "properties": {
                "split": {"type": "string"}, "sim_board": {"type": "string"}, "real_board": {"type": "string"},
                "policies": {
                  "type": "array", "minItems": 2,
                  "items": {
                    "type": "object", "additionalProperties": false, "required": ["policy", "sim", "real"],
                    "properties": {"policy": {"type": "string"}, "sim": {"type": "number"}, "real": {"type": "number"}}
                  }
                },
                "spearman": {"type": ["number", "null"], "minimum": -1, "maximum": 1},
                "mmrv": {"type": ["number", "null"], "minimum": 0},
                "mean_abs_gap": {"type": "number", "minimum": 0},
                "real_separates": {"type": "boolean"},
                "verdict": {"enum": ["tracks-real", "disagrees-real", "too-few-policies", "undecided"]}
              }
            }
          },
          "evidence": {
            "type": "object", "additionalProperties": false,
            "required": ["rows", "rows_with_n", "evaluators", "media"],
            "properties": {
              "rows": {"type": "integer", "minimum": 1}, "rows_with_n": {"type": "integer", "minimum": 0},
              "evaluators": {"type": "object", "additionalProperties": {"type": "integer", "minimum": 1}},
              "media": {"type": "array", "items": {"enum": ["sim", "real"]}}
            }
          }
        }
      }
    }
  },
  "$defs": {
    "badge": {"enum": ["saturated", "open", "collapses", "too-few-perturbed", "tracks-real", "disagrees-real", "too-few-policies", "real-robot", "unvalidated", "leaders-tied", "n-missing"]},
    "drop": {
      "type": "object", "additionalProperties": false, "required": ["policy", "base", "perturbed", "drop"],
      "properties": {"policy": {"type": "string"}, "base": {"type": "number"}, "perturbed": {"type": "number"}, "drop": {"type": "number"}}
    }
  }
}