{
  "schema_version": 1,
  "model": "Reflex-1",
  "repository": "gai-labs/reflex-1",
  "weights_sha256": "a06bd46ca119b88404f23b6588aa3dcd73aa90e0c51e4f931cd9c56a137acdda",
  "training": {
    "method": "full",
    "round": 8,
    "step": 2000,
    "trainable_parameters": 420778370
  },
  "classification": [
    {
      "task": "ag_news",
      "name": "AG News",
      "dataset": "fancyzhx/ag_news",
      "source_split": "test",
      "attempted": 500,
      "correct": 465,
      "accuracy": 0.93,
      "truncated": 0
    },
    {
      "task": "sst5",
      "name": "SST-5",
      "dataset": "SetFit/sst5",
      "source_split": "test",
      "attempted": 500,
      "correct": 299,
      "accuracy": 0.598,
      "truncated": 0
    },
    {
      "task": "emotion",
      "name": "Emotion",
      "dataset": "dair-ai/emotion",
      "source_split": "validation",
      "attempted": 500,
      "correct": 458,
      "accuracy": 0.916,
      "truncated": 0
    },
    {
      "task": "banking77",
      "name": "Banking77",
      "dataset": "mteb/banking77",
      "source_split": "test",
      "attempted": 495,
      "correct": 460,
      "accuracy": 0.9292929292929293,
      "truncated": 0
    },
    {
      "task": "boolq",
      "name": "BoolQ",
      "dataset": "google/boolq",
      "source_split": "validation",
      "attempted": 202,
      "correct": 171,
      "accuracy": 0.8465346534653465,
      "truncated": 1
    },
    {
      "task": "mnli",
      "name": "MNLI",
      "dataset": "nyu-mll/multi_nli",
      "source_split": "validation_matched",
      "attempted": 500,
      "correct": 436,
      "accuracy": 0.872,
      "truncated": 0
    },
    {
      "task": "race",
      "name": "RACE",
      "dataset": "ehovy/race",
      "source_split": "validation",
      "attempted": 500,
      "correct": 287,
      "accuracy": 0.574,
      "truncated": 26
    },
    {
      "task": "sciq",
      "name": "SciQ",
      "dataset": "allenai/sciq",
      "source_split": "validation",
      "attempted": 128,
      "correct": 123,
      "accuracy": 0.9609375,
      "truncated": 0
    }
  ],
  "classification_protocol": {
    "scope": "Reserved evaluation subsets from task families represented in training; not full official benchmark splits.",
    "precision": "float32",
    "device": "NVIDIA H200",
    "batch_size": 8,
    "max_state_tokens": 512,
    "max_request_tokens": 2048,
    "selected_before_final_predictions": true
  },
  "additional_decision_results": [
    {
      "cohort": "fresh_sciq_permutation",
      "task": "sciq",
      "metric": "accuracy_all_attempts",
      "value": 0.9609375,
      "attempted": 128,
      "truncated": 0,
      "errors": 0
    },
    {
      "cohort": "fresh_glaive",
      "task": "glaive_reserved_tool_routing",
      "metric": "accuracy_all_attempts",
      "value": 0.9875,
      "attempted": 80,
      "truncated": 0,
      "errors": 0
    },
    {
      "cohort": "fresh_decisions",
      "task": "decision-contractnli",
      "metric": "accuracy_all_attempts",
      "value": 0.7613307618129219,
      "attempted": 2074,
      "truncated": 221,
      "errors": 0
    },
    {
      "cohort": "fresh_decisions",
      "task": "decision-helpsteer2-correctness",
      "metric": "expected_score_mae",
      "value": 0.7760053580196521,
      "attempted": 1034,
      "truncated": 0,
      "errors": 0
    },
    {
      "cohort": "fresh_decisions",
      "task": "decision-helpsteer2-preference",
      "metric": "expected_score_mae",
      "value": 1.2689631955637537,
      "attempted": 446,
      "truncated": 0,
      "errors": 0
    },
    {
      "cohort": "fresh_decisions",
      "task": "decision-sharc-action",
      "metric": "accuracy_all_attempts",
      "value": 0.6323102948284195,
      "attempted": 8276,
      "truncated": 0,
      "errors": 0
    },
    {
      "cohort": "historical_typed",
      "task": "agent_trace_observability",
      "metric": "accuracy_all_attempts",
      "value": 0.738,
      "attempted": 500,
      "truncated": 0,
      "errors": 0
    },
    {
      "cohort": "historical_typed",
      "task": "customer_service",
      "metric": "accuracy_all_attempts",
      "value": 0.738,
      "attempted": 500,
      "truncated": 0,
      "errors": 0
    },
    {
      "cohort": "historical_typed",
      "task": "invoice_processing",
      "metric": "accuracy_all_attempts",
      "value": 0.8,
      "attempted": 500,
      "truncated": 0,
      "errors": 0
    },
    {
      "cohort": "historical_typed",
      "task": "security_incidents",
      "metric": "accuracy_all_attempts",
      "value": 0.712,
      "attempted": 500,
      "truncated": 0,
      "errors": 0
    },
    {
      "cohort": "historical_clinc_clean",
      "task": "clinc150_oos",
      "metric": "accuracy_all_attempts",
      "value": 0.8340757238307349,
      "attempted": 4490,
      "truncated": 0,
      "errors": 0
    },
    {
      "cohort": "historical_clinc_full",
      "task": "clinc150_oos",
      "metric": "accuracy_all_attempts",
      "value": 0.8378181818181818,
      "attempted": 5500,
      "truncated": 0,
      "errors": 0
    },
    {
      "cohort": "historical_hermes",
      "task": "hermes_tool_routing",
      "metric": "accuracy_all_attempts",
      "value": 0.8160919540229885,
      "attempted": 87,
      "truncated": 0,
      "errors": 0
    }
  ],
  "additional_decision_scope": "Fresh and historical cohorts are distinguished by cohort name. Common-choice adaptations; some cohorts use 1024 state / 4096 request tokens in the evaluation runtime, exceeding the default Hub interface budget. Not official end-to-end benchmark scores.",
  "native_browser": {
    "environment": "MiniWoB++ in Chromium",
    "success": 21,
    "attempted": 30,
    "errors": 0,
    "scope": "Frozen controller and 30 reserved episodes; not the full MiniWoB++ suite."
  },
  "native_dino": {
    "environment": "Chromium offline runner",
    "attempted": 16,
    "errors": 0,
    "success": 0,
    "mean_primary": 364.75,
    "mean_survival_seconds": 29.871996006957488,
    "scope": "16 unseeded native attempts with a 60-second survival target."
  },
  "public_jevbench": {
    "attempted": 231,
    "labelled": 231,
    "valid": 231,
    "strict_valid": 231,
    "renormalized": 0,
    "correct": 117,
    "accuracy_labelled": 0.5064935064935064,
    "state_truncated": 37,
    "backend_errors": 0,
    "official_benchmark_score": false,
    "scope": "Previously inspected public 231-item diagnostic, adapted to candidate scoring; excluded from training and checkpoint selection."
  },
  "cpu": {
    "hardware": "Intel Xeon Platinum 8558",
    "precision": "float32",
    "batch_size": 1,
    "scope": "Historical 120-input latency replay; two fresh processes per condition, interleaved with the previous checkpoint in ABBA order; 480 calls per thread condition. Tokenization and predict included; load and warmup excluded; cross-request caches disabled; shared host.",
    "memory_scope": "Peak whole-process RSS including imports, model loading, warmup and inference.",
    "conditions": {
      "1": {
        "intra_op_threads": 1,
        "inter_op_threads": 1,
        "blas_limit": 1,
        "timed_calls": 480,
        "unique_cases": 120,
        "peak_rss_bytes": 2332307456,
        "os_threads_observed_during_scoring": {
          "min": 2,
          "max": 2
        },
        "threads_with_observed_cpu_activity": [
          {
            "min_per_request": 1,
            "peak_per_request": 1,
            "distinct_threads": 1
          },
          {
            "min_per_request": 1,
            "peak_per_request": 1,
            "distinct_threads": 1
          }
        ],
        "tasks": {
          "ag_news": {
            "calls": 96,
            "median_ms": 1209.6750645,
            "p95_ms": 1385.0021889999998
          },
          "sst5": {
            "calls": 96,
            "median_ms": 973.6049575,
            "p95_ms": 1160.302423
          },
          "emotion": {
            "calls": 96,
            "median_ms": 956.856571,
            "p95_ms": 1165.7823429999999
          },
          "banking77": {
            "calls": 96,
            "median_ms": 1101.7479065,
            "p95_ms": 1276.865664
          },
          "boolq": {
            "calls": 96,
            "median_ms": 1600.8008725,
            "p95_ms": 2710.55801625
          }
        }
      },
      "4": {
        "intra_op_threads": 4,
        "inter_op_threads": 1,
        "blas_limit": 4,
        "timed_calls": 480,
        "unique_cases": 120,
        "peak_rss_bytes": 2331201536,
        "os_threads_observed_during_scoring": {
          "min": 11,
          "max": 11
        },
        "threads_with_observed_cpu_activity": [
          {
            "min_per_request": 4,
            "peak_per_request": 4,
            "distinct_threads": 4
          },
          {
            "min_per_request": 4,
            "peak_per_request": 4,
            "distinct_threads": 4
          }
        ],
        "tasks": {
          "ag_news": {
            "calls": 96,
            "median_ms": 389.4811355,
            "p95_ms": 480.36708525
          },
          "sst5": {
            "calls": 96,
            "median_ms": 320.6325725,
            "p95_ms": 404.241721
          },
          "emotion": {
            "calls": 96,
            "median_ms": 318.763468,
            "p95_ms": 391.02955475
          },
          "banking77": {
            "calls": 96,
            "median_ms": 375.223626,
            "p95_ms": 452.067269
          },
          "boolq": {
            "calls": 96,
            "median_ms": 486.721985,
            "p95_ms": 782.209749
          }
        }
      }
    }
  },
  "validation": {
    "development_regression_gate_passed": false,
    "full_suite_complete": false,
    "measured_tradeoffs": [
      {
        "task": "BoolQ",
        "previous": 0.8613861386138614,
        "current": 0.8465346534653465,
        "attempted": 202
      },
      {
        "task": "RACE",
        "previous": 0.578,
        "current": 0.574,
        "attempted": 500
      },
      {
        "task": "Public JevBench diagnostic",
        "previous": 0.5281385281385281,
        "current": 0.5064935064935064,
        "attempted": 231
      }
    ],
    "incomplete_panels": [
      "Python Dino and ViZDoom final runs returned execution errors.",
      "Historical WebGym final evidence is incomplete."
    ],
    "release_basis": "Latest completed full-parameter fine-tune, published with the measured trade-offs and incomplete evaluation coverage disclosed."
  },
  "source_reports_sha256": {
    "decision_and_resource_summary": "9e9d064cd80b82d29bb9d2b7fc24588c755a692ff55f12269dcf40a8bd7e3f05",
    "threads1-arm1-candidate": "308ed38935701f03c4f746d206af41ddefc6a065a78d5c360860cbcb2ef302ad",
    "threads1-arm2-candidate": "b74c5a5bdc665e9fdca13a5c739d8870a7f75da88fc8dab9945dcfeeed20906a",
    "threads4-arm1-candidate": "772a89f2c192751a0006f017272f70728ee42ccbb3c6a5d048c81eca9322464a",
    "threads4-arm2-candidate": "6f16018f8cd1dc6adbbf8b9f9779ad2669a79699c2a7ea51f33e32b05e72a5d1"
  }
}