{
  "version": "2026-11-18-scenario-v1",
  "created_on": "2026-11-18",
  "result_type": "analytical_scenario_not_measured",
  "calibrated_to_hardware": false,
  "confidence_interval": false,
  "hardware": {
    "gpu": "NVIDIA H200 NVL PCIe",
    "gpu_counts": [
      4,
      8
    ],
    "cpu": "2 × AMD EPYC 9354",
    "host_ram_gb": [
      512,
      1024,
      2048
    ],
    "hbm_bytes_per_gpu": 141000000000,
    "hbm_bandwidth_bytes_s_per_gpu": 4800000000000,
    "bf16_dense_flops_s_per_gpu": 835500000000000
  },
  "assumptions": {
    "weight_bytes_per_parameter": 2,
    "kv_bytes_per_element": 2,
    "reserved_hbm_fraction": 0.2,
    "compute_efficiency_range": [
      0.15,
      0.35
    ],
    "eight_gpu_additional_scaling_range": [
      0.85,
      0.95
    ],
    "eight_gpu_topology": "two_independent_four_gpu_groups_no_tp8_prediction",
    "prefill_included": false,
    "cpu_offload": false,
    "speculative_decode": false,
    "prefix_cache_reuse": false,
    "calibration": "none; efficiency bands are author assumptions",
    "weight_traffic_proxy": "Full BF16 parameter bytes are used as a conservative proxy; the entire input embedding table is not read per decode step",
    "runtime": "proposed modern vLLM / TensorRT-LLM; exact versions not benchmarked",
    "power_condition": "600W-capable GPU cabling, sustained clocks and validated cooling/power; H200-8 conditional",
    "units": "decimal GB and TB/s; counts are binary-exact integers"
  },
  "formulas": {
    "memory_tokens_s_per_replica": "B * TP * BW * eta_memory / (2 * P + B * kv_bytes_per_token * C)",
    "compute_tokens_s_per_replica": "TP * dense_BF16_peak * eta_compute / (2 * P + 4 * layers * hidden * C)",
    "replica_tokens_s": "min(memory_tokens_s, compute_tokens_s)",
    "node_tokens_s": "replica_tokens_s * replicas * additional_scaling",
    "capacity": "2 * P + B * kv_bytes_per_token * C <= TP * HBM * 0.8",
    "context": "C = input_tokens + 256; point at end of decode"
  },
  "models": {
    "llama31-8b": {
      "name": "Llama 3.1 8B Instruct",
      "hidden": 4096,
      "layers": 32,
      "heads": 32,
      "kv_heads": 8,
      "ffn": 14336,
      "vocab": 128256,
      "tp": 1,
      "memory_efficiency": [
        0.25,
        0.5
      ],
      "parameters": 8030261248,
      "kv_bytes_per_token": 131072
    },
    "llama31-70b": {
      "name": "Llama 3.1 70B Instruct",
      "hidden": 8192,
      "layers": 80,
      "heads": 64,
      "kv_heads": 8,
      "ffn": 28672,
      "vocab": 128256,
      "tp": 4,
      "memory_efficiency": [
        0.3,
        0.55
      ],
      "parameters": 70553706496,
      "kv_bytes_per_token": 327680
    }
  },
  "sources": [
    {
      "label": "NVIDIA H200: NVL, HBM, пропускная способность и вычислительные пики",
      "url": "https://www.nvidia.com/en-us/data-center/h200/"
    },
    {
      "label": "Meta: архитектура и параметры семейства Llama",
      "url": "https://github.com/meta-llama/llama-models/blob/0e0b8c519242d5833d8c11bffc1232b77ad7f301/models/sku_list.py"
    },
    {
      "label": "NVIDIA NeMo: конфигурации Llama 3.1 8B и 70B",
      "url": "https://docs.nvidia.com/nemo/megatron-bridge/0.3.0/apidocs/bridge/bridge.models.llama.llama_provider.html"
    },
    {
      "label": "MLCommons: Dell XE7745, 8 × H200 NVL, Llama 2 70B FP8, Interactive Server",
      "url": "https://github.com/mlcommons/inference_results_v5.0/blob/main/closed/Dell/results/XE7745_H200_NVL_141GBx8_TRT/llama2-70b-interactive-99.9/Server/performance/run_1/mlperf_log_summary.txt"
    }
  ],
  "rows": [
    {
      "model_id": "llama31-8b",
      "model": "Llama 3.1 8B Instruct",
      "precision": "BF16",
      "gpu_count": 4,
      "tp": 1,
      "replicas": 4,
      "input_tokens": 2048,
      "output_tokens": 256,
      "context_at_end": 2304,
      "active_sequences_per_replica": 1,
      "active_sequences_per_node": 4,
      "weight_bytes_per_replica": 16060522496,
      "kv_bytes_per_replica": 301989888,
      "hbm_budget_bytes_per_replica": 112800000000,
      "memory_fit": true,
      "replica_output_tokens_s": [
        73.338,
        146.677
      ],
      "node_output_tokens_s": [
        293.353,
        586.707
      ],
      "metric": "steady_state_decode_output_tokens_s_excludes_prefill",
      "status": "scenario_estimate_not_measured"
    },
    {
      "model_id": "llama31-8b",
      "model": "Llama 3.1 8B Instruct",
      "precision": "BF16",
      "gpu_count": 4,
      "tp": 1,
      "replicas": 4,
      "input_tokens": 2048,
      "output_tokens": 256,
      "context_at_end": 2304,
      "active_sequences_per_replica": 8,
      "active_sequences_per_node": 32,
      "weight_bytes_per_replica": 16060522496,
      "kv_bytes_per_replica": 2415919104,
      "hbm_budget_bytes_per_replica": 112800000000,
      "memory_fit": true,
      "replica_output_tokens_s": [
        519.581,
        1039.161
      ],
      "node_output_tokens_s": [
        2078.322,
        4156.645
      ],
      "metric": "steady_state_decode_output_tokens_s_excludes_prefill",
      "status": "scenario_estimate_not_measured"
    },
    {
      "model_id": "llama31-8b",
      "model": "Llama 3.1 8B Instruct",
      "precision": "BF16",
      "gpu_count": 4,
      "tp": 1,
      "replicas": 4,
      "input_tokens": 2048,
      "output_tokens": 256,
      "context_at_end": 2304,
      "active_sequences_per_replica": 32,
      "active_sequences_per_node": 128,
      "weight_bytes_per_replica": 16060522496,
      "kv_bytes_per_replica": 9663676416,
      "hbm_budget_bytes_per_replica": 112800000000,
      "memory_fit": true,
      "replica_output_tokens_s": [
        1492.758,
        2985.516
      ],
      "node_output_tokens_s": [
        5971.031,
        11942.063
      ],
      "metric": "steady_state_decode_output_tokens_s_excludes_prefill",
      "status": "scenario_estimate_not_measured"
    },
    {
      "model_id": "llama31-8b",
      "model": "Llama 3.1 8B Instruct",
      "precision": "BF16",
      "gpu_count": 4,
      "tp": 1,
      "replicas": 4,
      "input_tokens": 2048,
      "output_tokens": 256,
      "context_at_end": 2304,
      "active_sequences_per_replica": 64,
      "active_sequences_per_node": 256,
      "weight_bytes_per_replica": 16060522496,
      "kv_bytes_per_replica": 19327352832,
      "hbm_budget_bytes_per_replica": 112800000000,
      "memory_fit": true,
      "replica_output_tokens_s": [
        2170.235,
        4340.47
      ],
      "node_output_tokens_s": [
        8680.939,
        17361.879
      ],
      "metric": "steady_state_decode_output_tokens_s_excludes_prefill",
      "status": "scenario_estimate_not_measured"
    },
    {
      "model_id": "llama31-8b",
      "model": "Llama 3.1 8B Instruct",
      "precision": "BF16",
      "gpu_count": 4,
      "tp": 1,
      "replicas": 4,
      "input_tokens": 8192,
      "output_tokens": 256,
      "context_at_end": 8448,
      "active_sequences_per_replica": 1,
      "active_sequences_per_node": 4,
      "weight_bytes_per_replica": 16060522496,
      "kv_bytes_per_replica": 1107296256,
      "hbm_budget_bytes_per_replica": 112800000000,
      "memory_fit": true,
      "replica_output_tokens_s": [
        69.898,
        139.796
      ],
      "node_output_tokens_s": [
        279.593,
        559.186
      ],
      "metric": "steady_state_decode_output_tokens_s_excludes_prefill",
      "status": "scenario_estimate_not_measured"
    },
    {
      "model_id": "llama31-8b",
      "model": "Llama 3.1 8B Instruct",
      "precision": "BF16",
      "gpu_count": 4,
      "tp": 1,
      "replicas": 4,
      "input_tokens": 8192,
      "output_tokens": 256,
      "context_at_end": 8448,
      "active_sequences_per_replica": 8,
      "active_sequences_per_node": 32,
      "weight_bytes_per_replica": 16060522496,
      "kv_bytes_per_replica": 8858370048,
      "hbm_budget_bytes_per_replica": 112800000000,
      "memory_fit": true,
      "replica_output_tokens_s": [
        385.25,
        770.5
      ],
      "node_output_tokens_s": [
        1540.999,
        3081.999
      ],
      "metric": "steady_state_decode_output_tokens_s_excludes_prefill",
      "status": "scenario_estimate_not_measured"
    },
    {
      "model_id": "llama31-8b",
      "model": "Llama 3.1 8B Instruct",
      "precision": "BF16",
      "gpu_count": 4,
      "tp": 1,
      "replicas": 4,
      "input_tokens": 8192,
      "output_tokens": 256,
      "context_at_end": 8448,
      "active_sequences_per_replica": 32,
      "active_sequences_per_node": 128,
      "weight_bytes_per_replica": 16060522496,
      "kv_bytes_per_replica": 35433480192,
      "hbm_budget_bytes_per_replica": 112800000000,
      "memory_fit": true,
      "replica_output_tokens_s": [
        745.718,
        1491.436
      ],
      "node_output_tokens_s": [
        2982.872,
        5965.743
      ],
      "metric": "steady_state_decode_output_tokens_s_excludes_prefill",
      "status": "scenario_estimate_not_measured"
    },
    {
      "model_id": "llama31-8b",
      "model": "Llama 3.1 8B Instruct",
      "precision": "BF16",
      "gpu_count": 4,
      "tp": 1,
      "replicas": 4,
      "input_tokens": 8192,
      "output_tokens": 256,
      "context_at_end": 8448,
      "active_sequences_per_replica": 64,
      "active_sequences_per_node": 256,
      "weight_bytes_per_replica": 16060522496,
      "kv_bytes_per_replica": 70866960384,
      "hbm_budget_bytes_per_replica": 112800000000,
      "memory_fit": true,
      "replica_output_tokens_s": [
        883.495,
        1766.99
      ],
      "node_output_tokens_s": [
        3533.98,
        7067.96
      ],
      "metric": "steady_state_decode_output_tokens_s_excludes_prefill",
      "status": "scenario_estimate_not_measured"
    },
    {
      "model_id": "llama31-8b",
      "model": "Llama 3.1 8B Instruct",
      "precision": "BF16",
      "gpu_count": 4,
      "tp": 1,
      "replicas": 4,
      "input_tokens": 32768,
      "output_tokens": 256,
      "context_at_end": 33024,
      "active_sequences_per_replica": 1,
      "active_sequences_per_node": 4,
      "weight_bytes_per_replica": 16060522496,
      "kv_bytes_per_replica": 4328521728,
      "hbm_budget_bytes_per_replica": 112800000000,
      "memory_fit": true,
      "replica_output_tokens_s": [
        58.855,
        117.71
      ],
      "node_output_tokens_s": [
        235.421,
        470.841
      ],
      "metric": "steady_state_decode_output_tokens_s_excludes_prefill",
      "status": "scenario_estimate_not_measured"
    },
    {
      "model_id": "llama31-8b",
      "model": "Llama 3.1 8B Instruct",
      "precision": "BF16",
      "gpu_count": 4,
      "tp": 1,
      "replicas": 4,
      "input_tokens": 32768,
      "output_tokens": 256,
      "context_at_end": 33024,
      "active_sequences_per_replica": 8,
      "active_sequences_per_node": 32,
      "weight_bytes_per_replica": 16060522496,
      "kv_bytes_per_replica": 34628173824,
      "hbm_budget_bytes_per_replica": 112800000000,
      "memory_fit": true,
      "replica_output_tokens_s": [
        189.391,
        378.783
      ],
      "node_output_tokens_s": [
        757.565,
        1515.131
      ],
      "metric": "steady_state_decode_output_tokens_s_excludes_prefill",
      "status": "scenario_estimate_not_measured"
    },
    {
      "model_id": "llama31-8b",
      "model": "Llama 3.1 8B Instruct",
      "precision": "BF16",
      "gpu_count": 4,
      "tp": 1,
      "replicas": 4,
      "input_tokens": 32768,
      "output_tokens": 256,
      "context_at_end": 33024,
      "active_sequences_per_replica": 32,
      "active_sequences_per_node": 128,
      "weight_bytes_per_replica": 16060522496,
      "kv_bytes_per_replica": 138512695296,
      "hbm_budget_bytes_per_replica": 112800000000,
      "memory_fit": false,
      "replica_output_tokens_s": null,
      "node_output_tokens_s": null,
      "metric": "steady_state_decode_output_tokens_s_excludes_prefill",
      "status": "exceeds_scenario_hbm_budget"
    },
    {
      "model_id": "llama31-8b",
      "model": "Llama 3.1 8B Instruct",
      "precision": "BF16",
      "gpu_count": 4,
      "tp": 1,
      "replicas": 4,
      "input_tokens": 32768,
      "output_tokens": 256,
      "context_at_end": 33024,
      "active_sequences_per_replica": 64,
      "active_sequences_per_node": 256,
      "weight_bytes_per_replica": 16060522496,
      "kv_bytes_per_replica": 277025390592,
      "hbm_budget_bytes_per_replica": 112800000000,
      "memory_fit": false,
      "replica_output_tokens_s": null,
      "node_output_tokens_s": null,
      "metric": "steady_state_decode_output_tokens_s_excludes_prefill",
      "status": "exceeds_scenario_hbm_budget"
    },
    {
      "model_id": "llama31-8b",
      "model": "Llama 3.1 8B Instruct",
      "precision": "BF16",
      "gpu_count": 4,
      "tp": 1,
      "replicas": 4,
      "input_tokens": 130816,
      "output_tokens": 256,
      "context_at_end": 131072,
      "active_sequences_per_replica": 1,
      "active_sequences_per_node": 4,
      "weight_bytes_per_replica": 16060522496,
      "kv_bytes_per_replica": 17179869184,
      "hbm_budget_bytes_per_replica": 112800000000,
      "memory_fit": true,
      "replica_output_tokens_s": [
        36.101,
        72.201
      ],
      "node_output_tokens_s": [
        144.403,
        288.805
      ],
      "metric": "steady_state_decode_output_tokens_s_excludes_prefill",
      "status": "scenario_estimate_not_measured"
    },
    {
      "model_id": "llama31-8b",
      "model": "Llama 3.1 8B Instruct",
      "precision": "BF16",
      "gpu_count": 4,
      "tp": 1,
      "replicas": 4,
      "input_tokens": 130816,
      "output_tokens": 256,
      "context_at_end": 131072,
      "active_sequences_per_replica": 8,
      "active_sequences_per_node": 32,
      "weight_bytes_per_replica": 16060522496,
      "kv_bytes_per_replica": 137438953472,
      "hbm_budget_bytes_per_replica": 112800000000,
      "memory_fit": false,
      "replica_output_tokens_s": null,
      "node_output_tokens_s": null,
      "metric": "steady_state_decode_output_tokens_s_excludes_prefill",
      "status": "exceeds_scenario_hbm_budget"
    },
    {
      "model_id": "llama31-8b",
      "model": "Llama 3.1 8B Instruct",
      "precision": "BF16",
      "gpu_count": 4,
      "tp": 1,
      "replicas": 4,
      "input_tokens": 130816,
      "output_tokens": 256,
      "context_at_end": 131072,
      "active_sequences_per_replica": 32,
      "active_sequences_per_node": 128,
      "weight_bytes_per_replica": 16060522496,
      "kv_bytes_per_replica": 549755813888,
      "hbm_budget_bytes_per_replica": 112800000000,
      "memory_fit": false,
      "replica_output_tokens_s": null,
      "node_output_tokens_s": null,
      "metric": "steady_state_decode_output_tokens_s_excludes_prefill",
      "status": "exceeds_scenario_hbm_budget"
    },
    {
      "model_id": "llama31-8b",
      "model": "Llama 3.1 8B Instruct",
      "precision": "BF16",
      "gpu_count": 4,
      "tp": 1,
      "replicas": 4,
      "input_tokens": 130816,
      "output_tokens": 256,
      "context_at_end": 131072,
      "active_sequences_per_replica": 64,
      "active_sequences_per_node": 256,
      "weight_bytes_per_replica": 16060522496,
      "kv_bytes_per_replica": 1099511627776,
      "hbm_budget_bytes_per_replica": 112800000000,
      "memory_fit": false,
      "replica_output_tokens_s": null,
      "node_output_tokens_s": null,
      "metric": "steady_state_decode_output_tokens_s_excludes_prefill",
      "status": "exceeds_scenario_hbm_budget"
    },
    {
      "model_id": "llama31-8b",
      "model": "Llama 3.1 8B Instruct",
      "precision": "BF16",
      "gpu_count": 8,
      "tp": 1,
      "replicas": 8,
      "input_tokens": 2048,
      "output_tokens": 256,
      "context_at_end": 2304,
      "active_sequences_per_replica": 1,
      "active_sequences_per_node": 8,
      "weight_bytes_per_replica": 16060522496,
      "kv_bytes_per_replica": 301989888,
      "hbm_budget_bytes_per_replica": 112800000000,
      "memory_fit": true,
      "replica_output_tokens_s": [
        73.338,
        146.677
      ],
      "node_output_tokens_s": [
        498.701,
        1114.743
      ],
      "metric": "steady_state_decode_output_tokens_s_excludes_prefill",
      "status": "scenario_estimate_not_measured"
    },
    {
      "model_id": "llama31-8b",
      "model": "Llama 3.1 8B Instruct",
      "precision": "BF16",
      "gpu_count": 8,
      "tp": 1,
      "replicas": 8,
      "input_tokens": 2048,
      "output_tokens": 256,
      "context_at_end": 2304,
      "active_sequences_per_replica": 8,
      "active_sequences_per_node": 64,
      "weight_bytes_per_replica": 16060522496,
      "kv_bytes_per_replica": 2415919104,
      "hbm_budget_bytes_per_replica": 112800000000,
      "memory_fit": true,
      "replica_output_tokens_s": [
        519.581,
        1039.161
      ],
      "node_output_tokens_s": [
        3533.148,
        7897.625
      ],
      "metric": "steady_state_decode_output_tokens_s_excludes_prefill",
      "status": "scenario_estimate_not_measured"
    },
    {
      "model_id": "llama31-8b",
      "model": "Llama 3.1 8B Instruct",
      "precision": "BF16",
      "gpu_count": 8,
      "tp": 1,
      "replicas": 8,
      "input_tokens": 2048,
      "output_tokens": 256,
      "context_at_end": 2304,
      "active_sequences_per_replica": 32,
      "active_sequences_per_node": 256,
      "weight_bytes_per_replica": 16060522496,
      "kv_bytes_per_replica": 9663676416,
      "hbm_budget_bytes_per_replica": 112800000000,
      "memory_fit": true,
      "replica_output_tokens_s": [
        1492.758,
        2985.516
      ],
      "node_output_tokens_s": [
        10150.753,
        22689.919
      ],
      "metric": "steady_state_decode_output_tokens_s_excludes_prefill",
      "status": "scenario_estimate_not_measured"
    },
    {
      "model_id": "llama31-8b",
      "model": "Llama 3.1 8B Instruct",
      "precision": "BF16",
      "gpu_count": 8,
      "tp": 1,
      "replicas": 8,
      "input_tokens": 2048,
      "output_tokens": 256,
      "context_at_end": 2304,
      "active_sequences_per_replica": 64,
      "active_sequences_per_node": 512,
      "weight_bytes_per_replica": 16060522496,
      "kv_bytes_per_replica": 19327352832,
      "hbm_budget_bytes_per_replica": 112800000000,
      "memory_fit": true,
      "replica_output_tokens_s": [
        2170.235,
        4340.47
      ],
      "node_output_tokens_s": [
        14757.597,
        32987.57
      ],
      "metric": "steady_state_decode_output_tokens_s_excludes_prefill",
      "status": "scenario_estimate_not_measured"
    },
    {
      "model_id": "llama31-8b",
      "model": "Llama 3.1 8B Instruct",
      "precision": "BF16",
      "gpu_count": 8,
      "tp": 1,
      "replicas": 8,
      "input_tokens": 8192,
      "output_tokens": 256,
      "context_at_end": 8448,
      "active_sequences_per_replica": 1,
      "active_sequences_per_node": 8,
      "weight_bytes_per_replica": 16060522496,
      "kv_bytes_per_replica": 1107296256,
      "hbm_budget_bytes_per_replica": 112800000000,
      "memory_fit": true,
      "replica_output_tokens_s": [
        69.898,
        139.796
      ],
      "node_output_tokens_s": [
        475.308,
        1062.453
      ],
      "metric": "steady_state_decode_output_tokens_s_excludes_prefill",
      "status": "scenario_estimate_not_measured"
    },
    {
      "model_id": "llama31-8b",
      "model": "Llama 3.1 8B Instruct",
      "precision": "BF16",
      "gpu_count": 8,
      "tp": 1,
      "replicas": 8,
      "input_tokens": 8192,
      "output_tokens": 256,
      "context_at_end": 8448,
      "active_sequences_per_replica": 8,
      "active_sequences_per_node": 64,
      "weight_bytes_per_replica": 16060522496,
      "kv_bytes_per_replica": 8858370048,
      "hbm_budget_bytes_per_replica": 112800000000,
      "memory_fit": true,
      "replica_output_tokens_s": [
        385.25,
        770.5
      ],
      "node_output_tokens_s": [
        2619.699,
        5855.798
      ],
      "metric": "steady_state_decode_output_tokens_s_excludes_prefill",
      "status": "scenario_estimate_not_measured"
    },
    {
      "model_id": "llama31-8b",
      "model": "Llama 3.1 8B Instruct",
      "precision": "BF16",
      "gpu_count": 8,
      "tp": 1,
      "replicas": 8,
      "input_tokens": 8192,
      "output_tokens": 256,
      "context_at_end": 8448,
      "active_sequences_per_replica": 32,
      "active_sequences_per_node": 256,
      "weight_bytes_per_replica": 16060522496,
      "kv_bytes_per_replica": 35433480192,
      "hbm_budget_bytes_per_replica": 112800000000,
      "memory_fit": true,
      "replica_output_tokens_s": [
        745.718,
        1491.436
      ],
      "node_output_tokens_s": [
        5070.882,
        11334.912
      ],
      "metric": "steady_state_decode_output_tokens_s_excludes_prefill",
      "status": "scenario_estimate_not_measured"
    },
    {
      "model_id": "llama31-8b",
      "model": "Llama 3.1 8B Instruct",
      "precision": "BF16",
      "gpu_count": 8,
      "tp": 1,
      "replicas": 8,
      "input_tokens": 8192,
      "output_tokens": 256,
      "context_at_end": 8448,
      "active_sequences_per_replica": 64,
      "active_sequences_per_node": 512,
      "weight_bytes_per_replica": 16060522496,
      "kv_bytes_per_replica": 70866960384,
      "hbm_budget_bytes_per_replica": 112800000000,
      "memory_fit": true,
      "replica_output_tokens_s": [
        883.495,
        1766.99
      ],
      "node_output_tokens_s": [
        6007.766,
        13429.125
      ],
      "metric": "steady_state_decode_output_tokens_s_excludes_prefill",
      "status": "scenario_estimate_not_measured"
    },
    {
      "model_id": "llama31-8b",
      "model": "Llama 3.1 8B Instruct",
      "precision": "BF16",
      "gpu_count": 8,
      "tp": 1,
      "replicas": 8,
      "input_tokens": 32768,
      "output_tokens": 256,
      "context_at_end": 33024,
      "active_sequences_per_replica": 1,
      "active_sequences_per_node": 8,
      "weight_bytes_per_replica": 16060522496,
      "kv_bytes_per_replica": 4328521728,
      "hbm_budget_bytes_per_replica": 112800000000,
      "memory_fit": true,
      "replica_output_tokens_s": [
        58.855,
        117.71
      ],
      "node_output_tokens_s": [
        400.215,
        894.598
      ],
      "metric": "steady_state_decode_output_tokens_s_excludes_prefill",
      "status": "scenario_estimate_not_measured"
    },
    {
      "model_id": "llama31-8b",
      "model": "Llama 3.1 8B Instruct",
      "precision": "BF16",
      "gpu_count": 8,
      "tp": 1,
      "replicas": 8,
      "input_tokens": 32768,
      "output_tokens": 256,
      "context_at_end": 33024,
      "active_sequences_per_replica": 8,
      "active_sequences_per_node": 64,
      "weight_bytes_per_replica": 16060522496,
      "kv_bytes_per_replica": 34628173824,
      "hbm_budget_bytes_per_replica": 112800000000,
      "memory_fit": true,
      "replica_output_tokens_s": [
        189.391,
        378.783
      ],
      "node_output_tokens_s": [
        1287.861,
        2878.748
      ],
      "metric": "steady_state_decode_output_tokens_s_excludes_prefill",
      "status": "scenario_estimate_not_measured"
    },
    {
      "model_id": "llama31-8b",
      "model": "Llama 3.1 8B Instruct",
      "precision": "BF16",
      "gpu_count": 8,
      "tp": 1,
      "replicas": 8,
      "input_tokens": 32768,
      "output_tokens": 256,
      "context_at_end": 33024,
      "active_sequences_per_replica": 32,
      "active_sequences_per_node": 256,
      "weight_bytes_per_replica": 16060522496,
      "kv_bytes_per_replica": 138512695296,
      "hbm_budget_bytes_per_replica": 112800000000,
      "memory_fit": false,
      "replica_output_tokens_s": null,
      "node_output_tokens_s": null,
      "metric": "steady_state_decode_output_tokens_s_excludes_prefill",
      "status": "exceeds_scenario_hbm_budget"
    },
    {
      "model_id": "llama31-8b",
      "model": "Llama 3.1 8B Instruct",
      "precision": "BF16",
      "gpu_count": 8,
      "tp": 1,
      "replicas": 8,
      "input_tokens": 32768,
      "output_tokens": 256,
      "context_at_end": 33024,
      "active_sequences_per_replica": 64,
      "active_sequences_per_node": 512,
      "weight_bytes_per_replica": 16060522496,
      "kv_bytes_per_replica": 277025390592,
      "hbm_budget_bytes_per_replica": 112800000000,
      "memory_fit": false,
      "replica_output_tokens_s": null,
      "node_output_tokens_s": null,
      "metric": "steady_state_decode_output_tokens_s_excludes_prefill",
      "status": "exceeds_scenario_hbm_budget"
    },
    {
      "model_id": "llama31-8b",
      "model": "Llama 3.1 8B Instruct",
      "precision": "BF16",
      "gpu_count": 8,
      "tp": 1,
      "replicas": 8,
      "input_tokens": 130816,
      "output_tokens": 256,
      "context_at_end": 131072,
      "active_sequences_per_replica": 1,
      "active_sequences_per_node": 8,
      "weight_bytes_per_replica": 16060522496,
      "kv_bytes_per_replica": 17179869184,
      "hbm_budget_bytes_per_replica": 112800000000,
      "memory_fit": true,
      "replica_output_tokens_s": [
        36.101,
        72.201
      ],
      "node_output_tokens_s": [
        245.484,
        548.73
      ],
      "metric": "steady_state_decode_output_tokens_s_excludes_prefill",
      "status": "scenario_estimate_not_measured"
    },
    {
      "model_id": "llama31-8b",
      "model": "Llama 3.1 8B Instruct",
      "precision": "BF16",
      "gpu_count": 8,
      "tp": 1,
      "replicas": 8,
      "input_tokens": 130816,
      "output_tokens": 256,
      "context_at_end": 131072,
      "active_sequences_per_replica": 8,
      "active_sequences_per_node": 64,
      "weight_bytes_per_replica": 16060522496,
      "kv_bytes_per_replica": 137438953472,
      "hbm_budget_bytes_per_replica": 112800000000,
      "memory_fit": false,
      "replica_output_tokens_s": null,
      "node_output_tokens_s": null,
      "metric": "steady_state_decode_output_tokens_s_excludes_prefill",
      "status": "exceeds_scenario_hbm_budget"
    },
    {
      "model_id": "llama31-8b",
      "model": "Llama 3.1 8B Instruct",
      "precision": "BF16",
      "gpu_count": 8,
      "tp": 1,
      "replicas": 8,
      "input_tokens": 130816,
      "output_tokens": 256,
      "context_at_end": 131072,
      "active_sequences_per_replica": 32,
      "active_sequences_per_node": 256,
      "weight_bytes_per_replica": 16060522496,
      "kv_bytes_per_replica": 549755813888,
      "hbm_budget_bytes_per_replica": 112800000000,
      "memory_fit": false,
      "replica_output_tokens_s": null,
      "node_output_tokens_s": null,
      "metric": "steady_state_decode_output_tokens_s_excludes_prefill",
      "status": "exceeds_scenario_hbm_budget"
    },
    {
      "model_id": "llama31-8b",
      "model": "Llama 3.1 8B Instruct",
      "precision": "BF16",
      "gpu_count": 8,
      "tp": 1,
      "replicas": 8,
      "input_tokens": 130816,
      "output_tokens": 256,
      "context_at_end": 131072,
      "active_sequences_per_replica": 64,
      "active_sequences_per_node": 512,
      "weight_bytes_per_replica": 16060522496,
      "kv_bytes_per_replica": 1099511627776,
      "hbm_budget_bytes_per_replica": 112800000000,
      "memory_fit": false,
      "replica_output_tokens_s": null,
      "node_output_tokens_s": null,
      "metric": "steady_state_decode_output_tokens_s_excludes_prefill",
      "status": "exceeds_scenario_hbm_budget"
    },
    {
      "model_id": "llama31-70b",
      "model": "Llama 3.1 70B Instruct",
      "precision": "BF16",
      "gpu_count": 4,
      "tp": 4,
      "replicas": 1,
      "input_tokens": 2048,
      "output_tokens": 256,
      "context_at_end": 2304,
      "active_sequences_per_replica": 1,
      "active_sequences_per_node": 1,
      "weight_bytes_per_replica": 141107412992,
      "kv_bytes_per_replica": 754974720,
      "hbm_budget_bytes_per_replica": 451200000000,
      "memory_fit": true,
      "replica_output_tokens_s": [
        40.603,
        74.438
      ],
      "node_output_tokens_s": [
        40.603,
        74.438
      ],
      "metric": "steady_state_decode_output_tokens_s_excludes_prefill",
      "status": "scenario_estimate_not_measured"
    },
    {
      "model_id": "llama31-70b",
      "model": "Llama 3.1 70B Instruct",
      "precision": "BF16",
      "gpu_count": 4,
      "tp": 4,
      "replicas": 1,
      "input_tokens": 2048,
      "output_tokens": 256,
      "context_at_end": 2304,
      "active_sequences_per_replica": 8,
      "active_sequences_per_node": 8,
      "weight_bytes_per_replica": 141107412992,
      "kv_bytes_per_replica": 6039797760,
      "hbm_budget_bytes_per_replica": 451200000000,
      "memory_fit": true,
      "replica_output_tokens_s": [
        313.156,
        574.119
      ],
      "node_output_tokens_s": [
        313.156,
        574.119
      ],
      "metric": "steady_state_decode_output_tokens_s_excludes_prefill",
      "status": "scenario_estimate_not_measured"
    },
    {
      "model_id": "llama31-70b",
      "model": "Llama 3.1 70B Instruct",
      "precision": "BF16",
      "gpu_count": 4,
      "tp": 4,
      "replicas": 1,
      "input_tokens": 2048,
      "output_tokens": 256,
      "context_at_end": 2304,
      "active_sequences_per_replica": 32,
      "active_sequences_per_node": 32,
      "weight_bytes_per_replica": 141107412992,
      "kv_bytes_per_replica": 24159191040,
      "hbm_budget_bytes_per_replica": 451200000000,
      "memory_fit": true,
      "replica_output_tokens_s": [
        1115.289,
        2044.696
      ],
      "node_output_tokens_s": [
        1115.289,
        2044.696
      ],
      "metric": "steady_state_decode_output_tokens_s_excludes_prefill",
      "status": "scenario_estimate_not_measured"
    },
    {
      "model_id": "llama31-70b",
      "model": "Llama 3.1 70B Instruct",
      "precision": "BF16",
      "gpu_count": 4,
      "tp": 4,
      "replicas": 1,
      "input_tokens": 2048,
      "output_tokens": 256,
      "context_at_end": 2304,
      "active_sequences_per_replica": 64,
      "active_sequences_per_node": 64,
      "weight_bytes_per_replica": 141107412992,
      "kv_bytes_per_replica": 48318382080,
      "hbm_budget_bytes_per_replica": 451200000000,
      "memory_fit": true,
      "replica_output_tokens_s": [
        1946.092,
        3567.835
      ],
      "node_output_tokens_s": [
        1946.092,
        3567.835
      ],
      "metric": "steady_state_decode_output_tokens_s_excludes_prefill",
      "status": "scenario_estimate_not_measured"
    },
    {
      "model_id": "llama31-70b",
      "model": "Llama 3.1 70B Instruct",
      "precision": "BF16",
      "gpu_count": 4,
      "tp": 4,
      "replicas": 1,
      "input_tokens": 8192,
      "output_tokens": 256,
      "context_at_end": 8448,
      "active_sequences_per_replica": 1,
      "active_sequences_per_node": 1,
      "weight_bytes_per_replica": 141107412992,
      "kv_bytes_per_replica": 2768240640,
      "hbm_budget_bytes_per_replica": 451200000000,
      "memory_fit": true,
      "replica_output_tokens_s": [
        40.035,
        73.397
      ],
      "node_output_tokens_s": [
        40.035,
        73.397
      ],
      "metric": "steady_state_decode_output_tokens_s_excludes_prefill",
      "status": "scenario_estimate_not_measured"
    },
    {
      "model_id": "llama31-70b",
      "model": "Llama 3.1 70B Instruct",
      "precision": "BF16",
      "gpu_count": 4,
      "tp": 4,
      "replicas": 1,
      "input_tokens": 8192,
      "output_tokens": 256,
      "context_at_end": 8448,
      "active_sequences_per_replica": 8,
      "active_sequences_per_node": 8,
      "weight_bytes_per_replica": 141107412992,
      "kv_bytes_per_replica": 22145925120,
      "hbm_budget_bytes_per_replica": 451200000000,
      "memory_fit": true,
      "replica_output_tokens_s": [
        282.261,
        517.478
      ],
      "node_output_tokens_s": [
        282.261,
        517.478
      ],
      "metric": "steady_state_decode_output_tokens_s_excludes_prefill",
      "status": "scenario_estimate_not_measured"
    },
    {
      "model_id": "llama31-70b",
      "model": "Llama 3.1 70B Instruct",
      "precision": "BF16",
      "gpu_count": 4,
      "tp": 4,
      "replicas": 1,
      "input_tokens": 8192,
      "output_tokens": 256,
      "context_at_end": 8448,
      "active_sequences_per_replica": 32,
      "active_sequences_per_node": 32,
      "weight_bytes_per_replica": 141107412992,
      "kv_bytes_per_replica": 88583700480,
      "hbm_budget_bytes_per_replica": 451200000000,
      "memory_fit": true,
      "replica_output_tokens_s": [
        802.469,
        1471.193
      ],
      "node_output_tokens_s": [
        802.469,
        1471.193
      ],
      "metric": "steady_state_decode_output_tokens_s_excludes_prefill",
      "status": "scenario_estimate_not_measured"
    },
    {
      "model_id": "llama31-70b",
      "model": "Llama 3.1 70B Instruct",
      "precision": "BF16",
      "gpu_count": 4,
      "tp": 4,
      "replicas": 1,
      "input_tokens": 8192,
      "output_tokens": 256,
      "context_at_end": 8448,
      "active_sequences_per_replica": 64,
      "active_sequences_per_node": 64,
      "weight_bytes_per_replica": 141107412992,
      "kv_bytes_per_replica": 177167400960,
      "hbm_budget_bytes_per_replica": 451200000000,
      "memory_fit": true,
      "replica_output_tokens_s": [
        1158.244,
        2123.448
      ],
      "node_output_tokens_s": [
        1158.244,
        2123.448
      ],
      "metric": "steady_state_decode_output_tokens_s_excludes_prefill",
      "status": "scenario_estimate_not_measured"
    },
    {
      "model_id": "llama31-70b",
      "model": "Llama 3.1 70B Instruct",
      "precision": "BF16",
      "gpu_count": 4,
      "tp": 4,
      "replicas": 1,
      "input_tokens": 32768,
      "output_tokens": 256,
      "context_at_end": 33024,
      "active_sequences_per_replica": 1,
      "active_sequences_per_node": 1,
      "weight_bytes_per_replica": 141107412992,
      "kv_bytes_per_replica": 10821304320,
      "hbm_budget_bytes_per_replica": 451200000000,
      "memory_fit": true,
      "replica_output_tokens_s": [
        37.913,
        69.506
      ],
      "node_output_tokens_s": [
        37.913,
        69.506
      ],
      "metric": "steady_state_decode_output_tokens_s_excludes_prefill",
      "status": "scenario_estimate_not_measured"
    },
    {
      "model_id": "llama31-70b",
      "model": "Llama 3.1 70B Instruct",
      "precision": "BF16",
      "gpu_count": 4,
      "tp": 4,
      "replicas": 1,
      "input_tokens": 32768,
      "output_tokens": 256,
      "context_at_end": 33024,
      "active_sequences_per_replica": 8,
      "active_sequences_per_node": 8,
      "weight_bytes_per_replica": 141107412992,
      "kv_bytes_per_replica": 86570434560,
      "hbm_budget_bytes_per_replica": 451200000000,
      "memory_fit": true,
      "replica_output_tokens_s": [
        202.391,
        371.051
      ],
      "node_output_tokens_s": [
        202.391,
        371.051
      ],
      "metric": "steady_state_decode_output_tokens_s_excludes_prefill",
      "status": "scenario_estimate_not_measured"
    },
    {
      "model_id": "llama31-70b",
      "model": "Llama 3.1 70B Instruct",
      "precision": "BF16",
      "gpu_count": 4,
      "tp": 4,
      "replicas": 1,
      "input_tokens": 32768,
      "output_tokens": 256,
      "context_at_end": 33024,
      "active_sequences_per_replica": 32,
      "active_sequences_per_node": 32,
      "weight_bytes_per_replica": 141107412992,
      "kv_bytes_per_replica": 346281738240,
      "hbm_budget_bytes_per_replica": 451200000000,
      "memory_fit": false,
      "replica_output_tokens_s": null,
      "node_output_tokens_s": null,
      "metric": "steady_state_decode_output_tokens_s_excludes_prefill",
      "status": "exceeds_scenario_hbm_budget"
    },
    {
      "model_id": "llama31-70b",
      "model": "Llama 3.1 70B Instruct",
      "precision": "BF16",
      "gpu_count": 4,
      "tp": 4,
      "replicas": 1,
      "input_tokens": 32768,
      "output_tokens": 256,
      "context_at_end": 33024,
      "active_sequences_per_replica": 64,
      "active_sequences_per_node": 64,
      "weight_bytes_per_replica": 141107412992,
      "kv_bytes_per_replica": 692563476480,
      "hbm_budget_bytes_per_replica": 451200000000,
      "memory_fit": false,
      "replica_output_tokens_s": null,
      "node_output_tokens_s": null,
      "metric": "steady_state_decode_output_tokens_s_excludes_prefill",
      "status": "exceeds_scenario_hbm_budget"
    },
    {
      "model_id": "llama31-70b",
      "model": "Llama 3.1 70B Instruct",
      "precision": "BF16",
      "gpu_count": 4,
      "tp": 4,
      "replicas": 1,
      "input_tokens": 130816,
      "output_tokens": 256,
      "context_at_end": 131072,
      "active_sequences_per_replica": 1,
      "active_sequences_per_node": 1,
      "weight_bytes_per_replica": 141107412992,
      "kv_bytes_per_replica": 42949672960,
      "hbm_budget_bytes_per_replica": 451200000000,
      "memory_fit": true,
      "replica_output_tokens_s": [
        31.295,
        57.374
      ],
      "node_output_tokens_s": [
        31.295,
        57.374
      ],
      "metric": "steady_state_decode_output_tokens_s_excludes_prefill",
      "status": "scenario_estimate_not_measured"
    },
    {
      "model_id": "llama31-70b",
      "model": "Llama 3.1 70B Instruct",
      "precision": "BF16",
      "gpu_count": 4,
      "tp": 4,
      "replicas": 1,
      "input_tokens": 130816,
      "output_tokens": 256,
      "context_at_end": 131072,
      "active_sequences_per_replica": 8,
      "active_sequences_per_node": 8,
      "weight_bytes_per_replica": 141107412992,
      "kv_bytes_per_replica": 343597383680,
      "hbm_budget_bytes_per_replica": 451200000000,
      "memory_fit": false,
      "replica_output_tokens_s": null,
      "node_output_tokens_s": null,
      "metric": "steady_state_decode_output_tokens_s_excludes_prefill",
      "status": "exceeds_scenario_hbm_budget"
    },
    {
      "model_id": "llama31-70b",
      "model": "Llama 3.1 70B Instruct",
      "precision": "BF16",
      "gpu_count": 4,
      "tp": 4,
      "replicas": 1,
      "input_tokens": 130816,
      "output_tokens": 256,
      "context_at_end": 131072,
      "active_sequences_per_replica": 32,
      "active_sequences_per_node": 32,
      "weight_bytes_per_replica": 141107412992,
      "kv_bytes_per_replica": 1374389534720,
      "hbm_budget_bytes_per_replica": 451200000000,
      "memory_fit": false,
      "replica_output_tokens_s": null,
      "node_output_tokens_s": null,
      "metric": "steady_state_decode_output_tokens_s_excludes_prefill",
      "status": "exceeds_scenario_hbm_budget"
    },
    {
      "model_id": "llama31-70b",
      "model": "Llama 3.1 70B Instruct",
      "precision": "BF16",
      "gpu_count": 4,
      "tp": 4,
      "replicas": 1,
      "input_tokens": 130816,
      "output_tokens": 256,
      "context_at_end": 131072,
      "active_sequences_per_replica": 64,
      "active_sequences_per_node": 64,
      "weight_bytes_per_replica": 141107412992,
      "kv_bytes_per_replica": 2748779069440,
      "hbm_budget_bytes_per_replica": 451200000000,
      "memory_fit": false,
      "replica_output_tokens_s": null,
      "node_output_tokens_s": null,
      "metric": "steady_state_decode_output_tokens_s_excludes_prefill",
      "status": "exceeds_scenario_hbm_budget"
    },
    {
      "model_id": "llama31-70b",
      "model": "Llama 3.1 70B Instruct",
      "precision": "BF16",
      "gpu_count": 8,
      "tp": 4,
      "replicas": 2,
      "input_tokens": 2048,
      "output_tokens": 256,
      "context_at_end": 2304,
      "active_sequences_per_replica": 1,
      "active_sequences_per_node": 2,
      "weight_bytes_per_replica": 141107412992,
      "kv_bytes_per_replica": 754974720,
      "hbm_budget_bytes_per_replica": 451200000000,
      "memory_fit": true,
      "replica_output_tokens_s": [
        40.603,
        74.438
      ],
      "node_output_tokens_s": [
        69.025,
        141.433
      ],
      "metric": "steady_state_decode_output_tokens_s_excludes_prefill",
      "status": "scenario_estimate_not_measured"
    },
    {
      "model_id": "llama31-70b",
      "model": "Llama 3.1 70B Instruct",
      "precision": "BF16",
      "gpu_count": 8,
      "tp": 4,
      "replicas": 2,
      "input_tokens": 2048,
      "output_tokens": 256,
      "context_at_end": 2304,
      "active_sequences_per_replica": 8,
      "active_sequences_per_node": 16,
      "weight_bytes_per_replica": 141107412992,
      "kv_bytes_per_replica": 6039797760,
      "hbm_budget_bytes_per_replica": 451200000000,
      "memory_fit": true,
      "replica_output_tokens_s": [
        313.156,
        574.119
      ],
      "node_output_tokens_s": [
        532.365,
        1090.826
      ],
      "metric": "steady_state_decode_output_tokens_s_excludes_prefill",
      "status": "scenario_estimate_not_measured"
    },
    {
      "model_id": "llama31-70b",
      "model": "Llama 3.1 70B Instruct",
      "precision": "BF16",
      "gpu_count": 8,
      "tp": 4,
      "replicas": 2,
      "input_tokens": 2048,
      "output_tokens": 256,
      "context_at_end": 2304,
      "active_sequences_per_replica": 32,
      "active_sequences_per_node": 64,
      "weight_bytes_per_replica": 141107412992,
      "kv_bytes_per_replica": 24159191040,
      "hbm_budget_bytes_per_replica": 451200000000,
      "memory_fit": true,
      "replica_output_tokens_s": [
        1115.289,
        2044.696
      ],
      "node_output_tokens_s": [
        1895.991,
        3884.923
      ],
      "metric": "steady_state_decode_output_tokens_s_excludes_prefill",
      "status": "scenario_estimate_not_measured"
    },
    {
      "model_id": "llama31-70b",
      "model": "Llama 3.1 70B Instruct",
      "precision": "BF16",
      "gpu_count": 8,
      "tp": 4,
      "replicas": 2,
      "input_tokens": 2048,
      "output_tokens": 256,
      "context_at_end": 2304,
      "active_sequences_per_replica": 64,
      "active_sequences_per_node": 128,
      "weight_bytes_per_replica": 141107412992,
      "kv_bytes_per_replica": 48318382080,
      "hbm_budget_bytes_per_replica": 451200000000,
      "memory_fit": true,
      "replica_output_tokens_s": [
        1946.092,
        3567.835
      ],
      "node_output_tokens_s": [
        3308.356,
        6778.887
      ],
      "metric": "steady_state_decode_output_tokens_s_excludes_prefill",
      "status": "scenario_estimate_not_measured"
    },
    {
      "model_id": "llama31-70b",
      "model": "Llama 3.1 70B Instruct",
      "precision": "BF16",
      "gpu_count": 8,
      "tp": 4,
      "replicas": 2,
      "input_tokens": 8192,
      "output_tokens": 256,
      "context_at_end": 8448,
      "active_sequences_per_replica": 1,
      "active_sequences_per_node": 2,
      "weight_bytes_per_replica": 141107412992,
      "kv_bytes_per_replica": 2768240640,
      "hbm_budget_bytes_per_replica": 451200000000,
      "memory_fit": true,
      "replica_output_tokens_s": [
        40.035,
        73.397
      ],
      "node_output_tokens_s": [
        68.059,
        139.454
      ],
      "metric": "steady_state_decode_output_tokens_s_excludes_prefill",
      "status": "scenario_estimate_not_measured"
    },
    {
      "model_id": "llama31-70b",
      "model": "Llama 3.1 70B Instruct",
      "precision": "BF16",
      "gpu_count": 8,
      "tp": 4,
      "replicas": 2,
      "input_tokens": 8192,
      "output_tokens": 256,
      "context_at_end": 8448,
      "active_sequences_per_replica": 8,
      "active_sequences_per_node": 16,
      "weight_bytes_per_replica": 141107412992,
      "kv_bytes_per_replica": 22145925120,
      "hbm_budget_bytes_per_replica": 451200000000,
      "memory_fit": true,
      "replica_output_tokens_s": [
        282.261,
        517.478
      ],
      "node_output_tokens_s": [
        479.843,
        983.208
      ],
      "metric": "steady_state_decode_output_tokens_s_excludes_prefill",
      "status": "scenario_estimate_not_measured"
    },
    {
      "model_id": "llama31-70b",
      "model": "Llama 3.1 70B Instruct",
      "precision": "BF16",
      "gpu_count": 8,
      "tp": 4,
      "replicas": 2,
      "input_tokens": 8192,
      "output_tokens": 256,
      "context_at_end": 8448,
      "active_sequences_per_replica": 32,
      "active_sequences_per_node": 64,
      "weight_bytes_per_replica": 141107412992,
      "kv_bytes_per_replica": 88583700480,
      "hbm_budget_bytes_per_replica": 451200000000,
      "memory_fit": true,
      "replica_output_tokens_s": [
        802.469,
        1471.193
      ],
      "node_output_tokens_s": [
        1364.197,
        2795.267
      ],
      "metric": "steady_state_decode_output_tokens_s_excludes_prefill",
      "status": "scenario_estimate_not_measured"
    },
    {
      "model_id": "llama31-70b",
      "model": "Llama 3.1 70B Instruct",
      "precision": "BF16",
      "gpu_count": 8,
      "tp": 4,
      "replicas": 2,
      "input_tokens": 8192,
      "output_tokens": 256,
      "context_at_end": 8448,
      "active_sequences_per_replica": 64,
      "active_sequences_per_node": 128,
      "weight_bytes_per_replica": 141107412992,
      "kv_bytes_per_replica": 177167400960,
      "hbm_budget_bytes_per_replica": 451200000000,
      "memory_fit": true,
      "replica_output_tokens_s": [
        1158.244,
        2123.448
      ],
      "node_output_tokens_s": [
        1969.015,
        4034.551
      ],
      "metric": "steady_state_decode_output_tokens_s_excludes_prefill",
      "status": "scenario_estimate_not_measured"
    },
    {
      "model_id": "llama31-70b",
      "model": "Llama 3.1 70B Instruct",
      "precision": "BF16",
      "gpu_count": 8,
      "tp": 4,
      "replicas": 2,
      "input_tokens": 32768,
      "output_tokens": 256,
      "context_at_end": 33024,
      "active_sequences_per_replica": 1,
      "active_sequences_per_node": 2,
      "weight_bytes_per_replica": 141107412992,
      "kv_bytes_per_replica": 10821304320,
      "hbm_budget_bytes_per_replica": 451200000000,
      "memory_fit": true,
      "replica_output_tokens_s": [
        37.913,
        69.506
      ],
      "node_output_tokens_s": [
        64.451,
        132.062
      ],
      "metric": "steady_state_decode_output_tokens_s_excludes_prefill",
      "status": "scenario_estimate_not_measured"
    },
    {
      "model_id": "llama31-70b",
      "model": "Llama 3.1 70B Instruct",
      "precision": "BF16",
      "gpu_count": 8,
      "tp": 4,
      "replicas": 2,
      "input_tokens": 32768,
      "output_tokens": 256,
      "context_at_end": 33024,
      "active_sequences_per_replica": 8,
      "active_sequences_per_node": 16,
      "weight_bytes_per_replica": 141107412992,
      "kv_bytes_per_replica": 86570434560,
      "hbm_budget_bytes_per_replica": 451200000000,
      "memory_fit": true,
      "replica_output_tokens_s": [
        202.391,
        371.051
      ],
      "node_output_tokens_s": [
        344.065,
        704.996
      ],
      "metric": "steady_state_decode_output_tokens_s_excludes_prefill",
      "status": "scenario_estimate_not_measured"
    },
    {
      "model_id": "llama31-70b",
      "model": "Llama 3.1 70B Instruct",
      "precision": "BF16",
      "gpu_count": 8,
      "tp": 4,
      "replicas": 2,
      "input_tokens": 32768,
      "output_tokens": 256,
      "context_at_end": 33024,
      "active_sequences_per_replica": 32,
      "active_sequences_per_node": 64,
      "weight_bytes_per_replica": 141107412992,
      "kv_bytes_per_replica": 346281738240,
      "hbm_budget_bytes_per_replica": 451200000000,
      "memory_fit": false,
      "replica_output_tokens_s": null,
      "node_output_tokens_s": null,
      "metric": "steady_state_decode_output_tokens_s_excludes_prefill",
      "status": "exceeds_scenario_hbm_budget"
    },
    {
      "model_id": "llama31-70b",
      "model": "Llama 3.1 70B Instruct",
      "precision": "BF16",
      "gpu_count": 8,
      "tp": 4,
      "replicas": 2,
      "input_tokens": 32768,
      "output_tokens": 256,
      "context_at_end": 33024,
      "active_sequences_per_replica": 64,
      "active_sequences_per_node": 128,
      "weight_bytes_per_replica": 141107412992,
      "kv_bytes_per_replica": 692563476480,
      "hbm_budget_bytes_per_replica": 451200000000,
      "memory_fit": false,
      "replica_output_tokens_s": null,
      "node_output_tokens_s": null,
      "metric": "steady_state_decode_output_tokens_s_excludes_prefill",
      "status": "exceeds_scenario_hbm_budget"
    },
    {
      "model_id": "llama31-70b",
      "model": "Llama 3.1 70B Instruct",
      "precision": "BF16",
      "gpu_count": 8,
      "tp": 4,
      "replicas": 2,
      "input_tokens": 130816,
      "output_tokens": 256,
      "context_at_end": 131072,
      "active_sequences_per_replica": 1,
      "active_sequences_per_node": 2,
      "weight_bytes_per_replica": 141107412992,
      "kv_bytes_per_replica": 42949672960,
      "hbm_budget_bytes_per_replica": 451200000000,
      "memory_fit": true,
      "replica_output_tokens_s": [
        31.295,
        57.374
      ],
      "node_output_tokens_s": [
        53.201,
        109.01
      ],
      "metric": "steady_state_decode_output_tokens_s_excludes_prefill",
      "status": "scenario_estimate_not_measured"
    },
    {
      "model_id": "llama31-70b",
      "model": "Llama 3.1 70B Instruct",
      "precision": "BF16",
      "gpu_count": 8,
      "tp": 4,
      "replicas": 2,
      "input_tokens": 130816,
      "output_tokens": 256,
      "context_at_end": 131072,
      "active_sequences_per_replica": 8,
      "active_sequences_per_node": 16,
      "weight_bytes_per_replica": 141107412992,
      "kv_bytes_per_replica": 343597383680,
      "hbm_budget_bytes_per_replica": 451200000000,
      "memory_fit": false,
      "replica_output_tokens_s": null,
      "node_output_tokens_s": null,
      "metric": "steady_state_decode_output_tokens_s_excludes_prefill",
      "status": "exceeds_scenario_hbm_budget"
    },
    {
      "model_id": "llama31-70b",
      "model": "Llama 3.1 70B Instruct",
      "precision": "BF16",
      "gpu_count": 8,
      "tp": 4,
      "replicas": 2,
      "input_tokens": 130816,
      "output_tokens": 256,
      "context_at_end": 131072,
      "active_sequences_per_replica": 32,
      "active_sequences_per_node": 64,
      "weight_bytes_per_replica": 141107412992,
      "kv_bytes_per_replica": 1374389534720,
      "hbm_budget_bytes_per_replica": 451200000000,
      "memory_fit": false,
      "replica_output_tokens_s": null,
      "node_output_tokens_s": null,
      "metric": "steady_state_decode_output_tokens_s_excludes_prefill",
      "status": "exceeds_scenario_hbm_budget"
    },
    {
      "model_id": "llama31-70b",
      "model": "Llama 3.1 70B Instruct",
      "precision": "BF16",
      "gpu_count": 8,
      "tp": 4,
      "replicas": 2,
      "input_tokens": 130816,
      "output_tokens": 256,
      "context_at_end": 131072,
      "active_sequences_per_replica": 64,
      "active_sequences_per_node": 128,
      "weight_bytes_per_replica": 141107412992,
      "kv_bytes_per_replica": 2748779069440,
      "hbm_budget_bytes_per_replica": 451200000000,
      "memory_fit": false,
      "replica_output_tokens_s": null,
      "node_output_tokens_s": null,
      "metric": "steady_state_decode_output_tokens_s_excludes_prefill",
      "status": "exceeds_scenario_hbm_budget"
    }
  ]
}