{ "complete": true, "split": "validation", "provenance": { "source": { "repo": "RecursiveSI/flint", "revision": "2a247be0c0aec895ccb0ac7ca92229ec6df37288" }, "scripts": { "flint_quantize.py": "3f87c9cf1bed5a2346ebc085579a82e14a236325b0b2277171dfae1d07d7b692", "flint_reference.py": "2534d51af8e99699692f1a2cf5fc022bd7beaa7de07770dbffb9690aa4f62ff6", "flint_nvfp4.py": "31aec4a310b7bddaa90da083b80352e5607ce754c10052458859f7306b5cc566" }, "packages": { "torch": "2.8.0+cu128", "triton": "3.4.0", "safetensors": "0.8.0", "transformers": "5.20.2", "huggingface_hub": "2.2.0", "datasets": "5.1.0" }, "gpu": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "cuda": "12.8", "created_utc": "2026-10-21T23:38:51Z" }, "script_sha256": "e0e0aa655a97d9c78837196a94b280c54944ab5542686242dbaa7604d33dca1c", "normalization_script_sha256": "f06c3d51f9018814648a1fe42c39274dd103528b30185d914f76adbd68fef3ac", "flashinfer_version": "0.6.16.post1", "corpus_sha256": "ad58c37420891d4c5b08181aace2e75f00ac3f8a3253269b89a0f8ee88418923", "results": { "bf16": { "precision_by_layer": { "2": { "gate": "bf16", "up": "bf16", "down": "bf16" }, "3": { "gate": "bf16", "up": "bf16", "down": "bf16" }, "4": { "gate": "bf16", "up": "bf16", "down": "bf16" }, "5": { "gate": "bf16", "up": "bf16", "down": "bf16" }, "6": { "gate": "bf16", "up": "bf16", "down": "bf16" }, "7": { "gate": "bf16", "up": "bf16", "down": "bf16" }, "8": { "gate": "bf16", "up": "bf16", "down": "bf16" }, "9": { "gate": "bf16", "up": "bf16", "down": "bf16" }, "10": { "gate": "bf16", "up": "bf16", "down": "bf16" }, "11": { "gate": "bf16", "up": "bf16", "down": "bf16" }, "12": { "gate": "bf16", "up": "bf16", "down": "bf16" }, "13": { "gate": "bf16", "up": "bf16", "down": "bf16" }, "14": { "gate": "bf16", "up": "bf16", "down": "bf16" }, "15": { "gate": "bf16", "up": "bf16", "down": "bf16" }, "16": { "gate": "bf16", "up": "bf16", "down": "bf16" }, "17": { "gate": "bf16", "up": "bf16", "down": "bf16" }, "18": { "gate": "bf16", "up": "bf16", "down": "bf16" }, "19": { "gate": "bf16", "up": "bf16", "down": "bf16" }, "20": { "gate": "bf16", "up": "bf16", "down": "bf16" }, "21": { "gate": "bf16", "up": "bf16", "down": "bf16" }, "22": { "gate": "bf16", "up": "bf16", "down": "bf16" }, "23": { "gate": "bf16", "up": "bf16", "down": "bf16" }, "24": { "gate": "bf16", "up": "bf16", "down": "bf16" }, "25": { "gate": "bf16", "up": "bf16", "down": "bf16" }, "26": { "gate": "bf16", "up": "bf16", "down": "bf16" }, "27": { "gate": "bf16", "up": "bf16", "down": "bf16" }, "28": { "gate": "bf16", "up": "bf16", "down": "bf16" }, "29": { "gate": "bf16", "up": "bf16", "down": "bf16" }, "30": { "gate": "bf16", "up": "bf16", "down": "bf16" }, "31": { "gate": "bf16", "up": "bf16", "down": "bf16" }, "32": { "gate": "bf16", "up": "bf16", "down": "bf16" }, "33": { "gate": "bf16", "up": "bf16", "down": "bf16" }, "34": { "gate": "bf16", "up": "bf16", "down": "bf16" }, "35": { "gate": "bf16", "up": "bf16", "down": "bf16" }, "36": { "gate": "bf16", "up": "bf16", "down": "bf16" }, "37": { "gate": "bf16", "up": "bf16", "down": "bf16" }, "38": { "gate": "bf16", "up": "bf16", "down": "bf16" }, "39": { "gate": "bf16", "up": "bf16", "down": "bf16" }, "40": { "gate": "bf16", "up": "bf16", "down": "bf16" }, "41": { "gate": "bf16", "up": "bf16", "down": "bf16" }, "42": { "gate": "bf16", "up": "bf16", "down": "bf16" }, "43": { "gate": "bf16", "up": "bf16", "down": "bf16" }, "44": { "gate": "bf16", "up": "bf16", "down": "bf16" }, "45": { "gate": "bf16", "up": "bf16", "down": "bf16" }, "46": { "gate": "bf16", "up": "bf16", "down": "bf16" }, "47": { "gate": "bf16", "up": "bf16", "down": "bf16" }, "48": { "gate": "bf16", "up": "bf16", "down": "bf16" } }, "tensor_bytes": 79570574912, "aggregate": { "all": { "tokens": 49140, "nll": 2.214729754197971, "teacher_nll": 2.214729754197971, "kl": 0.0, "top1_agree": 1.0, "perplexity": 9.158933613767237, "teacher_perplexity": 9.158933613767237, "delta_nll": 0.0 }, "code": { "tokens": 16380, "nll": 1.585819314280127, "teacher_nll": 1.585819314280127, "kl": 0.0, "top1_agree": 1.0, "perplexity": 4.883290689738726, "teacher_perplexity": 4.883290689738726, "delta_nll": 0.0 }, "math": { "tokens": 16380, "nll": 1.5066873193514945, "teacher_nll": 1.5066873193514945, "kl": 0.0, "top1_agree": 1.0, "perplexity": 4.511759991246355, "teacher_perplexity": 4.511759991246355, "delta_nll": 0.0 }, "prose": { "tokens": 16380, "nll": 3.551682628962292, "teacher_nll": 3.551682628962292, "kl": 0.0, "top1_agree": 1.0, "perplexity": 34.871944693965524, "teacher_perplexity": 34.871944693965524, "delta_nll": 0.0 } }, "sequences": [ { "id": "prose-qualification-holdout-0", "domain": "prose", "tokens": 4095, "nll": 3.570623514736674, "teacher_nll": 3.570623514736674, "kl": 0.0, "top1_agree": 1.0 }, { "id": "prose-qualification-holdout-1", "domain": "prose", "tokens": 4095, "nll": 3.4886688269683876, "teacher_nll": 3.4886688269683876, "kl": 0.0, "top1_agree": 1.0 }, { "id": "prose-qualification-holdout-2", "domain": "prose", "tokens": 4095, "nll": 3.586854711586157, "teacher_nll": 3.586854711586157, "kl": 0.0, "top1_agree": 1.0 }, { "id": "prose-qualification-holdout-3", "domain": "prose", "tokens": 4095, "nll": 3.56058346255795, "teacher_nll": 3.56058346255795, "kl": 0.0, "top1_agree": 1.0 }, { "id": "code-qualification-holdout-0", "domain": "code", "tokens": 4095, "nll": 1.612040780315469, "teacher_nll": 1.612040780315469, "kl": 0.0, "top1_agree": 1.0 }, { "id": "code-qualification-holdout-1", "domain": "code", "tokens": 4095, "nll": 1.4962344744848826, "teacher_nll": 1.4962344744848826, "kl": 0.0, "top1_agree": 1.0 }, { "id": "code-qualification-holdout-2", "domain": "code", "tokens": 4095, "nll": 1.4386393615759918, "teacher_nll": 1.4386393615759918, "kl": 0.0, "top1_agree": 1.0 }, { "id": "code-qualification-holdout-3", "domain": "code", "tokens": 4095, "nll": 1.796362640744164, "teacher_nll": 1.796362640744164, "kl": 0.0, "top1_agree": 1.0 }, { "id": "math-qualification-holdout-0", "domain": "math", "tokens": 4095, "nll": 1.4128543704711765, "teacher_nll": 1.4128543704711765, "kl": 0.0, "top1_agree": 1.0 }, { "id": "math-qualification-holdout-1", "domain": "math", "tokens": 4095, "nll": 1.545063196724879, "teacher_nll": 1.545063196724879, "kl": 0.0, "top1_agree": 1.0 }, { "id": "math-qualification-holdout-2", "domain": "math", "tokens": 4095, "nll": 1.5522358872223831, "teacher_nll": 1.5522358872223831, "kl": 0.0, "top1_agree": 1.0 }, { "id": "math-qualification-holdout-3", "domain": "math", "tokens": 4095, "nll": 1.516595822987539, "teacher_nll": 1.516595822987539, "kl": 0.0, "top1_agree": 1.0 } ], "memory": { "swap_kib": 0, "cuda_peak_bytes": 14954638336 } }, "all-fp8": { "precision_by_layer": { "2": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "3": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "4": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "5": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "6": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "7": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "8": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "9": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "10": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "11": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "12": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "13": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "14": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "15": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "16": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "17": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "18": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "19": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "20": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "21": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "22": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "23": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "24": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "25": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "26": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "27": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "28": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "29": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "30": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "31": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "32": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "33": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "34": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "35": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "36": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "37": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "38": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "39": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "40": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "41": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "42": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "43": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "44": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "45": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "46": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "47": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "48": { "gate": "fp8", "up": "fp8", "down": "fp8" } }, "tensor_bytes": 41730416192, "aggregate": { "all": { "tokens": 49140, "nll": 2.21773667151141, "teacher_nll": 2.214729754197971, "kl": 0.07243157411187487, "top1_agree": 0.9358974358974359, "perplexity": 9.186515216841316, "teacher_perplexity": 9.158933613767237, "delta_nll": 0.0030069173134390503 }, "code": { "tokens": 16380, "nll": 1.6008225211002596, "teacher_nll": 1.585819314280127, "kl": 0.09259340757861169, "top1_agree": 0.9469474969474969, "perplexity": 4.957108073998569, "teacher_perplexity": 4.883290689738726, "delta_nll": 0.015003206820132675 }, "math": { "tokens": 16380, "nll": 1.5014515739948613, "teacher_nll": 1.5066873193514945, "kl": 0.021969766698331915, "top1_agree": 0.9597069597069597, "perplexity": 4.488199297540752, "teacher_perplexity": 4.511759991246355, "delta_nll": -0.005235745356633181 }, "prose": { "tokens": 16380, "nll": 3.5509359194391084, "teacher_nll": 3.551682628962292, "kl": 0.102731548058681, "top1_agree": 0.9010378510378511, "perplexity": 34.84591520021549, "teacher_perplexity": 34.871944693965524, "delta_nll": -0.0007467095231836751 } }, "sequences": [ { "id": "prose-qualification-holdout-0", "domain": "prose", "tokens": 4095, "nll": 3.5883770301082922, "teacher_nll": 3.570623514736674, "kl": 0.1001550619127695, "top1_agree": 0.8998778998778999 }, { "id": "prose-qualification-holdout-1", "domain": "prose", "tokens": 4095, "nll": 3.471363538059705, "teacher_nll": 3.4886688269683876, "kl": 0.1251851178787567, "top1_agree": 0.8981684981684982 }, { "id": "prose-qualification-holdout-2", "domain": "prose", "tokens": 4095, "nll": 3.5789717938582686, "teacher_nll": 3.586854711586157, "kl": 0.08583091795808376, "top1_agree": 0.9091575091575091 }, { "id": "prose-qualification-holdout-3", "domain": "prose", "tokens": 4095, "nll": 3.565031315730168, "teacher_nll": 3.56058346255795, "kl": 0.09975509448511405, "top1_agree": 0.896947496947497 }, { "id": "code-qualification-holdout-0", "domain": "code", "tokens": 4095, "nll": 1.6328039013276898, "teacher_nll": 1.612040780315469, "kl": 0.10738086557795859, "top1_agree": 0.9391941391941392 }, { "id": "code-qualification-holdout-1", "domain": "code", "tokens": 4095, "nll": 1.5051212597242642, "teacher_nll": 1.4962344744848826, "kl": 0.09163388843683387, "top1_agree": 0.9538461538461539 }, { "id": "code-qualification-holdout-2", "domain": "code", "tokens": 4095, "nll": 1.4643761817643348, "teacher_nll": 1.4386393615759918, "kl": 0.07029858778102348, "top1_agree": 0.9528693528693528 }, { "id": "code-qualification-holdout-3", "domain": "code", "tokens": 4095, "nll": 1.8009887415847499, "teacher_nll": 1.796362640744164, "kl": 0.10106028851863083, "top1_agree": 0.9418803418803419 }, { "id": "math-qualification-holdout-0", "domain": "math", "tokens": 4095, "nll": 1.405975469652113, "teacher_nll": 1.4128543704711765, "kl": 0.023144839301941885, "top1_agree": 0.957997557997558 }, { "id": "math-qualification-holdout-1", "domain": "math", "tokens": 4095, "nll": 1.5458229023021657, "teacher_nll": 1.545063196724879, "kl": 0.020068688932647055, "top1_agree": 0.95995115995116 }, { "id": "math-qualification-holdout-2", "domain": "math", "tokens": 4095, "nll": 1.5160463214357258, "teacher_nll": 1.5522358872223831, "kl": 0.02173219574240101, "top1_agree": 0.9631257631257631 }, { "id": "math-qualification-holdout-3", "domain": "math", "tokens": 4095, "nll": 1.5379616025894407, "teacher_nll": 1.516595822987539, "kl": 0.022933342816337708, "top1_agree": 0.9577533577533578 } ], "memory": { "swap_kib": 0, "cuda_peak_bytes": 14954638336 } }, "all-nvfp4": { "precision_by_layer": { "2": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "3": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "4": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "5": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "6": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "7": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "8": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "9": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "10": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "11": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "12": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "13": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "14": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "15": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "16": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "17": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "18": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "19": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "20": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "21": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "22": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "23": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "24": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "25": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "26": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "27": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "28": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "29": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "30": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "31": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "32": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "33": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "34": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "35": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "36": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "37": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "38": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "39": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "40": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "41": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "42": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "43": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "44": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "45": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "46": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "47": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "48": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" } }, "tensor_bytes": 25162207808, "aggregate": { "all": { "tokens": 49140, "nll": 2.2190047702218734, "teacher_nll": 2.214729754197971, "kl": 0.15325156021552402, "top1_agree": 0.8925315425315425, "perplexity": 9.198172014364316, "teacher_perplexity": 9.158933613767237, "delta_nll": 0.004275016023902456 }, "code": { "tokens": 16380, "nll": 1.5995975865112557, "teacher_nll": 1.585819314280127, "kl": 0.16933766659508404, "top1_agree": 0.9105616605616605, "perplexity": 4.951039658322216, "teacher_perplexity": 4.883290689738726, "delta_nll": 0.013778272231128819 }, "math": { "tokens": 16380, "nll": 1.5143714170461755, "teacher_nll": 1.5066873193514945, "kl": 0.06676194644345469, "top1_agree": 0.9306471306471307, "perplexity": 4.546562336862997, "teacher_perplexity": 4.511759991246355, "delta_nll": 0.007684097694681036 }, "prose": { "tokens": 16380, "nll": 3.5430453071081884, "teacher_nll": 3.551682628962292, "kl": 0.22365506760803333, "top1_agree": 0.8363858363858364, "perplexity": 34.57204152852661, "teacher_perplexity": 34.871944693965524, "delta_nll": -0.008637321854103597 } }, "sequences": [ { "id": "prose-qualification-holdout-0", "domain": "prose", "tokens": 4095, "nll": 3.587969711732224, "teacher_nll": 3.570623514736674, "kl": 0.22323825251634014, "top1_agree": 0.842002442002442 }, { "id": "prose-qualification-holdout-1", "domain": "prose", "tokens": 4095, "nll": 3.4460585225050555, "teacher_nll": 3.4886688269683876, "kl": 0.22454096511170105, "top1_agree": 0.8368742368742369 }, { "id": "prose-qualification-holdout-2", "domain": "prose", "tokens": 4095, "nll": 3.5779974633520775, "teacher_nll": 3.586854711586157, "kl": 0.22193228753058464, "top1_agree": 0.8356532356532357 }, { "id": "prose-qualification-holdout-3", "domain": "prose", "tokens": 4095, "nll": 3.560155530843397, "teacher_nll": 3.56058346255795, "kl": 0.22490876527350756, "top1_agree": 0.8310134310134311 }, { "id": "code-qualification-holdout-0", "domain": "code", "tokens": 4095, "nll": 1.6565769824352894, "teacher_nll": 1.612040780315469, "kl": 0.19423157339131003, "top1_agree": 0.9030525030525031 }, { "id": "code-qualification-holdout-1", "domain": "code", "tokens": 4095, "nll": 1.4670974128848904, "teacher_nll": 1.4962344744848826, "kl": 0.15822774912702586, "top1_agree": 0.9208791208791208 }, { "id": "code-qualification-holdout-2", "domain": "code", "tokens": 4095, "nll": 1.4386256086375104, "teacher_nll": 1.4386393615759918, "kl": 0.13105918493608562, "top1_agree": 0.9111111111111111 }, { "id": "code-qualification-holdout-3", "domain": "code", "tokens": 4095, "nll": 1.8360903420873322, "teacher_nll": 1.796362640744164, "kl": 0.19383215892591465, "top1_agree": 0.9072039072039072 }, { "id": "math-qualification-holdout-0", "domain": "math", "tokens": 4095, "nll": 1.403938700486161, "teacher_nll": 1.4128543704711765, "kl": 0.06598033006229098, "top1_agree": 0.9296703296703297 }, { "id": "math-qualification-holdout-1", "domain": "math", "tokens": 4095, "nll": 1.5686517051724724, "teacher_nll": 1.545063196724879, "kl": 0.07200618306213537, "top1_agree": 0.9252747252747253 }, { "id": "math-qualification-holdout-2", "domain": "math", "tokens": 4095, "nll": 1.5351991115472257, "teacher_nll": 1.5522358872223831, "kl": 0.06542188856336806, "top1_agree": 0.9328449328449329 }, { "id": "math-qualification-holdout-3", "domain": "math", "tokens": 4095, "nll": 1.549696150978843, "teacher_nll": 1.516595822987539, "kl": 0.06363938408602433, "top1_agree": 0.9347985347985348 } ], "memory": { "swap_kib": 0, "cuda_peak_bytes": 14954638336 } }, "published-mix": { "precision_by_layer": { "2": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "3": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "4": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "5": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "6": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "7": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "8": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "9": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "10": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "11": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "12": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "13": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "14": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "15": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "16": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "17": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "18": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "19": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "20": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "21": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "22": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "23": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "24": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "25": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "26": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "27": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "28": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "29": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "30": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "31": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "32": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "33": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "34": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "35": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "36": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "37": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "38": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "39": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "40": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "41": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "42": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "43": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "44": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "45": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "46": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "47": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "48": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" } }, "tensor_bytes": 30802448960, "aggregate": { "all": { "tokens": 49140, "nll": 2.2032232530436286, "teacher_nll": 2.214729754197971, "kl": 0.11295873393427079, "top1_agree": 0.9065730565730565, "perplexity": 9.054150334134441, "teacher_perplexity": 9.158933613767237, "delta_nll": -0.011506501154342264 }, "code": { "tokens": 16380, "nll": 1.564784675727397, "teacher_nll": 1.585819314280127, "kl": 0.13348741904174016, "top1_agree": 0.9209401709401709, "perplexity": 4.781645220758911, "teacher_perplexity": 4.883290689738726, "delta_nll": -0.0210346385527298 }, "math": { "tokens": 16380, "nll": 1.5041985815406864, "teacher_nll": 1.5066873193514945, "kl": 0.05023271923700532, "top1_agree": 0.9394993894993895, "perplexity": 4.500545364484122, "teacher_perplexity": 4.511759991246355, "delta_nll": -0.0024887378108080505 }, "prose": { "tokens": 16380, "nll": 3.540686501862802, "teacher_nll": 3.551682628962292, "kl": 0.15515606352406688, "top1_agree": 0.8592796092796092, "perplexity": 34.49058891881352, "teacher_perplexity": 34.871944693965524, "delta_nll": -0.01099612709949005 } }, "sequences": [ { "id": "prose-qualification-holdout-0", "domain": "prose", "tokens": 4095, "nll": 3.5968923231038707, "teacher_nll": 3.570623514736674, "kl": 0.14627862587953225, "top1_agree": 0.8632478632478633 }, { "id": "prose-qualification-holdout-1", "domain": "prose", "tokens": 4095, "nll": 3.4596483652117196, "teacher_nll": 3.4886688269683876, "kl": 0.17513312973236717, "top1_agree": 0.8542124542124542 }, { "id": "prose-qualification-holdout-2", "domain": "prose", "tokens": 4095, "nll": 3.5596329014816566, "teacher_nll": 3.586854711586157, "kl": 0.13868644857581283, "top1_agree": 0.8634920634920635 }, { "id": "prose-qualification-holdout-3", "domain": "prose", "tokens": 4095, "nll": 3.5465724176539606, "teacher_nll": 3.56058346255795, "kl": 0.16052604990855532, "top1_agree": 0.8561660561660561 }, { "id": "code-qualification-holdout-0", "domain": "code", "tokens": 4095, "nll": 1.6175182463630797, "teacher_nll": 1.612040780315469, "kl": 0.14147204016591167, "top1_agree": 0.9177045177045177 }, { "id": "code-qualification-holdout-1", "domain": "code", "tokens": 4095, "nll": 1.438487123598837, "teacher_nll": 1.4962344744848826, "kl": 0.13281468772335076, "top1_agree": 0.9274725274725275 }, { "id": "code-qualification-holdout-2", "domain": "code", "tokens": 4095, "nll": 1.4048096974690756, "teacher_nll": 1.4386393615759918, "kl": 0.11024958886506357, "top1_agree": 0.9250305250305251 }, { "id": "code-qualification-holdout-3", "domain": "code", "tokens": 4095, "nll": 1.798323635478596, "teacher_nll": 1.796362640744164, "kl": 0.14941335941263462, "top1_agree": 0.9135531135531135 }, { "id": "math-qualification-holdout-0", "domain": "math", "tokens": 4095, "nll": 1.3987592498024741, "teacher_nll": 1.4128543704711765, "kl": 0.05078120037676796, "top1_agree": 0.9438339438339438 }, { "id": "math-qualification-holdout-1", "domain": "math", "tokens": 4095, "nll": 1.5469760289267889, "teacher_nll": 1.545063196724879, "kl": 0.05077969653120262, "top1_agree": 0.9347985347985348 }, { "id": "math-qualification-holdout-2", "domain": "math", "tokens": 4095, "nll": 1.5395388960692764, "teacher_nll": 1.5522358872223831, "kl": 0.051088059443402785, "top1_agree": 0.9384615384615385 }, { "id": "math-qualification-holdout-3", "domain": "math", "tokens": 4095, "nll": 1.5315201513642065, "teacher_nll": 1.516595822987539, "kl": 0.048281920596647904, "top1_agree": 0.9409035409035409 } ], "memory": { "swap_kib": 0, "cuda_peak_bytes": 14954638336 } }, "budget28-swap1": { "precision_by_layer": { "2": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "3": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "4": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "5": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "6": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "7": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "8": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "9": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "10": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "11": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "12": { "gate": "fp8", "up": "nvfp4", "down": "fp8" }, "13": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "14": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "15": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "16": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "17": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "18": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "19": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "20": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "21": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "22": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "23": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "24": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "25": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "26": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "27": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "28": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "29": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "30": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "31": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "32": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "33": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "34": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "35": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "36": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "37": { "gate": "nvfp4", "up": "nvfp4", "down": "fp8" }, "38": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "39": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "40": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "41": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "42": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "43": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "44": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "45": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "46": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "47": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "48": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" } }, "tensor_bytes": 27982328384, "aggregate": { "all": { "tokens": 49140, "nll": 2.2250885697923932, "teacher_nll": 2.214729754197971, "kl": 0.1387744848271866, "top1_agree": 0.8979039479039479, "perplexity": 9.254302419253793, "teacher_perplexity": 9.158933613767237, "delta_nll": 0.01035881559442231 }, "code": { "tokens": 16380, "nll": 1.6103557565418938, "teacher_nll": 1.585819314280127, "kl": 0.1575198637551295, "top1_agree": 0.9144078144078144, "perplexity": 5.004591327277986, "teacher_perplexity": 4.883290689738726, "delta_nll": 0.024536442261766878 }, "math": { "tokens": 16380, "nll": 1.515096839465501, "teacher_nll": 1.5066873193514945, "kl": 0.060545078656830634, "top1_agree": 0.9348595848595849, "perplexity": 4.549861711688565, "teacher_perplexity": 4.511759991246355, "delta_nll": 0.008409520114006641 }, "prose": { "tokens": 16380, "nll": 3.5498131133697846, "teacher_nll": 3.551682628962292, "kl": 0.19825851206959968, "top1_agree": 0.8444444444444444, "perplexity": 34.8068119519279, "teacher_perplexity": 34.871944693965524, "delta_nll": -0.0018695155925074758 } }, "sequences": [ { "id": "prose-qualification-holdout-0", "domain": "prose", "tokens": 4095, "nll": 3.5897965204148066, "teacher_nll": 3.570623514736674, "kl": 0.18998560713324356, "top1_agree": 0.8451770451770452 }, { "id": "prose-qualification-holdout-1", "domain": "prose", "tokens": 4095, "nll": 3.454184816986941, "teacher_nll": 3.4886688269683876, "kl": 0.21330270371303162, "top1_agree": 0.8378510378510379 }, { "id": "prose-qualification-holdout-2", "domain": "prose", "tokens": 4095, "nll": 3.587775377359728, "teacher_nll": 3.586854711586157, "kl": 0.1919633207449255, "top1_agree": 0.8515262515262515 }, { "id": "prose-qualification-holdout-3", "domain": "prose", "tokens": 4095, "nll": 3.5674957387176627, "teacher_nll": 3.56058346255795, "kl": 0.19778241668719804, "top1_agree": 0.8432234432234432 }, { "id": "code-qualification-holdout-0", "domain": "code", "tokens": 4095, "nll": 1.659486316033481, "teacher_nll": 1.612040780315469, "kl": 0.18177173958593235, "top1_agree": 0.9081807081807082 }, { "id": "code-qualification-holdout-1", "domain": "code", "tokens": 4095, "nll": 1.5020296731420197, "teacher_nll": 1.4962344744848826, "kl": 0.15101925459245769, "top1_agree": 0.9218559218559218 }, { "id": "code-qualification-holdout-2", "domain": "code", "tokens": 4095, "nll": 1.4350260393377916, "teacher_nll": 1.4386393615759918, "kl": 0.1277437409347376, "top1_agree": 0.9186813186813186 }, { "id": "code-qualification-holdout-3", "domain": "code", "tokens": 4095, "nll": 1.8448809976542826, "teacher_nll": 1.796362640744164, "kl": 0.16954471990739034, "top1_agree": 0.9089133089133089 }, { "id": "math-qualification-holdout-0", "domain": "math", "tokens": 4095, "nll": 1.4089843853343829, "teacher_nll": 1.4128543704711765, "kl": 0.06082202776884421, "top1_agree": 0.9347985347985348 }, { "id": "math-qualification-holdout-1", "domain": "math", "tokens": 4095, "nll": 1.5700943328522063, "teacher_nll": 1.545063196724879, "kl": 0.06191048036273728, "top1_agree": 0.9291819291819292 }, { "id": "math-qualification-holdout-2", "domain": "math", "tokens": 4095, "nll": 1.5385928818885515, "teacher_nll": 1.5522358872223831, "kl": 0.06146017515324557, "top1_agree": 0.938949938949939 }, { "id": "math-qualification-holdout-3", "domain": "math", "tokens": 4095, "nll": 1.5427157577868638, "teacher_nll": 1.516595822987539, "kl": 0.05798763134249546, "top1_agree": 0.9365079365079365 } ], "memory": { "swap_kib": 0, "cuda_peak_bytes": 14954638336 } }, "budget32-conditional": { "precision_by_layer": { "2": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "3": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "4": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "5": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "6": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "7": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "8": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "9": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "10": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "11": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "12": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "13": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "14": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "15": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "16": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "17": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "18": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "19": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "20": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "21": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "22": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "23": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "24": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "25": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "26": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "27": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "28": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "29": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "30": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "31": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "32": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "33": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "34": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "35": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "36": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "37": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "38": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "39": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "40": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "41": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "42": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "43": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "44": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "45": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "46": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "47": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "48": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" } }, "tensor_bytes": 31859994176, "aggregate": { "all": { "tokens": 49140, "nll": 2.210038530593681, "teacher_nll": 2.214729754197971, "kl": 0.11078792841355273, "top1_agree": 0.908974358974359, "perplexity": 9.11606763377147, "teacher_perplexity": 9.158933613767237, "delta_nll": -0.0046912236042899735 }, "code": { "tokens": 16380, "nll": 1.578945990272494, "teacher_nll": 1.585819314280127, "kl": 0.1256440419321641, "top1_agree": 0.9242979242979243, "perplexity": 4.849841336412374, "teacher_perplexity": 4.883290689738726, "delta_nll": -0.006873324007632853 }, "math": { "tokens": 16380, "nll": 1.5118221361704072, "teacher_nll": 1.5066873193514945, "kl": 0.049465113265101, "top1_agree": 0.9401709401709402, "perplexity": 4.534986633575697, "teacher_perplexity": 4.511759991246355, "delta_nll": 0.005134816818912702 }, "prose": { "tokens": 16380, "nll": 3.539347465338142, "teacher_nll": 3.551682628962292, "kl": 0.1572546300433931, "top1_agree": 0.8624542124542125, "perplexity": 34.44443566783473, "teacher_perplexity": 34.871944693965524, "delta_nll": -0.012335163624149992 } }, "sequences": [ { "id": "prose-qualification-holdout-0", "domain": "prose", "tokens": 4095, "nll": 3.6023279863108355, "teacher_nll": 3.570623514736674, "kl": 0.147485230228112, "top1_agree": 0.8656898656898657 }, { "id": "prose-qualification-holdout-1", "domain": "prose", "tokens": 4095, "nll": 3.46047611446171, "teacher_nll": 3.4886688269683876, "kl": 0.1644654834081256, "top1_agree": 0.8598290598290599 }, { "id": "prose-qualification-holdout-2", "domain": "prose", "tokens": 4095, "nll": 3.5554149548503795, "teacher_nll": 3.586854711586157, "kl": 0.15823070167476294, "top1_agree": 0.864957264957265 }, { "id": "prose-qualification-holdout-3", "domain": "prose", "tokens": 4095, "nll": 3.539170805729644, "teacher_nll": 3.56058346255795, "kl": 0.15883710486257177, "top1_agree": 0.8593406593406593 }, { "id": "code-qualification-holdout-0", "domain": "code", "tokens": 4095, "nll": 1.6173301612937843, "teacher_nll": 1.612040780315469, "kl": 0.14581474144380172, "top1_agree": 0.9194139194139194 }, { "id": "code-qualification-holdout-1", "domain": "code", "tokens": 4095, "nll": 1.4841609367957482, "teacher_nll": 1.4962344744848826, "kl": 0.12464202101523186, "top1_agree": 0.9326007326007326 }, { "id": "code-qualification-holdout-2", "domain": "code", "tokens": 4095, "nll": 1.4110373812571841, "teacher_nll": 1.4386393615759918, "kl": 0.09008101636819059, "top1_agree": 0.926007326007326 }, { "id": "code-qualification-holdout-3", "domain": "code", "tokens": 4095, "nll": 1.8032554817432596, "teacher_nll": 1.796362640744164, "kl": 0.14203838890143222, "top1_agree": 0.9191697191697191 }, { "id": "math-qualification-holdout-0", "domain": "math", "tokens": 4095, "nll": 1.4158305093045636, "teacher_nll": 1.4128543704711765, "kl": 0.04934141346240946, "top1_agree": 0.9409035409035409 }, { "id": "math-qualification-holdout-1", "domain": "math", "tokens": 4095, "nll": 1.54967892033131, "teacher_nll": 1.545063196724879, "kl": 0.04675215886626052, "top1_agree": 0.9374847374847375 }, { "id": "math-qualification-holdout-2", "domain": "math", "tokens": 4095, "nll": 1.5434759258787274, "teacher_nll": 1.5522358872223831, "kl": 0.0511482006136751, "top1_agree": 0.9416361416361416 }, { "id": "math-qualification-holdout-3", "domain": "math", "tokens": 4095, "nll": 1.5383031891670274, "teacher_nll": 1.516595822987539, "kl": 0.050618680118058916, "top1_agree": 0.9406593406593406 } ], "memory": { "swap_kib": 0, "cuda_peak_bytes": 14954638336 } }, "budget36-swap1": { "precision_by_layer": { "2": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "3": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "4": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "5": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "6": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "7": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "8": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "9": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "10": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "11": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "12": { "gate": "fp8", "up": "nvfp4", "down": "fp8" }, "13": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "14": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "15": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "16": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "17": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "18": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "19": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "20": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "21": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "22": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "23": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "24": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "25": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "26": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "27": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "28": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "29": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "30": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "31": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "32": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "33": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "34": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "35": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "36": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "37": { "gate": "nvfp4", "up": "nvfp4", "down": "fp8" }, "38": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "39": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "40": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "41": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "42": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "43": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "44": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "45": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "46": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "47": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "48": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" } }, "tensor_bytes": 35737659968, "aggregate": { "all": { "tokens": 49140, "nll": 2.219416871241161, "teacher_nll": 2.214729754197971, "kl": 0.09010865478740386, "top1_agree": 0.9188848188848189, "perplexity": 9.201963371584448, "teacher_perplexity": 9.158933613767237, "delta_nll": 0.0046871170431899145 }, "code": { "tokens": 16380, "nll": 1.5947170897863432, "teacher_nll": 1.585819314280127, "kl": 0.10190384458427537, "top1_agree": 0.9322344322344323, "perplexity": 4.92693499469703, "teacher_perplexity": 4.883290689738726, "delta_nll": 0.008897775506216288 }, "math": { "tokens": 16380, "nll": 1.5039469321350476, "teacher_nll": 1.5066873193514945, "kl": 0.03564856964480746, "top1_agree": 0.9473748473748473, "perplexity": 4.499412947410116, "teacher_perplexity": 4.511759991246355, "delta_nll": -0.0027403872164468712 }, "prose": { "tokens": 16380, "nll": 3.5595865918020917, "teacher_nll": 3.551682628962292, "kl": 0.13277355013312875, "top1_agree": 0.877045177045177, "perplexity": 35.14866339553476, "teacher_perplexity": 34.871944693965524, "delta_nll": 0.00790396283979966 } }, "sequences": [ { "id": "prose-qualification-holdout-0", "domain": "prose", "tokens": 4095, "nll": 3.60210935525114, "teacher_nll": 3.570623514736674, "kl": 0.1251914977445125, "top1_agree": 0.873992673992674 }, { "id": "prose-qualification-holdout-1", "domain": "prose", "tokens": 4095, "nll": 3.4887589190323567, "teacher_nll": 3.4886688269683876, "kl": 0.1547056760543432, "top1_agree": 0.8717948717948718 }, { "id": "prose-qualification-holdout-2", "domain": "prose", "tokens": 4095, "nll": 3.587463203774849, "teacher_nll": 3.586854711586157, "kl": 0.11346375461899753, "top1_agree": 0.8842490842490842 }, { "id": "prose-qualification-holdout-3", "domain": "prose", "tokens": 4095, "nll": 3.560014889150021, "teacher_nll": 3.56058346255795, "kl": 0.13773327211466174, "top1_agree": 0.8781440781440781 }, { "id": "code-qualification-holdout-0", "domain": "code", "tokens": 4095, "nll": 1.634640592358488, "teacher_nll": 1.612040780315469, "kl": 0.11310800134917319, "top1_agree": 0.9255189255189256 }, { "id": "code-qualification-holdout-1", "domain": "code", "tokens": 4095, "nll": 1.4733869531070212, "teacher_nll": 1.4962344744848826, "kl": 0.10195107398343174, "top1_agree": 0.9423687423687424 }, { "id": "code-qualification-holdout-2", "domain": "code", "tokens": 4095, "nll": 1.4622409024081386, "teacher_nll": 1.4386393615759918, "kl": 0.0776867737409105, "top1_agree": 0.9374847374847375 }, { "id": "code-qualification-holdout-3", "domain": "code", "tokens": 4095, "nll": 1.8085999112717253, "teacher_nll": 1.796362640744164, "kl": 0.11486952926358605, "top1_agree": 0.9235653235653236 }, { "id": "math-qualification-holdout-0", "domain": "math", "tokens": 4095, "nll": 1.4042997626012352, "teacher_nll": 1.4128543704711765, "kl": 0.03463405605419215, "top1_agree": 0.9472527472527472 }, { "id": "math-qualification-holdout-1", "domain": "math", "tokens": 4095, "nll": 1.5480228545173766, "teacher_nll": 1.545063196724879, "kl": 0.03576130380123963, "top1_agree": 0.9457875457875458 }, { "id": "math-qualification-holdout-2", "domain": "math", "tokens": 4095, "nll": 1.5291209340823293, "teacher_nll": 1.5522358872223831, "kl": 0.03457320837965815, "top1_agree": 0.9474969474969475 }, { "id": "math-qualification-holdout-3", "domain": "math", "tokens": 4095, "nll": 1.5343441773392488, "teacher_nll": 1.516595822987539, "kl": 0.03762571034413991, "top1_agree": 0.9489621489621489 } ], "memory": { "swap_kib": 0, "cuda_peak_bytes": 14954638336 } }, "budget40-swap5": { "precision_by_layer": { "2": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "3": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "4": { "gate": "fp8", "up": "fp8", "down": "nvfp4" }, "5": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "6": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "7": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "8": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "9": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "10": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "11": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "12": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "13": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "14": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "15": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "16": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "17": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "18": { "gate": "fp8", "up": "nvfp4", "down": "fp8" }, "19": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "20": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "21": { "gate": "nvfp4", "up": "fp8", "down": "fp8" }, "22": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "23": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "24": { "gate": "nvfp4", "up": "fp8", "down": "nvfp4" }, "25": { "gate": "nvfp4", "up": "nvfp4", "down": "fp8" }, "26": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "27": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "28": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "29": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "30": { "gate": "nvfp4", "up": "fp8", "down": "nvfp4" }, "31": { "gate": "fp8", "up": "nvfp4", "down": "fp8" }, "32": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "33": { "gate": "nvfp4", "up": "fp8", "down": "fp8" }, "34": { "gate": "fp8", "up": "nvfp4", "down": "fp8" }, "35": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "36": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "37": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "38": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "39": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "40": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "41": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "42": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "43": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "44": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "45": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "46": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "47": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "48": { "gate": "fp8", "up": "fp8", "down": "fp8" } }, "tensor_bytes": 39967840832, "aggregate": { "all": { "tokens": 49140, "nll": 2.2186658260683534, "teacher_nll": 2.214729754197971, "kl": 0.07798120706536613, "top1_agree": 0.9294261294261295, "perplexity": 9.195054876034725, "teacher_perplexity": 9.158933613767237, "delta_nll": 0.003936071870382474 }, "code": { "tokens": 16380, "nll": 1.5962667831571111, "teacher_nll": 1.585819314280127, "kl": 0.09327435911211193, "top1_agree": 0.9406593406593406, "perplexity": 4.934576152393011, "teacher_perplexity": 4.883290689738726, "delta_nll": 0.010447468876984223 }, "math": { "tokens": 16380, "nll": 1.5076653056965643, "teacher_nll": 1.5066873193514945, "kl": 0.027212030440114065, "top1_agree": 0.9548840048840049, "perplexity": 4.516174589266453, "teacher_perplexity": 4.511759991246355, "delta_nll": 0.0009779863450698478 }, "prose": { "tokens": 16380, "nll": 3.5520653893513847, "teacher_nll": 3.551682628962292, "kl": 0.11345723164387238, "top1_agree": 0.8927350427350428, "perplexity": 34.88529484787707, "teacher_perplexity": 34.871944693965524, "delta_nll": 0.0003827603890926845 } }, "sequences": [ { "id": "prose-qualification-holdout-0", "domain": "prose", "tokens": 4095, "nll": 3.599587125859709, "teacher_nll": 3.570623514736674, "kl": 0.11613119326813899, "top1_agree": 0.8925518925518926 }, { "id": "prose-qualification-holdout-1", "domain": "prose", "tokens": 4095, "nll": 3.4782555157448347, "teacher_nll": 3.4886688269683876, "kl": 0.1251666544412373, "top1_agree": 0.8927960927960928 }, { "id": "prose-qualification-holdout-2", "domain": "prose", "tokens": 4095, "nll": 3.572577172583276, "teacher_nll": 3.586854711586157, "kl": 0.1002090939672002, "top1_agree": 0.897924297924298 }, { "id": "prose-qualification-holdout-3", "domain": "prose", "tokens": 4095, "nll": 3.5578417432177196, "teacher_nll": 3.56058346255795, "kl": 0.11232198489891304, "top1_agree": 0.8876678876678876 }, { "id": "code-qualification-holdout-0", "domain": "code", "tokens": 4095, "nll": 1.6341117285983466, "teacher_nll": 1.612040780315469, "kl": 0.10246821173235901, "top1_agree": 0.9374847374847375 }, { "id": "code-qualification-holdout-1", "domain": "code", "tokens": 4095, "nll": 1.4997853697729053, "teacher_nll": 1.4962344744848826, "kl": 0.09997353378378573, "top1_agree": 0.9474969474969475 }, { "id": "code-qualification-holdout-2", "domain": "code", "tokens": 4095, "nll": 1.4425236144199767, "teacher_nll": 1.4386393615759918, "kl": 0.06728265899732727, "top1_agree": 0.9433455433455433 }, { "id": "code-qualification-holdout-3", "domain": "code", "tokens": 4095, "nll": 1.8086464198372156, "teacher_nll": 1.796362640744164, "kl": 0.1033730319349757, "top1_agree": 0.9343101343101343 }, { "id": "math-qualification-holdout-0", "domain": "math", "tokens": 4095, "nll": 1.4159224659240872, "teacher_nll": 1.4128543704711765, "kl": 0.02953591391498789, "top1_agree": 0.9543345543345544 }, { "id": "math-qualification-holdout-1", "domain": "math", "tokens": 4095, "nll": 1.5524857913472567, "teacher_nll": 1.545063196724879, "kl": 0.026335953079591596, "top1_agree": 0.9545787545787546 }, { "id": "math-qualification-holdout-2", "domain": "math", "tokens": 4095, "nll": 1.5277823465647715, "teacher_nll": 1.5522358872223831, "kl": 0.02601443173247816, "top1_agree": 0.9538461538461539 }, { "id": "math-qualification-holdout-3", "domain": "math", "tokens": 4095, "nll": 1.5344706189501416, "teacher_nll": 1.516595822987539, "kl": 0.026961823033398617, "top1_agree": 0.9567765567765568 } ], "memory": { "swap_kib": 0, "cuda_peak_bytes": 14954638336 } }, "budget44-depth": { "precision_by_layer": { "2": { "gate": "bf16", "up": "bf16", "down": "bf16" }, "3": { "gate": "bf16", "up": "bf16", "down": "bf16" }, "4": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "5": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "6": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "7": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "8": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "9": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "10": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "11": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "12": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "13": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "14": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "15": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "16": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "17": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "18": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "19": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "20": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "21": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "22": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "23": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "24": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "25": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "26": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "27": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "28": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "29": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "30": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "31": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "32": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "33": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "34": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "35": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "36": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "37": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "38": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "39": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "40": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "41": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "42": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "43": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "44": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "45": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "46": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "47": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "48": { "gate": "fp8", "up": "fp8", "down": "fp8" } }, "tensor_bytes": 43340635712, "aggregate": { "all": { "tokens": 49140, "nll": 2.21992336427918, "teacher_nll": 2.214729754197971, "kl": 0.066194986073489, "top1_agree": 0.9354090354090354, "perplexity": 9.206625282481305, "teacher_perplexity": 9.158933613767237, "delta_nll": 0.005193610081208888 }, "code": { "tokens": 16380, "nll": 1.592673438416296, "teacher_nll": 1.585819314280127, "kl": 0.08255041191611388, "top1_agree": 0.9466422466422466, "perplexity": 4.916876338938635, "teacher_perplexity": 4.883290689738726, "delta_nll": 0.0068541241361690375 }, "math": { "tokens": 16380, "nll": 1.5130973644672878, "teacher_nll": 1.5066873193514945, "kl": 0.021885212046868635, "top1_agree": 0.9585470085470086, "perplexity": 4.5407734658387175, "teacher_perplexity": 4.511759991246355, "delta_nll": 0.00641004511579335 }, "prose": { "tokens": 16380, "nll": 3.553999289953956, "teacher_nll": 3.551682628962292, "kl": 0.09414933425748451, "top1_agree": 0.9010378510378511, "perplexity": 34.95282481768223, "teacher_perplexity": 34.871944693965524, "delta_nll": 0.0023166609916640546 } }, "sequences": [ { "id": "prose-qualification-holdout-0", "domain": "prose", "tokens": 4095, "nll": 3.5772039113027274, "teacher_nll": 3.570623514736674, "kl": 0.08783619703681769, "top1_agree": 0.9001221001221001 }, { "id": "prose-qualification-holdout-1", "domain": "prose", "tokens": 4095, "nll": 3.499457275241577, "teacher_nll": 3.4886688269683876, "kl": 0.11149968565310896, "top1_agree": 0.8920634920634921 }, { "id": "prose-qualification-holdout-2", "domain": "prose", "tokens": 4095, "nll": 3.589149953914358, "teacher_nll": 3.586854711586157, "kl": 0.07451462410890722, "top1_agree": 0.9115995115995116 }, { "id": "prose-qualification-holdout-3", "domain": "prose", "tokens": 4095, "nll": 3.5501860193571617, "teacher_nll": 3.56058346255795, "kl": 0.10274683023110415, "top1_agree": 0.9003663003663004 }, { "id": "code-qualification-holdout-0", "domain": "code", "tokens": 4095, "nll": 1.6159166301126444, "teacher_nll": 1.612040780315469, "kl": 0.08506395109407194, "top1_agree": 0.9426129426129426 }, { "id": "code-qualification-holdout-1", "domain": "code", "tokens": 4095, "nll": 1.4902202620756642, "teacher_nll": 1.4962344744848826, "kl": 0.08821711451433809, "top1_agree": 0.9540903540903541 }, { "id": "code-qualification-holdout-2", "domain": "code", "tokens": 4095, "nll": 1.470807389199952, "teacher_nll": 1.4386393615759918, "kl": 0.05696805114420051, "top1_agree": 0.9538461538461539 }, { "id": "code-qualification-holdout-3", "domain": "code", "tokens": 4095, "nll": 1.793749472276923, "teacher_nll": 1.796362640744164, "kl": 0.09995253091184501, "top1_agree": 0.936019536019536 }, { "id": "math-qualification-holdout-0", "domain": "math", "tokens": 4095, "nll": 1.4172291178988596, "teacher_nll": 1.4128543704711765, "kl": 0.02296853600768088, "top1_agree": 0.9526251526251526 }, { "id": "math-qualification-holdout-1", "domain": "math", "tokens": 4095, "nll": 1.5529686668125846, "teacher_nll": 1.545063196724879, "kl": 0.023295361356711942, "top1_agree": 0.9594627594627595 }, { "id": "math-qualification-holdout-2", "domain": "math", "tokens": 4095, "nll": 1.5368641188438705, "teacher_nll": 1.5522358872223831, "kl": 0.021453292778851205, "top1_agree": 0.9628815628815629 }, { "id": "math-qualification-holdout-3", "domain": "math", "tokens": 4095, "nll": 1.5453275543138367, "teacher_nll": 1.516595822987539, "kl": 0.019823658044230516, "top1_agree": 0.9592185592185593 } ], "memory": { "swap_kib": 0, "cuda_peak_bytes": 14954638336 } }, "budget48-depth": { "precision_by_layer": { "2": { "gate": "bf16", "up": "bf16", "down": "bf16" }, "3": { "gate": "bf16", "up": "bf16", "down": "bf16" }, "4": { "gate": "bf16", "up": "bf16", "down": "bf16" }, "5": { "gate": "bf16", "up": "bf16", "down": "bf16" }, "6": { "gate": "bf16", "up": "bf16", "down": "bf16" }, "7": { "gate": "bf16", "up": "bf16", "down": "bf16" }, "8": { "gate": "bf16", "up": "bf16", "down": "bf16" }, "9": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "10": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "11": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "12": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "13": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "14": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "15": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "16": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "17": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "18": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "19": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "20": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "21": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "22": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "23": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "24": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "25": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "26": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "27": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "28": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "29": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "30": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "31": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "32": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "33": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "34": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "35": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "36": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "37": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "38": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "39": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "40": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "41": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "42": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "43": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "44": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "45": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "46": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "47": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "48": { "gate": "fp8", "up": "fp8", "down": "fp8" } }, "tensor_bytes": 47366184512, "aggregate": { "all": { "tokens": 49140, "nll": 2.219438334936818, "teacher_nll": 2.214729754197971, "kl": 0.05590792845375154, "top1_agree": 0.9401302401302402, "perplexity": 9.202160881845348, "teacher_perplexity": 9.158933613767237, "delta_nll": 0.004708580738847168 }, "code": { "tokens": 16380, "nll": 1.598353770015004, "teacher_nll": 1.585819314280127, "kl": 0.06995221888771569, "top1_agree": 0.9498168498168498, "perplexity": 4.944885301760146, "teacher_perplexity": 4.883290689738726, "delta_nll": 0.012534455734876992 }, "math": { "tokens": 16380, "nll": 1.5061622015431395, "teacher_nll": 1.5066873193514945, "kl": 0.019403194701151243, "top1_agree": 0.9621489621489622, "perplexity": 4.5093914076750625, "teacher_perplexity": 4.511759991246355, "delta_nll": -0.0005251178083549579 }, "prose": { "tokens": 16380, "nll": 3.553799033252311, "teacher_nll": 3.551682628962292, "kl": 0.07836837177238767, "top1_agree": 0.9084249084249084, "perplexity": 34.94582598107641, "teacher_perplexity": 34.871944693965524, "delta_nll": 0.002116404290018803 } }, "sequences": [ { "id": "prose-qualification-holdout-0", "domain": "prose", "tokens": 4095, "nll": 3.581407642480946, "teacher_nll": 3.570623514736674, "kl": 0.07361397169739627, "top1_agree": 0.9086691086691087 }, { "id": "prose-qualification-holdout-1", "domain": "prose", "tokens": 4095, "nll": 3.488058652400388, "teacher_nll": 3.4886688269683876, "kl": 0.08343997936109047, "top1_agree": 0.9035409035409036 }, { "id": "prose-qualification-holdout-2", "domain": "prose", "tokens": 4095, "nll": 3.5756893917172237, "teacher_nll": 3.586854711586157, "kl": 0.07240510084399082, "top1_agree": 0.9103785103785104 }, { "id": "prose-qualification-holdout-3", "domain": "prose", "tokens": 4095, "nll": 3.570040446410686, "teacher_nll": 3.56058346255795, "kl": 0.08401443518707313, "top1_agree": 0.9111111111111111 }, { "id": "code-qualification-holdout-0", "domain": "code", "tokens": 4095, "nll": 1.611734163426363, "teacher_nll": 1.612040780315469, "kl": 0.08207176428865892, "top1_agree": 0.944078144078144 }, { "id": "code-qualification-holdout-1", "domain": "code", "tokens": 4095, "nll": 1.5178017247145283, "teacher_nll": 1.4962344744848826, "kl": 0.07385789276406587, "top1_agree": 0.9548229548229549 }, { "id": "code-qualification-holdout-2", "domain": "code", "tokens": 4095, "nll": 1.4678809896056906, "teacher_nll": 1.4386393615759918, "kl": 0.04726901354224982, "top1_agree": 0.9560439560439561 }, { "id": "code-qualification-holdout-3", "domain": "code", "tokens": 4095, "nll": 1.7959982023134335, "teacher_nll": 1.796362640744164, "kl": 0.07661020495588818, "top1_agree": 0.9443223443223443 }, { "id": "math-qualification-holdout-0", "domain": "math", "tokens": 4095, "nll": 1.4153713189638577, "teacher_nll": 1.4128543704711765, "kl": 0.02133012669427054, "top1_agree": 0.9592185592185593 }, { "id": "math-qualification-holdout-1", "domain": "math", "tokens": 4095, "nll": 1.550073492774451, "teacher_nll": 1.545063196724879, "kl": 0.019280421235331396, "top1_agree": 0.9604395604395605 }, { "id": "math-qualification-holdout-2", "domain": "math", "tokens": 4095, "nll": 1.5292586651477185, "teacher_nll": 1.5522358872223831, "kl": 0.01899524313626272, "top1_agree": 0.9655677655677656 }, { "id": "math-qualification-holdout-3", "domain": "math", "tokens": 4095, "nll": 1.529945329286531, "teacher_nll": 1.516595822987539, "kl": 0.018006987738740314, "top1_agree": 0.9633699633699634 } ], "memory": { "swap_kib": 0, "cuda_peak_bytes": 14954638336 } }, "budget28-depth": { "precision_by_layer": { "2": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "3": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "4": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "5": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "6": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "7": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "8": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "9": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "10": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "11": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "12": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "13": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "14": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "15": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "16": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "17": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "18": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "19": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "20": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "21": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "22": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "23": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "24": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "25": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "26": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "27": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "28": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "29": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "30": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "31": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "32": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "33": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "34": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "35": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "36": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "37": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "38": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "39": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "40": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "41": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "42": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "43": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "44": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "45": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "46": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "47": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "48": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" } }, "tensor_bytes": 27982328384, "aggregate": { "all": { "tokens": 49140, "nll": 2.2117175335194164, "teacher_nll": 2.214729754197971, "kl": 0.1379022409373005, "top1_agree": 0.8984330484330484, "perplexity": 9.131386394523139, "teacher_perplexity": 9.158933613767237, "delta_nll": -0.0030122206785545558 }, "code": { "tokens": 16380, "nll": 1.5829401942545387, "teacher_nll": 1.585819314280127, "kl": 0.15345898432536584, "top1_agree": 0.9151404151404151, "perplexity": 4.869251329922203, "teacher_perplexity": 4.883290689738726, "delta_nll": -0.002879120025588211 }, "math": { "tokens": 16380, "nll": 1.5169169920486407, "teacher_nll": 1.5066873193514945, "kl": 0.06299899023240012, "top1_agree": 0.9334554334554335, "perplexity": 4.558150695555302, "teacher_perplexity": 4.511759991246355, "delta_nll": 0.010229672697146253 }, "prose": { "tokens": 16380, "nll": 3.53529541425507, "teacher_nll": 3.551682628962292, "kl": 0.19724874825413555, "top1_agree": 0.8467032967032967, "perplexity": 34.305147447056434, "teacher_perplexity": 34.871944693965524, "delta_nll": -0.016387214707222153 } }, "sequences": [ { "id": "prose-qualification-holdout-0", "domain": "prose", "tokens": 4095, "nll": 3.576774416447268, "teacher_nll": 3.570623514736674, "kl": 0.18502871113964636, "top1_agree": 0.8517704517704517 }, { "id": "prose-qualification-holdout-1", "domain": "prose", "tokens": 4095, "nll": 3.4453719776046436, "teacher_nll": 3.4886688269683876, "kl": 0.1999274159525777, "top1_agree": 0.8402930402930403 }, { "id": "prose-qualification-holdout-2", "domain": "prose", "tokens": 4095, "nll": 3.567739048400059, "teacher_nll": 3.586854711586157, "kl": 0.194889590620849, "top1_agree": 0.8495726495726496 }, { "id": "prose-qualification-holdout-3", "domain": "prose", "tokens": 4095, "nll": 3.551296214568309, "teacher_nll": 3.56058346255795, "kl": 0.2091492753034692, "top1_agree": 0.8451770451770452 }, { "id": "code-qualification-holdout-0", "domain": "code", "tokens": 4095, "nll": 1.6328375978202028, "teacher_nll": 1.612040780315469, "kl": 0.16888136997618808, "top1_agree": 0.906959706959707 }, { "id": "code-qualification-holdout-1", "domain": "code", "tokens": 4095, "nll": 1.4438413806305117, "teacher_nll": 1.4962344744848826, "kl": 0.16586800363910942, "top1_agree": 0.9255189255189256 }, { "id": "code-qualification-holdout-2", "domain": "code", "tokens": 4095, "nll": 1.4312066908429977, "teacher_nll": 1.4386393615759918, "kl": 0.12033674994958918, "top1_agree": 0.9172161172161172 }, { "id": "code-qualification-holdout-3", "domain": "code", "tokens": 4095, "nll": 1.8238751077244424, "teacher_nll": 1.796362640744164, "kl": 0.15874981373657673, "top1_agree": 0.9108669108669109 }, { "id": "math-qualification-holdout-0", "domain": "math", "tokens": 4095, "nll": 1.4115624984105428, "teacher_nll": 1.4128543704711765, "kl": 0.06079979526545393, "top1_agree": 0.9343101343101343 }, { "id": "math-qualification-holdout-1", "domain": "math", "tokens": 4095, "nll": 1.5654080121945113, "teacher_nll": 1.545063196724879, "kl": 0.07012658806219788, "top1_agree": 0.9296703296703297 }, { "id": "math-qualification-holdout-2", "domain": "math", "tokens": 4095, "nll": 1.5408491942763183, "teacher_nll": 1.5522358872223831, "kl": 0.061726128159861865, "top1_agree": 0.9352869352869353 }, { "id": "math-qualification-holdout-3", "domain": "math", "tokens": 4095, "nll": 1.549848263313191, "teacher_nll": 1.516595822987539, "kl": 0.05934344944208678, "top1_agree": 0.9345543345543346 } ], "memory": { "swap_kib": 0, "cuda_peak_bytes": 14954638336 } }, "budget28-conditional": { "precision_by_layer": { "2": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "3": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "4": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "5": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "6": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "7": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "8": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "9": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "10": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "11": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "12": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "13": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "14": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "15": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "16": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "17": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "18": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "19": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "20": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "21": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "22": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "23": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "24": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "25": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "26": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "27": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "28": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "29": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "30": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "31": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "32": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "33": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "34": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "35": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "36": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "37": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "38": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "39": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "40": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "41": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "42": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "43": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "44": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "45": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "46": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "47": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "48": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" } }, "tensor_bytes": 27982328384, "aggregate": { "all": { "tokens": 49140, "nll": 2.2224484121527768, "teacher_nll": 2.214729754197971, "kl": 0.13521137943640685, "top1_agree": 0.8994505494505495, "perplexity": 9.22990182690099, "teacher_perplexity": 9.158933613767237, "delta_nll": 0.007718657954805863 }, "code": { "tokens": 16380, "nll": 1.5960918323169844, "teacher_nll": 1.585819314280127, "kl": 0.15048720305000907, "top1_agree": 0.9156898656898657, "perplexity": 4.9337129196633285, "teacher_perplexity": 4.883290689738726, "delta_nll": 0.01027251803685747 }, "math": { "tokens": 16380, "nll": 1.5213324331174694, "teacher_nll": 1.5066873193514945, "kl": 0.0603252738759704, "top1_agree": 0.9349206349206349, "perplexity": 4.57832143992999, "teacher_perplexity": 4.511759991246355, "delta_nll": 0.014645113765974882 }, "prose": { "tokens": 16380, "nll": 3.549920971023876, "teacher_nll": 3.551682628962292, "kl": 0.19482166138324108, "top1_agree": 0.8477411477411477, "perplexity": 34.810566335477304, "teacher_perplexity": 34.871944693965524, "delta_nll": -0.0017616579384158726 } }, "sequences": [ { "id": "prose-qualification-holdout-0", "domain": "prose", "tokens": 4095, "nll": 3.589803723159436, "teacher_nll": 3.570623514736674, "kl": 0.18862637355650738, "top1_agree": 0.8512820512820513 }, { "id": "prose-qualification-holdout-1", "domain": "prose", "tokens": 4095, "nll": 3.4418622362744675, "teacher_nll": 3.4886688269683876, "kl": 0.20619230410118242, "top1_agree": 0.842979242979243 }, { "id": "prose-qualification-holdout-2", "domain": "prose", "tokens": 4095, "nll": 3.6040337829217224, "teacher_nll": 3.586854711586157, "kl": 0.19182675237038488, "top1_agree": 0.852014652014652 }, { "id": "prose-qualification-holdout-3", "domain": "prose", "tokens": 4095, "nll": 3.563984141739879, "teacher_nll": 3.56058346255795, "kl": 0.19264121550488966, "top1_agree": 0.8446886446886447 }, { "id": "code-qualification-holdout-0", "domain": "code", "tokens": 4095, "nll": 1.6439844068590102, "teacher_nll": 1.612040780315469, "kl": 0.1714319664334494, "top1_agree": 0.9108669108669109 }, { "id": "code-qualification-holdout-1", "domain": "code", "tokens": 4095, "nll": 1.475341613941868, "teacher_nll": 1.4962344744848826, "kl": 0.1390051935473059, "top1_agree": 0.9223443223443224 }, { "id": "code-qualification-holdout-2", "domain": "code", "tokens": 4095, "nll": 1.4345243325891366, "teacher_nll": 1.4386393615759918, "kl": 0.12111187706063518, "top1_agree": 0.9177045177045177 }, { "id": "code-qualification-holdout-3", "domain": "code", "tokens": 4095, "nll": 1.8305169758779225, "teacher_nll": 1.796362640744164, "kl": 0.17039977515864577, "top1_agree": 0.9118437118437118 }, { "id": "math-qualification-holdout-0", "domain": "math", "tokens": 4095, "nll": 1.4194815079371135, "teacher_nll": 1.4128543704711765, "kl": 0.0626669808985695, "top1_agree": 0.9362637362637363 }, { "id": "math-qualification-holdout-1", "domain": "math", "tokens": 4095, "nll": 1.570239007691324, "teacher_nll": 1.545063196724879, "kl": 0.059914114885714465, "top1_agree": 0.9291819291819292 }, { "id": "math-qualification-holdout-2", "domain": "math", "tokens": 4095, "nll": 1.5442800039773459, "teacher_nll": 1.5522358872223831, "kl": 0.059936179899325155, "top1_agree": 0.9367521367521368 }, { "id": "math-qualification-holdout-3", "domain": "math", "tokens": 4095, "nll": 1.5513292128640945, "teacher_nll": 1.516595822987539, "kl": 0.05878381982027247, "top1_agree": 0.9374847374847375 } ], "memory": { "swap_kib": 0, "cuda_peak_bytes": 14954638336 } }, "budget32-depth": { "precision_by_layer": { "2": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "3": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "4": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "5": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "6": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "7": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "8": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "9": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "10": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "11": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "12": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "13": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "14": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "15": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "16": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "17": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "18": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "19": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "20": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "21": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "22": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "23": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "24": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "25": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "26": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "27": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "28": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "29": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "30": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "31": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "32": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "33": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "34": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "35": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "36": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "37": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "38": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "39": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "40": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "41": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "42": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "43": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "44": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "45": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "46": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "47": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "48": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" } }, "tensor_bytes": 31859994176, "aggregate": { "all": { "tokens": 49140, "nll": 2.209968472999628, "teacher_nll": 2.214729754197971, "kl": 0.1076129502004659, "top1_agree": 0.9095645095645095, "perplexity": 9.115429006376434, "teacher_perplexity": 9.158933613767237, "delta_nll": -0.00476128119834307 }, "code": { "tokens": 16380, "nll": 1.5728688767905346, "teacher_nll": 1.585819314280127, "kl": 0.12423731337809737, "top1_agree": 0.9235042735042736, "perplexity": 4.82045767459655, "teacher_perplexity": 4.883290689738726, "delta_nll": -0.012950437489592348 }, "math": { "tokens": 16380, "nll": 1.5087034377568516, "teacher_nll": 1.5066873193514945, "kl": 0.047924589849690086, "top1_agree": 0.942002442002442, "perplexity": 4.5208654093214875, "teacher_perplexity": 4.511759991246355, "delta_nll": 0.0020161184053570924 }, "prose": { "tokens": 16380, "nll": 3.548333104451497, "teacher_nll": 3.551682628962292, "kl": 0.1506769473736103, "top1_agree": 0.8631868131868132, "perplexity": 34.7553356619023, "teacher_perplexity": 34.871944693965524, "delta_nll": -0.0033495245107948435 } }, "sequences": [ { "id": "prose-qualification-holdout-0", "domain": "prose", "tokens": 4095, "nll": 3.599144261398595, "teacher_nll": 3.570623514736674, "kl": 0.1407006877391475, "top1_agree": 0.8647130647130647 }, { "id": "prose-qualification-holdout-1", "domain": "prose", "tokens": 4095, "nll": 3.4657273885240194, "teacher_nll": 3.4886688269683876, "kl": 0.17342653262891758, "top1_agree": 0.8617826617826618 }, { "id": "prose-qualification-holdout-2", "domain": "prose", "tokens": 4095, "nll": 3.5611716171469587, "teacher_nll": 3.586854711586157, "kl": 0.1355552160405123, "top1_agree": 0.8654456654456655 }, { "id": "prose-qualification-holdout-3", "domain": "prose", "tokens": 4095, "nll": 3.5672891507364164, "teacher_nll": 3.56058346255795, "kl": 0.1530253530858637, "top1_agree": 0.8608058608058609 }, { "id": "code-qualification-holdout-0", "domain": "code", "tokens": 4095, "nll": 1.6259151891910986, "teacher_nll": 1.612040780315469, "kl": 0.1325808134562221, "top1_agree": 0.9164835164835164 }, { "id": "code-qualification-holdout-1", "domain": "code", "tokens": 4095, "nll": 1.4427267800320636, "teacher_nll": 1.4962344744848826, "kl": 0.12487448638393765, "top1_agree": 0.9318681318681319 }, { "id": "code-qualification-holdout-2", "domain": "code", "tokens": 4095, "nll": 1.4258999699929114, "teacher_nll": 1.4386393615759918, "kl": 0.10263109003376757, "top1_agree": 0.9274725274725275 }, { "id": "code-qualification-holdout-3", "domain": "code", "tokens": 4095, "nll": 1.796933567946065, "teacher_nll": 1.796362640744164, "kl": 0.13686286363846217, "top1_agree": 0.9181929181929182 }, { "id": "math-qualification-holdout-0", "domain": "math", "tokens": 4095, "nll": 1.407740637729165, "teacher_nll": 1.4128543704711765, "kl": 0.04687176862042466, "top1_agree": 0.9443223443223443 }, { "id": "math-qualification-holdout-1", "domain": "math", "tokens": 4095, "nll": 1.547222171452747, "teacher_nll": 1.545063196724879, "kl": 0.046711798872848716, "top1_agree": 0.9416361416361416 }, { "id": "math-qualification-holdout-2", "domain": "math", "tokens": 4095, "nll": 1.5465935969090723, "teacher_nll": 1.5522358872223831, "kl": 0.04912073272342938, "top1_agree": 0.9423687423687424 }, { "id": "math-qualification-holdout-3", "domain": "math", "tokens": 4095, "nll": 1.5332573449364222, "teacher_nll": 1.516595822987539, "kl": 0.0489940591820576, "top1_agree": 0.9396825396825397 } ], "memory": { "swap_kib": 0, "cuda_peak_bytes": 14954638336 } }, "budget36-depth": { "precision_by_layer": { "2": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "3": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "4": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "5": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "6": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "7": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "8": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "9": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "10": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "11": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "12": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "13": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "14": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "15": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "16": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "17": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "18": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "19": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "20": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "21": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "22": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "23": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "24": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "25": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "26": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "27": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "28": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "29": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "30": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "31": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "32": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "33": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "34": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "35": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "36": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "37": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "38": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "39": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "40": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "41": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "42": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "43": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "44": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "45": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "46": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "47": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "48": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" } }, "tensor_bytes": 35737659968, "aggregate": { "all": { "tokens": 49140, "nll": 2.223855302175662, "teacher_nll": 2.214729754197971, "kl": 0.09127278831151743, "top1_agree": 0.9190883190883191, "perplexity": 9.24289642253355, "teacher_perplexity": 9.158933613767237, "delta_nll": 0.009125547977691273 }, "code": { "tokens": 16380, "nll": 1.6116677623092037, "teacher_nll": 1.585819314280127, "kl": 0.1089983913497101, "top1_agree": 0.9323565323565324, "perplexity": 5.011161689196273, "teacher_perplexity": 4.883290689738726, "delta_nll": 0.02584844802907682 }, "math": { "tokens": 16380, "nll": 1.503876333446293, "teacher_nll": 1.5066873193514945, "kl": 0.03497329444666558, "top1_agree": 0.9487789987789987, "perplexity": 4.49909530596853, "teacher_perplexity": 4.511759991246355, "delta_nll": -0.0028109859052014574 }, "prose": { "tokens": 16380, "nll": 3.5560218107714903, "teacher_nll": 3.551682628962292, "kl": 0.1298466791381766, "top1_agree": 0.8761294261294261, "perplexity": 35.023589170573715, "teacher_perplexity": 34.871944693965524, "delta_nll": 0.004339181809198234 } }, "sequences": [ { "id": "prose-qualification-holdout-0", "domain": "prose", "tokens": 4095, "nll": 3.6045262673253395, "teacher_nll": 3.570623514736674, "kl": 0.12269781500428588, "top1_agree": 0.8764346764346764 }, { "id": "prose-qualification-holdout-1", "domain": "prose", "tokens": 4095, "nll": 3.4700199392689015, "teacher_nll": 3.4886688269683876, "kl": 0.15376708143651122, "top1_agree": 0.8788766788766789 }, { "id": "prose-qualification-holdout-2", "domain": "prose", "tokens": 4095, "nll": 3.5796889234084057, "teacher_nll": 3.586854711586157, "kl": 0.11325538041972998, "top1_agree": 0.8774114774114774 }, { "id": "prose-qualification-holdout-3", "domain": "prose", "tokens": 4095, "nll": 3.5698521130833143, "teacher_nll": 3.56058346255795, "kl": 0.12966643969217936, "top1_agree": 0.8717948717948718 }, { "id": "code-qualification-holdout-0", "domain": "code", "tokens": 4095, "nll": 1.6474669259691996, "teacher_nll": 1.612040780315469, "kl": 0.12218621996878413, "top1_agree": 0.926984126984127 }, { "id": "code-qualification-holdout-1", "domain": "code", "tokens": 4095, "nll": 1.5146658239492712, "teacher_nll": 1.4962344744848826, "kl": 0.10590404228776948, "top1_agree": 0.936996336996337 }, { "id": "code-qualification-holdout-2", "domain": "code", "tokens": 4095, "nll": 1.4736719823145605, "teacher_nll": 1.4386393615759918, "kl": 0.08629191191059621, "top1_agree": 0.9387057387057387 }, { "id": "code-qualification-holdout-3", "domain": "code", "tokens": 4095, "nll": 1.8108663170037835, "teacher_nll": 1.796362640744164, "kl": 0.12161139123169057, "top1_agree": 0.9267399267399268 }, { "id": "math-qualification-holdout-0", "domain": "math", "tokens": 4095, "nll": 1.3993232566358405, "teacher_nll": 1.4128543704711765, "kl": 0.0360991838723603, "top1_agree": 0.947008547008547 }, { "id": "math-qualification-holdout-1", "domain": "math", "tokens": 4095, "nll": 1.542288848914215, "teacher_nll": 1.545063196724879, "kl": 0.03444656883258377, "top1_agree": 0.9462759462759462 }, { "id": "math-qualification-holdout-2", "domain": "math", "tokens": 4095, "nll": 1.535391583751264, "teacher_nll": 1.5522358872223831, "kl": 0.034275711528838626, "top1_agree": 0.9496947496947497 }, { "id": "math-qualification-holdout-3", "domain": "math", "tokens": 4095, "nll": 1.5385016444838526, "teacher_nll": 1.516595822987539, "kl": 0.035071713552879615, "top1_agree": 0.9521367521367521 } ], "memory": { "swap_kib": 0, "cuda_peak_bytes": 14954638336 } }, "budget36-conditional": { "precision_by_layer": { "2": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "3": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "4": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "5": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "6": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "7": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "8": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "9": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "10": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "11": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "12": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "13": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "14": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "15": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "16": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "17": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "18": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "19": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "20": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "21": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "22": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "23": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "24": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "25": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "26": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "27": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "28": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "29": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "30": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "31": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "32": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "33": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "34": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "35": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "36": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "37": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "38": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "39": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "40": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "41": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "42": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "43": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "44": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "45": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "46": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "47": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "48": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" } }, "tensor_bytes": 35737659968, "aggregate": { "all": { "tokens": 49140, "nll": 2.211066000828861, "teacher_nll": 2.214729754197971, "kl": 0.09881533177210929, "top1_agree": 0.9154049654049654, "perplexity": 9.125438935469404, "teacher_perplexity": 9.158933613767237, "delta_nll": -0.0036637533691097346 }, "code": { "tokens": 16380, "nll": 1.5794704765336127, "teacher_nll": 1.585819314280127, "kl": 0.1142164363947689, "top1_agree": 0.92997557997558, "perplexity": 4.852385678739899, "teacher_perplexity": 4.883290689738726, "delta_nll": -0.00634883774651418 }, "math": { "tokens": 16380, "nll": 1.5101053988904451, "teacher_nll": 1.5066873193514945, "kl": 0.03895971165544675, "top1_agree": 0.9462759462759462, "perplexity": 4.527207931861759, "teacher_perplexity": 4.511759991246355, "delta_nll": 0.00341807953895068 }, "prose": { "tokens": 16380, "nll": 3.5436221270625263, "teacher_nll": 3.551682628962292, "kl": 0.1432698472661122, "top1_agree": 0.86996336996337, "perplexity": 34.59198912447507, "teacher_perplexity": 34.871944693965524, "delta_nll": -0.008060501899765704 } }, "sequences": [ { "id": "prose-qualification-holdout-0", "domain": "prose", "tokens": 4095, "nll": 3.58719486148075, "teacher_nll": 3.570623514736674, "kl": 0.1392865496531802, "top1_agree": 0.8717948717948718 }, { "id": "prose-qualification-holdout-1", "domain": "prose", "tokens": 4095, "nll": 3.4693113334012993, "teacher_nll": 3.4886688269683876, "kl": 0.1504007330162039, "top1_agree": 0.8683760683760684 }, { "id": "prose-qualification-holdout-2", "domain": "prose", "tokens": 4095, "nll": 3.5602676931900565, "teacher_nll": 3.586854711586157, "kl": 0.13613470395406088, "top1_agree": 0.8798534798534798 }, { "id": "prose-qualification-holdout-3", "domain": "prose", "tokens": 4095, "nll": 3.557714620177999, "teacher_nll": 3.56058346255795, "kl": 0.14725740244100383, "top1_agree": 0.8598290598290599 }, { "id": "code-qualification-holdout-0", "domain": "code", "tokens": 4095, "nll": 1.6157172425471529, "teacher_nll": 1.612040780315469, "kl": 0.1348012829656566, "top1_agree": 0.9255189255189256 }, { "id": "code-qualification-holdout-1", "domain": "code", "tokens": 4095, "nll": 1.5071097101484026, "teacher_nll": 1.4962344744848826, "kl": 0.11607390501705864, "top1_agree": 0.936996336996337 }, { "id": "code-qualification-holdout-2", "domain": "code", "tokens": 4095, "nll": 1.4030494752821032, "teacher_nll": 1.4386393615759918, "kl": 0.08161759099971971, "top1_agree": 0.9335775335775336 }, { "id": "code-qualification-holdout-3", "domain": "code", "tokens": 4095, "nll": 1.792005478156792, "teacher_nll": 1.796362640744164, "kl": 0.12437296659664066, "top1_agree": 0.9238095238095239 }, { "id": "math-qualification-holdout-0", "domain": "math", "tokens": 4095, "nll": 1.4169574918909968, "teacher_nll": 1.4128543704711765, "kl": 0.039256950532414826, "top1_agree": 0.9465201465201465 }, { "id": "math-qualification-holdout-1", "domain": "math", "tokens": 4095, "nll": 1.5547984918540796, "teacher_nll": 1.545063196724879, "kl": 0.037581647774238724, "top1_agree": 0.9404151404151404 }, { "id": "math-qualification-holdout-2", "domain": "math", "tokens": 4095, "nll": 1.5365613138573802, "teacher_nll": 1.5522358872223831, "kl": 0.04003023478283259, "top1_agree": 0.9489621489621489 }, { "id": "math-qualification-holdout-3", "domain": "math", "tokens": 4095, "nll": 1.5321042979593242, "teacher_nll": 1.516595822987539, "kl": 0.03897001353230086, "top1_agree": 0.9492063492063492 } ], "memory": { "swap_kib": 0, "cuda_peak_bytes": 14954638336 } }, "budget40-depth": { "precision_by_layer": { "2": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "3": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "4": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "5": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "6": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "7": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "8": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "9": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "10": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "11": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "12": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "13": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "14": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "15": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "16": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "17": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "18": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "19": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "20": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "21": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "22": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "23": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "24": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "25": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "26": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "27": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "28": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "29": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "30": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "31": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "32": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "33": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "34": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "35": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "36": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "37": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "38": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "39": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "40": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "41": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "42": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "43": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "44": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "45": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "46": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "47": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "48": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" } }, "tensor_bytes": 39967840832, "aggregate": { "all": { "tokens": 49140, "nll": 2.2225935721111143, "teacher_nll": 2.214729754197971, "kl": 0.07734097852024803, "top1_agree": 0.9308506308506308, "perplexity": 9.231241736313887, "teacher_perplexity": 9.158933613767237, "delta_nll": 0.00786381791314339 }, "code": { "tokens": 16380, "nll": 1.614134958149895, "teacher_nll": 1.585819314280127, "kl": 0.09729672360901467, "top1_agree": 0.9414529914529914, "perplexity": 5.0235404706327325, "teacher_perplexity": 4.883290689738726, "delta_nll": 0.028315643869768037 }, "math": { "tokens": 16380, "nll": 1.5017716076638963, "teacher_nll": 1.5066873193514945, "kl": 0.024713811772319422, "top1_agree": 0.955921855921856, "perplexity": 4.489635902297988, "teacher_perplexity": 4.511759991246355, "delta_nll": -0.00491571168759819 }, "prose": { "tokens": 16380, "nll": 3.5518741505195517, "teacher_nll": 3.551682628962292, "kl": 0.11001240017941001, "top1_agree": 0.8951770451770452, "perplexity": 34.87862406271912, "teacher_perplexity": 34.871944693965524, "delta_nll": 0.00019152155725965514 } }, "sequences": [ { "id": "prose-qualification-holdout-0", "domain": "prose", "tokens": 4095, "nll": 3.59116698138152, "teacher_nll": 3.570623514736674, "kl": 0.10556535575270508, "top1_agree": 0.8942612942612943 }, { "id": "prose-qualification-holdout-1", "domain": "prose", "tokens": 4095, "nll": 3.4707406478341536, "teacher_nll": 3.4886688269683876, "kl": 0.13352662218650474, "top1_agree": 0.8920634920634921 }, { "id": "prose-qualification-holdout-2", "domain": "prose", "tokens": 4095, "nll": 3.5792850219577517, "teacher_nll": 3.586854711586157, "kl": 0.09330803332020221, "top1_agree": 0.9010989010989011 }, { "id": "prose-qualification-holdout-3", "domain": "prose", "tokens": 4095, "nll": 3.566303950904781, "teacher_nll": 3.56058346255795, "kl": 0.10764958945822803, "top1_agree": 0.8932844932844933 }, { "id": "code-qualification-holdout-0", "domain": "code", "tokens": 4095, "nll": 1.6441088330614697, "teacher_nll": 1.612040780315469, "kl": 0.11154779642638415, "top1_agree": 0.9357753357753358 }, { "id": "code-qualification-holdout-1", "domain": "code", "tokens": 4095, "nll": 1.5216690654137486, "teacher_nll": 1.4962344744848826, "kl": 0.09650421681221442, "top1_agree": 0.9465201465201465 }, { "id": "code-qualification-holdout-2", "domain": "code", "tokens": 4095, "nll": 1.477832281836951, "teacher_nll": 1.4386393615759918, "kl": 0.07622636381231968, "top1_agree": 0.9457875457875458 }, { "id": "code-qualification-holdout-3", "domain": "code", "tokens": 4095, "nll": 1.8129296522874099, "teacher_nll": 1.796362640744164, "kl": 0.10490851738514044, "top1_agree": 0.9377289377289377 }, { "id": "math-qualification-holdout-0", "domain": "math", "tokens": 4095, "nll": 1.4050527554583054, "teacher_nll": 1.4128543704711765, "kl": 0.025840590572182513, "top1_agree": 0.9553113553113554 }, { "id": "math-qualification-holdout-1", "domain": "math", "tokens": 4095, "nll": 1.5450425144516942, "teacher_nll": 1.545063196724879, "kl": 0.023052263863939652, "top1_agree": 0.9543345543345544 }, { "id": "math-qualification-holdout-2", "domain": "math", "tokens": 4095, "nll": 1.5158889481727311, "teacher_nll": 1.5522358872223831, "kl": 0.024994288965313718, "top1_agree": 0.9570207570207571 }, { "id": "math-qualification-holdout-3", "domain": "math", "tokens": 4095, "nll": 1.5411022125728546, "teacher_nll": 1.516595822987539, "kl": 0.024968103687841813, "top1_agree": 0.9570207570207571 } ], "memory": { "swap_kib": 0, "cuda_peak_bytes": 14954638336 } }, "budget40-conditional": { "precision_by_layer": { "2": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "3": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "4": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "5": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "6": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "7": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "8": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "9": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "10": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "11": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "12": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "13": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "14": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "15": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "16": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "17": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "18": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "19": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "20": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "21": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "22": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "23": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "24": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "25": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "26": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "27": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "28": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "29": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "30": { "gate": "nvfp4", "up": "nvfp4", "down": "nvfp4" }, "31": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "32": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "33": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "34": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "35": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "36": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "37": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "38": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "39": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "40": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "41": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "42": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "43": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "44": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "45": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "46": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "47": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "48": { "gate": "fp8", "up": "fp8", "down": "fp8" } }, "tensor_bytes": 39967840832, "aggregate": { "all": { "tokens": 49140, "nll": 2.2112540609767684, "teacher_nll": 2.214729754197971, "kl": 0.08093327798213132, "top1_agree": 0.9278795278795279, "perplexity": 9.127155228243405, "teacher_perplexity": 9.158933613767237, "delta_nll": -0.003475693221202558 }, "code": { "tokens": 16380, "nll": 1.58970221944781, "teacher_nll": 1.585819314280127, "kl": 0.10097118896751758, "top1_agree": 0.94004884004884, "perplexity": 4.902288904657061, "teacher_perplexity": 4.883290689738726, "delta_nll": 0.0038829051676831927 }, "math": { "tokens": 16380, "nll": 1.501633097517039, "teacher_nll": 1.5066873193514945, "kl": 0.02864558382075785, "top1_agree": 0.9518315018315018, "perplexity": 4.489014085234806, "teacher_perplexity": 4.511759991246355, "delta_nll": -0.0050542218344553635 }, "prose": { "tokens": 16380, "nll": 3.5424268659654556, "teacher_nll": 3.551682628962292, "kl": 0.11318306115811852, "top1_agree": 0.8917582417582418, "perplexity": 34.5506673656692, "teacher_perplexity": 34.871944693965524, "delta_nll": -0.009255762996836392 } }, "sequences": [ { "id": "prose-qualification-holdout-0", "domain": "prose", "tokens": 4095, "nll": 3.6013154808854884, "teacher_nll": 3.570623514736674, "kl": 0.11052129783909836, "top1_agree": 0.8947496947496948 }, { "id": "prose-qualification-holdout-1", "domain": "prose", "tokens": 4095, "nll": 3.4538758032196752, "teacher_nll": 3.4886688269683876, "kl": 0.13668285830990298, "top1_agree": 0.8920634920634921 }, { "id": "prose-qualification-holdout-2", "domain": "prose", "tokens": 4095, "nll": 3.5637225326892077, "teacher_nll": 3.586854711586157, "kl": 0.09356140222887126, "top1_agree": 0.9010989010989011 }, { "id": "prose-qualification-holdout-3", "domain": "prose", "tokens": 4095, "nll": 3.550793647067451, "teacher_nll": 3.56058346255795, "kl": 0.11196668625460149, "top1_agree": 0.8791208791208791 }, { "id": "code-qualification-holdout-0", "domain": "code", "tokens": 4095, "nll": 1.6111260536389473, "teacher_nll": 1.612040780315469, "kl": 0.11032203337648412, "top1_agree": 0.9357753357753358 }, { "id": "code-qualification-holdout-1", "domain": "code", "tokens": 4095, "nll": 1.4951144709255233, "teacher_nll": 1.4962344744848826, "kl": 0.09977835434624273, "top1_agree": 0.9482295482295482 }, { "id": "code-qualification-holdout-2", "domain": "code", "tokens": 4095, "nll": 1.4474246419829764, "teacher_nll": 1.4386393615759918, "kl": 0.07929879127113519, "top1_agree": 0.9431013431013431 }, { "id": "code-qualification-holdout-3", "domain": "code", "tokens": 4095, "nll": 1.8051437112437936, "teacher_nll": 1.796362640744164, "kl": 0.11448557687620832, "top1_agree": 0.9330891330891331 }, { "id": "math-qualification-holdout-0", "domain": "math", "tokens": 4095, "nll": 1.4102511390807138, "teacher_nll": 1.4128543704711765, "kl": 0.029556334721008645, "top1_agree": 0.9504273504273504 }, { "id": "math-qualification-holdout-1", "domain": "math", "tokens": 4095, "nll": 1.5434705280122303, "teacher_nll": 1.545063196724879, "kl": 0.027982906223944545, "top1_agree": 0.9484737484737484 }, { "id": "math-qualification-holdout-2", "domain": "math", "tokens": 4095, "nll": 1.5248470017615983, "teacher_nll": 1.5522358872223831, "kl": 0.029806509846878285, "top1_agree": 0.9540903540903541 }, { "id": "math-qualification-holdout-3", "domain": "math", "tokens": 4095, "nll": 1.5279637212136143, "teacher_nll": 1.516595822987539, "kl": 0.02723658449119992, "top1_agree": 0.9543345543345544 } ], "memory": { "swap_kib": 0, "cuda_peak_bytes": 14954638336 } }, "budget44-conditional": { "precision_by_layer": { "2": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "3": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "4": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "5": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "6": { "gate": "bf16", "up": "bf16", "down": "bf16" }, "7": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "8": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "9": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "10": { "gate": "bf16", "up": "bf16", "down": "bf16" }, "11": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "12": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "13": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "14": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "15": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "16": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "17": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "18": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "19": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "20": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "21": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "22": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "23": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "24": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "25": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "26": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "27": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "28": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "29": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "30": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "31": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "32": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "33": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "34": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "35": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "36": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "37": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "38": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "39": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "40": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "41": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "42": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "43": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "44": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "45": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "46": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "47": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "48": { "gate": "fp8", "up": "fp8", "down": "fp8" } }, "tensor_bytes": 43340635712, "aggregate": { "all": { "tokens": 49140, "nll": 2.220883278078697, "teacher_nll": 2.214729754197971, "kl": 0.07120300278258629, "top1_agree": 0.9367114367114368, "perplexity": 9.21546709214557, "teacher_perplexity": 9.158933613767237, "delta_nll": 0.006153523880726208 }, "code": { "tokens": 16380, "nll": 1.6065521288267421, "teacher_nll": 1.585819314280127, "kl": 0.09249036310147381, "top1_agree": 0.9474969474969475, "perplexity": 4.98559188131849, "teacher_perplexity": 4.883290689738726, "delta_nll": 0.02073281454661524 }, "math": { "tokens": 16380, "nll": 1.5020923887926434, "teacher_nll": 1.5066873193514945, "kl": 0.02246947362928018, "top1_agree": 0.9605616605616606, "perplexity": 4.491076323788055, "teacher_perplexity": 4.511759991246355, "delta_nll": -0.004594930558851074 }, "prose": { "tokens": 16380, "nll": 3.554005316616706, "teacher_nll": 3.551682628962292, "kl": 0.0986491716170049, "top1_agree": 0.9020757020757021, "perplexity": 34.95303546720431, "teacher_perplexity": 34.871944693965524, "delta_nll": 0.002322687654413791 } }, "sequences": [ { "id": "prose-qualification-holdout-0", "domain": "prose", "tokens": 4095, "nll": 3.576349109840626, "teacher_nll": 3.570623514736674, "kl": 0.09326929702572478, "top1_agree": 0.9074481074481074 }, { "id": "prose-qualification-holdout-1", "domain": "prose", "tokens": 4095, "nll": 3.472557887635097, "teacher_nll": 3.4886688269683876, "kl": 0.11587396829556196, "top1_agree": 0.8971916971916972 }, { "id": "prose-qualification-holdout-2", "domain": "prose", "tokens": 4095, "nll": 3.585051614577371, "teacher_nll": 3.586854711586157, "kl": 0.08426370783748789, "top1_agree": 0.9086691086691087 }, { "id": "prose-qualification-holdout-3", "domain": "prose", "tokens": 4095, "nll": 3.5820626544137286, "teacher_nll": 3.56058346255795, "kl": 0.10118971330924494, "top1_agree": 0.894993894993895 }, { "id": "code-qualification-holdout-0", "domain": "code", "tokens": 4095, "nll": 1.6318553607833546, "teacher_nll": 1.612040780315469, "kl": 0.10678286492751778, "top1_agree": 0.9416361416361416 }, { "id": "code-qualification-holdout-1", "domain": "code", "tokens": 4095, "nll": 1.5190067352974954, "teacher_nll": 1.4962344744848826, "kl": 0.08591921168842304, "top1_agree": 0.9526251526251526 }, { "id": "code-qualification-holdout-2", "domain": "code", "tokens": 4095, "nll": 1.4650198250608713, "teacher_nll": 1.4386393615759918, "kl": 0.07217798363303672, "top1_agree": 0.9545787545787546 }, { "id": "code-qualification-holdout-3", "domain": "code", "tokens": 4095, "nll": 1.8103265941652478, "teacher_nll": 1.796362640744164, "kl": 0.10508139215691768, "top1_agree": 0.9411477411477411 }, { "id": "math-qualification-holdout-0", "domain": "math", "tokens": 4095, "nll": 1.40823189257411, "teacher_nll": 1.4128543704711765, "kl": 0.02408212613273453, "top1_agree": 0.9604395604395605 }, { "id": "math-qualification-holdout-1", "domain": "math", "tokens": 4095, "nll": 1.539740735474378, "teacher_nll": 1.545063196724879, "kl": 0.021373734383938223, "top1_agree": 0.9582417582417583 }, { "id": "math-qualification-holdout-2", "domain": "math", "tokens": 4095, "nll": 1.527646278781914, "teacher_nll": 1.5522358872223831, "kl": 0.022497680037304013, "top1_agree": 0.9628815628815629 }, { "id": "math-qualification-holdout-3", "domain": "math", "tokens": 4095, "nll": 1.5327506483401716, "teacher_nll": 1.516595822987539, "kl": 0.02192435396314395, "top1_agree": 0.9606837606837607 } ], "memory": { "swap_kib": 0, "cuda_peak_bytes": 14954638336 } }, "budget48-conditional": { "precision_by_layer": { "2": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "3": { "gate": "bf16", "up": "bf16", "down": "bf16" }, "4": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "5": { "gate": "bf16", "up": "bf16", "down": "bf16" }, "6": { "gate": "bf16", "up": "bf16", "down": "bf16" }, "7": { "gate": "bf16", "up": "bf16", "down": "bf16" }, "8": { "gate": "bf16", "up": "bf16", "down": "bf16" }, "9": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "10": { "gate": "bf16", "up": "bf16", "down": "bf16" }, "11": { "gate": "bf16", "up": "bf16", "down": "bf16" }, "12": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "13": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "14": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "15": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "16": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "17": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "18": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "19": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "20": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "21": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "22": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "23": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "24": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "25": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "26": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "27": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "28": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "29": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "30": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "31": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "32": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "33": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "34": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "35": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "36": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "37": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "38": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "39": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "40": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "41": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "42": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "43": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "44": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "45": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "46": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "47": { "gate": "fp8", "up": "fp8", "down": "fp8" }, "48": { "gate": "fp8", "up": "fp8", "down": "fp8" } }, "tensor_bytes": 47366184512, "aggregate": { "all": { "tokens": 49140, "nll": 2.221172728381314, "teacher_nll": 2.214729754197971, "kl": 0.06295151559269836, "top1_agree": 0.9396418396418397, "perplexity": 9.218134897964124, "teacher_perplexity": 9.158933613767237, "delta_nll": 0.006442974183343164 }, "code": { "tokens": 16380, "nll": 1.6024605643909346, "teacher_nll": 1.585819314280127, "kl": 0.07698328460729312, "top1_agree": 0.9495726495726495, "perplexity": 4.9652346856740985, "teacher_perplexity": 4.883290689738726, "delta_nll": 0.016641250110807748 }, "math": { "tokens": 16380, "nll": 1.5092922030787765, "teacher_nll": 1.5066873193514945, "kl": 0.018815637222048855, "top1_agree": 0.9614163614163614, "perplexity": 4.523527921820188, "teacher_perplexity": 4.511759991246355, "delta_nll": 0.00260488372728207 }, "prose": { "tokens": 16380, "nll": 3.5517654176742313, "teacher_nll": 3.551682628962292, "kl": 0.09305562494875311, "top1_agree": 0.9079365079365079, "perplexity": 34.874831816858496, "teacher_perplexity": 34.871944693965524, "delta_nll": 8.27887119392301e-05 } }, "sequences": [ { "id": "prose-qualification-holdout-0", "domain": "prose", "tokens": 4095, "nll": 3.579152346880008, "teacher_nll": 3.570623514736674, "kl": 0.08813064690765735, "top1_agree": 0.9091575091575091 }, { "id": "prose-qualification-holdout-1", "domain": "prose", "tokens": 4095, "nll": 3.4873519394423935, "teacher_nll": 3.4886688269683876, "kl": 0.1052800552312271, "top1_agree": 0.904029304029304 }, { "id": "prose-qualification-holdout-2", "domain": "prose", "tokens": 4095, "nll": 3.590019969451122, "teacher_nll": 3.586854711586157, "kl": 0.08358880894233602, "top1_agree": 0.9145299145299145 }, { "id": "prose-qualification-holdout-3", "domain": "prose", "tokens": 4095, "nll": 3.5505374149234012, "teacher_nll": 3.56058346255795, "kl": 0.09522298871379195, "top1_agree": 0.904029304029304 }, { "id": "code-qualification-holdout-0", "domain": "code", "tokens": 4095, "nll": 1.6449129926707136, "teacher_nll": 1.612040780315469, "kl": 0.08681660701358129, "top1_agree": 0.947008547008547 }, { "id": "code-qualification-holdout-1", "domain": "code", "tokens": 4095, "nll": 1.5195604561886071, "teacher_nll": 1.4962344744848826, "kl": 0.08596276142731109, "top1_agree": 0.9538461538461539 }, { "id": "code-qualification-holdout-2", "domain": "code", "tokens": 4095, "nll": 1.4590187482635908, "teacher_nll": 1.4386393615759918, "kl": 0.053211267345263116, "top1_agree": 0.9540903540903541 }, { "id": "code-qualification-holdout-3", "domain": "code", "tokens": 4095, "nll": 1.7863500604408273, "teacher_nll": 1.796362640744164, "kl": 0.08194250264301696, "top1_agree": 0.9433455433455433 }, { "id": "math-qualification-holdout-0", "domain": "math", "tokens": 4095, "nll": 1.4171055084619766, "teacher_nll": 1.4128543704711765, "kl": 0.019379515520163943, "top1_agree": 0.958974358974359 }, { "id": "math-qualification-holdout-1", "domain": "math", "tokens": 4095, "nll": 1.5520651210649838, "teacher_nll": 1.545063196724879, "kl": 0.019911735824176244, "top1_agree": 0.9594627594627595 }, { "id": "math-qualification-holdout-2", "domain": "math", "tokens": 4095, "nll": 1.5341272659208602, "teacher_nll": 1.5522358872223831, "kl": 0.01809579213314732, "top1_agree": 0.9631257631257631 }, { "id": "math-qualification-holdout-3", "domain": "math", "tokens": 4095, "nll": 1.5338709168672853, "teacher_nll": 1.516595822987539, "kl": 0.017875505410707915, "top1_agree": 0.9641025641025641 } ], "memory": { "swap_kib": 0, "cuda_peak_bytes": 14954638336 } } } }