Publish 35.74 GB FP8/NVFP4 policy with held-out quality, exact export replay and generation checks
95e7465 verified | { | |
| "complete": true, | |
| "split": "validation", | |
| "provenance": { | |
| "source": { | |
| "repo": "RecursiveSI/flint", | |
| "revision": "2a247be0c0aec895ccb0ac7ca92229ec6df37288" | |
| }, | |
| "scripts": { | |
| "flint_quantize.py": "3f87c9cf1bed5a2346ebc085579a82e14a236325b0b2277171dfae1d07d7b692", | |
| "flint_reference.py": "2534d51af8e99699692f1a2cf5fc022bd7beaa7de07770dbffb9690aa4f62ff6", | |
| "flint_nvfp4.py": "31aec4a310b7bddaa90da083b80352e5607ce754c10052458859f7306b5cc566" | |
| }, | |
| "packages": { | |
| "torch": "2.8.0+cu128", | |
| "triton": "3.4.0", | |
| "safetensors": "0.8.0", | |
| "transformers": "5.20.2", | |
| "huggingface_hub": "2.2.0", | |
| "datasets": "5.1.0" | |
| }, | |
| "gpu": "NVIDIA RTX PRO 6000 Blackwell Server Edition", | |
| "cuda": "12.8", | |
| "created_utc": "2026-10-21T23:38:51Z" | |
| }, | |
| "script_sha256": "e0e0aa655a97d9c78837196a94b280c54944ab5542686242dbaa7604d33dca1c", | |
| "normalization_script_sha256": "f06c3d51f9018814648a1fe42c39274dd103528b30185d914f76adbd68fef3ac", | |
| "flashinfer_version": "0.6.16.post1", | |
| "corpus_sha256": "ad58c37420891d4c5b08181aace2e75f00ac3f8a3253269b89a0f8ee88418923", | |
| "results": { | |
| "bf16": { | |
| "precision_by_layer": { | |
| "2": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "3": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "4": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "5": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "6": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "7": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "8": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "9": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "10": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "11": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "12": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "13": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "14": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "15": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "16": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "17": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "18": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "19": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "20": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "21": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "22": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "23": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "24": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "25": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "26": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "27": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "28": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "29": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "30": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "31": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "32": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "33": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "34": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "35": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "36": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "37": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "38": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "39": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "40": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "41": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "42": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "43": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "44": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "45": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "46": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "47": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "48": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| } | |
| }, | |
| "tensor_bytes": 79570574912, | |
| "aggregate": { | |
| "all": { | |
| "tokens": 49140, | |
| "nll": 2.214729754197971, | |
| "teacher_nll": 2.214729754197971, | |
| "kl": 0.0, | |
| "top1_agree": 1.0, | |
| "perplexity": 9.158933613767237, | |
| "teacher_perplexity": 9.158933613767237, | |
| "delta_nll": 0.0 | |
| }, | |
| "code": { | |
| "tokens": 16380, | |
| "nll": 1.585819314280127, | |
| "teacher_nll": 1.585819314280127, | |
| "kl": 0.0, | |
| "top1_agree": 1.0, | |
| "perplexity": 4.883290689738726, | |
| "teacher_perplexity": 4.883290689738726, | |
| "delta_nll": 0.0 | |
| }, | |
| "math": { | |
| "tokens": 16380, | |
| "nll": 1.5066873193514945, | |
| "teacher_nll": 1.5066873193514945, | |
| "kl": 0.0, | |
| "top1_agree": 1.0, | |
| "perplexity": 4.511759991246355, | |
| "teacher_perplexity": 4.511759991246355, | |
| "delta_nll": 0.0 | |
| }, | |
| "prose": { | |
| "tokens": 16380, | |
| "nll": 3.551682628962292, | |
| "teacher_nll": 3.551682628962292, | |
| "kl": 0.0, | |
| "top1_agree": 1.0, | |
| "perplexity": 34.871944693965524, | |
| "teacher_perplexity": 34.871944693965524, | |
| "delta_nll": 0.0 | |
| } | |
| }, | |
| "sequences": [ | |
| { | |
| "id": "prose-qualification-holdout-0", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.570623514736674, | |
| "teacher_nll": 3.570623514736674, | |
| "kl": 0.0, | |
| "top1_agree": 1.0 | |
| }, | |
| { | |
| "id": "prose-qualification-holdout-1", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.4886688269683876, | |
| "teacher_nll": 3.4886688269683876, | |
| "kl": 0.0, | |
| "top1_agree": 1.0 | |
| }, | |
| { | |
| "id": "prose-qualification-holdout-2", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.586854711586157, | |
| "teacher_nll": 3.586854711586157, | |
| "kl": 0.0, | |
| "top1_agree": 1.0 | |
| }, | |
| { | |
| "id": "prose-qualification-holdout-3", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.56058346255795, | |
| "teacher_nll": 3.56058346255795, | |
| "kl": 0.0, | |
| "top1_agree": 1.0 | |
| }, | |
| { | |
| "id": "code-qualification-holdout-0", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.612040780315469, | |
| "teacher_nll": 1.612040780315469, | |
| "kl": 0.0, | |
| "top1_agree": 1.0 | |
| }, | |
| { | |
| "id": "code-qualification-holdout-1", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.4962344744848826, | |
| "teacher_nll": 1.4962344744848826, | |
| "kl": 0.0, | |
| "top1_agree": 1.0 | |
| }, | |
| { | |
| "id": "code-qualification-holdout-2", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.4386393615759918, | |
| "teacher_nll": 1.4386393615759918, | |
| "kl": 0.0, | |
| "top1_agree": 1.0 | |
| }, | |
| { | |
| "id": "code-qualification-holdout-3", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.796362640744164, | |
| "teacher_nll": 1.796362640744164, | |
| "kl": 0.0, | |
| "top1_agree": 1.0 | |
| }, | |
| { | |
| "id": "math-qualification-holdout-0", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.4128543704711765, | |
| "teacher_nll": 1.4128543704711765, | |
| "kl": 0.0, | |
| "top1_agree": 1.0 | |
| }, | |
| { | |
| "id": "math-qualification-holdout-1", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.545063196724879, | |
| "teacher_nll": 1.545063196724879, | |
| "kl": 0.0, | |
| "top1_agree": 1.0 | |
| }, | |
| { | |
| "id": "math-qualification-holdout-2", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.5522358872223831, | |
| "teacher_nll": 1.5522358872223831, | |
| "kl": 0.0, | |
| "top1_agree": 1.0 | |
| }, | |
| { | |
| "id": "math-qualification-holdout-3", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.516595822987539, | |
| "teacher_nll": 1.516595822987539, | |
| "kl": 0.0, | |
| "top1_agree": 1.0 | |
| } | |
| ], | |
| "memory": { | |
| "swap_kib": 0, | |
| "cuda_peak_bytes": 14954638336 | |
| } | |
| }, | |
| "all-fp8": { | |
| "precision_by_layer": { | |
| "2": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "3": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "4": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "5": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "6": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "7": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "8": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "9": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "10": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "11": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "12": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "13": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "14": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "15": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "16": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "17": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "18": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "19": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "20": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "21": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "22": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "23": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "24": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "25": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "26": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "27": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "28": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "29": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "30": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "31": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "32": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "33": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "34": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "35": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "36": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "37": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "38": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "39": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "40": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "41": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "42": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "43": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "44": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "45": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "46": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "47": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "48": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| } | |
| }, | |
| "tensor_bytes": 41730416192, | |
| "aggregate": { | |
| "all": { | |
| "tokens": 49140, | |
| "nll": 2.21773667151141, | |
| "teacher_nll": 2.214729754197971, | |
| "kl": 0.07243157411187487, | |
| "top1_agree": 0.9358974358974359, | |
| "perplexity": 9.186515216841316, | |
| "teacher_perplexity": 9.158933613767237, | |
| "delta_nll": 0.0030069173134390503 | |
| }, | |
| "code": { | |
| "tokens": 16380, | |
| "nll": 1.6008225211002596, | |
| "teacher_nll": 1.585819314280127, | |
| "kl": 0.09259340757861169, | |
| "top1_agree": 0.9469474969474969, | |
| "perplexity": 4.957108073998569, | |
| "teacher_perplexity": 4.883290689738726, | |
| "delta_nll": 0.015003206820132675 | |
| }, | |
| "math": { | |
| "tokens": 16380, | |
| "nll": 1.5014515739948613, | |
| "teacher_nll": 1.5066873193514945, | |
| "kl": 0.021969766698331915, | |
| "top1_agree": 0.9597069597069597, | |
| "perplexity": 4.488199297540752, | |
| "teacher_perplexity": 4.511759991246355, | |
| "delta_nll": -0.005235745356633181 | |
| }, | |
| "prose": { | |
| "tokens": 16380, | |
| "nll": 3.5509359194391084, | |
| "teacher_nll": 3.551682628962292, | |
| "kl": 0.102731548058681, | |
| "top1_agree": 0.9010378510378511, | |
| "perplexity": 34.84591520021549, | |
| "teacher_perplexity": 34.871944693965524, | |
| "delta_nll": -0.0007467095231836751 | |
| } | |
| }, | |
| "sequences": [ | |
| { | |
| "id": "prose-qualification-holdout-0", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.5883770301082922, | |
| "teacher_nll": 3.570623514736674, | |
| "kl": 0.1001550619127695, | |
| "top1_agree": 0.8998778998778999 | |
| }, | |
| { | |
| "id": "prose-qualification-holdout-1", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.471363538059705, | |
| "teacher_nll": 3.4886688269683876, | |
| "kl": 0.1251851178787567, | |
| "top1_agree": 0.8981684981684982 | |
| }, | |
| { | |
| "id": "prose-qualification-holdout-2", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.5789717938582686, | |
| "teacher_nll": 3.586854711586157, | |
| "kl": 0.08583091795808376, | |
| "top1_agree": 0.9091575091575091 | |
| }, | |
| { | |
| "id": "prose-qualification-holdout-3", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.565031315730168, | |
| "teacher_nll": 3.56058346255795, | |
| "kl": 0.09975509448511405, | |
| "top1_agree": 0.896947496947497 | |
| }, | |
| { | |
| "id": "code-qualification-holdout-0", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.6328039013276898, | |
| "teacher_nll": 1.612040780315469, | |
| "kl": 0.10738086557795859, | |
| "top1_agree": 0.9391941391941392 | |
| }, | |
| { | |
| "id": "code-qualification-holdout-1", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.5051212597242642, | |
| "teacher_nll": 1.4962344744848826, | |
| "kl": 0.09163388843683387, | |
| "top1_agree": 0.9538461538461539 | |
| }, | |
| { | |
| "id": "code-qualification-holdout-2", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.4643761817643348, | |
| "teacher_nll": 1.4386393615759918, | |
| "kl": 0.07029858778102348, | |
| "top1_agree": 0.9528693528693528 | |
| }, | |
| { | |
| "id": "code-qualification-holdout-3", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.8009887415847499, | |
| "teacher_nll": 1.796362640744164, | |
| "kl": 0.10106028851863083, | |
| "top1_agree": 0.9418803418803419 | |
| }, | |
| { | |
| "id": "math-qualification-holdout-0", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.405975469652113, | |
| "teacher_nll": 1.4128543704711765, | |
| "kl": 0.023144839301941885, | |
| "top1_agree": 0.957997557997558 | |
| }, | |
| { | |
| "id": "math-qualification-holdout-1", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.5458229023021657, | |
| "teacher_nll": 1.545063196724879, | |
| "kl": 0.020068688932647055, | |
| "top1_agree": 0.95995115995116 | |
| }, | |
| { | |
| "id": "math-qualification-holdout-2", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.5160463214357258, | |
| "teacher_nll": 1.5522358872223831, | |
| "kl": 0.02173219574240101, | |
| "top1_agree": 0.9631257631257631 | |
| }, | |
| { | |
| "id": "math-qualification-holdout-3", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.5379616025894407, | |
| "teacher_nll": 1.516595822987539, | |
| "kl": 0.022933342816337708, | |
| "top1_agree": 0.9577533577533578 | |
| } | |
| ], | |
| "memory": { | |
| "swap_kib": 0, | |
| "cuda_peak_bytes": 14954638336 | |
| } | |
| }, | |
| "all-nvfp4": { | |
| "precision_by_layer": { | |
| "2": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "3": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "4": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "5": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "6": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "7": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "8": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "9": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "10": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "11": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "12": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "13": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "14": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "15": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "16": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "17": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "18": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "19": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "20": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "21": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "22": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "23": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "24": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "25": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "26": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "27": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "28": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "29": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "30": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "31": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "32": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "33": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "34": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "35": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "36": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "37": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "38": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "39": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "40": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "41": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "42": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "43": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "44": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "45": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "46": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "47": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "48": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| } | |
| }, | |
| "tensor_bytes": 25162207808, | |
| "aggregate": { | |
| "all": { | |
| "tokens": 49140, | |
| "nll": 2.2190047702218734, | |
| "teacher_nll": 2.214729754197971, | |
| "kl": 0.15325156021552402, | |
| "top1_agree": 0.8925315425315425, | |
| "perplexity": 9.198172014364316, | |
| "teacher_perplexity": 9.158933613767237, | |
| "delta_nll": 0.004275016023902456 | |
| }, | |
| "code": { | |
| "tokens": 16380, | |
| "nll": 1.5995975865112557, | |
| "teacher_nll": 1.585819314280127, | |
| "kl": 0.16933766659508404, | |
| "top1_agree": 0.9105616605616605, | |
| "perplexity": 4.951039658322216, | |
| "teacher_perplexity": 4.883290689738726, | |
| "delta_nll": 0.013778272231128819 | |
| }, | |
| "math": { | |
| "tokens": 16380, | |
| "nll": 1.5143714170461755, | |
| "teacher_nll": 1.5066873193514945, | |
| "kl": 0.06676194644345469, | |
| "top1_agree": 0.9306471306471307, | |
| "perplexity": 4.546562336862997, | |
| "teacher_perplexity": 4.511759991246355, | |
| "delta_nll": 0.007684097694681036 | |
| }, | |
| "prose": { | |
| "tokens": 16380, | |
| "nll": 3.5430453071081884, | |
| "teacher_nll": 3.551682628962292, | |
| "kl": 0.22365506760803333, | |
| "top1_agree": 0.8363858363858364, | |
| "perplexity": 34.57204152852661, | |
| "teacher_perplexity": 34.871944693965524, | |
| "delta_nll": -0.008637321854103597 | |
| } | |
| }, | |
| "sequences": [ | |
| { | |
| "id": "prose-qualification-holdout-0", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.587969711732224, | |
| "teacher_nll": 3.570623514736674, | |
| "kl": 0.22323825251634014, | |
| "top1_agree": 0.842002442002442 | |
| }, | |
| { | |
| "id": "prose-qualification-holdout-1", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.4460585225050555, | |
| "teacher_nll": 3.4886688269683876, | |
| "kl": 0.22454096511170105, | |
| "top1_agree": 0.8368742368742369 | |
| }, | |
| { | |
| "id": "prose-qualification-holdout-2", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.5779974633520775, | |
| "teacher_nll": 3.586854711586157, | |
| "kl": 0.22193228753058464, | |
| "top1_agree": 0.8356532356532357 | |
| }, | |
| { | |
| "id": "prose-qualification-holdout-3", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.560155530843397, | |
| "teacher_nll": 3.56058346255795, | |
| "kl": 0.22490876527350756, | |
| "top1_agree": 0.8310134310134311 | |
| }, | |
| { | |
| "id": "code-qualification-holdout-0", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.6565769824352894, | |
| "teacher_nll": 1.612040780315469, | |
| "kl": 0.19423157339131003, | |
| "top1_agree": 0.9030525030525031 | |
| }, | |
| { | |
| "id": "code-qualification-holdout-1", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.4670974128848904, | |
| "teacher_nll": 1.4962344744848826, | |
| "kl": 0.15822774912702586, | |
| "top1_agree": 0.9208791208791208 | |
| }, | |
| { | |
| "id": "code-qualification-holdout-2", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.4386256086375104, | |
| "teacher_nll": 1.4386393615759918, | |
| "kl": 0.13105918493608562, | |
| "top1_agree": 0.9111111111111111 | |
| }, | |
| { | |
| "id": "code-qualification-holdout-3", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.8360903420873322, | |
| "teacher_nll": 1.796362640744164, | |
| "kl": 0.19383215892591465, | |
| "top1_agree": 0.9072039072039072 | |
| }, | |
| { | |
| "id": "math-qualification-holdout-0", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.403938700486161, | |
| "teacher_nll": 1.4128543704711765, | |
| "kl": 0.06598033006229098, | |
| "top1_agree": 0.9296703296703297 | |
| }, | |
| { | |
| "id": "math-qualification-holdout-1", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.5686517051724724, | |
| "teacher_nll": 1.545063196724879, | |
| "kl": 0.07200618306213537, | |
| "top1_agree": 0.9252747252747253 | |
| }, | |
| { | |
| "id": "math-qualification-holdout-2", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.5351991115472257, | |
| "teacher_nll": 1.5522358872223831, | |
| "kl": 0.06542188856336806, | |
| "top1_agree": 0.9328449328449329 | |
| }, | |
| { | |
| "id": "math-qualification-holdout-3", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.549696150978843, | |
| "teacher_nll": 1.516595822987539, | |
| "kl": 0.06363938408602433, | |
| "top1_agree": 0.9347985347985348 | |
| } | |
| ], | |
| "memory": { | |
| "swap_kib": 0, | |
| "cuda_peak_bytes": 14954638336 | |
| } | |
| }, | |
| "published-mix": { | |
| "precision_by_layer": { | |
| "2": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "3": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "4": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "5": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "6": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "7": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "8": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "9": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "10": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "11": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "12": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "13": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "14": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "15": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "16": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "17": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "18": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "19": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "20": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "21": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "22": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "23": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "24": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "25": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "26": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "27": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "28": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "29": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "30": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "31": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "32": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "33": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "34": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "35": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "36": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "37": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "38": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "39": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "40": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "41": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "42": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "43": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "44": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "45": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "46": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "47": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "48": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| } | |
| }, | |
| "tensor_bytes": 30802448960, | |
| "aggregate": { | |
| "all": { | |
| "tokens": 49140, | |
| "nll": 2.2032232530436286, | |
| "teacher_nll": 2.214729754197971, | |
| "kl": 0.11295873393427079, | |
| "top1_agree": 0.9065730565730565, | |
| "perplexity": 9.054150334134441, | |
| "teacher_perplexity": 9.158933613767237, | |
| "delta_nll": -0.011506501154342264 | |
| }, | |
| "code": { | |
| "tokens": 16380, | |
| "nll": 1.564784675727397, | |
| "teacher_nll": 1.585819314280127, | |
| "kl": 0.13348741904174016, | |
| "top1_agree": 0.9209401709401709, | |
| "perplexity": 4.781645220758911, | |
| "teacher_perplexity": 4.883290689738726, | |
| "delta_nll": -0.0210346385527298 | |
| }, | |
| "math": { | |
| "tokens": 16380, | |
| "nll": 1.5041985815406864, | |
| "teacher_nll": 1.5066873193514945, | |
| "kl": 0.05023271923700532, | |
| "top1_agree": 0.9394993894993895, | |
| "perplexity": 4.500545364484122, | |
| "teacher_perplexity": 4.511759991246355, | |
| "delta_nll": -0.0024887378108080505 | |
| }, | |
| "prose": { | |
| "tokens": 16380, | |
| "nll": 3.540686501862802, | |
| "teacher_nll": 3.551682628962292, | |
| "kl": 0.15515606352406688, | |
| "top1_agree": 0.8592796092796092, | |
| "perplexity": 34.49058891881352, | |
| "teacher_perplexity": 34.871944693965524, | |
| "delta_nll": -0.01099612709949005 | |
| } | |
| }, | |
| "sequences": [ | |
| { | |
| "id": "prose-qualification-holdout-0", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.5968923231038707, | |
| "teacher_nll": 3.570623514736674, | |
| "kl": 0.14627862587953225, | |
| "top1_agree": 0.8632478632478633 | |
| }, | |
| { | |
| "id": "prose-qualification-holdout-1", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.4596483652117196, | |
| "teacher_nll": 3.4886688269683876, | |
| "kl": 0.17513312973236717, | |
| "top1_agree": 0.8542124542124542 | |
| }, | |
| { | |
| "id": "prose-qualification-holdout-2", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.5596329014816566, | |
| "teacher_nll": 3.586854711586157, | |
| "kl": 0.13868644857581283, | |
| "top1_agree": 0.8634920634920635 | |
| }, | |
| { | |
| "id": "prose-qualification-holdout-3", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.5465724176539606, | |
| "teacher_nll": 3.56058346255795, | |
| "kl": 0.16052604990855532, | |
| "top1_agree": 0.8561660561660561 | |
| }, | |
| { | |
| "id": "code-qualification-holdout-0", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.6175182463630797, | |
| "teacher_nll": 1.612040780315469, | |
| "kl": 0.14147204016591167, | |
| "top1_agree": 0.9177045177045177 | |
| }, | |
| { | |
| "id": "code-qualification-holdout-1", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.438487123598837, | |
| "teacher_nll": 1.4962344744848826, | |
| "kl": 0.13281468772335076, | |
| "top1_agree": 0.9274725274725275 | |
| }, | |
| { | |
| "id": "code-qualification-holdout-2", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.4048096974690756, | |
| "teacher_nll": 1.4386393615759918, | |
| "kl": 0.11024958886506357, | |
| "top1_agree": 0.9250305250305251 | |
| }, | |
| { | |
| "id": "code-qualification-holdout-3", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.798323635478596, | |
| "teacher_nll": 1.796362640744164, | |
| "kl": 0.14941335941263462, | |
| "top1_agree": 0.9135531135531135 | |
| }, | |
| { | |
| "id": "math-qualification-holdout-0", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.3987592498024741, | |
| "teacher_nll": 1.4128543704711765, | |
| "kl": 0.05078120037676796, | |
| "top1_agree": 0.9438339438339438 | |
| }, | |
| { | |
| "id": "math-qualification-holdout-1", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.5469760289267889, | |
| "teacher_nll": 1.545063196724879, | |
| "kl": 0.05077969653120262, | |
| "top1_agree": 0.9347985347985348 | |
| }, | |
| { | |
| "id": "math-qualification-holdout-2", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.5395388960692764, | |
| "teacher_nll": 1.5522358872223831, | |
| "kl": 0.051088059443402785, | |
| "top1_agree": 0.9384615384615385 | |
| }, | |
| { | |
| "id": "math-qualification-holdout-3", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.5315201513642065, | |
| "teacher_nll": 1.516595822987539, | |
| "kl": 0.048281920596647904, | |
| "top1_agree": 0.9409035409035409 | |
| } | |
| ], | |
| "memory": { | |
| "swap_kib": 0, | |
| "cuda_peak_bytes": 14954638336 | |
| } | |
| }, | |
| "budget28-swap1": { | |
| "precision_by_layer": { | |
| "2": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "3": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "4": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "5": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "6": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "7": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "8": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "9": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "10": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "11": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "12": { | |
| "gate": "fp8", | |
| "up": "nvfp4", | |
| "down": "fp8" | |
| }, | |
| "13": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "14": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "15": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "16": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "17": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "18": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "19": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "20": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "21": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "22": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "23": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "24": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "25": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "26": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "27": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "28": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "29": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "30": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "31": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "32": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "33": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "34": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "35": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "36": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "37": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "fp8" | |
| }, | |
| "38": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "39": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "40": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "41": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "42": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "43": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "44": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "45": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "46": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "47": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "48": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| } | |
| }, | |
| "tensor_bytes": 27982328384, | |
| "aggregate": { | |
| "all": { | |
| "tokens": 49140, | |
| "nll": 2.2250885697923932, | |
| "teacher_nll": 2.214729754197971, | |
| "kl": 0.1387744848271866, | |
| "top1_agree": 0.8979039479039479, | |
| "perplexity": 9.254302419253793, | |
| "teacher_perplexity": 9.158933613767237, | |
| "delta_nll": 0.01035881559442231 | |
| }, | |
| "code": { | |
| "tokens": 16380, | |
| "nll": 1.6103557565418938, | |
| "teacher_nll": 1.585819314280127, | |
| "kl": 0.1575198637551295, | |
| "top1_agree": 0.9144078144078144, | |
| "perplexity": 5.004591327277986, | |
| "teacher_perplexity": 4.883290689738726, | |
| "delta_nll": 0.024536442261766878 | |
| }, | |
| "math": { | |
| "tokens": 16380, | |
| "nll": 1.515096839465501, | |
| "teacher_nll": 1.5066873193514945, | |
| "kl": 0.060545078656830634, | |
| "top1_agree": 0.9348595848595849, | |
| "perplexity": 4.549861711688565, | |
| "teacher_perplexity": 4.511759991246355, | |
| "delta_nll": 0.008409520114006641 | |
| }, | |
| "prose": { | |
| "tokens": 16380, | |
| "nll": 3.5498131133697846, | |
| "teacher_nll": 3.551682628962292, | |
| "kl": 0.19825851206959968, | |
| "top1_agree": 0.8444444444444444, | |
| "perplexity": 34.8068119519279, | |
| "teacher_perplexity": 34.871944693965524, | |
| "delta_nll": -0.0018695155925074758 | |
| } | |
| }, | |
| "sequences": [ | |
| { | |
| "id": "prose-qualification-holdout-0", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.5897965204148066, | |
| "teacher_nll": 3.570623514736674, | |
| "kl": 0.18998560713324356, | |
| "top1_agree": 0.8451770451770452 | |
| }, | |
| { | |
| "id": "prose-qualification-holdout-1", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.454184816986941, | |
| "teacher_nll": 3.4886688269683876, | |
| "kl": 0.21330270371303162, | |
| "top1_agree": 0.8378510378510379 | |
| }, | |
| { | |
| "id": "prose-qualification-holdout-2", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.587775377359728, | |
| "teacher_nll": 3.586854711586157, | |
| "kl": 0.1919633207449255, | |
| "top1_agree": 0.8515262515262515 | |
| }, | |
| { | |
| "id": "prose-qualification-holdout-3", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.5674957387176627, | |
| "teacher_nll": 3.56058346255795, | |
| "kl": 0.19778241668719804, | |
| "top1_agree": 0.8432234432234432 | |
| }, | |
| { | |
| "id": "code-qualification-holdout-0", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.659486316033481, | |
| "teacher_nll": 1.612040780315469, | |
| "kl": 0.18177173958593235, | |
| "top1_agree": 0.9081807081807082 | |
| }, | |
| { | |
| "id": "code-qualification-holdout-1", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.5020296731420197, | |
| "teacher_nll": 1.4962344744848826, | |
| "kl": 0.15101925459245769, | |
| "top1_agree": 0.9218559218559218 | |
| }, | |
| { | |
| "id": "code-qualification-holdout-2", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.4350260393377916, | |
| "teacher_nll": 1.4386393615759918, | |
| "kl": 0.1277437409347376, | |
| "top1_agree": 0.9186813186813186 | |
| }, | |
| { | |
| "id": "code-qualification-holdout-3", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.8448809976542826, | |
| "teacher_nll": 1.796362640744164, | |
| "kl": 0.16954471990739034, | |
| "top1_agree": 0.9089133089133089 | |
| }, | |
| { | |
| "id": "math-qualification-holdout-0", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.4089843853343829, | |
| "teacher_nll": 1.4128543704711765, | |
| "kl": 0.06082202776884421, | |
| "top1_agree": 0.9347985347985348 | |
| }, | |
| { | |
| "id": "math-qualification-holdout-1", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.5700943328522063, | |
| "teacher_nll": 1.545063196724879, | |
| "kl": 0.06191048036273728, | |
| "top1_agree": 0.9291819291819292 | |
| }, | |
| { | |
| "id": "math-qualification-holdout-2", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.5385928818885515, | |
| "teacher_nll": 1.5522358872223831, | |
| "kl": 0.06146017515324557, | |
| "top1_agree": 0.938949938949939 | |
| }, | |
| { | |
| "id": "math-qualification-holdout-3", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.5427157577868638, | |
| "teacher_nll": 1.516595822987539, | |
| "kl": 0.05798763134249546, | |
| "top1_agree": 0.9365079365079365 | |
| } | |
| ], | |
| "memory": { | |
| "swap_kib": 0, | |
| "cuda_peak_bytes": 14954638336 | |
| } | |
| }, | |
| "budget32-conditional": { | |
| "precision_by_layer": { | |
| "2": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "3": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "4": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "5": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "6": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "7": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "8": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "9": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "10": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "11": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "12": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "13": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "14": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "15": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "16": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "17": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "18": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "19": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "20": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "21": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "22": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "23": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "24": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "25": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "26": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "27": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "28": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "29": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "30": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "31": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "32": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "33": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "34": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "35": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "36": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "37": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "38": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "39": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "40": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "41": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "42": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "43": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "44": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "45": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "46": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "47": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "48": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| } | |
| }, | |
| "tensor_bytes": 31859994176, | |
| "aggregate": { | |
| "all": { | |
| "tokens": 49140, | |
| "nll": 2.210038530593681, | |
| "teacher_nll": 2.214729754197971, | |
| "kl": 0.11078792841355273, | |
| "top1_agree": 0.908974358974359, | |
| "perplexity": 9.11606763377147, | |
| "teacher_perplexity": 9.158933613767237, | |
| "delta_nll": -0.0046912236042899735 | |
| }, | |
| "code": { | |
| "tokens": 16380, | |
| "nll": 1.578945990272494, | |
| "teacher_nll": 1.585819314280127, | |
| "kl": 0.1256440419321641, | |
| "top1_agree": 0.9242979242979243, | |
| "perplexity": 4.849841336412374, | |
| "teacher_perplexity": 4.883290689738726, | |
| "delta_nll": -0.006873324007632853 | |
| }, | |
| "math": { | |
| "tokens": 16380, | |
| "nll": 1.5118221361704072, | |
| "teacher_nll": 1.5066873193514945, | |
| "kl": 0.049465113265101, | |
| "top1_agree": 0.9401709401709402, | |
| "perplexity": 4.534986633575697, | |
| "teacher_perplexity": 4.511759991246355, | |
| "delta_nll": 0.005134816818912702 | |
| }, | |
| "prose": { | |
| "tokens": 16380, | |
| "nll": 3.539347465338142, | |
| "teacher_nll": 3.551682628962292, | |
| "kl": 0.1572546300433931, | |
| "top1_agree": 0.8624542124542125, | |
| "perplexity": 34.44443566783473, | |
| "teacher_perplexity": 34.871944693965524, | |
| "delta_nll": -0.012335163624149992 | |
| } | |
| }, | |
| "sequences": [ | |
| { | |
| "id": "prose-qualification-holdout-0", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.6023279863108355, | |
| "teacher_nll": 3.570623514736674, | |
| "kl": 0.147485230228112, | |
| "top1_agree": 0.8656898656898657 | |
| }, | |
| { | |
| "id": "prose-qualification-holdout-1", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.46047611446171, | |
| "teacher_nll": 3.4886688269683876, | |
| "kl": 0.1644654834081256, | |
| "top1_agree": 0.8598290598290599 | |
| }, | |
| { | |
| "id": "prose-qualification-holdout-2", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.5554149548503795, | |
| "teacher_nll": 3.586854711586157, | |
| "kl": 0.15823070167476294, | |
| "top1_agree": 0.864957264957265 | |
| }, | |
| { | |
| "id": "prose-qualification-holdout-3", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.539170805729644, | |
| "teacher_nll": 3.56058346255795, | |
| "kl": 0.15883710486257177, | |
| "top1_agree": 0.8593406593406593 | |
| }, | |
| { | |
| "id": "code-qualification-holdout-0", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.6173301612937843, | |
| "teacher_nll": 1.612040780315469, | |
| "kl": 0.14581474144380172, | |
| "top1_agree": 0.9194139194139194 | |
| }, | |
| { | |
| "id": "code-qualification-holdout-1", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.4841609367957482, | |
| "teacher_nll": 1.4962344744848826, | |
| "kl": 0.12464202101523186, | |
| "top1_agree": 0.9326007326007326 | |
| }, | |
| { | |
| "id": "code-qualification-holdout-2", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.4110373812571841, | |
| "teacher_nll": 1.4386393615759918, | |
| "kl": 0.09008101636819059, | |
| "top1_agree": 0.926007326007326 | |
| }, | |
| { | |
| "id": "code-qualification-holdout-3", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.8032554817432596, | |
| "teacher_nll": 1.796362640744164, | |
| "kl": 0.14203838890143222, | |
| "top1_agree": 0.9191697191697191 | |
| }, | |
| { | |
| "id": "math-qualification-holdout-0", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.4158305093045636, | |
| "teacher_nll": 1.4128543704711765, | |
| "kl": 0.04934141346240946, | |
| "top1_agree": 0.9409035409035409 | |
| }, | |
| { | |
| "id": "math-qualification-holdout-1", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.54967892033131, | |
| "teacher_nll": 1.545063196724879, | |
| "kl": 0.04675215886626052, | |
| "top1_agree": 0.9374847374847375 | |
| }, | |
| { | |
| "id": "math-qualification-holdout-2", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.5434759258787274, | |
| "teacher_nll": 1.5522358872223831, | |
| "kl": 0.0511482006136751, | |
| "top1_agree": 0.9416361416361416 | |
| }, | |
| { | |
| "id": "math-qualification-holdout-3", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.5383031891670274, | |
| "teacher_nll": 1.516595822987539, | |
| "kl": 0.050618680118058916, | |
| "top1_agree": 0.9406593406593406 | |
| } | |
| ], | |
| "memory": { | |
| "swap_kib": 0, | |
| "cuda_peak_bytes": 14954638336 | |
| } | |
| }, | |
| "budget36-swap1": { | |
| "precision_by_layer": { | |
| "2": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "3": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "4": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "5": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "6": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "7": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "8": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "9": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "10": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "11": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "12": { | |
| "gate": "fp8", | |
| "up": "nvfp4", | |
| "down": "fp8" | |
| }, | |
| "13": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "14": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "15": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "16": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "17": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "18": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "19": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "20": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "21": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "22": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "23": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "24": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "25": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "26": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "27": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "28": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "29": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "30": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "31": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "32": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "33": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "34": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "35": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "36": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "37": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "fp8" | |
| }, | |
| "38": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "39": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "40": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "41": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "42": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "43": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "44": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "45": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "46": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "47": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "48": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| } | |
| }, | |
| "tensor_bytes": 35737659968, | |
| "aggregate": { | |
| "all": { | |
| "tokens": 49140, | |
| "nll": 2.219416871241161, | |
| "teacher_nll": 2.214729754197971, | |
| "kl": 0.09010865478740386, | |
| "top1_agree": 0.9188848188848189, | |
| "perplexity": 9.201963371584448, | |
| "teacher_perplexity": 9.158933613767237, | |
| "delta_nll": 0.0046871170431899145 | |
| }, | |
| "code": { | |
| "tokens": 16380, | |
| "nll": 1.5947170897863432, | |
| "teacher_nll": 1.585819314280127, | |
| "kl": 0.10190384458427537, | |
| "top1_agree": 0.9322344322344323, | |
| "perplexity": 4.92693499469703, | |
| "teacher_perplexity": 4.883290689738726, | |
| "delta_nll": 0.008897775506216288 | |
| }, | |
| "math": { | |
| "tokens": 16380, | |
| "nll": 1.5039469321350476, | |
| "teacher_nll": 1.5066873193514945, | |
| "kl": 0.03564856964480746, | |
| "top1_agree": 0.9473748473748473, | |
| "perplexity": 4.499412947410116, | |
| "teacher_perplexity": 4.511759991246355, | |
| "delta_nll": -0.0027403872164468712 | |
| }, | |
| "prose": { | |
| "tokens": 16380, | |
| "nll": 3.5595865918020917, | |
| "teacher_nll": 3.551682628962292, | |
| "kl": 0.13277355013312875, | |
| "top1_agree": 0.877045177045177, | |
| "perplexity": 35.14866339553476, | |
| "teacher_perplexity": 34.871944693965524, | |
| "delta_nll": 0.00790396283979966 | |
| } | |
| }, | |
| "sequences": [ | |
| { | |
| "id": "prose-qualification-holdout-0", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.60210935525114, | |
| "teacher_nll": 3.570623514736674, | |
| "kl": 0.1251914977445125, | |
| "top1_agree": 0.873992673992674 | |
| }, | |
| { | |
| "id": "prose-qualification-holdout-1", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.4887589190323567, | |
| "teacher_nll": 3.4886688269683876, | |
| "kl": 0.1547056760543432, | |
| "top1_agree": 0.8717948717948718 | |
| }, | |
| { | |
| "id": "prose-qualification-holdout-2", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.587463203774849, | |
| "teacher_nll": 3.586854711586157, | |
| "kl": 0.11346375461899753, | |
| "top1_agree": 0.8842490842490842 | |
| }, | |
| { | |
| "id": "prose-qualification-holdout-3", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.560014889150021, | |
| "teacher_nll": 3.56058346255795, | |
| "kl": 0.13773327211466174, | |
| "top1_agree": 0.8781440781440781 | |
| }, | |
| { | |
| "id": "code-qualification-holdout-0", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.634640592358488, | |
| "teacher_nll": 1.612040780315469, | |
| "kl": 0.11310800134917319, | |
| "top1_agree": 0.9255189255189256 | |
| }, | |
| { | |
| "id": "code-qualification-holdout-1", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.4733869531070212, | |
| "teacher_nll": 1.4962344744848826, | |
| "kl": 0.10195107398343174, | |
| "top1_agree": 0.9423687423687424 | |
| }, | |
| { | |
| "id": "code-qualification-holdout-2", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.4622409024081386, | |
| "teacher_nll": 1.4386393615759918, | |
| "kl": 0.0776867737409105, | |
| "top1_agree": 0.9374847374847375 | |
| }, | |
| { | |
| "id": "code-qualification-holdout-3", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.8085999112717253, | |
| "teacher_nll": 1.796362640744164, | |
| "kl": 0.11486952926358605, | |
| "top1_agree": 0.9235653235653236 | |
| }, | |
| { | |
| "id": "math-qualification-holdout-0", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.4042997626012352, | |
| "teacher_nll": 1.4128543704711765, | |
| "kl": 0.03463405605419215, | |
| "top1_agree": 0.9472527472527472 | |
| }, | |
| { | |
| "id": "math-qualification-holdout-1", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.5480228545173766, | |
| "teacher_nll": 1.545063196724879, | |
| "kl": 0.03576130380123963, | |
| "top1_agree": 0.9457875457875458 | |
| }, | |
| { | |
| "id": "math-qualification-holdout-2", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.5291209340823293, | |
| "teacher_nll": 1.5522358872223831, | |
| "kl": 0.03457320837965815, | |
| "top1_agree": 0.9474969474969475 | |
| }, | |
| { | |
| "id": "math-qualification-holdout-3", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.5343441773392488, | |
| "teacher_nll": 1.516595822987539, | |
| "kl": 0.03762571034413991, | |
| "top1_agree": 0.9489621489621489 | |
| } | |
| ], | |
| "memory": { | |
| "swap_kib": 0, | |
| "cuda_peak_bytes": 14954638336 | |
| } | |
| }, | |
| "budget40-swap5": { | |
| "precision_by_layer": { | |
| "2": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "3": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "4": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "nvfp4" | |
| }, | |
| "5": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "6": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "7": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "8": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "9": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "10": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "11": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "12": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "13": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "14": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "15": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "16": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "17": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "18": { | |
| "gate": "fp8", | |
| "up": "nvfp4", | |
| "down": "fp8" | |
| }, | |
| "19": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "20": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "21": { | |
| "gate": "nvfp4", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "22": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "23": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "24": { | |
| "gate": "nvfp4", | |
| "up": "fp8", | |
| "down": "nvfp4" | |
| }, | |
| "25": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "fp8" | |
| }, | |
| "26": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "27": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "28": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "29": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "30": { | |
| "gate": "nvfp4", | |
| "up": "fp8", | |
| "down": "nvfp4" | |
| }, | |
| "31": { | |
| "gate": "fp8", | |
| "up": "nvfp4", | |
| "down": "fp8" | |
| }, | |
| "32": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "33": { | |
| "gate": "nvfp4", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "34": { | |
| "gate": "fp8", | |
| "up": "nvfp4", | |
| "down": "fp8" | |
| }, | |
| "35": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "36": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "37": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "38": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "39": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "40": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "41": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "42": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "43": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "44": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "45": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "46": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "47": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "48": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| } | |
| }, | |
| "tensor_bytes": 39967840832, | |
| "aggregate": { | |
| "all": { | |
| "tokens": 49140, | |
| "nll": 2.2186658260683534, | |
| "teacher_nll": 2.214729754197971, | |
| "kl": 0.07798120706536613, | |
| "top1_agree": 0.9294261294261295, | |
| "perplexity": 9.195054876034725, | |
| "teacher_perplexity": 9.158933613767237, | |
| "delta_nll": 0.003936071870382474 | |
| }, | |
| "code": { | |
| "tokens": 16380, | |
| "nll": 1.5962667831571111, | |
| "teacher_nll": 1.585819314280127, | |
| "kl": 0.09327435911211193, | |
| "top1_agree": 0.9406593406593406, | |
| "perplexity": 4.934576152393011, | |
| "teacher_perplexity": 4.883290689738726, | |
| "delta_nll": 0.010447468876984223 | |
| }, | |
| "math": { | |
| "tokens": 16380, | |
| "nll": 1.5076653056965643, | |
| "teacher_nll": 1.5066873193514945, | |
| "kl": 0.027212030440114065, | |
| "top1_agree": 0.9548840048840049, | |
| "perplexity": 4.516174589266453, | |
| "teacher_perplexity": 4.511759991246355, | |
| "delta_nll": 0.0009779863450698478 | |
| }, | |
| "prose": { | |
| "tokens": 16380, | |
| "nll": 3.5520653893513847, | |
| "teacher_nll": 3.551682628962292, | |
| "kl": 0.11345723164387238, | |
| "top1_agree": 0.8927350427350428, | |
| "perplexity": 34.88529484787707, | |
| "teacher_perplexity": 34.871944693965524, | |
| "delta_nll": 0.0003827603890926845 | |
| } | |
| }, | |
| "sequences": [ | |
| { | |
| "id": "prose-qualification-holdout-0", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.599587125859709, | |
| "teacher_nll": 3.570623514736674, | |
| "kl": 0.11613119326813899, | |
| "top1_agree": 0.8925518925518926 | |
| }, | |
| { | |
| "id": "prose-qualification-holdout-1", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.4782555157448347, | |
| "teacher_nll": 3.4886688269683876, | |
| "kl": 0.1251666544412373, | |
| "top1_agree": 0.8927960927960928 | |
| }, | |
| { | |
| "id": "prose-qualification-holdout-2", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.572577172583276, | |
| "teacher_nll": 3.586854711586157, | |
| "kl": 0.1002090939672002, | |
| "top1_agree": 0.897924297924298 | |
| }, | |
| { | |
| "id": "prose-qualification-holdout-3", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.5578417432177196, | |
| "teacher_nll": 3.56058346255795, | |
| "kl": 0.11232198489891304, | |
| "top1_agree": 0.8876678876678876 | |
| }, | |
| { | |
| "id": "code-qualification-holdout-0", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.6341117285983466, | |
| "teacher_nll": 1.612040780315469, | |
| "kl": 0.10246821173235901, | |
| "top1_agree": 0.9374847374847375 | |
| }, | |
| { | |
| "id": "code-qualification-holdout-1", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.4997853697729053, | |
| "teacher_nll": 1.4962344744848826, | |
| "kl": 0.09997353378378573, | |
| "top1_agree": 0.9474969474969475 | |
| }, | |
| { | |
| "id": "code-qualification-holdout-2", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.4425236144199767, | |
| "teacher_nll": 1.4386393615759918, | |
| "kl": 0.06728265899732727, | |
| "top1_agree": 0.9433455433455433 | |
| }, | |
| { | |
| "id": "code-qualification-holdout-3", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.8086464198372156, | |
| "teacher_nll": 1.796362640744164, | |
| "kl": 0.1033730319349757, | |
| "top1_agree": 0.9343101343101343 | |
| }, | |
| { | |
| "id": "math-qualification-holdout-0", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.4159224659240872, | |
| "teacher_nll": 1.4128543704711765, | |
| "kl": 0.02953591391498789, | |
| "top1_agree": 0.9543345543345544 | |
| }, | |
| { | |
| "id": "math-qualification-holdout-1", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.5524857913472567, | |
| "teacher_nll": 1.545063196724879, | |
| "kl": 0.026335953079591596, | |
| "top1_agree": 0.9545787545787546 | |
| }, | |
| { | |
| "id": "math-qualification-holdout-2", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.5277823465647715, | |
| "teacher_nll": 1.5522358872223831, | |
| "kl": 0.02601443173247816, | |
| "top1_agree": 0.9538461538461539 | |
| }, | |
| { | |
| "id": "math-qualification-holdout-3", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.5344706189501416, | |
| "teacher_nll": 1.516595822987539, | |
| "kl": 0.026961823033398617, | |
| "top1_agree": 0.9567765567765568 | |
| } | |
| ], | |
| "memory": { | |
| "swap_kib": 0, | |
| "cuda_peak_bytes": 14954638336 | |
| } | |
| }, | |
| "budget44-depth": { | |
| "precision_by_layer": { | |
| "2": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "3": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "4": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "5": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "6": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "7": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "8": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "9": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "10": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "11": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "12": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "13": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "14": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "15": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "16": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "17": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "18": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "19": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "20": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "21": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "22": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "23": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "24": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "25": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "26": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "27": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "28": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "29": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "30": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "31": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "32": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "33": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "34": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "35": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "36": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "37": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "38": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "39": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "40": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "41": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "42": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "43": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "44": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "45": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "46": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "47": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "48": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| } | |
| }, | |
| "tensor_bytes": 43340635712, | |
| "aggregate": { | |
| "all": { | |
| "tokens": 49140, | |
| "nll": 2.21992336427918, | |
| "teacher_nll": 2.214729754197971, | |
| "kl": 0.066194986073489, | |
| "top1_agree": 0.9354090354090354, | |
| "perplexity": 9.206625282481305, | |
| "teacher_perplexity": 9.158933613767237, | |
| "delta_nll": 0.005193610081208888 | |
| }, | |
| "code": { | |
| "tokens": 16380, | |
| "nll": 1.592673438416296, | |
| "teacher_nll": 1.585819314280127, | |
| "kl": 0.08255041191611388, | |
| "top1_agree": 0.9466422466422466, | |
| "perplexity": 4.916876338938635, | |
| "teacher_perplexity": 4.883290689738726, | |
| "delta_nll": 0.0068541241361690375 | |
| }, | |
| "math": { | |
| "tokens": 16380, | |
| "nll": 1.5130973644672878, | |
| "teacher_nll": 1.5066873193514945, | |
| "kl": 0.021885212046868635, | |
| "top1_agree": 0.9585470085470086, | |
| "perplexity": 4.5407734658387175, | |
| "teacher_perplexity": 4.511759991246355, | |
| "delta_nll": 0.00641004511579335 | |
| }, | |
| "prose": { | |
| "tokens": 16380, | |
| "nll": 3.553999289953956, | |
| "teacher_nll": 3.551682628962292, | |
| "kl": 0.09414933425748451, | |
| "top1_agree": 0.9010378510378511, | |
| "perplexity": 34.95282481768223, | |
| "teacher_perplexity": 34.871944693965524, | |
| "delta_nll": 0.0023166609916640546 | |
| } | |
| }, | |
| "sequences": [ | |
| { | |
| "id": "prose-qualification-holdout-0", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.5772039113027274, | |
| "teacher_nll": 3.570623514736674, | |
| "kl": 0.08783619703681769, | |
| "top1_agree": 0.9001221001221001 | |
| }, | |
| { | |
| "id": "prose-qualification-holdout-1", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.499457275241577, | |
| "teacher_nll": 3.4886688269683876, | |
| "kl": 0.11149968565310896, | |
| "top1_agree": 0.8920634920634921 | |
| }, | |
| { | |
| "id": "prose-qualification-holdout-2", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.589149953914358, | |
| "teacher_nll": 3.586854711586157, | |
| "kl": 0.07451462410890722, | |
| "top1_agree": 0.9115995115995116 | |
| }, | |
| { | |
| "id": "prose-qualification-holdout-3", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.5501860193571617, | |
| "teacher_nll": 3.56058346255795, | |
| "kl": 0.10274683023110415, | |
| "top1_agree": 0.9003663003663004 | |
| }, | |
| { | |
| "id": "code-qualification-holdout-0", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.6159166301126444, | |
| "teacher_nll": 1.612040780315469, | |
| "kl": 0.08506395109407194, | |
| "top1_agree": 0.9426129426129426 | |
| }, | |
| { | |
| "id": "code-qualification-holdout-1", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.4902202620756642, | |
| "teacher_nll": 1.4962344744848826, | |
| "kl": 0.08821711451433809, | |
| "top1_agree": 0.9540903540903541 | |
| }, | |
| { | |
| "id": "code-qualification-holdout-2", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.470807389199952, | |
| "teacher_nll": 1.4386393615759918, | |
| "kl": 0.05696805114420051, | |
| "top1_agree": 0.9538461538461539 | |
| }, | |
| { | |
| "id": "code-qualification-holdout-3", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.793749472276923, | |
| "teacher_nll": 1.796362640744164, | |
| "kl": 0.09995253091184501, | |
| "top1_agree": 0.936019536019536 | |
| }, | |
| { | |
| "id": "math-qualification-holdout-0", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.4172291178988596, | |
| "teacher_nll": 1.4128543704711765, | |
| "kl": 0.02296853600768088, | |
| "top1_agree": 0.9526251526251526 | |
| }, | |
| { | |
| "id": "math-qualification-holdout-1", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.5529686668125846, | |
| "teacher_nll": 1.545063196724879, | |
| "kl": 0.023295361356711942, | |
| "top1_agree": 0.9594627594627595 | |
| }, | |
| { | |
| "id": "math-qualification-holdout-2", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.5368641188438705, | |
| "teacher_nll": 1.5522358872223831, | |
| "kl": 0.021453292778851205, | |
| "top1_agree": 0.9628815628815629 | |
| }, | |
| { | |
| "id": "math-qualification-holdout-3", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.5453275543138367, | |
| "teacher_nll": 1.516595822987539, | |
| "kl": 0.019823658044230516, | |
| "top1_agree": 0.9592185592185593 | |
| } | |
| ], | |
| "memory": { | |
| "swap_kib": 0, | |
| "cuda_peak_bytes": 14954638336 | |
| } | |
| }, | |
| "budget48-depth": { | |
| "precision_by_layer": { | |
| "2": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "3": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "4": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "5": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "6": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "7": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "8": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "9": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "10": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "11": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "12": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "13": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "14": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "15": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "16": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "17": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "18": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "19": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "20": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "21": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "22": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "23": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "24": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "25": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "26": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "27": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "28": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "29": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "30": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "31": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "32": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "33": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "34": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "35": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "36": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "37": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "38": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "39": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "40": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "41": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "42": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "43": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "44": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "45": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "46": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "47": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "48": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| } | |
| }, | |
| "tensor_bytes": 47366184512, | |
| "aggregate": { | |
| "all": { | |
| "tokens": 49140, | |
| "nll": 2.219438334936818, | |
| "teacher_nll": 2.214729754197971, | |
| "kl": 0.05590792845375154, | |
| "top1_agree": 0.9401302401302402, | |
| "perplexity": 9.202160881845348, | |
| "teacher_perplexity": 9.158933613767237, | |
| "delta_nll": 0.004708580738847168 | |
| }, | |
| "code": { | |
| "tokens": 16380, | |
| "nll": 1.598353770015004, | |
| "teacher_nll": 1.585819314280127, | |
| "kl": 0.06995221888771569, | |
| "top1_agree": 0.9498168498168498, | |
| "perplexity": 4.944885301760146, | |
| "teacher_perplexity": 4.883290689738726, | |
| "delta_nll": 0.012534455734876992 | |
| }, | |
| "math": { | |
| "tokens": 16380, | |
| "nll": 1.5061622015431395, | |
| "teacher_nll": 1.5066873193514945, | |
| "kl": 0.019403194701151243, | |
| "top1_agree": 0.9621489621489622, | |
| "perplexity": 4.5093914076750625, | |
| "teacher_perplexity": 4.511759991246355, | |
| "delta_nll": -0.0005251178083549579 | |
| }, | |
| "prose": { | |
| "tokens": 16380, | |
| "nll": 3.553799033252311, | |
| "teacher_nll": 3.551682628962292, | |
| "kl": 0.07836837177238767, | |
| "top1_agree": 0.9084249084249084, | |
| "perplexity": 34.94582598107641, | |
| "teacher_perplexity": 34.871944693965524, | |
| "delta_nll": 0.002116404290018803 | |
| } | |
| }, | |
| "sequences": [ | |
| { | |
| "id": "prose-qualification-holdout-0", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.581407642480946, | |
| "teacher_nll": 3.570623514736674, | |
| "kl": 0.07361397169739627, | |
| "top1_agree": 0.9086691086691087 | |
| }, | |
| { | |
| "id": "prose-qualification-holdout-1", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.488058652400388, | |
| "teacher_nll": 3.4886688269683876, | |
| "kl": 0.08343997936109047, | |
| "top1_agree": 0.9035409035409036 | |
| }, | |
| { | |
| "id": "prose-qualification-holdout-2", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.5756893917172237, | |
| "teacher_nll": 3.586854711586157, | |
| "kl": 0.07240510084399082, | |
| "top1_agree": 0.9103785103785104 | |
| }, | |
| { | |
| "id": "prose-qualification-holdout-3", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.570040446410686, | |
| "teacher_nll": 3.56058346255795, | |
| "kl": 0.08401443518707313, | |
| "top1_agree": 0.9111111111111111 | |
| }, | |
| { | |
| "id": "code-qualification-holdout-0", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.611734163426363, | |
| "teacher_nll": 1.612040780315469, | |
| "kl": 0.08207176428865892, | |
| "top1_agree": 0.944078144078144 | |
| }, | |
| { | |
| "id": "code-qualification-holdout-1", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.5178017247145283, | |
| "teacher_nll": 1.4962344744848826, | |
| "kl": 0.07385789276406587, | |
| "top1_agree": 0.9548229548229549 | |
| }, | |
| { | |
| "id": "code-qualification-holdout-2", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.4678809896056906, | |
| "teacher_nll": 1.4386393615759918, | |
| "kl": 0.04726901354224982, | |
| "top1_agree": 0.9560439560439561 | |
| }, | |
| { | |
| "id": "code-qualification-holdout-3", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.7959982023134335, | |
| "teacher_nll": 1.796362640744164, | |
| "kl": 0.07661020495588818, | |
| "top1_agree": 0.9443223443223443 | |
| }, | |
| { | |
| "id": "math-qualification-holdout-0", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.4153713189638577, | |
| "teacher_nll": 1.4128543704711765, | |
| "kl": 0.02133012669427054, | |
| "top1_agree": 0.9592185592185593 | |
| }, | |
| { | |
| "id": "math-qualification-holdout-1", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.550073492774451, | |
| "teacher_nll": 1.545063196724879, | |
| "kl": 0.019280421235331396, | |
| "top1_agree": 0.9604395604395605 | |
| }, | |
| { | |
| "id": "math-qualification-holdout-2", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.5292586651477185, | |
| "teacher_nll": 1.5522358872223831, | |
| "kl": 0.01899524313626272, | |
| "top1_agree": 0.9655677655677656 | |
| }, | |
| { | |
| "id": "math-qualification-holdout-3", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.529945329286531, | |
| "teacher_nll": 1.516595822987539, | |
| "kl": 0.018006987738740314, | |
| "top1_agree": 0.9633699633699634 | |
| } | |
| ], | |
| "memory": { | |
| "swap_kib": 0, | |
| "cuda_peak_bytes": 14954638336 | |
| } | |
| }, | |
| "budget28-depth": { | |
| "precision_by_layer": { | |
| "2": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "3": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "4": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "5": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "6": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "7": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "8": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "9": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "10": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "11": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "12": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "13": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "14": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "15": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "16": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "17": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "18": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "19": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "20": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "21": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "22": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "23": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "24": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "25": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "26": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "27": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "28": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "29": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "30": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "31": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "32": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "33": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "34": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "35": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "36": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "37": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "38": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "39": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "40": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "41": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "42": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "43": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "44": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "45": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "46": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "47": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "48": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| } | |
| }, | |
| "tensor_bytes": 27982328384, | |
| "aggregate": { | |
| "all": { | |
| "tokens": 49140, | |
| "nll": 2.2117175335194164, | |
| "teacher_nll": 2.214729754197971, | |
| "kl": 0.1379022409373005, | |
| "top1_agree": 0.8984330484330484, | |
| "perplexity": 9.131386394523139, | |
| "teacher_perplexity": 9.158933613767237, | |
| "delta_nll": -0.0030122206785545558 | |
| }, | |
| "code": { | |
| "tokens": 16380, | |
| "nll": 1.5829401942545387, | |
| "teacher_nll": 1.585819314280127, | |
| "kl": 0.15345898432536584, | |
| "top1_agree": 0.9151404151404151, | |
| "perplexity": 4.869251329922203, | |
| "teacher_perplexity": 4.883290689738726, | |
| "delta_nll": -0.002879120025588211 | |
| }, | |
| "math": { | |
| "tokens": 16380, | |
| "nll": 1.5169169920486407, | |
| "teacher_nll": 1.5066873193514945, | |
| "kl": 0.06299899023240012, | |
| "top1_agree": 0.9334554334554335, | |
| "perplexity": 4.558150695555302, | |
| "teacher_perplexity": 4.511759991246355, | |
| "delta_nll": 0.010229672697146253 | |
| }, | |
| "prose": { | |
| "tokens": 16380, | |
| "nll": 3.53529541425507, | |
| "teacher_nll": 3.551682628962292, | |
| "kl": 0.19724874825413555, | |
| "top1_agree": 0.8467032967032967, | |
| "perplexity": 34.305147447056434, | |
| "teacher_perplexity": 34.871944693965524, | |
| "delta_nll": -0.016387214707222153 | |
| } | |
| }, | |
| "sequences": [ | |
| { | |
| "id": "prose-qualification-holdout-0", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.576774416447268, | |
| "teacher_nll": 3.570623514736674, | |
| "kl": 0.18502871113964636, | |
| "top1_agree": 0.8517704517704517 | |
| }, | |
| { | |
| "id": "prose-qualification-holdout-1", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.4453719776046436, | |
| "teacher_nll": 3.4886688269683876, | |
| "kl": 0.1999274159525777, | |
| "top1_agree": 0.8402930402930403 | |
| }, | |
| { | |
| "id": "prose-qualification-holdout-2", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.567739048400059, | |
| "teacher_nll": 3.586854711586157, | |
| "kl": 0.194889590620849, | |
| "top1_agree": 0.8495726495726496 | |
| }, | |
| { | |
| "id": "prose-qualification-holdout-3", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.551296214568309, | |
| "teacher_nll": 3.56058346255795, | |
| "kl": 0.2091492753034692, | |
| "top1_agree": 0.8451770451770452 | |
| }, | |
| { | |
| "id": "code-qualification-holdout-0", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.6328375978202028, | |
| "teacher_nll": 1.612040780315469, | |
| "kl": 0.16888136997618808, | |
| "top1_agree": 0.906959706959707 | |
| }, | |
| { | |
| "id": "code-qualification-holdout-1", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.4438413806305117, | |
| "teacher_nll": 1.4962344744848826, | |
| "kl": 0.16586800363910942, | |
| "top1_agree": 0.9255189255189256 | |
| }, | |
| { | |
| "id": "code-qualification-holdout-2", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.4312066908429977, | |
| "teacher_nll": 1.4386393615759918, | |
| "kl": 0.12033674994958918, | |
| "top1_agree": 0.9172161172161172 | |
| }, | |
| { | |
| "id": "code-qualification-holdout-3", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.8238751077244424, | |
| "teacher_nll": 1.796362640744164, | |
| "kl": 0.15874981373657673, | |
| "top1_agree": 0.9108669108669109 | |
| }, | |
| { | |
| "id": "math-qualification-holdout-0", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.4115624984105428, | |
| "teacher_nll": 1.4128543704711765, | |
| "kl": 0.06079979526545393, | |
| "top1_agree": 0.9343101343101343 | |
| }, | |
| { | |
| "id": "math-qualification-holdout-1", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.5654080121945113, | |
| "teacher_nll": 1.545063196724879, | |
| "kl": 0.07012658806219788, | |
| "top1_agree": 0.9296703296703297 | |
| }, | |
| { | |
| "id": "math-qualification-holdout-2", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.5408491942763183, | |
| "teacher_nll": 1.5522358872223831, | |
| "kl": 0.061726128159861865, | |
| "top1_agree": 0.9352869352869353 | |
| }, | |
| { | |
| "id": "math-qualification-holdout-3", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.549848263313191, | |
| "teacher_nll": 1.516595822987539, | |
| "kl": 0.05934344944208678, | |
| "top1_agree": 0.9345543345543346 | |
| } | |
| ], | |
| "memory": { | |
| "swap_kib": 0, | |
| "cuda_peak_bytes": 14954638336 | |
| } | |
| }, | |
| "budget28-conditional": { | |
| "precision_by_layer": { | |
| "2": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "3": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "4": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "5": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "6": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "7": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "8": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "9": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "10": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "11": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "12": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "13": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "14": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "15": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "16": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "17": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "18": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "19": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "20": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "21": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "22": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "23": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "24": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "25": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "26": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "27": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "28": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "29": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "30": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "31": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "32": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "33": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "34": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "35": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "36": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "37": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "38": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "39": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "40": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "41": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "42": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "43": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "44": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "45": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "46": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "47": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "48": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| } | |
| }, | |
| "tensor_bytes": 27982328384, | |
| "aggregate": { | |
| "all": { | |
| "tokens": 49140, | |
| "nll": 2.2224484121527768, | |
| "teacher_nll": 2.214729754197971, | |
| "kl": 0.13521137943640685, | |
| "top1_agree": 0.8994505494505495, | |
| "perplexity": 9.22990182690099, | |
| "teacher_perplexity": 9.158933613767237, | |
| "delta_nll": 0.007718657954805863 | |
| }, | |
| "code": { | |
| "tokens": 16380, | |
| "nll": 1.5960918323169844, | |
| "teacher_nll": 1.585819314280127, | |
| "kl": 0.15048720305000907, | |
| "top1_agree": 0.9156898656898657, | |
| "perplexity": 4.9337129196633285, | |
| "teacher_perplexity": 4.883290689738726, | |
| "delta_nll": 0.01027251803685747 | |
| }, | |
| "math": { | |
| "tokens": 16380, | |
| "nll": 1.5213324331174694, | |
| "teacher_nll": 1.5066873193514945, | |
| "kl": 0.0603252738759704, | |
| "top1_agree": 0.9349206349206349, | |
| "perplexity": 4.57832143992999, | |
| "teacher_perplexity": 4.511759991246355, | |
| "delta_nll": 0.014645113765974882 | |
| }, | |
| "prose": { | |
| "tokens": 16380, | |
| "nll": 3.549920971023876, | |
| "teacher_nll": 3.551682628962292, | |
| "kl": 0.19482166138324108, | |
| "top1_agree": 0.8477411477411477, | |
| "perplexity": 34.810566335477304, | |
| "teacher_perplexity": 34.871944693965524, | |
| "delta_nll": -0.0017616579384158726 | |
| } | |
| }, | |
| "sequences": [ | |
| { | |
| "id": "prose-qualification-holdout-0", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.589803723159436, | |
| "teacher_nll": 3.570623514736674, | |
| "kl": 0.18862637355650738, | |
| "top1_agree": 0.8512820512820513 | |
| }, | |
| { | |
| "id": "prose-qualification-holdout-1", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.4418622362744675, | |
| "teacher_nll": 3.4886688269683876, | |
| "kl": 0.20619230410118242, | |
| "top1_agree": 0.842979242979243 | |
| }, | |
| { | |
| "id": "prose-qualification-holdout-2", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.6040337829217224, | |
| "teacher_nll": 3.586854711586157, | |
| "kl": 0.19182675237038488, | |
| "top1_agree": 0.852014652014652 | |
| }, | |
| { | |
| "id": "prose-qualification-holdout-3", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.563984141739879, | |
| "teacher_nll": 3.56058346255795, | |
| "kl": 0.19264121550488966, | |
| "top1_agree": 0.8446886446886447 | |
| }, | |
| { | |
| "id": "code-qualification-holdout-0", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.6439844068590102, | |
| "teacher_nll": 1.612040780315469, | |
| "kl": 0.1714319664334494, | |
| "top1_agree": 0.9108669108669109 | |
| }, | |
| { | |
| "id": "code-qualification-holdout-1", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.475341613941868, | |
| "teacher_nll": 1.4962344744848826, | |
| "kl": 0.1390051935473059, | |
| "top1_agree": 0.9223443223443224 | |
| }, | |
| { | |
| "id": "code-qualification-holdout-2", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.4345243325891366, | |
| "teacher_nll": 1.4386393615759918, | |
| "kl": 0.12111187706063518, | |
| "top1_agree": 0.9177045177045177 | |
| }, | |
| { | |
| "id": "code-qualification-holdout-3", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.8305169758779225, | |
| "teacher_nll": 1.796362640744164, | |
| "kl": 0.17039977515864577, | |
| "top1_agree": 0.9118437118437118 | |
| }, | |
| { | |
| "id": "math-qualification-holdout-0", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.4194815079371135, | |
| "teacher_nll": 1.4128543704711765, | |
| "kl": 0.0626669808985695, | |
| "top1_agree": 0.9362637362637363 | |
| }, | |
| { | |
| "id": "math-qualification-holdout-1", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.570239007691324, | |
| "teacher_nll": 1.545063196724879, | |
| "kl": 0.059914114885714465, | |
| "top1_agree": 0.9291819291819292 | |
| }, | |
| { | |
| "id": "math-qualification-holdout-2", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.5442800039773459, | |
| "teacher_nll": 1.5522358872223831, | |
| "kl": 0.059936179899325155, | |
| "top1_agree": 0.9367521367521368 | |
| }, | |
| { | |
| "id": "math-qualification-holdout-3", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.5513292128640945, | |
| "teacher_nll": 1.516595822987539, | |
| "kl": 0.05878381982027247, | |
| "top1_agree": 0.9374847374847375 | |
| } | |
| ], | |
| "memory": { | |
| "swap_kib": 0, | |
| "cuda_peak_bytes": 14954638336 | |
| } | |
| }, | |
| "budget32-depth": { | |
| "precision_by_layer": { | |
| "2": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "3": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "4": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "5": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "6": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "7": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "8": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "9": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "10": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "11": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "12": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "13": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "14": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "15": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "16": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "17": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "18": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "19": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "20": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "21": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "22": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "23": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "24": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "25": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "26": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "27": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "28": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "29": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "30": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "31": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "32": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "33": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "34": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "35": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "36": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "37": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "38": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "39": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "40": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "41": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "42": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "43": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "44": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "45": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "46": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "47": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "48": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| } | |
| }, | |
| "tensor_bytes": 31859994176, | |
| "aggregate": { | |
| "all": { | |
| "tokens": 49140, | |
| "nll": 2.209968472999628, | |
| "teacher_nll": 2.214729754197971, | |
| "kl": 0.1076129502004659, | |
| "top1_agree": 0.9095645095645095, | |
| "perplexity": 9.115429006376434, | |
| "teacher_perplexity": 9.158933613767237, | |
| "delta_nll": -0.00476128119834307 | |
| }, | |
| "code": { | |
| "tokens": 16380, | |
| "nll": 1.5728688767905346, | |
| "teacher_nll": 1.585819314280127, | |
| "kl": 0.12423731337809737, | |
| "top1_agree": 0.9235042735042736, | |
| "perplexity": 4.82045767459655, | |
| "teacher_perplexity": 4.883290689738726, | |
| "delta_nll": -0.012950437489592348 | |
| }, | |
| "math": { | |
| "tokens": 16380, | |
| "nll": 1.5087034377568516, | |
| "teacher_nll": 1.5066873193514945, | |
| "kl": 0.047924589849690086, | |
| "top1_agree": 0.942002442002442, | |
| "perplexity": 4.5208654093214875, | |
| "teacher_perplexity": 4.511759991246355, | |
| "delta_nll": 0.0020161184053570924 | |
| }, | |
| "prose": { | |
| "tokens": 16380, | |
| "nll": 3.548333104451497, | |
| "teacher_nll": 3.551682628962292, | |
| "kl": 0.1506769473736103, | |
| "top1_agree": 0.8631868131868132, | |
| "perplexity": 34.7553356619023, | |
| "teacher_perplexity": 34.871944693965524, | |
| "delta_nll": -0.0033495245107948435 | |
| } | |
| }, | |
| "sequences": [ | |
| { | |
| "id": "prose-qualification-holdout-0", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.599144261398595, | |
| "teacher_nll": 3.570623514736674, | |
| "kl": 0.1407006877391475, | |
| "top1_agree": 0.8647130647130647 | |
| }, | |
| { | |
| "id": "prose-qualification-holdout-1", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.4657273885240194, | |
| "teacher_nll": 3.4886688269683876, | |
| "kl": 0.17342653262891758, | |
| "top1_agree": 0.8617826617826618 | |
| }, | |
| { | |
| "id": "prose-qualification-holdout-2", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.5611716171469587, | |
| "teacher_nll": 3.586854711586157, | |
| "kl": 0.1355552160405123, | |
| "top1_agree": 0.8654456654456655 | |
| }, | |
| { | |
| "id": "prose-qualification-holdout-3", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.5672891507364164, | |
| "teacher_nll": 3.56058346255795, | |
| "kl": 0.1530253530858637, | |
| "top1_agree": 0.8608058608058609 | |
| }, | |
| { | |
| "id": "code-qualification-holdout-0", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.6259151891910986, | |
| "teacher_nll": 1.612040780315469, | |
| "kl": 0.1325808134562221, | |
| "top1_agree": 0.9164835164835164 | |
| }, | |
| { | |
| "id": "code-qualification-holdout-1", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.4427267800320636, | |
| "teacher_nll": 1.4962344744848826, | |
| "kl": 0.12487448638393765, | |
| "top1_agree": 0.9318681318681319 | |
| }, | |
| { | |
| "id": "code-qualification-holdout-2", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.4258999699929114, | |
| "teacher_nll": 1.4386393615759918, | |
| "kl": 0.10263109003376757, | |
| "top1_agree": 0.9274725274725275 | |
| }, | |
| { | |
| "id": "code-qualification-holdout-3", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.796933567946065, | |
| "teacher_nll": 1.796362640744164, | |
| "kl": 0.13686286363846217, | |
| "top1_agree": 0.9181929181929182 | |
| }, | |
| { | |
| "id": "math-qualification-holdout-0", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.407740637729165, | |
| "teacher_nll": 1.4128543704711765, | |
| "kl": 0.04687176862042466, | |
| "top1_agree": 0.9443223443223443 | |
| }, | |
| { | |
| "id": "math-qualification-holdout-1", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.547222171452747, | |
| "teacher_nll": 1.545063196724879, | |
| "kl": 0.046711798872848716, | |
| "top1_agree": 0.9416361416361416 | |
| }, | |
| { | |
| "id": "math-qualification-holdout-2", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.5465935969090723, | |
| "teacher_nll": 1.5522358872223831, | |
| "kl": 0.04912073272342938, | |
| "top1_agree": 0.9423687423687424 | |
| }, | |
| { | |
| "id": "math-qualification-holdout-3", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.5332573449364222, | |
| "teacher_nll": 1.516595822987539, | |
| "kl": 0.0489940591820576, | |
| "top1_agree": 0.9396825396825397 | |
| } | |
| ], | |
| "memory": { | |
| "swap_kib": 0, | |
| "cuda_peak_bytes": 14954638336 | |
| } | |
| }, | |
| "budget36-depth": { | |
| "precision_by_layer": { | |
| "2": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "3": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "4": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "5": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "6": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "7": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "8": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "9": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "10": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "11": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "12": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "13": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "14": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "15": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "16": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "17": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "18": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "19": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "20": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "21": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "22": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "23": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "24": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "25": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "26": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "27": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "28": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "29": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "30": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "31": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "32": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "33": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "34": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "35": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "36": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "37": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "38": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "39": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "40": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "41": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "42": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "43": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "44": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "45": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "46": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "47": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "48": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| } | |
| }, | |
| "tensor_bytes": 35737659968, | |
| "aggregate": { | |
| "all": { | |
| "tokens": 49140, | |
| "nll": 2.223855302175662, | |
| "teacher_nll": 2.214729754197971, | |
| "kl": 0.09127278831151743, | |
| "top1_agree": 0.9190883190883191, | |
| "perplexity": 9.24289642253355, | |
| "teacher_perplexity": 9.158933613767237, | |
| "delta_nll": 0.009125547977691273 | |
| }, | |
| "code": { | |
| "tokens": 16380, | |
| "nll": 1.6116677623092037, | |
| "teacher_nll": 1.585819314280127, | |
| "kl": 0.1089983913497101, | |
| "top1_agree": 0.9323565323565324, | |
| "perplexity": 5.011161689196273, | |
| "teacher_perplexity": 4.883290689738726, | |
| "delta_nll": 0.02584844802907682 | |
| }, | |
| "math": { | |
| "tokens": 16380, | |
| "nll": 1.503876333446293, | |
| "teacher_nll": 1.5066873193514945, | |
| "kl": 0.03497329444666558, | |
| "top1_agree": 0.9487789987789987, | |
| "perplexity": 4.49909530596853, | |
| "teacher_perplexity": 4.511759991246355, | |
| "delta_nll": -0.0028109859052014574 | |
| }, | |
| "prose": { | |
| "tokens": 16380, | |
| "nll": 3.5560218107714903, | |
| "teacher_nll": 3.551682628962292, | |
| "kl": 0.1298466791381766, | |
| "top1_agree": 0.8761294261294261, | |
| "perplexity": 35.023589170573715, | |
| "teacher_perplexity": 34.871944693965524, | |
| "delta_nll": 0.004339181809198234 | |
| } | |
| }, | |
| "sequences": [ | |
| { | |
| "id": "prose-qualification-holdout-0", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.6045262673253395, | |
| "teacher_nll": 3.570623514736674, | |
| "kl": 0.12269781500428588, | |
| "top1_agree": 0.8764346764346764 | |
| }, | |
| { | |
| "id": "prose-qualification-holdout-1", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.4700199392689015, | |
| "teacher_nll": 3.4886688269683876, | |
| "kl": 0.15376708143651122, | |
| "top1_agree": 0.8788766788766789 | |
| }, | |
| { | |
| "id": "prose-qualification-holdout-2", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.5796889234084057, | |
| "teacher_nll": 3.586854711586157, | |
| "kl": 0.11325538041972998, | |
| "top1_agree": 0.8774114774114774 | |
| }, | |
| { | |
| "id": "prose-qualification-holdout-3", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.5698521130833143, | |
| "teacher_nll": 3.56058346255795, | |
| "kl": 0.12966643969217936, | |
| "top1_agree": 0.8717948717948718 | |
| }, | |
| { | |
| "id": "code-qualification-holdout-0", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.6474669259691996, | |
| "teacher_nll": 1.612040780315469, | |
| "kl": 0.12218621996878413, | |
| "top1_agree": 0.926984126984127 | |
| }, | |
| { | |
| "id": "code-qualification-holdout-1", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.5146658239492712, | |
| "teacher_nll": 1.4962344744848826, | |
| "kl": 0.10590404228776948, | |
| "top1_agree": 0.936996336996337 | |
| }, | |
| { | |
| "id": "code-qualification-holdout-2", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.4736719823145605, | |
| "teacher_nll": 1.4386393615759918, | |
| "kl": 0.08629191191059621, | |
| "top1_agree": 0.9387057387057387 | |
| }, | |
| { | |
| "id": "code-qualification-holdout-3", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.8108663170037835, | |
| "teacher_nll": 1.796362640744164, | |
| "kl": 0.12161139123169057, | |
| "top1_agree": 0.9267399267399268 | |
| }, | |
| { | |
| "id": "math-qualification-holdout-0", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.3993232566358405, | |
| "teacher_nll": 1.4128543704711765, | |
| "kl": 0.0360991838723603, | |
| "top1_agree": 0.947008547008547 | |
| }, | |
| { | |
| "id": "math-qualification-holdout-1", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.542288848914215, | |
| "teacher_nll": 1.545063196724879, | |
| "kl": 0.03444656883258377, | |
| "top1_agree": 0.9462759462759462 | |
| }, | |
| { | |
| "id": "math-qualification-holdout-2", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.535391583751264, | |
| "teacher_nll": 1.5522358872223831, | |
| "kl": 0.034275711528838626, | |
| "top1_agree": 0.9496947496947497 | |
| }, | |
| { | |
| "id": "math-qualification-holdout-3", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.5385016444838526, | |
| "teacher_nll": 1.516595822987539, | |
| "kl": 0.035071713552879615, | |
| "top1_agree": 0.9521367521367521 | |
| } | |
| ], | |
| "memory": { | |
| "swap_kib": 0, | |
| "cuda_peak_bytes": 14954638336 | |
| } | |
| }, | |
| "budget36-conditional": { | |
| "precision_by_layer": { | |
| "2": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "3": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "4": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "5": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "6": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "7": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "8": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "9": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "10": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "11": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "12": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "13": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "14": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "15": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "16": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "17": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "18": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "19": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "20": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "21": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "22": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "23": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "24": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "25": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "26": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "27": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "28": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "29": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "30": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "31": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "32": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "33": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "34": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "35": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "36": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "37": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "38": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "39": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "40": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "41": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "42": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "43": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "44": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "45": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "46": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "47": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "48": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| } | |
| }, | |
| "tensor_bytes": 35737659968, | |
| "aggregate": { | |
| "all": { | |
| "tokens": 49140, | |
| "nll": 2.211066000828861, | |
| "teacher_nll": 2.214729754197971, | |
| "kl": 0.09881533177210929, | |
| "top1_agree": 0.9154049654049654, | |
| "perplexity": 9.125438935469404, | |
| "teacher_perplexity": 9.158933613767237, | |
| "delta_nll": -0.0036637533691097346 | |
| }, | |
| "code": { | |
| "tokens": 16380, | |
| "nll": 1.5794704765336127, | |
| "teacher_nll": 1.585819314280127, | |
| "kl": 0.1142164363947689, | |
| "top1_agree": 0.92997557997558, | |
| "perplexity": 4.852385678739899, | |
| "teacher_perplexity": 4.883290689738726, | |
| "delta_nll": -0.00634883774651418 | |
| }, | |
| "math": { | |
| "tokens": 16380, | |
| "nll": 1.5101053988904451, | |
| "teacher_nll": 1.5066873193514945, | |
| "kl": 0.03895971165544675, | |
| "top1_agree": 0.9462759462759462, | |
| "perplexity": 4.527207931861759, | |
| "teacher_perplexity": 4.511759991246355, | |
| "delta_nll": 0.00341807953895068 | |
| }, | |
| "prose": { | |
| "tokens": 16380, | |
| "nll": 3.5436221270625263, | |
| "teacher_nll": 3.551682628962292, | |
| "kl": 0.1432698472661122, | |
| "top1_agree": 0.86996336996337, | |
| "perplexity": 34.59198912447507, | |
| "teacher_perplexity": 34.871944693965524, | |
| "delta_nll": -0.008060501899765704 | |
| } | |
| }, | |
| "sequences": [ | |
| { | |
| "id": "prose-qualification-holdout-0", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.58719486148075, | |
| "teacher_nll": 3.570623514736674, | |
| "kl": 0.1392865496531802, | |
| "top1_agree": 0.8717948717948718 | |
| }, | |
| { | |
| "id": "prose-qualification-holdout-1", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.4693113334012993, | |
| "teacher_nll": 3.4886688269683876, | |
| "kl": 0.1504007330162039, | |
| "top1_agree": 0.8683760683760684 | |
| }, | |
| { | |
| "id": "prose-qualification-holdout-2", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.5602676931900565, | |
| "teacher_nll": 3.586854711586157, | |
| "kl": 0.13613470395406088, | |
| "top1_agree": 0.8798534798534798 | |
| }, | |
| { | |
| "id": "prose-qualification-holdout-3", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.557714620177999, | |
| "teacher_nll": 3.56058346255795, | |
| "kl": 0.14725740244100383, | |
| "top1_agree": 0.8598290598290599 | |
| }, | |
| { | |
| "id": "code-qualification-holdout-0", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.6157172425471529, | |
| "teacher_nll": 1.612040780315469, | |
| "kl": 0.1348012829656566, | |
| "top1_agree": 0.9255189255189256 | |
| }, | |
| { | |
| "id": "code-qualification-holdout-1", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.5071097101484026, | |
| "teacher_nll": 1.4962344744848826, | |
| "kl": 0.11607390501705864, | |
| "top1_agree": 0.936996336996337 | |
| }, | |
| { | |
| "id": "code-qualification-holdout-2", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.4030494752821032, | |
| "teacher_nll": 1.4386393615759918, | |
| "kl": 0.08161759099971971, | |
| "top1_agree": 0.9335775335775336 | |
| }, | |
| { | |
| "id": "code-qualification-holdout-3", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.792005478156792, | |
| "teacher_nll": 1.796362640744164, | |
| "kl": 0.12437296659664066, | |
| "top1_agree": 0.9238095238095239 | |
| }, | |
| { | |
| "id": "math-qualification-holdout-0", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.4169574918909968, | |
| "teacher_nll": 1.4128543704711765, | |
| "kl": 0.039256950532414826, | |
| "top1_agree": 0.9465201465201465 | |
| }, | |
| { | |
| "id": "math-qualification-holdout-1", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.5547984918540796, | |
| "teacher_nll": 1.545063196724879, | |
| "kl": 0.037581647774238724, | |
| "top1_agree": 0.9404151404151404 | |
| }, | |
| { | |
| "id": "math-qualification-holdout-2", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.5365613138573802, | |
| "teacher_nll": 1.5522358872223831, | |
| "kl": 0.04003023478283259, | |
| "top1_agree": 0.9489621489621489 | |
| }, | |
| { | |
| "id": "math-qualification-holdout-3", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.5321042979593242, | |
| "teacher_nll": 1.516595822987539, | |
| "kl": 0.03897001353230086, | |
| "top1_agree": 0.9492063492063492 | |
| } | |
| ], | |
| "memory": { | |
| "swap_kib": 0, | |
| "cuda_peak_bytes": 14954638336 | |
| } | |
| }, | |
| "budget40-depth": { | |
| "precision_by_layer": { | |
| "2": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "3": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "4": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "5": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "6": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "7": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "8": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "9": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "10": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "11": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "12": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "13": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "14": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "15": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "16": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "17": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "18": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "19": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "20": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "21": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "22": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "23": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "24": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "25": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "26": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "27": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "28": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "29": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "30": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "31": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "32": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "33": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "34": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "35": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "36": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "37": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "38": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "39": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "40": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "41": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "42": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "43": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "44": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "45": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "46": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "47": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "48": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| } | |
| }, | |
| "tensor_bytes": 39967840832, | |
| "aggregate": { | |
| "all": { | |
| "tokens": 49140, | |
| "nll": 2.2225935721111143, | |
| "teacher_nll": 2.214729754197971, | |
| "kl": 0.07734097852024803, | |
| "top1_agree": 0.9308506308506308, | |
| "perplexity": 9.231241736313887, | |
| "teacher_perplexity": 9.158933613767237, | |
| "delta_nll": 0.00786381791314339 | |
| }, | |
| "code": { | |
| "tokens": 16380, | |
| "nll": 1.614134958149895, | |
| "teacher_nll": 1.585819314280127, | |
| "kl": 0.09729672360901467, | |
| "top1_agree": 0.9414529914529914, | |
| "perplexity": 5.0235404706327325, | |
| "teacher_perplexity": 4.883290689738726, | |
| "delta_nll": 0.028315643869768037 | |
| }, | |
| "math": { | |
| "tokens": 16380, | |
| "nll": 1.5017716076638963, | |
| "teacher_nll": 1.5066873193514945, | |
| "kl": 0.024713811772319422, | |
| "top1_agree": 0.955921855921856, | |
| "perplexity": 4.489635902297988, | |
| "teacher_perplexity": 4.511759991246355, | |
| "delta_nll": -0.00491571168759819 | |
| }, | |
| "prose": { | |
| "tokens": 16380, | |
| "nll": 3.5518741505195517, | |
| "teacher_nll": 3.551682628962292, | |
| "kl": 0.11001240017941001, | |
| "top1_agree": 0.8951770451770452, | |
| "perplexity": 34.87862406271912, | |
| "teacher_perplexity": 34.871944693965524, | |
| "delta_nll": 0.00019152155725965514 | |
| } | |
| }, | |
| "sequences": [ | |
| { | |
| "id": "prose-qualification-holdout-0", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.59116698138152, | |
| "teacher_nll": 3.570623514736674, | |
| "kl": 0.10556535575270508, | |
| "top1_agree": 0.8942612942612943 | |
| }, | |
| { | |
| "id": "prose-qualification-holdout-1", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.4707406478341536, | |
| "teacher_nll": 3.4886688269683876, | |
| "kl": 0.13352662218650474, | |
| "top1_agree": 0.8920634920634921 | |
| }, | |
| { | |
| "id": "prose-qualification-holdout-2", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.5792850219577517, | |
| "teacher_nll": 3.586854711586157, | |
| "kl": 0.09330803332020221, | |
| "top1_agree": 0.9010989010989011 | |
| }, | |
| { | |
| "id": "prose-qualification-holdout-3", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.566303950904781, | |
| "teacher_nll": 3.56058346255795, | |
| "kl": 0.10764958945822803, | |
| "top1_agree": 0.8932844932844933 | |
| }, | |
| { | |
| "id": "code-qualification-holdout-0", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.6441088330614697, | |
| "teacher_nll": 1.612040780315469, | |
| "kl": 0.11154779642638415, | |
| "top1_agree": 0.9357753357753358 | |
| }, | |
| { | |
| "id": "code-qualification-holdout-1", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.5216690654137486, | |
| "teacher_nll": 1.4962344744848826, | |
| "kl": 0.09650421681221442, | |
| "top1_agree": 0.9465201465201465 | |
| }, | |
| { | |
| "id": "code-qualification-holdout-2", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.477832281836951, | |
| "teacher_nll": 1.4386393615759918, | |
| "kl": 0.07622636381231968, | |
| "top1_agree": 0.9457875457875458 | |
| }, | |
| { | |
| "id": "code-qualification-holdout-3", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.8129296522874099, | |
| "teacher_nll": 1.796362640744164, | |
| "kl": 0.10490851738514044, | |
| "top1_agree": 0.9377289377289377 | |
| }, | |
| { | |
| "id": "math-qualification-holdout-0", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.4050527554583054, | |
| "teacher_nll": 1.4128543704711765, | |
| "kl": 0.025840590572182513, | |
| "top1_agree": 0.9553113553113554 | |
| }, | |
| { | |
| "id": "math-qualification-holdout-1", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.5450425144516942, | |
| "teacher_nll": 1.545063196724879, | |
| "kl": 0.023052263863939652, | |
| "top1_agree": 0.9543345543345544 | |
| }, | |
| { | |
| "id": "math-qualification-holdout-2", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.5158889481727311, | |
| "teacher_nll": 1.5522358872223831, | |
| "kl": 0.024994288965313718, | |
| "top1_agree": 0.9570207570207571 | |
| }, | |
| { | |
| "id": "math-qualification-holdout-3", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.5411022125728546, | |
| "teacher_nll": 1.516595822987539, | |
| "kl": 0.024968103687841813, | |
| "top1_agree": 0.9570207570207571 | |
| } | |
| ], | |
| "memory": { | |
| "swap_kib": 0, | |
| "cuda_peak_bytes": 14954638336 | |
| } | |
| }, | |
| "budget40-conditional": { | |
| "precision_by_layer": { | |
| "2": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "3": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "4": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "5": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "6": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "7": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "8": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "9": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "10": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "11": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "12": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "13": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "14": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "15": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "16": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "17": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "18": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "19": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "20": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "21": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "22": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "23": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "24": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "25": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "26": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "27": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "28": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "29": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "30": { | |
| "gate": "nvfp4", | |
| "up": "nvfp4", | |
| "down": "nvfp4" | |
| }, | |
| "31": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "32": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "33": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "34": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "35": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "36": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "37": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "38": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "39": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "40": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "41": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "42": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "43": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "44": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "45": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "46": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "47": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "48": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| } | |
| }, | |
| "tensor_bytes": 39967840832, | |
| "aggregate": { | |
| "all": { | |
| "tokens": 49140, | |
| "nll": 2.2112540609767684, | |
| "teacher_nll": 2.214729754197971, | |
| "kl": 0.08093327798213132, | |
| "top1_agree": 0.9278795278795279, | |
| "perplexity": 9.127155228243405, | |
| "teacher_perplexity": 9.158933613767237, | |
| "delta_nll": -0.003475693221202558 | |
| }, | |
| "code": { | |
| "tokens": 16380, | |
| "nll": 1.58970221944781, | |
| "teacher_nll": 1.585819314280127, | |
| "kl": 0.10097118896751758, | |
| "top1_agree": 0.94004884004884, | |
| "perplexity": 4.902288904657061, | |
| "teacher_perplexity": 4.883290689738726, | |
| "delta_nll": 0.0038829051676831927 | |
| }, | |
| "math": { | |
| "tokens": 16380, | |
| "nll": 1.501633097517039, | |
| "teacher_nll": 1.5066873193514945, | |
| "kl": 0.02864558382075785, | |
| "top1_agree": 0.9518315018315018, | |
| "perplexity": 4.489014085234806, | |
| "teacher_perplexity": 4.511759991246355, | |
| "delta_nll": -0.0050542218344553635 | |
| }, | |
| "prose": { | |
| "tokens": 16380, | |
| "nll": 3.5424268659654556, | |
| "teacher_nll": 3.551682628962292, | |
| "kl": 0.11318306115811852, | |
| "top1_agree": 0.8917582417582418, | |
| "perplexity": 34.5506673656692, | |
| "teacher_perplexity": 34.871944693965524, | |
| "delta_nll": -0.009255762996836392 | |
| } | |
| }, | |
| "sequences": [ | |
| { | |
| "id": "prose-qualification-holdout-0", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.6013154808854884, | |
| "teacher_nll": 3.570623514736674, | |
| "kl": 0.11052129783909836, | |
| "top1_agree": 0.8947496947496948 | |
| }, | |
| { | |
| "id": "prose-qualification-holdout-1", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.4538758032196752, | |
| "teacher_nll": 3.4886688269683876, | |
| "kl": 0.13668285830990298, | |
| "top1_agree": 0.8920634920634921 | |
| }, | |
| { | |
| "id": "prose-qualification-holdout-2", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.5637225326892077, | |
| "teacher_nll": 3.586854711586157, | |
| "kl": 0.09356140222887126, | |
| "top1_agree": 0.9010989010989011 | |
| }, | |
| { | |
| "id": "prose-qualification-holdout-3", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.550793647067451, | |
| "teacher_nll": 3.56058346255795, | |
| "kl": 0.11196668625460149, | |
| "top1_agree": 0.8791208791208791 | |
| }, | |
| { | |
| "id": "code-qualification-holdout-0", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.6111260536389473, | |
| "teacher_nll": 1.612040780315469, | |
| "kl": 0.11032203337648412, | |
| "top1_agree": 0.9357753357753358 | |
| }, | |
| { | |
| "id": "code-qualification-holdout-1", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.4951144709255233, | |
| "teacher_nll": 1.4962344744848826, | |
| "kl": 0.09977835434624273, | |
| "top1_agree": 0.9482295482295482 | |
| }, | |
| { | |
| "id": "code-qualification-holdout-2", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.4474246419829764, | |
| "teacher_nll": 1.4386393615759918, | |
| "kl": 0.07929879127113519, | |
| "top1_agree": 0.9431013431013431 | |
| }, | |
| { | |
| "id": "code-qualification-holdout-3", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.8051437112437936, | |
| "teacher_nll": 1.796362640744164, | |
| "kl": 0.11448557687620832, | |
| "top1_agree": 0.9330891330891331 | |
| }, | |
| { | |
| "id": "math-qualification-holdout-0", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.4102511390807138, | |
| "teacher_nll": 1.4128543704711765, | |
| "kl": 0.029556334721008645, | |
| "top1_agree": 0.9504273504273504 | |
| }, | |
| { | |
| "id": "math-qualification-holdout-1", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.5434705280122303, | |
| "teacher_nll": 1.545063196724879, | |
| "kl": 0.027982906223944545, | |
| "top1_agree": 0.9484737484737484 | |
| }, | |
| { | |
| "id": "math-qualification-holdout-2", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.5248470017615983, | |
| "teacher_nll": 1.5522358872223831, | |
| "kl": 0.029806509846878285, | |
| "top1_agree": 0.9540903540903541 | |
| }, | |
| { | |
| "id": "math-qualification-holdout-3", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.5279637212136143, | |
| "teacher_nll": 1.516595822987539, | |
| "kl": 0.02723658449119992, | |
| "top1_agree": 0.9543345543345544 | |
| } | |
| ], | |
| "memory": { | |
| "swap_kib": 0, | |
| "cuda_peak_bytes": 14954638336 | |
| } | |
| }, | |
| "budget44-conditional": { | |
| "precision_by_layer": { | |
| "2": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "3": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "4": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "5": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "6": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "7": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "8": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "9": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "10": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "11": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "12": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "13": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "14": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "15": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "16": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "17": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "18": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "19": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "20": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "21": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "22": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "23": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "24": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "25": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "26": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "27": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "28": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "29": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "30": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "31": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "32": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "33": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "34": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "35": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "36": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "37": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "38": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "39": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "40": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "41": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "42": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "43": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "44": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "45": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "46": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "47": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "48": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| } | |
| }, | |
| "tensor_bytes": 43340635712, | |
| "aggregate": { | |
| "all": { | |
| "tokens": 49140, | |
| "nll": 2.220883278078697, | |
| "teacher_nll": 2.214729754197971, | |
| "kl": 0.07120300278258629, | |
| "top1_agree": 0.9367114367114368, | |
| "perplexity": 9.21546709214557, | |
| "teacher_perplexity": 9.158933613767237, | |
| "delta_nll": 0.006153523880726208 | |
| }, | |
| "code": { | |
| "tokens": 16380, | |
| "nll": 1.6065521288267421, | |
| "teacher_nll": 1.585819314280127, | |
| "kl": 0.09249036310147381, | |
| "top1_agree": 0.9474969474969475, | |
| "perplexity": 4.98559188131849, | |
| "teacher_perplexity": 4.883290689738726, | |
| "delta_nll": 0.02073281454661524 | |
| }, | |
| "math": { | |
| "tokens": 16380, | |
| "nll": 1.5020923887926434, | |
| "teacher_nll": 1.5066873193514945, | |
| "kl": 0.02246947362928018, | |
| "top1_agree": 0.9605616605616606, | |
| "perplexity": 4.491076323788055, | |
| "teacher_perplexity": 4.511759991246355, | |
| "delta_nll": -0.004594930558851074 | |
| }, | |
| "prose": { | |
| "tokens": 16380, | |
| "nll": 3.554005316616706, | |
| "teacher_nll": 3.551682628962292, | |
| "kl": 0.0986491716170049, | |
| "top1_agree": 0.9020757020757021, | |
| "perplexity": 34.95303546720431, | |
| "teacher_perplexity": 34.871944693965524, | |
| "delta_nll": 0.002322687654413791 | |
| } | |
| }, | |
| "sequences": [ | |
| { | |
| "id": "prose-qualification-holdout-0", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.576349109840626, | |
| "teacher_nll": 3.570623514736674, | |
| "kl": 0.09326929702572478, | |
| "top1_agree": 0.9074481074481074 | |
| }, | |
| { | |
| "id": "prose-qualification-holdout-1", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.472557887635097, | |
| "teacher_nll": 3.4886688269683876, | |
| "kl": 0.11587396829556196, | |
| "top1_agree": 0.8971916971916972 | |
| }, | |
| { | |
| "id": "prose-qualification-holdout-2", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.585051614577371, | |
| "teacher_nll": 3.586854711586157, | |
| "kl": 0.08426370783748789, | |
| "top1_agree": 0.9086691086691087 | |
| }, | |
| { | |
| "id": "prose-qualification-holdout-3", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.5820626544137286, | |
| "teacher_nll": 3.56058346255795, | |
| "kl": 0.10118971330924494, | |
| "top1_agree": 0.894993894993895 | |
| }, | |
| { | |
| "id": "code-qualification-holdout-0", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.6318553607833546, | |
| "teacher_nll": 1.612040780315469, | |
| "kl": 0.10678286492751778, | |
| "top1_agree": 0.9416361416361416 | |
| }, | |
| { | |
| "id": "code-qualification-holdout-1", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.5190067352974954, | |
| "teacher_nll": 1.4962344744848826, | |
| "kl": 0.08591921168842304, | |
| "top1_agree": 0.9526251526251526 | |
| }, | |
| { | |
| "id": "code-qualification-holdout-2", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.4650198250608713, | |
| "teacher_nll": 1.4386393615759918, | |
| "kl": 0.07217798363303672, | |
| "top1_agree": 0.9545787545787546 | |
| }, | |
| { | |
| "id": "code-qualification-holdout-3", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.8103265941652478, | |
| "teacher_nll": 1.796362640744164, | |
| "kl": 0.10508139215691768, | |
| "top1_agree": 0.9411477411477411 | |
| }, | |
| { | |
| "id": "math-qualification-holdout-0", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.40823189257411, | |
| "teacher_nll": 1.4128543704711765, | |
| "kl": 0.02408212613273453, | |
| "top1_agree": 0.9604395604395605 | |
| }, | |
| { | |
| "id": "math-qualification-holdout-1", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.539740735474378, | |
| "teacher_nll": 1.545063196724879, | |
| "kl": 0.021373734383938223, | |
| "top1_agree": 0.9582417582417583 | |
| }, | |
| { | |
| "id": "math-qualification-holdout-2", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.527646278781914, | |
| "teacher_nll": 1.5522358872223831, | |
| "kl": 0.022497680037304013, | |
| "top1_agree": 0.9628815628815629 | |
| }, | |
| { | |
| "id": "math-qualification-holdout-3", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.5327506483401716, | |
| "teacher_nll": 1.516595822987539, | |
| "kl": 0.02192435396314395, | |
| "top1_agree": 0.9606837606837607 | |
| } | |
| ], | |
| "memory": { | |
| "swap_kib": 0, | |
| "cuda_peak_bytes": 14954638336 | |
| } | |
| }, | |
| "budget48-conditional": { | |
| "precision_by_layer": { | |
| "2": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "3": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "4": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "5": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "6": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "7": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "8": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "9": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "10": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "11": { | |
| "gate": "bf16", | |
| "up": "bf16", | |
| "down": "bf16" | |
| }, | |
| "12": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "13": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "14": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "15": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "16": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "17": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "18": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "19": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "20": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "21": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "22": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "23": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "24": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "25": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "26": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "27": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "28": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "29": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "30": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "31": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "32": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "33": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "34": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "35": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "36": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "37": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "38": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "39": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "40": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "41": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "42": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "43": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "44": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "45": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "46": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "47": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| }, | |
| "48": { | |
| "gate": "fp8", | |
| "up": "fp8", | |
| "down": "fp8" | |
| } | |
| }, | |
| "tensor_bytes": 47366184512, | |
| "aggregate": { | |
| "all": { | |
| "tokens": 49140, | |
| "nll": 2.221172728381314, | |
| "teacher_nll": 2.214729754197971, | |
| "kl": 0.06295151559269836, | |
| "top1_agree": 0.9396418396418397, | |
| "perplexity": 9.218134897964124, | |
| "teacher_perplexity": 9.158933613767237, | |
| "delta_nll": 0.006442974183343164 | |
| }, | |
| "code": { | |
| "tokens": 16380, | |
| "nll": 1.6024605643909346, | |
| "teacher_nll": 1.585819314280127, | |
| "kl": 0.07698328460729312, | |
| "top1_agree": 0.9495726495726495, | |
| "perplexity": 4.9652346856740985, | |
| "teacher_perplexity": 4.883290689738726, | |
| "delta_nll": 0.016641250110807748 | |
| }, | |
| "math": { | |
| "tokens": 16380, | |
| "nll": 1.5092922030787765, | |
| "teacher_nll": 1.5066873193514945, | |
| "kl": 0.018815637222048855, | |
| "top1_agree": 0.9614163614163614, | |
| "perplexity": 4.523527921820188, | |
| "teacher_perplexity": 4.511759991246355, | |
| "delta_nll": 0.00260488372728207 | |
| }, | |
| "prose": { | |
| "tokens": 16380, | |
| "nll": 3.5517654176742313, | |
| "teacher_nll": 3.551682628962292, | |
| "kl": 0.09305562494875311, | |
| "top1_agree": 0.9079365079365079, | |
| "perplexity": 34.874831816858496, | |
| "teacher_perplexity": 34.871944693965524, | |
| "delta_nll": 8.27887119392301e-05 | |
| } | |
| }, | |
| "sequences": [ | |
| { | |
| "id": "prose-qualification-holdout-0", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.579152346880008, | |
| "teacher_nll": 3.570623514736674, | |
| "kl": 0.08813064690765735, | |
| "top1_agree": 0.9091575091575091 | |
| }, | |
| { | |
| "id": "prose-qualification-holdout-1", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.4873519394423935, | |
| "teacher_nll": 3.4886688269683876, | |
| "kl": 0.1052800552312271, | |
| "top1_agree": 0.904029304029304 | |
| }, | |
| { | |
| "id": "prose-qualification-holdout-2", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.590019969451122, | |
| "teacher_nll": 3.586854711586157, | |
| "kl": 0.08358880894233602, | |
| "top1_agree": 0.9145299145299145 | |
| }, | |
| { | |
| "id": "prose-qualification-holdout-3", | |
| "domain": "prose", | |
| "tokens": 4095, | |
| "nll": 3.5505374149234012, | |
| "teacher_nll": 3.56058346255795, | |
| "kl": 0.09522298871379195, | |
| "top1_agree": 0.904029304029304 | |
| }, | |
| { | |
| "id": "code-qualification-holdout-0", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.6449129926707136, | |
| "teacher_nll": 1.612040780315469, | |
| "kl": 0.08681660701358129, | |
| "top1_agree": 0.947008547008547 | |
| }, | |
| { | |
| "id": "code-qualification-holdout-1", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.5195604561886071, | |
| "teacher_nll": 1.4962344744848826, | |
| "kl": 0.08596276142731109, | |
| "top1_agree": 0.9538461538461539 | |
| }, | |
| { | |
| "id": "code-qualification-holdout-2", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.4590187482635908, | |
| "teacher_nll": 1.4386393615759918, | |
| "kl": 0.053211267345263116, | |
| "top1_agree": 0.9540903540903541 | |
| }, | |
| { | |
| "id": "code-qualification-holdout-3", | |
| "domain": "code", | |
| "tokens": 4095, | |
| "nll": 1.7863500604408273, | |
| "teacher_nll": 1.796362640744164, | |
| "kl": 0.08194250264301696, | |
| "top1_agree": 0.9433455433455433 | |
| }, | |
| { | |
| "id": "math-qualification-holdout-0", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.4171055084619766, | |
| "teacher_nll": 1.4128543704711765, | |
| "kl": 0.019379515520163943, | |
| "top1_agree": 0.958974358974359 | |
| }, | |
| { | |
| "id": "math-qualification-holdout-1", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.5520651210649838, | |
| "teacher_nll": 1.545063196724879, | |
| "kl": 0.019911735824176244, | |
| "top1_agree": 0.9594627594627595 | |
| }, | |
| { | |
| "id": "math-qualification-holdout-2", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.5341272659208602, | |
| "teacher_nll": 1.5522358872223831, | |
| "kl": 0.01809579213314732, | |
| "top1_agree": 0.9631257631257631 | |
| }, | |
| { | |
| "id": "math-qualification-holdout-3", | |
| "domain": "math", | |
| "tokens": 4095, | |
| "nll": 1.5338709168672853, | |
| "teacher_nll": 1.516595822987539, | |
| "kl": 0.017875505410707915, | |
| "top1_agree": 0.9641025641025641 | |
| } | |
| ], | |
| "memory": { | |
| "swap_kib": 0, | |
| "cuda_peak_bytes": 14954638336 | |
| } | |
| } | |
| } | |
| } | |