{ "schema": "ts3_capacity_slice_diagnostics_v1", "sample_id": "normal_rodv2_0150__gas_inertia_30deg", "native_nodes": 33555, "groups": [ { "schema": "ts3_capacity_slice_diagnostics_v1", "variant": "S001", "sample_id": "normal_rodv2_0150__gas_inertia_30deg", "native_nodes": 33555, "layers": 16, "slices": 1, "parameters": 7593108, "checkpoint_sha256": "12ae0df85a6d2813c498115a15c1e6acdb9c971778899c793cf24f90cd42ab7a", "extraction_source_sha256": "26c5dec82b68ba4bc23d7c3d998b78c5bbed6aae912e6247ea64949db69bd206", "CPU_forward_seconds": 2.23619759200119, "CPU_threads": 2, "CUDA_initialized": false, "hook_max_abs_prediction_difference_on_19_node_test": 0.0, "metrics": [ { "layer": 1, "head": 1, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.2540487527688479, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 1, "head": 2, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.21913158601583174, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 1, "head": 3, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.2386139606767624, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 1, "head": 4, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.265112322420502, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 1, "head": 5, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.3261312054467814, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 1, "head": 6, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.2512414299165503, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 1, "head": 7, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.26226526181424353, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 1, "head": 8, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.20823700558376596, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 2, "head": 1, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.1952633417734538, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 2, "head": 2, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.22093848585590392, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 2, "head": 3, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.2616302477219166, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 2, "head": 4, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.228754824930891, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 2, "head": 5, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.23986824690543815, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 2, "head": 6, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.2139619009491463, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 2, "head": 7, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.23959849262864033, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 2, "head": 8, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.2391342255615331, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 3, "head": 1, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.2642847764556146, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 3, "head": 2, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.24409534543830794, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 3, "head": 3, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.24545704631577062, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 3, "head": 4, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.26532588906865184, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 3, "head": 5, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.25326000736143706, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 3, "head": 6, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.2696471363906173, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 3, "head": 7, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.21981474937309353, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 3, "head": 8, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.23900513001904655, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 4, "head": 1, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.26025139317457496, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 4, "head": 2, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.2232577529338082, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 4, "head": 3, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.2515941210385798, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 4, "head": 4, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.25740544922849945, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 4, "head": 5, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.24361990230249211, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 4, "head": 6, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.24452029489697782, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 4, "head": 7, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.2783163832603736, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 4, "head": 8, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.23115725637801426, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 5, "head": 1, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.23506016660434695, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 5, "head": 2, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.21951092761903457, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 5, "head": 3, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.21191543324599296, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 5, "head": 4, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.25740010854676604, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 5, "head": 5, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.23329855678089742, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 5, "head": 6, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.21993211512729235, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 5, "head": 7, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.2500507934078644, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 5, "head": 8, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.2015224415209554, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 6, "head": 1, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.25888373958893285, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 6, "head": 2, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.3121101227312728, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 6, "head": 3, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.24833664781141, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 6, "head": 4, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.1982480740280318, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 6, "head": 5, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.21511633658918078, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 6, "head": 6, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.3122770995256832, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 6, "head": 7, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.2355961652821313, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 6, "head": 8, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.25636396281401036, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 7, "head": 1, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.25861446895186985, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 7, "head": 2, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.22552927325073607, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 7, "head": 3, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.19555732246430213, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 7, "head": 4, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.23594978703693364, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 7, "head": 5, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.26987804417485134, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 7, "head": 6, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.2594847046260536, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 7, "head": 7, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.25112727931009465, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 7, "head": 8, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.24545650876696878, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 8, "head": 1, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.25077961530583825, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 8, "head": 2, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.25278669664072706, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 8, "head": 3, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.2104124125686699, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 8, "head": 4, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.28950949316597496, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 8, "head": 5, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.2745009338685503, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 8, "head": 6, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.23597586832404988, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 8, "head": 7, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.25016182132841785, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 8, "head": 8, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.2533802078269516, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 9, "head": 1, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.213387302776203, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 9, "head": 2, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.27655209501154004, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 9, "head": 3, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.21338502383676197, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 9, "head": 4, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.22324070009386768, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 9, "head": 5, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.2509320549363274, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 9, "head": 6, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.20162999681818566, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 9, "head": 7, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.22244529234844823, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 9, "head": 8, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.2344441799723526, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 10, "head": 1, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.21331118329447343, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 10, "head": 2, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.22153484402729431, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 10, "head": 3, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.21991356233498452, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 10, "head": 4, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.2590222281724737, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 10, "head": 5, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.23518023579894087, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 10, "head": 6, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.2375113400587913, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 10, "head": 7, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.22989347472792027, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 10, "head": 8, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.26983730468867995, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 11, "head": 1, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.22731358167267537, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 11, "head": 2, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.2447776325603274, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 11, "head": 3, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.20239413880998444, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 11, "head": 4, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.22786610386096634, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 11, "head": 5, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.23333239533439495, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 11, "head": 6, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.19617502376723917, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 11, "head": 7, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.2338218022271089, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 11, "head": 8, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.21062952958567968, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 12, "head": 1, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.22034855370094905, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 12, "head": 2, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.2558659933933427, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 12, "head": 3, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.2535990123447824, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 12, "head": 4, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.21011617314901077, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 12, "head": 5, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.17319599727928173, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 12, "head": 6, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.21602209823687687, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 12, "head": 7, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.26818922546501084, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 12, "head": 8, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.21730959750720513, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 13, "head": 1, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.1840827354418969, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 13, "head": 2, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.2057232068366599, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 13, "head": 3, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.23250147530808762, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 13, "head": 4, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.2789946855956538, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 13, "head": 5, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.2241172550664811, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 13, "head": 6, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.28756128343791887, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 13, "head": 7, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.2057958503642909, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 13, "head": 8, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.2380991887398022, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 14, "head": 1, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.2944562853698001, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 14, "head": 2, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.24485952579137368, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 14, "head": 3, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.16300126882193172, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 14, "head": 4, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.20139555083814248, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 14, "head": 5, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.2286514528579672, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 14, "head": 6, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.23222885615146516, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 14, "head": 7, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.2054154746250281, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 14, "head": 8, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.1714878449469573, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 15, "head": 1, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.14243787053945062, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 15, "head": 2, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.17299072827330578, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 15, "head": 3, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.22072597172111286, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 15, "head": 4, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.22118647650452003, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 15, "head": 5, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.21116405005417088, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 15, "head": 6, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.17092261636989214, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 15, "head": 7, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.1876016115220343, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 15, "head": 8, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.21037958957306696, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 16, "head": 1, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.21553471681390696, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 16, "head": 2, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.17559998983092112, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 16, "head": 3, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.1935668751141409, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 16, "head": 4, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.18039522842940672, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 16, "head": 5, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.1549242430886563, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 16, "head": 6, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.21747203976918403, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 16, "head": 7, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.17900566441898444, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 }, { "layer": 16, "head": 8, "slices": 1, "native_nodes": 33555, "mass": [ 1.0 ], "mean_node_assignment_entropy": 0.0, "normalized_node_entropy": null, "effective_mass_slots": 1.0, "node_slot_mutual_information_nats": -0.0, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "centered_token_RMS": 0.0, "token_RMS": 0.18998670201619106, "mean_token_offdiagonal_cosine": null, "temperature": 0.5 } ] }, { "schema": "ts3_capacity_slice_diagnostics_v1", "variant": "S002", "sample_id": "normal_rodv2_0150__gas_inertia_30deg", "native_nodes": 33555, "layers": 16, "slices": 2, "parameters": 7593636, "checkpoint_sha256": "240ccea940878b4ceea31ce558398b0b67cc7d044788cf0383cc4a9c780909d9", "extraction_source_sha256": "26c5dec82b68ba4bc23d7c3d998b78c5bbed6aae912e6247ea64949db69bd206", "CPU_forward_seconds": 2.386923731000934, "CPU_threads": 2, "CUDA_initialized": false, "hook_max_abs_prediction_difference_on_19_node_test": 0.0, "metrics": [ { "layer": 1, "head": 1, "slices": 2, "native_nodes": 33555, "mass": [ 0.5042140080141534, 0.49578599204357715 ], "mean_node_assignment_entropy": 0.6930692083658823, "normalized_node_entropy": 0.999887509902298, "effective_mass_slots": 1.9999289689318223, "node_slot_mutual_information_nats": 4.2456029282655905e-05, "normalized_node_slot_mutual_information": 6.125110290192429e-05, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.0006861305265950541, "token_RMS": 0.24192836974842627, "mean_token_offdiagonal_cosine": 0.9999841714024447, "temperature": 0.4999331831932068 }, { "layer": 1, "head": 2, "slices": 2, "native_nodes": 33555, "mass": [ 0.4600009767109654, 0.5399990232925873 ], "mean_node_assignment_entropy": 0.689803852431914, "normalized_node_entropy": 0.9951765970896247, "effective_mass_slots": 1.9936037189696592, "node_slot_mutual_information_nats": 0.00014006263162624766, "normalized_node_slot_mutual_information": 0.0002020676640610452, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.001292212381407502, "token_RMS": 0.21715894261343935, "mean_token_offdiagonal_cosine": 0.9999369866255833, "temperature": 0.5032339692115784 }, { "layer": 1, "head": 3, "slices": 2, "native_nodes": 33555, "mass": [ 0.4749450222139582, 0.5250549778340027 ], "mean_node_assignment_entropy": 0.6915659191281882, "normalized_node_entropy": 0.9977187219740551, "effective_mass_slots": 1.997489517352185, "node_slot_mutual_information_nats": 0.0003252316325720317, "normalized_node_slot_mutual_information": 0.0004692100634518916, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.0023231290922085426, "token_RMS": 0.21118175368272424, "mean_token_offdiagonal_cosine": 0.9997579818140211, "temperature": 0.5007084608078003 }, { "layer": 1, "head": 4, "slices": 2, "native_nodes": 33555, "mass": [ 0.465696845306332, 0.534303154927255 ], "mean_node_assignment_entropy": 0.6905728983109469, "normalized_node_entropy": 0.9962860957655216, "effective_mass_slots": 1.995295017709964, "node_slot_mutual_information_nats": 0.00021901964928505713, "normalized_node_slot_mutual_information": 0.0003159785618807919, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.0014549882287594137, "token_RMS": 0.25718473356699034, "mean_token_offdiagonal_cosine": 0.9999383346153118, "temperature": 0.49949538707733154 }, { "layer": 1, "head": 5, "slices": 2, "native_nodes": 33555, "mass": [ 0.5024882296863247, 0.4975117704264721 ], "mean_node_assignment_entropy": 0.6930471037882483, "normalized_node_entropy": 0.9998556197377646, "effective_mass_slots": 1.9999752348351236, "node_slot_mutual_information_nats": 8.769411259401672e-05, "normalized_node_slot_mutual_information": 0.00012651586135454631, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.000792551544658364, "token_RMS": 0.3321923770391577, "mean_token_offdiagonal_cosine": 0.9999886160723943, "temperature": 0.5047767758369446 }, { "layer": 1, "head": 6, "slices": 2, "native_nodes": 33555, "mass": [ 0.4812272187939097, 0.5187727815586912 ], "mean_node_assignment_entropy": 0.6917717675803213, "normalized_node_entropy": 0.9980156985151222, "effective_mass_slots": 1.998590496028148, "node_slot_mutual_information_nats": 0.0006704125392777982, "normalized_node_slot_mutual_information": 0.0009672008457658569, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.0033503263691521223, "token_RMS": 0.235061479331822, "mean_token_offdiagonal_cosine": 0.9996012102949072, "temperature": 0.5008800625801086 }, { "layer": 1, "head": 7, "slices": 2, "native_nodes": 33555, "mass": [ 0.49136696396224316, 0.5086330359560458 ], "mean_node_assignment_entropy": 0.6926246684596926, "normalized_node_entropy": 0.999246174384161, "effective_mass_slots": 1.9997018902135144, "node_slot_mutual_information_nats": 0.0003734460972253384, "normalized_node_slot_mutual_information": 0.0005387688325063334, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.00259084259740307, "token_RMS": 0.2605537705657692, "mean_token_offdiagonal_cosine": 0.9998055340303605, "temperature": 0.5015496611595154 }, { "layer": 1, "head": 8, "slices": 2, "native_nodes": 33555, "mass": [ 0.5359515830961039, 0.4640484169589622 ], "mean_node_assignment_entropy": 0.6900384551664502, "normalized_node_entropy": 0.9955150572913191, "effective_mass_slots": 1.9948321586678603, "node_slot_mutual_information_nats": 0.0005214606426122703, "normalized_node_slot_mutual_information": 0.0007523086831154944, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.0034692385371676033, "token_RMS": 0.23146197491502488, "mean_token_offdiagonal_cosine": 0.9995510398385676, "temperature": 0.5013393759727478 }, { "layer": 2, "head": 1, "slices": 2, "native_nodes": 33555, "mass": [ 0.4820827097557746, 0.5179172899777764 ], "mean_node_assignment_entropy": 0.6921743577484679, "normalized_node_entropy": 0.998596513354218, "effective_mass_slots": 1.9987160203834897, "node_slot_mutual_information_nats": 0.0003306268395238954, "normalized_node_slot_mutual_information": 0.000476993701765915, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.001754025030339331, "token_RMS": 0.18462859657504468, "mean_token_offdiagonal_cosine": 0.999832079057639, "temperature": 0.501605749130249 }, { "layer": 2, "head": 2, "slices": 2, "native_nodes": 33555, "mass": [ 0.42785047376729607, 0.5721495262540198 ], "mean_node_assignment_entropy": 0.68214116706297, "normalized_node_entropy": 0.9841216789079568, "effective_mass_slots": 1.9792136860265344, "node_slot_mutual_information_nats": 0.0005584704977321664, "normalized_node_slot_mutual_information": 0.0008057026175609875, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.0022800678301575163, "token_RMS": 0.250179015715364, "mean_token_offdiagonal_cosine": 0.9998563411056156, "temperature": 0.49829161167144775 }, { "layer": 2, "head": 3, "slices": 2, "native_nodes": 33555, "mass": [ 0.44709229571882186, 0.5529077045653904 ], "mean_node_assignment_entropy": 0.6866909918764929, "normalized_node_entropy": 0.9906856886033397, "effective_mass_slots": 1.9888135113599623, "node_slot_mutual_information_nats": 0.0008472435995797722, "normalized_node_slot_mutual_information": 0.001222314139538652, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.0033515306088047433, "token_RMS": 0.26904457460020953, "mean_token_offdiagonal_cosine": 0.9996897878172876, "temperature": 0.5005266666412354 }, { "layer": 2, "head": 4, "slices": 2, "native_nodes": 33555, "mass": [ 0.49525872907256996, 0.5047412710020358 ], "mean_node_assignment_entropy": 0.6929922571055289, "normalized_node_entropy": 0.9997764927005961, "effective_mass_slots": 1.9999100820265403, "node_slot_mutual_information_nats": 0.00010996345700098331, "normalized_node_slot_mutual_information": 0.0001586437340943254, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.001079663662111404, "token_RMS": 0.24896081939408923, "mean_token_offdiagonal_cosine": 0.9999649787186048, "temperature": 0.500329852104187 }, { "layer": 2, "head": 5, "slices": 2, "native_nodes": 33555, "mass": [ 0.5443703964594633, 0.4556296035112274 ], "mean_node_assignment_entropy": 0.686762518492555, "normalized_node_entropy": 0.9907888796976241, "effective_mass_slots": 1.992130227307525, "node_slot_mutual_information_nats": 0.0024420136873692933, "normalized_node_slot_mutual_information": 0.003523081036550651, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.009890496209105317, "token_RMS": 0.22214716891535063, "mean_token_offdiagonal_cosine": 0.9960841560956606, "temperature": 0.4995597302913666 }, { "layer": 2, "head": 6, "slices": 2, "native_nodes": 33555, "mass": [ 0.5229890277427989, 0.47701097241973506 ], "mean_node_assignment_entropy": 0.6911825385782782, "normalized_node_entropy": 0.9971656207559265, "effective_mass_slots": 1.9978863905028492, "node_slot_mutual_information_nats": 0.0009072784212154072, "normalized_node_slot_mutual_information": 0.0013089260789930363, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.00570643447863503, "token_RMS": 0.2370230968369557, "mean_token_offdiagonal_cosine": 0.9988551742509028, "temperature": 0.5016686320304871 }, { "layer": 2, "head": 7, "slices": 2, "native_nodes": 33555, "mass": [ 0.44600257707626234, 0.5539974231964038 ], "mean_node_assignment_entropy": 0.6706684879659378, "normalized_node_entropy": 0.967570101668958, "effective_mass_slots": 1.9883484084254945, "node_slot_mutual_information_nats": 0.016635860660180746, "normalized_node_slot_mutual_information": 0.02400047367536256, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.03218502261804558, "token_RMS": 0.2445290413018073, "mean_token_offdiagonal_cosine": 0.9690770061808683, "temperature": 0.4976445436477661 }, { "layer": 2, "head": 8, "slices": 2, "native_nodes": 33555, "mass": [ 0.5056391693301961, 0.49436083022483396 ], "mean_node_assignment_entropy": 0.6914522280937553, "normalized_node_entropy": 0.9975547004824852, "effective_mass_slots": 1.9998728006883193, "node_slot_mutual_information_nats": 0.001631350787805852, "normalized_node_slot_mutual_information": 0.0023535416915178064, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.004362027315214179, "token_RMS": 0.2592769887502321, "mean_token_offdiagonal_cosine": 0.9994602823536304, "temperature": 0.49933499097824097 }, { "layer": 3, "head": 1, "slices": 2, "native_nodes": 33555, "mass": [ 0.5039826648119149, 0.4960173352120655 ], "mean_node_assignment_entropy": 0.6844691080842433, "normalized_node_entropy": 0.9874801878748298, "effective_mass_slots": 1.9999365538450993, "node_slot_mutual_information_nats": 0.008646348895064104, "normalized_node_slot_mutual_information": 0.012474044672704751, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.02316211675784357, "token_RMS": 0.2146648107752578, "mean_token_offdiagonal_cosine": 0.9783036985774928, "temperature": 0.49662914872169495 }, { "layer": 3, "head": 2, "slices": 2, "native_nodes": 33555, "mass": [ 0.4968169179363802, 0.5031830821169095 ], "mean_node_assignment_entropy": 0.6927497183619685, "normalized_node_entropy": 0.9994265832580381, "effective_mass_slots": 1.9999594720578004, "node_slot_mutual_information_nats": 0.00037719802155999904, "normalized_node_slot_mutual_information": 0.000544181715137739, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.004196973025686096, "token_RMS": 0.22998143069030477, "mean_token_offdiagonal_cosine": 0.9993465555364317, "temperature": 0.5023812055587769 }, { "layer": 3, "head": 3, "slices": 2, "native_nodes": 33555, "mass": [ 0.4966322805421076, 0.5033677194419054 ], "mean_node_assignment_entropy": 0.692675479615084, "normalized_node_entropy": 0.999319479386066, "effective_mass_slots": 1.9999546340441425, "node_slot_mutual_information_nats": 0.0004490177096698744, "normalized_node_slot_mutual_information": 0.0006477956230120481, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.003644584412729649, "token_RMS": 0.23115153613483505, "mean_token_offdiagonal_cosine": 0.9995215411590237, "temperature": 0.5009551644325256 }, { "layer": 3, "head": 4, "slices": 2, "native_nodes": 33555, "mass": [ 0.47575331040314034, 0.5242466892229429 ], "mean_node_assignment_entropy": 0.6894188713487905, "normalized_node_entropy": 0.9946211867901664, "effective_mass_slots": 1.997648852951779, "node_slot_mutual_information_nats": 0.0025520441584740983, "normalized_node_slot_mutual_information": 0.0036818214515602294, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.011868086756828109, "token_RMS": 0.2366617545057558, "mean_token_offdiagonal_cosine": 0.9951524105160227, "temperature": 0.49851495027542114 }, { "layer": 3, "head": 5, "slices": 2, "native_nodes": 33555, "mass": [ 0.48390251005429513, 0.5160974901588641 ], "mean_node_assignment_entropy": 0.689896198591436, "normalized_node_entropy": 0.9953098244360122, "effective_mass_slots": 1.9989635726282653, "node_slot_mutual_information_nats": 0.002732633963523856, "normalized_node_slot_mutual_information": 0.00394235746774062, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.011222691054460693, "token_RMS": 0.2511336541489108, "mean_token_offdiagonal_cosine": 0.9960347210576634, "temperature": 0.49738621711730957 }, { "layer": 3, "head": 6, "slices": 2, "native_nodes": 33555, "mass": [ 0.5024537418819792, 0.49754625779117667 ], "mean_node_assignment_entropy": 0.6856602024639149, "normalized_node_entropy": 0.9891985738296126, "effective_mass_slots": 1.9999759168488083, "node_slot_mutual_information_nats": 0.007474936447934177, "normalized_node_slot_mutual_information": 0.010784053744394801, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.02239984513762149, "token_RMS": 0.28911883836751057, "mean_token_offdiagonal_cosine": 0.9888734108163888, "temperature": 0.497834712266922 }, { "layer": 3, "head": 7, "slices": 2, "native_nodes": 33555, "mass": [ 0.4781503229787108, 0.5218496771323097 ], "mean_node_assignment_entropy": 0.6913719420219512, "normalized_node_entropy": 0.997438872164841, "effective_mass_slots": 1.9980906700961503, "node_slot_mutual_information_nats": 0.0008201176032537338, "normalized_node_slot_mutual_information": 0.0011831795991599043, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.005577410068680002, "token_RMS": 0.24968364225174464, "mean_token_offdiagonal_cosine": 0.9990057434450658, "temperature": 0.4978897273540497 }, { "layer": 3, "head": 8, "slices": 2, "native_nodes": 33555, "mass": [ 0.48646405582864244, 0.5135359441518179 ], "mean_node_assignment_entropy": 0.6925859765824695, "normalized_node_entropy": 0.9991903538047685, "effective_mass_slots": 1.9992671576250423, "node_slot_mutual_information_nats": 0.00019471564135009611, "normalized_node_slot_mutual_information": 0.00028091529015929765, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.0025148418818458284, "token_RMS": 0.24771374701812032, "mean_token_offdiagonal_cosine": 0.9997964701071063, "temperature": 0.5000970363616943 }, { "layer": 4, "head": 1, "slices": 2, "native_nodes": 33555, "mass": [ 0.45661518764449666, 0.5433848123315229 ], "mean_node_assignment_entropy": 0.6757865376616299, "normalized_node_entropy": 0.9749538865839562, "effective_mass_slots": 1.9924757453205066, "node_slot_mutual_information_nats": 0.01359142095805832, "normalized_node_slot_mutual_information": 0.019608275614825064, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.028240902044139597, "token_RMS": 0.2906699397970188, "mean_token_offdiagonal_cosine": 0.9821637618447405, "temperature": 0.4920922517776489 }, { "layer": 4, "head": 2, "slices": 2, "native_nodes": 33555, "mass": [ 0.4962281712422053, 0.5037718290713163 ], "mean_node_assignment_entropy": 0.6648532091597974, "normalized_node_entropy": 0.9591804277739525, "effective_mass_slots": 1.9999430933040312, "node_slot_mutual_information_nats": 0.028265517647359295, "normalized_node_slot_mutual_information": 0.04077852213800474, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.02802369802512623, "token_RMS": 0.2212684681098458, "mean_token_offdiagonal_cosine": 0.9729485646011269, "temperature": 0.4936912953853607 }, { "layer": 4, "head": 3, "slices": 2, "native_nodes": 33555, "mass": [ 0.4689569163617319, 0.531043083278562 ], "mean_node_assignment_entropy": 0.6908865579650835, "normalized_node_entropy": 0.9967386109930714, "effective_mass_slots": 1.996146544883127, "node_slot_mutual_information_nats": 0.00033203650924906736, "normalized_node_slot_mutual_information": 0.00047902742528771193, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.0020067103612623626, "token_RMS": 0.3123387511435472, "mean_token_offdiagonal_cosine": 0.999921196744911, "temperature": 0.5023165345191956 }, { "layer": 4, "head": 4, "slices": 2, "native_nodes": 33555, "mass": [ 0.45281051695035973, 0.5471894828053954 ], "mean_node_assignment_entropy": 0.6858315548315785, "normalized_node_entropy": 0.9894457830406855, "effective_mass_slots": 1.991099205011485, "node_slot_mutual_information_nats": 0.0028552957351801744, "normalized_node_slot_mutual_information": 0.0041193209974158445, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.011426446514694058, "token_RMS": 0.2434976127272645, "mean_token_offdiagonal_cosine": 0.9955960697831879, "temperature": 0.4945114850997925 }, { "layer": 4, "head": 5, "slices": 2, "native_nodes": 33555, "mass": [ 0.5170352786846016, 0.4829647218021119 ], "mean_node_assignment_entropy": 0.6842337178934098, "normalized_node_entropy": 0.9871405916138404, "effective_mass_slots": 1.9988393091052368, "node_slot_mutual_information_nats": 0.008332948753552727, "normalized_node_slot_mutual_information": 0.012021903842732388, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.019591366242444626, "token_RMS": 0.24615421967641415, "mean_token_offdiagonal_cosine": 0.9880668869267277, "temperature": 0.5004425644874573 }, { "layer": 4, "head": 6, "slices": 2, "native_nodes": 33555, "mass": [ 0.46491861025954195, 0.5350813902271716 ], "mean_node_assignment_entropy": 0.6883030232582842, "normalized_node_entropy": 0.9930113582836074, "effective_mass_slots": 1.995079200532154, "node_slot_mutual_information_nats": 0.00238072581040194, "normalized_node_slot_mutual_information": 0.0034346613203832376, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.012322606532589, "token_RMS": 0.20216779654448677, "mean_token_offdiagonal_cosine": 0.9926116955907268, "temperature": 0.49627915024757385 }, { "layer": 4, "head": 7, "slices": 2, "native_nodes": 33555, "mass": [ 0.4931798479184364, 0.5068201523675522 ], "mean_node_assignment_entropy": 0.6858961063381179, "normalized_node_entropy": 0.9895389111790519, "effective_mass_slots": 1.9998139448037267, "node_slot_mutual_information_nats": 0.007158042296355305, "normalized_node_slot_mutual_information": 0.010326872123425247, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.02189361072771912, "token_RMS": 0.27332720016535494, "mean_token_offdiagonal_cosine": 0.9872363269006852, "temperature": 0.496490478515625 }, { "layer": 4, "head": 8, "slices": 2, "native_nodes": 33555, "mass": [ 0.49678590836458886, 0.5032140914773181 ], "mean_node_assignment_entropy": 0.6930783603590659, "normalized_node_entropy": 0.9999007134374783, "effective_mass_slots": 1.9999586787011705, "node_slot_mutual_information_nats": 4.8159338030484555e-05, "normalized_node_slot_mutual_information": 6.947923814907533e-05, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.0007094135770329997, "token_RMS": 0.2642967734072945, "mean_token_offdiagonal_cosine": 0.999987666231101, "temperature": 0.5021019577980042 }, { "layer": 5, "head": 1, "slices": 2, "native_nodes": 33555, "mass": [ 0.511099831527897, 0.48890016840904343 ], "mean_node_assignment_entropy": 0.6919881687139462, "normalized_node_entropy": 0.9983278993574456, "effective_mass_slots": 1.999507195232786, "node_slot_mutual_information_nats": 0.000912579100337263, "normalized_node_slot_mutual_information": 0.001316573342475481, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.005961273457402709, "token_RMS": 0.2810144760023797, "mean_token_offdiagonal_cosine": 0.9990999884245373, "temperature": 0.4994153678417206 }, { "layer": 5, "head": 2, "slices": 2, "native_nodes": 33555, "mass": [ 0.44016006386434603, 0.5598399359296001 ], "mean_node_assignment_entropy": 0.6557787934595677, "normalized_node_entropy": 0.9460888132442661, "effective_mass_slots": 1.9856937507550063, "node_slot_mutual_information_nats": 0.030189556221893343, "normalized_node_slot_mutual_information": 0.043554323047964076, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.033566218119827025, "token_RMS": 0.22175285692726396, "mean_token_offdiagonal_cosine": 0.9549860203134904, "temperature": 0.49588319659233093 }, { "layer": 5, "head": 3, "slices": 2, "native_nodes": 33555, "mass": [ 0.4814601295059368, 0.5185398705651163 ], "mean_node_assignment_entropy": 0.6821050175939818, "normalized_node_entropy": 0.9840695262483167, "effective_mass_slots": 1.9986252502253063, "node_slot_mutual_information_nats": 0.010354551728185002, "normalized_node_slot_mutual_information": 0.01493846042888075, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.017898203681436495, "token_RMS": 0.2028902183965227, "mean_token_offdiagonal_cosine": 0.986710042372661, "temperature": 0.4978090226650238 }, { "layer": 5, "head": 4, "slices": 2, "native_nodes": 33555, "mass": [ 0.44952573029477344, 0.5504742700382879 ], "mean_node_assignment_entropy": 0.6853115339424313, "normalized_node_entropy": 0.9886955514827541, "effective_mass_slots": 1.9898180195869657, "node_slot_mutual_information_nats": 0.0027316531686615297, "normalized_node_slot_mutual_information": 0.003940942479856612, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.01239042202625434, "token_RMS": 0.27224975844641164, "mean_token_offdiagonal_cosine": 0.9962134327422634, "temperature": 0.4997653067111969 }, { "layer": 5, "head": 5, "slices": 2, "native_nodes": 33555, "mass": [ 0.524723439926434, 0.4752765607565637 ], "mean_node_assignment_entropy": 0.685248691449087, "normalized_node_entropy": 0.9886048889292492, "effective_mass_slots": 1.9975555035238308, "node_slot_mutual_information_nats": 0.006675493318202386, "normalized_node_slot_mutual_information": 0.009630701105657993, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.018021987981046293, "token_RMS": 0.20747376690289146, "mean_token_offdiagonal_cosine": 0.9876081136963643, "temperature": 0.4964768886566162 }, { "layer": 5, "head": 6, "slices": 2, "native_nodes": 33555, "mass": [ 0.39972376721774994, 0.6002762327373978 ], "mean_node_assignment_entropy": 0.6084363283175358, "normalized_node_entropy": 0.8777880735603981, "effective_mass_slots": 1.9599118648071974, "node_slot_mutual_information_nats": 0.06446317697874371, "normalized_node_slot_mutual_information": 0.09300070574718114, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.0687537014332024, "token_RMS": 0.23770068073728515, "mean_token_offdiagonal_cosine": 0.8541499814360151, "temperature": 0.49004611372947693 }, { "layer": 5, "head": 7, "slices": 2, "native_nodes": 33555, "mass": [ 0.48575298843094605, 0.5142470115033299 ], "mean_node_assignment_entropy": 0.6922224951911718, "normalized_node_entropy": 0.998665961004088, "effective_mass_slots": 1.999188145609097, "node_slot_mutual_information_nats": 0.0005186757625753602, "normalized_node_slot_mutual_information": 0.0007482909504967736, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.0027703040746419137, "token_RMS": 0.23026345906857199, "mean_token_offdiagonal_cosine": 0.9997968322123119, "temperature": 0.49945351481437683 }, { "layer": 5, "head": 8, "slices": 2, "native_nodes": 33555, "mass": [ 0.522482357358563, 0.47751764270538477 ], "mean_node_assignment_entropy": 0.690994292361562, "normalized_node_entropy": 0.996894038872604, "effective_mass_slots": 1.9979785148384952, "node_slot_mutual_information_nats": 0.001141634472895836, "normalized_node_slot_mutual_information": 0.0016470303925547084, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.005680118936359478, "token_RMS": 0.23616471638247746, "mean_token_offdiagonal_cosine": 0.9990775665323047, "temperature": 0.5009503960609436 }, { "layer": 6, "head": 1, "slices": 2, "native_nodes": 33555, "mass": [ 0.4632820827633058, 0.5367179167712965 ], "mean_node_assignment_entropy": 0.6891350356425255, "normalized_node_entropy": 0.9942116984243107, "effective_mass_slots": 1.9946095981794143, "node_slot_mutual_information_nats": 0.001313305413856436, "normalized_node_slot_mutual_information": 0.001894699207743308, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.008214608295225276, "token_RMS": 0.24137978024030568, "mean_token_offdiagonal_cosine": 0.9979401237389319, "temperature": 0.5015740990638733 }, { "layer": 6, "head": 2, "slices": 2, "native_nodes": 33555, "mass": [ 0.5277905708834731, 0.4722094290428093 ], "mean_node_assignment_entropy": 0.6073543287136728, "normalized_node_entropy": 0.8762270780976611, "effective_mass_slots": 1.9969115312742178, "node_slot_mutual_information_nats": 0.08424742392458306, "normalized_node_slot_mutual_information": 0.12154334070366621, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.07952627866532866, "token_RMS": 0.2900534145266648, "mean_token_offdiagonal_cosine": 0.8620810849720089, "temperature": 0.48686307668685913 }, { "layer": 6, "head": 3, "slices": 2, "native_nodes": 33555, "mass": [ 0.515187312442559, 0.484812687513921 ], "mean_node_assignment_entropy": 0.6194909858031434, "normalized_node_entropy": 0.8937365730936103, "effective_mass_slots": 1.999077453102765, "node_slot_mutual_information_nats": 0.07319481488886037, "normalized_node_slot_mutual_information": 0.10559779645894453, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.07698315863742101, "token_RMS": 0.24076106858179364, "mean_token_offdiagonal_cosine": 0.802865726670559, "temperature": 0.49018317461013794 }, { "layer": 6, "head": 4, "slices": 2, "native_nodes": 33555, "mass": [ 0.5622172934441289, 0.4377827063382711 ], "mean_node_assignment_entropy": 0.6834994437152186, "normalized_node_entropy": 0.9860812578983111, "effective_mass_slots": 1.9845359196447334, "node_slot_mutual_information_nats": 0.0018856494600867357, "normalized_node_slot_mutual_information": 0.002720417124922085, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.008944369225846432, "token_RMS": 0.2033263332023758, "mean_token_offdiagonal_cosine": 0.99641500660384, "temperature": 0.5003695487976074 }, { "layer": 6, "head": 5, "slices": 2, "native_nodes": 33555, "mass": [ 0.5391300071442185, 0.46086999249429894 ], "mean_node_assignment_entropy": 0.6505560550293261, "normalized_node_entropy": 0.9385539944110964, "effective_mass_slots": 1.9938784904910527, "node_slot_mutual_information_nats": 0.03952567708637633, "normalized_node_slot_mutual_information": 0.057023498320293665, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.06328878645121551, "token_RMS": 0.22762430285509716, "mean_token_offdiagonal_cosine": 0.8736729240708406, "temperature": 0.4951658546924591 }, { "layer": 6, "head": 6, "slices": 2, "native_nodes": 33555, "mass": [ 0.4725176549925035, 0.5274823447716892 ], "mean_node_assignment_entropy": 0.6047604671069973, "normalized_node_entropy": 0.872484926820958, "effective_mass_slots": 1.9969796429267075, "node_slot_mutual_information_nats": 0.08687539344733619, "normalized_node_slot_mutual_information": 0.12533469930174948, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.07019496046889293, "token_RMS": 0.3055208580769206, "mean_token_offdiagonal_cosine": 0.8945386580703343, "temperature": 0.4912426173686981 }, { "layer": 6, "head": 7, "slices": 2, "native_nodes": 33555, "mass": [ 0.5113766678957696, 0.4886233322720932 ], "mean_node_assignment_entropy": 0.6904031942324196, "normalized_node_entropy": 0.9960412645330116, "effective_mass_slots": 1.9994823079472948, "node_slot_mutual_information_nats": 0.0024851067947583294, "normalized_node_slot_mutual_information": 0.0035852512488773088, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.01363034069710379, "token_RMS": 0.2962560426617858, "mean_token_offdiagonal_cosine": 0.9957674562755447, "temperature": 0.5031158328056335 }, { "layer": 6, "head": 8, "slices": 2, "native_nodes": 33555, "mass": [ 0.5024619187857173, 0.4975380813928035 ], "mean_node_assignment_entropy": 0.6665661895616702, "normalized_node_entropy": 0.9616517361048744, "effective_mass_slots": 1.999975755764756, "node_slot_mutual_information_nats": 0.02656886880717957, "normalized_node_slot_mutual_information": 0.03833077527014744, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.03724667497542421, "token_RMS": 0.2275346146001166, "mean_token_offdiagonal_cosine": 0.9515221684804617, "temperature": 0.4992968738079071 }, { "layer": 7, "head": 1, "slices": 2, "native_nodes": 33555, "mass": [ 0.5987704363416905, 0.4012295637155961 ], "mean_node_assignment_entropy": 0.5175828168397971, "normalized_node_entropy": 0.746714163104116, "effective_mass_slots": 1.961102987334172, "node_slot_mutual_information_nats": 0.1559242467588764, "normalized_node_slot_mutual_information": 0.224951137553378, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.09118071669138735, "token_RMS": 0.2902531216864184, "mean_token_offdiagonal_cosine": 0.8147939424905357, "temperature": 0.4865035116672516 }, { "layer": 7, "head": 2, "slices": 2, "native_nodes": 33555, "mass": [ 0.5402961981303644, 0.4597038021689466 ], "mean_node_assignment_entropy": 0.6640676451211599, "normalized_node_entropy": 0.9580470984311095, "effective_mass_slots": 1.9935083742261726, "node_slot_mutual_information_nats": 0.025828443474864016, "normalized_node_slot_mutual_information": 0.037262567315067226, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.033685338138372364, "token_RMS": 0.22138159622513753, "mean_token_offdiagonal_cosine": 0.9555404499414862, "temperature": 0.4971703886985779 }, { "layer": 7, "head": 3, "slices": 2, "native_nodes": 33555, "mass": [ 0.4641189427795717, 0.5358810569113475 ], "mean_node_assignment_entropy": 0.6898097952693174, "normalized_node_entropy": 0.9951851707916753, "effective_mass_slots": 1.9948524057783084, "node_slot_mutual_information_nats": 0.0007602702696904995, "normalized_node_slot_mutual_information": 0.0010968381478177985, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.004540515469127806, "token_RMS": 0.20266096353355875, "mean_token_offdiagonal_cosine": 0.9994391942247833, "temperature": 0.5016118884086609 }, { "layer": 7, "head": 4, "slices": 2, "native_nodes": 33555, "mass": [ 0.4691462312054126, 0.5308537690228454 ], "mean_node_assignment_entropy": 0.6909406753008165, "normalized_node_entropy": 0.9968166858049594, "effective_mass_slots": 1.996193386546182, "node_slot_mutual_information_nats": 0.00030138494239428937, "normalized_node_slot_mutual_information": 0.00043480656179084717, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.002155095483266746, "token_RMS": 0.2109053376141382, "mean_token_offdiagonal_cosine": 0.9998084907968443, "temperature": 0.5016125440597534 }, { "layer": 7, "head": 5, "slices": 2, "native_nodes": 33555, "mass": [ 0.5125331971759396, 0.48746680302300904 ], "mean_node_assignment_entropy": 0.6537827475378056, "normalized_node_entropy": 0.9432091276915534, "effective_mass_slots": 1.9993717086553007, "node_slot_mutual_information_nats": 0.03905023799570173, "normalized_node_slot_mutual_information": 0.05633758470193266, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.05585096731410679, "token_RMS": 0.24264030829152683, "mean_token_offdiagonal_cosine": 0.9000764285958744, "temperature": 0.4989388585090637 }, { "layer": 7, "head": 6, "slices": 2, "native_nodes": 33555, "mass": [ 0.5176571770141385, 0.4823428230693489 ], "mean_node_assignment_entropy": 0.6894378053511631, "normalized_node_entropy": 0.9946485027814934, "effective_mass_slots": 1.9987530259114599, "node_slot_mutual_information_nats": 0.003085693715636606, "normalized_node_slot_mutual_information": 0.004451715021251171, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.012671115460505302, "token_RMS": 0.2295311859146926, "mean_token_offdiagonal_cosine": 0.9939132322036138, "temperature": 0.5019489526748657 }, { "layer": 7, "head": 7, "slices": 2, "native_nodes": 33555, "mass": [ 0.475345727274198, 0.5246542727160323 ], "mean_node_assignment_entropy": 0.690492052054995, "normalized_node_entropy": 0.9961694592729853, "effective_mass_slots": 1.9975691596075473, "node_slot_mutual_information_nats": 0.001438969086559605, "normalized_node_slot_mutual_information": 0.0020759935651720636, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.006840080643193043, "token_RMS": 0.2588375090594834, "mean_token_offdiagonal_cosine": 0.9987184958060372, "temperature": 0.49851441383361816 }, { "layer": 7, "head": 8, "slices": 2, "native_nodes": 33555, "mass": [ 0.5186260192871378, 0.48137398078125077 ], "mean_node_assignment_entropy": 0.6436857340400958, "normalized_node_entropy": 0.9286422163906184, "effective_mass_slots": 1.998612445997724, "node_slot_mutual_information_nats": 0.0487674287440788, "normalized_node_slot_mutual_information": 0.07035652760598837, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.057275630916846586, "token_RMS": 0.22892829619928529, "mean_token_offdiagonal_cosine": 0.8826146928204126, "temperature": 0.4953497052192688 }, { "layer": 8, "head": 1, "slices": 2, "native_nodes": 33555, "mass": [ 0.3917474589252717, 0.6082525409639299 ], "mean_node_assignment_entropy": 0.5265011525873127, "normalized_node_entropy": 0.7595806018600394, "effective_mass_slots": 1.9533060206615307, "node_slot_mutual_information_nats": 0.14302217963925545, "normalized_node_slot_mutual_information": 0.20633738930268433, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.08803822077976209, "token_RMS": 0.26922324466628955, "mean_token_offdiagonal_cosine": 0.7881547311358783, "temperature": 0.4847285747528076 }, { "layer": 8, "head": 2, "slices": 2, "native_nodes": 33555, "mass": [ 0.3392884196609485, 0.6607115804704997 ], "mean_node_assignment_entropy": 0.4149279088471957, "normalized_node_entropy": 0.5986144364202771, "effective_mass_slots": 1.8975476881650228, "node_slot_mutual_information_nats": 0.2256344532274251, "normalized_node_slot_mutual_information": 0.32552170672489894, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.14371872468997818, "token_RMS": 0.2763505556320342, "mean_token_offdiagonal_cosine": 0.5073632240632654, "temperature": 0.47846946120262146 }, { "layer": 8, "head": 3, "slices": 2, "native_nodes": 33555, "mass": [ 0.4883754614337051, 0.5116245387803423 ], "mean_node_assignment_entropy": 0.631179198894123, "normalized_node_entropy": 0.91059910015682, "effective_mass_slots": 1.9994595046141566, "node_slot_mutual_information_nats": 0.06169769744941245, "normalized_node_slot_mutual_information": 0.089010962144535, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.06286851114211231, "token_RMS": 0.23994218835202483, "mean_token_offdiagonal_cosine": 0.8661534176069832, "temperature": 0.4961528778076172 }, { "layer": 8, "head": 4, "slices": 2, "native_nodes": 33555, "mass": [ 0.4131453881601146, 0.5868546118609793 ], "mean_node_assignment_entropy": 0.49320289183052646, "normalized_node_entropy": 0.7115413661959964, "effective_mass_slots": 1.969900286369221, "node_slot_mutual_information_nats": 0.18478003358210193, "normalized_node_slot_mutual_information": 0.2665812381041946, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.09898391776449551, "token_RMS": 0.28268061556494506, "mean_token_offdiagonal_cosine": 0.7595057998681202, "temperature": 0.48845916986465454 }, { "layer": 8, "head": 5, "slices": 2, "native_nodes": 33555, "mass": [ 0.4421336887348535, 0.5578663114889637 ], "mean_node_assignment_entropy": 0.6229330227692652, "normalized_node_entropy": 0.8987023827551907, "effective_mass_slots": 1.986620849490856, "node_slot_mutual_information_nats": 0.06350210703712622, "normalized_node_slot_mutual_information": 0.09161417490846215, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.06261593549708945, "token_RMS": 0.25402915740990756, "mean_token_offdiagonal_cosine": 0.8786554828356593, "temperature": 0.4959590435028076 }, { "layer": 8, "head": 6, "slices": 2, "native_nodes": 33555, "mass": [ 0.5128555392449845, 0.48714446082073964 ], "mean_node_assignment_entropy": 0.6910646315841774, "normalized_node_entropy": 0.9969955169202513, "effective_mass_slots": 1.999338976815404, "node_slot_mutual_information_nats": 0.0017519827524757892, "normalized_node_slot_mutual_information": 0.0025275768287198173, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.009791847866166518, "token_RMS": 0.19527178732201875, "mean_token_offdiagonal_cosine": 0.9955728873369736, "temperature": 0.5023624897003174 }, { "layer": 8, "head": 7, "slices": 2, "native_nodes": 33555, "mass": [ 0.4275753198739687, 0.5724246802290582 ], "mean_node_assignment_entropy": 0.5617765233802108, "normalized_node_entropy": 0.8104722043684729, "effective_mass_slots": 1.9790551143699944, "node_slot_mutual_information_nats": 0.12084299245828356, "normalized_node_slot_mutual_information": 0.1743395859457481, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.10108506478973082, "token_RMS": 0.19497600440434543, "mean_token_offdiagonal_cosine": 0.46341089117485923, "temperature": 0.4869719445705414 }, { "layer": 8, "head": 8, "slices": 2, "native_nodes": 33555, "mass": [ 0.4351426251531269, 0.5648573749294723 ], "mean_node_assignment_entropy": 0.5431439524894311, "normalized_node_entropy": 0.7835910867453331, "effective_mass_slots": 1.9831975562759137, "node_slot_mutual_information_nats": 0.14156651703602408, "normalized_node_slot_mutual_information": 0.2042373120837949, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.10051929233703012, "token_RMS": 0.2634412238061746, "mean_token_offdiagonal_cosine": 0.7089419205966974, "temperature": 0.48995932936668396 }, { "layer": 9, "head": 1, "slices": 2, "native_nodes": 33555, "mass": [ 0.7078052789963157, 0.29219472085419534 ], "mean_node_assignment_entropy": 0.38458851492402063, "normalized_node_entropy": 0.5548439432637357, "effective_mass_slots": 1.8296141930601486, "node_slot_mutual_information_nats": 0.2195166062385187, "normalized_node_slot_mutual_information": 0.31669551921308625, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.1364333297227635, "token_RMS": 0.25906355222290167, "mean_token_offdiagonal_cosine": 0.4453526755790203, "temperature": 0.4778680205345154 }, { "layer": 9, "head": 2, "slices": 2, "native_nodes": 33555, "mass": [ 0.3032155919705719, 0.6967844080958738 ], "mean_node_assignment_entropy": 0.3925039763906173, "normalized_node_entropy": 0.5662635402679423, "effective_mass_slots": 1.8470029474049574, "node_slot_mutual_information_nats": 0.22106032060483194, "normalized_node_slot_mutual_information": 0.3189226282739154, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.12176532218518558, "token_RMS": 0.29413052300838083, "mean_token_offdiagonal_cosine": 0.6837354514548791, "temperature": 0.4808181822299957 }, { "layer": 9, "head": 3, "slices": 2, "native_nodes": 33555, "mass": [ 0.6142049566162199, 0.38579504345372295 ], "mean_node_assignment_entropy": 0.45889402782240957, "normalized_node_entropy": 0.6620441382329523, "effective_mass_slots": 1.9480520950338391, "node_slot_mutual_information_nats": 0.20793591987086363, "normalized_node_slot_mutual_information": 0.2999881204203798, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.1262848980417765, "token_RMS": 0.2609945179290185, "mean_token_offdiagonal_cosine": 0.5327983484408905, "temperature": 0.4873066842556 }, { "layer": 9, "head": 4, "slices": 2, "native_nodes": 33555, "mass": [ 0.6051532627968332, 0.39484673710102797 ], "mean_node_assignment_entropy": 0.5434442541265433, "normalized_node_entropy": 0.7840243304279656, "effective_mass_slots": 1.9559319744565706, "node_slot_mutual_information_nats": 0.12742253899612876, "normalized_node_slot_mutual_information": 0.18383186510719554, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.07692255500258735, "token_RMS": 0.22989498439690723, "mean_token_offdiagonal_cosine": 0.7786446684183475, "temperature": 0.49547088146209717 }, { "layer": 9, "head": 5, "slices": 2, "native_nodes": 33555, "mass": [ 0.6740170904970475, 0.325982909173888 ], "mean_node_assignment_entropy": 0.4094020235457551, "normalized_node_entropy": 0.5906422690993676, "effective_mass_slots": 1.880047840908561, "node_slot_mutual_information_nats": 0.22189520026411363, "normalized_node_slot_mutual_information": 0.32012710501810016, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.1335439037097161, "token_RMS": 0.2676296061390079, "mean_token_offdiagonal_cosine": 0.5215311480900793, "temperature": 0.4836619794368744 }, { "layer": 9, "head": 6, "slices": 2, "native_nodes": 33555, "mass": [ 0.2762443545955408, 0.7237556454379873 ], "mean_node_assignment_entropy": 0.4183674399788328, "normalized_node_entropy": 0.6035766309268732, "effective_mass_slots": 1.8028543804430515, "node_slot_mutual_information_nats": 0.1710037358353826, "normalized_node_slot_mutual_information": 0.2467062416631928, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.10182006331043592, "token_RMS": 0.23491482931534527, "mean_token_offdiagonal_cosine": 0.6247702363856047, "temperature": 0.47997188568115234 }, { "layer": 9, "head": 7, "slices": 2, "native_nodes": 33555, "mass": [ 0.5832546737752805, 0.41674532641900525 ], "mean_node_assignment_entropy": 0.4137096452059014, "normalized_node_entropy": 0.5968568535064865, "effective_mass_slots": 1.9723381556356343, "node_slot_mutual_information_nats": 0.2655100747908378, "normalized_node_slot_mutual_information": 0.38305006820679954, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.1464918297662878, "token_RMS": 0.2584948352861449, "mean_token_offdiagonal_cosine": 0.36977659535172785, "temperature": 0.4835028052330017 }, { "layer": 9, "head": 8, "slices": 2, "native_nodes": 33555, "mass": [ 0.5960960802289772, 0.403903919761253 ], "mean_node_assignment_entropy": 0.46813004235903727, "normalized_node_entropy": 0.6753688906025235, "effective_mass_slots": 1.963174592627239, "node_slot_mutual_information_nats": 0.2064328107159356, "normalized_node_slot_mutual_information": 0.2978195922966504, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.10853833608059993, "token_RMS": 0.2882980067507666, "mean_token_offdiagonal_cosine": 0.7193477686696804, "temperature": 0.48967835307121277 }, { "layer": 10, "head": 1, "slices": 2, "native_nodes": 33555, "mass": [ 0.49149941174981493, 0.5085005882324218 ], "mean_node_assignment_entropy": 0.662522625204052, "normalized_node_entropy": 0.9558181058586233, "effective_mass_slots": 1.9997109669705084, "node_slot_mutual_information_nats": 0.030480028397629755, "normalized_node_slot_mutual_information": 0.043973385815415225, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.04691950579007945, "token_RMS": 0.22300772340051264, "mean_token_offdiagonal_cosine": 0.9115469279736157, "temperature": 0.4936632513999939 }, { "layer": 10, "head": 2, "slices": 2, "native_nodes": 33555, "mass": [ 0.5100188830031703, 0.4899811171256133 ], "mean_node_assignment_entropy": 0.6815651930842362, "normalized_node_entropy": 0.9832907241051565, "effective_mass_slots": 1.999598501292378, "node_slot_mutual_information_nats": 0.01138121796904945, "normalized_node_slot_mutual_information": 0.016419626723224005, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.03229711454595413, "token_RMS": 0.23146106587877308, "mean_token_offdiagonal_cosine": 0.9610599612946255, "temperature": 0.49642258882522583 }, { "layer": 10, "head": 3, "slices": 2, "native_nodes": 33555, "mass": [ 0.5642958202645918, 0.43570417979313886 ], "mean_node_assignment_entropy": 0.6814649768914202, "normalized_node_entropy": 0.983146142700764, "effective_mass_slots": 1.9834868621785209, "node_slot_mutual_information_nats": 0.0033913605038229377, "normalized_node_slot_mutual_information": 0.0048926989807320485, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.014329396025129466, "token_RMS": 0.24091608469002673, "mean_token_offdiagonal_cosine": 0.9929861580333048, "temperature": 0.49738094210624695 }, { "layer": 10, "head": 4, "slices": 2, "native_nodes": 33555, "mass": [ 0.5614765803560985, 0.4385234197638035 ], "mean_node_assignment_entropy": 0.624822028941969, "normalized_node_entropy": 0.9014276425927591, "effective_mass_slots": 1.9849014781216272, "node_slot_mutual_information_nats": 0.06074725077732013, "normalized_node_slot_mutual_information": 0.08763975744407798, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.05475745909020932, "token_RMS": 0.24885145173265485, "mean_token_offdiagonal_cosine": 0.9111381520256104, "temperature": 0.4812485873699188 }, { "layer": 10, "head": 5, "slices": 2, "native_nodes": 33555, "mass": [ 0.506860465745673, 0.4931395339203776 ], "mean_node_assignment_entropy": 0.6711705368817491, "normalized_node_entropy": 0.9682944051500826, "effective_mass_slots": 1.9998117391881536, "node_slot_mutual_information_nats": 0.021882508841728265, "normalized_node_slot_mutual_information": 0.03156978698817026, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.03972021528747488, "token_RMS": 0.23049236216088484, "mean_token_offdiagonal_cosine": 0.9423668285683661, "temperature": 0.48747801780700684 }, { "layer": 10, "head": 6, "slices": 2, "native_nodes": 33555, "mass": [ 0.5358969931763413, 0.46410300676059907 ], "mean_node_assignment_entropy": 0.6873411322543694, "normalized_node_entropy": 0.9916236429023839, "effective_mass_slots": 1.994847834087904, "node_slot_mutual_information_nats": 0.0032266415383268576, "normalized_node_slot_mutual_information": 0.004655059746070494, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.0168446566757172, "token_RMS": 0.24260558710323096, "mean_token_offdiagonal_cosine": 0.9906743938055653, "temperature": 0.49887579679489136 }, { "layer": 10, "head": 7, "slices": 2, "native_nodes": 33555, "mass": [ 0.40619310049306545, 0.593806900063813 ], "mean_node_assignment_entropy": 0.6292505222674541, "normalized_node_entropy": 0.9078166079520463, "effective_mass_slots": 1.9649032013350982, "node_slot_mutual_information_nats": 0.04619246043331271, "normalized_node_slot_mutual_information": 0.06664163359359991, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.056476364121572094, "token_RMS": 0.24928128257242715, "mean_token_offdiagonal_cosine": 0.8975803228244666, "temperature": 0.4915263056755066 }, { "layer": 10, "head": 8, "slices": 2, "native_nodes": 33555, "mass": [ 0.5148620644041686, 0.48513793579655634 ], "mean_node_assignment_entropy": 0.6880714887135596, "normalized_node_entropy": 0.9926773245441388, "effective_mass_slots": 1.99911654108951, "node_slot_mutual_information_nats": 0.004633864799944543, "normalized_node_slot_mutual_information": 0.00668525376702992, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.017884067865251527, "token_RMS": 0.21951054218545935, "mean_token_offdiagonal_cosine": 0.9867261009967108, "temperature": 0.49870622158050537 }, { "layer": 11, "head": 1, "slices": 2, "native_nodes": 33555, "mass": [ 0.5127008504784777, 0.4872991493265704 ], "mean_node_assignment_entropy": 0.6675769132637207, "normalized_node_entropy": 0.9631099021775316, "effective_mass_slots": 1.9993547883865954, "node_slot_mutual_information_nats": 0.025247609441074625, "normalized_node_slot_mutual_information": 0.03642460093493974, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.044622657944221156, "token_RMS": 0.22333687706306948, "mean_token_offdiagonal_cosine": 0.9203138067607559, "temperature": 0.4844403564929962 }, { "layer": 11, "head": 2, "slices": 2, "native_nodes": 33555, "mass": [ 0.7324882321618137, 0.26751176772250307 ], "mean_node_assignment_entropy": 0.4278256043397414, "normalized_node_entropy": 0.6172218777462688, "effective_mass_slots": 1.7874110849307503, "node_slot_mutual_information_nats": 0.1529426473129915, "normalized_node_slot_mutual_information": 0.2206495988188826, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.11449598759947885, "token_RMS": 0.2750222061238643, "mean_token_offdiagonal_cosine": 0.6543553963005658, "temperature": 0.49226686358451843 }, { "layer": 11, "head": 3, "slices": 2, "native_nodes": 33555, "mass": [ 0.3591595536609411, 0.6408404464911569 ], "mean_node_assignment_entropy": 0.4396938288837866, "normalized_node_entropy": 0.6343441064401194, "effective_mass_slots": 1.9211675470004008, "node_slot_mutual_information_nats": 0.21323926973564455, "normalized_node_slot_mutual_information": 0.3076392369703984, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.12853582635097288, "token_RMS": 0.21443134484779128, "mean_token_offdiagonal_cosine": 0.28691210398941647, "temperature": 0.49134117364883423 }, { "layer": 11, "head": 4, "slices": 2, "native_nodes": 33555, "mass": [ 0.7161369222555337, 0.28386307820331375 ], "mean_node_assignment_entropy": 0.4100508094215161, "normalized_node_entropy": 0.5915782692649267, "effective_mass_slots": 1.8158704453992425, "node_slot_mutual_information_nats": 0.1865141275735373, "normalized_node_slot_mutual_information": 0.26908300690607373, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.15929024562846933, "token_RMS": 0.25475565070971257, "mean_token_offdiagonal_cosine": 0.22008099411615153, "temperature": 0.48838570713996887 }, { "layer": 11, "head": 5, "slices": 2, "native_nodes": 33555, "mass": [ 0.5035572447598806, 0.496442755613148 ], "mean_node_assignment_entropy": 0.6290010627536549, "normalized_node_entropy": 0.9074567139485855, "effective_mass_slots": 1.9999493840287708, "node_slot_mutual_information_nats": 0.06412080950042331, "normalized_node_slot_mutual_information": 0.09250677388404666, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.05630677848474879, "token_RMS": 0.2370967911217805, "mean_token_offdiagonal_cosine": 0.8887427446950656, "temperature": 0.477323055267334 }, { "layer": 11, "head": 6, "slices": 2, "native_nodes": 33555, "mass": [ 0.44817372648784537, 0.5518262735583391 ], "mean_node_assignment_entropy": 0.6719862191721602, "normalized_node_entropy": 0.9694711859453996, "effective_mass_slots": 1.9892657374991152, "node_slot_mutual_information_nats": 0.015779375344701085, "normalized_node_slot_mutual_information": 0.022764826558125836, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.03695956345011119, "token_RMS": 0.19636882107556403, "mean_token_offdiagonal_cosine": 0.9322247553356584, "temperature": 0.46780136227607727 }, { "layer": 11, "head": 7, "slices": 2, "native_nodes": 33555, "mass": [ 0.9234832567849612, 0.07651674323759765 ], "mean_node_assignment_entropy": 0.17530348302784926, "normalized_node_entropy": 0.2529094656148407, "effective_mass_slots": 1.3101983097355292, "node_slot_mutual_information_nats": 0.09487502420569455, "normalized_node_slot_mutual_information": 0.1368757269257759, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.15584572156591198, "token_RMS": 0.2872121721888198, "mean_token_offdiagonal_cosine": 0.45773336517639607, "temperature": 0.46444171667099 }, { "layer": 11, "head": 8, "slices": 2, "native_nodes": 33555, "mass": [ 0.5233258517394921, 0.4766741482494059 ], "mean_node_assignment_entropy": 0.5942104549527607, "normalized_node_entropy": 0.8572644766047227, "effective_mass_slots": 1.9978240130299687, "node_slot_mutual_information_nats": 0.09784813982760987, "normalized_node_slot_mutual_information": 0.14116502608950263, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.07971564285103, "token_RMS": 0.2365461892255117, "mean_token_offdiagonal_cosine": 0.7752908870150608, "temperature": 0.467622846364975 }, { "layer": 12, "head": 1, "slices": 2, "native_nodes": 33555, "mass": [ 0.9266710399266878, 0.07332895973388112 ], "mean_node_assignment_entropy": 0.21463395135154842, "normalized_node_entropy": 0.30965133722128196, "effective_mass_slots": 1.2997423279909741, "node_slot_mutual_information_nats": 0.04753208423218333, "normalized_node_slot_mutual_information": 0.06857430220488739, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.10370483561480968, "token_RMS": 0.20054045626807232, "mean_token_offdiagonal_cosine": 0.4964293365990137, "temperature": 0.4704362154006958 }, { "layer": 12, "head": 2, "slices": 2, "native_nodes": 33555, "mass": [ 0.888928348636009, 0.11107165119013299 ], "mean_node_assignment_entropy": 0.2375664496459821, "normalized_node_entropy": 0.342735938785856, "effective_mass_slots": 1.4172948698413885, "node_slot_mutual_information_nats": 0.11118358386796343, "normalized_node_slot_mutual_information": 0.160404005074573, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.14066392256523647, "token_RMS": 0.2948151254881917, "mean_token_offdiagonal_cosine": 0.6296289323976418, "temperature": 0.4668295085430145 }, { "layer": 12, "head": 3, "slices": 2, "native_nodes": 33555, "mass": [ 0.8957331067589246, 0.10426689329769587 ], "mean_node_assignment_entropy": 0.23537058498922983, "normalized_node_entropy": 0.3395679757350962, "effective_mass_slots": 1.3970437563025984, "node_slot_mutual_information_nats": 0.09898781641469559, "normalized_node_slot_mutual_information": 0.14280923184990846, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.11662983590287661, "token_RMS": 0.2891002178749927, "mean_token_offdiagonal_cosine": 0.7162513528572804, "temperature": 0.4603922963142395 }, { "layer": 12, "head": 4, "slices": 2, "native_nodes": 33555, "mass": [ 0.7829943676043545, 0.21700563230991698 ], "mean_node_assignment_entropy": 0.2593946500431306, "normalized_node_entropy": 0.3742273752503527, "effective_mass_slots": 1.6872363252522515, "node_slot_mutual_information_nats": 0.2636972298104788, "normalized_node_slot_mutual_information": 0.38043468574373507, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.14534992053689175, "token_RMS": 0.2447901865374678, "mean_token_offdiagonal_cosine": 0.3627913375034625, "temperature": 0.4504719376564026 }, { "layer": 12, "head": 5, "slices": 2, "native_nodes": 33555, "mass": [ 0.7789297496130313, 0.22107025061503238 ], "mean_node_assignment_entropy": 0.3439192935728929, "normalized_node_entropy": 0.4961706593036482, "effective_mass_slots": 1.6959773932535709, "node_slot_mutual_information_nats": 0.18433991425843504, "normalized_node_slot_mutual_information": 0.265946280138541, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.13230814153795592, "token_RMS": 0.19667677256895744, "mean_token_offdiagonal_cosine": 0.09792635275821364, "temperature": 0.46245044469833374 }, { "layer": 12, "head": 6, "slices": 2, "native_nodes": 33555, "mass": [ 0.4831575954569699, 0.5168424045172734 ], "mean_node_assignment_entropy": 0.6669981059532297, "normalized_node_entropy": 0.962274859741056, "effective_mass_slots": 1.9988654409067794, "node_slot_mutual_information_nats": 0.02558163409618608, "normalized_node_slot_mutual_information": 0.036906496648403675, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.045442082458358464, "token_RMS": 0.17802800286199638, "mean_token_offdiagonal_cosine": 0.8718905447186956, "temperature": 0.4662950336933136 }, { "layer": 12, "head": 7, "slices": 2, "native_nodes": 33555, "mass": [ 0.8038725992514539, 0.1961274005265607 ], "mean_node_assignment_entropy": 0.3611464382306531, "normalized_node_entropy": 0.5210241754700756, "effective_mass_slots": 1.6404765287131133, "node_slot_mutual_information_nats": 0.1338403276877293, "normalized_node_slot_mutual_information": 0.19309077702604088, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.12380505857726033, "token_RMS": 0.3110698177053975, "mean_token_offdiagonal_cosine": 0.70428703513926, "temperature": 0.47574982047080994 }, { "layer": 12, "head": 8, "slices": 2, "native_nodes": 33555, "mass": [ 0.6778103766218772, 0.322189623307958 ], "mean_node_assignment_entropy": 0.5583958584202173, "normalized_node_entropy": 0.8055949357957833, "effective_mass_slots": 1.8748130175974178, "node_slot_mutual_information_nats": 0.07011307208134943, "normalized_node_slot_mutual_information": 0.10115178139325325, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.07543300946726664, "token_RMS": 0.20028804175728646, "mean_token_offdiagonal_cosine": 0.716543870336831, "temperature": 0.4728742241859436 }, { "layer": 13, "head": 1, "slices": 2, "native_nodes": 33555, "mass": [ 0.919960479280878, 0.08003951835689531 ], "mean_node_assignment_entropy": 0.17091319292998539, "normalized_node_entropy": 0.24657561586258855, "effective_mass_slots": 1.321630076103367, "node_slot_mutual_information_nats": 0.10795268793778293, "normalized_node_slot_mutual_information": 0.15574280753847325, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.1835337443170427, "token_RMS": 0.28499069651633374, "mean_token_offdiagonal_cosine": 0.24713088657093885, "temperature": 0.4479334354400635 }, { "layer": 13, "head": 2, "slices": 2, "native_nodes": 33555, "mass": [ 0.18588661459994696, 0.8141133854009412 ], "mean_node_assignment_entropy": 0.3300420946849819, "normalized_node_entropy": 0.4761500932866291, "effective_mass_slots": 1.6164028803934816, "node_slot_mutual_information_nats": 0.1501611415162365, "normalized_node_slot_mutual_information": 0.21663673419970023, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.10378706188256397, "token_RMS": 0.2062273722884165, "mean_token_offdiagonal_cosine": 0.503897038320315, "temperature": 0.4719248116016388 }, { "layer": 13, "head": 3, "slices": 2, "native_nodes": 33555, "mass": [ 0.8168503134776611, 0.18314968308833646 ], "mean_node_assignment_entropy": 0.2211854923425939, "normalized_node_entropy": 0.31910321291924404, "effective_mass_slots": 1.6098420186698061, "node_slot_mutual_information_nats": 0.25495055678939016, "normalized_node_slot_mutual_information": 0.36781590395193325, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.18718179722622472, "token_RMS": 0.27979528115093494, "mean_token_offdiagonal_cosine": 0.11305711867973828, "temperature": 0.4634672999382019 }, { "layer": 13, "head": 4, "slices": 2, "native_nodes": 33555, "mass": [ 0.5723972426150385, 0.42760275730724717 ], "mean_node_assignment_entropy": 0.6524541568195571, "normalized_node_entropy": 0.9412923764509651, "effective_mass_slots": 1.9790709536673938, "node_slot_mutual_information_nats": 0.030173362451433983, "normalized_node_slot_mutual_information": 0.04353096037562906, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.05387187546094471, "token_RMS": 0.24735445672120873, "mean_token_offdiagonal_cosine": 0.9051363363241719, "temperature": 0.4715731739997864 }, { "layer": 13, "head": 5, "slices": 2, "native_nodes": 33555, "mass": [ 0.38999883126101825, 0.6100011650976322 ], "mean_node_assignment_entropy": 0.07749556059864576, "normalized_node_entropy": 0.1118024609665764, "effective_mass_slots": 1.9517912992344881, "node_slot_mutual_information_nats": 0.5912520053005172, "normalized_node_slot_mutual_information": 0.8529963359627114, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.20998043016860235, "token_RMS": 0.29614683892747323, "mean_token_offdiagonal_cosine": -0.005483366422687598, "temperature": 0.4435587525367737 }, { "layer": 13, "head": 6, "slices": 2, "native_nodes": 33555, "mass": [ 0.6436144891456314, 0.35638551106042254 ], "mean_node_assignment_entropy": 0.6384021330595877, "normalized_node_entropy": 0.9210195914580034, "effective_mass_slots": 1.9180521265018327, "node_slot_mutual_information_nats": 0.012908020561987588, "normalized_node_slot_mutual_information": 0.018622337252472265, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.02021470587545041, "token_RMS": 0.2372581310375359, "mean_token_offdiagonal_cosine": 0.9869446651929206, "temperature": 0.47083744406700134 }, { "layer": 13, "head": 7, "slices": 2, "native_nodes": 33555, "mass": [ 0.47497061057233575, 0.5250293894631908 ], "mean_node_assignment_entropy": 0.6609542752839884, "normalized_node_entropy": 0.9535554552065689, "effective_mass_slots": 1.9974946415272206, "node_slot_mutual_information_nats": 0.030939440781079353, "normalized_node_slot_mutual_information": 0.04463617778274094, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.05112748041055814, "token_RMS": 0.18631850283479995, "mean_token_offdiagonal_cosine": 0.8494896959333409, "temperature": 0.4870256185531616 }, { "layer": 13, "head": 8, "slices": 2, "native_nodes": 33555, "mass": [ 0.803545642888967, 0.19645435747139317 ], "mean_node_assignment_entropy": 0.23763145487097656, "normalized_node_entropy": 0.3428297215015874, "effective_mass_slots": 1.641232784000196, "node_slot_mutual_information_nats": 0.2578162021431162, "normalized_node_slot_mutual_information": 0.3719501562927003, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.1587654734716992, "token_RMS": 0.26400099362483487, "mean_token_offdiagonal_cosine": 0.2773871874831301, "temperature": 0.46389275789260864 }, { "layer": 14, "head": 1, "slices": 2, "native_nodes": 33555, "mass": [ 0.3824697532005094, 0.6175302469948866 ], "mean_node_assignment_entropy": 0.6613542158740104, "normalized_node_entropy": 0.9541324475124439, "effective_mass_slots": 1.944996651650554, "node_slot_mutual_information_nats": 0.0039060396784926787, "normalized_node_slot_mutual_information": 0.005635224073676909, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.00419918591223028, "token_RMS": 0.2306163664839568, "mean_token_offdiagonal_cosine": 0.9993656361807104, "temperature": 0.4652848243713379 }, { "layer": 14, "head": 2, "slices": 2, "native_nodes": 33555, "mass": [ 0.45697102150038127, 0.5430289787554098 ], "mean_node_assignment_entropy": 0.6223531001681086, "normalized_node_entropy": 0.8978657312944026, "effective_mass_slots": 1.992598588351772, "node_slot_mutual_information_nats": 0.0670865100148974, "normalized_node_slot_mutual_information": 0.09678537530904027, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.09018360600545205, "token_RMS": 0.22082587796940734, "mean_token_offdiagonal_cosine": 0.6738909594129775, "temperature": 0.462272584438324 }, { "layer": 14, "head": 3, "slices": 2, "native_nodes": 33555, "mass": [ 0.18697530381528998, 0.8130246955722087 ], "mean_node_assignment_entropy": 0.20598052237710368, "normalized_node_entropy": 0.2971670781531656, "effective_mass_slots": 1.6189977419559354, "node_slot_mutual_information_nats": 0.27582675760212555, "normalized_node_slot_mutual_information": 0.39793389533706874, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.19413717382844983, "token_RMS": 0.27060512077045323, "mean_token_offdiagonal_cosine": -0.04097480672397699, "temperature": 0.4374889135360718 }, { "layer": 14, "head": 4, "slices": 2, "native_nodes": 33555, "mass": [ 0.41880439506892814, 0.5811956054171192 ], "mean_node_assignment_entropy": 0.5222271562998898, "normalized_node_entropy": 0.7534145286113967, "effective_mass_slots": 1.973686582649937, "node_slot_mutual_information_nats": 0.15767599938528065, "normalized_node_slot_mutual_information": 0.22747838238035564, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.12159189603483506, "token_RMS": 0.2513118328905744, "mean_token_offdiagonal_cosine": 0.5321348609667319, "temperature": 0.46139034628868103 }, { "layer": 14, "head": 5, "slices": 2, "native_nodes": 33555, "mass": [ 0.6016748447795505, 0.39832515522289197 ], "mean_node_assignment_entropy": 0.5642063851415223, "normalized_node_entropy": 0.8139777538815628, "effective_mass_slots": 1.9587895955529002, "node_slot_mutual_information_nats": 0.10812034404699211, "normalized_node_slot_mutual_information": 0.15598468417580408, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.10309303547528544, "token_RMS": 0.25204242578998015, "mean_token_offdiagonal_cosine": 0.6866507160991224, "temperature": 0.44987380504608154 }, { "layer": 14, "head": 6, "slices": 2, "native_nodes": 33555, "mass": [ 0.4336118274155325, 0.5663881723570976 ], "mean_node_assignment_entropy": 0.6580432413861479, "normalized_node_entropy": 0.9493557210382947, "effective_mass_slots": 1.9823962044032355, "node_slot_mutual_information_nats": 0.026263075857481777, "normalized_node_slot_mutual_information": 0.03788960929807962, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.02506238068690977, "token_RMS": 0.2278496913465894, "mean_token_offdiagonal_cosine": 0.9938760261550089, "temperature": 0.4568415880203247 }, { "layer": 14, "head": 7, "slices": 2, "native_nodes": 33555, "mass": [ 0.6076562140506239, 0.39234378367411976 ], "mean_node_assignment_entropy": 0.06997781960917461, "normalized_node_entropy": 0.10095665332237867, "effective_mass_slots": 1.9538171147290302, "node_slot_mutual_information_nats": 0.5998071343030708, "normalized_node_slot_mutual_information": 0.8653387781488606, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.2674636617711381, "token_RMS": 0.3308459297922284, "mean_token_offdiagonal_cosine": -0.3079296646637574, "temperature": 0.47246354818344116 }, { "layer": 14, "head": 8, "slices": 2, "native_nodes": 33555, "mass": [ 0.03724639408332164, 0.9627536048478473 ], "mean_node_assignment_entropy": 0.1073133295584667, "normalized_node_entropy": 0.15482040837528294, "effective_mass_slots": 1.1724458809136755, "node_slot_mutual_information_nats": 0.05177873371130168, "normalized_node_slot_mutual_information": 0.07470092234880513, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.1369348499566186, "token_RMS": 0.21098477021783044, "mean_token_offdiagonal_cosine": 0.1849115987839036, "temperature": 0.4523463249206543 }, { "layer": 15, "head": 1, "slices": 2, "native_nodes": 33555, "mass": [ 0.5312088092825413, 0.4687911909830195 ], "mean_node_assignment_entropy": 0.6854618880211547, "normalized_node_entropy": 0.9889124665665059, "effective_mass_slots": 1.9961053041580246, "node_slot_mutual_information_nats": 0.00573604607068523, "normalized_node_slot_mutual_information": 0.008275365220488206, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.019018388844555357, "token_RMS": 0.1624888615385138, "mean_token_offdiagonal_cosine": 0.9742515846196356, "temperature": 0.4309997260570526 }, { "layer": 15, "head": 2, "slices": 2, "native_nodes": 33555, "mass": [ 0.5550495128192129, 0.44495048678377813 ], "mean_node_assignment_entropy": 0.6775895277767761, "normalized_node_entropy": 0.9775550514818494, "effective_mass_slots": 1.9878904047610173, "node_slot_mutual_information_nats": 0.009484450548150392, "normalized_node_slot_mutual_information": 0.01368316977137318, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.017077304863107663, "token_RMS": 0.17255675213149438, "mean_token_offdiagonal_cosine": 0.9823813387459581, "temperature": 0.4435434937477112 }, { "layer": 15, "head": 3, "slices": 2, "native_nodes": 33555, "mass": [ 0.4437500371687495, 0.5562499627673028 ], "mean_node_assignment_entropy": 0.6593461194453554, "normalized_node_entropy": 0.9512353767531964, "effective_mass_slots": 1.9873570641635812, "node_slot_mutual_information_nats": 0.02745952811284247, "normalized_node_slot_mutual_information": 0.039615725033548904, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.05174004218580709, "token_RMS": 0.1719364667273214, "mean_token_offdiagonal_cosine": 0.820030450005162, "temperature": 0.4615800976753235 }, { "layer": 15, "head": 4, "slices": 2, "native_nodes": 33555, "mass": [ 0.2427772148747806, 0.7572227849550529 ], "mean_node_assignment_entropy": 0.3153661297157314, "normalized_node_entropy": 0.45497715140523126, "effective_mass_slots": 1.7406525635542247, "node_slot_mutual_information_nats": 0.23889394972750333, "normalized_node_slot_mutual_information": 0.3446511165702464, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.1822514357011611, "token_RMS": 0.24549410081031153, "mean_token_offdiagonal_cosine": -0.10260867922265689, "temperature": 0.47071462869644165 }, { "layer": 15, "head": 5, "slices": 2, "native_nodes": 33555, "mass": [ 0.11436368224250332, 0.8856363166588249 ], "mean_node_assignment_entropy": 0.20321762284774766, "normalized_node_entropy": 0.29318105670368927, "effective_mass_slots": 1.4269545980377931, "node_slot_mutual_information_nats": 0.15232489876924668, "normalized_node_slot_mutual_information": 0.21975837605830556, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.20543810412856595, "token_RMS": 0.2979972724673081, "mean_token_offdiagonal_cosine": 0.06331323648009132, "temperature": 0.4536850154399872 }, { "layer": 15, "head": 6, "slices": 2, "native_nodes": 33555, "mass": [ 0.07959192661206027, 0.9204080726542251 ], "mean_node_assignment_entropy": 0.18325880049039295, "normalized_node_entropy": 0.26438656266674987, "effective_mass_slots": 1.3201846095974137, "node_slot_mutual_information_nats": 0.09451278208459846, "normalized_node_slot_mutual_information": 0.13635312201406946, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.20878901216383366, "token_RMS": 0.3248784028232303, "mean_token_offdiagonal_cosine": 0.2618331269685727, "temperature": 0.4483626186847687 }, { "layer": 15, "head": 7, "slices": 2, "native_nodes": 33555, "mass": [ 0.6293411747795727, 0.3706588254833236 ], "mean_node_assignment_entropy": 0.6337080265686755, "normalized_node_entropy": 0.9142474273021597, "effective_mass_slots": 1.9334488705515538, "node_slot_mutual_information_nats": 0.025597361160107135, "normalized_node_slot_mutual_information": 0.03692918600553033, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.01984440122772632, "token_RMS": 0.2024062057624229, "mean_token_offdiagonal_cosine": 0.9964787267964564, "temperature": 0.4674021303653717 }, { "layer": 15, "head": 8, "slices": 2, "native_nodes": 33555, "mass": [ 0.2949467233906043, 0.705053276404452 ], "mean_node_assignment_entropy": 0.4744211842225798, "normalized_node_entropy": 0.6844450897705852, "effective_mass_slots": 1.8340409015467982, "node_slot_mutual_information_nats": 0.13210049118787398, "normalized_node_slot_mutual_information": 0.190580723535742, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.12070232850925103, "token_RMS": 0.24283992303619065, "mean_token_offdiagonal_cosine": 0.5203567974299271, "temperature": 0.45695915818214417 }, { "layer": 16, "head": 1, "slices": 2, "native_nodes": 33555, "mass": [ 0.7602081061581921, 0.2397918939737001 ], "mean_node_assignment_entropy": 0.46889705608143784, "normalized_node_entropy": 0.6764754574961246, "effective_mass_slots": 1.7347094401116212, "node_slot_mutual_information_nats": 0.08194287361647357, "normalized_node_slot_mutual_information": 0.1182185774026775, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.1231957236640604, "token_RMS": 0.19791840336065686, "mean_token_offdiagonal_cosine": 0.22947239494269422, "temperature": 0.45334184169769287 }, { "layer": 16, "head": 2, "slices": 2, "native_nodes": 33555, "mass": [ 0.589811854264898, 0.41018814694054956 ], "mean_node_assignment_entropy": 0.0648293937501511, "normalized_node_entropy": 0.09352904486718094, "effective_mass_slots": 1.9678212218983342, "node_slot_mutual_information_nats": 0.6120975582221437, "normalized_node_slot_mutual_information": 0.8830701117873303, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.2970859131030645, "token_RMS": 0.32750767476076637, "mean_token_offdiagonal_cosine": -0.6517027508105785, "temperature": 0.4401816725730896 }, { "layer": 16, "head": 3, "slices": 2, "native_nodes": 33555, "mass": [ 0.4895868268467385, 0.5104131729649709 ], "mean_node_assignment_entropy": 0.6028805354684523, "normalized_node_entropy": 0.869772758768819, "effective_mass_slots": 1.9995662790980646, "node_slot_mutual_information_nats": 0.09004976112289764, "normalized_node_slot_mutual_information": 0.1299143438052402, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.11354117307422594, "token_RMS": 0.1958360451906228, "mean_token_offdiagonal_cosine": 0.32780449234606945, "temperature": 0.4834662079811096 }, { "layer": 16, "head": 4, "slices": 2, "native_nodes": 33555, "mass": [ 0.6038263658131844, 0.39617363404737393 ], "mean_node_assignment_entropy": 0.6560856736682517, "normalized_node_entropy": 0.9465315477994815, "effective_mass_slots": 1.9570332414027838, "node_slot_mutual_information_nats": 0.015344000499632027, "normalized_node_slot_mutual_information": 0.022136713428216902, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.04390383181365209, "token_RMS": 0.15348262031073437, "mean_token_offdiagonal_cosine": 0.8554124184693805, "temperature": 0.48540765047073364 }, { "layer": 16, "head": 5, "slices": 2, "native_nodes": 33555, "mass": [ 0.6043864082708242, 0.39561359154585507 ], "mean_node_assignment_entropy": 0.10498240090127922, "normalized_node_entropy": 0.15145758916089258, "effective_mass_slots": 1.9565701121318693, "node_slot_mutual_information_nats": 0.5662105966127693, "normalized_node_slot_mutual_information": 0.8168692198320235, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.26441160110454537, "token_RMS": 0.30214132192640536, "mean_token_offdiagonal_cosine": -0.548616193463079, "temperature": 0.44579437375068665 }, { "layer": 16, "head": 6, "slices": 2, "native_nodes": 33555, "mass": [ 0.16328129377633122, 0.8367187062002227 ], "mean_node_assignment_entropy": 0.2871311501807132, "normalized_node_entropy": 0.4142426864504592, "effective_mass_slots": 1.5606012705083034, "node_slot_mutual_information_nats": 0.1579400266347576, "normalized_node_slot_mutual_information": 0.22785929318383558, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.19880953305202043, "token_RMS": 0.3141780921908674, "mean_token_offdiagonal_cosine": 0.23650301352663303, "temperature": 0.4727310240268707 }, { "layer": 16, "head": 7, "slices": 2, "native_nodes": 33555, "mass": [ 0.5828488398116392, 0.4171511583832031 ], "mean_node_assignment_entropy": 0.03674364160106068, "normalized_node_entropy": 0.05300986952205166, "effective_mass_slots": 1.9726065736444556, "node_slot_mutual_information_nats": 0.6426121604083077, "normalized_node_slot_mutual_information": 0.9270933770360086, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.21051076201249763, "token_RMS": 0.2634958683040356, "mean_token_offdiagonal_cosine": -0.2789892069678855, "temperature": 0.45187437534332275 }, { "layer": 16, "head": 8, "slices": 2, "native_nodes": 33555, "mass": [ 0.497620936804993, 0.5023790633513238 ], "mean_node_assignment_entropy": 0.592515894920781, "normalized_node_entropy": 0.854819743250097, "effective_mass_slots": 1.999977360178554, "node_slot_mutual_information_nats": 0.10061996566437059, "normalized_node_slot_mutual_information": 0.14516392547840523, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "centered_token_RMS": 0.12503749699440525, "token_RMS": 0.19300239788370754, "mean_token_offdiagonal_cosine": 0.16062638221876924, "temperature": 0.46243053674697876 } ] }, { "schema": "ts3_capacity_slice_diagnostics_v1", "variant": "S004", "sample_id": "normal_rodv2_0150__gas_inertia_30deg", "native_nodes": 33555, "layers": 16, "slices": 4, "parameters": 7594692, "checkpoint_sha256": "bd179825f525bf029b1de8d8b7427c749ed1d003cc64167b7eeee3c236f8228a", "extraction_source_sha256": "26c5dec82b68ba4bc23d7c3d998b78c5bbed6aae912e6247ea64949db69bd206", "CPU_forward_seconds": 2.5409307339996303, "CPU_threads": 2, "CUDA_initialized": false, "hook_max_abs_prediction_difference_on_19_node_test": 0.0, "metrics": [ { "layer": 1, "head": 1, "slices": 4, "native_nodes": 33555, "mass": [ 0.26650896228438004, 0.2565022153895048, 0.24424643704387083, 0.2327423849580647 ], "mean_node_assignment_entropy": 1.3840772185087895, "normalized_node_entropy": 0.9984006696750104, "effective_mass_slots": 3.994827829665348, "node_slot_mutual_information_nats": 0.0009232633265388035, "normalized_node_slot_mutual_information": 0.0006659937113160897, "centered_token_effective_rank": 1.318361177515854, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.0045721703626135195, "token_RMS": 0.2550508159915972, "mean_token_offdiagonal_cosine": 0.9995763951938371, "temperature": 0.4986083507537842 }, { "layer": 1, "head": 2, "slices": 4, "native_nodes": 33555, "mass": [ 0.22819919737125008, 0.2628817104326064, 0.24961032274271724, 0.2593087694503177 ], "mean_node_assignment_entropy": 1.3845512898959085, "normalized_node_entropy": 0.9987426398946224, "effective_mass_slots": 3.9940952841728627, "node_slot_mutual_information_nats": 0.00026580164160372455, "normalized_node_slot_mutual_information": 0.0001917353551009195, "centered_token_effective_rank": 1.4455035535083922, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.001905039493173866, "token_RMS": 0.22039073276179846, "mean_token_offdiagonal_cosine": 0.9999032847487581, "temperature": 0.5020051598548889 }, { "layer": 1, "head": 3, "slices": 4, "native_nodes": 33555, "mass": [ 0.2421525122542581, 0.27288894676234926, 0.2233432742086897, 0.2616152670011846 ], "mean_node_assignment_entropy": 1.3830609346822742, "normalized_node_entropy": 0.9976675758566858, "effective_mass_slots": 3.988488635946335, "node_slot_mutual_information_nats": 0.0003514364778336443, "normalized_node_slot_mutual_information": 0.0002535078318790414, "centered_token_effective_rank": 2.037549546633109, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.0019201164379624056, "token_RMS": 0.24678031035201442, "mean_token_offdiagonal_cosine": 0.9999200918093815, "temperature": 0.5009380578994751 }, { "layer": 1, "head": 4, "slices": 4, "native_nodes": 33555, "mass": [ 0.2453369992795564, 0.2691500216726929, 0.2506070949204933, 0.23490588414946142 ], "mean_node_assignment_entropy": 1.3848275833696284, "normalized_node_entropy": 0.9989419435068052, "effective_mass_slots": 3.995102903256932, "node_slot_mutual_information_nats": 0.00024175352862543242, "normalized_node_slot_mutual_information": 0.00017438830843265971, "centered_token_effective_rank": 2.1892253440689795, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.0016016322394129894, "token_RMS": 0.24208671051701242, "mean_token_offdiagonal_cosine": 0.9999450681412253, "temperature": 0.5020498633384705 }, { "layer": 1, "head": 5, "slices": 4, "native_nodes": 33555, "mass": [ 0.2506064144687705, 0.25590138765849363, 0.2437066643328838, 0.249785533713488 ], "mean_node_assignment_entropy": 1.3854835754801504, "normalized_node_entropy": 0.9994151417891615, "effective_mass_slots": 3.9994007520057036, "node_slot_mutual_information_nats": 0.0006609624182276619, "normalized_node_slot_mutual_information": 0.0004767836014955124, "centered_token_effective_rank": 1.2601508066880092, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.002792474934499636, "token_RMS": 0.3221881215413532, "mean_token_offdiagonal_cosine": 0.9999001068939535, "temperature": 0.5022108554840088 }, { "layer": 1, "head": 6, "slices": 4, "native_nodes": 33555, "mass": [ 0.25465545753504726, 0.2700853337964115, 0.23092738612231758, 0.24433182200133544 ], "mean_node_assignment_entropy": 1.3840857742019932, "normalized_node_entropy": 0.9984068413030887, "effective_mass_slots": 3.9934427405803667, "node_slot_mutual_information_nats": 0.0005679269161118494, "normalized_node_slot_mutual_information": 0.00040967267273096374, "centered_token_effective_rank": 1.7537284392285364, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.003330095262757454, "token_RMS": 0.24848504416320538, "mean_token_offdiagonal_cosine": 0.999772714172491, "temperature": 0.5063515901565552 }, { "layer": 1, "head": 7, "slices": 4, "native_nodes": 33555, "mass": [ 0.2524640740389583, 0.2562278263922545, 0.2391682453747745, 0.25213985361803876 ], "mean_node_assignment_entropy": 1.3855841356225347, "normalized_node_entropy": 0.9994876805985259, "effective_mass_slots": 3.9986550688237927, "node_slot_mutual_information_nats": 0.0003739361643844141, "normalized_node_slot_mutual_information": 0.00026973792498321723, "centered_token_effective_rank": 1.1741751415436483, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.002779710829043673, "token_RMS": 0.2673410520270505, "mean_token_offdiagonal_cosine": 0.9998572350526881, "temperature": 0.5009306073188782 }, { "layer": 1, "head": 8, "slices": 4, "native_nodes": 33555, "mass": [ 0.24751835662647892, 0.22624905995304087, 0.2484781366922411, 0.27775444630813784 ], "mean_node_assignment_entropy": 1.3827155977901142, "normalized_node_entropy": 0.9974184679458082, "effective_mass_slots": 3.98933696718472, "node_slot_mutual_information_nats": 0.0009094456653744309, "normalized_node_slot_mutual_information": 0.0006560263756968276, "centered_token_effective_rank": 1.2280546673620758, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.004293467144824423, "token_RMS": 0.20885283840919272, "mean_token_offdiagonal_cosine": 0.9994413209508289, "temperature": 0.4986470341682434 }, { "layer": 2, "head": 1, "slices": 4, "native_nodes": 33555, "mass": [ 0.2510049637513089, 0.25782915778938836, 0.23125858677604164, 0.25990729176985705 ], "mean_node_assignment_entropy": 1.3848738623121906, "normalized_node_entropy": 0.9989753267072713, "effective_mass_slots": 3.995850798795906, "node_slot_mutual_information_nats": 0.00038266013838739177, "normalized_node_slot_mutual_information": 0.0002760309419986873, "centered_token_effective_rank": 2.0478574960866966, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.002362438486721461, "token_RMS": 0.20146837157942463, "mean_token_offdiagonal_cosine": 0.9998177860424771, "temperature": 0.5008584856987 }, { "layer": 2, "head": 2, "slices": 4, "native_nodes": 33555, "mass": [ 0.22627305028211553, 0.2650048193971014, 0.24120085511676886, 0.2675212752875016 ], "mean_node_assignment_entropy": 1.383441798565582, "normalized_node_entropy": 0.9979423110745368, "effective_mass_slots": 3.9905642707438207, "node_slot_mutual_information_nats": 0.0004908435762021046, "normalized_node_slot_mutual_information": 0.0003540687966194902, "centered_token_effective_rank": 1.7806729604734473, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.0026672031661534607, "token_RMS": 0.23288344133229816, "mean_token_offdiagonal_cosine": 0.999827345049792, "temperature": 0.49950912594795227 }, { "layer": 2, "head": 3, "slices": 4, "native_nodes": 33555, "mass": [ 0.21789061089259043, 0.2524894613730998, 0.30176009005822396, 0.22785983749845312 ], "mean_node_assignment_entropy": 1.377416127705519, "normalized_node_entropy": 0.9935957083406157, "effective_mass_slots": 3.9673261847158536, "node_slot_mutual_information_nats": 0.0006762349777695853, "normalized_node_slot_mutual_information": 0.00048780042445191956, "centered_token_effective_rank": 2.0700458107283954, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.0034118492019352665, "token_RMS": 0.260794645253883, "mean_token_offdiagonal_cosine": 0.9997737518750526, "temperature": 0.4969361424446106 }, { "layer": 2, "head": 4, "slices": 4, "native_nodes": 33555, "mass": [ 0.25402378121097013, 0.2587824820646176, 0.2269790819008861, 0.26021465483684864 ], "mean_node_assignment_entropy": 1.3838009936638944, "normalized_node_entropy": 0.9982014155680602, "effective_mass_slots": 3.9940658611698554, "node_slot_mutual_information_nats": 0.0010087312212752053, "normalized_node_slot_mutual_information": 0.0007276457652618032, "centered_token_effective_rank": 1.6036044544966253, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.004635533459638113, "token_RMS": 0.2502636405108087, "mean_token_offdiagonal_cosine": 0.999546681101671, "temperature": 0.5000073313713074 }, { "layer": 2, "head": 5, "slices": 4, "native_nodes": 33555, "mass": [ 0.2563805349336582, 0.23874156596212626, 0.2491334419299515, 0.2557444574158445 ], "mean_node_assignment_entropy": 1.3853623258351127, "normalized_node_entropy": 0.9993276786583587, "effective_mass_slots": 3.9983797764590943, "node_slot_mutual_information_nats": 0.0005268973422571932, "normalized_node_slot_mutual_information": 0.00038007609136601374, "centered_token_effective_rank": 1.2320094627359064, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.0030535976712250832, "token_RMS": 0.2339310418074867, "mean_token_offdiagonal_cosine": 0.9997770405357268, "temperature": 0.5011473298072815 }, { "layer": 2, "head": 6, "slices": 4, "native_nodes": 33555, "mass": [ 0.2534533094030222, 0.2355846799031897, 0.2801744795768383, 0.23078753117468048 ], "mean_node_assignment_entropy": 1.3826023147531676, "normalized_node_entropy": 0.9973367515079984, "effective_mass_slots": 3.9881881064895475, "node_slot_mutual_information_nats": 0.0007347043608103565, "normalized_node_slot_mutual_information": 0.0005299771689302985, "centered_token_effective_rank": 1.864254466045362, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.004701367762939605, "token_RMS": 0.25963909540694585, "mean_token_offdiagonal_cosine": 0.9995988241246305, "temperature": 0.5021385550498962 }, { "layer": 2, "head": 7, "slices": 4, "native_nodes": 33555, "mass": [ 0.23092486330976503, 0.26822111740044446, 0.23628289956544518, 0.2645711201022588 ], "mean_node_assignment_entropy": 1.3813992063929854, "normalized_node_entropy": 0.9964688922755549, "effective_mass_slots": 3.991227901716658, "node_slot_mutual_information_nats": 0.0026997219562137076, "normalized_node_slot_mutual_information": 0.0019474377390042835, "centered_token_effective_rank": 1.0767061938666709, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.012032139001101914, "token_RMS": 0.21373745835508304, "mean_token_offdiagonal_cosine": 0.9961799743808609, "temperature": 0.498592734336853 }, { "layer": 2, "head": 8, "slices": 4, "native_nodes": 33555, "mass": [ 0.24574716149952422, 0.24549954824850148, 0.2472439348706544, 0.2615093555687224 ], "mean_node_assignment_entropy": 1.3857101196872115, "normalized_node_entropy": 0.999578558891196, "effective_mass_slots": 3.998586899838115, "node_slot_mutual_information_nats": 0.00023090397588032552, "normalized_node_slot_mutual_information": 0.00016656201046204522, "centered_token_effective_rank": 1.5155684609864815, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.0016599840223258469, "token_RMS": 0.2320978812833312, "mean_token_offdiagonal_cosine": 0.9999328944758302, "temperature": 0.5016647577285767 }, { "layer": 3, "head": 1, "slices": 4, "native_nodes": 33555, "mass": [ 0.2516327776739768, 0.26669845383216295, 0.22039589783799327, 0.2612728709653919 ], "mean_node_assignment_entropy": 1.3811754610449378, "normalized_node_entropy": 0.9963074941235298, "effective_mass_slots": 3.989502584198183, "node_slot_mutual_information_nats": 0.002491096470940146, "normalized_node_slot_mutual_information": 0.0017969462625006733, "centered_token_effective_rank": 1.1220757829228907, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.01106798091568486, "token_RMS": 0.27981033797425975, "mean_token_offdiagonal_cosine": 0.9981072016863867, "temperature": 0.49863627552986145 }, { "layer": 3, "head": 2, "slices": 4, "native_nodes": 33555, "mass": [ 0.23164640819061852, 0.2574804626692782, 0.26738714513152, 0.24348598376034164 ], "mean_node_assignment_entropy": 1.3842200445083443, "normalized_node_entropy": 0.9985036968556443, "effective_mass_slots": 3.9940910418057247, "node_slot_mutual_information_nats": 0.0005959848688814606, "normalized_node_slot_mutual_information": 0.00042991220739007117, "centered_token_effective_rank": 1.0600390415399252, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.00550393629512904, "token_RMS": 0.2212213503940899, "mean_token_offdiagonal_cosine": 0.9991856441827637, "temperature": 0.49928978085517883 }, { "layer": 3, "head": 3, "slices": 4, "native_nodes": 33555, "mass": [ 0.2468800224293861, 0.24020249748741562, 0.25517587065208225, 0.2577416092632531 ], "mean_node_assignment_entropy": 1.3842438252105729, "normalized_node_entropy": 0.9985208510062313, "effective_mass_slots": 3.9984564966078024, "node_slot_mutual_information_nats": 0.0016645855920254249, "normalized_node_slot_mutual_information": 0.00120074468937515, "centered_token_effective_rank": 1.0747301045736195, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.008436181769401245, "token_RMS": 0.2352567603800912, "mean_token_offdiagonal_cosine": 0.9982974869865862, "temperature": 0.501215398311615 }, { "layer": 3, "head": 4, "slices": 4, "native_nodes": 33555, "mass": [ 0.2478813682014053, 0.29634551536644327, 0.22410825761352882, 0.23166485869427975 ], "mean_node_assignment_entropy": 1.3664705960726231, "normalized_node_entropy": 0.9857001762372797, "effective_mass_slots": 3.9755130251259363, "node_slot_mutual_information_nats": 0.01368320663053324, "normalized_node_slot_mutual_information": 0.009870347174664644, "centered_token_effective_rank": 1.0065685560710906, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.0295439823928713, "token_RMS": 0.25264492330428756, "mean_token_offdiagonal_cosine": 0.9819637739500527, "temperature": 0.4928109347820282 }, { "layer": 3, "head": 5, "slices": 4, "native_nodes": 33555, "mass": [ 0.2461956641185907, 0.24863063933161542, 0.2631341097007972, 0.2420395868645395 ], "mean_node_assignment_entropy": 1.3848209688105482, "normalized_node_entropy": 0.9989371721110138, "effective_mass_slots": 3.998000195673694, "node_slot_mutual_information_nats": 0.000973316210553854, "normalized_node_slot_mutual_information": 0.0007020992350914416, "centered_token_effective_rank": 1.13472479048237, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.006704262591125094, "token_RMS": 0.23767317674491076, "mean_token_offdiagonal_cosine": 0.9989610101777973, "temperature": 0.49965840578079224 }, { "layer": 3, "head": 6, "slices": 4, "native_nodes": 33555, "mass": [ 0.2511298296846268, 0.24871270599077114, 0.24471391547083268, 0.2554435492454494 ], "mean_node_assignment_entropy": 1.3851247059425655, "normalized_node_entropy": 0.9991562721380616, "effective_mass_slots": 3.9995160636192875, "node_slot_mutual_information_nats": 0.0010486637629809614, "normalized_node_slot_mutual_information": 0.0007564510052062962, "centered_token_effective_rank": 1.236091619807539, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.007363030283832002, "token_RMS": 0.2523033568799037, "mean_token_offdiagonal_cosine": 0.9988863794357029, "temperature": 0.5025554895401001 }, { "layer": 3, "head": 7, "slices": 4, "native_nodes": 33555, "mass": [ 0.2507423230937724, 0.2819695062150438, 0.21400093496902708, 0.25328723603789877 ], "mean_node_assignment_entropy": 1.3786422816700346, "normalized_node_entropy": 0.9944801914626021, "effective_mass_slots": 3.981200922608558, "node_slot_mutual_information_nats": 0.0029412314610337553, "normalized_node_slot_mutual_information": 0.0021216500214699995, "centered_token_effective_rank": 1.0426754720909488, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.011171358689468667, "token_RMS": 0.24792866218540233, "mean_token_offdiagonal_cosine": 0.9973008282903147, "temperature": 0.4960341155529022 }, { "layer": 3, "head": 8, "slices": 4, "native_nodes": 33555, "mass": [ 0.23616119942878613, 0.24531685802499933, 0.26813567916605513, 0.25038626323716506 ], "mean_node_assignment_entropy": 1.384781602210294, "normalized_node_entropy": 0.9989087751115321, "effective_mass_slots": 3.9956935781289675, "node_slot_mutual_information_nats": 0.00043557348587874856, "normalized_node_slot_mutual_information": 0.00031419985400999475, "centered_token_effective_rank": 1.106662763180705, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.0033940291593270197, "token_RMS": 0.22279304796086538, "mean_token_offdiagonal_cosine": 0.9996916703345423, "temperature": 0.5001254677772522 }, { "layer": 4, "head": 1, "slices": 4, "native_nodes": 33555, "mass": [ 0.2604388806272274, 0.2559071713602502, 0.23969436745195272, 0.2439595805454709 ], "mean_node_assignment_entropy": 1.3828649803404534, "normalized_node_entropy": 0.997526224678093, "effective_mass_slots": 3.9977079540901994, "node_slot_mutual_information_nats": 0.00285620506816886, "normalized_node_slot_mutual_information": 0.002060316443804569, "centered_token_effective_rank": 1.1093534791047632, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.010432049945231388, "token_RMS": 0.22754151566885586, "mean_token_offdiagonal_cosine": 0.9973655692632973, "temperature": 0.4963186979293823 }, { "layer": 4, "head": 2, "slices": 4, "native_nodes": 33555, "mass": [ 0.2548021782063599, 0.22187308643111492, 0.22352166825800251, 0.2998030673723039 ], "mean_node_assignment_entropy": 1.3619784121049796, "normalized_node_entropy": 0.9824597504708396, "effective_mass_slots": 3.9688970292273886, "node_slot_mutual_information_nats": 0.016509817602131882, "normalized_node_slot_mutual_information": 0.011909315990288492, "centered_token_effective_rank": 1.0140676163056146, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.02170829890657091, "token_RMS": 0.25324180073496166, "mean_token_offdiagonal_cosine": 0.9907744449676458, "temperature": 0.4920943081378937 }, { "layer": 4, "head": 3, "slices": 4, "native_nodes": 33555, "mass": [ 0.23327367954914233, 0.281883838105319, 0.25396068436105423, 0.23088179795606323 ], "mean_node_assignment_entropy": 1.3814981685823564, "normalized_node_entropy": 0.9965402784054754, "effective_mass_slots": 3.9867982210083315, "node_slot_mutual_information_nats": 0.0014902893082635593, "normalized_node_slot_mutual_information": 0.0010750164972608404, "centered_token_effective_rank": 1.0443717737959086, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.007802303615072778, "token_RMS": 0.3002404942522416, "mean_token_offdiagonal_cosine": 0.9992194856631188, "temperature": 0.5005096197128296 }, { "layer": 4, "head": 4, "slices": 4, "native_nodes": 33555, "mass": [ 0.23338579502044962, 0.27873787836335023, 0.24805096197352114, 0.2398253645867247 ], "mean_node_assignment_entropy": 1.3827793322223252, "normalized_node_entropy": 0.9974644426204505, "effective_mass_slots": 3.9905148888684714, "node_slot_mutual_information_nats": 0.0011409351830700132, "normalized_node_slot_mutual_information": 0.0008230107652954248, "centered_token_effective_rank": 1.097927834353243, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.006537159497525987, "token_RMS": 0.2563359103348738, "mean_token_offdiagonal_cosine": 0.9991504578712123, "temperature": 0.49733424186706543 }, { "layer": 4, "head": 5, "slices": 4, "native_nodes": 33555, "mass": [ 0.2468888109770533, 0.2483544479182395, 0.255886794274334, 0.2488699466633985 ], "mean_node_assignment_entropy": 1.3745874784171477, "normalized_node_entropy": 0.9915552691902421, "effective_mass_slots": 3.99961523301604, "node_slot_mutual_information_nats": 0.011610686330030262, "normalized_node_slot_mutual_information": 0.00837533979482597, "centered_token_effective_rank": 1.0324095696496363, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.023909450376584856, "token_RMS": 0.22707672736177886, "mean_token_offdiagonal_cosine": 0.9881545312769808, "temperature": 0.49762800335884094 }, { "layer": 4, "head": 6, "slices": 4, "native_nodes": 33555, "mass": [ 0.2311712487752703, 0.249509412266515, 0.2746252176623429, 0.24469412119417713 ], "mean_node_assignment_entropy": 1.3838318758594146, "normalized_node_entropy": 0.9982236923632246, "effective_mass_slots": 3.9921693951966395, "node_slot_mutual_information_nats": 0.0005029153560132116, "normalized_node_slot_mutual_information": 0.00036277674505358397, "centered_token_effective_rank": 1.1325733531740099, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.0040382328862564616, "token_RMS": 0.23880329212417079, "mean_token_offdiagonal_cosine": 0.9996204442040151, "temperature": 0.49993064999580383 }, { "layer": 4, "head": 7, "slices": 4, "native_nodes": 33555, "mass": [ 0.2538975980318777, 0.2313872107475743, 0.23569289054063797, 0.2790223006448275 ], "mean_node_assignment_entropy": 1.3826168239596432, "normalized_node_entropy": 0.9973472176881132, "effective_mass_slots": 3.9888901367387604, "node_slot_mutual_information_nats": 0.0008962070297657831, "normalized_node_slot_mutual_information": 0.0006464767187264615, "centered_token_effective_rank": 1.217171978086131, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.00653994797933469, "token_RMS": 0.2923413981093935, "mean_token_offdiagonal_cosine": 0.9993776137407028, "temperature": 0.4983014166355133 }, { "layer": 4, "head": 8, "slices": 4, "native_nodes": 33555, "mass": [ 0.2590835673325713, 0.26450919771741566, 0.2354506033842831, 0.24095663168962872 ], "mean_node_assignment_entropy": 1.3847839496078829, "normalized_node_entropy": 0.9989104684009624, "effective_mass_slots": 3.995308431651421, "node_slot_mutual_information_nats": 0.00033683104862558544, "normalized_node_slot_mutual_information": 0.0002429722417347807, "centered_token_effective_rank": 1.5306201181155208, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.0026160024151602266, "token_RMS": 0.26505270492171157, "mean_token_offdiagonal_cosine": 0.9998797929506029, "temperature": 0.5009399652481079 }, { "layer": 5, "head": 1, "slices": 4, "native_nodes": 33555, "mass": [ 0.2461153014577318, 0.23622041889752918, 0.2541583547074719, 0.2635059249057373 ], "mean_node_assignment_entropy": 1.3834107587973186, "normalized_node_entropy": 0.9979199206146647, "effective_mass_slots": 3.9967609721396635, "node_slot_mutual_information_nats": 0.002073517327221497, "normalized_node_slot_mutual_information": 0.0014957265825898958, "centered_token_effective_rank": 1.0435658438945432, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.009158274231379684, "token_RMS": 0.26732900703624046, "mean_token_offdiagonal_cosine": 0.998437388257089, "temperature": 0.49988555908203125 }, { "layer": 5, "head": 2, "slices": 4, "native_nodes": 33555, "mass": [ 0.25287973229388794, 0.26899580598108425, 0.24150943088826365, 0.2366150307630466 ], "mean_node_assignment_entropy": 1.3812472843000443, "normalized_node_entropy": 0.9963593036505111, "effective_mass_slots": 3.9950777674370985, "node_slot_mutual_information_nats": 0.0038157609207445375, "normalized_node_slot_mutual_information": 0.002752489678788025, "centered_token_effective_rank": 1.0085029439856565, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.011945048194787554, "token_RMS": 0.24253081037032248, "mean_token_offdiagonal_cosine": 0.9967743896473368, "temperature": 0.49723154306411743 }, { "layer": 5, "head": 3, "slices": 4, "native_nodes": 33555, "mass": [ 0.22723778918432133, 0.23898730843242316, 0.26145623737698853, 0.27231866524163023 ], "mean_node_assignment_entropy": 1.3815461943381497, "normalized_node_entropy": 0.9965749216653343, "effective_mass_slots": 3.989845599116397, "node_slot_mutual_information_nats": 0.002206338851566514, "normalized_node_slot_mutual_information": 0.0015915370598378303, "centered_token_effective_rank": 1.159325466167698, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.00872831492472058, "token_RMS": 0.23433962934516084, "mean_token_offdiagonal_cosine": 0.99827022148907, "temperature": 0.4985033869743347 }, { "layer": 5, "head": 4, "slices": 4, "native_nodes": 33555, "mass": [ 0.23323156976865028, 0.24848476457553192, 0.24645757538882226, 0.27182609011867226 ], "mean_node_assignment_entropy": 1.3844677029855612, "normalized_node_entropy": 0.9986823446841018, "effective_mass_slots": 3.9938792016568514, "node_slot_mutual_information_nats": 0.0002952865974574781, "normalized_node_slot_mutual_information": 0.00021300425489643963, "centered_token_effective_rank": 1.3449157832958467, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.0024151980520172667, "token_RMS": 0.2475057108827082, "mean_token_offdiagonal_cosine": 0.9998744187635805, "temperature": 0.5001603364944458 }, { "layer": 5, "head": 5, "slices": 4, "native_nodes": 33555, "mass": [ 0.2564261095332169, 0.24218637962807416, 0.2406172439835676, 0.2607702670194516 ], "mean_node_assignment_entropy": 1.3844560725654471, "normalized_node_entropy": 0.9986739551093907, "effective_mass_slots": 3.9975513831561114, "node_slot_mutual_information_nats": 0.0012259469005824197, "normalized_node_slot_mutual_information": 0.000884333756931726, "centered_token_effective_rank": 1.4787193982710265, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.005746501291729503, "token_RMS": 0.24936165487032874, "mean_token_offdiagonal_cosine": 0.9993400929238637, "temperature": 0.49955999851226807 }, { "layer": 5, "head": 6, "slices": 4, "native_nodes": 33555, "mass": [ 0.16898452662627614, 0.25945326595756874, 0.2234420625303918, 0.34812014458201146 ], "mean_node_assignment_entropy": 1.2902302581888752, "normalized_node_entropy": 0.9307043975469885, "effective_mass_slots": 3.8677948563201423, "node_slot_mutual_information_nats": 0.06245428188405966, "normalized_node_slot_mutual_information": 0.04505124137820715, "centered_token_effective_rank": 1.022353763248688, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.05750432019107066, "token_RMS": 0.24593172044150763, "mean_token_offdiagonal_cosine": 0.9430625672243514, "temperature": 0.49001261591911316 }, { "layer": 5, "head": 7, "slices": 4, "native_nodes": 33555, "mass": [ 0.2641117946079495, 0.2533129651008962, 0.24152039278816526, 0.2410548475704895 ], "mean_node_assignment_entropy": 1.3849750674620642, "normalized_node_entropy": 0.9990483307911888, "effective_mass_slots": 3.997119886187113, "node_slot_mutual_information_nats": 0.0005990058596199166, "normalized_node_slot_mutual_information": 0.00043209139156854214, "centered_token_effective_rank": 1.2943610834092314, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.004384438225388055, "token_RMS": 0.244812491019166, "mean_token_offdiagonal_cosine": 0.9995814572555944, "temperature": 0.4997832477092743 }, { "layer": 5, "head": 8, "slices": 4, "native_nodes": 33555, "mass": [ 0.2765137847303687, 0.2446571559610958, 0.23102795251736324, 0.24780110677873798 ], "mean_node_assignment_entropy": 1.3796425775238732, "normalized_node_entropy": 0.9952017523964796, "effective_mass_slots": 3.991350363436958, "node_slot_mutual_information_nats": 0.004487033072650526, "normalized_node_slot_mutual_information": 0.003236710181108841, "centered_token_effective_rank": 1.0601734760565695, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.01070276978330896, "token_RMS": 0.2441709180915225, "mean_token_offdiagonal_cosine": 0.9977532933968146, "temperature": 0.48933830857276917 }, { "layer": 6, "head": 1, "slices": 4, "native_nodes": 33555, "mass": [ 0.2524788585110262, 0.2207143924956065, 0.24490522726462116, 0.28190152212531105 ], "mean_node_assignment_entropy": 1.3751127338770868, "normalized_node_entropy": 0.991934160913869, "effective_mass_slots": 3.9848089854634905, "node_slot_mutual_information_nats": 0.007376643831931595, "normalized_node_slot_mutual_information": 0.005321123737365936, "centered_token_effective_rank": 1.0062586304322956, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.019979091625798813, "token_RMS": 0.28936835583791826, "mean_token_offdiagonal_cosine": 0.9938309092979662, "temperature": 0.4965653419494629 }, { "layer": 6, "head": 2, "slices": 4, "native_nodes": 33555, "mass": [ 0.24840924312675466, 0.2701550837645604, 0.244279486407219, 0.23715618662819246 ], "mean_node_assignment_entropy": 1.364395939743864, "normalized_node_entropy": 0.9842036280387548, "effective_mass_slots": 3.995210017162105, "node_slot_mutual_information_nats": 0.020700208095649764, "normalized_node_slot_mutual_information": 0.014932043782481743, "centered_token_effective_rank": 1.020196557373124, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.035314315820352396, "token_RMS": 0.31903062311161867, "mean_token_offdiagonal_cosine": 0.9855624633046723, "temperature": 0.4885939657688141 }, { "layer": 6, "head": 3, "slices": 4, "native_nodes": 33555, "mass": [ 0.2788623889060831, 0.25358779996060665, 0.23993416988108912, 0.22761564089917619 ], "mean_node_assignment_entropy": 1.3640992769502673, "normalized_node_entropy": 0.9839896310681857, "effective_mass_slots": 3.9885368336902967, "node_slot_mutual_information_nats": 0.01932517834925962, "normalized_node_slot_mutual_information": 0.01394016948438581, "centered_token_effective_rank": 1.0056401747681096, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.03476471266184616, "token_RMS": 0.2819738505018836, "mean_token_offdiagonal_cosine": 0.9796942772341625, "temperature": 0.49506425857543945 }, { "layer": 6, "head": 4, "slices": 4, "native_nodes": 33555, "mass": [ 0.2788971184667257, 0.23100156372305072, 0.24538690136017555, 0.24471441634613292 ], "mean_node_assignment_entropy": 1.382347152787629, "normalized_node_entropy": 0.9971526910568453, "effective_mass_slots": 3.990215137122605, "node_slot_mutual_information_nats": 0.0014979957389089638, "normalized_node_slot_mutual_information": 0.0010805755118983803, "centered_token_effective_rank": 1.0982088025465873, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.007923927626345311, "token_RMS": 0.22011357380133023, "mean_token_offdiagonal_cosine": 0.9982890772541723, "temperature": 0.498247355222702 }, { "layer": 6, "head": 5, "slices": 4, "native_nodes": 33555, "mass": [ 0.2524813076930578, 0.24918952359117244, 0.2569019081149011, 0.24142726069945475 ], "mean_node_assignment_entropy": 1.3847612949129433, "normalized_node_entropy": 0.9988941264929414, "effective_mass_slots": 3.9989733741966136, "node_slot_mutual_information_nats": 0.0012763768141972331, "normalized_node_slot_mutual_information": 0.0009207112500740011, "centered_token_effective_rank": 1.0387723074630995, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.008012190405989782, "token_RMS": 0.24781938184682037, "mean_token_offdiagonal_cosine": 0.9987860307135127, "temperature": 0.5009581446647644 }, { "layer": 6, "head": 6, "slices": 4, "native_nodes": 33555, "mass": [ 0.24433654838721383, 0.25665212651022745, 0.25521674477881745, 0.2437945803783633 ], "mean_node_assignment_entropy": 1.3521832942717829, "normalized_node_entropy": 0.9753940665094015, "effective_mass_slots": 3.998863841085703, "node_slot_mutual_information_nats": 0.033826986772609535, "normalized_node_slot_mutual_information": 0.02440101303253017, "centered_token_effective_rank": 1.0047251154665777, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.040493260707927116, "token_RMS": 0.31511905486177805, "mean_token_offdiagonal_cosine": 0.9779523586971033, "temperature": 0.4921133816242218 }, { "layer": 6, "head": 7, "slices": 4, "native_nodes": 33555, "mass": [ 0.2523206435613564, 0.2600112346223704, 0.22539694314652417, 0.2622711786946176 ], "mean_node_assignment_entropy": 1.3766545909688543, "normalized_node_entropy": 0.9930463757038953, "effective_mass_slots": 3.9929768556451903, "node_slot_mutual_information_nats": 0.007882440863323126, "normalized_node_slot_mutual_information": 0.005685979171808396, "centered_token_effective_rank": 1.0142160452963573, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.024941997241358788, "token_RMS": 0.29066839717413995, "mean_token_offdiagonal_cosine": 0.9903763841795826, "temperature": 0.49553951621055603 }, { "layer": 6, "head": 8, "slices": 4, "native_nodes": 33555, "mass": [ 0.24983437533812736, 0.24874006492506107, 0.25118732261570675, 0.2502382367383064 ], "mean_node_assignment_entropy": 1.378584206204674, "normalized_node_entropy": 0.9944382988696657, "effective_mass_slots": 3.999975345126634, "node_slot_mutual_information_nats": 0.0077039911778793435, "normalized_node_slot_mutual_information": 0.005557254933689426, "centered_token_effective_rank": 1.0144026423509236, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.018160661055319265, "token_RMS": 0.25251321705181967, "mean_token_offdiagonal_cosine": 0.9931030318750448, "temperature": 0.49776458740234375 }, { "layer": 7, "head": 1, "slices": 4, "native_nodes": 33555, "mass": [ 0.2885127727213746, 0.22347223538832664, 0.2576465566691015, 0.230368434936985 ], "mean_node_assignment_entropy": 1.3348099043783217, "normalized_node_entropy": 0.9628618147880381, "effective_mass_slots": 3.979283808973063, "node_slot_mutual_information_nats": 0.046291951231326456, "normalized_node_slot_mutual_information": 0.03339258423725421, "centered_token_effective_rank": 1.01595914341168, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.050061991850712445, "token_RMS": 0.27363811653221287, "mean_token_offdiagonal_cosine": 0.9574896541902755, "temperature": 0.4920286238193512 }, { "layer": 7, "head": 2, "slices": 4, "native_nodes": 33555, "mass": [ 0.24658665033463945, 0.22897828066603043, 0.2603658615090669, 0.26406920725134725 ], "mean_node_assignment_entropy": 1.3767002618149184, "normalized_node_entropy": 0.9930793202554602, "effective_mass_slots": 3.993869674046062, "node_slot_mutual_information_nats": 0.008060342212185212, "normalized_node_slot_mutual_information": 0.005814307868693791, "centered_token_effective_rank": 1.047004322302084, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.017885314110175684, "token_RMS": 0.23437953923498542, "mean_token_offdiagonal_cosine": 0.9923415410032522, "temperature": 0.49763843417167664 }, { "layer": 7, "head": 3, "slices": 4, "native_nodes": 33555, "mass": [ 0.2460165468063036, 0.2414525300255477, 0.2685498994165198, 0.24398102377649747 ], "mean_node_assignment_entropy": 1.3837819779665288, "normalized_node_entropy": 0.998187698641916, "effective_mass_slots": 3.996303532433888, "node_slot_mutual_information_nats": 0.0015878390025738653, "normalized_node_slot_mutual_information": 0.0011453837273716968, "centered_token_effective_rank": 1.1568457670856096, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.007338013930119417, "token_RMS": 0.2476044039045436, "mean_token_offdiagonal_cosine": 0.9989947073297939, "temperature": 0.500519335269928 }, { "layer": 7, "head": 4, "slices": 4, "native_nodes": 33555, "mass": [ 0.2368470559797314, 0.2543739968450356, 0.25097116083994725, 0.2578077867758148 ], "mean_node_assignment_entropy": 1.3849515747526961, "normalized_node_entropy": 0.9990313843835377, "effective_mass_slots": 3.997949197748147, "node_slot_mutual_information_nats": 0.0008299543283571964, "normalized_node_slot_mutual_information": 0.0005986854968426288, "centered_token_effective_rank": 1.0878712510205446, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.005440448300318107, "token_RMS": 0.28254846436188463, "mean_token_offdiagonal_cosine": 0.9995101419174394, "temperature": 0.5010110139846802 }, { "layer": 7, "head": 5, "slices": 4, "native_nodes": 33555, "mass": [ 0.29188873705503715, 0.23207604850951435, 0.2453967961983822, 0.2306384183067871 ], "mean_node_assignment_entropy": 1.33248934282063, "normalized_node_entropy": 0.9611878834623584, "effective_mass_slots": 3.9808489660272874, "node_slot_mutual_information_nats": 0.04900576177584681, "normalized_node_slot_mutual_information": 0.03535018474450006, "centered_token_effective_rank": 1.0102481990983296, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.053525727805881225, "token_RMS": 0.24034347570152048, "mean_token_offdiagonal_cosine": 0.9342414720079338, "temperature": 0.49164336919784546 }, { "layer": 7, "head": 6, "slices": 4, "native_nodes": 33555, "mass": [ 0.24636424454735353, 0.26431947265282735, 0.2465864390752085, 0.24272984365888653 ], "mean_node_assignment_entropy": 1.3822425722724176, "normalized_node_entropy": 0.9970772521615108, "effective_mass_slots": 3.9977637853191172, "node_slot_mutual_information_nats": 0.0034925788484823084, "normalized_node_slot_mutual_information": 0.0025193630923095564, "centered_token_effective_rank": 1.0494481140132186, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.014428217447128095, "token_RMS": 0.28690402424458583, "mean_token_offdiagonal_cosine": 0.996641926863365, "temperature": 0.49815502762794495 }, { "layer": 7, "head": 7, "slices": 4, "native_nodes": 33555, "mass": [ 0.22172321922252225, 0.25724924528382076, 0.23835964847687477, 0.2826678866597406 ], "mean_node_assignment_entropy": 1.3800457267039803, "normalized_node_entropy": 0.9954925630579191, "effective_mass_slots": 3.983675169115551, "node_slot_mutual_information_nats": 0.0021590758377676433, "normalized_node_slot_mutual_information": 0.0015574440020252817, "centered_token_effective_rank": 1.1502955197713787, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.006690527148666156, "token_RMS": 0.2757030931621071, "mean_token_offdiagonal_cosine": 0.9992779112532945, "temperature": 0.4999774098396301 }, { "layer": 7, "head": 8, "slices": 4, "native_nodes": 33555, "mass": [ 0.28545093817924744, 0.2384132969776434, 0.2524354530600247, 0.22370031193362816 ], "mean_node_assignment_entropy": 1.3713017649853403, "normalized_node_entropy": 0.9891851279533166, "effective_mass_slots": 3.9835473190363415, "node_slot_mutual_information_nats": 0.010870943541357647, "normalized_node_slot_mutual_information": 0.007841728168450293, "centered_token_effective_rank": 1.011304990053398, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.025809695988221824, "token_RMS": 0.2455268863833329, "mean_token_offdiagonal_cosine": 0.9860531096597898, "temperature": 0.49665534496307373 }, { "layer": 8, "head": 1, "slices": 4, "native_nodes": 33555, "mass": [ 0.22935353074782902, 0.26619396088871067, 0.2529113650619495, 0.25154114305560055 ], "mean_node_assignment_entropy": 1.2585741256668812, "normalized_node_entropy": 0.9078693248453863, "effective_mass_slots": 3.9943552371932265, "node_slot_mutual_information_nats": 0.1263080480839489, "normalized_node_slot_mutual_information": 0.09111199729753891, "centered_token_effective_rank": 1.000034940042019, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.07682253415626732, "token_RMS": 0.26888862078340764, "mean_token_offdiagonal_cosine": 0.8938556169022251, "temperature": 0.49109411239624023 }, { "layer": 8, "head": 2, "slices": 4, "native_nodes": 33555, "mass": [ 0.2041367157249022, 0.37876168614631706, 0.21316617960111695, 0.2039354188363006 ], "mean_node_assignment_entropy": 1.0796633595698406, "normalized_node_entropy": 0.7788124873404634, "effective_mass_slots": 3.8413394235971343, "node_slot_mutual_information_nats": 0.26615775443876766, "normalized_node_slot_mutual_information": 0.1919922362114763, "centered_token_effective_rank": 1.00070603938578, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.13067887813569568, "token_RMS": 0.30856496832914143, "mean_token_offdiagonal_cosine": 0.7705803442557843, "temperature": 0.48332032561302185 }, { "layer": 8, "head": 3, "slices": 4, "native_nodes": 33555, "mass": [ 0.2554603105591619, 0.2726784242865244, 0.23545875461787905, 0.23640251071162488 ], "mean_node_assignment_entropy": 1.351835077812626, "normalized_node_entropy": 0.9751428814300108, "effective_mass_slots": 3.9925426586942874, "node_slot_mutual_information_nats": 0.032593207944721536, "normalized_node_slot_mutual_information": 0.023511029734256263, "centered_token_effective_rank": 1.0059711523326993, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.043859789068554096, "token_RMS": 0.23847593988025334, "mean_token_offdiagonal_cosine": 0.9586978853750526, "temperature": 0.5008258819580078 }, { "layer": 8, "head": 4, "slices": 4, "native_nodes": 33555, "mass": [ 0.2156656194878502, 0.3194098445133758, 0.24283438438879645, 0.2220901515968216 ], "mean_node_assignment_entropy": 1.1012786473842608, "normalized_node_entropy": 0.7944046216090893, "effective_mass_slots": 3.9481515518013333, "node_slot_mutual_information_nats": 0.2719688604313004, "normalized_node_slot_mutual_information": 0.19618406311022987, "centered_token_effective_rank": 1.0050625710612555, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.09851241993131898, "token_RMS": 0.30342447827907754, "mean_token_offdiagonal_cosine": 0.8717993464734825, "temperature": 0.485787570476532 }, { "layer": 8, "head": 5, "slices": 4, "native_nodes": 33555, "mass": [ 0.2590766677054789, 0.22395431044457514, 0.2588236251747425, 0.2581453968233047 ], "mean_node_assignment_entropy": 1.363799140809723, "normalized_node_entropy": 0.9837731286074083, "effective_mass_slots": 3.992588896271172, "node_slot_mutual_information_nats": 0.020640725865626264, "normalized_node_slot_mutual_information": 0.014889136423343784, "centered_token_effective_rank": 1.0048602973393042, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.03486771319027079, "token_RMS": 0.24883618606608732, "mean_token_offdiagonal_cosine": 0.9737892871994399, "temperature": 0.5012246966362 }, { "layer": 8, "head": 6, "slices": 4, "native_nodes": 33555, "mass": [ 0.24305803281169164, 0.2517773286318605, 0.2540121408550488, 0.2511524979229958 ], "mean_node_assignment_entropy": 1.36996905310612, "normalized_node_entropy": 0.9882237795437742, "effective_mass_slots": 3.9994469765097107, "node_slot_mutual_information_nats": 0.01618704258297421, "normalized_node_slot_mutual_information": 0.011676483030557685, "centered_token_effective_rank": 1.005381200942641, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.028780517954970324, "token_RMS": 0.2309728768519107, "mean_token_offdiagonal_cosine": 0.9792436122216589, "temperature": 0.4978821575641632 }, { "layer": 8, "head": 7, "slices": 4, "native_nodes": 33555, "mass": [ 0.23089696274879631, 0.2644091729358929, 0.24189579755871612, 0.2627980665169571 ], "mean_node_assignment_entropy": 1.1928535046807465, "normalized_node_entropy": 0.8604619178549664, "effective_mass_slots": 3.9935587577087523, "node_slot_mutual_information_nats": 0.19182924792267975, "normalized_node_slot_mutual_information": 0.13837555233775478, "centered_token_effective_rank": 1.0034477999625082, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.11540710499665781, "token_RMS": 0.2591656226431233, "mean_token_offdiagonal_cosine": 0.7542296672101237, "temperature": 0.48085930943489075 }, { "layer": 8, "head": 8, "slices": 4, "native_nodes": 33555, "mass": [ 0.2407348393035031, 0.2649366717698446, 0.24869917206074296, 0.24562931662388485 ], "mean_node_assignment_entropy": 1.2900651210200058, "normalized_node_entropy": 0.9305852762596915, "effective_mass_slots": 3.997387867921885, "node_slot_mutual_information_nats": 0.09557599376141912, "normalized_node_slot_mutual_information": 0.06894350611381694, "centered_token_effective_rank": 1.0007982270986382, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.07552895887285314, "token_RMS": 0.25798033393289216, "mean_token_offdiagonal_cosine": 0.8910172144437651, "temperature": 0.49577510356903076 }, { "layer": 9, "head": 1, "slices": 4, "native_nodes": 33555, "mass": [ 0.24270389268915551, 0.2200972098664544, 0.30799601391355286, 0.2292028837442185 ], "mean_node_assignment_entropy": 1.2479599852386882, "normalized_node_entropy": 0.9002128409658597, "effective_mass_slots": 3.9636787613123197, "node_slot_mutual_information_nats": 0.1292125889220861, "normalized_node_slot_mutual_information": 0.0932071806291589, "centered_token_effective_rank": 1.0107843550514641, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.09182207192125871, "token_RMS": 0.2497079469947716, "mean_token_offdiagonal_cosine": 0.8359179850529191, "temperature": 0.4827040433883667 }, { "layer": 9, "head": 2, "slices": 4, "native_nodes": 33555, "mass": [ 0.23819538043128308, 0.28136307929066995, 0.2546922206194776, 0.22574931983975466 ], "mean_node_assignment_entropy": 1.3098906410358417, "normalized_node_entropy": 0.9448863659646372, "effective_mass_slots": 3.9862917272163254, "node_slot_mutual_information_nats": 0.0729707660386012, "normalized_node_slot_mutual_information": 0.05263728114687937, "centered_token_effective_rank": 1.0184994264635754, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.061600834482763295, "token_RMS": 0.2912642457777962, "mean_token_offdiagonal_cosine": 0.9399404132180504, "temperature": 0.48448479175567627 }, { "layer": 9, "head": 3, "slices": 4, "native_nodes": 33555, "mass": [ 0.2088906093501526, 0.2120143131467451, 0.3623645109704147, 0.21673056622110878 ], "mean_node_assignment_entropy": 1.120168996544603, "normalized_node_entropy": 0.8080311281362326, "effective_mass_slots": 3.877565494653417, "node_slot_mutual_information_nats": 0.23503851032996037, "normalized_node_slot_mutual_information": 0.16954444663548163, "centered_token_effective_rank": 1.005168988006721, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.10632075514054515, "token_RMS": 0.261893225916769, "mean_token_offdiagonal_cosine": 0.7862259312213107, "temperature": 0.4674690067768097 }, { "layer": 9, "head": 4, "slices": 4, "native_nodes": 33555, "mass": [ 0.24654169440615853, 0.23302181855503554, 0.2734423707073215, 0.24699411627664028 ], "mean_node_assignment_entropy": 1.3373688368144794, "normalized_node_entropy": 0.9647076943558454, "effective_mass_slots": 3.9932119028593602, "node_slot_mutual_information_nats": 0.04722705844338093, "normalized_node_slot_mutual_information": 0.03406712150601946, "centered_token_effective_rank": 1.0371800077862166, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.043325583989134345, "token_RMS": 0.24900495367853354, "mean_token_offdiagonal_cosine": 0.9596534295625222, "temperature": 0.4930136501789093 }, { "layer": 9, "head": 5, "slices": 4, "native_nodes": 33555, "mass": [ 0.25402279068231404, 0.2690440129472609, 0.23582334122444973, 0.24110985501585944 ], "mean_node_assignment_entropy": 1.3390505481064638, "normalized_node_entropy": 0.9659207926264219, "effective_mass_slots": 3.994765234048196, "node_slot_mutual_information_nats": 0.04593426444091597, "normalized_node_slot_mutual_information": 0.03313456775789587, "centered_token_effective_rank": 1.0239222886870112, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.056371949122431625, "token_RMS": 0.21748796881271787, "mean_token_offdiagonal_cosine": 0.9140342701029164, "temperature": 0.49132731556892395 }, { "layer": 9, "head": 6, "slices": 4, "native_nodes": 33555, "mass": [ 0.1947084506050767, 0.24656760217384616, 0.3493277241808525, 0.20939622325782467 ], "mean_node_assignment_entropy": 1.307455653299491, "normalized_node_entropy": 0.9431298935987078, "effective_mass_slots": 3.8908060361558494, "node_slot_mutual_information_nats": 0.05116069010441682, "normalized_node_slot_mutual_information": 0.036904636951049605, "centered_token_effective_rank": 1.0339034874986468, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.04770302835341671, "token_RMS": 0.21699738071578992, "mean_token_offdiagonal_cosine": 0.9458592662209057, "temperature": 0.48825603723526 }, { "layer": 9, "head": 7, "slices": 4, "native_nodes": 33555, "mass": [ 0.25271417272417757, 0.2564309401011044, 0.24190373604953724, 0.2489511510366975 ], "mean_node_assignment_entropy": 1.3041642159424076, "normalized_node_entropy": 0.9407556234224773, "effective_mass_slots": 3.9990743670553677, "node_slot_mutual_information_nats": 0.08189871016230765, "normalized_node_slot_mutual_information": 0.059077431503181906, "centered_token_effective_rank": 1.0045017873616973, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.08125161333281465, "token_RMS": 0.22254169097489496, "mean_token_offdiagonal_cosine": 0.8283197777095963, "temperature": 0.4861452877521515 }, { "layer": 9, "head": 8, "slices": 4, "native_nodes": 33555, "mass": [ 0.22305721046294527, 0.23560701211870844, 0.3132630817604356, 0.22807269582000048 ], "mean_node_assignment_entropy": 1.2066749272443944, "normalized_node_entropy": 0.8704319667502692, "effective_mass_slots": 3.9589160812251394, "node_slot_mutual_information_nats": 0.16929534369235877, "normalized_node_slot_mutual_information": 0.12212077639527932, "centered_token_effective_rank": 1.0064837165200882, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.08143347111644593, "token_RMS": 0.2445064901172706, "mean_token_offdiagonal_cosine": 0.8761908724017333, "temperature": 0.48101961612701416 }, { "layer": 10, "head": 1, "slices": 4, "native_nodes": 33555, "mass": [ 0.20764845717776606, 0.22391871027842736, 0.3506243203186701, 0.2178085119908834 ], "mean_node_assignment_entropy": 1.3267500938225165, "normalized_node_entropy": 0.9570478904283558, "effective_mass_slots": 3.8998234582112516, "node_slot_mutual_information_nats": 0.03418119116317686, "normalized_node_slot_mutual_information": 0.024656517491396458, "centered_token_effective_rank": 1.0061695626527807, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.043635204142360294, "token_RMS": 0.23144057283644276, "mean_token_offdiagonal_cosine": 0.9542847992757354, "temperature": 0.48982563614845276 }, { "layer": 10, "head": 2, "slices": 4, "native_nodes": 33555, "mass": [ 0.28183737899139444, 0.2456949628597577, 0.20281671171811136, 0.26965094634325243 ], "mean_node_assignment_entropy": 1.349358597530616, "normalized_node_entropy": 0.9733564785191532, "effective_mass_slots": 3.970116584387583, "node_slot_mutual_information_nats": 0.02943686308177984, "normalized_node_slot_mutual_information": 0.021234208193705594, "centered_token_effective_rank": 1.000967069204227, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.05134299841934475, "token_RMS": 0.1746315911913486, "mean_token_offdiagonal_cosine": 0.8957084673537065, "temperature": 0.4897426962852478 }, { "layer": 10, "head": 3, "slices": 4, "native_nodes": 33555, "mass": [ 0.28498189850715744, 0.24620861287906404, 0.21120395980982964, 0.25760552907572687 ], "mean_node_assignment_entropy": 1.374577923817713, "normalized_node_entropy": 0.991548377003631, "effective_mass_slots": 3.9774103058582444, "node_slot_mutual_information_nats": 0.006053006776494341, "normalized_node_slot_mutual_information": 0.004366321429457838, "centered_token_effective_rank": 1.0141959283043294, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.018777744569562785, "token_RMS": 0.26150105193328294, "mean_token_offdiagonal_cosine": 0.9932128669647681, "temperature": 0.49610424041748047 }, { "layer": 10, "head": 4, "slices": 4, "native_nodes": 33555, "mass": [ 0.2650945207313638, 0.24963293230501626, 0.22042139256929538, 0.2648511542313466 ], "mean_node_assignment_entropy": 1.3683691320100753, "normalized_node_entropy": 0.9870696804282355, "effective_mass_slots": 3.9892030870139132, "node_slot_mutual_information_nats": 0.015222351378055166, "normalized_node_slot_mutual_information": 0.010980605421894734, "centered_token_effective_rank": 1.0060652450870708, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.02654798081015556, "token_RMS": 0.2517023757941238, "mean_token_offdiagonal_cosine": 0.9857887435827623, "temperature": 0.4897175431251526 }, { "layer": 10, "head": 5, "slices": 4, "native_nodes": 33555, "mass": [ 0.2635758867710185, 0.24064576416794722, 0.23524667118713338, 0.26053167811370503 ], "mean_node_assignment_entropy": 1.3680555385627893, "normalized_node_entropy": 0.9868434705726081, "effective_mass_slots": 3.9951938154754445, "node_slot_mutual_information_nats": 0.017036553990658954, "normalized_node_slot_mutual_information": 0.012289275978080376, "centered_token_effective_rank": 1.0089667092727415, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.03498654837516822, "token_RMS": 0.23247537485480646, "mean_token_offdiagonal_cosine": 0.9715559231242555, "temperature": 0.4931207001209259 }, { "layer": 10, "head": 6, "slices": 4, "native_nodes": 33555, "mass": [ 0.1875325310724962, 0.19355062640445134, 0.4192052845206018, 0.19971155797475107 ], "mean_node_assignment_entropy": 1.0692350304379272, "normalized_node_entropy": 0.7712900379787787, "effective_mass_slots": 3.7356089477278007, "node_slot_mutual_information_nats": 0.24867581307116948, "normalized_node_slot_mutual_information": 0.17938168115340355, "centered_token_effective_rank": 1.006763941038076, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.1274525259250211, "token_RMS": 0.26844250029760247, "mean_token_offdiagonal_cosine": 0.6639436063301779, "temperature": 0.4826725125312805 }, { "layer": 10, "head": 7, "slices": 4, "native_nodes": 33555, "mass": [ 0.2561778392189218, 0.24220373729420674, 0.22655561798429938, 0.27506280555719415 ], "mean_node_assignment_entropy": 1.3799146353524872, "normalized_node_entropy": 0.9953980006365678, "effective_mass_slots": 3.9898122014286663, "node_slot_mutual_information_nats": 0.0038295271304731315, "normalized_node_slot_mutual_information": 0.002762419900041665, "centered_token_effective_rank": 1.0050145564352058, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.015799298496973185, "token_RMS": 0.22766199707107804, "mean_token_offdiagonal_cosine": 0.9936455106007379, "temperature": 0.49855145812034607 }, { "layer": 10, "head": 8, "slices": 4, "native_nodes": 33555, "mass": [ 0.260343713937831, 0.24282399190065493, 0.2481633321980644, 0.24866896222989865 ], "mean_node_assignment_entropy": 1.3848468619314165, "normalized_node_entropy": 0.9989558500495488, "effective_mass_slots": 3.9986986277078165, "node_slot_mutual_information_nats": 0.0011221031798889225, "normalized_node_slot_mutual_information": 0.0008094263464957425, "centered_token_effective_rank": 1.033508133205256, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.0077900692281848635, "token_RMS": 0.25420678786548195, "mean_token_offdiagonal_cosine": 0.9987727810420451, "temperature": 0.4997904300689697 }, { "layer": 11, "head": 1, "slices": 4, "native_nodes": 33555, "mass": [ 0.4585462473215062, 0.17577811583411435, 0.16072430104597896, 0.20495133612162264 ], "mean_node_assignment_entropy": 0.8087087745004473, "normalized_node_entropy": 0.5833600692475932, "effective_mass_slots": 3.6030495866436145, "node_slot_mutual_information_nats": 0.4730718197696435, "normalized_node_slot_mutual_information": 0.3412491841829911, "centered_token_effective_rank": 1.0002600541281486, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.15901139572021558, "token_RMS": 0.3036156548200119, "mean_token_offdiagonal_cosine": 0.643762527779258, "temperature": 0.4816339910030365 }, { "layer": 11, "head": 2, "slices": 4, "native_nodes": 33555, "mass": [ 0.5113067017188941, 0.15866698753824224, 0.14704805464937454, 0.18297825651003247 ], "mean_node_assignment_entropy": 0.6870676218666654, "normalized_node_entropy": 0.4956145254112058, "effective_mass_slots": 3.4134837871533423, "node_slot_mutual_information_nats": 0.5406657863727491, "normalized_node_slot_mutual_information": 0.39000792438914844, "centered_token_effective_rank": 1.0006374414823513, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.15923533216303065, "token_RMS": 0.3153229816311633, "mean_token_offdiagonal_cosine": 0.6371146877470543, "temperature": 0.46340593695640564 }, { "layer": 11, "head": 3, "slices": 4, "native_nodes": 33555, "mass": [ 0.26171322874941355, 0.2435152246404593, 0.2240522894380505, 0.2707192572084913 ], "mean_node_assignment_entropy": 1.2572516605345538, "normalized_node_entropy": 0.9069153679013077, "effective_mass_slots": 3.989667049063142, "node_slot_mutual_information_nats": 0.1264561205352881, "normalized_node_slot_mutual_information": 0.09121880899315858, "centered_token_effective_rank": 1.0151841468075256, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.0931570793792971, "token_RMS": 0.2249608545594439, "mean_token_offdiagonal_cosine": 0.7758578320260243, "temperature": 0.48950356245040894 }, { "layer": 11, "head": 4, "slices": 4, "native_nodes": 33555, "mass": [ 0.1505713872879082, 0.3117461661493311, 0.30163769922598765, 0.23604474740205303 ], "mean_node_assignment_entropy": 1.2429411059958846, "normalized_node_entropy": 0.8965924848686531, "effective_mass_slots": 3.8603099135681576, "node_slot_mutual_information_nats": 0.10780636274706668, "normalized_node_slot_mutual_information": 0.07776585245573489, "centered_token_effective_rank": 1.0033199202216498, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.11330828201816885, "token_RMS": 0.25884613700708914, "mean_token_offdiagonal_cosine": 0.7535086573951583, "temperature": 0.4868343472480774 }, { "layer": 11, "head": 5, "slices": 4, "native_nodes": 33555, "mass": [ 0.20552331040298472, 0.17997731742818388, 0.17310158571864254, 0.4413977862344484 ], "mean_node_assignment_entropy": 0.9355002520476177, "normalized_node_entropy": 0.6748207871897367, "effective_mass_slots": 3.663443122317935, "node_slot_mutual_information_nats": 0.36290319694624396, "normalized_node_slot_mutual_information": 0.26177932127854847, "centered_token_effective_rank": 1.0112221036860511, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.12172696968988783, "token_RMS": 0.28785189658924887, "mean_token_offdiagonal_cosine": 0.7637130876733287, "temperature": 0.4725141227245331 }, { "layer": 11, "head": 6, "slices": 4, "native_nodes": 33555, "mass": [ 0.5475226114787974, 0.15737297806239073, 0.145386623625254, 0.1497177866242011 ], "mean_node_assignment_entropy": 0.6787203345144888, "normalized_node_entropy": 0.48959323037727565, "effective_mass_slots": 3.272248394700139, "node_slot_mutual_information_nats": 0.5067569965952488, "normalized_node_slot_mutual_information": 0.36554790296187534, "centered_token_effective_rank": 1.0002114585784478, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.16608785678532945, "token_RMS": 0.2865138331217642, "mean_token_offdiagonal_cosine": 0.5718941072476922, "temperature": 0.47412464022636414 }, { "layer": 11, "head": 7, "slices": 4, "native_nodes": 33555, "mass": [ 0.5512953838836271, 0.10803437426353156, 0.1131163795576444, 0.22755386298838762 ], "mean_node_assignment_entropy": 0.5700817033475787, "normalized_node_entropy": 0.4112270231605425, "effective_mass_slots": 3.164768007853115, "node_slot_mutual_information_nats": 0.5819980503055193, "normalized_node_slot_mutual_information": 0.4198228504914091, "centered_token_effective_rank": 1.0011838761196856, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.1869307764868092, "token_RMS": 0.3147318477711589, "mean_token_offdiagonal_cosine": 0.48677535601199384, "temperature": 0.4612300395965576 }, { "layer": 11, "head": 8, "slices": 4, "native_nodes": 33555, "mass": [ 0.13375331571642346, 0.21843554792050596, 0.2064894260750089, 0.44132170997950815 ], "mean_node_assignment_entropy": 0.9433069783596211, "normalized_node_entropy": 0.6804521498576891, "effective_mass_slots": 3.6259223075642963, "node_slot_mutual_information_nats": 0.34480170725818615, "normalized_node_slot_mutual_information": 0.24872185657571663, "centered_token_effective_rank": 1.0261999218544526, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.12186893149006979, "token_RMS": 0.23663310853008848, "mean_token_offdiagonal_cosine": 0.6598332511577337, "temperature": 0.46404287219047546 }, { "layer": 12, "head": 1, "slices": 4, "native_nodes": 33555, "mass": [ 0.14181599557375324, 0.16939411562829437, 0.14511751903027004, 0.5436723699157627 ], "mean_node_assignment_entropy": 0.6448031788205494, "normalized_node_entropy": 0.46512717421692307, "effective_mass_slots": 3.284414600903878, "node_slot_mutual_information_nats": 0.5443852535536328, "normalized_node_slot_mutual_information": 0.3926909528174535, "centered_token_effective_rank": 1.0034646628670079, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.1717899173218555, "token_RMS": 0.2937998215415296, "mean_token_offdiagonal_cosine": 0.540811037745293, "temperature": 0.47362300753593445 }, { "layer": 12, "head": 2, "slices": 4, "native_nodes": 33555, "mass": [ 0.12137238367913354, 0.1990336213901234, 0.1301395378734585, 0.5494544573934179 ], "mean_node_assignment_entropy": 0.5764923648444166, "normalized_node_entropy": 0.4158513379356954, "effective_mass_slots": 3.2273481503210477, "node_slot_mutual_information_nats": 0.5951684290875113, "normalized_node_slot_mutual_information": 0.4293232705691137, "centered_token_effective_rank": 1.0054928448924954, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.19189905074926195, "token_RMS": 0.34182309322628596, "mean_token_offdiagonal_cosine": 0.4748717808347618, "temperature": 0.46855634450912476 }, { "layer": 12, "head": 3, "slices": 4, "native_nodes": 33555, "mass": [ 0.09235160485185814, 0.25101524882917736, 0.09002578060260898, 0.5666073651466205 ], "mean_node_assignment_entropy": 0.4519758540809605, "normalized_node_entropy": 0.32603166164207775, "effective_mass_slots": 3.021018212001153, "node_slot_mutual_information_nats": 0.6536180767731797, "normalized_node_slot_mutual_information": 0.4714857789980241, "centered_token_effective_rank": 1.0169723464067182, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.18291459888510003, "token_RMS": 0.37285837811129763, "mean_token_offdiagonal_cosine": 0.6302422533629711, "temperature": 0.44195908308029175 }, { "layer": 12, "head": 4, "slices": 4, "native_nodes": 33555, "mass": [ 0.09181666914049151, 0.2701042460532354, 0.0764969298281465, 0.5615821540868988 ], "mean_node_assignment_entropy": 0.4324942664400215, "normalized_node_entropy": 0.31197866670296454, "effective_mass_slots": 2.9846244735838625, "node_slot_mutual_information_nats": 0.6609796679959763, "normalized_node_slot_mutual_information": 0.47679604457311425, "centered_token_effective_rank": 1.0123931281220548, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.19062594641295633, "token_RMS": 0.3486910906332836, "mean_token_offdiagonal_cosine": 0.5380124467256544, "temperature": 0.44755518436431885 }, { "layer": 12, "head": 5, "slices": 4, "native_nodes": 33555, "mass": [ 0.1424884713617073, 0.18776508675334605, 0.14176557550403565, 0.5279808661758458 ], "mean_node_assignment_entropy": 0.6399278204155376, "normalized_node_entropy": 0.4616103465201897, "effective_mass_slots": 3.339617687651314, "node_slot_mutual_information_nats": 0.5659285152464235, "normalized_node_slot_mutual_information": 0.4082311312218347, "centered_token_effective_rank": 1.0101932974796737, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.1509550707063564, "token_RMS": 0.26918725397141274, "mean_token_offdiagonal_cosine": 0.5760896860096335, "temperature": 0.46996793150901794 }, { "layer": 12, "head": 6, "slices": 4, "native_nodes": 33555, "mass": [ 0.1520854438162564, 0.21559462676917202, 0.15107147931461984, 0.48124845022597035 ], "mean_node_assignment_entropy": 0.7403100986085386, "normalized_node_entropy": 0.5340208539912791, "effective_mass_slots": 3.506856715538401, "node_slot_mutual_information_nats": 0.5144100150106533, "normalized_node_slot_mutual_information": 0.3710683888197434, "centered_token_effective_rank": 1.0083504481714058, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.1587233978456532, "token_RMS": 0.2934949430263035, "mean_token_offdiagonal_cosine": 0.6038064051702593, "temperature": 0.47580069303512573 }, { "layer": 12, "head": 7, "slices": 4, "native_nodes": 33555, "mass": [ 0.1408869061291533, 0.16179183516255188, 0.13610430205641352, 0.5612169571656491 ], "mean_node_assignment_entropy": 0.6452122728045836, "normalized_node_entropy": 0.4654222731479349, "effective_mass_slots": 3.210503673192854, "node_slot_mutual_information_nats": 0.5212155595653555, "normalized_node_slot_mutual_information": 0.37597755150955225, "centered_token_effective_rank": 1.0043236275632128, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.1580174093709721, "token_RMS": 0.35926451618410293, "mean_token_offdiagonal_cosine": 0.7105756086851261, "temperature": 0.4806678593158722 }, { "layer": 12, "head": 8, "slices": 4, "native_nodes": 33555, "mass": [ 0.2736311769351361, 0.39499835027341534, 0.25682187072789503, 0.07454860243298093 ], "mean_node_assignment_entropy": 1.180584175821631, "normalized_node_entropy": 0.8516114679049257, "effective_mass_slots": 3.5402237822553966, "node_slot_mutual_information_nats": 0.08360576465243064, "normalized_node_slot_mutual_information": 0.06030881102689574, "centered_token_effective_rank": 1.0120450826075122, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.10903802793267084, "token_RMS": 0.2580348714946853, "mean_token_offdiagonal_cosine": 0.7990477385425976, "temperature": 0.4818078279495239 }, { "layer": 13, "head": 1, "slices": 4, "native_nodes": 33555, "mass": [ 0.14839058914924722, 0.15691704544485557, 0.5555676136099974, 0.1391247519038325 ], "mean_node_assignment_entropy": 0.5806483954192873, "normalized_node_entropy": 0.41884928028576984, "effective_mass_slots": 3.2371149100374885, "node_slot_mutual_information_nats": 0.5940340778250811, "normalized_node_slot_mutual_information": 0.4285050090986465, "centered_token_effective_rank": 1.0158546578904064, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.22077862050715433, "token_RMS": 0.3191487264923451, "mean_token_offdiagonal_cosine": 0.31595609506632555, "temperature": 0.4608343839645386 }, { "layer": 13, "head": 2, "slices": 4, "native_nodes": 33555, "mass": [ 0.1932289526059684, 0.20214484331371924, 0.1737148176314152, 0.43091138639272086 ], "mean_node_assignment_entropy": 1.0837194403302963, "normalized_node_entropy": 0.7817383311397407, "effective_mass_slots": 3.697483286680801, "node_slot_mutual_information_nats": 0.22393295509484124, "normalized_node_slot_mutual_information": 0.1615334819034692, "centered_token_effective_rank": 1.0108570396033159, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.08795615927324113, "token_RMS": 0.2094313224292055, "mean_token_offdiagonal_cosine": 0.7974823888171785, "temperature": 0.47702354192733765 }, { "layer": 13, "head": 3, "slices": 4, "native_nodes": 33555, "mass": [ 0.1454144630855361, 0.13506664591776868, 0.5824205598480846, 0.1370983309931629 ], "mean_node_assignment_entropy": 0.5962033036503875, "normalized_node_entropy": 0.4300697747690155, "effective_mass_slots": 3.120651986637019, "node_slot_mutual_information_nats": 0.5418386464157661, "normalized_node_slot_mutual_information": 0.3908539640730071, "centered_token_effective_rank": 1.0227692076556927, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.18988925984780533, "token_RMS": 0.3047101638812328, "mean_token_offdiagonal_cosine": 0.48879737603498574, "temperature": 0.45988520979881287 }, { "layer": 13, "head": 4, "slices": 4, "native_nodes": 33555, "mass": [ 0.25949578135028856, 0.21909157394510217, 0.2836002247266204, 0.2378124202046926 ], "mean_node_assignment_entropy": 1.295005861628896, "normalized_node_entropy": 0.9341492672470737, "effective_mass_slots": 3.9814914611934924, "node_slot_mutual_information_nats": 0.08665062646367017, "normalized_node_slot_mutual_information": 0.06250521454452947, "centered_token_effective_rank": 1.0018730603100088, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.09005129262570431, "token_RMS": 0.20893875585486782, "mean_token_offdiagonal_cosine": 0.7927627609893572, "temperature": 0.4763529598712921 }, { "layer": 13, "head": 5, "slices": 4, "native_nodes": 33555, "mass": [ 0.164635836841732, 0.1566287041823984, 0.16561264873193218, 0.5131228104312011 ], "mean_node_assignment_entropy": 0.7836330158416346, "normalized_node_entropy": 0.5652717329157944, "effective_mass_slots": 3.412828500007631, "node_slot_mutual_information_nats": 0.44390840377488716, "normalized_node_slot_mutual_information": 0.32021222636748264, "centered_token_effective_rank": 1.0189968840695052, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.15483085902255841, "token_RMS": 0.28352006052198386, "mean_token_offdiagonal_cosine": 0.6170667867059527, "temperature": 0.47587844729423523 }, { "layer": 13, "head": 6, "slices": 4, "native_nodes": 33555, "mass": [ 0.18227717045302133, 0.19030916457838956, 0.18148121423913868, 0.4459324501222797 ], "mean_node_assignment_entropy": 0.8851669128947308, "normalized_node_entropy": 0.6385129577961106, "effective_mass_slots": 3.6541593141057014, "node_slot_mutual_information_nats": 0.4106991440300938, "normalized_node_slot_mutual_information": 0.2962568091947792, "centered_token_effective_rank": 1.0202470943274973, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.17443705628237835, "token_RMS": 0.30588759739475846, "mean_token_offdiagonal_cosine": 0.6128089002101053, "temperature": 0.4682173430919647 }, { "layer": 13, "head": 7, "slices": 4, "native_nodes": 33555, "mass": [ 0.1873463429676014, 0.1819236271914898, 0.22847421427880876, 0.4022558155078111 ], "mean_node_assignment_entropy": 1.0448207261724507, "normalized_node_entropy": 0.7536788401335002, "effective_mass_slots": 3.771302093609117, "node_slot_mutual_information_nats": 0.282599598576708, "normalized_node_slot_mutual_information": 0.20385251971191423, "centered_token_effective_rank": 1.0082319693422606, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.13095255837178582, "token_RMS": 0.19853110763886944, "mean_token_offdiagonal_cosine": 0.4980547619393003, "temperature": 0.4834880828857422 }, { "layer": 13, "head": 8, "slices": 4, "native_nodes": 33555, "mass": [ 0.13371988919638114, 0.15655550194483442, 0.13805617562273756, 0.5716684330871373 ], "mean_node_assignment_entropy": 0.6326175899411959, "normalized_node_entropy": 0.45633712989364555, "effective_mass_slots": 3.1657576588820726, "node_slot_mutual_information_nats": 0.5197748236991193, "normalized_node_slot_mutual_information": 0.37493828026482734, "centered_token_effective_rank": 1.019638559578829, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.14739149021621872, "token_RMS": 0.28760709418037217, "mean_token_offdiagonal_cosine": 0.667069654201641, "temperature": 0.47054052352905273 }, { "layer": 14, "head": 1, "slices": 4, "native_nodes": 33555, "mass": [ 0.27971758323603213, 0.20861436179730136, 0.2406952526481971, 0.27097280238996657 ], "mean_node_assignment_entropy": 1.2924065075589097, "normalized_node_entropy": 0.9322742296339319, "effective_mass_slots": 3.974626520696821, "node_slot_mutual_information_nats": 0.08752427907574112, "normalized_node_slot_mutual_information": 0.06313542168997668, "centered_token_effective_rank": 1.0017140543871246, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.09657028564533122, "token_RMS": 0.24107854633252596, "mean_token_offdiagonal_cosine": 0.7871564268707618, "temperature": 0.4545113146305084 }, { "layer": 14, "head": 2, "slices": 4, "native_nodes": 33555, "mass": [ 0.21105196071219898, 0.5350908546336806, 0.11144044120300457, 0.14241674376810265 ], "mean_node_assignment_entropy": 0.5721364192785859, "normalized_node_entropy": 0.4127091874025923, "effective_mass_slots": 3.2707694801836076, "node_slot_mutual_information_nats": 0.612888852989193, "normalized_node_slot_mutual_information": 0.4421058544118169, "centered_token_effective_rank": 1.0060359543921986, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.20859513686315964, "token_RMS": 0.3411429475311927, "mean_token_offdiagonal_cosine": 0.4736215723169906, "temperature": 0.4749888777732849 }, { "layer": 14, "head": 3, "slices": 4, "native_nodes": 33555, "mass": [ 0.2754357645449855, 0.5552852557285872, 0.07281589349947296, 0.09646308662814748 ], "mean_node_assignment_entropy": 0.4313171899559085, "normalized_node_entropy": 0.3111295854997761, "effective_mass_slots": 2.998641330444966, "node_slot_mutual_information_nats": 0.6668421062749399, "normalized_node_slot_mutual_information": 0.48102489988940345, "centered_token_effective_rank": 1.0083167491519256, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.23954147464037406, "token_RMS": 0.35358257805909205, "mean_token_offdiagonal_cosine": 0.32020237679539304, "temperature": 0.44645580649375916 }, { "layer": 14, "head": 4, "slices": 4, "native_nodes": 33555, "mass": [ 0.20300361259091762, 0.5641980672130243, 0.10175410130113874, 0.1310442188320128 ], "mean_node_assignment_entropy": 0.5064587231568689, "normalized_node_entropy": 0.3653327441566856, "effective_mass_slots": 3.1438658029041244, "node_slot_mutual_information_nats": 0.6389944669444545, "normalized_node_slot_mutual_information": 0.4609370743081256, "centered_token_effective_rank": 1.0077975716891472, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.19421886190179077, "token_RMS": 0.3341476223223819, "mean_token_offdiagonal_cosine": 0.5025997343443834, "temperature": 0.4651615619659424 }, { "layer": 14, "head": 5, "slices": 4, "native_nodes": 33555, "mass": [ 0.20940655866970662, 0.5448465417403586, 0.1043656458703308, 0.14138125393677334 ], "mean_node_assignment_entropy": 0.5273717564059286, "normalized_node_entropy": 0.3804183088358678, "effective_mass_slots": 3.224235250343804, "node_slot_mutual_information_nats": 0.6433240340453059, "normalized_node_slot_mutual_information": 0.46406019680092275, "centered_token_effective_rank": 1.0100773585613168, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.2090711896230244, "token_RMS": 0.31637972293279293, "mean_token_offdiagonal_cosine": 0.43987737536455845, "temperature": 0.46011459827423096 }, { "layer": 14, "head": 6, "slices": 4, "native_nodes": 33555, "mass": [ 0.21531704674888308, 0.32403588704470637, 0.22945609082187665, 0.23119097534634286 ], "mean_node_assignment_entropy": 1.3477834734521486, "normalized_node_entropy": 0.9722202666707583, "effective_mass_slots": 3.9438514441195283, "node_slot_mutual_information_nats": 0.024374296280553542, "normalized_node_slot_mutual_information": 0.017582338184556453, "centered_token_effective_rank": 1.0321414011847438, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.04060753573414922, "token_RMS": 0.1955704750996497, "mean_token_offdiagonal_cosine": 0.9451192855024374, "temperature": 0.48356035351753235 }, { "layer": 14, "head": 7, "slices": 4, "native_nodes": 33555, "mass": [ 0.20170664850984638, 0.5535596780949568, 0.10954210168786536, 0.13519157149975447 ], "mean_node_assignment_entropy": 0.5304557245176649, "normalized_node_entropy": 0.3826429215863986, "effective_mass_slots": 3.1997124875467677, "node_slot_mutual_information_nats": 0.6326052336098398, "normalized_node_slot_mutual_information": 0.45632821668466006, "centered_token_effective_rank": 1.004762041000781, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.236608757452312, "token_RMS": 0.3388121552823691, "mean_token_offdiagonal_cosine": 0.29266375531892846, "temperature": 0.4691873788833618 }, { "layer": 14, "head": 8, "slices": 4, "native_nodes": 33555, "mass": [ 0.19919434594347887, 0.5827544990998953, 0.09630089020049712, 0.12175026442923358 ], "mean_node_assignment_entropy": 0.48194036560497727, "normalized_node_entropy": 0.34764648773125734, "effective_mass_slots": 3.058200437713095, "node_slot_mutual_information_nats": 0.6358862851146143, "normalized_node_slot_mutual_information": 0.4586949950520798, "centered_token_effective_rank": 1.007560376360548, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.18634033970380043, "token_RMS": 0.31288950221770484, "mean_token_offdiagonal_cosine": 0.46646519357553845, "temperature": 0.4547710120677948 }, { "layer": 15, "head": 1, "slices": 4, "native_nodes": 33555, "mass": [ 0.09983243911574698, 0.7096514052009639, 0.09610760248849157, 0.09440855325076569 ], "mean_node_assignment_entropy": 0.8507256898426198, "normalized_node_entropy": 0.613668866946395, "effective_mass_slots": 2.512717341466718, "node_slot_mutual_information_nats": 0.0706390838676687, "normalized_node_slot_mutual_information": 0.05095532799441261, "centered_token_effective_rank": 1.0254476906024896, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.08307319997225222, "token_RMS": 0.16800448562168238, "mean_token_offdiagonal_cosine": 0.7554402015509055, "temperature": 0.435411274433136 }, { "layer": 15, "head": 2, "slices": 4, "native_nodes": 33555, "mass": [ 0.21220190721205814, 0.24208047857713175, 0.28935006839732375, 0.2563675456820382 ], "mean_node_assignment_entropy": 1.2620942887785838, "normalized_node_entropy": 0.9104085857775731, "effective_mass_slots": 3.9754093192707667, "node_slot_mutual_information_nats": 0.11803342742776546, "normalized_node_slot_mutual_information": 0.0851431202045823, "centered_token_effective_rank": 1.0069426010540536, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.1146209491185834, "token_RMS": 0.18056876336134972, "mean_token_offdiagonal_cosine": 0.49316166941895173, "temperature": 0.436725378036499 }, { "layer": 15, "head": 3, "slices": 4, "native_nodes": 33555, "mass": [ 0.19133900106153054, 0.36775728499386673, 0.20821589465810794, 0.2326878193371201 ], "mean_node_assignment_entropy": 1.2556267918688728, "normalized_node_entropy": 0.9057432729182707, "effective_mass_slots": 3.8585781782637723, "node_slot_mutual_information_nats": 0.09467197615116674, "normalized_node_slot_mutual_information": 0.06829139525222323, "centered_token_effective_rank": 1.0146370441406536, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.10799640675206523, "token_RMS": 0.20543666767235574, "mean_token_offdiagonal_cosine": 0.6431244048997712, "temperature": 0.45789864659309387 }, { "layer": 15, "head": 4, "slices": 4, "native_nodes": 33555, "mass": [ 0.1994661053730059, 0.20043401723335405, 0.3900419173089141, 0.21005796032408597 ], "mean_node_assignment_entropy": 1.1905415593323294, "normalized_node_entropy": 0.8587942018104826, "effective_mass_slots": 3.8141067936350246, "node_slot_mutual_information_nats": 0.1481649478825835, "normalized_node_slot_mutual_information": 0.10687841777188747, "centered_token_effective_rank": 1.000869788541138, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.12020008329060015, "token_RMS": 0.23415251509968124, "mean_token_offdiagonal_cosine": 0.6947514411985415, "temperature": 0.45678749680519104 }, { "layer": 15, "head": 5, "slices": 4, "native_nodes": 33555, "mass": [ 0.021476566903509246, 0.5662011926075421, 0.3871258098316091, 0.025196437744203704 ], "mean_node_assignment_entropy": 0.1767841151990843, "normalized_node_entropy": 0.12752278315283108, "effective_mass_slots": 2.37424507199755, "node_slot_mutual_information_nats": 0.6878954073358325, "normalized_node_slot_mutual_information": 0.49621164640684956, "centered_token_effective_rank": 1.0428926171550366, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.25938600733118655, "token_RMS": 0.3777712982107039, "mean_token_offdiagonal_cosine": 0.33240270047548, "temperature": 0.43129467964172363 }, { "layer": 15, "head": 6, "slices": 4, "native_nodes": 33555, "mass": [ 0.021452106651789465, 0.5567235950896571, 0.39924738354667577, 0.02257691390007903 ], "mean_node_assignment_entropy": 0.17428718424397857, "normalized_node_entropy": 0.1257216281996445, "effective_mass_slots": 2.3646895668819385, "node_slot_mutual_information_nats": 0.6863595677236889, "normalized_node_slot_mutual_information": 0.4951037723108293, "centered_token_effective_rank": 1.026364292013025, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.29310525833172085, "token_RMS": 0.3995613934787931, "mean_token_offdiagonal_cosine": 0.20262109137493048, "temperature": 0.4456261396408081 }, { "layer": 15, "head": 7, "slices": 4, "native_nodes": 33555, "mass": [ 0.08027507754686074, 0.5500052892457085, 0.28618483160289127, 0.08353480141780306 ], "mean_node_assignment_entropy": 0.4280436156266798, "normalized_node_entropy": 0.3087682007743963, "effective_mass_slots": 2.994301655437967, "node_slot_mutual_information_nats": 0.6686674186147763, "normalized_node_slot_mutual_information": 0.48234158441978114, "centered_token_effective_rank": 1.0364998967745391, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.18233093048900734, "token_RMS": 0.2851561728858121, "mean_token_offdiagonal_cosine": 0.4819646986582029, "temperature": 0.4690152406692505 }, { "layer": 15, "head": 8, "slices": 4, "native_nodes": 33555, "mass": [ 0.14234468642550901, 0.5561255822158854, 0.15484422267464729, 0.14668550868880162 ], "mean_node_assignment_entropy": 0.7525072128900402, "normalized_node_entropy": 0.5428192121348182, "effective_mass_slots": 3.235580810061405, "node_slot_mutual_information_nats": 0.42170123837920415, "normalized_node_slot_mutual_information": 0.30419314267320624, "centered_token_effective_rank": 1.0055266600435373, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.1655261764495373, "token_RMS": 0.27769158813299266, "mean_token_offdiagonal_cosine": 0.5194571713305637, "temperature": 0.44576194882392883 }, { "layer": 16, "head": 1, "slices": 4, "native_nodes": 33555, "mass": [ 0.43211585807210307, 0.004659750088872325, 0.559051053199427, 0.004173345723135783 ], "mean_node_assignment_entropy": 0.05534365622142275, "normalized_node_entropy": 0.039922009187655115, "effective_mass_slots": 2.086631891914011, "node_slot_mutual_information_nats": 0.6802075748666809, "normalized_node_slot_mutual_information": 0.4906660475176345, "centered_token_effective_rank": 1.0341262351578768, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.2675476509272149, "token_RMS": 0.37248448847200893, "mean_token_offdiagonal_cosine": 0.289055687147491, "temperature": 0.42034947872161865 }, { "layer": 16, "head": 2, "slices": 4, "native_nodes": 33555, "mass": [ 0.4033423409540126, 0.019651935738093863, 0.5570624343025333, 0.019943283916306634 ], "mean_node_assignment_entropy": 0.17339572713299783, "normalized_node_entropy": 0.12507857782305593, "effective_mass_slots": 2.333680408904624, "node_slot_mutual_information_nats": 0.6740508688659266, "normalized_node_slot_mutual_information": 0.48622492290988467, "centered_token_effective_rank": 1.0084773646755645, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.26906451703224976, "token_RMS": 0.35519031260008355, "mean_token_offdiagonal_cosine": 0.2576509461123661, "temperature": 0.4411679208278656 }, { "layer": 16, "head": 3, "slices": 4, "native_nodes": 33555, "mass": [ 0.2561312097737918, 0.10444636608319152, 0.5457584660155682, 0.09366395775113 ], "mean_node_assignment_entropy": 0.47466938418799576, "normalized_node_entropy": 0.3424015833149198, "effective_mass_slots": 3.1177769686852743, "node_slot_mutual_information_nats": 0.66245085364071, "normalized_node_slot_mutual_information": 0.4778572806900564, "centered_token_effective_rank": 1.0224555057630471, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.23938231606295027, "token_RMS": 0.33556182756815633, "mean_token_offdiagonal_cosine": 0.30112826197495246, "temperature": 0.461539089679718 }, { "layer": 16, "head": 4, "slices": 4, "native_nodes": 33555, "mass": [ 0.4260104210077936, 0.014919361970184137, 0.5450580610378837, 0.014012151694901847 ], "mean_node_assignment_entropy": 0.13126207639805684, "normalized_node_entropy": 0.09468557333813243, "effective_mass_slots": 2.263303410377061, "node_slot_mutual_information_nats": 0.6855633556705312, "normalized_node_slot_mutual_information": 0.49452942672053596, "centered_token_effective_rank": 1.0600760173646606, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.21616765846642386, "token_RMS": 0.29057181194981496, "mean_token_offdiagonal_cosine": 0.3069622163914389, "temperature": 0.4353942275047302 }, { "layer": 16, "head": 5, "slices": 4, "native_nodes": 33555, "mass": [ 0.5512828548311317, 0.03996146202857329, 0.36525549565409876, 0.04350018751751793 ], "mean_node_assignment_entropy": 0.295273220438276, "normalized_node_entropy": 0.21299460541680723, "effective_mass_slots": 2.6148458761065343, "node_slot_mutual_information_nats": 0.6659319369502924, "normalized_node_slot_mutual_information": 0.48036835150388435, "centered_token_effective_rank": 1.025084806210658, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.23661011283525782, "token_RMS": 0.30042412031260524, "mean_token_offdiagonal_cosine": 0.2118928793243644, "temperature": 0.4268903136253357 }, { "layer": 16, "head": 6, "slices": 4, "native_nodes": 33555, "mass": [ 0.41662998530414813, 0.010845029928968316, 0.5592651006443309, 0.013259879997304025 ], "mean_node_assignment_entropy": 0.11950637477791629, "normalized_node_entropy": 0.08620562712335886, "effective_mass_slots": 2.217046573598412, "node_slot_mutual_information_nats": 0.6766695631224837, "normalized_node_slot_mutual_information": 0.48811391151865435, "centered_token_effective_rank": 1.0208504345615594, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.2812864736268358, "token_RMS": 0.3969449624947642, "mean_token_offdiagonal_cosine": 0.22267578687601652, "temperature": 0.4557431638240814 }, { "layer": 16, "head": 7, "slices": 4, "native_nodes": 33555, "mass": [ 0.5687591491486714, 0.07841366852076194, 0.2627861700425322, 0.09004101242295284 ], "mean_node_assignment_entropy": 0.4440979741401049, "normalized_node_entropy": 0.32034897248038224, "effective_mass_slots": 2.969923046716539, "node_slot_mutual_information_nats": 0.644438068142377, "normalized_node_slot_mutual_information": 0.46486380253453563, "centered_token_effective_rank": 1.03393813526521, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.18381658032132378, "token_RMS": 0.2848783677384167, "mean_token_offdiagonal_cosine": 0.4098031422616532, "temperature": 0.45863592624664307 }, { "layer": 16, "head": 8, "slices": 4, "native_nodes": 33555, "mass": [ 0.3641101402489375, 0.043521842442930704, 0.5479358015258662, 0.044432215533205824 ], "mean_node_assignment_entropy": 0.29326450502589785, "normalized_node_entropy": 0.21154562353480966, "effective_mass_slots": 2.6439345653064206, "node_slot_mutual_information_nats": 0.6790036682860482, "normalized_node_slot_mutual_information": 0.48979761249084824, "centered_token_effective_rank": 1.0193039284353167, "centered_token_rank_upper_bound": 3, "centered_token_RMS": 0.24954408429060293, "token_RMS": 0.333394683577899, "mean_token_offdiagonal_cosine": 0.25952403881273983, "temperature": 0.45406410098075867 } ] }, { "schema": "ts3_capacity_slice_diagnostics_v1", "variant": "S008", "sample_id": "normal_rodv2_0150__gas_inertia_30deg", "native_nodes": 33555, "layers": 16, "slices": 8, "parameters": 7596804, "checkpoint_sha256": "90f0f2a5c763afc70a0541d53710f294086cea9d2829b701fb2a67cbba445e0b", "extraction_source_sha256": "26c5dec82b68ba4bc23d7c3d998b78c5bbed6aae912e6247ea64949db69bd206", "CPU_forward_seconds": 2.6894523539995134, "CPU_threads": 2, "CUDA_initialized": false, "hook_max_abs_prediction_difference_on_19_node_test": 0.0, "metrics": [ { "layer": 1, "head": 1, "slices": 8, "native_nodes": 33555, "mass": [ 0.12693902632839685, 0.12440053443229424, 0.12430659411290491, 0.12243358819495546, 0.13223752448050244, 0.12371219086444532, 0.11244304573224174, 0.13352749563421654 ], "mean_node_assignment_entropy": 2.077732655418441, "normalized_node_entropy": 0.9991781994217475, "effective_mass_slots": 7.990449367844712, "node_slot_mutual_information_nats": 0.0005143440604524763, "normalized_node_slot_mutual_information": 0.0002473472084418269, "centered_token_effective_rank": 1.9024930506045257, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.0028062636448927735, "token_RMS": 0.2491805842689873, "mean_token_offdiagonal_cosine": 0.9998559377624832, "temperature": 0.49945521354675293 }, { "layer": 1, "head": 2, "slices": 8, "native_nodes": 33555, "mass": [ 0.11159493785366502, 0.12099120040272764, 0.1292102431464064, 0.13109496476761792, 0.12787224882659926, 0.13175168850217273, 0.13094408932949494, 0.11654062715133244 ], "mean_node_assignment_entropy": 2.0772776677425955, "normalized_node_entropy": 0.9989593966005449, "effective_mass_slots": 7.98664115431915, "node_slot_mutual_information_nats": 0.000492622464056236, "normalized_node_slot_mutual_information": 0.00023690132864147777, "centered_token_effective_rank": 1.6440714677410266, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.0027275884798629983, "token_RMS": 0.23049363422716576, "mean_token_offdiagonal_cosine": 0.9998429755246807, "temperature": 0.5008018016815186 }, { "layer": 1, "head": 3, "slices": 8, "native_nodes": 33555, "mass": [ 0.12289287555974458, 0.1304744274949569, 0.11748762720455588, 0.12923921757861873, 0.12690926780978323, 0.12550333047935422, 0.12849190094275226, 0.11900135272084965 ], "mean_node_assignment_entropy": 2.0781345953410426, "normalized_node_entropy": 0.9993714916661051, "effective_mass_slots": 7.994820429009295, "node_slot_mutual_information_nats": 0.0006592902810407253, "normalized_node_slot_mutual_information": 0.00031705160632124847, "centered_token_effective_rank": 1.8270262333345653, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.003369946361497821, "token_RMS": 0.22770576007219034, "mean_token_offdiagonal_cosine": 0.9997514291110046, "temperature": 0.500562846660614 }, { "layer": 1, "head": 4, "slices": 8, "native_nodes": 33555, "mass": [ 0.12608680537495345, 0.13900252823177917, 0.13113865669904995, 0.12504307638091633, 0.11531210730526491, 0.11855396066816318, 0.1246852804384454, 0.12017758508216889 ], "mean_node_assignment_entropy": 2.0769095941605746, "normalized_node_entropy": 0.9987823906233902, "effective_mass_slots": 7.987538758461948, "node_slot_mutual_information_nats": 0.0009730779209080609, "normalized_node_slot_mutual_information": 0.0004679515636308675, "centered_token_effective_rank": 1.4452006106362834, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.004410834366188294, "token_RMS": 0.27614371625025436, "mean_token_offdiagonal_cosine": 0.9997139481535265, "temperature": 0.4989807903766632 }, { "layer": 1, "head": 5, "slices": 8, "native_nodes": 33555, "mass": [ 0.12378471577075142, 0.12650847951842037, 0.12156492251963325, 0.1272212741534306, 0.12000142249651059, 0.1271515074618313, 0.12793204839748093, 0.12583562988621913 ], "mean_node_assignment_entropy": 2.078234399903055, "normalized_node_entropy": 0.9994194875149961, "effective_mass_slots": 7.998089251481738, "node_slot_mutual_information_nats": 0.000968269684331613, "normalized_node_slot_mutual_information": 0.0004656392906094467, "centered_token_effective_rank": 1.5205209972646716, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.004183453199392223, "token_RMS": 0.33864237186097146, "mean_token_offdiagonal_cosine": 0.999832874076297, "temperature": 0.5004823207855225 }, { "layer": 1, "head": 6, "slices": 8, "native_nodes": 33555, "mass": [ 0.12584492989565005, 0.13852371814762174, 0.12044442333375407, 0.1296440763389846, 0.11811032486981525, 0.1233951579581215, 0.12143734142244209, 0.12260002790216255 ], "mean_node_assignment_entropy": 2.0775033596860877, "normalized_node_entropy": 0.9990679314830931, "effective_mass_slots": 7.990750270782259, "node_slot_mutual_information_nats": 0.0007812969079643572, "normalized_node_slot_mutual_information": 0.00037572439152735304, "centered_token_effective_rank": 1.9392114380891547, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.0037706599355107743, "token_RMS": 0.2438370730452003, "mean_token_offdiagonal_cosine": 0.9997324437001847, "temperature": 0.5005094408988953 }, { "layer": 1, "head": 7, "slices": 8, "native_nodes": 33555, "mass": [ 0.12311026712027734, 0.12791022810956756, 0.12636861594560658, 0.12692674611226187, 0.11273947122834724, 0.12280365834269362, 0.12440166055707635, 0.13573935257151307 ], "mean_node_assignment_entropy": 2.077552069051923, "normalized_node_entropy": 0.999091355736605, "effective_mass_slots": 7.990711905165583, "node_slot_mutual_information_nats": 0.0007277862772316901, "normalized_node_slot_mutual_information": 0.00034999121766306657, "centered_token_effective_rank": 1.641096051695453, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.0029735172086666557, "token_RMS": 0.2687263862110276, "mean_token_offdiagonal_cosine": 0.9998612412095483, "temperature": 0.497981458902359 }, { "layer": 1, "head": 8, "slices": 8, "native_nodes": 33555, "mass": [ 0.1325960823414827, 0.11803950058389857, 0.11468858065450267, 0.13063857885408714, 0.1286233230559326, 0.1354708454926791, 0.12282522940130317, 0.11711785966407487 ], "mean_node_assignment_entropy": 2.077019241808402, "normalized_node_entropy": 0.998835119995979, "effective_mass_slots": 7.986105568436427, "node_slot_mutual_information_nats": 0.0006839859333873655, "normalized_node_slot_mutual_information": 0.00032892770471192035, "centered_token_effective_rank": 1.9219587477143503, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.0035733116883737777, "token_RMS": 0.2097275715519198, "mean_token_offdiagonal_cosine": 0.999677141753948, "temperature": 0.5009063482284546 }, { "layer": 2, "head": 1, "slices": 8, "native_nodes": 33555, "mass": [ 0.1222628496410754, 0.12899135330064945, 0.116772879021096, 0.12807052577439834, 0.12782181676290125, 0.12723539508440654, 0.12876875436669258, 0.12007642589779267 ], "mean_node_assignment_entropy": 2.0777946835903505, "normalized_node_entropy": 0.9992080286670839, "effective_mass_slots": 7.995093199190805, "node_slot_mutual_information_nats": 0.0010333198122136622, "normalized_node_slot_mutual_information": 0.0004969217895776552, "centered_token_effective_rank": 1.2991740415936783, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.0054005153710487835, "token_RMS": 0.2206725968659773, "mean_token_offdiagonal_cosine": 0.9993186754264372, "temperature": 0.5021759271621704 }, { "layer": 2, "head": 2, "slices": 8, "native_nodes": 33555, "mass": [ 0.12004994187637832, 0.134583460151563, 0.11783262205992526, 0.13369690645821952, 0.12168451334041591, 0.1223839557638486, 0.1297548541483057, 0.12001374605368652 ], "mean_node_assignment_entropy": 2.0773890095644574, "normalized_node_entropy": 0.9990129406986261, "effective_mass_slots": 7.990206065245122, "node_slot_mutual_information_nats": 0.0008275402747917049, "normalized_node_slot_mutual_information": 0.0003979627501926276, "centered_token_effective_rank": 1.8376819565392175, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.004889781496501351, "token_RMS": 0.2544660107502035, "mean_token_offdiagonal_cosine": 0.9995839145333802, "temperature": 0.5005964636802673 }, { "layer": 2, "head": 3, "slices": 8, "native_nodes": 33555, "mass": [ 0.1095310168250043, 0.13233251968397858, 0.14990565341735912, 0.1194366102612706, 0.1337874647593502, 0.1147468353040331, 0.11416361707099912, 0.12609628291803116 ], "mean_node_assignment_entropy": 2.0740647038248863, "normalized_node_entropy": 0.9974142875636668, "effective_mass_slots": 7.960974789188174, "node_slot_mutual_information_nats": 0.000486749486963145, "normalized_node_slot_mutual_information": 0.00023407702366565883, "centered_token_effective_rank": 2.017201511415138, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.0033740930508868953, "token_RMS": 0.270011101132689, "mean_token_offdiagonal_cosine": 0.9998265224282329, "temperature": 0.4980795979499817 }, { "layer": 2, "head": 4, "slices": 8, "native_nodes": 33555, "mass": [ 0.13461014972018548, 0.12876905544653075, 0.11581608026957764, 0.12789682194561702, 0.1237709974151506, 0.12318280369838257, 0.11509306511062212, 0.13086102648563666 ], "mean_node_assignment_entropy": 2.0769006248099524, "normalized_node_entropy": 0.9987780772775028, "effective_mass_slots": 7.989176414105161, "node_slot_mutual_information_nats": 0.0011870525722144087, "normalized_node_slot_mutual_information": 0.000570851619736072, "centered_token_effective_rank": 1.343564997472566, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.00624675406713497, "token_RMS": 0.23706309083193433, "mean_token_offdiagonal_cosine": 0.9992717468495409, "temperature": 0.4990162253379822 }, { "layer": 2, "head": 5, "slices": 8, "native_nodes": 33555, "mass": [ 0.13588331999606088, 0.12627615259973893, 0.12765341692928442, 0.1357204697105821, 0.13288518712868228, 0.10188133904563032, 0.11229118948419053, 0.12740892520774727 ], "mean_node_assignment_entropy": 2.0747721023537578, "normalized_node_entropy": 0.9977544743468452, "effective_mass_slots": 7.9667777477465265, "node_slot_mutual_information_nats": 0.0005080110501731738, "normalized_node_slot_mutual_information": 0.0002443016742672108, "centered_token_effective_rank": 2.010085765782335, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.0026922799052322846, "token_RMS": 0.21310624436716658, "mean_token_offdiagonal_cosine": 0.9998258085505743, "temperature": 0.5009616613388062 }, { "layer": 2, "head": 6, "slices": 8, "native_nodes": 33555, "mass": [ 0.1264562542990323, 0.1228097599425502, 0.1322486374145861, 0.1146962108843516, 0.13806206228622575, 0.1357528856597844, 0.10773938313790467, 0.12223480638222622 ], "mean_node_assignment_entropy": 2.07489197208968, "normalized_node_entropy": 0.997812119504701, "effective_mass_slots": 7.97554161271771, "node_slot_mutual_information_nats": 0.0014875880955917964, "normalized_node_slot_mutual_information": 0.000715378656131914, "centered_token_effective_rank": 1.356133370437805, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.008251963014174331, "token_RMS": 0.22747389490484302, "mean_token_offdiagonal_cosine": 0.998544356029125, "temperature": 0.49885374307632446 }, { "layer": 2, "head": 7, "slices": 8, "native_nodes": 33555, "mass": [ 0.11726252267731406, 0.131548075670813, 0.12332917662397651, 0.13171748017220292, 0.11520229516290834, 0.13338049735744598, 0.11969398244440493, 0.12786596970419792 ], "mean_node_assignment_entropy": 2.0766082181872942, "normalized_node_entropy": 0.9986374594160254, "effective_mass_slots": 7.988665149479064, "node_slot_mutual_information_nats": 0.0014154624874027633, "normalized_node_slot_mutual_information": 0.0006806935703801078, "centered_token_effective_rank": 1.2633182449707574, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.008157376692102218, "token_RMS": 0.20492527501752236, "mean_token_offdiagonal_cosine": 0.9982682985415176, "temperature": 0.4988676905632019 }, { "layer": 2, "head": 8, "slices": 8, "native_nodes": 33555, "mass": [ 0.12435850373475815, 0.13070988683200704, 0.13141181784195013, 0.13382003096026257, 0.120533412759989, 0.11189769529968593, 0.12417432954944276, 0.12309432315779344 ], "mean_node_assignment_entropy": 2.077398918619538, "normalized_node_entropy": 0.9990177059468343, "effective_mass_slots": 7.988759563120025, "node_slot_mutual_information_nats": 0.0006365804354877547, "normalized_node_slot_mutual_information": 0.0003061304791350401, "centered_token_effective_rank": 1.4008311910645055, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.00391598907247003, "token_RMS": 0.2376643677532529, "mean_token_offdiagonal_cosine": 0.9997303067904681, "temperature": 0.5010787844657898 }, { "layer": 3, "head": 1, "slices": 8, "native_nodes": 33555, "mass": [ 0.1231469514226171, 0.13079321821273093, 0.11927758374742274, 0.12987013758450075, 0.12055428629038036, 0.10916070260552709, 0.13999028226765625, 0.12720683748592226 ], "mean_node_assignment_entropy": 2.0758435810122844, "normalized_node_entropy": 0.9982697466625367, "effective_mass_slots": 7.980936033189512, "node_slot_mutual_information_nats": 0.0012121209628013219, "normalized_node_slot_mutual_information": 0.0005829069673303409, "centered_token_effective_rank": 1.248482935033934, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.006670590116372056, "token_RMS": 0.27786851601916635, "mean_token_offdiagonal_cosine": 0.999452542889199, "temperature": 0.49972817301750183 }, { "layer": 3, "head": 2, "slices": 8, "native_nodes": 33555, "mass": [ 0.12266087229712641, 0.1317844550314452, 0.12530881901590965, 0.12031840079766133, 0.1337538724281965, 0.11272793949658426, 0.12054447913388348, 0.13290116181873277 ], "mean_node_assignment_entropy": 2.0771931628018105, "normalized_node_entropy": 0.9989187583142112, "effective_mass_slots": 7.987691042003741, "node_slot_mutual_information_nats": 0.0007085742375725346, "normalized_node_slot_mutual_information": 0.0003407521795491913, "centered_token_effective_rank": 1.258968528160236, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.005396137438801256, "token_RMS": 0.22157701386045198, "mean_token_offdiagonal_cosine": 0.9993474382412708, "temperature": 0.49908411502838135 }, { "layer": 3, "head": 3, "slices": 8, "native_nodes": 33555, "mass": [ 0.1158542152504739, 0.1213573182640355, 0.13364889284188125, 0.12989839088932764, 0.12172419065277901, 0.11354962692892581, 0.14427635071063927, 0.11969101468042581 ], "mean_node_assignment_entropy": 2.0748538083596664, "normalized_node_entropy": 0.9977937666300235, "effective_mass_slots": 7.976848303025241, "node_slot_mutual_information_nats": 0.0016895755933736822, "normalized_node_slot_mutual_information": 0.0008125141099224132, "centered_token_effective_rank": 1.6270056995189988, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.007505050615699642, "token_RMS": 0.24254524465156982, "mean_token_offdiagonal_cosine": 0.9989619669088136, "temperature": 0.4973592460155487 }, { "layer": 3, "head": 4, "slices": 8, "native_nodes": 33555, "mass": [ 0.11675888078789334, 0.11543942224498016, 0.12898368174899905, 0.11856871222933685, 0.1319887855025948, 0.14103654751970734, 0.13257708326148623, 0.11464688674585774 ], "mean_node_assignment_entropy": 2.075015054619817, "normalized_node_entropy": 0.997871309689984, "effective_mass_slots": 7.978192050724799, "node_slot_mutual_information_nats": 0.001696771113743889, "normalized_node_slot_mutual_information": 0.000815974423773984, "centered_token_effective_rank": 1.1737979188425312, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.009185963962811797, "token_RMS": 0.2344379375694248, "mean_token_offdiagonal_cosine": 0.9983784714560657, "temperature": 0.499014675617218 }, { "layer": 3, "head": 5, "slices": 8, "native_nodes": 33555, "mass": [ 0.12014472469401029, 0.12380259890143162, 0.12524632043373815, 0.12727797898383275, 0.12925926587755182, 0.12708647209363266, 0.12282765336837628, 0.12435498557837171 ], "mean_node_assignment_entropy": 2.0776405291941638, "normalized_node_entropy": 0.9991338960727807, "effective_mass_slots": 7.998145064152041, "node_slot_mutual_information_nats": 0.0015691186193729223, "normalized_node_slot_mutual_information": 0.000754586550245284, "centered_token_effective_rank": 1.1816582689675705, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.0070513638113684045, "token_RMS": 0.2550315128043259, "mean_token_offdiagonal_cosine": 0.9991368547076895, "temperature": 0.49947041273117065 }, { "layer": 3, "head": 6, "slices": 8, "native_nodes": 33555, "mass": [ 0.12337485067851674, 0.1270242346734621, 0.12433232209101072, 0.13309040323483787, 0.120690809221359, 0.11748488675666946, 0.1320945017074192, 0.12190799136383668 ], "mean_node_assignment_entropy": 2.0778789267228746, "normalized_node_entropy": 0.9992485410502577, "effective_mass_slots": 7.993387122629134, "node_slot_mutual_information_nats": 0.0007356634554422925, "normalized_node_slot_mutual_information": 0.0003537793396432781, "centered_token_effective_rank": 1.3907823184118704, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.006381972123568442, "token_RMS": 0.2656444776532458, "mean_token_offdiagonal_cosine": 0.9993587321396398, "temperature": 0.5007231831550598 }, { "layer": 3, "head": 7, "slices": 8, "native_nodes": 33555, "mass": [ 0.12372984317589439, 0.12729290240861818, 0.11860707990948409, 0.12480084808026247, 0.132156965228897, 0.114509908591843, 0.1256251803434849, 0.1332772722850523 ], "mean_node_assignment_entropy": 2.077738614231206, "normalized_node_entropy": 0.9991810650049561, "effective_mass_slots": 7.991063048273548, "node_slot_mutual_information_nats": 0.0005851840403372499, "normalized_node_slot_mutual_information": 0.00028141403766730587, "centered_token_effective_rank": 1.4073727637116222, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.004263148142429237, "token_RMS": 0.25839796116191577, "mean_token_offdiagonal_cosine": 0.9996955877689973, "temperature": 0.5005813837051392 }, { "layer": 3, "head": 8, "slices": 8, "native_nodes": 33555, "mass": [ 0.11483324833104373, 0.11727388185891499, 0.13643301758160803, 0.12282166065424215, 0.12949338462145454, 0.13040396989373296, 0.11620594392304555, 0.13253489296210005 ], "mean_node_assignment_entropy": 2.0762313307593185, "normalized_node_entropy": 0.9984562148749208, "effective_mass_slots": 7.984573352431383, "node_slot_mutual_information_nats": 0.0012800183507186702, "normalized_node_slot_mutual_information": 0.0006155587089428985, "centered_token_effective_rank": 1.363839289005122, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.006766553621205362, "token_RMS": 0.2333935293646008, "mean_token_offdiagonal_cosine": 0.9990665041814907, "temperature": 0.5001466870307922 }, { "layer": 4, "head": 1, "slices": 8, "native_nodes": 33555, "mass": [ 0.12857752727904453, 0.1299867719596144, 0.11992952663547465, 0.1292861155986857, 0.12478235052925449, 0.11789131292535125, 0.12660706674838595, 0.12293932848072783 ], "mean_node_assignment_entropy": 2.0776435760341054, "normalized_node_entropy": 0.999135361293072, "effective_mass_slots": 7.9955257524229015, "node_slot_mutual_information_nats": 0.001238528242666348, "normalized_node_slot_mutual_information": 0.000595606184565221, "centered_token_effective_rank": 1.6131490466118121, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.006045051155964301, "token_RMS": 0.25823540321769867, "mean_token_offdiagonal_cosine": 0.9994172788583898, "temperature": 0.4966502785682678 }, { "layer": 4, "head": 2, "slices": 8, "native_nodes": 33555, "mass": [ 0.11982314988888601, 0.11666080264030172, 0.12125162017974248, 0.13651408620868136, 0.11912540899665429, 0.12929331946103872, 0.12264872766068646, 0.1346828850279567 ], "mean_node_assignment_entropy": 2.0761781152349945, "normalized_node_entropy": 0.9984306236172406, "effective_mass_slots": 7.987482624786726, "node_slot_mutual_information_nats": 0.001697529165726408, "normalized_node_slot_mutual_information": 0.0008163389697192895, "centered_token_effective_rank": 1.2985632804879395, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.00720103975924304, "token_RMS": 0.20193252314732796, "mean_token_offdiagonal_cosine": 0.9985567602516703, "temperature": 0.49690544605255127 }, { "layer": 4, "head": 3, "slices": 8, "native_nodes": 33555, "mass": [ 0.11149039158983107, 0.13590276756237415, 0.12224589367037007, 0.1138397502312538, 0.1160786711477768, 0.1319953027498152, 0.1330629529243005, 0.13538427013937718 ], "mean_node_assignment_entropy": 2.0755803079143087, "normalized_node_entropy": 0.9981431390649204, "effective_mass_slots": 7.976376745997994, "node_slot_mutual_information_nats": 0.0009039585843062525, "normalized_node_slot_mutual_information": 0.00043471218891587954, "centered_token_effective_rank": 1.5885601553639719, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.005532423139525553, "token_RMS": 0.2657083629087871, "mean_token_offdiagonal_cosine": 0.9995577984930986, "temperature": 0.5021750330924988 }, { "layer": 4, "head": 4, "slices": 8, "native_nodes": 33555, "mass": [ 0.11267028457760331, 0.14060579235748072, 0.12700034754919484, 0.11766847787049395, 0.11121352641482304, 0.126579105677734, 0.14359230958830135, 0.12067015606739574 ], "mean_node_assignment_entropy": 2.070469077469458, "normalized_node_entropy": 0.9956851567930448, "effective_mass_slots": 7.968104833474949, "node_slot_mutual_information_nats": 0.004977599537903998, "normalized_node_slot_mutual_information": 0.002393719389621765, "centered_token_effective_rank": 1.0584375900882814, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.013611641195413944, "token_RMS": 0.2328748608461083, "mean_token_offdiagonal_cosine": 0.9961126822717061, "temperature": 0.49191704392433167 }, { "layer": 4, "head": 5, "slices": 8, "native_nodes": 33555, "mass": [ 0.11975536204595855, 0.12323796222878185, 0.12595935730531982, 0.11714811124192888, 0.12210473476721338, 0.13576604568435155, 0.12972431688736988, 0.12630410976336018 ], "mean_node_assignment_entropy": 2.07582004810655, "normalized_node_entropy": 0.9982584297270698, "effective_mass_slots": 7.992329301075251, "node_slot_mutual_information_nats": 0.0026621962290920287, "normalized_node_slot_mutual_information": 0.0012802457658614562, "centered_token_effective_rank": 1.0948559020521214, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.010636567564637735, "token_RMS": 0.22969839255301466, "mean_token_offdiagonal_cosine": 0.9980602333992667, "temperature": 0.4995660185813904 }, { "layer": 4, "head": 6, "slices": 8, "native_nodes": 33555, "mass": [ 0.11047445274695183, 0.12891513175908223, 0.1375411507705341, 0.1267683865620436, 0.11938761667698297, 0.12075382031421515, 0.13687309335710182, 0.11928634741297069 ], "mean_node_assignment_entropy": 2.0762564379205104, "normalized_node_entropy": 0.9984682888672348, "effective_mass_slots": 7.980493923340915, "node_slot_mutual_information_nats": 0.0007438667812840194, "normalized_node_slot_mutual_information": 0.0003577243054801634, "centered_token_effective_rank": 1.2793187238214472, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.0063788513259140376, "token_RMS": 0.22523369777425784, "mean_token_offdiagonal_cosine": 0.9991058976528803, "temperature": 0.5011121034622192 }, { "layer": 4, "head": 7, "slices": 8, "native_nodes": 33555, "mass": [ 0.12295890067235099, 0.11782456977816634, 0.11756310022888906, 0.1351931865869256, 0.11985548587895918, 0.13550030971124974, 0.12786334615904071, 0.12324110094911384 ], "mean_node_assignment_entropy": 2.076951960067351, "normalized_node_entropy": 0.9988027643179267, "effective_mass_slots": 7.988498205956329, "node_slot_mutual_information_nats": 0.0010508228398227182, "normalized_node_slot_mutual_information": 0.0005053389666216978, "centered_token_effective_rank": 1.5102400011700345, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.006167371738437753, "token_RMS": 0.2981348857799369, "mean_token_offdiagonal_cosine": 0.999533060534351, "temperature": 0.5009337663650513 }, { "layer": 4, "head": 8, "slices": 8, "native_nodes": 33555, "mass": [ 0.12342765970035417, 0.14013711669293694, 0.11746041985621948, 0.1240846434840412, 0.11703846837130127, 0.11368420078602194, 0.14417018281100777, 0.11999730833164539 ], "mean_node_assignment_entropy": 2.0749351412852923, "normalized_node_entropy": 0.9978328794995107, "effective_mass_slots": 7.972717926930957, "node_slot_mutual_information_nats": 0.0010903130730408606, "normalized_node_slot_mutual_information": 0.0005243297544974852, "centered_token_effective_rank": 1.450566686609337, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.005737932015255425, "token_RMS": 0.236165071949579, "mean_token_offdiagonal_cosine": 0.9994053480152412, "temperature": 0.499461829662323 }, { "layer": 5, "head": 1, "slices": 8, "native_nodes": 33555, "mass": [ 0.1209726202185526, 0.11361429867370884, 0.1201114096476875, 0.1193089651097361, 0.1379178820954778, 0.11705483531271571, 0.13262837024283755, 0.13839161879742595 ], "mean_node_assignment_entropy": 2.0756787638286958, "normalized_node_entropy": 0.9981904863513978, "effective_mass_slots": 7.978797080348952, "node_slot_mutual_information_nats": 0.0011088944594246364, "normalized_node_slot_mutual_information": 0.0005332655124937237, "centered_token_effective_rank": 1.2826704681252554, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.00610699938632878, "token_RMS": 0.2588626660303623, "mean_token_offdiagonal_cosine": 0.9993755072947056, "temperature": 0.499160498380661 }, { "layer": 5, "head": 2, "slices": 8, "native_nodes": 33555, "mass": [ 0.1214709558566518, 0.13245318154120939, 0.11575596484736844, 0.12448080511006811, 0.12452085794657003, 0.12457814634805499, 0.12926375283056854, 0.12747633585345813 ], "mean_node_assignment_entropy": 2.077508312994509, "normalized_node_entropy": 0.9990703135209249, "effective_mass_slots": 7.994260069543471, "node_slot_mutual_information_nats": 0.0012154798581867077, "normalized_node_slot_mutual_information": 0.0005845222545687946, "centered_token_effective_rank": 1.1419080184534958, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.006549503658804712, "token_RMS": 0.2529832667901814, "mean_token_offdiagonal_cosine": 0.9992368692237782, "temperature": 0.5004388093948364 }, { "layer": 5, "head": 3, "slices": 8, "native_nodes": 33555, "mass": [ 0.11908203275223955, 0.12207620550447171, 0.13681278573522587, 0.12715307884689878, 0.12678095769121045, 0.12714695233516946, 0.12024531522978082, 0.12070267223251362 ], "mean_node_assignment_entropy": 2.076592944034622, "normalized_node_entropy": 0.9986301141012539, "effective_mass_slots": 7.99253218092396, "node_slot_mutual_information_nats": 0.001914684299367586, "normalized_node_slot_mutual_information": 0.0009207685145218587, "centered_token_effective_rank": 1.0924330365709216, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.00936897363982317, "token_RMS": 0.2043674694983765, "mean_token_offdiagonal_cosine": 0.9977298236916076, "temperature": 0.4997388422489166 }, { "layer": 5, "head": 4, "slices": 8, "native_nodes": 33555, "mass": [ 0.12479707984571005, 0.12450227327184091, 0.1278430262659616, 0.12910558771552422, 0.12288590040923301, 0.12868082829895539, 0.118918575336256, 0.12326672892646168 ], "mean_node_assignment_entropy": 2.078265733812678, "normalized_node_entropy": 0.9994345559403378, "effective_mass_slots": 7.997327974946369, "node_slot_mutual_information_nats": 0.0008417489439844772, "normalized_node_slot_mutual_information": 0.00040479567571997574, "centered_token_effective_rank": 1.1064939878661424, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.006480395826895947, "token_RMS": 0.2742324474538729, "mean_token_offdiagonal_cosine": 0.9993767300083266, "temperature": 0.5014272928237915 }, { "layer": 5, "head": 5, "slices": 8, "native_nodes": 33555, "mass": [ 0.12433948962481682, 0.11857002841397807, 0.12171206042678998, 0.13722924377639892, 0.11430391820638622, 0.12753657316676623, 0.12555320461901942, 0.13075548163484024 ], "mean_node_assignment_entropy": 2.0770100586433333, "normalized_node_entropy": 0.9988307038270774, "effective_mass_slots": 7.988633348663897, "node_slot_mutual_information_nats": 0.001009641281405127, "normalized_node_slot_mutual_information": 0.0004855348232533184, "centered_token_effective_rank": 1.579739917964827, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.005975055858540138, "token_RMS": 0.23226353762115773, "mean_token_offdiagonal_cosine": 0.9993018715867367, "temperature": 0.4997410476207733 }, { "layer": 5, "head": 6, "slices": 8, "native_nodes": 33555, "mass": [ 0.10629380168622234, 0.12715708588416097, 0.13897776756684874, 0.13594337952059724, 0.13423356138643822, 0.12205576385995738, 0.11744021466722113, 0.11789842560862912 ], "mean_node_assignment_entropy": 2.0724168076442275, "normalized_node_entropy": 0.9966218170144213, "effective_mass_slots": 7.971887894474179, "node_slot_mutual_information_nats": 0.0035045321982845046, "normalized_node_slot_mutual_information": 0.0016853237410335842, "centered_token_effective_rank": 1.1367008287672764, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.014073765947295706, "token_RMS": 0.2126618560859741, "mean_token_offdiagonal_cosine": 0.9953797451189667, "temperature": 0.4964098334312439 }, { "layer": 5, "head": 7, "slices": 8, "native_nodes": 33555, "mass": [ 0.12369852269540475, 0.12385510858978215, 0.11714842207824183, 0.11671771115676566, 0.1355097852519631, 0.13213360354186632, 0.1304758881919964, 0.12046095839161895 ], "mean_node_assignment_entropy": 2.0768488433063146, "normalized_node_entropy": 0.998753175638, "effective_mass_slots": 7.988999476690389, "node_slot_mutual_information_nats": 0.001216686689819113, "normalized_node_slot_mutual_information": 0.000585102617905881, "centered_token_effective_rank": 1.500441873367683, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.00635525076820583, "token_RMS": 0.2194775122580083, "mean_token_offdiagonal_cosine": 0.999068941098549, "temperature": 0.5012567639350891 }, { "layer": 5, "head": 8, "slices": 8, "native_nodes": 33555, "mass": [ 0.1290721573625018, 0.11462326496284421, 0.12480900629349036, 0.12998576529280756, 0.12249824911036233, 0.12000362359274874, 0.12148244406552237, 0.13752548922269076 ], "mean_node_assignment_entropy": 2.076867033320409, "normalized_node_entropy": 0.9987619231857093, "effective_mass_slots": 7.9888796594796645, "node_slot_mutual_information_nats": 0.0011834987889955961, "normalized_node_slot_mutual_information": 0.0005691426112606801, "centered_token_effective_rank": 1.2255162540071423, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.0061539380554903, "token_RMS": 0.20285673560620882, "mean_token_offdiagonal_cosine": 0.9990377360263842, "temperature": 0.4990001916885376 }, { "layer": 6, "head": 1, "slices": 8, "native_nodes": 33555, "mass": [ 0.11935980625127542, 0.13055151737810086, 0.12045652015969288, 0.13401332936083005, 0.12604223776038614, 0.13217506904242168, 0.1264923467514963, 0.11090917359377549 ], "mean_node_assignment_entropy": 2.076742771047752, "normalized_node_entropy": 0.9987021656641988, "effective_mass_slots": 7.986468546225012, "node_slot_mutual_information_nats": 0.0010059068244974156, "normalized_node_slot_mutual_information": 0.0004837389290995955, "centered_token_effective_rank": 1.1498039264740076, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.006231646700892927, "token_RMS": 0.2551093130813718, "mean_token_offdiagonal_cosine": 0.9993272393471122, "temperature": 0.4998006224632263 }, { "layer": 6, "head": 2, "slices": 8, "native_nodes": 33555, "mass": [ 0.12773503615659532, 0.12420523391828116, 0.12486989807219286, 0.11835170424170154, 0.12432346697172311, 0.11976716238008919, 0.1240594382491504, 0.13668806011573584 ], "mean_node_assignment_entropy": 2.0732647132264743, "normalized_node_entropy": 0.9970295734073046, "effective_mass_slots": 7.993131237512542, "node_slot_mutual_information_nats": 0.005317864338358014, "normalized_node_slot_mutual_information": 0.002557352169689792, "centered_token_effective_rank": 1.1023818858003862, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.017278049798751242, "token_RMS": 0.3263766267684143, "mean_token_offdiagonal_cosine": 0.9969115412432367, "temperature": 0.4957628846168518 }, { "layer": 6, "head": 3, "slices": 8, "native_nodes": 33555, "mass": [ 0.13398027457346806, 0.13713824508855746, 0.11967330598214791, 0.12560023751446786, 0.11802235818303586, 0.11495762955674363, 0.12728994963705903, 0.12333799915233075 ], "mean_node_assignment_entropy": 2.0729200891110384, "normalized_node_entropy": 0.9968638442398678, "effective_mass_slots": 7.986827403925801, "node_slot_mutual_information_nats": 0.0048735209658068435, "normalized_node_slot_mutual_information": 0.002343668176345975, "centered_token_effective_rank": 1.1263704155887229, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.017267643912427723, "token_RMS": 0.2600675858911649, "mean_token_offdiagonal_cosine": 0.995041875579467, "temperature": 0.49488455057144165 }, { "layer": 6, "head": 4, "slices": 8, "native_nodes": 33555, "mass": [ 0.14473643004427972, 0.1161837589907728, 0.12234917147524434, 0.12284284790814358, 0.11519043547259662, 0.12403781371488956, 0.1253617859236655, 0.12929775620218253 ], "mean_node_assignment_entropy": 2.0758699422009226, "normalized_node_entropy": 0.9982824237145768, "effective_mass_slots": 7.981459803559846, "node_slot_mutual_information_nats": 0.0012513853075577508, "normalized_node_slot_mutual_information": 0.0006017891258182925, "centered_token_effective_rank": 1.188219100917642, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.006826537734033062, "token_RMS": 0.19176776427785408, "mean_token_offdiagonal_cosine": 0.9985773817710254, "temperature": 0.49924197793006897 }, { "layer": 6, "head": 5, "slices": 8, "native_nodes": 33555, "mass": [ 0.1259720769180855, 0.12869543867634511, 0.12092497424923207, 0.11898636248375154, 0.12454438560868923, 0.13226429669006343, 0.12543527930708512, 0.1231771864673097 ], "mean_node_assignment_entropy": 2.076396100635347, "normalized_node_entropy": 0.9985354524359321, "effective_mass_slots": 7.996048146265003, "node_slot_mutual_information_nats": 0.002551337278450916, "normalized_node_slot_mutual_information": 0.0012269338797520936, "centered_token_effective_rank": 1.0850329875402482, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.012617437694957875, "token_RMS": 0.23230212690709434, "mean_token_offdiagonal_cosine": 0.9966513343645171, "temperature": 0.4971904754638672 }, { "layer": 6, "head": 6, "slices": 8, "native_nodes": 33555, "mass": [ 0.13003319326575583, 0.12417835568110752, 0.12065109004186209, 0.12388738224011138, 0.12037182631923907, 0.12325798990944604, 0.1247459071327954, 0.1328742553179798 ], "mean_node_assignment_entropy": 2.0688845764956723, "normalized_node_entropy": 0.99492317289399, "effective_mass_slots": 7.995789959751787, "node_slot_mutual_information_nats": 0.010030571632357876, "normalized_node_slot_mutual_information": 0.004823685317094741, "centered_token_effective_rank": 1.040160253338806, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.019814352585905932, "token_RMS": 0.278773372945944, "mean_token_offdiagonal_cosine": 0.994252097114005, "temperature": 0.48860764503479004 }, { "layer": 6, "head": 7, "slices": 8, "native_nodes": 33555, "mass": [ 0.12657032931411413, 0.12517048899506764, 0.1185571146025119, 0.13421823960594192, 0.13044989694472203, 0.1303840178856994, 0.11917660201142964, 0.11547331060609703 ], "mean_node_assignment_entropy": 2.075933190848663, "normalized_node_entropy": 0.9983128398847229, "effective_mass_slots": 7.989985333516212, "node_slot_mutual_information_nats": 0.0022557333228556686, "normalized_node_slot_mutual_information": 0.001084778426150619, "centered_token_effective_rank": 1.1196316596627167, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.009761969883095042, "token_RMS": 0.2721533778313987, "mean_token_offdiagonal_cosine": 0.9985949287886624, "temperature": 0.5008766651153564 }, { "layer": 6, "head": 8, "slices": 8, "native_nodes": 33555, "mass": [ 0.1265039825471427, 0.11467041238807137, 0.13218037879155264, 0.11828427642090916, 0.12482392616850878, 0.1212529685971652, 0.12549152342826594, 0.13679253168813768 ], "mean_node_assignment_entropy": 2.0753452775070724, "normalized_node_entropy": 0.9980301133305944, "effective_mass_slots": 7.988555703410113, "node_slot_mutual_information_nats": 0.002664702903985461, "normalized_node_slot_mutual_information": 0.0012814512216740815, "centered_token_effective_rank": 1.0463706420569716, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.010423449466435471, "token_RMS": 0.23461369516411198, "mean_token_offdiagonal_cosine": 0.9977469114684292, "temperature": 0.49978500604629517 }, { "layer": 7, "head": 1, "slices": 8, "native_nodes": 33555, "mass": [ 0.14951477889407322, 0.12825291848381626, 0.11967559955739936, 0.12185121663586426, 0.1244321539065483, 0.11582004400965194, 0.11275862580732304, 0.1276946625758738 ], "mean_node_assignment_entropy": 1.9807274574172635, "normalized_node_entropy": 0.9525285600561639, "effective_mass_slots": 7.972421001057799, "node_slot_mutual_information_nats": 0.09526075350599728, "normalized_node_slot_mutual_information": 0.04581073889148274, "centered_token_effective_rank": 1.0347390494710573, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.07023377743343522, "token_RMS": 0.2737363184398099, "mean_token_offdiagonal_cosine": 0.9246447685966415, "temperature": 0.4817546010017395 }, { "layer": 7, "head": 2, "slices": 8, "native_nodes": 33555, "mass": [ 0.11331607950566831, 0.11443511071090211, 0.12710224398667297, 0.12457076411318484, 0.1263698434309248, 0.13529459297559315, 0.1345069775907671, 0.12440438735078298 ], "mean_node_assignment_entropy": 2.06727917781452, "normalized_node_entropy": 0.9941511393220073, "effective_mass_slots": 7.985483841122576, "node_slot_mutual_information_nats": 0.010346195770338351, "normalized_node_slot_mutual_information": 0.004975468443311169, "centered_token_effective_rank": 1.038773582291777, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.023448920073679005, "token_RMS": 0.21414962131915574, "mean_token_offdiagonal_cosine": 0.9864380177047762, "temperature": 0.4924135208129883 }, { "layer": 7, "head": 3, "slices": 8, "native_nodes": 33555, "mass": [ 0.11371970669893905, 0.1284766339210813, 0.12888899222138456, 0.11986253909903229, 0.12856196475142675, 0.1304201317615524, 0.12864784855740274, 0.12142218307022579 ], "mean_node_assignment_entropy": 2.076584603307537, "normalized_node_entropy": 0.9986261030593863, "effective_mass_slots": 7.9919209108055185, "node_slot_mutual_information_nats": 0.0018465419443938202, "normalized_node_slot_mutual_information": 0.0008879989686568096, "centered_token_effective_rank": 1.2697328348729844, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.008157879988338553, "token_RMS": 0.2093324351859815, "mean_token_offdiagonal_cosine": 0.9984175422904608, "temperature": 0.500234842300415 }, { "layer": 7, "head": 4, "slices": 8, "native_nodes": 33555, "mass": [ 0.12776522009769897, 0.12523152830614595, 0.12406621418350447, 0.1399663955235229, 0.11581004358203023, 0.12248491530885748, 0.11220993717382712, 0.13246574606621536 ], "mean_node_assignment_entropy": 2.0764369977261232, "normalized_node_entropy": 0.998555119779282, "effective_mass_slots": 7.982701181019401, "node_slot_mutual_information_nats": 0.0008398503215518716, "normalized_node_slot_mutual_information": 0.00040388263133062883, "centered_token_effective_rank": 1.7344361514576798, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.004515895766313129, "token_RMS": 0.2844138165292205, "mean_token_offdiagonal_cosine": 0.9997349236408272, "temperature": 0.500999927520752 }, { "layer": 7, "head": 5, "slices": 8, "native_nodes": 33555, "mass": [ 0.12489692951421982, 0.1252178452534854, 0.11744050057345398, 0.1281141509066181, 0.11517342621208818, 0.1236405082430006, 0.13159165375368764, 0.1339249855674267 ], "mean_node_assignment_entropy": 2.0679006641774778, "normalized_node_entropy": 0.9944500110866137, "effective_mass_slots": 7.990738421296555, "node_slot_mutual_information_nats": 0.010382509515208582, "normalized_node_slot_mutual_information": 0.0049929316631912996, "centered_token_effective_rank": 1.0559603097907329, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.02471526554472225, "token_RMS": 0.24844352580227533, "mean_token_offdiagonal_cosine": 0.9888337133733399, "temperature": 0.49490007758140564 }, { "layer": 7, "head": 6, "slices": 8, "native_nodes": 33555, "mass": [ 0.12332923614446781, 0.1303672622675566, 0.12730835584645797, 0.12206552497894926, 0.1285406448434405, 0.12468185136862119, 0.12290097333722881, 0.12080615093461662 ], "mean_node_assignment_entropy": 2.0764154543206423, "normalized_node_entropy": 0.9985447595911983, "effective_mass_slots": 7.9974428668779405, "node_slot_mutual_information_nats": 0.0027063946226584257, "normalized_node_slot_mutual_information": 0.0013015007002659563, "centered_token_effective_rank": 1.1463745860310726, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.011340838606137802, "token_RMS": 0.2823325895924088, "mean_token_offdiagonal_cosine": 0.998180718843102, "temperature": 0.5004761815071106 }, { "layer": 7, "head": 7, "slices": 8, "native_nodes": 33555, "mass": [ 0.11045276757422405, 0.11937953480717109, 0.11279532316277802, 0.13919320870482288, 0.13017242704790832, 0.1258622476379503, 0.12790787200084155, 0.13423661892086539 ], "mean_node_assignment_entropy": 2.074580911739425, "normalized_node_entropy": 0.9976625310964576, "effective_mass_slots": 7.977008650650769, "node_slot_mutual_information_nats": 0.0019825736381169357, "normalized_node_slot_mutual_information": 0.0009534163853028312, "centered_token_effective_rank": 1.3972113220539601, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.007383927211312428, "token_RMS": 0.25277466679412447, "mean_token_offdiagonal_cosine": 0.9990401244567818, "temperature": 0.5001633167266846 }, { "layer": 7, "head": 8, "slices": 8, "native_nodes": 33555, "mass": [ 0.12854678642197348, 0.1282690440450708, 0.12677010716499895, 0.11763224447360605, 0.12274227075100017, 0.1250570818353314, 0.12468666973802874, 0.12629579579736025 ], "mean_node_assignment_entropy": 2.075072311581805, "normalized_node_entropy": 0.997898844468356, "effective_mass_slots": 7.997169727061722, "node_slot_mutual_information_nats": 0.004015383384381455, "normalized_node_slot_mutual_information": 0.0019309912319716892, "centered_token_effective_rank": 1.0573466919087402, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.01272173605823721, "token_RMS": 0.25191176404286625, "mean_token_offdiagonal_cosine": 0.997097506209643, "temperature": 0.49664899706840515 }, { "layer": 8, "head": 1, "slices": 8, "native_nodes": 33555, "mass": [ 0.11925862746202472, 0.13820509467389966, 0.12627505550035828, 0.12586817015631332, 0.13039403577894307, 0.12313565519603527, 0.1174626447114943, 0.1194007165433575 ], "mean_node_assignment_entropy": 2.0740661207410542, "normalized_node_entropy": 0.9974149689563098, "effective_mass_slots": 7.989562308684476, "node_slot_mutual_information_nats": 0.004069857647353903, "normalized_node_slot_mutual_information": 0.0019571878149871667, "centered_token_effective_rank": 1.0401211075919785, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.012133045306214344, "token_RMS": 0.23550907604653573, "mean_token_offdiagonal_cosine": 0.9969946647404606, "temperature": 0.4956398904323578 }, { "layer": 8, "head": 2, "slices": 8, "native_nodes": 33555, "mass": [ 0.12459300137127037, 0.1247079583813316, 0.12483105654866414, 0.11002161981558199, 0.11363168368061899, 0.14903086406088428, 0.12539878691309453, 0.12778502900507 ], "mean_node_assignment_entropy": 1.9821367703672734, "normalized_node_entropy": 0.9532062963241772, "effective_mass_slots": 7.970644576998263, "node_slot_mutual_information_nats": 0.09362859457217199, "normalized_node_slot_mutual_information": 0.04502583635822528, "centered_token_effective_rank": 1.0285474394901586, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.08180260827454616, "token_RMS": 0.27914778106878996, "mean_token_offdiagonal_cosine": 0.9088073759828615, "temperature": 0.47270864248275757 }, { "layer": 8, "head": 3, "slices": 8, "native_nodes": 33555, "mass": [ 0.12906228469818223, 0.1330299442541562, 0.11648284616288797, 0.11518340249199072, 0.13431435470129158, 0.12893458677286754, 0.12942456317616724, 0.11356801771026062 ], "mean_node_assignment_entropy": 2.0717406091693062, "normalized_node_entropy": 0.9962966342856129, "effective_mass_slots": 7.983794814063887, "node_slot_mutual_information_nats": 0.005673229868318419, "normalized_node_slot_mutual_information": 0.002728246865615377, "centered_token_effective_rank": 1.046134355663392, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.014906581239868537, "token_RMS": 0.2083894206901716, "mean_token_offdiagonal_cosine": 0.9942455629230705, "temperature": 0.49683961272239685 }, { "layer": 8, "head": 4, "slices": 8, "native_nodes": 33555, "mass": [ 0.129703148325533, 0.11868154705014852, 0.13082921666420264, 0.1223847851485809, 0.13560559186196827, 0.11572427653923083, 0.121458995616031, 0.1256124391257008 ], "mean_node_assignment_entropy": 2.03812466837817, "normalized_node_entropy": 0.9801307839275497, "effective_mass_slots": 7.9899736323997725, "node_slot_mutual_information_nats": 0.04006279131944579, "normalized_node_slot_mutual_information": 0.01926613012024462, "centered_token_effective_rank": 1.0220593777668143, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.04740976922649875, "token_RMS": 0.32128749256687644, "mean_token_offdiagonal_cosine": 0.9761702920795214, "temperature": 0.491964727640152 }, { "layer": 8, "head": 5, "slices": 8, "native_nodes": 33555, "mass": [ 0.12019808692550929, 0.13633140057209903, 0.11504649977946207, 0.11750605964475018, 0.11461627701856113, 0.13822514598326255, 0.13031542081646194, 0.12776110937535506 ], "mean_node_assignment_entropy": 2.0749970373080378, "normalized_node_entropy": 0.9978626451945326, "effective_mass_slots": 7.9801020434231535, "node_slot_mutual_information_nats": 0.0019541614683067365, "normalized_node_slot_mutual_information": 0.0009397530198074748, "centered_token_effective_rank": 1.0733276686624589, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.009161651947476094, "token_RMS": 0.23171304152366828, "mean_token_offdiagonal_cosine": 0.9982300198541444, "temperature": 0.5034866333007812 }, { "layer": 8, "head": 6, "slices": 8, "native_nodes": 33555, "mass": [ 0.1192909327469574, 0.11986354913508691, 0.12381286267893375, 0.12070955026657779, 0.11909355568585604, 0.1279958881276799, 0.12492917739959539, 0.14430448429637083 ], "mean_node_assignment_entropy": 2.0731103716051975, "normalized_node_entropy": 0.9969553507767649, "effective_mass_slots": 7.984684006131611, "node_slot_mutual_information_nats": 0.004414835844992471, "normalized_node_slot_mutual_information": 0.002123087259969825, "centered_token_effective_rank": 1.0351390352049776, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.012533149277500887, "token_RMS": 0.20720692315487002, "mean_token_offdiagonal_cosine": 0.9958299612126577, "temperature": 0.4940723180770874 }, { "layer": 8, "head": 7, "slices": 8, "native_nodes": 33555, "mass": [ 0.11924306464641998, 0.1157906993839464, 0.11538505310953472, 0.12669812973548183, 0.11442605301686716, 0.14617407292909193, 0.12260941123042265, 0.13967351605037412 ], "mean_node_assignment_entropy": 2.0663129603683936, "normalized_node_entropy": 0.9936864869493583, "effective_mass_slots": 7.96896920136099, "node_slot_mutual_information_nats": 0.009242189233749531, "normalized_node_slot_mutual_information": 0.004444553524829273, "centered_token_effective_rank": 1.0093591233297183, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.025016784587496288, "token_RMS": 0.22441794153254999, "mean_token_offdiagonal_cosine": 0.9861362286436446, "temperature": 0.4940938353538513 }, { "layer": 8, "head": 8, "slices": 8, "native_nodes": 33555, "mass": [ 0.12544677487749792, 0.12731194745995145, 0.12204058144539662, 0.12980920131014492, 0.12360347399784793, 0.11775380684235753, 0.12679186129251158, 0.12724235305583986 ], "mean_node_assignment_entropy": 2.0619763540178964, "normalized_node_entropy": 0.991601020123975, "effective_mass_slots": 7.996772456081751, "node_slot_mutual_information_nats": 0.01706166326713987, "normalized_node_slot_mutual_information": 0.008204925661606695, "centered_token_effective_rank": 1.0101186653517387, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.030607523323695054, "token_RMS": 0.2693610784158357, "mean_token_offdiagonal_cosine": 0.9858107240535586, "temperature": 0.4933362603187561 }, { "layer": 9, "head": 1, "slices": 8, "native_nodes": 33555, "mass": [ 0.1286714373951057, 0.1250322567581278, 0.15913850967545407, 0.11178449901924672, 0.12028913800388845, 0.11928625256622483, 0.13373653914259295, 0.10206136737663297 ], "mean_node_assignment_entropy": 2.0350676292555434, "normalized_node_entropy": 0.9786606588668775, "effective_mass_slots": 7.937580725959216, "node_slot_mutual_information_nats": 0.03654090511113672, "normalized_node_slot_mutual_information": 0.017572460864477044, "centered_token_effective_rank": 1.0270327314499565, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.04822886007098757, "token_RMS": 0.23917449856389764, "mean_token_offdiagonal_cosine": 0.9562171398410196, "temperature": 0.4836384057998657 }, { "layer": 9, "head": 2, "slices": 8, "native_nodes": 33555, "mass": [ 0.1067047483429719, 0.1399871586791859, 0.14602829769426923, 0.10952192581218607, 0.11147126446820428, 0.13737594607338752, 0.13570231892620374, 0.11320834004533502 ], "mean_node_assignment_entropy": 1.998671531873764, "normalized_node_entropy": 0.9611578358000757, "effective_mass_slots": 7.94161239218914, "node_slot_mutual_information_nats": 0.07344479483306321, "normalized_node_slot_mutual_information": 0.03531948042825589, "centered_token_effective_rank": 1.0144166082571489, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.05314387517327944, "token_RMS": 0.28095044286781035, "mean_token_offdiagonal_cosine": 0.9593621807149495, "temperature": 0.48139992356300354 }, { "layer": 9, "head": 3, "slices": 8, "native_nodes": 33555, "mass": [ 0.1102462531845823, 0.11854726127761697, 0.11377513922530019, 0.10899148718923699, 0.13104708413775962, 0.18941889808859746, 0.11662363131912076, 0.11135024565933022 ], "mean_node_assignment_entropy": 1.9450624022342056, "normalized_node_entropy": 0.9353772939742876, "effective_mass_slots": 7.855827547985925, "node_slot_mutual_information_nats": 0.11619321694589901, "normalized_node_slot_mutual_information": 0.05587712595759466, "centered_token_effective_rank": 1.043215578582435, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.06520783147111092, "token_RMS": 0.2638271899692479, "mean_token_offdiagonal_cosine": 0.9364247139942543, "temperature": 0.4661524295806885 }, { "layer": 9, "head": 4, "slices": 8, "native_nodes": 33555, "mass": [ 0.11334566993181192, 0.11247292985270088, 0.13452167532213433, 0.11388830180304751, 0.10867615582632638, 0.16375356668145943, 0.11693422722645051, 0.13640747358832375 ], "mean_node_assignment_entropy": 2.0594897757689177, "normalized_node_entropy": 0.990405228754447, "effective_mass_slots": 7.924925383305992, "node_slot_mutual_information_nats": 0.010523128594391196, "normalized_node_slot_mutual_information": 0.005060555145921675, "centered_token_effective_rank": 1.1861922438013388, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.01979120473763761, "token_RMS": 0.23632451015862693, "mean_token_offdiagonal_cosine": 0.9921153256002864, "temperature": 0.4947497844696045 }, { "layer": 9, "head": 5, "slices": 8, "native_nodes": 33555, "mass": [ 0.12892054186001778, 0.12384075617589498, 0.12780386430951454, 0.1095817891030176, 0.11739635317518501, 0.1333942283990175, 0.13098175546936797, 0.12808071149943606 ], "mean_node_assignment_entropy": 2.0290897913350356, "normalized_node_entropy": 0.9757859264924926, "effective_mass_slots": 7.985769161778523, "node_slot_mutual_information_nats": 0.04857131152615768, "normalized_node_slot_mutual_information": 0.023357863422753547, "centered_token_effective_rank": 1.0207629641894902, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.050069579418639924, "token_RMS": 0.23348325941189166, "mean_token_offdiagonal_cosine": 0.9481575513941038, "temperature": 0.48777955770492554 }, { "layer": 9, "head": 6, "slices": 8, "native_nodes": 33555, "mass": [ 0.10473734635747042, 0.14908090007121294, 0.19582807231794933, 0.1071939013715189, 0.10954179189063419, 0.09629538999377767, 0.14385839967014644, 0.09346419813522477 ], "mean_node_assignment_entropy": 2.0228558755148374, "normalized_node_entropy": 0.9727880466794528, "effective_mass_slots": 7.743115262233026, "node_slot_mutual_information_nats": 0.023948219770562496, "normalized_node_slot_mutual_information": 0.011516659300369847, "centered_token_effective_rank": 1.0284382236632443, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.03137852291068363, "token_RMS": 0.2345121603687056, "mean_token_offdiagonal_cosine": 0.9824939410689336, "temperature": 0.4880354702472687 }, { "layer": 9, "head": 7, "slices": 8, "native_nodes": 33555, "mass": [ 0.12462711462479309, 0.13173586538519735, 0.1348369705353519, 0.11778604410152431, 0.11385523878130077, 0.13093352967224023, 0.12374440807310165, 0.12248082916665727 ], "mean_node_assignment_entropy": 1.9530631760792776, "normalized_node_entropy": 0.9392248528908075, "effective_mass_slots": 7.988398273163784, "node_slot_mutual_information_nats": 0.12492709716524963, "normalized_node_slot_mutual_information": 0.060077234517653115, "centered_token_effective_rank": 1.015722515715929, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.08926954353821866, "token_RMS": 0.21746118570570308, "mean_token_offdiagonal_cosine": 0.8139818308419071, "temperature": 0.47380000352859497 }, { "layer": 9, "head": 8, "slices": 8, "native_nodes": 33555, "mass": [ 0.11729961644828701, 0.13287072139846962, 0.1392924861578628, 0.11727233242669291, 0.10330435424097212, 0.10776043326359822, 0.15867919189617596, 0.12352086425942221 ], "mean_node_assignment_entropy": 1.9631156931389213, "normalized_node_entropy": 0.9440590917276074, "effective_mass_slots": 7.928701853296385, "node_slot_mutual_information_nats": 0.10737362838816478, "normalized_node_slot_mutual_information": 0.051635800399286585, "centered_token_effective_rank": 1.010944137918443, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.06945804137946578, "token_RMS": 0.2664695968303987, "mean_token_offdiagonal_cosine": 0.9275048696434744, "temperature": 0.48024916648864746 }, { "layer": 10, "head": 1, "slices": 8, "native_nodes": 33555, "mass": [ 0.1196597813687316, 0.13231664045994584, 0.13386403174051992, 0.12209768748384636, 0.11888155649185643, 0.127697114816074, 0.1288317986946226, 0.11665138846279664 ], "mean_node_assignment_entropy": 2.07588704698665, "normalized_node_entropy": 0.9982906493777582, "effective_mass_slots": 7.99047473477724, "node_slot_mutual_information_nats": 0.0023631271437674073, "normalized_node_slot_mutual_information": 0.0011364239371011132, "centered_token_effective_rank": 1.0442278796143916, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.010206335434251557, "token_RMS": 0.2259158049302568, "mean_token_offdiagonal_cosine": 0.9976746508990677, "temperature": 0.4942673444747925 }, { "layer": 10, "head": 2, "slices": 8, "native_nodes": 33555, "mass": [ 0.1345001985511718, 0.12396930353512016, 0.11329298938369631, 0.1338454008206776, 0.12255064510602813, 0.11947981453482787, 0.1295023657171706, 0.12285928257956552 ], "mean_node_assignment_entropy": 2.0730697303129677, "normalized_node_entropy": 0.9969358064465131, "effective_mass_slots": 7.988209326376678, "node_slot_mutual_information_nats": 0.004896890001995224, "normalized_node_slot_mutual_information": 0.0023549063072190856, "centered_token_effective_rank": 1.0181030176331811, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.017422565221514568, "token_RMS": 0.19429478173977424, "mean_token_offdiagonal_cosine": 0.9909455808636604, "temperature": 0.4939288794994354 }, { "layer": 10, "head": 3, "slices": 8, "native_nodes": 33555, "mass": [ 0.14156699009254106, 0.12433734596321824, 0.10664271072936544, 0.12359732617841288, 0.12491516271782888, 0.12948265167186626, 0.12499667790346042, 0.12446113466004148 ], "mean_node_assignment_entropy": 2.0758459459122456, "normalized_node_entropy": 0.9982708839390187, "effective_mass_slots": 7.9795311309578105, "node_slot_mutual_information_nats": 0.001033708304227332, "normalized_node_slot_mutual_information": 0.0004971086147448373, "centered_token_effective_rank": 1.283879149096402, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.005895681331124762, "token_RMS": 0.27493972989461957, "mean_token_offdiagonal_cosine": 0.9994938085385916, "temperature": 0.4982563853263855 }, { "layer": 10, "head": 4, "slices": 8, "native_nodes": 33555, "mass": [ 0.12332206261755724, 0.13032485473717295, 0.11700407921748338, 0.1325963578792962, 0.1240957096698671, 0.1283326422443441, 0.11748539616233492, 0.12683889756298086 ], "mean_node_assignment_entropy": 2.0757496492570136, "normalized_node_entropy": 0.9982245750366996, "effective_mass_slots": 7.992786251069582, "node_slot_mutual_information_nats": 0.0027897670137284614, "normalized_node_slot_mutual_information": 0.0013415943453138882, "centered_token_effective_rank": 1.023922605142201, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.01246606905487204, "token_RMS": 0.30110444471381365, "mean_token_offdiagonal_cosine": 0.9980435479601919, "temperature": 0.49411994218826294 }, { "layer": 10, "head": 5, "slices": 8, "native_nodes": 33555, "mass": [ 0.11625917649690093, 0.12278314597750364, 0.12130824770085095, 0.12013093394998502, 0.13279879717016127, 0.1320365492996667, 0.12619112224550258, 0.12849202713256194 ], "mean_node_assignment_entropy": 2.0746189812062115, "normalized_node_entropy": 0.997680838640105, "effective_mass_slots": 7.992238046528487, "node_slot_mutual_information_nats": 0.003851845298121326, "normalized_node_slot_mutual_information": 0.0018523460366237027, "centered_token_effective_rank": 1.0842947926158821, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.01300838330483675, "token_RMS": 0.22234995321750498, "mean_token_offdiagonal_cosine": 0.996137273679559, "temperature": 0.4944462180137634 }, { "layer": 10, "head": 6, "slices": 8, "native_nodes": 33555, "mass": [ 0.11732288799250717, 0.12054427098346847, 0.13233457137714677, 0.1290895715979808, 0.13239761888203722, 0.1232151587306787, 0.11962288156265012, 0.1254730386206262 ], "mean_node_assignment_entropy": 2.073151260736931, "normalized_node_entropy": 0.9969750142926243, "effective_mass_slots": 7.9924505008958056, "node_slot_mutual_information_nats": 0.005346148001606821, "normalized_node_slot_mutual_information": 0.002570953736592201, "centered_token_effective_rank": 1.0855773028110727, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.018920612252241523, "token_RMS": 0.2180649133938501, "mean_token_offdiagonal_cosine": 0.992979879297553, "temperature": 0.49412477016448975 }, { "layer": 10, "head": 7, "slices": 8, "native_nodes": 33555, "mass": [ 0.11402090013431306, 0.12276451663948032, 0.10865574367571972, 0.14438128192846475, 0.12436027134043008, 0.10947608918178066, 0.12133814873149686, 0.15500304849754235 ], "mean_node_assignment_entropy": 2.0611264280820514, "normalized_node_entropy": 0.9911922921463862, "effective_mass_slots": 7.940453652450292, "node_slot_mutual_information_nats": 0.01084398061415337, "normalized_node_slot_mutual_information": 0.005214852351845042, "centered_token_effective_rank": 1.008036521997243, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.022398588573589864, "token_RMS": 0.250132650442035, "mean_token_offdiagonal_cosine": 0.991283466377032, "temperature": 0.4893980324268341 }, { "layer": 10, "head": 8, "slices": 8, "native_nodes": 33555, "mass": [ 0.123127666649547, 0.12360026948614708, 0.11477856879515151, 0.12301585996622585, 0.12482755562686615, 0.12291671570327736, 0.13148052441507174, 0.13625283940078925 ], "mean_node_assignment_entropy": 2.0758996310733644, "normalized_node_entropy": 0.9982967010442573, "effective_mass_slots": 7.990867030585961, "node_slot_mutual_information_nats": 0.0023996372838781532, "normalized_node_slot_mutual_information": 0.001153981603127758, "centered_token_effective_rank": 1.0396662239470646, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.011801182541680041, "token_RMS": 0.2755831971799838, "mean_token_offdiagonal_cosine": 0.997910756393666, "temperature": 0.49924707412719727 }, { "layer": 11, "head": 1, "slices": 8, "native_nodes": 33555, "mass": [ 0.3441485783812753, 0.0808887673400075, 0.08001807996740357, 0.16953359822788575, 0.07563555932515932, 0.08609252247835587, 0.08312203059642685, 0.08056086359567514 ], "mean_node_assignment_entropy": 1.5700700444414277, "normalized_node_entropy": 0.7550440889879875, "effective_mass_slots": 6.616347093620422, "node_slot_mutual_information_nats": 0.3194733746638796, "normalized_node_slot_mutual_information": 0.15363421777454697, "centered_token_effective_rank": 1.0266254305602156, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.08778350331204517, "token_RMS": 0.27689596752788204, "mean_token_offdiagonal_cosine": 0.8963569677912674, "temperature": 0.47368553280830383 }, { "layer": 11, "head": 2, "slices": 8, "native_nodes": 33555, "mass": [ 0.4695182051960973, 0.04538497346416038, 0.04923981327475818, 0.20303861769475515, 0.05165178399566894, 0.04453352995276798, 0.05244929976478239, 0.08418377607870992 ], "mean_node_assignment_entropy": 1.1417449020556591, "normalized_node_entropy": 0.5490632360519849, "effective_mass_slots": 5.062637108002666, "node_slot_mutual_information_nats": 0.48014261307251394, "normalized_node_slot_mutual_information": 0.23089978893306143, "centered_token_effective_rank": 1.0327320804901203, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.10383389583804696, "token_RMS": 0.33939817574443193, "mean_token_offdiagonal_cosine": 0.8934327859398599, "temperature": 0.45082399249076843 }, { "layer": 11, "head": 3, "slices": 8, "native_nodes": 33555, "mass": [ 0.09891874437624093, 0.10823568735638615, 0.10970287626372455, 0.19422667014923692, 0.09810998274279571, 0.11309992360566469, 0.1119229450101354, 0.1657831706433618 ], "mean_node_assignment_entropy": 1.8540460547301094, "normalized_node_entropy": 0.8916076829129589, "effective_mass_slots": 7.748770022952514, "node_slot_mutual_information_nats": 0.19348806932409812, "normalized_node_slot_mutual_information": 0.09304809269501878, "centered_token_effective_rank": 1.0096373609058282, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.08876302825181727, "token_RMS": 0.23707131905161122, "mean_token_offdiagonal_cosine": 0.8494266771985267, "temperature": 0.47917792201042175 }, { "layer": 11, "head": 4, "slices": 8, "native_nodes": 33555, "mass": [ 0.060134953557345876, 0.10429988640843889, 0.10788441671300282, 0.24957697234710316, 0.1031675969902884, 0.10107261028699771, 0.10721452734857073, 0.1666490366889741 ], "mean_node_assignment_entropy": 1.828935349112209, "normalized_node_entropy": 0.8795319860902363, "effective_mass_slots": 7.355529690539711, "node_slot_mutual_information_nats": 0.16651702009646963, "normalized_node_slot_mutual_information": 0.08007775970559487, "centered_token_effective_rank": 1.0133935950682755, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.10357812484293595, "token_RMS": 0.2431463802047821, "mean_token_offdiagonal_cosine": 0.839113081190531, "temperature": 0.4785913825035095 }, { "layer": 11, "head": 5, "slices": 8, "native_nodes": 33555, "mass": [ 0.10116315777418404, 0.08790247145334666, 0.08613335130864738, 0.257025133086354, 0.08344000963878034, 0.08982404297507213, 0.09142711446651748, 0.20308471923797264 ], "mean_node_assignment_entropy": 1.613444345680371, "normalized_node_entropy": 0.77590271875447, "effective_mass_slots": 7.185309569536492, "node_slot_mutual_information_nats": 0.35859425845294135, "normalized_node_slot_mutual_information": 0.17244738612043792, "centered_token_effective_rank": 1.0183428487315427, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.10372355822441541, "token_RMS": 0.26691417785236093, "mean_token_offdiagonal_cosine": 0.8216988884715564, "temperature": 0.45768776535987854 }, { "layer": 11, "head": 6, "slices": 8, "native_nodes": 33555, "mass": [ 0.40734694502456753, 0.09094933801763744, 0.08897545032987002, 0.10132685501118444, 0.08237109903472091, 0.08476287729209954, 0.07830866719750823, 0.06595876826079694 ], "mean_node_assignment_entropy": 1.4216946963891661, "normalized_node_entropy": 0.6836906293795969, "effective_mass_slots": 6.201161326300661, "node_slot_mutual_information_nats": 0.4030418888150771, "normalized_node_slot_mutual_information": 0.19382217808801092, "centered_token_effective_rank": 1.026783865340098, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.09099604043831581, "token_RMS": 0.25745498659218585, "mean_token_offdiagonal_cosine": 0.8761077964214442, "temperature": 0.4671856462955475 }, { "layer": 11, "head": 7, "slices": 8, "native_nodes": 33555, "mass": [ 0.5695679467565408, 0.015032324449535162, 0.015558491816032489, 0.29180139865114646, 0.016371904466820946, 0.014305942798145503, 0.016289991967186515, 0.0610719995687165 ], "mean_node_assignment_entropy": 0.5073433230539317, "normalized_node_entropy": 0.24398056539934487, "effective_mass_slots": 3.234148726253459, "node_slot_mutual_information_nats": 0.6664224253717712, "normalized_node_slot_mutual_information": 0.3204814427403499, "centered_token_effective_rank": 1.04333732822982, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.14191568718919934, "token_RMS": 0.35706673778395837, "mean_token_offdiagonal_cosine": 0.7791420914285914, "temperature": 0.4326547086238861 }, { "layer": 11, "head": 8, "slices": 8, "native_nodes": 33555, "mass": [ 0.06779852360820046, 0.09566105063074863, 0.0917607654828088, 0.26478558514184497, 0.09370698189037316, 0.09557699609307495, 0.09833632258462703, 0.19237377452091628 ], "mean_node_assignment_entropy": 1.6332912753169835, "normalized_node_entropy": 0.7854470744090076, "effective_mass_slots": 7.166604030841566, "node_slot_mutual_information_nats": 0.33614063122775883, "normalized_node_slot_mutual_information": 0.16164947390452453, "centered_token_effective_rank": 1.0171192290279167, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.10460340976211774, "token_RMS": 0.2682502170014959, "mean_token_offdiagonal_cosine": 0.8296449797662214, "temperature": 0.458250492811203 }, { "layer": 12, "head": 1, "slices": 8, "native_nodes": 33555, "mass": [ 0.08389663558292758, 0.06598470919064577, 0.06760331620684973, 0.511730995693219, 0.06594551263299078, 0.05425103911802488, 0.09083226492090422, 0.05975552655697078 ], "mean_node_assignment_entropy": 1.1323284089081078, "normalized_node_entropy": 0.5445348600631392, "effective_mass_slots": 5.134001664983436, "node_slot_mutual_information_nats": 0.5035569978460932, "normalized_node_slot_mutual_information": 0.24215972786583106, "centered_token_effective_rank": 1.009283741691327, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.11049739082581554, "token_RMS": 0.2459006121600717, "mean_token_offdiagonal_cosine": 0.7997905442480281, "temperature": 0.4750920236110687 }, { "layer": 12, "head": 2, "slices": 8, "native_nodes": 33555, "mass": [ 0.08145860171037568, 0.05157251644246954, 0.05354011667790823, 0.5545704244326454, 0.04515193069521688, 0.04764266858918225, 0.11660789213469377, 0.04945584929243696 ], "mean_node_assignment_entropy": 0.8986718691028018, "normalized_node_entropy": 0.432169816313676, "effective_mass_slots": 4.595272759857982, "node_slot_mutual_information_nats": 0.6263562450863002, "normalized_node_slot_mutual_information": 0.30121368287194583, "centered_token_effective_rank": 1.022653283588419, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.12054900153494753, "token_RMS": 0.3557182936540786, "mean_token_offdiagonal_cosine": 0.8391053718881586, "temperature": 0.46118229627609253 }, { "layer": 12, "head": 3, "slices": 8, "native_nodes": 33555, "mass": [ 0.08647970580724824, 0.048370669529425994, 0.03867628224682223, 0.561221303026157, 0.029564568026501988, 0.033226513567905284, 0.16686503138474762, 0.03559592596540648 ], "mean_node_assignment_entropy": 0.7872125352751307, "normalized_node_entropy": 0.37856920692235313, "effective_mass_slots": 4.232995477901584, "node_slot_mutual_information_nats": 0.6556973579667433, "normalized_node_slot_mutual_information": 0.3153237755542054, "centered_token_effective_rank": 1.0537207172842464, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.11749243364065606, "token_RMS": 0.3609580449898506, "mean_token_offdiagonal_cosine": 0.8527938931028325, "temperature": 0.43108418583869934 }, { "layer": 12, "head": 4, "slices": 8, "native_nodes": 33555, "mass": [ 0.0900367924487563, 0.05130707776616244, 0.039076040734878435, 0.5437150124373151, 0.043273687811007794, 0.037326177436995074, 0.15060943882131558, 0.044655772321380474 ], "mean_node_assignment_entropy": 0.8704663772662244, "normalized_node_entropy": 0.41860584191418787, "effective_mass_slots": 4.5253661416738575, "node_slot_mutual_information_nats": 0.6392321118992401, "normalized_node_slot_mutual_information": 0.3074056659380042, "centered_token_effective_rank": 1.0321507971036625, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.12104753946689015, "token_RMS": 0.31412344340949083, "mean_token_offdiagonal_cosine": 0.8035399046271928, "temperature": 0.4495494067668915 }, { "layer": 12, "head": 5, "slices": 8, "native_nodes": 33555, "mass": [ 0.08253727817822305, 0.0628947918584262, 0.06332526055811745, 0.5099631493106088, 0.062401765522857505, 0.05978965169801917, 0.10002884944299834, 0.05905925395951672 ], "mean_node_assignment_entropy": 1.1018403770882916, "normalized_node_entropy": 0.5298732159588346, "effective_mass_slots": 5.139520420771232, "node_slot_mutual_information_nats": 0.5351193946738533, "normalized_node_slot_mutual_information": 0.2573380323264907, "centered_token_effective_rank": 1.0292464913324517, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.0918280355585745, "token_RMS": 0.23004932235865472, "mean_token_offdiagonal_cosine": 0.834372608089024, "temperature": 0.46094509959220886 }, { "layer": 12, "head": 6, "slices": 8, "native_nodes": 33555, "mass": [ 0.11040160922013582, 0.09618994114822521, 0.08746803105921064, 0.3548937001507289, 0.07791942479023678, 0.08152203274153327, 0.11301130359400369, 0.07859395733434829 ], "mean_node_assignment_entropy": 1.5747954286248604, "normalized_node_entropy": 0.7573165184305652, "effective_mass_slots": 6.677785418069891, "node_slot_mutual_information_nats": 0.3239909797163496, "normalized_node_slot_mutual_information": 0.15580672657651146, "centered_token_effective_rank": 1.0123136152321819, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.08341553028004632, "token_RMS": 0.2311913876767494, "mean_token_offdiagonal_cosine": 0.8516838732745585, "temperature": 0.47789451479911804 }, { "layer": 12, "head": 7, "slices": 8, "native_nodes": 33555, "mass": [ 0.09510046449824201, 0.06329350597342395, 0.058622043643278866, 0.48578800351511453, 0.059701432537822036, 0.057474169455526386, 0.11889706960450588, 0.06112331072900474 ], "mean_node_assignment_entropy": 1.1626101528863801, "normalized_node_entropy": 0.5590973006854468, "effective_mass_slots": 5.322367962758909, "node_slot_mutual_information_nats": 0.509308157218729, "normalized_node_slot_mutual_information": 0.2449254509012523, "centered_token_effective_rank": 1.0536910568109716, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.09739431113655998, "token_RMS": 0.3517122340179223, "mean_token_offdiagonal_cosine": 0.9051068503094021, "temperature": 0.4664011299610138 }, { "layer": 12, "head": 8, "slices": 8, "native_nodes": 33555, "mass": [ 0.15395634768572636, 0.1479180219138438, 0.1429022912584959, 0.04296905663105361, 0.11685848941307916, 0.13059568348975034, 0.14758118511938229, 0.117218924293911 ], "mean_node_assignment_entropy": 2.0030642663169966, "normalized_node_entropy": 0.9632702945324737, "effective_mass_slots": 7.647634860873916, "node_slot_mutual_information_nats": 0.031332165205796514, "normalized_node_slot_mutual_information": 0.015067586454238788, "centered_token_effective_rank": 1.0118999973266793, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.06152054854767172, "token_RMS": 0.21359910704841473, "mean_token_offdiagonal_cosine": 0.9122638876188438, "temperature": 0.4727741479873657 }, { "layer": 13, "head": 1, "slices": 8, "native_nodes": 33555, "mass": [ 0.03001529634564825, 0.24230516324840548, 0.04056738586237741, 0.022434048705206353, 0.569919041860426, 0.02090498339004024, 0.06126950134519494, 0.01258457760286607 ], "mean_node_assignment_entropy": 0.602173977522111, "normalized_node_entropy": 0.28958447037451057, "effective_mass_slots": 3.6377520240953194, "node_slot_mutual_information_nats": 0.6891919375500651, "normalized_node_slot_mutual_information": 0.3314312635080451, "centered_token_effective_rank": 1.0486913004878837, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.1503737328901854, "token_RMS": 0.3555051383563703, "mean_token_offdiagonal_cosine": 0.7072730701608825, "temperature": 0.41820311546325684 }, { "layer": 13, "head": 2, "slices": 8, "native_nodes": 33555, "mass": [ 0.11389413456983599, 0.13572899864609347, 0.11540161608526131, 0.13590523663615706, 0.11876753743151193, 0.12337544666479142, 0.12101886973657616, 0.13590816053374657 ], "mean_node_assignment_entropy": 2.0634890780227084, "normalized_node_entropy": 0.9923284865973003, "effective_mass_slots": 7.98008386373751, "node_slot_mutual_information_nats": 0.013459842624076845, "normalized_node_slot_mutual_information": 0.00647281606830052, "centered_token_effective_rank": 1.156383800446124, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.0206679197445542, "token_RMS": 0.2270921482976985, "mean_token_offdiagonal_cosine": 0.9915724676577954, "temperature": 0.4785255491733551 }, { "layer": 13, "head": 3, "slices": 8, "native_nodes": 33555, "mass": [ 0.044982612369212534, 0.16931195558884032, 0.05743429373179371, 0.039648094373890934, 0.5575418557818863, 0.032764272737783456, 0.07276873253654442, 0.02554818299069446 ], "mean_node_assignment_entropy": 0.7961268415568074, "normalized_node_entropy": 0.3828560820775332, "effective_mass_slots": 4.2818608572827515, "node_slot_mutual_information_nats": 0.6582608532865349, "normalized_node_slot_mutual_information": 0.31655655621594053, "centered_token_effective_rank": 1.0396436573769494, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.14140164292417076, "token_RMS": 0.28756755008309093, "mean_token_offdiagonal_cosine": 0.7493723093164962, "temperature": 0.44515034556388855 }, { "layer": 13, "head": 4, "slices": 8, "native_nodes": 33555, "mass": [ 0.12098416786664752, 0.1168416234897068, 0.11896587558620254, 0.12066473965916792, 0.17194385649043817, 0.1206690122863696, 0.11294864332324221, 0.11698208100091255 ], "mean_node_assignment_entropy": 2.0635440400229683, "normalized_node_entropy": 0.9923549177323711, "effective_mass_slots": 7.92529478218172, "node_slot_mutual_information_nats": 0.006515475538410698, "normalized_node_slot_mutual_information": 0.0031332814160994877, "centered_token_effective_rank": 1.1995212677552074, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.018857371653550947, "token_RMS": 0.26507362375855675, "mean_token_offdiagonal_cosine": 0.9945836738053551, "temperature": 0.48087307810783386 }, { "layer": 13, "head": 5, "slices": 8, "native_nodes": 33555, "mass": [ 0.13012329816858204, 0.1325220156001931, 0.12519853430335381, 0.12907195145580955, 0.09861858858059547, 0.11494247541775239, 0.14115699435968795, 0.12836614232707388 ], "mean_node_assignment_entropy": 2.028339501631097, "normalized_node_entropy": 0.9754251134141251, "effective_mass_slots": 7.9612341501792905, "node_slot_mutual_information_nats": 0.046244530199490264, "normalized_node_slot_mutual_information": 0.022238918129014837, "centered_token_effective_rank": 1.019976675836448, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.05550073262435775, "token_RMS": 0.2533087581393552, "mean_token_offdiagonal_cosine": 0.9467368937954689, "temperature": 0.4780655801296234 }, { "layer": 13, "head": 6, "slices": 8, "native_nodes": 33555, "mass": [ 0.11861409044453594, 0.13640052466993982, 0.1105853733757208, 0.13534887462206027, 0.08108092413726994, 0.11521370820495734, 0.12272890171671993, 0.18002760280298372 ], "mean_node_assignment_entropy": 1.9472222068833416, "normalized_node_entropy": 0.9364159404598201, "effective_mass_slots": 7.827214755688334, "node_slot_mutual_information_nats": 0.11038452536649324, "normalized_node_slot_mutual_information": 0.053083735779040596, "centered_token_effective_rank": 1.0260462424114214, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.08478105411347554, "token_RMS": 0.27258416988134715, "mean_token_offdiagonal_cosine": 0.8938477039255517, "temperature": 0.46261516213417053 }, { "layer": 13, "head": 7, "slices": 8, "native_nodes": 33555, "mass": [ 0.07009956025130044, 0.1333145045524766, 0.08150159584092163, 0.0798680607875055, 0.37812052301088844, 0.07461014567457906, 0.09736396794384611, 0.08512164164779172 ], "mean_node_assignment_entropy": 1.496777880644545, "normalized_node_entropy": 0.719798008572726, "effective_mass_slots": 6.417503437756631, "node_slot_mutual_information_nats": 0.3622512888031806, "normalized_node_slot_mutual_information": 0.17420604597066144, "centered_token_effective_rank": 1.0350672781704036, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.08670913357044559, "token_RMS": 0.22814541755302076, "mean_token_offdiagonal_cosine": 0.8434305443219868, "temperature": 0.46913984417915344 }, { "layer": 13, "head": 8, "slices": 8, "native_nodes": 33555, "mass": [ 0.06655233625942629, 0.12625404099664994, 0.07430801419995035, 0.08021323082255565, 0.3810131318662114, 0.07508642413349376, 0.09782403306676137, 0.09874878867599136 ], "mean_node_assignment_entropy": 1.4628414213780894, "normalized_node_entropy": 0.7034780214097107, "effective_mass_slots": 6.39326378861931, "node_slot_mutual_information_nats": 0.3924034816574349, "normalized_node_slot_mutual_information": 0.18870618567158157, "centered_token_effective_rank": 1.0955346639753842, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.08596071936940855, "token_RMS": 0.27890632253053005, "mean_token_offdiagonal_cosine": 0.9110543082866044, "temperature": 0.4735002815723419 }, { "layer": 14, "head": 1, "slices": 8, "native_nodes": 33555, "mass": [ 0.038988941667051635, 0.04049485067527055, 0.035696208924224405, 0.042096440665628704, 0.11261954639124197, 0.437210629784981, 0.2540045709482509, 0.03888881122256276 ], "mean_node_assignment_entropy": 1.0516914083047784, "normalized_node_entropy": 0.5057566597689446, "effective_mass_slots": 4.907098729226606, "node_slot_mutual_information_nats": 0.5389914686793684, "normalized_node_slot_mutual_information": 0.2592001063150613, "centered_token_effective_rank": 1.1145402478898259, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.11890614169530175, "token_RMS": 0.3477970021951826, "mean_token_offdiagonal_cosine": 0.8710004065416003, "temperature": 0.46126216650009155 }, { "layer": 14, "head": 2, "slices": 8, "native_nodes": 33555, "mass": [ 0.011964377982856362, 0.011755182981094282, 0.01054254348130045, 0.013305051929138114, 0.04798863746343741, 0.5558827607842416, 0.3371523801868664, 0.011409064465676884 ], "mean_node_assignment_entropy": 0.39513014422291404, "normalized_node_entropy": 0.19001743319204634, "effective_mass_slots": 3.0053439087235647, "node_slot_mutual_information_nats": 0.7052618627145608, "normalized_node_slot_mutual_information": 0.33915926395546997, "centered_token_effective_rank": 1.0957563262705656, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.1442516990055652, "token_RMS": 0.3416851861529948, "mean_token_offdiagonal_cosine": 0.7886941339394531, "temperature": 0.43038520216941833 }, { "layer": 14, "head": 3, "slices": 8, "native_nodes": 33555, "mass": [ 0.011604855549113283, 0.19555264905266648, 0.17302986436599893, 0.01756227418844672, 0.547289169842389, 0.00628128444995477, 0.0324445428176792, 0.01623536000976032 ], "mean_node_assignment_entropy": 0.5798076104306755, "normalized_node_entropy": 0.2788285214126719, "effective_mass_slots": 3.615539301312184, "node_slot_mutual_information_nats": 0.7054334183719018, "normalized_node_slot_mutual_information": 0.3392417647874974, "centered_token_effective_rank": 1.1241010819619688, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.146600153693124, "token_RMS": 0.35089472428626317, "mean_token_offdiagonal_cosine": 0.7524352926700486, "temperature": 0.4129311442375183 }, { "layer": 14, "head": 4, "slices": 8, "native_nodes": 33555, "mass": [ 0.09427350553552238, 0.09929484591751762, 0.09719566462483496, 0.09255001394775066, 0.1065672660002524, 0.2062537795510038, 0.21164119013121224, 0.0922237339747492 ], "mean_node_assignment_entropy": 1.8022452857328628, "normalized_node_entropy": 0.8666967787307714, "effective_mass_slots": 7.474353075157045, "node_slot_mutual_information_nats": 0.20923228466914923, "normalized_node_slot_mutual_information": 0.10061945982868317, "centered_token_effective_rank": 1.0266575481558673, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.07807406360049027, "token_RMS": 0.28726751547277235, "mean_token_offdiagonal_cosine": 0.9239402693742301, "temperature": 0.48112455010414124 }, { "layer": 14, "head": 5, "slices": 8, "native_nodes": 33555, "mass": [ 0.015418734775872491, 0.01762944295353377, 0.014886921981431557, 0.014825315755508666, 0.026813485818410362, 0.5457996673156534, 0.3487837279003734, 0.0158427016238661 ], "mean_node_assignment_entropy": 0.4180261100650024, "normalized_node_entropy": 0.20102806531762768, "effective_mass_slots": 3.0683831330766536, "node_slot_mutual_information_nats": 0.7031246460461802, "normalized_node_slot_mutual_information": 0.338131479992544, "centered_token_effective_rank": 1.1574773758129255, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.1298373352650677, "token_RMS": 0.34311694936178033, "mean_token_offdiagonal_cosine": 0.8457444475768069, "temperature": 0.4221843481063843 }, { "layer": 14, "head": 6, "slices": 8, "native_nodes": 33555, "mass": [ 0.11613382830973834, 0.14569948649347814, 0.13009531067440325, 0.1192935936413833, 0.08385357192104521, 0.12999844824048176, 0.15333414577972407, 0.12159161505250102 ], "mean_node_assignment_entropy": 1.9717864147591493, "normalized_node_entropy": 0.9482288274217512, "effective_mass_slots": 7.8967495443814, "node_slot_mutual_information_nats": 0.09466480996671378, "normalized_node_slot_mutual_information": 0.04552415062855803, "centered_token_effective_rank": 1.273161180713678, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.06395660213898592, "token_RMS": 0.24575279022153096, "mean_token_offdiagonal_cosine": 0.9539056038644357, "temperature": 0.47066375613212585 }, { "layer": 14, "head": 7, "slices": 8, "native_nodes": 33555, "mass": [ 0.01812620003574529, 0.02350571824308172, 0.020926437688256524, 0.018227345573415855, 0.014708403638915533, 0.5424517653749545, 0.3434007421428658, 0.018653388018159113 ], "mean_node_assignment_entropy": 0.44836918967104167, "normalized_node_entropy": 0.2156200021419383, "effective_mass_slots": 3.158000301189633, "node_slot_mutual_information_nats": 0.7015698215359422, "normalized_node_slot_mutual_information": 0.337383767455753, "centered_token_effective_rank": 1.10194828105329, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.14731236510097354, "token_RMS": 0.3313646963612883, "mean_token_offdiagonal_cosine": 0.779597030542836, "temperature": 0.44194450974464417 }, { "layer": 14, "head": 8, "slices": 8, "native_nodes": 33555, "mass": [ 0.051879460643484275, 0.04846886821298506, 0.05128457507395717, 0.05359739845641781, 0.09852275466468702, 0.5266232013976319, 0.11350689768833734, 0.05611684392470738 ], "mean_node_assignment_entropy": 1.035603272319943, "normalized_node_entropy": 0.49801990176812155, "effective_mass_slots": 4.874548461088643, "node_slot_mutual_information_nats": 0.5484242044736263, "normalized_node_slot_mutual_information": 0.2637362933658585, "centered_token_effective_rank": 1.0162823639699232, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.09588838456966489, "token_RMS": 0.24213434796354008, "mean_token_offdiagonal_cosine": 0.8386136839333934, "temperature": 0.47348904609680176 }, { "layer": 15, "head": 1, "slices": 8, "native_nodes": 33555, "mass": [ 0.11494310543445126, 0.1639895645150785, 0.14780763148320022, 0.11628517672150879, 0.11541900010564397, 0.11298882708080016, 0.11272722159456415, 0.11583947340802808 ], "mean_node_assignment_entropy": 1.9992898472920309, "normalized_node_entropy": 0.9614551826626221, "effective_mass_slots": 7.91909782395596, "node_slot_mutual_information_nats": 0.06998744092926801, "normalized_node_slot_mutual_information": 0.03365684465105474, "centered_token_effective_rank": 1.0402620501381377, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.056913326179092234, "token_RMS": 0.191406689860943, "mean_token_offdiagonal_cosine": 0.9118107853570427, "temperature": 0.45973721146583557 }, { "layer": 15, "head": 2, "slices": 8, "native_nodes": 33555, "mass": [ 0.12107343559177386, 0.12725202599153168, 0.15933699827414213, 0.1357130284605856, 0.1132186809616035, 0.1161512318861465, 0.10980825667284655, 0.1174463427643221 ], "mean_node_assignment_entropy": 2.013886461339129, "normalized_node_entropy": 0.9684746702291283, "effective_mass_slots": 7.944555690209763, "node_slot_mutual_information_nats": 0.05860041389003978, "normalized_node_slot_mutual_information": 0.028180842171067044, "centered_token_effective_rank": 1.1036737646854045, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.056157206662742175, "token_RMS": 0.194266869789224, "mean_token_offdiagonal_cosine": 0.9109595532690756, "temperature": 0.4506489336490631 }, { "layer": 15, "head": 3, "slices": 8, "native_nodes": 33555, "mass": [ 0.12917364075893156, 0.041589692320534215, 0.19069211221348775, 0.1393424509783435, 0.11885429099568523, 0.12671661297018422, 0.12294263836102413, 0.13068856147225186 ], "mean_node_assignment_entropy": 2.000673659443905, "normalized_node_entropy": 0.9621206556389656, "effective_mass_slots": 7.582031827342772, "node_slot_mutual_information_nats": 0.025107555391727043, "normalized_node_slot_mutual_information": 0.012074181884163189, "centered_token_effective_rank": 1.0198996983973787, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.04709671765679588, "token_RMS": 0.17478858694613483, "mean_token_offdiagonal_cosine": 0.9217785213364385, "temperature": 0.47531211376190186 }, { "layer": 15, "head": 4, "slices": 8, "native_nodes": 33555, "mass": [ 0.009364835138826797, 0.5564538586541917, 0.3823754855103036, 0.014018349315661104, 0.008749445957570207, 0.009764105745799811, 0.009697541530826387, 0.00957637798799102 ], "mean_node_assignment_entropy": 0.2717701412993601, "normalized_node_entropy": 0.1306938117047599, "effective_mass_slots": 2.647113120155725, "node_slot_mutual_information_nats": 0.7016995162632036, "normalized_node_slot_mutual_information": 0.3374461374357028, "centered_token_effective_rank": 1.1517070006983368, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.1360706411952726, "token_RMS": 0.3002513368165196, "mean_token_offdiagonal_cosine": 0.8001953709455278, "temperature": 0.4358973503112793 }, { "layer": 15, "head": 5, "slices": 8, "native_nodes": 33555, "mass": [ 0.006173917747215816, 0.559102233814772, 0.39342279660472584, 0.012365436345031657, 0.006510919040110594, 0.007309460762261243, 0.0076547350688889225, 0.007460501111991888 ], "mean_node_assignment_entropy": 0.2247098194571195, "normalized_node_entropy": 0.1080625807232802, "effective_mass_slots": 2.510258765680079, "node_slot_mutual_information_nats": 0.6956760222692084, "normalized_node_slot_mutual_information": 0.33454944913104906, "centered_token_effective_rank": 1.083136702064588, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.15935522328915147, "token_RMS": 0.37839159325940586, "mean_token_offdiagonal_cosine": 0.7702446647885479, "temperature": 0.4417566955089569 }, { "layer": 15, "head": 6, "slices": 8, "native_nodes": 33555, "mass": [ 0.003713099969040733, 0.5637819038384477, 0.4086785438581561, 0.008228884417182258, 0.0033608194209625588, 0.003890422919001965, 0.004465315562048237, 0.0038810081307807044 ], "mean_node_assignment_entropy": 0.13835868027999293, "normalized_node_entropy": 0.06653646063462915, "effective_mass_slots": 2.3059917758948476, "node_slot_mutual_information_nats": 0.6971521751662595, "normalized_node_slot_mutual_information": 0.3352593286191055, "centered_token_effective_rank": 1.0940757146505011, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.1884865036439505, "token_RMS": 0.4331707295518698, "mean_token_offdiagonal_cosine": 0.7221710916551246, "temperature": 0.4192703664302826 }, { "layer": 15, "head": 7, "slices": 8, "native_nodes": 33555, "mass": [ 0.027898106786462398, 0.4635702483620047, 0.36280111315811064, 0.032979132590839434, 0.02820092865551159, 0.02685961546522409, 0.027663525075620064, 0.03002733001855304 ], "mean_node_assignment_entropy": 0.7339124007514378, "normalized_node_entropy": 0.35293726033700434, "effective_mass_slots": 3.8148770446777016, "node_slot_mutual_information_nats": 0.6049960340233228, "normalized_node_slot_mutual_information": 0.29094159268097947, "centered_token_effective_rank": 1.1188966378091312, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.09366487735973973, "token_RMS": 0.26559482748354935, "mean_token_offdiagonal_cosine": 0.8777110322346472, "temperature": 0.45172619819641113 }, { "layer": 15, "head": 8, "slices": 8, "native_nodes": 33555, "mass": [ 0.005678090727083935, 0.5523752872784948, 0.40632508439358234, 0.009751821331865461, 0.006105084435454206, 0.006048330150043636, 0.00703273453085665, 0.006683566288848701 ], "mean_node_assignment_entropy": 0.184366150055876, "normalized_node_entropy": 0.08866137679780094, "effective_mass_slots": 2.4563093091005554, "node_slot_mutual_information_nats": 0.7142937924867431, "normalized_node_slot_mutual_information": 0.3435027040527982, "centered_token_effective_rank": 1.2623399603244876, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.12892757855836165, "token_RMS": 0.34447707231505054, "mean_token_offdiagonal_cosine": 0.814473654749438, "temperature": 0.43090498447418213 }, { "layer": 16, "head": 1, "slices": 8, "native_nodes": 33555, "mass": [ 0.13278888632258756, 0.12018891707353595, 0.14261440260166774, 0.10149117853998196, 0.10501450277624019, 0.0903898055890497, 0.11597692434362497, 0.1915353825389939 ], "mean_node_assignment_entropy": 1.9026194423275584, "normalized_node_entropy": 0.9149665447149646, "effective_mass_slots": 7.791425422189903, "node_slot_mutual_information_nats": 0.15040438184177396, "normalized_node_slot_mutual_information": 0.07232921860369912, "centered_token_effective_rank": 1.0167050009323024, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.13418429874572324, "token_RMS": 0.23476661165925325, "mean_token_offdiagonal_cosine": 0.6828085647522977, "temperature": 0.4380427598953247 }, { "layer": 16, "head": 2, "slices": 8, "native_nodes": 33555, "mass": [ 0.04474579618420351, 0.03627680234118451, 0.04113708587518687, 0.03179811837786961, 0.03342188940698653, 0.5192713027186975, 0.03828054213614645, 0.2550684625620108 ], "mean_node_assignment_entropy": 0.788583618159735, "normalized_node_entropy": 0.37922855841510855, "effective_mass_slots": 4.168282824402981, "node_slot_mutual_information_nats": 0.6389205401319566, "normalized_node_slot_mutual_information": 0.3072558315901573, "centered_token_effective_rank": 1.0131757688881964, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.16794854389136388, "token_RMS": 0.2937340990580954, "mean_token_offdiagonal_cosine": 0.6944983793905201, "temperature": 0.45505326986312866 }, { "layer": 16, "head": 3, "slices": 8, "native_nodes": 33555, "mass": [ 0.02007400265818261, 0.020441042589421785, 0.021270076287895078, 0.019458689253404375, 0.020457927167391924, 0.522721077885968, 0.02196944295066923, 0.35360774103344716 ], "mean_node_assignment_entropy": 0.5091140689078854, "normalized_node_entropy": 0.24483211415340275, "effective_mass_slots": 3.276167365437268, "node_slot_mutual_information_nats": 0.6775601841287867, "normalized_node_slot_mutual_information": 0.3258375725154712, "centered_token_effective_rank": 1.063708165295879, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.13986867803860345, "token_RMS": 0.30895014455023256, "mean_token_offdiagonal_cosine": 0.755639095470384, "temperature": 0.45216891169548035 }, { "layer": 16, "head": 4, "slices": 8, "native_nodes": 33555, "mass": [ 0.022572916196390676, 0.021881015072384498, 0.02232442762601883, 0.021457594648246297, 0.019238258383883242, 0.4978736909291311, 0.02254184503174702, 0.3721102519741992 ], "mean_node_assignment_entropy": 0.5602279318814237, "normalized_node_entropy": 0.26941268636427, "effective_mass_slots": 3.3638632017250094, "node_slot_mutual_information_nats": 0.6528621440412453, "normalized_node_slot_mutual_information": 0.31396032586414696, "centered_token_effective_rank": 1.1159291935733067, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.12205726454122956, "token_RMS": 0.24844560364068077, "mean_token_offdiagonal_cosine": 0.7864563838834354, "temperature": 0.4385828673839569 }, { "layer": 16, "head": 5, "slices": 8, "native_nodes": 33555, "mass": [ 0.022000306404289814, 0.02039627450342204, 0.02165793606565855, 0.01940259802283399, 0.019761427236259267, 0.5273212271214918, 0.020750853208340806, 0.34870937745543407 ], "mean_node_assignment_entropy": 0.5093643894602768, "normalized_node_entropy": 0.24495249289325868, "effective_mass_slots": 3.272772751422811, "node_slot_mutual_information_nats": 0.6762731723689914, "normalized_node_slot_mutual_information": 0.3252186506876637, "centered_token_effective_rank": 1.0388673804881623, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.14535476925667967, "token_RMS": 0.2808914050473951, "mean_token_offdiagonal_cosine": 0.7215648701064836, "temperature": 0.4385589063167572 }, { "layer": 16, "head": 6, "slices": 8, "native_nodes": 33555, "mass": [ 0.0057640434945097066, 0.006781259979230916, 0.0054923789136611095, 0.007881566918276036, 0.007841152939614557, 0.5496850604486848, 0.00589510660572875, 0.4106594304596683 ], "mean_node_assignment_entropy": 0.20706383609941198, "normalized_node_entropy": 0.09957665649602226, "effective_mass_slots": 2.4425434850407846, "node_slot_mutual_information_nats": 0.6859760721736472, "normalized_node_slot_mutual_information": 0.3298847591644702, "centered_token_effective_rank": 1.0833666554044845, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.1644609859212718, "token_RMS": 0.42600418668590306, "mean_token_offdiagonal_cosine": 0.7838943014635019, "temperature": 0.4309614300727844 }, { "layer": 16, "head": 7, "slices": 8, "native_nodes": 33555, "mass": [ 0.38414367685204664, 0.09380607235859766, 0.10067699814045719, 0.09638246782734655, 0.09368328522197483, 0.038075549844188906, 0.10298186820741471, 0.09025008158155719 ], "mean_node_assignment_entropy": 1.5270496815034167, "normalized_node_entropy": 0.7343556675653502, "effective_mass_slots": 6.319042738342526, "node_slot_mutual_information_nats": 0.3165180497311675, "normalized_node_slot_mutual_information": 0.1522130068996672, "centered_token_effective_rank": 1.0484205126754234, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.10605745595896791, "token_RMS": 0.25221313781574645, "mean_token_offdiagonal_cosine": 0.8332381034788661, "temperature": 0.45203495025634766 }, { "layer": 16, "head": 8, "slices": 8, "native_nodes": 33555, "mass": [ 0.029268343928976755, 0.022783317163444994, 0.025651451747254844, 0.02344894626031717, 0.021118028317799138, 0.3318886416344763, 0.02197958886165562, 0.5238616816358023 ], "mean_node_assignment_entropy": 0.5737388247181215, "normalized_node_entropy": 0.2759100523954321, "effective_mass_slots": 3.461170366164967, "node_slot_mutual_information_nats": 0.6678679632728509, "normalized_node_slot_mutual_information": 0.3211765995274515, "centered_token_effective_rank": 1.0833592879690814, "centered_token_rank_upper_bound": 7, "centered_token_RMS": 0.15069266967586895, "token_RMS": 0.2983739053705019, "mean_token_offdiagonal_cosine": 0.7203757058076466, "temperature": 0.42688247561454773 } ] }, { "schema": "ts3_capacity_slice_diagnostics_v1", "variant": "S016", "sample_id": "normal_rodv2_0150__gas_inertia_30deg", "native_nodes": 33555, "layers": 16, "slices": 16, "parameters": 7601028, "checkpoint_sha256": "9aa66ae258c757a3c6270ef933226b9db098de969148805449f5d50cbe81042d", "extraction_source_sha256": "26c5dec82b68ba4bc23d7c3d998b78c5bbed6aae912e6247ea64949db69bd206", "CPU_forward_seconds": 3.080885680999927, "CPU_threads": 2, "CUDA_initialized": false, "hook_max_abs_prediction_difference_on_19_node_test": 0.0, "metrics": [ { "layer": 1, "head": 1, "slices": 16, "native_nodes": 33555, "mass": [ 0.06196547627211452, 0.06252067272097978, 0.061271520117843155, 0.06074767164989247, 0.05672587689848828, 0.06144894267972367, 0.05488775054576002, 0.06722852262698703, 0.07827178000662192, 0.06273535812106258, 0.0628686491539334, 0.06190307984993721, 0.06531027508093283, 0.05835839225585175, 0.059713735127791864, 0.06404229638393902 ], "mean_node_assignment_entropy": 2.7684796652171157, "normalized_node_entropy": 0.9985179709526677, "effective_mass_slots": 15.95029282164913, "node_slot_mutual_information_nats": 0.0009975225693019674, "normalized_node_slot_mutual_information": 0.0003597802159766914, "centered_token_effective_rank": 1.5336025440787653, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.004444306179755319, "token_RMS": 0.24187517656654955, "mean_token_offdiagonal_cosine": 0.9996424197162365, "temperature": 0.49677082896232605 }, { "layer": 1, "head": 2, "slices": 16, "native_nodes": 33555, "mass": [ 0.056193628831931824, 0.06102480751933701, 0.06007013031326078, 0.06429190475067906, 0.06642686511425853, 0.06680091720864326, 0.06771580858799726, 0.05813831834819469, 0.056965334437351224, 0.0565569175053968, 0.05669666191932431, 0.07150767220692403, 0.054075520376839555, 0.06462216720622266, 0.0698956102669026, 0.0690177355278597 ], "mean_node_assignment_entropy": 2.766538971808691, "normalized_node_entropy": 0.9978180137636126, "effective_mass_slots": 15.938088699909047, "node_slot_mutual_information_nats": 0.0021727884612703363, "normalized_node_slot_mutual_information": 0.000783667784493869, "centered_token_effective_rank": 1.1702596102466027, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.007105558303939282, "token_RMS": 0.25043550437776047, "mean_token_offdiagonal_cosine": 0.9991476112760687, "temperature": 0.48905861377716064 }, { "layer": 1, "head": 3, "slices": 16, "native_nodes": 33555, "mass": [ 0.061494590330720744, 0.0661924561465387, 0.06073568202072241, 0.06989627129871094, 0.06494325637271163, 0.06778646361273193, 0.06924098596596323, 0.06016912024216722, 0.06057821985791007, 0.06074357103473684, 0.053394844163735994, 0.0599153725844198, 0.0622183123094157, 0.0565315258007143, 0.06658251501443294, 0.05957681346751861 ], "mean_node_assignment_entropy": 2.7689695543256914, "normalized_node_entropy": 0.9986946610995495, "effective_mass_slots": 15.959675084126111, "node_slot_mutual_information_nats": 0.0010956793512799656, "normalized_node_slot_mutual_information": 0.00039518279162401075, "centered_token_effective_rank": 1.3029059806624943, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.004639203523706783, "token_RMS": 0.2362423077687346, "mean_token_offdiagonal_cosine": 0.999591160338025, "temperature": 0.4974597990512848 }, { "layer": 1, "head": 4, "slices": 16, "native_nodes": 33555, "mass": [ 0.06449706399716984, 0.06650979064061911, 0.06349121883218449, 0.0610283661465534, 0.05351533986347174, 0.06096095401772069, 0.05998505435516521, 0.058094228816878826, 0.05520791641633995, 0.05915976853342068, 0.06630700284365641, 0.06905649070703065, 0.06891810999159893, 0.07297795952307878, 0.06594195428288137, 0.0543487808000862 ], "mean_node_assignment_entropy": 2.7660095140250585, "normalized_node_entropy": 0.9976270522339109, "effective_mass_slots": 15.938158405401396, "node_slot_mutual_information_nats": 0.002706619751741357, "normalized_node_slot_mutual_information": 0.0009762067233523433, "centered_token_effective_rank": 1.2494122129996512, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.009363138182286165, "token_RMS": 0.30549207217720464, "mean_token_offdiagonal_cosine": 0.9990308834660027, "temperature": 0.49832624197006226 }, { "layer": 1, "head": 5, "slices": 16, "native_nodes": 33555, "mass": [ 0.05880463925122131, 0.060225813630249474, 0.06410802667981341, 0.06775013944321463, 0.05825416738559585, 0.0626968795319567, 0.06588249093657403, 0.06336570718865372, 0.0621449934507674, 0.06117980287351617, 0.06191887789006348, 0.06207731471704648, 0.06074055762004788, 0.06351235073063682, 0.06339406510458372, 0.06394417373925079 ], "mean_node_assignment_entropy": 2.771046926267253, "normalized_node_entropy": 0.9994439146490927, "effective_mass_slots": 15.98868372269644, "node_slot_mutual_information_nats": 0.0008342784095227707, "normalized_node_slot_mutual_information": 0.00030090233103480826, "centered_token_effective_rank": 2.2414619506864675, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.003757863316331967, "token_RMS": 0.2994463238032624, "mean_token_offdiagonal_cosine": 0.9998509338466202, "temperature": 0.49947014451026917 }, { "layer": 1, "head": 6, "slices": 16, "native_nodes": 33555, "mass": [ 0.062474791688622244, 0.065339595824147, 0.06248375369215612, 0.06463040551402462, 0.05706694840123696, 0.06193157952840061, 0.06276719924403878, 0.05925496841525911, 0.07611242235395205, 0.06287056725123052, 0.0626018483971647, 0.057380456622951416, 0.06732627671107044, 0.055546375120375344, 0.05990506315964946, 0.06230774771346099 ], "mean_node_assignment_entropy": 2.7685020169366346, "normalized_node_entropy": 0.998526032631394, "effective_mass_slots": 15.957300050114945, "node_slot_mutual_information_nats": 0.001414390981908209, "normalized_node_slot_mutual_information": 0.0005101337138692612, "centered_token_effective_rank": 1.5051311105160508, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.006526341053386524, "token_RMS": 0.24857638962548056, "mean_token_offdiagonal_cosine": 0.9992886687261996, "temperature": 0.49799880385398865 }, { "layer": 1, "head": 7, "slices": 16, "native_nodes": 33555, "mass": [ 0.061393808731466486, 0.06458560861241905, 0.058654646024345516, 0.060758333076127435, 0.05511859124480586, 0.058579129923755094, 0.061498767523656496, 0.06966116881083223, 0.0680793424526202, 0.061903198750589986, 0.0696118898103559, 0.06509428702982042, 0.05937232529230204, 0.06744844408095496, 0.06093912844479111, 0.0573013298241237 ], "mean_node_assignment_entropy": 2.7688296173232914, "normalized_node_entropy": 0.9986441894946997, "effective_mass_slots": 15.962069408341375, "node_slot_mutual_information_nats": 0.001385628469735778, "normalized_node_slot_mutual_information": 0.0004997598304505925, "centered_token_effective_rank": 1.7968850242197874, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.005176377162921299, "token_RMS": 0.250871079575196, "mean_token_offdiagonal_cosine": 0.9995633142735654, "temperature": 0.4943622052669525 }, { "layer": 1, "head": 8, "slices": 16, "native_nodes": 33555, "mass": [ 0.06354122263038146, 0.06016149531287739, 0.05963347400732215, 0.0651007909754619, 0.06169768435298517, 0.06432826618875322, 0.06139570873509912, 0.059255777596605816, 0.06255483368397505, 0.06249244939899448, 0.06699353895205497, 0.05943351157199447, 0.06378055932662238, 0.0631299638237253, 0.06393542777246769, 0.0625652955987382 ], "mean_node_assignment_entropy": 2.77054893882097, "normalized_node_entropy": 0.9992643036442984, "effective_mass_slots": 15.990928423871624, "node_slot_mutual_information_nats": 0.0014726491205294678, "normalized_node_slot_mutual_information": 0.0005311458957893392, "centered_token_effective_rank": 1.5555588905268598, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.005567231212264541, "token_RMS": 0.22049954234554459, "mean_token_offdiagonal_cosine": 0.9993255154514404, "temperature": 0.4998367726802826 }, { "layer": 2, "head": 1, "slices": 16, "native_nodes": 33555, "mass": [ 0.06632590327445752, 0.0633764404089098, 0.057753843062748375, 0.06433257384656207, 0.06248799426132246, 0.06458668622516196, 0.06493700789438951, 0.06227903314419962, 0.06660537911703078, 0.05925401188893905, 0.05951827862497734, 0.060451475475352595, 0.0632829026040564, 0.0622537327491523, 0.064533205488798, 0.05802153166693807 ], "mean_node_assignment_entropy": 2.7695335202991407, "normalized_node_entropy": 0.9988980688278309, "effective_mass_slots": 15.984955583314601, "node_slot_mutual_information_nats": 0.002114483560983871, "normalized_node_slot_mutual_information": 0.0007626387368681667, "centered_token_effective_rank": 1.1064364195238914, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.010513547559292987, "token_RMS": 0.21393542281463385, "mean_token_offdiagonal_cosine": 0.9974554042326813, "temperature": 0.5004863739013672 }, { "layer": 2, "head": 2, "slices": 16, "native_nodes": 33555, "mass": [ 0.059524812410799155, 0.06718162484150621, 0.06409950453908499, 0.06532807237195411, 0.06370030876608226, 0.06360496555321388, 0.06278421749431497, 0.06342002006871852, 0.06281059627103265, 0.06323549470948472, 0.0591063939649992, 0.064508263323239, 0.060348087447070806, 0.06299640818644327, 0.05980213355534462, 0.05754909606195566 ], "mean_node_assignment_entropy": 2.7698832120881174, "normalized_node_entropy": 0.9990241934802799, "effective_mass_slots": 15.987431588520995, "node_slot_mutual_information_nats": 0.001919675747234617, "normalized_node_slot_mutual_information": 0.0006923766701625493, "centered_token_effective_rank": 1.2045875132724009, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.009631653855005239, "token_RMS": 0.21941488987643115, "mean_token_offdiagonal_cosine": 0.998035943552185, "temperature": 0.5005801916122437 }, { "layer": 2, "head": 3, "slices": 16, "native_nodes": 33555, "mass": [ 0.05657775838801232, 0.06089056143805, 0.07182654997925346, 0.057096692667270695, 0.06541290394045915, 0.05214292084887866, 0.05542339961162588, 0.06342949402750186, 0.05765441864375883, 0.07709150829021737, 0.0666608689861066, 0.06704741017478018, 0.05848660854207228, 0.060440301975076306, 0.06052613740889882, 0.0692924651232229 ], "mean_node_assignment_entropy": 2.765860373080707, "normalized_node_entropy": 0.9975732610087087, "effective_mass_slots": 15.916321938887462, "node_slot_mutual_information_nats": 0.0014847466535852227, "normalized_node_slot_mutual_information": 0.0005355091585259711, "centered_token_effective_rank": 1.6530899731108195, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.008138493140664857, "token_RMS": 0.23242436446839793, "mean_token_offdiagonal_cosine": 0.9988244686264198, "temperature": 0.49638259410858154 }, { "layer": 2, "head": 4, "slices": 16, "native_nodes": 33555, "mass": [ 0.06490825328283703, 0.06522263886287394, 0.06155088438988709, 0.0635579220866228, 0.060090790118679634, 0.0648739243130795, 0.06019483725878259, 0.06516316030676861, 0.06747278823234022, 0.05965953524042803, 0.06294860762418107, 0.05979134776714022, 0.05928122506895624, 0.06529380683249413, 0.06244750838632947, 0.05754277037769987 ], "mean_node_assignment_entropy": 2.7703436232102083, "normalized_node_entropy": 0.9991902516909327, "effective_mass_slots": 15.984353572085965, "node_slot_mutual_information_nats": 0.0012667188270860663, "normalized_node_slot_mutual_information": 0.0004568722425094381, "centered_token_effective_rank": 1.5736916903716451, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.006448292625705881, "token_RMS": 0.2436343521043639, "mean_token_offdiagonal_cosine": 0.9992779602492976, "temperature": 0.4996732175350189 }, { "layer": 2, "head": 5, "slices": 16, "native_nodes": 33555, "mass": [ 0.06242907333560094, 0.06187154994029932, 0.06525915899143153, 0.06334830351421453, 0.06286690999591267, 0.05731507710809414, 0.05925982126572274, 0.06510790164915908, 0.062865794000856, 0.06589352392440687, 0.06440550003874108, 0.06194032790088526, 0.05998781304037654, 0.06603158103428962, 0.06230393090025712, 0.0591137330729868 ], "mean_node_assignment_entropy": 2.770272645792437, "normalized_node_entropy": 0.9991646519987742, "effective_mass_slots": 15.98726615587983, "node_slot_mutual_information_nats": 0.0015198943209449034, "normalized_node_slot_mutual_information": 0.0005481859998756277, "centered_token_effective_rank": 1.3498741037959723, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.007455420248095231, "token_RMS": 0.25222211462984917, "mean_token_offdiagonal_cosine": 0.9990853118473317, "temperature": 0.4968326687812805 }, { "layer": 2, "head": 6, "slices": 16, "native_nodes": 33555, "mass": [ 0.06309324820241316, 0.061616290553198574, 0.06403353487673911, 0.06061936558301004, 0.05938267494188107, 0.0679576171697243, 0.0570444226201672, 0.06175138574821319, 0.05982777018184954, 0.06489321354928164, 0.06232764583351342, 0.05905746821341296, 0.06573927113987564, 0.06574636961975568, 0.06455173944140437, 0.06235798219311275 ], "mean_node_assignment_entropy": 2.770246619278784, "normalized_node_entropy": 0.9991552649182295, "effective_mass_slots": 15.98375509946375, "node_slot_mutual_information_nats": 0.001326280904915933, "normalized_node_slot_mutual_information": 0.00047835472108698587, "centered_token_effective_rank": 1.340428116651783, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.008785408829303095, "token_RMS": 0.20928411271749664, "mean_token_offdiagonal_cosine": 0.9982822244627503, "temperature": 0.49954286217689514 }, { "layer": 2, "head": 7, "slices": 16, "native_nodes": 33555, "mass": [ 0.06460801482060737, 0.06375108309719624, 0.06413298661696663, 0.06412392658128893, 0.06057752293686258, 0.06533650160257692, 0.05873742522467819, 0.06166330967667454, 0.06516270527968356, 0.06246108609237662, 0.0631524043901811, 0.05804593107485324, 0.06307642136111627, 0.06605567744748636, 0.05853515694767454, 0.0605798464866291 ], "mean_node_assignment_entropy": 2.770582264126903, "normalized_node_entropy": 0.9992763232076999, "effective_mass_slots": 15.987423013640946, "node_slot_mutual_information_nats": 0.0012200873569843118, "normalized_node_slot_mutual_information": 0.0004400534948431473, "centered_token_effective_rank": 1.3237298138538345, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.007752824721636567, "token_RMS": 0.23291118040218922, "mean_token_offdiagonal_cosine": 0.9988759104719087, "temperature": 0.5003067851066589 }, { "layer": 2, "head": 8, "slices": 16, "native_nodes": 33555, "mass": [ 0.06483077808255376, 0.058946209232141215, 0.06560387432224402, 0.06303625259953095, 0.06079075454548723, 0.05413054983100918, 0.05973513223604977, 0.057766341672597706, 0.06507450938613364, 0.07423469591067029, 0.06273151751906857, 0.057904630260062706, 0.06245243949919713, 0.06704830784895188, 0.0662149801133127, 0.0594990273565387 ], "mean_node_assignment_entropy": 2.7685270601176986, "normalized_node_entropy": 0.9985350650496763, "effective_mass_slots": 15.957041110590007, "node_slot_mutual_information_nats": 0.0013731206430507292, "normalized_node_slot_mutual_information": 0.0004952485855678879, "centered_token_effective_rank": 1.2053859697770255, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.007234064134099025, "token_RMS": 0.2012305446339131, "mean_token_offdiagonal_cosine": 0.9988193312981571, "temperature": 0.4995689392089844 }, { "layer": 3, "head": 1, "slices": 16, "native_nodes": 33555, "mass": [ 0.06309062485120866, 0.06983733624430834, 0.05888263681380853, 0.06987245633094646, 0.06031447572298065, 0.057666325307144634, 0.06600279323018408, 0.06257166323661272, 0.05906316957054453, 0.061714739233117724, 0.05833187080956379, 0.059902888061063794, 0.06740613677170482, 0.05982654818271653, 0.06313765991155534, 0.06237867568135084 ], "mean_node_assignment_entropy": 2.767856478183313, "normalized_node_entropy": 0.9982932037418643, "effective_mass_slots": 15.970951116899581, "node_slot_mutual_information_nats": 0.002915038744805365, "normalized_node_slot_mutual_information": 0.0010513779852824722, "centered_token_effective_rank": 1.0709948202335347, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.013346926545407659, "token_RMS": 0.27160400033697113, "mean_token_offdiagonal_cosine": 0.9976435603064272, "temperature": 0.4986949861049652 }, { "layer": 3, "head": 2, "slices": 16, "native_nodes": 33555, "mass": [ 0.06303296422799837, 0.06430626167787949, 0.05945817430292311, 0.056744862952347465, 0.0684471912766432, 0.06102404199210885, 0.0592728704094194, 0.06739937639797325, 0.05846773273848756, 0.06776121942266197, 0.06228640522714024, 0.0600380868854279, 0.06651208161177552, 0.06760390771839075, 0.059511134081946596, 0.058133689271051045 ], "mean_node_assignment_entropy": 2.7663372244279527, "normalized_node_entropy": 0.9977452487771867, "effective_mass_slots": 15.969682518766554, "node_slot_mutual_information_nats": 0.004354857749548913, "normalized_node_slot_mutual_information": 0.001570682919762772, "centered_token_effective_rank": 1.0899039755484594, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.013626845879714885, "token_RMS": 0.21526047184239303, "mean_token_offdiagonal_cosine": 0.9957704709524926, "temperature": 0.49857327342033386 }, { "layer": 3, "head": 3, "slices": 16, "native_nodes": 33555, "mass": [ 0.06583496679795728, 0.06717534953632008, 0.05976447651448344, 0.06354821843779704, 0.06112315215091559, 0.061619629075209764, 0.06923411158125138, 0.05782104520921806, 0.06422068517432539, 0.05712881496978524, 0.06269238396331106, 0.057800840135569936, 0.06165649993582448, 0.06293972916795643, 0.06478532391828319, 0.06265477328979654 ], "mean_node_assignment_entropy": 2.7688908067670654, "normalized_node_entropy": 0.9986662589214717, "effective_mass_slots": 15.978084213066195, "node_slot_mutual_information_nats": 0.0023272398425575602, "normalized_node_slot_mutual_information": 0.0008393743449542511, "centered_token_effective_rank": 1.1835627788060739, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.01067238964611937, "token_RMS": 0.1873769608485331, "mean_token_offdiagonal_cosine": 0.9965726624661438, "temperature": 0.4963184893131256 }, { "layer": 3, "head": 4, "slices": 16, "native_nodes": 33555, "mass": [ 0.05787461753121397, 0.06328617495600805, 0.06206115625644384, 0.05987593538469644, 0.062073765206716223, 0.06439866433032507, 0.06392757568495619, 0.05992101075960651, 0.0652719696377246, 0.057246733058175715, 0.06429603818213953, 0.06265425713775304, 0.06121653595068046, 0.06524440279881888, 0.06363227510606367, 0.06701888807074641 ], "mean_node_assignment_entropy": 2.770407042346937, "normalized_node_entropy": 0.9992131253094467, "effective_mass_slots": 15.985702201880557, "node_slot_mutual_information_nats": 0.0012876680008115926, "normalized_node_slot_mutual_information": 0.0004644280597705726, "centered_token_effective_rank": 1.3157223444929365, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.008097958376084076, "token_RMS": 0.2601525385256469, "mean_token_offdiagonal_cosine": 0.9990732796747079, "temperature": 0.4982866048812866 }, { "layer": 3, "head": 5, "slices": 16, "native_nodes": 33555, "mass": [ 0.06168676972957646, 0.059060077581817215, 0.06136760175929768, 0.06658297961181153, 0.058786619016313636, 0.06006934531282835, 0.06438447932346783, 0.0657407952048192, 0.06816175151519192, 0.05420536984697999, 0.06677239686661463, 0.06351613950119202, 0.05842850802441571, 0.061337269777344654, 0.06519769778675835, 0.0647021988190565 ], "mean_node_assignment_entropy": 2.7694622914299627, "normalized_node_entropy": 0.9988723784437481, "effective_mass_slots": 15.972301970302487, "node_slot_mutual_information_nats": 0.0013938038220993043, "normalized_node_slot_mutual_information": 0.0005027084655286874, "centered_token_effective_rank": 1.382893770484574, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.00791469258019498, "token_RMS": 0.2568030525966295, "mean_token_offdiagonal_cosine": 0.9990162373392875, "temperature": 0.49928414821624756 }, { "layer": 3, "head": 6, "slices": 16, "native_nodes": 33555, "mass": [ 0.06243521312470675, 0.06776691839829445, 0.06574654221088427, 0.06643577918250432, 0.06737084095250422, 0.0589538922025752, 0.0626367596151967, 0.06575324806348522, 0.054351214703376165, 0.05857068435780925, 0.06103414246300936, 0.06254058178709083, 0.06044168616483365, 0.06433968209135445, 0.06316078331289837, 0.05846203119339841 ], "mean_node_assignment_entropy": 2.7693126272875443, "normalized_node_entropy": 0.9988183985147316, "effective_mass_slots": 15.973004383409949, "node_slot_mutual_information_nats": 0.0015874439464536039, "normalized_node_slot_mutual_information": 0.0005725493773094548, "centered_token_effective_rank": 1.0908251890197567, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.011459378626762932, "token_RMS": 0.1981077566273855, "mean_token_offdiagonal_cosine": 0.9965244400314759, "temperature": 0.49964720010757446 }, { "layer": 3, "head": 7, "slices": 16, "native_nodes": 33555, "mass": [ 0.05916364041619605, 0.062458580088427126, 0.06455100346381609, 0.06311387213202635, 0.06600677213431595, 0.058442053677312415, 0.06607948140169181, 0.06153845175459033, 0.06304153322312005, 0.05758892572486992, 0.06034768074084778, 0.06231099191096639, 0.06391182220842778, 0.059746680304912106, 0.06652131402786506, 0.06517719693394217 ], "mean_node_assignment_entropy": 2.770471851168574, "normalized_node_entropy": 0.999236500150842, "effective_mass_slots": 15.984563866761963, "node_slot_mutual_information_nats": 0.0011516470649790556, "normalized_node_slot_mutual_information": 0.0004153688773749033, "centered_token_effective_rank": 1.157446872120497, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.00886362007828137, "token_RMS": 0.22433057749124036, "mean_token_offdiagonal_cosine": 0.9983620994515223, "temperature": 0.499933123588562 }, { "layer": 3, "head": 8, "slices": 16, "native_nodes": 33555, "mass": [ 0.05918751807475104, 0.05829501506127112, 0.0665947216481188, 0.05943183455488007, 0.06218203870970115, 0.06267604556336007, 0.06047582853945743, 0.059749769727957446, 0.06256220230152418, 0.06442026995747478, 0.06747206133577849, 0.06054953031542759, 0.06329435400921264, 0.06497417400138529, 0.0630666591043871, 0.06506797707777161 ], "mean_node_assignment_entropy": 2.770777508506228, "normalized_node_entropy": 0.9993467427321533, "effective_mass_slots": 15.98563389555903, "node_slot_mutual_information_nats": 0.0009129288689231352, "normalized_node_slot_mutual_information": 0.00032926948796994443, "centered_token_effective_rank": 1.4469306971703035, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.005300888618658299, "token_RMS": 0.22830093378332011, "mean_token_offdiagonal_cosine": 0.9994501089411066, "temperature": 0.49954038858413696 }, { "layer": 4, "head": 1, "slices": 16, "native_nodes": 33555, "mass": [ 0.062469537077784416, 0.06541021800026989, 0.05964505882516576, 0.06140147003693102, 0.05954834209937414, 0.05964163821376555, 0.06832826299765767, 0.061754230098670364, 0.057237526638739475, 0.06466610975493584, 0.05608917864306267, 0.06616985510594531, 0.06452552747142917, 0.06333408086876936, 0.059017925711373355, 0.07076103786982717 ], "mean_node_assignment_entropy": 2.7692961327867582, "normalized_node_entropy": 0.9988124493811101, "effective_mass_slots": 15.969313460695856, "node_slot_mutual_information_nats": 0.0013728392046279403, "normalized_node_slot_mutual_information": 0.0004951470781136696, "centered_token_effective_rank": 1.6722452422294662, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.007349659572400314, "token_RMS": 0.280638279756316, "mean_token_offdiagonal_cosine": 0.999311290172485, "temperature": 0.4952206611633301 }, { "layer": 4, "head": 2, "slices": 16, "native_nodes": 33555, "mass": [ 0.058098873622063785, 0.06211083377139026, 0.06293801855295342, 0.06745139266171517, 0.05770679180285664, 0.06431176581500078, 0.061365010813955505, 0.06689599314239639, 0.0599237419204125, 0.06624471695319034, 0.06294250395390462, 0.061487305470753525, 0.06485804034869676, 0.05733867690750959, 0.05839422956961684, 0.06793210448353323 ], "mean_node_assignment_entropy": 2.7693501008496217, "normalized_node_entropy": 0.9988319142452751, "effective_mass_slots": 15.975490274706164, "node_slot_mutual_information_nats": 0.001705589065496227, "normalized_node_slot_mutual_information": 0.0006151612216464621, "centered_token_effective_rank": 1.2377630728941564, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.008325046329022593, "token_RMS": 0.24472130898828387, "mean_token_offdiagonal_cosine": 0.9988761735130044, "temperature": 0.4981158971786499 }, { "layer": 4, "head": 3, "slices": 16, "native_nodes": 33555, "mass": [ 0.061001147789653876, 0.06542075850172772, 0.06371908620301861, 0.059626784035637634, 0.06129344680784108, 0.05989176250495724, 0.06524850167150896, 0.06154622855904187, 0.06535372632563852, 0.058173691924545384, 0.056355776246334574, 0.061552383175591315, 0.056274158436199394, 0.06722680885289789, 0.06653550069592089, 0.07078023807664259 ], "mean_node_assignment_entropy": 2.768611311720062, "normalized_node_entropy": 0.9985654523919054, "effective_mass_slots": 15.968290370081062, "node_slot_mutual_information_nats": 0.0019935921827745062, "normalized_node_slot_mutual_information": 0.000719036388910946, "centered_token_effective_rank": 1.1971250611755337, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.010843355288820352, "token_RMS": 0.2585268083158846, "mean_token_offdiagonal_cosine": 0.9981649295562665, "temperature": 0.5019899606704712 }, { "layer": 4, "head": 4, "slices": 16, "native_nodes": 33555, "mass": [ 0.0634419595157402, 0.059850889719348786, 0.06044269728622392, 0.06325731306575376, 0.06321495385834151, 0.06166001108116017, 0.06246792140178523, 0.06219241737313552, 0.06455964814766635, 0.06303132379588809, 0.06140826424080593, 0.06555567905340122, 0.06297017376058912, 0.06602696638591526, 0.06023272013383586, 0.05968706125479273 ], "mean_node_assignment_entropy": 2.7655312760854978, "normalized_node_entropy": 0.9974545643579686, "effective_mass_slots": 15.992999634981768, "node_slot_mutual_information_nats": 0.0066198275996107725, "normalized_node_slot_mutual_information": 0.0023875981123745883, "centered_token_effective_rank": 1.0245595723994396, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.020679092886092994, "token_RMS": 0.2700235390856299, "mean_token_offdiagonal_cosine": 0.9938138459855598, "temperature": 0.49345311522483826 }, { "layer": 4, "head": 5, "slices": 16, "native_nodes": 33555, "mass": [ 0.06104504115343307, 0.05799443177660146, 0.06219617048270957, 0.060340817810001966, 0.061270742302874555, 0.07357810929122297, 0.06462536994848002, 0.06350507475030509, 0.06220478000461549, 0.061726342996929605, 0.06141625536330107, 0.05994196378717105, 0.06171006089532416, 0.06661032734047707, 0.06539189020281949, 0.05644262168568109 ], "mean_node_assignment_entropy": 2.7696023515499957, "normalized_node_entropy": 0.9989228944538976, "effective_mass_slots": 15.971527557679464, "node_slot_mutual_information_nats": 0.0012052578045005369, "normalized_node_slot_mutual_information": 0.0004347048643864111, "centered_token_effective_rank": 1.1821521174474878, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.008494963736728572, "token_RMS": 0.17394794490025683, "mean_token_offdiagonal_cosine": 0.9974870537829279, "temperature": 0.5008520483970642 }, { "layer": 4, "head": 6, "slices": 16, "native_nodes": 33555, "mass": [ 0.05974052880537206, 0.06351993353056261, 0.0667450904860624, 0.05883065420513074, 0.061839024252071585, 0.06091437657090186, 0.06590407575672011, 0.05826923797527949, 0.06297544767488099, 0.05677447252462916, 0.05537618789091904, 0.06498495740087426, 0.06229509536375091, 0.06930679325605536, 0.07025710292312609, 0.06226702153764864 ], "mean_node_assignment_entropy": 2.7685479856682833, "normalized_node_entropy": 0.9985426123466904, "effective_mass_slots": 15.965680838564118, "node_slot_mutual_information_nats": 0.001893485286881269, "normalized_node_slot_mutual_information": 0.0006829304583449557, "centered_token_effective_rank": 1.1493482824656645, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.010882423140282375, "token_RMS": 0.22255599365304876, "mean_token_offdiagonal_cosine": 0.9975431101335205, "temperature": 0.4991495609283447 }, { "layer": 4, "head": 7, "slices": 16, "native_nodes": 33555, "mass": [ 0.061547806449462075, 0.06132064082706548, 0.06047422151672309, 0.06631783128013431, 0.058429416992362, 0.06460400855182856, 0.0662509449049464, 0.0621578408561446, 0.05765426883291086, 0.06403195768774594, 0.06155126687056563, 0.06653885541558976, 0.058597649102873746, 0.06494069184001627, 0.06174585731937924, 0.06383674178461776 ], "mean_node_assignment_entropy": 2.7679126874035633, "normalized_node_entropy": 0.9983134769326911, "effective_mass_slots": 15.984253551278961, "node_slot_mutual_information_nats": 0.0036913971945740087, "normalized_node_slot_mutual_information": 0.0013313901066408385, "centered_token_effective_rank": 1.0963155251539751, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.015087447934422425, "token_RMS": 0.23430772361822733, "mean_token_offdiagonal_cosine": 0.995626327430271, "temperature": 0.4959760904312134 }, { "layer": 4, "head": 8, "slices": 16, "native_nodes": 33555, "mass": [ 0.060135360314693805, 0.06294180758407354, 0.06638099078709048, 0.062287004827462765, 0.055800614157957215, 0.0631524110551812, 0.06454868680086848, 0.0633932026024592, 0.062268599128868805, 0.05790213484601832, 0.06672771281393856, 0.06361415384579053, 0.059846135519412744, 0.0632287627700819, 0.06111558228242665, 0.06665684086395667 ], "mean_node_assignment_entropy": 2.7701407414110273, "normalized_node_entropy": 0.9991170775495414, "effective_mass_slots": 15.982144498646607, "node_slot_mutual_information_nats": 0.0013313888372881166, "normalized_node_slot_mutual_information": 0.00048019701826262223, "centered_token_effective_rank": 1.2779269357529675, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.0077893794823062955, "token_RMS": 0.17960790161379714, "mean_token_offdiagonal_cosine": 0.9982486078910832, "temperature": 0.49910008907318115 }, { "layer": 5, "head": 1, "slices": 16, "native_nodes": 33555, "mass": [ 0.06179159383190014, 0.054976151779131927, 0.059383185942910005, 0.06256614360104036, 0.06705905395157379, 0.06245882043631914, 0.06317256554059719, 0.06903903553376982, 0.0665802115603622, 0.061582210374880327, 0.06398066950704102, 0.06070985879786252, 0.059388363597499085, 0.060157036295707705, 0.0612152834047221, 0.06593981578972759 ], "mean_node_assignment_entropy": 2.770224977707233, "normalized_node_entropy": 0.9991474593712412, "effective_mass_slots": 15.976625021305205, "node_slot_mutual_information_nats": 0.000901740156363573, "normalized_node_slot_mutual_information": 0.0003252340129390413, "centered_token_effective_rank": 1.1720599091554251, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.006481311620641054, "token_RMS": 0.2395413071883456, "mean_token_offdiagonal_cosine": 0.9992273307141136, "temperature": 0.5003553628921509 }, { "layer": 5, "head": 2, "slices": 16, "native_nodes": 33555, "mass": [ 0.06485186819491719, 0.06811494061318404, 0.06317674476021076, 0.061146675133968036, 0.05588215248446372, 0.06142260976265625, 0.060881797241158324, 0.06759296480342557, 0.06537520568426507, 0.06325128105061187, 0.06588603153651615, 0.059959411255059415, 0.05949944670054955, 0.06190394220795722, 0.06095065273318982, 0.06010427559939516 ], "mean_node_assignment_entropy": 2.7700018764486822, "normalized_node_entropy": 0.9990669926014093, "effective_mass_slots": 15.97989845572141, "node_slot_mutual_information_nats": 0.0013297094087691974, "normalized_node_slot_mutual_information": 0.00047959129246367917, "centered_token_effective_rank": 1.160883871447239, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.007675101880581353, "token_RMS": 0.22215408883919355, "mean_token_offdiagonal_cosine": 0.9987796788181836, "temperature": 0.4983445107936859 }, { "layer": 5, "head": 3, "slices": 16, "native_nodes": 33555, "mass": [ 0.06256587214455965, 0.05786526254224912, 0.06622584184947836, 0.06100544254478018, 0.06529763150438966, 0.06280674370025988, 0.06066310626417598, 0.05899688324087104, 0.062240483080034305, 0.060495216653112895, 0.060952575080681796, 0.06648771919331187, 0.06626951033850395, 0.06209200056267822, 0.06185104643698155, 0.06418466516757239 ], "mean_node_assignment_entropy": 2.7705793794344706, "normalized_node_entropy": 0.9992752827748331, "effective_mass_slots": 15.987067319639344, "node_slot_mutual_information_nats": 0.0012007234382327425, "normalized_node_slot_mutual_information": 0.00043306943745438084, "centered_token_effective_rank": 1.078501550248671, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.0075712801485367685, "token_RMS": 0.18830131303307943, "mean_token_offdiagonal_cosine": 0.9983011233864088, "temperature": 0.500033974647522 }, { "layer": 5, "head": 4, "slices": 16, "native_nodes": 33555, "mass": [ 0.06060999430701605, 0.05711588277922108, 0.06823413808461078, 0.06133397764344095, 0.05951457447607601, 0.06383971879215732, 0.05358060431776166, 0.06537983349888446, 0.062422640239973924, 0.06656392515327372, 0.06792039849855491, 0.05889456382546157, 0.06503067099056431, 0.06164413399526977, 0.06403896051698124, 0.0638759828393416 ], "mean_node_assignment_entropy": 2.769210988540356, "normalized_node_entropy": 0.998781740085599, "effective_mass_slots": 15.969477958416705, "node_slot_mutual_information_nats": 0.0014682842616458025, "normalized_node_slot_mutual_information": 0.0005295716057229281, "centered_token_effective_rank": 1.165090622504987, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.00878099288441474, "token_RMS": 0.2617187251952176, "mean_token_offdiagonal_cosine": 0.9988468643523567, "temperature": 0.4997671842575073 }, { "layer": 5, "head": 5, "slices": 16, "native_nodes": 33555, "mass": [ 0.06567131405519473, 0.06296671119101835, 0.0625619731340549, 0.06612142478233123, 0.057219647186609976, 0.06496086924970638, 0.06116074963087404, 0.06439722921660984, 0.0685600958008504, 0.057984201577781405, 0.06053535491183695, 0.06081916851562886, 0.06159691742419166, 0.06284756684985837, 0.0601400420768013, 0.062456734167616476 ], "mean_node_assignment_entropy": 2.7695786063129195, "normalized_node_entropy": 0.998914330169954, "effective_mass_slots": 15.983002918253836, "node_slot_mutual_information_nats": 0.0019472336584414052, "normalized_node_slot_mutual_information": 0.0007023160856213722, "centered_token_effective_rank": 1.3628627383781045, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.009167608568363406, "token_RMS": 0.22209693664915042, "mean_token_offdiagonal_cosine": 0.9982145192410797, "temperature": 0.4976766109466553 }, { "layer": 5, "head": 6, "slices": 16, "native_nodes": 33555, "mass": [ 0.06031265640488379, 0.06702964863821924, 0.07441689813812946, 0.061199868187883394, 0.06717920845646187, 0.0670393214855388, 0.053685670259980826, 0.05776274684717236, 0.06298941047091967, 0.05910173171117842, 0.05766414587421063, 0.06564268662497533, 0.062169474594582566, 0.05794512536895735, 0.06442005060731948, 0.0614413567650089 ], "mean_node_assignment_entropy": 2.7677086840306173, "normalized_node_entropy": 0.9982398982690727, "effective_mass_slots": 15.951640811466085, "node_slot_mutual_information_nats": 0.00185301210148614, "normalized_node_slot_mutual_information": 0.0006683328423803227, "centered_token_effective_rank": 1.1762313076804645, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.009568763584008958, "token_RMS": 0.24006257813152188, "mean_token_offdiagonal_cosine": 0.9984897715834599, "temperature": 0.4975244998931885 }, { "layer": 5, "head": 7, "slices": 16, "native_nodes": 33555, "mass": [ 0.062426040789206536, 0.06131840596873951, 0.06370842563552662, 0.0593905770066793, 0.06379611694824765, 0.06633934760634136, 0.062283970376513, 0.0638676698340014, 0.0644315265163484, 0.06287904355506786, 0.05978580807228679, 0.061326843459059775, 0.06258635818503099, 0.06325679155320013, 0.06187004087032921, 0.06073303383524843 ], "mean_node_assignment_entropy": 2.7705426139865685, "normalized_node_entropy": 0.9992620224424921, "effective_mass_slots": 15.993916095638296, "node_slot_mutual_information_nats": 0.001665791919516657, "normalized_node_slot_mutual_information": 0.0006008074353598971, "centered_token_effective_rank": 1.2834867449441962, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.01026354509123892, "token_RMS": 0.2285929359081222, "mean_token_offdiagonal_cosine": 0.9978963553647566, "temperature": 0.5000150799751282 }, { "layer": 5, "head": 8, "slices": 16, "native_nodes": 33555, "mass": [ 0.06318885848756572, 0.059675173324215985, 0.05990414021933883, 0.06300006782492552, 0.0558435439228603, 0.06035939606075634, 0.06245703422151491, 0.06624034179940502, 0.06731791099889536, 0.06536428491754125, 0.0632694592824584, 0.057204750365759574, 0.057030845044458024, 0.06895588900360473, 0.06518809721906323, 0.06500020749359599 ], "mean_node_assignment_entropy": 2.7694276628185817, "normalized_node_entropy": 0.9988598888122701, "effective_mass_slots": 15.97127786557052, "node_slot_mutual_information_nats": 0.0013643128365172785, "normalized_node_slot_mutual_information": 0.0004920718408661582, "centered_token_effective_rank": 1.194364708372454, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.006932780553488764, "token_RMS": 0.23109398657721275, "mean_token_offdiagonal_cosine": 0.9990831891076658, "temperature": 0.4987824261188507 }, { "layer": 6, "head": 1, "slices": 16, "native_nodes": 33555, "mass": [ 0.05698223584407205, 0.0684180341925205, 0.061590686810609925, 0.061682558205129134, 0.06512537379500018, 0.06415948813986252, 0.0631958549935218, 0.057925445006250215, 0.06375192975126637, 0.061595157353808924, 0.05901410263928211, 0.06507721880836363, 0.06312973475062338, 0.06699575247947194, 0.06430217561550297, 0.05705425137879596 ], "mean_node_assignment_entropy": 2.7694739621294, "normalized_node_entropy": 0.9988765877587985, "effective_mass_slots": 15.977801108174434, "node_slot_mutual_information_nats": 0.0017263659980768509, "normalized_node_slot_mutual_information": 0.0006226549160461997, "centered_token_effective_rank": 1.153227288891257, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.008673517660630669, "token_RMS": 0.21276344880249737, "mean_token_offdiagonal_cosine": 0.9983094308613109, "temperature": 0.4952649772167206 }, { "layer": 6, "head": 2, "slices": 16, "native_nodes": 33555, "mass": [ 0.060551075711835246, 0.06018998371843804, 0.06097284355087725, 0.059669729249254636, 0.06330604275294129, 0.06606265450222022, 0.0609212753589019, 0.06883926984106459, 0.06104604765470844, 0.06036442038803658, 0.06520366849520134, 0.06241218677042334, 0.06072991108331697, 0.06316292974634384, 0.06449350327455398, 0.06207445820330277 ], "mean_node_assignment_entropy": 2.7702866381412807, "normalized_node_entropy": 0.999169698671846, "effective_mass_slots": 15.98772798901039, "node_slot_mutual_information_nats": 0.001534789116130142, "normalized_node_slot_mutual_information": 0.0005535581616628278, "centered_token_effective_rank": 1.1272259927806092, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.010938956527666457, "token_RMS": 0.266789431463725, "mean_token_offdiagonal_cosine": 0.9982591010311109, "temperature": 0.5002872943878174 }, { "layer": 6, "head": 3, "slices": 16, "native_nodes": 33555, "mass": [ 0.06574600777188973, 0.06940539624514798, 0.06445704853935705, 0.06000319575154556, 0.05995437832891435, 0.0584983613435859, 0.06742844746613659, 0.06435814623631711, 0.066903837196137, 0.06347721876381425, 0.062245502177697504, 0.06554016767600097, 0.05824563429392391, 0.057840563992246255, 0.05625683160310281, 0.05963926248928506 ], "mean_node_assignment_entropy": 2.768653394842455, "normalized_node_entropy": 0.9985806306699009, "effective_mass_slots": 15.96994388795302, "node_slot_mutual_information_nats": 0.0020550537873740282, "normalized_node_slot_mutual_information": 0.0007412039769511482, "centered_token_effective_rank": 1.2192358358092508, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.011476968665372136, "token_RMS": 0.20924693716486917, "mean_token_offdiagonal_cosine": 0.996844587431832, "temperature": 0.4952865242958069 }, { "layer": 6, "head": 4, "slices": 16, "native_nodes": 33555, "mass": [ 0.06471777493779447, 0.05733529134636717, 0.06871693784332439, 0.060693485388620266, 0.06249106685189532, 0.058875003881611494, 0.06311340690338564, 0.07024046512873797, 0.05897832016172361, 0.06748889174294034, 0.060486233380397374, 0.06511956950161962, 0.06287218002360156, 0.05787724006963796, 0.06003724139807139, 0.06095689172603799 ], "mean_node_assignment_entropy": 2.7686615790317224, "normalized_node_entropy": 0.9985835824922182, "effective_mass_slots": 15.971513052078961, "node_slot_mutual_information_nats": 0.0021451221061341563, "normalized_node_slot_mutual_information": 0.000773689256155259, "centered_token_effective_rank": 1.2286231364403861, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.01023441835874035, "token_RMS": 0.21751579184430783, "mean_token_offdiagonal_cosine": 0.997668236124877, "temperature": 0.5000637769699097 }, { "layer": 6, "head": 5, "slices": 16, "native_nodes": 33555, "mass": [ 0.06361496159630603, 0.06416462900261644, 0.06253606049157152, 0.060302662207062104, 0.06142043818141962, 0.062451319994340795, 0.059747990095776145, 0.05783539229756655, 0.061693227118026335, 0.062135046668658225, 0.06130284231293221, 0.06719987771404853, 0.062431215207439265, 0.06497395595671913, 0.06975284820818596, 0.058437532962429914 ], "mean_node_assignment_entropy": 2.770502821986844, "normalized_node_entropy": 0.9992476705123247, "effective_mass_slots": 15.982417863525898, "node_slot_mutual_information_nats": 0.0009864125081180397, "normalized_node_slot_mutual_information": 0.00035577310843318505, "centered_token_effective_rank": 1.1979639411454477, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.008026712736619254, "token_RMS": 0.28279516303670943, "mean_token_offdiagonal_cosine": 0.9991508096000986, "temperature": 0.5018625855445862 }, { "layer": 6, "head": 6, "slices": 16, "native_nodes": 33555, "mass": [ 0.06063481624944699, 0.0688109132830254, 0.06513787795309516, 0.06045456665418592, 0.062159028496232976, 0.05840619836158572, 0.06257396401997616, 0.060493504592734997, 0.06490995971758805, 0.06672898817126326, 0.06140237998307826, 0.06750876223356211, 0.06023067256707475, 0.05900631585054187, 0.06559309848879688, 0.05594895299756657 ], "mean_node_assignment_entropy": 2.7696359946714604, "normalized_node_entropy": 0.9989350286450219, "effective_mass_slots": 15.97506752546015, "node_slot_mutual_information_nats": 0.001393232529066868, "normalized_node_slot_mutual_information": 0.0005025024151224897, "centered_token_effective_rank": 1.2287111873409011, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.009613678370885491, "token_RMS": 0.296798065130999, "mean_token_offdiagonal_cosine": 0.9989821153323741, "temperature": 0.4977319538593292 }, { "layer": 6, "head": 7, "slices": 16, "native_nodes": 33555, "mass": [ 0.0661942282254455, 0.06420885875183856, 0.06791167582174719, 0.06321525285910022, 0.06385975284550481, 0.0605198010432571, 0.0643593328458029, 0.06362740103715472, 0.0612587120493333, 0.06197133487579791, 0.059960142040222486, 0.06540809511958513, 0.0588409102266351, 0.05898087441392777, 0.059810545609997774, 0.05987308240273442 ], "mean_node_assignment_entropy": 2.7699564006840873, "normalized_node_entropy": 0.9990505906863939, "effective_mass_slots": 15.985533365540777, "node_slot_mutual_information_nats": 0.001727747898575771, "normalized_node_slot_mutual_information": 0.0006231533312953982, "centered_token_effective_rank": 1.0933903979536541, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.011115438760934012, "token_RMS": 0.20033132177098228, "mean_token_offdiagonal_cosine": 0.9967881224492203, "temperature": 0.49843502044677734 }, { "layer": 6, "head": 8, "slices": 16, "native_nodes": 33555, "mass": [ 0.062092275791077796, 0.0664836796351702, 0.06655829079863283, 0.05879936394440028, 0.06938443135582838, 0.06344644016179771, 0.0690231596230306, 0.07094704569021115, 0.06219284172272419, 0.0588800782200397, 0.054835334160083314, 0.06256268325720703, 0.05606783132102535, 0.060255955106707414, 0.05958019917975984, 0.05889039065568391 ], "mean_node_assignment_entropy": 2.765044767919723, "normalized_node_entropy": 0.9972790936284398, "effective_mass_slots": 15.955695008705806, "node_slot_mutual_information_nats": 0.004771051419933059, "normalized_node_slot_mutual_information": 0.0017207930558409178, "centered_token_effective_rank": 1.0832614785071724, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.016559465787004297, "token_RMS": 0.24902819977442764, "mean_token_offdiagonal_cosine": 0.9953928198377924, "temperature": 0.4963773190975189 }, { "layer": 7, "head": 1, "slices": 16, "native_nodes": 33555, "mass": [ 0.07358526441616534, 0.059393110327425946, 0.06659748942357135, 0.06076901729896303, 0.06004187786939726, 0.058503378559230886, 0.059989840768565186, 0.05840913268332687, 0.06200841463388407, 0.061969474530991844, 0.06423229892187086, 0.060332917796084004, 0.06728591349099684, 0.0633762566363593, 0.06431762997925308, 0.05918798285475823 ], "mean_node_assignment_entropy": 2.7692948181369372, "normalized_node_entropy": 0.9988119752214158, "effective_mass_slots": 15.969664613018706, "node_slot_mutual_information_nats": 0.00139614280613376, "normalized_node_slot_mutual_information": 0.0005035520756954942, "centered_token_effective_rank": 1.3008604356773488, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.008153295848788246, "token_RMS": 0.22580564119729493, "mean_token_offdiagonal_cosine": 0.9986275310201843, "temperature": 0.5022180676460266 }, { "layer": 7, "head": 2, "slices": 16, "native_nodes": 33555, "mass": [ 0.06002978356795186, 0.06725098753666899, 0.05586250996295679, 0.06821664654205851, 0.057260439767845166, 0.0627133687970056, 0.057783577260673, 0.06824386369916033, 0.057483313140513345, 0.06430401147597593, 0.06427299424821618, 0.059817494596992336, 0.06262274455906933, 0.06140912145802524, 0.06674883404899028, 0.06598030919190527 ], "mean_node_assignment_entropy": 2.768797565310335, "normalized_node_entropy": 0.9986326291746641, "effective_mass_slots": 15.966874966173872, "node_slot_mutual_information_nats": 0.0017186962514825765, "normalized_node_slot_mutual_information": 0.000619888639702091, "centered_token_effective_rank": 1.2199972801247285, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.009133192386221275, "token_RMS": 0.2255261326553594, "mean_token_offdiagonal_cosine": 0.9983057773899622, "temperature": 0.49674177169799805 }, { "layer": 7, "head": 3, "slices": 16, "native_nodes": 33555, "mass": [ 0.06710368913827755, 0.0575672626928341, 0.06989645725536624, 0.05369649320927432, 0.06735844466929072, 0.055996094955259916, 0.0647644365547471, 0.06415315742049749, 0.06151035985314194, 0.06595001056026531, 0.0615717992472979, 0.06060866188330651, 0.06502397471602832, 0.06008006704147145, 0.06665623935519308, 0.0580628516697889 ], "mean_node_assignment_entropy": 2.767391532451168, "normalized_node_entropy": 0.9981255100163523, "effective_mass_slots": 15.958981376668593, "node_slot_mutual_information_nats": 0.0026302340166819427, "normalized_node_slot_mutual_information": 0.0009486563930611244, "centered_token_effective_rank": 1.0687530476769376, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.013102615260912808, "token_RMS": 0.1864079190789801, "mean_token_offdiagonal_cosine": 0.9949755164064774, "temperature": 0.49561262130737305 }, { "layer": 7, "head": 4, "slices": 16, "native_nodes": 33555, "mass": [ 0.06117507753744688, 0.05883432471639814, 0.06255666395023991, 0.06278788033717969, 0.061842464599572426, 0.06322758230391122, 0.06105142572813925, 0.06229074599734541, 0.06959261143716912, 0.05758697975200986, 0.06507045028839749, 0.06389608732226922, 0.06633840536768364, 0.06518096899980938, 0.061908920144243436, 0.0566594118001768 ], "mean_node_assignment_entropy": 2.7700996400907143, "normalized_node_entropy": 0.999102253381794, "effective_mass_slots": 15.979847554330911, "node_slot_mutual_information_nats": 0.0012287604228684579, "normalized_node_slot_mutual_information": 0.0004431816421282375, "centered_token_effective_rank": 1.3707131963162158, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.007416660685707289, "token_RMS": 0.2084894731260097, "mean_token_offdiagonal_cosine": 0.9987128303348508, "temperature": 0.4991716146469116 }, { "layer": 7, "head": 5, "slices": 16, "native_nodes": 33555, "mass": [ 0.06340136217879584, 0.05943371806631783, 0.06235312343380913, 0.06533097206664731, 0.05979660121093136, 0.06025405339991046, 0.06539327748960255, 0.067632874909296, 0.06201303711397213, 0.05848185035850142, 0.06414381854903359, 0.06277033570343986, 0.05949741912978787, 0.061649213018426775, 0.06870735355699636, 0.059140989380552664 ], "mean_node_assignment_entropy": 2.7705602678570007, "normalized_node_entropy": 0.9992683897303233, "effective_mass_slots": 15.981719836951756, "node_slot_mutual_information_nats": 0.0008852910299537164, "normalized_node_slot_mutual_information": 0.00031930124466442736, "centered_token_effective_rank": 1.3266691713567518, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.0073637264113183025, "token_RMS": 0.25082665903442886, "mean_token_offdiagonal_cosine": 0.9991189731299123, "temperature": 0.4992014169692993 }, { "layer": 7, "head": 6, "slices": 16, "native_nodes": 33555, "mass": [ 0.06345484768992617, 0.06598521316050152, 0.06170951895334738, 0.05943865504017567, 0.06841029983761591, 0.053739482729032804, 0.062175440193976715, 0.06523815053303504, 0.06235033770757538, 0.061505696375321, 0.07048668469820128, 0.060908522664936654, 0.059046383098382886, 0.0626189739141525, 0.06243706273472606, 0.060494730609142025 ], "mean_node_assignment_entropy": 2.7696717447240076, "normalized_node_entropy": 0.9989479227509022, "effective_mass_slots": 15.971096619997342, "node_slot_mutual_information_nats": 0.0011088826468075297, "normalized_node_slot_mutual_information": 0.00039994487386926276, "centered_token_effective_rank": 1.1843653680177344, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.007759506193237485, "token_RMS": 0.22030253495476898, "mean_token_offdiagonal_cosine": 0.9986969801239195, "temperature": 0.4992240071296692 }, { "layer": 7, "head": 7, "slices": 16, "native_nodes": 33555, "mass": [ 0.0591810308270253, 0.06711779049146799, 0.059863434549664765, 0.058830661441108864, 0.059225397837457776, 0.0631548842805728, 0.05882108745925243, 0.06152010551587051, 0.058532730332764335, 0.06522313103417297, 0.0666453455556631, 0.07477976445815297, 0.061862278392619194, 0.05709745866267199, 0.06328990680283965, 0.06485499218727984 ], "mean_node_assignment_entropy": 2.7695023612398733, "normalized_node_entropy": 0.99888683057276, "effective_mass_slots": 15.961964600129797, "node_slot_mutual_information_nats": 0.0007063184524191968, "normalized_node_slot_mutual_information": 0.0002547505321483856, "centered_token_effective_rank": 1.7392099115357142, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.004993563943756101, "token_RMS": 0.2470311028462081, "mean_token_offdiagonal_cosine": 0.9996192494908863, "temperature": 0.49839770793914795 }, { "layer": 7, "head": 8, "slices": 16, "native_nodes": 33555, "mass": [ 0.06495665310847477, 0.07207367860059449, 0.06262426353117301, 0.056034708598268104, 0.06572802289187989, 0.06347091281188438, 0.06008335779916599, 0.06328160027931583, 0.05498663879616785, 0.061152382956484966, 0.06278938557661214, 0.0628029482630393, 0.06068803994488812, 0.05834290918255328, 0.06163863962061838, 0.06934585800834889 ], "mean_node_assignment_entropy": 2.7694445889653334, "normalized_node_entropy": 0.9988659936292651, "effective_mass_slots": 15.963925378908783, "node_slot_mutual_information_nats": 0.0008869238747735153, "normalized_node_slot_mutual_information": 0.00031989016894544366, "centered_token_effective_rank": 1.2807666004880045, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.006185452234182271, "token_RMS": 0.2312688416287033, "mean_token_offdiagonal_cosine": 0.9992689049032095, "temperature": 0.49709028005599976 }, { "layer": 8, "head": 1, "slices": 16, "native_nodes": 33555, "mass": [ 0.057687448301314954, 0.06503288815556545, 0.05630870184090503, 0.05541767444860703, 0.0558793677135607, 0.06920390434906214, 0.05419050560462867, 0.06683228247443852, 0.06990058066157147, 0.07239602751982158, 0.06364329992745622, 0.06710665473313965, 0.06042731450588919, 0.06544354932529527, 0.05881619761245401, 0.06171360272437339 ], "mean_node_assignment_entropy": 2.7676612459467194, "normalized_node_entropy": 0.9982227885969754, "effective_mass_slots": 15.936544827194256, "node_slot_mutual_information_nats": 0.0009536427645659984, "normalized_node_slot_mutual_information": 0.0003439539218047518, "centered_token_effective_rank": 1.2818097209999912, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.005702346778736424, "token_RMS": 0.22414864242632054, "mean_token_offdiagonal_cosine": 0.9993354722448957, "temperature": 0.4965299367904663 }, { "layer": 8, "head": 2, "slices": 16, "native_nodes": 33555, "mass": [ 0.06042838787321113, 0.06096000491633552, 0.06183009769969025, 0.06547011997650089, 0.056650206170539855, 0.07122620541878259, 0.06925568191323225, 0.0634836356657803, 0.06218801598700054, 0.06303625683618132, 0.06301971864720746, 0.05811945641716695, 0.06492544876411835, 0.055220214004506656, 0.06358194852953092, 0.06060460116378402 ], "mean_node_assignment_entropy": 2.7698182004623226, "normalized_node_entropy": 0.9990007454927464, "effective_mass_slots": 15.967371545131916, "node_slot_mutual_information_nats": 0.0007291611887305649, "normalized_node_slot_mutual_information": 0.0002629893077475719, "centered_token_effective_rank": 1.7009351362802163, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.004749755264067415, "token_RMS": 0.21495254845531048, "mean_token_offdiagonal_cosine": 0.999512118755024, "temperature": 0.4958185851573944 }, { "layer": 8, "head": 3, "slices": 16, "native_nodes": 33555, "mass": [ 0.06256773390620836, 0.06368442958612464, 0.05309784025033777, 0.059686627884461614, 0.060333238639111775, 0.06731081540887694, 0.05762952690889398, 0.06270938212821636, 0.060194269151734515, 0.06606331851703633, 0.06738023692177174, 0.06768574188590369, 0.05821505977941899, 0.0644082507567938, 0.06269011682379869, 0.0663434114952749 ], "mean_node_assignment_entropy": 2.7696505876210638, "normalized_node_entropy": 0.9989402919390281, "effective_mass_slots": 15.966943412192418, "node_slot_mutual_information_nats": 0.0008699606826483297, "normalized_node_slot_mutual_information": 0.00031377199065628066, "centered_token_effective_rank": 1.534885081964246, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.0056628194730729805, "token_RMS": 0.20746562141129463, "mean_token_offdiagonal_cosine": 0.9992324836254904, "temperature": 0.5000587701797485 }, { "layer": 8, "head": 4, "slices": 16, "native_nodes": 33555, "mass": [ 0.06510250480650452, 0.06523909165527138, 0.06140203092709648, 0.0671882210450083, 0.05709310302094808, 0.06295767956913059, 0.06117496100230626, 0.06052132700309724, 0.06489582772398204, 0.05749779129446645, 0.06216456985250704, 0.06003810908007579, 0.0600672410912351, 0.06174383471911292, 0.06650478394538044, 0.06640892353165861 ], "mean_node_assignment_entropy": 2.7702762138224393, "normalized_node_entropy": 0.9991659388935717, "effective_mass_slots": 15.981108480907526, "node_slot_mutual_information_nats": 0.0011310908751034354, "normalized_node_slot_mutual_information": 0.00040795479907662104, "centered_token_effective_rank": 1.5931779269788922, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.006165108527354211, "token_RMS": 0.19266070984620998, "mean_token_offdiagonal_cosine": 0.9989399597849612, "temperature": 0.4979405999183655 }, { "layer": 8, "head": 5, "slices": 16, "native_nodes": 33555, "mass": [ 0.06296591498118687, 0.06466449603422834, 0.05926657903424906, 0.06149951258793333, 0.06301445547791622, 0.05886716385025183, 0.058125327862582676, 0.06226395459079899, 0.05853542228604176, 0.061034270201998024, 0.06047183735147085, 0.06733540061124656, 0.060045673826415465, 0.07443832314837318, 0.06506864315234963, 0.06240302458274491 ], "mean_node_assignment_entropy": 2.769026341316254, "normalized_node_entropy": 0.9987151426769675, "effective_mass_slots": 15.969065183761668, "node_slot_mutual_information_nats": 0.0016270834279183966, "normalized_node_slot_mutual_information": 0.0005868462981426215, "centered_token_effective_rank": 1.1417727803297042, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.010290750639145089, "token_RMS": 0.2663998499422394, "mean_token_offdiagonal_cosine": 0.9984984794405684, "temperature": 0.4999732971191406 }, { "layer": 8, "head": 6, "slices": 16, "native_nodes": 33555, "mass": [ 0.06573448532720717, 0.05897625753914985, 0.06444238761581193, 0.06394973232778187, 0.06586889047078504, 0.05983557234114532, 0.06889249249258067, 0.06501555490737575, 0.06782766001088363, 0.05844634307525922, 0.05698958652427345, 0.06422060286944571, 0.05929072672633598, 0.063694001012624, 0.06156313929158926, 0.05525256722383928 ], "mean_node_assignment_entropy": 2.7692277414830646, "normalized_node_entropy": 0.9987877824324405, "effective_mass_slots": 15.969436987412815, "node_slot_mutual_information_nats": 0.0014489657337279915, "normalized_node_slot_mutual_information": 0.0005226039196168529, "centered_token_effective_rank": 1.2579076413708965, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.007880156046356232, "token_RMS": 0.2532447653759036, "mean_token_offdiagonal_cosine": 0.9989952674702387, "temperature": 0.49835121631622314 }, { "layer": 8, "head": 7, "slices": 16, "native_nodes": 33555, "mass": [ 0.06273198923607587, 0.06415596142602187, 0.0599511174893365, 0.06331419239683578, 0.057927850592782076, 0.0632677124102946, 0.06226925156181233, 0.0659161857556762, 0.06255007361105863, 0.0648238651037012, 0.06077703132008657, 0.06096304318316824, 0.060856724859967064, 0.062499724638042854, 0.06547774001667334, 0.0625175363207526 ], "mean_node_assignment_entropy": 2.77121542041942, "normalized_node_entropy": 0.9995046860685303, "effective_mass_slots": 15.99158930842471, "node_slot_mutual_information_nats": 0.0008474953849271216, "normalized_node_slot_mutual_information": 0.0003056693472526604, "centered_token_effective_rank": 1.2670641942932293, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.006685942244638562, "token_RMS": 0.2120746895425065, "mean_token_offdiagonal_cosine": 0.998966528157445, "temperature": 0.4999127686023712 }, { "layer": 8, "head": 8, "slices": 16, "native_nodes": 33555, "mass": [ 0.06005440697056173, 0.06574029608208738, 0.059671120233794936, 0.056597741934727705, 0.06337965345352937, 0.06271433669525862, 0.058061132503881875, 0.06614653840061539, 0.05919779178702677, 0.06535993772813521, 0.06272721369768144, 0.06901063871950315, 0.0594609056170483, 0.06449875082160637, 0.06463894713084382, 0.06274058815930608 ], "mean_node_assignment_entropy": 2.769619803947804, "normalized_node_entropy": 0.9989291890758399, "effective_mass_slots": 15.977913449590282, "node_slot_mutual_information_nats": 0.0015875552486006583, "normalized_node_slot_mutual_information": 0.0005725895210733538, "centered_token_effective_rank": 1.1180715003251245, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.009746807720461693, "token_RMS": 0.30644330854069923, "mean_token_offdiagonal_cosine": 0.9989780495843099, "temperature": 0.4966300427913666 }, { "layer": 9, "head": 1, "slices": 16, "native_nodes": 33555, "mass": [ 0.0666970553523008, 0.05840439008113933, 0.06968784965125109, 0.06414769096348577, 0.07179645144260866, 0.058308257769297565, 0.05628608370716322, 0.05880078163634663, 0.07126185468937456, 0.06389040422488869, 0.05332724470792291, 0.05597208814808996, 0.060237209910435005, 0.06015492886251565, 0.06747770185027654, 0.06355000718841874 ], "mean_node_assignment_entropy": 2.7672007388331488, "normalized_node_entropy": 0.9980566957647149, "effective_mass_slots": 15.938002613365093, "node_slot_mutual_information_nats": 0.00150562011309896, "normalized_node_slot_mutual_information": 0.0005430376676576374, "centered_token_effective_rank": 1.4182228280115028, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.007482644649660443, "token_RMS": 0.2698698106363424, "mean_token_offdiagonal_cosine": 0.9992109070115257, "temperature": 0.49486976861953735 }, { "layer": 9, "head": 2, "slices": 16, "native_nodes": 33555, "mass": [ 0.05361556613734823, 0.07974190609925125, 0.06646519098422546, 0.05868769661369222, 0.05582903616631054, 0.06257761462383842, 0.07666980330239037, 0.048452842797328526, 0.06564481500072693, 0.06570853427287861, 0.059563181293186565, 0.07631136159941436, 0.06067195443747083, 0.05897923427313508, 0.05697340126977985, 0.05410786121040072 ], "mean_node_assignment_entropy": 2.7611685291571377, "normalized_node_entropy": 0.9958810360184189, "effective_mass_slots": 15.851987854202525, "node_slot_mutual_information_nats": 0.0021263799710680154, "normalized_node_slot_mutual_information": 0.0007669294598263609, "centered_token_effective_rank": 1.8823012889752782, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.006724083833723707, "token_RMS": 0.22155385560475352, "mean_token_offdiagonal_cosine": 0.999229065772173, "temperature": 0.49354931712150574 }, { "layer": 9, "head": 3, "slices": 16, "native_nodes": 33555, "mass": [ 0.059623162019423304, 0.05537366268720859, 0.06576518249940702, 0.06507027580181722, 0.06401802009511964, 0.06703921259826208, 0.06911887861949252, 0.06185268011394316, 0.06568813468656425, 0.06246315689382495, 0.05838358213350947, 0.06720589279491222, 0.04741782266260828, 0.07004549336618149, 0.06286002386324169, 0.058074818865617155 ], "mean_node_assignment_entropy": 2.7677304443205015, "normalized_node_entropy": 0.9982477466346488, "effective_mass_slots": 15.934130775924102, "node_slot_mutual_information_nats": 0.0007329539545701813, "normalized_node_slot_mutual_information": 0.0002643572588645888, "centered_token_effective_rank": 1.4460862738395523, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.005564066639486414, "token_RMS": 0.185642783034247, "mean_token_offdiagonal_cosine": 0.9990637724179088, "temperature": 0.49685394763946533 }, { "layer": 9, "head": 4, "slices": 16, "native_nodes": 33555, "mass": [ 0.0764628842058924, 0.05511478758852131, 0.062434968241297385, 0.06122199379106407, 0.06023908391563801, 0.06446384046920063, 0.06408066827975377, 0.0645486651624329, 0.05293078160860147, 0.05461383700779344, 0.0679224745150766, 0.053839865890310055, 0.06369973783622891, 0.06931141551654503, 0.06831369289736726, 0.060801302999004915 ], "mean_node_assignment_entropy": 2.765724861605668, "normalized_node_entropy": 0.9975243855754531, "effective_mass_slots": 15.92251230356168, "node_slot_mutual_information_nats": 0.0020091143728624417, "normalized_node_slot_mutual_information": 0.0007246348355768461, "centered_token_effective_rank": 1.254367004771194, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.00817384777540236, "token_RMS": 0.2552283554192998, "mean_token_offdiagonal_cosine": 0.9989470871512185, "temperature": 0.49468958377838135 }, { "layer": 9, "head": 5, "slices": 16, "native_nodes": 33555, "mass": [ 0.060251887252790494, 0.06287393954565443, 0.06339516827056436, 0.06243301839049394, 0.06106299059883242, 0.06691340595847926, 0.0690393996121471, 0.0646228243006917, 0.057404511584230725, 0.05933545336788963, 0.06968246578666194, 0.058406616826647884, 0.05854631104813107, 0.06059712953649038, 0.06619933835408262, 0.059235539259462594 ], "mean_node_assignment_entropy": 2.7697098731644907, "normalized_node_entropy": 0.9989616746789026, "effective_mass_slots": 15.971786114092964, "node_slot_mutual_information_nats": 0.0011139246428482075, "normalized_node_slot_mutual_information": 0.00040176338954027967, "centered_token_effective_rank": 1.545511420797043, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.007363623845725265, "token_RMS": 0.2516708266189686, "mean_token_offdiagonal_cosine": 0.9991498581419959, "temperature": 0.4986382722854614 }, { "layer": 9, "head": 6, "slices": 16, "native_nodes": 33555, "mass": [ 0.05486126553959453, 0.07906665082467489, 0.07815756382596349, 0.06464718872951317, 0.06139960856659991, 0.0663546327004719, 0.07240903459332806, 0.05224144501746758, 0.06746319115101945, 0.059207171208478175, 0.05711072355624266, 0.06625032116967206, 0.05690829251419501, 0.05906292244940752, 0.054625612154054565, 0.050234375845664765 ], "mean_node_assignment_entropy": 2.7618166890081026, "normalized_node_entropy": 0.9961148102690915, "effective_mass_slots": 15.857885807844704, "node_slot_mutual_information_nats": 0.0018502148983259303, "normalized_node_slot_mutual_information": 0.0006673239645984244, "centered_token_effective_rank": 1.3740382132463924, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.00842598827805454, "token_RMS": 0.2278617084059256, "mean_token_offdiagonal_cosine": 0.9985928446123197, "temperature": 0.49409356713294983 }, { "layer": 9, "head": 7, "slices": 16, "native_nodes": 33555, "mass": [ 0.05816617937712214, 0.06799992801042307, 0.065715537292847, 0.06202186835165653, 0.06328749350138756, 0.06304190135418684, 0.06489944186129898, 0.05680283891030723, 0.06087434677624468, 0.061723420178569605, 0.059309389876073246, 0.06564662142868653, 0.06785477732842911, 0.05755688144384509, 0.05978506784206122, 0.0653143062186195 ], "mean_node_assignment_entropy": 2.7697305290052783, "normalized_node_entropy": 0.99896912469867, "effective_mass_slots": 15.975601387777159, "node_slot_mutual_information_nats": 0.0013321161069996812, "normalized_node_slot_mutual_information": 0.000480459325364188, "centered_token_effective_rank": 1.3676446703207754, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.008246377195283126, "token_RMS": 0.24137571949678907, "mean_token_offdiagonal_cosine": 0.9987935558634654, "temperature": 0.5001112818717957 }, { "layer": 9, "head": 8, "slices": 16, "native_nodes": 33555, "mass": [ 0.06993521652963651, 0.06334344107769246, 0.05716941384758205, 0.060553570711218356, 0.062401708438959196, 0.056409462001005384, 0.07004681731026213, 0.0610493609089154, 0.05555654544565916, 0.05812240603885464, 0.07131206522347533, 0.057313383597008535, 0.07259060504086201, 0.05533516897567781, 0.06503304214355683, 0.06382779297330769 ], "mean_node_assignment_entropy": 2.767776141865636, "normalized_node_entropy": 0.9982642285400853, "effective_mass_slots": 15.934574632604873, "node_slot_mutual_information_nats": 0.0007151117411856589, "normalized_node_slot_mutual_information": 0.0002579220406725055, "centered_token_effective_rank": 1.5408410855452335, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.004669818268656544, "token_RMS": 0.22786796728732503, "mean_token_offdiagonal_cosine": 0.999556473678719, "temperature": 0.4976971447467804 }, { "layer": 10, "head": 1, "slices": 16, "native_nodes": 33555, "mass": [ 0.06345787510828722, 0.05730993396797828, 0.06382357452429455, 0.0596760961241968, 0.0631945051464891, 0.06581249246741527, 0.06317622042773839, 0.05840695760583508, 0.06343130453546203, 0.06177464348937516, 0.060697250189088954, 0.05482964001328415, 0.07200044226520304, 0.06468076540634309, 0.06691956413112389, 0.06080873414136901 ], "mean_node_assignment_entropy": 2.769926750169025, "normalized_node_entropy": 0.9990398965236338, "effective_mass_slots": 15.968534121764215, "node_slot_mutual_information_nats": 0.0006934183499427427, "normalized_node_slot_mutual_information": 0.0002500978036809507, "centered_token_effective_rank": 1.250674372362319, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.005735960353749153, "token_RMS": 0.21318906475135732, "mean_token_offdiagonal_cosine": 0.9993510862391256, "temperature": 0.49935418367385864 }, { "layer": 10, "head": 2, "slices": 16, "native_nodes": 33555, "mass": [ 0.060912773874855455, 0.059630229060317876, 0.062425514388978795, 0.06714732749791459, 0.06679129318799476, 0.058921618328317764, 0.053779066622372695, 0.062450718165727365, 0.06519359927410749, 0.05470036753715461, 0.05590540693819994, 0.05804182944130727, 0.07097862790756548, 0.059443021513607155, 0.07407582166655073, 0.06960278506375624 ], "mean_node_assignment_entropy": 2.765991841849573, "normalized_node_entropy": 0.9976206783439272, "effective_mass_slots": 15.931987711770152, "node_slot_mutual_information_nats": 0.0023370521771695074, "normalized_node_slot_mutual_information": 0.0008429133965753008, "centered_token_effective_rank": 1.1429945709130283, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.01026339457139059, "token_RMS": 0.31924162389873056, "mean_token_offdiagonal_cosine": 0.9990316287664331, "temperature": 0.48956406116485596 }, { "layer": 10, "head": 3, "slices": 16, "native_nodes": 33555, "mass": [ 0.06027119128468441, 0.05352369818856989, 0.06176772916628732, 0.05789526965091098, 0.06369289935740925, 0.07106088995833615, 0.05656204424601982, 0.05490463731810173, 0.06552660911609191, 0.06774282161917439, 0.06764367140897286, 0.06293527412354155, 0.06839841040759732, 0.061840410800013734, 0.05870237328393007, 0.06753207015473411 ], "mean_node_assignment_entropy": 2.767232581919409, "normalized_node_entropy": 0.9980681807303734, "effective_mass_slots": 15.94690253730782, "node_slot_mutual_information_nats": 0.0020320301646044037, "normalized_node_slot_mutual_information": 0.0007328999603528893, "centered_token_effective_rank": 1.2713548337726743, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.008277547481680303, "token_RMS": 0.23284002947026314, "mean_token_offdiagonal_cosine": 0.9986744961194705, "temperature": 0.49321940541267395 }, { "layer": 10, "head": 4, "slices": 16, "native_nodes": 33555, "mass": [ 0.06133311043327335, 0.06232868740192168, 0.06464485353370428, 0.06155130094140124, 0.06434344797937921, 0.06222234336872194, 0.06346908855038115, 0.06796555097059866, 0.06438339843839794, 0.05544459682283805, 0.06117505737458375, 0.05779088993070112, 0.06533567411829225, 0.0564372785974501, 0.07062948200457002, 0.06094523997464739 ], "mean_node_assignment_entropy": 2.769719309972987, "normalized_node_entropy": 0.9989650782881075, "effective_mass_slots": 15.970353336223525, "node_slot_mutual_information_nats": 0.0010147770075219498, "normalized_node_slot_mutual_information": 0.00036600343909001483, "centered_token_effective_rank": 1.410927570755938, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.005723993901574341, "token_RMS": 0.18848963431494328, "mean_token_offdiagonal_cosine": 0.9990329847291797, "temperature": 0.49856501817703247 }, { "layer": 10, "head": 5, "slices": 16, "native_nodes": 33555, "mass": [ 0.06001180581123441, 0.05394050536732474, 0.056414104912626, 0.05962865237136069, 0.06589591418678972, 0.059443683573797694, 0.057539113052661184, 0.06215996427683268, 0.06358292629205956, 0.05924934182605273, 0.06459557135371405, 0.064663352714159, 0.06801229734775864, 0.07197169713124205, 0.0654885948915548, 0.06740247504859208 ], "mean_node_assignment_entropy": 2.7683868724594904, "normalized_node_entropy": 0.9984845030398536, "effective_mass_slots": 15.956005566112289, "node_slot_mutual_information_nats": 0.0014484104249405938, "normalized_node_slot_mutual_information": 0.0005224036343084177, "centered_token_effective_rank": 1.1733700465681787, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.006851535703700514, "token_RMS": 0.28503220957040937, "mean_token_offdiagonal_cosine": 0.9995034511802818, "temperature": 0.4932941198348999 }, { "layer": 10, "head": 6, "slices": 16, "native_nodes": 33555, "mass": [ 0.06032006180394319, 0.06590299539211396, 0.06449717722989695, 0.06065542087360779, 0.06113139185735284, 0.06552099938633597, 0.07086639424547, 0.06704442683752221, 0.05848928179085086, 0.07122656730572774, 0.06106301848505332, 0.06416430689473299, 0.05794528716912285, 0.05865044068162638, 0.05365468845595157, 0.0588675416490881 ], "mean_node_assignment_entropy": 2.768312996168623, "normalized_node_entropy": 0.9984578578002351, "effective_mass_slots": 15.955536603167626, "node_slot_mutual_information_nats": 0.001492895284819884, "normalized_node_slot_mutual_information": 0.0005384481559940408, "centered_token_effective_rank": 1.1278048961786786, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.00946111201103898, "token_RMS": 0.2707196004715663, "mean_token_offdiagonal_cosine": 0.9990151649413558, "temperature": 0.4928092658519745 }, { "layer": 10, "head": 7, "slices": 16, "native_nodes": 33555, "mass": [ 0.06379902560861717, 0.06924463130890071, 0.05523827511946556, 0.07709832786070467, 0.06153819416156293, 0.05486992718481197, 0.06140482716818553, 0.06955691811245203, 0.05456888173133548, 0.05985986587831967, 0.06414251364662082, 0.061554377915948574, 0.0630581954699784, 0.06520048422363384, 0.0637650890382138, 0.05510046591951993 ], "mean_node_assignment_entropy": 2.7672767494299024, "normalized_node_entropy": 0.9980841107924627, "effective_mass_slots": 15.929477753845465, "node_slot_mutual_information_nats": 0.0008945901408097257, "normalized_node_slot_mutual_information": 0.0003226551899435877, "centered_token_effective_rank": 1.3636447222028323, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.005160002104804555, "token_RMS": 0.3059201850788943, "mean_token_offdiagonal_cosine": 0.9997024543255004, "temperature": 0.4939707815647125 }, { "layer": 10, "head": 8, "slices": 16, "native_nodes": 33555, "mass": [ 0.07026425382696096, 0.06908364775478884, 0.05849990098447278, 0.07364065493583999, 0.05572075833186381, 0.06132280902995063, 0.06607179042479797, 0.06715050370706796, 0.05293545533205318, 0.06039093474045256, 0.06604936413967633, 0.05640804126332474, 0.0634463999456486, 0.06428651867123247, 0.060143561240253995, 0.05458540581372132 ], "mean_node_assignment_entropy": 2.767182259534929, "normalized_node_entropy": 0.9980500307667397, "effective_mass_slots": 15.930622760444146, "node_slot_mutual_information_nats": 0.0010609571849879984, "normalized_node_slot_mutual_information": 0.000382659417344425, "centered_token_effective_rank": 1.4583295806316636, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.006616239333911984, "token_RMS": 0.26809115314586396, "mean_token_offdiagonal_cosine": 0.9993630632425485, "temperature": 0.4949597120285034 }, { "layer": 11, "head": 1, "slices": 16, "native_nodes": 33555, "mass": [ 0.08514064553153086, 0.04989956910549182, 0.05167156659924865, 0.05492490695079203, 0.04967237767062481, 0.059330270493675355, 0.06432226386668963, 0.056968858407211095, 0.0748388615834999, 0.06056412120706774, 0.0665206571497748, 0.05831897288145651, 0.06309137331992676, 0.06199643364232497, 0.07814092440357188, 0.06459819736840952 ], "mean_node_assignment_entropy": 2.7603144413296263, "normalized_node_entropy": 0.9955729889501104, "effective_mass_slots": 15.815554857987342, "node_slot_mutual_information_nats": 0.0006794990988154836, "normalized_node_slot_mutual_information": 0.0002450774950374045, "centered_token_effective_rank": 1.186829424055562, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.005070476712100773, "token_RMS": 0.24692614650080763, "mean_token_offdiagonal_cosine": 0.9995681626081728, "temperature": 0.4911801218986511 }, { "layer": 11, "head": 2, "slices": 16, "native_nodes": 33555, "mass": [ 0.062181641946578235, 0.060086637881580716, 0.05570090540245257, 0.06920736214730819, 0.06481695597070711, 0.054155084095099275, 0.07161177165496546, 0.06086534805370503, 0.06162360702700843, 0.055646376002698165, 0.06476967393943404, 0.05937541943815618, 0.06843570459619078, 0.06576995423823848, 0.06407576304219409, 0.06167779442102201 ], "mean_node_assignment_entropy": 2.7687928599651115, "normalized_node_entropy": 0.9986309320801091, "effective_mass_slots": 15.952698922280478, "node_slot_mutual_information_nats": 0.0008351663793462194, "normalized_node_slot_mutual_information": 0.00030122259844999537, "centered_token_effective_rank": 1.2109336590489956, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.005049888440052824, "token_RMS": 0.277363266189242, "mean_token_offdiagonal_cosine": 0.9996643661497574, "temperature": 0.49854138493537903 }, { "layer": 11, "head": 3, "slices": 16, "native_nodes": 33555, "mass": [ 0.04456047519670658, 0.07370430828828319, 0.07134846520817345, 0.06651644741564283, 0.07235545869658555, 0.06148919065060621, 0.05225367835600602, 0.08032156347211508, 0.05088392502192928, 0.05755671708299353, 0.0636595300474122, 0.0678464705407877, 0.0577170892473021, 0.0697278390956465, 0.047435437774999256, 0.06262340384197296 ], "mean_node_assignment_entropy": 2.758713804688455, "normalized_node_entropy": 0.9949956813139897, "effective_mass_slots": 15.798649143843695, "node_slot_mutual_information_nats": 0.0012106344642872635, "normalized_node_slot_mutual_information": 0.0004366440844891255, "centered_token_effective_rank": 1.3058013152266634, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.007201778903371245, "token_RMS": 0.2479448438112888, "mean_token_offdiagonal_cosine": 0.9991209682600792, "temperature": 0.4875433146953583 }, { "layer": 11, "head": 4, "slices": 16, "native_nodes": 33555, "mass": [ 0.0559016329126002, 0.06393685851686315, 0.067569080471655, 0.07449061927361898, 0.06287224678229138, 0.056484408059350746, 0.06626521654081778, 0.06568567740514868, 0.05672402922566434, 0.05980292319987236, 0.056867345939417425, 0.06947152826376066, 0.05636463166704265, 0.07153154042807693, 0.05566296400398243, 0.06036929723256712 ], "mean_node_assignment_entropy": 2.7665419489380936, "normalized_node_entropy": 0.9978190875360688, "effective_mass_slots": 15.928422910394024, "node_slot_mutual_information_nats": 0.001563168852160235, "normalized_node_slot_mutual_information": 0.0005637939877709161, "centered_token_effective_rank": 1.1322766588289792, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.006913964258606584, "token_RMS": 0.29620473472456693, "mean_token_offdiagonal_cosine": 0.9994294648396659, "temperature": 0.49563848972320557 }, { "layer": 11, "head": 5, "slices": 16, "native_nodes": 33555, "mass": [ 0.058213144490827554, 0.061932931021533126, 0.06173079661684142, 0.0688531160619953, 0.06457958699722632, 0.05908476964933757, 0.06460898565690487, 0.06449654302528568, 0.053956918183266186, 0.0631055554717828, 0.06734486618843204, 0.06367910864458072, 0.06385361253360383, 0.06764266363107364, 0.05856768366828119, 0.05834971785871749 ], "mean_node_assignment_entropy": 2.7693589515347887, "normalized_node_entropy": 0.9988351064551747, "effective_mass_slots": 15.968610458989653, "node_slot_mutual_information_nats": 0.0012659974507136518, "normalized_node_slot_mutual_information": 0.00045661206098066384, "centered_token_effective_rank": 1.2450599606476298, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.0062272107416671526, "token_RMS": 0.23133906492510906, "mean_token_offdiagonal_cosine": 0.999373367881612, "temperature": 0.5013400912284851 }, { "layer": 11, "head": 6, "slices": 16, "native_nodes": 33555, "mass": [ 0.07606778546730357, 0.06817804112211102, 0.06869790822072136, 0.04826724691134134, 0.05546724736155403, 0.0746857800188034, 0.04666645907830364, 0.06557255220398109, 0.0683411094192829, 0.07436681347798527, 0.05541846767198913, 0.06958636506479342, 0.052820898851926805, 0.05166664535454232, 0.06371133622387681, 0.06048534352372876 ], "mean_node_assignment_entropy": 2.759973829252883, "normalized_node_entropy": 0.9954501391116142, "effective_mass_slots": 15.817797071198502, "node_slot_mutual_information_nats": 0.0011618737823124725, "normalized_node_slot_mutual_information": 0.0004190573859702768, "centered_token_effective_rank": 1.5061144870293204, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.005360488547689633, "token_RMS": 0.25204654432099627, "mean_token_offdiagonal_cosine": 0.9995457957541457, "temperature": 0.49047908186912537 }, { "layer": 11, "head": 7, "slices": 16, "native_nodes": 33555, "mass": [ 0.12578940013404966, 0.0472423804964692, 0.05461123860465434, 0.03216888380843789, 0.05085458133501244, 0.06953117639701212, 0.037091393561900825, 0.061707403252020336, 0.08115992121916914, 0.09557294093994995, 0.04620840329115658, 0.05957156489930038, 0.056466143068374136, 0.03653768967360262, 0.08270407228587263, 0.06278280702047244 ], "mean_node_assignment_entropy": 2.6961757677849434, "normalized_node_entropy": 0.9724398523870829, "effective_mass_slots": 14.979658582471677, "node_slot_mutual_information_nats": 0.010517418487100638, "normalized_node_slot_mutual_information": 0.003793356873573499, "centered_token_effective_rank": 1.035010724117654, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.021968304718309965, "token_RMS": 0.2871843246090883, "mean_token_offdiagonal_cosine": 0.9944235171004804, "temperature": 0.47058844566345215 }, { "layer": 11, "head": 8, "slices": 16, "native_nodes": 33555, "mass": [ 0.03886768257126352, 0.0578512618728143, 0.07054430440272523, 0.09330065731958916, 0.07508257291539382, 0.047099271149993065, 0.06408524094397879, 0.085024223061765, 0.046392583719927515, 0.05263134994566432, 0.052840203690939204, 0.06986469130853731, 0.05196385667074403, 0.08441078582893202, 0.046799084268185055, 0.06324222997650271 ], "mean_node_assignment_entropy": 2.7407185602569535, "normalized_node_entropy": 0.9885052688387617, "effective_mass_slots": 15.52354213470573, "node_slot_mutual_information_nats": 0.0016391587865172141, "normalized_node_slot_mutual_information": 0.000591201563134489, "centered_token_effective_rank": 1.7115766324157047, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.005878870842704958, "token_RMS": 0.25387311665296847, "mean_token_offdiagonal_cosine": 0.999460985731566, "temperature": 0.4815791845321655 }, { "layer": 12, "head": 1, "slices": 16, "native_nodes": 33555, "mass": [ 0.06421876977438046, 0.07320180086216652, 0.05399346883011689, 0.05634911037562963, 0.06893260388025459, 0.06879642096353242, 0.060883128427989326, 0.06418046819312866, 0.07669310633044825, 0.05303817350273604, 0.05882255190400835, 0.0577215626351774, 0.05525295576823129, 0.05640774565257466, 0.07208496681716381, 0.05942316654674912 ], "mean_node_assignment_entropy": 2.7648197756265334, "normalized_node_entropy": 0.997197944812034, "effective_mass_slots": 15.895400798980207, "node_slot_mutual_information_nats": 0.0012100338307363678, "normalized_node_slot_mutual_information": 0.00043642745172780834, "centered_token_effective_rank": 1.2343906733942764, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.005411703039305571, "token_RMS": 0.2565151148021963, "mean_token_offdiagonal_cosine": 0.9995326519304738, "temperature": 0.4903039038181305 }, { "layer": 12, "head": 2, "slices": 16, "native_nodes": 33555, "mass": [ 0.060186032189944165, 0.05373706254564673, 0.05724700490877892, 0.0638513191829467, 0.041569287970277986, 0.06495104155010144, 0.06307583621987162, 0.06718083568513526, 0.046576297933752286, 0.06982387093345022, 0.07342884565692885, 0.059447953597459434, 0.06460084082807714, 0.08093203553018725, 0.06997366769647584, 0.06341806752789023 ], "mean_node_assignment_entropy": 2.7602231657145193, "normalized_node_entropy": 0.9955400682307931, "effective_mass_slots": 15.813795593697733, "node_slot_mutual_information_nats": 0.0006595321960523037, "normalized_node_slot_mutual_information": 0.00023787595713781654, "centered_token_effective_rank": 1.9992126069883167, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.003169584801523412, "token_RMS": 0.22636291300143388, "mean_token_offdiagonal_cosine": 0.9997969185875352, "temperature": 0.4852485656738281 }, { "layer": 12, "head": 3, "slices": 16, "native_nodes": 33555, "mass": [ 0.051908367323131144, 0.049155201231459895, 0.06368944118702274, 0.09152119566455112, 0.05232223855846953, 0.07358251390838531, 0.04939756836304187, 0.05855439205242773, 0.05896947331512282, 0.07183886386655053, 0.056058896630069854, 0.05735919758679703, 0.06202765694164231, 0.06682901665215307, 0.06550426409769797, 0.07128171261359462 ], "mean_node_assignment_entropy": 2.757902383218698, "normalized_node_entropy": 0.9947030228813674, "effective_mass_slots": 15.778493610160105, "node_slot_mutual_information_nats": 0.0007454656752061695, "normalized_node_slot_mutual_information": 0.00026886990819322085, "centered_token_effective_rank": 1.4188903101690349, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.004165682677982728, "token_RMS": 0.25450493870708474, "mean_token_offdiagonal_cosine": 0.9997227130661669, "temperature": 0.48990947008132935 }, { "layer": 12, "head": 4, "slices": 16, "native_nodes": 33555, "mass": [ 0.05711893813020958, 0.05889403427791585, 0.06321234922085056, 0.07885346109928201, 0.06209392023990698, 0.06695054267370618, 0.05405643867100338, 0.05375757677720105, 0.06445412068255718, 0.06438571572972115, 0.05814025127360157, 0.06484715157284092, 0.07570276114006651, 0.05593232836320634, 0.05410151381909874, 0.06749889600665074 ], "mean_node_assignment_entropy": 2.7654199723092114, "normalized_node_entropy": 0.9974144200064485, "effective_mass_slots": 15.89751109369483, "node_slot_mutual_information_nats": 0.0007425896763479933, "normalized_node_slot_mutual_information": 0.0002678326108706476, "centered_token_effective_rank": 1.627795907341706, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.0030418808919749, "token_RMS": 0.21102084736782015, "mean_token_offdiagonal_cosine": 0.9997804133473686, "temperature": 0.4912574589252472 }, { "layer": 12, "head": 5, "slices": 16, "native_nodes": 33555, "mass": [ 0.06119867474897628, 0.046003453470927526, 0.06727592369158228, 0.07474152467220094, 0.05653330137487709, 0.057593739289979684, 0.0595760392309603, 0.06620979525571856, 0.056325480948415885, 0.0713077156602306, 0.05946360092247639, 0.0679529887370542, 0.06655287695320046, 0.06221146134845885, 0.06346350905559418, 0.0635899143616847 ], "mean_node_assignment_entropy": 2.7659802719183504, "normalized_node_entropy": 0.9976165053733277, "effective_mass_slots": 15.90828314157255, "node_slot_mutual_information_nats": 0.0008596539594551977, "normalized_node_slot_mutual_information": 0.000310054626046644, "centered_token_effective_rank": 1.1962750215554867, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.005438564177477146, "token_RMS": 0.24380043032526624, "mean_token_offdiagonal_cosine": 0.9995134668411351, "temperature": 0.4971550405025482 }, { "layer": 12, "head": 6, "slices": 16, "native_nodes": 33555, "mass": [ 0.059899232856515196, 0.045156437329667065, 0.07185230047341129, 0.07280067547505205, 0.05385605053363623, 0.05708987197769752, 0.06508751817979057, 0.05915597583474087, 0.046749867341891786, 0.07108529970025433, 0.06519988717078594, 0.06898476328851214, 0.08229223233082134, 0.06766693583856022, 0.053276654081859026, 0.05984629748821827 ], "mean_node_assignment_entropy": 2.759180225361398, "normalized_node_entropy": 0.9951639070119455, "effective_mass_slots": 15.806636402898881, "node_slot_mutual_information_nats": 0.0012496519854345856, "normalized_node_slot_mutual_information": 0.00045071668055588095, "centered_token_effective_rank": 1.3105869235456484, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.004647864989831054, "token_RMS": 0.30344714289211294, "mean_token_offdiagonal_cosine": 0.9997776273159257, "temperature": 0.48331940174102783 }, { "layer": 12, "head": 7, "slices": 16, "native_nodes": 33555, "mass": [ 0.057587749380776536, 0.05505967850950503, 0.06006011940386785, 0.07682315259661196, 0.061570511403839276, 0.065673143627915, 0.05441271231353345, 0.06292028746207821, 0.06399141357687897, 0.0651277756620148, 0.05831229704378206, 0.06681580039041264, 0.06750381515629067, 0.05500108414680552, 0.06459787957644501, 0.06454257964022095 ], "mean_node_assignment_entropy": 2.7677674374154626, "normalized_node_entropy": 0.9982610890733106, "effective_mass_slots": 15.936685708426982, "node_slot_mutual_information_nats": 0.0008562913933189975, "normalized_node_slot_mutual_information": 0.0003088418366743047, "centered_token_effective_rank": 1.3131770475520146, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.006137619508715644, "token_RMS": 0.27920570516313015, "mean_token_offdiagonal_cosine": 0.9995441770334631, "temperature": 0.4965461194515228 }, { "layer": 12, "head": 8, "slices": 16, "native_nodes": 33555, "mass": [ 0.06942503001713536, 0.07624234939572744, 0.06583057931568705, 0.04930936265561961, 0.05843742347875147, 0.05464643812784849, 0.07285928074772854, 0.06027232424665929, 0.06034792406675162, 0.05244300980156979, 0.07140801411980827, 0.05691907211700756, 0.06643076099887225, 0.06798041355830159, 0.06346684041878645, 0.053981176929859515 ], "mean_node_assignment_entropy": 2.763511465852847, "normalized_node_entropy": 0.996726071806423, "effective_mass_slots": 15.878800447031413, "node_slot_mutual_information_nats": 0.001473448512922193, "normalized_node_slot_mutual_information": 0.0005314342156495164, "centered_token_effective_rank": 1.2618539459707376, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.006577772176645445, "token_RMS": 0.28456610240816516, "mean_token_offdiagonal_cosine": 0.9994858999095755, "temperature": 0.49197542667388916 }, { "layer": 13, "head": 1, "slices": 16, "native_nodes": 33555, "mass": [ 0.06392421094536106, 0.12040927059811889, 0.08116371102721019, 0.0822147200229608, 0.025120775837958027, 0.03813563891286964, 0.08301199784649767, 0.019811200745616927, 0.053421729348456695, 0.03169745627478167, 0.04007391917836096, 0.21299862882949222, 0.02897253669465313, 0.0910318181746906, 0.015803514842502968, 0.012208870883224494 ], "mean_node_assignment_entropy": 2.492448449356024, "normalized_node_entropy": 0.8989607543893307, "effective_mass_slots": 12.251441153263416, "node_slot_mutual_information_nats": 0.013195125879422953, "normalized_node_slot_mutual_information": 0.004759135667537279, "centered_token_effective_rank": 1.015490948404232, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.02370229387679143, "token_RMS": 0.3017551370611859, "mean_token_offdiagonal_cosine": 0.9935263482448042, "temperature": 0.4425434172153473 }, { "layer": 13, "head": 2, "slices": 16, "native_nodes": 33555, "mass": [ 0.05682549587033346, 0.058292282195635516, 0.059471750422044334, 0.058091471237874996, 0.0663088335558903, 0.055194526604587534, 0.0563159291234604, 0.06514001405975392, 0.07538609971673811, 0.05752270334473649, 0.06865396486400155, 0.057168518452548604, 0.06079453679770031, 0.061147502246223284, 0.07002575891443522, 0.07366061310972583 ], "mean_node_assignment_entropy": 2.767092090749655, "normalized_node_entropy": 0.9980175092519001, "effective_mass_slots": 15.920718200591281, "node_slot_mutual_information_nats": 0.0005292017514841874, "normalized_node_slot_mutual_information": 0.00019086918562399768, "centered_token_effective_rank": 1.3644600451852558, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.0030589287498771055, "token_RMS": 0.22539430961430604, "mean_token_offdiagonal_cosine": 0.9998213849676091, "temperature": 0.4943270683288574 }, { "layer": 13, "head": 3, "slices": 16, "native_nodes": 33555, "mass": [ 0.05366527778596783, 0.045781802621852836, 0.05053074841922272, 0.055053983543708204, 0.07428458907346294, 0.06160587487011513, 0.042559622820772615, 0.07557865521925924, 0.04775572130580077, 0.064141505881711, 0.06429801973693418, 0.046228397825119165, 0.08579169965535513, 0.043627446418928684, 0.09422492845911788, 0.09487172677311416 ], "mean_node_assignment_entropy": 2.7348085993243716, "normalized_node_entropy": 0.9863737010064408, "effective_mass_slots": 15.43024151389, "node_slot_mutual_information_nats": 0.0015207191569781564, "normalized_node_slot_mutual_information": 0.0005484834965893079, "centered_token_effective_rank": 1.2200448329468858, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.006079364835995343, "token_RMS": 0.2853077506533872, "mean_token_offdiagonal_cosine": 0.9995265707652371, "temperature": 0.48670899868011475 }, { "layer": 13, "head": 4, "slices": 16, "native_nodes": 33555, "mass": [ 0.061706263596069046, 0.06724619411255134, 0.06181078417221954, 0.07138113968375187, 0.05708188942176026, 0.06328110795369839, 0.07166066627986668, 0.06334032223956397, 0.06581355029409838, 0.06118912289659875, 0.058091725070973724, 0.07070279234573146, 0.05802070752037138, 0.06428195389700689, 0.05227341530704406, 0.05211836518782241 ], "mean_node_assignment_entropy": 2.7676936195923973, "normalized_node_entropy": 0.9982344649214943, "effective_mass_slots": 15.929005841328104, "node_slot_mutual_information_nats": 0.00044809443032001184, "normalized_node_slot_mutual_information": 0.00016161590311816157, "centered_token_effective_rank": 1.5641739651455486, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.0025173271074061176, "token_RMS": 0.2997159512821772, "mean_token_offdiagonal_cosine": 0.9999294651268436, "temperature": 0.5000612735748291 }, { "layer": 13, "head": 5, "slices": 16, "native_nodes": 33555, "mass": [ 0.06561385684704535, 0.08374590303028878, 0.08620749136682523, 0.06998184692409351, 0.044938329486347066, 0.04789444909827522, 0.08515727030921845, 0.0342021154124035, 0.08832398737383713, 0.05358441463460547, 0.05403614705904481, 0.08927183554893171, 0.0389590427440452, 0.08843911088577878, 0.035690190722784794, 0.03395400872705786 ], "mean_node_assignment_entropy": 2.711829542481541, "normalized_node_entropy": 0.9780857581685765, "effective_mass_slots": 15.078497280848898, "node_slot_mutual_information_nats": 0.0014401653218159005, "normalized_node_slot_mutual_information": 0.0005194298419610144, "centered_token_effective_rank": 1.3120521167131105, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.006320840854466804, "token_RMS": 0.2674905134979824, "mean_token_offdiagonal_cosine": 0.9994372340314082, "temperature": 0.4864058792591095 }, { "layer": 13, "head": 6, "slices": 16, "native_nodes": 33555, "mass": [ 0.06107684807334833, 0.06210034896732442, 0.06042169956394643, 0.0763803736612172, 0.05379089175358097, 0.05470014197862622, 0.07205806431148078, 0.051379836665311276, 0.04468874852066867, 0.056856468902417044, 0.06574182283916517, 0.09379664118719641, 0.07887313135946371, 0.05848806134211806, 0.06772658858382209, 0.04192033238023977 ], "mean_node_assignment_entropy": 2.75044762171487, "normalized_node_entropy": 0.9920142860182217, "effective_mass_slots": 15.675439809262933, "node_slot_mutual_information_nats": 0.0016475224177003867, "normalized_node_slot_mutual_information": 0.0005942181054424358, "centered_token_effective_rank": 1.7269546232904918, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.005302627658039287, "token_RMS": 0.314662063199948, "mean_token_offdiagonal_cosine": 0.9996993813553509, "temperature": 0.480284720659256 }, { "layer": 13, "head": 7, "slices": 16, "native_nodes": 33555, "mass": [ 0.0570499522049458, 0.05139044963401807, 0.06679772983338879, 0.045901402292628526, 0.08210197755182363, 0.0598072678153809, 0.031794688192623204, 0.07781527238739637, 0.0548442037074741, 0.0668326492115329, 0.05902169281296236, 0.03927684624978726, 0.08367638868455861, 0.049524651974164294, 0.0922231224679304, 0.08194170513725586 ], "mean_node_assignment_entropy": 2.7347294139996707, "normalized_node_entropy": 0.9863451409376265, "effective_mass_slots": 15.418946929086978, "node_slot_mutual_information_nats": 0.0008676592669587535, "normalized_node_slot_mutual_information": 0.00031294193040568675, "centered_token_effective_rank": 1.4699126909696119, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.004215235529192724, "token_RMS": 0.24587682109250183, "mean_token_offdiagonal_cosine": 0.9997258501378589, "temperature": 0.4848255515098572 }, { "layer": 13, "head": 8, "slices": 16, "native_nodes": 33555, "mass": [ 0.06213968479052472, 0.0554991342784564, 0.05833517429308535, 0.06336877054232738, 0.06796967409825364, 0.062399524549776424, 0.06746320676548687, 0.07119136166086745, 0.060879820164372433, 0.055810063591551946, 0.06120232394099182, 0.06083454861959425, 0.06267333970721709, 0.05787370998677154, 0.07038246229026733, 0.06197720078040637 ], "mean_node_assignment_entropy": 2.769487940351758, "normalized_node_entropy": 0.9988816293368177, "effective_mass_slots": 15.95775781986085, "node_slot_mutual_information_nats": 0.00045715432322435134, "normalized_node_slot_mutual_information": 0.00016488356875918049, "centered_token_effective_rank": 1.6552598204835118, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.0028435459408189083, "token_RMS": 0.2519964175079138, "mean_token_offdiagonal_cosine": 0.9998735407027942, "temperature": 0.49789175391197205 }, { "layer": 14, "head": 1, "slices": 16, "native_nodes": 33555, "mass": [ 0.011058253153737825, 0.07446760665755783, 0.02514895545952927, 0.016651252326084408, 0.0909321161682285, 0.011178919641491713, 0.10903909418372777, 0.1602693540491125, 0.013439538413965432, 0.09486806321772036, 0.1291479651218483, 0.050806896101764976, 0.020950773525264237, 0.057407858260691995, 0.07819582713803135, 0.056437526734979086 ], "mean_node_assignment_entropy": 2.5099842071104375, "normalized_node_entropy": 0.9052854420769613, "effective_mass_slots": 12.317516143605358, "node_slot_mutual_information_nats": 0.0010381189482169262, "normalized_node_slot_mutual_information": 0.0003744222646113565, "centered_token_effective_rank": 1.2355649254509617, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.00438995621889468, "token_RMS": 0.26742410961797203, "mean_token_offdiagonal_cosine": 0.9997131339972344, "temperature": 0.4494197368621826 }, { "layer": 14, "head": 2, "slices": 16, "native_nodes": 33555, "mass": [ 0.02364837135484012, 0.07439956342347101, 0.028160206446705287, 0.020721022302825784, 0.07192202355907493, 0.02708163122571433, 0.08846851481737314, 0.1342645464375928, 0.026010775960889698, 0.10255422357177259, 0.11517376603561158, 0.06558260183307883, 0.02968934185218969, 0.09121315791112729, 0.05069607999195216, 0.05041417333967301 ], "mean_node_assignment_entropy": 2.6136429043361495, "normalized_node_entropy": 0.9426724141850976, "effective_mass_slots": 13.661683078684183, "node_slot_mutual_information_nats": 0.0009521544188504372, "normalized_node_slot_mutual_information": 0.0003434171145590097, "centered_token_effective_rank": 1.337305573401815, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.003944215440265048, "token_RMS": 0.2703899660536535, "mean_token_offdiagonal_cosine": 0.9997791303078594, "temperature": 0.4744930863380432 }, { "layer": 14, "head": 3, "slices": 16, "native_nodes": 33555, "mass": [ 0.015138792121071536, 0.048254845435591644, 0.014602380552728935, 0.014563752271039593, 0.08128705589902008, 0.027910026459176288, 0.08717028637003249, 0.1297320544915092, 0.017303254792025175, 0.1374416567115417, 0.1021543925244768, 0.0782235874311346, 0.040968235633515725, 0.07585601796138093, 0.0782041963073824, 0.05118946499446433 ], "mean_node_assignment_entropy": 2.5644470169880083, "normalized_node_entropy": 0.9249287485077737, "effective_mass_slots": 13.037690345788576, "node_slot_mutual_information_nats": 0.0033974031010362005, "normalized_node_slot_mutual_information": 0.001225354151441428, "centered_token_effective_rank": 1.0551603062361783, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.010123472273951576, "token_RMS": 0.3361999486943214, "mean_token_offdiagonal_cosine": 0.9990390759125689, "temperature": 0.4476169943809509 }, { "layer": 14, "head": 4, "slices": 16, "native_nodes": 33555, "mass": [ 0.054872852734718466, 0.06937523341879655, 0.061806537276887445, 0.05493266480094877, 0.06027252764784265, 0.045210946033438275, 0.06512235937046944, 0.07278741930058133, 0.06233701547950555, 0.07220235701609859, 0.07034501674952938, 0.05591716894208541, 0.05998671379619234, 0.058160275605667756, 0.06427528957070705, 0.07239562220912527 ], "mean_node_assignment_entropy": 2.7640982855718326, "normalized_node_entropy": 0.9969377222810422, "effective_mass_slots": 15.881974072878645, "node_slot_mutual_information_nats": 0.001086474415195049, "normalized_node_slot_mutual_information": 0.00039186281271365845, "centered_token_effective_rank": 1.3918910377577793, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.004112887841088435, "token_RMS": 0.26973768502839385, "mean_token_offdiagonal_cosine": 0.999755127676019, "temperature": 0.4914207458496094 }, { "layer": 14, "head": 5, "slices": 16, "native_nodes": 33555, "mass": [ 0.092655447724057, 0.04159897014836213, 0.06274133422787366, 0.09480174085858895, 0.05569716023328123, 0.11388696108355889, 0.04276041034545243, 0.03379105602657097, 0.09426619955269844, 0.03985024428486628, 0.03600561338111996, 0.06420624280802771, 0.07921511720654471, 0.05646379151093038, 0.042084698150892795, 0.04997501257341286 ], "mean_node_assignment_entropy": 2.7001659921996093, "normalized_node_entropy": 0.9738790216308509, "effective_mass_slots": 14.887176687442496, "node_slot_mutual_information_nats": 0.0003342251976277133, "normalized_node_slot_mutual_information": 0.00012054625878940894, "centered_token_effective_rank": 1.4579115652538541, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.0016276262610732332, "token_RMS": 0.29064703775164347, "mean_token_offdiagonal_cosine": 0.9999672234939979, "temperature": 0.48591935634613037 }, { "layer": 14, "head": 6, "slices": 16, "native_nodes": 33555, "mass": [ 0.09578513777028642, 0.04889499740715337, 0.10383722053374748, 0.10403736610692191, 0.04735747884463276, 0.08028937222514063, 0.04067767388555798, 0.03685363142097108, 0.10374597629182226, 0.03252116525513798, 0.044223325080961995, 0.045745396545797144, 0.059736599875640416, 0.052601258505222924, 0.04610338580990722, 0.05759001460074579 ], "mean_node_assignment_entropy": 2.6955893063010583, "normalized_node_entropy": 0.9722283311184645, "effective_mass_slots": 14.830527883129863, "node_slot_mutual_information_nats": 0.0010984448410225056, "normalized_node_slot_mutual_information": 0.0003961802312083087, "centered_token_effective_rank": 1.1638164212254443, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.0034360687442681935, "token_RMS": 0.21621079923660017, "mean_token_offdiagonal_cosine": 0.9997485141306995, "temperature": 0.48432543873786926 }, { "layer": 14, "head": 7, "slices": 16, "native_nodes": 33555, "mass": [ 0.14478085749711708, 0.03188948233306159, 0.09682509929514592, 0.13973325019092536, 0.02720131511819281, 0.14877558441644204, 0.026439164012210893, 0.014707345376797314, 0.1386137681719426, 0.016809956247916504, 0.02085788139165549, 0.03545069578091925, 0.06418833960707541, 0.04130227067283681, 0.02367035872060786, 0.028754631009032236 ], "mean_node_assignment_entropy": 2.4669402723024834, "normalized_node_entropy": 0.8897606242550155, "effective_mass_slots": 11.829552918323307, "node_slot_mutual_information_nats": 0.0036606127778369135, "normalized_node_slot_mutual_information": 0.0013202869753000221, "centered_token_effective_rank": 1.0397908988966595, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.008581749465730468, "token_RMS": 0.27763291798078904, "mean_token_offdiagonal_cosine": 0.9989834774416461, "temperature": 0.44885966181755066 }, { "layer": 14, "head": 8, "slices": 16, "native_nodes": 33555, "mass": [ 0.02940451785680026, 0.08621169534388215, 0.04576409564627885, 0.03406544896626753, 0.06988285017109472, 0.026578794573473247, 0.08195930959579398, 0.10221214582576, 0.040013397213461824, 0.09001437115511157, 0.09820641023007648, 0.052785357712673515, 0.03733695651722494, 0.059262721605666646, 0.07164850138619624, 0.07465342637959153 ], "mean_node_assignment_entropy": 2.6920179383957374, "normalized_node_entropy": 0.9709402324269154, "effective_mass_slots": 14.780044385796984, "node_slot_mutual_information_nats": 0.0012599802183870246, "normalized_node_slot_mutual_information": 0.00045444180317128836, "centered_token_effective_rank": 1.029292316522976, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.006161239438771192, "token_RMS": 0.2264603540888748, "mean_token_offdiagonal_cosine": 0.9992689499632951, "temperature": 0.47909015417099 }, { "layer": 15, "head": 1, "slices": 16, "native_nodes": 33555, "mass": [ 0.06994787302929001, 0.09315298570928125, 0.06267172568087037, 0.055590920732582115, 0.056859764470182435, 0.05136336377877234, 0.06398250099852727, 0.04948253759165365, 0.05608332048046342, 0.08772632053837728, 0.042415092555763576, 0.07529198222163182, 0.04381328753193344, 0.08045306678385657, 0.05058206351538597, 0.06058319448656482 ], "mean_node_assignment_entropy": 2.745366057828376, "normalized_node_entropy": 0.9901814992634703, "effective_mass_slots": 15.579943353202893, "node_slot_mutual_information_nats": 0.0006183467413065635, "normalized_node_slot_mutual_information": 0.0002230214443082075, "centered_token_effective_rank": 1.5425739579643083, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.0022246242272635245, "token_RMS": 0.26788654331564293, "mean_token_offdiagonal_cosine": 0.9999267984326214, "temperature": 0.47735488414764404 }, { "layer": 15, "head": 2, "slices": 16, "native_nodes": 33555, "mass": [ 0.06722454990236294, 0.0863088279034409, 0.06202745779187504, 0.0526849535392269, 0.05746883316982203, 0.07507743517841992, 0.06607107643758002, 0.05025542777455203, 0.06351425357028617, 0.07745012936311797, 0.044062122173475236, 0.07145437705340958, 0.04174119209560671, 0.07847842202623027, 0.05928403206659205, 0.046896909878397286 ], "mean_node_assignment_entropy": 2.751183205573864, "normalized_node_entropy": 0.9922795918146038, "effective_mass_slots": 15.665101462305755, "node_slot_mutual_information_nats": 0.00025219581574731365, "normalized_node_slot_mutual_information": 9.096041317789904e-05, "centered_token_effective_rank": 1.7735247176965898, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.0013023221479088825, "token_RMS": 0.29215146731532055, "mean_token_offdiagonal_cosine": 0.9999789981451979, "temperature": 0.484426885843277 }, { "layer": 15, "head": 3, "slices": 16, "native_nodes": 33555, "mass": [ 0.06580001353504779, 0.06137473049901527, 0.06619197507606074, 0.06676172279437631, 0.06277663722321238, 0.05112511648153552, 0.0570083946317119, 0.06364342772983755, 0.05875298234726665, 0.06503585385645205, 0.06560950463363212, 0.059109064418505566, 0.06261874301154502, 0.058845717519757076, 0.06182554151245031, 0.07352057430516268 ], "mean_node_assignment_entropy": 2.7690405601871975, "normalized_node_entropy": 0.9987202710506168, "effective_mass_slots": 15.950861518494516, "node_slot_mutual_information_nats": 0.00047228128368459465, "normalized_node_slot_mutual_information": 0.0001703394664691096, "centered_token_effective_rank": 1.1580120800779357, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.002261890256464313, "token_RMS": 0.3029710543018248, "mean_token_offdiagonal_cosine": 0.9999411152771222, "temperature": 0.49634915590286255 }, { "layer": 15, "head": 4, "slices": 16, "native_nodes": 33555, "mass": [ 0.07497166912853406, 0.10592271884970757, 0.05040873431491969, 0.044265192875122636, 0.060122694389850476, 0.06276756624591687, 0.05862423473883894, 0.03984979326350912, 0.057471502819984616, 0.09609142841240845, 0.029391419371741837, 0.09136187788551459, 0.038828024192767464, 0.09776592675850157, 0.05247863821209037, 0.039678578813701994 ], "mean_node_assignment_entropy": 2.705466877165652, "normalized_node_entropy": 0.9757909117440592, "effective_mass_slots": 14.966968837765464, "node_slot_mutual_information_nats": 0.0003788183163497827, "normalized_node_slot_mutual_information": 0.00013662982659893452, "centered_token_effective_rank": 1.3358797408657919, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.0020737371149793632, "token_RMS": 0.2947193261574839, "mean_token_offdiagonal_cosine": 0.9999472528435133, "temperature": 0.47304463386535645 }, { "layer": 15, "head": 5, "slices": 16, "native_nodes": 33555, "mass": [ 0.05285581551617667, 0.050715977487514705, 0.07531422063865954, 0.06918752168204884, 0.056326235807025476, 0.06873549676309167, 0.06895339191981319, 0.06471179670227781, 0.06242677725506262, 0.04815785127098381, 0.08068875387175536, 0.0492797773702767, 0.06688657617283869, 0.052792912800569435, 0.061355715761364456, 0.07161117863515648 ], "mean_node_assignment_entropy": 2.7601007008657903, "normalized_node_entropy": 0.995495898373307, "effective_mass_slots": 15.811690624053949, "node_slot_mutual_information_nats": 0.0006488784813289961, "normalized_node_slot_mutual_information": 0.00023403344178822612, "centered_token_effective_rank": 1.1545334824749662, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.0028503982262405797, "token_RMS": 0.35473591429543466, "mean_token_offdiagonal_cosine": 0.9999340935190867, "temperature": 0.49191707372665405 }, { "layer": 15, "head": 6, "slices": 16, "native_nodes": 33555, "mass": [ 0.07396392281041062, 0.11314870932030795, 0.060270149613582256, 0.04924358228464948, 0.042924497969867116, 0.07268889859805003, 0.06302733784950876, 0.035758844781802995, 0.07028194641928402, 0.07258829571525416, 0.03760098257830505, 0.07015330283621565, 0.037660564091642215, 0.09833191119177724, 0.05225888967098822, 0.05009816435761467 ], "mean_node_assignment_entropy": 2.7181544520354146, "normalized_node_entropy": 0.9803669870804377, "effective_mass_slots": 15.155011383931884, "node_slot_mutual_information_nats": 0.00017680964080568629, "normalized_node_slot_mutual_information": 6.377059799293063e-05, "centered_token_effective_rank": 1.6742419580818255, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.0011988259916518708, "token_RMS": 0.3436693771563137, "mean_token_offdiagonal_cosine": 0.9999881868918338, "temperature": 0.48171132802963257 }, { "layer": 15, "head": 7, "slices": 16, "native_nodes": 33555, "mass": [ 0.0670807053450056, 0.05208422357313343, 0.055754432396141966, 0.07186370901935521, 0.061007414220311466, 0.05414504826373074, 0.053325949790687006, 0.06649645336395298, 0.0589488092910389, 0.0487225640196014, 0.07360360073099156, 0.055593170891187704, 0.08902875777306775, 0.05465233428907491, 0.06686298994566996, 0.07082983706573352 ], "mean_node_assignment_entropy": 2.759362388664226, "normalized_node_entropy": 0.9952296085353508, "effective_mass_slots": 15.796360740484706, "node_slot_mutual_information_nats": 0.00041719195728839864, "normalized_node_slot_mutual_information": 0.00015047019196968325, "centered_token_effective_rank": 1.38448386503345, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.002108305461637007, "token_RMS": 0.21644739769273466, "mean_token_offdiagonal_cosine": 0.9998990495084307, "temperature": 0.4960327446460724 }, { "layer": 15, "head": 8, "slices": 16, "native_nodes": 33555, "mass": [ 0.03612301540583918, 0.016764352657955575, 0.05856148523174869, 0.08481622890643792, 0.05553168951232731, 0.038184434548186895, 0.04029232874277309, 0.12498285245822556, 0.04917320335982472, 0.025043698372670443, 0.12706235995739054, 0.032590525185670176, 0.1264778884934022, 0.02293395060768889, 0.05619985726196388, 0.10526212908426388 ], "mean_node_assignment_entropy": 2.598324340996581, "normalized_node_entropy": 0.9371474103442129, "effective_mass_slots": 13.457345455591991, "node_slot_mutual_information_nats": 0.0012007465035499543, "normalized_node_slot_mutual_information": 0.0004330777565090703, "centered_token_effective_rank": 1.1008839666736334, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.004464202352437935, "token_RMS": 0.2528040767894473, "mean_token_offdiagonal_cosine": 0.9996686426729231, "temperature": 0.4518086016178131 }, { "layer": 16, "head": 1, "slices": 16, "native_nodes": 33555, "mass": [ 0.05839887232270029, 0.06243493192351943, 0.06983855927204556, 0.05674796672311719, 0.0642943940457218, 0.05641619981305554, 0.06299658183004547, 0.06249338330612455, 0.053645138201490986, 0.05344688469842976, 0.0748535744649361, 0.05913668467744135, 0.06777217006287228, 0.05105631474792433, 0.06847979324889102, 0.0779885506113921 ], "mean_node_assignment_entropy": 2.763794456084384, "normalized_node_entropy": 0.9968281389573377, "effective_mass_slots": 15.8862010097565, "node_slot_mutual_information_nats": 0.0016564153107938573, "normalized_node_slot_mutual_information": 0.0005974255386337122, "centered_token_effective_rank": 1.0675334238569403, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.004619976912009848, "token_RMS": 0.27659710338963184, "mean_token_offdiagonal_cosine": 0.9997033413337714, "temperature": 0.49661779403686523 }, { "layer": 16, "head": 2, "slices": 16, "native_nodes": 33555, "mass": [ 0.04470577911874733, 0.04765833187784493, 0.036056638119020365, 0.05758988969155669, 0.06815939899897579, 0.02721738413891073, 0.03818071228297292, 0.061689522056981215, 0.02899700262028829, 0.022388741635024825, 0.1398775025240765, 0.05432095028911879, 0.05786955529811964, 0.07027806233380986, 0.08821150541856203, 0.15679902358766357 ], "mean_node_assignment_entropy": 2.6217599352617085, "normalized_node_entropy": 0.9456000142508592, "effective_mass_slots": 13.779115365643863, "node_slot_mutual_information_nats": 0.001394131285140876, "normalized_node_slot_mutual_information": 0.0005028265728552248, "centered_token_effective_rank": 1.050251030675871, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.004139515593399872, "token_RMS": 0.33168816772016196, "mean_token_offdiagonal_cosine": 0.9998353073497844, "temperature": 0.4728732705116272 }, { "layer": 16, "head": 3, "slices": 16, "native_nodes": 33555, "mass": [ 0.05849718565928689, 0.049630335634905966, 0.04941214741735145, 0.0800623216459922, 0.07375986656814373, 0.03809667293455842, 0.05684418330467808, 0.06512263634977211, 0.03536096894171301, 0.04967892291575266, 0.09374549685776314, 0.04503447513895798, 0.06604196702318457, 0.06800201653728115, 0.08556885216310188, 0.08514195129557314 ], "mean_node_assignment_entropy": 2.7344168592490585, "normalized_node_entropy": 0.9862324106404479, "effective_mass_slots": 15.40606070342891, "node_slot_mutual_information_nats": 0.00034412491989010974, "normalized_node_slot_mutual_information": 0.0001241168288429433, "centered_token_effective_rank": 1.4395665395595456, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.001638616432800868, "token_RMS": 0.31519873992875597, "mean_token_offdiagonal_cosine": 0.999972071918377, "temperature": 0.4771174192428589 }, { "layer": 16, "head": 4, "slices": 16, "native_nodes": 33555, "mass": [ 0.07711090211109972, 0.059760206383909026, 0.07873983661307075, 0.0610869090970272, 0.05210144647657027, 0.07876523136648508, 0.058871293843742845, 0.05856847471866056, 0.06041437303734828, 0.06911186475205165, 0.06203396080216944, 0.0551293886108861, 0.05738009194672916, 0.05577791511908123, 0.05685270052220583, 0.058295404804905734 ], "mean_node_assignment_entropy": 2.7635074389053154, "normalized_node_entropy": 0.9967246193921147, "effective_mass_slots": 15.863871147575571, "node_slot_mutual_information_nats": 0.0005368299629786577, "normalized_node_slot_mutual_information": 0.00019362048134747884, "centered_token_effective_rank": 1.5739752198912755, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.0017427504739300338, "token_RMS": 0.2567938995836829, "mean_token_offdiagonal_cosine": 0.9999531490274963, "temperature": 0.49712568521499634 }, { "layer": 16, "head": 5, "slices": 16, "native_nodes": 33555, "mass": [ 0.06152945017738254, 0.05172625361891657, 0.06423503151850397, 0.06830574780405056, 0.0559261866039321, 0.04907682027760463, 0.060358589931250047, 0.07940826893425648, 0.05334836913458975, 0.049534551455023, 0.0638152396229514, 0.054977949049127846, 0.05758869936799367, 0.08196409833651508, 0.08393453451106915, 0.0642702095226095 ], "mean_node_assignment_entropy": 2.75800680087095, "normalized_node_entropy": 0.9947406835886387, "effective_mass_slots": 15.773237877655207, "node_slot_mutual_information_nats": 0.0003078978403383559, "normalized_node_slot_mutual_information": 0.00011105067183914197, "centered_token_effective_rank": 1.4855780457913914, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.001774491164597561, "token_RMS": 0.28531992961522895, "mean_token_offdiagonal_cosine": 0.9999602442908103, "temperature": 0.4877684712409973 }, { "layer": 16, "head": 6, "slices": 16, "native_nodes": 33555, "mass": [ 0.047078805713024925, 0.058646623981330276, 0.08640942684545583, 0.044534282693163396, 0.027323932868069518, 0.1073891806237887, 0.06824751010733876, 0.06395656597910047, 0.1417252325153248, 0.1304473109246899, 0.01871575452523333, 0.0613377659886769, 0.04904855769580015, 0.046531286282899785, 0.034292280196925956, 0.014315482942328343 ], "mean_node_assignment_entropy": 2.611094357123024, "normalized_node_entropy": 0.9417532200786357, "effective_mass_slots": 13.636688547610587, "node_slot_mutual_information_nats": 0.0016694907157832617, "normalized_node_slot_mutual_information": 0.0006021414941176694, "centered_token_effective_rank": 1.0416073406461652, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.005248847455898772, "token_RMS": 0.3227258650431878, "mean_token_offdiagonal_cosine": 0.9997274984275767, "temperature": 0.4403572380542755 }, { "layer": 16, "head": 7, "slices": 16, "native_nodes": 33555, "mass": [ 0.06732588908637162, 0.058100156606602574, 0.06610644126441116, 0.053410334944072234, 0.05259510850567982, 0.059942652021980625, 0.07449258093244361, 0.06716686366896145, 0.08940136792842365, 0.07502116187507679, 0.04781025532877866, 0.06255143283143465, 0.052053860668661946, 0.07497849507203774, 0.054061527355601685, 0.044981872043796486 ], "mean_node_assignment_entropy": 2.7552797105452345, "normalized_node_entropy": 0.9937570936663971, "effective_mass_slots": 15.732644711599603, "node_slot_mutual_information_nats": 0.00045812408977052854, "normalized_node_slot_mutual_information": 0.00016523333810592795, "centered_token_effective_rank": 1.0876787460664463, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.002244629240947576, "token_RMS": 0.26088169642468345, "mean_token_offdiagonal_cosine": 0.9999210473702379, "temperature": 0.48806920647621155 }, { "layer": 16, "head": 8, "slices": 16, "native_nodes": 33555, "mass": [ 0.036546473301899376, 0.027611122008858822, 0.0377639007647628, 0.06063118084139413, 0.053772486335979704, 0.017396503001520326, 0.04106026015794947, 0.09324940744481593, 0.03205386634847789, 0.018962453791380125, 0.12298920236304768, 0.0470148784055864, 0.05826880651616496, 0.10713811427523563, 0.1315722971832658, 0.11396904704383723 ], "mean_node_assignment_entropy": 2.5995012483262836, "normalized_node_entropy": 0.9375718899362498, "effective_mass_slots": 13.463501700745988, "node_slot_mutual_information_nats": 0.00048119812882818636, "normalized_node_slot_mutual_information": 0.00017355553853636826, "centered_token_effective_rank": 1.2965033286527996, "centered_token_rank_upper_bound": 15, "centered_token_RMS": 0.001830101973769551, "token_RMS": 0.2666948298593486, "mean_token_offdiagonal_cosine": 0.9999521819293914, "temperature": 0.4560678005218506 } ] }, { "schema": "ts3_capacity_slice_diagnostics_v1", "variant": "S032", "sample_id": "normal_rodv2_0150__gas_inertia_30deg", "native_nodes": 33555, "layers": 16, "slices": 32, "parameters": 7609476, "checkpoint_sha256": "9db98de06286698677e199dd13fadc3c775e71cbdfc24de3ed669f23e3191837", "extraction_source_sha256": "26c5dec82b68ba4bc23d7c3d998b78c5bbed6aae912e6247ea64949db69bd206", "CPU_forward_seconds": 4.674703202999808, "CPU_threads": 2, "CUDA_initialized": false, "hook_max_abs_prediction_difference_on_19_node_test": 0.0, "metrics": [ { "layer": 1, "head": 1, "slices": 32, "native_nodes": 33555, "mass": [ 0.030520623663226356, 0.029909614383987832, 0.03139007164235043, 0.02951526125717307, 0.030878502619719916, 0.02975128870415778, 0.027653060720312296, 0.03296599694483416, 0.037018077800333754, 0.03146530200745741, 0.029281505155098846, 0.03204919559621026, 0.031549911387736565, 0.030208813163602143, 0.029695120961253377, 0.03288124175554922, 0.030398127813774257, 0.03197104672577569, 0.03088199923903243, 0.031633613573010075, 0.03315681397693956, 0.03147484327398208, 0.029862364420297872, 0.0322666011445947, 0.03462420821766076, 0.03292402582812897, 0.03260336172031612, 0.028452154977857455, 0.030008450643642527, 0.02995939596814666, 0.03222230291669885, 0.030827101359107493 ], "mean_node_assignment_entropy": 3.463541271028904, "normalized_node_entropy": 0.9993667631255314, "effective_mass_slots": 31.947494144848665, "node_slot_mutual_information_nats": 0.0005524761976372083, "normalized_node_slot_mutual_information": 0.00015941093410807826, "centered_token_effective_rank": 2.083948413723655, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.0029748926831587256, "token_RMS": 0.2575480432725003, "mean_token_offdiagonal_cosine": 0.9998690035590085, "temperature": 0.4981365203857422 }, { "layer": 1, "head": 2, "slices": 32, "native_nodes": 33555, "mass": [ 0.026879376788350104, 0.030551104642081113, 0.032374549432956166, 0.03298975475934033, 0.03210812518235744, 0.033061281873257124, 0.03252621770489058, 0.02984449815994811, 0.03197093084998537, 0.0279536983006856, 0.027680941254084823, 0.03405859716871628, 0.028269471455492537, 0.02897273186611456, 0.033824644637818826, 0.03659717748226103, 0.030751136798401062, 0.0314861819834725, 0.033182991521979496, 0.03146492047944513, 0.029975743010038414, 0.03098336687074779, 0.030313261477780438, 0.033545410301635686, 0.03031349881846225, 0.029042093165251026, 0.03201092376904247, 0.02946983805988464, 0.03459134386063306, 0.03198845047981653, 0.03254645529875207, 0.02867128268148483 ], "mean_node_assignment_entropy": 3.462854556447931, "normalized_node_entropy": 0.9991686191814362, "effective_mass_slots": 31.921070900831655, "node_slot_mutual_information_nats": 0.0004117650957056185, "normalized_node_slot_mutual_information": 0.00011881029231713303, "centered_token_effective_rank": 2.3507433626642475, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.002432177789532408, "token_RMS": 0.24946858760577914, "mean_token_offdiagonal_cosine": 0.9999061811871434, "temperature": 0.49991610646247864 }, { "layer": 1, "head": 3, "slices": 32, "native_nodes": 33555, "mass": [ 0.0310872753706002, 0.033312018295757514, 0.029903301875469333, 0.034508516621707785, 0.0334772498619004, 0.03320590336879929, 0.03480899441538198, 0.030135563458409703, 0.03162127866332976, 0.03073314205972213, 0.028023681163314895, 0.03207144466958881, 0.030517865668006953, 0.028743964612801497, 0.03332760433006253, 0.03143981898087198, 0.030367246143338914, 0.029986431876247693, 0.030626631124413346, 0.030240408554195807, 0.029999696613779155, 0.032856331855744776, 0.030410171372558123, 0.03184985327089602, 0.028776373664693496, 0.027937585057036, 0.029742533191283598, 0.032410104899561594, 0.03259819986837407, 0.03180333569511547, 0.03410543899096669, 0.029372034355778218 ], "mean_node_assignment_entropy": 3.4634341966307063, "normalized_node_entropy": 0.9993358679848742, "effective_mass_slots": 31.946073794633886, "node_slot_mutual_information_nats": 0.0006150907147914886, "normalized_node_slot_mutual_information": 0.00017747766478530566, "centered_token_effective_rank": 2.0220471062288534, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.003265156486776528, "token_RMS": 0.2591176719222872, "mean_token_offdiagonal_cosine": 0.9998379881411961, "temperature": 0.5012420415878296 }, { "layer": 1, "head": 4, "slices": 32, "native_nodes": 33555, "mass": [ 0.031098581484938465, 0.034616934911755506, 0.03313421913805804, 0.030830225403927297, 0.02787971540584976, 0.029657226261303814, 0.03156356682600971, 0.029065704113059827, 0.03218737037569582, 0.029368190777160127, 0.032044012674312713, 0.034282458417429565, 0.03374794760577113, 0.03278633840032385, 0.032470333274115824, 0.02947444589082787, 0.03290786701137281, 0.028386676769452165, 0.03158699535321921, 0.03041717153286245, 0.028991913458801758, 0.02987107673240541, 0.03285391047744589, 0.03108478710771433, 0.027990290487492926, 0.03495942521859763, 0.032803368122784435, 0.027768743841965354, 0.02951178310211121, 0.03437196973202224, 0.029889126031183127, 0.032397624014844445 ], "mean_node_assignment_entropy": 3.46278484983726, "normalized_node_entropy": 0.9991485061051298, "effective_mass_slots": 31.9295688395812, "node_slot_mutual_information_nats": 0.0007476534967003623, "normalized_node_slot_mutual_information": 0.00021572719839858112, "centered_token_effective_rank": 1.8716431696128506, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.00413773986407433, "token_RMS": 0.2636840693767636, "mean_token_offdiagonal_cosine": 0.9997542910033834, "temperature": 0.5008943676948547 }, { "layer": 1, "head": 5, "slices": 32, "native_nodes": 33555, "mass": [ 0.030178632385470333, 0.030924510761191744, 0.030447926091352478, 0.03172571437069493, 0.029794966999450823, 0.03168251929834381, 0.033080388033173334, 0.03250958771024229, 0.032533143550066514, 0.03098776198767049, 0.030533294982274497, 0.03328881044883901, 0.02938569521578924, 0.032252279205092056, 0.03156006147272051, 0.03213780362876761, 0.0300237479662841, 0.03202607648031107, 0.030589553765200977, 0.02967788595170551, 0.03145579498129654, 0.0345990073900032, 0.0282938449058891, 0.02968142036603373, 0.03262200292242593, 0.02995426290951788, 0.029916638925653813, 0.032094203340476114, 0.031964331440824084, 0.02901993099927032, 0.03219874213330171, 0.03285945957722793 ], "mean_node_assignment_entropy": 3.464045264582006, "normalized_node_entropy": 0.9995121849254714, "effective_mass_slots": 31.967443013049415, "node_slot_mutual_information_nats": 0.0006727144668787766, "normalized_node_slot_mutual_information": 0.00019410436506005476, "centered_token_effective_rank": 2.152530711205396, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.003787563154631868, "token_RMS": 0.3132728752678877, "mean_token_offdiagonal_cosine": 0.9998655227964245, "temperature": 0.5027831792831421 }, { "layer": 1, "head": 6, "slices": 32, "native_nodes": 33555, "mass": [ 0.029974082655313206, 0.03370533568867582, 0.029675551294512194, 0.03202963455125963, 0.028907654425172315, 0.030776813473525264, 0.031107577949822295, 0.029594737902114917, 0.03674348657061208, 0.030859795907400456, 0.030421038298706526, 0.02919700731731637, 0.03210436131257296, 0.02809898421976556, 0.029654552481014954, 0.030433098854495133, 0.03418477478302097, 0.029239111219721108, 0.030202424779563204, 0.03280023225900793, 0.03182688471251265, 0.03147965392042043, 0.030562020463899785, 0.03396517213613279, 0.029380985339369385, 0.03229705398288076, 0.03167141559166713, 0.030340644450378904, 0.032373383415205396, 0.03217549154396438, 0.03327822959462144, 0.030938808691806256 ], "mean_node_assignment_entropy": 3.4632503687342457, "normalized_node_entropy": 0.999282826465954, "effective_mass_slots": 31.947224738320212, "node_slot_mutual_information_nats": 0.0008349456661349564, "normalized_node_slot_mutual_information": 0.00024091439438892674, "centered_token_effective_rank": 1.8846492045708727, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.0037793256435589974, "token_RMS": 0.254184619757627, "mean_token_offdiagonal_cosine": 0.9997798626129853, "temperature": 0.5002501606941223 }, { "layer": 1, "head": 7, "slices": 32, "native_nodes": 33555, "mass": [ 0.02893510098893691, 0.030186720620582665, 0.030428592070344308, 0.030154073693167967, 0.0289861586450589, 0.029026980832720935, 0.031007398751436235, 0.03328601801652761, 0.032412911642239, 0.030368612732640004, 0.03490682555829701, 0.03261369505139172, 0.028579676550782626, 0.03375259282456267, 0.029240721314557086, 0.03264907213831921, 0.032359394592926465, 0.034343322308134826, 0.033351803528909055, 0.03145062866266981, 0.03185683765264475, 0.03507479893704486, 0.03144159502898096, 0.02813524107982071, 0.03512310026891584, 0.03190906127805709, 0.0313298139252649, 0.026927321000659435, 0.02909035817118404, 0.03164800654720965, 0.027414631446557862, 0.03200893387816832 ], "mean_node_assignment_entropy": 3.4623142898017143, "normalized_node_entropy": 0.9990127311791853, "effective_mass_slots": 31.921258249546828, "node_slot_mutual_information_nats": 0.0009579008484434937, "normalized_node_slot_mutual_information": 0.00027639176074255176, "centered_token_effective_rank": 2.2388053561929473, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.004000360504729507, "token_RMS": 0.28121765097468343, "mean_token_offdiagonal_cosine": 0.9997957539013967, "temperature": 0.4950971007347107 }, { "layer": 1, "head": 8, "slices": 32, "native_nodes": 33555, "mass": [ 0.031821412388793206, 0.02940274317296575, 0.028954222418768646, 0.031707672743150894, 0.03258234816558261, 0.03213901942813162, 0.03002437230545404, 0.029182134079257717, 0.029891149399655263, 0.031167625655568475, 0.034084469749424466, 0.03135394856679006, 0.03215026751131756, 0.03332538994046095, 0.03197950503658856, 0.030638031784854157, 0.030444472912666234, 0.03257935821755804, 0.03057657513013097, 0.0307929107805817, 0.03503493462071531, 0.03127342864555727, 0.031170663055942304, 0.030058195335825762, 0.026989397826845875, 0.03169424367729686, 0.028640401011826633, 0.032259685691096905, 0.03193400900501765, 0.03351199408012537, 0.031072391132582516, 0.03156302666968541 ], "mean_node_assignment_entropy": 3.463537342679347, "normalized_node_entropy": 0.9993656296434466, "effective_mass_slots": 31.956715485444388, "node_slot_mutual_information_nats": 0.0008450033957854153, "normalized_node_slot_mutual_information": 0.00024381644172679053, "centered_token_effective_rank": 1.900312632268519, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.004020766533125376, "token_RMS": 0.20470863915011522, "mean_token_offdiagonal_cosine": 0.9996115076412665, "temperature": 0.5006352663040161 }, { "layer": 2, "head": 1, "slices": 32, "native_nodes": 33555, "mass": [ 0.03310120608932374, 0.0319155952247475, 0.02904822829299784, 0.033435830446588814, 0.032302232469290364, 0.03175511897973591, 0.03446941571911061, 0.030144451715295263, 0.0346766207020243, 0.030406582548397253, 0.029023240743899663, 0.030096549242501765, 0.03273518103731494, 0.03261387288540429, 0.0314752934725669, 0.02805069743817573, 0.03159521417610426, 0.03157603232296315, 0.03082264297758524, 0.031733565684782646, 0.030781512395948542, 0.0318534668572186, 0.03173587338471844, 0.029312724207496435, 0.02925422018211523, 0.031579435739631255, 0.031018189620379274, 0.02997650846638004, 0.032152290595632925, 0.03101822496078894, 0.029054530406896165, 0.03128545097146317 ], "mean_node_assignment_entropy": 3.4634555474673254, "normalized_node_entropy": 0.999342028534096, "effective_mass_slots": 31.961032031718016, "node_slot_mutual_information_nats": 0.001061864264776169, "normalized_node_slot_mutual_information": 0.0003063892617779568, "centered_token_effective_rank": 1.461614413523449, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.006287690929926089, "token_RMS": 0.22017685763084222, "mean_token_offdiagonal_cosine": 0.9991736787345638, "temperature": 0.5032981038093567 }, { "layer": 2, "head": 2, "slices": 32, "native_nodes": 33555, "mass": [ 0.029323788048515483, 0.03380308128380461, 0.03262726738826917, 0.033007581728896215, 0.029890431608267005, 0.03204285593748075, 0.03176241288131741, 0.03211468051679931, 0.030696655620589482, 0.03160494420826746, 0.031019742263589542, 0.03027495144582056, 0.029202716593146147, 0.03318844661310886, 0.03039206681447499, 0.02838600603246046, 0.029917117703948604, 0.030176405311341328, 0.03050541137906482, 0.027191318199038507, 0.03147831719639031, 0.030801487868981202, 0.032694055789821, 0.03193010285295765, 0.032061394553666515, 0.03130937719956593, 0.03291790827541473, 0.029745155691127987, 0.03350769459308659, 0.03149072542862092, 0.031488239560223535, 0.03344765913883268 ], "mean_node_assignment_entropy": 3.463551120799172, "normalized_node_entropy": 0.9993696051684753, "effective_mass_slots": 31.961009379982478, "node_slot_mutual_information_nats": 0.0009655822028880046, "normalized_node_slot_mutual_information": 0.00027860813113543304, "centered_token_effective_rank": 1.5510922195745798, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.00590578194389888, "token_RMS": 0.21790934302526996, "mean_token_offdiagonal_cosine": 0.9992532981965502, "temperature": 0.498690664768219 }, { "layer": 2, "head": 3, "slices": 32, "native_nodes": 33555, "mass": [ 0.028352432634452206, 0.03169337932751495, 0.03623885082858381, 0.029159861140373902, 0.03254906144817466, 0.028465824155574086, 0.028864701490417942, 0.0317195846370419, 0.029876880964946773, 0.03593953496916268, 0.03354967588436948, 0.03277705654195499, 0.029166008092794966, 0.033165717052553646, 0.030333826936489422, 0.03449823933834913, 0.029885575690642145, 0.030115241669712363, 0.029775310783186126, 0.02994865990454652, 0.030257460946693734, 0.028346314770436698, 0.03124655940303627, 0.03129309753818939, 0.03359625385226267, 0.03215949108507448, 0.032722886233318406, 0.033654958123793335, 0.029578342924990574, 0.031947650355964814, 0.029335735990238285, 0.029785825280441273 ], "mean_node_assignment_entropy": 3.462019821946952, "normalized_node_entropy": 0.9989277657164319, "effective_mass_slots": 31.927285831025454, "node_slot_mutual_information_nats": 0.001441177440611785, "normalized_node_slot_mutual_information": 0.00041583590932233415, "centered_token_effective_rank": 1.416288061360247, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.009482174258277548, "token_RMS": 0.25481267022301307, "mean_token_offdiagonal_cosine": 0.9985851757551527, "temperature": 0.5004467964172363 }, { "layer": 2, "head": 4, "slices": 32, "native_nodes": 33555, "mass": [ 0.03184112203657858, 0.03200207098429231, 0.03381931715689443, 0.03352719060388314, 0.03051423397852765, 0.02981049553583461, 0.03078794128501172, 0.032932955123880475, 0.03197231042573392, 0.03065543417140093, 0.03149847991224986, 0.03130218404496619, 0.02963655805508056, 0.030467064505643647, 0.0315460966380693, 0.027512604404852013, 0.03272707891837706, 0.03285348170336029, 0.029159149100065604, 0.03329462013470356, 0.029739886902316, 0.031592844772016467, 0.03085872945315373, 0.027999206695213278, 0.03209734516982403, 0.03187970021533382, 0.0306773094304662, 0.029324451083465333, 0.031225859771649593, 0.03234529728498718, 0.03209925041106717, 0.03229972998152421 ], "mean_node_assignment_entropy": 3.462335642112365, "normalized_node_entropy": 0.9990188921537227, "effective_mass_slots": 31.963209069784906, "node_slot_mutual_information_nats": 0.002249882687024307, "normalized_node_slot_mutual_information": 0.0006491789190303807, "centered_token_effective_rank": 1.1794409615709425, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.010793397455865852, "token_RMS": 0.20577312636245218, "mean_token_offdiagonal_cosine": 0.9971796983098355, "temperature": 0.49768611788749695 }, { "layer": 2, "head": 5, "slices": 32, "native_nodes": 33555, "mass": [ 0.03206286248068903, 0.03109542910060137, 0.033681911808226134, 0.03434575861890267, 0.030729965759032425, 0.026542290286697374, 0.03008520251472782, 0.03134303753894428, 0.03315675122847395, 0.03532908794891593, 0.03293821419746176, 0.029340427564774854, 0.030338164195340644, 0.03283453652649927, 0.03536180771921719, 0.029250182835333857, 0.029798222552624692, 0.033332555236594896, 0.03177980566726207, 0.033408334741077184, 0.032157986207291066, 0.028140667157261264, 0.02693987240210773, 0.03061269850489138, 0.02993547104020407, 0.032046611689784954, 0.03200168920959264, 0.030101231676005265, 0.02843462604395973, 0.03213282518911479, 0.03124599315796723, 0.02949577906053675 ], "mean_node_assignment_entropy": 3.4621146050984968, "normalized_node_entropy": 0.9989551143529707, "effective_mass_slots": 31.9239503208214, "node_slot_mutual_information_nats": 0.001241916743819349, "normalized_node_slot_mutual_information": 0.0003583414255010288, "centered_token_effective_rank": 1.461444131129282, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.006461513644315284, "token_RMS": 0.23332303149836242, "mean_token_offdiagonal_cosine": 0.9992159123462527, "temperature": 0.4997066855430603 }, { "layer": 2, "head": 6, "slices": 32, "native_nodes": 33555, "mass": [ 0.03232316022061312, 0.029907994619050957, 0.032299348413267016, 0.02929562547257593, 0.031396192438415314, 0.033694183910933075, 0.02900722208087996, 0.03047062804884437, 0.0323252040608072, 0.031242378996836413, 0.030294281579781673, 0.03203574552479257, 0.03248091612192713, 0.03270872073802298, 0.030856908116483544, 0.028736184670635243, 0.030546365885196953, 0.02786003116420006, 0.034317748392015505, 0.03186042338460928, 0.03166193891349403, 0.03183518348242563, 0.028634486897987182, 0.029584514175730017, 0.030512608002886264, 0.030480659976568295, 0.03250714907009943, 0.031199402837886255, 0.03368984850640047, 0.03060534346774898, 0.0338945696002999, 0.03173503113127584 ], "mean_node_assignment_entropy": 3.4626647407682536, "normalized_node_entropy": 0.9991138499534855, "effective_mass_slots": 31.95868277583581, "node_slot_mutual_information_nats": 0.0017791645068703232, "normalized_node_slot_mutual_information": 0.0005133583621974947, "centered_token_effective_rank": 1.2094488047112784, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.010788272263453755, "token_RMS": 0.19767957680509454, "mean_token_offdiagonal_cosine": 0.9969522643470661, "temperature": 0.4978848695755005 }, { "layer": 2, "head": 7, "slices": 32, "native_nodes": 33555, "mass": [ 0.03206208545873931, 0.0330794134131888, 0.03170047251577305, 0.0333285198325044, 0.029540274307218364, 0.03221616774088019, 0.03048847974497992, 0.03103615262126127, 0.03163710644310738, 0.03227496260019124, 0.031419200231194176, 0.029681417637318262, 0.03153562299799518, 0.03437089248381487, 0.029844045797963823, 0.029124630616947002, 0.032607971564059625, 0.029199918612811147, 0.03277727185551873, 0.032827504968342545, 0.029117686458896765, 0.031919699019694575, 0.0333216978599237, 0.0316018185850703, 0.030939741940585214, 0.03089958857586728, 0.029830141162582775, 0.029788431063274428, 0.031157372782658826, 0.030694727266226487, 0.029118976073852772, 0.030858007834669448 ], "mean_node_assignment_entropy": 3.463422066716301, "normalized_node_entropy": 0.9993323680314024, "effective_mass_slots": 31.967964016394482, "node_slot_mutual_information_nats": 0.0013122101359228644, "normalized_node_slot_mutual_information": 0.00037862381114002984, "centered_token_effective_rank": 1.2439180392921148, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.008790367446678275, "token_RMS": 0.1785262745715735, "mean_token_offdiagonal_cosine": 0.997522584543242, "temperature": 0.4991089403629303 }, { "layer": 2, "head": 8, "slices": 32, "native_nodes": 33555, "mass": [ 0.031189526958526236, 0.030162724162570022, 0.031170889764364725, 0.031355595243188525, 0.029191050283647456, 0.027555752387890656, 0.030222226247178106, 0.029342898202891002, 0.03218619965339911, 0.03721801684454645, 0.033382115600514906, 0.029318256601258266, 0.030156581699592826, 0.035735451173736166, 0.035008286283523704, 0.03193266079059578, 0.028192853820755148, 0.032708276741039, 0.03366467813103436, 0.030582970961641934, 0.03284208881529277, 0.030378198175412267, 0.029641615822766938, 0.03156890957202025, 0.02891570398748596, 0.030205402689421252, 0.031047473004166354, 0.03118152922948326, 0.030391685519494574, 0.032355075375726655, 0.033187272468441074, 0.028008033576678314 ], "mean_node_assignment_entropy": 3.4618881855848955, "normalized_node_entropy": 0.998889783491084, "effective_mass_slots": 31.923876230427272, "node_slot_mutual_information_nats": 0.0014660154143117943, "normalized_node_slot_mutual_information": 0.00042300263361888096, "centered_token_effective_rank": 1.2659720762541136, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.007832441278353895, "token_RMS": 0.22140515247485798, "mean_token_offdiagonal_cosine": 0.9987368212388698, "temperature": 0.499784380197525 }, { "layer": 3, "head": 1, "slices": 32, "native_nodes": 33555, "mass": [ 0.030634246007076517, 0.03430051909990841, 0.02967580263531345, 0.033391596893029854, 0.030889904158054725, 0.028364643387939443, 0.03284675851208924, 0.030978487066301748, 0.029293620387002582, 0.03161748400669378, 0.029315992524743862, 0.028439828986614263, 0.03491444181461587, 0.02993799508191926, 0.03049889300909488, 0.03145784134071056, 0.03172209219011486, 0.031681174163795, 0.03362562234053143, 0.03419852586427845, 0.03141829387383762, 0.031414493896103776, 0.030742518783932363, 0.0311984374889397, 0.028927838636473135, 0.03429143989538892, 0.031417435598538164, 0.032824095537198526, 0.02940202510186154, 0.030543652213216182, 0.03139740832837199, 0.028636891574984228 ], "mean_node_assignment_entropy": 3.4628679662309647, "normalized_node_entropy": 0.9991724884269326, "effective_mass_slots": 31.94798858695696, "node_slot_mutual_information_nats": 0.0012412575860070163, "normalized_node_slot_mutual_information": 0.00035815123275962567, "centered_token_effective_rank": 1.331089723610033, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.006976019360178705, "token_RMS": 0.283072349853265, "mean_token_offdiagonal_cosine": 0.999513518864199, "temperature": 0.499459832906723 }, { "layer": 3, "head": 2, "slices": 32, "native_nodes": 33555, "mass": [ 0.030723353112295477, 0.03151002430777945, 0.03163001244229081, 0.029561162927505637, 0.033420232603374456, 0.02999635333476488, 0.03048820195200857, 0.033787922107816774, 0.02744293727320409, 0.034187799824601295, 0.032319400837663534, 0.029157066208104623, 0.03310256825174097, 0.03532546293292977, 0.02948807210419585, 0.028976352511781763, 0.031155745485707875, 0.02851464540213724, 0.03047115139112558, 0.03184614049648496, 0.02972050865543411, 0.028634694582616017, 0.0323116870040515, 0.03197685348353209, 0.029789522681410257, 0.0329774835563429, 0.032817141896727416, 0.031216342321503644, 0.032272513863184984, 0.033506985844589346, 0.03167329023851268, 0.029998370062772447 ], "mean_node_assignment_entropy": 3.462757163897777, "normalized_node_entropy": 0.9991405176316108, "effective_mass_slots": 31.944572942602463, "node_slot_mutual_information_nats": 0.0012451415463519133, "normalized_node_slot_mutual_information": 0.00035927190682534413, "centered_token_effective_rank": 1.2375886408069612, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.007517631789075521, "token_RMS": 0.20910613468891534, "mean_token_offdiagonal_cosine": 0.9986831571877781, "temperature": 0.4987931251525879 }, { "layer": 3, "head": 3, "slices": 32, "native_nodes": 33555, "mass": [ 0.028613063509751853, 0.031826128627460454, 0.031023445107223754, 0.03140615371607256, 0.030957648348639982, 0.029544429553081765, 0.03543389218124836, 0.03125102396101324, 0.03338500150869197, 0.029388101364806074, 0.030718001105849407, 0.030279258543753424, 0.03202784264654769, 0.03076578098966963, 0.03351081025292241, 0.029691481853833482, 0.030164153692957153, 0.032174142471465654, 0.02963540542567119, 0.029770542741445773, 0.031015745331905865, 0.033200309788851246, 0.030282845479179345, 0.03280702982136166, 0.03202146365333777, 0.033158211081924766, 0.029819966116433012, 0.032399079917133915, 0.02942943781927381, 0.031755022127683656, 0.031415790810652613, 0.031128790508664288 ], "mean_node_assignment_entropy": 3.4626137297692976, "normalized_node_entropy": 0.9990991312904406, "effective_mass_slots": 31.9640975822563, "node_slot_mutual_information_nats": 0.001999592618258106, "normalized_node_slot_mutual_information": 0.0005769604708318295, "centered_token_effective_rank": 1.2655225481494308, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.009714935723868843, "token_RMS": 0.19112392249748042, "mean_token_offdiagonal_cosine": 0.9973685825190388, "temperature": 0.4970782995223999 }, { "layer": 3, "head": 4, "slices": 32, "native_nodes": 33555, "mass": [ 0.029231450140849255, 0.03116557245030211, 0.032949086423225575, 0.030920865568020824, 0.03175471478307403, 0.03226182688632127, 0.03275405352921595, 0.027558734446526408, 0.03239423768410845, 0.028730395989335165, 0.0317400358738318, 0.031022325838507864, 0.03013498217951006, 0.03088712956145186, 0.03377701492438621, 0.03309542645163631, 0.031950609217604786, 0.03392283018978737, 0.031082840462514583, 0.031278461444972525, 0.030619140829812697, 0.03525622522572507, 0.030889352097121476, 0.03041515523011861, 0.030835214449472157, 0.030666057943505104, 0.030339587583633885, 0.029461066608474428, 0.028939643300233752, 0.031402229451530116, 0.03123076501953417, 0.03133296847672065 ], "mean_node_assignment_entropy": 3.4629653937262237, "normalized_node_entropy": 0.999200600059784, "effective_mass_slots": 31.96004138698498, "node_slot_mutual_information_nats": 0.0015210221329948759, "normalized_node_slot_mutual_information": 0.00043887421767081216, "centered_token_effective_rank": 1.3417574299434174, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.008880786091616992, "token_RMS": 0.2175755370887468, "mean_token_offdiagonal_cosine": 0.9983026988404609, "temperature": 0.500282883644104 }, { "layer": 3, "head": 5, "slices": 32, "native_nodes": 33555, "mass": [ 0.031231425412419994, 0.030485254753064052, 0.031206622762896145, 0.03424323677007487, 0.029776712388051123, 0.030770116206244657, 0.033251973560473776, 0.032300382478248964, 0.03305960889939552, 0.027522642410000433, 0.03402764974116187, 0.030977919358538632, 0.029504742617801446, 0.031027110119483062, 0.033779241138273464, 0.032176137750660534, 0.030692328852557633, 0.0319047722395389, 0.03079520180713729, 0.031027330716786292, 0.03095721285937307, 0.03423831366771931, 0.03232178780706962, 0.03125256717142876, 0.02934177227394891, 0.030152685971406884, 0.030133689124752772, 0.028179547432302148, 0.028836605001994243, 0.032135757728700594, 0.03150951873032023, 0.031180132311067876 ], "mean_node_assignment_entropy": 3.461838490539599, "normalized_node_entropy": 0.9988754445320028, "effective_mass_slots": 31.956056059998023, "node_slot_mutual_information_nats": 0.002523220365740819, "normalized_node_slot_mutual_information": 0.0007280475017448633, "centered_token_effective_rank": 1.134536117779493, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.012063430651015197, "token_RMS": 0.2120984655663508, "mean_token_offdiagonal_cosine": 0.996807204288293, "temperature": 0.4986465275287628 }, { "layer": 3, "head": 6, "slices": 32, "native_nodes": 33555, "mass": [ 0.0322426820401729, 0.03367045626161955, 0.03236215168443962, 0.03318466432714556, 0.03297621613975396, 0.031160652832655594, 0.0311569454098175, 0.03118940497439223, 0.028053479246720792, 0.031527109520245886, 0.03129293682918812, 0.031242226611924426, 0.03171615961163619, 0.03242384293447609, 0.031154463897361928, 0.03226129750114291, 0.02863086914323629, 0.03197253032688358, 0.03144792769680994, 0.03135607559797333, 0.032416348669618573, 0.031129971020575378, 0.03250223195979366, 0.029267736612660238, 0.029169567354271033, 0.034012337403896226, 0.03139092449080507, 0.03059690837757924, 0.030104675765393957, 0.029360046151832842, 0.029095300036601723, 0.029931859411893188 ], "mean_node_assignment_entropy": 3.462971913682627, "normalized_node_entropy": 0.9992024813215379, "effective_mass_slots": 31.966165242436603, "node_slot_mutual_information_nats": 0.0017060935690187407, "normalized_node_slot_mutual_information": 0.0004922745462631779, "centered_token_effective_rank": 1.2815831763555694, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.011779312279421163, "token_RMS": 0.19109937478834157, "mean_token_offdiagonal_cosine": 0.996204297234233, "temperature": 0.49925631284713745 }, { "layer": 3, "head": 7, "slices": 32, "native_nodes": 33555, "mass": [ 0.03006524517704781, 0.031164159834098485, 0.03156517788539113, 0.03326064525720278, 0.031731421210958656, 0.02862379240054449, 0.03263330587384785, 0.03239368336141163, 0.03136115152066297, 0.026915254041934455, 0.03002508495148977, 0.031139820885685354, 0.03159373091738335, 0.02926833266360432, 0.03292154703173254, 0.0317781563135506, 0.031103048060711617, 0.03192423277115026, 0.03321026943078909, 0.031797431492846334, 0.031405926326928416, 0.03061273264811231, 0.03141292318581842, 0.03228008937284366, 0.03052579685379524, 0.0316390485436306, 0.03159672395604997, 0.03252685376715107, 0.028871849129086653, 0.030948560999495726, 0.031339252635841044, 0.03236475147616666 ], "mean_node_assignment_entropy": 3.4638556504078863, "normalized_node_entropy": 0.9994574738397345, "effective_mass_slots": 31.969293744891132, "node_slot_mutual_information_nats": 0.0009202212372141361, "normalized_node_slot_mutual_information": 0.00026551972308990813, "centered_token_effective_rank": 1.5834531962159974, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.0063335462299208105, "token_RMS": 0.24088874840340252, "mean_token_offdiagonal_cosine": 0.999316121770049, "temperature": 0.49817073345184326 }, { "layer": 3, "head": 8, "slices": 32, "native_nodes": 33555, "mass": [ 0.0267763646221533, 0.028234939301096988, 0.032651085810434126, 0.029790529987750225, 0.03251173885400676, 0.03251962952427938, 0.029950106626057602, 0.03224524460797329, 0.032895545512237986, 0.03210143210065781, 0.03298858157627594, 0.03098371512249763, 0.03193846373274676, 0.033498697683066375, 0.033816171591598256, 0.03050316337314524, 0.03023172789795213, 0.028657022332504774, 0.029709039517929366, 0.029710899163473293, 0.030599454514967615, 0.030774326573526106, 0.03167742964098456, 0.03164275053662633, 0.03469506345773672, 0.028179700832329374, 0.030246169313692936, 0.030120562039889675, 0.0318627268211943, 0.03233020406456742, 0.031161322073536078, 0.03499619134076882 ], "mean_node_assignment_entropy": 3.462669412571693, "normalized_node_entropy": 0.9991151979510162, "effective_mass_slots": 31.94294644060195, "node_slot_mutual_information_nats": 0.0012819751966031134, "normalized_node_slot_mutual_information": 0.00036989985173639314, "centered_token_effective_rank": 1.4303772642436554, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.008128530192029072, "token_RMS": 0.23397465623788286, "mean_token_offdiagonal_cosine": 0.9987827842960941, "temperature": 0.499138206243515 }, { "layer": 4, "head": 1, "slices": 32, "native_nodes": 33555, "mass": [ 0.031169980005786486, 0.03050776739186918, 0.03110239136003201, 0.03241952232946548, 0.03051183169829501, 0.029926829569728947, 0.03190931576795264, 0.03143092971148016, 0.02964712956998258, 0.03352834844220424, 0.028860838890171162, 0.03233157540010262, 0.03202700334669243, 0.03118509364069716, 0.030582090559472554, 0.033493948763983296, 0.02998714114504215, 0.03268815754041145, 0.029980900348778223, 0.03147008245051209, 0.029998534640113834, 0.02897378008237025, 0.031243021107878136, 0.03204754844264601, 0.03126697997010122, 0.03311813222523321, 0.030847389044495743, 0.03126512155194358, 0.030715961641636324, 0.03313570055400645, 0.031114838349547848, 0.03151211439475202 ], "mean_node_assignment_entropy": 3.4638573067169443, "normalized_node_entropy": 0.9994579517495074, "effective_mass_slots": 31.976607170229894, "node_slot_mutual_information_nats": 0.0011473028227197979, "normalized_node_slot_mutual_information": 0.00033104161854715237, "centered_token_effective_rank": 1.8032344097887045, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.006586270535483101, "token_RMS": 0.2547536467932466, "mean_token_offdiagonal_cosine": 0.9993420845516533, "temperature": 0.4977344572544098 }, { "layer": 4, "head": 2, "slices": 32, "native_nodes": 33555, "mass": [ 0.030078381364253358, 0.030599183126875495, 0.031740447253181994, 0.0331933973189163, 0.029307281939503493, 0.03388989288629105, 0.029448377916506393, 0.03363571810449907, 0.03224262146254514, 0.035351107736366405, 0.0330317412604231, 0.028279209712770315, 0.031012531821278076, 0.028911479755141415, 0.02981249871790729, 0.0338218071190589, 0.030724466962659614, 0.028147127713512386, 0.03418739818812589, 0.03026304994555055, 0.03142361083535488, 0.02980168504220525, 0.02874047454408252, 0.033065092392713206, 0.029744630468549377, 0.03383081018077627, 0.031667004958197034, 0.029679704439036537, 0.03107457858391107, 0.03001748104919053, 0.029821788183006973, 0.033455418555099016 ], "mean_node_assignment_entropy": 3.4594902069555937, "normalized_node_entropy": 0.9981978731157538, "effective_mass_slots": 31.93855476309606, "node_slot_mutual_information_nats": 0.004323686313354003, "normalized_node_slot_mutual_information": 0.001247552160527061, "centered_token_effective_rank": 1.179910140432528, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.013440345549492118, "token_RMS": 0.22821015423998756, "mean_token_offdiagonal_cosine": 0.9965781087588612, "temperature": 0.4986511170864105 }, { "layer": 4, "head": 3, "slices": 32, "native_nodes": 33555, "mass": [ 0.02955676437150329, 0.03244167948376225, 0.03137971188025412, 0.03049744807257016, 0.029178376794982697, 0.03127758823304085, 0.031870709680827926, 0.03224912368342882, 0.030489865316219895, 0.029453110749618323, 0.028720511529373174, 0.03174170746360179, 0.027184681869202854, 0.031548519079888336, 0.03313217082812621, 0.034914515363647834, 0.032374832402421276, 0.03175101568854401, 0.03227459425263461, 0.029031622655231203, 0.032876154081958396, 0.03407739999097197, 0.030538995235307953, 0.03055526909614494, 0.03382412327192318, 0.028454545718738334, 0.030537716953150208, 0.03192578441167958, 0.03208328579550945, 0.03313558545191745, 0.030388899705166113, 0.030533691308920616 ], "mean_node_assignment_entropy": 3.463001654880283, "normalized_node_entropy": 0.9992110628172116, "effective_mass_slots": 31.951392952219972, "node_slot_mutual_information_nats": 0.001214122871460166, "normalized_node_slot_mutual_information": 0.00035032180913708997, "centered_token_effective_rank": 1.3039487217095782, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.007648521923997801, "token_RMS": 0.2349536570074449, "mean_token_offdiagonal_cosine": 0.9989763068338527, "temperature": 0.4999978840351105 }, { "layer": 4, "head": 4, "slices": 32, "native_nodes": 33555, "mass": [ 0.03144330226274596, 0.029336620915712763, 0.031178733207992606, 0.030874923373457482, 0.03189195551343441, 0.031366951972847903, 0.030844959522793303, 0.030662719818280913, 0.03233470325963967, 0.031888011177787216, 0.030542135357106252, 0.0296234129652583, 0.030947168593838503, 0.030806696505873237, 0.02908802667951605, 0.02939653740231308, 0.029659979068039247, 0.0321751483137793, 0.030559033543457664, 0.031809432607967335, 0.029925815407655806, 0.03496007274221804, 0.03112152695706892, 0.029274174252242147, 0.03212608444984465, 0.030777391800169814, 0.0347548847448793, 0.03223144995980381, 0.032599442404453526, 0.03282094392778399, 0.030937734620559324, 0.03204002679693257 ], "mean_node_assignment_entropy": 3.4637434104962193, "normalized_node_entropy": 0.9994250882469441, "effective_mass_slots": 31.96920491128646, "node_slot_mutual_information_nats": 0.0010296824285003225, "normalized_node_slot_mutual_information": 0.000297103546657584, "centered_token_effective_rank": 1.4245480502272327, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.0065668782810947264, "token_RMS": 0.2582816650727403, "mean_token_offdiagonal_cosine": 0.999341083832916, "temperature": 0.4972933828830719 }, { "layer": 4, "head": 5, "slices": 32, "native_nodes": 33555, "mass": [ 0.03139430857454322, 0.030099547704238505, 0.031212588559462566, 0.03250919447446026, 0.029908723307093748, 0.038629638767690065, 0.031391379916731, 0.031206798655716607, 0.03202045290519816, 0.031098853143698375, 0.03223222975076941, 0.02946430672858826, 0.030308817676958925, 0.031614935980292906, 0.033038649403392566, 0.0281229547224245, 0.031351758161041514, 0.03120570480318374, 0.02887377319284335, 0.033650958571417054, 0.031432632933355645, 0.0328037157300052, 0.027422211548132257, 0.03119439821989521, 0.02972214800515077, 0.034219952513239116, 0.030430328763267567, 0.028716758889304493, 0.03335027752677013, 0.02728999838940194, 0.031761631096036434, 0.03232037156321815 ], "mean_node_assignment_entropy": 3.4621899982825437, "normalized_node_entropy": 0.9989768682275189, "effective_mass_slots": 31.926392693000984, "node_slot_mutual_information_nats": 0.0012430265843605959, "normalized_node_slot_mutual_information": 0.0003586616577900357, "centered_token_effective_rank": 1.3652129401236293, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.007100790165417102, "token_RMS": 0.17280567648600997, "mean_token_offdiagonal_cosine": 0.9983355330260624, "temperature": 0.5005728006362915 }, { "layer": 4, "head": 6, "slices": 32, "native_nodes": 33555, "mass": [ 0.02900809788663729, 0.03212570589723521, 0.033280320523572895, 0.02878831851649588, 0.028791567111964324, 0.031756056856345694, 0.032837683459234, 0.03080290090162243, 0.030524561444227768, 0.030294033832711516, 0.030323883702981544, 0.03195312721747753, 0.032467944633186245, 0.03424841332966952, 0.03368218289628556, 0.03206714953438409, 0.03468504499934359, 0.028556411921136463, 0.030570016372152314, 0.032928686348865435, 0.032643050722019576, 0.03037921268214831, 0.03133709460542421, 0.030015546578601403, 0.031045719477882455, 0.028723865850867305, 0.031341158481923446, 0.03321666547801274, 0.030868092422322108, 0.03149742043907837, 0.02952662125244268, 0.029713444885256714 ], "mean_node_assignment_entropy": 3.462367083363145, "normalized_node_entropy": 0.9990279641810387, "effective_mass_slots": 31.955436979762954, "node_slot_mutual_information_nats": 0.001975254493447487, "normalized_node_slot_mutual_information": 0.0005699379724380661, "centered_token_effective_rank": 1.131663192402545, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.01227923065519137, "token_RMS": 0.2090860927957742, "mean_token_offdiagonal_cosine": 0.9965340808999389, "temperature": 0.49868011474609375 }, { "layer": 4, "head": 7, "slices": 32, "native_nodes": 33555, "mass": [ 0.030450215671867058, 0.030080378494547262, 0.031050319549219742, 0.033552008568230283, 0.02880973418997752, 0.030764675387734364, 0.032483874126156366, 0.031549526342153196, 0.028435322631890435, 0.031116357337249906, 0.032998828290605166, 0.03243772618965507, 0.029726764590220896, 0.030097040332739442, 0.031571499227083064, 0.03185009519794253, 0.03187196913825171, 0.032272918315859954, 0.029316319106472817, 0.032145911871814065, 0.02987304055790462, 0.03171806122556455, 0.03200338053385346, 0.031059297389816977, 0.03596009620920661, 0.029072432010708648, 0.03281718376602613, 0.029197133230851723, 0.031244030568735946, 0.035131259505363464, 0.030482804889144993, 0.028859795667447573 ], "mean_node_assignment_entropy": 3.4632526645114856, "normalized_node_entropy": 0.9992834888872418, "effective_mass_slots": 31.951714016169124, "node_slot_mutual_information_nats": 0.0009731617015731331, "normalized_node_slot_mutual_information": 0.00028079511216852487, "centered_token_effective_rank": 1.8115874215676546, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.006064307434543511, "token_RMS": 0.26207985062041406, "mean_token_offdiagonal_cosine": 0.9994970221408978, "temperature": 0.4991609454154968 }, { "layer": 4, "head": 8, "slices": 32, "native_nodes": 33555, "mass": [ 0.030471021486739077, 0.03282837832443424, 0.03209780985807283, 0.030627028669730647, 0.027155125788614908, 0.028896719905182958, 0.03338489473790859, 0.031574736491418544, 0.02966874114438534, 0.028364076670700542, 0.03023685247489813, 0.03012956217399849, 0.030139779845496047, 0.03071605724942718, 0.030556330180167263, 0.03317182463990818, 0.03301410449061206, 0.03141352015495691, 0.028639670520035028, 0.03276037491048667, 0.03238072657727028, 0.03047133717986248, 0.031119483485296112, 0.033697740164199755, 0.03551700268878327, 0.030188282225417, 0.03183366150200154, 0.031958541792647135, 0.031679166596304584, 0.033449644269539655, 0.031135956975009007, 0.030721846634874293 ], "mean_node_assignment_entropy": 3.462950352175883, "normalized_node_entropy": 0.9991962599857671, "effective_mass_slots": 31.951020369693033, "node_slot_mutual_information_nats": 0.0012537645913193174, "normalized_node_slot_mutual_information": 0.0003617599916677114, "centered_token_effective_rank": 1.5925904693606947, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.007535648264497471, "token_RMS": 0.19124164249705633, "mean_token_offdiagonal_cosine": 0.9985195094415306, "temperature": 0.5001669526100159 }, { "layer": 5, "head": 1, "slices": 32, "native_nodes": 33555, "mass": [ 0.029773267273832907, 0.028497688828646523, 0.02889085088384265, 0.031723995288441796, 0.032035969633893824, 0.029856183583666523, 0.03275061989756255, 0.03417729727303266, 0.033169667816616406, 0.03211969023229308, 0.031569338820718156, 0.030400860347458793, 0.029436720940938757, 0.03296632463404735, 0.030508811744059044, 0.03112636826745916, 0.033737957937527194, 0.031340949314561356, 0.03277331721406769, 0.030625409464336707, 0.03141067703804766, 0.03284575331347201, 0.029289911438910406, 0.032958843353972, 0.030798827347694947, 0.030725896826385563, 0.030214158134684, 0.033774930298175194, 0.029074856045892796, 0.030486108880862537, 0.0304883229828102, 0.030450425008368767 ], "mean_node_assignment_entropy": 3.4632728301496885, "normalized_node_entropy": 0.9992893074604882, "effective_mass_slots": 31.962360032391256, "node_slot_mutual_information_nats": 0.0012861313384742168, "normalized_node_slot_mutual_information": 0.0003710990607897275, "centered_token_effective_rank": 1.5278002393939563, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.007018777848785056, "token_RMS": 0.20539077787963297, "mean_token_offdiagonal_cosine": 0.9988251488553702, "temperature": 0.497806191444397 }, { "layer": 5, "head": 2, "slices": 32, "native_nodes": 33555, "mass": [ 0.03158029916811018, 0.03177347824709095, 0.03224853468688896, 0.029851575746395125, 0.029922099455340648, 0.03133827127569559, 0.030551083703351755, 0.03374275560229195, 0.0327957782383679, 0.030988706831755503, 0.033157398880544996, 0.03131943532022727, 0.029639160520567958, 0.03031212235028636, 0.030112677302715, 0.030755485038781925, 0.033815199991979836, 0.031211979296310987, 0.030869966553866358, 0.030020086615311817, 0.03233166983529577, 0.02882314576123328, 0.028373367380283662, 0.030954033432903347, 0.032774299470425504, 0.030848504520254384, 0.031213759809106283, 0.032632960396498385, 0.029170363845872356, 0.03290440934068923, 0.034672672686494155, 0.029294719206811225 ], "mean_node_assignment_entropy": 3.462623787349366, "normalized_node_entropy": 0.9991020332946182, "effective_mass_slots": 31.96246367412628, "node_slot_mutual_information_nats": 0.0019384167518854412, "normalized_node_slot_mutual_information": 0.0005593088470242436, "centered_token_effective_rank": 1.300575428138547, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.009121948251647749, "token_RMS": 0.2076230828964597, "mean_token_offdiagonal_cosine": 0.9980392094762613, "temperature": 0.4966803789138794 }, { "layer": 5, "head": 3, "slices": 32, "native_nodes": 33555, "mass": [ 0.028928638588136966, 0.030353654819804215, 0.03255334750829244, 0.031962260321413256, 0.03248612086561047, 0.032143033225038795, 0.031465725986515425, 0.030207839333361387, 0.031154184411662185, 0.029353752614040166, 0.0313242647730257, 0.03287322475685792, 0.03225419113359487, 0.03136570816225399, 0.030078657481537954, 0.03186181648053599, 0.03126816660667599, 0.030015059714078585, 0.033192322100328804, 0.032406388292559, 0.029884750208388745, 0.031913484255388445, 0.032402947450462176, 0.02925031964469109, 0.03225913766364665, 0.02980228672510438, 0.03226173980800748, 0.029051197186609777, 0.03190771807903834, 0.03138202331669328, 0.03237540620764807, 0.03026063260550853 ], "mean_node_assignment_entropy": 3.464002997557718, "normalized_node_entropy": 0.9994999892402048, "effective_mass_slots": 31.975898357577734, "node_slot_mutual_information_nats": 0.0009794451365241486, "normalized_node_slot_mutual_information": 0.00028260812825724055, "centered_token_effective_rank": 1.2714743204428252, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.006545832107943555, "token_RMS": 0.18337921233584678, "mean_token_offdiagonal_cosine": 0.9987060003050465, "temperature": 0.4997660219669342 }, { "layer": 5, "head": 4, "slices": 32, "native_nodes": 33555, "mass": [ 0.03013700526721155, 0.02937260388616362, 0.033627641593587505, 0.03211799927212375, 0.02810961283752263, 0.031106487334533948, 0.02856264220687489, 0.03234432558450671, 0.032209712011075434, 0.03139017768750433, 0.033280350816328005, 0.029345609369529395, 0.032111935523403703, 0.03128654237394183, 0.030986136419029975, 0.03224630305903525, 0.03059141013220079, 0.032122322214114256, 0.03289296982587583, 0.03091223920853549, 0.031366886863644014, 0.030966814853125527, 0.030876324968719213, 0.03207569824712915, 0.03030843707746595, 0.030397326606080865, 0.03273724691167689, 0.03138489981155969, 0.03235222758063881, 0.031002031409517076, 0.031718996398882544, 0.03005908252195361 ], "mean_node_assignment_entropy": 3.4636292757262126, "normalized_node_entropy": 0.9993921559136077, "effective_mass_slots": 31.973368223498174, "node_slot_mutual_information_nats": 0.0012740375513482505, "normalized_node_slot_mutual_information": 0.0003676095314472878, "centered_token_effective_rank": 1.374422104446845, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.007895213465434089, "token_RMS": 0.2534166567682975, "mean_token_offdiagonal_cosine": 0.9990198971284563, "temperature": 0.5010371208190918 }, { "layer": 5, "head": 5, "slices": 32, "native_nodes": 33555, "mass": [ 0.03160960748092266, 0.030293084478144265, 0.03241107307775261, 0.03450969438714014, 0.029968590014134697, 0.03329568861573912, 0.03102251592656616, 0.033402157099780144, 0.03546070559531774, 0.029147611379972016, 0.031245970974587912, 0.029651263422876564, 0.031811678902673626, 0.032020752454342244, 0.03189879512832606, 0.03170176704676891, 0.030644615623456606, 0.033062181795758934, 0.026991043402865417, 0.0286212283465137, 0.03014771475862851, 0.030524216499776335, 0.033184970787477515, 0.03141896552685884, 0.028543453187198933, 0.03243036059341573, 0.031176954906113528, 0.031621987660956835, 0.03476484519415766, 0.027311764806930896, 0.028864375776757677, 0.03124036519010924 ], "mean_node_assignment_entropy": 3.4623221301380127, "normalized_node_entropy": 0.9990149934220446, "effective_mass_slots": 31.935119878749667, "node_slot_mutual_information_nats": 0.0013842107044004237, "normalized_node_slot_mutual_information": 0.00039939878375678205, "centered_token_effective_rank": 1.9764677102622255, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.007221469610757128, "token_RMS": 0.21082309209785474, "mean_token_offdiagonal_cosine": 0.998893145535032, "temperature": 0.49816906452178955 }, { "layer": 5, "head": 6, "slices": 32, "native_nodes": 33555, "mass": [ 0.03093457160512855, 0.03348017181779825, 0.037797794748842053, 0.03049439617875764, 0.03503506567216483, 0.03367764447175822, 0.028285782633568842, 0.030411462478573924, 0.0322549954601295, 0.028266840944636457, 0.027964399450472046, 0.03255567243870893, 0.03116455046406465, 0.030117326708475268, 0.03289694123829402, 0.030091465029695088, 0.033469359584973137, 0.031017834139645605, 0.030588979224963474, 0.032516076620776414, 0.029832704852244533, 0.030598480739792988, 0.02989071424800146, 0.027633601460051314, 0.03242330582194322, 0.02866445380126892, 0.031214205468676606, 0.030280675611376425, 0.03302668654607591, 0.02731483579492395, 0.03388629796527711, 0.03221270640718876 ], "mean_node_assignment_entropy": 3.4618305477938556, "normalized_node_entropy": 0.9988731527400239, "effective_mass_slots": 31.915936070010762, "node_slot_mutual_information_nats": 0.0012749005799497226, "normalized_node_slot_mutual_information": 0.00036785854886398566, "centered_token_effective_rank": 1.617879691449437, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.006897989877947991, "token_RMS": 0.19327441928847744, "mean_token_offdiagonal_cosine": 0.9987470976287735, "temperature": 0.49954020977020264 }, { "layer": 5, "head": 7, "slices": 32, "native_nodes": 33555, "mass": [ 0.029281782056076608, 0.030561922205163263, 0.029280210978702217, 0.02954357457814194, 0.03323916732196907, 0.029887645486578504, 0.032193289554564526, 0.03324531755681996, 0.02975069141716782, 0.03083680092565679, 0.03028476363842586, 0.031673084285025306, 0.03172192912265123, 0.031625155185426586, 0.02939704237265712, 0.029457492549363716, 0.03162286015086719, 0.03051462101570639, 0.03238880511497069, 0.03173157044866369, 0.03268069296935386, 0.03397657874149522, 0.028512715103473574, 0.03151986995794355, 0.03347256701279194, 0.033592613845244285, 0.033431303295796876, 0.029480094931916462, 0.030293295758225012, 0.0311159557616692, 0.03245127443393498, 0.03123531248023579 ], "mean_node_assignment_entropy": 3.4629311520602344, "normalized_node_entropy": 0.999190720003441, "effective_mass_slots": 31.963029390896864, "node_slot_mutual_information_nats": 0.0016487512950540406, "normalized_node_slot_mutual_information": 0.0004757290634067441, "centered_token_effective_rank": 1.272715787785331, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.010767257096062018, "token_RMS": 0.2328666406319816, "mean_token_offdiagonal_cosine": 0.9978703849944145, "temperature": 0.5007913708686829 }, { "layer": 5, "head": 8, "slices": 32, "native_nodes": 33555, "mass": [ 0.031960050157082455, 0.030918456411294194, 0.029229724853498146, 0.03216768772075001, 0.02877261567216368, 0.030919311148594805, 0.031008035628420107, 0.03253566036773828, 0.033036946700981645, 0.03297387379367471, 0.03325359680651153, 0.02953475514969995, 0.029513692112442206, 0.032401517339937826, 0.0317402698484299, 0.03251439904178766, 0.03004455389606226, 0.03534818200394234, 0.029117851445681468, 0.027360496538170954, 0.031022813907269214, 0.031600151042050245, 0.03195543640902822, 0.032956654395985274, 0.03003319352004811, 0.03415592940616645, 0.031003014186727202, 0.03230785878466487, 0.03150448624620298, 0.0304081597861839, 0.028593378764791063, 0.0301072470327547 ], "mean_node_assignment_entropy": 3.4632225933648155, "normalized_node_entropy": 0.9992748121884069, "effective_mass_slots": 31.9515547686771, "node_slot_mutual_information_nats": 0.0009982488311481852, "normalized_node_slot_mutual_information": 0.0002880337276541382, "centered_token_effective_rank": 1.7859571375047967, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.005653407926438627, "token_RMS": 0.18284284928546254, "mean_token_offdiagonal_cosine": 0.9990645441797151, "temperature": 0.5000309944152832 }, { "layer": 6, "head": 1, "slices": 32, "native_nodes": 33555, "mass": [ 0.028823677215110685, 0.032763117836570974, 0.029425400752485942, 0.03200763954706473, 0.03208984612625893, 0.031303210310430346, 0.03091407392627524, 0.02896292092848412, 0.033525038858165934, 0.030893345530773664, 0.030803633444460844, 0.031809911380655594, 0.03130608295133378, 0.03290616896660171, 0.0320800710326268, 0.027996531339211553, 0.032524819378070106, 0.03239637451611915, 0.033675605838588515, 0.02731713258843542, 0.033325191650879916, 0.030228965419450488, 0.02873653126923547, 0.029530488449046002, 0.03374731140040531, 0.032098676676282835, 0.031171718038892786, 0.03218954843630638, 0.03208028185785687, 0.0317707505231442, 0.02930972073291004, 0.03228621342058571 ], "mean_node_assignment_entropy": 3.462781415232488, "normalized_node_entropy": 0.9991475150876754, "effective_mass_slots": 31.95313688421991, "node_slot_mutual_information_nats": 0.001488941811359723, "normalized_node_slot_mutual_information": 0.00042961779348418057, "centered_token_effective_rank": 1.2009444567007814, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.009023029886169516, "token_RMS": 0.2095520748506767, "mean_token_offdiagonal_cosine": 0.9981105778173739, "temperature": 0.4968530237674713 }, { "layer": 6, "head": 2, "slices": 32, "native_nodes": 33555, "mass": [ 0.02819575192036243, 0.029426965389787625, 0.030719268240335447, 0.02930574565903829, 0.03354911933804461, 0.03292651492189169, 0.029834423861723778, 0.03591142293171811, 0.028101192226914033, 0.029321728448861314, 0.03281134459746699, 0.030817191596203317, 0.02883375873426987, 0.03331958766298924, 0.03414243792407542, 0.03155987579328322, 0.030506454857912528, 0.030972247294478776, 0.03017487971098532, 0.033123415339509986, 0.02908227900394232, 0.03200148507611857, 0.03035654757350057, 0.029108006408738674, 0.02994110722135297, 0.032262128980331, 0.03120036804578136, 0.030492503527063956, 0.031226072932416308, 0.03352887468256257, 0.03528138127101905, 0.03196591909726683 ], "mean_node_assignment_entropy": 3.46097927865175, "normalized_node_entropy": 0.9986275283860684, "effective_mass_slots": 31.9368093030667, "node_slot_mutual_information_nats": 0.002779962559857019, "normalized_node_slot_mutual_information": 0.0008021276397925418, "centered_token_effective_rank": 1.3726754832808001, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.012247073967838138, "token_RMS": 0.24639217745185824, "mean_token_offdiagonal_cosine": 0.9975347906371376, "temperature": 0.49182015657424927 }, { "layer": 6, "head": 3, "slices": 32, "native_nodes": 33555, "mass": [ 0.031175949865589376, 0.03474864085614148, 0.030620619731037507, 0.0316479429439423, 0.030345094038362014, 0.02926964762901926, 0.034260305376050904, 0.03129218319518931, 0.033420810387684234, 0.032399208112361, 0.03108597422939284, 0.03315800816917576, 0.028560004195312492, 0.03061262737049811, 0.029558154363298964, 0.031776958335018375, 0.03021925175697113, 0.031916403026943006, 0.03247664457800722, 0.02973134605620057, 0.03102192115652021, 0.031396446151778515, 0.032952220598270424, 0.031144822382670263, 0.030560338005785472, 0.028631961360993427, 0.02993832456194738, 0.031127945090251467, 0.031111107509893536, 0.030453649929065033, 0.03127168102578192, 0.032113808175711835 ], "mean_node_assignment_entropy": 3.4627358978857927, "normalized_node_entropy": 0.999134381557605, "effective_mass_slots": 31.966471841700297, "node_slot_mutual_information_nats": 0.0019517006881470955, "normalized_node_slot_mutual_information": 0.0005631417808178784, "centered_token_effective_rank": 1.5542520272299365, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.010927234032400568, "token_RMS": 0.21557567899784405, "mean_token_offdiagonal_cosine": 0.9976337641075692, "temperature": 0.4994584023952484 }, { "layer": 6, "head": 4, "slices": 32, "native_nodes": 33555, "mass": [ 0.03242465175095418, 0.030569464727823202, 0.03287212401378698, 0.030646789039971842, 0.030395084191926935, 0.03075902136843449, 0.030871021368398855, 0.034253302816650746, 0.030236392956481856, 0.03316380731331806, 0.03178379106614419, 0.03334914626523957, 0.03060488985190003, 0.030557861868827765, 0.030522419875974788, 0.030941565746294104, 0.03175669867168452, 0.030678490107631188, 0.03076731517173782, 0.0336050264817661, 0.02888763619473307, 0.030339925128740643, 0.03296233102015351, 0.03224070312660962, 0.030540867685281366, 0.028275883920360848, 0.03075045860405651, 0.03190608927193208, 0.030463537209778426, 0.030089549105900303, 0.03251767537759617, 0.030266478898414752 ], "mean_node_assignment_entropy": 3.4631452306981747, "normalized_node_entropy": 0.9992524900413043, "effective_mass_slots": 31.97086769519528, "node_slot_mutual_information_nats": 0.0016798729247318178, "normalized_node_slot_mutual_information": 0.00048470886756684647, "centered_token_effective_rank": 1.4589731401322033, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.009409501553264256, "token_RMS": 0.23333736072823022, "mean_token_offdiagonal_cosine": 0.9985452842759984, "temperature": 0.49977654218673706 }, { "layer": 6, "head": 5, "slices": 32, "native_nodes": 33555, "mass": [ 0.0317453653552332, 0.03286010830053504, 0.03101638563109428, 0.029806367373181392, 0.029456151236526444, 0.03201592098263741, 0.030945696247814576, 0.028556742556110043, 0.03172408308405854, 0.03117861880455513, 0.0294703142765497, 0.03459435708031464, 0.029744403164224838, 0.0303030205168626, 0.03394768394606423, 0.030925157093507506, 0.03318517567328148, 0.03164404340797788, 0.03488403562738916, 0.029783523966455724, 0.03280022721268562, 0.03047101280560811, 0.0319096244206604, 0.031677068452357876, 0.028054708081048197, 0.030908653244926844, 0.02905661342819594, 0.03203138681263233, 0.03068622445137823, 0.030586648921269523, 0.032731549798301, 0.0312991279232184 ], "mean_node_assignment_entropy": 3.4634258421295376, "normalized_node_entropy": 0.9993334573853931, "effective_mass_slots": 31.95777567454944, "node_slot_mutual_information_nats": 0.0009896791797525673, "normalized_node_slot_mutual_information": 0.00028556104894001717, "centered_token_effective_rank": 1.4766172853248172, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.007519377654720804, "token_RMS": 0.2408090014289525, "mean_token_offdiagonal_cosine": 0.9990184932238887, "temperature": 0.5017659068107605 }, { "layer": 6, "head": 6, "slices": 32, "native_nodes": 33555, "mass": [ 0.03132164406727135, 0.03402824937034969, 0.03189713051080517, 0.030709740187461535, 0.0305862997395016, 0.0304237381659638, 0.030577641545798097, 0.029546775423226795, 0.033536267180771254, 0.03330290520414122, 0.029912906217192718, 0.034000745017457305, 0.02959951687914491, 0.029702000609276955, 0.032854495466259494, 0.02798354029488926, 0.030088866478954337, 0.032963012772314046, 0.032235643732252, 0.030138789646848182, 0.031176047148067803, 0.031585323397470395, 0.03393010018817218, 0.029508577028297487, 0.031940440133029706, 0.029613415729085178, 0.028227204358894357, 0.0334905018435183, 0.02794129206032791, 0.032562524465517356, 0.02939852816186653, 0.035216137017394705 ], "mean_node_assignment_entropy": 3.4621352491955664, "normalized_node_entropy": 0.9989610709802639, "effective_mass_slots": 31.94075617523602, "node_slot_mutual_information_nats": 0.00174756817755517, "normalized_node_slot_mutual_information": 0.0005042415886748414, "centered_token_effective_rank": 1.3485222390646074, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.010444380357589315, "token_RMS": 0.27646220097660507, "mean_token_offdiagonal_cosine": 0.9986176758159534, "temperature": 0.49861258268356323 }, { "layer": 6, "head": 7, "slices": 32, "native_nodes": 33555, "mass": [ 0.03132016393394189, 0.0320713869842096, 0.03209492287566216, 0.03165570757689179, 0.032440222455892224, 0.030711216566746413, 0.030832578997763167, 0.03144169468901726, 0.03135659329003663, 0.03114507880382159, 0.0312048113434028, 0.03345153915435364, 0.029337001996035193, 0.032229777515953704, 0.03143482656663442, 0.02938204148520329, 0.028727701964860717, 0.030312300655136536, 0.03338156932907468, 0.03159286145744239, 0.02965926883569854, 0.03148436700999337, 0.032251795394852095, 0.030081372455177675, 0.030544727480763686, 0.028558085808196555, 0.03385492102168761, 0.03197904747401908, 0.03155350245933937, 0.030195954800675707, 0.031280370351721466, 0.032432589402349866 ], "mean_node_assignment_entropy": 3.4630511366902907, "normalized_node_entropy": 0.9992253402495941, "effective_mass_slots": 31.97380640760689, "node_slot_mutual_information_nats": 0.001865881152366189, "normalized_node_slot_mutual_information": 0.000538379497081377, "centered_token_effective_rank": 1.3936561542442694, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.010910624627460955, "token_RMS": 0.2141022081387929, "mean_token_offdiagonal_cosine": 0.9974886032761541, "temperature": 0.4980800449848175 }, { "layer": 6, "head": 8, "slices": 32, "native_nodes": 33555, "mass": [ 0.032278224286743894, 0.03201348088662825, 0.03380067500822449, 0.02983968714797502, 0.03277937137595215, 0.03169929050314339, 0.03341420332932682, 0.03471112164830135, 0.03172007102323323, 0.030524306572699507, 0.02854579312924194, 0.030705802024161295, 0.02919383051009598, 0.030293118127213293, 0.03111990793580235, 0.030365634499490483, 0.031909114589231624, 0.03261285530839844, 0.028200135530295822, 0.030470741827625433, 0.02922534331476177, 0.03095884835429411, 0.030841199524124445, 0.030504440970584527, 0.030904666997904198, 0.02961368190154884, 0.0355669797345554, 0.030552797025310233, 0.030948454672183424, 0.03210537274581961, 0.03173320315558435, 0.030847646511792514 ], "mean_node_assignment_entropy": 3.4613758171061906, "normalized_node_entropy": 0.998741945198417, "effective_mass_slots": 31.957462504411005, "node_slot_mutual_information_nats": 0.0030299046577217936, "normalized_node_slot_mutual_information": 0.0008742456848123207, "centered_token_effective_rank": 1.3186575365508941, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.010800358868353746, "token_RMS": 0.2417236933311779, "mean_token_offdiagonal_cosine": 0.9980092349929357, "temperature": 0.49664855003356934 }, { "layer": 7, "head": 1, "slices": 32, "native_nodes": 33555, "mass": [ 0.035765923521863224, 0.02883744153510954, 0.03397881724101352, 0.034105129813076826, 0.028979464292841543, 0.029541451487684105, 0.029343724224370597, 0.028943855737824053, 0.03280146509118077, 0.02985425168064672, 0.03183784925109218, 0.029900975985336795, 0.03321983738357043, 0.03171958134489778, 0.03284620553801302, 0.028884495283801603, 0.031216317298888022, 0.03105410536074759, 0.03286397428884758, 0.028428538074857644, 0.03084872949936718, 0.028930859576570458, 0.031159836436702195, 0.03142345085248269, 0.031004988030598375, 0.03319225557217289, 0.03136040561407418, 0.03520237771773939, 0.031004200296232517, 0.0312052399707194, 0.03130021936227829, 0.02924403254225275 ], "mean_node_assignment_entropy": 3.4613385091777267, "normalized_node_entropy": 0.9987311804057408, "effective_mass_slots": 31.940504259103704, "node_slot_mutual_information_nats": 0.0025364211834415507, "normalized_node_slot_mutual_information": 0.0007318564525913682, "centered_token_effective_rank": 1.4822930016336795, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.01171257949289937, "token_RMS": 0.22458473913305638, "mean_token_offdiagonal_cosine": 0.9972555243095024, "temperature": 0.49545732140541077 }, { "layer": 7, "head": 2, "slices": 32, "native_nodes": 33555, "mass": [ 0.03171484097288851, 0.031429170132140756, 0.030324329343289824, 0.0348499833775906, 0.02862511412743626, 0.031438436441322215, 0.028635911649056118, 0.03176652689885852, 0.029938378206238698, 0.03059169040603452, 0.032117116688625096, 0.02981628113393101, 0.03351521556324077, 0.032155955454684054, 0.03281018489389177, 0.033028911053412734, 0.0332379794938676, 0.02999771868874172, 0.031495743613307675, 0.031420349707068425, 0.03361037039515639, 0.029141624971076283, 0.0295056335575298, 0.02831305310673009, 0.031217404509679587, 0.031106904058074245, 0.030338335237012695, 0.029581052257571575, 0.03550460802569082, 0.03125895748611286, 0.030082776123079684, 0.03142944236609748 ], "mean_node_assignment_entropy": 3.4629109196754535, "normalized_node_entropy": 0.9991848821712033, "effective_mass_slots": 31.951728438267594, "node_slot_mutual_information_nats": 0.0013153579091720324, "normalized_node_slot_mutual_information": 0.00037953206651131335, "centered_token_effective_rank": 1.5923463103940205, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.008125877623420236, "token_RMS": 0.18715493949682185, "mean_token_offdiagonal_cosine": 0.9981339732181308, "temperature": 0.5002118945121765 }, { "layer": 7, "head": 3, "slices": 32, "native_nodes": 33555, "mass": [ 0.03070005116024754, 0.03120126025502825, 0.03398796185426999, 0.028058510377110515, 0.03351770715890423, 0.030242791252923763, 0.031176561643409047, 0.03217262629396806, 0.029433522703113026, 0.033684861065711336, 0.029363215561706926, 0.030052666482727267, 0.03206213978730501, 0.028974154066766823, 0.03156693048820714, 0.031027732279932708, 0.03271191903951892, 0.03436432919248377, 0.03326027009998645, 0.030588515500593976, 0.02919991489629146, 0.03231708580516184, 0.03465371653298839, 0.03338523815888226, 0.028601437210126316, 0.0285271381260168, 0.03225211352232032, 0.03107581292324167, 0.02861731427686569, 0.03298146812318246, 0.031218601895140707, 0.0290224321656714 ], "mean_node_assignment_entropy": 3.4616676862930573, "normalized_node_entropy": 0.9988261608441131, "effective_mass_slots": 31.941992589661606, "node_slot_mutual_information_nats": 0.002253839947470304, "normalized_node_slot_mutual_information": 0.0006503207430345699, "centered_token_effective_rank": 1.184899659683561, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.01102143262362149, "token_RMS": 0.1860134380746528, "mean_token_offdiagonal_cosine": 0.9963959725914332, "temperature": 0.49781903624534607 }, { "layer": 7, "head": 4, "slices": 32, "native_nodes": 33555, "mass": [ 0.031222335429149668, 0.02962570243081123, 0.03040824896927838, 0.031029228043549996, 0.03164121932990548, 0.030496240765269335, 0.028913709802397353, 0.031028264258215085, 0.0334383612136992, 0.02843505312137571, 0.03223591573356543, 0.031067145431799763, 0.03289270169362415, 0.032608766717365255, 0.03074639767642624, 0.027200952287629577, 0.03236296675963855, 0.03136564009807501, 0.03347743473621758, 0.028933121741535214, 0.030726676399362565, 0.0318748443012061, 0.031167834302577847, 0.02990066409177635, 0.02991540706080192, 0.031860675762674345, 0.03498825377447028, 0.032291687513915195, 0.031047845475243536, 0.031046827181989436, 0.03329438183563794, 0.0327554956478758 ], "mean_node_assignment_entropy": 3.463092589718382, "normalized_node_entropy": 0.9992373010652055, "effective_mass_slots": 31.957651905406966, "node_slot_mutual_information_nats": 0.001319058687380359, "normalized_node_slot_mutual_information": 0.00038059988538502985, "centered_token_effective_rank": 1.791909935643139, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.006946722044930349, "token_RMS": 0.20494215656427808, "mean_token_offdiagonal_cosine": 0.9988456142985656, "temperature": 0.501105546951294 }, { "layer": 7, "head": 5, "slices": 32, "native_nodes": 33555, "mass": [ 0.030991714845300875, 0.029155952581058065, 0.03088516025121069, 0.032465736370135695, 0.029616123154945922, 0.030738318455728185, 0.03369660223722502, 0.033050559951938084, 0.032234038190585616, 0.030248808549699583, 0.03228272225431884, 0.03205606648883021, 0.029501307545269483, 0.030035809713599402, 0.03398093894482627, 0.029538285978790137, 0.03264918349652199, 0.02938272022643444, 0.02984393856244895, 0.03420976188659419, 0.031667394720039, 0.030492855062742397, 0.030594638839692673, 0.03039274193555219, 0.030105552232222395, 0.0305908673997586, 0.03359629727479261, 0.029419004478725124, 0.031381737684292296, 0.032194704751659926, 0.03050599540177871, 0.032494460510967334 ], "mean_node_assignment_entropy": 3.4633873547167924, "normalized_node_entropy": 0.9993223522654923, "effective_mass_slots": 31.96514185928118, "node_slot_mutual_information_nats": 0.0012586374486018848, "normalized_node_slot_mutual_information": 0.00036316600107501535, "centered_token_effective_rank": 1.4693618975021572, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.00789511687436374, "token_RMS": 0.23746614971116545, "mean_token_offdiagonal_cosine": 0.9989052687859544, "temperature": 0.4988757371902466 }, { "layer": 7, "head": 6, "slices": 32, "native_nodes": 33555, "mass": [ 0.031838558186547815, 0.03265625302110996, 0.030854713029558854, 0.03115547250419513, 0.03155834992492391, 0.02686421031382414, 0.02951942666406968, 0.031858139918822054, 0.030623921101340823, 0.030843565895467343, 0.031847270713479876, 0.030411733251557393, 0.029638243167526657, 0.030982546315913833, 0.03136677151614437, 0.030276168598023935, 0.028009747598953617, 0.030110152072581692, 0.03257179324490323, 0.028232931919676806, 0.031068870464414512, 0.03469850563757414, 0.031895240960467, 0.030032797934352293, 0.031940898646711145, 0.032762289381967784, 0.03575485683500216, 0.03289409838455869, 0.03187927956733258, 0.032034156422641874, 0.03251416169527728, 0.031304875644366015 ], "mean_node_assignment_entropy": 3.4628959430504596, "normalized_node_entropy": 0.9991805608306816, "effective_mass_slots": 31.951094134143432, "node_slot_mutual_information_nats": 0.0013104823868381743, "normalized_node_slot_mutual_information": 0.00037812528813275327, "centered_token_effective_rank": 1.2238509219384013, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.010028096023062667, "token_RMS": 0.22820787791787026, "mean_token_offdiagonal_cosine": 0.9980819099617215, "temperature": 0.5030057430267334 }, { "layer": 7, "head": 7, "slices": 32, "native_nodes": 33555, "mass": [ 0.028385486560129156, 0.031903433900383665, 0.028743081790830974, 0.03133327227461633, 0.03010565728657453, 0.03269429807007899, 0.030776892839028215, 0.0320201431473376, 0.031003269963238444, 0.032371110463107755, 0.03272024577786343, 0.037657248645337, 0.0302384565221186, 0.03001172407349146, 0.03097026823620037, 0.030914929937313165, 0.03576092674102589, 0.030772773894678283, 0.02922962490325446, 0.030690406562296685, 0.03212733721628956, 0.034546645956771455, 0.028604438222002173, 0.03165044837053008, 0.031219504724989845, 0.02916053825798688, 0.02785130081160682, 0.02939089210255724, 0.030699115660418486, 0.030674850316380998, 0.03050218205713041, 0.03526949450476899 ], "mean_node_assignment_entropy": 3.4624298222584766, "normalized_node_entropy": 0.9990460667996719, "effective_mass_slots": 31.926486264025478, "node_slot_mutual_information_nats": 0.0010061334402404754, "normalized_node_slot_mutual_information": 0.0002903087449414972, "centered_token_effective_rank": 1.8868443204778864, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.006079805053209361, "token_RMS": 0.25983976753209487, "mean_token_offdiagonal_cosine": 0.9994973716922612, "temperature": 0.4998946487903595 }, { "layer": 7, "head": 8, "slices": 32, "native_nodes": 33555, "mass": [ 0.033781273644281525, 0.03308695749107833, 0.03374543095696143, 0.02935923609142642, 0.032027810656567866, 0.03244386818240526, 0.030477166577236627, 0.030704096881188414, 0.029433997273844804, 0.029662663823363057, 0.03018392349105268, 0.03096062134168104, 0.030088202362610054, 0.03043572675898025, 0.031045330801043082, 0.033377269248021126, 0.03038621305672205, 0.031130683508629044, 0.03234125903133548, 0.028863887136907268, 0.029961905258356184, 0.02973540554872534, 0.031782143972864045, 0.03081591060720112, 0.02968785720432574, 0.03285089335484132, 0.030286908785533202, 0.03384048980084124, 0.03166253127706129, 0.032694609122826256, 0.0324303924395686, 0.03071533457075336 ], "mean_node_assignment_entropy": 3.463494551148109, "normalized_node_entropy": 0.9993532826174646, "effective_mass_slots": 31.967004893160855, "node_slot_mutual_information_nats": 0.0012097226166010167, "normalized_node_slot_mutual_information": 0.00034905216396430153, "centered_token_effective_rank": 1.599664095217325, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.006150977050676235, "token_RMS": 0.21277767818429177, "mean_token_offdiagonal_cosine": 0.9991932328743143, "temperature": 0.4988841414451599 }, { "layer": 8, "head": 1, "slices": 32, "native_nodes": 33555, "mass": [ 0.031140836051272096, 0.034224453145581764, 0.03129702004872017, 0.03106522807528642, 0.03162150040292212, 0.031521594374894546, 0.029001010634308155, 0.03034070564450308, 0.03650508679109836, 0.0336991849693318, 0.032038118995169895, 0.03172967525993129, 0.03239319499528207, 0.03186407444831814, 0.028630529693796213, 0.030584966501457256, 0.031143210889143482, 0.031437464723578064, 0.029724556854017765, 0.032294391434583625, 0.02757272301540123, 0.026216080338051485, 0.02622605076823242, 0.02895176162017056, 0.03400317808509654, 0.030204533230868046, 0.027973242885748296, 0.027553212695015806, 0.03558770150219323, 0.035439701435576516, 0.03430004717623658, 0.033714963569559935 ], "mean_node_assignment_entropy": 3.4610295115139014, "normalized_node_entropy": 0.9986420225262914, "effective_mass_slots": 31.889068779830062, "node_slot_mutual_information_nats": 0.0012337680729288536, "normalized_node_slot_mutual_information": 0.000355990216084318, "centered_token_effective_rank": 1.5117951925460502, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.006260236658716288, "token_RMS": 0.2548578498885641, "mean_token_offdiagonal_cosine": 0.9993893655300122, "temperature": 0.4958502948284149 }, { "layer": 8, "head": 2, "slices": 32, "native_nodes": 33555, "mass": [ 0.029616099563106003, 0.03008808915530662, 0.03108284392513056, 0.0316140745175971, 0.02722180241230676, 0.03630346395919076, 0.034330160240458846, 0.03326593345055289, 0.03140855712255188, 0.03312304321220754, 0.032909689855089734, 0.028659545587140674, 0.03289909228208461, 0.027267129121190337, 0.03350068152221726, 0.032159717492354, 0.028520962406816345, 0.03303093683824438, 0.030225387809462562, 0.027429596896348135, 0.03561122174825918, 0.028532892900847, 0.03170867839400982, 0.03200883735923285, 0.029272217530662267, 0.03321325321878998, 0.03493397815746684, 0.032488513814465526, 0.02851523027282512, 0.0321179299942695, 0.02760054652886239, 0.029339892833796597 ], "mean_node_assignment_entropy": 3.4611033571848737, "normalized_node_entropy": 0.998663329882952, "effective_mass_slots": 31.898370572129895, "node_slot_mutual_information_nats": 0.001451572057759254, "normalized_node_slot_mutual_information": 0.0004188351618444527, "centered_token_effective_rank": 1.4301345704612267, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.007720371290946426, "token_RMS": 0.20252386729493352, "mean_token_offdiagonal_cosine": 0.9985638704548101, "temperature": 0.49074462056159973 }, { "layer": 8, "head": 3, "slices": 32, "native_nodes": 33555, "mass": [ 0.029990461115651647, 0.03353900303441053, 0.02631649519010852, 0.030380250640672003, 0.030966155371717378, 0.03354809928628251, 0.028483431884718358, 0.030676344403811937, 0.028528101235237337, 0.03250458084469828, 0.032656934403628, 0.03280533386777124, 0.029620106004521578, 0.03203214930289683, 0.03112311928330822, 0.03337546178696097, 0.029094943125925, 0.03336274260928433, 0.031458005347184924, 0.03142215474699519, 0.03180973397640309, 0.0312540889112716, 0.031214203369502445, 0.03181500049512497, 0.03290336527049124, 0.032562541669186616, 0.029507555154523718, 0.03497957186674889, 0.02862931742887839, 0.03242166396943765, 0.029164742266573594, 0.031854341836650917 ], "mean_node_assignment_entropy": 3.4632687423031574, "normalized_node_entropy": 0.9992881279573045, "effective_mass_slots": 31.942774708632506, "node_slot_mutual_information_nats": 0.0006772692412702064, "normalized_node_slot_mutual_information": 0.00019541859514543152, "centered_token_effective_rank": 1.6490274196246801, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.005261546483981582, "token_RMS": 0.1972621692488605, "mean_token_offdiagonal_cosine": 0.999278789423702, "temperature": 0.4984058737754822 }, { "layer": 8, "head": 4, "slices": 32, "native_nodes": 33555, "mass": [ 0.03291463428968654, 0.03466703354015629, 0.029982861811485274, 0.032844456272415404, 0.027741162276144996, 0.03143200553485141, 0.02842470921216317, 0.03176285570842368, 0.03141805505475234, 0.029100026198329895, 0.03290879184408206, 0.02944375742612635, 0.029354608051879647, 0.03268221695525087, 0.03292953166319661, 0.033885377753903996, 0.027958222999369453, 0.031658537833417266, 0.027681644261216394, 0.03300143349801947, 0.03068658279793759, 0.03141814669778185, 0.029475793257164734, 0.030072617481474176, 0.033143295224180365, 0.03459747635147829, 0.033574475135280626, 0.033811158652904044, 0.03209681834238969, 0.02698434922117543, 0.03192850971804559, 0.030418855036123023 ], "mean_node_assignment_entropy": 3.461373463762128, "normalized_node_entropy": 0.9987412661668552, "effective_mass_slots": 31.92582700429131, "node_slot_mutual_information_nats": 0.002041842418961881, "normalized_node_slot_mutual_information": 0.0005891511864226062, "centered_token_effective_rank": 1.521777121232482, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.009829174096701091, "token_RMS": 0.1981165319584999, "mean_token_offdiagonal_cosine": 0.9976145757539754, "temperature": 0.49465397000312805 }, { "layer": 8, "head": 5, "slices": 32, "native_nodes": 33555, "mass": [ 0.030765123098407003, 0.032316881611181646, 0.02878827362394452, 0.030808607145928535, 0.030125177807405406, 0.030505196462399096, 0.029719122701118164, 0.032444408549057745, 0.029534411061993122, 0.030649308908595855, 0.03193430372666067, 0.03296404183593695, 0.029387718510710346, 0.0357135871520471, 0.0316750684835493, 0.03173645705849514, 0.031177860668789185, 0.030609233269387182, 0.031456002804589885, 0.03321655707289771, 0.030454443787928043, 0.032342845154808224, 0.0278725125169238, 0.02996510862998832, 0.0330450130567996, 0.03324375814730347, 0.03025650531603094, 0.031045132470939592, 0.030920037742625885, 0.03038412006689848, 0.03425998854369143, 0.030683193008915408 ], "mean_node_assignment_entropy": 3.463528533767702, "normalized_node_entropy": 0.9993630879288173, "effective_mass_slots": 31.958672214609116, "node_slot_mutual_information_nats": 0.0009150410423486299, "normalized_node_slot_mutual_information": 0.00026402503480124726, "centered_token_effective_rank": 1.2686313520118688, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.007182742554142366, "token_RMS": 0.24737715062781845, "mean_token_offdiagonal_cosine": 0.9991499267269044, "temperature": 0.5008052587509155 }, { "layer": 8, "head": 6, "slices": 32, "native_nodes": 33555, "mass": [ 0.031291590071076646, 0.02778184448487223, 0.03202201012783435, 0.03207068229289821, 0.03203985021595201, 0.030540874452087176, 0.03370667453047602, 0.03087153200156162, 0.03369833823677023, 0.02935134787664801, 0.02803426565400642, 0.0315469746115559, 0.029425443020958585, 0.03025330628196674, 0.030784934005755424, 0.028495025631657053, 0.029485120717727503, 0.03046136934001006, 0.03191880549962955, 0.031808260219809176, 0.032280886518299766, 0.03325128461618818, 0.032708899515098526, 0.03466134547144774, 0.03165534302451282, 0.03053564992595218, 0.03023667651957317, 0.03127446175382072, 0.0326971541484006, 0.03237998061061945, 0.03330865236563488, 0.02942141647868479 ], "mean_node_assignment_entropy": 3.46314281875826, "normalized_node_entropy": 0.9992517941025535, "effective_mass_slots": 31.954392367653757, "node_slot_mutual_information_nats": 0.001166828912176232, "normalized_node_slot_mutual_information": 0.00033667565703250277, "centered_token_effective_rank": 1.5629988943455935, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.006520212984195002, "token_RMS": 0.2280246178011631, "mean_token_offdiagonal_cosine": 0.9991825567105708, "temperature": 0.4975663423538208 }, { "layer": 8, "head": 7, "slices": 32, "native_nodes": 33555, "mass": [ 0.029789087166053548, 0.029967766862199978, 0.02807458410220785, 0.030343482799460576, 0.027161117123542324, 0.03411943524362051, 0.03049647832710383, 0.03431446326849642, 0.03004629480498609, 0.03175501550531542, 0.03001898891964047, 0.02968000919592663, 0.028956169853434622, 0.03160687128700579, 0.032961773764685676, 0.03239293954045251, 0.030513406875875664, 0.03493450157640766, 0.034301555311026154, 0.029635166246657702, 0.03373491178613504, 0.03244200668492965, 0.03424881299236833, 0.032353714153874454, 0.03183567931517823, 0.032031461723107744, 0.032334982632709196, 0.035118381360929055, 0.02866945744228024, 0.03199227969783187, 0.024963196550653155, 0.029206007868195855 ], "mean_node_assignment_entropy": 3.4614298692107326, "normalized_node_entropy": 0.9987575413390514, "effective_mass_slots": 31.908480457167006, "node_slot_mutual_information_nats": 0.0014419503017890634, "normalized_node_slot_mutual_information": 0.0004160589099198852, "centered_token_effective_rank": 1.1404638079928566, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.009232973889324573, "token_RMS": 0.23442131627157975, "mean_token_offdiagonal_cosine": 0.9984200647268424, "temperature": 0.4971316456794739 }, { "layer": 8, "head": 8, "slices": 32, "native_nodes": 33555, "mass": [ 0.03007662276877694, 0.030559487714464238, 0.029937321241129584, 0.030231992757543777, 0.032342557248875504, 0.030176110316310515, 0.02996581512910077, 0.032758232510912755, 0.029891993967374096, 0.03095487549988715, 0.03011215076826356, 0.032212594840878205, 0.030173789217208823, 0.030886259580936132, 0.032546517147230614, 0.03101536264106567, 0.03295452131940691, 0.0293648075910221, 0.032841044435236784, 0.033807336474027616, 0.032419880957461615, 0.0305689349069452, 0.031428646981702835, 0.029640402662913484, 0.030679336274321664, 0.03221650069212837, 0.03347448008941247, 0.03145666795347961, 0.03131631505964961, 0.030638782693079538, 0.03213491211605754, 0.031215746302144874 ], "mean_node_assignment_entropy": 3.4640998893919215, "normalized_node_entropy": 0.9995279463139464, "effective_mass_slots": 31.976841771181952, "node_slot_mutual_information_nats": 0.0009120567638576382, "normalized_node_slot_mutual_information": 0.0002631639540453302, "centered_token_effective_rank": 1.1317891319590552, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.007945533607816479, "token_RMS": 0.2731338005587273, "mean_token_offdiagonal_cosine": 0.9992147687182787, "temperature": 0.5008169412612915 }, { "layer": 9, "head": 1, "slices": 32, "native_nodes": 33555, "mass": [ 0.030638877902806668, 0.028675489311430265, 0.03542153114185676, 0.02958374207631484, 0.03693129997198085, 0.026585504641852727, 0.029256817784186467, 0.02600500050906079, 0.037653075414664644, 0.03465404968163088, 0.0277682884533978, 0.02857548519591517, 0.029810721065442174, 0.02800982820605347, 0.03579240262856997, 0.02965639702816991, 0.028551995752965208, 0.02961956887000667, 0.030500208962203013, 0.03277585136263812, 0.029754552729522182, 0.031190421494372626, 0.033249927563652464, 0.026338513619057594, 0.035131425773157324, 0.03184981489646472, 0.033876542246041065, 0.0324399004530896, 0.03369179344083101, 0.029945827958401224, 0.041559949029204284, 0.02450519463810928 ], "mean_node_assignment_entropy": 3.4569300037656108, "normalized_node_entropy": 0.9974591546265824, "effective_mass_slots": 31.77608688504166, "node_slot_mutual_information_nats": 0.0017840183910484697, "normalized_node_slot_mutual_information": 0.0005147588971240669, "centered_token_effective_rank": 1.3194442896522898, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.008699371322352678, "token_RMS": 0.30279357400778867, "mean_token_offdiagonal_cosine": 0.999162551404059, "temperature": 0.49225014448165894 }, { "layer": 9, "head": 2, "slices": 32, "native_nodes": 33555, "mass": [ 0.028879663261435477, 0.035504305999292524, 0.03320434083944238, 0.030993216106205437, 0.028990337375905062, 0.031017162460034997, 0.03655720005499377, 0.025632549323793264, 0.032600365862729935, 0.03301690583564302, 0.028839095697948198, 0.0376792904883823, 0.029236857007593337, 0.03141185101307623, 0.026952114380810605, 0.029243382919180626, 0.03355393625885768, 0.03137579959598104, 0.029732588628583703, 0.02835070622762666, 0.0327904520723694, 0.029664366427576413, 0.03068468747148436, 0.036825427546877385, 0.026732188446780037, 0.028494777198314152, 0.03391643189023689, 0.028461048074067748, 0.03030282464497332, 0.03741006562293393, 0.029292585168240893, 0.03265347576456878 ], "mean_node_assignment_entropy": 3.4595319501002315, "normalized_node_entropy": 0.9982099176413057, "effective_mass_slots": 31.843064981707002, "node_slot_mutual_information_nats": 0.0012876682313480714, "normalized_node_slot_mutual_information": 0.000371542514335225, "centered_token_effective_rank": 1.7274065921772603, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.005501210205948663, "token_RMS": 0.23681235205193157, "mean_token_offdiagonal_cosine": 0.9994939392682491, "temperature": 0.49609413743019104 }, { "layer": 9, "head": 3, "slices": 32, "native_nodes": 33555, "mass": [ 0.031935686511787624, 0.02684415432939123, 0.03227801919655133, 0.031021209138249465, 0.03333366646837582, 0.032750869587101536, 0.031051943028844464, 0.0322114132019988, 0.03267245392265547, 0.03177017897013322, 0.028776169609599844, 0.03209315935021283, 0.026638421708321876, 0.03415188750987915, 0.031593376168940396, 0.03261588292426483, 0.03185048696098314, 0.029385456687744, 0.03274191158929555, 0.030065133300102107, 0.03520643660292693, 0.029994303409263852, 0.033876562623007075, 0.030469879801834042, 0.029232945033844385, 0.031885883417699365, 0.029238712989851874, 0.02987866160452739, 0.02887861919689953, 0.031357510763535094, 0.03415016637469187, 0.030048838016431212 ], "mean_node_assignment_entropy": 3.4626680430603924, "normalized_node_entropy": 0.999114802793584, "effective_mass_slots": 31.93420053812511, "node_slot_mutual_information_nats": 0.0010095096058315711, "normalized_node_slot_mutual_information": 0.00029128290041259596, "centered_token_effective_rank": 1.4572067043057921, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.005352190048709334, "token_RMS": 0.22364845738643688, "mean_token_offdiagonal_cosine": 0.9994860486958087, "temperature": 0.497781366109848 }, { "layer": 9, "head": 4, "slices": 32, "native_nodes": 33555, "mass": [ 0.04093429610824828, 0.02862756100085369, 0.030218188843887335, 0.030176793670773312, 0.03270904192599121, 0.035761631926100615, 0.03154540571845283, 0.036363787287181475, 0.02488113972396569, 0.026405655612774675, 0.03271909311664671, 0.02601227529564543, 0.03547050615803413, 0.03341725389894324, 0.03377055737062691, 0.03412010581924403, 0.0271624611163571, 0.03330505548672027, 0.03458725854209338, 0.030128677610117532, 0.03109394498347055, 0.032072934244983226, 0.02559791768621695, 0.0275815646172307, 0.027358290903362392, 0.03862972853441891, 0.028939143811412087, 0.03512947539545095, 0.023791553373231366, 0.02690898861190394, 0.030305835161363114, 0.03427387660178045 ], "mean_node_assignment_entropy": 3.4556184639720753, "normalized_node_entropy": 0.99708072423537, "effective_mass_slots": 31.72691124731744, "node_slot_mutual_information_nats": 0.0015467920160503468, "normalized_node_slot_mutual_information": 0.00044630983416849544, "centered_token_effective_rank": 1.4538447414712259, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.0071076058459839395, "token_RMS": 0.2386587258583269, "mean_token_offdiagonal_cosine": 0.9991042255846404, "temperature": 0.4931093156337738 }, { "layer": 9, "head": 5, "slices": 32, "native_nodes": 33555, "mass": [ 0.03330427082406748, 0.02983467623884517, 0.03434993340486014, 0.028200395633828052, 0.03332656891013183, 0.03473797465025526, 0.0351918362744953, 0.03886271499828972, 0.02316231133973895, 0.0280630310690673, 0.03338321140783987, 0.024704832039110673, 0.033889802295346584, 0.030119994119022858, 0.03594275595113189, 0.03375705658952075, 0.026026582917343697, 0.034489249507712726, 0.0334210845130404, 0.03113991978673053, 0.030287031104782355, 0.03611707087634502, 0.02964282597868529, 0.02711646368430386, 0.02795754942730471, 0.03804652924879646, 0.027514577873030074, 0.03365588993485781, 0.022359905003970663, 0.024770306604935878, 0.032326651469368174, 0.034296995842466586 ], "mean_node_assignment_entropy": 3.4552186421938584, "normalized_node_entropy": 0.9969653600560354, "effective_mass_slots": 31.702635647261065, "node_slot_mutual_information_nats": 0.0011811786798729962, "normalized_node_slot_mutual_information": 0.0003408161247713088, "centered_token_effective_rank": 1.609090964081888, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.006556171250467886, "token_RMS": 0.23683484114431663, "mean_token_offdiagonal_cosine": 0.9992343458820679, "temperature": 0.4932563602924347 }, { "layer": 9, "head": 6, "slices": 32, "native_nodes": 33555, "mass": [ 0.029542360887388822, 0.036016845459990274, 0.035843806370766396, 0.03119573416689339, 0.03124188367061449, 0.03036818473614145, 0.03272649010002498, 0.02897131644103872, 0.030334047018102786, 0.030213063831574745, 0.02938143503991658, 0.031065961891806635, 0.030341553236695785, 0.02934128210318238, 0.02852445219659651, 0.027885827489753465, 0.03093440183380769, 0.03266289732909526, 0.032434930545744874, 0.030600979102928385, 0.032145501869727734, 0.03229806225348862, 0.032271459336472935, 0.029972676340351247, 0.03417585863358634, 0.03216438219906001, 0.031158080842718427, 0.03230519546000191, 0.032634022222914276, 0.03143422895796279, 0.028433509090297217, 0.0313795695066643 ], "mean_node_assignment_entropy": 3.4630037158811198, "normalized_node_entropy": 0.9992116574963489, "effective_mass_slots": 31.943059567394826, "node_slot_mutual_information_nats": 0.0009512134074540768, "normalized_node_slot_mutual_information": 0.0002744621731522179, "centered_token_effective_rank": 1.7867515301794017, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.005357777317043847, "token_RMS": 0.23100966166142736, "mean_token_offdiagonal_cosine": 0.9994925320679435, "temperature": 0.4992658793926239 }, { "layer": 9, "head": 7, "slices": 32, "native_nodes": 33555, "mass": [ 0.030684438433135664, 0.03367044876824058, 0.03382018236869441, 0.031331199438718746, 0.03136254628200704, 0.03101808772328145, 0.031837286288341954, 0.030665655089302187, 0.029578348531521942, 0.030086709524769484, 0.02941127731694477, 0.031551055390137424, 0.03506353858810053, 0.02957974461112252, 0.028857449292714717, 0.03414801010655773, 0.028777474267157068, 0.03445105106032627, 0.03177728250976904, 0.0306943628697757, 0.03179812340298702, 0.0292459050440727, 0.03096991862955729, 0.032170334743729136, 0.026181047227477338, 0.03140970447732544, 0.03405281421540391, 0.03223245864274152, 0.028326747371565212, 0.033900294145545615, 0.02802295419214744, 0.033323549183654266 ], "mean_node_assignment_entropy": 3.462468658279894, "normalized_node_entropy": 0.9990572725067732, "effective_mass_slots": 31.928187758313527, "node_slot_mutual_information_nats": 0.0010205901280726515, "normalized_node_slot_mutual_information": 0.0002944800633101293, "centered_token_effective_rank": 1.555787565384361, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.0070649257098221, "token_RMS": 0.2244074234851838, "mean_token_offdiagonal_cosine": 0.9990107885893187, "temperature": 0.49889230728149414 }, { "layer": 9, "head": 8, "slices": 32, "native_nodes": 33555, "mass": [ 0.03062116322952062, 0.03345056864212476, 0.036448309763670944, 0.027938944304778543, 0.0329389946746446, 0.029078577740461794, 0.03818959646818446, 0.029011666055004354, 0.027923006100979492, 0.030732095248152347, 0.032060848500325916, 0.025622796097631836, 0.03395466153525042, 0.027511278551442135, 0.036084064462135704, 0.028441213278732433, 0.029510916432724094, 0.02858512470957473, 0.03140988342542795, 0.029883950965701334, 0.028985392785324565, 0.03309987926648353, 0.03442070332862744, 0.02793447434471432, 0.03377222029136716, 0.035061492102178744, 0.03203724697010205, 0.03612574666462069, 0.027103409671306946, 0.027823509726434114, 0.03282872050414407, 0.03140954419097892 ], "mean_node_assignment_entropy": 3.4590597588715406, "normalized_node_entropy": 0.9980736720525091, "effective_mass_slots": 31.838498468518843, "node_slot_mutual_information_nats": 0.0016164423405911066, "normalized_node_slot_mutual_information": 0.00046640666973074763, "centered_token_effective_rank": 1.3783134262266163, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.008316924420341997, "token_RMS": 0.2520071242522932, "mean_token_offdiagonal_cosine": 0.9988921349182546, "temperature": 0.4920760989189148 }, { "layer": 10, "head": 1, "slices": 32, "native_nodes": 33555, "mass": [ 0.03181384601722556, 0.030461980770391316, 0.028933748384489952, 0.031147760885218503, 0.03267736996655456, 0.03378987420645072, 0.03366037061349981, 0.02901910963953829, 0.030524468500816047, 0.029995707197067917, 0.029855982070996073, 0.026554176602149007, 0.03602654525019021, 0.031040413827014883, 0.03243996062082996, 0.028842761926658283, 0.029886973691895226, 0.030677574343680715, 0.03049502003576588, 0.038263426258918015, 0.030987348597407375, 0.036785847751523354, 0.03141209520366264, 0.029873790602446614, 0.028677641003635634, 0.02963339818408123, 0.02830059756125881, 0.03191736882274676, 0.03409130761846445, 0.029462798362321106, 0.03089224962253113, 0.03185848594822057 ], "mean_node_assignment_entropy": 3.4620654472513404, "normalized_node_entropy": 0.998940930396508, "effective_mass_slots": 31.9017802224268, "node_slot_mutual_information_nats": 0.0005963673280100501, "normalized_node_slot_mutual_information": 0.0001720752373336602, "centered_token_effective_rank": 1.7698513887021898, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.003706203069940427, "token_RMS": 0.1918985636728906, "mean_token_offdiagonal_cosine": 0.9996263491027894, "temperature": 0.49717310070991516 }, { "layer": 10, "head": 2, "slices": 32, "native_nodes": 33555, "mass": [ 0.03340658537411793, 0.030070938436904816, 0.0316785335407547, 0.03241982739504786, 0.03288931660254219, 0.03045571099502144, 0.027907449949708715, 0.028193598072029424, 0.03521912152108875, 0.03427719108059818, 0.02956675650467817, 0.030836113773296335, 0.031642695345043374, 0.029344157886851956, 0.03462812782539605, 0.03145691298360246, 0.030471771677717013, 0.032334743991811694, 0.02416933471551508, 0.029395455680426747, 0.0339903678640117, 0.028941168103407612, 0.02901835876672843, 0.027799883802913906, 0.03396785705648846, 0.03353340248793093, 0.03147630975818318, 0.03520949586499841, 0.030159855199273607, 0.03290004268956955, 0.03317642658643592, 0.029462487547435093 ], "mean_node_assignment_entropy": 3.4610469249178535, "normalized_node_entropy": 0.9986470469725968, "effective_mass_slots": 31.896687947977224, "node_slot_mutual_information_nats": 0.0014552534004259599, "normalized_node_slot_mutual_information": 0.0004198973728062666, "centered_token_effective_rank": 1.4379179331961525, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.006964975680738367, "token_RMS": 0.30659400337869447, "mean_token_offdiagonal_cosine": 0.9994899486110134, "temperature": 0.4963543117046356 }, { "layer": 10, "head": 3, "slices": 32, "native_nodes": 33555, "mass": [ 0.032911380500849234, 0.02889857209276343, 0.03275846145001303, 0.03414223799599947, 0.03162477551090026, 0.03389876809822138, 0.033046331211942316, 0.02959992870202169, 0.03149528880842999, 0.03408330767101137, 0.032624248087332886, 0.029643814284212897, 0.03160185152333107, 0.028925835826374383, 0.025626466204562853, 0.030399418363022637, 0.0298885046074095, 0.032204337843121066, 0.030254341490847613, 0.026396230685661614, 0.034100544510175815, 0.03075157813561325, 0.030906016139402116, 0.029701774600172183, 0.03488002786310974, 0.029702479429704, 0.028387786842124365, 0.03273768614188798, 0.029772963886546263, 0.0335687274733069, 0.03531198715384494, 0.03015432638775226 ], "mean_node_assignment_entropy": 3.461404180854085, "normalized_node_entropy": 0.9987501292461026, "effective_mass_slots": 31.911172591488377, "node_slot_mutual_information_nats": 0.0015520055954025835, "normalized_node_slot_mutual_information": 0.00044781415518384607, "centered_token_effective_rank": 1.3717365118597196, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.006852281053181219, "token_RMS": 0.24003871510368693, "mean_token_offdiagonal_cosine": 0.9991958194643644, "temperature": 0.49326762557029724 }, { "layer": 10, "head": 4, "slices": 32, "native_nodes": 33555, "mass": [ 0.031171368905141196, 0.03243050410057958, 0.03318923115452735, 0.0322943036885375, 0.03181520850121344, 0.03124236553838562, 0.031938640029570545, 0.03295106786944226, 0.03126102157876355, 0.028012257157832565, 0.03118099375577837, 0.026687631577102355, 0.030456716153615813, 0.02730787127417382, 0.03350299874115516, 0.028465224193158455, 0.02580558591422497, 0.032089168990272074, 0.031026927071340818, 0.03153890096125207, 0.030792614503875607, 0.03233947037885582, 0.0348391430971764, 0.029077976463453333, 0.03512186325377101, 0.03014674093493796, 0.03005306614564812, 0.03520392241466287, 0.029936776270349127, 0.029136423034821887, 0.034434084220777025, 0.034549932113668635 ], "mean_node_assignment_entropy": 3.4619150263774126, "normalized_node_entropy": 0.9988975281067356, "effective_mass_slots": 31.906882008141704, "node_slot_mutual_information_nats": 0.0009066970778626171, "normalized_node_slot_mutual_information": 0.0002616174755641824, "centered_token_effective_rank": 1.4161779267082526, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.004936513803763182, "token_RMS": 0.17549960096319975, "mean_token_offdiagonal_cosine": 0.9992060300373102, "temperature": 0.4985625147819519 }, { "layer": 10, "head": 5, "slices": 32, "native_nodes": 33555, "mass": [ 0.03014404498142815, 0.03072085371661463, 0.026004474989336022, 0.029977284894537028, 0.03137321904814558, 0.03215465786679635, 0.030050153809281367, 0.03326523349698255, 0.0295228502941152, 0.02766289142590622, 0.030416489648532094, 0.029816580670474272, 0.031251680969496966, 0.034321511154038205, 0.03333865811041083, 0.03226870415134697, 0.03349400475108309, 0.02822497202034342, 0.03932560598037803, 0.037837088866387744, 0.02549038999201182, 0.0347490079848604, 0.03294343347593891, 0.030480118037033963, 0.027483387467121246, 0.03148849661990882, 0.034016798689046626, 0.027156291239717432, 0.036068079384736856, 0.03024095479649317, 0.026381485909334576, 0.03233059573201944 ], "mean_node_assignment_entropy": 3.4596022568616034, "normalized_node_entropy": 0.9982302038845002, "effective_mass_slots": 31.83285229260724, "node_slot_mutual_information_nats": 0.0008965906157980541, "normalized_node_slot_mutual_information": 0.0002587013670238869, "centered_token_effective_rank": 1.5239166297812818, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.004789787774292948, "token_RMS": 0.2861046594956669, "mean_token_offdiagonal_cosine": 0.9997384742464744, "temperature": 0.4946182072162628 }, { "layer": 10, "head": 6, "slices": 32, "native_nodes": 33555, "mass": [ 0.02782387840394344, 0.029982621527763063, 0.033260531636181735, 0.029527296609216238, 0.02872051344169996, 0.032948486427947905, 0.028858148677360904, 0.03613752332969672, 0.02581094983554938, 0.027996688238380474, 0.027430190026011825, 0.029755746796194284, 0.03643602941653187, 0.032694707461886054, 0.03425245013000837, 0.035870222887711224, 0.027748850250047412, 0.028377146698282472, 0.03828533428216403, 0.032966964409941, 0.027512848031010593, 0.03700891095362776, 0.03257236795433586, 0.039098048690251866, 0.026960537531455466, 0.028667576165072983, 0.0355284857244352, 0.02893739011021551, 0.031295224611449395, 0.0275675190926584, 0.02533608112234976, 0.03463072983092584 ], "mean_node_assignment_entropy": 3.4573721032786464, "normalized_node_entropy": 0.9975867175815897, "effective_mass_slots": 31.764289033392476, "node_slot_mutual_information_nats": 0.0009705691145205364, "normalized_node_slot_mutual_information": 0.0002800470496717541, "centered_token_effective_rank": 1.2542953395793282, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.006675078540664884, "token_RMS": 0.2954631905092522, "mean_token_offdiagonal_cosine": 0.9995704196932713, "temperature": 0.49140191078186035 }, { "layer": 10, "head": 7, "slices": 32, "native_nodes": 33555, "mass": [ 0.028826630635362165, 0.03386206792084928, 0.03000857639197949, 0.035003150531261906, 0.030022544614862996, 0.027659079633489965, 0.03071831234383425, 0.03692874801889864, 0.02690641667131535, 0.029375386563610027, 0.030507864876238248, 0.02918534737544956, 0.032446873784698065, 0.030185027579613048, 0.029943715773479557, 0.028505520930316955, 0.029635900497767365, 0.03084062898528503, 0.03487529169350103, 0.03307267873080436, 0.03068203800222572, 0.031082544099712155, 0.031346441638499006, 0.03569856136143563, 0.031754665113535904, 0.029429931086621987, 0.03192737611886899, 0.03158462603322938, 0.031691705478923436, 0.02948499117282513, 0.03189972454551266, 0.03490763195736089 ], "mean_node_assignment_entropy": 3.462322701441198, "normalized_node_entropy": 0.9990151582652991, "effective_mass_slots": 31.91168690144512, "node_slot_mutual_information_nats": 0.0006496018028312456, "normalized_node_slot_mutual_information": 0.00018743545989943365, "centered_token_effective_rank": 1.7655427941061392, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.003481180514338255, "token_RMS": 0.2776837063530833, "mean_token_offdiagonal_cosine": 0.9998420399034558, "temperature": 0.4967355728149414 }, { "layer": 10, "head": 8, "slices": 32, "native_nodes": 33555, "mass": [ 0.032247732350117866, 0.032004640920795725, 0.02997769020062599, 0.0327133212661538, 0.028864587471610723, 0.03072920486390937, 0.031594399963756056, 0.03412840952995441, 0.029229360766405765, 0.03248561441997154, 0.029614971533824037, 0.027739239695510128, 0.033737818243137735, 0.031182941581513784, 0.030850075703875317, 0.02863386334162225, 0.03113368337338638, 0.030344331840247762, 0.03201473216090316, 0.031225874911005105, 0.0296317824028902, 0.03407063847517331, 0.033645708881004384, 0.030984293869132436, 0.03336980596413432, 0.02797565965531187, 0.03224821342886684, 0.0314166205514274, 0.02959835868145071, 0.03166936519793748, 0.03168541171392746, 0.03325164687210971 ], "mean_node_assignment_entropy": 3.4634765297261922, "normalized_node_entropy": 0.9993480827342588, "effective_mass_slots": 31.951343647882286, "node_slot_mutual_information_nats": 0.0007377049198780128, "normalized_node_slot_mutual_information": 0.00021285664590947983, "centered_token_effective_rank": 1.7756621653667481, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.004428538015101027, "token_RMS": 0.2835508983327935, "mean_token_offdiagonal_cosine": 0.9997606367869744, "temperature": 0.5007973909378052 }, { "layer": 11, "head": 1, "slices": 32, "native_nodes": 33555, "mass": [ 0.037710669969288055, 0.027771855838016182, 0.029892127884649303, 0.029276305744614584, 0.03061405416439834, 0.030967163886625675, 0.03130194997749951, 0.0285097951967907, 0.02865224543544241, 0.031065872855869727, 0.030877131726878657, 0.03267679651825843, 0.03385445589085589, 0.03052616041054306, 0.034445425884465385, 0.030969636559468626, 0.030587253362915144, 0.03223312574264865, 0.028786660529281197, 0.032134129263870305, 0.03215549087806628, 0.02996472944667186, 0.036092261700263556, 0.03201775699198876, 0.029193426505277398, 0.03137960443774166, 0.028825119189055413, 0.028405842948230346, 0.03751226864758484, 0.034011231930078524, 0.027649584394089866, 0.029939866079967577 ], "mean_node_assignment_entropy": 3.4613755315633328, "normalized_node_entropy": 0.998741862808164, "effective_mass_slots": 31.895725234878288, "node_slot_mutual_information_nats": 0.0010964640717410212, "normalized_node_slot_mutual_information": 0.00031637265576273837, "centered_token_effective_rank": 1.7411718307974258, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.004650906385583423, "token_RMS": 0.24710473210559555, "mean_token_offdiagonal_cosine": 0.9996502458585381, "temperature": 0.49406659603118896 }, { "layer": 11, "head": 2, "slices": 32, "native_nodes": 33555, "mass": [ 0.034113894807761266, 0.022390227689239987, 0.019893854194531817, 0.030978547659694412, 0.017843626602715892, 0.02269133128794228, 0.03220079994275156, 0.022806249392617033, 0.031127775487459607, 0.01662791419514506, 0.040603530944290755, 0.029021929286896623, 0.03486380613701938, 0.028072850415218054, 0.05058232270738502, 0.03601011833471054, 0.021929468381764138, 0.025503919237533246, 0.0300072032566367, 0.04808402575403696, 0.03380329019765157, 0.0260480808401948, 0.026669817488524532, 0.04558030208465888, 0.028541668838814552, 0.0462383181027121, 0.02962404455239656, 0.034664174512077546, 0.056647899298594366, 0.025323635402852047, 0.039987577768071825, 0.01151779518308538 ], "mean_node_assignment_entropy": 3.412039826419415, "normalized_node_entropy": 0.984506587378186, "effective_mass_slots": 30.358736140002325, "node_slot_mutual_information_nats": 0.0010444961243152306, "normalized_node_slot_mutual_information": 0.0003013778757554651, "centered_token_effective_rank": 2.000172025143984, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.0038724316004291083, "token_RMS": 0.26453336944631767, "mean_token_offdiagonal_cosine": 0.9998250405727117, "temperature": 0.4689115583896637 }, { "layer": 11, "head": 3, "slices": 32, "native_nodes": 33555, "mass": [ 0.024739385775709947, 0.032955879928448964, 0.029767191787589645, 0.034380895747453295, 0.029885900421938196, 0.02970290491253364, 0.030673010234411952, 0.03613297495052314, 0.030057971845108267, 0.026049482911623123, 0.036575797416026644, 0.030381215036952305, 0.03085975631674042, 0.03503593532578092, 0.03055814915498897, 0.030925817195963903, 0.02666598842244064, 0.027145825265270464, 0.03322477981824533, 0.03546492562853683, 0.028497925147496363, 0.030946795257357117, 0.025157918426165316, 0.0321579819152415, 0.031018756699655772, 0.03481310356634284, 0.02959418418842063, 0.03735845809187082, 0.036008040037440785, 0.031034803195329027, 0.03663948144763153, 0.02558876346475347 ], "mean_node_assignment_entropy": 3.4584022947744963, "normalized_node_entropy": 0.9978839680140353, "effective_mass_slots": 31.79458623420051, "node_slot_mutual_information_nats": 0.0008937363109136065, "normalized_node_slot_mutual_information": 0.00025787778872349136, "centered_token_effective_rank": 1.828105970645824, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.003967785464975111, "token_RMS": 0.24660009219389603, "mean_token_offdiagonal_cosine": 0.9997532076585147, "temperature": 0.49458399415016174 }, { "layer": 11, "head": 4, "slices": 32, "native_nodes": 33555, "mass": [ 0.03576403080083729, 0.03330081505732864, 0.03483770881028578, 0.02723027112678797, 0.03178117359574702, 0.034064596254645564, 0.03355972124362175, 0.026764627630836098, 0.027508159893832856, 0.037775018462788294, 0.02757519823283228, 0.03362580684603877, 0.035560444802669754, 0.03101209574049435, 0.0367893716157473, 0.028741667323694858, 0.028959278412792742, 0.030972044685925727, 0.02981638449616536, 0.025844401936751065, 0.030736437518465467, 0.03417131205344321, 0.038868643340007, 0.03096156267985211, 0.027480645200942334, 0.03286378678917494, 0.026779833330265315, 0.025002742543896502, 0.03315435764108493, 0.03358106356801921, 0.023192123590183498, 0.0317246745887718 ], "mean_node_assignment_entropy": 3.456783449623759, "normalized_node_entropy": 0.9974168680398482, "effective_mass_slots": 31.756994552905375, "node_slot_mutual_information_nats": 0.0013295523311760427, "normalized_node_slot_mutual_information": 0.0003836277109580075, "centered_token_effective_rank": 1.279105724223499, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.005354975140658722, "token_RMS": 0.32415666249271297, "mean_token_offdiagonal_cosine": 0.9997218277236045, "temperature": 0.48449820280075073 }, { "layer": 11, "head": 5, "slices": 32, "native_nodes": 33555, "mass": [ 0.029442681898902076, 0.029648155981216555, 0.028230616365656682, 0.03801446595535764, 0.034648143586019454, 0.029242976766449918, 0.032950553752902706, 0.033534278830860575, 0.025454055831156036, 0.03251271046139661, 0.03380392860866014, 0.031375902616328, 0.03353027792298044, 0.03337507755204873, 0.02821212169296419, 0.025227564710509208, 0.031566043846079375, 0.032068824888023977, 0.029531807943372575, 0.031711167821388914, 0.02605438882329881, 0.03479292807242116, 0.02783157615646922, 0.028719738433161127, 0.02680686383106113, 0.03638839049062454, 0.0311716244651071, 0.03395541226129128, 0.03369511841790649, 0.036753441781336264, 0.026743941852300272, 0.033005218049354496 ], "mean_node_assignment_entropy": 3.4591094760604877, "normalized_node_entropy": 0.9980880174008973, "effective_mass_slots": 31.817076300896797, "node_slot_mutual_information_nats": 0.0008936601920277454, "normalized_node_slot_mutual_information": 0.00025785582545566143, "centered_token_effective_rank": 1.5102918911659453, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.0049493577250807365, "token_RMS": 0.2714542487073673, "mean_token_offdiagonal_cosine": 0.9996661493081489, "temperature": 0.49564021825790405 }, { "layer": 11, "head": 6, "slices": 32, "native_nodes": 33555, "mass": [ 0.036726104580626424, 0.030557048021181654, 0.03224026439171615, 0.029437972374472474, 0.02762175232023245, 0.0323741444649799, 0.02872462781705989, 0.03137532596015225, 0.03380067944698754, 0.0301746994945855, 0.03086512800954996, 0.032398064451070394, 0.031089742868725723, 0.029864633303552515, 0.03610765749585556, 0.03389221102303479, 0.028307934586780825, 0.02860488322066792, 0.031221093365739663, 0.031496313818641736, 0.03527614791770572, 0.030164227103380115, 0.03422175598566853, 0.03262731640378599, 0.0306382489901504, 0.03289745552375115, 0.027660107393177997, 0.028890682622732606, 0.03287960305052815, 0.031157642012458412, 0.029330838187327773, 0.027375693848397404 ], "mean_node_assignment_entropy": 3.4619350179217907, "normalized_node_entropy": 0.9989032964471224, "effective_mass_slots": 31.906698482631914, "node_slot_mutual_information_nats": 0.0008809536070644164, "normalized_node_slot_mutual_information": 0.0002541894800330156, "centered_token_effective_rank": 1.415485624080571, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.003995670311358614, "token_RMS": 0.2860515892316643, "mean_token_offdiagonal_cosine": 0.9998093126229923, "temperature": 0.49326902627944946 }, { "layer": 11, "head": 7, "slices": 32, "native_nodes": 33555, "mass": [ 0.04814456547903972, 0.0233685051976542, 0.029181519713440962, 0.04368351000868924, 0.05098873389308356, 0.02244108056419476, 0.019406579177526334, 0.04667177117995739, 0.032105056077159744, 0.05150941086430927, 0.017156657010634367, 0.034853146834771635, 0.02508798805293739, 0.024325581242274446, 0.01812859973539238, 0.022752692512673818, 0.04068434795995056, 0.042506694554896245, 0.02104389068405926, 0.0217902737495761, 0.027494957823611072, 0.03487377553199038, 0.03188267947442307, 0.026850641441061446, 0.019214567251646684, 0.03603859244777946, 0.025130596684882864, 0.025195732263112953, 0.017084789894201497, 0.03480370940071312, 0.019531764716438416, 0.066067588496845 ], "mean_node_assignment_entropy": 3.396512960032378, "normalized_node_entropy": 0.9800264807507612, "effective_mass_slots": 29.8904902666948, "node_slot_mutual_information_nats": 0.0010274184941168052, "normalized_node_slot_mutual_information": 0.0002964503132759843, "centered_token_effective_rank": 1.720452359204702, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.0032077102681424847, "token_RMS": 0.3161938955201239, "mean_token_offdiagonal_cosine": 0.9999018852218677, "temperature": 0.47661665081977844 }, { "layer": 11, "head": 8, "slices": 32, "native_nodes": 33555, "mass": [ 0.0244476821674349, 0.03242253439747671, 0.03455164534409077, 0.029037634937270018, 0.02988184716959386, 0.029423920661621788, 0.035639155782551585, 0.03420513537096926, 0.028051209965679974, 0.03099592463599776, 0.03207034447362652, 0.029136280742499018, 0.02845092692472526, 0.03908892960913925, 0.03060012701196609, 0.032285443850104754, 0.030594489729161536, 0.025357361612867856, 0.03349754289525345, 0.03037898900441691, 0.028219954606526976, 0.0355937850169339, 0.026057145955611972, 0.03236712197019746, 0.036537316098982434, 0.02916627213755222, 0.03569312409524581, 0.03793222695298192, 0.02916901037807508, 0.028017693744854945, 0.030896205919407692, 0.030233017042292566 ], "mean_node_assignment_entropy": 3.4572574684715165, "normalized_node_entropy": 0.9975536409680377, "effective_mass_slots": 31.800866316398633, "node_slot_mutual_information_nats": 0.002236063597585325, "normalized_node_slot_mutual_information": 0.0006451915726697365, "centered_token_effective_rank": 1.2748012811296878, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.0072268727381054765, "token_RMS": 0.25229975678268207, "mean_token_offdiagonal_cosine": 0.9992548960767007, "temperature": 0.4879787862300873 }, { "layer": 12, "head": 1, "slices": 32, "native_nodes": 33555, "mass": [ 0.02834254362931409, 0.0278074364308748, 0.03131822509963952, 0.03447106348373126, 0.02699417252298386, 0.035137964164328336, 0.03335966429866172, 0.029502844257659644, 0.03168203585158471, 0.033672429514140435, 0.030332339319606293, 0.030381483126738676, 0.03127891906632732, 0.03165751046790725, 0.02883547522863694, 0.030075734372415463, 0.03179395735017359, 0.03445265025145792, 0.029562835723845583, 0.03203569498496534, 0.02841725024386226, 0.03229957906829895, 0.030871511194613507, 0.02993840172319534, 0.026375340305700636, 0.032658728725365564, 0.039393947682880626, 0.034405004323548004, 0.03004015087343074, 0.032189465436605384, 0.03066169641425714, 0.030053944608290705 ], "mean_node_assignment_entropy": 3.4618845013889414, "normalized_node_entropy": 0.9988887204568375, "effective_mass_slots": 31.89052568606409, "node_slot_mutual_information_nats": 0.00042446385163064093, "normalized_node_slot_mutual_information": 0.00012247437875683088, "centered_token_effective_rank": 1.7470940222659739, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.00293042458570934, "token_RMS": 0.2565200366548523, "mean_token_offdiagonal_cosine": 0.9998681277394054, "temperature": 0.4951874613761902 }, { "layer": 12, "head": 2, "slices": 32, "native_nodes": 33555, "mass": [ 0.0276699623454658, 0.023154415874438748, 0.02570848664497276, 0.041650681841006634, 0.027768585007988008, 0.03086223809364664, 0.029184794058875325, 0.03490256019410691, 0.0296502457457303, 0.03498595891793214, 0.02703837375266908, 0.031605939316460475, 0.03349529733117256, 0.031006626874783536, 0.03395439620415503, 0.03999212175799747, 0.03260467479005733, 0.03257614052562356, 0.027884703429562646, 0.03284791820112079, 0.025698959805996156, 0.03214671019419768, 0.029973149352078732, 0.033035566207649296, 0.022763794436084943, 0.0342769052459735, 0.030911860745519538, 0.03064816877908695, 0.032528165491047215, 0.03301957785292756, 0.038160014503684454, 0.028293006518455184 ], "mean_node_assignment_entropy": 3.455954664021485, "normalized_node_entropy": 0.9971777310641761, "effective_mass_slots": 31.711326524287706, "node_slot_mutual_information_nats": 0.0007192566493081642, "normalized_node_slot_mutual_information": 0.00020753360021666015, "centered_token_effective_rank": 1.3682841906591654, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.003838130617805164, "token_RMS": 0.2321233696959089, "mean_token_offdiagonal_cosine": 0.9997198446081593, "temperature": 0.4895017147064209 }, { "layer": 12, "head": 3, "slices": 32, "native_nodes": 33555, "mass": [ 0.024028196393546468, 0.02634539523214266, 0.028636777709163154, 0.04438086947082869, 0.025457411522142596, 0.039098454265676374, 0.025646819914851435, 0.02889315702449661, 0.03364198275894091, 0.03330954453630217, 0.027777525610225937, 0.024527603761194325, 0.029509731061282553, 0.03369762463967805, 0.030762456579630962, 0.03568072755052371, 0.03524529780699055, 0.039737004168845645, 0.029127540350661976, 0.033730089085043644, 0.024017154848592466, 0.03111944982529157, 0.03315109567220734, 0.03375847849963257, 0.021843040112482063, 0.03971062606868641, 0.030157654661246144, 0.029588236295562745, 0.033863248028923584, 0.033753018789831564, 0.03464930408810907, 0.02515448329007416 ], "mean_node_assignment_entropy": 3.4511487516111155, "normalized_node_entropy": 0.9957910378638987, "effective_mass_slots": 31.560688160896465, "node_slot_mutual_information_nats": 0.0007635489734640899, "normalized_node_slot_mutual_information": 0.00022031366349850023, "centered_token_effective_rank": 1.779923792211385, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.0028646940061651505, "token_RMS": 0.2422376524489739, "mean_token_offdiagonal_cosine": 0.9998630852209084, "temperature": 0.48939719796180725 }, { "layer": 12, "head": 4, "slices": 32, "native_nodes": 33555, "mass": [ 0.025822540010628003, 0.026793612476264536, 0.030243476677861942, 0.05139577114298738, 0.024092678154796925, 0.03873674118156681, 0.026097737506338408, 0.025896568434106718, 0.033232996930028104, 0.03555090208977101, 0.030046037507050972, 0.024145424108540953, 0.02758498394423564, 0.031145317032666785, 0.025950030329416715, 0.032526423081571346, 0.03621997682619443, 0.03975321640768575, 0.02803878070346122, 0.03893956609637615, 0.027248769394821355, 0.025518091010422985, 0.038367237353558924, 0.027728842631828374, 0.020275354723322144, 0.03739561778394001, 0.03331967789539325, 0.032925676742121476, 0.032044157040275044, 0.0353215566380975, 0.031799750120799934, 0.025842488339444757 ], "mean_node_assignment_entropy": 3.4462555589958077, "normalized_node_entropy": 0.9943791609198549, "effective_mass_slots": 31.402838121151202, "node_slot_mutual_information_nats": 0.0006427158577531067, "normalized_node_slot_mutual_information": 0.0001854485961362207, "centered_token_effective_rank": 1.8047599941638623, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.002784617061147449, "token_RMS": 0.21266436704095626, "mean_token_offdiagonal_cosine": 0.9998390040865835, "temperature": 0.4851617217063904 }, { "layer": 12, "head": 5, "slices": 32, "native_nodes": 33555, "mass": [ 0.03230972303894411, 0.028464326796371276, 0.03407802993695083, 0.03422473150003891, 0.02604385455758059, 0.028757979783168883, 0.03191389626927586, 0.031377911883430074, 0.03133949556451381, 0.03209253435752117, 0.03343087418060103, 0.033066980649385895, 0.028124175696450433, 0.03330553243317808, 0.028303107726198252, 0.028077950000356604, 0.03279773979238925, 0.03195459243993151, 0.030342485782326893, 0.03389745930726278, 0.033631684611902704, 0.024331571657120108, 0.034595060195302556, 0.027718798281640734, 0.02866828235371682, 0.0307898374613818, 0.034026604429089426, 0.03796266687304257, 0.031713790121396544, 0.032696741772420074, 0.03192127934046286, 0.02804030147237494 ], "mean_node_assignment_entropy": 3.461021080732062, "normalized_node_entropy": 0.9986395899168613, "effective_mass_slots": 31.86458444724714, "node_slot_mutual_information_nats": 0.00047410690874416517, "normalized_node_slot_mutual_information": 0.00013679833721928067, "centered_token_effective_rank": 1.5784000467561543, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.003276450166917247, "token_RMS": 0.27149910958915846, "mean_token_offdiagonal_cosine": 0.999855254954067, "temperature": 0.49467504024505615 }, { "layer": 12, "head": 6, "slices": 32, "native_nodes": 33555, "mass": [ 0.03053079798641112, 0.02956988751131116, 0.030375768695842083, 0.031496280510238873, 0.03219256368027885, 0.03078934198411659, 0.03345685225396998, 0.03097123642665914, 0.030505742027915957, 0.03076931158733775, 0.031007107495739802, 0.03232194574833174, 0.03364855758229643, 0.032212097874995654, 0.03058775192288444, 0.03150955823904719, 0.03368129283617202, 0.030371638849564148, 0.032105764730845546, 0.02964555654514344, 0.028903870326134493, 0.03138919759604476, 0.02970143766623285, 0.03376741479200216, 0.029866415948883607, 0.03158347751851445, 0.03001181555704013, 0.03308515612098716, 0.033107729602314784, 0.030785134440195525, 0.031800777591390786, 0.02824851840822188 ], "mean_node_assignment_entropy": 3.4640226683771265, "normalized_node_entropy": 0.9995056650389269, "effective_mass_slots": 31.967517502983757, "node_slot_mutual_information_nats": 0.0006976408502317177, "normalized_node_slot_mutual_information": 0.00020129659899017184, "centered_token_effective_rank": 1.3867988783544454, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.003306105093937401, "token_RMS": 0.2970321358507854, "mean_token_offdiagonal_cosine": 0.9998745714345297, "temperature": 0.49648627638816833 }, { "layer": 12, "head": 7, "slices": 32, "native_nodes": 33555, "mass": [ 0.02972622127036964, 0.035201356979871076, 0.03148827051332812, 0.030923362729582182, 0.02710376084750126, 0.038081868697857645, 0.030450694546583085, 0.02537156226716557, 0.030942383646463213, 0.030233675286211566, 0.035450580991352745, 0.029648544571737545, 0.0336187692014649, 0.03377364185298587, 0.026878532422632932, 0.029023800029375924, 0.035437661505022144, 0.035425336343878656, 0.03245036773402491, 0.029738602712317454, 0.031082961538501527, 0.031483984818578545, 0.03364077304810343, 0.03421140744379577, 0.029785827206423574, 0.029872216810123288, 0.03210125383583049, 0.033045557021624, 0.03055880473808149, 0.028608302776596973, 0.025652723625300095, 0.028987193371556037 ], "mean_node_assignment_entropy": 3.4606883145510294, "normalized_node_entropy": 0.9985435738930311, "effective_mass_slots": 31.85700874531024, "node_slot_mutual_information_nats": 0.0005690980564470038, "normalized_node_slot_mutual_information": 0.00016420698876312795, "centered_token_effective_rank": 1.8265242178117975, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.003790882146258127, "token_RMS": 0.25565539025988926, "mean_token_offdiagonal_cosine": 0.9998036012802975, "temperature": 0.4920627176761627 }, { "layer": 12, "head": 8, "slices": 32, "native_nodes": 33555, "mass": [ 0.0429434289819099, 0.03669490852982867, 0.03869867859814317, 0.01979048769283252, 0.03616265320919777, 0.02251549181923206, 0.03390175089847356, 0.02945093488871092, 0.02777671215575865, 0.02597799410780129, 0.03390183829980127, 0.036005864040755405, 0.0312764876366834, 0.023838208313804326, 0.026058761457753172, 0.02300173892978051, 0.024008399081370493, 0.021256797654589498, 0.030155839026418354, 0.03229751233574913, 0.0464406932492518, 0.025508007533141086, 0.033097139027037946, 0.026061900721585637, 0.05753135959985688, 0.019900826029973544, 0.03560909576088994, 0.03658990867568047, 0.026619063668933324, 0.02659101386037878, 0.026976797181012164, 0.043359707333752585 ], "mean_node_assignment_entropy": 3.431479907406163, "normalized_node_entropy": 0.9901158090649982, "effective_mass_slots": 30.950293054381593, "node_slot_mutual_information_nats": 0.0009025603187615694, "normalized_node_slot_mutual_information": 0.00026042385919609564, "centered_token_effective_rank": 1.5442747871039182, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.003445500145101186, "token_RMS": 0.2930407179592093, "mean_token_offdiagonal_cosine": 0.9998596494734543, "temperature": 0.4815036952495575 }, { "layer": 13, "head": 1, "slices": 32, "native_nodes": 33555, "mass": [ 0.046994905030451166, 0.02050257438701909, 0.023974784558622, 0.019327025920165986, 0.03036446298839367, 0.02895517533587026, 0.04834740961731288, 0.01517860791686883, 0.021688105509844166, 0.0215468049487353, 0.01939339389953212, 0.0664239464999089, 0.017397613217075808, 0.021348672522105094, 0.01786495848653072, 0.036021855244039155, 0.09250290523478996, 0.015204308146030496, 0.010626049727537853, 0.024450126751222635, 0.049909972903616935, 0.0586364344350806, 0.04927834595215381, 0.039924634893157944, 0.01241353731904961, 0.010794632760312443, 0.020662441590198317, 0.03609303235369323, 0.015232695056054192, 0.05474549243716525, 0.0236865389621963, 0.03050855567256654 ], "mean_node_assignment_entropy": 3.3101259517157406, "normalized_node_entropy": 0.9551004590516319, "effective_mass_slots": 27.424739043618906, "node_slot_mutual_information_nats": 0.0013195391265874434, "normalized_node_slot_mutual_information": 0.00038073851083733173, "centered_token_effective_rank": 1.4856885796300416, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.003523866506661258, "token_RMS": 0.3415448627695244, "mean_token_offdiagonal_cosine": 0.9998901887301596, "temperature": 0.4663889408111572 }, { "layer": 13, "head": 2, "slices": 32, "native_nodes": 33555, "mass": [ 0.025992364229334926, 0.03281900602366624, 0.030795255377766558, 0.034973877894695866, 0.026161946428959405, 0.029278097514381193, 0.02922333314645141, 0.03197234910280689, 0.03691127642432946, 0.02861590866466301, 0.03539104857801894, 0.029424809471256794, 0.026469121703073424, 0.03436034074498082, 0.027426097474602593, 0.029324728175731285, 0.027990141056223786, 0.027695605750988783, 0.036257492638641454, 0.028174965559987726, 0.02936441037970974, 0.034590197707822117, 0.02601398420093392, 0.0321696510618116, 0.0388066606961647, 0.04331398805620871, 0.028769047203097805, 0.02997995264436303, 0.03683052820686016, 0.029389060098933776, 0.0325149187252243, 0.02899983541318635 ], "mean_node_assignment_entropy": 3.4569173162563396, "normalized_node_entropy": 0.9974554937852411, "effective_mass_slots": 31.732953237868177, "node_slot_mutual_information_nats": 0.00043835900286426366, "normalized_node_slot_mutual_information": 0.00012648367191226082, "centered_token_effective_rank": 1.984245622208863, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.001969556195159507, "token_RMS": 0.2362026410617831, "mean_token_offdiagonal_cosine": 0.9999300462037364, "temperature": 0.4914398491382599 }, { "layer": 13, "head": 3, "slices": 32, "native_nodes": 33555, "mass": [ 0.04254752004952362, 0.0247076687919407, 0.05229502141285833, 0.022716398756868383, 0.03200932629267677, 0.03195929836141532, 0.024932997120146582, 0.022006807695602967, 0.014836034118838307, 0.023898023439701988, 0.04793094549899289, 0.03873036445539393, 0.059577316283844405, 0.01839812627030838, 0.03649995547998655, 0.030570693853468195, 0.036615930614772876, 0.022244072670480335, 0.024203740822207092, 0.022261753769358586, 0.04564073906081512, 0.01661795305892783, 0.03083715881159241, 0.01822718385764653, 0.016713153776250454, 0.020935153073517846, 0.047264695596191356, 0.020067877879924433, 0.01324665058571293, 0.04371475643811257, 0.037134207183398854, 0.060658474497895645 ], "mean_node_assignment_entropy": 3.380511632651146, "normalized_node_entropy": 0.9754094736186523, "effective_mass_slots": 29.450798814511856, "node_slot_mutual_information_nats": 0.0022094015102309683, "normalized_node_slot_mutual_information": 0.0006374985204285608, "centered_token_effective_rank": 1.1244824641852047, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.005265344578639553, "token_RMS": 0.31531647846542576, "mean_token_offdiagonal_cosine": 0.9997165806196361, "temperature": 0.4742530584335327 }, { "layer": 13, "head": 4, "slices": 32, "native_nodes": 33555, "mass": [ 0.027729187345301814, 0.030248654322625718, 0.0254451620003236, 0.03613145983954324, 0.027811575324151506, 0.029799741914077003, 0.036882050537408916, 0.03804197066329334, 0.03683694139258408, 0.032225217700732875, 0.026326030320589916, 0.030131902285070095, 0.025966424753659352, 0.03196452965951858, 0.02769323764583994, 0.032285405181025253, 0.03071382867652246, 0.03772469043760555, 0.02862481388091739, 0.03471237058087799, 0.027436809830513, 0.03266170286424841, 0.029456225676053223, 0.03188934954236613, 0.03522001520518645, 0.03328581019417795, 0.027695233722216963, 0.03303613768017047, 0.03850153545474193, 0.02677953550182677, 0.031034614535104393, 0.025707835082207312 ], "mean_node_assignment_entropy": 3.456485141389733, "normalized_node_entropy": 0.9973307944778711, "effective_mass_slots": 31.755679286808565, "node_slot_mutual_information_nats": 0.0015864431277838875, "normalized_node_slot_mutual_information": 0.0004577507266212381, "centered_token_effective_rank": 1.7828367193191392, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.0039026066335433682, "token_RMS": 0.32319764097559694, "mean_token_offdiagonal_cosine": 0.9998595856126455, "temperature": 0.48596644401550293 }, { "layer": 13, "head": 5, "slices": 32, "native_nodes": 33555, "mass": [ 0.025427292188541523, 0.02920602154680015, 0.023364738198925622, 0.03003584304770349, 0.027105948890624183, 0.024398576338224377, 0.048589771705015, 0.025316714423368202, 0.057174389251381394, 0.032527827694257154, 0.01965609906626261, 0.034644535074440416, 0.01444577959623849, 0.04048878555392175, 0.026236108009667514, 0.03445269287808244, 0.03340494338955357, 0.02359269152876562, 0.017250013191820818, 0.034935341109794384, 0.02801383841394792, 0.0628337762732519, 0.02789395049729498, 0.042575059975135915, 0.026601762147549547, 0.021846592852785004, 0.0177971518646652, 0.05493257951039686, 0.04023884339517585, 0.030149271669352825, 0.02490388093206206, 0.01995917965321201 ], "mean_node_assignment_entropy": 3.402248986702671, "normalized_node_entropy": 0.9816815481970889, "effective_mass_slots": 30.050704838580053, "node_slot_mutual_information_nats": 0.000637129530259628, "normalized_node_slot_mutual_information": 0.00018383672274189603, "centered_token_effective_rank": 1.8724185188614944, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.0025906807561231573, "token_RMS": 0.2855681761960353, "mean_token_offdiagonal_cosine": 0.9999265226754055, "temperature": 0.4754674732685089 }, { "layer": 13, "head": 6, "slices": 32, "native_nodes": 33555, "mass": [ 0.031396816645359926, 0.028581378331613016, 0.02883504861695163, 0.029000022942778998, 0.030546372875209324, 0.029432014001619707, 0.03158179099466571, 0.02675805972299825, 0.028973110204066007, 0.03196811335581263, 0.031642942382748535, 0.0354806519763922, 0.03681366437161542, 0.02825414218056257, 0.03401425311558649, 0.02844451879992284, 0.03300217855985386, 0.02673162888219843, 0.029298464634660158, 0.03154639972243672, 0.03520749653372461, 0.030700384115270415, 0.033783831521802095, 0.03359237688752742, 0.024865775265386893, 0.02476691214445184, 0.036240229340423646, 0.03035709957420524, 0.0247669079247876, 0.037972305374140355, 0.0335419268153176, 0.04190318257858911 ], "mean_node_assignment_entropy": 3.4566219109570233, "normalized_node_entropy": 0.9973702578331659, "effective_mass_slots": 31.752604490483115, "node_slot_mutual_information_nats": 0.001352842213621308, "normalized_node_slot_mutual_information": 0.00039034775053934176, "centered_token_effective_rank": 1.651334130877403, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.0043388249179281135, "token_RMS": 0.33344990765574267, "mean_token_offdiagonal_cosine": 0.9998260887760146, "temperature": 0.4838605523109436 }, { "layer": 13, "head": 7, "slices": 32, "native_nodes": 33555, "mass": [ 0.02727682618239725, 0.037191835557951726, 0.04819871859105684, 0.03152883528370808, 0.02939969207139497, 0.03192914519295254, 0.016507666127045986, 0.022516062711005397, 0.023156308482834467, 0.03069421990827723, 0.047898249454699726, 0.028569573081720073, 0.05402616344413393, 0.02428482741038653, 0.036433825897772575, 0.02151263251144624, 0.018915530588328528, 0.02748039305180138, 0.04069156933838432, 0.017556439395458274, 0.03607960290304241, 0.024809758750450644, 0.02200434587117617, 0.015431788810609998, 0.035903178010540555, 0.058118693094249274, 0.038295736889095516, 0.02583538886609846, 0.01991665259979509, 0.038130267481235644, 0.03398913666733623, 0.035716935863457214 ], "mean_node_assignment_entropy": 3.409966313187589, "normalized_node_entropy": 0.9839082979268313, "effective_mass_slots": 30.28990664778103, "node_slot_mutual_information_nats": 0.0008482298983438774, "normalized_node_slot_mutual_information": 0.0002447474135748923, "centered_token_effective_rank": 1.8324515580344094, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.00337232704771668, "token_RMS": 0.2972842515339255, "mean_token_offdiagonal_cosine": 0.9998836180604735, "temperature": 0.4738651216030121 }, { "layer": 13, "head": 8, "slices": 32, "native_nodes": 33555, "mass": [ 0.023224320774419157, 0.022963825279199816, 0.017870790018328007, 0.0390993571818994, 0.027194663250271797, 0.023803277596422766, 0.03801824688093217, 0.056264211743572684, 0.04138039630858416, 0.031494908450739496, 0.021801875441136275, 0.018296897235268044, 0.020607602715296852, 0.03307598538740064, 0.03212315172985065, 0.04294921999658172, 0.02742609812240676, 0.04577048179958443, 0.028818658824925132, 0.05339029676653806, 0.02105345605406147, 0.029263472242935584, 0.03025509211219622, 0.043908887243778456, 0.03565393265538254, 0.028153799783486737, 0.027681344304238787, 0.026340393762302246, 0.05335987353556701, 0.015106065311853408, 0.026696181835362726, 0.016953235707656403 ], "mean_node_assignment_entropy": 3.407417187694526, "normalized_node_entropy": 0.9831727757853421, "effective_mass_slots": 30.199341692410687, "node_slot_mutual_information_nats": 0.00040293818416570204, "normalized_node_slot_mutual_information": 0.00011626338401613243, "centered_token_effective_rank": 1.66918330774402, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.001999330216774946, "token_RMS": 0.2545790420064731, "mean_token_offdiagonal_cosine": 0.9999423356807525, "temperature": 0.4673958122730255 }, { "layer": 14, "head": 1, "slices": 32, "native_nodes": 33555, "mass": [ 0.05410558258421578, 0.015737758632935278, 0.019888058811983484, 0.016514285510807514, 0.02279532257222708, 0.060470355231819484, 0.028160260153612122, 0.02782967292269737, 0.00706760321322325, 0.030089276274709212, 0.019680584456109377, 0.012533801818476617, 0.008171556602426305, 0.02177482395892375, 0.03504372870219551, 0.007942549686328258, 0.03627531334997609, 0.027385831750444196, 0.018513962505000873, 0.02275750998009594, 0.10186729508877743, 0.07224466513313332, 0.021386271158951864, 0.04636757028092217, 0.14261793836012024, 0.022631311034049598, 0.0142558885349127, 0.01587719566661561, 0.022778038236138682, 0.02040661646328248, 0.01774929295184493, 0.009080078155124282 ], "mean_node_assignment_entropy": 3.170573298354076, "normalized_node_entropy": 0.9148340748620779, "effective_mass_slots": 23.844966143988994, "node_slot_mutual_information_nats": 0.0009998335581982154, "normalized_node_slot_mutual_information": 0.00028849098322538644, "centered_token_effective_rank": 1.416867631957022, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.0020289565718817648, "token_RMS": 0.2744179085009858, "mean_token_offdiagonal_cosine": 0.9999442392167883, "temperature": 0.4687247574329376 }, { "layer": 14, "head": 2, "slices": 32, "native_nodes": 33555, "mass": [ 0.01794211635769364, 0.033051963755007456, 0.02645465222658761, 0.02732491506919467, 0.024565602172926044, 0.0183644895431422, 0.029153774163183786, 0.026880734717558596, 0.07969454691155961, 0.025003256547361758, 0.03614870834717619, 0.06710990674703794, 0.07029808677023525, 0.03054074908044251, 0.012441475024482094, 0.07157460850544275, 0.01604554543709914, 0.022783127691274233, 0.031064064830603783, 0.027442312998925585, 0.008410562383103408, 0.012202409954083523, 0.02794406174366644, 0.012729229566890531, 0.007363020420097222, 0.021695217883708116, 0.041924332189406284, 0.024696876212117427, 0.024002120637803527, 0.04290839355167503, 0.033025546821514, 0.049213591229096736 ], "mean_node_assignment_entropy": 3.312634037734799, "normalized_node_entropy": 0.9558241397039955, "effective_mass_slots": 27.474217047327173, "node_slot_mutual_information_nats": 0.0006139652362726622, "normalized_node_slot_mutual_information": 0.0001771529203297581, "centered_token_effective_rank": 1.4967387541592319, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.002407481287784565, "token_RMS": 0.29506444576722024, "mean_token_offdiagonal_cosine": 0.9999327569822515, "temperature": 0.4695081114768982 }, { "layer": 14, "head": 3, "slices": 32, "native_nodes": 33555, "mass": [ 0.01846051539888797, 0.0294578455785444, 0.017702902218729652, 0.037201813523588374, 0.021034580653647415, 0.019949187871250235, 0.016569605193595884, 0.015830224540959996, 0.10153394063617999, 0.019741883609544375, 0.018521546828959512, 0.03167056866243288, 0.09982928359519394, 0.015207843844809506, 0.022163926375936314, 0.1014343344629357, 0.015541737508558869, 0.023027205308464915, 0.029618623865108575, 0.02853633772935416, 0.011838937210283519, 0.010112750914484605, 0.048789372637639224, 0.016581591384352418, 0.013038333883916207, 0.018485118630863287, 0.030249538317693538, 0.020622167957222527, 0.0208844543498655, 0.01530399262900125, 0.02686827050161431, 0.08419156439303731 ], "mean_node_assignment_entropy": 3.204954700682131, "normalized_node_entropy": 0.9247544505895764, "effective_mass_slots": 24.669504476009614, "node_slot_mutual_information_nats": 0.0006131435613223601, "normalized_node_slot_mutual_information": 0.0001769158350545534, "centered_token_effective_rank": 1.8086769856159939, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.0018311215499137556, "token_RMS": 0.3079779731041337, "mean_token_offdiagonal_cosine": 0.9999654463907688, "temperature": 0.4686199426651001 }, { "layer": 14, "head": 4, "slices": 32, "native_nodes": 33555, "mass": [ 0.006308472755965556, 0.017590219648051166, 0.011406028116153252, 0.018489067643023768, 0.00967871264103956, 0.003774823927081985, 0.010435970838252033, 0.009048147972452802, 0.21599446480233078, 0.01089803321316734, 0.013655741940176295, 0.042271245307604226, 0.20284071882159482, 0.010092865710603713, 0.004529445687369414, 0.17134750729165565, 0.004861411791349768, 0.009296844915308269, 0.01549759790214727, 0.015039468447898028, 0.0012050084764677839, 0.001629469514698303, 0.021633499006572413, 0.003137366061282474, 0.0009990708867055765, 0.008360114268877524, 0.028979239773020235, 0.011989415438537394, 0.00816245282484065, 0.010896505879838037, 0.020018030094009737, 0.07993303823912487 ], "mean_node_assignment_entropy": 2.539996847846831, "normalized_node_entropy": 0.7328881712524443, "effective_mass_slots": 12.695668373888799, "node_slot_mutual_information_nats": 0.0012640145169746653, "normalized_node_slot_mutual_information": 0.0003647174950502016, "centered_token_effective_rank": 1.276710805894231, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.0049177726830325645, "token_RMS": 0.2644371028750099, "mean_token_offdiagonal_cosine": 0.9996470075214642, "temperature": 0.4525659680366516 }, { "layer": 14, "head": 5, "slices": 32, "native_nodes": 33555, "mass": [ 0.00849126122525258, 0.019168474511058138, 0.014235617874034505, 0.02346686028846684, 0.012824770595921105, 0.007642596762562417, 0.011658478647580385, 0.011100569324749154, 0.1775689747875617, 0.012613044066470926, 0.01563807862403207, 0.04376211207059719, 0.19153372684601627, 0.014558423771114245, 0.006650629798134958, 0.13619194997472053, 0.006572973985816701, 0.012240368429992713, 0.020190506569698126, 0.018785336050837056, 0.0019320847110490013, 0.002555959221541009, 0.02333544153298818, 0.004686316432973398, 0.0016515411994990497, 0.011005254159525376, 0.0299328139456742, 0.016557873682857376, 0.011801117373847727, 0.015241074301480065, 0.022441420122934444, 0.0939643490365109 ], "mean_node_assignment_entropy": 2.7419898268083647, "normalized_node_entropy": 0.7911710250608831, "effective_mass_slots": 15.550801195629814, "node_slot_mutual_information_nats": 0.0021223343242859904, "normalized_node_slot_mutual_information": 0.0006123762409511655, "centered_token_effective_rank": 1.3315093247888667, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.003009395276046462, "token_RMS": 0.31566416115854645, "mean_token_offdiagonal_cosine": 0.9999117118947137, "temperature": 0.43758776783943176 }, { "layer": 14, "head": 6, "slices": 32, "native_nodes": 33555, "mass": [ 0.04417092067394486, 0.02531519901840622, 0.03679825231441796, 0.024264743329571053, 0.0297589686757081, 0.043823719292439466, 0.03251785889940705, 0.037383065781482656, 0.023101825013561356, 0.029549658327579782, 0.03276710427477079, 0.02479794338332205, 0.019143349756037775, 0.034756701720389714, 0.02459661902688003, 0.022718967766073718, 0.03187441665125779, 0.03453289519097851, 0.0250497207197098, 0.029860229220842485, 0.05277350338098456, 0.04079734552487889, 0.023547494659266092, 0.03450774710457214, 0.04206659410589055, 0.03276183700976962, 0.02626106601900661, 0.028948829695526878, 0.036059212002521865, 0.030606858575953366, 0.025841001907316072, 0.01904635110043179 ], "mean_node_assignment_entropy": 3.435369686214534, "normalized_node_entropy": 0.9912381619843965, "effective_mass_slots": 31.070523458236302, "node_slot_mutual_information_nats": 0.000889884754851078, "normalized_node_slot_mutual_information": 0.0002567664645572682, "centered_token_effective_rank": 1.8769924836595628, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.0027923137214215846, "token_RMS": 0.2630162081309891, "mean_token_offdiagonal_cosine": 0.9998861740640618, "temperature": 0.48200154304504395 }, { "layer": 14, "head": 7, "slices": 32, "native_nodes": 33555, "mass": [ 0.018932355490820257, 0.01259858690739575, 0.011235877900795003, 0.015228892372225288, 0.014440478147077607, 0.02707334047824815, 0.011500445135257503, 0.013407449082185136, 0.0010812209397299051, 0.012209885513942594, 0.0097668734539676, 0.0027941932306809383, 0.0013918202732921573, 0.012199435101006297, 0.0983673114636548, 0.0016439425495864371, 0.05499634282457244, 0.013092414271061475, 0.010734193098017061, 0.012318960351739293, 0.13749866095081364, 0.09136446936963888, 0.01061791257369076, 0.06661201267979341, 0.25586214384047146, 0.014808387942351273, 0.005514863436641134, 0.014215672007151145, 0.022400487204945634, 0.012839224768102817, 0.009189064047271432, 0.004063082121945712 ], "mean_node_assignment_entropy": 2.6818243482059194, "normalized_node_entropy": 0.7738109375383914, "effective_mass_slots": 14.626831797464973, "node_slot_mutual_information_nats": 0.001033288177451741, "normalized_node_slot_mutual_information": 0.0002981439458837644, "centered_token_effective_rank": 1.391796452212866, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.0029598008778765578, "token_RMS": 0.3284267797384058, "mean_token_offdiagonal_cosine": 0.9999167589024156, "temperature": 0.4401369094848633 }, { "layer": 14, "head": 8, "slices": 32, "native_nodes": 33555, "mass": [ 0.01221366443493237, 0.02559879338678243, 0.016898750422921823, 0.02319588383459606, 0.012514779943322802, 0.009223471443673824, 0.014077221895905991, 0.01369827179283441, 0.1779100190733206, 0.015276695101883698, 0.01924714738861972, 0.03726358409777268, 0.1273635945871146, 0.01355464373831932, 0.008729278906650265, 0.15249805790128643, 0.008273964999900026, 0.014458014298190998, 0.020490967201335725, 0.022158426004960843, 0.0038504303009041802, 0.0045092462941991995, 0.032463741753344176, 0.007702202751850642, 0.003508118551004927, 0.010792558644071828, 0.03140479652448376, 0.015444566351279, 0.013658912488315024, 0.014886986275777085, 0.02551896756806613, 0.09161424198088108 ], "mean_node_assignment_entropy": 2.8693147405566184, "normalized_node_entropy": 0.8279092293901272, "effective_mass_slots": 17.716509338155923, "node_slot_mutual_information_nats": 0.005182195269194612, "normalized_node_slot_mutual_information": 0.0014952654831570625, "centered_token_effective_rank": 1.022928357060449, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.006263569187384846, "token_RMS": 0.23381915407891898, "mean_token_offdiagonal_cosine": 0.9992607847235381, "temperature": 0.4529479444026947 }, { "layer": 15, "head": 1, "slices": 32, "native_nodes": 33555, "mass": [ 0.03663350032125406, 0.019017550027442837, 0.02589584205051485, 0.026315367348118006, 0.02323084217924339, 0.020113089968048056, 0.018964121405841822, 0.023725383226652737, 0.020006761327885494, 0.024489363893586767, 0.02496047199683908, 0.026259842168052938, 0.029920348137197454, 0.01901521940217823, 0.02747825112933774, 0.06857125620460247, 0.028001781751368, 0.14539008342288073, 0.020099802201938795, 0.020416083888014334, 0.03337535592338098, 0.016391171364277722, 0.09423341524916724, 0.03067261068756295, 0.017763152536637045, 0.01767854231665492, 0.024542940737999615, 0.04670924444882316, 0.007659284446822573, 0.029576668512193267, 0.013707286346581091, 0.019185365516275518 ], "mean_node_assignment_entropy": 3.2376094875871337, "normalized_node_entropy": 0.9341766304154031, "effective_mass_slots": 25.491381821820113, "node_slot_mutual_information_nats": 0.0007309396815080582, "normalized_node_slot_mutual_information": 0.00021090461074012678, "centered_token_effective_rank": 1.290203641428389, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.001795491956595787, "token_RMS": 0.2566438195003532, "mean_token_offdiagonal_cosine": 0.9999497472873925, "temperature": 0.47357478737831116 }, { "layer": 15, "head": 2, "slices": 32, "native_nodes": 33555, "mass": [ 0.031806015034418954, 0.03190658577236151, 0.034553433859682244, 0.03478078568497743, 0.029239735039315463, 0.03125100262838342, 0.030852916204865848, 0.03565204624164888, 0.03120288949344885, 0.03035962925736772, 0.032331716309333056, 0.03117335685004937, 0.03102053197545118, 0.030150589807897987, 0.03212844909876119, 0.03494583634894734, 0.034341242866577575, 0.033178041766673404, 0.02566297424649352, 0.03059238503328904, 0.031034903360841314, 0.033622758906274135, 0.03138022010732676, 0.035051689762025384, 0.035232122455387885, 0.02612363539576184, 0.024345870232041645, 0.02942177869003161, 0.030672211005269033, 0.025755859528754404, 0.03002821934679913, 0.030200567350264475 ], "mean_node_assignment_entropy": 3.461035032035044, "normalized_node_entropy": 0.9986436154119864, "effective_mass_slots": 31.86586846250596, "node_slot_mutual_information_nats": 0.0005004507927686141, "normalized_node_slot_mutual_information": 0.00014439957538724598, "centered_token_effective_rank": 1.4242600539340027, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.0016773204850164332, "token_RMS": 0.28376237014888867, "mean_token_offdiagonal_cosine": 0.9999648093494803, "temperature": 0.48624086380004883 }, { "layer": 15, "head": 3, "slices": 32, "native_nodes": 33555, "mass": [ 0.025797159705578065, 0.03005446173853516, 0.030870071542938478, 0.029051852827377406, 0.026294563243515603, 0.029670146550756154, 0.03251735294403431, 0.02926615290266047, 0.035033271710688534, 0.029883700957922303, 0.02758048366221794, 0.026384371584424263, 0.026380313008428597, 0.03054211716391196, 0.027119509251784097, 0.01839286931594044, 0.026260525236804676, 0.013325776764630357, 0.027940110628390576, 0.03427847484450034, 0.02503600859446411, 0.05325570054348953, 0.018033259900042828, 0.026270707542524398, 0.03508615238255359, 0.02612203174421816, 0.030052942425931838, 0.023589483756839013, 0.09441893819884419, 0.02493385312141471, 0.049328065195847086, 0.03722957065538507 ], "mean_node_assignment_entropy": 3.3928627578394264, "normalized_node_entropy": 0.9789732550303585, "effective_mass_slots": 29.761101287159075, "node_slot_mutual_information_nats": 0.00033945705767823853, "normalized_node_slot_mutual_information": 9.794660274143071e-05, "centered_token_effective_rank": 1.8930249742122651, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.0011152513653180426, "token_RMS": 0.28462598364276726, "mean_token_offdiagonal_cosine": 0.9999849665282705, "temperature": 0.4791676998138428 }, { "layer": 15, "head": 4, "slices": 32, "native_nodes": 33555, "mass": [ 0.032467492711453445, 0.02987140544128567, 0.03382513028949903, 0.031014340665597193, 0.03450249025411932, 0.02935861339663548, 0.03063208947687866, 0.029136588588809936, 0.030293414931599455, 0.033545323262900675, 0.03131808452052831, 0.031094942986077016, 0.03225817765406082, 0.030118666925485454, 0.02998821624877902, 0.035040849315580144, 0.03354762458809929, 0.041012568751078324, 0.02993637246109302, 0.02703155407510257, 0.029923976730428178, 0.024325506235544327, 0.037827183742416434, 0.03225353392877152, 0.03658965567023899, 0.029275634220043995, 0.029743500550365932, 0.031231100570991682, 0.023185909487337047, 0.03424787371229387, 0.02565308788403005, 0.029749090980331513 ], "mean_node_assignment_entropy": 3.458825986655447, "normalized_node_entropy": 0.9980062196491379, "effective_mass_slots": 31.791158416434598, "node_slot_mutual_information_nats": 0.00036222725213264084, "normalized_node_slot_mutual_information": 0.00010451669206531943, "centered_token_effective_rank": 1.4407188033771388, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.0013461273548620865, "token_RMS": 0.3106905270897712, "mean_token_offdiagonal_cosine": 0.9999809049328575, "temperature": 0.49542805552482605 }, { "layer": 15, "head": 5, "slices": 32, "native_nodes": 33555, "mass": [ 0.014786506542635379, 0.004861898339672547, 0.008866358699307296, 0.009772545226682113, 0.009848776337404913, 0.007509367317050105, 0.007108657956878291, 0.008666577991575553, 0.006374433511725735, 0.008678710301820372, 0.012769289774204879, 0.009223162643128491, 0.010526401610583436, 0.004791494756975884, 0.011120980752529605, 0.077968440918165, 0.010042786692933743, 0.5101162261378301, 0.006892982700631753, 0.0038041013766262214, 0.0123770273797244, 0.0018881864963018037, 0.14003318284173744, 0.010905159836891174, 0.00590923048376553, 0.00555950030662267, 0.012020583845249783, 0.04316047480930214, 0.0003997878797615363, 0.015309136879598884, 0.0014805856417346097, 0.007227443779660926 ], "mean_node_assignment_entropy": 2.022896998992723, "normalized_node_entropy": 0.5836846937351936, "effective_mass_slots": 7.575434894916722, "node_slot_mutual_information_nats": 0.002013762624905713, "normalized_node_slot_mutual_information": 0.0005810490704958027, "centered_token_effective_rank": 1.1309727056900576, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.002634097341952576, "token_RMS": 0.3471744275379933, "mean_token_offdiagonal_cosine": 0.9999406933949712, "temperature": 0.44751599431037903 }, { "layer": 15, "head": 6, "slices": 32, "native_nodes": 33555, "mass": [ 0.008130797634456063, 0.019104828431784904, 0.01062885554261187, 0.01011133479439316, 0.005861353942761654, 0.011646958974494945, 0.009944700780712516, 0.011351479016934958, 0.01506187800362966, 0.009091552910193049, 0.006912062413179493, 0.010703683026593483, 0.009195836942968771, 0.02031029241034383, 0.008623625109999963, 0.0031593219659508667, 0.009893255601392127, 0.0005375806929114381, 0.010089955758511567, 0.04353463802169695, 0.00937893256839826, 0.19690149724716632, 0.0016894924687013483, 0.009143240620039217, 0.010010898196876866, 0.01283571333627892, 0.010342188127896208, 0.003893075921884851, 0.3852094032253386, 0.005370012824324387, 0.11093386297204814, 0.010397690193287139 ], "mean_node_assignment_entropy": 2.263730351902992, "normalized_node_entropy": 0.6531745105200549, "effective_mass_slots": 9.643838669927328, "node_slot_mutual_information_nats": 0.002588879732596361, "normalized_node_slot_mutual_information": 0.0007469927903349431, "centered_token_effective_rank": 1.8452813931628023, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.0030986038699490036, "token_RMS": 0.3339713966488057, "mean_token_offdiagonal_cosine": 0.9999159961619214, "temperature": 0.44210320711135864 }, { "layer": 15, "head": 7, "slices": 32, "native_nodes": 33555, "mass": [ 0.026863932011380067, 0.03480176884737052, 0.0224183355504293, 0.03029454714507464, 0.022207372340788217, 0.02276315486713336, 0.01852824544746969, 0.027368484515790575, 0.022824551920653403, 0.022511485490058134, 0.017624797628568813, 0.03488655142761426, 0.02991778186459004, 0.038672669120094066, 0.02375524315515106, 0.01986256702488749, 0.03444215267443689, 0.00774271175305002, 0.028207115477222724, 0.06515252843539152, 0.03433960269469917, 0.07309563067262778, 0.013787666717973015, 0.027161607501084394, 0.024432826751706816, 0.03479793581075222, 0.024630510082740115, 0.014045947859505883, 0.08396436682673436, 0.02621377404030363, 0.06865318961482594, 0.024030945064826652 ], "mean_node_assignment_entropy": 3.337189666837378, "normalized_node_entropy": 0.9629093965704355, "effective_mass_slots": 28.14630408514025, "node_slot_mutual_information_nats": 0.00022638552786036215, "normalized_node_slot_mutual_information": 6.532105567463494e-05, "centered_token_effective_rank": 1.9439218989880498, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.0010585145488621682, "token_RMS": 0.25498568210468686, "mean_token_offdiagonal_cosine": 0.9999831132835514, "temperature": 0.4643689692020416 }, { "layer": 15, "head": 8, "slices": 32, "native_nodes": 33555, "mass": [ 0.02513715919951509, 0.034701200820879755, 0.026542053987442638, 0.028643680469640463, 0.021321304989560957, 0.025069174418791255, 0.023469042653727772, 0.029385840186979408, 0.02863356111005821, 0.024458891991539767, 0.017945000307137377, 0.028747343262811562, 0.02550281321105588, 0.03637668211110222, 0.023082956144811855, 0.01807931797306098, 0.030559897246712136, 0.008569697160584647, 0.028011294688422728, 0.061500089845506714, 0.029477620042051168, 0.08230184807456423, 0.014711333486021027, 0.026617617648183153, 0.021300143878748762, 0.029997942977918097, 0.022624357358162973, 0.017197286066946267, 0.09305411789806638, 0.021724684504816242, 0.06685747577563303, 0.02839857021874313 ], "mean_node_assignment_entropy": 3.329460674756684, "normalized_node_entropy": 0.960679280861258, "effective_mass_slots": 27.93200712993232, "node_slot_mutual_information_nats": 0.0003125654874303052, "normalized_node_slot_mutual_information": 9.018733573374859e-05, "centered_token_effective_rank": 1.7325389621935987, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.0013480118310415842, "token_RMS": 0.2540177761698748, "mean_token_offdiagonal_cosine": 0.9999711118786644, "temperature": 0.4694807827472687 }, { "layer": 16, "head": 1, "slices": 32, "native_nodes": 33555, "mass": [ 0.00025874607027359827, 0.00026668163162569115, 0.00024304521538227972, 0.0002658609604687832, 0.0002745124209362778, 0.00018619773444833443, 0.0002490272018327986, 0.0002273432031949887, 0.0002151090580020963, 0.006266580521671823, 0.000237707038967157, 0.0003142999817519817, 0.001041377395652848, 7.804537795298471e-05, 0.00021798768792784842, 0.00021620832797686798, 0.00030018958387554825, 0.0002527526168024253, 0.00027253131077711174, 0.00025213687325691223, 0.0002793846625674487, 0.00023312803439309493, 0.0002471394955048298, 0.0003170151365080789, 0.00023022985309820024, 0.9857801907500573, 0.00020831462635881486, 0.00024631025189770925, 0.000268628462877886, 0.00034239455806312776, 1.2554837543390366e-07, 0.000210798966577423 ], "mean_node_assignment_entropy": 0.10984976926918258, "normalized_node_entropy": 0.031695943473489314, "effective_mass_slots": 1.116640742671117, "node_slot_mutual_information_nats": 0.00047507211179569075, "normalized_node_slot_mutual_information": 0.00013707683595045805, "centered_token_effective_rank": 1.9285910561636526, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.006390910772946775, "token_RMS": 0.3581437725513339, "mean_token_offdiagonal_cosine": 0.9997399206787109, "temperature": 0.424912691116333 }, { "layer": 16, "head": 2, "slices": 32, "native_nodes": 33555, "mass": [ 0.006021761582675012, 0.005270635354844846, 0.00635329006898123, 0.005716562226557342, 0.005477717061291294, 0.004320933179574032, 0.008174925993638388, 0.005637756241642293, 0.006652976938561793, 0.001910465980498746, 0.005034756817735733, 0.006765961365888347, 0.0034767272625498185, 0.018232519337453637, 0.005692765759388819, 0.004814145404339223, 0.007632029918372862, 0.005201972295636859, 0.007401756372584762, 0.00546692987324426, 0.009591473794162425, 0.007449047071541445, 0.004579254954796872, 0.008107727652338196, 0.005130025486787524, 9.18854406121654e-05, 0.006064559429253361, 0.004300130256374646, 0.005392008127025442, 0.006914414752762131, 0.8116090249320715, 0.005513859365450174 ], "mean_node_assignment_entropy": 1.1108301433636083, "normalized_node_entropy": 0.3205178278201308, "effective_mass_slots": 3.038178442061032, "node_slot_mutual_information_nats": 0.0004279957739916629, "normalized_node_slot_mutual_information": 0.00012349347613184114, "centered_token_effective_rank": 1.7611444422944078, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.001071869045106312, "token_RMS": 0.36552757632304567, "mean_token_offdiagonal_cosine": 0.9999916629440876, "temperature": 0.43320295214653015 }, { "layer": 16, "head": 3, "slices": 32, "native_nodes": 33555, "mass": [ 0.00019371714764537289, 0.00018288841153939236, 0.00020467397187582648, 0.00017933988050857635, 0.00019343270416427173, 0.0002073799242453251, 0.00021758174622354585, 0.00017735591623507124, 0.0001851239408042827, 2.167135904294637e-05, 0.0002109229204748077, 0.00020498287170160436, 8.892341649017613e-05, 0.0010656822516472997, 0.00016710128711546659, 0.00018311875867612956, 0.00018864005921786906, 0.00016479135726854975, 0.00020415755171669685, 0.0001836086820557664, 0.00021255165247314691, 0.00017950930245553862, 0.00019795651924308259, 0.00020348929925103904, 0.00019294468406933044, 2.2419139907335225e-07, 0.00017952720610379567, 0.00017459948652506421, 0.0001851537920116515, 0.0002223628196005814, 0.9936645422921844, 0.0001620445085291209 ], "mean_node_assignment_entropy": 0.05875149378789327, "normalized_node_entropy": 0.01695209774652247, "effective_mass_slots": 1.060597321771799, "node_slot_mutual_information_nats": 8.076672879174518e-05, "normalized_node_slot_mutual_information": 2.3304351819334925e-05, "centered_token_effective_rank": 1.5603511793136249, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.0028051558471720688, "token_RMS": 0.3249679232004386, "mean_token_offdiagonal_cosine": 0.9999306976267045, "temperature": 0.4213255047798157 }, { "layer": 16, "head": 4, "slices": 32, "native_nodes": 33555, "mass": [ 0.04066197732598091, 0.024976902667584566, 0.036704078156708106, 0.026397773441226468, 0.025404675017402293, 0.04097905060055181, 0.029005754895751423, 0.021609571060428604, 0.026865091038563358, 0.05228724552482487, 0.025915806598527262, 0.022364552964411553, 0.026868700393897608, 0.021620962737596366, 0.02401765211900404, 0.027915621625328627, 0.025125711016338072, 0.020307557745400225, 0.026672975708650168, 0.02084118667537561, 0.018226633187327347, 0.019397024847672552, 0.02926143587214006, 0.023434762194066587, 0.029068382725568113, 0.1827448681776258, 0.02111292214834633, 0.02315739890372606, 0.031159390489422446, 0.023801063644297165, 0.0032966135629478996, 0.028796656543397012 ], "mean_node_assignment_entropy": 3.2398438281000916, "normalized_node_entropy": 0.9348213248109435, "effective_mass_slots": 25.533919331614932, "node_slot_mutual_information_nats": 0.00016391003933424741, "normalized_node_slot_mutual_information": 4.7294440179886733e-05, "centered_token_effective_rank": 1.4983388069061405, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.0008634529753287569, "token_RMS": 0.28674987356332704, "mean_token_offdiagonal_cosine": 0.9999917923814458, "temperature": 0.4752122461795807 }, { "layer": 16, "head": 5, "slices": 32, "native_nodes": 33555, "mass": [ 0.000356838263069346, 0.00028390138100180734, 0.00026867812756797865, 0.0002460836277302498, 0.00030528586381944206, 0.0003459870463958175, 0.00025695523615849343, 0.00021144501843391246, 0.00023100446303945572, 1.9602752019717724e-05, 0.0002352119318933719, 0.0002604607026811531, 8.389133247990412e-05, 0.0014755232025135208, 0.000230327626196696, 0.00026774819475352515, 0.0002337705691195436, 0.00022254722810556023, 0.0002458135324119771, 0.00024887573253963574, 0.000263966277940511, 0.00018233518100164902, 0.00028281187413008743, 0.00023273296632436637, 0.0002727916744093778, 4.203800883743763e-07, 0.0002296077214449121, 0.00026091094014609953, 0.0003460344192301673, 0.0003232342273525221, 0.9913043435187262, 0.0002708596167038698 ], "mean_node_assignment_entropy": 0.0777818965771253, "normalized_node_entropy": 0.02244311129255138, "effective_mass_slots": 1.0809525614058375, "node_slot_mutual_information_nats": 6.075712611580686e-05, "normalized_node_slot_mutual_information": 1.7530800909187976e-05, "centered_token_effective_rank": 2.1298091074069796, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.002216084863422318, "token_RMS": 0.3122137979765272, "mean_token_offdiagonal_cosine": 0.9999580540251705, "temperature": 0.4346104562282562 }, { "layer": 16, "head": 6, "slices": 32, "native_nodes": 33555, "mass": [ 0.0011558891820827907, 0.0009795430393472802, 0.0008439590482522374, 0.0008747555483898642, 0.0008570471977672232, 0.001432204108494855, 0.0007715539898581258, 0.0006389249284690513, 0.0007936058513690555, 0.008708477998103804, 0.0006504685988583391, 0.0008423091138015992, 0.0014104157694941684, 0.0004163726628221993, 0.0006718558722940034, 0.0007294051959966199, 0.0007988328466225471, 0.000632292488447291, 0.0007344084432722748, 0.0007949114539858962, 0.0004996507817574291, 0.0005757921972155772, 0.0009041602875641294, 0.0007832681062098966, 0.0008257035658799718, 0.9677132082125803, 0.0006494026201623952, 0.0007103524663521114, 0.0009986969117283923, 0.0007983855727389444, 1.1794845059736077e-06, 0.0008029667952543006 ], "mean_node_assignment_entropy": 0.23982355922040968, "normalized_node_entropy": 0.06919845191512514, "effective_mass_slots": 1.2712687248999075, "node_slot_mutual_information_nats": 0.00019183856986274006, "normalized_node_slot_mutual_information": 5.535291067844121e-05, "centered_token_effective_rank": 1.3880179094045637, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.003093937132842096, "token_RMS": 0.3541984680526773, "mean_token_offdiagonal_cosine": 0.9999358971747622, "temperature": 0.44044029712677 }, { "layer": 16, "head": 7, "slices": 32, "native_nodes": 33555, "mass": [ 0.005428074625218177, 0.004953664688977996, 0.005116048860004927, 0.004755242582461365, 0.0053722231906290965, 0.0046238437390368195, 0.00582846516923433, 0.00518822754969738, 0.0055295927458664925, 0.0014760129462295874, 0.005195944185707745, 0.005269304807866171, 0.003201990787438341, 0.014307446418530743, 0.004620521544258042, 0.004567823622972739, 0.0058434076493292615, 0.005302529740077211, 0.005783819691089999, 0.005269786596368716, 0.006989717501977653, 0.006074701895155462, 0.004661449479962506, 0.006054316393932795, 0.005075734652797541, 8.314402018152754e-05, 0.0049100042076334815, 0.005153122126088579, 0.004770003220061154, 0.00602286814425694, 0.8376608325958679, 0.004910134919915604 ], "mean_node_assignment_entropy": 0.9853917639683948, "normalized_node_entropy": 0.28432396224200623, "effective_mass_slots": 2.6851734831072025, "node_slot_mutual_information_nats": 0.002353573863160374, "normalized_node_slot_mutual_information": 0.0006790978681494703, "centered_token_effective_rank": 1.081470421198374, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.002464288768382414, "token_RMS": 0.24168794456229067, "mean_token_offdiagonal_cosine": 0.9998929542613118, "temperature": 0.46964284777641296 }, { "layer": 16, "head": 8, "slices": 32, "native_nodes": 33555, "mass": [ 0.01079262692870961, 0.010275777339776544, 0.011141554311094377, 0.010807364201277267, 0.010515696799176901, 0.010437483774286334, 0.009948771926662532, 0.010823830941448027, 0.009631696805766489, 0.003256221267125081, 0.010898521491063412, 0.01149035487729224, 0.007982741444642392, 0.022838752236505357, 0.011386961668325668, 0.010498675566115015, 0.010328626375789171, 0.010838133797733409, 0.010167118664867004, 0.010511824937439893, 0.012305683121745738, 0.010043742397397361, 0.010770159353038723, 0.00920717843485303, 0.009362702875482336, 0.0004575872194734414, 0.01056489838749912, 0.010283401613407512, 0.011277779500868775, 0.013227951324806716, 0.677292172683367, 0.010634007996536327 ], "mean_node_assignment_entropy": 1.7175062595308057, "normalized_node_entropy": 0.4955675526641693, "effective_mass_slots": 5.575866154734545, "node_slot_mutual_information_nats": 0.0009414098255735581, "normalized_node_slot_mutual_information": 0.00027163345735982326, "centered_token_effective_rank": 1.4855453562697047, "centered_token_rank_upper_bound": 31, "centered_token_RMS": 0.0013960373066260222, "token_RMS": 0.3326439573684776, "mean_token_offdiagonal_cosine": 0.9999821214183071, "temperature": 0.4561024308204651 } ] }, { "schema": "ts3_capacity_slice_diagnostics_v1", "variant": "H064", "sample_id": "normal_rodv2_0150__gas_inertia_30deg", "native_nodes": 33555, "layers": 8, "slices": 64, "parameters": 3894340, "checkpoint_sha256": "1f789493374fdce8048d88cff1f4bd8f3ca4dbb240202602d7684d0ce86fccfe", "extraction_source_sha256": "26c5dec82b68ba4bc23d7c3d998b78c5bbed6aae912e6247ea64949db69bd206", "CPU_forward_seconds": 3.804767802999777, "CPU_threads": 2, "CUDA_initialized": false, "hook_max_abs_prediction_difference_on_19_node_test": 0.0, "metrics": [ { "layer": 1, "head": 1, "slices": 64, "native_nodes": 33555, "mass": [ 0.014947750748359549, 0.015296904393947083, 0.015102227123998491, 0.015507443208592141, 0.015596458822102434, 0.015097737298744647, 0.01391671177459432, 0.016268406665159968, 0.017945166190328737, 0.015512271243798548, 0.014780447770849884, 0.015999009542532444, 0.015810977033282683, 0.015049794038990547, 0.015272021339019734, 0.015766148613228774, 0.01556305451111291, 0.01617679112558792, 0.015489337565700702, 0.01621518641201808, 0.017270108569880264, 0.0160714008483046, 0.01490632416154496, 0.01661475665668602, 0.01645327734889585, 0.016869647504914192, 0.016040088113238045, 0.014453457543886493, 0.015190776788012457, 0.0149467999880517, 0.015952469187803513, 0.015607728458626796, 0.015717809570162507, 0.014264228066945463, 0.01621523185892217, 0.015393658260955147, 0.014691981841741909, 0.016027444883855733, 0.016729248196276292, 0.016393852482655732, 0.01516416175031432, 0.01569553751880371, 0.01634635341372727, 0.0157785340518008, 0.015366785718990436, 0.017535553795398347, 0.015525101918092678, 0.014786073293316526, 0.0145311906216315, 0.01602815506645929, 0.014081233450866744, 0.015698077718986386, 0.015599025124407813, 0.01642127952450586, 0.016094528495318408, 0.01613610877672516, 0.015686189380021007, 0.014850015874362004, 0.014646268796912339, 0.01584291601684841, 0.013790033748166614, 0.014581636298848077, 0.01619079470664436, 0.01650030960292379 ], "mean_node_assignment_entropy": 4.1568712150752045, "normalized_node_entropy": 0.9995162479338463, "effective_mass_slots": 63.91181474235223, "node_slot_mutual_information_nats": 0.0006330234639637666, "normalized_node_slot_mutual_information": 0.00015220996870447993, "centered_token_effective_rank": 1.5607343700740708, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.0036306036476613313, "token_RMS": 0.260923545840969, "mean_token_offdiagonal_cosine": 0.9998049707818691, "temperature": 0.4991128146648407 }, { "layer": 1, "head": 2, "slices": 64, "native_nodes": 33555, "mass": [ 0.013717595027015002, 0.01512285263848374, 0.016194470369981024, 0.01584450213274401, 0.015725257121234123, 0.01612667023562169, 0.01611139367013557, 0.015025858001511768, 0.01601964407229238, 0.014431220314027904, 0.014325322839691668, 0.016262104388949785, 0.01463425900873711, 0.014524447022114438, 0.016601290319027774, 0.017947158652102828, 0.01546940464668521, 0.015592082826699048, 0.01638441264314066, 0.015626264000823575, 0.01513823053441945, 0.014702977611892999, 0.014938555440291343, 0.016356839422586734, 0.015765553525775435, 0.014708520222309271, 0.015670802465507755, 0.015307892591811438, 0.016851498883185934, 0.015270315305857344, 0.015799229175702787, 0.01456965107999208, 0.016085217530586435, 0.01527484037451676, 0.015458187712832448, 0.016077576322677263, 0.017243299462098804, 0.01699215703491344, 0.017466856858793952, 0.015988957611398475, 0.014285044474276056, 0.01365006311438981, 0.015602264598799105, 0.015557373339520394, 0.015610276700490554, 0.016081862190840738, 0.016884647055654373, 0.015681806651589778, 0.016018377290573736, 0.014531093312230618, 0.01512133518402924, 0.01763915547579549, 0.01489352148890522, 0.014546836487396013, 0.014767363057208832, 0.014246118116766912, 0.01638792502416142, 0.016102061167191713, 0.01655876957437415, 0.015552245708180512, 0.01646225324943895, 0.01464802068653316, 0.015211886556375277, 0.016606330582346073 ], "mean_node_assignment_entropy": 4.156688203579513, "normalized_node_entropy": 0.9994722429709697, "effective_mass_slots": 63.8890767527817, "node_slot_mutual_information_nats": 0.000460200355616891, "normalized_node_slot_mutual_information": 0.00011065479514397101, "centered_token_effective_rank": 1.940317234040259, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.0025880133172498762, "token_RMS": 0.243651283819415, "mean_token_offdiagonal_cosine": 0.9998941722004197, "temperature": 0.5004549026489258 }, { "layer": 1, "head": 3, "slices": 64, "native_nodes": 33555, "mass": [ 0.015303766095741302, 0.016547258862774004, 0.014779769456242462, 0.016642854845998114, 0.016818527838501386, 0.01694356558677639, 0.01690882432205549, 0.015616058018194538, 0.015620174051450202, 0.015141768078801974, 0.014609524310456713, 0.01628812004581278, 0.015069405041688326, 0.01471965287112209, 0.016280785041833896, 0.015696359349650902, 0.015657524394312457, 0.015484237504589844, 0.015482756759482335, 0.015192064994979706, 0.015633815085767118, 0.01627592122321858, 0.01503799849028422, 0.016273169213554272, 0.01457832547476716, 0.014733893830274224, 0.0147760876437066, 0.015871948566586955, 0.01595429238964457, 0.015936856326190393, 0.01678881076347866, 0.015135801886336724, 0.01482563543253756, 0.016327529137440924, 0.01488612657722356, 0.016565961936177057, 0.01552928883627872, 0.015937278520456327, 0.016124236839889342, 0.016147271161941992, 0.015634681360448877, 0.015523590667376156, 0.015638636122107277, 0.01590046152151092, 0.015100024964795296, 0.016203592655670403, 0.01613312946709386, 0.015314260597635384, 0.015754738092037243, 0.01546448165260232, 0.016282830015740785, 0.015559754451988546, 0.016044869647012874, 0.01422630120598911, 0.014811457455160455, 0.015276304431228547, 0.014984255248350148, 0.016419490515706153, 0.015948136841855812, 0.01579007465195845, 0.015486667336067017, 0.015227380282721715, 0.01486137792864414, 0.014270256345500722 ], "mean_node_assignment_entropy": 4.157094535491253, "normalized_node_entropy": 0.9995699451433067, "effective_mass_slots": 63.94248689406605, "node_slot_mutual_information_nats": 0.0008895015671619433, "normalized_node_slot_mutual_information": 0.00021387991663458282, "centered_token_effective_rank": 1.3952731478386446, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.004226304854324172, "token_RMS": 0.24808231359537036, "mean_token_offdiagonal_cosine": 0.9997086497723514, "temperature": 0.49919256567955017 }, { "layer": 1, "head": 4, "slices": 64, "native_nodes": 33555, "mass": [ 0.015547320476536503, 0.01796825369058121, 0.01651796955368817, 0.01569521007294729, 0.014513299160422832, 0.014688035816280608, 0.01570555243744942, 0.015179280992144934, 0.016491239302547547, 0.014847921721821544, 0.015677955246586956, 0.01723451109331325, 0.01719025708038336, 0.016029172872694553, 0.016218771544944163, 0.01502349005045927, 0.0166300042050417, 0.01423696393535405, 0.01608156912581671, 0.01478948228978783, 0.014721985547322486, 0.01535590329860974, 0.015646870633843422, 0.01599369427724072, 0.014318190162301712, 0.01760786634786192, 0.0169190647862402, 0.01410267342330513, 0.014777077731833633, 0.016651358180755013, 0.015204387430877112, 0.01625742482357565, 0.016850383824225715, 0.016304151341069287, 0.01629659721688615, 0.014400280538195337, 0.015555548083857024, 0.015879963419225714, 0.016337097775967657, 0.014485898335823588, 0.015525177680372286, 0.015194874557964432, 0.016245688000260106, 0.01726134093893261, 0.015346835053832143, 0.014941593095901123, 0.01670547364854991, 0.015949902479135632, 0.016054019301540226, 0.014464351361151992, 0.014522853359217004, 0.015431339555754292, 0.015720845330403868, 0.015232433482805844, 0.014369303489448353, 0.013953536532229752, 0.014846228447570262, 0.015038884329817597, 0.01655376709768952, 0.016978045298815476, 0.014799554722286197, 0.016009967584862936, 0.01440684349449456, 0.014514463490635409 ], "mean_node_assignment_entropy": 4.156214773462093, "normalized_node_entropy": 0.9993584070905347, "effective_mass_slots": 63.88162834657084, "node_slot_mutual_information_nats": 0.0008170402700873325, "normalized_node_slot_mutual_information": 0.00019645665764359566, "centered_token_effective_rank": 1.6679910428507838, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.004437958093325549, "token_RMS": 0.27838870241040126, "mean_token_offdiagonal_cosine": 0.999744636360574, "temperature": 0.5001382231712341 }, { "layer": 1, "head": 5, "slices": 64, "native_nodes": 33555, "mass": [ 0.014820643563549053, 0.015034255738471565, 0.01581239843497647, 0.016609706510163122, 0.014558679196745437, 0.0156748657163514, 0.01648182346933939, 0.01603964459305903, 0.01591532080607871, 0.015394343533906367, 0.015514493639221616, 0.016727742048724542, 0.015325629842516038, 0.016073105951587677, 0.015640480735208358, 0.016381669360347732, 0.01484887772917663, 0.016152193207244345, 0.014976303877910973, 0.014761241677315299, 0.015521682481760373, 0.017662510061432567, 0.014569716226082502, 0.015062621222219717, 0.01656783800158606, 0.014554214037759411, 0.014582930205020992, 0.01639097761754138, 0.015768501568558434, 0.014731556555358108, 0.015957090512613056, 0.016934932663680374, 0.01615793278637617, 0.01602735780629133, 0.0156330373987218, 0.015935101964561074, 0.01740720232299469, 0.01476657521132296, 0.01592372482648915, 0.016377006092938247, 0.01612285429287345, 0.015028939747300558, 0.015309590228443715, 0.015181934400408185, 0.01641610527141601, 0.0152841113925462, 0.015711548260988056, 0.015358383030380991, 0.014283739344132646, 0.014776764948527765, 0.014047879398101143, 0.01639627294029001, 0.015482190900320287, 0.014890126433879061, 0.014305636886190309, 0.015118000185140163, 0.01669281663442778, 0.01582904200414741, 0.015061448748159508, 0.015259302899046052, 0.01566236373664365, 0.015472666826612627, 0.016581989893021686, 0.01645036235070343 ], "mean_node_assignment_entropy": 4.156568663327538, "normalized_node_entropy": 0.9994434996161844, "effective_mass_slots": 63.92140174267552, "node_slot_mutual_information_nats": 0.0010855675313869995, "normalized_node_slot_mutual_information": 0.00026102381568034975, "centered_token_effective_rank": 1.3815544169960814, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.005573477855896619, "token_RMS": 0.2895745481805774, "mean_token_offdiagonal_cosine": 0.9996413535556669, "temperature": 0.4983595311641693 }, { "layer": 1, "head": 6, "slices": 64, "native_nodes": 33555, "mass": [ 0.0147422835443709, 0.01727805183821549, 0.015168588139358789, 0.01583649727871558, 0.015263505182652942, 0.014937010095008588, 0.015964262733729164, 0.015642537554050723, 0.01895534742027943, 0.015410749905584085, 0.015344305314576593, 0.015067055874986116, 0.015929926894194386, 0.01385408705146213, 0.014812665482373213, 0.01556193575880852, 0.016958140461222546, 0.013892162976432062, 0.015128239325711631, 0.01631546122294112, 0.015396324901964674, 0.016031693753741562, 0.015812122801768674, 0.017158554950980197, 0.014931320293410011, 0.016115238197173382, 0.015888201547230103, 0.015742534526123224, 0.01596969274040453, 0.015950574418894848, 0.016650084937786518, 0.015828007612904198, 0.0157910832237092, 0.015534141890714623, 0.016974853067944027, 0.014148185616618625, 0.015338605634741582, 0.016139685340206822, 0.015181424079656895, 0.016045597754973953, 0.015231910988304339, 0.01743851524096739, 0.015573463165160308, 0.01561093648417956, 0.015807138415001584, 0.015516459912693398, 0.015642088975697965, 0.015624367808577575, 0.015319755139622315, 0.013495255310508608, 0.014726292759772918, 0.015508032455954897, 0.016407446478757325, 0.014056170503635943, 0.01595708718194485, 0.013492563989409222, 0.01637480133280409, 0.015647277813263246, 0.014185263023847325, 0.01682520986126239, 0.01546018025257063, 0.015590913889166664, 0.015630396765828433, 0.01618773095379677 ], "mean_node_assignment_entropy": 4.156248601820063, "normalized_node_entropy": 0.9993665410912489, "effective_mass_slots": 63.88587658867047, "node_slot_mutual_information_nats": 0.0008497114826200303, "normalized_node_slot_mutual_information": 0.00020431242369372106, "centered_token_effective_rank": 1.5003088124518296, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.004080270551664558, "token_RMS": 0.250280391361639, "mean_token_offdiagonal_cosine": 0.9997375129121167, "temperature": 0.5008915066719055 }, { "layer": 1, "head": 7, "slices": 64, "native_nodes": 33555, "mass": [ 0.014195165919222414, 0.01490980302308528, 0.014915434030904498, 0.014890447902951, 0.013243489188238292, 0.014065160544933602, 0.015702583518941157, 0.016221897345746243, 0.016530633927770474, 0.015257094558586513, 0.017459318216242293, 0.01585199051418522, 0.013859014309256616, 0.017007289718625054, 0.014918801533242857, 0.01505889783950192, 0.01647082979167348, 0.016726729163996885, 0.016106381730125578, 0.015794429358696106, 0.01578643559383162, 0.017119621303592517, 0.015953099813393892, 0.013772443291413245, 0.017584532098279832, 0.016142823339322387, 0.015738338481003703, 0.012991524848716747, 0.014127278298493728, 0.01562626661052515, 0.012970832830777629, 0.016096666679474604, 0.01897116951547833, 0.015787025930554522, 0.01671935172992738, 0.014616650601813155, 0.015901793786572273, 0.014020424435197455, 0.013715677716852682, 0.018246707323682707, 0.016241854430771526, 0.014918294827066747, 0.015162881632739182, 0.019387607534867457, 0.016101073235775174, 0.014338790477981929, 0.016699943736954943, 0.016198357813212383, 0.013436297315095554, 0.016907790978206702, 0.013756425067368296, 0.014306692703958863, 0.012838033368363321, 0.01897676455906412, 0.014884041869856618, 0.014566488640535261, 0.017536340139233404, 0.013680255247884786, 0.01768436771718281, 0.016107322731210887, 0.015196149677094813, 0.016242323084864136, 0.018127675889238366, 0.015630170957115525 ], "mean_node_assignment_entropy": 4.151842742362337, "normalized_node_entropy": 0.9983071558261631, "effective_mass_slots": 63.701075250928305, "node_slot_mutual_information_nats": 0.002358699993697755, "normalized_node_slot_mutual_information": 0.0005671474639754301, "centered_token_effective_rank": 1.3959926848731572, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.007307562593284872, "token_RMS": 0.26407068046383975, "mean_token_offdiagonal_cosine": 0.9992803991017056, "temperature": 0.4885198175907135 }, { "layer": 1, "head": 8, "slices": 64, "native_nodes": 33555, "mass": [ 0.01578780266981849, 0.014380798026122238, 0.014201534151583532, 0.01579744555386381, 0.015962348629725644, 0.016150243079056374, 0.014796739380466406, 0.014809689793534241, 0.015342762700368495, 0.015832105024704286, 0.017291922725996882, 0.01530439522855107, 0.01586330149638046, 0.01731916331412553, 0.015823528513222412, 0.015455024099334655, 0.015221912776483737, 0.017481605255358214, 0.015430172009724135, 0.015513426322346206, 0.018218318128386868, 0.015662446043466186, 0.014998702180945577, 0.01488966838978625, 0.013967476107902861, 0.015696900134461813, 0.014112869283647087, 0.015974833837467048, 0.016128392047256412, 0.016696369255383765, 0.01503588387307161, 0.01586374538867218, 0.014389551639908819, 0.015446349945806957, 0.014029462701102436, 0.015439539151539604, 0.016644547853689358, 0.01670785357756811, 0.01541606291166571, 0.014627878160653474, 0.018274242772507457, 0.015501035333863227, 0.015575966654941296, 0.017264085790888197, 0.015726817596324633, 0.014436029469795029, 0.01588030578658782, 0.014756262685696693, 0.015026719059010782, 0.017855200165864112, 0.01614634352832624, 0.01639800384141657, 0.01467774074486894, 0.01599084590029484, 0.014211543439523178, 0.01589656007738298, 0.014975422659713728, 0.013822539067227515, 0.01586088391381433, 0.015238604627576292, 0.015670343031328228, 0.015755137072099232, 0.014526761261458467, 0.016819834157923326 ], "mean_node_assignment_entropy": 4.155624619968292, "normalized_node_entropy": 0.9992165051707231, "effective_mass_slots": 63.86872165721098, "node_slot_mutual_information_nats": 0.0012051326438458076, "normalized_node_slot_mutual_information": 0.00028977314814829203, "centered_token_effective_rank": 1.4868222942570066, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.005999451202619677, "token_RMS": 0.21296636406157424, "mean_token_offdiagonal_cosine": 0.999218075584682, "temperature": 0.4993128776550293 }, { "layer": 2, "head": 1, "slices": 64, "native_nodes": 33555, "mass": [ 0.016472894860193692, 0.0161861279008197, 0.01461753829549862, 0.01631254977554265, 0.016478519986067624, 0.016125411692229055, 0.01668433844143917, 0.015053580176946198, 0.017244601182055633, 0.015433595606630063, 0.014564485854944268, 0.015334917531788544, 0.015764403715368896, 0.015952616972362225, 0.015983285110001414, 0.014737305559124673, 0.01631070456096067, 0.016067426297949654, 0.01527873719716979, 0.014609620664277059, 0.01562946324407801, 0.015574180617159133, 0.01587592290636391, 0.015543573473640826, 0.01518347887712266, 0.015064272583513507, 0.015762809309161973, 0.015151055126542964, 0.016318414702755055, 0.015547150744822177, 0.014238557333301243, 0.01645309497217737, 0.015526779681125539, 0.014471945167337217, 0.01662614083880951, 0.015845579653201203, 0.015876777271191005, 0.015359959217997701, 0.015032807398560265, 0.016172081822755084, 0.014703149653720288, 0.015953059541096215, 0.015262948332881495, 0.015292320042574366, 0.014728822916215462, 0.014285813566397767, 0.01476931170499497, 0.01642995905838857, 0.01629494227815351, 0.0163498439807059, 0.014629033083772539, 0.01590542897891246, 0.01696806574608637, 0.015151564605481897, 0.017061160201260372, 0.015571577903652108, 0.01606275001778086, 0.01620146525512219, 0.01526162611139374, 0.01657411798983568, 0.0164380195646792, 0.01503684565684707, 0.015230268276389675, 0.01337119920231705 ], "mean_node_assignment_entropy": 4.1561055128224975, "normalized_node_entropy": 0.99933213546005, "effective_mass_slots": 63.92200639140043, "node_slot_mutual_information_nats": 0.0015581772449033693, "normalized_node_slot_mutual_information": 0.0003746624306746865, "centered_token_effective_rank": 1.135223022430275, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.008054894972378051, "token_RMS": 0.22346080558406856, "mean_token_offdiagonal_cosine": 0.9987069699175094, "temperature": 0.5012396574020386 }, { "layer": 2, "head": 2, "slices": 64, "native_nodes": 33555, "mass": [ 0.014787602710859988, 0.016490115686771483, 0.015239258722079225, 0.016791424950058254, 0.01581366531353268, 0.01567934110284411, 0.01677207230948186, 0.0155653501982226, 0.015769105189319045, 0.015322783501434923, 0.01504338801630433, 0.01600349734060887, 0.015405548633897941, 0.01606685938586991, 0.013798319341592915, 0.014496357275572464, 0.015714005716289364, 0.013955978179666071, 0.015035028278388027, 0.014043020425499542, 0.014935968090068022, 0.01563917887336743, 0.017472585742079435, 0.016221428995488778, 0.015301039744395556, 0.015838108440624356, 0.016448865965760343, 0.014912481796656444, 0.015881719410107496, 0.015143132233341917, 0.015301549519370449, 0.016999041588513192, 0.015084993162205865, 0.014482107390322801, 0.016105050732445617, 0.016803619190062447, 0.015810270682826578, 0.016360997626783596, 0.014120377698390588, 0.016208422316326822, 0.016496674780448143, 0.016341531402072694, 0.01552153817091968, 0.016007432676496482, 0.015547027870248867, 0.01427891314423261, 0.014563668981769582, 0.01614752009684741, 0.01413472095273203, 0.016632266172667654, 0.015462842040599912, 0.015341315802862287, 0.015463995148951384, 0.014596155768219005, 0.016404806986237988, 0.016433513666166046, 0.01429032658173637, 0.015823297609726776, 0.015207294680676647, 0.017288990287717215, 0.015944991026819195, 0.016422994955325024, 0.017111836823938305, 0.015672682978006336 ], "mean_node_assignment_entropy": 4.1561465243867115, "normalized_node_entropy": 0.9993419966567684, "effective_mass_slots": 63.90193659366808, "node_slot_mutual_information_nats": 0.0012031431677144155, "normalized_node_slot_mutual_information": 0.00028929478025683765, "centered_token_effective_rank": 1.1890659972832214, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.007831680454927192, "token_RMS": 0.20049171342889924, "mean_token_offdiagonal_cosine": 0.9984628088242095, "temperature": 0.4993506669998169 }, { "layer": 2, "head": 3, "slices": 64, "native_nodes": 33555, "mass": [ 0.013554414824948033, 0.01647361289668729, 0.017042595547871093, 0.014593994681604208, 0.016351288052714585, 0.013971241156418932, 0.01481600844947561, 0.016063399016640657, 0.016128813794998205, 0.016264634963078915, 0.0165148918505138, 0.014566081782242009, 0.013771022144095, 0.0169598649108413, 0.01569764231259622, 0.017566720232636186, 0.014568590985383767, 0.016266097759819922, 0.015322533556215899, 0.014981046184256835, 0.01578979628626049, 0.014535902642355318, 0.014521770836762399, 0.015651575232185827, 0.01604510414729267, 0.016522141912239593, 0.015351258946499132, 0.017089601090940328, 0.015094511535125817, 0.016316158926294014, 0.01380195317323904, 0.013815821305771083, 0.015520871135737453, 0.013831986967657962, 0.015815848736651712, 0.01700622823170961, 0.01372769187498605, 0.016414902069483903, 0.015734519008305598, 0.014318673455741603, 0.016214050300504816, 0.015165643075475783, 0.016621982348624036, 0.01641832643884218, 0.01623920201972147, 0.01595340763566889, 0.015182829000368868, 0.016144931860708867, 0.014396130242732603, 0.015387759279088352, 0.017075284734432716, 0.016336666677863786, 0.015789807448919956, 0.016053647231552077, 0.01603851619607742, 0.014477624815118915, 0.01603691425627438, 0.01673812120822592, 0.014742752071289614, 0.015276063617938918, 0.01651310416102529, 0.015992402228245196, 0.014543112922243907, 0.018280907669646553 ], "mean_node_assignment_entropy": 4.155044554478499, "normalized_node_entropy": 0.9990770289031372, "effective_mass_slots": 63.85535867520002, "node_slot_mutual_information_nats": 0.0015759504800261226, "normalized_node_slot_mutual_information": 0.00037893599037004477, "centered_token_effective_rank": 1.2278045285045507, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.010459158125566637, "token_RMS": 0.2843383945543439, "mean_token_offdiagonal_cosine": 0.998653265801726, "temperature": 0.4992547631263733 }, { "layer": 2, "head": 4, "slices": 64, "native_nodes": 33555, "mass": [ 0.016565397561857657, 0.01635344241518317, 0.016071526293361522, 0.01710352792843539, 0.015378436740588287, 0.01501808599997026, 0.015565043384817945, 0.016287788869281575, 0.016934064030619523, 0.015417349971042599, 0.016206634543795038, 0.01554226742188913, 0.014991656133929755, 0.01644074493207869, 0.016170983778174303, 0.015100795387524532, 0.01626604885565659, 0.016495196308872372, 0.01512915189341188, 0.01671168804368538, 0.014955366621408159, 0.015216769079822504, 0.015122208482917659, 0.01449427828883813, 0.0158345135159745, 0.01573637802436373, 0.015199553884561389, 0.014972460576372101, 0.01579696991542087, 0.015872321111159663, 0.01609524897559356, 0.015241230209597424, 0.015136943751927496, 0.015916534891120857, 0.014650786007768243, 0.013639207799534403, 0.014799932847217208, 0.01589956443066827, 0.01558062314527558, 0.01431958572804426, 0.015698212044955525, 0.014539778860672507, 0.01600097334949124, 0.015454577390593577, 0.016285482721938638, 0.015560585021086714, 0.01490120115687019, 0.014529728916110233, 0.0159214210366649, 0.016730012921132107, 0.01638235593695627, 0.01511240207739069, 0.015129241558890099, 0.0157680904379495, 0.015781639760348742, 0.015334687428639136, 0.015152437369418409, 0.01681445768957028, 0.016911056923669322, 0.01596446939880194, 0.016253922044739046, 0.015074775043157232, 0.015362181577422397, 0.015106001605808895 ], "mean_node_assignment_entropy": 4.155761152027952, "normalized_node_entropy": 0.9992493341916221, "effective_mass_slots": 63.93345915146768, "node_slot_mutual_information_nats": 0.0020816897096453957, "normalized_node_slot_mutual_information": 0.0005005405701291665, "centered_token_effective_rank": 1.0919356354453251, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.010732855731303035, "token_RMS": 0.26554028793780343, "mean_token_offdiagonal_cosine": 0.9983552225563554, "temperature": 0.49767038226127625 }, { "layer": 2, "head": 5, "slices": 64, "native_nodes": 33555, "mass": [ 0.016641984009514922, 0.01535192721098275, 0.017624212072551864, 0.01630897636914876, 0.016392046643048858, 0.014487540836577408, 0.013878448999504488, 0.01599124687350648, 0.016382085016898498, 0.01869482804972952, 0.017296545565652344, 0.014806652447754705, 0.015802002652026317, 0.015973945745135, 0.017263054779717277, 0.015309239382424141, 0.014470573239535817, 0.01721750343699442, 0.014823288719608743, 0.01807708190713811, 0.016154323308863943, 0.014544556372177404, 0.01466056124372856, 0.015408857515204719, 0.015327327608986701, 0.01590909503243232, 0.015685735733252874, 0.01615876350227109, 0.013995952933061951, 0.016695080823100568, 0.016541379596219256, 0.015046169421567708, 0.014613805330746878, 0.015204124275282426, 0.01541163761389649, 0.013516578309331215, 0.015446484078045459, 0.015076283144224349, 0.015239645250257082, 0.01349556302437216, 0.01567955065644626, 0.015083134762033533, 0.016224122099492894, 0.016878224877026383, 0.016905065684274977, 0.01495622989254339, 0.01580787486889522, 0.013722994172621716, 0.014698792544861493, 0.015588673295941958, 0.016724892604483826, 0.016326742372616422, 0.014182470405326926, 0.01606017238029028, 0.015442332814290348, 0.014706947625814678, 0.016718310710820413, 0.014871092637935067, 0.01774529869636406, 0.015368702633670253, 0.015234702514800101, 0.015250019779984775, 0.014745684870496678, 0.014152858919219668 ], "mean_node_assignment_entropy": 4.155109774926409, "normalized_node_entropy": 0.9990927111059312, "effective_mass_slots": 63.83876559088294, "node_slot_mutual_information_nats": 0.0012508420457919556, "normalized_node_slot_mutual_information": 0.0003007639360665767, "centered_token_effective_rank": 1.2917198976649216, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.006909899444887489, "token_RMS": 0.24818584176718708, "mean_token_offdiagonal_cosine": 0.9992333162009972, "temperature": 0.5022552013397217 }, { "layer": 2, "head": 6, "slices": 64, "native_nodes": 33555, "mass": [ 0.015413959963226918, 0.01499429274835564, 0.01735450866894588, 0.014471446265286242, 0.01550762005274661, 0.016214555407431733, 0.013658083636897089, 0.015426286902057176, 0.015423068054178054, 0.016042796767234465, 0.015310999319388088, 0.015951441273713826, 0.015892337124576567, 0.015483161346436507, 0.015846623972223715, 0.01484081097701577, 0.015271018395726886, 0.014731440545760418, 0.01578873157453482, 0.015782441094494393, 0.015628687362335796, 0.015645799751186306, 0.014269259682799742, 0.014906773290753302, 0.016375691618816433, 0.015627309231018457, 0.015806911498915464, 0.015569690760208959, 0.017035557363210618, 0.015466586131124583, 0.01647403328320767, 0.016047966196153864, 0.01506351939984129, 0.015447687356082066, 0.017134898520824032, 0.015833113175382326, 0.016268036330476538, 0.016321810040189413, 0.01680912830453461, 0.01601439522058349, 0.014738671268366867, 0.0151371075876247, 0.016584534862836023, 0.015555664089152673, 0.01659923391348669, 0.015450325521213467, 0.015715244870409182, 0.01660205745144473, 0.015405224116316204, 0.015077864549967478, 0.01576070445403068, 0.015265984499846178, 0.015599359664885596, 0.016481123862208277, 0.014860464230200881, 0.01538634065512555, 0.01628886308993424, 0.014508869540406498, 0.014354832032219413, 0.014993297539689142, 0.015190974109245607, 0.016151080122449292, 0.015967377979819616, 0.015172321206665647 ], "mean_node_assignment_entropy": 4.156241598721058, "normalized_node_entropy": 0.9993648572018814, "effective_mass_slots": 63.933554255671055, "node_slot_mutual_information_nats": 0.001602730565217314, "normalized_node_slot_mutual_information": 0.00038537523972003075, "centered_token_effective_rank": 1.1896166863831616, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.009389458716392324, "token_RMS": 0.21382563511854982, "mean_token_offdiagonal_cosine": 0.9980973675050786, "temperature": 0.49897441267967224 }, { "layer": 2, "head": 7, "slices": 64, "native_nodes": 33555, "mass": [ 0.014868424318907354, 0.016986877245531808, 0.01643063502016792, 0.0167829766168451, 0.014500367175579523, 0.016379122036484096, 0.014694367867335622, 0.015742892740429504, 0.015836775480686172, 0.016060384640431566, 0.016357141198273157, 0.014574598872928928, 0.015537757417174605, 0.017708755317631893, 0.014556060935411008, 0.01473332630869236, 0.016182126446080723, 0.014808145223895673, 0.016516702909634848, 0.016603765995445007, 0.014797509316038026, 0.016240588346288894, 0.01565046369743017, 0.016133462945217215, 0.01612530848364697, 0.01569563889960106, 0.0150660162659773, 0.015015718813348884, 0.015619892446537111, 0.015378252477355267, 0.014969387737335534, 0.015662640043412356, 0.014659007657288777, 0.015069556751318589, 0.0168496808307219, 0.013975283867235316, 0.01506387690794666, 0.015796203477020335, 0.017110305820412588, 0.01607075846184896, 0.0173051707587016, 0.015215215630021105, 0.016162184706603633, 0.01447659200003265, 0.016053676250653133, 0.015352215014943209, 0.015437465005462251, 0.015928241761153997, 0.015340875506161257, 0.015892974181413527, 0.01676270141784058, 0.015263104660461768, 0.015109030225494192, 0.015417311086472457, 0.01579142652805592, 0.015708285272471145, 0.014385143816920732, 0.016710884387047986, 0.0152248716783131, 0.014797319925104723, 0.014056657211980784, 0.015190073350232664, 0.016220444182857965, 0.015387378626805165 ], "mean_node_assignment_entropy": 4.155403042012087, "normalized_node_entropy": 0.9991632269342918, "effective_mass_slots": 63.913257920453574, "node_slot_mutual_information_nats": 0.0021237770438977677, "normalized_node_slot_mutual_information": 0.0005106604348641886, "centered_token_effective_rank": 1.1271395416170835, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.010836305649226398, "token_RMS": 0.22337231132636431, "mean_token_offdiagonal_cosine": 0.9976656766057739, "temperature": 0.49729689955711365 }, { "layer": 2, "head": 8, "slices": 64, "native_nodes": 33555, "mass": [ 0.015073082754906084, 0.015671412778801515, 0.01591415632060407, 0.015980107629081343, 0.014373100047198063, 0.01379291073862819, 0.015450399053092775, 0.014995373627819003, 0.015491315194091566, 0.01759146760758931, 0.016233876182870765, 0.014875331135349291, 0.015119780009525567, 0.01851665373342596, 0.017664514331782944, 0.014909974874568055, 0.01443818141371145, 0.016525314020437307, 0.017269687697007163, 0.015653264587999028, 0.01691342253055334, 0.015549777515150407, 0.014218758189399308, 0.015217970431182934, 0.014676126369224222, 0.015374917808085826, 0.014896635546466532, 0.015027780280004554, 0.015321033253417589, 0.016454067984447405, 0.016116996621983654, 0.013419069097582887, 0.014758876200325175, 0.015441615835127506, 0.014748475150926966, 0.015400940107078697, 0.013981631605226323, 0.01659310234386117, 0.015717210427771477, 0.015892377977511057, 0.01656185374706939, 0.014618435621898854, 0.01687179030008162, 0.014880614195420345, 0.015644106689039542, 0.014663983223733044, 0.015010685128684531, 0.015806490200417814, 0.017518379693458694, 0.015134909104113428, 0.015951236508061715, 0.016813296857944297, 0.017209512653916455, 0.016409788173063423, 0.014517888891678722, 0.016007892531138112, 0.016070117096392646, 0.016980474944971808, 0.014986278916605183, 0.01395572468529065, 0.015603665457468082, 0.015446446471625692, 0.016149335402999232, 0.01592640508776174 ], "mean_node_assignment_entropy": 4.154986708316808, "normalized_node_entropy": 0.9990631198413695, "effective_mass_slots": 63.86471258482796, "node_slot_mutual_information_nats": 0.0017802718128061557, "normalized_node_slot_mutual_information": 0.000428064885961641, "centered_token_effective_rank": 1.1246274484887808, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.009384929303781004, "token_RMS": 0.19703318371471065, "mean_token_offdiagonal_cosine": 0.9977261520788385, "temperature": 0.4983305335044861 }, { "layer": 3, "head": 1, "slices": 64, "native_nodes": 33555, "mass": [ 0.01593902875776019, 0.016494602822167005, 0.015295207997246402, 0.016937315276867097, 0.015735925321491227, 0.014483574080050453, 0.015457643284192473, 0.01543692873080052, 0.014972622329853514, 0.015867034815564906, 0.015240353559141204, 0.014307779974090842, 0.01653402442887765, 0.015101856660614638, 0.015503188115095679, 0.01595473974785512, 0.015220632002528104, 0.015092881356334477, 0.01641249772876772, 0.016694685666526442, 0.015252566620243447, 0.014998573234082468, 0.01580290421071937, 0.01549342593957794, 0.014663689054943974, 0.016284950335267633, 0.015510896598661271, 0.015787790838732812, 0.014775791029415158, 0.015366202143550079, 0.015758209404272056, 0.01463301632325192, 0.015195997167534647, 0.015780086593344116, 0.01607480714403758, 0.01597041656516993, 0.01671925783604206, 0.015470520447789201, 0.015424836944439534, 0.015618345340914528, 0.016115897439915376, 0.015844973860464788, 0.017164730045246688, 0.0159882191970507, 0.015972961343945876, 0.015235092295487784, 0.01615095436792294, 0.015722582593472513, 0.016128447882730067, 0.015996813397472102, 0.015897441722120688, 0.015975120754235548, 0.016371980587453774, 0.015418356240812631, 0.014484798042662913, 0.01626945516601651, 0.01595897610845926, 0.015853843430392728, 0.01425571779935171, 0.0150542320256725, 0.014696505926621597, 0.015856227053088156, 0.015844342393672665, 0.014477523845985295 ], "mean_node_assignment_entropy": 4.156203207716099, "normalized_node_entropy": 0.9993556261164698, "effective_mass_slots": 63.94598482329763, "node_slot_mutual_information_nats": 0.0018355321499337052, "normalized_node_slot_mutual_information": 0.000441352188350269, "centered_token_effective_rank": 1.1807417062404442, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.010068877681762584, "token_RMS": 0.27612030271230076, "mean_token_offdiagonal_cosine": 0.9988206654254865, "temperature": 0.4950060546398163 }, { "layer": 3, "head": 2, "slices": 64, "native_nodes": 33555, "mass": [ 0.015731521946175577, 0.016342950854053566, 0.015324077061002079, 0.014963052379315176, 0.016794545616387806, 0.01471809040785009, 0.015004501324671557, 0.016314233014935432, 0.014354706055235887, 0.015653994052768454, 0.015254842927211746, 0.015025738399765022, 0.016073376252623325, 0.016707196282068827, 0.015238952714821617, 0.013910750224642348, 0.016462061326129793, 0.014113076869967335, 0.015541912738198833, 0.015415335588979838, 0.014965010441688707, 0.014517496306615151, 0.01643195805105007, 0.015349048418937257, 0.01576438129226873, 0.016925845089414457, 0.015950282879402457, 0.016201907696182693, 0.016077949817668056, 0.01726267307410007, 0.015695649914520098, 0.015420802910285546, 0.015529453752010355, 0.014703644400970705, 0.014686189096039649, 0.016681592759078573, 0.015149114364815032, 0.015262888653714262, 0.014793313208802582, 0.015443981349642533, 0.01460556362127915, 0.017033479154118836, 0.01577792362271274, 0.016732166530493652, 0.016293564470294175, 0.0164058512011511, 0.015363661180462812, 0.014901726938214588, 0.01607094755802304, 0.01586326883572626, 0.016193587020080637, 0.014789238765795795, 0.01577613081482191, 0.01566606538977811, 0.0164372948568296, 0.016489382179998246, 0.014074850114704404, 0.01570622386590408, 0.015713904751541052, 0.015914025343510482, 0.01529560623144543, 0.016045199422660625, 0.015848136255374665, 0.015244102548858374 ], "mean_node_assignment_entropy": 4.155640286898904, "normalized_node_entropy": 0.9992202722712397, "effective_mass_slots": 63.92461885064374, "node_slot_mutual_information_nats": 0.0020642718146381966, "normalized_node_slot_mutual_information": 0.000496352451670898, "centered_token_effective_rank": 1.1203162572931051, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.010919998564061378, "token_RMS": 0.20152249676464698, "mean_token_offdiagonal_cosine": 0.9970361637141772, "temperature": 0.49846160411834717 }, { "layer": 3, "head": 3, "slices": 64, "native_nodes": 33555, "mass": [ 0.01518398783756846, 0.014865248191076444, 0.0157565616948008, 0.01506891255450842, 0.01563957130641629, 0.015522957645909536, 0.016319426522880123, 0.01528080720462781, 0.01624930534220931, 0.0152926618966613, 0.015847799073276932, 0.0152603668833104, 0.016019194350706815, 0.015572502818405451, 0.015459671821525717, 0.015217852089018397, 0.015219496288884284, 0.015672153979401096, 0.014660882047094285, 0.015405749344945424, 0.015746422894043682, 0.016146330345372627, 0.015126771368102991, 0.01698961730165491, 0.015961449340623105, 0.01618992551026588, 0.016325442509503928, 0.016337257348896937, 0.015622601395487039, 0.015185826551682626, 0.015905551048865253, 0.01584978835789631, 0.015418761495499138, 0.015303919538372613, 0.015704417187232603, 0.01576919171727684, 0.01449483169198054, 0.0156913337176309, 0.015122162042130986, 0.01637896181527881, 0.015096689559099635, 0.015732959530428286, 0.0151876402455126, 0.0159046640477807, 0.01539111352375635, 0.015188155473172309, 0.015067689313667487, 0.015848138890555426, 0.014232733151055967, 0.016191615591569793, 0.01483123153634064, 0.015464876771457255, 0.0160902392517455, 0.016115390808039684, 0.016627739917644, 0.015104527671721081, 0.015253461452236813, 0.015245328801877673, 0.01606248932389753, 0.016133719042077884, 0.015872722953855504, 0.016275536798877742, 0.01581256643893546, 0.016483097847130895 ], "mean_node_assignment_entropy": 4.156586434886352, "normalized_node_entropy": 0.9994477727728127, "effective_mass_slots": 63.962874185093916, "node_slot_mutual_information_nats": 0.00171638929761464, "normalized_node_slot_mutual_information": 0.0004127043879839221, "centered_token_effective_rank": 1.1243948505376329, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.010028666217036582, "token_RMS": 0.1670340136343344, "mean_token_offdiagonal_cosine": 0.9963946751922099, "temperature": 0.4965437352657318 }, { "layer": 3, "head": 4, "slices": 64, "native_nodes": 33555, "mass": [ 0.015054652586716987, 0.015668906110533568, 0.01658941637305536, 0.015229396909284371, 0.015029790491196548, 0.01636565286824914, 0.016396698498696804, 0.014757550560712877, 0.01644147400439092, 0.014307201975705931, 0.016967325289739953, 0.015512141669475761, 0.015260143736534174, 0.016205090779526376, 0.01623077932617545, 0.01606427558366503, 0.016282211855245966, 0.01682819722731273, 0.014479384992414322, 0.015199860450862338, 0.015723601911334102, 0.01676538866848717, 0.015631058352600133, 0.01536776709820364, 0.016246571019486164, 0.014592915724959065, 0.014381878576965148, 0.014799207125251552, 0.014908820019310223, 0.015569056196307846, 0.014714975031199621, 0.015083815916238075, 0.014711307396027303, 0.015170057576564363, 0.015783971788661703, 0.014620267173114094, 0.01545024127908292, 0.01547068230904369, 0.015932579437414253, 0.01610584165814905, 0.016554700150878246, 0.014838488587468193, 0.015197134497386033, 0.014400956381404876, 0.014739966277749728, 0.016171468891044646, 0.015250869699609633, 0.01749711333478577, 0.015391824929055586, 0.015669666376781435, 0.01486039783257723, 0.01703325535382089, 0.015464903013049366, 0.015410234459769244, 0.016366992340201263, 0.016111002690677112, 0.017757459845315626, 0.014637245331611804, 0.015329648834623916, 0.014865669364565097, 0.015948443187642446, 0.0158340271122697, 0.01578684939051275, 0.01698152655989571 ], "mean_node_assignment_entropy": 4.1560720966477, "normalized_node_entropy": 0.9993241005684389, "effective_mass_slots": 63.91664701930648, "node_slot_mutual_information_nats": 0.0015077475412583752, "normalized_node_slot_mutual_information": 0.00036253665011433097, "centered_token_effective_rank": 1.1752381166577397, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.009440933159654202, "token_RMS": 0.22340703566901893, "mean_token_offdiagonal_cosine": 0.998197958596965, "temperature": 0.4980366826057434 }, { "layer": 3, "head": 5, "slices": 64, "native_nodes": 33555, "mass": [ 0.01563430572262514, 0.015760529975055306, 0.015231223163437673, 0.01630102190168386, 0.015426281022693118, 0.0158208315170643, 0.015813034748198635, 0.01572261794037687, 0.017683364254675198, 0.014665763013063752, 0.015824446925537326, 0.01667139186132369, 0.015131790165150642, 0.015763867427856553, 0.017076363828521145, 0.016743301863197446, 0.014297795218195321, 0.01553628014955981, 0.015499227675158964, 0.014902846196827618, 0.015643722681691414, 0.016153191068910385, 0.016192629791833956, 0.015535685853598824, 0.014938917613308364, 0.01544640576262918, 0.015719073984842465, 0.014797524334742093, 0.01415368350242968, 0.014956100467321033, 0.015137022640125893, 0.016905472919149508, 0.015774173537223993, 0.017433513948953937, 0.0156702803047614, 0.015655981753126394, 0.015785944769432347, 0.015344076778230662, 0.015755567962900216, 0.014580139997669897, 0.01566462441496168, 0.014609379284616742, 0.016200567144612463, 0.01637104493771939, 0.016391119386874792, 0.015688112690282096, 0.015695798526791573, 0.015161896934403897, 0.01560245807805949, 0.016966673778293695, 0.014469950887742474, 0.015488718113927342, 0.015224963580020191, 0.015130251681041058, 0.015514479161325785, 0.01501808533368119, 0.0155029607184299, 0.016570301199640793, 0.014565123517952739, 0.014916224814418521, 0.015879754609349383, 0.01493284379758941, 0.015619227829740515, 0.015730045348180465 ], "mean_node_assignment_entropy": 4.15661905321029, "normalized_node_entropy": 0.999455615821844, "effective_mass_slots": 63.9318389560276, "node_slot_mutual_information_nats": 0.001198446304450762, "normalized_node_slot_mutual_information": 0.00028816542336713654, "centered_token_effective_rank": 1.1745466445111605, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.007918773447372303, "token_RMS": 0.2163858170143318, "mean_token_offdiagonal_cosine": 0.9986852635505735, "temperature": 0.5015000104904175 }, { "layer": 3, "head": 6, "slices": 64, "native_nodes": 33555, "mass": [ 0.016806318939471543, 0.01602566274988471, 0.017143858308612307, 0.016966472087796284, 0.015627929463293146, 0.0157375129745201, 0.016818190444743218, 0.015732972468193294, 0.015310664141680832, 0.015932650169497267, 0.015945192066320602, 0.014885450240972322, 0.014645673813651667, 0.015457629267891808, 0.016654055366036417, 0.015881902205206442, 0.015542685192546302, 0.016344764309689223, 0.01379097399059386, 0.014803143279415194, 0.015630365537531977, 0.014707517571222177, 0.016661662588068348, 0.016040006264846595, 0.015612932628084759, 0.016529120300439754, 0.014106282729235291, 0.015101967106756884, 0.013880476130736317, 0.01531023031946884, 0.013307231118603642, 0.01500935883011051, 0.015037354889292908, 0.015035804441400485, 0.0173639579896606, 0.01513705197011189, 0.01548327582722746, 0.015061256364504168, 0.01775972983919083, 0.01509463606443078, 0.016752057671663507, 0.014964049195918218, 0.01525361255583434, 0.014559990652999087, 0.01438120484319916, 0.014952682732765284, 0.01446766774224917, 0.01662875248066223, 0.01598420120004663, 0.01662448922072548, 0.014188510585216035, 0.017991725955923958, 0.014917040264810975, 0.015204806531475688, 0.016663843597187058, 0.01714682619585186, 0.01814936830144142, 0.01597270162090839, 0.015219185249709262, 0.01386928161160168, 0.01495816863966697, 0.015011224416468303, 0.01716449766401371, 0.015050191009638385 ], "mean_node_assignment_entropy": 4.1533576517006985, "normalized_node_entropy": 0.9986714145244715, "effective_mass_slots": 63.85424060404475, "node_slot_mutual_information_nats": 0.0032453436710513373, "normalized_node_slot_mutual_information": 0.0007803402033676913, "centered_token_effective_rank": 1.0597777230603433, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.018305605666858776, "token_RMS": 0.19714260590252564, "mean_token_offdiagonal_cosine": 0.991738303559669, "temperature": 0.4972086548805237 }, { "layer": 3, "head": 7, "slices": 64, "native_nodes": 33555, "mass": [ 0.015203713739826229, 0.015148430526997067, 0.016449642378790798, 0.014795734881916206, 0.016895320674790576, 0.014017066536292228, 0.015424184418405387, 0.01598230762630057, 0.015547249813341664, 0.015619783489123228, 0.014960144497309839, 0.015877394768033487, 0.016331299748690804, 0.015520950816926978, 0.014622573585740333, 0.01590538859689885, 0.01600084688040969, 0.016132064854870246, 0.014976488786450203, 0.014927276575388594, 0.0162859228005678, 0.015302171669189887, 0.01592556479102634, 0.017033854731547463, 0.014191709486777363, 0.0157146339639333, 0.017271323280883315, 0.017770959107912196, 0.014897603381049338, 0.015346570654947496, 0.015009623513290707, 0.014945844722257528, 0.015604647476491986, 0.016211589858837392, 0.015304735451914518, 0.016867491594182542, 0.014447498853348785, 0.01583409885452754, 0.015209362010211041, 0.015238643717617484, 0.015819569086652368, 0.01433830235556996, 0.014607010763224574, 0.017366998345897684, 0.016691564908494022, 0.016045997975883453, 0.014979566380047413, 0.015605393139640739, 0.014498065146317142, 0.016401748738532368, 0.014581252559059166, 0.016341331735366574, 0.015734134140771232, 0.01601434381873819, 0.015701686693354054, 0.016351355516661645, 0.01445753439542585, 0.01604635399950699, 0.01636396376145475, 0.015567299438330452, 0.015193201643137939, 0.0147033283273248, 0.01634137787126341, 0.015496905616646534 ], "mean_node_assignment_entropy": 4.155024709968104, "normalized_node_entropy": 0.9990722573070145, "effective_mass_slots": 63.91427655645659, "node_slot_mutual_information_nats": 0.002518046749624858, "normalized_node_slot_mutual_information": 0.0006054622597350594, "centered_token_effective_rank": 1.065459004311328, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.013715953088714416, "token_RMS": 0.2214526868288701, "mean_token_offdiagonal_cosine": 0.9962856536343937, "temperature": 0.49136456847190857 }, { "layer": 3, "head": 8, "slices": 64, "native_nodes": 33555, "mass": [ 0.014324006820748423, 0.014273469804392995, 0.01673307450472389, 0.015085841199784756, 0.015186877155321497, 0.015801625996866936, 0.01558533376275676, 0.015256329485709427, 0.016414608064921794, 0.015888784995107996, 0.01772447262767543, 0.014899019806780007, 0.01602193036038529, 0.01665983193178874, 0.016141859539138492, 0.016048057790889487, 0.014409394503930938, 0.014372400070285677, 0.014767106755931809, 0.01544747797256321, 0.0153097167194518, 0.015146821241610995, 0.015526729123812597, 0.01576277094611027, 0.01647567875467373, 0.01469399131753462, 0.015472841962190538, 0.015388885782172563, 0.01596135262720761, 0.016441823672150743, 0.015133630849322215, 0.01676875839556379, 0.01581074341561525, 0.015999143817182395, 0.01575058479599778, 0.014899363884189687, 0.01594890396322624, 0.015243806899084613, 0.015827554455403067, 0.015194717187513815, 0.015601799866669474, 0.01600137531980506, 0.015794574875940225, 0.015280833723021591, 0.01477630192378122, 0.01610975000666486, 0.014959841104222382, 0.01662501106012322, 0.015324859973501163, 0.015893803449185805, 0.015623920445565388, 0.015240574629058375, 0.015801949421480364, 0.015660429123171244, 0.01536577585865339, 0.015329476162977767, 0.01656673971689586, 0.015597448901153547, 0.015662119381413955, 0.015396407416312133, 0.016055022620248636, 0.015787689962134713, 0.015753682029940496, 0.01599128996264209 ], "mean_node_assignment_entropy": 4.156716345447453, "normalized_node_entropy": 0.999479009659856, "effective_mass_slots": 63.945809457179756, "node_slot_mutual_information_nats": 0.0013196520046685478, "normalized_node_slot_mutual_information": 0.0003173092338057489, "centered_token_effective_rank": 1.2685939236393344, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.008342382113707072, "token_RMS": 0.20595180652192036, "mean_token_offdiagonal_cosine": 0.9983919739470547, "temperature": 0.4992532730102539 }, { "layer": 4, "head": 1, "slices": 64, "native_nodes": 33555, "mass": [ 0.014564716333259441, 0.014920152376212026, 0.01940499302493848, 0.014099206943456274, 0.01512696283459086, 0.01444185876261569, 0.015412307907639158, 0.015166977654313778, 0.012657864773908283, 0.015512388702157251, 0.01363485830692643, 0.014046688656712755, 0.016213567631416032, 0.014521936163991233, 0.017619909331032645, 0.017918687375244893, 0.015849693195713676, 0.016954083407233617, 0.018064634606820597, 0.01356416538456543, 0.015288290705000603, 0.01741887228467673, 0.01631223782697159, 0.017484329045185244, 0.014697619138590546, 0.01833442689493163, 0.013903462644896801, 0.015350635922255086, 0.013571280079172552, 0.014699465407089404, 0.0149423440791208, 0.014624519570229786, 0.01536898237346799, 0.01632192332451327, 0.018894225931689622, 0.015662076573659606, 0.01753957272963683, 0.013978296508528254, 0.014049212300947075, 0.014277227643686868, 0.015489800959590152, 0.0141570871244023, 0.018643810767710937, 0.013340665705528679, 0.015311948789166785, 0.021910952843615772, 0.01576291104164427, 0.014194727458993469, 0.016233710042195294, 0.013605517255467898, 0.015997546763443685, 0.012728898952372463, 0.014402262711123957, 0.013280152316069065, 0.01735222947272658, 0.016973364474802806, 0.01420971909347789, 0.015258555275748462, 0.019116614566153635, 0.014374135572931225, 0.013526250944990343, 0.0180723340259017, 0.015446995521907106, 0.018195154060984473 ], "mean_node_assignment_entropy": 4.139818923093778, "normalized_node_entropy": 0.9954160384209473, "effective_mass_slots": 63.5612505822051, "node_slot_mutual_information_nats": 0.012185093997994834, "normalized_node_slot_mutual_information": 0.00292989578061217, "centered_token_effective_rank": 1.016292251656279, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.03081363789979084, "token_RMS": 0.20601437404235753, "mean_token_offdiagonal_cosine": 0.9776233260676555, "temperature": 0.48561009764671326 }, { "layer": 4, "head": 2, "slices": 64, "native_nodes": 33555, "mass": [ 0.013170842043422646, 0.023083122892227817, 0.017562134465642053, 0.017138894128533654, 0.013060504450173966, 0.018682064914293483, 0.018227252973077895, 0.02000432291294757, 0.012348920160982518, 0.0160461850746891, 0.017460796269354335, 0.014832410416723346, 0.016585533213269478, 0.013007666751483685, 0.013779885381667798, 0.022643171803757377, 0.02312118527703592, 0.018598073482793197, 0.01855299607737509, 0.016397586706712132, 0.01762035615831578, 0.012737462951769382, 0.014412501339502705, 0.015335151087534337, 0.0163358374839756, 0.01985860012347888, 0.012746733067424824, 0.014321381731408175, 0.01624310029374449, 0.014202401495641762, 0.014554567492953216, 0.01298717583011047, 0.013178701385036344, 0.017414440560323784, 0.01540539550395565, 0.013718797349804135, 0.013761333685055777, 0.012866238086323254, 0.013253525249113794, 0.013734156224971565, 0.021434422732654067, 0.015818246580952326, 0.013737703761025639, 0.013245135664688974, 0.018434591973129664, 0.014899203981418729, 0.01748291472859282, 0.016306106505671, 0.015502642731458025, 0.01381861302408238, 0.014070006609662442, 0.012767355298370038, 0.017318182787382643, 0.01308602575463068, 0.01316037314408712, 0.013425771898509955, 0.01361362924215078, 0.021446353429158217, 0.01543883675366036, 0.011996124612147772, 0.013468373955659443, 0.013580409080681609, 0.014074783888107715, 0.01288278551297347 ], "mean_node_assignment_entropy": 4.122742740474195, "normalized_node_entropy": 0.9913100844238495, "effective_mass_slots": 63.02347124858314, "node_slot_mutual_information_nats": 0.020764476035598278, "normalized_node_slot_mutual_information": 0.004992801100535894, "centered_token_effective_rank": 1.0364624405693939, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.031456904889991165, "token_RMS": 0.19425329647940967, "mean_token_offdiagonal_cosine": 0.9740520216929748, "temperature": 0.47903984785079956 }, { "layer": 4, "head": 3, "slices": 64, "native_nodes": 33555, "mass": [ 0.013314496592156819, 0.017733759210934726, 0.016399590310357114, 0.014598566155218453, 0.013994131898117516, 0.016765213678928872, 0.01712331044572447, 0.017475749576951954, 0.015084889749568241, 0.014785406622531081, 0.014500982869339317, 0.016309172746125453, 0.013638609667151674, 0.01697308889024971, 0.016125187743247008, 0.018177583862960364, 0.01872384115845973, 0.016667476477388154, 0.017116864096534855, 0.01366327852679625, 0.017290048242601157, 0.016625217650201703, 0.015488771897355005, 0.014496474413859286, 0.015910775242466327, 0.014395092806504488, 0.013888214405799806, 0.015769714836740063, 0.016347946464707885, 0.016546191871743537, 0.014892206401034752, 0.01461337415812055, 0.013204192924140472, 0.015134650689895711, 0.01671151050421, 0.01429591929161736, 0.015031196318945379, 0.016190165281351285, 0.01655770015588213, 0.01524147853647174, 0.015715887364304834, 0.01695233025828038, 0.014817562255763587, 0.01566601872717263, 0.015053066253249113, 0.015041084442664597, 0.017067839985294538, 0.01627489809599648, 0.016182609141120533, 0.015522495205546745, 0.016076868408887075, 0.015465176652885463, 0.016449134974462754, 0.014742090501198735, 0.014945295757461839, 0.01589207279163805, 0.013820755939290644, 0.015976563000568884, 0.01717287848531898, 0.014391136824315314, 0.012750899681279963, 0.014260236612149054, 0.014069457519219388, 0.01789159907474659 ], "mean_node_assignment_entropy": 4.154202181459002, "normalized_node_entropy": 0.9988744810068361, "effective_mass_slots": 63.775530219753286, "node_slot_mutual_information_nats": 0.00116739644655528, "normalized_node_slot_mutual_information": 0.00028069951069945005, "centered_token_effective_rank": 1.5584371996502062, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.00619030604637498, "token_RMS": 0.22107590257921858, "mean_token_offdiagonal_cosine": 0.9992603893056696, "temperature": 0.4979887008666992 }, { "layer": 4, "head": 4, "slices": 64, "native_nodes": 33555, "mass": [ 0.01862136667294281, 0.014573225845137174, 0.015555670731976426, 0.015414485308705135, 0.016810635679944487, 0.015247603853898865, 0.01398775837929691, 0.014864711727957544, 0.016627033260049898, 0.016611732501270713, 0.014826545929429367, 0.012976828012618218, 0.01742446898166193, 0.01631139070156425, 0.014431874097368345, 0.014104368959236715, 0.01533682771710419, 0.014563265041981212, 0.015723551241141144, 0.015168167163569753, 0.01500605965060096, 0.01767595072037225, 0.01608055759506638, 0.016184428498934943, 0.014059011835836384, 0.015814296950463602, 0.017541913079846424, 0.016411301351871636, 0.014891920022076935, 0.014336690961406459, 0.015157025154180976, 0.016700887271942636, 0.016238628530042473, 0.015624118334917798, 0.01465835333086418, 0.015769022035660482, 0.015981690676094895, 0.016005208941584255, 0.015006379246730605, 0.017315993426169747, 0.013774485334519738, 0.014293205783225184, 0.015902058263976115, 0.017637177741185835, 0.014881638262097268, 0.015094601238711796, 0.012794422896501472, 0.015559888297822113, 0.014358199766784742, 0.01617700157934366, 0.01654328342852996, 0.015447963225569078, 0.013373913552133012, 0.018161661275210766, 0.015811304420020725, 0.01601023846933866, 0.014981563851812383, 0.014610977412866615, 0.015772182784553807, 0.018875631026761566, 0.016782819521447584, 0.018101994067706937, 0.014114184558947784, 0.015298653325761791 ], "mean_node_assignment_entropy": 4.146757641936248, "normalized_node_entropy": 0.9970844476316396, "effective_mass_slots": 63.77989367612694, "node_slot_mutual_information_nats": 0.008680352603303909, "normalized_node_slot_mutual_information": 0.002087183608992359, "centered_token_effective_rank": 1.0387395937336776, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.023150023305156937, "token_RMS": 0.2373003477951989, "mean_token_offdiagonal_cosine": 0.9925741492433614, "temperature": 0.4810405671596527 }, { "layer": 4, "head": 5, "slices": 64, "native_nodes": 33555, "mass": [ 0.016956537879665952, 0.013392472617309293, 0.01587930759269316, 0.014970081640625875, 0.016017122508830587, 0.016999824355381235, 0.015101159500484672, 0.014899375422764306, 0.016143465327103684, 0.01617569608155612, 0.014324472301794074, 0.01402229804372449, 0.01596660655440943, 0.01656200680090718, 0.015725929702358324, 0.01359765668091689, 0.013835466512306945, 0.01445017101151824, 0.014387598593549026, 0.0161177061108761, 0.014859395636507339, 0.016632184725226463, 0.013800798832477604, 0.018342392508652727, 0.013890484910646506, 0.017883663862580612, 0.017449568642291968, 0.015662076180369758, 0.015366264235135598, 0.013360773893152303, 0.014460009004020901, 0.016811004481741176, 0.016008760889923007, 0.015274365816080496, 0.016376912865258733, 0.015849426859578155, 0.016366965070698812, 0.01611465791593102, 0.015914210171559903, 0.01786477621111837, 0.015189940953409906, 0.014030640071329835, 0.01658946117151691, 0.01619120176441356, 0.014588874741759239, 0.015821230414194032, 0.014610786365175536, 0.015813472126949883, 0.014807256326915352, 0.015042182197287788, 0.015176898358030861, 0.014321300120322879, 0.014622923885650445, 0.017165619708351194, 0.01684562659487539, 0.015686608955898018, 0.01745406543199573, 0.014931535450712506, 0.01557374953251649, 0.016547841907231433, 0.018099563631085194, 0.016774124343455005, 0.015250912525432554, 0.015050535050744388 ], "mean_node_assignment_entropy": 4.150351754266725, "normalized_node_entropy": 0.9979486489709024, "effective_mass_slots": 63.81336822856143, "node_slot_mutual_information_nats": 0.005610947498001728, "normalized_node_slot_mutual_information": 0.0013491476883425717, "centered_token_effective_rank": 1.0550900494354125, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.018855334175936243, "token_RMS": 0.20395779017318963, "mean_token_offdiagonal_cosine": 0.9914578895980652, "temperature": 0.49708953499794006 }, { "layer": 4, "head": 6, "slices": 64, "native_nodes": 33555, "mass": [ 0.014309009388610976, 0.014224509991185842, 0.016806315387844942, 0.015068011223296218, 0.014983113300021405, 0.014722110555375267, 0.016152388194522894, 0.014322474558778454, 0.014598711119025764, 0.014411005485098464, 0.014487576070601752, 0.01660567264995038, 0.014782543440623205, 0.016624885331436873, 0.016639032569229426, 0.014720837955215022, 0.015833521135914096, 0.014730724656235273, 0.015516364794946688, 0.01610672161137959, 0.01525856552842903, 0.01620404771336934, 0.016399423924465282, 0.015187650415649482, 0.015768078548467082, 0.014435375647208868, 0.015962845870661293, 0.015516680756128904, 0.015052837806302377, 0.016741959839857223, 0.015328040922144152, 0.015366354676618819, 0.015202580821315847, 0.015274602980128815, 0.017412129452505077, 0.015133369100245273, 0.014971715010727034, 0.016224013878310947, 0.016254022675704458, 0.014947304207731148, 0.014974001730273065, 0.01733167493950968, 0.015606613318233096, 0.016069614523534642, 0.015526566168063267, 0.017295757283761513, 0.015849993515950105, 0.015997128938997116, 0.016574697669014288, 0.015034212640565344, 0.015154635580985152, 0.015310209288870084, 0.015379240084148087, 0.01533549565865184, 0.0159035554890385, 0.015115958998667254, 0.014774367625579057, 0.016478744641583644, 0.01589163877525134, 0.01592220393234439, 0.01540901273410729, 0.016277879095247597, 0.016258122457707316, 0.018241545352948803 ], "mean_node_assignment_entropy": 4.1553961015694965, "normalized_node_entropy": 0.9991615581106074, "effective_mass_slots": 63.907042228133825, "node_slot_mutual_information_nats": 0.0020334607546947936, "normalized_node_slot_mutual_information": 0.000488943957773418, "centered_token_effective_rank": 1.1931839237791446, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.00930415584094395, "token_RMS": 0.2124714309952296, "mean_token_offdiagonal_cosine": 0.998084637023086, "temperature": 0.4996194541454315 }, { "layer": 4, "head": 7, "slices": 64, "native_nodes": 33555, "mass": [ 0.01596916081319155, 0.01593248533497707, 0.013538342275168409, 0.015326880540178636, 0.015162565779330043, 0.014245417749562228, 0.0159069300306823, 0.01479437536977852, 0.01707500158619697, 0.016468211022087554, 0.014629258717961344, 0.01705339114872661, 0.015419645923435591, 0.017100436031188906, 0.014151460678523289, 0.015559827396875121, 0.015683518953492605, 0.014278928076979114, 0.014510154455174118, 0.014238263512699974, 0.014941027279065612, 0.016352168583945205, 0.016231746256580172, 0.014491128887508784, 0.01621230319844546, 0.01360860060369772, 0.01719301395621159, 0.014400349730092018, 0.01571382931779863, 0.017850625312899205, 0.015895039230088837, 0.015271741578017096, 0.015239721079010168, 0.014938015548362184, 0.013913176361165553, 0.0158173435995048, 0.015839587735416567, 0.01648423101016376, 0.016734492382851494, 0.01712093399797368, 0.015045118066292818, 0.015192036507971782, 0.015119129264314517, 0.016840108027097463, 0.015276401933166598, 0.01466113650881841, 0.01593548594873369, 0.01555456962922707, 0.014572180515134935, 0.016995483082031516, 0.01516781682163841, 0.017854824500007718, 0.015760823913643533, 0.016437360178526574, 0.01573021871054018, 0.015762430833267654, 0.01573564991978877, 0.015074184827751897, 0.015761641602041184, 0.017576864506894448, 0.014307500400046596, 0.015100856594943401, 0.01615811247112694, 0.017086734437007492 ], "mean_node_assignment_entropy": 4.152293261357473, "normalized_node_entropy": 0.9984154827461812, "effective_mass_slots": 63.86190498795943, "node_slot_mutual_information_nats": 0.004429756175740707, "normalized_node_slot_mutual_information": 0.0010651312111813963, "centered_token_effective_rank": 1.0630998900546513, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.020863914330638725, "token_RMS": 0.297679377796353, "mean_token_offdiagonal_cosine": 0.9955322952225499, "temperature": 0.4899030029773712 }, { "layer": 4, "head": 8, "slices": 64, "native_nodes": 33555, "mass": [ 0.014942697777948922, 0.016121099393794264, 0.01580417189261936, 0.014818568456700956, 0.014685557399740555, 0.014971183086261807, 0.016150524292428185, 0.015948119451228163, 0.015636634392376017, 0.014511927462211666, 0.015402976205874185, 0.015510206944483339, 0.014961389065645895, 0.015580011623357985, 0.01589507869146559, 0.016591842972200666, 0.01666136770697224, 0.01607548603930957, 0.015703611897457028, 0.015294204748203309, 0.01613103117512917, 0.01567820188878186, 0.016048235483101213, 0.015750532979102626, 0.017228322289412255, 0.014836125658813105, 0.016067521255577158, 0.016139358333908015, 0.01572254046485785, 0.017566373774310265, 0.015072537676241532, 0.014623423453463144, 0.01638189815775819, 0.01536510311890876, 0.015662225893214445, 0.015522325997515756, 0.01630070344528988, 0.015038538195588046, 0.015310290452709605, 0.015411401166901328, 0.015227908609486864, 0.01631984427763362, 0.015714390554819647, 0.01575241634007996, 0.015939633537768304, 0.015743740930217395, 0.01574461489742101, 0.015193955869930297, 0.015503164511210045, 0.015649713562906117, 0.0157466047273727, 0.015805191919873694, 0.016132571935851305, 0.014395849871006618, 0.01485043576142926, 0.01557972790039726, 0.015353013071155356, 0.015262116397788043, 0.015327305836105396, 0.014402087260220289, 0.016734893398861507, 0.015684465594970512, 0.01592583072884834, 0.014887172372845455 ], "mean_node_assignment_entropy": 4.155606584517753, "normalized_node_entropy": 0.9992121685615476, "effective_mass_slots": 63.948871101412244, "node_slot_mutual_information_nats": 0.0024772905195771244, "normalized_node_slot_mutual_information": 0.0005956624579058602, "centered_token_effective_rank": 1.1251833122059154, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.012420901948852203, "token_RMS": 0.1722235748544519, "mean_token_offdiagonal_cosine": 0.9947668804206774, "temperature": 0.4996313750743866 }, { "layer": 5, "head": 1, "slices": 64, "native_nodes": 33555, "mass": [ 0.020940610825390428, 0.017298146101074788, 0.02220943120654188, 0.012767032124274198, 0.01345144131373114, 0.022847898125358736, 0.013492074737669751, 0.0120517936209751, 0.023072102787218753, 0.017511859084068042, 0.01492791835142619, 0.01394630541755468, 0.01525631429647799, 0.010689319032915364, 0.01347299055518998, 0.021409007780461495, 0.021860185487026822, 0.01278272600279131, 0.018825233786367853, 0.01517291605403732, 0.009557314360293076, 0.011120054385074025, 0.011867557382126435, 0.01071096562288203, 0.014880718460500533, 0.015494554251767701, 0.010739007839166164, 0.025060573717957547, 0.012636127816203711, 0.023550596102797947, 0.012216927039818212, 0.024028922994904892, 0.015345507335445686, 0.013669739309418017, 0.015528406034985752, 0.013238478261650479, 0.01278148127614013, 0.011443982919950925, 0.022941230313366437, 0.01150939604590825, 0.01678060895069876, 0.015893677851642328, 0.009818986989502234, 0.018144737482248555, 0.01678893641042622, 0.014846654497742022, 0.021158078492255088, 0.013103887704029656, 0.012284811502321167, 0.016058223817212725, 0.01664624071666548, 0.017611455596842438, 0.010215405781155654, 0.011444021580037773, 0.026049645106493984, 0.013298998415140392, 0.017819805368209336, 0.010698407205692383, 0.012910759641819846, 0.015109584129172179, 0.015398247871226534, 0.01476873126127422, 0.019730721999499896, 0.011112523323302164 ], "mean_node_assignment_entropy": 4.1140530735442296, "normalized_node_entropy": 0.9892206611927097, "effective_mass_slots": 61.800182601233466, "node_slot_mutual_information_nats": 0.009853245627565954, "normalized_node_slot_mutual_information": 0.0023692047672583777, "centered_token_effective_rank": 1.0612117135740446, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.016952866559154112, "token_RMS": 0.22817314278611428, "mean_token_offdiagonal_cosine": 0.9944181582385136, "temperature": 0.47326600551605225 }, { "layer": 5, "head": 2, "slices": 64, "native_nodes": 33555, "mass": [ 0.02084715520390485, 0.015254705813020188, 0.031403845517516286, 0.01230320355589451, 0.01198938567765338, 0.02828842353185653, 0.010928966479733722, 0.009101500188979839, 0.0270818877971318, 0.015009545538790392, 0.0118725455478622, 0.013864275325362248, 0.020968166971669398, 0.00898405507039276, 0.010653874714323296, 0.022956884578408818, 0.02643872354512127, 0.012192392778947712, 0.02001471219790011, 0.015696160327279565, 0.008115732511653975, 0.010261914803381582, 0.011127055333873152, 0.008681805502840364, 0.01159850153738985, 0.014298247417895506, 0.009033927294206262, 0.02724332466506155, 0.013093896709813294, 0.020415037553665836, 0.009724993963347424, 0.03106363683807421, 0.015886600208019797, 0.010502582657019225, 0.017129494741030572, 0.010711767542646536, 0.013703427847040168, 0.009726050917187966, 0.030356899286783937, 0.00922485391172644, 0.017339711923847752, 0.018413847384494475, 0.008948777218765188, 0.014860904107105935, 0.016163954675428523, 0.013994758147449346, 0.02205240135095684, 0.011204013398860548, 0.010698100896292695, 0.011747136236462787, 0.014598335587226534, 0.02118516735512735, 0.008236792318399697, 0.008539451886942096, 0.03591805468171151, 0.011274510955918459, 0.017208907025212716, 0.008465165666061255, 0.01470130263108745, 0.013435540150369739, 0.014309075584288478, 0.016269268189805165, 0.023223067614545616, 0.009461593823214612 ], "mean_node_assignment_entropy": 4.0465326660077405, "normalized_node_entropy": 0.9729854350074273, "effective_mass_slots": 58.889649308021106, "node_slot_mutual_information_nats": 0.029132675896555327, "normalized_node_slot_mutual_information": 0.007004927840630968, "centered_token_effective_rank": 1.0143294957622908, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.0402411684210453, "token_RMS": 0.18143649638764997, "mean_token_offdiagonal_cosine": 0.9518414948467193, "temperature": 0.4751253128051758 }, { "layer": 5, "head": 3, "slices": 64, "native_nodes": 33555, "mass": [ 0.016590215857628515, 0.01832746768575936, 0.019495687562838527, 0.013033722470815757, 0.015312568638532035, 0.018312536914724783, 0.013898481829989784, 0.012880583582999693, 0.015285628118151623, 0.01597682170452215, 0.0166015842799311, 0.01759245622480676, 0.014694188149028928, 0.014726428931012512, 0.016228834364905725, 0.01697382096263908, 0.01416013322936016, 0.014095436445502735, 0.015205910730029543, 0.017359539994898077, 0.012566084889721416, 0.013649013223363283, 0.01519809805263112, 0.011961051402329032, 0.017545707048599864, 0.014845796576264655, 0.015083258762464956, 0.01540420324242012, 0.014827640528990906, 0.019042332626983575, 0.016455672151958502, 0.017077302850750674, 0.014779686782525187, 0.019522013680265852, 0.01496961584328304, 0.014272690218149052, 0.018691324404406264, 0.012160755866438243, 0.01630972476403829, 0.01665208845974822, 0.01571809612136851, 0.015965208934983335, 0.012035425848628304, 0.018415726980047623, 0.015008894046744953, 0.014436632064726932, 0.016652779058876487, 0.015683684258489598, 0.015454184852352868, 0.018728672165932662, 0.0166929923908319, 0.01845786241305743, 0.014180529029656053, 0.01383869540214987, 0.015235698227860136, 0.014827490485777195, 0.018358181941476606, 0.014266945074423826, 0.015310132774010783, 0.015432416885894727, 0.013841955069934908, 0.014203047468445638, 0.014585052015996205, 0.01490358704654715 ], "mean_node_assignment_entropy": 4.1495534862594345, "normalized_node_entropy": 0.9977567060883327, "effective_mass_slots": 63.55910743059462, "node_slot_mutual_information_nats": 0.0024168123682910547, "normalized_node_slot_mutual_information": 0.000581120536415436, "centered_token_effective_rank": 1.186893233522367, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.0104260890113598, "token_RMS": 0.1642824826232821, "mean_token_offdiagonal_cosine": 0.9960141027877969, "temperature": 0.4870678782463074 }, { "layer": 5, "head": 4, "slices": 64, "native_nodes": 33555, "mass": [ 0.015408487667401357, 0.014071223643878864, 0.016629665932834433, 0.015253544280466762, 0.01584294779469596, 0.015319610259968483, 0.013706105755873256, 0.0173124497446615, 0.015330251961510177, 0.016288934015609116, 0.016064875970844796, 0.015028528347744652, 0.01681154167309854, 0.01655216308766089, 0.016606146485948995, 0.01588402527579162, 0.014578700075077285, 0.016578810596057286, 0.016393327669020852, 0.015187916322940864, 0.015806232931625558, 0.016036827362754087, 0.015044600413919312, 0.01570186246313613, 0.01492462664692173, 0.015110412433287182, 0.016696848243673686, 0.014655189803740276, 0.016530163467653132, 0.015525293150160924, 0.015195610984491532, 0.016466043969318483, 0.01503917885711937, 0.014841852269760323, 0.015531975389744811, 0.01608004042110763, 0.01671726298552479, 0.014321718449995639, 0.01574084174860507, 0.018128367323129193, 0.014939480343665093, 0.016398317012849835, 0.014485024842566162, 0.013901887444445938, 0.01573128911827968, 0.014667206914295268, 0.014956251681827962, 0.014619828456538994, 0.015912021013153567, 0.015565443133335544, 0.015107982374939774, 0.015529713423090282, 0.015403827444504935, 0.015204946723347662, 0.015163528287569273, 0.016529535817049277, 0.014869191329701437, 0.015367723459794489, 0.018093909606650624, 0.013825861847791679, 0.015058194086562901, 0.014663242996560534, 0.01712535828404154, 0.0179360284891483 ], "mean_node_assignment_entropy": 4.15517175785425, "normalized_node_entropy": 0.9991076148497006, "effective_mass_slots": 63.876126909724164, "node_slot_mutual_information_nats": 0.0017739329362891354, "normalized_node_slot_mutual_information": 0.00042654070834232224, "centered_token_effective_rank": 1.1197685871356118, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.009043860196068926, "token_RMS": 0.21411952199753975, "mean_token_offdiagonal_cosine": 0.9982803359706054, "temperature": 0.4987984597682953 }, { "layer": 5, "head": 5, "slices": 64, "native_nodes": 33555, "mass": [ 0.016452762477694966, 0.0160114901949116, 0.01627644578042404, 0.01578867103634706, 0.01363845122235642, 0.018193826738663373, 0.015293530184809452, 0.015075888419769468, 0.0175478996710932, 0.015428699944248819, 0.01614002013852954, 0.016655909758926574, 0.015467088203391497, 0.016090049556755012, 0.014850781156484832, 0.015281412141591049, 0.015526368349902704, 0.015270270543449673, 0.013530617035762188, 0.015018396660593392, 0.014531318311271583, 0.014193131919240272, 0.016645954309092152, 0.014199997788186073, 0.014439595371097295, 0.016960024302557494, 0.014739658081335906, 0.01712671839166502, 0.015670887706960887, 0.015661564451296645, 0.014364821620195584, 0.014944304806678363, 0.017751065922440024, 0.01663113361910916, 0.01569154083766537, 0.01436102018190481, 0.016897606943676955, 0.014834722705632826, 0.014601919774647185, 0.01576881406157365, 0.01560421551859045, 0.016653101259795054, 0.015003146119765474, 0.0165510313511572, 0.014853689001825675, 0.017238180939263006, 0.01441178291191191, 0.015107575984070259, 0.015357470081880691, 0.01696008639342138, 0.01466415176442027, 0.015860775051572705, 0.016204374439591374, 0.015228013382069161, 0.01619242164105812, 0.015086751133525466, 0.015370197009199192, 0.015163472356173973, 0.01632419665044443, 0.015066719079933335, 0.01630466994016945, 0.01625177299634628, 0.01595639030222472, 0.015031434331384017 ], "mean_node_assignment_entropy": 4.15594381823953, "normalized_node_entropy": 0.999293256131219, "effective_mass_slots": 63.8741719745514, "node_slot_mutual_information_nats": 0.0009712669838988219, "normalized_node_slot_mutual_information": 0.0002335403435083352, "centered_token_effective_rank": 1.4062127495759607, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.005858523275580894, "token_RMS": 0.19491453989944113, "mean_token_offdiagonal_cosine": 0.9991767618352222, "temperature": 0.498394638299942 }, { "layer": 5, "head": 6, "slices": 64, "native_nodes": 33555, "mass": [ 0.01352478245670408, 0.015726085726214475, 0.014835527080294805, 0.015038671516711339, 0.017839848274408644, 0.013592332849011764, 0.015025817930771668, 0.01837699269650235, 0.014833378792142143, 0.013609462897547922, 0.014570277347753298, 0.016774970110083387, 0.014912652952818309, 0.015374482776944691, 0.01685850417418809, 0.014040799760635075, 0.01819138981975341, 0.01577778675112856, 0.016219612817077244, 0.016165579649665692, 0.018606563348996446, 0.015616360284757764, 0.014166393154304218, 0.014910674779955533, 0.018002991167340783, 0.013602774020302733, 0.017944162412967205, 0.013914381770294694, 0.016003438257343756, 0.013957732676351734, 0.01798872578347352, 0.013839697300435232, 0.014782377641946143, 0.01481329290169377, 0.015821443097961498, 0.020573588528683833, 0.016450027554018268, 0.018648081097678026, 0.015538448336124952, 0.015504739284482075, 0.012956956782628192, 0.01762519421097823, 0.015536763421128677, 0.0165595967099992, 0.016080470446088093, 0.014120552469738246, 0.017311276542779152, 0.016802004770092825, 0.015319320095269748, 0.015476841371045163, 0.01620668641775469, 0.013928617015752995, 0.01642914060589285, 0.015854047926542762, 0.013810354509425445, 0.01379850165738894, 0.016832954073476066, 0.01704777141541993, 0.014587430937198755, 0.01485571809637042, 0.012788264536260092, 0.012952687207322428, 0.014748129228607649, 0.016395870285118956 ], "mean_node_assignment_entropy": 4.15112697549008, "normalized_node_entropy": 0.9981350502733235, "effective_mass_slots": 63.658528854796224, "node_slot_mutual_information_nats": 0.0024063366842765888, "normalized_node_slot_mutual_information": 0.0005786016668525044, "centered_token_effective_rank": 1.092211715345537, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.011210175540949389, "token_RMS": 0.19151535641798756, "mean_token_offdiagonal_cosine": 0.9966233098813849, "temperature": 0.48205310106277466 }, { "layer": 5, "head": 7, "slices": 64, "native_nodes": 33555, "mass": [ 0.015137572551176403, 0.015369017050502873, 0.015189823656003063, 0.01636802989455511, 0.014762387476637216, 0.01502570053760971, 0.016287949388132616, 0.016593238290478693, 0.015716875876857776, 0.01559960455678802, 0.015222222085960566, 0.015403100553105653, 0.015253396509952133, 0.016579667192783455, 0.01525364740828149, 0.015134954206354615, 0.015577224574732214, 0.015564452247776568, 0.016242260032008283, 0.016494396951362088, 0.015458516778920921, 0.016595551117670693, 0.015865167326816653, 0.01532506413181605, 0.015033172942379373, 0.0159742766930474, 0.016344865967593547, 0.015695190818425513, 0.015817494050496525, 0.014909192891809503, 0.015241797326870664, 0.01545757986369249, 0.015645983331171887, 0.01650044018786549, 0.015615407284557367, 0.01558856838805352, 0.015662939100375146, 0.015878542394890345, 0.015212895986230634, 0.015908781231679783, 0.016331178991887577, 0.015199848046911275, 0.016782713432884698, 0.015519177729399624, 0.01606137634779147, 0.015803449949899927, 0.015141553111122362, 0.015411795292650823, 0.01567573083034239, 0.015957912871695422, 0.015202216680656112, 0.015422916619018033, 0.015295366373403277, 0.015135929993247066, 0.015297951783822254, 0.015437326700743185, 0.015528304474474092, 0.01568306537625101, 0.014894855857748083, 0.014935312665651441, 0.016026304871972964, 0.015168618072796297, 0.016201878327061995, 0.015378266961801303 ], "mean_node_assignment_entropy": 4.156877075545564, "normalized_node_entropy": 0.9995176570791005, "effective_mass_slots": 63.968691448687395, "node_slot_mutual_information_nats": 0.0015166920043911958, "normalized_node_slot_mutual_information": 0.0003646873388818534, "centered_token_effective_rank": 1.1659324655183962, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.007584215870317025, "token_RMS": 0.2055229633994745, "mean_token_offdiagonal_cosine": 0.998701300146399, "temperature": 0.49997249245643616 }, { "layer": 5, "head": 8, "slices": 64, "native_nodes": 33555, "mass": [ 0.016387683559709255, 0.01453397785621648, 0.01745937412027106, 0.01488053464143247, 0.015154784397312167, 0.016002629482886898, 0.014967973833489287, 0.014959196645348505, 0.017749613251399975, 0.015603060114170094, 0.015282659313383783, 0.014954869983035857, 0.015349302330388653, 0.016106551119480672, 0.014773090965318673, 0.01788365355164208, 0.016774492473661404, 0.017010122639279868, 0.016120712684375565, 0.015014095939423835, 0.013376530019488424, 0.015580387836434041, 0.014447323625568413, 0.015625737732293806, 0.01507645814613367, 0.016924812357537997, 0.01431111992041079, 0.017203399017780705, 0.01614562198189421, 0.014962321285765122, 0.014073138255106943, 0.018379074325578264, 0.015960223869909217, 0.013925627851421592, 0.01704646215390127, 0.014234277697768817, 0.014253050237649813, 0.015671219052937016, 0.0167784553542711, 0.014736224415266173, 0.017224092938027017, 0.016073989757393436, 0.015337926884351243, 0.014566971457642026, 0.017355997754101862, 0.015363837727165812, 0.016733947400691444, 0.014470164283567179, 0.01573985494357399, 0.015444281294435648, 0.01515141759628982, 0.015930250464799667, 0.013382206845075456, 0.014753496338466367, 0.01755641068116379, 0.014305303458720024, 0.0157971186160646, 0.01451801145556138, 0.015301395395434556, 0.014896130125546105, 0.01570428833570828, 0.017057453493421557, 0.016433793426197253, 0.01522181598510415 ], "mean_node_assignment_entropy": 4.154356510303154, "normalized_node_entropy": 0.9989115892498569, "effective_mass_slots": 63.83272483732811, "node_slot_mutual_information_nats": 0.0019094770295069452, "normalized_node_slot_mutual_information": 0.00045913217352684313, "centered_token_effective_rank": 1.198935082153997, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.010366377252548357, "token_RMS": 0.23166034467579333, "mean_token_offdiagonal_cosine": 0.9981091988740028, "temperature": 0.4966919422149658 }, { "layer": 6, "head": 1, "slices": 64, "native_nodes": 33555, "mass": [ 0.007689559415375253, 0.047626147479442826, 0.019570362945815132, 0.004159581597082316, 0.007267909448589969, 0.003894705440007531, 0.05939249293513695, 0.03237985542426786, 0.007525157161371183, 0.026526931877066333, 0.0037409850913362923, 0.005509239709124939, 0.023546818898600236, 0.00639189679497297, 0.003747446575891623, 0.005093769357087166, 0.03936108361905786, 0.0575332412182854, 0.0036477601153487673, 0.007188955343404326, 0.04273098297744674, 0.015144697111280367, 0.0035392581796901685, 0.0038729602312789894, 0.006214323436466397, 0.0036171729463821346, 0.01803372788765208, 0.021454042275153816, 0.004246430368526422, 0.006785532997732652, 0.006404953345147366, 0.002671236769337723, 0.017622902522824183, 0.005556851227517553, 0.006945679639414066, 0.006885377112903715, 0.006311951253539296, 0.003903079863462551, 0.003383003994943237, 0.008522887423333663, 0.02452845870765741, 0.005283220550084228, 0.01694601764978799, 0.00976091799251441, 0.030338072142090673, 0.052742378423946606, 0.013850749033788972, 0.0035534978480425356, 0.06598710702947863, 0.002792598205819646, 0.0019550728519084804, 0.012639569243877758, 0.025019216679415198, 0.0028130702018882674, 0.008969849320451907, 0.004158826636868227, 0.011501008448610227, 0.006440828769267795, 0.0036361042661388365, 0.009982678460212866, 0.02319837488581537, 0.01293928522584429, 0.011135775332174382, 0.07418636997672906 ], "mean_node_assignment_entropy": 3.6808387776679385, "normalized_node_entropy": 0.8850546418088882, "effective_mass_slots": 40.38365759395669, "node_slot_mutual_information_nats": 0.017586410459830226, "normalized_node_slot_mutual_information": 0.004228637859572477, "centered_token_effective_rank": 1.0535310232808583, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.022289124595540583, "token_RMS": 0.22596203973554735, "mean_token_offdiagonal_cosine": 0.9919373874884386, "temperature": 0.45872411131858826 }, { "layer": 6, "head": 2, "slices": 64, "native_nodes": 33555, "mass": [ 0.014237151593397323, 0.0130155239391624, 0.014915880434339411, 0.015246022537416156, 0.01702774538426943, 0.017802044972744062, 0.013334643781310603, 0.016697215672008967, 0.013795395059178837, 0.013355244741408867, 0.015939467349159762, 0.015131293661168561, 0.012738129728745861, 0.01859131482413288, 0.020360487997376463, 0.01689885170727018, 0.01394901463178791, 0.013254136679522808, 0.014476044658300036, 0.01421157979823992, 0.012592390343194933, 0.016040840877641634, 0.01717302029559117, 0.018793316133683367, 0.014097427789995043, 0.018033738659297614, 0.015448503138009259, 0.01379803783476637, 0.015582475458392397, 0.01694368687431275, 0.017448413949234975, 0.017563848821367196, 0.018019161890398958, 0.014529093767353531, 0.01598647066041319, 0.01665948243730404, 0.015089238890879842, 0.014592225692173734, 0.016228951169687197, 0.013792558131935726, 0.01573167079236709, 0.01581707858431803, 0.018645937302660665, 0.015228492312850504, 0.012725522905430501, 0.013083808763043445, 0.013711637385120701, 0.01770744597282016, 0.013905758611222854, 0.01629291042838717, 0.018534420343176154, 0.013188861010227291, 0.015308528824266246, 0.014679549624853369, 0.014829332217093401, 0.019283936884364684, 0.015980767633312416, 0.014675804075770132, 0.016588213756644963, 0.014004616434901682, 0.01682551855868362, 0.016589284869391905, 0.020438316488238014, 0.012832514627004963 ], "mean_node_assignment_entropy": 4.1483210752368525, "normalized_node_entropy": 0.9974603738765634, "effective_mass_slots": 63.511612551088035, "node_slot_mutual_information_nats": 0.0029016887625159526, "normalized_node_slot_mutual_information": 0.0006977086646474997, "centered_token_effective_rank": 1.1439735917167058, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.009289163275813072, "token_RMS": 0.27003183860271474, "mean_token_offdiagonal_cosine": 0.9988047188840716, "temperature": 0.49444645643234253 }, { "layer": 6, "head": 3, "slices": 64, "native_nodes": 33555, "mass": [ 0.01879520439571279, 0.020088118384297336, 0.01694775051070257, 0.01312718473426468, 0.01380363888447379, 0.01224308177057752, 0.022331605512205293, 0.01972685778518269, 0.017994546849458785, 0.01859250651011078, 0.015913941035267273, 0.015093515527833905, 0.015197439704436185, 0.011896279336771655, 0.01414622825769825, 0.015450162924615206, 0.015441073585790683, 0.025875857931100615, 0.015026764868535328, 0.016977382473638714, 0.01493708596564413, 0.012760356180158914, 0.015848537874447618, 0.010471712435315226, 0.0142334772699939, 0.013186214083974709, 0.012006128898811641, 0.017787633321245635, 0.010536532475267646, 0.015889478984577942, 0.018523596829565853, 0.013872237785076388, 0.011626484930862784, 0.016752483394741346, 0.018801705247548126, 0.012711010483913585, 0.013703478769221731, 0.016349278620802696, 0.013169588394198834, 0.011729758407855652, 0.017119304508341617, 0.012801305677021799, 0.013356959923514532, 0.0126777785593865, 0.02031196766656329, 0.014572587381144107, 0.01907358234141795, 0.014267743801837696, 0.022771610869387546, 0.015377271331701277, 0.013798380315979917, 0.011435639674257253, 0.01497078038654381, 0.01225234392562454, 0.01508191310964506, 0.013703395440928043, 0.024097647131290915, 0.017026651099922626, 0.01212764339153217, 0.018623033812681176, 0.013760087770569335, 0.018382304108469475, 0.013242849419755252, 0.019601296844920374 ], "mean_node_assignment_entropy": 4.136020546326182, "normalized_node_entropy": 0.9945027218666074, "effective_mass_slots": 62.661529675535874, "node_slot_mutual_information_nats": 0.0017271512129406474, "normalized_node_slot_mutual_information": 0.0004152920816291384, "centered_token_effective_rank": 1.2325957079210406, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.007367629881950452, "token_RMS": 0.2062390638878768, "mean_token_offdiagonal_cosine": 0.9987112132690059, "temperature": 0.4878517687320709 }, { "layer": 6, "head": 4, "slices": 64, "native_nodes": 33555, "mass": [ 0.015467907582297134, 0.02321135258317913, 0.023067426329500547, 0.011474658729462168, 0.013414173529097165, 0.011475948412529208, 0.02508334686321578, 0.024652332844355036, 0.014990636455362812, 0.024647838759136537, 0.010830886285563878, 0.014202604147048692, 0.022422802004440807, 0.01028227152325343, 0.012037282800011869, 0.013206587385704302, 0.023270772215333642, 0.02141373150082281, 0.009651794060709455, 0.015110233660069597, 0.023301882172957332, 0.017509749744322265, 0.011285942392469545, 0.01238168605405245, 0.0110228583954953, 0.0100207868572133, 0.01942424064369173, 0.01770871304496823, 0.011443046130952714, 0.014391916389798438, 0.013759114931130344, 0.011256609465430069, 0.015241198159166977, 0.011855641230158125, 0.01282251658651663, 0.013450625258693706, 0.012305326736722967, 0.012951164307288416, 0.010217572149261776, 0.014696076043608505, 0.02256887688846475, 0.011608826057496103, 0.01732001841021639, 0.013755949592319664, 0.020522596258428623, 0.024513120819096894, 0.014454260431096844, 0.010393671376119693, 0.019839176899753373, 0.010376810743925184, 0.011402380791973062, 0.015833472817799822, 0.017663980384414586, 0.010331591897377793, 0.013109102854989507, 0.011853998966960256, 0.015850761754574557, 0.012016265210990463, 0.012391227163483225, 0.01587535273000618, 0.020475210101100694, 0.015200012864223306, 0.019046434848465944, 0.026635645948252733 ], "mean_node_assignment_entropy": 4.109874083150155, "normalized_node_entropy": 0.9882158264064673, "effective_mass_slots": 61.2689573581469, "node_slot_mutual_information_nats": 0.005399226275108582, "normalized_node_slot_mutual_information": 0.0012982394952894237, "centered_token_effective_rank": 1.0680681636942397, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.011751019371790775, "token_RMS": 0.17272532809856037, "mean_token_offdiagonal_cosine": 0.9956398461470225, "temperature": 0.4926237463951111 }, { "layer": 6, "head": 5, "slices": 64, "native_nodes": 33555, "mass": [ 0.007495909147525203, 0.021107567015010046, 0.018976295312788577, 0.007688926001465315, 0.016789095662728522, 0.010040419605786666, 0.019327996106216865, 0.019565416310849053, 0.007734505206248259, 0.01529481319158008, 0.0056813126150699065, 0.009287252922913896, 0.01190959166204178, 0.02118413413889654, 0.016022989870471075, 0.006617075009397788, 0.02436079597057595, 0.017486181788311916, 0.005841719556928476, 0.006469949893181556, 0.02437349533457645, 0.03154750088535914, 0.007134098727730672, 0.01648379596322997, 0.008373029343880999, 0.010330135329548648, 0.03616932965781761, 0.013459010307073834, 0.010612638398695189, 0.008532970074693003, 0.009875541640385361, 0.0083136322004536, 0.04561725715898292, 0.0066000969137931485, 0.012498903205981021, 0.019145582135638516, 0.008686449558887258, 0.005474089093362287, 0.006880427039940437, 0.013324946987968215, 0.02397376874833966, 0.006923773515519327, 0.03167227530093413, 0.019562669472682585, 0.010937173042383648, 0.030165034978349506, 0.009166505094631483, 0.009673514940741654, 0.02683705842807154, 0.004659890667548548, 0.0059591223763874, 0.01653292119697862, 0.03602078464562208, 0.004947298754045759, 0.00914627493575222, 0.008464303515815971, 0.012167318208794527, 0.007343937773931943, 0.007961210956253836, 0.012091007254292794, 0.05831558191281458, 0.013630954571531241, 0.03822815835008526, 0.02330458410277978 ], "mean_node_assignment_entropy": 3.9436635600037078, "normalized_node_entropy": 0.9482506434919774, "effective_mass_slots": 52.46928093811959, "node_slot_mutual_information_nats": 0.016564313353240046, "normalized_node_slot_mutual_information": 0.003982875455075042, "centered_token_effective_rank": 1.0277354860286376, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.027592047117635396, "token_RMS": 0.23181628776347313, "mean_token_offdiagonal_cosine": 0.9861500093641756, "temperature": 0.46441784501075745 }, { "layer": 6, "head": 6, "slices": 64, "native_nodes": 33555, "mass": [ 0.007640128266141252, 0.04902180625088851, 0.03130351420946562, 0.0025549868815031053, 0.004176451786428317, 0.0025508180342362356, 0.037945010512845205, 0.035928983787207075, 0.007195777237313175, 0.053250654105159785, 0.002333744477505352, 0.004578396927544797, 0.035953156171540594, 0.0030885379185504303, 0.004529534027187453, 0.002960614376116337, 0.047000262817952713, 0.045140369712857904, 0.002167824208508134, 0.006010110927069863, 0.08382049051262333, 0.028977957717278305, 0.002483391350917485, 0.0030148710157015036, 0.002574855049511698, 0.0024610825034839284, 0.01663946987484597, 0.022562336257345682, 0.002395835707790266, 0.006278459780373932, 0.004727805719543385, 0.0037902288625482816, 0.0065526308480030136, 0.003913968840681683, 0.004754855907028795, 0.009885906213135384, 0.0035749567234149, 0.004029968926829337, 0.002830307268422536, 0.008268925333552416, 0.04605558004436759, 0.0019101347145579356, 0.012229680622946024, 0.0059698219422038385, 0.040195602170663404, 0.05248791740441479, 0.0060266988448806425, 0.0025521522259113207, 0.027510589392776105, 0.002189437070553516, 0.0021034168326724036, 0.011425681844366998, 0.02859177869099154, 0.00250370544056358, 0.00484634588985737, 0.002470493213786499, 0.012473911236188368, 0.00354619458036875, 0.0024071562955661945, 0.014474012181018507, 0.02565762330495535, 0.005506056320863662, 0.014340610709221765, 0.06365641239292796 ], "mean_node_assignment_entropy": 3.5573097153031257, "normalized_node_entropy": 0.855352180862325, "effective_mass_slots": 35.95995394192395, "node_slot_mutual_information_nats": 0.02509621348534541, "normalized_node_slot_mutual_information": 0.0060343637900664255, "centered_token_effective_rank": 1.0284526035913253, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.03578299032761833, "token_RMS": 0.24201780462075792, "mean_token_offdiagonal_cosine": 0.9826195211161688, "temperature": 0.45899835228919983 }, { "layer": 6, "head": 7, "slices": 64, "native_nodes": 33555, "mass": [ 0.007109390861740426, 0.04241946952911811, 0.038660072321284544, 0.001601398178969731, 0.0033357735216726385, 0.002048184786484838, 0.04478907407500264, 0.05144566008417293, 0.004203976495845088, 0.04116544428784825, 0.0016680489064834988, 0.003198338638622807, 0.03192409953509637, 0.0022484554903896203, 0.0024357994080795526, 0.002514720997235911, 0.038264695949149705, 0.044262722527463674, 0.00163386296804588, 0.00663636341687409, 0.06921690223074371, 0.029985734175115207, 0.0021501684373122797, 0.0018761354504948872, 0.0018400741855950002, 0.0017217998729670825, 0.016684050925937527, 0.024645208171676362, 0.0019106544343780462, 0.004788536550321718, 0.004963601605965022, 0.0019592496518458105, 0.006419731911900186, 0.002971584654396233, 0.0032689458532240826, 0.006177096370265671, 0.0032973345696576126, 0.0033527354872240813, 0.001627618721320977, 0.006628227803796184, 0.04032884528381674, 0.0014356584846369745, 0.011425841848027767, 0.005768014864979145, 0.039245299154742966, 0.06648269987138741, 0.005489521154115508, 0.002153038717682135, 0.036136473012379516, 0.0013543213838270074, 0.0011640013966324899, 0.01227865097659868, 0.023958692205296846, 0.0014624666439327966, 0.00453632383519483, 0.0018023049144829677, 0.007893621820052762, 0.0037125426584863324, 0.0018947400599974135, 0.012262806245195912, 0.0217474435826489, 0.004116310184298858, 0.014167710602468882, 0.11213172768452069 ], "mean_node_assignment_entropy": 3.4296256414089346, "normalized_node_entropy": 0.8246506508277168, "effective_mass_slots": 31.740507688402797, "node_slot_mutual_information_nats": 0.027968068785186873, "normalized_node_slot_mutual_information": 0.006724899023271753, "centered_token_effective_rank": 1.034498937165543, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.039000351936689895, "token_RMS": 0.19345385092392053, "mean_token_offdiagonal_cosine": 0.9606189560729645, "temperature": 0.4483592212200165 }, { "layer": 6, "head": 8, "slices": 64, "native_nodes": 33555, "mass": [ 0.01183658929120107, 0.034634523596567414, 0.023574296032171406, 0.006577904428495698, 0.011357873631773105, 0.00689444145575752, 0.040423707151487125, 0.04051137061673338, 0.0121039305401586, 0.025168971641752726, 0.005907858676442105, 0.007880772747023906, 0.021864752816165404, 0.008524347570697168, 0.008412542622888911, 0.006765434084880074, 0.027573424006346405, 0.03807640046560161, 0.004998093047630996, 0.008245676124074047, 0.032058388729998985, 0.02085132177537794, 0.006085519271223711, 0.008418063543147971, 0.007298251069238987, 0.006754372102259644, 0.02831320682216634, 0.01763596540099617, 0.0064379590483541384, 0.009975196326140665, 0.010234729034459515, 0.006801585191737285, 0.017921729393429956, 0.007943715288460192, 0.010741910525360186, 0.012562573900313904, 0.007316240619989052, 0.006295819422642288, 0.005652275873685896, 0.010066644185822335, 0.025999241042174447, 0.0057083269839459955, 0.019739322325750694, 0.011159908849256944, 0.0226312582166939, 0.03720368216039882, 0.012856951199700652, 0.006488409335943581, 0.0376201454083187, 0.00538223971478278, 0.005240252257812273, 0.01366038244434679, 0.02385985860757884, 0.00454880914327788, 0.008952353046625528, 0.005980891416704463, 0.02045230423184252, 0.007311781953852066, 0.00666612375572306, 0.013491302436474955, 0.029685892047247576, 0.014480135324304932, 0.02219650802189894, 0.04798554167730978 ], "mean_node_assignment_entropy": 3.9186992619974017, "normalized_node_entropy": 0.9422479986698155, "effective_mass_slots": 51.13158016128544, "node_slot_mutual_information_nats": 0.01570305139469541, "normalized_node_slot_mutual_information": 0.0037757857289919317, "centered_token_effective_rank": 1.0406356234759708, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.021135685183750814, "token_RMS": 0.24181668332776407, "mean_token_offdiagonal_cosine": 0.9922796747435201, "temperature": 0.47706717252731323 }, { "layer": 7, "head": 1, "slices": 64, "native_nodes": 33555, "mass": [ 0.012775110069819427, 0.029161300967405688, 0.007714496854031571, 0.005945991066184143, 0.006480804009280068, 0.005463725911413676, 0.005884708954370216, 0.007480410884037745, 0.008096065655747027, 0.013035951209584987, 0.011841130441279796, 0.0091720635526303, 0.00838070015384559, 0.0066550045142083755, 0.00808970329520058, 0.06271675772700726, 0.010453944088595, 0.043587482511506515, 0.00814438494605961, 0.005696053426982531, 0.008962459607079521, 0.01610900832373705, 0.04927404139453476, 0.006544849056181001, 0.007816970336284684, 0.00905363255027349, 0.007796480970904331, 0.04962719040100576, 0.006348774137413661, 0.015203250162558794, 0.008544989484744631, 0.007688034135604594, 0.006948296401847885, 0.0067290601596866825, 0.015105183580519767, 0.0070138107911013195, 0.013479800208555952, 0.045549205599530074, 0.06771505625471373, 0.05519062348210835, 0.00824855644013852, 0.008555410098578131, 0.009461772899184892, 0.00641674280180226, 0.023934958627326197, 0.012635860114988823, 0.008582295456851512, 0.0062646967982821974, 0.015001835265742109, 0.03504416686544483, 0.007630747766903573, 0.016004289542325537, 0.010073149627103187, 0.008065169773282015, 0.012946182663947002, 0.007556175140322871, 0.012235507077612832, 0.008287038563621207, 0.007236029257346458, 0.006319825753132183, 0.008362995242002494, 0.04520554283469796, 0.00880732904102988, 0.019647215131901714 ], "mean_node_assignment_entropy": 3.8151298473463453, "normalized_node_entropy": 0.9173448185190068, "effective_mass_slots": 45.589999099899515, "node_slot_mutual_information_nats": 0.004558527135629209, "normalized_node_slot_mutual_information": 0.0010960940820550053, "centered_token_effective_rank": 1.0974004967944713, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.008040099606183562, "token_RMS": 0.16870306449085426, "mean_token_offdiagonal_cosine": 0.9979000331028973, "temperature": 0.47620514035224915 }, { "layer": 7, "head": 2, "slices": 64, "native_nodes": 33555, "mass": [ 0.0008515571841628421, 0.07339908663061884, 0.0005620090842536773, 0.0007264346535475249, 0.0007131168935995747, 0.0005695948981775176, 0.00046166126710050695, 0.0007915879062489255, 0.00043574399357103935, 0.010936118888367157, 0.0013654455134911275, 0.0008040567102978102, 0.0014152535577233191, 0.0006143397053975944, 0.0006361316436196847, 0.2939968228656557, 0.001676078421638113, 0.14788982939369774, 0.0006727565291664208, 0.0004931128765288807, 0.0013031677489168665, 0.0065779749077435725, 0.049057586580978974, 0.0007178387406369625, 0.0026205255410943033, 0.0012140846414146122, 0.0009626138424902579, 0.11315226789068188, 0.0011754359971658905, 0.00419852490759742, 0.0017039452052987848, 0.0008945668181089451, 0.0005544733687988401, 0.0008296880111209449, 0.007446933192711791, 0.001099224325853775, 0.002035090090796316, 0.018925293514233305, 0.07508314247864127, 0.03951894631940994, 0.0007672334430439043, 0.0011605142267565075, 0.0017497369931848957, 0.0010447800445123772, 0.005136860035699305, 0.0017894987258142837, 0.0007966331124479957, 0.0004764937127576869, 0.0021094515122003625, 0.06921703918564873, 0.0043984088590977855, 0.0018146192490414224, 0.002048741489896672, 0.0007016115205682594, 0.0028865681769737936, 0.000464193815453348, 0.0030711925543862465, 0.001626556247401016, 0.0009927210925035777, 0.000679895269295818, 0.0015213553407637482, 0.019613581998688668, 0.0005251777554651124, 0.007325072750222144 ], "mean_node_assignment_entropy": 2.4563238136869225, "normalized_node_entropy": 0.5906210308039316, "effective_mass_slots": 11.88308395054889, "node_slot_mutual_information_nats": 0.01879205835650044, "normalized_node_slot_mutual_information": 0.004518534899836532, "centered_token_effective_rank": 1.0499718057511733, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.02608333529145475, "token_RMS": 0.24091518395930567, "mean_token_offdiagonal_cosine": 0.9881171858518673, "temperature": 0.43115341663360596 }, { "layer": 7, "head": 3, "slices": 64, "native_nodes": 33555, "mass": [ 0.030579532801798533, 0.01121668297452723, 0.018186101260674263, 0.010531561846845268, 0.008486085639903778, 0.008997388630011871, 0.012147556458264168, 0.009767375907617422, 0.030762133572093512, 0.010713571114120043, 0.01341339449817738, 0.01295516261193391, 0.01118828698736272, 0.014536459243857342, 0.016009459472973983, 0.01775390990952735, 0.0091376677328409, 0.018087718453085638, 0.014018712056135434, 0.012565233639183724, 0.01418879484651479, 0.011760796618697417, 0.03436962885990563, 0.009375795332548106, 0.00797864317981688, 0.016513106613910523, 0.01047021972539794, 0.014947155581110282, 0.005860705939699545, 0.01536815557481768, 0.010259572570168471, 0.010359408830936025, 0.02075988976871882, 0.013608269916461274, 0.017156886495177978, 0.008725300275462117, 0.020741477724419515, 0.030257538863713226, 0.035918732540191525, 0.023300710625862146, 0.0145112259309856, 0.009722871241519026, 0.00890661155773906, 0.009516598683557245, 0.014857037529385964, 0.013435430423056977, 0.013523288717794003, 0.012921776958148537, 0.019560453240586063, 0.018341884411668265, 0.007962547401150858, 0.020512246946656983, 0.011722699172557235, 0.019422456872058447, 0.015482736048095051, 0.015223847636449241, 0.020881060697055046, 0.012565932197865715, 0.009157330049677847, 0.00905014562457045, 0.009660872523529247, 0.0246669048165608, 0.024278912894264575, 0.04114034380816504 ], "mean_node_assignment_entropy": 4.059703200134567, "normalized_node_entropy": 0.9761522790525327, "effective_mass_slots": 58.11010131571727, "node_slot_mutual_information_nats": 0.0026363094669701326, "normalized_node_slot_mutual_information": 0.0006338984323744062, "centered_token_effective_rank": 1.0977204966644107, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.006074521529293038, "token_RMS": 0.1689963005137223, "mean_token_offdiagonal_cosine": 0.9986926096314849, "temperature": 0.4652578830718994 }, { "layer": 7, "head": 4, "slices": 64, "native_nodes": 33555, "mass": [ 0.018774484891365066, 0.012478755904724364, 0.01658943093940671, 0.015689361467929742, 0.01416383350312137, 0.014508184448585098, 0.014906908005155149, 0.015389180118991043, 0.020266677201906644, 0.014376536350113867, 0.0151418035875186, 0.017834529456380113, 0.01759363706295136, 0.016379898557676197, 0.017918951741100964, 0.010839635682017646, 0.01659954862849241, 0.012980929397839252, 0.0165618199378534, 0.017690725874468283, 0.017443546980415153, 0.016243569234186527, 0.015021315519576535, 0.014969659152239086, 0.013807129267658114, 0.017046317038634348, 0.016800431968488, 0.01210050579161007, 0.013512618577288473, 0.014388421079361204, 0.01496767788829018, 0.016699131815041332, 0.018976645428377032, 0.017306242454028214, 0.014855214942264994, 0.017680016805900357, 0.01497799190508726, 0.014399080621718366, 0.012185913434183033, 0.012809512312029286, 0.015686899534924973, 0.01617563959200564, 0.014295007612718864, 0.013179508227876616, 0.013958796792730386, 0.013544106154228408, 0.01545455864626615, 0.016524960525997782, 0.01884566067354794, 0.013236318045768861, 0.017371751950181397, 0.015021411977145465, 0.016837177172718418, 0.017152871509186558, 0.017749987254087163, 0.016909717068780004, 0.015142233060005924, 0.016056879672966092, 0.01515583914212384, 0.014879455682150593, 0.015277903410283433, 0.01179090497595905, 0.017975607461979768, 0.018901029274515965 ], "mean_node_assignment_entropy": 4.150188094676906, "normalized_node_entropy": 0.9979092971577982, "effective_mass_slots": 63.48795841210327, "node_slot_mutual_information_nats": 0.0006621619339899354, "normalized_node_slot_mutual_information": 0.0001592162897387875, "centered_token_effective_rank": 1.3581563650773014, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.0027245062156577428, "token_RMS": 0.2073910957570684, "mean_token_offdiagonal_cosine": 0.9998361869972996, "temperature": 0.49655693769454956 }, { "layer": 7, "head": 5, "slices": 64, "native_nodes": 33555, "mass": [ 0.025546264697453018, 0.002493829572662953, 0.028186479656600576, 0.021713498344548037, 0.014092658390585933, 0.021062188280957003, 0.02655684176253466, 0.017454761697523125, 0.04266577694449041, 0.0045922085536998294, 0.015669918647545262, 0.015390604034797825, 0.012053749828558178, 0.02663611393572321, 0.02375178007005695, 0.003424542538665991, 0.014004341211553533, 0.0028770965692213753, 0.017213621951285497, 0.03824366822460203, 0.01468804991110047, 0.006470547516022176, 0.007732272043911902, 0.015722752708679136, 0.007919055553847868, 0.01910597244978413, 0.01615362296783232, 0.00286484939887045, 0.011394029932082446, 0.010016858149355969, 0.011260830444876217, 0.017756738009564375, 0.0318099082256217, 0.014773499363048868, 0.0069684215780803326, 0.016962746580452895, 0.013417077955175504, 0.010110451789655268, 0.007355712818914688, 0.008543473381089496, 0.020752827478887747, 0.013632711325428638, 0.01011238219165119, 0.013506676273059914, 0.011358588028656417, 0.016518118766274827, 0.015258953244553168, 0.02506159663882044, 0.01774454754106273, 0.00470261094335929, 0.007837055206163939, 0.021600658169887474, 0.012503171486844161, 0.021453328749170213, 0.011235607575307107, 0.03206822246855985, 0.010450361181600646, 0.016221953144194588, 0.01217576211839892, 0.016976883880169002, 0.009681444257639893, 0.008869267319848242, 0.033981145897497456, 0.011639310537296638 ], "mean_node_assignment_entropy": 4.012135278255906, "normalized_node_entropy": 0.9647146115525762, "effective_mass_slots": 55.40559674545918, "node_slot_mutual_information_nats": 0.002545334680989164, "normalized_node_slot_mutual_information": 0.0006120236202776265, "centered_token_effective_rank": 1.0717217188250323, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.00784148568384131, "token_RMS": 0.28896460226553766, "mean_token_offdiagonal_cosine": 0.999354524220287, "temperature": 0.47716283798217773 }, { "layer": 7, "head": 6, "slices": 64, "native_nodes": 33555, "mass": [ 0.00024475453513234487, 0.004669178303996149, 0.00033041212626600284, 0.0003079978689595276, 0.0003489269592421326, 0.00028863693901179724, 0.00038608297325197217, 0.0004078192785641087, 0.0003775976676996216, 0.0007764071860558346, 0.0008182825673669366, 0.00033855371344611833, 0.00033575443641252505, 0.0003782147627405466, 0.0002794906269910673, 0.17068286644947936, 0.0008202238322628448, 0.0338035595478528, 0.0002659773980185468, 0.0003223514194382268, 0.00031600565642595565, 0.0015089771867395413, 0.07464044778215745, 0.00036750424875219793, 0.0003675783801439585, 0.0005271362267836734, 0.00037846444390660116, 0.028379086014246945, 0.0002312861242993142, 0.001528113546946454, 0.0002710180173843325, 0.00048337764882151193, 0.00037467277797000146, 0.00023756293160456633, 0.0006846990119863963, 0.0003661294854818182, 0.0027069711246636113, 0.09105344257894651, 0.2920185400644627, 0.08400808785217846, 0.0003455277955526398, 0.0003590905134957267, 0.0005926681614450104, 0.00023774727721018722, 0.01379231988957379, 0.0038691936445125075, 0.00025060338631990376, 0.00028702686286172723, 0.0009857496720423132, 0.018899871157422594, 0.0003271098193781348, 0.004815449503553922, 0.00044696561719231653, 0.0003232417288845545, 0.00038040654402780064, 0.0004088841411999276, 0.0006244876821583472, 0.0003741475945933121, 0.00024204499713732425, 0.0002552333893515471, 0.0003264743864601059, 0.15365967384326246, 0.0003315831487749438, 0.0012323095353982618 ], "mean_node_assignment_entropy": 2.1624110213883405, "normalized_node_entropy": 0.5199499428200995, "effective_mass_slots": 8.846925848181224, "node_slot_mutual_information_nats": 0.017659015499984942, "normalized_node_slot_mutual_information": 0.00424609568146827, "centered_token_effective_rank": 1.0606020737548583, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.032942637816332325, "token_RMS": 0.3076680953794162, "mean_token_offdiagonal_cosine": 0.9889377258188097, "temperature": 0.4470881223678589 }, { "layer": 7, "head": 7, "slices": 64, "native_nodes": 33555, "mass": [ 0.0013297289693347844, 0.03750038723801469, 0.0011509133152849112, 0.0014247197177347987, 0.0013703456544414449, 0.0014202475344484377, 0.0012004692338604378, 0.0016753733211331988, 0.0012497877416135295, 0.008403160151664844, 0.0027429443056527386, 0.0019600404490118773, 0.002014628380131531, 0.0013587466860821463, 0.0013781918268420403, 0.21601806855285208, 0.0031936450005704492, 0.10439011207917039, 0.0013124038520727194, 0.0011095904407086997, 0.001750971227309762, 0.0068749707280305, 0.06912679895647608, 0.0016545839306780872, 0.002957515844459364, 0.0019783338358070114, 0.0017587768236972305, 0.07944741756714255, 0.0017036695681289483, 0.0057422986260202155, 0.0025942333221884903, 0.0022241847464193622, 0.001440237545234284, 0.0016442267309713038, 0.006403441708620997, 0.0017138936666230822, 0.006786749222429783, 0.047446562559069795, 0.11380067635291496, 0.06772313169111413, 0.0014746579383986328, 0.0018177745566989119, 0.0026915921092604033, 0.0013417002527494105, 0.012441060022785293, 0.004261149756939014, 0.0015374083172385888, 0.0011718207798613592, 0.0032469065302799466, 0.0552000244390794, 0.0028837397732022407, 0.005366761094777155, 0.0027289420919064857, 0.001571305723513713, 0.0030414926785780855, 0.0014052423612628036, 0.003498344170477787, 0.002610029818277405, 0.0016625465637402097, 0.0013061340464098786, 0.002661343518004955, 0.059780994970520765, 0.0012545029406525715, 0.008068346683210173 ], "mean_node_assignment_entropy": 2.805499663788951, "normalized_node_entropy": 0.6745800753606623, "effective_mass_slots": 16.82460810983899, "node_slot_mutual_information_nats": 0.017342919336190477, "normalized_node_slot_mutual_information": 0.004170090620143219, "centered_token_effective_rank": 1.0328757137478124, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.018317827145944047, "token_RMS": 0.3111808800597224, "mean_token_offdiagonal_cosine": 0.9970477898826171, "temperature": 0.45496049523353577 }, { "layer": 7, "head": 8, "slices": 64, "native_nodes": 33555, "mass": [ 0.00034351315788183357, 0.029893974126697545, 0.00029740276663585516, 0.00032002473877956045, 0.0004896518328228004, 0.0004061355214622752, 0.00032046122900877485, 0.0004994228835690324, 0.00022242458255849234, 0.002461121575741036, 0.0010180298598039678, 0.00039218395141239515, 0.0006136887697058542, 0.0003365228768608309, 0.00033298951572481643, 0.30781033084478165, 0.001119438345234432, 0.10091648757304525, 0.0004199791023936843, 0.0002456489736485153, 0.0004928887709750773, 0.0025840152052631223, 0.05061522509273785, 0.0004570441515652578, 0.0008768806187746495, 0.0006577744506070655, 0.0005994237655561918, 0.09711352481965141, 0.0006071832817678014, 0.002598458963534858, 0.0007804074723872609, 0.0005754291841980862, 0.00025088867876463987, 0.00034011067348344894, 0.0020341646675555575, 0.000475806893184856, 0.001716511436386833, 0.042973150397831165, 0.14358171790720559, 0.07210555076532509, 0.000373194280834378, 0.0006654214581823752, 0.001562107037513826, 0.00043185007230363005, 0.012222833209452508, 0.002102981750794935, 0.00038637773778053705, 0.000294935684997815, 0.0010931519728913461, 0.03226361397346847, 0.0008507296564934531, 0.002210723806816944, 0.0006697110296436936, 0.00035223221531576843, 0.0010023492814884459, 0.00032455959877200413, 0.0009966385239801026, 0.000591924472958411, 0.0004361620956276456, 0.0003988211536833949, 0.000708195535710528, 0.06814795264225126, 0.00027373476811300466, 0.0017442093213679856 ], "mean_node_assignment_entropy": 2.305340431658859, "normalized_node_entropy": 0.5543172013858431, "effective_mass_slots": 10.158559072041703, "node_slot_mutual_information_nats": 0.012976176815613805, "normalized_node_slot_mutual_information": 0.003120110990264063, "centered_token_effective_rank": 1.0455754722743626, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.025648645067212986, "token_RMS": 0.2544062840120526, "mean_token_offdiagonal_cosine": 0.9907084844251717, "temperature": 0.4423738718032837 }, { "layer": 8, "head": 1, "slices": 64, "native_nodes": 33555, "mass": [ 0.005315652982481829, 0.0017769640485311047, 0.027081366091957746, 0.0027723017532665425, 0.00464064020393859, 0.002631644757983271, 0.06168618001795405, 0.01316944000818698, 0.0024448145354479425, 0.02880601296696777, 0.029452088162094252, 0.002598620242934778, 0.0250434974835462, 0.005066464400559224, 0.1625467148886594, 0.0023348746946841455, 0.0030350385741400765, 0.002105882680163619, 0.02049735564499263, 0.00621261258815397, 0.038274904694105646, 0.003893637202058119, 0.0010937314797263588, 0.004183847971944434, 0.0030384785585416832, 0.007596186158082011, 0.016114652068984926, 0.00286709756053781, 0.004050521840320712, 0.033601528502175855, 0.018285583602793257, 0.0022454468599106933, 0.003669831617645816, 0.0015716767716230531, 0.06323372410182057, 0.002844068720389167, 0.0019793270056840517, 0.043866042535340724, 0.12124916944131554, 0.0028556820745615284, 0.001236753869501172, 0.003806598772163121, 0.01711875739728169, 0.0015947800204660196, 0.0020885925591374995, 0.002404221761061345, 0.0023681651472111666, 0.0336921110402861, 0.002828541480460249, 0.004792584339357971, 0.004516874538746212, 0.004658836160167927, 0.0014618138085645248, 0.038174942391102314, 0.01676334942309534, 0.006666888988747107, 0.0023971411719594347, 0.012875273831870345, 0.012622038399124267, 0.004178513993098754, 0.0038357019846385473, 0.027189474565470616, 0.003405083423938906, 0.0035896554848504235 ], "mean_node_assignment_entropy": 3.304726880809787, "normalized_node_entropy": 0.7946188470727887, "effective_mass_slots": 27.433786159084793, "node_slot_mutual_information_nats": 0.0070484444753913245, "normalized_node_slot_mutual_information": 0.001694792648438046, "centered_token_effective_rank": 1.0418803212394068, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.009498349766741025, "token_RMS": 0.2694913025052547, "mean_token_offdiagonal_cosine": 0.9987494331148911, "temperature": 0.4412786364555359 }, { "layer": 8, "head": 2, "slices": 64, "native_nodes": 33555, "mass": [ 0.028794978284741226, 0.02667772761418263, 0.011926198760060045, 0.01461465060178009, 0.0143429575904853, 0.030969948611954803, 0.009444282134996807, 0.010830987750820016, 0.012783024622194859, 0.013574124738544052, 0.011598375016600538, 0.014999460875421802, 0.007590687799823545, 0.020460413195411524, 0.009514765806758151, 0.014303839982839335, 0.01217484332889212, 0.017953655891472367, 0.012535027616450957, 0.011187512026368382, 0.011772828437654393, 0.01777427688386997, 0.029906285812515133, 0.01122603662905435, 0.01549079530468429, 0.015502301673387526, 0.009619281000249957, 0.012276549732961752, 0.017272967445250417, 0.014059396898877717, 0.01769794167246047, 0.016972525864939473, 0.011732803526757868, 0.03367215444566578, 0.007777002669266902, 0.008752044231047033, 0.010563268009509385, 0.015551269298047161, 0.010426222821002328, 0.014165373527833609, 0.028328852306298148, 0.007550539102140526, 0.01572832023898938, 0.02691417397552916, 0.016024777093507275, 0.02364344963146924, 0.014459358190893105, 0.012676238640476723, 0.01657436103285949, 0.012933605168730291, 0.008900180670183366, 0.010432381078383283, 0.034349559208192514, 0.013361604031213677, 0.015237366606891732, 0.01122869465096557, 0.009448924516578168, 0.01588392312279172, 0.014394287189190913, 0.018518736875682486, 0.009451810067324982, 0.007133510798332993, 0.025909301555504423, 0.022427255937568995 ], "mean_node_assignment_entropy": 4.075966139338892, "normalized_node_entropy": 0.9800626893425922, "effective_mass_slots": 59.00619318908165, "node_slot_mutual_information_nats": 0.0016762683644042653, "normalized_node_slot_mutual_information": 0.0004030573427541813, "centered_token_effective_rank": 1.1666377055624813, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.003255236199492365, "token_RMS": 0.26634234034352755, "mean_token_offdiagonal_cosine": 0.9998486823256214, "temperature": 0.4726564884185791 }, { "layer": 8, "head": 3, "slices": 64, "native_nodes": 33555, "mass": [ 0.017116974164236232, 0.03668801348266179, 0.02614384228642572, 0.0031415191183366007, 0.0030211046153024194, 0.24295886087135166, 0.005648583324226137, 0.0028697429316440594, 0.0029524321689013287, 0.02966240230148941, 0.007843657321572934, 0.002374759031215583, 0.0073875494201817065, 0.0020985033300273247, 0.004568106771895426, 0.0021630869335315223, 0.0019863978629368934, 0.03132602471939863, 0.006990628895456519, 0.004433393100466157, 0.0027416310009090424, 0.047451994091831894, 0.07140085701514971, 0.0013993708826658404, 0.0028624367354666133, 0.0467524344119589, 0.0021395305302717104, 0.00357650997761219, 0.002547915593844899, 0.0025933707364551883, 0.0023933842688279327, 0.002145206148858579, 0.0027332950189971673, 0.05591809250444347, 0.0026892912058838875, 0.0018238343367788282, 0.0013205784289546576, 0.017905674330227387, 0.0019708897364054513, 0.002913758851478972, 0.04213025885346998, 0.0024670203880033195, 0.006505187353150617, 0.04049260228219499, 0.004554772290709659, 0.0036564282090001386, 0.002904684455108301, 0.00833640426385944, 0.0048390466483808625, 0.0026438885860470105, 0.006164518093923142, 0.006507358696165165, 0.05105850090014741, 0.012533519559875275, 0.009812433127829543, 0.004578576409516765, 0.002251033067190194, 0.02499301641487832, 0.014068162453327133, 0.0041197375212538604, 0.003480446947892014, 0.0029325693588389684, 0.018649431512774484, 0.0036647640123148467 ], "mean_node_assignment_entropy": 3.1842859843990374, "normalized_node_entropy": 0.7656589330774538, "effective_mass_slots": 24.34782776920581, "node_slot_mutual_information_nats": 0.008156652493386929, "normalized_node_slot_mutual_information": 0.0019612603504106534, "centered_token_effective_rank": 1.0567627409399192, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.008125128054355972, "token_RMS": 0.26754254731705845, "mean_token_offdiagonal_cosine": 0.9990904584100834, "temperature": 0.45806360244750977 }, { "layer": 8, "head": 4, "slices": 64, "native_nodes": 33555, "mass": [ 0.029114291752058406, 0.0005623217275020775, 0.310871031267546, 0.0010622905381329189, 0.0005709648219084945, 0.020035830132564443, 0.021681135997171728, 0.0014075419841384322, 0.0006155626286255775, 0.1361567805099729, 0.04461177526558943, 0.0006655884047299473, 0.032145353314076676, 0.000759728831005763, 0.010706181173877422, 0.0008242640911263982, 0.0006255639496060873, 0.0004591521784099633, 0.018299148986184052, 0.000656677239238898, 0.0006029602297221401, 0.00030983173524158544, 0.0004216520123183658, 0.0005212544810721872, 0.0009495688646181368, 0.0009897985570756528, 0.0006823579354846655, 0.0008651793481927238, 0.0014760190911614368, 0.0013779877398959375, 0.0003595863213936319, 0.0006622114040141331, 0.0011938036895312526, 0.00030890281319869686, 0.0012793467496955444, 0.0006461407025528201, 0.0005856214172382787, 0.15918312133082405, 0.0017537062180913666, 0.0005807500720107236, 0.000297082341414798, 0.0006205358519499883, 0.00124494571373646, 0.0006029253242705436, 0.0009432421844380317, 0.0014225098594907693, 0.0007719755866086728, 0.006018046021221127, 0.0007765424518779085, 0.0006062590083933206, 0.001249846027183025, 0.0010619124963666148, 0.00037970776249052446, 0.0007524922718544454, 0.005129159560682139, 0.0014792883118236806, 0.0006212445422265603, 0.08349689645452625, 0.07244525431394311, 0.0009173352965039958, 0.002850119225822934, 0.0012536932639934477, 0.007881026434075272, 0.00059697477431924 ], "mean_node_assignment_entropy": 2.3429858252739924, "normalized_node_entropy": 0.5633690051659874, "effective_mass_slots": 10.47464853838594, "node_slot_mutual_information_nats": 0.0059720875499240655, "normalized_node_slot_mutual_information": 0.0014359835153383617, "centered_token_effective_rank": 1.0387174582822263, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.0117138073030662, "token_RMS": 0.31302608019438044, "mean_token_offdiagonal_cosine": 0.9986569877952509, "temperature": 0.427405446767807 }, { "layer": 8, "head": 5, "slices": 64, "native_nodes": 33555, "mass": [ 0.006432584612785042, 0.004266538671194961, 0.006570355934505157, 0.033775286735793886, 0.01670593771389088, 0.004067222123567057, 0.014025382310065457, 0.00940271762864851, 0.015399625621605045, 0.004980133727277254, 0.0056697526652043405, 0.023578736571545945, 0.010122476941003182, 0.013681275548388138, 0.0042677520247304795, 0.015780064323035806, 0.02387839487423939, 0.006804507625050706, 0.017310991482716205, 0.009387362733507739, 0.009524531955058467, 0.0052700648895980945, 0.00940697939630424, 0.047044181423945924, 0.02190441152994314, 0.004506674980307022, 0.005683636120261716, 0.01794799925242357, 0.020867965999957952, 0.005847835881423775, 0.007425010576205848, 0.016156189010849503, 0.029868556170292323, 0.005896158557775998, 0.008921812972154636, 0.0673076002807371, 0.08316598894445645, 0.003689426010272783, 0.008020126468522385, 0.03344793386419872, 0.005339521454638024, 0.028664591897630404, 0.010352297149922779, 0.005015114027656522, 0.027365719937670695, 0.01211298222712825, 0.019949128247065864, 0.0060822059346673524, 0.011800079422623625, 0.019658238182518526, 0.011589118732341247, 0.007219738608958638, 0.008978570852250487, 0.004076927682464506, 0.009987143205647815, 0.010121331808525437, 0.0535440104108039, 0.00704920849381363, 0.009958205819285372, 0.00873355385540002, 0.02918899974609084, 0.011075971222222696, 0.012065089470953206, 0.012062067467085946 ], "mean_node_assignment_entropy": 3.83454373853969, "normalized_node_entropy": 0.9220128726105061, "effective_mass_slots": 46.35923058864167, "node_slot_mutual_information_nats": 0.0018766833455119425, "normalized_node_slot_mutual_information": 0.00045124695931483144, "centered_token_effective_rank": 1.3821358320275592, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.0036900247253768828, "token_RMS": 0.3261432935523687, "mean_token_offdiagonal_cosine": 0.9998835699192988, "temperature": 0.45632192492485046 }, { "layer": 8, "head": 6, "slices": 64, "native_nodes": 33555, "mass": [ 0.0002720673641187088, 0.01652317253880801, 0.00015495199184852303, 0.0002854339741352896, 0.00022537458638564577, 0.05302204173399659, 5.648161190679918e-05, 0.00019669768811326563, 0.0002046896960747844, 0.00010438263399951633, 8.242848675026958e-05, 0.00020703696553076899, 0.00014076950240778528, 0.00014917761825495254, 0.0001646716735032972, 0.0002483139793139825, 0.00018459458685757922, 0.025761376515545997, 7.401292145298388e-05, 0.0002950486881420643, 0.0004779608175373603, 0.13126411565906013, 0.22210550021772643, 0.00014261255600541925, 0.0002058976098602326, 0.021538181263898493, 0.00022163399135287435, 0.00024332177753221562, 0.00016848775432755272, 0.00013121947197254258, 0.000548660349769865, 0.00021972130818439324, 0.00018956127135249818, 0.16552001649355358, 0.0001880267753326495, 0.00019709791290804698, 0.0001764691636933688, 7.940540026561695e-05, 5.862445087715432e-05, 0.00040374938178120004, 0.15986864324423192, 0.0003039462833543952, 0.0012249536788540142, 0.03568053405025271, 0.0005828325171309831, 0.0002206259820289852, 0.0002424593466216641, 0.00027344066799310084, 0.00036251284069403305, 0.00022661015976982525, 0.0003509025759333892, 0.000550321296809846, 0.14597913257243683, 0.00868758012631814, 0.00035983898419442536, 0.00035931361525100183, 0.00023729597324445914, 0.00026255154996624515, 0.00013152368701770123, 0.00027320979956170147, 0.00016411515930734306, 0.0001780342578334676, 0.0006317376962164798, 0.00041489582510103435 ], "mean_node_assignment_entropy": 2.1424969215881626, "normalized_node_entropy": 0.5151616139825188, "effective_mass_slots": 8.551399487297362, "node_slot_mutual_information_nats": 0.0035980306907883453, "normalized_node_slot_mutual_information": 0.0008651435057611062, "centered_token_effective_rank": 1.0607815739348028, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.012734255125056075, "token_RMS": 0.3179358561462731, "mean_token_offdiagonal_cosine": 0.9984668683622386, "temperature": 0.4390506446361542 }, { "layer": 8, "head": 7, "slices": 64, "native_nodes": 33555, "mass": [ 0.009789892068295681, 0.006135343177242124, 0.006497863981072895, 0.01240876356142277, 0.024099316965328424, 0.00318708891194603, 0.02360424376899831, 0.016337802988381157, 0.01443880547527482, 0.009173931104257745, 0.011500571645000282, 0.015369373507439122, 0.009006014315270174, 0.03230767408514934, 0.015926822229599236, 0.012435454680584232, 0.02313137265870653, 0.006860817641931854, 0.016822204819594965, 0.018751367354953436, 0.022061013697083545, 0.006122917884405946, 0.005507521734078489, 0.02036899707738475, 0.016022835765686205, 0.008497589235525007, 0.016337772241413744, 0.014951478615108971, 0.0143657831139857, 0.03291104691961566, 0.03492724748747153, 0.014388826952403712, 0.012585601806079972, 0.007910981221989657, 0.030136224091099638, 0.01577278709802635, 0.017552799596797466, 0.00997953153264059, 0.06323517511599182, 0.018656942356811375, 0.005618145545347004, 0.01895161040244959, 0.014693242419028696, 0.006042685059267396, 0.006641383750624805, 0.010907890409399938, 0.01183942451181048, 0.012966073717965826, 0.014055823891824288, 0.01816885159043544, 0.019040134844399745, 0.011230743001974956, 0.007778815669066579, 0.012250791581003699, 0.016159525429911833, 0.015797474398274232, 0.017296332124364773, 0.009370632954272108, 0.010270253299940735, 0.017576353250157724, 0.010538446224477648, 0.02760895837463062, 0.009479902683023261, 0.025638706532225507 ], "mean_node_assignment_entropy": 4.013927481270278, "normalized_node_entropy": 0.965145545286093, "effective_mass_slots": 55.459823914999916, "node_slot_mutual_information_nats": 0.0017313839213279891, "normalized_node_slot_mutual_information": 0.0004163098328624629, "centered_token_effective_rank": 1.144214623527345, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.0046180504358116345, "token_RMS": 0.24858707325057033, "mean_token_offdiagonal_cosine": 0.999658420338961, "temperature": 0.46153876185417175 }, { "layer": 8, "head": 8, "slices": 64, "native_nodes": 33555, "mass": [ 0.00016526618968153346, 0.0002464781916244461, 0.0006625701389671695, 0.0003362649192510323, 0.0001888360150633087, 0.00023338212516589232, 0.0007965302518602933, 0.010019653685909414, 0.00017799175120727427, 0.000604865028755323, 0.004740095962467059, 0.00022315080040272224, 0.002460287430605652, 0.0004340011083704896, 0.45210349723960935, 0.000190836675096311, 0.00018075341137347158, 0.0003801653487628653, 0.0004179945957268662, 0.0004858703377082871, 0.05310755112026491, 0.007501369749708372, 0.00041591260154814696, 0.00018942068858715595, 0.00013005175231750385, 0.0003431080595198897, 0.013793548778828548, 0.00011547531182246118, 0.00024341753942028703, 0.06082367776543226, 0.05273372211675764, 0.00018121620019988162, 0.00011442761147152128, 0.0006763873688208147, 0.005940998188089377, 0.0001406572952131842, 0.0003291636771408007, 0.0005684635269015277, 0.016703664977589187, 0.00017895153170072872, 0.0005213099036886433, 0.00037500665395889354, 0.04494504188060441, 0.00040093980426407434, 0.00020063334465604993, 0.0002654216607775572, 0.00015443807826318818, 0.018875194767484754, 0.00020744404971025118, 0.00016664582256679204, 0.00018575141570443957, 0.00036192621993311186, 0.0008662539418774829, 0.23996870013387134, 0.0004939093617673474, 0.0006879619548661097, 0.00011772660624764152, 0.0003188875626281446, 0.00022629089941104258, 0.0003359155881315859, 0.00016704697745099732, 0.00035657796951872204, 0.00020404880071363344, 0.00031724959217327854 ], "mean_node_assignment_entropy": 1.8024544874341986, "normalized_node_entropy": 0.4333986917415628, "effective_mass_slots": 6.088108874728939, "node_slot_mutual_information_nats": 0.003883016462147504, "normalized_node_slot_mutual_information": 0.000933668098938402, "centered_token_effective_rank": 1.049433883998863, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.009445203620535641, "token_RMS": 0.3630399739605084, "mean_token_offdiagonal_cosine": 0.9994023628626794, "temperature": 0.42867615818977356 } ] }, { "variant": "S064", "sample_id": "normal_rodv2_0150__gas_inertia_30deg", "native_nodes": 33555, "layers": 16, "slices": 64, "parameters": 7626372, "checkpoint_sha256": "eaf499445d4b00bf103f6df0d2623cc490bc80d72e38ce701bf1e279a6f3df20", "metrics": [ { "layer": 1, "head": 1, "slices": 64, "native_nodes": 33555, "mass": [ 0.014813875112276801, 0.015056277727540302, 0.01546750988579777, 0.015481928484704479, 0.015137646048139787, 0.014870026658217388, 0.01413033830340285, 0.016494422099153236, 0.01759833455643077, 0.015197808935077016, 0.015144494120511758, 0.016309048039465365, 0.015567054438682707, 0.015357791358748521, 0.015537695072656807, 0.015354414099546805, 0.015681823657059824, 0.01571471439156864, 0.015493155299263879, 0.016063189976285926, 0.017262414779146043, 0.01582057075098994, 0.015518877617964853, 0.016142443730880973, 0.016025955864812414, 0.017170403593150012, 0.01622952562407816, 0.014432045669940061, 0.014916536663515284, 0.014978217058529879, 0.015265251295492575, 0.015831097085825457, 0.016095783454361173, 0.01409004094180998, 0.016118434342874747, 0.015575038683101826, 0.014818518527458302, 0.015623646691655805, 0.016410551654268458, 0.015627804375527698, 0.015480432311199787, 0.01542667727484453, 0.016157344700675612, 0.01588600686472506, 0.015375085157496157, 0.01693808150802026, 0.015834472597676733, 0.014983434633148834, 0.015103622603311639, 0.016096040139519062, 0.014287784978782895, 0.01564024350801717, 0.015285397645359716, 0.01747109243681513, 0.016398355779183228, 0.01633240703408426, 0.015509958287795101, 0.0147666974875517, 0.015392581794442585, 0.015932211090079668, 0.013868234772804135, 0.01494898550118571, 0.016127940258304038, 0.016332205139725598 ], "mean_node_assignment_entropy": 4.156899898848989, "normalized_node_entropy": 0.9995231449235451, "effective_mass_slots": 63.92399087220639, "node_slot_mutual_information_nats": 0.000794836082523176, "normalized_node_slot_mutual_information": 0.00019111767909596615, "centered_token_effective_rank": 1.5776641090544798, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.00412367879038308, "token_RMS": 0.2300708266372899, "mean_token_offdiagonal_cosine": 0.9996820906935482, "temperature": 0.494767963886261 }, { "layer": 1, "head": 2, "slices": 64, "native_nodes": 33555, "mass": [ 0.014205187488074736, 0.014994657111278262, 0.015910213590034642, 0.01600318192767326, 0.015393744274444301, 0.01614273290749794, 0.016260523612081846, 0.014673249958289255, 0.01646615431076847, 0.014662169342167535, 0.01449520488222027, 0.016295772064169125, 0.014887886900621846, 0.014347139193124503, 0.01691876822126958, 0.017516457982446804, 0.015383791330202052, 0.016066036901251204, 0.016101713014111275, 0.015953885212556168, 0.015255321267530968, 0.014365390097607783, 0.015142042730117392, 0.016582580409819716, 0.015379791039012616, 0.014694094882742061, 0.01567081421221791, 0.01490847574961317, 0.01710979521689228, 0.015969250251410962, 0.016189378159124287, 0.014352146716436753, 0.015529455864729006, 0.01600460841698896, 0.015268461916036709, 0.016169772458817427, 0.017096649488258505, 0.01682839138141076, 0.017105862280461317, 0.015997480388107384, 0.014573058761815043, 0.013447135574632741, 0.015430872130741033, 0.015574811228316856, 0.01556664675176077, 0.01544198502605078, 0.01665302140058428, 0.0159072226837928, 0.016124454838148956, 0.015025270568611299, 0.015227295291314332, 0.017555306008858032, 0.01461726281072513, 0.014923192648073588, 0.015061903232299185, 0.014452310467413385, 0.01649127456313353, 0.015332895850216226, 0.016773850460400915, 0.015521651764879496, 0.01588324692206866, 0.01475559718664855, 0.014617966668273143, 0.016743533997854068 ], "mean_node_assignment_entropy": 4.156820548727787, "normalized_node_entropy": 0.9995040652524865, "effective_mass_slots": 63.89618943258539, "node_slot_mutual_information_nats": 0.000439177584692807, "normalized_node_slot_mutual_information": 0.00010559988725098423, "centered_token_effective_rank": 2.135081534284321, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.00262596633681738, "token_RMS": 0.2466460547068915, "mean_token_offdiagonal_cosine": 0.9998883413639639, "temperature": 0.4980335831642151 }, { "layer": 1, "head": 3, "slices": 64, "native_nodes": 33555, "mass": [ 0.015528313641819003, 0.016819698290435607, 0.01511641078648167, 0.017196688500434636, 0.01672377536874617, 0.016949734749554735, 0.016703800339045814, 0.015230326601135501, 0.016014427711820527, 0.015133157138582393, 0.014114382678834102, 0.01594351229587196, 0.015752288010823463, 0.01432449522309839, 0.016193381467932092, 0.01557755128019343, 0.015575455752887069, 0.014809598829117072, 0.01539304227562797, 0.01515665668480719, 0.015360961417620245, 0.016256623299501814, 0.015654379388531457, 0.016843046136300346, 0.014312140796971363, 0.014433210753176415, 0.015183434863492232, 0.01626381592930802, 0.01574609369020662, 0.016484980668021624, 0.017268911561416748, 0.01513813393502193, 0.01422885112567677, 0.016635808631258187, 0.014977064457513983, 0.015694016404172388, 0.015114351872878207, 0.015990570824909157, 0.01606243949196428, 0.015127520126723182, 0.01649827638302665, 0.015638523303181034, 0.015432950531176512, 0.015988061249821228, 0.014932000386842165, 0.015914883146889873, 0.01575318531490859, 0.015780638964995774, 0.016422530806857513, 0.014796723740297767, 0.016356663829521414, 0.015445325271487502, 0.01643494847163307, 0.014201673081708397, 0.01497385601757749, 0.015198680403543965, 0.01434263265274677, 0.01695025666441861, 0.015838449299940542, 0.016586433411490506, 0.015873513843893624, 0.014584040643144669, 0.014754417705811885, 0.014268281847191913 ], "mean_node_assignment_entropy": 4.156661279255223, "normalized_node_entropy": 0.9994657690394476, "effective_mass_slots": 63.91263542525978, "node_slot_mutual_information_nats": 0.0008558000652900688, "normalized_node_slot_mutual_information": 0.00020577641836440558, "centered_token_effective_rank": 1.863568672421625, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.003971780208045802, "token_RMS": 0.22663704492327708, "mean_token_offdiagonal_cosine": 0.9996906221595183, "temperature": 0.49933868646621704 }, { "layer": 1, "head": 4, "slices": 64, "native_nodes": 33555, "mass": [ 0.01587545280883927, 0.017313254223876962, 0.01666292174440474, 0.015604548485325978, 0.014695612326566358, 0.015056620914760475, 0.015543517357979379, 0.015080708371578865, 0.01604735173652485, 0.01508951896990048, 0.01610445134728068, 0.017395292512623967, 0.017028560353083044, 0.01705349824677044, 0.016350247392952974, 0.014919899498721585, 0.016293967740157556, 0.014512125684736366, 0.01587111109151888, 0.014892671482351369, 0.014793137278124155, 0.015230038234079459, 0.01583278776514088, 0.015604146996341205, 0.014023924694259029, 0.017394921125938767, 0.016187972714340405, 0.014001174388150002, 0.014882759580499963, 0.01767814926405659, 0.015239442608203623, 0.016005949407617206, 0.016531688885935647, 0.01669760803447659, 0.015502235486175332, 0.014658057046247886, 0.016186860262944622, 0.015287780181109692, 0.015637062085622284, 0.013793211513909581, 0.01692822364599818, 0.014714992322491748, 0.015906593946354512, 0.017332531625524023, 0.0152702500275416, 0.014135236804909374, 0.017118123321814928, 0.016030728042951695, 0.015969243768650904, 0.014580972220250737, 0.015089933342777605, 0.016008139164645666, 0.015322523319189213, 0.01588831975811299, 0.013565009630463807, 0.013948278528469017, 0.015133454997829105, 0.014733775533489287, 0.016641909352832197, 0.016658938411529427, 0.015389382219403613, 0.016058469343319316, 0.014580733632032428, 0.014433997209447406 ], "mean_node_assignment_entropy": 4.1557521217599716, "normalized_node_entropy": 0.9992471628711499, "effective_mass_slots": 63.87124132111026, "node_slot_mutual_information_nats": 0.0011170807446019282, "normalized_node_slot_mutual_information": 0.00026860114175162545, "centered_token_effective_rank": 1.531502939344212, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.005726023153592477, "token_RMS": 0.28680317539638195, "mean_token_offdiagonal_cosine": 0.9996112996462063, "temperature": 0.4990132451057434 }, { "layer": 1, "head": 5, "slices": 64, "native_nodes": 33555, "mass": [ 0.015257995202193723, 0.01528918762301221, 0.015932244684970728, 0.015956681603161976, 0.014933984491886964, 0.015844840613752936, 0.016095932979969915, 0.01591483418444082, 0.014965784076238785, 0.015702343481378982, 0.015323346066287956, 0.016522603099181805, 0.016035092414600043, 0.016483250629607017, 0.015791495080030884, 0.016157902806004775, 0.014462377378394273, 0.016206534615755438, 0.015404556425391847, 0.014721882523977974, 0.015755568196154125, 0.016828418961381826, 0.014313109777840038, 0.015103088744815132, 0.016069761362893228, 0.015054469453616366, 0.015395860571284893, 0.016339025093982756, 0.01575976712969199, 0.014892085825000527, 0.015979229199550596, 0.016257270127301722, 0.01571210761163331, 0.016509832027284288, 0.01496881731939736, 0.016426076113482718, 0.01694899503315944, 0.014561664416632225, 0.01672910506302923, 0.01666902596958902, 0.01581586841005355, 0.014641149827524645, 0.015673938073992793, 0.015452272008708702, 0.01628552509549211, 0.01525228858197247, 0.016235495365616543, 0.01554933632798232, 0.014653304996139076, 0.015852532600044468, 0.014410493861800058, 0.016487984872123218, 0.01490229674318125, 0.01463973604360821, 0.014499638157028934, 0.015370192093464652, 0.016275883208215904, 0.015385623043923437, 0.01581331420905756, 0.014923320872027213, 0.015457981645021792, 0.015461147000707576, 0.015944700494781635, 0.01573982860675411 ], "mean_node_assignment_entropy": 4.157291385853926, "normalized_node_entropy": 0.999617277650311, "effective_mass_slots": 63.941963978744454, "node_slot_mutual_information_nats": 0.0006844732701418721, "normalized_node_slot_mutual_information": 0.00016458103207578847, "centered_token_effective_rank": 2.4183542051080713, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.003610848102207068, "token_RMS": 0.33530401172850754, "mean_token_offdiagonal_cosine": 0.9999005880100135, "temperature": 0.5018079280853271 }, { "layer": 1, "head": 6, "slices": 64, "native_nodes": 33555, "mass": [ 0.015744863956030933, 0.016591928382543617, 0.015065060904761075, 0.016414511236683426, 0.013848331539039995, 0.015248249694088846, 0.015333331201484751, 0.014972304744081693, 0.01973376865314844, 0.015771102249475796, 0.015934228126807275, 0.013938288965278154, 0.016889672569298846, 0.013233471239262376, 0.014866408247252527, 0.015227917808861166, 0.01755990244485932, 0.015405544874246821, 0.01502332514824437, 0.01609298616521157, 0.016966965683848177, 0.014894081950254295, 0.015330105024536887, 0.016622553951816246, 0.014506130520381492, 0.01636976177926711, 0.016011271075678776, 0.0152470488998289, 0.016271659609104098, 0.016315941123569124, 0.016775646936295508, 0.015171202602212705, 0.015299295011761823, 0.016125393992576054, 0.016576794351293868, 0.013609268131588588, 0.014244602364810599, 0.01733595098000652, 0.015555206803634659, 0.01564181024130118, 0.01621960734940467, 0.01649212120365917, 0.01600889341330905, 0.016249768485137095, 0.01500476731514499, 0.015492278111885138, 0.01602977513631376, 0.015749843974172128, 0.015276836536129716, 0.014686291421204473, 0.015357347744174391, 0.014843051162573482, 0.015582761434235058, 0.014212958630595721, 0.016604364483731458, 0.014361703645694968, 0.01650219433908335, 0.01451854001119513, 0.01476203069568786, 0.01605938614642095, 0.014123095766308278, 0.016001086481578337, 0.014345749223671688, 0.017749657928405185 ], "mean_node_assignment_entropy": 4.155623290937571, "normalized_node_entropy": 0.9992161856063847, "effective_mass_slots": 63.846268711843905, "node_slot_mutual_information_nats": 0.0008548514910868832, "normalized_node_slot_mutual_information": 0.00020554833448126374, "centered_token_effective_rank": 2.1240386859840483, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.004352321747114685, "token_RMS": 0.22414422209871532, "mean_token_offdiagonal_cosine": 0.9996435228053292, "temperature": 0.4994603097438812 }, { "layer": 1, "head": 7, "slices": 64, "native_nodes": 33555, "mass": [ 0.014979918812770875, 0.015326706313053091, 0.014970554456652107, 0.01537687366806396, 0.013912635175237853, 0.015235456006193176, 0.01493024497656944, 0.016623142924986314, 0.016397509034104106, 0.015424196949447078, 0.016792857947224626, 0.01607347394379569, 0.014312396610896486, 0.016636665978926497, 0.0155887518756146, 0.015337765757663383, 0.015709635772886796, 0.016664957195729118, 0.016298611125863133, 0.015390261449197324, 0.01631841436347664, 0.016263172699402473, 0.015388727176758514, 0.014039422547512146, 0.016634324556287373, 0.01607833275747155, 0.01620424916836468, 0.013594153112197726, 0.014045946024338288, 0.015895900435911138, 0.014169921470863456, 0.015988484643024468, 0.016837507075734347, 0.016290757566526134, 0.015624373689440408, 0.014690842120066527, 0.015141456778368142, 0.014049395001983966, 0.014985526971587827, 0.01732134985489233, 0.016328776162908742, 0.014125532632317011, 0.016089461410548324, 0.01870323515892242, 0.01575623399973129, 0.014533820633256493, 0.016877919192588412, 0.016751253309380303, 0.014146798159446966, 0.017391588465361842, 0.014187365022034559, 0.014873972588839492, 0.012577437142454472, 0.01837830186320957, 0.01545368456285191, 0.015787056401108838, 0.01611346059447317, 0.01394615412383468, 0.01801827761422742, 0.016147917326339363, 0.014679016446440218, 0.014784034620790079, 0.016726331090361143, 0.016077497643030732 ], "mean_node_assignment_entropy": 4.154875584037797, "normalized_node_entropy": 0.9990364001003277, "effective_mass_slots": 63.81924919832369, "node_slot_mutual_information_nats": 0.001179272379634888, "normalized_node_slot_mutual_information": 0.0002835550689927634, "centered_token_effective_rank": 2.2795140867301154, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.004487007161162228, "token_RMS": 0.25823058198189913, "mean_token_offdiagonal_cosine": 0.9996975039342727, "temperature": 0.4932496249675751 }, { "layer": 1, "head": 8, "slices": 64, "native_nodes": 33555, "mass": [ 0.01632549038295365, 0.015050825097733719, 0.014199845769391687, 0.016282526873170913, 0.016605058145413604, 0.01647336821164367, 0.014952433452336555, 0.015209696304612056, 0.015629324950072417, 0.015940068793170694, 0.016892631986818073, 0.0153484996462069, 0.01594460146067848, 0.016748524324435837, 0.016198533834094902, 0.014686644281079013, 0.015604686014733919, 0.016627433541881852, 0.015218360575323728, 0.015132210435377132, 0.017584599868337953, 0.016337171345399623, 0.015412076728313278, 0.01538874647196928, 0.013366257126460158, 0.015411675380907298, 0.014432571479899972, 0.016956624628054796, 0.0166262877086432, 0.016603719373306472, 0.016441980445949852, 0.015985819908457644, 0.013319022072979553, 0.015719298927318524, 0.014174786344200547, 0.014928335478303686, 0.01624732938939418, 0.016309825399416764, 0.014988901500242235, 0.014281563567933827, 0.017843354229400782, 0.015703314418316492, 0.015005889568145362, 0.015901637245027894, 0.015811662836883768, 0.014573320853917339, 0.015658606858425825, 0.014584153605203988, 0.015637832809022582, 0.01688119774095069, 0.01668392617020641, 0.016275589389890554, 0.015629095647102707, 0.015288341902040979, 0.015293479722974073, 0.01622416265161755, 0.01467919619208569, 0.014088822325691303, 0.015280763470074971, 0.015446569793500287, 0.015167468572108983, 0.015708979470048873, 0.014311427483231056, 0.016733849742436635 ], "mean_node_assignment_entropy": 4.15605103580332, "normalized_node_entropy": 0.999319036505815, "effective_mass_slots": 63.88525316667718, "node_slot_mutual_information_nats": 0.0010375190821916291, "normalized_node_slot_mutual_information": 0.0002494706057842554, "centered_token_effective_rank": 2.07817117841267, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.004448497455820578, "token_RMS": 0.23148537801423766, "mean_token_offdiagonal_cosine": 0.9996501757767974, "temperature": 0.4987735152244568 }, { "layer": 2, "head": 1, "slices": 64, "native_nodes": 33555, "mass": [ 0.016019576327852218, 0.01651139026736715, 0.015198074080851559, 0.015731315470475887, 0.01576586938068335, 0.016890179873791773, 0.01587165634773284, 0.015870646917488913, 0.017375032071453272, 0.014518876473047867, 0.01420398106210954, 0.01651100320851167, 0.015951133507060063, 0.016001074281127712, 0.014799794748690823, 0.014317327909196006, 0.018065251713648336, 0.015654031775149055, 0.015197742348052523, 0.01655609778962189, 0.01387746774688232, 0.015771724821061824, 0.015101711738578761, 0.015816294332496313, 0.014615577030657719, 0.016509542886193853, 0.014766753727195204, 0.014869708004817666, 0.016831192669543756, 0.014984300866475486, 0.014863381150018048, 0.0173801617344669, 0.015293901597362413, 0.015170474127051577, 0.016646167602472763, 0.015238940376872198, 0.015389638564644726, 0.014459073807388619, 0.015391644063380575, 0.016723144416950032, 0.014348858195553947, 0.01607699945528516, 0.015150176911808982, 0.01455641792058243, 0.015434680500897233, 0.016145456357102888, 0.01488019801266053, 0.016656039858820044, 0.01562606424144316, 0.016252741123551167, 0.01556206846587785, 0.015110998328776833, 0.01594212856449019, 0.014616791050364347, 0.017106296181192502, 0.01521486229125997, 0.014999510983379583, 0.01660946568485553, 0.01643434131273484, 0.016013301624702086, 0.01517106001304898, 0.015658068086359677, 0.015862705241658392, 0.013859912554600647 ], "mean_node_assignment_entropy": 4.156304510508534, "normalized_node_entropy": 0.9993799842891821, "effective_mass_slots": 63.89681006131827, "node_slot_mutual_information_nats": 0.0009649288357298502, "normalized_node_slot_mutual_information": 0.0002320163410197027, "centered_token_effective_rank": 1.4153330829790047, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.006080991899682919, "token_RMS": 0.21050563652716664, "mean_token_offdiagonal_cosine": 0.9992143435416252, "temperature": 0.5001745820045471 }, { "layer": 2, "head": 2, "slices": 64, "native_nodes": 33555, "mass": [ 0.014809139407954686, 0.016754907680665117, 0.015602230647421464, 0.01645053517472511, 0.01540528711516062, 0.016318267656509913, 0.015873807074365833, 0.01605572648232073, 0.01669879018915688, 0.014993436937376517, 0.014887621730173013, 0.01525567820257667, 0.015294257435221028, 0.01656558904286145, 0.014566496791654728, 0.014853898204325732, 0.015231916861950846, 0.013428718304574365, 0.014906021974167051, 0.014877265308431436, 0.014316055036591642, 0.014714668301115792, 0.015660868826662396, 0.016402527110629207, 0.017341699280680745, 0.01599968822636604, 0.016692650844771128, 0.015362065169427917, 0.016544794119786814, 0.014975561596638595, 0.014452661536028907, 0.017268574853070918, 0.015761082726785385, 0.013793981158191367, 0.01679991107753773, 0.016040805261429484, 0.015318003586781946, 0.016697397783360882, 0.013987383128434859, 0.01592997911212306, 0.016514801754526135, 0.016128220596057775, 0.01581211740448305, 0.01629288452737796, 0.015509898116391065, 0.014267881959898783, 0.015001182814424705, 0.01596360377295129, 0.013906230696373886, 0.017141646007249375, 0.015850145050251365, 0.01496045129225735, 0.01588938607960704, 0.015557128025934648, 0.016693119765368265, 0.015733672034496646, 0.014859729945896112, 0.01575905540912251, 0.014885046509125206, 0.01660042975273862, 0.015546124294231464, 0.01598094769778079, 0.01632276563107914, 0.015933579728984296 ], "mean_node_assignment_entropy": 4.156120758167332, "normalized_node_entropy": 0.9993358011906148, "effective_mass_slots": 63.897414762161446, "node_slot_mutual_information_nats": 0.0011581448415691753, "normalized_node_slot_mutual_information": 0.00027847496992716394, "centered_token_effective_rank": 1.4801589194833986, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.006846597839312819, "token_RMS": 0.1836808414164069, "mean_token_offdiagonal_cosine": 0.9987083115298784, "temperature": 0.4983869791030884 }, { "layer": 2, "head": 3, "slices": 64, "native_nodes": 33555, "mass": [ 0.014372584310176652, 0.015213634301020846, 0.016677904163527917, 0.014144678618062264, 0.016125173716205116, 0.014093217828076912, 0.013628343430019814, 0.015205473952259602, 0.015499575692683462, 0.016392461148678653, 0.015988135740056583, 0.01556605200219809, 0.014545041176744029, 0.015823427388743466, 0.015318197542847295, 0.01747654426560936, 0.014888303971711578, 0.016243450655354204, 0.015765548087273477, 0.015320929209046003, 0.01572360730836993, 0.014015881306623767, 0.014132500676050474, 0.014924602997573233, 0.015672889861234243, 0.0164982239624857, 0.015045894242128136, 0.01662301225097848, 0.0157678780102784, 0.016627712394071306, 0.015929317263648473, 0.01333595506489974, 0.01542386787300595, 0.014098838965403369, 0.015063694473192253, 0.017201348600625735, 0.014579924517492258, 0.016235390144178675, 0.01559487112491519, 0.014581604870714856, 0.015790594881504548, 0.0150988891172886, 0.016819902927803623, 0.01614838347600551, 0.017606943860847678, 0.0166580545599555, 0.014631993250485337, 0.015494417721932139, 0.015287788686412064, 0.016162545011119175, 0.01680336547386822, 0.015454990066023942, 0.015400038094201096, 0.016067439718958812, 0.015241743623433215, 0.015252374069308084, 0.015543564657675556, 0.01763090219186579, 0.015924797037283587, 0.01580610465166213, 0.016331291732044825, 0.017295107016604406, 0.01491165565183316, 0.017271419168334617 ], "mean_node_assignment_entropy": 4.1561291745133015, "normalized_node_entropy": 0.999337824894047, "effective_mass_slots": 63.87531647218433, "node_slot_mutual_information_nats": 0.0008038285530380662, "normalized_node_slot_mutual_information": 0.00019327991119882823, "centered_token_effective_rank": 1.9998647366697861, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.005664781363301568, "token_RMS": 0.2531656826173679, "mean_token_offdiagonal_cosine": 0.9995006945453648, "temperature": 0.5004897713661194 }, { "layer": 2, "head": 4, "slices": 64, "native_nodes": 33555, "mass": [ 0.016498151369950195, 0.01619700380163707, 0.016211084026641807, 0.01621570690168503, 0.015884457408050733, 0.016697333916503285, 0.015734126590882865, 0.016353493597346808, 0.016879682402184636, 0.015320288708577557, 0.015465433605102984, 0.015535681725776226, 0.015404809931868494, 0.015917045472548103, 0.016313351578444282, 0.015223734961599756, 0.016664460950258093, 0.01668650767642568, 0.015073134696865744, 0.016351431317853913, 0.015003657917199414, 0.015099885603217312, 0.014508670250115123, 0.015532070475684168, 0.015408827115703475, 0.01608691475749784, 0.014595177614260112, 0.015503313359788492, 0.01576325235087072, 0.016055648649426966, 0.015787220809560402, 0.015019394659841492, 0.015381266426135728, 0.015690212845086225, 0.015227757551102405, 0.014129067993982504, 0.015579233759817948, 0.014863412918123558, 0.015469237857983527, 0.014958768681878527, 0.014813344834771798, 0.015418277198795438, 0.015318333548749724, 0.014733326693988738, 0.015854914448300474, 0.015714220755882455, 0.015303410731801118, 0.015392792000761555, 0.016388221754635737, 0.017034864548281237, 0.01577188013516441, 0.01570264401535748, 0.01538144777508264, 0.015759131705299332, 0.01737783595702009, 0.015206643976681929, 0.01546762330834205, 0.015762982953041724, 0.01620981918583771, 0.01501323712911696, 0.015786825378932366, 0.014950357107111149, 0.015174620345511312, 0.014173332347548891 ], "mean_node_assignment_entropy": 4.156485014686574, "normalized_node_entropy": 0.9994233863696018, "effective_mass_slots": 63.94344843764346, "node_slot_mutual_information_nats": 0.0015140598906260294, "normalized_node_slot_mutual_information": 0.00036405444930250985, "centered_token_effective_rank": 1.5341023487282843, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.006904330771102886, "token_RMS": 0.22201900552140463, "mean_token_offdiagonal_cosine": 0.9990536115303807, "temperature": 0.49961012601852417 }, { "layer": 2, "head": 5, "slices": 64, "native_nodes": 33555, "mass": [ 0.017213082410125365, 0.01590674854540231, 0.016408379982133868, 0.015525276430512954, 0.01636713283341042, 0.01499711320355408, 0.013720381078399822, 0.016907640565572535, 0.01634394707905653, 0.016898179462399084, 0.016911814868729936, 0.015425114343343895, 0.015324302477174527, 0.01605964628881033, 0.016480531374575697, 0.016152349208285798, 0.017380202060609484, 0.01654012679391474, 0.015275438285385988, 0.01839815862213152, 0.016007653489540148, 0.015541725583272362, 0.014735468335598464, 0.015490571268051619, 0.01509641089886449, 0.014647261632323008, 0.014823865291381216, 0.014954516115650976, 0.014605656956259903, 0.016453078794947434, 0.015853338556275638, 0.014731805128614498, 0.015026332955513798, 0.015002287219781854, 0.015395536757322843, 0.014415216765217589, 0.015523128394598693, 0.0155213009304338, 0.016242605064249113, 0.013972004994249321, 0.015027681693619009, 0.014767287307280247, 0.01596070210922761, 0.017000886821746605, 0.015473359848095504, 0.016293305384208614, 0.016091818455232006, 0.015700210075597305, 0.01498087639742087, 0.014638661647487764, 0.016402467846817134, 0.015194524943133595, 0.015202157567054668, 0.013913170811337866, 0.015357696816614952, 0.01626807274170594, 0.015293728289708014, 0.015437602781307775, 0.0173173859711576, 0.015124482461843195, 0.014872170655847021, 0.01647883911072129, 0.014282928721546559, 0.014644650427862999 ], "mean_node_assignment_entropy": 4.156325082637679, "normalized_node_entropy": 0.9993849308406317, "effective_mass_slots": 63.889309358819446, "node_slot_mutual_information_nats": 0.0008269620702741065, "normalized_node_slot_mutual_information": 0.000198842346297954, "centered_token_effective_rank": 1.5028493427322218, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.005240153413740438, "token_RMS": 0.17775108459925715, "mean_token_offdiagonal_cosine": 0.9991573361394706, "temperature": 0.5012573003768921 }, { "layer": 2, "head": 6, "slices": 64, "native_nodes": 33555, "mass": [ 0.01618747024836732, 0.014275806395895142, 0.015259779361066246, 0.013871186844990125, 0.015950253423880116, 0.016982919902731836, 0.013105831507939222, 0.014663237556643066, 0.0160432824350959, 0.015509478887314103, 0.01507324288476932, 0.014606531313200711, 0.016255886937255538, 0.015277784445793915, 0.016342766696484233, 0.016705671250775753, 0.014973726019101177, 0.01493386125541483, 0.016941531270719103, 0.01625268053681973, 0.016174620986090273, 0.015595854488257644, 0.01412264929495662, 0.013944107123916088, 0.015586347927577652, 0.01477525544938046, 0.015840709462513192, 0.015729753383424347, 0.016727894272212832, 0.015859208667309152, 0.018142044073215825, 0.015515260800947964, 0.016158159786837812, 0.016164230653493507, 0.015955838021221237, 0.015084692128718725, 0.017374067564013773, 0.01515987563765612, 0.016926777241256123, 0.014625382400558428, 0.01443710285549085, 0.015083657971201259, 0.015671034323445972, 0.015875691967591535, 0.015935006925805396, 0.015327723444256426, 0.016651350833229123, 0.016076275155213572, 0.015475946319009318, 0.014912827071005063, 0.017330970134929704, 0.016153610690166992, 0.01572468706466743, 0.01745391056275198, 0.013530899668391336, 0.01665694817599237, 0.016639391144685146, 0.014147405762232605, 0.014529689157337523, 0.01520744701235434, 0.01564294547134004, 0.01590342257985649, 0.01563868152604646, 0.015317715511043493 ], "mean_node_assignment_entropy": 4.155916747046778, "normalized_node_entropy": 0.9992867468852965, "effective_mass_slots": 63.87152070818981, "node_slot_mutual_information_nats": 0.0009568296716473412, "normalized_node_slot_mutual_information": 0.00023006890371017242, "centered_token_effective_rank": 1.7561202110039251, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.005893848814772404, "token_RMS": 0.22184168795199052, "mean_token_offdiagonal_cosine": 0.9993373736379727, "temperature": 0.4974064230918884 }, { "layer": 2, "head": 7, "slices": 64, "native_nodes": 33555, "mass": [ 0.015551896777096706, 0.015803685788197866, 0.015786794628162504, 0.015465646070521033, 0.014331398607815782, 0.018065392957327796, 0.013704922596652423, 0.016085492415405557, 0.015516132285096547, 0.01674383807133019, 0.01629857913804821, 0.014370869886812314, 0.015420181730895627, 0.01663502190379206, 0.015349074881265724, 0.015647951477696983, 0.01581133566244453, 0.014273449460178006, 0.015952437112393473, 0.0160255456793481, 0.015135284348342009, 0.016729129577226824, 0.015827597944461994, 0.01553179501114415, 0.01523120031537908, 0.015076818885488457, 0.01613507664369286, 0.015494537410385837, 0.016570474281451898, 0.015558823026470244, 0.01608239169686482, 0.0160430289886813, 0.015465243779913295, 0.01635662893030691, 0.017202565873091987, 0.015153595085367238, 0.017531605982785785, 0.01550598156338272, 0.016220476430238644, 0.01693228516661629, 0.015113040307880053, 0.014931955259676893, 0.01567989300385244, 0.014334603689328967, 0.016647874966603495, 0.015485111622864497, 0.0156927473934412, 0.016221231722771554, 0.015098044330970556, 0.016179559928451232, 0.016305263099003754, 0.013762693009568946, 0.01529684584432939, 0.015425021662995066, 0.014824892947154134, 0.015249954728455998, 0.015239970905021013, 0.016390430656914125, 0.01463340457324552, 0.014454038874715936, 0.014129671159420607, 0.015119071962982705, 0.01580556534028652, 0.01535492466781813 ], "mean_node_assignment_entropy": 4.155883338845584, "normalized_node_entropy": 0.9992787139109319, "effective_mass_slots": 63.90667487626452, "node_slot_mutual_information_nats": 0.0015404752400360877, "normalized_node_slot_mutual_information": 0.00037040599823538327, "centered_token_effective_rank": 1.2695872093084952, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.008619925552061657, "token_RMS": 0.20899698794739635, "mean_token_offdiagonal_cosine": 0.9983124586947274, "temperature": 0.49919393658638 }, { "layer": 2, "head": 8, "slices": 64, "native_nodes": 33555, "mass": [ 0.015596513992480488, 0.01557440539909966, 0.016241471656083153, 0.015643745748016157, 0.014848312238463612, 0.015052410540262698, 0.014776805998134935, 0.015220948545568356, 0.016715671857057083, 0.017485113583939244, 0.01677689721677111, 0.013880938212142325, 0.014951916343602307, 0.017116568638770917, 0.017537922881912225, 0.01503045509775151, 0.014127899824045326, 0.015776843523362137, 0.017904881844104152, 0.015639022690585546, 0.017078761941019344, 0.015909391456851083, 0.013388508707096193, 0.014787845856133604, 0.01518936288655315, 0.014440785506786008, 0.015462073825289748, 0.015032934583445994, 0.015638074705371947, 0.01665052892250274, 0.017677694959547956, 0.01415232260869834, 0.015105703178831495, 0.01581127482147632, 0.015571589631655322, 0.014397341394476395, 0.015645541417902475, 0.01606205613819338, 0.015572574681481268, 0.014872171853507591, 0.01633028885514219, 0.014607929698862009, 0.017129609125067228, 0.015881618517328173, 0.01595473524939084, 0.01429809731931036, 0.015681188342854938, 0.014631107234096424, 0.0162955110838531, 0.01525855734555772, 0.017121567969303288, 0.01616655879420431, 0.016309832154981756, 0.01725598688393769, 0.013897495727912268, 0.016325739081968424, 0.01636229311358203, 0.016219387980375893, 0.01457096407365655, 0.014037720122263713, 0.015422010195826369, 0.014827397201033182, 0.015586107150619458, 0.01548301188544165 ], "mean_node_assignment_entropy": 4.155646233606306, "normalized_node_entropy": 0.9992217021521195, "effective_mass_slots": 63.86341791742134, "node_slot_mutual_information_nats": 0.0011004742872460227, "normalized_node_slot_mutual_information": 0.0002646081328059423, "centered_token_effective_rank": 1.3786174536962985, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.005643830671401197, "token_RMS": 0.22092692897022184, "mean_token_offdiagonal_cosine": 0.999387482400934, "temperature": 0.4995062053203583 }, { "layer": 3, "head": 1, "slices": 64, "native_nodes": 33555, "mass": [ 0.016597559957815974, 0.017014676542246844, 0.014820756931276999, 0.015796594350232247, 0.014911909411158822, 0.014711283716814313, 0.016694721416101878, 0.01660381352903346, 0.014443800087272677, 0.015820150241597702, 0.015809662312528998, 0.015157449969333795, 0.015949408274553045, 0.01604508927929883, 0.014565447985408753, 0.01581335440114298, 0.01517020627781871, 0.016714050010542787, 0.016513987724945303, 0.016737509102923754, 0.014407835236215286, 0.014009757119190265, 0.016231523833490576, 0.01605768579772627, 0.01564011496057779, 0.017125771639360975, 0.01583568883140815, 0.015764730990448148, 0.014969606556258112, 0.015265937423190032, 0.015655525968521235, 0.014339039039830971, 0.01655950257131385, 0.017364530481571296, 0.015879375979136672, 0.016096947053337734, 0.015493956020521284, 0.014249999568892155, 0.016699657762277997, 0.014184475299584326, 0.015459043131715191, 0.014908512422684239, 0.01622601431498734, 0.01730844709761868, 0.015352323169152084, 0.013595133064909514, 0.015951965560362512, 0.015875536131831736, 0.016400923694282296, 0.01572529433450282, 0.014589692113998484, 0.0160675161853577, 0.01619885912490613, 0.015772892840205513, 0.01603951044980175, 0.015645863044012803, 0.015432002550320465, 0.015009417723031108, 0.015415107273120466, 0.015096308532982926, 0.015574351859750398, 0.014390273631782543, 0.015250851120674883, 0.01499106705284784 ], "mean_node_assignment_entropy": 4.155871001622694, "normalized_node_entropy": 0.9992757474358851, "effective_mass_slots": 63.90684424665626, "node_slot_mutual_information_nats": 0.001555462736432034, "normalized_node_slot_mutual_information": 0.00037400972935634536, "centered_token_effective_rank": 1.2167309243313524, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.009867415189366518, "token_RMS": 0.1651879345998392, "mean_token_offdiagonal_cosine": 0.9964612642076185, "temperature": 0.49777787923812866 }, { "layer": 3, "head": 2, "slices": 64, "native_nodes": 33555, "mass": [ 0.015480415985563298, 0.0166893287895589, 0.014925203557306494, 0.014962894832342087, 0.01617374233185419, 0.0149687942437469, 0.01583971596639223, 0.0173670044761679, 0.01509696396089665, 0.014297965438427264, 0.01554457552774335, 0.015295457265934722, 0.0160101340701458, 0.016160161901449123, 0.015689288456347646, 0.015014833591977222, 0.015545565635770794, 0.015315011489235454, 0.015633688671890474, 0.015795100139069044, 0.015355955670273947, 0.015898487438307473, 0.01663093484689459, 0.01658683947704243, 0.015295798852739986, 0.01538467888550056, 0.0157522277860186, 0.015254524752199027, 0.015450913065633952, 0.01619516400496276, 0.01612693068703642, 0.01574510169127933, 0.015203176294731688, 0.01617435117426612, 0.014069655348787301, 0.01784569710446459, 0.01369065351250221, 0.014656417422643843, 0.014459074710956088, 0.015332559436740641, 0.01534253837436109, 0.015201320625606255, 0.0154107358595577, 0.016607240409729218, 0.01711279373266821, 0.015179293382495993, 0.015194636648025398, 0.01474502419511939, 0.015146758302606585, 0.01607523706432578, 0.015165748637024717, 0.015227024874734178, 0.016094689731056076, 0.014870219808746301, 0.016973486720750726, 0.016122346720599263, 0.01381514017840573, 0.016841094813393846, 0.01628654253913467, 0.015233493997848499, 0.015555021177959006, 0.015862123463305625, 0.016753363949341765, 0.016269136376894325 ], "mean_node_assignment_entropy": 4.155996818623512, "normalized_node_entropy": 0.9993060000297417, "effective_mass_slots": 63.91547759971854, "node_slot_mutual_information_nats": 0.0015647293884644142, "normalized_node_slot_mutual_information": 0.0003762378881784718, "centered_token_effective_rank": 1.3067029681709847, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.00838706631705592, "token_RMS": 0.1958332544349854, "mean_token_offdiagonal_cosine": 0.9981760863925556, "temperature": 0.4990568161010742 }, { "layer": 3, "head": 3, "slices": 64, "native_nodes": 33555, "mass": [ 0.015091889747921524, 0.014787058001893446, 0.015605245117581587, 0.016005200544415777, 0.015135424576573171, 0.014845078107374265, 0.016024627608504254, 0.014823092200470395, 0.016823691636298394, 0.015951544681355533, 0.015372097499378835, 0.01448046719773909, 0.015818774061408792, 0.014627969740623378, 0.01629236300649781, 0.015051195662368198, 0.015418722094795048, 0.016008064538965396, 0.014823086121935697, 0.016121507174806007, 0.01571505560995263, 0.016162632854613473, 0.014843245950820167, 0.017083109820611588, 0.015260816049460788, 0.01652156120345758, 0.015447075959867345, 0.015981948628661938, 0.015471973645315851, 0.016313965805291222, 0.01579342616227669, 0.014992675586348096, 0.014829505164736323, 0.015979525265290128, 0.014828350610177309, 0.016077999845580112, 0.015696491641365076, 0.016104332968257362, 0.014985072789673495, 0.015456125775504495, 0.01543903666438937, 0.016827263328536953, 0.015705754195197454, 0.015408708785973226, 0.01579201933825834, 0.015213763974679153, 0.014490756721293682, 0.01567560453031719, 0.014409243475466483, 0.015862463169591452, 0.015234100152289454, 0.015302217318484208, 0.01653349529735321, 0.01579229686919374, 0.01560323312706527, 0.015931139012509737, 0.01572001590501035, 0.0153316418138642, 0.015595113118657224, 0.016253201939296597, 0.01580589464093339, 0.01693283257998483, 0.01725296001060365, 0.015235253526756882 ], "mean_node_assignment_entropy": 4.156743165925338, "normalized_node_entropy": 0.9994854586215959, "effective_mass_slots": 63.94553333035289, "node_slot_mutual_information_nats": 0.0012885133795057513, "normalized_node_slot_mutual_information": 0.0003098219771220044, "centered_token_effective_rank": 1.4628592555624733, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.008164398169148927, "token_RMS": 0.22778087755844756, "mean_token_offdiagonal_cosine": 0.9987841787222149, "temperature": 0.5004838705062866 }, { "layer": 3, "head": 4, "slices": 64, "native_nodes": 33555, "mass": [ 0.015843434597657958, 0.01579618302178492, 0.015187160713777707, 0.015539204433361644, 0.015219045209491187, 0.017108285233529398, 0.014974837016073909, 0.014228528322527917, 0.016240950884341066, 0.015454838675993723, 0.015900945937783614, 0.015920356604932736, 0.015607968143782394, 0.01620409539114557, 0.015367932281994089, 0.01656232725997082, 0.014374246584805793, 0.016527848621110654, 0.01600608958094877, 0.0161130813681363, 0.015648614708764413, 0.017485774973288872, 0.01479576535921501, 0.015053852897616456, 0.01474453163735825, 0.015301862396544083, 0.01568046892637748, 0.014632825419831881, 0.016220714450781774, 0.014719154861314502, 0.015676843992213373, 0.015358926834808636, 0.015839876906593536, 0.01609557317432464, 0.014844518540266942, 0.015332038346675243, 0.01630212719856276, 0.01573027541211191, 0.01587232957677223, 0.015493827716521935, 0.0166497164421564, 0.016652325331535277, 0.0162239295744808, 0.014854408340033733, 0.014441910324941012, 0.016454527110628524, 0.0160397838246321, 0.01664899542426485, 0.014945603789371148, 0.016822365810893417, 0.0159220271345687, 0.015653943887246585, 0.015756961398188597, 0.014749751326527694, 0.015185499275881743, 0.014954804600499689, 0.016920613478150343, 0.014726032908007251, 0.01430017444581424, 0.01629453380243147, 0.015027853380569518, 0.015539626229563851, 0.014513192109396144, 0.01571415706596366 ], "mean_node_assignment_entropy": 4.156479607829072, "normalized_node_entropy": 0.9994220862951845, "effective_mass_slots": 63.928862282443966, "node_slot_mutual_information_nats": 0.001291330489689102, "normalized_node_slot_mutual_information": 0.0003104993489371974, "centered_token_effective_rank": 1.361793260541112, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.008129451229070047, "token_RMS": 0.17020429376483184, "mean_token_offdiagonal_cosine": 0.9976899880094929, "temperature": 0.5005087852478027 }, { "layer": 3, "head": 5, "slices": 64, "native_nodes": 33555, "mass": [ 0.014579983121981796, 0.015010705662633423, 0.01597567967070768, 0.015691484154495276, 0.01640977283914462, 0.015255772128352605, 0.01502421680763486, 0.015549357043976952, 0.014927973892251641, 0.01360556969527089, 0.01713574244892645, 0.015716296371925724, 0.015867475512411293, 0.016818775590373175, 0.015841615449616635, 0.017038699941807612, 0.01429777052456097, 0.014612696794652847, 0.016480539877879702, 0.015434665151518834, 0.015540394430052788, 0.01635236565138567, 0.016449476933991337, 0.015138497091402348, 0.013999744733281222, 0.016069893618609027, 0.01688612683743767, 0.014299906951152849, 0.015009629143302894, 0.015423702737925715, 0.015593838753910618, 0.015855998193422996, 0.015740403531566364, 0.016917265763897918, 0.015678161283200872, 0.014872943910901538, 0.015952364269812218, 0.015596705836354531, 0.015570575431196873, 0.014509188504913199, 0.01567233912846607, 0.015863355918869435, 0.01633342924824665, 0.015255190605041505, 0.016622833851427883, 0.01691957002536468, 0.015202545227113494, 0.014729054036034394, 0.014100455416130817, 0.018257985033973054, 0.015774247759927883, 0.01465116075898288, 0.01599531455836705, 0.01575541171234036, 0.01588000005965678, 0.01471773454767637, 0.017215395107701023, 0.015469190653931444, 0.01485080845932117, 0.016100615850172247, 0.01573726962924319, 0.014899234574345305, 0.016202040124757292, 0.015062841037606104 ], "mean_node_assignment_entropy": 4.155966924080788, "normalized_node_entropy": 0.9992988119116522, "effective_mass_slots": 63.90094431494816, "node_slot_mutual_information_nats": 0.001367215205215544, "normalized_node_slot_mutual_information": 0.00032874576606540867, "centered_token_effective_rank": 1.2626028600148536, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.009186036876900146, "token_RMS": 0.21031693151342404, "mean_token_offdiagonal_cosine": 0.9980918620064447, "temperature": 0.49961262941360474 }, { "layer": 3, "head": 6, "slices": 64, "native_nodes": 33555, "mass": [ 0.015130966638660773, 0.016247039080202367, 0.015069190006869114, 0.01670200319472501, 0.015522626742738367, 0.01466872355682966, 0.015494611513215425, 0.016667333125264326, 0.016036071005341032, 0.014502927177892217, 0.015132226207854363, 0.015670434224780528, 0.015479100478451958, 0.01577980082039128, 0.015068425442659851, 0.015836234512247478, 0.014674063552364723, 0.016505090700471618, 0.016400121059532386, 0.015888177924054673, 0.01534297151651748, 0.015786059957423396, 0.015838409033748987, 0.01584893976651979, 0.014644856246321785, 0.016483670608793466, 0.01677921411388856, 0.015440887071313277, 0.014306050327273022, 0.014328208883285896, 0.015858137255639723, 0.014594856166282056, 0.01552194716494057, 0.01522996898670032, 0.015685426833663446, 0.01623987270411725, 0.015624325264246937, 0.014458374530321226, 0.01576122704123424, 0.0159099004334226, 0.016320050317134068, 0.016469482226360264, 0.01707753616047521, 0.015601747985077168, 0.01499593237645601, 0.015183036394373989, 0.015608624240510903, 0.01674300871282647, 0.014638155823146873, 0.016800605732353074, 0.01463095655642875, 0.016923502824324605, 0.01718827240927318, 0.014861931512500043, 0.01603735287259804, 0.01566164844010634, 0.017243341751221244, 0.015165819496836725, 0.014669340815783133, 0.014662597707392668, 0.014370979719150396, 0.01596145935580323, 0.01644616489765066, 0.014579980928690072 ], "mean_node_assignment_entropy": 4.154861686200176, "normalized_node_entropy": 0.9990330583767585, "effective_mass_slots": 63.91855350868756, "node_slot_mutual_information_nats": 0.0027479852878879996, "normalized_node_slot_mutual_information": 0.0006607507912119746, "centered_token_effective_rank": 1.1801115677861145, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.012918039648706869, "token_RMS": 0.1937614735599516, "mean_token_offdiagonal_cosine": 0.9956050558989881, "temperature": 0.49723005294799805 }, { "layer": 3, "head": 7, "slices": 64, "native_nodes": 33555, "mass": [ 0.015197659137634767, 0.016306915557068306, 0.015024995130466387, 0.015526998033651322, 0.015926811774445174, 0.014450491389037378, 0.01652088430580382, 0.015217218160986758, 0.01626545041857831, 0.014851870932347763, 0.013442417217057457, 0.016063550371560667, 0.016125936627153746, 0.01453401128443813, 0.016271582178492835, 0.016014627189153557, 0.015518972467597241, 0.01723146877895345, 0.016600399169526332, 0.0158854144753178, 0.0162770887146822, 0.01591625712540502, 0.014814550434356474, 0.016239769032052653, 0.014909618277385257, 0.016790411560326447, 0.01670813508328445, 0.01714978393868351, 0.014771762442450475, 0.014772766904502712, 0.015574143648329656, 0.015751637509496524, 0.015506704724107213, 0.015093602614335448, 0.015167822187866074, 0.015550853483847393, 0.014446425266345183, 0.015587303651664131, 0.01583759898314012, 0.016626437839951613, 0.016123650101005294, 0.014871875887048074, 0.01606698803256155, 0.015460485367695873, 0.01544389903488204, 0.015430486890011511, 0.015037679064459904, 0.01582545350297271, 0.015323371460017096, 0.015464241583343813, 0.015490630478518381, 0.015753099951110953, 0.015918334571425677, 0.016323929761399448, 0.014794073935868469, 0.015751316300850745, 0.01584116043499362, 0.016147422608315073, 0.016200692764748593, 0.015206773906852926, 0.01428138392587041, 0.015074867008839843, 0.016681921793498467, 0.015015913648057617 ], "mean_node_assignment_entropy": 4.156862268801588, "normalized_node_entropy": 0.9995140968097496, "effective_mass_slots": 63.930654838792044, "node_slot_mutual_information_nats": 0.0009367089840592868, "normalized_node_slot_mutual_information": 0.0002252309010097454, "centered_token_effective_rank": 1.2840992298250522, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.007506794192211382, "token_RMS": 0.19148432494347678, "mean_token_offdiagonal_cosine": 0.998504304056023, "temperature": 0.49691152572631836 }, { "layer": 3, "head": 8, "slices": 64, "native_nodes": 33555, "mass": [ 0.01571594455728071, 0.015302494642505292, 0.015516223127030434, 0.014907859739306395, 0.015114481228240963, 0.01633538076968542, 0.014694015872393443, 0.01569638021763249, 0.017035491720609107, 0.01468219322959693, 0.01695788593044372, 0.01601362156971341, 0.015110415126836934, 0.016094761158015915, 0.015816982551513238, 0.01643560348710592, 0.014734688286510377, 0.015891972590488208, 0.01552132204052318, 0.015065191061497, 0.014909320659108372, 0.015429633524562515, 0.016314354675250622, 0.01566622221572906, 0.015860732947243758, 0.015294749654286613, 0.016366950364794786, 0.015029313471048933, 0.01470776894235976, 0.015432347868799701, 0.01522421606896425, 0.01621061331459137, 0.01588786383043026, 0.01636366137870865, 0.015263292240957276, 0.016064979549568974, 0.016588301377709842, 0.013901450265670228, 0.015357425947959635, 0.015337473786884406, 0.015790411209871607, 0.01566403914124191, 0.016288227105610082, 0.015628166142015686, 0.01513493817378429, 0.0158490239301589, 0.015874823866207487, 0.01521138478758269, 0.015044151200030142, 0.015847457271341844, 0.015854729381779183, 0.015583473167433111, 0.016078945819048132, 0.015760662285060095, 0.015294724065355814, 0.014723624076514952, 0.01614287026604545, 0.015529893270569238, 0.014954181164869466, 0.01547470215803495, 0.015640934019078487, 0.016015817291425125, 0.01650725599179066, 0.016251983232668517 ], "mean_node_assignment_entropy": 4.156847416327814, "normalized_node_entropy": 0.9995105255447063, "effective_mass_slots": 63.953698949753175, "node_slot_mutual_information_nats": 0.001311951302687575, "normalized_node_slot_mutual_information": 0.0003154576063791967, "centered_token_effective_rank": 1.665348981473823, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.007136002071345998, "token_RMS": 0.22647724835968075, "mean_token_offdiagonal_cosine": 0.9990750771133061, "temperature": 0.49965760111808777 }, { "layer": 4, "head": 1, "slices": 64, "native_nodes": 33555, "mass": [ 0.015386930079815677, 0.01641140780141742, 0.01442755644841528, 0.015146220277402577, 0.015703354679179066, 0.01557069496325055, 0.016449949075235434, 0.015278113241852964, 0.015145868056672613, 0.015575639978928246, 0.014802274292337967, 0.016951988354761603, 0.01593177683433292, 0.015805169844433982, 0.015733180794964257, 0.016619218267459324, 0.015457819222364779, 0.01653301546453093, 0.015117793493255953, 0.01515285848989481, 0.015660858863273296, 0.0149229495024114, 0.015253621703473125, 0.01406149399448633, 0.016296613603056526, 0.014670514223497345, 0.015893040643956368, 0.016836945716479847, 0.015972577470099805, 0.01684944229707013, 0.016266307376675955, 0.015575994191586643, 0.0159411532600925, 0.015166132266126266, 0.014980901193166255, 0.016077503682125426, 0.016029567763319253, 0.01520827761903292, 0.015845207202857082, 0.015550745311625454, 0.015399517574836184, 0.017516300203191233, 0.01536952694923778, 0.015801925881375963, 0.01714611298108626, 0.014941806242790286, 0.014728980832137173, 0.015476099821674923, 0.014566539817925023, 0.015338256604405518, 0.01531767274716997, 0.016421611226191422, 0.0176051937240173, 0.014870263921684496, 0.014850251459144806, 0.015010204979898127, 0.016143439735064314, 0.015485466525292781, 0.015239743273713571, 0.014557478882954127, 0.014940471541829826, 0.015978422310192766, 0.015434983300663794, 0.015599051650295473 ], "mean_node_assignment_entropy": 4.156394892634424, "normalized_node_entropy": 0.9994017165966499, "effective_mass_slots": 63.929856844815525, "node_slot_mutual_information_nats": 0.0013916028927685176, "normalized_node_slot_mutual_information": 0.00033460976538064605, "centered_token_effective_rank": 1.3124171742512807, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.009006051916411573, "token_RMS": 0.25840480321927517, "mean_token_offdiagonal_cosine": 0.9989174200549477, "temperature": 0.49719563126564026 }, { "layer": 4, "head": 2, "slices": 64, "native_nodes": 33555, "mass": [ 0.014879716953118082, 0.014829818719994949, 0.016066785184426433, 0.01655903536406715, 0.015010052885082507, 0.015557147144650696, 0.01598579974193084, 0.016632920211715887, 0.014653022062478052, 0.016556363039561345, 0.01571467030871698, 0.015540866892451217, 0.01535277525883103, 0.01495091642470007, 0.015981463845577765, 0.015918608741717355, 0.01557546892820803, 0.015520281847907755, 0.016241952472768426, 0.016634913968848516, 0.015360514152805623, 0.014656801079750292, 0.016408021138896885, 0.01710396173088575, 0.015415217599471677, 0.015896273950774586, 0.0154112082620051, 0.014790523138256404, 0.01620480952221823, 0.015193780092682093, 0.015404986254449015, 0.016057098044920448, 0.014270394822634504, 0.015512431714772032, 0.015888321377595657, 0.014626464590173803, 0.015251519788689983, 0.014841539122307465, 0.014627788834869995, 0.014848826697945258, 0.016813666175930445, 0.015155070510792396, 0.015671797193095006, 0.01519517837315154, 0.01610421078503598, 0.016927142563950302, 0.016153632164542134, 0.01584464543138375, 0.016185308956836574, 0.01535964172698181, 0.016222724063518178, 0.01606648769601569, 0.015555240714989437, 0.015568535453458822, 0.015864393058839552, 0.015604300456430478, 0.01471881958790764, 0.016715040831543387, 0.01630920859611886, 0.015163346148906335, 0.014913940627636915, 0.01585505430902628, 0.014485233956777229, 0.015614318638049692 ], "mean_node_assignment_entropy": 4.155969796241382, "normalized_node_entropy": 0.999299502520293, "effective_mass_slots": 63.94256708840197, "node_slot_mutual_information_nats": 0.0020154949797897004, "normalized_node_slot_mutual_information": 0.0004846241020465337, "centered_token_effective_rank": 1.3025482235334924, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.009044206932846003, "token_RMS": 0.19161641355803696, "mean_token_offdiagonal_cosine": 0.9977649479733749, "temperature": 0.5000409483909607 }, { "layer": 4, "head": 3, "slices": 64, "native_nodes": 33555, "mass": [ 0.014419709245830158, 0.01553985297168097, 0.016062620439988688, 0.014526939202966552, 0.01507165998982599, 0.01426768317908013, 0.016007215834486825, 0.015880660615437317, 0.015322517708328046, 0.014207155493326344, 0.013972344817273122, 0.016030565152926456, 0.014483415621627445, 0.016375483339451196, 0.01558137853614483, 0.017255064046709526, 0.01710468545356787, 0.016606059813747916, 0.016843065852528313, 0.014081586921631162, 0.01578776210217873, 0.016566499882556564, 0.016089981608564622, 0.014966504922659293, 0.015433457310459569, 0.014833492314493784, 0.014875814236640583, 0.015814756702744397, 0.01572442490514798, 0.015778873295131463, 0.015288762605857731, 0.015141824695914918, 0.014706559774301133, 0.015500615997873597, 0.016846062387450098, 0.016193587825256255, 0.015135011668832571, 0.016882401657160446, 0.016532410229033772, 0.016182022345060963, 0.015210373547206274, 0.017863433314602523, 0.015833291916320727, 0.015825425639011396, 0.01508078254299132, 0.015577571683054956, 0.016274723239440646, 0.015943069880401665, 0.016054208859503757, 0.01486511494278255, 0.015189818856146089, 0.01535536955132794, 0.014506082100739042, 0.015341298202933382, 0.015661179066740166, 0.015592864079442808, 0.015098783826351948, 0.014703537320690137, 0.01665228914417856, 0.015879289883825465, 0.015199513530885741, 0.015382144958887711, 0.014357284612919246, 0.018632058605841485 ], "mean_node_assignment_entropy": 4.1557211552041515, "normalized_node_entropy": 0.9992397169883973, "effective_mass_slots": 63.894377568458054, "node_slot_mutual_information_nats": 0.0015102143341163554, "normalized_node_slot_mutual_information": 0.00036312978841818237, "centered_token_effective_rank": 1.4007190079012972, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.009382877969186389, "token_RMS": 0.21773676832308064, "mean_token_offdiagonal_cosine": 0.9981461785530897, "temperature": 0.501863420009613 }, { "layer": 4, "head": 4, "slices": 64, "native_nodes": 33555, "mass": [ 0.0145448693806882, 0.016158051833326106, 0.015470578861684429, 0.014423307898795618, 0.014952538152200474, 0.01618569620376065, 0.016780167641713514, 0.015082310876142799, 0.015729670364738862, 0.014069035695844934, 0.013631644047021803, 0.01593950128077577, 0.014366931848687602, 0.016872189023214593, 0.015668291797229474, 0.014945306653783309, 0.016237180290493636, 0.016250983645106735, 0.01660269119437757, 0.015868370427780998, 0.015162992493681485, 0.01648011829578038, 0.015527020396633931, 0.01323830314908155, 0.016243518487000907, 0.014172236085428759, 0.017050717075732403, 0.017377377841901964, 0.015835429773688306, 0.017357199968451795, 0.015452647663268766, 0.015012674546028823, 0.016316148227755428, 0.013608251229808958, 0.015429590951533099, 0.01467651813734401, 0.015626227574634173, 0.015029594833243624, 0.017101115010614377, 0.018022372823354563, 0.014791486722450066, 0.018311357762730615, 0.015360423652231782, 0.01714981867597473, 0.01672002619215387, 0.014749215206101085, 0.015540544216587281, 0.01658565484724507, 0.015607075870893047, 0.014769792973810914, 0.015141872158256013, 0.016203215993960813, 0.014775109741819578, 0.015895708000076197, 0.01485867335181725, 0.015879442752675478, 0.015344634572452848, 0.01577260319819093, 0.015290229497365004, 0.014884804980086909, 0.014953121057355027, 0.017083801200185673, 0.014982238280260915, 0.014849777544793528 ], "mean_node_assignment_entropy": 4.153372858073138, "normalized_node_entropy": 0.9986750708841562, "effective_mass_slots": 63.85935976236876, "node_slot_mutual_information_nats": 0.003310303519156399, "normalized_node_slot_mutual_information": 0.0007959597451540368, "centered_token_effective_rank": 1.1011048862492367, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.01653941635942295, "token_RMS": 0.1869327271774183, "mean_token_offdiagonal_cosine": 0.9946138329790218, "temperature": 0.4925941526889801 }, { "layer": 4, "head": 5, "slices": 64, "native_nodes": 33555, "mass": [ 0.015403456115911169, 0.016574198498377154, 0.015180945081069827, 0.014558669658731089, 0.015535451602559876, 0.015952125745069837, 0.016566499061588292, 0.01601500705002766, 0.015175456591888611, 0.015475357753534001, 0.014995633038722568, 0.01598111298834513, 0.016126148791984003, 0.01630515110681885, 0.016589370604275306, 0.01570147412696293, 0.015444848970613453, 0.015504708622028959, 0.01429779117738498, 0.015921049885148707, 0.015866502612771477, 0.015575996532702061, 0.014647928608248085, 0.014954339956325988, 0.016158777928042833, 0.01537857285767767, 0.015916368212190616, 0.01614482300898643, 0.015609206473186997, 0.015805759080583674, 0.015652116799690724, 0.015009012511614813, 0.014966149407509293, 0.014512110921795598, 0.01524675599350857, 0.015492122347927798, 0.017429620216249078, 0.014245125850363666, 0.01602438980442319, 0.01719655745017859, 0.01643333490066438, 0.01617472412922582, 0.015642659653784745, 0.016037313061340423, 0.01605548039961325, 0.015193976467910217, 0.015585758351927506, 0.0153006316083405, 0.015352028101458401, 0.015267808798402482, 0.014620785574153822, 0.015734354080833554, 0.01798620116472968, 0.015101892227561477, 0.015700903522399424, 0.01621987870380209, 0.015618658713933128, 0.014868588355301728, 0.015397253613338938, 0.014435979767616895, 0.013958021602007036, 0.017356503626042755, 0.0156839006837841, 0.01513666950794601 ], "mean_node_assignment_entropy": 4.156102911988173, "normalized_node_entropy": 0.9993315100915862, "effective_mass_slots": 63.924748777676996, "node_slot_mutual_information_nats": 0.0016036792271432176, "normalized_node_slot_mutual_information": 0.0003856033446960276, "centered_token_effective_rank": 1.2530221547476208, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.009437457107313432, "token_RMS": 0.2229976687920921, "mean_token_offdiagonal_cosine": 0.9983209309414958, "temperature": 0.5003040432929993 }, { "layer": 4, "head": 6, "slices": 64, "native_nodes": 33555, "mass": [ 0.014804280557281266, 0.015430803344290936, 0.016424301467395153, 0.015512938743046146, 0.01572807082206104, 0.01563151403695385, 0.015586658745155909, 0.014532701371978974, 0.016142689546501072, 0.014906111900820815, 0.015732901157305308, 0.016010246694647526, 0.015184885053198066, 0.015632305432856528, 0.017435823113332385, 0.015267590186810542, 0.014562553411032969, 0.015209614245281803, 0.016645534672126153, 0.016132239790611395, 0.015152319010267748, 0.015785804772984073, 0.01515742458079486, 0.015194481495540795, 0.015869499636904757, 0.014909332686089587, 0.015580453591249443, 0.01635101426552663, 0.016063411298968846, 0.016079335619303168, 0.01577364190263309, 0.0165252880802934, 0.01587041953542232, 0.014442821291227167, 0.01645718945042931, 0.015613708018450584, 0.014744359626477903, 0.014591932211072918, 0.01643603577704179, 0.014442071100832088, 0.0156513637165475, 0.016246828208232696, 0.016306230599304066, 0.015017449497041946, 0.015423610853565563, 0.01602152578272908, 0.016207734996075737, 0.015039675382084311, 0.01614883131690511, 0.015010450617136161, 0.015123858172493673, 0.01565181316793711, 0.015780019292957897, 0.015989203307116517, 0.016676038707064023, 0.014901132089815975, 0.015344129885183133, 0.016012294132597193, 0.016121406482557322, 0.014950591398089694, 0.01515344376786111, 0.01525858571283012, 0.0160758502674534, 0.016333624143100696 ], "mean_node_assignment_entropy": 4.156803596315902, "normalized_node_entropy": 0.99949998905906, "effective_mass_slots": 63.94778490708858, "node_slot_mutual_information_nats": 0.0012632932214540205, "normalized_node_slot_mutual_information": 0.0003037578109633931, "centered_token_effective_rank": 1.1918023305616507, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.009155716875975268, "token_RMS": 0.14548192166812915, "mean_token_offdiagonal_cosine": 0.9961096591108982, "temperature": 0.5014758110046387 }, { "layer": 4, "head": 7, "slices": 64, "native_nodes": 33555, "mass": [ 0.01549968932547267, 0.015029570394290122, 0.014994119740400873, 0.01678678727191263, 0.015033823516674483, 0.017361992970275516, 0.015953662391846927, 0.01504395167465302, 0.014581859390308375, 0.0166847336961831, 0.01585104052444237, 0.016688352477234544, 0.015428944903346354, 0.015531286408655948, 0.01383053897102181, 0.014704882921385944, 0.015864290107908113, 0.015023203988686801, 0.015359381374125463, 0.016521795045996882, 0.01551531251734188, 0.014805225659988354, 0.015420885520110568, 0.015283556346246016, 0.015949584186163712, 0.015707539440182775, 0.01696970749356662, 0.014021700965425168, 0.015413829992450202, 0.015578860220530006, 0.0164147985917328, 0.015061231263452619, 0.013861646059322661, 0.016463703914367706, 0.015354461093409471, 0.015586301030247244, 0.01473362953283449, 0.016061611071999462, 0.01616427372775238, 0.01626458870872329, 0.01742178426286384, 0.014893259810881347, 0.013940715584268364, 0.016677375947977476, 0.016165665975899378, 0.014738606517647928, 0.01617959941475207, 0.016821372151738963, 0.014643928541652964, 0.015499204967402432, 0.015331600304132886, 0.01555169047619867, 0.015551894227984525, 0.017043776975728425, 0.014145092569727902, 0.015558153790724453, 0.015465010790760231, 0.017883806158012053, 0.015124436815740713, 0.01688719101052, 0.01577536415707277, 0.016270132833652418, 0.016115799414710315, 0.013908182712519663 ], "mean_node_assignment_entropy": 4.154863092406241, "normalized_node_entropy": 0.9990333964978445, "effective_mass_slots": 63.89151885756203, "node_slot_mutual_information_nats": 0.0023235349347041634, "normalized_node_slot_mutual_information": 0.000558692054604993, "centered_token_effective_rank": 1.1690840214315188, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.014523696160398714, "token_RMS": 0.2527961248012264, "mean_token_offdiagonal_cosine": 0.9968345420664633, "temperature": 0.4955364167690277 }, { "layer": 4, "head": 8, "slices": 64, "native_nodes": 33555, "mass": [ 0.014885025542141153, 0.015543893199914161, 0.0161706909820695, 0.01578028290159917, 0.015020333303352605, 0.014355748842759418, 0.01642490942353104, 0.014302334504407673, 0.014755236354980281, 0.013655507090876042, 0.016380994568307314, 0.01642845606162418, 0.01429694293747137, 0.014891296613063858, 0.016374715925486446, 0.015163694110476541, 0.01639775183260543, 0.015956151442672498, 0.015614786469092394, 0.01693567167734415, 0.015995347446918426, 0.015474507595437274, 0.016951950694496933, 0.016748895099176465, 0.01825230386324095, 0.014755093342745478, 0.015127958339348829, 0.015099253490841428, 0.01642069134976692, 0.016958460050246885, 0.01641188024766238, 0.015038073274196349, 0.016086994556702074, 0.0161614072940008, 0.015470884992396747, 0.014721699596181869, 0.015076948114149155, 0.01440028070156189, 0.015110903005725606, 0.014876575214612628, 0.01629967037163705, 0.017349244906222736, 0.014726198306427931, 0.015576763108101799, 0.01616713773204404, 0.016364475097737606, 0.0169936553491972, 0.015516426586194208, 0.016098101947841285, 0.015659463114541337, 0.014263873687478881, 0.015330591131844911, 0.01642043688343545, 0.013970887887073317, 0.014776848186173279, 0.014935596099959064, 0.014452411902416456, 0.015882745667078594, 0.014491262001190415, 0.014867159633337563, 0.017955370812569017, 0.01559007771642366, 0.016779478965098165, 0.015057590922235005 ], "mean_node_assignment_entropy": 4.1556155276660265, "normalized_node_entropy": 0.9992143189341584, "effective_mass_slots": 63.879200550780354, "node_slot_mutual_information_nats": 0.001378280743055349, "normalized_node_slot_mutual_information": 0.000331406465493118, "centered_token_effective_rank": 1.5117928006687857, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.007328284051919301, "token_RMS": 0.23947661583910976, "mean_token_offdiagonal_cosine": 0.9990924012717357, "temperature": 0.49955376982688904 }, { "layer": 5, "head": 1, "slices": 64, "native_nodes": 33555, "mass": [ 0.015003966101706295, 0.01448580552294413, 0.015131430501139674, 0.015270123668953126, 0.017025967074091576, 0.014572336949683386, 0.01670295052441852, 0.015879695444789566, 0.015579298420887896, 0.015796316690846218, 0.015910169047003327, 0.01501113656181645, 0.015345399920549563, 0.016950619608194918, 0.016145264209865538, 0.01439786452329309, 0.016632058291643693, 0.01517300550480204, 0.01655297988456455, 0.015887452766405828, 0.015816907817338258, 0.016736630348769294, 0.01410946681548197, 0.01587687058097012, 0.016225644680315433, 0.014967983621965709, 0.015460467322769357, 0.015505069117832691, 0.0150055363887818, 0.015832505686697928, 0.016945676533265237, 0.016353861504568683, 0.014296642348537758, 0.014870855327423053, 0.015732740796630614, 0.015296366827451708, 0.015688215539879646, 0.01548741673136158, 0.015168160379361492, 0.015911116629185033, 0.015030964535668398, 0.015530864757668456, 0.014551557448261445, 0.014615215143379512, 0.016796873222507516, 0.014662725680995237, 0.015054560255915962, 0.016050663305136482, 0.015182513455707525, 0.015158828429216077, 0.016260308362243826, 0.016405352902107553, 0.01620840300499034, 0.01669794898638381, 0.016705226501683917, 0.015427533678867, 0.015292310534674308, 0.016827089964345402, 0.01513763907103354, 0.016053212690982435, 0.015829601762515168, 0.015475309133777239, 0.014786585441047656, 0.015516735377177905 ], "mean_node_assignment_entropy": 4.15632182213608, "normalized_node_entropy": 0.9993841468557173, "effective_mass_slots": 63.928935175641676, "node_slot_mutual_information_nats": 0.0014502564056346756, "normalized_node_slot_mutual_information": 0.00034871295407109995, "centered_token_effective_rank": 1.3592121662884042, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.007723536824576523, "token_RMS": 0.22605010760006194, "mean_token_offdiagonal_cosine": 0.9988848333114235, "temperature": 0.49868303537368774 }, { "layer": 5, "head": 2, "slices": 64, "native_nodes": 33555, "mass": [ 0.0153054626814265, 0.017830386619435933, 0.013750234447339709, 0.01570430279620166, 0.01540524644205146, 0.015628634419416, 0.017600618637187874, 0.015953396622942505, 0.015900115271265027, 0.014889026452630265, 0.015820773945675312, 0.015063465895269173, 0.014981669494628267, 0.015916896376477377, 0.015417252064296331, 0.012954361659698516, 0.016633004426729697, 0.013854384564463898, 0.014473629890073876, 0.015151094705046263, 0.017399910333184873, 0.015073648369712181, 0.013753712698711784, 0.01518708573341494, 0.01660924541620051, 0.016531856435099475, 0.015491890485076928, 0.01565854614607478, 0.015215044464478016, 0.01641976815682932, 0.018156090282506584, 0.014382327087706469, 0.014688615277637145, 0.01647861208561311, 0.0157094944037265, 0.01685705127033822, 0.01495255443395614, 0.01644195667866991, 0.014851277689665284, 0.0159278204724539, 0.015180941826839186, 0.015170888988085501, 0.016062271844436357, 0.016495823706210885, 0.01494430216355964, 0.0164989536209025, 0.014806581723887205, 0.015231250946687135, 0.015512033720349363, 0.01582436757063683, 0.016496851054207434, 0.01566369546561514, 0.01750923459648624, 0.018294057917560936, 0.014695245594967496, 0.014298360196049497, 0.015533331982267407, 0.017285192811162482, 0.014253632795143911, 0.016774798935825192, 0.01632620561465503, 0.01507827734693745, 0.014216062867512444, 0.013827171206147904 ], "mean_node_assignment_entropy": 4.154086555002946, "normalized_node_entropy": 0.9988466787210448, "effective_mass_slots": 63.83887058348006, "node_slot_mutual_information_nats": 0.002275706620585183, "normalized_node_slot_mutual_information": 0.0005471917760060709, "centered_token_effective_rank": 1.2287380379001056, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.009677279299959434, "token_RMS": 0.22376908269538395, "mean_token_offdiagonal_cosine": 0.9983478256523899, "temperature": 0.4979057013988495 }, { "layer": 5, "head": 3, "slices": 64, "native_nodes": 33555, "mass": [ 0.015903340923585883, 0.014734359126554136, 0.01770020955636478, 0.014673533355044219, 0.01581874154716277, 0.016567870497485954, 0.01426757574109176, 0.015950192033416324, 0.016285985262878267, 0.015402948924242748, 0.01522966079686446, 0.016286852415925857, 0.01542209748513447, 0.016762391652735933, 0.015723996463318755, 0.01626807702087713, 0.01459311291396689, 0.01601466686232952, 0.014397897743656754, 0.01472521814305489, 0.015649742865941906, 0.014655044019242244, 0.017040169578877704, 0.01528533376839926, 0.015889793088610518, 0.01676541249011161, 0.015648549422066626, 0.01471778246595059, 0.016008833147038062, 0.013980832793575705, 0.014568649295030817, 0.01496247908930641, 0.017442529718984863, 0.015495788234833757, 0.015521290782056921, 0.013519209030376712, 0.015216791161229565, 0.016169556203772743, 0.014802955248593094, 0.015591286299626698, 0.015922070174300224, 0.015498805855892264, 0.016692196672127003, 0.014688931550231647, 0.015689327922748072, 0.017032827320824374, 0.015986124362817083, 0.015399482632656777, 0.015065845218421166, 0.01729920516549684, 0.01559060885855572, 0.014479793451316772, 0.01526355816853764, 0.016208387247667398, 0.016072718499026025, 0.015541906248361223, 0.015617462665074638, 0.015016718048587355, 0.01561751866655158, 0.014818082884483616, 0.016073684908273128, 0.0159241934142193, 0.01548550931914182, 0.017354283478505705 ], "mean_node_assignment_entropy": 4.155826558869937, "normalized_node_entropy": 0.9992650612127174, "effective_mass_slots": 63.90279714864542, "node_slot_mutual_information_nats": 0.0015365753997214071, "normalized_node_slot_mutual_information": 0.00036946828485500847, "centered_token_effective_rank": 1.167506724330122, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.008970627670479405, "token_RMS": 0.18624993025471304, "mean_token_offdiagonal_cosine": 0.9976628743373378, "temperature": 0.49745258688926697 }, { "layer": 5, "head": 4, "slices": 64, "native_nodes": 33555, "mass": [ 0.015369246736881613, 0.015505100378325072, 0.015546100875194835, 0.015115885385271201, 0.016299029496086102, 0.01605444556531528, 0.01445099610309077, 0.015020187784387668, 0.015604926754694558, 0.01680038507348902, 0.015607569817852742, 0.01558716820380586, 0.016367662569746373, 0.015376257504771042, 0.016063336358157102, 0.016402284049425807, 0.01546520881877715, 0.014962592894177272, 0.01672748808667559, 0.01606603431924922, 0.015062929168504524, 0.015415330039846029, 0.016014359374170492, 0.01615460380368382, 0.01577315376384561, 0.014979175370030583, 0.015809501015868872, 0.015948410671592344, 0.015142416216254305, 0.016126831733090007, 0.014410050671990068, 0.015615307532096563, 0.015048679959619032, 0.015131126171428415, 0.015248902981223505, 0.01612538137685354, 0.01595218019842249, 0.015093177607061787, 0.016329461013788064, 0.015909659025979423, 0.014687640977724487, 0.016363715749628638, 0.01545205620099306, 0.01615793466295438, 0.014849841607185842, 0.01592597410420319, 0.015909541114435353, 0.01586314112168944, 0.015700482081878898, 0.015512540273733616, 0.01626703252719174, 0.014591332992077797, 0.016658429826776736, 0.015220457224380445, 0.015411540577050747, 0.015551321924114668, 0.015850346952520234, 0.01599641583183806, 0.016405335403151724, 0.015749539201766933, 0.014550420819087704, 0.014939269435891335, 0.015011663437988751, 0.01569148159111697 ], "mean_node_assignment_entropy": 4.156681057179911, "normalized_node_entropy": 0.999470524625092, "effective_mass_slots": 63.9572631780145, "node_slot_mutual_information_nats": 0.0015340402833263767, "normalized_node_slot_mutual_information": 0.000368858718213144, "centered_token_effective_rank": 1.2000565987848377, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.00853298912394354, "token_RMS": 0.3034676775934038, "mean_token_offdiagonal_cosine": 0.9992619853018481, "temperature": 0.49814629554748535 }, { "layer": 5, "head": 5, "slices": 64, "native_nodes": 33555, "mass": [ 0.015816922136308354, 0.014531959112105782, 0.01368246146180363, 0.016544180034823976, 0.015362035429193701, 0.015362633219133935, 0.0158532698810125, 0.016202440353077403, 0.01609016074334797, 0.015442177899722511, 0.016365617317344754, 0.016554355979600796, 0.016246872711435434, 0.015388373428165486, 0.01587758243273799, 0.017218772341978654, 0.014692352982294833, 0.017039716679859873, 0.013605639193084348, 0.015022133329069716, 0.015602248501753669, 0.014219390005489439, 0.016711001831889952, 0.01591472122543456, 0.014757598788345505, 0.015849907115860085, 0.015414384561418075, 0.013413947474029936, 0.01652339447349101, 0.016324061180132278, 0.014034350831774909, 0.015150371005650675, 0.016826556741746122, 0.01575330438442343, 0.013963413862571189, 0.01591203331493808, 0.01632286785951742, 0.014743099428575657, 0.0177296727163557, 0.017408297420316295, 0.014385258905021157, 0.017944173584147485, 0.016402255859786087, 0.015614943211639252, 0.015679203960703162, 0.015580773214856642, 0.01486628369160797, 0.014674501035891494, 0.015282756000848254, 0.015610677805289153, 0.015288192862396384, 0.015451997799587629, 0.01539989639006443, 0.014928076977101466, 0.01623922199706947, 0.016784063251385158, 0.014661585411406313, 0.0159554136536195, 0.015590783683359711, 0.015520090729940747, 0.014516723062829208, 0.01682523682323827, 0.015491869188084668, 0.015831741153851935 ], "mean_node_assignment_entropy": 4.155525320801065, "normalized_node_entropy": 0.9991926287680359, "effective_mass_slots": 63.87712329597719, "node_slot_mutual_information_nats": 0.0014359685946203626, "normalized_node_slot_mutual_information": 0.0003452774617218486, "centered_token_effective_rank": 1.4053706449763388, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.00856624009627201, "token_RMS": 0.21036064636173935, "mean_token_offdiagonal_cosine": 0.9983853520849425, "temperature": 0.4970945417881012 }, { "layer": 5, "head": 6, "slices": 64, "native_nodes": 33555, "mass": [ 0.014304283452975894, 0.015447595643165205, 0.016900069311382188, 0.016570161421430918, 0.015589237885696489, 0.014737277475453945, 0.014591612040967686, 0.014813510792152552, 0.015049151165876885, 0.015381328464070627, 0.015394592866098369, 0.01622092129920478, 0.01539636390635361, 0.015032744115725605, 0.015829029764284536, 0.015650208987186828, 0.01516953048973081, 0.016169486172062073, 0.015036227996971093, 0.01500382122865818, 0.015987473005583508, 0.016299857143432038, 0.01573503134543757, 0.016168635549622425, 0.015521267567270116, 0.016145009519272814, 0.015750271639246964, 0.016295131519819858, 0.015690746436356574, 0.013644576539145178, 0.015498812928115259, 0.01536351667022944, 0.01628596059433453, 0.015339450147394282, 0.015308670238722873, 0.014865571524431081, 0.014578958096476989, 0.01622904890651671, 0.015148643968201371, 0.015760326239561702, 0.014607456892716593, 0.014771648052252187, 0.01565397335478689, 0.0158770608931509, 0.015306519022628591, 0.016286816888002698, 0.0158908369040999, 0.01622645409755279, 0.014790179556497402, 0.015448293482101793, 0.01454639140774087, 0.015459107904776991, 0.015033241282666143, 0.016991656885825563, 0.016064476894028512, 0.01690885467315183, 0.015916464682249346, 0.015032973991616207, 0.017162024892778904, 0.015088885794670576, 0.01589628297740111, 0.015562092267546368, 0.01661160115894769, 0.01896259210545266 ], "mean_node_assignment_entropy": 4.155788021573508, "normalized_node_entropy": 0.9992557949516427, "effective_mass_slots": 63.91545984221892, "node_slot_mutual_information_nats": 0.0017732486105819234, "normalized_node_slot_mutual_information": 0.00042637616279163094, "centered_token_effective_rank": 1.2314026095304789, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.01060537398290804, "token_RMS": 0.24163152641987493, "mean_token_offdiagonal_cosine": 0.9981754725133952, "temperature": 0.49731889367103577 }, { "layer": 5, "head": 7, "slices": 64, "native_nodes": 33555, "mass": [ 0.01643987180443927, 0.01770365808626509, 0.015337079199151175, 0.015183495990643335, 0.015433487005119902, 0.016916811478790105, 0.016684197922362254, 0.014327877079905625, 0.015786532246879762, 0.01557406333840372, 0.015950210994094802, 0.014100136663811899, 0.015237112201343944, 0.015687105249746192, 0.015381875122528045, 0.014436027059569397, 0.015715013015857086, 0.015045972786829344, 0.01585237783746359, 0.016120905981923273, 0.016053081794711717, 0.016452554370494647, 0.015265649171900535, 0.015167807062832196, 0.015776644090076456, 0.014819593449760044, 0.01429297535045614, 0.016648686486095826, 0.015797350904734307, 0.016879738488259143, 0.016781906162171716, 0.016432546241165337, 0.014455052993546618, 0.01644962746794307, 0.017504997207777474, 0.01617861684079258, 0.015095515511047377, 0.015144307394151875, 0.016142150732718998, 0.015553719725478133, 0.01669223075550793, 0.014128867477748108, 0.015597249439363374, 0.014932204839860765, 0.01559319149673922, 0.014214677382785132, 0.01571374822331912, 0.015016326293706752, 0.01594579703703405, 0.014008701728267485, 0.016070046010182238, 0.01895300003648195, 0.015594396766648751, 0.014734665450830561, 0.014867441652245812, 0.015655393858020148, 0.01585849530324884, 0.015623380247941495, 0.013998863096342696, 0.017511860589255183, 0.015130639903251798, 0.016017175533322638, 0.015325614131266572, 0.013009700160615137 ], "mean_node_assignment_entropy": 4.155097179018265, "normalized_node_entropy": 0.9990896824302288, "effective_mass_slots": 63.868453501474185, "node_slot_mutual_information_nats": 0.001728375039514063, "normalized_node_slot_mutual_information": 0.0004155863497172008, "centered_token_effective_rank": 1.4782223475180796, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.007855327875762892, "token_RMS": 0.20750900699052777, "mean_token_offdiagonal_cosine": 0.9985998948396126, "temperature": 0.5009755492210388 }, { "layer": 5, "head": 8, "slices": 64, "native_nodes": 33555, "mass": [ 0.015259493748454655, 0.014206323958655802, 0.016031035543017665, 0.0162365703978285, 0.015490754819671482, 0.014950335509864153, 0.01504892434435241, 0.015700968273035097, 0.016106033028437723, 0.01449323511546312, 0.015954347077056025, 0.015075937214494418, 0.015316807222903005, 0.01726561530030106, 0.0162010524879612, 0.016391232409190448, 0.015623244002651278, 0.016800569986802127, 0.014934277653942838, 0.01590702205540002, 0.015666005052731604, 0.015068944636524176, 0.01584207185227249, 0.016349873995049544, 0.015393842194790572, 0.015034134535421746, 0.015992901195603936, 0.015391888728107457, 0.01601726187257347, 0.012841682066782063, 0.014221333321797308, 0.014715908453309213, 0.01775693871220206, 0.013483926449667687, 0.015521779496041216, 0.014493320467437083, 0.017082533708685708, 0.016847597321126548, 0.015972907243194088, 0.016241783016912096, 0.014326474649381742, 0.01772390894470796, 0.017075931676000806, 0.014001568022106775, 0.016167805541788007, 0.016028979193319855, 0.014683559314032644, 0.01603169484398987, 0.015679760651743157, 0.014882406708163381, 0.015620552332559191, 0.015182965513246058, 0.015622318680008901, 0.015178225106598412, 0.017023816513572893, 0.014575124955554998, 0.014578675600930308, 0.0154482298373128, 0.016227202859281552, 0.015199452500183631, 0.015973072074133874, 0.016202205783354334, 0.015377458184929749, 0.018258196248468486 ], "mean_node_assignment_entropy": 4.155364945999135, "normalized_node_entropy": 0.9991540667794645, "effective_mass_slots": 63.87047338410419, "node_slot_mutual_information_nats": 0.0014922332285030038, "normalized_node_slot_mutual_information": 0.00035880624643516853, "centered_token_effective_rank": 1.4559700673258484, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.00862261885183468, "token_RMS": 0.20014227395512899, "mean_token_offdiagonal_cosine": 0.9981634836818787, "temperature": 0.49922823905944824 }, { "layer": 6, "head": 1, "slices": 64, "native_nodes": 33555, "mass": [ 0.015653102097230986, 0.015221385583320373, 0.014725562822750367, 0.016288351348843166, 0.014397119137251311, 0.015556636891150027, 0.01495638389088743, 0.013649823229924796, 0.016044009603102566, 0.015974858777650005, 0.016209312928822477, 0.013681661226926697, 0.016880047111629756, 0.015322534135631487, 0.01654435064401597, 0.015872794957705216, 0.014903532936941483, 0.015466247121464157, 0.01637578209799109, 0.01511065291160044, 0.016226014657818404, 0.016836028137595247, 0.015989111576797204, 0.014961983848570214, 0.01694105124991149, 0.016039610204593983, 0.013701956483297192, 0.014906610652716668, 0.01603549750247836, 0.016677878001148876, 0.013822327920537808, 0.01677352683980869, 0.01393339566901968, 0.01716294464708094, 0.015565101361764542, 0.01598120544551515, 0.015795049206312785, 0.016480685988805536, 0.016692701654766554, 0.015209789387354409, 0.014477980669436308, 0.014555619175266515, 0.014482327051862637, 0.014867590989508406, 0.018751160920414567, 0.014587903162818378, 0.01705823958732189, 0.015149029455312966, 0.014342861194116378, 0.018911752625750306, 0.018047630387916107, 0.01586841411190281, 0.01625679115541507, 0.01822067078845771, 0.015001053466333786, 0.015193436001533635, 0.016347064539310546, 0.015533187899684706, 0.01503611921219397, 0.014690279487879607, 0.014254185830947496, 0.013943389453984172, 0.014354825827426738, 0.015501866867812136 ], "mean_node_assignment_entropy": 4.153179873019603, "normalized_node_entropy": 0.9986286677875394, "effective_mass_slots": 63.820010855518156, "node_slot_mutual_information_nats": 0.0028869179263457667, "normalized_node_slot_mutual_information": 0.0006941570292987479, "centered_token_effective_rank": 1.0889894887680793, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.01195661466433082, "token_RMS": 0.1795918786304567, "mean_token_offdiagonal_cosine": 0.9955484227485535, "temperature": 0.49518322944641113 }, { "layer": 6, "head": 2, "slices": 64, "native_nodes": 33555, "mass": [ 0.016813128734555087, 0.017048140024512923, 0.015254393311622929, 0.01596836129245349, 0.016309089682698737, 0.015842008565552117, 0.016857617727012185, 0.016728240373663602, 0.018331156705630073, 0.015832305632752574, 0.016333672737017454, 0.017508199856555634, 0.015333521106331594, 0.015471389255179874, 0.015111127498568952, 0.01631963365695868, 0.015561404544790736, 0.016192568681974284, 0.016727864988605616, 0.015661004325479042, 0.015858265794724817, 0.015230772615721384, 0.015097882824121014, 0.015599077423825481, 0.014155864337995712, 0.013706062244332693, 0.016533683456986758, 0.015000778241708904, 0.01633087995043989, 0.015975993870051292, 0.016787998262371505, 0.016205138470236483, 0.014316458975436337, 0.016206029697757526, 0.015408183056614141, 0.01367369569838602, 0.014874595011047539, 0.0140883035363854, 0.015585973913927197, 0.01617483469152318, 0.015989497009009198, 0.014231987893076248, 0.014601436119863571, 0.014973249762579787, 0.014999101062088365, 0.015291929183684088, 0.013825211827321951, 0.013818642129252021, 0.014788571859286236, 0.016215196318425946, 0.014782087502865372, 0.015550470960509261, 0.01414052214528441, 0.01606738191892326, 0.014995234904025594, 0.015709344187459708, 0.017477248250414534, 0.015409672455208532, 0.016665619038208606, 0.015740422122297034, 0.015609598823081557, 0.015060567743870865, 0.016007458996996363, 0.016034246952944763 ], "mean_node_assignment_entropy": 4.155497068613535, "normalized_node_entropy": 0.9991858355528953, "effective_mass_slots": 63.87697314992418, "node_slot_mutual_information_nats": 0.001461870234384044, "normalized_node_slot_mutual_information": 0.0003515054895948412, "centered_token_effective_rank": 1.3003340715602922, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.010007017290418387, "token_RMS": 0.28568273761842194, "mean_token_offdiagonal_cosine": 0.9987705963036798, "temperature": 0.496014267206192 }, { "layer": 6, "head": 3, "slices": 64, "native_nodes": 33555, "mass": [ 0.015778755900082157, 0.015711429349483035, 0.014017130033535078, 0.015655028043477737, 0.014623616817123514, 0.014810020061168997, 0.015278227587171246, 0.015399634182306608, 0.016416129100811878, 0.015861037042868457, 0.01767045604679836, 0.016316314179474142, 0.016126573534057626, 0.015565969848667007, 0.014079102860285448, 0.015308002392425434, 0.016154715176069458, 0.01591000951248711, 0.015970858447964236, 0.014880592576330082, 0.017189898367484358, 0.015832536613934755, 0.01464325976181254, 0.015647095966139942, 0.014936995527380502, 0.015750701569887816, 0.015608853658164707, 0.015306229207033577, 0.016151844220678082, 0.014657858315194137, 0.016115105884559614, 0.0156718808742116, 0.016791410720043275, 0.015477521371136895, 0.015905032047371568, 0.015092919103361609, 0.01615836412503354, 0.015967240380774114, 0.01570266564854734, 0.017050665560674028, 0.01575211727520696, 0.016718470721197917, 0.01613169231040856, 0.016807829015536478, 0.01542534043524215, 0.015628929787364416, 0.014285760042646872, 0.014997356957857982, 0.015266459067524892, 0.0151999506381643, 0.01558370813657991, 0.015309453999612277, 0.0142831290972291, 0.01607801106891224, 0.01529333806321782, 0.015756643236637106, 0.014873877493518909, 0.014479944798881678, 0.015917074629314486, 0.015640984882530247, 0.015647648755089616, 0.0168194745036544, 0.015398617436954755, 0.015512505561477504 ], "mean_node_assignment_entropy": 4.154313661331865, "normalized_node_entropy": 0.9989012862501258, "effective_mass_slots": 63.92976609410682, "node_slot_mutual_information_nats": 0.0034714146587040418, "normalized_node_slot_mutual_information": 0.0008346987854969291, "centered_token_effective_rank": 1.1955665885696887, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.016380343296956448, "token_RMS": 0.25281127702744866, "mean_token_offdiagonal_cosine": 0.9957781928082066, "temperature": 0.49519142508506775 }, { "layer": 6, "head": 4, "slices": 64, "native_nodes": 33555, "mass": [ 0.016951795137953498, 0.014942646805808167, 0.014948659616581749, 0.01579028434702837, 0.015391299499146338, 0.01569850205972118, 0.015290263858741068, 0.01591918662769409, 0.01650613187969779, 0.01562699972042813, 0.017494583145647714, 0.01598018752337457, 0.0157579874502715, 0.01536295819137206, 0.01553945798027748, 0.016000768111363962, 0.015662820135580322, 0.01602160694362656, 0.016321428980738826, 0.01601350347007299, 0.01624374725249299, 0.015336178821576653, 0.015894735791816068, 0.015940816319131128, 0.014932017186313819, 0.015335847588980132, 0.014438831361351781, 0.015267785959614094, 0.014879019825683631, 0.015649332651667795, 0.01687469536216821, 0.01503004531717868, 0.01542204152895386, 0.016012487928487827, 0.016356223394100367, 0.014190246488534291, 0.016325473531801988, 0.015551471844567545, 0.014441325419448231, 0.015460398612673556, 0.016340542058198824, 0.017505440000606186, 0.015658628158720947, 0.016507909997505007, 0.015435569855115166, 0.014812849432500885, 0.013864072383581403, 0.014863144951014757, 0.014697802301361378, 0.017108479044657583, 0.016711300004824868, 0.014102586683575877, 0.01449545080117263, 0.015630148438870853, 0.01484783235149112, 0.01664625586340385, 0.014903706554731435, 0.015590381084115196, 0.016602226041591554, 0.015907964669390037, 0.015452043644805243, 0.016321920923613353, 0.01476994929559037, 0.014420002007241079 ], "mean_node_assignment_entropy": 4.155185368588539, "normalized_node_entropy": 0.9991108875395108, "effective_mass_slots": 63.915649022638576, "node_slot_mutual_information_nats": 0.0023788614450168666, "normalized_node_slot_mutual_information": 0.0005719952682812978, "centered_token_effective_rank": 1.183923615594019, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.011142575541190967, "token_RMS": 0.19840469787689927, "mean_token_offdiagonal_cosine": 0.9968470262727086, "temperature": 0.4991694688796997 }, { "layer": 6, "head": 5, "slices": 64, "native_nodes": 33555, "mass": [ 0.015877596021249208, 0.01593579074800583, 0.014376461313574765, 0.014926967857123274, 0.015596955284118794, 0.016434483600589134, 0.015956339746299533, 0.015030284771189758, 0.016465528426749344, 0.016325186777817184, 0.015603230063623224, 0.01563575331539675, 0.016329986010079296, 0.015607555881042698, 0.0173868410968455, 0.015123304281138953, 0.015434342559142255, 0.015339901999572789, 0.01773196762523998, 0.015164605546823168, 0.01586813340203637, 0.015858037761800992, 0.014380283668890402, 0.015268685572563208, 0.01674648940378908, 0.014902060974881691, 0.013858377270016393, 0.01574883417661713, 0.01607779595762771, 0.016398220166542892, 0.01577135980972442, 0.017741709506141103, 0.0149809647237985, 0.016558281545802193, 0.015927518993275273, 0.015951586094610098, 0.014478815245844403, 0.01519087410546232, 0.015380996711787507, 0.015437420128148442, 0.01470708503765199, 0.013536733194809303, 0.014566619083232042, 0.013956017767550595, 0.01749977597325925, 0.014522378671691743, 0.015482700957620114, 0.014834798073429115, 0.015159341314289346, 0.01795437896820991, 0.015812384414845598, 0.016526834514020382, 0.01658595231406234, 0.016928080106832697, 0.014394871071019883, 0.015575953144616208, 0.018378572265246047, 0.01685708941603969, 0.014836122100580789, 0.015976045381584772, 0.014159224200019572, 0.01473299567222, 0.013135556693253654, 0.015070961386763584 ], "mean_node_assignment_entropy": 4.154443064378974, "normalized_node_entropy": 0.9989324011058491, "effective_mass_slots": 63.85205152686488, "node_slot_mutual_information_nats": 0.002125647996322755, "normalized_node_slot_mutual_information": 0.0005111103038284, "centered_token_effective_rank": 1.2132784428542143, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.011944558105508131, "token_RMS": 0.22508301049036722, "mean_token_offdiagonal_cosine": 0.9972723332934814, "temperature": 0.4962131381034851 }, { "layer": 6, "head": 6, "slices": 64, "native_nodes": 33555, "mass": [ 0.015499047417098729, 0.015526013621304647, 0.015432300410705136, 0.015000195207493109, 0.014806174440663284, 0.015475405410632675, 0.01570239779800998, 0.01731582117642896, 0.015310458655339643, 0.015072103351023759, 0.017644265610094648, 0.017385355506973257, 0.014310886401301737, 0.01518958858144237, 0.013869675961390389, 0.01487980879723333, 0.015406573616188048, 0.015700525587840686, 0.01602193792181978, 0.017649270521151852, 0.014361409706496133, 0.01547542404455029, 0.016979728017522476, 0.014642168188639981, 0.01505111187761404, 0.015064189315698192, 0.017517990468678024, 0.016244474513451505, 0.01694848895122341, 0.013596256988767255, 0.016390966249605152, 0.014578029494235803, 0.016528790805951224, 0.014505447628735084, 0.014830734580560948, 0.01462426046602014, 0.016207151755250934, 0.01604147610980243, 0.01674138001576921, 0.0163328609241259, 0.014721739804753824, 0.01772189782120564, 0.015563070896700426, 0.014655955699889311, 0.015152501279296418, 0.01662054875028207, 0.014340658800077233, 0.01652878328084817, 0.016115913674792665, 0.01487312213445701, 0.01561219253893839, 0.015718780697999635, 0.014199033765322965, 0.015488104711545818, 0.017839850475360776, 0.01555480771271878, 0.014002556281893971, 0.01707874560611265, 0.015437192293174033, 0.01600164140306381, 0.014795650490768494, 0.016364842626934942, 0.014574681113749818, 0.015177581993014513 ], "mean_node_assignment_entropy": 4.154646229333155, "normalized_node_entropy": 0.9989812519511623, "effective_mass_slots": 63.85972172604622, "node_slot_mutual_information_nats": 0.0020426003812907823, "normalized_node_slot_mutual_information": 0.0004911415734343529, "centered_token_effective_rank": 1.4159854360660584, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.008852739276518138, "token_RMS": 0.2637013546739283, "mean_token_offdiagonal_cosine": 0.9989050844564247, "temperature": 0.4957536458969116 }, { "layer": 6, "head": 7, "slices": 64, "native_nodes": 33555, "mass": [ 0.014567498292264933, 0.015146140768294708, 0.014877008368537182, 0.015289630908049969, 0.015179847429085019, 0.016127366241222522, 0.014402275400675056, 0.014235648263099828, 0.014037211644867771, 0.014786017040378326, 0.015402736968685993, 0.016706346366024378, 0.013970635303065365, 0.017770978397238877, 0.016179882292320025, 0.01488768095881937, 0.015396593554707854, 0.013801323240554879, 0.017009197020435, 0.015063750370504284, 0.014885999919046475, 0.015997922387695172, 0.01667949535660951, 0.01774646663754366, 0.01731840227453715, 0.015537193125787462, 0.016068882823396753, 0.014976175507271615, 0.01691297576427593, 0.013570100900727765, 0.014434648425607093, 0.015166226983616594, 0.017288586763228492, 0.015646901821534158, 0.016758306385598706, 0.016639034646060726, 0.015269504199555273, 0.016928067939521667, 0.017631475596503146, 0.016445816009698293, 0.014922786639863716, 0.017531109241802855, 0.01740920799400648, 0.015182920876596772, 0.015470979710664218, 0.014118424680063451, 0.014190471385963813, 0.01721337500922443, 0.015261698315468785, 0.01528734204433875, 0.015696927514346322, 0.01613886492423166, 0.01638134955710949, 0.0158411878174316, 0.01551303437034884, 0.01577143122438875, 0.013889196565986178, 0.016949856075087838, 0.015219713583440942, 0.015080753720535188, 0.016124642699803685, 0.014543580930725, 0.016174320696717516, 0.013346871945825496 ], "mean_node_assignment_entropy": 4.154049252051706, "normalized_node_entropy": 0.9988377092572508, "effective_mass_slots": 63.835037464251776, "node_slot_mutual_information_nats": 0.002252964112450684, "normalized_node_slot_mutual_information": 0.0005417233587222345, "centered_token_effective_rank": 1.2656578736883162, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.01099708033177114, "token_RMS": 0.2661352846236446, "mean_token_offdiagonal_cosine": 0.9983106794251678, "temperature": 0.4970402419567108 }, { "layer": 6, "head": 8, "slices": 64, "native_nodes": 33555, "mass": [ 0.016519279402060785, 0.013941252493717119, 0.016532774271079617, 0.015772906305234268, 0.014137698295497757, 0.01497539964892695, 0.015546032526192962, 0.01600314936574397, 0.015397031417952229, 0.01657794814539074, 0.015404334497897418, 0.014314423842907103, 0.015602826360197336, 0.014702495911845977, 0.016555092899752454, 0.016529112596316266, 0.016667461860383734, 0.01602752756992403, 0.015308221044401053, 0.017227470826936573, 0.015055656361663826, 0.015042248557802862, 0.016424058947416314, 0.014873594940408139, 0.015323947538442564, 0.015802253057147445, 0.015724520301139626, 0.015442810048873911, 0.01424972289170381, 0.017305860011429683, 0.014538609268175136, 0.01524297965776812, 0.015609726661766557, 0.015778643804953568, 0.014478997417979998, 0.014534154041936573, 0.01545377843994539, 0.018544241206809566, 0.016616694287110673, 0.015232331758289263, 0.014570904902657549, 0.017363671078276587, 0.016018112137471844, 0.014726856592590204, 0.017114984604758045, 0.016147361395955795, 0.0158303398996727, 0.01748496336664551, 0.015115441205658628, 0.016015792141552344, 0.01642992266453386, 0.015991048115058443, 0.014567491336113791, 0.015205609402088486, 0.015616806396542025, 0.015392234652425143, 0.014070464369265418, 0.015565994448100529, 0.014869390951305508, 0.014601033752457455, 0.01492557973471724, 0.015217968865128845, 0.016723640876135374, 0.01541711852540648 ], "mean_node_assignment_entropy": 4.151595106062033, "normalized_node_entropy": 0.998247611882431, "effective_mass_slots": 63.88585879143153, "node_slot_mutual_information_nats": 0.005502928661996975, "normalized_node_slot_mutual_information": 0.001323174648504796, "centered_token_effective_rank": 1.0537805096014867, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.016795279227340695, "token_RMS": 0.19411399293081563, "mean_token_offdiagonal_cosine": 0.9928281996052248, "temperature": 0.49587348103523254 }, { "layer": 7, "head": 1, "slices": 64, "native_nodes": 33555, "mass": [ 0.015010221563129752, 0.016175910921812038, 0.015399962326536999, 0.015515644647843826, 0.01698869171209168, 0.015115433900015194, 0.0152176653060381, 0.015611508077657484, 0.015447572314832262, 0.016363003157937407, 0.017339263348159484, 0.01576863874589286, 0.015974473783607862, 0.016168200293331873, 0.014804363284843447, 0.015275127826764516, 0.016095243086126643, 0.015459863524265047, 0.015046685297087095, 0.0163528040623493, 0.013964148511189477, 0.017129202020641696, 0.015366388905103437, 0.016729944583343297, 0.015145355723759418, 0.015408555565521537, 0.014596169612215975, 0.015618391090710088, 0.015562109602164217, 0.014885905641002735, 0.015898620046957197, 0.015285695433525601, 0.017101329753227836, 0.015643883202124495, 0.015622628341836796, 0.015206264167487835, 0.015397851321817936, 0.014828109700438687, 0.015227318420837452, 0.016146649388814847, 0.01604148363249079, 0.01507819036249143, 0.014539360988376136, 0.015527586931743826, 0.015892832002387, 0.015920789922334862, 0.015922288884228503, 0.01476118109052835, 0.015123676085732783, 0.016835168430243352, 0.01584508642851263, 0.015554539314573184, 0.015538723654478828, 0.014485997995776625, 0.015347784236012433, 0.016505272346092857, 0.015482406478279163, 0.016546925562810674, 0.01672376686213931, 0.014656816316470716, 0.016388442752472896, 0.014860642139374268, 0.015647931324076304, 0.014878307895119021 ], "mean_node_assignment_entropy": 4.155916056988479, "normalized_node_entropy": 0.999286580961349, "effective_mass_slots": 63.93466817442126, "node_slot_mutual_information_nats": 0.0019456952153760199, "normalized_node_slot_mutual_information": 0.00046784080638406127, "centered_token_effective_rank": 1.1106545614544252, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.01090556662904173, "token_RMS": 0.23659070988823336, "mean_token_offdiagonal_cosine": 0.9978578784063223, "temperature": 0.49525144696235657 }, { "layer": 7, "head": 2, "slices": 64, "native_nodes": 33555, "mass": [ 0.017272653743999937, 0.014060850817093443, 0.017064906220265304, 0.015254344647680038, 0.015093848662099252, 0.017363008631401916, 0.017613756262635248, 0.015310121148451664, 0.016951012000554794, 0.013249502211699248, 0.01583280495024197, 0.016031435643997352, 0.016566606839077118, 0.016058656263604122, 0.015621586314137043, 0.01440307873990506, 0.01596739434273121, 0.01468525520553489, 0.01524366523297244, 0.01682763374656896, 0.016984173597779328, 0.01449010296387651, 0.014310759723779341, 0.015436687096209555, 0.01483680909132326, 0.014955831197914913, 0.01515018818118683, 0.01498694743266108, 0.015558547671063303, 0.0163723819957267, 0.014491259243193309, 0.016961299810203848, 0.015481109873255272, 0.01464823158453712, 0.014172794388624075, 0.01517581130125238, 0.015614112863749278, 0.015466610526863383, 0.014901877448435708, 0.01659306810818798, 0.016378861079232566, 0.014980727210424417, 0.01731426587555629, 0.01527863490606048, 0.016862053783326527, 0.016970830414327082, 0.015006637906275702, 0.0158035891782534, 0.016656257781502502, 0.015156533792609386, 0.014795964948262344, 0.018609785179328216, 0.01541178370153579, 0.014813192633959428, 0.015212998329935869, 0.01629696442509569, 0.014237197545194542, 0.0152509762129089, 0.015644298064046585, 0.016462399967368944, 0.014723654457975377, 0.015783942546714898, 0.015269006732554288, 0.01401871759116965 ], "mean_node_assignment_entropy": 4.154226481364971, "normalized_node_entropy": 0.998880323899142, "effective_mass_slots": 63.8603938200883, "node_slot_mutual_information_nats": 0.002472872831686068, "normalized_node_slot_mutual_information": 0.0005946002285037565, "centered_token_effective_rank": 1.1750004928760815, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.011733211054024547, "token_RMS": 0.2583245760505941, "mean_token_offdiagonal_cosine": 0.9979336467841978, "temperature": 0.4937331974506378 }, { "layer": 7, "head": 3, "slices": 64, "native_nodes": 33555, "mass": [ 0.01448164189889674, 0.017177999599537985, 0.01658608239955091, 0.014585821102095057, 0.015895588531331918, 0.01628498890787039, 0.015747717431256684, 0.015682270455172342, 0.01519851897587974, 0.015041314309978873, 0.01555243494318563, 0.01509937707944259, 0.01651000414679812, 0.015166900539278069, 0.01496272410854926, 0.017724513146632735, 0.015295378253948553, 0.016269045754086997, 0.014507152766795312, 0.016510204564224338, 0.014989669670700147, 0.014304038940097194, 0.01629199119346643, 0.016211134436020035, 0.015634393450075345, 0.015520382464357246, 0.013819614791335887, 0.015979281859534844, 0.014827629044233438, 0.015236472492118056, 0.01529541643614799, 0.014443323720285966, 0.01658023687462847, 0.015755547226061498, 0.015999591207672137, 0.01602967155220509, 0.015422041965541675, 0.016978646321254103, 0.01753713967651351, 0.01787706170399855, 0.01480159764683934, 0.014354965189615335, 0.016311014599231063, 0.014808398423040034, 0.017595709457764806, 0.015521995224016441, 0.01337338333949169, 0.016577649791631306, 0.01487109661510295, 0.015725217947483532, 0.015061916420609537, 0.016777247905169438, 0.015350653858714649, 0.013688829515560642, 0.01506695540039297, 0.0170803910155738, 0.015551367644822876, 0.015225593825226844, 0.014568921490047128, 0.015355010379220713, 0.016908932913629323, 0.016077094268058766, 0.016180252588923447, 0.014152840466459628 ], "mean_node_assignment_entropy": 4.155273867960253, "normalized_node_entropy": 0.9991321671402932, "effective_mass_slots": 63.87062080746851, "node_slot_mutual_information_nats": 0.0015856194261685275, "normalized_node_slot_mutual_information": 0.0003812608804784231, "centered_token_effective_rank": 1.2312752857718015, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.008963907203389972, "token_RMS": 0.19702039925349113, "mean_token_offdiagonal_cosine": 0.9979604489031433, "temperature": 0.4999639689922333 }, { "layer": 7, "head": 4, "slices": 64, "native_nodes": 33555, "mass": [ 0.015102869867828871, 0.01450669250545812, 0.015379003898495001, 0.017087391476826738, 0.015021421205912426, 0.014619463211281557, 0.014869961423782465, 0.01556934208938478, 0.0161876386162235, 0.015902776988074706, 0.01468765574085954, 0.015043376462159098, 0.016055738774085653, 0.016607270375029588, 0.016226266061207424, 0.016058333077240524, 0.016805737783860082, 0.01715490187855107, 0.016594586073725, 0.016478380834762323, 0.01618519677569491, 0.014479657704258459, 0.016526689985468638, 0.015450360474855734, 0.016311428166738282, 0.017014408735507042, 0.014648083330362018, 0.014941162867975331, 0.015884031647198194, 0.015287949153194527, 0.013552123673078138, 0.013801235362533228, 0.01681495585994116, 0.015026251611460376, 0.017182181740598483, 0.015476678012513737, 0.013779019422672612, 0.014262966198713195, 0.015495598223934655, 0.013704829469418636, 0.015360138807021112, 0.014469804242641875, 0.014396526159741114, 0.01629524793533597, 0.015200832196278324, 0.014048760853354745, 0.01668062078321194, 0.017242871919201944, 0.014756760069682157, 0.015385852176282511, 0.016220406529920642, 0.015121766683027857, 0.01693607741402582, 0.016051069776995396, 0.015528120001967573, 0.015239887745422856, 0.01685793696601688, 0.016533509882133954, 0.015409804923195382, 0.014878886202446014, 0.016928176910702083, 0.015043207790884099, 0.017064088030055335, 0.016566028879771395 ], "mean_node_assignment_entropy": 4.155859521086146, "normalized_node_entropy": 0.999272986950361, "effective_mass_slots": 63.87083331924566, "node_slot_mutual_information_nats": 0.001003293517404913, "normalized_node_slot_mutual_information": 0.00024124109701935214, "centered_token_effective_rank": 1.362393912491212, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.007138590893880763, "token_RMS": 0.20506968308661339, "mean_token_offdiagonal_cosine": 0.9987952090440464, "temperature": 0.500993549823761 }, { "layer": 7, "head": 5, "slices": 64, "native_nodes": 33555, "mass": [ 0.016120856347079028, 0.01657114956395279, 0.014810286466581262, 0.016448319299836804, 0.015080186914202398, 0.014607506832062106, 0.01409322758369129, 0.01614293423065653, 0.014794787634988421, 0.016258768011103788, 0.015233511652094146, 0.016706079016716474, 0.01586474599248716, 0.015370756809948996, 0.017462143716847126, 0.014456946866298429, 0.015478761947900169, 0.015625029059318205, 0.01574779696981272, 0.01390564226240226, 0.01598069764126379, 0.015386459830831423, 0.015094267300103482, 0.015344860068448981, 0.014368688825042171, 0.016756368113614785, 0.015370950438753892, 0.015981453643050675, 0.015146550533352435, 0.01472159501053021, 0.016364744224149422, 0.015335473786351746, 0.01434768636616158, 0.01676981379522943, 0.01594980008982876, 0.015846825018386346, 0.01655532369725134, 0.01520965789948361, 0.01438760572441856, 0.0157766610960461, 0.01494532044202027, 0.016798537018727064, 0.016734682288142417, 0.01661995054531974, 0.01512877930939917, 0.014872590032997474, 0.017497718975479717, 0.01633499047053236, 0.015836115075171776, 0.015814480927014134, 0.014153312478697996, 0.014927559172824267, 0.015371447137299264, 0.016935362611529396, 0.016147650771327078, 0.01505758107053245, 0.01594795859654034, 0.014343992127637104, 0.01650318957098087, 0.016521662609234527, 0.01603726151303143, 0.015086814682254038, 0.01416777820463637, 0.016740344290199285 ], "mean_node_assignment_entropy": 4.15610567934656, "normalized_node_entropy": 0.999332175500623, "effective_mass_slots": 63.90003811191419, "node_slot_mutual_information_nats": 0.0012142784673336848, "normalized_node_slot_mutual_information": 0.00029197225384675974, "centered_token_effective_rank": 1.388277686602376, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.007521733343890986, "token_RMS": 0.2513803140294406, "mean_token_offdiagonal_cosine": 0.9991091213124191, "temperature": 0.5000736713409424 }, { "layer": 7, "head": 6, "slices": 64, "native_nodes": 33555, "mass": [ 0.015593591253805394, 0.015818823398301977, 0.016279685834314186, 0.015274077307501167, 0.016955677061749008, 0.015474878876375523, 0.015140709342722823, 0.014239150352888181, 0.01601451844839603, 0.016332579295593213, 0.01669396774932429, 0.015759832831300852, 0.016128872390633864, 0.014494437208384467, 0.015047698802030348, 0.01500950728567666, 0.015934372096870777, 0.014426945780216981, 0.015491945205683495, 0.01535367650947016, 0.015873627982658407, 0.018538607980524743, 0.0159382312095775, 0.015539158612068933, 0.01598706340758376, 0.015572759655911119, 0.016697226108317312, 0.015007846237989727, 0.01462716400299142, 0.01463908613446421, 0.016989978957344515, 0.016150520431942768, 0.01578120695938587, 0.016048701966522506, 0.014919833123329774, 0.01703714402935119, 0.015458281207793632, 0.015663640863189814, 0.01515908939818026, 0.015973588404059855, 0.01606168641728636, 0.017455911741960716, 0.015448450029197197, 0.015822132084351584, 0.016066348210982996, 0.015165614937294028, 0.015865861588508678, 0.014577886954642696, 0.015296613328955983, 0.014684651261788112, 0.017200842019791683, 0.015018073876817604, 0.015210984246740085, 0.015839134524958692, 0.01662345507727985, 0.014906954836539426, 0.0144650486935579, 0.014680853868193393, 0.016264510596862958, 0.015496349060551862, 0.01383083582769859, 0.014023220676036709, 0.014633617629430997, 0.016293258640280674 ], "mean_node_assignment_entropy": 4.156388606909451, "normalized_node_entropy": 0.9994002051992755, "effective_mass_slots": 63.90190320109452, "node_slot_mutual_information_nats": 0.0009605380851889223, "normalized_node_slot_mutual_information": 0.00023096058868117314, "centered_token_effective_rank": 1.5691498130646964, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.00632705578073291, "token_RMS": 0.22036574657211994, "mean_token_offdiagonal_cosine": 0.9992177338841154, "temperature": 0.5016274452209473 }, { "layer": 7, "head": 7, "slices": 64, "native_nodes": 33555, "mass": [ 0.013969915077859635, 0.015383133763368856, 0.014319308269688464, 0.017998227522980616, 0.015229485950911077, 0.016268101266155504, 0.01605158173954446, 0.016866056659598762, 0.017158325885011337, 0.016581386325273326, 0.014454882359430538, 0.015968373742450272, 0.014056320110872452, 0.016526814790603844, 0.016159158583073773, 0.014863024597493214, 0.015558849422575028, 0.01531219250386415, 0.014600732525656003, 0.01626006650072712, 0.015085961675872846, 0.01472760447342742, 0.015878883510969014, 0.015060188142534759, 0.015694084983458374, 0.014874303650658846, 0.016017465578785438, 0.015404126691645712, 0.01470737166004984, 0.014732689547419521, 0.016324344590598266, 0.01644452334023738, 0.016279300440182196, 0.016556793542845543, 0.015112538990440995, 0.016684991139027626, 0.018081504491956762, 0.01656127347853887, 0.014399570724346113, 0.014552591583380542, 0.01388224993287438, 0.014391181995111612, 0.015974069831357452, 0.017053468562740315, 0.014199905519750767, 0.014874620080902349, 0.014462814148698091, 0.01634992057150317, 0.015601213718608361, 0.015221190449328336, 0.015658580976539885, 0.014840059268444158, 0.015256039450516014, 0.016461619355102007, 0.015585989007070459, 0.01675272003346878, 0.014846592490920596, 0.01612867033113232, 0.015618379240556653, 0.016136625725145917, 0.016103496860156, 0.015578134844531744, 0.01607877810901571, 0.016177629360507532 ], "mean_node_assignment_entropy": 4.155742718797951, "normalized_node_entropy": 0.9992449019367037, "effective_mass_slots": 63.885566033690296, "node_slot_mutual_information_nats": 0.001350733403161719, "normalized_node_slot_mutual_information": 0.0003247827303840809, "centered_token_effective_rank": 1.593455017761704, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.00785643040363315, "token_RMS": 0.22481092761255575, "mean_token_offdiagonal_cosine": 0.9988714604263287, "temperature": 0.4997074007987976 }, { "layer": 7, "head": 8, "slices": 64, "native_nodes": 33555, "mass": [ 0.016604060007223444, 0.014566428815875762, 0.01681901819311774, 0.01522243738231614, 0.015623240140306267, 0.01599227025918413, 0.01614743919976221, 0.015325142641601336, 0.015030100801217184, 0.016603825512133852, 0.01591688851434407, 0.015064534930407672, 0.015420318325678134, 0.015587431931766335, 0.015797580957230647, 0.016447284178523286, 0.016931780255335115, 0.014852826541805655, 0.01598944589573373, 0.017290554828153577, 0.014623237746631561, 0.01607192386041107, 0.016071537425407106, 0.01676438272378494, 0.015012864919465118, 0.014831939616633712, 0.015855396936759032, 0.01517834644167927, 0.0159520793658995, 0.015909903282789517, 0.014834163275524705, 0.01543941050504225, 0.016288527155400762, 0.015822147976869646, 0.016465035837513912, 0.016166144482083113, 0.014131476235345745, 0.014882210190246258, 0.01543278543270526, 0.01609060719427187, 0.015746295963283703, 0.0155101981697344, 0.013235850534353957, 0.015260420520801718, 0.015896439136757687, 0.01548177894449655, 0.015352957068762496, 0.01543260827319728, 0.015482995600743944, 0.01651573514224715, 0.015937745729701393, 0.016040820069139237, 0.014769039115356566, 0.014937400456888189, 0.014481092550914062, 0.015230689314183615, 0.01568699134942552, 0.016027198838423358, 0.01649340303580597, 0.015740464182467606, 0.015345804999407473, 0.015799788355182302, 0.01621598021287292, 0.014321572518895136 ], "mean_node_assignment_entropy": 4.156213627249452, "normalized_node_entropy": 0.9993581314846525, "effective_mass_slots": 63.929690850870536, "node_slot_mutual_information_nats": 0.0015702717732484217, "normalized_node_slot_mutual_information": 0.00037757055001890285, "centered_token_effective_rank": 1.2142119911594982, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.009456524031697087, "token_RMS": 0.25595594136665895, "mean_token_offdiagonal_cosine": 0.9986319530904045, "temperature": 0.4975091814994812 }, { "layer": 8, "head": 1, "slices": 64, "native_nodes": 33555, "mass": [ 0.015007590997818152, 0.01735643958638187, 0.015194284406701146, 0.016281539879127564, 0.01640019138028473, 0.015722682779995417, 0.014477070498056382, 0.015544788883045602, 0.013960680989386196, 0.016323931941896068, 0.016069426013798193, 0.014678930407055549, 0.015319025630444719, 0.01491859071370258, 0.01383385341114893, 0.015661222063173722, 0.017576884908867812, 0.018100748490361765, 0.014996296834106809, 0.014595074714509089, 0.015340609513855987, 0.014308838828961844, 0.014715687684868002, 0.016954051208230087, 0.016500488643478556, 0.015186639007576717, 0.014332276174412534, 0.016496362012102278, 0.01550457207828809, 0.017805041422707467, 0.015802771840236204, 0.01699161333009887, 0.01511760729071818, 0.015340728506794488, 0.014344855345482532, 0.015487320686844134, 0.01657413356525166, 0.016261140788241837, 0.013797395929806307, 0.016713985081580555, 0.01658922154015071, 0.015950498423444596, 0.01673346221626844, 0.015018831219314872, 0.013644675917911239, 0.01628832082086354, 0.014324277932649485, 0.018103923037816952, 0.014886061816013192, 0.015780305598447945, 0.016308033258370103, 0.015080462082678702, 0.015163781385546273, 0.016757997008482687, 0.014636930506418313, 0.015595367179209048, 0.015286465768187887, 0.014759384100192875, 0.015122404669411064, 0.015876698944644926, 0.015225493957166025, 0.01578160221934776, 0.016451959700311568, 0.015038467741916502 ], "mean_node_assignment_entropy": 4.1548055821525836, "normalized_node_entropy": 0.9990195682048859, "effective_mass_slots": 63.860158327740734, "node_slot_mutual_information_nats": 0.0018900844253844085, "normalized_node_slot_mutual_information": 0.00045446923789392536, "centered_token_effective_rank": 1.2671511878112538, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.00911588379492441, "token_RMS": 0.2635482830507794, "mean_token_offdiagonal_cosine": 0.9989472509246343, "temperature": 0.49790358543395996 }, { "layer": 8, "head": 2, "slices": 64, "native_nodes": 33555, "mass": [ 0.016160856477884046, 0.016401986173748072, 0.014997209935759781, 0.014309632963877137, 0.014631676971201853, 0.013906122243436807, 0.01575471710987115, 0.01639441753636461, 0.016735424012730638, 0.014463371351292442, 0.01622531426510678, 0.015124672034017341, 0.01521427077422344, 0.015793309950566322, 0.016694706702370915, 0.015557481133412487, 0.016971834297296793, 0.014242077046693606, 0.015408116009798566, 0.01758431361338995, 0.01508062499324272, 0.015434247837127844, 0.014975535076579507, 0.014477854240016803, 0.015915656705918307, 0.015869288479473824, 0.0171033987642498, 0.014690027623922365, 0.01700828858147327, 0.015039891503044078, 0.015216757375105822, 0.01558217965066433, 0.01796879849943838, 0.017163021207985962, 0.016564996051556962, 0.016248845092168448, 0.014972802863022357, 0.015243160933330533, 0.01538137854388473, 0.01614021751760433, 0.01642800363721866, 0.015324534276549472, 0.014868039046870073, 0.016912520478652063, 0.015533266848111682, 0.015721544987827234, 0.016578847381092453, 0.01432515039941208, 0.01558051553176392, 0.016997927506838, 0.01594919663854082, 0.015482024995035865, 0.016194007387866322, 0.013400135147218075, 0.016210576194718607, 0.01468579612645684, 0.015738841278428266, 0.015872743592801962, 0.015602746256880408, 0.013525276245459673, 0.01529395574507584, 0.015186027930240405, 0.014672415411218305, 0.015267424823777622 ], "mean_node_assignment_entropy": 4.154898069695878, "normalized_node_entropy": 0.9990418067582283, "effective_mass_slots": 63.882559387342596, "node_slot_mutual_information_nats": 0.002148318398465854, "normalized_node_slot_mutual_information": 0.0005165613832862013, "centered_token_effective_rank": 1.1222232663419125, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.010461359548481415, "token_RMS": 0.24641071287376157, "mean_token_offdiagonal_cosine": 0.9981919495956254, "temperature": 0.4940894842147827 }, { "layer": 8, "head": 3, "slices": 64, "native_nodes": 33555, "mass": [ 0.015797153422216235, 0.015382145336828986, 0.015142944438327196, 0.015209885382301237, 0.016197023339871545, 0.016663258778345718, 0.016070620417500542, 0.014161806307553206, 0.01567818326593853, 0.016362084398877957, 0.015886191376088727, 0.017407377820471425, 0.016122441303782636, 0.0165212789875991, 0.016159823625383898, 0.016554297772758658, 0.014706145206028065, 0.014423937424451153, 0.015810364294248185, 0.015061982120636358, 0.014802890644254583, 0.01595797079129997, 0.016160728572114952, 0.01611019128756194, 0.014081902013107137, 0.01465865815247918, 0.014716698363231282, 0.014616338758073128, 0.014366276758803313, 0.014427664840846227, 0.016317052975482377, 0.01584922440928472, 0.015615288261560086, 0.014443042054367153, 0.016598582793387422, 0.015195062211983216, 0.014715348674319412, 0.016671284232935294, 0.01473957548050354, 0.016323201761100425, 0.015548437786963147, 0.01628590356525255, 0.01610893860726887, 0.014629948612581758, 0.015590621877920742, 0.014336117718165045, 0.015500398219517687, 0.015406231704120699, 0.014662291330464807, 0.016007736297758667, 0.015785482155267084, 0.017110047681969602, 0.015354654204470623, 0.015268053944014327, 0.015793411730204543, 0.016286565373176933, 0.0165514920762267, 0.016863480054542607, 0.016188181243418334, 0.015611707839008513, 0.016707640042385015, 0.01546530247260856, 0.01551625876772259, 0.015735168498458522 ], "mean_node_assignment_entropy": 4.156262488047889, "normalized_node_entropy": 0.9993698800232524, "effective_mass_slots": 63.918112418696, "node_slot_mutual_information_nats": 0.0013402826032615778, "normalized_node_slot_mutual_information": 0.0003222698441858714, "centered_token_effective_rank": 1.232546286329227, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.007740627404702585, "token_RMS": 0.24384599139703692, "mean_token_offdiagonal_cosine": 0.9990105826853835, "temperature": 0.4985952377319336 }, { "layer": 8, "head": 4, "slices": 64, "native_nodes": 33555, "mass": [ 0.015532981799178535, 0.014063453873431535, 0.01620844236730912, 0.015632674672462255, 0.017451272768012466, 0.015503181148425351, 0.01506175321112303, 0.015371405766287968, 0.014527665776986201, 0.015598449870614293, 0.0158079080127103, 0.01593340130090145, 0.014832856854602346, 0.014606810363728151, 0.015065666455112153, 0.015652014057144618, 0.016595669264106223, 0.014007407770331295, 0.016644352202258884, 0.01559256270279154, 0.016021453914352037, 0.016040485966109675, 0.017617604303747002, 0.015442265132243668, 0.01469047706440422, 0.01470124856527934, 0.016137256089005646, 0.015800667793777673, 0.014518746341624597, 0.015105871191450078, 0.015615659354957085, 0.016020777820347363, 0.015000975809768209, 0.014995074708271739, 0.017126692889524436, 0.015352685141916017, 0.016579359133035775, 0.014074669066428552, 0.015773353264354154, 0.01598148485139121, 0.015827242622098957, 0.016653863665517493, 0.01531471319134705, 0.016132733370093487, 0.016227292255174428, 0.015606851539556661, 0.015415324640978362, 0.017252677066895275, 0.015685071789684122, 0.015321294254355656, 0.016080826812043106, 0.01438147695717056, 0.015188441296556172, 0.015975573222827198, 0.013556189732544202, 0.016044331563328858, 0.0171532858136751, 0.016146069156643513, 0.017763364471460735, 0.016080806250283596, 0.013661944471808123, 0.014378242175390514, 0.01682418865824626, 0.01504145659539072 ], "mean_node_assignment_entropy": 4.155134573352115, "normalized_node_entropy": 0.9990986738668959, "effective_mass_slots": 63.88797848440753, "node_slot_mutual_information_nats": 0.0019966401982136617, "normalized_node_slot_mutual_information": 0.00048009048540040115, "centered_token_effective_rank": 1.2622998950471735, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.009145772668307066, "token_RMS": 0.32189915974328115, "mean_token_offdiagonal_cosine": 0.9992788798406785, "temperature": 0.49492332339286804 }, { "layer": 8, "head": 5, "slices": 64, "native_nodes": 33555, "mass": [ 0.01568235053820029, 0.01452054672760298, 0.01572627829938168, 0.01567915257337378, 0.014427944303592498, 0.01726336538827183, 0.017478887828527957, 0.014603125303373076, 0.014980937979894483, 0.015860307729780684, 0.016429961626957076, 0.016798812663425474, 0.015169662175328983, 0.015570319416630092, 0.016456247349298136, 0.015900560643792045, 0.013243654320376486, 0.014734748458747065, 0.01454659400707922, 0.015287688928927587, 0.015608498864314396, 0.01585379363784397, 0.01638740993547153, 0.016097551049069826, 0.015051261973645716, 0.015662424464260007, 0.014788516863432287, 0.014607461279078, 0.013606763225353532, 0.015400199820093934, 0.015304814471588851, 0.014954224561448381, 0.015111366152289309, 0.014164388095514551, 0.014081712191497857, 0.014541082367392023, 0.01576766089349319, 0.016027637986923415, 0.016189076744504035, 0.015158795385180598, 0.015587592027601809, 0.014684208770712909, 0.015529837122739622, 0.01623803897676294, 0.016087164286390277, 0.016044971368781698, 0.01717610343288219, 0.01560951824612375, 0.016613113707520714, 0.016504594613371226, 0.015195740640053511, 0.015886797047507575, 0.015505661450591781, 0.016152572509297147, 0.016598143419821212, 0.016289824677647634, 0.01633166868979037, 0.01671765522606423, 0.015342744057291694, 0.016194024167382057, 0.016462349818167077, 0.015881070139469707, 0.016360081484558784, 0.016280737871532024 ], "mean_node_assignment_entropy": 4.156088648758396, "normalized_node_entropy": 0.9993280805097752, "effective_mass_slots": 63.9032059498876, "node_slot_mutual_information_nats": 0.0012808827262960065, "normalized_node_slot_mutual_information": 0.00030798719286459736, "centered_token_effective_rank": 1.3364836485580145, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.007993456936263526, "token_RMS": 0.23274046987562302, "mean_token_offdiagonal_cosine": 0.9988423996886011, "temperature": 0.49784916639328003 }, { "layer": 8, "head": 6, "slices": 64, "native_nodes": 33555, "mass": [ 0.01646288988037381, 0.016276529045436087, 0.014600098267588402, 0.015193331479163616, 0.01549740846622294, 0.01452928369995548, 0.016291623391436406, 0.01719059509940855, 0.017471817206114536, 0.015376549937422736, 0.01527510868745418, 0.015558359380508922, 0.01535990397920331, 0.017074422093865733, 0.0163465303025397, 0.014357520377638398, 0.016806122590305464, 0.015411464536775998, 0.01587943718458493, 0.016258418039897395, 0.015366570431072413, 0.015759076972619298, 0.015430132847297629, 0.014694309767461661, 0.01684083626398101, 0.016165256417644946, 0.015119549102116459, 0.01457477044802636, 0.016511867296424152, 0.015898038319618314, 0.015432922856477268, 0.015777217187076215, 0.016202421732565503, 0.016592013456328155, 0.015473798047704004, 0.016603093075458722, 0.014977635054333815, 0.014713436194489173, 0.015165918736272155, 0.013448590488368602, 0.015322494414550211, 0.016561856249941567, 0.01694007316528485, 0.016687995650072937, 0.014330850724803097, 0.014902128930954531, 0.015556214909210652, 0.014735646169638779, 0.015399065925686278, 0.014525875700503087, 0.016151465458850663, 0.015194790548449424, 0.014333642424990311, 0.014099102197213574, 0.017685152727940134, 0.014423114497416057, 0.014297865254818647, 0.014684314424797366, 0.015871166343885516, 0.015782426882381455, 0.018352278807779048, 0.014569599835407208, 0.01554953567886309, 0.016078474590586197 ], "mean_node_assignment_entropy": 4.155617227176833, "normalized_node_entropy": 0.9992147275801271, "effective_mass_slots": 63.878955784161285, "node_slot_mutual_information_nats": 0.0013727495141635515, "normalized_node_slot_mutual_information": 0.0003300764860777483, "centered_token_effective_rank": 1.2206115388623855, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.00853238301627502, "token_RMS": 0.2796505255347698, "mean_token_offdiagonal_cosine": 0.9991019300939717, "temperature": 0.4971577823162079 }, { "layer": 8, "head": 7, "slices": 64, "native_nodes": 33555, "mass": [ 0.015940103671457848, 0.013978011223759973, 0.01571867136966652, 0.015251320394400228, 0.015075009585569281, 0.01669473728155871, 0.014583650406218555, 0.016314349222843354, 0.013756547295469683, 0.016692932344103798, 0.015820101525752764, 0.01601361831081991, 0.0161900650375696, 0.015841177923347, 0.016059252440611894, 0.015420270742824368, 0.015028006012668474, 0.017667077304974403, 0.01535163586691406, 0.015253352853307877, 0.015315429899897611, 0.014430650851642511, 0.015405128025669772, 0.015014483180380563, 0.016286125532658822, 0.016655048212410352, 0.016417809501495708, 0.016346047809945634, 0.013954259696312614, 0.015298351575471289, 0.014079618548235832, 0.01585084867260261, 0.015316324280814999, 0.015247043648729416, 0.015287085284797174, 0.015320767582488688, 0.016398550635624207, 0.016951234874165267, 0.014936612649221095, 0.015508665523245195, 0.014618689764855588, 0.015644602490262345, 0.014559804993101388, 0.0158080188567775, 0.014313254164063599, 0.017024218363068735, 0.01581464789576319, 0.014935409697168205, 0.014056849460829576, 0.016664358341741396, 0.01669254250127243, 0.016836273597200604, 0.015723009117784743, 0.01476935335136467, 0.01572854530661321, 0.018530123983943513, 0.015318643158925368, 0.015509200945547629, 0.015527358963350623, 0.01612961335653627, 0.014912236505718865, 0.014875398522581238, 0.017973205600738205, 0.015364664073102592 ], "mean_node_assignment_entropy": 4.155736700426948, "normalized_node_entropy": 0.9992434548243703, "effective_mass_slots": 63.882132980491264, "node_slot_mutual_information_nats": 0.0013030127896431765, "normalized_node_slot_mutual_information": 0.0003133083483055175, "centered_token_effective_rank": 1.1100604185999896, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.009104549129061475, "token_RMS": 0.22618315656395416, "mean_token_offdiagonal_cosine": 0.9983669951299443, "temperature": 0.49664101004600525 }, { "layer": 8, "head": 8, "slices": 64, "native_nodes": 33555, "mass": [ 0.015142722542529099, 0.015055034768861606, 0.01512684277810446, 0.016141037104812034, 0.015155243739872091, 0.015298222747344522, 0.015721989994236654, 0.01616741760664833, 0.015229276118564403, 0.01597083820874681, 0.01624364431405135, 0.01567973731691552, 0.014975604779197889, 0.01603804416832395, 0.015693623536700615, 0.015095745866351921, 0.017802595255712986, 0.015391783592266336, 0.01603560263873581, 0.016355094720317084, 0.016409429846904064, 0.015155274511846855, 0.015579556203454916, 0.014459921536830326, 0.016541730040261632, 0.01607132342435456, 0.015386878613467509, 0.015575692893870896, 0.01611075629627979, 0.016473368370347363, 0.015282248178334035, 0.015139078651433454, 0.01533203346435785, 0.015996901287706, 0.014254398873477458, 0.016192301249123513, 0.015480216559105375, 0.01605940911404854, 0.015537662564822217, 0.015620978121916223, 0.015362722078465106, 0.015918129803081316, 0.015371691851374726, 0.015842005914301148, 0.016505499245609182, 0.01566041389774715, 0.015738328797802396, 0.01573022993531557, 0.016165399329073203, 0.014316191519271278, 0.01585930850386793, 0.014916663920314491, 0.014228136563761599, 0.015752585175887224, 0.01492591636038061, 0.01563738792767767, 0.014313716182160211, 0.014969692513709708, 0.016233705653780007, 0.016966152009893515, 0.016252662888236125, 0.015201470193541533, 0.015625007355380582, 0.01552772096109415 ], "mean_node_assignment_entropy": 4.156771745129192, "normalized_node_entropy": 0.9994923304675414, "effective_mass_slots": 63.944890185936785, "node_slot_mutual_information_nats": 0.001249876434056496, "normalized_node_slot_mutual_information": 0.00030053175552288144, "centered_token_effective_rank": 1.2882163578489527, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.007696344293651689, "token_RMS": 0.26110785982447426, "mean_token_offdiagonal_cosine": 0.9991394940130307, "temperature": 0.49899494647979736 }, { "layer": 9, "head": 1, "slices": 64, "native_nodes": 33555, "mass": [ 0.017984955868654753, 0.015530387427163247, 0.016764808352439906, 0.016154821991066725, 0.015055590016719366, 0.016018118965200112, 0.015490066992667672, 0.014892589312222733, 0.014521664410447634, 0.015426953969157775, 0.015370522307948381, 0.014610660311560164, 0.016190673025457333, 0.015891886857353377, 0.016021576436324486, 0.014684822942688006, 0.015375400118745176, 0.01468952800018313, 0.01556449201878928, 0.015624808396762724, 0.014449193291787978, 0.015669112421978545, 0.014036279787535534, 0.014737460383044333, 0.014966053168789371, 0.017285554406790037, 0.016169906290745686, 0.016057502584219105, 0.014660161192430512, 0.016156848041098407, 0.015567390640053774, 0.015723915341972303, 0.014233767756013494, 0.015811795686614342, 0.01704371622304921, 0.014928004995178287, 0.014234162212076504, 0.016569587345286536, 0.01542727416510301, 0.014995659114311392, 0.016212984429975725, 0.01773139613539962, 0.015817008093593418, 0.01627866746809191, 0.01496431700288006, 0.016344777531833048, 0.016412188077347987, 0.014802502862600639, 0.01429928367977734, 0.016005392673334756, 0.014642448216841021, 0.016729743237314503, 0.014864957132135955, 0.01521607503203908, 0.01633190988060495, 0.015576429613436402, 0.015510468067127544, 0.01583092006337219, 0.014685827686537832, 0.015238814429780922, 0.017149761082936758, 0.01691325435795433, 0.017307581129436558, 0.014545619156505263 ], "mean_node_assignment_entropy": 4.154690788163388, "normalized_node_entropy": 0.9989919660850632, "effective_mass_slots": 63.894092466721176, "node_slot_mutual_information_nats": 0.002536119286292049, "normalized_node_slot_mutual_information": 0.0006098077862393994, "centered_token_effective_rank": 1.158742598000923, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.011648460560813274, "token_RMS": 0.20938429548896312, "mean_token_offdiagonal_cosine": 0.9969621988575018, "temperature": 0.4911794662475586 }, { "layer": 9, "head": 2, "slices": 64, "native_nodes": 33555, "mass": [ 0.01534279100685721, 0.014525223187486083, 0.016880560053323442, 0.01637392663164446, 0.0163999346816101, 0.016737970661118266, 0.014769957427669834, 0.01617264912917937, 0.015038135403056974, 0.014802884456253749, 0.014499720259518378, 0.014869178697103814, 0.014928813100126484, 0.016188827730441054, 0.016143219306817433, 0.0152695360427661, 0.015385745200055836, 0.015573569646208141, 0.01659029215999473, 0.01563199161497909, 0.01542102477116772, 0.014262509422029536, 0.01568861473182411, 0.013310620730548249, 0.01551408482161218, 0.017082160068591222, 0.015088515715250855, 0.01538373823732142, 0.01368754928679855, 0.01525213337383888, 0.01562909927075381, 0.015011766347842632, 0.016131038621490987, 0.017492445782154482, 0.01688057680014366, 0.015271143116375547, 0.016389907723299093, 0.017140786690606105, 0.016126342907724474, 0.013742252120826155, 0.014993628497177053, 0.014846703707350294, 0.015832081018869213, 0.015302532049782176, 0.015467142237114704, 0.016892746904335617, 0.016503423333696487, 0.01605403530260851, 0.015209241971959745, 0.017592099053232047, 0.015979329509056374, 0.016405023187215727, 0.01598341291077294, 0.01418958853507247, 0.015185326366818417, 0.01621385166387394, 0.016316117867000313, 0.016073466017908905, 0.014623872093265705, 0.01638541943676056, 0.015126872498826837, 0.014895508912017406, 0.015693914675992134, 0.015603425411829169 ], "mean_node_assignment_entropy": 4.155680418961631, "normalized_node_entropy": 0.9992299219925526, "effective_mass_slots": 63.89362514583272, "node_slot_mutual_information_nats": 0.0015391744691877207, "normalized_node_slot_mutual_information": 0.0003700932289600046, "centered_token_effective_rank": 1.3393230453587845, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.006877665824844621, "token_RMS": 0.32338360458771276, "mean_token_offdiagonal_cosine": 0.9995759245812551, "temperature": 0.49634426832199097 }, { "layer": 9, "head": 3, "slices": 64, "native_nodes": 33555, "mass": [ 0.015736209543766304, 0.016389565823055417, 0.015680046205513925, 0.01577681621106131, 0.016614872018007706, 0.01670212465994956, 0.016664646897393384, 0.015625127775347847, 0.014796246879381192, 0.01583728220534718, 0.01540557180516474, 0.015243277274268677, 0.016733444642262806, 0.014679633074547891, 0.01607775670102813, 0.01538238033148273, 0.015101933255686194, 0.016747660912443445, 0.014471477657615375, 0.015763468182400243, 0.01441862093460416, 0.015364406271523325, 0.014350093655247143, 0.015586686201089408, 0.014249135910406666, 0.015199364045244352, 0.01659064483984965, 0.015599034555565003, 0.01562040144930945, 0.015869908240577837, 0.015377131921551858, 0.01667733581081941, 0.01622642338044987, 0.015111369229137072, 0.015471993058902143, 0.015046934111146782, 0.01592454878981663, 0.015038879551165524, 0.015679079497843924, 0.014258713682694631, 0.016624009622239557, 0.016336036243228536, 0.016589333499084997, 0.015250851035078136, 0.015641026206968472, 0.01672150962612052, 0.01473605443389467, 0.015306349158605678, 0.01588667181785803, 0.015757999030895108, 0.015860462373153382, 0.015471662131667297, 0.01558331087503081, 0.014579664703473535, 0.015260303004962191, 0.015337074939880397, 0.01662761748387728, 0.01479083838971281, 0.015142140511493847, 0.015245501174018478, 0.01537480542932781, 0.016779069996778504, 0.016084207372825673, 0.015923253231465533 ], "mean_node_assignment_entropy": 4.156271440855654, "normalized_node_entropy": 0.9993720327184797, "effective_mass_slots": 63.93724506310911, "node_slot_mutual_information_nats": 0.001630615565491489, "normalized_node_slot_mutual_information": 0.00039208016498848734, "centered_token_effective_rank": 1.092445640840535, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.009806836132812927, "token_RMS": 0.29181205347373584, "mean_token_offdiagonal_cosine": 0.998941837144798, "temperature": 0.4939483404159546 }, { "layer": 9, "head": 4, "slices": 64, "native_nodes": 33555, "mass": [ 0.01328677893589198, 0.01555547623748978, 0.018104355828510436, 0.014367651036685029, 0.015107956843684087, 0.017773536911739125, 0.015247052743633499, 0.014741827643222051, 0.01588468482853039, 0.016386936717773404, 0.015736933689186442, 0.017053166594044888, 0.01497490667106453, 0.015058664005733233, 0.014023022971255752, 0.01656057845349817, 0.01593445245542366, 0.016381925083319374, 0.015123668346166024, 0.015530937153864343, 0.015598852910665397, 0.015402370072610375, 0.016215361023468572, 0.015119041034829962, 0.015373401048452643, 0.01725360831747315, 0.015381525888858076, 0.015281153808085179, 0.01542387651050589, 0.015093857303429398, 0.015466498030895556, 0.015622042797837807, 0.01563764673191013, 0.015479628923259572, 0.016908413143957572, 0.015302227393643118, 0.016689025173237126, 0.014583501845738107, 0.013553369083892675, 0.01591133614254036, 0.015290527233101495, 0.016101536208591914, 0.014192303938250208, 0.014866426899544023, 0.017557455202632043, 0.014705309944623959, 0.015597961446532077, 0.01566705095546045, 0.01736477127879072, 0.016160773443905496, 0.015688446174178514, 0.014352866082649716, 0.017475700436453894, 0.015586675249618542, 0.01606840614649653, 0.016516351695998896, 0.017292491319758484, 0.014421597242409746, 0.014880258177986196, 0.014741037053743971, 0.014718842958761405, 0.014664901474679774, 0.015319421039423775, 0.016637638343336855 ], "mean_node_assignment_entropy": 4.155025799130572, "normalized_node_entropy": 0.9990725191952298, "effective_mass_slots": 63.868906657650264, "node_slot_mutual_information_nats": 0.0018068500507562035, "normalized_node_slot_mutual_information": 0.0004344556013093245, "centered_token_effective_rank": 1.2140147884281343, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.007482367207410541, "token_RMS": 0.26857477852677397, "mean_token_offdiagonal_cosine": 0.99926158187035, "temperature": 0.49593910574913025 }, { "layer": 9, "head": 5, "slices": 64, "native_nodes": 33555, "mass": [ 0.01633042992046734, 0.014822927966431913, 0.01638408576185085, 0.015564172087960815, 0.01392622364860074, 0.01600974043555066, 0.014441814616732183, 0.01534073386758215, 0.01570556631095007, 0.015476713823594487, 0.01739255838596289, 0.014473904918608936, 0.01597447865560036, 0.015445756359016677, 0.016733116145126244, 0.014701391992064404, 0.015269115381942006, 0.013877420344825314, 0.01599416379473916, 0.015726127597650328, 0.01512811751360377, 0.016037610407950334, 0.01517038582992783, 0.01425931224680803, 0.016836620778297236, 0.016748711285992486, 0.015929346929887574, 0.017144704829652215, 0.016170876082368755, 0.01367122209810621, 0.016108813912321437, 0.014992002536218118, 0.014891930155993405, 0.015314677483765091, 0.015159449364721881, 0.015115436693066999, 0.014029908251234378, 0.016089975873860175, 0.01548367074693951, 0.015304384944285192, 0.017164836792493744, 0.015852695735730147, 0.01445476179408203, 0.015580493754358533, 0.017261300418115712, 0.014900858582871324, 0.017243424032231527, 0.015791924657654548, 0.016939175306902504, 0.0159428854924372, 0.01597246827651773, 0.013770396555334689, 0.014784324532140208, 0.015734449130108418, 0.016444724537887378, 0.01568249093668127, 0.015331074319621114, 0.016361704288651985, 0.015487667838131781, 0.015127826868103338, 0.01602069190933006, 0.01547046097031832, 0.017200124223407505, 0.01630163944748635 ], "mean_node_assignment_entropy": 4.155010696666473, "normalized_node_entropy": 0.9990688878202196, "effective_mass_slots": 63.89103577336161, "node_slot_mutual_information_nats": 0.002168369639214518, "normalized_node_slot_mutual_information": 0.000521382687551496, "centered_token_effective_rank": 1.1962560134141675, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.009952739926960202, "token_RMS": 0.25810302143256064, "mean_token_offdiagonal_cosine": 0.9985138993806859, "temperature": 0.4944228231906891 }, { "layer": 9, "head": 6, "slices": 64, "native_nodes": 33555, "mass": [ 0.016060585195439847, 0.015959669633447512, 0.01859089564426572, 0.015181197841072904, 0.014621692392633704, 0.015215702830686012, 0.01538036104784053, 0.01509365874964751, 0.014460406476925132, 0.014570198653257784, 0.014865016770253387, 0.015256019352544324, 0.01516753418420763, 0.016143786835615626, 0.015927239242458757, 0.015745159992100254, 0.014437838424024364, 0.016555371847280693, 0.01717291058415433, 0.01570823574978528, 0.01621964076233326, 0.01613219276718954, 0.01526170307477555, 0.014505415833734877, 0.01644956721713168, 0.014499947720548245, 0.014319053662083504, 0.015839573883426127, 0.014704630413760045, 0.01594724565413559, 0.015213616159034724, 0.015268927493753394, 0.01572747215955579, 0.01525249342730949, 0.016631262684680855, 0.015014689564738202, 0.01804509075007963, 0.015149860581955697, 0.01464454668338533, 0.015649890180551167, 0.015244169120173477, 0.015734905249578927, 0.01629567047140603, 0.015815709712659082, 0.016111729088951428, 0.016478918013602135, 0.015412519544484034, 0.014814967005826548, 0.014852235202901394, 0.015486567065446143, 0.016674909888648885, 0.015470497962989434, 0.016659099465360133, 0.0157547664202027, 0.01608943167608717, 0.017346006537039754, 0.015441656777322416, 0.01513035779422272, 0.015279351353713047, 0.01551588764534926, 0.014509927206415299, 0.015372313558583492, 0.016130090500499412, 0.01578801047995591 ], "mean_node_assignment_entropy": 4.156540729278416, "normalized_node_entropy": 0.9994367828971611, "effective_mass_slots": 63.90924512693751, "node_slot_mutual_information_nats": 0.0009233028124926435, "normalized_node_slot_mutual_information": 0.00022200739813699487, "centered_token_effective_rank": 1.5226368316137726, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.005873499340689882, "token_RMS": 0.23039244858542415, "mean_token_offdiagonal_cosine": 0.9994133284111213, "temperature": 0.49835851788520813 }, { "layer": 9, "head": 7, "slices": 64, "native_nodes": 33555, "mass": [ 0.01713059619230079, 0.015227222291028566, 0.01427264315201209, 0.015801302130919574, 0.015701616447956823, 0.016821335178052343, 0.01573350210777495, 0.014084700257531968, 0.015812304491909103, 0.014798563165303457, 0.015983132052083623, 0.016253953556470642, 0.015579712150013095, 0.016511135629731166, 0.01602803060232332, 0.014764886617248637, 0.015495997202886438, 0.014243827729217121, 0.016129170940954955, 0.015157306592452412, 0.015739336960526155, 0.015787778415491952, 0.01505636962716576, 0.01588604503079878, 0.015868477776980784, 0.016086391487796232, 0.015667710943232747, 0.01602639367371229, 0.014738929253018812, 0.015222964045301473, 0.015291764225126325, 0.016202990750425122, 0.015596318906504784, 0.014450998871801348, 0.014222744909868548, 0.016394429570673174, 0.015620461935595159, 0.015663845200469626, 0.01641986660815852, 0.015028375907488985, 0.014350195113342462, 0.014567373493013812, 0.019151067675256886, 0.01616387814527616, 0.015655427190431174, 0.015009205111870998, 0.01654612789072919, 0.015898763466664707, 0.015484506229190993, 0.014562794261458928, 0.015358405343904001, 0.015777928392399915, 0.01548404855321415, 0.01627947386859906, 0.0152779675031231, 0.015145710420938695, 0.015859233879159086, 0.015944412727319144, 0.01573802894773746, 0.015911875523359244, 0.015244170410869161, 0.015430005792527835, 0.016401407611573698, 0.01625486009126559 ], "mean_node_assignment_entropy": 4.15623812442439, "normalized_node_entropy": 0.9993640218101191, "effective_mass_slots": 63.91755529568296, "node_slot_mutual_information_nats": 0.001355929989375504, "normalized_node_slot_mutual_information": 0.00032603224524411074, "centered_token_effective_rank": 1.3148900221507374, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.009707868983968321, "token_RMS": 0.27467969887599153, "mean_token_offdiagonal_cosine": 0.9988037099554692, "temperature": 0.4989567995071411 }, { "layer": 9, "head": 8, "slices": 64, "native_nodes": 33555, "mass": [ 0.01647500180370241, 0.014724366776306572, 0.014959989633866909, 0.015184792878686041, 0.016012247657449703, 0.016976387221313432, 0.015870727165576474, 0.016808338541159498, 0.014574021966957226, 0.014605430176590127, 0.017298442561797227, 0.01621660433238895, 0.015125882409811641, 0.015425218258071383, 0.014776054846608298, 0.015960109403578674, 0.01727001352190989, 0.015195333195627934, 0.015171844402119692, 0.014902312457900106, 0.016135049302304683, 0.015155180905532186, 0.015659570598769712, 0.016431298644469686, 0.015212273431852772, 0.013853880927114324, 0.014181924492399405, 0.015169767723499955, 0.016659889584999782, 0.014388016581140107, 0.014904495959815885, 0.016162380802199065, 0.01573914391838093, 0.014129638556497025, 0.015743136665732613, 0.015009168252368884, 0.015155273819245946, 0.014692476611168653, 0.015595051958866117, 0.017875991879255045, 0.016086188294776167, 0.015267526255477835, 0.014798745272020268, 0.01581229459116328, 0.015895834610764298, 0.015228346349109998, 0.014013701030101076, 0.016404986192046656, 0.01692922084809189, 0.015402067405818525, 0.016101097244246982, 0.01568915002512081, 0.01707568798947487, 0.015633073418187884, 0.015582901248609837, 0.01570386721295596, 0.016393633649022944, 0.017373673941575365, 0.015094183466999927, 0.016391157318889703, 0.014853877103117988, 0.016315092829477545, 0.01454604578673098, 0.016020918158710786 ], "mean_node_assignment_entropy": 4.155436745796695, "normalized_node_entropy": 0.9991713309814442, "effective_mass_slots": 63.897960457399066, "node_slot_mutual_information_nats": 0.0018506973528280568, "normalized_node_slot_mutual_information": 0.000444998648851895, "centered_token_effective_rank": 1.2118037776755624, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.008026287026951543, "token_RMS": 0.22287080487052136, "mean_token_offdiagonal_cosine": 0.99873163715105, "temperature": 0.49523985385894775 }, { "layer": 10, "head": 1, "slices": 64, "native_nodes": 33555, "mass": [ 0.01443762044891251, 0.01592295795236121, 0.017847507523053308, 0.015910777106526986, 0.01526869887983194, 0.014860768627273781, 0.01506797868970489, 0.01486820106249846, 0.015776368731311146, 0.015979904288820443, 0.01480889988963503, 0.016469691632638308, 0.015921528167709555, 0.014697063600664176, 0.015510987289579628, 0.016498235609360742, 0.014857029980718823, 0.01555986369333716, 0.014209775398789974, 0.015173321134005435, 0.01684904870516234, 0.01499785619902084, 0.01560596824291832, 0.01627107704220102, 0.016207009673617073, 0.015758324713469652, 0.014390204480824642, 0.0157677671189165, 0.014574548970220156, 0.016257662281189727, 0.015860440405597764, 0.015511023151868548, 0.0172061136610514, 0.016074495762642103, 0.015924286462878384, 0.015904340294907636, 0.014749930838947012, 0.016570572671608844, 0.015510658445781797, 0.015387038776303808, 0.01632093312519928, 0.016669376823352268, 0.014933939071794307, 0.016024378296517965, 0.015674764602003066, 0.015077618348162836, 0.015533350333749066, 0.015668513490776113, 0.014627238861203967, 0.016084764208386507, 0.015158156219094374, 0.015366189567267567, 0.015192103182341592, 0.014844945342130232, 0.015285855103819794, 0.01581439495424172, 0.0155756842721081, 0.01565014205227984, 0.014865871937341811, 0.016813873140702852, 0.014322544673560159, 0.016262116717046145, 0.01646162284802656, 0.016746075352510367 ], "mean_node_assignment_entropy": 4.156648082409691, "normalized_node_entropy": 0.9994625958688469, "effective_mass_slots": 63.927523993716314, "node_slot_mutual_information_nats": 0.0011019216598446846, "normalized_node_slot_mutual_information": 0.0002649561523510103, "centered_token_effective_rank": 1.2024385050094637, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.006936319852741538, "token_RMS": 0.2901952938906436, "mean_token_offdiagonal_cosine": 0.9994903280177, "temperature": 0.496909499168396 }, { "layer": 10, "head": 2, "slices": 64, "native_nodes": 33555, "mass": [ 0.016615511861401776, 0.015643238858129002, 0.015571549207509462, 0.015869590184329218, 0.013796984950434946, 0.015080300723403058, 0.01697730042441132, 0.016171761548706982, 0.013804543076654541, 0.015474525037939219, 0.014979834565672163, 0.015359628136139161, 0.016555239465334673, 0.015248088045216428, 0.014329815369375264, 0.014573356257775175, 0.014022234143144442, 0.01552946161703019, 0.014297144074920543, 0.014929876080682941, 0.01641773495502851, 0.015531175094333993, 0.01618517144904986, 0.01752984547723074, 0.018481719052837336, 0.01503756052340228, 0.01572472296703472, 0.016134197728697564, 0.014468156507386126, 0.01562560333682588, 0.01620969733834209, 0.01629694367998595, 0.0156081977595799, 0.015682453666264813, 0.015370651154171477, 0.017372251135639758, 0.015278321420300639, 0.01632859758167322, 0.01632295998615189, 0.017748715610506637, 0.014362787516548122, 0.01645270511932641, 0.01574450254378136, 0.015844291819678905, 0.01613269187104789, 0.014621163250423548, 0.01600458550773036, 0.016068606402832118, 0.01601520288761163, 0.013684329027822096, 0.014700122967261375, 0.015004398383842977, 0.014765712615830269, 0.01706573289360131, 0.016303587870483888, 0.01512653842543973, 0.014710003563934681, 0.0157988267935032, 0.014833840099152945, 0.01537242041196607, 0.01640644261088639, 0.0144836721424196, 0.015491442701769175, 0.016821731906287087 ], "mean_node_assignment_entropy": 4.155484476892716, "normalized_node_entropy": 0.9991828078840317, "effective_mass_slots": 63.87571511659643, "node_slot_mutual_information_nats": 0.001454767131636281, "normalized_node_slot_mutual_information": 0.0003497975544099874, "centered_token_effective_rank": 1.1792330412521794, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.008714130628709484, "token_RMS": 0.27148812647821247, "mean_token_offdiagonal_cosine": 0.9989932214724033, "temperature": 0.4963994324207306 }, { "layer": 10, "head": 3, "slices": 64, "native_nodes": 33555, "mass": [ 0.017129565596151967, 0.017357014307055323, 0.014159028459745003, 0.015859630008521912, 0.015530191386800474, 0.016402746611050657, 0.017384578112661038, 0.016369945976637264, 0.014648429391096049, 0.014962013883452634, 0.016304204336362968, 0.01508236491058764, 0.014696611232629274, 0.014472024027530712, 0.015124168343655516, 0.014599794767338478, 0.014083075952914482, 0.01517821589526166, 0.01464013694004923, 0.015895094672161623, 0.016270538065274433, 0.01409405905644019, 0.016222955522948584, 0.015913639034153878, 0.017376210313718343, 0.014416983954507412, 0.016163989718386722, 0.016108345109600235, 0.01695992718133623, 0.014937583778640264, 0.016108658612984567, 0.014853833507534964, 0.013291831793838919, 0.015714908200504172, 0.015180066727060725, 0.014966779668453091, 0.0176278433398205, 0.01759927877380549, 0.014584611385179212, 0.015755138039087117, 0.01629585293807881, 0.014819002890774256, 0.015089254716119526, 0.01380471810506999, 0.014831346793330882, 0.015345983799547512, 0.017850040572299507, 0.015264388838803168, 0.015475587249179655, 0.014588320192081165, 0.017836188583136174, 0.015833545771373747, 0.014984837041218836, 0.01577522113912042, 0.0160444172764245, 0.01488681869574958, 0.01689640504348706, 0.017164440733795416, 0.015097328001471474, 0.014411152951417133, 0.015532072165498274, 0.01748932284817728, 0.015650376995087045, 0.01500735975532165 ], "mean_node_assignment_entropy": 4.155347770930429, "normalized_node_entropy": 0.9991499370483897, "effective_mass_slots": 63.84821062715656, "node_slot_mutual_information_nats": 0.0011607865220089053, "normalized_node_slot_mutual_information": 0.00027911015980549924, "centered_token_effective_rank": 1.3431888032519348, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.006206346666298598, "token_RMS": 0.345227708682379, "mean_token_offdiagonal_cosine": 0.9997292357379478, "temperature": 0.49885162711143494 }, { "layer": 10, "head": 4, "slices": 64, "native_nodes": 33555, "mass": [ 0.01555697498813931, 0.016111766066384987, 0.015635556189393014, 0.01607714061567155, 0.014638136198899329, 0.015451307745763929, 0.014842841507637443, 0.0157557214700899, 0.01549844024099233, 0.015713760424963214, 0.01405266869553416, 0.014833755899015873, 0.015495181893129104, 0.015326881274301185, 0.01692041041130499, 0.015925047131020182, 0.015120663715412944, 0.015078388903145152, 0.01605356385230001, 0.015863287298672424, 0.015339146904878273, 0.0167111503244259, 0.015472583787832445, 0.016333546719178412, 0.01505341767973897, 0.015087606834233554, 0.01571243702615938, 0.015244360343942946, 0.014837547873231901, 0.014471312954238053, 0.014528916320857756, 0.015439736831174436, 0.015263595645563104, 0.01643624549167906, 0.015007310254812117, 0.016023870524101713, 0.01575824764825436, 0.016176773619054948, 0.015648716912167588, 0.015452165090046115, 0.015898122397827206, 0.017172520986345037, 0.014879260753934908, 0.015871465711622228, 0.01718344616657477, 0.014924989921677673, 0.014695048208593114, 0.016489036246105048, 0.015724420473046157, 0.015034502658078753, 0.01701943079630534, 0.01547097292903718, 0.016275858535148086, 0.015109576478199719, 0.017691273321226216, 0.01524748912239604, 0.01649358662378502, 0.016407084350426024, 0.015184742319569016, 0.01576648012701227, 0.015164503481695929, 0.015419079956500466, 0.01588109592684951, 0.01504582871271195 ], "mean_node_assignment_entropy": 4.156185967458301, "normalized_node_entropy": 0.9993514807107317, "effective_mass_slots": 63.93384248615631, "node_slot_mutual_information_nats": 0.0016628700997092238, "normalized_node_slot_mutual_information": 0.00039983574108217227, "centered_token_effective_rank": 1.2256707628542824, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.006830841001304992, "token_RMS": 0.30148401564558214, "mean_token_offdiagonal_cosine": 0.9995156893191369, "temperature": 0.4938478171825409 }, { "layer": 10, "head": 5, "slices": 64, "native_nodes": 33555, "mass": [ 0.0149183890905113, 0.016124931536975712, 0.016875072111639442, 0.015338134416243729, 0.014953344222505133, 0.015872821779517814, 0.015202840403634947, 0.0159822258767173, 0.014076759545775048, 0.015984746299194448, 0.01558259993421327, 0.015452064097015349, 0.015106517300337238, 0.015192751637395599, 0.015288249210173781, 0.015928732080299327, 0.014529014112781154, 0.016138726481014616, 0.01480531153248157, 0.014405101128566453, 0.01626484728075654, 0.014846686853478633, 0.014600793779009573, 0.01737320223200643, 0.01631606841916997, 0.015240690218138456, 0.016173290068456036, 0.015850080121122608, 0.014607868892226867, 0.015762916323996002, 0.015419532241131274, 0.016404442712187217, 0.015291791703458195, 0.015738582193536174, 0.014965700709143458, 0.01697080638589935, 0.01530153259316898, 0.01667291156302584, 0.01676965672325722, 0.01650993425594461, 0.01632255820118667, 0.017268307611397315, 0.015255751805453539, 0.015862189977861537, 0.01575102799108124, 0.014716226811117058, 0.015981123404447756, 0.01673556955503762, 0.014102044609790098, 0.014175054974323314, 0.015323323818294751, 0.0151795740998008, 0.015527722191505746, 0.016648248314436717, 0.015109790540008187, 0.014814923642165188, 0.016715818384559298, 0.01589226002330739, 0.01555879068697037, 0.016150443159951938, 0.015079603149888546, 0.015084598043600727, 0.014864152173978386, 0.01704122884505162 ], "mean_node_assignment_entropy": 4.156414359272284, "normalized_node_entropy": 0.9994063973336338, "effective_mass_slots": 63.91726495415051, "node_slot_mutual_information_nats": 0.0011751526931584877, "normalized_node_slot_mutual_information": 0.00028256449378451, "centered_token_effective_rank": 1.1644974145700635, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.006537122079032434, "token_RMS": 0.24898070569583763, "mean_token_offdiagonal_cosine": 0.9993067075502754, "temperature": 0.49803194403648376 }, { "layer": 10, "head": 6, "slices": 64, "native_nodes": 33555, "mass": [ 0.015162426926392128, 0.015220617246276318, 0.01605107652267957, 0.01655856343928512, 0.01544372165227852, 0.0145256873403389, 0.013998976249024082, 0.014461478400676266, 0.015428834055698748, 0.015509505746374053, 0.015175379329612066, 0.014524073652271418, 0.01439671401098768, 0.016784415074217924, 0.016464520394863156, 0.0161379024691157, 0.01614728335517629, 0.015641110408715343, 0.01567605061079369, 0.014376345758328156, 0.01447605573647296, 0.016251601346171287, 0.013735056503737911, 0.015510459778037446, 0.015058464192147094, 0.01478423676611041, 0.014979474364322346, 0.015430802880114548, 0.015764590632212073, 0.016715572602606144, 0.015174684883265325, 0.015079286426884187, 0.016053219183484537, 0.017342002996199268, 0.015918976668728622, 0.017028041792062885, 0.0155365484911405, 0.014307844205900323, 0.015675046094174916, 0.01698500743698406, 0.015202075259066422, 0.01606252153661208, 0.015159803563058643, 0.01581290927111232, 0.0152564463986803, 0.016160142338235053, 0.014953276598327847, 0.015987610099652612, 0.01590717313607183, 0.017845821029069064, 0.016266690993010154, 0.015534296126004002, 0.01577494112224346, 0.015531386009740404, 0.01682316497495061, 0.015224543613101497, 0.016457560032105422, 0.015147547329444455, 0.016107779352160647, 0.016535834894289232, 0.015183520518468684, 0.0141472064649747, 0.017696998736233677, 0.015729094888848983 ], "mean_node_assignment_entropy": 4.1551100326171655, "normalized_node_entropy": 0.9990927730674607, "effective_mass_slots": 63.90123155296063, "node_slot_mutual_information_nats": 0.002228601709873601, "normalized_node_slot_mutual_information": 0.0005358654391585515, "centered_token_effective_rank": 1.2113904748468578, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.009774284148382769, "token_RMS": 0.29584560524466014, "mean_token_offdiagonal_cosine": 0.9989020966933273, "temperature": 0.487006813287735 }, { "layer": 10, "head": 7, "slices": 64, "native_nodes": 33555, "mass": [ 0.0150879206945763, 0.015373095951340465, 0.015345698507590658, 0.01621534361571602, 0.01674532836463817, 0.015819065378166457, 0.015961784825579382, 0.01775825767618407, 0.015832309083961202, 0.01686809556615626, 0.01461490103704326, 0.014248421992631991, 0.016923028228668747, 0.013726879925483866, 0.014366344554423887, 0.01386257537326027, 0.013990281101617635, 0.015090984242896514, 0.01710931528401284, 0.015091533267282414, 0.016819981045686813, 0.015536509556028309, 0.015923673755678994, 0.012946341580967623, 0.01562514353336467, 0.015607131897849175, 0.019468322591674327, 0.01778042465315155, 0.01659149023492043, 0.014901071220454294, 0.01643033079933994, 0.016643608947087597, 0.015234080435728426, 0.014591240420168166, 0.015596605710421025, 0.015855792961734904, 0.01400603390500147, 0.015898186283252968, 0.014696972610906595, 0.018020792526927502, 0.015314807415046292, 0.01615363671468642, 0.015794551310883986, 0.016296060624428458, 0.018022758499478427, 0.015876412438874034, 0.014567418882797894, 0.01960965248251939, 0.014863874330881313, 0.01584692534079825, 0.014935788179640487, 0.014901640468098432, 0.014819719145473328, 0.014888262226311661, 0.01612569011041192, 0.016622123915123133, 0.015518484298168121, 0.013713121114679224, 0.014909131004251768, 0.01428977917108783, 0.01637037988824845, 0.014222382864941341, 0.013503474202337341, 0.014629025928339737 ], "mean_node_assignment_entropy": 4.152374469087843, "normalized_node_entropy": 0.998435009077829, "effective_mass_slots": 63.77844585439035, "node_slot_mutual_information_nats": 0.0030408249094788786, "normalized_node_slot_mutual_information": 0.0007311638361861349, "centered_token_effective_rank": 1.0868028584561547, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.010992314602568935, "token_RMS": 0.2815345700400199, "mean_token_offdiagonal_cosine": 0.9984800930211192, "temperature": 0.4876847565174103 }, { "layer": 10, "head": 8, "slices": 64, "native_nodes": 33555, "mass": [ 0.014941977211577431, 0.014806471198698417, 0.014670790177359878, 0.015014560762617968, 0.01610196685076266, 0.01615614437717001, 0.01502895958870405, 0.01584577361653839, 0.016636105680777443, 0.016559022500047454, 0.015530295714607201, 0.01393639782579365, 0.016169680143698035, 0.015428064183297749, 0.01541467091639548, 0.014552252795511166, 0.01597647501413579, 0.015593603326277227, 0.016418646917417658, 0.015486467546100472, 0.01586609213324182, 0.01727176172251634, 0.015135516409059028, 0.01427361629226092, 0.015581300065216694, 0.015357007669883618, 0.01627215110192958, 0.016451535999887062, 0.015003367909400291, 0.015815593541138102, 0.01569093478375341, 0.015694243559896095, 0.015173346833623596, 0.015296473940233505, 0.015445490930806176, 0.016082903979874337, 0.014888516975661945, 0.015514274336527542, 0.015705282494537905, 0.01612926366923685, 0.015321844327440473, 0.015309819468437088, 0.015469030811083759, 0.016830144356783563, 0.015616414897313703, 0.0167430542958416, 0.016122589480049153, 0.016170908877107874, 0.015902542616994926, 0.01507199754955969, 0.015913821108424972, 0.015412405650907854, 0.01594117074403282, 0.015612058539230518, 0.015617914517891511, 0.01741980541607934, 0.015746975199249617, 0.015333884020326675, 0.015655782987378762, 0.014907040313799936, 0.016164339956763765, 0.014830420002943, 0.015323286296389281, 0.014645747775651849 ], "mean_node_assignment_entropy": 4.157353909502172, "normalized_node_entropy": 0.9996323114098548, "effective_mass_slots": 63.94261873668969, "node_slot_mutual_information_nats": 0.0006321894480167956, "normalized_node_slot_mutual_information": 0.00015200943025936035, "centered_token_effective_rank": 1.3899143703610548, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.004844077672495832, "token_RMS": 0.27076407472280073, "mean_token_offdiagonal_cosine": 0.9996959637212303, "temperature": 0.49734100699424744 }, { "layer": 11, "head": 1, "slices": 64, "native_nodes": 33555, "mass": [ 0.01556040950601324, 0.01600523239767395, 0.015792262412007048, 0.015296510263840038, 0.016008521066963308, 0.01747834731158157, 0.015515834736346204, 0.015400878114245843, 0.016507478222925944, 0.01709502710765271, 0.01600066125306873, 0.015773410382641045, 0.016657856457249594, 0.014482008729609083, 0.01666823241633748, 0.01523093847384988, 0.013687928069275771, 0.01469787377650408, 0.0160477650669202, 0.016010274080332853, 0.01627025923054294, 0.015208519205552041, 0.016243138130836943, 0.01488491005531088, 0.015257740104489107, 0.01656466178629881, 0.013777850304371224, 0.013449821360337096, 0.015268929751464719, 0.01589374390152226, 0.01498328423630732, 0.015576935830678568, 0.016640978370562935, 0.015857206941297077, 0.016379508603768897, 0.015167046204623435, 0.015107824195955171, 0.015673629653983837, 0.014831166286002043, 0.015774821989641266, 0.015480502181486763, 0.01678078312983573, 0.015338288891892322, 0.015463625744535837, 0.015623634079402676, 0.015233431344943722, 0.015195508666903852, 0.0157292775478924, 0.015707345897279932, 0.015318664534159272, 0.014361880531981687, 0.014736352161776467, 0.015037821070239061, 0.015585888417682195, 0.01593148983607543, 0.014778891966305312, 0.015710135607453816, 0.015593286842188818, 0.01670401367566888, 0.01659018003677797, 0.01562755923618851, 0.015892847668256868, 0.015442624644489717, 0.017406540253672723 ], "mean_node_assignment_entropy": 4.155690975316965, "normalized_node_entropy": 0.9992324602594675, "effective_mass_slots": 63.916677617968475, "node_slot_mutual_information_nats": 0.0018893475994623543, "normalized_node_slot_mutual_information": 0.00045429206870996775, "centered_token_effective_rank": 1.1506458474131265, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.009130787087759617, "token_RMS": 0.27261319013551855, "mean_token_offdiagonal_cosine": 0.9988981422582239, "temperature": 0.4934663772583008 }, { "layer": 11, "head": 2, "slices": 64, "native_nodes": 33555, "mass": [ 0.016630345929651062, 0.015460472330401053, 0.015703523624730018, 0.015540075692388565, 0.014653041794998264, 0.014577319804951493, 0.014967663275088208, 0.01476818205853401, 0.01490821228962824, 0.015280678911036625, 0.01571224244194048, 0.015599479595752386, 0.013852612178002762, 0.016432253068618032, 0.015122788522940397, 0.016538733231618995, 0.015263816409341458, 0.015466963104329722, 0.015084277630610564, 0.015149792883505748, 0.014443267473009806, 0.014873448273158965, 0.013653506974604592, 0.017544845057446846, 0.015597671600933793, 0.014243939335356259, 0.016893929588971898, 0.016257726680751067, 0.01587806829649496, 0.01640945630923253, 0.015217709974578003, 0.013919311810260983, 0.015109992122788034, 0.01646344105339347, 0.01656915144213549, 0.014665683511810577, 0.016065119625591137, 0.016425310953870925, 0.016017179786376273, 0.015375753392337323, 0.01540151220669873, 0.014669254517859654, 0.01600216899281988, 0.014308187281350686, 0.017375509551253197, 0.01844028674350279, 0.01599716114807575, 0.014153222781689978, 0.01523726816386131, 0.015503312903300268, 0.01682104975525792, 0.01667109028108872, 0.016678720691179292, 0.01566762197085503, 0.01626156716980956, 0.01610988490030918, 0.01640101312109436, 0.01641557744676848, 0.015915594518994002, 0.016546131517072536, 0.016066556561928652, 0.014856313529458091, 0.015398862188222059, 0.014765143854039256 ], "mean_node_assignment_entropy": 4.155107071361991, "normalized_node_entropy": 0.9990920610361015, "effective_mass_slots": 63.88617231277309, "node_slot_mutual_information_nats": 0.001995870873312988, "normalized_node_slot_mutual_information": 0.0004799055018638955, "centered_token_effective_rank": 1.0652190267656556, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.009247823164585636, "token_RMS": 0.3075352774272922, "mean_token_offdiagonal_cosine": 0.9992250718078651, "temperature": 0.48618364334106445 }, { "layer": 11, "head": 3, "slices": 64, "native_nodes": 33555, "mass": [ 0.015566174452728893, 0.01305821725174134, 0.014789355026743722, 0.016112467918654363, 0.013898041225873204, 0.01446890425655312, 0.016697911250403208, 0.015312563452851336, 0.01641077666002616, 0.015780635220609966, 0.0160630131728482, 0.014500380040015545, 0.014830991456118867, 0.01664073206739665, 0.015476233306109257, 0.014452150048675009, 0.014924505682621329, 0.016005288660132153, 0.015741084955245226, 0.01665931379195231, 0.01572963669027391, 0.014348260816250503, 0.015514954784669946, 0.016814008875854224, 0.016474204970463487, 0.015607396892441658, 0.016322837732876624, 0.015167679433143143, 0.015652767224136017, 0.015526235340968843, 0.016965257776494512, 0.016151016895346884, 0.013435886611221373, 0.016166967590914152, 0.016596647990181113, 0.015938677042339682, 0.015274203737364754, 0.01600205055578839, 0.014407863173894156, 0.01650404970209772, 0.01509016119681977, 0.015772910055143346, 0.014412152734902208, 0.016047355459011883, 0.01680405658171982, 0.015733633084202413, 0.01657339840760473, 0.014557167667478674, 0.016027097062781874, 0.015048919388067659, 0.01717807799158611, 0.014873919889415239, 0.015361606354437044, 0.016176517403291405, 0.017296717105917115, 0.014691777466909441, 0.016757277387921935, 0.01514156949301845, 0.015236218727269128, 0.015981985418054657, 0.01608026557133066, 0.016297103124325828, 0.015780247301207428, 0.015090521655709213 ], "mean_node_assignment_entropy": 4.154897709665329, "normalized_node_entropy": 0.9990417201891805, "effective_mass_slots": 63.89473175225646, "node_slot_mutual_information_nats": 0.0023392031277635894, "normalized_node_slot_mutual_information": 0.0005624594586760805, "centered_token_effective_rank": 1.1860442609346575, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.009432869037901262, "token_RMS": 0.233786141359872, "mean_token_offdiagonal_cosine": 0.998364743349413, "temperature": 0.4922720789909363 }, { "layer": 11, "head": 4, "slices": 64, "native_nodes": 33555, "mass": [ 0.014253940620305437, 0.015611929907804184, 0.015602243911364479, 0.01586864982904971, 0.015910300510727185, 0.015416673880340783, 0.015335300630295735, 0.01702993070333974, 0.016193456166191965, 0.016327388498628484, 0.014648316442858772, 0.01556787774710071, 0.016367056080662107, 0.01596412745242928, 0.0145373174360025, 0.014964712650802821, 0.015201216842326948, 0.014982714616689255, 0.015088136514920518, 0.015296471116456798, 0.015464219081280369, 0.01584876330949354, 0.015359150431384106, 0.01488625923150268, 0.014282592379373838, 0.01720751318855775, 0.016146662523446823, 0.014867803328446439, 0.0146090303941109, 0.015337324253337703, 0.015070698511134652, 0.01698246179396963, 0.015021697266077013, 0.014905426497032028, 0.014933185447593051, 0.016680226534396308, 0.015730763865059436, 0.01494377612747888, 0.015477251401247415, 0.015744480658880452, 0.016643419223648127, 0.016256628322121266, 0.015974621971142956, 0.016279657611229563, 0.01507012767800785, 0.013974925555536368, 0.015603570449548326, 0.017425922658769053, 0.015470211908377784, 0.016315467545082382, 0.01465117298333065, 0.01644936564895102, 0.016505830459415494, 0.015189328522346064, 0.014989202725322871, 0.015588262056449748, 0.014845102332974267, 0.015577345139969702, 0.016633134890131785, 0.016533679511459693, 0.016575361317490753, 0.01601496174545175, 0.015035179094359393, 0.016730473053824385 ], "mean_node_assignment_entropy": 4.155592787747835, "normalized_node_entropy": 0.9992088511396242, "effective_mass_slots": 63.92222921832075, "node_slot_mutual_information_nats": 0.002074388232235158, "normalized_node_slot_mutual_information": 0.0004987849359206811, "centered_token_effective_rank": 1.0619723307558702, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.008791096739227604, "token_RMS": 0.23231575851032538, "mean_token_offdiagonal_cosine": 0.9985909546032128, "temperature": 0.48833394050598145 }, { "layer": 11, "head": 5, "slices": 64, "native_nodes": 33555, "mass": [ 0.01569852575672519, 0.015077989006332022, 0.016506160141556297, 0.016747391978874104, 0.015630308370618143, 0.015549033341673691, 0.015060260221989379, 0.017017064690911864, 0.015710658965610948, 0.01624503831473899, 0.014936407468158315, 0.016962058884814002, 0.015849704452546215, 0.015242728999076346, 0.015944113436464076, 0.016249225026149495, 0.014555271515033495, 0.015176784644085726, 0.01521570660152244, 0.015456572786332145, 0.015453211698559548, 0.01578065012515728, 0.015391173238255835, 0.015292865630017763, 0.014703586377978388, 0.016346182236915602, 0.014242140924098142, 0.01556211143461181, 0.015279591945711218, 0.014994997620713584, 0.015569730738663332, 0.01674860453943931, 0.015445024495373457, 0.01486768283925794, 0.01521862855275743, 0.016545778326190606, 0.016287883238917163, 0.015500490729394656, 0.015418097323589132, 0.015603753772382854, 0.015502742440676374, 0.015393149335453023, 0.016266323027807863, 0.015736289655659562, 0.015718912618794573, 0.014443937067185032, 0.01588619531090232, 0.015364483321723106, 0.015911541914897916, 0.01515549918548696, 0.015447031130458849, 0.01459913953443213, 0.014917752019435971, 0.01585072296031953, 0.014833116801373739, 0.016950690174052603, 0.01573657949634519, 0.015299121536938421, 0.01656033295425852, 0.015375071598877185, 0.01578737117961824, 0.016334797187306287, 0.015773197193536367, 0.016072812095459875 ], "mean_node_assignment_entropy": 4.156711735231795, "normalized_node_entropy": 0.9994779011373115, "effective_mass_slots": 63.948865040915265, "node_slot_mutual_information_nats": 0.0013720450345537927, "normalized_node_slot_mutual_information": 0.0003299070945378472, "centered_token_effective_rank": 1.228761460194562, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.006906401884373816, "token_RMS": 0.2691143650924631, "mean_token_offdiagonal_cosine": 0.9993810609807096, "temperature": 0.4944804012775421 }, { "layer": 11, "head": 6, "slices": 64, "native_nodes": 33555, "mass": [ 0.01614024301436042, 0.015243414817054579, 0.014817093492399125, 0.0171355597687614, 0.015474391385086904, 0.014254505870104139, 0.015404883301685738, 0.016127340090972463, 0.014431666537915038, 0.014781905055926473, 0.015825716505609003, 0.015268194232141007, 0.015563890826822463, 0.014707824982305278, 0.016525586144650342, 0.01668731368619645, 0.0161562309912519, 0.017778718181264204, 0.01524684833965817, 0.01555309346448916, 0.014519794203808403, 0.016626017808085973, 0.015012219442284164, 0.0159360497129126, 0.015077266686032131, 0.016537049832995167, 0.015281398694936173, 0.01837594578473663, 0.015559504987654486, 0.016116616881372427, 0.01593120202242844, 0.014324996265595368, 0.017341518591583716, 0.01680482222446474, 0.01574400268348536, 0.014946630458172504, 0.016264407008563266, 0.014642461632160384, 0.016605962906296474, 0.016067960993207092, 0.01764439191758597, 0.014857611307912346, 0.01677938663371428, 0.01601974857315219, 0.016296000770820385, 0.015911520590566875, 0.01587729486899797, 0.016460691812328718, 0.016972400407364998, 0.01539283473762936, 0.01784885862811394, 0.014625168581163053, 0.014568399819178386, 0.013786369761125402, 0.014120985139502737, 0.016530514416949454, 0.01413338600872011, 0.014115080405723327, 0.015525084171561305, 0.013461603261635868, 0.013188038952398843, 0.015527708949322734, 0.015697171222660015, 0.0138195002350788 ], "mean_node_assignment_entropy": 4.155504422378227, "normalized_node_entropy": 0.9991876037595375, "effective_mass_slots": 63.840873050422864, "node_slot_mutual_information_nats": 0.0008892062716778781, "normalized_node_slot_mutual_information": 0.00021380891307950655, "centered_token_effective_rank": 1.1471614354659412, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.005645271244578177, "token_RMS": 0.21878550268643754, "mean_token_offdiagonal_cosine": 0.9993796620700311, "temperature": 0.49550798535346985 }, { "layer": 11, "head": 7, "slices": 64, "native_nodes": 33555, "mass": [ 0.013623977221372034, 0.01704558869916793, 0.01623184913656956, 0.016394833967116298, 0.014737655214588984, 0.014884021641768991, 0.01619988579301361, 0.016370131374526763, 0.015304572063768396, 0.01461831780627643, 0.01647080729549412, 0.01568774400019971, 0.014823827799728812, 0.017232296294490388, 0.016442459487307534, 0.01600277516549594, 0.014313548059187326, 0.013779553329129942, 0.016316834354259416, 0.016695248761973834, 0.013649350177092002, 0.01504679804026965, 0.013295717603210463, 0.016345772769892204, 0.015673134259206804, 0.016891030439722024, 0.015503166776720555, 0.01664798538837676, 0.016466044720149602, 0.015681051719234874, 0.016029406082917034, 0.013477457055719044, 0.016681448085728443, 0.01565957834299933, 0.017135701850534035, 0.015144713493120944, 0.0147665785198981, 0.016651973169146548, 0.01536277172296813, 0.015130936847717978, 0.014706308942195458, 0.015185939726012995, 0.015258196839179292, 0.013482937745771044, 0.015843492288449414, 0.015362202594310129, 0.015737862922911797, 0.014606596037552301, 0.016000653287159085, 0.014599692971990869, 0.015651911763398575, 0.01586221220506617, 0.01527606091120549, 0.01557664110106983, 0.01576857651320038, 0.016967410233297533, 0.016344226822712225, 0.01683365552196416, 0.01473723659243292, 0.01712706742730231, 0.016759872169019727, 0.017282619413181044, 0.014911610410233587, 0.015700471168154977 ], "mean_node_assignment_entropy": 4.154867576050453, "normalized_node_entropy": 0.9990344745863895, "effective_mass_slots": 63.864269320657904, "node_slot_mutual_information_nats": 0.0018924633807628766, "normalized_node_slot_mutual_information": 0.00045504125574842737, "centered_token_effective_rank": 1.0742897431107634, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.009814377106409857, "token_RMS": 0.29577403076390324, "mean_token_offdiagonal_cosine": 0.9988843126351727, "temperature": 0.489545613527298 }, { "layer": 11, "head": 8, "slices": 64, "native_nodes": 33555, "mass": [ 0.016464060120425554, 0.014687167553334091, 0.015337463878644701, 0.015568551090352361, 0.015966459935481892, 0.015258171673819163, 0.015454899059668807, 0.016098937245854376, 0.0161854341173204, 0.015289113204690334, 0.015801965750725015, 0.01511256815586697, 0.016782326083183467, 0.014763053225425801, 0.015544932444609458, 0.016052120656631037, 0.015290060435714443, 0.016070696358772826, 0.015101255711588263, 0.015538251320447763, 0.015905418039737107, 0.016212237908611343, 0.016559250187225927, 0.015096399954332403, 0.015274452036150903, 0.016495078837080197, 0.014826493617882717, 0.015757915880012054, 0.01635167226878111, 0.01557418253328837, 0.015431313485827509, 0.016005468048846967, 0.015530233592907393, 0.015580662290521678, 0.014536625878823494, 0.016378372851462215, 0.014888788023365448, 0.015511844237407884, 0.01632447063275078, 0.014640950545282251, 0.015306918800511094, 0.015625325833479056, 0.015171614368996417, 0.01644243999384205, 0.015410429609387407, 0.015459931592135453, 0.015047251787261474, 0.01594982575825018, 0.01542469352878427, 0.015396897066975737, 0.016206613405339944, 0.015991735239636033, 0.016078764150279135, 0.015205423820904267, 0.015987402813420568, 0.014572524456516843, 0.016889483267681474, 0.015126860757584437, 0.01626826928923776, 0.015382047163328397, 0.015500319067951091, 0.015025239826029192, 0.015946276749260396, 0.015334392639745491 ], "mean_node_assignment_entropy": 4.156726615793555, "normalized_node_entropy": 0.9994814791560875, "effective_mass_slots": 63.95927850097504, "node_slot_mutual_information_nats": 0.0015199916360133514, "normalized_node_slot_mutual_information": 0.0003654807325781941, "centered_token_effective_rank": 1.2427628674728646, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.006723131047419217, "token_RMS": 0.30576790608888127, "mean_token_offdiagonal_cosine": 0.9995608973839216, "temperature": 0.48922640085220337 }, { "layer": 12, "head": 1, "slices": 64, "native_nodes": 33555, "mass": [ 0.015195034726074202, 0.01438845742456121, 0.016603586370534246, 0.016389369634813442, 0.016411085384786174, 0.014067638610593609, 0.014799242190136282, 0.014474677943962197, 0.016077929514114297, 0.01563723237321587, 0.014780311900309108, 0.016382904094466952, 0.016952480479015746, 0.014552114329368404, 0.013811065729376083, 0.01744281887123285, 0.015475399152328158, 0.015965724423263487, 0.015277482362858471, 0.016515164236846056, 0.016636469254011795, 0.013844303070728389, 0.015122846796228257, 0.014555839599100333, 0.015342332431836981, 0.015817689036359564, 0.016414687384545556, 0.0172069150258942, 0.01715927970332123, 0.014187796574461755, 0.015861862337747137, 0.01612408846806035, 0.014528107507293954, 0.016467997544440796, 0.014303302700778283, 0.01505929725914927, 0.014716660539516792, 0.014847606090758556, 0.016414908147768806, 0.01596913456740842, 0.016082951604027697, 0.014946368728995971, 0.01753130528635665, 0.016513329828464714, 0.015397748762955122, 0.015535906411406536, 0.015393665706734111, 0.015644793528294648, 0.018142890471494904, 0.01704103878827333, 0.015036920439343689, 0.01689505343439371, 0.015233830346238363, 0.015370451648667013, 0.01424866359855193, 0.015610966383506042, 0.01723112847315734, 0.014047127150303337, 0.014623379361230256, 0.016358530626584673, 0.01456127674325095, 0.01549881647321941, 0.015680106395708167, 0.015592906046023163 ], "mean_node_assignment_entropy": 4.156172667914227, "normalized_node_entropy": 0.9993482828463514, "effective_mass_slots": 63.86653280547722, "node_slot_mutual_information_nats": 0.0006228130045844438, "normalized_node_slot_mutual_information": 0.0001497548721858554, "centered_token_effective_rank": 1.1628285136660708, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.004396210152627011, "token_RMS": 0.24527963304371622, "mean_token_offdiagonal_cosine": 0.9996766138303756, "temperature": 0.4967619478702545 }, { "layer": 12, "head": 2, "slices": 64, "native_nodes": 33555, "mass": [ 0.015276862248737266, 0.014648277438247795, 0.015978745441323706, 0.016028255377574865, 0.015870256352386432, 0.015585627775728667, 0.01653258326754691, 0.01546848724293533, 0.015510555569594858, 0.016391325397010336, 0.01596773020440754, 0.01511351529144127, 0.01568591499762481, 0.015743053128637264, 0.014286858926819601, 0.015218898625951414, 0.0159454288695803, 0.015469017471674612, 0.015342972437182094, 0.015491227907864282, 0.015468549707632735, 0.014779596234989706, 0.015240575358296024, 0.01467831820605378, 0.01468130171653136, 0.01638332961615374, 0.01601207405689567, 0.017542251839738652, 0.01580467687581401, 0.016565679845105534, 0.015976988537687514, 0.014320583047153492, 0.014981727168100537, 0.01661005637359644, 0.015352810469152515, 0.01385261179589822, 0.016465005277504892, 0.016136222429026206, 0.015392761375139464, 0.014600070599578139, 0.016641697344179925, 0.014297782617682601, 0.015827523690949936, 0.014951383148813644, 0.014426770193729621, 0.017275188229794902, 0.015448648574180716, 0.017133423570344246, 0.017146638240884464, 0.015137084297482782, 0.015350454972787411, 0.0157050083722348, 0.014867871377720925, 0.01516199435518602, 0.0169982146913045, 0.015328985882550812, 0.01463664808461695, 0.015496797206507828, 0.01622884434941646, 0.016615970813031036, 0.014977560547584172, 0.016567763995705056, 0.017037032521992474, 0.014339928226935411 ], "mean_node_assignment_entropy": 4.156470690695324, "normalized_node_entropy": 0.9994199421777448, "effective_mass_slots": 63.9094091277892, "node_slot_mutual_information_nats": 0.00099590754451917, "normalized_node_slot_mutual_information": 0.00023946514594361856, "centered_token_effective_rank": 1.2153574718847822, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.004850933631932097, "token_RMS": 0.28576128054232874, "mean_token_offdiagonal_cosine": 0.9997159530225509, "temperature": 0.4919143319129944 }, { "layer": 12, "head": 3, "slices": 64, "native_nodes": 33555, "mass": [ 0.015175429681842533, 0.015872084790200434, 0.016178506755938547, 0.01563433110458612, 0.014358070131539499, 0.01539758480152729, 0.017047994563028808, 0.014992771415541431, 0.015831152724570922, 0.01568381977087849, 0.01497141153492936, 0.016143605069120902, 0.014758807639052253, 0.01619048090136774, 0.01569899255386245, 0.01534918330530974, 0.016120345864348937, 0.015016391821762938, 0.01566775392384589, 0.014931412330423244, 0.015042901683513826, 0.015162010379540833, 0.014900069462915074, 0.015910242139658085, 0.014617684595186928, 0.015559661314096798, 0.016409186739405147, 0.016245360844611072, 0.01511318360096877, 0.015022079984394764, 0.015549186155207777, 0.014109318471324815, 0.01635325628092296, 0.01481504955281401, 0.015061562939714505, 0.015368835035516686, 0.016751008908271418, 0.01568106528210038, 0.015814978494988197, 0.013976655512684025, 0.016574818102054207, 0.017580488671048656, 0.015107590838602875, 0.017095925981871925, 0.014889584439139824, 0.014549348515341417, 0.016617626426767847, 0.015165976407524016, 0.0153433743095759, 0.015736383380349755, 0.01553112954934336, 0.016221789922468078, 0.01582148711372907, 0.016091160490945694, 0.016219146889535206, 0.015511605599063854, 0.01776515254374766, 0.01496243560996177, 0.014666488808275131, 0.016199426735831683, 0.016633064939604796, 0.01586410771362065, 0.014275532838852523, 0.01709292599318804 ], "mean_node_assignment_entropy": 4.156104821504052, "normalized_node_entropy": 0.9993319692331012, "effective_mass_slots": 63.9149329483672, "node_slot_mutual_information_nats": 0.0014482050403170632, "normalized_node_slot_mutual_information": 0.0003482197049759714, "centered_token_effective_rank": 1.189519735351105, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.006206698636013407, "token_RMS": 0.3360036902583164, "mean_token_offdiagonal_cosine": 0.9996683145666269, "temperature": 0.4908687174320221 }, { "layer": 12, "head": 4, "slices": 64, "native_nodes": 33555, "mass": [ 0.01588923502610748, 0.016046442428772796, 0.014739953068623048, 0.01518651860440129, 0.017118521034273478, 0.014388107882365485, 0.015740396981028335, 0.016542680188069807, 0.016637098563510568, 0.014749515962842066, 0.015057363168749586, 0.01592006578812779, 0.015360501251760618, 0.015305065703229273, 0.015990181546394602, 0.016327605792768814, 0.015948882614692746, 0.01579719728438705, 0.015927660507567016, 0.014955428615906319, 0.015257861690281836, 0.016057562769528445, 0.014795708919845765, 0.014893085834745225, 0.014981768315017107, 0.016324675500513926, 0.015285917620169452, 0.014335793065693495, 0.015867807615352423, 0.016637803290903596, 0.01700223785555988, 0.015902062926611848, 0.01601138604576449, 0.01583493304180905, 0.01744658666558929, 0.015679916111698822, 0.01697135629678333, 0.015629860186942988, 0.015351986809993239, 0.01577886390521836, 0.014792362970485006, 0.016116340798615184, 0.016619250765413096, 0.01434742616060158, 0.015892451333117688, 0.015531274520589038, 0.015336476994483815, 0.014977455283764258, 0.014488950402716947, 0.014372663796192273, 0.015792002263789098, 0.016185187665664295, 0.015879370398917146, 0.01583898711570452, 0.014834803957150726, 0.01585509109805818, 0.015094415999775786, 0.01589499400994396, 0.015314687353702748, 0.01526650247423442, 0.014753843213618468, 0.01484196507417064, 0.015963411294789814, 0.016334488396156057 ], "mean_node_assignment_entropy": 4.156470088043904, "normalized_node_entropy": 0.9994197972707091, "effective_mass_slots": 63.932756253899996, "node_slot_mutual_information_nats": 0.0013617594280779244, "normalized_node_slot_mutual_information": 0.0003274339289619688, "centered_token_effective_rank": 1.0856046045485772, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.00556284859596219, "token_RMS": 0.2844215685391605, "mean_token_offdiagonal_cosine": 0.9996462124195248, "temperature": 0.49069666862487793 }, { "layer": 12, "head": 5, "slices": 64, "native_nodes": 33555, "mass": [ 0.014932986261466588, 0.014244888947130075, 0.016365892619850463, 0.016018593617198835, 0.01582466082955802, 0.015779933448219784, 0.015404872661349609, 0.015617151439662857, 0.015699224602061915, 0.017128072688757105, 0.015260215687482656, 0.015190454006346056, 0.015697014689997266, 0.014916419140098613, 0.016120439908528156, 0.015734083488979908, 0.015795585561047997, 0.01635026621221919, 0.016155433498079828, 0.0154570622994692, 0.015316267323230173, 0.015874625782846888, 0.014888045331872605, 0.015379725813292796, 0.017205552604577345, 0.01629570618306799, 0.016162360929099243, 0.016707107639366257, 0.017540638459974882, 0.01484788184714351, 0.014123870233372754, 0.01593214179351848, 0.014629206534504509, 0.014619802255386071, 0.015660392093613595, 0.015951289322614144, 0.01475106438300197, 0.015675990839201955, 0.016740576713009415, 0.014838172217425514, 0.015649169654896416, 0.01522406049912612, 0.014528455316516383, 0.018118336273465794, 0.01419464504498072, 0.013483157351217813, 0.01629234792448435, 0.01552090445590848, 0.014687828648951438, 0.015665278657205038, 0.014030037920479625, 0.01593000280324299, 0.015973796854618586, 0.014840716366160557, 0.015598569090256498, 0.01681863774330805, 0.017532166546773243, 0.015145935309375563, 0.015404859847178085, 0.015741227650766517, 0.015498453433687987, 0.01529372013953222, 0.015597462162737377, 0.016396560688851276 ], "mean_node_assignment_entropy": 4.156475630348593, "normalized_node_entropy": 0.9994211299132907, "effective_mass_slots": 63.90023181159382, "node_slot_mutual_information_nats": 0.0008473587527886295, "normalized_node_slot_mutual_information": 0.0002037467117503355, "centered_token_effective_rank": 1.367665432761571, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.0043902602719423384, "token_RMS": 0.20753118845545065, "mean_token_offdiagonal_cosine": 0.9995475927425096, "temperature": 0.49759119749069214 }, { "layer": 12, "head": 6, "slices": 64, "native_nodes": 33555, "mass": [ 0.016452937468807553, 0.018168287073121733, 0.016066920270524657, 0.013893827834922774, 0.014474466859915762, 0.015500166409401519, 0.014702243610218471, 0.014964163893670834, 0.015567628496647494, 0.014123386739096917, 0.016538184684759687, 0.015580670480886865, 0.0154787929124121, 0.015559573179118378, 0.01571993095914909, 0.014000599058224225, 0.016213160501705598, 0.014954701947463801, 0.015113722266288782, 0.015925732721184506, 0.01654311219096237, 0.014197208245986088, 0.016484949029643477, 0.015531816443525817, 0.01581414635536737, 0.014732419046880173, 0.015216899422517638, 0.01594329839279494, 0.013988773749411026, 0.01572183183037858, 0.015419305880286045, 0.016545229257729596, 0.014995700150068764, 0.01526983950848867, 0.01623266797905227, 0.015997668161695164, 0.015259757562872284, 0.014861595229473452, 0.014351375867777664, 0.01596214235644394, 0.015449692732168103, 0.016467766490429216, 0.014908313713779065, 0.015510316539765062, 0.01591800948888876, 0.016402415546566813, 0.016769297286976896, 0.01609800529429355, 0.015305195606533328, 0.01643808241780931, 0.01599763000444757, 0.017770411760739582, 0.015327402428897203, 0.01571420416818226, 0.014962392945257239, 0.016518075976922116, 0.015955126866162038, 0.01546174839967197, 0.015489530317610786, 0.015969527348783853, 0.015448603714269357, 0.016132223798563522, 0.015407438868871302, 0.016509754152535822 ], "mean_node_assignment_entropy": 4.156712471199779, "normalized_node_entropy": 0.9994780781002051, "effective_mass_slots": 63.91137738982835, "node_slot_mutual_information_nats": 0.0007849242537707468, "normalized_node_slot_mutual_information": 0.00018873438806475446, "centered_token_effective_rank": 1.2110091037469013, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.004459814347759027, "token_RMS": 0.26408541296780863, "mean_token_offdiagonal_cosine": 0.9997106651249796, "temperature": 0.4979866147041321 }, { "layer": 12, "head": 7, "slices": 64, "native_nodes": 33555, "mass": [ 0.016314460928567812, 0.014948582839242412, 0.015015407295537065, 0.015826545471488995, 0.01658305539640624, 0.014894455891908005, 0.016260075486609148, 0.016422065024807742, 0.016161554858628052, 0.015813022233449193, 0.015137953641268628, 0.01586752212292044, 0.01739686005688441, 0.014778394969781124, 0.016677668573610423, 0.016420582046996257, 0.015391456078049093, 0.014728095298825877, 0.014995296659940864, 0.01697735880366817, 0.014996752484571541, 0.01651649663077437, 0.01634118025196818, 0.016576637051119177, 0.01519056269006684, 0.01599705653888776, 0.016228708849517364, 0.01624890949161877, 0.015992081037223906, 0.015707200555864408, 0.016631023057033834, 0.014925945200239195, 0.015123456149223113, 0.015695881494740726, 0.01543313714640495, 0.014902439554496843, 0.015216630777629152, 0.01673426471496282, 0.014668786203323009, 0.014771698110334042, 0.014067709973910991, 0.01517189572658376, 0.017874908877580784, 0.01443756605281864, 0.015191028020319, 0.015986269188764782, 0.014061477543896356, 0.015460018472416561, 0.01479422680766052, 0.014865254230171135, 0.01480723039870614, 0.015049201848975966, 0.016427628042221053, 0.015019514748523815, 0.016447846431714236, 0.015143781323876827, 0.014661307588376181, 0.01456742915487928, 0.016229514044564762, 0.01570954769799818, 0.01592351101880643, 0.015893791937422616, 0.01650847872190989, 0.015191600115525974 ], "mean_node_assignment_entropy": 4.156781666701997, "normalized_node_entropy": 0.999494716101522, "effective_mass_slots": 63.91452983910974, "node_slot_mutual_information_nats": 0.000765052857289561, "normalized_node_slot_mutual_information": 0.00018395632720493026, "centered_token_effective_rank": 1.3133881213827991, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.0054598718042084505, "token_RMS": 0.31968715306625745, "mean_token_offdiagonal_cosine": 0.9997236965472078, "temperature": 0.4981691539287567 }, { "layer": 12, "head": 8, "slices": 64, "native_nodes": 33555, "mass": [ 0.015188726326421563, 0.014602800802194607, 0.015989119811154234, 0.0163574195429605, 0.014888064332296393, 0.018148159321756695, 0.013983770684762233, 0.0153479679944911, 0.014487511689414293, 0.017085570989018705, 0.016965710559079677, 0.015214214713794652, 0.016190638038980777, 0.017536529335214848, 0.015112083446722399, 0.01558166612697383, 0.015488981785128624, 0.015396121224618016, 0.016337878281516213, 0.016932560304450953, 0.015258875441801448, 0.015442025352498669, 0.016600053360447845, 0.017197264444331107, 0.013882364049823652, 0.01757048361787203, 0.014338808013269193, 0.014873814343353654, 0.015841786132887007, 0.016048220520157152, 0.01670857044157502, 0.014921652259075584, 0.014252481031249762, 0.015986722273297744, 0.015078215903433652, 0.01569100320333652, 0.014837647482004522, 0.015360843636081091, 0.01543095328755851, 0.017469395612186227, 0.016672459636047327, 0.013825727882971975, 0.017079967621288113, 0.015222787958528331, 0.016482836702449134, 0.013688750654423473, 0.01444876407961579, 0.015097989006246358, 0.015165867775427729, 0.015337378475379302, 0.016377623836745663, 0.01512973810375545, 0.015582045762448716, 0.014787982557412455, 0.01627731661027376, 0.016608487673585637, 0.014395415561720274, 0.013251476890915838, 0.01680395250673435, 0.015874313216724723, 0.01595375330525021, 0.01592588180582013, 0.015773626011502784, 0.014609180553174422 ], "mean_node_assignment_entropy": 4.155386777379533, "normalized_node_entropy": 0.9991593161168374, "effective_mass_slots": 63.856313895496854, "node_slot_mutual_information_nats": 0.0012486865921754742, "normalized_node_slot_mutual_information": 0.00030024565902601603, "centered_token_effective_rank": 1.2107499513551299, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.006146770750068061, "token_RMS": 0.23418282316386627, "mean_token_offdiagonal_cosine": 0.999303193717698, "temperature": 0.4953628182411194 }, { "layer": 13, "head": 1, "slices": 64, "native_nodes": 33555, "mass": [ 0.014847429315936342, 0.015633534093381456, 0.015828039246758262, 0.0162449772096532, 0.014986839049417071, 0.015940725298287828, 0.014924205232019165, 0.016824976486646416, 0.015167512802978624, 0.016662174115167558, 0.01708627458833321, 0.014000489134127764, 0.01655068038692863, 0.015135871404911, 0.017492903003724733, 0.015193826398523442, 0.01503094524290008, 0.01754152597633298, 0.016984422072310806, 0.016168976855018672, 0.015588775541616159, 0.013626638211997126, 0.015533305837179798, 0.015173462580852271, 0.016820416286490017, 0.016948996647368635, 0.017353688547480144, 0.014620139554721369, 0.015934951842328578, 0.014560531114435261, 0.015500404510045883, 0.015425700546609912, 0.01628695510723604, 0.014804819722720531, 0.017250961606905493, 0.015200576372999732, 0.01413119606340101, 0.015360238797537458, 0.016939562113408566, 0.013406970917056142, 0.014996845233642012, 0.01543377170930688, 0.015341250572063283, 0.015893499311678654, 0.015574427211920563, 0.015559163334653294, 0.015857340479132235, 0.01472537655315626, 0.016910260553330055, 0.01688970996847124, 0.014720102018648804, 0.014835760809565519, 0.015363637970560905, 0.017186432166019214, 0.016005890144748805, 0.015644472306465193, 0.01520164500005491, 0.016196915666991715, 0.014812411528761811, 0.015160728284164612, 0.014294920973407585, 0.014729343967423711, 0.014661003811962958, 0.015290470668751008 ], "mean_node_assignment_entropy": 4.153900707130873, "normalized_node_entropy": 0.9988019917538116, "effective_mass_slots": 63.87783514347648, "node_slot_mutual_information_nats": 0.003071726215825521, "normalized_node_slot_mutual_information": 0.0007385940297566835, "centered_token_effective_rank": 1.0671720839062493, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.00891250886736794, "token_RMS": 0.29336856919883736, "mean_token_offdiagonal_cosine": 0.9991074975979719, "temperature": 0.4748539328575134 }, { "layer": 13, "head": 2, "slices": 64, "native_nodes": 33555, "mass": [ 0.016068376491997837, 0.01579354369415867, 0.014901731226600472, 0.015066395551682344, 0.016657221270607172, 0.016470426815625647, 0.014855232632953094, 0.014937286185287467, 0.014442889414330235, 0.017423604989088234, 0.016610481543931348, 0.01630745698013563, 0.01692062739048038, 0.014589964618456469, 0.015974941859644886, 0.017769278637535295, 0.016416161559658707, 0.01671729794605032, 0.015881300454806898, 0.014897584534694143, 0.01635228370560155, 0.01482423045329007, 0.016738438710510384, 0.015321380870935503, 0.015009886257554722, 0.016313470261274988, 0.015304254991993376, 0.015577018981672642, 0.014994915409618413, 0.016994267622889588, 0.016082029087426515, 0.01708757407394851, 0.016562636870063197, 0.013890939699287449, 0.015496904394589226, 0.014340417646287965, 0.014559788281807706, 0.015839018649751225, 0.015217804212820919, 0.016286989082233274, 0.015881056296610695, 0.01607757866948248, 0.015010967696727547, 0.014064499961009388, 0.014545251462692397, 0.015368896835062982, 0.014957473327083375, 0.016074006824840635, 0.015757275033909276, 0.0149295399724025, 0.017487764625191565, 0.014592486049776202, 0.015197018224170349, 0.014012614881691697, 0.014283541921288055, 0.014448607856505737, 0.015742101838539954, 0.018861344295471014, 0.014116672681642616, 0.015456482333136269, 0.01600491068085994, 0.017058924559491026, 0.014672851682846138, 0.013902079611195127 ], "mean_node_assignment_entropy": 4.155666270355344, "normalized_node_entropy": 0.9992265199718651, "effective_mass_slots": 63.857866250034284, "node_slot_mutual_information_nats": 0.0009935034390595732, "normalized_node_slot_mutual_information": 0.00023888708077289612, "centered_token_effective_rank": 1.132742668205183, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.004837534160819907, "token_RMS": 0.27711694686049454, "mean_token_offdiagonal_cosine": 0.9997769359811315, "temperature": 0.4969019591808319 }, { "layer": 13, "head": 3, "slices": 64, "native_nodes": 33555, "mass": [ 0.015065310356522074, 0.016768612971687847, 0.016057430913902496, 0.017368563984686846, 0.014430603507815717, 0.015191501635997586, 0.016705942001499625, 0.016029657990338772, 0.017247893638112848, 0.015287806305658522, 0.013547207520240252, 0.014272518878165122, 0.014941512246110625, 0.016967455590580386, 0.01520222998756403, 0.013926408179670533, 0.013596120980311356, 0.015576684677113258, 0.01718438722747233, 0.01653335214786941, 0.01491282707414139, 0.01490747046334531, 0.013874609328194678, 0.015530645492832345, 0.01581868908035406, 0.015987907798863384, 0.0156386962412762, 0.016671088028928416, 0.015027567267616634, 0.014973038044578982, 0.015124980688556939, 0.014839269673625807, 0.014440655994246088, 0.01625798217771288, 0.015671669645838614, 0.01691107020545887, 0.016307748403861472, 0.01544269632027408, 0.016794843309881027, 0.014236877571012602, 0.01567587262446111, 0.014859484930733262, 0.015078079854843869, 0.01633757051464041, 0.01587634043552391, 0.015088978680185147, 0.01725699806219818, 0.01609609550615347, 0.01617519737409619, 0.017239402742528238, 0.014265171856947067, 0.01439178345927275, 0.015822480802470465, 0.017490247616994276, 0.01742204473848102, 0.015573302882005624, 0.013765341839988315, 0.015096678295078523, 0.0146538478015772, 0.017940248517177175, 0.014832718495538784, 0.013669504388537345, 0.01606602155570301, 0.018055053574140355 ], "mean_node_assignment_entropy": 4.155394197691942, "normalized_node_entropy": 0.9991611003248231, "effective_mass_slots": 63.83274834148435, "node_slot_mutual_information_nats": 0.0008721578554862219, "normalized_node_slot_mutual_information": 0.0002097096354970543, "centered_token_effective_rank": 1.1855812675797308, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.005774659410181431, "token_RMS": 0.24040046253591985, "mean_token_offdiagonal_cosine": 0.9994169245681547, "temperature": 0.4855283200740814 }, { "layer": 13, "head": 4, "slices": 64, "native_nodes": 33555, "mass": [ 0.015671835701361426, 0.015379058982443657, 0.01588274550296265, 0.014410627944329166, 0.01723110887116939, 0.01615793732078331, 0.012847402468079463, 0.014132372398113168, 0.015155286911939956, 0.01569087674138803, 0.01754112347911623, 0.01536200007107628, 0.015008581919875093, 0.016277727126967505, 0.0163430342967538, 0.015113898455006429, 0.015827472300012392, 0.014422533954554069, 0.015573422958503455, 0.01416096231286629, 0.016715178314794587, 0.01719092734140276, 0.016226272092142113, 0.015412721934604398, 0.015106003100560117, 0.01642139021605875, 0.01406913858299514, 0.015234159441830514, 0.015255319906226295, 0.018110727260174844, 0.01395817838490982, 0.0148188454206273, 0.015449859688899198, 0.01621355439891955, 0.015539025491281996, 0.014705966476163953, 0.015771498136758592, 0.014622940436242066, 0.015671405792785878, 0.015484500661849834, 0.015230526493407365, 0.015851840540969882, 0.014886869982022622, 0.014171891280557755, 0.016320347151356968, 0.01553649450002138, 0.014708671783866043, 0.01650983516366675, 0.015449775227336784, 0.016278273537322033, 0.016933887269203026, 0.01581863798728431, 0.014661648735457304, 0.01671466273295197, 0.013813804245978981, 0.019119605038277865, 0.016114587931709334, 0.014716773254222609, 0.01743385469486276, 0.014837138408151358, 0.016063821657208693, 0.018484833202850905, 0.014484319501396725, 0.015690276919746296 ], "mean_node_assignment_entropy": 4.155435270971125, "normalized_node_entropy": 0.9991709763608548, "effective_mass_slots": 63.832930858741285, "node_slot_mutual_information_nats": 0.0008339438765956331, "normalized_node_slot_mutual_information": 0.00020052111585737293, "centered_token_effective_rank": 1.124668651784989, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.005961524776816091, "token_RMS": 0.31407152002950867, "mean_token_offdiagonal_cosine": 0.9996543513546046, "temperature": 0.49642714858055115 }, { "layer": 13, "head": 5, "slices": 64, "native_nodes": 33555, "mass": [ 0.016819468650185854, 0.015268378239305422, 0.014403865034696846, 0.016733531601534943, 0.014694927903393211, 0.016544957517203144, 0.01571312994655816, 0.01766359225973529, 0.013922768976800826, 0.015507341434615964, 0.014891084350618897, 0.01552299003139962, 0.01735180630256337, 0.014710148069560705, 0.014823092979417439, 0.014937649428069529, 0.01638435422619237, 0.017697556247076234, 0.01612107633290952, 0.01666549418919258, 0.016118958508594437, 0.013644698261548538, 0.01635258394823471, 0.0156824322950554, 0.01570289342873317, 0.015496735905887312, 0.016180360584622914, 0.013817297087210982, 0.014760480525013437, 0.015026231128664143, 0.015838156017510773, 0.01580636388934608, 0.015479006936307094, 0.013640756475069035, 0.016296161112871403, 0.015344490255589297, 0.013837364492786657, 0.016180148106548536, 0.01612823195736053, 0.013828299659305847, 0.016774729371788054, 0.016592759938169263, 0.016048956307733963, 0.015346044948597393, 0.01636951715359166, 0.016037323374971203, 0.015297579321487927, 0.01644902885964354, 0.016371722426612417, 0.01680045854247526, 0.015972031982602546, 0.016260192661838015, 0.016170695380226828, 0.015893798347825113, 0.01738682575174139, 0.01416915608857245, 0.016432607172115594, 0.015124244601336007, 0.016404300084499404, 0.01521294114636897, 0.015471727047584625, 0.014188829314942296, 0.01417645410895614, 0.013509211972918716 ], "mean_node_assignment_entropy": 4.155299791206022, "normalized_node_entropy": 0.9991384003633123, "effective_mass_slots": 63.85949345824453, "node_slot_mutual_information_nats": 0.0013854639828307924, "normalized_node_slot_mutual_information": 0.0003331336695600427, "centered_token_effective_rank": 1.1339195517445515, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.006153012582691129, "token_RMS": 0.28371227161506224, "mean_token_offdiagonal_cosine": 0.9995669360541657, "temperature": 0.48862650990486145 }, { "layer": 13, "head": 6, "slices": 64, "native_nodes": 33555, "mass": [ 0.016360368978106138, 0.01518261733857215, 0.014570216093900138, 0.015143148364886951, 0.016248217679342584, 0.016259529442455487, 0.015676256095691617, 0.01619406767989405, 0.01588546279315452, 0.01672629590332619, 0.014089101334697105, 0.014729443283740783, 0.01501513518622852, 0.015336192059651907, 0.01496451634338042, 0.016769477867107377, 0.01670973489199496, 0.017486875811865133, 0.01566225163210608, 0.015389557008070457, 0.01501638513541918, 0.015402481658321756, 0.015328592856292077, 0.01523513294264592, 0.015794947550406828, 0.015354636477090273, 0.013942396236274676, 0.016208552205337, 0.015848570557305902, 0.016994447265424845, 0.015996721060065357, 0.014156558011862351, 0.01616739857655433, 0.015186407975352898, 0.015443629252922381, 0.015087355437311477, 0.015679282036898185, 0.015296277728982547, 0.016041188101370108, 0.016841459154695242, 0.015075714508358252, 0.017520784856114583, 0.01677016330472964, 0.015269985106305861, 0.015355046556419063, 0.015786779235458386, 0.01613758981489883, 0.015320617796564809, 0.016043009799050958, 0.015383404718703367, 0.015279242627887768, 0.015195540015907264, 0.015436830316607652, 0.014985338482195867, 0.016872647050293524, 0.015723395731143022, 0.014897132799087082, 0.01543099032516514, 0.015310282672204065, 0.016111303451498093, 0.016268630694826106, 0.015588154827790385, 0.014122503481843072, 0.014694025873002139 ], "mean_node_assignment_entropy": 4.153571154811412, "normalized_node_entropy": 0.9987227511709783, "effective_mass_slots": 63.92235275485485, "node_slot_mutual_information_nats": 0.004097953773579022, "normalized_node_slot_mutual_information": 0.000985349597822445, "centered_token_effective_rank": 1.1231342096988255, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.009609243050414478, "token_RMS": 0.3291751387018542, "mean_token_offdiagonal_cosine": 0.9991442098285193, "temperature": 0.4758959412574768 }, { "layer": 13, "head": 7, "slices": 64, "native_nodes": 33555, "mass": [ 0.015531086666233901, 0.014878106212670588, 0.015186304371149534, 0.015874583304073812, 0.015992781321662868, 0.01549248622638505, 0.016709577282711656, 0.015932514478920592, 0.01544336401935844, 0.015431997883632488, 0.016485317783090772, 0.017454035329196566, 0.014830246943280471, 0.015081007735624147, 0.016366884170255246, 0.01575986753650717, 0.016321645492185177, 0.014960236917010471, 0.014369408516517216, 0.015506449910975696, 0.015083544243156529, 0.014874742466930288, 0.017037645382233516, 0.015820448662968573, 0.01493363296114893, 0.013902761194493603, 0.015919807326504352, 0.015159455391049225, 0.01468601864865992, 0.016105095704931686, 0.015325412455507122, 0.015884742860931687, 0.015340413563558534, 0.01439783428357799, 0.016146636339502068, 0.0157481788326626, 0.014803997231662638, 0.01730005303543489, 0.015119237051267833, 0.015316463891717098, 0.016193990470574245, 0.015444735325612, 0.01436900713677654, 0.017089409324530784, 0.015287190555916682, 0.015868217655324472, 0.015386344375170133, 0.01521458196402542, 0.01515000807938784, 0.014460878684559446, 0.01642123742472875, 0.0155164235759587, 0.017386234028872995, 0.014933530216687724, 0.014834375745883163, 0.015788902821566902, 0.01668123465490455, 0.01621171485459466, 0.01544481103876507, 0.016961860163918624, 0.015891191651099593, 0.015222802364621703, 0.01593959017968572, 0.015787704332729306 ], "mean_node_assignment_entropy": 4.156307992856062, "normalized_node_entropy": 0.9993808216167671, "effective_mass_slots": 63.92143435435901, "node_slot_mutual_information_nats": 0.0013467481868429942, "normalized_node_slot_mutual_information": 0.0003238244884140985, "centered_token_effective_rank": 1.2611043429161923, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.006110333674975644, "token_RMS": 0.25677789910022564, "mean_token_offdiagonal_cosine": 0.9994479233798585, "temperature": 0.49335938692092896 }, { "layer": 13, "head": 8, "slices": 64, "native_nodes": 33555, "mass": [ 0.015842514864727354, 0.016114441864997383, 0.017119044345163665, 0.0151206951220082, 0.015322942152478366, 0.01623841818900022, 0.015362292011740967, 0.01465756756333614, 0.01468935897833339, 0.016816043937913246, 0.017789402348201012, 0.018259820169316127, 0.01686725548427314, 0.015809793647302634, 0.015860134391484645, 0.0150402478203999, 0.01528566983327072, 0.01453050386600253, 0.015090905410735676, 0.015075433521051776, 0.016786607594405943, 0.01745283063698181, 0.015100224338574029, 0.014058948943155247, 0.016057321471051815, 0.017453382116251307, 0.016290455674185002, 0.015319842471007155, 0.01421020919546727, 0.01669743565875538, 0.016264179392299093, 0.017061814734293974, 0.015989169568398496, 0.016535511721914205, 0.015945788050069572, 0.014955424042780803, 0.015031598183762039, 0.016130027956866487, 0.014130833937817467, 0.015865704083196, 0.014861160101078427, 0.01479336968927714, 0.015424243205551276, 0.015696692773540778, 0.015172262136140128, 0.01517983862014179, 0.014249961438900071, 0.014378144554161018, 0.01429685947567562, 0.015123620221310552, 0.014979350712550564, 0.016134885928757746, 0.015274638461034827, 0.014480113729861202, 0.014751419038465323, 0.015401063989328879, 0.015292320157952342, 0.014460318255905885, 0.016459069952020133, 0.015209463827235389, 0.0162516224705823, 0.017235913598109793, 0.014757024579670493, 0.015906847885616523 ], "mean_node_assignment_entropy": 4.1558076270944175, "normalized_node_entropy": 0.9992605090829412, "effective_mass_slots": 63.878832579237894, "node_slot_mutual_information_nats": 0.0011804208699635055, "normalized_node_slot_mutual_information": 0.0002838312225430309, "centered_token_effective_rank": 1.0985340191598305, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.006972135637686323, "token_RMS": 0.25067521829442835, "mean_token_offdiagonal_cosine": 0.9994153346888052, "temperature": 0.49622172117233276 }, { "layer": 14, "head": 1, "slices": 64, "native_nodes": 33555, "mass": [ 0.015659281463840914, 0.01504444747323909, 0.01495845161293453, 0.01587323136730368, 0.014984854661131418, 0.015997700947691425, 0.014075602072211557, 0.013998354577841444, 0.016652484014885664, 0.01596223309446271, 0.014554059908494538, 0.01488753314334147, 0.015629150826945253, 0.01591794212055804, 0.01634583623302403, 0.015302691628012682, 0.015729417705320287, 0.015461475521852302, 0.01606902824680086, 0.016289296103029392, 0.01570122481936195, 0.01615082885866846, 0.01557147377676459, 0.015776741372804683, 0.015993509133032886, 0.01561124362181936, 0.015346104244189066, 0.0169678133601111, 0.01491310397597956, 0.013810756117895949, 0.01531232662547245, 0.015603830557132803, 0.015433672704235565, 0.017111708899156967, 0.015017250194066488, 0.015431509511702118, 0.014768058313199765, 0.015076882342736202, 0.016676889716576375, 0.016390654095507533, 0.015439772256042887, 0.015759302118123954, 0.015402683067271375, 0.01649892326578167, 0.01491040996990675, 0.01591368611810962, 0.015518982023985469, 0.015594960953704452, 0.015786503354897088, 0.01546996249197679, 0.016253598239686383, 0.01577745713426777, 0.01535353120363668, 0.016685499058712472, 0.015498468459219811, 0.016585809607744313, 0.016314411656038717, 0.01585414355592709, 0.015456307228477545, 0.016044344697572262, 0.015033775886859078, 0.015678686708030937, 0.01589454950706767, 0.015215576277145676 ], "mean_node_assignment_entropy": 4.157194116029736, "normalized_node_entropy": 0.9995938892014797, "effective_mass_slots": 63.94301105866311, "node_slot_mutual_information_nats": 0.0007981184334742153, "normalized_node_slot_mutual_information": 0.00019190691766921976, "centered_token_effective_rank": 1.1488038322677572, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.004611625955771545, "token_RMS": 0.35597103096825305, "mean_token_offdiagonal_cosine": 0.9998572459493231, "temperature": 0.4880652129650116 }, { "layer": 14, "head": 2, "slices": 64, "native_nodes": 33555, "mass": [ 0.01537469737750752, 0.014325308263376232, 0.015328079829528992, 0.016906268365160487, 0.014109279783344086, 0.016166895339350118, 0.014487197046016745, 0.01336496746384863, 0.01560546308464446, 0.01578927304442501, 0.014391754982062377, 0.014755699162654142, 0.01591430135488159, 0.014980570795998419, 0.015787228065771996, 0.016534202999677252, 0.015089097415778307, 0.015763742426798056, 0.016197799992900396, 0.015281236839399236, 0.016374512375674496, 0.016238627363495378, 0.015413118637026893, 0.01585816384963841, 0.01691844886297344, 0.01600793389645961, 0.015723796594666484, 0.015647935111177222, 0.015225882179237987, 0.014697642870066401, 0.014948846410525313, 0.015865589965636116, 0.01485072351212767, 0.01576249344523365, 0.01411563438814255, 0.016933165030392126, 0.016417629838865758, 0.015506159151401779, 0.016398510939244975, 0.01694816232172792, 0.016226288133483053, 0.016160934128149215, 0.015312525946044642, 0.01643012855568327, 0.016875594915471602, 0.015228839281842616, 0.01484729369896621, 0.014678997243015586, 0.015534548802936046, 0.015555360869784422, 0.015920746808802518, 0.01536312141115571, 0.01621692452183949, 0.015744828299657142, 0.014791255694889104, 0.017204380542930255, 0.015850121221438356, 0.015576987048812824, 0.015235046125618698, 0.0154281046650077, 0.016466210614165453, 0.015435325690118962, 0.015834068183747895, 0.016076327078105475 ], "mean_node_assignment_entropy": 4.155925881766165, "normalized_node_entropy": 0.9992889433210231, "effective_mass_slots": 63.92076808439899, "node_slot_mutual_information_nats": 0.001718435958816933, "normalized_node_slot_mutual_information": 0.00041319650597841006, "centered_token_effective_rank": 1.1168194514250553, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.007144175741483871, "token_RMS": 0.2584395154090686, "mean_token_offdiagonal_cosine": 0.9992301602914109, "temperature": 0.48224812746047974 }, { "layer": 14, "head": 3, "slices": 64, "native_nodes": 33555, "mass": [ 0.016092112069196512, 0.01699615785905642, 0.017478299907137625, 0.015891605304064783, 0.01601513304488547, 0.015943854681136106, 0.0160890401940115, 0.01654633129876806, 0.015288613859834401, 0.015009838818666691, 0.015858767629630066, 0.015808200608811815, 0.0159220361890336, 0.014871571182365336, 0.01558051649045303, 0.01661775793601008, 0.01679409719657221, 0.01636338996566469, 0.015223160282392435, 0.01648697746619793, 0.01568519182763011, 0.013694364555910587, 0.015446912502527744, 0.015113933766523029, 0.013703945442487951, 0.014186894397801275, 0.014555331735064476, 0.015972509422018787, 0.016513548231171727, 0.014527151159133922, 0.016775947868282227, 0.01564017537933533, 0.01575697754369445, 0.015464786149177274, 0.016160866056781662, 0.015352777835281991, 0.014853743345517903, 0.01565863457817161, 0.016780226722008833, 0.01570953588956067, 0.01568796211403158, 0.014553354732079408, 0.015858103610289876, 0.015188995033342709, 0.015014463780255498, 0.01585671626696966, 0.014953964914126823, 0.01569783430607028, 0.015328041439499323, 0.015512905257121628, 0.016758011391011973, 0.0156042934799626, 0.014944275258648256, 0.015003784519866288, 0.016009166146358084, 0.015299763830636496, 0.015156426931205582, 0.014982614269880616, 0.015751278614413007, 0.014936611644736062, 0.01524084755986254, 0.01661095815180462, 0.015758251194246675, 0.01586045973314323 ], "mean_node_assignment_entropy": 4.1554187657537724, "normalized_node_entropy": 0.9991670076949842, "effective_mass_slots": 63.92654468326132, "node_slot_mutual_information_nats": 0.0023159191246708843, "normalized_node_slot_mutual_information": 0.000556860839377099, "centered_token_effective_rank": 1.1209838535627028, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.005911111759079718, "token_RMS": 0.3327572490289617, "mean_token_offdiagonal_cosine": 0.9996857538394526, "temperature": 0.47490787506103516 }, { "layer": 14, "head": 4, "slices": 64, "native_nodes": 33555, "mass": [ 0.01617057212818408, 0.015127204455387537, 0.015064319501521468, 0.014788525059570536, 0.016455714754491597, 0.017696056111696723, 0.01629430108244742, 0.015510957763975645, 0.01486813510681936, 0.015213863370209297, 0.015510844068154592, 0.017196614415479334, 0.01471880669466182, 0.016239634231498958, 0.01479015402860008, 0.013950332977025347, 0.015747896430844913, 0.015316243510154308, 0.016349620691185167, 0.01629455201014168, 0.015421890587196356, 0.015682909969474124, 0.01575725559578747, 0.015761638752008148, 0.016169650725700598, 0.015699261624181195, 0.016090302664418545, 0.01473010406375482, 0.016177036197676492, 0.017034900969752274, 0.015753809718165066, 0.01587235688216204, 0.016030535974705374, 0.015395747471365963, 0.016809697797700415, 0.015788182021469664, 0.015240209445500539, 0.016009329725043007, 0.01572106244791071, 0.01612231023559679, 0.01562966527402037, 0.014363881895983917, 0.015993318110932013, 0.015670332471204352, 0.015948320156252787, 0.015374907802564477, 0.014421308700107965, 0.01495412310704449, 0.015010078488834531, 0.015078003750066255, 0.015036801280290878, 0.01530300778681125, 0.015956307146540033, 0.01596636114862122, 0.016152268482199805, 0.01582113911610498, 0.015711174049637983, 0.015030853082820709, 0.01534633087120576, 0.015110572537171638, 0.014582472676035189, 0.015263136982142864, 0.01673729344799997, 0.014965799874639283 ], "mean_node_assignment_entropy": 4.156865979507659, "normalized_node_entropy": 0.9995149890459573, "effective_mass_slots": 63.936698145913915, "node_slot_mutual_information_nats": 0.0010275229081671, "normalized_node_slot_mutual_information": 0.0002470670340020802, "centered_token_effective_rank": 1.1650412551600944, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.005570329278127225, "token_RMS": 0.33603581736954735, "mean_token_offdiagonal_cosine": 0.9997581629507375, "temperature": 0.49270814657211304 }, { "layer": 14, "head": 5, "slices": 64, "native_nodes": 33555, "mass": [ 0.015861176497148494, 0.01576768971539422, 0.015236711750677678, 0.01446383471434597, 0.014943548902755984, 0.01577677192692962, 0.015638083212672686, 0.015637401515744807, 0.014749680588893662, 0.01549504711877623, 0.016802336718993116, 0.014562422191637307, 0.014970755918058878, 0.016079667356820572, 0.01599835974449213, 0.0149058987482141, 0.016221176579066156, 0.01603179855113692, 0.01506878266133498, 0.014971584854018864, 0.016241501580444743, 0.017236389003604132, 0.015312022777423296, 0.016526045265669014, 0.016609642279685886, 0.014829413879442376, 0.016595013234822622, 0.016984696954770746, 0.016055420351719432, 0.01566572991856061, 0.015049986407741396, 0.015331715657377834, 0.015594500255751691, 0.014577616392042931, 0.01478736891747498, 0.015208595541307645, 0.016242945532107287, 0.016898722513053566, 0.01570040574109962, 0.01572916044543742, 0.01562624892233501, 0.016276102074100122, 0.014954202372795591, 0.014945436230881605, 0.015456151952704762, 0.014507982803023203, 0.016469161220299, 0.015114774099201289, 0.01624493031301667, 0.015524980251782387, 0.016087783437686864, 0.016131432742244566, 0.013704631579178063, 0.014360489040438872, 0.015235274501318077, 0.017587152502613734, 0.015219457319522291, 0.015503185463150831, 0.014666311343377722, 0.01636354962627235, 0.01581702072641308, 0.015602576299128501, 0.01628975211594805, 0.01595179067494137 ], "mean_node_assignment_entropy": 4.156670350143818, "normalized_node_entropy": 0.999467950127113, "effective_mass_slots": 63.924858206647166, "node_slot_mutual_information_nats": 0.0010379529104742602, "normalized_node_slot_mutual_information": 0.00024957491943624694, "centered_token_effective_rank": 1.0916688878580187, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.005639390227275853, "token_RMS": 0.2920960054509873, "mean_token_offdiagonal_cosine": 0.9996511293926794, "temperature": 0.4914403259754181 }, { "layer": 14, "head": 6, "slices": 64, "native_nodes": 33555, "mass": [ 0.01466458486580441, 0.01710475472706456, 0.012862686339708612, 0.014654480874779337, 0.015412478266646629, 0.015980074512882587, 0.01646778747040266, 0.01524106729986606, 0.013751287575860551, 0.018041254021588175, 0.014981025458769958, 0.016861083826865884, 0.015497147598093055, 0.015115615053677183, 0.015315717887291564, 0.015208505491393456, 0.015362448972248673, 0.01416077490903203, 0.016769140779599047, 0.015196716359445545, 0.014582349002253608, 0.016112109592078677, 0.018312172942566297, 0.01563774686861262, 0.01958669211019917, 0.014475202004379016, 0.015634134529437967, 0.013014840198931387, 0.013085192320761515, 0.01779184950910583, 0.01678894173357796, 0.016745310096449584, 0.016239114097097364, 0.013252282838182111, 0.01400209317149517, 0.015899882826112037, 0.016095743740246426, 0.016531708234547235, 0.015775383060492466, 0.016455047524218728, 0.0181723759711819, 0.015419985070761342, 0.015861548731863198, 0.014228577717457029, 0.01611064951186865, 0.016612463152432137, 0.014849238457207335, 0.01598093904826289, 0.014448604501773855, 0.016674083392890043, 0.014556693853600878, 0.015557241588225886, 0.016008003401822454, 0.015533805685263555, 0.015715666645767304, 0.016428456584003027, 0.014851318390890367, 0.016056982603358753, 0.01588997580763272, 0.01714482973312376, 0.016205446546887024, 0.014577606615667735, 0.0142488105416591, 0.014200267872398833 ], "mean_node_assignment_entropy": 4.154816002305699, "normalized_node_entropy": 0.9990220737220901, "effective_mass_slots": 63.77948148610664, "node_slot_mutual_information_nats": 0.0006155295176508702, "normalized_node_slot_mutual_information": 0.00014800356377261435, "centered_token_effective_rank": 1.1916724807887804, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.004321011287311815, "token_RMS": 0.25704009459448995, "mean_token_offdiagonal_cosine": 0.9997963231319639, "temperature": 0.4991358518600464 }, { "layer": 14, "head": 7, "slices": 64, "native_nodes": 33555, "mass": [ 0.01432905446206772, 0.015694330191019616, 0.015127439817130307, 0.013710770745736851, 0.015545528723839285, 0.01439913554704659, 0.016642921515038944, 0.016993466732437058, 0.013910008683931701, 0.017349594190767814, 0.01584191927809844, 0.016307964336308562, 0.015856402229355054, 0.014990109977698321, 0.015261616569660208, 0.015842878991287498, 0.014614296595897231, 0.016774796822468175, 0.01573232738180423, 0.014037799772450784, 0.015471513331271714, 0.01709401011233837, 0.016761028197097308, 0.016520780933993954, 0.015868129863982016, 0.016134261944575613, 0.015595691085956417, 0.015827106620026735, 0.014653418304637627, 0.014697764788504071, 0.016135087260464906, 0.015381971210343456, 0.01679604139665816, 0.01557789185604672, 0.016525008771338168, 0.01603373922929423, 0.01666662482440143, 0.014610885653876278, 0.015332470664671522, 0.015208012876234757, 0.016561355544419325, 0.015839579231279908, 0.01612200987844607, 0.015760194254263896, 0.015445118933591913, 0.01616794767968147, 0.015594052465477407, 0.016336976422929247, 0.014924517058884089, 0.014992187211114867, 0.015387463505943146, 0.01631912610849837, 0.014642985153462027, 0.014805335934132208, 0.015639283227125184, 0.01384150026555682, 0.016093666035353896, 0.01656344577874375, 0.014542842402553536, 0.01656685712054925, 0.015394500726613287, 0.015317335688082868, 0.015777451094840016, 0.01551046683160901 ], "mean_node_assignment_entropy": 4.156727243609892, "normalized_node_entropy": 0.999481630114007, "effective_mass_slots": 63.907253782701936, "node_slot_mutual_information_nats": 0.0007056290571121338, "normalized_node_slot_mutual_information": 0.00016966792356713843, "centered_token_effective_rank": 1.0979354175660756, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.00514823980599822, "token_RMS": 0.32439227641402446, "mean_token_offdiagonal_cosine": 0.999753479326564, "temperature": 0.49228718876838684 }, { "layer": 14, "head": 8, "slices": 64, "native_nodes": 33555, "mass": [ 0.015461971982369887, 0.014289345164110099, 0.015220510471968035, 0.015854340401409973, 0.019499795511753508, 0.0178847501447696, 0.016723365057773266, 0.015751612458431923, 0.01416528508321923, 0.014922102673178795, 0.015230455346718583, 0.016869918586118107, 0.013343497847541757, 0.015812663134559934, 0.016837731044894357, 0.014109675757916686, 0.01542382478886626, 0.016530981426579803, 0.016294470617211516, 0.014659501814271357, 0.01566578615720493, 0.017656139321397642, 0.01484247680582498, 0.01754762789420756, 0.01709697585918743, 0.018273132694811584, 0.014138814205540539, 0.014820015770394976, 0.015933611342438356, 0.017241730530332868, 0.01371503814204477, 0.013177870982496568, 0.013779865587059414, 0.0164755381507217, 0.01647339843745346, 0.01571200024220005, 0.014907829637700702, 0.014707019334405576, 0.015110545135721411, 0.015091489773061037, 0.015502196802219093, 0.015651343326980666, 0.014676415307202791, 0.017547018138846935, 0.015344421811998023, 0.016213755112909078, 0.016139290665572358, 0.013377751539651575, 0.016622343001771463, 0.013455100751682759, 0.014930060982311151, 0.015563793862639588, 0.014423322950944586, 0.01597910652109486, 0.014936477977562113, 0.01461740551770928, 0.017503078659015802, 0.015930323884159776, 0.0162796198898759, 0.015877979114538438, 0.013515398247802228, 0.0165029457524239, 0.01722227393167551, 0.014935870784497344 ], "mean_node_assignment_entropy": 4.154549418341301, "normalized_node_entropy": 0.9989579738281873, "effective_mass_slots": 63.77542687840723, "node_slot_mutual_information_nats": 0.0008185391711457157, "normalized_node_slot_mutual_information": 0.00019681706716421444, "centered_token_effective_rank": 1.1071843573088094, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.0046530913107271235, "token_RMS": 0.2386663571511329, "mean_token_offdiagonal_cosine": 0.9996309830351291, "temperature": 0.49394768476486206 }, { "layer": 15, "head": 1, "slices": 64, "native_nodes": 33555, "mass": [ 0.015350019494150704, 0.016848445792351213, 0.015233363882967274, 0.01595693646286656, 0.01634198091280272, 0.01639194356008638, 0.01636290370090195, 0.01618838410065713, 0.015340972143139193, 0.015297808444354754, 0.015645495170985218, 0.01588807853504698, 0.013817368061901373, 0.015581622871141057, 0.015514419538862232, 0.01512757138710071, 0.015600875765529302, 0.01582339312342782, 0.016516128019766273, 0.0154901239961872, 0.015762585169918675, 0.014801935221394268, 0.01509729022785495, 0.014861005041517918, 0.015769786831125288, 0.01679488698598161, 0.016391400159162465, 0.0161342909556832, 0.015787009014983262, 0.016516933380456143, 0.016054723704614842, 0.016880953535459788, 0.016282860317433038, 0.015929111796930143, 0.015645031639347176, 0.01446607208625759, 0.014952215484579805, 0.013712129807547945, 0.016780676429050723, 0.015249943521277344, 0.015039732751166336, 0.015204329535863587, 0.014780935837890419, 0.0157410037498008, 0.01582581103841686, 0.015528038945790109, 0.015158792378303457, 0.016506577759366103, 0.014910074618785593, 0.016206423725753538, 0.015431067702292313, 0.015190605122377871, 0.016235083192747878, 0.015706401886652992, 0.016114663261453374, 0.014138962564935248, 0.015733555184529863, 0.016030331185600197, 0.015990799423898364, 0.015106050184488101, 0.015599004549547895, 0.015166440310886188, 0.015281757733652363, 0.015184880965426088 ], "mean_node_assignment_entropy": 4.157230525200181, "normalized_node_entropy": 0.9996026437564205, "effective_mass_slots": 63.939008398186154, "node_slot_mutual_information_nats": 0.0006991099938700174, "normalized_node_slot_mutual_information": 0.00016810042019869796, "centered_token_effective_rank": 1.1074379831830834, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.003282024419054145, "token_RMS": 0.3074652984443287, "mean_token_offdiagonal_cosine": 0.9998898149634196, "temperature": 0.48993679881095886 }, { "layer": 15, "head": 2, "slices": 64, "native_nodes": 33555, "mass": [ 0.01571675594618527, 0.014353799816615677, 0.014897250940165875, 0.01634086608273886, 0.015995405876717328, 0.014318459058207373, 0.014492634734889067, 0.015373395960358896, 0.015239701398836085, 0.016017220859297732, 0.015807557267608293, 0.01596600829821117, 0.01694563803837678, 0.015543351448226385, 0.014986738996923574, 0.015329437503668062, 0.014654040512132175, 0.01577245741774514, 0.016052357413567465, 0.015702412177069882, 0.01614448777252161, 0.014287642037406878, 0.016233180990772086, 0.015751671649275826, 0.015221925438474017, 0.015941694663370434, 0.016017077270922442, 0.015485010880795397, 0.014865607990754369, 0.01635122009694705, 0.015651272344074766, 0.015044995263688497, 0.014274419002060633, 0.01581749823288566, 0.01575589464335408, 0.014774761790657537, 0.014629214692368466, 0.017459911741666032, 0.015620416046607745, 0.016647523777974894, 0.015860337550671236, 0.016607488323441028, 0.01752699035831987, 0.015207340352563624, 0.014946530490193304, 0.015372354517709018, 0.016448393606805195, 0.01598384727978827, 0.015879534503228262, 0.015869702673719087, 0.015627825878740398, 0.016007752861173147, 0.01453924137766449, 0.016278899574882405, 0.014792946151691841, 0.016007336862929274, 0.01574853544147479, 0.015072383081329249, 0.015786268060432693, 0.015401782011556352, 0.01653200773661866, 0.015756736004608803, 0.015641353926323705, 0.015623495413395397 ], "mean_node_assignment_entropy": 4.15716516649223, "normalized_node_entropy": 0.9995869283091138, "effective_mass_slots": 63.93432044969944, "node_slot_mutual_information_nats": 0.0006911469461270414, "normalized_node_slot_mutual_information": 0.00016618571195050572, "centered_token_effective_rank": 1.0886191695447414, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.0035022940288077543, "token_RMS": 0.32100324152409304, "mean_token_offdiagonal_cosine": 0.9998828299723055, "temperature": 0.48903751373291016 }, { "layer": 15, "head": 3, "slices": 64, "native_nodes": 33555, "mass": [ 0.015572101061139825, 0.016165069792535687, 0.015431653231719872, 0.015360656451345637, 0.014928562220448558, 0.01706510207261499, 0.01605053177332935, 0.015596992671578531, 0.01587085503468107, 0.01482500264018934, 0.01517269758530091, 0.015320402540814954, 0.016579769861167916, 0.01647971706969197, 0.015863678430561922, 0.01491943786450584, 0.015196637271687937, 0.014978081410640559, 0.015425550959384835, 0.016590395454839684, 0.016953406151094908, 0.015663723318335633, 0.014887294017645538, 0.014841662110066097, 0.014795957082465365, 0.01651251650969214, 0.0154940126853175, 0.01583182440812276, 0.014886790101523682, 0.015603373290155199, 0.016953509948529316, 0.015236056216045572, 0.017869508767373123, 0.01618598098399662, 0.015630274850166274, 0.015540254819622526, 0.016770554067920312, 0.01589659686341733, 0.015343004985344853, 0.016040042728255596, 0.014589199516741827, 0.014398312014533864, 0.014199241479788535, 0.01548401831702395, 0.014477129066269998, 0.01596467936376251, 0.015093702144117038, 0.015431403286334317, 0.014738889135067277, 0.015383701488978505, 0.014945657895590691, 0.015040895419569711, 0.015245881167090147, 0.017092342529184432, 0.015447859415006503, 0.016327327988778687, 0.015505193038551449, 0.016482112343314686, 0.015560631589935444, 0.01596525309649297, 0.015625964179263128, 0.015168388991007026, 0.016072453811349502, 0.015430523215533147 ], "mean_node_assignment_entropy": 4.157360113376931, "normalized_node_entropy": 0.9996338031264128, "effective_mass_slots": 63.93102864800832, "node_slot_mutual_information_nats": 0.0004447114960708376, "normalized_node_slot_mutual_information": 0.00010693051166795153, "centered_token_effective_rank": 1.125670027911283, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.002979100033502145, "token_RMS": 0.26308736482313183, "mean_token_offdiagonal_cosine": 0.9998701901995909, "temperature": 0.4933025538921356 }, { "layer": 15, "head": 4, "slices": 64, "native_nodes": 33555, "mass": [ 0.015442616471526982, 0.01524640468561628, 0.01594332392971267, 0.015761153893790902, 0.013591725864027024, 0.015150445771466476, 0.014882182728539696, 0.015542438531033844, 0.015783698280051744, 0.014232256753233334, 0.016561048056454582, 0.01442317686782937, 0.01525443910671482, 0.015343732305060927, 0.014963625360853695, 0.016402301990631495, 0.014643019087631503, 0.01631835549356503, 0.014411691987212299, 0.015865740510634385, 0.015748495980569873, 0.017257587629882613, 0.01611355658789347, 0.015263697271909875, 0.01380690788938172, 0.015027612835810536, 0.016729893146082482, 0.015657342016344996, 0.01688129567042839, 0.014330062981500096, 0.017412095567545402, 0.016618009820533852, 0.014867707215429392, 0.014732058404140911, 0.017187888562765888, 0.01643531914834267, 0.01590777005569521, 0.01367571251201809, 0.016562872256508012, 0.016116041524080117, 0.015766569994214597, 0.016311841957285533, 0.01721974859834872, 0.01510490883369903, 0.017476278862989067, 0.014779597414803744, 0.015714453612754684, 0.016797986266197256, 0.016732354351403887, 0.015895333134038687, 0.015580367828583179, 0.014247351795720275, 0.015915619190951615, 0.015366609975187136, 0.014737845880591849, 0.015496848125497554, 0.01757758731715522, 0.014304104893722419, 0.01579861064376139, 0.013683259609912932, 0.01680331164178586, 0.015269561523701783, 0.0146207040278767, 0.01670384023293038 ], "mean_node_assignment_entropy": 4.156252075488972, "normalized_node_entropy": 0.999367376332067, "effective_mass_slots": 63.867501127444896, "node_slot_mutual_information_nats": 0.0005585669641288149, "normalized_node_slot_mutual_information": 0.0001343069648588408, "centered_token_effective_rank": 1.1089442493051054, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.0037913764799503286, "token_RMS": 0.24186896809140687, "mean_token_offdiagonal_cosine": 0.9997542534112251, "temperature": 0.4999861419200897 }, { "layer": 15, "head": 5, "slices": 64, "native_nodes": 33555, "mass": [ 0.015685736373729695, 0.016780827545110292, 0.014883823093149553, 0.015085885193403752, 0.0161284977406421, 0.015245401975993667, 0.015929722148970028, 0.014899433270233334, 0.016417745980577752, 0.015976299575305594, 0.01512749157857076, 0.01695606964473095, 0.01601953265386102, 0.016653219589858364, 0.01539628684643954, 0.013904259559985529, 0.01536913426578236, 0.015039341975957701, 0.015838181681102806, 0.015196212397416744, 0.015086863751921066, 0.01590860136757552, 0.014822607642036049, 0.014897421984058338, 0.017207693493773907, 0.01643991077013759, 0.015366306309988621, 0.014876613977865204, 0.01670265799018314, 0.015150417065470767, 0.01503313447519051, 0.01667208241487807, 0.01710320133868521, 0.01530073024096605, 0.015645106908029983, 0.01508152931484553, 0.014786423816710751, 0.017435000984339845, 0.015992916879486868, 0.01426907975788404, 0.015272586567752312, 0.015664804492225157, 0.015235768495350433, 0.016754950660417853, 0.014160896181275327, 0.015211481001711588, 0.015425762943612616, 0.014518300547664051, 0.014850302593652928, 0.01537832073443223, 0.017428059648872844, 0.016732581410650843, 0.01571011594851239, 0.014969647535173028, 0.018305749208847488, 0.01552012851343809, 0.015120124242114046, 0.01554198145539994, 0.015653292945473875, 0.015400107752605596, 0.015327438402872668, 0.014812722435878039, 0.016913987245427888, 0.013779485388849757 ], "mean_node_assignment_entropy": 4.156668809368612, "normalized_node_entropy": 0.9994675796489881, "effective_mass_slots": 63.89549450883584, "node_slot_mutual_information_nats": 0.0005800410601155548, "normalized_node_slot_mutual_information": 0.00013947039349011467, "centered_token_effective_rank": 1.0614958531602348, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.0037266989922683444, "token_RMS": 0.33758952590726693, "mean_token_offdiagonal_cosine": 0.9998762718271512, "temperature": 0.48433974385261536 }, { "layer": 15, "head": 6, "slices": 64, "native_nodes": 33555, "mass": [ 0.015295568823918752, 0.014040745000810546, 0.015312115257435772, 0.016140232140909157, 0.017329735549161524, 0.014527710964546582, 0.015473555347844326, 0.014996763234068502, 0.014117900517785395, 0.016240865893072456, 0.016700016432461894, 0.015577874751601963, 0.016954606936889546, 0.01409018084386314, 0.01564691906250628, 0.016721552244585044, 0.015422285736665172, 0.017537158741745468, 0.015492035678835291, 0.013016695174651012, 0.015197787661343533, 0.01238415937142447, 0.01703799519681785, 0.016452940200631594, 0.017552246926806177, 0.013875217703821238, 0.015096012325192764, 0.016539515134359573, 0.01297132769673663, 0.01710658052300882, 0.013580470470201727, 0.015721375961203616, 0.012671942767513534, 0.016491007982170222, 0.014563917909375407, 0.016755342438307442, 0.015038665528464, 0.014945167362814316, 0.01491364523443135, 0.016642947538892252, 0.016543537945930827, 0.017835071723729997, 0.01742590425335922, 0.014684731582621804, 0.01528132404222243, 0.017787129885092824, 0.01763098914222854, 0.01643230490727319, 0.01670802841558353, 0.014999295490157043, 0.01493277275614799, 0.015248487945915271, 0.013730106895450614, 0.013959303775097553, 0.015624427276917026, 0.016202105825228196, 0.01525212953541877, 0.016120114943339828, 0.015217382335185232, 0.016316762331563987, 0.01640422881788081, 0.01750504246397623, 0.016065448887242085, 0.015920584348703866 ], "mean_node_assignment_entropy": 4.151078239310342, "normalized_node_entropy": 0.9981233316991869, "effective_mass_slots": 63.772408392344964, "node_slot_mutual_information_nats": 0.004242387158713612, "normalized_node_slot_mutual_information": 0.0010200784859011915, "centered_token_effective_rank": 1.017170156740436, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.010365017612045706, "token_RMS": 0.3838525288189469, "mean_token_offdiagonal_cosine": 0.9992631480768973, "temperature": 0.4711304306983948 }, { "layer": 15, "head": 7, "slices": 64, "native_nodes": 33555, "mass": [ 0.01435960143906689, 0.015465038091764732, 0.017162635730477636, 0.015970193646429653, 0.015652301037444555, 0.015225969239566466, 0.016225103581316725, 0.015610560328001469, 0.015461527224812746, 0.01608196462563823, 0.015859417190253203, 0.015439596813160445, 0.014782664400399522, 0.01577559760288059, 0.013859653442453039, 0.01619759079287054, 0.016449092017580227, 0.015883467870696017, 0.01547255506190857, 0.014683129934746965, 0.013719052409206706, 0.015387383509483375, 0.016067515018755092, 0.01758476548011213, 0.016938498666011225, 0.014566385699817701, 0.015199747222742572, 0.01591059670527816, 0.014698165605759775, 0.014709149929288913, 0.014367252441031259, 0.015983643860508587, 0.015252430772878182, 0.014582611163854687, 0.015570278015199201, 0.01648700093769171, 0.015095327387911795, 0.0162649925731843, 0.016599055915719002, 0.015181427917632922, 0.016623485728797453, 0.016898506834232635, 0.01770518047516107, 0.016479689692222153, 0.016574132488880903, 0.015138607964168325, 0.014949828544677397, 0.01604941300215149, 0.01745973258995207, 0.014342450512113221, 0.016218525816502852, 0.015959647363000425, 0.01304521880973968, 0.014935394389838793, 0.015709460425204366, 0.01673752657859305, 0.01593654113182624, 0.01568628596382016, 0.015134543527765337, 0.015601663317932685, 0.014344984327289319, 0.015037029147377945, 0.016159575982090082, 0.01548963375797576 ], "mean_node_assignment_entropy": 4.156723729050912, "normalized_node_entropy": 0.999480785041205, "effective_mass_slots": 63.88743813471811, "node_slot_mutual_information_nats": 0.00039902669581515227, "normalized_node_slot_mutual_information": 9.594563920580485e-05, "centered_token_effective_rank": 1.1290097724547157, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.0029299401047441238, "token_RMS": 0.23659578452751925, "mean_token_offdiagonal_cosine": 0.9998764769483577, "temperature": 0.49824902415275574 }, { "layer": 15, "head": 8, "slices": 64, "native_nodes": 33555, "mass": [ 0.01558207672082677, 0.01624647814276996, 0.014973064805441368, 0.015098169698340939, 0.014856440224382938, 0.015470096658714889, 0.015840895480368492, 0.015464490503289229, 0.01487545775380882, 0.01649217861531753, 0.014928880729438465, 0.015990180759592864, 0.015372499406022441, 0.015790441254079742, 0.015367416958690285, 0.014948692111621234, 0.016072065367958323, 0.016275572736383, 0.014535972649419455, 0.015234380806229355, 0.015246173092600549, 0.015064065472110873, 0.01595481658620018, 0.015466083661127004, 0.015307039898396817, 0.015555547130968732, 0.017045859719171837, 0.015467341108914593, 0.01602917671522242, 0.01596718850870181, 0.014983612975496158, 0.0147875846209702, 0.014807634474633303, 0.015146014958702226, 0.01680762959812672, 0.01623118374163082, 0.01599434406642715, 0.015573966200224085, 0.0159973811630491, 0.014191887445382596, 0.01588839915719962, 0.015471444196744837, 0.014338123245599674, 0.014926074055843517, 0.01613524381114072, 0.014359106622956056, 0.015102978921164398, 0.01592139711823065, 0.016190621403430777, 0.015622571519169539, 0.0162225069528946, 0.015237904265006711, 0.016630713653245158, 0.016007880671742226, 0.015792824507521244, 0.01507417418258638, 0.016779224626301405, 0.015302377814104226, 0.015895669648653882, 0.017560353274650747, 0.01504347261683656, 0.017019997827621015, 0.016567034091933122, 0.015869923359838653 ], "mean_node_assignment_entropy": 4.15749079899289, "normalized_node_entropy": 0.9996652263747561, "effective_mass_slots": 63.937452441766965, "node_slot_mutual_information_nats": 0.0004145008948972517, "normalized_node_slot_mutual_information": 9.966639758538376e-05, "centered_token_effective_rank": 1.2763709097106903, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.002585056580660126, "token_RMS": 0.31401163678303134, "mean_token_offdiagonal_cosine": 0.9999408060417799, "temperature": 0.48827069997787476 }, { "layer": 16, "head": 1, "slices": 64, "native_nodes": 33555, "mass": [ 0.01567285247902022, 0.016055173476464905, 0.014168300091023506, 0.01618068903776763, 0.01562810211240215, 0.016389173936005003, 0.014557216384433818, 0.016119245704773588, 0.014873465957962984, 0.014871738514678523, 0.015898942188202286, 0.015627065316734124, 0.015002665506136558, 0.016418592760350757, 0.015332723310934175, 0.015385896396855007, 0.0146417846911809, 0.015868479166845462, 0.015419844573388918, 0.015446503247664372, 0.01662326865680899, 0.015838291188456356, 0.016386715889270143, 0.01518435830630905, 0.01561800425045382, 0.013664454444543454, 0.015319507871299978, 0.016167173465521517, 0.015353864472292252, 0.016604745696501253, 0.0176518720005281, 0.016543717538923215, 0.016067705808517973, 0.01580567904479492, 0.01517687220445019, 0.015561618626360841, 0.01647800396502822, 0.01435426960261701, 0.015648541785110492, 0.015340267066170608, 0.015967634639997914, 0.015702721825936142, 0.01605381272464542, 0.015356165924470474, 0.015120614437915684, 0.01609014655790769, 0.015678135087238153, 0.016294635825370947, 0.016152936773161795, 0.015538936333805478, 0.015533989922795347, 0.014595059921315255, 0.016586245112998166, 0.01439122134541243, 0.01519729362676097, 0.015224727785131238, 0.015221460225292893, 0.01630905714547149, 0.015954085552989727, 0.015258765544322133, 0.015522219209885023, 0.014215387904916901, 0.016668294783836506, 0.0164190654709631 ], "mean_node_assignment_entropy": 4.157123888045708, "normalized_node_entropy": 0.999577002940765, "effective_mass_slots": 63.932187182803425, "node_slot_mutual_information_nats": 0.0006990582985482163, "normalized_node_slot_mutual_information": 0.00016808799010129805, "centered_token_effective_rank": 1.0710210863888512, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.0034116285797237376, "token_RMS": 0.2992133081625691, "mean_token_offdiagonal_cosine": 0.9998709631692626, "temperature": 0.4885954260826111 }, { "layer": 16, "head": 2, "slices": 64, "native_nodes": 33555, "mass": [ 0.01573039902302818, 0.015145926605873982, 0.01575550913761881, 0.015255343571367446, 0.015684579345607897, 0.01596158544708228, 0.015394727254433234, 0.015919871864175098, 0.014503347014861506, 0.015311424685951593, 0.01575473976217298, 0.015209397642604419, 0.016563713571161917, 0.014945829172237472, 0.016178024194206508, 0.015197720269337708, 0.015533040580218302, 0.015996666086804067, 0.015219177577171084, 0.01628995083972996, 0.01551347438666779, 0.015548731290795394, 0.016327666827273376, 0.015265657126486098, 0.015614542320443948, 0.016552592865593154, 0.014534467476460488, 0.01646405269648986, 0.014673233885168719, 0.015452733630018322, 0.01610803907114589, 0.014760965301131078, 0.01640911698768798, 0.01599121869666186, 0.015704473816557794, 0.015212316859933734, 0.015686948002583755, 0.01392293052742017, 0.015148885927693603, 0.01627154433138079, 0.015170086902525871, 0.016308489978849674, 0.016265199829912667, 0.01572812072790614, 0.016291920767522235, 0.01516505741904086, 0.015218486082801883, 0.016049676579179696, 0.015674239177589328, 0.01505309844664611, 0.016544412619862697, 0.01529317167810313, 0.01567565805975788, 0.015954357404453338, 0.01605323440607761, 0.015937406326640098, 0.016339774104758082, 0.01575287763301892, 0.016225289423398938, 0.014214995529931725, 0.01692729170340226, 0.014787808013545643, 0.01652135803191997, 0.014133423531458232 ], "mean_node_assignment_entropy": 4.157597804032009, "normalized_node_entropy": 0.9996909556479707, "effective_mass_slots": 63.94508223977514, "node_slot_mutual_information_nats": 0.0004268209541775647, "normalized_node_slot_mutual_information": 0.00010262874565657802, "centered_token_effective_rank": 1.2093826868002762, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.0023691295479003737, "token_RMS": 0.2846624624245445, "mean_token_offdiagonal_cosine": 0.9999425978329767, "temperature": 0.4913613498210907 }, { "layer": 16, "head": 3, "slices": 64, "native_nodes": 33555, "mass": [ 0.01570722745455865, 0.01576852592729968, 0.016687341756322686, 0.014124761332609698, 0.014744759763289433, 0.01711540005811368, 0.015653170186306294, 0.015435225235981329, 0.016622950467668922, 0.014834400565359165, 0.015220208926343683, 0.015554860405232011, 0.0156905749613163, 0.01493685411964061, 0.015666732571673826, 0.01610912410910124, 0.015949258105719726, 0.016402679416050572, 0.015201778364309635, 0.015021344169590197, 0.015400756900649085, 0.015374300379770698, 0.015200700318503799, 0.01413682071560631, 0.01492936306921292, 0.01640205488090183, 0.015400759639356392, 0.014735324838481962, 0.016131779893865272, 0.01518050965525109, 0.015414104116002284, 0.016970140713048733, 0.014285993767254729, 0.016849987764389427, 0.01732759065398728, 0.01491273435063337, 0.0154587266137378, 0.016841912676815198, 0.014970513029658765, 0.01406706870795673, 0.015056778089759639, 0.015956898270564266, 0.01608727712657693, 0.016721334131030722, 0.01453499142990913, 0.01653463710118887, 0.014318996848519921, 0.01463892546090621, 0.016493230232850414, 0.015671633257396157, 0.016514086756578252, 0.0159269176977264, 0.016043764811396074, 0.017519568927245245, 0.015699485028302762, 0.016818506306229095, 0.013887867549798729, 0.016310215543280004, 0.01652337219518923, 0.01517377182352256, 0.01489987088848339, 0.015712918448932897, 0.014702286816980218, 0.015814344715085554 ], "mean_node_assignment_entropy": 4.156998059752444, "normalized_node_entropy": 0.999546747631649, "effective_mass_slots": 63.903229756961416, "node_slot_mutual_information_nats": 0.0003718442811528888, "normalized_node_slot_mutual_information": 8.940965006703236e-05, "centered_token_effective_rank": 1.138959210743523, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.0023205645851609724, "token_RMS": 0.3417851151863231, "mean_token_offdiagonal_cosine": 0.9999533481170356, "temperature": 0.49255257844924927 }, { "layer": 16, "head": 4, "slices": 64, "native_nodes": 33555, "mass": [ 0.01305547409537292, 0.016136267867242947, 0.01352574314012837, 0.015446226375218858, 0.015455296880011453, 0.013386054355232038, 0.014616166786660542, 0.01629284405071944, 0.015250761331741953, 0.014962218556014121, 0.014937827971169533, 0.016134588469212313, 0.016095748355504192, 0.015428280021960335, 0.01599306935071723, 0.014470707936202152, 0.015271074108717258, 0.016245279378601717, 0.015426862724108733, 0.01701949612233211, 0.01699178118928723, 0.015876647179513047, 0.014952394459271703, 0.016192571363089745, 0.01395580199233618, 0.016724952150620837, 0.016998067223320524, 0.016022714768995597, 0.014918469523919701, 0.01750574953445529, 0.0179036968164096, 0.01466640613175459, 0.015054920573004569, 0.015092230097848254, 0.01744626172224423, 0.01570190350608208, 0.01593101013797643, 0.014215663097318107, 0.017409203697016512, 0.015655284282055978, 0.015960770846634432, 0.016817377129828604, 0.015813980250051898, 0.01594752617476772, 0.012574479905459622, 0.01379312944958341, 0.017117447514076414, 0.015497814342179733, 0.014688598427595689, 0.01635906165070202, 0.016884728730826207, 0.01720457920542889, 0.015361605213424645, 0.016465845948684422, 0.015336569510633438, 0.01650369392416238, 0.015278304665736746, 0.014999602755081758, 0.015217838581857391, 0.015397724381815506, 0.015466098674085764, 0.01656151956169043, 0.01350348758942976, 0.01688249768144442 ], "mean_node_assignment_entropy": 4.155726573823961, "normalized_node_entropy": 0.9992410198910521, "effective_mass_slots": 63.828364804496914, "node_slot_mutual_information_nats": 0.00047110713837028584, "normalized_node_slot_mutual_information": 0.00011327732204236702, "centered_token_effective_rank": 1.0412157350788536, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.0034669454285725077, "token_RMS": 0.26141919427243504, "mean_token_offdiagonal_cosine": 0.999823573128436, "temperature": 0.4962366819381714 }, { "layer": 16, "head": 5, "slices": 64, "native_nodes": 33555, "mass": [ 0.01438830360837393, 0.015568913283186527, 0.01586442376006655, 0.015060350827755071, 0.015464489838443424, 0.01564960292003568, 0.015990282493934752, 0.015126263992001893, 0.01638370205081623, 0.01620258346328773, 0.016369144219226247, 0.01685599651631185, 0.016768534910397315, 0.014192796138775382, 0.015934635973792906, 0.01567821164817093, 0.015738281556197657, 0.015724626389341693, 0.015273679760352028, 0.015579742502191884, 0.016244220860649944, 0.017592680570437023, 0.014645952448379866, 0.015320700661570274, 0.015992523535348906, 0.015244497827027616, 0.01648215710341868, 0.015407399561257836, 0.0148186246864914, 0.015502343731115647, 0.014821228461492445, 0.015576125049111234, 0.014917608277963519, 0.016141715229352493, 0.014991701114298272, 0.013996411208521324, 0.017095645438258568, 0.016262602490737795, 0.014028001553347776, 0.01533926464664436, 0.015171368506636718, 0.015446190661836081, 0.01650344061463596, 0.015147837512378293, 0.016704980092631903, 0.01607834901596844, 0.014735376530035057, 0.01565642693580849, 0.016143391999668487, 0.014808990430953366, 0.015411679125598416, 0.015817842608246403, 0.015367645201581483, 0.016248577882857298, 0.016470879439978953, 0.016116312746946092, 0.01510576736445648, 0.015725510643170366, 0.01539970179749124, 0.016211148361172933, 0.016155534240898318, 0.015577435599160682, 0.015293731620709279, 0.014465910964510139 ], "mean_node_assignment_entropy": 4.155910965035143, "normalized_node_entropy": 0.9992853566053779, "effective_mass_slots": 63.930669408143366, "node_slot_mutual_information_nats": 0.0018882406435194454, "normalized_node_slot_mutual_information": 0.00045402590206841486, "centered_token_effective_rank": 1.0492407924198925, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.005543853041587008, "token_RMS": 0.302445426343545, "mean_token_offdiagonal_cosine": 0.9996618822255734, "temperature": 0.4771822988986969 }, { "layer": 16, "head": 6, "slices": 64, "native_nodes": 33555, "mass": [ 0.01475251784819886, 0.015076511405156366, 0.015469040411547095, 0.01629831074309445, 0.01655208460410418, 0.013612512968238562, 0.015530403295090285, 0.016355618252082406, 0.014173000208054289, 0.01707005185445496, 0.01640527416739489, 0.016160353240846363, 0.017225645040179254, 0.013445697418735877, 0.017319331916123785, 0.015601926281462038, 0.015112277328293044, 0.01670463933870185, 0.016466451385655893, 0.01655806776620764, 0.01618287174930126, 0.017889856411228235, 0.015275417703281988, 0.014031592039789684, 0.014771926725171412, 0.01709188814851579, 0.016164273517257657, 0.015775782352660615, 0.014365562740666174, 0.016811363335747138, 0.015446094992123583, 0.014853136217011512, 0.014108392551975418, 0.01689552102232944, 0.013698215959914226, 0.015518707012991635, 0.015350510728798197, 0.015300555740119652, 0.01388960389564439, 0.01482105540152461, 0.015375768349258525, 0.01648984963737396, 0.01590294907547588, 0.015189333585238177, 0.01473672711568664, 0.015083677436828374, 0.014628712381796215, 0.01643927494199141, 0.015418223042339153, 0.015189713872069886, 0.016012579104954006, 0.01697860925650298, 0.015604115048882202, 0.016832132207352135, 0.01664576473275474, 0.015218597108720535, 0.015518004451246247, 0.016414760011580657, 0.0155808433913657, 0.01723245648108506, 0.015202602295635479, 0.015471849365029699, 0.013958858001509254, 0.01474255551362718 ], "mean_node_assignment_entropy": 4.155775968645037, "normalized_node_entropy": 0.999252896834954, "effective_mass_slots": 63.861704819173276, "node_slot_mutual_information_nats": 0.000943914482776087, "normalized_node_slot_mutual_information": 0.00022696345722072195, "centered_token_effective_rank": 1.0330735323228695, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.004660316101315654, "token_RMS": 0.3907153641262532, "mean_token_offdiagonal_cosine": 0.9998625690822568, "temperature": 0.48309779167175293 }, { "layer": 16, "head": 7, "slices": 64, "native_nodes": 33555, "mass": [ 0.013621901732086336, 0.016724260241923417, 0.016489497237568054, 0.017370768801607952, 0.015189427749569247, 0.0162945832847059, 0.014991289117174938, 0.015556643076680657, 0.013245761469161966, 0.015534156359978617, 0.01523076883936169, 0.015934843945048717, 0.018434185595887907, 0.011753068089783814, 0.017471380978089972, 0.014709928785906634, 0.013209072426999535, 0.017049779323187943, 0.015766675463699847, 0.016899690552746604, 0.017615369626800645, 0.018143497415485352, 0.017682598761224994, 0.014339402851426172, 0.013391520878967026, 0.017490891368945608, 0.015980612146489808, 0.014251700920295545, 0.014949061582926585, 0.014311768004684339, 0.014973811330802483, 0.015107790145491802, 0.0147912308063584, 0.015908563893741894, 0.015141666972003028, 0.016644532660905268, 0.015361414548809536, 0.01566178420267976, 0.012519026620027926, 0.014975814721343761, 0.016523375081192886, 0.015830420276543952, 0.0168321989380093, 0.016189722076331497, 0.01607923005242525, 0.01371035990378102, 0.016146032830872084, 0.016832133111113888, 0.015028006130710939, 0.015626020440555617, 0.01538626503933739, 0.017915508839769577, 0.01590937058767602, 0.01845723531602916, 0.01667292503388317, 0.015925410414768783, 0.015425782810717338, 0.015114231949616315, 0.01501127876640543, 0.01429878666171519, 0.01569729132585826, 0.014874385767275638, 0.015564000178913951, 0.014230286058289934 ], "mean_node_assignment_entropy": 4.153749157092538, "normalized_node_entropy": 0.9987655516723527, "effective_mass_slots": 63.74699569958389, "node_slot_mutual_information_nats": 0.0011728995544828535, "normalized_node_slot_mutual_information": 0.0002820227284522146, "centered_token_effective_rank": 1.0234184268655453, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.005050015987078586, "token_RMS": 0.2749041201446649, "mean_token_offdiagonal_cosine": 0.9997021070546267, "temperature": 0.4849376082420349 }, { "layer": 16, "head": 8, "slices": 64, "native_nodes": 33555, "mass": [ 0.015283287204319098, 0.016329627851449757, 0.016000158485591928, 0.015210392889628475, 0.015443441601679108, 0.016538797573893795, 0.016217576658580524, 0.01492374492743952, 0.016693750822855843, 0.016845423484204727, 0.015383691978386202, 0.015856160087928103, 0.015500128493845383, 0.014074037503603762, 0.01428414168722529, 0.015461875953950401, 0.015323937680078786, 0.0156384901514835, 0.016594342527029286, 0.015649893547328764, 0.015031473574217066, 0.015295484531940034, 0.015507751200228792, 0.016090853782816154, 0.016375918808651163, 0.016102344719902614, 0.015093574307652445, 0.016350645683578132, 0.015707286494081714, 0.015214402071305642, 0.01423639458518255, 0.014831828275999917, 0.016536172003168995, 0.015181663655485131, 0.01606342598933001, 0.01595987882628879, 0.015035450292356794, 0.016172300476062947, 0.014960935650716158, 0.015865097917070147, 0.014584636631861802, 0.014605581166780292, 0.015682559461345207, 0.015979182784270865, 0.015552070903859787, 0.01610454630343716, 0.01637394676811248, 0.015695046713805255, 0.016339656451114456, 0.01590176908358445, 0.01617256644044658, 0.015537972791411007, 0.014914709755645976, 0.015272835250086266, 0.015862440087585073, 0.015275429612137471, 0.015906015519958114, 0.01626804783926993, 0.01589751745620111, 0.015383264571295074, 0.01609643284179245, 0.015444086778244511, 0.01570024065656143, 0.014583660099023128 ], "mean_node_assignment_entropy": 4.15739115234062, "normalized_node_entropy": 0.9996412664195776, "effective_mass_slots": 63.94877585078104, "node_slot_mutual_information_nats": 0.0006912332145585509, "normalized_node_slot_mutual_information": 0.00016620645512355973, "centered_token_effective_rank": 1.0487138815420878, "centered_token_rank_upper_bound": 32, "centered_token_RMS": 0.0028976687383427536, "token_RMS": 0.31567334263937397, "mean_token_offdiagonal_cosine": 0.999915703887339, "temperature": 0.48512744903564453 } ], "source": "Reused frozen original 64-slice CPU extraction; no repeated inference" } ], "scope": "One original TRAIN case, all native nodes, every layer/head; not a 20-case collapse or causal conclusion.", "numerical_note": "CPU FP32 observation; not a bitwise reproduction of TF32-enabled GPU activations.", "degeneracy_note": "M=1 normalized entropy/MI and pair cosine are undefined (null), centered rank is 0. M=2 centered rank has upper bound 1 by definition.", "last_layer_head_means": [ { "variant": "S001", "slices": 1, "layers": 16, "normalized_node_entropy": null, "normalized_node_slot_mutual_information": null, "centered_token_effective_rank": 0.0, "centered_token_rank_upper_bound": 0, "relative_centered_token_RMS": 0.0, "mean_token_offdiagonal_cosine": null }, { "variant": "S002", "slices": 2, "layers": 16, "normalized_node_entropy": 0.5074798371643883, "normalized_node_slot_mutual_information": 0.4087906952442173, "centered_token_effective_rank": 1.0, "centered_token_rank_upper_bound": 1, "relative_centered_token_RMS": 0.6687604381089353, "mean_token_offdiagonal_cosine": 0.04131381878275044 }, { "variant": "S004", "slices": 4, "layers": 16, "normalized_node_entropy": 0.17914782152739017, "normalized_node_slot_mutual_information": 0.4840526694857542, "centered_token_effective_rank": 1.0280390534292367, "centered_token_rank_upper_bound": 3, "relative_centered_token_RMS": 0.7278841172182288, "mean_token_offdiagonal_cosine": 0.2823366198626278 }, { "variant": "S008", "slices": 8, "layers": 16, "normalized_node_entropy": 0.3954043466247261, "normalized_node_slot_mutual_information": 0.26848449560659093, "centered_token_effective_rank": 1.0579414956533544, "centered_token_rank_upper_bound": 7, "relative_centered_token_RMS": 0.48955334391600575, "mean_token_offdiagonal_cosine": 0.7473094255441419 }, { "variant": "S016", "slices": 16, "layers": 16, "normalized_node_entropy": 0.9741510088138351, "normalized_node_slot_mutual_information": 0.0003087463080348083, "centered_token_effective_rank": 1.2553367093925543, "centered_token_rank_upper_bound": 15, "relative_centered_token_RMS": 0.00988973522474868, "mean_token_offdiagonal_cosine": 0.9998781052059307 }, { "variant": "S032", "slices": 32, "layers": 16, "normalized_node_entropy": 0.2719400339956173, "normalized_node_slot_mutual_information": 0.00016934801764730546, "centered_token_effective_rank": 1.604158534869681, "centered_token_rank_upper_bound": 31, "relative_centered_token_RMS": 0.007830770215925095, "mean_token_offdiagonal_cosine": 0.9999278875638126 }, { "variant": "S064", "slices": 64, "layers": 16, "normalized_node_entropy": 0.9993750997054623, "normalized_node_slot_mutual_information": 0.0002003277813415233, "centered_token_effective_rank": 1.0768781690202374, "centered_token_rank_upper_bound": 32, "relative_centered_token_RMS": 0.012197914346460541, "mean_token_offdiagonal_cosine": 0.9998415930621884 }, { "variant": "H064", "slices": 64, "layers": 8, "normalized_node_entropy": 0.7424285247849378, "normalized_node_slot_mutual_information": 0.0010201827817272557, "centered_token_effective_rank": 1.1175705174390753, "centered_token_rank_upper_bound": 32, "relative_centered_token_RMS": 0.026402418349553022, "mean_token_offdiagonal_cosine": 0.9992195978911281 } ] }