{"type": "excalidraw", "version": 2, "source": "https://excalidraw.com", "elements": [{"type": "text", "x": 20, "y": 10, "width": 1300, "height": 28, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0001", "roundness": null, "seed": 1561692720, "version": 1, "versionNonce": 109525499, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0001", "text": "One token through the V4.1-Flash router (reference inference code)", "originalText": "One token through the V4.1-Flash router (reference inference code)", "fontSize": 22, "fontFamily": 1, "textAlign": "left", "verticalAlign": "middle", "lineHeight": 1.25, "autoResize": false, "containerId": null}, {"type": "rectangle", "x": 40, "y": 80, "width": 170, "height": 50, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#e9ecef", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0002", "roundness": {"type": 3}, "seed": 302003102, "version": 1, "versionNonce": 893149981, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0002"}, {"type": "text", "x": 40, "y": 80, "width": 170, "height": 50, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0003", "roundness": null, "seed": 1615892811, "version": 1, "versionNonce": 638019486, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0003", "text": "token hidden state\n(5120, fp32)", "originalText": "token hidden state\n(5120, fp32)", "fontSize": 12, "fontFamily": 1, "textAlign": "center", "verticalAlign": "middle", "lineHeight": 1.25, "autoResize": false, "containerId": null}, {"type": "rectangle", "x": 250, "y": 80, "width": 220, "height": 50, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#d0bfff", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0004", "roundness": {"type": 3}, "seed": 1083438815, "version": 1, "versionNonce": 1492026738, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0004"}, {"type": "text", "x": 250, "y": 80, "width": 220, "height": 50, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0005", "roundness": null, "seed": 1564070057, "version": 1, "versionNonce": 2036465043, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0005", "text": "W_gate: 384 x 5120 (fp32)\nlogits (384)", "originalText": "W_gate: 384 x 5120 (fp32)\nlogits (384)", "fontSize": 11, "fontFamily": 1, "textAlign": "center", "verticalAlign": "middle", "lineHeight": 1.25, "autoResize": false, "containerId": null}, {"type": "arrow", "x": 210, "y": 105, "width": 40, "height": 0, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0006", "roundness": null, "seed": 527603372, "version": 1, "versionNonce": 495439556, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0006", "points": [[0, 0], [40, 0]], "startArrowhead": null, "endArrowhead": "arrow", "startBinding": null, "endBinding": null, "lastCommittedPoint": null, "elbowed": false}, {"type": "rectangle", "x": 510, "y": 80, "width": 200, "height": 50, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#fff3bf", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0007", "roundness": {"type": 3}, "seed": 2096236325, "version": 1, "versionNonce": 2001409496, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0007"}, {"type": "text", "x": 510, "y": 80, "width": 200, "height": 50, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0008", "roundness": null, "seed": 2063281257, "version": 1, "versionNonce": 2078054028, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0008", "text": "affinity s_i = sqrt(softplus(logit))\n(384)", "originalText": "affinity s_i = sqrt(softplus(logit))\n(384)", "fontSize": 11, "fontFamily": 1, "textAlign": "center", "verticalAlign": "middle", "lineHeight": 1.25, "autoResize": false, "containerId": null}, {"type": "arrow", "x": 470, "y": 105, "width": 40, "height": 0, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0009", "roundness": null, "seed": 1339395519, "version": 1, "versionNonce": 368871839, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0009", "points": [[0, 0], [40, 0]], "startArrowhead": null, "endArrowhead": "arrow", "startBinding": null, "endBinding": null, "lastCommittedPoint": null, "elbowed": false}, {"type": "rectangle", "x": 750, "y": 80, "width": 230, "height": 50, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#ffd8a8", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0010", "roundness": {"type": 3}, "seed": 618979931, "version": 1, "versionNonce": 438892468, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0010"}, {"type": "text", "x": 750, "y": 80, "width": 230, "height": 50, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0011", "roundness": null, "seed": 1471609727, "version": 1, "versionNonce": 1137122203, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0011", "text": "+ bias_i for selection only\n(text bias or image bias, per token)", "originalText": "+ bias_i for selection only\n(text bias or image bias, per token)", "fontSize": 11, "fontFamily": 1, "textAlign": "center", "verticalAlign": "middle", "lineHeight": 1.25, "autoResize": false, "containerId": null}, {"type": "arrow", "x": 710, "y": 105, "width": 40, "height": 0, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0012", "roundness": null, "seed": 2055665571, "version": 1, "versionNonce": 693373551, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0012", "points": [[0, 0], [40, 0]], "startArrowhead": null, "endArrowhead": "arrow", "startBinding": null, "endBinding": null, "lastCommittedPoint": null, "elbowed": false}, {"type": "rectangle", "x": 1020, "y": 80, "width": 160, "height": 50, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#ffc9c9", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0013", "roundness": {"type": 3}, "seed": 99195380, "version": 1, "versionNonce": 881391735, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0013"}, {"type": "text", "x": 1020, "y": 80, "width": 160, "height": 50, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0014", "roundness": null, "seed": 1553714998, "version": 1, "versionNonce": 629653100, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0014", "text": "top-6 on s_i + bias_i\n-> 6 indices", "originalText": "top-6 on s_i + bias_i\n-> 6 indices", "fontSize": 11, "fontFamily": 1, "textAlign": "center", "verticalAlign": "middle", "lineHeight": 1.25, "autoResize": false, "containerId": null}, {"type": "arrow", "x": 980, "y": 105, "width": 40, "height": 0, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0015", "roundness": null, "seed": 116146606, "version": 1, "versionNonce": 1280285447, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0015", "points": [[0, 0], [40, 0]], "startArrowhead": null, "endArrowhead": "arrow", "startBinding": null, "endBinding": null, "lastCommittedPoint": null, "elbowed": false}, {"type": "rectangle", "x": 510, "y": 200, "width": 200, "height": 50, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#fff3bf", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0016", "roundness": {"type": 3}, "seed": 390887331, "version": 1, "versionNonce": 1121481225, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0016"}, {"type": "text", "x": 510, "y": 200, "width": 200, "height": 50, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0017", "roundness": null, "seed": 1574963608, "version": 1, "versionNonce": 717440071, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0017", "text": "gather the 6 unbiased s_i", "originalText": "gather the 6 unbiased s_i", "fontSize": 11, "fontFamily": 1, "textAlign": "center", "verticalAlign": "middle", "lineHeight": 1.25, "autoResize": false, "containerId": null}, {"type": "arrow", "x": 1100, "y": 130, "width": 390, "height": 95, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0018", "roundness": null, "seed": 1527703408, "version": 1, "versionNonce": 956887592, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0018", "points": [[0, 0], [0, 95], [-390, 95]], "startArrowhead": null, "endArrowhead": "arrow", "startBinding": null, "endBinding": null, "lastCommittedPoint": null, "elbowed": false}, {"type": "rectangle", "x": 250, "y": 200, "width": 220, "height": 50, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#c3fae8", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0019", "roundness": {"type": 3}, "seed": 1415900355, "version": 1, "versionNonce": 957956675, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0019"}, {"type": "text", "x": 250, "y": 200, "width": 220, "height": 50, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0020", "roundness": null, "seed": 838145800, "version": 1, "versionNonce": 1028162214, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0020", "text": "normalize to sum 1,\nthen x 1.5 (routed_scaling_factor)", "originalText": "normalize to sum 1,\nthen x 1.5 (routed_scaling_factor)", "fontSize": 11, "fontFamily": 1, "textAlign": "center", "verticalAlign": "middle", "lineHeight": 1.25, "autoResize": false, "containerId": null}, {"type": "arrow", "x": 510, "y": 225, "width": 40, "height": 0, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0021", "roundness": null, "seed": 1720926263, "version": 1, "versionNonce": 973838694, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0021", "points": [[0, 0], [-40, 0]], "startArrowhead": null, "endArrowhead": "arrow", "startBinding": null, "endBinding": null, "lastCommittedPoint": null, "elbowed": false}, {"type": "rectangle", "x": 40, "y": 200, "width": 170, "height": 50, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#b2f2bb", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0022", "roundness": {"type": 3}, "seed": 858641202, "version": 1, "versionNonce": 2116481899, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0022"}, {"type": "text", "x": 40, "y": 200, "width": 170, "height": 50, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0023", "roundness": null, "seed": 1527129487, "version": 1, "versionNonce": 124468791, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0023", "text": "6 mixing weights", "originalText": "6 mixing weights", "fontSize": 12, "fontFamily": 1, "textAlign": "center", "verticalAlign": "middle", "lineHeight": 1.25, "autoResize": false, "containerId": null}, {"type": "arrow", "x": 250, "y": 225, "width": 40, "height": 0, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0024", "roundness": null, "seed": 119988829, "version": 1, "versionNonce": 1200093500, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0024", "points": [[0, 0], [-40, 0]], "startArrowhead": null, "endArrowhead": "arrow", "startBinding": null, "endBinding": null, "lastCommittedPoint": null, "elbowed": false}, {"type": "rectangle", "x": 250, "y": 320, "width": 100, "height": 60, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#a5d8ff", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0025", "roundness": {"type": 3}, "seed": 2028255630, "version": 1, "versionNonce": 1113145427, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0025"}, {"type": "text", "x": 250, "y": 320, "width": 100, "height": 60, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0026", "roundness": null, "seed": 831698693, "version": 1, "versionNonce": 1478675320, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0026", "text": "expert 17\nSwiGLU, clamp 10\nx weight", "originalText": "expert 17\nSwiGLU, clamp 10\nx weight", "fontSize": 10, "fontFamily": 1, "textAlign": "center", "verticalAlign": "middle", "lineHeight": 1.25, "autoResize": false, "containerId": null}, {"type": "arrow", "x": 120, "y": 250, "width": 130, "height": 100, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 0.8, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0027", "roundness": null, "seed": 1920828234, "version": 1, "versionNonce": 1501175504, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0027", "points": [[0, 0], [0, 100], [130, 100]], "startArrowhead": null, "endArrowhead": "arrow", "startBinding": null, "endBinding": null, "lastCommittedPoint": null, "elbowed": false}, {"type": "rectangle", "x": 370, "y": 320, "width": 100, "height": 60, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#a5d8ff", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0028", "roundness": {"type": 3}, "seed": 1566099206, "version": 1, "versionNonce": 345908636, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0028"}, {"type": "text", "x": 370, "y": 320, "width": 100, "height": 60, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0029", "roundness": null, "seed": 946878465, "version": 1, "versionNonce": 438761610, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0029", "text": "expert 88\nSwiGLU, clamp 10\nx weight", "originalText": "expert 88\nSwiGLU, clamp 10\nx weight", "fontSize": 10, "fontFamily": 1, "textAlign": "center", "verticalAlign": "middle", "lineHeight": 1.25, "autoResize": false, "containerId": null}, {"type": "arrow", "x": 120, "y": 250, "width": 250, "height": 100, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 0.8, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0030", "roundness": null, "seed": 974295421, "version": 1, "versionNonce": 2018978167, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0030", "points": [[0, 0], [0, 100], [250, 100]], "startArrowhead": null, "endArrowhead": "arrow", "startBinding": null, "endBinding": null, "lastCommittedPoint": null, "elbowed": false}, {"type": "rectangle", "x": 490, "y": 320, "width": 100, "height": 60, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#a5d8ff", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0031", "roundness": {"type": 3}, "seed": 844846558, "version": 1, "versionNonce": 1450571438, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0031"}, {"type": "text", "x": 490, "y": 320, "width": 100, "height": 60, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0032", "roundness": null, "seed": 877776916, "version": 1, "versionNonce": 2072980150, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0032", "text": "expert 140\nSwiGLU, clamp 10\nx weight", "originalText": "expert 140\nSwiGLU, clamp 10\nx weight", "fontSize": 10, "fontFamily": 1, "textAlign": "center", "verticalAlign": "middle", "lineHeight": 1.25, "autoResize": false, "containerId": null}, {"type": "arrow", "x": 120, "y": 250, "width": 370, "height": 100, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 0.8, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0033", "roundness": null, "seed": 8196149, "version": 1, "versionNonce": 2059322682, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0033", "points": [[0, 0], [0, 100], [370, 100]], "startArrowhead": null, "endArrowhead": "arrow", "startBinding": null, "endBinding": null, "lastCommittedPoint": null, "elbowed": false}, {"type": "rectangle", "x": 610, "y": 320, "width": 100, "height": 60, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#a5d8ff", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0034", "roundness": {"type": 3}, "seed": 1477498383, "version": 1, "versionNonce": 364120812, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0034"}, {"type": "text", "x": 610, "y": 320, "width": 100, "height": 60, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0035", "roundness": null, "seed": 514982154, "version": 1, "versionNonce": 1668748296, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0035", "text": "expert 203\nSwiGLU, clamp 10\nx weight", "originalText": "expert 203\nSwiGLU, clamp 10\nx weight", "fontSize": 10, "fontFamily": 1, "textAlign": "center", "verticalAlign": "middle", "lineHeight": 1.25, "autoResize": false, "containerId": null}, {"type": "arrow", "x": 120, "y": 250, "width": 490, "height": 100, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 0.8, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0036", "roundness": null, "seed": 856070306, "version": 1, "versionNonce": 2053134994, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0036", "points": [[0, 0], [0, 100], [490, 100]], "startArrowhead": null, "endArrowhead": "arrow", "startBinding": null, "endBinding": null, "lastCommittedPoint": null, "elbowed": false}, {"type": "rectangle", "x": 730, "y": 320, "width": 100, "height": 60, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#a5d8ff", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0037", "roundness": {"type": 3}, "seed": 766744960, "version": 1, "versionNonce": 1863693998, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0037"}, {"type": "text", "x": 730, "y": 320, "width": 100, "height": 60, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0038", "roundness": null, "seed": 1428150522, "version": 1, "versionNonce": 372587780, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0038", "text": "expert 290\nSwiGLU, clamp 10\nx weight", "originalText": "expert 290\nSwiGLU, clamp 10\nx weight", "fontSize": 10, "fontFamily": 1, "textAlign": "center", "verticalAlign": "middle", "lineHeight": 1.25, "autoResize": false, "containerId": null}, {"type": "arrow", "x": 120, "y": 250, "width": 610, "height": 100, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 0.8, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0039", "roundness": null, "seed": 1700113407, "version": 1, "versionNonce": 1989259376, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0039", "points": [[0, 0], [0, 100], [610, 100]], "startArrowhead": null, "endArrowhead": "arrow", "startBinding": null, "endBinding": null, "lastCommittedPoint": null, "elbowed": false}, {"type": "rectangle", "x": 850, "y": 320, "width": 100, "height": 60, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#a5d8ff", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0040", "roundness": {"type": 3}, "seed": 1723943247, "version": 1, "versionNonce": 364725392, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0040"}, {"type": "text", "x": 850, "y": 320, "width": 100, "height": 60, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0041", "roundness": null, "seed": 682281554, "version": 1, "versionNonce": 730160160, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0041", "text": "expert 371\nSwiGLU, clamp 10\nx weight", "originalText": "expert 371\nSwiGLU, clamp 10\nx weight", "fontSize": 10, "fontFamily": 1, "textAlign": "center", "verticalAlign": "middle", "lineHeight": 1.25, "autoResize": false, "containerId": null}, {"type": "arrow", "x": 120, "y": 250, "width": 730, "height": 100, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 0.8, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0042", "roundness": null, "seed": 545625653, "version": 1, "versionNonce": 118321418, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0042", "points": [[0, 0], [0, 100], [730, 100]], "startArrowhead": null, "endArrowhead": "arrow", "startBinding": null, "endBinding": null, "lastCommittedPoint": null, "elbowed": false}, {"type": "rectangle", "x": 1000, "y": 320, "width": 180, "height": 60, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#b2f2bb", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0043", "roundness": {"type": 3}, "seed": 649187325, "version": 1, "versionNonce": 1998679711, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0043"}, {"type": "text", "x": 1000, "y": 320, "width": 180, "height": 60, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0044", "roundness": null, "seed": 627813882, "version": 1, "versionNonce": 2037347502, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0044", "text": "shared expert\nweight 1, always", "originalText": "shared expert\nweight 1, always", "fontSize": 11, "fontFamily": 1, "textAlign": "center", "verticalAlign": "middle", "lineHeight": 1.25, "autoResize": false, "containerId": null}, {"type": "rectangle", "x": 250, "y": 420, "width": 930, "height": 44, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#e9ecef", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0045", "roundness": {"type": 3}, "seed": 1504988819, "version": 1, "versionNonce": 669638768, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0045"}, {"type": "text", "x": 250, "y": 420, "width": 930, "height": 44, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0046", "roundness": null, "seed": 562571391, "version": 1, "versionNonce": 91898035, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0046", "text": "sum of 6 weighted routed outputs + shared output -> back into the residual stream", "originalText": "sum of 6 weighted routed outputs + shared output -> back into the residual stream", "fontSize": 12, "fontFamily": 1, "textAlign": "center", "verticalAlign": "middle", "lineHeight": 1.25, "autoResize": false, "containerId": null}, {"type": "arrow", "x": 300, "y": 380, "width": 0, "height": 40, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 0.8, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0047", "roundness": null, "seed": 61172930, "version": 1, "versionNonce": 441402618, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0047", "points": [[0, 0], [0, 40]], "startArrowhead": null, "endArrowhead": "arrow", "startBinding": null, "endBinding": null, "lastCommittedPoint": null, "elbowed": false}, {"type": "arrow", "x": 420, "y": 380, "width": 0, "height": 40, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 0.8, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0048", "roundness": null, "seed": 598077321, "version": 1, "versionNonce": 1863197322, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0048", "points": [[0, 0], [0, 40]], "startArrowhead": null, "endArrowhead": "arrow", "startBinding": null, "endBinding": null, "lastCommittedPoint": null, "elbowed": false}, {"type": "arrow", "x": 540, "y": 380, "width": 0, "height": 40, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 0.8, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0049", "roundness": null, "seed": 836682997, "version": 1, "versionNonce": 906419965, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0049", "points": [[0, 0], [0, 40]], "startArrowhead": null, "endArrowhead": "arrow", "startBinding": null, "endBinding": null, "lastCommittedPoint": null, "elbowed": false}, {"type": "arrow", "x": 660, "y": 380, "width": 0, "height": 40, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 0.8, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0050", "roundness": null, "seed": 120232147, "version": 1, "versionNonce": 1081622283, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0050", "points": [[0, 0], [0, 40]], "startArrowhead": null, "endArrowhead": "arrow", "startBinding": null, "endBinding": null, "lastCommittedPoint": null, "elbowed": false}, {"type": "arrow", "x": 780, "y": 380, "width": 0, "height": 40, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 0.8, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0051", "roundness": null, "seed": 913882254, "version": 1, "versionNonce": 1258282194, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0051", "points": [[0, 0], [0, 40]], "startArrowhead": null, "endArrowhead": "arrow", "startBinding": null, "endBinding": null, "lastCommittedPoint": null, "elbowed": false}, {"type": "arrow", "x": 900, "y": 380, "width": 0, "height": 40, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 0.8, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0052", "roundness": null, "seed": 1033108813, "version": 1, "versionNonce": 1400113411, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0052", "points": [[0, 0], [0, 40]], "startArrowhead": null, "endArrowhead": "arrow", "startBinding": null, "endBinding": null, "lastCommittedPoint": null, "elbowed": false}, {"type": "arrow", "x": 1090, "y": 380, "width": 0, "height": 40, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 0.8, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0053", "roundness": null, "seed": 1113963314, "version": 1, "versionNonce": 1799645191, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0053", "points": [[0, 0], [0, 40]], "startArrowhead": null, "endArrowhead": "arrow", "startBinding": null, "endBinding": null, "lastCommittedPoint": null, "elbowed": false}, {"type": "text", "x": 40, "y": 490, "width": 1200, "height": 37.2, "angle": 0, "strokeColor": "#495057", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0054", "roundness": null, "seed": 562957180, "version": 1, "versionNonce": 261582917, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0054", "text": "Bias update (training only): after each step, b_i += u * sign(mean load - load_i), u = 0.001; the two modality biases update against their own loads.\nThe bias never touches the mixing weights, so no gradient interference (Wang et al. 2024).", "originalText": "Bias update (training only): after each step, b_i += u * sign(mean load - load_i), u = 0.001; the two modality biases update against their own loads.\nThe bias never touches the mixing weights, so no gradient interference (Wang et al. 2024).", "fontSize": 12, "fontFamily": 1, "textAlign": "left", "verticalAlign": "middle", "lineHeight": 1.25, "autoResize": false, "containerId": null}], "appState": {"gridSize": 20, "gridStep": 5, "gridModeEnabled": false, "viewBackgroundColor": "#fdf8f6", "lockedMultiSelections": {}}, "files": {}}