{ "type": "excalidraw", "version": 2, "source": "https://excalidraw.com", "elements": [ { "id": "e1", "type": "rectangle", "x": 0, "y": 0, "width": 175, "height": 36, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#f1f3f5", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": { "type": 3 }, "seed": 2055665571, "version": 1, "versionNonce": 693373551, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "e2", "type": "text", "x": 6, "y": 9.875, "width": 0.0, "height": 16.25, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": null, "seed": 99195380, "version": 1, "versionNonce": 881391735, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "", "originalText": "", "fontSize": 13, "fontFamily": 1, "textAlign": "left", "verticalAlign": "top", "containerId": null, "lineHeight": 1.25, "baseline": 11, "autoResize": true }, { "id": "e3", "type": "rectangle", "x": 181, "y": 0, "width": 200, "height": 36, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#f1f3f5", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": { "type": 3 }, "seed": 1553714998, "version": 1, "versionNonce": 629653100, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "e4", "type": "text", "x": 187, "y": 9.875, "width": 107.25000000000001, "height": 16.25, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": null, "seed": 116146606, "version": 1, "versionNonce": 1280285447, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "A (activations)", "originalText": "A (activations)", "fontSize": 13, "fontFamily": 1, "textAlign": "left", "verticalAlign": "top", "containerId": null, "lineHeight": 1.25, "baseline": 11, "autoResize": true }, { "id": "e5", "type": "rectangle", "x": 387, "y": 0, "width": 235, "height": 36, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#f1f3f5", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": { "type": 3 }, "seed": 390887331, "version": 1, "versionNonce": 1121481225, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "e6", "type": "text", "x": 393, "y": 9.875, "width": 164.45000000000002, "height": 16.25, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": null, "seed": 1574963608, "version": 1, "versionNonce": 717440071, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "B (weights of expert e)", "originalText": "B (weights of expert e)", "fontSize": 13, "fontFamily": 1, "textAlign": "left", "verticalAlign": "top", "containerId": null, "lineHeight": 1.25, "baseline": 11, "autoResize": true }, { "id": "e7", "type": "rectangle", "x": 628, "y": 0, "width": 200, "height": 36, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#f1f3f5", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": { "type": 3 }, "seed": 1527703408, "version": 1, "versionNonce": 956887592, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "e8", "type": "text", "x": 634, "y": 9.875, "width": 71.5, "height": 16.25, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": null, "seed": 1415900355, "version": 1, "versionNonce": 957956675, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "C (output)", "originalText": "C (output)", "fontSize": 13, "fontFamily": 1, "textAlign": "left", "verticalAlign": "top", "containerId": null, "lineHeight": 1.25, "baseline": 11, "autoResize": true }, { "id": "e9", "type": "rectangle", "x": 0, "y": 40, "width": 175, "height": 36, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#ffffff", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": { "type": 3 }, "seed": 838145800, "version": 1, "versionNonce": 1028162214, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "e10", "type": "text", "x": 6, "y": 49.875, "width": 121.55000000000001, "height": 16.25, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": null, "seed": 1720926263, "version": 1, "versionNonce": 973838694, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "GEMM 1 gate + up", "originalText": "GEMM 1 gate + up", "fontSize": 13, "fontFamily": 1, "textAlign": "left", "verticalAlign": "top", "containerId": null, "lineHeight": 1.25, "baseline": 11, "autoResize": true }, { "id": "e11", "type": "rectangle", "x": 181, "y": 40, "width": 200, "height": 36, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#ffffff", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": { "type": 3 }, "seed": 858641202, "version": 1, "versionNonce": 2116481899, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "e12", "type": "text", "x": 187, "y": 49.875, "width": 121.55000000000001, "height": 16.25, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": null, "seed": 1527129487, "version": 1, "versionNonce": 124468791, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "X_e [n_e, 2048]", "originalText": "X_e [n_e, 2048]", "fontSize": 13, "fontFamily": 1, "textAlign": "left", "verticalAlign": "top", "containerId": null, "lineHeight": 1.25, "baseline": 11, "autoResize": true }, { "id": "e13", "type": "rectangle", "x": 387, "y": 40, "width": 235, "height": 36, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#e7f5ff", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": { "type": 3 }, "seed": 119988829, "version": 1, "versionNonce": 1200093500, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "e14", "type": "text", "x": 393, "y": 49.875, "width": 143.0, "height": 16.25, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": null, "seed": 2028255630, "version": 1, "versionNonce": 1113145427, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "W13_e^T [2048, 1536]", "originalText": "W13_e^T [2048, 1536]", "fontSize": 13, "fontFamily": 1, "textAlign": "left", "verticalAlign": "top", "containerId": null, "lineHeight": 1.25, "baseline": 11, "autoResize": true }, { "id": "e15", "type": "rectangle", "x": 628, "y": 40, "width": 200, "height": 36, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#ffffff", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": { "type": 3 }, "seed": 831698693, "version": 1, "versionNonce": 1478675320, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "e16", "type": "text", "x": 634, "y": 49.875, "width": 107.25000000000001, "height": 16.25, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": null, "seed": 1920828234, "version": 1, "versionNonce": 1501175504, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "H_e [n_e, 1536]", "originalText": "H_e [n_e, 1536]", "fontSize": 13, "fontFamily": 1, "textAlign": "left", "verticalAlign": "top", "containerId": null, "lineHeight": 1.25, "baseline": 11, "autoResize": true }, { "id": "e17", "type": "rectangle", "x": 0, "y": 80, "width": 175, "height": 36, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#ffffff", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": { "type": 3 }, "seed": 1566099206, "version": 1, "versionNonce": 345908636, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "e18", "type": "text", "x": 6, "y": 89.875, "width": 71.5, "height": 16.25, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": null, "seed": 946878465, "version": 1, "versionNonce": 438761610, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "SiLU x mul", "originalText": "SiLU x mul", "fontSize": 13, "fontFamily": 1, "textAlign": "left", "verticalAlign": "top", "containerId": null, "lineHeight": 1.25, "baseline": 11, "autoResize": true }, { "id": "e19", "type": "rectangle", "x": 181, "y": 80, "width": 200, "height": 36, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#ffffff", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": { "type": 3 }, "seed": 974295421, "version": 1, "versionNonce": 2018978167, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "e20", "type": "text", "x": 187, "y": 89.875, "width": 121.55000000000001, "height": 16.25, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": null, "seed": 844846558, "version": 1, "versionNonce": 1450571438, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "H_e [n_e, 1536]", "originalText": "H_e [n_e, 1536]", "fontSize": 13, "fontFamily": 1, "textAlign": "left", "verticalAlign": "top", "containerId": null, "lineHeight": 1.25, "baseline": 11, "autoResize": true }, { "id": "e21", "type": "rectangle", "x": 387, "y": 80, "width": 235, "height": 36, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#ffffff", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": { "type": 3 }, "seed": 877776916, "version": 1, "versionNonce": 2072980150, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "e22", "type": "text", "x": 393, "y": 89.875, "width": 71.5, "height": 16.25, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": null, "seed": 8196149, "version": 1, "versionNonce": 2059322682, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "no weights", "originalText": "no weights", "fontSize": 13, "fontFamily": 1, "textAlign": "left", "verticalAlign": "top", "containerId": null, "lineHeight": 1.25, "baseline": 11, "autoResize": true }, { "id": "e23", "type": "rectangle", "x": 628, "y": 80, "width": 200, "height": 36, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#ffffff", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": { "type": 3 }, "seed": 1477498383, "version": 1, "versionNonce": 364120812, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "e24", "type": "text", "x": 634, "y": 89.875, "width": 100.10000000000001, "height": 16.25, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": null, "seed": 514982154, "version": 1, "versionNonce": 1668748296, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "A_e [n_e, 768]", "originalText": "A_e [n_e, 768]", "fontSize": 13, "fontFamily": 1, "textAlign": "left", "verticalAlign": "top", "containerId": null, "lineHeight": 1.25, "baseline": 11, "autoResize": true }, { "id": "e25", "type": "rectangle", "x": 0, "y": 120, "width": 175, "height": 36, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#ffffff", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": { "type": 3 }, "seed": 856070306, "version": 1, "versionNonce": 2053134994, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "e26", "type": "text", "x": 6, "y": 129.875, "width": 85.80000000000001, "height": 16.25, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": null, "seed": 766744960, "version": 1, "versionNonce": 1863693998, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "GEMM 2 down", "originalText": "GEMM 2 down", "fontSize": 13, "fontFamily": 1, "textAlign": "left", "verticalAlign": "top", "containerId": null, "lineHeight": 1.25, "baseline": 11, "autoResize": true }, { "id": "e27", "type": "rectangle", "x": 181, "y": 120, "width": 200, "height": 36, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#ffffff", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": { "type": 3 }, "seed": 1428150522, "version": 1, "versionNonce": 372587780, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "e28", "type": "text", "x": 187, "y": 129.875, "width": 114.4, "height": 16.25, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": null, "seed": 1700113407, "version": 1, "versionNonce": 1989259376, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "A_e [n_e, 768]", "originalText": "A_e [n_e, 768]", "fontSize": 13, "fontFamily": 1, "textAlign": "left", "verticalAlign": "top", "containerId": null, "lineHeight": 1.25, "baseline": 11, "autoResize": true }, { "id": "e29", "type": "rectangle", "x": 387, "y": 120, "width": 235, "height": 36, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#e7f5ff", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": { "type": 3 }, "seed": 1723943247, "version": 1, "versionNonce": 364725392, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "e30", "type": "text", "x": 393, "y": 129.875, "width": 135.85000000000002, "height": 16.25, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": null, "seed": 682281554, "version": 1, "versionNonce": 730160160, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "W2_e^T [768, 2048]", "originalText": "W2_e^T [768, 2048]", "fontSize": 13, "fontFamily": 1, "textAlign": "left", "verticalAlign": "top", "containerId": null, "lineHeight": 1.25, "baseline": 11, "autoResize": true }, { "id": "e31", "type": "rectangle", "x": 628, "y": 120, "width": 200, "height": 36, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#ffffff", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": { "type": 3 }, "seed": 545625653, "version": 1, "versionNonce": 118321418, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "e32", "type": "text", "x": 634, "y": 129.875, "width": 107.25000000000001, "height": 16.25, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": null, "seed": 649187325, "version": 1, "versionNonce": 1998679711, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "Y_e [n_e, 2048]", "originalText": "Y_e [n_e, 2048]", "fontSize": 13, "fontFamily": 1, "textAlign": "left", "verticalAlign": "top", "containerId": null, "lineHeight": 1.25, "baseline": 11, "autoResize": true }, { "id": "e33", "type": "text", "x": 0, "y": 172, "width": 562.1, "height": 17.5, "angle": 0, "strokeColor": "#1971c2", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": null, "seed": 627813882, "version": 1, "versionNonce": 2037347502, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "Only the row count n_e changes from expert to expert. K and N stay fixed.", "originalText": "Only the row count n_e changes from expert to expert. K and N stay fixed.", "fontSize": 14, "fontFamily": 1, "textAlign": "left", "verticalAlign": "top", "containerId": null, "lineHeight": 1.25, "baseline": 12, "autoResize": true } ], "appState": { "viewBackgroundColor": "#fdf8f6", "gridSize": null }, "files": {} }