{ "type": "excalidraw", "version": 2, "source": "https://excalidraw.com", "elements": [ { "id": "e1", "type": "rectangle", "x": 0, "y": 30, "width": 110, "height": 40, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#e9ecef", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": { "type": 3 }, "seed": 1390851129, "version": 1, "versionNonce": 647892280, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "e2", "type": "text", "x": 6, "y": 41.25, "width": 61.60000000000001, "height": 17.5, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": null, "seed": 1695753999, "version": 1, "versionNonce": 207388625, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "x [2048]", "originalText": "x [2048]", "fontSize": 14, "fontFamily": 1, "textAlign": "left", "verticalAlign": "top", "containerId": null, "lineHeight": 1.25, "baseline": 12, "autoResize": true }, { "id": "e3", "type": "arrow", "x": 110, "y": 50, "width": 40, "height": 0, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": { "type": 2 }, "seed": 311111476, "version": 1, "versionNonce": 404285458, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 40, 0 ] ], "lastCommittedPoint": null, "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "e4", "type": "rectangle", "x": 150, "y": 0, "width": 200, "height": 100, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#e7f5ff", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": { "type": 3 }, "seed": 1570621945, "version": 1, "versionNonce": 249103478, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "e5", "type": "text", "x": 156, "y": 41.25, "width": 0.0, "height": 17.5, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": null, "seed": 922121677, "version": 1, "versionNonce": 161042649, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "", "originalText": "", "fontSize": 14, "fontFamily": 1, "textAlign": "left", "verticalAlign": "top", "containerId": null, "lineHeight": 1.25, "baseline": 12, "autoResize": true }, { "id": "e6", "type": "text", "x": 160, "y": 8, "width": 143.0, "height": 65.0, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": null, "seed": 369140571, "version": 1, "versionNonce": 1862494043, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "W13 [1536, 2048]\n= W_gate [768, 2048]\n+ W_up [768, 2048]\nstored as one matrix", "originalText": "W13 [1536, 2048]\n= W_gate [768, 2048]\n+ W_up [768, 2048]\nstored as one matrix", "fontSize": 13, "fontFamily": 1, "textAlign": "left", "verticalAlign": "top", "containerId": null, "lineHeight": 1.25, "baseline": 11, "autoResize": true }, { "id": "e7", "type": "arrow", "x": 350, "y": 50, "width": 35, "height": 0, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": { "type": 2 }, "seed": 1796035740, "version": 1, "versionNonce": 300026768, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 35, 0 ] ], "lastCommittedPoint": null, "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "e8", "type": "rectangle", "x": 385, "y": 30, "width": 100, "height": 40, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#e9ecef", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": { "type": 3 }, "seed": 1033639717, "version": 1, "versionNonce": 389609434, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "e9", "type": "text", "x": 391, "y": 41.25, "width": 61.60000000000001, "height": 17.5, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": null, "seed": 1823296039, "version": 1, "versionNonce": 253877687, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "h [1536]", "originalText": "h [1536]", "fontSize": 14, "fontFamily": 1, "textAlign": "left", "verticalAlign": "top", "containerId": null, "lineHeight": 1.25, "baseline": 12, "autoResize": true }, { "id": "e10", "type": "arrow", "x": 485, "y": 50, "width": 30, "height": 0, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": { "type": 2 }, "seed": 531725348, "version": 1, "versionNonce": 958804058, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 30, 0 ] ], "lastCommittedPoint": null, "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "e11", "type": "rectangle", "x": 515, "y": 15, "width": 120, "height": 70, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#ebfbee", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": { "type": 3 }, "seed": 265695474, "version": 1, "versionNonce": 1703729685, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "e12", "type": "text", "x": 521, "y": 41.25, "width": 0.0, "height": 17.5, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": null, "seed": 212984477, "version": 1, "versionNonce": 949539217, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "", "originalText": "", "fontSize": 14, "fontFamily": 1, "textAlign": "left", "verticalAlign": "top", "containerId": null, "lineHeight": 1.25, "baseline": 12, "autoResize": true }, { "id": "e13", "type": "text", "x": 523, "y": 20, "width": 92.95, "height": 48.75, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": null, "seed": 200071089, "version": 1, "versionNonce": 571981486, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "SiLU(h[:768])\n x h[768:]\n= a [768]", "originalText": "SiLU(h[:768])\n x h[768:]\n= a [768]", "fontSize": 13, "fontFamily": 1, "textAlign": "left", "verticalAlign": "top", "containerId": null, "lineHeight": 1.25, "baseline": 11, "autoResize": true }, { "id": "e14", "type": "arrow", "x": 635, "y": 50, "width": 30, "height": 0, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": { "type": 2 }, "seed": 1243862423, "version": 1, "versionNonce": 1800188483, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 30, 0 ] ], "lastCommittedPoint": null, "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "e15", "type": "rectangle", "x": 665, "y": 15, "width": 110, "height": 70, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#e7f5ff", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": { "type": 3 }, "seed": 619570853, "version": 1, "versionNonce": 505913793, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "e16", "type": "text", "x": 671, "y": 41.25, "width": 0.0, "height": 17.5, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": null, "seed": 1324919353, "version": 1, "versionNonce": 776213900, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "", "originalText": "", "fontSize": 14, "fontFamily": 1, "textAlign": "left", "verticalAlign": "top", "containerId": null, "lineHeight": 1.25, "baseline": 12, "autoResize": true }, { "id": "e17", "type": "text", "x": 673, "y": 22, "width": 78.65, "height": 32.5, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": null, "seed": 442620899, "version": 1, "versionNonce": 806899910, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "W2\n[2048, 768]", "originalText": "W2\n[2048, 768]", "fontSize": 13, "fontFamily": 1, "textAlign": "left", "verticalAlign": "top", "containerId": null, "lineHeight": 1.25, "baseline": 11, "autoResize": true }, { "id": "e18", "type": "arrow", "x": 775, "y": 50, "width": 30, "height": 0, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": { "type": 2 }, "seed": 1599435268, "version": 1, "versionNonce": 418461139, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 30, 0 ] ], "lastCommittedPoint": null, "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "e19", "type": "rectangle", "x": 805, "y": 30, "width": 85, "height": 40, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#e9ecef", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": { "type": 3 }, "seed": 269676600, "version": 1, "versionNonce": 255985077, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "e20", "type": "text", "x": 811, "y": 41.25, "width": 61.60000000000001, "height": 17.5, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": null, "seed": 884585952, "version": 1, "versionNonce": 2132084005, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "y [2048]", "originalText": "y [2048]", "fontSize": 14, "fontFamily": 1, "textAlign": "left", "verticalAlign": "top", "containerId": null, "lineHeight": 1.25, "baseline": 12, "autoResize": true }, { "id": "e21", "type": "text", "x": 0, "y": 120, "width": 762.3000000000001, "height": 17.5, "angle": 0, "strokeColor": "#1971c2", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": null, "seed": 1836494975, "version": 1, "versionNonce": 1349251824, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "Weights per expert: 1536 x 2048 + 2048 x 768 = 4,718,592. 128 experts: 604M weights in one layer.", "originalText": "Weights per expert: 1536 x 2048 + 2048 x 768 = 4,718,592. 128 experts: 604M weights in one layer.", "fontSize": 14, "fontFamily": 1, "textAlign": "left", "verticalAlign": "top", "containerId": null, "lineHeight": 1.25, "baseline": 12, "autoResize": true } ], "appState": { "viewBackgroundColor": "#fdf8f6", "gridSize": null }, "files": {} }