{ "type": "excalidraw", "version": 2, "source": "https://excalidraw.com", "elements": [ { "id": "e1", "type": "rectangle", "x": 0, "y": 0, "width": 400, "height": 36, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#fff3bf", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": { "type": 3 }, "seed": 1856188578, "version": 1, "versionNonce": 2126015576, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "e2", "type": "text", "x": 6, "y": 9.25, "width": 223.3, "height": 17.5, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": null, "seed": 1688291830, "version": 1, "versionNonce": 1321918115, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "x [M, 2048] (token order)", "originalText": "x [M, 2048] (token order)", "fontSize": 14, "fontFamily": 1, "textAlign": "left", "verticalAlign": "top", "containerId": null, "lineHeight": 1.25, "baseline": 12, "autoResize": true }, { "id": "e3", "type": "arrow", "x": 40, "y": 36, "width": 0, "height": 22, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": { "type": 2 }, "seed": 924195863, "version": 1, "versionNonce": 985979547, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 0, 22 ] ], "lastCommittedPoint": null, "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "e4", "type": "rectangle", "x": 0, "y": 58, "width": 520, "height": 36, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#e7f5ff", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": { "type": 3 }, "seed": 1471905176, "version": 1, "versionNonce": 853085646, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "e5", "type": "text", "x": 6, "y": 67.25, "width": 315.70000000000005, "height": 17.5, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": null, "seed": 600087727, "version": 1, "versionNonce": 1738163422, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "GEMM 1 read row p // top_k, write row p", "originalText": "GEMM 1 read row p // top_k, write row p", "fontSize": 14, "fontFamily": 1, "textAlign": "left", "verticalAlign": "top", "containerId": null, "lineHeight": 1.25, "baseline": 12, "autoResize": true }, { "id": "e6", "type": "arrow", "x": 40, "y": 94, "width": 0, "height": 22, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": { "type": 2 }, "seed": 1492725227, "version": 1, "versionNonce": 233596964, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 0, 22 ] ], "lastCommittedPoint": null, "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "e7", "type": "rectangle", "x": 0, "y": 116, "width": 400, "height": 36, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#e5dbff", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": { "type": 3 }, "seed": 557566592, "version": 1, "versionNonce": 61225319, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "e8", "type": "text", "x": 6, "y": 125.25, "width": 315.70000000000005, "height": 17.5, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": null, "seed": 303752167, "version": 1, "versionNonce": 1097767345, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "cache1 [M x top_k, 1536] (pair order)", "originalText": "cache1 [M x top_k, 1536] (pair order)", "fontSize": 14, "fontFamily": 1, "textAlign": "left", "verticalAlign": "top", "containerId": null, "lineHeight": 1.25, "baseline": 12, "autoResize": true }, { "id": "e9", "type": "arrow", "x": 40, "y": 152, "width": 0, "height": 22, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": { "type": 2 }, "seed": 1850017270, "version": 1, "versionNonce": 701138478, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 0, 22 ] ], "lastCommittedPoint": null, "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "e10", "type": "rectangle", "x": 0, "y": 174, "width": 520, "height": 36, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#ebfbee", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": { "type": 3 }, "seed": 237945867, "version": 1, "versionNonce": 362859751, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "e11", "type": "text", "x": 6, "y": 183.25, "width": 277.20000000000005, "height": 17.5, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": null, "seed": 1635874816, "version": 1, "versionNonce": 1210894223, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "SiLU x mul read row p, write row p", "originalText": "SiLU x mul read row p, write row p", "fontSize": 14, "fontFamily": 1, "textAlign": "left", "verticalAlign": "top", "containerId": null, "lineHeight": 1.25, "baseline": 12, "autoResize": true }, { "id": "e12", "type": "arrow", "x": 40, "y": 210, "width": 0, "height": 22, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": { "type": 2 }, "seed": 1040296615, "version": 1, "versionNonce": 1258676655, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 0, 22 ] ], "lastCommittedPoint": null, "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "e13", "type": "rectangle", "x": 0, "y": 232, "width": 400, "height": 36, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#e5dbff", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": { "type": 3 }, "seed": 194293563, "version": 1, "versionNonce": 1973335386, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "e14", "type": "text", "x": 6, "y": 241.25, "width": 308.0, "height": 17.5, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": null, "seed": 796080902, "version": 1, "versionNonce": 676598821, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "cache2 [M x top_k, 768] (pair order)", "originalText": "cache2 [M x top_k, 768] (pair order)", "fontSize": 14, "fontFamily": 1, "textAlign": "left", "verticalAlign": "top", "containerId": null, "lineHeight": 1.25, "baseline": 12, "autoResize": true }, { "id": "e15", "type": "arrow", "x": 40, "y": 268, "width": 0, "height": 22, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": { "type": 2 }, "seed": 1155503872, "version": 1, "versionNonce": 1914802141, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 0, 22 ] ], "lastCommittedPoint": null, "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "e16", "type": "rectangle", "x": 0, "y": 290, "width": 520, "height": 36, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#e7f5ff", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": { "type": 3 }, "seed": 15559427, "version": 1, "versionNonce": 1130620378, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "e17", "type": "text", "x": 6, "y": 299.25, "width": 392.70000000000005, "height": 17.5, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": null, "seed": 1563975175, "version": 1, "versionNonce": 1412727127, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "GEMM 2 read row p, write row p, x topk_weights[p]", "originalText": "GEMM 2 read row p, write row p, x topk_weights[p]", "fontSize": 14, "fontFamily": 1, "textAlign": "left", "verticalAlign": "top", "containerId": null, "lineHeight": 1.25, "baseline": 12, "autoResize": true }, { "id": "e18", "type": "arrow", "x": 40, "y": 326, "width": 0, "height": 22, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": { "type": 2 }, "seed": 1389567516, "version": 1, "versionNonce": 1049889717, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 0, 22 ] ], "lastCommittedPoint": null, "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "e19", "type": "rectangle", "x": 0, "y": 348, "width": 400, "height": 36, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#e5dbff", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": { "type": 3 }, "seed": 147947538, "version": 1, "versionNonce": 1329498207, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "e20", "type": "text", "x": 6, "y": 357.25, "width": 315.70000000000005, "height": 17.5, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": null, "seed": 935726745, "version": 1, "versionNonce": 1531516258, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "cache3 [M x top_k, 2048] (pair order)", "originalText": "cache3 [M x top_k, 2048] (pair order)", "fontSize": 14, "fontFamily": 1, "textAlign": "left", "verticalAlign": "top", "containerId": null, "lineHeight": 1.25, "baseline": 12, "autoResize": true }, { "id": "e21", "type": "arrow", "x": 40, "y": 384, "width": 0, "height": 22, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": { "type": 2 }, "seed": 785798162, "version": 1, "versionNonce": 4590954, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 0, 22 ] ], "lastCommittedPoint": null, "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "e22", "type": "rectangle", "x": 0, "y": 406, "width": 520, "height": 36, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#e9ecef", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": { "type": 3 }, "seed": 1440243342, "version": 1, "versionNonce": 1639073805, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "e23", "type": "text", "x": 6, "y": 415.25, "width": 415.8, "height": 17.5, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": null, "seed": 360307211, "version": 1, "versionNonce": 2038578867, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "moe_sum add rows t x top_k ... t x top_k + top_k - 1", "originalText": "moe_sum add rows t x top_k ... t x top_k + top_k - 1", "fontSize": 14, "fontFamily": 1, "textAlign": "left", "verticalAlign": "top", "containerId": null, "lineHeight": 1.25, "baseline": 12, "autoResize": true }, { "id": "e24", "type": "arrow", "x": 40, "y": 442, "width": 0, "height": 22, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": { "type": 2 }, "seed": 1197990395, "version": 1, "versionNonce": 863202765, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 0, 22 ] ], "lastCommittedPoint": null, "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "e25", "type": "rectangle", "x": 0, "y": 464, "width": 400, "height": 36, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#fff3bf", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": { "type": 3 }, "seed": 1065922394, "version": 1, "versionNonce": 21262380, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "e26", "type": "text", "x": 6, "y": 473.25, "width": 238.70000000000002, "height": 17.5, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": null, "seed": 390205077, "version": 1, "versionNonce": 1134595847, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "out [M, 2048] (token order)", "originalText": "out [M, 2048] (token order)", "fontSize": 14, "fontFamily": 1, "textAlign": "left", "verticalAlign": "top", "containerId": null, "lineHeight": 1.25, "baseline": 12, "autoResize": true }, { "id": "e27", "type": "text", "x": 570, "y": 10, "width": 264.0, "height": 18.75, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": null, "seed": 385487906, "version": 1, "versionNonce": 617896093, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "Two places change the row order:", "originalText": "Two places change the row order:", "fontSize": 15, "fontFamily": 1, "textAlign": "left", "verticalAlign": "top", "containerId": null, "lineHeight": 1.25, "baseline": 13, "autoResize": true }, { "id": "e28", "type": "text", "x": 570, "y": 40, "width": 269.5, "height": 140.0, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": null, "seed": 1715887401, "version": 1, "versionNonce": 178958210, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "GEMM 1: token order -> pair order,\n with p // top_k on the read side.\n\nmoe_sum: pair order -> token order,\n with the sum over top_k rows.\n\nEverything between them stays\nin pair order.", "originalText": "GEMM 1: token order -> pair order,\n with p // top_k on the read side.\n\nmoe_sum: pair order -> token order,\n with the sum over top_k rows.\n\nEverything between them stays\nin pair order.", "fontSize": 14, "fontFamily": 1, "textAlign": "left", "verticalAlign": "top", "containerId": null, "lineHeight": 1.25, "baseline": 12, "autoResize": true }, { "id": "e29", "type": "text", "x": 570, "y": 250, "width": 246.40000000000003, "height": 87.5, "angle": 0, "strokeColor": "#1971c2", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": null, "seed": 1692125396, "version": 1, "versionNonce": 96611648, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "GEMM 1 and GEMM 2 both use\nsorted_token_ids and expert_ids,\nso each block uses one expert's\nweights. GEMM 2 gets top_k = 1,\nso p // 1 = p.", "originalText": "GEMM 1 and GEMM 2 both use\nsorted_token_ids and expert_ids,\nso each block uses one expert's\nweights. GEMM 2 gets top_k = 1,\nso p // 1 = p.", "fontSize": 14, "fontFamily": 1, "textAlign": "left", "verticalAlign": "top", "containerId": null, "lineHeight": 1.25, "baseline": 12, "autoResize": true } ], "appState": { "viewBackgroundColor": "#fdf8f6", "gridSize": null }, "files": {} }