{ "type": "excalidraw", "version": 2, "source": "https://excalidraw.com", "elements": [ { "id": "el0051", "type": "text", "x": 20, "y": 10, "width": 1101.1000000000001, "height": 27.5, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": null, "seed": 108344815, "version": 1, "versionNonce": 765355855, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "Expert parallelism: dispatch, expert compute, combine (4 ranks, 8 experts, top-2 routing)", "fontSize": 22, "fontFamily": 1, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "Expert parallelism: dispatch, expert compute, combine (4 ranks, 8 experts, top-2 routing)", "autoResize": true, "lineHeight": 1.25 }, { "id": "el0052", "type": "text", "x": 20, "y": 44, "width": 854.7, "height": 17.5, "angle": 0, "strokeColor": "#2f9e44", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": null, "seed": 193993377, "version": 1, "versionNonce": 597176411, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "each expert stays whole on one rank; tokens travel to their experts and back with two all-to-alls per MoE layer", "fontSize": 14, "fontFamily": 1, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "each expert stays whole on one rank; tokens travel to their experts and back with two all-to-alls per MoE layer", "autoResize": true, "lineHeight": 1.25 }, { "id": "el0053", "type": "text", "x": 20, "y": 100, "width": 57.2, "height": 32.5, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": null, "seed": 867838003, "version": 1, "versionNonce": 1747160821, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "tokens\n+ router", "fontSize": 13, "fontFamily": 1, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "tokens\n+ router", "autoResize": true, "lineHeight": 1.25 }, { "id": "el0054", "type": "text", "x": 20, "y": 215, "width": 71.5, "height": 32.5, "angle": 0, "strokeColor": "#e03131", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": null, "seed": 383477192, "version": 1, "versionNonce": 370093381, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "dispatch\nall-to-all", "fontSize": 13, "fontFamily": 1, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "dispatch\nall-to-all", "autoResize": true, "lineHeight": 1.25 }, { "id": "el0055", "type": "text", "x": 20, "y": 300, "width": 71.5, "height": 32.5, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": null, "seed": 1089823834, "version": 1, "versionNonce": 1906694893, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "experts\n(resident)", "fontSize": 13, "fontFamily": 1, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "experts\n(resident)", "autoResize": true, "lineHeight": 1.25 }, { "id": "el0056", "type": "text", "x": 20, "y": 400, "width": 71.5, "height": 32.5, "angle": 0, "strokeColor": "#e03131", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": null, "seed": 267561757, "version": 1, "versionNonce": 351978998, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "combine\nall-to-all", "fontSize": 13, "fontFamily": 1, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "combine\nall-to-all", "autoResize": true, "lineHeight": 1.25 }, { "id": "el0057", "type": "text", "x": 20, "y": 480, "width": 50.050000000000004, "height": 16.25, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": null, "seed": 236823979, "version": 1, "versionNonce": 938757965, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "outputs", "fontSize": 13, "fontFamily": 1, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "outputs", "autoResize": true, "lineHeight": 1.25 }, { "id": "el0058", "type": "rectangle", "x": 120, "y": 90, "width": 180, "height": 60, "angle": 0, "strokeColor": "#1971c2", "backgroundColor": "#a5d8ff", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": { "type": 3 }, "seed": 1157887063, "version": 1, "versionNonce": 660254271, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "el0059", "type": "text", "x": 145.64999999999998, "y": 97, "width": 128.70000000000002, "height": 16.25, "angle": 0, "strokeColor": "#1971c2", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": null, "seed": 1475439159, "version": 1, "versionNonce": 851413501, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "rank 0: its tokens", "fontSize": 13, "fontFamily": 1, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "rank 0: its tokens", "autoResize": true, "lineHeight": 1.25 }, { "id": "el0060", "type": "text", "x": 134.1, "y": 118, "width": 151.8, "height": 15.0, "angle": 0, "strokeColor": "#1971c2", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": null, "seed": 1299647364, "version": 1, "versionNonce": 2089612784, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "token t: top-2 = e1, e5", "fontSize": 12, "fontFamily": 1, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "token t: top-2 = e1, e5", "autoResize": true, "lineHeight": 1.25 }, { "id": "el0061", "type": "rectangle", "x": 120, "y": 290, "width": 85, "height": 60, "angle": 0, "strokeColor": "#1971c2", "backgroundColor": "#a5d8ff", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": { "type": 3 }, "seed": 1471345650, "version": 1, "versionNonce": 539317727, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "el0062", "type": "rectangle", "x": 215, "y": 290, "width": 85, "height": 60, "angle": 0, "strokeColor": "#1971c2", "backgroundColor": "#a5d8ff", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": { "type": 3 }, "seed": 1150522804, "version": 1, "versionNonce": 1175332278, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "el0063", "type": "text", "x": 133.4, "y": 308, "width": 57.2, "height": 16.25, "angle": 0, "strokeColor": "#1971c2", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": null, "seed": 549238221, "version": 1, "versionNonce": 1810235514, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "expert 0", "fontSize": 13, "fontFamily": 1, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "expert 0", "autoResize": true, "lineHeight": 1.25 }, { "id": "el0064", "type": "text", "x": 228.4, "y": 308, "width": 57.2, "height": 16.25, "angle": 0, "strokeColor": "#1971c2", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": null, "seed": 408464828, "version": 1, "versionNonce": 285817019, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "expert 1", "fontSize": 13, "fontFamily": 1, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "expert 1", "autoResize": true, "lineHeight": 1.25 }, { "id": "el0065", "type": "text", "x": 140.425, "y": 354, "width": 139.15, "height": 27.5, "angle": 0, "strokeColor": "#868e96", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": null, "seed": 791491766, "version": 1, "versionNonce": 879384617, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "grouped GEMM on\nwhatever tokens arrived", "fontSize": 11, "fontFamily": 1, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "grouped GEMM on\nwhatever tokens arrived", "autoResize": true, "lineHeight": 1.25 }, { "id": "el0066", "type": "rectangle", "x": 120, "y": 470, "width": 180, "height": 50, "angle": 0, "strokeColor": "#1971c2", "backgroundColor": "#a5d8ff", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": { "type": 3 }, "seed": 1354973391, "version": 1, "versionNonce": 1302409011, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "el0067", "type": "text", "x": 127.5, "y": 479, "width": 165.0, "height": 30.0, "angle": 0, "strokeColor": "#1971c2", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": null, "seed": 414163140, "version": 1, "versionNonce": 279572640, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "rank 0: gate-weighted sum\nof its tokens' 2 outputs", "fontSize": 12, "fontFamily": 1, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "rank 0: gate-weighted sum\nof its tokens' 2 outputs", "autoResize": true, "lineHeight": 1.25 }, { "id": "el0068", "type": "rectangle", "x": 340, "y": 90, "width": 180, "height": 60, "angle": 0, "strokeColor": "#e8590c", "backgroundColor": "#ffd8a8", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": { "type": 3 }, "seed": 1010880910, "version": 1, "versionNonce": 929963159, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "el0069", "type": "text", "x": 365.65, "y": 97, "width": 128.70000000000002, "height": 16.25, "angle": 0, "strokeColor": "#e8590c", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": null, "seed": 537208337, "version": 1, "versionNonce": 1413912340, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "rank 1: its tokens", "fontSize": 13, "fontFamily": 1, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "rank 1: its tokens", "autoResize": true, "lineHeight": 1.25 }, { "id": "el0070", "type": "text", "x": 354.1, "y": 118, "width": 151.8, "height": 15.0, "angle": 0, "strokeColor": "#e8590c", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": null, "seed": 2052024545, "version": 1, "versionNonce": 1831551409, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "token t: top-2 = e2, e6", "fontSize": 12, "fontFamily": 1, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "token t: top-2 = e2, e6", "autoResize": true, "lineHeight": 1.25 }, { "id": "el0071", "type": "rectangle", "x": 340, "y": 290, "width": 85, "height": 60, "angle": 0, "strokeColor": "#e8590c", "backgroundColor": "#ffd8a8", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": { "type": 3 }, "seed": 785321486, "version": 1, "versionNonce": 1620369149, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "el0072", "type": "rectangle", "x": 435, "y": 290, "width": 85, "height": 60, "angle": 0, "strokeColor": "#e8590c", "backgroundColor": "#ffd8a8", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": { "type": 3 }, "seed": 1386847314, "version": 1, "versionNonce": 204931861, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "el0073", "type": "text", "x": 353.4, "y": 308, "width": 57.2, "height": 16.25, "angle": 0, "strokeColor": "#e8590c", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": null, "seed": 1068666995, "version": 1, "versionNonce": 1622203297, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "expert 2", "fontSize": 13, "fontFamily": 1, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "expert 2", "autoResize": true, "lineHeight": 1.25 }, { "id": "el0074", "type": "text", "x": 448.4, "y": 308, "width": 57.2, "height": 16.25, "angle": 0, "strokeColor": "#e8590c", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": null, "seed": 935543391, "version": 1, "versionNonce": 1175909006, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "expert 3", "fontSize": 13, "fontFamily": 1, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "expert 3", "autoResize": true, "lineHeight": 1.25 }, { "id": "el0075", "type": "text", "x": 360.425, "y": 354, "width": 139.15, "height": 27.5, "angle": 0, "strokeColor": "#868e96", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": null, "seed": 625414597, "version": 1, "versionNonce": 1379469920, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "grouped GEMM on\nwhatever tokens arrived", "fontSize": 11, "fontFamily": 1, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "grouped GEMM on\nwhatever tokens arrived", "autoResize": true, "lineHeight": 1.25 }, { "id": "el0076", "type": "rectangle", "x": 340, "y": 470, "width": 180, "height": 50, "angle": 0, "strokeColor": "#e8590c", "backgroundColor": "#ffd8a8", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": { "type": 3 }, "seed": 2146575530, "version": 1, "versionNonce": 432904614, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "el0077", "type": "text", "x": 347.5, "y": 479, "width": 165.0, "height": 30.0, "angle": 0, "strokeColor": "#e8590c", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": null, "seed": 1074265173, "version": 1, "versionNonce": 1055762953, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "rank 1: gate-weighted sum\nof its tokens' 2 outputs", "fontSize": 12, "fontFamily": 1, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "rank 1: gate-weighted sum\nof its tokens' 2 outputs", "autoResize": true, "lineHeight": 1.25 }, { "id": "el0078", "type": "rectangle", "x": 560, "y": 90, "width": 180, "height": 60, "angle": 0, "strokeColor": "#2f9e44", "backgroundColor": "#b2f2bb", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": { "type": 3 }, "seed": 911771569, "version": 1, "versionNonce": 959274523, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "el0079", "type": "text", "x": 585.65, "y": 97, "width": 128.70000000000002, "height": 16.25, "angle": 0, "strokeColor": "#2f9e44", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": null, "seed": 1703657719, "version": 1, "versionNonce": 2086116197, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "rank 2: its tokens", "fontSize": 13, "fontFamily": 1, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "rank 2: its tokens", "autoResize": true, "lineHeight": 1.25 }, { "id": "el0080", "type": "text", "x": 574.1, "y": 118, "width": 151.8, "height": 15.0, "angle": 0, "strokeColor": "#2f9e44", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": null, "seed": 2048067157, "version": 1, "versionNonce": 967907247, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "token t: top-2 = e3, e0", "fontSize": 12, "fontFamily": 1, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "token t: top-2 = e3, e0", "autoResize": true, "lineHeight": 1.25 }, { "id": "el0081", "type": "rectangle", "x": 560, "y": 290, "width": 85, "height": 60, "angle": 0, "strokeColor": "#2f9e44", "backgroundColor": "#b2f2bb", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": { "type": 3 }, "seed": 2019010751, "version": 1, "versionNonce": 1627050776, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "el0082", "type": "rectangle", "x": 655, "y": 290, "width": 85, "height": 60, "angle": 0, "strokeColor": "#2f9e44", "backgroundColor": "#b2f2bb", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": { "type": 3 }, "seed": 2127000273, "version": 1, "versionNonce": 1564573516, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "el0083", "type": "text", "x": 573.4, "y": 308, "width": 57.2, "height": 16.25, "angle": 0, "strokeColor": "#2f9e44", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": null, "seed": 2127065519, "version": 1, "versionNonce": 1529487537, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "expert 4", "fontSize": 13, "fontFamily": 1, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "expert 4", "autoResize": true, "lineHeight": 1.25 }, { "id": "el0084", "type": "text", "x": 668.4, "y": 308, "width": 57.2, "height": 16.25, "angle": 0, "strokeColor": "#2f9e44", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": null, "seed": 827365797, "version": 1, "versionNonce": 415070, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "expert 5", "fontSize": 13, "fontFamily": 1, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "expert 5", "autoResize": true, "lineHeight": 1.25 }, { "id": "el0085", "type": "text", "x": 580.425, "y": 354, "width": 139.15, "height": 27.5, "angle": 0, "strokeColor": "#868e96", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": null, "seed": 2111629093, "version": 1, "versionNonce": 2094855435, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "grouped GEMM on\nwhatever tokens arrived", "fontSize": 11, "fontFamily": 1, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "grouped GEMM on\nwhatever tokens arrived", "autoResize": true, "lineHeight": 1.25 }, { "id": "el0086", "type": "rectangle", "x": 560, "y": 470, "width": 180, "height": 50, "angle": 0, "strokeColor": "#2f9e44", "backgroundColor": "#b2f2bb", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": { "type": 3 }, "seed": 1800232337, "version": 1, "versionNonce": 1156744761, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "el0087", "type": "text", "x": 567.5, "y": 479, "width": 165.0, "height": 30.0, "angle": 0, "strokeColor": "#2f9e44", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": null, "seed": 790433298, "version": 1, "versionNonce": 472789488, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "rank 2: gate-weighted sum\nof its tokens' 2 outputs", "fontSize": 12, "fontFamily": 1, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "rank 2: gate-weighted sum\nof its tokens' 2 outputs", "autoResize": true, "lineHeight": 1.25 }, { "id": "el0088", "type": "rectangle", "x": 780, "y": 90, "width": 180, "height": 60, "angle": 0, "strokeColor": "#7048e8", "backgroundColor": "#d0bfff", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": { "type": 3 }, "seed": 135580670, "version": 1, "versionNonce": 805669125, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "el0089", "type": "text", "x": 805.65, "y": 97, "width": 128.70000000000002, "height": 16.25, "angle": 0, "strokeColor": "#7048e8", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": null, "seed": 356781762, "version": 1, "versionNonce": 458461442, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "rank 3: its tokens", "fontSize": 13, "fontFamily": 1, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "rank 3: its tokens", "autoResize": true, "lineHeight": 1.25 }, { "id": "el0090", "type": "text", "x": 794.1, "y": 118, "width": 151.8, "height": 15.0, "angle": 0, "strokeColor": "#7048e8", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": null, "seed": 77849033, "version": 1, "versionNonce": 534571061, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "token t: top-2 = e7, e4", "fontSize": 12, "fontFamily": 1, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "token t: top-2 = e7, e4", "autoResize": true, "lineHeight": 1.25 }, { "id": "el0091", "type": "rectangle", "x": 780, "y": 290, "width": 85, "height": 60, "angle": 0, "strokeColor": "#7048e8", "backgroundColor": "#d0bfff", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": { "type": 3 }, "seed": 493234676, "version": 1, "versionNonce": 254292498, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "el0092", "type": "rectangle", "x": 875, "y": 290, "width": 85, "height": 60, "angle": 0, "strokeColor": "#7048e8", "backgroundColor": "#d0bfff", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": { "type": 3 }, "seed": 170972131, "version": 1, "versionNonce": 209471221, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "el0093", "type": "text", "x": 793.4, "y": 308, "width": 57.2, "height": 16.25, "angle": 0, "strokeColor": "#7048e8", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": null, "seed": 6813519, "version": 1, "versionNonce": 41697135, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "expert 6", "fontSize": 13, "fontFamily": 1, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "expert 6", "autoResize": true, "lineHeight": 1.25 }, { "id": "el0094", "type": "text", "x": 888.4, "y": 308, "width": 57.2, "height": 16.25, "angle": 0, "strokeColor": "#7048e8", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": null, "seed": 935575578, "version": 1, "versionNonce": 828250389, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "expert 7", "fontSize": 13, "fontFamily": 1, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "expert 7", "autoResize": true, "lineHeight": 1.25 }, { "id": "el0095", "type": "text", "x": 800.425, "y": 354, "width": 139.15, "height": 27.5, "angle": 0, "strokeColor": "#868e96", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": null, "seed": 210544176, "version": 1, "versionNonce": 1533253232, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "grouped GEMM on\nwhatever tokens arrived", "fontSize": 11, "fontFamily": 1, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "grouped GEMM on\nwhatever tokens arrived", "autoResize": true, "lineHeight": 1.25 }, { "id": "el0096", "type": "rectangle", "x": 780, "y": 470, "width": 180, "height": 50, "angle": 0, "strokeColor": "#7048e8", "backgroundColor": "#d0bfff", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": { "type": 3 }, "seed": 951692618, "version": 1, "versionNonce": 1177721682, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "el0097", "type": "text", "x": 787.5, "y": 479, "width": 165.0, "height": 30.0, "angle": 0, "strokeColor": "#7048e8", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": null, "seed": 1071074677, "version": 1, "versionNonce": 1378179762, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "rank 3: gate-weighted sum\nof its tokens' 2 outputs", "fontSize": 12, "fontFamily": 1, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "rank 3: gate-weighted sum\nof its tokens' 2 outputs", "autoResize": true, "lineHeight": 1.25 }, { "id": "el0098", "type": "arrow", "x": 210, "y": 150, "width": 47, "height": 140, "angle": 0, "strokeColor": "#e03131", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": { "type": 2 }, "seed": 1798694399, "version": 1, "versionNonce": 1804655303, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 47, 140 ] ], "lastCommittedPoint": null, "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "el0099", "type": "arrow", "x": 257, "y": 350, "width": 47, "height": 120, "angle": 0, "strokeColor": "#868e96", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "dashed", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": { "type": 2 }, "seed": 286005641, "version": 1, "versionNonce": 490468705, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "points": [ [ 0, 0 ], [ -47, 120 ] ], "lastCommittedPoint": null, "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "el0100", "type": "arrow", "x": 210, "y": 150, "width": 487, "height": 140, "angle": 0, "strokeColor": "#e03131", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": { "type": 2 }, "seed": 266287815, "version": 1, "versionNonce": 2131283585, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 487, 140 ] ], "lastCommittedPoint": null, "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "el0101", "type": "arrow", "x": 697, "y": 350, "width": 487, "height": 120, "angle": 0, "strokeColor": "#868e96", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "dashed", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": { "type": 2 }, "seed": 415189450, "version": 1, "versionNonce": 1382371368, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "points": [ [ 0, 0 ], [ -487, 120 ] ], "lastCommittedPoint": null, "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "el0102", "type": "arrow", "x": 430, "y": 150, "width": 48, "height": 140, "angle": 0, "strokeColor": "#e03131", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": { "type": 2 }, "seed": 1884197971, "version": 1, "versionNonce": 1227142488, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "points": [ [ 0, 0 ], [ -48, 140 ] ], "lastCommittedPoint": null, "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "el0103", "type": "arrow", "x": 382, "y": 350, "width": 48, "height": 120, "angle": 0, "strokeColor": "#868e96", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "dashed", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": { "type": 2 }, "seed": 560825436, "version": 1, "versionNonce": 1794657683, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 48, 120 ] ], "lastCommittedPoint": null, "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "el0104", "type": "arrow", "x": 430, "y": 150, "width": 392, "height": 140, "angle": 0, "strokeColor": "#e03131", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": { "type": 2 }, "seed": 1585039921, "version": 1, "versionNonce": 1624556044, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 392, 140 ] ], "lastCommittedPoint": null, "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "el0105", "type": "arrow", "x": 822, "y": 350, "width": 392, "height": 120, "angle": 0, "strokeColor": "#868e96", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "dashed", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": { "type": 2 }, "seed": 1646179714, "version": 1, "versionNonce": 250369958, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "points": [ [ 0, 0 ], [ -392, 120 ] ], "lastCommittedPoint": null, "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "el0106", "type": "arrow", "x": 650, "y": 150, "width": 173, "height": 140, "angle": 0, "strokeColor": "#e03131", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": { "type": 2 }, "seed": 449726146, "version": 1, "versionNonce": 243426068, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "points": [ [ 0, 0 ], [ -173, 140 ] ], "lastCommittedPoint": null, "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "el0107", "type": "arrow", "x": 477, "y": 350, "width": 173, "height": 120, "angle": 0, "strokeColor": "#868e96", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "dashed", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": { "type": 2 }, "seed": 586570132, "version": 1, "versionNonce": 487795704, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 173, 120 ] ], "lastCommittedPoint": null, "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "el0108", "type": "arrow", "x": 650, "y": 150, "width": 488, "height": 140, "angle": 0, "strokeColor": "#e03131", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": { "type": 2 }, "seed": 1372922169, "version": 1, "versionNonce": 1228492814, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "points": [ [ 0, 0 ], [ -488, 140 ] ], "lastCommittedPoint": null, "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "el0109", "type": "arrow", "x": 162, "y": 350, "width": 488, "height": 120, "angle": 0, "strokeColor": "#868e96", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "dashed", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": { "type": 2 }, "seed": 295340501, "version": 1, "versionNonce": 1281314441, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 488, 120 ] ], "lastCommittedPoint": null, "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "el0110", "type": "arrow", "x": 870, "y": 150, "width": 47, "height": 140, "angle": 0, "strokeColor": "#e03131", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": { "type": 2 }, "seed": 577362405, "version": 1, "versionNonce": 1461266755, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 47, 140 ] ], "lastCommittedPoint": null, "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "el0111", "type": "arrow", "x": 917, "y": 350, "width": 47, "height": 120, "angle": 0, "strokeColor": "#868e96", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "dashed", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": { "type": 2 }, "seed": 274145508, "version": 1, "versionNonce": 1103162939, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "points": [ [ 0, 0 ], [ -47, 120 ] ], "lastCommittedPoint": null, "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "el0112", "type": "arrow", "x": 870, "y": 150, "width": 268, "height": 140, "angle": 0, "strokeColor": "#e03131", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": { "type": 2 }, "seed": 1583203552, "version": 1, "versionNonce": 1536412897, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "points": [ [ 0, 0 ], [ -268, 140 ] ], "lastCommittedPoint": null, "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "el0113", "type": "arrow", "x": 602, "y": 350, "width": 268, "height": 120, "angle": 0, "strokeColor": "#868e96", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "dashed", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": { "type": 2 }, "seed": 1225333559, "version": 1, "versionNonce": 1201622374, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 268, 120 ] ], "lastCommittedPoint": null, "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "el0114", "type": "text", "x": 282.59999999999997, "y": 208, "width": 514.8000000000001, "height": 15.0, "angle": 0, "strokeColor": "#e03131", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": null, "seed": 1691078081, "version": 1, "versionNonce": 2087208697, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "every rank sends to every rank: one all-to-all carrying k x h values per token", "fontSize": 12, "fontFamily": 1, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "every rank sends to every rank: one all-to-all carrying k x h values per token", "autoResize": true, "lineHeight": 1.25 }, { "id": "el0115", "type": "text", "x": 345.29999999999995, "y": 424, "width": 389.40000000000003, "height": 15.0, "angle": 0, "strokeColor": "#868e96", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": null, "seed": 1310166497, "version": 1, "versionNonce": 590551059, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "and one back (dashed), summed with the router gates at home", "fontSize": 12, "fontFamily": 1, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "and one back (dashed), summed with the router gates at home", "autoResize": true, "lineHeight": 1.25 }, { "id": "el0116", "type": "text", "x": 20, "y": 545, "width": 1194.0500000000002, "height": 16.25, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": null, "seed": 418802349, "version": 1, "versionNonce": 1543475404, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "per MoE layer per token, DeepSeek-V3 (h = 7168, k = 8, bf16): 8 x 7168 x 2 = 115 KB out and 115 KB back; the 64-way EP group spans 8 nodes, so this crosses InfiniBand.", "fontSize": 13, "fontFamily": 1, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "per MoE layer per token, DeepSeek-V3 (h = 7168, k = 8, bf16): 8 x 7168 x 2 = 115 KB out and 115 KB back; the 64-way EP group spans 8 nodes, so this crosses InfiniBand.", "autoResize": true, "lineHeight": 1.25 }, { "id": "el0117", "type": "text", "x": 20, "y": 570, "width": 922.35, "height": 16.25, "angle": 0, "strokeColor": "#2f9e44", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": null, "seed": 1416668992, "version": 1, "versionNonce": 1611823290, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "load balance: capacity factor (drop overflow), auxiliary loss, or a per-expert routing bias; the step time is the max over ranks.", "fontSize": 13, "fontFamily": 1, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "load balance: capacity factor (drop overflow), auxiliary loss, or a per-expert routing bias; the step time is the max over ranks.", "autoResize": true, "lineHeight": 1.25 } ], "appState": { "gridSize": 20, "gridStep": 5, "gridModeEnabled": false, "viewBackgroundColor": "#ffffff" }, "files": {} }