{"type": "excalidraw", "version": 2, "source": "https://excalidraw.com", "elements": [{"type": "text", "x": 20, "y": 10, "width": 1300, "height": 28, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0001", "roundness": null, "seed": 332163161, "version": 1, "versionNonce": 1039923490, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0001", "text": "Causal encoder-decoder: where the decoder's global KV comes from during prefill", "originalText": "Causal encoder-decoder: where the decoder's global KV comes from during prefill", "fontSize": 22, "fontFamily": 1, "textAlign": "left", "verticalAlign": "middle", "lineHeight": 1.25, "autoResize": false, "containerId": null}, {"type": "text", "x": 40, "y": 55, "width": 300, "height": 22, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0002", "roundness": null, "seed": 772116747, "version": 1, "versionNonce": 1073596140, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0002", "text": "prompt tokens (N)", "originalText": "prompt tokens (N)", "fontSize": 14, "fontFamily": 1, "textAlign": "left", "verticalAlign": "middle", "lineHeight": 1.25, "autoResize": false, "containerId": null}, {"type": "rectangle", "x": 40, "y": 80, "width": 36, "height": 26, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#e9ecef", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0003", "roundness": {"type": 3}, "seed": 1950521148, "version": 1, "versionNonce": 1691057261, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0003"}, {"type": "rectangle", "x": 82, "y": 80, "width": 36, "height": 26, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#e9ecef", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0004", "roundness": {"type": 3}, "seed": 1086180213, "version": 1, "versionNonce": 1578277432, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0004"}, {"type": "rectangle", "x": 124, "y": 80, "width": 36, "height": 26, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#e9ecef", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0005", "roundness": {"type": 3}, "seed": 1703554357, "version": 1, "versionNonce": 1504460393, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0005"}, {"type": "rectangle", "x": 166, "y": 80, "width": 36, "height": 26, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#e9ecef", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0006", "roundness": {"type": 3}, "seed": 1796231110, "version": 1, "versionNonce": 357601085, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0006"}, {"type": "rectangle", "x": 208, "y": 80, "width": 36, "height": 26, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#e9ecef", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0007", "roundness": {"type": 3}, "seed": 1612006853, "version": 1, "versionNonce": 1010451582, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0007"}, {"type": "rectangle", "x": 250, "y": 80, "width": 36, "height": 26, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#e9ecef", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0008", "roundness": {"type": 3}, "seed": 1773097240, "version": 1, "versionNonce": 689859745, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0008"}, {"type": "rectangle", "x": 292, "y": 80, "width": 36, "height": 26, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#e9ecef", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0009", "roundness": {"type": 3}, "seed": 1783575547, "version": 1, "versionNonce": 2077565418, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0009"}, {"type": "rectangle", "x": 334, "y": 80, "width": 36, "height": 26, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#e9ecef", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0010", "roundness": {"type": 3}, "seed": 670764889, "version": 1, "versionNonce": 1722504955, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0010"}, {"type": "text", "x": 376, "y": 80, "width": 40, "height": 26, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0011", "roundness": null, "seed": 641356317, "version": 1, "versionNonce": 698290297, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0011", "text": "...", "originalText": "...", "fontSize": 14, "fontFamily": 1, "textAlign": "center", "verticalAlign": "middle", "lineHeight": 1.25, "autoResize": false, "containerId": null}, {"type": "rectangle", "x": 40, "y": 130, "width": 340, "height": 220, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#b2f2bb", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0012", "roundness": {"type": 3}, "seed": 411718717, "version": 1, "versionNonce": 2138714820, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0012"}, {"type": "text", "x": 40, "y": 136, "width": 340, "height": 24, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0013", "roundness": null, "seed": 2076621967, "version": 1, "versionNonce": 1902900388, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0013", "text": "causal encoder: layers 0 to 19", "originalText": "causal encoder: layers 0 to 19", "fontSize": 15, "fontFamily": 1, "textAlign": "center", "verticalAlign": "middle", "lineHeight": 1.25, "autoResize": false, "containerId": null}, {"type": "text", "x": 40, "y": 162, "width": 340, "height": 60, "angle": 0, "strokeColor": "#495057", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0014", "roundness": null, "seed": 799340815, "version": 1, "versionNonce": 585338689, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0014", "text": "runs over all N prompt tokens\nglobal branch: Full at 2, 8, 14 (ratio 2), Reuse elsewhere\nwindow branch: every layer, own hidden state", "originalText": "runs over all N prompt tokens\nglobal branch: Full at 2, 8, 14 (ratio 2), Reuse elsewhere\nwindow branch: every layer, own hidden state", "fontSize": 11, "fontFamily": 1, "textAlign": "center", "verticalAlign": "middle", "lineHeight": 1.25, "autoResize": false, "containerId": null}, {"type": "rectangle", "x": 60, "y": 240, "width": 300, "height": 40, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#8ce99a", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0015", "roundness": {"type": 3}, "seed": 1148291136, "version": 1, "versionNonce": 855470886, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0015"}, {"type": "text", "x": 60, "y": 240, "width": 300, "height": 40, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0016", "roundness": null, "seed": 629449792, "version": 1, "versionNonce": 1352259829, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0016", "text": "H_20: final encoder hidden state", "originalText": "H_20: final encoder hidden state", "fontSize": 13, "fontFamily": 1, "textAlign": "center", "verticalAlign": "middle", "lineHeight": 1.25, "autoResize": false, "containerId": null}, {"type": "arrow", "x": 210, "y": 108, "width": 0, "height": 22, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0017", "roundness": null, "seed": 997743129, "version": 1, "versionNonce": 1270724628, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0017", "points": [[0, 0], [0, 22]], "startArrowhead": null, "endArrowhead": "arrow", "startBinding": null, "endBinding": null, "lastCommittedPoint": null, "elbowed": false}, {"type": "rectangle", "x": 430, "y": 240, "width": 320, "height": 40, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#fff3bf", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0018", "roundness": {"type": 3}, "seed": 1774683413, "version": 1, "versionNonce": 1147501514, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0018"}, {"type": "text", "x": 430, "y": 240, "width": 320, "height": 40, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0019", "roundness": null, "seed": 934605833, "version": 1, "versionNonce": 1319531301, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0019", "text": "C_l = H_20 W_KV_l,   Z_l = H_20 W_Z_l", "originalText": "C_l = H_20 W_KV_l,   Z_l = H_20 W_Z_l", "fontSize": 11, "fontFamily": 1, "textAlign": "center", "verticalAlign": "middle", "lineHeight": 1.25, "autoResize": false, "containerId": null}, {"type": "arrow", "x": 360, "y": 260, "width": 70, "height": 0, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0020", "roundness": null, "seed": 99320091, "version": 1, "versionNonce": 1151453400, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0020", "points": [[0, 0], [70, 0]], "startArrowhead": null, "endArrowhead": "arrow", "startBinding": null, "endBinding": null, "lastCommittedPoint": null, "elbowed": false}, {"type": "text", "x": 430, "y": 284, "width": 320, "height": 18, "angle": 0, "strokeColor": "#495057", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0021", "roundness": null, "seed": 2059392214, "version": 1, "versionNonce": 1643899184, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0021", "text": "one projection per decoder layer, no decoder pass", "originalText": "one projection per decoder layer, no decoder pass", "fontSize": 11, "fontFamily": 1, "textAlign": "center", "verticalAlign": "middle", "lineHeight": 1.25, "autoResize": false, "containerId": null}, {"type": "rectangle", "x": 800, "y": 130, "width": 540, "height": 220, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#f8f9fa", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "dashed", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0022", "roundness": {"type": 3}, "seed": 861674413, "version": 1, "versionNonce": 739798401, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0022"}, {"type": "text", "x": 800, "y": 136, "width": 540, "height": 24, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0023", "roundness": null, "seed": 1548208391, "version": 1, "versionNonce": 1025747016, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0023", "text": "decoder: layers 20 to 39 (not run over the prompt for the global branch)", "originalText": "decoder: layers 20 to 39 (not run over the prompt for the global branch)", "fontSize": 14, "fontFamily": 1, "textAlign": "center", "verticalAlign": "middle", "lineHeight": 1.25, "autoResize": false, "containerId": null}, {"type": "rectangle", "x": 820, "y": 180, "width": 220, "height": 40, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#ffc9c9", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0024", "roundness": {"type": 3}, "seed": 1383160529, "version": 1, "versionNonce": 2072732505, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0024"}, {"type": "text", "x": 820, "y": 180, "width": 220, "height": 40, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0025", "roundness": null, "seed": 616800822, "version": 1, "versionNonce": 1797086979, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0025", "text": "layer 20 Full: main KV", "originalText": "layer 20 Full: main KV", "fontSize": 12, "fontFamily": 1, "textAlign": "center", "verticalAlign": "middle", "lineHeight": 1.25, "autoResize": false, "containerId": null}, {"type": "arrow", "x": 750, "y": 260, "width": 70, "height": 60, "angle": 0, "strokeColor": "#e03131", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0026", "roundness": null, "seed": 2059822784, "version": 1, "versionNonce": 883719866, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0026", "points": [[0, 0], [30, 0], [30, -60], [70, -60]], "startArrowhead": null, "endArrowhead": "arrow", "startBinding": null, "endBinding": null, "lastCommittedPoint": null, "elbowed": false}, {"type": "rectangle", "x": 1060, "y": 180, "width": 260, "height": 40, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#fff3bf", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0027", "roundness": {"type": 3}, "seed": 2010272156, "version": 1, "versionNonce": 119272177, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0027"}, {"type": "text", "x": 1060, "y": 180, "width": 260, "height": 40, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0028", "roundness": null, "seed": 2067164821, "version": 1, "versionNonce": 310846306, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0028", "text": "24, 28, 32, 36 Reindex; rest Reuse", "originalText": "24, 28, 32, 36 Reindex; rest Reuse", "fontSize": 11, "fontFamily": 1, "textAlign": "center", "verticalAlign": "middle", "lineHeight": 1.25, "autoResize": false, "containerId": null}, {"type": "arrow", "x": 1040, "y": 200, "width": 20, "height": 0, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0029", "roundness": null, "seed": 1719113507, "version": 1, "versionNonce": 196908606, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0029", "points": [[0, 0], [20, 0]], "startArrowhead": null, "endArrowhead": "arrow", "startBinding": null, "endBinding": null, "lastCommittedPoint": null, "elbowed": false}, {"type": "rectangle", "x": 820, "y": 240, "width": 500, "height": 40, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#ffd8a8", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0030", "roundness": {"type": 3}, "seed": 2007447131, "version": 1, "versionNonce": 986130948, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0030"}, {"type": "text", "x": 820, "y": 240, "width": 500, "height": 40, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0031", "roundness": null, "seed": 1008366920, "version": 1, "versionNonce": 297487880, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0031", "text": "window KV per decoder layer: missing after early exit", "originalText": "window KV per decoder layer: missing after early exit", "fontSize": 12, "fontFamily": 1, "textAlign": "center", "verticalAlign": "middle", "lineHeight": 1.25, "autoResize": false, "containerId": null}, {"type": "text", "x": 820, "y": 290, "width": 500, "height": 50, "angle": 0, "strokeColor": "#495057", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0032", "roundness": null, "seed": 933604918, "version": 1, "versionNonce": 1091662261, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0032", "text": "rebuilt by decoder bounded replay: the last 128 prompt tokens' encoder outputs\nare run through the 20 decoder layers, windows truncated at the replay start", "originalText": "rebuilt by decoder bounded replay: the last 128 prompt tokens' encoder outputs\nare run through the 20 decoder layers, windows truncated at the replay start", "fontSize": 11, "fontFamily": 1, "textAlign": "left", "verticalAlign": "middle", "lineHeight": 1.25, "autoResize": false, "containerId": null}, {"type": "text", "x": 40, "y": 380, "width": 1300, "height": 60, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0033", "roundness": null, "seed": 1039776378, "version": 1, "versionNonce": 814425292, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0033", "text": "Prefill complexity: O(N L/2 + n_win L/2) instead of O(N L). Half the layers means half the experts per prompt token:\n8B active in prefill, 16B in decode. Decode runs all 40 layers as usual.", "originalText": "Prefill complexity: O(N L/2 + n_win L/2) instead of O(N L). Half the layers means half the experts per prompt token:\n8B active in prefill, 16B in decode. Decode runs all 40 layers as usual.", "fontSize": 13, "fontFamily": 1, "textAlign": "left", "verticalAlign": "middle", "lineHeight": 1.25, "autoResize": false, "containerId": null}], "appState": {"gridSize": 20, "gridStep": 5, "gridModeEnabled": false, "viewBackgroundColor": "#fdf8f6", "lockedMultiSelections": {}}, "files": {}}