{"type": "excalidraw", "version": 2, "source": "https://excalidraw.com", "elements": [{"type": "text", "x": 20, "y": 10, "width": 1300, "height": 28, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0001", "roundness": null, "seed": 1843591968, "version": 1, "versionNonce": 598321627, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0001", "text": "DeepSeek Sparse Attention, one decode step in one layer (V3.2 sizes)", "originalText": "DeepSeek Sparse Attention, one decode step in one layer (V3.2 sizes)", "fontSize": 22, "fontFamily": 1, "textAlign": "left", "verticalAlign": "middle", "lineHeight": 1.25, "autoResize": false, "containerId": null}, {"type": "rectangle", "x": 40, "y": 70, "width": 180, "height": 50, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#e9ecef", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0002", "roundness": {"type": 3}, "seed": 69062037, "version": 1, "versionNonce": 987587880, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0002"}, {"type": "text", "x": 40, "y": 70, "width": 180, "height": 50, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0003", "roundness": null, "seed": 365466112, "version": 1, "versionNonce": 133833464, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0003", "text": "h_t  (query token)", "originalText": "h_t  (query token)", "fontSize": 14, "fontFamily": 1, "textAlign": "center", "verticalAlign": "middle", "lineHeight": 1.25, "autoResize": false, "containerId": null}, {"type": "rectangle", "x": 40, "y": 150, "width": 180, "height": 50, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#a5d8ff", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0004", "roundness": {"type": 3}, "seed": 179796361, "version": 1, "versionNonce": 571630913, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0004"}, {"type": "text", "x": 40, "y": 150, "width": 180, "height": 50, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0005", "roundness": null, "seed": 1549226796, "version": 1, "versionNonce": 450612830, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0005", "text": "indexer q: 64 heads x 128\nweights w: 64 (FP8)", "originalText": "indexer q: 64 heads x 128\nweights w: 64 (FP8)", "fontSize": 12, "fontFamily": 1, "textAlign": "center", "verticalAlign": "middle", "lineHeight": 1.25, "autoResize": false, "containerId": null}, {"type": "arrow", "x": 130, "y": 120, "width": 0, "height": 30, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0006", "roundness": null, "seed": 1617563116, "version": 1, "versionNonce": 1938688888, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0006", "points": [[0, 0], [0, 30]], "startArrowhead": null, "endArrowhead": "arrow", "startBinding": null, "endBinding": null, "lastCommittedPoint": null, "elbowed": false}, {"type": "rectangle", "x": 300, "y": 70, "width": 260, "height": 50, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#ffd8a8", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0007", "roundness": {"type": 3}, "seed": 218099797, "version": 1, "versionNonce": 80920074, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0007"}, {"type": "text", "x": 300, "y": 70, "width": 260, "height": 50, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0008", "roundness": null, "seed": 1050375824, "version": 1, "versionNonce": 2101503088, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0008", "text": "cache: indexer keys k_I  (T x 128, FP8)", "originalText": "cache: indexer keys k_I  (T x 128, FP8)", "fontSize": 12, "fontFamily": 1, "textAlign": "center", "verticalAlign": "middle", "lineHeight": 1.25, "autoResize": false, "containerId": null}, {"type": "rectangle", "x": 300, "y": 150, "width": 260, "height": 50, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#b2f2bb", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0009", "roundness": {"type": 3}, "seed": 1132981884, "version": 1, "versionNonce": 14234933, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0009"}, {"type": "text", "x": 300, "y": 150, "width": 260, "height": 50, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0010", "roundness": null, "seed": 1962578963, "version": 1, "versionNonce": 301126734, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0010", "text": "cache: latents  (T x 576, FP8 + bf16 RoPE)", "originalText": "cache: latents  (T x 576, FP8 + bf16 RoPE)", "fontSize": 12, "fontFamily": 1, "textAlign": "center", "verticalAlign": "middle", "lineHeight": 1.25, "autoResize": false, "containerId": null}, {"type": "rectangle", "x": 640, "y": 150, "width": 320, "height": 60, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#fff3bf", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0011", "roundness": {"type": 3}, "seed": 394887584, "version": 1, "versionNonce": 283684098, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0011"}, {"type": "text", "x": 640, "y": 150, "width": 320, "height": 60, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0012", "roundness": null, "seed": 2035206439, "version": 1, "versionNonce": 1083162951, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0012", "text": "scores I(t,s) = sum_j w_j ReLU(q_j . k_s)\none score per cached token (T)", "originalText": "scores I(t,s) = sum_j w_j ReLU(q_j . k_s)\none score per cached token (T)", "fontSize": 12, "fontFamily": 1, "textAlign": "center", "verticalAlign": "middle", "lineHeight": 1.25, "autoResize": false, "containerId": null}, {"type": "arrow", "x": 220, "y": 175, "width": 420, "height": 0, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0013", "roundness": null, "seed": 319760307, "version": 1, "versionNonce": 1140563901, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0013", "points": [[0, 0], [420, 0]], "startArrowhead": null, "endArrowhead": "arrow", "startBinding": null, "endBinding": null, "lastCommittedPoint": null, "elbowed": false}, {"type": "arrow", "x": 430, "y": 120, "width": 210, "height": 45, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0014", "roundness": null, "seed": 1008396568, "version": 1, "versionNonce": 881402584, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0014", "points": [[0, 0], [0, 15], [170, 15], [170, 45], [210, 45]], "startArrowhead": null, "endArrowhead": "arrow", "startBinding": null, "endBinding": null, "lastCommittedPoint": null, "elbowed": false}, {"type": "rectangle", "x": 1000, "y": 150, "width": 180, "height": 60, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#d0bfff", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0015", "roundness": {"type": 3}, "seed": 991004122, "version": 1, "versionNonce": 1977145509, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0015"}, {"type": "text", "x": 1000, "y": 150, "width": 180, "height": 60, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0016", "roundness": null, "seed": 2121493856, "version": 1, "versionNonce": 1643084754, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0016", "text": "top-k selector\nk = 2048", "originalText": "top-k selector\nk = 2048", "fontSize": 13, "fontFamily": 1, "textAlign": "center", "verticalAlign": "middle", "lineHeight": 1.25, "autoResize": false, "containerId": null}, {"type": "arrow", "x": 960, "y": 180, "width": 40, "height": 0, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0017", "roundness": null, "seed": 329595264, "version": 1, "versionNonce": 2057332978, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0017", "points": [[0, 0], [40, 0]], "startArrowhead": null, "endArrowhead": "arrow", "startBinding": null, "endBinding": null, "lastCommittedPoint": null, "elbowed": false}, {"type": "rectangle", "x": 640, "y": 270, "width": 500, "height": 60, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#c3fae8", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0018", "roundness": {"type": 3}, "seed": 1234026409, "version": 1, "versionNonce": 200778942, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0018"}, {"type": "text", "x": 640, "y": 270, "width": 500, "height": 60, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0019", "roundness": null, "seed": 851649605, "version": 1, "versionNonce": 332738928, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0019", "text": "core attention, MQA mode: q (128 heads) over the 2048 selected latents,\neach latent is key and value", "originalText": "core attention, MQA mode: q (128 heads) over the 2048 selected latents,\neach latent is key and value", "fontSize": 12, "fontFamily": 1, "textAlign": "center", "verticalAlign": "middle", "lineHeight": 1.25, "autoResize": false, "containerId": null}, {"type": "arrow", "x": 1090, "y": 210, "width": 50, "height": 90, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0020", "roundness": null, "seed": 633185882, "version": 1, "versionNonce": 1424953947, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0020", "points": [[0, 0], [0, 90], [50, 90]], "startArrowhead": null, "endArrowhead": "arrow", "startBinding": null, "endBinding": null, "lastCommittedPoint": null, "elbowed": false}, {"type": "arrow", "x": 560, "y": 175, "width": 80, "height": 125, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "dashed", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0021", "roundness": null, "seed": 1090665198, "version": 1, "versionNonce": 1307461651, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0021", "points": [[0, 0], [0, 125], [80, 125]], "startArrowhead": null, "endArrowhead": "arrow", "startBinding": null, "endBinding": null, "lastCommittedPoint": null, "elbowed": false}, {"type": "text", "x": 575, "y": 232, "width": 60, "height": 20, "angle": 0, "strokeColor": "#495057", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0022", "roundness": null, "seed": 573124783, "version": 1, "versionNonce": 53554862, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0022", "text": "gather", "originalText": "gather", "fontSize": 11, "fontFamily": 1, "textAlign": "center", "verticalAlign": "middle", "lineHeight": 1.25, "autoResize": false, "containerId": null}, {"type": "rectangle", "x": 640, "y": 370, "width": 500, "height": 44, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#e9ecef", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0023", "roundness": {"type": 3}, "seed": 2071981132, "version": 1, "versionNonce": 260537060, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0023"}, {"type": "text", "x": 640, "y": 370, "width": 500, "height": 44, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0024", "roundness": null, "seed": 2086486749, "version": 1, "versionNonce": 1154370228, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0024", "text": "u_t  (attention output)", "originalText": "u_t  (attention output)", "fontSize": 13, "fontFamily": 1, "textAlign": "center", "verticalAlign": "middle", "lineHeight": 1.25, "autoResize": false, "containerId": null}, {"type": "arrow", "x": 890, "y": 330, "width": 0, "height": 40, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0025", "roundness": null, "seed": 427431139, "version": 1, "versionNonce": 934986292, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0025", "points": [[0, 0], [0, 40]], "startArrowhead": null, "endArrowhead": "arrow", "startBinding": null, "endBinding": null, "lastCommittedPoint": null, "elbowed": false}, {"type": "text", "x": 40, "y": 250, "width": 540, "height": 99.60000000000001, "angle": 0, "strokeColor": "#495057", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0026", "roundness": null, "seed": 2102877464, "version": 1, "versionNonce": 1249219059, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0026", "text": "Cost per query per layer:\n  indexer scan: T keys x 64 heads x 128 dims, FP8   (linear in T)\n  core attention: 2048 entries x 128 heads x 576   (constant in T)\n\nStored per token per layer: 656 B latent row + 132 B indexer key = 788 B\nAcross 61 layers: 48,068 B per token", "originalText": "Cost per query per layer:\n  indexer scan: T keys x 64 heads x 128 dims, FP8   (linear in T)\n  core attention: 2048 entries x 128 heads x 576   (constant in T)\n\nStored per token per layer: 656 B latent row + 132 B indexer key = 788 B\nAcross 61 layers: 48,068 B per token", "fontSize": 12, "fontFamily": 1, "textAlign": "left", "verticalAlign": "middle", "lineHeight": 1.25, "autoResize": false, "containerId": null}], "appState": {"gridSize": 20, "gridStep": 5, "gridModeEnabled": false, "viewBackgroundColor": "#fdf8f6", "lockedMultiSelections": {}}, "files": {}}