{"type": "excalidraw", "version": 2, "source": "https://excalidraw.com", "elements": [{"type": "text", "x": 20, "y": 10, "width": 1300, "height": 28, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0001", "roundness": null, "seed": 1553714998, "version": 1, "versionNonce": 629653100, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0001", "text": "MLA at decode time: naive up-projection versus weight absorption", "originalText": "MLA at decode time: naive up-projection versus weight absorption", "fontSize": 22, "fontFamily": 1, "textAlign": "left", "verticalAlign": "middle", "lineHeight": 1.25, "autoResize": false, "containerId": null}, {"type": "rectangle", "x": 40, "y": 50, "width": 640, "height": 520, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#f8f9fa", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "dashed", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0002", "roundness": {"type": 3}, "seed": 116146606, "version": 1, "versionNonce": 1280285447, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0002"}, {"type": "text", "x": 40, "y": 56, "width": 640, "height": 26, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0003", "roundness": null, "seed": 390887331, "version": 1, "versionNonce": 1121481225, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0003", "text": "Naive", "originalText": "Naive", "fontSize": 18, "fontFamily": 1, "textAlign": "center", "verticalAlign": "middle", "lineHeight": 1.25, "autoResize": false, "containerId": null}, {"type": "rectangle", "x": 720, "y": 50, "width": 640, "height": 520, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#f8f9fa", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "dashed", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0004", "roundness": {"type": 3}, "seed": 1574963608, "version": 1, "versionNonce": 717440071, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0004"}, {"type": "text", "x": 720, "y": 56, "width": 640, "height": 26, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0005", "roundness": null, "seed": 1527703408, "version": 1, "versionNonce": 956887592, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0005", "text": "Absorbed (what the kernels compute)", "originalText": "Absorbed (what the kernels compute)", "fontSize": 18, "fontFamily": 1, "textAlign": "center", "verticalAlign": "middle", "lineHeight": 1.25, "autoResize": false, "containerId": null}, {"type": "rectangle", "x": 60, "y": 110, "width": 200, "height": 44, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#b2f2bb", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0006", "roundness": {"type": 3}, "seed": 1415900355, "version": 1, "versionNonce": 957956675, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0006"}, {"type": "text", "x": 60, "y": 110, "width": 200, "height": 44, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0007", "roundness": null, "seed": 838145800, "version": 1, "versionNonce": 1028162214, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0007", "text": "cache: latents C  (T x 512)", "originalText": "cache: latents C  (T x 512)", "fontSize": 13, "fontFamily": 1, "textAlign": "center", "verticalAlign": "middle", "lineHeight": 1.25, "autoResize": false, "containerId": null}, {"type": "rectangle", "x": 60, "y": 170, "width": 200, "height": 44, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#d0bfff", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0008", "roundness": {"type": 3}, "seed": 1720926263, "version": 1, "versionNonce": 973838694, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0008"}, {"type": "text", "x": 60, "y": 170, "width": 200, "height": 44, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0009", "roundness": null, "seed": 858641202, "version": 1, "versionNonce": 2116481899, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0009", "text": "W_UK  (per head)", "originalText": "W_UK  (per head)", "fontSize": 13, "fontFamily": 1, "textAlign": "center", "verticalAlign": "middle", "lineHeight": 1.25, "autoResize": false, "containerId": null}, {"type": "rectangle", "x": 280, "y": 170, "width": 200, "height": 44, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#d0bfff", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0010", "roundness": {"type": 3}, "seed": 1527129487, "version": 1, "versionNonce": 124468791, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0010"}, {"type": "text", "x": 280, "y": 170, "width": 200, "height": 44, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0011", "roundness": null, "seed": 119988829, "version": 1, "versionNonce": 1200093500, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0011", "text": "W_UV  (per head)", "originalText": "W_UV  (per head)", "fontSize": 13, "fontFamily": 1, "textAlign": "center", "verticalAlign": "middle", "lineHeight": 1.25, "autoResize": false, "containerId": null}, {"type": "arrow", "x": 160, "y": 154, "width": 0, "height": 16, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0012", "roundness": null, "seed": 2028255630, "version": 1, "versionNonce": 1113145427, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0012", "points": [[0, 0], [0, 16]], "startArrowhead": null, "endArrowhead": "arrow", "startBinding": null, "endBinding": null, "lastCommittedPoint": null, "elbowed": false}, {"type": "arrow", "x": 260, "y": 132, "width": 120, "height": 38, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0013", "roundness": null, "seed": 831698693, "version": 1, "versionNonce": 1478675320, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0013", "points": [[0, 0], [120, 0], [120, 38]], "startArrowhead": null, "endArrowhead": "arrow", "startBinding": null, "endBinding": null, "lastCommittedPoint": null, "elbowed": false}, {"type": "rectangle", "x": 60, "y": 230, "width": 200, "height": 44, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#ffc9c9", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0014", "roundness": {"type": 3}, "seed": 1920828234, "version": 1, "versionNonce": 1501175504, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0014"}, {"type": "text", "x": 60, "y": 230, "width": 200, "height": 44, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0015", "roundness": null, "seed": 1566099206, "version": 1, "versionNonce": 345908636, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0015", "text": "K: 128 heads x 128  (T each)", "originalText": "K: 128 heads x 128  (T each)", "fontSize": 13, "fontFamily": 1, "textAlign": "center", "verticalAlign": "middle", "lineHeight": 1.25, "autoResize": false, "containerId": null}, {"type": "rectangle", "x": 280, "y": 230, "width": 200, "height": 44, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#ffc9c9", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0016", "roundness": {"type": 3}, "seed": 946878465, "version": 1, "versionNonce": 438761610, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0016"}, {"type": "text", "x": 280, "y": 230, "width": 200, "height": 44, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0017", "roundness": null, "seed": 974295421, "version": 1, "versionNonce": 2018978167, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0017", "text": "V: 128 heads x 128  (T each)", "originalText": "V: 128 heads x 128  (T each)", "fontSize": 13, "fontFamily": 1, "textAlign": "center", "verticalAlign": "middle", "lineHeight": 1.25, "autoResize": false, "containerId": null}, {"type": "arrow", "x": 160, "y": 214, "width": 0, "height": 16, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0018", "roundness": null, "seed": 844846558, "version": 1, "versionNonce": 1450571438, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0018", "points": [[0, 0], [0, 16]], "startArrowhead": null, "endArrowhead": "arrow", "startBinding": null, "endBinding": null, "lastCommittedPoint": null, "elbowed": false}, {"type": "arrow", "x": 380, "y": 214, "width": 0, "height": 16, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0019", "roundness": null, "seed": 877776916, "version": 1, "versionNonce": 2072980150, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0019", "points": [[0, 0], [0, 16]], "startArrowhead": null, "endArrowhead": "arrow", "startBinding": null, "endBinding": null, "lastCommittedPoint": null, "elbowed": false}, {"type": "rectangle", "x": 500, "y": 110, "width": 160, "height": 44, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#e9ecef", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0020", "roundness": {"type": 3}, "seed": 8196149, "version": 1, "versionNonce": 2059322682, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0020"}, {"type": "text", "x": 500, "y": 110, "width": 160, "height": 44, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0021", "roundness": null, "seed": 1477498383, "version": 1, "versionNonce": 364120812, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0021", "text": "h_t  (new token)", "originalText": "h_t  (new token)", "fontSize": 13, "fontFamily": 1, "textAlign": "center", "verticalAlign": "middle", "lineHeight": 1.25, "autoResize": false, "containerId": null}, {"type": "rectangle", "x": 500, "y": 170, "width": 160, "height": 44, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#d0bfff", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0022", "roundness": {"type": 3}, "seed": 514982154, "version": 1, "versionNonce": 1668748296, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0022"}, {"type": "text", "x": 500, "y": 170, "width": 160, "height": 44, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0023", "roundness": null, "seed": 856070306, "version": 1, "versionNonce": 2053134994, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0023", "text": "W_Q", "originalText": "W_Q", "fontSize": 13, "fontFamily": 1, "textAlign": "center", "verticalAlign": "middle", "lineHeight": 1.25, "autoResize": false, "containerId": null}, {"type": "rectangle", "x": 500, "y": 230, "width": 160, "height": 44, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#a5d8ff", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0024", "roundness": {"type": 3}, "seed": 766744960, "version": 1, "versionNonce": 1863693998, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0024"}, {"type": "text", "x": 500, "y": 230, "width": 160, "height": 44, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0025", "roundness": null, "seed": 1428150522, "version": 1, "versionNonce": 372587780, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0025", "text": "q: 128 heads x 128", "originalText": "q: 128 heads x 128", "fontSize": 13, "fontFamily": 1, "textAlign": "center", "verticalAlign": "middle", "lineHeight": 1.25, "autoResize": false, "containerId": null}, {"type": "arrow", "x": 580, "y": 154, "width": 0, "height": 16, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0026", "roundness": null, "seed": 1700113407, "version": 1, "versionNonce": 1989259376, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0026", "points": [[0, 0], [0, 16]], "startArrowhead": null, "endArrowhead": "arrow", "startBinding": null, "endBinding": null, "lastCommittedPoint": null, "elbowed": false}, {"type": "arrow", "x": 580, "y": 214, "width": 0, "height": 16, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0027", "roundness": null, "seed": 1723943247, "version": 1, "versionNonce": 364725392, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0027", "points": [[0, 0], [0, 16]], "startArrowhead": null, "endArrowhead": "arrow", "startBinding": null, "endBinding": null, "lastCommittedPoint": null, "elbowed": false}, {"type": "rectangle", "x": 60, "y": 310, "width": 600, "height": 44, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#fff3bf", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0028", "roundness": {"type": 3}, "seed": 682281554, "version": 1, "versionNonce": 730160160, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0028"}, {"type": "text", "x": 60, "y": 310, "width": 600, "height": 44, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0029", "roundness": null, "seed": 545625653, "version": 1, "versionNonce": 118321418, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0029", "text": "scores = q . K  per head, over T tokens;  softmax", "originalText": "scores = q . K  per head, over T tokens;  softmax", "fontSize": 13, "fontFamily": 1, "textAlign": "center", "verticalAlign": "middle", "lineHeight": 1.25, "autoResize": false, "containerId": null}, {"type": "arrow", "x": 160, "y": 274, "width": 0, "height": 36, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0030", "roundness": null, "seed": 649187325, "version": 1, "versionNonce": 1998679711, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0030", "points": [[0, 0], [0, 36]], "startArrowhead": null, "endArrowhead": "arrow", "startBinding": null, "endBinding": null, "lastCommittedPoint": null, "elbowed": false}, {"type": "arrow", "x": 580, "y": 274, "width": 0, "height": 36, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0031", "roundness": null, "seed": 627813882, "version": 1, "versionNonce": 2037347502, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0031", "points": [[0, 0], [0, 36]], "startArrowhead": null, "endArrowhead": "arrow", "startBinding": null, "endBinding": null, "lastCommittedPoint": null, "elbowed": false}, {"type": "rectangle", "x": 60, "y": 380, "width": 600, "height": 44, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#fff3bf", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0032", "roundness": {"type": 3}, "seed": 1504988819, "version": 1, "versionNonce": 669638768, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0032"}, {"type": "text", "x": 60, "y": 380, "width": 600, "height": 44, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0033", "roundness": null, "seed": 562571391, "version": 1, "versionNonce": 91898035, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0033", "text": "out = softmax . V  per head  ->  W_O", "originalText": "out = softmax . V  per head  ->  W_O", "fontSize": 13, "fontFamily": 1, "textAlign": "center", "verticalAlign": "middle", "lineHeight": 1.25, "autoResize": false, "containerId": null}, {"type": "arrow", "x": 380, "y": 274, "width": 0, "height": 106, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "dashed", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0034", "roundness": null, "seed": 61172930, "version": 1, "versionNonce": 441402618, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0034", "points": [[0, 0], [0, 106]], "startArrowhead": null, "endArrowhead": "arrow", "startBinding": null, "endBinding": null, "lastCommittedPoint": null, "elbowed": false}, {"type": "arrow", "x": 360, "y": 354, "width": 0, "height": 26, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0035", "roundness": null, "seed": 598077321, "version": 1, "versionNonce": 1863197322, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0035", "points": [[0, 0], [0, 26]], "startArrowhead": null, "endArrowhead": "arrow", "startBinding": null, "endBinding": null, "lastCommittedPoint": null, "elbowed": false}, {"type": "text", "x": 60, "y": 440, "width": 600, "height": 37.2, "angle": 0, "strokeColor": "#495057", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0036", "roundness": null, "seed": 836682997, "version": 1, "versionNonce": 906419965, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0036", "text": "Regenerates 128 keys and 128 values for every cached token, every step.\nThe cache is small but the work to use it is the full MHA work.", "originalText": "Regenerates 128 keys and 128 values for every cached token, every step.\nThe cache is small but the work to use it is the full MHA work.", "fontSize": 12, "fontFamily": 1, "textAlign": "left", "verticalAlign": "middle", "lineHeight": 1.25, "autoResize": false, "containerId": null}, {"type": "rectangle", "x": 740, "y": 110, "width": 200, "height": 44, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#b2f2bb", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0037", "roundness": {"type": 3}, "seed": 120232147, "version": 1, "versionNonce": 1081622283, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0037"}, {"type": "text", "x": 740, "y": 110, "width": 200, "height": 44, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0038", "roundness": null, "seed": 913882254, "version": 1, "versionNonce": 1258282194, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0038", "text": "cache: latents C  (T x 512)", "originalText": "cache: latents C  (T x 512)", "fontSize": 13, "fontFamily": 1, "textAlign": "center", "verticalAlign": "middle", "lineHeight": 1.25, "autoResize": false, "containerId": null}, {"type": "rectangle", "x": 960, "y": 110, "width": 200, "height": 44, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#ffd8a8", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0039", "roundness": {"type": 3}, "seed": 1033108813, "version": 1, "versionNonce": 1400113411, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0039"}, {"type": "text", "x": 960, "y": 110, "width": 200, "height": 44, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0040", "roundness": null, "seed": 1113963314, "version": 1, "versionNonce": 1799645191, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0040", "text": "cache: RoPE keys k_R  (T x 64)", "originalText": "cache: RoPE keys k_R  (T x 64)", "fontSize": 13, "fontFamily": 1, "textAlign": "center", "verticalAlign": "middle", "lineHeight": 1.25, "autoResize": false, "containerId": null}, {"type": "rectangle", "x": 1180, "y": 110, "width": 160, "height": 44, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#e9ecef", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0041", "roundness": {"type": 3}, "seed": 562957180, "version": 1, "versionNonce": 261582917, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0041"}, {"type": "text", "x": 1180, "y": 110, "width": 160, "height": 44, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0042", "roundness": null, "seed": 1519490801, "version": 1, "versionNonce": 1967786446, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0042", "text": "h_t  (new token)", "originalText": "h_t  (new token)", "fontSize": 13, "fontFamily": 1, "textAlign": "center", "verticalAlign": "middle", "lineHeight": 1.25, "autoResize": false, "containerId": null}, {"type": "rectangle", "x": 1180, "y": 170, "width": 160, "height": 44, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#d0bfff", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0043", "roundness": {"type": 3}, "seed": 1806584668, "version": 1, "versionNonce": 561623934, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0043"}, {"type": "text", "x": 1180, "y": 170, "width": 160, "height": 44, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0044", "roundness": null, "seed": 652132316, "version": 1, "versionNonce": 80336782, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0044", "text": "W_Q, then (W_UK)^T folded in", "originalText": "W_Q, then (W_UK)^T folded in", "fontSize": 13, "fontFamily": 1, "textAlign": "center", "verticalAlign": "middle", "lineHeight": 1.25, "autoResize": false, "containerId": null}, {"type": "arrow", "x": 1260, "y": 154, "width": 0, "height": 16, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0045", "roundness": null, "seed": 1890322093, "version": 1, "versionNonce": 786442398, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0045", "points": [[0, 0], [0, 16]], "startArrowhead": null, "endArrowhead": "arrow", "startBinding": null, "endBinding": null, "lastCommittedPoint": null, "elbowed": false}, {"type": "rectangle", "x": 1100, "y": 230, "width": 240, "height": 44, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#a5d8ff", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0046", "roundness": {"type": 3}, "seed": 16889874, "version": 1, "versionNonce": 643396776, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0046"}, {"type": "text", "x": 1100, "y": 230, "width": 240, "height": 44, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0047", "roundness": null, "seed": 740223520, "version": 1, "versionNonce": 607991153, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0047", "text": "q_abs: 128 heads x 512  (+ q_R x 64)", "originalText": "q_abs: 128 heads x 512  (+ q_R x 64)", "fontSize": 13, "fontFamily": 1, "textAlign": "center", "verticalAlign": "middle", "lineHeight": 1.25, "autoResize": false, "containerId": null}, {"type": "arrow", "x": 1260, "y": 214, "width": 0, "height": 16, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0048", "roundness": null, "seed": 2033636663, "version": 1, "versionNonce": 516841822, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0048", "points": [[0, 0], [0, 16]], "startArrowhead": null, "endArrowhead": "arrow", "startBinding": null, "endBinding": null, "lastCommittedPoint": null, "elbowed": false}, {"type": "rectangle", "x": 740, "y": 310, "width": 600, "height": 44, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#fff3bf", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0049", "roundness": {"type": 3}, "seed": 265236939, "version": 1, "versionNonce": 1400082662, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0049"}, {"type": "text", "x": 740, "y": 310, "width": 600, "height": 44, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0050", "roundness": null, "seed": 2072265938, "version": 1, "versionNonce": 455736474, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0050", "text": "scores = q_abs . C  +  q_R . k_R  per head;  softmax", "originalText": "scores = q_abs . C  +  q_R . k_R  per head;  softmax", "fontSize": 13, "fontFamily": 1, "textAlign": "center", "verticalAlign": "middle", "lineHeight": 1.25, "autoResize": false, "containerId": null}, {"type": "arrow", "x": 840, "y": 154, "width": 0, "height": 156, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0051", "roundness": null, "seed": 244051093, "version": 1, "versionNonce": 1067275002, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0051", "points": [[0, 0], [0, 156]], "startArrowhead": null, "endArrowhead": "arrow", "startBinding": null, "endBinding": null, "lastCommittedPoint": null, "elbowed": false}, {"type": "arrow", "x": 1060, "y": 154, "width": 0, "height": 156, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0052", "roundness": null, "seed": 821653593, "version": 1, "versionNonce": 1189349777, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0052", "points": [[0, 0], [0, 156]], "startArrowhead": null, "endArrowhead": "arrow", "startBinding": null, "endBinding": null, "lastCommittedPoint": null, "elbowed": false}, {"type": "arrow", "x": 1220, "y": 274, "width": 0, "height": 36, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0053", "roundness": null, "seed": 181242850, "version": 1, "versionNonce": 419812754, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0053", "points": [[0, 0], [0, 36]], "startArrowhead": null, "endArrowhead": "arrow", "startBinding": null, "endBinding": null, "lastCommittedPoint": null, "elbowed": false}, {"type": "rectangle", "x": 740, "y": 380, "width": 600, "height": 44, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#fff3bf", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0054", "roundness": {"type": 3}, "seed": 1942080813, "version": 1, "versionNonce": 119682499, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0054"}, {"type": "text", "x": 740, "y": 380, "width": 600, "height": 44, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0055", "roundness": null, "seed": 272167094, "version": 1, "versionNonce": 1903737355, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0055", "text": "o_lat = softmax . C  per head (512 wide)  ->  W_UV folded into W_O", "originalText": "o_lat = softmax . C  per head (512 wide)  ->  W_UV folded into W_O", "fontSize": 13, "fontFamily": 1, "textAlign": "center", "verticalAlign": "middle", "lineHeight": 1.25, "autoResize": false, "containerId": null}, {"type": "arrow", "x": 1040, "y": 354, "width": 0, "height": 26, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0056", "roundness": null, "seed": 1398499908, "version": 1, "versionNonce": 856430244, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0056", "points": [[0, 0], [0, 26]], "startArrowhead": null, "endArrowhead": "arrow", "startBinding": null, "endBinding": null, "lastCommittedPoint": null, "elbowed": false}, {"type": "text", "x": 740, "y": 440, "width": 600, "height": 37.2, "angle": 0, "strokeColor": "#495057", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": "a0057", "roundness": null, "seed": 1190502837, "version": 1, "versionNonce": 1942810372, "isDeleted": false, "boundElements": null, "updated": 1789071763372, "link": null, "locked": false, "id": "el0057", "text": "Attention runs over the latent as key and value: MQA with a 576-wide key and a\n512-wide value shared by all heads. No key or value is ever materialized.", "originalText": "Attention runs over the latent as key and value: MQA with a 576-wide key and a\n512-wide value shared by all heads. No key or value is ever materialized.", "fontSize": 12, "fontFamily": 1, "textAlign": "left", "verticalAlign": "middle", "lineHeight": 1.25, "autoResize": false, "containerId": null}], "appState": {"gridSize": 20, "gridStep": 5, "gridModeEnabled": false, "viewBackgroundColor": "#fdf8f6", "lockedMultiSelections": {}}, "files": {}}