{ "type": "excalidraw", "version": 2, "source": "generated", "elements": [ { "id": "el0001", "type": "text", "x": 20, "y": 10, "width": 880.0000000000001, "height": 25.0, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": null, "seed": 1390851129, "version": 1, "versionNonce": 647892280, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "FlashAttention-2 forward: one block sweeps K^T left to right and V top to bottom", "fontSize": 20, "fontFamily": 1, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "FlashAttention-2 forward: one block sweeps K^T left to right and V top to bottom", "autoResize": true, "lineHeight": 1.25 }, { "id": "el0002", "type": "rectangle", "x": 40, "y": 90, "width": 60, "height": 50, "angle": 0, "strokeColor": "#e03131", "backgroundColor": "#ffc9c9", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": { "type": 3 }, "seed": 1695753999, "version": 1, "versionNonce": 207388625, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "el0003", "type": "text", "x": 48.825, "y": 101.25, "width": 42.35, "height": 27.5, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": null, "seed": 311111476, "version": 1, "versionNonce": 404285458, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "Q_i\n(Br, d)", "fontSize": 11, "fontFamily": 1, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "Q_i\n(Br, d)", "autoResize": true, "lineHeight": 1.25 }, { "id": "el0004", "type": "rectangle", "x": 130, "y": 60, "width": 44, "height": 40, "angle": 0, "strokeColor": "#495057", "backgroundColor": "#f1f3f5", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": { "type": 3 }, "seed": 1570621945, "version": 1, "versionNonce": 249103478, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "el0005", "type": "rectangle", "x": 174, "y": 60, "width": 44, "height": 40, "angle": 0, "strokeColor": "#495057", "backgroundColor": "#f1f3f5", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": { "type": 3 }, "seed": 922121677, "version": 1, "versionNonce": 161042649, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "el0006", "type": "rectangle", "x": 218, "y": 60, "width": 44, "height": 40, "angle": 0, "strokeColor": "#495057", "backgroundColor": "#f1f3f5", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": { "type": 3 }, "seed": 369140571, "version": 1, "versionNonce": 1862494043, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "el0007", "type": "rectangle", "x": 262, "y": 60, "width": 44, "height": 40, "angle": 0, "strokeColor": "#1971c2", "backgroundColor": "#a5d8ff", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": { "type": 3 }, "seed": 1796035740, "version": 1, "versionNonce": 300026768, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "el0008", "type": "rectangle", "x": 306, "y": 60, "width": 44, "height": 40, "angle": 0, "strokeColor": "#495057", "backgroundColor": "#f1f3f5", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": { "type": 3 }, "seed": 1033639717, "version": 1, "versionNonce": 389609434, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "el0009", "type": "rectangle", "x": 350, "y": 60, "width": 44, "height": 40, "angle": 0, "strokeColor": "#495057", "backgroundColor": "#f1f3f5", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": { "type": 3 }, "seed": 1823296039, "version": 1, "versionNonce": 253877687, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "el0010", "type": "rectangle", "x": 394, "y": 60, "width": 44, "height": 40, "angle": 0, "strokeColor": "#495057", "backgroundColor": "#f1f3f5", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": { "type": 3 }, "seed": 531725348, "version": 1, "versionNonce": 958804058, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "el0011", "type": "rectangle", "x": 438, "y": 60, "width": 44, "height": 40, "angle": 0, "strokeColor": "#495057", "backgroundColor": "#f1f3f5", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": { "type": 3 }, "seed": 265695474, "version": 1, "versionNonce": 1703729685, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "el0012", "type": "text", "x": 142.64999999999998, "y": 42, "width": 326.70000000000005, "height": 13.75, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": null, "seed": 212984477, "version": 1, "versionNonce": 949539217, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "K^T (d, N): tiles K_1^T .. K_Tc^T, sweep left to right", "fontSize": 11, "fontFamily": 1, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "K^T (d, N): tiles K_1^T .. K_Tc^T, sweep left to right", "autoResize": true, "lineHeight": 1.25 }, { "id": "el0013", "type": "arrow", "x": 140, "y": 106, "width": 330, "height": 0, "angle": 0, "strokeColor": "#1971c2", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": { "type": 2 }, "seed": 200071089, "version": 1, "versionNonce": 571981486, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 330, 0 ] ], "lastCommittedPoint": null, "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "el0014", "type": "rectangle", "x": 262, "y": 110, "width": 44, "height": 40, "angle": 0, "strokeColor": "#e8590c", "backgroundColor": "#ffd8a8", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": { "type": 3 }, "seed": 1243862423, "version": 1, "versionNonce": 1800188483, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "el0015", "type": "text", "x": 280.975, "y": 123.125, "width": 6.050000000000001, "height": 13.75, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": null, "seed": 619570853, "version": 1, "versionNonce": 505913793, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "S", "fontSize": 11, "fontFamily": 1, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "S", "autoResize": true, "lineHeight": 1.25 }, { "id": "el0016", "type": "text", "x": 262.0, "y": 155, "width": 44.0, "height": 12.5, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": null, "seed": 1324919353, "version": 1, "versionNonce": 776213900, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "(Br, Bc)", "fontSize": 10, "fontFamily": 1, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "(Br, Bc)", "autoResize": true, "lineHeight": 1.25 }, { "id": "el0017", "type": "rectangle", "x": 560, "y": 60, "width": 60, "height": 30, "angle": 0, "strokeColor": "#495057", "backgroundColor": "#f1f3f5", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": { "type": 3 }, "seed": 442620899, "version": 1, "versionNonce": 806899910, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "el0018", "type": "rectangle", "x": 560, "y": 90, "width": 60, "height": 30, "angle": 0, "strokeColor": "#495057", "backgroundColor": "#f1f3f5", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": { "type": 3 }, "seed": 1599435268, "version": 1, "versionNonce": 418461139, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "el0019", "type": "rectangle", "x": 560, "y": 120, "width": 60, "height": 30, "angle": 0, "strokeColor": "#495057", "backgroundColor": "#f1f3f5", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": { "type": 3 }, "seed": 269676600, "version": 1, "versionNonce": 255985077, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "el0020", "type": "rectangle", "x": 560, "y": 150, "width": 60, "height": 30, "angle": 0, "strokeColor": "#1971c2", "backgroundColor": "#a5d8ff", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": { "type": 3 }, "seed": 884585952, "version": 1, "versionNonce": 2132084005, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "el0021", "type": "rectangle", "x": 560, "y": 180, "width": 60, "height": 30, "angle": 0, "strokeColor": "#495057", "backgroundColor": "#f1f3f5", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": { "type": 3 }, "seed": 1836494975, "version": 1, "versionNonce": 1349251824, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "el0022", "type": "rectangle", "x": 560, "y": 210, "width": 60, "height": 30, "angle": 0, "strokeColor": "#495057", "backgroundColor": "#f1f3f5", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": { "type": 3 }, "seed": 1999744785, "version": 1, "versionNonce": 1946412081, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "el0023", "type": "rectangle", "x": 560, "y": 240, "width": 60, "height": 30, "angle": 0, "strokeColor": "#495057", "backgroundColor": "#f1f3f5", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": { "type": 3 }, "seed": 1552984409, "version": 1, "versionNonce": 1287489454, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "el0024", "type": "rectangle", "x": 560, "y": 270, "width": 60, "height": 30, "angle": 0, "strokeColor": "#495057", "backgroundColor": "#f1f3f5", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": { "type": 3 }, "seed": 1066984056, "version": 1, "versionNonce": 772092315, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "el0025", "type": "text", "x": 505.3, "y": 306, "width": 169.4, "height": 27.5, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": null, "seed": 1048386556, "version": 1, "versionNonce": 351564608, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "V (N, d): tiles V_1 .. V_Tc,\nsweep top to bottom", "fontSize": 11, "fontFamily": 1, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "V (N, d): tiles V_1 .. V_Tc,\nsweep top to bottom", "autoResize": true, "lineHeight": 1.25 }, { "id": "el0026", "type": "arrow", "x": 630, "y": 70, "width": 0, "height": 220, "angle": 0, "strokeColor": "#1971c2", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": { "type": 2 }, "seed": 1289560150, "version": 1, "versionNonce": 2126508551, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 0, 220 ] ], "lastCommittedPoint": null, "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "el0027", "type": "rectangle", "x": 680, "y": 90, "width": 60, "height": 50, "angle": 0, "strokeColor": "#2f9e44", "backgroundColor": "#b2f2bb", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": { "type": 3 }, "seed": 1475216846, "version": 1, "versionNonce": 1927728187, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "el0028", "type": "text", "x": 688.825, "y": 101.25, "width": 42.35, "height": 27.5, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": null, "seed": 1236683273, "version": 1, "versionNonce": 314395343, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "O_i\n(Br, d)", "fontSize": 11, "fontFamily": 1, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "O_i\n(Br, d)", "autoResize": true, "lineHeight": 1.25 }, { "id": "el0029", "type": "text", "x": 660.5, "y": 150, "width": 99.00000000000001, "height": 37.5, "angle": 0, "strokeColor": "#495057", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": null, "seed": 507088657, "version": 1, "versionNonce": 1795823849, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "rescaled by\nexp(m_old - m_new)\nevery tile", "fontSize": 10, "fontFamily": 1, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "rescaled by\nexp(m_old - m_new)\nevery tile", "autoResize": true, "lineHeight": 1.25 }, { "id": "el0030", "type": "rectangle", "x": 130, "y": 200, "width": 400, "height": 110, "angle": 0, "strokeColor": "#495057", "backgroundColor": "#f8f9fa", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": { "type": 3 }, "seed": 708506837, "version": 1, "versionNonce": 1469118511, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "el0031", "type": "text", "x": 184.79999999999998, "y": 206, "width": 290.40000000000003, "height": 15.0, "angle": 0, "strokeColor": "#495057", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": null, "seed": 652768598, "version": 1, "versionNonce": 2100080515, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "resident in SRAM per block, independent of N", "fontSize": 12, "fontFamily": 1, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "resident in SRAM per block, independent of N", "autoResize": true, "lineHeight": 1.25 }, { "id": "el0032", "type": "text", "x": 150, "y": 230, "width": 387.20000000000005, "height": 55.0, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": null, "seed": 1811180650, "version": 1, "versionNonce": 168393880, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "Q_i (Br, d), one K_j^T (d, Bc), one V_j (Bc, d), S (Br, Bc)\nm, l: vectors of length Br O_i (Br, d) accumulator\n\nat Br = Bc = 64, d = 128, BF16 inputs, FP32 S and O: about 80 KB", "fontSize": 11, "fontFamily": 1, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "Q_i (Br, d), one K_j^T (d, Bc), one V_j (Bc, d), S (Br, Bc)\nm, l: vectors of length Br O_i (Br, d) accumulator\n\nat Br = Bc = 64, d = 128, BF16 inputs, FP32 S and O: about 80 KB", "autoResize": true, "lineHeight": 1.25 }, { "id": "el0033", "type": "text", "x": 40, "y": 350, "width": 1062.6000000000001, "height": 15.0, "angle": 0, "strokeColor": "#495057", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": null, "seed": 333377415, "version": 1, "versionNonce": 1347535309, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "The N x N matrix is never materialized: each block holds one (Br, Bc) tile of it at a time. Blocks over different query tiles run independently on different SMs.", "fontSize": 12, "fontFamily": 1, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "The N x N matrix is never materialized: each block holds one (Br, Bc) tile of it at a time. Blocks over different query tiles run independently on different SMs.", "autoResize": true, "lineHeight": 1.25 } ], "appState": { "gridSize": 20, "gridStep": 5, "gridModeEnabled": false, "viewBackgroundColor": "#fdf8f6" }, "files": {} }