{ "type": "excalidraw", "version": 2, "source": "https://excalidraw.com", "elements": [ { "id": "el0001", "type": "text", "x": 20, "y": 10, "width": 686.4000000000001, "height": 30.0, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": null, "seed": 1771794317, "version": 1, "versionNonce": 1610898137, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "Tensor-parallel MLP: Y = GeLU(X A) B on two ranks", "fontSize": 24, "fontFamily": 1, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "Tensor-parallel MLP: Y = GeLU(X A) B on two ranks", "autoResize": true, "lineHeight": 1.25 }, { "id": "el0002", "type": "text", "x": 20, "y": 48, "width": 891.0000000000001, "height": 18.75, "angle": 0, "strokeColor": "#2f9e44", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": null, "seed": 331397559, "version": 1, "versionNonce": 2121234372, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "A split by columns (each rank owns half the 4h outputs), B split by rows (each rank owns half the 4h inputs)", "fontSize": 15, "fontFamily": 1, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "A split by columns (each rank owns half the 4h outputs), B split by rows (each rank owns half the 4h inputs)", "autoResize": true, "lineHeight": 1.25 }, { "id": "el0003", "type": "text", "x": 20, "y": 95, "width": 193.60000000000002, "height": 20.0, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": null, "seed": 745789752, "version": 1, "versionNonce": 307078167, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "the split, as matrices", "fontSize": 16, "fontFamily": 1, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "the split, as matrices", "autoResize": true, "lineHeight": 1.25 }, { "id": "el0004", "type": "rectangle", "x": 20, "y": 125, "width": 60, "height": 90, "angle": 0, "strokeColor": "#1971c2", "backgroundColor": "#a5d8ff", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": { "type": 3 }, "seed": 272738100, "version": 1, "versionNonce": 137640753, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "el0005", "type": "rectangle", "x": 80, "y": 125, "width": 60, "height": 90, "angle": 0, "strokeColor": "#e8590c", "backgroundColor": "#ffd8a8", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": { "type": 3 }, "seed": 1396947072, "version": 1, "versionNonce": 856951296, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "el0006", "type": "text", "x": 41.2, "y": 160, "width": 17.6, "height": 20.0, "angle": 0, "strokeColor": "#1971c2", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": null, "seed": 162985700, "version": 1, "versionNonce": 1642857622, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "A1", "fontSize": 16, "fontFamily": 1, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "A1", "autoResize": true, "lineHeight": 1.25 }, { "id": "el0007", "type": "text", "x": 101.2, "y": 160, "width": 17.6, "height": 20.0, "angle": 0, "strokeColor": "#e8590c", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": null, "seed": 236108019, "version": 1, "versionNonce": 1135245888, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "A2", "fontSize": 16, "fontFamily": 1, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "A2", "autoResize": true, "lineHeight": 1.25 }, { "id": "el0008", "type": "text", "x": 37.099999999999994, "y": 222, "width": 85.80000000000001, "height": 32.5, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": null, "seed": 2037568382, "version": 1, "versionNonce": 1755621363, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "A (h x 4h)\ncolumn split", "fontSize": 13, "fontFamily": 1, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "A (h x 4h)\ncolumn split", "autoResize": true, "lineHeight": 1.25 }, { "id": "el0009", "type": "rectangle", "x": 200, "y": 110, "width": 90, "height": 60, "angle": 0, "strokeColor": "#1971c2", "backgroundColor": "#a5d8ff", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": { "type": 3 }, "seed": 2092065, "version": 1, "versionNonce": 625801321, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "el0010", "type": "rectangle", "x": 200, "y": 170, "width": 90, "height": 60, "angle": 0, "strokeColor": "#e8590c", "backgroundColor": "#ffd8a8", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": { "type": 3 }, "seed": 2083786155, "version": 1, "versionNonce": 1921346457, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "el0011", "type": "text", "x": 236.2, "y": 130, "width": 17.6, "height": 20.0, "angle": 0, "strokeColor": "#1971c2", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": null, "seed": 559218294, "version": 1, "versionNonce": 1089194840, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "B1", "fontSize": 16, "fontFamily": 1, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "B1", "autoResize": true, "lineHeight": 1.25 }, { "id": "el0012", "type": "text", "x": 236.2, "y": 190, "width": 17.6, "height": 20.0, "angle": 0, "strokeColor": "#e8590c", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": null, "seed": 430547703, "version": 1, "versionNonce": 1032685481, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "B2", "fontSize": 16, "fontFamily": 1, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "B2", "autoResize": true, "lineHeight": 1.25 }, { "id": "el0013", "type": "text", "x": 205.675, "y": 240, "width": 78.65, "height": 32.5, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": null, "seed": 1568640917, "version": 1, "versionNonce": 1706833314, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "B (4h x h)\nrow split", "fontSize": 13, "fontFamily": 1, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "B (4h x h)\nrow split", "autoResize": true, "lineHeight": 1.25 }, { "id": "el0014", "type": "rectangle", "x": 340, "y": 125, "width": 70, "height": 90, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#dee2e6", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": { "type": 3 }, "seed": 887284494, "version": 1, "versionNonce": 726986143, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "el0015", "type": "text", "x": 370.6, "y": 160, "width": 8.8, "height": 20.0, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": null, "seed": 626457296, "version": 1, "versionNonce": 310935238, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "X", "fontSize": 16, "fontFamily": 1, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "X", "autoResize": true, "lineHeight": 1.25 }, { "id": "el0016", "type": "text", "x": 321.375, "y": 222, "width": 107.25000000000001, "height": 48.75, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": null, "seed": 370743557, "version": 1, "versionNonce": 1197195974, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "X (tokens x h)\nfull copy on\nevery rank", "fontSize": 13, "fontFamily": 1, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "X (tokens x h)\nfull copy on\nevery rank", "autoResize": true, "lineHeight": 1.25 }, { "id": "el0017", "type": "text", "x": 417.2, "y": 122, "width": 52.800000000000004, "height": 20.0, "angle": 0, "strokeColor": "#1971c2", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": null, "seed": 523183233, "version": 1, "versionNonce": 873736315, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "rank 1", "fontSize": 16, "fontFamily": 1, "textAlign": "right", "verticalAlign": "top", "containerId": null, "originalText": "rank 1", "autoResize": true, "lineHeight": 1.25 }, { "id": "el0018", "type": "rectangle", "x": 480, "y": 100, "width": 60, "height": 60, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#dee2e6", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": { "type": 3 }, "seed": 1493712798, "version": 1, "versionNonce": 877554354, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "el0019", "type": "text", "x": 505.6, "y": 120.0, "width": 8.8, "height": 20.0, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": null, "seed": 1328289717, "version": 1, "versionNonce": 1348006062, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "X", "fontSize": 16, "fontFamily": 1, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "X", "autoResize": true, "lineHeight": 1.25 }, { "id": "el0020", "type": "arrow", "x": 540, "y": 130, "width": 50, "height": 0, "angle": 0, "strokeColor": "#868e96", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": { "type": 2 }, "seed": 2107237701, "version": 1, "versionNonce": 660830310, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 50, 0 ] ], "lastCommittedPoint": null, "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "el0021", "type": "rectangle", "x": 590, "y": 100, "width": 80, "height": 60, "angle": 0, "strokeColor": "#1971c2", "backgroundColor": "#a5d8ff", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": { "type": 3 }, "seed": 913150002, "version": 1, "versionNonce": 129548686, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "el0022", "type": "text", "x": 612.4, "y": 120.0, "width": 35.2, "height": 20.0, "angle": 0, "strokeColor": "#1971c2", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": null, "seed": 1930117755, "version": 1, "versionNonce": 300935830, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "X A1", "fontSize": 16, "fontFamily": 1, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "X A1", "autoResize": true, "lineHeight": 1.25 }, { "id": "el0023", "type": "text", "x": 587.1, "y": 164, "width": 85.80000000000001, "height": 15.0, "angle": 0, "strokeColor": "#1971c2", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": null, "seed": 254108198, "version": 1, "versionNonce": 696036427, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "(tokens x 2h)", "fontSize": 12, "fontFamily": 1, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "(tokens x 2h)", "autoResize": true, "lineHeight": 1.25 }, { "id": "el0024", "type": "arrow", "x": 670, "y": 130, "width": 50, "height": 0, "angle": 0, "strokeColor": "#868e96", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": { "type": 2 }, "seed": 2104724972, "version": 1, "versionNonce": 1321130850, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 50, 0 ] ], "lastCommittedPoint": null, "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "el0025", "type": "rectangle", "x": 720, "y": 100, "width": 80, "height": 60, "angle": 0, "strokeColor": "#2f9e44", "backgroundColor": "#b2f2bb", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": { "type": 3 }, "seed": 1312721111, "version": 1, "versionNonce": 144036457, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "el0026", "type": "text", "x": 742.4, "y": 120.0, "width": 35.2, "height": 20.0, "angle": 0, "strokeColor": "#2f9e44", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": null, "seed": 684393522, "version": 1, "versionNonce": 1279466450, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "GeLU", "fontSize": 16, "fontFamily": 1, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "GeLU", "autoResize": true, "lineHeight": 1.25 }, { "id": "el0027", "type": "text", "x": 720.4, "y": 164, "width": 79.2, "height": 30.0, "angle": 0, "strokeColor": "#2f9e44", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": null, "seed": 1259286311, "version": 1, "versionNonce": 289809888, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "elementwise:\nno sync", "fontSize": 12, "fontFamily": 1, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "elementwise:\nno sync", "autoResize": true, "lineHeight": 1.25 }, { "id": "el0028", "type": "arrow", "x": 800, "y": 130, "width": 50, "height": 0, "angle": 0, "strokeColor": "#868e96", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": { "type": 2 }, "seed": 789378130, "version": 1, "versionNonce": 122295265, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 50, 0 ] ], "lastCommittedPoint": null, "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "el0029", "type": "rectangle", "x": 850, "y": 100, "width": 90, "height": 60, "angle": 0, "strokeColor": "#1971c2", "backgroundColor": "#a5d8ff", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": { "type": 3 }, "seed": 1522518980, "version": 1, "versionNonce": 1717554678, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "el0030", "type": "text", "x": 868.6, "y": 120.0, "width": 52.800000000000004, "height": 20.0, "angle": 0, "strokeColor": "#1971c2", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": null, "seed": 315110289, "version": 1, "versionNonce": 840395618, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "(.) B1", "fontSize": 16, "fontFamily": 1, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "(.) B1", "autoResize": true, "lineHeight": 1.25 }, { "id": "el0031", "type": "text", "x": 858.7, "y": 164, "width": 72.60000000000001, "height": 15.0, "angle": 0, "strokeColor": "#1971c2", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": null, "seed": 412910240, "version": 1, "versionNonce": 297251411, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "partial sum", "fontSize": 12, "fontFamily": 1, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "partial sum", "autoResize": true, "lineHeight": 1.25 }, { "id": "el0032", "type": "arrow", "x": 940, "y": 130, "width": 50, "height": 0, "angle": 0, "strokeColor": "#868e96", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": { "type": 2 }, "seed": 1151796270, "version": 1, "versionNonce": 2051970771, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 50, 0 ] ], "lastCommittedPoint": null, "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "el0033", "type": "rectangle", "x": 990, "y": 100, "width": 60, "height": 60, "angle": 0, "strokeColor": "#1971c2", "backgroundColor": "#a5d8ff", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": { "type": 3 }, "seed": 1330070683, "version": 1, "versionNonce": 47268483, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "el0034", "type": "text", "x": 1011.2, "y": 120.0, "width": 17.6, "height": 20.0, "angle": 0, "strokeColor": "#1971c2", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": null, "seed": 1418128451, "version": 1, "versionNonce": 1369318616, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "Y1", "fontSize": 16, "fontFamily": 1, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "Y1", "autoResize": true, "lineHeight": 1.25 }, { "id": "el0035", "type": "text", "x": 417.2, "y": 252, "width": 52.800000000000004, "height": 20.0, "angle": 0, "strokeColor": "#e8590c", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": null, "seed": 1814954122, "version": 1, "versionNonce": 200843529, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "rank 2", "fontSize": 16, "fontFamily": 1, "textAlign": "right", "verticalAlign": "top", "containerId": null, "originalText": "rank 2", "autoResize": true, "lineHeight": 1.25 }, { "id": "el0036", "type": "rectangle", "x": 480, "y": 230, "width": 60, "height": 60, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#dee2e6", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": { "type": 3 }, "seed": 1511242350, "version": 1, "versionNonce": 1434729413, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "el0037", "type": "text", "x": 505.6, "y": 250.0, "width": 8.8, "height": 20.0, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": null, "seed": 996368890, "version": 1, "versionNonce": 520443306, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "X", "fontSize": 16, "fontFamily": 1, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "X", "autoResize": true, "lineHeight": 1.25 }, { "id": "el0038", "type": "arrow", "x": 540, "y": 260, "width": 50, "height": 0, "angle": 0, "strokeColor": "#868e96", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": { "type": 2 }, "seed": 1293945008, "version": 1, "versionNonce": 512364511, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 50, 0 ] ], "lastCommittedPoint": null, "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "el0039", "type": "rectangle", "x": 590, "y": 230, "width": 80, "height": 60, "angle": 0, "strokeColor": "#e8590c", "backgroundColor": "#ffd8a8", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": { "type": 3 }, "seed": 38868265, "version": 1, "versionNonce": 770932384, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "el0040", "type": "text", "x": 612.4, "y": 250.0, "width": 35.2, "height": 20.0, "angle": 0, "strokeColor": "#e8590c", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": null, "seed": 288568381, "version": 1, "versionNonce": 1195854406, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "X A2", "fontSize": 16, "fontFamily": 1, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "X A2", "autoResize": true, "lineHeight": 1.25 }, { "id": "el0041", "type": "text", "x": 587.1, "y": 294, "width": 85.80000000000001, "height": 15.0, "angle": 0, "strokeColor": "#e8590c", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": null, "seed": 2075648338, "version": 1, "versionNonce": 506180323, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "(tokens x 2h)", "fontSize": 12, "fontFamily": 1, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "(tokens x 2h)", "autoResize": true, "lineHeight": 1.25 }, { "id": "el0042", "type": "arrow", "x": 670, "y": 260, "width": 50, "height": 0, "angle": 0, "strokeColor": "#868e96", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": { "type": 2 }, "seed": 1712619377, "version": 1, "versionNonce": 156650592, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 50, 0 ] ], "lastCommittedPoint": null, "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "el0043", "type": "rectangle", "x": 720, "y": 230, "width": 80, "height": 60, "angle": 0, "strokeColor": "#2f9e44", "backgroundColor": "#b2f2bb", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": { "type": 3 }, "seed": 2090655203, "version": 1, "versionNonce": 1783722425, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "el0044", "type": "text", "x": 742.4, "y": 250.0, "width": 35.2, "height": 20.0, "angle": 0, "strokeColor": "#2f9e44", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": null, "seed": 577675681, "version": 1, "versionNonce": 502732955, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "GeLU", "fontSize": 16, "fontFamily": 1, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "GeLU", "autoResize": true, "lineHeight": 1.25 }, { "id": "el0045", "type": "text", "x": 720.4, "y": 294, "width": 79.2, "height": 30.0, "angle": 0, "strokeColor": "#2f9e44", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": null, "seed": 719291063, "version": 1, "versionNonce": 1336714650, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "elementwise:\nno sync", "fontSize": 12, "fontFamily": 1, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "elementwise:\nno sync", "autoResize": true, "lineHeight": 1.25 }, { "id": "el0046", "type": "arrow", "x": 800, "y": 260, "width": 50, "height": 0, "angle": 0, "strokeColor": "#868e96", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": { "type": 2 }, "seed": 405106597, "version": 1, "versionNonce": 1811779595, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 50, 0 ] ], "lastCommittedPoint": null, "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "el0047", "type": "rectangle", "x": 850, "y": 230, "width": 90, "height": 60, "angle": 0, "strokeColor": "#e8590c", "backgroundColor": "#ffd8a8", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": { "type": 3 }, "seed": 347913926, "version": 1, "versionNonce": 494716672, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "el0048", "type": "text", "x": 868.6, "y": 250.0, "width": 52.800000000000004, "height": 20.0, "angle": 0, "strokeColor": "#e8590c", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": null, "seed": 1515714011, "version": 1, "versionNonce": 1012007554, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "(.) B2", "fontSize": 16, "fontFamily": 1, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "(.) B2", "autoResize": true, "lineHeight": 1.25 }, { "id": "el0049", "type": "text", "x": 858.7, "y": 294, "width": 72.60000000000001, "height": 15.0, "angle": 0, "strokeColor": "#e8590c", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": null, "seed": 1477751378, "version": 1, "versionNonce": 183638333, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "partial sum", "fontSize": 12, "fontFamily": 1, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "partial sum", "autoResize": true, "lineHeight": 1.25 }, { "id": "el0050", "type": "arrow", "x": 940, "y": 260, "width": 50, "height": 0, "angle": 0, "strokeColor": "#868e96", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": { "type": 2 }, "seed": 129022301, "version": 1, "versionNonce": 469755974, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 50, 0 ] ], "lastCommittedPoint": null, "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "el0051", "type": "rectangle", "x": 990, "y": 230, "width": 60, "height": 60, "angle": 0, "strokeColor": "#e8590c", "backgroundColor": "#ffd8a8", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": { "type": 3 }, "seed": 330619942, "version": 1, "versionNonce": 1124287796, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "el0052", "type": "text", "x": 1011.2, "y": 250.0, "width": 17.6, "height": 20.0, "angle": 0, "strokeColor": "#e8590c", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": null, "seed": 1297110296, "version": 1, "versionNonce": 297814128, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "Y2", "fontSize": 16, "fontFamily": 1, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "Y2", "autoResize": true, "lineHeight": 1.25 }, { "id": "el0053", "type": "arrow", "x": 1050, "y": 130, "width": 50, "height": 65, "angle": 0, "strokeColor": "#868e96", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": { "type": 2 }, "seed": 2013241217, "version": 1, "versionNonce": 91003118, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 50, 0 ], [ 50, 65 ] ], "lastCommittedPoint": null, "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "el0054", "type": "arrow", "x": 1050, "y": 260, "width": 50, "height": 65, "angle": 0, "strokeColor": "#868e96", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": { "type": 2 }, "seed": 263086734, "version": 1, "versionNonce": 150650980, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 50, 0 ], [ 50, -65 ] ], "lastCommittedPoint": null, "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": null, "elbowed": false }, { "id": "el0055", "type": "rectangle", "x": 1100, "y": 165, "width": 120, "height": 60, "angle": 0, "strokeColor": "#e03131", "backgroundColor": "#ffc9c9", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": { "type": 3 }, "seed": 2080678479, "version": 1, "versionNonce": 1126008343, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "el0056", "type": "text", "x": 1117.65, "y": 177.5, "width": 84.7, "height": 35.0, "angle": 0, "strokeColor": "#e03131", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": null, "seed": 874970870, "version": 1, "versionNonce": 1518062352, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "all-reduce\nY = Y1 + Y2", "fontSize": 14, "fontFamily": 1, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "all-reduce\nY = Y1 + Y2", "autoResize": true, "lineHeight": 1.25 }, { "id": "el0057", "type": "arrow", "x": 1220, "y": 195, "width": 50, "height": 0, "angle": 0, "strokeColor": "#868e96", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": { "type": 2 }, "seed": 62206169, "version": 1, "versionNonce": 1116113553, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 50, 0 ] ], "lastCommittedPoint": null, "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "el0058", "type": "rectangle", "x": 1270, "y": 165, "width": 60, "height": 60, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#dee2e6", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": { "type": 3 }, "seed": 353801247, "version": 1, "versionNonce": 1269847415, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "el0059", "type": "text", "x": 1295.6, "y": 185.0, "width": 8.8, "height": 20.0, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": null, "seed": 1048274891, "version": 1, "versionNonce": 1796259239, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "Y", "fontSize": 16, "fontFamily": 1, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "Y", "autoResize": true, "lineHeight": 1.25 }, { "id": "el0060", "type": "text", "x": 1267.0, "y": 230, "width": 66.0, "height": 30.0, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": null, "seed": 2041994690, "version": 1, "versionNonce": 1981961362, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "full Y on\nboth ranks", "fontSize": 12, "fontFamily": 1, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "full Y on\nboth ranks", "autoResize": true, "lineHeight": 1.25 }, { "id": "el0061", "type": "text", "x": 503.95, "y": 305, "width": 12.100000000000001, "height": 27.5, "angle": 0, "strokeColor": "#7048e8", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": null, "seed": 1811524741, "version": 1, "versionNonce": 301855577, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "f", "fontSize": 22, "fontFamily": 1, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "f", "autoResize": true, "lineHeight": 1.25 }, { "id": "el0062", "type": "text", "x": 424.2, "y": 332, "width": 171.60000000000002, "height": 45.0, "angle": 0, "strokeColor": "#7048e8", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": null, "seed": 1255679756, "version": 1, "versionNonce": 650133667, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "forward: identity\nbackward: all-reduce dL/dX\n(X was used by both ranks)", "fontSize": 12, "fontFamily": 1, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "forward: identity\nbackward: all-reduce dL/dX\n(X was used by both ranks)", "autoResize": true, "lineHeight": 1.25 }, { "id": "el0063", "type": "text", "x": 1153.95, "y": 305, "width": 12.100000000000001, "height": 27.5, "angle": 0, "strokeColor": "#7048e8", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": null, "seed": 394164295, "version": 1, "versionNonce": 1772618230, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "g", "fontSize": 22, "fontFamily": 1, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "g", "autoResize": true, "lineHeight": 1.25 }, { "id": "el0064", "type": "text", "x": 1077.5, "y": 332, "width": 165.0, "height": 45.0, "angle": 0, "strokeColor": "#7048e8", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": null, "seed": 1614948864, "version": 1, "versionNonce": 1268466631, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "forward: all-reduce\nbackward: identity\n(dL/dY1 = dL/dY2 = dL/dY)", "fontSize": 12, "fontFamily": 1, "textAlign": "center", "verticalAlign": "top", "containerId": null, "originalText": "forward: all-reduce\nbackward: identity\n(dL/dY1 = dL/dY2 = dL/dY)", "autoResize": true, "lineHeight": 1.25 }, { "id": "el0065", "type": "text", "x": 20, "y": 400, "width": 839.3000000000001, "height": 35.0, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": null, "seed": 524684855, "version": 1, "versionNonce": 867435930, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "Why not split A by rows? X A = X1 A1 + X2 A2 would have to be summed BEFORE the GeLU,\nbecause GeLU(X1 A1 + X2 A2) is not GeLU(X1 A1) + GeLU(X2 A2). The column split puts the only sync at the end.", "fontSize": 14, "fontFamily": 1, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "Why not split A by rows? X A = X1 A1 + X2 A2 would have to be summed BEFORE the GeLU,\nbecause GeLU(X1 A1 + X2 A2) is not GeLU(X1 A1) + GeLU(X2 A2). The column split puts the only sync at the end.", "autoResize": true, "lineHeight": 1.25 }, { "id": "el0066", "type": "text", "x": 20, "y": 450, "width": 908.6, "height": 17.5, "angle": 0, "strokeColor": "#2f9e44", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 2, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "index": null, "roundness": null, "seed": 657417466, "version": 1, "versionNonce": 1651355327, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "Per MLP block: one all-reduce forward (g), one backward (f). Attention has the same shape, so 4 all-reduces per layer.", "fontSize": 14, "fontFamily": 1, "textAlign": "left", "verticalAlign": "top", "containerId": null, "originalText": "Per MLP block: one all-reduce forward (g), one backward (f). Attention has the same shape, so 4 all-reduces per layer.", "autoResize": true, "lineHeight": 1.25 } ], "appState": { "gridSize": 20, "gridStep": 5, "gridModeEnabled": false, "viewBackgroundColor": "#ffffff" }, "files": {} }