{
 "type": "excalidraw",
 "version": 2,
 "source": "https://excalidraw.com",
 "elements": [
  {
   "id": "el0001",
   "type": "text",
   "x": 20,
   "y": 10,
   "width": 1234.2,
   "height": 27.5,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 472702160,
   "version": 1,
   "versionNonce": 291810695,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "Context parallelism: the sequence is split across ranks; attention brings K and V together   (4 ranks)",
   "fontSize": 22,
   "fontFamily": 1,
   "textAlign": "left",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "Context parallelism: the sequence is split across ranks; attention brings K and V together   (4 ranks)",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0002",
   "type": "text",
   "x": 20,
   "y": 44,
   "width": 862.4000000000001,
   "height": 17.5,
   "angle": 0,
   "strokeColor": "#2f9e44",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 581533130,
   "version": 1,
   "versionNonce": 61468447,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "every other op is per-token and runs unchanged on a chunk; only attention needs the other ranks' keys and values",
   "fontSize": 14,
   "fontFamily": 1,
   "textAlign": "left",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "every other op is per-token and runs unchanged on a chunk; only attention needs the other ranks' keys and values",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0003",
   "type": "text",
   "x": 20,
   "y": 92,
   "width": 154.0,
   "height": 17.5,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1852813448,
   "version": 1,
   "versionNonce": 1459490934,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "sequence of s tokens",
   "fontSize": 14,
   "fontFamily": 1,
   "textAlign": "left",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "sequence of s tokens",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0004",
   "type": "rectangle",
   "x": 190,
   "y": 85,
   "width": 148,
   "height": 34,
   "angle": 0,
   "strokeColor": "#1971c2",
   "backgroundColor": "#a5d8ff",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 474040130,
   "version": 1,
   "versionNonce": 1586311449,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0005",
   "type": "text",
   "x": 194.7,
   "y": 93,
   "width": 138.60000000000002,
   "height": 15.0,
   "angle": 0,
   "strokeColor": "#1971c2",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 364914729,
   "version": 1,
   "versionNonce": 1381448837,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "chunk 0  (s/4 tokens)",
   "fontSize": 12,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "chunk 0  (s/4 tokens)",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0006",
   "type": "rectangle",
   "x": 340,
   "y": 85,
   "width": 148,
   "height": 34,
   "angle": 0,
   "strokeColor": "#e8590c",
   "backgroundColor": "#ffd8a8",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 521309656,
   "version": 1,
   "versionNonce": 54269625,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0007",
   "type": "text",
   "x": 344.7,
   "y": 93,
   "width": 138.60000000000002,
   "height": 15.0,
   "angle": 0,
   "strokeColor": "#e8590c",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 810142982,
   "version": 1,
   "versionNonce": 870860192,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "chunk 1  (s/4 tokens)",
   "fontSize": 12,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "chunk 1  (s/4 tokens)",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0008",
   "type": "rectangle",
   "x": 490,
   "y": 85,
   "width": 148,
   "height": 34,
   "angle": 0,
   "strokeColor": "#2f9e44",
   "backgroundColor": "#b2f2bb",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 714288432,
   "version": 1,
   "versionNonce": 866801548,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0009",
   "type": "text",
   "x": 494.7,
   "y": 93,
   "width": 138.60000000000002,
   "height": 15.0,
   "angle": 0,
   "strokeColor": "#2f9e44",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1297091983,
   "version": 1,
   "versionNonce": 447829456,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "chunk 2  (s/4 tokens)",
   "fontSize": 12,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "chunk 2  (s/4 tokens)",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0010",
   "type": "rectangle",
   "x": 640,
   "y": 85,
   "width": 148,
   "height": 34,
   "angle": 0,
   "strokeColor": "#7048e8",
   "backgroundColor": "#d0bfff",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 320006446,
   "version": 1,
   "versionNonce": 1955691852,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0011",
   "type": "text",
   "x": 644.7,
   "y": 93,
   "width": 138.60000000000002,
   "height": 15.0,
   "angle": 0,
   "strokeColor": "#7048e8",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 745174407,
   "version": 1,
   "versionNonce": 522724260,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "chunk 3  (s/4 tokens)",
   "fontSize": 12,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "chunk 3  (s/4 tokens)",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0012",
   "type": "text",
   "x": 209.5,
   "y": 124,
   "width": 561.0,
   "height": 15.0,
   "angle": 0,
   "strokeColor": "#2f9e44",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1153361532,
   "version": 1,
   "versionNonce": 1347152861,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "chunk i lives on rank i for every layer: its Q_i, K_i, V_i and all of its activations",
   "fontSize": 12,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "chunk i lives on rank i for every layer: its Q_i, K_i, V_i and all of its activations",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0013",
   "type": "text",
   "x": 20,
   "y": 170,
   "width": 118.80000000000001,
   "height": 22.5,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1832165780,
   "version": 1,
   "versionNonce": 874644406,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "Ring variant",
   "fontSize": 18,
   "fontFamily": 1,
   "textAlign": "left",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "Ring variant",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0014",
   "type": "text",
   "x": 20,
   "y": 196,
   "width": 198.00000000000003,
   "height": 15.0,
   "angle": 0,
   "strokeColor": "#868e96",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 235794730,
   "version": 1,
   "versionNonce": 907037888,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "(Ring Attention; Megatron p2p)",
   "fontSize": 12,
   "fontFamily": 1,
   "textAlign": "left",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "(Ring Attention; Megatron p2p)",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0015",
   "type": "rectangle",
   "x": 245.0,
   "y": 209.0,
   "width": 110,
   "height": 52,
   "angle": 0,
   "strokeColor": "#1971c2",
   "backgroundColor": "#a5d8ff",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 1045420615,
   "version": 1,
   "versionNonce": 2067821540,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0016",
   "type": "text",
   "x": 280.2,
   "y": 214.0,
   "width": 39.6,
   "height": 15.0,
   "angle": 0,
   "strokeColor": "#1971c2",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 889990869,
   "version": 1,
   "versionNonce": 179468053,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "rank 0",
   "fontSize": 12,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "rank 0",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0017",
   "type": "text",
   "x": 264.25,
   "y": 230.0,
   "width": 71.5,
   "height": 16.25,
   "angle": 0,
   "strokeColor": "#1971c2",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1625562413,
   "version": 1,
   "versionNonce": 983762456,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "Q0   K0 V0",
   "fontSize": 13,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "Q0   K0 V0",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0018",
   "type": "rectangle",
   "x": 350.0,
   "y": 314.0,
   "width": 110,
   "height": 52,
   "angle": 0,
   "strokeColor": "#e8590c",
   "backgroundColor": "#ffd8a8",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 187138019,
   "version": 1,
   "versionNonce": 1907915885,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0019",
   "type": "text",
   "x": 385.2,
   "y": 319.0,
   "width": 39.6,
   "height": 15.0,
   "angle": 0,
   "strokeColor": "#e8590c",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 780146022,
   "version": 1,
   "versionNonce": 187596408,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "rank 1",
   "fontSize": 12,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "rank 1",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0020",
   "type": "text",
   "x": 369.25,
   "y": 335.0,
   "width": 71.5,
   "height": 16.25,
   "angle": 0,
   "strokeColor": "#e8590c",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1391400372,
   "version": 1,
   "versionNonce": 844398025,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "Q1   K1 V1",
   "fontSize": 13,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "Q1   K1 V1",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0021",
   "type": "rectangle",
   "x": 245.0,
   "y": 419.0,
   "width": 110,
   "height": 52,
   "angle": 0,
   "strokeColor": "#2f9e44",
   "backgroundColor": "#b2f2bb",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 613134293,
   "version": 1,
   "versionNonce": 133910568,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0022",
   "type": "text",
   "x": 280.2,
   "y": 424.0,
   "width": 39.6,
   "height": 15.0,
   "angle": 0,
   "strokeColor": "#2f9e44",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1451266993,
   "version": 1,
   "versionNonce": 426844605,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "rank 2",
   "fontSize": 12,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "rank 2",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0023",
   "type": "text",
   "x": 264.25,
   "y": 440.0,
   "width": 71.5,
   "height": 16.25,
   "angle": 0,
   "strokeColor": "#2f9e44",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 358756423,
   "version": 1,
   "versionNonce": 1266985459,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "Q2   K2 V2",
   "fontSize": 13,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "Q2   K2 V2",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0024",
   "type": "rectangle",
   "x": 140.0,
   "y": 314.0,
   "width": 110,
   "height": 52,
   "angle": 0,
   "strokeColor": "#7048e8",
   "backgroundColor": "#d0bfff",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 99053356,
   "version": 1,
   "versionNonce": 2075013879,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0025",
   "type": "text",
   "x": 175.2,
   "y": 319.0,
   "width": 39.6,
   "height": 15.0,
   "angle": 0,
   "strokeColor": "#7048e8",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1499340982,
   "version": 1,
   "versionNonce": 1288580076,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "rank 3",
   "fontSize": 12,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "rank 3",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0026",
   "type": "text",
   "x": 159.25,
   "y": 335.0,
   "width": 71.5,
   "height": 16.25,
   "angle": 0,
   "strokeColor": "#7048e8",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1201758954,
   "version": 1,
   "versionNonce": 1600609289,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "Q3   K3 V3",
   "fontSize": 13,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "Q3   K3 V3",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0027",
   "type": "arrow",
   "x": 346.6690475583121,
   "y": 281.6690475583121,
   "width": 11.661904883375769,
   "height": 11.661904883375769,
   "angle": 0,
   "strokeColor": "#e03131",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 2
   },
   "seed": 1815322730,
   "version": 1,
   "versionNonce": 1023051530,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "points": [
    [
     0.0,
     0.0
    ],
    [
     11.661904883375769,
     11.661904883375769
    ]
   ],
   "lastCommittedPoint": null,
   "startBinding": null,
   "endBinding": null,
   "startArrowhead": null,
   "endArrowhead": "arrow",
   "elbowed": false
  },
  {
   "id": "el0028",
   "type": "arrow",
   "x": 358.3309524416879,
   "y": 386.6690475583121,
   "width": 11.661904883375769,
   "height": 11.661904883375769,
   "angle": 0,
   "strokeColor": "#e03131",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 2
   },
   "seed": 497573847,
   "version": 1,
   "versionNonce": 663021024,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "points": [
    [
     0.0,
     0.0
    ],
    [
     -11.661904883375769,
     11.661904883375769
    ]
   ],
   "lastCommittedPoint": null,
   "startBinding": null,
   "endBinding": null,
   "startArrowhead": null,
   "endArrowhead": "arrow",
   "elbowed": false
  },
  {
   "id": "el0029",
   "type": "arrow",
   "x": 253.33095244168788,
   "y": 398.3309524416879,
   "width": 11.661904883375769,
   "height": 11.661904883375769,
   "angle": 0,
   "strokeColor": "#e03131",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 2
   },
   "seed": 127749398,
   "version": 1,
   "versionNonce": 1126452690,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "points": [
    [
     0.0,
     0.0
    ],
    [
     -11.661904883375769,
     -11.661904883375769
    ]
   ],
   "lastCommittedPoint": null,
   "startBinding": null,
   "endBinding": null,
   "startArrowhead": null,
   "endArrowhead": "arrow",
   "elbowed": false
  },
  {
   "id": "el0030",
   "type": "arrow",
   "x": 241.66904755831212,
   "y": 293.3309524416879,
   "width": 11.661904883375769,
   "height": 11.661904883375769,
   "angle": 0,
   "strokeColor": "#e03131",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 2
   },
   "seed": 579639208,
   "version": 1,
   "versionNonce": 1604767600,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "points": [
    [
     0.0,
     0.0
    ],
    [
     11.661904883375769,
     -11.661904883375769
    ]
   ],
   "lastCommittedPoint": null,
   "startBinding": null,
   "endBinding": null,
   "startArrowhead": null,
   "endArrowhead": "arrow",
   "elbowed": false
  },
  {
   "id": "el0031",
   "type": "text",
   "x": 250.5,
   "y": 328,
   "width": 99.00000000000001,
   "height": 30.0,
   "angle": 0,
   "strokeColor": "#e03131",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 2033369193,
   "version": 1,
   "versionNonce": 520377131,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "K,V blocks\nrotate: 3 steps",
   "fontSize": 12,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "K,V blocks\nrotate: 3 steps",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0032",
   "type": "text",
   "x": 20,
   "y": 480,
   "width": 468.6,
   "height": 60.0,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1646097544,
   "version": 1,
   "versionNonce": 1378483512,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "step 0: rank i attends Q_i against its own K_i, V_i\nsteps 1 to 3: receive the next K,V block, attend against it, pass it on\nrunning max and denominator rescaled as in online softmax\nthe transfer of block j+1 overlaps the attention on block j",
   "fontSize": 12,
   "fontFamily": 1,
   "textAlign": "left",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "step 0: rank i attends Q_i against its own K_i, V_i\nsteps 1 to 3: receive the next K,V block, attend against it, pass it on\nrunning max and denominator rescaled as in online softmax\nthe transfer of block j+1 overlaps the attention on block j",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0033",
   "type": "text",
   "x": 560,
   "y": 170,
   "width": 178.20000000000002,
   "height": 22.5,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 818625084,
   "version": 1,
   "versionNonce": 1094334474,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "All-gather variant",
   "fontSize": 18,
   "fontFamily": 1,
   "textAlign": "left",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "All-gather variant",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0034",
   "type": "text",
   "x": 560,
   "y": 196,
   "width": 198.00000000000003,
   "height": 15.0,
   "angle": 0,
   "strokeColor": "#868e96",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1196929507,
   "version": 1,
   "versionNonce": 178648647,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "(Llama 3; Megatron all_gather)",
   "fontSize": 12,
   "fontFamily": 1,
   "textAlign": "left",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "(Llama 3; Megatron all_gather)",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0035",
   "type": "rectangle",
   "x": 560,
   "y": 230,
   "width": 85,
   "height": 40,
   "angle": 0,
   "strokeColor": "#1971c2",
   "backgroundColor": "#a5d8ff",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 346245595,
   "version": 1,
   "versionNonce": 890976047,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0036",
   "type": "text",
   "x": 584.125,
   "y": 241,
   "width": 35.75,
   "height": 16.25,
   "angle": 0,
   "strokeColor": "#1971c2",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 432808586,
   "version": 1,
   "versionNonce": 1461153561,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "K0 V0",
   "fontSize": 13,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "K0 V0",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0037",
   "type": "rectangle",
   "x": 655,
   "y": 230,
   "width": 85,
   "height": 40,
   "angle": 0,
   "strokeColor": "#e8590c",
   "backgroundColor": "#ffd8a8",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 1455884689,
   "version": 1,
   "versionNonce": 732273148,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0038",
   "type": "text",
   "x": 679.125,
   "y": 241,
   "width": 35.75,
   "height": 16.25,
   "angle": 0,
   "strokeColor": "#e8590c",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 239847227,
   "version": 1,
   "versionNonce": 574173551,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "K1 V1",
   "fontSize": 13,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "K1 V1",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0039",
   "type": "rectangle",
   "x": 750,
   "y": 230,
   "width": 85,
   "height": 40,
   "angle": 0,
   "strokeColor": "#2f9e44",
   "backgroundColor": "#b2f2bb",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 1782841896,
   "version": 1,
   "versionNonce": 2047949942,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0040",
   "type": "text",
   "x": 774.125,
   "y": 241,
   "width": 35.75,
   "height": 16.25,
   "angle": 0,
   "strokeColor": "#2f9e44",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1895827105,
   "version": 1,
   "versionNonce": 271720859,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "K2 V2",
   "fontSize": 13,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "K2 V2",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0041",
   "type": "rectangle",
   "x": 845,
   "y": 230,
   "width": 85,
   "height": 40,
   "angle": 0,
   "strokeColor": "#7048e8",
   "backgroundColor": "#d0bfff",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 1579341239,
   "version": 1,
   "versionNonce": 528159408,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0042",
   "type": "text",
   "x": 869.125,
   "y": 241,
   "width": 35.75,
   "height": 16.25,
   "angle": 0,
   "strokeColor": "#7048e8",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 2042887976,
   "version": 1,
   "versionNonce": 705054650,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "K3 V3",
   "fontSize": 13,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "K3 V3",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0043",
   "type": "arrow",
   "x": 745,
   "y": 270,
   "width": 0,
   "height": 45,
   "angle": 0,
   "strokeColor": "#e03131",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 2
   },
   "seed": 719992590,
   "version": 1,
   "versionNonce": 547918831,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "points": [
    [
     0,
     0
    ],
    [
     0,
     45
    ]
   ],
   "lastCommittedPoint": null,
   "startBinding": null,
   "endBinding": null,
   "startArrowhead": null,
   "endArrowhead": "arrow",
   "elbowed": false
  },
  {
   "id": "el0044",
   "type": "text",
   "x": 758,
   "y": 282,
   "width": 217.8,
   "height": 15.0,
   "angle": 0,
   "strokeColor": "#e03131",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 371723808,
   "version": 1,
   "versionNonce": 838489609,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "all-gather K, V (small under GQA)",
   "fontSize": 12,
   "fontFamily": 1,
   "textAlign": "left",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "all-gather K, V (small under GQA)",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0045",
   "type": "rectangle",
   "x": 560,
   "y": 315,
   "width": 370,
   "height": 40,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "#dee2e6",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 105491074,
   "version": 1,
   "versionNonce": 1233969861,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0046",
   "type": "text",
   "x": 587.7,
   "y": 326,
   "width": 314.6,
   "height": 16.25,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 542766288,
   "version": 1,
   "versionNonce": 1108307326,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "every rank: full K, V for the whole sequence",
   "fontSize": 13,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "every rank: full K, V for the whole sequence",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0047",
   "type": "arrow",
   "x": 745,
   "y": 355,
   "width": 0,
   "height": 40,
   "angle": 0,
   "strokeColor": "#868e96",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 2
   },
   "seed": 547069279,
   "version": 1,
   "versionNonce": 398549576,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "points": [
    [
     0,
     0
    ],
    [
     0,
     40
    ]
   ],
   "lastCommittedPoint": null,
   "startBinding": null,
   "endBinding": null,
   "startArrowhead": null,
   "endArrowhead": "arrow",
   "elbowed": false
  },
  {
   "id": "el0048",
   "type": "rectangle",
   "x": 560,
   "y": 395,
   "width": 370,
   "height": 40,
   "angle": 0,
   "strokeColor": "#2f9e44",
   "backgroundColor": "#b2f2bb",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 2122451966,
   "version": 1,
   "versionNonce": 525242227,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0049",
   "type": "text",
   "x": 580.55,
   "y": 406,
   "width": 328.90000000000003,
   "height": 16.25,
   "angle": 0,
   "strokeColor": "#2f9e44",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1976000270,
   "version": 1,
   "versionNonce": 920194676,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "rank i: attention(Q_i, K_all, V_all), any mask",
   "fontSize": 13,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "rank i: attention(Q_i, K_all, V_all), any mask",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0050",
   "type": "text",
   "x": 560,
   "y": 480,
   "width": 462.00000000000006,
   "height": 60.0,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 965184143,
   "version": 1,
   "versionNonce": 823904405,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "latency exposed but small: attention is O(s^2), the gather O(s)\ncausal balance: cut into 2c chunks, rank i gets chunks i and 2c-1-i\nall-to-all (Ulysses) variant instead: re-shard to heads, attend on the\nfull sequence for 1/c of the heads, re-shard back",
   "fontSize": 12,
   "fontFamily": 1,
   "textAlign": "left",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "latency exposed but small: attention is O(s^2), the gather O(s)\ncausal balance: cut into 2c chunks, rank i gets chunks i and 2c-1-i\nall-to-all (Ulysses) variant instead: re-shard to heads, attend on the\nfull sequence for 1/c of the heads, re-shard back",
   "autoResize": true,
   "lineHeight": 1.25
  }
 ],
 "appState": {
  "gridSize": 20,
  "gridStep": 5,
  "gridModeEnabled": false,
  "viewBackgroundColor": "#ffffff"
 },
 "files": {}
}