{
 "type": "excalidraw",
 "version": 2,
 "source": "https://excalidraw.com",
 "elements": [
  {
   "id": "el0067",
   "type": "text",
   "x": 20,
   "y": 10,
   "width": 844.8000000000001,
   "height": 30.0,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 478541269,
   "version": 1,
   "versionNonce": 743878023,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "Tensor-parallel attention: split by heads   (4 heads, two ranks)",
   "fontSize": 24,
   "fontFamily": 1,
   "textAlign": "left",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "Tensor-parallel attention: split by heads   (4 heads, two ranks)",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0068",
   "type": "text",
   "x": 20,
   "y": 48,
   "width": 981.7500000000001,
   "height": 18.75,
   "angle": 0,
   "strokeColor": "#2f9e44",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 718272210,
   "version": 1,
   "versionNonce": 1420501013,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "Q, K, V projections split by columns so each rank owns whole heads; the output projection split by rows; one all-reduce",
   "fontSize": 15,
   "fontFamily": 1,
   "textAlign": "left",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "Q, K, V projections split by columns so each rank owns whole heads; the output projection split by rows; one all-reduce",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0069",
   "type": "text",
   "x": 57.199999999999996,
   "y": 122,
   "width": 52.800000000000004,
   "height": 20.0,
   "angle": 0,
   "strokeColor": "#1971c2",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1523385535,
   "version": 1,
   "versionNonce": 77690241,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "rank 1",
   "fontSize": 16,
   "fontFamily": 1,
   "textAlign": "right",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "rank 1",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0070",
   "type": "rectangle",
   "x": 120,
   "y": 100,
   "width": 60,
   "height": 60,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "#dee2e6",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 205332454,
   "version": 1,
   "versionNonce": 2145572132,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0071",
   "type": "text",
   "x": 145.6,
   "y": 120.0,
   "width": 8.8,
   "height": 20.0,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 939292003,
   "version": 1,
   "versionNonce": 1850031758,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "X",
   "fontSize": 16,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "X",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0072",
   "type": "arrow",
   "x": 180,
   "y": 130,
   "width": 50,
   "height": 0,
   "angle": 0,
   "strokeColor": "#868e96",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 2
   },
   "seed": 2109618383,
   "version": 1,
   "versionNonce": 1028844885,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "points": [
    [
     0,
     0
    ],
    [
     50,
     0
    ]
   ],
   "lastCommittedPoint": null,
   "startBinding": null,
   "endBinding": null,
   "startArrowhead": null,
   "endArrowhead": "arrow",
   "elbowed": false
  },
  {
   "id": "el0073",
   "type": "rectangle",
   "x": 230,
   "y": 100,
   "width": 110,
   "height": 60,
   "angle": 0,
   "strokeColor": "#1971c2",
   "backgroundColor": "#a5d8ff",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 830519272,
   "version": 1,
   "versionNonce": 748644894,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0074",
   "type": "text",
   "x": 245.675,
   "y": 113.75,
   "width": 78.65,
   "height": 32.5,
   "angle": 0,
   "strokeColor": "#1971c2",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1355857005,
   "version": 1,
   "versionNonce": 697724755,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "W_Q W_K W_V\nheads 1, 2",
   "fontSize": 13,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "W_Q W_K W_V\nheads 1, 2",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0075",
   "type": "text",
   "x": 245.4,
   "y": 164,
   "width": 79.2,
   "height": 15.0,
   "angle": 0,
   "strokeColor": "#1971c2",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 395484037,
   "version": 1,
   "versionNonce": 621553333,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "column split",
   "fontSize": 12,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "column split",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0076",
   "type": "arrow",
   "x": 340,
   "y": 130,
   "width": 50,
   "height": 0,
   "angle": 0,
   "strokeColor": "#868e96",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 2
   },
   "seed": 308095559,
   "version": 1,
   "versionNonce": 715189010,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "points": [
    [
     0,
     0
    ],
    [
     50,
     0
    ]
   ],
   "lastCommittedPoint": null,
   "startBinding": null,
   "endBinding": null,
   "startArrowhead": null,
   "endArrowhead": "arrow",
   "elbowed": false
  },
  {
   "id": "el0077",
   "type": "rectangle",
   "x": 390,
   "y": 100,
   "width": 130,
   "height": 60,
   "angle": 0,
   "strokeColor": "#2f9e44",
   "backgroundColor": "#b2f2bb",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 1610156447,
   "version": 1,
   "versionNonce": 1127697047,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0078",
   "type": "text",
   "x": 419.25,
   "y": 113.75,
   "width": 71.5,
   "height": 32.5,
   "angle": 0,
   "strokeColor": "#2f9e44",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 590743293,
   "version": 1,
   "versionNonce": 1700884046,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "attention\nheads 1, 2",
   "fontSize": 13,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "attention\nheads 1, 2",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0079",
   "type": "text",
   "x": 375.8,
   "y": 164,
   "width": 158.4,
   "height": 30.0,
   "angle": 0,
   "strokeColor": "#2f9e44",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 356468233,
   "version": 1,
   "versionNonce": 1484446879,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "softmax(QK^T)V per head:\nentirely local",
   "fontSize": 12,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "softmax(QK^T)V per head:\nentirely local",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0080",
   "type": "arrow",
   "x": 520,
   "y": 130,
   "width": 50,
   "height": 0,
   "angle": 0,
   "strokeColor": "#868e96",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 2
   },
   "seed": 1511724507,
   "version": 1,
   "versionNonce": 1776652827,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "points": [
    [
     0,
     0
    ],
    [
     50,
     0
    ]
   ],
   "lastCommittedPoint": null,
   "startBinding": null,
   "endBinding": null,
   "startArrowhead": null,
   "endArrowhead": "arrow",
   "elbowed": false
  },
  {
   "id": "el0081",
   "type": "rectangle",
   "x": 570,
   "y": 100,
   "width": 100,
   "height": 60,
   "angle": 0,
   "strokeColor": "#1971c2",
   "backgroundColor": "#a5d8ff",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 1039474775,
   "version": 1,
   "versionNonce": 728882324,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0082",
   "type": "text",
   "x": 584.25,
   "y": 113.75,
   "width": 71.5,
   "height": 32.5,
   "angle": 0,
   "strokeColor": "#1971c2",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 478353329,
   "version": 1,
   "versionNonce": 1293113949,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "concat of\nheads 1, 2",
   "fontSize": 13,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "concat of\nheads 1, 2",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0083",
   "type": "text",
   "x": 577.1,
   "y": 164,
   "width": 85.80000000000001,
   "height": 30.0,
   "angle": 0,
   "strokeColor": "#1971c2",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1161562206,
   "version": 1,
   "versionNonce": 1462902050,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "sharded along\nthe head dim",
   "fontSize": 12,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "sharded along\nthe head dim",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0084",
   "type": "arrow",
   "x": 670,
   "y": 130,
   "width": 50,
   "height": 0,
   "angle": 0,
   "strokeColor": "#868e96",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 2
   },
   "seed": 1486153381,
   "version": 1,
   "versionNonce": 1314983161,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "points": [
    [
     0,
     0
    ],
    [
     50,
     0
    ]
   ],
   "lastCommittedPoint": null,
   "startBinding": null,
   "endBinding": null,
   "startArrowhead": null,
   "endArrowhead": "arrow",
   "elbowed": false
  },
  {
   "id": "el0085",
   "type": "rectangle",
   "x": 720,
   "y": 100,
   "width": 90,
   "height": 60,
   "angle": 0,
   "strokeColor": "#1971c2",
   "backgroundColor": "#a5d8ff",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 158885319,
   "version": 1,
   "versionNonce": 113066257,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0086",
   "type": "text",
   "x": 730.35,
   "y": 121.25,
   "width": 69.30000000000001,
   "height": 17.5,
   "angle": 0,
   "strokeColor": "#1971c2",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 529834903,
   "version": 1,
   "versionNonce": 49506686,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "(.) W_O,1",
   "fontSize": 14,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "(.) W_O,1",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0087",
   "type": "text",
   "x": 728.7,
   "y": 164,
   "width": 72.60000000000001,
   "height": 30.0,
   "angle": 0,
   "strokeColor": "#1971c2",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1263307675,
   "version": 1,
   "versionNonce": 1462701600,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "row split:\npartial sum",
   "fontSize": 12,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "row split:\npartial sum",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0088",
   "type": "arrow",
   "x": 810,
   "y": 130,
   "width": 50,
   "height": 0,
   "angle": 0,
   "strokeColor": "#868e96",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 2
   },
   "seed": 1813428335,
   "version": 1,
   "versionNonce": 807569473,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "points": [
    [
     0,
     0
    ],
    [
     50,
     0
    ]
   ],
   "lastCommittedPoint": null,
   "startBinding": null,
   "endBinding": null,
   "startArrowhead": null,
   "endArrowhead": "arrow",
   "elbowed": false
  },
  {
   "id": "el0089",
   "type": "rectangle",
   "x": 860,
   "y": 100,
   "width": 60,
   "height": 60,
   "angle": 0,
   "strokeColor": "#1971c2",
   "backgroundColor": "#a5d8ff",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 1705571251,
   "version": 1,
   "versionNonce": 300690383,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0090",
   "type": "text",
   "x": 881.2,
   "y": 120.0,
   "width": 17.6,
   "height": 20.0,
   "angle": 0,
   "strokeColor": "#1971c2",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 129999902,
   "version": 1,
   "versionNonce": 2093614145,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "O1",
   "fontSize": 16,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "O1",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0091",
   "type": "text",
   "x": 57.199999999999996,
   "y": 252,
   "width": 52.800000000000004,
   "height": 20.0,
   "angle": 0,
   "strokeColor": "#e8590c",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 186319890,
   "version": 1,
   "versionNonce": 1961776406,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "rank 2",
   "fontSize": 16,
   "fontFamily": 1,
   "textAlign": "right",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "rank 2",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0092",
   "type": "rectangle",
   "x": 120,
   "y": 230,
   "width": 60,
   "height": 60,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "#dee2e6",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 1450873206,
   "version": 1,
   "versionNonce": 604411722,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0093",
   "type": "text",
   "x": 145.6,
   "y": 250.0,
   "width": 8.8,
   "height": 20.0,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 9326876,
   "version": 1,
   "versionNonce": 191031046,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "X",
   "fontSize": 16,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "X",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0094",
   "type": "arrow",
   "x": 180,
   "y": 260,
   "width": 50,
   "height": 0,
   "angle": 0,
   "strokeColor": "#868e96",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 2
   },
   "seed": 163198143,
   "version": 1,
   "versionNonce": 1098826828,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "points": [
    [
     0,
     0
    ],
    [
     50,
     0
    ]
   ],
   "lastCommittedPoint": null,
   "startBinding": null,
   "endBinding": null,
   "startArrowhead": null,
   "endArrowhead": "arrow",
   "elbowed": false
  },
  {
   "id": "el0095",
   "type": "rectangle",
   "x": 230,
   "y": 230,
   "width": 110,
   "height": 60,
   "angle": 0,
   "strokeColor": "#e8590c",
   "backgroundColor": "#ffd8a8",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 1077556139,
   "version": 1,
   "versionNonce": 988198268,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0096",
   "type": "text",
   "x": 245.675,
   "y": 243.75,
   "width": 78.65,
   "height": 32.5,
   "angle": 0,
   "strokeColor": "#e8590c",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1160815309,
   "version": 1,
   "versionNonce": 345723885,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "W_Q W_K W_V\nheads 3, 4",
   "fontSize": 13,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "W_Q W_K W_V\nheads 3, 4",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0097",
   "type": "text",
   "x": 245.4,
   "y": 294,
   "width": 79.2,
   "height": 15.0,
   "angle": 0,
   "strokeColor": "#e8590c",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 813232825,
   "version": 1,
   "versionNonce": 731183208,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "column split",
   "fontSize": 12,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "column split",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0098",
   "type": "arrow",
   "x": 340,
   "y": 260,
   "width": 50,
   "height": 0,
   "angle": 0,
   "strokeColor": "#868e96",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 2
   },
   "seed": 262208329,
   "version": 1,
   "versionNonce": 2009308257,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "points": [
    [
     0,
     0
    ],
    [
     50,
     0
    ]
   ],
   "lastCommittedPoint": null,
   "startBinding": null,
   "endBinding": null,
   "startArrowhead": null,
   "endArrowhead": "arrow",
   "elbowed": false
  },
  {
   "id": "el0099",
   "type": "rectangle",
   "x": 390,
   "y": 230,
   "width": 130,
   "height": 60,
   "angle": 0,
   "strokeColor": "#2f9e44",
   "backgroundColor": "#b2f2bb",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 1098267947,
   "version": 1,
   "versionNonce": 251295882,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0100",
   "type": "text",
   "x": 419.25,
   "y": 243.75,
   "width": 71.5,
   "height": 32.5,
   "angle": 0,
   "strokeColor": "#2f9e44",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 508462774,
   "version": 1,
   "versionNonce": 952138815,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "attention\nheads 3, 4",
   "fontSize": 13,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "attention\nheads 3, 4",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0101",
   "type": "text",
   "x": 375.8,
   "y": 294,
   "width": 158.4,
   "height": 30.0,
   "angle": 0,
   "strokeColor": "#2f9e44",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 323901245,
   "version": 1,
   "versionNonce": 407305660,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "softmax(QK^T)V per head:\nentirely local",
   "fontSize": 12,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "softmax(QK^T)V per head:\nentirely local",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0102",
   "type": "arrow",
   "x": 520,
   "y": 260,
   "width": 50,
   "height": 0,
   "angle": 0,
   "strokeColor": "#868e96",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 2
   },
   "seed": 180556440,
   "version": 1,
   "versionNonce": 1787441484,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "points": [
    [
     0,
     0
    ],
    [
     50,
     0
    ]
   ],
   "lastCommittedPoint": null,
   "startBinding": null,
   "endBinding": null,
   "startArrowhead": null,
   "endArrowhead": "arrow",
   "elbowed": false
  },
  {
   "id": "el0103",
   "type": "rectangle",
   "x": 570,
   "y": 230,
   "width": 100,
   "height": 60,
   "angle": 0,
   "strokeColor": "#e8590c",
   "backgroundColor": "#ffd8a8",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 2102327576,
   "version": 1,
   "versionNonce": 229643495,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0104",
   "type": "text",
   "x": 584.25,
   "y": 243.75,
   "width": 71.5,
   "height": 32.5,
   "angle": 0,
   "strokeColor": "#e8590c",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1542276928,
   "version": 1,
   "versionNonce": 1470939160,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "concat of\nheads 3, 4",
   "fontSize": 13,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "concat of\nheads 3, 4",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0105",
   "type": "text",
   "x": 577.1,
   "y": 294,
   "width": 85.80000000000001,
   "height": 30.0,
   "angle": 0,
   "strokeColor": "#e8590c",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1055273858,
   "version": 1,
   "versionNonce": 329201013,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "sharded along\nthe head dim",
   "fontSize": 12,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "sharded along\nthe head dim",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0106",
   "type": "arrow",
   "x": 670,
   "y": 260,
   "width": 50,
   "height": 0,
   "angle": 0,
   "strokeColor": "#868e96",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 2
   },
   "seed": 1773980233,
   "version": 1,
   "versionNonce": 1582512011,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "points": [
    [
     0,
     0
    ],
    [
     50,
     0
    ]
   ],
   "lastCommittedPoint": null,
   "startBinding": null,
   "endBinding": null,
   "startArrowhead": null,
   "endArrowhead": "arrow",
   "elbowed": false
  },
  {
   "id": "el0107",
   "type": "rectangle",
   "x": 720,
   "y": 230,
   "width": 90,
   "height": 60,
   "angle": 0,
   "strokeColor": "#e8590c",
   "backgroundColor": "#ffd8a8",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 1181240956,
   "version": 1,
   "versionNonce": 1965172037,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0108",
   "type": "text",
   "x": 730.35,
   "y": 251.25,
   "width": 69.30000000000001,
   "height": 17.5,
   "angle": 0,
   "strokeColor": "#e8590c",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1580042562,
   "version": 1,
   "versionNonce": 1216720807,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "(.) W_O,2",
   "fontSize": 14,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "(.) W_O,2",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0109",
   "type": "text",
   "x": 728.7,
   "y": 294,
   "width": 72.60000000000001,
   "height": 30.0,
   "angle": 0,
   "strokeColor": "#e8590c",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 60978357,
   "version": 1,
   "versionNonce": 2061697174,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "row split:\npartial sum",
   "fontSize": 12,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "row split:\npartial sum",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0110",
   "type": "arrow",
   "x": 810,
   "y": 260,
   "width": 50,
   "height": 0,
   "angle": 0,
   "strokeColor": "#868e96",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 2
   },
   "seed": 1555649821,
   "version": 1,
   "versionNonce": 2133871459,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "points": [
    [
     0,
     0
    ],
    [
     50,
     0
    ]
   ],
   "lastCommittedPoint": null,
   "startBinding": null,
   "endBinding": null,
   "startArrowhead": null,
   "endArrowhead": "arrow",
   "elbowed": false
  },
  {
   "id": "el0111",
   "type": "rectangle",
   "x": 860,
   "y": 230,
   "width": 60,
   "height": 60,
   "angle": 0,
   "strokeColor": "#e8590c",
   "backgroundColor": "#ffd8a8",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 1687590333,
   "version": 1,
   "versionNonce": 2065181317,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0112",
   "type": "text",
   "x": 881.2,
   "y": 250.0,
   "width": 17.6,
   "height": 20.0,
   "angle": 0,
   "strokeColor": "#e8590c",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1226504278,
   "version": 1,
   "versionNonce": 1528916665,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "O2",
   "fontSize": 16,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "O2",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0113",
   "type": "arrow",
   "x": 920,
   "y": 130,
   "width": 50,
   "height": 65,
   "angle": 0,
   "strokeColor": "#868e96",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 2
   },
   "seed": 158445899,
   "version": 1,
   "versionNonce": 1558116236,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "points": [
    [
     0,
     0
    ],
    [
     50,
     0
    ],
    [
     50,
     65
    ]
   ],
   "lastCommittedPoint": null,
   "startBinding": null,
   "endBinding": null,
   "startArrowhead": null,
   "endArrowhead": "arrow",
   "elbowed": false
  },
  {
   "id": "el0114",
   "type": "arrow",
   "x": 920,
   "y": 260,
   "width": 50,
   "height": 65,
   "angle": 0,
   "strokeColor": "#868e96",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 2
   },
   "seed": 1416335451,
   "version": 1,
   "versionNonce": 108343384,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "points": [
    [
     0,
     0
    ],
    [
     50,
     0
    ],
    [
     50,
     -65
    ]
   ],
   "lastCommittedPoint": null,
   "startBinding": null,
   "endBinding": null,
   "startArrowhead": null,
   "endArrowhead": null,
   "elbowed": false
  },
  {
   "id": "el0115",
   "type": "rectangle",
   "x": 970,
   "y": 165,
   "width": 120,
   "height": 60,
   "angle": 0,
   "strokeColor": "#e03131",
   "backgroundColor": "#ffc9c9",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 2000796160,
   "version": 1,
   "versionNonce": 782473781,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0116",
   "type": "text",
   "x": 987.65,
   "y": 177.5,
   "width": 84.7,
   "height": 35.0,
   "angle": 0,
   "strokeColor": "#e03131",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 112553845,
   "version": 1,
   "versionNonce": 271935172,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "all-reduce\nO = O1 + O2",
   "fontSize": 14,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "all-reduce\nO = O1 + O2",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0117",
   "type": "arrow",
   "x": 1090,
   "y": 195,
   "width": 50,
   "height": 0,
   "angle": 0,
   "strokeColor": "#868e96",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 2
   },
   "seed": 1078777587,
   "version": 1,
   "versionNonce": 373909989,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "points": [
    [
     0,
     0
    ],
    [
     50,
     0
    ]
   ],
   "lastCommittedPoint": null,
   "startBinding": null,
   "endBinding": null,
   "startArrowhead": null,
   "endArrowhead": "arrow",
   "elbowed": false
  },
  {
   "id": "el0118",
   "type": "rectangle",
   "x": 1140,
   "y": 165,
   "width": 60,
   "height": 60,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "#dee2e6",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 1697749638,
   "version": 1,
   "versionNonce": 1847915266,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0119",
   "type": "text",
   "x": 1165.6,
   "y": 185.0,
   "width": 8.8,
   "height": 20.0,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 161526657,
   "version": 1,
   "versionNonce": 1795162083,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "O",
   "fontSize": 16,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "O",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0120",
   "type": "text",
   "x": 143.95,
   "y": 305,
   "width": 12.100000000000001,
   "height": 27.5,
   "angle": 0,
   "strokeColor": "#7048e8",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1510061505,
   "version": 1,
   "versionNonce": 1577546070,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "f",
   "fontSize": 22,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "f",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0121",
   "type": "text",
   "x": 103.8,
   "y": 332,
   "width": 92.4,
   "height": 30.0,
   "angle": 0,
   "strokeColor": "#7048e8",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1017998792,
   "version": 1,
   "versionNonce": 512670355,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "identity fwd\nall-reduce bwd",
   "fontSize": 12,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "identity fwd\nall-reduce bwd",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0122",
   "type": "text",
   "x": 1023.95,
   "y": 305,
   "width": 12.100000000000001,
   "height": 27.5,
   "angle": 0,
   "strokeColor": "#7048e8",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 163313930,
   "version": 1,
   "versionNonce": 1469752785,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "g",
   "fontSize": 22,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "g",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0123",
   "type": "text",
   "x": 983.8,
   "y": 332,
   "width": 92.4,
   "height": 30.0,
   "angle": 0,
   "strokeColor": "#7048e8",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 762541788,
   "version": 1,
   "versionNonce": 2066544914,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "all-reduce fwd\nidentity bwd",
   "fontSize": 12,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "all-reduce fwd\nidentity bwd",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0124",
   "type": "text",
   "x": 20,
   "y": 380,
   "width": 793.1,
   "height": 17.5,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 325986910,
   "version": 1,
   "versionNonce": 540337992,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "W_QKV as a matrix  (h x 3h):  columns grouped by head, so a column split hands each rank complete heads",
   "fontSize": 14,
   "fontFamily": 1,
   "textAlign": "left",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "W_QKV as a matrix  (h x 3h):  columns grouped by head, so a column split hands each rank complete heads",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0125",
   "type": "rectangle",
   "x": 20,
   "y": 410,
   "width": 60,
   "height": 50,
   "angle": 0,
   "strokeColor": "#1971c2",
   "backgroundColor": "#a5d8ff",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 967059676,
   "version": 1,
   "versionNonce": 2076207049,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0126",
   "type": "text",
   "x": 36.8,
   "y": 427,
   "width": 26.400000000000002,
   "height": 15.0,
   "angle": 0,
   "strokeColor": "#1971c2",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 55270169,
   "version": 1,
   "versionNonce": 835414828,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "Q h1",
   "fontSize": 12,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "Q h1",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0127",
   "type": "rectangle",
   "x": 82,
   "y": 410,
   "width": 60,
   "height": 50,
   "angle": 0,
   "strokeColor": "#1971c2",
   "backgroundColor": "#a5d8ff",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 752142238,
   "version": 1,
   "versionNonce": 473051341,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0128",
   "type": "text",
   "x": 98.8,
   "y": 427,
   "width": 26.400000000000002,
   "height": 15.0,
   "angle": 0,
   "strokeColor": "#1971c2",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 79806338,
   "version": 1,
   "versionNonce": 375576787,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "Q h2",
   "fontSize": 12,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "Q h2",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0129",
   "type": "rectangle",
   "x": 144,
   "y": 410,
   "width": 60,
   "height": 50,
   "angle": 0,
   "strokeColor": "#e8590c",
   "backgroundColor": "#ffd8a8",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1849001177,
   "version": 1,
   "versionNonce": 1477037607,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0130",
   "type": "text",
   "x": 160.8,
   "y": 427,
   "width": 26.400000000000002,
   "height": 15.0,
   "angle": 0,
   "strokeColor": "#e8590c",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1669767181,
   "version": 1,
   "versionNonce": 284000029,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "Q h3",
   "fontSize": 12,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "Q h3",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0131",
   "type": "rectangle",
   "x": 206,
   "y": 410,
   "width": 60,
   "height": 50,
   "angle": 0,
   "strokeColor": "#e8590c",
   "backgroundColor": "#ffd8a8",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1497260189,
   "version": 1,
   "versionNonce": 1899300900,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0132",
   "type": "text",
   "x": 222.8,
   "y": 427,
   "width": 26.400000000000002,
   "height": 15.0,
   "angle": 0,
   "strokeColor": "#e8590c",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 959141570,
   "version": 1,
   "versionNonce": 1214837972,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "Q h4",
   "fontSize": 12,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "Q h4",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0133",
   "type": "rectangle",
   "x": 268,
   "y": 410,
   "width": 60,
   "height": 50,
   "angle": 0,
   "strokeColor": "#1971c2",
   "backgroundColor": "#a5d8ff",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 569330679,
   "version": 1,
   "versionNonce": 1351862010,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0134",
   "type": "text",
   "x": 284.8,
   "y": 427,
   "width": 26.400000000000002,
   "height": 15.0,
   "angle": 0,
   "strokeColor": "#1971c2",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 456278667,
   "version": 1,
   "versionNonce": 886511532,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "K h1",
   "fontSize": 12,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "K h1",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0135",
   "type": "rectangle",
   "x": 330,
   "y": 410,
   "width": 60,
   "height": 50,
   "angle": 0,
   "strokeColor": "#1971c2",
   "backgroundColor": "#a5d8ff",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 361956302,
   "version": 1,
   "versionNonce": 1939796778,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0136",
   "type": "text",
   "x": 346.8,
   "y": 427,
   "width": 26.400000000000002,
   "height": 15.0,
   "angle": 0,
   "strokeColor": "#1971c2",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 423969784,
   "version": 1,
   "versionNonce": 1354604736,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "K h2",
   "fontSize": 12,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "K h2",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0137",
   "type": "rectangle",
   "x": 392,
   "y": 410,
   "width": 60,
   "height": 50,
   "angle": 0,
   "strokeColor": "#e8590c",
   "backgroundColor": "#ffd8a8",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1972981325,
   "version": 1,
   "versionNonce": 979438246,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0138",
   "type": "text",
   "x": 408.8,
   "y": 427,
   "width": 26.400000000000002,
   "height": 15.0,
   "angle": 0,
   "strokeColor": "#e8590c",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1590666500,
   "version": 1,
   "versionNonce": 1178059420,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "K h3",
   "fontSize": 12,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "K h3",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0139",
   "type": "rectangle",
   "x": 454,
   "y": 410,
   "width": 60,
   "height": 50,
   "angle": 0,
   "strokeColor": "#e8590c",
   "backgroundColor": "#ffd8a8",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 70525857,
   "version": 1,
   "versionNonce": 853798968,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0140",
   "type": "text",
   "x": 470.8,
   "y": 427,
   "width": 26.400000000000002,
   "height": 15.0,
   "angle": 0,
   "strokeColor": "#e8590c",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1022492057,
   "version": 1,
   "versionNonce": 1754066707,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "K h4",
   "fontSize": 12,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "K h4",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0141",
   "type": "rectangle",
   "x": 516,
   "y": 410,
   "width": 60,
   "height": 50,
   "angle": 0,
   "strokeColor": "#1971c2",
   "backgroundColor": "#a5d8ff",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 413562236,
   "version": 1,
   "versionNonce": 1285329710,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0142",
   "type": "text",
   "x": 532.8,
   "y": 427,
   "width": 26.400000000000002,
   "height": 15.0,
   "angle": 0,
   "strokeColor": "#1971c2",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 193691629,
   "version": 1,
   "versionNonce": 577129774,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "V h1",
   "fontSize": 12,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "V h1",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0143",
   "type": "rectangle",
   "x": 578,
   "y": 410,
   "width": 60,
   "height": 50,
   "angle": 0,
   "strokeColor": "#1971c2",
   "backgroundColor": "#a5d8ff",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 311489100,
   "version": 1,
   "versionNonce": 767721127,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0144",
   "type": "text",
   "x": 594.8,
   "y": 427,
   "width": 26.400000000000002,
   "height": 15.0,
   "angle": 0,
   "strokeColor": "#1971c2",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 677418884,
   "version": 1,
   "versionNonce": 403428569,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "V h2",
   "fontSize": 12,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "V h2",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0145",
   "type": "rectangle",
   "x": 640,
   "y": 410,
   "width": 60,
   "height": 50,
   "angle": 0,
   "strokeColor": "#e8590c",
   "backgroundColor": "#ffd8a8",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1020695351,
   "version": 1,
   "versionNonce": 1634524011,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0146",
   "type": "text",
   "x": 656.8,
   "y": 427,
   "width": 26.400000000000002,
   "height": 15.0,
   "angle": 0,
   "strokeColor": "#e8590c",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 23799048,
   "version": 1,
   "versionNonce": 1648354477,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "V h3",
   "fontSize": 12,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "V h3",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0147",
   "type": "rectangle",
   "x": 702,
   "y": 410,
   "width": 60,
   "height": 50,
   "angle": 0,
   "strokeColor": "#e8590c",
   "backgroundColor": "#ffd8a8",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 514554807,
   "version": 1,
   "versionNonce": 954772287,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0148",
   "type": "text",
   "x": 718.8,
   "y": 427,
   "width": 26.400000000000002,
   "height": 15.0,
   "angle": 0,
   "strokeColor": "#e8590c",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1232134618,
   "version": 1,
   "versionNonce": 328195455,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "V h4",
   "fontSize": 12,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "V h4",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0149",
   "type": "text",
   "x": 113.14999999999998,
   "y": 468,
   "width": 557.7,
   "height": 16.25,
   "angle": 0,
   "strokeColor": "#2f9e44",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1559991334,
   "version": 1,
   "versionNonce": 1400982250,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "blue columns live on rank 1, orange on rank 2; nothing crosses ranks until W_O",
   "fontSize": 13,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "blue columns live on rank 1, orange on rank 2; nothing crosses ranks until W_O",
   "autoResize": true,
   "lineHeight": 1.25
  }
 ],
 "appState": {
  "gridSize": 20,
  "gridStep": 5,
  "gridModeEnabled": false,
  "viewBackgroundColor": "#ffffff"
 },
 "files": {}
}