{
 "type": "excalidraw",
 "version": 2,
 "source": "https://excalidraw.com",
 "elements": [
  {
   "id": "el0572",
   "type": "text",
   "x": 20,
   "y": 10,
   "width": 1282.6000000000001,
   "height": 27.5,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 573614336,
   "version": 1,
   "versionNonce": 1226892252,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "3D parallelism on four 8-GPU nodes:  TP = 8 (in the node), PP = 2 (across nodes), DP = 2 (across replicas)",
   "fontSize": 22,
   "fontFamily": 1,
   "textAlign": "left",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "3D parallelism on four 8-GPU nodes:  TP = 8 (in the node), PP = 2 (across nodes), DP = 2 (across replicas)",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0573",
   "type": "text",
   "x": 20,
   "y": 44,
   "width": 1008.7,
   "height": 17.5,
   "angle": 0,
   "strokeColor": "#2f9e44",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 167164860,
   "version": 1,
   "versionNonce": 116724656,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "chattiest split on the fastest link: TP all-reduces on NVLink, PP point-to-point over the network, DP gradient reduce once per step",
   "fontSize": 14,
   "fontFamily": 1,
   "textAlign": "left",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "chattiest split on the fastest link: TP all-reduces on NVLink, PP point-to-point over the network, DP gradient reduce once per step",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0574",
   "type": "rectangle",
   "x": 60,
   "y": 100,
   "width": 330,
   "height": 150,
   "angle": 0,
   "strokeColor": "#7048e8",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 704057952,
   "version": 1,
   "versionNonce": 1155930989,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0575",
   "type": "text",
   "x": 70,
   "y": 108,
   "width": 46.2,
   "height": 17.5,
   "angle": 0,
   "strokeColor": "#7048e8",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 508110515,
   "version": 1,
   "versionNonce": 33554571,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "node 0",
   "fontSize": 14,
   "fontFamily": 1,
   "textAlign": "left",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "node 0",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0576",
   "type": "text",
   "x": 308.5,
   "y": 108,
   "width": 71.5,
   "height": 16.25,
   "angle": 0,
   "strokeColor": "#7048e8",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1181684625,
   "version": 1,
   "versionNonce": 1228378938,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "PP stage 1",
   "fontSize": 13,
   "fontFamily": 1,
   "textAlign": "right",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "PP stage 1",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0577",
   "type": "rectangle",
   "x": 75,
   "y": 140,
   "width": 32,
   "height": 32,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "#d0bfff",
   "fillStyle": "solid",
   "strokeWidth": 1.5,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 1206804597,
   "version": 1,
   "versionNonce": 999388020,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0578",
   "type": "text",
   "x": 84.95,
   "y": 148,
   "width": 12.100000000000001,
   "height": 13.75,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1798106119,
   "version": 1,
   "versionNonce": 760291707,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "g0",
   "fontSize": 11,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "g0",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0579",
   "type": "rectangle",
   "x": 114,
   "y": 140,
   "width": 32,
   "height": 32,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "#d0bfff",
   "fillStyle": "solid",
   "strokeWidth": 1.5,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 642423406,
   "version": 1,
   "versionNonce": 2044789368,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0580",
   "type": "text",
   "x": 123.95,
   "y": 148,
   "width": 12.100000000000001,
   "height": 13.75,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 21664670,
   "version": 1,
   "versionNonce": 1696462556,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "g1",
   "fontSize": 11,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "g1",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0581",
   "type": "rectangle",
   "x": 153,
   "y": 140,
   "width": 32,
   "height": 32,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "#d0bfff",
   "fillStyle": "solid",
   "strokeWidth": 1.5,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 1999213036,
   "version": 1,
   "versionNonce": 230883757,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0582",
   "type": "text",
   "x": 162.95,
   "y": 148,
   "width": 12.100000000000001,
   "height": 13.75,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1313961279,
   "version": 1,
   "versionNonce": 639061103,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "g2",
   "fontSize": 11,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "g2",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0583",
   "type": "rectangle",
   "x": 192,
   "y": 140,
   "width": 32,
   "height": 32,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "#d0bfff",
   "fillStyle": "solid",
   "strokeWidth": 1.5,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 1468475368,
   "version": 1,
   "versionNonce": 1529926396,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0584",
   "type": "text",
   "x": 201.95,
   "y": 148,
   "width": 12.100000000000001,
   "height": 13.75,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1365982612,
   "version": 1,
   "versionNonce": 2090649829,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "g3",
   "fontSize": 11,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "g3",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0585",
   "type": "rectangle",
   "x": 231,
   "y": 140,
   "width": 32,
   "height": 32,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "#d0bfff",
   "fillStyle": "solid",
   "strokeWidth": 1.5,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 1910486845,
   "version": 1,
   "versionNonce": 161996212,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0586",
   "type": "text",
   "x": 240.95,
   "y": 148,
   "width": 12.100000000000001,
   "height": 13.75,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1832087099,
   "version": 1,
   "versionNonce": 1526743364,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "g4",
   "fontSize": 11,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "g4",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0587",
   "type": "rectangle",
   "x": 270,
   "y": 140,
   "width": 32,
   "height": 32,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "#d0bfff",
   "fillStyle": "solid",
   "strokeWidth": 1.5,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 1198895831,
   "version": 1,
   "versionNonce": 691759774,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0588",
   "type": "text",
   "x": 279.95,
   "y": 148,
   "width": 12.100000000000001,
   "height": 13.75,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1233570472,
   "version": 1,
   "versionNonce": 1269436283,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "g5",
   "fontSize": 11,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "g5",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0589",
   "type": "rectangle",
   "x": 309,
   "y": 140,
   "width": 32,
   "height": 32,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "#d0bfff",
   "fillStyle": "solid",
   "strokeWidth": 1.5,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 2129857771,
   "version": 1,
   "versionNonce": 1388497274,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0590",
   "type": "text",
   "x": 318.95,
   "y": 148,
   "width": 12.100000000000001,
   "height": 13.75,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 544658235,
   "version": 1,
   "versionNonce": 356279431,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "g6",
   "fontSize": 11,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "g6",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0591",
   "type": "rectangle",
   "x": 348,
   "y": 140,
   "width": 32,
   "height": 32,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "#d0bfff",
   "fillStyle": "solid",
   "strokeWidth": 1.5,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 1721889981,
   "version": 1,
   "versionNonce": 73008815,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0592",
   "type": "text",
   "x": 357.95,
   "y": 148,
   "width": 12.100000000000001,
   "height": 13.75,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 655930729,
   "version": 1,
   "versionNonce": 386211946,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "g7",
   "fontSize": 11,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "g7",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0593",
   "type": "arrow",
   "x": 75,
   "y": 192,
   "width": 305,
   "height": 0,
   "angle": 0,
   "strokeColor": "#1971c2",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 2
   },
   "seed": 111303672,
   "version": 1,
   "versionNonce": 1925719020,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "points": [
    [
     0,
     0
    ],
    [
     305,
     0
    ]
   ],
   "lastCommittedPoint": null,
   "startBinding": null,
   "endBinding": null,
   "startArrowhead": null,
   "endArrowhead": null,
   "elbowed": false
  },
  {
   "id": "el0594",
   "type": "text",
   "x": 76.5,
   "y": 198,
   "width": 297.0,
   "height": 30.0,
   "angle": 0,
   "strokeColor": "#1971c2",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1442706805,
   "version": 1,
   "versionNonce": 1166795012,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "TP group of 8: NVLink, 450 GB/s per direction\n4 all-reduces of the hidden state per layer",
   "fontSize": 12,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "TP group of 8: NVLink, 450 GB/s per direction\n4 all-reduces of the hidden state per layer",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0595",
   "type": "rectangle",
   "x": 540,
   "y": 100,
   "width": 330,
   "height": 150,
   "angle": 0,
   "strokeColor": "#7048e8",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 515267672,
   "version": 1,
   "versionNonce": 81222541,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0596",
   "type": "text",
   "x": 550,
   "y": 108,
   "width": 46.2,
   "height": 17.5,
   "angle": 0,
   "strokeColor": "#7048e8",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1126990975,
   "version": 1,
   "versionNonce": 868929081,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "node 1",
   "fontSize": 14,
   "fontFamily": 1,
   "textAlign": "left",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "node 1",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0597",
   "type": "text",
   "x": 788.5,
   "y": 108,
   "width": 71.5,
   "height": 16.25,
   "angle": 0,
   "strokeColor": "#7048e8",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1658373707,
   "version": 1,
   "versionNonce": 1047478478,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "PP stage 2",
   "fontSize": 13,
   "fontFamily": 1,
   "textAlign": "right",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "PP stage 2",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0598",
   "type": "rectangle",
   "x": 555,
   "y": 140,
   "width": 32,
   "height": 32,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "#d0bfff",
   "fillStyle": "solid",
   "strokeWidth": 1.5,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 572988033,
   "version": 1,
   "versionNonce": 1614260807,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0599",
   "type": "text",
   "x": 564.95,
   "y": 148,
   "width": 12.100000000000001,
   "height": 13.75,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1875085664,
   "version": 1,
   "versionNonce": 1118933879,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "g0",
   "fontSize": 11,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "g0",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0600",
   "type": "rectangle",
   "x": 594,
   "y": 140,
   "width": 32,
   "height": 32,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "#d0bfff",
   "fillStyle": "solid",
   "strokeWidth": 1.5,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 1087423013,
   "version": 1,
   "versionNonce": 1362205172,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0601",
   "type": "text",
   "x": 603.95,
   "y": 148,
   "width": 12.100000000000001,
   "height": 13.75,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 269095006,
   "version": 1,
   "versionNonce": 966779406,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "g1",
   "fontSize": 11,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "g1",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0602",
   "type": "rectangle",
   "x": 633,
   "y": 140,
   "width": 32,
   "height": 32,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "#d0bfff",
   "fillStyle": "solid",
   "strokeWidth": 1.5,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 979952423,
   "version": 1,
   "versionNonce": 862361391,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0603",
   "type": "text",
   "x": 642.95,
   "y": 148,
   "width": 12.100000000000001,
   "height": 13.75,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 393498348,
   "version": 1,
   "versionNonce": 1152244634,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "g2",
   "fontSize": 11,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "g2",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0604",
   "type": "rectangle",
   "x": 672,
   "y": 140,
   "width": 32,
   "height": 32,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "#d0bfff",
   "fillStyle": "solid",
   "strokeWidth": 1.5,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 1474631864,
   "version": 1,
   "versionNonce": 1328749774,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0605",
   "type": "text",
   "x": 681.95,
   "y": 148,
   "width": 12.100000000000001,
   "height": 13.75,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1196796385,
   "version": 1,
   "versionNonce": 1611153693,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "g3",
   "fontSize": 11,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "g3",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0606",
   "type": "rectangle",
   "x": 711,
   "y": 140,
   "width": 32,
   "height": 32,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "#d0bfff",
   "fillStyle": "solid",
   "strokeWidth": 1.5,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 1398641769,
   "version": 1,
   "versionNonce": 1899153460,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0607",
   "type": "text",
   "x": 720.95,
   "y": 148,
   "width": 12.100000000000001,
   "height": 13.75,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1988921626,
   "version": 1,
   "versionNonce": 1602485132,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "g4",
   "fontSize": 11,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "g4",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0608",
   "type": "rectangle",
   "x": 750,
   "y": 140,
   "width": 32,
   "height": 32,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "#d0bfff",
   "fillStyle": "solid",
   "strokeWidth": 1.5,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 349901259,
   "version": 1,
   "versionNonce": 595935209,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0609",
   "type": "text",
   "x": 759.95,
   "y": 148,
   "width": 12.100000000000001,
   "height": 13.75,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1839912162,
   "version": 1,
   "versionNonce": 1484518197,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "g5",
   "fontSize": 11,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "g5",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0610",
   "type": "rectangle",
   "x": 789,
   "y": 140,
   "width": 32,
   "height": 32,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "#d0bfff",
   "fillStyle": "solid",
   "strokeWidth": 1.5,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 2118550733,
   "version": 1,
   "versionNonce": 366637979,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0611",
   "type": "text",
   "x": 798.95,
   "y": 148,
   "width": 12.100000000000001,
   "height": 13.75,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1081385598,
   "version": 1,
   "versionNonce": 91465265,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "g6",
   "fontSize": 11,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "g6",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0612",
   "type": "rectangle",
   "x": 828,
   "y": 140,
   "width": 32,
   "height": 32,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "#d0bfff",
   "fillStyle": "solid",
   "strokeWidth": 1.5,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 1367712321,
   "version": 1,
   "versionNonce": 1002246191,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0613",
   "type": "text",
   "x": 837.95,
   "y": 148,
   "width": 12.100000000000001,
   "height": 13.75,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 305358517,
   "version": 1,
   "versionNonce": 168442767,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "g7",
   "fontSize": 11,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "g7",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0614",
   "type": "arrow",
   "x": 555,
   "y": 192,
   "width": 305,
   "height": 0,
   "angle": 0,
   "strokeColor": "#1971c2",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 2
   },
   "seed": 510120366,
   "version": 1,
   "versionNonce": 1326411266,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "points": [
    [
     0,
     0
    ],
    [
     305,
     0
    ]
   ],
   "lastCommittedPoint": null,
   "startBinding": null,
   "endBinding": null,
   "startArrowhead": null,
   "endArrowhead": null,
   "elbowed": false
  },
  {
   "id": "el0615",
   "type": "text",
   "x": 556.5,
   "y": 198,
   "width": 297.0,
   "height": 30.0,
   "angle": 0,
   "strokeColor": "#1971c2",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1254813618,
   "version": 1,
   "versionNonce": 888445648,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "TP group of 8: NVLink, 450 GB/s per direction\n4 all-reduces of the hidden state per layer",
   "fontSize": 12,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "TP group of 8: NVLink, 450 GB/s per direction\n4 all-reduces of the hidden state per layer",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0616",
   "type": "rectangle",
   "x": 60,
   "y": 360,
   "width": 330,
   "height": 150,
   "angle": 0,
   "strokeColor": "#e8590c",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 1309545279,
   "version": 1,
   "versionNonce": 291733184,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0617",
   "type": "text",
   "x": 70,
   "y": 368,
   "width": 46.2,
   "height": 17.5,
   "angle": 0,
   "strokeColor": "#e8590c",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1638043198,
   "version": 1,
   "versionNonce": 2077040091,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "node 2",
   "fontSize": 14,
   "fontFamily": 1,
   "textAlign": "left",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "node 2",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0618",
   "type": "text",
   "x": 308.5,
   "y": 368,
   "width": 71.5,
   "height": 16.25,
   "angle": 0,
   "strokeColor": "#e8590c",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 2050700362,
   "version": 1,
   "versionNonce": 1489583256,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "PP stage 1",
   "fontSize": 13,
   "fontFamily": 1,
   "textAlign": "right",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "PP stage 1",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0619",
   "type": "rectangle",
   "x": 75,
   "y": 400,
   "width": 32,
   "height": 32,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "#ffd8a8",
   "fillStyle": "solid",
   "strokeWidth": 1.5,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 672570039,
   "version": 1,
   "versionNonce": 372295096,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0620",
   "type": "text",
   "x": 84.95,
   "y": 408,
   "width": 12.100000000000001,
   "height": 13.75,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 942167835,
   "version": 1,
   "versionNonce": 1511679208,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "g0",
   "fontSize": 11,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "g0",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0621",
   "type": "rectangle",
   "x": 114,
   "y": 400,
   "width": 32,
   "height": 32,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "#ffd8a8",
   "fillStyle": "solid",
   "strokeWidth": 1.5,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 1461101782,
   "version": 1,
   "versionNonce": 1964994664,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0622",
   "type": "text",
   "x": 123.95,
   "y": 408,
   "width": 12.100000000000001,
   "height": 13.75,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1690725863,
   "version": 1,
   "versionNonce": 2128481248,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "g1",
   "fontSize": 11,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "g1",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0623",
   "type": "rectangle",
   "x": 153,
   "y": 400,
   "width": 32,
   "height": 32,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "#ffd8a8",
   "fillStyle": "solid",
   "strokeWidth": 1.5,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 718358223,
   "version": 1,
   "versionNonce": 1454790948,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0624",
   "type": "text",
   "x": 162.95,
   "y": 408,
   "width": 12.100000000000001,
   "height": 13.75,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1171501581,
   "version": 1,
   "versionNonce": 1039473327,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "g2",
   "fontSize": 11,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "g2",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0625",
   "type": "rectangle",
   "x": 192,
   "y": 400,
   "width": 32,
   "height": 32,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "#ffd8a8",
   "fillStyle": "solid",
   "strokeWidth": 1.5,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 406544219,
   "version": 1,
   "versionNonce": 512517003,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0626",
   "type": "text",
   "x": 201.95,
   "y": 408,
   "width": 12.100000000000001,
   "height": 13.75,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 580799201,
   "version": 1,
   "versionNonce": 866481682,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "g3",
   "fontSize": 11,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "g3",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0627",
   "type": "rectangle",
   "x": 231,
   "y": 400,
   "width": 32,
   "height": 32,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "#ffd8a8",
   "fillStyle": "solid",
   "strokeWidth": 1.5,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 1984900346,
   "version": 1,
   "versionNonce": 2010863875,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0628",
   "type": "text",
   "x": 240.95,
   "y": 408,
   "width": 12.100000000000001,
   "height": 13.75,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1347830664,
   "version": 1,
   "versionNonce": 1714730218,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "g4",
   "fontSize": 11,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "g4",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0629",
   "type": "rectangle",
   "x": 270,
   "y": 400,
   "width": 32,
   "height": 32,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "#ffd8a8",
   "fillStyle": "solid",
   "strokeWidth": 1.5,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 156581202,
   "version": 1,
   "versionNonce": 527714679,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0630",
   "type": "text",
   "x": 279.95,
   "y": 408,
   "width": 12.100000000000001,
   "height": 13.75,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 450598882,
   "version": 1,
   "versionNonce": 1788769217,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "g5",
   "fontSize": 11,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "g5",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0631",
   "type": "rectangle",
   "x": 309,
   "y": 400,
   "width": 32,
   "height": 32,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "#ffd8a8",
   "fillStyle": "solid",
   "strokeWidth": 1.5,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 833485866,
   "version": 1,
   "versionNonce": 99658622,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0632",
   "type": "text",
   "x": 318.95,
   "y": 408,
   "width": 12.100000000000001,
   "height": 13.75,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1020155571,
   "version": 1,
   "versionNonce": 1804289862,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "g6",
   "fontSize": 11,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "g6",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0633",
   "type": "rectangle",
   "x": 348,
   "y": 400,
   "width": 32,
   "height": 32,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "#ffd8a8",
   "fillStyle": "solid",
   "strokeWidth": 1.5,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 2100150367,
   "version": 1,
   "versionNonce": 928325849,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0634",
   "type": "text",
   "x": 357.95,
   "y": 408,
   "width": 12.100000000000001,
   "height": 13.75,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1698395576,
   "version": 1,
   "versionNonce": 2129296297,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "g7",
   "fontSize": 11,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "g7",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0635",
   "type": "arrow",
   "x": 75,
   "y": 452,
   "width": 305,
   "height": 0,
   "angle": 0,
   "strokeColor": "#1971c2",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 2
   },
   "seed": 2046307172,
   "version": 1,
   "versionNonce": 1086907277,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "points": [
    [
     0,
     0
    ],
    [
     305,
     0
    ]
   ],
   "lastCommittedPoint": null,
   "startBinding": null,
   "endBinding": null,
   "startArrowhead": null,
   "endArrowhead": null,
   "elbowed": false
  },
  {
   "id": "el0636",
   "type": "text",
   "x": 76.5,
   "y": 458,
   "width": 297.0,
   "height": 30.0,
   "angle": 0,
   "strokeColor": "#1971c2",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1058386762,
   "version": 1,
   "versionNonce": 953682382,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "TP group of 8: NVLink, 450 GB/s per direction\n4 all-reduces of the hidden state per layer",
   "fontSize": 12,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "TP group of 8: NVLink, 450 GB/s per direction\n4 all-reduces of the hidden state per layer",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0637",
   "type": "rectangle",
   "x": 540,
   "y": 360,
   "width": 330,
   "height": 150,
   "angle": 0,
   "strokeColor": "#e8590c",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 1637453851,
   "version": 1,
   "versionNonce": 681792102,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0638",
   "type": "text",
   "x": 550,
   "y": 368,
   "width": 46.2,
   "height": 17.5,
   "angle": 0,
   "strokeColor": "#e8590c",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1648576620,
   "version": 1,
   "versionNonce": 1868589567,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "node 3",
   "fontSize": 14,
   "fontFamily": 1,
   "textAlign": "left",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "node 3",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0639",
   "type": "text",
   "x": 788.5,
   "y": 368,
   "width": 71.5,
   "height": 16.25,
   "angle": 0,
   "strokeColor": "#e8590c",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 979749922,
   "version": 1,
   "versionNonce": 2053997711,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "PP stage 2",
   "fontSize": 13,
   "fontFamily": 1,
   "textAlign": "right",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "PP stage 2",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0640",
   "type": "rectangle",
   "x": 555,
   "y": 400,
   "width": 32,
   "height": 32,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "#ffd8a8",
   "fillStyle": "solid",
   "strokeWidth": 1.5,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 227209045,
   "version": 1,
   "versionNonce": 1054450262,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0641",
   "type": "text",
   "x": 564.95,
   "y": 408,
   "width": 12.100000000000001,
   "height": 13.75,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 503920679,
   "version": 1,
   "versionNonce": 945027577,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "g0",
   "fontSize": 11,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "g0",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0642",
   "type": "rectangle",
   "x": 594,
   "y": 400,
   "width": 32,
   "height": 32,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "#ffd8a8",
   "fillStyle": "solid",
   "strokeWidth": 1.5,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 432821620,
   "version": 1,
   "versionNonce": 1855243672,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0643",
   "type": "text",
   "x": 603.95,
   "y": 408,
   "width": 12.100000000000001,
   "height": 13.75,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 580495759,
   "version": 1,
   "versionNonce": 1053900713,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "g1",
   "fontSize": 11,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "g1",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0644",
   "type": "rectangle",
   "x": 633,
   "y": 400,
   "width": 32,
   "height": 32,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "#ffd8a8",
   "fillStyle": "solid",
   "strokeWidth": 1.5,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 926942407,
   "version": 1,
   "versionNonce": 1037167373,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0645",
   "type": "text",
   "x": 642.95,
   "y": 408,
   "width": 12.100000000000001,
   "height": 13.75,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 894203074,
   "version": 1,
   "versionNonce": 73913157,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "g2",
   "fontSize": 11,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "g2",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0646",
   "type": "rectangle",
   "x": 672,
   "y": 400,
   "width": 32,
   "height": 32,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "#ffd8a8",
   "fillStyle": "solid",
   "strokeWidth": 1.5,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 1931875296,
   "version": 1,
   "versionNonce": 2126085430,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0647",
   "type": "text",
   "x": 681.95,
   "y": 408,
   "width": 12.100000000000001,
   "height": 13.75,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1815004640,
   "version": 1,
   "versionNonce": 180887984,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "g3",
   "fontSize": 11,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "g3",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0648",
   "type": "rectangle",
   "x": 711,
   "y": 400,
   "width": 32,
   "height": 32,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "#ffd8a8",
   "fillStyle": "solid",
   "strokeWidth": 1.5,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 1320514258,
   "version": 1,
   "versionNonce": 721313367,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0649",
   "type": "text",
   "x": 720.95,
   "y": 408,
   "width": 12.100000000000001,
   "height": 13.75,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 535234445,
   "version": 1,
   "versionNonce": 759440445,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "g4",
   "fontSize": 11,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "g4",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0650",
   "type": "rectangle",
   "x": 750,
   "y": 400,
   "width": 32,
   "height": 32,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "#ffd8a8",
   "fillStyle": "solid",
   "strokeWidth": 1.5,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 2070539393,
   "version": 1,
   "versionNonce": 628889722,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0651",
   "type": "text",
   "x": 759.95,
   "y": 408,
   "width": 12.100000000000001,
   "height": 13.75,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 2038314940,
   "version": 1,
   "versionNonce": 1895710756,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "g5",
   "fontSize": 11,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "g5",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0652",
   "type": "rectangle",
   "x": 789,
   "y": 400,
   "width": 32,
   "height": 32,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "#ffd8a8",
   "fillStyle": "solid",
   "strokeWidth": 1.5,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 296676632,
   "version": 1,
   "versionNonce": 2059721018,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0653",
   "type": "text",
   "x": 798.95,
   "y": 408,
   "width": 12.100000000000001,
   "height": 13.75,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1576211304,
   "version": 1,
   "versionNonce": 1141259551,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "g6",
   "fontSize": 11,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "g6",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0654",
   "type": "rectangle",
   "x": 828,
   "y": 400,
   "width": 32,
   "height": 32,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "#ffd8a8",
   "fillStyle": "solid",
   "strokeWidth": 1.5,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 1213202260,
   "version": 1,
   "versionNonce": 14295021,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0655",
   "type": "text",
   "x": 837.95,
   "y": 408,
   "width": 12.100000000000001,
   "height": 13.75,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1799380663,
   "version": 1,
   "versionNonce": 229000884,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "g7",
   "fontSize": 11,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "g7",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0656",
   "type": "arrow",
   "x": 555,
   "y": 452,
   "width": 305,
   "height": 0,
   "angle": 0,
   "strokeColor": "#1971c2",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 2
   },
   "seed": 328799681,
   "version": 1,
   "versionNonce": 952206616,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "points": [
    [
     0,
     0
    ],
    [
     305,
     0
    ]
   ],
   "lastCommittedPoint": null,
   "startBinding": null,
   "endBinding": null,
   "startArrowhead": null,
   "endArrowhead": null,
   "elbowed": false
  },
  {
   "id": "el0657",
   "type": "text",
   "x": 556.5,
   "y": 458,
   "width": 297.0,
   "height": 30.0,
   "angle": 0,
   "strokeColor": "#1971c2",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 963909550,
   "version": 1,
   "versionNonce": 1766187823,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "TP group of 8: NVLink, 450 GB/s per direction\n4 all-reduces of the hidden state per layer",
   "fontSize": 12,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "TP group of 8: NVLink, 450 GB/s per direction\n4 all-reduces of the hidden state per layer",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0658",
   "type": "arrow",
   "x": 390,
   "y": 148,
   "width": 150,
   "height": 0,
   "angle": 0,
   "strokeColor": "#7048e8",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 2
   },
   "seed": 251129411,
   "version": 1,
   "versionNonce": 1168093690,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "points": [
    [
     0,
     0
    ],
    [
     150,
     0
    ]
   ],
   "lastCommittedPoint": null,
   "startBinding": null,
   "endBinding": null,
   "startArrowhead": null,
   "endArrowhead": "arrow",
   "elbowed": false
  },
  {
   "id": "el0659",
   "type": "arrow",
   "x": 540,
   "y": 160,
   "width": 150,
   "height": 0,
   "angle": 0,
   "strokeColor": "#7048e8",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "dashed",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 2
   },
   "seed": 2018254756,
   "version": 1,
   "versionNonce": 542545406,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "points": [
    [
     0,
     0
    ],
    [
     -150,
     0
    ]
   ],
   "lastCommittedPoint": null,
   "startBinding": null,
   "endBinding": null,
   "startArrowhead": null,
   "endArrowhead": "arrow",
   "elbowed": false
  },
  {
   "id": "el0660",
   "type": "text",
   "x": 392.4,
   "y": 168,
   "width": 145.20000000000002,
   "height": 55.0,
   "angle": 0,
   "strokeColor": "#7048e8",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1128891185,
   "version": 1,
   "versionNonce": 1325360388,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "PP: activations forward,\ngradients back; one\ntensor per microbatch,\nover InfiniBand",
   "fontSize": 11,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "PP: activations forward,\ngradients back; one\ntensor per microbatch,\nover InfiniBand",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0661",
   "type": "arrow",
   "x": 390,
   "y": 408,
   "width": 150,
   "height": 0,
   "angle": 0,
   "strokeColor": "#e8590c",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 2
   },
   "seed": 1786759935,
   "version": 1,
   "versionNonce": 1580987793,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "points": [
    [
     0,
     0
    ],
    [
     150,
     0
    ]
   ],
   "lastCommittedPoint": null,
   "startBinding": null,
   "endBinding": null,
   "startArrowhead": null,
   "endArrowhead": "arrow",
   "elbowed": false
  },
  {
   "id": "el0662",
   "type": "arrow",
   "x": 540,
   "y": 420,
   "width": 150,
   "height": 0,
   "angle": 0,
   "strokeColor": "#e8590c",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "dashed",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 2
   },
   "seed": 1425768744,
   "version": 1,
   "versionNonce": 1890487142,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "points": [
    [
     0,
     0
    ],
    [
     -150,
     0
    ]
   ],
   "lastCommittedPoint": null,
   "startBinding": null,
   "endBinding": null,
   "startArrowhead": null,
   "endArrowhead": "arrow",
   "elbowed": false
  },
  {
   "id": "el0663",
   "type": "text",
   "x": 392.4,
   "y": 428,
   "width": 145.20000000000002,
   "height": 55.0,
   "angle": 0,
   "strokeColor": "#e8590c",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1512374384,
   "version": 1,
   "versionNonce": 7050243,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "PP: activations forward,\ngradients back; one\ntensor per microbatch,\nover InfiniBand",
   "fontSize": 11,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "PP: activations forward,\ngradients back; one\ntensor per microbatch,\nover InfiniBand",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0664",
   "type": "text",
   "x": -19.30000000000001,
   "y": 165.0,
   "width": 69.30000000000001,
   "height": 17.5,
   "angle": 0,
   "strokeColor": "#7048e8",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 893724220,
   "version": 1,
   "versionNonce": 1751137010,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "replica A",
   "fontSize": 14,
   "fontFamily": 1,
   "textAlign": "right",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "replica A",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0665",
   "type": "text",
   "x": -19.30000000000001,
   "y": 425.0,
   "width": 69.30000000000001,
   "height": 17.5,
   "angle": 0,
   "strokeColor": "#e8590c",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 982103307,
   "version": 1,
   "versionNonce": 1027814502,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "replica B",
   "fontSize": 14,
   "fontFamily": 1,
   "textAlign": "right",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "replica B",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0666",
   "type": "arrow",
   "x": 225.0,
   "y": 250,
   "width": 0.0,
   "height": 110,
   "angle": 0,
   "strokeColor": "#e03131",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 2
   },
   "seed": 1016662928,
   "version": 1,
   "versionNonce": 833653894,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "points": [
    [
     0.0,
     0
    ],
    [
     0.0,
     110
    ]
   ],
   "lastCommittedPoint": null,
   "startBinding": null,
   "endBinding": null,
   "startArrowhead": "arrow",
   "endArrowhead": "arrow",
   "elbowed": false
  },
  {
   "id": "el0667",
   "type": "arrow",
   "x": 705.0,
   "y": 250,
   "width": 0.0,
   "height": 110,
   "angle": 0,
   "strokeColor": "#e03131",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 2
   },
   "seed": 348174172,
   "version": 1,
   "versionNonce": 2094308021,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "points": [
    [
     0.0,
     0
    ],
    [
     0.0,
     110
    ]
   ],
   "lastCommittedPoint": null,
   "startBinding": null,
   "endBinding": null,
   "startArrowhead": "arrow",
   "endArrowhead": "arrow",
   "elbowed": false
  },
  {
   "id": "el0668",
   "type": "text",
   "x": 239.0,
   "y": 280,
   "width": 369.6,
   "height": 45.0,
   "angle": 0,
   "strokeColor": "#e03131",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1028719303,
   "version": 1,
   "versionNonce": 639718191,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "DP: gradient all-reduce (or reduce-scatter + all-gather)\nbetween the same stage of each replica, once per step,\noverlapped with the backward pass",
   "fontSize": 12,
   "fontFamily": 1,
   "textAlign": "left",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "DP: gradient all-reduce (or reduce-scatter + all-gather)\nbetween the same stage of each replica, once per step,\noverlapped with the backward pass",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0669",
   "type": "text",
   "x": 719.0,
   "y": 295,
   "width": 105.60000000000001,
   "height": 15.0,
   "angle": 0,
   "strokeColor": "#e03131",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1353687842,
   "version": 1,
   "versionNonce": 441041742,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "same for stage 2",
   "fontSize": 12,
   "fontFamily": 1,
   "textAlign": "left",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "same for stage 2",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0670",
   "type": "text",
   "x": 20,
   "y": 540,
   "width": 931.7,
   "height": 17.5,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1722962933,
   "version": 1,
   "versionNonce": 1634127460,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "GPUs = TP x PP x DP = 8 x 2 x 2 = 32.   Llama 3 405B: TP 8, CP 1, PP 16, DP 64 on 8,192 GPUs; one replica spans 16 nodes.",
   "fontSize": 14,
   "fontFamily": 1,
   "textAlign": "left",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "GPUs = TP x PP x DP = 8 x 2 x 2 = 32.   Llama 3 405B: TP 8, CP 1, PP 16, DP 64 on 8,192 GPUs; one replica spans 16 nodes.",
   "autoResize": true,
   "lineHeight": 1.25
  }
 ],
 "appState": {
  "gridSize": 20,
  "gridStep": 5,
  "gridModeEnabled": false,
  "viewBackgroundColor": "#ffffff"
 },
 "files": {}
}