{
 "type": "excalidraw",
 "version": 2,
 "source": "https://excalidraw.com",
 "elements": [
  {
   "id": "el0194",
   "type": "text",
   "x": 20,
   "y": 10,
   "width": 1227.6000000000001,
   "height": 30.0,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1262743003,
   "version": 1,
   "versionNonce": 1808885496,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "FSDP: all-gather, compute, free; with the next unit's gather prefetched   (one rank, 3 units)",
   "fontSize": 24,
   "fontFamily": 1,
   "textAlign": "left",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "FSDP: all-gather, compute, free; with the next unit's gather prefetched   (one rank, 3 units)",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0195",
   "type": "text",
   "x": 92.25,
   "y": 96,
   "width": 57.75000000000001,
   "height": 37.5,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 720504104,
   "version": 1,
   "versionNonce": 2062944666,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "compute\nstream",
   "fontSize": 15,
   "fontFamily": 1,
   "textAlign": "right",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "compute\nstream",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0196",
   "type": "text",
   "x": 100.5,
   "y": 176,
   "width": 49.50000000000001,
   "height": 37.5,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 234726755,
   "version": 1,
   "versionNonce": 1129343760,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "comm\nstream",
   "fontSize": 15,
   "fontFamily": 1,
   "textAlign": "right",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "comm\nstream",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0197",
   "type": "text",
   "x": 84.0,
   "y": 281,
   "width": 66.0,
   "height": 37.5,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 680644849,
   "version": 1,
   "versionNonce": 1678940540,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "resident\nparams",
   "fontSize": 15,
   "fontFamily": 1,
   "textAlign": "right",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "resident\nparams",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0198",
   "type": "rectangle",
   "x": 260,
   "y": 80,
   "width": 100,
   "height": 50,
   "angle": 0,
   "strokeColor": "#2f9e44",
   "backgroundColor": "#b2f2bb",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 615498983,
   "version": 1,
   "versionNonce": 1397648777,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0199",
   "type": "text",
   "x": 286.9,
   "y": 96.25,
   "width": 46.2,
   "height": 17.5,
   "angle": 0,
   "strokeColor": "#2f9e44",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 915917922,
   "version": 1,
   "versionNonce": 641694045,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "fwd U1",
   "fontSize": 14,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "fwd U1",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0200",
   "type": "rectangle",
   "x": 160,
   "y": 160,
   "width": 95.0,
   "height": 50,
   "angle": 0,
   "strokeColor": "#1971c2",
   "backgroundColor": "#a5d8ff",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 1585771217,
   "version": 1,
   "versionNonce": 459710729,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0201",
   "type": "text",
   "x": 188.25,
   "y": 176.25,
   "width": 38.5,
   "height": 17.5,
   "angle": 0,
   "strokeColor": "#1971c2",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1130143450,
   "version": 1,
   "versionNonce": 1886332726,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "AG U1",
   "fontSize": 14,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "AG U1",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0202",
   "type": "rectangle",
   "x": 260,
   "y": 265,
   "width": 100,
   "height": 40,
   "angle": 0,
   "strokeColor": "#868e96",
   "backgroundColor": "#dee2e6",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 1008708578,
   "version": 1,
   "versionNonce": 1559977170,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0203",
   "type": "text",
   "x": 286.9,
   "y": 277.5,
   "width": 46.2,
   "height": 15.0,
   "angle": 0,
   "strokeColor": "#868e96",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 957567981,
   "version": 1,
   "versionNonce": 1271373043,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "U1 full",
   "fontSize": 12,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "U1 full",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0204",
   "type": "rectangle",
   "x": 360,
   "y": 80,
   "width": 100,
   "height": 50,
   "angle": 0,
   "strokeColor": "#2f9e44",
   "backgroundColor": "#b2f2bb",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 1849515577,
   "version": 1,
   "versionNonce": 401196687,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0205",
   "type": "text",
   "x": 386.9,
   "y": 96.25,
   "width": 46.2,
   "height": 17.5,
   "angle": 0,
   "strokeColor": "#2f9e44",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1982291010,
   "version": 1,
   "versionNonce": 228516014,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "fwd U2",
   "fontSize": 14,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "fwd U2",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0206",
   "type": "rectangle",
   "x": 260,
   "y": 160,
   "width": 95.0,
   "height": 50,
   "angle": 0,
   "strokeColor": "#1971c2",
   "backgroundColor": "#a5d8ff",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 390159449,
   "version": 1,
   "versionNonce": 1667404727,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0207",
   "type": "text",
   "x": 288.25,
   "y": 176.25,
   "width": 38.5,
   "height": 17.5,
   "angle": 0,
   "strokeColor": "#1971c2",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1798176963,
   "version": 1,
   "versionNonce": 173410990,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "AG U2",
   "fontSize": 14,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "AG U2",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0208",
   "type": "rectangle",
   "x": 360,
   "y": 265,
   "width": 100,
   "height": 40,
   "angle": 0,
   "strokeColor": "#868e96",
   "backgroundColor": "#dee2e6",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 1889270413,
   "version": 1,
   "versionNonce": 526824568,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0209",
   "type": "text",
   "x": 386.9,
   "y": 277.5,
   "width": 46.2,
   "height": 15.0,
   "angle": 0,
   "strokeColor": "#868e96",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1384463635,
   "version": 1,
   "versionNonce": 895071907,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "U2 full",
   "fontSize": 12,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "U2 full",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0210",
   "type": "rectangle",
   "x": 460,
   "y": 80,
   "width": 100,
   "height": 50,
   "angle": 0,
   "strokeColor": "#2f9e44",
   "backgroundColor": "#b2f2bb",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 2074196379,
   "version": 1,
   "versionNonce": 2034221949,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0211",
   "type": "text",
   "x": 486.9,
   "y": 96.25,
   "width": 46.2,
   "height": 17.5,
   "angle": 0,
   "strokeColor": "#2f9e44",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 102922772,
   "version": 1,
   "versionNonce": 1550451940,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "fwd U3",
   "fontSize": 14,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "fwd U3",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0212",
   "type": "rectangle",
   "x": 360,
   "y": 160,
   "width": 95.0,
   "height": 50,
   "angle": 0,
   "strokeColor": "#1971c2",
   "backgroundColor": "#a5d8ff",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 611410364,
   "version": 1,
   "versionNonce": 1910490815,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0213",
   "type": "text",
   "x": 388.25,
   "y": 176.25,
   "width": 38.5,
   "height": 17.5,
   "angle": 0,
   "strokeColor": "#1971c2",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1745614390,
   "version": 1,
   "versionNonce": 1537455456,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "AG U3",
   "fontSize": 14,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "AG U3",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0214",
   "type": "rectangle",
   "x": 460,
   "y": 265,
   "width": 100,
   "height": 40,
   "angle": 0,
   "strokeColor": "#868e96",
   "backgroundColor": "#dee2e6",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 1739895632,
   "version": 1,
   "versionNonce": 59359669,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0215",
   "type": "text",
   "x": 486.9,
   "y": 277.5,
   "width": 46.2,
   "height": 15.0,
   "angle": 0,
   "strokeColor": "#868e96",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1833752151,
   "version": 1,
   "versionNonce": 1959303282,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "U3 full",
   "fontSize": 12,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "U3 full",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0216",
   "type": "text",
   "x": 282.95,
   "y": 310,
   "width": 254.10000000000002,
   "height": 13.75,
   "angle": 0,
   "strokeColor": "#868e96",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 650525665,
   "version": 1,
   "versionNonce": 1884319038,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "each unit is freed right after its forward",
   "fontSize": 11,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "each unit is freed right after its forward",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0217",
   "type": "text",
   "x": 160,
   "y": 214,
   "width": 217.8,
   "height": 30.0,
   "angle": 0,
   "strokeColor": "#2f9e44",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 176990991,
   "version": 1,
   "versionNonce": 355165856,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "AG U2 runs while fwd U1 computes:\nforward prefetch",
   "fontSize": 12,
   "fontFamily": 1,
   "textAlign": "left",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "AG U2 runs while fwd U1 computes:\nforward prefetch",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0218",
   "type": "text",
   "x": 620.0,
   "y": 54,
   "width": 154.0,
   "height": 17.5,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1636089888,
   "version": 1,
   "versionNonce": 1912332199,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "backward, in reverse",
   "fontSize": 14,
   "fontFamily": 1,
   "textAlign": "left",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "backward, in reverse",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0219",
   "type": "rectangle",
   "x": 720.0,
   "y": 80,
   "width": 100,
   "height": 50,
   "angle": 0,
   "strokeColor": "#2f9e44",
   "backgroundColor": "#b2f2bb",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 418514305,
   "version": 1,
   "versionNonce": 1340181868,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0220",
   "type": "text",
   "x": 746.9,
   "y": 96.25,
   "width": 46.2,
   "height": 17.5,
   "angle": 0,
   "strokeColor": "#2f9e44",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1782436257,
   "version": 1,
   "versionNonce": 1043132484,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "bwd U3",
   "fontSize": 14,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "bwd U3",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0221",
   "type": "rectangle",
   "x": 720.0,
   "y": 265,
   "width": 100,
   "height": 40,
   "angle": 0,
   "strokeColor": "#868e96",
   "backgroundColor": "#dee2e6",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 1402108039,
   "version": 1,
   "versionNonce": 284476242,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0222",
   "type": "text",
   "x": 746.9,
   "y": 277.5,
   "width": 46.2,
   "height": 15.0,
   "angle": 0,
   "strokeColor": "#868e96",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 2084533065,
   "version": 1,
   "versionNonce": 2047548630,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "U3 full",
   "fontSize": 12,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "U3 full",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0223",
   "type": "rectangle",
   "x": 820.0,
   "y": 80,
   "width": 100,
   "height": 50,
   "angle": 0,
   "strokeColor": "#2f9e44",
   "backgroundColor": "#b2f2bb",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 2139939085,
   "version": 1,
   "versionNonce": 1653351288,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0224",
   "type": "text",
   "x": 846.9,
   "y": 96.25,
   "width": 46.2,
   "height": 17.5,
   "angle": 0,
   "strokeColor": "#2f9e44",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 547589596,
   "version": 1,
   "versionNonce": 1475659535,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "bwd U2",
   "fontSize": 14,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "bwd U2",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0225",
   "type": "rectangle",
   "x": 820.0,
   "y": 265,
   "width": 100,
   "height": 40,
   "angle": 0,
   "strokeColor": "#868e96",
   "backgroundColor": "#dee2e6",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 1333112033,
   "version": 1,
   "versionNonce": 235655899,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0226",
   "type": "text",
   "x": 846.9,
   "y": 277.5,
   "width": 46.2,
   "height": 15.0,
   "angle": 0,
   "strokeColor": "#868e96",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1623192593,
   "version": 1,
   "versionNonce": 200454806,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "U2 full",
   "fontSize": 12,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "U2 full",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0227",
   "type": "rectangle",
   "x": 920.0,
   "y": 80,
   "width": 100,
   "height": 50,
   "angle": 0,
   "strokeColor": "#2f9e44",
   "backgroundColor": "#b2f2bb",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 977240358,
   "version": 1,
   "versionNonce": 713426600,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0228",
   "type": "text",
   "x": 946.9,
   "y": 96.25,
   "width": 46.2,
   "height": 17.5,
   "angle": 0,
   "strokeColor": "#2f9e44",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 151763357,
   "version": 1,
   "versionNonce": 971232981,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "bwd U1",
   "fontSize": 14,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "bwd U1",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0229",
   "type": "rectangle",
   "x": 920.0,
   "y": 265,
   "width": 100,
   "height": 40,
   "angle": 0,
   "strokeColor": "#868e96",
   "backgroundColor": "#dee2e6",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 658957653,
   "version": 1,
   "versionNonce": 2054370368,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0230",
   "type": "text",
   "x": 946.9,
   "y": 277.5,
   "width": 46.2,
   "height": 15.0,
   "angle": 0,
   "strokeColor": "#868e96",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1759161856,
   "version": 1,
   "versionNonce": 1457694535,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "U1 full",
   "fontSize": 12,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "U1 full",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0231",
   "type": "rectangle",
   "x": 620.0,
   "y": 160,
   "width": 95.0,
   "height": 50,
   "angle": 0,
   "strokeColor": "#1971c2",
   "backgroundColor": "#a5d8ff",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 118026738,
   "version": 1,
   "versionNonce": 215070303,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0232",
   "type": "text",
   "x": 649.625,
   "y": 176.875,
   "width": 35.75,
   "height": 16.25,
   "angle": 0,
   "strokeColor": "#1971c2",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1322492583,
   "version": 1,
   "versionNonce": 207213727,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "AG U3",
   "fontSize": 13,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "AG U3",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0233",
   "type": "rectangle",
   "x": 720.0,
   "y": 160,
   "width": 47.0,
   "height": 50,
   "angle": 0,
   "strokeColor": "#1971c2",
   "backgroundColor": "#a5d8ff",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 783823964,
   "version": 1,
   "versionNonce": 411578354,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0234",
   "type": "text",
   "x": 728.375,
   "y": 178.125,
   "width": 30.250000000000004,
   "height": 13.75,
   "angle": 0,
   "strokeColor": "#1971c2",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 254203620,
   "version": 1,
   "versionNonce": 227795212,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "AG U2",
   "fontSize": 11,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "AG U2",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0235",
   "type": "rectangle",
   "x": 770.0,
   "y": 160,
   "width": 47.0,
   "height": 50,
   "angle": 0,
   "strokeColor": "#e03131",
   "backgroundColor": "#ffc9c9",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 2088188081,
   "version": 1,
   "versionNonce": 719741427,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0236",
   "type": "text",
   "x": 778.375,
   "y": 178.125,
   "width": 30.250000000000004,
   "height": 13.75,
   "angle": 0,
   "strokeColor": "#e03131",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 791936300,
   "version": 1,
   "versionNonce": 1808700932,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "RS U3",
   "fontSize": 11,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "RS U3",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0237",
   "type": "rectangle",
   "x": 820.0,
   "y": 160,
   "width": 47.0,
   "height": 50,
   "angle": 0,
   "strokeColor": "#1971c2",
   "backgroundColor": "#a5d8ff",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 331547934,
   "version": 1,
   "versionNonce": 1726618972,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0238",
   "type": "text",
   "x": 828.375,
   "y": 178.125,
   "width": 30.250000000000004,
   "height": 13.75,
   "angle": 0,
   "strokeColor": "#1971c2",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 927433061,
   "version": 1,
   "versionNonce": 660036178,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "AG U1",
   "fontSize": 11,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "AG U1",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0239",
   "type": "rectangle",
   "x": 870.0,
   "y": 160,
   "width": 47.0,
   "height": 50,
   "angle": 0,
   "strokeColor": "#e03131",
   "backgroundColor": "#ffc9c9",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 1183543151,
   "version": 1,
   "versionNonce": 823379054,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0240",
   "type": "text",
   "x": 878.375,
   "y": 178.125,
   "width": 30.250000000000004,
   "height": 13.75,
   "angle": 0,
   "strokeColor": "#e03131",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1598791423,
   "version": 1,
   "versionNonce": 121874930,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "RS U2",
   "fontSize": 11,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "RS U2",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0241",
   "type": "rectangle",
   "x": 920.0,
   "y": 160,
   "width": 95.0,
   "height": 50,
   "angle": 0,
   "strokeColor": "#e03131",
   "backgroundColor": "#ffc9c9",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 1198552465,
   "version": 1,
   "versionNonce": 1708838405,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0242",
   "type": "text",
   "x": 949.625,
   "y": 176.875,
   "width": 35.75,
   "height": 16.25,
   "angle": 0,
   "strokeColor": "#e03131",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1753206471,
   "version": 1,
   "versionNonce": 2035854475,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "RS U1",
   "fontSize": 13,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "RS U1",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0243",
   "type": "text",
   "x": 595.6,
   "y": 214,
   "width": 448.8,
   "height": 30.0,
   "angle": 0,
   "strokeColor": "#2f9e44",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1216150713,
   "version": 1,
   "versionNonce": 1235519311,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "blue AG: weights gathered again, prefetched during the previous unit\nred RS: reduce-scatter the gradient; each rank keeps only its shard",
   "fontSize": 12,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "blue AG: weights gathered again, prefetched during the previous unit\nred RS: reduce-scatter the gradient; each rank keeps only its shard",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0244",
   "type": "arrow",
   "x": 160,
   "y": 345,
   "width": 880.0,
   "height": 0,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 1.5,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 2
   },
   "seed": 713707393,
   "version": 1,
   "versionNonce": 1362280887,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "points": [
    [
     0,
     0
    ],
    [
     880.0,
     0
    ]
   ],
   "lastCommittedPoint": null,
   "startBinding": null,
   "endBinding": null,
   "startArrowhead": null,
   "endArrowhead": "arrow",
   "elbowed": false
  },
  {
   "id": "el0245",
   "type": "text",
   "x": 1046.0,
   "y": 337,
   "width": 28.6,
   "height": 16.25,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 2072419705,
   "version": 1,
   "versionNonce": 1415863786,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "time",
   "fontSize": 13,
   "fontFamily": 1,
   "textAlign": "left",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "time",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0246",
   "type": "text",
   "x": 160,
   "y": 355,
   "width": 962.5000000000001,
   "height": 17.5,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1193821734,
   "version": 1,
   "versionNonce": 1120573410,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "Per unit per step: 1 all-gather (fwd) + 1 all-gather (bwd) + 1 reduce-scatter = 3 x (params of the unit): the 1.5x of ZeRO-3.",
   "fontSize": 14,
   "fontFamily": 1,
   "textAlign": "left",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "Per unit per step: 1 all-gather (fwd) + 1 all-gather (bwd) + 1 reduce-scatter = 3 x (params of the unit): the 1.5x of ZeRO-3.",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0247",
   "type": "text",
   "x": 160,
   "y": 380,
   "width": 831.6,
   "height": 17.5,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1957595365,
   "version": 1,
   "versionNonce": 1854956635,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "Steady-state memory: the sharded state (16 bytes/param divided by D) plus one or two units gathered in full.",
   "fontSize": 14,
   "fontFamily": 1,
   "textAlign": "left",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "Steady-state memory: the sharded state (16 bytes/param divided by D) plus one or two units gathered in full.",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0248",
   "type": "text",
   "x": 160,
   "y": 405,
   "width": 754.6,
   "height": 17.5,
   "angle": 0,
   "strokeColor": "#2f9e44",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 451557609,
   "version": 1,
   "versionNonce": 191278636,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "Without prefetch every unit waits for its own gather: the extra all-gather would be fully exposed.",
   "fontSize": 14,
   "fontFamily": 1,
   "textAlign": "left",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "Without prefetch every unit waits for its own gather: the extra all-gather would be fully exposed.",
   "autoResize": true,
   "lineHeight": 1.25
  }
 ],
 "appState": {
  "gridSize": 20,
  "gridStep": 5,
  "gridModeEnabled": false,
  "viewBackgroundColor": "#ffffff"
 },
 "files": {}
}