{
 "type": "excalidraw",
 "version": 2,
 "source": "https://excalidraw.com",
 "elements": [
  {
   "id": "el0150",
   "type": "text",
   "x": 20,
   "y": 10,
   "width": 1333.2,
   "height": 30.0,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 945414556,
   "version": 1,
   "versionNonce": 217795102,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "DDP: bucketed gradient all-reduce overlapped with the backward pass   (one rank, 4 layers, 2 buckets)",
   "fontSize": 24,
   "fontFamily": 1,
   "textAlign": "left",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "DDP: bucketed gradient all-reduce overlapped with the backward pass   (one rank, 4 layers, 2 buckets)",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0151",
   "type": "text",
   "x": 92.25,
   "y": 86,
   "width": 57.75000000000001,
   "height": 37.5,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1413979597,
   "version": 1,
   "versionNonce": 12029424,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "compute\nstream",
   "fontSize": 15,
   "fontFamily": 1,
   "textAlign": "right",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "compute\nstream",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0152",
   "type": "text",
   "x": 100.5,
   "y": 166,
   "width": 49.50000000000001,
   "height": 37.5,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1021055601,
   "version": 1,
   "versionNonce": 1819591550,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "comm\nstream",
   "fontSize": 15,
   "fontFamily": 1,
   "textAlign": "right",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "comm\nstream",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0153",
   "type": "rectangle",
   "x": 160,
   "y": 70,
   "width": 220,
   "height": 50,
   "angle": 0,
   "strokeColor": "#1971c2",
   "backgroundColor": "#a5d8ff",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 1255987863,
   "version": 1,
   "versionNonce": 420437962,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0154",
   "type": "text",
   "x": 185.3,
   "y": 86.25,
   "width": 169.4,
   "height": 17.5,
   "angle": 0,
   "strokeColor": "#1971c2",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 98980963,
   "version": 1,
   "versionNonce": 181219263,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "forward, layers 1 to 4",
   "fontSize": 14,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "forward, layers 1 to 4",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0155",
   "type": "rectangle",
   "x": 380,
   "y": 70,
   "width": 110,
   "height": 50,
   "angle": 0,
   "strokeColor": "#2f9e44",
   "backgroundColor": "#b2f2bb",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 1692743095,
   "version": 1,
   "versionNonce": 1961986739,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0156",
   "type": "text",
   "x": 411.9,
   "y": 86.25,
   "width": 46.2,
   "height": 17.5,
   "angle": 0,
   "strokeColor": "#2f9e44",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1593275319,
   "version": 1,
   "versionNonce": 155340859,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "bwd L4",
   "fontSize": 14,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "bwd L4",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0157",
   "type": "rectangle",
   "x": 490,
   "y": 70,
   "width": 110,
   "height": 50,
   "angle": 0,
   "strokeColor": "#2f9e44",
   "backgroundColor": "#b2f2bb",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 293647116,
   "version": 1,
   "versionNonce": 329253120,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0158",
   "type": "text",
   "x": 521.9,
   "y": 86.25,
   "width": 46.2,
   "height": 17.5,
   "angle": 0,
   "strokeColor": "#2f9e44",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1949487058,
   "version": 1,
   "versionNonce": 789299093,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "bwd L3",
   "fontSize": 14,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "bwd L3",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0159",
   "type": "rectangle",
   "x": 600,
   "y": 70,
   "width": 110,
   "height": 50,
   "angle": 0,
   "strokeColor": "#2f9e44",
   "backgroundColor": "#b2f2bb",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 973357529,
   "version": 1,
   "versionNonce": 798231919,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0160",
   "type": "text",
   "x": 631.9,
   "y": 86.25,
   "width": 46.2,
   "height": 17.5,
   "angle": 0,
   "strokeColor": "#2f9e44",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 2014371816,
   "version": 1,
   "versionNonce": 51565850,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "bwd L2",
   "fontSize": 14,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "bwd L2",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0161",
   "type": "rectangle",
   "x": 710,
   "y": 70,
   "width": 110,
   "height": 50,
   "angle": 0,
   "strokeColor": "#2f9e44",
   "backgroundColor": "#b2f2bb",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 539318261,
   "version": 1,
   "versionNonce": 305526307,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0162",
   "type": "text",
   "x": 741.9,
   "y": 86.25,
   "width": 46.2,
   "height": 17.5,
   "angle": 0,
   "strokeColor": "#2f9e44",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 156064877,
   "version": 1,
   "versionNonce": 798362831,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "bwd L1",
   "fontSize": 14,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "bwd L1",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0163",
   "type": "text",
   "x": 384.4,
   "y": 124,
   "width": 211.20000000000002,
   "height": 15.0,
   "angle": 0,
   "strokeColor": "#e03131",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 453491930,
   "version": 1,
   "versionNonce": 1279174507,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "bucket 1 ready (grads of L4, L3)",
   "fontSize": 12,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "bucket 1 ready (grads of L4, L3)",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0164",
   "type": "text",
   "x": 604.4,
   "y": 124,
   "width": 211.20000000000002,
   "height": 15.0,
   "angle": 0,
   "strokeColor": "#e03131",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1797243929,
   "version": 1,
   "versionNonce": 546402529,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "bucket 2 ready (grads of L2, L1)",
   "fontSize": 12,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "bucket 2 ready (grads of L2, L1)",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0165",
   "type": "rectangle",
   "x": 600,
   "y": 150,
   "width": 176.0,
   "height": 50,
   "angle": 0,
   "strokeColor": "#e03131",
   "backgroundColor": "#ffc9c9",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 1776630050,
   "version": 1,
   "versionNonce": 2071145578,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0166",
   "type": "text",
   "x": 614.85,
   "y": 166.25,
   "width": 146.3,
   "height": 17.5,
   "angle": 0,
   "strokeColor": "#e03131",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1544789116,
   "version": 1,
   "versionNonce": 1589940333,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "all-reduce bucket 1",
   "fontSize": 14,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "all-reduce bucket 1",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0167",
   "type": "rectangle",
   "x": 820,
   "y": 150,
   "width": 176.0,
   "height": 50,
   "angle": 0,
   "strokeColor": "#e03131",
   "backgroundColor": "#ffc9c9",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 1023062721,
   "version": 1,
   "versionNonce": 531942588,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0168",
   "type": "text",
   "x": 834.85,
   "y": 166.25,
   "width": 146.3,
   "height": 17.5,
   "angle": 0,
   "strokeColor": "#e03131",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1089125974,
   "version": 1,
   "versionNonce": 1060080286,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "all-reduce bucket 2",
   "fontSize": 14,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "all-reduce bucket 2",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0169",
   "type": "arrow",
   "x": 600,
   "y": 120,
   "width": 0,
   "height": 30,
   "angle": 0,
   "strokeColor": "#e03131",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 1.5,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 2
   },
   "seed": 1682636248,
   "version": 1,
   "versionNonce": 1535755299,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "points": [
    [
     0,
     0
    ],
    [
     0,
     30
    ]
   ],
   "lastCommittedPoint": null,
   "startBinding": null,
   "endBinding": null,
   "startArrowhead": null,
   "endArrowhead": "arrow",
   "elbowed": false
  },
  {
   "id": "el0170",
   "type": "arrow",
   "x": 820,
   "y": 120,
   "width": 0,
   "height": 30,
   "angle": 0,
   "strokeColor": "#e03131",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 1.5,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 2
   },
   "seed": 1558951614,
   "version": 1,
   "versionNonce": 80355448,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "points": [
    [
     0,
     0
    ],
    [
     0,
     30
    ]
   ],
   "lastCommittedPoint": null,
   "startBinding": null,
   "endBinding": null,
   "startArrowhead": null,
   "endArrowhead": "arrow",
   "elbowed": false
  },
  {
   "id": "el0171",
   "type": "text",
   "x": 608.8,
   "y": 204,
   "width": 158.4,
   "height": 15.0,
   "angle": 0,
   "strokeColor": "#2f9e44",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 2146143696,
   "version": 1,
   "versionNonce": 1907825126,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "hidden behind bwd L2, L1",
   "fontSize": 12,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "hidden behind bwd L2, L1",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0172",
   "type": "text",
   "x": 838.7,
   "y": 204,
   "width": 138.60000000000002,
   "height": 30.0,
   "angle": 0,
   "strokeColor": "#e03131",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1581447923,
   "version": 1,
   "versionNonce": 709919005,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "exposed: nothing left\nto overlap with",
   "fontSize": 12,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "exposed: nothing left\nto overlap with",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0173",
   "type": "rectangle",
   "x": 996.0,
   "y": 70,
   "width": 99.0,
   "height": 50,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "#dee2e6",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 1947875463,
   "version": 1,
   "versionNonce": 803365649,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0174",
   "type": "text",
   "x": 1013.325,
   "y": 78.75,
   "width": 64.35000000000001,
   "height": 32.5,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1160444996,
   "version": 1,
   "versionNonce": 325692868,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "optimizer\nstep",
   "fontSize": 13,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "optimizer\nstep",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0175",
   "type": "arrow",
   "x": 996.0,
   "y": 175,
   "width": 0.0,
   "height": 55,
   "angle": 0,
   "strokeColor": "#e03131",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 1.5,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 2
   },
   "seed": 538950116,
   "version": 1,
   "versionNonce": 2229047,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "points": [
    [
     0.0,
     0
    ],
    [
     0.0,
     -55
    ]
   ],
   "lastCommittedPoint": null,
   "startBinding": null,
   "endBinding": null,
   "startArrowhead": null,
   "endArrowhead": "arrow",
   "elbowed": false
  },
  {
   "id": "el0176",
   "type": "arrow",
   "x": 160,
   "y": 235,
   "width": 946.0,
   "height": 0,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 1.5,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 2
   },
   "seed": 309609513,
   "version": 1,
   "versionNonce": 299010498,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "points": [
    [
     0,
     0
    ],
    [
     946.0,
     0
    ]
   ],
   "lastCommittedPoint": null,
   "startBinding": null,
   "endBinding": null,
   "startArrowhead": null,
   "endArrowhead": "arrow",
   "elbowed": false
  },
  {
   "id": "el0177",
   "type": "text",
   "x": 1112.0,
   "y": 227,
   "width": 28.6,
   "height": 16.25,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 690009422,
   "version": 1,
   "versionNonce": 1714185519,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "time",
   "fontSize": 13,
   "fontFamily": 1,
   "textAlign": "left",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "time",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0178",
   "type": "text",
   "x": 160,
   "y": 242,
   "width": 950.95,
   "height": 16.25,
   "angle": 0,
   "strokeColor": "#2f9e44",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 842169929,
   "version": 1,
   "versionNonce": 1470350162,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "hooks fire as each parameter's gradient is produced; a bucket launches its all-reduce the moment it fills, in reverse parameter order",
   "fontSize": 13,
   "fontFamily": 1,
   "textAlign": "left",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "hooks fire as each parameter's gradient is produced; a bucket launches its all-reduce the moment it fills, in reverse parameter order",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0179",
   "type": "text",
   "x": 20,
   "y": 300,
   "width": 960.3000000000001,
   "height": 22.5,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 2128790331,
   "version": 1,
   "versionNonce": 1028242670,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "With gradient accumulation over k microbatches:  no_sync() for the first k-1, the collective once",
   "fontSize": 18,
   "fontFamily": 1,
   "textAlign": "left",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "With gradient accumulation over k microbatches:  no_sync() for the first k-1, the collective once",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0180",
   "type": "rectangle",
   "x": 160,
   "y": 340,
   "width": 143.0,
   "height": 55,
   "angle": 0,
   "strokeColor": "#2f9e44",
   "backgroundColor": "#b2f2bb",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 835596019,
   "version": 1,
   "versionNonce": 1794371043,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0181",
   "type": "text",
   "x": 188.6,
   "y": 351.25,
   "width": 85.80000000000001,
   "height": 32.5,
   "angle": 0,
   "strokeColor": "#2f9e44",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1386724011,
   "version": 1,
   "versionNonce": 382958290,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "fwd + bwd\nmicrobatch 1",
   "fontSize": 13,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "fwd + bwd\nmicrobatch 1",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0182",
   "type": "text",
   "x": 172.1,
   "y": 400,
   "width": 118.80000000000001,
   "height": 30.0,
   "angle": 0,
   "strokeColor": "#868e96",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 733205241,
   "version": 1,
   "versionNonce": 227352617,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "no_sync(): grads\naccumulate locally",
   "fontSize": 12,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "no_sync(): grads\naccumulate locally",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0183",
   "type": "rectangle",
   "x": 323.0,
   "y": 340,
   "width": 143.0,
   "height": 55,
   "angle": 0,
   "strokeColor": "#2f9e44",
   "backgroundColor": "#b2f2bb",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 1631523918,
   "version": 1,
   "versionNonce": 27764601,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0184",
   "type": "text",
   "x": 351.6,
   "y": 351.25,
   "width": 85.80000000000001,
   "height": 32.5,
   "angle": 0,
   "strokeColor": "#2f9e44",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 576040299,
   "version": 1,
   "versionNonce": 1415654805,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "fwd + bwd\nmicrobatch 2",
   "fontSize": 13,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "fwd + bwd\nmicrobatch 2",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0185",
   "type": "text",
   "x": 335.1,
   "y": 400,
   "width": 118.80000000000001,
   "height": 30.0,
   "angle": 0,
   "strokeColor": "#868e96",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 435240054,
   "version": 1,
   "versionNonce": 253932491,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "no_sync(): grads\naccumulate locally",
   "fontSize": 12,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "no_sync(): grads\naccumulate locally",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0186",
   "type": "text",
   "x": 496.0,
   "y": 358,
   "width": 33.0,
   "height": 25.0,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 925146572,
   "version": 1,
   "versionNonce": 566862937,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "...",
   "fontSize": 20,
   "fontFamily": 1,
   "textAlign": "left",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "...",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0187",
   "type": "rectangle",
   "x": 536.0,
   "y": 340,
   "width": 143.0,
   "height": 55,
   "angle": 0,
   "strokeColor": "#1971c2",
   "backgroundColor": "#a5d8ff",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 755471033,
   "version": 1,
   "versionNonce": 578703884,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0188",
   "type": "text",
   "x": 564.6,
   "y": 351.25,
   "width": 85.80000000000001,
   "height": 32.5,
   "angle": 0,
   "strokeColor": "#1971c2",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1200962277,
   "version": 1,
   "versionNonce": 310939419,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "fwd + bwd\nmicrobatch k",
   "fontSize": 13,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "fwd + bwd\nmicrobatch k",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0189",
   "type": "text",
   "x": 544.8,
   "y": 400,
   "width": 125.4,
   "height": 30.0,
   "angle": 0,
   "strokeColor": "#e03131",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1196967519,
   "version": 1,
   "versionNonce": 885848550,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "all-reduce\noverlapped as above",
   "fontSize": 12,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "all-reduce\noverlapped as above",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0190",
   "type": "rectangle",
   "x": 699.0,
   "y": 340,
   "width": 99.0,
   "height": 55,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "#dee2e6",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": {
    "type": 3
   },
   "seed": 325415290,
   "version": 1,
   "versionNonce": 67138105,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false
  },
  {
   "id": "el0191",
   "type": "text",
   "x": 716.325,
   "y": 351.25,
   "width": 64.35000000000001,
   "height": 32.5,
   "angle": 0,
   "strokeColor": "#1e1e1e",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1797965518,
   "version": 1,
   "versionNonce": 1706847584,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "optimizer\nstep",
   "fontSize": 13,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "optimizer\nstep",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0192",
   "type": "text",
   "x": 699.0,
   "y": 400,
   "width": 99.00000000000001,
   "height": 30.0,
   "angle": 0,
   "strokeColor": "#868e96",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1992657064,
   "version": 1,
   "versionNonce": 1432004044,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "on the sum\n(scaled by 1/k)",
   "fontSize": 12,
   "fontFamily": 1,
   "textAlign": "center",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "on the sum\n(scaled by 1/k)",
   "autoResize": true,
   "lineHeight": 1.25
  },
  {
   "id": "el0193",
   "type": "text",
   "x": 160,
   "y": 445,
   "width": 936.6500000000001,
   "height": 16.25,
   "angle": 0,
   "strokeColor": "#2f9e44",
   "backgroundColor": "transparent",
   "fillStyle": "solid",
   "strokeWidth": 2,
   "strokeStyle": "solid",
   "roughness": 1,
   "opacity": 100,
   "groupIds": [],
   "frameId": null,
   "index": null,
   "roundness": null,
   "seed": 1510480095,
   "version": 1,
   "versionNonce": 22448991,
   "isDeleted": false,
   "boundElements": null,
   "updated": 1,
   "link": null,
   "locked": false,
   "text": "activations: one microbatch at a time.  gradient buffer: one, reused.  communication: once per step instead of once per microbatch.",
   "fontSize": 13,
   "fontFamily": 1,
   "textAlign": "left",
   "verticalAlign": "top",
   "containerId": null,
   "originalText": "activations: one microbatch at a time.  gradient buffer: one, reused.  communication: once per step instead of once per microbatch.",
   "autoResize": true,
   "lineHeight": 1.25
  }
 ],
 "appState": {
  "gridSize": 20,
  "gridStep": 5,
  "gridModeEnabled": false,
  "viewBackgroundColor": "#ffffff"
 },
 "files": {}
}