{ "type": "excalidraw", "version": 2, "source": "https://excalidraw.com", "elements": [ { "id": "e1", "type": "rectangle", "x": 0, "y": 40, "width": 110, "height": 40, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#e9ecef", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": { "type": 3 }, "seed": 1999744785, "version": 1, "versionNonce": 1946412081, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "e2", "type": "text", "x": 6, "y": 51.25, "width": 61.60000000000001, "height": 17.5, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": null, "seed": 1552984409, "version": 1, "versionNonce": 1287489454, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "x [2048]", "originalText": "x [2048]", "fontSize": 14, "fontFamily": 1, "textAlign": "left", "verticalAlign": "top", "containerId": null, "lineHeight": 1.25, "baseline": 12, "autoResize": true }, { "id": "e3", "type": "arrow", "x": 110, "y": 60, "width": 35, "height": 0, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": { "type": 2 }, "seed": 1066984056, "version": 1, "versionNonce": 772092315, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 35, 0 ] ], "lastCommittedPoint": null, "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "e4", "type": "rectangle", "x": 145, "y": 10, "width": 180, "height": 100, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#d0bfff", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": { "type": 3 }, "seed": 1048386556, "version": 1, "versionNonce": 351564608, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "e5", "type": "text", "x": 151, "y": 51.25, "width": 0.0, "height": 17.5, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": null, "seed": 1289560150, "version": 1, "versionNonce": 2126508551, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "", "originalText": "", "fontSize": 14, "fontFamily": 1, "textAlign": "left", "verticalAlign": "top", "containerId": null, "lineHeight": 1.25, "baseline": 12, "autoResize": true }, { "id": "e6", "type": "text", "x": 155, "y": 16, "width": 107.25000000000001, "height": 65.0, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": null, "seed": 1475216846, "version": 1, "versionNonce": 1927728187, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "router\nscores = W_r x\nW_r [128, 2048]\n-> 128 scores", "originalText": "router\nscores = W_r x\nW_r [128, 2048]\n-> 128 scores", "fontSize": 13, "fontFamily": 1, "textAlign": "left", "verticalAlign": "top", "containerId": null, "lineHeight": 1.25, "baseline": 11, "autoResize": true }, { "id": "e7", "type": "arrow", "x": 325, "y": 60, "width": 35, "height": 0, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": { "type": 2 }, "seed": 1236683273, "version": 1, "versionNonce": 314395343, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 35, 0 ] ], "lastCommittedPoint": null, "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "e8", "type": "rectangle", "x": 360, "y": 10, "width": 190, "height": 100, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#d0bfff", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": { "type": 3 }, "seed": 507088657, "version": 1, "versionNonce": 1795823849, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "e9", "type": "text", "x": 366, "y": 51.25, "width": 0.0, "height": 17.5, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": null, "seed": 708506837, "version": 1, "versionNonce": 1469118511, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "", "originalText": "", "fontSize": 14, "fontFamily": 1, "textAlign": "left", "verticalAlign": "top", "containerId": null, "lineHeight": 1.25, "baseline": 12, "autoResize": true }, { "id": "e10", "type": "text", "x": 370, "y": 16, "width": 143.0, "height": 65.0, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": null, "seed": 652768598, "version": 1, "versionNonce": 2100080515, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "softmax over 128\nkeep the top 8:\n 8 expert ids\n 8 weights, sum = 1", "originalText": "softmax over 128\nkeep the top 8:\n 8 expert ids\n 8 weights, sum = 1", "fontSize": 13, "fontFamily": 1, "textAlign": "left", "verticalAlign": "top", "containerId": null, "lineHeight": 1.25, "baseline": 11, "autoResize": true }, { "id": "e11", "type": "arrow", "x": 550, "y": 60, "width": 35, "height": 0, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": { "type": 2 }, "seed": 1811180650, "version": 1, "versionNonce": 168393880, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "points": [ [ 0, 0 ], [ 35, 0 ] ], "lastCommittedPoint": null, "startBinding": null, "endBinding": null, "startArrowhead": null, "endArrowhead": "arrow", "elbowed": false }, { "id": "e12", "type": "rectangle", "x": 585, "y": 10, "width": 300, "height": 100, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "#ebfbee", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": { "type": 3 }, "seed": 333377415, "version": 1, "versionNonce": 1347535309, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false }, { "id": "e13", "type": "text", "x": 591, "y": 51.25, "width": 0.0, "height": 17.5, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": null, "seed": 1460814403, "version": 1, "versionNonce": 1504004732, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "", "originalText": "", "fontSize": 14, "fontFamily": 1, "textAlign": "left", "verticalAlign": "top", "containerId": null, "lineHeight": 1.25, "baseline": 12, "autoResize": true }, { "id": "e14", "type": "text", "x": 595, "y": 16, "width": 214.50000000000003, "height": 65.0, "angle": 0, "strokeColor": "#1e1e1e", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": null, "seed": 2133201996, "version": 1, "versionNonce": 1959386987, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "run the 8 chosen experts\ny = w0 f_a(x) + w1 f_b(x)\n + ... (8 terms)\nthe other 120 experts: no work", "originalText": "run the 8 chosen experts\ny = w0 f_a(x) + w1 f_b(x)\n + ... (8 terms)\nthe other 120 experts: no work", "fontSize": 13, "fontFamily": 1, "textAlign": "left", "verticalAlign": "top", "containerId": null, "lineHeight": 1.25, "baseline": 11, "autoResize": true }, { "id": "e15", "type": "text", "x": 0, "y": 130, "width": 608.3000000000001, "height": 17.5, "angle": 0, "strokeColor": "#1971c2", "backgroundColor": "transparent", "fillStyle": "solid", "strokeWidth": 1.5, "strokeStyle": "solid", "roughness": 1, "opacity": 100, "groupIds": [], "frameId": null, "roundness": null, "seed": 295334610, "version": 1, "versionNonce": 401991736, "isDeleted": false, "boundElements": null, "updated": 1, "link": null, "locked": false, "text": "Expert work per token: 8 x 4,718,592 = 37.7M multiply-adds per layer, not 604M.", "originalText": "Expert work per token: 8 x 4,718,592 = 37.7M multiply-adds per layer, not 604M.", "fontSize": 14, "fontFamily": 1, "textAlign": "left", "verticalAlign": "top", "containerId": null, "lineHeight": 1.25, "baseline": 12, "autoResize": true } ], "appState": { "viewBackgroundColor": "#fdf8f6", "gridSize": null }, "files": {} }