{"cells":[{"cell_type":"markdown","id":"cb2c0c7c","metadata":{"papermill":{"duration":0.004501,"end_time":"2026-08-10T19:08:41.647217+00:00","exception":false,"start_time":"2026-08-10T19:08:41.642716+00:00","status":"completed"},"tags":[]},"source":["# DeepSeek-V3 FULL model architecture pretraining on arXiv — Kaggle\n","\n","This notebook keeps the complete DeepSeek-V3 **model architecture** while reducing only widths/counts so it can run on Kaggle.\n","\n","Included:\n","- Multi-Head Latent Attention (MLA)\n","- decoupled RoPE / non-RoPE QK dimensions + YaRN\n","- DeepSeekMoE shared + routed experts\n","- sigmoid grouped top-k routing with correction bias\n","- dense early layers -> MoE later layers\n","- RMSNorm + SiLU gated MLP\n","- **sequential Multi-Token Prediction (MTP)**\n"," - `enorm`\n"," - `hnorm`\n"," - `eh_proj` (`2d -> d`)\n"," - appended full Transformer/MoE MTP layer(s)\n"," - shared token embedding\n"," - shared LM output projection\n"," - weighted MTP auxiliary cross-entropy objective\n","\n","The released DeepSeek-V3 uses 61 main layers plus 1 MTP module; this notebook preserves that layout pattern as\n","`num_hidden_layers` main layers followed by `num_nextn_predict_layers` MTP layer(s).\n","\n","Dataset:\n","`/kaggle/input/datasets/organizations/Cornell-University/arxiv/arxiv-metadata-oai-snapshot.json`\n","\n","Training text:\n","`title + abstract`\n"]},{"cell_type":"code","execution_count":1,"id":"6d192e5f","metadata":{"execution":{"iopub.execute_input":"2026-08-10T19:08:41.655254Z","iopub.status.busy":"2026-08-10T19:08:41.654791Z","iopub.status.idle":"2026-08-10T19:08:58.911562Z","shell.execute_reply":"2026-08-10T19:08:58.910475Z"},"papermill":{"duration":17.263416,"end_time":"2026-08-10T19:08:58.913822+00:00","exception":false,"start_time":"2026-08-10T19:08:41.650406+00:00","status":"completed"},"tags":[]},"outputs":[{"name":"stdout","output_type":"stream","text":["\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m44.1/44.1 kB\u001b[0m \u001b[31m1.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n","\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m10.0/10.0 MB\u001b[0m \u001b[31m88.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n","\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.0/3.0 MB\u001b[0m \u001b[31m85.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n","\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m566.4/566.4 kB\u001b[0m \u001b[31m34.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n","\u001b[?25h"]}],"source":["\n","# 1. INSTALL\n","!pip install -q \"transformers==4.46.3\" \"tokenizers>=0.20,<0.21\" sentencepiece\n"]},{"cell_type":"code","execution_count":2,"id":"e5007046","metadata":{"execution":{"iopub.execute_input":"2026-08-10T19:08:58.92368Z","iopub.status.busy":"2026-08-10T19:08:58.923028Z","iopub.status.idle":"2026-08-10T19:08:58.940082Z","shell.execute_reply":"2026-08-10T19:08:58.939278Z"},"papermill":{"duration":0.024008,"end_time":"2026-08-10T19:08:58.941651+00:00","exception":false,"start_time":"2026-08-10T19:08:58.917643+00:00","status":"completed"},"tags":[]},"outputs":[{"name":"stdout","output_type":"stream","text":["/kaggle/working/deepseek_v3_exact/modeling_deepseek.py\n","/kaggle/working/deepseek_v3_exact/configuration_deepseek.py\n"]}],"source":["\n","# 2. RESTORE THE SUPPLIED ARCHITECTURE FILES\n","import base64, gzip\n","from pathlib import Path\n","\n","PKG_DIR = Path(\"/kaggle/working/deepseek_v3_exact\")\n","PKG_DIR.mkdir(parents=True, exist_ok=True)\n","(PKG_DIR / \"__init__.py\").write_text(\"\", encoding=\"utf-8\")\n","\n","MODEL_BLOB = \"\"\"H4sIAMS4eGoC/+19aZfbRpLgd/4KDPVmDUos1GFPz05tUz2yLLn1rGul6vb21tagUCRIYkgCLACsw36a374RkfcFkjrcx0jPlgggj8jIyMjIuLJYrau6jVZZO+8V7PdtVpdFOWt607paRe39Gh4i/u1l0bTD6M26LaoyWw6js816mQ+jP5Xw3BMNtFU9nhsPSVkm0005ZtWirImem983bbFskvE8Hy/WVVG2vG/8JrouS+0lNCjef//02c9FO39ZzYq2eVk1zTB6WsM/z8q2rtb37M2r98/wR483UWdlM63qVV43SQZQ3WQIWSOafPL07OT5a0/ZcQYgpgSsKPsUXw2jH+7LbFWM6clTcVVN8iXgMc3atkxXWbMwW4l7Efx50rZ5iZC8ggJPq/Imr9u8HtK3dF3n66zO0+8m2AgrRy2538fZpsmWTrFBF2DVpl1vWgug77Mmf4Ul3tBXxPPbDCiAPj6lXl6+8n17n19v8nKcP11mTVNMi7y2S3UCY+DmbZ2f1VlR5hMCxVNvfU9E4UXpy5fpyyd/efbu9Zt3r9iv9wzEoklZrVmdZ4DmtKpTgBrw1s6zMj1Oj7/1Q+npJZtM0qbN6jadVOOmrXH5DINfoN+UhppCk7DaJhKgKaBrzmjkJM1usmKZXcH68nx2gD5Jj49YwWU1m0E37AGIYpmN87TO201dGuAFx5awsZnolOia3inIWAvJuCqnxWxT0ypKJ3m+bvJ8IWr+wJ///O1TKmeu/EmBAF1t2nyCfAEfRYFys1rf48ty3es9iM7mRQOMapE30eVWar+MsmiZZ9NIsJ2oKKN2nkfP/080q7P1PIEWXwDjy2H08CFr6assfVssl1FZtdFVjhgaA3TtvK42s3mUlZOoARCXANt6nWc1gphB4UkuOuE9FFM/2uLBKZvSKXWxAyWy8lRHR970rrfb4o9GskZyC8DFW2sMer0eUlJeQ11OUsksb1P2Mk5TYHh5mkK59Omb189f/Jg+f/Mu/eHNUyjft+e8D61N8mmUYgubcp1N0knWZrHVJxtmk18v87JJizK9ylrYAUaRWS5pNqt4UqxGB8fDaAI7VD5io4Od49uTAcNtOSnGQCpi4GVV/pLXldVjAksKX8SDIcxi2uJ2NnqeLZt8oD5Re6vsLmWA6XDZoCZQLB4kRZuveL3xhldDUJ4nMPJYziWDbLxZ4XjspmBkMMIjc4D6MIdRDMM/Yt0M+GrHVR6pHjgWhvKFAke98wxtyBvt9cbIwbU1/O7V+9fALGLY0YEdb5Y5nzSaXahftGkKY1lOh9G8mEygzab4BTZIqD86zg9+N1Ck3O/35W+nA1gVESwC2JoBLqT36OxfXmb3eY0fvU00mzXQ5SCRUAzUJ4Anuc2L2byFWQDQ32Y1UC+stZhhtCrzJtbgHVh1b7K6yGA/SwHKYgn8YYQD6smRc0ZuDbyF1dxoAy5K2AJTmk9owCiV0FtZ0vjmlG0rDvZ0WWWS4vGPgNOpsq5u45NBguwuxlWzAHwjfZ3Vm3ywW7fRQ06wdXNdt7Hs6pEfRapVTpX6JDx0R6QhB+nOt3EnyG/LSewQi5dQK9h275+trvLJBHjXLgQLCBnSalhXTUE8IhfVm9HJ0Xf/cxhdgUQ0Oj6CP1A8v4HFNXoNxKNNsocMTVqCXgC18Lf5OtAvFA18MasjYFAW/9Ho7Sad1vk1vD9OjqJDjTGY9R4+jDhFwbooZ3kMoxOwDqOTASM0GBTMExv1AFoTJdRMW6umzmewncOGdrWZTgEpfQFQfyhhG0aArgaLlS3nvApfD6LvN8VyEs3zOkcWgLt/dMkg/c+iTWhnvoxuq3qRmF0DQ2vTcdXAai5TEtvtsV+nwFhGAdwOjcJ8oqlhAXfCXloFNVaNOx3QWLZZcqqGPSaEKM6BESIG7ASmDOlKcRjPgBjR8nqCHPl+oRNksAf+Qu1HcrfkZOCSi9uOXAdG/zayDO6mzTB+VZs0HEKQJXPik7WR33F8a4z5QfRDgWSF2wPJoesMSAmW6AZEqjbaNDmKZRNZBj6uNm0mhMGqnoBsA0RVXbUZl9sa2BSicbYcb5ZUTvYFlCGBHMNKiAnuIQN/wLbqg+Nt5I/Tx5AGCwCaBMG54YuK5sy3FrpblMQgWoQX21oMbFp3kpRsjvYgujuNzq9guCCRa2LiPM8mjUaA+Ey754XiQNMg+cH+jv3APIgWosehwqcuIXrXAwPfWhGju8QkzbtE7DLWDuWhdzVl56e83QuJXtnS0K2oZma3iriFPYieVusCcBM4GjfJcpmtMnVOZo8v8e+XcEDO6vdAvPDB2vvgNNPOIyp28FjtkM6W2dVGHNxZOaWAKBYsEuV3QDQTGBlBsqR+ooZ1lERP63xStA0uRlyF77ASLeA6OtwcLrKiWmDtG5T23L3bWCIKobh1GSJucFOXpbTNvWfxfiRV9ZJDnk6zMfCEEWyuXGC2uK5ZDjmu8SIsM3RJIkwIEdQ9+EfdIowehbRh4W+vbeTrzvHJO8cncSiuIn199tOncalt7XxOTsX7iqCzXdhVg/zqalmN/+1fSU0ET/mq+qUALHzlXX813mVKI46o4QHTlDf40Uo7LlmSAv6JfUh7KPs77Oxv4DR34G/wIDo2yw7o7CbPlYfa7wM4uJmFt5wH5Vg/13HQZOSf71j4N3pO+brBfI4N5gWavYAIkYRxb4HxIiqmBfBTfIcUOokAUXAYuQI0VdOorhjuGtIx32d1mWJx4Cd1nY+ZUaJYMSrBI4wsz1U+ulKni9v2ONsRxwUE5yHZTpNlNYtDGhxYZ0avUOdE1FsXg8HAWIknepMI2aCnzgfPBRaI7qOralNOGoWSHRChLZhldYtAob5yNme/Pgod0A5pqVpSSFa1GktwKmTXssNgV7p+GwEVXY3zYrlDT56x7dQVn2PU5wOwqGeHakDB2B61RbwYl/pTkFjW0U223MCq/s9N0+ISHqMeTk4C6oJWyMphV8O/R8fQGP+lbEH0Ivo9MOdT+2QK7FqH6ig5RiphFTRyoReD6BGV76n+2YEL5n61JrNHDCMhLNBIFATwOhqRwvFUFz+iRyPo8ujoGIf7ts5vkGfBcp7B2qyL9p5RAesDLWgwReb+QajXlWNCcQ1IhD5pAWA/7ImNFGHljXHmhniOtW5gUoZiP+SIkbU8CuG/ACb2ERX/pqU1/FPVxawos2Ua6vO7o3/7ndZn3mbQTtOOvj2x3jZA4kCTahCCSq03abZcpsws9WkyIxbeBj5U31bEUoGLAaIeXPz2FGkYw5K/LVGSraoRH7Lvo0CDLCRe/B0e6NkwhOxmCjwpHIbqbJv1wHjrEx2Z1OhlAJZQhowgYFAgeIqyJXt0EJ4AvRFkvxXE8iftG3zL2rYX+wl5GPiGVGtZHnQeZEyP5/W2ZeWzUwghV7gT4BQcBHYXOXC2UR4e4pkBBD+fUcWUxg1Ud0ABAGgEATN4DMAYIOIuqNHwQ/Nr7wsfSGwVdMcotx8y1PlClkwll2JHMQOzjsjhroyhzs7MI9rh3vUF9/Mugs9yPDEANM8qsTyswBxziDtPGbsaK80+jdNMLI8z+/f50To0ku3zdJ4tpyTYac/xndJsvauYsR7f08GRGdoRzw2elfAVmdoT4TtxdwwTdHeeJMkwOo3ukmYOh9Xzg+MLWrjMlnN3osrYJaLTC10E06b64A7Y5d2xNssfPfpsvV7e0xGqvkeWhdyK0OD7EMNqXQwjoIohiqkwH4LHFWiw2pTN9SbPf8lJlDlWuHsCbRWAOyYLRm95rUhpB7nKD6rDd9TwLfL7COa5qQB+RvRPal1/dB3F3F59RqUuB6fRmWyB1VSm60WgtOpFlYXRBUrDF+DI0TqrWzHjDD+RZPGatRxEfn8zezSi49dtzVhJ2LIoLp20eAct2nmaiHbIZl2V21EePQcxM7+DnQfdkFv0EBxnJYh2Rp8bPB5DO2s8EFTTKYhV6G6owACob+ewSNCRQGqis+inPx/QmlcDNUgHRgr7z+UwqqQ/NDf4k2742DPyb4wGvomyerZZ0WlqnY/RT7ahwcI3GCCCli0rhGdejOfYpqyOk3yuo/0CcWNu9UVplWgq6el4z/EUresKuNXyPrqqq2wCx9aW4coAg2bommHfwnlZtSayqQcvdB6I5tlNztRbyFKic3I+4x5jpl0ZYLlIEIfwouDAGP0uOhvz2qqxTaSaHJ2T2hZn3uINzN/U6GfXoUl8osenwChBZmLzfbEq4Bi9vFfj4mOx0LodSY13XNaYThg1vyNurXGqS3J+jPW1O7iE0a7WdYFnfbFIvQuR70YTWGq0ZtEsEmKiSU/33EM2NnKRmkigYwN8tmkj0xq5OA9XolpXgB7AGKxSqHzNdrEeY9L4fFPkt7FWBvc20nbTJgUd5fF3w+jbAUgKVFUr6+tgoXWwoOdP7+CaCemoX7kG0QOwhmJurIsC1yiTNEKLslAVFoEKC6MC38R5R0PRgEed8url21186phvuOEMylQcEcnuq3xSICjyfbcnneWbgU0T9ZBfufFR60+WMF7CYjMeuTsKSCC5/sFs1YEZMWuxB+rJLYjO4O5LvVfns0+eJjBmWAI4938yb1bmtxHb4x4GYB5GV0XWeLX4m/UXaXZS3ZaBhj3NuB0GG4bDSDpFXsBCds7NeYavF0F3J43QdPBMeGOtl9hEPQjdA6FU4GjDV7bmVjYll9Cr6tmP0Mjui+dzLYm2WqcLtRjQJpHfQXttk8LfKUhfZvkSxGqMyxCltKrWF+skxb45ykBe2fvZ1iNWNX1j+l9eUX/rjGyRrvJ2Xk1Uce2lPa4ZwLDWh0MvPG1aBdU73UcV32IdptUBkW0GcLZzy7+3hFNOSg0ANVxpnRvvnXWOA2Y6OpeH7eLfbgoRtLnnq3V7H8feOR7aHQ98yjjh1mfgfRT1yyrb3KXtuO9x2IPVDfOX6/ovXNad8CqYMXajCcA8CFmc+bketahr0UETawpW5xvWGEQHj4krB4JuAFxcfBj1g/9q7gXwlCwykOcwhK0s8FzLVjKfm2GUjchWQw70/zIY7BtAcNX8oot8jpe/kjiIMh88IPkNcBWxCY1oCnaMMiBxBWMF5gNNpYlRlhTJwrR+5nxZTv3o9GzquoY6qfoLkNtzmORM1jBCzcxsVRUTm+RwoA0LXQKIE15K45e45Zp16qyAXfh11b7Aow0ey/LJs7quPEQ57Rcl8GKkChLvOVQqfgzmExk9x/tp9KsD/Ie+RbXGvDF2gqzlYBHZXHbf5Ue4wGDDdDyvCooOYe+4SNr8ojxVhtHB8UDEdATWrCZnH5lrj3hjKpHvIs6BxQuCzqYJIBpqfCIVSedHFyIIDLo5cJxigCufQzu8iQsPjMXkTuolqXEHVH0sIAiPrF1BwALQAhXkE0vTR3CcO40eG28uBKC0NYeBNUL4iBWmy2KRxzqIuwyaBc+N0YW7TjF6TOJiyI3ZwQYYIXBAQsii9hVx2POCfxIgZjg8ui1w/hYmA79McnhoFHOa9UAgj1cO3fupKDcx0a40CnfpGXEA8E2L5TL+L4W15KqqlhhmyBT2/YOinPYHoT5SIonFDlSqoNFpNF3sQp6WbAD9SRmC8wdgX3PYlI8VOF+IgZ5V65+2cU+N3XUzTxSpsC6pXYBNaKtOY50gCz+Ojpn2xhXQTi1jVVnBtp4xQVZDlRGKagTVoRdGfnBy1IFk/elQ76O3vfzDLlkbiGqFjiirzbItMFCZtEesTGQJ3FJtz+Z3qPfiOfVXz5yDi1DlPIlWxV0+4dtVtKIyKLaiExtJH/Oslt8bpQf6bY4/nvNO92FIJ5p51gDXrGOhzujnaxLB+wMiIN4Kf4l0ZdIPIBFRYhUbjSjenULFbqt6yaJnYo/LpKxhNeEpqQ2izspF+NCGrHNra2vRiAQUn70gykOipA/MFuIu/XMv6/dvCGZoMKqcgsWU8sWnWuIDXVV5GlazbN87NIoooscjE0kP/TMQbAQJp4h+z112RSuPcCfetynSHxmS8/ZxIJMt0FeNmawDVOLWvQhsIu5+YJHu8adQaxdtHn12atyR5j4jvf02cyRVWIAYoYfiXFbndrIfxrElxygaSlhhHo7Z0dfVi3ahgHTR3j48Pj0mDKNtk8MaHoXnpntC9j6UFxOMh2zv7QO04aeXMttN94HduwcLnSTOWmzCsPdB3u0brfiak9Uya1ND8GQ/RRMeDVDN9naTHO59h5UA7ILM2fxCb0MhQADZ57Av5zWOXF/VlmGVwyNqNR761GgDJMpN7tEyLTD9BeKe1pYhed9i/Hms8DJSwA58kMjGUKzIQerHKkd7QsS7AGAi0Z+JwnPZzYULBRuET3I89w2VQlNVjzw6dctR7j4pykl+l2KOI/Sa0xrWgH9owDJw6OSeEddDtUq6tpR7sRwYW0FHHQMpQ/8iGnR0Ikh5b6ZH0AtViVkpFkzBsQfcM/by7yIrDRxbM6GWQp6jxiXDsvmIGnNECpZx2TZqsTZJmd+mXF8q37HFfnQxjOCsay4mTX2KLRn6AdW1Nv/MTYP2bL7mRrwmPwtpSiGYCAM2wUiSrJ7hwVSX4OmcmnInEHQ3osogpVqjOL648FeSTNZ4azFZMd1BWV0fnhQxnDGzkegtMUWVwMHxINgoWxvCtOC2jPPHdPR+NmUUFhPbcbIn0R199NqKXPXQRE+Wfi9IQ7cTszmWKS5t1kumCt4BStZyAGWuggZx6Hs/Xm9iz2vKE+b70FZLlPm6AtKYgkMnO5N29pkLPkq1ChjAPBvU0EeVQMsdwLGUOIRoQldjkQujTt4Lx4J/0BYJuIOhOhYyEuo5NuZ7YOU6kHWtwVFNB3679rY1sq4awCpB5VspOlm5mmaDzrYfpwY+9qVIZJJyVlauuTGKJnDkIk1w2oFw94UaWmIyCZ9zcCFhfxgtTMEnNGpj1j0SkQ72eROdQoePosUF9FpE/7zj8bLByJ9FFzb0Rw9P9/F1C2HnehMX3QThJQKNSHp71TRRqNQ9IhXmKDrXNhpK4sikQjVtfMpQe8TH1z131jzl5YQ3qZp/pLXWswRLrR+/ROmVJiUG9Ek/L6SRp1t94Z0S1Hejt6XCrjHL53I4p3yIF71tsq2/Zd/W04g8YKoZi+K0ueL9G7PbGB7pvFGeb2+AeEYZib8eMN1KaFc4GuwuWGAtdaig+uxkhCAN3KLmysBwqI11PnZ3L6qowffQIyB9vq1hK4OX9RlcO2wozoR7WIaazB1RytDpReOUAmEYJcbuHPBzgymBumKLMGUruwBPMWQUWm2WqVFIna+4XdOqEBAmWW8MPqsf57whx/cJkQf5Os/adHHD4g7Ek3nwOo10z1E0IkLBU9S+kD+F/tG0GlBa14L5PWsJH9HblGUbZP2R0maZZzd5zKJWMYaWfWtG1NmA3INFvAPXhcwqNthYS2uJXHSR36cUsqtlsTIcgwdoO+K1gnmvzBrGsLSabl9GvQ6lELJ8HBpyezco2NU2deqBzk+H0Snzr6AfpxeOLbYLaALEht0NmDaHIk2twYZh2/G2/KnZZmQu7V3TysgKPvvWKzSkHSBsmKZYNk1wfaO6etFET5bL6C/VJnqd55NvWN6H7SauUydJMpzTMdkpbaEy3/o5LIILnhnwc1nFZDfoKyN+63uaKuDVrLF0wAlPGp9W5Tj3WXtflEARgCTmhsTsuWlKc5CmiUgm/IEmC5kxhmpgyUvZ/aVQxNT5uFqtWJIeNKFQqua+02cf1m+Oxucmmmxq4ZXOVbqYSWNJDvcY4EEJ7hsKE0mit8BkYMOnVJPNhuWdXNfVTTHJo8yAx9MnRZGMMY0z6xDjUXCMSdBezRxMJW+Z1NWabUZcHeV82s/N2TWBslVnGD4t1tbbNTGp0PDvEpyNYSYg2OVtpnuCindm2et0WdWZZRDSXlqlFyk1pPFTUcX6YtZb3Pi60d9aOYA9PdwE2oaeyyBMZRim6x1qgNweGp/lrN7w1N7QFrok9BxRVUe0d3XzUo7rtv+07TqJK4J7aA9Q9+rez6J4nWYfBZFLSBwGZ6nh221ZjBAKYgbkbTLypMh2+/M2c7XDYHygfwKCnYXAEJoi901X11kYGAervuAHY2UFqdWsGp6IYNZcgnunKdABchvZNgGeRjxB9XImrKwDsb2wDwJs65HNZgYeDNFsDoOBIdWWofhp5iYwJ92T/REzxgQUHLaTFbuppi0lrOBx7TbWME3DwVHyL0G7DSFT+DgFbTZOAo9QE+jrEvfN4n2Zbt+fnqWjufM+K9K/sHU6ZheuSkdG+vsyCekR+aFg/O1oNl/qiYVYChQlvKr5I0/5010nI7Cz8LBtliMgnLrdP5RuftKVi6dLrBl6U+KNLBlmuPuOxWeJJ/0P0wcW6F98buoyux9Ffea23z/14zQwIZ2Jgb1m7R1n6PPNkidZkrVA/B7HWyfX3f7hrOlB7ISlCt0Ts1uTmX7FLmEXmZ8HtYtbtDkARn/19r/I70/Diwa+XgS9rjC4uSgD7mF01tuWRac/DNeVGX62FsJUP12FGIveXkLtYd6SfkTATHA8hJDoVPuwF/n7crL9N6B4lneKEe/29dB4nJmYq/2fUY/GPOyn/T+Vi7K6LaN31dtn0tmbFtCv+nL60DcyjJF6jumiWCy/rcjl8RGkyh1ihAZT6p52J8/n2UpEcM/QDuoQIujQEXotTbcMjT/GUHnzI1m7Zptq08RO7iHdfy+Qt69Lfy0LmfdFabo4vbhQyg29GVGcSi+rchasCExB6Ui1qnS1oFueG1MkmNAbRppgiJ55VNg0OctgFCpgEiRp7el6x3OTHvwI0N67lS4uLgxJsb/OaNkRSvvIXljPJpmLeyhJt+hyhf5boSLUWyMt4SRf1/mY8lFIBeFVDoS6qm7w5okyuvku+fZfPbo+QFJ0ad8kV5RNC9SZXPYDUh8ROCfv1FXnM5/+/ZUv1+oYRNHlAQdKl0FoFa+0MHZDYxEbqhSrbc0OpefG0IbpXcXquDYILtxrUmZhOyQPMzJhdkFzDMPo3KtKG/r3ggsR8OQ7G1MWk7xp8km6uBHIcdUeIQwb1TEnRjfoVvFzR9u4xxCkJMP6xH/cyTgONBieBcKCRzUgtSGxR8MSGwOzbZU7koirzPy4DUCb3QUnKWZVksav4PyoSQnQ1s1uU3IjsmtCZ3rfwrP6xLgHx2TsHa70U9swE/RodoQAvzTbp6RgyPqjpq034xY53TxrovEcowooedE4jzDtNhqxkDf+LoleTKP7ahNlxBW7jTZ9b799FKOzTVsd1PkMyaa4yYE1jyt12cPi8EYYYIbR2mN4QX1DAbLLLyyxk7K8MetdoGOevoswGJGPctLfImVpk/loZM0VhT1tGoxTxAKzdh6r4kNrtnSuwdLPCX6jROBYpxZ17ZJqVCNu5JOS5fiT3ckSoZR3enOYvUmtEeInyluES2rmutWHee1ZpnqT3MaM//nXOwowbAvYUt/PLU5Z/bVmy8Bp+qzjUQ3uNJqFORpf7Y6xIMAfwyXYTdTq+IsJKvunbOoxQSb8hr8/YCToe5Zabozric4GzHbesw7KkhwtNmqthc16gp5tPc9R29uCvT6GBughuwTdccycZFwXY8bYV1m72ixjnXqGGhTaxnGCSbVkTJ2h7PRtJzALevdcfov+CcDooidt9fqioru49LSveSvwQTfzarOcoNhawYaADPDXXbv/wC+aaPqenqJfPYP7EJJsedyqdZmwRpUW1gKlnB3Ovs/XQvHxdqzus/8p7BJgQdQed2KTMOdC/WHb1mLRsvH4yMKEnlZos8bsgdqeh7dgrLW0yFbDZZmIAPpsKcg8DoKiEgR0pl7GwCB9jdkOZ51AcJeFLiDWI78HxDASoWUjI9DMI4hRg+wcLMU+zh3MvnS+5FnwrIld17sttu6/6C+1Xi/3XvB2/7usemOE4RwKJkL1mpY0bmpgdqivpd4InBNcm2SwYc3iGWtfzJlFFuTqSLoWqLp3VdNuae0PbQo2NshPdFt7DnLtXPKrk12d16xqscetzUrWoEpg1Gcz1/gMS9yQMM9MnsWhKOd5jfE+NKZLT/uXmBALhXSBSZ4vlDcAq+6ebfebsq02mEmb+WtW5fIesHy9KTA3BDuSRLdyLZSG2xZl8KVwTLxHqczzCaXa5fmJlvfk10U11purJcg8T96+AEB6PLjVGCaqhiZ5tmQ45kqkSLnx4xUuKl22yGPRQL17PrbV9hQWD0mjJnVrXV57dlF9Lzh788Ob0+i9ZBFClYXudhFNfqQ2OTx3vaue0hXqV5vVmiX0PUmOE61FwgYuifL38CWa5SWFLDSUcmqZT2HjWRazEqeE+RDh9jTETMQwmbeY4xf3d5iAnF/PjJ1VbVutDmoKOmXVMQnMkOUEvgX6WGWTXCRJJjgVGI9HCCEjOky1UVxh3jLulocDAMqYYCZdLCCuzhoDmb7L+c/TaN626+b08HAGM7rBtPCrwx+y6uDJi5fZ1SF1dSDn/7DO6cDZHLbZ7PAGOz/SEfR9fotnXwY6UoiJsSE/FV+nzA8Y94uF+B3nd+N8zUbY0uEbSEmWRGfeAfoUTjZjugjstsYkzySbxAL9A/tCa9V9iveHYZh2igVFTiZkdEWjF5uhHyJI3VWdwvLKlulJenwU/7b68U5V91cdeUBH/iDM2jUGNqlyJmLzREruOP8Rle7G44MIGq9vYWfK7eOHztXNOz9cssWDOCEjWVfr2MSUExim4VcQSO+rKeDvxRTwgG+Xahlx2aPR9nrabfRk8lp1lX09uhOquxmQ2p0K6LgT06779z6IKIYJeC6KSOyiONw/sTNMHEadCZ8Cq9ev9ouv9ou/iv3iSxs7PqPa/fPo3b+q3f/7qN0927As9U8jeymZhGut1ecJyAwWeZ0fOaMVvt/aAjWlga/qf+PAezbP8QwuWCgZI2GzxnNhmU9hUAWGiqLayz4DG5s69I4xVfbFKRs8NHLWwmIT7UtUkuhnoYsQWzXs4MwbFxv+6c9sOEYyeJR6xZ0v2U1VTEDUlSoDfTiHXBt2iFtPEuIVhvI1uB8aqzHfWt6aZIOh23XUNQlMM5vWLH9cIHLNzgzG4ueP8EJahaUXZfT22fMzPETDwXR5jxIRabv5dOU1JXElPQxXSbPDtGgUr9Uplph1DQ65jX7U0eUsJdFZUcE5WlgKDDQmnRFdeqS6SqI/ImkMESqcd61tASTeQz2mTB10vmf6J6ZJZ6IdowM63+RwRLlvKcoQL5hqUFGGqQvG0KuucWD6NlKeoNcn3hyhBvw/IsoKhXDhPU4F6V/0UEjoEHmFxOJLxCIGweiowTGucYCxEyvD9SsNNi31aTp3olj/Cfdjd00COhszygpJhiP31DrA/VGodbiahGn3uOYwX+IwrzcYOvpLPrHNSCIrq+adCvwPBMA85XUyRpvkXO+xIMGOjlLGxOOdH2jF9VVmoyuaFH0dEP1pXuL6n8TbO2R18Y2sPFtv2Od4Fz9Qq70u51mSixM9PUHI5zd0Ju0FU48aJ8juPqwT/I7xw2eBlQwkvGrEYgsuaH41HFtbpJRYZuyeM6+xhDTOwOXJAYYOgswWh7mL5VWAwI0Uo2I/G4OH4NaBShBakcQtFDuC3dprpdGn80PSqfro3CaqWG9pENjsrb2iq1LnhmFX7LLXKAUl3zccJaQ9uGFIVAkCOOxQ9ZjfmFBq5kNh+9hI3+2ssDjdg2DkOhUMO66v2EPEDFrRHPlYEyf3McKZ0Xx7WOQ067DXBfvvwkwnDTZbCdLUivtJ00eWfpIMkSNrl0ujtsw1OtIvfjcJUOm6tf1GmPnwz1OQsBrDjMbvDAGR1JabDyh/aYDZchMYDCFCHS7MGN0QqhnOtLQ6dYM3D8JX1Ry/k5DfSdNYLn38TgVUHotalLzGKZKoaX/iKLANtth9FSn+eUHDZHca8kpsM0FLI5OqbBQ9efsixE937Q+Daj6yN4MR79ofO01+ZI+Wqnx7n3TFK6cKqnKgXelK/Wc8PkS6OlzGnkstB5dcGDxC0LEqt6c5PcqbXKupbchFcjqWhji9YFmVB2bhxLcROLe8ekaskGcn6XCWKzRHEkJ3g2fafQ0A6v/+6T/OYNLoTEPKKDoCsWaT6AduT8U7Z6PDiC6XcvUrOkfgDHcHw6KtduDpJMz8Eh32B3GSf0dmIJqDS53T4S54fBnBGRrkoz2t2XQR7SQHlrRspPswCIT8SLaii3WBXYVtacL2nuwySiIlB3b9VPtUuggEGaQ4MQr1w8f4z2kWCEvu8yaDdcXpsHTVJWF1S1nsiEjXOafX7qfxRuhYG3+ctv/rQIqNG0BjSrvIlvF0qKKsrdeYzC5RmwGP0MHQ9CfKG6NG1vMMaokpscuUzRnU9r3GZnQc9EJyYMo21JHmiwDiTI2NodfdbzfTOkZGBnq6ii70oosgHeBwrkd+/HXVWXjrePoRov1aCPluEUvG94v3JsMYsX+GXZRkSsYdJO1M+lBbXZG+WXYQcZgtm3Bo1PTbktFviX73hKCdAXzchTlwcfTi8Z6xFv6TDYw/bGUseqYMPo0Lk5OEOQPl/CAaSCdZm8VmZ4Oeuy3o/suBbH5amik5KJ540DjTMBXHiEXRpCTVp9ld0cSOCMyaMBL+ias+dobGynijUOU5UWsTsBU+rexvCCFe566LC6OR1o/v1LLbYKzy2wdkW+ICYwmMx3Vo1/m93AAZFW/fAdVWZxK6F63XDBkMpJ6hdbUxa+XaDfV93DUSdrDJ2F07njwwEr+PIsttnvJ7w6v8BmAd6bPDXrkXC54Jp8kMTuOr8fqevCi5qyS8RpsBdDlJgojRgT8/PbAyyZsUpUMkUs4eD7rldpBSD3jof7PEW0PhxLhZwQGSvEO5VJt0e1WZL1BxdaBP3GkQZmO7M+Wv0NQyFtkpInrZ9ZYNIw6C6GeBwxDn8XMOS6SJw7KmtVzjfQTMgZEyrPfk7OzZ67MXb16nT18+ef/+2XuZQaWfZ7O87p/6EqOyNvq2vuzEKG0dsYa9Dz33FsEfMAo0r8lEtfs96FuypJpZKbYlRd2aONNKTwZ/kYSEd83b2BPXzjMRqhCXX5LV6MKXS05eTiXBH7m5V+0Maavl2rHwmBcz2jd58U0omE7QuRHQF9gl1zacflVM8mOJM7ZNLVJAXJNjJt9lNt5W+5/1y8HY62mdX1NS+153KhTr9i97yNYsM0Nkd45AX64/I28kVBB57upVk9I9ofAu1CfdR6CWxxfv/B8z3YnPQ3oXx27ZADpwX3S6eG8rGnT27pmRqM9Rn+e4fusDMEo4r/h1Rhd+7b2r4jazfAt9rNbg5eBUObIqjwpUttIFSZciH7cUEcmsSTLh5S66X72vTj1mZkZeBtW9uivO4BKZlh2tIwJBqtqqfTwMHVdtDx86p/r6EgEpTm+J7+Ijzf07vkTS6UbBz/Oc7q+pauKt5ELEbtzVrR8N14hT2BTGMClYmGk5id7nupt8cwkCz0S78sQUW1BLyhtEBeoKNcdMVWqOSS6HncbyAq2YNIZLTCgMhS/NBQxQoW3DsDeg05YEii7LzUq0QIi4nmaNrlWbtcwN4XQbo1LX6WlgjsRynYsvaQHGDsEOjCEy/y0KLGNmGQSQtYAGS3bpfWRd6Gip0v/R0hnBgbKYbEgDbl4xELxv0rfX2s7omp78PRRXGn5/q+zYmlo2XAANKmnTPLLEszjMKUdm+x18btTlMqBvdSP9YdhBjiPzcdjNV0bOm6F/0Y7kL7OAm1sufFeonO1Hodl+ED3foI8eyNwlOa1tp5ROQgkJSLvebiqk4V3Lbx+huvHJuk3SdFLY4qAgWnk0YkmfTeI1m1JiSLAJh9iH7uGUl4cjlhIt0/dnT96dpT+8efr+7N2L1z8iBozLVrhfnRHRe375ts7P0Nkwn7zC75cXSfSUrHJkrsLTFDvFTarxRp5upGGV4keLMfcsaKSVdllc1Vl9H8lDEduPcIfDvhkocbMZz9ErEl0egVML/6omu6HMQDiBv4gbHJhgo6VuBK6wIRdJlX07b8cJR5Y+YtiLlg1an9/en+G2EHHxTJ5AL2IRQbq+Z5+qenYII24O0eF0mR9C0Xm7Wj6wKg4AQ1fspgfq9U8NBShnqN6p89lmmdWyU1aDWHmNoatCSBPfXQQjtlZ0KafmucZDdpdASnBk74lD1lU+B6xVNXeSeJvVGcwJCBDafaHsPpD4/NI+T19eWJs+kQKvsKkzLfUSy7Ekgq1lL1rUd75Mohcif5NM+ZSJ7qcYSCxCGC2XQO7+IZx9oL9qXNC4qQ3ZwZBFjzN7rwajIF0039v+AueX/2WReoIrAN09W/by8kI4yACWJTCMhkTelB4Ffff+He++ZdeNIZW0eM8IV9eSy+IVCj9aqD3DJ1u1dEVInd1ypnTA+ZW4AQV9xhvhu4+kTbHw1TrhGVzDC37YG7gql1dnb9/TFbV/hKZ8V9zQv2+YSWidAZZJPrsCiYhaAelzcYAjOHsrEgT0JBz84zfokUwyyITKifnL79CdXFAWvykXh5Qg2/e8xr/E8s3Z4CWJ0V1RBVsCNC/okfTyFSsF4GqqqKp+Sjahl68S2RidxEF0Ijs91lcDAvApfyo7mqMmrWV3w2AxOTB+pzKLKUgM7O2hudpDU7XtcoNd1AW28isnK7K6JMDXlLjipBpnV/yVuoJgt1vSTS1D4CIwbTuTwMVy5HY4as9L2oZC0a9ntCm9FE4YyKAidsHTGbloAHuYFGM9FQYb7M8084c8oQUPKplWy2V125g0Yi8YEnHEaNlZ27jgMdfWwVz7nc/JbbKnpo5kCv4MIlBP982XglRPSqpeOctdcGx836ugiTOVtCS6WlbjBUzsf8SLg+MBbTLPVldxm/5aPFp8GND5Di1WeCUj+V9TY0CkB9gZJe+bEDXhEaZELj5kTn7sfCXypJxMkDzwG7lRP7tOoviE2yj4pqKjtc3H87IYw9TVOQa/f/RK3EeH7KhtrXWlTy1bX4G01p611bEE0a4hLlcPaR3zL8EmvkCbnKQN7nMSPewavvFOZ0Gmhl6RswGyufEJfbHiXiiJ0/ppQopTkAxAoto06U4a1OkG82vy+VJiqq94B0Oci3MOzUHsB0Ez4MnbXal4AIaBl9/yGYnVjajnc5hNGTo88EgTlgQVW898DTFks4Shxpw8VRfRYcL0lGQrlMCmBd5D16dntpJ5bosmndVwLsDzEPkKrquiJAGKX22FRYHk+F2kjEXTDb59/27Ab8pIm0WxxhNWkzJDbUrmC/IcHKFix1A59XkdAZLmN3NiACJLsGBHgQB1CZe6TIXLSJxJrXShjIbfTqy4JJUSle7nnZmBV01BF+2N85g1NVTLzBLt+dYmYnWyNksYv07jVZ6V5HGO3Y/gf8ekxCvjYgz7KWq9YEHWB96drYs4ZMz3gy155xeAXCjZeH7drQPQOjt3m7iQwzIO4i9ev/3T2Xv/SdxU67MNvJgodb4ymlw6ivuQ3tzC0wtmYsbq7NAsKmhJrnBbpV0Ij6j3CZwXJzw9L1MyFjNALTsHCC05T1SHsVH8XkQTx23S84LB1b7VVUuMgudPOr98smkrEryQpFHtgIpuXR8hPwJ3wxUM5zOQH6xDna94mmI6sDSFClPlKmyBd/4zelwwIQaR9OKH5g8XcZIczpZV0wBSHtD7Awro380nfucJCyrbX2GrMn4YpBCUxRBdyiwA/9mexYTnhFVmHAs5B5OxYK4vMSL9+OLy1ETAgfCPZ0TRCnQ8fIirAseXTx4+HFp1jvx1ePEgjk2DkI1p+fWAubb9PdHRi6nHGMLtSGqel/dKZbFEm8OlXPqXPOcMhWcwWkTDh9GJzwhiQ4EL8zYryUzD0+kJQhHKoSEV4gu5Ronp/FJmIwRQKfgVTtx5OmF7ZmoZEUy/HZTmoXoxvcc+oemapeZjUzApMti7V0BleJ8OUxcBRStdW1bfFTekacuumsPjfzs+So6//e53cNYQpivLcwYXQyYWATYoWVOL0Xqz+8SlcFodmYg6IikRJmcbjQdqSSr3GQU+iYUHOYLGzEVn9JCDhNHN3mXMixIEcWIUX6CjK9EJcoie11FE9XmALCS4vmVnLhsVn0xOahE0II+SuF2SobdlJryAIa8DW7CKD3iY2cTSr8WGla9RARnLqUoOyA69LHpIxPjXMA5VwozIoNIDxgdD5k1N2yAsnTzdYXu/xpMsRi4DhoumlVA5673nNfVe3WsqSpwJeWZB0+WMUsqKPjGHI+wWl8qIRKZcwjefbuuTNdlnt1XEViCz7WaoAMknp9biyVDTyuTfTYvsks3rRSREvf9lVSB7LUJ6GZpwWkbc5fNSkiazjwMtkG6Y1gM1gOwUmd6JZlo347I8C9LI92T5D7DTPZZLgX9RMZC5EnmDPRkX2B1MPBXqMp9l43s3cwvHn41YpWVG6Yu75xNxZqvcqCndRKdoFG4YA6A7G8rKsw3RyqSAv6GjD2cwmq0rEzMjsF22OnZRBO8Bf9W0ErSNj3Gp8PYXt/Oq4Rk3J1X5jWkVEAnZiloZ6w0/g1lxk5dMKyxMPoMQ3z0eSDu4cLjQIDFNcjty7sQU5ht27vb75+zYqNKmwtcObvdG4nioj0reoa7hGPd9nJbxvCJcV7A989S5lFs3E9tEjoIBN0FymwrSubkTk3Qz3SzxYCXFDvI1wVDhulpSesZoXt2yLL3lDWaT1+HhQisq3irdznOTY5IdkixLh2Rpbr+hSnmNEcIqK8OyqhbAa4GI6+Iu6e3v5vLbuLjsJtl9vOPRF3Y6koP13PnW4XDEx2H7r33CUMz4dD4KA3ajMxt8HWSLwjzgs1op2gY+HmraFtmGSKoyZnHj26JgR9F6iVYt2sb+5k2NjjmGNIJBXaFlh9EtDfywIaQgHvz8UOz0uDWz2WQz/FDO9DPc9Pl1Pw2h2K/1u+T5MUzlS8gysDW598cY92yHZeYRo2mjRp1af5aJQ5oNVGn1rvdlbRI2wCGDBM+IQz0yW/NLmFOzS/d21UCMQIfhRerdRUIeptMr2ZEmDhGP2YB5nn3UCZhtbvRmReIAd3xzQYJ+V+2asuRtuVJEjJcXV6OdAcvChFgyFxZ2UuZ3bYkneuRcvC+6yjyEA/vWdtw9nagPRXq+sAe6tNeNFPkSlm2zzaCZgCVq1lPQyavFpJ8J7NyUXoEnHlnX1TgnGcrj4Y/rWU+mjlGaTLBR52z7lnTD7KKbgWUrTagVHnGqsxbP6tZy23j3CyjJjS3Cat+hrh7sGlkg5Tlh5FKaD8dl/68bYKDXRXZ07nHGd5ow5Hmnc72uU7Ur2mD/UAZvDcsouaWSJsN0liXT5J9K+MwCGYbR91mTaxLLzyA7vAXcalELvvTocbenq22dcZsIhSSptHOcKYTbHnQLoAEg/Q6tJpxmQ/uC6u9h4Dm0jLTfuvOo3qXbjSzW84qw9rj1b9CJ8bitG62wP+k6OtcV+U2uWbqYbk1bWm4GSSym9+1UCdvutl/A1P8LOwtTJhySfu/xBpN5F4hRpilk2mKV94MpF0KDCOVyGerJ5EeqLk/qcqol2FaNb8dDdweivreTZhtKCYPCaCFQmBd08FFjr2qzL/1GBYs7K+iOdnCWxtdMeyWXCbsERRqTreaHEekgHZuw3ZBrBba11CC03JfZqhhTg8yBVG/B7nnQERugjdr64MmArqbQShytGyC8V0gw/waDsvir4HKjpW7SiSec3eh6WxC9f+ieBCCKVB/tXEmPyF+CTOApwQL02T9dOQ6sQemPyaYUIfRH5iQ4S9Jz96FRZOQKcbGcCs89IX5J3I7cP5lEIoGUyPI2r6vNbK5iAZvusP04HMjnhsyLKONw+irio2hMi0K5S4yt0tlAuxMVfPfJw5XWzu8mPNeXZe/chg+X0GI/3/Wm3NAoYvhRC8bYaakhZSMMRsWYW6/WgNDCWJhDC7gjOA2CwpA7k9iAdGF1amsin1sVT7DSFC14vZkAlBnmWQlx8bOuiDg/NYQW5/B9cbqLjOeJcHXQ8qgriogsLqSlUDFHBtguqXWEre0ZurZH+NrWEDZP+b3j2HaKZbOQZ5OggUxMfefb3AO7upeszAZPOkj02O1sS4SYZxkgtZhdHl8MjQWNDuSoEaXcK8zNXLfLiIvklBUYVce6WhVwNV7oNxgpP2/mNMuqcHsj18HY18+hqol8NfUJ2C/ub0tUaDaZWPp8dn+mMBUafKm310Ldc5ESXRhsZjs9GXXiHUgtaStTZBz4dlVbNPXvmG7rgf2TJ/7UtQCeMyAzPLiDuPEqIW/Ir8cK3lXoGLrYH1qL4MI38pvgedpxYw6oJmKL6zYmjXTFA1u8bqQNpyPA2B2oX+5gJdX4zeQ8zt2uKoxpq1Ulxcgo4VNMfs3kA71kTg3ci7V/scWisbf9gnktjBwjkLfwFuuFaURYpZ8tVunzq31Zi3pUynp+33BXHmKtjGEz/0xlpMYOObdVSQYMBt3e5nmpuDl37IFqdJeIEYT3Oe0Blp81tCYSiKlpFvLUdivGhb81W2etNe0JjbIr22GCogFOKZY1IKVVjzhM0Q3NZud6GexBK6JWCPcjkyU+QguuCi2zK+DYXSUI2C+VqyeQIf69ck2zQ9eqK6LPm1zzAfohX8O5fIE5lq9gTTYiNiwtmA+WHh42jOoNeirCKXC6WapU0Z4IMxYURlQJ64bqPzr+kEQvAencySID8s9mmt9BRm55ohpl4Ebg2KKmfKEkF4l0U3wKOYdMvMigBmDWNxQDUpStXEXmNTM7mtYov4xq8fcjOAnjUnWm2a8w4FucedixwoEop6LdnDWeJUMi7AZqWRIqpqQKfopofIY+O+t7xLfO4tjRak2aUc5XMP2hNqhH0fHAPpbP04JHDK6zom5cykhs4dkaFddJHl8gzo6lGlHTibJPBIgrZ1/VebYwBXQXbdYbulnjAAZnJSbk8VRMFyShwNKs8wv7VtqfxIWe7W0l8okL5y6Mh87I44rfsG1lH9/U/JrFVVHGIZwMNZjky8FHjJZ31z1k9aDXsUbdvV10Hr6BgBiBoXOxDwxlDEW8eHeSWAE5cOBiyg6jrh74ZyFm6HZrnUKvml94Y6wFf+Z1b073QJgRNKWp9Cb2TuVXfgVqulk3O6bsc2jDbBh8+jC8X4ZD4dGCKVS63466FLVYcS8N9JEvRXnt3gO0l0JZowSvbjmgMxYDULogSZ/xXiiylEDuvG5RBNlY3KoMMq3KYQ2QlVLP1fp4CmxjYxrGHIUP2+FnhbXYNanRXuwDdjFd7NwsfkaBJcCVDhZc2FbSCYozTKq/yu+rcuKI+ZZr5APm4Mqd9FFup+bZeUM0LkIn5hkefKHGwXi+KRf8jjdzl2DvSF2FAqXcinArtlk53ujDmgDGC6eyAeXZPPVlBW2LcpM7m+ZE25AUgtUmfIDpkHkX/r2I7e6snd/v2bmYSOqUmJloCjct/w1UBorYD6OmWQ1vMMMvfO1alKDLTwmcE/Ny4i5OIUr5vcGocbpg+kDcde2cWAd+ZzGOVl55sM0zzL1VSwd/B9Gyzm71sxC/hBo1iLHe0iDBUNvYUcUY1R+yy/AC8rMljKPYnHw7+ASPKs8Z8qP9qty2TO8qAR/neLsBaJxRBVT+FlCDdqvLH6dBxYhRzmybqwR5i/zJO1RlhjDHp7fQhfbP63P27zyNsPQrsRyetb2GdmihHzP1f0MjAcEoffrm9fMXP6bP37zD/npq8Xx1cfsEFzeh0dgD4H8grzg/5WnGxLo7ly/D3pcJGcU/L1n7eIRneiyRz4qFsEL/5WyDEYIi/Jd2skSGm/IYYe7UQ4HJLI68VfqbJhmGMoyr7e2CIgwPjo+OeDpXLQhILm8MpmPMlrQFIjhIBORgbRZoxtMSuMcMNqwBC0KjTahoWMiZDAcVKQtbrSc+Df7B+QajacPeEfE0WlT9s7sMnaG1N5eXl+v7dq5peR8/fsztW1pfmCaxqtvICGUf+m0BPaMpVwOvZz+zEtzFb5+c/TE9e4Mc8c/P3p09+yH9+dmLH/949n5gNNoKEDDpvQ7SDg2evfnp2esX//fZu4EJ6Bpqrim5yh/z+yFNJsWkwdoeo1T+B1hSJb1qsyXlQFjlf+gbTcjDvIQvZq0OxULmYT2j/rrtW/0/iH6kvIltbrye8Zf8CMm2f/Ey5kc7SbLslgW2AEffHgWwlrBly7bQWO8BVjHmnSGXOTheM0qkYFsgtiUIVulmnfJ22N3TzRq2Q2HP0PUM+6Dx/5UvvlmxW8JFuWF0hTkvKGBNlIRa2tXDQLo6C/vq5Pt5nXz/ah65wpwudA8y9Bz2HRD1x+k0z1AJ1oioGxoCSZFSTYa/0YI58GTSVbJh7HrAkdJBLaffNBdz0+3JYohCow7vrK2eK3v7w/hIZuR7OQyRyUj7PfTNu70qND2KOspy9Yl+0AjlWpaFuU7A0aVsM7t9rGmOr1GxcYfcnx9Er9CAinaaA6af0UxFmm3LzAtQTNvUGtl5kiTSOBBWOvC6DCqpnaG6x131uk0xHdDpj1tVDF2g6o9S3YCaEaMHoSExdG7aHEo9iF5raDQ+8GnsXAIQbqV+h0nbV9l74cxH2O2ix/YNNH6NO4eYUWnYQrwX4zPPBCP2j7+Ia+4bicF6y5uHfUCC358LUOBFJiHCD8me7ofW5Ac4gfydYAz2eG4vAS9viIMTJVuhBHCihrOFdjgaaZxMwflIa6urXVnD8BqphCdWxnO9oHk8ykDMxhDGCP5az6Unc0au03UBwloeZVNMgwybTuLTWQU99Rze2ekJJi8HjfkaHsB4pYuipWEWV6XhILWCxKC9KNEv4NQa8J+mY2caRtSR9RqhHHFgO7d/PoqkUwzwbcJNspt3lyjt2+c1/aaws3EZA86AKT8kQJWtiqig/mfUpY0adeh9rG/W9UenRriJFQ8T3H/N5I6BeByPNYDyBnXGxSineo/nJFUIVocDcunzMmKc9S7doXd1/osHOxhpfS2G2gdZDP7jhpSTi10gNJe25oIgk+1sSu7VRglJcOS2nHQcHWDyE5a2iNrlWZctA3F+N8bcblZB7WjcYnqp25ynVIwyVN+Q2ukWb7i0em2qVS464ud6rAedLDcNSGZLnpiGpTmC5dKK0jydS57MEpbRSqS70dT8eHWAY4eDrztckReOZvGKGmZxZRp77HFa2SZfKAHHcjY5iEPdHOgkNbANgA+iEzW5Ou3BqJpVtlziJRSYaEf29g2fRgXKvFoiOvHeDn6zGBJREv2ck+5gUjTjrJ74kE3VrygvOk/Xp4GQWItH8BYO4O896NsDYVpTDkq+BZS8Qa3mLUY1xnqvj0eebgdwDs7bbxp+Daoe4IiLDI227iJL7FUJk8CXRnZVsQvaZpVuRIbVXaw2K07eIg/UbU4ZFSmLEpwLtItQzISayXbCdtjHfqS9tbjR9iOfz9ZjB4ZtK2KXe2XtNnWnLMtBhO1ryMfjvv6Ji7+G796WqLbtcZcPYMZylMCMonwlTNHpuKpZlG6kdv+uWESLB6BfSjxIxpsVkCWe2dAcH6yfMAV5Oi2Wy9QO2UO3AMyN5pjuLQNTpy+JFS1Jc+MhAn16HmAnl4YsYuasA/KnRSbSmm4oTVy+EmkSj5tWQ17UtPm6My7TnEJXlnFnkRkwpeb5174Z0nxq9vGhI2DR3xKR36laLR8UdvQKyWY9QYW00eKvbmi7QdWnUbdrj5Pm/DTaGuvVlzovKK2vJ/Xec2DsmwQHVbvOix+C6lIDJ9z0jDJ5MWYX9yjn7TqvahkU48qfV3m2onRCummbasAaoTSDGKbYc3MM0bei7F4aVlOPfGfUQKiNFGLpqJGwG+FZMuf4SMFNniuqGNfODLzNIfCqLAKvhtLbctp3/TyMoW1JTyYvH/MkKNPMX+KCKJnHluz2mJSMLWstMxkcNSkYhA1BJEHW0349r+r3vJ2nRjOXF8g+GhVXJrJnRzQiSonIPLXM7ulikorMoNyhl4iQyZdMDP3x7dnByQCqc3jeFziKomXXXFmj4TuAAmGIJWuAuaC4EZZK1Mzgy0VfVYdSfmY9nkBQ5Qvj90ZOKUG2SGCr35BFncHniYMIkVkUeWTmpBhn+VXr6hZ7pn5ZwlF/19RLg5mh4BSSLXSss/yJWoNicLQVJgp3PEPHDFaXulbOAKvhGW2DG4ieW0+zOw/l1MiUHjFC6QLJLpXzATqQ9858TKq8IJFuDfT6fLFbqA6VSmDNQZu9+4Q4r2aMUabbIrcsGH6TuK2/deeyr6m7vvo1/ZX9mmy2lNe7eTjt7c/U4b8U9F0KCAiHdT6rkQFg/lqfF9Nu7ksOD+Qp7ynttfsRDk3Hl+yaT71z3AWly1H8Ks/Kg/fXG9yT8OtgKLZP6WbhtvyYNWzJDW7jZDA84BbDgT+o7a/kZqChOd3DNeCrQ8CXdggI+QN4JizsG0A7fDCjw74J0DxJz4660pG5Na1sZnpsiMig5MmtKw787Ppc1eg/wcLeO6fcUyatzqGtJRcRI2yNVjMzutkytZSWk344a8M24E8tA6nhI4oTdnDcgcrdpirQsv/wyvh/fh1rloHQIAYJRtkOkqyerbI7rw6LRVHBabfDD8FvevHjQpNOqiXGhFjeHj13LDygTPfF5dFfJkyOi64Soi+6/WJ28GexPEu2YMQmnLq6WuYr8pkPhyKKStYed9oLOhX4mh9FfbUj9j1pSzzd4BLBRRh3uEUkE9a8CE1B9WdEahnfV6Arl4z2HwmeKTAHH/aRmvtx35uRZf8uVngBbaCHned0ZGL9NBimBD2+ev/M4120NwHYfj/GekpE0CMaT9gEyTeDPRDX2YloetDzE1gIUV2T2om5LQ5aPoD9+j0DVabvlsK9Qp1wytoSArZ95B201jnw758+Q/n/JQG848i3TNXubihmQ+hk4pNVQp4ptmvKYG/flG0non28VMyxdIqmnkHu47jiq76bE4uvpt+h5f8DWm2TQERCAQA=\"\"\"\n","CONFIG_BLOB = \"\"\"H4sIAMS4eGoC/41XTW/jNhC961cI7iVZbA1/JG5aQIcg8bZBsWmQLvZSFANaGsmEJVIhqSTeov+9I1k2PyQX60MgzrxHDjnDN0yuZBUbxYTOpapQ6WkqRc6LRjHDpYDG8FLHvKqlMvGTQoJygdldB4ryAdvDl7IouCiiqP1AFSdHy7RAAwfjBYBgFQJcRvfr9dOf6/Xv8PS8/vJ8+/C4voe7Px4/PfwKt893vz18XcPn2yea5Z9/oyhKS6Z1fI9Ya8Td1+UhpIswxstfoph+lcywBLOvkfiTrGfB63LSuXe412Ak8EJIhcAMcJGjQpG2+L8mNdMGCASvrGxQT/6OOlqGeQwE5QbgorO0P41l/vE0epUp24Dm3zCZL35e3Mysa8uzDMXB99N8dWM9XBikA804M9hzb66WCwuoJMIQtJhdOZOIpoJ+iZLtKTvJau57Bb4bAbWiOVJzBDmYytSUJq3hDXmxNclsuvQnYMag6Apliywj7iJY/3RmowDQW0aLA77XqIy3tAAlG+P4Ftcr68W6PxVr6uE6ZSVVGOQsNVIli+m1hexeaTeKARXsLrmeO8f54jjm10tnpZcdBVIfwoeMV8nqysmtNXsbI5LwSJ7XyHoHFZqtzJKJkKx5B5NO3K0XtJk6CSkH65V/wP3xAP2lAt65pLZGupxCrvDFPaucq7aegUpDIyisS5Zi4uaWrjN0i9ZKbpIvqkHr1KlU3RE3Ik0mmheV5NlkUNaUgNZbNo6rYu9QS827ksFqg1lGM1HiV8ubq5lb/wShRH6j4CkpBVLpzZx8qUpDFyPd42SOPzoJa2hHKUu3GERdUyrogCgwniWPUjiujdTW5USBrt05PsMR3qTK3B18YqVGtxop/WaLhiXzGf2ms8DX12kQib1PG84Gk1pvRnNQvSfevB8+7N6YKvTB0gvfUZCmVodI0ezAB53JDzHOeHy6I2hEcUY+bCBcBB7YgsDG9K4Na8zuUwcySLSBbUgZ08aeOeYKwvWFsw3UtwzXC6S0XyqwBrRAP1tOYAoJvqh2BN/kE3qdJVz/5btHNZfAo3af6moxMdyhD3zxcC9nYYFOt9jAFNwHF/p6BhQK+WFWcX5WR9sJ64zCRHRi3p1/9zUyyxFhB8OiCdS/L5rAOrxJtiv0V8gafPBooyDOqD0Iz2shbWSewQe7LYWg7jCyTSEfe1LEXMetiFq5O/P2+K4bdZYWWKNR4aM6t7pHg1D2gqbWyV5gCy6Y0+fae+UMfeCp6xHq9B1e1mNL6m7ocTAC6u/tEdYPfaDfqQjqG86B+8bl4XubPdIfYlqQadSxwv4VnsWbfZzKqqZ/Sza85GZPDd1sKfsipm3EuqnrkhOse+gTMgvL8fvF/9Tn/q/vWU5D1+zicjr8P2Dw6nAHHz2Y9wJxBz7Me5DgWdjY+2TE5pP810P3goj+A0RD51IeDgAA\"\"\"\n","\n","def unpack(blob):\n"," return gzip.decompress(base64.b64decode(blob)).decode(\"utf-8\")\n","\n","(PKG_DIR / \"modeling_deepseek.py\").write_text(unpack(MODEL_BLOB), encoding=\"utf-8\")\n","(PKG_DIR / \"configuration_deepseek.py\").write_text(unpack(CONFIG_BLOB), encoding=\"utf-8\")\n","\n","print(PKG_DIR / \"modeling_deepseek.py\")\n","print(PKG_DIR / \"configuration_deepseek.py\")\n"]},{"cell_type":"code","execution_count":3,"id":"73e343c5","metadata":{"execution":{"iopub.execute_input":"2026-08-10T19:08:58.950019Z","iopub.status.busy":"2026-08-10T19:08:58.949335Z","iopub.status.idle":"2026-08-10T19:09:32.852269Z","shell.execute_reply":"2026-08-10T19:09:32.851426Z"},"papermill":{"duration":33.912431,"end_time":"2026-08-10T19:09:32.857464+00:00","exception":false,"start_time":"2026-08-10T19:08:58.945033+00:00","status":"completed"},"tags":[]},"outputs":[{"name":"stdout","output_type":"stream","text":["PyTorch: 2.10.0+cu128\n","CUDA: True\n","GPU: Tesla T4\n","DeepseekV3RMSNorm\n","DeepseekV3YarnRotaryEmbedding\n","DeepseekV3MLP\n","MoEGate\n","DeepseekV3MoE\n","DeepseekV3Attention\n","DeepseekV3DecoderLayer\n","DeepseekV3MTPSharedHead\n","DeepseekV3MTPDecoderLayer\n","DeepseekV3Model\n","DeepseekV3ForCausalLM\n"]}],"source":["\n","# 3. IMPORT + VERIFY\n","import os, sys\n","os.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\"\n","sys.path.insert(0, \"/kaggle/working\")\n","\n","import torch\n","import torch.nn.functional as F\n","\n","from deepseek_v3_exact.configuration_deepseek import DeepseekV3Config\n","from deepseek_v3_exact.modeling_deepseek import (\n"," DeepseekV3RMSNorm,\n"," DeepseekV3YarnRotaryEmbedding,\n"," DeepseekV3MLP,\n"," MoEGate,\n"," DeepseekV3MoE,\n"," DeepseekV3Attention,\n"," DeepseekV3DecoderLayer,\n"," DeepseekV3MTPSharedHead,\n"," DeepseekV3MTPDecoderLayer,\n"," DeepseekV3Model,\n"," DeepseekV3ForCausalLM,\n",")\n","\n","print(\"PyTorch:\", torch.__version__)\n","print(\"CUDA:\", torch.cuda.is_available())\n","if torch.cuda.is_available():\n"," print(\"GPU:\", torch.cuda.get_device_name(0))\n","\n","for cls in [\n"," DeepseekV3RMSNorm,\n"," DeepseekV3YarnRotaryEmbedding,\n"," DeepseekV3MLP,\n"," MoEGate,\n"," DeepseekV3MoE,\n"," DeepseekV3Attention,\n"," DeepseekV3DecoderLayer,\n"," DeepseekV3MTPSharedHead,\n"," DeepseekV3MTPDecoderLayer,\n"," DeepseekV3Model,\n"," DeepseekV3ForCausalLM,\n","]:\n"," print(cls.__name__)\n"]},{"cell_type":"code","execution_count":4,"id":"8e56fbbb","metadata":{"execution":{"iopub.execute_input":"2026-08-10T19:09:32.86642Z","iopub.status.busy":"2026-08-10T19:09:32.865457Z","iopub.status.idle":"2026-08-10T19:09:32.885927Z","shell.execute_reply":"2026-08-10T19:09:32.884923Z"},"papermill":{"duration":0.026378,"end_time":"2026-08-10T19:09:32.887453+00:00","exception":false,"start_time":"2026-08-10T19:09:32.861075+00:00","status":"completed"},"tags":[]},"outputs":[{"name":"stdout","output_type":"stream","text":["Device: cuda\n"]}],"source":["\n","# 4. SETTINGS\n","import re, gc, json, math, time, random\n","from pathlib import Path\n","\n","import numpy as np\n","import matplotlib.pyplot as plt\n","\n","from torch.utils.data import IterableDataset, DataLoader\n","from tokenizers import Tokenizer, models, trainers, pre_tokenizers, decoders\n","from tokenizers.processors import TemplateProcessing\n","\n","SEED = 42\n","random.seed(SEED)\n","np.random.seed(SEED)\n","torch.manual_seed(SEED)\n","if torch.cuda.is_available():\n"," torch.cuda.manual_seed_all(SEED)\n","\n","DATA_PATH = Path(\"/kaggle/input/datasets/organizations/Cornell-University/arxiv/arxiv-metadata-oai-snapshot.json\")\n","OUT_DIR = Path(\"/kaggle/working/arxiv_deepseek_v3\")\n","OUT_DIR.mkdir(parents=True, exist_ok=True)\n","\n","TOKENIZER_PATH = OUT_DIR / \"tokenizer.json\"\n","TOKENIZER_CONFIG_PATH = OUT_DIR / \"tokenizer_config.json\"\n","CONFIG_PATH = OUT_DIR / \"config.json\"\n","CHECKPOINT_PATH = OUT_DIR / \"model.pt\"\n","LOSS_PATH = OUT_DIR / \"training_loss.png\"\n","\n","assert DATA_PATH.exists(), DATA_PATH\n","\n","DEVICE = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n","\n","VOCAB_TARGET = 24000\n","TOKENIZER_TRAIN_DOCS = 200_000\n","SEQ_LEN = 512\n","\n","BATCH_SIZE = 2\n","GRAD_ACCUM_STEPS = 8\n","MAX_TRAIN_STEPS = 3000\n","LEARNING_RATE = 3e-4\n","MIN_LR_RATIO = 0.10\n","WARMUP_STEPS = 150\n","WEIGHT_DECAY = 0.10\n","GRAD_CLIP = 1.0\n","LOG_EVERY = 10\n","SAVE_EVERY = 250\n","\n","HIDDEN_SIZE = 384\n","INTERMEDIATE_SIZE = 1024\n","MOE_INTERMEDIATE_SIZE = 384\n","\n","NUM_HIDDEN_LAYERS = 6\n","NUM_ATTENTION_HEADS = 6\n","NUM_KEY_VALUE_HEADS = 6\n","\n","Q_LORA_RANK = 128\n","KV_LORA_RANK = 96\n","QK_NOPE_HEAD_DIM = 48\n","QK_ROPE_HEAD_DIM = 16\n","V_HEAD_DIM = 64\n","\n","FIRST_K_DENSE_REPLACE = 2\n","N_ROUTED_EXPERTS = 8\n","N_SHARED_EXPERTS = 1\n","NUM_EXPERTS_PER_TOK = 2\n","\n","N_GROUP = 4\n","TOPK_GROUP = 2\n","\n","ROUTED_SCALING_FACTOR = 1.0\n","TOPK_METHOD = \"noaux_tc\"\n","SCORING_FUNC = \"sigmoid\"\n","NORM_TOPK_PROB = True\n","\n","MAX_POSITION_EMBEDDINGS = 4096\n","ROPE_THETA = 10000.0\n","ROPE_SCALING = {\n"," \"type\": \"yarn\",\n"," \"factor\": 4.0,\n"," \"original_max_position_embeddings\": 1024,\n"," \"beta_fast\": 32,\n"," \"beta_slow\": 1,\n"," \"mscale\": 1.0,\n"," \"mscale_all_dim\": 1.0,\n","}\n","\n","print(\"Device:\", DEVICE)\n"]},{"cell_type":"code","execution_count":5,"id":"e980a8fb","metadata":{"execution":{"iopub.execute_input":"2026-08-10T19:09:32.895805Z","iopub.status.busy":"2026-08-10T19:09:32.895377Z","iopub.status.idle":"2026-08-10T19:09:32.913112Z","shell.execute_reply":"2026-08-10T19:09:32.912514Z"},"papermill":{"duration":0.02364,"end_time":"2026-08-10T19:09:32.914672+00:00","exception":false,"start_time":"2026-08-10T19:09:32.891032+00:00","status":"completed"},"tags":[]},"outputs":[{"name":"stdout","output_type":"stream","text":["====================================================================================================\n","Title: Calculation of prompt diphoton production cross sections at Tevatron and LHC energies\n","Abstract: A fully differential calculation in perturbative quantum chromodynamics is presented for the production of massive photon pairs at hadron colliders. All next-to-leading order perturbative contributions from quark-antiquark, gluon-(anti)quark, and gluon-gluon subprocesses are included, as well as all-orders resummation of initial-state gluon radiation valid at next-to-next-to-leading logarithmic accuracy. The region of phase space is specified in which the calculation is most reliable. Good agreement is demonstrated with data from the Fermilab Tevatron, and predictions are made for more detailed tests with CDF and DO data. Predictions are shown for distributions of diphoton pairs produced at the energy of the Large Hadron Collider (LHC). Distributions of the diphoton pairs from the decay of a Higgs boson are contrasted with those produced from QCD processes at the LHC, showing that enhanced sensitivity to the signal can be obtained with judicious selection of events.\n","====================================================================================================\n","Title: Sparsity-certifying Graph Decompositions\n","Abstract: We describe a new algorithm, the $(k,\\ell)$-pebble game with colors, and use it obtain a characterization of the family of $(k,\\ell)$-sparse graphs and algorithmic solutions to a family of problems concerning tree decompositions of graphs. Special instances of sparse graphs appear in rigidity theory and have received increased attention in recent years. In particular, our colored pebbles generalize and strengthen the previous results of Lee and Streinu and give a new proof of the Tutte-Nash-Williams characterization of arboricity. We also present a new decomposition that certifies sparsity based on the $(k,\\ell)$-pebble game with colors. Our work also exposes connections between pebble game algorithms and previous sparse graph algorithms by Gabow, Gabow and Westermann and Hendrickson.\n","====================================================================================================\n","Title: The evolution of the Earth-Moon system based on the dark matter field fluid model\n","Abstract: The evolution of Earth-Moon system is described by the dark matter field fluid model proposed in the Meeting of Division of Particle and Field 2004, American Physical Society. The current behavior of the Earth-Moon system agrees with this model very well and the general pattern of the evolution of the Moon-Earth system described by this model agrees with geological and fossil evidence. The closest distance of the Moon to Earth was about 259000 km at 4.5 billion years ago, which is far beyond the Roche's limit. The result suggests that the tidal friction may not be the primary cause for the evolution of the Earth-Moon system. The average dark matter field fluid constant derived from Earth-Moon system data is 4.39 x 10^(-22) s^(-1)m^(-1). This model predicts that the Mars's rotation is also slowing with the angular acceleration rate about -4.38 x 10^(-22) rad s^(-2).\n"]}],"source":["\n","# 5. STREAM title + abstract\n","_space_re = re.compile(r\"\\s+\")\n","\n","def clean_text(x):\n"," if x is None:\n"," return \"\"\n"," return _space_re.sub(\" \", str(x).replace(\"\\x00\", \" \")).strip()\n","\n","def iter_arxiv_documents(max_docs=None):\n"," count = 0\n"," with DATA_PATH.open(\"r\", encoding=\"utf-8\") as f:\n"," for line in f:\n"," if not line.strip():\n"," continue\n"," try:\n"," obj = json.loads(line)\n"," except json.JSONDecodeError:\n"," continue\n","\n"," title = clean_text(obj.get(\"title\"))\n"," abstract = clean_text(obj.get(\"abstract\"))\n","\n"," if not title and not abstract:\n"," continue\n","\n"," text = \"\"\n"," if title:\n"," text += \"Title: \" + title\n"," if abstract:\n"," if text:\n"," text += \"\\n\"\n"," text += \"Abstract: \" + abstract\n","\n"," yield text\n"," count += 1\n","\n"," if max_docs is not None and count >= max_docs:\n"," return\n","\n","for x in iter_arxiv_documents(max_docs=3):\n"," print(\"=\" * 100)\n"," print(x[:1200])\n"]},{"cell_type":"code","execution_count":6,"id":"d645ca65","metadata":{"execution":{"iopub.execute_input":"2026-08-10T19:09:32.923448Z","iopub.status.busy":"2026-08-10T19:09:32.922764Z","iopub.status.idle":"2026-08-10T19:12:41.286473Z","shell.execute_reply":"2026-08-10T19:12:41.28571Z"},"papermill":{"duration":188.370094,"end_time":"2026-08-10T19:12:41.288383+00:00","exception":false,"start_time":"2026-08-10T19:09:32.918289+00:00","status":"completed"},"tags":[]},"outputs":[{"name":"stdout","output_type":"stream","text":["\n","\n","\n","vocab: 24000\n","bos: 0 eos: 1 unk: 2\n"]}],"source":["\n","# 6. TRAIN TOKENIZER\n","BOS = \"<｜begin▁of▁sentence｜>\"\n","EOS = \"<｜end▁of▁sentence｜>\"\n","UNK = \"<|unk|>\"\n","\n","if TOKENIZER_PATH.exists():\n"," tokenizer = Tokenizer.from_file(str(TOKENIZER_PATH))\n","else:\n"," tokenizer = Tokenizer(models.BPE(unk_token=UNK))\n"," tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=False)\n"," tokenizer.decoder = decoders.ByteLevel()\n","\n"," trainer = trainers.BpeTrainer(\n"," vocab_size=VOCAB_TARGET,\n"," min_frequency=2,\n"," special_tokens=[BOS, EOS, UNK],\n"," show_progress=True,\n"," )\n","\n"," tokenizer.train_from_iterator(\n"," iter_arxiv_documents(max_docs=TOKENIZER_TRAIN_DOCS),\n"," trainer=trainer,\n"," length=TOKENIZER_TRAIN_DOCS,\n"," )\n","\n"," tokenizer.post_processor = TemplateProcessing(\n"," single=f\"{BOS} $A {EOS}\",\n"," special_tokens=[\n"," (BOS, tokenizer.token_to_id(BOS)),\n"," (EOS, tokenizer.token_to_id(EOS)),\n"," ],\n"," )\n","\n"," tokenizer.save(str(TOKENIZER_PATH))\n","\n","BOS_ID = tokenizer.token_to_id(BOS)\n","EOS_ID = tokenizer.token_to_id(EOS)\n","UNK_ID = tokenizer.token_to_id(UNK)\n","VOCAB_SIZE = tokenizer.get_vocab_size()\n","\n","assert BOS_ID == 0\n","assert EOS_ID == 1\n","\n","tokenizer_cfg = {\n"," \"add_bos_token\": True,\n"," \"add_eos_token\": False,\n"," \"bos_token\": BOS,\n"," \"eos_token\": EOS,\n"," \"pad_token\": EOS,\n"," \"bos_token_id\": BOS_ID,\n"," \"eos_token_id\": EOS_ID,\n"," \"pad_token_id\": EOS_ID,\n"," \"model_max_length\": MAX_POSITION_EMBEDDINGS,\n","}\n","\n","TOKENIZER_CONFIG_PATH.write_text(\n"," json.dumps(tokenizer_cfg, indent=2, ensure_ascii=False),\n"," encoding=\"utf-8\",\n",")\n","\n","print(\"vocab:\", VOCAB_SIZE)\n","print(\"bos:\", BOS_ID, \"eos:\", EOS_ID, \"unk:\", UNK_ID)\n"]},{"cell_type":"code","execution_count":7,"id":"a01712e3","metadata":{"execution":{"iopub.execute_input":"2026-08-10T19:12:41.297654Z","iopub.status.busy":"2026-08-10T19:12:41.29738Z","iopub.status.idle":"2026-08-10T19:12:41.307143Z","shell.execute_reply":"2026-08-10T19:12:41.306075Z"},"papermill":{"duration":0.016374,"end_time":"2026-08-10T19:12:41.308705+00:00","exception":false,"start_time":"2026-08-10T19:12:41.292331+00:00","status":"completed"},"tags":[]},"outputs":[{"name":"stdout","output_type":"stream","text":["DeepseekV3Config {\n"," \"attention_bias\": false,\n"," \"attention_dropout\": 0.0,\n"," \"bos_token_id\": 0,\n"," \"eos_token_id\": 1,\n"," \"ep_size\": 1,\n"," \"first_k_dense_replace\": 2,\n"," \"hidden_act\": \"silu\",\n"," \"hidden_size\": 384,\n"," \"initializer_range\": 0.006,\n"," \"intermediate_size\": 1024,\n"," \"kv_lora_rank\": 96,\n"," \"max_position_embeddings\": 4096,\n"," \"model_type\": \"deepseek_v3\",\n"," \"moe_intermediate_size\": 384,\n"," \"moe_layer_freq\": 1,\n"," \"mtp_loss_weight\": 0.3,\n"," \"n_group\": 4,\n"," \"n_layers\": 6,\n"," \"n_positions\": 4096,\n"," \"n_routed_experts\": 8,\n"," \"n_shared_experts\": 1,\n"," \"norm_topk_prob\": true,\n"," \"num_attention_heads\": 6,\n"," \"num_experts_per_tok\": 2,\n"," \"num_hidden_layers\": 6,\n"," \"num_key_value_heads\": 6,\n"," \"num_nextn_predict_layers\": 1,\n"," \"pad_token_id\": 1,\n"," \"q_lora_rank\": 128,\n"," \"qk_nope_head_dim\": 48,\n"," \"qk_rope_head_dim\": 16,\n"," \"rms_norm_eps\": 1e-06,\n"," \"rope_scaling\": {\n"," \"beta_fast\": 32,\n"," \"beta_slow\": 1,\n"," \"factor\": 4.0,\n"," \"mscale\": 1.0,\n"," \"mscale_all_dim\": 1.0,\n"," \"original_max_position_embeddings\": 1024,\n"," \"type\": \"yarn\"\n"," },\n"," \"rope_theta\": 10000.0,\n"," \"routed_scaling_factor\": 1.0,\n"," \"scoring_func\": \"sigmoid\",\n"," \"tie_word_embeddings\": false,\n"," \"topk_group\": 2,\n"," \"topk_method\": \"noaux_tc\",\n"," \"transformers_version\": \"4.46.3\",\n"," \"use_cache\": false,\n"," \"v_head_dim\": 64,\n"," \"vocab_size\": 24000\n","}\n","\n"]}],"source":["\n","# 7. FULL DEEPSEEK CONFIG — ONLY SIZES REDUCED\n","config = DeepseekV3Config(\n"," vocab_size=VOCAB_SIZE,\n"," hidden_size=HIDDEN_SIZE,\n"," intermediate_size=INTERMEDIATE_SIZE,\n"," moe_intermediate_size=MOE_INTERMEDIATE_SIZE,\n","\n"," num_hidden_layers=NUM_HIDDEN_LAYERS,\n"," num_nextn_predict_layers=1,\n"," mtp_loss_weight=0.3,\n","\n"," num_attention_heads=NUM_ATTENTION_HEADS,\n"," num_key_value_heads=NUM_KEY_VALUE_HEADS,\n","\n"," n_shared_experts=N_SHARED_EXPERTS,\n"," n_routed_experts=N_ROUTED_EXPERTS,\n"," ep_size=1,\n","\n"," routed_scaling_factor=ROUTED_SCALING_FACTOR,\n","\n"," kv_lora_rank=KV_LORA_RANK,\n"," q_lora_rank=Q_LORA_RANK,\n"," qk_rope_head_dim=QK_ROPE_HEAD_DIM,\n"," qk_nope_head_dim=QK_NOPE_HEAD_DIM,\n"," v_head_dim=V_HEAD_DIM,\n","\n"," topk_method=TOPK_METHOD,\n"," n_group=N_GROUP,\n"," topk_group=TOPK_GROUP,\n"," num_experts_per_tok=NUM_EXPERTS_PER_TOK,\n","\n"," moe_layer_freq=1,\n"," first_k_dense_replace=FIRST_K_DENSE_REPLACE,\n","\n"," norm_topk_prob=NORM_TOPK_PROB,\n"," scoring_func=SCORING_FUNC,\n","\n"," hidden_act=\"silu\",\n"," max_position_embeddings=MAX_POSITION_EMBEDDINGS,\n","\n"," initializer_range=0.006,\n"," rms_norm_eps=1e-6,\n"," use_cache=False,\n","\n"," pad_token_id=EOS_ID,\n"," bos_token_id=BOS_ID,\n"," eos_token_id=EOS_ID,\n","\n"," tie_word_embeddings=False,\n","\n"," rope_theta=ROPE_THETA,\n"," rope_scaling=ROPE_SCALING,\n","\n"," attention_bias=False,\n"," attention_dropout=0.0,\n",")\n","\n","config._attn_implementation = \"eager\"\n","\n","CONFIG_PATH.write_text(\n"," json.dumps(config.to_dict(), indent=2),\n"," encoding=\"utf-8\",\n",")\n","\n","print(config)\n"]},{"cell_type":"code","execution_count":8,"id":"3c48d675","metadata":{"execution":{"iopub.execute_input":"2026-08-10T19:12:41.318609Z","iopub.status.busy":"2026-08-10T19:12:41.317737Z","iopub.status.idle":"2026-08-10T19:12:42.640296Z","shell.execute_reply":"2026-08-10T19:12:42.639392Z"},"papermill":{"duration":1.329975,"end_time":"2026-08-10T19:12:42.642813+00:00","exception":false,"start_time":"2026-08-10T19:12:41.312838+00:00","status":"completed"},"tags":[]},"outputs":[{"name":"stderr","output_type":"stream","text":["DeepseekV3ForCausalLM has generative capabilities, as `prepare_inputs_for_generation` is explicitly overwritten. However, it doesn't directly inherit from `GenerationMixin`. From 👉v4.50👈 onwards, `PreTrainedModel` will NOT inherit from `GenerationMixin`, and this model will lose the ability to call `generate` and other related functions.\n"," - If you're using `trust_remote_code=True`, you can get rid of this warning by loading the model with an auto class. See https://huggingface.co/docs/transformers/en/model_doc/auto#auto-classes\n"," - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).\n"," - If you are not the owner of the model architecture class, please contact the model code owner to update it.\n"]},{"name":"stdout","output_type":"stream","text":["Total parameters: 43,489,608 (43.49M)\n","Trainable: 43,489,608 (43.49M)\n","Main layers: 6\n","MTP layers: 1\n","Total stack: 7\n","\n","FIRST MAIN LAYER\n","DeepseekV3DecoderLayer(\n"," (self_attn): DeepseekV3Attention(\n"," (q_a_proj): Linear(in_features=384, out_features=128, bias=False)\n"," (q_a_layernorm): DeepseekV3RMSNorm()\n"," (q_b_proj): Linear(in_features=128, out_features=384, bias=False)\n"," (kv_a_proj_with_mqa): Linear(in_features=384, out_features=112, bias=False)\n"," (kv_a_layernorm): DeepseekV3RMSNorm()\n"," (kv_b_proj): Linear(in_features=96, out_features=672, bias=False)\n"," (o_proj): Linear(in_features=384, out_features=384, bias=False)\n"," (rotary_emb): DeepseekV3YarnRotaryEmbedding()\n"," )\n"," (mlp): DeepseekV3MLP(\n"," (gate_proj): Linear(in_features=384, out_features=1024, bias=False)\n"," (up_proj): Linear(in_features=384, out_features=1024, bias=False)\n"," (down_proj): Linear(in_features=1024, out_features=384, bias=False)\n"," (act_fn): SiLU()\n"," )\n"," (input_layernorm): DeepseekV3RMSNorm()\n"," (post_attention_layernorm): DeepseekV3RMSNorm()\n",")\n","\n","FIRST MoE MAIN LAYER\n","DeepseekV3DecoderLayer(\n"," (self_attn): DeepseekV3Attention(\n"," (q_a_proj): Linear(in_features=384, out_features=128, bias=False)\n"," (q_a_layernorm): DeepseekV3RMSNorm()\n"," (q_b_proj): Linear(in_features=128, out_features=384, bias=False)\n"," (kv_a_proj_with_mqa): Linear(in_features=384, out_features=112, bias=False)\n"," (kv_a_layernorm): DeepseekV3RMSNorm()\n"," (kv_b_proj): Linear(in_features=96, out_features=672, bias=False)\n"," (o_proj): Linear(in_features=384, out_features=384, bias=False)\n"," (rotary_emb): DeepseekV3YarnRotaryEmbedding()\n"," )\n"," (mlp): DeepseekV3MoE(\n"," (experts): ModuleList(\n"," (0-7): 8 x DeepseekV3MLP(\n"," (gate_proj): Linear(in_features=384, out_features=384, bias=False)\n"," (up_proj): Linear(in_features=384, out_features=384, bias=False)\n"," (down_proj): Linear(in_features=384, out_features=384, bias=False)\n"," (act_fn): SiLU()\n"," )\n"," )\n"," (gate): MoEGate()\n"," (shared_experts): DeepseekV3MLP(\n"," (gate_proj): Linear(in_features=384, out_features=384, bias=False)\n"," (up_proj): Linear(in_features=384, out_features=384, bias=False)\n"," (down_proj): Linear(in_features=384, out_features=384, bias=False)\n"," (act_fn): SiLU()\n"," )\n"," )\n"," (input_layernorm): DeepseekV3RMSNorm()\n"," (post_attention_layernorm): DeepseekV3RMSNorm()\n",")\n","\n","MTP LAYER\n","DeepseekV3MTPDecoderLayer(\n"," (self_attn): DeepseekV3Attention(\n"," (q_a_proj): Linear(in_features=384, out_features=128, bias=False)\n"," (q_a_layernorm): DeepseekV3RMSNorm()\n"," (q_b_proj): Linear(in_features=128, out_features=384, bias=False)\n"," (kv_a_proj_with_mqa): Linear(in_features=384, out_features=112, bias=False)\n"," (kv_a_layernorm): DeepseekV3RMSNorm()\n"," (kv_b_proj): Linear(in_features=96, out_features=672, bias=False)\n"," (o_proj): Linear(in_features=384, out_features=384, bias=False)\n"," (rotary_emb): DeepseekV3YarnRotaryEmbedding()\n"," )\n"," (mlp): DeepseekV3MoE(\n"," (experts): ModuleList(\n"," (0-7): 8 x DeepseekV3MLP(\n"," (gate_proj): Linear(in_features=384, out_features=384, bias=False)\n"," (up_proj): Linear(in_features=384, out_features=384, bias=False)\n"," (down_proj): Linear(in_features=384, out_features=384, bias=False)\n"," (act_fn): SiLU()\n"," )\n"," )\n"," (gate): MoEGate()\n"," (shared_experts): DeepseekV3MLP(\n"," (gate_proj): Linear(in_features=384, out_features=384, bias=False)\n"," (up_proj): Linear(in_features=384, out_features=384, bias=False)\n"," (down_proj): Linear(in_features=384, out_features=384, bias=False)\n"," (act_fn): SiLU()\n"," )\n"," )\n"," (input_layernorm): DeepseekV3RMSNorm()\n"," (post_attention_layernorm): DeepseekV3RMSNorm()\n"," (embed_tokens): Embedding(24000, 384, padding_idx=1)\n"," (enorm): DeepseekV3RMSNorm()\n"," (hnorm): DeepseekV3RMSNorm()\n"," (eh_proj): Linear(in_features=768, out_features=384, bias=False)\n"," (shared_head): DeepseekV3MTPSharedHead(\n"," (norm): DeepseekV3RMSNorm()\n"," (head): Linear(in_features=384, out_features=24000, bias=False)\n"," )\n",")\n","\n","MTP STRUCTURE\n","embed_tokens: Embedding(24000, 384, padding_idx=1)\n","enorm: DeepseekV3RMSNorm()\n","hnorm: DeepseekV3RMSNorm()\n","eh_proj: Linear(in_features=768, out_features=384, bias=False)\n","self_attn: DeepseekV3Attention\n","mlp: DeepseekV3MoE\n","shared_head: DeepseekV3MTPSharedHead(\n"," (norm): DeepseekV3RMSNorm()\n"," (head): Linear(in_features=384, out_features=24000, bias=False)\n",")\n","\n","MTP embedding shared: True\n","MTP output head shared: True\n"]}],"source":["# 8. INSTANTIATE FULL DeepseekV3ForCausalLM + MTP\n","model = DeepseekV3ForCausalLM(config).to(DEVICE)\n","model.config.use_cache = False\n","\n","total_params = sum(p.numel() for p in model.parameters())\n","trainable_params = sum(p.numel() for p in model.parameters() if p.requires_grad)\n","\n","print(f\"Total parameters: {total_params:,} ({total_params/1e6:.2f}M)\")\n","print(f\"Trainable: {trainable_params:,} ({trainable_params/1e6:.2f}M)\")\n","print(f\"Main layers: {config.num_hidden_layers}\")\n","print(f\"MTP layers: {config.num_nextn_predict_layers}\")\n","print(f\"Total stack: {len(model.model.layers)}\")\n","\n","assert len(model.model.layers) == (\n"," config.num_hidden_layers + config.num_nextn_predict_layers\n",")\n","\n","print(\"\\nFIRST MAIN LAYER\")\n","print(model.model.layers[0])\n","\n","print(\"\\nFIRST MoE MAIN LAYER\")\n","print(model.model.layers[FIRST_K_DENSE_REPLACE])\n","\n","mtp_layer = model.model.layers[config.num_hidden_layers]\n","\n","print(\"\\nMTP LAYER\")\n","print(mtp_layer)\n","\n","print(\"\\nMTP STRUCTURE\")\n","print(\"embed_tokens:\", mtp_layer.embed_tokens)\n","print(\"enorm:\", mtp_layer.enorm)\n","print(\"hnorm:\", mtp_layer.hnorm)\n","print(\"eh_proj:\", mtp_layer.eh_proj)\n","print(\"self_attn:\", type(mtp_layer.self_attn).__name__)\n","print(\"mlp:\", type(mtp_layer.mlp).__name__)\n","print(\"shared_head:\", mtp_layer.shared_head)\n","\n","# Physical parameter sharing used by the MTP path.\n","assert mtp_layer.embed_tokens.weight is model.model.embed_tokens.weight\n","assert mtp_layer.shared_head.head.weight is model.lm_head.weight\n","print(\"\\nMTP embedding shared:\", True)\n","print(\"MTP output head shared:\", True)\n"]},{"cell_type":"code","execution_count":9,"id":"279feee7","metadata":{"execution":{"iopub.execute_input":"2026-08-10T19:12:42.657293Z","iopub.status.busy":"2026-08-10T19:12:42.656986Z","iopub.status.idle":"2026-08-10T19:12:42.686676Z","shell.execute_reply":"2026-08-10T19:12:42.685814Z"},"papermill":{"duration":0.039927,"end_time":"2026-08-10T19:12:42.688259+00:00","exception":false,"start_time":"2026-08-10T19:12:42.648332+00:00","status":"completed"},"tags":[]},"outputs":[{"name":"stdout","output_type":"stream","text":["input_ids torch.Size([2, 512])\n","attention_mask torch.Size([2, 512])\n","labels torch.Size([2, 512])\n"]}],"source":["\n","# 9. STREAMING DATASET\n","class ArxivTokenDataset(IterableDataset):\n"," def __init__(self, seq_len):\n"," super().__init__()\n"," self.seq_len = seq_len\n","\n"," def __iter__(self):\n"," buffer = []\n","\n"," with DATA_PATH.open(\"r\", encoding=\"utf-8\") as f:\n"," for line in f:\n"," if not line.strip():\n"," continue\n","\n"," try:\n"," obj = json.loads(line)\n"," except json.JSONDecodeError:\n"," continue\n","\n"," title = clean_text(obj.get(\"title\"))\n"," abstract = clean_text(obj.get(\"abstract\"))\n","\n"," if not title and not abstract:\n"," continue\n","\n"," text = \"\"\n"," if title:\n"," text += \"Title: \" + title\n"," if abstract:\n"," if text:\n"," text += \"\\n\"\n"," text += \"Abstract: \" + abstract\n","\n"," ids = tokenizer.encode(text).ids\n"," buffer.extend(ids)\n","\n"," while len(buffer) >= self.seq_len:\n"," chunk = buffer[:self.seq_len]\n"," buffer = buffer[self.seq_len:]\n","\n"," x = torch.tensor(chunk, dtype=torch.long)\n","\n"," yield {\n"," \"input_ids\": x,\n"," \"attention_mask\": torch.ones_like(x),\n"," \"labels\": x.clone(),\n"," }\n","\n","dataset = ArxivTokenDataset(SEQ_LEN)\n","\n","loader = DataLoader(\n"," dataset,\n"," batch_size=BATCH_SIZE,\n"," num_workers=0,\n"," pin_memory=torch.cuda.is_available(),\n",")\n","\n","batch = next(iter(loader))\n","for k, v in batch.items():\n"," print(k, v.shape)\n"]},{"cell_type":"code","execution_count":10,"id":"663ae575","metadata":{"execution":{"iopub.execute_input":"2026-08-10T19:12:42.697813Z","iopub.status.busy":"2026-08-10T19:12:42.69758Z","iopub.status.idle":"2026-08-10T19:12:44.105506Z","shell.execute_reply":"2026-08-10T19:12:44.104506Z"},"papermill":{"duration":1.415114,"end_time":"2026-08-10T19:12:44.107496+00:00","exception":false,"start_time":"2026-08-10T19:12:42.692382+00:00","status":"completed"},"tags":[]},"outputs":[{"name":"stdout","output_type":"stream","text":["patched DeepseekV3MoE mixed-precision forward\n"]},{"name":"stderr","output_type":"stream","text":["/tmp/ipykernel_23/3836598894.py:105: UserWarning: Converting a tensor with requires_grad=True to a scalar may lead to unexpected behavior.\n","Consider using tensor.detach() first. (Triggered internally at /pytorch/torch/csrc/autograd/generated/python_variable_methods.cpp:836.)\n"," print(\"total loss:\", float(out.loss))\n"]},{"name":"stdout","output_type":"stream","text":["total loss: 13.133403778076172\n","main loss: 10.100336074829102\n","weighted MTP loss: 3.0330679416656494\n","finite model gradients: True\n","finite MTP gradients: True\n","\n","FULL DeepSeek-V3 + MTP forward/backward passed\n"]}],"source":["# 10. FIX MIXED-PRECISION MoE + FULL FORWARD/BACKWARD SANITY TEST\n","\n","def fixed_moe_forward(self, hidden_states):\n"," identity = hidden_states\n"," orig_shape = hidden_states.shape\n","\n"," topk_idx, topk_weight = self.gate(hidden_states)\n","\n"," hidden_states = hidden_states.view(\n"," -1,\n"," hidden_states.shape[-1]\n"," )\n","\n"," if self.training:\n"," # Keep accumulator in hidden-state dtype.\n"," y = torch.zeros_like(hidden_states)\n","\n"," for expert_id, expert in enumerate(self.experts):\n"," if expert is None:\n"," continue\n","\n"," token_idx, route_idx = torch.where(\n"," topk_idx == expert_id\n"," )\n","\n"," if token_idx.numel() == 0:\n"," continue\n","\n"," expert_out = expert(\n"," hidden_states[token_idx]\n"," )\n","\n"," route_weight = (\n"," topk_weight[token_idx, route_idx]\n"," .to(expert_out.dtype)\n"," .unsqueeze(-1)\n"," )\n","\n"," contribution = expert_out * route_weight\n","\n"," # IMPORTANT:\n"," # index_add_ requires source and destination\n"," # to have exactly the same dtype.\n"," contribution = contribution.to(y.dtype)\n","\n"," y.index_add_(\n"," 0,\n"," token_idx,\n"," contribution\n"," )\n","\n"," y = y.view(*orig_shape)\n","\n"," else:\n"," y = self.moe_infer(\n"," hidden_states,\n"," topk_idx,\n"," topk_weight\n"," ).view(*orig_shape)\n","\n"," if self.config.n_shared_experts is not None:\n"," shared_out = self.shared_experts(identity)\n"," y = y + shared_out.to(y.dtype)\n","\n"," return y\n","\n","\n","# Patch every DeepSeek-V3 MoE module.\n","for module in model.modules():\n"," if module.__class__.__name__ == \"DeepseekV3MoE\":\n"," module.forward = fixed_moe_forward.__get__(\n"," module,\n"," module.__class__\n"," )\n","\n","print(\"patched DeepseekV3MoE mixed-precision forward\")\n","\n","\n","# ============================================================\n","# FULL FORWARD + BACKWARD TEST — MAIN LM + MTP\n","# ============================================================\n","\n","model.train()\n","model.zero_grad(set_to_none=True)\n","\n","tiny = {\n"," k: v[:1, :64].to(DEVICE)\n"," for k, v in batch.items()\n","}\n","\n","amp_dtype = (\n"," torch.bfloat16\n"," if DEVICE.type == \"cuda\"\n"," and torch.cuda.is_bf16_supported()\n"," else torch.float16\n",")\n","\n","with torch.autocast(\n"," device_type=DEVICE.type,\n"," dtype=amp_dtype if DEVICE.type == \"cuda\" else torch.float32,\n"," enabled=DEVICE.type == \"cuda\",\n","):\n"," out = model(**tiny)\n","\n","print(\"total loss:\", float(out.loss))\n","print(\"main loss:\", float(model._last_main_loss))\n","print(\"weighted MTP loss:\", float(model._last_mtp_loss))\n","\n","assert model._last_mtp_loss is not None\n","assert torch.isfinite(out.loss)\n","\n","out.loss.backward()\n","\n","grad_found = any(\n"," p.grad is not None\n"," and torch.isfinite(p.grad).all()\n"," for p in model.parameters()\n",")\n","\n","mtp_layer = model.model.layers[\n"," config.num_hidden_layers\n","]\n","\n","mtp_grad_found = (\n"," mtp_layer.eh_proj.weight.grad is not None\n"," and torch.isfinite(\n"," mtp_layer.eh_proj.weight.grad\n"," ).all()\n",")\n","\n","print(\"finite model gradients:\", grad_found)\n","print(\"finite MTP gradients:\", bool(mtp_grad_found))\n","\n","assert grad_found\n","assert mtp_grad_found\n","\n","model.zero_grad(set_to_none=True)\n","\n","print(\"\\nFULL DeepSeek-V3 + MTP forward/backward passed\")"]},{"cell_type":"code","execution_count":11,"id":"ddaa46c6","metadata":{"execution":{"iopub.execute_input":"2026-08-10T19:12:44.118927Z","iopub.status.busy":"2026-08-10T19:12:44.118229Z","iopub.status.idle":"2026-08-10T19:12:44.125636Z","shell.execute_reply":"2026-08-10T19:12:44.124969Z"},"papermill":{"duration":0.014638,"end_time":"2026-08-10T19:12:44.127198+00:00","exception":false,"start_time":"2026-08-10T19:12:44.11256+00:00","status":"completed"},"tags":[]},"outputs":[],"source":["\n","# 11. OPTIMIZER\n","optimizer = torch.optim.AdamW(\n"," model.parameters(),\n"," lr=LEARNING_RATE,\n"," betas=(0.9, 0.95),\n"," eps=1e-8,\n"," weight_decay=WEIGHT_DECAY,\n",")\n","\n","use_scaler = DEVICE.type == \"cuda\" and amp_dtype == torch.float16\n","scaler = torch.amp.GradScaler(\"cuda\", enabled=use_scaler)\n","\n","def lr_for_step(step):\n"," if step < WARMUP_STEPS:\n"," return LEARNING_RATE * (step + 1) / max(WARMUP_STEPS, 1)\n","\n"," p = (step - WARMUP_STEPS) / max(MAX_TRAIN_STEPS - WARMUP_STEPS, 1)\n"," p = min(max(p, 0.0), 1.0)\n","\n"," cosine = 0.5 * (1.0 + math.cos(math.pi * p))\n"," min_lr = LEARNING_RATE * MIN_LR_RATIO\n"," return min_lr + (LEARNING_RATE - min_lr) * cosine\n"]},{"cell_type":"code","execution_count":12,"id":"57299947","metadata":{"execution":{"iopub.execute_input":"2026-08-10T19:12:44.138752Z","iopub.status.busy":"2026-08-10T19:12:44.138065Z","iopub.status.idle":"2026-08-10T20:14:31.97725Z","shell.execute_reply":"2026-08-10T20:14:31.976192Z"},"papermill":{"duration":3707.84765,"end_time":"2026-08-10T20:14:31.979559+00:00","exception":false,"start_time":"2026-08-10T19:12:44.131909+00:00","status":"completed"},"tags":[]},"outputs":[{"name":"stdout","output_type":"stream","text":["step 1/3000 | loss 13.1164 | ppl 497015.10 | lr 2.00e-06 | grad 2.679 | ~5,041 tok/s\n","step 10/3000 | loss 12.9827 | ppl 434821.78 | lr 2.00e-05 | grad 2.379 | ~6,684 tok/s\n","step 20/3000 | loss 12.7633 | ppl 349180.39 | lr 4.00e-05 | grad 2.081 | ~6,866 tok/s\n","step 30/3000 | loss 12.4610 | ppl 258064.95 | lr 6.00e-05 | grad 2.332 | ~6,948 tok/s\n","step 40/3000 | loss 12.0765 | ppl 175697.16 | lr 8.00e-05 | grad 2.206 | ~7,008 tok/s\n","step 50/3000 | loss 11.5823 | ppl 107184.25 | lr 1.00e-04 | grad 2.339 | ~7,028 tok/s\n","step 60/3000 | loss 11.0564 | ppl 63349.86 | lr 1.20e-04 | grad 2.314 | ~7,035 tok/s\n","step 70/3000 | loss 10.5462 | ppl 38033.32 | lr 1.40e-04 | grad 2.286 | ~7,025 tok/s\n","step 80/3000 | loss 10.2554 | ppl 28436.75 | lr 1.60e-04 | grad 2.136 | ~7,002 tok/s\n","step 90/3000 | loss 9.7151 | ppl 16565.53 | lr 1.80e-04 | grad 2.129 | ~6,981 tok/s\n","step 100/3000 | loss 9.3415 | ppl 11401.49 | lr 2.00e-04 | grad 1.769 | ~6,968 tok/s\n","step 110/3000 | loss 9.0938 | ppl 8899.87 | lr 2.20e-04 | grad 1.468 | ~6,953 tok/s\n","step 120/3000 | loss 8.8818 | ppl 7199.70 | lr 2.40e-04 | grad 1.111 | ~6,941 tok/s\n","step 130/3000 | loss 8.5364 | ppl 5096.90 | lr 2.60e-04 | grad 1.144 | ~6,928 tok/s\n","step 140/3000 | loss 8.1818 | ppl 3575.44 | lr 2.80e-04 | grad 0.829 | ~6,913 tok/s\n","step 150/3000 | loss 8.1109 | ppl 3330.73 | lr 3.00e-04 | grad 0.673 | ~6,899 tok/s\n","step 160/3000 | loss 7.9064 | ppl 2714.47 | lr 3.00e-04 | grad 0.677 | ~6,884 tok/s\n","step 170/3000 | loss 7.7934 | ppl 2424.59 | lr 3.00e-04 | grad 0.657 | ~6,873 tok/s\n","step 180/3000 | loss 7.8721 | ppl 2622.99 | lr 3.00e-04 | grad 0.679 | ~6,864 tok/s\n","step 190/3000 | loss 7.4898 | ppl 1789.75 | lr 3.00e-04 | grad 0.799 | ~6,854 tok/s\n","step 200/3000 | loss 7.6194 | ppl 2037.34 | lr 3.00e-04 | grad 0.712 | ~6,846 tok/s\n","step 210/3000 | loss 7.5523 | ppl 1905.07 | lr 3.00e-04 | grad 0.806 | ~6,839 tok/s\n","step 220/3000 | loss 7.4779 | ppl 1768.54 | lr 3.00e-04 | grad 0.870 | ~6,833 tok/s\n","step 230/3000 | loss 7.4064 | ppl 1646.53 | lr 2.99e-04 | grad 0.717 | ~6,828 tok/s\n","step 240/3000 | loss 7.2867 | ppl 1460.79 | lr 2.99e-04 | grad 0.766 | ~6,823 tok/s\n","step 250/3000 | loss 7.3491 | ppl 1554.76 | lr 2.99e-04 | grad 0.792 | ~6,818 tok/s\n","saved: /kaggle/working/arxiv_deepseek_v3/model.pt\n","step 260/3000 | loss 7.5157 | ppl 1836.74 | lr 2.99e-04 | grad 0.910 | ~6,793 tok/s\n","step 270/3000 | loss 7.2807 | ppl 1452.07 | lr 2.99e-04 | grad 1.132 | ~6,790 tok/s\n","step 280/3000 | loss 7.4386 | ppl 1700.32 | lr 2.99e-04 | grad 0.954 | ~6,786 tok/s\n","step 290/3000 | loss 7.1439 | ppl 1266.39 | lr 2.98e-04 | grad 0.796 | ~6,781 tok/s\n","step 300/3000 | loss 7.0833 | ppl 1191.87 | lr 2.98e-04 | grad 0.910 | ~6,776 tok/s\n","step 310/3000 | loss 7.0577 | ppl 1161.77 | lr 2.98e-04 | grad 0.903 | ~6,773 tok/s\n","step 320/3000 | loss 7.1978 | ppl 1336.51 | lr 2.98e-04 | grad 0.880 | ~6,769 tok/s\n","step 330/3000 | loss 7.1818 | ppl 1315.29 | lr 2.97e-04 | grad 0.967 | ~6,767 tok/s\n","step 340/3000 | loss 7.0196 | ppl 1118.39 | lr 2.97e-04 | grad 0.934 | ~6,764 tok/s\n","step 350/3000 | loss 6.9698 | ppl 1064.05 | lr 2.97e-04 | grad 1.209 | ~6,762 tok/s\n","step 360/3000 | loss 6.9903 | ppl 1086.05 | lr 2.96e-04 | grad 1.028 | ~6,760 tok/s\n","step 370/3000 | loss 6.7530 | ppl 856.62 | lr 2.96e-04 | grad 0.924 | ~6,758 tok/s\n","step 380/3000 | loss 6.9199 | ppl 1012.25 | lr 2.96e-04 | grad 1.325 | ~6,757 tok/s\n","step 390/3000 | loss 7.0018 | ppl 1098.62 | lr 2.95e-04 | grad 1.324 | ~6,755 tok/s\n","step 400/3000 | loss 7.2475 | ppl 1404.61 | lr 2.95e-04 | grad 1.297 | ~6,752 tok/s\n","step 410/3000 | loss 6.9177 | ppl 1009.97 | lr 2.95e-04 | grad 1.263 | ~6,751 tok/s\n","step 420/3000 | loss 6.9796 | ppl 1074.44 | lr 2.94e-04 | grad 1.144 | ~6,749 tok/s\n","step 430/3000 | loss 6.8233 | ppl 918.98 | lr 2.94e-04 | grad 1.135 | ~6,748 tok/s\n","step 440/3000 | loss 6.8065 | ppl 903.71 | lr 2.93e-04 | grad 1.037 | ~6,747 tok/s\n","step 450/3000 | loss 7.1300 | ppl 1248.87 | lr 2.93e-04 | grad 1.096 | ~6,746 tok/s\n","step 460/3000 | loss 6.5324 | ppl 687.03 | lr 2.92e-04 | grad 1.062 | ~6,744 tok/s\n","step 470/3000 | loss 6.6535 | ppl 775.53 | lr 2.92e-04 | grad 1.280 | ~6,743 tok/s\n","step 480/3000 | loss 6.8433 | ppl 937.60 | lr 2.91e-04 | grad 1.320 | ~6,742 tok/s\n","step 490/3000 | loss 6.6490 | ppl 772.00 | lr 2.91e-04 | grad 1.033 | ~6,741 tok/s\n","step 500/3000 | loss 6.8324 | ppl 927.39 | lr 2.90e-04 | grad 1.526 | ~6,740 tok/s\n","saved: /kaggle/working/arxiv_deepseek_v3/model.pt\n","step 510/3000 | loss 6.5901 | ppl 727.88 | lr 2.90e-04 | grad 1.402 | ~6,726 tok/s\n","step 520/3000 | loss 6.6333 | ppl 759.97 | lr 2.89e-04 | grad 1.629 | ~6,725 tok/s\n","step 530/3000 | loss 6.5039 | ppl 667.73 | lr 2.88e-04 | grad 1.177 | ~6,724 tok/s\n","step 540/3000 | loss 6.6162 | ppl 747.09 | lr 2.88e-04 | grad 1.419 | ~6,723 tok/s\n","step 550/3000 | loss 6.4342 | ppl 622.76 | lr 2.87e-04 | grad 1.425 | ~6,722 tok/s\n","step 560/3000 | loss 6.5324 | ppl 687.05 | lr 2.87e-04 | grad 1.154 | ~6,721 tok/s\n","step 570/3000 | loss 6.5285 | ppl 684.38 | lr 2.86e-04 | grad 1.008 | ~6,721 tok/s\n","step 580/3000 | loss 6.7247 | ppl 832.73 | lr 2.85e-04 | grad 1.315 | ~6,720 tok/s\n","step 590/3000 | loss 6.3837 | ppl 592.11 | lr 2.84e-04 | grad 1.372 | ~6,718 tok/s\n","step 600/3000 | loss 6.4873 | ppl 656.72 | lr 2.84e-04 | grad 1.492 | ~6,717 tok/s\n","step 610/3000 | loss 6.5270 | ppl 683.32 | lr 2.83e-04 | grad 1.263 | ~6,716 tok/s\n","step 620/3000 | loss 6.5489 | ppl 698.49 | lr 2.82e-04 | grad 1.295 | ~6,715 tok/s\n","step 630/3000 | loss 6.3458 | ppl 570.07 | lr 2.82e-04 | grad 1.372 | ~6,714 tok/s\n","step 640/3000 | loss 6.4917 | ppl 659.66 | lr 2.81e-04 | grad 1.020 | ~6,713 tok/s\n","step 650/3000 | loss 6.3312 | ppl 561.83 | lr 2.80e-04 | grad 1.055 | ~6,712 tok/s\n","step 660/3000 | loss 6.4597 | ppl 638.84 | lr 2.79e-04 | grad 1.138 | ~6,712 tok/s\n","step 670/3000 | loss 6.4374 | ppl 624.78 | lr 2.79e-04 | grad 1.289 | ~6,711 tok/s\n","step 680/3000 | loss 7.2661 | ppl 1430.98 | lr 2.78e-04 | grad 1.321 | ~6,711 tok/s\n","step 690/3000 | loss 6.4765 | ppl 649.70 | lr 2.77e-04 | grad 1.330 | ~6,710 tok/s\n","step 700/3000 | loss 6.4246 | ppl 616.86 | lr 2.76e-04 | grad 1.063 | ~6,710 tok/s\n","step 710/3000 | loss 6.1733 | ppl 479.77 | lr 2.75e-04 | grad 1.163 | ~6,710 tok/s\n","step 720/3000 | loss 6.2114 | ppl 498.40 | lr 2.74e-04 | grad 1.481 | ~6,709 tok/s\n","step 730/3000 | loss 6.3212 | ppl 556.25 | lr 2.73e-04 | grad 1.183 | ~6,709 tok/s\n","step 740/3000 | loss 6.4906 | ppl 658.90 | lr 2.73e-04 | grad 1.197 | ~6,708 tok/s\n","step 750/3000 | loss 6.3294 | ppl 560.80 | lr 2.72e-04 | grad 1.132 | ~6,708 tok/s\n","saved: /kaggle/working/arxiv_deepseek_v3/model.pt\n","step 760/3000 | loss 6.3411 | ppl 567.42 | lr 2.71e-04 | grad 1.242 | ~6,699 tok/s\n","step 770/3000 | loss 6.2170 | ppl 501.20 | lr 2.70e-04 | grad 1.166 | ~6,699 tok/s\n","step 780/3000 | loss 6.2341 | ppl 509.83 | lr 2.69e-04 | grad 1.241 | ~6,698 tok/s\n","step 790/3000 | loss 6.1881 | ppl 486.91 | lr 2.68e-04 | grad 1.501 | ~6,697 tok/s\n","step 800/3000 | loss 6.2557 | ppl 520.97 | lr 2.67e-04 | grad 1.341 | ~6,696 tok/s\n","step 810/3000 | loss 6.2544 | ppl 520.31 | lr 2.66e-04 | grad 1.229 | ~6,696 tok/s\n","step 820/3000 | loss 6.2109 | ppl 498.13 | lr 2.65e-04 | grad 1.183 | ~6,695 tok/s\n","step 830/3000 | loss 6.1596 | ppl 473.22 | lr 2.64e-04 | grad 1.113 | ~6,695 tok/s\n","step 840/3000 | loss 6.2934 | ppl 540.99 | lr 2.63e-04 | grad 1.067 | ~6,694 tok/s\n","step 850/3000 | loss 6.2290 | ppl 507.26 | lr 2.62e-04 | grad 1.329 | ~6,693 tok/s\n","step 860/3000 | loss 6.2089 | ppl 497.16 | lr 2.61e-04 | grad 1.377 | ~6,691 tok/s\n","step 870/3000 | loss 6.0180 | ppl 410.77 | lr 2.60e-04 | grad 1.350 | ~6,690 tok/s\n","step 880/3000 | loss 6.1586 | ppl 472.75 | lr 2.59e-04 | grad 1.270 | ~6,688 tok/s\n","step 890/3000 | loss 6.1773 | ppl 481.68 | lr 2.58e-04 | grad 1.213 | ~6,686 tok/s\n","step 900/3000 | loss 6.2242 | ppl 504.81 | lr 2.57e-04 | grad 1.241 | ~6,685 tok/s\n","step 910/3000 | loss 6.2066 | ppl 496.01 | lr 2.55e-04 | grad 1.115 | ~6,683 tok/s\n","step 920/3000 | loss 6.1780 | ppl 482.04 | lr 2.54e-04 | grad 1.184 | ~6,682 tok/s\n","step 930/3000 | loss 6.1252 | ppl 457.23 | lr 2.53e-04 | grad 1.532 | ~6,680 tok/s\n","step 940/3000 | loss 6.0881 | ppl 440.57 | lr 2.52e-04 | grad 1.263 | ~6,678 tok/s\n","step 950/3000 | loss 5.9666 | ppl 390.19 | lr 2.51e-04 | grad 1.063 | ~6,677 tok/s\n","step 960/3000 | loss 5.9958 | ppl 401.75 | lr 2.50e-04 | grad 1.187 | ~6,677 tok/s\n","step 970/3000 | loss 6.2251 | ppl 505.25 | lr 2.49e-04 | grad 1.178 | ~6,676 tok/s\n","step 980/3000 | loss 5.9769 | ppl 394.20 | lr 2.47e-04 | grad 1.066 | ~6,675 tok/s\n","step 990/3000 | loss 6.0831 | ppl 438.37 | lr 2.46e-04 | grad 1.163 | ~6,673 tok/s\n","step 1000/3000 | loss 6.1596 | ppl 473.24 | lr 2.45e-04 | grad 1.129 | ~6,673 tok/s\n","saved: /kaggle/working/arxiv_deepseek_v3/model.pt\n","step 1010/3000 | loss 6.0497 | ppl 423.97 | lr 2.44e-04 | grad 1.217 | ~6,667 tok/s\n","step 1020/3000 | loss 6.0977 | ppl 444.82 | lr 2.43e-04 | grad 1.347 | ~6,666 tok/s\n","step 1030/3000 | loss 6.1506 | ppl 468.99 | lr 2.41e-04 | grad 1.099 | ~6,666 tok/s\n","step 1040/3000 | loss 6.1190 | ppl 454.43 | lr 2.40e-04 | grad 1.233 | ~6,666 tok/s\n","step 1050/3000 | loss 6.0286 | ppl 415.12 | lr 2.39e-04 | grad 1.102 | ~6,665 tok/s\n","step 1060/3000 | loss 6.1682 | ppl 477.31 | lr 2.38e-04 | grad 1.161 | ~6,665 tok/s\n","step 1070/3000 | loss 5.9567 | ppl 386.33 | lr 2.36e-04 | grad 1.195 | ~6,664 tok/s\n","step 1080/3000 | loss 6.1369 | ppl 462.63 | lr 2.35e-04 | grad 1.384 | ~6,664 tok/s\n","step 1090/3000 | loss 5.9757 | ppl 393.74 | lr 2.34e-04 | grad 1.233 | ~6,663 tok/s\n","step 1100/3000 | loss 6.0371 | ppl 418.67 | lr 2.33e-04 | grad 1.240 | ~6,661 tok/s\n","step 1110/3000 | loss 6.0438 | ppl 421.49 | lr 2.31e-04 | grad 1.108 | ~6,660 tok/s\n","step 1120/3000 | loss 6.1363 | ppl 462.34 | lr 2.30e-04 | grad 1.132 | ~6,659 tok/s\n","step 1130/3000 | loss 5.9254 | ppl 374.43 | lr 2.29e-04 | grad 1.207 | ~6,658 tok/s\n","step 1140/3000 | loss 5.9043 | ppl 366.62 | lr 2.27e-04 | grad 1.138 | ~6,658 tok/s\n","step 1150/3000 | loss 5.9187 | ppl 371.92 | lr 2.26e-04 | grad 1.130 | ~6,658 tok/s\n","step 1160/3000 | loss 5.8558 | ppl 349.24 | lr 2.25e-04 | grad 1.096 | ~6,658 tok/s\n","step 1170/3000 | loss 6.0014 | ppl 404.01 | lr 2.23e-04 | grad 1.180 | ~6,658 tok/s\n","step 1180/3000 | loss 5.8507 | ppl 347.48 | lr 2.22e-04 | grad 1.098 | ~6,658 tok/s\n","step 1190/3000 | loss 5.8144 | ppl 335.08 | lr 2.21e-04 | grad 1.192 | ~6,658 tok/s\n","step 1200/3000 | loss 5.9614 | ppl 388.14 | lr 2.19e-04 | grad 1.100 | ~6,658 tok/s\n","step 1210/3000 | loss 5.9792 | ppl 395.14 | lr 2.18e-04 | grad 1.103 | ~6,658 tok/s\n","step 1220/3000 | loss 5.9390 | ppl 379.54 | lr 2.17e-04 | grad 1.076 | ~6,658 tok/s\n","step 1230/3000 | loss 5.9117 | ppl 369.33 | lr 2.15e-04 | grad 1.235 | ~6,658 tok/s\n","step 1240/3000 | loss 5.9350 | ppl 378.05 | lr 2.14e-04 | grad 1.225 | ~6,659 tok/s\n","step 1250/3000 | loss 5.8545 | ppl 348.81 | lr 2.12e-04 | grad 1.133 | ~6,659 tok/s\n","saved: /kaggle/working/arxiv_deepseek_v3/model.pt\n","step 1260/3000 | loss 5.9045 | ppl 366.68 | lr 2.11e-04 | grad 1.101 | ~6,654 tok/s\n","step 1270/3000 | loss 5.8829 | ppl 358.84 | lr 2.10e-04 | grad 1.185 | ~6,654 tok/s\n","step 1280/3000 | loss 5.8733 | ppl 355.41 | lr 2.08e-04 | grad 1.090 | ~6,654 tok/s\n","step 1290/3000 | loss 5.8289 | ppl 339.98 | lr 2.07e-04 | grad 1.254 | ~6,654 tok/s\n","step 1300/3000 | loss 5.8483 | ppl 346.63 | lr 2.05e-04 | grad 1.113 | ~6,654 tok/s\n","step 1310/3000 | loss 5.7959 | ppl 328.95 | lr 2.04e-04 | grad 1.173 | ~6,654 tok/s\n","step 1320/3000 | loss 5.7627 | ppl 318.20 | lr 2.03e-04 | grad 1.263 | ~6,654 tok/s\n","step 1330/3000 | loss 5.8646 | ppl 352.34 | lr 2.01e-04 | grad 1.048 | ~6,654 tok/s\n","step 1340/3000 | loss 5.8821 | ppl 358.56 | lr 2.00e-04 | grad 1.257 | ~6,654 tok/s\n","step 1350/3000 | loss 5.9555 | ppl 385.86 | lr 1.98e-04 | grad 1.284 | ~6,654 tok/s\n","step 1360/3000 | loss 5.8348 | ppl 341.99 | lr 1.97e-04 | grad 1.113 | ~6,654 tok/s\n","step 1370/3000 | loss 5.8825 | ppl 358.70 | lr 1.95e-04 | grad 1.048 | ~6,654 tok/s\n","step 1380/3000 | loss 5.6409 | ppl 281.70 | lr 1.94e-04 | grad 1.183 | ~6,654 tok/s\n","step 1390/3000 | loss 5.7605 | ppl 317.50 | lr 1.92e-04 | grad 1.277 | ~6,654 tok/s\n","step 1400/3000 | loss 5.9464 | ppl 382.38 | lr 1.91e-04 | grad 1.071 | ~6,654 tok/s\n","step 1410/3000 | loss 5.8017 | ppl 330.86 | lr 1.90e-04 | grad 1.152 | ~6,654 tok/s\n","step 1420/3000 | loss 5.6980 | ppl 298.26 | lr 1.88e-04 | grad 1.114 | ~6,654 tok/s\n","step 1430/3000 | loss 5.7453 | ppl 312.73 | lr 1.87e-04 | grad 1.223 | ~6,654 tok/s\n","step 1440/3000 | loss 5.8510 | ppl 347.57 | lr 1.85e-04 | grad 1.078 | ~6,654 tok/s\n","step 1450/3000 | loss 5.8443 | ppl 345.25 | lr 1.84e-04 | grad 1.172 | ~6,654 tok/s\n","step 1460/3000 | loss 6.0460 | ppl 422.43 | lr 1.82e-04 | grad 1.163 | ~6,654 tok/s\n","step 1470/3000 | loss 5.7834 | ppl 324.87 | lr 1.81e-04 | grad 1.144 | ~6,654 tok/s\n","step 1480/3000 | loss 5.7060 | ppl 300.65 | lr 1.79e-04 | grad 1.128 | ~6,654 tok/s\n","step 1490/3000 | loss 5.8565 | ppl 349.50 | lr 1.78e-04 | grad 1.119 | ~6,654 tok/s\n","step 1500/3000 | loss 5.6485 | ppl 283.88 | lr 1.76e-04 | grad 1.114 | ~6,654 tok/s\n","saved: /kaggle/working/arxiv_deepseek_v3/model.pt\n","step 1510/3000 | loss 5.7114 | ppl 302.30 | lr 1.75e-04 | grad 1.127 | ~6,649 tok/s\n","step 1520/3000 | loss 5.8942 | ppl 362.93 | lr 1.73e-04 | grad 1.181 | ~6,649 tok/s\n","step 1530/3000 | loss 5.7875 | ppl 326.19 | lr 1.72e-04 | grad 1.079 | ~6,649 tok/s\n","step 1540/3000 | loss 5.6268 | ppl 277.78 | lr 1.70e-04 | grad 1.139 | ~6,649 tok/s\n","step 1550/3000 | loss 5.7625 | ppl 318.13 | lr 1.69e-04 | grad 1.160 | ~6,649 tok/s\n","step 1560/3000 | loss 5.8814 | ppl 358.30 | lr 1.67e-04 | grad 1.085 | ~6,649 tok/s\n","step 1570/3000 | loss 6.0495 | ppl 423.89 | lr 1.66e-04 | grad 1.177 | ~6,649 tok/s\n","step 1580/3000 | loss 5.5570 | ppl 259.05 | lr 1.64e-04 | grad 1.138 | ~6,649 tok/s\n","step 1590/3000 | loss 6.0528 | ppl 425.30 | lr 1.63e-04 | grad 1.208 | ~6,649 tok/s\n","step 1600/3000 | loss 5.5548 | ppl 258.48 | lr 1.61e-04 | grad 1.179 | ~6,649 tok/s\n","step 1610/3000 | loss 5.6675 | ppl 289.30 | lr 1.60e-04 | grad 1.056 | ~6,649 tok/s\n","step 1620/3000 | loss 6.1958 | ppl 490.69 | lr 1.58e-04 | grad 1.304 | ~6,649 tok/s\n","step 1630/3000 | loss 5.6584 | ppl 286.70 | lr 1.57e-04 | grad 1.110 | ~6,649 tok/s\n","step 1640/3000 | loss 5.6982 | ppl 298.34 | lr 1.55e-04 | grad 1.221 | ~6,649 tok/s\n","step 1650/3000 | loss 5.6893 | ppl 295.69 | lr 1.54e-04 | grad 1.109 | ~6,649 tok/s\n","step 1660/3000 | loss 5.5665 | ppl 261.51 | lr 1.53e-04 | grad 1.254 | ~6,649 tok/s\n","step 1670/3000 | loss 5.6063 | ppl 272.15 | lr 1.51e-04 | grad 1.195 | ~6,649 tok/s\n","step 1680/3000 | loss 5.6737 | ppl 291.12 | lr 1.50e-04 | grad 1.120 | ~6,648 tok/s\n","step 1690/3000 | loss 5.5944 | ppl 268.91 | lr 1.48e-04 | grad 1.128 | ~6,648 tok/s\n","step 1700/3000 | loss 5.7267 | ppl 306.95 | lr 1.47e-04 | grad 1.222 | ~6,648 tok/s\n","step 1710/3000 | loss 5.7528 | ppl 315.07 | lr 1.45e-04 | grad 1.145 | ~6,648 tok/s\n","step 1720/3000 | loss 5.5061 | ppl 246.18 | lr 1.44e-04 | grad 1.106 | ~6,648 tok/s\n","step 1730/3000 | loss 5.6459 | ppl 283.12 | lr 1.42e-04 | grad 1.085 | ~6,648 tok/s\n","step 1740/3000 | loss 5.7193 | ppl 304.69 | lr 1.41e-04 | grad 1.271 | ~6,648 tok/s\n","step 1750/3000 | loss 5.8168 | ppl 335.88 | lr 1.39e-04 | grad 1.072 | ~6,648 tok/s\n","saved: /kaggle/working/arxiv_deepseek_v3/model.pt\n","step 1760/3000 | loss 5.6225 | ppl 276.57 | lr 1.38e-04 | grad 1.155 | ~6,644 tok/s\n","step 1770/3000 | loss 5.6449 | ppl 282.83 | lr 1.36e-04 | grad 1.201 | ~6,644 tok/s\n","step 1780/3000 | loss 5.7061 | ppl 300.69 | lr 1.35e-04 | grad 1.128 | ~6,644 tok/s\n","step 1790/3000 | loss 5.6970 | ppl 297.96 | lr 1.33e-04 | grad 1.207 | ~6,644 tok/s\n","step 1800/3000 | loss 5.7161 | ppl 303.71 | lr 1.32e-04 | grad 1.167 | ~6,644 tok/s\n","step 1810/3000 | loss 5.5561 | ppl 258.80 | lr 1.31e-04 | grad 1.107 | ~6,644 tok/s\n","step 1820/3000 | loss 5.8004 | ppl 330.42 | lr 1.29e-04 | grad 1.164 | ~6,645 tok/s\n","step 1830/3000 | loss 5.6464 | ppl 283.28 | lr 1.28e-04 | grad 1.315 | ~6,645 tok/s\n","step 1840/3000 | loss 5.5052 | ppl 245.96 | lr 1.26e-04 | grad 1.335 | ~6,645 tok/s\n","step 1850/3000 | loss 5.6940 | ppl 297.09 | lr 1.25e-04 | grad 1.212 | ~6,645 tok/s\n","step 1860/3000 | loss 5.4939 | ppl 243.21 | lr 1.23e-04 | grad 1.250 | ~6,645 tok/s\n","step 1870/3000 | loss 5.5044 | ppl 245.78 | lr 1.22e-04 | grad 1.272 | ~6,645 tok/s\n","step 1880/3000 | loss 5.7341 | ppl 309.22 | lr 1.21e-04 | grad 1.152 | ~6,646 tok/s\n","step 1890/3000 | loss 5.6768 | ppl 292.01 | lr 1.19e-04 | grad 1.111 | ~6,646 tok/s\n","step 1900/3000 | loss 5.7370 | ppl 310.14 | lr 1.18e-04 | grad 1.143 | ~6,646 tok/s\n","step 1910/3000 | loss 5.8363 | ppl 342.51 | lr 1.16e-04 | grad 1.183 | ~6,647 tok/s\n","step 1920/3000 | loss 5.5762 | ppl 264.08 | lr 1.15e-04 | grad 1.222 | ~6,647 tok/s\n","step 1930/3000 | loss 5.4621 | ppl 235.59 | lr 1.14e-04 | grad 1.176 | ~6,648 tok/s\n","step 1940/3000 | loss 5.6545 | ppl 285.58 | lr 1.12e-04 | grad 1.239 | ~6,648 tok/s\n","step 1950/3000 | loss 5.7086 | ppl 301.44 | lr 1.11e-04 | grad 1.135 | ~6,648 tok/s\n","step 1960/3000 | loss 5.6069 | ppl 272.29 | lr 1.10e-04 | grad 1.220 | ~6,648 tok/s\n","step 1970/3000 | loss 5.5838 | ppl 266.09 | lr 1.08e-04 | grad 1.223 | ~6,649 tok/s\n","step 1980/3000 | loss 5.8805 | ppl 358.01 | lr 1.07e-04 | grad 1.183 | ~6,649 tok/s\n","step 1990/3000 | loss 5.5730 | ppl 263.22 | lr 1.06e-04 | grad 1.183 | ~6,649 tok/s\n","step 2000/3000 | loss 5.6335 | ppl 279.63 | lr 1.04e-04 | grad 1.175 | ~6,649 tok/s\n","saved: /kaggle/working/arxiv_deepseek_v3/model.pt\n","step 2010/3000 | loss 5.5325 | ppl 252.78 | lr 1.03e-04 | grad 1.179 | ~6,645 tok/s\n","step 2020/3000 | loss 5.7508 | ppl 314.45 | lr 1.02e-04 | grad 1.133 | ~6,645 tok/s\n","step 2030/3000 | loss 5.5753 | ppl 263.83 | lr 1.00e-04 | grad 1.167 | ~6,645 tok/s\n","step 2040/3000 | loss 5.8067 | ppl 332.53 | lr 9.89e-05 | grad 1.156 | ~6,645 tok/s\n","step 2050/3000 | loss 5.5273 | ppl 251.46 | lr 9.76e-05 | grad 1.175 | ~6,646 tok/s\n","step 2060/3000 | loss 5.4955 | ppl 243.59 | lr 9.63e-05 | grad 1.150 | ~6,646 tok/s\n","step 2070/3000 | loss 5.8467 | ppl 346.10 | lr 9.51e-05 | grad 1.176 | ~6,646 tok/s\n","step 2080/3000 | loss 5.4880 | ppl 241.77 | lr 9.38e-05 | grad 1.230 | ~6,646 tok/s\n","step 2090/3000 | loss 5.6720 | ppl 290.63 | lr 9.25e-05 | grad 1.201 | ~6,646 tok/s\n","step 2100/3000 | loss 5.6712 | ppl 290.38 | lr 9.13e-05 | grad 1.154 | ~6,646 tok/s\n","step 2110/3000 | loss 5.4292 | ppl 227.97 | lr 9.00e-05 | grad 1.222 | ~6,646 tok/s\n","step 2120/3000 | loss 5.6699 | ppl 290.01 | lr 8.88e-05 | grad 1.189 | ~6,646 tok/s\n","step 2130/3000 | loss 5.5298 | ppl 252.10 | lr 8.76e-05 | grad 1.151 | ~6,646 tok/s\n","step 2140/3000 | loss 5.4532 | ppl 233.50 | lr 8.64e-05 | grad 1.186 | ~6,646 tok/s\n","step 2150/3000 | loss 5.7031 | ppl 299.79 | lr 8.52e-05 | grad 1.270 | ~6,646 tok/s\n","step 2160/3000 | loss 5.5276 | ppl 251.54 | lr 8.40e-05 | grad 1.148 | ~6,646 tok/s\n","step 2170/3000 | loss 5.7070 | ppl 300.97 | lr 8.28e-05 | grad 1.233 | ~6,646 tok/s\n","step 2180/3000 | loss 5.7117 | ppl 302.39 | lr 8.16e-05 | grad 1.153 | ~6,646 tok/s\n","step 2190/3000 | loss 5.7107 | ppl 302.09 | lr 8.04e-05 | grad 1.236 | ~6,646 tok/s\n","step 2200/3000 | loss 5.6536 | ppl 285.32 | lr 7.93e-05 | grad 1.207 | ~6,646 tok/s\n","step 2210/3000 | loss 5.5917 | ppl 268.19 | lr 7.81e-05 | grad 1.167 | ~6,645 tok/s\n","step 2220/3000 | loss 5.6408 | ppl 281.68 | lr 7.70e-05 | grad 1.291 | ~6,645 tok/s\n","step 2230/3000 | loss 5.5456 | ppl 256.10 | lr 7.59e-05 | grad 1.207 | ~6,645 tok/s\n","step 2240/3000 | loss 5.4620 | ppl 235.56 | lr 7.48e-05 | grad 1.154 | ~6,645 tok/s\n","step 2250/3000 | loss 5.6015 | ppl 270.84 | lr 7.37e-05 | grad 1.184 | ~6,645 tok/s\n","saved: /kaggle/working/arxiv_deepseek_v3/model.pt\n","step 2260/3000 | loss 5.5472 | ppl 256.51 | lr 7.26e-05 | grad 1.150 | ~6,642 tok/s\n","step 2270/3000 | loss 5.5462 | ppl 256.27 | lr 7.15e-05 | grad 1.176 | ~6,642 tok/s\n","step 2280/3000 | loss 5.6084 | ppl 272.72 | lr 7.04e-05 | grad 1.282 | ~6,641 tok/s\n","step 2290/3000 | loss 5.4918 | ppl 242.68 | lr 6.94e-05 | grad 1.225 | ~6,641 tok/s\n","step 2300/3000 | loss 5.7022 | ppl 299.52 | lr 6.83e-05 | grad 1.151 | ~6,640 tok/s\n","step 2310/3000 | loss 5.6940 | ppl 297.09 | lr 6.73e-05 | grad 1.201 | ~6,640 tok/s\n","step 2320/3000 | loss 5.4859 | ppl 241.27 | lr 6.63e-05 | grad 1.251 | ~6,640 tok/s\n","step 2330/3000 | loss 5.6592 | ppl 286.91 | lr 6.53e-05 | grad 1.191 | ~6,639 tok/s\n","step 2340/3000 | loss 5.5106 | ppl 247.29 | lr 6.43e-05 | grad 1.216 | ~6,639 tok/s\n","step 2350/3000 | loss 5.5447 | ppl 255.87 | lr 6.33e-05 | grad 1.122 | ~6,639 tok/s\n","step 2360/3000 | loss 5.5587 | ppl 259.47 | lr 6.23e-05 | grad 1.213 | ~6,639 tok/s\n","step 2370/3000 | loss 5.6986 | ppl 298.46 | lr 6.14e-05 | grad 1.142 | ~6,639 tok/s\n","step 2380/3000 | loss 5.4915 | ppl 242.62 | lr 6.04e-05 | grad 1.285 | ~6,639 tok/s\n","step 2390/3000 | loss 5.4914 | ppl 242.60 | lr 5.95e-05 | grad 1.184 | ~6,639 tok/s\n","step 2400/3000 | loss 5.4512 | ppl 233.04 | lr 5.86e-05 | grad 1.071 | ~6,638 tok/s\n","step 2410/3000 | loss 5.4972 | ppl 244.00 | lr 5.76e-05 | grad 1.220 | ~6,638 tok/s\n","step 2420/3000 | loss 5.5554 | ppl 258.63 | lr 5.68e-05 | grad 1.205 | ~6,638 tok/s\n","step 2430/3000 | loss 5.5720 | ppl 262.95 | lr 5.59e-05 | grad 1.209 | ~6,638 tok/s\n","step 2440/3000 | loss 5.4827 | ppl 240.51 | lr 5.50e-05 | grad 1.216 | ~6,638 tok/s\n","step 2450/3000 | loss 5.4213 | ppl 226.17 | lr 5.41e-05 | grad 1.212 | ~6,638 tok/s\n","step 2460/3000 | loss 5.4808 | ppl 240.03 | lr 5.33e-05 | grad 1.234 | ~6,638 tok/s\n","step 2470/3000 | loss 5.6078 | ppl 272.54 | lr 5.25e-05 | grad 1.160 | ~6,638 tok/s\n","step 2480/3000 | loss 5.4667 | ppl 236.68 | lr 5.17e-05 | grad 1.222 | ~6,638 tok/s\n","step 2490/3000 | loss 5.4458 | ppl 231.78 | lr 5.09e-05 | grad 1.194 | ~6,637 tok/s\n","step 2500/3000 | loss 5.5292 | ppl 251.95 | lr 5.01e-05 | grad 1.255 | ~6,638 tok/s\n","saved: /kaggle/working/arxiv_deepseek_v3/model.pt\n","step 2510/3000 | loss 5.6539 | ppl 285.42 | lr 4.93e-05 | grad 1.240 | ~6,635 tok/s\n","step 2520/3000 | loss 5.5083 | ppl 246.72 | lr 4.85e-05 | grad 1.270 | ~6,635 tok/s\n","step 2530/3000 | loss 5.4778 | ppl 239.33 | lr 4.78e-05 | grad 1.213 | ~6,635 tok/s\n","step 2540/3000 | loss 5.5789 | ppl 264.78 | lr 4.71e-05 | grad 1.184 | ~6,634 tok/s\n","step 2550/3000 | loss 5.7843 | ppl 325.17 | lr 4.63e-05 | grad 1.400 | ~6,634 tok/s\n","step 2560/3000 | loss 5.5379 | ppl 254.14 | lr 4.56e-05 | grad 1.179 | ~6,634 tok/s\n","step 2570/3000 | loss 5.4020 | ppl 221.85 | lr 4.50e-05 | grad 1.260 | ~6,635 tok/s\n","step 2580/3000 | loss 5.5811 | ppl 265.37 | lr 4.43e-05 | grad 1.207 | ~6,634 tok/s\n","step 2590/3000 | loss 5.6406 | ppl 281.63 | lr 4.36e-05 | grad 1.357 | ~6,634 tok/s\n","step 2600/3000 | loss 5.7351 | ppl 309.55 | lr 4.30e-05 | grad 1.336 | ~6,634 tok/s\n","step 2610/3000 | loss 5.4842 | ppl 240.86 | lr 4.23e-05 | grad 1.270 | ~6,634 tok/s\n","step 2620/3000 | loss 5.4888 | ppl 241.97 | lr 4.17e-05 | grad 1.250 | ~6,634 tok/s\n","step 2630/3000 | loss 5.4768 | ppl 239.09 | lr 4.11e-05 | grad 1.267 | ~6,634 tok/s\n","step 2640/3000 | loss 5.5501 | ppl 257.26 | lr 4.05e-05 | grad 1.259 | ~6,634 tok/s\n","step 2650/3000 | loss 5.5204 | ppl 249.73 | lr 4.00e-05 | grad 1.172 | ~6,634 tok/s\n","step 2660/3000 | loss 5.5492 | ppl 257.02 | lr 3.94e-05 | grad 1.243 | ~6,634 tok/s\n","step 2670/3000 | loss 5.5542 | ppl 258.32 | lr 3.89e-05 | grad 1.237 | ~6,634 tok/s\n","step 2680/3000 | loss 5.5924 | ppl 268.39 | lr 3.84e-05 | grad 1.362 | ~6,634 tok/s\n","step 2690/3000 | loss 5.6925 | ppl 296.63 | lr 3.79e-05 | grad 1.264 | ~6,634 tok/s\n","step 2700/3000 | loss 5.4128 | ppl 224.26 | lr 3.74e-05 | grad 1.242 | ~6,634 tok/s\n","step 2710/3000 | loss 5.7116 | ppl 302.36 | lr 3.69e-05 | grad 1.206 | ~6,634 tok/s\n","step 2720/3000 | loss 5.4440 | ppl 231.36 | lr 3.64e-05 | grad 1.196 | ~6,634 tok/s\n","step 2730/3000 | loss 5.4478 | ppl 232.25 | lr 3.60e-05 | grad 1.223 | ~6,634 tok/s\n","step 2740/3000 | loss 5.7168 | ppl 303.94 | lr 3.55e-05 | grad 1.247 | ~6,634 tok/s\n","step 2750/3000 | loss 5.5220 | ppl 250.12 | lr 3.51e-05 | grad 1.262 | ~6,634 tok/s\n","saved: /kaggle/working/arxiv_deepseek_v3/model.pt\n","step 2760/3000 | loss 5.6485 | ppl 283.86 | lr 3.47e-05 | grad 1.251 | ~6,632 tok/s\n","step 2770/3000 | loss 5.6437 | ppl 282.50 | lr 3.44e-05 | grad 1.239 | ~6,632 tok/s\n","step 2780/3000 | loss 5.3976 | ppl 220.88 | lr 3.40e-05 | grad 1.270 | ~6,632 tok/s\n","step 2790/3000 | loss 5.4855 | ppl 241.17 | lr 3.36e-05 | grad 1.253 | ~6,632 tok/s\n","step 2800/3000 | loss 5.4087 | ppl 223.35 | lr 3.33e-05 | grad 1.238 | ~6,632 tok/s\n","step 2810/3000 | loss 5.5288 | ppl 251.84 | lr 3.30e-05 | grad 1.238 | ~6,632 tok/s\n","step 2820/3000 | loss 5.5302 | ppl 252.20 | lr 3.27e-05 | grad 1.244 | ~6,632 tok/s\n","step 2830/3000 | loss 5.7789 | ppl 323.40 | lr 3.24e-05 | grad 1.276 | ~6,632 tok/s\n","step 2840/3000 | loss 5.5855 | ppl 266.54 | lr 3.21e-05 | grad 1.204 | ~6,632 tok/s\n","step 2850/3000 | loss 5.4023 | ppl 221.92 | lr 3.19e-05 | grad 1.308 | ~6,631 tok/s\n","step 2860/3000 | loss 5.3907 | ppl 219.35 | lr 3.16e-05 | grad 1.175 | ~6,631 tok/s\n","step 2870/3000 | loss 5.3404 | ppl 208.60 | lr 3.14e-05 | grad 1.264 | ~6,631 tok/s\n","step 2880/3000 | loss 5.5792 | ppl 264.87 | lr 3.12e-05 | grad 1.229 | ~6,631 tok/s\n","step 2890/3000 | loss 5.5227 | ppl 250.31 | lr 3.10e-05 | grad 1.211 | ~6,631 tok/s\n","step 2900/3000 | loss 5.5317 | ppl 252.58 | lr 3.08e-05 | grad 1.183 | ~6,631 tok/s\n","step 2910/3000 | loss 5.4105 | ppl 223.75 | lr 3.07e-05 | grad 1.212 | ~6,631 tok/s\n","step 2920/3000 | loss 5.5058 | ppl 246.12 | lr 3.05e-05 | grad 1.254 | ~6,631 tok/s\n","step 2930/3000 | loss 5.8188 | ppl 336.57 | lr 3.04e-05 | grad 1.301 | ~6,631 tok/s\n","step 2940/3000 | loss 5.5108 | ppl 247.36 | lr 3.03e-05 | grad 1.239 | ~6,631 tok/s\n","step 2950/3000 | loss 5.5735 | ppl 263.36 | lr 3.02e-05 | grad 1.291 | ~6,631 tok/s\n","step 2960/3000 | loss 5.5097 | ppl 247.07 | lr 3.01e-05 | grad 1.255 | ~6,631 tok/s\n","step 2970/3000 | loss 5.4979 | ppl 244.18 | lr 3.01e-05 | grad 1.310 | ~6,631 tok/s\n","step 2980/3000 | loss 5.5305 | ppl 252.27 | lr 3.00e-05 | grad 1.277 | ~6,631 tok/s\n","step 2990/3000 | loss 5.4396 | ppl 230.35 | lr 3.00e-05 | grad 1.223 | ~6,631 tok/s\n","step 3000/3000 | loss 5.4824 | ppl 240.41 | lr 3.00e-05 | grad 1.298 | ~6,630 tok/s\n","saved: /kaggle/working/arxiv_deepseek_v3/model.pt\n"]}],"source":["\n","# 12. PRETRAIN\n","model.train()\n","optimizer.zero_grad(set_to_none=True)\n","\n","train_iter = iter(loader)\n","global_step = 0\n","micro_step = 0\n","running_loss = 0.0\n","steps = []\n","losses = []\n","start = time.time()\n","\n","while global_step < MAX_TRAIN_STEPS:\n"," try:\n"," batch = next(train_iter)\n"," except StopIteration:\n"," train_iter = iter(loader)\n"," batch = next(train_iter)\n","\n"," batch = {\n"," k: v.to(DEVICE, non_blocking=True)\n"," for k, v in batch.items()\n"," }\n","\n"," with torch.autocast(\n"," device_type=DEVICE.type,\n"," dtype=amp_dtype if DEVICE.type == \"cuda\" else torch.float32,\n"," enabled=DEVICE.type == \"cuda\",\n"," ):\n"," out = model(**batch)\n"," loss = out.loss\n"," main_loss_value = float(model._last_main_loss) if model._last_main_loss is not None else float('nan')\n"," mtp_loss_value = float(model._last_mtp_loss) if model._last_mtp_loss is not None else 0.0\n"," backward_loss = loss / GRAD_ACCUM_STEPS\n","\n"," if use_scaler:\n"," scaler.scale(backward_loss).backward()\n"," else:\n"," backward_loss.backward()\n","\n"," running_loss += float(loss.detach())\n"," micro_step += 1\n","\n"," if micro_step % GRAD_ACCUM_STEPS:\n"," continue\n","\n"," if use_scaler:\n"," scaler.unscale_(optimizer)\n","\n"," grad_norm = torch.nn.utils.clip_grad_norm_(\n"," model.parameters(),\n"," GRAD_CLIP,\n"," )\n","\n"," lr = lr_for_step(global_step)\n"," for group in optimizer.param_groups:\n"," group[\"lr\"] = lr\n","\n"," if use_scaler:\n"," scaler.step(optimizer)\n"," scaler.update()\n"," else:\n"," optimizer.step()\n","\n"," optimizer.zero_grad(set_to_none=True)\n","\n"," global_step += 1\n"," mean_loss = running_loss / GRAD_ACCUM_STEPS\n"," running_loss = 0.0\n","\n"," steps.append(global_step)\n"," losses.append(mean_loss)\n","\n"," if global_step == 1 or global_step % LOG_EVERY == 0:\n"," elapsed = time.time() - start\n"," tokens_done = global_step * BATCH_SIZE * SEQ_LEN * GRAD_ACCUM_STEPS\n"," tok_s = tokens_done / max(elapsed, 1e-9)\n","\n"," print(\n"," f\"step {global_step:5d}/{MAX_TRAIN_STEPS} | \"\n"," f\"loss {mean_loss:.4f} | \"\n"," f\"ppl {math.exp(min(mean_loss, 20)):.2f} | \"\n"," f\"lr {lr:.2e} | \"\n"," f\"grad {float(grad_norm):.3f} | \"\n"," f\"~{tok_s:,.0f} tok/s\"\n"," )\n","\n"," if global_step % SAVE_EVERY == 0 or global_step == MAX_TRAIN_STEPS:\n"," torch.save(\n"," {\n"," \"model_state_dict\": model.state_dict(),\n"," \"optimizer_state_dict\": optimizer.state_dict(),\n"," \"config\": config.to_dict(),\n"," \"step\": global_step,\n"," \"losses\": losses,\n"," },\n"," CHECKPOINT_PATH,\n"," )\n"," print(\"saved:\", CHECKPOINT_PATH)\n"]},{"cell_type":"code","execution_count":13,"id":"b7f49b08","metadata":{"execution":{"iopub.execute_input":"2026-08-10T20:14:32.013693Z","iopub.status.busy":"2026-08-10T20:14:32.012989Z","iopub.status.idle":"2026-08-10T20:14:32.502527Z","shell.execute_reply":"2026-08-10T20:14:32.501838Z"},"papermill":{"duration":0.508819,"end_time":"2026-08-10T20:14:32.504931+00:00","exception":false,"start_time":"2026-08-10T20:14:31.996112+00:00","status":"completed"},"tags":[]},"outputs":[{"data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAA90AAAHqCAYAAAAZLi26AAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjAsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvlHJYcgAAAAlwSFlzAAAPYQAAD2EBqD+naQAAuq5JREFUeJzs3Xd4U2UbBvA7XenehdJSVhllW4bsKVOGgyEiCuIGRcTJpyBDZaiICIIooiwFZaPsvWSXXSjQFtpCd5vuNjnn+6NtmjRJm7RJk7T377q4yNlPkjdpnvMuiSiKIoiIiIiIiIjI6GzMHQARERERERFRdcWkm4iIiIiIiMhEmHQTERERERERmQiTbiIiIiIiIiITYdJNREREREREZCJMuomIiIiIiIhMhEk3ERERERERkYkw6SYiIiIiIiIyESbdRERERERERCbCpJuIyMJMmDABrq6u5g5DL2vXrkVISAjs7e3h6elp0LFHjhyBRCLBkSNHlOsmTJiABg0aGDXGioqKioJEIsE333xT7r6zZs2CRCKpgqiITMuSPoOWpPj74LfffqvQ8RKJBLNmzTJqTERkPZh0E1GV+O233yCRSJT/HB0dERAQgIEDB2LJkiXIyMgwd4gaTpw4gcGDByMwMBCOjo6oV68ehg0bhg0bNpg7NA0XL16ERCLBZ599pnOfiIgISCQSTJs2DQBw7NgxDB8+HEFBQXB0dIS/vz8GDRqEkydP6nXN8PBwTJgwAcHBwfj555+xcuVKozyX6uKrr77Ctm3bTHqN7OxszJo1S+3GRXXVoEEDSCQS9OvXT+v2n3/+Wfn9cv78eWWSpM+/qKgo5U2g4n/29vZo1KgRXnrpJdy7d6+Kn63x7N+/HxKJBLNnz9bYFhkZCWdnZ4wcOdIMkRnfhg0bsHjxYnOHQUSkwc7cARBRzTJnzhw0bNgQBQUFePToEY4cOYKpU6di0aJF2LFjB9q0aWPuEAEAf/31F5577jk89thjePfdd+Hl5YXIyEgcO3YMP//8M8aOHWvuENW0a9cOISEh+OOPP/DFF19o3af4ZsG4ceMAALdv34aNjQ3efPNN+Pv7IzU1FevWrUPPnj3xzz//YNCgQWVe88iRIxAEAd9//z0aN25s3CdkZT777DN88sknauu++uorjBw5Ek8//bTJrpudna1Mpnr37m2y61gKR0dHHD58GI8ePYK/v7/atvXr18PR0RG5ubkAAD8/P6xdu1Ztn2+//RYxMTH47rvv1Nb7+fkhKioKADBlyhR07NgRBQUFuHjxIlauXIl//vkHV69eRUBAgOmenIn0798fY8eOxbx58/D888+jadOmym2TJk2Cvb09lixZolz3888/QxAEc4RaaRs2bMC1a9cwdepUo5+7fv36yMnJgb29fYWOz8nJgZ0df3YT1VT89BNRlRo8eDA6dOigXJ4+fToOHTqEoUOHYvjw4bh58yacnJzMGGGhWbNmoUWLFvjvv//g4OCgti0hIcFMUZXthRdewIwZM/Dff/+hc+fOGtv/+OMPhISEoF27dgCAV199Fa+++qraPpMmTUKjRo2wePHicpPu4tfB0Gbl5padnQ1nZ2ejntPOzq5a/aCWy+UQBEGj7FfWb7/9hpdffhmiKFbo+G7duuHcuXPYuHEj3n33XeX6mJgYHD9+HM888ww2b94MAHBxcVHeYCr2559/IjU1VWO9qh49eihrfl9++WU0bdoUU6ZMwe+//47p06dXKG5zEAQB+fn5cHR0xHfffYfdu3fjzTffxKFDhwAUvhZ79uzBkiVL1G4mVDSpNIWsrCy4uLiY5Ny5ublwcHCAjY1+jT6LW2hVVGWOJSLrx+blRGR2ffv2xYwZMxAdHY1169apbQsPD8fIkSPh7e0NR0dHdOjQATt27NA4R1paGqZOnYqgoCBIpVI0btwYCxYsUKuxUe2j+91336F+/fpwcnJCr169cO3aNbXz3b17Fx07dtSadNSqVUttWRAELF68GC1btoSjoyNq166NN954A6mpqRrH7t69Gz169ICLiwvc3NwwZMgQXL9+vdzXKCwsDH5+fujduzcyMzO17vPCCy8AgNbm7xcuXMCtW7eU++ji7OwMPz8/pKWllblfgwYN8PnnnwMorCVU7a+oq+9igwYNMGHChDLPq6/t27djyJAhCAgIgFQqRXBwMObOnQuFQqG2X+/evdGqVStcuHABPXv2hLOzM/73v/8BKPzRPWvWLDRt2hSOjo6oU6cOnn32Wdy9e1fjeitXrkRwcDCkUik6duyIc+fOqW0v3adbIpEgKysLv//+u7K5supzj42NxcSJE1G7dm1IpVK0bNkSv/76q8Z1y4oxKioKfn5+AIDZs2crr1P82vfu3Vtr7XfpPruqn4vFixcrn+eNGzcA6P8ZrAqOjo549tlnNcr4H3/8AS8vLwwcONDo1+zbty+AwqbY5vDNN9+ga9eu8PHxgZOTE9q3b4+///5bYz+JRIK3334b69evR8uWLSGVSrFnzx4Ahd9ZCxYswOHDh/H7778jLS0N7733Hjp27IjJkyernUe1fBQUFMDb2xsvv/yyxvVkMhkcHR3xwQcflBm/alzNmjWDo6Mj2rdvj2PHjqntV/wZunHjBsaOHQsvLy90795duX3dunVo3749nJyc4O3tjTFjxuDBgwfK7b1798Y///yD6Oho5Weh+HkUdx34888/8dlnnyEwMBDOzs6QyWRISUnBBx98gNatW8PV1RXu7u4YPHgwLl++rBaftj7dxeNvxMbG4umnn4arqyv8/PzwwQcfaHwXlf5eLH6+d+7cwYQJE+Dp6QkPDw+8/PLLyM7OVjs2JycHU6ZMga+vL9zc3DB8+HDExsaynziRFak+t+WJyKq9+OKL+N///od9+/bhtddeAwBcv34d3bp1Q2BgID755BO4uLhg06ZNePrpp7F582Y888wzAAprLnv16oXY2Fi88cYbqFevHk6dOoXp06fj4cOHGn381qxZg4yMDEyePBm5ubn4/vvv0bdvX1y9ehW1a9cGUNiU8ODBg4iJiUHdunXLjP2NN95Q1uBNmTIFkZGRWLp0KS5duoSTJ08qa47Wrl2L8ePHY+DAgViwYAGys7OxfPlydO/eHZcuXdI5eNG5c+cwcOBAdOjQAdu3b9fZEqBhw4bo2rUrNm3ahO+++w62trbKbcVJirZm8TKZDPn5+UhKSsKaNWtw7do1ZWKqy+LFi7FmzRps3boVy5cvh6ura5V2Dfjtt9/g6uqKadOmwdXVFYcOHcLMmTMhk8nw9ddfq+2bnJyMwYMHY8yYMRg3bhxq164NhUKBoUOH4uDBgxgzZgzeffddZGRkYP/+/bh27RqCg4OVx2/YsAEZGRl44403IJFIsHDhQjz77LO4d++ezlrBtWvX4tVXX8Xjjz+O119/HQCU54yPj0fnzp2VyYifnx92796NV155BTKZTNk0trwY+/Xrh+XLl+Ott97CM888g2effRYAKvw+rF69Grm5uXj99dchlUrh7e2t92ewKo0dOxYDBgzA3bt3la/phg0bMHLkSJPU0hbfhPHx8TH6ufXx/fffY/jw4XjhhReQn5+PP//8E6NGjcKuXbswZMgQtX0PHTqETZs24e2334avr6/ad8qrr76K33//HR988AH27t2LxMRE/Pvvv2XW9Nrb2+OZZ57Bli1b8NNPP6ndhNy2bRvy8vIwZsyYcp/D0aNHsXHjRkyZMgVSqRQ//vgjBg0ahLNnz6JVq1Zq+44aNQpNmjTBV199pWwR8eWXX2LGjBkYPXo0Xn31VSQmJuKHH35Az549cenSJXh6euLTTz9Fenq6WveB0gNSzp07Fw4ODvjggw+Ql5cHBwcH3LhxA9u2bcOoUaPQsGFDxMfH46effkKvXr1w48aNcrsUKBQKDBw4EJ06dcI333yDAwcO4Ntvv0VwcDDeeuutcl+b0aNHo2HDhpg3bx4uXryIX375RXmTpNiECROwadMmvPjii+jcuTOOHj2q8d4TkYUTiYiqwOrVq0UA4rlz53Tu4+HhIYaGhiqXn3jiCbF169Zibm6ucp0gCGLXrl3FJk2aKNfNnTtXdHFxEW/fvq12vk8++US0tbUV79+/L4qiKEZGRooARCcnJzEmJka535kzZ0QA4nvvvadct2rVKhGA6ODgIPbp00ecMWOGePz4cVGhUKhd4/jx4yIAcf369Wrr9+zZo7Y+IyND9PT0FF977TW1/R49eiR6eHiorR8/frzo4uIiiqIonjhxQnR3dxeHDBmi9jrosmzZMhGAuHfvXuU6hUIhBgYGil26dNF6zMCBA0UAyuf7xhtviDk5OeVe6/PPPxcBiImJiWrrAYiff/65xv7169cXx48fr1w+fPiwCEA8fPiwct348ePF+vXrl3vt7OxsjXVvvPGG6OzsrPY69erVSwQgrlixQm3fX3/9VQQgLlq0SOM8giCIolhSXnx8fMSUlBTl9u3bt4sAxJ07dyrXFb8WqlxcXNSeb7FXXnlFrFOnjpiUlKS2fsyYMaKHh4fyuekTY2Jios7Xu1evXmKvXr001pd+jYufp7u7u5iQkKC2r76fQX0Vfw9URP369cUhQ4aIcrlc9Pf3F+fOnSuKoijeuHFDBCAePXq03O+ZIUOG6CxfxeXx119/FRMTE8W4uDjxn3/+ERs0aCBKJJIyv7tMqXRZz8/PF1u1aiX27dtXbT0A0cbGRrx+/brOc127dk20t7cXAYhTp07Vuk/p8rF3716N8i6Kovjkk0+KjRo1Kjf+4u+W8+fPK9dFR0eLjo6O4jPPPKNcV/wZev7559WOj4qKEm1tbcUvv/xSbf3Vq1dFOzs7tfW63t/i97ZRo0Yar2dubq7G93pkZKQolUrFOXPmqK0DIK5evVq5bvz48SIAtf1EURRDQ0PF9u3ba7wOqp/T4uc7ceJEtf2eeeYZ0cfHR7l84cIFre/XhAkTdH72icjysHk5EVkMV1dX5SjmKSkpOHToEEaPHo2MjAwkJSUhKSkJycnJGDhwICIiIhAbGwugcNCzHj16wMvLS7lfUlIS+vXrB4VCodGM8emnn0ZgYKBy+fHHH0enTp3w77//KtdNnDgRe/bsQe/evXHixAnMnTsXPXr0QJMmTXDq1Cnlfn/99Rc8PDzQv39/tWu3b98erq6uOHz4MIDCEYTT0tLw/PPPq+1na2uLTp06KfdTdfjwYQwcOBBPPPEEtmzZAqlUWu5r+Nxzz8He3l6t+e3Ro0cRGxurs2n5/PnzsW/fPqxatQqdO3dGfn4+5HJ5udcyJ9Xa/uLy0aNHD2RnZyM8PFxtX6lUqtE8dvPmzfD19cU777yjce7SU38999xz8PLyUi736NEDACo0orUoiti8eTOGDRsGURTVysLAgQORnp6OixcvGhyjMYwYMULZXB0w7DOoS2pqqtpzLO4aobouKSlJozltWWxtbTF69Gj88ccfAAoHUAsKClK+L5U1ceJE+Pn5ISAgAEOGDFF2E1Adi6IqqZb11NRUpKeno0ePHspyoqpXr15o0aKFznO5u7sra6sHDBig1/X79u0LX19fbNy4US2O/fv347nnntPrHF26dEH79u2Vy/Xq1cNTTz2FvXv3ajTDfvPNN9WWt2zZAkEQMHr0aLUy4+/vjyZNmmj97tRl/PjxGi2FpFKpsrZfoVAgOTkZrq6uaNasmdbXWJvSMffo0UPv7wdtxyYnJ0MmkwGAsovApEmT1PbT9r1ARJaLzcuJyGJkZmYq+0vfuXMHoihixowZmDFjhtb9ExISEBgYiIiICFy5ckUtYSi9n6omTZpo7NO0aVNs2rRJbd3AgQMxcOBAZGdn48KFC9i4cSNWrFiBoUOHIjw8HLVq1UJERATS09M1+nmXvnZERASAkv6hpbm7u6st5+bmYsiQIWjfvj02bdqkMUhXeno6cnJylMsODg7w9vaGj48PBg4ciK1bt2LFihVwdHTEhg0bYGdnh9GjR2u99mOPPaZ8PG7cOLRr1w4TJkzQ2m/UUly/fh2fffYZDh06pPxxWiw9PV1tOTAwUKNv/t27d9GsWTO9Bj+rV6+e2nJxAq6tz355EhMTkZaWhpUrV+qcYq24zBgSozE0bNhQbdmQz6AuoaGhiI6O1lhf+rP6+eefG9Q3dezYsViyZAkuX76MDRs2YMyYMUa7ETFz5kz06NEDtra28PX1RfPmzct9D1JSUpCfn1+h63l7e5c5YN2uXbvwxRdfICwsDHl5ecr12p5v6fewtLfffhs2NjaoX78+3n//ffTr16/cJvl2dnYYMWIENmzYgLy8PEilUmzZsgUFBQV6J926vnOzs7ORmJioNhJ96ecQEREBURS1ngMwbOA3ba9P8QwMP/74IyIjI9VuAujTpcDR0VGjPHt5een9/VDW94u7uzuio6NhY2OjEXtNnzGCyNow6SYiixATE4P09HTlD4niAdA++OADnYMjqe7bv39/fPTRR1r3U50ipyKcnZ3Ro0cP9OjRA76+vpg9ezZ2796N8ePHQxAE1KpVC+vXr9d6bPGPseLns3btWo2pjgBo/KiXSqV48sknsX37duzZswdDhw5V2/7uu+/i999/Vy736tVLOVfzuHHjsGvXLuzatQvDhw/H5s2bMWDAAJ03JVQ5ODhg+PDhmD9/PnJycow6knzpGq2KSktLQ69eveDu7o45c+YgODgYjo6OuHjxIj7++GON6Y4q+xxU+8arEiswAndxbOPGjcP48eO17mOsvvESiURrjLreh9KvkyGfQV3Wr1+vdnNo3759+Prrr7F//361/Ro1alTmeUrr1KkTgoODMXXqVERGRhp1Cr/WrVvrnAtcl2effRZHjx6t0PUOHz6sc7q348ePY/jw4ejZsyd+/PFH1KlTB/b29li9erXWARPLKutbtmzBjh07sHjxYjRp0gRDhgzB119/Xe74DQAwZswY/PTTT9i9ezeefvppbNq0CSEhIWjbtq3ez1Nf2sqhRCLB7t27tX4WS/fbNuTcQOHUfjNmzMDEiRMxd+5ceHt7w8bGBlOnTtVr6jRd3w/6Mub3CxFZLibdRGQRiufTLf5xX/wj3N7evtwfwMHBwcjMzNT7h3JxrbOq27dv6xzITFVxE9OHDx8qr33gwAF069atzB+8xQM+1apVS684JRIJ1q9fj6eeegqjRo3C7t271X6Yf/TRR2rTHqk2fx4+fDjc3NywYcMG2NvbIzU1tdxRy1Xl5ORAFEVkZGRUKGH18vLSGP08Pz9f+ZpV1pEjR5CcnIwtW7agZ8+eyvWGjC4dHByMM2fOoKCgwGRTJGmrifTz84ObmxsUCoVe5bq8GMuq3fXy8tLaxFVbzbM2hnwGdenWrZvackxMDABU+Hyqnn/+eXzxxRdo3ry5WmsNc/j2228r1PIBQJmJ6+bNm+Ho6Ii9e/eqdS9ZvXq1QdfIyMjAlClT0K5dO7z99tuwtbXFiBEj8MUXX+D5558vt4a8Z8+eqFOnDjZu3Iju3bvj0KFD+PTTT/W+vq7v3OLZEsoSHBwMURTRsGHDcm+gVqS1w99//40+ffpg1apVauvT0tLg6+tr8PmMrX79+hAEAZGRkWq1/Xfu3DFjVERkKPbpJiKzO3ToEObOnYuGDRsqk8NatWqhd+/e+Omnn7Qma4mJicrHo0ePxunTp7F3716N/dLS0jT6J2/btk2tL+rZs2dx5swZDB48WLnu4MGDWmMt7vfdrFkz5bUVCgXmzp2rsa9cLlcmnwMHDoS7uzu++uorFBQUlPl8ijk4OGDLli3o2LEjhg0bhrNnzyq3tWjRAv369VP+U+0v6eTkhGeeeQb//vsvli9fDhcXFzz11FMa59c233haWho2b96MoKAgnU3myxMcHKzRj37lypVGq+kurhlSrQnKz8/Hjz/+qPc5RowYgaSkJCxdulRjm7FqmFxcXDRuPhQnO5s3b9aYpg5QLwf6xFg837i2Kd6Cg4MRHh6uds7Lly/j5MmTesVvyGfQHF599VV8/vnn+Pbbb80aBwC0b99e7fNoyD/VG2al2draQiKRqH12oqKisG3bNoPi++yzz/Dw4UP89NNPys/P999/D1tbW7z99tvlHm9jY4ORI0di586dWLt2LeRyud5NywHg9OnTav2jHzx4gO3bt2PAgAHl1hQ/++yzsLW1xezZszU+m6IoIjk5Wbns4uKi0b2kPLa2thrn/euvv8odr6CqFN+ILv399sMPP5gjHCKqINZ0E1GV2r17N8LDwyGXyxEfH49Dhw5h//79qF+/Pnbs2AFHR0flvsuWLUP37t3RunVrvPbaa2jUqBHi4+Nx+vRpxMTEKOdR/fDDD7Fjxw4MHToUEyZMQPv27ZGVlYWrV6/i77//RlRUlFqNRePGjdG9e3e89dZbyMvLw+LFi+Hj46PWPP2pp55Cw4YNMWzYMAQHByMrKwsHDhzAzp07lUkwUNis+4033sC8efMQFhaGAQMGwN7eHhEREfjrr7/w/fffY+TIkXB3d8fy5cvx4osvol27dhgzZgz8/Pxw//59/PPPP+jWrZvW5MrJyQm7du1C3759MXjwYBw9elRjih1txo0bhzVr1mDv3r144YUX4OLiorHP4MGDUbduXXTq1Am1atXC/fv3sXr1asTFxakNmmSoV199FW+++SZGjBiB/v374/Lly9i7d6/Rao26du0KLy8vjB8/HlOmTIFEIsHatWsNSpZfeuklrFmzBtOmTcPZs2fRo0cP5Xs8adIkrTcpDNW+fXscOHAAixYtQkBAABo2bIhOnTph/vz5OHz4MDp16oTXXnsNLVq0QEpKCi5evIgDBw4gJSVF7xidnJzQokULbNy4EU2bNoW3tzdatWqFVq1aYeLEiVi0aBEGDhyIV155BQkJCVixYgVatmyp0Q9eF30/g+ZQv379aj9H8ZAhQ7Bo0SIMGjQIY8eORUJCApYtW4bGjRvjypUrep3jwoULWLZsGSZPnqw2GFxgYCDmzJmDadOmYfPmzRgxYkSZ53nuuefwww8/4PPPP0fr1q3RvHlzvZ9Hq1atMHDgQLUpw4DC+eXLExwcjC+++ALTp09HVFQUnn76abi5uSEyMhJbt27F66+/rpwrvH379ti4cSOmTZuGjh07wtXVVfldrcvQoUMxZ84cvPzyy+jatSuuXr2K9evXG9zlwVTat2+PESNGYPHixUhOTlZOGXb79m0AphlUkYhMoMrHSyeiGql4Kp/ifw4ODqK/v7/Yv39/8fvvvxdlMpnW4+7evSu+9NJLor+/v2hvby8GBgaKQ4cOFf/++2+1/TIyMsTp06eLjRs3Fh0cHERfX1+xa9eu4jfffCPm5+eLolgy5cvXX38tfvvtt2JQUJAolUrFHj16iJcvX1Y73x9//CGOGTNGDA4OFp2cnERHR0exRYsW4qeffqo11pUrV4rt27cXnZycRDc3N7F169biRx99JMbFxantd/jwYXHgwIGih4eH6OjoKAYHB4sTJkxQm05HdcqwYklJSWKLFi1Ef39/MSIiotzXWy6Xi3Xq1BEBiP/++6/WfZYuXSp2795d9PX1Fe3s7EQ/Pz9x2LBh4rFjx8o9vyjqnjJMoVCIH3/8sejr6ys6OzuLAwcOFO/cuWPUKcNOnjwpdu7cWXRychIDAgLEjz76SDm1ker5evXqJbZs2VLrObKzs8VPP/1UbNiwoWhvby/6+/uLI0eOFO/evSuKonp5KQ06pv9RFR4eLvbs2VN0cnISAag99/j4eHHy5MliUFCQ8tpPPPGEuHLlSoNiFEVRPHXqlNi+fXvRwcFBI65169aJjRo1Eh0cHMTHHntM3Lt3r84pw7Q9T1HU/zOoD2NMGabP+SszZdhff/1VofhMZdWqVWKTJk1EqVQqhoSEiKtXr9Za3gCIkydPVlsnl8vFdu3aiQEBAWJ6errGueVyufjYY4+JdevWFTMyMkRR1P0ZFARBDAoKEgGIX3zxhd7xF8e1bt065fMIDQ1V+5yKou7vk2KbN28Wu3fvLrq4uIguLi5iSEiIOHnyZPHWrVvKfTIzM8WxY8eKnp6eIgDl8yjrvc3NzRXff/99sU6dOqKTk5PYrVs38fTp0xpT7umaMqz0d7Xqcyn9Omj7zij9fIvLcGRkpHJdVlaWOHnyZNHb21t0dXUVn376afHWrVsiAHH+/PlaXy8isiwSUeRIDURUM0RFRaFhw4b4+uuvlTUjRERkOhKJBJMnT9bakocqLiwsDKGhoVi3bp1BY3YQkXmwTzcRERERkYVSnQGg2OLFi2FjY6M2mCQRWS726SYiIiIislALFy7EhQsX0KdPH9jZ2WH37t3YvXs3Xn/9dQQFBZk7PCLSA5NuIiIiIiIL1bVrV+zfvx9z585FZmYm6tWrh1mzZhk0bRsRmRf7dBMRERERERGZCPt0ExEREREREZkIk24iIiIiIiIiE6n2fboFQUBcXBzc3NwgkUjMHQ4RERERERFVA6IoIiMjAwEBAbCx0V2fXe2T7ri4OI7sSERERERERCbx4MED1K1bV+f2ap90u7m5ASh8Idzd3c0cjW6CICA1NRVeXl5l3iUhAlheSH8sK2QIlhcyBMsLGYLlhQxhLeVFJpMhKChImXPqUu2T7uIm5e7u7hafdMvlcri7u1t0wSLLwPJC+mJZIUOwvJAhWF7IECwvZAhrKy/ldWO2/GdAREREREREZKWYdBMRERERERGZCJNuIiIiIiIiIhNh0k1ERERERERkIky6iYiIiIiIiEyESTcRERERERGRiTDpJiIiIiIiIjIRJt1EREREREREJsKkm4iIiIiIiMhEmHQTERERERERmQiTbiIiIiIiIiITYdJNREREREREZCJMuomIiIiIiIhMhEk3ERERERERkYkw6bYQf5y9j/D4LHOHQUREREREREZkZ+4ACEjJysesnTdQoBCx+DlbPB1a19whERERERERkRGwptsCZOXJ0a6eFwDgh0N3IFcIZo6IiIiIiIiIjMGsSfexY8cwbNgwBAQEQCKRYNu2bWrbZ82ahZCQELi4uMDLywv9+vXDmTNnzBOsCQV5O+OH5x+Do50N7iZm4VxUqrlDIiIiIiIiIiMwa9KdlZWFtm3bYtmyZVq3N23aFEuXLsXVq1dx4sQJNGjQAAMGDEBiYmIVR2p6vq5S9GlSWNt9PKL6PT8iIiIiIqKayKx9ugcPHozBgwfr3D527Fi15UWLFmHVqlW4cuUKnnjiCVOHV+U6N/DA7pvJOBaRiI8GhZg7HCIiIiIiIqokqxlILT8/HytXroSHhwfatm2rc7+8vDzk5eUpl2UyGQBAEAQIguX2lRYEAY/XdwcAXIuVIUGWA19XqZmjIkslCAJEUbToMk2WgWWFDMHyQoZgeSFDsLyQIaylvOgbn8Un3bt27cKYMWOQnZ2NOnXqYP/+/fD19dW5/7x58zB79myN9ampqZDL5aYMtVIEQYCDIhdN/JwQkZiDfZejMai57udJNZsgCMjIyIAoirCx4XiIpBvLChmC5YUMwfJChmB5IUNYS3nJyMjQaz+LT7r79OmDsLAwJCUl4eeff8bo0aNx5swZ1KpVS+v+06dPx7Rp05TLMpkMQUFB8PLygru7e1WFbTBBECCRSNC7WW1EJEbh0sNcjO3mbe6wyEIVlxcvLy+L/iIi82NZIUOwvJAhWF7IECwvZAhrKS92dvql0xafdLu4uKBx48Zo3LgxOnfujCZNmmDVqlWYPn261v2lUimkUs1m2TY2Nhb9hgGARCJBj6Z++PlEFI5HJEEikUAikZg7LLJQEonEKso1mR/LChmC5YUMwfJChmB5IUNYQ3nRNzbLfQY6CIKg1me7uulY3wuO9jZIyMjDrXj9misQERERERGRZTJrTXdmZibu3LmjXI6MjERYWBi8vb3h4+ODL7/8EsOHD0edOnWQlJSEZcuWITY2FqNGjTJj1KYltbdFp4Y+OHo7EcdvJyHE33KbxBMREREREVHZzFrTff78eYSGhiI0NBQAMG3aNISGhmLmzJmwtbVFeHg4RowYgaZNm2LYsGFITk7G8ePH0bJlS3OGbXI9m/oBAI5xvm4iIiIiIiKrZtaa7t69e0MURZ3bt2zZUoXRWI6eTQpHLT8bmYJ8uQAHO6vrBUBERERERESwwj7dNUHjWq7wdnFAnlzAyTtJ5g6HiIiIiIiIKohJtwWSSCQY3jYAALDzSpyZoyEiIiIiIqKKYtJtoQa0rA0AOHY7EYKguwk+ERERERERWS4m3RaqQ31vuDjYIikzH9fjZOYOh4iIiIiIiCqASbeFcrCzQbfGhQOqHb6VYOZoiIiIiIiIqCKYdFuwHkWjmJ+PTjVzJERERERERFQRTLotWOu6ngCA67HpZU6tRkRERERERJaJSbcFC/F3g62NBMlZ+YhNyzF3OERERERERGQgJt0WzNHeFo8FeQIAjtxKNG8wREREREREZDAm3RauSyMfAOAI5kRERERERFaISbeFa1LbFQAQEZ9h5kiIiIiIiIjIUEy6LVwzfzcAwK34DA6mRkREREREZGWYdFu4hr4usLWRICNXjnhZnrnDISIiIiIiIgMw6bZwUjtbNPR1AVBY201ERERERETWg0m3FWhWu7CJ+e1HTLqJiIiIiIisCZNuK1A8mNpt1nQTERERERFZFSbdVkBZ082km4iIiIiIyKow6bYCTYqS7oiETAgCRzAnIiIiIiKyFky6rUADH2c42NogO1+B2LQcc4dDREREREREemLSbQXsbG0QXIv9uomIiIiIiKwNk24r0bRoMDVOG0ZERERERGQ9mHRbiaZF/bovRqeaORIiIiIiIiLSF5NuK9E12AcAcORWIgdTIyIiIiIishJMuq1EywAPAIBcEJGeU2DmaIiIiIiIiEgfTLqthIOdDdwc7QAAyVn5Zo6GiIiIiIiI9MGk24r4ukoBAAkZuWaOhIiIiIiIiPTBpNuKBPsVjmB+I05m5kiIiIiIiIhIH0y6rUjbuoX9uq/EpJs5EiIiIiIiItIHk24r0ibIEwBwJSbNrHEQERERERGRfph0W5Himu6o5GzIcjmCORERERERkaVj0m1FPJ0d4CYtHME8MSPPzNEQERERERFReZh0WxlPF3sAQFo2a7qJiIiIiIgsHZNuK+Pp5AAASM/hXN1ERERERESWjkm3lfF2KUy6r8Zw2jAiIiIiIiJLx6TbyvRs6gcAuHA/1cyREBERERERUXnMmnQfO3YMw4YNQ0BAACQSCbZt26bcVlBQgI8//hitW7eGi4sLAgIC8NJLLyEuLs58AVuA5nXcAAAxKdlmjoSIiIiIiIjKY9akOysrC23btsWyZcs0tmVnZ+PixYuYMWMGLl68iC1btuDWrVsYPny4GSK1HPW8nQEAMak5UAiimaMhIiIiIiKistiZ8+KDBw/G4MGDtW7z8PDA/v371dYtXboUjz/+OO7fv4969epVRYgWp46HE+xsJMhXCIiX5SLA08ncIREREREREZEOVtWnOz09HRKJBJ6enuYOxWxsbSQI9CpMtO+ziTkREREREZFFM2tNtyFyc3Px8ccf4/nnn4e7u7vO/fLy8pCXl6dclskKR/kWBAGCIJg8zooSBAGiKOoVY5CXE6KTs3E/OQuPN/CqgujI0hhSXqhmY1khQ7C8kCFYXsgQLC9kCGspL/rGZxVJd0FBAUaPHg1RFLF8+fIy9503bx5mz56tsT41NRVyudxUIVaaIAjIyMiAKIqwsSm7AYKfsy0AICIuBSkNnasiPLIwhpQXqtlYVsgQLC9kCJYXMgTLCxnCWspLRkaGXvtZfNJdnHBHR0fj0KFDZdZyA8D06dMxbdo05bJMJkNQUBC8vLzKPdacBEGARCKBl5dXuQWrQa0UAAmQFUjg7e1dNQGSRTGkvFDNxrJChmB5IUOwvJAhWF7IENZSXuzs9EunLTrpLk64IyIicPjwYfj4+JR7jFQqhVQq1VhvY2Nj0W8YAEgkEr3i9PdwBAAkZORZ/HMi09G3vBCxrJAhWF7IECwvZAiWFzKENZQXfWMza9KdmZmJO3fuKJcjIyMRFhYGb29v1KlTByNHjsTFixexa9cuKBQKPHr0CADg7e0NBwcHc4VtdrXcC5PueFleOXsSERERERGROZk16T5//jz69OmjXC5uFj5+/HjMmjULO3bsAAA89thjascdPnwYvXv3rqowLU5tt5KabiIiIiIiIrJcZk26e/fuDVEUdW4va1tNVtu9sPl8UmYe0rLz4elcc2v9iYiIiIiILJnlNpAnnbxdHFC3aK7u4xFJZo6GiIiIiIiIdGHSbYUkEgkeC/IEACSyiTkREREREZHFYtJtpXxdC5uYJ2cx6SYiIiIiIrJUTLqtlK9rYT/uBI5gTkREREREZLGYdFupIG9nAEB0craZIyEiIiIiIiJdmHRbqYa+LgCAyOQsM0dCREREREREujDptlINipLuxIw8ZOQWmDkaIiIiIiIi0oZJt5Vyd7RX9utmE3MiIiIiIiLLxKTbihU3Mb+XxCbmRERERERElohJtxUrHkwtJpU13URERERERJaISbcV83EpbF6els0+3URERERERJaISbcV8ypKulOy8s0cCREREREREWnDpNuKeTsz6SYiIiIiIrJkTLqtmL+HIwAgNjXHzJEQERERERGRNky6rViwnysAIDIpCwpBNHM0REREREREVBqTbisW6OkEqZ0N8hUCRzAnIiIiIiKyQEy6rZiNjQSNimq77yRkmjkaIiIiIiIiKo1Jt5UL9nMBANxNZNJNRERERERkaZh0W7nift13E7LMHAkRERERERGVxqTbygXXKmpezppuIiIiIiIii8Ok28o1LqrpjojPgChyBHMiIiIiIiJLwqTbygXXcoGdjQSyXDniZXnmDoeIiIiIiIhUMOm2clI7W3i5OAAAUrLyzRwNERERERERqWLSXQ24O9oBAGS5BWaOhIiIiIiIiFQx6a4G3J3sAQCyHCbdREREREREloRJdzXg7liUdOfKzRwJERERERERqWLSXQ14F/XpTs7kQGpERERERESWhEl3NVDb3REA8EiWa+ZIiIiIiIiISBWT7mqgtrsUABDPpJuIiIiIiMiiMOmuBvyLa7rTmXQTERERERFZEibd1UBtj8KkO17GPt1ERERERESWhEl3NVBc0x0vy4UgiGaOhoiIiIiIiIox6a4G/NykkEgAuSAiOSvf3OEQERERERFRESbd1YC9rQ18XTmYGhERERERkaVh0l1N+BUl3SfuJJk5EiIiIiIiIirGpLuauPFQBgCYvzvczJEQERERERFRMbMm3ceOHcOwYcMQEBAAiUSCbdu2qW3fsmULBgwYAB8fH0gkEoSFhZklTiIiIiIiIqKKMGvSnZWVhbZt22LZsmU6t3fv3h0LFiyo4sisT8cGXsrHosgRzImIiIiIiCyBnTkvPnjwYAwePFjn9hdffBEAEBUVVUURWa9lY9vh8a8OAgCy8hVwlZr1rSUiIiIiIiKYOek2hby8POTl5SmXZbLCvs6CIEAQBHOFVS5BECCKYoVj9HV1gKO9DXILBCRn5MLZ3tnIEZIlqWx5oZqDZYUMwfJChmB5IUOwvJAhrKW86BtftUu6582bh9mzZ2usT01NhVwuN0NE+hEEARkZGRBFETY2FWv17+loh0cF+Yh8mAQXuBo5QrIkxigvVDOwrJAhWF7IECwvZAiWFzKEtZSXjIwMvfardkn39OnTMW3aNOWyTCZDUFAQvLy84O7ubsbIyiYIAiQSCby8vCpcsHzdHPEoIx8KO0d4e3sbOUKyJMYoL1QzsKyQIVheyBAsL2QIlhcyhLWUFzs7/dLpapd0S6VSSKVSjfU2NjYW/YYBgEQiqVScvm6Fzztelm/xz5Uqr7LlhWoOlhUyBMsLGYLlhQzB8kKGsIbyom9slvsMyGDNarsBAG48TDdzJERERERERASYuaY7MzMTd+7cUS5HRkYiLCwM3t7eqFevHlJSUnD//n3ExcUBAG7dugUA8Pf3h7+/v1litmR1vQsHT0vKyDdzJERERERERASYuab7/PnzCA0NRWhoKABg2rRpCA0NxcyZMwEAO3bsQGhoKIYMGQIAGDNmDEJDQ7FixQqzxWzJXBxsAQBZ+ZY7YBwREREREVFNYtaa7t69e0MURZ3bJ0yYgAkTJlRdQFbO2aHw7czOV5g5EiIiIiIiIgLYp7tacZEW1nQz6SYiIiIiIrIMTLqrkeKa7psPZcgtYOJNRERERERkbky6qxGpXcnbeSIiyYyREBEREREREcCku1ppWjRlGACkZHEEcyIiIiIiInNj0l2NONjZYGynegCAyOQsM0dDRERERERETLqrmdaBHgCAKzFp5g2EiIiIiIiImHRXN/V9nAEA8bI8M0dCRERERERETLqrGR8XKQAgOZNJNxERERERkbkx6a5mvF0cAABpOQVQCKKZoyEiIiIiIqrZmHRXM57O9gAAUQRkOQVmjoaIiIiIiKhmY9Jdzdjb2sDORgIASMhgE3MiIiIiIiJzYtJdDcmLmpW/tf6CmSMhIiIiIiKq2Zh0V2P3EjlXNxERERERkTkx6SYiIiIiIiIyESbd1VAjPxdzh0BERERERERg0l0t/Tq+IwDAzkaCtOx8M0dDRERERERUczHproYa+LqglpsUckHEg5Qcc4dDRERERERUYxmcdOfk5CA7O1u5HB0djcWLF2Pfvn1GDYwqx8vZAQAgy+Vc3UREREREROZicNL91FNPYc2aNQCAtLQ0dOrUCd9++y2eeuopLF++3OgBUsW4O9kBAGQ5TLqJiIiIiIjMxeCk++LFi+jRowcA4O+//0bt2rURHR2NNWvWYMmSJUYPkCrG3dEeAGu6iYiIiIiIzMngpDs7Oxtubm4AgH379uHZZ5+FjY0NOnfujOjoaKMHSBXj7lSYdKezppuIiIiIiMhsDE66GzdujG3btuHBgwfYu3cvBgwYAABISEiAu7u70QOkivEoSrplOXIzR0JERERERFRzGZx0z5w5Ex988AEaNGiATp06oUuXLgAKa71DQ0ONHiBVjLtjYZ/upYfvIINNzImIiIiIiMzC4KR75MiRuH//Ps6fP489e/Yo1z/xxBP47rvvjBocVVxx83IA+PVElPkCISIiIiIiqsHsKnKQv78//P39AQAymQyHDh1Cs2bNEBISYtTgqOKKB1IDgOwCNjEnIiIiIiIyB4NrukePHo2lS5cCKJyzu0OHDhg9ejTatGmDzZs3Gz1AqhgfVwflY6mtwW8zERERERERGYHB2dixY8eUU4Zt3boVoigiLS0NS5YswRdffGH0AKlifF2lyseXY9LNGAkREREREVHNZXDSnZ6eDm9vbwDAnj17MGLECDg7O2PIkCGIiIgweoBUMQGeTsrHR28nIi4tx4zREBERERER1UwGJ91BQUE4ffo0srKysGfPHuWUYampqXB0dDR6gFQxfm5S1HIrqe2OSWXSTUREREREVNUMTrqnTp2KF154AXXr1kVAQAB69+4NoLDZeevWrY0dH1VC35Baysfs1k1ERERERFT1DB69fNKkSXj88cfx4MED9O/fHzY2hdlco0aN2KfbwgiiqHwskUjMGAkREREREVHNVKEpwzp06IAOHTpAFEWIogiJRIIhQ4YYOzaqJAe7kurtxQcisGbi42aMhoiIiIiIqOapUKPjNWvWoHXr1nBycoKTkxPatGmDtWvXGjs2qiQne1vl42O3E80YCRERERERUc1kcE33okWLMGPGDLz99tvo1q0bAODEiRN48803kZSUhPfee8/oQVLFqCbdREREREREVPUMTrp/+OEHLF++HC+99JJy3fDhw9GyZUvMmjWLSbcFGdG+LpYcumPuMIiIiIiIiGosg5uXP3z4EF27dtVY37VrVzx8+NCgcx07dgzDhg1DQEAAJBIJtm3bprZdFEXMnDkTderUgZOTE/r168e5wA1Q38fF3CEQERERERHVaAYn3Y0bN8amTZs01m/cuBFNmjQx6FxZWVlo27Ytli1bpnX7woULsWTJEqxYsQJnzpyBi4sLBg4ciNzcXEPDJiIiIiIiIqpyBjcvnz17Np577jkcO3ZM2af75MmTOHjwoNZkvCyDBw/G4MGDtW4TRRGLFy/GZ599hqeeegpA4QButWvXxrZt2zBmzBhDQ6+Rejb14yBqREREREREZmJw0j1ixAicOXMG3333nbI5ePPmzXH27FmEhoYaLbDIyEg8evQI/fr1U67z8PBAp06dcPr0aZ1Jd15eHvLy8pTLMpkMACAIAgRBMFp8xiYIAkRRNHqMH/Rvoky6n1p6AlsnaXYNIOtjqvJC1Q/LChmC5YUMwfJChmB5IUNYS3nRN74KzdPdvn17rFu3riKH6u3Ro0cAgNq1a6utr127tnKbNvPmzcPs2bM11qempkIulxs3SCMSBAEZGRkQRRE2NhWayU2r3Kxs5ePLMem4ei8OgZ6ORjs/mYepygtVPywrZAiWFzIEywsZguWFDGEt5SUjI0Ov/fRKuotri/Xh7u6u976mMH36dEybNk25LJPJEBQUBC8vL7PHVhZBECCRSODl5WXUgpUmSNWWnV3d4e3tarTzk3mYqrxQ9cOyQoZgeSFDsLyQIVheyBDWUl7s7PSrw9ZrL09PT0gkkjL3EUUREokECoVCrwuXx9/fHwAQHx+POnXqKNfHx8fjscce03mcVCqFVCrVWG9jY2PRbxgASCQSo8fZ0Fc9wZbYSCz+dSD9mKK8UPXEskKGYHkhQ7C8kCFYXsgQ1lBe9I1Nr6T78OHDlQqmIho2bAh/f38cPHhQmWTLZDKcOXMGb731VpXHY61sbSR4uVsDrD4ZBQDILbDsfhFERERERETViV5Jd69evUxy8czMTNy5c0e5HBkZibCwMHh7e6NevXqYOnUqvvjiCzRp0gQNGzbEjBkzEBAQgKefftok8dQEeXLjtEQgIiIiIiKi8lVoIDVjOX/+PPr06aNcLu6LPX78ePz222/46KOPkJWVhddffx1paWno3r079uzZA0dHDgRWUazpJiIiIiIiqjpmTbp79+4NURR1bpdIJJgzZw7mzJlThVFVb7kFrOkmIiIiIiKqKpbbK52MZkALf+Vj1nQTERERERFVHSbdNUCXYB94OdsDALLyLXeuciIiIiIiourG4KT7888/R3R0tCliIRN6onltAEBiRp6ZIyEiIiIiIqo5DE66t2/fjuDgYDzxxBPYsGED8vKYxFkDf/fCwecepueYORIiIiIiIqKaw+CkOywsDOfOnUPLli3x7rvvwt/fH2+99RbOnTtnivjISPw9CpPuR+m8SUJERERERFRVKtSnOzQ0FEuWLEFcXBxWrVqFmJgYdOvWDW3atMH333+P9PR0Y8dJlVRc0x0vyzVzJERERERERDVHpQZSE0URBQUFyM/PhyiK8PLywtKlSxEUFISNGzcaK0YyguKa7qjkLE4bRkREREREVEUqlHRfuHABb7/9NurUqYP33nsPoaGhuHnzJo4ePYqIiAh8+eWXmDJlirFjpUpo5u+GAA9HZOTKMeyHE+YOh4iIiIiIqEYwOOlu3bo1OnfujMjISKxatQoPHjzA/Pnz0bhxY+U+zz//PBITE40aKFWOva0NOjb0BgBEJGRClltg5oiIiIiIiIiqPztDDxg9ejQmTpyIwMBAnfv4+vpCEIRKBUbGJ1F5XCDn+0NERERERGRqBtd0z5gxQ5lwi6IIURSNHhSZhkRSknZvvRRrxkiIiIiIiIhqhgr16V61ahVatWoFR0dHODo6olWrVvjll1+MHRsZmeoNki/+uWnGSIiIiIiIiGoGg5uXz5w5E4sWLcI777yDLl26AABOnz6N9957D/fv38ecOXOMHiQRERERERGRNTI46V6+fDl+/vlnPP/888p1w4cPR5s2bfDOO+8w6bZg7AhARERERERUtQxuXl5QUIAOHTporG/fvj3kcrlRgiLTCPZzNXcIRERERERENYrBSfeLL76I5cuXa6xfuXIlXnjhBaMERabxes9G5g6BiIiIiIioRjG4eTlQOJDavn370LlzZwDAmTNncP/+fbz00kuYNm2acr9FixYZJ0oyCkd7W3OHQEREREREVKMYnHRfu3YN7dq1AwDcvXsXQOG83L6+vrh27ZpyP9XpqchyNKvthlvxGQCAnHwFnByYiBMREREREZmKwUn34cOHTREHVZFdU7qjyae7AQCRSVloEeBu5oiIiIiIiIiqrwrN010sJiYGMTExxoqFqoC9rQ1C/N0AAAkZuWaOhoiIiIiIqHozOOkWBAFz5syBh4cH6tevj/r168PT0xNz586FIAimiJGMrJa7IwAgMSPPzJEQERERERFVbwY3L//000+xatUqzJ8/H926dQMAnDhxArNmzUJubi6+/PJLowdJxlXLTQoA+PDvK2hexx2tAj3MHBEREREREVH1ZHDS/fvvv+OXX37B8OHDlevatGmDwMBATJo0iUm3FfB1lSofj/7pNG7MGWTGaIiIiIiIiKovg5uXp6SkICQkRGN9SEgIUlJSjBIUmZZqX+7sfIUZIyEiIiIiIqreDE6627Zti6VLl2qsX7p0Kdq2bWuUoMi0XuvRyNwhEBERERER1QgGNy9fuHAhhgwZggMHDqBLly4AgNOnT+PBgwf4999/jR4gGV/zOurThEXEZ6BJbTczRUNERERERFR9GVzT3atXL9y+fRvPPPMM0tLSkJaWhmeffRa3bt1Cjx49TBEjmdj0LVfNHQIREREREVG1ZFBNd0FBAQYNGoQVK1ZwwLRq5Hx0qrlDICIiIiIiqpYMqum2t7fHlStXTBULVaFV4zuoLV+JSTNPIERERERERNWYwc3Lx40bh1WrVpkiFqpCnRv5qC0fu51opkiIiIiIiIiqL4MHUpPL5fj1119x4MABtG/fHi4uLmrbFy1aZLTgyHScHWzVlkXRTIEQERERERFVYwYn3deuXUO7du0AALdv3zZ6QFQ1JBIJejTxxfGIJAAAc24iIiIiIiLjMzjpPnz4sCniIDN4pXtDZdItsKqbiIiIiIjI6Azu0z1x4kRkZGRorM/KysLEiRONEhRVDXvbkrefOTcREREREZHxGZx0//7778jJydFYn5OTgzVr1hglKKoaEpXH3x+MwD9XHpotFiIiIiIioupI76RbJpMhPT0doigiIyMDMplM+S81NRX//vsvatWqZfQAMzIyMHXqVNSvXx9OTk7o2rUrzp07Z/TrEDB5w0Vzh0BERERERFSt6N2n29PTExKJBBKJBE2bNtXYLpFIMHv2bKMGBwCvvvoqrl27hrVr1yIgIADr1q1Dv379cOPGDQQGBhr9ejWJY6kRzImIiIiIiMi49E66Dx8+DFEU0bdvX2zevBne3t7KbQ4ODqhfvz4CAgKMGlxOTg42b96M7du3o2fPngCAWbNmYefOnVi+fDm++OILo16vpgkN8jR3CERERERERNWa3kl3r169AACRkZEICgqCjY3B3cENJpfLoVAo4OjoqLbeyckJJ06cMPn1qzuJRIKPB4VgwZ5wc4dCRERERERULRk8ZVj9+vWRlpaGs2fPIiEhAYIgqG1/6aWXjBacm5sbunTpgrlz56J58+aoXbs2/vjjD5w+fRqNGzfWekxeXh7y8vKUyzKZDAAgCIJGrJZEEASIoljlMTrZq988seTXiEqYq7yQ9WFZIUOwvJAhWF7IECwvZAhrKS/6xmdw0r1z50688MILyMzMhLu7OySSkjGwJRKJUZNuAFi7di0mTpyIwMBA2Nraol27dnj++edx4cIFrfvPmzdPa9/y1NRUyOVyo8ZmTIIgICMjA6IoVkkrgmKKfPWR6FNSUqrs2lRx5iovZH1YVsgQLC9kCJYXMgTLCxnCWsqLtqm0tZGIomEzNDdt2hRPPvkkvvrqKzg7O1couIrIysqCTCZDnTp18NxzzyEzMxP//POPxn7aarqDgoKQmpoKd3f3KovXUIIgIDU1FV5eXlVasHZcjsPUjZeVy1ve6oLH2Nfb4pmrvJD1YVkhQ7C8kCFYXsgQLC9kCGspLzKZDF5eXkhPTy8z1zS4pjs2NhZTpkyp0oQbAFxcXODi4oLU1FTs3bsXCxcu1LqfVCqFVCrVWG9jY2PRbxhQ2FKgquPMk6s3ibj+MAPt6nvr2JssiTnKC1knlhUyBMsLGYLlhQzB8kKGsIbyom9sBj+DgQMH4vz58wYHVFF79+7Fnj17EBkZif3796NPnz4ICQnByy+/XGUxVGcDW/qrLQuCQQ0fiIiIiIiIqAwG13QPGTIEH374IW7cuIHWrVvD3t5ebfvw4cONFhwApKenY/r06YiJiYG3tzdGjBiBL7/8UuO6VDGezg5YPaEjXv7tHABg0f7bGPN4EKR2nMObiIiIiIiosgxOul977TUAwJw5czS2SSQSKBSKykelYvTo0Rg9erRRz0nqpHYlDR7Scwrw1/kYjOtc34wRERERERERVQ8GJ92WPmw7Ga5zIx+15ZjUHB17EhERERERkSEq1Ss9NzfXWHGQGdnYSPBGr0bKZT83zYHoiIiIiIiIyHAGJ90KhQJz585FYGAgXF1dce/ePQDAjBkzsGrVKqMHSFUjyKtkNHoOpkZERERERGQcBifdX375JX777TcsXLgQDg4OyvWtWrXCL7/8YtTgqOqM6lBX+Tg737j98omIiIiIiGoqg5PuNWvWYOXKlXjhhRdga1sywnXbtm0RHh5u1OCo6kjtbPFq94YAgOwCuZmjISIiIiIiqh4MTrpjY2PRuHFjjfWCIKCgoMAoQZF5ODsU3kRJysg3cyRERERERETVg8FJd4sWLXD8+HGN9X///TdCQ0ONEhSZRx1PJwDA5osxkOXyBgoREREREVFlGTxl2MyZMzF+/HjExsZCEARs2bIFt27dwpo1a7Br1y5TxEhVpIvK1GFn76WgX4vaZoyGiIiIiIjI+hlc0/3UU09h586dOHDgAFxcXDBz5kzcvHkTO3fuRP/+/U0RI1WRBr4uysevrjmPfDnnZCciIiIiIqoMg2u6AaBHjx7Yv3+/sWMhC3MmMhk9mviZOwwiIiIiIiKrZXBNN1VvI9qVTB22aP9tM0ZCRERERERk/Zh0k5ovn2mlfHzpfpr5AiEiIiIiIqoGmHSTGkd72/J3IiIiIiIiIr0w6aYyiaJo7hCIiIiIiIisVqWTboVCgbCwMKSmphojHrIAG17tpHycmSc3YyRERERERETWzeCke+rUqVi1ahWAwoS7V69eaNeuHYKCgnDkyBFjx0dm0CXYB3Y2EgDAS7+eNXM0RERERERE1svgpPvvv/9G27ZtAQA7d+5EZGQkwsPD8d577+HTTz81eoBU9SQSCeRCYbNyDqZGRERERERUcQYn3UlJSfD39wcA/Pvvvxg1ahSaNm2KiRMn4urVq0YPkMzDSWVAtbuJmZArBOTLBTNGREREREREZH0MTrpr166NGzduQKFQYM+ePejfvz8AIDs7G7a2HPm6uvB2cVA+fuLbo2j86W40/Ww3Nl+IMWNURERERERE1sXgpPvll1/G6NGj0apVK0gkEvTr1w8AcObMGYSEhBg9QDKPb0e31br+/b8uV3EkRERERERE1svO0ANmzZqFVq1a4cGDBxg1ahSkUikAwNbWFp988onRAyTz6NzIx9whEBERERERWT2Dk24AGDlypNpyWloaxo8fb5SAiIiIiIiIiKoLg5uXL1iwABs3blQujx49Gj4+Pqhbty6uXLli1OCIiIiIiIiIrJnBSfeKFSsQFBQEANi/fz/279+P3bt3Y9CgQfjggw+MHiARERERERGRtTK4efmjR4+USfeuXbswevRoDBgwAA0aNECnTp2MHiARERERERGRtTK4ptvLywsPHjwAAOzZs0c5erkoilAoFMaNjizS5PUX8TenDiMiIiIiIiqXwTXdzz77LMaOHYsmTZogOTkZgwcPBgBcunQJjRs3NnqAZHn+ufoQ/1x9iJHt65o7FCIiIiIiIotmcNL93XffoUGDBnjw4AEWLlwIV1dXAMDDhw8xadIkowdIREREREREZK0MTrrt7e21Dpj23nvvGSUgshyfDWmOL/65ae4wiIiIiIiIrJbBfboB4O7du3jnnXfQr18/9OvXD1OmTMG9e/eMHRuZ2as9Gpk7BCIiIiIiIqtmcNK9d+9etGjRAmfPnkWbNm3Qpk0bnDlzBi1atMD+/ftNESMRERERERGRVTK4efknn3yC9957D/Pnz9dY//HHH6N///5GC47Mj03MiYiIiIiIKs7gmu6bN2/ilVde0Vg/ceJE3LhxwyhBkeUY17k+Qvzd0LyOu7lDISIiIiIisjoGJ91+fn4ICwvTWB8WFoZatWoZIyayII72ttgztSe2Tupq7lCIiIiIiIisjsHNy1977TW8/vrruHfvHrp2LUzETp48iQULFmDatGlGD5Asg6O9LdoGeeLygzTluutx6WgZ4GG+oIiIiIiIiCycwUn3jBkz4Obmhm+//RbTp08HAAQEBGDWrFmYMmWK0QMky/Fy1waYujFMuTxkyQlEzR9ivoCIiIiIiIgsnEHNy+VyOdauXYuxY8ciJiYG6enpSE9PR0xMDN59911IJBKjBqdQKDBjxgw0bNgQTk5OCA4Oxty5cyGKolGvQ/oZ3Nrf3CEQERERERFZFYOSbjs7O7z55pvIzc0FALi5ucHNzc0kgQHAggULsHz5cixduhQ3b97EggULsHDhQvzwww8muybpZm+jWVx+OnrXDJEQERERERFZB4MHUnv88cdx6dIlU8Si4dSpU3jqqacwZMgQNGjQACNHjsSAAQNw9uzZKrk+qbOx0WzJMG93OKKTszTWZ+QWID2noCrCIiIiIiIislgG9+meNGkS3n//fcTExKB9+/ZwcXFR296mTRujBde1a1esXLkSt2/fRtOmTXH58mWcOHECixYt0nlMXl4e8vLylMsymQwAIAgCBEEwWmzGJggCRFG06Bh1uRGXjiAvJ+WyIIhoPWsfAODmnAGQ2tmaK7Rqy5rLC1UtlhUyBMsLGYLlhQzB8kKGsJbyom98BifdY8aMAQC1QdMkEglEUYREIoFCoTD0lDp98sknkMlkCAkJga2tLRQKBb788ku88MILOo+ZN28eZs+erbE+NTUVcrncaLEZmyAIyMjIgCiKsNHSjNuSvbX+En4Y0QxdGnoCAHLyS8rA7fvxCPR0NFNk1Zc1lxeqWiwrZAiWFzIEywsZguWFDGEt5SUjI0Ov/QxOuiMjIw0OpqI2bdqE9evXY8OGDWjZsiXCwsIwdepUBAQEYPz48VqPmT59utrUZTKZDEFBQfDy8oK7u3tVhW4wQRAgkUjg5eVl0QVLl89338P5z/oBAGS5Jc3K3T084O3touswqiBrLy9UdVhWyBAsL2QIlhcyBMsLGcJayoudnX7ptMFJd/369Q0OpqI+/PBDfPLJJ8ra9datWyM6Ohrz5s3TmXRLpVJIpVKN9TY2Nhb9hgGFLQasIU5t8uSCMm5BVOn7LbHO52MNrLm8UNViWSFDsLyQIVheyBAsL2QIaygv+sZm8DOYN28efv31V431v/76KxYsWGDo6cqUnZ2t8URsbW0tvm1/TaQ6XZxcIWh9TEREREREVNMYnHT/9NNPCAkJ0VjfsmVLrFixwihBFRs2bBi+/PJL/PPPP4iKisLWrVuxaNEiPPPMM0a9DlWe6tzpBULJ4zw5k24iIiIiIqq5DG5e/ujRI9SpU0djvZ+fHx4+fGiUoIr98MMPmDFjBiZNmoSEhAQEBATgjTfewMyZM416Haq8rHwF1p+JRl6BgDm7bijXM+kmIiIiIqKazOCkOygoCCdPnkTDhg3V1p88eRIBAQFGCwwA3NzcsHjxYixevNio56WK69PMD4dvJeLAtF44dTcJM7dfV277dOs1jf3zdSTdsWk5SMrIQ9sgT1OFSkREREREZHYGJ92vvfYapk6dioKCAvTt2xcAcPDgQXz00Ud4//33jR4gWZZV4zsiI1cOD2d7/Hcvudz983X06e42/xAA4OD7vRDs52rUGImIiIiIiCyFwUn3hx9+iOTkZEyaNAn5+fkAAEdHR3z88ceYPn260QMky2JjI4GHsz0AwM5GUs7eumu6i12NSWfSTURERERE1ZbBSbdEIsGCBQswY8YM3Lx5E05OTmjSpInWabqoerPRI+nOUJmzWxsRYpnbiYiIiIiIrFmFJz1zdXVFx44d0apVKybcNZS3s0O5+1yIToUoiihQaWauUBndXGTOTURERERE1ZjlzjROFq9vSC2M7lAXc55qqXOfrDw5Jv52Dr0WHsbD9BwAwLLDd5TbBSbdRERERERUjTHppgqzsZFg4ci2eKlLA537FAgiDt9KRFx6Ltacjka+XMCi/beV20VWdRMRERERUTXGpJtMKjdfoXysEESNPt66RjcnIiIiIiKqDph0k0kdDE9QPhZFEZl5crXtOSpJORERERERUXXDpJuMomuwT7n75MsFZOSqJ925BUy6iYiIiIio+jJ4yjAibZaNbYdTd5PRzN8V/RYd07rP76ej8fvpaLV1OUy6iYiIiIioGmNNNxmFl4sDhrSpg8a13BDx5WDMf7a1Xsfl5Bf26Y6Iz8Dg74/jnysPTRkmERERERFRlWLSTUZnb2sDb5fy5/AGSmq6F+y5hZsPZZi84aIpQyMiIiIiIqpSTLrJJPSdf7u4T3d2vrycPYmIiIiIiKwP+3STSQh6zr+99VIstl6KNXE0RERERERE5sGabjIJhUpV968TOhh07J5rj5SPBUHErycicTUmXW2O7xMRSXhtzXnEy3IrHywREREREZGJsKabTEK1prtvSG24OdppTBemy5vrLuDy5wPg4WSPzRdjMGfXDeW2Jc+HYnjbAIxbdUa57ueXNJP6AoWAKzHpaFvXA3a2vLdERERERETmwWyETKJ08/Ivnm4FG4n+x1+NSUdMajYux6Sprf/wr8tqyw/Tc7QeP2PbNYxYfgoL997S/6JERERERERGxppuMoneTWsBAB4L8gQAPPVYIIa0roPGn+7W63jVmuyK+PPcAwDAymP38L8nm1fqXERERERERBXFpJtMwsvFATfmDITUzla5zhjNvPPkAr4/EKFcvhYrw47LcRjeNqDS5yYiIiIiIjI2Ni8nk3F2sIOtIW3K9fTdgdtqy1P+uISUrHzIFQIA4EFKttGvSaaRJ1dgyh+XsOVijLlDISIiIiIyCSbdVKVmD29pmvPuvI7QOfsRnZyFuDTt/bzJ8mw4cx87Lsdh2qbL5e9MRERERGSFmHRTlRrftQHcHI3fq2F7WBwy8uT4/mAE7GyNX7uuS26BosquVR0lZeaZOwQiIiIiIpNi0k1VTyx/l4o6eisR721UrzUVRdNccPGB2wiZsQen7iaZ5Pw1QVGPACIiIiKiaotJN1W5r55tDQCY2q+J0c+dnJWP+6X6dHeedxA/Hrlj9GstLhrQ7fPt141+7pqi9NRyRERERETVDZNuqnLD2gbg8ucDMLVf0yq5XrwsDwv3mG++7htxMry3MYwDvGkhVzDpJiIiIqLqjUk3mYWHkz0AYO0rj6N3Mz+c+qSvmSOqOEk5XciH/HAcWy/FYtL6i1UTkBVhTTcRERERVXecp5vMqkcTP/Ro4gcAGNW+Lv66YP6poy5Ep0KuENCpkY9e+0tQdtZdnFfeScisbGjVzvawWHOHQERERERkUqzpJovx9ai2uPvVk1q3vdq9YaXPr8+AagpBxIjlp/Dcyv8QlZSl13lTsvMxc/s1XItNL3M/U8xZbs1SsvKRml1g7jCIiIiIiEyKSTdZFF2J6fiuDTCmY1Clzr32v+hy98mTl0wBtuf6I0z87RxWnYgs85jEjDysOR2NoT+cKHM/Jt3qsvLk5g6BiIiIiMjkmHSTxdnxdjf0auqnts7d0R4DW/krl5vUcjX4vDO3X8d/95LL3KdAXlIbvnBPOA6FJ2DurhsGX0sbJt3qVG9wEBERERFVV0y6yeK0qeuJlS+1V1vnIrWFg21JcW0R4F6hc49Z+R9ORCRh66UYrD+jWfOdpyhJBAUjj/HFpFtdbgEn6a4O9Om2QURERFSTcSA1skhSO1u1ZTtbG9irJN2NfA2v6S42btUZ5eOhbQKUI6kDQIEJp7CyLW+Y8xqGNd3W70FKNkauOIWJ3RrijV7B5g6HiIiIyCKxppss1s63u+ONno2w8+3uAADViuKsfDl6NPGt9DUKFIW1rbLcAoiiiHy59trX9JzKD/ilraY7Ni0HgrGr1K1EHmu6rd783eGIl+Vh3u5wc4dCREREZLGYdJPFal3XA9OfbI7WdT0AAM383ZTbRrWvi+Xj2sPetnK1x7KcApy5l4w2s/Zh1o7ryiS8tNk7r1fqOoBm0v33hRh0m38In2y5UulzW6PcKqzpzi1QYOJv5/DbybIHxSPDKGroDSMiIiIiQ7B5OVkNN0d7RM57EqII2BQlsLXdHRGTmlPhc/b99qjy8e+nozGqg/YR0k/eSarwNYqVTrq/238bALDpfAwWjmxb6fNbm6qs6f7rQgwOhSfgUHgCJnSr/PRzRERERET6svia7gYNGkAikWj8mzx5srlDIzOQSCTKhBsAjD2Gk65pv+JleXhx1Rm9m4ILgoiY1Gy1daWT7po+AFVV1nRn5HI+cCIiIiIyD4tPus+dO4eHDx8q/+3fvx8AMGrUKDNHRpbAzVGzscak3sHwdnHAM6GBRr3W8Ygk3EnMhFwh4M+z97Xucz4qBQDw3qYwdF9wGLuuxCm3VcVAaqIoWk0fcbkJB60jMrejtxPRbf4ho7SSISIiIutm8Um3n58f/P39lf927dqF4OBg9OrVy9yhkQX4fkwoQvzdsGJcO+W6VoEeuPBZP3z33GOYWNSU+JnQQHwyOKTS1/tk8xU0/nQ3PtlyVev2kStOAwC2hxUm28sO31VuEyGatHZbFEU899N/GLb0hFX0tVV9KVRHkCeqDsb/ehaxaTl44Zcz5e9MRERE1ZpV9enOz8/HunXrMG3aNEg4/RKhcHC1PVN7qq2T2tkoy8eMoc3xSo+GCPR0AlA4AvpX/1Z8pOWL99MM2l9lljPcjs/EpPUXsXxce90HVEKeXMDZopr2BynZaODrYpLrGIugknULNbypPRERERFVX1aVdG/btg1paWmYMGGCzn3y8vKQl5enXJbJZAAAQRAgCJY7RZEgCEVNgy03Rkv3SvcGuB4rQ/fGPmqvYx13qXJ5Qpf6lUq69fHt3pLzx8vy1LbtvvZIGYtqmmno+66tvBSo9JEWRcsu7wAgVxkpXhQNfw0Modrk3tJfF2Mz7XdLzX1dDWFNrw3/FpEhWF7IECwvZAhrKS/6xmdVSfeqVaswePBgBAQE6Nxn3rx5mD17tsb61NRUyOVyU4ZXKYIgICMjA6IowsbG4lv9W6S3OtcGUBsZ6WlmjeMHlSbliRl5GtuTkpOhEEQ8TM9VrktJSTHoGtrKS0ZuSflOS0uHm0Tz2pYkMytL+VgQBINfA0Pk5JQMamfK61giU3635OfnKx/XtNfVENb02vBvERmC5YUMwfJChrCW8pKRkaHXflaTdEdHR+PAgQPYsmVLmftNnz4d06ZNUy7LZDIEBQXBy8sL7u7upg6zwgRBgEQigZeXl0UXrJrA29keKdmmG+06TSHFgMXH1dblSJwQ6OWk1/HzdocjI6cA7/cKUCsvYmZJku3u4QFvb8tuXu7opPIlJZHA29vbhNdKVT425XUskSm/WxwcHJSPa9rraghrem34t4gMwfJChmB5IUNYS3mxs9MvnbaapHv16tWoVasWhgwZUuZ+UqkUUqlUY72NjY1Fv2FA8XRYlh+ntVv83GOYujFMuezlbI9UlSR7z3s98fiXB012/dIJNwAcu5MEhSDC2cEOI9vX1XlsvlzAz8cjAQAvhPrCx6ekvIhQmUoNEqsqR4Vzr5suXhuVMSCs6XUxFtN9t9Ts11Vf1vba8G8RGYLlhQzB8kKGsIbyom9slvsMVAiCgNWrV2P8+PF6300g0uXpUlOJvdytIXxdS27U1HJzxJ6pPao0ptjUHMzcfh0f/HUZCbJc/HL8HlKz8tX2uRabjlaf71UuC6KIAoWA7WGxeJSeiwKVfstyC+//AgCqA6ybeiC1mj4nOhERERGZj1Uk3QcOHMD9+/cxceJEc4dC1YyDrQ0m9Q5GboFCbX2TWm4I8XdDXT2bfFdWkkrT8Amrz+GLf27i8a8OqO3z8eYryFcZfCwtR45VJyLx7p9hGLj4mNrAZMVzYCdn5uHi/VRYItVEmykxEREREVVXVlFtPGDAANZUkUl0DvaBna0NsvPVB9mztZHgnyk9kJ0vR+tZ+7Qe27uZH47cSjRKHNn5JUn/jYeFI+4XKERcj0tHywAPANCYe3v8+uvKx+k5BZCr1XQXPu624BByCwT8+XpndG7kY5RYjUUt6ebnm4iIiIiqKauo6SYyleIeqYKWnM/WRgJnB837Uu3re+Hsp0/gt5cfx4yhLYwSx64rD7WuH7LkBKb+eQmiKJY7N31x7Xbh48Ja79yCwv+P3jbOzQFjUn3NmXMTERERUXVlFTXdRKZiU5THBno6ITYtB+6O6h8JWxv1RNff3RGb3+qqXG5Rx/Qj4m8Li8OOy3FabwyoUu3HXaCw/CxWtabb9H26TXr6Gquc+0BEREREBNZ0Uw3laF9Y9Hs29QMA/DqhIwa0qI1Nb3Yp8zhnqa3asptKkr5sbDu0q+eJV7o3NHK02mviS1Ot6Z65/RoElYPO3Es2ekyVpZoIMycmIiIiouqKNd1UIx16vzfORaVgSOs6AIBm/m5Y+VKHco9zKdXc3FVastzM3w1bJnWDKIqo7+OMVoEeePGXM8jKV5Q+jUmo9umOSMjEkdsJyuWL99MQL8tFbXdHAFBrri5XCAh/lIEWddxhYyOBQhDx4qozCPB0wjej2posXtU+6qyJJiIiIqLqijXdVCMFeDrhqccCYWdr2EfgpS711ZZVa7qLm6JLJBK81KUB2tXzwqEPeuPrkW3gYGf6j5rq6OUAkJpVoLYcl5YDAEjPLkC3+Yfw+przEEURn269hqE/nMD3ByMAAJdj0nDqbjL+vhBj0nhLNynnYGpEREREVB0x6SYqR3E/748HhWBk+7pq21xVku58uebc2LXdHTGqQxBsq6Dza+nRzf8r1aS8ePPV2HTEpedi34143I7PxMbzDwAA3x+MwLXYdFx5kKY8pnQiXywpMw/7rj/SuKYhSh9aiVOVi+k8EREREZkLm5cTlePA+70QmZiFTlqm3JLa2aKulxPScwpQ38fZDNGVKCiVtf5Vqqa6OEFOziqZE3zF0btq+wz94YTacr5C0Noa4MnvjyMhIw+zh7fE+K4NKhRv6ZrtwmXT3JxgJToRERERmQtruonKUcvNUWvCXezwB71x7tN+cLS31bmPakX3pjfKHqytovIKyu47np5T2Nw8NStfue5m0Zzgus+pvaY7IaMwcd9345EhIaop3bzclDXdRERERETmwqSbqJLsbW3KTLgBoGuwLwCglpsUjzf0Nuj8pact0+VeUlaZ29/fFAYASMku6esd/iijzGPytDSZV1WZGuTSLdf334iv+MkMwL7jRERERFSV2LycqAp8PbINfj8dhRHt6pa/cym13aSIS88td7/5u8PL3C7LlQMAUlSal5cnT15Ye55boEBmnhw/Hr6LyKRM5fbK5K+lk9/JGy5iSJshFT9hWdeC6pzggC3nlyYiIiKiKsKkm6gKeLk4YGq/psrlnW93x7ClJ8o4okSvZn6ITcvFsduJlY5j3r83cTs+s/wdixQPDtdz4WFlk3JVYiWGKCvdvBxQn8rMVARRhK2J+o4TEREREZXG5uVEZtC6rgd+KWde8Nd7NkLbuh74eFAIPhrYDO3re2HzW5XrD/7TsXs4G5mi9/55cgEKQdSacAOaNd25WvqVKwQRC/aE48itBLX12vpw52sZLf16XDomr7+IyHKaz+tLW7JPRERERGQqTLqJzKRXMz+1eb5VPdnaH/97sjm2v90dns4OaBXogc1vdUX7+uX3B+/RxNdoMebJFUjK1N0cXTV9XXzgNkJm7MGZUlOVbb4Qg+VH7mLC6nNq67Ulv7laBm57Ztkp/HP1Id5ce0FnHP/behUDvjuqNenXiJk5NxERERFVISbdRGZib2uDq7MGImp+ST/mul5O2Da5GxaNfqzc49sHuWHR6DYa64O8S6YuG9Y2AJ0MHLhN1d3ELCTIdCfd8bKSvuaLD0QAAD7fcV25TiGI+PHIHa3HClqquov7kKsqrv2+l6S7WfyGM/dxOz4Te69rH01dNdFm0k1EREREVYlJN5GFeSzIs9zR0AEgyNMRTz8WqJa0A0Cgp5PyccsAd6wY177CsXz09xWEP9I9rVh0cjYepGSrrVMd8XzDmWhEJatv33juPk7dTdLavFzXFGUAUKAQsepEJDJyC5AnV+D03WStSXp5LL15+d8XYjDgu6O4X+p1IyIiIiLrxKSbyAI42BV+FEPreZW7768TOmBQy9p4u2eQcl0jPxflY09ne+VjqZ0N7O0q9zH/8O8rZW4/HpGktlzc91oURfx9IUZt2/awWHy8+SrG/nxGR/PyspPoubtuYOb26/h8+3U8//N/+Hz7dY197iVm4t+rD3VODWbpSfcHf13G7fhMzNh+zdyhlMvEY94RERERVQscvZzIAux+twe2XozFqz0alrtv35Da6N3UDykpJQOi7Xi7O47dTkT3Jr44fbekT7WdrQ3sTTw/lggRlx+kqa1beewuwh9m4HJMutr6d/8MUz5O1DI4m7Y+3aUduZWA1KK5xv889wCtAj2U2yQSCfp+exQAsGp8BzzRvLbG8dpq2C1RZp7c3CEQERERkREw6SayAMF+rvhgYLMKH+8qtcOTresAUB9ILTNXDnsb7TXdfm5SrYmvoT7dqlkj+9W/Zc8ZDgD7bsRrrBu29ATufDkYdra6a+dtSlWvfrbtmsq2kvUX76cqk27VPFtXDbilsfQaeSIiIiLSD5uXE1Uzzg4l99LScwpgY6O9pluWU6C27K5jJPWq9tOxewAKk2Nt/ZqTs/J1HitRmX9bUM+0ta83ggKFgG/23tIYtb2yrKVGnoiIiIjKxqSbqBrzcXHQuv7XCR3UBzx7rRMOvt+7iqIq28Gb8Ziz8wYaTv8XPb8+bNCx56JKmtyrjViuss+Bm+o17IIgQparfgOiPApBRFRR3/XfT0Vh6eE7eG7lf3ofv+/6I2w690C5nCdX4HB4AlJVbihk58ktvlbewsMjIiIisgiWUbVFREb1+8THse/6I4zrXF9j26TewegbUhvPhAZi66VYDG7lj67Bxpvbu7Iu3k/DxftpFTr2t1NRyseqCatcpdr4o7+voF/z2rC3lcDN0R6TN1zE7muPlH3AkzLz8Mpv5zCqQ5DW1w8APvz7MrZcjMWIdnWx+WKM1n10EUURrxfPOS4B6ng4Yt/1eKz9L1ptv4iETEzfchXzR2hOC1eVbj6U4fdTUXi3XxPU8XDSuZ8oipBwZDUiIiIiDUy6iaqhXk390Kupn8b68V3q48OivuNzn26FviG10DekVlWHVyVU+0QXyNUHaGs3dz8A4Mz/nsDua4Vze7/y+3m0DHBHm7oeuByTjssx6RpJd3p2ARwdbLDlYiwA6JVwFyf/xQlp8bzjQOENgLL8ee6B2ZPuoT+cgEIQcTcxE3+92VXnfoIImHjMPiIiIiKrxKSbqAbx93BSJn+uUjsMaxtQ7jFhM/vjnT8uaUwNZul+Ph6JAoUIN0c7HLmdqHWfTl8dVFu+HidDikoT76/3hqN1oCeSMvPQr3ltdJ53EI18XUqfRmnt6ShsOh+D1S93hK+rFKIoYvzqc8jKk+OvN7rAxkaC7DzD5xavqAM34vHz8Xv4tF89eHtX7ByKolYCN+LU52s/H5WivGEBFN7ksAWzbiIiIqLSmHQT1SCq83lrM+eplthw5j4+HhSCuPQctKvnBU9nBywf1x6tPt+r3O+1Hg3x8/FIU4dbaarNzfX1MD1X+XjZ4bvKx8WjpN8r6sutzYyiecMXH7iNL55ujTy5gGNFCf/9lGw08HXBWZV+56b26przAID5+xVY93odrftsPHcfv56IwqoJHVDXyxkAIFcIeJCag4YqNxhKd98eueK02rJCEGFva7zYiYiIiKoLDqRGVANseK0TPhkcggEtNOetVvVSlwbYM7Un+oTUwgud6qN5HXcAhbXinw1pDqCwWfqnQ1oYJa4nW/sb5TyWJq1oHvG8Us3akzLz8EZxf24jkSsE5aBuOuPJ0T3n98ebr+JWfAa+2HVTue7Lf2+izzdHsD0sVrmuvEHTFBxunYiIiEgrJt1ENUDXYF+82Su4UgNdvdqjEaLmD8GLOgYXK0+wllp2G4kEWyfp7idsrYoT0Oz8kmR3/u5wnI9KNfhc+aUS9/TsAoz/9Sy2XirsTz51Yxh6f3MEOy/HKfcRRRFylb7jOqZqV5NdUNLsffXJKADAu3+GlZxTo65bnYJDmRMRERFpxaSbiCrtpS7aE/Gwmf2Vjwe30mze7OsqRWg9L1z4rB/mPt0K0weHmCzGqlSgKExAPy9qbg4Ae64/wpvrDK/ljk7OwqZzD5TTia05HYWjtxPx3sbLmPLHJey68hAAsLJofnMAeGPtBfRYWDLdmq2OudpVHbudiI3n7iMuLUfr9nJruhXqO1yNSce0jWGIS8tBvCwXp+8adx7zYgdvxuN6XLpJzk01R1p2PjacuY/0HMOmDyQiItIH+3QTUYX4ujogKTMfjzfwxtt9GmPN6cIpr0LreeJS0ZRfUjtbTO3XBDvC4jChWwMsP3oXCkHEjKEtsOtKHN7p2xgA4OMqxYud6+PUXe2DtfVrXgsHbiZUyfMyhqTMPPT55ggiy2n2rY8XfjmDhIw8tDvnidrujrjxsGRAsx0qtduO9oX3UAsUAvbdUJ+L3FbPFg4fb76qc1t5Sbe8VPPyYUtPACjsI3/6XmHCveG1Tkadnu52fAZe+b2w33rU/CFGOy/VPG+uu4D/7qXgwM14/Dqho7nDISKiaoZJNxFVyKY3umDdf/fxRq9GcJaWfJV0DfZRJt0OdjaY2q8ppvZrCgC4NLM/BEGEp7MDXuneUOOc9bydtV7LzdHe+E/AhMIepBntXAkZeQBQ7tzlD9NzMWL5KQzXMiK9tpruyw/SDJpjvLzm5YKOrPxeUqby8V/nY9Dc3x1eLg56X1eb6OQsZOTKEZOaXanzEBX7717hAIeHwq3n5h4REVkPNi8nogpp5OeKmcNaoLa7I5xUhq1W7YNcOtlzd7SHp7PuhKuulzNe1ZKM+2hJ0paNbaf1HB8NalZu7NVRTGoOLkSn4vMd1zW2Fb8POfkKPL3sJBbuCcdTy04qWyfoQxQBQRDx64lIXLqv2TddtaY7T17SP9zetuTPzNZLsQgtmiO9Mnp9fQRDfziBxKIbEoXxmb5PeUR8Bm4+lJW/IxEREZEKJt1EVGmqyXVtd8dKnev9AZpJc75C0FjXIsBd6/FPhNRGkLeT1m3PdQiqVGzWqvjt2R4Wi7AHafjxyN2yD9BCBLDzShzm7LqBZ348pbH9cHgCLkSnIjo5C21m7VOuj0nV3kdcVUVHPo9KLqnp/ufqQ0TEZ5R7zKL9t7Hp3AO9zv/DwQi8++clCIKIAoWA/t8dw+Dvj6sNkGdwzElZuBrDPuhlEUUR07dcxbf7bpk7FCIiIqNg83IiMop/p/RAWk6+crqsirKz1WwKHZeWi0BPJ8SqDPKlOoe0KmcHW0igvQ9zvxa1kZUvVw4+VlMU10LLciv+3igEETfidNfyFs9j/lyHII2p0soSL8vFwMXH4Olkjya13fAgJRsfDw5Bn2a1yj129cmSueLf3nAJQNl9u6/GpGPJwQgAwOiO5d+A+Xb/bQDAmI711G7yyHLkcHao2J/P3t8cAQCc/d8TqFXJG1TV1d3ELPxx9j4AYFr/ppWadYEsl1wh4PCtRLSr5wkfV6m5wyEiMinWdBORUbQIcEfXYF90augNAKjjUbGEwk5L/+NnQgPxWJCnXsc7O9hidIe6Gus9ne3RIsAdc59qhfGlRlsP9NReM66Ntv7Rqy184KWz0TLsuByHvAL9k2FtflIZIV2XjefLr0VWbQr+87F7SMsuQFRyNvbfiEf4owy8vPqczmMElVpxbRXkZTUzT83OLzc2bXLlCrXaeH2mYNNGdRq3GB2jxBtTalZ+lTS7N7YCldeJ879XX6tOROK1Nee1tpwhIqpumHQTkVH5uEoRNrM/jnzYu0LHq9ZqLRvbDrve6Y4nW/ur7VOcNG94tRPe6NkITz1WMniYs4Md3uwVrJYI92jii/+mP4FATyd4uThg9lOt8PzjhTWdTWq5wsdV/4G9tM03rq123tJM3XjZoBpoU1Lt/1161HNtdl2JQ/svDuC/e8nlzgeena8oc3sxQ5NR1bEKhAq+jJl5Jc3SHe1sy9iz8k5EJCF07n589PcVk17HFFRvbBUomHRXV8Utju6ncEBEIqr+mHQTkdF5OjtAaoSkwsnBBq0CPSCRSNRGz54xtAUAoGtjX0x/sjm+HxOKFzrVw6vdG8LJwRZ2tjboE1ILk3oHw9HeBjOHtoCjvXo8M4a2wNcj22DjG10Mimnp2Hbo0sgHbet6KNfJdSQGL3drgHb1PMs8n68BCX9lqQ5wZk6LD9yGKIoQRVHnqOfbw2IxYvkp7L76EG9vuISUrHxM/O1cuTWfCRl5yjnNi/187B4+/Ouy2rWKk/0rMWlIyMjVOI9qbasE6km3vIJZtyynJOkubzT4yvr+YGHT+L8u6D9CvaVQbUyibTwHqh7YioGIahL26SYii1XLTXsTdTtbzfuFXz7TWmPdR4NCMLVfUzjYae7v7GCHUWUMrBY+dxAGLT6mNlgXADSt7YY/Xu+M+bvDcbloQCxtNcjPhgZiWv+m6LnwsNbzh/i7YWT7uhjY0h89tOzTyNcF94wwz7eq1Er2tzeWZYfvYteVhwj0dNLZN//dP8MAABeiS0ZKz85XlFszPmrFaaRk5eGP1zqjUyMfAMCX/94EALV+ozGpOTgXmYKPNhfWBEfNH4J8uYBF+2+jvo8zZpUaBT5fUXLDorxkQdCxXbVPvalrcFVDKFAIaqPIWzJBENVauxQw6a62dN1wIyKqjiz+r3BsbCzGjRsHHx8fODk5oXXr1jh//ry5wyIiE/rt5Y744ulWaBVYUptc0d9n2hLu0qb1b6qxztHeFn+/1RUbX++s9Rh7lSblfm6atdWjOgTBzdEe3jrmpO4S7INXezTSGdNzegz0Zai/LajWMzo5G6fuJmP9mfsGHTdz+7Uytydl5kEQgdtaRjLPzCtJevt8c0SZcBf7/VQUVhy9i+lbrmrcSMlTq+kWy2yeXjqZWHnsLtKzC9SSbrmWZPJBSjZGrTiFfdcf6Ty3vlRjaDFzD346etcs85qLooh8uaDxeiVl5mnsG5WUhcfm7MOiogHsACbdluTnY/ewSY8xG/SlT9cSIqLqwqKT7tTUVHTr1g329vbYvXs3bty4gW+//RZeXl7mDo2ITKh3s1oY17l++Tsa8XqXZvTHG70Kk+D5zxbWmvu6SpW1paUNbFnYz7y2uxTt6nnhf0+GYOnYUOX24trQhSPbaD3epqg2z8mhpNl7cbN5ABrN4anQlouxeu1XoBDxMD0HXxXVcgPAuv/KTvDvJmZqXf/dgQjkFpTUdKdk5aPLvEP4dOtV5brkzDwcu50IURQ1+p1/9W842s7Zh//uJivXaWs2/em2azgXlYrX114o+8npkCdXKONUr+kWMW93OLovOIzjEYkVOndFyBUCus4/hKaf7ca0TZeV6zede4AOXxxQS64BYP7ucMhy5fhHZXaBAjkTM0sQnZyFL/+9adQxAnS1CCEiqo4sunn5ggULEBQUhNWrVyvXNWzY0IwREZG5OJk4CfVyccD0wc0xfXBzvfZvFeiBg+/3Qm13R0gkErzeMxiiKOJtFE5dVVDU77d9fW+smfg4Xvr1LHo08cXxiCQAJf1WfV2l+PTJ5nCws0E9H2fl+fuG1MK5qBS0qeuBr/4NN+IzrRnm7LqBP8/dx+147Ym0Nrqau15+kIY/zpbU8K3/LxqPZLlYf+Y+fF2liE7OwrawOADAotFtEeznqvU8Sw7dUT7W1rw8MUOz9lfv2AURXeYdQna+HFc+H4hcHQPKrf/vPtoEeiK7QI46HvqP2l8Rl2PS8DC9sL/81kux+O65xwAAn24rvFmx5GCEWisTbX3l2afbMqhOBakQRK2zOBiKNd1EVJNYdNK9Y8cODBw4EKNGjcLRo0cRGBiISZMm4bXXXtN5TF5eHvLySn64yGSF88oKggChokPOVgFBKGx+Z8kxkuWoieXlgwFNcf2hDOM61avy5/16z4ZYeSwS7/dvonbthkVJsrZ46rhLleu7N/bB3S8HQSKRoNH/dgMAJJKS417p3gAAlAk5ADjaSbBkzGMAUG7S/cOYx/DLiUhlH3MqZEjCvefawzL7at9JKGmurjpC+vdF836XnOeRXs24s3ILtJQblenQ9Cjj/159iBVH7+GbUW0Q5OWMlKIB5Cb+dha3tDSvBwrLXfsv9kMuiLg0ox88nOzVtpf+bknPKYAoivB0NnzAv7hU9WnRCv8Oi2o3HH44eBtjOgbBx1Wq9UZEvlxR5msRm5aDi9GpeLJ1HaMkgsVxkjqFymuSL5crB8qszN8i1c8bX/OaoSb+dqGKs5byom98Fp1037t3D8uXL8e0adPwv//9D+fOncOUKVPg4OCA8ePHaz1m3rx5mD17tsb61NRUyOVyLUdYBkEQkJGRAVEUYVPRSWCpxqiJ5cUBwPpxhc2vU1JSqvTar3X0w6Am7gjylJZ77VXPt0BCRj587Qu07tvY1wl3knLQq4GLxvZAZwWCPKXoUM8DkvwspKQUDqT2cqcArD4Tp/OadZwEvNm1Dt7aVJJ0+7jY491e9fBIlosjt1NwI8H080JbszfXXSxze15+yd+PzBzN0c6L5efnIzuv/AHrJm24hPMfdIJcIUCWq4C3iz0UKqPLF5cNQRQhQeFUesU18cVdE97+IwwAMOj7E9g4oWQgwRN3Spqxl5acka2sYbwW+QjN/V0giCJEsXCqLtXvFgESdF50FgBwampHvcZHUBWXnKa2nJKSgm8ORamt+3Z/BPZff4hfx7ZEbp5mTX9yShpSHHS/nn0WnYVcEBGXlI5RobUNik+Xsj7juQUCHO3VX4eUrAJ4u9jrOKJ6SE0ruYmTlJSi7BZTmb9FcoVmeafqrSb+drF0coWAO0k5aFrLWfm3xVJYS3nJyNB+k7s0i066BUFAhw4d8NVXXwEAQkNDce3aNaxYsUJn0j19+nRMmzZNuSyTyRAUFAQvLy+4u7tXSdwVIQgCJBIJvLy8LLpgkWVgeal6Ptq7dmvo4+1d5vadU3ogOTMfAZ6aTXu9ARz9SDNxmPGUd5lJd/06vvDJkwMoqRF/snUdjOveFIIgYETbRDyxTDOpnNw7GO/0bYyQmXvLjNla2dpIjDYtkSAp+ZyJEt1dHWzt7CGxyde5XdWai8lIzMjDxvMPsOHVTrBVmWbvwqMCdGzghSe/P4F63s747eUOeHLJCTT0dcHqCR0Rlaw+sv2eCP3+6EemlCS2Pt6e8PJyxVPLTiFfLuCfKd0hgaj8bslSqdEvsHOGv7eztlPq5OKs3tLA29sbf148o7HflbhMeHt7w8ZW8yfJzD2RaFLLFTcfyvDxoGZ4snUdte3FNxAuPczBG0+U/dnTl7eOz/CtRxkYvOQEnu8YhC+faQUA+PHIXXyz7zZmDWuBl7qUPw7FhehUzNp5AzOHNkfHBsaJt6LuJGTCRWqrVzcDF5VGNG6ennB3LLzJUJm/RYW3kwrpes2peuFvF8vz3qbL2B4Wh/f7N8HkPo3NHY4aaykvdnb6pdMWnXTXqVMHLVq0UFvXvHlzbN68WecxUqkUUqlUY72NjY1Fv2FAYU2GNcRJloHlxTo5OdigrrfhtWKb3uiC2Tuv43pcYZeZIG8nPEgprL32dJHCz90JG17thLG/FCY1BYqSO8MeTvYY2LI29l6PBwB88XQr7LoShzd6B8PRwaL/DFSKMecBVp2n+/Q93bVyIoBH6bprwlWp9vEe87N6MvrGuot494kmSMjIQ0JGHl7+7Tzup+TgfkoObGxsEC9TT+xdpfq9j6r9xhUCkFUg4FpRmUrKykdtNykkEgkkEgn+t7Vk2rR8hWZNgyiKSM8pgEQiwa4rcejcyAeNfF2UU36VfvXL+q6ysbHR2sc3MikLkUVT5739Rxii2gaWeQ5dTkQkISEjF8+2q6tzn2L5ClHrQIbLj94DAPxx7gHmjSgcIPGbfYWDwc3aeQMTupU/5szzP5+BXBDx3MoziJo/pNz9gcJyfCwiEaFBnhVq5q9NQkYuBiw+DgD6xaFSAyaIErXXuqJ/i1Q/n/w7VnPwt4tl2V40HsmKo/fwzhOaM7mYmzWUF31js+hfW926dcOtW7fU1t2+fRv161fdqMZERJbg8Ybe+GdKD9yIk+Hn4/cwrX9TRCdnQ2pvo5yDuWtjX+X+pae8clYZJX1c5/pGHR0+xN8N2yZ3w8zt19C7WS00r+OOPt8cUW4/8kFv2NvZoNv8Q0a7ZlWL1HPO9EPhCUa75v2Ukr7h56JK5itPy87HrUcytX1/UEng9VUgCGo3EyQqNY9nIlPwz9WSUcSztAzMNnnDRfx79RH83KTKZL5tXQ90auSD9/o1NXgu8soMrKV65J2EDLy94RLqeTujf4vaGNUhCONWFd7UaFPXA41ruZV5rpAZe7BiXDsMaqVeq26MLuMVeY6/n4rCnF030MjPBYfe7135IADciJOVv5MKucp7ufLYPbSt64HBpVodGKq812LN6SgcvJmAFePaq83yQERkjSz3tgGA9957D//99x+++uor3LlzBxs2bMDKlSsxefJkc4dGRGQWLQLc8d1zjyHI2xndm/hqNFHt08wPADSSakPnOfdzK2kxtOud7mrbVOcoB4B5z7aGo70tFo5siydb10FDXxcMbuWv3N7A1wWBnk5YM/Hxcq/7w/Oh5e5TWos6ltt1qDKuxWofGK/TVwcxa+eNSp+/QC6oTYWmOid2Rq76GCjvbwrDgj2F3RdyCxS4FpuOf68WzieuWnt+OSYdK4/dw7RNYRqjkZc3OntlWiaozgM+bdNlhD/KwL4b8fjw7ytIyy5pFfBQz1YIU/4M01hXkf6Oqq9vRe24XFgTdS9Rvxs/+shRuYlS1pzzxfJV+l+vOHoXb60vewwEfeiaMiwpMw9n7iVj5vbrOHo7EevPRFf6WkRE5mbRSXfHjh2xdetW/PHHH2jVqhXmzp2LxYsX44UXXjB3aEREFumX8R1x9tMn0L6+l9r6sn5Wn/+sH/o1r6VcXjPxcfRvUdK3vFWgh/JxizruuPL5QPz9Zhd8+mRzfD/mMYTWU78WADSppTltVs+mflqv/8GAprj1xSBcnz0Qw9oGlBGpdj2a+GJAC+MMomVJIhK0j75euhVDRckFEQdvltTMD/7+OMavPgdRFBGXpj7w3t3ELCw/chfZ+XK8ue4Chv5wosxz7772CI/S80qte6hj76J4DKwZV3XgZgJ+PFJY2y/LUR94LTOv5AaCvol9cSIqiiIW7AnH2tNRsDGwqnvDmfsImbEHe649Mug4Q+y++hC/nogEAFyPS8euK7rHflCVo3azpfzXJK9As8yJoojcAgXe3HgTPx65q2fEJVRrulVvjPRceBjPrfxPuVz6BpCxnI9KQfcFh7D/RnylzxWVlIVN5x5AboQp7kRRNGrXGCKyDBbdvBwAhg4diqFDh5o7DCIiq2BrI0EtN0eDjvF1lSK0nhcOFCVgPZv6wc3RDhvO3EfbuoUJ96LRbfHtvtv4elQbODnYokMDb3QoYyCot3o3hlwQMaClv9p6GwlQ+vdk35DakNrZQs9uyWom9Q7G230Lr9Vm1j6t+7Sr54mL99MMP3k1993+2zgfXdJsPTNPjuMRScjKr4/Zu25qPeborUQcuZWo1/kPhqsnMzO3X9exJ3AhOkWvREOuEGBX1J3i4v1UtW0L99zCyPZ1EZWsPmWb6k2KD/66gvOf9Sv3OsWhRCVnY3lRQvlka/8yjtD0v62F85G/ue4CTnzcB3W9yh+I7vdTUUjKzMP7A5rpdY3iGudtYbG4UjRloJ+rFJ0a+UAhiChQCFr7p+eUauFQ3sj02uZLL1CI+OtCDM4/kOH8Axle6d5Ir2bg95OzsS0sVi2G9zddxi/jO2DViUi1KflMaeJv5yDLleO1Nef17l+vS++i7jSCKGLM4/UqfB5RFDH6p9OQ5cjx77s9jDYNniVIycrH8iN3MKCxGzhunuWRWNjI5dWRRdd0ExGRcUx9ojHcHe3wto7RSWu5qQ9AGVrPCyc+7oNNb3YBADzbri5OftIXLQM8tB2uwcnBFh8NCsFjQZ5q60v/uH+le0M0r6Pex/b1no30ugYAfDQoBM4OdnB3tMfJT/pq3ceuAgOwdG7kDW8X4wxaZalUE25VUSm6m2Ab0qw4Orn8+cqLjVh+GgV6zHWaml1Si/3sj6c0tneZpzluQJZKTXdSpmYTd21NwBWCiKsx6Th2u+QGw4GbFe+v333B4XKb1+fLBXy+4zp+OHQHD1J0v3b5Wlo6FCfcABD2IA0AMGTJcYTO2a/WlLyYas11vlzA7qsPNVo3qO2v5Zq5cgXSVd6Pjl8eAFCYXC3YE457idpbajy7/BQW7b+ttu5geAJGrjiNL/7RfrPHUELR+6fttSqmbZyCW48yMOyHEzhcwbEZrujoEqKvjDw5zkWl4lZ8RpnvR1U5H5WCs5HGmc5t+pYr+Pl4JMb8ftUo5yPLU6AQEPYgjS01dGDSTURUA9T3ccGlmQPwwUDtNWhPhwZiUEt/fD6sZMaIul7OkNoZdwCj2cNbKh//8VpnzBjaQuMO+/+ebI4bcwZi5Yvt8YYBCXigpxO6NS6Z221a/6ZYOjYUL+oxlVNpQV7OWDrW8P7l+lCN0RLdSjBe32FD6PND7e8LMQafI1NH8+QTEUk4eDMeITP2aN0+bOkJfL5DZQT3Ugnc9TjNBOtBSjZ+OX5PLdEvdjtefVq30n32H6aXJFm6XovIpCy0nrUXw5eewLhfNKdfA0oS5PBHGcgpUOCZH08iJlU9iVc9/5/nHuCt9RfR+5sjUAgivtl7S+1mA6A90c8tUKjVgGfmySGKImZsu4blR+7imVI3Ra7HpePtDRe13vgACqdT06Y40tSs/DJvRhRLzy7Ak0uOY9jSE5i68ZLO/bTV601afwFXY9Px8m/ntB4T9iAN28Ni1eNT6RPvW8kbdSmZJc3sCxQCbj3SbypAU8iTKzByxWmM/um0WheNiiq+GaQQRLM9r2kbw/DiqjM6xxPILVAwYayE6Vuu4ullJ/Htvlvl76yFMcbAsGQW37yciIiMo6ymiva2NljxYnuTx/Bcx3roGuyLQE+nMvvIOjvYYUBLfwxo6Y83ewXDzlaCf68+xO+nonHjYeHIywuLpm1SNe+ZNpi68RJe7xmMQUWDuYmiiD3XH+GfK5p9in1cHLDxjc5YfuQeOjX0xkebrwAoTFx8XDSnnzSGhSPbIsDDEQ2n/2uS81fWzUfmSbrv65FQLdgTjpe7NTDoR/vSw+oju8fLctHpq4MGx1fakCXq/dozcgswYvkpJGTkaR3t3rlU0+uhP5xAm7oeWPliB/h7OCIuraSFgeqgdqopwHf7byNPLqjVbJcW9iBNLREMf5SBV38/jz1TeyrX5clLftxuOv8AQGFiPW1TWOEUQoeB317uiDy5gM6NfPDTMc0+27n5gkYNuEIQcS6qsGY0vahv/YYz93HyThIO30qocNNxURTRZf5B5BYIuDijP7xdHJCalY/b8Rno1Ej9JtakDRcQXlQ+igf706ZwYDz1BCtBVnZrhKeXnQQA1PN2xuydNxDo6YRvR7dVbnd1VP9ZnZFbADdH7VNE/nPlIep4OqKdypgYyVklSfewH04gK1+Bta88jh5NtI+HYUq5BaoDKxZonZYwQZaL1OwCNPMve0aA0t5cfxFHP+yj9/7bw2JxLTYd/3uyud7NoEVRVNtXrhCw5VLhDZM7iZloWls95vScArSbux+tAj2wfXI3vWOzZLLcAhy4EY8BLf1hZyPR2tXEmIpviv545C4+GhRi0LGzdlzHb6eisHdqT4PLk7Vg0k1ERFUqyLv8vq2qvIpqj57rWA/PhNbF/hvx6BPiB2ctc4zX83HGlknqP5gkEgk6NfRWJt2fDWmOnHwFJnZvCImkMMEv/uFcnHRL7WxM9odfamdT5g/Hul5OiEnV3rTUxcFWa7NYY/r3RpJJz6+LviPsb70Ui+lb9G+ieupustrykoMRhoSlt+TMfCQUNSHX1jxZW8J5JSYdo346hff7N4OHU0lypmuwvBw9aoIOhScoE95i4aVuUqieX/UGQfGcvQAwYXVhbe/jDb3xIEWzPM7ZdR0BHk5q6+SCCKl9SSPKrDy5sm97Re0Ii8WzoYHKJDAyKRPeLt6YueM6dl6Ow6dPNsdrKi1iTt5J1nUqdaU+gtvDYpGhZ43uv1cfIuxBGsIepGHhyJKbf6pl+FB4PCb+dh4fDwrBW72D1Y6/HZ+ByRsKu2qo9idPzylJuos/57suP0THBt4oUAg6E3igMKkURM0uPBWlWuOrq/b38aKbV8c+7IN6Pvp/ryfraO2gy7tFswnkFgiYNbxluX3dZ26/hiO3ErFrSne4F71mpb83T91JQnRKNp4v6oN/IiIJCkHE5aIa+erg8+3XsfVSLIL97uB+SjbGdKyHuU+3MndYWv12KgoAsORQBJaNbWfeYEyEzcuJiMhqONjZYEibOloT7rKM7hCEgS1rY8GI1ni1RyO880QTuEjtNM4z/9nWaBngrvdAVgDgaG/Yn1JpOT+KhxTNf9xYywjwv+sx7Vpl5VdiFPGqYEjCrc36M/crHYMst0BjXaJKIlGgJUl5QUdz8AcpOZi6MQyn7pbc7Lh4PxW/HL+n1uQcAE7f1S+hLJ1kl2bICPi6+vQeuJmAk3fVb9AUKAS1Lilr/6v8dF9RydmYs6tkirziG1Y7i6ZSW7g3vNxziKKIZYfvqL1+pdO2d0tNEycIIu4lZuKDvy5j15U4tdYDqiO+qza9L37bRVHExN/OA4Byqj1VsSo31VSbOufka74vTg626L7gEFrP2qezmbcoihiy5AT6fntE2UriWmw6nl52Eq/+fh53EzMRk5qNPt8cwW8nI7WeozTV51XeCPeXHmjvGpCRW4DsfOONPr/2v2g8rzKyvS5rTkfjfko2Fu4Jx6k7hWVUNQ65QsTYX85g+paruFQ0IKNqF4xPt17Vayo9S7etqCvE3cQsFChEo3wetREEEVFaWveQOtZ0ExFRtedob4ufXuxQ7n5jHq+nNvqwva2kzB+cy8a2Q8+mvniYnot8uVDuVFpASU3Uby93xPzd4RjWNgBf7y3pA/de/6YIruWK3s388PiX6s2g3Rzt8eML7TDJCPMkF/twYDOsPR2NRaPbYqyOxJDUrT2t+eN11IrTysflDZqmzc/HS5Kh4pHe/zh7H64qtZv69q29r2UQu+x8ufImk7H6Tt4tNXe4XCGigY8L7hRNdzd/d/kJsT4OqbQcKD19WYGicJR2e1vtN7OSM/NwLylL+Rm7OWcQnBxsodrYRFsf3w/+vowtFwuTlr8vxCBkmrtyW3GtHKDeV3/9mWg82dpfZ//0YqqtATrNO4j5z7ZG35BaWlsybL0Uq2y5EP5QpnXWiKx8BW4VxfHL8Uhci0vHmXspyv7zh8Lj0TekFiKTsjBr5w1M6NawzPgA9S4Oqt0RtNFWE56Tr0Drohklni81ont5TcTvJGTi+4MRmNK3MU7fU7/RdDaqcKYDoWhqtQHfHcP9lGxsm9xNY+DOdf/dx7r/7uOfKd3Vbnbmqjyf+ynZcLCzwTyVsrr+zH2M61wfzeu4o6Ii4jPg6miHOqVagxQLfyTDkVuJeLlbA6OPnQIUfsaDvJz16rZTGXJF4d+98m706eufKw8xqGVchaYPtXRMuomIiHQor4anVaA73Bztlc0+N7/VFbN3Xscng0Mw9ufCBPa1Hg1xOz4Tl2PSUM/bGQ5FyUHvZrXQu1nh/OiqSbejvS1GdwjSej2pnQ3qGdg8vzyT+zTGpN7BWqeFIu3u6JhD3djuJmahTV39ZgxQ9fGWKxrropOzlUmEtnm3jaFAENSSNVNIyMjVaO3Q++sjOPZRH63NjgcuPoZgv5JWI0mZebgel67WZ/n301EaxxUn3MX6LTqqNR7VOcVjUnPQ6+sjOmMXBBETfz+ndlMkMSMPr/x+HoGeTojVMmJ56a4C2qg2idZWsy6IwL1E/WoikzPz8CA1Rz3pLqe8yLV8T95VGb3+j7PqrUskKBwZ/fTdZLzVOxip2QVQCCL8PQqnu3zhl/8QL8vDqTtJav3ci41ccQoxqTn4cEAzZVL59LKTOqd+G7LkBBr5uSiXVW86iSLw13nNARojEjL1SrrjZbm4EJ2KgS39leUvXpaL/t8dAwCdMQ1afFx5fdXuB3KFgJsPM9AiwL3CU8btvvrQoJkmiu24HAc3qR36hNTS+5jwRxllJtwFCgFp2QXwc9N/jJR3/riEwa38UY1mzAPApJuIiEgvblI7ZZ/PPVN7ICUzH/V9XNT2aV/fCzve7q7WNHFY2wC0qesJedGP2IrOh2pnI4G3qwO8XUtGSNb1Q11fxaPDSyQSg2pbDkzrifm7w7VOo7VqfAe88vv5Mo//cGAztRsNpc0c2kKtSbGl2XoptvydjKSsQdN00dYyVlBZWV7NZUUpBNHkIxCXbgYOALFpOfh67y0kyDSnu0vKzEdSZkkT+RsPZXhznXpCYqpmt8W2h8ViWJsAbLkUq3Oee30/x7LcApy6k4zezfzgYGuDpMw8nV0XVGXp0cw7/JFMmQyqKq87gup0f1l5cihEscybeBKJBCOLWobU83HGjG3XIMuV4/LnA+Bkb4v4ogHttCXcAHDpfhoA4EpsmsY2XTcpVG86hD8sSRIFUdR6o0gURchyC7D00B08/VggWgS4IzEjD7+ejMSYjkHK7/7B3x9HSlY+Zg1roWxBcLNosE99lJ4FYcb26/jj7H1MeaIJpvVvWuaxNx/KsC0sFpN6F04JWvy3payEO0+u0Ppdn5knx5Q/Ckf7v/vVkxoJf1p2Pp5adhJDWtdRGyTNppy/Z8+v/A/no1PVWiIIgogbD2UI8XeDnY4WKpdj0hBaquWCtWOfbiIiIn2o/LYI8XdH18a+undV+SHiUjTqr52tjc4fGI9raTJa2s53CgcFclXph96jie4YdHF3tMOh93shct6TmP5k8zL37dHEF3++3hl9mpWMnuziYIvGtdy09v0O8XfDE81rK5cHtKitsQ8A1HZ3LPO6tdxNM3K8Nute6VRl1zInQShMJARBVKvlNSa5ouxky5RWHL2rHJ26LNoSc11TSBnLu3+G4c9zD/DBX5crdZ4jtxLRZtY+vLnuAkJm7EGfb4/gqp5zg8eXGpl90f7b+J9K3+XD4QlaE25AszvC5QdpeG1NyY214ublX/5zAy0/34s2s/bhYZrm61xM9aZkTGoOZEXT+u29/girTujX5xzQrIHPzpdjupZWHqWp3tATRO019QpBxOwdN7Dy2D08tayw29C0TWFYfuSuWleSlKIbA/tuxBeeTxDxw6GSGRM++Osyfj0RiZXH7mLcL2ew/oz6DZ7SCXBxq4AlByNw6m5SmV0VBn9/HD8dvYe2s/fhySUncOpuEuQKQetI88XSsktuSmRoGZuicB/Nmx3r/otGdHI2fjyiPpOBTTmZ5Pmi+J9edhIXogtvfn134DaG/nACs3fqvrE6YvlpndusFWu6iYiI9NA3pBa2h8Uh0FN7H73SPhjQFKnZBWpNW3Xp0cQXZ6M0B6z66pnWiEjIwEyV+cxVp1orb1A2AKjtLlX7wT3nqVZo9P/27jysqSvvA/g3gYQdwr7JIiKgCAiKyCJuCKi11dq61qq17tY6Vat2U9vO6DhL27fTxXc6o77TjnbVTkdra62oVXTUCpaqKLi1U3BHoCgiOe8fmGtCFhIkEuT7eR6fh9x7c3MuOYb87jnn92uiTQo7GaL83bDi4Th08HRG7whvXK6uxcv/PiqtzzQUvPzhkQSdxw5GStQ0laTIz810UN6SmjuFs625WnMTqSu+RbmB962lPL3hsDQKaasar0MHGrKuW9vdZnEH9Mvfnb1cg32nzMzWrqW4vErK4j8xNRzRAW5Yo7VOvbHGI92PvLtXZ+mNJmjVzkugyc5uiCbIBgA3rTJrz35yBJ0NJJA05uNDutPCM36/QwqCzbXg40IYGqy9VS+w80TDTB7Nte4/1fAZralScFDrM1tzY2LE23tQqDU75ZNGbfyu5BLGJN9Z437o7BXsO3UZvSO89erQj/vrfrg62KPgpUFGb9hqHCurxLi/7sdvsqLg5mhvNP9DRU0d/N0dcfXXm3hhU5HBY678ehPerro3PrXfM413d5YaLd+oVgu9aeevbj6GjbPSpZsS/9h31mazqVsDg24iIiIjEkNVOHyuAnHBHnhleDd0DXTHA2YmeJkzoLPZr/NY7zC8v/8sBncL1Nk+LiXUyDMaaH8RG3R7VHnb7RGXt8YlwdXRHl/9WI5/3s7Y3dHHBdmxhkefAeDz2Wl4f08pFg3tBp9Gga+3qwPeGJMoPQ7zdjZQiqrhi6dm2vvQuABkd/XHU7enLWo0FeL4aE2htzaFnf437tQIbymBU3ZXf2kUqy2bsu5AkzkK7tb3Nh5wA7pJ0DSMlehrC5pK2mbI8fI70581085N3Qj7tfYW6tUC/7vrlME147fU6mZn+24cHJ68i3wJlgbcGoaavmTjD0ZLpQHAe7tP4dXNx6TH35+rQOWNOp2A2xjtqgRnLtdgzP/uw8nfDjYYwFbX3kJNXT3cGwXd12oMj1K/9s0JRPm7osxIM67W3ETljTokvrLNaPsu/3oTndGQP+HJdQcxtlco/nfXKZ1jrvx602SixJQV2/USSh4+V4EfmrFc5n7BoJuIiMiI1Y/1wIYDP2FMcgjcHRWY3rdT009qBk8XJfYtGWjxeu9b9WqMSQ7BfyuuY/VjPfB7rfJJQ+MbAvjjWusLv53f1+RrxAV7YFFWOLxcmg56lz0YCx9XBzyeGo7nNv6AXyquo1twQ9KvzXMzcOJ8NZLDPSGTyfDS50W42uhLYmaUL3ad0F/fmh7p3WSyOA8nBYYlBOL9fXdf/svQCNKTfTri+3NXUa8WCDJzZoOts3bATa2jOTc6tNfFP/z2XqyZlGzy+ItVtVi394zBgBtomCnwr8JfDO5rSpWBEVRbYDDg1vro1A64NV7YaHjkuLGM3+/Q21ZXr0a9kRsX6/efQ0KICpuPlGFhbjTcHRXYcMD4Z5+7iXruFTV1OmvaDam5eQuXqmvx569P4MjP13DkZ91ZGrW36rHlhzKT5zBWweH5TebP+LB2foh7jUE3ERGREX7ujpg70PwR67vRnARrdWqBlSPjpcdzB3RGTW09hsTdGTGflB6OS9W16B/j1+wkboYEejjhtyPiAAAfT09FnfpOjWaVsxK9Ot5Zp64X2Apg7aRkRDy3RWfzgBg//G1iT8hkMkxMDcMv125II/faCl4aBJlMhheGdkXMi1ubfQ0KOxnsDUwvd1LaYe/iAbC3k6PkQrXB0VFtUf6u8HF1wF4z62i3RfEdPJqV1I1s3+S1B0zu/+2WYyaXslTU1OllezeXOdnZbUVTn57fHGv+jJic13ch2t/N4D7tcmb/2HcWf328p8nZQtpT9hurqLnZZM6MN745aXLE/s9fn8DqRiPf5rLkMyR15Q5sn53UrNexRUykRkRE1EY5NkrC4+Jgj1eGd0NqJ29pm4O9HZ4f2hVpnSxPumYuudx09vPGU08DPBx11qZr5MYGSDcGlj/UDX99/E5tde2SM5pjHA2sGd+3ZCBWjYzXqdn78kOxeNrAzZONs9IN1ne2l8vh7eoADycFeoR56mQRDvO+Mwo/d0AkhsYF4ounMvDuhB5mrfdfPDgGwSon/PHRhCaPvZeSQlUm9zsrW76WMLUdpjKY/++uU80uo1fZRoLun67UNJnFveZm80dmf7py3WA1CEOm/t9Bk1O7TU1qWfzZD3jr2xLjBwBNTpH/YP/dzzAyx7XrdSan+Lc1HOkmIiJqY14d3g0fH/oZs/tbZ7p7S3s8NRx/3nZCeqzJur5sWFe8sf0k/jIuCTfr1ejb2dfYKZAc7okhcYGIbJRo6Ys5GRj2l++kxwEejhiVHIJRySHYfuw8QrycEeXvhsobddh54iIKbtc0dlLYoVuwh049YY3GydW0a/x+80xfvLf7NDIifRCnVUPbwd4O3y3qj9W7Thn9QpzQwQNPpHfEjNvLFO42m3VjKx6O06thba7lD3aTfo//nJqCk+erMbCLnzQV1kVpO18Z+3T2we6Tl1q7GdQCbHV6eWN9VulPCbdVhpbtaNt+3Lzg3pibTdx8sNROE+29mxsZtsZ2PkGJiIjILI/1DsNjvcNauxlmm5yuG3RrRqonpXfExLRws6a9B3k44YF4/SR2cR08kBsbgK0/luuN1mqXL3N3VGDT7HSEL94M4M6aTUPTyxtvC9eqx66wk2NmP8M3O2QymckvpJ/PyTC6ryVE+btix4J+6P/HPIufG+nnijMrh0qP0zr5oPzanUznzibKELWEEYnBOHP5V7Oyn0f4uDDovk/sOmk6QCTb05JlAa/+ehMT//4fo/t/vY+Cbk4vJyIiIqtyc1QgOdzT4L6mAu4PnkzBqJ4dMDfL+Nr6VY/G49XhutPRjZneNwIA8OIDDTXKPZz0kw41TiTXLdgDr4/ujk9mpDZ5/pgAw+sylQbWxH42Kw3eZiStA4DXR3fX29Z4yre/uyM6+rhg35KBZp1Tm6F6u9pvjak1vf8zRr9tpqyZnIzf3c4HoKFyVkDRVNHf29xMJIq6l+YPikLPMMP9ujlig9zxSI8OLXa+tqCZSc+pDRmRGGx839t7TD53wecncN6KZQ7vJQbdREREZHWvje6O3NgAswJXbemRPlj1SILJjLzujgo81jtMr7asIYtzY7B38QBMSA0H0JD0LdBDtzyaoURDwxOD0TPcS297Y4O6+mPVyHhsmdsHu5/tL23vYiAYTwr1xEvDukqPP5zW2+h5H0wIwqbZ6TrbtAPhPz6agA6eDevNAzz065yve6KXyXbbGbj54efmgF4dvZAR6aNzI0L751eHdsID8YGYZWT0v7EQLyekd/JB1yB3ne119Wqza6a7mkgUZYy7geeEeOmvwXc1c0T/gfhAjO8dBnsDJeeAhlkHX8zJQLDKCUsGx0jbXxjaxeg5N8/tg99rJUYEoJOb4G6Zk3PAFrXXfAIdPFv2/VrxcFzTB1nB2F6hGBDjZ3Dfmcs1BrdrHD9fYzDvRlt0f1wFERER2bQOns54d0IPswJXa5LJZHplwL76TaZOUGoqKZw55x+VHIKuQe4I8XLGv+akY1hCEP4yznAW3mHxQZg7sDP+PqknUiK88T9jE+HWKPCb0DsMcrkM3UNUOpmJtUfPTY2Q9urohb5RxtfLA/rr2DXX8uG03vjHlF46U+43z83Ayw/FYsvcDOR2aVifb2iafmPdQ1TYMb8flPZyND68tk6tF8A+nBhssN3GAuMfl+cYfe3diwboXGOghyN8DNyk2fZMpqlLANAwwv2XcUnwclEaDQhkkCGugwf2LB6gU2qwqVrS2m2clBaOT2emYe/iASavDQCeHtgZg7r6622P8LmzNOJ3Vg66zL1pYglnpZ1OhQZrcVTI0YLFHVqEOTcRAWD5g7GI9nfD4G4BJo8z1N/vBXcne7NmIRnj6WwbM1vuFoNuIiIiatfcHRXoG+WL/3uiF7bO69Oi547voMKbYxMRYqT2uFwuwzODojAgpiFgejAhCIVLs1H6uyFNntucmwM+rg5YP9X4CLqGsWn+MpmsoYRbWjhclHYYkxyCQA8nPJ4arjOV3sFAJnkAOpnflfZyqXxctyAPZETeyah/s16tk2H/q3mZ+PPo7noj9ANj/OBk5LVcHOyxZnIyZvXrpDcrwMNJgbwF/aTHCju5XgmoA89nIdBD94bM9L4RSAxV4Yn0jti5sB8+mZGKWf0jpf3m3GwAgE63k/FpB8YDYvyQaCJrfEyAG+zkDTeJXBzssdJI0Pxojw74zaAojEzSn8a7fX5fbJqdjk9mpCLc23AftMQqEwHw9mf6mnWO9U+annWh7YunMpCglbDQmLjgpo8xpXBpNqZn2lZiSpWBpS+GdPZzxVe/yTQ5jRvQryJxr7go7e/qhkxLlrpsTQy6iYiIiABkRvkiJsC96QOtTC6XGf+SqvW92dA68cZ8XJXSucb2Cml2m/zdHVGwNNvoqGPjtfGOCjkW5cYYrXMvl8vw/pMp0uPaOjWmZHTEq8O7IW9BP0QbmI4/vHsQ/jYpGZU3jJeZ6h/th2dzY9A9RIXCl7Lh4+qA4d0bEvBp3/gY1NUf8kZf5rXL0mksGdwFG2el46VhXRHm7YKe4V46741eDXojPp+TgW+eyURiqCfGpYQCAOYMiDR7HTsAjOkViuOv5GJWv074bFaatF0z3T4nNgAbZ6WhcGk21k5OlurZdw9RoWe4F1TO5uUPMGVUcggC3PWXLwBAuNaoemMdb+9bNjgCKRHeKHwpW9qmYShvgJPCzqyA7Yunmp+kcFxKKBzs7fBsTnSzz2GuBxP0k0Ea4+1q3vulueGlXV3r0AtZesc1NwFaZhOzZDQURpZaOCgYbgIMuomIiIhsmtCKtFMiGqbne7koobRwrePLD3W7q3aYWlupHXRvmNYbx17ONZrl3ZDaW/VQ2svxWO8wo8Gb5rdQed28MlMezgrsWzIAr2kloft8djpm9uuEhTnROtOJtafna0ZzF5oRhBkLNLJjdad6uzrYI9Kv4UbCb4d3Q+HSbCSFekJhb9konqPCDs/mxiAp9E4CN83NA5lMhsRQT3g4KdAv2k8vyDa0rt2UmAA37Fk8QG/7jgX98GRGR4vOtWl2OjZMTcHQrg2zGTycFTrBtNJejsKl2RjVU3eZRIC7o85xk9PDm3wtPwM3TwDo5W7Q0JxeLpch2t9wIkRA9732N5D3oamA+u3xSXj5oViTx2gzdy27pmrD9bo7/y+08y4o7eRIDvdEVhfdPmlsxkhjppIoAoCboz2OLMvGyd8anp1zN8t17icMuomIiIhsmPas0FWPJOCpAZHYOCsNs29Pcx4aF2j0udpTMxV2crNGx5tDczPA1cEevSO8DU8JNTG71dj0ewDStGjNdY6xYMTe3k6u05aEEBUW5cbAUWEHDyethHDD79yQGJUcgoKXBkm/X1MM3YhY8XAc5gww/lyZTCbdpHhuSBco7GSYa+B4cycDmztzVyaTIT3S28yzAm+MSUSwygn/nJqCQA9HvHd7Xa6T0k4nL0LvCC88NyRG57l2chk+m5WGuGAPfDitNzycFOjV0UvnvRiTfOd9vFWvhqPCDqseScDhFwch3NsZ0/tGNMz60HrOzH6dsGZSssFs/hqN35O0Tg3XvOxBwwGvTGuhwV/GJUqVFgI9HDEkrmGd9APxgdgw7U4SSC8XB50bD3kL+uF/xiYabRPQkFtB5aw0e5r/rXqB96ek4PkhxpPvrZ7QQ/qddva7c8NA+/f8dFZnfDwjDY4KO2mZA9AwJX/+oCi8M95wvgkNtdp0TzzwfJbJRJdNBe3tBet0ExEREbURXi5KzM9uGIEN83ZBfIf+eonhgIZAaN+pKxh/eyqzhnYd8cwoX+w60TJ1kv3cHLFvyUA4Oxgf1TI0ffvjGanYePi/0jUZ8sVTGTh18VfE317b6+/uiOJXcxH9wlYADYH+uieSLW7zsge74mLVDTyR0RGOjUb9zJ2K3XiKNNAwam5uxuXYIA/8uDz3rm6GmJtwCwD+8UQKIp7bIj12Utihg6cTTl6o1jtWcwlpnXyQ36gMnfbos3YwqiFDQ3Z+U9O+J6WF49XNxwDoTo32dFEib+GdzP/a+5R2cvSP8cOxskqj59UekX5/SgrSI71x7XodnLRGjp0Udrhe11ADWntmRWd/N3w8Iw3Xb9bD7vYyj1WP1Osl76u9VY+UCG+8991pAOYlkdMEnz6uDk1m7QaAunqBjM4+6Bnuid9uOaa3f+3kZPSLvpMVvFuwB9ZOTpaqGGhoj/Bvmp2OuGVfAwBuqdV46vbyj5gANxwvr5KOc1HaSTWybzURdDf+v9OYoaC7T2cf7D55SW/74RcHwdnBDtuOnser/z6KF7Itm1Fhyxh0ExEREdkwQ0G1hrER4rWTe6G4vEoKVDXs5TLpS/TUPh1bLOgGDJcqA4C/Pt4T/9h3Vqc8mkZyuBeSm8ho7+aoQEKj0lkO9nY4s3Jos9sKNGTU/3xO89cCA8D0zE54J68UtbdvZrz4QFeLSxw1N+B+bXQCth09j0lp4WY/Ry6X4Q+PxGP7sQtYNDgGXi5KlF+7gYfe+g6Pp4bj//LP4EZdw7WYyrslb4FM5fZ2ciSFqvD9uQqTmbfrtRqieV1Tpd06+7tJQW1cBw/IZDKonJU6icSWPxSLYJUT8oovYELvML1zaAfohl4rWOUEF61jjJWO01DayeGsbDhPVld/HDx71eTxQEMZPc1zNT6ZkYpH3s0HAL2cBAB0gvB1T/TC/lOX8VD3OwnWtGvc12sF02+NT8LAP+2UHscGe+A/p68AANQWJGCbPygKf9p2Qmdb41kvubEBeOexJHRcsgWNed6eFv9AfBCGdAvAlStXzH5tW8egm4iIiMgGrZmcjG1Hz2OKhetngYbRp8aBKtDwRXz+R4X43cPdkBHpg8npulnIrWFQV3+D5azuB05KO/z18Z54/O//AYBmvVeNBXk44pdrN5pMYDUisQNGJBovFWfMoz1D8GjPO1O7PZwUOLI0B0p7OWb3j0TC8oaRUFOhlmMTNwrMTTj918d7YvMPZTqBYWOB7o5IClVBaS+Xyum5GAiE9ywegJPnq9DB0wnbjp5HUqhKJ9eAdvCnVgukR/ogXSuDvjn+OTUF7+0+jeUPxuLG7ZFy4E6d+/enpGDhJ4VY9mAspv/jEADg309lINzHRRoNfzKjI27Vq5Ee6YNPDv2MD/afM/haMq215umR3rhYVYvuISq8MLQLfvjvtSbb3jfK12DJvemZEVi96xQWD74zbb2TrytOrxiCDQd+QlywB7xclFjx5XE8kR6OP3xVbPbv56mBnbEu/wwuVRsvjSeT3T8ZyS3BoJuIiIjIBvWP9kN/rZGrlpAe6YN9z92ZKrx0mPmJnciwPp198JusKHQJbJmbF98u6IeqG7cMTse3Fs1ou6NWpmlTI5zDEoLw4YGf0DvC8BpxmV5BNsO8XR3weGq4yWPkchk+ndmQrV0TrBkbfQ6+PStk7+IBOsnEGmtixrRRaZ18pNJ2V7VqrmtG4zM6N0zFF0IgPdIb1bX16BrorjMzwN5OjjkDGqZ17ym5M8U6q4sfBnX1R0VNHdbsOYMFWksu3p+SAiEafhdP9oloXuNvWzKkC+YMiNQZ9QYafrdje91ZjvLm7TXqj/TogL2ll/XO0y/aF4tyY/S2/+GRBCz69AguVNVa1K4ZfW2rZFtLY9BNRERERNRMMpkMT2cZLo3WHI4KuybXyVqLdgkzU7OKHRV2+GRmmvEDWnggs/HIaFNT8k0tyQB0p6w3l8pZgRAvJ6jVgG+jdfUymQzvT0mRfjZGeynCexPv5CWYlhmh8zyZTGb27AFzNA64TRmRGIxnPiqUHo9JDsH1unq8Nqq7wWUG/WP8MC0zQlqvb47S3w25q1rebQHTyRERERERkU4QZawetymakfK4YI8mjrx7mvJinf1cLX5u1xaYlSCTybBjfj9sn9/XYL32hkDZdCA5OjkEvm4OeqXSbGn6deO2rBwZjzfGJJpc1/9g9yDIZcCAGOMzdTRr4r1clPd9wA1wpJuIiIiIiG7bPr8vrt+sl5JaWeKLORlYu/eMyZJpLWXpsFh0D/FEVlfzl2B880wmTl+qQY8w08n7zGVvJ8fdlKFWOSuxb8lAmw86H04Kxmff/xfDmqhFruHn5oijL+cazFyuieE/nJ6KP35djGdz9Keo348YdBMREREREYCGpFrN1dnfDb8dEdeCrTHOxcEe4xqVxGtKpJ8bIv2smzjQUrYecAPA70bE4YH4QKRGmJ94rvESCV83B1ysqkV214ZM9Q0lznq1aDttGYNuIiIiIiIiMshRYYcBMXdXgeCreZk4+ksl0iMNJ9+73zHoJiIiIiIiIqvxclEio7NlJdruJ0ykRkRERERERGQlDLqJiIiIiIiIrMTmg+5ly5ZJKfc1/2Ji2keWOyIiIiIiImrb2sSa7tjYWHzzzTfSY3v7NtFsIiIiIiIiaufaRPRqb2+PgICA1m4GERERERERkUVsfno5AJw8eRJBQUGIiIjA+PHjce7cudZuEhEREREREVGTbH6kOyUlBWvXrkV0dDTKysqwfPly9OnTB0VFRXBz0y9uX1tbi9raWulxZWUlAECtVkOtVt+zdltKrVZDCGHTbSTbwf5C5mJfIUuwv5Al2F/IEuwvZIm20l/MbZ/NB92DBw+Wfo6Pj0dKSgrCwsLw0UcfYcqUKXrHr1ixAsuXL9fbfvXqVdy6dcuqbb0barUaVVVVEEJALm8TExCoFbG/kLnYV8gS7C9kCfYXsgT7C1mirfSXqqoqs46z+aC7MZVKhaioKJSUlBjcv2TJEjzzzDPS48rKSoSEhMDT0xPu7u73qpkWU6vVkMlk8PT0tOmORbaB/YXMxb5ClmB/IUuwv5Al2F/IEm2lv5ib4LvNBd3V1dUoLS3FhAkTDO53cHCAg4OD3na5XG7TbxgAyGSyNtFOsg3sL2Qu9hWyBPsLWYL9hSzB/kKWaAv9xdy22e4V3LZgwQLs3LkTZ86cwd69ezFixAjY2dlh7Nixrd00IiIiIiIiIpNsfqT7559/xtixY3H58mX4+voiIyMD+/btg6+vb2s3jYiIiIiIiMgkmw+6N2zY0NpNICIiIiIiImoWm59eTkRERERERNRWMegmIiIiIiIishKbn15+t4QQABpKh9kyTS06e3t7m87QR7aB/YXMxb5ClmB/IUuwv5Al2F/IEm2lv2hiTE3Macx9H3RrCpaHhIS0ckuIiIiIiIjoflNVVQUPDw+j+2WiqbC8jVOr1fjll1/g5uYGmUzW2s0xqrKyEiEhIfjpp5/g7u7e2s0hG8f+QuZiXyFLsL+QJdhfyBLsL2SJttJfhBCoqqpCUFCQyRH5+36kWy6Xo0OHDq3dDLO5u7vbdMci28L+QuZiXyFLsL+QJdhfyBLsL2SJttBfTI1wa9juBHkiIiIiIiKiNo5BNxEREREREZGVMOi2EQ4ODli6dCkcHBxauynUBrC/kLnYV8gS7C9kCfYXsgT7C1nifusv930iNSIiIiIiIqLWwpFuIiIiIiIiIith0E1ERERERERkJQy6iYiIiIiIiKyEQbcNeOuttxAeHg5HR0ekpKTgP//5T2s3iVrBsmXLIJPJdP7FxMRI+2/cuIHZs2fD29sbrq6uGDlyJM6fP69zjnPnzmHo0KFwdnaGn58fFi5ciFu3bt3rS6EWtmvXLgwbNgxBQUGQyWTYtGmTzn4hBF566SUEBgbCyckJWVlZOHnypM4xV65cwfjx4+Hu7g6VSoUpU6agurpa55gjR46gT58+cHR0REhICFatWmXtSyMraKq/TJo0Se+zJjc3V+cY9pf2Y8WKFUhOToabmxv8/PwwfPhwFBcX6xzTUn9/8vLykJSUBAcHB0RGRmLt2rXWvjxqYeb0l379+ul9xsyYMUPnGPaX9uGdd95BfHy8VGs7NTUVX375pbS/XX22CGpVGzZsEEqlUvz9738XP/74o5g6dapQqVTi/Pnzrd00useWLl0qYmNjRVlZmfTv4sWL0v4ZM2aIkJAQsX37dnHw4EHRu3dvkZaWJu2/deuW6Natm8jKyhKHDx8WW7ZsET4+PmLJkiWtcTnUgrZs2SKef/558dlnnwkAYuPGjTr7V65cKTw8PMSmTZtEYWGhePDBB0XHjh3F9evXpWNyc3NFQkKC2Ldvn9i9e7eIjIwUY8eOlfZfu3ZN+Pv7i/Hjx4uioiKxfv164eTkJFavXn2vLpNaSFP9ZeLEiSI3N1fns+bKlSs6x7C/tB85OTlizZo1oqioSBQUFIghQ4aI0NBQUV1dLR3TEn9/Tp06JZydncUzzzwjjh49Kt58801hZ2cntm7dek+vl+6OOf2lb9++YurUqTqfMdeuXZP2s7+0H//617/E5s2bxYkTJ0RxcbF47rnnhEKhEEVFRUKI9vXZwqC7lfXq1UvMnj1belxfXy+CgoLEihUrWrFV1BqWLl0qEhISDO6rqKgQCoVCfPzxx9K2Y8eOCQAiPz9fCNHwRVsul4vy8nLpmHfeeUe4u7uL2tpaq7ad7p3GQZRarRYBAQHiD3/4g7StoqJCODg4iPXr1wshhDh69KgAIA4cOCAd8+WXXwqZTCb++9//CiGEePvtt4Wnp6dOX1m0aJGIjo628hWRNRkLuh966CGjz2F/ad8uXLggAIidO3cKIVru78+zzz4rYmNjdV5r9OjRIicnx9qXRFbUuL8I0RB0P/3000afw/7Svnl6eor33nuv3X22cHp5K7p58yYOHTqErKwsaZtcLkdWVhby8/NbsWXUWk6ePImgoCBERERg/PjxOHfuHADg0KFDqKur0+krMTExCA0NlfpKfn4+4uLi4O/vLx2Tk5ODyspK/Pjjj/f2QuieOX36NMrLy3X6hoeHB1JSUnT6hkqlQs+ePaVjsrKyIJfLsX//fumYzMxMKJVK6ZicnBwUFxfj6tWr9+hq6F7Jy8uDn58foqOjMXPmTFy+fFnax/7Svl27dg0A4OXlBaDl/v7k5+frnENzDL/vtG2N+4vGBx98AB8fH3Tr1g1LlixBTU2NtI/9pX2qr6/Hhg0b8OuvvyI1NbXdfbbYt3YD2rNLly6hvr5epyMBgL+/P44fP95KraLWkpKSgrVr1yI6OhplZWVYvnw5+vTpg6KiIpSXl0OpVEKlUuk8x9/fH+Xl5QCA8vJyg31Js4/uT5r31tB7r903/Pz8dPbb29vDy8tL55iOHTvqnUOzz9PT0yrtp3svNzcXDz/8MDp27IjS0lI899xzGDx4MPLz82FnZ8f+0o6p1WrMmzcP6enp6NatGwC02N8fY8dUVlbi+vXrcHJyssYlkRUZ6i8AMG7cOISFhSEoKAhHjhzBokWLUFxcjM8++wwA+0t788MPPyA1NRU3btyAq6srNm7ciK5du6KgoKBdfbYw6CayEYMHD5Z+jo+PR0pKCsLCwvDRRx/ZzAcGEbV9Y8aMkX6Oi4tDfHw8OnXqhLy8PAwcOLAVW0atbfbs2SgqKsJ3333X2k2hNsBYf5k2bZr0c1xcHAIDAzFw4ECUlpaiU6dO97qZ1Mqio6NRUFCAa9eu4ZNPPsHEiROxc+fO1m7WPcfp5a3Ix8cHdnZ2eln6zp8/j4CAgFZqFdkKlUqFqKgolJSUICAgADdv3kRFRYXOMdp9JSAgwGBf0uyj+5PmvTX1ORIQEIALFy7o7L916xauXLnC/kOIiIiAj48PSkpKALC/tFdz5szBv//9b+zYsQMdOnSQtrfU3x9jx7i7u/PGchtkrL8YkpKSAgA6nzHsL+2HUqlEZGQkevTogRUrViAhIQFvvPFGu/tsYdDdipRKJXr06IHt27dL29RqNbZv347U1NRWbBnZgurqapSWliIwMBA9evSAQqHQ6SvFxcU4d+6c1FdSU1Pxww8/6HxZ3rZtG9zd3dG1a9d73n66Nzp27IiAgACdvlFZWYn9+/fr9I2KigocOnRIOubbb7+FWq2WvgylpqZi165dqKurk47Ztm0boqOjOVX4Pvfzzz/j8uXLCAwMBMD+0t4IITBnzhxs3LgR3377rd6ygZb6+5OamqpzDs0x/L7TtjTVXwwpKCgAAJ3PGPaX9kutVqO2trb9fba0dia39m7Dhg3CwcFBrF27Vhw9elRMmzZNqFQqnSx91D7Mnz9f5OXlidOnT4s9e/aIrKws4ePjIy5cuCCEaCirEBoaKr799ltx8OBBkZqaKlJTU6Xna8oqZGdni4KCArF161bh6+vLkmH3gaqqKnH48GFx+PBhAUD8+c9/FocPHxZnz54VQjSUDFOpVOLzzz8XR44cEQ899JDBkmGJiYli//794rvvvhOdO3fWKQFVUVEh/P39xYQJE0RRUZHYsGGDcHZ2ZgmoNshUf6mqqhILFiwQ+fn54vTp0+Kbb74RSUlJonPnzuLGjRvSOdhf2o+ZM2cKDw8PkZeXp1PiqaamRjqmJf7+aMr6LFy4UBw7dky89dZbNlnWh0xrqr+UlJSIl19+WRw8eFCcPn1afP755yIiIkJkZmZK52B/aT8WL14sdu7cKU6fPi2OHDkiFi9eLGQymfj666+FEO3rs4VBtw148803RWhoqFAqlaJXr15i3759rd0kagWjR48WgYGBQqlUiuDgYDF69GhRUlIi7b9+/bqYNWuW8PT0FM7OzmLEiBGirKxM5xxnzpwRgwcPFk5OTsLHx0fMnz9f1NXV3etLoRa2Y8cOAUDv38SJE4UQDWXDXnzxReHv7y8cHBzEwIEDRXFxsc45Ll++LMaOHStcXV2Fu7u7mDx5sqiqqtI5prCwUGRkZAgHBwcRHBwsVq5cea8ukVqQqf5SU1MjsrOzha+vr1AoFCIsLExMnTpV70Yv+0v7YaivABBr1qyRjmmpvz87duwQ3bt3F0qlUkREROi8BrUNTfWXc+fOiczMTOHl5SUcHBxEZGSkWLhwoU6dbiHYX9qLJ554QoSFhQmlUil8fX3FwIEDpYBbiPb12SITQoh7N65ORERERERE1H5wTTcRERERERGRlTDoJiIiIiIiIrISBt1EREREREREVsKgm4iIiIiIiMhKGHQTERERERERWQmDbiIiIiIiIiIrYdBNREREREREZCUMuomIiIiIiIishEE3ERFRGxAeHo7XX3/9rs6xbNkydO/evUXaQ0REROaRCSFEazeCiIiIGqxduxbz5s1DRUWFzvaLFy/CxcUFzs7OzT53dXU1amtr4e3tfZetbDnLli3Dpk2bUFBQ0NpNISIisgr71m4AERERNc3X1/euz+Hq6gpXV9cWaI1hN2/ehFKptNr5iYiI2iJOLyciImohtbW1mDt3Lvz8/ODo6IiMjAwcOHBA2p+XlweZTIbNmzcjPj4ejo6O6N27N4qKiqT9kydPxrVr1yCTySCTybBs2TIA+tPLZTIZVq9ejQceeADOzs7o0qUL8vPzUVJSgn79+sHFxQVpaWkoLS2VntN4ernmNbT/hYeHS/uLioowePBguLq6wt/fHxMmTMClS5ek/f369cOcOXMwb948+Pj4ICcnx+DvJS8vD7169YKLiwtUKhXS09Nx9uxZrF27FsuXL0dhYaH0+mvXrgUAVFRU4Mknn4Svry/c3d0xYMAAFBYW6l3L6tWrERISAmdnZ4waNQrXrl2z9G0jIiKyKgbdRERELeTZZ5/Fp59+inXr1uH7779HZGQkcnJycOXKFZ3jFi5ciD/96U84cOAAfH19MWzYMNTV1SEtLQ2vv/463N3dUVZWhrKyMixYsMDo673yyit4/PHHUVBQgJiYGIwbNw7Tp0/HkiVLcPDgQQghMGfOHKPP17xGWVkZSkpKEBkZiczMTAANQe+AAQOQmJiIgwcPYuvWrTh//jxGjRqlc45169ZBqVRiz549ePfdd/Ve49atWxg+fDj69u2LI0eOID8/H9OmTYNMJsPo0aMxf/58xMbGSu0YPXo0AODRRx/FhQsX8OWXX+LQoUNISkrCwIEDdX6XJSUl+Oijj/DFF19g69atOHz4MGbNmtX0G0VERHQvCSIiIrpr1dXVQqFQiA8++EDadvPmTREUFCRWrVolhBBix44dAoDYsGGDdMzly5eFk5OT+PDDD4UQQqxZs0Z4eHjonT8sLEy89tpr0mMA4oUXXpAe5+fnCwDib3/7m7Rt/fr1wtHRUXq8dOlSkZCQoHdutVotRowYIXr06CFqamqEEEK88sorIjs7W+e4n376SQAQxcXFQggh+vbtKxITE03+Xi5fviwAiLy8PIP7DbVp9+7dwt3dXdy4cUNne6dOncTq1aul59nZ2Ymff/5Z2v/ll18KuVwuysrKTLaJiIjoXuKabiIiohZQWlqKuro6pKenS9sUCgV69eqFY8eO6Rybmpoq/ezl5YXo6Gi9Y8wRHx8v/ezv7w8AiIuL09l248YNVFZWwt3d3eh5nnvuOeTn5+PgwYNwcnICABQWFmLHjh0G14CXlpYiKioKANCjRw+TbfTy8sKkSZOQk5ODQYMGISsrC6NGjUJgYKDR5xQWFqK6ulov4dv169d1psuHhoYiODhYepyamgq1Wo3i4mIEBASYbBcREdG9wqCbiIiojVIoFNLPMpnM6Da1Wm30HO+//z5ee+015OXl6QSw1dXVGDZsGH7/+9/rPUc7YHZxcWmynWvWrMHcuXOxdetWfPjhh3jhhRewbds29O7d2+Dx1dXVCAwMRF5ent4+lUrV5OsRERHZEgbdRERELaBTp07S2uawsDAAQF1dHQ4cOIB58+bpHLtv3z6EhoYCAK5evYoTJ06gS5cuAAClUon6+vp70ub8/Hw8+eSTWL16tV4AnJSUhE8//RTh4eGwt7/7rwuJiYlITEzEkiVLkJqain/+85/o3bu3wetNSkpCeXk57O3tdRK7NXbu3Dn88ssvCAoKAtDwe5XL5YiOjr7r9hIREbUUJlIjIiJqAS4uLpg5cyYWLlyIrVu34ujRo5g6dSpqamowZcoUnWNffvllbN++HUVFRZg0aRJ8fHwwfPhwAA1Zyqurq7F9+3ZcunQJNTU1VmlveXk5RowYgTFjxiAnJwfl5eUoLy/HxYsXAQCzZ8/GlStXMHbsWBw4cAClpaX46quvMHnyZItuCpw+fRpLlixBfn4+zp49i6+//honT56UbjKEh4fj9OnTKCgowKVLl1BbW4usrCykpqZi+PDh+Prrr3HmzBns3bsXzz//PA4ePCid29HRERMnTkRhYSF2796NuXPnYtSoUZxaTkRENoVBNxERUQtZuXIlRo4ciQkTJiApKQklJSX46quv4OnpqXfc008/jR49eqC8vBxffPGFVN86LS0NM2bMwOjRo+Hr64tVq1ZZpa3Hjx/H+fPnsW7dOgQGBkr/kpOTAQBBQUHYs2cP6uvrkZ2djbi4OMybNw8qlQpyuflfH5ydnXH8+HGMHDkSUVFRmDZtGmbPno3p06cDAEaOHInc3Fz0798fvr6+WL9+PWQyGbZs2YLMzExMnjwZUVFRGDNmDM6ePSutXQeAyMhIPPzwwxgyZAiys7MRHx+Pt99+u2V/UURERHdJJoQQrd0IIiKi9iAvLw/9+/fH1atXuTb5Li1btgybNm1CQUFBazeFiIjIJI50ExEREREREVkJg24iIiIiIiIiK+H0ciIiIiIiIiIr4Ug3ERERERERkZUw6CYiIiIiIiKyEgbdRERERERERFbCoJuIiIiIiIjIShh0ExEREREREVkJg24iIiIiIiIiK2HQTURERERERGQlDLqJiIiIiIiIrIRBNxEREREREZGV/D8VCK4J6pprywAAAABJRU5ErkJggg==\n","text/plain":["
"]},"metadata":{},"output_type":"display_data"}],"source":["\n","# 13. LOSS GRAPH\n","plt.figure(figsize=(10, 5))\n","plt.plot(steps, losses)\n","plt.xlabel(\"optimizer step\")\n","plt.ylabel(\"cross entropy loss\")\n","plt.title(\"DeepSeek-V3 full architecture + MTP — arXiv pretraining\")\n","plt.grid(alpha=0.25)\n","plt.tight_layout()\n","plt.savefig(LOSS_PATH, dpi=160)\n","plt.show()\n"]},{"cell_type":"code","execution_count":14,"id":"472eb8d0","metadata":{"execution":{"iopub.execute_input":"2026-08-10T20:14:32.541525Z","iopub.status.busy":"2026-08-10T20:14:32.541287Z","iopub.status.idle":"2026-08-10T20:14:41.02567Z","shell.execute_reply":"2026-08-10T20:14:41.025035Z"},"papermill":{"duration":8.504765,"end_time":"2026-08-10T20:14:41.027465+00:00","exception":false,"start_time":"2026-08-10T20:14:32.5227+00:00","status":"completed"},"tags":[]},"outputs":[{"name":"stdout","output_type":"stream","text":["====================================================================================================\n","Title: Quantum Entanglement\n","Abstract: We study the entanglement entropy of entanglement in a interacting finite-time quantum computing (LD) that are of the main body of interest in quantum information processing. We derive the eigenvalues of the entanglement measure which are expressed in terms of the probability distribution and the path integral of the entanglement. We show that the fidelity of the entanglement is derived from the state of the creation of the entanglement. The problem of this system is studied using an alternative approach in the limit of finite-time coupling which is given by the quantum limit. As a result, we obtain that the entanglement has a unique state where the effects are experimentally observed.\n","====================================================================================================\n","Title: Black Hole Thermodynamics\n","Abstract: We consider a simple class of light-earth black holes which are renormalized by a massive black hole. We give an explicit expression for the magnetic fields, and obtain the flow time of a black hole in the sense of the anisotropy which is of zero and we find that the first time is not the middle of the matter, which is the temperature of the field. The examples are discussed as well.\n","====================================================================================================\n","Title: Neural Networks and Scientific Discovery\n","Abstract: A network is a new problem in which the study of nodes is able to a common control of the information of the network to the past with the building in the literature. In this paper, we analyze the impact of the it on the individual nodes on the biology, with a new mechanism of their ability to wave packets. The finding is that it is generally now the same as the interval between the patterns of the execution time, the size of the given network, and the amount of information from the system is in its universal frame.\n"]}],"source":["\n","# 14. GENERATION\n","@torch.no_grad()\n","def generate(prompt, max_new_tokens=180, temperature=0.8, top_p=0.95):\n"," model.eval()\n","\n"," prompt_ids = tokenizer.encode(prompt, add_special_tokens=False).ids\n"," ids = [BOS_ID] + prompt_ids\n","\n"," for _ in range(max_new_tokens):\n"," context = ids[-SEQ_LEN:]\n","\n"," input_ids = torch.tensor([context], dtype=torch.long, device=DEVICE)\n"," attention_mask = torch.ones_like(input_ids)\n","\n"," with torch.autocast(\n"," device_type=DEVICE.type,\n"," dtype=amp_dtype if DEVICE.type == \"cuda\" else torch.float32,\n"," enabled=DEVICE.type == \"cuda\",\n"," ):\n"," logits = model(\n"," input_ids=input_ids,\n"," attention_mask=attention_mask,\n"," use_cache=False,\n"," ).logits[0, -1].float()\n","\n"," if temperature <= 0:\n"," next_id = int(torch.argmax(logits))\n"," else:\n"," probs = torch.softmax(logits / temperature, dim=-1)\n"," sorted_probs, sorted_ids = torch.sort(probs, descending=True)\n"," cumulative = torch.cumsum(sorted_probs, dim=-1)\n","\n"," cutoff = cumulative > top_p\n"," cutoff[1:] = cutoff[:-1].clone()\n"," cutoff[0] = False\n","\n"," sorted_probs[cutoff] = 0\n"," sorted_probs /= sorted_probs.sum()\n","\n"," pick = torch.multinomial(sorted_probs, 1)\n"," next_id = int(sorted_ids[pick])\n","\n"," ids.append(next_id)\n","\n"," if next_id == EOS_ID:\n"," break\n","\n"," model.train()\n"," return tokenizer.decode(ids, skip_special_tokens=True)\n","\n","for prompt in [\n"," \"Title: Quantum Entanglement\\nAbstract:\",\n"," \"Title: Black Hole Thermodynamics\\nAbstract:\",\n"," \"Title: Neural Networks and Scientific Discovery\\nAbstract:\",\n","]:\n"," print(\"=\" * 100)\n"," print(generate(prompt))\n"]},{"cell_type":"code","execution_count":15,"id":"de11fd01","metadata":{"execution":{"iopub.execute_input":"2026-08-10T20:14:41.064999Z","iopub.status.busy":"2026-08-10T20:14:41.064416Z","iopub.status.idle":"2026-08-10T20:14:42.518295Z","shell.execute_reply":"2026-08-10T20:14:42.517323Z"},"papermill":{"duration":1.474579,"end_time":"2026-08-10T20:14:42.520112+00:00","exception":false,"start_time":"2026-08-10T20:14:41.045533+00:00","status":"completed"},"tags":[]},"outputs":[{"name":"stdout","output_type":"stream","text":["reloaded step: 3000\n","/kaggle/working/arxiv_deepseek_v3/model.pt\n","/kaggle/working/arxiv_deepseek_v3/tokenizer.json\n","/kaggle/working/arxiv_deepseek_v3/config.json\n"]}],"source":["\n","# 15. RELOAD CHECK\n","checkpoint = torch.load(CHECKPOINT_PATH, map_location=\"cpu\")\n","\n","reload_config = DeepseekV3Config(**checkpoint[\"config\"])\n","reload_config._attn_implementation = \"eager\"\n","reload_config.use_cache = False\n","\n","reloaded = DeepseekV3ForCausalLM(reload_config)\n","reloaded.load_state_dict(checkpoint[\"model_state_dict\"])\n","reloaded = reloaded.to(DEVICE)\n","\n","print(\"reloaded step:\", checkpoint[\"step\"])\n","print(CHECKPOINT_PATH)\n","print(TOKENIZER_PATH)\n","print(CONFIG_PATH)\n"]},{"cell_type":"markdown","id":"f845adb9","metadata":{"papermill":{"duration":0.017512,"end_time":"2026-08-10T20:14:42.556257+00:00","exception":false,"start_time":"2026-08-10T20:14:42.538745+00:00","status":"completed"},"tags":[]},"source":["Saved files:\n","\n","- `/kaggle/working/arxiv_deepseek_v3/model.pt`\n","- `/kaggle/working/arxiv_deepseek_v3/config.json`\n","- `/kaggle/working/arxiv_deepseek_v3/tokenizer.json`\n","- `/kaggle/working/arxiv_deepseek_v3/tokenizer_config.json`\n","- `/kaggle/working/arxiv_deepseek_v3/training_loss.png`\n","- `/kaggle/working/deepseek_v3_exact/modeling_deepseek.py`\n","- `/kaggle/working/deepseek_v3_exact/configuration_deepseek.py`\n","\n","The saved state dict includes the appended MTP layer(s), including `enorm`, `hnorm`, `eh_proj`,\n","the MTP Transformer/MoE block, and the shared embedding/output projection paths.\n"]}],"metadata":{"accelerator":"GPU","kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.12.13"},"papermill":{"default_parameters":{},"duration":3966.689917,"end_time":"2026-08-10T20:14:45.574553+00:00","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2026-08-10T19:08:38.884636+00:00","version":"2.7.0"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceType":"datasetVersion","sourceId":18745237}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat":4,"nbformat_minor":5}