372 lines
87 KiB
Plaintext
372 lines
87 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"# 504 Batch Normalization\n",
|
|
"\n",
|
|
"View more, visit my tutorial page: https://mofanpy.com/tutorials/\n",
|
|
"My Youtube Channel: https://www.youtube.com/user/MorvanZhou\n",
|
|
"\n",
|
|
"Dependencies:\n",
|
|
"* torch: 0.1.11\n",
|
|
"* matplotlib\n",
|
|
"* numpy"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 2,
|
|
"metadata": {
|
|
"collapsed": true
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"import torch\n",
|
|
"from torch.autograd import Variable\n",
|
|
"from torch import nn\n",
|
|
"from torch.nn import init\n",
|
|
"import torch.utils.data as Data\n",
|
|
"import torch.nn.functional as F\n",
|
|
"import matplotlib.pyplot as plt\n",
|
|
"import numpy as np\n",
|
|
"%matplotlib inline\n",
|
|
"\n",
|
|
"torch.manual_seed(1) # reproducible\n",
|
|
"np.random.seed(1)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 3,
|
|
"metadata": {
|
|
"collapsed": true
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Hyper parameters\n",
|
|
"N_SAMPLES = 2000\n",
|
|
"BATCH_SIZE = 64\n",
|
|
"EPOCH = 12\n",
|
|
"LR = 0.03\n",
|
|
"N_HIDDEN = 8\n",
|
|
"ACTIVATION = F.tanh\n",
|
|
"B_INIT = -0.2 # use a bad bias constant initializer"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 4,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"<matplotlib.legend.Legend at 0x7f1a8e78f240>"
|
|
]
|
|
},
|
|
"execution_count": 4,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
},
|
|
{
|
|
"data": {
|
|
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD8CAYAAAB5Pm/hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJztvXlwpOd95/d5+kbjvgYzA8xwZsjhMaTEQ2OKFmVblmRb\n0notbeKotEllubaqWC7biZ3EtdZmq3a3svlDjje7karWdnFtZeXEhzayHakUKbYkS3ZsS1zxFocz\n4hycAxgAgxvd6LvfJ3983yZAEEcDjasbv08Vphvdb3c/8+Lt7/N7fs/vcN57DMMwjNYlst8DMAzD\nMHYXE3rDMIwWx4TeMAyjxTGhNwzDaHFM6A3DMFocE3rDMIwWx4TeMAyjxTGhNwzDaHFM6A3DMFqc\n2H4PAGBgYMCfOnVqv4dhGIbRVDz//PPT3vvBzY47EEJ/6tQpnnvuuf0ehmEYRlPhnLtRz3HmujEM\nw2hxNhV659znnHN3nHOvrniszzn3defc5fC2N3zcOec+65y74px7xTn32G4O3jAMw9iceiz6/wB8\naNVjnwK+6b0/C3wz/B3gw8DZ8Odp4Ld3ZpiGYRjGdtnUR++9/2vn3KlVD38UeF94//PAt4FfDx//\nfa/ax991zvU4545578e3OrByuczo6CiFQmGrL20qUqkUIyMjxOPx/R6KYRgtynY3Y4dWiPcEMBTe\nHwZurThuNHzsbULvnHsaWf2cPHnybR8wOjpKZ2cnp06dwjm3zWEebLz3zMzMMDo6yunTp/d7OIZh\n7BVBAMUsVMoQi0OyAyK7t2Xa8DuH1vuWu5d475/x3p/33p8fHHx7dFChUKC/v79lRR7AOUd/f3/L\nr1oMw1hBqQC3L8HkNZgd1e3tS3p8l9iu0E86544BhLd3wsfHgBMrjhsJH9sWrSzyNQ7D/9EwjJAg\ngDvXwAHpLmjr1K1Dj/tgVz52u0L/ZeCp8P5TwJdWPP6PwuibJ4CF7fjnDcMwWpJ8BpbmoFiAxWnI\nzkNxCaJJqJSgkN2Vj60nvPKPgO8A9znnRp1znwQ+DfyEc+4y8MHwd4CvAteAK8C/B35xV0a9B8zP\nz/Nbv/VbW37dRz7yEebn53dhRIZhNDWlAoxegMmr8IO/gYvfhtf/BsYuwtQbUC3LZ78L1BN18w/X\neeoDaxzrgV9qdFDbYoc3N2pC/4u/+Na5qlKpEIutf9q++tWvbvszDcNoUYJAgn7nKoy+BkEVIqHb\ndmkO+k9ArhuO3bsrH38gSiA0TKkg/1alBM6B9xBLwJEzkEht6y0/9alPcfXqVR555BHi8TipVIre\n3l4uXbrE66+/zsc+9jFu3bpFoVDgV37lV3j66aeB5XIO2WyWD3/4w7z3ve/l7/7u7xgeHuZLX/oS\nbW1tO/k/NwyjGVicgqvPQWZKBqmLQiwGsSSU8pCZgWr1wPnoDw67tLnx6U9/mrvvvpuXXnqJ3/zN\n3+SFF17gM5/5DK+//joAn/vc53j++ed57rnn+OxnP8vMzMzb3uPy5cv80i/9EhcuXKCnp4c/+ZM/\naeA/ahhGUxIEctmUMlBYhMBDpSB/fXZWPvqlOYjEILe4K0NofqEvZmXJx1dZ7vHUjm5uPP7442+J\ndf/sZz/Lww8/zBNPPMGtW7e4fPny215z+vRpHnnkEQDe9a53cf369R0Zi2EYTUQxKzGfn4JSEYIy\nuAhEo3ree0i2QyGzaxZ987tuKmW5a9bCuR3b3Ghvb3/z/re//W2+8Y1v8J3vfId0Os373ve+NWPh\nk8nkm/ej0Sj5fH5HxmIYRhNRKkJuTj5552TRV0uAl7D7BBTzkExrf3EXaH6hj8U1I66F99s+cZ2d\nnWQymTWfW1hYoLe3l3Q6zaVLl/jud7+7rc8wDOMQEJShXIFoAopzUM6xnGPqgDCQJHoMoib0a5Ps\n0MZrufBW9025oMdTHdt62/7+fp588kkeeugh2traGBoaevO5D33oQ/zO7/wODzzwAPfddx9PPPFE\no/8LwzBalqh88om2MBJwpWHqgRhEokCgEMtdoPmFPhJRdM2da9rIWB1147a/DfGHf/iHaz6eTCb5\n2te+tuZzNT/8wMAAr776ZmVnfu3Xfm3b4zAMo5mpQnsfLM2He4YOCXyoTbWChuV9jKNvChIpGL5f\nJ7EWR5/qaEjkDcMwdoRIKORz4+BXCnkARKBa0Wati+yaZrWG0INOUFvXfo/CMAxD1JI4F6fh9sXQ\nN/+2g8I9Rg+lnELDd4EDLfTe+5Yv+uXX20g2DKN5qSVxlnJw/UVYmIZycdVBobZVSjJUuwbWjyBs\nkAPr20ilUszMzLS0ENbq0adS28veNQzjAFJL4qwUYOIKjF+BShG5albil3+Saeg/qezYXeDAWvQj\nIyOMjo4yNTW130PZVWodpgzDaBHyGZgfV/GyuQlFAFYraxwY+uSjceg5rr3GwxZHH4/HreuSYRjN\nRakAo6/CrdcgPw+ZacjNg19L6APF1nf0QO9RBZBsMxx8Mw6s0BuGYTQVNZeND+S2mRuHfBZYS+RD\nogno6IfOARg+t2tRNwfWR28YhtFU5DOwNKtKlVM3VPpgI5EnKqEfPgcPvl9++l3CLHrDMIxGKRXg\n5suqNT87Glah3EjkgUQShk7ByLldFXkwoTcMw2iMIIDJK5CdgaCiDNi3RdisxKnkQTyljdhd8suv\nxFw3hmEYjVDMQn5RiU9TN0JrfiOhD6NtvIf2bug+sutDNIveMAyjESplKOZg4rISpNgs96eqeSDV\nBQ+8Tw1Hdhmz6A3DMBohEoP5CbUEdFE2F3og1Qnn3gf9e5NDYxa9YRhGQ3iVN1icrsNtA0ST0NkP\nIw/sWeFFE3rDMIxGWFqEuTE1GNkw0sbJ+k/3QPdRiO6d/JrQG4ZhbJcgUGXKSkkRNC6+qhRxjQjE\nk9AxoF6xybZd6ya1Fib0hmEY2yU7o7j54hLkc+uIPGqQlGyHeEKhlX0jEv49woTeMAxjOwSBKlOW\ni3LfBIX1j3Wh1Lb3Qt8wdPTtSfx8DYu6MQzD2A4LkzB6AcYvQjW/8bE+UIXK3uMKq9zFujZrYRa9\nYRjGVsnMwvf+DGZGIbuwycERuWziSegagrPv3vWSB2uMYPs45/4759wF59yrzrk/cs6lnHOnnXPP\nOueuOOe+4JxL7NRgDcMw9p1SCb7/F1DMb16dkigk2iTsiSScfnjPRR4aEHrn3DDw3wLnvfcPAVHg\nE8BvAP/We38PMAd8cicGahiGse+UCnD1WZi5BdlZyG1izUfCxiKd/XLZuOjejHP1MBp8fQxoc87F\ngDQwDrwf+GL4/OeBjzX4GYZhGPtPrd58IaMom9wssE6UTY1oDNLdkOyEY/es02lq99m20Hvvx4B/\nDdxEAr8APA/Me/9mO5VRYLjRQRqGYew7xaxq2WRnlQH7tmbfq4koymbwlModxHavVeBmNOK66QU+\nCpwGjgPtwIe28PqnnXPPOeeea/W+sIZhtAD5LNy5ro3YYi5s+L0eUSVHveOD8ND7tRkbS+xpSOVK\nGnHdfBB4w3s/5b0vA38KPAn0hK4cgBFgbK0Xe++f8d6f996fHxwcbGAYhmEYu0wQwNxtlSOevSmh\n36h4WaodTj2sJKnCkg49cmZPQypX0sin3gSecM6lnXMO+ADwGvAt4GfDY54CvtTYEA3DMPaRmsjP\n3oSx1yAzw6aRNule6B+G/pMwdAaG74dEaq9G/Da2HUfvvX/WOfdF4AX0v34ReAb4f4A/ds79z+Fj\nv7cTAzUMw9hzSgW4fUkCP3YRlhaguoHLxsXUOSqRhEgcBk7smxW/koYSprz3/wL4F6sevgY83sj7\nGoZh7Du1FoFzY4CHahkibuPXOAexGMSS0Dt8IEQeLDPWMAxjbYpZuWkWJqGQ02ZsMbv56+IpRdr0\nDO36EOvFhN4wDGMtSkWYuakom/wC5Jc2f000BskOOH4vtHXu/hjr5GCsKwzDMA4alSJk5qCwKMve\nbxY3H1Vy1LH7oP9g+OZrmEVvGIaxFuUiVAqwcEeumw2JqA/s8Qdg5Ny+lTpYj4Mz5RiGYRwUijm4\n/RrMTyryZrMWgal21Zjv6gf8vmXArodZ9IZhGCsp5ODVb8DkNZU8CEobH++cYuQHTspd44N9y4Bd\nD7PoDcMwagQBjL4K8xMQVKC8SdEygFibwilT7RL6nuMHyj8PZtEbhmEsU8wqbv7OG7A4BdXNfPMO\n2jqgZxgG74JEu34/YBysaccwDGM/KeZh4irk5qCymcgjSz7RBn3HFD8fTx44tw2Y0BuGYYgggPlx\nWJqHQh2JUYQZsO29UA0g3ravhcs2wlw3hmEYpQJMXoVbF2D+NhBs/pp0lzpHtffB2ceVDXsARR5M\n6A3DOOxUKnD9ebhzSwXMqhuUH66RaJeL5sgpGHnoQIs8mNAbhnGYKRXg5itw4xXIZ1T2YNMMWKf+\nr8k0tHXDyIMHWuTBhN4wjMNKEMD4ZTX6zsyoQUhudvPXRZOQbIOhs3D34xL8A44JvWEYh5O52/CD\nv4bpWypzUC1s/ppEu3zz978XuoZ0vwkwoTcM4/BRqcCrf6ns13JR2aybUWsk0nNcLpsDGkq5Fgfb\nsWQYhrEbzN2GO9fAhxuvwWZC7yTsXUMqXhaLH9hQyrUwi94wjMPH7K2wOmUxLHOwUaSNUzXK4Qdg\n8LSKl516BCLNI5/NM1LDMIydoJhTiYNiGGVTKbBh3LyLQlsXdA1CZ5/CKZtI5MGE3jCMw0QhBxf/\nSre5RfBVNrbmgVgChs6onk2TWfI1msPBZBiG0ShBAGOvQTkPmcnQv75ZclQUogllwJ58sClFHkzo\nDcM4LBSzkFuAmVGYHaujC5SDSBTiCeg5Bp0DezLM3aA5pyfDMIytUirC4h2JfLkA1Y26RgF4iXyq\nA7qONE2EzVqY0BuGcTioFFVjvpCB6iZdowCIKZSy97iafjcxzTtFGYZhbIVSUa6bTXOjIoCDKNB/\nHLqPQkfv7o9vFzGL3jCM1iUI5JuvlGFpBiolhVduRjQBbT3QfUwRN23NUepgPUzoDcNoTUoFZb9W\nSnLbXH5WdW3YxG0Ti0H7APQMwsBdcPz+pvbPQ4OuG+dcj3Pui865S865i865H3bO9Tnnvu6cuxze\nNveaxzCM5iMIJPIOiEbh+suqa+M3E/kUpHuhvRvOnIcTD0IitSdD3k0anaY+A/y/3vv7gYeBi8Cn\ngG96788C3wx/NwzD2DuKWUXWFJfgwrdg7KL6wG5IVBE2fcdlybd1Nb3Lpsa2hd451w38KPB7AN77\nkvd+Hvgo8PnwsM8DH2t0kIZhGFsin1Wd+avPwdQN9YENyhu8wCmUsv+kfPPJlKpUNrnLpkYj/4vT\nwBTwvzvnXnTO/a5zrh0Y8t6Ph8dMAEONDtIwDKNugkDVKYtLaihSWoJgk65RiXZZ8V2DCqccPANt\nzVGCuB4aEfoY8Bjw2977R4ElVrlpvPeedXKMnXNPO+eec849NzU11cAwDMMwVlDMAoGaiWRmIL9Z\nlE1EDUQiURUt6xuBRFvT1Jqvh0aEfhQY9d4/G/7+RST8k865YwDh7Z21Xuy9f8Z7f957f35wcLCB\nYRiGYawgtwi3X4fsjJKj2CQDNhKTyPePyJpvslrz9bDt/4n3fgK45Zy7L3zoA8BrwJeBp8LHngK+\n1NAIDcMw6qWQgyvPwvQNWMpAdbPsqIj6v3YegQd/HI6eheH7WyLSZiWNxtH/N8AfOOcSwDXg59Dk\n8R+dc58EbgAfb/AzDMMwNicI4OZL2oSdGQ2t+U2EPhJT16iT74R0T8tE2aymIaH33r8EnF/jqQ80\n8r6GYRhborYBe/MCTN8E5yAS2bhFoItCIgmDd0G6Q9mzLYplxhqG0dzUMmCnb8LMTcgvqhdssJFv\n3kE0DvG0/POxpHzzLYoJvWEYzUstA7ZchIVJRdlUKptnwOIk7M4r4iaeaqkom9WY0BuG0bzkM7Aw\nARPXID8H1CPyoNoIThZ9qrPlomxWY0JvGEZzUirA6AW4+SoUshL9SrW+10aiqmvTPwKnH225KJvV\nmNAbhtF8BAFMXoWlOZUf9oFq21TqsOZdVIXO2jrhrocVbdPimNAbhtFcBAFMX4cbL4WlDgpQyilR\nijos+khUFvyRM3Dq0ZZ22dQwoTcMo3koFWDiMrz+dzB1Xb9Xq2GkTR3hkZE4tPcppPLRD0EyvetD\nPgiY0BuG0RwEAdy+JL/82CUo5yXy5Tx19AeEaBK6h+DEORh5h+4fEkzoDcNoDvIZhVJOXoOgqqzW\ncoG6RN7FFEZ58iH55YfOHgqXTQ0TesMwmoPcvH7yixBxEu9cPdmsEUh3qiXgkTMw/IAmiUPE4frf\nGobRvJRLMDOu22oRinnWqYL+VlKdqi9/9w9J4Iu5lq1psx4m9IZhHEyCQLXlK2VFyiyOQyUP1bLC\nKX2dLptEEgZOQDyp92zhmjbrYUJvGMbBo1a/plJSgbJCBm5dgHS3yhxsWn4YICpxT6RV4iCRlp+/\nhWvarIcJvWEYB4ta/RqHNlAB5idh9rbuR6JQybFpzHw0qmSoRBpSXVApQizR0jVt1qN5hX7lsi4W\nh2SHypIahtHcFLOy5NNd4fc8A7dekVAHgcTasbF7PpKU9d/Wqd6vlaKOb/GaNuvRnEJfW9aVC6pa\nVy1r1h4+B6nDkQBhGC1LpSx3TbkIc2Oy5ucnITsrv3w1ADbws7uouka19ypBqmsQTjyoNoGHUOSh\nGYW+tqyrlmBpFqphzencPCxOwbkfOzTZbobRksTiipOfG1Nd+eycast7J6Nus7h57/VTKUJHLwye\nPtQiD401B98fillZ8plpwKlbu4sobKqQhdFX69uNNwzjYJLsUMZrdk6bsLk5WFqAoERdyVEgoW/r\ngoG7YODUoRZ5aEaLvlIO3TUVLcumryuuNhrVxTFRgcFT0HVkv0dqGMZ2qJRU1mDxDixMwdRNCAp1\nvjgi4y+egr7j0DciH/0hp/mEPhbX8q1cgtnLUPXahS9WdIHE4jB+BToHDv0sbhhNR801G0/o+1up\nWfER6rLmY0m1COw9HiZF+UMZZbOa5lPCZIdm69lbEHjFyZZyUFyST25hSs9lpvd7pIZhbJViVi7Y\nhdCan58Ie7/W6bJxXhb90dOaKHoOt2++RvOdgUhEfreqlwsnMyOfvQ9F34eZb+NXzFdvGM1GqajW\ngEEVfBWiEUXR1IMLSxC/6++prk3fCXPbhDSf6wYgGoO73qHGA7l5ICKrvlrRRVEuKwQz3QldR5UC\nbXH2hnHwCcphiQIPS4sy6DatMx/THl3nAJw5D52DcufGo+a2CWlOoY/FIdUeJk5E1U6slNf9aknV\n7RanZBkcuw/6T0r0j5xp+d6QhtG0BIEs+mIe7lyFal7CvyFhzHwkBj3H5ObJzkn0D2ly1Fo0p9An\nO6CwFKZFj0GlwHKaXAQiYTZdNqp+kpUijDyoTZ7h++2PbxgHjVoS5NIsZCZDl2wZfGX917gY4OSy\nHTgJ3Uegox+O3gM9R+17voLmPBNBIIEvZFTN7i250MGyFVDKaYavVmD+tnz5hex+jNgwjPWoRdoQ\nVqTMzoUlCzZLjAq0sk/3KEky1QH9J0zk16A5LfrMHVWhW5pf/5igqp/MLBzpUcJFNK4Z3/z1hrH/\n1OpVLc7A1A25YCevQWZeUXQbRdq4mJqPpNohFtN3feAkDN1tIr8GzSn0haxCr4B142uDMlTDTdq5\nMcXU+gCS7bqIzF9vGPtHzVVTyIZ9YC/KBVsuQCHHpuGU0QjE04qy6TkKD/yI+eQ3oOGz4pyLOude\ndM59Jfz9tHPuWefcFefcF5xzicaH+bZPhWpBO+sbUS3L6s/MhSKflh/PoYvMwi8NY+9Z6apZmpNf\nvpwP69OU2bBgWQ0fkctm5EG4+3ET+U3YiTPzK8DFFb//BvBvvff3AHPAJ3fgM95KW5dKkDoI/1mH\nAAqLMD8OE1chtwi3X5dLv1Iyf71h7Ae1MsSlIkxcgZkxrbIzc1DO1fEGYUORjj7oPWbumjpo6Ow4\n50aAvwf8bvi7A94PfDE85PPAxxr5jDVJtsHd74JYG5s2HwDF4eYW4I3nYPIyXHtOmz2HsKWYYew7\ntTj5yaswP6bEx0qJuix5nMKr2/vgxENw+lFzwdZBo9Pg/wb8E5Ydav3AvPdvxkSNAsNrvdA597Rz\n7jnn3HNTU1NrHbI+yQ5I94UzeZ1twSp5xedOXlWxpMkrSrwyDGNvicVlwc/dlj++tFRHvHyIiylG\n/vTDcOYxxc8bm7JtoXfO/TRwx3v//HZe771/xnt/3nt/fnBwcBtvEChypr0XqHMboFKAXFYROIsz\n5qM3jL0kCJTMWCrC/B1VpSxk5Z+vi4hcNQ9/GIYflOAbddHIdPgk8DPOuY8AKaAL+AzQ45yLhVb9\nCDDW+DBXUatJf+QuKC6Gv5fqeKGHoAiZKTUkWJgKJwrDMHaVWpRNKafv3Rvf0yZssFE/wJVEoGsA\n7n0P9AzZ5usW2faZ8t7/U+/9iPf+FPAJ4C+99/8V8C3gZ8PDngK+1PAoV1Mpq/Z8NA4nHpYrZyuU\nS6qKd+PlOicIwzC2TS3KJp+B6y/C638LMxNhI5E6/fLtPXDkHrlrhu83v/wW2Y0p8deB/945dwX5\n7H9vxz8hFlcRI1AD4Yd+XPHx9VItQXYGbr4CL35l48QrwzAao5hV8cEr31X54dlx8MX6Xx9NqDT5\nve8+9C0Bt8uOnDHv/be99z8d3r/mvX/ce3+P9/6/8H4rf9E6SXaECVBeu/UdfXD/j1G/Jyr0zRez\nkJ2GC98Kd/0Nw9hRgkCumusvKWZ+YUbFyraCA+5/EkbeYSK/TZrzrEUiMHSPqtUVl1SpMhKFobPK\nlqtH8MtFNSdZnFHm7OVn5Uc0DGNnKBW0ar78HZi6Lks+O7G194imYOg+uP9HzV3TAM0bm5RIwV3v\nhCOnwsbBAQyehAvf1o5+bn6TOtaBEqfuvAEnHlQD4usvwsg5SHVaLRzDaIQgUAjz/Lg6PuFUSnwr\nJNplzA2d1vsZ26Z5hR60jEv36AcUerW0CJf+OmwtWMdGTymrBiblIvTMK7GqcwCGz0EqvbvjN4xW\npeaXL2aV+VpYBDYoOfw2onD8Pjj1iAy5qiU3NkJzC/1qEim47z3QloapMbj8N1DIK6RyIypFGLsI\nc+Nw18P6fXEKzv2Y6uMYhrE18lnFyU9dV+mRuqPbInLDprvVMAinBkPROhMjjTVpPf9EqhM6BuH0\nO+G+J1UuoR6qoc/+xkuKyCnlYew1S6oyjK0SBDB9fbmceLHWN2ITInEZa8m0ShDnF/V495Bq2xjb\npvWEPhJRMgUR6D4ahl3Waw0EsuZvvw7ZKVichulbuuDMR2gYG1PLfJ26IWu+6hVpk1us48WhFCU7\noK0bEp3yz/ccVUMR6/3aEK3luqmRSCmpIhKV3z7VBROX2dxH6BUpEE2o2mU0ocYGS3NaPloNe8N4\nO0Gg+lFjF5Wxnp2DsUva7yrWI/JIyOMpJUZVA0gkob0bcJYFuwO0ptCDLoxIVK3F8lm1EiwsbP66\nahGys4rTd+jCi7dpY8micgzjrZQKcnG+8YICGnIL2t/KZaBaT8lhACcXaRCoU1RbB5x6DE68A9o6\nTeR3gNYVegg3Ut2yhX/r+6q1sSlV+RbH31Ap1KkbslSqZfBVlUg169447NRCKKdvKHkxv6h68vns\nFkQe+eYjMcBDIg0PfQBGHrLv1w7S2lNl5xFZ9cUs9A5D+wBE6m14VYHZm/B3X4DpmxL6Yk49aCtl\n61BlGMWsxL1aUYGypRmYn4LKUv3vEU1A71GJfd8I3P9eOHqPifwO09pCH4vBXY8ATq6Xrv6tLQOD\nEsxPKumjkJXVEY2qbEK5YB2qjMNNqajvwNI8TN2CpUydK+YVtHfrOxmLyS3a1qWChcaO0tquG5C4\nn35UF08xq4tq4kp9/noAqhL7tk69x5FTamBiHaqMw0oQyNi5+X0YvwzjVyTw1RKbNvVeSSwF7f3y\nyZcrcol6r6KFxo7S+kKf7NBmagLVs47EobAEU0U1GK+H0pLCLI+eVZZeLC6RtwvSOGyUCvLLj70G\n07eVEJVfgKBC/SIfBee1hxZPQSypyLh4QtFtFkq547S+0Nfi6u9cUzxvNAb9I0qIyk4rUsDXEXa5\nNA8zN9UNJ56G4/fK+ggCi8AxDge1uvILd8JeDkWFTwYlVDiqDlxMwh6NyQ8fT0nc23u1EWuhlLtC\n6ws9LEfdFLLyK5Zyqr0RBJCfV+beplQhM6OLO9kBfUfVfzaesggco/UJAliYgJlbqitfXApdmGGj\n77qISOBT7fLN952Awbsg0QHH7oHOLe6hGXVzOIQedAG1dUEb4N4J06PgbyjVOpIIrZJNqBahHFok\n0zflX0x3aSk7cs4uUqM1KebkqpmfgNFXYXE2LBpYYGs++YT2uvpOSuDPvlvfn1SHfXd2mcMj9Cvp\n6IcjpxUONjsOxS3UoS/lFUtfzik55Pj9isaJp7T8jMVl8Zs7x2gFCjm4+FfKIcnNKbqmlK9/f6uG\ni8gn3z8C/cPwwJMqUWLsCYdT6Gt++/EfQGoRCvNQrHcJGii00ntF41QrqqezOAl3Py5r38olGM1O\nEGhf6sqzynTtGlCfh2hMK9utEm9TDZv+U4qTj1ggw15yOIUeoL1LUTTlgjZpA9ROcMNmJSt4M7yy\n1s6wV9Uvj9+v5+5c076ALUmNZqNW1mD0glycBMp+XZrTHlXdPvmQaELW+9BpRbCVlixibY85vEKf\n7FDp04ETCg2bvqXom8wiUG/t7ECx+W/G0zvoPgYdPZo8ClntCxhGsxAEEvkbL8sIijjILoZNRPJs\nrXkI2v/qOiJhj8XlCqqULYRyjzm8Qh+JaAm5OKXyCNm5sBRxFDJTbGmTqVqEhUl9MRbuwIM/rk0n\nS6gymoUgNFoWZ7Thite+U2FMdZ+CClsX+Ziia6JxrQQKS1oM9A7bSnePObxCD9qUHbpbS8lIRPXn\nXQCTMTUM3zJeX5YL31Snqr7hHR+yYew4pYJcjZWSDJW5SW22RlNamVbLyh/ZCpG4Ml67jqgiZTSm\nzNdYEnp65GyWAAAgAElEQVSGduf/YazL4Z5WIxEJfbJDbcv6jkNbD/SeVOgk0frfywcQeF3Ipby+\nMHduaLPWMA4qtSQoh0Id40mtTMtldVqrlKBcb5EyF95Gw/ryab2Xi0BHn4IUBu/SatfYUw63RQ9v\nTaY6clq3o68pkQoUglkvQVW17F1UvsjrLyok7Z73QNp8ksYBpJiVGMficmPO3NJ3oJBVaYN6XZgu\nqh+89qU6+mVIdfRCW68mkKN3w9BZc9vsAyb0sJxMlQx0sQ+eUBZgJKYlaz0V+VxMS9xS+KXJLSgJ\naywPs2Nw/qOyagzjIJHP6vqMOJgZU5nhxTthgbItRNe4qK77dA+0tcuaH34QOrr13FHLfN1PTOhX\nUsxqqZruglOPwItfgXSvxHszy8aXlzdfKx6CMlTi0NcHxaKSTt7198MGC4ZxAAgCmLstUc8swMyo\nsl+3GicfiS+7fXqHlRTV1g3H7pVFb5mv+46d/ZVUyuBCP2MsASffKXdO74hKqroEdZ0yh6pc5he1\nmZVsU8TBwp3dHL1hbI18RoEImRmYvS1DZ7t7StWyomviCb1vZ79Cl9u6TOQPANs2L51zJ4DfB4bQ\nGu8Z7/1nnHN9wBeAU8B14OPe+7nGh7oHxOLKeAVd8PE2GDwlS2WuKxTuWTUvrmzgzgkqKpHgq3Dn\nKnQdlcWzMKnEkUhkOZytVu7YyiYYu83Ka85XYfyqBL5cgqVFRZ1tJay4RiQalgFJ6ztjLpoDRyN+\nhArwP3jvX3DOdQLPO+e+Dvxj4Jve+0875z4FfAr49caHugckO2TJlwsKBwPduggMntRGVTkH1Tqi\nEColTX/lIuSXoOcIXH1OETnH7tceQKWkFYT3VjbB2F1WhlDidS1Xy3Lb+EDuGhfRBFAvkbg6rkXi\n+p6MnFMVymRavn9LGDwwbHva9d6Pe+9fCO9ngIvAMPBR4PPhYZ8HPtboIPeMWg0cj6yeSkmV+6Jx\n+Rm7jkCirY769QBeX55qUZX+Zse1RL76PfjOF5SUFY1qckl3yd1jfWiN3WB1CGXNiCnmYHZCdZqK\nS/WX/wBtsCaSEGtTuGT/Sf2ebNeE4ZwlDB4gdmRn0Dl3CngUeBYY8t6Ph09NINdO87Ay3LL7KMzf\nlhtn6g1Z47Xomi0RyLKfuqZJI5GCiIeBU4q77x1WFqKVTTB2g5VBBrViZbNjWkkWsypWVt5ir9dU\nZ+iuiet+Z5/EvZST2FtLwANFw0LvnOsA/gT4Ve/9oqttZgLee++cWzNGyzn3NPA0wMmTJxsdxs7y\nZu36LnWoL2She0hWUTylZuHVrRZ3qoarg4Ss9uycfPfew+ytsLOOWUHGLlALMigXFVkze0t9k3ML\nclNu2S8fvlfCgY8DHrqG9BnVShiXby0BDxINCb1zLo5E/g+8938aPjzpnDvmvR93zh0D1gw18d4/\nAzwDcP78+S2Ww9tD3oyx75BFn5mRFe6AylbrfwQKuyyFDcdT3Qo/q5YVY1+LRTaMnSQWVzLf1HW5\nDKdva+N1K64aQJnigSLQUmmtSNO9isGfH4d0JxQ79P2wloAHim3/JZxM998DLnrv/82Kp74MPBXe\nfwr40vaHd4Co+e+7BhUuGYkuh2JuBV/V8rlcUtasr8rimri2vBdgGDtJskOW9uyojJPMne2JfLJN\nkTVdR2DwtMqHHL9nuYxIez+ceEiuTwsqOFA0YtE/CfzXwPedcy+Fj/2PwKeB/+ic+yRwA/h4Y0M8\nQCRScO+T+sJcexGiRbljgi0mmPiq+tTW0s2T7WrLln7CrCBjZ6mFVBJRmeHMNCzNbvFNIjJCInHo\nOyKhdxGtaouhbz+RhGP3Qbp7h/8Dxk6wbaH33v8Ny1WMVvOB7b7vgSeegMf+vpavt16RpVSqAFsI\nSwNU6TKnL2IkDFGbvK6Klyb2xk5QC6ksF2BuVHWYZm+zZZ98JALJFPQc1+ZrPKnG3j1Dy5UpKxVV\nqzQOJJaPvx3ae+Dd/wDu+SF49S/VHDw7C4UMW9ugDaNx5icVa784rfDNk++ERGK3Rm+0MjULvpCH\nidfDaK/rMDeuSJuNEv3WwsXCCBun/aUgrv2k/hMyekATSTxqbscDjAn9donElDX77o/Dt55RWdfA\nQynLlqNxKnn95Ofh/7ujFofv/s9VNtkw6qVmwecXFVUzeU19FSrlsLTBFssbuBik+2TYpLsUOHD2\ncS0IygW1zFyZ7Gcr0QOLCX2jpDvgkQ/D3/4R+gZ4Reds2ZUT0WurBZh+A/72/4Sf+AVIdFipBGNj\ngkDiPnZJyXyT12BuQuU3gu2E60bkTkx1QN8x1ZA/eq+uu66jEvzCimvSipYdeEzod4Khs3DXO9VA\nOb8E2WlZO4WtWPdBOE+4MDJiDl7+c+g7qfeKxRW2Fk9ZqQRjmVIBJq8qvHF+HLLzKqg3e2ObIg9q\nI5hUNM3gKTj5Dgl/blHXYS3k2GgabBreCWIxOPd+xRSDan1UA9bfq16PQKuBQlY+/8vPwrXvyVpb\nWoClOVlRVirhcFGz2DMzug2C5cdvX4KZm0rAy0yryf30DVWl3DYOOo/IF3/8fom8JUE1NWbR7xSd\nffCej8Mr35Q4J9KqYhkEW4tZ9hX5U6thNuPCpNw1Q6cgl4HMD1RbpK0Leo+bG6fVWVmMbHUBvHIR\npm/K0MjPKaKmlIfKNkXeRSHVLoOlVremUtKP+eGbGhP6nSSehqEzipPPzMDoRXWZWsoCpS2+WVWV\nBXOLcPMlRfT4QJZVIiULf/h+WVzmxmlNVhcjAxkB8+Mw/rrKacyOKwCgkIPc/PYseRdVnPzACWVr\nJ1Iq+ZHuhvZeax7SApjQ7yTFrOKKhx+Q37QS1haplKBYZcsbtJUCZEuAU8x9/7CsrXib4pknrsiV\nc/rR5YqERuuwshgZqKHHGy+oxHU2dOPVkqFKebYcVVMjlpCodw4ocqytQ9Z8PCmBN39802PqsJPU\nikfFk3D8Xj3WOagv5vStsMNUlS3H2oO+0HOTqvPddUQljgs5NS8vLCr2vtaQ2WgNVnY8q1bgxsth\nvsYSENHGaCbH1q6nNXBOrsZySZv9vcPhZ0es9lKLYEK/k6zsUBWJKWph9pYajHfltawu5mX1bzn8\nEsXZ5zP6Epbycu1EohL/hUk48Q7F4Jsrp/kJAq0IcwuavPMZbcbn5rV/48uhS3C7Iu/k+sFBslPG\nSeegVo2g90332OZri2BCv5Os7FAVT+nLc+SMHg/C+PrMlEIwt9LJ5y1U9R7FJU0qbR3aG5i+IXEo\nZODux82V08yUCsq2zs1rg3V2TH75hTuKuqmEjei3XJgMwMlSj8Yk4tGE/PF9tZ4ICzJSBk7KUDG/\nfEtgarCT1Cpc3rmmTdRalES6Bx77CNx4BUZfg9ELUGqk7nw4acQSsuaZUEjn0pw+IxqXdW+WffNR\nC5mcH9f1k2qDWxdUSC+/FIbVbtdIiKvhTSwFvUO6TffCPe+Go2d0zYIyYds6TeRbCBP6nWZlh6qV\nmYMeWU29x+DEg0qGWthOudgagVxAPpCPPh9WwUx0wOKMLMKRc/ZlbTaWFmD8B7Kuk23ym8/dhkKh\ngWvFLfd1TXfD8ft0raS74cjdchFFosqANVoSE/rdYK3MQceytd85AOc/Cq//DYxfUTz0dqy0WuZj\nKfyA8dcVkeMDuXfKJejs14qirbO+jdpaUSwrubD3lApw/QVZ8/G0Nl0nfhC2sGzgbxBLyNDoHFCW\na/8JhUwm0mFXqJJ1NmtxTOj3ktXW/vD9cPU/wavfVgVLGnTnVCtQXYCbL8tam7wCxx+QW2fg5OYx\n9xsl55gbaPeoZb7eeEV7LbkMlCaVBFVYbOCNXdjMphOIKKkvnpLIJ9uXD7P+ri2PCf1es9raP/ko\nvPGy3DCZ7bR3W0kYilktht2rApi+rq4/U28o4ufEubDs7CoLca3kHNDG8p1rmpTMDbTz1CbXpVkY\ne02im50GIpDLN/7+0Zgm+mhCm6yRqCz5Glba4FBgQr/f5OagLQ3xo/KVFjNhnZztbriFVCuyzOfH\nFaHT0a+ICqrQ3vd2K311ck6NeEqbdIWsJc40ykq3WCSqSKzbF+WCy8wqDJeoXDXFHFtv2r2CSFyf\nEUto8ognoW9EK7t85u0rNpvEWxoT+v2mXFI0RUcf9I8o43FxEqoNCj1e4lzIqVZ+7f0W7ijl/fqL\n2qytWfcrk3NW45z5cBtlpVusWtYEnM9qFVcpqFFIIaPJuFqh4SSoml8+3av7Ayfh0Y/IurcSw4cO\nE/r9Jtkmd4lzqisSBLK0yrlt9PZcTaCfQmZ547ZcUB2ejh5Z+sl2OHqPlvZ+HXExH25jVCow+qoE\nPNkGi/PqJDY7CvMT2iivliTy5QLbX82FWdkupuJkLqr6Ne19cO97lt0ztjI7dJjQ7zdtPfoilgva\nOIsnoS2sM1KtyOKvFhr8kCqUcpC5o/Zvc7clLOWi9gZuv66SDfHkcrJXDfPhbo+amyafVQOQhQlI\ndUnYl+a0kpodU7ZzcUnumnIjpYXRZN3er79Ze6/cdScegkT7211yxqHChH6/SbUtb5YWstoww4Ux\nzz3yj1fLNOyzx0vs5yYk9vkFReb4cAWRm1P6+5G7NbmYD3drrPS/+yosTGnPZXZUYZLVkkQ4VtEE\ne/v18Fjkuik1uvEalhj2gVYLR+/WxFLL0LaJ+lBjQr/fJDvkn28PXSnFLNy+LLEo5VW8LOIVctfI\n5lyN4iIUAWISoGhcPvrAyxLsPqoCaeWS3D3RuEQqSFg8PaydZ1Apyf9eDgV7+qbyGfpHNFF2JGDq\npiqa4iX+i5OaSKvFBgYTQddEZHnVFY3Dsfu0YvBe47CJ+tBjQr/frCybEEtKSNp7JfouogiczKwa\nQQSEBa22WY72LVTCiaQsX2419OVf+GsoFmTRJ1PKtHVYPD2snWcQjYWbp4FcYkvzkJ3R8xNXIN2p\nSWFpTueXiNw3fqv9CVaRaNPk7NFkk0zLD997XOPDae+ls99E3jChPxCsTKRauCNh7eiD8cvQmdfG\n7OIdLfljiTCvagfE3legEm7YlpcAJ0Gauym30eBpRWt0HdHG8PwknHpELp9Grftmy8BdrwnI1HWY\nfEMrMA9kp+RuizjIZcMN2KR6Ab9Z/2gHIpgqFYVPtrXr7/O+n9OmrkXTGGtgQn9QWJlIVchqCT50\nN8yNwfC5MA3eARE9V8ztjGC8xR3ktTlby9ScuCJfvotqL6FY0HhGzmlM27XumzEDt5gNN6bjio4p\n5dRBbGkO5m9DJaw75CK6DSrgvCaAYi4MTw0ajJoMhbutF7oGdL20d8EjP6Ua8oaxDib0B43VpY4H\nT0HXoJJc5sYkFPk5WW+lgqI6gkajclYSyAr1HvKTy5u2lV7tF8yNKzyzuARnf3j9csjrWezNmoGb\nzypKJhrX5ukbL0nI42kJebWqcxaEm+ZBFU2iDcbDrySWUCTNOz4gK94H8sH3HN+5zzBaEhP6g8Z6\npY5HHpC1Oz8hUWxrl1VXLst9UN6BTkM1qkWoOr1faSmcUJbABUrAcRF45c81jrNPQir91tdvZLFX\nS82VgRsE8rtff1FJTol2baaWi4qJz0xr49oTum9CV9iO4nT+eo8rQqv7qB6Op2yj1agLE/qDyFql\njhNp+XmrlTCELqnHljJhtEU0LIAVCnTDrHyPqqJ1iOjzE20a09XnFA300AeBQC6NIJD4JVIaU7US\nWv2hJd81uPMZuLvl7y8VVBt+7DVVGc3Ny5ovlwAPuSiUwybuOHZe4IFoUhFZXUfgzLvg1KPLLf7M\nD2/Uya4IvXPuQ8BngCjwu977T+/G57Q0a5U6PnaPrMm2TlnFuUW5D7oG1Uu0WtXjO+K7X40HqqrH\nUi5qfJWK/PbjlyDdp81BhyzgVDt0D0p0XZgXkOqSaG0nA3ctMQcVAJu4Jms60f72CKGNXEhrvV/t\nMRxMXIaZm3JVBRXlH1QrcpUFZai4FRFQO+iiqRFLKURz5B36f919Xsl1hrFFdlzonXNR4N8BPwGM\nAt9zzn3Ze//aTn/WoaOjX5b+5BWJp4tItBYmofuIMiMnryqMLzOzS4IP2lQMFImzNANEJarJdolk\nNCq3Qrkgd0PnAJSKsHBFxxVzEs50r6JTYOMM3LVcQaAxzI7yZjneciHclAxXD4OnQrdWQZNTtaxx\ndR9VKOTKyWEllaJq+89P6L0XJ7VHEgnDXRvOVK4DF1WY7eAZtYvsOaZIKMPYBrth0T8OXPHeXwNw\nzv0x8FHAhL5RIhE1/w6qsjSrFSXMtvXIKu0+qu5BN16W4BczcjOUc7s0oJoVW5Uvv1pUnZWS1ySU\nmZabZ25Mj89PwOK0mlDHEhC7rY5bkdDF09YNY5e0wdnRq5UL6P9Szuu4SFSW7uSV5Q3r2uRQLujY\nnqEwTHRcfvRC2KO3UoTpUT03dEb7BJlpVXqcv62JauScVijZBa1MFqc0eQUldsVqfxtOP+ke6Dyi\nbOWe49a/1WiI3RD6YeDWit9HgXevPsg59zTwNMDJkyd3YRgtSiIFd70TjpxS2zmQOyTVvhzG196j\nDdLCktwPWQ+VHahtvhnVSrhFUFF4ZrmkMZSLcjG5iKz9ckGRPAN3QWZO+w2zoxJdvBLHuo5I3DoH\nJN6xmP5v1dBVEg3L+ZYKuk9Uk0hmejlLNRLTZBFLyNc9dTVsvbgEt14N6wgVdVwsrNc+/rpuqxXV\npynvwXl7k0hYOz6uldDpR9To3fq3Gg2yb5ux3vtngGcAzp8/vxemUuvgIrL4Vi/laz79IND9jgGJ\nY6koa3Y3NgtXU/NZVwr60YAlnqmwXMDipKz7zKwmgGpYaiHZLuu6UlRyVqkQZpMil00kJldPdhaW\nFqF7QOeilJOrCqcN6WhM+xWRqMYQVCA7j6KICgoZfYt17sKOSzG9l0JodigDeSOiWonV9gk6BjSh\nRZNw7F64/0ck/IbRILsh9GPAiRW/j4SPGXtFIik3TmFRrpHpG+B61WEqKCt6p9EU/C3hJbaFpVBI\nw/2FwqIEOZUORT6iuai4pAgXvCapaEw/sYSELwjLNVSLy4XfFic1iQTVsBm7WxHK2a0IpnIFgrUs\ndK9N2L0i1am9gWhUY6xWYeCU/n846D0KD/+UibyxY+yG0H8POOucO40E/hPAf7kLn2OsR7JDYpdq\nl4D2H1dDk5pFXSmGZXH3yMoHJPal8OOioYUe+uZz2dB6z4XCF4SuptDqLkeXrfml+XBTtCorfXFG\n0T/UmnU4PdfWoUmlnNdexb4T+t7jYf/eeFqTVaoDhh8IO04FYWnhB03kjR1lx4Xee19xzv0y8Oco\nvPJz3vsLO/05xgasTLqqlKFnOBTPUujr9aF75I78+pFa/HqjpZDrJawLU/O3+6oSv9b1h1fD0gxr\n5AhUVrtXwv2IzF761tcjHK+LyzVTay7TNww9I/o7jTywHK1ksfHGLrErPnrv/VeBr+7Gext1Uku6\nmp/Q70NnFIVSLmhDslQEorJ2c6FfO6iGNdJ32ze9XZpoKyeSUB34WHI5ozioytVUDjet23sk8gcp\nE9hoSSwztpVxEfnql+YlLCMPyGVTrchnHZRhNpClWWtIHY8rESjYrRj8ViesDV/LXm3v0eQaVLXh\nnEpDNKIN5fYeawhi7Am2Rmx1am4cT1gALQBCF8KR07I6E2neTDpyMd1Gk5DqgZj5iusmEldsfkef\nznM1zCTG6/dUp26DMOGs57i5aYw9wSz6w8BatXNSHZAZUsu7mZsS/FJB/vJ4QpuFnX2K1Z8rExbB\nN9bCRcN94IjKUcSTKvdQzmvSTKSUuRxLKLnNB8oPaDNr3tgbTOgPC2vVzqk1jz5yWu6dO9eX49px\nEqpUu1wMQRVKWVmpe7Zpe9CpuWnC++09CuVMpBXRFOnTc9Ek9J+U4FfK2hRv7zW3jbFnmNAfZiIR\nZZ/euRbWOu9VKd7AS5QW7ki0hoqqGbM4pZDAoBKWWq9VbWyiTdJtEVUdIeegktNGaySmePcg0IRY\nLck1U61IyCNxnc9EWpNkIbvc/H3gpJU0MPYUE/rDzmq3zrGzqJDXlPzNXUckXLG4rFQfhPH3QDUm\nV09AWEAtzChtNRzLJRziaRVVS6VDX3wVSilNgMl26D+x3Ans3I+oZ2sh+9ZyFVbSwNhjTOiNtd06\nzkmgnJPP+fi9ErXMtIqCpbtlzc7dUlOSUijyLqLjIhFt7FZ2q6DabhAJWwECVGWVx+LLpY89UCrp\n90gkbBcYJn71jcjv3jOkhLWRc8tJT2uVqzCMPcSE3lib1S0NE+3anC3l5VuOJ8NQzaPQVlB2a61P\naiHsr0oFZcESTgABB8e/X3M5RWWdV6panYDG6WKQ7lzuAUtUSWaxuHISEilNjvHk8iRw9w9pFWRJ\nT8YBw4TeWJu1Whq29UBHIcxo9RDzmgQG7tLz0zfkg84tqiFINC43R7WoDUkfbvQGgcoTBDV3z3aJ\nKSa9lmHrIquSvSLL7+9i4XNe/vZYKix5HF+RMxDX/ycSUS/WWl19CBPNPPQcVQvH/rs0AVTL8sOv\ntOAN44BhQm+sz1phmWcee6vPOZGCqZsS+Y4BbUqmu+HUO1WYbGFa1SJr/v14IrTsAyAVxpqH7p1I\nIhTdCMuWf1gAzVeVZRoEuo2Fl24QQDwIO0c5hYgGlfDznD4zFgsblqRkyLd1qAzw0bNKEhu7qEJq\n7X3Q2avia7GEwiadkzsmGpfbpqNH5yASs7IFRtNgQm9szFr++9U+55ol7SLaxI2EAtk1FLp8jsD4\nD2TFR2MSVYcsZO+h2qba8M5BEAvr4ISRKwSyrhNJIOyole7W5ybSep9cOJG0dUJnLOx+NR+KfBLa\nu/Ve1RK0D8Cxu9Udq1rR56d7ZKn3jSgzODMbPl/VeybSYU37+HKnJxN2o4kwoTcaJ56UGKe7FGVS\nK7MQjanoWM8QxKLayI0lJbjzE7KmS3kVXAvKcn0UsmECUqCVQFDRSiESk/XcOQDdx6CjW+8VT+o1\nN15ZbhXY2QuTb6hiZ1DRBBOJw7H74KH3a5z5Ra1KggBmb2mDuT2cvLoTmniyMypfvDSnFYOFRRpN\nigm90TirN26T7Xq8XIB4VCGa2Vn1XY0lZEUnO9Q6sJTTJJFfkKU+P6Xqk0FVwr80JzGPxtSCsKMX\nzv6QMk9rfWST7XDqUfWBTXVAfh6O3rNczsGh92rvk8ivbtzSPQQX/0oTQzzcS+gfUfOPckF7FbXW\nhibyRhNiQm80zlobt95L1I+ckUgP3SMrf+amfOBL8xLonuOylKdvaAM3kZBPfX4c8Kr62NErq77v\nuNxInWFnqdX7B3e/S0leE1eWk5Xcis7fuUUdv9oVlUrDuR+Dsde0yojFw9VCSuNPpPb0dBrGTmNC\nb+wM69XTqVnAK3vdZuZg+rqe7+iTGA+e0uQQDevCDJ6Shd3erWPiqWXhrb3nWvsHsaRWBrVVxUpc\nre7+GiTTcPqx9cdvGE2MCb2xc6wlvKufr7lMeo9pBZDPLK8ABu6SyNdK/CbTyw3P6xXeWBjSuRbe\n6/ntjt8wmhQTemN/2GwFUGOrwrt6v6BGuaDHrZCYcQgxoTf2j92woDfbLzBXjHEIMaE3Wo96VwuG\ncUgwoTdaE/O3G8abmIljGIbR4pjQG4ZhtDgm9IZhGC2OCb1hGEaL4/x6ySV7OQjnpoAb6zw9AEzv\n4XAapdnGCzbmvaDZxgs25r2g0fHe5b0f3OygAyH0G+Gce857f36/x1EvzTZesDHvBc02XrAx7wV7\nNV5z3RiGYbQ4JvSGYRgtTjMI/TP7PYAt0mzjBRvzXtBs4wUb816wJ+M98D56wzAMozGawaI3DMMw\nGuBACb1z7gvOuZfCn+vOuZfWOe66c+774XHP7fU4V43lXzrnxlaM+yPrHPch59wPnHNXnHOf2utx\nrhrLbzrnLjnnXnHO/Zlzrmed4/b1PG92zpxzyfCaueKce9Y5d2qvx7hqPCecc99yzr3mnLvgnPuV\nNY55n3NuYcX18s/3Y6yrxrTh39mJz4bn+RXn3GP7Mc4V47lvxfl7yTm36Jz71VXH7Pt5ds59zjl3\nxzn36orH+pxzX3fOXQ5ve9d57VPhMZedc081PBjv/YH8Af5X4J+v89x1YGC/xxiO5V8Cv7bJMVHg\nKnAGSAAvA+f2ccw/CcTC+78B/MZBO8/1nDPgF4HfCe9/AvjCPl8Lx4DHwvudwOtrjPl9wFf2c5xb\n/TsDHwG+hrrvPgE8u99jXnWdTKB48gN1noEfBR4DXl3x2P8CfCq8/6m1vntAH3AtvO0N7/c2MpYD\nZdHXcM454OPAH+33WHaIx4Er3vtr3vsS8MfAR/drMN77v/DeV8JfvwuM7NdYNqCec/ZR4PPh/S8C\nHwivnX3Bez/uvX8hvJ8BLgLD+zWeHeSjwO978V2gxzl3bL8HFfIB4Kr3fr2Ey33De//XwOyqh1de\ns58HPrbGS38K+Lr3ftZ7Pwd8HfhQI2M5kEIP/Agw6b2/vM7zHvgL59zzzrmn93Bc6/HL4ZL2c+ss\nxYaBWyt+H+XgCMDPI2ttLfbzPNdzzt48Jpy4FoD+PRndJoRupEeBZ9d4+oedcy87577mnHtwTwe2\nNpv9nQ/y9fsJ1jcID9p5Bhjy3o+H9yeAoTWO2fHzvef16J1z3wCOrvHUP/Pefym8/w/Z2Jp/r/d+\nzDl3BPi6c+5SOHvuChuNGfht4F+hL8u/Qi6nn9+tsdRLPefZOffPgArwB+u8zZ6e51bBOdcB/Anw\nq977xVVPv4DcDNlwP+f/Bs7u9RhX0ZR/Z+dcAvgZ4J+u8fRBPM9vwXvvnXN7Eva450Lvvf/gRs87\n52LAfwa8a4P3GAtv7zjn/gwt83ftwtxszDWcc/8e+MoaT40BJ1b8PhI+tmvUcZ7/MfDTwAd86Bhc\n4z329Dyvop5zVjtmNLxuuoGZvRne2jjn4kjk/8B7/6ern18p/N77rzrnfss5N+C937f6LHX8nff8\n+pRgLF4AAAG9SURBVK2TDwMveO8nVz9xEM9zyKRz7pj3fjx0f91Z45gxtMdQYwT4diMfehBdNx8E\nLnnvR9d60jnX7pzrrN1HG4uvrnXsXrDKV/kP1hnL94CzzrnToRXyCeDLezG+tXDOfQj4J8DPeO9z\n6xyz3+e5nnP2ZaAWkfCzwF+uN2ntBeH+wO8BF733/2adY47W9hGcc4+j7+C+TU51/p2/DPyjMPrm\nCWBhhfthP1l35X/QzvMKVl6zTwFfWuOYPwd+0jnXG7qCfzJ8bPvs5670OjvV/wH4hVWPHQe+Gt4/\ngyIwXgYuIFfEfo73/wC+D7wS/hGPrR5z+PtHUBTG1QMw5ivIB/hS+FOLXDlQ53mtcwb8T2iCAkgB\n/1f4//lPwJl9Pq/vRS68V1ac248Av1C7poFfDs/ny2gj/D37POY1/86rxuyAfxf+Hb4PnN/PMYdj\nakfC3b3isQN1ntEkNA6UkZ/9k2gP6ZvAZeAbQF947Hngd1e89ufD6/oK8HONjsUyYw3DMFqcg+i6\nMQzDMHYQE3rDMIwWx4TeMAyjxTGhNwzDaHFM6A3DMFocE3rDMIwWx4TeMAyjxTGhNwzDaHH+f45b\nUsONmxd/AAAAAElFTkSuQmCC\n",
|
|
"text/plain": [
|
|
"<matplotlib.figure.Figure at 0x7f1a8e78f3c8>"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
}
|
|
],
|
|
"source": [
|
|
"# training data\n",
|
|
"x = np.linspace(-7, 10, N_SAMPLES)[:, np.newaxis]\n",
|
|
"noise = np.random.normal(0, 2, x.shape)\n",
|
|
"y = np.square(x) - 5 + noise\n",
|
|
"\n",
|
|
"# test data\n",
|
|
"test_x = np.linspace(-7, 10, 200)[:, np.newaxis]\n",
|
|
"noise = np.random.normal(0, 2, test_x.shape)\n",
|
|
"test_y = np.square(test_x) - 5 + noise\n",
|
|
"\n",
|
|
"train_x, train_y = torch.from_numpy(x).float(), torch.from_numpy(y).float()\n",
|
|
"test_x = Variable(torch.from_numpy(test_x).float(), volatile=True) # not for computing gradients\n",
|
|
"test_y = Variable(torch.from_numpy(test_y).float(), volatile=True)\n",
|
|
"\n",
|
|
"train_dataset = Data.TensorDataset(train_x, train_y)\n",
|
|
"train_loader = Data.DataLoader(dataset=train_dataset, batch_size=BATCH_SIZE, shuffle=True, num_workers=2,)\n",
|
|
"\n",
|
|
"# show data\n",
|
|
"plt.scatter(train_x.numpy(), train_y.numpy(), c='#FF9359', s=50, alpha=0.2, label='train')\n",
|
|
"plt.legend(loc='upper left')"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 5,
|
|
"metadata": {
|
|
"collapsed": true
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"class Net(nn.Module):\n",
|
|
" def __init__(self, batch_normalization=False):\n",
|
|
" super(Net, self).__init__()\n",
|
|
" self.do_bn = batch_normalization\n",
|
|
" self.fcs = []\n",
|
|
" self.bns = []\n",
|
|
" self.bn_input = nn.BatchNorm1d(1, momentum=0.5) # for input data\n",
|
|
"\n",
|
|
" for i in range(N_HIDDEN): # build hidden layers and BN layers\n",
|
|
" input_size = 1 if i == 0 else 10\n",
|
|
" fc = nn.Linear(input_size, 10)\n",
|
|
" setattr(self, 'fc%i' % i, fc) # IMPORTANT set layer to the Module\n",
|
|
" self._set_init(fc) # parameters initialization\n",
|
|
" self.fcs.append(fc)\n",
|
|
" if self.do_bn:\n",
|
|
" bn = nn.BatchNorm1d(10, momentum=0.5)\n",
|
|
" setattr(self, 'bn%i' % i, bn) # IMPORTANT set layer to the Module\n",
|
|
" self.bns.append(bn)\n",
|
|
"\n",
|
|
" self.predict = nn.Linear(10, 1) # output layer\n",
|
|
" self._set_init(self.predict) # parameters initialization\n",
|
|
"\n",
|
|
" def _set_init(self, layer):\n",
|
|
" init.normal(layer.weight, mean=0., std=.1)\n",
|
|
" init.constant(layer.bias, B_INIT)\n",
|
|
"\n",
|
|
" def forward(self, x):\n",
|
|
" pre_activation = [x]\n",
|
|
" if self.do_bn: x = self.bn_input(x) # input batch normalization\n",
|
|
" layer_input = [x]\n",
|
|
" for i in range(N_HIDDEN):\n",
|
|
" x = self.fcs[i](x)\n",
|
|
" pre_activation.append(x)\n",
|
|
" if self.do_bn: x = self.bns[i](x) # batch normalization\n",
|
|
" x = ACTIVATION(x)\n",
|
|
" layer_input.append(x)\n",
|
|
" out = self.predict(x)\n",
|
|
" return out, layer_input, pre_activation"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 6,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Net (\n",
|
|
" (bn_input): BatchNorm1d(1, eps=1e-05, momentum=0.5, affine=True)\n",
|
|
" (fc0): Linear (1 -> 10)\n",
|
|
" (fc1): Linear (10 -> 10)\n",
|
|
" (fc2): Linear (10 -> 10)\n",
|
|
" (fc3): Linear (10 -> 10)\n",
|
|
" (fc4): Linear (10 -> 10)\n",
|
|
" (fc5): Linear (10 -> 10)\n",
|
|
" (fc6): Linear (10 -> 10)\n",
|
|
" (fc7): Linear (10 -> 10)\n",
|
|
" (predict): Linear (10 -> 1)\n",
|
|
") Net (\n",
|
|
" (bn_input): BatchNorm1d(1, eps=1e-05, momentum=0.5, affine=True)\n",
|
|
" (fc0): Linear (1 -> 10)\n",
|
|
" (bn0): BatchNorm1d(10, eps=1e-05, momentum=0.5, affine=True)\n",
|
|
" (fc1): Linear (10 -> 10)\n",
|
|
" (bn1): BatchNorm1d(10, eps=1e-05, momentum=0.5, affine=True)\n",
|
|
" (fc2): Linear (10 -> 10)\n",
|
|
" (bn2): BatchNorm1d(10, eps=1e-05, momentum=0.5, affine=True)\n",
|
|
" (fc3): Linear (10 -> 10)\n",
|
|
" (bn3): BatchNorm1d(10, eps=1e-05, momentum=0.5, affine=True)\n",
|
|
" (fc4): Linear (10 -> 10)\n",
|
|
" (bn4): BatchNorm1d(10, eps=1e-05, momentum=0.5, affine=True)\n",
|
|
" (fc5): Linear (10 -> 10)\n",
|
|
" (bn5): BatchNorm1d(10, eps=1e-05, momentum=0.5, affine=True)\n",
|
|
" (fc6): Linear (10 -> 10)\n",
|
|
" (bn6): BatchNorm1d(10, eps=1e-05, momentum=0.5, affine=True)\n",
|
|
" (fc7): Linear (10 -> 10)\n",
|
|
" (bn7): BatchNorm1d(10, eps=1e-05, momentum=0.5, affine=True)\n",
|
|
" (predict): Linear (10 -> 1)\n",
|
|
")\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"nets = [Net(batch_normalization=False), Net(batch_normalization=True)]\n",
|
|
"\n",
|
|
"print(*nets) # print net architecture"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 7,
|
|
"metadata": {
|
|
"collapsed": true
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"opts = [torch.optim.Adam(net.parameters(), lr=LR) for net in nets]\n",
|
|
"\n",
|
|
"loss_func = torch.nn.MSELoss()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 10,
|
|
"metadata": {
|
|
"scrolled": true
|
|
},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Epoch: 0\n",
|
|
"Epoch: 1\n",
|
|
"Epoch: 2\n",
|
|
"Epoch: 3\n",
|
|
"Epoch: 4\n",
|
|
"Epoch: 5\n",
|
|
"Epoch: 6\n",
|
|
"Epoch: 7\n",
|
|
"Epoch: 8\n",
|
|
"Epoch: 9\n",
|
|
"Epoch: 10\n",
|
|
"Epoch: 11\n"
|
|
]
|
|
},
|
|
{
|
|
"data": {
|
|
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAk0AAAE/CAYAAABb+PcPAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAHMdJREFUeJzt3XuUbGV55/HvwyEEFBC0ARUMTdAJQxLEMUYSJBox3hLB\nEZEBHWCtzEqcRIM6GlRGjVGj4yycTMDERYxyGWAQUWEZw1UBMwb1tKIRIUNiOA6R2xGQiwQQnvmj\nqqFP0139dnft+/ezVq9TtbvOrufXdemn3/3WuyMzkSRJ0mRbNV2AJElSF9g0SZIkFbBpkiRJKmDT\nJEmSVMCmSZIkqYBNkyRJUgGbJkmSpAKdaZoi4oaIeNES2w+OiOsi4scR8aWI2LOJ+qq0VPaI2CYi\nPj3+XkbECxoqr1LLZD8gIi6JiNsj4raIODcintJUjVVZJvu+EbExIu4Yf10aEfs2VWNVlnu9L/j+\nu8fP+2Vv01XLPO6z47z3LPh6V1M1VmnCe/3jIuLPI2JzRPwoIq5sor4qLfPYv3bR4/7j8XPh2U3V\nWYUJj/trIuLaiLg7Ir4bEa9sor55nWmalhIRM8BngHcBTwQ2Auc0WlS9/hZ4HXBz04XUbGfgFGAW\n2BO4G/hkkwXV6AfAqxk932eAC4D/3WhFNYuIvYHDgZuarqUBO2Xm9uOv9zVdTM1OYfS8/7fjf9/c\nbDn1yMwzFzzm2wO/B3wP+EbDpVUuInYH/hfwFmBH4G3AWRGxa1M1bd3UHU/Jq4BrMvNcgIj4I2Bz\nROyTmdc1WlnFMvMB4E8BIuKhhsupVWb+zcLrEXEycEVD5dQqM+8E7gSIiAAeAp7eaFH1+yhwPPDn\nTReiekTEPsAhwB6Zedd481yDJTXpGOD0HMbpPPYA7lzwnv/XEXEvsDdwaxMFdXqkCfh54FvzVzLz\nXuCfxts1HL8GXNN0EXWKiDuBfwVOAv6k4XJqExGHA/dn5hearqUhmyLixoj45HikfSh+GdgEvHd8\neO7vI+Kwpouq23j6ya8BpzddS002AtdGxCERsWF8aO5+4NtNFdT1kabtgdsWbfsRsEMDtagBEbEf\n8G7g0KZrqVNm7hQRj2f0V+empuupQ0TswKhB/I2ma2nAZuA5wNXAkxiNtp0JvKTJomq0B/ALwHnA\nU4FfYTTq8N3MvLbRyup1NPDlzPznpgupQ2Y+FBGnA2cB2wIPAIePB0ga0fWRpnsYHedcaEdGc1zU\ncxHxdOBvgOMy88tN11O38RvHx4DTmzzGX6M/As7IzBsarqN2mXlPZm7MzJ9k5i3AG4AXjxvJIbgP\neBB4f2Y+kJlXAF8CXtxsWbU7Gjit6SLqMp4Y/mHgBcA2wPOBj0fE/k3V1PWm6RrgmfNXxn95783A\nDtUM0XiY+lLgfZl5RtP1NGgr4HHA7k0XUoODgT+IiJsj4mbgacCnIuL4hutqwvx8lq6/h5da6nDM\nEOb0PCIiDmQ0yvbppmup0f7AleM/GB7OzK8DXwUa+9Rs115wPxUR285/AZ8FfiEiDhtffzfw7Z5O\nAt8ie0RsHRE/Pc4NsM14ezRaZTUWZ98T+CJwcmZ+rOniKrY4+29ExLPGx/d3BD4C3AH08RDF4tf7\nSxgdotl//PUD4HcZHarqm8WP+3Mj4uciYquIeBLwZ8DlmfmjpgutyOLH/krg+8A7xu99BwK/DlzU\naJXVeMx7/Xj7McB5mdnnIymLH/evAwfNjyxFxLOAg2hwTlN0ZQJ+RNzA6OPlC30AuBw4efy9rwLH\n9m34fkL21y2xfa8+5V8m+7wtjmuPP47bG8tk/y6wgdEcj/uArwHvyMzG3kSqsNxzPjP/66Lb/KfM\nvLTG0iq3TPbvMfojd1fgLuAS4A8zs3fLjUx4vzsb+DiwH6N5fCdk5mfrra5aE7K/n9HSModl5mV1\n11WHCdlvBt4E7MZoDvNHM/PEeqt7VGeaJkmSpCZ17fCcJElSI2yaJEmSCtg0SZIkFbBpkiRJKmDT\nJEmSVKCS06jMzMzk7OxsFbtuzNzc3ObM3GWl2w05O/Qvv9mHmR18zZt9siFnh/7lL81eSdM0OzvL\nxo0bq9h1YyKi6PxeQ84O/ctv9jJ9yw6+5ktuZ/ZhZof+5S/N7uE5SZKkApWMNG3hqx0+Tc5zX910\nBY9a6ufYpvqmpQ85+5Bhrcy+paFkL7Wen9Hi/9umn62P/WA40iRJklSg+pEmSUtr81/OkqTHcKRJ\nkiSpgE2TJElSAZsmSZKkAjZNkiRJBWyaJEmSCtg0SZIkFbBpkiRJKmDTJEmSVMCmSZIkqUDRiuAR\ncWBm/p+VtklaRpfPwShJAspHmk4q3CZJktRLE0eaIuJXgF8FdomItyz41o7AhioLkyRJapOVDs9t\nA2w/vt0OC7bfBXh2UUmSNBgTm6bMvAK4IiJOzcxNNdUkbWmp+UDPtWeXJNWrdE7TxyNip/krEbFz\nRFxUUU2SJEmtU9o0zWTmnfNXMvMOYNdqSpIkSWqf0qbp4Yj4mfkrEbEnkNWUJEmS1D5F6zQBJwB/\nGxFXAAEcBPxuZVVJkiS1TFHTlJkXRsS/Aw4Yb3pTZm6urixJkqR2KT6NSmZuzszPA9cC/zkirqmu\nLEmSpHYpPY3KU4EjgKOAXwQ+CPyHCusaNk+5IUlS66y0IvjvAEcCuwOfAn4bOD8z31tDbVrJ4ubK\ntYu6zfWoJKnVVhppOhn4O+CozNwIEBF+ak6SJA3OSk3TU4DDgRMj4smMRpt+qvKqJPWbo2qSOmji\nRPDM/GFmfiwznw8cDNwJ3BIR10bEn9RSoSRJUguUrtNEZt4InMho1OkZjOY6SVI3OdolaZWKlhyI\niMdFxLsi4pQFm79RUU2SJEmtUzrS9ElgDvjV8fV/Ac4FPl9FURoQl1eQJHVE6eKWe2fmh4EHATLz\nx4xOpyJJkjQIpU3TAxGxHeOT9EbE3sD9lVUlSZLUMqWH594DXAg8LSLOBA4Ejq2qKEmSpLZZsWmK\niACuA17F6IS9ARznCXslSdKQrNg0ZWZGxBcy8xeBv66hJkmSpNYpPTz3jYh4TmZ+vdJqJE1PW85N\n6CckJfVEadP0XOB1EXEDcC+jQ3SZmftVVZgkSVKblDZNL6m0CkmSpJab2DRFxLbA64GnA38P/FVm\n/qSOwiRJktpkpZGm0xgtaPll4GXAvsBxVRclqQKea02S1mWlpmnf8afmiIi/Ar5WfUmSHmGjI0mt\nsVLT9OD8hcz8yWjJJqkFbCYkSTVbqWl6ZkTcNb4cwHbj6/Ofntux0uq0OjYSUrv4mpR6ZWLTlJkb\n6ipEkiSpzUpP2CtJkjRoNk2SJEkFbJokSZIKlK4ILqnNPL+bJFXOpqnv/PRO/9ggSVIjbJpUH3/Z\na5LFzw+be0ktY9PUNBsJqR6+1iStk03TEPX1kJ0jFZKkCvnpOUmSpAKONGmkr6NPGiYPxUmqgE1T\n3Xwzr4+NoCRpiiIzp7/TiNuATVPfcbP2zMxdVrrRkLNDL/ObvUAPs4OvebNPMOTs0Mv8ZY97FU2T\nJElS3zgRXJIkqYBNkyRJUgGbJkmSpAI2TZIkSQVsmiRJkgrYNEmSJBWwaZIkSSpg0yRJklTApkmS\nJKmATZMkSVKBSk7YOzMzk7Ozs1XsujFzc3ObS85LM+Ts0L/8Zh9mdvA1b/bJhpwd+pe/NHslTdPs\n7CwbN26sYteNiYiiExMOOTv0L7/Zy/QtO/iaL7md2YeZHfqXvzS7h+ckSZIKVDLStNDF36v6Hqrz\n4p9tuoJHLfVzbFN9a7U4Vx8yrVVfH+MmtOVn2ZY6usaf22Rt+vm0qZY6ONIkSZJUoPKRJkmSpKZN\nY1RsxaYpIs7IzP+40jZJq9PlQ9eSNEQlI00/v/BKRGwAnl1NOZLWYmjzCiSpCcvOaYqId0TE3cB+\nEXHX+Otu4Fbg/NoqlCRJaoFlR5oy84PAByPig5n5jhprkiRJA9OFEfOST899LSKeMH8lInaKiFdW\nWJMkSVLrlMxpek9mfnb+SmbeGRHvAT5XXVkq4RpHkiTVp2SkaanbuFSBJEkalJLmZ2NEfAT46Pj6\nG4C56kqSJGlLjqyrDUqapjcC7wLOGV+/BPi9yirSoHVhIqAk1ck13dpjxaYpM+8F3j5/PSK2BV4B\nnFthXZIkSa1SdO65iNgQES+PiDOAG4AjKq1KkiSpZSaONEXE84GjgJcDXwMOBH42M39cQ22SJEmt\nsWzTFBE3At8H/gJ4a2beHRH/bMMkSZqWIc/XcQ5n90w6PPdp4KmMDsW9IiIeD2QtVUmSJLXMpNOo\nvCki3gy8ADgS+DDwhIh4DfCFzLynnhIlrYUf0W4nRxek7po4ETxHvpSZvwPsxah5OpTRZHBJkqTB\nKFrZOyK2A34mMz8PfH58XZIkaTBWXHIgIg4BrgYuHF/fn0cXupQkSRqEohP2Ar8MXA6QmVdHhEfg\n1TrO4ZEkVamkaXowM38UEQu3PVxRPdLUOOFWkjRNJU3TNRFxFLAhIp4B/AHwlWrLkiRJapfSE/ae\nANwPnAVcBLy/yqIkqdSQF0eUVO9RhZVOo7IB+OPMfCujxknSGviLXVKJobxXdDXnSus0PQQ8r6Za\nJEmSWqvk8Nw3I+IC4Fzg3vmNmfmZyqoauK524NKQ+bqV+q+kadoW+CHwwgXbErBpkiRJg1HSNL0t\nMzdXXokkSVKLLds0RcQrgE8AD0bEw8BrMtOlBqQaudaUJLXHpIngHwAOysynAocBH6ynJEmSpPaZ\ndHjuJ5l5HUBmfjUidqipJklSAUcipXpNapp2jYi3LHc9Mz9SXVmSJEntMqlp+ktghwnXJUlqhKNs\n7dXnx2bZpikz31tnIVId+vxilsDnuFSlkiUHpF7zl4wkqYRNk1rPlZYlSW1g0yRJkirVlz9+Jy1u\nefSk/5iZp0+/HEkr6cubjyR1zaSRpucss/0QYHfApkmSpsB5df2z+DH18eyHSZ+ee+P85YgI4LXA\n8cBVjFYLlyRJGoyJc5oiYmvgWOCtjJqlV2fmP9RQl6SB8zCkpLWq6v1j0pym3weOAy4DXpqZN1RT\ngiRpJTaRGqK2HbqeNNJ0EnAr8DzgwNEROgACyMzcr+LaJGmwbJKk9pnUNO1VWxWSKte2v9jq5sRc\nSes1aSL4psXbImIG+GFmZqVVSZIkLaHJUdhJc5oOAD4E3A68DzgDmAG2ioijM/PCekqUJK3H0EcZ\nh8DHuB6TDs+dDLwTeALwReBlmXlVROwDnA3YNGlVnKPRf75xS0vz/a8ftprwva0z8+LMPBe4OTOv\nAsjM6+opTZIkqT0mNU0PL7h836LvOadJkiQNyqTDc8+MiLsYLTGw3fgy4+vbVl6ZJElSi0z69NyG\nOguRJElqs0mH5yRJkjRm0yRJklRg4gl7JWm9/Ki1pL6IKhb3jojbgPkVxWeAzVO/k3osrH3PzNxl\npf/Qo+zwaP1F2WGL/GbvLrOPrPY1P+Tsi/9/1ww5O/iaL85eSdO0xR1EbMzMX6r0Tiqy3tq7nB3W\nV7/Zzd5FZh/m+92Qs4PP+9XU75wmSZKkAjZNkiRJBepomk6p4T6qst7au5wd1le/2bvL7PX/3zYY\n8vvdkLODz/tilc9pkiRJ6gMPz0mSJBWovGmKiHMi4urx1w0RcXXV97kWEfGJiLg1Ir6zYNsTI+KS\niLh+/O/O69j/f4mIjIiZ6VRcrYjYEBHfjIjPT2FfZjd76w05O0wv/5Czj/fVqfxmX132ypumzDwi\nM/fPzP2B84DPVH2fa3Qq8NJF294OXJaZzwAuG19ftYh4GvBi4PvrKbBmxwHXrncnZjf7uiuqz5Cz\nwxTyDzk7dDa/2VehtsNzERHAa4Cz67rP1cjMK4HbF20+FDhtfPk04JVr3P3/AP4Q6MQEsojYA/hN\n4ONT2J3Zzd56Q84OU80/5OzQsfxmX332Ouc0HQTckpnX13if67VbZt40vnwzsNtqdxARhwL/kpnf\nmmpl1fpTRk/+h9ezE7ObfSpV1WPI2WEK+YecHTqb3+yrzD6Vc89FxKXAk5f41gmZef748pG0dJSp\nRGZmRCzZQU/KD7yT0ZBlJ0TEbwG3ZuZcRLyg4PZmfyyzm70zVpN/yNnHt+9NfrOXZ9/i/1ax5MDM\nzEzOzs5Ofb9Nmpubux3YnJk/N+l2Pc1+f2ZuW3LbvuU3+zCzA8zNzW0uORdVT7MXPfZmn62hovoM\n+TVfmn0qI02Lzc7OsnHjxip23ZiIuA84f6Xb9TT7d1a+1Ujf8pu9TN+yA0TEppVv1dvsRY+92YeZ\nHfqXvzS76zSV2xH4UNNFSJKkZlQy0rSFk4+q/C4q84azFl77v5m5+NN167fUz2fL++2mvuYqYfYt\nmX0YFucfcnYYTv6BZXekSZIkqYBNkyRJUgGbJkmSpAI2TZIkSQVsmiRJkgrYNEmSJBWwaZIkSSpg\n0yRJklTApkmSJKlA9SuCD9nAVkqVJKnPHGmSJEkqYNMkSZJUwKZJkiSpgHOaNOL8K0mSJnKkSZIk\nqYAjTXVbakRnKBzNkiR1mCNNkiRJBRxpkiRJ/TeFox2ONEmSJBVYsWmKiJ8u2SZJktRnJSNNf1e4\nTZIkqbeWndMUEU8Gdge2i4hnATH+1o7A42qoTZIkqTUmTQR/CXAssAdwIo82TXcB76y2LEmSpHZZ\ntmnKzNOA0yLisMw8r8aaJEmSWqdkTtOzI2Kn+SsRsXNEvL/CmiRJklqnpGl6WWbeOX8lM+8AXl5d\nSZIkSe1T0jRtWLjEQERsB7jkgCRJGpSSFcHPBC6LiE8ymgx+LHBalUVJkiS1zYpNU2b+t4j4FvAi\nIIGLgD2rLkySJKlNSk+jcgujhulw4IXAtZVVJEmS1EKTFrf8N8CR46/NwDlAZOav11SbJElSa0w6\nPHcd8GXgtzLzHwEi4s21VCVJktQykw7PvQq4CfhSRPxlRBzMo6uCS5IkDcqkFcE/B3wuIh4PHAq8\nCdg1Iv4C+GxmXlxTjcNz8lGP3faGs+qvQ5IkPWLFieCZeW9mnpWZr2B0HrpvAsdXXpkkSVKLlKzT\n9IjxauCnjL9Up6VGnxZzNEqSpMqULjkgSZI0aKsaaVLLlYxGgSNSkiStgU3TEJU2V5Ik6RE2TZKk\nYVn8h6Oj7yrknCZJkqQCjjSpWf7Fp6a4HpqkVbJpkobMxkGSinl4TpIkqYBNkyRJUgEPz6ldPFwk\nSWopmya1n42UJKkFbJrUTTZS0rD5HqAGOKdJkiSpgCNNUh38q1iSOs+mSf3hQpmSpArZNEmOAkmS\nCtg0SU1xZEySOsWmScOy1KiSJEkFbJrUX0NqkIaUVZIaEpk5/Z1G3AZsmvqOm7VnZu6y0o2GnB16\nmd/sBXqYHXzNm32CIWeHXuYve9yraJokSZL6xsUtJUmSCtg0SZIkFbBpkiRJKmDTJEmSVMCmSZIk\nqYBNkyRJUgGbJkmSpAI2TZIkSQVsmiRJkgrYNEmSJBWo5IS9MzMzOTs7W8WuGzM3N7e55Lw0Q84O\n/ctv9mFmB1/zZp9syNmhf/lLs1fSNM3OzrJx48Yqdt2YiCg6MeGQs0P/8pu9TN+yg6/5ktuZfZjZ\noX/5S7N7eE6SJKlAJSNNVXnbZY/d9t8Prr+OPqrjZzuUx28oObtm8ePS9seka/VWzZ/HdEz7/Wlo\n73edapokqeuG9ktG6hMPz0mSJBVwpEmS1FuO7GmaHGmSJEkqYNMkSZJUwKZJkiSpgE2TJElSAZsm\nSZKkAn56TpI0dX5qTX204khTRBxXsk2SJKnPSkaajgH+56Jtxy6xTZK0wFKjLZK6a9mmKSKOBI4C\n9oqICxZ8awfg9qoLkyRJapNJI01fAW4CZoATF2y/G/h2lUVJkiS1zbJNU2ZuAjZFxGuBH2TmvwJE\nxHbAHsANtVQoST3nYbxuc9L7cJQsOfAp4OEF1x8Czq2mHEmSpHYqaZq2zswH5q+ML29TXUmSJEnt\nU/Lpudsi4pDMvAAgIg4FNldbliRpLTxUNEw+7vUoaZpeD5wZEScDAfw/4OhKq5IkqSKLGwybC5Va\nsWnKzH8CDoiI7cfX74mI3SqvTJIkqUVWc+65rYEjIuIy4JsV1SNJktRKE0eaxssLHMpokctnMVrY\n8pXAlVUX5kdwJUlaO3+PTt+kFcHPAg4CLgZOAr4I/GNmXl5PaVI9nEApSSoxaaRpX+AO4Frg2sx8\nKCKynrL6wV/GkiT1x6QVwfePiH2AI4FLI2IzsENE7JaZt9RWoTrJYWFJa+X7h9pq4kTwzLwuM9+T\nmfsAxwGnAV+PiK/UUp0kSVJLlKzTBEBmzgFzEfE2RnOdJEmSOmEaU2aKm6Z5mZnU8Ok5SZK6ygU0\nm1XVId5VN02SpPo5z0dq3moWt5QkSRqsSes0TTy/XGaePv1yJEmS2mnS4bnnLLP9EGB3wKZJkiRt\noc9rFE5ap+mN85cjIoDXAscDVwEfqL40SVKf9PmXqYZhpXPPbQ0cC7yVUbP06sz8hxrqUoWGPKF0\nyNkltYtNZPdMmtP0+4wWtLwMeGlm3lBXUX221o+h+uLqNh+/etkcS6rCpJGmk4BbgecBB46O0AEQ\njJZr2q/i2iRJklpjUtO0V21VSJKkIo6kNmfSRPBNi7dFxAzww/Gq4JIkSYMxaU7TAcCHgNuB9wFn\nADPAVhFxdGZeWE+JkqriXCup/RxZao9Jh+dOBt4JPAH4IvCyzLwqIvYBzgZsmiRJ0mBMapq2zsyL\nASLijzPzKoDMvG7BpHBJHeFfq1tylE3Sak0699zDCy7ft+h7zmmSJEmDMmmk6ZkRcRejJQa2G19m\nfH3byiuTJElqkUmfnttQZyESeMhEktReE0+jIkmS1s85hf1g06TWc/RJUok+NCZ9yNBnNk0NsyGo\njm8+kqRpsmmSpJ7zjzNpOmyaJHWaI4rSsNX5HmDT1ELTfAIM6RfKkLJ2nSMfmrfW162v925Z/Hh1\n9fVu09QjXXwT6WLN09KXN5GVDPkxltQvnW+ahvKLR5INmNQX6xltbvJ9oPNNk6Sl2WBI6pIuvGfZ\nNHVYF55gVelj9tJMfR1N7eNjKqlfbJqkjmlzc9Hm2lQtH3sNwVZNFyBJktQFkZnT32nEbcCm8dUZ\nYPPU76QeC2vfMzN3Wek/9Cg7PFp/UXbYIr/Zu8vsI6t9zQ85++L/3zVDzg6+5ouzV9I0bXEHERsz\n85cqvZOKrLf2LmeH9dVvdrN3kdmH+X435Ozg83419Xt4TpIkqYBNkyRJUoE6mqZTariPqqy39i5n\nh/XVb/buMnv9/7cNhvx+N+Ts4PO+WOVzmiRJkvrAw3OSJEkFKm+aIuKciLh6/HVDRFxd9X2uRUR8\nIiJujYjvLNj2xIi4JCKuH/+78yr3eXhEXBMRD0dEZz5dsNTPYo376Vx+s5t9nfsx+wCzj/c12PxD\nyl5505SZR2Tm/pm5P3Ae8Jmq73ONTgVeumjb24HLMvMZwGXj66vxHeBVwJXrrq5ep/LYn8VadDH/\nqZh9vcxu9q44lelkh2HnH0z22k6jEhEBvAZ4YV33uRqZeWVEzC7afCjwgvHl04DLgeNXsc9rAUbR\nu2OZn8Va9tO5/GaP2Snsx+xm74RpZR/va7D5h5S9zjlNBwG3ZOb1Nd7neu2WmTeNL98M7NZkMZIk\nqTlTGWmKiEuBJy/xrRMy8/zx5SOBs6dxf03IzIyIx3zUsDB7bw05v9nNvojZe27I+YecfaGpNE2Z\n+aJJ34+IrRkd73z2NO6vRrdExFMy86aIeApw6+IbrJS974ac3+zDZPbhGnL+IWdfqK7Dcy8CrsvM\nG2u6v2m5ADhmfPkYYDDdtCRJWiQzK/9iNEv99XXc1zpqPBu4CXgQuBH4beBJjD41dz1wKfDEVe7z\n34/3dT9wC3BR0znX+rNY4346l9/sZje72c1v9uW+XBFckiSpgCuCS5IkFbBpkiRJKmDTJEmSVMCm\nSZIkqYBNkyRJUgGbJkmSpAI2TZIkSQVsmiRJkgr8fyHKwy7zLMiUAAAAAElFTkSuQmCC\n",
|
|
"text/plain": [
|
|
"<matplotlib.figure.Figure at 0x7f1a8af019b0>"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
}
|
|
],
|
|
"source": [
|
|
"f, axs = plt.subplots(4, N_HIDDEN+1, figsize=(10, 5))\n",
|
|
"plt.ion() # something about plotting\n",
|
|
"def plot_histogram(l_in, l_in_bn, pre_ac, pre_ac_bn):\n",
|
|
" for i, (ax_pa, ax_pa_bn, ax, ax_bn) in enumerate(zip(axs[0, :], axs[1, :], axs[2, :], axs[3, :])):\n",
|
|
" [a.clear() for a in [ax_pa, ax_pa_bn, ax, ax_bn]]\n",
|
|
" if i == 0: p_range = (-7, 10);the_range = (-7, 10)\n",
|
|
" else:p_range = (-4, 4);the_range = (-1, 1)\n",
|
|
" ax_pa.set_title('L' + str(i))\n",
|
|
" ax_pa.hist(pre_ac[i].data.numpy().ravel(), bins=10, range=p_range, color='#FF9359', alpha=0.5);ax_pa_bn.hist(pre_ac_bn[i].data.numpy().ravel(), bins=10, range=p_range, color='#74BCFF', alpha=0.5)\n",
|
|
" ax.hist(l_in[i].data.numpy().ravel(), bins=10, range=the_range, color='#FF9359');ax_bn.hist(l_in_bn[i].data.numpy().ravel(), bins=10, range=the_range, color='#74BCFF')\n",
|
|
" for a in [ax_pa, ax, ax_pa_bn, ax_bn]: a.set_yticks(());a.set_xticks(())\n",
|
|
" ax_pa_bn.set_xticks(p_range);ax_bn.set_xticks(the_range)\n",
|
|
" axs[0, 0].set_ylabel('PreAct');axs[1, 0].set_ylabel('BN PreAct');axs[2, 0].set_ylabel('Act');axs[3, 0].set_ylabel('BN Act')\n",
|
|
" plt.pause(0.01)\n",
|
|
" \n",
|
|
"# training\n",
|
|
"losses = [[], []] # recode loss for two networks\n",
|
|
"for epoch in range(EPOCH):\n",
|
|
" print('Epoch: ', epoch)\n",
|
|
" layer_inputs, pre_acts = [], []\n",
|
|
" for net, l in zip(nets, losses):\n",
|
|
" net.eval() # set eval mode to fix moving_mean and moving_var\n",
|
|
" pred, layer_input, pre_act = net(test_x)\n",
|
|
" l.append(loss_func(pred, test_y).data)\n",
|
|
" layer_inputs.append(layer_input)\n",
|
|
" pre_acts.append(pre_act)\n",
|
|
" net.train() # free moving_mean and moving_var\n",
|
|
" plot_histogram(*layer_inputs, *pre_acts) # plot histogram\n",
|
|
"\n",
|
|
" for step, (b_x, b_y) in enumerate(train_loader):\n",
|
|
" b_x, b_y = Variable(b_x), Variable(b_y)\n",
|
|
" for net, opt in zip(nets, opts): # train for each network\n",
|
|
" pred, _, _ = net(b_x)\n",
|
|
" loss = loss_func(pred, b_y)\n",
|
|
" opt.zero_grad()\n",
|
|
" loss.backward()\n",
|
|
" opt.step() # it will also learns the parameters in Batch Normalization\n",
|
|
" \n",
|
|
"plt.ioff()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 11,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAY8AAAEKCAYAAADq59mMAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XecVfWd//HXZzoMDAwwIFVAQaTIgCOQJa4gVrLWuJZN\nomgiUddEN9lssPxiEjcbszHGFnXtJZZYV7NiwRZLbKBIRzoOIH0oIjAzfH5/nHOZOzDlXrh3zszl\n/Xw8zuOe873nnPu54zgfvuV8v+buiIiIJCMr6gBERKTlUfIQEZGkKXmIiEjSlDxERCRpSh4iIpI0\nJQ8REUla2pKHmfU0szfNbI6ZzTazK8LyDmY2xcwWhK/FYbmZ2a1mttDMZpjZ8Lh7XRCev8DMLkhX\nzCIikhhL13MeZtYV6Orun5hZW2AacDowAdjg7jeY2SSg2N1/bmbjgR8B44GRwC3uPtLMOgBTgTLA\nw/sc6e4b0xK4iIg0Km01D3df5e6fhPtbgLlAd+A04KHwtIcIEgph+cMe+ABoHyagE4Ep7r4hTBhT\ngJPSFbeIiDQupyk+xMx6A8OAD4Eu7r4qfOtLoEu43x34Iu6y8rCsvvK6PmciMBGgsLDwyAEDBqTm\nC4iIHACmTZu2zt1LEjk37cnDzNoAzwBXuvtmM9v9nru7maWs3czd7wbuBigrK/OpU6em6tYiIhnP\nzJYlem5aR1uZWS5B4njU3Z8Ni1eHzVGxfpE1YfkKoGfc5T3CsvrKRUQkIukcbWXAfcBcd78p7q0X\ngNiIqQuA5+PKzw9HXY0CNoXNW68AJ5hZcTgy64SwTEREIpLOZqvRwPeAmWY2PSy7GrgBeNLMvg8s\nA84O35tMMNJqIbANuBDA3TeY2fXAx+F5v3b3DWmMW0REGpG2obpRU5+HHMgqKyspLy9n+/btUYci\nzVBBQQE9evQgNze3VrmZTXP3skTu0SSjrUSkaZWXl9O2bVt69+5N/CAVEXdn/fr1lJeX06dPn32+\nj6YnEclA27dvp2PHjkocshczo2PHjvtdK1XyEMlQShxSn1T8bih5iIhI0pQ8RCQtysvLOe200+jX\nrx+HHHIIV1xxBTt37tzrvJUrV3LWWWc1er/x48dTUVGxT7H88pe/5MYbb9yna6VuSh4iknLuzpln\nnsnpp5/OggUL+Pzzz9m6dSvXXHNNrfOqqqro1q0bTz/9dKP3nDx5Mu3bt09XyJIkJQ8RSbk33niD\ngoICLrzwQgCys7P54x//yP33388dd9zBqaeeyrHHHsu4ceNYunQpgwcPBmDbtm2cffbZDBw4kDPO\nOIORI0cSG3Lfu3dv1q1bx9KlSzn88MO5+OKLGTRoECeccAJff/01APfccw9HHXUUQ4cO5dvf/jbb\ntm2L5gdwANBQXZFMd/u/pO/elz9WZ/Hs2bM58sgja5UVFRXRq1cvqqqq+OSTT5gxYwYdOnRg6dKl\nu8+54447KC4uZs6cOcyaNYvS0tI6779gwQIef/xx7rnnHs4++2yeeeYZvvvd73LmmWdy8cUXA3Dt\ntddy33338aMf/Sg131VqUc1DRJrc8ccfT4cOHfYqf/fddzn33HMBGDx4MEcccUSd1/fp02d3Yjny\nyCN3J6BZs2Zx9NFHM2TIEB599FFmz56dni8gSh4iknoDBw5k2rRptco2b97M8uXLycnJobCwcL/u\nn5+fv3s/OzubqqoqACZMmMDtt9/OzJkzue666/SEfRqp2Uok09XTtJRO48aNY9KkSTz88MOcf/75\nVFdX89Of/pQJEybQunXreq8bPXo0Tz75JGPHjmXOnDnMnDkzqc/dsmULXbt2pbKykkcffZTu3etc\n+kdSQDUPEUk5M+O5557jqaeeol+/fvTv35+CggL+67/+q8HrLrvsMtauXcvAgQO59tprGTRoEO3a\ntUv4c6+//npGjhzJ6NGj0WJw6aWJEUUy0Ny5czn88MOjDiNp1dXVVFZWUlBQwKJFizjuuOOYP38+\neXl5UYeWcer6HdHEiCLSIm3bto2xY8dSWVmJu3PHHXcocTRTSh4i0my0bdsWtRi0DOrzEBGRpCl5\niIhI0pQ8REQkaWlLHmZ2v5mtMbNZcWV/MbPp4bY0tra5mfU2s6/j3rsr7pojzWymmS00s1tNixSI\niEQunTWPB4GT4gvc/Rx3L3X3UuAZ4Nm4txfF3nP3S+LK7wQuBvqFW617ikjzlJ2dTWlpKUOHDmX4\n8OH8/e9/b/D8iooK7rjjjkbvO2bMmEY71ZcuXYqZcdttt+0uu/zyy3nwwQcTij1V4mPd1ynlb775\n5loTPO7P1PSplLbk4e5vAxvqei+sPZwNPN7QPcysK1Dk7h948EDKw8DpqY5VRFKvVatWTJ8+nc8+\n+4zf/va3XHXVVQ2en2jySFTnzp255ZZb6lxDJBGxKU9SZV+nlN8zeTSXqemj6vM4Gljt7gviyvqY\n2adm9jczOzos6w6Ux51THpaJSAuyefNmiouLAdi6dSvjxo1j+PDhDBkyhOeffx6ASZMmsWjRIkpL\nS/nZz34GwO9+9zuGDBnC0KFDmTRp0u77PfXUU4wYMYL+/fvzzjvv1PmZJSUljBs3joceemiv96ZP\nn86oUaM44ogjOOOMM9i4cSMQ1BSuvPJKysrKuOWWW5gwYQKXXnopo0aNom/fvrz11ltcdNFFHH74\n4UyYMGH3/S699FLKysoYNGgQ1113XZ3xxKaUv+uuuygtLaW0tJQ+ffowduzYeu9x6623snLlSsaO\nHbv7vNh9AG666SYGDx7M4MGDufnmmwEanLI+laJ6zuM8atc6VgG93H29mR0J/K+ZDUr2pmY2EZgI\n0KtXr5QEKtLS/ez19N379+Pqf+/rr7+mtLSU7du3s2rVKt544w0ACgoKeO655ygqKmLdunWMGjWK\nU089lRtuuIFZs2Yxffp0AF566SWef/55PvzwQ1q3bs2GDTUNGVVVVXz00UdMnjyZX/3qV7z22mt1\nxvDzn/+ck08+mYsuuqhW+fnnn89tt93GMcccwy9+8Qt+9atf7f7ju3Pnzt1NTRMmTGDjxo28//77\nvPDCC5x66qm899573HvvvRx11FFMnz6d0tJSfvOb39ChQweqq6sZN24cM2bMqHdG4EsuuYRLLrmE\nyspKjj32WH7yk58A1HmPH//4x9x00028+eabdOrUqdZ9pk2bxgMPPMCHH36IuzNy5EiOOeYYiouL\n652yPpWavOZhZjnAmcBfYmXuvsPd14f704BFQH9gBdAj7vIeYVmd3P1udy9z97KSkpJ0hC8iCYo1\nW82bN4+XX36Z888/H3fH3bn66qs54ogjOO6441ixYgWrV6/e6/rXXnuNCy+8cPdEivFTuJ955plA\n7enY69K3b19GjhzJY4/VTA65adMmKioqOOaYYwC44IILePvtt3e/f84559S6xymnnIKZMWTIELp0\n6cKQIUPIyspi0KBBuz/7ySefZPjw4QwbNozZs2czZ86cRn8+V1xxBcceeyynnHLKPt3j3Xff5Ywz\nzqCwsJA2bdpw5pln7q6F1TdlfSpFUfM4Dpjn7rubo8ysBNjg7tVm1pegY3yxu28ws81mNgr4EDgf\nuK3Ou4pIs/WNb3yDdevWsXbtWiZPnszatWuZNm0aubm59O7dO+mp02NTssdPx16fq6++mrPOOmt3\nsmjMntPFxz4rKyur1lTwWVlZVFVVsWTJEm688UY+/vhjiouLmTBhQqPf58EHH2TZsmXcfvvtAPt0\nj4bsOWV9i2q2MrPHgTFAJzMrB65z9/uAc9m7o/wfgV+bWSWwC7jE3WN11MsIRm61Al4KNxFJUENN\nS01l3rx5VFdX07FjRzZt2kTnzp3Jzc3lzTffZNmyZUAwNcmWLVt2X3P88cfz61//mu985zu7m63q\nWkCqMQMGDGDgwIH89a9/5aijjqJdu3YUFxfzzjvvcPTRR/PII48knFjqsnnzZgoLC2nXrh2rV6/m\npZdeYsyYMfWeP23aNG688UbeeecdsrKyGr1H7OeyZ7PV0UcfzYQJE5g0aRLuznPPPccjjzyyz98j\nWWlLHu5+Xj3lE+ooe4Zg6G5d508FBqc0OBFJu1ifB4C789BDD5Gdnc13vvMdTjnlFIYMGUJZWdnu\nqdM7duzI6NGjGTx4MCeffDK///3vmT59OmVlZeTl5TF+/PhGp3SvzzXXXMOwYcN2Hz/00ENccskl\nbNu2jb59+/LAAw/s8/ccOnQow4YNY8CAAfTs2ZPRo0c3eP7tt9/Ohg0bdneAl5WVce+999Z7j4kT\nJ3LSSSfRrVs33nzzzd3lw4cPZ8KECYwYMQKAH/zgBwwbNiwtTVR10ZTsIhmopU7JLk1nf6dk1/Qk\nIiKSNCUPERFJmpKHSIbK1CZp2X+p+N1Q8hDJQAUFBaxfv14JRPbi7qxfv56CgoL9uo9WEhTJQD16\n9KC8vJy1a9dGHYo0QwUFBfTo0aPxExug5CGSgXJzc+nTp0/UYUgGU7OViIgkTclDRESSpuQhIiJJ\nU/IQEZGkKXmIiEjSlDxERCRpSh4iIpI0JQ8REUmakoeIiCRNyUNERJKm5CEiIklT8hARkaSlLXmY\n2f1mtsbMZsWV/dLMVpjZ9HAbH/feVWa20Mzmm9mJceUnhWULzWxSuuIVEZHEpbPm8SBwUh3lf3T3\n0nCbDGBmA4FzgUHhNXeYWbaZZQN/Ak4GBgLnheeKiEiE0jYlu7u/bWa9Ezz9NOAJd98BLDGzhcCI\n8L2F7r4YwMyeCM+dk+JwRUQkCVH0eVxuZjPCZq3isKw78EXcOeVhWX3ldTKziWY21cymahEcEZH0\naerkcSdwCFAKrAL+kMqbu/vd7l7m7mUlJSWpvLWIiMRp0pUE3X11bN/M7gH+LzxcAfSMO7VHWEYD\n5SIiEpEmrXmYWde4wzOA2EisF4BzzSzfzPoA/YCPgI+BfmbWx8zyCDrVX2jKmEVEZG9pq3mY2ePA\nGKCTmZUD1wFjzKwUcGAp8EMAd59tZk8SdIRXAf/q7tXhfS4HXgGygfvdfXa6YhYRkcSYu0cdQ1qU\nlZX51KlTow5DRKTFMLNp7l6WyLl6wlxERJKm5CEiIklT8hARkaQpeYiISNKUPEREJGlKHiIikjQl\nDxERSZqSh4iIJE3JQ0REkqbkISIiSVPyEBGRpCl5iIhI0pQ8REQkaUoeIiKSNCUPERFJmpKHiIgk\nTclDRESSpuQhIiJJS1vyMLP7zWyNmc2KK/u9mc0zsxlm9pyZtQ/Le5vZ12Y2PdzuirvmSDObaWYL\nzexWM7N0xSwiIolJZ83jQeCkPcqmAIPd/Qjgc+CquPcWuXtpuF0SV34ncDHQL9z2vKeIiDSxtCUP\nd38b2LBH2avuXhUefgD0aOgeZtYVKHL3D9zdgYeB09MRr4iIJC7KPo+LgJfijvuY2adm9jczOzos\n6w6Ux51THpbVycwmmtlUM5u6du3a1EcsIiJARMnDzK4BqoBHw6JVQC93Hwb8BHjMzIqSva+73+3u\nZe5eVlJSkrqARUSklpym/kAzmwD8EzAubIrC3XcAO8L9aWa2COgPrKB201aPsExERCLUpDUPMzsJ\n+A/gVHffFldeYmbZ4X5fgo7xxe6+CthsZqPCUVbnA883ZcwiIrK3tNU8zOxxYAzQyczKgesIRlfl\nA1PCEbcfhCOr/hH4tZlVAruAS9w91tl+GcHIrVYEfSTx/SQiIhIBC1uOMk5ZWZlPnTo16jBERFoM\nM5vm7mWJnKsnzEVEJGlKHnvYvn41Fau/jDoMEZFmLankYWZZ+zKEtqVYPHMWf/w4iz9/sp3qnTui\nDkdEpNlqNHmY2WNmVmRmhcAsYI6Z/Sz9oTWtzRsruOfLw9iQW8KyvN787Z3Pog5JRKTZSqTmMdDd\nNxNMC/IS0Af4XlqjikBRcXvGFX2x+/hVH87KmZ9GGJGISPOVSPLINbNcguTxgrtXAhk5RGtsWR96\n+moAqi2Hx7/oRNV6PZMoIrKnRJLH/wBLgULgbTM7GNiczqCikp1tnFvWjlzfCcCX+T159b3PoWpn\nxJGJiDQvjSYPd7/V3bu7+3gPLAPGNkFskejcvoDx3bfuPn6r9TEs+dvLEUYkItL8JNJhfkXYYW5m\ndp+ZfQIc2wSxReYfBnTg0LzgAXe3LP6ycwQ75n0QcVQiIs1HIs1WF4Ud5icAxQSd5TekNaqIZRmc\nU1ZMgQfDddfnHcSLc7bBxlURRyYi0jwkkjxiy76OBx5x99lxZRmrfSvjtMNqjt9vdyzz33hJ/R8i\nIiSWPKaZ2asEyeMVM2tLMHlhxjuyRz6Dir7affxk4else+cvEUYkItI8JJI8vg9MAo4Kp1HPAy5M\na1TNhBmcNbSQQgtqG5tzO/C/G/vCAvV/iMiBLZHRVrsIFmG61sxuBP7B3WekPbJmok0enDU4d/fx\np+1G89mHn0KF5r8SkQNXIqOtbgCuAOaE24/N7L/SHVhzMrizcWTnqt3Hz5Z8l82vPqD+DxE5YCXS\nbDUeON7d73f3+4GTCJaRPaCcdngO7XODBLItpy1P552Iv/toI1eJiGSmRGfVbR+33y4dgTR3rXLg\nnME1Cy/ObTucj7+ohIUfRhiViEg0EkkevwU+NbMHzewhYBrwm/SG1Twd2gG+2aNmWq8XunyPDW8/\nC5tWRxiViEjTS6TD/HFgFPAs8AzwDXdPaLyqmd1vZmvMbFZcWQczm2JmC8LX4rDczOxWM1toZjPM\nbHjcNReE5y8wswuS/ZKpdPKhRkmrYKTyjuxW/KXTBHa9fBtUV0YZlohIk6o3eZjZ8NgGdAXKw61b\n/B/2RjxI0EcSbxLwurv3A14PjwFOBvqF20TgzjCODsB1wEhgBHBdLOFEIS8bzh2URVY4sfDiwsN5\np3oAvPdYVCGJiDS5nAbe+0MD7zkJzG/l7m+bWe89ik8DxoT7DwFvAT8Pyx92dwc+MLP2ZtY1PHeK\nu28AMLMpBAnp8cY+P116tYNjexuvLQ2OXy45m8PmXcNB3T+GQ46KKiwRkSZTb/Jw93TNnNvF3WOT\nRH0JdAn3uwNfxJ1XHpbVV74XM5tIUGuhV69eKQx5b+P6wNx1zoqtRlVWHk90u4wfvX4D2SUHQ1Hn\ntH62iEjUklrDPNXCWkbKFpZy97vdvczdy0pKSlJ12zrlZMG5g4wcC8Jf0aoPrxWdAC/fCtVVjVwt\nItKyRZE8VofNUYSva8LyFUDPuPN6hGX1lUfuoDZw4iE1c0S+0el0vtgM/D2yFjURkSYRRfJ4AYiN\nmLoAeD6u/Pxw1NUoYFPYvPUKcIKZFYcd5SeEZc3CP/aCPuFTMLssm8e7XUbljNdg8dRoAxMRSaNE\npid5PZGyeq59HHgfOMzMys3s+wRrgRxvZguA46hZG2QysBhYCNwDXAYQdpRfD3wcbr+OdZ43B1kG\n5wyE/Oyg+Wptfjcmdz4XXr8LNq+NODoRkfSwoNuhjjfMCoDWwJsEI55i7TNFwMvuPqApAtxXZWVl\nPnVq0/3r/8MV8PS8muMfLvtPDm27E878BWQ3NKhNRKR5MLNp7l6WyLkN1Tx+SPA0+YDwNbY9D9y+\nv0FmmhHdYEDHmuO/dPshX69dAR9o/Q8RyTz1Jg93v8Xd+wD/7u593b1PuA11dyWPPZjBPx8OrcNK\nRkVuCX/t8j349EVY8km0wYmIpFgiHeZfhqsHYmbXmtmzSTxhfkApyocz4xrzPm4/hlltjoTX7oIt\n66MLTEQkxRJJHv/P3beY2TcJOrjvI5w6RPY2tAuUdqk5fqbrD9haZfDKbXr+Q0QyRiLJozp8/RZw\nt7u/SLAUrdTjjMOgKPwJbc1pxzMHfR//8nP48KloAxMRSZFEkscKM/sf4BxgspnlJ3jdAat1Lvzz\nwJrjWUUj+KTom/DJX2Hpp9EFJiKSIokkgbMJHso70d0rgA7Az9IaVQYY0BFGxc3A9b8HXUBFTgd4\n7U7Yqv4PEWnZElnPYxvBFCLfDIuqgAXpDCpT/NOh0KFVsL89u5C/dLuEXdu/glduh13VDV8sItKM\nJfKE+XUEU6ZfFRblAn9OZ1CZIj8Hzh1Y83TlwsLBvF98PKyar/4PEWnREmm2OgM4FfgKwN1XAm3T\nGVQm6dMejjm45vjFzuexNu8gmPYCLPssusBERPZDIsljZ/zU6WZWmN6QMs+JfeGg8KdWmZXPE90u\npZqssP9jY7TBiYjsg0SSx5PhaKv2ZnYx8Bpwb3rDyizB2h+QHbZfLW/Vj7c6ngJfb4ZX1f8hIi1P\nIh3mNwJPA88AhwG/cPdb0x1YpuneFo7vW3M8peTbrMg/GFbOhSl3QsWq+i8WEWlmEukw/527T3H3\nn7n7v7v7FDP7XVMEl2nG9IJeRcF+teXwRLdLqbIcWPB3+PO/w4t/gBVzoZ6ZjkVEmotEmq2Or6Ps\n5FQHciDIDpuvcsOf+pcFvXil01nhuw5LpsFz18OT18L89zSdiYg0W/UmDzO71MxmEizkNCNuWwLM\naLoQM0tJa/jWoTXHf+t0Ckv6fKv2SWuXwJQ/wcNXBk+lb9/atEGKiDSiocWg2gHFwG+BSXFvbWlO\nK/nVp6kXg0rGLod7p8OC8KfYPh8u77eKdnNehHnvQHVl7Qty8+HwMTD0JGjXZa/7iYikQjKLQdWb\nPFq65pw8ACq2wx8+hO1hy1TXNnDZkVBQuRlmvQYzXg1GY9Vi0LcMSk+GrocFi4iIiKRIs04eZnYY\nEL+8Xl/gF0B74GIgtvD31e4+ObzmKuD7BDP8/tjdX2nsc5p78gCYvx7u/yyoiQD06wAXDQ2G9lK1\nEz7/O0x/CTZ8sffFnftC6Xg4dCRkZTdp3CKSmZp18qj14WbZwApgJHAhsDUcGhx/zkDgcWAE0I3g\nOZP+7t7gwxEtIXkAfLwSnpxbc3zkQXDOwLhKhTt8MQumT4bldTyR3rYTHHEiDBwL+a2bJGYRyUzJ\nJI+cdAfTiHHAIndfZvU3wZwGPOHuO4AlZraQIJG830QxptVR3aBiB7y6ODie9iW0L4CTDglPMINe\nQ4JtfTl89hLMf7emX2TLOnjvUfjomSCBDD0RijpH8l1E5MAR9boc5xLUKmIuD0d03W9mxWFZdyC+\n3aY8LNuLmU00s6lmNnXt2rV1ndIsHdcbRnSrOX59KXywoo4TO/aAYy+GC26FEd+GVkU171VuDxLL\nI/8GL90Mqz5Pc9QiciCLrNnKzPKAlcAgd19tZl2AdQRzaF0PdHX3i8zsduADd/9zeN19wEvu/nRD\n928pzVYx1bvgwRkwL1zqw4AJQ2FgpwYuqtoJn78XNGltqCPbdDkUho2HvkepX0REGpVMs1WUNY+T\ngU/cfTWAu69292p33wXcQ9A0BUGfSM+463qEZRklOwu+Oxh6hPMVO/DnmfDFngOu4uXkBU1V5/03\nnPJz6Dmk9vurF8LLtwa1kemTYee2dIUvIgeYKGseTwCvuPsD4XFXd18V7v8bMNLdzzWzQcBj1HSY\nvw70y5QO8z1t2QG3T4UN24Pjwly4vAw6JdoXvm552C/yHuza4wn13FbQbxS07Qh5hVBQCHmtg472\n/Dbha2GQlDQMWOSA0+xHW4XTui8H+rr7prDsEaCU4B/dS4EfxiWTa4CLCFYxvNLdX2rsM1pq8gBY\n8xX8aSpsC//2d2oVJJDCvCRu8lUFzJoCM6ck/4R6VnaQRPILaxJK7DUvvqyOc/JaQ3bU4zBEZF80\n++TRFFpy8gBYWgH/8ylU7QqOexXBD4dDXrJdF1U7g6fWP3sJNq5MeZx1yi2oSShZOTW1GDN2r6to\nWeGuxdVyrPa5tc4P92PXxM6t6977ItU1rX36/ypd/y+2kFpkU9V2m+3fvATiSjT2rofB8H9KOoKW\nNFRX6tG7PfzLIHhkZvArtXwzPDYbzh8CWcn8P5aTB4PHwaCx8MVsWLcUdnwFO7aF21fBtjPc3/7V\n3s1dyarcHmxbm/0sNiKZKSc3/R+R9k+QfTakM5zaH54PR93OXgsvfA6n9d+Hf6RZVs3zIo2p2hmX\nYL6q2d8Zf1zH+zu2wc6vmvG/7EQkVZQ8mrlv9gzmwfrb8uD4vfLgIcIxBzd83X7JyQu2wuLGz92T\n7wpqHbHEUl1NUHfysFbuYXLxmiRT6zX+/YbOj9uvdf6+2NcLnYabhOp5r8HEn+qmm338bk2e/5v6\nA5u4KS/hj0vgxETuVdgh0Q/cZ0oeLcD4Q4ME8tma4PjFhcFMvKUHRRtXnSwr6DTPax1MnSIiGSnq\nJ8wlAVkWLCLVt31N2RNzYNHG6GISkQObkkcLkZMFFxwBncPnPao9eCL9S60TJSIRUPJoQVrnwg9K\noSh83mN7Fdw3HTZtjzYuETnwKHm0MMWt4KJSyA+f96jYAfd9VrOolIhIU1DyaIG6t4XvxT3vsWor\nPDyz5oFCEZF0U/JooQ7rCP98eM3xgg3w1Fw9YiEiTUPJowUr6won9q05/uRLeGVxdPGIyIFDyaOF\nG9cbRu6xkNT75VFFIyIHCiWPFs4MzjgMBnSsKXtuPsxZF11MIpL5lDwyQH0LSS3fFGlYIpLBlDwy\nRH4OXDQUOhQEx5W74P7PYJ0WDxSRNFDyyCBt8+EHw4KHCQG+qgweIty6M9q4RCTzKHlkmJLWcOER\nwXQmAOu+hgc+g50NLtorIpIcJY8MFFtIKjZz8/LN8Ngs2KVnQEQkRSJLHma21Mxmmtl0M5salnUw\nsylmtiB8LQ7LzcxuNbOFZjbDzIZHFXdLEVtIKmb2Ovjf+XqIUERSI+qax1h3L41bM3cS8Lq79wNe\nD48BTgb6hdtE4M4mj7QF+mZPOKZXzfH7K+Ct5dHFIyKZo7ktBnUaMCbcfwh4C/h5WP6wuzvwgZm1\nN7Ou7r4qkihbkD0Xkpq8ED5eCQe3q9m6FCa5LrqIHPCiTB4OvGpmDvyPu98NdIlLCF8CXcL97sAX\ncdeWh2W1koeZTSSomdCrVy+kZiGpLTthcUVQtnZbsE0Nf3r52dCrHfQqqkkosRFbIiJ1iTJ5fNPd\nV5hZZ2CKmc2Lf9PdPUwsCQsT0N0AZWVlat0PxRaSenpu0PexZ8f5jupgYsUFG2rKSloHSSSWUA5q\no9qJiNQfXY5PAAAPUklEQVSILHm4+4rwdY2ZPQeMAFbHmqPMrCsQNrawAugZd3mPsEwS1DoXzj8C\nKquhfAss21SzbanjOZC6aic942omvdpBoWonIgesSJKHmRUCWe6+Jdw/Afg18AJwAXBD+Pp8eMkL\nwOVm9gQwEtik/o59k5sNfdoHGwSjryq2xyWTzbBiS921k4Ubgy2mpHWQRA5W7UTkgBNVzaML8JyZ\nxWJ4zN1fNrOPgSfN7PvAMuDs8PzJwHhgIbANuLDpQ85MZsHqhMWtoPSgoGzP2snyTbC5gdrJNNVO\nRA445hk68L+srMynTp0adRgZYXftZHNNQlm5BaoT+NXp0x5O7w/d2qY/ThHZP2Y2Le7RiQY1t6G6\n0gzVqp2E499itZPlcc1dm3fsfe2SCrjlYxjTC47rEzSbiUjLp+Qh+6TOvpMdNc1cyzYFyWWXB9sb\ny2DGGjjrcDikONrYRWT/KXlISphBcUGwxWona76Cp+cFtQ8IJmm86xMY0Q2+daieJRFpyaKenkQy\nWOdCuGQ4nHkYFMQ1V320Em78IKiJZGiXm0jGU/KQtMoy+EYP+PdRMLikpnzLTnhkJjw0EzZtjy4+\nEdk3Sh7SJNoVBE+5nz8EivJqymevDWoh75dryniRlkTJQ5rUkM5BLWRkt5qy7dXw7Hy4a1rQTyIi\nzZ+ShzS5VrnBqKtLhkOnVjXlSzbBTR/ClCVQtSu6+ESkcUoeEplDiuEnI+HY3jXTmlQ7vLoYbv4o\nGO4rIs2TkodEKjcbTj4ErjgqmNokZvVX8Kep8Px82F4VXXwiUjclD2kWurWFy8vg1H6QG/5WOvBu\nOfzhA5i7LtLwRGQPSh7SbGQZHN0r6FDv36GmvGIH3P8ZPDoLttYxQaOIND0lD2l2OrSCH5TCeYNq\nz8o7fTX8/v1gjRE9XCgSLSUPaZbMYPhB8LNRwWvMtir4yxy4Zzqs/zq6+EQOdEoe0qwV5gU1kB+U\nBvNmxSzYEPSFvLUMqjWsV6TJKXlIi3BYR/jpSDi6J8QWK6zcBS8uhNumBqsfikjTUfKQFiM/B07t\nDz86Crq2qSlfsQVu/ThIJKu2wlc71Scikm6akl1anJ5FwXMhf1te8zT6Lg+asN5aFpyTbVCUD23z\ngtd2+Xsft82H1jlB/4qIJKfJk4eZ9QQeJljH3IG73f0WM/slcDGwNjz1anefHF5zFfB9oBr4sbu/\n0tRxS/OSnRU8mT6kMzw9FxZX1H6/2mHj9mBr8D5hkinKDyZs3L0fJppY0mmlJCNSSxQ1jyrgp+7+\niZm1BaaZ2ZTwvT+6+43xJ5vZQOBcYBDQDXjNzPq7e3WTRi3NUklr+OFwmLYKPl0dTO++eUcw2WIi\nEk0yOVk1tZZYkunYCg7vBJ1a7//3EGlpmjx5uPsqYFW4v8XM5gLdG7jkNOAJd98BLDGzhcAI4P20\nBystQpbBUd2CLWZndZBENu+AzTvr3t+SRJKp2lV3knlhQdD/MrgEjugMXQpVQ5EDQ6R9HmbWGxgG\nfAiMBi43s/OBqQS1k40EieWDuMvKaTjZiJCXHdQIGqsV7KgKEsqWMKFsituPTzY7Gkgyq7YG25Ql\nwSzBQzoHW4+2SiSSuSJLHmbWBngGuNLdN5vZncD1BP0g1wN/AC5K8p4TgYkAvXr1Sm3AkpHyc6Ak\nJ2j+akgsyWyOSyyLK+DzDbWnj1/3Nby5LNja58PgzjCkBHq3r5k5WCQTRJI8zCyXIHE86u7PArj7\n6rj37wH+LzxcAfSMu7xHWLYXd78buBugrKxMgzUlZepKMsccHMz4O289zFoDc9cHzWUxFTvg3S+C\nrU1e0LQ1uAQOLQ46/EVasihGWxlwHzDX3W+KK+8a9ocAnAHMCvdfAB4zs5sIOsz7AR81Ycgi9SrI\ngdIuwVZZHdREZq6FOWvh67ip5LfuhA9WBFurHBjYKaiVHNYhmJZepKWJouYxGvgeMNPMpodlVwPn\nmVkpQbPVUuCHAO4+28yeBOYQjNT6V420kuYoNxsGlQRb9S5YtDFIJLPWwNbKmvO+roJpXwZbXjYM\n6Bj0kQzoGCQjkZbAPEMfxS0rK/OpU6dGHYYIuxyWVsCstUEyqahnWHBOVjAV/eAwAbXOrfu8TLPx\n6yChdilUc17UzGyau5clcq7+nSOSZlkGfYuD7ZR+UL4lqI3MXAtrt9WcV7UL5qwLtqx5wTK9Q8JE\nUpQfXfypVlkNiypg/jqYv6HmZ5CfDX3aB9/7kGLo3laDDJoz1TxEIuIeLLc7cy3MXBMM962LAQe3\nCx5IPLhdMD1LXgvqJ3EPRqHNC5PFoo21R6jVpyAb+oSJ5NDi4HkaJZP0Us1DpAUwg4PaBNvxfWDd\ntrBpaw0s31xzngNLNwUbBH9Au7aBXkXBEOCD20GHgub1TMmOqiBJzFsP89fDhgae4M/Ngla5wfDn\neNurg+WHY0sQt8qBvnE1k4OUTCKlmodIM7Rpe00fyeKNQQJpSJvcIInEth5NXDuJ1aLmrw8SxpKK\nYOqX+nRuHUyzf1jHICHkZAWLey3aGDRpLdq4dzLZU+vc4NpDw2Sip/v3XzI1DyUPkWZu687gX99L\nN8GyTbDmq8aTSZZBtza1E0pximsn26uCRbnmh7WLigb+2OdnB3/kYwmjQ6uG7x1r6lq0MdgWbmx8\n/frC3JpaySHFQYJSMkmOkgdKHpK5vq6C5WEiWbYpaOLaXtX4dW3y4pJJUdB3kswzJu5Bv0ysKWrp\npmAkWX0OahM8xzKgY9C8lrMfI6ncYc22mmSyaCN8VdnwNW3zaieTTq2UTBqj5IGShxw4dnlQG1kW\nl1DWbGv8uljtpHdc7aT9HrWTbZVB7WLeevh8fTBFS30KsqFfBxjQKRhy3L6g/nP3V6yZLFYrWVwR\nxNqQovwwkbSHbm2DZNomVw9pxlPyQMlDDmzbKoMaSSyZfLEpsRmEi/KgV7tgQsklFUENp6G/EN3b\nhk1RHYLkE9VzGrscvtxaUytZXFH7Cf+GFOQESaRtXphQ8mr22+YH78XK8jN8iJGSB0oeIvH2tXay\np9Y50D/st+jfofk+f7IrbGJbGCaTJRsTn36/IblZcYmlrmQT91rQAhcQ01BdEaklK25Y8MhwQYNt\nlbWTyReb95563ghGbh3WMei76FnUMobHZllQK+reFo7pFSSTFVvCRFIRPOW/tTLohG+o32ZPlbuC\nYccNDT2Oybaa5FKQHYx+i9/yGznesywnq3klIyUPkQNU69zgwcPDOwXHsaafZZuCP47d2gR9GG3y\noo0zFbIsSHw9i2DMwTXluzxo3tq6A7bsDJLJlp01iaVW2c7EHm6MqXbYtCPYUsGoP8nsmWy6toEj\nu6bmc+uj5CEiQNiB3jbYDhRZFgzxLcyFLo2c6x7UzPZMKLHXPfcbWkBsXzjBPRO57+ASJQ8RkWbB\nLOjHKEhg8TAI1naJJZKd1TXbjup9O27oocs9NcUDokoeIiJpkJcdPAzZ2AORiareVXdyqausc2Fq\nPrMhSh4iIi1Adha0CucBaw40e76IiCRNyUNERJKm5CEiIklT8hARkaS1mORhZieZ2XwzW2hmk6KO\nR0TkQNYikoeZZQN/Ak4GBgLnmdnAaKMSETlwtYjkAYwAFrr7YnffCTwBnBZxTCIiB6yW8pxHd+CL\nuONyYOSeJ5nZRGBieLjVzObv4+d1Atbt47XNnb5by5XJ30/frXk4uPFTAi0leSTE3e8G7t7f+5jZ\n1ESnJW5p9N1arkz+fvpuLU9LabZaAfSMO+4RlomISARaSvL4GOhnZn3MLA84F3gh4phERA5YLaLZ\nyt2rzOxy4BUgG7jf3Wen8SP3u+mrGdN3a7ky+fvpu7UwGbsMrYiIpE9LabYSEZFmRMlDRESSpuQR\nJ5OnQDGznmb2ppnNMbPZZnZF1DGlmpllm9mnZvZ/UceSSmbW3syeNrN5ZjbXzL4RdUypZGb/Fv5O\nzjKzx82sIOqY9pWZ3W9ma8xsVlxZBzObYmYLwtfiKGNMFSWP0AEwBUoV8FN3HwiMAv41w74fwBXA\n3KiDSINbgJfdfQAwlAz6jmbWHfgxUObugwkGxJwbbVT75UHgpD3KJgGvu3s/4PXwuMVT8qiR0VOg\nuPsqd/8k3N9C8Aeoe7RRpY6Z9QC+BdwbdSypZGbtgH8E7gNw953uXhFtVCmXA7QysxygNbAy4nj2\nmbu/DWzYo/g04KFw/yHg9CYNKk2UPGrUNQVKxvxxjWdmvYFhwIfRRpJSNwP/AeyKOpAU6wOsBR4I\nm+TuNbMmWKG6abj7CuBGYDmwCtjk7q9GG1XKdXH3VeH+l0CXKINJFSWPA4yZtQGeAa50981Rx5MK\nZvZPwBp3nxZ1LGmQAwwH7nT3YcBXZEizB0DY/n8aQZLsBhSa2XejjSp9PHg2IiOej1DyqJHxU6CY\nWS5B4njU3Z+NOp4UGg2camZLCZobjzWzP0cbUsqUA+XuHqslPk2QTDLFccASd1/r7pXAs8A/RBxT\nqq02s64A4euaiONJCSWPGhk9BYqZGUG7+Vx3vynqeFLJ3a9y9x7u3pvgv9sb7p4R/3p19y+BL8zs\nsLBoHDAnwpBSbTkwysxah7+j48igAQGhF4ALwv0LgOcjjCVlWsT0JE0hgilQmtpo4HvATDObHpZd\n7e6TI4xJEvMj4NHwHzWLgQsjjidl3P1DM3sa+IRgROCntODpPMzscWAM0MnMyoHrgBuAJ83s+8Ay\n4OzoIkwdTU8iIiJJU7OViIgkTclDRESSpuQhIiJJU/IQEZGkKXmIiEjSlDxE0sjMrjSz1lHHIZJq\nGqorkkbhU+9l7r4u6lhEUkk1D5EUMbNCM3vRzD4L16a4jmC+pjfN7M3wnBPM7H0z+8TMngrnGsPM\nlprZf5vZTDP7yMwOjfK7iDRGyUMkdU4CVrr70HBtipsJphcf6+5jzawTcC1wnLsPB6YCP4m7fpO7\nDwFuD68VabaUPERSZyZwvJn9zsyOdvdNe7w/imChsffCKWIuAA6Oe//xuNeMWi1QMo/mthJJEXf/\n3MyGA+OB/zSz1/c4xYAp7n5efbeoZ1+k2VHNQyRFzKwbsM3d/wz8nmDq9C1A2/CUD4DRsf6MsI+k\nf9wtzol7fb9pohbZN6p5iKTOEOD3ZrYLqAQuJWh+etnMVob9HhOAx80sP7zmWuDzcL/YzGYAO4D6\naicizYKG6oo0AxrSKy2Nmq1ERCRpqnmIiEjSVPMQEZGkKXmIiEjSlDxERCRpSh4iIpI0JQ8REUna\n/weQ2pRl6ztibwAAAABJRU5ErkJggg==\n",
|
|
"text/plain": [
|
|
"<matplotlib.figure.Figure at 0x7f1a8c6594a8>"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"data": {
|
|
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD8CAYAAABn919SAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzsvXd4XOWZ9/95pqiMercsy5bkKrnKyDbEgAsQSkIJSwgh\nFEOAsEk2ZJO8u9kk176bzRbykl82YbMJSyek0UJJAgkQDBgwxjY2tlyw3C25qJfRaFRmzu+PW0cz\no2LJRdJIuj/XpWs0Z8555pn2Pfe5n7sYy7JQFEVRxj6O0Z6AoiiKcnZQQVcURRknqKAriqKME1TQ\nFUVRxgkq6IqiKOMEFXRFUZRxggq6oijKOEEFXVEUZZyggq4oijJOcI3kk2VmZloFBQUj+ZSKoihj\nns2bN9dalpU12H4jKugFBQVs2rRpJJ9SURRlzGOMOTSU/dTloiiKMk5QQVcURRknqKAriqKME0bU\nh94fnZ2dVFZW4vf7R3sqShQSFxfHlClTcLvdoz0VRYl6Rl3QKysrSUpKoqCgAGPMaE9HiSIsy6Ku\nro7KykoKCwtHezqKEvWMusvF7/eTkZGhYq70wRhDRkaGXr0pyhAZdUEHVMyVAdHvhqIMnVF3uSiK\noowrAgFoaAC/H+LiIC0NnM4ReeqosNBHm8rKSq6++mpmzpzJ9OnTueeee+jo6Oiz39GjR7nuuusG\nHe+KK66gsbHxtObyL//yL/zoRz86rWMVRRllvF5Ytw42boQdO+R23TrZPgJMeEG3LItrr72Wa665\nhoqKCvbs2YPX6+W73/1uxH5dXV1MnjyZZ599dtAxX375ZVJTU4dryoqiRCOBAGzaBMZAdjZkZsqt\nMbI9GBz2KUSPy+VnNw7v+F/9Tb+b33jjDeLi4rjtttsAcDqd/Nd//ReFhYUUFhby5z//Ga/XSyAQ\n4IknnuDTn/405eXl+Hw+1qxZQ3l5ObNnz+bo0aP8z//8D2VlZT0lDrxeL5dffjnnn38+7733Hnl5\nebz44ovEx8fz0EMP8eCDD9LR0cGMGTN48skn8Xg8w/seKIoyfDQ0QFubiHg4iYlQXQ319SLyw8iE\nt9B37NjBOeecE7EtOTmZqVOn0tXVxYcffsizzz7LW2+9FbHPz3/+c9LS0ti5cyc/+MEP2Lx5c7/j\nV1RU8JWvfIUdO3aQmprKc889B8C1117Lxo0b+eijjyguLuaRRx4ZnheoKMrI4PeDYwBJdTjk8WEm\neiz0KOWSSy4hPT29z/Z33nmHe+65B4B58+axYMGCfo8vLCxk0aJFAJxzzjkcPHgQgPLycr73ve/R\n2NiI1+vl0ksvHZ4XoCjKyOB2i5UeCEBsLCQlhRZDg0FZIB1mJrygl5SU9PGLNzc3c/jwYVwuFwkJ\nCWc0fmxsbM//TqeTtrY2ANasWcMLL7zAwoULefzxx3nzzTfP6HkURRlFvF4oL4dDh8Qaj42Vv5IS\nEfP4eOjHMDzbRI+gD+DjHm4uuugivv3tb/PLX/6SW265hUAgwDe/+U3WrFlzUp/28uXLefrpp1m1\nahU7d+5k+/btp/S8LS0t5Obm0tnZya9//Wvy8vLO9KUoijIaBALwwQfQ2gozZoioB4PQ3AzvvQdL\nl8KSJQO7Y84iE96Hbozh+eef55lnnmHmzJnMmjWLuLg4/uM//uOkx335y1+mpqaGkpISvve97zF3\n7lxSUlKG/Lw/+MEPWLZsGcuXL2fOnDln+jIURRktjhyR8MSDB+HYMRHuQACKimDaNJg7VxZGRwBj\nWdaIPBFAWVmZ1bvBxa5duyguLh6xOZwtAoEAnZ2dxMXFsW/fPi6++GI+/vhjYmJiRntq446x+h1R\nJgAdHfDkk7B/P0yZAh6P+M19PrAsmDoV5s+Xx84AY8xmy7LKBtsvelwuYwyfz8eqVavo7OzEsix+\n/vOfq5gryngnPAu0qwu2bIE9e8S90tEhC6NFRSLs9fXQ1DQii6E2KuinSVJSkrbTU5SJhNcrCUJt\nbWJ9b98ulnhGBrjCpHT/fpgzB9rbxVofgcVQmwnvQ1cURRmU3lmgdhSL0wk7d4qoBwIi4g0NUFkp\nC6MjtBhqoxa6oijKYPTOAm1shAMHRNDr62HXLskCzcsTS37SJCgoEB/6CKKCriiKMhjhWaCBgIQm\nWpZUUiwqEn+61yuhi+npYrEvXTqi1jmooCuKogxOXFyouFZjo/jOg0Gx3J1OmDlTxPvIEZg+HT71\nqUi/+gihPnQkg3PRokUsXLiQxYsX89577510/8bGRn7+858POu7KlSsHXTg9ePAgxhj++7//u2fb\nV7/6VR5//PEhzf1sET7X0y3/+5Of/ASfz9dz/0zKCCtKVJGWJtme1dXw4YfiI3e5xO1y4IAIfEcH\npKaKZT4KYg4q6ADEx8ezdetWPvroI/7zP/+Tf/qnfzrp/kMV9KGSnZ3NT3/6035rsA+Frq6uszYX\nOP3yv70FXcsIK+OK/HzYsAFqa8Xd4nZLNMuSJeKGKSmRJKKMjFGbYtS4XP7PX4d3/PsuGtp+zc3N\npKWlAeD1ern66qtpaGigs7OTf/u3f+Pqq6/m29/+Nvv27WPRokVccskl3Hffffzwhz/kV7/6FQ6H\ng8svv5x7770XgGeeeYYvf/nLNDY28sgjj3DBBRf0ec6srCyWL1/OE088wZ133hnx2NatW7n77rvx\n+XxMnz6dRx99lLS0NFauXMmiRYt45513+PznP8/27duJj49ny5YtVFdX8+ijj/LLX/6S9evXs2zZ\nsh6L/2//9m/ZuHEjbW1tXHfddXz/+9/vMx+7/O+zzz7LAw88AEBTUxMFBQWsXbu23zHuv/9+jh49\nyqpVq8jMzGTt2rU942RmZvLjH/+YRx99FIA77riDr3/96xw8eHDA8sKKEjXY4YrHj4tbJTVV4s7T\n02UR1OGQhdG2NkhOHtEwxd5EjaCPJm1tbSxatAi/38+xY8d44403AIiLi+P5558nOTmZ2tpazj33\nXK666iruvfdeysvL2bp1KwCvvPIKL774Ihs2bMDj8VBfX98zdldXFx988AEvv/wy3//+93n99df7\nncM//uM/cvnll3P77bdHbL/lllv47//+b1asWME///M/8/3vf5+f/OQnAHR0dPS4SdasWUNDQwPr\n16/npZde4qqrruLdd9/l4YcfZsmSJWzdupVFixbx7//+76SnpxMIBLjooovYtm3bgJUi7777bu6+\n+246OztZvXo13/jGNwD6HeNrX/saP/7xj1m7di2ZvWo+b968mccee4wNGzZgWRbLli1jxYoVpKWl\nUVFRwW9/+1seeughrr/+ep577jluuummU/0IFWV4CA9XtMU6NVX+P3QolBna3Cwhi2VlI74QGo66\nXAi5XHbv3s2f//xnbrnlFizLwrIsvvOd77BgwQIuvvhiqqqqOHHiRJ/jX3/9dW677baeYl7h5Xav\nvfZaILJ0bn8UFRWxbNkyfvObUJGypqYmGhsbWbFiBQC33norb7/9ds/jn/vc5yLGuPLKKzHGMH/+\nfHJycpg/fz4Oh4O5c+f2PPfTTz/N4sWLKS0tZceOHezcuXPQ9+eee+5h9erVXHnllac1xjvvvMNn\nPvMZEhISSExM5Nprr2XdunXAwOWFFSUqsMMVExMl7jwQkO3p6eKCyc+XKJcZM2DFihGr2TIQaqH3\n4rzzzqO2tpaamhpefvllampq2Lx5M263m4KCAvynWKTeLp/rdDoH9XV/5zvf4brrrusR8MHoXdrX\nfi6HwxFRttfhcNDV1cWBAwf40Y9+xMaNG0lLS2PNmjWDvp7HH3+cQ4cO8bOf/QzgtMY4GQOVF1aU\nqCA8XDEpSRZG29rk1uWS6JeEBLHUh7kb0VCIGkEfqo97uNm9ezeBQICMjAyamprIzs7G7Xazdu1a\nDh06BEjaf0tLS88xl1xyCf/6r//KF77whR6XS39NMQZjzpw5lJSU8Ic//IElS5aQkpJCWloa69at\n44ILLuDJJ58cstj3R3NzMwkJCaSkpHDixAleeeUVVq5cOeD+mzdv5kc/+hHr1q3D0f2lPtkY9vvS\n2+VywQUXsGbNGr797W9jWRbPP/88Tz755Gm/DkUZMcLDFZ1OKC6WJKK6OnGzNDeLmI+yq8UmagR9\nNLF96CBNo5944gmcTidf+MIXuPLKK5k/fz5lZWU9ZW4zMjJYvnw58+bN4/LLL+e+++5j69atlJWV\nERMTwxVXXDFo+d2B+O53v0tpaWnP/SeeeKJnUbSoqIjHHnvstF/nwoULKS0tZc6cOeTn57N8+fKT\n7v+zn/2M+vp6Vq1aBUBZWRkPP/zwgGPcddddXHbZZUyePJm1a9f2bF+8eDFr1qxh6dKlgCyKlpaW\nqntFiX7S0sTVUlUlFnlsLCxYACdOiM985UqxzKNAzEHL5ypjAP2OKKOG1wvr1sG2bdDZKeGKMTFS\nEvfCC0euzvnZLJ9rjPl74A7AArYDtwG5wO+ADGAzcLNlWacXSK0oihJt2BEuHg+sXh2KZOnsFFfM\nSTqajRaDXicYY/KArwFllmXNA5zADcAPgf+yLGsG0AB8cTgnqiiKMqKER7jY8ec5OdKsor1dYs+j\njKE6flxAvDHGBXiAY8BqwO6u/ARwzdmfnqIoyggRCEgWaGWl3DY3Q0uL+MsbG0MhiyACfwbRXcPF\noC4Xy7KqjDE/Ag4DbcCriIul0bIsOw6vEtAux4qijE3Cm1c4HCLmW7aIoOfliYslPl7S+z0eiXwZ\nwU5EQ2UoLpc04GqgEJgMJACXDfUJjDF3GWM2GWM21dTUnPZEFUVRhoX+mlds2AA1NRLdUlUlRbk6\nOqSZRXOziPsopvgPxFBcLhcDByzLqrEsqxP4PbAcSO12wQBMAar6O9iyrActyyqzLKssKyvrrExa\nURTlrBHuKw8EpJpiR4c0p8jPl3DFlhaoqIBjx0TQoyTuvDdDmdFh4FxjjMcYY4CLgJ3AWuC67n1u\nBV4cnikOL6dbOVFLwyrKOCE8G7SlRQTbdqfExoqwl5RIFcUpU6C0dNRT/AdiUEG3LGsDsvj5IRKy\n6AAeBP4R+IYxZi8SuvjIMM4zRO+Fi/CFitNgIEEfLE1fS8MqyjghPBu0vT3yvmXJ/aQkEfSYmKgM\nV7QZUhy6ZVn/F/i/vTbvB5ae9RmdjN4LF8Gg+LLKyk77jBleCtftdhMXF0daWhq7d+9mz549XHPN\nNRw5cgS/388999zDXXfdBYRKzHq9Xi0BqyhjGbt5hdcbav4cGwtNTSLgds0kn09cMFHoO7eJPifQ\nQPReuMjMlFtjZLt9Rj1F7r33XqZPn87WrVu57777+PDDD/npT3/Knj17AHj00UfZvHkzmzZt4v77\n76eurq7PGBUVFXzlK19hx44dpKam8txzz53RS1UUZQRxOsUotCwxFtvbRW+am6WJRUODLIzGxMCq\nVVHpO7eJ3pn1JnzhIpzERNl+loL8ly5dSmFhYc/9+++/n4ULF3Luuedy5MgRKioq+hyjJWAVZYyT\nmCip/PPmSZii2w2TJ4u4Hz8ufvTbb5c66FHM2CnOFb5w0ZuzGOQfXpL2zTff5PXXX2f9+vV4PB5W\nrlzZb6lYLQGrKGMce21u40ZISZEmz62toVT/pKSoXQgNZ+wIevhCRW/OIMi/dynccJqamkhLS8Pj\n8bB7927ef//903oORVGimPAWc3v3ihV+7JhEttiBD9XV4gWIgprnJ2PsCHr4wkX4mdLrPaMg//BS\nuPHx8eTk5PQ8dtlll/HAAw9QXFzM7NmzOffcc8/0VSiKEk30bjGXmip/Pp8kES1eLB6AKE31783Y\nEXR74WLTJjlb9o5yOYOFivC2b+HExsbyyiuv9PuY7SfPzMykvLy8Z/u3vvWt056HoigjjL02l50N\nXV2hMGiPRyzy5mYR+ChN9e/N2BF0CC1c1NfL2TIuTizzKF51VhQlijlZizmHQ3zoZ+gFGEnGlqCD\nvMlR7sdSFCWKCQTEMvf7ZeGzs1O2j4EWc4MRFYJuWRZSVUBRIhnJjlrKBKB3cmJnpyyEulyQlSXi\nXVoKR49GZYu5wRh1QY+Li6Ouro6MjAwVdSUCy7Koq6sjbgz4LpUxQO/kxEBAardkZMDmzdJWLjZW\n/OUpKWeUgT5ajLqgT5kyhcrKSrS0rtIfcXFxTJkyZbSnoYwHwhdAfT5xrdix5nV1Uh9q5Up5fIyu\nzY26oLvd7ojMTEVRlGHBXgANBETM29slmai9XR6rrIRXX5WM0DEo5jCWUv8VRVHOBDs5saVFLPPj\nx2V7SooU4MrPlzroa9eedm2o0UYFXVGUiUFamvjIDx0SMW9slIJbfr/UbklIkEXRpqaobAA9FEbd\n5aIoijIi2JUUDx6UyBa/X1wu+fkwZ04oWTEubkxkhfaHCrqiKOMbu/DW22+LhX7ppRKmeOCAZIG6\n3WKp+3zyeELCmMgK7Q8VdEVRxi8DFd4qK5OkofZ2cb1UVkr44rRpoQz0MYj60BVFGZ/0V3grPV3u\nHzkiJXILC8UiT0uTfqFxcWMmK7Q/1EJXFGV8MljhrWAQLrgA9u2TUrmTJ4/Z+HMbFXRFUcYnQym8\n5fPBpEnSqWgMC7mNCrqiKOMTux9oICCLoDk5klAUDEoNl8zMMVV4ayiooCuKMv7weqG8XGLOOzvh\nxAlpAj11qljlxsCyZbIIOk7EHHRRVFGU8Ya9GOpyiWg3Nsp2p1MWQ/Py4LzzROzHGWqhK4oyvghf\nDG1slMShmBhJ629thaIiWfwcI31CTwUVdEVRxhfhi6Ht7WKZJyXJfacz1NBiCH1CLQvaA9AZgK4g\ndHb/BS15DCAIYIFFaJv9v8NAUdpZfn0nQQVdUZTxhV2ECyTz0w5XBNkeGxv6v1dG6NEW2FULh5qg\n3g+NfhH00yXWCf+28vSPP1VU0BVFGT8EAvLn9YolnpUVCle0LBHz5OQ+fUKrWuBPe6FibNbk6kEF\nXVGU8UFTE7z5ptwCVFSIiyU/XxZDAWbNkrou8fE94YofHoendoob5Wwz0g0UVdAVRRn7NDXBY4/J\nwqfHI5mhdknctja46Sa539ERqtXicLChCp7dffKhXQ5xnbid4HbIfYcBg0Q/2o0z7f/t7SDHjCRD\nEnRjTCrwMDAPOencDnwMPAUUAAeB6y3LahiWWSqKogxEICCWeXu7hCT6/VJJsblZlDU9XcIYL7pI\nIl+6Oe6F5z/uO1xRKpROgrwkSIuDBHdIoKOdocah/xT4s2VZc4CFwC7g28BfLcuaCfy1+76iKMrI\nUlsr1RLb28VS37tXtmdni888Lk786Zs29SyWBoLiZgmE+UScBj4/F+5eDOfmQX4yJMaMHTGHIQi6\nMSYFuBB4BMCyrA7LshqBq4Enund7ArhmuCapKIrSL01N8Mc/wo4dsH+/3B44EApbNEaEPi1NXC/d\nnYg2HYPKlsihri+BxZPGloD3ZigWeiFQAzxmjNlijHnYGJMA5FiWdax7n+NAznBNUlEUpQ+23/zw\nYXG7tLXB0aPiP6+sFGvc74fERIls6Y47D1rw9uHIoRZkQ+k4ULChCLoLWAz8wrKsUqCVXu4Vy7Is\nBljQNcbcZYzZZIzZVFNTc6bzVRRFifSbT58u2Z5Tp0q6f02N+M8rK2Uh1C6+1R13vqcOqn2hoRwG\nrpw5ti1zm6EIeiVQaVnWhu77zyICf8IYkwvQfVvd38GWZT1oWVaZZVllWVlZZ2POiqJMdBoaxEJP\nSBCxLiyU9P78fAlJDASkLO5VV4mFHhZ3/vaRyKEWZEPq2Ow414dBo1wsyzpujDlijJltWdbHwEXA\nzu6/W4F7u29fHNaZKoqi2LS2SghiXZ3Emns8MHu2bE9IgJYWKC4Wl4vP1xN3fszn6JM8dOHU0XkJ\nw8FQ49D/Dvi1MSYG2A/chlj3TxtjvggcAq4fnikqiqKE4fXC1q3iUmloEFFPTJSiW0lJ4i8/5xxY\nvrxP3Pm6nZFDFaZINMt4YUiCblnWVqCsn4cuOrvTURRFOQmBAHzwQagOy6RJ4nppaYFt26SNXFyc\nxJwnRyp1Szt8eDxyuAvGkXUOmimqKMpYIRCAjz6C114TS9zlkpT+YFBizpubJbHo6qv7iDnAe1WR\ncefp8TB3nC3rqaArihL9eL2wYQO8/rpEsdip/fPny2MdHbBkCXziE/2KeWcA1ldGbrsgXyJcxhPa\nsUhRlOjG7kDk9YqvPDVVRNsYOHgw1BsUQre9+PA4tHaG7sc5oSx3+Kc+0qigK4oS3dgdiNxucbXE\nxkr0ip3S39oaamTRXQ43nEAQ3uqVSLQsD+LGoX9CBV1RlOjG7kAUGys1zQsLZXtTk4h5dbX40Zcs\n6bfh84ajUNMrkWh5/gjNfYQZh+coRVHGFXYHoqQkiSe3rFDMeXW1ZIpOniyZor1o64RX90duO2eS\nVFEcj6iFrihKdJOWFuo6VFwsgt7QID71uDiJbFm6tI91blnw1K5I37nbAZdOH+H5jyBqoSuKEt04\nnVBaGupGlJws2aButwj51Kn9ivlf9sOOXuWjVk6DlNiRm/pIo4KuKEp04/XCli0Sdx4XJz71lBRY\ntarfEMVAEP5YAe/0ClPMS4JV00ZozqOECrqiKNGLHbJojGSF2ni98OGHcOGFEdZ5Uzv8phz2N0YO\n43HBLfNHviXcSKM+dEVRohc7ZDExMXJ7YmJEwwqAinr4yYa+Yh7vgtsWSmboeEctdEVRohc7ZLE/\nuhtWALxfJf1Bg726MiTHwhcXwuSkYZ5nlKCCrihK9BAIiFVuJw653T19QPvQXaDr9QOyANqbmelw\n41zpCzpRUEFXFCU68HrFX97WJtZ3Z6fUaGltlf8nTZKIF3vf+HjWtab3EXMDXFwof+OtVstgqKAr\nijL6hC9+ZmdLU4pdu6TWeXOzxCHGxMCiRRKTHh9PedEyXqqIdMe4HXDjPJg3zqooDhVdFFUUZfQJ\nX/wMBETM29vFEq+vD4UnfvQRzJlDTdmF/G5/5Cqn2wF3lk5cMQcVdEVRooHwxc+WFnGzHO/uRpGS\nIolEs2ZBQgLBbdv4zU5DeyB0uMPAmgVQmDryU48mVNAVRRl97HotIO6W+nqorZVtgYC4WwA8Ht7t\nyqWyJdI5fuVMmJUxwnOOQtSHrijK6GPXa6muhooK2L9fXC6NjRLpMns2AI0OD39JKY04dH4WLJ8y\nGpOOPsaWhR4IyFm7slJuA4HBj1EUJfqx67V8/LFEtiQkyK3HA7m50siitZUX8lbRbtw9h8W54JrZ\nspaqjCULvXdIUzAoZ/Sysr5ZZIqijC0CAfGZp6XJ77mwEN56S4TeGKivp7xoGTsSCyMOu2K6JA8p\nwtgQ9N4hTTa2yPeq56AoyhjATiKqq4Pdu+X/qqpQV6Irr5Rol5YW/LEeXsi/OOLwqcnSeUgJMTYE\n3Q5pChdzkDN5dbUsoGRmjs7cFEU5dWxjzOuF8nJZCLUsMcxSUsR/fvgwLF8OXi9/8ebS5EroOdxh\n4LriiZc4NBhjQ9B713MIBCS0yY5TbW1VQVeUsUL4FbdliVUeGyu/58pK+U0XF4vIe70ccabybmJx\nxBArpkKuelr7MDYEvXdI065dYrE7nWKdx8ZCRob60hUl2gkEYO9esb4nTYLt20NWOcj/bW2wdSvk\n5BCoOspzeUuxCJniaXGS1q/0ZWwIuh3S1NQkXwZjRMB9PsjJkSwy9aUrSnRju1kOH4YjR8Qa37cv\n0hCLjYWCAhH1lBTeK7mUqqbIUonXzoaYcV7X/HQZG+rndEo0S0uLrIS3t4tlbllQUiKC3qs2sqIo\nUUS4myUvT8IS7e1Hj4pxBvKb7q6y2DhpGn/2RubxL8yGOepdHZCxYaGDnMVLS0XMExPlTJ6cHLLI\nw2ojK4oSZYQHNrS0iHXe2Cj/d3XJlXdGhmSEdnRgBYM8P3k1Hb6QqyXOBVfNGsXXMAYYO4IOkmSQ\nlCRfikBAqrC1t4u4d3bKmV1RlOjDDmwIBGDPHpg2TTJAm5tlW0KCPH7BBRATw7bEInb6PBFDaMz5\n4AxZ0I0xTmATUGVZ1qeNMYXA74AMYDNws2VZHcMzzW66fekd1bXEHDkUWhhtbRVRv/DCYX16RVFO\nEzuwoaVFfrcZGbIQmp0tFRSnTBHrPBDAm5zJ84550BU6fFqKxpwPhVPxod8D7Aq7/0PgvyzLmgE0\nAF88mxPrF6eTXYVL+KFZzpGOOBHzri7pBp6RAa+/DseOaWkARYk27MCG+vpQkwqHQyzzZctgxQoo\nKoKSEp7P+AStXSFXi9PAdXM05nwoDEnQjTFTgE8BD3ffN8Bq4NnuXZ4ArhmOCYZzuAme3Oeh2Z3I\nA8U38nH+Qjnru1ySYPSHP8AvfgEbN8rfunWysq4oyuhi12rx++HAgZDRZVkwbx6kp0NGBh95CthW\nE6nclxTBJI1IHhJDtdB/AvwDYDf3ywAaLcuyL4oqgWG9IGrywyMfQaclH3YHLh5NWM6HuYvk7G8X\nwU9IEHHPzJQV9U2bBu5JqCjK8BMISHGtV1+V36THIyHIgQDMmSP3vV5a4pJ5/mhkiOKUJFg5dXSm\nPRYZVNCNMZ8Gqi3L2nw6T2CMucsYs8kYs6mmpuZ0hgBkMWTZ5MhtQePgt9mreStmhiyOxsSIqLe3\ny2JLYqKGMyrKaOL1wptvwlNPiajbZTpyc8VdumEDHD9OMGjxVNIyWjsjXS2fKwHn2AiujgqGsii6\nHLjKGHMFEAckAz8FUo0xrm4rfQpQ1d/BlmU9CDwIUFZWZp3uRI2BK2ZAkjvIS3sjP+E/Zn2C5lnt\nfOrg6zgSEkLRL6DhjIoyWtix516vLIpmdHeg8PnkseJiWfOaMYN1sTP5eF/k7/qSQnW1nCqDnvss\ny/ony7KmWJZVANwAvGFZ1heAtcB13bvdCrw4bLMM44JpDr4ww4/TinSjvD19Fb9bdhtdDpe4WGK7\n45uCQQ1nVJThpr9eBXbsudsdWggFcbF0doqxlZNDpTOVV/ZHSlFhCqycNsKvYRxwJnHo/wj8zhjz\nb8AW4JGzM6XBWTQtjoT4Lp7YFogodr8lcQatJpZba35PjNcrGWhud6iNlVPzhRXlrDNQr4LcXLnv\ncvWNOHMx0cEAAAAgAElEQVQ4oL0dv3Hx62OZBMKu3eNd8Pl56mo5HU5J0C3LehN4s/v//cDSsz+l\noTEz28XfzvfxcLmFl5ie7XsS8nl4ypXc/qcfEdfeKgWAmpul5svKlaEiQIqinDkn61Wwe7dsN0bE\nvrpa3C5OJwSDWF1dPJNYRm1HpKF1XbEU4FJOnTF9DszL8fDVc11kxnRFbD+QXsRDn/oubVOLxF9X\nXi4VGh97TMRdUZSzg+1W6V3pNDFRWsht3y6VE91uCVfcuFEWR9vaWJdYzDYrssfBssmwoFfbA2Xo\njGlBB8iwfHylawN5nZGRLIcT8/jfxXfQOnma+NOTkuTL98wzYilo0pGinDnhvQoCAanPcvSoNHne\nvFlCipOSxO0yc6bEm3d2sv/Km/kTMyKGyk3UWi1nytiq5dKb7su9RIfhS8FNPNy2gMPxk3oerkrO\n44G5t3DXO/eTtGuX7N/aKrHqkyZpP1JFOVPslH67T0FjozSsaGyUCJbYWPmtFRWJq2XuXJr9Fr+q\nmUTQiiy8dct8LYt7poxtCz3sci8+1smdB56n0HcsYpfjCdn8Ytnf0pSUJeKdmSmC3tQEL7+s1rqi\nnAlpaZL/sXmzZH02NcnvLDlZLHM7muXoUcjKIpCSyq9iF9PSFSk9ny+BTM8Az6EMmbEt6OGXe0lJ\nxMU6uePYy8yo3ROxW01qHr/4xFdpSMyU/SsqxI/38cfwxhtaIkBRThenE6ZPl7yPujpJHOrokIXQ\n9HTZ1twshldzM3/yT+GASY8YYnUBlGT1P7xyaoxtQQ9vTed0QnExMQS4/eBLzNm/MWLXOk8Gv1h6\nN/XNHWKhZ2SIdZGcrCUCFOVMcLlgwQJxq+TmwuTJoQ5jJ05IrfP9+/nAl8q69pyIQ2ekwaVFozTv\nccjYFnS7gpttXXs8UFqKe85MbnXvYm7b4YjdG2JS+MV5X6XOeEKWg8ulJQIU5UywE/eyssTNUlsr\nv6tZs8SH7nKx35PH7+MWRhyWEgtfmKdVFM8mY1vQ7dZ0liW+8Npa+UtJwXXdtdx82RQWeCJdKY1x\nqTyQfTm1R+pk0ebAAVnQ0RIBinLqBALy5/WKgdTVJTXP4+Lkd1lURP2cRfxy5d8RcIRiMNwOWLMA\nEmNOMrZyyoztKBcQ6/rCC8W69vvli5SeDg4HzkCAG/2bcQRmstUZquzVmJTJAxd+nS/VvUZWZxPs\n3CkF9rVEgKIMnfAMUbdb1qSqqyXCpaMDYmLwTyvisenX0uqOXPH8XAlMSR6leY9jxr6gg1jXmf10\njm1owOn3cUPWMUyjYYvJ7XmoKSGdB2I/zZfq/0r2kd0SWpWe3ncMRVH60l+GaF6eZIfu2AFlZQTj\n4/lt3FKOkxFx6CWFsDCnnzGVM2Zsu1wGozsKxmngho4tnNO4O+LhZlcCv0hdzYnOGCkNYJ12MUhF\nmVj0lyHqcMDs2ZCWhhUTw4vJS9hJZPjKgiyLiwtHeK4TiPEt6GFRMI7YWK6vfJUlvr0Ru3hjEvnF\n4js4vv0AvPiiZLhpCztFOTnhIcPhOJ0wezZvmWm81xGZw5/nCfC5uUYXQYeR8S3o4VEwSUk44uO4\n7vhaljXtjNitNSaRB/Kv5uj75XDvvfDBB9rCTlFs+iuNGx4y3Ist8QX8ydM7osViTalTM0GHmfEt\n6OFRMHV1kJ2No76ea7c+xXmV70fs2ur28L+rv0VVzgyJfNEWdooiBs26dfD++/Dee5Jd/cc/inUe\nHjLczd5mF0+5FkRsi3PCFxcZUjXmYNgZH4uiJ6N3FMwnPoHj6FE+s349juNO3p20pGdXn9vD/y7/\nKnfu+BX5zc2Qmiqr9nbbLEWZSNgLn21tkrrf1iZGkh3ue/31sGeP/EYcDo5ZHp5wn0cgzE50Grh1\ngRTeUoaf8S/o0DcKxuHAJCRwdcMGHImJrEss7nmoze3hwXk3c6d/M1PtYzU+XZmI1NaKkB8+LPVa\nMjJC9c0rKuDpp+GznwWXi7rWIA8fycbfFelT+VwJzNDgsRFjYgh6b9LSICUFc+wYV7ZsxtHWyltZ\nZT0P+13xPGSW8EXvLgq0hZ0yEWlqEtfK5s2SeJeRIeLe0QHHj0vV0sOHobKS5oVLeXDqZ2juJeZX\nzIDSSQOMrwwLE1PQnU7pXnTgAKaykk9VbcCRf4i15/xNzy5+ZywPtZdwe6zFdI1PVyYSTU3SDKah\nQbI+fT5Zh/L5xP04dWpPCzlfi5+HPMup73JHDLF8CqycOkrzn8CM70XRk5GSArffDnl5mJgYLm8r\n56Ldr0Ts0uGM4REWsqdh4r5NygQjEIA335TqidOni6vS5ZIyuNXVIuxtbdDYSIcniUc++Q8cT82L\nGGJxjsVVs8Qzo4wsE9NCt0lOhhUrIDYWk5jIpW4XzuA+XnVM79ml03Lw2DYpvl+s66LKeKehQSz0\nhASxwouLoaZGtrW2ytWt309XYjKPX/YPHE6NNMOLg9Vcn+PAYfTHMhqo6enxSIW4nBxMejqXpDdy\nRVxlxC5dQXhiG2yvHqU5KspIYddDspPqPB6JEsvLk+1uN8HkZH576TeomDwv4tAiGrg5+BHOdg0i\nGC1U0HuX4AVWxZ3gKkdkRmnAgl+Vw9YTIz1BRRlB4uJExOPjxbUCcv/cc6GwkOCUfJ699O/ZVnRu\nxGF5/hrWJB3AHezSIIJRZGK7XCCUfLRpU088LcEgF8Q04spM5fe1oUvHoAW/KReLvSz3JGMqylgi\nEBBXi98vVRM9HsjPhyNHJCHP6YTWVqx583hh/nVsjJ0ecXhWewN3JO0jvq1dTgQaRDBqqKBD3+Sj\nri44cIDzjm7EZSbzjGsBVvcKjwU8vRMCQViWd/JhFSXqCS+B223MALL4mZcnfnO/Hys/nz9Mv4z1\nJ2IjDk8JtHKn2Upiq1/EvKys/xovyoiggm5jJx8FApLq7HBAYiJLdm3AFXOM3035JEEjX1QLeHa3\nWOrL80d32opy2gQCUrfI65VIFpdLrPPqahH4hQvB48HyJPBKYwbrDkeGrSTFWHypqJ00a2ZEHwJl\n9FBB741dFjQjA7ZuBWModdXibFzHr1PPJ2hCyRMv7BE3zAUab6uMRY4ckSJ08fEixK2tcOiQrCvZ\nnYjy8nh90nmsrYoU8wQ3fKnUkJWYDqiLJVrQ02lv/H653KyslJoVXV0QCLDAf4RbK1/BaUWW1H2p\nAt4+PMBYihKt2Na5wyGWdUwMbN8uWaD79knXocpK1nbk8mpVpJsl3gV3lUKO1meJOlTQe9PVJV/s\n7dulY/n+/dJay++npHk/t5ltuHqJ+h8q4C0VdWUs0dAg/vKYGLndtUuSh3JyZGHU6eTNyefysnN2\nxGFxTrizFCYnjdK8lZNirBHs0lNWVmZt2rTp9Af42Y1nbzJnwB7PPB7L/xZdjsgOt5868WtW1v9p\nlGalKGcHC3g98zO8mvXZiO0xQT93Hv5PCtoqRmdiY5Wv/uaMhzDGbLYsq2yw/Qa10I0x+caYtcaY\nncaYHcaYe7q3pxtjXjPGVHTfpp3xrMcIs3zl3H7kPtzB9ojtf8r5AmvTPz1Ks1KUM8cCXsm6oY+Y\nu4Pt3H7k/6mYRzlDcbl0Ad+0LKsEOBf4ijGmBPg28FfLsmYCf+2+P2GY6dvRr6i/nHMjazOuHKVZ\nKcrpE8DJs5PuYG3mVRHbY4J+bj9yH9N9uwc4UokWBhV0y7KOWZb1Yff/LcAuIA+4Gniie7cngGuG\na5LRygzfTr545P/hDkamOr+c/XneyLhqgKMUJfrwO+J5NP//8EHa6ojtcQEfdx7+T2b4dg5wpBJN\nnJIP3RhTALwNzAMOW5aV2r3dAA32/YE4Yx/6cGPHoBsjyUaNjVBeHurYUlYmi0lVVRIJU1AAe/aw\nb9YyHpl1PZ2OyCjQywqDXFSk687KCFNZCTt2hPIq3nkHDh6ErCwph5uXJ6GKlgWTJ1NfVMLjdXkc\n64xM2fdYHdw5t4Mp2m5o1DlrPvSwAROB54CvW5bVHP6YJWeFfs8Mxpi7jDGbjDGbampqhvp0o0N4\nD9LqaglbbG6WVf/8fAnxsqMCjBHB9/uZ7mjijoY3iAl0RAz35wMOXjswSq9FmbiEN3BuaZHvsF1f\nxbLkO+zxQHs75V1p/KS6sI+Yp8UE+PIyl4r5GGNIgm6McSNi/mvLsn7fvfmEMSa3+/FcoN9ahJZl\nPWhZVpllWWVZWVlnY87Di10GYMkSWLAAZs6E5culfnpbm5QVjY2VePX2dvk/IYGi+v3csf/3xFpd\nEcO9uh9e2z9Kr0WZmIQXnGtvDwm8XaslPp4ubyt/SFvCE8nn0xaMlIH8ZPi7pU5ykvTqcqwxlCgX\nAzwC7LIs68dhD70E3Nr9/63Ai2d/eqOEXQZg7lyYNEl+FMXFYt3U1cltba1YPy4X7N4N27ZR2HSI\nLx56gdhelvqrB+DFPVL/RVHOCoGAfAcrK+U2EJYbEX6l2dwsbePa20XQJ0/myHEfP510FW9PPq/P\nsPOy4O7FkBTb5yFlDDCoD90Ycz6wDtgO2JL0HWAD8DQwFTgEXG9ZVv3Jxop6H3p/hBcvamuDLVsk\nAWPaNPkx7d0rP6CMDKmrXlTEQXcWD+deRrsz8lcxIw1umi9p04py2vRXUMsujJUY5iIJBiXz85VX\noL2drtY2XkuYx5szLyboiOz/6SDIFdPhwmkO7TQUhQzVhz62EotGi2BQura8/ba4WLKzYds28aPX\n1MiPq6hIhD4YhHnzODhnGQ93zqPdRC6UpsfDTfPkslZRTpneC/c2Xq9Y5MuXS3chu2ro/v3Q2MiR\no16eyv8kJxL7dm1O7Wjhpn0vMO3TF8hCvxJ1DFXQtTjXUHA4xApPTBQxb2wMFfBKSJDogZwcWWxq\naYGMDApaj3J3WzWPZ6ymKRB6m+vb4GebYNU0uLgQXOqmVE4Fu3hcdnZoWyAg4r1vnxTcSk6WfT76\niEBiEm984kZeT5lGkL6m9zm+fVzVvBlP23Ep1GU3gFbGJCroQ8XvD33R29tF4EFE3I5+SUgQiygQ\ngIwMphw7xj0pO3ly4U0ccEU2yvjrQdh5vJPrc+qZkuqUhSyns+/zKko44d9DAJ9P6rC0tkqIbWp3\n5HBSErWdLn476/McDhb0GSa5y8t1zRspbq+SDbGx8r2tr5f1I2VMoqfioRIeCmZ/+UFE3O6KfuCA\nXPZmZ8s2n4+kzlbu2voo53X2jV885ndz/8Fs/vBhMx1vvxvRBk9R+iX8exgIiJgbI9ttf7oxfOzI\n5P6L/4nD6YV9hliy/x2+tefJkJj7fPKdTkmRE4YyZlELfaiEh4IlJYV6LloWzJghgn70qDzW0SH7\n+XzQ0ICrpYVrP/4BxQVlPLvkZppjQqXqLGN421nE9mAuf7NxD7NXzNdLXmVgwr+HXV0h11+lNDa3\nHA7emXMJfyi4BMtEfo8S6eCz7Vspee8RCcc1ATkBxMZCSYmMqf1AxzQq6EMlvPdoXZ1Y4R9/LI/N\nmiXhYVOnSuy6xwPvvisi73bLj6a6muLWt/jmzvW8sPw2tpREplg3EM/DwYUs2ernmoVxxKj3RRmI\nggLxd1dXywKoMeB0EkxI5Pezr2ZD4YV9DilpqOCznkMkxjlg/nxZxI+JETFPThbjQ/uBjnk0yuVU\nCQZDvUdjusvndnSID3PvXvlxbNwIH30Usnj27xcrKj9faqx3dLBr+VX8fv5naYzvW6QyNxFuXQAZ\n8SP82pToJjxcEWQBdP9+WLyYQG09v4stZWvBsohDjBXkih0vsqJpO2bWLMmrmDNHcicGC3tUogaN\nchku7KSj3gQCctm7ebNY5llZYtU3N8sx7e0h36cxFHcc41vv/Ii/zLuad3KXYIVFIBzzwk83BLmx\nxGJOtprqCvL92rRJrHE7wiUjA7xeuta9y68v/jrliZH+8rjONm6sfoPiSR2QOwdWrpTvrsMhY9iG\nifYDHTfoJ3i2cDph+nQR7kBALPakJPFzZmaKFX/ihOw3dSp0dBDb5uWqqrX8XcIusq3IBdG2gIPH\nthm2HvCN0gtSogo7XDHcgnY66Zo5i1+W3dZHzFP8jXx14y8odjVJSO0VV4iI26JtGyZTpoREXhnz\nqIV+NnG5xIfudIZCGWfMkMvbrCwR9aQkEfQ9eyRmfeZM8k9U8HeH1vLMrGvY5s7rGS5oHPxmXzyd\nwTaWTFf/y4Smd7giEOwK8JRjPrumTIvYnm78fCluB+mF2TBvnvypYE8I9FM+m9gRAhkZkrGXmCgW\nussl7pb582HFip6ypcyZI2GPVVXEuR3ctPMpPlWzHhO2rmEZw9MH43nviBaCmbDYV3wnTkhSWyCA\n1erjpSoPW+MixTwbH19O2kN6apx8DydPVjGfQKiFfjYJDylLTITSUvGhZ2bC4cMSKrZnj/zApkyR\naIVDh2Rx6/BhTGIiK2trSZnfxu+yVhIMCzt7fo+DGBeU5Y7ey1NGAXsh1OuVuizHj0NSEq+llvJu\n9ryIXTM7mri76kWSFhaDV6NWJiIq6GeT8NDG6upQBEF+Plx9dai+el6eWOZ2PZipU0XwY2LAGEq3\nv45rkeHX6RcSMKFF0Wd2gccNJZrINzEIXwjNzZXEn127eDc4mdeyz43YNbmrlTsPPEdSXRUkxkg0\nS1mZWucTDBX0s41dT72/CILmZlmgSk2NrAcTDIrAe72ycNXczPy63azp8PP4pE/2iHrQgie3Wdy1\nMEhhhka/jFsCAVkEPXpUumNlZYnLJTaWLSWreKF9ZsTu8aaLO1MOkD57GhyLkSQh9ZtPSFTQh4OB\nQhvD07bD68GACP2hQ2KpO51w/DhzfD5udLj5VfZFWN01Tbssw6NbLb4y38ekbM8IvBhlRAmPNa+q\ngvfek5P9tGnsTp/J76YWEV5jy02A2xP2MsnVLoZCR4f6zScw+qmPJOE+drsejN8vrpeGBmmo0Z1V\nSmMjNDWx4K1n+Mz+lyOG8ePi4a3QWNmrsYEytgl3sWRkyBVdYiIkJ3MgkMgv8z9FMMwF57CC3JKw\nnwJXq2zwetVvPsFRQR9JwjvJ2A0zNm6U+4sWiW/9nHPE0kpNlRDIkhLO8zRwad3GiKGanB4e3uHC\n9/Z6Leo1XgiPNW9pEWFPSuJoQg6PLbglogm5sYLccOTPzDm6XToWVVfL90j95hMadbmMNOE+9sxM\nqaGRkSGWus8nWabFxWLBt7fLD7yujotcH9Dc6GP99BU9Q51wpfJEcB53bPwQ94rz9Ycczdh+cXtd\npb9yyeGx5j4ftLdTmz2Nh2bcQFtMpHvt6raPKC2Ih+bu74vHo9meigr6qGD72P1+OO88iW6xG063\ntcHBg3J/926xvFJSMF4v11S9RsvNkyhPm90z1H5S+W3XLG6qrsHhcp5cMJTRYagt49xuEX2vF3bs\noKm2hQcvuQtvXGR7q0+eWM/yHB8kp8rn7fFoDXMFUJfL6GInIqWmhqJf4uPFSj94UMLUMjPlh+7x\n4MDixvWPUNh2LGKY7Y5JvLS5GWvjRtixQ9w469apKyYa6F2DJTNTrsiamuDll+WEHQjIZ1VeLjX1\nX3uN1sZWHrrwazR4Iv3h5zdu5+KWbVIEDuQEoTXMlW7UQh9NeicigZQGaG0VCz07W0T+wAFZJLUs\n3C2NrNn7e/5n/m1UE7oMfzduJslx8ayOOyEbbKvwwgv1Mnw06d0yzu4w1NYmi57BoIQl+v0i7JMm\n0VpZzf9e8DVOJOdFDHXOwfVc2fgeZunS0GcaDGoNc6UH/aWPJuGLpNXVsrhVVyeJRlOmiIj7/fKD\nnzlTfKWJiXiy07gjYQ8ppiNiuFf8U3inPUvuJCaKaNTXj8ILU3oI94uHdxjKyJArMMuSuvpPPw17\n9+I9Xs8Dq7/JsV5iXtJQwWePvIZjzhxxsYBGtSh9UAt9tOkvESkYFLdJXFyo7G5lpQjB7t2wbRtp\n+/bxxfj3+fm8W/G7Q4W7XmybihOL82Jr9XJ8tAmvweJyyX07mczvh4oKOWnv3g2VlbSkZvPgJf/A\n8fisiGFmBGu5KbsaZ3ysRL/YtYFsP7xegSndqKBHA70TkQIBSSbpDltj61ax5lNTYckSKCyE998n\nt/koaw68wMNFf0OXK6bn8N+3TcOJxdJgtV6OjzR2NEtdnQg1SMbn3r2S9BMfL/5v+7G6OmhqoiZ1\nMg9f+A3qe4t50wFuSzmI23LIZz9vnoyjNcyVflBBj0bCa8Ls3y/umORkiYKxf9ApKbB/P9OTq1hT\n/mseW3AzgbA45WfaCmjrbGRFMCgioxEvZ4+BQhDDC2nt3Cn7xsbKY/X1ckxNjRRjM0Y+w/Z2jsw8\nh0dW/z2tCZHdq2Ye28FtHz+N+9ylcoJfulQ7CiknRQU9WrFdMeXlUoI3N1dE3eGQy/T6ernNzWV2\nRge3bv8NT8y/MULU/5iwiNat+7jcvROzpJ/2YkOJjVYiGSgEsbQUtmwRoY6Pl1BUt1taEbpcYl37\nfOJKC2swvr1wKb8r+Swd7sgrqTkNe7ll73O4r7hM1k/UGleGgAp6NONwSF2Oqipxt4CIQkWFWO6N\njfKY10txQQE3HX2VX+V+koAz9LGudU6nxWvxN398GdfqleK/Dbcmta/k0OmvDRxICOJLL8l7OGWK\nuFEOHJDw07o62WfbNintMH8+NDYSrDrKX+ZfwxvzPt3nac45upnPdm7DeeH5sGyZCrkyZFTQo53w\n0Mb4eImSSEiQy3U7HK6jAw4eZF5GC7cf3Mcvz/sS7c6QT32TZwbVPsMtf15LytScSGsyXJg01FEY\n6MqldwgiyAl2716JVImNlX327BFxT0yUzwrkKmv/fsjJwbt8Fb+b+wU+zprT56lX73mVy068hykt\nFRfLRP4clFNGBT3aGcifPnOmPFZbK0Lf3AxOJ7Pcbr70/v/wyLK7aXWFol8OZ07nJ5053BzcQ9Gb\nb4obYNKkyOdKTJTx7bIE45HB3Ewnu3Lp3QYuPAwxK0uO6eyUyBafTwTe7j6VmAheLzsLl/BMzAV4\ns9wR03IGA1xd9QbnJZ6AyaXwqU/JZ6Qop4B+Y8YCA/nTFy+GzZtDoWw5OVBdTX5SF1858jyPZl9C\nbUIoasLrTuQBq5SVVgyfbN3d/4c/nkMdB3MzDeRS8XphwwZxf9khiElJ8r7bYYjGyILnoUMi6J2d\n4g5rb4eYGHyWiz8tvIEPCj4BVuS0kgI+bvaupzCrAxLyZT4q5sppYCzLGnyvgQ425jLgp4ATeNiy\nrHtPtn9ZWZm1adOm036+CU9trSyqhYtNMCjWeVWVCMH27T3dj9pO1PGbxTezO21mn6EmddTzuZi9\nTGmvFUsyKUks1WPHxPpPSBhfC6WBgJRDMCZyjcDrFSvazgXo/f6CWNvvvSedpyorZZtdlqGyUlxe\nFRUwbZoIfmWlvGcJCQTdMWxefAV/KryEVndCn2lNo5Gbp7WQEmc0FFEZEGPMZsuyygbb77TNAGOM\nE/gf4BKgEthojHnJsqydpzumMgj9lQpwOMSamzpVhNjvl7h1h4P4ulpu2/AQr866lL8WXRQx1PGY\ndO63yljauoPL9r1HYodXrP7qajlJxMaKlWk3TIiNlb+EBPkba0Lfn/8bIt1MvV0qIK9/40Y5aToc\n4tfes0fGO3BAToDubvdJY6N8Prm54POxb+mlvDxlBYc9fRvBOrC4OO4oqzv34qy0dN1COSucyXXd\nUmCvZVn7AYwxvwOuBlTQh4uBepbGx8OcOfDuuyL2qaniDrAsHAkeLtv/OkXHdvF06Y00hRV7soyD\nDenz2ZY4nVXbXuITL/6W2NkzxNosKRGffXm5CKHtAigslFj49PS+ETGBgFxFnOiuJ5OTI1ZsNAh/\nf2JtY7uZ4uLkJNbYGMrQ3bVLFj0dDnk/mprkvW5vh7/+VWrYJyfLCSEpCfx+DmZO5y8ln2Zv5qx+\nny7L4efzngPku3wQNwHWLZQR40wEPQ84Ena/Elh2ZtNRBqW/UgEpKSLmvZoJ43KJjz0jg1mJLXzz\nz9/nhUWf48OiT0QM2Rbj4eWyG3iz5HJWHHiL8/a/Tfz774sgZ2SIkGVlyYmjqUl8xWlpkRExXq+4\nNLZtE1G0LInFnj9f9hntUMjw9n+9sRcym5slbtzplHopFRWhVoFpaRKS6PdLlue0afJ+FxTA8eME\nmprZPmkh60rO53DqtH6fxmUFuCj+OCtiT+A2Ya7O8bxuoYwow77yYoy5C7gLYOrUqcP9dBOD3qUC\namsj3Qkej4QmZmaKSC1YAMEg8evX8/mNj7No11r+8Ik11KRFFoDyeVJ4Ze5VvD7nMhaXv855O18j\nzxOQhViHQ0Sxo0Niq4NB+b++XsRuwwYJ3UtPF+EH8T3v2SP3V6wYXZdCf+4qEOt47155Pbt3y61t\nbXd1yf3aWnk/HQ55b48dE596VRXVk4rYNOMyNi+eRXN86oBPPze5natq15OeltT3Qa2YqJwlzkTQ\nq4D8sPtTurdFYFnWg8CDIIuiZ/B8ykD0506wBTgjQ/527BAxio+nuGo3M//4Pd4puZTX519Fe69u\nOJ3OGDYsvIINC68gt7GSRclvsihwlHTaxSoNBEKWq98vgrdvn1jvdq9Ue7+mppALJydn4LDB8O22\nT7qz8+wtzPZ2V1mWuFY+/ljmFhcnc588WUS/qkrCOjMz5WrH65X3LxikuqGD8vzz2L7gi1Sm9W+N\n28xI7uLSWS4KEl2wjr4nFK2YqJxFzkTQNwIzjTGFiJDfANx4VmalnBoDuRNsce3slFu3W8TD7cbV\n2srKqvUsfe9Z1i35G94555qIqo02x1KncOy8m3gFyGmtZvbR7cxqPkCRvwN3jEus9L/8RYSyq0uu\nCExYW3qfT/z5b78NF1wgVnDvsME5c0Lb29tFZINBEVe75snKlXJ7OoSfLEpKRETtcE+7M1RXV6hp\nRGKizMuywOHAmzWZfa489sYUszdhCrUXDS6+sxPbWTnDzYwM+yd2kvUPrZionCXONGzxCuAnSNji\no+hYL/8AAA1HSURBVJZl/fvJ9tewxWFioJC8pibxaefmykKlvVjZ0SEVALOzZdGzs5O2C1fzXvEl\nrE8upiluYNeBjSvYyTTvUfJbj5HfcJCpuz4g5fhBTPfCIC4XzJol4pmSIlE4+/aJ6yInp6dnJn6/\ndGcqLZXoma1bZfuhQ/LYrFki9LGxcPvtIdEdiqUfFycnsS1bemqn9FjlRUVihR8+HFrUrK+n8/JP\ncSx5MpU+F5Vd8RzJKOJ4yuQhfQwxgQ5KO49wwdJJ5GT1DVEEQi4de34apqgMgaGGLZ6RoJ8qKujD\nyEBJM3PmiH/7o49EMA8eFGGbOlUs99paEcKsLEhIINAVYFdiAetzFlMRn4dlhi42iW1N5NQfIfvE\nfrI7m8gKtpBRV0lKWyPu9FQR6RkzZK6TJolIHz8uPuyrr5Y57NghcwI58RQViT/76FGYPRuuvFJO\nBr1fa0yMjLlzp7wuY+TEUFUlJ4ukJHGd1NTQtf8AzZ0OmqbNpjZlEjVZBVTHp1NjkqhNzSXoODX3\nThENLOk8zPzGCmJXX9g3NFJRzhAV9InIQNZfZ6f0r+zsDLktOjvlLylJMk7fekss+rg4+bMsvNOL\n+cifzEdNcRy0kk9J3HuT6G8mtekEKV2teILteDp9JAT8JDRU46k6SHxiLO68XNx+H+6WRtwJ8bib\nGnDm5mDS06CxCZKT4OKLMbs/JmgMXQlJdOKgq62drvKddB6qpH3yVHy+dtpc8fjSsmnzttPmSaIl\nNYem+FQa3Ul4+0nwORUcVoDpbceY13qAud6DpCycIycZS+PJleFBBV2JZLC0d/tkYDdmcLlCnXFa\nWmhze6iYVMyermQ+7kym0Yod7Vc0Yhgs8tqqmR6oZUb9PgqTg8TWVYf6gs6cKVcHWqlSGSaGPVNU\nGWP0F78e7r91OMQPXV4uiUnhwtTRQfzuHSyYksOCrqNY5Vuo23eUSkcqh1OncqRwIVWTZtIZVuFx\nLJMe8DKlqZIpwUbypqYxpeEQnoN7uhO26mHqXJiaJ2J+7JgstM6bp5a5MuqooE8kesev92ag9Pjc\n3J6+l5w4gTl8mEyPm8wj5Syq+hD2/ImAv536WQupnlZMNQlUJ+dS48mk0emhOS4VK4rEzlhBklob\nSelsIbW5muymY2Q5/WTFQxatxM8vhoPbZAE18xyI7V5stSso2o1GUlNDpRGi6PUpExcVdCXEQOnx\nTqcsSNqNG1pbxYK3y+9aFs66OrKajpGVUMDc88+RxUl3NaxbR+BEDc1Fs2mMTaHleAM+Two+46Y1\nK4/WnDx8Lg/txk1nfAIdXUE6Y+Pp6rLosBwEMBFhkJYxEAxigkHcVgB3sAtXuw+XAbe/lZgOP/Ex\nhnh/C56qQ8RbHcQ31ZPYcIIURwcpGYkktzXiLDtHIlyqqkSgc3MhGCfWttsNxcWwfr08npkp741l\nwTnnhN4jjSFXogwVdCXEydLjY2NFzDo6JPW/OyoGEIFPTRXhW7FC6pscOiRCvGgRzrffJq22irTO\ngyKMHR65EjjcFaqFUlAAQZecKFasknHLy0Nhl21tEvYYEyM9ObdskeqHycmhDNWiIimklZ7eE4bI\n7NmSWBXrk1jz2kY5zu2G886TeZeXh+Zhi7XHIx2GZsyQ11lSIiczr1csdY0hV6IQFXQlxEDp8bYl\nOnmy7NPRIdExNklJEjGTliZCGJ6V2dkpgupyifDNnSsunZoaWLtWTgZOp1j0LpcIe0WFHGPXkklN\nDXUGio0N1T1xOCRqJz1djj1+XMS3qSkUB5+YKCeBOXPkdt8+qcmyerWcgCwrlAwVLsxer4w1c2Zo\n+9SpGkOuRDUq6EqIk1VzLCuT26wsEc62tsiaLcGgCLDtfrAXYWtqZNzYWLG0wy3gvDyxqmNi5PjK\nSnGDHDkiwm9fMRw+LDHkdrar2x2qGbN9u1jc8+eL2HZ0yGOVlVLD5vDhUImC1lYR6fPPj1wnGGoG\n52BrEIoyyqigK5EMFg2zbJmIZn9VFZcs6SuAOTlw+eUimLW1IcHs6BBrPD9fhHrrVhHevDx5brsQ\nWHm5pOh3N+1g924RY7u8b0ODiH9zs4j1kiVy3PHjcsUwbZo81t4esu4zMk7tNSvKGEEFXenLySzR\nxES49FLxpx8/LtvsIlYDCWB/gunzidUNka3c7Of3+SQk0BgRb7u+SkeHbJszR2qzb9smt+E+8NhY\neY6GBnHXpHaXMrDdKP0tYqr1rYwDVNCVU8fhECv5VFLc+yv5ay/A2pUbbYJBsf7b2sQ1Y4yIuS34\n9fVyPz4+5LpJDas/43TKYqbTqYWwlAmFCroyOoQvwNpVISEU6x0TI4IcDIqQ21Z8fLwIcnu7WOHh\nkSk2Xq9Y4eefL7Hk6kZRJgj67VZGB3sB1rJCZXOPHpX7JSUi3K2tIu5paRIXblkSC9/QEOrxecMN\nYsFXV4vVb9c6LyuTSJfMTIlqOZlLSFHGCWqhK6NHuG995sxQDRmfT1wusbGyqGl3CiotFdFvb5f6\n6LZIZ2frgqaioIKujDa2bz0zU0Q9XJgvvBA+/DDSD56S0rcIli5oKgqggq5EE/0Js4YTKsqQUUFX\nohu1vhVlyKipoyiKMk5QQVcURRknqKAriqKME1TQFUVRxgkq6IqiKOMEFXRFUZRxgrEsa+SezJga\n4NAQds0Eaod5OmeLsTRX0PkON2NpvmNprjCx5zvNsqyswXYaUUEfKsaYTZZllY32PIbCWJor6HyH\nm7E037E0V9D5DgV1uSiKoowTVNAVRVHGCdEq6A+O9gROgbE0V9D5Djdjab5jaa6g8x2UqPShK4qi\nKKdOtFroiqIoyikSFYJujHnKGLO1+++gMWbrAPsdNMZs795v00jPs3sO/2KMqQqb7xUD7HeZMeZj\nY8xeY8y3R3qeYfO4zxiz2xizzRjzvDEmdYD9RvW9Hez9MsbEdn9P9hpjNhhjCkZ6jt3zyDfGrDXG\n7DTG7DDG3NPPPiuNMU1h35F/Ho25hs3npJ+tEe7vfm+3GWMWj8Y8u+cyO+x922qMaTbGfL3XPqP6\n/hpjHjXGVBtjysO2pRtjXjPGVHTfpg1w7K3d+1QYY24965OzLCuq/oD/D/jnAR47CGSO8vz+BfjW\nIPs4gX1AERADfASUjNJ8Pwm4uv//IfDDaHtvh/J+AV8GHuj+/wbgqVGaay6wuPv/JGBPP3NdCfxx\nNOZ3Op8t8P+3dz4hVlVxHP98oSKwsLFIJ6dFgqtWlYiGtRkxG8KpiLBNfyYIFy5cheEudy3aVYv+\nkIUU9NchlByLaKVFQ6OGgqObZhhHyLDCRQnfFue8uL7um3nhm3tvj98HhnfuPec9vvM95/7Ovb9z\n3swIcAgQsAE4Vrfmwrg4T9qD3Rh/gQeBe4GThXOvALtzeXfZdQasAM7l14FcHuiltkbcobeQJOBJ\n4IO6tVwj64Fp2+ds/wl8CIzWIcT2YdtX8uFRYKgOHYvQjV+jwL5c/hgYzuOlUmzP2Z7M5d+BU8Dq\nqnX0mFHgPSeOArdIGqxbFDAMnLXdzZcRK8P2t8DFttPF8bkPeLTkrQ8BE7Yv2v4VmAC29lJbowI6\n8AAwb/tMh3oDhyX9IOmFCnW1szM/mr7T4dFqNfBz4XiGZlz0Y6Q7sTLq9LYbv/5pkyeoS8Ctlajr\nQE773AMcK6neKGlK0iFJd1cq7N8s1rdNHa/b6Xxz1yR/AVbansvl88DKkjZL7nNl/7FI0hFgVUnV\nHtsHcvkpFr4732R7VtLtwISk03m2rEwr8Aawl3SR7CWliMZ6reG/0I23kvYAV4D9HT6mEm/7BUk3\nAZ8Au2z/1lY9SUoT/JHXWD4H1latscD/rm8l3QBsA14qqW6av1dh25Jq2T5YWUC3vXmheknXAY8D\n9y3wGbP59YKkz0iP6j0fmItpbSHpTeCLkqpZ4M7C8VA+tyR04e2zwCPAsHMyr+QzKvG2A9341Woz\nk8fKcuCXauRdjaTrScF8v+1P2+uLAd72QUmvS7rNdi1/h6SLvq10vHbJw8Ck7fn2iqb5m5mXNGh7\nLqerLpS0mSXl/1sMAd/0UkSTUi6bgdO2Z8oqJS2TdHOrTFrsO1nWdilpyy0+1kHD98BaSXflO43t\nwHgV+tqRtBV4Edhm+3KHNnV7241f40BrV8ATwNedJqelJOft3wZO2X61Q5tVrfy+pPWk66yuyaeb\nvh0Hns67XTYAlwrpg7ro+LTeJH8LFMfnM8CBkjZfAlskDeRU7ZZ8rnfUtVJcsgL8LrCj7dwdwMFc\nXkPa/TAF/ERKJ9Sh833gBHA8d+Jgu9Z8PELaAXG2Lq1ZxzQpb/dj/mntFGmUt2V+AS+TJiKAG4GP\n8u/zHbCmJj83kdJtxwuejgA7WuMX2Jl9nCItRN9fY/+X9m2bXgGvZe9PAOvq0pv1LCMF6OWFc43x\nlzTRzAF/kfLgz5PWc74CzgBHgBW57TrgrcJ7x/IYngae67W2+KZoEARBn9CklEsQBEFwDURAD4Ig\n6BMioAdBEPQJEdCDIAj6hAjoQRAEfUIE9CAIgj4hAnoQBEGfEAE9CIKgT/gbsZLUZjoBiQwAAAAA\nSUVORK5CYII=\n",
|
|
"text/plain": [
|
|
"<matplotlib.figure.Figure at 0x7f1a8bdbecf8>"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
}
|
|
],
|
|
"source": [
|
|
"# plot training loss\n",
|
|
"plt.figure(2)\n",
|
|
"plt.plot(losses[0], c='#FF9359', lw=3, label='Original')\n",
|
|
"plt.plot(losses[1], c='#74BCFF', lw=3, label='Batch Normalization')\n",
|
|
"plt.xlabel('step');plt.ylabel('test loss');plt.ylim((0, 2000));plt.legend(loc='best')\n",
|
|
"\n",
|
|
"# evaluation\n",
|
|
"# set net to eval mode to freeze the parameters in batch normalization layers\n",
|
|
"[net.eval() for net in nets] # set eval mode to fix moving_mean and moving_var\n",
|
|
"preds = [net(test_x)[0] for net in nets]\n",
|
|
"plt.figure(3)\n",
|
|
"plt.plot(test_x.data.numpy(), preds[0].data.numpy(), c='#FF9359', lw=4, label='Original')\n",
|
|
"plt.plot(test_x.data.numpy(), preds[1].data.numpy(), c='#74BCFF', lw=4, label='Batch Normalization')\n",
|
|
"plt.scatter(test_x.data.numpy(), test_y.data.numpy(), c='r', s=50, alpha=0.2, label='train')\n",
|
|
"plt.legend(loc='best')\n",
|
|
"plt.show()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {
|
|
"collapsed": true
|
|
},
|
|
"outputs": [],
|
|
"source": []
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "Python 3",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.5.2"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 2
|
|
}
|