Skip to content

Instantly share code, notes, and snippets.

@nzw0301

nzw0301/CIFAR10-warm-restart.ipynb

Created January 12, 2018 14:09
Show Gist options
  • Save nzw0301/66d88fc81687cab6856ad2f1a947383e to your computer and use it in GitHub Desktop.
Save nzw0301/66d88fc81687cab6856ad2f1a947383e to your computer and use it in GitHub Desktop.
Download ZIP
Display the source blob
Display the rendered blob
Raw
CIFAR10-warm-restart.ipynb
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"Using TensorFlow backend.\n",
"/home/nzw/.pyenv/versions/miniconda3-latest/lib/python3.6/importlib/_bootstrap.py:219: RuntimeWarning: compiletime version 3.5 of module 'tensorflow.python.framework.fast_tensor_util' does not match runtime version 3.6\n",
" return f(*args, **kwds)\n"
]
}
],
"source": [
"'''\n",
"Based on https://github.com/keras-team/keras/blob/master/examples/cifar10_cnn.py\n",
"- Change the batch size from 32 to 128\n",
"- Change the number of epochs\n",
"'''\n",
"\n",
"import keras\n",
"from keras.datasets import cifar10\n",
"from keras.models import Sequential\n",
"from keras.layers import Dense, Dropout, Activation, Flatten\n",
"from keras.layers import Conv2D, MaxPooling2D\n",
"from keras.preprocessing.image import ImageDataGenerator\n",
"\n",
"from keras_contrib.callbacks import LearningRateWarmRestarter\n",
"\n",
"from __future__ import print_function\n",
"import numpy as np\n",
"\n",
"import matplotlib.pyplot as plt\n",
"\n",
"%matplotlib inline\n",
"\n",
"np.random.seed(7)"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"batch_size = 128\n",
"num_classes = 10\n",
"epochs = 40\n",
"num_predictions = 20\n",
"\n",
"(x_train, y_train), (x_test, y_test) = cifar10.load_data()\n",
"x_train = x_train.astype('float32')\n",
"x_test = x_test.astype('float32')\n",
"\n",
"x_train /= 255\n",
"x_test /= 255\n",
"\n",
"y_train = keras.utils.to_categorical(y_train, num_classes)\n",
"y_test = keras.utils.to_categorical(y_test, num_classes)"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [],
"source": [
"def create_model():\n",
" model = Sequential()\n",
" model.add(Conv2D(32, (3, 3), padding='same',\n",
" input_shape=x_train.shape[1:]))\n",
" model.add(Activation('relu'))\n",
" model.add(Conv2D(32, (3, 3)))\n",
" model.add(Activation('relu'))\n",
" model.add(MaxPooling2D(pool_size=(2, 2)))\n",
" model.add(Dropout(0.25))\n",
"\n",
" model.add(Conv2D(64, (3, 3), padding='same'))\n",
" model.add(Activation('relu'))\n",
" model.add(Conv2D(64, (3, 3)))\n",
" model.add(Activation('relu'))\n",
" model.add(MaxPooling2D(pool_size=(2, 2)))\n",
" model.add(Dropout(0.25))\n",
"\n",
" model.add(Flatten())\n",
" model.add(Dense(512))\n",
" model.add(Activation('relu'))\n",
" model.add(Dropout(0.5))\n",
" model.add(Dense(num_classes))\n",
" model.add(Activation('softmax'))\n",
"\n",
" opt = keras.optimizers.SGD(lr=0.01, decay=0.0005, momentum=0.9, nesterov=False)\n",
"\n",
" model.compile(loss='categorical_crossentropy',\n",
" optimizer=opt,\n",
" metrics=['accuracy'])\n",
" return model\n"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Using real-time data augmentation.\n",
"Epoch 1/40\n",
"391/391 [==============================] - 17s 44ms/step - loss: 2.0405 - acc: 0.2455 - val_loss: 1.7301 - val_acc: 0.3815\n",
"Epoch 2/40\n",
"391/391 [==============================] - 15s 38ms/step - loss: 1.7017 - acc: 0.3739 - val_loss: 1.5146 - val_acc: 0.4629\n",
"Epoch 3/40\n",
"391/391 [==============================] - 16s 41ms/step - loss: 1.5202 - acc: 0.4433 - val_loss: 1.3368 - val_acc: 0.5188\n",
"Epoch 4/40\n",
"391/391 [==============================] - 16s 41ms/step - loss: 1.4117 - acc: 0.4860 - val_loss: 1.2133 - val_acc: 0.5712\n",
"Epoch 5/40\n",
"391/391 [==============================] - 17s 43ms/step - loss: 1.3376 - acc: 0.5197 - val_loss: 1.1535 - val_acc: 0.5869\n",
"Epoch 6/40\n",
"391/391 [==============================] - 17s 43ms/step - loss: 1.2804 - acc: 0.5407 - val_loss: 1.0958 - val_acc: 0.6137\n",
"Epoch 7/40\n",
"391/391 [==============================] - 17s 44ms/step - loss: 1.2197 - acc: 0.5635 - val_loss: 1.0672 - val_acc: 0.6194\n",
"Epoch 8/40\n",
"391/391 [==============================] - 17s 44ms/step - loss: 1.1747 - acc: 0.5790 - val_loss: 0.9996 - val_acc: 0.6489\n",
"Epoch 9/40\n",
"391/391 [==============================] - 16s 42ms/step - loss: 1.1441 - acc: 0.5901 - val_loss: 0.9804 - val_acc: 0.6565\n",
"Epoch 10/40\n",
"391/391 [==============================] - 17s 43ms/step - loss: 1.1099 - acc: 0.6041 - val_loss: 0.9411 - val_acc: 0.6686\n",
"Epoch 11/40\n",
"391/391 [==============================] - 17s 43ms/step - loss: 1.0854 - acc: 0.6114 - val_loss: 0.9278 - val_acc: 0.6741\n",
"Epoch 12/40\n",
"391/391 [==============================] - 17s 45ms/step - loss: 1.0600 - acc: 0.6231 - val_loss: 0.9047 - val_acc: 0.6796\n",
"Epoch 13/40\n",
"391/391 [==============================] - 17s 44ms/step - loss: 1.0364 - acc: 0.6294 - val_loss: 0.8925 - val_acc: 0.6886\n",
"Epoch 14/40\n",
"391/391 [==============================] - 17s 44ms/step - loss: 1.0204 - acc: 0.6363 - val_loss: 0.8963 - val_acc: 0.6900\n",
"Epoch 15/40\n",
"391/391 [==============================] - 17s 43ms/step - loss: 1.0030 - acc: 0.6426 - val_loss: 0.8438 - val_acc: 0.7084\n",
"Epoch 16/40\n",
"391/391 [==============================] - 17s 42ms/step - loss: 0.9880 - acc: 0.6496 - val_loss: 0.8477 - val_acc: 0.7037\n",
"Epoch 17/40\n",
"391/391 [==============================] - 17s 44ms/step - loss: 0.9793 - acc: 0.6519 - val_loss: 0.8369 - val_acc: 0.7091\n",
"Epoch 18/40\n",
"391/391 [==============================] - 17s 44ms/step - loss: 0.9573 - acc: 0.6610 - val_loss: 0.8247 - val_acc: 0.7125\n",
"Epoch 19/40\n",
"391/391 [==============================] - 18s 46ms/step - loss: 0.9453 - acc: 0.6660 - val_loss: 0.8056 - val_acc: 0.7170\n",
"Epoch 20/40\n",
"391/391 [==============================] - 17s 43ms/step - loss: 0.9336 - acc: 0.6689 - val_loss: 0.7921 - val_acc: 0.7253\n",
"Epoch 21/40\n",
"391/391 [==============================] - 15s 39ms/step - loss: 0.9246 - acc: 0.6719 - val_loss: 0.7914 - val_acc: 0.7241\n",
"Epoch 22/40\n",
"391/391 [==============================] - 16s 41ms/step - loss: 0.9069 - acc: 0.6779 - val_loss: 0.7748 - val_acc: 0.7306\n",
"Epoch 23/40\n",
"391/391 [==============================] - 17s 44ms/step - loss: 0.9056 - acc: 0.6792 - val_loss: 0.7701 - val_acc: 0.7323\n",
"Epoch 24/40\n",
"391/391 [==============================] - 15s 40ms/step - loss: 0.8981 - acc: 0.6820 - val_loss: 0.7651 - val_acc: 0.7309\n",
"Epoch 25/40\n",
"391/391 [==============================] - 15s 39ms/step - loss: 0.8875 - acc: 0.6860 - val_loss: 0.7442 - val_acc: 0.7437\n",
"Epoch 26/40\n",
"391/391 [==============================] - 17s 44ms/step - loss: 0.8805 - acc: 0.6878 - val_loss: 0.7534 - val_acc: 0.7386\n",
"Epoch 27/40\n",
"391/391 [==============================] - 17s 43ms/step - loss: 0.8670 - acc: 0.6940 - val_loss: 0.7412 - val_acc: 0.7421\n",
"Epoch 28/40\n",
"391/391 [==============================] - 17s 44ms/step - loss: 0.8653 - acc: 0.6935 - val_loss: 0.7319 - val_acc: 0.7437\n",
"Epoch 29/40\n",
"391/391 [==============================] - 17s 44ms/step - loss: 0.8601 - acc: 0.6952 - val_loss: 0.7489 - val_acc: 0.7422\n",
"Epoch 30/40\n",
"391/391 [==============================] - 16s 41ms/step - loss: 0.8504 - acc: 0.6987 - val_loss: 0.7161 - val_acc: 0.7522\n",
"Epoch 31/40\n",
"391/391 [==============================] - 18s 47ms/step - loss: 0.8437 - acc: 0.7019 - val_loss: 0.7148 - val_acc: 0.7513\n",
"Epoch 32/40\n",
"391/391 [==============================] - 15s 39ms/step - loss: 0.8342 - acc: 0.7078 - val_loss: 0.6997 - val_acc: 0.7557\n",
"Epoch 33/40\n",
"391/391 [==============================] - 15s 39ms/step - loss: 0.8354 - acc: 0.7041 - val_loss: 0.7134 - val_acc: 0.7559\n",
"Epoch 34/40\n",
"391/391 [==============================] - 18s 45ms/step - loss: 0.8331 - acc: 0.7049 - val_loss: 0.7083 - val_acc: 0.7539\n",
"Epoch 35/40\n",
"391/391 [==============================] - 18s 45ms/step - loss: 0.8228 - acc: 0.7122 - val_loss: 0.6853 - val_acc: 0.7621\n",
"Epoch 36/40\n",
"391/391 [==============================] - 15s 39ms/step - loss: 0.8185 - acc: 0.7098 - val_loss: 0.6955 - val_acc: 0.7594\n",
"Epoch 37/40\n",
"391/391 [==============================] - 17s 43ms/step - loss: 0.8125 - acc: 0.7137 - val_loss: 0.6954 - val_acc: 0.7588\n",
"Epoch 38/40\n",
"391/391 [==============================] - 16s 41ms/step - loss: 0.8126 - acc: 0.7132 - val_loss: 0.6907 - val_acc: 0.7602\n",
"Epoch 39/40\n",
"391/391 [==============================] - 17s 43ms/step - loss: 0.8019 - acc: 0.7171 - val_loss: 0.6874 - val_acc: 0.7623\n",
"Epoch 40/40\n",
"391/391 [==============================] - 15s 38ms/step - loss: 0.7972 - acc: 0.7200 - val_loss: 0.6755 - val_acc: 0.7669\n"
]
}
],
"source": [
"print('Using real-time data augmentation.')\n",
"# This will do preprocessing and realtime data augmentation:\n",
"datagen = ImageDataGenerator(\n",
" featurewise_center=False, # set input mean to 0 over the dataset\n",
" samplewise_center=False, # set each sample mean to 0\n",
" featurewise_std_normalization=False, # divide inputs by std of the dataset\n",
" samplewise_std_normalization=False, # divide each input by its std\n",
" zca_whitening=False, # apply ZCA whitening\n",
" rotation_range=0, # randomly rotate images in the range (degrees, 0 to 180)\n",
" width_shift_range=0.1, # randomly shift images horizontally (fraction of total width)\n",
" height_shift_range=0.1, # randomly shift images vertically (fraction of total height)\n",
" horizontal_flip=True, # randomly flip images\n",
" vertical_flip=False) # randomly flip images\n",
"\n",
"# (std, mean, and principal components if ZCA whitening is applied).\n",
"datagen.fit(x_train)\n",
"\n",
"# Fit the model on the batches generated by datagen.flow().\n",
"model = create_model()\n",
"hist = model.fit_generator(datagen.flow(x_train, y_train,\n",
" batch_size=batch_size),\n",
" epochs=epochs,\n",
" validation_data=(x_test, y_test),\n",
" workers=4)"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"10000/10000 [==============================] - 1s 106us/step\n",
"Test loss: 0.6755105630874634\n",
"Test accuracy: 0.7669\n"
]
}
],
"source": [
"scores = model.evaluate(x_test, y_test, verbose=1)\n",
"print('Test loss:', scores[0])\n",
"print('Test accuracy:', scores[1])"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Epoch 1/40\n",
"391/391 [==============================] - 16s 41ms/step - loss: 2.0546 - acc: 0.2359 - val_loss: 1.6990 - val_acc: 0.3935\n",
"Epoch 2/40\n",
"391/391 [==============================] - 14s 36ms/step - loss: 1.6966 - acc: 0.3731 - val_loss: 1.4856 - val_acc: 0.4540\n",
"Epoch 3/40\n",
"391/391 [==============================] - 17s 45ms/step - loss: 1.5238 - acc: 0.4426 - val_loss: 1.3598 - val_acc: 0.5095\n",
"Epoch 4/40\n",
"391/391 [==============================] - 15s 38ms/step - loss: 1.4380 - acc: 0.4778 - val_loss: 1.2942 - val_acc: 0.5294\n",
"Epoch 5/40\n",
"391/391 [==============================] - 17s 42ms/step - loss: 1.3956 - acc: 0.4925 - val_loss: 1.2574 - val_acc: 0.5511\n",
"Epoch 6/40\n",
"391/391 [==============================] - 17s 43ms/step - loss: 1.4110 - acc: 0.4916 - val_loss: 1.2505 - val_acc: 0.5566\n",
"Epoch 7/40\n",
"391/391 [==============================] - 16s 42ms/step - loss: 1.3411 - acc: 0.5178 - val_loss: 1.1696 - val_acc: 0.5819\n",
"Epoch 8/40\n",
"391/391 [==============================] - 16s 42ms/step - loss: 1.2848 - acc: 0.5383 - val_loss: 1.1356 - val_acc: 0.5931\n",
"Epoch 9/40\n",
"391/391 [==============================] - 17s 43ms/step - loss: 1.2403 - acc: 0.5560 - val_loss: 1.0811 - val_acc: 0.6153\n",
"Epoch 10/40\n",
"391/391 [==============================] - 15s 40ms/step - loss: 1.2098 - acc: 0.5679 - val_loss: 1.0347 - val_acc: 0.6337\n",
"Epoch 11/40\n",
"391/391 [==============================] - 16s 40ms/step - loss: 1.1754 - acc: 0.5787 - val_loss: 1.0254 - val_acc: 0.6399\n",
"Epoch 12/40\n",
"391/391 [==============================] - 17s 43ms/step - loss: 1.1499 - acc: 0.5891 - val_loss: 1.0023 - val_acc: 0.6490\n",
"Epoch 13/40\n",
"391/391 [==============================] - 15s 39ms/step - loss: 1.1362 - acc: 0.5948 - val_loss: 1.0023 - val_acc: 0.6455\n",
"Epoch 14/40\n",
"391/391 [==============================] - 14s 36ms/step - loss: 1.1291 - acc: 0.5956 - val_loss: 0.9897 - val_acc: 0.6541\n",
"Epoch 15/40\n",
"391/391 [==============================] - 15s 40ms/step - loss: 1.1190 - acc: 0.5983 - val_loss: 0.9908 - val_acc: 0.6504\n",
"Epoch 16/40\n",
"391/391 [==============================] - 14s 37ms/step - loss: 1.1409 - acc: 0.5921 - val_loss: 1.0039 - val_acc: 0.6444\n",
"Epoch 17/40\n",
"391/391 [==============================] - 14s 37ms/step - loss: 1.1228 - acc: 0.5998 - val_loss: 0.9734 - val_acc: 0.6553\n",
"Epoch 18/40\n",
"391/391 [==============================] - 15s 39ms/step - loss: 1.1000 - acc: 0.6074 - val_loss: 0.9597 - val_acc: 0.6650\n",
"Epoch 19/40\n",
"391/391 [==============================] - 15s 38ms/step - loss: 1.0823 - acc: 0.6130 - val_loss: 0.9445 - val_acc: 0.6648\n",
"Epoch 20/40\n",
"391/391 [==============================] - 15s 38ms/step - loss: 1.0632 - acc: 0.6207 - val_loss: 0.9186 - val_acc: 0.6777\n",
"Epoch 21/40\n",
"391/391 [==============================] - 15s 37ms/step - loss: 1.0480 - acc: 0.6282 - val_loss: 0.9000 - val_acc: 0.6823\n",
"Epoch 22/40\n",
"391/391 [==============================] - 14s 36ms/step - loss: 1.0389 - acc: 0.6271 - val_loss: 0.8920 - val_acc: 0.6892\n",
"Epoch 23/40\n",
"391/391 [==============================] - 16s 42ms/step - loss: 1.0257 - acc: 0.6322 - val_loss: 0.8795 - val_acc: 0.6892\n",
"Epoch 24/40\n",
"391/391 [==============================] - 14s 37ms/step - loss: 1.0127 - acc: 0.6407 - val_loss: 0.9037 - val_acc: 0.6793\n",
"Epoch 25/40\n",
"391/391 [==============================] - 16s 40ms/step - loss: 1.0083 - acc: 0.6420 - val_loss: 0.8663 - val_acc: 0.6966\n",
"Epoch 26/40\n",
"391/391 [==============================] - 15s 39ms/step - loss: 0.9929 - acc: 0.6479 - val_loss: 0.8512 - val_acc: 0.7001\n",
"Epoch 27/40\n",
"391/391 [==============================] - 15s 38ms/step - loss: 0.9877 - acc: 0.6462 - val_loss: 0.8488 - val_acc: 0.7038\n",
"Epoch 28/40\n",
"391/391 [==============================] - 15s 40ms/step - loss: 0.9863 - acc: 0.6489 - val_loss: 0.8486 - val_acc: 0.6991\n",
"Epoch 29/40\n",
"391/391 [==============================] - 18s 46ms/step - loss: 0.9726 - acc: 0.6559 - val_loss: 0.8419 - val_acc: 0.7045\n",
"Epoch 30/40\n",
"391/391 [==============================] - 15s 40ms/step - loss: 0.9688 - acc: 0.6554 - val_loss: 0.8403 - val_acc: 0.7047\n",
"Epoch 31/40\n",
"391/391 [==============================] - 15s 39ms/step - loss: 0.9637 - acc: 0.6561 - val_loss: 0.8335 - val_acc: 0.7073\n",
"Epoch 32/40\n",
"391/391 [==============================] - 16s 42ms/step - loss: 0.9617 - acc: 0.6581 - val_loss: 0.8363 - val_acc: 0.7063\n",
"Epoch 33/40\n",
"391/391 [==============================] - 17s 45ms/step - loss: 0.9620 - acc: 0.6583 - val_loss: 0.8335 - val_acc: 0.7068\n",
"Epoch 34/40\n",
"391/391 [==============================] - 17s 44ms/step - loss: 0.9626 - acc: 0.6549 - val_loss: 0.8364 - val_acc: 0.7058\n",
"Epoch 35/40\n",
"391/391 [==============================] - 15s 39ms/step - loss: 0.9555 - acc: 0.6584 - val_loss: 0.8344 - val_acc: 0.7062\n",
"Epoch 36/40\n",
"391/391 [==============================] - 16s 41ms/step - loss: 0.9761 - acc: 0.6520 - val_loss: 0.8453 - val_acc: 0.7019\n",
"Epoch 37/40\n",
"391/391 [==============================] - 16s 41ms/step - loss: 0.9723 - acc: 0.6519 - val_loss: 0.8287 - val_acc: 0.7108\n",
"Epoch 38/40\n",
"391/391 [==============================] - 16s 41ms/step - loss: 0.9632 - acc: 0.6586 - val_loss: 0.8401 - val_acc: 0.7047\n",
"Epoch 39/40\n",
"391/391 [==============================] - 17s 42ms/step - loss: 0.9597 - acc: 0.6581 - val_loss: 0.8228 - val_acc: 0.7120\n",
"Epoch 40/40\n",
"391/391 [==============================] - 15s 40ms/step - loss: 0.9484 - acc: 0.6631 - val_loss: 0.8000 - val_acc: 0.7215\n",
"10000/10000 [==============================] - 1s 94us/step\n",
"Test loss: 0.8000454781532288\n",
"Test accuracy: 0.7215\n"
]
}
],
"source": [
"model = create_model()\n",
"warm_restart_hist = model.fit_generator(datagen.flow(x_train, y_train,\n",
" batch_size=batch_size),\n",
" epochs=epochs,\n",
" validation_data=(x_test, y_test),\n",
" callbacks=[LearningRateWarmRestarter(max_lr=0.01, num_restart_epochs=5, factor=2)], \n",
" workers=4)\n",
"\n",
"\n",
"scores = model.evaluate(x_test, y_test, verbose=1)\n",
"print('Test loss:', scores[0])\n",
"print('Test accuracy:', scores[1])"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"<matplotlib.legend.Legend at 0x7f6cfefa3860>"
]
},
"execution_count": 12,
"metadata": {},
"output_type": "execute_result"
},
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAEICAYAAACktLTqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvAOZPmwAAIABJREFUeJzt3Xd8VfX9x/HXJzd7QxIgC8ImBMIKgiLDAaIV3IobZ9XWttr2V+2wape1tXW0Dty1VnHWXRd7KQmywt6EkYSQhOzkJt/fH98LBkwgJDc5yc3n+eh95I5zv+dzD/Z9z/2e7/keMcaglFLKt/g5XYBSSinv03BXSikfpOGulFI+SMNdKaV8kIa7Ukr5IA13pZTyQRruStUjIv1ExNR7/JmIXN2UZZuxrt+IyNPNff9x2r1ZROZ5u13VsWi4K0TkKhHJFJFSEdknIp+IyOme1+4XkX/XW9aISJln2VIRKTqmrX6eZR4/5nn/Y96bIyJ/ERG/estcKSJLRaRcRL5ooM6RIrLC8/pyEUn3/tY4mjFmijHm1Za2IyJni8iOY9r+nTHmtpa2rVRDNNw7ORG5G3gU+CPQHegJPAlccJy3DTPGhHtu0ce8dj1wELhSRAIaeG+aMSYcOBO41rP8YQXA34C/NFBnEPAe8CLQBXgN+G8j61Cq09Nw78REJAp4EPiBMeYdY0yZMabGGPOBMebnzWhPsIF9LyDA9xpb1hizCVgCDK/33GfGmDeBfQ285Sy7iHnCGFMF/B0IAiY2UMc1IrLsmOd+LiLveO5PF5GVIlIiIrtE5DfH+UyLRGSm575LRP4uIgUishWYesyyN4vIek+7W0XkZs/zUcAHQM96v3i6icjvReSleu+/UESyRaRIROaIyMB6r+WIyN0iskZEikXkNc8X3gmJyOmeX2bFIvK1iIyp99pNIrLDU/M2EZnheX6AiCzwvOeAiPynKetS7YeGe+d2KhAMvOul9iZh9/5nA28C1zW2oIikAuOALU1sOw1YdfiBsfNmrPE8f6z/AkNEpE+9564CDgdUKXANEAVMA34sIuc3oYbbgSnAMOAU4PJjXs/FfqFFArcAT4hIujGm2LOeXfV+8eTVf6Nne/wbuBOIA74APjjml8nlwGSgDzAK+0V6XCISC3wEPALEAI8DH4tIFxGJxP5SmmyMicD+e6z2vPUPnvd1AZKAf55oXap90XDv3GKAA8YY90m+b4Vn77LomL7164GPPGH2H+B7IhJzzHtXi0gZsA74HHimiesMB4qPea4YiDh2QWNMKfAhcHgvdBA2ED/0vD7HGLPWGFNnjFkFvE4DvwAacDnwd2NMjjGmAHjomPV+YIzZZqw5wJfA+CZ+vhnA+57aajxtRwJj6i3zqDFmv2fdH1LvV89xTAOyjTGvGWPcxph/A9v49leVwX4RBhtj9hlj1nmerwFSgHhjTKUxZnETP4dqJzTcO7cCIFZE/E/yfSONMdGe248ARCQMuAQ4fPBxEbZ75cpj3puODeSrsL8cQpu4zlJs2NUXCZQ0svx/6q37auAdY0ylp9ZTRWSeiOSLSDFwMxDbhBoSgN31Hu+s/6KInC8iX4nIQc+B5ilNbPdw20faM8bUATlAYr1l9te7X479wjupduvVnWiMOYTdRj8A9ovIhyIywLPMT4EAINPTFXQ9qkPRcO/clgKVwIVeaOsSbNjMEpH92GDvQQNdM5495teATODXTWw/G9sdAhzp3x/qeb4h/wMSRWQoNsDq9xm/DrwNJBtjooDnsMcITmQfkFzvcc969YQAbwF/Arp7DjR/Vq/dEw2Z3Av0qteeH7Y7ZE8T6mpyux49D7drjPnEGHM2EI/tInvG8/w+Y8zNxph4bPjPEpHeLaxFtSEN907M031yH/BPz8G8UBEJEJFzReThk2zueuBZbOAO99wmABme/uSG/Am4TUTi4MgBy2DAH/ATkeB6vyrmAC4R+YHnQOKPsV0H8xv5bNXYAP8b9ktnTr2XI4CDxphKERmLp/umCd4AfiIiiZ7upl/Uey0ICATygVpPH/5Z9V7Pxf5K+k43Ur22p4vIJE8/+8+xv0q+amJtjfkQSBORK8QOR70K6Iftd48XkWkiEgpUA2VALYCIXC4ih381FGG/nGpbWItqQxrunZwx5m/A3dg96Hxst8MPsQclm0REemIPph7uEz58+xp7YLDBn/TGmJXYXw8/8zx1A1ABPAGc4bn/tGfZSuzwzJuxYXMNcIGnf7ox/wHOBmYbY+oH0+3An0SkBPglNlib4ilsP/oaYDl2T/3wZykC7sIenD4IXIqnj9/z+lrsl80Oz7GKbsdsi2zsdnoK++8wFZh+gs93QsaYfGA69ouowFPj+caYg4AL+yWyz/Paadh/e7B9/cs9x0fewY6o2tWSWlTbEr1Yh1JK+R7dc1dKKR+k4a6UUj5Iw10ppXyQhrtSSvmgkz15xWtiY2NNSkqKU6tXSqkOKSsr64AxJu5EyzkW7ikpKWRmZjq1eqWU6pBE5Ngzjhuk3TJKKeWDNNyVUsoHabgrpZQPcqzPXSnVvtTU1JCTk0NlZaXTpSggODiYpKQkAgKad7ExDXelFAA5OTlERESQkpKCnXRTOcUYQ0FBATk5OfTu3bzJOLVbRikFQGVlJTExMRrs7YCIEBMT06JfURruSqkjNNjbj5b+W3S8cM9dB5/9BqpKna5EKaXarY4X7kU7YcnjkLvW6UqUUgqAefPmsWTJkpN+38qVK/n4449boaKOGO7xniut7Vt9/OWUUp1SbW3LLxh1sm00J9zdbreG+1Ei4iE0FvatcroSpZQXPfzwwzz++OMA3HXXXZx55pkAfPnll1xzzTUA3H777WRkZJCWlsZvf/vbI+9NSUnhwQcf5PTTT+fNN99k0qRJ3HXXXUyYMIHU1FSWL1/OxRdfTP/+/fn1rxu+bG94eDj33XcfY8aMYenSpWRlZTFx4kRGjRrFOeecw759+wB4/PHHGTx4MOnp6cyYMYMdO3bw9NNP8/e//53hw4ezcOFCPvjgA8aMGcOIESM4++yzyc3NBeD+++/n1ltvZcqUKVx33XXcd999zJ49m+HDhzN79myvbs+ONxRSxO6979dwV6q1PPBBNuv2HvJqm4MTIvnttLRGX58wYQKPPPIIP/rRj8jMzKSqqoqamhoWLVrE+PHjAfjDH/5A165dqa2t5ayzzmL16tWkp6cDdlz4okWLAHj66acJDAxkwYIFPPbYY1xwwQVkZWXRtWtX+vbty1133UVMTMxR6y8rK2PIkCE8+OCD1NTUMHHiRN577z3i4uKYPXs2v/rVr3jhhRd46KGH2L59O0FBQRQVFREdHc1tt91GeHg4P/uZvWJkYWEhy5YtQ0R47rnnePjhh3nkkUcAyMrKYtGiRYSEhPDSSy+RmZnJP/7xD69ua+iI4Q4Qnw5LngB3FfgHOV2NUsoLRo0aRVZWFiUlJQQFBTFy5EgyMzNZuHDhkT36N954g1mzZuF2u9m3bx/r1q07Eu5XXHHFUe1Nnz4dgKFDh5KWlkZ8fDwAffr0Yffu3d8Jd5fLxSWXXALAxo0bWbt2LZMnTwZsN83h96enp3P11Vdz4YUXcuGFFzb4WXJycrjiiivYt28f1dXVR41Vnz59OiEhIS3aVk3R4cL9q20FfL0hjDvr3JC3HhKGO12SUj7neHvYrSUgIICUlBRefPFFTjvtNNLT05k7dy5bt24lNTWV7du389e//pXly5fTpUsXZs6cedQ48LCwsKPaCwqyO35+fn5H7h9+7Ha7v7P+4OBgXC4XYE8iSktLY+nSpd9Z7qOPPmLBggW8//77/O53vyM7O/s7y9x5553cfffdTJ8+nXnz5nH//fc3Wmdr6XB97u46w1t7utoH2u+ulE+ZMGECf/3rX5kwYQLjx4/n6aefZvjw4YgIhw4dIiwsjKioKHJzc/nkk09arY6BAweSn59/JNxramrIzs6mrq6O3bt3c8YZZ/Dwww9TVFREaWkpERERlJSUHHl/cXExiYmJALz88suNrufY93lThwv3tIRIdpluVLnCYb+OmFHKl4wfP559+/Zx6qmn0r17d4KDg4/0tw8bNowRI0aQlpbGjTfeyLhx41qtjsDAQN566y1+8YtfMGzYMIYPH86SJUuora3lmmuuYejQoYwYMYK77rqL6Ohopk2bxrvvvnvkgOr999/PZZddxvjx44mNjW10PWeccQbr1q1rlQOqYozxaoNNlZGRYZp7sY7T/zyHF/kt/WOC4OYvvFyZUp3T+vXrSU1NdboMVU9D/yYikmWMyTjRezvcnjvAkIQoVrp7wf61UNfyMa1KKeVrOma4J0aytCwR3BVwYLPT5SilVLvTIcM9LTGKbJNiH2i/u1JKfUfHDPeESLaYRNx+QTpiRimlGtAhw71bRDAxEaHsDeyt4a6UUg3okOEOMCQxitV1KbZbxqERP0op1V513HBP8BxUrSy20wArpVQLTZo0ieYO0W5vOmy4pyVGsaYuxT7QrhmlOr2GphTozDpsuA9JjGKjSaZOXDq3u1I+YseOHaSmpnLLLbeQlpbGlClTqKioYOXKlYwdO5b09HQuuugiCgsLAbun/ctf/pKJEyfy2GOPMXPmTG6//XbOOOMM+vTpw/z587nxxhtJTU1l5syZR9bT2NTBvqTDTRx2WEJUMKGhYeQG9iJe99yV8q5P7oH9a7zbZo+hcO5DJ1xs8+bNvPbaazz77LNcfvnlvP322zz88MM88cQTTJw4kfvuu48HHniARx99FICioiLmz58PwMyZMyksLGTOnDm8//77TJs2jcWLF/Pcc88xevRoVq5cyfDhw487dbCv6LB77iLCkMQo1pkUHeuulA/p3bs3w4fb2V5HjRrF1q1bKSoqYuLEiQBcf/31LFiw4Mjyx071O23aNESEoUOH0r17d4YOHYqfnx9paWns2LEDsFMHjxw5khEjRpCdnc26deva5sO1oQ675w6QlhDF0h1JnOWaAyX7IaKH0yUp5RuasIfdWupPz+tyuSgqKjru8ic71e+Jpg72FSfccxeRF0QkT0QavSK1iEwSkZUiki0i871bYuOGJEay2t3LPtB+d6V8UlRUFF26dGHhwoUAvPLKK0f24pujLacOdlJT9txfAv4B/KuhF0UkGngSmGqM2SUi3bxX3vENSYhinfGE+/5VMGBKW61aKdWGXn75ZW677TbKy8vp06cPL774YrPbqj91cJ8+fVp16mAnNWnKXxFJAT40xgxp4LU7gARjTMNXnW1ES6b8PayuzpD+wGfMD7qbmL4j4Ip/t6g9pToznfK3/XF6yt8BQBcRmSciWSJynRfabBI/P2FwQiTrpbd2yyilVD3eCHd/YBTwPeAc4DciMqChBUXkVhHJFJHM/Px8L6zads18VZFkz1KtKPRKm0op1dF5I9xzgP8ZY8qMMQeABcCwhhY0xswyxmQYYzLi4uK8sGp7UHWVu6d94O1xuUp1Mk5dmU19V0v/LbwR7u8B40XEX0RCgTHAei+02yRDEqPI1mkIlGqx4OBgCgoKNODbAWMMBQUFBAcHN7uNE46WEZHXgElArIjkAL8FAjwFPG2MWS8i/wNWA3XAc8aYRodNeluf2DDKArpwKCCOSO13V6rZkpKSyMnJwVtdpqplgoODSUpKavb7Txjuxpgrm7DMX4C/NLuKFvB3+ZEaH8mm4j5k6J67Us0WEBBA7969nS5DeUmHnX6gviEJUSyvTMYUbIbqMqfLUUopx/lGuCdGsqKmJ2LqIDfb6XKUUspxPhHuaQl6UFUpperziXAf0D2CfFcs5f5RGu5KKYWPhHugvx8De0SyzdVHp/9VSil8JNwB0uKjWF7VE5O7DtzVTpejlFKO8plwH5IYyYrqZKSuBvI3OF2OUko5ymfCPS0xirXGM0ZX+92VUp2cz4R7ao9IdtGdar9Q7XdXSnV6PhPuIYEu+naLZGeATv+rlFI+E+5gz1RdUdPL7rnXup0uRymlHONT4Z6WGMXiyj5QUw65bTZ3mVJKtTs+Fe5DEiJZXjfQPti1zNlilFLKQT4V7oMTItlHDIeCesCupU6Xo5RSjvGpcI8IDqB3bBgbAtLsnrtedEAp1Un5VLgDpCVEMr+yH5Tuh8IdTpejlFKO8LlwH53SlS/KPCczab+7UqqT8rlwPyetB5tMElWucO13V0p1Wj4X7j2ighnVK4ZVMgh2f+V0OUop5QifC3eA84bGM6+ir51ArPyg0+UopVSb88lwP3doj2/Hu+veu1KqE/LJcI+PCsEvaSQ1+Gu/u1KqU/LJcAeYnN6LVXV9qNy62OlSlFKqzflsuJ87NJ7MuoEE5K6Emgqny1FKqTbls+GeGB1CQdcRuIwb9n7jdDlKKdWmfDbcAZLTJwFQtGGBs4UopVQb8+lwP3NUKpvrEineuNDpUpRSqk35dLgndQllW+hQYgq/gbo6p8tRSqk249PhDhDQ+zTCTRn7t2i/u1Kq8zhhuIvICyKSJyLHvbSRiIwWkVoRudR75bXcoFOmALA583OHK1FKqbbTlD33l4Cpx1tARFzAn4FPvVCTVyWkDKLAryt1O/VkJqVU53HCcDfGLABONEHLncDbQJ43ivIqEYpiRtG3ci27D5Y7XY1SSrWJFve5i0gicBHwdBOWvVVEMkUkMz8/v6WrbrIuqRNIkgMszNR+d6VU5+CNA6qPAr8wxtSeaEFjzCxjTIYxJiMuLs4Lq26aroMmALBvzbw2W6dSSjnJ3wttZACviwhALHCeiLiNMf/1Qtve0X0I1a5Q4gq/YU9RBYnRIU5XpJRSrarFe+7GmN7GmBRjTArwFnBHuwp2AJc/tQkZjPbbyCdr9jldjVJKtbqmDIV8DVgKDBSRHBG5SURuE5HbWr887wnpezoD/XYzb9UWp0tRSqlWd8JuGWPMlU1tzBgzs0XVtKaeY/HD4Nqbyd6i8SRo14xSyof5/BmqRyRlYMRFht9GPlm73+lqlFKqVXWecA8MQ+KHMTF4C28s3427VueaUUr5rs4T7gA9TyWtbjPbcgt5ftF2p6tRSqlW08nCfSyuuipu7FPM37/YxK4CPWNVKeWbOl24A/ygbz7+fn786r9rMMY4XJRSSnlf5wr38G7QtS+ReZn839SBLNx8gP+u3ON0VUop5XWdK9wB+kyCbfO4ekQsI3pG87sP13OwrNrpqpRSyqs6X7inXQQ15bi2fMZDF6dzqKKGP3y03umqlFLKqzpfuPc6DcK7Q/a7DOwRwW0T+/L2ihwWbT7gdGVKKeU1nS/c/Vww+ALY/BlUlfDDM/vROzaMX767horqE05sqZRSHULnC3ewXTPuStj0KcEBLv540VB2HSznsS83O12ZUkp5RecM9+SxEBEP2e8CcGrfGC7PSOLZhdtYt/eQw8UppVTLdc5w9/ODwRfC5s+h0ob5L89LpUtoAPe+s5raOh37rpTq2DpnuIPtmqmtgo2fABAdGsh909JYlVPMY19s0pOblFIdWucN96TREJl0pGsGYFp6PBePTOTxOVv47fvZugevlOqwvHGZvY7Jzw/SLoSvnoGKIgiJRkT466XDiAsP4pkF29hXXMnjM0YQEuhyulqllDopnXfPHWzXTF0NbPz4yFN+fsK956Vy/7TBfLE+l6ueW0ZBaZWDRSql1Mnr3OGeOAqieh7VNXPYzHG9eerqUazbe4hLnlrCzoIyBwpUSqnm6dzhLmK7ZrbOgfKD33l56pAe/OeWsRRX1HDxk0tYubvIgSKVUurkde5wB0/XjBs2fNTgy6N6deHt208jNMjFjFlL+WJdbhsXqJRSJ0+cGvKXkZFhMjMzHVn3UYyBx4dD175w7TuNLpZfUsVNLy9n7Z5ihiZFM6BbOP27h9O/ewQDukeQEBWMiLRh4UqpzkhEsowxGSdarvOOljlMxO69L34cygogLKbBxeIignj91rE8MWcLq3YXMXdjPm9m5Rx5PSzQRb/uEYzsGc3/nTNIR9gopRyl4Q423Bf9HTZ8AKNmNrpYaKA/v5g66MjjwrJqNueVsim3hC15pWzcX8KLi3cQFxHEHZP6tUHhSinVMA13gB7ptlsm+93jhvuxuoQFckrvrpzSu+uR5258aTnPzN/GNWN7ERkc0ArFKqXUiekBVfi2a2b7Aihr2bzud08eQHFFDS8s2u6l4pRS6uRpuB+WdhGYOlj3XouaGZIYxdS0Hjy/cDuFevk+pZRDNNwP654GsQMaPKHpZN01eQCl1W5mLdzmhcKUUurkabgfdrhrZudiKGnZWPaBPSKYPiyBlxbv4IBOXaCUcsAJw11EXhCRPBFZ28jrV4vIas9tiYgM836ZbeRw14wX9t5/fFZ/qty1PDVvqxcKU0qpk9OUPfeXgKnHeX07MNEYkw78Dpjlhbqc0S0Vkk6B+X+Gkv0taqpPXDiXjEzilWU72V9c6aUClVKqaU4Y7saYBcB3J1759vUlxphCz8NlQJKXanPGBf+EmnJ474f27NUW+NFZ/THG8M+5W7xUnFJKNY23+9xvAj7xcpttK24ATP4dbPkcMl9oUVPJXUO5PCOZ15fvYvfBci8VqJRSJ+a1cBeRM7Dh/ovjLHOriGSKSGZ+fr63Vu19o2+GvmfCZ7+GAy3b6/7hmf0QEZ6Ys9lLxSml1Il5JdxFJB14DrjAGFPQ2HLGmFnGmAxjTEZcXJw3Vt06/Pxs94wrEN69FWrdzW4qPiqEa8b04u0Ve9h+QOeEV0q1jRaHu4j0BN4BrjXGbGp5Se1EZAJMexT2ZMHCR1rU1O2T+hLo8uOxL3xn8yil2remDIV8DVgKDBSRHBG5SURuE5HbPIvcB8QAT4rIShFpB/P4eknaRZB+hR09syer2c3ERQRx/WkpvLdqL5tyS7xYoFJKNUzncz+RiiJ4ahwEBMP3F0JgaLOaKSyrZvzDcxnXL4anrxmlc78rpZqlqfO56xmqJxISDRc9BQVb4PP7mt1Ml7BAbpvYh0+zc/npG6uorKn1YpFKKXU0Dfem6D0BTv0hLH8WNn/R7GZ+cEY/7p48gHe+2cMVzyzVk5uUUq1Gw72pzvwNxKXCez9o8GLaTSEi/Ois/sy6dhRb8kqZ9o9FrNhVeOI3KqXUSdJwb6qAYLh4FpQXwPt3tujs1SlpPXjnjnGEBLiY8cwy3szc7cVClVJKw/3kxKfD2ffDhg9bfPbqwB4RvPeDcWSkdOHnb63mgQ+ycdfWeaVMpZTScD9ZY++wZ69++kvI29CiprqEBfKvG09h5mkpvLh4BzNfXE5RuV7gQynVchruJ8vPDy58GoIi4K0boaZlB0X9XX7cPz2Nhy9J56vtBcyYtYyKah1Jo5RqGQ335ojoDhc+BXnZLRoeWd/lo5OZdV0GG3NL+PV/1+LU+QdKKd+g4d5c/SfbLpqvn4GN//NKk2cM7MadZ/bn7RU5vKEHWZVSLaDh3hJn3w/dh8J7d7T44h6H/fis/pzeL5b73ssme2+xV9pUSnU+Gu4t4R8Elz4P1eXw7vehruWjXVx+wqMzhhMdGsAdr67gUGWNFwpVSnU2Gu4tFTcQzn0Its2DpU94pcnY8CD+cdVIcgor+Pmbq7T/XSl10jTcvWHk9ZA6Hb58EPas8EqTo1O6cs/UQXyancvzi7Z7pU2lVOeh4e4NIjD9cQjvAW/fBMV7vNLszeN7M2Vwdx76ZANZO5s35YFSqnPScPeWkC5wyXP2wOqTp8LqN1p8gW0R4S+XDSMhOoQfvPoNBaVVXipWKeXrNNy9qdepcNsi6DYI3rkF3rgOyhq96mCTRIUE8OTVIzlYXs1PZq+ktk7735VSJ6bh7m0xfeGGT+DsB2DT/+DJsbDxkxY1OSQxigemp7Fw8wEe+WyjlwpVSvkyDffW4OeC038Ct8yF8G7w2gx474dQeajZTc4YncyM0ck8OW8rryzb6cVilVK+SMO9NfUYYgN+/E9h5av2cn07FjerKRHh9xcO4axB3bjvvbX8b613TppSSvkmDffW5h8IZ90HN34KLn945UI4sLl5Tbn8eOKqEQxLiuZHr3/D8h06gkYp1TAN97aSfIoN+IAQ+OjuZo+kCQ3054WZo0mKDuGml5azKbfEy4UqpXyBhntbCu8GZ/0Wti+ANW82u5muYYG8fOMpBAW4uP6Fr9lbVOHFIpVSvkDDva2NugESM+zFPiqaf/3U5K6hvHzDKZRWupn54tcUl+scNEqpb2m4tzU/Pzj/b/ZarF/+rkVNDU6I5JlrR7H9QBm3/CuTyhq9yIdSytJwd0L8MBhzm70Oa05mi5o6rV8sf7t8OF/vOMhPXl9JlVsDXikF4tSMgxkZGSYzs2XB1qFVlcA/ToGwGLhlnh1J0wIvLNrOgx+uI9Dfj6GJUYzq1YWRPaMZ2bML3SKDvVOzUspxIpJljMk44XIa7g7K/i+8eT1MfQjG3t7i5uZtzGPR5gOs2FXI2j2HqK6188snRocwqlcXRvfuysUjEgkLatkXiVLKORruHYEx8OplsGsp/HA5RCZ4rekqdy3Zew+xYmchK3YVkrWzkNxDVcRFBPHTyQO4LCMZl594bX1Kqbah4d5RHNxu558ZMBUuf7nVVmOMYcWuQv748QaydhYysHsE9543iEkDu7XaOpVS3tfUcD/hAVUReUFE8kRkbSOvi4g8LiJbRGS1iIxsTsGdVtfeMOFnsO6/sPnzVluNiDCqV1feuu1Unrp6JJXuWma+uJxrn/+KdXubP+eNUqp9OuGeu4hMAEqBfxljhjTw+nnAncB5wBjgMWPMmBOtWPfc63FXwdOnQ2013LHMnsXayqrddfx72U4en7OZ4ooaLh2ZxO2T+hIRHHBkGcPR/21EBgcQHODyei27D5azYX8JEwbEEuTv/faV8iVe7ZYRkRTgw0bC/RlgnjHmNc/jjcAkY8y+47Wp4X6M7Qvg5WkwaiZM+T0ERbTJaovLa/jnvC28tHjHkQOwjQny92N8/1jOTu3Omand6BbRvFE4xhg27C/h0+z9fJady7p99pfDkMRInrhyJL1jw5rVrlKdQVuG+4fAQ8aYRZ7HXwK/MMZ8J7lF5FbgVoCePXuO2rlTp649ysc/h69nQVAUjL7RjoWP6NEmq97bGXRnAAAXq0lEQVR9sJz5m/I59r+Gw4dcDbA1r5TP1+WyxzPdwfDkaCYP7s7Zqd0Z0D0ckcYP0NbW2T7/T9fu57N1uew6WI4IjOrZhSlp3YkND+LBD9dR467j9xcN4aIRSa3yOZXq6Noy3D8C/nRMuP+fMSbreG3qnnsj9mTB4sdh/fvg5w/pV8BpP4K4AU5XBti97o25JXyxLpfP1+exancRAMldQ0iKDqW6to5qt+fmuV/lrqOsyk1FTS2BLj9O6xfDOWk9OOuYvf+9RRX8+PVvWL6jkEtGJvHgBWk6bFOpY2i3TEdXsBWW/tPOA++uhIHnwbgfQ8+xTld2lLxDlXy5IY85G/IoLKsmKMCPQJcfgf5+BPq7jtwPDvBjZM8uTBoYd1S//rHctXU8PmcLT8zZTO+YMJ64agRpCVFt+ImUat/aMty/B/yQbw+oPm6MOeVEbWq4N1HZAdtV8/WzUHEQhl4GFz4FrsYD0hcs2XqAu2avpLCshl99L5XrTu113G4fpToLr4W7iLwGTAJigVzgt0AAgDHmabH/j/sHMBUoB25oqL/9WBruJ6m6DBY/BvP/DP3PsWPi22BUjZMKSqv42ZurmLsxn4kD4rh6TE8mDIhrlRE7SnUUehKTr1r+PHz0U0g5Ha58rc1G1Tilrs7wwuLtPDFnC8UVNYQFujgrtTvnDe3BxAHdCAnUoFedi4a7L1v9Brx7GyQMh6vfgtCuTlfU6mpq61i6tYBP1u7j0+xcDpZVExLg4sxB3Th3aA/OGNhND76qTkHD3ddt+AjenAkx/eHadyGiu9MVtRl3bR1fbz/Ix2v38b+1uRworSIkwMXUIT24aEQi4/rF6rw5ymdpuHcGW+fC61dBRDxc9x5EJztdUZurrTN8vf0g76/ay0er93Ko0k23iCAuGJ7AxSOTSI2PdLpEpbxKw72z2PWVnVkyKMIGfGw/pytyTGVNLXM25PHOij3M25iHu84wqEcEF49M5JKRScSEBzldolItpuHemexbDa9cBCJwxq9g0PkQHud0VY46WFbNh6v38s6KPazcXURksD/3nJvKjNHJ+GmXjerANNw7m/xN8Ma1kL8BxA96ngqp0yH1fIjq3Kfyb9xfwn3vreWr7QcZ1asLf7xoKAN7+PYoI+W7NNw7I2MgN9tOXbD+A8hbZ59PGAmp02DwBRDT19kaHWKM4a2sHP748XpKKt3cMqEPPzqzvw6lVB2OhruCA1u+Dfq9K+xzQy6Bs+6DLimOluaUg2XV/PHj9byVlUNy1xAevGAIZ+gFS1QHouGujla0G7JesvPVmFoY830Y/1MI6eJ0ZY5Ytq2AX727hq35ZXwvPZ57pg4iuWuo02UpdUIa7qphxXtg7h/thGQh0TDxF5BxE/gHOl1Zm6ty1zJr/jaemLsFd20dU4f04KbT+zCqV+f8wlMdg4a7Or79a+CzX8O2edClN5x9v+2T74STc+0vruTlpTt4ddlODlW6GdkzmlvG92FKWg89GUq1Oxru6sSMgS1fwue/sQdf44fbgB9wDnQb3OmCvqzKzZuZu3l+8XZ2H6wguWsIN47rzeUZya0ytUFlTS3/XraTnl1DmZLWNhdlUR2fhrtqurpa203z9bOwf7V9LjIJBkyxM1D2ngCBjfRH19ZAWb6dmjhukE9079TWGT7L3s+zC7exYlcR4UH+DE+OJi0hksEJkaQlRNE7NqzZe/XGGP63dj9/+Hg9OYX2qla//l4qN4/v482PoXyUhrtqnkP7YPNn9rZtHlSXgisIeo+389iU5UFpng300lyoKPz2vYkZcMPH4O87Z4Jm7SzkrazdrNlTzKb9pUeuMxsS4CI1PoK0hCjSk6KYODCuSdeU3bD/EA+8v46l2woY1COCe89LZfbyXXy8Zj+3TujDPVMH6UlW6rg03FXLuatg5xLY/Dls/hRKcu2Zr2Hd7N/w7t/erzwEX/wWRl4H059wuvJWUVNbx+bcUtbtO0T23mKy9x5i/d5DlFS5ARiWFMVZqd05c1A30hIij7q4SGFZNX/7fBOvfrWTyJAAfjplIFeOTsbf5UdtneGBD7L519KdXDg8gYcvHUagv59TH1O1cxruqu198QAs+huc/yhk3OB0NW2irs5eU3bOhjy+WJ/Lyt1FGAPxUcGcOagbZ6d2Z3dhOX/7fBMllW6uGdOTuyYPIDr06O4rYwxPztvKXz7dyPj+sTx1zSjCdQpj1QANd9X26mrtJGbbF8ANn0DyaKcranMHSquYuyGPL9fnsXBzPmXVtQCM6xfDfeennXDagzczd3PPO2tIjY/gxZmnEBfhO11cyjs03JUzyg/CrElQWw23zu9U88wfq8pdy1fbDuLyE07rG9Pka8DO3ZDHHa+uIC4iiJdvPIXesWGtXKnqSJoa7tqxp7wrtCvMeBUqiuzFRGprnK7IMUH+LiYMiGNcv9iTurj3GYO68dqtYymtcnPpU0t4KyuHMk+/vlJNpeGuvK/HUHtQddcS+PRXTlfTIQ1Pjuat206lS1ggP3tzFRm//4K7Zq9kwaZ8auuc+bWtOhY9YqNaR/pldrKyZU9C4kgYNsPpijqcPnHhfH7XBDJ3FvLOij18tHov736zR680pZpE+9xV66mtgX9dCHsy4cZP7QW9G1JTAdVlEBrT6c6KPRmVNbXM3ZDH2/WuNNW/WzhJXUKICA4gPNifiGB/IoMDCA+y90MDXfiJICIIdvP6iYD9HxHBAfSLCycqNMDpj6eaSA+oqvahNB9mTQRx2eGRpXn25Kf6f6uK7bJRydB7IvSZaM+KjdBT8htz+EpTn6/LpbC8mtJKNyWVbkqq3FS76066vdjwIPp1C6Nft3D6xYXTt1s4/bqF0z0i2GdOqjLGsKOgnMVbDpC9t5hT+8ZyTlp3gvw71pz+Gu6q/diTBS9Pt2e7BkZAeDd7AlT9v/7BsHsZbF8IlUX2fXGDbNj3ngApp9tZLNUJVblrKal0U1rppqzajTF2GiGDwRioMwaDfa6ovJoteaVszS9lS569Hao8+uBtoL8fIQEuQgJcBAf4ERzgIiTQPk6MDmFA9wj6dQ+nf7dwEqJC2tWXQd6hShZvPcDiLQUs2XKAvcWVgD3DuKKmlq5hgVw6KokZo5PpExd+3Laq3LWs2Flkh7hWubksI5khiVFt8TGOouGu2pfqcvu3sTlqDqurtfPbbJsP2+fDzqXgrrB7/sljPPPdTOmUE5u1BWMM+aVVNvDzSikoq6aippbK6loqa+qoqKm1j2tqKatys+tgBQdKq468PzTQRf9u4fTrFkHv2FCq3HUUV9RQVF5j/1bUcKiihqJy225EcADRIQFEhQQQHRpA5OH7IYH4u8Tz3mqKyu17i8trKKqwjw0QFugiLMifsEB/woLs/dBAF8EBLrL3HmJLXikA0aEBnNonhtP6xTKubwwpMWEs2nKA177exefrcnHXGcb07spVY3pyTloPggNcGGPYml/Kgk0HWLg5n2XbDlJRU4u/n+DvEipr6hieHM21Y3vxvfR4ggPa5heAhrvyDe4qyFkOW+fY+W72r7HPRyZB/8l2BsveEyBQx4I7pbCsms15pWzOK2Fzrt3735RbQl5JFSIQdSSwbXhHhwYSFeJPSICLkkr3UeF/+FbqGfoZHOBHdEgg0aHffgFEhwQSFRqAnwhlVfbXSVmVm/LqWkqr3JRX1VJW7aZPXDjj+sYwrl8sg+MjG/1FkVdSyVtZObz+9W52HSw/8kWwcncR+zx7+n1iwxjfP5bx/eMY2zeG2jrDOyty+PeynWzNLyM6NIArMpK5akxPesW07n+LGu7KNx3a65nr5jPYOhdqyuzEZv0nw2l3Qs+xTleoPCpragl0+TWrm6amto7aOtNme8Ngp5JYsrWA15bvImtHISN7RTO+fxyn94tt9CpdxhiWbivg38t28ml2LnXGMKF/HLeM78Pp/WNbpU4Nd+X7Dk9stulTWD0bKg5C0ikw7scw8Dzw09M4VNvZX1zJ68t38Z+vdpFXUsX4/rHce24qgxO8O1zVq+EuIlOBxwAX8Jwx5qFjXu8JvAxEe5a5xxjz8fHa1HBXXlVdBt+8Ckv/AUU7Iaaf3ZNPnwEBJ56KVylvqXLX8srSnTwxZwuHKmu4aEQiP50ykMToEK+077VwFxEXsAmYDOQAy4ErjTHr6i0zC/jGGPOUiAwGPjbGpByvXQ131Spq3bD+fVj8GOxbaackHvN9GHUDhMU4XZ3qRIrLa3hy/hZeXLwDgBvGpXDHpH5EhbTsnAJvzi1zCrDFGLPNGFMNvA5ccMwyBjj82yMK2HsyxSrlNS5/GHIx3DoPrv8A4tNhzu/gr/3h5Wmw/Dk7L71SrSwqNIB7z01l7s8mcX56PLMWbGPiX+by3MJtVLlrW339TdlzvxSYaoy52fP4WmCMMeaH9ZaJBz4DugBhwNnGmKzjtat77qrN5K2HNW/BuvegYDMg9sDr4AsgdRpEJTldoeoEsvcW89AnG1i4+QBXj+nJHy4a2qx2vNktcxlwzjHhfoox5s56y9ztaesRETkVeB4YYoypO6atW4FbAXr27Dlq586dJ/mxlGoBYyB/A6x73wZ9XrZ9PmEkRCeDn3+9m+vb+wGh9ksgcZSOrVcttmBTPj27hpLSzKmcvRnupwL3G2PO8Ty+F8AY86d6y2Rj9+53ex5vA8YaY/Iaa1f33JXjCrbakN/0qT0rts5tb7Xub+/Xue3B2roa6D4ERs2E9MshuO3PTFQKvBvu/tgDqmcBe7AHVK8yxmTXW+YTYLYx5iURSQW+BBLNcRrXcFcdRuUhWPsWZL5oz571D4Ehl9igT8rQvXnVprw9FPI84FHsMMcXjDF/EJEHgUxjzPueETLPAuHYg6v/Z4z57HhtarirDmnvN5D1ku3Dry6Fbmkw7Aro2heiEu2Zs6ExOsZetRo9iUmp1lRVYgM+6yU75LI+VyBEJtigj0ywB2yjkiC657f3g45/LVWlGtPUcNeLdSjVHEERdgrjjBvstMaHcqB4j50eof793csge6/tu68vONpOcRydDH3OgJHX6clWyqs03JVqqfA4e0sY0fDrdbV27vqi3VDsuRXthuIcOLAZNn4Mi/4Gp9+tIa+8RsNdqdbm5/J00yQAY777+vYFMPdP8MnPYdHfYbwn5P2DGm+zpgL2roQDG+1Qzh5D9cCuOoqGu1JO6z0BUsbbkJ/3J/j4Z9+G/IhrbR9+0S479fHur+3f/auP7uqJSPDMdX+OvZJVS6dALs23I4S69rHTKqsORw+oKtWeGAPb5tmQ3/0VRMSDqbPdOmBPqEoYCcmjIWk0xA6wy236n50CubrUToGccjoMmAr9z4YuvZu2V2+MnWUz83l7olddjX1+7A9g8gPg0uustgc6WkapjswY2DYXvnoGgiIh+RQb5t2H2PlzGuKuhl1LYNNnNuwPbrXPh3e3700+xV7NKn740f36FUV2yuTMF+wZvMFRMOwqGHENfPMKfPU09BoHl71kL4moHKXhrlRnV7DVXsHqcHdO4Xb7vF+AnVAteYwd0rn2bagpt9MrZNwIaRcffTnE1W/A+z+y17C9/BX7q0E5RsNdKXW00nzI+doG/e6vYe8KED8YepkN9YThjb93/xqYfY0d4nnun+3yegDXERruSqnjq62xwzSbOvSyohDeudVe4nD4NfC9v0KAdy5AoZpOT2JSSh2fK+DkDpKGdIErZ8P8P8P8hyB3jR3N4x8E/sH2ryvo28eBYdA9zQ4FVW1Ow10p1XR+fnDGvfaErXe/b4dtHk9EPAy91F7usMeQE7dfvAfWf2Bn6yza6ZmuIdlO3RCdDFE9v53GIbDhi1YrS7tllFLN466yM2a6K6G22v51V9rn3ZW2jz/7XdjyuR2Tf3iStaGXeU7o8ijcaS+NuO59e0wA7LI9hsKhPZ6zenO+O4VD8hg4+wHodWrbfeZ2QPvclVLtQ1kBZL8Dq16HPZmA2BOtkkbDli/sTJsAPdLt1bEGXwCx/Y9uo64WSvZ7pm7YBQe32UnbSvbBoPNtyMf2a+tP5ggNd6VU+1Ow1Y6pXz0bCnfY4Zep02HwdHs27MmoLoOlT8LiR+10DBk3wMR77Dw/jSnZb0/U2pNlf20cpd7oH1eA7VKKSrTdQpGJ9nyBdjCVs4a7Uqr9MgYqi+3Y+ZYqzYN5D9k9+YBQOP3H9qzawFC7l79zCexYZP8ePrHLP/jokT7H5qC7CtwVRz/nFwCR8Z6pnOPtvP1Hbl2PfhzWrfGTzVpIw10p1bnkb4Iv7oeNH0F4D7v3XbzbvhYcDb1O+/bWY9jxw9cYe+nF4sPTN+fUu7/HdgeVH7TLNCQ4yk7l3H8y9DsbInp47WNquCulOqcdi+0UyoFh0Ot0G+bdBrdOl0qt247/Ly+odztgu302fwGl++1yPdI9QT/ZHmtowV69hrtSSjnJGHtm75bPbdDv/gpMrd2rn/BzOO3OZjWrJzEppZSTROwcPvHpMP6ndoK2bXNt0EfEt/rqNdyVUqothERD2kX21gacH9ejlFLK6zTclVLKB2m4K6WUD9JwV0opH6ThrpRSPkjDXSmlfJCGu1JK+SANd6WU8kGOTT8gIvnAzma+PRY44MVyvElra572XBu07/q0tubpqLX1MsYcZ15jy7FwbwkRyWzK3ApO0Nqapz3XBu27Pq2teXy9Nu2WUUopH6ThrpRSPqijhvsspws4Dq2tedpzbdC+69Pamsena+uQfe5KKaWOr6PuuSullDoODXellPJBHS7cRWSqiGwUkS0ico/T9dQnIjtEZI2IrBQRR68hKCIviEieiKyt91xXEflcRDZ7/nZpR7XdLyJ7PNtupYic51BtySIyV0TWi0i2iPzY87zj2+44tTm+7UQkWES+FpFVntoe8DzfW0S+8my32SIS2I5qe0lEttfbbsPburZ6NbpE5BsR+dDzuOXbzRjTYW6AC9gK9AECgVXAYKfrqlffDiDW6To8tUwARgJr6z33MHCP5/49wJ/bUW33Az9rB9stHhjpuR8BbAIGt4dtd5zaHN92gADhnvsBwFfAWOANYIbn+aeB29tRbS8Blzr935ynrruB/wAfeh63eLt1tD33U4Atxphtxphq4HXgAodrapeMMQuAg8c8fQHwsuf+y8CFbVqURyO1tQvGmH3GmBWe+yXAeiCRdrDtjlOb44xV6nkY4LkZ4EzgLc/zTm23xmprF0QkCfge8JznseCF7dbRwj0R2F3vcQ7t5D9uDwN8JiJZInKr08U0oLsxZh/YoAC6OVzPsX4oIqs93TaOdBnVJyIpwAjsnl672nbH1AbtYNt5uhZWAnnA59hf2UXGGLdnEcf+/3psbcaYw9vtD57t9ncRCXKiNuBR4P+AOs/jGLyw3TpauEsDz7Wbb2BgnDFmJHAu8AMRmeB0QR3IU0BfYDiwD3jEyWJEJBx4G/iJMeaQk7Ucq4Ha2sW2M8bUGmOGA0nYX9mpDS3WtlV5VnpMbSIyBLgXGASMBroCv2jrukTkfCDPGJNV/+kGFj3p7dbRwj0HSK73OAnY61At32GM2ev5mwe8i/0PvD3JFZF4AM/fPIfrOcIYk+v5P2Ad8CwObjsRCcCG56vGmHc8T7eLbddQbe1p23nqKQLmYfu1o0XE3/OS4/9/rVfbVE83lzHGVAEv4sx2GwdMF5Ed2G7mM7F78i3ebh0t3JcD/T1HkgOBGcD7DtcEgIiEiUjE4fvAFGDt8d/V5t4Hrvfcvx54z8FajnI4OD0uwqFt5+nvfB5Yb4z5W72XHN92jdXWHradiMSJSLTnfghwNvaYwFzgUs9iTm23hmrbUO/LWrB92m2+3Ywx9xpjkowxKdg8m2OMuRpvbDenjxI346jyedhRAluBXzldT726+mBH76wCsp2uDXgN+xO9BvuL5yZsX96XwGbP367tqLZXgDXAamyQxjtU2+nYn8CrgZWe23ntYdsdpzbHtx2QDnzjqWEtcJ/n+T7A18AW4E0gqB3VNsez3dYC/8YzosapGzCJb0fLtHi76fQDSinlgzpat4xSSqkm0HBXSikfpOGulFI+SMNdKaV8kIa7Ukr5IA13pZTyQRruSinlg/4fibbHrG9P00gAAAAASUVORK5CYII=\n",
"text/plain": [
"<matplotlib.figure.Figure at 0x7f6cfebf8f28>"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"plt.plot(np.arange(40), warm_restart_hist.history['val_loss'], label='warm restart')\n",
"plt.plot(np.arange(40), hist.history['val_loss'], label='normal')\n",
"plt.title('CIFAR10 validation loss')\n",
"plt.legend()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.4"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment