3-2. VGG16 Triplet Network (margin 0.2).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import warnings\n",
    "warnings.simplefilter('ignore')\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "import sys, os, time, gc, random, pickle\n",
    "from sklearn.neighbors import NearestNeighbors\n",
    "import requests, shutil\n",
    "import tensorflow as tf\n",
    "import keras\n",
    "\n",
    "from keras import backend as K\n",
    "from keras.preprocessing.image import load_img, img_to_array\n",
    "from keras.models import Sequential, load_model, Model\n",
    "from keras.layers import Dense, Dropout, Activation, Flatten, Input, Lambda, MaxPooling2D\n",
    "from keras.callbacks import ModelCheckpoint, LearningRateScheduler, ReduceLROnPlateau\n",
    "from keras.applications.vgg16 import VGG16\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train:\t\t (113783, 4)\n",
      "Validation:\t (22255, 4)\n",
      "Test:\t\t (22391, 4)\n",
      "\n",
      "Train Landmarks:\t 14943\n",
      "Validation Landmarks:\t 7674\n",
      "Test Landmarks:\t\t 14436\n"
     ]
    }
   ],
   "source": [
    "train_df = pd.read_csv('./data/triplet/train.csv')\n",
    "val_df = pd.read_csv('./data/triplet/validation.csv')\n",
    "test_df = pd.read_csv('./data/triplet/test.csv')\n",
    "\n",
    "print('Train:\\t\\t', train_df.shape)\n",
    "print('Validation:\\t', val_df.shape)\n",
    "print('Test:\\t\\t', test_df.shape)\n",
    "\n",
    "print('\\nTrain Landmarks:\\t', len(train_df['landmark_id'].unique()))\n",
    "print('Validation Landmarks:\\t', len(val_df['landmark_id'].unique()))\n",
    "print('Test Landmarks:\\t\\t', len(test_df['landmark_id'].unique()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>image_id</th>\n",
       "      <th>id</th>\n",
       "      <th>url</th>\n",
       "      <th>landmark_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>465272</td>\n",
       "      <td>a2ccf8ed2e969f6a</td>\n",
       "      <td>https://lh4.googleusercontent.com/-TPHkS5gzvm4...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>64516</td>\n",
       "      <td>e205ca7c8dd7c027</td>\n",
       "      <td>https://lh3.googleusercontent.com/-V3RjsZtGpxE...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>928409</td>\n",
       "      <td>4e8ab93c1620e8a3</td>\n",
       "      <td>http://mw2.google.com/mw-panoramio/photos/medi...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>88809</td>\n",
       "      <td>896bf928214d1ca4</td>\n",
       "      <td>http://lh5.ggpht.com/-Cy0l41uUaGA/R--yB8vy41I/...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1001133</td>\n",
       "      <td>375d2a153bdca926</td>\n",
       "      <td>http://lh6.ggpht.com/-UqzFpnqE9bU/S_0u1RovfdI/...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   image_id                id  \\\n",
       "0    465272  a2ccf8ed2e969f6a   \n",
       "1     64516  e205ca7c8dd7c027   \n",
       "2    928409  4e8ab93c1620e8a3   \n",
       "3     88809  896bf928214d1ca4   \n",
       "4   1001133  375d2a153bdca926   \n",
       "\n",
       "                                                 url  landmark_id  \n",
       "0  https://lh4.googleusercontent.com/-TPHkS5gzvm4...            0  \n",
       "1  https://lh3.googleusercontent.com/-V3RjsZtGpxE...            0  \n",
       "2  http://mw2.google.com/mw-panoramio/photos/medi...            0  \n",
       "3  http://lh5.ggpht.com/-Cy0l41uUaGA/R--yB8vy41I/...            0  \n",
       "4  http://lh6.ggpht.com/-UqzFpnqE9bU/S_0u1RovfdI/...            0  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Helper Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# training set triplet generator\n",
    "def train_triplet_generator(df, batch_size=100, img_size=(224, 224), seed=42, \n",
    "                            prefix='./data/triplet/train/'):\n",
    "    \"\"\" training set triplet generator\n",
    "        it will generate 7400 triplet images in total\n",
    "    \"\"\"\n",
    "    # get images with only one training image landmark id and the rest landmark ids\n",
    "    np.random.seed(seed)\n",
    "    grouped = df[['landmark_id', 'image_id']].groupby('landmark_id').count().reset_index()\n",
    "    unique_neg_ids = list(grouped[grouped['image_id'] == 1]['landmark_id'].values)\n",
    "    rest_ids = list(grouped[grouped['image_id'] > 1]['landmark_id'].values)\n",
    "    size = 7400 * 2 - len(unique_neg_ids) \n",
    "    zeros = np.zeros((batch_size, 3, 1), dtype=K.floatx())\n",
    "    \n",
    "    while True:\n",
    "        # get positive and negative image landmark ids\n",
    "        np.random.shuffle(rest_ids)\n",
    "        candidate_ids = list(np.random.choice(rest_ids, size=size, replace=False))\n",
    "        pos_landmark_ids = candidate_ids[:7400]\n",
    "        neg_landmark_ids = candidate_ids[7400:] + unique_neg_ids\n",
    "        np.random.shuffle(neg_landmark_ids)\n",
    "        \n",
    "        # transform landmark id into image id\n",
    "        anc_img_ids = []\n",
    "        pos_img_ids = []\n",
    "        neg_img_ids = []\n",
    "        \n",
    "        for i in range(len(pos_landmark_ids)):\n",
    "            tmp_pos_ids = df[df['landmark_id'] == pos_landmark_ids[i]]['image_id'].values\n",
    "            anc_img_ids.append(tmp_pos_ids[0])\n",
    "            pos_img_ids.append(tmp_pos_ids[1])\n",
    "            \n",
    "            tmp_neg_ids = df[df['landmark_id'] == neg_landmark_ids[i]]['image_id'].values\n",
    "            neg_img_ids.append(tmp_neg_ids[0])\n",
    "        \n",
    "        # iterator to read batch images\n",
    "        for j in range(len(pos_img_ids) // batch_size):\n",
    "            batch_anc_img_ids = anc_img_ids[j * batch_size: (j + 1) * batch_size]\n",
    "            batch_pos_img_ids = pos_img_ids[j * batch_size: (j + 1) * batch_size]\n",
    "            batch_neg_img_ids = neg_img_ids[j * batch_size: (j + 1) * batch_size]\n",
    "            \n",
    "            # get images\n",
    "            anc_imgs = []\n",
    "            pos_imgs = []\n",
    "            neg_imgs = []\n",
    "            \n",
    "            # iteratively read images\n",
    "            for k in range(batch_size):\n",
    "                anc_path = prefix + str(batch_anc_img_ids[k]) + '.jpg'\n",
    "                pos_path = prefix + str(batch_pos_img_ids[k]) + '.jpg'\n",
    "                neg_path = prefix + str(batch_neg_img_ids[k]) + '.jpg'\n",
    "                \n",
    "                tmp_anc_img = load_img(anc_path, target_size=img_size)\n",
    "                tmp_anc_img = img_to_array(tmp_anc_img)\n",
    "                anc_imgs.append(tmp_anc_img)\n",
    "                \n",
    "                tmp_pos_img = load_img(pos_path, target_size=img_size)\n",
    "                tmp_pos_img = img_to_array(tmp_pos_img)\n",
    "                pos_imgs.append(tmp_pos_img)\n",
    "                \n",
    "                tmp_neg_img = load_img(neg_path, target_size=img_size)\n",
    "                tmp_neg_img = img_to_array(tmp_neg_img)\n",
    "                neg_imgs.append(tmp_neg_img)\n",
    "        \n",
    "            # transform list to array\n",
    "            anc_imgs = np.array(anc_imgs, dtype=K.floatx()) / 255.0\n",
    "            pos_imgs = np.array(pos_imgs, dtype=K.floatx()) / 255.0\n",
    "            neg_imgs = np.array(neg_imgs, dtype=K.floatx()) / 255.0\n",
    "\n",
    "            yield [anc_imgs, pos_imgs, neg_imgs], zeros"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# validation set triplet generator\n",
    "def val_triplet_generator(df, batch_size=128, img_size=(224, 224), \n",
    "                          seed=42, prefix='./data/triplet/validation'):\n",
    "    \"\"\" validation set triplet collector \"\"\"\n",
    "    \n",
    "     # get images with only one image landmark id and the rest landmark ids\n",
    "    grouped = df[['landmark_id', 'image_id']].groupby('landmark_id').count().reset_index()\n",
    "    unique_neg_ids = list(grouped[grouped['image_id'] == 1]['landmark_id'].values)\n",
    "    rest_ids = list(grouped[grouped['image_id'] > 1]['landmark_id'].values)\n",
    "    size = 3072 * 2 - len(unique_neg_ids) \n",
    "    zeros = np.zeros((batch_size, 3, 1), dtype=K.floatx())\n",
    "    \n",
    "    while True:\n",
    "        # get positive and negative image landmark ids\n",
    "        np.random.seed(seed)\n",
    "        candidate_ids = list(np.random.choice(rest_ids, size=size, replace=False))\n",
    "        pos_landmark_ids = candidate_ids[:3072]\n",
    "        neg_landmark_ids = candidate_ids[3072:] + unique_neg_ids\n",
    "        np.random.shuffle(neg_landmark_ids)\n",
    "        \n",
    "        # transform landmark id into image id\n",
    "        anc_img_ids = []\n",
    "        pos_img_ids = []\n",
    "        neg_img_ids = []\n",
    "        \n",
    "        for i in range(len(pos_landmark_ids)):\n",
    "            tmp_pos_ids = df[df['landmark_id'] == pos_landmark_ids[i]]['image_id'].values\n",
    "            anc_img_ids.append(tmp_pos_ids[0])\n",
    "            pos_img_ids.append(tmp_pos_ids[1])\n",
    "            \n",
    "            tmp_neg_ids = df[df['landmark_id'] == neg_landmark_ids[i]]['image_id'].values\n",
    "            neg_img_ids.append(tmp_neg_ids[0])\n",
    "        \n",
    "        # iterator to read batch images\n",
    "        for j in range(len(pos_img_ids) // batch_size):\n",
    "            batch_anc_img_ids = anc_img_ids[j * batch_size: (j + 1) * batch_size]\n",
    "            batch_pos_img_ids = pos_img_ids[j * batch_size: (j + 1) * batch_size]\n",
    "            batch_neg_img_ids = neg_img_ids[j * batch_size: (j + 1) * batch_size]\n",
    "            \n",
    "            # get images\n",
    "            anc_imgs = []\n",
    "            pos_imgs = []\n",
    "            neg_imgs = []\n",
    "            \n",
    "            # iteratively read images\n",
    "            for k in range(batch_size):\n",
    "                anc_path = prefix + str(batch_anc_img_ids[k]) + '.jpg'\n",
    "                pos_path = prefix + str(batch_pos_img_ids[k]) + '.jpg'\n",
    "                neg_path = prefix + str(batch_neg_img_ids[k]) + '.jpg'\n",
    "                \n",
    "                tmp_anc_img = load_img(anc_path, target_size=img_size)\n",
    "                tmp_anc_img = img_to_array(tmp_anc_img)\n",
    "                anc_imgs.append(tmp_anc_img)\n",
    "                \n",
    "                tmp_pos_img = load_img(pos_path, target_size=img_size)\n",
    "                tmp_pos_img = img_to_array(tmp_pos_img)\n",
    "                pos_imgs.append(tmp_pos_img)\n",
    "                \n",
    "                tmp_neg_img = load_img(neg_path, target_size=img_size)\n",
    "                tmp_neg_img = img_to_array(tmp_neg_img)\n",
    "                neg_imgs.append(tmp_neg_img)\n",
    "        \n",
    "            # transform list to array\n",
    "            anc_imgs = np.array(anc_imgs, dtype=K.floatx()) / 255.0\n",
    "            pos_imgs = np.array(pos_imgs, dtype=K.floatx()) / 255.0\n",
    "            neg_imgs = np.array(neg_imgs, dtype=K.floatx()) / 255.0\n",
    "            \n",
    "            yield [anc_imgs, pos_imgs, neg_imgs], zeros"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Define Triplet Loss Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define base network for triplet network\n",
    "def base_net(input_shape=(224, 224, 3)):\n",
    "    \"\"\" define triplet network \"\"\"\n",
    "    # load pre-trained VGG16 model\n",
    "    vgg16 = VGG16(include_top=False, weights='imagenet', input_shape=input_shape, pooling='avg')\n",
    "    \n",
    "    # frozen shallow layers\n",
    "    vgg16.trainable = True\n",
    "    \n",
    "    set_trainable = False\n",
    "    for layer in vgg16.layers:\n",
    "        if layer.name == 'block5_conv1':\n",
    "            set_trainable = True\n",
    "        if set_trainable:\n",
    "            layer.trainable = True\n",
    "        else:\n",
    "            layer.trainable = False\n",
    "    \n",
    "    # define sequential model\n",
    "    model = Sequential(name='base_net')\n",
    "    model.add(vgg16)\n",
    "    model.add(Lambda(lambda x: K.l2_normalize(x, axis=1), name='l2_norm'))\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define triplet network\n",
    "def triplet_net(base_model, input_shape=(224, 224, 3)):\n",
    "    \"\"\" function to define triplet networks \"\"\"\n",
    "    # define input: anchor, positive, negative\n",
    "    anchor = Input(shape=input_shape, name='anchor_input')\n",
    "    positive = Input(shape=input_shape, name='positive_input')\n",
    "    negative = Input(shape=input_shape, name='negative_input')\n",
    "    \n",
    "    # extract vector represent using CNN based model\n",
    "    anc_vec = base_model(anchor)\n",
    "    pos_vec = base_model(positive)\n",
    "    neg_vec = base_model(negative)\n",
    "    \n",
    "    # stack outputs\n",
    "    stacks = Lambda(lambda x: K.stack(x, axis=1), name='output')([anc_vec, pos_vec, neg_vec])\n",
    "\n",
    "    # define inputs and outputs\n",
    "    inputs=[anchor, positive, negative]\n",
    "    outputs = stacks\n",
    "    \n",
    "    # define the triplet model\n",
    "    model = Model(inputs=inputs, outputs=outputs, name='triplet_net')\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define triplet loss\n",
    "def triplet_loss(y_true, y_pred):\n",
    "    \"\"\" function to compute triplet loss\n",
    "        margin is predefined coded, manually change if needed\n",
    "    \"\"\"\n",
    "    # define triplet margin\n",
    "    margin = K.constant(0.2)\n",
    "    zero = K.constant(0.0)\n",
    "    \n",
    "    # get the prediction vector\n",
    "    anchor, positive, negative = y_pred[:, 0], y_pred[:, 1], y_pred[:, 2]\n",
    "    \n",
    "    # compute distance\n",
    "    pos_distance = K.sum(K.square(anchor - positive), axis=1)\n",
    "    neg_distance = K.sum(K.square(anchor - negative), axis=1)\n",
    "    \n",
    "    # compute loss\n",
    "    partial_loss = pos_distance - neg_distance + margin\n",
    "    full_loss = K.sum(K.maximum(partial_loss, zero), axis=0)\n",
    "    \n",
    "    return full_loss"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Build Triplet Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# For reproduciable purpose\n",
    "seed = 42\n",
    "K.clear_session()\n",
    "os.environ['PYTHONHASHSEED'] = '0'\n",
    "np.random.seed(seed)\n",
    "random.seed(seed)\n",
    "session_conf = tf.ConfigProto(intra_op_parallelism_threads=1, inter_op_parallelism_threads=1)\n",
    "tf.set_random_seed(seed)\n",
    "sess = tf.Session(graph=tf.get_default_graph(), config=session_conf)\n",
    "K.set_session(sess)\n",
    "\n",
    "# Define Parameters\n",
    "img_size = (224, 224, 3)  # target image size\n",
    "\n",
    "# triplet image generator\n",
    "train_generator = train_triplet_generator(train_df, batch_size=100, img_size=img_size[:2], \n",
    "                                          seed=42, prefix='./data/triplet/train/')\n",
    "\n",
    "val_generator = val_triplet_generator(val_df, batch_size=64, img_size=img_size[:2], \n",
    "                                      seed=42, prefix='./data/triplet/validation/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/ubuntu/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py:3148: calling l2_normalize (from tensorflow.python.ops.nn_impl) with dim is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "dim is deprecated, use axis instead\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "vgg16 (Model)                (None, 512)               14714688  \n",
      "_________________________________________________________________\n",
      "l2_norm (Lambda)             (None, 512)               0         \n",
      "=================================================================\n",
      "Total params: 14,714,688\n",
      "Trainable params: 7,079,424\n",
      "Non-trainable params: 7,635,264\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Define triplet network model\n",
    "base_model = base_net(input_shape=img_size)\n",
    "base_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "anchor_input (InputLayer)       (None, 224, 224, 3)  0                                            \n",
      "__________________________________________________________________________________________________\n",
      "positive_input (InputLayer)     (None, 224, 224, 3)  0                                            \n",
      "__________________________________________________________________________________________________\n",
      "negative_input (InputLayer)     (None, 224, 224, 3)  0                                            \n",
      "__________________________________________________________________________________________________\n",
      "base_net (Sequential)           (None, 512)          14714688    anchor_input[0][0]               \n",
      "                                                                 positive_input[0][0]             \n",
      "                                                                 negative_input[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "output (Lambda)                 (None, 3, 512)       0           base_net[1][0]                   \n",
      "                                                                 base_net[2][0]                   \n",
      "                                                                 base_net[3][0]                   \n",
      "==================================================================================================\n",
      "Total params: 14,714,688\n",
      "Trainable params: 7,079,424\n",
      "Non-trainable params: 7,635,264\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "triplet_model = triplet_net(base_model=base_model, input_shape=img_size)\n",
    "triplet_model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Fit Triplet Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Learning rate:  0.001\n"
     ]
    }
   ],
   "source": [
    "# define learning scheduler\n",
    "def lr_schedule(epoch):\n",
    "    \"\"\" Learning rate schedule \"\"\"\n",
    "    lr = 1e-3\n",
    "    if epoch > 80:\n",
    "        lr *= 2e-1\n",
    "    elif epoch > 60:\n",
    "        lr *= 4e-1\n",
    "    elif epoch > 40:\n",
    "        lr *= 6e-1\n",
    "    elif epoch > 20:\n",
    "        lr *= 8e-1\n",
    "    print('Learning rate: ', lr)\n",
    "    return lr\n",
    "\n",
    "# define optimizer\n",
    "opt = keras.optimizers.Adam(lr=lr_schedule(0))\n",
    "\n",
    "# Create call backs\n",
    "lr_scheduler = LearningRateScheduler(lr_schedule)\n",
    "lr_reducer = ReduceLROnPlateau(factor=np.sqrt(0.1), cooldown=0, patience=5, min_lr=0.5e-6)\n",
    "callbacks = [lr_reducer, lr_scheduler]\n",
    "\n",
    "# compile the model\n",
    "triplet_model.compile(optimizer=opt, loss=triplet_loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/100\n",
      "Learning rate:  0.001\n",
      " - 77s - loss: 7.7332 - val_loss: 4.2107\n",
      "Epoch 2/100\n",
      "Learning rate:  0.001\n",
      " - 58s - loss: 5.6798 - val_loss: 3.6927\n",
      "Epoch 3/100\n",
      "Learning rate:  0.001\n",
      " - 58s - loss: 4.7835 - val_loss: 3.6454\n",
      "Epoch 4/100\n",
      "Learning rate:  0.001\n",
      " - 58s - loss: 4.3902 - val_loss: 3.2572\n",
      "Epoch 5/100\n",
      "Learning rate:  0.001\n",
      " - 60s - loss: 3.8070 - val_loss: 3.1818\n",
      "Epoch 6/100\n",
      "Learning rate:  0.001\n",
      " - 59s - loss: 3.5787 - val_loss: 2.9858\n",
      "Epoch 7/100\n",
      "Learning rate:  0.001\n",
      " - 58s - loss: 3.3480 - val_loss: 3.0683\n",
      "Epoch 8/100\n",
      "Learning rate:  0.001\n",
      " - 59s - loss: 3.2973 - val_loss: 2.7818\n",
      "Epoch 9/100\n",
      "Learning rate:  0.001\n",
      " - 59s - loss: 2.9821 - val_loss: 2.6965\n",
      "Epoch 10/100\n",
      "Learning rate:  0.001\n",
      " - 59s - loss: 2.8711 - val_loss: 2.4352\n",
      "Epoch 11/100\n",
      "Learning rate:  0.001\n",
      " - 59s - loss: 2.7547 - val_loss: 2.4549\n",
      "Epoch 12/100\n",
      "Learning rate:  0.001\n",
      " - 59s - loss: 2.5552 - val_loss: 2.3653\n",
      "Epoch 13/100\n",
      "Learning rate:  0.001\n",
      " - 59s - loss: 2.4377 - val_loss: 2.4918\n",
      "Epoch 14/100\n",
      "Learning rate:  0.001\n",
      " - 58s - loss: 2.4175 - val_loss: 2.1328\n",
      "Epoch 15/100\n",
      "Learning rate:  0.001\n",
      " - 59s - loss: 2.1246 - val_loss: 2.6063\n",
      "Epoch 16/100\n",
      "Learning rate:  0.001\n",
      " - 59s - loss: 2.2620 - val_loss: 2.2274\n",
      "Epoch 17/100\n",
      "Learning rate:  0.001\n",
      " - 59s - loss: 2.0758 - val_loss: 2.1308\n",
      "Epoch 18/100\n",
      "Learning rate:  0.001\n",
      " - 58s - loss: 2.0344 - val_loss: 2.1936\n",
      "Epoch 19/100\n",
      "Learning rate:  0.001\n",
      " - 59s - loss: 1.9755 - val_loss: 2.1215\n",
      "Epoch 20/100\n",
      "Learning rate:  0.001\n",
      " - 59s - loss: 2.0538 - val_loss: 2.0722\n",
      "Epoch 21/100\n",
      "Learning rate:  0.001\n",
      " - 59s - loss: 1.9367 - val_loss: 2.0892\n",
      "Epoch 22/100\n",
      "Learning rate:  0.0008\n",
      " - 59s - loss: 1.8953 - val_loss: 2.0480\n",
      "Epoch 23/100\n",
      "Learning rate:  0.0008\n",
      " - 59s - loss: 1.6609 - val_loss: 1.9786\n",
      "Epoch 24/100\n",
      "Learning rate:  0.0008\n",
      " - 59s - loss: 1.7405 - val_loss: 1.9201\n",
      "Epoch 25/100\n",
      "Learning rate:  0.0008\n",
      " - 59s - loss: 1.7180 - val_loss: 2.0296\n",
      "Epoch 26/100\n",
      "Learning rate:  0.0008\n",
      " - 59s - loss: 1.5015 - val_loss: 2.1107\n",
      "Epoch 27/100\n",
      "Learning rate:  0.0008\n",
      " - 59s - loss: 1.6364 - val_loss: 1.8464\n",
      "Epoch 28/100\n",
      "Learning rate:  0.0008\n",
      " - 59s - loss: 1.6187 - val_loss: 1.9587\n",
      "Epoch 29/100\n",
      "Learning rate:  0.0008\n",
      " - 59s - loss: 1.4990 - val_loss: 1.7847\n",
      "Epoch 30/100\n",
      "Learning rate:  0.0008\n",
      " - 59s - loss: 1.3793 - val_loss: 1.8119\n",
      "Epoch 31/100\n",
      "Learning rate:  0.0008\n",
      " - 59s - loss: 1.3830 - val_loss: 1.8721\n",
      "Epoch 32/100\n",
      "Learning rate:  0.0008\n",
      " - 59s - loss: 1.3360 - val_loss: 1.8716\n",
      "Epoch 33/100\n",
      "Learning rate:  0.0008\n",
      " - 59s - loss: 1.4114 - val_loss: 1.8452\n",
      "Epoch 34/100\n",
      "Learning rate:  0.0008\n",
      " - 59s - loss: 1.4221 - val_loss: 1.8763\n",
      "Epoch 35/100\n",
      "Learning rate:  0.0008\n",
      " - 59s - loss: 1.3269 - val_loss: 1.8478\n",
      "Epoch 36/100\n",
      "Learning rate:  0.0008\n",
      " - 59s - loss: 1.2382 - val_loss: 1.8724\n",
      "Epoch 37/100\n",
      "Learning rate:  0.0008\n",
      " - 60s - loss: 1.2443 - val_loss: 1.8216\n",
      "Epoch 38/100\n",
      "Learning rate:  0.0008\n",
      " - 59s - loss: 1.3180 - val_loss: 1.7071\n",
      "Epoch 39/100\n",
      "Learning rate:  0.0008\n",
      " - 59s - loss: 1.3271 - val_loss: 1.8952\n",
      "Epoch 40/100\n",
      "Learning rate:  0.0008\n",
      " - 59s - loss: 1.1610 - val_loss: 1.6942\n",
      "Epoch 41/100\n",
      "Learning rate:  0.0008\n",
      " - 59s - loss: 1.2403 - val_loss: 1.9045\n",
      "Epoch 42/100\n",
      "Learning rate:  0.0006\n",
      " - 59s - loss: 1.0876 - val_loss: 1.6120\n",
      "Epoch 43/100\n",
      "Learning rate:  0.0006\n",
      " - 59s - loss: 1.0502 - val_loss: 1.7563\n",
      "Epoch 44/100\n",
      "Learning rate:  0.0006\n",
      " - 59s - loss: 1.0752 - val_loss: 1.8270\n",
      "Epoch 45/100\n",
      "Learning rate:  0.0006\n",
      " - 60s - loss: 1.0311 - val_loss: 1.6308\n",
      "Epoch 46/100\n",
      "Learning rate:  0.0006\n",
      " - 59s - loss: 0.8716 - val_loss: 1.6669\n",
      "Epoch 47/100\n",
      "Learning rate:  0.0006\n",
      " - 59s - loss: 0.8397 - val_loss: 1.7649\n",
      "Epoch 48/100\n",
      "Learning rate:  0.0006\n",
      " - 59s - loss: 0.8891 - val_loss: 1.5974\n",
      "Epoch 49/100\n",
      "Learning rate:  0.0006\n",
      " - 59s - loss: 0.9445 - val_loss: 1.6254\n",
      "Epoch 50/100\n",
      "Learning rate:  0.0006\n",
      " - 60s - loss: 0.8980 - val_loss: 1.6149\n",
      "Epoch 51/100\n",
      "Learning rate:  0.0006\n",
      " - 59s - loss: 0.8600 - val_loss: 1.6353\n",
      "Epoch 52/100\n",
      "Learning rate:  0.0006\n",
      " - 59s - loss: 1.0008 - val_loss: 1.6684\n",
      "Epoch 53/100\n",
      "Learning rate:  0.0006\n",
      " - 59s - loss: 0.8027 - val_loss: 1.5662\n",
      "Epoch 54/100\n",
      "Learning rate:  0.0006\n",
      " - 59s - loss: 0.7801 - val_loss: 1.6854\n",
      "Epoch 55/100\n",
      "Learning rate:  0.0006\n",
      " - 59s - loss: 0.7548 - val_loss: 1.6016\n",
      "Epoch 56/100\n",
      "Learning rate:  0.0006\n",
      " - 59s - loss: 0.8114 - val_loss: 1.6352\n",
      "Epoch 57/100\n",
      "Learning rate:  0.0006\n",
      " - 59s - loss: 0.7686 - val_loss: 1.4522\n",
      "Epoch 58/100\n",
      "Learning rate:  0.0006\n",
      " - 59s - loss: 0.7300 - val_loss: 1.5663\n",
      "Epoch 59/100\n",
      "Learning rate:  0.0006\n",
      " - 59s - loss: 0.7487 - val_loss: 1.4708\n",
      "Epoch 60/100\n",
      "Learning rate:  0.0006\n",
      " - 59s - loss: 0.7510 - val_loss: 1.5861\n",
      "Epoch 61/100\n",
      "Learning rate:  0.0006\n",
      " - 59s - loss: 0.6935 - val_loss: 1.6596\n",
      "Epoch 62/100\n",
      "Learning rate:  0.0004\n",
      " - 59s - loss: 0.6797 - val_loss: 1.4774\n",
      "Epoch 63/100\n",
      "Learning rate:  0.0004\n",
      " - 59s - loss: 0.7001 - val_loss: 1.5134\n",
      "Epoch 64/100\n",
      "Learning rate:  0.0004\n",
      " - 59s - loss: 0.5844 - val_loss: 1.4197\n",
      "Epoch 65/100\n",
      "Learning rate:  0.0004\n",
      " - 59s - loss: 0.6446 - val_loss: 1.4926\n",
      "Epoch 66/100\n",
      "Learning rate:  0.0004\n",
      " - 59s - loss: 0.6026 - val_loss: 1.5527\n",
      "Epoch 67/100\n",
      "Learning rate:  0.0004\n",
      " - 59s - loss: 0.6460 - val_loss: 1.4644\n",
      "Epoch 68/100\n",
      "Learning rate:  0.0004\n",
      " - 59s - loss: 0.5630 - val_loss: 1.5861\n",
      "Epoch 69/100\n",
      "Learning rate:  0.0004\n",
      " - 59s - loss: 0.6012 - val_loss: 1.4482\n",
      "Epoch 70/100\n",
      "Learning rate:  0.0004\n",
      " - 59s - loss: 0.5075 - val_loss: 1.4659\n",
      "Epoch 71/100\n",
      "Learning rate:  0.0004\n",
      " - 59s - loss: 0.5888 - val_loss: 1.5850\n",
      "Epoch 72/100\n",
      "Learning rate:  0.0004\n",
      " - 59s - loss: 0.5064 - val_loss: 1.4944\n",
      "Epoch 73/100\n",
      "Learning rate:  0.0004\n",
      " - 59s - loss: 0.4329 - val_loss: 1.4240\n",
      "Epoch 74/100\n",
      "Learning rate:  0.0004\n",
      " - 59s - loss: 0.5468 - val_loss: 1.5128\n",
      "Epoch 75/100\n",
      "Learning rate:  0.0004\n",
      " - 58s - loss: 0.4864 - val_loss: 1.4003\n",
      "Epoch 76/100\n",
      "Learning rate:  0.0004\n",
      " - 59s - loss: 0.5130 - val_loss: 1.4395\n",
      "Epoch 77/100\n",
      "Learning rate:  0.0004\n",
      " - 58s - loss: 0.5056 - val_loss: 1.5506\n",
      "Epoch 78/100\n",
      "Learning rate:  0.0004\n",
      " - 59s - loss: 0.5362 - val_loss: 1.5528\n",
      "Epoch 79/100\n",
      "Learning rate:  0.0004\n",
      " - 59s - loss: 0.5445 - val_loss: 1.4497\n",
      "Epoch 80/100\n",
      "Learning rate:  0.0004\n",
      " - 59s - loss: 0.4415 - val_loss: 1.4614\n",
      "Epoch 81/100\n",
      "Learning rate:  0.0004\n",
      " - 59s - loss: 0.4713 - val_loss: 1.4815\n",
      "Epoch 82/100\n",
      "Learning rate:  0.0002\n",
      " - 59s - loss: 0.4706 - val_loss: 1.3981\n",
      "Epoch 83/100\n",
      "Learning rate:  0.0002\n",
      " - 59s - loss: 0.4149 - val_loss: 1.3690\n",
      "Epoch 84/100\n",
      "Learning rate:  0.0002\n",
      " - 59s - loss: 0.4281 - val_loss: 1.3740\n",
      "Epoch 85/100\n",
      "Learning rate:  0.0002\n",
      " - 59s - loss: 0.3734 - val_loss: 1.4188\n",
      "Epoch 88/100\n",
      "Learning rate:  0.0002\n",
      " - 59s - loss: 0.3628 - val_loss: 1.4053\n",
      "Epoch 89/100\n",
      "Learning rate:  0.0002\n",
      " - 59s - loss: 0.3606 - val_loss: 1.3607\n",
      "Epoch 90/100\n",
      "Learning rate:  0.0002\n",
      " - 58s - loss: 0.3287 - val_loss: 1.3653\n",
      "Epoch 91/100\n",
      "Learning rate:  0.0002\n",
      " - 59s - loss: 0.3738 - val_loss: 1.4178\n",
      "Epoch 92/100\n",
      "Learning rate:  0.0002\n",
      " - 59s - loss: 0.3550 - val_loss: 1.3949\n",
      "Epoch 93/100\n",
      "Learning rate:  0.0002\n",
      " - 59s - loss: 0.3267 - val_loss: 1.4047\n",
      "Epoch 94/100\n",
      "Learning rate:  0.0002\n",
      " - 59s - loss: 0.3786 - val_loss: 1.4003\n",
      "Epoch 95/100\n",
      "Learning rate:  0.0002\n",
      " - 59s - loss: 0.3837 - val_loss: 1.3409\n",
      "Epoch 96/100\n",
      "Learning rate:  0.0002\n",
      " - 59s - loss: 0.3656 - val_loss: 1.4185\n",
      "Epoch 97/100\n",
      "Learning rate:  0.0002\n",
      " - 59s - loss: 0.3338 - val_loss: 1.4077\n",
      "Epoch 98/100\n",
      "Learning rate:  0.0002\n",
      " - 59s - loss: 0.4139 - val_loss: 1.4140\n",
      "Epoch 99/100\n",
      "Learning rate:  0.0002\n",
      " - 59s - loss: 0.2979 - val_loss: 1.4310\n",
      "Epoch 100/100\n",
      "Learning rate:  0.0002\n",
      " - 59s - loss: 0.3261 - val_loss: 1.4065\n"
     ]
    }
   ],
   "source": [
    "# fit the mode\n",
    "history = triplet_model.fit_generator(train_generator, steps_per_epoch=74, epochs=100, \n",
    "                                      validation_data=val_generator, validation_steps=48, \n",
    "                                      verbose=2, callbacks=callbacks)\n",
    "\n",
    "base_model.save('./models/vgg16-base-0.2-model.h5')\n",
    "pickle.dump(history.history, open('./models/vgg16-triplet-0.2-history.p', 'wb'))\n",
    "_ = gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAl8AAAG/CAYAAABmE7ICAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvFvnyVgAAIABJREFUeJzs3Xd4VUXCx/HvpHcSSAgk9B5QaqhSpYPooqzrUgRF0HVduyu4rm3Xd+V91bXhigVZQEWxYMOCSFRQqoIKoTcB6S0BElLm/ePcIIS0G3JvbpLf53nuk9zTZu4dkJ8zc+YYay0iIiIi4h1+5V0BERERkapE4UtERETEixS+RERERLxI4UtERETEixS+RERERLxI4UtERETEixS+RKowY8w4Y4w1xjQp77r4AmNMA9f3UdirbTnWTW0lUkkElHcFRER80L+ADwrYvtHbFRGRykfhS0TkfFuttUvLuxIiUjlp2FFEimWMGW2MWWOMyTDGHDTGzDLG1M53zEhjzA/GmHRjzHFjzE/GmBvP2t/RGLPAGHPIGHPKGLPVGPN8EWXWMsZkG2NuLWDfX40xWcaYONf7gcaYb40xx1zlbzDGPFCW30G+8vOGJ282xjxpjNlvjDlpjPnIGNMg37GBxph/GmO2G2NOu37+0xgTmO+4cGPMY8aYLcaYTGPMXmPMO8aY+HzFxxpjXnN9x3uMMc8YY0LOuk6AMeYfruvktddiY0x3T30fIuIe9XyJSJGMMROBacCbwGQgAfgfoLMxpr21Nt31D/ts4BngHpz/sWsBRLuuEQF8BiwHxgFpQAOgW2HlWmv3GmO+AEa7rnu2McCn1toDxphGOEOEbwOPAKeBpkCjC/jYfsaY/P99tNbanHzbJgOrgeuAmjjfy+fGmFbW2izXMf8FrnbtW4zzmf/mqt9IAGNMELAAaAM8BiwFqgEDgRhg31llzgLeAK4EugIPAUeAB1377wXucJWxGogCkoHq7n8NIuIR1lq99NKrir5wgpAFmhSy3x/nH/5F+bZ3d513q+v93cDhIspJdh3f2s36jXKd1/ysbW1d2652vR/heh9VBt9HA9e1CnqlF3DcOsDvrO2XuLaPd72/yPX+oXzl3H/29wFc73p/eQna6uF82z8CNuZ7/255/9nSSy+9Cn9p2FFEitIcp0fntbM3WmsXAzuAXq5NK4AYY8xsY8xlxpjofNfZBBwFprmGMOuWsPz3gHScnq48Y4Bj/DYhfjWQBcwxxowwxtQs4bWL8k+gY75XjwKOe9tam5v3xlq7BNiF0yMF0NP1c3a+8/Le531/A4C91tqCJvnn93G+9z8B9c56vwIYYox51BjT3dWrJiI+ROFLRIqSN1T1awH79ubtt9Z+BfweqIsTmA4YY74wxrR27T8G9AH2AM8DO40xPxtjriqqcGvtSeAdYJRx+AN/BOZaazNcx2zGGZ7zwxmS22uMWWqM6VXYdUtgh7V2Zb7XDwUct6+QbYmu3wv7/vbm218D2F3Cuh3O9z4TCD7r/f/gDEFeDnwDHDLGvGqMiS3h9UXEwxS+RKQoef/Q1ypgX62z9mOtfdta2wtnjtJwoDbwqTHGz7V/tbX2KpzA0RXYArxljLmomDrMwhnm6w70dV131tkHWGsXWWsH4cwx6wdkAx97IXDknwyfty0vSBX2/dXKt/8gvwW2C2KtzbLWTrHWXozzXd0BXAVMLYvri8iFU/gSkaJswOnJuebsjcaYbkB9ICX/CdbadGvtRziT9Gvj9OqcvT/bOss4/B3nv0FJxdRhEc5Q3hjXaztOj855rLWZ1tovgf8FwoGGxVz7Qo3IC5cAxphLgDrAd65NX7t+XpPvvFGunymun58DtYwxw8qyctbavdbal4EvcOafiYgP0N2OIgIwyBizN9+2Y9baBa4lG6YZY2bjzFVKBB7Fmcc1HcAY8whOj88inKHFOsCtwGrr3JF4GTARmAdswwlGt+Lc9fgdRbDW5hpjXgNuBAKBf1trbd5+Y8xNOHOr5gO/ALE4dyHuAX52HdMLWAhcb62dWYLvo5ExpksB2zdaa88e9osE5hljpgFxOIuzbgJmuur+szHmDeAh192T3+L0+v0deMNa+5PrOrOBCcAbxph/Actc1x4IPGWtXV+COuP6rO8Da4Dvce6CbAcMwgnDIuIDFL5EBODZAratBS6y1r5ojDmJs4TE+zgT4OcDf7XWnnAduwwnTP0bZ1hxP05vzt9d+zcBp1zva+OErhVAf2vtrhLUbxbOEgp5v59tDTAYJ/jUxBnKWwyMstaech1jcO7cLGlv/2TXK7/f4yxpkedfQBNgBk6gXATcYn9bZgKcuxS34tzReD9OKJwCPJx3gLU2yxgzAGeu1kTXz0PAEs6f41Wcr131/DMQBuzE6Ql81M3riIiHmLP+B1JERErAtZDqNmCCa1hPRKTENOdLRERExIsUvkRERES8SMOOIiIiIl6kni8RERERL/Lpux1jY2NtgwYNPFrGiRMnCA8P92gZ4j61i+9S2/gmtYvvUtv4Jk+0y6pVqw5aa+OKO86nw1eDBg1YuXKlR8tISUmhd+/eHi1D3Kd28V1qG9+kdvFdahvf5Il2McbsKMlxXh12NMbcYYxZ63qm2xvGmBBvli8iIiJS3rwWvowxiTiLMCZbay/CWfAw/yM3RERERCo1b0+4DwBCXY/ZCMNZ6VlERESkyvDqUhPGmNtwHnFxCvjcWjuqgGMm4jxeg/j4+A5z5szxaJ3S09OJiIjwaBniPrWL71Lb+Ca1i+9S2/gmT7RLnz59Vllrk4s7zmvhyxgTA7wD/AE4CswF3rbWzi7snOTkZKsJ91WT2sV3qW18k9rFd6ltfJOHJtyXKHx5827HfsA2a+0BAGPMu0A3oNDwJSIiUpzjx4+zf/9+srKyij+4HFSrVo3U1NTyrobk4267BAYGUrNmTaKioi64bG+Gr51AF2NMGM6wY1/As91aIiJSqR0/fpx9+/aRmJhIaGgoxpjyrtJ50tLSiIyMLO9qSD7utIu1llOnTrF7926ACw5gXptwb61dBrwNfA/85Cr7RW+VLyIilc/+/ftJTEwkLCzMJ4OXVA7GGMLCwkhMTGT//v0XfD2vLrJqrX0QeNCbZYqISOWVlZVFaGhoeVdDqojQ0NAyGd7Wsx1FRKRCU4+XeEtZ/VlT+BIRERHxIoUvERERES9S+BIREalErrnmGkaMGOHWOV26dOHuu+/2UI0kP69OuBcREanqips3NHbsWGbMmFHq60+bNg13F1CfP38+gYGBpS6zpCZNmsQXX3yBpxdQ93VVOnztPHSSAydzy7saIiJShfz6669nfv/oo4+YMGHCOdsKu3szKyurRAGpWrVqbtepevXqbp8jpVelhx0nzlrJG+tPl3c1RESkCqlVq9aZV3R09HnbqlWrxvr16zHGMHfuXHr16kVISAj//e9/2bdvH3/4wx/OrG120UUX8dprr51z/fzDjl26dOGOO+7gnnvuoXr16tSqVYvJkyef0zuWf9ixVq1aTJkyheuvv57IyEjq1q3LM888c04569at45JLLiEkJISWLVuyYMECAgICuJBnMh88eJBRo0YRExNDWFgYAwcOZMOGDWf2Hzp0iJEjRxIXF0dISAhNmjThP//5z5n9zz77LE2aNCE4OJi4uDgGDx5c6rp4UpXu+YoIDuBEmvceLC4iIp738IdrWbfnuFfLbJkQxYPDWpX5dSdNmsQTTzxBmzZtCA4O5tSpU3Tp0oXJkycTFRXFJ598wtixY6lfvz7du3cv9DrTp0/nnnvuYdmyZSxfvpxrr72WTp06MXz48ELPefzxx/nHP/7Bfffdx7x587jtttvo3r077du3Jzs7myuuuILGjRuzfPlyjh8/zh133EFu7oWNJo0aNYrdu3fz4YcfEhkZyb333svgwYNJTU0lODiYSZMmsWnTJj755BNiY2PZunUrR48eBWDJkiXcddddzJo1i65du3L48GEWLlx4QfXxlKodvkICOHSkvGshIiJSsDvvvJPf/e5352y74447zvz+5z//mQULFjBnzpwiw1f79u25//77AWjatCkvvPACCxcuLDJ8XXbZZdx0000A3H333Tz99NN8+eWXtG/fno8//pgdO3awZMkSatasCcCUKVPo27dvqT/rTz/9xOeff86yZcvo1KkTAK+99hr16tVj7ty5jB49mh07dpCcnExysvPs6gYNGpw5f8eOHURFRTFs2DDCwsKoV68ebdu2LXV9PKlKh6/w4ABOZavnS0SkMvFED1R5yQsZebKzs3n00Ud5++232b17N6dPnyYzM7PY4bXWrVuf8z4hIaHYx+QUdc769etp0KDBmeAF0Llz52I/T1FSU1MJCgqiY8eOZ7bVqFGDpKQk1q1bBzhh85prrmHp0qX079+fyy+//EzoHDJkCP/85z9p0KABAwcOZODAgQwfPpzw8PALqpcnVOk5X5HBAWTklHctRERECpY/ODz66KNMnTqVyZMns2jRIlavXs2QIUM4fbro+cv5J+obY4odIizNOZ6Sd4foFVdcwY4dO7jtttv49ddfGThwIH/6058AiI6OZs2aNbz22mskJCTwyCOP0LJlyzJ5FmNZq9LhK0I9XyIiUoEsXryY4cOHM3LkSNq0aUOjRo3YuHGj1+vRokULduzYwYEDB85sW758+QVdMykpidOnT7NixYoz2w4dOkRqaiotW7Y8s61mzZqMGzeOWbNm8fzzz/Pyyy+fCYWBgYH079+fKVOmsGbNGg4cOMCnn356QfXyhCo/7JiZAzm5Fn8/PRtMRER8W7Nmzfj444/57rvviI6O5sknn2TPnj3Ur1/fq/UYOnQo9erVY+zYsTz22GOkpaUxadIkjDHFrmN26tQpVq9efc62iIgILr74YgYOHMj48eN54YUXiIiIYNKkScTHx/P73/8egPvuu48uXbrQsmVLMjMzmTdvHs2bN8fPz493332XPXv20L17d2JiYvj888/JyMggKSnJY99DaVXpnq/IECd7njidXc41ERERKd7DDz9M69at6d+/P71796ZmzZpur2ZfFgICAnj//fc5evQoHTt25IYbbuCBBx4AICQkpMhz161bR7t27c55jRs3DoDZs2fTunVrhg4dSteuXcnNzeWTTz4hKCgIcHq27r33Xlq3bk3Pnj3Jycnh3XffBSAmJoa5c+dy6aWXkpSUxLPPPsvMmTPPmUPmK4y7q+B6U3JysvXkKrhzlu9k0rs/8e2kS0mILnhROykfKSkp9O7du7yrIQVQ2/imqtouqampPtmzcba0tDQiIyPLuxoet2zZMrp06cLPP/9Mq1a+f9NDadulqD9zxphV1trkAneepUoPO0a4er7SM9XzJSIi4o65c+cSExNDkyZN2LJlC7fffjudOnWqEMGrvFXp8BUe7Hz8tAyFLxEREXccO3aMyZMns2vXLmrUqEHfvn158skny7taFUKVDl+RrvB1Qj1fIiIibrnhhhu44YYbyrsaFVKVnnCvYUcRERHxtqodvlw9X+kadhQREREvUfgC0tTzJSIiIl5SpcNXuOZ8iYiIiJdV6fAV6O9HkJ/mfImIiIj3VOnwBRASYLTUhIiIiHhNlQ9foQEadhQRkYrp5ZdfJjo6utD3BXnsscdo0qRJmZctJVflw1dIgNGwo4iIeM3ll19O3759C9yXmpqKMYbPP/+8VNceNWoUGzduvJDqnSc7OxtjDPPmzfN4WQW5//77adu2rcfL8aYqH75CA7TUhIiIeM/48eNZtGgR27dvP2/fK6+8Qv369enXr1+prh0aGkrNmjUvsIa+V1Zlo/AVYLTUhIiIeM3QoUOJj4/n1VdfPWd7VlYWs2bN4vrrr8fPz/nn+e6776ZZs2aEhobSsGFDJk2aRGZmZqHXLmgo8F//+hfx8fFERkYybtw4Tp48ec7+ZcuW0b9/f2JjY4mKiqJHjx4sX778zP4GDRoAMHz4cIwxZ4YsCyrr+eefp3HjxgQFBdG0aVOmT59+Zl9eD9rLL7/MVVddRXh4OI0bN+aNN94o4TdXsMOHDzNmzBhiYmIICwtjwIABpKamntl/5MgRRo0aRVxcHCEhITRu3JjnnnvunDo3bdqU4OBg4uLiGDRoELm5uRdUp+JU6ccLAYT4wyGFLxGRyuOTSbD3J++WWetiGPxYiQ4NCAhg7NixzJgxgwcffPBM0Prwww85ePAg11133Zljo6KimDFjBgkJCaxdu5Ybb7yR0NBQHnzwwRKV9frrr/PQQw/x3HPP0atXL+bMmcPjjz9+To9VWloaY8eO5ZlnngHg2WefZfDgwWzevJmYmBhWrFhBQkICr776KoMGDSIgoODoMHfuXG6//Xaeeuop+vXrx/z585k4cSK1a9dm8ODBZ457+OGHmTJlClOmTGHatGmMGzeOHj16UKdOnRJ9pvzGjBnDtm3b+OCDD6hWrRqTJ09m0KBBbNiwgZCQEO677z7Wr1/P/PnzqVmzJlu3buXQoUOAEzxvu+02Zs6cSbdu3Thy5AhffvllqerhDvV8ac6XiIh42fjx49m5cydffPHFmW2vvPIKAwYMoG7dume2PfDAA3Tr1o0GDRowdOhQJk2a5FZP0VNPPcX111/PhAkTaNasGQ888ADt27c/55h+/foxevRokpKSSEpKYurUqfj5+fHZZ58BEBcXB0B0dDS1atUiNja2wLIef/xxxo0bx80330yzZs24/fbbueaaa5gyZco5x40bN46RI0fSpEkTHn30UQAWL15c4s90ttTUVObPn8/LL79Mjx49aN26NbNnz+bw4cPMmTMHgB07dtC+fXs6duxI/fr16dOnDyNGjDizLzIykmHDhlG/fn3atm3LnXfeeSYQe4p6vgKM5nyJiFQmJeyBKk9NmzalV69eTJ8+nQEDBrBnzx4+++yzM4Ehz5tvvskzzzzDli1bSE9PJzs7261gkJqayi233HLOtq5duzJ37twz7/ft28ff//53UlJS2LdvHzk5OZw8eZKdO3e69ZlSU1O5+eabz9nWvXt3HnjggXO2tW7d+szvQUFBxMbGsn//frfKOrvMgIAAOnfufGZbTEwMrVq1Yt26dQDcfPPNXH311axYsYL+/fszbNgwevbsCcCgQYNISEigYcOGDBw4kAEDBnDllVcSERFRqvqUVJXv+QoJgNM5uWRm55R3VUREpAoZP3488+bN4/Dhw8yYMYPq1atzxRVXnNm/ePFiRo0axZAhQ/jwww/54YcfeOSRRzh9+nSZ1mP06NH88MMPPPXUU3z77besXr2ahISEMivHGHPO+8DAwPP2e2KOVV65l112GTt27ODOO+9k3759DB48mAkTJgDOsO7q1auZM2cOderU4dFHHyUpKYm9e/eWeX3OVuXDV2iA0zgnMhW+RETEe0aMGEFISAizZ89m+vTpXHvttecEkyVLllC/fn3+9re/0bFjR5o2bVrgHZJFSUpKYunSpedsy/9+8eLF3HrrrQwZMoRWrVoRHh5+Tvjw9/fH39+fnJyi/51MSkpiyZIl5127ZcuWbtXZHUlJSWRnZ7Ns2bIz244ePcratWvPKTcuLo5rr72WmTNn8uKLLzJ9+nSysrIAZw5e3759eeyxx1izZg1Hjx5l/vz5HqszaNiRUNc3kJ6RTfXwoPKtjIiIVBmhoaGMHDmShx56iCNHjjB+/Phz9jdr1oydO3fyxhtv0KlTJz755BPeeustt8q47bbbGD9+PB06dKBHjx689dZbrFq16pwJ982aNWPWrFkkJyeTlpbGPffcQ3Bw8Jn9xhjq1avHwoULueSSSwgODiYmJua8su655x5GjhxJu3bt6NevHx9//DFz5szhww8/dPObOd+pU6dYvXr1OdvCw8NJSkpi6NChTJgwgWnTphEVFcXkyZOpXr06f/jDHwBnnbCOHTvSsmVLsrKyeO+992jatCmBgYG8//777Nixg549exITE8PChQs5efIkSUlJF1znoqjny9XzlZaZVc41ERGRquaGG27gyJEjdOvW7bx/8IcPH84dd9zBrbfeStu2bUlJSeHhhx926/qjRo3i/vvvZ/LkybRv354NGzZw2223nXPMjBkzOHr0KO3atWPkyJHceOON50z6B3jyySdZsGABdevWpWPHjgWWNWLECP7973/z+OOP06pVK6ZOncq0adPOudOxtDZu3Ei7du3OeY0ZMwaAmTNn0r59ey677DK6dOnC6dOn+fTTTwkJCQGceWWTJ0+mTZs2dO/enYyMDN5//33AmR/27rvv0rdvX1q0aMG///1vXn31Vbp27XrBdS6KsdZ6tIALkZycbFeuXOnRMqa+vZD/W5nBWzd2pVPD6h4tS0ouJSWF3r17l3c1pABqG99UVdslNTXV470UFyotLY3IyMjyrobkU9p2KerPnDFmlbU2ubhrqOcrb9hRPV8iIiLiBVU+fIXkDTtquQkRERHxAq+FL2NMc2PM6rNex40xt3ur/ML81vOl8CUiIiKe57W7Ha21G4C2AMYYf2A38J63yi9MyJmlJhS+RERExPPKa9ixL7DFWrujnMo/I9gfjEGr3IuIVFCefgiySJ6y+rNWLnc7GmOmA99ba58rYN9EYCJAfHx8h/yPWihr6enp3LPU0D0xgFFJwcWfIF6Rnp7u8cc7SOmobXxTVW2XsLAwoqOjiY2NJSAg4LzV1H1BTk4O/v7+5V0NyceddrHWkp2dzcGDBzl69CgnT54s8Lg+ffqU6G5Hr4cvY0wQsAdoZa3dV9Sx3lhqIiUlhcnf5dCjaSz/O6KNR8uSkquqt81XBGob31RV2yU3N5eDBw9y7NgxsrN9cwQjIyPjzJpT4jvcbZeAgACqVatGbGxsoc/XLOlSE+Wxwv1gnF6vIoOXN4UHB2jCvYhIBeTn50fNmjXPWbHd16SkpNCuXbvyrobkU57tUh5zvv4IvFEO5RYqIjhAS02IiIiIV3g1fBljwoH+wLveLLc4kSHq+RIRERHv8Gr4staesNbWsNYe82a5xYkIDtBSEyIiIuIVVX6Fe3DN+dKwo4iIiHiBwheuOV/q+RIREREvUPjCmfN1IjOb8ljzTERERKoWhS+cYcdcC6eycsq7KiIiIlLJKXzhDDuCHjEkIiIinqfwhTPsCGjel4iIiHicwhe/9XxpuQkRERHxNIUvnDlfoGFHERER8TyFL37r+dKwo4iIiHiawhe/zflSz5eIiIh4msIXZ835Oq3wJSIiIp6l8MVvc77S1PMlIiIiHqbwBQQH+BHob0jXnC8RERHxMIUvwBhDRHCAlpoQERERj1P4cokICdCEexEREfE4hS+X8KAALTUhIiIiHqfw5RKpni8RERHxAoUvl4jgAC01ISIiIh6n8OUSERKoni8RERHxOIUvl4hgf835EhEREY9T+HKJCNacLxEREfE8hS+XiOBATmXlkJNry7sqIiIiUokpfLmEB/sDaJV7ERER8SiFL5fIEOf5jgpfIiIi4kkKXy4RwYEAesSQiIiIeJTCl0uEq+crTZPuRURExIMUvlwiNOdLREREvEDhyyVv2FHLTYiIiIgnKXy55A07as6XiIiIeJLCl0tEsGvOl8KXiIiIeJDCl0t4kGvOl4YdRURExIMUvlwC/P0IDfQnPTOrvKsiIiIilZjC11kiQgJIz8wp72qIiIhIJabwdZbI4AAtNSEiIiIepfB1lvDgANIzNOwoIiIinqPwdZaI4ABOaNhRREREPEjh6ywRIQFaakJEREQ8yqvhyxgTbYx52xiz3hiTaozp6s3yi+PM+dKwo4iIiHhOgJfLexr41Fo7whgTBIR5ufwiOXO+1PMlIiIinuO18GWMqQb0BMYBWGtPA6e9VX5JRIRozpeIiIh4lrHWeqcgY9oCLwLrgDbAKuA2a+2JfMdNBCYCxMfHd5gzZ45H65Wenk5ERAQAH205zdubsnhpQBiBfsaj5UrRzm4X8S1qG9+kdvFdahvf5Il26dOnzyprbXJxx3lz2DEAaA/8xVq7zBjzNDAJ+PvZB1lrX8QJaSQnJ9vevXt7tFIpKSnklbEzeDtvb1pL+07dqBER7NFypWhnt4v4FrWNb1K7+C61jW8qz3bx5oT7XcAua+0y1/u3ccKYzwgPcrKohh5FRETEU7wWvqy1e4FfjDHNXZv64gxB+oyIECd8pemORxEREfEQb9/t+BfgNdedjluB67xcfpEig52vQ3c8ioiIiKd4NXxZa1cDxU5EKy/heeFLC62KiIiIh2iF+7PkDTsqfImIiIinKHydJVI9XyIiIuJhCl9nOdPzpTlfIiIi4iEKX2cJDfTHz6jnS0RERDxH4essxhjn+Y4KXyIiIuIhCl/5ROrh2iIiIuJBCl/5RISo50tEREQ8R+ErHw07ioiIiCcpfOUTofAlIiIiHqTwlU/18CAOpmeWdzVERESkklL4yicxOpRfj2aQk2vLuyoiIiJSCSl85ZMYE0p2rmV/WkZ5V0VEREQqIYWvfOrEhAGw68ipcq6JiIiIVEYKX/kkRocCsFvhS0RERDxA4SufOjFO+Np15GQ510REREQqI4WvfEIC/YmNCGL3UfV8iYiISNlT+CpAYkyY5nyJiIiIRyh8FaBOdKjmfImIiIhHKHwVIDEmlN1HT2Gt1voSERGRsqXwVYA6MaFkZudyQCvdi4iISBlT+CqAlpsQERERT1H4KoAWWhURERFPUfgqQKJrrS8tNyEiIiJlTeGrABHBAUSHBWqhVRERESlzCl+FSNRyEyIiIuIBCl+FqBMTqjlfIiIiUuYUvgqRGB2mtb5ERESkzCl8FSIxJpSTp3M4cjKrvKsiIiIilYjCVyHqxGitLxERESl7Cl+FyFtoVXc8ioiISFlS+CpEXddCq1rrS0RERMqSwlchokIDiAgO0B2PIiIiUqYUvgphjNFyEyIiIlLmFL6KkBgdqjlfIiIiUqYUvoqQGBOqOV8iIiJSphS+ilAnJpS0jGyOndJaXyIiIlI2FL6KkBjtuuNR875ERESkjCh8FeHMQqsaehQREZEyEuDNwowx24E0IAfIttYme7N8dyXGaKFVERERKVteDV8ufay1B8uhXLfVCA8iJNBPw44iIiL0fcVcAAAgAElEQVRSZjTsWARjjGu5CYUvERERKRvGWuu9wozZBhwBLDDNWvtiAcdMBCYCxMfHd5gzZ45H65Senk5ERESh+59YmUHaactD3UI9Wg85V3HtIuVHbeOb1C6+S23jmzzRLn369FlVkilV3h527G6t3W2MqQksMMast9Z+ffYBrkD2IkBycrLt3bu3RyuUkpJCUWV8fuQnPvnp1yKPkbJXXLtI+VHb+Ca1i+9S2/im8mwXrw47Wmt3u37uB94DOnmz/NJIjA7lyMksTmRml3dVREREpBLwWvgyxoQbYyLzfgcGAD97q/zS0nITIiIiUpa82fMVDyw2xqwBlgMfW2s/9WL5pXImfGnSvYiIiJQBr835stZuBdp4q7yyUifGWeVea32JiIhIWdBSE8WIiwgmyN+PXRp2FBERkTKg8FUMPz9DQnSI1voSERGRMqHwVQKJMaGa8yUiIiJlQuGrBOpEh/HL4ZN4c0FaERERqZwUvkqgdd1qHDpxmi0H0su7KiIiIlLBKXyVQM+mcQB8tbFCPA9cREREfJjCVwnUrR5Go7hwvtp4oLyrIiIiIhWcwlcJ9WoWx7Kth8jIyinvqoiIiEgFpvBVQr2axZGZncuybYfLuyoiIiJSgSl8lVDnhjUICvDjqw0aehQREZHSU/gqodAgfzo3rM5XG/eXd1VERESkAlP4ckOvZnFsOXBCz3kUERGRUlP4ckOvZs6SE19ryQkREREpJYUvNzSpGUFCtRANPYqIiEipKXy5wRhDr+ZxfLv5EFk5ueVdHREREamAFL7c1LNpHGmZ2fyw82h5V0VEREQqIIUvN3VrEou/n9HQo4iIiJSKwpebqoUG0r5etCbdi4iISKkofJVCz6Zx/LT7GAfTM8u7KiIiIlLBKHyVQq/mzpIT32zSavciIiLiHoWvUrgooRrVw4M09CgiIiJuU/gqBT8/Q4+msXy98QC5uba8qyMiIiIViMJXKXVrXINDJ06z/dCJ8q6KiIiIVCAKX6XUKqEaAOt+PV7ONREREZGKROGrlJrGRxDgZ1i3R+FLRERESk7hq5SCA/xpUjOCtQpfIiIi4gaFrwvQMiFKw44iIiLiFoWvC9AqoRoH0jLZn5ZR3lURERGRCuKCwpcxJtQY088YU7+sKlSRtKwdBaB5XyIiIlJiboUvY8wMY8zNrt+DgOXA58AGY8xgD9TPs3Z/T+TxjaU+vWWCK3xp6FFERERKyN2er4HAUtfvlwORQC3gIderYnn/Fhpsn1Pq06uFBlInJlQ9XyIiIlJi7oavGGC/6/dBwDvW2v3AHKBlWVbMKxLbE3V8I9jSr1LfsnaUwpeIiIiUmLvhay9wkTHGH6cX7AvX9gggqywr5hWJHQjMToMj20p9iVYJ1dh26AQnMrPLsGIiIiJSWbkbvqYDbwI/AznAQtf2zsD6MqyXd9RJdn7u/r7Ul2iZEIW1sH5vWhlVSkRERCozt8KXtfYR4HrgRaC7tfa0a1c2MKWM6+Z5cUnk+AXDrpWlvsSZSfd7jpVVrURERKQSC3D3BGvtOwVs+2/ZVMfL/ANIi2xM9O5Vpb5EQrUQosMCdcejiIiIlIi7S01cbYwZcNb7B4wxu4wxnxljapd99TwvLbIZ/LoGsk8Xf3ABjDGadC8iIiIl5u6cr4fyfjHGtAfuA54BAoEnSnIBY4y/MeYHY8xHbpbtEcejmkFOJuxfW+prtEqIYv3eNLJzcsuwZiIiIlIZuRu+6gMbXL8PB+ZZa/8XuBPoW8Jr3AakulmuxxyPaur8coHzvjKzc9l68EQZ1UpEREQqK3fDVwbOwqrghK28pSaOnbW9UMaYOsBQ4GU3y/WYzOA4CI+7sDsea1cD9JghERERKZ67E+6/AZ4wxiwGkoERru3NgF9KcP5TwF8pIqgZYyYCEwHi4+NJSUlxs4ruST9xgoMhDQjd9DUrSllWTq4lwA8+XfYz0cc2lW0Fq6j09HSPt72UjtrGN6ldfJfaxjeVZ7u4G75uAf6DE7pustbucW0fDHxW1InGmMuA/dbaVcaY3oUdZ619EWcpC5KTk23v3oUeWiZSUlKIbT0QFj1K7y7tIKRaqa7Tct1i0gIC6N27SxnXsGpKSUnB020vpaO28U1qF9+ltvFN5dkuboUva+0uYFgB228vwemXAJcbY4YAIUCUMWa2tXa0O3XwiDodAAt7foBGvUt1iVYJUXz6816stRhjyrJ2IiIiUom4O+cLAGPMpcaYW4wxfzbG9CnJOdbaydbaOtbaBsA1wJc+EbwAEto5Py9k0n3tKI6czOLXYxllVCkRERGpjNzq+TLGJALvAR2AvCHHBGPMSmD4WcOQFUtoDNRocsGPGQJn0n1CdGhZ1UxEREQqGXd7vp7BeaZjE2ttXWttXaCpa9szJb2ItTbFWnuZm2V7VmIy7F4J1pbq9Ba1ojAGrXQvIiIiRXI3fPUH/myt3Za3wVq7FbjVta/iSuwA6fvg+O5SnR4eHEDDGuGs1TMeRUREpAilmfNVUNdQ6bqLfEmdDs7PC3jOY1JClHq+REREpEjuhq+FwLPGmLp5G4wx9XDW7/qyLCvmdfEXgX/QBU26b5UQxS+HT3HsVFYZVkxEREQqE3fD161AOLDVGLPDGLMD2AKEAX8p68p5VUAw1Lr4gibdd6gXA8D8n34tq1qJiIhIJeNW+LLW/gK0B4YAj7teg4GrgCfLvHbelpjsrPWVm1Oq0zs1rE7rOtX4T8oWPWRbRERECuT2nC/rWGCtfdb1+gKohhPAKrbEDpB1Ag6sL9Xpxhj+3KcJOw+f5KMf1fslIiIi5yvVIquVVp1k5+cFzPvqnxRP8/hIpi7aTG5uxb8PQURERMqWwtfZqjdynu14AXc8+vkZbu7TmE370/l83d4yrJyIiIhUBgpfZzMG6nWF1a/DuxNLHcIua51Aw9hwnlu0GVvKRVtFRESkcirR44WMMR8Uc0hUGdTFNwx7GhY/BT/Mhh/fdCbhd74JWl4BAUEluoS/n+FPvRrz13d+JGXjAfo0r+nhSouIiEhFUdKer0PFvLYBMz1RQa+LrAWDH4O7UmHw/0HGUXj3Bnilv1uPHvpdu0QSo0N57kv1fomIiMhvStTzZa29ztMV8TnBkdB5InS8ARY/CV/+A/anQnzLEp0eFODHjb0a8cD7a1m69TBdG9fwcIVFRESkItCcr+L4+UHbkc7vmxe4derVyXWJiwxm6qLNHqiYiIiIVEQKXyURlQA1W8Em98JXSKA/E3o0ZPHmg3y/84iHKiciIiIVicJXSTXtBzuXQmaaW6eN6lyf6LBAnl+0xUMVExERkYpE4aukmvSH3CzY9rVbp4UHB3Bdt4Z8kbqP9XuPe6hyIiIiUlEofJVU3c4QFOH20CPA2G71CQ/yV++XiIiIKHyVWEAQNOoNmxe6teQEQHRYEKO71OejH/ew/eAJj1RPREREKgaFL3c06QfHdsLBjW6fOr57QwL8/Zj2tXq/REREqjKFL3c06ef8LMXQY82oEK5OrsPbq3ax91hGGVdMREREKgqFL3dE14W4FrD5i1KdfmPPxuRaeOmbrWVcMREREakoFL7c1aQf7FgCp92fu1W3ehhXtE3g9WU7OXzitAcqJyIiIr5O4ctdTfpBzmnYvrhUp9/cuzEZ2Tm8umRbGVdMREREKgKFL3fV7waBYaWa9wXQpGYkA1vWYsa320nLyCrjyomIiIivU/hyV0AwNOzpPOfRzSUn8vy5TxPSMrK54rklvLF8JxlZOWVcSREREfFVCl+l0aQfHNkOh0s3cf7iOtV4cUwHwoL9mfzuT3SfsoipizZz7KR6wkRERCo7ha/SuIAlJ/IMaFWLD2/pzus3dKZlQhT/99kGuj22kM/X7i2jSoqIiIgvUvgqjeoNoUYTZ+jxAhhj6NYklpnXd2L+rT2IjwrhuUWby6iSIiIi4osUvkqrSX/Y9k2p1/zKr2VCFNd0qsuPu46xTY8gEhERqbQUvkqry5+cHrDZV8FHd0Bm+gVfclibBIyBD1bvKYMKioiIiC9S+CqtmPow8SvoegusfBVe6A47l17QJWtXC6Vjg+p8sGY3tpR3UoqIiIhvU/i6EIEhMPBRGPcx2Bx4dTAseBByS790xBVtE9hy4ATrfj1ehhUVERERX6HwVRYaXAJ/+hbajoIlT8EPs0t9qSEX1SbAz2joUUREpJJS+CorwZFw+bNQtzMsehQy00p1mZjwIHo2i+PDNXvIzdXQo4iISGWj8FWWjIGB/wPp+2DJ06W+zOVtEthzLIOVO46UYeVERETEFyh8lbU6yXDx7+HbZ+HYrlJdon/LeEIC/fhgze4yrpyIiIiUN4UvT+j7gPPcx4WPFH7M6ROFPhsyPDiAfknxfPzjr2Tl5HqokiIiIlIeFL48IboedP0z/Pgm7F517j5r4dvn4LF6sOrVQi9xRdtEjpzMYvHmgx6urIiIiHiT18KXMSbEGLPcGLPGGLPWGPOwt8ouF93vgPA4+Oxvv/VwnTwMc0bC538D4w/LXiy096tns1iiQgL4UHc9ioiIVCre7PnKBC611rYB2gKDjDFdvFi+d4VEQZ+/wc7vIPVD+GUFTOvpPIx70BQYPAUOpMLu7ws8PTjAn8EX1eaztXs5dbr064aJiIiIb/Fa+LKOvGfwBLpelXsthXZjoGZL+Oh2eHWQczfk+M+gy01w0VUQEAo/zCr09CvaJnDidA5frt/vxUqLiIiIJxlvPsbGGOMPrAKaAFOttfcWcMxEYCJAfHx8hzlz5ni0Tunp6URERHjs+jGHV9P6x4c4GNuZDc3/Qnbgb2W1SH2K2IPL+LbbDHL9g887N9da7kw5RWSQ4d6OIUQEGY/V09d4ul2k9NQ2vknt4rvUNr7JE+3Sp0+fVdba5OKO82r4OlOoMdHAe8BfrLU/F3ZccnKyXblypUfrkpKSQu/evT1aBmn7IKKm0/N1tu1LYMYQGD4N2lxTcP027GfizFU0rxXJ7PGdqRYW6Nm6+givtIuUitrGN6ldfJfaxjd5ol2MMSUKX+Vyt6O19iiwCBhUHuV7XWT8+cELoH43qN4Ivi986LF385pMG9OBDXvTGDN9GcdOZXmwoiIiIuJp3rzbMc7V44UxJhToD6z3Vvk+yRjneZA7FsOhLYUe1qdFTf4zuj2pvx7n2leWcTxDAUxERKSi8mbPV21gkTHmR2AFsMBa+5EXy/dNbUeC8YPVrxd5WN+keP4zqgPrfj3Ota8sVwATERGpoLx5t+OP1tp21trW1tqLrLVFLP9ehUQlQJN+TvjKLXpJiX4t45k6sj0/7z7GuOnLOZGZ7aVKioiISFnRCve+oN1oSNsDW74s9tABrWrx3Mh2rNl1jImzVpKRpTXAREREKhKFL1/QbDCE1Shyza+zDbqoNv83ojVLNh/ilte/1/MfRUREKhCFL18QEAStr4H18+FEyZ7leGX7OvzjilZ8kbqfu95aQ05u5V6vVkREpLJQ+PIV7UZDbhZ88wTklqwna0zXBtw7qAUfrNnD/fN+ojzWbBMRERH3BJR3BcQlvqUTwJY+D0e2w+/+A6HRxZ72p96NSc/MYuqiLUSFBDJ5SJLn6yoiIiKlpp4vX3L5czD4f2HT5/Bib/j1xxKddveA5vyxUz2mfb2VNb8c9WwdRURE5IIofPkSY6DzjXDdJ5CdCa/0hx9eK8FphvuGtKBGeBCPzk/V8KOIiIgPU/jyRXU7wY1fQ52O8P7N8PX/FXtKZEggt/dryvJth/kidb8XKikiIiKlofDlqyLiYMw8aD4UljwDmWnFnnJNp3o0igvnX5+kavkJERERH6Xw5cv8A6DHnZB5HH58s9jDA/39mDSoBVsPnGDOil+8UEERERFxl8KXr0vsAAntYPlLUIK5XP1bxtOpQXWe/mIj6Xr8kIiIiM9R+PJ1xkCniXBgPWz/pgSHG+4bmsTB9NNM+2pL6cvNzYVF/4LD20p/DRERETmPwldF0OpKCK0Oy18s0eFt60YzrE0CL32zlb3HMkpX5r6f4avHYNkLpTtfRERECqTwVREEhkD7a2H9x3C0ZHO5/jqwObm58MTnG0pX5s6lzs+Nn5VouFNERERKRuGroki+3vm56tUSHV63ehhju9Vn7qpdzPtht/vl/eIKX0e2waELGL4UERGRcyh8VRQx9aHZYFg1A7JKNpR414DmdG1Ug7vmruGLdftKXpa1sOM7Z50xgE2fuV9fERERKZDCV0XSaQKcPATr5pXo8JBAf14am8xFCVHc/Pr3fLvlIORkw751RZ947BdI2wMXXw1xLZzHHYmIiEiZUPiqSBr1hhpNSzzxHiAiOIAZ13WiQY0wJvx3BYffmAj/6Qq/rCj8pJ3LnJ/1ukDT/rB9CWSmX1DVRURExKHwVZHkLTuxe5Xzys2Fw1sh9UNImQLfTXV6tvKJCQ9i1vjO3B30HtU3v+NsTP2g8HJ2fgdBkRDfCpoOhNws2Jrimc8kIiJSxQSUdwXETW2ugYUPwxt/dHqjsk64dhjAOkOEI16FsOrnnBa/5R2uy36Tj/z6EJt7iEYr5/Fh2PUk1Y6kZe0oosOCfjv4l2VQtyP4+Tu9X0GRznWTLvPaxxQREamsFL4qmpAo6PM32DDf6ZnKe8W1gLXvwUd3wEt94I9zoGaSc87WFPjwVmjUmxb9p7P0rcfpcmQqr338BVttAgANaoTxyriONI7Ihn1rIely51z/QGjcBzYtcCbiG1MuH1tERKSy0LBjRdT1Zhj3EQye4qz/ldgBgsKh3WgY9zFknYKX+znrgu1bB2+OgdhmcPVMmtSuzuixfwLgg/7HmXl9JyYNbsGvxzKYsWQ77FoBWKfHK0+zgc4E/H0/l8vHFRERqUwUviqbup1gYooTtuaMhBlDIDAMRs2FkGrOMdF1odbFRGz7nJ7N4ripV2MGtqrFB2v2kL39OzD+UCf5t2s26e/83KglJ0RERC6UwldlFJUA182HNn90hgpHvQXV6px7TPOhztyuEwcBGNGhDsdOZXFsw9dQu7XTk5YnMh5qt3WGHkVEROSCKHxVVoGhMPwF+OtWqN3m/P3NBwMWNn4KwCVNYqkb5U/EoR+hXtfzj286AHYth5OHPVtvERGRSk7hq7Lz8y94e+02EFUHNnwCgL+f4cZm6QTbTI7Gtj//+KYDwObCli89WFkREZHKT+GrqjLG6f3a8qUzQR8YHLUdgA8O1z3/+MT2EFZDq92LiIhcIIWvqqz5YMg6CVu/AqDG4R/Y61+LmT9nYq0991g/f2yTvmSkfsbPv2joUUREpLQUvqqyBj2cBVQ3fOxMzN+5lFO1OrF5fzqrfzl63uFfZLUjJOsoz81+i4ysnHKosIiISMWn8FWVBQRB036w4VM4tBlOHqTWxb0ICfTj7VW7zjk0ZcN+7lldgxz8uPLkW0z/an05VVpERKRiU/iq6poPhRP74bvnAAht1J1BrjW/8nq3th88wa1v/ECt+Nrk9H2IAf6r6PDNDezZu7fw6+5cemY4U0RERH6j8FXVNe3nLKr6/SwIiYbYZozoUJe0jGwWrNvHicxsJs5aiZ+f4aVrkwnqcRuHBk6lHRsw0wfA0V/Ovd7hrfDmaJg+EGZeDu9M0PIUIiIiZ1H4qupCY6DBJWBznEcK+fnRrXENEqqF8NbKX7jn7TVs3p/Oc39sT93qYQDU6Dqaj1pPJTzzAKenXQq/roGM47DgAZjaGTZ/CZfeD73uhbXvwtROsHZeOX9QERER36AHaws0HwLbvoa6nQHw8zNc1aEOz365GYC/DUmie9PYc04ZcvnVTNiUyRMZ/yDu1SGYwFA4cQDajoJL/w5RtZ0Dk4bB+3+GuWPh52Ew5AlnxXwREZEqSj1fAq2udIJX0uVnNl3Vvg6B/oYr2iZwQ4+G550SEujPtZcP5rJTD3MgtBHEtXCeKfm7538LXgC1LoYbvoS+D8LGz+GZdjD/r3B4m+c/V1GsdXrjXIvMioiIeIt6vsTpiRp/7uKpDWLDSbmnD7WiQjDGFHhav6SazG7WjL477ufLG3oTFxlc8PX9A6DHnU4v2NePw8pXYMVL0GIodP2L8zDw3Gw4sh0OboJDm2i0ZTU0DHaGQgspv9Qy0+Dju+DHN52lNu5aD8ERZVuGiIhIIRS+pFCJ0aFF7jfG8OCwlgx86mtGvrSU0V3qc3mbBGLCgwo+IbYpXDkN+j0Iy1+CldMh9UOITHDuuMzNPnNoHeMPr74L1RtBm5HQ5hqILmDlfXftWQ1vX+cEvTYjYc3rTgjrOP7Cry0iIlICXht2NMbUNcYsMsasM8asNcbc5q2yxXMaxUXw9DXtCArw48EP1tL5fxZy82ur+HL9PrJzcgs+KSoB+j1I5q0/8Vn9u0jJaMyxdjfB7/4DNyyEe3ew5JLX4HcvQFQiLPonPHUx/Pdy2LWydBW1Fpa+AK/0h6wMGPuRM0RaqzWseNnZLyIi4gXe7PnKBu6y1n5vjIkEVhljFlhr13mxDuIBQy6uzZCLa7Nuz3HeXrWLeat3M/+nvdSJCeWuAc24ok0ifn7nDh1u2pfGrXPWkPprB0IDO1FjXRBv9+xGrWohAOQEhELbwdD2j3BkB6yZ4/SUvdwPOk1wJvWHRJWsgtmn4d0bYN370GywE7rCqjv7Ok2AD/4CO7+D+t3K8msREREpkNd6vqy1v1prv3f9ngakAoneKl88r2VCFA8Ma8nSyX15YXQHqoUGcsebaxjyzDcsWr8fay3WWmZ9t53Lnl3M/uMZvDI2mTkTu3D0ZBajX1nGofTM8y8cUx963wu3rHDC0vKXnCUtUj8qvlLZmfDWtU7w6v8I/PGN34IXwEUjIKSac00REREvMOc9QNkbhRrTAPgauMhaezzfvonARID4+PgOc+bM8Whd0tPTiYjQZGtPyLWW5XtzeHfTafaftDSP8SPY3/DjwRwujvVn/MVBRAc7+X/94RyeWJlBQoQf93YMITfzRKHtEnl8A803TCXixA4OxHZmS+PxZISev3yFyc2i1dopxB5awcamN7EncXCB12u8+RUSd3/M0i4vczq4eoHHyG/0d8Y3qV18l9rGN3miXfr06bPKWptc3HFeD1/GmAjgK+BRa+27RR2bnJxsV64s5RyfEkpJSaF3794eLaOqy8rJZc6KX3j6i00cz8jivsEtGNutwXl3US5av58JM1fSvl4M45tmMLBvn8IvmpPlPBIpZYozUb/jeOhxN0TEuQrNgLfGwKbP4bJ/Q/L1hV/r0BZ4tj30+Rv0+uv5+4/scB6X1Prqsr/zsgLS3xnfpHbxXWob3+SJdjHGlCh8efVuR2NMIPAO8FpxwUsqj0B/P8Z0qc+I9nVIy8iiZlRIgcf1aVGTf/+hLbfO+YGTaf40b3OCBrHhBV/UPxC63wEXXw1fPQbLX3QekdTtFug0Ed67CTYvgMueguTriq5gjcbQ+FJY+Sp0v9NZGiPP4a3w6lBI2+OUedGVpfwWREREHN6829EArwCp1tonvVWu+I7QIP9Cg1eeYW0S+Nfwi1l3OIfej6fw+xe+5c0VO0nLyCr4hGqJcPmzcPMy5zmVX02Bx5s5wWvY08UHrzwdJzgBa8P837Yd2Q4zhkF2BtRsCZ/cC6eOlOx6IiIihfDmCveXAGOAS40xq12vIV4sXyqIazrV44leodw7qAWHTpzm3nd+ouOjX3DXW2s4XlgIi2sGV8+ECYsg6TJnmYoO40peaLOBUK2us/grOA8M/+8wOJ0O174Pw6fByUPO8ysrm9wc507S9P3FHJcLMy6j4dZZ3qmXiEgl5bVhR2vtYkATZqREYkL8GN67MTf1asTqX44yd9Uu3li+k1rVgrlnYIvCT0xs74Qwd/n5O71kCx+BLYvgo9vh1DEY+z7Ubu0c0+0WWPI0tP4DNOhesutmpsHqNyDjqGu+mHF+Gj+o0QTqdYXw2PPPyzoF25fAloWQc9opr0FPCK9x7nHWwtGdsGuFU8ZFVzkPS3fH9/+Fj+5wPvcfighW696D7d+Q6B8Gp09CUJh75YiICKAV7sXHGWNoVy+GdvViOHLiNDO/28FNvRoTGRJY9oW1Hwspj8Gs4RAcCWPmQUK73/b3muQ8D/LD2+GmxRBYxBBqThasmuFc7+TBosuNbQ71u0K9bk7v2uYvYMcSZ7gzIAT8ApyFYAHiL4aGPSGiphO4dq2A9H2/XWvBg06PX5c/QbU6xX/mk4dh4T8gMBxSP4Cdy6Be5wI+TzZ8+SiExhBw6gis/xha/77464uIyHkUvqTC+FPvxnzy815eX7aTG3s1LvsCwmOdOxrXzoPR70CdDufuDwpz7pycfSUsfhL63Hf+Nax1QswXD8PhLVC/u7O+WO02gAWb6xyTmwX71joha8d38PO7TlgDqNHUuTuzSV+of4kTvvashm0psO1rJ4jlZEJMQ2jUG+p0dF4A302Fpf+BZS/Axb+HbrdCfMvCP3PKv5wes+s/gzdHw4K/O7/nv6tzzevO5/nDa5x6/w5CV892P3yl7XUeqF6/q3vniYhUMgpfUmG0rhNN9yaxvLx4G2O7NSAk0L/sCxn6bxjwz8KH7pr0dYYdv3kSWl0JNVs4vUK/roEdi53FXHevgrgWMPItaDqgkOUpQpyHhtfrAj1w5l3tX+f0uMU0OP/wuh2dV897nGU0sk6eu1hsnqtegr5/h++ed4YTf3wTrnwJLh5x/rH71jpBLnm883DzPvfBh7fB+o+ch6DnycpwlvRITIYWQ9m77CMabp3jzIsryfM2c3Oduix4ADKPww1fnh9sRUSqEG9OuBe5YH/q3ZgDaZm8+/1uzxQQEFT8nKmB/wPBEc4DumddCVPqw8uXusLF/7d33+FRVtkDx793JpPeK5CQBELovYN0EGkK0qwIKov+bFhX1nUta6/rWlBZQUFRBHRdEVBAAekQqkCAhBoghAAhhfTM/f1xQ0sjIJkk5M9OvsEAACAASURBVHyeJ08yM2+5My9vcrjlnHS48X24f5WZxF/evGAWK9RqUXLgVZTNteTA6yzfcBj4Ojy2w8wp+36CKWB+Ia3N6k1Xn/M9eK3vNEHjkhfMsOlZMdMg7TD0fQ6UIimkN6BhWzkSIJ+INwsXfnrU9P55BMMvz0gtTSFEjSbBl6hWukYF0DLMh09/30uBvZL+gHsEwsA34XgspB2FVrfCyM/hiT2mBFK7sRfnCqss7v5w+7dm3tqcuyFuyfnXdv4AB1aYGplnAzmrE/R7EU7Gnx8CzUmHFe9AvZ5QvyeAqSYQ2R22fF16EFWQZ3oHP+4KSX/ATR/C2HnQ5++QsNb0EF6OgjyTx+3MycvbTwghqiAJvkS1opTigV5RHDyZycLtiSVus/FgChsOnCIzN7/iGtJyNDybBA+uhcHvmOSrXsVLHFU6Fy8zfy24CXx7h5kzlpsJi/5hJu8XTcfR8AYzT23Z6ybwWvuJWTDQt0iKjdZ3mAS0h9YWP6fWMPce+PVFc7wH10PbMaYXsM0YCG4GS543dTfLIycdvr4FfnwIfvg/6TUTQlR7EnyJaqd/01rUD/Jg8tK9XFgeKzM3n6fnbmPEx6sZ9ckamj//C/3/tZwn52xlxpoDJRft/jOcXK7u8SqKm69ZuelXD76+1QQwqQkw6E0z3HkhpaD/P03AteRFWP0+NBoMYUWqZTS9CZw9YctXxc+35iOz6OD6f5rUFV61zr9msUL/l0wC2/VTLt329CT4YjDsW2baEfcLbJt9uZ+AEEJUKRJ8iWrHYlHc3yOKnYlp/B5n0jhsP5LKkA9WMntjAg/0imLq2PY83CeaUF83lu46znP/28E902OojELyVYJHgEkW61XLDDk2HwERXUveNrSdeX3Df0yvU59ni2/j7AHNhpmVoblnzj9/aK2Z+9bkJrPSsiQN+kKD62H5W2UPI56Ih6n94EQc3DbLBHJhHWHhX01QJoQQ1ZQEX6JaGtYmlFrerkxeGs9nK/YxfPJqzuTkM3N8J/46oDF9m4Tw2PUN+fzujsQ824+XhzVna8Jpftt1iSzu1zKvEBj7I3S8zywaKEuff4DVxazsLC1VRes7TAWAnT+axxnJZm6ZXwQM/bDsxQb9Xzb7Ln+95NcTNsDU680Q6bifoGF/02s29COTgHbBE5U7/Ji0E2aOMsO4QghxmST4EtWSs5OF8d3rsW7/KV6eH0vPRkH8PLEHXaOKZ4tXSnFLh7qE+7vz7uI9Nbf3C0zi1UFvXjwUWBL/evBwjAmiShPexQxlbplpUmV8Px6yTsGo6WYVZVmCG5uFCRumQvIe81xuplmV+d14mD7EDJeOX2x64s4Kagi9/2a22/lD+d7zpdjtcDjGJJv9+Dp4v43Jq1aalAMmEW/cIpgxFH5/yxyjMiVuI/zgbMjPrdx2CCHKpQosyRLiytzWMZw1e0/Sq1EQd3aOQJXR02KzWnikbzRPztnKop1J3NCs5OAjLimd33YdJyLAgwbBnkQEuGOz1tD/o/iGl/26Uqb3a+nLJj/YvmWmyPnZckyX0usZ2DYHfnzYBINxiwrzlwWYFaS9nwXPoOL7dXnYrJac/2TpJZfK6nXLz4UTe+DYHyY3255FcOY4KKvJu5Z12qTHuG0WRF538b7pSTBjmKk+MP5Xk8z2t5dNZYDhU4qnADmdYD6XqN7lqzhwufJzYcXbsOId6tvzYW00dHvs6p9HCHFVSfAlqi0PFyemjutQ7u2Hta7D5KXx/GvxHq5vEoLFcvEf6MMpmdw6ZS0nz5zvPbBZFZEBHrSP9Of5G5tWTGLX6qzVrbD0Fdj8JbS63axmLC/PIOjxpFn56BEMrW6DpkNNVv+yUnVYnczw46c9zfyv4f+B5FjYv8Kkzzi4ygQlnsHgGWKGWz1DzNy0Y9vg+C5TYQDAxcfMQWs0EBr0M8FT6hH4cpipZDBqOjQaYLbNOg1fjTDlnO760SxCCP2P6QH8eRJ80h1GfWHat3sh7F5gAjwwpaPuXlD+vG/lkbgNfnjApPJoeQsnD8cTsPxNaD6yfMlvhRCVRoIvUWM4WS1M7BfNxFlbWLj9GINb1j73WmZuPhNmbCQ3385PD3fDrjVxSRnEJ2cQl5TBrA2HSE7P5uM729XcnrCS+NY12fBPHzIpNy43uOj6iKkCENSo+MrLsoQ0M9n+l71qio9npRS2JwIaDwZXX1POKCPJ5GPbu8wkpw1pDl36moS2tVqAf1TxQM8nFO7+GWaOgFm3w82fQOMh8M2tkLzL5E6rWxj0KwUd7jUF3WePNQsEwBROr9sJrn/JlIL67WWzSrPVLZf3+ZTkgt4u3APg1m+g8SDiFn5LwMaJ8Mvf4JYSVqE6kt0OOamXX+T9rJwMk8hYiGuUBF+iRhnSsg4f/BbPe0v2MKB5LawWhdaap+ZsI/ZYGtPGdqB5qJmv1DLM99x+X609yLM/bOfpudt4e1SrYr1mNdrIz02wYbmCoNRiKbv2ZFm6PQYn40zty8juUK/7pYdKy8sjwCSF/eY2+P4vJjfZ8Z0wcprpKSuqThu4bzmsm2IC0uj+JhkvmEBk98+w6FmT98zNt/j+AGdOmCHTkoZaz8rPNTU4434xiyEGvH5uqDPbLQR6PGECvbglEN3vT34IV6ggH2aPMcOt9/xcWNf0MsT+BHPGmtqkQ94ru4C9ENWUBF+iRrFaFI/2i+ahrzfz07ajDG0dykdL45n/RyKTBjamd+PgEve7s3MEKWdyeWfxHvw8nHl2cJMy55jVKJWVzd/JGUZ8VnHHd/GCO+aahLG755ui6s2Hl769mx/0err48xaL6RWc0ssUMh/4RvFtTsTD5wOhIBdu+6bkNCAF+SYQjPvFHK/D+OLbdH0Ets6ChU9B5JqrG7jkZZlSU86e0Paukns5tYafJpohVxcfmHUnTFhWfF5eafb/bsp2edWBrd+Yagu3fHXpBSJXS9pRiPncDFHb80xlBXse2Nyh+xOOa4e45knwJWqcQc1r07hWPP9eEoeLk5W3F+1hWOs63Nejfpn7PdSnAacyc5m6cj/+Hs482LuBg1osKo3N1eQXO30Q/Mv+91GmOq3N8OT6KdDmTjPkeVbKAZhxE2i76S2bMRSGfXxxMXS73SxM2PkD9H+l5MALTOLfQW+Z1Zir34eef73yNp87d4Ep0P7by5BWWFP14Gq48T2wuV287W8vw+avoMdfoeEAE1DOvRvu/P7SQfqRTaan0T/KzI87sAL+ez9M6Q23fW16FytS6mGT0Pf0IRNsWZzAagOLDTJPQtxiuOuH8tVfFeISZPKKqHEsFsWj/Rqy78QZHpi5kZZhPrw+ouUle7KUUvxjcFNubhPKW7/s5ut1hxzUYlGpLNY/F3id1edZ0zs2/8nzqSlSD5uVlblnzB/2exdDaHv47l4zp0tr87XgSdj6NfT+O3R9qOzzRPWBpsPM/ikHzj9/It48N3M07Pnl0u3V2gxfftrDVEXwDDFDsb2fNUXVpw0wqznPWvepmYvWdqwp1h7WDoa8C/uXw68vlH2u5D0wc6QZQh3zvfnedCjc84v5/KcNhO3fXbrN+bmmdNayNy4vD1zqEfhiCGSeMtfgmSMw6SA8FQ9PxJpgMCvFvOfju8p/XCFKIT1foka6oVkILcN8SEzNZsqY9uVexWixKN4c2ZLUrDz+/sMf+LrbGNSi9qV3FMLNz5Rc+t+DJniJ6mMCr6zTpvrA2d6wu34w2/z6Tzi13+RMi5kK1000iwzK44ZXTU/NT4+Z1Zg7/wdJ281r7gEQv9hs0+n+kocPUw6afff+anp6Rn4OzW4229brYdr6/V/MUOro6ZBxHBY+bRYmDH73/DHb3AlHN8PqD6B264t78846nWBWlyqLKYPlXef8a7Vbwl+Wmnluc++Bg2ug3/NmSLio9CSYfZcp3A6Qk2aS+V5qekDaUXMdzpyAMf8tXkoLzHN3LzQ9ip8PNPVSQ9uWfVwwx0zehS037dLbihpFgi9RIyml+Gp8J7QdfNxtl7WvzWrho9vbMmbqOh6dtQUfNxvXNSie3FWIYlrdDhunm94Zj3+bgGHMfy/+Q+7kYtJn+EWaBK4AHSdAvxfLv5rUJ9TMP1v8HOxdavKXDXjdrEx184PvJ5j0GCfiYOCb54cE7XYT6C1+3pzrhtfMEKeT88XHbzQA/vKbWQ06YyigTJA34rPiw4s3vAZJO+B/D5lVrcHNIO2w6e06sdvMI8tJh3HzISCq+HvxDDI9boufM3nVdi80PWoNbzi/zZGNZn5ZVgqMmAoJ62DNh2B1NkXhS/vc0hJN4JVReB3qlpG6JqQp3LPQ5HmbfpOZm1evu+m1TEuE9KOmBy051rzfY9sh4xgAHWw+EOVpAm4hkOBL1GDerpcXdF3IzdnK1HEduOXTNUyYEcPXf+lMq7qlrGIT4iyLBQa/bXqMcjPMhP7wTsW3U8oMUwY2MkOH3Z+4/DQeXR6CwIZmrlTRieKjvzRDgav+DSn7TX6yzFNmXtmBFVC/N9z0ftmrRwOjTaLZHx82c8Fu+7r4HDAwgduo6eY9fz4I7Pkmme5ZXrVN+o6ykvM6OcPA182Chx8fhq9Hm3xmA143vXjzHjXDouMXm1655iPMZPmV75p5W72fKX7M5D0meEw/Znqy6nYs48Ms5F/frOD88mbzZXM3KTUuZLFBUGOTWDekGfhFkjfvGZy/HG7m4PV8+vLSqohrkgRfQlwhHzcbM+7pyIhPVjPu8/XMub8rDYKvPDdRfoGdpPQcjqdlczw9h+PpOSSn59CsjnepGflFNVS7FYyeAZ61yu5pAWg56srPY7Ga5LElvmYxQ6ABDczw4qc9Te+PxQlufL/01YxFuXqbYcdL8QqB22fByn+Z9x3U0ASWQY3Op+Qoj7od4b4V5jgr3ja9YHlnTKqRUdPPr6pUygx/2vNh+RsmIOr5lOlp3PG9ybl2dBPYPEzgFd65/G3wrgPjFpyvS+pV2zx39rtvRLGewo1HbfRI+59py6G1pofQs+SV1VdN2lE4sMoM0fqGmxQoJQ3XXo4T8WbxSe3W5V/Berm0NsPdednmPw2ehUmSz36mWpuVtzlppsc0NcG068Qek3rmRDygTVmyuh0hrKO556pYyhIJvoT4E4K9Xfnynk6M/GQNd01dx9z/60od3xL+938Jq+NP8NTcbRw5nVXi6w/0iuLJ/o0kv9i1osmNld0Co+1dJliYM9ZUFrjxvYopgwTmD+CoL/78cZyczZBqs2HwyzMQ3BT6Pl98uNNiMYGkPd+UwNq9ABK3mFWltVqa+WAtRl1Z+giPALOqtJzsVle4+WOTQmTBk/BJNzPnrkHfK09EW+wkBWbF6J6fTTqSs9UVLuTqawKx6P6mtmp58+Id3WwWa8T+BBQuZPANhzptzZC5b4RZEXrmhCnVdSYZstPAydX0htrczXd3f1MFo7QKDBnJpmdzz8Lir7kVlu7KSTPXtChnL9MbG3md+SwOr4fYH81rFptZcTxuvhnWrwIk+BLiT4oM9GD6PR249dO13PHZOrpEBZCTZyc7v4CcvALyCjTdowMZ1b4uPm4XD3Vm5Rbwxs+7+GL1AeoHevDKzc2p5e1KsJcrwd4u+LjZeHHeDiYv28uhU5m8PaqVlDgSV1f9nvBkfOXla7tSQY1Mr1VZLBZTikpZ4NAa6PY4tBxt9q0MbceYYeA5Y82KVpQJCur1hPq9TLvyssyQdG6mmU+Wm2F6eM5+P9vjk5128c/piZB9+nyN0n4vmqHP/FxIPWQWNpw+ZHqHVr5rgqno/tD+bvO96FCo1ialyIq3Ye9vJm9b9ycgspsp03Vkown2iha4d/Mz5cJcvU1AlpdlhpnzMiE7FVa+B50mmGtxYS3UPb+YhSbZaWaeYEQX01OZcez8d5Q5rot34Xcf06sa2ND0jhXtrU1PgsMbTCCWeqTKBF4gwZcQV0WzOj58NrY9j8/eyqIdx3BxsuJqs+DiZCXfbufl+bG8u3gPw9uGMrZLJNEhXmw+lMITs7ey78QZxnWN5OkBjXFzLh5YvXpzCyICPHh94S6OpWYz5a72+Hs4l9AKIa5QdQu8LofFCsMmV3YrzqvVHB5Ya4KXfctg33KzOGDVe+Xb38nVDB+eC0C8ITDYzB2M7F5yb1rR4e3TCbBphvn65lbwDjO9RnlZZhg3N9MEexlJ4BFkehY73GtW3oIJ6s7KSDaBn0eQWUlbdHFG0fMuew1WfwgbZ0D3x0xqkt9eNgs9gpuZlb8hzcr3WVyKVwg0GWK+qphr+I4TwrE61Q9g1aSSVzNtP5LKF6sPMDvmMF+tPUSrur78cfg0tbxdmTm+U5mrJZVS3N8zijA/Nx6fvZXhk1fx+d0dqRfocVXanZmbT1ZuAQGeVed/hUJc06w20zsV3hl6TTK1LA+uNqtAbR7gXOTLxctUFnDxMvv+Wb51oc/fzQKA3Qth85dmwYWzuwncnN3NUGGd1tD6jpIXUpzlGVR2Sayi5x02Gbo8CEtehCUvwK8vgS4wC0T6/KPKzc2qKBJ8CeEAzUN9eHtUK/42sDGzNiTww+YjjGwXxrNDmpZ71eWQlnWo7ePK+OkxDPtoFR/d3pZu0ZeX4mL7kVQ++C2OxNRsTmbkcupMLll5BQA8O7gJ47tfhWSiQojL4+IJDfs7/rxWGzS9yXw5UkgzuGM2HFgJG6aauYcX9qbVABJ8CeFAAZ4uPNi7wRWXJmoX4c8PD17HhBkbuWvaOp4Z1IR7u9W7ZHb+3Hw7H/4Wx0fL9uLrZqN5qA8Ngjzx93AmwNOF1XtP8ObPu+nZMIjokD+5IkoIIcojspv5qoEk+BKimokI8OD7B7ryxOytvDw/lu1HUnl9RMtSJ+LvOJrKE7O3sutYOsPbhPL8jc2KJZYd1T6M699dzpNzt/Hd/V1wskrlMSGEqCgSfAlRDXm4OPHxnW35aGk87yzeQ3xyBh/f0Q5vVxvpOXlk5OSTkZ3P73uSmbxsL34eznx2V3v6NQ0p8XiBni68OLQ5j3yzmakr93NfzxIyjWNykdkvp2aeEEKIYiT4EqKaUkrxUJ9omtT25tFZW+j+5tISt7u5TSjP39gUX/eyV0je2LI287cd5Z3Fe+jbJKRYwtgVcck8MXsrYW759O6lLznUKYQQomQSfAlRzfVtEsKPD3djwR+JuNqseLk44eXqhKerEyHerjQs5xwupRQvDWtO/3/9zlNztzL3/q5YLYrcfDtvL9rNlN/3EejpzKbjBUxffYBx19Wr4HcmhBDXJgm+hLgG1Av0uOJJ/BcK9nLlxZuaMXHWFqat3E/fJsE8Mmsz24+kMaZzBM8MasKtHyzm1YW76FQ/gCa1va9C64UQomaRWbVCiIvc1KoO/ZqE8Nai3Qx+fyWHU7KYMqYdLw1rjpuzlXtbuODrZuPhbzaTlVtQ2c0VQohqR4IvIcRFlFK8enNzvF1ttI3w5eeJPeh/QWFvb2fFu6Nbszc5g5fn76zElgohRPUkw45CiGKCvV1ZPakPzk4l//+sW3QgE3rU59Pl++geHcSA5ldQnFgIIWoo6fkSQpSotMDrrCeub0TLMB8mfb+NxNQsB7VKCCGqPwm+hBBXxNnJwr9vbUNuvp3hk1fz+sJdbD+Sii4hD9iZnHxWxZ9g/rZEMnPzK6G1QghRdciwoxDiitUL9GDq2A58vHwv/1mxj0+W76VeoAeDW9SmfpAHWxJOs/FgCrGJadgLYzIvFydubhvKbR3DZbWkEKJGcljwpZSaBgwBjmutmzvqvEKIitUlKoAuUQGcOpPLLzuO8dO2o0xeFo9dg7uzlTbhvjzUuwFtI/xwdrIwe0MCszYkMGPNQdqE+zK2SyRDW9eRpK1CiBrDkT1fXwAfAjMceE4hhIP4ezhzW8dwbusYTnJ6DicycogO9ixWJ7JrVCDPn8nlu02H+Wb9IR79dgu5+XZGd6hbSS0XQgjHcticL63178ApR51PCFF5grxcaFLbu9QC3X4ezozvXp/Fj/Wkc31/Xpy3g4RTmQ5uZXFfrjnA8MmryMiReWlCiIqjSpocW2EnUyoS+KmsYUel1ARgAkBISEi7WbNmVWibMjIy8PT0vPSGwqHkulRdV/vanMiy8+zKLCK8LTzd0RVLJQ0/7jxZwFsbstHATVE2hkeXXQuzqpF7puqSa1M1VcR16d2790atdftLbVflJtxrracAUwDat2+ve/XqVaHnW7ZsGRV9DnH55LpUXRVxbXRQAk/N3Ua8NZwJPaL+1LFiE9Nws1mJDPQo9z5HT2fxxAcriQr2pF6gB4viknlmdGdq+bj+qbY4ktwzVZdcm6qpMq+LpJoQQlS6ke3C6N80hLd/2cOuY2lXdIyNB1MYM3UdA/+9gl5vL2PstPX8tisJu73s3v2c/AIemLmJ7LwCPrmzHc8NaYrdDm8v2n1F7RBCiEuR4EsIUemUUrw6vAXebk489u1WcvLLXzNy86EU7pq2nhEfr2bH0TQmDWzMY/0aEpuYxj1fxNDr7WV8tmIfpzNzS9z/n/N2siXhNG+PakWDYE/q+rsz7rpIvtt0mB1HU6/WWxRCiHMcFnwppb4B1gCNlFKHlVL3OurcQoiqL9DThdeGtyQ2MY1/L4krc1utNaviT3DXtPXcPHk124+kMmlgY1b8tTf394xiYr9oVk3qw4e3t6GWtysvz4+lwytLGD99A//bcoQzhRPq58QkMHPdIe7rWZ+BLWqfO/6DvRrg42bj1QWxJSaNFUKIP8Nhc7601rc56lxCiOrp+qYhjG4fxifL9xJ3PIPejYLp1SiIOr5uAOTm25m39SifrdxPbGIagZ7O/HVAI8Z2icTD5eJfZzarhSEt6zCkZR1iE9P47+YjzNt6lCWxx3G1WejZMIhlu5PpUj+Ap/o3umhfH3cbE/tG8+K8nSzbnUzvxsEO+wyEENe+KjfhXghRsz13YzPcnZ1YvDOJxTuTAGhcy4u2EX78GptEUprJH/bGiBYMbR2Kq816yWM2qe1Nk9reTBrQmI2HUpi39SgL/kgk0NOFD25vU2JKjDs6RTB99QFeXRBL9+jAUtNmCCHE5ZLgSwhRpXi6OPHCTc14/samxB/PYOnu4yzdlcycmAQ61vPn9REt6dUw6Ioy4lssig6R/nSI9Of5G5uRb7fj4lRy8ObsZGHSwCbc/9VGZscc5vZO4Vf0fs7k5OPiZJHgTQhxjgRfQogqSSlFdIgX0SFeTOgRhdb6qpYgsloUVkvZvWY3NAuhY6Q/b/2yi8MpmbQM86VlmA+1fVwv2ZbE1Cw+/C2e2TEJ+Hs4M6pdXUa3r0t4gPtVew9CiOpJgi8hRLVQGbUflVK8NKw5T83dypTf95FfmLYi0NOFVmE+tI3wo12EH63CfHFzNoFccnoOk5fFM3PdIbTWjGgbxvHC5z5cGk+X+gHc2rEuNzSrVa4hUyHEtUeCLyGEKEOjWl78+FA3svMKiE1MY9vhVLYdTmVLQgq/7joOgJNF0ayON/WDPPl5+zFyC+yMaBvKw32iqetveroSU7OYG3OY2RsTmDhrC54uTtzQrBZDW9eha1RAsWFJu11z5HQWx9KyySuwk1+gySuwk1eg8XO30bGevxQjF6KakuBLCCHKwdVmpU24H23C/c49l3Iml02HUth4MIWYgyn8GpvEDc1CmNivIfWKZNiv7ePGw32jebB3A9buO8kPW46wcPsxvtt0mEBPZwa3qE2Ynzt7ktLZk5RO3PEMMnNLz3fWJtyXp25oRNeowAp7z0KIiiHBlxBCXCE/D2f6Ngmhb5OQcu9jsSi6Ngika4NA/jm0Oct2JzNv61FmbUggJ99OkJcLDUM8Gd2+Lo1qeRHq64bNasFmVdisFpysiq0Jqbz/axy3/2cd1zUI4MkiqTKEEFWbBF9CCFFJXG1WBjSvxYDmtTiTk09uvh0/j0sX9G5Wx4fhbUP5au1BJi/by82TV9M8wMr67F3U9XcnzM+Nun7u+Hs6sz/5DDsT04hNTGPn0TSOns7i0esbMrp9XQe8QyFESST4EkKIKsDDxQkPl/Jv72qzMr57fW7tGM7nK/czY2Ucn/6+j4JSall6ujjRpLYXgV4uPP3dNlycLAxtHXqVWi+EuBwSfAkhRDXm6eLEw32jaWE9QrfuPUhKz+HwqUwOp2SRnJFDZIA7TWv7EObnhsWiyM4rYOy09Tw+eyvuzk5c37T4kKnWmk2HUnC1WWlWx6fUc9vtmnX7T6EUdK4fUJFvU4hrigRfQghxjXCyWgj1dSPU141OpWzjarMydVwH7vxsHQ/O3MS0cR3oFn1+0v72I6m8tjCWVfEnAVMdYHT7MIa1Dj03JHo8PZs5MYeZHZPAwZOZANzeKZznhjSV9BlClIMEX0IIUcN4ujgx/e6O3DJlDX+ZEcOMeztSy9uVdxbt5octR/Fzt/HckKbYnCzMiUngxXk7eW3BLvo1DabArvk19jj5dk3Hev482i+aXcfS+XT5PrYcOs3kO9oSWWSlJ8DR01kcTsmiQ6SfpMgQNZ4EX0IIUQP5uNv48t5O3DJlDWOnrSe/QKMUPNArivt7ReHtagNgTOcIYhPTmBNzmP9uPoxFKe7pVo9bOtQlKsjz3PE6Rvrz+OytDPlgJW+MaMnglrU5mZHDgu3HmLflKOsPnAJgeJtQXhvRotSyTruOpfHOoj2MahdG/2a1Kv6DEKISSPAlhBA1VJCXCzPHd+L+LzfSMMSLx/s3pLaPW7HtmtT25rkbm/L3wU0AU5qpqL5NQlgwsTsPfb2JB7/exKe/+7DjaBoFdk10sCdPXN+Q3AI7H/wWz+GULD4Z0w7/C1Z2aq35ev0h/jlvJ3kFdhbvTOKe6+oxaWBjnJ2kLqa4tkjwJYQQNVhtHzf+91C3cm1bUtB1oVBfN76d0IV3Fu1m+Z5kJvSoz02t6tC4lte5ocboEC+enLOVmyevYtq4DkQFeZKalcczcaipTAAAEPVJREFU3//B/D8S6R4dyBsjWjLl931MW7WfjQdP8eHtbc9VCihLgV3z0k87ScvK49F+Da9KHc3jadnEHEzhREYOKWfySMnMJSUzl5w8O/f3iqJ1Xd8/fQ5R80jwJYQQ4qpxdrLwt0FN+NugJiW+flOrOoT6ujFhRgw3f7SKpwY05tPle0lMzebpAY25r0d9LBbFCzc1o3N9f56au41B76/grZGtGNC89GFIu13zzPd/8G1MAs5WC/O2HeWuLpE81LtBibnTktKyOXQqEy9XJ3zcbPi42XCzWcktsBNzIIXf9ySzfE8yu46lX7Sfl4sTvh42MrLz+T0umc/uak/XBlJlQFweCb6EEEI4VLsIP3548Dru/mID//hhO6G+bsy+rwvtIvwu2m5A89o0re3DQ99s4v6vNnJ7p3CeGdQET5eL/3RprXn+xx18G5PAI32juaNTOP9avIfPV+1ndkwCD/VuQL+mIWw6mML6/adYf+DUuVWaF3K2WkBBbr4dm1XRPsKfpwc05roGAdT2ccPX3YatsAbn8bRsxkxdz7gvNvDhbW1kfpq4LBJ8CSGEcLi6/u58939d+XHLEW5qFYqPu63E7cID3JlzfxfeWbSH/6zYx/Ldybw5siXXFfY2aa15dUEsX649yH096vNYv2iUUrw+oiV3X1eP1xfG8trCXby2cBcAfu42OkT6M6ZzBNEhXmRk55OalXfuy641HSP96RIVgIdL6X8ig71d+fa+zoz9fAP/N3MTb41syfC2YRdtk5tvZ+vh02xMyidvZxIKsFhAKUXDEFM6StRMEnwJIYSoFD5uNsZ0ibzkdi5OVp4Z1IQbmoXw1Jxt3PHZOu7oFM7fBjXh0+V7+c+K/YzrGsmkgY0vSmPRqJYXn9/dkXX7TrLvxBnaRfjRIMgTyyXmrpWXr7szM8d3YsKMGB6fvZW0rDy6NghkRdwJVsYls27/qfPF0TfHXLSvk0Uxsl0YD/ZuUK75bOLaIsGXEEKIaqFdhD8LJnbnnUW7+Wzlfhb8kUhKZh63dazLc0Oalpo/rFP9ADpVUAZ+Txcnpo3rwMPfbOaFeTvPPV8v0IOR7cLo1iCQxL07aNeuPXatsWsosNuZtzWRr9cd4rtNhxnVvi4P9m5wrieswK5JTs/hyOks8gvstKrrK8lrrzESfAkhhKg2XG1W/j64KQOa1+Lv/93OgOa1eGVYi6vWm3Wlbfr4jrZ8tfYgrjYr3aIDCfM735u1LHkXzUMvLtPULsKf+3rWZ/LSvczacIg5MQm0CvMlKT2bY6nZ5BWcr9HpbLXQOtyXrlEBdKkfQOtw31LzpNUUBXbN9NUHCPB0rpY1SiX4EkIIUe20i/Dn50d7VHYzznGyWhh3Xb3L2qe2jxsvDWvO/b2i+HhZPLuPpdM23I9QXzfq+LoR6ud2rn7mmr0n+fevcby3JA4vFyce7tuAcV3rVascaAV2TWZuPpm5BZzJMd9z8u2E+bkR7OVS7soHialZPDprC+v2m8S9Ofl2RrevW5FNv+ok+BJCCCEqUaivGy8Pa1Hq632bmOLnqZl5rNt/km/WH+LVBbv4Zn0Czw5uQp/GwRcFLsdSs1nwRyK/xyXj6mQlyMuFYC8Xggq/3Jyt2KwWnCwKm9WCzWoh3N8dN+fSe9N2HE1l5rpDHEnJIjrYk4a1vGgU4kV0iCfuzmWHEnFJ6Xy8fC/zth69qEfvQt6uTjQI9iQ62ByzYz1/mtfxKdajuWjHMf763TZy8+28MaIFP21LZNJ32/BwdmJwy9pltqMqkeBLCCGEqAZ83G30b1aL/s1qsXT3cV76aSf3To+hR8MgHunTgO1HUpn/RyIbDqQAEBXkgUUp1u4/yenMvDKP7exkoUv9AHo3CqJ342AiAjzIzivgp22JfLX2IFsSTuNqs1Av0JO1+06Sk28/t2+9QA861/enS1QgXeoHEOTlAsDmQyl8vGwvi3Ym4WazckuHukT4e+DuYsXd2Yq7sxPOVguHTmUSdzyduKQMlsQm8W1MAgCBns70bBhM78ZBdKznz0e/xTN9zUGah3rz/q1tqB/kyY2t6jB22nomztqMm7OFPo1DKujTv7ok+BJCCCGqmd6NgunWIJAZaw7y3pI9jPxkDQCNQrx4/PqGDGpRmwbB52tv5uQXcCIjl+T0HLLzCsgv0OTZ7eTl28nJt7Ml4TRLdx/nhXk7eWHeTuoFepCSmcvpzDzqB3nw3JCmjGgbho+7jQK7JuFUJruT0tlzLJ2th0/z09ZEvllvgqaGIZ54u9qIOZiCj5uNR/pGM65r5EXlpMpyPD2bVfEnWLormSWxSXy36fC518Z3q8dTAxqdm/Pm7uzE1HEduOM/67j/q018cXcHukadT0Oy/8QZVu89ScKpzFIT/1YGCb6EEEKIashmtXBvt3oMa12HJbFJtA33IzrEq8RtXZyshPq6lZpb7MZWdfjHkKYcPHmGpbuOs3xPMs3qeHN7p3C61A+4aFjTalFEBnoQGejBDYXJZfML7Ow4msbqvSdZvfcEianZ/H1QE27rFF4sKe6lBHu5cnObMG5uE0Z+gcmVtir+JG3D/egWXbyagLerjen3dOTWKWsYPz2Gx/o1ZGdiGmv2nuRYWjYAdXxceez6hlVm1agEX0IIIUQ1FuDpwi0dwq/KsSICPBh3Xb3LXjzgZLXQqq4vrer68n+9oq5KW84et12EP+0i/Mvczt/Dma/u7cToT9fwyoJYAjyc6RIVQJeoALpGBRIZ4F7uCf2OIMGXEEIIIaq9YG9XFk7swdHULOoHelSpYKsoCb6EEEIIcU1wc7YSFeR56Q0rWfVJECKEEEIIcQ2Q4EsIIYQQwoEk+BJCCCGEcCAJvoQQQgghHEiCLyGEEEIIB5LgSwghhBDCgST4EkIIIYRwIAm+hBBCCCEcyKHBl1JqgFJqt1IqXik1yZHnFkIIIYSoChwWfCmlrMBHwECgKXCbUqqpo84vhBBCCFEVOLLnqyMQr7Xep7XOBWYBQx14fiGEEEKISqe01o45kVIjgQFa6/GFj8cAnbTWDxXZbgIwASAkJKTdrFmzKrRdGRkZeHpW/TpQNY1cl6pLrk3VJNel6pJrUzVVxHXp3bv3Rq11+0ttV+UKa2utpwBTANq3b6979epVoedbtmwZFX0OcfnkulRdcm2qJrkuVZdcm6qpMq+LI4cdjwB1L3gcVvicEEIIIUSN4cjgawMQrZSqp5RyBm4FfnTg+YUQQgghKp3D5nwBKKUGAe8BVmCa1vqVS2yfDBys4GYFAicq+Bzi8sl1qbrk2lRNcl2qLrk2VVNFXJcIrXXQpTZyaPBVFSmlYsozOU44llyXqkuuTdUk16XqkmtTNVXmdZEM90IIIYQQDiTBlxBCCCGEA0nwVZjWQlQ5cl2qLrk2VZNcl6pLrk3VVGnXpcbP+RJCCCGEcCTp+RJCCCGEcCAJvoQQQgghHKjGBl9KqQFKqd1KqXil1KTKbk9NppSqq5RaqpTaqZTaoZSaWPi8v1JqsVIqrvC7X2W3tSZSSlmVUpuVUj8VPq6nlFpXeO98W5g0WTiYUspXKTVXKbVLKRWrlOoi90zlU0o9Vvh7bLtS6hullKvcM5VDKTVNKXVcKbX9gudKvEeU8X7hNdqmlGpbkW2rkcGXUsoKfAQMBJoCtymlmlZuq2q0fOAJrXVToDPwYOH1mAT8qrWOBn4tfCwcbyIQe8HjN4B/aa0bACnAvZXSKvFv4GetdWOgFeYayT1TiZRSocAjQHutdXNMQvFbkXumsnwBDCjyXGn3yEAguvBrAvBxRTasRgZfQEcgXmu9T2udC8wChlZym2osrXWi1npT4c/pmD8ioZhrMr1ws+nAsMppYc2llAoDBgOfFT5WQB9gbuEmcl0qgVLKB+gBTAXQWudqrU8j90xV4AS4KaWcAHcgEblnKoXW+nfgVJGnS7tHhgIztLEW8FVK1a6ottXU4CsUSLjg8eHC50QlU0pFAm2AdUCI1jqx8KVjQEglNasmew/4K2AvfBwAnNZa5xc+lnunctQDkoHPC4eEP1NKeSD3TKXSWh8B3gYOYYKuVGAjcs9UJaXdIw6NC2pq8CWqIKWUJ/Ad8KjWOu3C17TJiSJ5URxIKTUEOK613ljZbRHFOAFtgY+11m2AMxQZYpR7xvEK5w8NxQTHdQAPig97iSqiMu+Rmhp8HQHqXvA4rPA5UUmUUjZM4DVTa/194dNJZ7t9C78fr6z21VDXATcppQ5ghub7YOYZ+RYOqYDcO5XlMHBYa72u8PFcTDAm90zl6gfs11ona63zgO8x95HcM1VHafeIQ+OCmhp8bQCiC1egOGMmRP5YyW2qsQrnEU0FYrXW717w0o/A2MKfxwL/c3TbajKt9d+01mFa60jMPfKb1voOYCkwsnAzuS6VQGt9DEhQSjUqfKovsBO5ZyrbIaCzUsq98Pfa2esi90zVUdo98iNwV+Gqx85A6gXDk1ddjc1wr5QahJnPYgWmaa1fqeQm1VhKqW7ACuAPzs8tegYz72s2EA4cBEZrrYtOnhQOoJTqBTyptR6ilKqP6QnzBzYDd2qtcyqzfTWRUqo1ZiGEM7APuBvzH2q5ZyqRUupF4BbMKu7NwHjM3CG5ZxxMKfUN0AsIBJKA54EfKOEeKQyWP8QME2cCd2utYyqsbTU1+BJCCCGEqAw1ddhRCCGEEKJSSPAlhBBCCOFAEnwJIYQQQjiQBF9CCCGEEA4kwZcQQgghhANJ8CWEECVQSmml1MhLbymEEJdHgi8hRJWjlPqiMPgp+rW2stsmhBB/ltOlNxFCiEqxBBhT5LncymiIEEJcTdLzJYSoqnK01seKfJ2Cc0OCDyml5iulMpVSB5VSd164s1KqhVJqiVIqSyl1qrA3zafINmOVUn8opXKUUklKqelF2uCvlJqjlDqjlNpXwjmeKzx3jlLqmFJqRoV8EkKIa4oEX0KI6upFTD221sAUYIZSqj2AUsoD+AXIADoCNwNdgWlnd1ZK3Qd8CnwOtAQGAduLnOM5TO23VsC3wDSlVHjh/iOAJ4EHgGhgCLC+At6nEOIaI+WFhBBVjlLqC+BOILvISx9prZ9WSmngM631Xy7YZwlwTGt9p1LqL8DbQJjWOr3w9V6YAsfRWut4pdRh4Cut9aRS2qCB17XWfyt87ASkARO01l8ppR4H7gOaa63zrtqbF0Jc82TOlxCiqvodmFDkudMX/LymyGtrgMGFPzcBtp0NvAqtxhRub6qUSsMUO/71Em3YdvYHrXW+UioZCC58ag4wEdivlPoF+Bn4UQomCyEuRYYdhRBVVabWOr7I14mrcNzL6e4v2qOlKfy9qbVOABpher/SgHeAjYVDnkIIUSoJvoQQ1VXnEh7HFv4cC7RQSnld8HpXzO+8WK31ceAI0PfPNEBrna21nq+1fgzoADQDrvszxxRCXPtk2FEIUVW5KKVqFXmuQGudXPjzcKXUBmAZMBITSHUqfG0mZkL+DKXUc4AfZnL991rr+MJtXgH+pZRKAuYD7kBfrfU75WmcUmoc5nfoOszE/lswPWVxl/k+hRA1jARfQoiqqh+QWOS5I0BY4c8vACOA94Fk4G6t9QYArXWmUuoG4D3MCsRszKrFiWcPpLX+WCmVCzwBvAGcAhZcRvtOA09jJvbbgJ3AcK31/ss4hhCiBpLVjkKIaqdwJeIorfXcym6LEEJcLpnzJYQQQgjhQBJ8CSGEEEI4kAw7CiGEEEI4kPR8CSGEEEI4kARfQgghhBAOJMGXEEIIIYQDSfAlhBBCCOFAEnwJIYQQQjjQ/wMOTrG6J3en3gAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 720x504 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Visualize the training process\n",
    "train_loss = history.history['loss']\n",
    "val_loss = history.history['val_loss']\n",
    "\n",
    "fig, ax = plt.subplots(figsize=(10, 7))\n",
    "ax.plot(train_loss, label='Training Loss')\n",
    "ax.plot(val_loss, label='Validation Loss')\n",
    "ax.set_title('Loss vs. Epochs', fontsize=16)\n",
    "ax.set_xlabel('Epochs', fontsize=14)\n",
    "ax.set_ylabel('Loss', fontsize=14)\n",
    "ax.legend(fontsize=14)\n",
    "ax.grid(True)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Extract Features using Triplet Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train:\t\t (113783, 4)\n",
      "Validation:\t (22255, 4)\n",
      "Test:\t\t (22391, 4)\n",
      "\n",
      "Train Landmarks:\t 14943\n",
      "Validation Landmarks:\t 7674\n",
      "Test Landmarks:\t\t 14436\n"
     ]
    }
   ],
   "source": [
    "train_df = pd.read_csv('./data/triplet/train.csv')\n",
    "val_df = pd.read_csv('./data/triplet/validation.csv')\n",
    "test_df = pd.read_csv('./data/triplet/test.csv')\n",
    "\n",
    "print('Train:\\t\\t', train_df.shape)\n",
    "print('Validation:\\t', val_df.shape)\n",
    "print('Test:\\t\\t', test_df.shape)\n",
    "\n",
    "print('\\nTrain Landmarks:\\t', len(train_df['landmark_id'].unique()))\n",
    "print('Validation Landmarks:\\t', len(val_df['landmark_id'].unique()))\n",
    "print('Test Landmarks:\\t\\t', len(test_df['landmark_id'].unique()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "vgg16 (Model)                (None, 512)               14714688  \n",
      "_________________________________________________________________\n",
      "l2_norm (Lambda)             (None, 512)               0         \n",
      "=================================================================\n",
      "Total params: 14,714,688\n",
      "Trainable params: 7,079,424\n",
      "Non-trainable params: 7,635,264\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Load trained model\n",
    "base_model = load_model('./models/vgg16-base-0.2-model.h5')\n",
    "base_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define train_imgs and test_imgs\n",
    "train_imgs = np.zeros(shape=(len(train_df), 512), dtype=np.float32)\n",
    "val_imgs = np.zeros(shape=(len(val_df), 512), dtype=np.float32)\n",
    "test_imgs = np.zeros(shape=(len(test_df), 512), dtype=np.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Process:          0\n",
      "================================================================================\n",
      "Process:      20000\n",
      "================================================================================\n",
      "Process:      40000\n",
      "================================================================================\n",
      "Process:      60000\n",
      "================================================================================\n",
      "Process:      80000\n",
      "================================================================================\n",
      "Process:     100000\n",
      "========================================================"
     ]
    }
   ],
   "source": [
    "# Process training images\n",
    "img_ids = train_df['image_id'].values\n",
    "steps = 20000\n",
    "for i in range(0, len(train_df), steps):\n",
    "    tmp_imgs = []\n",
    "    print('\\nProcess: {:10d}'.format(i))\n",
    "    \n",
    "    start = i\n",
    "    end = min(len(train_df), i + steps)\n",
    "    for idx in range(start, end):\n",
    "        if idx % 250 == 0:\n",
    "            print('=', end='')\n",
    "            \n",
    "        img_id = img_ids[idx]\n",
    "        path = './data/triplet/train/' + str(img_id) + '.jpg'\n",
    "        img = load_img(path, target_size=img_size[:2])\n",
    "        img = img_to_array(img)\n",
    "        tmp_imgs.append(img)\n",
    "        \n",
    "    tmp_imgs = np.array(tmp_imgs, dtype=np.float32) / 255.0\n",
    "    tmp_prediction = base_model.predict(tmp_imgs)\n",
    "    train_imgs[start: end, ] = tmp_prediction\n",
    "    _ = gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Process:          0\n",
      "================================================================================\n",
      "Process:       4000\n",
      "================================================================================\n",
      "Process:       8000\n",
      "================================================================================\n",
      "Process:      12000\n",
      "================================================================================\n",
      "Process:      16000\n",
      "================================================================================\n",
      "Process:      20000\n",
      "=============================================="
     ]
    }
   ],
   "source": [
    "# Process validation images\n",
    "img_ids = val_df['image_id'].values\n",
    "steps = 4000\n",
    "for i in range(0, len(val_df), steps):\n",
    "    tmp_imgs = []\n",
    "    print('\\nProcess: {:10d}'.format(i))\n",
    "    \n",
    "    start = i\n",
    "    end = min(len(val_df), i + steps)\n",
    "    for idx in range(start, end):\n",
    "        if idx % 50 == 0:\n",
    "            print('=', end='')\n",
    "            \n",
    "        img_id = img_ids[idx]\n",
    "        path = './data/triplet/validation/' + str(img_id) + '.jpg'\n",
    "        img = load_img(path, target_size=img_size[:2])\n",
    "        img = img_to_array(img)\n",
    "        tmp_imgs.append(img)\n",
    "        \n",
    "    tmp_imgs = np.array(tmp_imgs, dtype=np.float32) / 255.0\n",
    "    tmp_prediction = base_model.predict(tmp_imgs)\n",
    "    val_imgs[start: end, ] = tmp_prediction\n",
    "    _ = gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Process:          0\n",
      "================================================================================\n",
      "Process:       4000\n",
      "================================================================================\n",
      "Process:       8000\n",
      "================================================================================\n",
      "Process:      12000\n",
      "================================================================================\n",
      "Process:      16000\n",
      "================================================================================\n",
      "Process:      20000\n",
      "================================================"
     ]
    }
   ],
   "source": [
    "# Process test images\n",
    "img_ids = test_df['image_id'].values\n",
    "steps = 4000\n",
    "for i in range(0, len(test_df), steps):\n",
    "    tmp_imgs = []\n",
    "    print('\\nProcess: {:10d}'.format(i))\n",
    "    \n",
    "    start = i\n",
    "    end = min(len(test_df), i + steps)\n",
    "    for idx in range(start, end):\n",
    "        if idx % 50 == 0:\n",
    "            print('=', end='')\n",
    "            \n",
    "        img_id = img_ids[idx]\n",
    "        path = './data/triplet/test/' + str(img_id) + '.jpg'\n",
    "        img = load_img(path, target_size=img_size[:2])\n",
    "        img = img_to_array(img)\n",
    "        tmp_imgs.append(img)\n",
    "        \n",
    "    tmp_imgs = np.array(tmp_imgs, dtype=np.float32) / 255.0\n",
    "    tmp_prediction = base_model.predict(tmp_imgs)\n",
    "    test_imgs[start: end, ] = tmp_prediction\n",
    "    _ = gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train:\t\t (113783, 512)\n",
      "Validation:\t (22255, 512)\n",
      "Test:\t\t (22391, 512)\n"
     ]
    }
   ],
   "source": [
    "print('Train:\\t\\t', train_imgs.shape)\n",
    "print('Validation:\\t', val_imgs.shape)\n",
    "print('Test:\\t\\t', test_imgs.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save to disk\n",
    "np.save('./data/triplet/train-triplet-vgg16-0.2-features.npy', train_imgs)\n",
    "np.save('./data/triplet/validation-triplet-vgg16-0.2-features.npy', val_imgs)\n",
    "np.save('./data/triplet/test-triplet-vgg16-0.2-features.npy', test_imgs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load Features and Labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train:\t\t (113783, 512) (113783, 4)\n",
      "Validation:\t (22255, 512) (22255, 4)\n",
      "Test:\t\t (22391, 512) (22391, 4)\n"
     ]
    }
   ],
   "source": [
    "# Already normalized\n",
    "train_feature = np.load('./data/triplet/train-triplet-vgg16-0.2-features.npy')\n",
    "val_feature = np.load('./data/triplet/validation-triplet-vgg16-0.2-features.npy')\n",
    "test_feature = np.load('./data/triplet/test-triplet-vgg16-0.2-features.npy')\n",
    "\n",
    "train_df = pd.read_csv('./data/triplet/train.csv')\n",
    "val_df = pd.read_csv('./data/triplet/validation.csv')\n",
    "test_df = pd.read_csv('./data/triplet/test.csv')\n",
    "\n",
    "print('Train:\\t\\t', train_feature.shape, train_df.shape)\n",
    "print('Validation:\\t', val_feature.shape, val_df.shape)\n",
    "print('Test:\\t\\t', test_feature.shape, test_df.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Helper function\n",
    "def accuracy(true_label, prediction, top=1):\n",
    "    \"\"\" function to calculate the prediction accuracy \"\"\"\n",
    "    prediction = prediction[:, :top]\n",
    "    count = 0\n",
    "    for i in range(len(true_label)):\n",
    "        if true_label[i] in prediction[i]:\n",
    "            count += 1\n",
    "            \n",
    "    return count / len(true_label)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Implement KNN Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Merge train and validation features\n",
    "train_val_feature = np.concatenate((train_feature, val_feature), axis=0)\n",
    "train_val_df = pd.concat((train_df, val_df), axis=0)\n",
    "train_val_df = train_val_df.reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "NearestNeighbors(algorithm='auto', leaf_size=30, metric='minkowski',\n",
       "         metric_params=None, n_jobs=-1, n_neighbors=50, p=2, radius=1.0)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Implement KNN model\n",
    "knn = NearestNeighbors(n_neighbors=50, algorithm='auto', leaf_size=30, \n",
    "                       metric='minkowski', p=2, n_jobs=-1)\n",
    "knn.fit(train_val_feature)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Search the first 50 neighbors\n",
    "distance, neighbor_index = knn.kneighbors(test_feature, return_distance=True)\n",
    "\n",
    "# Save the results\n",
    "np.save('./result/knn-triplet-vgg16-0.2-distance.npy', distance)\n",
    "np.save('./result/knn-triplet-vgg16-0.2-neighbor.npy', neighbor_index)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Search Neighbors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "knn_distance = np.load('./result/knn-triplet-vgg16-0.2-distance.npy')\n",
    "knn_neighbor = np.load('./result/knn-triplet-vgg16-0.2-neighbor.npy')\n",
    "\n",
    "# Get the first 50 neighbors\n",
    "predictions = []\n",
    "for neighbors in knn_neighbor:\n",
    "    predictions.append(train_val_df.loc[neighbors]['landmark_id'].values)\n",
    "\n",
    "predictions = np.array(predictions)\n",
    "np.save('./result/knn-triplet-vgg16-0.2-test-prediction.npy', predictions)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Compute Accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Top  1 accuracy:\t 0.36867491402795766\n",
      "Top  5 accuracy:\t 0.5401277298914743\n",
      "Top 10 accuracy:\t 0.6123442454557635\n",
      "Top 20 accuracy:\t 0.6807199321155821\n"
     ]
    }
   ],
   "source": [
    "print('Top  1 accuracy:\\t', accuracy(test_df['landmark_id'].values, predictions, top=1))\n",
    "print('Top  5 accuracy:\\t', accuracy(test_df['landmark_id'].values, predictions, top=5))\n",
    "print('Top 10 accuracy:\\t', accuracy(test_df['landmark_id'].values, predictions, top=10))\n",
    "print('Top 20 accuracy:\\t', accuracy(test_df['landmark_id'].values, predictions, top=20))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "knn_acc = []\n",
    "for i in range(1, 51):\n",
    "    tmp_acc = accuracy(test_df['landmark_id'].values, predictions, top=i)\n",
    "    knn_acc.append(tmp_acc)\n",
    "\n",
    "np.save('./result/knn-triplet-vgg16-0.2-accuracy.npy', knn_acc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Environment (conda_tensorflow_p36)",
   "language": "python",
   "name": "conda_tensorflow_p36"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}