relu_attn vs. flash_attn

relu_v_flash.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7821e58b",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import numpy as np\n",
    "\n",
    "import triton\n",
    "import triton.language as tl\n",
    "\n",
    "try:\n",
    "    from flash_attn.flash_attn_interface import \\\n",
    "        flash_attn_qkvpacked_func as flash_attn_func\n",
    "    HAS_FLASH = True\n",
    "except BaseException:\n",
    "    HAS_FLASH = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a2c13728",
   "metadata": {},
   "outputs": [],
   "source": [
    "# relu_attn bf16 fwd implementation\n",
    "# from https://gist.github.com/mitchellnw/17d529b1a5eabd38ca345e41f5002074\n",
    "\n",
    "@triton.jit\n",
    "def relu_attn_(q_ptr,\n",
    "               k_ptr,\n",
    "               v_ptr,\n",
    "               o_ptr,\n",
    "               Dh: tl.constexpr, # head dim\n",
    "               L: tl.constexpr,  # seqlen\n",
    "               Nh: tl.constexpr, # num heads\n",
    "               B: tl.constexpr,  # batchsize\n",
    "               sm_scale: tl.constexpr, # 1/sqrt(Dh)\n",
    "               relu_scale: tl.constexpr, # 1/L\n",
    "               is_causal: tl.constexpr,\n",
    "               is_squared: tl.constexpr,\n",
    "               BLOCK_SIZE: tl.constexpr,  # Number of elements each program should process.\n",
    "               ):\n",
    "    # Q, K, V is of size [B, L, Nh, Dh]\n",
    "    pid = tl.program_id(axis=0) # current program id\n",
    "    currB = (pid * BLOCK_SIZE) // (Nh * L) # current batch idx\n",
    "    currL = (BLOCK_SIZE * pid) % L\n",
    "    currNh = ((BLOCK_SIZE * pid) // L) % Nh\n",
    "    # Common offsets\n",
    "    block_start = currB*Nh*L*Dh + currL*Nh*Dh + currNh*Dh\n",
    "    bsz_offset = tl.arange(0, BLOCK_SIZE)\n",
    "    common_offset = tl.arange(0, Dh)[None, :] + bsz_offset[:, None]*(Dh*Nh)\n",
    "    # Always keep q in mem\n",
    "    q = tl.load(q_ptr + block_start + common_offset)\n",
    "    # Accum.\n",
    "    acc = tl.zeros((BLOCK_SIZE, Dh), dtype=tl.float32)\n",
    "    # Loop over seqlen in BLOCK_SIZE chunks\n",
    "    upper = currL + 1 if is_causal else L\n",
    "    for l in range(0, upper, BLOCK_SIZE):\n",
    "      common_kv_offset = currB*Nh*L*Dh + l*Nh*Dh + currNh*Dh + common_offset\n",
    "      k = tl.load(k_ptr + common_kv_offset)\n",
    "      v = tl.load(v_ptr + common_kv_offset)\n",
    "      qk = tl.dot((q * sm_scale).to(tl.bfloat16), tl.trans(k)) # TODO: why is bfloat cast required\n",
    "      # causal masking and relu\n",
    "      mask = (qk >= 0)\n",
    "      if is_causal:\n",
    "        mask *= ((currL + bsz_offset)[:, None] >= (l + bsz_offset)[None, :])\n",
    "      qk = tl.where(mask, qk, 0.)\n",
    "      if is_squared:\n",
    "        qk *= qk\n",
    "      acc += tl.dot((relu_scale * qk).to(tl.bfloat16), v) # TODO: why is bfloat cast required\n",
    "    tl.store(o_ptr + block_start + common_offset, acc)\n",
    "\n",
    "\n",
    "def relu_attn(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, is_causal: bool = True, is_squared: bool = False):\n",
    "    output = torch.empty_like(q)\n",
    "    B, L, Nh, Dh = q.shape\n",
    "    BLOCK_SIZE = min(L, 64)\n",
    "    grid = lambda meta: ((B * Nh * L) // BLOCK_SIZE, )\n",
    "    relu_attn_[grid](q, k, v, output, Dh, L, Nh, B, 1./np.sqrt(Dh), 1./L, is_causal=is_causal, is_squared=is_squared, BLOCK_SIZE=BLOCK_SIZE, num_warps=4, num_stages=1)\n",
    "    return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "27de504b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# benchmarking code\n",
    "\n",
    "# default vals\n",
    "# BATCH, N_HEADS, N_CTX, D_HEAD = 4, 48, 4096, 64\n",
    "BATCH, N_CTX, N_HEADS, D_HEAD = 4, 256, 4, 128\n",
    "\n",
    "# vary seq length for fixed head and batch=4\n",
    "configs = []\n",
    "\n",
    "for xval in [\"N_CTX\", \"H\"]:\n",
    "    x_names = [xval]\n",
    "    # x_vals=[2**i for i in range(10, 15)] if xval == \"N_CTX\" else [2*i for i in range(24, 48)]\n",
    "    x_vals=[256, 512, 1024, 2048] if xval == \"N_CTX\" else [2*i for i in range(2, 24)]\n",
    "    \n",
    "    args={\n",
    "        \"BATCH\": BATCH,\n",
    "        \"D_HEAD\": D_HEAD,\n",
    "        \"dtype\": torch.bfloat16,\n",
    "    }\n",
    "    if xval == \"N_CTX\":\n",
    "        args[\"H\"] = N_HEADS\n",
    "    else:\n",
    "        args[\"N_CTX\"] = N_CTX\n",
    "    \n",
    "    # for mode in [\"fwd\", \"bwd\"]:\n",
    "    for mode in [\"fwd\"]:\n",
    "        args[\"mode\"] = mode\n",
    "\n",
    "        configs.append(\n",
    "            triton.testing.Benchmark(\n",
    "                x_names=x_names,\n",
    "                x_vals=x_vals,\n",
    "                x_log=True,\n",
    "                line_arg=\"provider\",\n",
    "                line_vals=[\"relu\"] + ([\"flash\"] if HAS_FLASH else []),\n",
    "                line_names=[\"ReLU\"] + ([\"Flash-2\"] if HAS_FLASH else []),\n",
    "                styles=[(\"red\", \"-\"), (\"blue\", \"-\")],\n",
    "                ylabel=\"ms\",\n",
    "                plot_name=f\"fused-attention-batch{BATCH}-d{D_HEAD}-{mode}-xval{xval}\",\n",
    "                args=args,\n",
    "            ))\n",
    "\n",
    "@triton.testing.perf_report(configs)\n",
    "def bench_flash_attention(BATCH, H, N_CTX, D_HEAD, mode, provider, dtype=torch.bfloat16, device=\"cuda\"):\n",
    "    assert mode in [\"fwd\", \"bwd\"]\n",
    "    warmup = 25\n",
    "    rep = 100\n",
    "    if provider == \"relu\":\n",
    "        q = torch.randn((BATCH, N_CTX, H, D_HEAD), dtype=dtype, device=\"cuda\", requires_grad=True)\n",
    "        k = torch.randn((BATCH, N_CTX, H, D_HEAD), dtype=dtype, device=\"cuda\", requires_grad=True)\n",
    "        v = torch.randn((BATCH, N_CTX, H, D_HEAD), dtype=dtype, device=\"cuda\", requires_grad=True)\n",
    "        fn = lambda: relu_attn(q, k, v, is_causal=True, is_squared=False)\n",
    "        if mode == \"bwd\":\n",
    "            o = fn()\n",
    "            do = torch.randn_like(o)\n",
    "            fn = lambda: o.backward(do, retain_graph=True)\n",
    "        ms = triton.testing.do_bench(fn, warmup=warmup, rep=rep)\n",
    "    if provider == \"flash\":\n",
    "        qkv = torch.randn((BATCH, N_CTX, 3, H, D_HEAD), dtype=dtype, device=device, requires_grad=True)\n",
    "        fn = lambda: flash_attn_func(qkv, causal=True)\n",
    "        if mode == \"bwd\":\n",
    "            o = fn()\n",
    "            do = torch.randn_like(o)\n",
    "            fn = lambda: o.backward(do, retain_graph=True)\n",
    "        ms = triton.testing.do_bench(fn, warmup=warmup, rep=rep)\n",
    "\n",
    "    ms, min_ms, max_ms = ms\n",
    "    return ms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6635b136",
   "metadata": {},
   "outputs": [],
   "source": [
    "bench_flash_attention.run(save_path=\".\", print_data=True)"
   ]
  },
  {
   "attachments": {
    "fused-attention-batch4-d128-fwd-xvalN_CTX.png": {
     "image/png": "iVBORw0KGgoAAAANSUhEUgAAAoAAAAHgCAYAAAA10dzkAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/OQEPoAAAACXBIWXMAAA9hAAAPYQGoP6dpAABd8klEQVR4nO3deVhUZf8G8HsAAZFNRDZF0UTNBTA2ydySwuW1LDRTSzTT8lVTSXOpXFoEzYxc0qxcetMw/ZnlkmWYmokbSm6JaCZugCuDoIDM+f3xxODIoCzDnDMz9+e65npnDmfO+R7fkrvnnOf5qiRJkkBEREREFsNK7gKIiIiIyLgYAImIiIgsDAMgERERkYVhACQiIiKyMAyARERERBaGAZCIiIjIwjAAEhEREVkYBkAiIiIiC8MASERERGRhGACJiIiILAwDIBEREZGFYQAkIiIisjAMgEREREQWhgGQiIiIyMIwABIRERFZGAZAIiIiIgvDAEhERERkYRgAiYiIiCwMAyARERGRhWEAJCIiIrIwDIBEREREFoYBkIiIiMjCMAASERERWRgGQCIiIiILwwBIREREZGEYAImIiIgsDAMgERERkYVhACQiIiKyMAyARERERBaGAZCIiIjIwjAAEhEREVkYBkAiIiIiC8MASERERGRhGACJiIiILAwDIBEREZGFYQAkIiIisjAMgEREREQWhgGQiIiIyMIwABIRERFZGAZAIiIiIgvDAEhERERkYRgAiYiIiCwMAyARERGRhWEAJCIiIrIwDIBEREREFoYBkIiIiMjCMAASERERWRgGQCIiIiILwwBIREREZGEYAImIiIgsDAMgERERkYVhACQiIiKyMAyARERERBaGAZCIiIjIwjAAEhEREVkYBkAiIiIiC2MjdwGmTKPR4NKlS3BycoJKpZK7HCIiIqoASZKQm5sLHx8fWFlZ5lgYA2A1XLp0Cb6+vnKXQURERFVw/vx5NGzYUO4yZKGYALho0SJ89NFHyMzMRGBgIBYsWICwsDC9+37xxRf4+uuvcezYMQBAcHAwZs2apbO/JEmYPn06vvjiC9y8eRMdOnTA4sWL4e/vr93n+vXrGDNmDDZu3AgrKytER0fj008/haOjY4VqdnJyAiD+AXJ2dq7qpRMREZERqdVq+Pr6an+PWyJFBMA1a9YgNjYWS5YsQXh4OBISEhAVFYW0tDR4eHiU2X/Hjh0YMGAAHn/8cdjb22P27Nl4+umncfz4cTRo0AAAMGfOHMyfPx8rV65EkyZN8O677yIqKgonTpyAvb09AGDQoEG4fPkytm3bhqKiIgwdOhQjRozA6tWrK1R3yW1fZ2dnBkAiIiITY8mPb6kkSZLkLiI8PByhoaFYuHAhAPFsna+vL8aMGYPJkyc/9PvFxcWoW7cuFi5ciMGDB0OSJPj4+ODNN9/EhAkTAAA5OTnw9PTEihUr8OKLL+Kvv/5Cq1atcODAAYSEhAAAtm7dip49e+LChQvw8fF56HnVajVcXFyQk5PDAEhERGQi+PtbAbOACwsLkZKSgsjISO02KysrREZGIjk5uULHyM/PR1FREdzc3AAAZ8+eRWZmps4xXVxcEB4erj1mcnIyXF1dteEPACIjI2FlZYV9+/YZ4tKIiIiIFEn2W8BXr15FcXExPD09dbZ7enri5MmTFTrGpEmT4OPjow18mZmZ2mPcf8ySn2VmZpa5vWxjYwM3NzftPvcrKChAQUGB9rNara5QfURERERKInsArK74+HgkJiZix44d2mf7akpcXBxmzpxZqe9IkoS7d++iuLi4hqqih7G2toaNjY1FP+tBRER0L9kDoLu7O6ytrZGVlaWzPSsrC15eXg/87ty5cxEfH49ff/0VAQEB2u0l38vKyoK3t7fOMYOCgrT7ZGdn6xzv7t27uH79ernnnTJlCmJjY7WfS2YRlaewsBCXL19Gfn7+A6+Dap6DgwO8vb1ha2srdylERESykz0A2traIjg4GElJSejTpw8AMQkkKSkJo0ePLvd7c+bMwYcffoiff/5Z5zk+AGjSpAm8vLyQlJSkDXxqtRr79u3DyJEjAQARERG4efMmUlJSEBwcDADYvn07NBoNwsPD9Z7Tzs4OdnZ2FboujUaDs2fPwtraGj4+PrC1teUIlAwkSUJhYSGuXLmCs2fPwt/f32IX/SQiIiohewAEgNjYWMTExCAkJARhYWFISEhAXl4ehg4dCgAYPHgwGjRogLi4OADA7NmzMW3aNKxevRp+fn7aZ/YcHR3h6OgIlUqFcePG4YMPPoC/v792GRgfHx9tyHz00UfRvXt3DB8+HEuWLEFRURFGjx6NF198sUIzgB+msLBQO5vZwcGh2sejqqtduzZq1aqFc+fOobCwsMYfFSAiIlI6RQTA/v3748qVK5g2bRoyMzMRFBSErVu3aidxZGRk6IzaLF68GIWFhejbt6/OcaZPn44ZM2YAAN566y3k5eVhxIgRuHnzJp544gls3bpV55f/qlWrMHr0aHTr1k27EPT8+fMNem0cbVIG/v9ARERUShHrAJqqB60jdOfOHZw9exZNmjThiJMC8P8PIiIqwXUAFbAOIBEREREZFwMglTFkyBCoVCqoVCrUqlULTZo0wVtvvYU7d+5U6Pv//PMPVCoVUlNTy/xsx44dUKlUuHnzZpmf+fn5ISEhoXrFExER0UMp4hlAUp7u3btj+fLlKCoqQkpKCmJiYqBSqTB79my5SyMiIkshSQBX0KgRHAEkvezs7ODl5QVfX1/06dMHkZGR2LZtGwCxxE1cXByaNGmC2rVrIzAwEOvWrZO5YiIiMiv79wOhocDRo3JXYpY4AmhMkgTIsSi0g0O1/gvq2LFj2LNnDxo3bgxAdET55ptvsGTJEvj7+2PXrl146aWXUL9+fXTu3NlQVRMRkaXKywNeeglITwfmzAH+9z+5KzI7DIDGlJ8PODoa/7y3bgF16lTqK5s2bYKjoyPu3r2LgoICWFlZYeHChSgoKMCsWbPw66+/IiIiAgDQtGlT7N69G59//jkDIBERVd+ECSL8NWgAGHh5NhIYAEmvrl27YvHixcjLy8Mnn3wCGxsbREdH4/jx48jPz8dTTz2ls39hYSHatWsnU7VERGQ2Nm8GliwR71euBOrWlbceM8UAaEwODmI0To7zVlKdOnXQrFkzAMCyZcsQGBiIr776Cm3atAEAbN68GQ0aNND5TkXa5JWst5STkwNXV1edn928eRMuLi6VrpWIiMxEdjbwyivi/bhxQLduspZjzhgAjUmlqvStWCWwsrLC1KlTERsbi1OnTsHOzg4ZGRlVut1b0os3JSVF+0whAPz999/IyclB8+bNDVk6ERGZCkkChg8XIbB1a+Df9q9UMxgAqUL69euHiRMn4vPPP8eECRMwfvx4aDQaPPHEE8jJycEff/wBZ2dnxMTEaL+TlpZW5jitW7fGq6++ijfffBM2NjZo27Ytzp8/j0mTJqF9+/Z4/PHHjXlZRESkFF99Bfz4I2BrC6xaBbBrU41iAKQKsbGxwejRozFnzhycPXsW9evXR1xcHP7++2+4urrisccew9SpU3W+8+KLL5Y5zvnz5/Hpp58iPj4ekyZNwrlz5+Dl5YWnnnoKH374IVRc74mIyPKcPi1u+QLABx8AgYGylmMJ2Au4GtgL2HTw/w8iIoW6exfo2BHYuxfo3BlISgKsrWv0lOwFzIWgiYiISE5xcSL8OTuLWb81HP5IYAAkIiIieezfD8ycKd5/9hlwz+RAqlkMgERERGR8Jd0+iouB/v2BgQPlrsiiMAASERGR8d3b7WPx4mq1LKXKYwAkIiIi42K3D9kxABIREZHxsNuHIjAAEhERkXGw24diMAASERGRcbDbh2IwABIREVHNY7cPRWEApErp0qULxpX8C1xN//zzD1QqFVJTUw1yPCIiUqi7d4GXXxZLv3TuDMTGyl2RxWMApDKGDBkClUpV5nX69Gm5SwMArF+/Hk899RTq168PZ2dnRERE4Oeff5a7LCIiKg+7fSgOAyDp1b17d1y+fFnn1aRJE7nLAgDs2rULTz31FLZs2YKUlBR07doVvXv3xuHDh+UujYiI7sduH4rEAEh62dnZwcvLS+dlree/2P73v/8hJCQETk5O8PLywsCBA5Gdna39+Y0bNzBo0CDUr18ftWvXhr+/P5YvX65zjL///htdu3aFg4MDAgMDkZyc/MDaEhIS8NZbbyE0NBT+/v6YNWsW/P39sXHjRsNcPBERGQa7fSiWjdwFWBJJAvLzjX9eB4eaW2C9qKgI77//Plq0aIHs7GzExsZiyJAh2LJlCwDg3XffxYkTJ/DTTz/B3d0dp0+fxu3bt3WO8fbbb2Pu3Lnw9/fH22+/jQEDBuD06dOwsanYP54ajQa5ublwc3Mz+PUREVE1sNuHYjEAGlF+PuDoaPzz3roF1KlTue9s2rQJjvcU26NHD6xdu7bMfq+ULOYJoGnTppg/fz5CQ0Nx69YtODo6IiMjA+3atUNISAgAwM/Pr8wxJkyYgF69egEAZs6cidatW+P06dNo2bJlhWqdO3cubt26hRdeeKEyl0hERDWJ3T4UjbeASa+uXbsiNTVV+5o/f77e/VJSUtC7d280atQITk5O6Ny5MwAgIyMDADBy5EgkJiYiKCgIb731Fvbs2VPmGAEBAdr33t7eAKC9jezo6Kh9vf7662W+u3r1asycORPfffcdPDw8qnfRRERkGOz2oXgcATQiBwcxGifHeSurTp06aNas2QP3ycvLQ1RUFKKiorBq1SrUr18fGRkZiIqKQmFhIQAxcnju3Dls2bIF27ZtQ7du3TBq1CjMnTtXe5xatWpp36v+vT2g0WgAQGeJGGdnZ53zJyYm4tVXX8XatWsRGRlZ+YskIiLDY7cPk8AAaEQqVeVvxSrZyZMnce3aNcTHx8PX1xcAcPDgwTL71a9fHzExMYiJiUHHjh0xceJEnQD4IOWF0G+//RavvPIKEhMTtbePiYhIAdjtwyQwAFKVNWrUCLa2tliwYAFef/11HDt2DO+//77OPtOmTUNwcDBat26NgoICbNq0CY8++mi1zrt69WrExMTg008/RXh4ODIzMwEAtWvXhouLS7WOTURE1cBuHyaDzwBSldWvXx8rVqzA2rVr0apVK8THx5cZ2bO1tcWUKVMQEBCATp06wdraGomJidU679KlS3H37l2MGjUK3t7e2tfYsWOrdVwiIqoGdvswKSpJkiS5izBVarUaLi4uyMnJKfN82p07d3D27Fk0adIE9hz+lh3//yAiqmHvvw9Mmya6fRw5ougFnx/0+9tScASQiIiIqofdPkwOAyARERFVHbt9mCQGQCIiIqo6dvswSQyAREREVDXs9mGyGACJiIio8tjtw6QpJgAuWrQIfn5+sLe3R3h4OPbv31/uvsePH0d0dDT8/PygUqmQkJBQZp+Sn93/GjVqlHafLl26lPm5vnZj1cFJ1srA/x+IiAyI3T5MniIC4Jo1axAbG4vp06fj0KFDCAwMRFRUlLYf7P3y8/PRtGlTxMfHw8vLS+8+Bw4cwOXLl7Wvbdu2AQD69euns9/w4cN19pszZ45BrqmkvVl+fr5BjkfVU/L/w71t54iIqIrY7cPkKaITyLx58zB8+HAMHToUALBkyRJs3rwZy5Ytw+TJk8vsHxoaitDQUADQ+3NALFJ8r/j4eDzyyCPo3LmzznYHB4dyQ2R1WFtbw9XVVRtiHRwctH1uyXgkSUJ+fj6ys7Ph6uoKa2truUsiIjJt7PZhFmQPgIWFhUhJScGUKVO026ysrBAZGYnk5GSDneObb75BbGxsmRC2atUqfPPNN/Dy8kLv3r3x7rvvwsHBQe9xCgoKUFBQoP2sVqsfeN6SYFneSCYZj6ura40EfSIii8JuH2ZD9gB49epVFBcXw9PTU2e7p6cnTp48aZBzbNiwATdv3sSQIUN0tg8cOBCNGzeGj48Pjhw5gkmTJiEtLQ3r16/Xe5y4uDjMLFnosgJUKhW8vb3h4eGBoqKi6lwCVUOtWrU48kdEZAhxccDevaLbx8qVAP9uNVmyB0Bj+Oqrr9CjRw/4+PjobB8xYoT2fdu2beHt7Y1u3brhzJkzeOSRR8ocZ8qUKYi957921Go1fH19H3p+a2trBhAiIjJt7PZhVmQPgO7u7rC2tkZWVpbO9qysLIPcsjt37hx+/fXXckf17hUeHg4AOH36tN4AaGdnBzs7u2rXREREZFLY7cPsyD4L2NbWFsHBwUhKStJu02g0SEpKQkRERLWPv3z5cnh4eKBXr14P3Tc1NRUA4O3tXe3zEhERmQ12+zA7so8AAkBsbCxiYmIQEhKCsLAwJCQkIC8vTzsrePDgwWjQoAHi/l1nqLCwECdOnNC+v3jxIlJTU+Ho6IhmzZppj6vRaLB8+XLExMTAxkb3Us+cOYPVq1ejZ8+eqFevHo4cOYLx48ejU6dOCAgIMNKVExERKRy7fZglRQTA/v3748qVK5g2bRoyMzMRFBSErVu3aieGZGRkwMqqdLDy0qVLaNeunfbz3LlzMXfuXHTu3Bk7duzQbv/111+RkZGBV0pWKr+Hra0tfv31V23Y9PX1RXR0NN55552au1AiIiJTwm4fZkslsUVClanVari4uCAnJwfOzs5yl0NERGQ4kgT06SMWfG7dGjh40GwWfObvbwU8A0hEREQKxG4fZo0BkIiIiHSx24fZYwAkIiKiUuz2YREYAImIiKgUu31YBAZAIiIiEtjtw2IwABIRERG7fVgYBkAiIiJitw8LwwBIRERk6djtw+IwABIREVkydvuwSAyARERElkqSgOHDRQhs3VrMACaLwABIRERkqdjtw2IxABIREVkidvuwaAyARERElobdPiweAyAREZGlYbcPi8cASEREZEnY7YPAAEhERGQ52O2D/sUASEREZCnY7YP+xQBIRERkCdjtg+7BAEhERGTu2O2D7sMASEREZM7Y7YP0YAAkIiIyZ+z2QXowABIREZkrdvugcjAAEhERmSN2+6AHYAAkIiIyR+z2QQ/AAEhERGRu2O2DHoIBkIiIyJyw2wdVAAMgERGROWG3D6oABkAiIiJzwW4fVEEMgEREROaA3T6oEhgAiYiITJ0ZdvvQaIBt2+SuwnwxABIREZk6M+z2MWUK8PTTwPTpcldinhgAiYiITJkZdvtYuhSYM0e8b95c3lrMFQMgERGRqTLDbh+//AL897/i/cyZwKBB8tZjrhgAiYiITNWsWWbV7ePYMaBvX7GE4csvA+++K3dF5osBkIiIyBTt3w+89554bwbdPjIzgV69gNxcoFMn4IsvuIRhTWIAJCIiMjVm1u0jPx/o3RvIyBDP/H3/PWBnJ3dV5o0BkIiIyNSYUbeP4mKRZQ8eBOrVE2tZu7nJXZX5U0wAXLRoEfz8/GBvb4/w8HDs37+/3H2PHz+O6Oho+Pn5QaVSISEhocw+M2bMgEql0nm1bNlSZ587d+5g1KhRqFevHhwdHREdHY2srCxDXxoREZHhmFm3j0mTxIifrS3www9As2ZyV2QZFBEA16xZg9jYWEyfPh2HDh1CYGAgoqKikJ2drXf//Px8NG3aFPHx8fDy8ir3uK1bt8bly5e1r927d+v8fPz48di4cSPWrl2LnTt34tKlS3j++ecNem1EREQGY2bdPhYvBj7+WLxfsQLo0EHWciyKIgLgvHnzMHz4cAwdOhStWrXCkiVL4ODggGXLlundPzQ0FB999BFefPFF2D3gIQEbGxt4eXlpX+7u7tqf5eTk4KuvvsK8efPw5JNPIjg4GMuXL8eePXuwd+9eg18jERFRtZhZt4+ffgJGjxbvP/gAGDBA3nosjewBsLCwECkpKYiMjNRus7KyQmRkJJKTk6t17PT0dPj4+KBp06YYNGgQMjIytD9LSUlBUVGRznlbtmyJRo0aVfu8REREBmdG3T7+/BN44QXR7m3IEGDqVLkrsjyyB8CrV6+iuLgYnp6eOts9PT2RmZlZ5eOGh4djxYoV2Lp1KxYvXoyzZ8+iY8eOyM3NBQBkZmbC1tYWrq6uFT5vQUEB1Gq1zouIiKjGmVG3j0uXgP/8B7h1C+jaFfj8c5Oew2KybOQuoKb06NFD+z4gIADh4eFo3LgxvvvuOwwbNqxKx4yLi8PMmTMNVSIREdHDmVG3j1u3RPi7cAFo2RL4v/8TA5pkfLKPALq7u8Pa2rrM7NusrKwHTvCoLFdXVzRv3hynT58GAHh5eaGwsBA3b96s8HmnTJmCnJwc7ev8+fMGq4+IiEgvM+n2UVwslis8fBioX19MZjbxCcwmTfYAaGtri+DgYCQlJWm3aTQaJCUlISIiwmDnuXXrFs6cOQNvb28AQHBwMGrVqqVz3rS0NGRkZJR7Xjs7Ozg7O+u8iIiIaowZdft4801g40bx6OKPPwJNm8pdkWVTxC3g2NhYxMTEICQkBGFhYUhISEBeXh6GDh0KABg8eDAaNGiAuH9nPBUWFuLEiRPa9xcvXkRqaiocHR3R7N8FhCZMmIDevXujcePGuHTpEqZPnw5ra2sM+HeakYuLC4YNG4bY2Fi4ubnB2dkZY8aMQUREBNq3by/DnwIREdE9zKjbx4IFwKefivdffw3w16z8FBEA+/fvjytXrmDatGnIzMxEUFAQtm7dqp0YkpGRASur0sHKS5cuoV27dtrPc+fOxdy5c9G5c2fs2LEDAHDhwgUMGDAA165dQ/369fHEE09g7969qF+/vvZ7n3zyCaysrBAdHY2CggJERUXhs88+M85FExERPYiZdPvYtKl0/kp8PNCvn6zl0L9UkiRJchdhqtRqNVxcXJCTk8PbwUREZDibN4vZEgDw668mu+Dz4cNAx45iMHPYMOCLL5SRY/n7WwHPABIREdE9zKTbx4ULIsPm5YlLMOFBTLPEAEhERKQUZtLtIzcX6N1brPnXqhWwbh1Qq5bcVdG9GACJiIiUwgy6fdy9C7z4IpCaCnh4iLvZ9/VcIAVgACQiIlICM+j2IUniErZsEdl140bAz0/uqkgfBkAiIiK5mUm3j/nzgUWLxLN+33wDhIXJXRGVhwGQiIhIbmbQ7eOHH4Dx48X7OXOA6Gh566EHYwAkIiKSkxl0+0hJEetUSxLw2mui6wcpGwMgERGRXMyg20dGhljuJT8fiIoCFi7kci+mgAGQiIhILibe7UOtFuEvMxNo2xb47jvARhE9xuhhGACJiIjksHkzsGSJeL9yJVC3rrz1VNLdu8ALLwBHjwJeXqLlm4U21TBJDIBERETGZuLdPiQJGD0a+PlnwMFBhL9GjeSuiiqDAZCIiMiYzKDbx7x5wOefizvWq1cDwcFyV0SVxQBIRERkTCbe7WP9emDiRPH+44+BZ5+Vtx6qGgZAIiIiYzHxbh8HDohJy5IE/Pe/pZdCpocBkIiIyBhMvNvHuXNA797A7dtAjx7Ap5+a3KRlugcDIBERkTGYcLePnBygVy8gKwsICADWrOFyL6aOAZCIiKimmXC3j6IioF8/4PhxwMdHrF7j5CR3VVRdDIBEREQ1yYS7fZQ867dtG1CnDrBxI9CwodxVkSEwABIREdUkE+728dFHwJdfAlZWQGIi8NhjcldEhsIASEREVFNMuNvHunXApEnifUKCaPlG5oMBkIiIqCaYcLePvXvFhGUAeOMNYMwYeeshw2MAJCIiMjQT7vZx9izwzDPAnTti2Zd58+SuiGoCAyAREZGhmWi3jxs3gJ49gStXgHbtRJs3E1qthiqBAZCIiMiQTLTbR2Eh0LcvcPKkmOm7aRPg6Ch3VVRTGACJiIgMxUS7fUgS8PrrwPbtIvRt2iTW/CPzxQBIRERkKCba7SMuDli+XCz38t13JjNoSdXAAEhERGQIJtrtY80a4O23xfsFC0SfXzJ/DIBERETVZaLdPvbsAWJixPvx40XXD7IMDIBERETVZYLdPs6cAZ59FigoEP/70UdyV0TGxABIRERUHSbY7eP6daBXL+DqVSA4WKxUYyKPK5KBMAASERFVlQl2+ygsBJ5/HkhLA3x9gY0bgTp15K6KjI0BkIiIqCpMsNtHSck7dwJOTmLw0ttb7qpIDgyAREREVWGC3T4++AD4+mtxu3ftWqBtW7krIrkwABIREVWWCXb7WLUKmDZNvP/sMyAqSt56SF4MgERERJVhgt0+fv+99FHFiROBESPkrYfkxwBIRERUGSbW7SM9HejTR0z+iI4G4uPlroiUgAGQiIiookys28e1a0DPnmLZl7Aw8fyfFX/zExgAiYiIKsbEun0UFIiRv9OnRU798UfAwUHuqkgpFBMAFy1aBD8/P9jb2yM8PBz79+8vd9/jx48jOjoafn5+UKlUSEhIKLNPXFwcQkND4eTkBA8PD/Tp0wdpaWk6+3Tp0gUqlUrn9frrrxv60oiIyByYULcPSRLP/O3eDbi4AFu2AJ6ecldFSqKIALhmzRrExsZi+vTpOHToEAIDAxEVFYXs7Gy9++fn56Np06aIj4+Hl5eX3n127tyJUaNGYe/evdi2bRuKiorw9NNPIy8vT2e/4cOH4/Lly9rXnDlzDH59RERk4kys28eMGcDq1YCNDbBuHdCqldwVkdKoJEmS5C4iPDwcoaGhWLhwIQBAo9HA19cXY8aMweTJkx/4XT8/P4wbNw7jSqbjl+PKlSvw8PDAzp070alTJwBiBDAoKEjvCGJFqNVquLi4ICcnB87OzlU6BhERKVx2tlgwLztbLP3yySdyV/RAX38NxMSI9198Abz6qrz1KBF/fytgBLCwsBApKSmIjIzUbrOyskJkZCSSk5MNdp6cnBwAgJubm872VatWwd3dHW3atMGUKVOQn59f7jEKCgqgVqt1XkREZMZMrNvHzp2lgW/yZIY/Kp+N3AVcvXoVxcXF8Lzv4QRPT0+cPHnSIOfQaDQYN24cOnTogDZt2mi3Dxw4EI0bN4aPjw+OHDmCSZMmIS0tDevXr9d7nLi4OMycOdMgNRERkQkwoW4faWnAc88BRUVAv37Ahx/KXREpmewB0BhGjRqFY8eOYffu3TrbR9yzEmbbtm3h7e2Nbt264cyZM3jkkUfKHGfKlCmIvWfBT7VaDV9f35ornIiI5GNC3T6uXBHLvdy4AbRvLx5T5HIv9CCyB0B3d3dYW1sjKytLZ3tWVla5EzwqY/To0di0aRN27dqFhg0bPnDf8PBwAMDp06f1BkA7OzvY2dlVuyYiIlI4E+r2ceeOWO7l77+BJk2AH34AateWuypSOtn/+8DW1hbBwcFISkrSbtNoNEhKSkJERESVjytJEkaPHo3vv/8e27dvR5MmTR76ndTUVACAt7d3lc9LRERmwES6fWg0wJAhwJ49gKurmKzs4SF3VWQKZB8BBIDY2FjExMQgJCQEYWFhSEhIQF5eHoYOHQoAGDx4MBo0aIC4fx++LSwsxIkTJ7TvL168iNTUVDg6OqJZs2YAxG3f1atX44cffoCTkxMyMzMBAC4uLqhduzbOnDmD1atXo2fPnqhXrx6OHDmC8ePHo1OnTggICJDhT4GIiBTBhLp9TJsGrFkjlntZvx549FG5KyKTISnEggULpEaNGkm2trZSWFiYtHfvXu3POnfuLMXExGg/nz17VgJQ5tW5c2ftPvp+DkBavny5JEmSlJGRIXXq1Elyc3OT7OzspGbNmkkTJ06UcnJyKlxzTk6OBKBS3yEiIgW7dUuS/P0lCZCk/v0lSaORu6JyLVsmygQk6d9fbVRB/P0tSYpYB9BUcR0hIiIzM3KkWPC5QQPg6FHFLvi8fTsQFSUeVXznHeD99+WuyLTw97cCngEkIiJSBBPp9vHXX8Dzz4vwN2BA6d1qospgACQiIsrOFs1zAbH0S7duspZTnqwssdxLTg7QoQOwbJmiWxKTgjEAEhGRZTORbh+3bwPPPgv88w/wyCPAhg2KXpeaFI4BkIiILJsJdPvQaIDBg4F9+8Sd6S1bAHd3uasiU8YASERElstEun1MnQqsWwfUqgV8/z3QvLncFZGpYwAkIiLLZCLdPr74Apg9W7z/6itRKlF1MQASEZFlMoFuH9u2iZVpAGD6dJFXiQyBAZCIiCyPCXT7OH4c6NsXKC4GXnpJBEAiQ2EAJCIiy5KXJxJVcTHQvz8wcKDcFZWRmQn06gWo1UDHjsCXX3K5FzIsBkAiIrIsEyYA6emi28fixYpLVvn5wDPPAOfOAf7+YtKHnZ3cVZG5YQAkIiLLofBuHxqNGJw8cACoV08s91KvntxVkTliACQiIstgAt0+Jk0SI362tmKh52bN5K6IzBUDIBERmT8T6PaxZAkwd654v2IF8MQTspZDZo4BkIiIzJ/Cu31s3QqMHi3ev/8+MGCAvPWQ+WMAJCIi86bwbh9HjgAvvCAmJcfEAG+/LXdFZAkYAImIyHwpvNvHpUtiuZfcXKBLF2DpUsVNSiYzxQBIRETmS8HdPvLygN69gQsXgBYtgPXrxR1qImNgACQiIvOk4G4fxcVi/elDhwB3d7Hci8JWpCEzxwBIRETmR+HdPiZMEHNS7OyAH34AmjaVuyKyNAyARERkfhTc7WPRIiAhQbxfuRJ4/HFZyyELVeUAuHLlSmzevFn7+a233oKrqysef/xxnDt3ziDFERERVZqCu31s2QK88YZ4P2uWGJwkkkOVA+CsWbNQu3ZtAEBycjIWLVqEOXPmwN3dHePHjzdYgURERBWm4G4fqaki8Gk0osTJk+WuiCyZTVW/eP78eTT7t0fNhg0bEB0djREjRqBDhw7o0qWLoeojIiKqGAV3+7h4EfjPf4Bbt0QmXbJEUXelyQJVeQTQ0dER165dAwD88ssveOqppwAA9vb2uH37tmGqIyIiqiiFdvu4dUuEv4sXgUcfBdatA2rVkrsqsnRVHgF86qmn8Oqrr6Jdu3Y4deoUevbsCQA4fvw4Gitoqj0REVkAhXb7KC4GXnxR3P718BCPJ7q6yl0VUTVGABctWoSIiAhcuXIF//d//4d69eoBAFJSUjBQYdPtiYjIjCm428f48SL02duLwckmTeSuiEhQSZIkVfXLd+7cwZEjR5CdnQ2NRqPzs2eeeabaxSmdWq2Gi4sLcnJy4OzsLHc5RESW6b33gOnTRbePI0cUs+Dz/PnA2LHi/bp1QHS0vPVQKf7+rsYt4K1bt2Lw4MG4du0a7s+QKpUKxcXF1S6OiIjogRTa7ePHH0vvSM+Zw/BHylPlW8BjxoxBv379cOnSJWg0Gp0Xwx8REdU4hXb7SEkBBgwQk5JHjBBrUhMpTZUDYFZWFmJjY+Hp6WnIeoiIiCpGgd0+zp8HevcG8vOBp58GFi5URFlEZVQ5APbt2xc7duwwYClEREQVpMBuH2o10KsXcPmyWIbwu++43AspV5UngeTn56Nfv36oX78+2rZti1r3/VP+RkmvGzPGh0iJiGSQnQ20bSv+d9w44JNP5K4Id++Kkb+tWwFPT2DfPsU8jkh68Pd3NSaBfPvtt/jll19gb2+PHTt2QHXPGLdKpbKIAEhEREamwG4fkiT6+27dCtSuDWzcyPBHylflAPj2229j5syZmDx5MqysqnwnmYiIqOIU2O0jIaH0EcRVq4DQULkrInq4Kie3wsJC9O/fn+GPiIiMQ4HdPjZsAN58U7yfOxd47jlZyyGqsCqnt5iYGKxZs8aQtRAREemnwG4fBw+KlWckCRg5UnT9IDIVVb4FXFxcjDlz5uDnn39GQEBAmUkg8+bNq3ZxREREAIBZs4C9e0W3j5UrAWtrWcs5d05M+rh9G+jeXXT94HIvZEqqPAJ49OhRtGvXDlZWVjh27BgOHz6sfaWmplb6eIsWLYKfnx/s7e0RHh6O/fv3l7vv8ePHER0dDT8/P6hUKiQkJFTpmHfu3MGoUaNQr149ODo6Ijo6GllZWZWunYiIapDCun3k5AD/+Q+QmQkEBABr1gA2VR5OIZJHlf+R/e233wxWxJo1axAbG4slS5YgPDwcCQkJiIqKQlpaGjw8PMrsn5+fj6ZNm6Jfv34YX86Ye0WOOX78eGzevBlr166Fi4sLRo8ejeeffx5//PGHwa6NiIiqQWHdPoqKgH79gGPHAG9vYNMmMShJZHIkBQgLC5NGjRql/VxcXCz5+PhIcXFxD/1u48aNpU8++aTSx7x586ZUq1Ytae3atdp9/vrrLwmAlJycXKG6c3JyJABSTk5OhfYnIqJKev11SQIkqUEDSbp+XdZSNBpJGj5clOPgIEkpKbKWQ9XA39+SJPsU3sLCQqSkpCAyMlK7zcrKCpGRkUhOTq6xY6akpKCoqEhnn5YtW6JRo0ZVPi8RERmQwrp9zJ0LfPGFeNYvMRF47DFZyyGqFtmfWrh69SqKi4vL9BT29PTEyZMna+yYmZmZsLW1haura5l9MjMz9R63oKAABQUF2s9qtbpK9RER0UNkZwOvvCLejxsHdOsmaznr1gFvvSXeJySICSBEpkz2EUBTEhcXBxcXF+3L19dX7pKIiMyPwrp97NsnVqABgDFjRNcPIlMnewB0d3eHtbV1mdm3WVlZ8PLyqrFjenl5obCwEDdv3qzweadMmYKcnBzt6/z581Wqj4iIHkBB3T7OngWeeQa4cwfo1UsRbYeJDEL2AGhra4vg4GAkJSVpt2k0GiQlJSEiIqLGjhkcHIxatWrp7JOWloaMjIxyz2tnZwdnZ2edFxERGZCCun3cvClCX3Y2EBQknvuTeflBIoOR/RlAAIiNjUVMTAxCQkIQFhaGhIQE5OXlYejQoQCAwYMHo0GDBoj79zZAYWEhTpw4oX1/8eJFpKamwtHREc2aNavQMV1cXDBs2DDExsbCzc0Nzs7OGDNmDCIiItC+fXsZ/hSIiCycgrp9FBYC0dHAX38BDRqI5V4cHWUrh8jgFBEA+/fvjytXrmDatGnIzMxEUFAQtm7dqp3EkZGRodNz+NKlS2jXrp3289y5czF37lx07twZO3bsqNAxAeCTTz6BlZUVoqOjUVBQgKioKHz22WfGuWgiItKlkG4fJa3dtm8H6tQR4a9BA1lKIaoxKkmSJLmLMFVqtRouLi7Iycnh7WAiourYvx94/HGx4PM33wCDBslWSnw8MGUKYGUlHkXs1Uu2UqiG8Pe3Ap4BJCIiC6egbh/ffSfCHyD6+zL8kbliACQiInlNmACkp4v7rIsXi5WWZZCcDAweLN6PGweMGiVLGURGwQBIRETyUUi3j7//Fsu9FBSI/507V5YyiIyGAZCIiOShkG4fN24APXsCV6+K9m6rV3O5FzJ/DIBERGR8Cun2UVgIPP88kJYG+PoCGzeKmb9E5o4BkIiIjE8B3T4kCRgxAtixA3ByEsu9+PgYvQwiWTAAEhGRcSmk28eHH5YuN7h2LRAQIEsZRLJgACQiIuNRSLeP1auBd98V7xctAqKiZCmDSDYMgEREZDwK6Paxezfwb1dQTJgAvPaa0Usgkh0DIBERGcf+/cB774n3n30GNG5s9BLS04E+fcTkj+eeA2bPNnoJRIrAAEhERDVPAd0+rl0TnT2uXQNCQ0XHOSv+FiQLxX/0iYio5snc7aOgQIz4pacDjRqJCcgODkYtgUhRGACJiKhmydztQ5KAV18Ffv9dPHq4eTPg5WXUEogUhwGQiIhqjgK6fbz3nrjda20NrFsHtGlj9BKIFIcBkIiIaoYCun188w0wY4Z4v3gx8NRTRi+BSJEYAImIqGbI3O1j167SwcdJk0QWJSKBAZCIiAxP5m4faWliuZeiIqBvX7H8IBGVYgAkIiLDkrnbx9WrYrmXGzeA8HDg66+53AvR/fivBBERGZaM3T7u3BEjf2fOAH5+4g507dpGOz2RyWAAJCIiw5Gx24dGI1q8/fEH4OICbNkCeHgY7fREJoUBkIiIDEPmbh/TpwOJiYCNDbB+PfDoo0Y9PZFJYQAkIiLDkLHbx4oVYq4JACxdCjz5pNFOTWSSGACJiKj6ZOz2sX176RIvU6eK28BE9GAMgEREVD0ydvv46y8gOlpMPO7fH3j/faOdmsikMQASEVHV/f23SGAydPvIzhbLvdy8CTz+uLgNzOVeiCqG/6oQEVHl3bkjhttatwZ27xZrrRix28ft28CzzwJnzwJNmwIbNhi90QiRSWMAJCKiyvnlF6BtW2DaNBEEu3UDDh0yWrcPjQaIiRFLDdatK5Z7qV/fKKcmMhsMgEREVDEXLgAvvABERYlWb97ewLffAtu2AS1bGq2Md94B1q4FatUSy720aGG0UxOZDQZAIiJ6sKIiYN48sbDe2rXiQbtx44CTJ4EXXzTqci9ffVX6mOGXXwJduhjt1ERmxUbuAoiISMF27wZGjgSOHROfIyJEh4+gIKOX8uuvwOuvi/fTpgGDBxu9BCKzwRFAIiIqKztbLKjXsaMIf/XqiSG33btlCX/Hj5cu9zJwIDBjhtFLIDIrHAEkIqJSxcXAF18AU6aI9VUAscpyXJwIgTLIyhLLvajVwBNPAMuWGfWuM5FZYgAkIiIhJUXc7j1wQHwOChIt3dq3l62k/HzgmWeAc+cAf3+x3IudnWzlEJkN3gImIrJ0N28Co0cDoaEi/Dk7A/Pni/cyhj+NBnj5ZWD/fsDNTXSbk2kQksjscASQiMhSSRLwzTfAhAnimT9APGA3d65Y4kVmkyeLZV5sbcXIn7+/3BURmQ8GQCIiS3T8OPDf/wK7donPLVsCixYBTz4pb13/+vxz4KOPxPvly8VcFCIyHN4CJiKyJLduAZMmief7du0SLdzi4oA//1RM+Pv5Z2DUKPF+5kwxKElEhsURQCIiSyBJwPffA2PHio4egGimm5AA+PnJWZmOo0eBfv3EZOSXXwbefVfuiojMk2JGABctWgQ/Pz/Y29sjPDwc+/fvf+D+a9euRcuWLWFvb4+2bdtiy5YtOj9XqVR6Xx+V3FMA4OfnV+bn8fHxNXJ9RESyOXNGrKMSHS3Cn58fsHGjeLBOQeHv8mVRZm4u0LmzWI2Gy70Q1QxFBMA1a9YgNjYW06dPx6FDhxAYGIioqChklzyUfJ89e/ZgwIABGDZsGA4fPow+ffqgT58+OFayUj2Ay5cv67yWLVsGlUqF6OhonWO99957OvuNGTOmRq+ViMho7twR91BbtwZ++knMpnjnHfH833/+I3d1OvLygN69gfPngebNxeQPLvdCVHNUkiRJchcRHh6O0NBQLFy4EACg0Wjg6+uLMWPGYPLkyWX279+/P/Ly8rBp0ybttvbt2yMoKAhLlizRe44+ffogNzcXSUlJ2m1+fn4YN24cxo0bV6W61Wo1XFxckJOTA2dn5yodg4ioRmzdKpZ2OXNGfI6MBBYuBFq0kLcuPYqLxeDkDz8A7u7A3r3AI4/IXRWZM/7+VsAIYGFhIVJSUhAZGandZmVlhcjISCQnJ+v9TnJyss7+ABAVFVXu/llZWdi8eTOGDRtW5mfx8fGoV68e2rVrh48++gh3796txtUQEcns/Hmgb1+gRw8R/ry9gcRE4JdfFBn+AOCtt0T4s7MTd6UZ/ohqnuyTQK5evYri4mJ4enrqbPf09MTJkyf1ficzM1Pv/pmZmXr3X7lyJZycnPD888/rbH/jjTfw2GOPwc3NDXv27MGUKVNw+fJlzJs3T+9xCgoKUFBQoP2sVqsfen1EREZRVAR8+qlokpuXB1hbA2+8IT4reITjs8+Akr9yV6wAOnSQtRwiiyF7ADSGZcuWYdCgQbC3t9fZHhsbq30fEBAAW1tbvPbaa4iLi4OdnodP4uLiMHPmzBqvl4ioUn7/XbRwO35cfH78cZGsAgPlreshtmwBSh67/vBD4MUX5a2HyJLIfgvY3d0d1tbWyMrK0tmelZUFLy8vvd/x8vKq8P6///470tLS8Oqrrz60lvDwcNy9exf//POP3p9PmTIFOTk52tf58+cfekwiohqTnQ3ExACdOonwV68e8NVXIhAqPPz9+SfQv79o9zZ0KDBlitwVEVkW2QOgra0tgoODdSZnaDQaJCUlISIiQu93IiIidPYHgG3btund/6uvvkJwcDACK/CXYWpqKqysrODh4aH353Z2dnB2dtZ5EREZXXExsHixeKbv66/FWikjRgBpacArrwBWsv/V/kAXL4rlXm7dEmtPL1nC5V6IjE0Rt4BjY2MRExODkJAQhIWFISEhAXl5eRg6dCgAYPDgwWjQoAHi4uIAAGPHjkXnzp3x8ccfo1evXkhMTMTBgwexdOlSneOq1WqsXbsWH3/8cZlzJicnY9++fejatSucnJyQnJyM8ePH46WXXkLdunVr/qKJiKri4EFxu/fgQfG5XTsRBsPD5a2rgm7dEsu9XLwIPPoo8H//J1anISLjUkQA7N+/P65cuYJp06YhMzMTQUFB2Lp1q3aiR0ZGBqzu+S/axx9/HKtXr8Y777yDqVOnwt/fHxs2bECbNm10jpuYmAhJkjBgwIAy57Szs0NiYiJmzJiBgoICNGnSBOPHj9d5LpCISDFu3ADeflsMl0mSmNjx4YciDFpby11dhRQXAwMGAIcPA/XrA5s3A66ucldFZJkUsQ6gqeI6QkRU4yQJ+N//gAkTgCtXxLZBg4C5c4FynpNWqrFjgfnzAXt74LffgPbt5a6ILBV/fytkBJCIiPQ4dgz473/FpA5A3DNdtAjo2lXeuqpg/nzxAkSeZfgjkpeynxQmIrJEt24BEyeK5/t+/x1wcADi44HUVJMMfxs3AuPHi/fx8WKdaiKSF0cAiYiUQpJEE9xx44ALF8S2Pn2AhASgcWMZC6u6Q4fE+n4aDfDqq6LrBxHJjwGQiEgJTp8WqyJv3So+N2kCLFgg1ksxUefPA//5D5CfL1oRf/YZl3shUgreAiYiktOdO6JdW5s2IvzZ2gLvvisWdjbh8JebK8Lf5ctAq1bAunVArVpyV0VEJTgCSEQkl59+EqN+Z86Iz089BSxcCDRvLm9d1XT3rrjte+QI4OkplntxcZG7KiK6F0cAiYiM7fx5IDoa6NlThD8fH+C774Cffzb58CdJYrmXLVuA2rWBH38E/PzkroqI7scASERkLEVFwEcfieVc1q8XCzjHxgInTwL9+pnFA3Kfflr6rN+qVUBYmNwVEZE+vAVMRGQMu3aJNf2OHxefO3QQLdzatpW3LgP64QeRZwGRc597Tt56iKh8HAEkIqpJWVnA4MFA584i/Lm7A8uXi0BoRuHv4EFg4EBxC/j110uDIBEpEwMgEVFNKC4W90JbtBCtL1Qq4LXXgLQ0YMgQwMp8/vrNyAB69xbLvXTvLlavMYO72URmjbeAiYgM7cABYORIICVFfH7sMXG718weiDt3DliyBPjiC+DaNTGguWYNYMPfLESKx39NiYgM5cYNYOpU4PPPxb1QFxfgww/FPVFra7mrMwhJAn77TYzy/fij6PABAP7+wKZNgLOzvPURUcUwABIRVZckAStXij5nV66IbS+9JGZCeHnJW5uB3LoFfP21WKbwr79Kt3frBoweLRZ95sgfkengv65ERNVx9KiY3bt7t/jcqhWwaBHQpYusZRnKqVPiclasANRqsa1OHSAmBhg1SlwuEZkeBkAioqrIzQVmzgQSEsSEDwcHYPp0YNw40c7NhBUXiyYlCxeKtalL+PuL0b6YGHb2IDJ1DIBERJUhSaKx7fjxwMWLYttzz4kg2KiRrKVV140bwLJlYvLy33+LbSqVaEk8erToVGdGk5eJLBoDIBFRRaWniyT0yy/ic9OmYjZEz57y1lVNR46I0b5vvgFu3xbbXF2BYcPE3e2mTWUtj4hqAAMgEdHD3L4NxMeLV2GhuMU7ebJ41a4td3VVUlQEbNgg8uvvv5duDwgAxowRizo7OMhWHhHVMAZAIqIH2bJFJKKSe6JPPy2Gy/z95a2rirKygKVLxfp9ly6JbdbWwPPPi8t84gku4kxkCRgAiYj0ycgQEzq+/158btBAPOcXHW1yCUmSgP37xWjfd9+J0T8A8PAQzUlee01cHhFZDgZAIqJ7FRaKoDdzpuhtZm0tguD06YCTk9zVVcqdO6Izx8KFoldvifbtxaOMffsCdnby1UdE8mEAJCIqsXOnmPVw4oT4/MQTYkps27by1lVJGRmlLdquXhXb7OyAF18UwS8kRN76iEh+DIBERJmZwMSJYhosANSvL7p4DB5sMrd7JQnYsUOM9m3YUNqizddXtCV+9VVxWUREAAMgEVmy4mIxVPb220BOjgh7r78u+vfWrSt3dRVy65bIrQsXAsePl27v2lWM9j3zDFu0EVFZ/GuBiCzT/v1iaOzQIfE5OBhYvBgIDZW3rgpKTxd3p5cvF9kVEMu2DB4sgl/r1vLWR0TKxgBIRJbl+nVg6lSxFookiZ5ms2aJqbDW1nJX90AaDbB1q5jNu3Vr6fZmzUpbtLm6ylYeEZkQBkAisgwaDfD11+JZv5KZEYMHA3PmAJ6e8tb2EDdvipG+RYuAM2fENpVKNCAZPVosTcgWbURUGQyARGT+jh4Vs3t37xafW7US9087d5a3roc4erS0RVt+vtjm6gq88oq4e92smazlEZEJYwAkIvOVmwvMmAF8+qmY8OHgID6PGwfUqiVzcfrdvStm8S5cKFalKdGmjejUMWgQUKeObOURkZlgACQi8yNJwNq1wPjxpf3OoqOBTz4R66IoUHa2WLdvyRLgwgWxzdoaeO45cZu3UyeTWZGGiEwAAyARmZdTp0Ri2rZNfH7kETFrokcPeesqx/79YrRvzRrRhAQQ6/WNGCFWpGnYUN76iMg8MQASkXm4fRuIiwNmzxZJys4OmDxZvOzt5a5OR0GB6Mm7cKEIgCXCwsRt3n792KKNiGoWAyARmb7Nm0VyOntWfI6KEulKYbMkLlwQt3iXLgWuXBHbbG1Fi7ZRo0QAJCIyBgZAIjJdGRnA2LFi1gQANGggJnw8/7xiHpiTJGDXLnEXesMGMRcFELd2S1q0eXjIWiIRWSAGQCIyPYWFYkLHe++J9VFsbMSEj2nTAEdHuasDAOTlAatWiYHIo0dLt3fpIh5RfPZZtmgjIvnwrx8iMi2//Sbul/71l/jcsaNY069NG3nr+teZM2LB5mXLdFu0vfyyKLttW3nrIyICGACJyFRkZgITJohhNUBMlZ07VyQrmW/3ajTAzz+L0b6ffhK3fQExAXnUKGDoULZoIyJlUUzzoEWLFsHPzw/29vYIDw/H/nunxumxdu1atGzZEvb29mjbti22bNmi8/MhQ4ZApVLpvLp3766zz/Xr1zFo0CA4OzvD1dUVw4YNw61btwx+bURUDcXFIlm1aCHCn0olunqkpYlWbjKGv5s3gYQEUVrPnsCWLSL89egh3p86Je5MM/wRkdIoIgCuWbMGsbGxmD59Og4dOoTAwEBERUUhOztb7/579uzBgAEDMGzYMBw+fBh9+vRBnz59cOzYMZ39unfvjsuXL2tf3377rc7PBw0ahOPHj2Pbtm3YtGkTdu3ahREjRtTYdRJRJe3bB4SGihm+ajUQEiLWTVm0CKhbV7ayjh0TEzgaNhQB7/RpwMVFNBhJTxfhr0cP9uclIuVSSVLJzQr5hIeHIzQ0FAsXLgQAaDQa+Pr6YsyYMZg8eXKZ/fv374+8vDxs2rRJu619+/YICgrCkiVLAIgRwJs3b2JDyezA+/z1119o1aoVDhw4gJCQEADA1q1b0bNnT1y4cAE+Pj4PrVutVsPFxQU5OTlwdnau7GUTUXmuXQOmThWtMSRJDKHNmiVWR7a2lqWku3eBH38Us3l37Cjd3rp1aYs2hcw/IaKH4O9vBYwAFhYWIiUlBZGRkdptVlZWiIyMRHJyst7vJCcn6+wPAFFRUWX237FjBzw8PNCiRQuMHDkS165d0zmGq6urNvwBQGRkJKysrLBv3z695y0oKIBardZ5EZEBaTTA8uVAy5ZisTxJAmJixO3ekSNlCX9Xroj1pZs2Fd3kduwQZURHi/koR48Cr73G8EdEpkX2SSBXr15FcXExPD09dbZ7enri5MmTer+TmZmpd//MzEzt5+7du+P5559HkyZNcObMGUydOhU9evRAcnIyrK2tkZmZCY/7Ft+ysbGBm5ubznHuFRcXh5kzZ1blMonoYY4cESFvzx7xuXVrMbu3UydZyjl4UIz2JSaWtmhzdy9t0abQlsJERBUiewCsKS+++KL2fdu2bREQEIBHHnkEO3bsQLdu3ap0zClTpiA2Nlb7Wa1Ww5e/BYiqR60GZswA5s8XEz7q1AFmzgTeeAOoVcuopRQUAGvXijkn994ICAkRt3lfeEFxXeWIiKpE9gDo7u4Oa2trZGVl6WzPysqCl5eX3u94eXlVan8AaNq0Kdzd3XH69Gl069YNXl5eZSaZ3L17F9evXy/3OHZ2drBjg04iw5AkkbbGjwcuXRLb+vYVCzw3bGjUUi5eLG3RVvLXQq1aQP/+IvixRRsRmRvZnwG0tbVFcHAwkpKStNs0Gg2SkpIQERGh9zsRERE6+wPAtm3byt0fAC5cuIBr167B29tbe4ybN28iJSVFu8/27duh0WgQHh5enUsiooc5dQp4+mmRsC5dEj17t24VgdBI4a+kRdsLLwCNGwMffCDCX4MGwPvvA+fPA//7H8MfEZkpSQESExMlOzs7acWKFdKJEyekESNGSK6urlJmZqYkSZL08ssvS5MnT9bu/8cff0g2NjbS3Llzpb/++kuaPn26VKtWLeno0aOSJElSbm6uNGHCBCk5OVk6e/as9Ouvv0qPPfaY5O/vL925c0d7nO7du0vt2rWT9u3bJ+3evVvy9/eXBgwYUOG6c3JyJABSTk6Ogf4kiMxcXp4kvfOOJNnaShIgSXZ2kjRzpiTdvm20Em7dkqSlSyUpIECUUPLq1EmS1q6VpMJCo5VCRDLh729JUkQAlCRJWrBggdSoUSPJ1tZWCgsLk/bu3av9WefOnaWYmBid/b/77jupefPmkq2trdS6dWtp8+bN2p/l5+dLTz/9tFS/fn2pVq1aUuPGjaXhw4drA2WJa9euSQMGDJAcHR0lZ2dnaejQoVJubm6Fa+Y/QESVsHGjJPn5lSau7t0l6fRpo53+zBlJevNNSXJ1LS2hdm1JGjFCkv7802hlEJEC8Pe3JCliHUBTxXWEiCrg3Dlg7Fjghx/E54YNgU8/BZ57rsa7eGg0wLZtYjZvSZcOQCzpUtKiTcb1pIlIJvz9rYBJIERkpgoLgXnzgPfeA27fBmxsgNhY4N13a3zRvJwcYOVK0TDk1KnS7d27A6NHs0sHEREDIBEZ3vbtYoitZC3PTp3Emn6tW9foaU+cEEu4fP01kJcntjk7i5G+//4XaN68Rk9PRGQyGACJyHAuXwYmTABWrxafPTyAuXOBl16qsdu9d+8CGzeK4Ld9e+n2Vq3EaN/LL7NLBxHR/RgAiaj67t4VI3zvvisWdlapxJDbBx+IPr414OpV4MsvgcWLgYwMsc3KCnj2WbF2X5cuNf6IIRGRyWIAJKLq2btXtHBLTRWfQ0NFKgsOrpHTpaSI0b5vvxWdOwDRom34cNGirVGjGjktEZFZYQAkoqq5dg2YPFkMwwFipC8uTiQxa2uDnqqwEFi3TgS/5OTS7cHBYrSvf3+2aCMiqgwGQCKqHI0GWL4cmDRJhEAAGDIEmD1bPPNnQJcuAZ9/Ll4l3R9r1RLdO0aPBsLDeZuXiKgqGACJqOL+/FPc7i0ZhmvTRjz717GjwU4hScAff4i1+9avF48XAoCPj7jFO3w48IC230REVAEMgET0cGo1MG2aSGUajZhWO2MG8MYbYkjOAPLzxXN9CxeWPk4IiGw5erRYN9pApyIisngMgERUPkkC1qwRCzhfviy29esnFnhu2NAgpzh7VgwifvUVcOOG2Fa7NjBokAh+gYEGOQ0REd2DAZCI9EtLE4s5JyWJz82aidYaTz9d7UNrNMCvv4rRvk2bSlu0NWlS2qLNza3apyEionIwABKRrvx84MMPgY8+AoqKADs74O23gYkTqz3VVq0ubdGWlla6/emnxWzeHj0MPoGYiIj0YAAkolIbN4okdu6c+NyzJzB/PvDII9U67F9/idC3ciVw65bY5uRU2qKtRYtq1k1ERJXCAEhEwD//AGPHAj/+KD77+gKffgr06VPldVaKi8Xt3YULxe3eEo8+Wtqizcmp2pUTEVEVMAASWbKCAuDjj0XLttu3ARsb4M03RUu3OnWqdMhr18SEjs8+Kx1ItLICnnlGBL8nn+TafUREcmMAJLJUSUlixkXJw3idO4vU1qpVlQ53+LBYJebbb4E7d8Q2Nzexbt/IkUDjxgaqm4iIqo0BkMjSXL4sRvm+/VZ89vQUo4ADB1Z6aK6wUCzWvGABsGdP6fZ27cSjhC++KJZ0ISIiZWEAJLIEajVw5AiwaxcQHw/k5or7sv/9L/D++6KPbyVcvlzaoi0zU2yzsRFLBI4ZA7Rvz9u8RERKxgBIZE4kSTx49+ef4pWaKv7377919wsLAxYvBh57rFKH3rNHTOpYt660RZu3d2mLNm9vw10KERHVHAZAIlN15w5w4kRpyEtNFaN8N2/q379hQyAoSMzsHTpUjABWwO3bpS3aDh8u3f7EE6Ut2mxtq3cpRERkXAyARKYgO1t3RC81FTh5Uqy1cr9atcREjqAg0Uet5FWvXqVO+c8/YpDwyy+B69fFNnt70aJt1CjxnB8REZkmBkAiJSkuBk6dKhv2Sh60u1+9eiLclYS9oCCgZcsqD8lJkpgcvHChWBNaoxHb/fzE44KvvFLpHElERArEAEgkl5KJGfeGvaNHS9dQuZdKBfj764a9wECgQQODzLbIzQW+/loEv5MnS7c/9ZS4zdurF1u0ERGZEwZAoppW0YkZJerUAQICdMNe27ZVXpj5QdLSROhbuVKEQEB05xgyRIz4tWxp8FMSEZECMAASGVJVJ2bcG/YeeaTCEzSqorgY2LJFrN23bVvp9pYtS1u0OTvX2OmJiEgBGACJqkqGiRlVkZcHnD4NpKcDx46J0b5//hE/U6lKW7R168a1+4iILAUDINHDyDwxoyLu3BF3lNPTRanp6aWvixfL7u/mBrz6qmjR5udXY2UREZFCMQAS3UtBEzPuV1QEnD2rG+5KAl9GhnjUsDxubqJUf3+ga1dgwAC2aCMismQMgGSZFDoxo7hYhLl7w13J+7Nn9d9dLuHkBDRvXhr0/P1LP7u5GbRMIiIycQyAZP4UNjFDowEuXdJ/u/bMGaCwsPzv1q6tP+D5+wMeHnyGj4iIKoYBkMyLQiZmSJIo5f6Ad+qUmJBx+3b537W1FXnz/oDn7w/4+NToBGEiIrIQDIBkmhQyMePatbIBr+R9ybp6+lhbA02a6L9l6+vLRZeJiKhmMQCS8sk8MUOt1h/w0tNLe+Tqo1IBjRvrv2Xr5ycGIImIiOTAAEjKIePEjPx8cWtW3y3b7OwHf7dBA/3P5DVtCtjbV7oUIiKiGscASPKQYWJGQYGYZKHvlq2+tfLu5eGh/5m8Zs1qpEMbERFRjWIApJpXnYkZQUFilK+CEzOKikSXC323bDMyxAzc8tStq/+ZvGbNABeXyl82ERGRUjEAkuEYaWJGcTFw/rz+Z/LOngXu3i3/u05O5S+jYoSubERERIqgmAC4aNEifPTRR8jMzERgYCAWLFiAsLCwcvdfu3Yt3n33Xfzzzz/w9/fH7Nmz0bNnTwBAUVER3nnnHWzZsgV///03XFxcEBkZifj4ePj4+GiP4efnh3PnzukcNy4uDpMnT66ZizQnNTwxQ5LEWnn6nsmryFp5zZrpv2Xr6cm18oiIiBQRANesWYPY2FgsWbIE4eHhSEhIQFRUFNLS0uDh4VFm/z179mDAgAGIi4vDf/7zH6xevRp9+vTBoUOH0KZNG+Tn5+PQoUN49913ERgYiBs3bmDs2LF45plncPDgQZ1jvffeexg+fLj2s5OTU41fr0mpwYkZkgRkZ+l/Ju/0aTExozy2tmKShb5btlwrj4iI6MFUkvSgDqLGER4ejtDQUCxcuBAAoNFo4OvrizFjxugdjevfvz/y8vKwadMm7bb27dsjKCgIS5Ys0XuOAwcOICwsDOfOnUOjRo0AiBHAcePGYdy4cVWqW61Ww8XFBTk5OXB2dq7SMRSlhiZmXL9e/jIqanX55ZSslafvlm2jRlwrj4iIqsbsfn9XgewjgIWFhUhJScGUKVO026ysrBAZGYnk5GS930lOTkZsbKzOtqioKGzYsKHc8+Tk5EClUsHV1VVne3x8PN5//300atQIAwcOxPjx42FjI/sfS80z8MSM3Nx/A953ZUf0HrZWXqNG+m/XNmnCtfKIiIhqguxJ5+rVqyguLoanp6fOdk9PT5w8eVLvdzIzM/Xun1nOZIM7d+5g0qRJGDBggE7Sf+ONN/DYY4/Bzc0Ne/bswZQpU3D58mXMmzdP73EKCgpQUFCg/ax+0PCVUhhwYkbJWnnp6UD6F7ojellZDy7Dx0d/yHvkEa6VR0REZGyyB8CaVlRUhBdeeAGSJGHx4sU6P7t3FDEgIAC2trZ47bXXEBcXBzs7uzLHiouLw8yZM2u85iozwMSMAvcG+PusqnQEb1HpaN6FCw8+fclaefqWUeFaeURERMohewB0d3eHtbU1su4bQsrKyoKXl5fe73h5eVVo/5Lwd+7cOWzfvv2h9/nDw8Nx9+5d/PPPP2jRokWZn0+ZMkUnNKrVavj6+j7wmDWimhMz7rYOxD/OAUi/6FD6TN5OMZr3sLXyXF1LR/HuH83jWnlERESmQfYAaGtri+DgYCQlJaFPnz4AxCSQpKQkjB49Wu93IiIikJSUpDN5Y9u2bYiIiNB+Lgl/6enp+O2331CvAou8paamwsrKSu/MYwCws7PTOzJYo6o4MUMTEITzDSNwqnYg0vO8kX7aSozoJT18rTxHR/0Br2StPC6jQkREZNpkD4CAuBUbExODkJAQhIWFISEhAXl5eRg6dCgAYPDgwWjQoAHi4uIAAGPHjkXnzp3x8ccfo1evXkhMTMTBgwexdOlSACL89e3bF4cOHcKmTZtQXFysfT7Qzc0Ntra2SE5Oxr59+9C1a1c4OTkhOTkZ48ePx0svvYS6devK8wdR4ocfgLVrHzoxQ3q0FS4174J0jw5It2+L9MLGOHW+NtLTgTPbROuz8tjb679dy7XyiIiIzJ8iAmD//v1x5coVTJs2DZmZmQgKCsLWrVu1Ez0yMjJgdc/SIo8//jhWr16Nd955B1OnToW/vz82bNiANm3aAAAuXryIH3/8EQAQFBSkc67ffvsNXbp0gZ2dHRITEzFjxgwUFBSgSZMmGD9+fJnZxbI4fBhYtQoAIAG4UrcF0ps8jfT6j+NUrdZIL/BFepYz0k9bIf9I+YepVUtMstDX9aJBA66VR0REZKkUsQ6gqaqpdYTWz07HutWFSL/dAKcynaHOLT+pWVsDfn76b9k2agRYwoo2RERElcF1ABUyAki6jt7xx7f3jOypVICvr/6uF35+D22dS0RERKSDAVCBuncX/WxLAh/XyiMiIiJDYgBUoPBw8SIiIiKqCZwGQERERGRhGACJiIiILAwDIBEREZGFYQAkIiIisjAMgEREREQWhgGQiIiIyMIwABIRERFZGAZAIiIiIgvDAEhERERkYRgAiYiIiCwMAyARERGRhWEAJCIiIrIwDIBEREREFsZG7gJMmSRJAAC1Wi1zJURERFRRJb+3S36PWyIGwGrIzc0FAPj6+spcCREREVVWbm4uXFxc5C5DFirJkuNvNWk0Gly6dAlOTk5QqVRyl1NjQkNDceDAAbnLMDpzuG4lX4NSapOjDmOcs6bOoVar4evri/Pnz8PZ2dngxyfLZcx/FyVJQm5uLnx8fGBlZZlPw3EEsBqsrKzQsGFDucuocdbW1hb5F705XLeSr0EptclRhzHOWdPncHZ2VsT/f2Q+jP3voqWO/JWwzNhLlTJq1Ci5S5CFOVy3kq9BKbXJUYcxzqmUP1+iiuI/s8bFW8BERFRharUaLi4uyMnJ4QggkQnjCCAREVWYnZ0dpk+fDjs7O7lLIaJq4AggERERkYXhCCARERGRhWEAJCIiIrIwDIBkVOfPn0eXLl3QqlUrBAQEYO3atXKXZBSWet3Gwj9fIrrXzZs3ERISgqCgILRp0wZffPGF3CUpDp8BJKO6fPkysrKyEBQUhMzMTAQHB+PUqVOoU6eO3KXVKEu9bmPhny8R3au4uBgFBQVwcHBAXl4e2rRpg4MHD6JevXpyl6YYXAiajMrb2xve3t4AAC8vL7i7u+P69etm/4vaUq/bWPjnS0T3sra2hoODAwCgoKAAkiRZdN9ffXgL2EwsXrwYAQEB2tX5IyIi8NNPPxn0HLt27ULv3r3h4+MDlUqFDRs26N1v0aJF8PPzg729PcLDw7F//369+6WkpKC4uNhgvZTj4+OhUqkwbtw4gxyvhNKvu6ZdvHgRL730EurVq4fatWujbdu2OHjwoMGOb+l/vuaCt9wsQ1xcHEJDQ+Hk5AQPDw/06dMHaWlpBj2Hof5OuHnzJgIDA9GwYUNMnDgR7u7uBq3T1DEAmomGDRsiPj4eKSkpOHjwIJ588kk8++yzOH78uN79//jjDxQVFZXZfuLECWRlZen9Tl5eHgIDA7Fo0aJy61izZg1iY2Mxffp0HDp0CIGBgYiKikJ2drbOftevX8fgwYOxdOnSSlxl+Q4cOIDPP/8cAQEBD9zP3K67pt24cQMdOnRArVq18NNPP+HEiRP4+OOPUbduXb3788/Xcjk5OWHXrl1ITU3Fvn37MGvWLFy7dk3ussjAdu7ciVGjRmHv3r3Ytm0bioqK8PTTTyMvL0/v/nL+neDq6oo///wTZ8+exerVq8s9n8WSyGzVrVtX+vLLL8tsLy4ulgIDA6W+fftKd+/e1W4/efKk5OnpKc2ePfuhxwYgff/992W2h4WFSaNGjdI5l4+PjxQXF6fddufOHaljx47S119/Xckr0i83N1fy9/eXtm3bJnXu3FkaO3as3v3M7bqNYdKkSdITTzxRoX3550slrl27JjVu3Fi6cuWK3KVQDcvOzpYASDt37izzM7n/TrjXyJEjpbVr11bgiiwHRwDNUHFxMRITE5GXl4eIiIgyP7eyssKWLVtw+PBhDB48GBqNBmfOnMGTTz6JPn364K233qrSeQsLC5GSkoLIyEidc0VGRiI5ORkAIEkShgwZgieffBIvv/xy1S7wPqNGjUKvXr10zquPuV23Mfz4448ICQlBv3794OHhgXbt2pV7a49/vqatIrfdeMuN7peTkwMAcHNzK/MzOf9OyMrKQm5urrbGXbt2oUWLFlU6n7liADQjR48ehaOjI+zs7PD666/j+++/R6tWrfTu6+Pjg+3bt2P37t0YOHAgnnzySURGRmLx4sVVPv/Vq1dRXFwMT09Pne2enp7IzMwEIG4HrFmzBhs2bEBQUBCCgoJw9OjRKp8zMTERhw4dQlxcXIX2N5frNpa///4bixcvhr+/P37++WeMHDkSb7zxBlauXKl3f/75mq6H3XbjLTe6n0ajwbhx49ChQwe0adNG7z5y/Z1w7tw5dOzYEYGBgejYsSPGjBmDtm3bVvmc5oizgM1IixYtkJqaipycHKxbtw4xMTHYuXNnuSGwUaNG+N///ofOnTujadOm+Oqrr6BSqWq0xieeeAIajcYgxzp//jzGjh2Lbdu2wd7evsLfM/XrNiaNRoOQkBDMmjULANCuXTscO3YMS5YsQUxMjN7v8M/XNPXo0QM9evQo9+fz5s3D8OHDMXToUADAkiVLsHnzZixbtgyTJ0/W2dfT0xOBgYH4/fff0bdv3xqtm+QzatQoHDt2DLt3737gfnL8nRAWFobU1NQaPYep4wigGbG1tUWzZs0QHByMuLg4BAYG4tNPPy13/6ysLIwYMQK9e/dGfn4+xo8fX63zu7u7w9rausx/9WdlZcHLy6tax9YnJSUF2dnZeOyxx2BjYwMbGxvs3LkT8+fPh42NDYqLi/V+z9Sv25i8vb3L/AfEo48+ioyMjHK/wz9f88NbbnS/0aNHY9OmTfjtt9/QsGHDB+7LvxOUiQHQjGk0GhQUFOj92dWrV9GtWzc8+uijWL9+PZKSkrBmzRpMmDChyueztbVFcHAwkpKSdGpISkrS+yxidXXr1g1Hjx5Famqq9hUSEoJBgwYhNTUV1tbWZb5jDtdtTB06dCizxMOpU6fQuHFjvfvzz9c88ZYblZAkCaNHj8b333+P7du3o0mTJg/cn38nKJi8c1DIUCZPnizt3LlTOnv2rHTkyBFp8uTJkkqlkn755Zcy+xYXF0shISFSz549pYKCAu321NRUyc3NTZo3b57ec+Tm5kqHDx+WDh8+LAGQ5s2bJx0+fFg6d+6cdp/ExETJzs5OWrFihXTixAlpxIgRkqurq5SZmWn4i9bjYbOAzfW6a8r+/fslGxsb6cMPP5TS09OlVatWSQ4ODtI333xTZl/++ZoP3Dfz8uLFixIAac+ePTr7TZw4UQoLCzNydSSnkSNHSi4uLtKOHTuky5cva1/5+fll9uXfCcrGAGgmXnnlFalx48aSra2tVL9+falbt256w1+JX375Rbp9+3aZ7YcOHZLOnz+v9zu//fabBKDMKyYmRme/BQsWSI0aNZJsbW2lsLAwae/evdW6tsp4UACUJPO97pq0ceNGqU2bNpKdnZ3UsmVLaenSpeXuyz9f83B/ACwoKJCsra3LLMcxePBg6ZlnnjFucSQrff+uApCWL1+ud3/+naBc7AVMREQ6VCoVvv/+e/Tp00e7LTw8HGFhYViwYAEAccutUaNGGD16dJlJIESkfJwFTEREuHXrFk6fPq39fPbsWaSmpsLNzQ2NGjVCbGwsYmJiEBISgrCwMCQkJCAvL087K5iITAtHAImICDt27EDXrl3LbI+JicGKFSsAAAsXLsRHH32EzMxMBAUFYf78+QgPDzdypURkCAyARERERBaGy8AQERERWRgGQCIiIiILwwBIREREZGEYAImIiIgsDAMgERERkYVhACQiIiKyMAyARERERBaGAZCIiIjIwjAAEhEREVkYBkAiMgtDhgyBSqVCfHy8zvYNGzZApVJV6BiSJGHp0qUIDw+Ho6MjXF1dERISgoSEBOTn58PPzw8qlarc15AhQ/Dnn3/C1tYWP/74o86x/+///g/29vY4duyYwa6ZiKiqbOQugIjIUOzt7TF79my89tprqFu3bqW///LLL2P9+vV45513sHDhQtSvXx9//vknEhIS4OfnhwMHDqC4uBgAsGfPHkRHRyMtLQ3Ozs4AgNq1a8PFxQXTpk3DiBEj0KFDB9SrVw/Z2dl4/fXXMXPmTLRp08ag10xEVBXsBUxEZmHIkCG4du0aTp8+jd69e2POnDkAxAjgc889h4f9Vffdd9+hf//+2LBhA5599lmdn0mSBLVaDRcXF+22HTt2oGvXrrhx4wZcXV119i8uLkZERASaNm2KxMREPPfcc8jKysLvv/8Oa2trw1wwEVE18BYwEZkNa2trzJo1CwsWLMCFCxcq9d1Vq1ahRYsWZcIfAKhUKp3wV5E6Vq5ciR9++AEDBw7Ezz//jBUrVjD8EZFiMAASkVl57rnnEBQUhOnTp1fqe+np6WjRooXB6nj00Ucxbtw4fPvtt5gxYwaaN29usGMTEVUXAyARmZ3Zs2dj5cqV+Ouvvyr8HUM/DXPr1i2sWbMGDg4O+P333w16bCKi6mIAJCKz06lTJ0RFRWHKlCkV/k7z5s1x8uRJg9UwceJE2NvbY8+ePfj111/x9ddfG+zYRETVxQBIRGYpPj4eGzduRHJycoX2HzhwIE6dOoUffvihzM8kSUJOTk6Fz71t2zZ8+eWXWLlyJQIDA/HBBx9g3LhxuHz5coWPQURUkxgAicgstW3bFoMGDcL8+fMrtP8LL7yA/v37Y8CAAZg1axYOHjyIc+fOYdOmTYiMjMRvv/1WoeOo1WoMGzYMEydORGhoKABg/PjxaNWqFUaMGFHl6yEiMiQGQCIyW++99x40Gk2F9lWpVFi9ejXmzZuHDRs2oHPnzggICMCMGTPw7LPPIioqqkLHGTduHFxcXDBjxgztNisrKyxfvhzbt2/nrWAiUgSuA0hERERkYTgCSERERGRhGACJyCL06NEDjo6Oel+zZs2SuzwiIqPiLWAisggXL17E7du39f7Mzc0Nbm5uRq6IiEg+DIBEREREFoa3gImIiIgsDAMgERERkYVhACQiIiKyMP8PC+e1QLb3PtoAAAAASUVORK5CYII="
    }
   },
   "cell_type": "markdown",
   "id": "7617edec",
   "metadata": {},
   "source": [
    "![fused-attention-batch4-d128-fwd-xvalN_CTX.png](attachment:fused-attention-batch4-d128-fwd-xvalN_CTX.png)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ff2a774a",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}