{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gzip\n",
    "import math\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy\n",
    "import random\n",
    "import sklearn\n",
    "import string\n",
    "from collections import defaultdict\n",
    "from gensim.models import Word2Vec\n",
    "from nltk.stem.porter import *\n",
    "from sklearn import linear_model\n",
    "from sklearn.manifold import TSNE"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Data is available at http://cseweb.ucsd.edu/~jmcauley/pml/data/. Download and save to your own directory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataDir = \"/home/jmcauley/pml_data/\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Bag-of-words models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def parseData(fname):\n",
    "    for l in open(fname):\n",
    "        yield eval(l)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = list(parseData(dataDir + \"beer_50000.json\"))[:5000]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "How many unique words are there?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "36225"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wordCount = defaultdict(int)\n",
    "for d in data:\n",
    "    for w in d['review/text'].split():\n",
    "        wordCount[w] += 1\n",
    "\n",
    "len(wordCount)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Ignore capitalization and remove punctuation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "19426"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wordCount = defaultdict(int)\n",
    "punctuation = set(string.punctuation)\n",
    "for d in data:\n",
    "    r = ''.join([c for c in d['review/text'].lower() if not c in punctuation])\n",
    "    for w in r.split():\n",
    "        wordCount[w] += 1\n",
    "\n",
    "len(wordCount)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "With stemming"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "14847"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wordCount = defaultdict(int)\n",
    "punctuation = set(string.punctuation)\n",
    "stemmer = PorterStemmer()\n",
    "for d in data:\n",
    "  r = ''.join([c for c in d['review/text'].lower() if not c in punctuation])\n",
    "  for w in r.split():\n",
    "    w = stemmer.stem(w)\n",
    "    wordCount[w] += 1\n",
    "    \n",
    "len(wordCount)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Just build our feature vector by taking the most popular words (lowercase, punctuation removed, but no stemming)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "wordCount = defaultdict(int)\n",
    "punctuation = set(string.punctuation)\n",
    "for d in data:\n",
    "  r = ''.join([c for c in d['review/text'].lower() if not c in punctuation])\n",
    "  for w in r.split():\n",
    "    wordCount[w] += 1\n",
    "\n",
    "counts = [(wordCount[w], w) for w in wordCount]\n",
    "counts.sort()\n",
    "counts.reverse()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "words = [x[1] for x in counts[:1000]]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Sentiment analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "wordId = dict(zip(words, range(len(words))))\n",
    "wordSet = set(words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def feature(datum):\n",
    "    feat = [0]*len(words)\n",
    "    r = ''.join([c for c in datum['review/text'].lower() if not c in punctuation])\n",
    "    for w in r.split():\n",
    "        if w in words:\n",
    "            feat[wordId[w]] += 1\n",
    "    feat.append(1) # offset\n",
    "    return feat"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Extract bag-of-word features. For a bigger dataset, replace this with a sparse matrix to save memory (see examples in Chapter 6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = [feature(d) for d in data]\n",
    "y = [d['review/overall'] for d in data]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Regularized regression\n",
    "clf = linear_model.Ridge(1.0, fit_intercept=False) # MSE + 1.0 l2\n",
    "clf.fit(X, y)\n",
    "theta = clf.coef_\n",
    "predictions = clf.predict(X)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# N-grams"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Simple example..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentence = \"Dark red color, light beige foam\"\n",
    "unigrams = sentence.split()\n",
    "bigrams = list(zip(unigrams[:-1], unigrams[1:]))\n",
    "trigrams = list(zip(unigrams[:-2], unigrams[1:-1], unigrams[2:]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('Dark', 'red', 'color,'),\n",
       " ('red', 'color,', 'light'),\n",
       " ('color,', 'light', 'beige'),\n",
       " ('light', 'beige', 'foam')]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trigrams"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Extract n-grams up to length 5 (same dataset as example above)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "wordCount = defaultdict(int)\n",
    "punctuation = set(string.punctuation)\n",
    "for d in data:\n",
    "    r = ''.join([c for c in d['review/text'].lower() if not c in punctuation])\n",
    "    ws = r.split()\n",
    "    ws2 = [' '.join(x) for x in list(zip(ws[:-1],ws[1:]))]\n",
    "    ws3 = [' '.join(x) for x in list(zip(ws[:-2],ws[1:-1],ws[2:]))]\n",
    "    ws4 = [' '.join(x) for x in list(zip(ws[:-3],ws[1:-2],ws[2:-1],ws[3:]))]\n",
    "    ws5 = [' '.join(x) for x in list(zip(ws[:-4],ws[1:-3],ws[2:-2],ws[3:-1],ws[4:]))]\n",
    "    for w in ws + ws2 + ws3 + ws4 + ws5:\n",
    "        wordCount[w] += 1\n",
    "\n",
    "counts = [(wordCount[w], w) for w in wordCount]\n",
    "counts.sort()\n",
    "counts.reverse()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "words = [x[1] for x in counts[:1000]]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A few of our 1000 most popular n-grams. Note the combination of n-grams of different lengths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['pint',\n",
       " 'hoppy',\n",
       " 'stout',\n",
       " 'though',\n",
       " 'lots',\n",
       " 'and the',\n",
       " 'malty',\n",
       " 'mouthfeel is',\n",
       " 'even',\n",
       " 'quickly']"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "words[200:210]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "wordId = dict(zip(words, range(len(words))))\n",
    "wordSet = set(words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def feature(datum):\n",
    "    feat = [0]*len(words)\n",
    "    r = ''.join([c for c in datum['review/text'].lower() if not c in punctuation])\n",
    "    ws = r.split()\n",
    "    ws2 = [' '.join(x) for x in list(zip(ws[:-1],ws[1:]))]\n",
    "    ws3 = [' '.join(x) for x in list(zip(ws[:-2],ws[1:-1],ws[2:]))]\n",
    "    ws4 = [' '.join(x) for x in list(zip(ws[:-3],ws[1:-2],ws[2:-1],ws[3:]))]\n",
    "    ws5 = [' '.join(x) for x in list(zip(ws[:-4],ws[1:-3],ws[2:-2],ws[3:-1],ws[4:]))]\n",
    "    for w in ws + ws2 + ws3 + ws4 + ws5:\n",
    "        if w in words:\n",
    "            feat[wordId[w]] += 1\n",
    "    feat.append(1) #offset\n",
    "    return feat"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Same as the model in the previous example above, except using n-grams rather than just unigrams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = [feature(d) for d in data]\n",
    "y = [d['review/overall'] for d in data]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "clf = linear_model.Ridge(1.0, fit_intercept=False) # MSE + 1.0 l2\n",
    "clf.fit(X, y)\n",
    "theta = clf.coef_\n",
    "predictions = clf.predict(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "wordSort = list(zip(theta[:-1], words))\n",
    "wordSort.sort()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Some of the most negative and positive n-grams..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(-0.39143632655636557, 'a lot of'),\n",
       " (-0.28212293516954884, 'a bit more'),\n",
       " (-0.2661141170411719, 'water'),\n",
       " (-0.2530229314150494, 'the background'),\n",
       " (-0.24105277280700568, 'corn'),\n",
       " (-0.21580888155062322, 'as it warms'),\n",
       " (-0.20470888977263463, 'hint of'),\n",
       " (-0.2032403642895844, 'yellow'),\n",
       " (-0.19776086681438282, 'little more'),\n",
       " (-0.19515009289478227, 'watery'),\n",
       " (-0.1833549486926261, 'straw'),\n",
       " (-0.1784682882644964, 'a hint'),\n",
       " (-0.1749628781772528, 'around the'),\n",
       " (-0.17387445409086819, 'kind'),\n",
       " (-0.17381459136980834, 'down the'),\n",
       " (-0.17231514207712764, 'lot'),\n",
       " (-0.16887649424485385, 'weak'),\n",
       " (-0.16647751584896267, 'bad'),\n",
       " (-0.16513650187976975, 'old'),\n",
       " (-0.1581538428300185, 'up to')]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wordSort[:20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0.1596617274119243, 'wonderful'),\n",
       " (0.16316670449503792, 'off white head'),\n",
       " (0.17379190414356988, 'a little more'),\n",
       " (0.17560247184995886, 'favorite'),\n",
       " (0.17674628573616408, 'not too'),\n",
       " (0.17724972085645002, 'i really'),\n",
       " (0.17919486420127723, 'easy to'),\n",
       " (0.18200914314264174, 'i am'),\n",
       " (0.18394566780123306, 'background'),\n",
       " (0.18752400440800757, 'touch of'),\n",
       " (0.19751810949893422, 'the best'),\n",
       " (0.19933817230459436, 'hint'),\n",
       " (0.21249123771083625, 'this one is'),\n",
       " (0.2141045147587393, 'a hint of'),\n",
       " (0.22043777880133708, 'not bad'),\n",
       " (0.2376891974695213, 'of these'),\n",
       " (0.23871898220027807, 'a bad'),\n",
       " (0.24874277211071016, 'lot of'),\n",
       " (0.2748653479324936, 'a lot'),\n",
       " (0.3121741564883301, 'bit more')]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wordSort[-20:]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TF-IDF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = []"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Small set of Goodreads fantasy reviews"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "z = gzip.open(dataDir + \"goodreads_reviews_fantasy_paranormal.json.gz\")\n",
    "for l in z.readlines():\n",
    "    dataset.append(eval(l))\n",
    "    if len(dataset) == 50000:\n",
    "        break"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For example..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'book_id': '18245960',\n",
       " 'date_added': 'Sun Jul 30 07:44:10 -0700 2017',\n",
       " 'date_updated': 'Wed Aug 30 00:00:26 -0700 2017',\n",
       " 'n_comments': 1,\n",
       " 'n_votes': 28,\n",
       " 'rating': 5,\n",
       " 'read_at': 'Sat Aug 26 12:05:52 -0700 2017',\n",
       " 'review_id': 'dfdbb7b0eb5a7e4c26d59a937e2e5feb',\n",
       " 'review_text': 'This is a special book. It started slow for about the first third, then in the middle third it started to get interesting, then the last third blew my mind. This is what I love about good science fiction - it pushes your thinking about where things can go. \\n It is a 2015 Hugo winner, and translated from its original Chinese, which made it interesting in just a different way from most things I\\'ve read. For instance the intermixing of Chinese revolutionary history - how they kept accusing people of being \"reactionaries\", etc. \\n It is a book about science, and aliens. The science described in the book is impressive - its a book grounded in physics and pretty accurate as far as I could tell. Though when it got to folding protons into 8 dimensions I think he was just making stuff up - interesting to think about though. \\n But what would happen if our SETI stations received a message - if we found someone was out there - and the person monitoring and answering the signal on our side was disillusioned? That part of the book was a bit dark - I would like to think human reaction to discovering alien civilization that is hostile would be more like Enders Game where we would band together. \\n I did like how the book unveiled the Trisolaran culture through the game. It was a smart way to build empathy with them and also understand what they\\'ve gone through across so many centuries. And who know a 3 body problem was an unsolvable math problem? But I still don\\'t get who made the game - maybe that will come in the next book. \\n I loved this quote: \\n \"In the long history of scientific progress, how many protons have been smashed apart in accelerators by physicists? How many neutrons and electrons? Probably no fewer than a hundred million. Every collision was probably the end of the civilizations and intelligences in a microcosmos. In fact, even in nature, the destruction of universes must be happening at every second--for example, through the decay of neutrons. Also, a high-energy cosmic ray entering the atmosphere may destroy thousands of such miniature universes....\"',\n",
       " 'started_at': 'Tue Aug 15 13:23:18 -0700 2017',\n",
       " 'user_id': '8842281e1d1347389f2ab93d60773d4d'}"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Similar process to extract bag-of-words representations as in previous examples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "wordCount = defaultdict(int)\n",
    "punctuation = set(string.punctuation)\n",
    "for d in dataset:\n",
    "    r = ''.join([c for c in d['review_text'].lower() if not c in punctuation])\n",
    "    for w in r.split():\n",
    "        wordCount[w] += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "counts = [(wordCount[w], w) for w in wordCount]\n",
    "counts.sort()\n",
    "counts.reverse()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "words = [x[1] for x in counts[:1000]]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Document frequency (df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = defaultdict(int)\n",
    "for d in dataset:\n",
    "    r = ''.join([c for c in d['review_text'].lower() if not c in punctuation])\n",
    "    for w in set(r.split()):\n",
    "        df[w] += 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Term frequency (tf)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here we extract frequencies for terms in a single specific review"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'book_id': '76620',\n",
       " 'date_added': 'Tue Sep 02 17:03:40 -0700 2008',\n",
       " 'date_updated': 'Wed Dec 14 12:30:43 -0800 2016',\n",
       " 'n_comments': 2,\n",
       " 'n_votes': 30,\n",
       " 'rating': 5,\n",
       " 'read_at': 'Tue May 05 00:00:00 -0700 2015',\n",
       " 'review_id': '9206654986a733bd753753aa0c882813',\n",
       " 'review_text': \"I read this after hearing from a few people that it was among their all-time favorites. I was almost put off when I saw it was a story about rabbits, originally written as a tale by a father to his children - but I'm glad I wasn't. \\n I found the folk tales about El-ahrairah to be very impressive. The author clearly had a vivid imagination to create so much of the rabbits culture and history. But I think this book was worth reading as it's really a story about survival, leadership, and human nature. \\n Oh and Fiver rocks. And BigWig is the man.\",\n",
       " 'started_at': 'Sun Apr 19 00:00:00 -0700 2015',\n",
       " 'user_id': '8842281e1d1347389f2ab93d60773d4d'}"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rev = dataset[9] # Query review\n",
    "rev"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf = defaultdict(int)\n",
    "r = ''.join([c for c in rev['review_text'].lower() if not c in punctuation])\n",
    "for w in r.split():\n",
    "    # Note = rather than +=, different versions of tf could be used instead\n",
    "    tf[w] = 1\n",
    "    \n",
    "tfidf = dict(zip(words,[tf[w] * math.log2(len(dataset) / df[w]) for w in words]))\n",
    "tfidfQuery = [tf[w] * math.log2(len(dataset) / df[w]) for w in words]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Find the highest tf-idf words in our example review"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "maxTf = [(tf[w],w) for w in words]\n",
    "maxTf.sort(reverse=True)\n",
    "maxTfIdf = [(tfidf[w],w) for w in words]\n",
    "maxTfIdf.sort(reverse=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(6.475214154215886, 'nature'),\n",
       " (6.1604918290613755, 'tales'),\n",
       " (5.77675046027207, 'children'),\n",
       " (5.7117950184313, 'saw'),\n",
       " (5.316168825598678, 'history'),\n",
       " (5.286585714095165, 'father'),\n",
       " (4.98210658996402, 'worth'),\n",
       " (4.910068021149542, 'glad'),\n",
       " (4.881720019873613, 'human'),\n",
       " (4.731589561708275, 'tale')]"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "maxTfIdf[:10]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Cosine similarity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "def Cosine(x1,x2):\n",
    "    numer = 0\n",
    "    norm1 = 0\n",
    "    norm2 = 0\n",
    "    for a1,a2 in zip(x1,x2):\n",
    "        numer += a1*a2\n",
    "        norm1 += a1**2\n",
    "        norm2 += a2**2\n",
    "    if norm1*norm2:\n",
    "        return numer / math.sqrt(norm1*norm2)\n",
    "    return 0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Find the other reviews in the corpus with the highest cosine similarity between tf-idf vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "similarities = []\n",
    "for rev2 in dataset:\n",
    "    tf = defaultdict(int)\n",
    "    r = ''.join([c for c in rev2['review_text'].lower() if not c in punctuation])\n",
    "    for w in r.split():\n",
    "        # Note = rather than +=\n",
    "        tf[w] = 1\n",
    "    tfidf2 = [tf[w] * math.log2(len(dataset) / df[w]) for w in words]\n",
    "    similarities.append((Cosine(tfidfQuery, tfidf2), rev2['review_text']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "similarities.sort(reverse=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(1.0,\n",
       "  \"I read this after hearing from a few people that it was among their all-time favorites. I was almost put off when I saw it was a story about rabbits, originally written as a tale by a father to his children - but I'm glad I wasn't. \\n I found the folk tales about El-ahrairah to be very impressive. The author clearly had a vivid imagination to create so much of the rabbits culture and history. But I think this book was worth reading as it's really a story about survival, leadership, and human nature. \\n Oh and Fiver rocks. And BigWig is the man.\"),\n",
       " (0.33623828530476574,\n",
       "  'Need a review \\n For a full explanation of the provenance and history of Arabian Nights, as well as an alternative translation, see \\n https://www.theatlantic.com/entertain... \\n For children, see \\n Aladdin and Other Tales from the Arabian Nights, Dawood, 1960 \\n https://www.goodreads.com/review/show...'),\n",
       " (0.2944751254505903,\n",
       "  'Awesome! Love this author and his story world. Found quite a few errors/typos in the text, but the story is worth reading.'),\n",
       " (0.2944520286327877,\n",
       "  \"At first I was iffy about this book but it intrigued by the viewpoint it was written from then before I knew it is was sucked in and couldn't put it down!! I almost cried when I saw there was 3 more books.\"),\n",
       " (0.2902791374813246,\n",
       "  \"I was delighted by this book ...the only fault is that it was too short ! What a fantastic idea; a refuge for the children who have had adventures & now cannot fit back into the identity assigned to them. How many of us are not comfortable in the families we were born to? I loved the way the different doorways were sorted; one would think that adventures shared would be a bonding moment. Rivalries will be ever present ; guess that is human nature. I don't want to describe too much & ruin the magic; this author is the heir apparent to Neil Gaiman. Im so glad she has more books to discover !\"),\n",
       " (0.27867963900816956,\n",
       "  \"Loved this tale and look forward to reading much more of Gem Sivad's tales.\"),\n",
       " (0.2755108699963019,\n",
       "  \"I'm really liking these books. It was good to learn more about the vampire lore in these ones and I'm glad that Jane found some companionship in this book, but I wasn't liking the kidnapped children storyline.\"),\n",
       " (0.2685687782658902,\n",
       "  'An amazing story. The first taste of the Tolkien universe, this children tales has it all and is superbly written. A must absolutely read.'),\n",
       " (0.260579392737578,\n",
       "  \"I'm enjoying reading this series. I wish I would have read this before I saw the movie...but oh well.\"),\n",
       " (0.2602744050799704,\n",
       "  \"See full review for Enchanted at: https://toomanybooksnotenoughshelves.... \\n Sunday Woodcutter is the seventh daughter of a seventh daughter. Because she is such a thing, she has magical properties. Her gift bestowed upon her by her Fairy Godmother Joy is that whatever she writes comes true. It has had harsh implications in the past, so Sunday just writes about her family history. She had six older sisters and three older brothers. Jack Junior died tragically thanks in large part to the royal family of Arilland. Monday was married off to a prince and they never see her anymore. Tuesday died. Wednesday is a little bit strange. Thursday married a pirate king and sends them gifts whenever she can. Friday has a pure heart and a talent with needle and thread and uses it to make clothes for the poor children. Saturday is tomboyish and spends her days with her father and brother in the wood with her axe. Sunday is very lonely, until she meets an enchanted frog who she fasts befriends. \\n But the irony - of course - is that the frog (Grumble) is actually the crown prince Rumbold. Thanks to one of her Sunday's kisses, he becomes his human self without any remembrance of his year prior to becoming a frog. All he knows is he is desperately in love with Sunday and needs her to be his at any cost. So, like in most fairy tales, a few balls are held with all the maidens in the kingdom invited. But maybe they aren't meant to live happily ever after... \\n As I've said in my previous posts, I really like fairy-tale retellings and this book seemed to be right up my alley. But...it wasn't to be.\")]"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "similarities[:10]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# word2vec (gensim)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A few utility data structures (used later)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "beerStyles = {} # Style of each item\n",
    "categories = set() # Set of item categories\n",
    "reviewsPerUser = defaultdict(list)\n",
    "beerIdToName = {} # Map an ID to the name of the product"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "z = open(dataDir + \"beer_50000.json\")\n",
    "\n",
    "reviews = []\n",
    "reviewDicts = []\n",
    "\n",
    "for l in z:\n",
    "    d = eval(l)\n",
    "    reviews.append(d['review/text'])\n",
    "    beerStyles[d['beer/beerId']] = d['beer/style']\n",
    "    categories.add(d['beer/style'])\n",
    "    beerIdToName[d['beer/beerId']] = d['beer/name']\n",
    "    reviewsPerUser[d['user/profileName']].append((d['review/timeUnix'], d['beer/beerId']))\n",
    "    reviewDicts.append(d)\n",
    "    if len(reviews) == 50000:\n",
    "        break"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Tokenize the reviews, so that each review becomes a list of words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "reviewTokens = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "punctuation = set(string.punctuation)\n",
    "for r in reviews:\n",
    "    r = ''.join([c for c in r.lower() if not c in punctuation])\n",
    "    tokens = []\n",
    "    for w in r.split():\n",
    "        tokens.append(w)\n",
    "    reviewTokens.append(tokens)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Example of a tokenized review"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['a',\n",
       " 'lot',\n",
       " 'of',\n",
       " 'foam',\n",
       " 'but',\n",
       " 'a',\n",
       " 'lot',\n",
       " 'in',\n",
       " 'the',\n",
       " 'smell',\n",
       " 'some',\n",
       " 'banana',\n",
       " 'and',\n",
       " 'then',\n",
       " 'lactic',\n",
       " 'and',\n",
       " 'tart',\n",
       " 'not',\n",
       " 'a',\n",
       " 'good',\n",
       " 'start',\n",
       " 'quite',\n",
       " 'dark',\n",
       " 'orange',\n",
       " 'in',\n",
       " 'color',\n",
       " 'with',\n",
       " 'a',\n",
       " 'lively',\n",
       " 'carbonation',\n",
       " 'now',\n",
       " 'visible',\n",
       " 'under',\n",
       " 'the',\n",
       " 'foam',\n",
       " 'again',\n",
       " 'tending',\n",
       " 'to',\n",
       " 'lactic',\n",
       " 'sourness',\n",
       " 'same',\n",
       " 'for',\n",
       " 'the',\n",
       " 'taste',\n",
       " 'with',\n",
       " 'some',\n",
       " 'yeast',\n",
       " 'and',\n",
       " 'banana']"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reviewTokens[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Fit the word2vec model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = Word2Vec(reviewTokens,\n",
    "                 min_count=5, # Words/items with fewer instances are discarded\n",
    "                 size=10, # Model dimensionality\n",
    "                 window=3, # Window size\n",
    "                 sg=1) # Skip-gram model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Extract word representation for a particular word"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.2489907 , -0.08647301, -0.6366861 ,  0.45150977, -0.922045  ,\n",
       "       -0.3455544 , -0.7728421 ,  0.8667233 ,  0.01321169, -1.0097654 ],\n",
       "      dtype=float32)"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.wv['yeast']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Find similar words to a given query"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('piny', 0.990902841091156),\n",
       " ('citrus', 0.9905515909194946),\n",
       " ('floral', 0.989111602306366),\n",
       " ('citrussy', 0.9866609573364258),\n",
       " ('citric', 0.9859722256660461),\n",
       " ('flowery', 0.9844235181808472),\n",
       " ('pine', 0.9834136962890625),\n",
       " ('piney', 0.9828003644943237),\n",
       " ('spicy', 0.9817010760307312),\n",
       " ('herbal', 0.9794316291809082)]"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.wv.similar_by_word(\"grassy\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# item2vec"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Almost the same as word2vec, but \"documents\" are made up of sequences of item IDs rather than words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "reviewLists = []\n",
    "for u in reviewsPerUser:\n",
    "    rl = list(reviewsPerUser[u])\n",
    "    rl.sort()\n",
    "    reviewLists.append([x[1] for x in rl])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "model10 = Word2Vec(reviewLists,\n",
    "                 min_count=5, # Words/items with fewer instances are discarded\n",
    "                 size=10, # Model dimensionality\n",
    "                 window=3, # Window size\n",
    "                 sg=1) # Skip-gram model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Crystal Bitter Ale'"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "beerIdToName['7360']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('55890', 0.9990268349647522),\n",
       " ('10788', 0.9987408518791199),\n",
       " ('32374', 0.9984040260314941),\n",
       " ('4370', 0.9983484745025635),\n",
       " ('18898', 0.998065710067749),\n",
       " ('48047', 0.9975799918174744),\n",
       " ('39606', 0.9974957704544067),\n",
       " ('10480', 0.9967398643493652),\n",
       " ('36376', 0.9967268109321594),\n",
       " ('13009', 0.996677041053772)]"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model10.wv.similar_by_word('7360')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Summerbrau Lager\n",
      "Pilot Rock Porter\n",
      "Puget Sound Vanilla Porter\n",
      "Pinnacle Peak Pale Ale\n",
      "Ferdinand Svetly Lezak\n",
      "Matso's Monsoonal Blonde\n",
      "Leviathan Imperial Stout\n",
      "Ter Dolen Tripel\n",
      "La Délivrance\n",
      "Pale Ale\n"
     ]
    }
   ],
   "source": [
    "for b in model10.wv.similar_by_word('7360'):\n",
    "    print(beerIdToName[b[0]])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# t-SNE embedding"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Visualize the embeddings from the model above using t-SNE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = []\n",
    "beers = []\n",
    "for b in beerIdToName:\n",
    "    try:\n",
    "        X.append(list(model10.wv[b]))\n",
    "        beers.append(b)\n",
    "    except Exception as e:\n",
    "        pass"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Fit a model with just two components for the sake of visualization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_embedded = TSNE(n_components=2).fit_transform(X)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Generate scatterplots using the embedded points (one scatter plot per category)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "scatterPlotsX = defaultdict(list)\n",
    "scatterPlotsY = defaultdict(list)\n",
    "for xy, b in zip(X, beers):\n",
    "    cat = beerStyles[b]\n",
    "    try:\n",
    "        scatterPlotsX[cat].append(xy[0])\n",
    "        scatterPlotsY[cat].append(xy[1])\n",
    "    except Exception as e:\n",
    "        pass"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Plot data from a few categories (more interesting with a larger dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWwAAAELCAYAAAD9brxbAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl8VPW5+PHPEwgabEIIBGUNiIKgcl24RSQgi3KlCnVjcUOtUmvr3mqtXEWsRUUtLr/eylXQQK+0aFEWcVc0LIWCSIOolEASAmiCgGxhS57fH+dkmJlMZk6SmWQmPO/Xa16Zs33PM0ue853v+Z7zFVXFGGNM/Etq6ACMMcZ4YwnbGGMShCVsY4xJEJawjTEmQVjCNsaYBGEJ2xhjEoQl7DgmIlkiUiEiET8nEckRke9F5Pn6iC0SEWkrIrtF5CsRGehh/RtEJLceQgsXwwUisjnM8j0i0rn+IooNEflERH4WpbImiMjMMMs3ichg9/nvROR/o7HfY5Ul7AhEZL2IdG3AECJ2lBeRM4CRQHdVvdOdt9ZNmLtF5IiIlLkJZ7eIPCAiySLyjIhsdudtFJE/+pVZICLfiUiK37ybReQTv+kKvzIr//4GQFW3qWoasAC4M1qvtR5UG4OqpqpqAYCIvCIij9ZbVPHN0+emqo+r6s9jHUxjZgk7sreBnzR0EBFkANtVdXvlDFU9Q1XT3KSZC/zSTThpqvoE8CBwDtDbXWcg8LlfmYrz/bg7aF8a9LyXW2Zl2U8Hrb8WaBWF12jMMc8SdmTvAJf4z3B/7r8hIiUiki8id/gtmyAis0VkplvjXCMip7q12u9EpFBELvJb/xMRmSQiy0XkBxF5U0TS/XcHXOduVyIiD4aIsSlQEeF1SNB0b+BNVf0OQFWLVPUvQes8BfxaRNLClBlcbrAKNz4vkkTkBRHZJSLrKn9KA4jIje683SKyQUR+7reslYjMF5GdbrPQp37Lwn1Wx4vIqyKyQ0TWAv8ZLjj3F8XJIjIOuBa4341nrod9efleXBhm31H7zrlOqe47JyLnicgS9/1cLSIX+C3rLCKL3O3eA1oHxXm9++usNPi7Kn7NJ3K0uW9sqO+2+9nkuJ/NlyJyn/g1V4nIb0WkWI42uw0K99k1GqpqjzAPoBlQCqS40wKsBMYDTYDOwAbgInf5BGA/cCHOATEH2Aj8zl3/FmCjX/mfAJuBHkAK8AYw012WhZPwprpx9AIO4DR9VG6fBEwCPgvzGj4BfhY0bzxQCNwGnBFim03AYDee37vzbgY+9lunAjg5wvs3EPge6BhhvRuAwzjNJ02AUcAuIN1dPgzo7D7vD+wDznKnJwH/474XTYB+Hj+rJ4BPgRZAeyAPKAoTY3nl6wVeAR71WxbV70XQfuvzO9ce2A78lzs9xJ1u5U4vxTmQJ7ufw25ghrusJ7AH6OcufwY4BAz2i7Ny3bDfbfez+QRIA9oBayo/G6AbUASc6E53Aro0dK6ol3zU0AEkwgN4C7jUff5joCBo+QPANPf5BOA9v2WXul9qcad/5H5R09zpT4BJfuv3AA66/6RZbpJo67d8OTDKfd7S/ZL/APQJE3+ohC04yToXKAOKgbF+yysT9unATpxmjVAJexeww11nB24SCdrXbPd1/DFMjDcAxUHzlgPXVrP+m8Ad7vOJ7nTXoHUifVb5/vEC4wifsH0HKKom7Gh8L8orvxc1fB3R+M4dcL8T9wM5Qft6F7ge6IiTgFP8lv0fR5PwQ8Brfsua43yXq0vY4b7b+cCFfstu5mjC7gp8i3MwaVqfuaChH9Yk4s07HG3HzgLauz/VdojITpyaTBu/9b/ze16G076sftPg/BNV8u+ZUIhTO/H/qelf3v7KbVV1J84/xQzgv2vygtTxZ1XtD6Tj1FKni0j3oPW+xDlx+LtqijpbVTNUtaX79wP/hSJyNs57111V73XnZcvRk5R5fqtvCSq7EKd2hYgME5FlbpPHTpwad+V79BTOP/j7bnPJb935kT6rdjgHKv/91VY0vhdC4PcimmWDt+9cFjAqaF/9gLY479dOVS0L2rZSO/9yVXU/zq+rcEJ+t6n62fiXm49zbuUR4DsReU1E2kbYT6NgCdsb/xOPm3F+Xmb4JaoWqjq8DuV39HuehVOL2V7NugFUtQKYj1NLqhVVPaiq/4NTS+4ZYpVHcGqf7UMsi9SG3QP4UlU3+O1vsR49SXmm37rB5XcCtopIM5yf7ZOBTFVtiXMQFbe8var6G1XtCowA7nXbNCN9Vlup+t57FdwzIhbfi1iWHfy6D+N85zbj1IL995WqqpOBbUBL8es5hPMZVdrmX66INKf2J5y3AR2q2Q+q+le3slH5mT1Ry/0kFEvYHqhqMfCDiJwOrAD2iMj97omRJiJyuoj0rsMurhOR09wv+ETgdb/aUaSECM7PzmY12aGI3CVOv+PK13ADTu3m8+B13RrN3/DePc9fshufFyeKyB0i0lRERgKn4Rwsm7mP7apaISLDgKF+r+USOdr1cg9wBKcJINJn9TrwOxFJF5EOwO01eF3fASf7TcfiexHLsqv7zv0FGC4iQ0Ukyd3fBSLSTlWLcNrSJ4rTLTQb8D9ovAFcKiLni0gy8Cjhv7/hls3m6GfTHviVbyORbiIyyD2QH8L5BRHppHujYAnbu3eAn7g12kuBs3DaeUuAl3BOjngVXDubiXOiaCtOYrorzLrB0+B8WcN9lqG22Y9zUmgbzknV24ArVLWwmm0exWl+8Z+vwBoJ7If9x6DtmuD9n+kfwKk4Nb3fA1eq6i5V3YtzsHhdRHYAY4C5ftudCnwoInuAJcCfVPVTD5/VRJyTV5tw2mlnRIjP/7VPA053mw3mxOh74cyMftlKNd85t3LyU5xun6U4TR6/4ej361rgPJymjofcMirjXIeTWGe55X5PYLNGuJiCpx/FaSLbBLyPc3CtPPAfh1OjLnX3k0n1TXaNihytyJlwRGQAMFFVo9p9SJwLUWaq6vQ6lHEqTn/n7upe2BEPxLlC8zmcZowxDR2PSVwi8gtgdLT//xKN1bC9WwK819BBhKKq/wb+BHwmIs82dDzg9BvGqQn2wanJG+OZiJzkNq2IeyL818Ccho6roVkNu4GJyMfAX+pSwzamsRGRTjjnLzrjdB2dBTyoqkcaMq6GZgnbGGMShDWJGGNMgvB6j4caExGruhtjTC2oasgujzFL2O5OY1m8McY0OiLVd0+3JhFjjEkQlrCNMSZBWMI2xpgEYQnbGGMShCVsY4xJEJawjTEmQVjCNsaYBGEJ2xhjEoQlbGOMSRAxvdLRGJMYKioqWLJkCcXFxXTo0IHs7OywV9yZhmEJ2xjDkiVL+PjjjwFYv349AP3792/IkEwI1iRijKG4uDjstIkPlrCNMXTo0CHstIkP1iRijCE7OxsgoA3bxJ+YjTgjImq3VzXGmJoRkWrvh21NIsYYkyAsYRtjTIKwhG2MMQnCErYxxiQIS9jGGJMgLGEbY0yCsIRtjDEJwhK2McYkCEvYxhiTICxhG2NMgrCEbYwxCcIStjHGJAhL2MYYkyAsYRtjTIKwhG2MMQnCErYxxiQIS9jGGJMgLGEbY0yCsIRtjDEJwhK2McYkCEvYxhiTICxhG2NMgmjqZSURaQ9k+a+vqp/FKihjjDFVRUzYIvIkMBpYB5S7sxWwhG2MMfVIVDX8CiLfAL1U9WCNChbRSGUbY4wJJCKoqoRa5qUNeyOQHN2QjDHG1JSXNuz9wBci8hHgq2Wr6p0xi8oYY0wVXhL2PPdhjDGmAUVswwYQkWZAN3fyG1U97GEba8M2xpgaCteG7aWXyEAgBygABOgoIjdYtz5jjKlfXnqJrAKuUdVv3OluwCxVPTfCdlbDNsaYGqprL5HkymQNoKrrsV4jxhhT77ycdFwpIi8Df3GnrwVWxi4kY4wxoXhpEjkO+BWQ7c7KBf4n0oU01iRijDE1F65JxFMvkVru1BK2McbUUK16iYjIbFUdJSJ5OPcOCaCqvaIYozHGmAiqrWGLSFtV3SYiWaGWq2ph2IKthm2MMTVWq14iqrrNfbod2Owm6OOA/wC2Rj1KY4wxYXnp1vcZcLx7T+z3geuBV2MZlDHGmKq8JGxR1f3AFTi9Q0YCp8c2LGOMMcE8JWwR6YvT//ptd16T2IVkjDEmFC8J+y7gd8CbqvqliJwMfBLbsIwxxgSzftjGGBNH6nq3vm7Ab4DOBA7COzhaARpjjInMy6Xpa4AXgVUcHYQXVV0VYTurYRtjTA3VqYYNHFHVP0c5JmOMMTXk5aTjfBH5pYi0FZGMykfMIzPGGBPAS5PIphCzVVVPjrCdNYkYY0wN2d36jDEmQdRpxBkRaS4i/y0i/+tOnyoil0Y7SGOMMeF5acN+BTgEnO9ObwEei1lExhhjQvKSsLuq6mTgMIB7X5GQ1XVjjDGx4yVhHxKRFNxBDESkKxB2eDBjjDHR56Uf9gTgXaCjiPwf0A+4MZZBGWOMqcpTLxERaQWch9MU8g9V3e5hG+slYowxNVSnXiKu9ji3VG0GDBCRK6IVnDHGGG+83PxpOtAL+BKocGcrMCeGcRljjAnipQ37PFXtGfNIjDHGhOWlSWSZiFjCNsaYBualhj0DJ2l/i9OdT3DuJdIrppEZY4wJ4CVhT8MZKT2Po23Yxhhj6pmXhF2qqvNiHokxxpiwvNxe9X+AdGA+flc4qmrYXiLWD9sYY2quriPOpOAk6qF+86xbnzHG1DO7H7YxxgSpqKhgyZIlFBcX06FDB7KzsxGpn3ve1aqGLSL3q+pkEXkB98ZP/lT1zijGaIwxcWPJkiV8/PHHAKxfvx6A/v37N2RIQPgmka/cvyvrIxBjjIkXxcXFYacbSrUJW1Xnu39z6i8cY4xpeB06dPDVrCun40G4JpH5hGgKqaSqI2ISkTHGNLDs7GyAgDbseFDtSUcRucB9egVwEvAXd/pq4DtVvSdswXbS0RhjaqxOo6aLyEpV7R1pXojtLGEbY0wN1bUf9gkicrKqbnQL6wKcEM0ATew0ZPckY0x0eUnY9wCLRGQjzo2fsoCfxzQqEzXx2j3JGFNzERO2qr4rIqcCp7mzvlZVG4Q3QcRr9yRjTM15qWHjJug1MY7FxEAsuidZM4sxDcNTwjaJKxbdk6yZxZiGYQm7kRORqCdTa2YxpmGEu3DmnHAbqurn0Q/HREOsmyzi9SowYxq7cDXsZ9y/xwO9cdqwBWcE9ZVA39iGZmor1k0W8XoVWF1Z23x4dX1/GvL9LS8v58knn2TZsmX07duXBx54gKQkL0Paxpdw9xIZBCAic4BzVDXPnT4DeKReojO1Eusmi1g0s8QDa5sPr67vT/D2BQUFXHfddQFJO1aJ9cknn2T8+PEALFiwAIAHH3ywzuXWNy/vRPfKZA2gqmuBHrELydRVcBOFNVl4k2ht8xUVFeTm5jJr1ixyc3OJ9ZXFdX1/gtffuHEjixcvDphXmVgXLFjA+PHjeeKJJ2oXrKvyPSotLaV///6+g8OyZcvqVG5D8XLS8V8i8jJH7yVyLfCv2IVk6qqxNlnEWjTb5qP18z9cOfX9i6Cu70/w9lA1iQcn0rom1sr3KD09nSFDhgCQm5tL376J2aLrJWHfBNwG3OVOfwb8OWYRmTprrE0WdeElgUbzQBetZBqunPr+RVDX9yc7O5uCggI2btzomxec9Pv27etrsqichtofAIPfk3PPPZeLL76YBx54oEaxxwsvVzoeEJEXgYWq+k09xGRM1HlJoNE80EUrmYYrp75769T1/RERrrvuOhYvXlxt0q9MpP5t2FD7A2Dwe3TFFVckdGUmYsIWkRHAU0AzoIuInAU8avfDNomkvmuj0Uqm4cpp6KYvVaW0tJSysjJSUlLIzMyMWOuNlPSTkpJCngys7efX0O9RtHlpEpkA/BhYBKCqX7h37Ks31t3K1FV910ajlSjCldPQTV+lpaWUlJQAsGfPHgDatGkTk33V9vNr6Pco2rwk7MOq+kNQgqzXG11bdytTV/Vd04pWoojnhFNWVhZ2OpoaW025trwk7C9F5BqgiXvXvjuBpbENK1Cidbcy8SeeE1+iSklJ8dWsK6djxT4/h5d+2HcApwMHgVnAbuDuWAYVzPoVG1NVfffDDpaZmUmbNm1ITU2lTZs2ZGZm1uv+j0URhwirdcFRHCJMVaucWbY2bNMQ4ul8Sm5urq+pEGDw4MFWC20EajVEWDyNmm4/h0y8iKfzKTVpKoynA42pvXBt2E+7f0OOmh7LoIyJV/F0PqUmPSfi6UADdgCprXA3f/oUQESeCRohfb6IrIx5ZMbEoXgawacmPSeCDyzLly/3leE1UUYzycbbASRR2KjpxtRAbbqXRUp0tU1eNWkqDD7Q7Nu3z7fP/v37e7pLXjSTbDz9UkkktR01/daYRmVMA4mUXGtzPiVSoquP5FV5YFm+fDn79u2rsi8vtx+NZpw2CEbt2KjpxviJxU/1UInO/8Bw5MiRgOWxSF7+Bxr/niWV+/Jyl7zgJPvxxx+zadOmWt2z2i6EqZ1wvUSuqGZRV7fbyZwYxWRMg4lFbTdUbdL/wADwww8/kJ6ezsCBA2OavKpLlNXdJS/Utp988gkffvghixcv9vX9rulgANbzq3bC1bCHu3/bAOcDH+E0iQzCudLRErZpdGLxUz1UkvzrX/8asM63337LlClT+MMf/sCAAQPqvM/qVJcoq7tLXqhtJ0+eTG5urm9+og4GkIjC9RK5CUBE3gd6quo2d7ot8Gq9RGdMDIVqr47FT/VQSTL4wFBZk2+o5FfdXfJC8VIbN7Hh5aRjx8pk7foO6BSjeIypN9W1V9fHT/VQzQuQGMkvXG28sQx2G6+8JOyPROQ9nPuIAIwGPoxdSMbUj4bsWlZZ6+7Xrx9NmzalRYsW1TZFxJtwtfHGMthtvPLSS+R2EbkcqGxY+19VfTO2YRkTe/HQtawmTRHREOsrDKM9JqMJ5KWGDfA5sEdVPxSR5iKSqqp7Im5lTAPwmpSOxa5lsb7C0Nq3Y8vLEGHjgJ8DGUBXoD3wIjAktqEZUztek9Kx2LUs1s1AXnqbmNrzUsP+Fc4QYcsBVPXfIhKbcYCMiQK77Ll6sW4GilYTj528DM1Lwj6oqocqf1KKSFPqeYgwY2oiHtqm41U0m4Fi2R5uJy9D85KwPxWRB4EUEbkI+CUwP7ZhGVN7jbFtOlo1zmg2A8WyPdxOXobmJWE/ANwM5OHc9Gkh8HIsgzKmLhpj23Q81jhj2fRkJy9D89Ktr0JEcnDasBX4JmpjfxljPInHGmcsm57s5GVoXnqJXILTKyQf514iXUTkVlV9J9bBGWMc8VjjjGXTU333T08UEQfhFZGvgUtVdYM73RV4W1VPi7CdVcSNiZKKigqeeOIJ6zVxDAg3CK+XhP1PVf1Pv2kBVvjPq2Y7S9jGGFNDtR01vfJ+2CtFZCEwG6cNeyTwz6hHaYwxJiwv98MG5w59F7jPS4GUmEVkTCOhqpSWllJWVkZKSgqZmZkNNjK4XYjSOES8H7YxpnZKS0spKSkBYM8e59Y7bdo0zEXC8dgt0NScl14iXYA7gM7+66vqiNiFZUziKysrCztdn+KxW6CpOS8XzrwFTMO5urEituEY03ikpKT4ataV0w0lHrsFmprzkrAPqOrzMY/EmEYmMzMTIKANu6HYhSiNg5dufdcApwLvAwcr56vq5xG2s259xhhTQ7Xq1ufnTOB6YDBHm0TUnTbGmBqL9cg3jZWXhD0SOFlVD8U6GGPMsSHWI980Vl46Yq4F0mMdiDHm2GGDTNSOlxp2OvC1iPyTwDZs69ZnjKkVG2Sidrwk7Akxj8IYc0xpjINM1IeIvUQARCQLOLVy1HSgSaRR062XiDHG1Fy4XiIR27DdUdPfAKa6s9rjXExjjDGmHnk56fgroB+wG5xR0wEbNd0YY+qZl4R90L9Ln42abowxDcNGTU9gdvGBMccWL5emJ+GMmj4UZ0zH94CXI51RtJOOsZebm+u7+ABg8ODBdvGBMQmuTpemq2oF8JL7MHHELj4w5tgSl0NOVFRUkJuby6xZs8jNzcVq6qEFX2xgFx8Y07h5acOud3afAW/s4gNjji1xmbDtp743ImIHMmOOIeFGTZ9PmO57sbyXiN1nwBhjqgpXw37a/XsFcBLwF3f6apxR1GPGfuobY0xVXrr1rVTV3pHmhdjOuvUZY0wN1eleIsAJInKyX2FdgBOiFZwxxhhvvJx0vAdYJCIbcS6cyQJujWlUxhhjqvB6e9XjgNPcya9V9WC49d1tQjaJdO7cmcLCwprGaRqBrKwsCgoKGjoMY+JauCYRrwn7fKAzfjVyVZ0RYZuQCdsNJuI+TeNjn70xkdXp0nQRmQl0Bb4Ayt3ZCoRN2MYYY6LLSxt2b6CndfkwxpiG5XXU9JNiHYgxxpjwvNSwWwPrRGQFNmq6McY0GC8J+5FYB5HIcnJyePnll8nNzQUgNTWVvLw8Onfu3LCBGWManYhNIqr6KfA1kOo+vnLnNWoDBw4kIyODw4cPR1zXf5SXPXv2xDxZd+nSJWDggmCffvopHTt2jGkMxpj652XU9FHACmAkMApYLiJXxTqwUA4cOMDMmTOZOXMmBw4ciNl+CgsLWbx4MUlJScybNy9m+4mlhh4qrLy8PPJKxpga8XLScTzwn6p6g6qOBX4MPBSLYAoKCnjttddYvXp1lWVHjhzhoosuYuzYsYwdO5aLLrqII0eOxCIMZsyYQd++fbnxxht59dVXA5bt2LGDESNG0KJFC8477zzy8/MDliclJbFx40YABg0axPTp033LcnJyAm6HmpSUxNSpU+nWrRsZGRncfvvtAWW99NJL9OzZk7S0NM444wy++OILxo4dS1FREcOHDyctLY2nn36amli4cCHnnHMOLVq0ICsri4kTJ1Z57Z07dyYzM5PHHnssoDavqjzxxBOccsopZGZmMmbMGHbt2gU4B7mkpCSmT59OVlYWQ4YMqVFcxhgPVDXsA8gLmk4KnlfNdhpKdfNXr16tqampCmhSUpK+9tprVZbj9P/2PVavXh2wzo4dO3TMmDF6xhln6K9//Ws9cuRIyH1Fcsopp+iLL76oq1at0uTkZC0pKfEtGz16tI4ePVrLysp07dq12r59e+3fv79veVJSkubn56uq6sCBA3XatGm+Za+++mrAuiKiw4cP1927d2tRUZFmZmbqe++9p6qqs2fP1g4dOuiqVatUVTU/P1+LiopUVbVz58768ccfVxv/okWLtGPHjiGXffrpp7p27VpVVc3Ly9OTTjpJ586dq6qqX375pf7oRz/SpUuX6uHDh/U3v/mNNmvWTD/66CNVVX322We1b9++unXrVj106JD+4he/0KuvvlpVVQsKClRE9IYbbtD9+/frgQMHquy7us/eGHOU+38SOq9Wt0CPJt6ncAbevdF9vANM9rBduGCquO222wKScZ8+fQKWFxYWalJSkm95UlKSFhYWBqxz7bXXBpQxZcqUGr5Vqrm5udqsWTPdsWOHqqr26NFDn332WVVVLS8v1+TkZF2/fr1v/QcffLBKEq5Jwl66dKlvetSoUfrkk0+qqup//dd/6fPPPx8yxs6dO/uSaCjhEnawu+++W++9915VVX300Uf1mmuu8S3bv39/QMLu0aNHwIFi69atmpycrOXl5VpQUKBJSUlaUFBQ7b4sYRsTWbiE7eWk433AVKCX+/hfVb2/DpX6kNLT0wOmW7RoETDdqVMn/vSnP9G8eXOaN2/On/70Jzp16hSwzldffRV22osZM2YwdOhQWrZsCcDVV19NTk4OAKWlpZSXlwcMqJCVlVXjffg78cQTfc+bN2/O3r17Adi8eTNdu3atU9mhrFixgsGDB9OmTRvS09OZOnUq27dvB2Dr1q0BJytTUlJo1aqVb7qwsJDLL7+cjIwMMjIy6NmzJ8nJyXz33dHbo9tgE8bEjpdL07sAC1V1jjudIiKdVbUgmoHcf//9LFq0iGXLltG5c2eee+65Kuv84he/4NZbb62Mq8rySy65hM8//9w3/ZOf/KRGMRw4cIDZs2dTUVFB27ZtATh06BC7du0iLy+P008/naZNm7J582a6desGQFFRUbXlnXDCCezfv983/e2333qOpWPHjlXaxyvV5YTiNddcw5133sl7771HcnIy99xzD99//z0Abdu2DRjpp6yszLcMnIPm9OnT6du3b5VyK2/o1dAnO41pzLycdHwdqPCbLnfnRVV6ejpLly5lz549bNq0idNOOy3keiJSbVKYOHEiL774InfccQfz58/npz/9aY1iePPNN2natClfffUVa9asYc2aNXz11VdkZ2czY8YMkpKSuPzyy3nkkUcoKytj3bp1vtp3KGeddRZz5syhrKyMDRs2MG3aNM+x3HLLLTz99NO+A1B+fj6bN28GnFp55YnN6qgqBw8eDHgA7N27l5YtW5KcnMyKFSt47bXXfNtcddVVzJ8/n3/84x8cPnyYRx55JKDMW2+9lQcffNB3kCotLQ3oRaN29wJjYqu6thI92hb9RYh5azxsF659Ji5dfPHFet9991WZP3v2bG3btq2Wl5draWmpXnrppdqiRQvt06ePPvzww9W2YW/fvl2HDh2qaWlpmp2drRMnTqz2BKWq6k033aQPPfSQb3rq1KnavXt3TU1N1TPPPFO/+OILVVWdO3eudurUSVu2bKnPPPNMlXgXLVqkSUlJvoeI+Pb197//XbOysjQtLU2HDx+ud9xxh15//fW+bXNycrRTp07aunVrfeyxx7RDhw66ePFiVVWtqKjQKVOmaPfu3TUtLU1POeUUHT9+vKqqrw27vLy82vc3nj97Y+IFYdqwvQwR9gHwgqrOc6d/CtypqmH7bR2Lt1dVVZo0aUJRUVGjaMvdt28f6enpbNiwoc5t9dC4P3tjoqWuQ4T9AnhQRDaLSBHwW2zEmZDy8vJISUnhpJMS915ZCxYsoKysjH379vHrX/+aXr16RSVZG2PqzksvkXxVPQ/ogXOb1fNVdUPsQ0ssc+bMYciQIUyePJmmTb3coiU+zZ07l3bt2tGhQwfy8/P561//2tAhGWNcXppETgQmAe3A1K47AAARX0lEQVRUdZiI9AT6qmrYM2jHYpOICc8+e2Miq2uTyKs4F860c6fXA3dHJzRjjDFeeUnYrVV1Nm7XPlU9wtGhwowxxtQTL42t+0SkFc7l3ojIecAPMY3KGHPMqKioYMmSJRQXF9OhQweys7PtAqxqeEnY9wLzgK4isgTIBBrk9qrGmMZnyZIlvjtCVl5p639XS3NUxIStqp+LyAVAd0CAb1Q18l39jTHGg+Li4rDT5igvAxiMBFJU9UvgMuBvInJOzCMzxhwTgi8yawwXncWKl5OOD6nqHhHJBoYA04A/xzashtO5c2eaN29OWloabdu25aabbgq4gZNXEydOZOzYsTGI0JjGJTs7m8GDB9OtWzcGDx5MdnZ2Q4cUt7wk7MoeIZcAL6nq20Cz2IVUvSNHjvhuyhSr0WZEhLfffpvdu3fz+eefs3LlSh577LEalRGN4bFsiC1zrBAR+vfvz9VXX03//v3thGMYXhL2FhGZCowGForIcR63q7HK25hu27atyrKKigpmzpzJW2+9xVtvvcXMmTOpqKgIUUrdVV7c0bZtW4YNG8batWvZtm0bI0aMoFWrVnTr1o2XX37Zt/7EiRMZOXIk119/Penp6bz44otMmjSJv/3tb6SmpnL22WcDsHv3bm655RbatWtHx44deeihh3z7ysnJITs7m3vvvZfWrVtXGbrLGGO89BIZBVwMPK2qu0SkLXBftAP59ttveeWVVzh06BAiwuWXX86ZZ57pW15SUhJw7+mioiJKSkoC7ttRVlbGwoULKSkpoWvXrlx44YUkJdX+2LJ582YWLlzIlVdeyZgxYzjzzDP5+9//zrp167jooos45ZRTGDhwIADz5s3jjTfeYObMmRw8eJDt27eTn5/PjBkzfOXdcMMNtG3blo0bN7J3714uvfRSOnXqxLhx4wBYvnw511xzDSUlJZ5GazfGHFu83Etkv6rOUdV/u9PbVPX9aAeyatUqDh06VLlPli9fHrA8JSUl4KeSiJCSkhKwzjvvvMPatWspKSlh2bJlrFixolaxXHbZZWRkZDBgwAAGDRrEuHHjWLp0KZMnTyY5OZn/+I//4JZbbglIxn379mX48OEAHHfccVXKLCkp4Z133mHKlCkcf/zxtG7dmrvvvptZs2b51mnfvj2//OUvSUpKClmGMebYFjd3KTr++OPDTrdo0YKf/OQnvP++c6wYOnRolWHEKoe6qlRaWlqrWObOncugQYN80ytWrCAjI4PmzZv75mVlZbFq1SrftP/QWqEUFhZy+PBh30g2lfe39R/mLFIZxphjW9wk7H79+lFQUEBxcTHp6elcfPHFVdbp3bs35557LhB6KKpTTz01oP371FNPrVUswTcoateuHTt27GDfvn2ccMIJgNMk0759e986wfEET3fs2JHjjz+e77//vtqTKnayxRgTTtwk7OOPP56bb76ZQ4cO0axZ9Z1QwiW1gQMHkpqaSmlpKV27dvWNu1hXHTp04Pzzz+d3v/sdTz31FN988w3Tpk0LaM4IduKJJ/Lhhx86o0SIcNJJJzF06FDuuecefv/73/OjH/2ITZs2UVxczIABA6ISpzGmcYtJb4+6CJesIxERevfuzbBhw2qdrKs7IMyaNYtNmzbRrl07rrzySn7/+98HNJsEGzlyJKpKq1at6N27N+D0BDl06BA9e/YkIyODkSNH1mhgXmPMsS3i/bBrXbDdD9sEsc/emMjqej9sY4wxccAStjHGJAhL2MYYkyAsYRtjTIKwhG2MMQnCErYxxiQIS9jGGJMgLGEbY0yCsITdQHJycqI20OigQYOYPn16VMoyxsQvS9hBFi9eTL9+/UhPT6d169b0798/4K580VQfN3uaOHEizZo1Iy0tjbS0NE4//XTmzJkT8/0aY6IvoRJ2RUUFO3fuZOfOnTEZbWbPnj0MHz6cu+66i507d7JlyxYmTJiQ8PemHjNmDLt372b37t1MmTKF6667rta3ng3HhjUzJrbiKmEfOnSIXbt2UVZWVmWZqlJQUMCWLVvYsmULBQUFUb8vxfr16xERRo0ahYhw3HHHceGFF3LGGWcAgcN4tWzZklNOOYVly5aRk5NDp06dOOmkkwIGNdi9ezdjx46lTZs2dOnShT/84Q/V7vu+++5jwIAB7NmzB4Dp06fTs2dPWrVqxbBhwwJG2/nggw/o0aMHLVu25I477qjR+zB06FBSU1PJz8/3zVuwYAFnn302LVu2JDs7m7y8PN+ybdu2cdVVV9GmTRu6du3KCy+84FsWPDRaTk6O5ziMMTUXNwm7rKyMDRs2UFxcTH5+Prt27QpYfuDAgYDRy/fv38+BAwcC1ikvL2fz5s38+9//Ztu2bTVO6N26daNJkybceOONvPvuu1ViAGcwg7POOosdO3Zw9dVXM2bMGFauXEl+fj4zZ87k9ttv98V5++23s2fPHgoKCli0aBEzZszglVdeCShPVRk3bhxr167lgw8+IDU1lblz5/LEE0/w1ltvUVpa6hugFJxBGq688komTZrE9u3b6dq1K0uWLPH8Gt9++20OHz5Mz549AVi9ejU333wzL730Ejt27ODWW29lxIgRHD58GFVl+PDhnH322Wzbto2PPvqI5557jg8++MBX3rx58xg1ahS7du3i2muvrdH7bYypocqRT6L9cIquqrr5W7Zs0by8PN9jw4YNAcsPHjwYsDwvL08PHjwYsE5RUVHA8tLS0pD7Cufrr7/Wm266STt27KjJyck6YsQILSkpUVXVV199Vbt16+ZbNy8vT5OSkgL206pVK12zZo2Wl5drs2bN9Ouvv/Ytmzp1qg4aNMhXVp8+fXT06NF61VVX6eHDh33rDRs2TKdPn+6bLi8v1+bNm2tRUZHOmDFD+/btGxBzhw4ddNq0aSFfzyOPPKLNmjXTli1b6gknnKBNmzbVp556yrf8tttu04cffjhgm+7du+tnn32my5cv16ysrIBljz/+uP7sZz/zlX3BBRdU+14Gq+6zN8Yc5f6fhMyrcVPDDh4st0mTJgHTzZo1o127dogIIkK7du2q3Dv74MGDYae96N69O9OnT6eoqIi1a9eydetW7r77bt/yE0880fe8ckzJ1q1bB8zbu3cv27dv58iRIwFDgGVlZbFlyxbf9IYNG5g3bx4TJkygadOjY0kUFhZy1113kZGRQUZGBq1atUJE2LJlC1u3bq0ylFikocVGjx7Njh072Lt3L/n5+eTk5PDSSy/59vXMM8/49tWyZUuKi4vZunUrhYWFbNmyJWDZ448/TklJied9G2OiJ24SdmZmpi8BJicn+8Y+9JeRkUHPnj19AwAES01NDTtdU926dePGG29k7dq1Nd62devWJCcnU1hY6JtXWFgYMKxYz549eeWVV7j44otZv369b36nTp2YOnUqO3bsYMeOHezcuZO9e/dy3nnn0bZt24D2bHBGd/eqU6dODBs2jPnz5wNOwh0/fnyVfY0ePZqOHTty8sknByz74YcffNuCDWtmTH2Km4TdpEkTunbtSo8ePejevXu1PTMqa9ihtGnThnbt2pGRkUGnTp1IS0urUQzffPMNf/zjH3214M2bNzNr1iz69u1b7TZaTTt5UlISo0aNYvz48ezdu5fCwkKmTJnC9ddfH7De6NGjmTRpEhdeeCEbN24E4NZbb2XSpEmsW7cOgB9++IE33ngDgEsuuYR169bx1ltvUV5eznPPPcd3330X9nX5x1hcXMy7777rO5E6btw4XnzxRd8I8/v27WPhwoXs27ePH//4x6SmpjJ58mQOHDhAeXk5X375JStXrgy7P2NMbMRNwq4U3BRSEyJCRkYG7dq1q3GyBqdGvnz5cvr06UNqairnn38+vXr14umnnw67z+qmn3/+eZo3b87JJ5/MgAEDuO6667jpppuqlDF27FgefvhhhgwZQlFREZdddhkPPPAAY8aMIT09nV69evHuu+8C0KpVK15//XV++9vf0rp1a/Lz8+nXr1/Y1zV79mxfP+w+ffrQv39/Hn74YQDOPfdcXnrpJW6//XYyMjLo1q2br7dHUlISCxYs4IsvvqBLly60adOGcePGsXv3bm9vqDEmqmyIMFNv7LM3JjIbIswYYxoBS9jGGJMgLGEbY0yCsIRtjDEJwhK2McYkCEvYxhiTIJpGXiW6srKy7Oq4Y1RWVlZDh2BMQqv3ftjGGGOqZ/2wjTGmEbCEbYwxCcIStjHGJAhL2MYYkyBi2kvEeoMYY0z0xKyXiDHGmOiyJhFjjEkQlrCNMSZBWMI2xpgEYQn7GCIid4rIOhGZKSKXisj9Ndg2S0SujlFcE0Tk3mhv78acV8OyXhGRKyKs4ytXRM4VkWdrFnF0iMjihtivaTj1fi8R06BuA4ao6lZ3ekHwCiLSRFXLQ2zbBbgGmBXD+GIhVmfVFUBVVwGrYrSP8AGoZjfEfk3DsRr2MUJE/gycDLwjIneJyA0i8oK77BUR+bOI/AN4UkQGiMhqEflcRFaJyAnA40C2O++uEOX/RkRWiMgXIjLBnZclIl+55X8jIn8RkSEistid7u1XxFkistSdf0u4ct354911PwO6+80/1113NfArv/lJIjJZRJa7y8f5Lft/bpzvA22qef+qK/cCEZnvPp8gIq+KyGcisklELheRJ0XkXyKyUESauOudIyKLROSfIvKOiJzozv9ERJ5wY/xaRPq583u68z53Y+jqzt/jF8dTIpInImtEZJRfbJ+IyOvu65sZ+tthEoaq2uMYeQAbgZbu8xuA593nrwDz/NabB/R1nzfHObBf4L9OULkXAVPd5wLMB7KBLOAQ0NNdthJ42X0+AnjTfT4BWA00A1oBRcBJYco9B1gDHAekAv8G7nXXWwP0c59PBv7lPh8HPOg+bwb8043vcuA9d35bYCdwRYjXWF25vvfFfR2fue9XL2AfMNRdNsd9zU2BJUArd/4oYJr7/BPgKff5MOAD9/nzwNXu86bAce7z3e7fK/1eQxugEDjRjW2n+7oEWAqc39DfQ3vU/mFNIscWcR+hvO73fAkwRUT+D5ijqlsiXAQ1FLhIRD53yz8BOBXYDGxS1XXuel8CH7nP83ASZqW5qnoI+F5EPgZ+DPSvptw0nGR/EDgoIvMARKQF0EJVl7hlzgQu9ovxTBEZ6U6nuWUNwG3mUdVt7r4DRCg32DuqWuG2cSep6vt+r7czzq+BM4APxHlTk4CtftvPcf+u8nt/lgHjRaSD+7o3BO2zn99rKBGRRcB/AnuAFaq6zX0dX7gxLK0mdhPnLGGbSvsqn6jqkyKyALgEWCIiQyNsK8DjqvpSwEyRLOCg36wKv+kKAr9//m3N4jcdqtwqTTJB21Y3/w5V/SCorEvClFUbBwFUVUXksN/8ytcrwFpV7Rdue6DcXR9VneU2V10KLBSRn6vqojAx+L8H/u+/r0yTmKwN21QhIier6peqOhmn6eA0nNpaWjWbvAf8zG3rRkTaiUhmZXEed/tTEWkmIq1wfsr/E3i/mnI/Ay4TkeNEJBUYDqCqPwA7ReR8t8zrgmL8pYg0dcs6VUSau2WNdtu42wKDggNzy91VTbnhhHrt3wCZInKeG0dTEekZbnsR6aKqm1T1BWAuTnOLf/m5fq8hE+eXyQqPMZoEYkfbY0t1PSaC598tIoNwamRfAu+465S7J91eVdXnfBurfiAipwHL3KaTPThJrSKo7HA9Nv4FLMJpw35UVb8Fvg1VrqquFpHZ7jbfEZicfgZMF5EKnIRf6WWc5oDP3aaIEuAyVX1TRAa7r7OI6psLqis3nCqvV1UPi8hVwAtuU0sT4FlgXYj1K6dHicj1wGFgG/AH/+XuazgPp529ArjPbRrpESkek1jsXiLGGJMgrEnEGGMShCVsY4xJEJawjTEmQVjCNsaYBGEJ2xhjEoQlbGOMSRCWsI0xJkFYwjbGmATx/wEplbYBQJzREwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f146c353c50>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.scatter(scatterPlotsX['American Adjunct Lager'],\n",
    "            scatterPlotsY['American Adjunct Lager'], color='k', lw=0, label=\"Adjunct Lager\")\n",
    "plt.scatter(scatterPlotsX['American Porter'],\n",
    "            scatterPlotsY['American Porter'], color='grey', lw=0, label = \"Porter\")\n",
    "plt.scatter(scatterPlotsX['Smoked Beer'],\n",
    "            scatterPlotsY['Smoked Beer'], color='lightgrey', lw = 0, label = \"Smoked Beer\")\n",
    "plt.legend(loc='lower left')\n",
    "plt.xticks([])\n",
    "plt.yticks([])\n",
    "plt.xlabel(\"first embedded dimension \")\n",
    "plt.ylabel(\"second embedded dimension\")\n",
    "plt.title(\"\\emph{TSNE}-based item embeddings\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Exercises"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 8.1 / 8.2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Simple sentiment analysis pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "def parseData(fname):\n",
    "    for l in open(fname):\n",
    "        yield eval(l)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = list(parseData(dataDir + \"beer_50000.json\"))[:5000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "random.shuffle(data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Add a couple of \"options\" for the representation (in this case whether we should convert to lower case, remove puncuation). More could be added."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "def feature(datum, words, wordId, tolower=True, removePunct=True):\n",
    "    feat = [0]*len(words)\n",
    "    r = datum['review/text']\n",
    "    if tolower:\n",
    "        r = r.lower()\n",
    "    if removePunct:\n",
    "        r = ''.join([c for c in r if not c in punctuation])\n",
    "    for w in r.split():\n",
    "        if w in words:\n",
    "            feat[wordId[w]] += 1\n",
    "    feat.append(1) # offset\n",
    "    return feat"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Condense the pipeline code (see Chapter 3) into a single function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pipeline(dSize = 1000, tolower=True, removePunct=True):\n",
    "    wordCount = defaultdict(int)\n",
    "    punctuation = set(string.punctuation)\n",
    "    for d in data: # Strictly, should just use the *training* data to extract word counts\n",
    "        r = d['review/text']\n",
    "        if tolower:\n",
    "            r = r.lower()\n",
    "        if removePunct:\n",
    "            r = ''.join([c for c in r if not c in punctuation])\n",
    "        for w in r.split():\n",
    "            wordCount[w] += 1\n",
    "\n",
    "    counts = [(wordCount[w], w) for w in wordCount]\n",
    "    counts.sort()\n",
    "    counts.reverse()\n",
    "    \n",
    "    words = [x[1] for x in counts[:dSize]]\n",
    "    \n",
    "    wordId = dict(zip(words, range(len(words))))\n",
    "    wordSet = set(words)\n",
    "    \n",
    "    X = [feature(d, words, wordId, tolower, removePunct) for d in data]\n",
    "    y = [d['review/overall'] for d in data]\n",
    "    \n",
    "    Ntrain,Nvalid,Ntest = 4000,500,500\n",
    "    Xtrain,Xvalid,Xtest = X[:Ntrain],X[Ntrain:Ntrain+Nvalid],X[Ntrain+Nvalid:]\n",
    "    ytrain,yvalid,ytest = y[:Ntrain],y[Ntrain:Ntrain+Nvalid],y[Ntrain+Nvalid:]\n",
    "    \n",
    "    bestModel = None\n",
    "    bestVal = None\n",
    "    bestLamb = None\n",
    "    \n",
    "    ls = [0.01, 0.1, 1, 10, 100, 1000, 10000]\n",
    "    errorTrain = []\n",
    "    errorValid = []\n",
    "\n",
    "    for l in ls:\n",
    "        model = sklearn.linear_model.Ridge(l)\n",
    "        model.fit(Xtrain, ytrain)\n",
    "        predictTrain = model.predict(Xtrain)\n",
    "        MSEtrain = sum((ytrain - predictTrain)**2)/len(ytrain)\n",
    "        errorTrain.append(MSEtrain)\n",
    "        predictValid = model.predict(Xvalid)\n",
    "        MSEvalid = sum((yvalid - predictValid)**2)/len(yvalid)\n",
    "        errorValid.append(MSEvalid)\n",
    "        print(\"l = \" + str(l) + \", validation MSE = \" + str(MSEvalid))\n",
    "        if bestVal == None or MSEvalid < bestVal:\n",
    "            bestVal = MSEvalid\n",
    "            bestModel = model\n",
    "            bestLamb = l\n",
    "            \n",
    "    predictTest = bestModel.predict(Xtest)\n",
    "    MSEtest = sum((ytest - predictTest)**2)/len(ytest)\n",
    "    MSEtest\n",
    "    \n",
    "    plt.xticks([])\n",
    "    plt.xlabel(r\"$\\lambda$\")\n",
    "    plt.ylabel(r\"error (MSE)\")\n",
    "    plt.title(r\"Validation Pipeline\")\n",
    "    plt.xscale('log')\n",
    "    plt.plot(ls, errorTrain, color='k', linestyle='--', label='training error')\n",
    "    plt.plot(ls, errorValid, color='grey',zorder=4,label=\"validation error\")\n",
    "    plt.plot([bestLamb], [MSEtest], linestyle='', marker='x', color='k', label=\"test error\")\n",
    "    plt.legend(loc='best')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Run models with different feature representation options and dictionary sizes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "l = 0.01, validation MSE = 0.8124083444098931\n",
      "l = 0.1, validation MSE = 0.8041193851382906\n",
      "l = 1, validation MSE = 0.7387154350328736\n",
      "l = 10, validation MSE = 0.5361335778700643\n",
      "l = 100, validation MSE = 0.3996322733152027\n",
      "l = 1000, validation MSE = 0.40242091430095334\n",
      "l = 10000, validation MSE = 0.4792032394453763\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYkAAAEfCAYAAAC04jrjAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl0VFW2+PHvDglgQIYwD2EMQyItxKnBMYAyCTIICMr4635OD7Vf91OxHYCmbcfWVrvbBgURBQEHJhGlQRJsVEAJg5AAMgRIABEJCVNCkv37o4p6RVKVuVJVyf6slbXqnnvq3H2S1N117zn3XlFVjDHGGE9C/B2AMcaYwGVJwhhjjFeWJIwxxnhlScIYY4xXliSMMcZ4ZUnCGGOMV5YkTFAQkdYikiciIc7lz0RkbHHqlmJbT4jIzLLEW4pt3igiSeXU1n4R6eV8XeF9MZWLJQlTIURkpYhM9VA+WESOFHOH7rqoR1UHqOp7xalbRFy3iMihS96o+pyq3luc95eEiIwXkRwRyRCRdBHZLCK3O7f5H1WNLu9t+qovpuqwJGEqyrvAGA/lY4D3VDWvguO5SChmQiknX6tqHVWtB8wGFolI3QrcvjElYknCVJQlQAMRufFigYjUAwYCc53LA5zfrk+JSIqITPHWmIisFZH/53wdIiIvi8hxEfkRuD1f3QkistP5Df5HEbnXWR4OfAY0F5FM5/qmIjJFRN5ze/8dIvKDiPwiIl+KSGe3dftF5A8islVETorIByJSvZi/k9nAZUD7/Ec0znYni8gOETkhIrPc2xWRgSKS6Nzmf0TkV15+T66+uJ2GG+f8/f4kIn90qyvObf7o/F0ucP6NTBVmScJUCFU9D3wIjHMrvgtIUtUfnMungbGqWhfHjv5+EbmjGM3fCwwAugLXAMPzrT8GDFDVOsBE4FUR6aaqZ4H+QJqqXu78hn/0YsgAItIRmA88DDQCVgLLRSTUrf0RQB+grTOGCUUF7Hz/fwGZwB73bbq5G7gNaA90Ap5yvjcWmOV8fwQwA1gmImFeNpe/3RuADsCtwDMi0slZ/jBwB3AT0Bw4CfyzqL6Yys2ShKlI7wIj3L4Rj3WWAaCq61R1h/P1D8AC4JZitDsC+JuqpqlqOvCc+0pVXamqB5yvvwJW4dgRFsdI4FNV/VJVc4GXcXz7v96tzmuqesy57eVAt0La6yEivwBpOJLkEFXN9FL3Dbc+PQuMdpb/F/AvVf1OHd4DsoDuxeiPAlNVNVtVtwFbcSQ2gPuAJ1X1iKpeAP4EDC/tBABTOYQWXcWY8qGq60XkODBERL4DrgWGXlwvItcBzwNdgOrOnw+L0XRzwH3wOcV9pYj0B54BOuL4YnQZsK2YYTd3b09V1XlaqIVbnWNur88CzQpp7xtVvbmY2z7s9jrFGQtAa2CciDzkXBYgzG19UfLHW9ut3cUicnF8SIALQBPgSDHbNpWMfUMwFe09YDyOAesvVPW427r5OMYuWjgHdmfg2FEV5QgQ6bbc+uIL51HLR8CLQCNVrY/jlNHFdosatE5zb88pkkt34L6Sv09pzteHgGdVNcL5U19Va6vqwjJu7yDQP1+7tVTVEkQVZknCVLS5OM6F/xa3U01OtYGTqnrBeVRxd7713hLGIuBhEWkhIvWBx93WXTwi+VlV85xHFX3c1h/DMaBep5C2bxeRniISKiL/C5wHvim8m+Xiv519igD+iOP0G8BbOMZrrgMQkVrOQf9axWizsKQ7A/iLiLRyttuomGNCphKzJGEqlKqmAF8D4cCyfKsfBKaLyCkcg7T5vxmrl9dvAV/gOL/+HfCx2/ZO4xiQ/dA5FjAKWOq2fhfwAbDPOXupab54d+M46vk7cBzHgPogVc3xEEd5m49j/ORHHIPbzzpj+h7HuMTfnX3ajePozBV2IW3mX+e+/BqO380q59/ga+C6snTABD/x9UOHRKQf8DccCWmWqr6Qb30rHFMBGwEngDGqmlagIWOqEBHZD/xGVb/0dyymavPpkYRzVsTfgb7AFcBo9znmTi8Dc1S1K47ZFM/7MiZjjDHF5+vTTdcBe1Q1xTmlbgEwOF+dGGAtgKrGe1hvTFVkzxU2AcHXSaIFl05NPMylUwcBtgDDAERkGFDbOfhoTJWlqu3sVJMJBIEwcP0oECci3+O4wCkVyPVvSMYYY8D3F9OlAq3clls6y1ycc7DvBMdUPuBOVc3I35CI2OG3McaUgqoW53ojj3x9JLEJiHLeWKw6jumHl0x7FJEGInKxA0/gmOnkkaqW6WfKlCllrudpXf6ywpaL89qX/Stp3wKlf77625VH/yrqb1ea/pX0/7Wy9a84fQ2m/pXms1dWPk0S6rjXzSQcc713AAtUNUlEponIQGe1OGCXiCQDjXHOBfeFuLi4MtfztC5/WWHL3l6Xh+K0V9K+eSr3R/989bfzVF6Z+lfS/9fK1r/i9rWsKvW+pawZtKJ+HKFWXlOmTPF3CD5VmftXmfumav0Lds59Z6n3vYEwcG0o/6OKQFOZ+1eZ+wbWv6rO51dclxcR0WCJ1RhjAoWIoGUYuLZbhRsTJNq0aUNKSkrRFU2V1Lp1aw4cOFDu7dqRhDFBwvmN0N9hmADl7f+jrEcSNiZhjDHGK0sSxhhjvLIkYYwxxitLEsaYgPDAAw/w7LPFu5a2JHVN2djAtTFBIpAHrtu2bcusWbPo1auXv0Opsmzg2hgTtHJzK+eNnT3tlEuayAP9d2NJwhhTJuPGjePgwYMMGjSIOnXq8PLLL5OSkkJISAizZ8+mdevW9O7dG4CRI0fSrFkz6tevT1xcHDt37nS1M3HiRJ555hkAEhISiIyM5JVXXqFJkya0aNGCOXPmlKruL7/8wqBBg6hbty6//vWvefrpp7npppu89ufbb7/lhhtuoH79+sTGxpKQkOBa17NnT5566iluvPFGatWqxf79+z2WHTlyhMGDB9OgQQM6duzI22+/7Wpj2rRpjBgxgrFjx1KvXj3efffdMv3+fc0upjPGlMncuXP56quvmD17Nj179gRwXfS3bt06kpOTCQlxfB8dMGAAc+bMISwsjMcff5x77rmHxMREj+0ePXqUzMxM0tLSWLVqFcOHD2fo0KHUrVu3RHUffPBBLr/8cn766Sf27dtH3759adOmjcdtpqWlMXDgQObNm0ffvn1Zs2YNd955J7t27aJBgwYAvP/++3z++ed07NiRvLw8j2W9e/fmyiuv5OjRo+zcuZPbbruNqKgo1y1Ali1bxkcffcR7771HVlZWqX/3FcGOJIwx5SL/aRYRYdq0aVx22WXUqFEDgAkTJhAeHk5YWBjPPPMMW7duJTMz02N71atX5+mnn6ZatWr079+f2rVrs2vXrhLVzcvL45NPPuFPf/oTNWrUIDo6mvHjx3vtw/vvv8/tt99O3759AejduzfXXHMNn332mavOhAkT6Ny5MyEhIYSGhhYoO3r0KF9//TUvvPACYWFhdO3ald/+9rfMnTvX1UaPHj0YNGgQgOt3E6gsSRhTSUydOhURKfAzderUYtf3Vre0WrZs6Xqdl5fH5MmTiYqKol69erRt2xYR4eeff/b43gYNGriOQADCw8M5ffp0ieoeP36c3NzcS+KIjIz0Gm9KSgqLFi0iIiKCiIgI6tevz/r16zl69Gih73cvS0tLIyIigvDwcFdZ69atSU1N9Vg/0NnpJmMqialTp5ZoJ1/S+oX5v+eGeS+fP38+y5cv58svv6RVq1acOnWK+vXr+3TGVqNGjQgNDeXw4cNERUUBcOjQIa/1IyMjGTduHDNmzPBax1Nf3cuaN2/OL7/8wpkzZ6hVqxYABw8epEWLFoW2EajsSMIYU2ZNmzZl3759l5Tl3/lnZmZSo0YN6tevz5kzZ3jiiSd8vrMMCQlh2LBhTJ06lXPnzpGcnHzJaZ/8xowZw/Lly1m1ahV5eXmcP3+ehIQE0tLSir3Nli1bcv311/PEE0+QlZXFtm3bmDVrFmPHji2PLlU4SxLGmDKbPHky06dPJyIigldeeQUo+G153LhxtGrVihYtWtClSxeuv/76Em2jJAnFve4bb7xBeno6zZo1Y/z48dx9991exwFatmzJ0qVL+ctf/kKjRo1o3bo1L7/8smuAuqijiIs++OAD9u/fT/PmzbnzzjuZPn26a1A/2NjFdMYEiUC+mC6YTJ48mWPHjvHOO+/4O5RyZRfTGWNMKezatYvt27cDsHHjRmbNmsWwYcP8HFXw8HmSEJF+IpIsIrtF5HEP6yNF5EsR2SwiW0Skv6d2qlev7jrkM8aY4srMzGTYsGHUrl2b0aNH8+ijj7qmn5qi+fR0k4iEALuB3kAasAkYparJbnVmAJtVdYaIRAOfqWrb/G398Y9/1Jo1axISEkJYWBjVq1cnLCzM4+vQ0FCv64t6r/s0OmMCiZ1uMoXx1ekmX0+BvQ7Yo6opACKyABgMJLvVyQPqOF/XA1Lx4LnnniMvL4/c3Fyys7O5cOECFy5c8Pr64nJWVhanT58usM7be0Wk0CTinmyKWu9pXbVq1Xz46zbGmPLl6yTRAnCflHwYR+JwNw1YJSIPA+HArd4aExFCQ0NdVzmWN1UlNze3yASU//Xp06eLlYCys7MREY9JpFmzZsTExNCqVSs7mjHGBIxAuJhuNPCOqr4qIt2B94Er/BGIexK67LLLfLKNi0nIPYlkZWVx8OBBVq1axalTp+jcuTMxMTG0adPGjjyMMX7l6ySRCrRyW25JwdNJvwH6AqjqtyJSU0QaqmqBa/Xdrw6Ni4tz3SwrmFSrVo1q1apRs2bNS8pbt27NTTfdxMmTJ9m5cydr167ll19+oVOnTsTExNC2bVufHUEZYyqP+Ph44uPjy609Xw9cVwN24Ri4PgJsBEarapJbnRXAIlV91zlw/W9VbemhrSp3ncSpU6dISkoiKSmJn376iY4dOxIdHU379u0JCwvzd3imgtnAtSmMrwaufX4xnYj0A17DMd12lqo+LyLTgE2q+qkzMbwF1MYxiP2oqq7x0E6VSxLuMjMzXQnjyJEjREVFERMTQ1RUFNWrV/d3eKYCVMYkkZCQwJgxY1z3U+rSpQv//Oc/ufnmm4usW1IPPPAALVu25MknnyxTzIEqaJNEeanqScLdmTNnSE5OZufOnaSmptKuXTuio6Pp2LFjwN922JReZU0SY8eO5eDBg+Va99133+Xtt9/mq6++Ko8wg0KwToE1PlCrVi2uvvpqrr76as6ePeu6ovTTTz+lTZs2xMTE0KlTpwLjHsZUFaoakHdazc3NLTAZxVNZSdvwJZtrGeTCw8OJjY3l7rvv5n/+53+IiYkhKSmJV199lXnz5rF582bOnj3r7zBNJfbiiy8yYsSIS8oeeeQRfve73wEwZ84cYmJiqFOnDlFRUcycOdNrW23btuXLL78E4Pz580yYMIGIiAi6dOnCpk2bLqn7wgsvEBUVRZ06dejSpQtLliwBIDk5mQceeIBvvvmGyy+/nIiICODSR54CvPXWW3To0IGGDRsyZMgQjhw54loXEhLCjBkz6NixIxEREUyaNMlrzKrK888/T1RUFI0aNWLUqFGkp6cDeHyMq7dHuy5btowuXboQERFBr169SE7+v8vJ2rZty4svvkjXrl2pXbt2xd59QlWD4scRqimu8+fP6/bt23XRokX63HPP6dy5c3XTpk2amZnp79BMKQXqZyAlJUVr1aqlp0+fVlXV3NxcbdasmW7cuFFVVT/77DPdv3+/qqquW7dOw8PDNTExUVVV4+PjNTIy0tVWmzZtdM2aNaqq+vjjj+vNN9+s6enpevjwYe3SpcsldT/66CM9evSoqqouWrRIa9Wq5VqeM2eO3nTTTZfEOWHCBH366adVVXXNmjXasGFD3bJli2ZnZ+tDDz2kN998s6uuiOigQYM0IyNDDx48qI0aNdIvvvjCY///9re/aY8ePTQtLU2zs7P1/vvv19GjR6uq6oEDB1REdPz48Xr27Fk9f/68x7Ldu3drrVq1dM2aNZqTk6MvvviiRkVF6YULF1y/l9jYWE1NTdXz5897jMPb/4ezvPT73rK8uSJ/AvUDEgyysrJ0x44d+tFHH+lzzz2n77zzjn777bd66tQpf4dmSiCQPwM33XSTvvfee6qqumrVKo2KivJad8iQIfr666+rauFJol27drpq1SrXupkzZ15SN79u3brpsmXLVLXoJPGb3/xGH3/8cde606dPa1hYmKakpKiqI0l8/fXXrvUjR47UF154weN2o6Oj9csvv3Qtp6WlaVhYmObm5uqBAwc0JCREDxw44FrvqWz69Ol61113uZbz8vK0RYsWmpCQ4Pq9zJkzx2vfVX2XJGxMogqoXr06MTExxMTEkJOTw969e0lKSiI+Pp5GjRoRHR1NTEyMxwfMm+Axbdq0cmlnypQpJX7P6NGj+eCDDxgzZgwffPABd999t2vdypUr+dOf/sTu3bvJy8vj3LlzXHnllUW2mZaWdsljR1u3bn3J+rlz5/Lqq69y4MABwDGhw9ujUD21ffXVV7uWa9WqRYMGDUhNTaVVK8elXU2aNHGtL+zRqSkpKQwdOtR1pwRVJSwsjGPHjrnquPfDU1laWtol/RMRIiMjL3nkqac2KoIliSomNDSUTp060alTJ3Jzc9m3bx87d+7kq6++on79+sTExBAdHe06j2uCR2l27uVlxIgR/O///i+pqaksXryYb7/9FoDs7GyGDx/O+++/z+DBgwkJCWHo0KEeZ+Hk16xZMw4dOkR0dDTg2BlfdPDgQe69917Wrl1Ljx49AIiNjXW1W9SgdfPmzS9p78yZM5w4caJUO+JWrVoxe/ZsVxzuLm6jOI88/eGHHy5Zf+jQoUvi8ddAvA1cV2HVqlWjQ4cODB48mD/84Q/07t2bkydPMnv2bGbMmMG6deuK/c3MVG0NGzbklltuYeLEibRr145OnToBjiSRnZ1Nw4YNCQkJYeXKlaxatapYbY4cOZLnnnuO9PR0Dh8+zN///nfXujNnzhASEkLDhg3Jy8vjnXfeuWQn26RJEw4fPsyFCxc8tj169Gjeeecdtm3bRlZWFn/84x/p3r07kZGRJe77fffdxx//+EfX1Nzjx4+zbNky13pPCTF/2ciRI1mxYgVr164lJyeHl19+mZo1a3pMPBXNkoQBHAmjXbt2DBw4kN///vf07duX06dPM3fuXP75z38SHx/PTz/9VKxvgKZquvvuu1mzZg333HOPq6x27dq8/vrrjBgxgoiICBYsWMDgwYO9tuH+bXnKlCm0atWKtm3b0q9fP8aNG+daFx0dzR/+8Ae6d+9O06ZN2bFjBzfeeKNrfa9evbjiiito2rQpjRs3LrCd3r17M336dIYNG0aLFi3Yv38/CxYs8BiHp2V3jzzyCIMHD6ZPnz7UrVuX66+/no0bNxb63vxlHTt25P3332fSpEk0atSIFStWsHz5ctetePw5ndcupjOFUlUOHz7Mzp072blzJ2FhYa4xjKZNmwbkXPTKqjJeTGfKj11xbUnC71SVtLQ0du7cSVKS4/ZbFxNG8+bNLWH4mCUJUxhLEpYkAoqqcvToUZKSkti5cyc5OTmuW5xHRkZawvABSxKmMJYkLEkELFXl+PHjriOMc+fOuRKGPUSp/FiSMIWxJGFJImj8/PPProSRmZlJ586d6dWrF+Hh4f4OLahZkjCFsSRhSSIo/fLLL3zzzTccPnyYcePG+eyJf1WBJQlTGEsSliSClqqyevVq9u3bZ4miDCxJmML4KknYyWLjcyLCrbfeSps2bXjvvfc4f/68v0MyxhSTJQlTIUSEPn360KpVK95//31LFMYECUsSpsKICH379qV58+bMmzePrKwsf4dkjCmCJQlToUSE/v3707RpU0sUlYj7w4LK4t133+Wmm24qh4hMefF5khCRfiKSLCK7ReRxD+tfEZFEEdksIrtE5Bdfx2T8S0QYMGAAjRo1Yv78+WRnZ/s7JBMgVMv3saO5ubnFKitpG1WJT5OEiIQAfwf6AlcAo0Wks3sdVf29qsaq6lXAG8AnvozJBAYRYeDAgTRo0MASRRmsWLHC9ajMi9LT01mxYkWFtTFu3DgOHjzIoEGDqFOnDi+//DIA3377LTfccAP169cnNjaWhIQE13vmzJlD+/btqVOnDu3bt+eDDz7w+tjR/DIyMvjtb39L8+bNiYyM5Omnn3bN6nn33Xe58cYb+f3vf0/Dhg2ZNm2axzJV5c9//jNt2rShadOmTJgwgYyMDMDzI0ertLI8saioH6A7sNJteTLweCH11wO9vazz8MwlE+zy8vJ0yZIlOmfOHM3OzvZ3OAHN02fg5MmT+uCDD+rJkyc9LhdHebTRpk2bS57Olpqaqg0aNNDPP/9cVVVXr16tDRo00J9//lnPnDmjderU0T179qiq6tGjR3Xnzp2q6vmJcvkNGTJEH3jgAT137pweP35cf/3rX+vMmTNd7w8NDdV//OMfmpubq+fPn/dYNmvWLO3QoYMeOHBAz5w5o8OGDdOxY8eqqudHjgYDb/tIAvnxpcCdwEy35THA617qtgJScV674WF9WX5/JoDl5ubqJ598onPnzrVEUQhvn4GLO/X9+/eXeOdeXm24P3ZUVfWFF17QcePGXVKnb9++OnfuXD1z5ozWr19fP/nkEz137twldYpKEseOHdMaNWpcsuP+4IMPtGfPnq73t27dukCb+ct69+6tb775pmt5165dhT5yNBj4KkkE0sD1KOAjZ6dMFRISEsLgwYOpVasWCxcuJCcnx98hBZV69erx6KOP0rZtWx599FHq1avnlzbcpaSksGjRIiIiIoiIiKB+/fqsX7+eI0eOEB4ezsKFC3nzzTdp1qwZgwYNYteuXcVu98KFCzRr1szV7v3333/Jw7E8PTgof1n+x4W2bt2anJycIh85WhX5+vGlqTiOEC5q6SzzZBTwYGGNTZ061fU6Li6OuLi4skVnAkZISAhDhgxh8eLFLFy4kLvuusv1wBVTuPT0dF566SX279/PSy+9xLPPPlvinXxZ28g/2BwZGcm4ceOYMWOGx/q33XYbt912G1lZWTz55JPce++9JCQkFDloHRkZSc2aNTlx4oTXusV5yE/+x5empKQQFhZGkyZNOHTokNd2gkF8fDzx8fHl12BZDkOK+gGqAT8CrYHqwBYg2kO9zsC+Itoqw4GYCRa5ubm6aNEinTdvnl64cMHf4QQUT5+BQBmT6NGjh7711luu5UOHDmmzZs30iy++0NzcXD137pzGx8dramqqHjt2TJcuXapnzpzR3NxcnTJlisbFxamq6ueff65t27Yt9LTjkCFD9JFHHtGMjAzNy8vTvXv3akJCgqp6Pl3lqeztt9/Wjh076v79+zUzM1OHDx/uOj12cUwiNze32P0PBN72kQTymIQjPvoBu4A9wGRn2TRgoFudKcBfiminTL9AEzxycnJ04cKFOn/+fM3JyfF3OAHD02fg008/LbAzP3nypH766afFbrc82li6dKm2atVK69evr3/9619VVXXjxo16yy23aEREhDZu3FgHDhyohw4d0iNHjugtt9yi9erV0/r162vPnj01KSlJVVWzs7N14MCBGhERoY0aNfK4rYyMDH3ggQe0ZcuWWq9ePb3qqqt04cKFqlr8JJGXl6fTp0/XyMhIbdy4sY4bN07T09NVVV1jEpYkHD92gz8TkHJzc/nwww8REYYPH061atX8HZLf2Q3+TGHsBn+mSqlWrRojRowgLy+Pjz/+uMpf0GSMv1iSMAHrYqLIycnhk08+IS8vz98hGVPlWJIwAS00NJSRI0eSnZ3N4sWLLVEYU8EsSZiAFxoayl133cXZs2dZsmSJJQpjKpAlCRMUQkNDGTVqFKdPn2bZsmWWKIypIJYkTNAICwtj9OjRnDp1iuXLl9tMH2MqgCUJE1QuJoqTJ09aojCmAth1EiYoZWdnM2/ePBo2bMjAgQOD9hYKJdGmTZtLbiVhjLvWrVtz4MCBAuVlvU7CkoQJWllZWcybN48mTZowYMCAKpEojCkpu5jOVFk1atTgnnvu4ciRI6xcudJOPRnjA5YkTFCrUaMGY8aMITU1lS+++MIShTHlzJKECXo1a9Zk7NixHDx4kFWrVlmiMKYcWZIwlcLFRHHgwAFWr15ticKYcmJJwlQal112GWPHjmXv3r18+eWXliiMKQeWJEylEh4ezrhx49i9ezdr1661RGFMGVmSMJXOxUSRnJxMQkKCv8MxJqhZkjCVUq1atRg3bhw7duxg3bp1/g7HmKBlScJUWrVr12b8+PFs27aNr776yt/hGBOULEmYSu1iotiyZQvr16/3dzjGBB2fJwkR6SciySKyW0Qe91JnpIjsEJHtIvK+r2MyVcvll1/O+PHj2bx5M19//bW/wzEmqPj03k0iEgLsBnoDacAmYJSqJrvViQIWAj1VNUNEGqrqzx7asns3mTLJyMhgzpw5XHfddXTv3t3f4RhTIQL93k3XAXtUNUVVLwALgMH56vwX8A9VzQDwlCCMKQ916tRh/PjxbNiwgQ0bNvg7HGOCgq+TRAvgkNvyYWeZu45AJxH5j4h8LSJ9fRyTqcLq1q3L+PHj+eabb9i0aZO/wzEm4AXCwHUoEAXcDNwNvCUidfwbkqnM6tWrx/jx41m/fj3fffedv8MxJqCF+rj9VKCV23JLZ5m7w8C3qpoHHBCR3UAH4Pv8jU2dOtX1Oi4ujri4uHIO11QV9evXZ9y4cbz77ruEhIRw1VVX+TskY8pFfHw88fHx5dZekQPXIlITGAjcBDQHzgE/ACtUdUcR760G7MIxcH0E2AiMVtUktzp9nWUTRKQhjuTQTVVP5mvLBq5NuTtx4gRz584lLi6O2NhYf4djTLnz6cC1iEwD1gM9gA3ADGARkAM8LyL/FpErvb1fVXOBScAqYAewQFWTRGSaiAx01vkCOCEiO4A1wP/mTxDG+EqDBg0YO3Ysa9euZcuWLf4Ox5iAU+iRhIjcrqorClnfGGilqj4/sWtHEsaXfv75Z+bOncutt97KlVd6/d5jTNDx2zOuRSRUVXNKu+FSbM+ShPGp48ePM3fuXPr06cOvfvX0VelJAAAb/klEQVQrf4djTLnw9emm/7i9fi/f6o2l3agxgahRo0aMHTuWVatWsWNHocNtxlQZRU2BreX2+op860qdmYwJVI0bN2bMmDGsXLmSnTt3+jscY/yuqCRR2PkdO/djKqUmTZowZswYPvvsM5KSkop+gzGVWFHXSdQTkaE4kkk9ERnmLBegrk8jM8aPmjZtyj333MO8efMICQmhU6dO/g7JGL8oanbTO4W9WVUnlntE3mOxgWtT4dLS0pg/fz533HEHHTt29Hc4xpSY32Y3VTRLEsZfUlNTmT9/PkOGDKFDhw7+DseYEvH17KZBItLabfkZEdkqIstEpG1pN2pMMGnRogWjR49myZIl/Pjjj/4Ox5gKVdTA9bPAcQDnFdJjgP8HLAP+5dvQjAkcLVu2ZNSoUSxevJh9+/b5OxxjKkyRs5tU9azz9TBglqp+r6pvA418G5oxgSUyMpKRI0fy8ccfs3//fn+HY0yFKCpJiIjUdj5hrjeOeytdVNN3YRkTmFq3bs2IESP46KOPOHDggL/DMcbnikoSfwO2AN8BSRfv0SQisTju6mpMldOmTRuGDx/Ohx9+SEpKir/DMcaninOr8BZAY2Cr85kPiEgzIExVD/o+RFccNrvJBJR9+/bx8ccfc9ddd9GqVaui32CMH/h0CqyIFPokFlXdXNoNl5QlCROI9u7dyyeffEKvXr246qqrELG71ZjA4uskkYfjAUM/XyxyW62q2qu0Gy4pSxImUP30008sWbKEWrVqcccdd3D55Zf7OyRjXHydJH4HDAdOAQuAxap6urQbKwtLEiaQ5ebm8tVXX7Fp0yb69etHly5d7KjCBIQKueJaRNoBo4DBQArwF1Wt0Md4WZIwwSAtLY3FixfTuHFjbr/9dsLDw/0dkqnifHrF9UWqug9YiuMxpNcBdhMbYzxo3rw59957L3Xq1OHNN99k165d/g7JmDIp6nST+xHEIRynnFao6rmKCe+SWOxIwgSVlJQUlixZQps2bejbty81a9qlRabiVcTA9TYcRxEZ5HuGhKq+UowA++G43iIExxXbL+RbPx54CTjsLPq7qs720I4lCRN0srKyWLVqFXv37mXw4MG0bWu3PDO+kZubS1paGpGRkZeU+zpJTKWQhwup6rRCG3dcqb0bx9XaacAmYJSqJrvVGQ9craoPF9GWJQkTtPbs2cPy5cuJjo7m1ltvJSwszN8hmUrkq6++4qGHHuLaa6/lrbfeumRdQN8qXES6A1NUtb9zeTKOqbMvuNUZD1yjqg8V0ZYlCRPUzp07x8qVK0lNTWXo0KG0bNnS3yGZIJeamspjjz3GunXrePnllxk5cmSBWXW+vlX4UyJSv5D1vZx3h/WmBY6xjIsOO8vyGyYiW0RkkYjYJ8dUSpdddhnDhg2jd+/eLFiwgDVr1pCTk+PvsEyQmjlzJl27dqVNmzYkJydz1113+WTadVGPL90OfCoi54HNOG4bXhPoAHQDVgN/KWMMy4D5qnpBRO4F3sVxesqYSikmJoZWrVqxfPly3n77bYYMGULTpk39HZYJMp06deLbb78lKirKp9sp7nUSHYAbgGbAOSAJWFfULCfn6aapqtrPuVzgdFO++iHAL6paz8M6nTJlims5Li6OuLi4ImM3JlCpKlu3buXf//433bt354YbbiAkpFiz0o3xKj4+nvj4eNfytGnTAnpMohqwC8eRwRFgIzBaVZPc6jRV1aPO10OBR1X1eg9t2ZiEqZROnTrF0qVLyc7OZsiQITRs2NDfIZkAcubMGcLCwqhevXqp3l8hF9OVlqrmApNwXIS3A1igqkkiMs1tLONhEflBRBKddSf4MiZjAk3dunUZO3YsV155JbNnz2bDhg3YFyKjqixcuJDo6Gg+//xzv8Xh0yOJ8mRHEqYqOHHiBEuWLCE0NJTBgwdTr16BM6+mCti+fTsPP/wwv/zyC2+88QY333xzqdvy+ZGEiFQTkf8p7QaMMcXXoEEDJk6cSPv27XnrrbdITEy0o4oqJDs7m4cffpjevXszfPhwvv/++zIliOzs7DLHVNTsJlQ1V0RGA6+WeWvGmCKFhIRw44030qFDBxYvXkxycjKDBg2idu3a/g7N+FhYWBht2rRh586dpR6bUlUOHTpEYmIiycnJRb+hCMWd3fQqEAYsBM64BWMPHTLGh3Jzc0lISGDz5s3079+fK664wt8hmQCVmZnJtm3bSExMBCA2NpauXbty+eWXV8itwtd6KLaHDhlTQQ4fPsySJUto1qwZAwYM4LLLLvN3SKaMsrOzSz1j6aLc3Fz27NlDYmIiBw8eJDo6mtjYWFq2bOm6sC6gb8tRnixJmKruwoULrF69mqSkJAYNGkSHDh38HZIphQsXLvDGG2/w2muvsWPHjlKdRjx+/DiJiYls27aNBg0aEBsbS0xMjMekU1EPHaoLTAEujqAkAH9S1VOl3XBJWZIwxmH//v0sXbqU9u3b06dPH2rUqOHvkEwxrV69mocffpjIyEhee+01OnfuXOz3ZmVlsWPHDhITE0lPT6dr167ExsbSoEGDQt9XUUniYxzPun7XWTQW6Kqqw0q74ZKyJGHM/8nKyuKLL75g//79DB48mDZt2vg7JFOIQ4cO8bvf/Y7Nmzfz6quvMnjw4GLdZ0lVOXjwoGsQum3btsTGxhIVFVXsq/MrKklsUdVuRZX5kiUJYwravXs3n376KVdccQW9evWyW5AHqF27drFw4UIeffTRYo0nZWRksHXrVrZs2UJISIhrELpWrVol3nZFJYlvcNwu4z/O5RuAl1W1R2k3XFKWJIzx7OzZs3z22WccO3aMIUOG0KKFpxstm0CXm5vL7t27SUxM5NChQ8TExBAbG0uLFi3KdHfXikoSXYG5QF1n0UlgvKpuK+2GS8qShDGF++GHH/j888+5+uqrufnmm6lWrZq/Q6qSVLVEO/WffvqJxMREtm/fTsOGDYmNjSU6OrrMM58u8nmScN6ZdbiqLhKROgCqmlHaDZaWJQljipaZmcny5cvJzMxk6NChNG7c2N8hVRkZGRlMnz6dEydOMHt2gScwX+L8+fP88MMPbNmyhYyMDNcgdERERLnHVVFHEt+p6jWl3Uh5sCRhTPGoKomJiaxZs4brr7+eHj162C3IfSgvL4958+YxefJk+vTpw/PPP0+TJk0K1FNVUlJSSExMZNeuXbRr147Y2Fjat2/v079PRSWJ54GfKXjF9S+l3XBJWZIwpmTS09NZsmQJeXl5DBkyxCffUqu6zZs3M2nSJHJycnjjjTf49a9/XaBORkYGW7ZsYcuWLYSFhREbG8uvfvWrUg1Cl0ZFJYn9HopVVduVdsMlZUnCmJJTVTZs2MC6devo2bMn11xzjU8ecVlVvfTSS0RERDBx4sRLjgZycnJcg9CHDx/miiuuIDY2lubNm1f477+ixiR6qOr60m6kPFiSMKb0fv75Z5YsWUKNGjW44447qFu3btFvMiV27Ngx1yB048aNXYPQ/pyaXFFHEomqGlvajZQHSxLGlE1eXh7/+c9/2LBhA3369OHKK6/0+q12xYoV3HDDDZc8zyI9PZ3169dz++23V1TIQeH8+fNs376dLVu2cPr0adcgdP369f0dGlBxSeJl4BvgE3/tqS1JGFM+jh49yuLFi4mIiGDgwIEez42np6fz5JNP8uyzz1KvXr0Cy1XJ4cOHeeyxxxg/fjx9+/YFHKfxDhw4QGJiIrt37yYqKopu3brRrl27gJskUFFJIhOoBeQC5wDBMSZRp7QbLilLEsaUn5ycHOLj49m6dSsDBgwgOjq6QJ2LieHRRx/lpZdeqnIJIisri1deeYW//vWv3H///TzxxBPk5OS4BqFr1KjhGoQODw/3d7he2V1gjTGldujQIZYsWULLli3p378/NWvWvGT9gQMHaNu2Lfv3769S94dasWIFv/vd74iJieGll14iOzubLVu2kJaW5hqEbtasWVBMAqioIwkB7gHaqup0EYkEmqnqxmK8tx/wNxyPSp2lqi94qXcn8CFwjaeHGVmSMMY3srOzWb16Nbt27eKOO+6gffv2QNU9ksjKymLgwIHcf//9hIeHs337dpo2bUpsbCydO3cOuvtjVVSSeBPIA3qparSI1AdWqeq1RbwvBNgN9AbSgE3AKFVNzlevNrACx9PvJlmSMKbi7du3j6VLl9KxY0euvfZapk2bVuXGJM6dO+cahD579izdunWjW7duQd3nikoSm1X1KvdZTiKyVVW7FvG+7sAUVe3vXJ6MYyzjhXz1XgVWAY8Bf7AkYYx/nD9/ns8//9z1XG33x6VW1tlNqsr+/ftJTExkz549dOjQwTUIHQynk4pS1iQRWsx6F0SkGqDOjTbCcWRRlBbAIbflw8B17hVEJBZoqaorReSxYsZjjPGBmjVrMmTIEJKTk1mxYgVpaWn07NmT0NBQ6tWrVyBBqCp5eXmun6KWi1vmqzruy5mZmSQlJdG0aVPCw8OJjY21R8N6UNwk8TqwGGgsIs8Cw4Gnyrpx51jHK8B492Jv9adOnep6HRcXR1xcXFlDMMZ40LlzZyIjI1mxYgUvvfQS1apV87ijBQgJCUFECAkJueSntGVF1SmqfmhoaKFtqCqrVq1iyZIlDBkyhJEjR1aq26vHx8cTHx9fbu0Ve3aTiHTGMbYgwBpVTSrGe7oDU1W1n3P5ktNNzrvK/gicdrbbFDgB3JH/lJOdbjKm4qkqZ8+eRUS87niDyZYtW/jtb39L3bp1mTlzpmuQvjIL6CmwzlNUu3AklyPARmC0twQjImuB36tqood1liSMMaWWlJTELbfcwvPPP8/EiRODLsGVVkWNSZSKquaKyCQcg9IXp8Amicg0YJOqfpr/LRRyuskYY0orOjqaXbt2BcztMoKFXUxnjDGVWFmPJALrJiPGGFMO9u7d6+8QKg1LEsaYSuPo0aOMGDGCoUOHkpOT4+9wKgVLEsaYoKeqzJ49myuvvJIOHTqwYcMGQkN9OuRaZdhv0RgT1A4cOMBvfvMb0tPTWbVqFd26dfN3SJWKHUkYY4KaqtK/f382bNhgCcIHbHaTMcZUYja7yRhjjM9YkjDGBIWEhAT++7//GzujULEsSRhjAlp6ejr33Xcf99xzD3369Kkyt9MIFJYkjDEBa8mSJXTp0gURYceOHQwePNjfIVU5NgXWGBOQFi5cyDPPPMP8+fO5+eab/R1OlWWzm4wxASk7O5u8vDxq1qzp71CCWkDfKrw8WZIwxpiSsymwxpiglpOTw549e/wdhvHCkoQxxm+2bNlC9+7d+fOf/+zvUIwXliSMMRXu3LlzPPHEE/Tt25dJkyYxZ84cf4dkvLDZTcaYCvX1118zYcIEYmNj2bp1K02bNvV3SKYQNnBtjKlQCQkJnDp1ijvuuMPfoVQJNrvJGGOMVwE/u0lE+olIsojsFpHHPay/T0S2iUiiiKwTkc6+jskYY0zx+PRIQkRCgN1AbyAN2ASMUtVktzq1VfW08/Ug4EFV7e+hLTuSMCZIXHxS3JEjR3jqqaf8HU6VVtYjCV8PXF8H7FHVFAARWQAMBlxJ4mKCcKoN5Pk4JmOMD/3444/ce++9ZGZm8vbbb/s7HFNGvj7d1AI45LZ82Fl2CRF5UER+BJ4HHvZxTMYYH8jJyeHFF1+ke/fuDBw4kG+++YauXbv6OyxTRgExBVZV/wn8U0RGAU8DEzzVmzp1qut1XFwccXFxFRCdMaY4nnrqKb7//ns2btxIu3bt/B1OlRUfH098fHy5tefrMYnuwFRV7edcngyoqr7gpb4AJ1W1nod1NiZhTAA7c+YM4eHh9ryHABPos5s2AVEi0lpEqgOjgGXuFUQkym1xII6BbmNMkKlVq5YliErIp6ebVDVXRCYBq3AkpFmqmiQi04BNqvopMElEbgWygZPAeF/GZIwpm/T0dE6dOkXr1q39HYqpAHYxnTGm2D755BMeeughHnvsMR555BF/h2OKIdCnwBpjKoEjR44wadIkfvjhBxYsWMBNN93k75BMBbG7wBpjCjV37ly6du1KdHQ0W7dutQRRxdiRhDGmUDVr1mT16tVceeWV/g7F+IGNSRhjTCUW6FNgjTEB7sSJE7zyyisMHDgQ+yJm8rMkYUwVpKqsX7+esWPH0r59exITE3niiSf8HZYJQHa6yZgqaNSoUSQmJnLfffcxfvx4GjRo4O+QjI/YQ4eMMSV29OhRmjRpYldIVwE2JmGM8ej06dN89913Htc1bdrUEoQpFksSxlQy27Zt48EHH6RVq1bMnDnT3+GYIGdJwphKQFWZO3cu119/PQMGDKBJkyZs27bNkoQpM7uYzphKQERITk5m8uTJDBgwgNBQ+2ib8mED18YYU4nZDf6MqSL27t3LzJkzCQkJ4bnnnvN3OKaKsDEJYwLYhQsXWLx4MX379qV79+7k5uYyceJEf4dlqhA7kjAmQGVnZ9OpUydatmzJ/fffz9KlS6lZs6a/wzJVjI1JGBPAUlNTadGihb/DMEHMrrg2JsgdOXKE06dP06FDB3+HYiqhgL/iWkT6iUiyiOwWkcc9rP8fEdkhIltE5N8iEunrmIzxt7y8PP79739z5513EhMTw9q1a/0dkjEe+fRIQkRCgN1AbyAN2ASMUtVktzq3ABtU9byI3A/EqeooD23ZkYQJeqdPn+bNN99kxowZ1K5dmwceeIC7776byy+/3N+hmUoq0KfAXgfsUdUUABFZAAwGXElCVRPc6n8L3OPjmIzxGxFhz549zJs3j+uuu87un2QCnq+TRAvgkNvyYRyJw5vfACt9GpExflSrVi27VYYJKgFznYSIjAGuBl7ydyzGlJaq8s033zBhwgQ+/PBDf4djTJn5+kgiFWjlttzSWXYJEbkVeAK4WVUveGts6tSprtdxcXHExcWVV5zGlElGRgbz5s3jX//6F2fOnOG+++6z/0/jF/Hx8cTHx5dbe74euK4G7MIxcH0E2AiMVtUktzqxwIdAX1XdW0hbNnBt/CovL4+QkIIH31u2bKFnz5707t2b+++/n169enmsZ4w/BPx1EiLSD3gNx6mtWar6vIhMAzap6qci8m+gC44kIkCKqg7x0I4lCVMhTp48yfTp0zl27BhHjx7l2LFjHDt2jHr16rFnz54C9XNycjh+/DjNmjXzQ7TGFC7gk0R5sSRhSuvcuXO8/vrrrp39xR2/qrJjx44C9U+fPs2//vUvmjRpQtOmTWnSpAlNmjShYcOGVKtWzQ89MKb0LEmYKicnJ4f333+/wE4/IyODTZs2FaifnZ3Nk08+WWCnf/HHmMrMkoQJeqrKypUrXTv9iz/Hjx/n888/L/DtPS8vj4kTJ9K4ceMCO/5f/epXdu2BMW6qVJJISEgoUN6jRw/CwsIKlH/99dfk5OSUqP6FCwUnVl1//fUe669fv95j++VVPyEhgaysLFT1kp9bb72V6tWrF6i/YsUKzp8/X6D+kCFDqFGjRoH6CxYs8Fh/zJgxHu80OmvWLM6dO1eg/n333cdll11WoP7f/vY3zp49i6qSl5fHiRMnOHbsGLNmzSI8PLxA/aFDh1KnTp0C3/J79eplp3iMKYNAv+K6XD311FMFypYvX07dunULlP/lL38hIyOjRPUzMzMLlC9btsxj/eeff95j++VV/7XXXiMzMxMRueTnxhtv9JgkFi5cyOnTpwvUHzBggMcksW7dOs6cOVOg/qhRBe6IAkBycjJnz54tUD8vL89j/YyMDM6dO4eIEBISQqtWrbj22mu9zvpZvHixx3JjjH8F1ZFEsMRqjDGBIuDvAmuMMSZ4WZIwxhjjlSUJY4wxXlmSMMYY45UlCWOMMV5ZkjDGGOOVJQljjDFeWZIwxhjjlSUJY4wxXlmSMMYY45UlCWOMMV5ZkjDGGOOVJQljjDFeWZIwxhjjlc+ThIj0E5FkEdktIo97WH+TiHwvIhdEZJiv4zHGGFN8Pk0SIhIC/B3oC1wBjBaRzvmqpQDjgXm+jCXQxcfH+zsEn6rM/avMfQPrX1Xn6yOJ64A9qpqiqheABcBg9wqqelBVfwCq9BOFKvs/amXuX2XuG1j/qjpfJ4kWwCG35cPOMr8o7j9DYfU8rctfVtiyt9floTjtlbRvnsr90T9f/e08lVem/pX0/7Wy9a+4fS2ryrxvqVID15X5D1nc9mwnWnR5ZeqfJYn4Yq8ri8q8b/HpM65FpDswVVX7OZcnA6qqL3io+w6wXFU/8dJWlT4dZYwxpVWWZ1yHlmcgHmwCokSkNXAEGAWMLqS+146UpZPGGGNKx6enm1Q1F5gErAJ2AAtUNUlEponIQAARuUZEDgHDgX+JyHZfxmSMMab4fHq6yRhjTHCrUgPXxhhjSsaShDHGGK+COkmIyGARmSkiH4jIbf6Op7yJSFsReVtEFvk7lvImIuEiMkdEZojI3f6Op7xV5r8dVO7Pnoh0FpE3RWSRiNzv73h8wfn52yQiA4qsWxnGJESkHvCSqv6Xv2PxBRFZpKoj/R1HeRKRMcBJVV0hIgtUdZS/Y/KFyvi3c1eZP3siIsC7qjrO37GUNxGZBmQCO1X1s8LqBsSRhIjMEpFjIrItX3mhNwd08xTwD99GWXrl0L+AV4o+tuT/rsbPrbBAS6my/w3L0L+A/uxB6fomIoOAT4FCd6CBoKT9E5FbgZ3AcQq57MBFVf3+A9wIdAO2uZWFAD8CrYEwYAvQ2bluLPAK0Bx4Hujl7z74qH/NnMsf+rsPPujjPcAA5+v5/o6/vPvnVifg/3al7V8wfPbK8rdz1vvU3/GXd/+APzv3L18Ai4tqPyCOJFT1P8DJfMVebw6oqu+p6u+BO4HewHARubciYy6JMvQvS0TeBLoF+rfUkvYRWIzj7/YPYHnFRVo6Je2fiEQEy98OStW/hwiCzx6Uqm+3iMhrIvIvYEXFRltypdi/POXcv8wD3iqqfV9fcV0Wnm4OeJ17BVV9A3ijIoMqR8Xp3y/AAxUZVDnz2kdVPQv8P38EVY4K61+w/+2g8P4F82cPCu9bApDgj6DKUXH2L3OL01BAHEkYY4wJTIGcJFKBVm7LLZ1llUVl7x9U/j5a/4JXZe4blGP/AilJCJeOtLtuDigi1XHcHHCZXyIrH5W9f1D5+2j9C97+Vea+gS/75++Reedo+3wgDcgCDgITneX9gV3AHmCyv+O0/lXdPlr/grd/lblvFdG/SnExnTHGGN8IpNNNxhhjAowlCWOMMV5ZkjDGGOOVJQljjDFeWZIwxhjjlSUJY4wxXlmSMMYY45UlCWOMMV5ZkjCmDESknog84u84jPEVSxLGlIGqpgMRIjLQ37EY4wuWJIwpu4+A4f4OwhhfsCRhTBmp6nYgWkSq+TsWY8qbJQljyscRoJe/gzCmvFmSMKaMnAPX72GnnEwlZEnCmDIQkYeBJFX9GLhaRKSo9xgTTCxJGFNKItIfuFxVVzmLEoA4/0VkTPmzhw4ZU0oiUkNVs9yWqwM1VTXDj2EZU64sSRhjjPHKTjcZY4zxypKEMcYYryxJGGOM8cqShDHGGK8sSRhjjPHKkoQxxhivLEkYY4zx6v8DXZuvXlW+PjIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f146c374eb8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "pipeline(2000, False, False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "l = 0.01, validation MSE = 0.4421864459799068\n",
      "l = 0.1, validation MSE = 0.441830005249123\n",
      "l = 1, validation MSE = 0.4384260891567443\n",
      "l = 10, validation MSE = 0.41488435972033527\n",
      "l = 100, validation MSE = 0.36584997844103806\n",
      "l = 1000, validation MSE = 0.3710655728890064\n",
      "l = 10000, validation MSE = 0.4509858498087078\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZAAAAEfCAYAAABvWZDBAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xd8VFXawPHfE6qAdJAWAog0UQKrWEBE0RVEBEGqFEXE8rrrrvuqIBbKi6vIqutaVopSXCkiSEcsEFYRUTEgJRRJKKEISyIhkP68f2QyOwmTZDKZyUwmz/fzmQ9zzz333Ockwzy599x7j6gqxhhjTFGFBToAY4wxpZMlEGOMMV6xBGKMMcYrlkCMMcZ4xRKIMcYYr1gCMcYY4xVLIKZUE5EIEckSkTDH8hoRGeFJXS/2NV5EZhQnXi/22VVE9viorVgRudXxvsT7YkKPJRATUCKyVkQmuinvKyLHPfyyd97MpKp3qup8T+oWEtfNInIk14aqf1XVsZ5sXxQiMkpEMkTkrIgkisg2Eent2OfXqtrW1/v0V19M2WIJxATaXGC4m/LhwHxVzSrheHIIHiYbH9msqtVVtSbwPrBYRGqU4P6NKTJLICbQPgXqiEjXnAIRqQncBcxzLN/p+Kv8NxE5JCIv5teYiGwQkdGO92EiMl1ETonIAaB3nrr3i8hux1/+B0RkrKO8CrAGaCQiSY71DUTkRRGZ77L93SKyU0TOiMhXItLGZV2siPxFRLaLSIKILBCRih7+TN4HLgEuz3sk5Gh3nIjsEpH/iMhs13ZF5C4R+cmxz69F5Kp8fk7Ovric2hvp+Pn+KiLPutQVxz4POH6WCx2/I1PGWQIxAaWqKcDHwEiX4sHAHlXd6Vg+B4xQ1RpkJ4FHRORuD5ofC9wJdACuAe7Ns/4kcKeqVgceAF4XkUhVPQ/0Ao6p6qWOI4MTOSEDiEgr4CPgj0A9YC2wUkTKu7Q/EPg90NwRw/2FBezY/iEgCdjvuk8Xw4DbgcuB1sBzjm07ArMd29cG3gNWiEiFfHaXt90uwBXAbcALItLaUf5H4G7gJqARkAC8U1hfTOizBGKCwVxgoMtf0iMcZQCo6iZV3eV4vxNYCNzsQbsDgTdU9ZiqJgJ/dV2pqmtVNc7x/t/AerK/JD0xCFilql+paiYwneyjhhtd6vxdVU869r0SiCygvRtE5AxwjOwE2k9Vk/Kp+w+XPk0FhjrKHwL+qao/aLb5QCpwvQf9UWCiqqap6g5gO9lJD+BhYIKqHlfVdGAycK+3FyOY0FG+8CrG+JeqfiMip4B+IvIDcC1wT856EekMvAy0Byo6Xh970HQjwHUg/JDrShHpBbwAtCL7j6lLgB0eht3ItT1VVceppsYudU66vD8PNCygvW9VtZuH+z7q8v6QIxaACGCkiPzBsSxABZf1hckbbzWXdpeJSM54lADpwGXAcQ/bNiHI/oIwwWI+MIrswfPPVPWUy7qPyB4raewYZH6P7C+xwhwHwl2WI3LeOI52lgDTgHqqWovs01A57RY2gH7MtT2HcHJ/uftL3j4dc7w/AkxV1dqOVy1Vraaqi4q5v8NArzztVlVVSx5lnCUQEyzmkX3ufQwup68cqgEJqpruOBoZlmd9fslkMfBHEWksIrWAZ1zW5RzJnFbVLMfRyO9d1p8ke3C/egFt9xaRW0SkvIj8L5ACfFtwN33ifxx9qg08S/YpPYCZZI8PdQYQkaqOCxCqetBmQQn5PeAlEWnqaLeeh2NQJsRZAjFBQVUPAZuBKsCKPKsfA6aIyG9kDxjn/Yta83k/E/iM7PP5PwCfuOzvHNmDwx87xh6GAMtd1u8FFgAHHVdZNcgT7z6yj5beAk6RPbjfR1Uz3MThax+RPV5zgOyB9qmOmH4kexzkLUef9pF9VOcMu4A2865zXf472T+b9Y7fwWagc3E6YEKD+HtCKRHpCbxBdrKaraqv5Fk/CniV/x76v6Wq77usm0D2h3mqqs7za7DGBDkRiQUeVNWvAh2LMX4dRHdcpfEW0IPs87Tfi8hyVY3JU3Whqv4xz7a1yB7g7ET24fWPjm1/82fMxhhjPOPvU1idgf2qeshx+d9CoK+beu7Ov94BrFfV3xyXK64HevovVGNKBZuD2gQNfyeQxuS+jPIouS9zzNFfRKJFZLGI5KzPu218PtsaU2aoags7fWWCRTAMoq8AmqlqJPAFjsdXGGOMCW7+vpEwHmjqstzEUeakqgkui7OAnEH2eKB7nm035N2BiNghvTHGeEFVPbmfKl/+PgL5HmjpeFhbRbIvlcx1iWaeyyP7AjlzH3wG3C4iNRwD6rc7yi6iqsV6vfjii8Wu526dJ2Wuy+7eexpbMPYvv74WVCeY+1fU311J9q+ofQuW/vnrd+eL/pWmz6Y3/fMFvx6BqGqmiDxO9gB4zmW8e0RkEvC9qq4i+0avu8l+NMIZHA+cU9UEEZlC9vX7CkzS7MF0n+vevXux67lb50mZ63J+74srUP3Lr6++7FtR2itu/wLxu/O0vaL2zV15KH023ZWHUv+C5bul2Bk20K/sLoSuF198MdAh+JX1r3QL5f6Fct9UVR3fncX6/g2GQXRTAF//NRtsrH+lWyj3L5T75it+vxPd30RES3sfjDGmpIkIGuSD6MYYY0KUJRBjjClDoqKiyMrKKryiByyBGGNMGbFlyxbuu+8+UlJSfNKeJRBjjCkDVJVnnnmGiRMnUqVKFZ+0aQnEGGPKgDVr1nDq1Cnuv/9+n7VpCcQYY0JcZmYm48aN469//Svly/vu/nFLIMYYE+I++eQTatSowd13+3YmYrsPxBhjQlxGRganT5+mQYP/PnrQF/eBWAIxxpgyyG4kNMYYEzCWQIwxxnjFEogxxhivWAIxxpgQ9PTTT7Nlyxa/7sMG0Y0xJsTs3LmTHj16sG/fPmrUqOG2jg2iG2OMucj48eMZP358vsnDV/w6pa0xxpiStWnTJnbu3MmSJUv8vi87AjHGmBCR88DEKVOmUKlSJb/vzxKIMcaEiDNnztC2bVuGDRtWIvuzQXRjjCmDbBDdGGNMwFgCMcYY4xVLIMYYY7xiCcQYY0qxM2fOcOrUqYDs2wbRjTGmFHviiScoV64cr732WpG2s/lAsARijCm7Dh48SOfOndm9ezf169cv0rZ2FZYxxpRhzz33HE888USRk4ev2BGIMcaUQj/++CN9+vRh3759VKtWrcjb2xGIMcaUUePHj+eFF17wKnn4ih2BGGNMKbRjxw7atm1LhQoVvNreBtGxBGKMMd6wU1jGGGMCxhKIMcYYr1gCMcYY4xVLIMYYUwqcPXuWP/zhD2RlZQU6FCe/JxAR6SkiMSKyT0SeKaDeABHJEpFOjuUIETkvItscr3f8HasxxgSradOmkZSURFhY8Pzd79c50UUkDHgL6AEcA74XkeWqGpOnXjXgj8CWPE0cUNVO/ozRGGOC3fHjx3n33Xf56aefAh1KLv5OZZ2B/ap6SFXTgYVAXzf1pgAvA6l5yot1iZkxxoSCiRMnMnr0aJo2bRroUHLx6xEI0Bg44rJ8lOyk4iQiHYEmqrpWRJ7Os30zEfkROAs8r6pf+zVaY4wJMjExMSxdupS9e/f6rM2jR4/6pB1/J5ACiYgArwGjXIsd/x4HmqpqgmNc5FMRaaeq50o6TmOMCZTNmzfz7LPPUrt2bZ+0l5yczMcff+yTtvydQOIB12OuJo6yHJcCVwIbHcmkAbBcRO5W1W1AGoCqbhORX4BWwLa8O5k4caLzfffu3enevbtve2GMMQEyevRon7SzceNGNm7c6JO2cvj1USYiUg7YS/Yg+nFgKzBUVffkU38D8KSq/iQidYEzqpolIi2AKOAqVU3Ms409ysQYYzzw+eefc+LECe677z7KlSsX3I8yUdVM4HFgPbALWKiqe0Rkkojc5W4T/nsKqxuwQ0S2AYuBh/MmD2OMMZ7ZvXs3u3btYsCAAT67FNgepmiMMSHu9OnTfPDBB9x33300atQIsIcpGmNMSEpISPBZW6mpqSxatIgePXo4k4evWAIxxpggcuHCBa6++mpiYmIKr1wIVWXFihWEh4fTqZPv78kO6GW8vvDkk0/y2muvISLOF+D1cnG29bbtgrYJCwujUqVKhb4qVqxIuXLl/PVjNsaUkDfffJPOnTvTpk2bYre1ZcsWEhISfHYlV16lfgykRo0aevjwYSA72+a8irPsy7aK23ZWVhapqanOV1paWq5l11e5cuU8TjYFrS9fvrwzkRljSs6ZM2do3bo1X3/9Na1bty5WW3FxcSxZsoQxY8ZQs2bNi9bbjITYIHoOVSUjIyPf5FLQK29SysrK8jr55K1ricgYzz311FMkJSXxz3/+s1jtnD17lpkzZ9KvXz8uv/xyt3UsgWAJxB8yMzO9Sjx5X+np6VSoUKHABFO/fn2uvPJKKleuHOhuGxNQhw8fpmPHjuzcuZOGDRt63U5mZiZz5szhiiuuoFu3bvnWswSCJZBgpqqFJpkjR47wyy+/0KpVKzp06EDz5s2D6nHVxpSUs2fPsnXrVm677bZitbNmzRp+++03hgwZUuAZAEsgWAIJBefPn2fnzp1ER0eTnJxMhw4d6NChA3Xq1Al0aMaUKjt27CAqKoqHHnqo0KN6SyBYAgk1J0+eJDo6mp9//pk6derQoUMHrrzySipVqhTo0IwJaidPnmTevHmMHDmSyy67rND6lkCwBBKqMjMzOXDgANHR0cTGxtK6dWsiIyNp1qyZDcwbk0dKSgozZsyge/fuXH311R5tYwkESyBlQXJyMj///DPR0dGkpKTQoUMHIiMjqVWrVqBDMybgVJWFCxdSs2ZNevXq5fF2lkCwBFLWHD9+nOjoaHbu3Em9evWIjIykXbt2VKxYMdChGVNkn3/+OZUrV+amm27yuo1NmzZx4MABRo0aVaSbiS2BYAmkrMrMzGTfvn1ER0dz+PBh2rRpQ2RkJE2bNrVTXKZUSE9Pp127drz77rteX3l14MABVqxYwUMPPcSll15apG0tgWAJxMC5c+fYsWMH0dHRZGRkOK/icnf3rTHB4t1332XZsmWsX7/eq+0TExOZNWsWAwcOJCIiosjbWwLBEoj5L1XNdYqrQYMGREZG0rZtWypUqBDo8IxxOnfuHFdccQWrV6/26iGHGRkZvP/++1x11VXccMMNXsVgCQRLIMa9jIwM9u7dS3R0NEePHqVt27ZERkYSHh5up7hMwE2ePJm9e/fyr3/9q8jb5jxhNz09nQEDBnj9ebYEgiUQU7ikpCS2b9/O9u3bycrKIjIykquvvpoaNWoEOjRTBmVmZnLttdeyZMkSWrRoUeTtf/zxR7777jvGjBlTrItHLIFgCcR4TlWJj48nOjqaXbt20bhxYzp06ECbNm3sFJcpUVlZWV49sic+Pp6PPvqIBx54gLp16xYrBksgWAIx3klPTycmJobt27dz7Ngx2rVrR2RkJI0bN7ZTXCYonT9/nhkzZnDHHXfQtm3bYrdnCQRLIKb4fvvtN+dVXGFhYc6ruIp6WaQx/pKVlcW//vUvGjZsWOyHLeawBIIlEOM7qsqRI0eIjo5mz549NGnShMjISFq3bk358qV+8k5Tin355ZfEx8czfPhwnz2t2hIIlkCMf6SlpRETE0N0dDQnTpzgyiuvpGPHjjRs2NBOcZkiy5nwzZuxtpiYGNauXcvYsWOpWrWqz2KyBIIlEON/iYmJzqu4ypcv77yKq1q1aoEOzZQSn332GW+88QZr164t0nb/+c9/eP/99xk2bBiNGzf2aUyWQLAEYkqOqnL48GHnKa6IiAgiIyNp1apVkZ5BZMqWrKwsOnXqxAsvvED//v093i4tLY1Zs2bRuXNnrrnmGp/HZQkESyAmMNLS0ti9ezfR0dGcOnWK9u3bExkZWaypSE1o+vDDD3n77bfZvHmzx6c/VZWlS5dSvnx57r77br+cNrUEgiUQE3gJCQlER0ezfft2KleuTGRkJFdddZVPz1eb0iklJYU2bdowf/78Ij1xd8uWLWzfvp3Ro0f77R4lSyBYAjHBQ1WJi4sjOjqavXv30rx5c7p37+7R7HAmNL3++uts2LCBFStWeLzNoUOH+Pjjj3nwwQf9OueNJRAsgZjglJqayvbt24mKiuKOO+7weJY4E1rWrl1Ls2bNPL7xLykpiZkzZ9KnTx+uuOIKv8ZmCQRLICa4nTx5ksWLF3P55Zdzxx132GC7yVdmZibz5s2jRYsW3HzzzX7fnyUQLIGY4JeSksKyZcu4cOECAwcOtDvcjVvr1q3jzJkzDB06tETuNfJFAvHNLY3GmHxVrlyZIUOG0LJlS2bOnMmhQ4cCHZIJMj///DP79u3jnnvuKVU3qtoRiDEl6MCBA3z66ad07dqV6667rlR9WRj/+PXXX5k7dy4jRoygQYMGJbZfO4VF/gmkWbNm9peeKVBERARxcXElvt+EhAQWL15M3bp16dOnT7HmdDDBZ+HChfTt25dLLrmk0LopKSnMnDmTbt260aFDhxKI7r8sgZB/AnH8cAIQkSktAvkZSU9PZ/Xq1Rw/fpxBgwZRp06dgMRhfOubb75h2LBh7N27l8qVKxdYV1VZtGgRl156Kb179y6hCP/LxkCMKaUqVKhA3759ufbaa3n//ffZu3dvoEPy2OrVq0lMTMxVlpiYyOrVqwMUUXBQVZ5++mkmT55caPKA7GSTnJxMz549SyA6//B7AhGRniISIyL7ROSZAuoNEJEsEenkUjZeRPaLyB4R+b2/YzWmJIkI11xzDUOHDmXNmjV89dVXZGVlBTqsQnXp0oUJEyY4k0hiYiITJkygS5cuAY4ssFasWEFSUhLDhw8vtO7Bgwf57rvvGDhwYKm+tLvQU1giUhm4C7gJaARcAHYCq1V1VyHbhgH7gB7AMeB7YIiqxuSpVw1YDVQAHlfVbSLSFvgIuBZoAnwBXJH3fJWdwjLeCqbPyLlz5/jkk08oV64c/fv3p0qVKoEOqUA5SeOpp57i1VdfZerUqdSsWTPQYQVMRkYGV111FX/729+48847C6z722+/MXPmTAYMGEDz5s1LKMKL+f0UlohMAr4BbgC+A94DFgMZwMsi8rmIFHSLbWdgv6oeUtV0YCHQ1029KcDLQKpLWV9goapmqGocsN/RnnF49NFHmTp1qs/rmpJXrVo1RowYQf369Zk5cybHjx8PdEgFqlmzJk899RTNmzfnqaeeKtPJA2D9+vU0aNCAXr16FVgvIyODxYsXc+ONNwY0efiMqub7AnoXsr4+cE0B6wcAM1yWhwNv5qnTEfjY8X4D0Mnx/h/AMJd6s4D+bvah7uRXHiyaNWumX375ZaDDKNOC9TOyc+dOnTZtmv7000+BDiVfCQkJ+thjj2lsbKw+9thjmpCQEOiQAu7cuXOF1lmxYoUuWrRIs7KySiCigjk+/wXmgMJeBR6BqGq+o2IiUl5Vf1XVH7xLXSDZF8G/BvzF2zZCVWZmZqBD8At1c8rIXVlBQvVnk+PKK69k1KhRfP3116xatYqMjIxAh5RLzumrqVOn0qxZM6ZOnZprTKSsKuzpyz/99BOHDx+mb9++IXP/T2GnsL52eT8/z+qtHrQfDzR1WW7iKMtxKXAlsFFEYoHrgRWOgfTCtnWaOHGi87Vx40YPwgqskSNHcvjwYfr06UP16tWZPn06hw4dIiwsjPfff5+IiAh69OgBwKBBg2jYsCG1atWie/fu7N6929nOAw88wAsvvABAVFQU4eHhvPbaa1x22WU0btyYOXPmeFX3zJkz9OnThxo1anDdddfx/PPPF/go6i1bttClSxdq1apFx44diYqKcq675ZZbeO655+jatStVq1YlNjbWbdnx48fp27cvderUoVWrVsyaNcvZxqRJkxg4cCAjRoygZs2azJ07t1g//9Kgfv36jBkzhuTkZObMmcPZs2cDHZLTN998k2vMo2bNmkydOpVvvvkmwJEFr2PHjvHFF18waNAgKlWqFJAYNm7cmOu70icKOjwBfnJ5vy2/dQVsXw44AEQAFYFooG0B9TcAHR3v2wE/ObZr7mhH3GxT0OFZ0GrWrJl+9dVXzuW4uDgVER01apSeP39eU1JSVFX1gw8+0OTkZE1LS9M///nPGhkZ6dzm/vvv1+eff15VVTdu3Kjly5fXiRMnakZGhq5Zs0arVKmiiYmJRa47ePBgHTp0qKakpOju3bs1PDxcb7rpJrf9iI+P1zp16ui6detUVfWLL77QOnXq6OnTp1VVtXv37hoREaF79uzRzMxMTU9Pd1vWrVs3ffzxxzUtLU2jo6O1Xr16umHDBlVVnThxolasWFFXrFihqur82RRXsH9GVFWzsrL03//+t06fPl0PHjwY6HCMF5KTk/WNN97QnTt3BjqUXPD3KSygoHMLhZ53UNVM4HFgPbCL7EHxPSIySUTuyqdNcWy7m+wB+93AGuAxR6dDRt7uiAiTJk3ikksucf6Vcv/991OlShUqVKjACy+8wPbt20lKSnLbXsWKFXn++ecpV64cvXr1olq1avneX5Bf3aysLJYuXcrkyZOpVKkSbdu2ZdSoUfn24cMPP6R3797ccccdAPTo0YNrrrmGNWvWOOvcf//9tGnThrCwMMqXL39R2YkTJ9i8eTOvvPIKFSpUoEOHDowZM4Z58+Y527jhhhvo06cPQMD+ggsEEaFr167cc889fPLJJ2zevDlorhwr6zz5PeT8f2rbti1XXnllCURVsgpLIDVF5B4RGeB439/xGgDU8GQHqrpOVVur6hWq+rKj7EVVXeWm7q2qus1l+a+q2lJV26rq+iL0yyMTJ05ERC565Xd4566+zw4FHZo0aeJ8n5WVxbhx42jZsiU1a9akefPmiAinT592u22dOnUIC/vvr7RKlSqcO3euSHVPnTpFZmZmrjjCw8PzjffQoUMsXryY2rVrU7t2bWrVqsU333zDiRMnCtzetezYsWPUrl0716WrERERxMfHu61fFrVo0YIxY8awa9culixZQmpqauEbGb966KGHWLduXYF1oqKiyMzM5LbbbiuhqEpWYQkkCrib7PtAooA+jtddwCb/huZ/EydOdHtYVlAC8bRuYfIbRHMt/+ijj1i5ciVfffUViYmJxMXFuZ6684t69epRvnx5jh496iw7cuRIvvXDw8MZOXIkZ86c4cyZMyQkJJCUlMRTTz3lrOOur65ljRo14syZMyQnJzvLDh8+TOPGjQtso6ypWbMmDzzwAJUqVWLWrFn5/iFh/C86OppVq1YVePPkvn37iI6OZsCAAbn+WAslhV2F9UBBr5IKMhQ1aNCAgwcP5irLmxiSkpKoVKkStWrVIjk5mfHjx/v9izQsLIz+/fszceJELly4QExMTK5TSXkNHz6clStXsn79erKyskhJSSEqKopjx455vM8mTZpw4403Mn78eFJTU9mxYwezZ89mxIgRvuhSSClfvjx33303119/PR988AF79uwJdEhl0vjx43nuuefyndvlzJkzLF++nHvvvZdq1aqVcHQlp7CrsPqISITL8gsisl1EVohICNwFEzjjxo1jypQp1K5dm9deew24+K/skSNH0rRpUxo3bkz79u258cYbi7SPoiQb17r/+Mc/SExMpGHDhowaNYphw4blO+7QpEkTli9fzksvvUS9evWIiIhg+vTpzkdyFHb0kWPBggXExsbSqFEjBgwYwJQpU7jllls8jr+s+d3vfsewYcP47LPP+OKLL0rFI1BCxVdffcX+/fsZO3as2/Xp6eksWrSIm2++OeRPvRb4KBMR2QFcr6rnHYPerwFDyb75b6Cq3lEyYebPHmXif+PGjePkyZN88MEHgQ7Fp0LhM5KcnMwnn3wCwIABAwq9F8EUT1ZWFp07d+app55i8ODBF61XVT799FMA+vXrF9SnXkviabyqqucd7/sDs1X1R1WdBdQrzo5N8Nq7dy8///wzAFu3bmX27Nn0798/wFEZd6pWrcrw4cNp1KgRM2fOzHXhgfG95ORk7rjjDgYOHOh2/ffff8/Jkye56667gjp5+IonRyA3AueBWGCAOu48F5HdqtquRKIsgB2B+N4PP/zA0KFDOX78OJdddhkPP/wwTz/9dKDD8rlQ+4zs2bOHVatW0aNHDzp16lT4Bsanjhw5wsKFC3nwwQepXbt2oMMplN8nlBKR0cCzwFngV1Xt6SjvCExX1R7F2bkvWAIx3grFz8jp06dZtGgR4eHh3Hnnnc77box/nTt3jpkzZ9K7d29atWoV6HA8UiIzEopIY7IfmrhdVbMcZQ2BCqp6uDg79wVLIMZbofoZSU1NZcWKFSQmJjJo0CBq1PDoli3jpczMTObPn09ERESpuvCjJI5ACjwOdr3pL1AsgRhvhfJnRFX59ttv2bx5M/3796dFixaBDilkrV+/nlOnTjF06NBSdb9HSSSQLLInj8q5Y8l1Z6qqtxZn575gCcR4qyx8RmJjY1m6dCnXXXcdXbp0KRMDu77266+/curUKbePItm1axdffPEFY8eO5ZJLLglAdN4riQTyJ+Be4DeyJ4Napqrun40RIJZAjLfKymfk7NmzLF68mEsvvZR+/fqVqWeJ+cL//M//UKlSJef9WjlOnTrFnDlzGD58OA0bNgxQdN4rkTEQx45aAEPIniXwEPCSqkYXZ8e+YgnEeKssfUYyMjJYt24dcXFxDB48mHr17Cp8T+zfv58bb7yRmJgY6tSp4yxPTU1l5syZdOnShY4dOwYwQu+VxH0gAKjqQWA52U/V7QyUjssMQlDOXB452rdvz6ZN7h9LlrduUdk0uKGjfPny3HXXXXTp0oU5c+awa9euQIdUKkyYMIEnn3wyV/JQVZYvX06zZs1KbfLwlQKv8ctz5HGE7NNYL6nqhRKIzeTD9Tz2zp07Pa5bkLlz5zJr1iz+/e9/O8veffdd7wI0Qatjx440aNCAxYsXEx8fz2233VaqBn5L0tatW/nmm29yTbYGsHnzZn777Te7uZbCj0AOAIOAdcC3ZM8Q+KiIPCkiT/o7OFNyVDUoB1jdTV9b1CltQ30K3KJq2LAhDz30EL/++ivz58/P9RRk81+TJk1i4sSJuaYZiI1Pkh/eAAAdI0lEQVSN5dtvv2XQoEF2jw2FJ5DJwDIgC6hG9hS0ri/jhWnTpl30KIQnnniCP/3pTwDMmTOHdu3aUb16dVq2bMmMGTPybat58+Z89dVXAKSkpHD//fdTu3Zt2rdvz/fff5+r7iuvvELLli2pXr067du3dz6zJyYmhkcffZRvv/2WSy+91HkXres0uAAzZ87kiiuuoG7duvTr14/jx48714WFhfHee+/RqlUrateuzeOPP55vzKrKyy+/TMuWLalXrx5Dhgxxzqftbmrf/Kb7XbFiBe3bt6d27drceuutxMTE5Pq5TJs2jQ4dOlCtWjV72GAeVapUYdiwYYSHhzNjxoxcj+832WbPns0DD/z3oeNnz55l6dKl9O/f3+6tyVGc6QyD4UUpnNL20KFDWrVqVT137pyqqmZmZmrDhg1169atqqq6Zs0ajY2NVVXVTZs2aZUqVfSnn35S1ezpaMPDw51tNWvWTL/88ktVVX3mmWe0W7dumpiYqEePHtX27dvnqrtkyRI9ceKEqqouXrxYq1at6lyeM2fORdPWuk6D++WXX2rdunU1Ojpa09LS9A9/+IN269bNWVdEtE+fPnr27Fk9fPiw1qtXTz/77DO3/X/jjTf0hhtu0GPHjmlaWpo+8sgjOnToUFV1P7Wvu7J9+/Zp1apV9csvv9SMjAydNm2atmzZUtPT050/l44dO2p8fHy+U+AG82ekJMXExOi0adP0+++/16ysrECHE5TS09N15syZ+u9//zvQofgMPpjStrAv5+eAWgWsvxW4q7hBFKsDpTCBqKredNNNOn/+fFVVXb9+vbZs2TLfuv369dM333xTVQtOIC1atND169c7182YMSNX3bwiIyOd84wXlkAefPBBfeaZZ5zrzp07pxUqVNBDhw6panYC2bx5s3P9oEGD9JVXXnG737Zt2+aaD/7YsWNaoUIFzczM1Li4OA0LC9O4uDjnendlU6ZM0cGDBzuXs7KytHHjxhoVFeX8ucyZMyffvqsG/2ekJJ0+fVrfeecdXbZsmaalpQU6nKCzatUqXbhwYUglWF8kkMJO4v0MrBKRFGAbcAqoDFwBRAJfAC/59JCoBE2aNMkn7bz44otF3mbo0KEsWLCA4cOHs2DBAoYNG+Zct3btWiZPnsy+ffvIysriwoULXH311YW2eezYsVxT0UZERORaP2/ePF5//XXi4uKA7CeLejqr3bFjx/jd737nXK5atSp16tQhPj6epk2bAnDZZZc51xc0ne6hQ4e45557nIO3qkqFChU4efKks45rP9yVHTt2LFf/RITw8PBcT6N114Zxr06dOjz44IOsXLmS999/n0GDBlGrVq1AhxUUoqOjiY2NZcyYMUE5ThhIBSYQVV0OLBeRK4AuQEOyH6z4ITBWS/nVWN588fvKwIED+d///V/i4+NZtmwZW7ZsASAtLY17772XDz/8kL59+xIWFsY999yTc7RVoIYNG3LkyBHatm0LZH9R5zh8+DBjx45lw4YN3HDDDUD2FTk57Rb2H6NRo0a52ktOTuY///mPV1/STZs25f3333fG4SpnH55Mg5v3CrQjR47kisf+sxdNxYoV6d+/P9999x2zZ8+mX79+tGzZMtBhlZi4uDiaNm2a66q0EydO8PnnnzNq1CgqV64cwOiCk6f3gexX1Tmq+ldVfUNVPyvtySPQ6taty80338wDDzxAixYtaN26NZCdQNLS0qhbty5hYWGsXbuW9evXe9TmoEGD+Otf/0piYiJHjx7lrbfecq5LTk4mLCyMunXrkpWVxQcffJDrC/iyyy7j6NGjpKenu2176NChfPDBB+zYsYPU1FSeffZZrr/+eq/uM3n44Yd59tlnOXw4+1mcp06dYsWKFc717pJl3rJBgwaxevVqNmzYQEZGBtOnT6dy5cpuk5LxnIhw/fXXM3DgQJYvX86mTZs8+uOltDt9+jRdu3blhx9+cJZduHCBxYsX06tXL+rXrx/A6IKXXQAeQMOGDePLL7/kvvvuc5ZVq1aNN998k4EDB1K7dm0WLlxI3759823D9a/sF198kaZNm9K8eXN69uzJyJEjnevatm3LX/7yF66//noaNGjArl276Nq1q3P9rbfeypVXXkmDBg3c/mfp0aMHU6ZMoX///jRu3JjY2FgWLlzoNg53y66eeOIJ+vbty+9//3tq1KjBjTfeyNatWwvcNm9Zq1at+PDDD3n88cepV68eq1evZuXKlc5LK+3oo3giIiIYO3Ys+/fvZ+HChaSkpAQ6JL9RVcaMGcOQIUNo0aIFhw4dYs+ePSxZsoRWrVrRvn37QIcYtDx6lEkws0eZGG/ZZ6RwmZmZfPbZZ/zyyy8MHjy41PwlrqqkpKSQnJzM+fPnSU5OzvXetez06dNkZGRwySWXULVqVapWrUqVKlWoU6cOt956K+XKlQt0d/yipOYDKQf8UVVfL86O/MUSiPGWfUY8t337dtavX0+vXr0C8hd5TkJwlwzcJYbz589ToUIFZzJw/df1/cmTJxk6dChr1qxx+7TdUFaSD1Pcqqqdi7Mjf7EEYrxln5GiOXHiBIsXL6ZVq1bcfvvtxfrLXFVJTU11mwxylvMmiPwSgmtiyFmuUqWKR3eKjx07lmuuuYaxY8d63ZfSqiQTyOtABWAR4HzugdqEUqYUs89I0V24cIFly5aRmprKwIEDqVatGlBwQsibDFwTgrujgvzK/PHokIyMDMqVK1cmx8xKMoFscFOsahNKmVLMPiPeUVWioqL44YcfqFatmjNBlC9f/qIvf3dHB/5MCMZzJZZAgpklEOMt+4wUz6+//kpmZqYlhFKqJI9AagAvAt0cRVHAZFX9rTg79wVLIMZb9hkxZVmJTSgFvA8kkf1o90Fk343+QXF2bIwxJW3NmjWcOnUq0GGEDE+PQKJVNbKwskCwIxDjLfuMlC27du2ie/fufPPNN7RqZZOqluQRyAURcd62LCJdAHuUiTGmVEhJSWHo0KG8/PLLljx8yNME8gjwtojEiUgc8BbwsN+iKgNcJ4Iqjrlz53LTTTf5ICJjQte4ceNo3bo1o0ePDnQoIaXQyyZEJAxoraodRKQ6gKqe9XtkxiOqvp2KNjMz86IbxNyVFbUNYwJl7dq1LF26lO3bt5fJ+z38qdAjEFXNAp52vD9b2pPH6tWrndOn5khMTGT16tUl1sbIkSM5fPgwffr0oXr16kyfPh2ALVu20KVLF2rVqkXHjh2JiopybjNnzhwuv/xyqlevzuWXX86CBQvynYo2r7NnzzJmzBgaNWpEeHg4zz//vPPc/9y5c+natStPPvkkdevWZdKkSW7LVJX/+7//o1mzZjRo0ID777+fs2ezPwr5TTlrTDDYvn078+bNs/lN/MGTWaeAl4H/BcKB2jmv4s5m5YsXRZyRMCEhQR977DFNSEhwu+wJX7TRrFmzXLPyxcfHa506dXTdunWqqvrFF19onTp19PTp05qcnKzVq1fX/fv3q6rqiRMndPfu3arqfibBvPr166ePPvqoXrhwQU+dOqXXXXedzpgxw7l9+fLl9e2339bMzExNSUlxWzZ79my94oorNC4uTpOTk7V///46YsQIVXU/DW1pkN9nxJiyAH9Paav//ZKOdfM66OG2PYEYYB/wjJv1DwM7gJ+ATUAbR3kEcJ7smRC3Ae/k035BPxy3cr7wY2Nji/zF76s2XKeiVVV95ZVXdOTIkbnq3HHHHTpv3jxNTk7WWrVq6dKlS/XChQu56hSWQE6ePKmVKlXK9aW+YMECveWWW5zbR0REXNRm3rIePXrou+++61zeu3dvgdPQlgaWQExZ5osE4ukYyHBV/aaoRzeObd8CegDHgO9FZLmqxrhU+5eqvueo3wd4HejlWHdAVTsVdb+FqVmzJk899RTNmzcnNjaWmjVrBqQNV4cOHWLx4sWsXLkSyE7sGRkZ3HrrrVSpUoVFixbx6quvMnr0aLp27cr06dOdk1AV1m56ejoNGzZ0tquqzmloAbeTQuUtyzuFbEREBBkZGYVOQ2uMCV2ejoG8VVi9fHQG9qvqIVVNBxYCuWZHUlXXibOrAVkuy34Z8UpMTOTVV18lNjaWV1999aLxjJJoI+9gXnh4OCNHjuTMmTOcOXOGhIQEkpKSePrppwG4/fbbWb9+PSdOnKB169bOp4cWNigYHh5O5cqV+c9//uNsNzExkR07duQbi7uyvFPaHjp0iAoVKuSaB90GKI0pWzy9jPdLERkgRf+GaAwccVk+6ijLRUQeE5EDZI+1/NFlVTMR+VFENrjeh1IciYmJTJgwgalTp9KsWTOmTp3KhAkTipQAfNFGgwYNOHjwoHN5+PDhrFy5kvXr15OVlUVKSgpRUVEcO3aMX3/9lRUrVjifYFqtWjXnvM2FTUXboEEDfv/73/PnP/+ZpKQkVJWDBw+yadMmj2OF7CltX3/9deLi4jh37hwTJkxgyJAhzjjUbsgzQWLy5Mm5/kAyfuTJeS6yH2OSBaST/RiTJOCsB9sNAGa4LA8H3iyg/hBgjuN9RaCW430n4DBQzc02BZ3fu8iqVasuGq9ISEjQVatWua3vrzaWL1+uTZs21Vq1aunf/vY3VVXdunWr3nzzzVq7dm2tX7++3nXXXXrkyBE9fvy43nzzzVqzZk2tVauW3nLLLbpnzx5VVU1LS9O77rpLa9eurfXq1XO7r7Nnz+qjjz6qTZo00Zo1a2qnTp100aJFqup+DMVdWVZWlk6ZMkXDw8O1fv36OnLkSE1MTFRVdY6BZGZmetz/YJDfZ8SUXitWrNCIiAivxjXLGnwwBuLXp/GKyPXARFXt6Vge5wj6lXzqC5CgqhcNKDgeKf8XzTMHiYjoiy++6Fzu3r073bt3t8dUmELZZyS0HD9+nI4dO7JkyRK6dvXJCYuQsnHjRjZu3OhcdlyeXyJP4xXgPqC5qk4RkXCgoapuLWS7csBesgfRjwNbgaGquselTktVPeB43wd4XlU7i0hd4IyqZolIC7KfAHyVqibm2Ye664N9OZjC2GckdGRlZdGrVy+uu+46Jk+eHOhwSgVfPAvL0wf4v0P2KaxbgSnAOeBt4NqCNlLVTBF5HFhP9njLbFXdIyKTgO9VdRXwuIjcBqQBCcAox+bdgMkikubY98N5k4cxxgD8/e9/JykpiRdeeCHQoZQpnh6BbFPVTiLyk6p2dJRtV9UOfo+w8NjsCMR4xT4joWPdunW0atWKFi1aBDqUUqMkj0DSHaej1LHjeuS+3NYYYwKmZ8+egQ6hTPL0Mt43gWVAfRGZCnwNvOS3qIwxxgQ9j6/CEpE2ZA+GC/Cl60B4INkpLOMt+4yYsqzE5kQPZpZAjLfsM1J6ZWVlOW9iNd4pyTGQUiciIsIerWEK5PpsL1N6ZGVl0bt3b8aPH0+3bt0CHU6ZFrIJJC4uLtAhGGP84LXXXiMpKYkbb7wx0KGUeSF7CssYE3q2bdtGz5492bp1K82aNQt0OKWaL05h2UlEY0ypkJyczNChQ3nzzTcteQQJOwIxxpQKf/rTn0hISGDu3LmBDiUk2FVYWAIxpqyIj4+nevXqXHrppYEOJSRYAsESiDHGeMPGQIwxxgSMJRBjjDFesQRijAlK8fHxZGXZM1uDmSUQY0zQOXfuHN27d2fDhg2BDsUUwAbRjTFBZ/To0YgIs2fPDnQoIcuehWWMCTmLFy/m66+/Ztu2bYEOxRTCjkCMMUHj0KFDXHvttaxZs4Zrrrkm0OGENLsPBEsgxoSSP//5zzRo0IBnnnkm0KGEPEsgWAIxJpRkZGQQFhZmc32UABsDMcaElPLl7SupNLE0b4wxxiuWQIwxxnjFEogxJmA2bNjAyZMnAx2G8ZIlEGNMQMTFxTF48GDi4+MDHYrxkiUQY0yJy8jI4L777uPpp5+mU6dOgQ7HeMkSiDGmxE2dOpUqVarw5JNPBjoUUwx2zZwxpkRt3ryZd999l23bttn9HqWc/faMMSVq69atvPfeezRq1CjQoZhisjvRjTGmDLIpbY0xxgSMJRBjjDFesQRijDHGK5ZAjDF+9frrr/PDDz8EOgzjB5ZAjDF+s2nTJqZNm0Z4eHigQzF+YAnEGOMXCQkJjBgxglmzZnHZZZcFOhzjB35PICLSU0RiRGSfiFw0zZiIPCwiO0TkJxHZJCJtXNaNF5H9IrJHRH7v71iNMb6hqjzyyCPcfffd9O7dO9DhGD/x630gIhIG7AN6AMeA74EhqhrjUqeaqp5zvO8DPKaqvUSkHfAv4FqgCfAFcEXemz7sPhBjgs+cOXP429/+xtatW7nkkksCHY5xozTcB9IZ2K+qh1Q1HVgI9HWtkJM8HKoBWY73dwMLVTVDVeOA/Y72jDFBrnHjxixYsMCSR4jz97OwGgNHXJaP4iYJiMhjwJNABeBWl22/dakW7ygzxgS522+/PdAhmBIQFIPoqvqOqrYEngGeD3Q8xhhjCufvI5B4oKnLchNHWX4WAf902db12r98t504caLzfffu3enevXvRIzXGmBC2ceNGNm7c6NM2/T2IXg7YS/Yg+nFgKzBUVfe41Gmpqgcc7/sAz6tqZ5dB9OvIPnX1OTaIbkxQUlVEijUea0pY0A+iq2om8DiwHthF9qD4HhGZJCJ3Oao9LiI7RWQb8CdglGPb3cBiYDewhuyrsyxTGBNkVJXBgwezdu3aQIdiSpg9zt0YUyyzZs3iH//4B9999x2VK1cOdDjGQ744ArEEYozx2t69e+natStRUVG0a9cu0OGYIgj6U1jGmNCVlpbGsGHDmDx5siWPMsoSiDHGK2+99RZNmjThkUceCXQoJkDsFJYxxisxMTHUrVuXunXrBjoU4wUbA8ESiDH+ZpfohiYbAzHG+E1SUhJvv/027du3JzY2NtDhmCBkCcQYk8vevXv54x//SEREBBs3buSdd96hWbNmgQ7LBCFLIMYYpxkzZtCtWzeqV6/O9u3b+fjjj7n55pvtFJZxy8ZAjDFOv/32G5UqVbIbAssAGwMxxnhlz549uPvDq0aNGpY8jMcsgRhTRqSnp7N48WJuuukmbr/9dk6ePBnokEwpZwnEmBB34sQJpkyZQrNmzXjnnXd44okniIuLo0GDBoEOzZRy/p4PxBgTYJ988glHjx5l7dq1XH311YEOx4QQG0Q3xpgyyAbRjTEAHD58mJdeeonMzMxAh2LKEEsgxpRSqsqGDRvo378/HTt25Ndff+X8+fOBDsuUITYGYkwp9OmnnzJhwgQAHn/8cebNm0e1atUCHJUpa2wMxJhSaMOGDagqt9xyi90lbrxiT+PFEogxxnjDBtGNCVEJCQm89tprXHvttVy4cCHQ4RjjliUQY4LIzz//zMMPP0yLFi348ccfefPNN+3RIiZo2SC6MUHi2WefZe7cuTz88MPs2bPH7hQ3Qc/GQIwJEidOnKBOnTpUqFAh0KGYMsDGQIwpheLi4tyWN2jQwJKHKVUsgRhTAlJTU5k/fz7XXXcdPXv2JCMjI9AhGVNslkCM8aOjR4/y3HPP0bRpU+bPn8+ECRPYtWsX5cvb8KMp/exTbMq048ePc/LkSVJTU0lLSyM1NZXU1FTatm1LixYtLqq/fv16Nm/e7KyX8xo8eDC33377RfWnT59OZmYmUVFRtGnTpiS6ZEyJCYkEsm7duovKWrduTfPmzS8qj4mJcXsOujTVj42Nvai8TZs2buvv2bMn3/ruviB3797NwYMHLypv27Ytl19++UXlu3btclu/Xbt2+db/5ZdfnMsZGRmkpqbSoUMH2rVrd1H9lStXsmHDhou+sEeMGEGfPn0uqv/SSy8xc+bMixLCtGnT+NOf/nRR/blz57JgwQIqVapExYoVqVSpEpUqVeKRRx5x+/NJT08HoHr16rm2adSo0UV1Ad544w235caEgpBIIO7+kz700ENuv1A3bdrE0qVLS3X9ZcuWXVQ+ZswYt/W//vrrfOu7+4LcvHkzn3766UXlDz74oNuEsGXLFrf1R48enW/95cuXO5fLlStHpUqVqF69utsEUrVqVRo3buz8Ys95tW7d+qK6Of0aMmTIRQkhv8HpcePGMW7cOLfr3Onduze9e/f2uL4xocwu4zXGmDLILuM1xhgTMJZAjDHGeMUSiDHGGK9YAjHGGOMVSyDGGGO84vcEIiI9RSRGRPaJyDNu1v9ZRHaJSLSIfC4i4S7rMkVkm4j8JCIXXytqjDEmYPyaQEQkDHgLuAO4EhgqInlvx90G/E5VI4FPgFdd1iWraidV7aiq/fwZa7DauHFjoEPwK+tf6RbK/QvlvvmKv49AOgP7VfWQqqYDC4G+rhVUNUpVUxyLW4DGLqvL/GTPof4htv6VbqHcv1Dum6/4O4E0Bo64LB8ld4LI60FgrctyJRHZKiKbRaRvfhsVl6cflILquVvnSZnrcn7viytQ/cuvr77+j1lS/QvE787T9oraN3flofTZdFceSv0Llu+WoBlEF5HhwO/IfQorQlU7A/cBb4jIxc/q8IFQ/yVbAim8niWQ0PpsuisPpf4Fy3eLXx9lIiLXAxNVtadjeRygqvpKnnq3AX8Huqnqf/Jp6wNgpaouzVNuzzExxhgvFPdRJv5OIOWAvUAP4DiwFRiqqntc6nQEPgbuUNVfXMprAudVNU1E6gLfAH1VNcZvARtjjPGYX5/Gq6qZIvI4sJ7s02WzVXWPiEwCvlfVVcA0oCrwsYgIcMhxxVVb4D0RyXRs+1dLHsYYEzxK/dN4jTHGBEbQDKIbY4wpXSyBGGOM8UrIJhAR6SsiM0RkgYhcPFl1KSYizUVklogsDnQsviYiVURkjoi8JyLDAh2Pr4Xy7w5C+/8dgIi0EZF3RWSxiDwS6Hj8wfF/8HsRubPQuqE+BuK4mutVVX0o0LH4mogsVtVBgY7Dlxz3AyWo6moRWaiqQwIdkz+E4u/OVSj/vwNwXPAzV1VHBjoWX3Nc5JQE7FbVNQXVDfojEBGZLSInRWRHnvICH9Lo4jngbf9G6R0f9C3oedHHJvz36QWZJRaol0L9d1iM/gXt/ztX3vRPRPoAq4ACv1yDQVH757gnbzdwCk8eJaWqQf0CugKRwA6XsjDgABABVACigTaOdSOA14BGwMvArYHugx/61tCx/HGg++CHPt4H3Ol4/1Gg4/d1/1zqBP3vztv+Bfv/O1/8/hz1VgU6fl/3D/g/x3fMZ8CywtoP+iMQVf0aSMhTnO9DGlV1vqo+CQwg+wbGe0VkbEnG7Kli9C1VRN4FIoP9r9ui9hFYRvbv7G1gZclF6p2i9k9EapeW3x141b8/EOT/71x50b+bReTvIvJPYHXJRlt0XnzHPOf4jvkXMLOw9v16I6EfuXtIY2fXCqr6D+AfJRmUj3jStzPAoyUZlI/l20dVPQ+MDkRQPlRQ/0r77w4K7l9p/X/nqqD+RQFRgQjKhzz5jpnnSUNBfwRijDEmOJXWBBIPNHVZbuIoCwWh3Lccod5H61/pZv3zUGlJIELuKwK+B1qKSISIVASGACsCElnxhXLfcoR6H61/1r9g5r/+BfoqAQ+uIvgIOAakAoeBBxzlvch+0u9+YFyg47S+lc0+Wv+sf8H88nf/Qv5GQmOMMf5RWk5hGWOMCTKWQIwxxnjFEogxxhivWAIxxhjjFUsgxhhjvGIJxBhjjFcsgRhjjPGKJRBjjDFesQRijB+ISE0ReSLQcRjjT5ZAjPEDVU0EaovIXYGOxRh/sQRijP8sAe4NdBDG+IslEGP8RFV/BtqKSLlAx2KMP1gCMca/jgO3BjoIY/zBEogxfuIYRJ+PncYyIcoSiDF+ICJ/BPao6ifA70RECtvGmNLGEogxPiYivYBLVXW9oygK6B64iIzxD5tQyhgfE5FKqprqslwRqKyqZwMYljE+ZwnEGGOMV+wUljHGGK9YAjHGGOMVSyDGGGO8YgnEGGOMVyyBGGOM8YolEGOMMV6xBGKMMcYr/w+71OoMjLwEFQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f148ba1ec50>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "pipeline(1000, True, True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 8.3"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Use item2vec to make recommendations (following code from exercises in Chapter 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "def recScore(i, userHistory):\n",
    "    historyInVocab = [w for w in userHistory if w in model10.wv]\n",
    "    if len(historyInVocab) == 0:\n",
    "        return 0\n",
    "    sc = model10.wv.distance(str(i), historyInVocab[-1])\n",
    "    return sc\n",
    "\n",
    "def rec(userHistory):\n",
    "    historyInVocab = [w for w in userHistory if w in model10.wv]\n",
    "    if len(historyInVocab) == 0:\n",
    "        return 0\n",
    "    return model10.wv.most_similar(positive = historyInVocab, topn=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.20018476247787476"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recScore(20539, reviewLists[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('41688', 0.9940300583839417),\n",
       " ('44596', 0.9929537773132324),\n",
       " ('24082', 0.9924867749214172),\n",
       " ('26393', 0.9917975068092346),\n",
       " ('6824', 0.9912790060043335),\n",
       " ('8950', 0.9910286068916321),\n",
       " ('24473', 0.9906283020973206),\n",
       " ('8679', 0.9903963208198547),\n",
       " ('5972', 0.990136981010437),\n",
       " ('25918', 0.989347517490387)]"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rec(reviewLists[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 8.4"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "(see tf-idf retrieval examples above)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 8.5"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Predict the rating using item2vec item similarity scores. Adapts models from Chapter 4."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "ratingMean = sum([d['review/overall'] for d in reviewDicts]) / len(reviewDicts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "itemAverages = defaultdict(list)\n",
    "reviewsPerUser = defaultdict(list)\n",
    "    \n",
    "for d in reviewDicts:\n",
    "    i = d['beer/beerId']\n",
    "    u = d['user/profileName']\n",
    "    itemAverages[i].append(d['review/overall'])\n",
    "    reviewsPerUser[u].append(d)\n",
    "    \n",
    "for i in itemAverages:\n",
    "    itemAverages[i] = sum(itemAverages[i]) / len(itemAverages[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predictRating(user,item):\n",
    "    ratings = []\n",
    "    similarities = []\n",
    "    if not str(item) in model10.wv:\n",
    "        return ratingMean\n",
    "    for d in reviewsPerUser[user]:\n",
    "        i2 = d['beer/beerId']\n",
    "        if i2 == item: continue\n",
    "        ratings.append(d['review/overall'] - itemAverages[i2])\n",
    "        if str(i2) in model10.wv:\n",
    "            similarities.append(model10.wv.distance(str(item), str(i2)))\n",
    "    if (sum(similarities) > 0):\n",
    "        weightedRatings = [(x*y) for x,y in zip(ratings,similarities)]\n",
    "        return itemAverages[item] + sum(weightedRatings) / sum(similarities)\n",
    "    else:\n",
    "        return ratingMean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "u,i = reviewDicts[0]['user/profileName'],reviewDicts[0]['beer/beerId']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3.88871"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictRating(u,i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3.88871"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ratingMean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "alwaysPredictMean = [ratingMean for _ in reviewDicts]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels = [d['review/overall'] for d in reviewDicts]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions = [predictRating(d['user/profileName'],d['beer/beerId']) for d in reviewDicts]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "def MSE(predictions, labels):\n",
    "    differences = [(x-y)**2 for x,y in zip(predictions,labels)]\n",
    "    return sum(differences) / len(differences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.4924295358999677"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "MSE(alwaysPredictMean, labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.4199912117748809"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "MSE(predictions, labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}