In [1]:
import gzip
import math
import matplotlib.pyplot as plt
import numpy
import random
import sklearn
import string
from collections import defaultdict
from gensim.models import Word2Vec
from nltk.stem.porter import *
from sklearn import linear_model
from sklearn.manifold import TSNE
Data is available at http://cseweb.ucsd.edu/~jmcauley/pml/data/. Download and save to your own directory
In [2]:
dataDir = "/home/jmcauley/pml_data/"
Bag-of-words models¶
In [3]:
def parseData(fname):
for l in open(fname):
yield eval(l)
In [4]:
data = list(parseData(dataDir + "beer_50000.json"))[:5000]
How many unique words are there?
In [5]:
wordCount = defaultdict(int)
for d in data:
for w in d['review/text'].split():
wordCount[w] += 1
len(wordCount)
Out[5]:
Ignore capitalization and remove punctuation
In [6]:
wordCount = defaultdict(int)
punctuation = set(string.punctuation)
for d in data:
r = ''.join([c for c in d['review/text'].lower() if not c in punctuation])
for w in r.split():
wordCount[w] += 1
len(wordCount)
Out[6]:
With stemming
In [7]:
wordCount = defaultdict(int)
punctuation = set(string.punctuation)
stemmer = PorterStemmer()
for d in data:
r = ''.join([c for c in d['review/text'].lower() if not c in punctuation])
for w in r.split():
w = stemmer.stem(w)
wordCount[w] += 1
len(wordCount)
Out[7]:
Just build our feature vector by taking the most popular words (lowercase, punctuation removed, but no stemming)
In [8]:
wordCount = defaultdict(int)
punctuation = set(string.punctuation)
for d in data:
r = ''.join([c for c in d['review/text'].lower() if not c in punctuation])
for w in r.split():
wordCount[w] += 1
counts = [(wordCount[w], w) for w in wordCount]
counts.sort()
counts.reverse()
In [9]:
words = [x[1] for x in counts[:1000]]
Sentiment analysis¶
In [10]:
wordId = dict(zip(words, range(len(words))))
wordSet = set(words)
In [11]:
def feature(datum):
feat = [0]*len(words)
r = ''.join([c for c in datum['review/text'].lower() if not c in punctuation])
for w in r.split():
if w in words:
feat[wordId[w]] += 1
feat.append(1) # offset
return feat
Extract bag-of-word features. For a bigger dataset, replace this with a sparse matrix to save memory (see examples in Chapter 6)
In [12]:
X = [feature(d) for d in data]
y = [d['review/overall'] for d in data]
In [13]:
# Regularized regression
clf = linear_model.Ridge(1.0, fit_intercept=False) # MSE + 1.0 l2
clf.fit(X, y)
theta = clf.coef_
predictions = clf.predict(X)
N-grams¶
Simple example...
In [14]:
sentence = "Dark red color, light beige foam"
unigrams = sentence.split()
bigrams = list(zip(unigrams[:-1], unigrams[1:]))
trigrams = list(zip(unigrams[:-2], unigrams[1:-1], unigrams[2:]))
In [15]:
trigrams
Out[15]:
Extract n-grams up to length 5 (same dataset as example above)
In [16]:
wordCount = defaultdict(int)
punctuation = set(string.punctuation)
for d in data:
r = ''.join([c for c in d['review/text'].lower() if not c in punctuation])
ws = r.split()
ws2 = [' '.join(x) for x in list(zip(ws[:-1],ws[1:]))]
ws3 = [' '.join(x) for x in list(zip(ws[:-2],ws[1:-1],ws[2:]))]
ws4 = [' '.join(x) for x in list(zip(ws[:-3],ws[1:-2],ws[2:-1],ws[3:]))]
ws5 = [' '.join(x) for x in list(zip(ws[:-4],ws[1:-3],ws[2:-2],ws[3:-1],ws[4:]))]
for w in ws + ws2 + ws3 + ws4 + ws5:
wordCount[w] += 1
counts = [(wordCount[w], w) for w in wordCount]
counts.sort()
counts.reverse()
In [17]:
words = [x[1] for x in counts[:1000]]
A few of our 1000 most popular n-grams. Note the combination of n-grams of different lengths
In [18]:
words[200:210]
Out[18]:
In [19]:
wordId = dict(zip(words, range(len(words))))
wordSet = set(words)
In [20]:
def feature(datum):
feat = [0]*len(words)
r = ''.join([c for c in datum['review/text'].lower() if not c in punctuation])
ws = r.split()
ws2 = [' '.join(x) for x in list(zip(ws[:-1],ws[1:]))]
ws3 = [' '.join(x) for x in list(zip(ws[:-2],ws[1:-1],ws[2:]))]
ws4 = [' '.join(x) for x in list(zip(ws[:-3],ws[1:-2],ws[2:-1],ws[3:]))]
ws5 = [' '.join(x) for x in list(zip(ws[:-4],ws[1:-3],ws[2:-2],ws[3:-1],ws[4:]))]
for w in ws + ws2 + ws3 + ws4 + ws5:
if w in words:
feat[wordId[w]] += 1
feat.append(1) #offset
return feat
Same as the model in the previous example above, except using n-grams rather than just unigrams
In [21]:
X = [feature(d) for d in data]
y = [d['review/overall'] for d in data]
In [22]:
clf = linear_model.Ridge(1.0, fit_intercept=False) # MSE + 1.0 l2
clf.fit(X, y)
theta = clf.coef_
predictions = clf.predict(X)
In [23]:
wordSort = list(zip(theta[:-1], words))
wordSort.sort()
Some of the most negative and positive n-grams...
In [24]:
wordSort[:20]
Out[24]:
In [25]:
wordSort[-20:]
Out[25]:
TF-IDF¶
In [26]:
dataset = []
Small set of Goodreads fantasy reviews
In [27]:
z = gzip.open(dataDir + "goodreads_reviews_fantasy_paranormal.json.gz")
for l in z.readlines():
dataset.append(eval(l))
if len(dataset) == 50000:
break
For example...
In [28]:
dataset[0]
Out[28]:
Similar process to extract bag-of-words representations as in previous examples
In [29]:
wordCount = defaultdict(int)
punctuation = set(string.punctuation)
for d in dataset:
r = ''.join([c for c in d['review_text'].lower() if not c in punctuation])
for w in r.split():
wordCount[w] += 1
In [30]:
counts = [(wordCount[w], w) for w in wordCount]
counts.sort()
counts.reverse()
In [31]:
words = [x[1] for x in counts[:1000]]
Document frequency (df)¶
In [32]:
df = defaultdict(int)
for d in dataset:
r = ''.join([c for c in d['review_text'].lower() if not c in punctuation])
for w in set(r.split()):
df[w] += 1
Term frequency (tf)¶
Here we extract frequencies for terms in a single specific review
In [33]:
rev = dataset[9] # Query review
rev
Out[33]:
In [34]:
tf = defaultdict(int)
r = ''.join([c for c in rev['review_text'].lower() if not c in punctuation])
for w in r.split():
# Note = rather than +=, different versions of tf could be used instead
tf[w] = 1
tfidf = dict(zip(words,[tf[w] * math.log2(len(dataset) / df[w]) for w in words]))
tfidfQuery = [tf[w] * math.log2(len(dataset) / df[w]) for w in words]
Find the highest tf-idf words in our example review
In [35]:
maxTf = [(tf[w],w) for w in words]
maxTf.sort(reverse=True)
maxTfIdf = [(tfidf[w],w) for w in words]
maxTfIdf.sort(reverse=True)
In [36]:
maxTfIdf[:10]
Out[36]:
Cosine similarity
In [37]:
def Cosine(x1,x2):
numer = 0
norm1 = 0
norm2 = 0
for a1,a2 in zip(x1,x2):
numer += a1*a2
norm1 += a1**2
norm2 += a2**2
if norm1*norm2:
return numer / math.sqrt(norm1*norm2)
return 0
Find the other reviews in the corpus with the highest cosine similarity between tf-idf vectors
In [38]:
similarities = []
for rev2 in dataset:
tf = defaultdict(int)
r = ''.join([c for c in rev2['review_text'].lower() if not c in punctuation])
for w in r.split():
# Note = rather than +=
tf[w] = 1
tfidf2 = [tf[w] * math.log2(len(dataset) / df[w]) for w in words]
similarities.append((Cosine(tfidfQuery, tfidf2), rev2['review_text']))
In [39]:
similarities.sort(reverse=True)
In [40]:
similarities[:10]
Out[40]:
word2vec (gensim)¶
A few utility data structures (used later)
In [41]:
beerStyles = {} # Style of each item
categories = set() # Set of item categories
reviewsPerUser = defaultdict(list)
beerIdToName = {} # Map an ID to the name of the product
In [42]:
z = open(dataDir + "beer_50000.json")
reviews = []
reviewDicts = []
for l in z:
d = eval(l)
reviews.append(d['review/text'])
beerStyles[d['beer/beerId']] = d['beer/style']
categories.add(d['beer/style'])
beerIdToName[d['beer/beerId']] = d['beer/name']
reviewsPerUser[d['user/profileName']].append((d['review/timeUnix'], d['beer/beerId']))
reviewDicts.append(d)
if len(reviews) == 50000:
break
Tokenize the reviews, so that each review becomes a list of words
In [43]:
reviewTokens = []
In [44]:
punctuation = set(string.punctuation)
for r in reviews:
r = ''.join([c for c in r.lower() if not c in punctuation])
tokens = []
for w in r.split():
tokens.append(w)
reviewTokens.append(tokens)
Example of a tokenized review
In [45]:
reviewTokens[0]
Out[45]:
Fit the word2vec model
In [46]:
model = Word2Vec(reviewTokens,
min_count=5, # Words/items with fewer instances are discarded
size=10, # Model dimensionality
window=3, # Window size
sg=1) # Skip-gram model
Extract word representation for a particular word
In [47]:
model.wv['yeast']
Out[47]:
Find similar words to a given query
In [48]:
model.wv.similar_by_word("grassy")
Out[48]:
item2vec¶
Almost the same as word2vec, but "documents" are made up of sequences of item IDs rather than words
In [49]:
reviewLists = []
for u in reviewsPerUser:
rl = list(reviewsPerUser[u])
rl.sort()
reviewLists.append([x[1] for x in rl])
In [50]:
model10 = Word2Vec(reviewLists,
min_count=5, # Words/items with fewer instances are discarded
size=10, # Model dimensionality
window=3, # Window size
sg=1) # Skip-gram model
In [51]:
beerIdToName['7360']
Out[51]:
In [52]:
model10.wv.similar_by_word('7360')
Out[52]:
In [53]:
for b in model10.wv.similar_by_word('7360'):
print(beerIdToName[b[0]])
t-SNE embedding¶
Visualize the embeddings from the model above using t-SNE
In [54]:
X = []
beers = []
for b in beerIdToName:
try:
X.append(list(model10.wv[b]))
beers.append(b)
except Exception as e:
pass
Fit a model with just two components for the sake of visualization
In [55]:
X_embedded = TSNE(n_components=2).fit_transform(X)
Generate scatterplots using the embedded points (one scatter plot per category)
In [56]:
scatterPlotsX = defaultdict(list)
scatterPlotsY = defaultdict(list)
for xy, b in zip(X, beers):
cat = beerStyles[b]
try:
scatterPlotsX[cat].append(xy[0])
scatterPlotsY[cat].append(xy[1])
except Exception as e:
pass
Plot data from a few categories (more interesting with a larger dataset)
In [57]:
plt.scatter(scatterPlotsX['American Adjunct Lager'],
scatterPlotsY['American Adjunct Lager'], color='k', lw=0, label="Adjunct Lager")
plt.scatter(scatterPlotsX['American Porter'],
scatterPlotsY['American Porter'], color='grey', lw=0, label = "Porter")
plt.scatter(scatterPlotsX['Smoked Beer'],
scatterPlotsY['Smoked Beer'], color='lightgrey', lw = 0, label = "Smoked Beer")
plt.legend(loc='lower left')
plt.xticks([])
plt.yticks([])
plt.xlabel("first embedded dimension ")
plt.ylabel("second embedded dimension")
plt.title("\emph{TSNE}-based item embeddings")
plt.show()
Exercises¶
8.1 / 8.2¶
Simple sentiment analysis pipeline
In [58]:
def parseData(fname):
for l in open(fname):
yield eval(l)
In [59]:
data = list(parseData(dataDir + "beer_50000.json"))[:5000]
In [60]:
random.shuffle(data)
Add a couple of "options" for the representation (in this case whether we should convert to lower case, remove puncuation). More could be added.
In [61]:
def feature(datum, words, wordId, tolower=True, removePunct=True):
feat = [0]*len(words)
r = datum['review/text']
if tolower:
r = r.lower()
if removePunct:
r = ''.join([c for c in r if not c in punctuation])
for w in r.split():
if w in words:
feat[wordId[w]] += 1
feat.append(1) # offset
return feat
Condense the pipeline code (see Chapter 3) into a single function
In [62]:
def pipeline(dSize = 1000, tolower=True, removePunct=True):
wordCount = defaultdict(int)
punctuation = set(string.punctuation)
for d in data: # Strictly, should just use the *training* data to extract word counts
r = d['review/text']
if tolower:
r = r.lower()
if removePunct:
r = ''.join([c for c in r if not c in punctuation])
for w in r.split():
wordCount[w] += 1
counts = [(wordCount[w], w) for w in wordCount]
counts.sort()
counts.reverse()
words = [x[1] for x in counts[:dSize]]
wordId = dict(zip(words, range(len(words))))
wordSet = set(words)
X = [feature(d, words, wordId, tolower, removePunct) for d in data]
y = [d['review/overall'] for d in data]
Ntrain,Nvalid,Ntest = 4000,500,500
Xtrain,Xvalid,Xtest = X[:Ntrain],X[Ntrain:Ntrain+Nvalid],X[Ntrain+Nvalid:]
ytrain,yvalid,ytest = y[:Ntrain],y[Ntrain:Ntrain+Nvalid],y[Ntrain+Nvalid:]
bestModel = None
bestVal = None
bestLamb = None
ls = [0.01, 0.1, 1, 10, 100, 1000, 10000]
errorTrain = []
errorValid = []
for l in ls:
model = sklearn.linear_model.Ridge(l)
model.fit(Xtrain, ytrain)
predictTrain = model.predict(Xtrain)
MSEtrain = sum((ytrain - predictTrain)**2)/len(ytrain)
errorTrain.append(MSEtrain)
predictValid = model.predict(Xvalid)
MSEvalid = sum((yvalid - predictValid)**2)/len(yvalid)
errorValid.append(MSEvalid)
print("l = " + str(l) + ", validation MSE = " + str(MSEvalid))
if bestVal == None or MSEvalid < bestVal:
bestVal = MSEvalid
bestModel = model
bestLamb = l
predictTest = bestModel.predict(Xtest)
MSEtest = sum((ytest - predictTest)**2)/len(ytest)
MSEtest
plt.xticks([])
plt.xlabel(r"$\lambda$")
plt.ylabel(r"error (MSE)")
plt.title(r"Validation Pipeline")
plt.xscale('log')
plt.plot(ls, errorTrain, color='k', linestyle='--', label='training error')
plt.plot(ls, errorValid, color='grey',zorder=4,label="validation error")
plt.plot([bestLamb], [MSEtest], linestyle='', marker='x', color='k', label="test error")
plt.legend(loc='best')
plt.show()
Run models with different feature representation options and dictionary sizes
In [63]:
pipeline(2000, False, False)
In [64]:
pipeline(1000, True, True)
8.3¶
Use item2vec to make recommendations (following code from exercises in Chapter 4)
In [65]:
def recScore(i, userHistory):
historyInVocab = [w for w in userHistory if w in model10.wv]
if len(historyInVocab) == 0:
return 0
sc = model10.wv.distance(str(i), historyInVocab[-1])
return sc
def rec(userHistory):
historyInVocab = [w for w in userHistory if w in model10.wv]
if len(historyInVocab) == 0:
return 0
return model10.wv.most_similar(positive = historyInVocab, topn=10)
In [66]:
recScore(20539, reviewLists[0])
Out[66]:
In [67]:
rec(reviewLists[0])
Out[67]:
8.4¶
(see tf-idf retrieval examples above)
8.5¶
Predict the rating using item2vec item similarity scores. Adapts models from Chapter 4.
In [68]:
ratingMean = sum([d['review/overall'] for d in reviewDicts]) / len(reviewDicts)
In [69]:
itemAverages = defaultdict(list)
reviewsPerUser = defaultdict(list)
for d in reviewDicts:
i = d['beer/beerId']
u = d['user/profileName']
itemAverages[i].append(d['review/overall'])
reviewsPerUser[u].append(d)
for i in itemAverages:
itemAverages[i] = sum(itemAverages[i]) / len(itemAverages[i])
In [70]:
def predictRating(user,item):
ratings = []
similarities = []
if not str(item) in model10.wv:
return ratingMean
for d in reviewsPerUser[user]:
i2 = d['beer/beerId']
if i2 == item: continue
ratings.append(d['review/overall'] - itemAverages[i2])
if str(i2) in model10.wv:
similarities.append(model10.wv.distance(str(item), str(i2)))
if (sum(similarities) > 0):
weightedRatings = [(x*y) for x,y in zip(ratings,similarities)]
return itemAverages[item] + sum(weightedRatings) / sum(similarities)
else:
return ratingMean
In [71]:
u,i = reviewDicts[0]['user/profileName'],reviewDicts[0]['beer/beerId']
In [72]:
predictRating(u,i)
Out[72]:
In [73]:
ratingMean
Out[73]:
In [74]:
alwaysPredictMean = [ratingMean for _ in reviewDicts]
In [75]:
labels = [d['review/overall'] for d in reviewDicts]
In [76]:
predictions = [predictRating(d['user/profileName'],d['beer/beerId']) for d in reviewDicts]
In [77]:
def MSE(predictions, labels):
differences = [(x-y)**2 for x,y in zip(predictions,labels)]
return sum(differences) / len(differences)
In [78]:
MSE(alwaysPredictMean, labels)
Out[78]:
In [79]:
MSE(predictions, labels)
Out[79]: