In [1]:
import json
import gzip
import math
from collections import defaultdict
import numpy
from sklearn import linear_model
In [2]:
def assertFloat(x):
    assert type(float(x)) == float

def assertFloatList(items, N):
    assert len(items) == N
    assert [type(float(x)) for x in items] == [float]*N
In [3]:
answers = {}
In [6]:
f = open("/home/julian/Downloads/spoilers.json.gz", 'r')
In [7]:
dataset = []
for l in f:
    d = eval(l)
    dataset.append(d)
In [8]:
f.close()
In [9]:
dataset[0]
Out[9]:
{'user_id': 'b0d7e561ca59e313b728dc30a5b1862e',
 'timestamp': '2013-05-06',
 'review_sentences': [[0,
   'The author did an excellent job of making a very readable novel about the emotional lives of Hadley and Ernest Hemingway.'],
  [0,
   'The many other creative people interacting with them in Paris in the 1920s were very colorful and interesting too.'],
  [0,
   'She captured a wonderful snapshot of the 1920s in Europe--the men returning from war, the writing, the music, the art, the fashions, the eating and drinking.'],
  [0,
   'Mostly told from the viewpoint of Hadley, the book shows she was very attracted to Ernest who was extroverted, interesting, ambitious, and a gifted writer.'],
  [0,
   'Hadley had a sweetness, and more traditional values than some of their friends in Paris.'],
  [0,
   'They both came from families with domineering mothers and suicidal fathers.'],
  [0,
   'Ernest was still suffering the traumatic effects of his time in World War I. He was also very self absorbed, and hurt Hadley very deeply.'],
  [0, 'Their love story spiraled downward as his career took off.'],
  [0,
   'I enjoyed this historical fiction about the five years of their marriage, and was sorry to see the book end.'],
  [0, 'First read August 3, 2011'],
  [0, 'Reread May 7, 2013 for a bookgroup']],
 'rating': 4,
 'has_spoiler': False,
 'book_id': '8683812',
 'review_id': 'd293fbf3512a0b05faffec43a7811c55'}
In [10]:
reviewsPerUser = defaultdict(list)
reviewsPerItem = defaultdict(list)

for d in dataset:
    u,i = d['user_id'],d['book_id']
    reviewsPerUser[u].append(d)
    reviewsPerItem[i].append(d)
    
for u in reviewsPerUser:
    reviewsPerUser[u].sort(key=lambda x: x['timestamp'])
    
for i in reviewsPerItem:
    reviewsPerItem[i].sort(key=lambda x: x['timestamp'])
In [11]:
[d['timestamp'] for d in reviewsPerUser['b0d7e561ca59e313b728dc30a5b1862e']]
Out[11]:
['2012-03-13',
 '2013-05-06',
 '2013-09-03',
 '2015-04-05',
 '2016-02-10',
 '2016-05-29']
In [12]:
def MSE(y, ypred):
    diffs = [(a-b)**2 for (a,b) in zip(y,ypred)]
    return sum(diffs) / len(diffs)
In [13]:
### 1a
In [14]:
y, ypred = [], []
for i in reviewsPerItem:
    ratings = [d['rating'] for d in reviewsPerItem[i]]
    if len(ratings) < 2: continue
    y.append(ratings[-1])
    ypred.append(sum(ratings[:-1]) / (len(ratings) - 1))
In [15]:
answers['Q1a'] = MSE(y,ypred)
In [14]:
assertFloat(answers['Q1a'])
In [15]:
### 1b
In [24]:
y, ypred = [], []
for u in reviewsPerUser:
    ratings = [d['rating'] for d in reviewsPerUser[u]]
    if len(ratings) < 2: continue
    y.append(ratings[-1])
    ypred.append(sum(ratings[:-1]) / (len(ratings) - 1))
In [26]:
answers['Q1b'] = MSE(y,ypred)
In [27]:
answers['Q1b']
Out[27]:
1.970416294395752
In [18]:
assertFloat(answers['Q1b'])
In [19]:
### 2
In [20]:
answers['Q2'] = []

for N in [1,2,3]:
    y, ypred = [], []
    for u in reviewsPerUser:
        ratings = [d['rating'] for d in reviewsPerUser[u][-(N+1):]]
        if len(ratings) < 2: continue
        y.append(ratings[-1])
        ypred.append(sum(ratings[:-1]) / (len(ratings) - 1))
    answers['Q2'].append(MSE(y,ypred))
In [21]:
assertFloatList(answers['Q2'], 3)
In [22]:
### 3a
In [23]:
def feature3(N, u):
    ratings = [d['rating'] for d in reviewsPerUser[u][-(N+1):-1]]
    if len(ratings) < N:
        raise Exception
    return [1] + ratings
In [24]:
answers['Q3a'] = [feature3(2,dataset[0]['user_id']), feature3(3,dataset[0]['user_id'])]
In [25]:
assert len(answers['Q3a']) == 2
assert len(answers['Q3a'][0]) == 3
assert len(answers['Q3a'][1]) == 4
In [26]:
### 3b
In [27]:
answers['Q3b'] = []

for N in [1,2,3]:
    y = []
    X = []
    for d in dataset:
        u = d['user_id']
        ratings = [d['rating'] for d in reviewsPerUser[u][-(N+1):]]
        if len(ratings) < N+1: continue
        y.append(ratings[-1])
        X.append(feature3(N,d['user_id']))
    theta,residuals,rank,s = numpy.linalg.lstsq(X,y)
    mse = residuals[0] / len(y)
    answers['Q3b'].append(mse)

/home/julian/.local/lib/python3.7/site-packages/ipykernel_launcher.py:12: FutureWarning: `rcond` parameter will change to the default of machine precision times ``max(M, N)`` where M and N are the input matrix dimensions.
To use the future default and silence this warning we advise to pass `rcond=None`, to keep using the old, explicitly pass `rcond=-1`.
  if sys.path[0] == '':
In [28]:
assertFloatList(answers['Q3b'], 3)
In [29]:
### 4a
In [30]:
globalAverage = [d['rating'] for d in dataset]
globalAverage = sum(globalAverage) / len(globalAverage)
In [31]:
def featureMeanValue(N, u):
    ratings = [d['rating'] for d in reviewsPerUser[u][-(N+1):-1]]
    nMissing = N - len(ratings)
    if len(ratings) > 0:
        av = sum(ratings) / len(ratings)
    else:
        av = globalAverage
    return [1] + [av]*nMissing + ratings
In [32]:
def featureMissingValue(N, u):
    ratings = [d['rating'] for d in reviewsPerUser[u][-(N+1):-1]]
    nMissing = N - len(ratings)
    feat = [0]*nMissing + ratings
    indicator = [1]*nMissing + [0]*len(ratings)
    return [1] + indicator + feat
In [33]:
answers['Q4a'] = [featureMeanValue(10, dataset[0]['user_id']), featureMissingValue(10, dataset[0]['user_id'])]
In [34]:
answers['Q4a']
Out[34]:
[[1, 4.2, 4.2, 4.2, 4.2, 4.2, 5, 4, 4, 4, 4],
 [1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4, 4, 4, 4]]
In [35]:
assert len(answers['Q4a']) == 2
assert len(answers['Q4a'][0]) == 11
assert len(answers['Q4a'][1]) == 21
In [36]:
### 4b
In [37]:
answers['Q4b'] = []

for featFunc in [featureMeanValue, featureMissingValue]:
    y = []
    X = []
    for d in dataset:
        u = d['user_id']
        ratings = [d['rating'] for d in reviewsPerUser[u]]
        y.append(ratings[-1])
        X.append(featFunc(10,d['user_id']))
    theta,residuals,rank,s = numpy.linalg.lstsq(X,y)
    mse = residuals[0] / len(y)
    answers['Q4b'].append(mse)

/home/julian/.local/lib/python3.7/site-packages/ipykernel_launcher.py:11: FutureWarning: `rcond` parameter will change to the default of machine precision times ``max(M, N)`` where M and N are the input matrix dimensions.
To use the future default and silence this warning we advise to pass `rcond=None`, to keep using the old, explicitly pass `rcond=-1`.
  # This is added back by InteractiveShellApp.init_path()
In [38]:
assertFloatList(answers["Q4b"], 2)
In [39]:
### 5
In [40]:
def feature5(sentence):
    return [1, len(sentence), sentence.count('!'), sum([c.isupper() for c in sentence])]
In [41]:
y = []
X = []

for d in dataset:
    for spoiler,sentence in d['review_sentences']:
        X.append(feature5(sentence))
        y.append(spoiler)
In [42]:
len(X)
Out[42]:
261176
In [43]:
mod = linear_model.LogisticRegression(class_weight='balanced')
mod.fit(X,y)
predictions = mod.predict(X) # Binary vector of predictions
In [44]:
def rates(predictions, y):
    TP = [a and b for (a,b) in zip(predictions,y)]
    TN = [not a and not b for (a,b) in zip(predictions,y)]
    FP = [a and not b for (a,b) in zip(predictions,y)]
    FN = [not a and b for (a,b) in zip(predictions,y)]

    TP = sum(TP)
    TN = sum(TN)
    FP = sum(FP)
    FN = sum(FN)
    
    return TP, TN, FP, FN
In [45]:
TP, TN, FP, FN = rates(predictions, y)
In [46]:
BER = 0.5 * (FP / (TN + FP) + FN / (FN + TP))
In [47]:
BER
Out[47]:
0.470265288006232
In [48]:
answers['Q5a'] = X[0]
In [49]:
answers['Q5b'] = [TP, TN, FP, FN, BER]
In [50]:
assert len(answers['Q5a']) == 4
assertFloatList(answers['Q5b'], 5)
In [51]:
### 6
In [52]:
def feature6(review):
    sentences = review['review_sentences']
    if len(sentences) < 6:
        raise Exception
    return [s[0] for s in sentences[:5]] + feature5(sentences[5][1])
In [53]:
y = []
X = []

for d in dataset:
    sentences = d['review_sentences']
    if len(sentences) < 6: continue
    X.append(feature6(d))
    y.append(sentences[5][0])

mod = linear_model.LogisticRegression(class_weight='balanced')
mod.fit(X,y)
predictions = mod.predict(X) # Binary vector of predictions

TP, TN, FP, FN = rates(predictions, y)
BER = 0.5 * (FP / (TN + FP) + FN / (FN + TP))

/usr/local/lib/python3.7/dist-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):
STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.

Increase the number of iterations (max_iter) or scale the data as shown in:
    https://scikit-learn.org/stable/modules/preprocessing.html
Please also refer to the documentation for alternative solver options:
    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression
  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,
In [54]:
answers['Q6a'] = X[0]
In [55]:
answers['Q6b'] = BER
In [56]:
assert len(answers['Q6a']) == 9
assertFloat(answers['Q6b'])
In [57]:
### 7
In [58]:
len(X)
Out[58]:
11907
In [59]:
Xtrain, Xvalid, Xtest = X[:len(X)//2], X[len(X)//2:(3*len(X))//4], X[(3*len(X))//4:]
ytrain, yvalid, ytest = y[:len(X)//2], y[len(X)//2:(3*len(X))//4], y[(3*len(X))//4:]
In [60]:
models = {}
bers = {}
bestC = None

for c in [0.01, 0.1, 1, 10, 100]:
    mod = linear_model.LogisticRegression(C=c, class_weight='balanced')
    mod.fit(Xtrain,ytrain)
    predictions = mod.predict(Xvalid)
    TP, TN, FP, FN = rates(predictions, yvalid)
    BER = 0.5 * (FP / (TN + FP) + FN / (FN + TP))
    models[c] = mod
    bers[c] = BER

/usr/local/lib/python3.7/dist-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):
STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.

Increase the number of iterations (max_iter) or scale the data as shown in:
    https://scikit-learn.org/stable/modules/preprocessing.html
Please also refer to the documentation for alternative solver options:
    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression
  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,
/usr/local/lib/python3.7/dist-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):
STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.

Increase the number of iterations (max_iter) or scale the data as shown in:
    https://scikit-learn.org/stable/modules/preprocessing.html
Please also refer to the documentation for alternative solver options:
    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression
  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,
/usr/local/lib/python3.7/dist-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):
STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.

Increase the number of iterations (max_iter) or scale the data as shown in:
    https://scikit-learn.org/stable/modules/preprocessing.html
Please also refer to the documentation for alternative solver options:
    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression
  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,
/usr/local/lib/python3.7/dist-packages/sklearn/linear_model/_logistic.py:818: ConvergenceWarning: lbfgs failed to converge (status=1):
STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.

Increase the number of iterations (max_iter) or scale the data as shown in:
    https://scikit-learn.org/stable/modules/preprocessing.html
Please also refer to the documentation for alternative solver options:
    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression
  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG,
In [61]:
bestC = min(bers)
In [62]:
predictions = models[bestC].predict(Xtest)
TP, TN, FP, FN = rates(predictions, ytest)
BER = 0.5 * (FP / (TN + FP) + FN / (FN + TP))
In [63]:
answers['Q7'] = list(bers.values()) + [bestC] + [BER]
In [64]:
assertFloatList(answers['Q7'], 7)
In [65]:
### 8
In [66]:
def Jaccard(s1, s2):
    numer = len(s1.intersection(s2))
    denom = len(s1.union(s2))
    if denom == 0:
        return 0
    return numer / denom
In [67]:
dataTrain = dataset[:15000]
dataTest = dataset[15000:]
In [68]:
itemAverages = defaultdict(list)
ratingMean = []

for d in dataTrain:
    itemAverages[d['book_id']].append(d['rating'])
    ratingMean.append(d['rating'])

for i in itemAverages:
    itemAverages[i] = sum(itemAverages[i]) / len(itemAverages[i])

ratingMean = sum(ratingMean) / len(ratingMean)
In [69]:
reviewsPerUser = defaultdict(list)
usersPerItem = defaultdict(set)

for d in dataTrain:
    u,i = d['user_id'], d['book_id']
    reviewsPerUser[u].append(d)
    usersPerItem[i].add(u)
In [70]:
def predictRating(user,item):
    ratings = []
    similarities = []
    for d in reviewsPerUser[user]:
        i2 = d['book_id']
        if i2 == item: continue
        ratings.append(d['rating'] - itemAverages[i2])
        similarities.append(Jaccard(usersPerItem[item],usersPerItem[i2]))
    if (sum(similarities) > 0):
        weightedRatings = [(x*y) for x,y in zip(ratings,similarities)]
        return itemAverages[item] + sum(weightedRatings) / sum(similarities)
    else:
        # User hasn't rated any similar items
        return ratingMean
In [71]:
alwaysPredictMean = [ratingMean for d in dataTest]
In [72]:
simPredictions = [predictRating(d['user_id'], d['book_id']) for d in dataTest]
In [73]:
labels = [d['rating'] for d in dataTest]
In [74]:
MSE(alwaysPredictMean, labels)
Out[74]:
1.4967523599999757
In [75]:
MSE(simPredictions, labels)
Out[75]:
1.533136609277726
In [76]:
answers["Q8"] = MSE(simPredictions, labels)
In [77]:
assertFloat(answers["Q8"])
In [78]:
### 9
In [79]:
MSE0 = []
MSE15 = []
MSE5 = []

for d in dataTest:
    u,i = d['user_id'], d['book_id']
    y = d['rating']
    ypred = predictRating(d['user_id'], d['book_id'])
    err = math.fabs(y - ypred)**2
    count = len(usersPerItem[i])
    if count == 0:
        MSE0.append(err)
    elif count <= 5:
        MSE15.append(err)
    else:
        MSE5.append(err)
        
        
MSE0 = sum(MSE0) / len(MSE0)
MSE15 = sum(MSE15) / len(MSE15)
MSE5 = sum(MSE5) / len(MSE5)
In [80]:
MSE0, MSE15, MSE5
Out[80]:
(1.742012484444442, 1.5287575880478592, 1.4979280057145445)
In [81]:
answers["Q9"] = [MSE0, MSE15, MSE5]
In [82]:
assertFloatList(answers["Q9"], 3)
In [83]:
### 10
In [84]:
itsMSE = 100.0
In [85]:
answers["Q10"] = ("describe your solution", itsMSE)
In [86]:
assert type(answers["Q10"][0]) == str
assertFloat(answers["Q10"][1])
In [87]:
f = open("answers_midterm.txt", 'w')
f.write(str(answers) + '\n')
f.close()
In [ ]: