%matplotlib inline
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from scipy import stats
from ast import literal_eval
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
from sklearn.metrics.pairwise import linear_kernel, cosine_similarity
from nltk.stem.snowball import SnowballStemmer
import re
#from nltk.stem.wordnet import WordNetLemmatizer
#from nltk.corpus import wordnet
#from surprise import Reader, Dataset, SVD, evaluate

import requests
from bs4 import BeautifulSoup
import time

import uncurl
import json

context = uncurl.parse_context(cURL)

s = requests.session()
response = s.post(context.url, json=json.loads(context.data[1:]), headers=context.headers)

from benedict import benedict

response_json = benedict(response.json()[0])

senscrtq = pd.json_normalize(response_json['data.user.collection.products'])

senscrtq.loc[senscrtq.originalTitle.isnull(),"originalTitle"] = senscrtq.loc[senscrtq.originalTitle.isnull(),"title"]

senscrtq = senscrtq[["originalTitle","dateReleaseOriginal","otherUserInfos.rating"]]

senscrtq.dateReleaseOriginal = pd.to_datetime(senscrtq.dateReleaseOriginal)

senscrtq["decade"] = ((senscrtq.dateReleaseOriginal.dt.year//10)*10).astype("Int64")

senscrtq["year"] = senscrtq.dateReleaseOriginal.dt.year

senscrtq.rename(columns={"originalTitle":"original_title","otherUserInfos.rating":"note"},inplace=True)

senscrtq.to_csv('my_note3.csv',index=False)

def weighted_rating(x):
    """IMDB weighting formula"""
    v = x['vote_count']
    R = x['vote_average']
    return (v/(v+m) * R) + (m/(m+v) * C)

movies['wr'] = movies.apply(weighted_rating, axis=1)

movies = movies[pd.to_numeric(movies['id'], errors='coerce').notnull()]

movies['id'] = movies.id.astype(int)

def get_crew(x,job):
    """
    Retrieve a crew member from his/her job label
    """
    list_ = literal_eval(x)
    if isinstance(list_, list):
        job_list = [i['name'] for i in list_ if i['job']==job]
        if len(job_list)>0:
            return job_list[0]
        else:
            return ""
    else:
        return ""

keywords = pd.read_csv('keywords.csv')

movies['Director'] = movies['Director'].str.lower().str.replace(" ", "")
movies['Director of Photography'] = movies['Director of Photography'].str.lower().str.replace(" ", "")
#movies['genres'] = movies['genres'].apply(lambda x: [stemmer.stem(i) for i in x])
movies['genres'] = movies['genres'].apply(lambda x: [str.lower(i.replace(" ", "")) for i in x])
#movies['keywords'] = movies['keywords'].apply(lambda x: [stemmer.stem(i,) for i in x])
movies['keywords'] = movies['keywords'].apply(lambda x: [str.lower(i) for i in x])
movies['keywords'] = movies['keywords'].apply(lambda x: [''] if len(x) < 1 else x)
movies['keywords'] = movies['keywords'].apply(lambda x: np.hstack([re.sub("[^w]", " ",  word).split() for word in x]).tolist())

movies['soup'] = movies['keywords']+movies['genres']

from sklearn.metrics.pairwise import linear_kernel, cosine_similarity
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer

def get_recommendations_CV(df, title,n=100):
    count  = CountVectorizer(analyzer='word',ngram_range=(1, 2),min_df=1, stop_words='english')
    count_matrix = count.fit_transform(df['soup'])
    cosine_sim = cosine_similarity(count_matrix, count_matrix)
    idx = df[df['original_title'] == title ].index[0]
    sim_scores = list(enumerate(cosine_sim[idx]))
    sim_scores = sorted(sim_scores, key=lambda x: x[1], reverse=True)
    sim_scores = sim_scores[1:1+n]
    movie_indices = [i[0] for i in sim_scores]
    return df[['id','original_title','year','wr']].iloc[movie_indices].sort_values('wr',ascending=False)

from sentence_transformers import SentenceTransformer
from sentence_transformers import models, losses

model = SentenceTransformer('bert-base-nli-mean-tokens')

text_embeddings = model.encode(movies['soup'], show_progress_bar = True)
similarities = cosine_similarity(text_embeddings)

with open('similarities.npy', 'wb') as f:
    np.save(f, similarities)

def get_recommendations_BERT(df, title,n=100):
    similarities = np.load('similarities.npy')
    idx = df[df['original_title'] == title ].index[0]
    sim_scores = list(enumerate(similarities[idx]))
    sim_scores = sorted(sim_scores, key=lambda x: x[1], reverse=True)
    sim_scores = sim_scores[1:1+n]
    movie_indices = [i[0] for i in sim_scores]
    return df[['id','original_title','year','wr']].iloc[movie_indices].sort_values('wr',ascending=False)

match = pd.read_csv('links_small.csv')
match = match.dropna(how="any")
match['tmdbId'] = match['tmdbId'].astype('Int64')#.astype(str)
match.rename(columns={'tmdbId':'id'},inplace=True)
match.set_index('id',inplace=True)

from surprise import Reader, Dataset, SVD
from surprise.model_selection import cross_validate, GridSearchCV

reader = Reader()

inner_df = pd.merge(my_notes,movies, on=['original_title','year'],how='inner')
inner_df['id'] = inner_df.id.astype("int64",errors='ignore')

#inner_df= inner_df[['original_title','year','note','id']]
#inner_df.drop_duplicates(inplace=True)

inner_df['rating'] = inner_df['note']/2
inner_df['userId'] = -1
inner_df = inner_df.merge(match.reset_index(),how='inner',on='id')

def helper(x,userId,algo):
    try:
        return algo.predict(userId,match['movieId'].to_dict()[x]).est
    except:
        return np.nan

def hybrid(df, userId, title, algo = svd):
    ds = get_recommendations_CV(df, title, n=50)
    df = df.loc[ds.index,['id','original_title','year','wr']]
    #df = ds.reset_index().rename(columns={'index':'id'})
    df['est'] = df['id'].apply(lambda x: helper(x,userId,algo))
    df.dropna(axis=0,how='any', inplace=True)
    df.sort_values('est', ascending=False, inplace =True)
    df = df.head(10)
    df['year'] = df['year'].astype(int)
    df['link'] = df['id'].apply(lambda x: 'https://www.themoviedb.org/movie/{}'.format(str(x)))
    return df.head(10)

def hybrid_bis(df, userId, title, algo = svd):
    ds = get_recommendations_BERT(df, title, n=50)
    df = df.loc[ds.index,['id','original_title','year','wr']]
    #df = ds.reset_index().rename(columns={'index':'id'})
    df['est'] = df['id'].apply(lambda x: helper(x,userId,algo))
    df.dropna(axis=0,how='any', inplace=True)
    df.sort_values('est', ascending=False, inplace =True)
    df = df.head(10)
    df['year'] = df['year'].astype(int)
    df['link'] = df['id'].apply(lambda x: 'https://www.themoviedb.org/movie/{}'.format(str(x)))
    return df.head(10)

response = hybrid(movies,-1, 'This Is Spinal Tap', algo = svd)
response2 = hybrid_bis(movies,-1, 'This Is Spinal Tap', algo = svd)