import pandas as pd

import numpy as np

rating_df = pd.read_csv(r"C:UsersSumanth ReddyOneDriveDocumentsanime-recommendation/animelist.csv",low_memory=False,usecols=["user_id", "anime_id", "rating"],nrows=50000000)

rating_df.head(-15)

rating_df.shape

rating_df.isnull().sum()

rating_df.info()

# User should rate atleast 400 animes

n_ratings = rating_df['user_id'].value_counts()

rating_df = rating_df[rating_df['user_id'].isin(n_ratings[n_ratings >= 400].index)].copy()

​

print('Users who rated more than 400 animes are :',len(rating_df))

# Scaling between (0 , 1.0)

​

min_rating = min(rating_df['rating'])

max_rating = max(rating_df['rating'])

rating_df['rating'] = rating_df["rating"].apply(lambda x: (x - min_rating) / (max_rating - min_rating)).values.astype(np.float64)

​

​

# Average rating by the users

​

AvgRating = np.mean(rating_df['rating'])

print('Average rating by the user:', AvgRating)

# Removing Duplicated Rows from the data

​

duplicates = rating_df.duplicated()

​

if duplicates.sum() > 0:

    print('> {} duplicates'.format(duplicates.sum()))

    rating_df = rating_df[~duplicates]

​

print('> {} duplicates'.format(rating_df.duplicated().sum()))

g = rating_df.groupby('user_id')['rating'].count()

top_users = g.dropna().sort_values(ascending=False)[:20]

top_ratings = rating_df.join(top_users,rsuffix='_r',how='inner',on='user_id')

​

​

g = rating_df.groupby('anime_id')['rating'].count()

top_animes = g.dropna().sort_values(ascending=False)[:20]

top_ratings = top_ratings.join(top_animes,rsuffix='_r',how='inner',on='anime_id')

​

​

pd.crosstab(top_ratings.user_id, top_ratings.anime_id, top_ratings.rating,  aggfunc=np.sum)

# Now encoding the categorical data

​

user_ids = rating_df["user_id"].unique().tolist()

user2user_encoded = {x:i for i ,x in enumerate(user_ids)}

user_encoded2user = {x:i for i, x in enumerate(user_ids)}

rating_df["user"] = rating_df["user_id"].map(user2user_encoded)

n_users = len(user2user_encoded)

​

​

anime_ids = rating_df["anime_id"].unique().tolist()

anime2anime_encoded =  {x: i for i, x in enumerate(anime_ids)}

anime_encoded2anime = {i: x for i, x in enumerate(anime_ids)}

rating_df['anime'] = rating_df['anime_id'].map(anime2anime_encoded)

n_animes = len(anime2anime_encoded)

​

​

print("  Number of users : {} ".format(n_users))

print('-'*30)

print("  Number of animes : {}".format(n_animes))

print('-'*30)

print("  Minimum rating : {} ".format(min(rating_df['rating'])))

print('-'*30)

print("  Max Rating : {}".format(max(rating_df['rating'])))

rating_df = rating_df.sample(frac=1, random_state=69)

​

​

X = rating_df[['user','anime']].values

​

y = rating_df['rating']

test_set_size = 30000  # 30k for the test set

train_indices = rating_df.shape[0] - test_set_size

​

​

X_train, X_test, y_train, y_test = (

    X[:train_indices],

    X[train_indices:],

    y[:train_indices],

    y[train_indices:],

)

​

​

​

print(' >> Train set ratings : {}'.format(len(y_train)))

print('-'*30)

print(' >> Test set ratings : {}'.format(len(y_test)))

X_train_array = [X_train[:, 0], X_train[:, 1]]

X_test_array = [X_test[:, 0], X_test[:, 1]]

​

import keras

from keras import layers 

from keras.models import Model

from keras.optimizers import Adam

# Embedding layers

from keras.layers import Add, Activation, Lambda, BatchNormalization, Concatenate, Dropout, Input, Embedding, Dot, Reshape, Dense, Flatten

​

def RecommenderNet():

    embedding_size = 128

    user = Input(name = 'user', shape = [1])

    user_embedding = Embedding(name = 'user_embedding',

                       input_dim = n_users, 

                       output_dim = embedding_size)(user)

    anime = Input(name = 'anime', shape = [1])

    anime_embedding = Embedding(name = 'anime_embedding',

                       input_dim = n_animes, 

                       output_dim = embedding_size)(anime)

    #x = Concatenate()([user_embedding, anime_embedding])

    x = Dot(name = 'dot_product', normalize = True, axes = 2)([user_embedding, anime_embedding])

    x = Flatten()(x)

    x = Dense(1, kernel_initializer='he_normal')(x)

    x = BatchNormalization()(x)

    x = Activation("sigmoid")(x)

    model = Model(inputs=[user, anime], outputs=x)

    model.compile(loss='binary_crossentropy', metrics=["mae", "mse"], optimizer='Adam')

    return model

​

​

model = RecommenderNet()

​

model.summary()

# Callbacks

from tensorflow.keras.callbacks import Callback, ModelCheckpoint, LearningRateScheduler, TensorBoard, EarlyStopping, ReduceLROnPlateau

​

start_lr = 0.00001

min_lr = 0.00001

max_lr = 0.00005

batch_size = 30000

​

​

rampup_epochs = 5

sustain_epochs = 0

exp_decay = .8

​

def lrfn(epoch):

    if epoch < rampup_epochs:

        return (max_lr - start_lr)/rampup_epochs * epoch + start_lr

    elif epoch < rampup_epochs + sustain_epochs:

        return max_lr

    else:

        return (max_lr - min_lr) * exp_decay**(epoch-rampup_epochs-sustain_epochs) + min_lr

​

​

lr_callback = LearningRateScheduler(lambda epoch: lrfn(epoch), verbose=0)

​

checkpoint_filepath = './weights.h5'

​

model_checkpoints = ModelCheckpoint(filepath=checkpoint_filepath,

                                        save_weights_only=True,

                                        monitor='val_loss',

                                        mode='min',

                                        save_best_only=True)

​

early_stopping = EarlyStopping(patience = 3, monitor='val_loss', 

                               mode='min', restore_best_weights=True)

​

​

my_callbacks = [

    model_checkpoints,

    lr_callback,

    early_stopping,   

]

# Model training

history = model.fit(

    x=X_train_array,

    y=y_train,

    batch_size=batch_size,

    epochs=20,

    verbose=1,

    validation_data=(X_test_array, y_test),

    callbacks=my_callbacks

)

​

model.load_weights(checkpoint_filepath)

import matplotlib.pyplot as plt

%matplotlib inline 

​

plt.plot(history.history['loss'][0:-2])

plt.plot(history.history["val_loss"][0:-2])

​

plt.title("model loss")

plt.ylabel("loss")

plt.xlabel("epoch")

​

plt.legend(["train","test"],loc="upper left")

​

plt.show()

def extract_weights(name,model):

    weight_layer = model.get_layer(name)

    weights = weight_layer.get_weights()[0]

    weights = weights / np.linalg.norm(weights, axis = 1).reshape((-1,1))

    return weights

​

anime_weights = extract_weights('anime_embedding',model)

user_weights = extract_weights('user_embedding',model)

​

df = pd.read_csv(r'C:/Users/Sumanth Reddy/OneDrive/Documents/anime-recommendation/anime.csv.zip')

df = df.replace("Unknown",np.nan)

df.head()

df.info()

df.columns

# Fixing the names

​

def getAnimeName(anime_id):

    try:

        name = df[df.anime_id == anime_id].eng_version.values[0]

        if name is np.nan:

            name = df[df.anime_id == anime_id].Name.values[0]

    except:

        print('Error')

    return name

​

df['anime_id'] = df['MAL_ID']

df['eng_version'] = df['English name']

df["eng_version"] = df.anime_id.apply(lambda x:getAnimeName(x))

​

​

df.sort_values(by = ["Score"],

              inplace=True,

              ascending = False,

              kind = 'quicksort',

              na_position ='last')

​

​

​

df = df[["anime_id", "eng_version",

        "Score", "Genders", "Episodes",

        "Type", "Premiered", "Members"]]

​

def getAnimeFrame(anime):

    if isinstance(anime, int):

        return df[df.anime_id == anime]

    if isinstance(anime, str):

        return df[df.eng_version == anime]

​

cols = ["MAL_ID", "Name", "Genders", "sypnopsis"]

sypnopsis_df = pd.read_csv(r'C:/Users/Sumanth Reddy/OneDrive/Documents/anime-recommendation/anime_with_synopsis.csv.zip', usecols=cols)

​

def getSypnopsis(anime):

    if isinstance(anime, int):

        return sypnopsis_df[sypnopsis_df.MAL_ID == anime].sypnopsis.values[0]

    if isinstance(anime, str):

        return sypnopsis_df[sypnopsis_df.Name == anime].sypnopsis.values[0]

#pd.reset_option('all')

pd.set_option("max_colwidth", None)

​

def find_similar_animes(name, n=10, return_dist=False, neg=False):

    try:

        index = getAnimeFrame(name).anime_id.values[0]

        encoded_index = anime2anime_encoded.get(index)

        weights = anime_weights

        dists = np.dot(weights, weights[encoded_index])

        sorted_dists = np.argsort(dists)

        n = n + 1            

        if neg:

            closest = sorted_dists[:n]

        else:

            closest = sorted_dists[-n:]

​

        print('Animes closest to {} are : '.format(name))

​

        if return_dist:

            return dists, closest

        rindex = df

​

        SimilarityArr = []

​

        for close in closest:

            decoded_id = anime_encoded2anime.get(close)

            sypnopsis = getSypnopsis(decoded_id)

            anime_frame = getAnimeFrame(decoded_id)

            anime_name = anime_frame.eng_version.values[0]

            genre = anime_frame.Genders.values[0]

            similarity = dists[close]

            SimilarityArr.append({"anime_id": decoded_id, "name": anime_name,

                                  "similarity": similarity,"genre": genre,

                                  'sypnopsis': sypnopsis})

​

        Frame = pd.DataFrame(SimilarityArr).sort_values(by="similarity", ascending=False)

        return Frame[Frame.anime_id != index].drop(['anime_id'], axis=1)

​

    except:

        print('{}!, Not Found in Anime list'.format(name))

find_similar_animes('Dragon Ball Z', n=5, neg=False)

find_similar_animes('Your Name.', n=5, neg=False)

find_similar_animes('Sword Art Online', n=5, neg=False)

find_similar_animes('Black Clover', n=5, neg=False)

print(' || Picking up random user || ')

print('-'*30)

​

ratings_per_user = rating_df.groupby('user_id').size()

random_user = ratings_per_user[ratings_per_user < 500].sample(1, random_state=None).index[0]

print(' >> user_id:', random_user)

#pd.reset_option('all')

​

​

pd.set_option("max_colwidth", None)

​

def find_similar_users(item_input, n=10,return_dist=False, neg=False):

    try:

        index = item_input

        encoded_index = user2user_encoded.get(index)

        weights = user_weights

        dists = np.dot(weights, weights[encoded_index])

        sorted_dists = np.argsort(dists)

        n = n + 1

        if neg:

            closest = sorted_dists[:n]

        else:

            closest = sorted_dists[-n:]

​

        print('> users similar to # {}'.format(item_input))

​

        if return_dist:

            return dists, closest

        rindex = df

        SimilarityArr = []

        for close in closest:

            similarity = dists[close]

​

            if isinstance(item_input, int):

                decoded_id = user_encoded2user.get(close)

                SimilarityArr.append({"similar_users": decoded_id, 

                                      "similarity": similarity})

​

        Frame = pd.DataFrame(SimilarityArr).sort_values(by="similarity", 

                                                        ascending=False)

        return Frame

    except:

        print('{}!, not Found in User list'.format(name))

similar_users = find_similar_users(int(random_user), 

                                   n=5, 

                                   neg=False)

​

similar_users = similar_users[similar_users.similarity > 0.4]

similar_users = similar_users[similar_users.similar_users != random_user]

similar_users.head(3)

from wordcloud import WordCloud

from collections import defaultdict

import matplotlib.pyplot as plt

​

def showWordCloud(all_genres):

    genres_cloud = WordCloud(width=700, height=400, 

                             background_color='white', 

                             colormap='gnuplot').generate_from_frequencies(all_genres)

    plt.figure(figsize=(10,8)) 

    plt.imshow(genres_cloud, interpolation='bilinear')

    plt.axis('off')

    plt.show()

​

def getFavGenre(frame, plot=False):

        frame.dropna(inplace=False)

        all_genres = defaultdict(int)

        genres_list = []

        for genres in frame['Genders']:

            if isinstance(genres, str):

                for genre in genres.split(','):

                    genres_list.append(genre)

                    all_genres[genre.strip()] += 1    

        if plot:

            showWordCloud(all_genres)

        return genres_list

​

def get_user_preferences(user_id, plot=False, verbose=0):

    animes_watched_by_user = rating_df[rating_df.user_id==user_id]

    user_rating_percentile = np.percentile(animes_watched_by_user.rating, 75)

    animes_watched_by_user = animes_watched_by_user[animes_watched_by_user.rating >= user_rating_percentile]

    top_animes_user = (

        animes_watched_by_user.sort_values(by="rating", ascending=False)#.head(10)

        .anime_id.values

    )

    anime_df_rows = df[df["anime_id"].isin(top_animes_user)]

    anime_df_rows = anime_df_rows[["eng_version", "Genders"]]

    if verbose != 0:

        print("> User #{} has rated {} movies (avg. rating = {:.1f})".format(

          user_id, len(animes_watched_by_user),

          animes_watched_by_user['rating'].mean(),

        ))

        print('> preferred genres')

    if plot:

        getFavGenre(anime_df_rows, plot)

    return anime_df_rows#.eng_version.values

user_pref = get_user_preferences(random_user, plot=True, verbose=1)

print('> animes highly rated by this user')

​

pd.DataFrame(user_pref).head(10)

print("Showing recommendations for user: {}".format(random_user))

print("===" * 25)

​

animes_watched_by_user = rating_df[rating_df.user_id==random_user]

anime_not_watched_df = df[

    ~df["anime_id"].isin(animes_watched_by_user.anime_id.values)

]

​

anime_not_watched = list(

    set(anime_not_watched_df['anime_id']).intersection(set(anime2anime_encoded.keys()))

)

​

anime_not_watched = [[anime2anime_encoded.get(x)] for x in anime_not_watched]

​

user_encoder = user2user_encoded.get(random_user)

​

user_anime_array = np.hstack(

    ([[user_encoder]] * len(anime_not_watched), anime_not_watched)

)

​

user_anime_array = [user_anime_array[:, 0], user_anime_array[:, 1]]

ratings = model.predict(user_anime_array).flatten()

​

top_ratings_indices = (-ratings).argsort()[:10]

​

recommended_anime_ids = [

    anime_encoded2anime.get(anime_not_watched[x][0]) for x in top_ratings_indices

]

​

Results = []

top_rated_ids = []

​

for index, anime_id in enumerate(anime_not_watched):

    rating = ratings[index]

    id_ = anime_encoded2anime.get(anime_id[0])

    if id_ in recommended_anime_ids:

        top_rated_ids.append(id_)

        try:

            condition = (df.anime_id == id_)

            name = df[condition]['eng_version'].values[0]

            genre = df[condition].Genders.values[0]

            score = df[condition].Score.values[0]

            sypnopsis = getSypnopsis(int(id_))

        except:

            continue

        Results.append({#"anime_id": id_, 

                        "name": name, 

                        "pred_rating": rating,

                        "genre": genre, 

                        'sypnopsis': sypnopsis})

​

print("---" * 25)

print("> Top 10 anime recommendations")

print("---" * 25)

​

​

Results = pd.DataFrame(Results).sort_values(by='pred_rating', ascending=False)

Results

model.save('anime_model.h5')

​

from IPython.display import FileLink

FileLink(r'./anime_model.h5')






Thanks & Regards,
Sai Sumanth Reddy