Project 10

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import plotly.express as px
import nltk

import warnings
warnings.simplefilter(action='ignore')


df = pd.read_csv('../input/sms-spam-collection-dataset/spam.csv',usecols=['v1','v2'],encoding='cp1252')


df.head(2)

df.shape

df.info()

df['v1'].value_counts()

# renaming columns
df.rename(columns={'v1':'target','v2':'msg'},inplace=True)

df.head(2)


df['target'].value_counts()

df['target']=df['target'].apply(lambda x: 1 if x=='spam' else 0)


df.duplicated().sum()


df.drop_duplicates(keep='first',inplace=True)

df.reset_index(drop=True,inplace=True)


df.shape

EDA

df['target'].value_counts().plot(kind='pie',labels=['Ham','Spam'],autopct='%.2f%%',ylabel='',title='Target Data')



df['msg_chars']=df['msg'].apply(len)



print(df['msg'].str.split().apply(len).head())

 use this code, give better result then above
print(df['msg'].apply(lambda x : len(nltk.word_tokenize(x))).head())


df['msg_words']=df['msg'].apply(lambda x : len(nltk.word_tokenize(x)))


# creating new feature for no of sentence in msg
df['msg_sentences']=df['msg'].apply(lambda x : len(nltk.sent_tokenize(x)))


df.head()


df.describe()


# Spam 
df[df['target']==1].describe().style.background_gradient()


# Ham
df[df['target']==0].describe().style.background_gradient(cmap='YlOrRd') 



sns.displot(df[df['target']==1]['msg_chars'],label='Spam')


sns.displot(df[df['target']==0]['msg_chars'],label='Ham')



plt.figure(figsize=(15,5))
sns.distplot(df[df['target']==1]['msg_chars'],label='Spam')
sns.distplot(df[df['target']==0]['msg_chars'],label='Ham')
plt.legend()



plt.figure(figsize=(15,5))
sns.distplot(df[df['target']==1]['msg_words'],label='Spam')
sns.distplot(df[df['target']==0]['msg_words'],label='Ham')
plt.legend()



plt.figure(figsize=(15,5))
sns.distplot(df[df['target']==1]['msg_sentences'],label='Spam')
sns.distplot(df[df['target']==0]['msg_sentences'],label='Ham')
plt.legend()



plt.figure(figsize=(15,15))
plt.subplot(311)
sns.histplot(df[df['target']==0]['msg_chars'],color='g',label='Ham')
sns.histplot(df[df['target']==1]['msg_chars'],color='r',label='Spam')
plt.legend()

plt.subplot(312)
sns.histplot(df[df['target']==0]['msg_words'],color='g',label='Ham')
sns.histplot(df[df['target']==1]['msg_words'],color='r',label='Spam')
plt.legend()


plt.figure(figsize=(15,15))
plt.subplot(311)
sns.histplot(df[df['target']==0]['msg_chars'],color='g',label='Ham')
sns.histplot(df[df['target']==1]['msg_chars'],color='r',label='Spam')
plt.legend()

plt.subplot(312)
sns.histplot(df[df['target']==0]['msg_words'],color='g',label='Ham')
sns.histplot(df[df['target']==1]['msg_words'],color='r',label='Spam')
plt.legend()



sns.pairplot(df,hue='target')


# pearson coeff. correlation
sns.heatmap(df.corr(),annot=True)

• Data/Text Preprocessing

• So, basic text preprocessing are
  • lower case(converts to lower case)
  • tokenization (break into words)
  • removing special characters (remove ?!~{}@#$%^)
  • removing stop words(is,if,and,the,an,a) and punctuations (?!.)
  • stemming (get root words, (loved,loving,loves=love))

def text_prep(text):
    from nltk.corpus import stopwords
    from nltk.stem.porter import PorterStemmer
    ps = PorterStemmer()

    # lower case converting
    text = text.lower()
    
    # tokenization
    text = nltk.word_tokenize(text)
    
    # removing special chars
    ls = []
    for i in text:
       if i.isalnum():
            ls.append(i)
    ls2 = []
    # remove stopwords
for i in ls:
        if i not in stopwords.words('english'):
            ls2.append(i) 

    ls3 = []
    # apply stemming
    for i in ls2:
        ls3.append(ps.stem(i))
    
    return " ".join(ls3)

df['trans_msg']=df['msg'].apply(text_prep)



from wordcloud import WordCloud
wc = WordCloud(width=1200,height=500,min_font_size=10,background_color='white')


spam_wc = wc.generate(df['msg'].str.cat(sep=' ')) # object of words

plt.figure(figsize=(15,6))
plt.imshow(spam_wc)
plt.title('Frequently used words in Messages(in original msgs)',fontdict={'fontsize':20},loc='center',color='r')
plt.show()


spam_wc = wc.generate(df['trans_msg'].str.cat(sep=' ')) # object of words

plt.figure(figsize=(15,6))
plt.imshow(spam_wc)
plt.title('Frequently used words in Messages(in transformed msgs)',fontdict={'fontsize':20},loc='center',color='r')
plt.show()


spam_wc = wc.generate(df[df['target']==1]['trans_msg'].str.cat(sep=' ')) # object of words

plt.figure(figsize=(15,6))
plt.imshow(spam_wc)
plt.title('Frequently used words in Spam Messages(on transformed msgs)',fontdict={'fontsize':20},loc='center',color='r')
plt.show()


spam_wc = wc.generate(df[df['target']==1]['msg'].str.cat(sep=' ')) # object of words

plt.figure(figsize=(15,6))
plt.imshow(spam_wc)
plt.title('Frequently used words in Spam Messages(on original msgs)',fontdict={'fontsize':20},loc='center',color='r')
plt.show()




spam_wc = wc.generate(df[df['target']==0]['msg'].str.cat(sep=' ')) # object of words

plt.figure(figsize=(15,6))
plt.imshow(spam_wc)
plt.title('Frequently used words in Ham Messages(on original msgs)',fontdict={'fontsize':20},loc='center',color='r')
plt.show()


spam_wc = wc.generate(df[df['target']==0]['trans_msg'].str.cat(sep=' ')) # object of words
plt.figure(figsize=(15,6))
plt.imshow(spam_wc)
plt.title('Frequently used words in Ham Messages(on transformed msgs)',fontdict={'fontsize':20},loc='center',color='r')
plt.show()



from collections import Counter


spam_corpus =[] # list of all words
for i in (df[df['target']==1]['trans_msg'].tolist()):
    for word in i.split():
        spam_corpus.append(word)

# len(spam_corpus) # entire spam msgs words converted in to list

spam30 = pd.DataFrame(Counter(spam_corpus).most_common(30)) #most common 30 words in entire spam msgs only

plt.figure(figsize=(15,5))
sns.barplot(spam30[0],spam30[1])
plt.xticks(rotation=90)


plt.title('Most 30 words occured in Spam Messages',fontdict={'fontsize':20},loc='center',color='r')
plt.ylabel('Count')
plt.xlabel('Message word')
plt.show()

ham_corpus =[]
for i in (df[df['target']==0]['trans_msg'].tolist()):
    for word in i.split():
        ham_corpus.append(word)

# len(spam_corpus) # entire spam msgs words converted in to list

ham30 = pd.DataFrame(Counter(ham_corpus).most_common(30)) #most common 30 words in entire spam msgs only

plt.figure(figsize=(15,5))
sns.barplot(ham30[0],ham30[1])
plt.xticks(rotation=90)
plt.title('Most 30 words occured in Ham Messages',fontdict={'fontsize':20},loc='center',color='r')
plt.ylabel('Count')
plt.xlabel('Message word')
plt.show()