#Importing the necessary dependencies for the dataset
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
import string
import re
import nltk
from nltk.stem import WordNetLemmatizer
from nltk.corpus import stopwords
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.naive_bayes import MultinomialNB
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import accuracy_score
from sklearn.linear_model import SGDClassifier

dataset = "dataset.csv"

import chardet
with open(dataset, 'rb') as rawdata:
    result = chardet.detect(rawdata.read(100000))
result

data=pd.read_csv(dataset,encoding='Windows-1252')

#Getting Used to the dataset and exploring it's content
data.head()

#Basic Visualization of the given data
sns.countplot(data['Sentiment'])

#Sample row from dataset displayed in order to understand what the data looks like before preprocessing.
print(data.loc[[9]])
print(data['SentimentText'].loc[[9]])

#Removing Punctuation
punc = string.punctuation
plist = punc
def rmp(text):
    trans = str.maketrans('', '', plist)
    return text.translate(trans)
data['SentimentText']= data['SentimentText'].apply(lambda x: rmp(x))
#rmp refers to the function used to remove punctuation

#Removing Hashtags and Mentions
def rmhashmention(text):
    chars = ['@','#']
    for separator in  string.punctuation:
        if separator not in chars :
            text = text.replace(separator,' ')
    words = []
    for word in text.split():
        word = word.strip()
        if word:
            if word[0] not in chars:
                words.append(word)
    return ' '.join(words)
data['SentimentText']= data['SentimentText'].apply(lambda x: rmhashmention(x))

#Displaying the same sample row after basic preprocessing like removal or punctuation, hashtags, and mentions
print(data.loc[[9]])
print(data['SentimentText'].loc[[9]])

#First 5 rows of data displayed before removal of stopwords, links, numbers, and also before lemmatization
data.head(5)

#Removal of stopwords and Lemmatization
lemmatizer = WordNetLemmatizer()
nltk.download('stopwords')
stopws = set(stopwords.words('english'))
  
vc = {'VB', 'VBD', 'VBG', 'VBN', 'VBP', 'VBZ'}

def rem(text):
  text = text.lower()
  temp =[]
  words = nltk.word_tokenize(text)
  tags = nltk.pos_tag(words)
  for i, word in enumerate(words):
      if tags[i][1] in vc: 
          lemmatized = lemmatizer.lemmatize(word, 'v')
      else:
          lemmatized = lemmatizer.lemmatize(word)
      if lemmatized not in stopws and lemmatized.isalpha():
          temp.append(lemmatized)
          
  final = ' '.join(temp)
  final = final.replace("n't", " not")
  final = final.replace("'m", " am")
  final = final.replace("'s", " is")
  final = final.replace("'re", " are")
  final = final.replace("'ll", " will")
  final = final.replace("'ve", " have")
  final = final.replace("'d", " would")
  return final

#applying the function rem to text
data['SentimentText']= data['SentimentText'].apply(lambda x: rem(x))

#Removal of links and numbers
data['SentimentText'] = data['SentimentText'].str.strip().str.replace('   ', ' ').str.replace('  ', ' ')
data['SentimentText'] = data['SentimentText'].str.replace('\d+', '')
data['SentimentText'] = data['SentimentText'].str.replace(r'http?://[^\s<>"]+|www\.[^\s<>"]+', '')

#First 5 rows of data after the data has been manipulated
data.head(5)

#using countvectorizer to convert into Vector representation
count = CountVectorizer()
x = count.fit_transform(data['SentimentText'])

#Splitting dataset into training and testing
y = data.iloc[:,[1]].values
x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.30, random_state=50)

print("X_train.shape : ", x_train.shape)
print("X_test.shape : ", x_test.shape)
print("y_train.shape : ", y_train.shape)
print("y_test.shape : ", y_test.shape)

#using Stochastic Gradient Descent Classifier model
model = SGDClassifier(loss = 'hinge', penalty = 'l2', random_state=0)
model.fit(x_train,y_train)

ypred = model.predict(x_test)

#Printing the accuracy of the model that uses Countvectorizer and SGD Classifier
print("Accuracy Score for SGD Classifier Model using CountVectorizer is : ", accuracy_score(y_test, ypred))

#Printing the classification report for the model that uses Countvectorizer and SGD Classifier
from sklearn.metrics import classification_report
print("Classification Report for SGD Classifier Model using CountVectorizer : \n\n", classification_report(y_test, ypred))

#Using TF-IDF Vectorizer to convert into a matrix of tfidf features
tf = TfidfVectorizer(analyzer='word',min_df=0, stop_words='english')
X = tf.fit_transform(data['SentimentText'])
Y = data.iloc[:,[1]].values
X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.30, random_state=50)

model2 = SGDClassifier(loss = 'hinge', penalty = 'l2', random_state=0)
model2.fit(X_train, Y_train)
y2pred = model2.predict(X_test)

##Printing the accuracy of the model that uses TF-IDF vectorizer and SGD Classifier
print("Accuracy Score for SGD Classifier Model using TF-IDF vectorizer is : ", accuracy_score(Y_test, y2pred))

##Printing the classification report for the model that uses Countvectorizer and SGD Classifier
print("Classification Report for SGD Classifier Model using TF-IDF vectorizer : \n\n", classification_report(Y_test, y2pred))