Performing Twitter Sentiment Analysis-For beginner-

Learn how to check whether tweets are racist or no

Import Basic Libraries

import sklearn
import numpy as np
import pandas as pd

Connection with google drive

from google.colab import drive
drive.mount('/content/gdrive')

Import Data

# training data
train = pd.read_csv("/content/gdrive/My Drive/Colab Notebooks/data/train.csv")
# test data
test = pd.read_csv("/content/gdrive/My Drive/Colab Notebooks/data/test.csv")

Data Exploration (Exploratory Data Analysis)

train.head()

test.tail()

# non-racist/sexist related tweets
sum(train["label"] == 0)

29720

# racist/sexist related tweets
sum(train["label"] == 1)

2242

# check if there are any missing values
train.isnull().sum()
#train.isnull().values.any()

Data cleaning

#install tweet-preprocessor to clean tweets
!pip install tweet-preprocessor

# remove special characters using the regular expression library
import re
#set up punctuations we want to be replaced
REPLACE_NO_SPACE = re.compile("(\.)|(\;)|(\:)|(\!)|(\')|(\?)|(\,)|(\")|(\|)|(\()|(\))|(\[)|(\])|(\%)|(\$)|(\>)|(\<)|(\{)|(\})")
REPLACE_WITH_SPACE = re.compile("(<br\s/><br\s/?)|(-)|(/)|(:).")
import preprocessor as p
# custum function to clean the dataset (combining tweet_preprocessor and reguar expression)
def clean_tweets(df):
tempArr = []
for line in df:
# send to tweet_processor
tmpL = p.clean(line)
# remove puctuation
tmpL = REPLACE_NO_SPACE.sub("", tmpL.lower()) # convert all tweets to lower cases
tmpL = REPLACE_WITH_SPACE.sub(" ", tmpL)
tempArr.append(tmpL)
return tempArr
# clean training data
train_tweet = clean_tweets(train["tweet"])
train_tweet = pd.DataFrame(train_tweet)
# append cleaned tweets to the training data
train["clean_tweet"] = train_tweet
# compare the cleaned and uncleaned tweets
train.head(10)

# clean the test data and append the cleaned tweets to the test data
test_tweet = clean_tweets(test["tweet"])
test_tweet = pd.DataFrame(test_tweet)
# append cleaned tweets to the training data
test["clean_tweet"] = test_tweet
# compare the cleaned and uncleaned tweets
test.tail()

Test and Train split

from sklearn.model_selection import train_test_split
# extract the labels from the train data
y = train.label.values
# use 70% for the training and 30% for the test
x_train, x_test, y_train, y_test = train_test_split(train.clean_tweet.values, y,
stratify=y,
random_state=1,
test_size=0.3, shuffle=True)

Vectorize tweets using CountVectorizer

CountVectorizer Example

from sklearn.feature_extraction.text import CountVectorizer
documents = ["This is Import Data's Youtube channel",
"Data science is my passion and it is fun!",
"Please subscribe to my channel"]
# initializing the countvectorizer
vectorizer = CountVectorizer()
# tokenize and make the document into a matrix
document_term_matrix = vectorizer.fit_transform(documents)
# check the result
pd.DataFrame(document_term_matrix.toarray(), columns = vectorizer.get_feature_names())

from sklearn.feature_extraction.text import CountVectorizer
# vectorize tweets for model building
vectorizer = CountVectorizer(binary=True, stop_words='english')
# learn a vocabulary dictionary of all tokens in the raw documents
vectorizer.fit(list(x_train) + list(x_test))
# transform documents to document-term matrix
x_train_vec = vectorizer.transform(x_train)
x_test_vec = vectorizer.transform(x_test)

Model building

Apply Support Vetor Classifier (SVC)

from sklearn import svm
# classify using support vector classifier
svm = svm.SVC(kernel = 'linear', probability=True)
# fit the SVC model based on the given training data
prob = svm.fit(x_train_vec, y_train).predict_proba(x_test_vec)
# perform classification and prediction on samples in x_test
y_pred_svm = svm.predict(x_test_vec)

Accuracy score for SVC

from sklearn.metrics import accuracy_score
print("Accuracy score for SVC is: ", accuracy_score(y_test, y_pred_svm) * 100, '%')

References:

Twitter Sentiment Analysis | NLP | Text Analytics