Task1_part2.R

library(readr)
library(dplyr)
library(fastDummies)
library(caret)
library(e1071)
library(rpart)
library(xgboost)
library(pROC)
evaluate_metrics <- function(true_labels, predicted_labels, positive_class = 1) {
  # Convert inputs to factors to handle any data type
  true_labels <- factor(true_labels)
  predicted_labels <- factor(predicted_labels, levels = levels(true_labels))
  
  # True Positives (TP): predicted = positive & true = positive
  TP <- sum(predicted_labels == positive_class & true_labels == positive_class)
  # False Positives (FP): predicted = positive & true != positive
  FP <- sum(predicted_labels == positive_class & true_labels != positive_class)
  # False Negatives (FN): predicted != positive & true = positive
  FN <- sum(predicted_labels != positive_class & true_labels == positive_class)
  
  # Recall = TP / (TP + FN)
  recall <- ifelse((TP + FN) == 0, NA, TP / (TP + FN))
  # Precision = TP / (TP + FP)
  precision <- ifelse((TP + FP) == 0, NA, TP / (TP + FP))
  # F1 = 2 * (Precision * Recall) / (Precision + Recall)
  f1 <- ifelse(is.na(precision) | is.na(recall) | (precision + recall) == 0, NA,
               2 * precision * recall / (precision + recall))
  
  return(list(Recall = recall, Precision = precision, F1 = f1))
}


evaluate_model_auc <- function(probabilities, true_labels, positive_class = "1") {
  # Ensure labels are factors with correct levels
  true_labels <- factor(true_labels, levels = c("0", "1"))

  # Compute ROC and AUC
  roc_obj <- roc(response = true_labels, predictor = probabilities,
                 levels = c("0", "1"), direction = "<", quiet = TRUE)

  # Plot ROC curve
  plot(roc_obj, main = paste("ROC Curve - AUC =", round(auc(roc_obj), 4)),
       col = "blue", lwd = 2)

  # Return and print AUC
  auc_value <- auc(roc_obj)
  cat("AUC:", round(auc_value, 4), "\n")
  return(auc_value)
}

original_df <- read_csv("original_df.csv")
unique_tweet <- read_csv("unique_tweets_data.csv")

### Joining tokens with original dataframe to get the models with the anotator information
data_with_anotator <- original_df %>%
  inner_join(unique_tweet, by = "id_EXIST")

data_with_anotator <-  data_with_anotator %>%  select(-tweet, -label_task1_1.y, -mentions, -hashtags, -emojis, -id_EXIST, -annotator_id)

set.seed(123)

# One hot encode data 
data_with_anotator <- fastDummies::dummy_cols(data_with_anotator, select_columns = c("age", "country", "ethnicity", "education", "gender", "continent", "label_task1_1.x"), remove_selected_columns = TRUE, remove_first_dummy = TRUE)
sapply(train_data, class)
# ------------------------
# 8. Prepare Labels and Partition Data
# ------------------------
labels <- as.factor(data_with_anotator$label_task1_1.x_YES)
labels_nr <- as.numeric(data_with_anotator$label_task1_1.x_YES)

table(labels)
table(labels_nr)
table(data_with_anotator$label_task1_1.x_YES)

data_with_anotator <-  data_with_anotator %>%  select(-label_task1_1.x_YES)
colnames(data_with_anotator)
set.seed(123)
train_index <- createDataPartition(labels, p = 0.8, list = FALSE)

train_data <- data_with_anotator[train_index, ]
train_labels <- labels[train_index]
test_data <- data_with_anotator[-train_index, ]
test_labels <- labels[-train_index]

sum(is.na(train_data))

# ------------------------
# 9. Train and Evaluate Model
# ------------------------
# Support Vector Machine (linear)
print('Support Vector Machine (linear)')
svm_model <- svm(train_data, y = train_labels, kernel = "linear", probability = TRUE)

# Predict on test data
svm_pred <- predict(svm_model, test_data, probability = TRUE)
svm_prob <- attr(svm_pred, "probabilities")[, "1"]

# Evaluate
cm = confusionMatrix(svm_pred, test_labels, positive = "1")

evaluate_model_auc(svm_prob, test_labels)
precision <- cm$byClass["Precision"]
recall <- cm$byClass["Recall"]
f1 <- cm$byClass["F1"]
cat("Precision:", precision, "\nRecall:", recall, "\nF1-score:", f1, "\n")

# ------------------------
# Confusion Matrix and Statistics
# 
#           Reference
# Prediction    0    1
#          0 1755  623
#          1  295  770
#                                           
#                Accuracy : 0.7334          
#                  95% CI : (0.7183, 0.7481)
#     No Information Rate : 0.5954          
#     P-Value [Acc > NIR] : < 2.2e-16       
#                                           
#                   Kappa : 0.4249          
#                                           
#  Mcnemar's Test P-Value : < 2.2e-16       
#                                           
#             Sensitivity : 0.5528          
#             Specificity : 0.8561          
#          Pos Pred Value : 0.7230          
#          Neg Pred Value : 0.7380          
#              Prevalence : 0.4046          
#          Detection Rate : 0.2236          
#    Detection Prevalence : 0.3093          
#       Balanced Accuracy : 0.7044          
#                                           
#        'Positive' Class : 1 

# ------------------------
# 10. Decision Tree and Evaluate Model
# ------------------------
print('Decision Tree')
tree_model <- rpart(train_labels ~ ., data = as.data.frame(train_data), method = "class")
predictions <- predict(tree_model, newdata = as.data.frame(test_data), type = "class")
probabilities <- predict(tree_model, newdata = as.data.frame(test_data), type = "prob")[, "1"]
cm = confusionMatrix(predictions, test_labels, positive = "1")
evaluate_model_auc(probabilities, test_labels)
precision <- cm$byClass["Precision"]
recall <- cm$byClass["Recall"]
f1 <- cm$byClass["F1"]
cat("Precision:", precision, "\nRecall:", recall, "\nF1-score:", f1, "\n")
# ------------------------
# Confusion Matrix and Statistics
# 
#           Reference
# Prediction    0    1
#          0 1772  689
#          1  278  704
#                                           
#                Accuracy : 0.7191          
#                  95% CI : (0.7038, 0.7341)
#     No Information Rate : 0.5954          
#     P-Value [Acc > NIR] : < 2.2e-16       
#                                           
#                   Kappa : 0.3881          
#                                           
#  Mcnemar's Test P-Value : < 2.2e-16       
#                                           
#             Sensitivity : 0.5054          
#             Specificity : 0.8644          
#          Pos Pred Value : 0.7169          
#          Neg Pred Value : 0.7200          
#              Prevalence : 0.4046          
#          Detection Rate : 0.2045          
#    Detection Prevalence : 0.2852          
#       Balanced Accuracy : 0.6849          
#                                           
#        'Positive' Class : 1               
#                                           
# ------------------------

# ------------------------
# 11. Logistic Regression
# ------------------------
print('Logistic Regression')
# Levels 0,1 
train_labels_lr <- factor(train_labels, levels = c(0, 1))
test_labels_lr <- factor(test_labels, levels = c(0, 1))

train_df <- as.data.frame(train_data)
train_df$label <- train_labels_lr

# Remove columns
cols_to_remove <- grep("label_task1", names(train_df), value = TRUE)
train_df <- train_df[, !(names(train_df) %in% cols_to_remove)]

# Adding label
train_df$label <- train_labels_lr

# TTraining model
logit_model <- glm(label ~ ., data = train_df, family = "binomial")

# Prepare datframe test
test_df <- as.data.frame(test_data)
test_df <- test_df[, !(names(test_df) %in% cols_to_remove)]

#  Making the prediction
probabilities <- predict(logit_model, newdata = test_df, type = "response")

# Convert the probabilities into classes
predictions <- ifelse(probabilities > 0.5, 1, 0)
predictions <- factor(predictions, levels = c(0, 1))

# Model evaluation
cm = confusionMatrix(predictions, test_labels_lr,  positive = "1")
evaluate_model_auc(probabilities, test_labels_lr)
precision <- cm$byClass["Precision"]
recall <- cm$byClass["Recall"]
f1 <- cm$byClass["F1"]
cat("Precision:", precision, "\nRecall:", recall, "\nF1-score:", f1, "\n")
# ------------------------
# Confusion Matrix and Statistics
# 
#           Reference
# Prediction    0    1
#          0 1792  688
#          1  258  705
#                                         
#                Accuracy : 0.7252        
#                  95% CI : (0.71, 0.7401)
#     No Information Rate : 0.5954        
#     P-Value [Acc > NIR] : < 2.2e-16     
#                                         
#                   Kappa : 0.4           
#                                         
#  Mcnemar's Test P-Value : < 2.2e-16     
#                                         
#             Sensitivity : 0.5061        
#             Specificity : 0.8741        
#          Pos Pred Value : 0.7321        
#          Neg Pred Value : 0.7226        
#              Prevalence : 0.4046        
#          Detection Rate : 0.2048        
#    Detection Prevalence : 0.2797        
#       Balanced Accuracy : 0.6901        
#                                         
#        'Positive' Class : 1  
# ------------------------

# ------------------------
# 11. XGBoost
# ------------------------

print('XGBoost')
# Train and test data
set.seed(123)
train_index <- createDataPartition(labels_nr, p = 0.8, list = FALSE)
train_data <- data_with_anotator[train_index, ]
train_labels <- labels_nr[train_index]
test_data <- data_with_anotator[-train_index, ]
test_labels <- labels_nr[-train_index]

# Criation of DMatrix
dtrain <- xgb.DMatrix(data = as.matrix(train_data), label = train_labels)
dtest <- xgb.DMatrix(data = as.matrix(test_data), label = test_labels)

# Paramters
params <- list(
  objective = "binary:logistic",
  eval_metric = "logloss",
  eta = 0.1,
  max_depth = 6,
  verbosity = 1
)

# Train model
xgb_model <- xgb.train(
  params = params,
  data = dtrain,
  nrounds = 100,
  watchlist = list(train = dtrain, eval = dtest),
  early_stopping_rounds = 10,
  verbose = 1
)

# Evalauation
xgb_pred_prob <- predict(xgb_model, newdata = dtest)
xgb_pred_class <- ifelse(xgb_pred_prob > 0.5, 1, 0)

cm = confusionMatrix(
  factor(xgb_pred_class, levels = c(0, 1)),
  factor(test_labels, levels = c(0, 1)),  positive = "1"
)
evaluate_model_auc(xgb_pred_prob, test_labels)

precision <- cm$byClass["Precision"]
recall <- cm$byClass["Recall"]
f1 <- cm$byClass["F1"]
cat("Precision:", precision, "\nRecall:", recall, "\nF1-score:", f1, "\n")
# ------------------------
# Confusion Matrix and Statistics
# 
#           Reference
# Prediction    0    1
#          0 1801  654
#          1  274  715
#                                           
#                Accuracy : 0.7305          
#                  95% CI : (0.7154, 0.7453)
#     No Information Rate : 0.6025          
#     P-Value [Acc > NIR] : < 2.2e-16       
#                                           
#                   Kappa : 0.4096          
#                                           
#  Mcnemar's Test P-Value : < 2.2e-16       
#                                           
#             Sensitivity : 0.5223          
#             Specificity : 0.8680          
#          Pos Pred Value : 0.7230          
#          Neg Pred Value : 0.7336          
#              Prevalence : 0.3975          
#          Detection Rate : 0.2076          
#    Detection Prevalence : 0.2872          
#       Balanced Accuracy : 0.6951          
#                                           
#        'Positive' Class : 1   
# ------------------------