Plotting lift curve in MLR

Question

I would like to know how to plot lift curves in MLR especially for a Benchmark experiment with multiple algorithms and tasks. Help with ROC curve plotting will also be appreciated. Thanks.

Answer 1

I am not a mlr user but here is a general way. First some data:

Two class problem

iris2 = iris[iris$Species!="setosa",]
iris2$Species = factor(iris2$Species)

1st model:

log_model = glm(Species~., data = iris2, family = "binomial")
prob = predict(log_model, iris2,  type = "response") #get the logistic regression prob

2nd model:

library(e1071)
svm_model = svm(Species~., data = iris2, probability = TRUE)
prob_svm = predict(svm_model, iris2, probability = TRUE)
prob_svm  = attr(prob_svm , "probabilities")[,2] #get the probability for svm model

make a data frame from classes (1/0 coding) and additional columns for predicted probabilities for each model

for_lift = data.frame(Class = as.factor(ifelse(iris2$Species == "versicolor", 1, 0)),  glm = prob, svm = prob_svm)

make a lift object

library(caret)
lift_obj = lift(Class ~ glm+svm, data = for_lift)

xyplot(lift_obj, auto.key = list(columns = 2,
                                            lines = TRUE,
                                            points = FALSE))

You can use the same data frame to plot ROC curves

library(pROC)

plot(pROC::roc(response = for_lift$Class,
               predictor = for_lift$glm,
               levels=c(0, 1)),
     lwd=1.5) 

plot(
  pROC::roc(response = for_lift$Class,
            predictor = for_lift$svm ,
            levels=c(0, 1)), 
  add=T, lty=2, lwd=1.5)
legend(0.9, 0.9, c("logistic", "svm"), lty = c(1,2))

You can also check the ROCR package: https://cran.r-project.org/web/packages/ROCR/ROCR.pdf it has methods to plot both types of plots

Additionally if you are a ggplot2 user you can use the lift_obj to plot lift and ROC curves with it also.

library(ggplot2)


 p1 = ggplot(lift_obj$data)+
  geom_line(aes(CumTestedPct, CumEventPct, color = liftModelVar))+
  xlab("% Samples tested")+
  ylab("% Samples found")+
  scale_color_discrete(guide = guide_legend(title = "method"))+
  geom_polygon(data = data.frame(x = c(0, lift_obj$pct, 100, 0),
                              y = c(0, 100, 100, 0)),
                              aes(x = x, y = y), alpha = 0.1)

p2 = ggplot(lift_obj$data)+
  geom_line(aes(1-Sp , Sn, color = liftModelVar))+
  scale_color_discrete(guide = guide_legend(title = "method"))
library(cowplot)
plot_grid(p1, p2, labels=c("lift", "ROC"))