t ts P ALEPlot t t P rt P ts r P ts t r P ts

t ts P ALEPlot 2 2 2 t t P rt P ts r P ts t r P ts t t r 1 t2 1 s r s r s r 1 1 tr s r t r s s rt t r s 2 s t t r r r t s s r t r t 2 t t r r t t2 t s s t t t s t t st 2 t t r t r t r s s t t r t s r t r t rs rst t s ts t r t s s s 2 r t r t s s r 1 t r2 r s t s r r 2 r t t s 3 t s rt t st t r t s rs s t t t 1 s r s r t r 1 t E[Y X = x] f(x) r Y s s r r s s r X = (X 1,...,X d ) s d s t r r t rs f( ) s t tt t t r ts Y r t r t2 t t Y s t rt r ss t ss t s tt s t X tr t t t t s {y i,x i = (x i,1,...,x i,d ) : i = 1,...,n} r s t2 t t t t s2 t r tt 2 f st ˆf s r s X Y t t r r s r s t t s s t r r s t r s t s 3 t t f(x 1,...,x d ) r t r x 1,...,x d s s t r r r t r t ts r t r t rs t tr t t s tt str t ts r t s s d = 2 r d s t 2 s r s s r t r t t r st t tt t s 2 r s 3 t r s t rt P t r s r s 3 t t r t r s 2 x 1 t r s s f( ) P t ts t t f 1,PD (x 1 ) = E[f(x 1,X 2 )] rs s x 1 st t s ˆf 1,PD (x 1 ) = 1 n n f(x 1,x i,2 ) i=1 r str t s f 1,PD (x 1 ) s t t s x 1 = 0.3 r t 2 1 t n = 200 s r t s (X 1,X 2 ) r str t t s t x 2 = x 1 t t t N(0,0.05 2 ) r s t t r t rs s t t r str t s t r t P ts s t t t 1 t t s t t r f(x 1,X 2 ) s X 2 r s r ts r str t s t r

s q t t t r r t t r rt s t r r q r s r t r s r 1tr t 2 t t tr t r t t s r tr f(x) = β 0 + β 1 x 1 + β 2 x 2 2 t rr t r t t s 1tr t t r r 2 t r ts 1 t2 r tr s r s r r r ss tr t 1 t t 1tr t r 2 P t s t t r s 3 t t r t rs r rr t r str t t r s t t t t f 1,PD (x 1 ) f 1,M (x 1 ) t x 1 = 0.3 1tr t r t t s r q r t t f 1,PD (x 1 ) rs s t r s t2 X 2 s ss tr t r t t t t t s t2 X 2 X 1 = x 1 t t str t X 2 X 1 r ts r t r t s t P ts t t s 1tr t 2 s t t s t2 t r s t2 s str t r t t t x 1 s t t t f 1,M (x 1 ) = E[f(X 1,X 2 ) X 1 = x 1 ] rs s x 1 r st t f 1,M (x 1 ) s ˆf 1,M (x 1 ) = 1 n(x 1 ) i N(x 1 ) f(x 1,x i,2 ) r N(x 1 ) {1,2,...,n} s t s s t r s i r x i,1 s t s s r r t 2 s t r x 1 n(x 1 ) s t r s r t s t r t r s st t r s t t s r t2 2 s t st t f 1,M (x 1 ) t s r t r s t r s r s r s r t s f 1,M (x 1 ) t s 3 t t x 1 X 1 X 2 r t 2 s f 1,M (x 1 ) s r r ss Y t X 1 r r 3 r t s r X 2 s q t 2 Y s X 1 X 2 f 1,M (x 1 ) r t t t r ts s q t tt r s r r ss t r 2 Y s X 1 t t X 2 rr t t X 1 X 2 r s t f 1,M (x 2 ) t r s t Y s X 2 r r s s r s r t t s s r t r r s r t 2 rr t P ts t s r 2 s 2 t r r r t rt 2 s ss r s 3 t ts t r t rs

st t t r r t t ts t t ALEPlot P t ts ts r s 2 st t t s 3 t r t t s t t 1tr t r P ts t r ts t t r t r x j j {1,...,d} s s f j,ale (x j ) = xj z 0,j E[ f(x 1,...,X d ) X j ) X j = z j ]dz j c 1 r z 0,j s r 1 t r X j st t c 1 s s s t t f j,ale (X j ) s 3 r t r s t t t r str t X j t t t x j s t st t f j,ale (x j ) rs s x j t s 3 s t t f( ) x j st t t t s t 2 r t t r t s t t r t t t r ˆf j,ale (x) = k j (x) k=1 1 n j (k) i:x i,j N j (k) [f(z k,j,x i,\j ) f(z k 1,j,x i,\j )] ĉ 1 r t t t s s s t st t ĉ 1 s s s t t 1 n n i=1 ˆf j,ale (x i,j ) = 0 t x i,\j = (x i,l : l = 1,...,d;l j) r t s s r t \j t s r s t t jt t {N j (k) = (z k 1,j,z k,j ] : k = 1,2,...,K} s t 2 rt t t s r {x i,j : i = 1,2,...,n} t t r s ALEPlot t s s z k,j s t k K q t t r str t {x i,j : i = 1,2,...,n} t z 0,j s st t s st s r t z K,j s s t r st s r t r k = 1,2,...,K t n j (k) t t r s r t s {x i,j : i = 1,...,n} t t t t kt t r N j (k) s t t K k=1 n j(k) = n r rt r x r t r t r x j t k j (x) t t 1 t t r t x s x (z kj (x) 1,j,z kj (x),j] r str t s t t t t t st t r ˆf j,ale (x j ) r t rst r t r j = 1 r t s d = 2 r t rs 2 t s t t t r t t ss s r t t2 f( ) t t st r r t f( ) s s ss 2 r t st t r ts t t t r s s t st t rs s t r s s t s s s t r t t t r t K s t r t t ALEPlot t s t2 2 s K r 100 t r r s t tt r r s ts t t t t r t 2 st r K t r 2 t r t rs r s r t r t 50 st t s K s 2 s t r t r s r r t r r t rs t q t K s t r t r s s t t 2 t r t r 2

r str t t t t ts t t t st t r ˆf j,ale (x j ) r j = 1 t d = 2 r t rs ts r s tt r t {(x i,1,x i,2 ) : i = 1,2,...,n} r n = 30 tr s r t s r {x i,1 : i = 1,2,...,n} s rt t t K = 5 t r s {N 1 (k) = (z k 1,1,z k,1 ] : k = 1,2,...,5} r t K s s 2 s r r t 5 rs tr s r t s t t 5 t r s r n 1 (1) = 4,n 1 (2) = 6,n 1 (3) = 6,n 1 (4) = 5, n 1 (5) = 9 r 3 t s s ts s t N 1 (4) r r t s ts r ss t t r s f(z 4,j,x i,\j ) f(z 3,j,x i,\j ) r t t r t r s rr s t k = 4 j = 1 ALEPlot s s t s 3 t ts r t rs t r s r r t r t ts t s ss t 1t s t 1 s r s r s t s sts t r r2 t s ALEPlot PDPlot r t ts P ts r s t 2 tt s r s r t tr t s t t t s t t t t t ALEPlot s t yaimpute s r r r t r r t rs r t r st rs t2 r t r r st t ALEPlot t st t yaimpute 1 str t s t s t ALEPlot t s 3 t ts t s t 1 1 s 3 t ts t t t s X = {X 1,X 2,X 3,X 4 } s s t d = 4 r t r r s r X i : i {1, 2, 3, 4} s r str t t t r [0, 1] 4 r t rs r t 1 t rr t r t rs s t r s s t t t r s s r s Y = 4x 1 +3.87x 2 2 +2.97 exp( 5+10x 3) 1+exp( 5+10x 3 ) +ǫ, r ǫ s N(0,1 2 ) str t ts r s s t t t t r t r s r 1 t 2 t s r r t n = 5000 s r t s {y i,x i = (x i,1,...,x i,4 ) : i = 1,...,n} r t s s t tr t s t t t r t r s t t 2 s 2 t 8 s t s 2 r r t t t t t 2 r t r 0.1

s r t rs r s s r 1 t 2 t t r t s 10 r ss t t t tt t ts r t r r t rs s K = 100 r t s t t ts t r t r t s str ts t ts ## R code for Example 1 ## Load relevant packages library(aleplot) library(nnet) ## Generate some data and fit a neural network supervised learning model n = 5000 x1 <- runif(n, min = 0, max = 1) x2 <- runif(n, min = 0, max = 1) x3 <- runif(n, min = 0, max = 1) x4 <- runif(n, min = 0, max = 1) y = 4*x1 + 3.87*x2^2 + 2.97*exp(-5+10*x3)/(1+exp(-5+10*x3))+ rnorm(n, 0, 1) DAT <- data.frame(y, x1, x2, x3, x4) nnet.dat <- nnet(y~., data = DAT, linout = T, skip = F, size = 8, decay = 0.1, maxit = 1000, trace = F) ## Define the predictive function yhat <- function(x.model, newdata) as.numeric(predict(x.model, newdata, type = "raw")) ## Calculate and plot the ALE main effects of x1, x2, x3, and x4 par(mfrow = c(2,2), mar = c(4,4,2,2) + 0.1) ALE.1 = ALEPlot(DAT[,2:5], nnet.dat, pred.fun = yhat, J = 1, K = 100, ALE.2 = ALEPlot(DAT[,2:5], nnet.dat, pred.fun = yhat, J = 2, K = 100, ALE.3 = ALEPlot(DAT[,2:5], nnet.dat, pred.fun = yhat, J = 3, K = 100, ALE.4 = ALEPlot(DAT[,2:5], nnet.dat, pred.fun = yhat, J = 4, K = 100, ## Manually plot the ALE main effects on the same scale for easier ## comparison of the relative importance of the four predictor variables plot(ale.1$x.values, ALE.1$f.values, type="l", xlab="x1", ylab="ale_main_x1", xlim = c(0,1), ylim = c(-2,2), main = "(a)") plot(ale.2$x.values, ALE.2$f.values, type="l", xlab="x2", ylab="ale_main_x2", xlim = c(0,1), ylim = c(-2,2), main = "(b)") plot(ale.3$x.values, ALE.3$f.values, type="l", xlab="x3", ylab="ale_main_x3", xlim = c(0,1), ylim = c(-2,2), main = "(c)") plot(ale.4$x.values, ALE.4$f.values, type="l", xlab="x4", ylab="ale_main_x4", xlim = c(0,1), ylim = c(-2,2), main = "(d)") t t t t rst r t t ALEPlot t s t t r r t r r s 1 t r s s r s t t s r s r s t s r t ALEPlot s t s r s r t r t r s s r s t t t s t r t t r s s r t s r s r t r st s r s r ts r s 2 t r t t t t s r tt s rt t t t ss t t s r t t r t t s t r t t r t s r r t st t t s ss t 1t s t tt s s t r t r s t t s t r t r t r st s ALEPlot s t r t t s 1 s s rt r s st r s t P t st t s t t t r ts t r t r s s t s s

t tt s r t s t t r ts t t s t (a) (b) ALE_main_x1 2 1 0 1 2 ALE_main_x2 2 1 0 1 2 x1 (c) x2 (d) ALE_main_x3 2 1 0 1 2 ALE_main_x4 2 1 0 1 2 x3 r t ts r t tt r t r 1 ˆf 1,ALE (x 1 ) s x 1 ˆf 2,ALE (x 2 ) s x 2 ˆf 3,ALE (x 3 ) s x 3 ˆf 4,ALE (x 4 ) s x 4 r st t ts r t 2 t r t rr t r q r t s r t s s r s t 2 r x 1 x 2 x 3 r r ˆf 4,ALE (x 4 ) rr t 2 t s t t x 4 s t Y x4 r s s t t ts r x 1 x 2 x 3 x 4 r 1 st t s f 1,ALE (x 1 ) f 2,ALE (x 2 ) f 3,ALE (x 3 ) r 2 t r t rr t r q r t s r t s s q t t s t t t st t f 4,ALE (x 4 ) s st 3 r r s t t t t t Y t t X 4 t r r t t t s t t t r s t Y X 4 t s r ts s t s 2 ss ss r rt t ts r t rt

[f(z k,j,z m 1,l,x i,\{j,l} ) f(z k 1,j,z m 1,l,x i,\{j,l} )]] ĝ j (x j ) ĝ l (x l ) ĉ 2 st t t r r t t r r ts t t ALEPlot P s r r t t r t rs {x j,x l } {j,l} {1,...,d} s s f {j,l},ale (x j,x l ) = xl xj z 0,l E[ 2 f(x 1,...,X d ) z 0,j X j X l X j = z j,x l = z l ]dz j dz l g j (x j ) g l (x l ) c 2 r z 0,j z 0,l r r 1 t r s X j X l r s t 2 t s g j (x j ) g l (x l ) t s t s r s x j x l r s t 2 t st t c 2 r t s t tf {j,l},ale (x j,x l ) s 2 t r t s s t tf {j,l},ale (X j,x l ) s 3 r t r s t t t r str t {X j,x l } t ts x j x l f {j,l},ale (X j,x l ) r t 3 r t r t st t f {j,l},ale (x j,x l ) s {x j,x l } s s t t r t t t t t r t rs r str t t t t s t t s r r t st t r ˆf {j,l},ale (x j,x l ) r K = 5 r s {x i,j : i = 1,2,...,n} {x i,l : i = 1,2,...,n} r rt t t 5 t r s t r rt s r t r s t r r t r s {N {j,l} (k,m) = N j (k) N l (m) : k = 1,2,...,5;m = 1,2,...,5} t r rs s t r N {j,l} (4,3) s r r t r s q r (k,m) = (4,3) r t r ss t r rs t s t r s t q t s r s r t r r t n {j,l} (4,3) = 2 s r t s r N {j,l} (4,3) s st t t s r r t {X j,x l } t 2 (x j,x l ) (z 0,j,z K,j ] (z 0,l,z K,l ] ˆf {j,l},ale (x j,x l ) = k j (x j ) k=1 k l (x l ) m=1 1 n {j,l} (k,m) i:x i,{j,l} N {j,l} (k,m) [[f(z k,j,z m,l,x i,\{j,l} ) f(z k 1,j,z m,l,x i,\{j,l} )] r t t s ĝ j (x j ) ĝ l (x l ) t st t ĉ 2 r t t 2 t r t s r r t st t t t t s s s s str t r

rt t t {X j,x l } s t r K 2 r t r s {N {j,l} (k,m) = N j (k) N l (m) : k = 1,2,...,K;m = 1,2,...,K} t s t r ss r t t t r s t s rt t s t t n {j,l} (k,m) t t r tr s r t s t t t N {j,l} (k,m) s t t K K k=1 m=1 n {j,l}(k,m) = n r s t (x j,x l ) t {X j,x l } s t k j (x j ) k l (x l ) t t s t t r s t x j x l r s t 2 (x j,x l ) N {j,l} (k j (x j ),k l (x l )) 1 2 1 2 t r t t r t t tr str t t t s 3 t s r r t r t ts s t ALEPlot X = {X 1,X 2,X 3,X 4 } r r t r t r r s t X j U[0,1] t tr r s s s r t s Y = 4x 1 +3.87x 2 2 +2.97 exp( 5+10x 3) 1+exp( 5+10x 3 ) +13.86(x 1 0.5)(x 2 0.5)+ǫ, r ǫ N(0,1) t ts r s s t t t r t r s r 1 t 2 t s r r t n = 5000 s r t s r t r t r t r s t t 2 s 2 t 6 s t s 2 r r t t t t t 2 r t r 0.1 r r 1 t 2 t r t 10 r ss t s K = 500 r t t ts K = 100 r t s r r t ts s t t r t t ## R code for Example 2 ## Load relevant packages library(aleplot) library(nnet) ## Generate some data and fit a neural network supervised learning model n = 5000 x1 <- runif(n, min = 0, max = 1) x2 <- runif(n, min = 0, max = 1) x3 <- runif(n, min = 0, max = 1) x4 <- runif(n, min = 0, max = 1) y = 4*x1 + 3.87*x2^2 + 2.97*exp(-5+10*x3)/(1+exp(-5+10*x3))+ 13.86*(x1-0.5)*(x2-0.5)+ rnorm(n, 0, 1) DAT <- data.frame(y, x1, x2, x3, x4) nnet.dat <- nnet(y~., data = DAT, linout = T, skip = F, size = 6, decay = 0.1, maxit = 1000, trace = F) ## Define the predictive function yhat <- function(x.model, newdata) as.numeric(predict(x.model, newdata, type = "raw")) ## Calculate and plot the ALE main effects of x1, x2, x3, and x4 ALE.1 = ALEPlot(DAT[,2:5], nnet.dat, pred.fun = yhat, J = 1, K = 500, ALE.2 = ALEPlot(DAT[,2:5], nnet.dat, pred.fun = yhat, J = 2, K = 500, ALE.3 = ALEPlot(DAT[,2:5], nnet.dat, pred.fun = yhat, J = 3, K = 500, ALE.4 = ALEPlot(DAT[,2:5], nnet.dat, pred.fun = yhat, J = 4, K = 500, ## Calculate and plot the ALE second-order effects of {x1, x2} and {x1, x4} ALE.12 = ALEPlot(DAT[,2:5], nnet.dat, pred.fun = yhat, J = c(1,2), K = 100, ALE.14 = ALEPlot(DAT[,2:5], nnet.dat, pred.fun = yhat, J = c(1,4), K = 100,

## Manually plot the ALE main effects on the same scale for easier comparison ## of the relative importance of the four predictor variables par(mfrow = c(3,2)) plot(ale.1$x.values, ALE.1$f.values, type="l", xlab="x1", ylab="ale_main_x1", xlim = c(0,1), ylim = c(-2,2), main = "(a)") plot(ale.2$x.values, ALE.2$f.values, type="l", xlab="x2", ylab="ale_main_x2", xlim = c(0,1), ylim = c(-2,2), main = "(b)") plot(ale.3$x.values, ALE.3$f.values, type="l", xlab="x3", ylab="ale_main_x3", xlim = c(0,1), ylim = c(-2,2), main = "(c)") plot(ale.4$x.values, ALE.4$f.values, type="l", xlab="x4", ylab="ale_main_x4", xlim = c(0,1), ylim = c(-2,2), main = "(d)") ## Manually plot the ALE second-order effects of {x1, x2} and {x1, x4} image(ale.12$x.values[[1]], ALE.12$x.values[[2]], ALE.12$f.values, xlab = "x1", ylab = "x2", main = "(e)") contour(ale.12$x.values[[1]], ALE.12$x.values[[2]], ALE.12$f.values, add=true, drawlabels=true) image(ale.14$x.values[[1]], ALE.14$x.values[[2]], ALE.14$f.values, xlab = "x1", ylab = "x4", main = "(f)") contour(ale.14$x.values[[1]], ALE.14$x.values[[2]], ALE.14$f.values, add=true, drawlabels=true) r s s t t ts x 1 x 4 r s s t s r r t ts r {x 1,x 2 } {x 1,x 4 } r s t 2 r r s s t t t t t t t r t t r t t ts st t r t rr t r q r t s st t 3 r r t s s r2 r r s s t t t t r t t x 1 x 2 s s t t t r t t x 1 x 4 s st 2 t t ˆf {1,2},ALE (x 1,x 2 ) s s r t t t r t 2 r r t t ˆf{1,4},ALE (x 1,x 4 ) s s t t t r t t 3 r t t t t t s t t r s s t r s s r s ts r t t t t t t tr s str t r t t x 1 x 2 t r t t x 1 x 4 r t r r t t t {x 1,x 2 } t r t t s r t f x 2 x 1 s 1 r r 1 s 2 x 1 = 0.2 r s x 2 r s s ˆf {1,2},ALE (x 1,x 2 ) 1 s 2 x 1 = 0.8 r s x 2 r s s ˆf {1,2},ALE (x 1,x 2 ) s r t s t t {x 1,x 2 } s t r r t t r t s rs s t t 2 t s r r ts 3 r ts s t tt r s s tr t r t r r t t r r r t rst t f x 2 r r t 1 s x 1 s t t t ˆf 2,ALE (x 2 )+ ˆf {1,2},ALE (x 1,x 2 ) rs s x 2 t t x 2 t t t r t t {x 1,x 2 } r 1 x 1 = 0.2 t tr t x 2 Y s r 3.87x 2 2 4.16x 2 +constant r 1 x 1 = 0.8 t tr t x 2 Y s str r 3.87x 2 2 +4.16x 2 +constant r s q t s 2 t t st t ˆf 2,ALE (x 2 ) + ˆf {1,2},ALE (x 1,x 2 ) t x 1 = 0.2 x 1 = 0.8

(a) (b) ALE_main_x1 2 1 0 1 2 ALE_main_x2 2 1 0 1 2 x1 x2 (c) (d) ALE_main_x3 2 1 0 1 2 ALE_main_x4 2 1 0 1 2 x3 x4 (e) (f) x2 x4 x1 x1 r t ts ˆf 1,ALE (x 1 ) ˆf 2,ALE (x 2 ) ˆf 3,ALE (x 3 ) ˆf 4,ALE (x 4 ) s r r t ts ˆf {1,2},ALE (x 1,x 2 ) ˆf {1,4},ALE (x 1,x 4 ) r t tt r t r 1 r s t r t rr t r q r t s st t 3 r r t s s r x 1 x 4 r r s s t t r t t x 1 x 2 r s s t t t t r t t x 1 x 4 s

2 s ss s t r rt s r t r st s ts 2 s t 1 t t s s t r 1 t t t t s ts ts t 1tr t r P ts t r r r 2 r t t t rt r t r t r t t t f ts ts t s r r t r t t r rs P ts tt s r rst t ts t r t rs t t r t 1tr t r s ts 2 t2 t s ss r rt2 r t r t r t rs t t s ss r rt2 r t s s ts r t rs st s P ts ts r s r ss t t 2 1 s t P ts t t t 1 s s t t K 2 t r r r r t r t ts st t s r 2 s ts s r r t r t ts r t ALEPlot s t s r t s t 2 r ss r t r t t t r r t t ts s r r t r t ts 2 r t s r 1 1 t t t r 1 s r 2 r 3 rs s r t r s s r t s t r t r r r t r r s t r t t s s t r t rs t2 r r r r s t r2 t tt r s t s ts s s 1 t 1 r r t s r s ts s t r rs s rst t t t t r t s t s s r r n = 30,162 s s t tr t s t t r r s s t ss t s r r s ts rs r s s s t r2 t r r t t r rs r r t $50 1994 d = 12 r t r r s t t s r x 1 r r ss x 2 t r t 8 t r s t x 3 tr t s r r s st t r t t r t t r t r t r t r t r s r t s t ss t s r ss t s r r s r st r s r r ss r t r t r r t st t s x 4 t r t t r s t x 5 t r t t r s r t s st t s x 6 t r t t r s r x 7 t r t t r s s 1 x 8 t r t t r s t s x 9 r t ss x 10 r rs r s t r x 11 r t tr2 x 12 t r t t r s t t t t r s t r t r rt r t rs t s t st tr s t 2 2 t tr t s r t rs t r t rs s r 0.02 t r t t 3 r t t r tr s t r r ss t s s f(x) r str t ts s t s t r t r t2 t t rs s r $50k rs r t tt st tr r s s t t ts r t t rs r r t rs t s r r t r t t f {1,11},ALE (x 1,x 11 ) r 4 rs r 6 s K = 500 r t ts ts K = 50 r t t r t t s t r t t ts

## R code for Example 3 ## Load relevant packages library(aleplot) library(gbm) ## Read data and fit a boosted tree supervised learning model data = read.csv("adult_data.csv", header = TRUE, strip.white = TRUE, na.strings = "?") data = na.omit(data) gbm.data <- gbm(income==">50k" ~., data= data[,-c(3,4)], distribution = "bernoulli", n.trees=6000, shrinkage=0.02, interaction.depth=3) ## Define the predictive function; note the additional arguments for the ## predict function in gbm yhat <- function(x.model, newdata) as.numeric(predict(x.model, newdata, n.trees = 6000, type="link")) ## Calculate and plot the ALE main and interaction effects for x_1, x_3, ## x_11, and {x_1, x_11} par(mfrow = c(2,2), mar = c(4,4,2,2)+ 0.1) ALE.1=ALEPlot(data[,-c(3,4,15)], gbm.data, pred.fun=yhat, J=1, K=500, ALE.3=ALEPlot(data[,-c(3,4,15)], gbm.data, pred.fun=yhat, J=3, K=500, ALE.11=ALEPlot(data[,-c(3,4,15)], gbm.data, pred.fun=yhat, J=11, K=500, ALE.1and11=ALEPlot(data[,-c(3,4,15)], gbm.data, pred.fun=yhat, J=c(1,11), K=50, NA.plot = FALSE) r t r r t t t r s ts t ts ts r r t r r t t s r t2 r r t r 2 r s s t t t s r 2 rs t r 2 s t 2 r s s t t t t r st s rr r ss t s t r s r r s t st r s r st r s t P Pr ss r s t 2 r s s t rs r r t t rs r t t st st r s s t r 2 30 50 rs r s r r 4 rs r 6 t r t t r s r s t r st r t s s r t r s r t2 r r t t t s ss t t r s rs r r t r t t r t t t t t t t s r t2 r s s s r 2 r s t r s r t2 r t rs r s r r t r 2 r s s ˆf {1,11},ALE (x 1,x 11 ) r s s 2 ts r 2 r s t r s s 2 ts r 2 r s r t rs r P r s t s s s 2 r s r s 2 rs 2 r t s r r s s t 2 r s r r s t r r t r 1 t s r t r r t r r t t t r t ts s r r 2 t ˆf {1,11},ALE (x 1,x 11 ) s x 1 r x 11 ts s t 2 r s tr t r t s t t t t ˆf {1,11},ALE (x 1,x 11 ) r s s 2 r 2 r s r t rs r s t 2 t t s r s rs r s ss t t r s t r t2 2 r r r t t s st t t r ˆf 11,ALE (x 11 )+ ˆf {1,11},ALE (75,x 11 ) r s s r r s s r x 11 = 35 t x 11 = 80 rs r r r t s st r s s 2 t ts r

2 r s s t 2 r s rs r r s ss t t r s t r t2 r r t (a) (b) f_1(x_1) 1.0 0.5 0.0 0.5 f_3(x_3) 0.5 0.0 0.5 1.0 1.5 20 40 60 80 x_1 (x1) (c) 5 10 15 x_3 (x5) (d) f_11(x_11) 1.0 0.5 0.0 0.5 1.0 x_11 (x13) 20 40 60 80 0 20 40 60 80 100 20 30 40 50 60 70 80 90 x_11 (x13) x_1 (x1) r r t t 1 s t st tr s s f(x) t ts r t rs r t s tt t s r r t r t t r 4 rs r 6 tt r t r t s t t r t t t t2 s t t tr s r t s r P P ts 1 r P P ts r s ss t t s ts t r t t tt t 2 t s r r t 1tr t r r s r t t 2 s r t 1 st s 3 t t s t s s t s 1 r ts r r P ts r s t 1tr t r 1 t 1 t t r t rs

s X = {X 1,X 2 } s r str t t x 2 = x 1 t t N(0,0.05 2 ) r s t t r t rs tr r s s s r t s Y = X 1 +X 2 2 +ǫ t ǫ N(0,0.12 ) r t n = 200 s r t s r t s t r t r s t t 2 s 2 t 10 s t s 2 r r t t t t t 2 r t r 0.0001 s r t rs r s s r 1 t 2 t t r t s 10 r ss t s t r t P ts ts r x 1 x 2 ## R code for Example 4 ## Load relevant packages library(aleplot) library(nnet) ## Generate some data and fit a neural network supervised learning model n = 200 x <- runif(n, min = 0, max = 1) x1 <- x + rnorm(n, 0, 0.05) x2 <- x + rnorm(n, 0, 0.05) y = x1 + x2^2 + rnorm(n, 0, 0.1) DAT = data.frame(y, x1, x2) nnet.dat <- nnet(y ~., data = DAT, linout = T, skip = F, size = 10, decay = 0.0001, maxit = 1000, trace = F) ## Define the predictive function yhat <- function(x.model, newdata) as.numeric(predict(x.model, newdata, type= "raw")) ## Calculate and plot the ALE and PD main effects of x1 and x2 par(mfrow = c(2,2), mar = c(4,4,2,2) + 0.1) ALE.1 = ALEPlot(DAT[,2:3], nnet.dat, pred.fun = yhat, J = 1, K = 50, PD.1 = PDPlot(DAT[,2:3], nnet.dat, pred.fun = yhat, J = 1, K = 50) ALE.2 = ALEPlot(DAT[,2:3], nnet.dat, pred.fun = yhat, J = 2, K = 50, PD.2 = PDPlot(DAT[,2:3], nnet.dat, pred.fun = yhat, J = 2, K = 50) ## Manually plot the ALE main effects on the same scale for easier ## comparison of the relative importance of the four predictor variables ## We also plot the true linear and quadratic effects in black for reference plot(ale.1$x.values, ALE.1$f.values, type="l", xlab="x1", ylab="ale_main_x1", xlim = c(0,1), ylim = c(-1,1), col = "blue", main = "(a)") curve(x - 0.5, from = 0, to = 1, add = TRUE) plot(pd.1$x.values, PD.1$f.values, type="l", xlab="x2", ylab="pd_x1", xlim = c(0,1), ylim = c(-1,1), col = "blue", main = "(b)") curve(x - 0.5, from = 0, to = 1, add = TRUE) plot(ale.2$x.values, ALE.2$f.values, type="l", xlab="x3", ylab="ale_main_x2", xlim = c(0,1), ylim = c(-1,1), col = "blue", main = "(c)") curve(x^2 - (1/3+0.05^2), from = 0, to = 1, add = TRUE) plot(pd.2$x.values, PD.2$f.values, type="l", xlab="x4", ylab="pd_x2", xlim = c(0,1), ylim = c(-1,1), col = "blue", main = "(d)") curve(x^2 - (1/3+0.05^2), from = 0, to = 1, add = TRUE) r s ts r t2 r t t 1 1 r t r s r r s t t t ts tt x 1 x 2 t s r s r t t tr r q r t ts tt t r t P ts r t s r s r t r r 2 t P ts s str t r 2 t P t r q r s 1tr t f r ts t t {x 1,x 2 } tr t r s t 1tr t f s r2 r t

(a) (b) ALE_main_x1 1.0 0.5 0.0 0.5 1.0 PD_x1 1.0 0.5 0.0 0.5 1.0 x1 (c) x2 (d) ALE_main_x2 1.0 0.5 0.0 0.5 1.0 PD_x2 1.0 0.5 0.0 0.5 1.0 x3 r r s ˆf 1,ALE (x 1 ) ˆf 1,PD (x 1 ) ˆf 2,ALE (x 2 ) ˆf 2,PD (x 2 ) r r t r s tt r r 3 t t 1 t t r s t tr t t r r X 1 q r t r X 2 t r s r t st t t t s r ts t s P ts r t s x4 1 r s ts s r r2 t t 1 r t s r t t t r r t s t 1 1 r t r r t r t s t n = 200 s r t s r r t r t s str t r t r t t s t r t rs s t r s ts r r t s r r r t ts ts t s r r s r t t tr r q r t ts r X 1 X 2 r s t 2 t r t P ts r t s

r r s ˆf 1,ALE (x 1 ) ˆf 1,PD (x 1 ) ˆf 2,ALE (x 2 ) ˆf 2,PD (x 2 ) r r t r s tt r t r r t s t 1 t t r s t tr t t r r X 1 q r t r X 2 t r 2 r s r t st t t t s r t 50 t r r t s

r s 2 s 3 t ts r t r r s 1 s r s r s s tt r t r r 2 t r 1 t r t st s t t st s 2 t tr t s r t rs r 3 st r r ss s r rs http://cran.r-project.org/package=gbm s 2 r t t st s t rt t r r r