Decision Tree Regression With Hyper Parameter Tuning

import pandas as pd
import numpy as np

# Reading our csv data
combine_data= pd.read_csv('data/Real_combine.csv')
combine_data.head(5)

combine_data.drop(['Unnamed: 0'],axis=1,inplace=True)

combine_data.head(2)

# combine data top 5 rows
combine_data.head()

# combine data bottom 5 features
combine_data.tail()

# To get statistical data 
combine_data.describe()

combine_data.isnull().sum()

import seaborn as sns
sns.heatmap(combine_data.isnull(),yticklabels=False)

# To check outliers 
import matplotlib.pyplot as plt


a4_dims = (11.7, 8.27)
fig, ax = plt.subplots(figsize=a4_dims)
g = sns.boxplot(data=combine_data,linewidth=2.5,ax=ax)
g.set_yscale("log")

sns.pairplot(combine_data)

combine_data.corr()

relation =combine_data.corr()
relation_index=relation.index

relation_index

sns.heatmap(combine_data[relation_index].corr(),annot=True)

from sklearn.ensemble import RandomForestRegressor
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_error as mse

X_train, X_test, y_train, y_test = train_test_split(
    combine_data.iloc[:,:-1],
    combine_data.iloc[:,-1],
    test_size=0.3,
    random_state=0)

# size of train data set
X_train.shape

# size of test data set
X_test.shape

# Feature selection by ExtraTreesRegressor(model based)

from sklearn.ensemble import ExtraTreesRegressor
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score as acc

X_train, X_test, y_train, y_test = train_test_split(
    combine_data.iloc[:,:-1],
    combine_data.iloc[:,-1],
    test_size=0.3,
    random_state=0)

reg= ExtraTreesRegressor()

reg.fit(X_train,y_train)

Let us print the features importance.

reg.feature_importances_

feat_importances = pd.Series(reg.feature_importances_, index=X_train.columns)
feat_importances.nlargest(5).plot(kind='barh')
plt.show()

# Traning model with all features 

from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(combine_data.iloc[:,:-1], combine_data.iloc[:,-1], test_size=0.3, random_state=0)

X_train

X_test

from sklearn.tree import DecisionTreeRegressor

reg_decision_model=DecisionTreeRegressor()

# fit independent varaibles to the dependent variables
reg_decision_model.fit(X_train,y_train)

reg_decision_model.score(X_train,y_train)

reg_decision_model.score(X_test,y_test)

prediction=reg_decision_model.predict(X_test)

# checking difference between labled y and predicted y
sns.distplot(y_test-prediction)

checking predicted y and labeled y using a scatter plot.

plt.scatter(y_test,prediction)

# Hyper parameters range intialization for tuning 

parameters={"splitter":["best","random"],
            "max_depth" : [1,3,5,7,9,11,12],
           "min_samples_leaf":[1,2,3,4,5,6,7,8,9,10],
           "min_weight_fraction_leaf":[0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9],
           "max_features":["auto","log2","sqrt",None],
           "max_leaf_nodes":[None,10,20,30,40,50,60,70,80,90] }

# calculating different regression metrics

from sklearn.model_selection import GridSearchCV

tuning_model=GridSearchCV(reg_decision_model,param_grid=parameters,scoring='neg_mean_squared_error',cv=3,verbose=3)

# function for calculating how much time take for hyperparameter tuning

def timer(start_time=None):
    if not start_time:
        start_time=datetime.now()
        return start_time
    elif start_time:
        thour,temp_sec=divmod((datetime.now()-start_time).total_seconds(),3600)
        tmin,tsec=divmod(temp_sec,60)
        #print(thour,":",tmin,':',round(tsec,2))

X=combine_data.iloc[:,:-1]

y=combine_data.iloc[:,-1]

%%capture
from datetime import datetime

start_time=timer(None)

tuning_model.fit(X,y)

timer(start_time)

# best hyperparameters 
tuning_model.best_params_

# best model score
tuning_model.best_score_

tuned_hyper_model= DecisionTreeRegressor(max_depth=5,max_features='auto',max_leaf_nodes=50,min_samples_leaf=2,min_weight_fraction_leaf=0.1,splitter='random')

# fitting model


tuned_hyper_model.fit(X_train,y_train)

# prediction 

tuned_pred=tuned_hyper_model.predict(X_test)

plt.scatter(y_test,tuned_pred)

# With hyperparameter tuned 

from sklearn import metrics

print('MAE:', metrics.mean_absolute_error(y_test,tuned_pred))
print('MSE:', metrics.mean_squared_error(y_test, tuned_pred))
print('RMSE:', np.sqrt(metrics.mean_squared_error(y_test, tuned_pred)))

# without hyperparameter tuning 

from sklearn import metrics

print('MAE:', metrics.mean_absolute_error(y_test,prediction))
print('MSE:', metrics.mean_squared_error(y_test, prediction))
print('RMSE:', np.sqrt(metrics.mean_squared_error(y_test, prediction)))