How to optimise learning rates in XGBoost example 1?
0

How to optimise learning rates in XGBoost example 1?

This recipe helps you optimise learning rates in XGBoost example 1
In [2]:
def Snippet_193():
    print()
    print(format('How to optimise learning rates in XGBoost','*^82'))

    import warnings
    warnings.filterwarnings("ignore")

    # load libraries
    from sklearn import datasets
    from sklearn.model_selection import train_test_split
    from xgboost import XGBClassifier
    from sklearn.model_selection import GridSearchCV
    from sklearn.model_selection import StratifiedKFold
    import matplotlib
    matplotlib.use('Agg')
    from matplotlib import pyplot

    # load the iris datasets
    dataset = datasets.load_wine()
    X = dataset.data; y = dataset.target
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25)

    # grid search
    model = XGBClassifier()
    learning_rate = [0.0001, 0.001, 0.01, 0.1, 0.2, 0.3]
    param_grid = dict(learning_rate=learning_rate)
    kfold = StratifiedKFold(n_splits=10, shuffle=True, random_state=7)
    grid_search = GridSearchCV(model, param_grid, scoring="neg_log_loss", n_jobs=-1, cv=kfold)
    grid_result = grid_search.fit(X, y)

    # summarize results
    print(); print("Best: %f using %s" % (grid_result.best_score_, grid_result.best_params_))
    means = grid_result.cv_results_['mean_test_score']
    stds = grid_result.cv_results_['std_test_score']
    params = grid_result.cv_results_['params']

    for mean, stdev, param in zip(means, stds, params):
	     print("%f (%f) with: %r" % (mean, stdev, param))

    # plot
    pyplot.errorbar(learning_rate, means, yerr=stds)
    pyplot.title("XGBoost learning_rate vs Log Loss")
    pyplot.xlabel('learning_rate')
    pyplot.ylabel('Log Loss')
    pyplot.savefig('learning_rate.png')

Snippet_193()
********************How to optimise learning rates in XGBoost*********************

Best: -0.077694 using {'learning_rate': 0.3}
-1.086580 (0.000540) with: {'learning_rate': 0.0001}
-0.986720 (0.005130) with: {'learning_rate': 0.001}
-0.443082 (0.032684) with: {'learning_rate': 0.01}
-0.083225 (0.059937) with: {'learning_rate': 0.1}
-0.080207 (0.059523) with: {'learning_rate': 0.2}
-0.077694 (0.059774) with: {'learning_rate': 0.3}