Recipe: How to optimise learning rates in XGBoost example 1?

How to optimise learning rates in XGBoost example 1?

This recipe helps you optimise learning rates in XGBoost example 1
In [2]:
def Snippet_193():
    print(format('How to optimise learning rates in XGBoost','*^82'))

    import warnings

    # load libraries
    from sklearn import datasets
    from sklearn.model_selection import train_test_split
    from xgboost import XGBClassifier
    from sklearn.model_selection import GridSearchCV
    from sklearn.model_selection import StratifiedKFold
    import matplotlib
    from matplotlib import pyplot

    # load the iris datasets
    dataset = datasets.load_wine()
    X =; y =
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25)

    # grid search
    model = XGBClassifier()
    learning_rate = [0.0001, 0.001, 0.01, 0.1, 0.2, 0.3]
    param_grid = dict(learning_rate=learning_rate)
    kfold = StratifiedKFold(n_splits=10, shuffle=True, random_state=7)
    grid_search = GridSearchCV(model, param_grid, scoring="neg_log_loss", n_jobs=-1, cv=kfold)
    grid_result =, y)

    # summarize results
    print(); print("Best: %f using %s" % (grid_result.best_score_, grid_result.best_params_))
    means = grid_result.cv_results_['mean_test_score']
    stds = grid_result.cv_results_['std_test_score']
    params = grid_result.cv_results_['params']

    for mean, stdev, param in zip(means, stds, params):
	     print("%f (%f) with: %r" % (mean, stdev, param))

    # plot
    pyplot.errorbar(learning_rate, means, yerr=stds)
    pyplot.title("XGBoost learning_rate vs Log Loss")
    pyplot.ylabel('Log Loss')

********************How to optimise learning rates in XGBoost*********************

Best: -0.077694 using {'learning_rate': 0.3}
-1.086580 (0.000540) with: {'learning_rate': 0.0001}
-0.986720 (0.005130) with: {'learning_rate': 0.001}
-0.443082 (0.032684) with: {'learning_rate': 0.01}
-0.083225 (0.059937) with: {'learning_rate': 0.1}
-0.080207 (0.059523) with: {'learning_rate': 0.2}
-0.077694 (0.059774) with: {'learning_rate': 0.3}

Stuck at work?
Can't find the recipe you are looking for. Let us know and we will find an expert to create the recipe for you. Click here
Companies using this Recipe