6. Model Fine Tuning

from sklearn.model_selection import GridSearchCV

from sklearn.ensemble import RandomForestRegressor

param_grid = [

    # try 12 (3×4) combinations of hyperparameters

    {'n_estimators': [3, 10, 30], 'max_features': [2, 4, 6, 8]},

    # then try 6 (2×3) combinations with bootstrap set as False

    {'bootstrap': [False], 'n_estimators': [3, 10], 'max_features': [2, 3, 4]},

  ]

forest_reg = RandomForestRegressor(random_state=42)

# train across 5 folds, that's a total of (12+6)*5=90 rounds of training 

grid_search = GridSearchCV(forest_reg, param_grid, cv=5,

                           scoring='neg_mean_squared_error',

                           return_train_score=True)

grid_search.fit(housing_prepared, housing_labels)

print(grid_search.best_params_)

{'max_features': 6, 'n_estimators': 30}

print(grid_search.best_estimator_)

RandomForestRegressor(max_features=6, n_estimators=30, random_state=42)

cvres = grid_search.cv_results_

for mean_score, params in zip(cvres["mean_test_score"], cvres["params"]):

    print(np.sqrt(-mean_score), params)

64246.880700613496 {'max_features': 2, 'n_estimators': 3}

55869.85367924813 {'max_features': 2, 'n_estimators': 10}

53472.1282558969 {'max_features': 2, 'n_estimators': 30}

61376.36445082522 {'max_features': 4, 'n_estimators': 3}

53846.329115303764 {'max_features': 4, 'n_estimators': 10}

51270.1941502407 {'max_features': 4, 'n_estimators': 30}

59860.61532587693 {'max_features': 6, 'n_estimators': 3}

53114.42460001889 {'max_features': 6, 'n_estimators': 10}

50811.43543872171 {'max_features': 6, 'n_estimators': 30}

59220.31563298743 {'max_features': 8, 'n_estimators': 3}

52884.78697544277 {'max_features': 8, 'n_estimators': 10}

50944.39369116168 {'max_features': 8, 'n_estimators': 30}

62805.52917192821 {'bootstrap': False, 'max_features': 2, 'n_estimators': 3}

54462.1410888642 {'bootstrap': False, 'max_features': 2, 'n_estimators': 10}

61117.32056104296 {'bootstrap': False, 'max_features': 3, 'n_estimators': 3}

53022.992252269294 {'bootstrap': False, 'max_features': 3, 'n_estimators': 10}

60234.58052562756 {'bootstrap': False, 'max_features': 4, 'n_estimators': 3}

52712.989031117104 {'bootstrap': False, 'max_features': 4, 'n_estimators': 10}

print(pd.DataFrame(grid_search.cv_results_))

mean_fit_time std_fit_time mean_score_time std_score_time param_max_features param_n_estimators param_bootstrap params split0_test_score split1_test_score split2_test_score split3_test_score split4_test_score mean_test_score std_test_score rank_test_score \

0 0.075160 0.005952 0.001628 0.003255 2 3 NaN {'max_features': 2, 'n_estimators': 3} -3.931932e+09 -4.108663e+09 -4.407950e+09 -3.913400e+09 -4.276364e+09 -4.127662e+09 1.924854e+08 18

1 0.208825 0.009360 0.013174 0.003575 2 10 NaN {'max_features': 2, 'n_estimators': 10} -2.909367e+09 -3.120571e+09 -3.345757e+09 -2.963266e+09 -3.268242e+09 -3.121441e+09 1.684421e+08 11

2 0.637935 0.024080 0.027345 0.003750 2 30 NaN {'max_features': 2, 'n_estimators': 30} -2.677180e+09 -2.901491e+09 -3.061026e+09 -2.678830e+09 -2.977815e+09 -2.859269e+09 1.563689e+08 8

3 0.102063 0.006219 0.004827 0.003942 4 3 NaN {'max_features': 4, 'n_estimators': 3} -3.514530e+09 -3.714699e+09 -3.961517e+09 -3.738894e+09 -3.905650e+09 -3.767058e+09 1.576876e+08 16

4 0.331181 0.008400 0.008935 0.001370 4 10 NaN {'max_features': 4, 'n_estimators': 10} -2.758664e+09 -2.966146e+09 -3.048343e+09 -2.706431e+09 -3.017552e+09 -2.899427e+09 1.397445e+08 9

5 0.977282 0.027483 0.031115 0.003655 4 30 NaN {'max_features': 4, 'n_estimators': 30} -2.469417e+09 -2.730887e+09 -2.779078e+09 -2.440168e+09 -2.723613e+09 -2.628633e+09 1.435127e+08 3

6 0.141438 0.008595 0.003201 0.003920 6 3 NaN {'max_features': 6, 'n_estimators': 3} -3.469825e+09 -3.777163e+09 -3.470435e+09 -3.499229e+09 -3.699814e+09 -3.583293e+09 1.294923e+08 13

7 0.830213 0.267011 0.017223 0.005046 6 10 NaN {'max_features': 6, 'n_estimators': 10} -2.718094e+09 -2.860399e+09 -2.994504e+09 -2.614934e+09 -2.917779e+09 -2.821142e+09 1.371676e+08 7

8 2.980916 0.041966 0.054408 0.002776 6 30 NaN {'max_features': 6, 'n_estimators': 30} -2.477282e+09 -2.632966e+09 -2.757774e+09 -2.360187e+09 -2.680800e+09 -2.581802e+09 1.438206e+08 1

9 0.397616 0.008179 0.008668 0.002151 8 3 NaN {'max_features': 8, 'n_estimators': 3} -3.409629e+09 -3.627721e+09 -3.639449e+09 -3.213483e+09 -3.644948e+09 -3.507046e+09 1.713356e+08 12

10 1.280724 0.012595 0.021398 0.001839 8 10 NaN {'max_features': 8, 'n_estimators': 10} -2.637820e+09 -2.920688e+09 -2.894457e+09 -2.586931e+09 -2.944107e+09 -2.796801e+09 1.522524e+08 5

11 3.818793 0.024259 0.060393 0.003006 8 30 NaN {'max_features': 8, 'n_estimators': 30} -2.417068e+09 -2.704235e+09 -2.752111e+09 -2.386532e+09 -2.716711e+09 -2.595331e+09 1.590898e+08 2

12 0.232447 0.007231 0.007618 0.001725 2 3 False {'bootstrap': False, 'max_features': 2, 'n_est... -3.776348e+09 -4.007550e+09 -4.036863e+09 -3.709783e+09 -4.192129e+09 -3.944534e+09 1.773098e+08 17

13 0.776798 0.071837 0.025901 0.005939 2 10 False {'bootstrap': False, 'max_features': 2, 'n_est... -2.800426e+09 -2.985411e+09 -3.058220e+09 -2.822708e+09 -3.163859e+09 -2.966125e+09 1.385473e+08 10

14 0.290882 0.004773 0.007681 0.000649 3 3 False {'bootstrap': False, 'max_features': 3, 'n_est... -3.577790e+09 -3.742428e+09 -3.790774e+09 -3.648047e+09 -3.917596e+09 -3.735327e+09 1.172880e+08 15

15 0.948928 0.015138 0.020933 0.002874 3 10 False {'bootstrap': False, 'max_features': 3, 'n_est... -2.698069e+09 -2.810563e+09 -2.967153e+09 -2.671411e+09 -2.909993e+09 -2.811438e+09 1.152570e+08 6

16 0.361425 0.006885 0.006889 0.002728 4 3 False {'bootstrap': False, 'max_features': 4, 'n_est... -3.477450e+09 -3.607213e+09 -3.844638e+09 -3.350849e+09 -3.860872e+09 -3.628205e+09 2.005349e+08 14

17 1.178464 0.028682 0.023459 0.002601 4 10 False {'bootstrap': False, 'max_features': 4, 'n_est... -2.629701e+09 -2.787807e+09 -2.989189e+09 -2.614631e+09 -2.871968e+09 -2.778659e+09 1.429733e+08 4

split0_train_score split1_train_score split2_train_score split3_train_score split4_train_score mean_train_score std_train_score

0 -1.114486e+09 -1.060851e+09 -1.105913e+09 -1.094676e+09 -1.133282e+09 -1.101842e+09 2.406866e+07

1 -5.759053e+08 -5.710866e+08 -5.772770e+08 -5.820883e+08 -5.967966e+08 -5.806308e+08 8.810527e+06

2 -4.385939e+08 -4.261753e+08 -4.364716e+08 -4.336798e+08 -4.521519e+08 -4.374145e+08 8.482200e+06

3 -9.783463e+08 -9.627673e+08 -9.841863e+08 -1.031772e+09 -1.008554e+09 -9.931251e+08 2.429626e+07

4 -5.340533e+08 -5.253397e+08 -5.299331e+08 -5.298511e+08 -5.321210e+08 -5.302596e+08 2.909844e+06

5 -4.023370e+08 -4.024745e+08 -4.028804e+08 -4.097805e+08 -4.081646e+08 -4.051274e+08 3.185886e+06

6 -9.272504e+08 -9.323462e+08 -9.092923e+08 -9.485986e+08 -9.375958e+08 -9.310167e+08 1.296387e+07

7 -5.176219e+08 -5.102643e+08 -5.036363e+08 -5.320044e+08 -5.129550e+08 -5.152964e+08 9.500832e+06

8 -3.969466e+08 -3.888514e+08 -3.956343e+08 -4.058589e+08 -3.912175e+08 -3.957017e+08 5.861017e+06

9 -9.121501e+08 -9.226590e+08 -8.740418e+08 -9.727259e+08 -8.892866e+08 -9.141727e+08 3.386559e+07

10 -5.215995e+08 -5.260928e+08 -4.925520e+08 -5.271685e+08 -5.086615e+08 -5.152149e+08 1.310370e+07

11 -3.976878e+08 -3.992145e+08 -3.905255e+08 -3.967421e+08 -3.932722e+08 -3.954884e+08 3.157049e+06

12 -0.000000e+00 -2.843179e+01 -0.000000e+00 -0.000000e+00 -0.000000e+00 -5.686359e+00 1.137272e+01

13 -0.000000e+00 -2.558861e+00 -4.822180e+03 -8.711582e+02 -5.676576e+03 -2.274495e+03 2.464517e+03

14 -0.000000e+00 -0.000000e+00 -0.000000e+00 -1.076625e+01 -4.861637e+01 -1.187652e+01 1.883722e+01

15 -2.365811e+02 -0.000000e+00 -2.365632e+02 -9.689629e-01 -4.375473e+00 -9.569775e+01 1.150327e+02

16 -0.000000e+00 -0.000000e+00 -0.000000e+00 -0.000000e+00 -0.000000e+00 0.000000e+00 0.000000e+00

17 -0.000000e+00 -1.832084e+00 -0.000000e+00 -0.000000e+00 -2.422407e+01 -5.211231e+00 9.532865e+00

final_model = grid_search.best_estimator_

X_test = strat_test_set.drop("median_house_value", axis=1)

y_test = strat_test_set["median_house_value"].copy()

X_test_prepared = full_pipeline.transform(X_test)

final_predictions = final_model.predict(X_test_prepared)

final_mse = mean_squared_error(y_test, final_predictions)

final_rmse = np.sqrt(final_mse)

print(final_rmse)

49040.18609727207

Evaluate System on the Test Set

final_model = lin_reg

X_test = strat_test_set.drop("median_house_value", axis=1)

y_test = strat_test_set["median_house_value"].copy()

X_test_prepared = full_pipeline.transform(X_test)

final_predictions = final_model.predict(X_test_prepared)

final_mse = mean_squared_error(y_test, final_predictions)

final_rmse = np.sqrt(final_mse)

print(final_rmse)

66741.36252150308