Hyperparameter Tuning — GridSearch & RandomSearch

from sklearn.model_selection import GridSearchCV, train_test_split
from sklearn.svm import SVC

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

# Every key is a hyperparameter name, every value is a list of options to try
param_grid = {
    "C": [0.1, 1, 10, 100],
    "kernel": ["linear", "rbf"],
    "gamma": ["scale", "auto"]
}

grid_search = GridSearchCV(
    SVC(),
    param_grid,
    cv=5,             # 5-fold cross-validation for EVERY combination
    scoring="accuracy"
)
grid_search.fit(X_train, y_train)

print("Best hyperparameters:", grid_search.best_params_)
print("Best cross-validated score:", grid_search.best_score_)

# The final, refit model with the best hyperparameters, ready to use directly
best_model = grid_search.best_estimator_
print("Test accuracy of best model:", best_model.score(X_test, y_test))

from sklearn.model_selection import RandomizedSearchCV
from sklearn.ensemble import GradientBoostingClassifier
from scipy.stats import randint, uniform

# A much larger space than we'd want to grid-search exhaustively
param_distributions = {
    "n_estimators": randint(50, 500),
    "max_depth": randint(2, 10),
    "learning_rate": uniform(0.01, 0.3),
    "subsample": uniform(0.6, 0.4)
}

random_search = RandomizedSearchCV(
    GradientBoostingClassifier(random_state=42),
    param_distributions,
    n_iter=30,        # only try 30 random combinations, not every possibility
    cv=5,
    scoring="accuracy",
    random_state=42
)
random_search.fit(X_train, y_train)

print("Best hyperparameters found:", random_search.best_params_)
print("Best cross-validated score:", random_search.best_score_)

# Best hyperparameters: {'C': 10, 'gamma': 'scale', 'kernel': 'rbf'}
# Best cross-validated score: 0.891
# Test accuracy of best model: 0.87
# -> Test accuracy is close to but not identical to best_score_, which is
#    expected: best_score_ is a CV average on training data, the test
#    score is one final check on data the search never saw at all.