k-Nearest Neighbors Algorithm

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from math import sqrt

plt.rcParams['font.sans-serif'] = ['Simhei']

wine_data = {'color_intensity': [14.13, 13.2, 13.16, 14.27, 13.24, 12.07, 12.43, 11.79, 12.37, 12.04],
             'alcohol_content': [5.64, 4.28, 5.68, 4.80, 4.22, 2.76, 3.94, 3.1, 2.12, 2.6],
             'variety': [0, 0, 0, 0, 0, 1, 1, 1, 1, 1]
            }

df = pd.DataFrame(wine_data)

features = df.iloc[:, 1:3].values
new_point = np.array([12.3, 4.1])

dist_color = ((new_point - features) ** 2)[:, 0]
dist_alcohol = ((new_point - features) ** 2)[:, 1]

total_distance = np.sqrt(dist_color + dist_alcohol)
nearest_indices = np.argsort(total_distance)[:3]
labels = df.iloc[:, :1].values

predicted_class = pd.Series([labels[i] for i in nearest_indices]).value_counts().index[0]

plt.scatter(features[labels.ravel() == 1, 0], features[labels.ravel() == 1, 1], color='red', label='Cabernet Sauvignon')
plt.scatter(features[labels.ravel() == 0, 0], features[labels.ravel() == 0, 1], color='purple', label='Pinot Noir')
plt.scatter(new_point[0], new_point[1], color='yellow', label='New Sample')
plt.legend()
plt.show()

Using scikit-learn for kNN

from sklearn.neighbors import KNeighborsClassifier
import pandas as pd
import numpy as np

wine_data = {'color_intensity': [14.13, 13.2, 13.16, 14.27, 13.24, 12.07, 12.43, 11.79, 12.37, 12.04],
             'alcohol_content': [5.64, 4.28, 5.68, 4.80, 4.22, 2.76, 3.94, 3.1, 2.12, 2.6],
             'variety': [0, 0, 0, 0, 0, 1, 1, 1, 1, 1]
            }

df = pd.DataFrame(wine_data)
knn = KNeighborsClassifier(n_neighbors=3)
knn.fit(df.iloc[:, 1:3], df.iloc[:, 0])
prediction = knn.predict([[12.01, 4.1]])

random_features = np.concatenate([np.random.normal(11, 2, (10, 1)), np.random.normal(5, 2, (10, 1))], axis=1)
multiple_predictions = knn.predict(random_features)

accuracy = knn.score([[12.01, 4.1]], [0])

actual_labels = [0, 1, 1, 0, 1, 0, 1, 0, 0, 0]
model_accuracy = knn.score(random_features, np.array(actual_labels))

probability_predictions = knn.predict_proba(random_features)

Splitting Data into Training and Test Sets

from sklearn.neighbors import KNeighborsClassifier
import pandas as pd
import numpy as np
from sklearn.datasets import load_breast_cancer
from sklearn.model_selection import train_test_split
import matplotlib.pyplot as plt
from sklearn.model_selection import cross_val_score as CVS

wine_data = {'color_intensity': [14.13, 13.2, 13.16, 14.27, 13.24, 12.07, 12.43, 11.79, 12.37, 12.04],
             'alcohol_content': [5.64, 4.28, 5.68, 4.80, 4.22, 2.76, 3.94, 3.1, 2.12, 2.6],
             'variety': [0, 0, 0, 0, 0, 1, 1, 1, 1, 1]
            }
df = pd.DataFrame(wine_data)
X = df.loc[:, ['color_intensity', 'alcohol_content']]
y = df.loc[:, 'variety']

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=234)

clf = KNeighborsClassifier(n_neighbors=5)
clf.fit(X_train, y_train)
score = clf.score(X_test, y_test)

scores_list = []
k_values = range(1, 9)
for k in k_values:
    knn_model = KNeighborsClassifier(n_neighbors=k)
    knn_model.fit(X_train, y_train)
    scores_list.append(knn_model.score(X_test, y_test))
plt.plot(k_values, scores_list)
plt.show()

Cross-Validation for Optimal k

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=100)

means = []
vars = []
for i in range(1, 7):
    knn_cv = KNeighborsClassifier(n_neighbors=i)
    cv_results = CVS(knn_cv, X_train, y_train, cv=5)
    means.append(cv_results.mean())
    vars.append(cv_results.var())

means = np.array(means)
vars = np.array(vars)
plt.plot(range(1, 7), means, color='k')
plt.plot(range(1, 7), vars, color='r', linestyle='--')
plt.show()

final_model = KNeighborsClassifier(n_neighbors=5)
final_model.fit(X_train, y_train)
final_score = final_model.score(X_test, y_test)

Feature Normalization

data = [[-1, 2], [-0.5, 6], [0, 10], [1, 18]]
df = pd.DataFrame(data)
normalized = (df - np.min(df, axis=0)) / (np.max(df, axis=0) - np.min(df, axis=0))

Distance Weighting in kNN

KNeighborsClassifier(n_neighbors=i, weights='distance')

Decision Trees

import pandas as pd
import numpy as np
from sklearn.neighbors import KNeighborsClassifier

sample_data = {'companionship': [0, 0, 0, 1, 1],
               'gaming': [1, 1, 0, 1, 1],
               'label': ['yes', 'yes', 'no', 'no', 'no']}
df = pd.DataFrame(sample_data)

def compute_entropy(dataset):
    n_samples = dataset.shape[0]
    label_counts = dataset.iloc[:, -1].value_counts()
    probabilities = label_counts / n_samples
    entropy = (-probabilities * np.log2(probabilities)).sum()
    return entropy

print(compute_entropy(df))

def find_best_split(dataset):
    base_entropy = compute_entropy(dataset)
    best_gain = 0
    best_column = -1
    for col in range(dataset.shape[1] - 1):
        unique_values = dataset.iloc[:, col].value_counts().index
        weighted_entropy = 0
        for val in unique_values:
            subset = dataset[dataset.iloc[:, col] == val]
            subset_entropy = compute_entropy(subset)
            weighted_entropy += (subset.shape[0] / dataset.shape[0]) * subset_entropy
        info_gain = base_entropy - weighted_entropy
        if info_gain > best_gain:
            best_gain = info_gain
            best_column = col
    return best_column

print(find_best_split(df))

def split_dataset(dataset, column, value):
    col_name = dataset.columns[column]
    subset = dataset.loc[dataset[col_name] == value, :].drop(col_name, axis=1)
    return subset

print(split_dataset(df, 1, 1))

Implementing Decision Trees with scikit-learn

import pandas as pd
import numpy as np
from sklearn import tree
from sklearn.model_selection import train_test_split
from sklearn.datasets import load_wine
from sklearn.tree import DecisionTreeClassifier
import graphviz

wine = load_wine()
data = np.concatenate((wine.data, wine.target.reshape(-1, 1)), axis=1)
feature_names = ['alcohol', 'malic_acid', 'ash', 'alcalinity_of_ash', 'magnesium', 'total_phenols', 'flavanoids',
                 'nonflavanoid_phenols', 'proanthocyanins', 'color_intensity', 'hue', 'od280/od315', 'proline', 'label']
wine_df = pd.DataFrame(data=data, columns=feature_names)

X_train, X_test, y_train, y_test = train_test_split(wine_df.iloc[:, :-1], wine_df.iloc[:, -1], test_size=0.3, random_state=420)

clf = DecisionTreeClassifier(criterion='entropy')
clf.fit(X_train, y_train)
accuracy = clf.score(X_test, y_test)

feature_names = ['alcohol', 'malic_acid', 'ash', 'alcalinity_of_ash', 'magnesium', 'total_phenols', 'flavanoids',
                 'nonflavanoid_phenols', 'proanthocyanins', 'color_intensity', 'hue', 'od280/od315', 'proline']
dot_data = tree.export_graphviz(clf, feature_names=feature_names, class_names=["Gin", "Sherry", "Vermouth"], filled=True)
graph = graphviz.Source(dot_data, filename='decision_tree.pdf')
graph.render('wine')

for feature, importance in zip(feature_names, clf.feature_importances_):
    print(feature, importance)

Preventing Overfitting and Pruning

clf = tree.DecisionTreeClassifier(criterion='entropy', min_samples_split=20)
clf.fit(X_train, y_train)

dot_data = tree.export_graphviz(clf, feature_names=feature_names, class_names=["Gin", "Sherry", "Vermouth"], filled=True, rounded=True)
graph = graphviz.Source(dot_data)

Determining Optimal Pruning Praameters

test_scores = []
for depth in range(1, 11):
    clf = tree.DecisionTreeClassifier(criterion='entropy', max_depth=depth, random_state=30, splitter='random')
    clf.fit(X_train, y_train)
    test_scores.append(clf.score(X_test, y_test))

plt.plot(range(1, 11), test_scores, color='red')
plt.ylabel('score')
plt.xlabel('max_depth')
plt.xticks(range(1, 11))
plt.show()

Addressing Class Imbalance

import matplotlib.pyplot as plt
from sklearn.tree import DecisionTreeClassifier
from sklearn.datasets import make_blobs
from sklearn.model_selection import train_test_split
from sklearn import metrics

class_1_samples = 1000
class_2_samples = 100
centroids = [[0, 0], [2.0, 2.0]]
std_devs = [2.5, 0.5]

X, y = make_blobs(n_samples=[class_1_samples, class_2_samples], centers=centroids, cluster_std=std_devs, random_state=420, shuffle=False)

plt.scatter(X[:, 0], X[:, 1], c=y, cmap='rainbow', s=10)

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=420)

clf_unweighted = DecisionTreeClassifier()
clf_unweighted.fit(X_train, y_train)
clf_weighted = DecisionTreeClassifier(class_weight='balanced')
clf_weighted.fit(X_train, y_train)

score_unweighted = clf_unweighted.score(X_test, y_test)
score_weighted = clf_weighted.score(X_test, y_test)

conf_matrix_before = metrics.confusion_matrix(y_test, clf_unweighted.predict(X_test))
conf_matrix_after = metrics.confusion_matrix(y_test, clf_weighted.predict(X_test))

precision_before = metrics.precision_score(y_test, clf_unweighted.predict(X_test))
precision_after = metrics.precision_score(y_test, clf_weighted.predict(X_test))

recall_before = metrics.recall_score(y_test, clf_unweighted.predict(X_test))
recall_after = metrics.recall_score(y_test, clf_weighted.predict(X_test))

f1_before = metrics.f1_score(y_test, clf_unweighted.predict(X_test))
f1_after = metrics.f1_score(y_test, clf_weighted.predict(X_test))

Linear Regression

from sklearn.linear_model import LinearRegression
from sklearn.model_selection import train_test_split
from sklearn.model_selection import cross_val_score
from sklearn.datasets import fetch_california_housing
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np

housing = fetch_california_housing()
X = pd.DataFrame(housing.data, columns=housing.feature_names)
y = housing.target
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=420)

lr = LinearRegression()
lr.fit(X_train, y_train)
train_score = lr.score(X_train, y_train)

from sklearn.metrics import mean_squared_error
y_pred_train = lr.predict(X_train)
y_pred_test = lr.predict(X_test)
mse_train = mean_squared_error(y_train, y_pred_train)
mse_test = mean_squared_error(y_test, y_pred_test)

cv_scores_mse = cross_val_score(LinearRegression(), X_train, y_train, cv=10, scoring='neg_mean_squared_error')
mean_cv_mse = cv_scores_mse.mean()

from sklearn.metrics import r2_score
r2_train = r2_score(y_train, y_pred_train)
r2_test = r2_score(y_test, y_pred_test)
cv_scores_r2 = cross_val_score(lr, X_train, y_train, cv=10, scoring='r2')
mean_cv_r2 = cv_scores_r2.mean()

coefficients = lr.coef_
intercept = lr.intercept_

feature_importance = list(zip(X.columns, lr.coef_))

from sklearn.preprocessing import StandardScaler
scaler = StandardScaler()
X_train_scaled = scaler.fit_transform(X_train)
lr_scaled = LinearRegression()
lr_scaled.fit(X_train_scaled, y_train)
scaled_score = lr_scaled.score(X_train_scaled, y_train)

plt.scatter(range(len(y_test)), sorted(y_test), s=2, label='True')
plt.scatter(range(len(y_test)), y_pred_test[np.argsort(y_test)], s=2, c='r', label='Predict', alpha=0.3)
plt.legend()
plt.show()

from sklearn.preprocessing import PolynomialFeatures
X.columns = ['median_income', 'house_age', 'avg_rooms', 'avg_bedrooms', 'population', 'avg_occupancy', 'latitude', 'longitude']
poly = PolynomialFeatures(degree=2).fit(X, y)
X_poly = poly.transform(X)
reg_poly = LinearRegression().fit(X_poly, y)
feature_names_poly = poly.get_feature_names(X.columns)
coeff_poly = list(zip(feature_names_poly, reg_poly.coef_))

poly_4 = PolynomialFeatures(degree=4).fit(X, y)
X_poly_4 = poly_4.transform(X)
lr_poly_4 = LinearRegression().fit(X_poly_4, y)
score_poly_4 = lr_poly_4.score(X_poly_4, y)

Logistic Regression

import numpy as np
import pandas as pd
from sklearn.datasets import load_breast_cancer
import matplotlib.pyplot as plt

X = load_breast_cancer().data
y = load_breast_cancer().target

from sklearn.linear_model import LogisticRegression as LR
from sklearn.model_selection import train_test_split

lr_l1 = LR(penalty='l1', solver='liblinear', C=0.5, max_iter=1000).fit(X, y)
lr_l2 = LR(penalty='l2', solver='liblinear', C=0.5, max_iter=1000).fit(X, y)

score_l1 = lr_l1.score(X, y)
score_l2 = lr_l2.score(X, y)

coef_l1 = lr_l1.coef_
coef_l2 = lr_l2.coef_

probabilities = lr_l2.predict_proba(X)

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=420)

l1_scores = []
l2_scores = []
l1_test_scores = []
l2_test_scores = []

c_values = np.linspace(0.05, 1, 19)
for c in c_values:
    lrl1 = LR(penalty='l1', solver='liblinear', C=c, max_iter=1000).fit(X_train, y_train)
    lrl2 = LR(penalty='l2', solver='liblinear', C=c, max_iter=1000).fit(X_train, y_train)
    
    l1_scores.append(lrl1.score(X_train, y_train))
    l2_scores.append(lrl2.score(X_train, y_train))
    
    l1_test_scores.append(lrl1.score(X_test, y_test))
    l2_test_scores.append(lrl2.score(X_test, y_test))

plt.figure(figsize=(6, 6))
plt.plot(c_values, l1_scores, 'green', label='L1')
plt.plot(c_values, l2_scores, 'black', label='L2')
plt.plot(c_values, l1_test_scores, 'lightgreen', label='L1 test')
plt.plot(c_values, l2_test_scores, 'gray', label='L2 test')
plt.legend(loc=4)
plt.show()

l2_iter_scores = []
l2_iter_test_scores = []
for i in range(1, 201, 10):
    lrl2_iter = LR(penalty='l2', solver='liblinear', C=0.9, max_iter=i).fit(X_train, y_train)
    l2_iter_scores.append(lrl2_iter.score(X_train, y_train))
    l2_iter_test_scores.append(lrl2_iter.score(X_test, y_test))

plt.plot(range(1, 201, 10), l2_iter_scores, label='L2 train')
plt.plot(range(1, 201, 10), l2_iter_test_scores, label='L2 test')
plt.legend(loc=4)
plt.show()

from sklearn.model_selection import GridSearchCV
from sklearn.preprocessing import StandardScaler

data_df = pd.DataFrame(X, columns=load_breast_cancer().feature_names)
data_df['label'] = y

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=420)

scaler = StandardScaler().fit(X_train)
X_train_scaled = scaler.transform(X_train)
X_test_scaled = scaler.transform(X_test)

param_grid = {'C': list(np.linspace(0.05, 1, 19)),
              'solver': ['liblinear', 'sag', 'newton-cg', 'lbfgs']}

model = LR(penalty='l2', max_iter=10000)
gs = GridSearchCV(model, param_grid, cv=5)
gs.fit(X_train_scaled, y_train)

best_score = gs.best_score_
best_params = gs.best_params_

optimal_model = LR(penalty='l2', max_iter=10000, C=best_params['C'], solver=best_params['solver'])
optimal_model.fit(X_train_scaled, y_train)
train_score_optimal = optimal_model.score(X_train_scaled, y_train)
test_score_optimal = optimal_model.score(X_test_scaled, y_test)

from sklearn.metrics import recall_score
y_pred_test_optimal = optimal_model.predict(X_test_scaled)
recall = recall_score(y_pred_test_optimal, y_test, average='micro')

Logistic Regression Case Study

import numpy as np
import pandas as pd
from sklearn.datasets import load_iris
from sklearn.model_selection import train_test_split, GridSearchCV
from sklearn.linear_model import LogisticRegression as LR
from sklearn.preprocessing import StandardScaler

iris = load_iris()
X = pd.DataFrame(iris.data, columns=iris.feature_names)
y = iris.target

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=420)

scaler = StandardScaler().fit(X_train)
X_train_scaled = scaler.transform(X_train)
X_test_scaled = scaler.transform(X_test)

param_grid = {'C': list(np.linspace(0.05, 1, 20)),
              'solver': ['liblinear', 'sag', 'newton-cg', 'lbfgs']}

model = LR(penalty='l2', max_iter=10000)
gs = GridSearchCV(model, param_grid, cv=5)
gs.fit(X_train_scaled, y_train)

best_score = gs.best_score_
best_params = gs.best_params_

final_model = LR(penalty='l2', max_iter=10000, C=best_params['C'], solver=best_params['solver'])
final_model.fit(X_train_scaled, y_train)
scores = final_model.score(X_train_scaled, y_train), final_model.score(X_test_scaled, y_test)

from sklearn.metrics import recall_score
y_pred = final_model.predict(X_test_scaled)
recall = recall_score(y_pred, y_test, average='micro')