KNN

import numpy as np
import matplotlib.pyplot as plt
from tpknnsource import (rand_gauss, rand_bi_gauss, rand_checkers, rand_tri_gauss,
                         rand_clown, plot_2d, ErrorCurve,
                         frontiere_new, LOOCurve)
import numpy as np
from scipy import stats
from sklearn.neighbors import KNeighborsClassifier
from sklearn.base import BaseEstimator, ClassifierMixin
import matplotlib.pyplot as plt
import seaborn as sns
from matplotlib import rc
from sklearn import linear_model
from sklearn.preprocessing import PolynomialFeatures
from sklearn.metrics import accuracy_score
from sklearn import linear_model

# Plot
plt.close('all')
rc('font', **{'family': 'sans-serif', 'sans-serif': ['Computer Modern Roman']})
params = {'axes.labelsize': 12,
          'font.size': 16,
          'legend.fontsize': 16,
          'text.usetex': False,
          'figure.figsize': (8, 6)}
plt.rcParams.update(params)
# Seaborn
sns.set_context("poster")
sns.set_palette("colorblind")
sns.set_style("white")
sns.axes_style()
# Seed initialization
np.random.seed(seed=44)
# For saving files
saving_activated = True # False

n = 10
mu = [1., 1.]
sigmas = [1., 1.]
rand_gauss(n, mu, sigmas)

n1 = 20
n2 = 20
mu1 = [1., 1.]
mu2 = [-1., -1.]
sigmas1 = [0.9, 0.9]
sigmas2 = [0.9, 0.9]
X1, y1 = rand_bi_gauss(n1, n2, mu1, mu2, sigmas1, sigmas2)

rand_bi_gauss()

n1 = 20
n2 = 20
n3 = 20
mu1 = [1., 1.]
mu2 = [-1., -1.]
mu3 = [1., -1.]
sigmas1 = [0.9, 0.9]
sigmas2 = [0.9, 0.9]
sigmas3 = [0.9, 0.9]
X2, y2 = rand_tri_gauss(n1, n2, n3, mu1, mu2, mu3, sigmas1, sigmas2, sigmas3)

n1 = 50
n2 = 50
sigmas1 = 1.
sigmas2 = 5.
X3, y3 = rand_clown(n1, n2, sigmas1, sigmas2)

n1 = 150
n2 = 150
sigma = 0.1
X4, y4 = rand_checkers(n1, n2, sigma)

plt.close("all")
plt.figure(1, figsize=(15, 15))
plt.subplot(221)
plt.title('data set #1')
plot_2d(X1, y1)
plt.subplot(222)
plt.title('Second data set #2')
plot_2d(X2, y2)
plt.subplot(223)
plt.title('Third data set #3')
plot_2d(X3, y3)
plt.subplot(224)
plt.title('Fourth data set #4')
plot_2d(X4, y4)
plt.show()

from sklearn.base import BaseEstimator, ClassifierMixin


class KNNClassifier_homemade(BaseEstimator, ClassifierMixin):
    """ Homemade kNN classifier class """

    def __init__(self, n_neighbors=1):
        self.n_neighbors = n_neighbors
        pass

    def fit(self, X_tr, y_tr):
        self.X_tr = X_tr
        self.y_tr = y_tr

    def predict(self, x):
        X = self.X_tr
        y = self.y_tr
        XSize = X.shape[0]
        k = self.n_neighbors

        # calcul the distances
        diffMat = np.tile(x, (XSize, 1)) - X
        sqDiffMat = diffMat**2
        sqDistances = sqDiffMat.sum(axis=1)
        distances = sqDistances**0.5
        # print(diffMat)
        # print(distances)
        sortedDistIndicies = distances.argsort()
        # print(sortedDistIndicies)

        # chose the k nearest neighbours and get their label and accumule their distances by label
        classCount = {}
        # classCount = {label : (count of each label, total distance) }
        # 1:(0,0),2:(0,0),3:(0,0)}

        for i in range(k):
            print(classCount)

            voteIlabel = int(y[sortedDistIndicies[i]])
            print(voteIlabel)
            # count=
            classCount[voteIlabel] = (classCount.get(voteIlabel, (0, 0))[
                                      0]+1, classCount.get(voteIlabel, (0, 0))[1]+distances[voteIlabel])

        # chose the label the most frequent and si ex aequo chose the class with shortest distance
        sortedClassCount = sorted(classCount.items(), key=lambda kv: kv[1],
                                  reverse=True)
        # print(classCount.items())
        # print(sortedClassCount)

        return sortedClassCount[0][0]

# test my function
clf = KNNClassifier_homemade(n_neighbors=9)
clf.fit(X2, y2)
clf.predict(np.array([0.70322977, -0.08537583]))

import numpy as np
import matplotlib.pyplot as plt
from matplotlib.colors import ListedColormap
from sklearn import neighbors, datasets

n_neighbors = 5

clf = KNeighborsClassifier(n_neighbors=5)
clf.fit(X2, y2)
print(clf.predict([[0.70322977, -0.08537583]]))

# k=5
clf.fit(X1, y1)

def f(x): return clf.predict(x.reshape(1, -1))

frontiere_new(f, X1, y1, w=None, step=50, alpha_choice=1,
              colorbar=True, samples=True)

clf = KNeighborsClassifier(n_neighbors=5)
clf.fit(X2, y2)

def f(x): return clf.predict(x.reshape(1, -1))

frontiere_new(f, X2, y2, w=None, step=50, alpha_choice=1, colorbar=True,
              samples=True)

clf = KNeighborsClassifier(n_neighbors=5)
clf.fit(X3, y3)

def f(x): return clf.predict(x.reshape(1, -1))

frontiere_new(f, X3, y3, w=None, step=50, alpha_choice=1, colorbar=True,
              samples=True)

clf = KNeighborsClassifier(n_neighbors=5)
clf.fit(X4, y4)


def f(x): return clf.predict(x.reshape(1, -1))


frontiere_new(f, X4, y4, w=None, step=50, alpha_choice=1, colorbar=True,
              samples=True)

knn0 = KNeighborsClassifier(n_neighbors=5)
knn0 = knn0.fit(X2, y2)

knn1 = KNeighborsClassifier(n_neighbors=30)
knn1 = knn1.fit(X2, y2)

knn2 = KNeighborsClassifier(n_neighbors=60)
knn2 = knn2.fit(X2, y2)

knn3 = KNeighborsClassifier(n_neighbors=len(X2))
knn3 = knn3.fit(X2, y2)

knn4 = KNeighborsClassifier(n_neighbors=1)
knn4 = knn4.fit(X2, y2)

plt.figure(num=None, figsize=(20, 30))
# k=1
plt.subplot(511)
plt.title("k=1")


def f(x): return knn4.predict(x.reshape(1, -1))


frontiere_new(f, X=X2, y=y2)

# k=5
plt.subplot(512)

plt.title("k=5")


def f(x): return knn0.predict(x.reshape(1, -1))


frontiere_new(f, X=X2, y=y2)

# k=30
plt.subplot(513)
plt.title("k=30")


def f(x): return knn1.predict(x.reshape(1, -1))


frontiere_new(f, X=X2, y=y2)

# k=60
plt.subplot(514)
plt.title("k=60")


def f(x): return knn2.predict(x.reshape(1, -1))


frontiere_new(f, X=X2, y=y2)

# k=n
plt.subplot(515)
plt.title("k=n")


def f(x): return knn3.predict(x.reshape(1, -1))


frontiere_new(f, X=X2, y=y2)

def weight_func(dist):
    """ Weight function to replace lambda d: d ** -2.
    The lambda function is not valid because:
    if d==0 then 0^-2 is not valid. """

    # Dist could be multidimensional, flatten it so all values
    # can be looped

    h = 10**(-j)
    weight = np.exp((-(dist**2)/h))
    #print(j,h,weight,dist)
    return weight


"""# my first methode to split
train = []
i = 0
while i < len(X2):
    train.append(i)
    i = i+2"""


plt.figure(num=None, figsize=(20, 30))

j = -2
knn = KNeighborsClassifier(n_neighbors=7, weights=weight_func)
knn0 = knn.fit(X2[::2], y2[::2])

plt.subplot(511)
plt.title("j=-2")


def f(x): return knn0.predict(x.reshape(1, -1))


frontiere_new(f, X=X2[::2], y=y2[::2])

j = -1
knn = KNeighborsClassifier(n_neighbors=7, weights=weight_func)
knn1 = knn.fit(X2[::2], y2[::2])

plt.subplot(512)
plt.title("j=-1")


def f(x): return knn1.predict(x.reshape(1, -1))


frontiere_new(f, X=X2[::2], y=y2[::2])


j = 0
knn2 = KNeighborsClassifier(n_neighbors=7, weights=weight_func)
knn2 = knn.fit(X2[::2], y2[::2])
plt.subplot(513)
plt.title("j=0")


def f(x): return knn2.predict(x.reshape(1, -1))


frontiere_new(f, X=X2[::2], y=y2[::2])

j = 1
knn3 = KNeighborsClassifier(n_neighbors=7, weights=weight_func)
knn3 = knn3.fit(X2[::2], y2[::2])
plt.subplot(514)
plt.title("j=1")


def f(x): return knn3.predict(x.reshape(1, -1))


frontiere_new(f, X=X2[::2], y=y2[::2])

j = 2
knn4 = KNeighborsClassifier(n_neighbors=7, weights=weight_func)
knn4 = knn4.fit(X2[::2], y2[::2])

plt.subplot(515)
plt.title("j=2")


def f(x): return knn4.predict(x.reshape(1, -1))


frontiere_new(f, X=X2[::2], y=y2[::2])

knn8 = KNeighborsClassifier(n_neighbors=1)
knn8 = knn8.fit(X2[::2], y2[::2])
pred = knn8.predict(X2[::2])

print("Le taux d'erreur sur les mêmes données d’apprentissage est de:", round(
    (1 - accuracy_score(pred, y2[::2]))*100, 2), "%")

pred = knn8.predict(X2[1::2])

print("Le taux d'erreur sur des données de test est de:", round(
    (1 - accuracy_score(pred, y2[1::2]))*100, 2), "%")

from sklearn.metrics import accuracy_score

knn8 = KNeighborsClassifier(n_neighbors=1)
knn8 = knn8.fit(X4[::2], y4[::2])
pred = knn8.predict(X4[1::2])

print("Le taux d'erreur est de:", round(
    (1 - accuracy_score(pred, y4[1::2]))*100, 2), "%")

curve = ErrorCurve(k_range=list(range(1, 50)))
ErrorCurve.fit_curve(curve, X4[::2], y4[::2], X4[1::2], y4[1::2])
ErrorCurve.plot(curve)

i = 100
X4_10, y4_10 = rand_checkers(i, i, 0.1)
curve = ErrorCurve(k_range=list(range(1, 50)))
curve.fit_curve(X4_10[::2], y4_10[::2], X4_10[1::2], y4_10[1::2])
ErrorCurve.plot(curve)

i = 200
X4_10, y4_10 = rand_checkers(i, i, 0.1)
curve = ErrorCurve(k_range=list(range(1, 50)))
curve.fit_curve(X4_10[::2], y4_10[::2], X4_10[1::2], y4_10[1::2])
ErrorCurve.plot(curve)

i = 500
X4_10, y4_10 = rand_checkers(i, i, 0.1)
curve = ErrorCurve(k_range=list(range(1, 50)))
curve.fit_curve(X4_10[::2], y4_10[::2], X4_10[1::2], y4_10[1::2])
ErrorCurve.plot(curve)

i = 1000
X4_10, y4_10 = rand_checkers(i, i, 0.1)
curve = ErrorCurve(k_range=list(range(1, 50)))
curve.fit_curve(X4_10[::2], y4_10[::2], X4_10[1::2], y4_10[1::2])
ErrorCurve.plot(curve)

from sklearn.datasets import load_digits
digits = load_digits()
print(digits.data.shape)

import matplotlib.pyplot as plt 
plt.gray() 
plt.matshow(digits.images[0]) 
plt.show() 

digits.DESCR

digits.data

digits.images

digits.target

digits.target_names

images_and_labels = list(zip(digits.images, digits.target))
for index, (image, label) in enumerate(images_and_labels[:4]):
    plt.subplot(2, 4, index + 1)
    plt.axis('off')
    plt.imshow(image)
    plt.title('image: %i' % label)

for index, (image, label) in enumerate(images_and_labels[:100]):
    plt.subplot(10, 10, index + 1)
    plt.axis('off')
    plt.imshow(image, cmap=plt.cm.gray_r, interpolation='nearest')

from sklearn import metrics


def split_data(X, y):
    return [
        X[::2],
        y[::2],
        X[1::2],
        y[1::2],
    ]


X_train, y_train, X_test, y_test = split_data(digits.data, digits.target)

knn = KNeighborsClassifier(n_neighbors=30).fit(X_train, y_train)
y_pred = knn.predict(X_test)
print("Number of elements in test sample:", len(y_test))
print("Errors rate:", 1-accuracy_score(y_pred, y_test))
print((metrics.classification_report(y_test, knn.predict(X_test))))

plt.figure(figsize=(12, 7))
sns.heatmap(metrics.confusion_matrix(
    y_test, y_pred), annot=True, fmt='3.0f')
plt.title('Confusion_matrix', y=1.05, size=15)

k14 = [1, 6, 11, 16, 21, 26, 31, 36, 41, 46, 100, 200]
loo = LOOCurve(k_range=k14)
loo.fit_curve(digits.data, digits.target)
loo.plot()

# Another way to find the best k
from sklearn.model_selection import GridSearchCV

knn = KNeighborsClassifier()

knn_param_grid = {"n_neighbors": [6, 11, 16, 21, 26, 31, 36, 41, 46, 100, 200]}

knnCV = GridSearchCV(knn, param_grid=knn_param_grid, cv=10,
                     scoring="accuracy", n_jobs=-1, verbose=1)

knnCV.fit(digits.data, digits.target)

# Best param
param_knnCV = knnCV.best_estimator_
result_knnCV = knnCV.best_score_*100
print("Accuracy CV : %.2f%% ", (result_knnCV))

# the best k found is 6 (after 1)
param_knnCV

from sklearn.base import BaseEstimator, ClassifierMixin
import numpy as np


class LDAClassifier_Homemade(BaseEstimator, ClassifierMixin):
    """ Homemade LDA classifier class """

    def __init__(self):
        pass

    def fit(self, X_tr, y_tr):
        # m= num of y_tr=1
        # n= num of y_tr
        y = y_tr
        X = X_tr
        m = np.sum(y[y == 1])
        n = y.shape[0]
        self.pi_positif = m/n

        # calcul Pi+ , mu+, mu-, sigma(variance)
        # numpy.where(condition[, x, y])
        # Return elements chosen from x or y depending on condition.
        mu_positif = np.sum(X_tr[np.where(y == 1), :][0], axis=0).T/m
        mu_negatif = np.sum(X_tr[np.where(y == -1), :][0], axis=0).T/(n-m)

        # variance
        sigma_positif = np.dot((X[np.where(y == 1), :][0] - mu_positif.T).T,
                               (X[np.where(y == 1), :][0] - mu_positif.T)) / (m-1)
        sigma_negatif = np.dot((X[np.where(y == -1), :][0]-mu_negatif.T).T,
                               (X[np.where(y == -1), :][0]-mu_negatif.T)) / (n-m-1)
        sigma = ((m-1)*sigma_positif+(n-m-1)*sigma_negatif)/(n-2)

        self.mu_positif = mu_positif
        self.mu_negatif = mu_negatif
        self.sigma = sigma
        self.m = m
        self.n = n
        return self

    def predict(self, X):
        # We use the mu, sigma calculated in fit()
        mu_positif = self.mu_positif
        mu_negatif = self.mu_negatif
        sigma = self.sigma
        m = self.m
        n = self.n
        pred = []

        # for each point in X, we predict its label
        for x in X:
            #print(np.dot(np.dot(x, np.linalg.inv(sigma)), (mu_positif-mu_negatif)))
            # print(1/2*np.dot(np.dot(mu_positif, np.linalg.inv(sigma)), mu_positif.T) - \
             #   1/2*np.dot(np.dot(mu_negatif, np.linalg.inv(sigma)), mu_negatif.T) \
              #      + np.log(1-m/n) - np.log(m/n))

            # both ends have negative values
            if np.dot(
                np.dot(x, np.linalg.inv(sigma)), (mu_positif-mu_negatif)) \
                > 1/2*np.dot(np.dot(mu_positif, np.linalg.inv(sigma)), mu_positif.T) - \
                1/2*np.dot(np.dot(mu_negatif, np.linalg.inv(sigma)), mu_negatif.T) \
                    + np.log(1-m/n) - np.log(m/n):
                pred.append(1)
            else:
                pred.append(-1)
        return pred

LDA = LDAClassifier_Homemade().fit(X1, y1)
y_pred = LDA.predict(X1)

from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
from sklearn.model_selection import train_test_split

#un jeu de données simulé. m est différent de n − m
X1, y1 = rand_bi_gauss(1450, 390, [0,1], [0,1.1], [0.1,0.1], [0.1,0.1])

#Echantillon de validation relativement grand
X_1_train,X_1_test,y_1_train,y_1_test = train_test_split(X1,y1,test_size=0.8,random_state=1)

lda_homemade= LDAClassifier_Homemade()
lda_homemade.fit(X_1_train,y_1_train)

lda_homemade_pred = lda_homemade.predict(X_1_test)
result = accuracy_score(y_1_test, lda_homemade_pred)*100
print(" lda_homemade Model Accuracy  " , result, "%")

lda= LinearDiscriminantAnalysis()
lda.fit(X_1_train,y_1_train)

y_pred_lda=lda.predict(X_1_test)
result = accuracy_score(y_1_test, y_pred_lda)*100
print(" lda_homemade Model  Accuracy : " , result, "%")

def f(x): return lda.predict(x.reshape(1, -1))

frontiere_new(f, X1, y1, w=None, step=50, alpha_choice=1, colorbar=True,samples=True)

# un jeu de données simulé. m est différent de n − m
X1, y1 = rand_bi_gauss(1450, 390, [0, 1], [1, 2], [0.25, 0.25], [0.1, 0.1])

# Echantillon de validation relativement grand
X_1_train, X_1_test, y_1_train, y_1_test = train_test_split(
    X1, y1, test_size=0.8, random_state=1)

lda_homemade = LDAClassifier_Homemade()
lda_homemade.fit(X_1_train, y_1_train)

lda_homemade_pred = lda_homemade.predict(X_1_test)
result = accuracy_score(y_1_test, lda_homemade_pred)*100
print(" lda_homemade Model Accuracy  ", result, "%")

lda = LinearDiscriminantAnalysis()
lda.fit(X_1_train, y_1_train)

y_pred_lda = lda.predict(X_1_test)
result = accuracy_score(y_1_test, y_pred_lda)*100
print(" lda_homemade Model  Accuracy : ", result, "%")

def f(x): return lda_homemade.predict(x.reshape(1, -1))


frontiere_new(f, X1, y1, w=None, step=50, alpha_choice=1,
              colorbar=True, samples=True)

# un jeu de données simulé. m est différent de n − m
X1, y1 = rand_bi_gauss(1450, 390, [0, 1], [1, 2], [0.1, 0.1], [0.1, 0.1])

lda_homemade = LDAClassifier_Homemade()
lda_homemade.fit(X1, y1)


def f(x): return lda_homemade.predict(x.reshape(1, -1))


frontiere_new(f, X1, y1, w=None, step=50, alpha_choice=1,
              colorbar=True, samples=True)

n1 = 50
n2 = 50
sigmas1 = 5
sigmas2 = 5
X3, y3 = rand_clown(n1, n2, sigmas1, sigmas2)


def f(x): return lda_homemade.predict(x.reshape(1, -1))


frontiere_new(f, X3, y3, w=None, step=50, alpha_choice=1,
              colorbar=True, samples=True)

# un jeu de données simulé. (y+ y-) class déséquilibrées
X1, y1 = rand_bi_gauss(250, 10, [0, 1], [1, 2], [0.25, 0.25], [0.25, 0.25])

# 20% training data %80 test data avec stratification 
X_1_train, X_1_test, y_1_train, y_1_test = train_test_split(
    X1, y1, test_size=0.8, random_state=1, stratify=y1)

from sklearn.linear_model import LogisticRegression

lr = LogisticRegression(solver='liblinear', C=1e9)
lr.fit(X_1_train, y_1_train)

y_pred_lr = lr.predict(X_1_test)
result_lr = accuracy_score(y_1_test, y_pred_lr)*100
print("Logistic Regression Model Accuracy:  ", result_lr, "%")

lda = LinearDiscriminantAnalysis()
lda.fit(X1, y1)
y_pred_lr = lda.predict(X_1_test)
result_lr = accuracy_score(y_1_test, y_pred_lr)*100
print("LDA Model Accuracy:  ", result_lr, "%")

lda.coef_

lda.intercept_

lr.coef_

lr.intercept_

def f(x): return lr.predict(x.reshape(1, -1))


frontiere_new(f, X1, y1, w=None, step=50, alpha_choice=1,
              colorbar=True, samples=True)

def f(x): return lda.predict(x.reshape(1, -1))


frontiere_new(f, X1, y1, w=None, step=50, alpha_choice=1,
              colorbar=True, samples=True)

from sklearn import metrics


def split_data(X, y):
    return [
        X[::2],
        y[::2],
        X[1::2],
        y[1::2],
    ]


X_train, y_train, X_test, y_test = split_data(digits.data, digits.target)

lr = LogisticRegression(solver='liblinear', C=1e9,
                        multi_class='auto').fit(X_train, y_train)
y_pred = lr.predict(X_test)
print("Number of elements in test sample:", len(y_test))
print("Errors rate:", 1-accuracy_score(y_pred, y_test))
print((metrics.classification_report(y_test, lr.predict(X_test))))

plt.figure(figsize=(12, 7))
sns.heatmap(metrics.confusion_matrix(
    y_test, y_pred), annot=True)
plt.title('Confusion_matrix', y=1.05, size=15)