🤯🤯🤯 💣🐍💣💥🐍🐍💥🐍🐍💣🐍💣💥💣💣💥💣💥💥🐍💥💥💥💥💥💥 🤯🤯🤯

src/classifier/data_loader.py

@@ -13,8 +13,10 @@ def load_df():
     df = pd.read_csv(DATA_FILE_PATH)
     tr = df[df["split"] == "train"]
     tt = df[df["split"] == "test"]
+    new_df = pd.concat([tr, tt])
+    new_df["Genre"] = pd.Categorical(new_df["Genre"], categories=GENRES, ordered=True)
 
-    return pd.concat([tr,tt]).reset_index(drop=True)
+    return new_df.reset_index(drop=True)
 
 @cache
 def load_df_train():

src/classifier/data_models.py

@@ -44,6 +44,9 @@ def extract_train_data_from(T, train_data):
     indices = train_data.index.values
     return T[indices]
 
+def extract_genres_from(dataframe):
+    return dataframe["Genre"].cat.codes.to_numpy()
+
 def vectorize_data(dataframe, Q=1000):
     tokens = np.hstack(dataframe["tokens"].apply(lambda x: x.split()).values)
     unique_tokens = pd.Series(tokens).value_counts().index[:Q].values

src/classifier/knn.py

-from turtle import distance
 import numpy as np
-import matplotlib.pyplot as plt
-import pandas as pd
-from classifier.data_loader import *
 from classifier.data_loader import GENRES
+from classifier.data_models import extract_genres_from
 
 class KNNPredictor:
-    def __init__(self, distances, df_train, df_test, df_all):
+    def __init__(self, distances, df_train, df_test, genres_train = None, genres_test = None):
         self.distances = distances
-        self.df_test = df_test
         self.df_train = df_train
+        self.df_test = df_test
+        self.genres_train = genres_train if not genres_train is None else extract_genres_from(df_train)
+        self.genres_test = genres_test if not genres_test is None else extract_genres_from(df_test)
         self.nodes = self.get_nodos_cercanos()
-        self.df_all = df_all
 
     def predict_for(self, neighbours):
         generos_inferidos = self.inferir_generos(neighbours)
-
-        res = sum(
-            self.df_all.loc[self.df_test.iloc[i].name, "Genre"] == genero
-            for i, genero in enumerate(generos_inferidos)
-        )
-        res /= len(generos_inferidos)
-
-        return res
+        return np.sum(np.equal(generos_inferidos, self.genres_test))/len(self.genres_test)
 
     def get_nodos_cercanos(self):
         T = self.distances[self.df_test.index.values, :][:, self.df_train.index.values]
         indices = np.argsort(T, axis=1, stable=True)
-
-        return [ [ self.df_train.iloc()[i]["Genre"] for i in row ] for row in indices ]
+        return self.genres_train[indices]
 
     def inferir_generos(self, k):
-        return [
-            max(
-                ((g, sum(entry == g for entry in entries[:k])) for g in GENRES),
-                key=lambda x: x[1]
-            )[0]
-            for entries in self.nodes
-        ]
+        counts = np.zeros((self.nodes.shape[0], len(GENRES)), dtype=int)
+        np.add.at(counts, (np.arange(self.nodes.shape[0])[:, None], self.nodes[:, :k]), 1)
+
+        maxes = np.empty(self.nodes.shape[0], dtype=np.int8)
+        np.argmax(counts, axis=1, out=maxes)
+
+        return maxes

src/experiments/3a.py

@@ -20,7 +20,7 @@ if __name__ == "__main__":
         M = extract_train_data_from(X, x_train)
 
         dists = distances(X, M, center = with_mean)
-        knnp = KNNPredictor(dists, x_train, x_test, x_all)
+        knnp = KNNPredictor(dists, x_train, x_test)
         result = knnp.predict_for(K)
 
-        print("Para (Q = {}) es {} \n".format(Q, result))
\ No newline at end of file
+        print("Para (Q = {}) es {} \n".format(Q, result))

src/experiments/3b.py

@@ -48,7 +48,7 @@ if __name__ == "__main__":
 
         for fold, (x_train, x_test) in enumerate(cross_validations(df_train, KFOLDS)):
             dist_matrix = distances(X, extract_train_data_from(X, x_train), center = CENTERING)
-            knn = KNNPredictor(dist_matrix, x_train, x_test, df_train)
+            knn = KNNPredictor(dist_matrix, x_train, x_test)
 
             for K in range(1, MAXIMUM_K + 1):
                 print("Para (K = {}) y (Q = {}) en el fold-{} \n".format(K, Q, 1 + fold))

src/experiments/3d.py

@@ -14,7 +14,7 @@ from multiprocessing.pool import AsyncResult
 import sys
 import numpy as np
 import matplotlib.pyplot as plt
-from classifier.data_models import distances, vectorize_data, extract_train_data_from, with_mean, with_tf_idf, do_not_center, with_tf_idf_and_mean
+from classifier.data_models import distances, vectorize_data, extract_train_data_from, extract_genres_from, with_mean, with_tf_idf, do_not_center, with_tf_idf_and_mean
 from classifier.cv import cross_validations
 from classifier.knn import KNNPredictor
 from classifier.data_loader import *
@@ -73,35 +73,32 @@ def pca_for_fold(i_fold):
 
     print("[FOLD-{}] Generando PCA para P = {}".format(1 + i_fold, P_MAX))
     pca = PCA(X_tn, P_MAX, iterations=ITERS, epsilon = EPSILON, center = CENTERING)
-    # Por algun motivo, al usar mean center, si se centran los datos antes de transformar, la accuracy empeora
-    # Por otro lado con tf-idf, si se centra antes de transformar, mejora la accuracy
-    # return pca.transform(X_all)
     return pca.transform(CENTERING(X_all, X_tn))
 
 def accuracies_for_fold_and_P(i_fold, P, i_P):
     df_train, df_test = folds[i_fold]
+    train_genres, test_genres = genres_per_fold[i_fold]
     accuracy = np.zeros((MAXIMUM_K,), dtype=np.float64)
 
     print("[FOLD-{}] Generando KNN para P = {}".format(1 + i_fold, P))
     X_pca = X_pcas_per_fold[i_fold][:,:P]
 
     dist_matrix = distances(X_pca)
-    knn = KNNPredictor(dist_matrix, df_train, df_test, df_all)
+    knn = KNNPredictor(dist_matrix, df_train, df_test, train_genres, test_genres)
 
     for K in range(1, MAXIMUM_K + 1):
-        print("[FOLD-{}] Ejecutando Predictor con (P = {}) y (K = {})".format(1 + i_fold, P, K))
+        # print("[FOLD-{}] Ejecutando Predictor con (P = {}) y (K = {})".format(1 + i_fold, P, K))
         accuracy[K-1] = knn.predict_for(K)
 
     return (i_fold, i_P), accuracy
 
 if __name__ == "__main__":
-    # pcas_per_fold = []
     X_pcas_per_fold = []
+    genres_per_fold = [ (extract_genres_from(tr), extract_genres_from(tst)) for (tr, tst) in folds]
 
     X_all = vectorize_data(df_all, Q)
 
     with mp.Pool(processes=min(KFOLDS, MAX_PROCESSES)) as pool:
-        # pcas_per_fold = pool.map(pca_for_fold, range(KFOLDS))
         X_pcas_per_fold = pool.map(pca_for_fold, range(KFOLDS))
 
     with mp.Pool(processes=MAX_PROCESSES) as pool:
@@ -141,4 +138,4 @@ if __name__ == "__main__":
     plt.ylabel("P")
     plt.xticks(range(1, MAXIMUM_K + 1, 2))
     plt.title("Acuracies para PCA ({})".format(CTR_NAME))
-    plt.show()
+    plt.show()
\ No newline at end of file

src/experiments/full_dataset_analisys.py

@@ -12,7 +12,7 @@ from experiments.helpers import max_cpus_config_value, serialize_results
 import multiprocessing as mp
 import sys
 import numpy as np
-from classifier.data_models import distances, vectorize_data, extract_train_data_from, with_mean, with_tf_idf, do_not_center, with_tf_idf_and_mean
+from classifier.data_models import distances, vectorize_data, extract_train_data_from, extract_genres_from, with_mean, with_tf_idf, do_not_center, with_tf_idf_and_mean
 from classifier.knn import KNNPredictor
 from classifier.data_loader import *
 from classifier.pca import PCA
@@ -32,7 +32,7 @@ if len(selected_options := set(CENTERING_OPTIONS.keys()) & set(sys.argv)) > 1:
     print("Se especificaron múltiples opciones de centrado")
     exit(1)
 
-MAX_PROCESSES  = max_cpus_config_value()
+MAX_PROCESSES    = max_cpus_config_value()
 BEST_KNN_RESULTS = 10
 BEST_PCA_RESULTS = 50
 
@@ -69,10 +69,10 @@ def accuracies_for_P(P):
     X_pca = X_all_pca[:,:P]
 
     dist_matrix = distances(X_pca)
-    knn = KNNPredictor(dist_matrix, df_train, df_test, df_all)
+    knn = KNNPredictor(dist_matrix, df_train, df_test, genres_train, genres_test)
 
     for K in range(1, MAXIMUM_K + 1):
-        print("Ejecutando Predictor con (P = {}) y (K = {})".format(P, K))
+        # print("Ejecutando Predictor con (P = {}) y (K = {})".format(P, K))
         accuracy[K-1] = knn.predict_for(K)
 
     return P, accuracy
@@ -82,10 +82,10 @@ def accuracies_for_KNN():
 
     print("Generando KNN (sin PCA)")
     dist_matrix = distances(X_all, X_tn, center=CENTERING)
-    knn = KNNPredictor(dist_matrix, df_train, df_test, df_all)
+    knn = KNNPredictor(dist_matrix, df_train, df_test, genres_train, genres_test)
 
     for K in range(1, MAXIMUM_K + 1):
-        print("Ejecutando Predictor (sin PCA) con (K = {})".format(K))
+        # print("Ejecutando Predictor (sin PCA) con (K = {})".format(K))
         accuracy[K-1] = knn.predict_for(K)
 
     return accuracy
@@ -95,6 +95,9 @@ if __name__ == "__main__":
     df_train = load_df_train()
     df_test = load_df_real_world()
 
+    genres_train = extract_genres_from(df_train)
+    genres_test = extract_genres_from(df_test)
+
     X_all = vectorize_data(df_all, Q)
     X_tn = extract_train_data_from(X_all, df_train)
 
@@ -107,7 +110,6 @@ if __name__ == "__main__":
 
     print("Generando PCA para P = {}".format(P_MAX))
     pca = PCA(X_tn, P_MAX, iterations=ITERS, epsilon = EPSILON, center=CENTERING)
-    # X_all_pca = pca.transform(X_all)
     X_all_pca = pca.transform(CENTERING(X_all, X_tn))
 
     with mp.Pool(processes=MAX_PROCESSES) as pool: