updated report

README.md

@@ -1,10 +1,62 @@
 # Information Modelling & Analysis: Project 2
 
-Student: enter your name here
+Student: Claudio Maggioni
 
 Please follow the instructions provided in the project slides 
 and consider the submission instructions available on iCorsi.
 
 For your convencience, I the following resources are available in the `resources` folder:
 - **defects4j-checkout-closure-1f**: The output of the command `defects4j checkout -p Closure -v 1f -w ...`
-- **modified_classes** The list of buggy classes in: `framework/projects/Closure/modified_classes/`
+- **modified_classes** The list of buggy classes in: `framework/projects/Closure/modified_classes/`
+
+## Setup
+
+To install the required libraries run:
+
+```shell
+python3 -m venv .env
+source .env/bin/activate
+pip install -r requirements.txt
+```
+
+## Data pre-processing
+
+To extract and label the feature vectors run:
+
+```shell
+python3 ./extract_feature_vectors.py
+python3 ./label_feature_vectors.py
+```
+
+The labeled feature vectors are stored in file
+`./metrics/feature_vectors_labeled.csv` from the repository root.
+
+## Training
+
+To train the classifiers with the grid search procedure defined in the report
+to later extract the optimal combination of hyperparameters run:
+
+```shell
+python3 ./train_classifiers.py
+```
+
+and answer `y` to run again training when prompted. Answering `n` simply
+computes again data about the best hyperparameter configuration from the metrics
+produced by a previous training.
+
+Raw cross validation training metrics are stored in `./models/models.csv`. The
+optimal hyperparameter configurations found are stored in `./models/best.csv`.
+
+## Evaluation
+
+To run the 20-times 5-fold cross validation procedure delete the file
+`./models/evaluation.csv` and run:
+
+```shell
+python3 ./evaluate_classifiers.py
+```
+
+Raw data from the repeated cross validation procedure is stored in
+`./models/evaluation.csv`. P-values for each metric of each classifier pair are
+stored in `./models/model_stats.csv`.
+

evaluate_classifiers.py

@@ -2,13 +2,22 @@
 import os
 import pandas as pd
 import glob
+import re
+import itertools
+import statsmodels.stats.power as pw
+import numpy as np
 from train_classifiers import perform_grid_search, load_dataset
+import seaborn as sns
+import matplotlib.pyplot as plt
 
 from sklearn.neural_network import MLPClassifier
 from sklearn.naive_bayes import GaussianNB
 from sklearn.svm import SVC
 from sklearn.tree import DecisionTreeClassifier
 from sklearn.ensemble import RandomForestClassifier
+from sklearn.base import BaseEstimator, ClassifierMixin
+
+from scipy.stats import wilcoxon
 
 DIR: str = os.path.dirname(os.path.realpath(__file__))
 OUT_DIR: str = DIR + '/models'
@@ -37,6 +46,20 @@ RANDOM_STATES: list[int] = [
 ]
 
 
+class BiasedClassifier(BaseEstimator, ClassifierMixin):
+    def __init__(self, to_return=None):
+        self.to_return = to_return
+
+    def fit(self, X, y=None):
+        pass
+    
+    def predict(self, X, y=None):
+        return np.array([self.to_return] * len(X))
+    
+    def predict_proba(self, X, y=None):
+        return np.array([self.to_return] * len(X))
+
+
 def clean_output():
     filelist = glob.glob(OUT_DIR + '/models.csv')
     for f in filelist:
@@ -65,6 +88,7 @@ def main():
         df_best.loc[:, 'classifier'] = df_best['classifier'].apply(lambda x: string_to_classifier[x])
         df_best.loc[:, 'params'] = df_best['params'].apply(lambda x: eval(x))
         classifiers = [(e['classifier'], unit_grid(e['params'])) for e in df_best.to_dict('records')]
+        classifiers.append((BiasedClassifier(), { 'to_return': [1] }), )
 
         dfs: list[pd.DataFrame] = []
 
@@ -78,7 +102,122 @@ def main():
     else:
         df = pd.read_csv(OUT_DIR + '/evaluation.csv')
 
-    # TODO: statistical analysis
+    metrics_columns = list(df.filter(regex='^split\d_test'))
+    df = pd.melt(df, id_vars=['classifier'], value_vars=metrics_columns, var_name="metric")
+    df.loc[:, 'metric'] = df['metric'].apply(lambda x: re.sub('split\d_test_', '', x))
+
+    classifier_list = df['classifier'].unique()
+    metric_list = df['metric'].unique()
+
+    df_stats = pd.DataFrame(columns=['classifier_a', 'classifier_b', 'metric', 'pvalue'])
+
+    short_names = {
+        'MLPClassifier': "MLP",
+        'GaussianNB': "NB",
+        'SVC': "SVP",
+        'DecisionTreeClassifier': "DT",
+        'RandomForestClassifier': "RF",
+        'BiasedClassifier': 'Biased'
+    }
+
+    observations = {}
+
+    i = 1
+    for classifier_a in classifier_list:
+        for classifier_b in classifier_list:
+            if classifier_a >= classifier_b:
+                continue
+
+            for metric in metric_list:
+                if metric == 'accuracy':
+                    continue
+
+                series_a = list(df.loc[(df['classifier'] == classifier_a) & \
+                    (df['metric'] == metric), 'value'])
+                series_b = list(df.loc[(df['classifier'] == classifier_b) & \
+                    (df['metric'] == metric), 'value'])
+
+                df_stats.loc[i, 'classifier_a'] = classifier_a
+                df_stats.loc[i, 'classifier_b'] = classifier_b
+                df_stats.loc[i, 'metric'] = metric
+                pvalue = wilcoxon(series_a, series_b).pvalue
+                df_stats.loc[i, 'pvalue'] = pvalue
+
+                i += 1
+
+                if metric not in observations:
+                    observations[metric] = []
+                      
+                sa = short_names[classifier_a]
+                sb = short_names[classifier_b]
+
+                meana = np.round(np.mean(series_a), decimals=4)
+                meanb = np.round(np.mean(series_b), decimals=4)
+                pvalue = np.round(pvalue, decimals=4)
+
+                s = f"- Mean *{metric}* for *{sa}*: {meana},"
+                s += f" mean *{metric}* for *{sb}*: {meanb} $\\Rightarrow$ "
+
+                if pvalue < 0.05:
+                    better = sa if meana > meanb else sb 
+                    worse = sa if better == sb else sb 
+                    s += f"*{better}* is better than *{worse}* (*p-value* $= {pvalue}$)"
+                else:
+                    eff_size = (np.mean(series_a) - np.mean(series_b)) / np.sqrt((np.std(series_a) ** 2 + np.std(series_b) ** 2) / 2.0)
+                    power = pw.FTestAnovaPower().solve_power(effect_size=eff_size, nobs=len(series_a) + len(series_b), alpha=0.05)
+                    power = np.round(power, decimals=4)
+                    if power >= 0.8:
+                        s += f"*{sa}* is as effective as *{sb}* (*p-value* $= {pvalue}$, *5% corrected ANOVA power* $= {power}$)"
+                    else:
+                        s += f"statistical test inconclusive (*p-value* $= {pvalue}$, *5% corrected ANOVA power* $= {power}$)"
+                
+                observations[metric].append(s)
+
+    for metric in metric_list:
+        if metric == 'accuracy':
+            continue
+
+        print(metric + ":")
+        print("\n".join(observations[metric]))
+
+    df_stats.to_csv(OUT_DIR + '/model_stats.csv')
+
+    for metric in metric_list:
+        if metric == 'accuracy':
+            continue
+
+        print(metric)
+
+        dft = df_stats.loc[df_stats['metric'] == metric, :].copy()
+        dft.pvalue = dft.pvalue.apply(lambda x: '{0:.4g}'.format(round(x, 4)))
+
+        dft = dft \
+            .pivot(index=['classifier_a'], columns=['classifier_b'], values=['pvalue']) \
+            .reset_index(drop=False)
+        dft.columns = sorted([x[1] for x in dft.columns])
+        print(dft.replace({ np.nan: '--' }).to_markdown(index=False) + '\n')
+
+    dfg = df.loc[df['metric'] != 'accuracy', :].sort_values(by=['classifier'])
+    # Order by metric list
+    dfg = pd.concat([dfg[dfg['metric'] == met] for met in metric_list if met != 'accuracy'])
+
+    f, ax = plt.subplots(figsize=(8, 10))
+    plt.yticks(np.arange(0.0, 1.0 + 1, 0.1))
+    sns.boxplot(x="metric", y="value", hue="classifier", data=dfg, ax=ax)
+    
+    ax.set(ylabel="Metric value", ylim=[0, 1], xlabel="Metric")
+    ax.set_title("Distribution of metrics for each classifier")
+    sns.despine(offset=10, trim=True)
+    f.savefig(OUT_DIR + '/boxplot.svg')
+
+    # Print table of mean and standard deviation
+    dftab = dfg.groupby(['classifier', 'metric']) \
+        .agg({'value': ['mean', 'std']}) \
+        .reset_index(drop=False)
+    dftab.columns = [x[1] if x[0] == 'value' else x[0] for x in dftab.columns]
+    print(dftab.to_markdown(index=False))
+
+    print()
 
 
 if __name__ == '__main__':

grid_search_table.py

@@ -0,0 +1,30 @@
+#!/usr/bin/env python3
+
+import os
+import pandas as pd
+
+from train_classifiers import get_classifiers
+
+def main():
+    i = 0
+    df = pd.DataFrame(columns=['Classifier', 'Parameter', 'Values'])
+    for clazz, grid in get_classifiers():
+        for name, values in grid.items():
+            df.loc[i, 'Classifier'] = type(clazz).__name__
+            df.loc[i, 'Parameter'] = name
+            df.loc[i, 'Values'] = ', '.join([str(x) for x in values])
+            i += 1
+    
+    n1 = '5, 10, 15, ..., 100'
+    n2 = ', '.join([str(x) for x in range(15, 101, 15)])
+    n3 = ', '.join([str(x) for x in range(20, 101, 20)])
+
+    df.loc[(df.Classifier == 'MLPClassifier') & (df.Parameter == 'hidden_layer_sizes'), 'Values'] = f'$[{n1}]$, $[{n2}]^2$, $[{n3}]^3$'
+
+    for i in set(df['Classifier']):
+        print(i)
+        print(df.loc[df.Classifier == i, ['Parameter', 'Values']].to_markdown(index=False))
+        print()
+
+if __name__ == '__main__':
+    main()

metric_stats.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python3
+
+import pandas as pd
+import os
+
+def main():
+    dfs = pd.read_csv(os.path.dirname(__file__) + '/metrics/feature_vectors_labeled.csv')
+    metrics = ['MTH', 'FLD', 'RFC', 'INT', 'SZ', 'CPX', 'EX', 'RET', 'BCM', 
+        'NML', 'WRD', 'DCM']
+    df = dfs.agg(dict([(m, ['min', 'max', 'mean']) for m in metrics])).reset_index()
+    df = pd.melt(df, id_vars=['index'], value_vars=metrics, var_name='metric') \
+        .pivot(index='metric', columns=['index'], values=['value']) \
+        .reset_index()
+    df.columns = sorted([c[1] for c in df.columns])
+    df = df.reindex([df.columns[0]] + list(reversed(sorted(df.columns[1:]))), axis=1)
+    print(df.to_markdown(index=False))
+
+    print()
+    print(dfs.groupby('buggy').count().loc[:, 'class_name'])
+
+if __name__ == '__main__':
+    main()

models/evaluation.csv

@@ -1,101 +1,121 @@
-mean_fit_time,std_fit_time,mean_score_time,std_score_time,param_criterion,param_splitter,params,split0_test_precision,split1_test_precision,split2_test_precision,split3_test_precision,split4_test_precision,mean_test_precision,std_test_precision,rank_test_precision,split0_test_accuracy,split1_test_accuracy,split2_test_accuracy,split3_test_accuracy,split4_test_accuracy,mean_test_accuracy,std_test_accuracy,rank_test_accuracy,split0_test_recall,split1_test_recall,split2_test_recall,split3_test_recall,split4_test_recall,mean_test_recall,std_test_recall,rank_test_recall,split0_test_f1,split1_test_f1,split2_test_f1,split3_test_f1,split4_test_f1,mean_test_f1,std_test_f1,rank_test_f1,classifier,param_activation,param_hidden_layer_sizes,param_learning_rate,param_max_iter,param_solver,param_class_weight,param_max_features,param_gamma,param_kernel
-0.0021428585052490233,0.00020663926272374776,0.004525613784790039,0.00020776142457162384,gini,best,"{'criterion': 'gini', 'splitter': 'best'}",0.7884615384615384,0.8863636363636364,0.8269230769230769,0.86,0.8301886792452831,0.838387386198707,0.03304562031604819,1,0.8505747126436781,0.896551724137931,0.896551724137931,0.9080459770114943,0.896551724137931,0.8896551724137932,0.020040914682945644,1,0.9534883720930233,0.9069767441860465,1.0,0.9772727272727273,1.0,0.9675475687103594,0.03483499704423961,1,0.8631578947368421,0.896551724137931,0.9052631578947368,0.9148936170212766,0.9072164948453608,0.8974165777272294,0.018096736889836287,1,DecisionTreeClassifier,,,,,,,,,
-0.0013436317443847657,0.00023391341934461652,0.0057566165924072266,0.002408419907602289,,,{},0.8,0.8666666666666667,0.8,0.896551724137931,0.6923076923076923,0.8111052166224579,0.07035294426445517,1,0.6781609195402298,0.632183908045977,0.6436781609195402,0.7586206896551724,0.6091954022988506,0.664367816091954,0.05211854735059599,1,0.46511627906976744,0.3023255813953488,0.37209302325581395,0.5909090909090909,0.4090909090909091,0.4279069767441861,0.09715364127553945,1,0.5882352941176471,0.44827586206896547,0.5079365079365079,0.7123287671232877,0.5142857142857142,0.5542124291064245,0.09068240332458721,1,GaussianNB,,,,,,,,,
-0.8683316230773925,0.0859394739172999,0.004524374008178711,0.0003576277923583903,,,"{'activation': 'logistic', 'hidden_layer_sizes': (60, 80, 100), 'learning_rate': 'constant', 'max_iter': 500000, 'solver': 'lbfgs'}",0.7924528301886793,0.8085106382978723,0.82,0.9166666666666666,0.7818181818181819,0.8238896633942799,0.048202697589338095,1,0.8620689655172413,0.8390804597701149,0.8735632183908046,0.9540229885057471,0.8505747126436781,0.8758620689655172,0.04073573596935482,1,0.9767441860465116,0.8837209302325582,0.9534883720930233,1.0,0.9772727272727273,0.9582452431289641,0.04006049624755327,1,0.875,0.8444444444444444,0.8817204301075269,0.9565217391304348,0.8686868686868686,0.885274696473855,0.03777991211164225,1,MLPClassifier,logistic,"(60, 80, 100)",constant,500000,lbfgs,,,,
-0.15126414299011232,0.01811107239857173,0.013306856155395508,0.002210692523930667,gini,,"{'class_weight': 'balanced', 'criterion': 'gini', 'max_features': 'sqrt'}",0.8076923076923077,0.8837209302325582,0.8571428571428571,0.8979591836734694,0.8113207547169812,0.8515672066916349,0.036774852827802504,1,0.8735632183908046,0.8850574712643678,0.9080459770114943,0.9425287356321839,0.8735632183908046,0.8965517241379312,0.02621092931262383,1,0.9767441860465116,0.8837209302325582,0.9767441860465116,1.0,0.9772727272727273,0.9628964059196617,0.040584808213141546,1,0.8842105263157894,0.8837209302325582,0.9130434782608695,0.9462365591397849,0.88659793814433,0.9027618864186664,0.024346426830158686,1,RandomForestClassifier,,,,,,balanced,sqrt,,
-0.004443168640136719,0.0003713448777735096,0.005470561981201172,0.0002904982371310523,,,"{'gamma': 'scale', 'kernel': 'rbf'}",0.717391304347826,0.8064516129032258,0.6818181818181818,0.8421052631578947,0.725,0.7545532724454256,0.059839100951909266,1,0.735632183908046,0.7241379310344828,0.6896551724137931,0.7931034482758621,0.7011494252873564,0.728735632183908,0.03605606239392671,1,0.7674418604651163,0.5813953488372093,0.6976744186046512,0.7272727272727273,0.6590909090909091,0.6865750528541227,0.06346193424695262,1,0.7415730337078652,0.6756756756756758,0.6896551724137931,0.7804878048780488,0.6904761904761905,0.7155735754303147,0.03945456568462695,1,SVC,,,,,,,,scale,rbf
-0.0019789695739746093,0.00015863653469972515,0.0042498111724853516,0.00023481695648380676,gini,best,"{'criterion': 'gini', 'splitter': 'best'}",0.8571428571428571,0.8723404255319149,0.8541666666666666,0.8333333333333334,0.8367346938775511,0.8507435953104647,0.014272053717617132,1,0.896551724137931,0.896551724137931,0.8850574712643678,0.8620689655172413,0.8735632183908046,0.8827586206896552,0.013404487114586916,1,0.9545454545454546,0.9318181818181818,0.9318181818181818,0.9090909090909091,0.9318181818181818,0.9318181818181819,0.014373989364401745,1,0.9032258064516128,0.9010989010989012,0.8913043478260869,0.8695652173913043,0.8817204301075268,0.8893829405750864,0.012522273686114336,1,DecisionTreeClassifier,,,,,,,,,
-0.0010324954986572266,2.87824059371415e-05,0.005038261413574219,0.000740872543736187,,,{},0.75,0.8,0.7391304347826086,0.6666666666666666,0.7692307692307693,0.7450055741360089,0.04428926038920083,1,0.632183908045977,0.5977011494252874,0.6206896551724138,0.5632183908045977,0.6551724137931034,0.6137931034482758,0.03135214183215139,1,0.4090909090909091,0.2727272727272727,0.38636363636363635,0.2727272727272727,0.45454545454545453,0.3590909090909091,0.07385489458759965,1,0.5294117647058824,0.4067796610169491,0.5074626865671641,0.3870967741935484,0.5714285714285714,0.4804358915824231,0.07147892377055552,1,GaussianNB,,,,,,,,,
-0.936018705368042,0.1435024964787475,0.0041712760925292965,0.0005843526359750852,,,"{'activation': 'logistic', 'hidden_layer_sizes': (60, 80, 100), 'learning_rate': 'constant', 'max_iter': 500000, 'solver': 'lbfgs'}",0.8235294117647058,0.8863636363636364,0.8478260869565217,0.7884615384615384,0.8723404255319149,0.8437042198156635,0.03497320291265929,1,0.8735632183908046,0.8850574712643678,0.8620689655172413,0.8390804597701149,0.896551724137931,0.871264367816092,0.01977546038400606,1,0.9545454545454546,0.8863636363636364,0.8863636363636364,0.9318181818181818,0.9318181818181818,0.9181818181818182,0.02727272727272728,1,0.8842105263157896,0.8863636363636365,0.8666666666666666,0.8541666666666667,0.9010989010989012,0.8785012794223321,0.016354180500685624,1,MLPClassifier,logistic,"(60, 80, 100)",constant,500000,lbfgs,,,,
-0.17002401351928711,0.004173933521887817,0.014090013504028321,0.0019074009971583154,gini,,"{'class_weight': 'balanced', 'criterion': 'gini', 'max_features': 'sqrt'}",0.84,0.8913043478260869,0.9333333333333333,0.8723404255319149,0.84,0.875395621338267,0.03499971511583098,1,0.8850574712643678,0.9080459770114943,0.9425287356321839,0.896551724137931,0.8850574712643678,0.903448275862069,0.021318663208036073,1,0.9545454545454546,0.9318181818181818,0.9545454545454546,0.9318181818181818,0.9545454545454546,0.9454545454545453,0.011134044285378127,1,0.8936170212765958,0.9111111111111111,0.9438202247191012,0.9010989010989012,0.8936170212765958,0.9086528558964609,0.018718983557465988,1,RandomForestClassifier,,,,,,balanced,sqrt,,
-0.004750776290893555,0.000524355936126991,0.006039905548095703,0.0009508248484377138,,,"{'gamma': 'scale', 'kernel': 'rbf'}",0.7209302325581395,0.78125,0.7894736842105263,0.7073170731707317,0.723404255319149,0.7444750490517091,0.03393029940101563,1,0.7126436781609196,0.7011494252873564,0.7471264367816092,0.6896551724137931,0.735632183908046,0.7172413793103448,0.021318663208036076,1,0.7045454545454546,0.5681818181818182,0.6818181818181818,0.6590909090909091,0.7727272727272727,0.6772727272727274,0.0664942674469445,1,0.7126436781609196,0.6578947368421052,0.7317073170731707,0.6823529411764706,0.7472527472527473,0.7063702841010826,0.03249433165775736,1,SVC,,,,,,,,scale,rbf
-0.001989936828613281,0.00011800323750563928,0.004775238037109375,0.0007958635643620779,gini,best,"{'criterion': 'gini', 'splitter': 'best'}",0.851063829787234,0.8936170212765957,0.7777777777777778,0.8125,0.8085106382978723,0.828693853427896,0.03994366880311656,1,0.8850574712643678,0.9310344827586207,0.8390804597701149,0.8505747126436781,0.8390804597701149,0.8689655172413794,0.03531561263387866,1,0.9302325581395349,0.9767441860465116,0.9545454545454546,0.9069767441860465,0.8837209302325582,0.9304439746300212,0.03304054899887746,1,0.888888888888889,0.9333333333333332,0.8571428571428572,0.8571428571428572,0.8444444444444444,0.8761904761904763,0.03212472543663111,1,DecisionTreeClassifier,,,,,,,,,
-0.0009866714477539062,2.2157591981926944e-05,0.004707145690917969,0.0006455812049327525,,,{},0.8333333333333334,0.8421052631578947,0.782608695652174,0.782608695652174,0.9523809523809523,0.8386073880353055,0.06205833034255249,1,0.6896551724137931,0.6551724137931034,0.6436781609195402,0.6551724137931034,0.7241379310344828,0.6735632183908046,0.02961861776258651,1,0.46511627906976744,0.37209302325581395,0.4090909090909091,0.4186046511627907,0.46511627906976744,0.4260042283298097,0.035515453449528465,1,0.5970149253731343,0.5161290322580645,0.537313432835821,0.5454545454545454,0.625,0.564182387184313,0.0403944433699087,1,GaussianNB,,,,,,,,,
-1.7184324264526367,0.41735925133386115,0.004328155517578125,0.0004639785456082588,,,"{'activation': 'logistic', 'hidden_layer_sizes': (60, 80, 100), 'learning_rate': 'constant', 'max_iter': 500000, 'solver': 'lbfgs'}",0.7547169811320755,0.8163265306122449,0.8269230769230769,0.7547169811320755,0.8,0.7905367139598946,0.030478655990839956,1,0.8160919540229885,0.8620689655172413,0.8850574712643678,0.8160919540229885,0.8160919540229885,0.839080459770115,0.029078415265916133,1,0.9302325581395349,0.9302325581395349,0.9772727272727273,0.9302325581395349,0.8372093023255814,0.9210359408033826,0.04570167345332005,1,0.8333333333333334,0.8695652173913043,0.8958333333333334,0.8333333333333334,0.8181818181818183,0.8500494071146246,0.028465530806464814,1,MLPClassifier,logistic,"(60, 80, 100)",constant,500000,lbfgs,,,,
-0.17911319732666015,0.005977100154862893,0.013374042510986329,0.0025833323949244315,gini,,"{'class_weight': 'balanced', 'criterion': 'gini', 'max_features': 'sqrt'}",0.8888888888888888,0.851063829787234,0.875,0.8478260869565217,0.925,0.8775557611265288,0.028183695479612123,1,0.9080459770114943,0.8850574712643678,0.9080459770114943,0.8735632183908046,0.896551724137931,0.8942528735632184,0.013404487114586881,1,0.9302325581395349,0.9302325581395349,0.9545454545454546,0.9069767441860465,0.8604651162790697,0.9164904862579281,0.03179692771335431,1,0.9090909090909092,0.888888888888889,0.9130434782608695,0.8764044943820224,0.891566265060241,0.8957988071365861,0.013534100682265786,1,RandomForestClassifier,,,,,,balanced,sqrt,,
-0.004246282577514649,0.00020742593961810567,0.005492544174194336,0.000505823063241848,,,"{'gamma': 'scale', 'kernel': 'rbf'}",0.717391304347826,0.7021276595744681,0.7142857142857143,0.8085106382978723,0.7446808510638298,0.7373992335139421,0.03817981082531593,1,0.735632183908046,0.7241379310344828,0.7701149425287356,0.8390804597701149,0.7701149425287356,0.7678160919540229,0.04008182936589124,1,0.7674418604651163,0.7674418604651163,0.9090909090909091,0.8837209302325582,0.813953488372093,0.8283298097251585,0.05867179006586292,1,0.7415730337078652,0.7333333333333334,0.8,0.8444444444444444,0.7777777777777778,0.7794257178526842,0.04052353688322142,1,SVC,,,,,,,,scale,rbf
-0.0023474693298339844,0.0004159644612266083,0.0050795555114746095,0.0007537542711967322,gini,best,"{'criterion': 'gini', 'splitter': 'best'}",0.7272727272727273,0.84,0.7818181818181819,0.7719298245614035,0.9333333333333333,0.8108708133971293,0.07098545529627763,1,0.7931034482758621,0.896551724137931,0.8620689655172413,0.8505747126436781,0.9540229885057471,0.871264367816092,0.05312285061283374,1,0.9302325581395349,0.9767441860465116,1.0,1.0,0.9767441860465116,0.9767441860465116,0.025475467790937966,1,0.8163265306122448,0.9032258064516129,0.8775510204081634,0.8712871287128713,0.9545454545454545,0.8845871881460694,0.04500549685151739,1,DecisionTreeClassifier,,,,,,,,,
-0.001402568817138672,0.00027041629894149975,0.004859399795532226,0.0005202325826066542,,,{},0.625,0.782608695652174,0.7894736842105263,0.875,0.9523809523809523,0.8048926664487306,0.10932842678338679,1,0.5517241379310345,0.6551724137931034,0.632183908045977,0.7701149425287356,0.7241379310344828,0.6666666666666666,0.0755479389662298,1,0.23255813953488372,0.4186046511627907,0.3488372093023256,0.6363636363636364,0.46511627906976744,0.4202959830866807,0.13345321620743475,1,0.33898305084745767,0.5454545454545454,0.48387096774193555,0.7368421052631579,0.625,0.5460301338614193,0.13369969035673737,1,GaussianNB,,,,,,,,,
-1.0924948692321776,0.1974697751548748,0.004323148727416992,0.00040777369083626287,,,"{'activation': 'logistic', 'hidden_layer_sizes': (60, 80, 100), 'learning_rate': 'constant', 'max_iter': 500000, 'solver': 'lbfgs'}",0.8235294117647058,0.8541666666666666,0.7843137254901961,0.7818181818181819,0.9555555555555556,0.8398767082590611,0.06373550073933776,1,0.8850574712643678,0.896551724137931,0.8390804597701149,0.8505747126436781,0.9770114942528736,0.8896551724137931,0.04854876340676525,1,0.9767441860465116,0.9534883720930233,0.9302325581395349,0.9772727272727273,1.0,0.9675475687103594,0.023758736496393568,1,0.8936170212765957,0.9010989010989011,0.851063829787234,0.8686868686868686,0.9772727272727273,0.8983478696244653,0.04330043338122906,1,MLPClassifier,logistic,"(60, 80, 100)",constant,500000,lbfgs,,,,
-0.18673672676086425,0.00425871820676779,0.01402268409729004,0.002114995380243859,gini,,"{'class_weight': 'balanced', 'criterion': 'gini', 'max_features': 'sqrt'}",0.8695652173913043,0.875,0.9130434782608695,0.88,0.9347826086956522,0.8944782608695652,0.025214242681993553,1,0.896551724137931,0.9195402298850575,0.9425287356321839,0.9310344827586207,0.9655172413793104,0.9310344827586207,0.022988505747126443,1,0.9302325581395349,0.9767441860465116,0.9767441860465116,1.0,1.0,0.9767441860465116,0.025475467790937966,1,0.898876404494382,0.923076923076923,0.9438202247191011,0.9361702127659575,0.9662921348314606,0.9336471799775647,0.022344067534023297,1,RandomForestClassifier,,,,,,balanced,sqrt,,
-0.004868793487548828,0.0006423848080842837,0.006475830078125,0.0013074481211800904,,,"{'gamma': 'scale', 'kernel': 'rbf'}",0.6944444444444444,0.75,0.8095238095238095,0.8076923076923077,0.8695652173913043,0.7862451558103731,0.059473003340029494,1,0.6666666666666666,0.735632183908046,0.8045977011494253,0.8620689655172413,0.896551724137931,0.793103448275862,0.0835213890785515,1,0.5813953488372093,0.6976744186046512,0.7906976744186046,0.9545454545454546,0.9302325581395349,0.7909090909090909,0.1405536516486544,1,0.6329113924050633,0.7228915662650603,0.8,0.875,0.898876404494382,0.7859358726329011,0.0982617507617456,1,SVC,,,,,,,,scale,rbf
-0.0022649288177490233,0.0002710619149921518,0.00461268424987793,0.0002385657809741055,gini,best,"{'criterion': 'gini', 'splitter': 'best'}",0.8431372549019608,0.875,0.7678571428571429,0.8148148148148148,0.8113207547169812,0.8224259934581799,0.03564266639355289,1,0.896551724137931,0.9195402298850575,0.8505747126436781,0.8850574712643678,0.8850574712643678,0.8873563218390805,0.022288183252488877,1,0.9772727272727273,0.9767441860465116,1.0,1.0,1.0,0.9908033826638478,0.011264749940598078,1,0.9052631578947369,0.923076923076923,0.8686868686868687,0.8979591836734693,0.8958333333333334,0.8981638933330662,0.017580824817308816,1,DecisionTreeClassifier,,,,,,,,,
-0.001177835464477539,0.00022100300951762638,0.004941177368164062,0.0005995174698562829,,,{},0.9375,0.8095238095238095,0.95,0.7619047619047619,0.8333333333333334,0.8584523809523809,0.07345189858605215,1,0.6551724137931034,0.6551724137931034,0.7126436781609196,0.6206896551724138,0.5977011494252874,0.6482758620689656,0.03887709086273048,1,0.3409090909090909,0.3953488372093023,0.4418604651162791,0.36363636363636365,0.23255813953488372,0.354862579281184,0.06991112119272025,1,0.5,0.53125,0.6031746031746031,0.4923076923076923,0.3636363636363636,0.4980737318237319,0.07779102983602866,1,GaussianNB,,,,,,,,,
-1.8019145488739015,0.5072924351385918,0.004171037673950195,0.0005703277922583684,,,"{'activation': 'logistic', 'hidden_layer_sizes': (60, 80, 100), 'learning_rate': 'constant', 'max_iter': 500000, 'solver': 'lbfgs'}",0.8269230769230769,0.84,0.7735849056603774,0.8461538461538461,0.7543859649122807,0.8082095587299163,0.03713877412907954,1,0.8850574712643678,0.896551724137931,0.8390804597701149,0.9080459770114943,0.8390804597701149,0.8735632183908045,0.029078415265916147,1,0.9772727272727273,0.9767441860465116,0.9534883720930233,1.0,1.0,0.9815010570824525,0.017376069138229963,1,0.8958333333333334,0.9032258064516129,0.8541666666666667,0.9166666666666666,0.86,0.8859784946236559,0.024589534747539915,1,MLPClassifier,logistic,"(60, 80, 100)",constant,500000,lbfgs,,,,
-0.18393878936767577,0.0022903507457411342,0.012638616561889648,0.0008147810915167272,gini,,"{'class_weight': 'balanced', 'criterion': 'gini', 'max_features': 'sqrt'}",0.9772727272727273,0.8367346938775511,0.8269230769230769,0.88,0.8958333333333334,0.8833527662813376,0.053564016249746035,1,0.9770114942528736,0.8850574712643678,0.896551724137931,0.9310344827586207,0.9425287356321839,0.9264367816091955,0.03299471286070649,1,0.9772727272727273,0.9534883720930233,1.0,1.0,1.0,0.9861522198731502,0.018552926367390112,1,0.9772727272727273,0.8913043478260869,0.9052631578947368,0.9361702127659575,0.945054945054945,0.9310130781628907,0.03034325907898099,1,RandomForestClassifier,,,,,,balanced,sqrt,,
-0.004398775100708008,0.00020901091710458136,0.005275821685791016,0.00036392815452945213,,,"{'gamma': 'scale', 'kernel': 'rbf'}",0.8108108108108109,0.7560975609756098,0.673469387755102,0.75,0.6904761904761905,0.7361707900035427,0.04935365888461486,1,0.7586206896551724,0.7471264367816092,0.7011494252873564,0.7241379310344828,0.6896551724137931,0.7241379310344828,0.02621092931262383,1,0.6818181818181818,0.7209302325581395,0.7674418604651163,0.6818181818181818,0.6744186046511628,0.7052854122621564,0.035106597889331737,1,0.7407407407407407,0.7380952380952381,0.7173913043478259,0.7142857142857143,0.6823529411764705,0.7185731877291979,0.020996172957040878,1,SVC,,,,,,,,scale,rbf
-0.00206756591796875,0.0001717024992325036,0.004828643798828125,0.00045015684067216416,gini,best,"{'criterion': 'gini', 'splitter': 'best'}",0.8723404255319149,0.82,0.7735849056603774,0.8333333333333334,0.7962962962962963,0.8191109921643843,0.03355465034778286,1,0.896551724137931,0.8735632183908046,0.8275862068965517,0.8620689655172413,0.8735632183908046,0.8666666666666668,0.022524043611799355,1,0.9318181818181818,0.9534883720930233,0.9318181818181818,0.9090909090909091,1.0,0.9452431289640593,0.030769271598132378,1,0.9010989010989012,0.8817204301075269,0.8453608247422681,0.8695652173913043,0.8865979381443299,0.876868662296866,0.018715699648497057,1,DecisionTreeClassifier,,,,,,,,,
-0.0016726970672607422,0.0002812474893888989,0.005605936050415039,0.0007809245863278931,,,{},0.7368421052631579,0.7391304347826086,0.84,0.8148148148148148,0.8,0.7861574709721163,0.04136526157514908,1,0.5977011494252874,0.632183908045977,0.6896551724137931,0.6896551724137931,0.6436781609195402,0.6505747126436783,0.03531561263387867,1,0.3181818181818182,0.3953488372093023,0.4772727272727273,0.5,0.37209302325581395,0.41257928118393233,0.0673417112152376,1,0.44444444444444436,0.5151515151515151,0.6086956521739131,0.6197183098591549,0.5079365079365079,0.5391892859131071,0.06610173097256672,1,GaussianNB,,,,,,,,,
-1.5445869445800782,0.3960126734423616,0.004444551467895508,0.0005046720090545574,,,"{'activation': 'logistic', 'hidden_layer_sizes': (60, 80, 100), 'learning_rate': 'constant', 'max_iter': 500000, 'solver': 'lbfgs'}",0.7962962962962963,0.8297872340425532,0.7708333333333334,0.7884615384615384,0.8431372549019608,0.8057031314071363,0.02676946578992968,1,0.8620689655172413,0.8620689655172413,0.7931034482758621,0.8390804597701149,0.9080459770114943,0.8528735632183908,0.03735190071096992,1,0.9772727272727273,0.9069767441860465,0.8409090909090909,0.9318181818181818,1.0,0.9313953488372093,0.05584897837553966,1,0.8775510204081632,0.8666666666666666,0.8043478260869567,0.8541666666666667,0.9148936170212766,0.863525159369946,0.03587436535354627,1,MLPClassifier,logistic,"(60, 80, 100)",constant,500000,lbfgs,,,,
-0.18384242057800293,0.006076069359904161,0.012868118286132813,0.0007459176849424791,gini,,"{'class_weight': 'balanced', 'criterion': 'gini', 'max_features': 'sqrt'}",0.8431372549019608,0.8888888888888888,0.8666666666666667,0.8913043478260869,0.8775510204081632,0.8735096357383533,0.01754316812431488,1,0.896551724137931,0.9080459770114943,0.8735632183908046,0.9080459770114943,0.9310344827586207,0.903448275862069,0.018675950355484944,1,0.9772727272727273,0.9302325581395349,0.8863636363636364,0.9318181818181818,1.0,0.9451374207188159,0.039740395854914776,1,0.9052631578947369,0.9090909090909092,0.8764044943820225,0.9111111111111111,0.9347826086956522,0.9073304562348863,0.01861022331533133,1,RandomForestClassifier,,,,,,balanced,sqrt,,
-0.004279470443725586,0.0002558823265505084,0.005733108520507813,0.0006832044460948641,,,"{'gamma': 'scale', 'kernel': 'rbf'}",0.717391304347826,0.7906976744186046,0.782608695652174,0.7272727272727273,0.7837837837837838,0.7603508370950232,0.03132133010229837,1,0.7241379310344828,0.7931034482758621,0.7931034482758621,0.7241379310344828,0.7471264367816092,0.7563218390804598,0.03118312635919665,1,0.75,0.7906976744186046,0.8181818181818182,0.7272727272727273,0.6744186046511628,0.7521141649048626,0.05002357207543973,1,0.7333333333333332,0.7906976744186046,0.8,0.7272727272727273,0.725,0.7552607470049331,0.03297644647693505,1,SVC,,,,,,,,scale,rbf
-0.0022994041442871093,0.00023470608880696303,0.004636621475219727,0.0002043977704776254,gini,best,"{'criterion': 'gini', 'splitter': 'best'}",0.8367346938775511,0.8571428571428571,0.8076923076923077,0.8936170212765957,0.82,0.8430373759978623,0.030255523340888345,1,0.8850574712643678,0.896551724137931,0.8620689655172413,0.9195402298850575,0.8735632183908046,0.8873563218390805,0.019775460384006044,1,0.9534883720930233,0.9545454545454546,0.9545454545454546,0.9545454545454546,0.9534883720930233,0.9541226215644821,0.000517862524901312,1,0.8913043478260869,0.9032258064516128,0.875,0.9230769230769231,0.8817204301075269,0.89486550149243,0.017005245505571434,1,DecisionTreeClassifier,,,,,,,,,
-0.0012067317962646484,0.0001254314321262609,0.004871988296508789,6.468794751091687e-05,,,{},0.8636363636363636,0.782608695652174,0.7142857142857143,0.7272727272727273,0.8333333333333334,0.7842273668360625,0.058061123902532125,1,0.6896551724137931,0.6436781609195402,0.5632183908045977,0.6091954022988506,0.5977011494252874,0.6206896551724138,0.043007556169815435,1,0.4418604651162791,0.4090909090909091,0.22727272727272727,0.36363636363636365,0.23255813953488372,0.33488372093023255,0.08925031719584993,1,0.5846153846153846,0.537313432835821,0.3448275862068965,0.4848484848484849,0.3636363636363636,0.46304825042859016,0.09447537704369392,1,GaussianNB,,,,,,,,,
-1.4954755306243896,0.6153562765333447,0.004317426681518554,0.0006817551917294555,,,"{'activation': 'logistic', 'hidden_layer_sizes': (60, 80, 100), 'learning_rate': 'constant', 'max_iter': 500000, 'solver': 'lbfgs'}",0.8163265306122449,0.7843137254901961,0.7884615384615384,0.875,0.8541666666666666,0.8236536922461293,0.03579746610560267,1,0.8620689655172413,0.8275862068965517,0.8390804597701149,0.9080459770114943,0.896551724137931,0.8666666666666668,0.0313521418321514,1,0.9302325581395349,0.9090909090909091,0.9318181818181818,0.9545454545454546,0.9534883720930233,0.9358350951374208,0.016878929096162457,1,0.8695652173913043,0.8421052631578948,0.8541666666666667,0.9130434782608695,0.9010989010989011,0.8759959053151274,0.027089215903989865,1,MLPClassifier,logistic,"(60, 80, 100)",constant,500000,lbfgs,,,,
-0.18642067909240723,0.0027648207169169367,0.013179731369018555,0.0014817875495277486,gini,,"{'class_weight': 'balanced', 'criterion': 'gini', 'max_features': 'sqrt'}",0.851063829787234,0.82,0.9333333333333333,0.9333333333333333,0.9318181818181818,0.8939097356544166,0.04867013678331812,1,0.8850574712643678,0.8620689655172413,0.9425287356321839,0.9425287356321839,0.9425287356321839,0.9149425287356323,0.034559302019248055,1,0.9302325581395349,0.9318181818181818,0.9545454545454546,0.9545454545454546,0.9534883720930233,0.94492600422833,0.011367446138494802,1,0.888888888888889,0.8723404255319149,0.9438202247191012,0.9438202247191012,0.942528735632184,0.9182796998982381,0.031199003994897112,1,RandomForestClassifier,,,,,,balanced,sqrt,,
-0.004606199264526367,0.00030640835253414975,0.005708837509155273,0.0004622477857420305,,,"{'gamma': 'scale', 'kernel': 'rbf'}",0.7727272727272727,0.6181818181818182,0.6666666666666666,0.7647058823529411,0.8857142857142857,0.7415991851285969,0.09286416779158137,1,0.7816091954022989,0.6436781609195402,0.6206896551724138,0.7011494252873564,0.8160919540229885,0.7126436781609196,0.07589689204238703,1,0.7906976744186046,0.7727272727272727,0.5,0.5909090909090909,0.7209302325581395,0.6750528541226215,0.11202002052077396,1,0.7816091954022988,0.6868686868686869,0.5714285714285715,0.6666666666666667,0.7948717948717948,0.7002889830476038,0.08182231409702262,1,SVC,,,,,,,,scale,rbf
-0.002074527740478516,5.369457420975792e-05,0.004720640182495117,0.00042156321788053064,gini,best,"{'criterion': 'gini', 'splitter': 'best'}",0.9148936170212766,0.8775510204081632,0.8,0.84,0.8571428571428571,0.8579174989144593,0.038230424320156525,1,0.9540229885057471,0.9310344827586207,0.8735632183908046,0.896551724137931,0.9080459770114943,0.9126436781609195,0.027777117180677144,1,1.0,1.0,1.0,0.9767441860465116,0.9767441860465116,0.9906976744186047,0.011392975547828756,1,0.9555555555555556,0.9347826086956522,0.888888888888889,0.9032258064516129,0.9130434782608695,0.9190992675705159,0.023564101742434385,1,DecisionTreeClassifier,,,,,,,,,
-0.001164388656616211,0.00016108427312301984,0.004695367813110351,0.0003719933595314699,,,{},0.9166666666666666,0.75,0.8125,0.8260869565217391,0.92,0.8450507246376813,0.06511536141307574,1,0.735632183908046,0.5977011494252874,0.7241379310344828,0.6781609195402298,0.7471264367816092,0.6965517241379311,0.05469138967066811,1,0.5116279069767442,0.27906976744186046,0.5909090909090909,0.4418604651162791,0.5348837209302325,0.4716701902748414,0.10751188547866333,1,0.6567164179104478,0.4067796610169491,0.6842105263157896,0.5757575757575758,0.6764705882352942,0.5999869538472113,0.10399779742560995,1,GaussianNB,,,,,,,,,
-1.9322119235992432,0.6113837699210615,0.004147720336914062,0.0007077111722888735,,,"{'activation': 'logistic', 'hidden_layer_sizes': (60, 80, 100), 'learning_rate': 'constant', 'max_iter': 500000, 'solver': 'lbfgs'}",0.82,0.8235294117647058,0.7884615384615384,0.82,0.8723404255319149,0.8248662751516317,0.026937885140533396,1,0.8735632183908046,0.8850574712643678,0.8390804597701149,0.8735632183908046,0.9080459770114943,0.8758620689655172,0.02228818325248888,1,0.9534883720930233,0.9767441860465116,0.9318181818181818,0.9534883720930233,0.9534883720930233,0.9538054968287526,0.0142121580957787,1,0.8817204301075269,0.8936170212765957,0.8541666666666667,0.8817204301075269,0.9111111111111112,0.8844671318538856,0.01858402022510766,1,MLPClassifier,logistic,"(60, 80, 100)",constant,500000,lbfgs,,,,
-0.19132018089294434,0.0022244393787120087,0.012306642532348634,0.0004950663228963199,gini,,"{'class_weight': 'balanced', 'criterion': 'gini', 'max_features': 'sqrt'}",0.8958333333333334,0.8958333333333334,0.8461538461538461,0.9111111111111111,0.8775510204081632,0.8852965288679574,0.022273678027797773,1,0.9425287356321839,0.9425287356321839,0.9080459770114943,0.9310344827586207,0.9310344827586207,0.9310344827586207,0.012591323161038287,1,1.0,1.0,1.0,0.9534883720930233,1.0,0.9906976744186047,0.018604651162790687,1,0.945054945054945,0.945054945054945,0.9166666666666666,0.9318181818181819,0.9347826086956522,0.9346754694580781,0.010468615732609477,1,RandomForestClassifier,,,,,,balanced,sqrt,,
-0.004273223876953125,0.00024021906028259382,0.005591773986816406,0.0009051024902601005,,,"{'gamma': 'scale', 'kernel': 'rbf'}",0.7608695652173914,0.7209302325581395,0.78,0.7272727272727273,0.6808510638297872,0.7339847177756091,0.03428952260559052,1,0.7816091954022989,0.7241379310344828,0.8160919540229885,0.735632183908046,0.7011494252873564,0.7517241379310345,0.0415068277822248,1,0.813953488372093,0.7209302325581395,0.8863636363636364,0.7441860465116279,0.7441860465116279,0.7819238900634249,0.060831031066091654,1,0.7865168539325844,0.7209302325581395,0.8297872340425532,0.735632183908046,0.711111111111111,0.7567955231104868,0.044793791674645,1,SVC,,,,,,,,scale,rbf
-0.0024343490600585937,0.00020424280894957623,0.006906652450561523,0.0006866707168031187,gini,best,"{'criterion': 'gini', 'splitter': 'best'}",0.78,0.7777777777777778,0.803921568627451,0.7777777777777778,0.7843137254901961,0.7847581699346404,0.009874465157950648,1,0.8275862068965517,0.8505747126436781,0.8505747126436781,0.8505747126436781,0.8390804597701149,0.8436781609195403,0.009195402298850562,1,0.9069767441860465,0.9767441860465116,0.9318181818181818,0.9767441860465116,0.9302325581395349,0.9445031712473572,0.027755209074470696,1,0.8387096774193548,0.8659793814432991,0.8631578947368421,0.8659793814432991,0.851063829787234,0.8569780329660059,0.010667238198752621,1,DecisionTreeClassifier,,,,,,,,,
-0.0013713836669921875,0.000203211862934177,0.006127309799194336,0.00048237270682944654,,,{},0.7,0.8636363636363636,0.8666666666666667,0.75,0.8666666666666667,0.8093939393939393,0.07070678210619315,1,0.5977011494252874,0.6896551724137931,0.6206896551724138,0.6206896551724138,0.632183908045977,0.632183908045977,0.030842316931031583,1,0.32558139534883723,0.4418604651162791,0.29545454545454547,0.3488372093023256,0.3023255813953488,0.3428118393234672,0.0529635231418738,1,0.44444444444444453,0.5846153846153846,0.44067796610169496,0.4761904761904763,0.44827586206896547,0.47884082668419314,0.054348897991092215,1,GaussianNB,,,,,,,,,
-1.30067138671875,0.3074056367912712,0.004353380203247071,0.00042067031364921955,,,"{'activation': 'logistic', 'hidden_layer_sizes': (60, 80, 100), 'learning_rate': 'constant', 'max_iter': 500000, 'solver': 'lbfgs'}",0.78,0.7241379310344828,0.86,0.7166666666666667,0.84,0.7841609195402299,0.05838684030347587,1,0.8275862068965517,0.8045977011494253,0.9080459770114943,0.8045977011494253,0.896551724137931,0.8482758620689654,0.045048087223598675,1,0.9069767441860465,0.9767441860465116,0.9772727272727273,1.0,0.9767441860465116,0.9675475687103594,0.0315775848667449,1,0.8387096774193548,0.8316831683168318,0.9148936170212766,0.8349514563106796,0.9032258064516129,0.8646927451039511,0.03648073624019725,1,MLPClassifier,logistic,"(60, 80, 100)",constant,500000,lbfgs,,,,
-0.1895288944244385,0.002014413546974869,0.013449954986572265,0.0011055928745813503,gini,,"{'class_weight': 'balanced', 'criterion': 'gini', 'max_features': 'sqrt'}",0.8297872340425532,0.8076923076923077,0.8723404255319149,0.7678571428571429,0.84,0.8235354220247837,0.03476541641217675,1,0.8620689655172413,0.8735632183908046,0.896551724137931,0.8505747126436781,0.896551724137931,0.8758620689655172,0.018390804597701173,1,0.9069767441860465,0.9767441860465116,0.9318181818181818,1.0,0.9767441860465116,0.9584566596194503,0.03392493645746217,1,0.8666666666666666,0.8842105263157894,0.9010989010989012,0.8686868686868687,0.9032258064516129,0.8847777538439677,0.015452898133897993,1,RandomForestClassifier,,,,,,balanced,sqrt,,
-0.004441690444946289,0.00011799946083353337,0.006065845489501953,0.0010027435685364772,,,"{'gamma': 'scale', 'kernel': 'rbf'}",0.7307692307692307,0.7755102040816326,0.7894736842105263,0.6875,0.75,0.7466506238122779,0.03585783913849692,1,0.7816091954022989,0.8160919540229885,0.7471264367816092,0.7126436781609196,0.735632183908046,0.7586206896551724,0.03634801908239517,1,0.8837209302325582,0.8837209302325582,0.6818181818181818,0.7674418604651163,0.6976744186046512,0.782875264270613,0.08723486075604465,1,0.8,0.826086956521739,0.7317073170731707,0.7252747252747254,0.7228915662650603,0.761192113026939,0.04322901021574641,1,SVC,,,,,,,,scale,rbf
-0.002072000503540039,0.00021045339651355993,0.004689788818359375,0.0001804676263642823,gini,best,"{'criterion': 'gini', 'splitter': 'best'}",0.8163265306122449,0.8297872340425532,0.84,0.7962962962962963,0.7884615384615384,0.8141743198825265,0.019473734664295157,1,0.8620689655172413,0.8505747126436781,0.8850574712643678,0.8620689655172413,0.8505747126436781,0.8620689655172413,0.012591323161038327,1,0.9302325581395349,0.8863636363636364,0.9545454545454546,0.9772727272727273,0.9534883720930233,0.9403805496828752,0.030836021021931347,1,0.8695652173913043,0.8571428571428571,0.8936170212765958,0.8775510204081632,0.8631578947368421,0.8722068021911525,0.012670763879318698,1,DecisionTreeClassifier,,,,,,,,,
-0.001321697235107422,0.0001672977988012324,0.004941225051879883,0.0004893872700529387,,,{},0.782608695652174,0.85,0.8461538461538461,0.875,0.9130434782608695,0.8533612040133779,0.04265998708448138,1,0.6551724137931034,0.6551724137931034,0.7011494252873564,0.7701149425287356,0.7241379310344828,0.7011494252873562,0.04361762289887419,1,0.4186046511627907,0.38636363636363635,0.5,0.6363636363636364,0.4883720930232558,0.4859408033826639,0.08634856159563532,1,0.5454545454545454,0.53125,0.6285714285714286,0.7368421052631579,0.6363636363636364,0.6156963431305537,0.07393821140149666,1,GaussianNB,,,,,,,,,
-1.2938672065734864,0.41128834648806695,0.004164218902587891,0.0005306523186815521,,,"{'activation': 'logistic', 'hidden_layer_sizes': (60, 80, 100), 'learning_rate': 'constant', 'max_iter': 500000, 'solver': 'lbfgs'}",0.7843137254901961,0.9024390243902439,0.84,0.8775510204081632,0.7692307692307693,0.8347069079038745,0.051532265568785755,1,0.8390804597701149,0.8735632183908046,0.8850574712643678,0.9195402298850575,0.8275862068965517,0.8689655172413794,0.0329947128607065,1,0.9302325581395349,0.8409090909090909,0.9545454545454546,0.9772727272727273,0.9302325581395349,0.9266384778012686,0.04629900468563652,1,0.851063829787234,0.8705882352941177,0.8936170212765958,0.9247311827956989,0.8421052631578948,0.8764211064623083,0.029947133184237494,1,MLPClassifier,logistic,"(60, 80, 100)",constant,500000,lbfgs,,,,
-0.1928084373474121,0.0018819907485268688,0.013821220397949219,0.0008887381291720887,gini,,"{'class_weight': 'balanced', 'criterion': 'gini', 'max_features': 'sqrt'}",0.7843137254901961,0.9024390243902439,0.8958333333333334,0.86,0.7843137254901961,0.8453799617407938,0.0519099128757062,1,0.8390804597701149,0.8735632183908046,0.9310344827586207,0.9080459770114943,0.8390804597701149,0.87816091954023,0.0367816091954023,1,0.9302325581395349,0.8409090909090909,0.9772727272727273,0.9772727272727273,0.9302325581395349,0.931183932346723,0.049799018580322194,1,0.851063829787234,0.8705882352941177,0.9347826086956522,0.9148936170212766,0.851063829787234,0.8844784241171029,0.03429752912239728,1,RandomForestClassifier,,,,,,balanced,sqrt,,
-0.004432106018066406,0.00040587701626017183,0.005803823471069336,0.00020614299614554025,,,"{'gamma': 'scale', 'kernel': 'rbf'}",0.7111111111111111,0.825,0.8409090909090909,0.7777777777777778,0.7608695652173914,0.7831335090030741,0.04648099349397003,1,0.7241379310344828,0.7931034482758621,0.8390804597701149,0.7816091954022989,0.7816091954022989,0.7839080459770115,0.036637649311515454,1,0.7441860465116279,0.75,0.8409090909090909,0.7954545454545454,0.813953488372093,0.7889006342494714,0.037116391304827384,1,0.7272727272727273,0.7857142857142856,0.8409090909090909,0.7865168539325843,0.7865168539325844,0.7853859623522546,0.03595172186115807,1,SVC,,,,,,,,scale,rbf
-0.002194929122924805,0.00025078630992778853,0.0047760486602783205,0.0003426548102886713,gini,best,"{'criterion': 'gini', 'splitter': 'best'}",0.8936170212765957,0.8269230769230769,0.8333333333333334,0.875,0.9090909090909091,0.8675928681247831,0.032502175910018874,1,0.9310344827586207,0.8850574712643678,0.8620689655172413,0.9080459770114943,0.9080459770114943,0.8988505747126437,0.023443767878587523,1,0.9767441860465116,0.9772727272727273,0.9090909090909091,0.9545454545454546,0.9090909090909091,0.9453488372093023,0.030720205594027078,1,0.9333333333333332,0.8958333333333334,0.8695652173913043,0.9130434782608695,0.9090909090909091,0.90417325428195,0.021073418855955303,1,DecisionTreeClassifier,,,,,,,,,
-0.0010171890258789062,5.780556663777061e-06,0.004647302627563477,0.00040967131925084886,,,{},0.8421052631578947,0.8148148148148148,0.9,0.8695652173913043,0.8666666666666667,0.8586303924061361,0.02859538868506317,1,0.6551724137931034,0.6896551724137931,0.7701149425287356,0.6896551724137931,0.6206896551724138,0.6850574712643678,0.04962536355154688,1,0.37209302325581395,0.5,0.6136363636363636,0.45454545454545453,0.29545454545454547,0.4471458773784355,0.1088214915607599,1,0.5161290322580645,0.6197183098591549,0.7297297297297297,0.5970149253731344,0.44067796610169496,0.5806539926643557,0.09774869299001981,1,GaussianNB,,,,,,,,,
-1.2683510780334473,0.14120442535349456,0.00443415641784668,0.000605298136657173,,,"{'activation': 'logistic', 'hidden_layer_sizes': (60, 80, 100), 'learning_rate': 'constant', 'max_iter': 500000, 'solver': 'lbfgs'}",0.8775510204081632,0.84,0.8695652173913043,0.8936170212765957,0.9523809523809523,0.8866228422914032,0.037202143822820034,1,0.9310344827586207,0.8850574712643678,0.8850574712643678,0.9195402298850575,0.9310344827586207,0.9103448275862069,0.021069313540026825,1,1.0,0.9545454545454546,0.9090909090909091,0.9545454545454546,0.9090909090909091,0.9454545454545455,0.03401506715249039,1,0.9347826086956522,0.8936170212765958,0.888888888888889,0.9230769230769231,0.9302325581395349,0.9141196000155191,0.019098510426465782,1,MLPClassifier,logistic,"(60, 80, 100)",constant,500000,lbfgs,,,,
-0.18851003646850586,0.001857404059599005,0.013000822067260743,0.000793564808041002,gini,,"{'class_weight': 'balanced', 'criterion': 'gini', 'max_features': 'sqrt'}",0.8958333333333334,0.8775510204081632,0.8695652173913043,0.8979591836734694,0.9555555555555556,0.8992928620723651,0.03011869270863508,1,0.9425287356321839,0.9195402298850575,0.8850574712643678,0.9425287356321839,0.9655172413793104,0.9310344827586207,0.027200366818848805,1,1.0,0.9772727272727273,0.9090909090909091,1.0,0.9772727272727273,0.9727272727272727,0.033402132856134255,1,0.945054945054945,0.9247311827956989,0.888888888888889,0.9462365591397849,0.9662921348314608,0.9342407421421557,0.026212060948629195,1,RandomForestClassifier,,,,,,balanced,sqrt,,
-0.00439605712890625,0.0004693768456373561,0.0059528350830078125,0.0002539582508022052,,,"{'gamma': 'scale', 'kernel': 'rbf'}",0.78125,0.7857142857142857,0.7272727272727273,0.7727272727272727,0.7804878048780488,0.7694904181184669,0.021518618958659787,1,0.7126436781609196,0.7701149425287356,0.7241379310344828,0.7701149425287356,0.7586206896551724,0.7471264367816092,0.024110548233796558,1,0.5813953488372093,0.75,0.7272727272727273,0.7727272727272727,0.7272727272727273,0.7117336152219874,0.06731349664174523,1,0.6666666666666666,0.7674418604651163,0.7272727272727273,0.7727272727272727,0.7529411764705882,0.7374099407204742,0.03872421296266713,1,SVC,,,,,,,,scale,rbf
-0.0022286891937255858,6.970233930780428e-05,0.004866743087768554,0.00039462011857565807,gini,best,"{'criterion': 'gini', 'splitter': 'best'}",0.7924528301886793,0.9148936170212766,0.8,0.9130434782608695,0.7884615384615384,0.841770292786473,0.059068909383946856,1,0.8505747126436781,0.9425287356321839,0.8505747126436781,0.9310344827586207,0.8390804597701149,0.8827586206896552,0.044457654266238426,1,0.9545454545454546,0.9772727272727273,0.9302325581395349,0.9545454545454546,0.9318181818181818,0.9496828752642706,0.01735483446510314,1,0.865979381443299,0.945054945054945,0.8602150537634408,0.9333333333333332,0.8541666666666667,0.8917498760523369,0.03909392622320531,1,DecisionTreeClassifier,,,,,,,,,
-0.0015948295593261718,0.00012828066395255346,0.005991125106811523,0.0005817005776371807,,,{},0.9285714285714286,0.9166666666666666,0.8214285714285714,0.8571428571428571,0.7692307692307693,0.8586080586080588,0.0593943365061509,1,0.7701149425287356,0.7241379310344828,0.7126436781609196,0.6666666666666666,0.6551724137931034,0.7057471264367816,0.041506827782224795,1,0.5909090909090909,0.5,0.5348837209302325,0.4090909090909091,0.45454545454545453,0.49788583509513745,0.06291194932924528,1,0.7222222222222223,0.6470588235294118,0.647887323943662,0.5538461538461539,0.5714285714285714,0.6284886189940042,0.06055492920625583,1,GaussianNB,,,,,,,,,
-1.1026182651519776,0.14324052655804487,0.004889726638793945,0.0010175996327541174,,,"{'activation': 'logistic', 'hidden_layer_sizes': (60, 80, 100), 'learning_rate': 'constant', 'max_iter': 500000, 'solver': 'lbfgs'}",0.803921568627451,0.86,0.8636363636363636,0.8541666666666666,0.82,0.8403449197860964,0.023918321397641523,1,0.8505747126436781,0.9080459770114943,0.8735632183908046,0.8850574712643678,0.8620689655172413,0.8758620689655172,0.019775460384006058,1,0.9318181818181818,0.9772727272727273,0.8837209302325582,0.9318181818181818,0.9318181818181818,0.9312896405919661,0.029590757115799084,1,0.8631578947368421,0.9148936170212766,0.8735632183908046,0.8913043478260869,0.8723404255319149,0.883051900701385,0.01834415464041346,1,MLPClassifier,logistic,"(60, 80, 100)",constant,500000,lbfgs,,,,
-0.19236540794372559,0.002878943916901777,0.013001012802124023,0.0010413261131587454,gini,,"{'class_weight': 'balanced', 'criterion': 'gini', 'max_features': 'sqrt'}",0.8936170212765957,0.9555555555555556,0.8076923076923077,0.8888888888888888,0.7884615384615384,0.8668430623749772,0.06118046233689892,1,0.9195402298850575,0.9655172413793104,0.8735632183908046,0.896551724137931,0.8390804597701149,0.8988505747126437,0.042637326416072215,1,0.9545454545454546,0.9772727272727273,0.9767441860465116,0.9090909090909091,0.9318181818181818,0.9498942917547568,0.026396176455870207,1,0.9230769230769231,0.9662921348314608,0.8842105263157894,0.8988764044943819,0.8541666666666667,0.9053245310770445,0.03778016080904847,1,RandomForestClassifier,,,,,,balanced,sqrt,,
-0.004561805725097656,0.00034113593490975005,0.006715822219848633,0.0013517431568828573,,,"{'gamma': 'scale', 'kernel': 'rbf'}",0.8085106382978723,0.813953488372093,0.7,0.8333333333333334,0.7,0.7711594920006597,0.05868452505524505,1,0.8275862068965517,0.8045977011494253,0.6896551724137931,0.7701149425287356,0.7241379310344828,0.7632183908045976,0.0506790981265485,1,0.8636363636363636,0.7954545454545454,0.6511627906976745,0.6818181818181818,0.7954545454545454,0.7575052854122621,0.07897007315826317,1,0.8351648351648351,0.8045977011494252,0.674698795180723,0.7499999999999999,0.7446808510638298,0.7618284365117626,0.055195947968182155,1,SVC,,,,,,,,scale,rbf
-0.0021810054779052733,0.00016655870743876805,0.004390907287597656,0.00018454117977101525,gini,best,"{'criterion': 'gini', 'splitter': 'best'}",0.7735849056603774,0.84,0.8125,0.7818181818181819,0.8367346938775511,0.808927556271222,0.02733319282764477,1,0.8390804597701149,0.896551724137931,0.8505747126436781,0.8620689655172413,0.8735632183908046,0.864367816091954,0.019775460384006058,1,0.9534883720930233,0.9767441860465116,0.9069767441860465,1.0,0.9318181818181818,0.9538054968287526,0.032669833944515725,1,0.8541666666666667,0.9032258064516129,0.8571428571428572,0.8775510204081634,0.8817204301075268,0.8747613561553654,0.017894069465452515,1,DecisionTreeClassifier,,,,,,,,,
-0.0014212608337402343,0.000418630747539305,0.0058075904846191405,0.0006960161121911706,,,{},0.8260869565217391,0.7692307692307693,0.7272727272727273,0.8571428571428571,0.782608695652174,0.7924684011640534,0.04516784197918672,1,0.6781609195402298,0.5862068965517241,0.6206896551724138,0.6781609195402298,0.6436781609195402,0.6413793103448275,0.03516565181788127,1,0.4418604651162791,0.23255813953488372,0.37209302325581395,0.4186046511627907,0.4090909090909091,0.37484143763213534,0.07460815759375224,1,0.5757575757575758,0.3571428571428571,0.49230769230769234,0.5625000000000001,0.537313432835821,0.5050043116087892,0.07922104738541158,1,GaussianNB,,,,,,,,,
-1.2932703495025635,0.3070269149927146,0.00411520004272461,0.0005692206222851011,,,"{'activation': 'logistic', 'hidden_layer_sizes': (60, 80, 100), 'learning_rate': 'constant', 'max_iter': 500000, 'solver': 'lbfgs'}",0.7735849056603774,0.7636363636363637,0.74,0.86,0.8269230769230769,0.7928288692439637,0.044017718586866796,1,0.8390804597701149,0.8390804597701149,0.7816091954022989,0.9195402298850575,0.8850574712643678,0.8528735632183908,0.04677468953879495,1,0.9534883720930233,0.9767441860465116,0.8604651162790697,1.0,0.9772727272727273,0.9535940803382663,0.048832632973398654,1,0.8541666666666667,0.8571428571428571,0.7956989247311828,0.924731182795699,0.8958333333333334,0.8655145929339477,0.04358651041683021,1,MLPClassifier,logistic,"(60, 80, 100)",constant,500000,lbfgs,,,,
-0.19373579025268556,0.0015811447448819665,0.01365489959716797,0.0011622480912881331,gini,,"{'class_weight': 'balanced', 'criterion': 'gini', 'max_features': 'sqrt'}",0.8367346938775511,0.8571428571428571,0.7916666666666666,0.8958333333333334,0.8461538461538461,0.8455062794348509,0.03359847095844642,1,0.8850574712643678,0.9080459770114943,0.8275862068965517,0.9425287356321839,0.9080459770114943,0.8942528735632184,0.0380527479476939,1,0.9534883720930233,0.9767441860465116,0.8837209302325582,1.0,1.0,0.9627906976744185,0.043133109281375356,1,0.8913043478260869,0.9130434782608695,0.8351648351648352,0.945054945054945,0.9166666666666666,0.9002468545946808,0.03676075749165613,1,RandomForestClassifier,,,,,,balanced,sqrt,,
-0.0045135498046875,0.00022925561941214,0.005884504318237305,0.00025680262489860436,,,"{'gamma': 'scale', 'kernel': 'rbf'}",0.6666666666666666,0.7441860465116279,0.7674418604651163,0.75,0.782608695652174,0.7421806538591169,0.0401000777003621,1,0.7126436781609196,0.7471264367816092,0.7701149425287356,0.7816091954022989,0.7931034482758621,0.760919540229885,0.028527985393082433,1,0.8372093023255814,0.7441860465116279,0.7674418604651163,0.8372093023255814,0.8181818181818182,0.8008456659619452,0.03813475310235768,1,0.7422680412371134,0.7441860465116278,0.7674418604651162,0.7912087912087912,0.8,0.7690209478845297,0.02360926055055637,1,SVC,,,,,,,,scale,rbf
-0.002174854278564453,0.00011694590492603858,0.0051247596740722655,0.0005745828873581929,gini,best,"{'criterion': 'gini', 'splitter': 'best'}",0.8695652173913043,0.8478260869565217,0.8627450980392157,0.82,0.8627450980392157,0.8525763000852514,0.017771955811906083,1,0.896551724137931,0.8620689655172413,0.9195402298850575,0.8735632183908046,0.9195402298850575,0.8942528735632184,0.02344376787858752,1,0.9302325581395349,0.8863636363636364,1.0,0.9534883720930233,1.0,0.9540169133192389,0.04329394717822427,1,0.898876404494382,0.8666666666666666,0.9263157894736842,0.8817204301075269,0.9263157894736842,0.8999790160431889,0.02379729229571924,1,DecisionTreeClassifier,,,,,,,,,
-0.0010544776916503907,3.80763950719563e-05,0.005653572082519531,0.0005627015960399458,,,{},0.8260869565217391,0.9,0.8695652173913043,0.8928571428571429,0.8148148148148148,0.8606648263170001,0.03452729566153893,1,0.6781609195402298,0.6781609195402298,0.6896551724137931,0.7586206896551724,0.6896551724137931,0.6988505747126437,0.03032392174315613,1,0.4418604651162791,0.4090909090909091,0.45454545454545453,0.5813953488372093,0.5,0.47737843551797043,0.05962769648938314,1,0.5757575757575758,0.5625000000000001,0.5970149253731344,0.7042253521126761,0.6197183098591549,0.6118432326205083,0.05010644823946961,1,GaussianNB,,,,,,,,,
-1.2718564510345458,0.32077819343379,0.004101800918579102,0.00046629674815160143,,,"{'activation': 'logistic', 'hidden_layer_sizes': (60, 80, 100), 'learning_rate': 'constant', 'max_iter': 500000, 'solver': 'lbfgs'}",0.803921568627451,0.9512195121951219,0.9148936170212766,0.8723404255319149,0.7586206896551724,0.8601991626061875,0.07058179850298994,1,0.8620689655172413,0.9195402298850575,0.9425287356321839,0.9080459770114943,0.8390804597701149,0.8942528735632184,0.03805274794769391,1,0.9534883720930233,0.8863636363636364,0.9772727272727273,0.9534883720930233,1.0,0.9541226215644819,0.038033526964117256,1,0.8723404255319148,0.9176470588235294,0.945054945054945,0.9111111111111112,0.8627450980392156,0.9017797277121431,0.03033798337188889,1,MLPClassifier,logistic,"(60, 80, 100)",constant,500000,lbfgs,,,,
-0.18883519172668456,0.0008676456576691585,0.016011619567871095,0.0017325355898029666,gini,,"{'class_weight': 'balanced', 'criterion': 'gini', 'max_features': 'sqrt'}",0.8723404255319149,0.8666666666666667,0.88,0.8863636363636364,0.8979591836734694,0.8806659824471375,0.010929385542622658,1,0.9080459770114943,0.8735632183908046,0.9310344827586207,0.896551724137931,0.9425287356321839,0.9103448275862067,0.024545007475934022,1,0.9534883720930233,0.8863636363636364,1.0,0.9069767441860465,1.0,0.9493657505285412,0.04671347706089014,1,0.9111111111111112,0.8764044943820225,0.9361702127659575,0.896551724137931,0.9462365591397849,0.9132948203073614,0.02551249787656514,1,RandomForestClassifier,,,,,,balanced,sqrt,,
-0.004232358932495117,0.00012370137460271448,0.0057848930358886715,0.0007042767085316053,,,"{'gamma': 'scale', 'kernel': 'rbf'}",0.7380952380952381,0.7560975609756098,0.8095238095238095,0.72,0.7560975609756098,0.7559628339140534,0.029937905118780302,1,0.735632183908046,0.735632183908046,0.7931034482758621,0.7586206896551724,0.735632183908046,0.7517241379310345,0.02252404361179935,1,0.7209302325581395,0.7045454545454546,0.7727272727272727,0.8372093023255814,0.7045454545454546,0.7479915433403805,0.05114509679539022,1,0.7294117647058824,0.7294117647058823,0.7906976744186046,0.7741935483870969,0.7294117647058823,0.7506253033846698,0.026500187782749762,1,SVC,,,,,,,,scale,rbf
-0.002027750015258789,0.00014001793569640337,0.004712867736816406,0.0003577187549815837,gini,best,"{'criterion': 'gini', 'splitter': 'best'}",0.7818181818181819,0.7857142857142857,0.8269230769230769,0.8888888888888888,0.8148148148148148,0.8196318496318495,0.038608750101800454,1,0.8505747126436781,0.8620689655172413,0.896551724137931,0.896551724137931,0.8850574712643678,0.8781609195402298,0.018675950355484996,1,0.9772727272727273,1.0,1.0,0.9090909090909091,1.0,0.9772727272727273,0.035208939510976534,1,0.8686868686868686,0.88,0.9052631578947368,0.8988764044943819,0.8979591836734693,0.8901571229498912,0.013632529322759126,1,DecisionTreeClassifier,,,,,,,,,
-0.0010700702667236328,7.968338420942486e-05,0.004435539245605469,0.00013211435868668643,,,{},0.875,0.8260869565217391,0.8,0.8620689655172413,0.8260869565217391,0.8378485757121439,0.02711472538797047,1,0.632183908045977,0.6666666666666666,0.6781609195402298,0.735632183908046,0.6666666666666666,0.6758620689655171,0.033629284685811014,1,0.3181818181818182,0.4318181818181818,0.46511627906976744,0.5681818181818182,0.4318181818181818,0.44302325581395346,0.07998657303214383,1,0.4666666666666667,0.5671641791044776,0.5882352941176471,0.6849315068493151,0.5671641791044776,0.5748323651685168,0.06945340883021155,1,GaussianNB,,,,,,,,,
-1.085054349899292,0.15827320568582814,0.0044189453125,0.0005321580576924798,,,"{'activation': 'logistic', 'hidden_layer_sizes': (60, 80, 100), 'learning_rate': 'constant', 'max_iter': 500000, 'solver': 'lbfgs'}",0.7592592592592593,0.8461538461538461,0.803921568627451,0.8269230769230769,0.8235294117647058,0.8119574325456677,0.029559934015403234,1,0.8160919540229885,0.9080459770114943,0.8620689655172413,0.8850574712643678,0.8735632183908046,0.8689655172413794,0.030497699221658854,1,0.9318181818181818,1.0,0.9534883720930233,0.9772727272727273,0.9545454545454546,0.9634249471458773,0.023263980883870366,1,0.836734693877551,0.9166666666666666,0.8723404255319148,0.8958333333333334,0.8842105263157896,0.8811571291450511,0.026595971410552314,1,MLPClassifier,logistic,"(60, 80, 100)",constant,500000,lbfgs,,,,
-0.19232707023620604,0.00213496251133884,0.014331436157226563,0.001831241405672576,gini,,"{'class_weight': 'balanced', 'criterion': 'gini', 'max_features': 'sqrt'}",0.8148148148148148,0.88,0.8571428571428571,0.8775510204081632,0.8979591836734694,0.865493575207861,0.028453138052690893,1,0.8850574712643678,0.9310344827586207,0.9080459770114943,0.9195402298850575,0.9425287356321839,0.9172413793103449,0.019775460384006016,1,1.0,1.0,0.9767441860465116,0.9772727272727273,1.0,0.9908033826638478,0.011264749940598078,1,0.8979591836734693,0.9361702127659575,0.9130434782608695,0.9247311827956989,0.9462365591397849,0.923628123327156,0.016974046610632783,1,RandomForestClassifier,,,,,,balanced,sqrt,,
-0.0048328876495361325,0.00019966701802444185,0.00673069953918457,0.0008874171344694884,,,"{'gamma': 'scale', 'kernel': 'rbf'}",0.7446808510638298,0.7954545454545454,0.7727272727272727,0.7959183673469388,0.7222222222222222,0.7662006517629619,0.02890928495781439,1,0.7586206896551724,0.7931034482758621,0.7816091954022989,0.8275862068965517,0.7701149425287356,0.7862068965517242,0.023668115266636796,1,0.7954545454545454,0.7954545454545454,0.7906976744186046,0.8863636363636364,0.8863636363636364,0.8308668076109937,0.04534624987074944,1,0.7692307692307692,0.7954545454545455,0.7816091954022988,0.8387096774193548,0.7959183673469388,0.7961845109707815,0.023443282540081302,1,SVC,,,,,,,,scale,rbf
-0.0021222591400146484,0.00019748915384906377,0.005195093154907226,0.00046344108327378027,gini,best,"{'criterion': 'gini', 'splitter': 'best'}",0.8636363636363636,0.7959183673469388,0.7924528301886793,0.9523809523809523,0.8431372549019608,0.849505153690979,0.05822401991154598,1,0.8735632183908046,0.8390804597701149,0.8620689655172413,0.9425287356321839,0.9080459770114943,0.8850574712643677,0.03634801908239516,1,0.8837209302325582,0.9069767441860465,0.9767441860465116,0.9302325581395349,1.0,0.9395348837209301,0.04313310928137536,1,0.8735632183908046,0.8478260869565216,0.875,0.9411764705882352,0.9148936170212766,0.8904918785913676,0.033200362403427226,1,DecisionTreeClassifier,,,,,,,,,
-0.001080465316772461,6.112791999116283e-05,0.005242586135864258,0.0002060961688989271,,,{},0.8333333333333334,0.8125,0.6666666666666666,0.8947368421052632,0.8260869565217391,0.8066647597254004,0.07547005012658593,1,0.6896551724137931,0.6206896551724138,0.6091954022988506,0.6781609195402298,0.6781609195402298,0.6551724137931034,0.03331350976135501,1,0.46511627906976744,0.3023255813953488,0.4186046511627907,0.3953488372093023,0.4418604651162791,0.4046511627906977,0.05620021383067244,1,0.5970149253731343,0.44067796610169485,0.5142857142857143,0.5483870967741935,0.5757575757575758,0.5352246556584626,0.05479365049210219,1,GaussianNB,,,,,,,,,
-1.1959696292877198,0.14080829208907036,0.004356956481933594,0.0005870528652363635,,,"{'activation': 'logistic', 'hidden_layer_sizes': (60, 80, 100), 'learning_rate': 'constant', 'max_iter': 500000, 'solver': 'lbfgs'}",0.7872340425531915,0.8085106382978723,0.8113207547169812,0.9130434782608695,0.8431372549019608,0.832649233746175,0.04398488044286019,1,0.8160919540229885,0.8390804597701149,0.8850574712643678,0.9425287356321839,0.9080459770114943,0.8781609195402298,0.04574654883248827,1,0.8604651162790697,0.8837209302325582,1.0,0.9767441860465116,1.0,0.9441860465116279,0.05992604058941919,1,0.8222222222222222,0.8444444444444444,0.8958333333333334,0.9438202247191011,0.9148936170212766,0.8842427683480756,0.04484137320821887,1,MLPClassifier,logistic,"(60, 80, 100)",constant,500000,lbfgs,,,,
-0.1951068878173828,0.0022881152175649425,0.013331174850463867,0.0009673283529581934,gini,,"{'class_weight': 'balanced', 'criterion': 'gini', 'max_features': 'sqrt'}",0.8837209302325582,0.7959183673469388,0.8269230769230769,0.9534883720930233,0.8431372549019608,0.8606376002995114,0.05437943771859323,1,0.8850574712643678,0.8390804597701149,0.896551724137931,0.9540229885057471,0.9080459770114943,0.8965517241379309,0.03706785171631516,1,0.8837209302325582,0.9069767441860465,1.0,0.9534883720930233,1.0,0.9488372093023255,0.04743273966132824,1,0.8837209302325582,0.8478260869565216,0.9052631578947368,0.9534883720930233,0.9148936170212766,0.9010384328396233,0.034906332402911494,1,RandomForestClassifier,,,,,,balanced,sqrt,,
-0.004303455352783203,0.00023048815720343765,0.005551338195800781,0.00023452182629605712,,,"{'gamma': 'scale', 'kernel': 'rbf'}",0.8108108108108109,0.6888888888888889,0.7391304347826086,0.8809523809523809,0.75,0.7739565030869378,0.06607514803168148,1,0.7701149425287356,0.7011494252873564,0.7586206896551724,0.8735632183908046,0.7586206896551724,0.7724137931034484,0.05602785106645491,1,0.6976744186046512,0.7209302325581395,0.7906976744186046,0.8604651162790697,0.7674418604651163,0.7674418604651163,0.05696487773914367,1,0.75,0.7045454545454545,0.7640449438202247,0.8705882352941177,0.7586206896551724,0.769559864662994,0.054710645038602346,1,SVC,,,,,,,,scale,rbf
-0.0020093917846679688,0.00011963995388653626,0.004314136505126953,0.00013841124847784088,gini,best,"{'criterion': 'gini', 'splitter': 'best'}",0.8809523809523809,0.8541666666666666,0.803921568627451,0.803921568627451,0.8979591836734694,0.8481842737094837,0.03874391219806552,1,0.8735632183908046,0.896551724137931,0.8620689655172413,0.8620689655172413,0.9425287356321839,0.8873563218390805,0.030323921743156134,1,0.8604651162790697,0.9534883720930233,0.9534883720930233,0.9534883720930233,1.0,0.9441860465116279,0.04557190219131496,1,0.8705882352941177,0.9010989010989011,0.8723404255319148,0.8723404255319148,0.9462365591397849,0.8925209093193267,0.02917018690474438,1,DecisionTreeClassifier,,,,,,,,,
-0.0011737823486328125,0.0002477355771924818,0.006713438034057617,0.0003002526020621795,,,{},0.8181818181818182,0.7391304347826086,0.7916666666666666,0.9,0.75,0.7997957839262186,0.05762797654884553,1,0.6666666666666666,0.632183908045977,0.6666666666666666,0.6896551724137931,0.632183908045977,0.657471264367816,0.022288183252488873,1,0.4186046511627907,0.3953488372093023,0.4418604651162791,0.4186046511627907,0.4090909090909091,0.41670190274841434,0.015192595537608194,1,0.5538461538461539,0.5151515151515151,0.5671641791044777,0.5714285714285715,0.5294117647058824,0.5474004368473201,0.021785726074865806,1,GaussianNB,,,,,,,,,
-1.7255743980407714,0.5048302426773646,0.004402303695678711,0.00029247624846939784,,,"{'activation': 'logistic', 'hidden_layer_sizes': (60, 80, 100), 'learning_rate': 'constant', 'max_iter': 500000, 'solver': 'lbfgs'}",0.8541666666666666,0.8478260869565217,0.7413793103448276,0.84,0.8076923076923077,0.8182128743320647,0.04161339992707591,1,0.896551724137931,0.8735632183908046,0.8275862068965517,0.896551724137931,0.8620689655172413,0.871264367816092,0.025598916610712757,1,0.9534883720930233,0.9069767441860465,1.0,0.9767441860465116,0.9545454545454546,0.9583509513742072,0.03082333525345895,1,0.9010989010989011,0.8764044943820224,0.8514851485148515,0.9032258064516129,0.875,0.8814428700894776,0.019106107705665402,1,MLPClassifier,logistic,"(60, 80, 100)",constant,500000,lbfgs,,,,
-0.19639902114868163,0.002088271265498995,0.021457910537719727,0.005576984026873168,gini,,"{'class_weight': 'balanced', 'criterion': 'gini', 'max_features': 'sqrt'}",0.8888888888888888,0.8837209302325582,0.82,0.8571428571428571,0.851063829787234,0.8601633012103077,0.024840030190334587,1,0.9080459770114943,0.8850574712643678,0.8735632183908046,0.9080459770114943,0.8735632183908046,0.8896551724137932,0.015591563179598297,1,0.9302325581395349,0.8837209302325582,0.9534883720930233,0.9767441860465116,0.9090909090909091,0.9306553911205073,0.03259929854950761,1,0.9090909090909092,0.8837209302325582,0.8817204301075269,0.9130434782608695,0.8791208791208791,0.8933393253625486,0.014601660842130238,1,RandomForestClassifier,,,,,,balanced,sqrt,,
-0.0047051429748535155,0.00017161371311103895,0.006136894226074219,0.0006322596042438573,,,"{'gamma': 'scale', 'kernel': 'rbf'}",0.7446808510638298,0.7317073170731707,0.6808510638297872,0.7333333333333333,0.7446808510638298,0.7270506832727902,0.023736918079198085,1,0.7701149425287356,0.7241379310344828,0.7011494252873564,0.7471264367816092,0.7586206896551724,0.7402298850574713,0.024759378423606884,1,0.813953488372093,0.6976744186046512,0.7441860465116279,0.7674418604651163,0.7954545454545454,0.7637420718816068,0.040713458952669083,1,0.7777777777777778,0.7142857142857143,0.711111111111111,0.7499999999999999,0.7692307692307692,0.7444810744810745,0.02748450695266498,1,SVC,,,,,,,,scale,rbf
-0.002054643630981445,5.7839659351053614e-05,0.004293632507324219,0.00013169879756414914,gini,best,"{'criterion': 'gini', 'splitter': 'best'}",0.7659574468085106,0.8775510204081632,0.851063829787234,0.875,0.875,0.8489144594007815,0.04258508944297005,1,0.7931034482758621,0.9195402298850575,0.8850574712643678,0.9080459770114943,0.9195402298850575,0.885057471264368,0.047669980122592444,1,0.8372093023255814,0.9772727272727273,0.9302325581395349,0.9545454545454546,0.9767441860465116,0.9352008456659618,0.051956005796795726,1,0.8,0.9247311827956989,0.888888888888889,0.9130434782608695,0.923076923076923,0.8899480946044761,0.04676015587235548,1,DecisionTreeClassifier,,,,,,,,,
-0.0013537883758544921,0.00027163213604338267,0.005893182754516601,0.0004974941364007879,,,{},0.7619047619047619,0.8260869565217391,0.8181818181818182,0.85,0.8888888888888888,0.8290124850994417,0.04160521142398222,1,0.632183908045977,0.6666666666666666,0.6666666666666666,0.6551724137931034,0.6666666666666666,0.657471264367816,0.013404487114586881,1,0.37209302325581395,0.4318181818181818,0.4186046511627907,0.38636363636363635,0.37209302325581395,0.3961945031712474,0.0246157259643505,1,0.5,0.5671641791044776,0.5538461538461539,0.53125,0.5245901639344261,0.5353700993770115,0.02339424823081782,1,GaussianNB,,,,,,,,,
-1.4838696479797364,0.6199237407095957,0.0040858745574951175,0.0006339836598317698,,,"{'activation': 'logistic', 'hidden_layer_sizes': (60, 80, 100), 'learning_rate': 'constant', 'max_iter': 500000, 'solver': 'lbfgs'}",0.8809523809523809,0.7692307692307693,0.7924528301886793,0.875,0.8269230769230769,0.8289118114589812,0.04410755304470653,1,0.8735632183908046,0.8160919540229885,0.8620689655172413,0.9080459770114943,0.896551724137931,0.871264367816092,0.03201928339582557,1,0.8604651162790697,0.9090909090909091,0.9767441860465116,0.9545454545454546,1.0,0.9401691331923889,0.04988533262452812,1,0.8705882352941177,0.8333333333333333,0.875,0.9130434782608695,0.9052631578947368,0.8794456409566115,0.028354615586541464,1,MLPClassifier,logistic,"(60, 80, 100)",constant,500000,lbfgs,,,,
-0.1956347942352295,0.0014039843468034515,0.013931798934936523,0.001354689586326171,gini,,"{'class_weight': 'balanced', 'criterion': 'gini', 'max_features': 'sqrt'}",0.8809523809523809,0.8888888888888888,0.8888888888888888,0.9166666666666666,0.8936170212765957,0.8938027693346842,0.012134391020530794,1,0.8735632183908046,0.896551724137931,0.9080459770114943,0.9540229885057471,0.9310344827586207,0.9126436781609195,0.02777711718067714,1,0.8604651162790697,0.9090909090909091,0.9302325581395349,1.0,0.9767441860465116,0.935306553911205,0.04942988079513129,1,0.8705882352941177,0.8988764044943819,0.9090909090909092,0.9565217391304348,0.9333333333333332,0.9136821242686354,0.029389707317364476,1,RandomForestClassifier,,,,,,balanced,sqrt,,
-0.005793619155883789,0.0005572281654045837,0.008275651931762695,0.0013568136907738026,,,"{'gamma': 'scale', 'kernel': 'rbf'}",0.8205128205128205,0.8205128205128205,0.8048780487804879,0.7948717948717948,0.6888888888888889,0.7859328747133626,0.04949324075410711,1,0.7931034482758621,0.7816091954022989,0.7931034482758621,0.7586206896551724,0.7011494252873564,0.7655172413793103,0.03455930201924807,1,0.7441860465116279,0.7272727272727273,0.7674418604651163,0.7045454545454546,0.7209302325581395,0.7328752642706131,0.021444969992358884,1,0.7804878048780488,0.7710843373493976,0.7857142857142858,0.746987951807229,0.7045454545454545,0.7577639668588831,0.029743429397742827,1,SVC,,,,,,,,scale,rbf
-0.005517721176147461,0.0014116957467791389,0.008818578720092774,0.0013795761354392928,gini,best,"{'criterion': 'gini', 'splitter': 'best'}",0.8478260869565217,0.8936170212765957,0.8076923076923077,0.8367346938775511,0.8461538461538461,0.8464047911913644,0.02765228701358383,1,0.8735632183908046,0.9310344827586207,0.8735632183908046,0.8735632183908046,0.9080459770114943,0.8919540229885058,0.023668115266636754,1,0.9069767441860465,0.9767441860465116,0.9767441860465116,0.9318181818181818,1.0,0.9584566596194503,0.03392493645746217,1,0.8764044943820224,0.9333333333333332,0.8842105263157894,0.8817204301075268,0.9166666666666666,0.8984670901610677,0.02243808310022765,1,DecisionTreeClassifier,,,,,,,,,
-0.0017252922058105468,0.000320580731560803,0.008783674240112305,0.0013091521896116177,,,{},0.8275862068965517,0.8333333333333334,0.7058823529411765,0.8636363636363636,0.7727272727272727,0.8006331059069396,0.055716471037288065,1,0.7241379310344828,0.7816091954022989,0.5862068965517241,0.6781609195402298,0.632183908045977,0.6804597701149425,0.06834974136698166,1,0.5581395348837209,0.6976744186046512,0.27906976744186046,0.4318181818181818,0.38636363636363635,0.47061310782241017,0.14457495305699658,1,0.6666666666666666,0.759493670886076,0.4,0.5757575757575758,0.515151515151515,0.5834138856923666,0.12352662876664627,1,GaussianNB,,,,,,,,,
-1.233776044845581,0.23447687891633837,0.004335212707519531,0.0006214690301318999,,,"{'activation': 'logistic', 'hidden_layer_sizes': (60, 80, 100), 'learning_rate': 'constant', 'max_iter': 500000, 'solver': 'lbfgs'}",0.7222222222222222,0.8125,0.7547169811320755,0.8269230769230769,0.82,0.7872724560554748,0.04140310304928609,1,0.7816091954022989,0.8505747126436781,0.8160919540229885,0.8850574712643678,0.8620689655172413,0.8390804597701148,0.03634801908239515,1,0.9069767441860465,0.9069767441860465,0.9302325581395349,0.9772727272727273,0.9318181818181818,0.9306553911205073,0.025675137939879418,1,0.8041237113402061,0.8571428571428572,0.8333333333333334,0.8958333333333334,0.8723404255319149,0.8525547321363292,0.03162463540687007,1,MLPClassifier,logistic,"(60, 80, 100)",constant,500000,lbfgs,,,,
-0.19530963897705078,0.0027913324156435,0.014107561111450196,0.0008964366429279065,gini,,"{'class_weight': 'balanced', 'criterion': 'gini', 'max_features': 'sqrt'}",0.8125,0.8297872340425532,0.84,0.9565217391304348,0.875,0.8627617946345977,0.05113192701931878,1,0.8505747126436781,0.8620689655172413,0.896551724137931,0.9770114942528736,0.9080459770114943,0.8988505747126437,0.044457654266238454,1,0.9069767441860465,0.9069767441860465,0.9767441860465116,1.0,0.9545454545454546,0.9490486257928119,0.03723812068719494,1,0.8571428571428572,0.8666666666666666,0.9032258064516129,0.9777777777777777,0.9130434782608695,0.9035713172599568,0.04269425052296095,1,RandomForestClassifier,,,,,,balanced,sqrt,,
-0.004642105102539063,0.000382758187848364,0.006401586532592774,0.0011292886347330732,,,"{'gamma': 'scale', 'kernel': 'rbf'}",0.72,0.7441860465116279,0.6666666666666666,0.8222222222222222,0.7391304347826086,0.7384410740366251,0.05008872025357937,1,0.7586206896551724,0.7471264367816092,0.6666666666666666,0.8275862068965517,0.7471264367816092,0.7494252873563217,0.05109450751929395,1,0.8372093023255814,0.7441860465116279,0.6511627906976745,0.8409090909090909,0.7727272727272727,0.7692389006342495,0.06976872316076294,1,0.7741935483870969,0.7441860465116278,0.6588235294117646,0.8314606741573033,0.7555555555555555,0.7528438708046696,0.05578269669633417,1,SVC,,,,,,,,scale,rbf
-0.0024424076080322267,0.00044691371294081517,0.004960823059082031,0.00034843031152216207,gini,best,"{'criterion': 'gini', 'splitter': 'best'}",0.7962962962962963,0.8,0.8235294117647058,0.8297872340425532,0.84,0.817922588420711,0.017020775966639777,1,0.8620689655172413,0.8505747126436781,0.8735632183908046,0.8620689655172413,0.8850574712643678,0.8666666666666666,0.011721883939293793,1,0.9772727272727273,0.9302325581395349,0.9545454545454546,0.9069767441860465,0.9545454545454546,0.9447145877378436,0.02402997441354085,1,0.8775510204081632,0.8602150537634408,0.8842105263157896,0.8666666666666666,0.8936170212765958,0.8764520576861312,0.011961870283472129,1,DecisionTreeClassifier,,,,,,,,,
-0.001541900634765625,0.00020927183956067987,0.006117963790893554,0.0011798443445466258,,,{},0.8695652173913043,0.8333333333333334,0.8636363636363636,0.76,0.8571428571428571,0.8367355543007717,0.04029347027048997,1,0.6896551724137931,0.6896551724137931,0.6781609195402298,0.6551724137931034,0.6091954022988506,0.664367816091954,0.030323921743156134,1,0.45454545454545453,0.46511627906976744,0.4318181818181818,0.4418604651162791,0.2727272727272727,0.4132135306553911,0.07114125637203969,1,0.5970149253731344,0.5970149253731343,0.5757575757575758,0.5588235294117647,0.41379310344827586,0.548480811872777,0.06885185952651231,1,GaussianNB,,,,,,,,,
-1.1970837593078614,0.19940989694591937,0.004441261291503906,0.0006258874336289021,,,"{'activation': 'logistic', 'hidden_layer_sizes': (60, 80, 100), 'learning_rate': 'constant', 'max_iter': 500000, 'solver': 'lbfgs'}",0.7457627118644068,0.8125,0.9148936170212766,0.8,0.8695652173913043,0.8285443092553976,0.05841901774746935,1,0.8275862068965517,0.8505747126436781,0.9425287356321839,0.8505747126436781,0.8850574712643678,0.871264367816092,0.04008182936589125,1,1.0,0.9069767441860465,0.9772727272727273,0.9302325581395349,0.9090909090909091,0.9447145877378436,0.03746846609226435,1,0.8543689320388349,0.8571428571428572,0.945054945054945,0.8602150537634408,0.888888888888889,0.8811341353777935,0.03427994057319923,1,MLPClassifier,logistic,"(60, 80, 100)",constant,500000,lbfgs,,,,
-0.19360728263854982,0.0027234169199152124,0.014816474914550782,0.0014398728068045806,gini,,"{'class_weight': 'balanced', 'criterion': 'gini', 'max_features': 'sqrt'}",0.7719298245614035,0.8333333333333334,0.8936170212765957,0.8478260869565217,0.8695652173913043,0.8432542967038318,0.04107353328001852,1,0.8505747126436781,0.8735632183908046,0.9195402298850575,0.8735632183908046,0.8850574712643678,0.8804597701149426,0.02252404361179934,1,1.0,0.9302325581395349,0.9545454545454546,0.9069767441860465,0.9090909090909091,0.9401691331923889,0.03450431121940526,1,0.8712871287128713,0.8791208791208791,0.9230769230769231,0.8764044943820224,0.888888888888889,0.887755662836317,0.01856608067167711,1,RandomForestClassifier,,,,,,balanced,sqrt,,
-0.0045832633972167965,0.00037759465520098416,0.006492757797241211,0.0010056513369830899,,,"{'gamma': 'scale', 'kernel': 'rbf'}",0.75,0.7435897435897436,0.8222222222222222,0.6739130434782609,0.8055555555555556,0.7590561129691563,0.05238839483175277,1,0.7701149425287356,0.7241379310344828,0.8275862068965517,0.6896551724137931,0.7471264367816092,0.7517241379310345,0.046320555585310064,1,0.8181818181818182,0.6744186046511628,0.8409090909090909,0.7209302325581395,0.6590909090909091,0.7427061310782241,0.07412055319973548,1,0.7826086956521738,0.7073170731707318,0.8314606741573033,0.6966292134831461,0.7250000000000001,0.748603131292671,0.050960151788453534,1,SVC,,,,,,,,scale,rbf
+mean_fit_time,std_fit_time,mean_score_time,std_score_time,param_criterion,param_splitter,params,split0_test_precision,split1_test_precision,split2_test_precision,split3_test_precision,split4_test_precision,mean_test_precision,std_test_precision,rank_test_precision,split0_test_accuracy,split1_test_accuracy,split2_test_accuracy,split3_test_accuracy,split4_test_accuracy,mean_test_accuracy,std_test_accuracy,rank_test_accuracy,split0_test_recall,split1_test_recall,split2_test_recall,split3_test_recall,split4_test_recall,mean_test_recall,std_test_recall,rank_test_recall,split0_test_f1,split1_test_f1,split2_test_f1,split3_test_f1,split4_test_f1,mean_test_f1,std_test_f1,rank_test_f1,classifier,param_activation,param_hidden_layer_sizes,param_learning_rate,param_max_iter,param_solver,param_class_weight,param_max_features,param_gamma,param_kernel,param_to_return
+0.001976490020751953,0.00010017382944249402,0.0041100502014160155,0.00020298226930069274,gini,best,"{'criterion': 'gini', 'splitter': 'best'}",0.7692307692307693,0.8666666666666667,0.8269230769230769,0.8431372549019608,0.8301886792452831,0.8272292893935514,0.032190216489620026,1,0.8275862068965517,0.8850574712643678,0.896551724137931,0.896551724137931,0.896551724137931,0.8804597701149426,0.026808974229173804,1,0.9302325581395349,0.9069767441860465,1.0,0.9772727272727273,1.0,0.9628964059196617,0.03782583118243111,1,0.8421052631578948,0.8863636363636364,0.9052631578947368,0.9052631578947369,0.9072164948453608,0.8892423420312732,0.02476527569918078,1,DecisionTreeClassifier,,,,,,,,,,
+0.0010249614715576172,1.540634219388853e-05,0.0037468910217285157,0.00011609066524702597,,,{},0.8,0.8666666666666667,0.8,0.896551724137931,0.6923076923076923,0.8111052166224579,0.07035294426445517,1,0.6781609195402298,0.632183908045977,0.6436781609195402,0.7586206896551724,0.6091954022988506,0.664367816091954,0.05211854735059599,1,0.46511627906976744,0.3023255813953488,0.37209302325581395,0.5909090909090909,0.4090909090909091,0.4279069767441861,0.09715364127553945,1,0.5882352941176471,0.44827586206896547,0.5079365079365079,0.7123287671232877,0.5142857142857142,0.5542124291064245,0.09068240332458721,1,GaussianNB,,,,,,,,,,
+1.310411024093628,0.3313412936813181,0.004386615753173828,0.000617457196577433,,,"{'activation': 'logistic', 'hidden_layer_sizes': (60, 80, 100), 'learning_rate': 'constant', 'max_iter': 500000, 'solver': 'lbfgs'}",0.7962962962962963,0.8085106382978723,0.7924528301886793,0.9148936170212766,0.7719298245614035,0.8168166412731056,0.050433300547092344,1,0.8735632183908046,0.8390804597701149,0.8620689655172413,0.9425287356321839,0.8505747126436781,0.8735632183908045,0.03634801908239516,1,1.0,0.8837209302325582,0.9767441860465116,0.9772727272727273,1.0,0.9675475687103594,0.04315641873741724,1,0.8865979381443299,0.8444444444444444,0.875,0.945054945054945,0.8712871287128713,0.8844768912713181,0.03328836897029522,1,MLPClassifier,logistic,"(60, 80, 100)",constant,500000,lbfgs,,,,,
+0.17238092422485352,0.022403166733080074,0.01240386962890625,0.001147519995702171,gini,,"{'class_weight': 'balanced', 'criterion': 'gini', 'max_features': 'sqrt'}",0.8076923076923077,0.9069767441860465,0.84,0.88,0.8148148148148148,0.8498967733386337,0.03814245452050048,1,0.8735632183908046,0.9080459770114943,0.896551724137931,0.9310344827586207,0.8850574712643678,0.8988505747126437,0.019775460384006016,1,0.9767441860465116,0.9069767441860465,0.9767441860465116,1.0,1.0,0.9720930232558139,0.03417892664348623,1,0.8842105263157894,0.9069767441860465,0.9032258064516129,0.9361702127659575,0.8979591836734693,0.9057084946785752,0.017075730914147384,1,RandomForestClassifier,,,,,,balanced,sqrt,,,
+0.006538248062133789,0.0004504782516144641,0.009292316436767579,0.0022072603975237647,,,"{'gamma': 'scale', 'kernel': 'rbf'}",0.717391304347826,0.8064516129032258,0.6818181818181818,0.8421052631578947,0.725,0.7545532724454256,0.059839100951909266,1,0.735632183908046,0.7241379310344828,0.6896551724137931,0.7931034482758621,0.7011494252873564,0.728735632183908,0.03605606239392671,1,0.7674418604651163,0.5813953488372093,0.6976744186046512,0.7272727272727273,0.6590909090909091,0.6865750528541227,0.06346193424695262,1,0.7415730337078652,0.6756756756756758,0.6896551724137931,0.7804878048780488,0.6904761904761905,0.7155735754303147,0.03945456568462695,1,SVC,,,,,,,,scale,rbf,
+0.00013699531555175782,3.0689055760546595e-05,0.012158870697021484,0.0042352282228720825,,,{'to_return': 1},0.4942528735632184,0.4942528735632184,0.4942528735632184,0.5057471264367817,0.5057471264367817,0.4988505747126437,0.005631010902949855,1,0.4942528735632184,0.4942528735632184,0.4942528735632184,0.5057471264367817,0.5057471264367817,0.4988505747126437,0.005631010902949855,1,1.0,1.0,1.0,1.0,1.0,1.0,0.0,1,0.6615384615384615,0.6615384615384615,0.6615384615384615,0.6717557251908397,0.6717557251908397,0.6656253669994129,0.005005416503162351,1,BiasedClassifier,,,,,,,,,,1
+0.003063344955444336,0.0008433942655856669,0.011542463302612304,0.004006498105823139,gini,best,"{'criterion': 'gini', 'splitter': 'best'}",0.84,0.8541666666666666,0.8367346938775511,0.8,0.8333333333333334,0.8328469387755101,0.01788837437307085,1,0.8850574712643678,0.8850574712643678,0.8735632183908046,0.8390804597701149,0.8620689655172413,0.8689655172413794,0.017203022467926193,1,0.9545454545454546,0.9318181818181818,0.9318181818181818,0.9090909090909091,0.9090909090909091,0.9272727272727271,0.01700753357624521,1,0.8936170212765958,0.8913043478260869,0.8817204301075268,0.8510638297872342,0.8695652173913043,0.8774541692777497,0.015684079777382086,1,DecisionTreeClassifier,,,,,,,,,,
+0.004034900665283203,0.001735953947161072,0.016949176788330078,0.0021505846834181247,,,{},0.75,0.8,0.7391304347826086,0.6666666666666666,0.7692307692307693,0.7450055741360089,0.04428926038920083,1,0.632183908045977,0.5977011494252874,0.6206896551724138,0.5632183908045977,0.6551724137931034,0.6137931034482758,0.03135214183215139,1,0.4090909090909091,0.2727272727272727,0.38636363636363635,0.2727272727272727,0.45454545454545453,0.3590909090909091,0.07385489458759965,1,0.5294117647058824,0.4067796610169491,0.5074626865671641,0.3870967741935484,0.5714285714285714,0.4804358915824231,0.07147892377055552,1,GaussianNB,,,,,,,,,,
+1.5442252159118652,0.7110808431621647,0.004485130310058594,0.0010939685128482305,,,"{'activation': 'logistic', 'hidden_layer_sizes': (60, 80, 100), 'learning_rate': 'constant', 'max_iter': 500000, 'solver': 'lbfgs'}",0.8666666666666667,0.8666666666666667,0.7592592592592593,0.803921568627451,0.8125,0.8218028322440087,0.0408497443717608,1,0.8735632183908046,0.8735632183908046,0.8160919540229885,0.8505747126436781,0.8390804597701149,0.850574712643678,0.02180881144943712,1,0.8863636363636364,0.8863636363636364,0.9318181818181818,0.9318181818181818,0.8863636363636364,0.9045454545454545,0.022268088570756142,1,0.8764044943820225,0.8764044943820225,0.836734693877551,0.8631578947368421,0.8478260869565218,0.860105532866992,0.015732832630266848,1,MLPClassifier,logistic,"(60, 80, 100)",constant,500000,lbfgs,,,,,
+0.18745131492614747,0.00897341090736586,0.01860365867614746,0.006105705204878399,gini,,"{'class_weight': 'balanced', 'criterion': 'gini', 'max_features': 'sqrt'}",0.84,0.9111111111111111,0.9333333333333333,0.8541666666666666,0.8936170212765957,0.8864456264775413,0.03480631998683805,1,0.8850574712643678,0.9195402298850575,0.9425287356321839,0.8850574712643678,0.9195402298850575,0.9103448275862069,0.022288183252488842,1,0.9545454545454546,0.9318181818181818,0.9545454545454546,0.9318181818181818,0.9545454545454546,0.9454545454545453,0.011134044285378127,1,0.8936170212765958,0.9213483146067416,0.9438202247191012,0.8913043478260869,0.9230769230769231,0.9146333663010898,0.019769606709709422,1,RandomForestClassifier,,,,,,balanced,sqrt,,,
+0.007366752624511719,0.0013313799198813722,0.006848001480102539,0.0005562254578038502,,,"{'gamma': 'scale', 'kernel': 'rbf'}",0.7209302325581395,0.78125,0.7894736842105263,0.7073170731707317,0.723404255319149,0.7444750490517091,0.03393029940101563,1,0.7126436781609196,0.7011494252873564,0.7471264367816092,0.6896551724137931,0.735632183908046,0.7172413793103448,0.021318663208036076,1,0.7045454545454546,0.5681818181818182,0.6818181818181818,0.6590909090909091,0.7727272727272727,0.6772727272727274,0.0664942674469445,1,0.7126436781609196,0.6578947368421052,0.7317073170731707,0.6823529411764706,0.7472527472527473,0.7063702841010826,0.03249433165775736,1,SVC,,,,,,,,scale,rbf,
+0.00011134147644042969,4.903430361465795e-05,0.004900646209716797,0.0005161370158997369,,,{'to_return': 1},0.5057471264367817,0.5057471264367817,0.5057471264367817,0.5057471264367817,0.5057471264367817,0.5057471264367817,0.0,1,0.5057471264367817,0.5057471264367817,0.5057471264367817,0.5057471264367817,0.5057471264367817,0.5057471264367817,0.0,1,1.0,1.0,1.0,1.0,1.0,1.0,0.0,1,0.6717557251908397,0.6717557251908397,0.6717557251908397,0.6717557251908397,0.6717557251908397,0.6717557251908397,0.0,1,BiasedClassifier,,,,,,,,,,1
+0.002200460433959961,0.00020534606753037414,0.004897880554199219,0.00023687623875931181,gini,best,"{'criterion': 'gini', 'splitter': 'best'}",0.8297872340425532,0.9111111111111111,0.7777777777777778,0.7959183673469388,0.8478260869565217,0.8324841154469805,0.04637820829132765,1,0.8620689655172413,0.9310344827586207,0.8390804597701149,0.8390804597701149,0.8735632183908046,0.8689655172413794,0.03378606541769901,1,0.9069767441860465,0.9534883720930233,0.9545454545454546,0.9069767441860465,0.9069767441860465,0.9257928118393235,0.023047306680611702,1,0.8666666666666666,0.9318181818181819,0.8571428571428572,0.8478260869565216,0.8764044943820224,0.8759716573932499,0.029503563665100513,1,DecisionTreeClassifier,,,,,,,,,,
+0.0013114452362060548,0.0002328472352982698,0.005080604553222656,0.0012423829742128443,,,{},0.8333333333333334,0.8421052631578947,0.782608695652174,0.782608695652174,0.9523809523809523,0.8386073880353055,0.06205833034255249,1,0.6896551724137931,0.6551724137931034,0.6436781609195402,0.6551724137931034,0.7241379310344828,0.6735632183908046,0.02961861776258651,1,0.46511627906976744,0.37209302325581395,0.4090909090909091,0.4186046511627907,0.46511627906976744,0.4260042283298097,0.035515453449528465,1,0.5970149253731343,0.5161290322580645,0.537313432835821,0.5454545454545454,0.625,0.564182387184313,0.0403944433699087,1,GaussianNB,,,,,,,,,,
+1.7335651874542237,0.46873655713973056,0.0058174610137939455,0.002204283396201828,,,"{'activation': 'logistic', 'hidden_layer_sizes': (60, 80, 100), 'learning_rate': 'constant', 'max_iter': 500000, 'solver': 'lbfgs'}",0.8125,0.8809523809523809,0.8333333333333334,0.803921568627451,0.8837209302325582,0.8428856426291447,0.033613480134915025,1,0.8505747126436781,0.8735632183908046,0.8620689655172413,0.8620689655172413,0.8850574712643678,0.8666666666666666,0.011721883939293793,1,0.9069767441860465,0.8604651162790697,0.9090909090909091,0.9534883720930233,0.8837209302325582,0.9027484143763214,0.03092756625439281,1,0.8571428571428572,0.8705882352941177,0.8695652173913043,0.8723404255319148,0.8837209302325582,0.8706715331185505,0.00845373188121867,1,MLPClassifier,logistic,"(60, 80, 100)",constant,500000,lbfgs,,,,,
+0.19207940101623536,0.0037181516810966433,0.014526033401489257,0.00295937253763726,gini,,"{'class_weight': 'balanced', 'criterion': 'gini', 'max_features': 'sqrt'}",0.8695652173913043,0.851063829787234,0.9148936170212766,0.8478260869565217,0.925,0.8816697502312673,0.03227993647718546,1,0.896551724137931,0.8850574712643678,0.9425287356321839,0.8735632183908046,0.896551724137931,0.8988505747126435,0.023443767878587488,1,0.9302325581395349,0.9302325581395349,0.9772727272727273,0.9069767441860465,0.8604651162790697,0.9210359408033826,0.03794263429529879,1,0.898876404494382,0.888888888888889,0.945054945054945,0.8764044943820224,0.891566265060241,0.9001581995760958,0.023589904475264656,1,RandomForestClassifier,,,,,,balanced,sqrt,,,
+0.0051666259765625,0.00034187932701668506,0.00728912353515625,0.001704487097268627,,,"{'gamma': 'scale', 'kernel': 'rbf'}",0.717391304347826,0.7021276595744681,0.7142857142857143,0.8085106382978723,0.7446808510638298,0.7373992335139421,0.03817981082531593,1,0.735632183908046,0.7241379310344828,0.7701149425287356,0.8390804597701149,0.7701149425287356,0.7678160919540229,0.04008182936589124,1,0.7674418604651163,0.7674418604651163,0.9090909090909091,0.8837209302325582,0.813953488372093,0.8283298097251585,0.05867179006586292,1,0.7415730337078652,0.7333333333333334,0.8,0.8444444444444444,0.7777777777777778,0.7794257178526842,0.04052353688322142,1,SVC,,,,,,,,scale,rbf,
+8.912086486816407e-05,4.704524797218547e-06,0.005401611328125,0.0014136162828282612,,,{'to_return': 1},0.4942528735632184,0.4942528735632184,0.5057471264367817,0.4942528735632184,0.4942528735632184,0.496551724137931,0.004597701149425305,1,0.4942528735632184,0.4942528735632184,0.5057471264367817,0.4942528735632184,0.4942528735632184,0.496551724137931,0.004597701149425305,1,1.0,1.0,1.0,1.0,1.0,1.0,0.0,1,0.6615384615384615,0.6615384615384615,0.6717557251908397,0.6615384615384615,0.6615384615384615,0.6635819142689371,0.004086905460951274,1,BiasedClassifier,,,,,,,,,,1
+0.002162599563598633,0.00011463611565533277,0.005778264999389648,0.0011629495747886357,gini,best,"{'criterion': 'gini', 'splitter': 'best'}",0.7547169811320755,0.84,0.7818181818181819,0.8,0.9333333333333333,0.8219736992567181,0.06220078905101413,1,0.8160919540229885,0.896551724137931,0.8620689655172413,0.8735632183908046,0.9540229885057471,0.8804597701149424,0.04516524759629539,1,0.9302325581395349,0.9767441860465116,1.0,1.0,0.9767441860465116,0.9767441860465116,0.025475467790937966,1,0.8333333333333334,0.9032258064516129,0.8775510204081634,0.888888888888889,0.9545454545454545,0.8915089007254906,0.039235128453130444,1,DecisionTreeClassifier,,,,,,,,,,
+0.0011775970458984375,4.386730812718879e-05,0.004958534240722656,0.0002850278946466431,,,{},0.625,0.782608695652174,0.7894736842105263,0.875,0.9523809523809523,0.8048926664487306,0.10932842678338679,1,0.5517241379310345,0.6551724137931034,0.632183908045977,0.7701149425287356,0.7241379310344828,0.6666666666666666,0.0755479389662298,1,0.23255813953488372,0.4186046511627907,0.3488372093023256,0.6363636363636364,0.46511627906976744,0.4202959830866807,0.13345321620743475,1,0.33898305084745767,0.5454545454545454,0.48387096774193555,0.7368421052631579,0.625,0.5460301338614193,0.13369969035673737,1,GaussianNB,,,,,,,,,,
+1.1879018306732179,0.22662159651745153,0.0044406414031982425,0.000705439773300126,,,"{'activation': 'logistic', 'hidden_layer_sizes': (60, 80, 100), 'learning_rate': 'constant', 'max_iter': 500000, 'solver': 'lbfgs'}",0.8367346938775511,0.9333333333333333,0.8260869565217391,0.7857142857142857,0.9333333333333333,0.8630405205560485,0.059864961476919436,1,0.8850574712643678,0.9540229885057471,0.8505747126436781,0.8620689655172413,0.9540229885057471,0.9011494252873563,0.044576366504977726,1,0.9534883720930233,0.9767441860465116,0.8837209302325582,1.0,0.9767441860465116,0.9581395348837208,0.04001081519554709,1,0.8913043478260869,0.9545454545454545,0.853932584269663,0.88,0.9545454545454545,0.9068655682373317,0.04077389911655387,1,MLPClassifier,logistic,"(60, 80, 100)",constant,500000,lbfgs,,,,,
+0.20467829704284668,0.0022406586232805925,0.015773487091064454,0.0018224619895604826,gini,,"{'class_weight': 'balanced', 'criterion': 'gini', 'max_features': 'sqrt'}",0.9069767441860465,0.875,0.8571428571428571,0.88,0.9555555555555556,0.8949350313768918,0.03425948006212634,1,0.9080459770114943,0.9195402298850575,0.9080459770114943,0.9310344827586207,0.9770114942528736,0.9287356321839081,0.025598916610712757,1,0.9069767441860465,0.9767441860465116,0.9767441860465116,1.0,1.0,0.9720930232558139,0.03417892664348623,1,0.9069767441860465,0.923076923076923,0.9130434782608695,0.9361702127659575,0.9772727272727273,0.9313080171125048,0.025018995413057674,1,RandomForestClassifier,,,,,,balanced,sqrt,,,
+0.005266332626342773,0.0008829188156004573,0.006360149383544922,0.000921209104419299,,,"{'gamma': 'scale', 'kernel': 'rbf'}",0.6944444444444444,0.75,0.8095238095238095,0.8076923076923077,0.8695652173913043,0.7862451558103731,0.059473003340029494,1,0.6666666666666666,0.735632183908046,0.8045977011494253,0.8620689655172413,0.896551724137931,0.793103448275862,0.0835213890785515,1,0.5813953488372093,0.6976744186046512,0.7906976744186046,0.9545454545454546,0.9302325581395349,0.7909090909090909,0.1405536516486544,1,0.6329113924050633,0.7228915662650603,0.8,0.875,0.898876404494382,0.7859358726329011,0.0982617507617456,1,SVC,,,,,,,,scale,rbf,
+0.00010089874267578125,2.7320480583703417e-05,0.005885505676269531,0.0009496829264005458,,,{'to_return': 1},0.4942528735632184,0.4942528735632184,0.4942528735632184,0.5057471264367817,0.4942528735632184,0.496551724137931,0.004597701149425305,1,0.4942528735632184,0.4942528735632184,0.4942528735632184,0.5057471264367817,0.4942528735632184,0.496551724137931,0.004597701149425305,1,1.0,1.0,1.0,1.0,1.0,1.0,0.0,1,0.6615384615384615,0.6615384615384615,0.6615384615384615,0.6717557251908397,0.6615384615384615,0.6635819142689372,0.004086905460951274,1,BiasedClassifier,,,,,,,,,,1
+0.0025852203369140627,0.0002888902644231681,0.008424186706542968,0.0021211058855906766,gini,best,"{'criterion': 'gini', 'splitter': 'best'}",0.86,0.875,0.7818181818181819,0.8461538461538461,0.8431372549019608,0.8412218565747978,0.03178166606803471,1,0.9080459770114943,0.9195402298850575,0.8620689655172413,0.9080459770114943,0.9080459770114943,0.9011494252873563,0.020040914682945644,1,0.9772727272727273,0.9767441860465116,1.0,1.0,1.0,0.9908033826638478,0.011264749940598078,1,0.9148936170212766,0.923076923076923,0.8775510204081634,0.9166666666666666,0.9148936170212766,0.9094163688388612,0.016214675368913384,1,DecisionTreeClassifier,,,,,,,,,,
+0.0013147830963134766,0.0002624367468080042,0.006475019454956055,0.002189931541185622,,,{},0.9375,0.8095238095238095,0.95,0.7619047619047619,0.8333333333333334,0.8584523809523809,0.07345189858605215,1,0.6551724137931034,0.6551724137931034,0.7126436781609196,0.6206896551724138,0.5977011494252874,0.6482758620689656,0.03887709086273048,1,0.3409090909090909,0.3953488372093023,0.4418604651162791,0.36363636363636365,0.23255813953488372,0.354862579281184,0.06991112119272025,1,0.5,0.53125,0.6031746031746031,0.4923076923076923,0.3636363636363636,0.4980737318237319,0.07779102983602866,1,GaussianNB,,,,,,,,,,
+1.0479557514190674,0.08324623938687833,0.0047226428985595705,0.0006157492109823827,,,"{'activation': 'logistic', 'hidden_layer_sizes': (60, 80, 100), 'learning_rate': 'constant', 'max_iter': 500000, 'solver': 'lbfgs'}",0.8775510204081632,0.7924528301886793,0.7884615384615384,0.9361702127659575,0.8775510204081632,0.8544373244465003,0.05646881047053954,1,0.9195402298850575,0.8620689655172413,0.8505747126436781,0.9655172413793104,0.9310344827586207,0.9057471264367816,0.04325261545339492,1,0.9772727272727273,0.9767441860465116,0.9534883720930233,1.0,1.0,0.9815010570824525,0.017376069138229963,1,0.9247311827956989,0.875,0.8631578947368421,0.967032967032967,0.9347826086956522,0.9129409306522321,0.038626110932591155,1,MLPClassifier,logistic,"(60, 80, 100)",constant,500000,lbfgs,,,,,
+0.19396591186523438,0.003609898885404677,0.012253856658935547,0.00026434699497171824,gini,,"{'class_weight': 'balanced', 'criterion': 'gini', 'max_features': 'sqrt'}",0.9545454545454546,0.8541666666666666,0.8431372549019608,0.88,0.9347826086956522,0.8933263969619467,0.04403729741077802,1,0.9540229885057471,0.896551724137931,0.9080459770114943,0.9310344827586207,0.9655172413793104,0.9310344827586207,0.026210929312623858,1,0.9545454545454546,0.9534883720930233,1.0,1.0,1.0,0.9816067653276956,0.022529499881196128,1,0.9545454545454546,0.9010989010989011,0.9148936170212766,0.9361702127659575,0.9662921348314606,0.93460006405261,0.024146814667758826,1,RandomForestClassifier,,,,,,balanced,sqrt,,,
+0.00482325553894043,0.00015945875182292734,0.006249189376831055,0.00037919171963596884,,,"{'gamma': 'scale', 'kernel': 'rbf'}",0.8108108108108109,0.7560975609756098,0.673469387755102,0.75,0.6904761904761905,0.7361707900035427,0.04935365888461486,1,0.7586206896551724,0.7471264367816092,0.7011494252873564,0.7241379310344828,0.6896551724137931,0.7241379310344828,0.02621092931262383,1,0.6818181818181818,0.7209302325581395,0.7674418604651163,0.6818181818181818,0.6744186046511628,0.7052854122621564,0.035106597889331737,1,0.7407407407407407,0.7380952380952381,0.7173913043478259,0.7142857142857143,0.6823529411764705,0.7185731877291979,0.020996172957040878,1,SVC,,,,,,,,scale,rbf,
+0.00012807846069335939,4.9965708754241704e-05,0.005942535400390625,0.0008432287970806463,,,{'to_return': 1},0.5057471264367817,0.4942528735632184,0.4942528735632184,0.5057471264367817,0.4942528735632184,0.4988505747126437,0.005631010902949855,1,0.5057471264367817,0.4942528735632184,0.4942528735632184,0.5057471264367817,0.4942528735632184,0.4988505747126437,0.005631010902949855,1,1.0,1.0,1.0,1.0,1.0,1.0,0.0,1,0.6717557251908397,0.6615384615384615,0.6615384615384615,0.6717557251908397,0.6615384615384615,0.6656253669994128,0.005005416503162351,1,BiasedClassifier,,,,,,,,,,1
+0.00256495475769043,0.00015650071578116354,0.005503225326538086,0.0001899081074143049,gini,best,"{'criterion': 'gini', 'splitter': 'best'}",0.8367346938775511,0.82,0.7647058823529411,0.8541666666666666,0.7777777777777778,0.8106770041349873,0.034213960093917035,1,0.8735632183908046,0.8735632183908046,0.8045977011494253,0.8850574712643678,0.8505747126436781,0.8574712643678162,0.028712634475394943,1,0.9318181818181818,0.9534883720930233,0.8863636363636364,0.9318181818181818,0.9767441860465116,0.936046511627907,0.0298819840039179,1,0.8817204301075268,0.8817204301075269,0.8210526315789474,0.8913043478260869,0.8659793814432991,0.8683554442126773,0.025008397111140102,1,DecisionTreeClassifier,,,,,,,,,,
+0.0013147830963134766,0.00016054524336622277,0.005717658996582031,0.0003844313276936607,,,{},0.7368421052631579,0.7391304347826086,0.84,0.8148148148148148,0.8,0.7861574709721163,0.04136526157514908,1,0.5977011494252874,0.632183908045977,0.6896551724137931,0.6896551724137931,0.6436781609195402,0.6505747126436783,0.03531561263387867,1,0.3181818181818182,0.3953488372093023,0.4772727272727273,0.5,0.37209302325581395,0.41257928118393233,0.0673417112152376,1,0.44444444444444436,0.5151515151515151,0.6086956521739131,0.6197183098591549,0.5079365079365079,0.5391892859131071,0.06610173097256672,1,GaussianNB,,,,,,,,,,
+1.0477601528167724,0.20298421513586734,0.0041754722595214845,0.0005256725239497262,,,"{'activation': 'logistic', 'hidden_layer_sizes': (60, 80, 100), 'learning_rate': 'constant', 'max_iter': 500000, 'solver': 'lbfgs'}",0.7884615384615384,0.8478260869565217,0.74,0.8666666666666667,0.8367346938775511,0.8159377971924556,0.04592229517397828,1,0.8390804597701149,0.8735632183908046,0.7701149425287356,0.8735632183908046,0.8850574712643678,0.8482758620689654,0.04201302731608524,1,0.9318181818181818,0.9069767441860465,0.8409090909090909,0.8863636363636364,0.9534883720930233,0.9039112050739957,0.038789885092825646,1,0.8541666666666667,0.8764044943820224,0.7872340425531915,0.8764044943820225,0.8913043478260869,0.8571028091619979,0.03689206102089107,1,MLPClassifier,logistic,"(60, 80, 100)",constant,500000,lbfgs,,,,,
+0.2054218292236328,0.002629685939615766,0.017171573638916016,0.003591558498269382,gini,,"{'class_weight': 'balanced', 'criterion': 'gini', 'max_features': 'sqrt'}",0.86,0.9090909090909091,0.8863636363636364,0.9111111111111111,0.8775510204081632,0.8888233353947639,0.01934641753747082,1,0.9080459770114943,0.9195402298850575,0.8850574712643678,0.9195402298850575,0.9310344827586207,0.9126436781609195,0.015591563179598299,1,0.9772727272727273,0.9302325581395349,0.8863636363636364,0.9318181818181818,1.0,0.9451374207188159,0.039740395854914776,1,0.9148936170212766,0.9195402298850575,0.8863636363636365,0.9213483146067416,0.9347826086956522,0.915385681314473,0.015947258584666963,1,RandomForestClassifier,,,,,,balanced,sqrt,,,
+0.006590557098388672,0.0006515370283574501,0.006815481185913086,0.00030984676738491626,,,"{'gamma': 'scale', 'kernel': 'rbf'}",0.717391304347826,0.7906976744186046,0.782608695652174,0.7272727272727273,0.7837837837837838,0.7603508370950232,0.03132133010229837,1,0.7241379310344828,0.7931034482758621,0.7931034482758621,0.7241379310344828,0.7471264367816092,0.7563218390804598,0.03118312635919665,1,0.75,0.7906976744186046,0.8181818181818182,0.7272727272727273,0.6744186046511628,0.7521141649048626,0.05002357207543973,1,0.7333333333333332,0.7906976744186046,0.8,0.7272727272727273,0.725,0.7552607470049331,0.03297644647693505,1,SVC,,,,,,,,scale,rbf,
+0.00026397705078125,0.00024138971123202716,0.008141565322875976,0.002164818796181482,,,{'to_return': 1},0.5057471264367817,0.4942528735632184,0.5057471264367817,0.5057471264367817,0.4942528735632184,0.5011494252873564,0.005631010902949855,1,0.5057471264367817,0.4942528735632184,0.5057471264367817,0.5057471264367817,0.4942528735632184,0.5011494252873564,0.005631010902949855,1,1.0,1.0,1.0,1.0,1.0,1.0,0.0,1,0.6717557251908397,0.6615384615384615,0.6717557251908397,0.6717557251908397,0.6615384615384615,0.6676688197298885,0.005005416503162351,1,BiasedClassifier,,,,,,,,,,1
+0.0022379875183105467,7.523311769264513e-05,0.007052278518676758,0.0017106026771576644,gini,best,"{'criterion': 'gini', 'splitter': 'best'}",0.8333333333333334,0.8367346938775511,0.8235294117647058,0.875,0.8541666666666666,0.8445528211284514,0.018157315267629374,1,0.8735632183908046,0.8735632183908046,0.8735632183908046,0.9080459770114943,0.896551724137931,0.8850574712643677,0.014539207632958049,1,0.9302325581395349,0.9318181818181818,0.9545454545454546,0.9545454545454546,0.9534883720930233,0.94492600422833,0.011367446138494802,1,0.8791208791208791,0.8817204301075268,0.8842105263157896,0.9130434782608695,0.9010989010989011,0.8918388429807932,0.013097368687406876,1,DecisionTreeClassifier,,,,,,,,,,
+0.0011313915252685546,7.793182372957246e-05,0.007006502151489258,0.0022371042748052664,,,{},0.8636363636363636,0.782608695652174,0.7142857142857143,0.7272727272727273,0.8333333333333334,0.7842273668360625,0.058061123902532125,1,0.6896551724137931,0.6436781609195402,0.5632183908045977,0.6091954022988506,0.5977011494252874,0.6206896551724138,0.043007556169815435,1,0.4418604651162791,0.4090909090909091,0.22727272727272727,0.36363636363636365,0.23255813953488372,0.33488372093023255,0.08925031719584993,1,0.5846153846153846,0.537313432835821,0.3448275862068965,0.4848484848484849,0.3636363636363636,0.46304825042859016,0.09447537704369392,1,GaussianNB,,,,,,,,,,
+1.3931004047393798,0.09965616673795333,0.005034875869750976,0.0005051953074006412,,,"{'activation': 'logistic', 'hidden_layer_sizes': (60, 80, 100), 'learning_rate': 'constant', 'max_iter': 500000, 'solver': 'lbfgs'}",0.8235294117647058,0.8541666666666666,0.8301886792452831,0.7924528301886793,0.8958333333333334,0.8392341842397336,0.03447723549096509,1,0.8850574712643678,0.8850574712643678,0.896551724137931,0.8505747126436781,0.9425287356321839,0.8919540229885058,0.02961861776258649,1,0.9767441860465116,0.9318181818181818,1.0,0.9545454545454546,1.0,0.9726215644820296,0.026487035540576458,1,0.8936170212765957,0.8913043478260869,0.9072164948453608,0.865979381443299,0.945054945054945,0.9006344380892575,0.0258989288751693,1,MLPClassifier,logistic,"(60, 80, 100)",constant,500000,lbfgs,,,,,
+0.20398807525634766,0.0024655745813987003,0.014611768722534179,0.0022107979105587586,gini,,"{'class_weight': 'balanced', 'criterion': 'gini', 'max_features': 'sqrt'}",0.851063829787234,0.8113207547169812,0.8979591836734694,0.8936170212765957,0.9111111111111111,0.8730143801130783,0.0368402036759761,1,0.8850574712643678,0.8735632183908046,0.9425287356321839,0.9195402298850575,0.9310344827586207,0.9103448275862069,0.026611119316759108,1,0.9302325581395349,0.9772727272727273,1.0,0.9545454545454546,0.9534883720930233,0.9631078224101481,0.02369940195787108,1,0.888888888888889,0.88659793814433,0.9462365591397849,0.9230769230769231,0.9318181818181819,0.9153236982136217,0.02371393637250391,1,RandomForestClassifier,,,,,,balanced,sqrt,,,
+0.005649709701538086,0.0005415109101334089,0.0069865226745605465,0.0005735623344444109,,,"{'gamma': 'scale', 'kernel': 'rbf'}",0.7727272727272727,0.6181818181818182,0.6666666666666666,0.7647058823529411,0.8857142857142857,0.7415991851285969,0.09286416779158137,1,0.7816091954022989,0.6436781609195402,0.6206896551724138,0.7011494252873564,0.8160919540229885,0.7126436781609196,0.07589689204238703,1,0.7906976744186046,0.7727272727272727,0.5,0.5909090909090909,0.7209302325581395,0.6750528541226215,0.11202002052077396,1,0.7816091954022988,0.6868686868686869,0.5714285714285715,0.6666666666666667,0.7948717948717948,0.7002889830476038,0.08182231409702262,1,SVC,,,,,,,,scale,rbf,
+0.00010442733764648438,2.756184771846471e-05,0.004592609405517578,8.964569010581436e-05,,,{'to_return': 1},0.4942528735632184,0.5057471264367817,0.5057471264367817,0.5057471264367817,0.4942528735632184,0.5011494252873564,0.005631010902949855,1,0.4942528735632184,0.5057471264367817,0.5057471264367817,0.5057471264367817,0.4942528735632184,0.5011494252873564,0.005631010902949855,1,1.0,1.0,1.0,1.0,1.0,1.0,0.0,1,0.6615384615384615,0.6717557251908397,0.6717557251908397,0.6717557251908397,0.6615384615384615,0.6676688197298885,0.005005416503162351,1,BiasedClassifier,,,,,,,,,,1
+0.002202606201171875,8.199014921929533e-05,0.004653835296630859,0.0003465135831855013,gini,best,"{'criterion': 'gini', 'splitter': 'best'}",0.9148936170212766,0.8775510204081632,0.8301886792452831,0.8235294117647058,0.8235294117647058,0.8539384280408269,0.036569267675365204,1,0.9540229885057471,0.9310344827586207,0.896551724137931,0.8850574712643678,0.8850574712643678,0.9103448275862069,0.027586206896551693,1,1.0,1.0,1.0,0.9767441860465116,0.9767441860465116,0.9906976744186047,0.011392975547828756,1,0.9555555555555556,0.9347826086956522,0.9072164948453608,0.8936170212765957,0.8936170212765957,0.916957740329952,0.0244621645317452,1,DecisionTreeClassifier,,,,,,,,,,
+0.0012935638427734376,0.00027419992467017824,0.005270767211914063,0.0005503262196998468,,,{},0.9166666666666666,0.75,0.8125,0.8260869565217391,0.92,0.8450507246376813,0.06511536141307574,1,0.735632183908046,0.5977011494252874,0.7241379310344828,0.6781609195402298,0.7471264367816092,0.6965517241379311,0.05469138967066811,1,0.5116279069767442,0.27906976744186046,0.5909090909090909,0.4418604651162791,0.5348837209302325,0.4716701902748414,0.10751188547866333,1,0.6567164179104478,0.4067796610169491,0.6842105263157896,0.5757575757575758,0.6764705882352942,0.5999869538472113,0.10399779742560995,1,GaussianNB,,,,,,,,,,
+1.244570779800415,0.3043393347584673,0.004804563522338867,0.0002211505453833038,,,"{'activation': 'logistic', 'hidden_layer_sizes': (60, 80, 100), 'learning_rate': 'constant', 'max_iter': 500000, 'solver': 'lbfgs'}",0.9130434782608695,0.7368421052631579,0.8148148148148148,0.8541666666666666,0.7818181818181819,0.8201370493647382,0.06037272378729973,1,0.9425287356321839,0.8160919540229885,0.8850574712643678,0.896551724137931,0.8620689655172413,0.8804597701149426,0.041506827782224795,1,0.9767441860465116,0.9767441860465116,1.0,0.9534883720930233,1.0,0.9813953488372092,0.017403057612902048,1,0.9438202247191011,0.8399999999999999,0.8979591836734693,0.9010989010989011,0.8775510204081634,0.8920858659799269,0.0338123459925887,1,MLPClassifier,logistic,"(60, 80, 100)",constant,500000,lbfgs,,,,,
+0.1939687728881836,0.003765016415698473,0.012894678115844726,0.0003168130408899265,gini,,"{'class_weight': 'balanced', 'criterion': 'gini', 'max_features': 'sqrt'}",0.8958333333333334,0.8958333333333334,0.8301886792452831,0.9318181818181818,0.8775510204081632,0.886244909627659,0.033101913995759895,1,0.9425287356321839,0.9425287356321839,0.896551724137931,0.9425287356321839,0.9310344827586207,0.9310344827586207,0.017806819982562814,1,1.0,1.0,1.0,0.9534883720930233,1.0,0.9906976744186047,0.018604651162790687,1,0.945054945054945,0.945054945054945,0.9072164948453608,0.942528735632184,0.9347826086956522,0.9349275458566174,0.014358480908354091,1,RandomForestClassifier,,,,,,balanced,sqrt,,,
+0.005279970169067383,0.0008765591193435004,0.007296323776245117,0.0008316393011371611,,,"{'gamma': 'scale', 'kernel': 'rbf'}",0.7608695652173914,0.7209302325581395,0.78,0.7272727272727273,0.6808510638297872,0.7339847177756091,0.03428952260559052,1,0.7816091954022989,0.7241379310344828,0.8160919540229885,0.735632183908046,0.7011494252873564,0.7517241379310345,0.0415068277822248,1,0.813953488372093,0.7209302325581395,0.8863636363636364,0.7441860465116279,0.7441860465116279,0.7819238900634249,0.060831031066091654,1,0.7865168539325844,0.7209302325581395,0.8297872340425532,0.735632183908046,0.711111111111111,0.7567955231104868,0.044793791674645,1,SVC,,,,,,,,scale,rbf,
+0.00011925697326660156,3.990600008791067e-05,0.005774450302124023,0.0013703348492437616,,,{'to_return': 1},0.4942528735632184,0.4942528735632184,0.5057471264367817,0.4942528735632184,0.4942528735632184,0.496551724137931,0.004597701149425305,1,0.4942528735632184,0.4942528735632184,0.5057471264367817,0.4942528735632184,0.4942528735632184,0.496551724137931,0.004597701149425305,1,1.0,1.0,1.0,1.0,1.0,1.0,0.0,1,0.6615384615384615,0.6615384615384615,0.6717557251908397,0.6615384615384615,0.6615384615384615,0.6635819142689371,0.004086905460951274,1,BiasedClassifier,,,,,,,,,,1
+0.003740835189819336,0.0008560569652560701,0.011518573760986328,0.0018415989280215705,gini,best,"{'criterion': 'gini', 'splitter': 'best'}",0.8478260869565217,0.7777777777777778,0.803921568627451,0.7636363636363637,0.8,0.7986323593996228,0.028668806393340435,1,0.8735632183908046,0.8505747126436781,0.8505747126436781,0.8390804597701149,0.8505747126436781,0.8528735632183908,0.011262021805899698,1,0.9069767441860465,0.9767441860465116,0.9318181818181818,0.9767441860465116,0.9302325581395349,0.9445031712473572,0.027755209074470696,1,0.8764044943820224,0.8659793814432991,0.8631578947368421,0.8571428571428571,0.8602150537634408,0.8645799362936923,0.006605500264160805,1,DecisionTreeClassifier,,,,,,,,,,
+0.0017208576202392579,0.0001298221367926076,0.007273340225219726,0.0012454420689525531,,,{},0.7,0.8636363636363636,0.8666666666666667,0.75,0.8666666666666667,0.8093939393939393,0.07070678210619315,1,0.5977011494252874,0.6896551724137931,0.6206896551724138,0.6206896551724138,0.632183908045977,0.632183908045977,0.030842316931031583,1,0.32558139534883723,0.4418604651162791,0.29545454545454547,0.3488372093023256,0.3023255813953488,0.3428118393234672,0.0529635231418738,1,0.44444444444444453,0.5846153846153846,0.44067796610169496,0.4761904761904763,0.44827586206896547,0.47884082668419314,0.054348897991092215,1,GaussianNB,,,,,,,,,,
+0.9770734310150146,0.1658866872160389,0.005283164978027344,0.0020174377396788377,,,"{'activation': 'logistic', 'hidden_layer_sizes': (60, 80, 100), 'learning_rate': 'constant', 'max_iter': 500000, 'solver': 'lbfgs'}",0.8125,0.6842105263157895,0.8297872340425532,0.8076923076923077,0.8571428571428571,0.7982665850387014,0.05959896913691404,1,0.8505747126436781,0.7471264367816092,0.8505747126436781,0.8735632183908046,0.9080459770114943,0.8459770114942529,0.05371642043930081,1,0.9069767441860465,0.9069767441860465,0.8863636363636364,0.9767441860465116,0.9767441860465116,0.9307610993657505,0.038292073187628534,1,0.8571428571428572,0.78,0.8571428571428571,0.8842105263157894,0.9130434782608695,0.8583079437724747,0.04427588831345019,1,MLPClassifier,logistic,"(60, 80, 100)",constant,500000,lbfgs,,,,,
+0.19642195701599122,0.0017260620755782006,0.013675546646118164,0.0016129140021678028,gini,,"{'class_weight': 'balanced', 'criterion': 'gini', 'max_features': 'sqrt'}",0.8297872340425532,0.8,0.8571428571428571,0.7818181818181819,0.84,0.8217496546007185,0.027268760773085165,1,0.8620689655172413,0.8505747126436781,0.896551724137931,0.8620689655172413,0.896551724137931,0.8735632183908045,0.019233563828369583,1,0.9069767441860465,0.9302325581395349,0.9545454545454546,1.0,0.9767441860465116,0.9536997885835096,0.032891405458087795,1,0.8666666666666666,0.8602150537634408,0.9032258064516128,0.8775510204081634,0.9032258064516129,0.8821768707482993,0.018057694846154125,1,RandomForestClassifier,,,,,,balanced,sqrt,,,
+0.005113554000854492,0.00048391269672356493,0.006591796875,0.0005771967753280567,,,"{'gamma': 'scale', 'kernel': 'rbf'}",0.7307692307692307,0.7755102040816326,0.7894736842105263,0.6875,0.75,0.7466506238122779,0.03585783913849692,1,0.7816091954022989,0.8160919540229885,0.7471264367816092,0.7126436781609196,0.735632183908046,0.7586206896551724,0.03634801908239517,1,0.8837209302325582,0.8837209302325582,0.6818181818181818,0.7674418604651163,0.6976744186046512,0.782875264270613,0.08723486075604465,1,0.8,0.826086956521739,0.7317073170731707,0.7252747252747254,0.7228915662650603,0.761192113026939,0.04322901021574641,1,SVC,,,,,,,,scale,rbf,
+0.0001235485076904297,4.161545588715187e-05,0.0059563636779785155,0.001140669767182086,,,{'to_return': 1},0.4942528735632184,0.4942528735632184,0.5057471264367817,0.4942528735632184,0.4942528735632184,0.496551724137931,0.004597701149425305,1,0.4942528735632184,0.4942528735632184,0.5057471264367817,0.4942528735632184,0.4942528735632184,0.496551724137931,0.004597701149425305,1,1.0,1.0,1.0,1.0,1.0,1.0,0.0,1,0.6615384615384615,0.6615384615384615,0.6717557251908397,0.6615384615384615,0.6615384615384615,0.6635819142689371,0.004086905460951274,1,BiasedClassifier,,,,,,,,,,1
+0.002730274200439453,0.00028384909290979447,0.009071731567382812,0.0016121569596659678,gini,best,"{'criterion': 'gini', 'splitter': 'best'}",0.8,0.8125,0.8269230769230769,0.7962962962962963,0.7735849056603774,0.8018608557759501,0.017749719885155765,1,0.8505747126436781,0.8390804597701149,0.8850574712643678,0.8620689655172413,0.8390804597701149,0.8551724137931036,0.017203022467926186,1,0.9302325581395349,0.8863636363636364,0.9772727272727273,0.9772727272727273,0.9534883720930233,0.9449260042283297,0.034092179205079426,1,0.8602150537634408,0.8478260869565218,0.8958333333333334,0.8775510204081632,0.8541666666666667,0.8671184322256252,0.0174412378751083,1,DecisionTreeClassifier,,,,,,,,,,
+0.0012081146240234375,3.769378885249778e-05,0.006778430938720703,0.0010731698438494657,,,{},0.782608695652174,0.85,0.8461538461538461,0.875,0.9130434782608695,0.8533612040133779,0.04265998708448138,1,0.6551724137931034,0.6551724137931034,0.7011494252873564,0.7701149425287356,0.7241379310344828,0.7011494252873562,0.04361762289887419,1,0.4186046511627907,0.38636363636363635,0.5,0.6363636363636364,0.4883720930232558,0.4859408033826639,0.08634856159563532,1,0.5454545454545454,0.53125,0.6285714285714286,0.7368421052631579,0.6363636363636364,0.6156963431305537,0.07393821140149666,1,GaussianNB,,,,,,,,,,
+1.1008864879608153,0.38655884335243584,0.004242610931396484,0.0004474385257136033,,,"{'activation': 'logistic', 'hidden_layer_sizes': (60, 80, 100), 'learning_rate': 'constant', 'max_iter': 500000, 'solver': 'lbfgs'}",0.8163265306122449,0.8409090909090909,0.9130434782608695,0.8431372549019608,0.8,0.8426832709368333,0.038653465483181944,1,0.8620689655172413,0.8390804597701149,0.9310344827586207,0.896551724137931,0.8505747126436781,0.8758620689655172,0.03362928468581103,1,0.9302325581395349,0.8409090909090909,0.9545454545454546,0.9772727272727273,0.9302325581395349,0.9266384778012686,0.04629900468563652,1,0.8695652173913043,0.8409090909090909,0.9333333333333332,0.9052631578947369,0.8602150537634408,0.8818571706583812,0.033147712247499295,1,MLPClassifier,logistic,"(60, 80, 100)",constant,500000,lbfgs,,,,,
+0.1944653034210205,0.0021298287659961,0.01570138931274414,0.003420572297116351,gini,,"{'class_weight': 'balanced', 'criterion': 'gini', 'max_features': 'sqrt'}",0.8333333333333334,0.8809523809523809,0.875,0.8431372549019608,0.7692307692307693,0.840330747683689,0.03991020263529254,1,0.8735632183908046,0.8620689655172413,0.9080459770114943,0.896551724137931,0.8275862068965517,0.8735632183908046,0.028155054514749183,1,0.9302325581395349,0.8409090909090909,0.9545454545454546,0.9772727272727273,0.9302325581395349,0.9266384778012686,0.04629900468563652,1,0.8791208791208791,0.8604651162790699,0.9130434782608695,0.9052631578947369,0.8421052631578948,0.8799995789426902,0.026640173487975972,1,RandomForestClassifier,,,,,,balanced,sqrt,,,
+0.00528559684753418,0.00022586531460856187,0.006539630889892578,0.0005519733850803031,,,"{'gamma': 'scale', 'kernel': 'rbf'}",0.7111111111111111,0.825,0.8409090909090909,0.7777777777777778,0.7608695652173914,0.7831335090030741,0.04648099349397003,1,0.7241379310344828,0.7931034482758621,0.8390804597701149,0.7816091954022989,0.7816091954022989,0.7839080459770115,0.036637649311515454,1,0.7441860465116279,0.75,0.8409090909090909,0.7954545454545454,0.813953488372093,0.7889006342494714,0.037116391304827384,1,0.7272727272727273,0.7857142857142856,0.8409090909090909,0.7865168539325843,0.7865168539325844,0.7853859623522546,0.03595172186115807,1,SVC,,,,,,,,scale,rbf,
+0.00013723373413085938,4.043865868458364e-05,0.005619573593139649,0.00020906818485107281,,,{'to_return': 1},0.4942528735632184,0.5057471264367817,0.5057471264367817,0.5057471264367817,0.4942528735632184,0.5011494252873564,0.005631010902949855,1,0.4942528735632184,0.5057471264367817,0.5057471264367817,0.5057471264367817,0.4942528735632184,0.5011494252873564,0.005631010902949855,1,1.0,1.0,1.0,1.0,1.0,1.0,0.0,1,0.6615384615384615,0.6717557251908397,0.6717557251908397,0.6717557251908397,0.6615384615384615,0.6676688197298885,0.005005416503162351,1,BiasedClassifier,,,,,,,,,,1
+0.0026053905487060545,0.00012377423322753301,0.0060803890228271484,0.0012691567708560461,gini,best,"{'criterion': 'gini', 'splitter': 'best'}",0.9130434782608695,0.8269230769230769,0.8695652173913043,0.875,0.9111111111111111,0.8791285767372724,0.03164496879137249,1,0.9425287356321839,0.8850574712643678,0.8850574712643678,0.9080459770114943,0.9195402298850575,0.9080459770114941,0.021808811449437075,1,0.9767441860465116,0.9772727272727273,0.9090909090909091,0.9545454545454546,0.9318181818181818,0.9498942917547568,0.026396176455870207,1,0.9438202247191011,0.8958333333333334,0.888888888888889,0.9130434782608695,0.9213483146067416,0.9125868479617869,0.01946673649737717,1,DecisionTreeClassifier,,,,,,,,,,
+0.0010817527770996093,4.825167957714567e-05,0.004527091979980469,0.00010537399941841686,,,{},0.8421052631578947,0.8148148148148148,0.9,0.8695652173913043,0.8666666666666667,0.8586303924061361,0.02859538868506317,1,0.6551724137931034,0.6896551724137931,0.7701149425287356,0.6896551724137931,0.6206896551724138,0.6850574712643678,0.04962536355154688,1,0.37209302325581395,0.5,0.6136363636363636,0.45454545454545453,0.29545454545454547,0.4471458773784355,0.1088214915607599,1,0.5161290322580645,0.6197183098591549,0.7297297297297297,0.5970149253731344,0.44067796610169496,0.5806539926643557,0.09774869299001981,1,GaussianNB,,,,,,,,,,
+1.5829660892486572,0.4694879594316442,0.004829883575439453,0.0011689629137479325,,,"{'activation': 'logistic', 'hidden_layer_sizes': (60, 80, 100), 'learning_rate': 'constant', 'max_iter': 500000, 'solver': 'lbfgs'}",0.8723404255319149,0.9111111111111111,0.8297872340425532,0.8235294117647058,0.9090909090909091,0.8691718183082389,0.03740816198191002,1,0.9080459770114943,0.9195402298850575,0.8505747126436781,0.8735632183908046,0.9080459770114943,0.8919540229885058,0.025804533701889256,1,0.9534883720930233,0.9318181818181818,0.8863636363636364,0.9545454545454546,0.9090909090909091,0.927061310782241,0.026289281428004012,1,0.9111111111111112,0.9213483146067416,0.8571428571428571,0.8842105263157896,0.9090909090909091,0.8965807436534817,0.02319240202332199,1,MLPClassifier,logistic,"(60, 80, 100)",constant,500000,lbfgs,,,,,
+0.19327588081359864,0.0014020858403075208,0.013039779663085938,0.0014272566515328624,gini,,"{'class_weight': 'balanced', 'criterion': 'gini', 'max_features': 'sqrt'}",0.875,0.8958333333333334,0.8888888888888888,0.8979591836734694,0.9555555555555556,0.9026473922902495,0.027645466738354288,1,0.9195402298850575,0.9310344827586207,0.896551724137931,0.9425287356321839,0.9655172413793104,0.9310344827586207,0.022988505747126443,1,0.9767441860465116,0.9772727272727273,0.9090909090909091,1.0,0.9772727272727273,0.968076109936575,0.03079831086240959,1,0.923076923076923,0.9347826086956522,0.8988764044943819,0.9462365591397849,0.9662921348314608,0.9338529260476406,0.02256546141738577,1,RandomForestClassifier,,,,,,balanced,sqrt,,,
+0.005579805374145508,0.00014887911437197273,0.006759786605834961,0.0006649470252304854,,,"{'gamma': 'scale', 'kernel': 'rbf'}",0.78125,0.7857142857142857,0.7272727272727273,0.7727272727272727,0.7804878048780488,0.7694904181184669,0.021518618958659787,1,0.7126436781609196,0.7701149425287356,0.7241379310344828,0.7701149425287356,0.7586206896551724,0.7471264367816092,0.024110548233796558,1,0.5813953488372093,0.75,0.7272727272727273,0.7727272727272727,0.7272727272727273,0.7117336152219874,0.06731349664174523,1,0.6666666666666666,0.7674418604651163,0.7272727272727273,0.7727272727272727,0.7529411764705882,0.7374099407204742,0.03872421296266713,1,SVC,,,,,,,,scale,rbf,
+0.00011072158813476562,4.851020752159076e-05,0.008330488204956054,0.0013405276019831799,,,{'to_return': 1},0.4942528735632184,0.5057471264367817,0.5057471264367817,0.5057471264367817,0.5057471264367817,0.5034482758620691,0.004597701149425305,1,0.4942528735632184,0.5057471264367817,0.5057471264367817,0.5057471264367817,0.5057471264367817,0.5034482758620691,0.004597701149425305,1,1.0,1.0,1.0,1.0,1.0,1.0,0.0,1,0.6615384615384615,0.6717557251908397,0.6717557251908397,0.6717557251908397,0.6717557251908397,0.6697122724603641,0.004086905460951274,1,BiasedClassifier,,,,,,,,,,1
+0.0037708282470703125,0.001065721309113856,0.0054857730865478516,0.0012596444672464688,gini,best,"{'criterion': 'gini', 'splitter': 'best'}",0.7924528301886793,0.9148936170212766,0.7843137254901961,0.8936170212765957,0.7884615384615384,0.8347477464876573,0.05720806857413431,1,0.8505747126436781,0.9425287356321839,0.8390804597701149,0.9195402298850575,0.8390804597701149,0.87816091954023,0.04397960107979078,1,0.9545454545454546,0.9772727272727273,0.9302325581395349,0.9545454545454546,0.9318181818181818,0.9496828752642706,0.01735483446510314,1,0.865979381443299,0.945054945054945,0.851063829787234,0.9230769230769231,0.8541666666666667,0.8878683492058137,0.03867674594419678,1,DecisionTreeClassifier,,,,,,,,,,
+0.003181314468383789,0.0008955317507939424,0.00865950584411621,0.0010530350346202528,,,{},0.9285714285714286,0.9166666666666666,0.8214285714285714,0.8571428571428571,0.7692307692307693,0.8586080586080588,0.0593943365061509,1,0.7701149425287356,0.7241379310344828,0.7126436781609196,0.6666666666666666,0.6551724137931034,0.7057471264367816,0.041506827782224795,1,0.5909090909090909,0.5,0.5348837209302325,0.4090909090909091,0.45454545454545453,0.49788583509513745,0.06291194932924528,1,0.7222222222222223,0.6470588235294118,0.647887323943662,0.5538461538461539,0.5714285714285714,0.6284886189940042,0.06055492920625583,1,GaussianNB,,,,,,,,,,
+1.6712066650390625,0.5435026584710643,0.005199718475341797,0.0007911147565816734,,,"{'activation': 'logistic', 'hidden_layer_sizes': (60, 80, 100), 'learning_rate': 'constant', 'max_iter': 500000, 'solver': 'lbfgs'}",0.8163265306122449,0.8461538461538461,0.8163265306122449,0.8125,0.8076923076923077,0.8197998430141288,0.013553902752863936,1,0.8505747126436781,0.9080459770114943,0.8620689655172413,0.8390804597701149,0.8620689655172413,0.864367816091954,0.02344376787858753,1,0.9090909090909091,1.0,0.9302325581395349,0.8863636363636364,0.9545454545454546,0.936046511627907,0.03914346489990062,1,0.8602150537634408,0.9166666666666666,0.8695652173913043,0.8478260869565218,0.875,0.8738546049555869,0.02330709981695928,1,MLPClassifier,logistic,"(60, 80, 100)",constant,500000,lbfgs,,,,,
+0.24261794090270997,0.0027998896084300315,0.013295841217041016,0.0005744904596258133,gini,,"{'class_weight': 'balanced', 'criterion': 'gini', 'max_features': 'sqrt'}",0.851063829787234,0.9555555555555556,0.8076923076923077,0.9130434782608695,0.7884615384615384,0.8631633419515012,0.06297323624199293,1,0.8735632183908046,0.9655172413793104,0.8735632183908046,0.9310344827586207,0.8390804597701149,0.8965517241379309,0.045398661283060915,1,0.9090909090909091,0.9772727272727273,0.9767441860465116,0.9545454545454546,0.9318181818181818,0.9498942917547568,0.026396176455870207,1,0.8791208791208791,0.9662921348314608,0.8842105263157894,0.9333333333333332,0.8541666666666667,0.903424708053626,0.04057845422811123,1,RandomForestClassifier,,,,,,balanced,sqrt,,,
+0.005695199966430664,0.0010135756854755902,0.007267904281616211,0.0006372510619464208,,,"{'gamma': 'scale', 'kernel': 'rbf'}",0.8085106382978723,0.813953488372093,0.7,0.8333333333333334,0.7,0.7711594920006597,0.05868452505524505,1,0.8275862068965517,0.8045977011494253,0.6896551724137931,0.7701149425287356,0.7241379310344828,0.7632183908045976,0.0506790981265485,1,0.8636363636363636,0.7954545454545454,0.6511627906976745,0.6818181818181818,0.7954545454545454,0.7575052854122621,0.07897007315826317,1,0.8351648351648351,0.8045977011494252,0.674698795180723,0.7499999999999999,0.7446808510638298,0.7618284365117626,0.055195947968182155,1,SVC,,,,,,,,scale,rbf,
+9.436607360839844e-05,1.1031749002050198e-05,0.005015230178833008,0.000552628337985854,,,{'to_return': 1},0.5057471264367817,0.5057471264367817,0.4942528735632184,0.5057471264367817,0.5057471264367817,0.5034482758620691,0.004597701149425304,1,0.5057471264367817,0.5057471264367817,0.4942528735632184,0.5057471264367817,0.5057471264367817,0.5034482758620691,0.004597701149425304,1,1.0,1.0,1.0,1.0,1.0,1.0,0.0,1,0.6717557251908397,0.6717557251908397,0.6615384615384615,0.6717557251908397,0.6717557251908397,0.6697122724603641,0.004086905460951274,1,BiasedClassifier,,,,,,,,,,1
+0.0027812957763671876,0.0004336907641480536,0.007120752334594726,0.0010501252301157967,gini,best,"{'criterion': 'gini', 'splitter': 'best'}",0.7454545454545455,0.8113207547169812,0.8297872340425532,0.8269230769230769,0.8367346938775511,0.8100440610029415,0.033348221568675535,1,0.8160919540229885,0.8850574712643678,0.8620689655172413,0.896551724137931,0.8735632183908046,0.8666666666666666,0.02777711718067719,1,0.9534883720930233,1.0,0.9069767441860465,1.0,0.9318181818181818,0.9584566596194503,0.03697613374460007,1,0.8367346938775511,0.8958333333333334,0.8666666666666666,0.9052631578947368,0.8817204301075268,0.877243656375963,0.024097961788466172,1,DecisionTreeClassifier,,,,,,,,,,
+0.0011383533477783204,0.00010591363725564466,0.004954624176025391,0.00026727245489039053,,,{},0.8260869565217391,0.7692307692307693,0.7272727272727273,0.8571428571428571,0.782608695652174,0.7924684011640534,0.04516784197918672,1,0.6781609195402298,0.5862068965517241,0.6206896551724138,0.6781609195402298,0.6436781609195402,0.6413793103448275,0.03516565181788127,1,0.4418604651162791,0.23255813953488372,0.37209302325581395,0.4186046511627907,0.4090909090909091,0.37484143763213534,0.07460815759375224,1,0.5757575757575758,0.3571428571428571,0.49230769230769234,0.5625000000000001,0.537313432835821,0.5050043116087892,0.07922104738541158,1,GaussianNB,,,,,,,,,,
+1.995161199569702,0.7529384156967494,0.005998659133911133,0.0014307593401110862,,,"{'activation': 'logistic', 'hidden_layer_sizes': (60, 80, 100), 'learning_rate': 'constant', 'max_iter': 500000, 'solver': 'lbfgs'}",0.7884615384615384,0.7636363636363637,0.7755102040816326,0.8431372549019608,0.8571428571428571,0.8055776436448705,0.037485406728924545,1,0.8505747126436781,0.8390804597701149,0.8160919540229885,0.9080459770114943,0.896551724137931,0.8620689655172413,0.03486379514506461,1,0.9534883720930233,0.9767441860465116,0.8837209302325582,1.0,0.9545454545454546,0.9536997885835096,0.038916716962248216,1,0.8631578947368421,0.8571428571428571,0.826086956521739,0.9148936170212766,0.9032258064516128,0.8729014263748655,0.03230410377327937,1,MLPClassifier,logistic,"(60, 80, 100)",constant,500000,lbfgs,,,,,
+0.21839437484741211,0.0018267138045557631,0.014053153991699218,0.0007263422290151501,gini,,"{'class_weight': 'balanced', 'criterion': 'gini', 'max_features': 'sqrt'}",0.8367346938775511,0.84,0.7647058823529411,0.86,0.8269230769230769,0.8256727306307138,0.03232497194300956,1,0.8850574712643678,0.896551724137931,0.8160919540229885,0.9195402298850575,0.8850574712643678,0.8804597701149424,0.03455930201924806,1,0.9534883720930233,0.9767441860465116,0.9069767441860465,1.0,0.9772727272727273,0.9628964059196617,0.0315931511332815,1,0.8913043478260869,0.9032258064516129,0.8297872340425532,0.924731182795699,0.8958333333333334,0.888976380889857,0.031740334968446686,1,RandomForestClassifier,,,,,,balanced,sqrt,,,
+0.005539226531982422,0.0003132011288837995,0.006952476501464844,0.0005806192810137669,,,"{'gamma': 'scale', 'kernel': 'rbf'}",0.6666666666666666,0.7441860465116279,0.7674418604651163,0.75,0.782608695652174,0.7421806538591169,0.0401000777003621,1,0.7126436781609196,0.7471264367816092,0.7701149425287356,0.7816091954022989,0.7931034482758621,0.760919540229885,0.028527985393082433,1,0.8372093023255814,0.7441860465116279,0.7674418604651163,0.8372093023255814,0.8181818181818182,0.8008456659619452,0.03813475310235768,1,0.7422680412371134,0.7441860465116278,0.7674418604651162,0.7912087912087912,0.8,0.7690209478845297,0.02360926055055637,1,SVC,,,,,,,,scale,rbf,
+0.000119781494140625,2.4980806525700427e-05,0.0058937549591064455,0.000389733609456147,,,{'to_return': 1},0.4942528735632184,0.4942528735632184,0.4942528735632184,0.4942528735632184,0.5057471264367817,0.496551724137931,0.004597701149425304,1,0.4942528735632184,0.4942528735632184,0.4942528735632184,0.4942528735632184,0.5057471264367817,0.496551724137931,0.004597701149425304,1,1.0,1.0,1.0,1.0,1.0,1.0,0.0,1,0.6615384615384615,0.6615384615384615,0.6615384615384615,0.6615384615384615,0.6717557251908397,0.6635819142689371,0.004086905460951274,1,BiasedClassifier,,,,,,,,,,1
+0.002464008331298828,0.0001832424088848076,0.0056017875671386715,0.0003938460344191295,gini,best,"{'criterion': 'gini', 'splitter': 'best'}",0.8333333333333334,0.8478260869565217,0.88,0.82,0.8627450980392157,0.8487809036658142,0.021154244114916888,1,0.8735632183908046,0.8620689655172413,0.9310344827586207,0.8735632183908046,0.9195402298850575,0.8919540229885057,0.027777117180677165,1,0.9302325581395349,0.8863636363636364,1.0,0.9534883720930233,1.0,0.9540169133192389,0.04329394717822427,1,0.8791208791208791,0.8666666666666666,0.9361702127659575,0.8817204301075269,0.9263157894736842,0.897998795626943,0.02779198606682888,1,DecisionTreeClassifier,,,,,,,,,,
+0.0011179447174072266,5.294245469650377e-05,0.005008077621459961,0.0004570369266305592,,,{},0.8260869565217391,0.9,0.8695652173913043,0.8928571428571429,0.8148148148148148,0.8606648263170001,0.03452729566153893,1,0.6781609195402298,0.6781609195402298,0.6896551724137931,0.7586206896551724,0.6896551724137931,0.6988505747126437,0.03032392174315613,1,0.4418604651162791,0.4090909090909091,0.45454545454545453,0.5813953488372093,0.5,0.47737843551797043,0.05962769648938314,1,0.5757575757575758,0.5625000000000001,0.5970149253731344,0.7042253521126761,0.6197183098591549,0.6118432326205083,0.05010644823946961,1,GaussianNB,,,,,,,,,,
+1.6133128643035888,0.4708061606935623,0.008328628540039063,0.0028030366175952363,,,"{'activation': 'logistic', 'hidden_layer_sizes': (60, 80, 100), 'learning_rate': 'constant', 'max_iter': 500000, 'solver': 'lbfgs'}",0.8367346938775511,0.8837209302325582,0.8775510204081632,0.8723404255319149,0.88,0.8700694140100375,0.017072292916992474,1,0.8850574712643678,0.8735632183908046,0.9195402298850575,0.9080459770114943,0.9310344827586207,0.903448275862069,0.021318663208036076,1,0.9534883720930233,0.8636363636363636,0.9772727272727273,0.9534883720930233,1.0,0.9495771670190274,0.04631613731716571,1,0.8913043478260869,0.8735632183908046,0.9247311827956989,0.9111111111111112,0.9361702127659575,0.9073760145779317,0.022564850176524636,1,MLPClassifier,logistic,"(60, 80, 100)",constant,500000,lbfgs,,,,,
+0.21513056755065918,0.0033775328263650475,0.019371461868286134,0.0007154497423913386,gini,,"{'class_weight': 'balanced', 'criterion': 'gini', 'max_features': 'sqrt'}",0.8367346938775511,0.8888888888888888,0.8979591836734694,0.9069767441860465,0.88,0.8821119021251912,0.024408504303677008,1,0.8850574712643678,0.896551724137931,0.9425287356321839,0.9080459770114943,0.9310344827586207,0.9126436781609195,0.021318663208036076,1,0.9534883720930233,0.9090909090909091,1.0,0.9069767441860465,1.0,0.9539112050739957,0.041134499812313964,1,0.8913043478260869,0.8988764044943819,0.9462365591397849,0.9069767441860465,0.9361702127659575,0.9159128536824515,0.021473522878435014,1,RandomForestClassifier,,,,,,balanced,sqrt,,,
+0.0042136192321777345,0.00013865276693192082,0.005751371383666992,0.0007100369798845391,,,"{'gamma': 'scale', 'kernel': 'rbf'}",0.7380952380952381,0.7560975609756098,0.8095238095238095,0.72,0.7560975609756098,0.7559628339140534,0.029937905118780302,1,0.735632183908046,0.735632183908046,0.7931034482758621,0.7586206896551724,0.735632183908046,0.7517241379310345,0.02252404361179935,1,0.7209302325581395,0.7045454545454546,0.7727272727272727,0.8372093023255814,0.7045454545454546,0.7479915433403805,0.05114509679539022,1,0.7294117647058824,0.7294117647058823,0.7906976744186046,0.7741935483870969,0.7294117647058823,0.7506253033846698,0.026500187782749762,1,SVC,,,,,,,,scale,rbf,
+0.0001003265380859375,1.7100047998968138e-05,0.004994964599609375,0.0005322873543987106,,,{'to_return': 1},0.4942528735632184,0.5057471264367817,0.5057471264367817,0.4942528735632184,0.5057471264367817,0.5011494252873563,0.005631010902949855,1,0.4942528735632184,0.5057471264367817,0.5057471264367817,0.4942528735632184,0.5057471264367817,0.5011494252873563,0.005631010902949855,1,1.0,1.0,1.0,1.0,1.0,1.0,0.0,1,0.6615384615384615,0.6717557251908397,0.6717557251908397,0.6615384615384615,0.6717557251908397,0.6676688197298883,0.00500541650316235,1,BiasedClassifier,,,,,,,,,,1
+0.0019986629486083984,0.0002167012159107283,0.004934978485107422,0.0005398000378015275,gini,best,"{'criterion': 'gini', 'splitter': 'best'}",0.8269230769230769,0.7719298245614035,0.7962962962962963,0.8888888888888888,0.8301886792452831,0.8228453531829898,0.039318047646042784,1,0.8850574712643678,0.8505747126436781,0.8735632183908046,0.896551724137931,0.896551724137931,0.8804597701149426,0.017203022467926186,1,0.9772727272727273,1.0,1.0,0.9090909090909091,1.0,0.9772727272727273,0.035208939510976534,1,0.8958333333333334,0.8712871287128713,0.8865979381443299,0.8988764044943819,0.9072164948453608,0.8919622599060555,0.012261478591669811,1,DecisionTreeClassifier,,,,,,,,,,
+0.0010799407958984376,9.17237641314175e-05,0.004824399948120117,0.0006967275913509627,,,{},0.875,0.8260869565217391,0.8,0.8620689655172413,0.8260869565217391,0.8378485757121439,0.02711472538797047,1,0.632183908045977,0.6666666666666666,0.6781609195402298,0.735632183908046,0.6666666666666666,0.6758620689655171,0.033629284685811014,1,0.3181818181818182,0.4318181818181818,0.46511627906976744,0.5681818181818182,0.4318181818181818,0.44302325581395346,0.07998657303214383,1,0.4666666666666667,0.5671641791044776,0.5882352941176471,0.6849315068493151,0.5671641791044776,0.5748323651685168,0.06945340883021155,1,GaussianNB,,,,,,,,,,
+2.259204959869385,0.46340751411518283,0.004901695251464844,0.0013836243357564788,,,"{'activation': 'logistic', 'hidden_layer_sizes': (60, 80, 100), 'learning_rate': 'constant', 'max_iter': 500000, 'solver': 'lbfgs'}",0.8148148148148148,0.88,0.8571428571428571,0.875,0.9361702127659575,0.872625576944726,0.03919117138438418,1,0.8850574712643678,0.9310344827586207,0.9080459770114943,0.9080459770114943,0.9655172413793104,0.9195402298850575,0.027200366818848812,1,1.0,1.0,0.9767441860465116,0.9545454545454546,1.0,0.9862579281183933,0.01823582127090724,1,0.8979591836734693,0.9361702127659575,0.9130434782608695,0.9130434782608695,0.967032967032967,0.9254498639988264,0.02411495243431271,1,MLPClassifier,logistic,"(60, 80, 100)",constant,500000,lbfgs,,,,,
+0.19242162704467775,0.0016003622769180863,0.012227916717529297,0.0003004062970458725,gini,,"{'class_weight': 'balanced', 'criterion': 'gini', 'max_features': 'sqrt'}",0.8461538461538461,0.88,0.8571428571428571,0.8775510204081632,0.9166666666666666,0.8755028780743066,0.024155051344107174,1,0.9080459770114943,0.9310344827586207,0.9080459770114943,0.9195402298850575,0.9540229885057471,0.9241379310344827,0.017203022467926148,1,1.0,1.0,0.9767441860465116,0.9772727272727273,1.0,0.9908033826638478,0.011264749940598078,1,0.9166666666666666,0.9361702127659575,0.9130434782608695,0.9247311827956989,0.9565217391304348,0.9294266559239255,0.015702817562029407,1,RandomForestClassifier,,,,,,balanced,sqrt,,,
+0.004063701629638672,0.00014276759337124057,0.005324411392211914,0.00046126971092070505,,,"{'gamma': 'scale', 'kernel': 'rbf'}",0.7446808510638298,0.7954545454545454,0.7727272727272727,0.7959183673469388,0.7222222222222222,0.7662006517629619,0.02890928495781439,1,0.7586206896551724,0.7931034482758621,0.7816091954022989,0.8275862068965517,0.7701149425287356,0.7862068965517242,0.023668115266636796,1,0.7954545454545454,0.7954545454545454,0.7906976744186046,0.8863636363636364,0.8863636363636364,0.8308668076109937,0.04534624987074944,1,0.7692307692307692,0.7954545454545455,0.7816091954022988,0.8387096774193548,0.7959183673469388,0.7961845109707815,0.023443282540081302,1,SVC,,,,,,,,scale,rbf,
+7.653236389160156e-05,3.520208134706831e-06,0.004015016555786133,0.00011267535926929854,,,{'to_return': 1},0.5057471264367817,0.5057471264367817,0.4942528735632184,0.5057471264367817,0.5057471264367817,0.5034482758620691,0.004597701149425304,1,0.5057471264367817,0.5057471264367817,0.4942528735632184,0.5057471264367817,0.5057471264367817,0.5034482758620691,0.004597701149425304,1,1.0,1.0,1.0,1.0,1.0,1.0,0.0,1,0.6717557251908397,0.6717557251908397,0.6615384615384615,0.6717557251908397,0.6717557251908397,0.6697122724603641,0.004086905460951274,1,BiasedClassifier,,,,,,,,,,1
+0.0019217967987060548,7.137690033364649e-05,0.004196548461914062,7.035224470287863e-05,gini,best,"{'criterion': 'gini', 'splitter': 'best'}",0.8478260869565217,0.8,0.8076923076923077,0.9090909090909091,0.86,0.8449218607479478,0.039388995346916054,1,0.8735632183908046,0.8505747126436781,0.8735632183908046,0.9195402298850575,0.9195402298850575,0.8873563218390803,0.027586206896551717,1,0.9069767441860465,0.9302325581395349,0.9767441860465116,0.9302325581395349,1.0,0.9488372093023255,0.034178926643486214,1,0.8764044943820224,0.8602150537634408,0.8842105263157894,0.9195402298850575,0.924731182795699,0.8930202974284018,0.02505501170677674,1,DecisionTreeClassifier,,,,,,,,,,
+0.0009679317474365235,3.4980937724421564e-05,0.003993892669677734,9.065972355155905e-05,,,{},0.8333333333333334,0.8125,0.6666666666666666,0.8947368421052632,0.8260869565217391,0.8066647597254004,0.07547005012658593,1,0.6896551724137931,0.6206896551724138,0.6091954022988506,0.6781609195402298,0.6781609195402298,0.6551724137931034,0.03331350976135501,1,0.46511627906976744,0.3023255813953488,0.4186046511627907,0.3953488372093023,0.4418604651162791,0.4046511627906977,0.05620021383067244,1,0.5970149253731343,0.44067796610169485,0.5142857142857143,0.5483870967741935,0.5757575757575758,0.5352246556584626,0.05479365049210219,1,GaussianNB,,,,,,,,,,
+1.3330429077148438,0.18693852518890453,0.005749130249023437,0.001121520059448941,,,"{'activation': 'logistic', 'hidden_layer_sizes': (60, 80, 100), 'learning_rate': 'constant', 'max_iter': 500000, 'solver': 'lbfgs'}",0.8780487804878049,0.8837209302325582,0.7777777777777778,0.8913043478260869,0.8269230769230769,0.851554982649461,0.04328137770023995,1,0.8620689655172413,0.8850574712643678,0.8505747126436781,0.9195402298850575,0.896551724137931,0.8827586206896552,0.024545007475934067,1,0.8372093023255814,0.8837209302325582,0.9767441860465116,0.9534883720930233,1.0,0.9302325581395348,0.060643743304210664,1,0.8571428571428572,0.8837209302325582,0.8659793814432991,0.9213483146067417,0.9052631578947368,0.8866909282640385,0.02390372234008688,1,MLPClassifier,logistic,"(60, 80, 100)",constant,500000,lbfgs,,,,,
+0.20228543281555175,0.0013756265053154787,0.013142728805541992,0.0002900245586164067,gini,,"{'class_weight': 'balanced', 'criterion': 'gini', 'max_features': 'sqrt'}",0.8260869565217391,0.8478260869565217,0.8076923076923077,0.9333333333333333,0.8431372549019608,0.8516151878811724,0.043231927854269554,1,0.8505747126436781,0.8735632183908046,0.8735632183908046,0.9540229885057471,0.9080459770114943,0.8919540229885057,0.0360560623939267,1,0.8837209302325582,0.9069767441860465,0.9767441860465116,0.9767441860465116,1.0,0.9488372093023255,0.04509469634805887,1,0.853932584269663,0.8764044943820224,0.8842105263157894,0.9545454545454545,0.9148936170212766,0.8967973353068412,0.034853990562134375,1,RandomForestClassifier,,,,,,balanced,sqrt,,,
+0.004331207275390625,0.0002960434040205934,0.0056801319122314455,0.00035792606637237385,,,"{'gamma': 'scale', 'kernel': 'rbf'}",0.8108108108108109,0.6888888888888889,0.7391304347826086,0.8809523809523809,0.75,0.7739565030869378,0.06607514803168148,1,0.7701149425287356,0.7011494252873564,0.7586206896551724,0.8735632183908046,0.7586206896551724,0.7724137931034484,0.05602785106645491,1,0.6976744186046512,0.7209302325581395,0.7906976744186046,0.8604651162790697,0.7674418604651163,0.7674418604651163,0.05696487773914367,1,0.75,0.7045454545454545,0.7640449438202247,0.8705882352941177,0.7586206896551724,0.769559864662994,0.054710645038602346,1,SVC,,,,,,,,scale,rbf,
+8.244514465332032e-05,6.836255206661167e-06,0.0047031879425048825,0.0004015594204932739,,,{'to_return': 1},0.4942528735632184,0.4942528735632184,0.4942528735632184,0.4942528735632184,0.4942528735632184,0.4942528735632184,0.0,1,0.4942528735632184,0.4942528735632184,0.4942528735632184,0.4942528735632184,0.4942528735632184,0.4942528735632184,0.0,1,1.0,1.0,1.0,1.0,1.0,1.0,0.0,1,0.6615384615384615,0.6615384615384615,0.6615384615384615,0.6615384615384615,0.6615384615384615,0.6615384615384615,0.0,1,BiasedClassifier,,,,,,,,,,1
+0.0020009517669677735,0.00011523040014140854,0.0043304443359375,0.00032293260643118544,gini,best,"{'criterion': 'gini', 'splitter': 'best'}",0.8636363636363636,0.851063829787234,0.7884615384615384,0.84,0.88,0.8446323463770271,0.031080559417356046,1,0.8735632183908046,0.8850574712643678,0.8505747126436781,0.896551724137931,0.9310344827586207,0.8873563218390805,0.026611119316759135,1,0.8837209302325582,0.9302325581395349,0.9534883720930233,0.9767441860465116,1.0,0.9488372093023255,0.04001081519554709,1,0.8735632183908046,0.888888888888889,0.8631578947368421,0.9032258064516129,0.9361702127659575,0.8930012042468214,0.025508725334762155,1,DecisionTreeClassifier,,,,,,,,,,
+0.0011005878448486328,0.00016774134961461894,0.004408931732177735,0.0003319931791854012,,,{},0.8181818181818182,0.7391304347826086,0.7916666666666666,0.9,0.75,0.7997957839262186,0.05762797654884553,1,0.6666666666666666,0.632183908045977,0.6666666666666666,0.6896551724137931,0.632183908045977,0.657471264367816,0.022288183252488873,1,0.4186046511627907,0.3953488372093023,0.4418604651162791,0.4186046511627907,0.4090909090909091,0.41670190274841434,0.015192595537608194,1,0.5538461538461539,0.5151515151515151,0.5671641791044777,0.5714285714285715,0.5294117647058824,0.5474004368473201,0.021785726074865806,1,GaussianNB,,,,,,,,,,
+2.010102367401123,0.718723534169696,0.005715608596801758,0.001989253789442101,,,"{'activation': 'logistic', 'hidden_layer_sizes': (60, 80, 100), 'learning_rate': 'constant', 'max_iter': 500000, 'solver': 'lbfgs'}",0.8571428571428571,0.8333333333333334,0.7777777777777778,0.7777777777777778,0.8723404255319149,0.8236744343127322,0.03948370428555789,1,0.9080459770114943,0.8735632183908046,0.8505747126436781,0.8505747126436781,0.896551724137931,0.8758620689655172,0.023443767878587537,1,0.9767441860465116,0.9302325581395349,0.9767441860465116,0.9767441860465116,0.9318181818181818,0.9584566596194503,0.022403166170783254,1,0.9130434782608695,0.8791208791208791,0.8659793814432991,0.8659793814432991,0.9010989010989012,0.8850444042734497,0.018993332270735984,1,MLPClassifier,logistic,"(60, 80, 100)",constant,500000,lbfgs,,,,,
+0.2506169319152832,0.0041195909902483575,0.025205612182617188,0.0014503115712025032,gini,,"{'class_weight': 'balanced', 'criterion': 'gini', 'max_features': 'sqrt'}",0.9090909090909091,0.8888888888888888,0.8367346938775511,0.8571428571428571,0.86,0.8703714698000413,0.02551895790120753,1,0.9195402298850575,0.9080459770114943,0.8850574712643678,0.9080459770114943,0.9080459770114943,0.9057471264367816,0.011262021805899657,1,0.9302325581395349,0.9302325581395349,0.9534883720930233,0.9767441860465116,0.9772727272727273,0.9535940803382663,0.020919552232619824,1,0.9195402298850575,0.9090909090909092,0.8913043478260869,0.9130434782608695,0.9148936170212766,0.9095745164168398,0.009733189701493498,1,RandomForestClassifier,,,,,,balanced,sqrt,,,
+0.0062087059020996095,0.00026685007959969035,0.007111692428588867,0.00014544390043446138,,,"{'gamma': 'scale', 'kernel': 'rbf'}",0.7446808510638298,0.7317073170731707,0.6808510638297872,0.7333333333333333,0.7446808510638298,0.7270506832727902,0.023736918079198085,1,0.7701149425287356,0.7241379310344828,0.7011494252873564,0.7471264367816092,0.7586206896551724,0.7402298850574713,0.024759378423606884,1,0.813953488372093,0.6976744186046512,0.7441860465116279,0.7674418604651163,0.7954545454545454,0.7637420718816068,0.040713458952669083,1,0.7777777777777778,0.7142857142857143,0.711111111111111,0.7499999999999999,0.7692307692307692,0.7444810744810745,0.02748450695266498,1,SVC,,,,,,,,scale,rbf,
+0.000115203857421875,3.0487537499630993e-05,0.005177640914916992,0.00044141144631256536,,,{'to_return': 1},0.4942528735632184,0.4942528735632184,0.4942528735632184,0.4942528735632184,0.5057471264367817,0.496551724137931,0.004597701149425304,1,0.4942528735632184,0.4942528735632184,0.4942528735632184,0.4942528735632184,0.5057471264367817,0.496551724137931,0.004597701149425304,1,1.0,1.0,1.0,1.0,1.0,1.0,0.0,1,0.6615384615384615,0.6615384615384615,0.6615384615384615,0.6615384615384615,0.6717557251908397,0.6635819142689371,0.004086905460951274,1,BiasedClassifier,,,,,,,,,,1
+0.0022040843963623048,0.00015991703635098497,0.0045887470245361325,0.0004738532785959043,gini,best,"{'criterion': 'gini', 'splitter': 'best'}",0.7346938775510204,0.8958333333333334,0.851063829787234,0.8571428571428571,0.8541666666666666,0.8385801128962223,0.05443094793302604,1,0.7701149425287356,0.9310344827586207,0.8850574712643678,0.896551724137931,0.896551724137931,0.8758620689655172,0.05507654506073997,1,0.8372093023255814,0.9772727272727273,0.9302325581395349,0.9545454545454546,0.9534883720930233,0.9305496828752643,0.04898479480022706,1,0.782608695652174,0.9347826086956522,0.888888888888889,0.9032258064516128,0.9010989010989011,0.8821209801574458,0.05201407819527025,1,DecisionTreeClassifier,,,,,,,,,,
+0.0011530399322509765,0.00015324113015629455,0.004853343963623047,0.0003699877070622847,,,{},0.7619047619047619,0.8260869565217391,0.8181818181818182,0.85,0.8888888888888888,0.8290124850994417,0.04160521142398222,1,0.632183908045977,0.6666666666666666,0.6666666666666666,0.6551724137931034,0.6666666666666666,0.657471264367816,0.013404487114586881,1,0.37209302325581395,0.4318181818181818,0.4186046511627907,0.38636363636363635,0.37209302325581395,0.3961945031712474,0.0246157259643505,1,0.5,0.5671641791044776,0.5538461538461539,0.53125,0.5245901639344261,0.5353700993770115,0.02339424823081782,1,GaussianNB,,,,,,,,,,
+1.4172007083892821,0.2765746568926641,0.0058556079864501955,0.0012508237731777398,,,"{'activation': 'logistic', 'hidden_layer_sizes': (60, 80, 100), 'learning_rate': 'constant', 'max_iter': 500000, 'solver': 'lbfgs'}",0.8604651162790697,0.8085106382978723,0.8571428571428571,0.875,0.8235294117647058,0.844929604696901,0.024815534248618338,1,0.8620689655172413,0.8275862068965517,0.9080459770114943,0.9080459770114943,0.8850574712643678,0.87816091954023,0.030497699221658858,1,0.8604651162790697,0.8636363636363636,0.9767441860465116,0.9545454545454546,0.9767441860465116,0.9264270613107822,0.05319383428227083,1,0.8604651162790697,0.8351648351648351,0.9130434782608695,0.9130434782608695,0.8936170212765957,0.883066785848448,0.03070486603190395,1,MLPClassifier,logistic,"(60, 80, 100)",constant,500000,lbfgs,,,,,
+0.20352258682250976,0.0019300123353234574,0.014762353897094727,0.0016365459841071802,gini,,"{'class_weight': 'balanced', 'criterion': 'gini', 'max_features': 'sqrt'}",0.925,0.8888888888888888,0.9130434782608695,0.9166666666666666,0.8936170212765957,0.9074432110186041,0.013857145153956448,1,0.896551724137931,0.896551724137931,0.9425287356321839,0.9540229885057471,0.9310344827586207,0.9241379310344827,0.023668115266636754,1,0.8604651162790697,0.9090909090909091,0.9767441860465116,1.0,0.9767441860465116,0.9446088794926004,0.05191383475506909,1,0.891566265060241,0.8988764044943819,0.9438202247191011,0.9565217391304348,0.9333333333333332,0.9248235933474985,0.025366775981023187,1,RandomForestClassifier,,,,,,balanced,sqrt,,,
+0.004777431488037109,0.0002970876887188734,0.006045913696289063,0.0005675497983209845,,,"{'gamma': 'scale', 'kernel': 'rbf'}",0.8205128205128205,0.8205128205128205,0.8048780487804879,0.7948717948717948,0.6888888888888889,0.7859328747133626,0.04949324075410711,1,0.7931034482758621,0.7816091954022989,0.7931034482758621,0.7586206896551724,0.7011494252873564,0.7655172413793103,0.03455930201924807,1,0.7441860465116279,0.7272727272727273,0.7674418604651163,0.7045454545454546,0.7209302325581395,0.7328752642706131,0.021444969992358884,1,0.7804878048780488,0.7710843373493976,0.7857142857142858,0.746987951807229,0.7045454545454545,0.7577639668588831,0.029743429397742827,1,SVC,,,,,,,,scale,rbf,
+7.796287536621094e-05,4.08802704798752e-06,0.003885698318481445,0.00011026155861765243,,,{'to_return': 1},0.4942528735632184,0.5057471264367817,0.4942528735632184,0.5057471264367817,0.4942528735632184,0.4988505747126437,0.005631010902949855,1,0.4942528735632184,0.5057471264367817,0.4942528735632184,0.5057471264367817,0.4942528735632184,0.4988505747126437,0.005631010902949855,1,1.0,1.0,1.0,1.0,1.0,1.0,0.0,1,0.6615384615384615,0.6717557251908397,0.6615384615384615,0.6717557251908397,0.6615384615384615,0.6656253669994128,0.005005416503162351,1,BiasedClassifier,,,,,,,,,,1
+0.00205535888671875,0.00015819509488697322,0.0038710117340087892,5.018162300930847e-05,gini,best,"{'criterion': 'gini', 'splitter': 'best'}",0.8478260869565217,0.9130434782608695,0.7884615384615384,0.8913043478260869,0.8301886792452831,0.85416482615006,0.04422215958714269,1,0.8735632183908046,0.9425287356321839,0.8505747126436781,0.9080459770114943,0.896551724137931,0.8942528735632184,0.03118312635919663,1,0.9069767441860465,0.9767441860465116,0.9534883720930233,0.9318181818181818,1.0,0.9538054968287526,0.032669833944515725,1,0.8764044943820224,0.9438202247191011,0.8631578947368421,0.9111111111111111,0.9072164948453608,0.9003420439588876,0.02831695717286607,1,DecisionTreeClassifier,,,,,,,,,,
+0.0009009838104248047,3.2133305695536605e-05,0.0037467479705810547,0.00011888928377511985,,,{},0.8275862068965517,0.8333333333333334,0.7058823529411765,0.8636363636363636,0.7727272727272727,0.8006331059069396,0.055716471037288065,1,0.7241379310344828,0.7816091954022989,0.5862068965517241,0.6781609195402298,0.632183908045977,0.6804597701149425,0.06834974136698166,1,0.5581395348837209,0.6976744186046512,0.27906976744186046,0.4318181818181818,0.38636363636363635,0.47061310782241017,0.14457495305699658,1,0.6666666666666666,0.759493670886076,0.4,0.5757575757575758,0.515151515151515,0.5834138856923666,0.12352662876664627,1,GaussianNB,,,,,,,,,,
+2.279987907409668,0.40810481956110456,0.0045074462890625,0.000777673991761876,,,"{'activation': 'logistic', 'hidden_layer_sizes': (60, 80, 100), 'learning_rate': 'constant', 'max_iter': 500000, 'solver': 'lbfgs'}",0.7692307692307693,0.8809523809523809,0.8541666666666666,0.8333333333333334,0.9069767441860465,0.8489319788738394,0.04694146023927249,1,0.8275862068965517,0.8735632183908046,0.896551724137931,0.8620689655172413,0.896551724137931,0.871264367816092,0.025598916610712757,1,0.9302325581395349,0.8604651162790697,0.9534883720930233,0.9090909090909091,0.8863636363636364,0.9079281183932346,0.03253410606328727,1,0.8421052631578948,0.8705882352941177,0.9010989010989011,0.8695652173913043,0.896551724137931,0.8759818682160297,0.021316066089476535,1,MLPClassifier,logistic,"(60, 80, 100)",constant,500000,lbfgs,,,,,
+0.19093751907348633,0.002536299273939237,0.012502145767211915,0.0001547722187985144,gini,,"{'class_weight': 'balanced', 'criterion': 'gini', 'max_features': 'sqrt'}",0.8163265306122449,0.8666666666666667,0.851063829787234,0.9565217391304348,0.8936170212765957,0.8768391574946351,0.04703283029865054,1,0.8620689655172413,0.8850574712643678,0.8850574712643678,0.9770114942528736,0.9195402298850575,0.9057471264367816,0.04008182936589127,1,0.9302325581395349,0.9069767441860465,0.9302325581395349,1.0,0.9545454545454546,0.9443974630021141,0.03161083076797578,1,0.8695652173913043,0.8863636363636364,0.888888888888889,0.9777777777777777,0.9230769230769231,0.9091344886997061,0.03847184911466428,1,RandomForestClassifier,,,,,,balanced,sqrt,,,
+0.00411839485168457,0.00013571286477059825,0.00532684326171875,0.00040210005787527365,,,"{'gamma': 'scale', 'kernel': 'rbf'}",0.72,0.7441860465116279,0.6666666666666666,0.8222222222222222,0.7391304347826086,0.7384410740366251,0.05008872025357937,1,0.7586206896551724,0.7471264367816092,0.6666666666666666,0.8275862068965517,0.7471264367816092,0.7494252873563217,0.05109450751929395,1,0.8372093023255814,0.7441860465116279,0.6511627906976745,0.8409090909090909,0.7727272727272727,0.7692389006342495,0.06976872316076294,1,0.7741935483870969,0.7441860465116278,0.6588235294117646,0.8314606741573033,0.7555555555555555,0.7528438708046696,0.05578269669633417,1,SVC,,,,,,,,scale,rbf,
+7.4005126953125e-05,2.7828667567090748e-06,0.004036712646484375,0.00016477490067278447,,,{'to_return': 1},0.4942528735632184,0.4942528735632184,0.4942528735632184,0.5057471264367817,0.5057471264367817,0.4988505747126437,0.005631010902949855,1,0.4942528735632184,0.4942528735632184,0.4942528735632184,0.5057471264367817,0.5057471264367817,0.4988505747126437,0.005631010902949855,1,1.0,1.0,1.0,1.0,1.0,1.0,0.0,1,0.6615384615384615,0.6615384615384615,0.6615384615384615,0.6717557251908397,0.6717557251908397,0.6656253669994129,0.005005416503162351,1,BiasedClassifier,,,,,,,,,,1
+0.0018951892852783203,8.00446129539477e-05,0.004139852523803711,0.00016271933705261097,gini,best,"{'criterion': 'gini', 'splitter': 'best'}",0.8,0.8,0.7636363636363637,0.8297872340425532,0.8571428571428571,0.8101132909643548,0.031507161767741265,1,0.8735632183908046,0.8505747126436781,0.8275862068965517,0.8620689655172413,0.896551724137931,0.8620689655172413,0.022988505747126454,1,1.0,0.9302325581395349,0.9545454545454546,0.9069767441860465,0.9545454545454546,0.9492600422832981,0.030920339343146964,1,0.888888888888889,0.8602150537634408,0.8484848484848485,0.8666666666666666,0.9032258064516128,0.8734962528510917,0.019846326846579894,1,DecisionTreeClassifier,,,,,,,,,,
+0.0011829853057861329,0.00017642217052784282,0.004505681991577149,0.00041280835182465797,,,{},0.8695652173913043,0.8333333333333334,0.8636363636363636,0.76,0.8571428571428571,0.8367355543007717,0.04029347027048997,1,0.6896551724137931,0.6896551724137931,0.6781609195402298,0.6551724137931034,0.6091954022988506,0.664367816091954,0.030323921743156134,1,0.45454545454545453,0.46511627906976744,0.4318181818181818,0.4418604651162791,0.2727272727272727,0.4132135306553911,0.07114125637203969,1,0.5970149253731344,0.5970149253731343,0.5757575757575758,0.5588235294117647,0.41379310344827586,0.548480811872777,0.06885185952651231,1,GaussianNB,,,,,,,,,,
+1.1361024379730225,0.159419282040439,0.00465998649597168,0.00041844547523993013,,,"{'activation': 'logistic', 'hidden_layer_sizes': (60, 80, 100), 'learning_rate': 'constant', 'max_iter': 500000, 'solver': 'lbfgs'}",0.7857142857142857,0.8297872340425532,0.7884615384615384,0.75,0.8888888888888888,0.8085703894214532,0.04745202573749175,1,0.8620689655172413,0.8620689655172413,0.8390804597701149,0.8045977011494253,0.896551724137931,0.8528735632183908,0.03032392174315613,1,1.0,0.9069767441860465,0.9318181818181818,0.9069767441860465,0.9090909090909091,0.9309725158562367,0.035765647062131915,1,0.88,0.8666666666666666,0.8541666666666667,0.8210526315789475,0.8988764044943819,0.8641524738813324,0.026155417637202708,1,MLPClassifier,logistic,"(60, 80, 100)",constant,500000,lbfgs,,,,,
+0.2012843132019043,0.003481437204419735,0.012103986740112305,0.0006385231718857603,gini,,"{'class_weight': 'balanced', 'criterion': 'gini', 'max_features': 'sqrt'}",0.8,0.8297872340425532,0.8775510204081632,0.851063829787234,0.851063829787234,0.8418931828050369,0.025850631232723163,1,0.8735632183908046,0.8620689655172413,0.9195402298850575,0.8850574712643678,0.8735632183908046,0.8827586206896552,0.019775460384006037,1,1.0,0.9069767441860465,0.9772727272727273,0.9302325581395349,0.9090909090909091,0.9447145877378436,0.03746846609226435,1,0.888888888888889,0.8666666666666666,0.9247311827956989,0.888888888888889,0.8791208791208791,0.8896593012722045,0.019343957876412386,1,RandomForestClassifier,,,,,,balanced,sqrt,,,
+0.004258394241333008,0.0002193918903849466,0.005930662155151367,0.0003742918078827082,,,"{'gamma': 'scale', 'kernel': 'rbf'}",0.75,0.7435897435897436,0.8222222222222222,0.6739130434782609,0.8055555555555556,0.7590561129691563,0.05238839483175277,1,0.7701149425287356,0.7241379310344828,0.8275862068965517,0.6896551724137931,0.7471264367816092,0.7517241379310345,0.046320555585310064,1,0.8181818181818182,0.6744186046511628,0.8409090909090909,0.7209302325581395,0.6590909090909091,0.7427061310782241,0.07412055319973548,1,0.7826086956521738,0.7073170731707318,0.8314606741573033,0.6966292134831461,0.7250000000000001,0.748603131292671,0.050960151788453534,1,SVC,,,,,,,,scale,rbf,
+7.4005126953125e-05,2.5793320343586964e-06,0.004357624053955078,0.0004015861681529256,,,{'to_return': 1},0.5057471264367817,0.4942528735632184,0.5057471264367817,0.4942528735632184,0.5057471264367817,0.5011494252873563,0.005631010902949855,1,0.5057471264367817,0.4942528735632184,0.5057471264367817,0.4942528735632184,0.5057471264367817,0.5011494252873563,0.005631010902949855,1,1.0,1.0,1.0,1.0,1.0,1.0,0.0,1,0.6717557251908397,0.6615384615384615,0.6717557251908397,0.6615384615384615,0.6717557251908397,0.6676688197298883,0.00500541650316235,1,BiasedClassifier,,,,,,,,,,1

models/model_stats.csv

@@ -0,0 +1,46 @@
+,classifier_a,classifier_b,metric,pvalue
+1,DecisionTreeClassifier,GaussianNB,precision,0.08929133280531223
+2,DecisionTreeClassifier,GaussianNB,recall,3.877480505802584e-18
+3,DecisionTreeClassifier,GaussianNB,f1,3.896340037647931e-18
+4,DecisionTreeClassifier,MLPClassifier,precision,0.4012348497407896
+5,DecisionTreeClassifier,MLPClassifier,recall,0.011820059675817408
+6,DecisionTreeClassifier,MLPClassifier,f1,0.4710651684151138
+7,DecisionTreeClassifier,RandomForestClassifier,precision,8.283473187323235e-12
+8,DecisionTreeClassifier,RandomForestClassifier,recall,0.3276029575034267
+9,DecisionTreeClassifier,RandomForestClassifier,f1,1.4515097813437996e-10
+10,DecisionTreeClassifier,SVC,precision,6.472995016722292e-16
+11,DecisionTreeClassifier,SVC,recall,3.864155888689142e-18
+12,DecisionTreeClassifier,SVC,f1,3.896559845095909e-18
+13,GaussianNB,MLPClassifier,precision,0.03476088049603166
+14,GaussianNB,MLPClassifier,recall,3.873544128513129e-18
+15,GaussianNB,MLPClassifier,f1,3.896120241954008e-18
+16,GaussianNB,RandomForestClassifier,precision,5.027978595522601e-10
+17,GaussianNB,RandomForestClassifier,recall,3.8656827355135645e-18
+18,GaussianNB,RandomForestClassifier,f1,3.896120241954008e-18
+19,GaussianNB,SVC,precision,7.361006463422299e-13
+20,GaussianNB,SVC,recall,3.881639684405151e-18
+21,GaussianNB,SVC,f1,4.265842540306607e-18
+22,MLPClassifier,RandomForestClassifier,precision,2.9302015489842885e-09
+23,MLPClassifier,RandomForestClassifier,recall,0.00010909237805840521
+24,MLPClassifier,RandomForestClassifier,f1,1.1542838431590428e-11
+25,MLPClassifier,SVC,precision,3.6744416439536415e-16
+26,MLPClassifier,SVC,recall,5.645631221640026e-18
+27,MLPClassifier,SVC,f1,5.112831740936498e-18
+28,RandomForestClassifier,SVC,precision,4.0161556854627e-18
+29,RandomForestClassifier,SVC,recall,3.8584897469079895e-18
+30,RandomForestClassifier,SVC,f1,3.896559845095909e-18
+31,BiasedClassifier,DecisionTreeClassifier,precision,3.881858705649312e-18
+32,BiasedClassifier,DecisionTreeClassifier,recall,1.0267247842714985e-14
+33,BiasedClassifier,DecisionTreeClassifier,f1,3.881858705649312e-18
+34,BiasedClassifier,GaussianNB,precision,3.876167958900271e-18
+35,BiasedClassifier,GaussianNB,recall,3.7861845925093915e-18
+36,BiasedClassifier,GaussianNB,f1,4.22499456189738e-16
+37,BiasedClassifier,MLPClassifier,precision,3.887338045617697e-18
+38,BiasedClassifier,MLPClassifier,recall,2.2417567344517687e-15
+39,BiasedClassifier,MLPClassifier,f1,3.886680137925553e-18
+40,BiasedClassifier,RandomForestClassifier,precision,3.8798879361158955e-18
+41,BiasedClassifier,RandomForestClassifier,recall,4.647240654275836e-14
+42,BiasedClassifier,RandomForestClassifier,f1,3.8798879361158955e-18
+43,BiasedClassifier,SVC,precision,3.890848669938197e-18
+44,BiasedClassifier,SVC,recall,3.803119001919453e-18
+45,BiasedClassifier,SVC,f1,4.337936940303975e-17

report/build.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set -e
+
+SCRIPT_DIR=$(cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd)
+
+cd "$SCRIPT_DIR"
+pandoc main.md -o main.pdf  

report/main.md

@@ -0,0 +1,432 @@
+---
+author: Claudio Maggioni
+title: Information Modelling & Analysis -- Project 2
+geometry: margin=2cm
+---
+
+<!--The following shows a minimal submission report for project 2. If you
+choose to use this template, replace all template instructions (the
+yellow bits) with your own values. In addition, for any section, if
+**and only if** anything was unclear or warnings were raised by the
+code, and you had to take assumptions about the correct implementation
+(e.g., about details of a metric), describe your assumptions in one or
+two sentences.
+
+You may - at your own risk - also choose not to use this template. As
+long as your submission is a latex-generated, english PDF containing all
+expected info, you'll be fine.-->
+
+# Code Repository
+
+The code and result files, part of this submission, can be found at:
+
+- Repository: [https://github.com/infoMA2023/project-02-bug-prediction-maggicl](https://github.com/infoMA2023/project-02-bug-prediction-maggicl)
+- Commit ID: **786d6a06847b217bdad53804f8a6c79e30134ef3** 
+
+# Data Pre-Processing
+
+I use the sources of the [Closure]{.smallcaps} repository that were already downloaded using the command:
+
+```shell
+defects4j checkout -p Closure -v 1f -w ./resources
+```
+
+and used the code in the following subfolder for the project:
+
+```
+./resources/defects4j-checkout-closure-1f/src/com/google/javascript/jscomp/
+```
+
+relative to the root folder of the repository. The resulting CSV of extracted, labelled feature vectors can be found in
+the repository at the path:
+
+```
+./metrics/feature_vectors_labeled.csv
+```
+
+relative to the root folder of the repository. 
+
+Unlabeled feature vectors can be computed by running the script `./extract_feature_vectors.py`.
+The resulting CSV of unlabeled feature vectors is located in `./metrics/feature_vectors.csv`.
+
+Labels for feature vectors can be computed by running the script `./label_feature_vectors.py`.
+
+## Feature Vector Extraction
+
+I extracted **291** feature vectors in total. Aggregate metrics
+about the extracted feature vectors, i.e. the distribution of the values of each
+code metric, can be found in Table [1](#tab:metrics){reference-type="ref"
+reference="tab:metrics"}. 
+
+::: {#tab:metrics}
+| **Metric** | **Minimum** | **Average** | **Maximum** |
+|:------|-:|---------:|-----:|
+| `BCM` |0|  13.4124  |  221 |
+| `CPX` |0|   5.8247  |   96 |
+| `DCM` |0|   4.8652  |176.2 |
+| `EX`  |0|   0.1134  |    2 |
+| `FLD` |0|   6.5773  |  167 |
+| `INT` |0|   0.6667  |    3 |
+| `MTH` |0|  11.6529  |  209 |
+| `NML` |0|  13.5622  |   28 |
+| `RET` |0|   3.6735  |   86 |
+| `RFC` |0| 107.2710  |  882 |
+| `SZ`  |0|  18.9966  |  347 |
+| `WRD` |0| 314.4740  | 3133 |
+
+  : Distribution of values for each extracted code metric.
+:::
+
+
+## Feature Vector Labelling
+
+After feature vectors are labeled, I determine that the dataset contains
+**75** buggy classes and **216** non-buggy classes.
+
+# Classifiers
+
+<!--In every subsection below, describe in a concise way which different
+hyperparameters you tried for the corresponding classifier, and report
+the corresponding precision, recall and F1 values (for example in a
+table or an [itemize]{.smallcaps}-environment). Furthermore, for every
+type of classifiers, explicitly mention which hyperparameter
+configuration you chose (based on above reported results) to be used in
+further steps, and (in one or two sentences), explain why these
+hyperparameters may outperform the other ones you tested..-->
+
+In this section I explain how I define and perform training for each classifier.
+
+Since the dataset has an unbalanced number of feature vectors of each class, in order
+to increase classification performance I upsample the dataset by performing sampling with
+replacement over the least frequent class until the number of feature vectors matches the
+most frequest class[^1].
+
+[^1]: Upsampling due to unbalanced classes was suggested by *Michele Cattaneo*, who is attending this class.
+
+Other than for the `GaussianNB` (Naive Bayes) classifier, the classifiers chosen for the 
+project offer to select hyperparameter values. In order to choose them, I perform a grid
+search over each classifier. The hyperparameter values I have considered in the grid search
+for each classifier are the following:
+
+- For *DecisionTreeClassifier*:
+
+  | **Parameter** | **Values** |
+  |:------------|:--------------|
+  | criterion   | gini, entropy |
+  | splitter    | best, random  |
+
+- For *SVC*:
+
+  | **Parameter** | **Values** |
+  |:------------|:---------------------------|
+  | kernel      | linear, poly, rbf, sigmoid |
+  | gamma       | scale, auto                |
+
+- For *MLPClassifier*:
+
+  | **Parameter** | **Values** |
+  |:------------|:---------------------------|
+  | max_iter           | 500000 |
+  | hidden_layer_sizes | $[5, 10, 15, ..., 100]$, $[15, 30, 45, 60, 75, 90]^2$, $[20, 40, 60, 80, 100]^3$ |
+  | activation         | identity, logistic, tanh, relu |
+  | solver             | lbfgs, sgd, adam |
+  | learning_rate      | constant, invscaling, adaptive |
+
+  Note that the $[...]^2$ denotes a cartesian product of the array with itself, and $[...]^3$
+  denotes the cartesian product of $[...]^2$ with the array (i.e. $[...]^3 = [...]^2 \times [...] = ([...] \times [...]) \times [...]$). 
+
+  Note also the high upper bound on iterations (500000). This is to allow convergence of the less optimal hyperparameter configurations and avoid `ConvergenceWarning` errors.
+
+- For *RandomForestClassifier*:
+
+  | **Parameter** | **Values** |
+  |:-------------|:-----------------------------|
+  | criterion    | gini, entropy                |
+  | max_features | sqrt, log2                   |
+  | class_weight | balanced, balanced_subsample |
+
+The script `./train_classifiers.py`, according to the random seed $3735924759$, performs upscaling of the dataset and the grid search training, by recording precision, accuracy, recall and the F1 score of each configuration of hyperparameters. These metrics are then collected and stored in `./models/models.csv`.
+
+The metrics for each classifier and each hyperparameter configuration in decreasing order of
+accuracy are reported in the following sections.
+
+For each classifier, I then choose the hyperparameter configuration with highest accuracy. Namely, these configurations are:
+
+| **Classifier** | **Hyper-parameter configuration** | **Precision** | **Accuracy** | **Recall** | **F1 Score** |
+|:----|:--------|-:|-:|-:|--:|
+| DecisionTreeClassifier | `criterion`: gini, `splitter`: best | 0.7885 | 0.8506 | 0.9535 | 0.8632 |
+| GaussianNB | -- | 0.8 | 0.6782 | 0.4651 | 0.5882 |
+| MLPClassifier | `activation`: logistic, `hidden_layer_sizes`: (60, 80, 100), `learning_rate`: constant, `max_iter`: 500000, `solver`: lbfgs | 0.8958 | 0.9425 | 1 | 0.9451 |
+| RandomForestClassifier | `class_weight`: balanced, `criterion`: gini, `max_features`: sqrt | 0.8367 | 0.8851 | 0.9535 | 0.8913 |
+| SVC | `gamma`: scale, `kernel`: rbf | 0.7174 | 0.7356 | 0.7674 | 0.7416 |
+
+## Decision Tree (DT)
+
+| criterion   | splitter   |   precision |   accuracy |   recall |       f1 |
+|:------------|:-----------|------------:|-----------:|---------:|---------:|
+| gini        | best       |    0.788462 |   0.850575 | 0.953488 | 0.863158 |
+| gini        | random     |    0.784314 |   0.83908  | 0.930233 | 0.851064 |
+| entropy     | random     |    0.736842 |   0.816092 | 0.976744 | 0.84     |
+| entropy     | best       |    0.745455 |   0.816092 | 0.953488 | 0.836735 |
+
+## Naive Bayes (NB)
+
+|   precision |   accuracy |   recall |       f1 |
+|------------:|-----------:|---------:|---------:|
+|         0.8 |   0.678161 | 0.465116 | 0.588235 |
+
+## Support Vector Machine (SVP)
+
+| gamma   | kernel   |   precision |   accuracy |   recall |       f1 |
+|:--------|:---------|------------:|-----------:|---------:|---------:|
+| scale   | rbf      |    0.717391 |   0.735632 | 0.767442 | 0.741573 |
+| scale   | linear   |    0.75     |   0.735632 | 0.697674 | 0.722892 |
+| auto    | linear   |    0.75     |   0.735632 | 0.697674 | 0.722892 |
+| auto    | rbf      |    0.702128 |   0.724138 | 0.767442 | 0.733333 |
+| scale   | sigmoid  |    0.647059 |   0.678161 | 0.767442 | 0.702128 |
+| auto    | sigmoid  |    0.647059 |   0.678161 | 0.767442 | 0.702128 |
+| auto    | poly     |    0.772727 |   0.643678 | 0.395349 | 0.523077 |
+| scale   | poly     |    0.833333 |   0.597701 | 0.232558 | 0.363636 |
+
+## Multi-Layer Perceptron (MLP)
+
+For sake of brevity, only the top 100 results by accuracy are shown.
+
+| activation   | hidden_layer_sizes   | learning_rate   |   max_iter | solver   |   precision |   accuracy |   recall |       f1 |
+|:-------------|:---------------------|:----------------|-----------:|:---------|------------:|-----------:|---------:|---------:|
+| logistic     | (60, 80, 100)        | constant        |     500000 | lbfgs    |    0.895833 |   0.942529 | 1        | 0.945055 |
+| logistic     | (40, 80, 100)        | adaptive        |     500000 | lbfgs    |    0.86     |   0.91954  | 1        | 0.924731 |
+| tanh         | (40, 80, 100)        | invscaling      |     500000 | adam     |    0.86     |   0.91954  | 1        | 0.924731 |
+| tanh         | (60, 100, 80)        | adaptive        |     500000 | lbfgs    |    0.86     |   0.91954  | 1        | 0.924731 |
+| tanh         | (100, 60, 20)        | constant        |     500000 | adam     |    0.86     |   0.91954  | 1        | 0.924731 |
+| tanh         | (100, 80, 80)        | constant        |     500000 | adam     |    0.86     |   0.91954  | 1        | 0.924731 |
+| relu         | (75, 30)             | adaptive        |     500000 | lbfgs    |    0.86     |   0.91954  | 1        | 0.924731 |
+| logistic     | (20, 40, 60)         | adaptive        |     500000 | lbfgs    |    0.875    |   0.91954  | 0.976744 | 0.923077 |
+| logistic     | (40, 60, 80)         | adaptive        |     500000 | lbfgs    |    0.843137 |   0.908046 | 1        | 0.914894 |
+| logistic     | (80, 40, 20)         | constant        |     500000 | lbfgs    |    0.843137 |   0.908046 | 1        | 0.914894 |
+| tanh         | 30                   | invscaling      |     500000 | lbfgs    |    0.843137 |   0.908046 | 1        | 0.914894 |
+| tanh         | 60                   | constant        |     500000 | lbfgs    |    0.843137 |   0.908046 | 1        | 0.914894 |
+| tanh         | 85                   | adaptive        |     500000 | lbfgs    |    0.843137 |   0.908046 | 1        | 0.914894 |
+| tanh         | (30, 30)             | constant        |     500000 | adam     |    0.843137 |   0.908046 | 1        | 0.914894 |
+| tanh         | (45, 45)             | adaptive        |     500000 | lbfgs    |    0.843137 |   0.908046 | 1        | 0.914894 |
+| tanh         | (60, 60)             | invscaling      |     500000 | lbfgs    |    0.843137 |   0.908046 | 1        | 0.914894 |
+| tanh         | (75, 45)             | invscaling      |     500000 | lbfgs    |    0.843137 |   0.908046 | 1        | 0.914894 |
+| tanh         | (75, 75)             | adaptive        |     500000 | lbfgs    |    0.843137 |   0.908046 | 1        | 0.914894 |
+| tanh         | (90, 90)             | invscaling      |     500000 | adam     |    0.843137 |   0.908046 | 1        | 0.914894 |
+| tanh         | (20, 40, 60)         | invscaling      |     500000 | adam     |    0.843137 |   0.908046 | 1        | 0.914894 |
+| tanh         | (20, 100, 20)        | invscaling      |     500000 | lbfgs    |    0.843137 |   0.908046 | 1        | 0.914894 |
+| tanh         | (40, 20, 100)        | constant        |     500000 | adam     |    0.843137 |   0.908046 | 1        | 0.914894 |
+| tanh         | (40, 80, 60)         | invscaling      |     500000 | adam     |    0.843137 |   0.908046 | 1        | 0.914894 |
+| tanh         | (40, 80, 100)        | adaptive        |     500000 | adam     |    0.843137 |   0.908046 | 1        | 0.914894 |
+| tanh         | (60, 20, 40)         | adaptive        |     500000 | lbfgs    |    0.843137 |   0.908046 | 1        | 0.914894 |
+| tanh         | (60, 60, 80)         | constant        |     500000 | adam     |    0.843137 |   0.908046 | 1        | 0.914894 |
+| tanh         | (60, 80, 80)         | adaptive        |     500000 | lbfgs    |    0.843137 |   0.908046 | 1        | 0.914894 |
+| tanh         | (80, 20, 40)         | adaptive        |     500000 | lbfgs    |    0.843137 |   0.908046 | 1        | 0.914894 |
+| tanh         | (80, 40, 80)         | constant        |     500000 | adam     |    0.843137 |   0.908046 | 1        | 0.914894 |
+| tanh         | (80, 60, 60)         | constant        |     500000 | lbfgs    |    0.843137 |   0.908046 | 1        | 0.914894 |
+| relu         | (20, 20, 80)         | constant        |     500000 | adam     |    0.843137 |   0.908046 | 1        | 0.914894 |
+| relu         | (20, 40, 100)        | constant        |     500000 | lbfgs    |    0.843137 |   0.908046 | 1        | 0.914894 |
+| relu         | (20, 60, 20)         | adaptive        |     500000 | adam     |    0.843137 |   0.908046 | 1        | 0.914894 |
+| relu         | (20, 60, 100)        | adaptive        |     500000 | adam     |    0.843137 |   0.908046 | 1        | 0.914894 |
+| relu         | (20, 100, 20)        | constant        |     500000 | adam     |    0.843137 |   0.908046 | 1        | 0.914894 |
+| relu         | (20, 100, 40)        | adaptive        |     500000 | adam     |    0.843137 |   0.908046 | 1        | 0.914894 |
+| relu         | (40, 20, 80)         | constant        |     500000 | lbfgs    |    0.843137 |   0.908046 | 1        | 0.914894 |
+| relu         | (40, 80, 60)         | constant        |     500000 | lbfgs    |    0.843137 |   0.908046 | 1        | 0.914894 |
+| relu         | (60, 20, 100)        | constant        |     500000 | adam     |    0.843137 |   0.908046 | 1        | 0.914894 |
+| relu         | (80, 20, 60)         | constant        |     500000 | lbfgs    |    0.843137 |   0.908046 | 1        | 0.914894 |
+| relu         | (80, 60, 20)         | adaptive        |     500000 | adam     |    0.843137 |   0.908046 | 1        | 0.914894 |
+| relu         | (100, 20, 60)        | invscaling      |     500000 | adam     |    0.843137 |   0.908046 | 1        | 0.914894 |
+| logistic     | (20, 60, 80)         | invscaling      |     500000 | lbfgs    |    0.857143 |   0.908046 | 0.976744 | 0.913043 |
+| logistic     | (60, 20, 20)         | adaptive        |     500000 | lbfgs    |    0.857143 |   0.908046 | 0.976744 | 0.913043 |
+| tanh         | (15, 45)             | constant        |     500000 | lbfgs    |    0.857143 |   0.908046 | 0.976744 | 0.913043 |
+| tanh         | (45, 90)             | invscaling      |     500000 | lbfgs    |    0.857143 |   0.908046 | 0.976744 | 0.913043 |
+| tanh         | (90, 30)             | constant        |     500000 | lbfgs    |    0.857143 |   0.908046 | 0.976744 | 0.913043 |
+| tanh         | (20, 80, 100)        | invscaling      |     500000 | adam     |    0.857143 |   0.908046 | 0.976744 | 0.913043 |
+| tanh         | (20, 80, 100)        | adaptive        |     500000 | lbfgs    |    0.857143 |   0.908046 | 0.976744 | 0.913043 |
+| tanh         | (40, 40, 40)         | adaptive        |     500000 | lbfgs    |    0.857143 |   0.908046 | 0.976744 | 0.913043 |
+| tanh         | (40, 60, 100)        | adaptive        |     500000 | adam     |    0.857143 |   0.908046 | 0.976744 | 0.913043 |
+| tanh         | (60, 80, 60)         | constant        |     500000 | lbfgs    |    0.857143 |   0.908046 | 0.976744 | 0.913043 |
+| tanh         | (100, 40, 60)        | invscaling      |     500000 | lbfgs    |    0.857143 |   0.908046 | 0.976744 | 0.913043 |
+| tanh         | (100, 80, 100)       | adaptive        |     500000 | adam     |    0.857143 |   0.908046 | 0.976744 | 0.913043 |
+| relu         | (30, 30)             | adaptive        |     500000 | lbfgs    |    0.857143 |   0.908046 | 0.976744 | 0.913043 |
+| relu         | (20, 20, 40)         | adaptive        |     500000 | lbfgs    |    0.857143 |   0.908046 | 0.976744 | 0.913043 |
+| relu         | (20, 40, 40)         | adaptive        |     500000 | adam     |    0.857143 |   0.908046 | 0.976744 | 0.913043 |
+| relu         | (40, 20, 100)        | adaptive        |     500000 | adam     |    0.857143 |   0.908046 | 0.976744 | 0.913043 |
+| relu         | (60, 80, 20)         | invscaling      |     500000 | lbfgs    |    0.857143 |   0.908046 | 0.976744 | 0.913043 |
+| logistic     | (40, 80, 60)         | adaptive        |     500000 | lbfgs    |    0.87234  |   0.908046 | 0.953488 | 0.911111 |
+| logistic     | 35                   | adaptive        |     500000 | lbfgs    |    0.826923 |   0.896552 | 1        | 0.905263 |
+| logistic     | (15, 60)             | invscaling      |     500000 | lbfgs    |    0.826923 |   0.896552 | 1        | 0.905263 |
+| logistic     | (45, 45)             | constant        |     500000 | lbfgs    |    0.826923 |   0.896552 | 1        | 0.905263 |
+| logistic     | (20, 20, 60)         | adaptive        |     500000 | lbfgs    |    0.826923 |   0.896552 | 1        | 0.905263 |
+| logistic     | (60, 60, 80)         | adaptive        |     500000 | lbfgs    |    0.826923 |   0.896552 | 1        | 0.905263 |
+| logistic     | (80, 40, 100)        | invscaling      |     500000 | lbfgs    |    0.826923 |   0.896552 | 1        | 0.905263 |
+| logistic     | (100, 100, 100)      | constant        |     500000 | lbfgs    |    0.826923 |   0.896552 | 1        | 0.905263 |
+| tanh         | 60                   | constant        |     500000 | adam     |    0.826923 |   0.896552 | 1        | 0.905263 |
+| tanh         | (15, 15)             | invscaling      |     500000 | adam     |    0.826923 |   0.896552 | 1        | 0.905263 |
+| tanh         | (15, 45)             | adaptive        |     500000 | adam     |    0.826923 |   0.896552 | 1        | 0.905263 |
+| tanh         | (30, 30)             | invscaling      |     500000 | adam     |    0.826923 |   0.896552 | 1        | 0.905263 |
+| tanh         | (30, 60)             | constant        |     500000 | adam     |    0.826923 |   0.896552 | 1        | 0.905263 |
+| tanh         | (60, 90)             | invscaling      |     500000 | lbfgs    |    0.826923 |   0.896552 | 1        | 0.905263 |
+| tanh         | (75, 15)             | constant        |     500000 | lbfgs    |    0.826923 |   0.896552 | 1        | 0.905263 |
+| tanh         | (75, 45)             | constant        |     500000 | lbfgs    |    0.826923 |   0.896552 | 1        | 0.905263 |
+| tanh         | (90, 15)             | adaptive        |     500000 | adam     |    0.826923 |   0.896552 | 1        | 0.905263 |
+| tanh         | (90, 45)             | invscaling      |     500000 | lbfgs    |    0.826923 |   0.896552 | 1        | 0.905263 |
+| tanh         | (20, 40, 20)         | invscaling      |     500000 | adam     |    0.826923 |   0.896552 | 1        | 0.905263 |
+| tanh         | (20, 40, 40)         | invscaling      |     500000 | adam     |    0.826923 |   0.896552 | 1        | 0.905263 |
+| tanh         | (20, 60, 20)         | adaptive        |     500000 | adam     |    0.826923 |   0.896552 | 1        | 0.905263 |
+| tanh         | (20, 80, 60)         | constant        |     500000 | adam     |    0.826923 |   0.896552 | 1        | 0.905263 |
+| tanh         | (20, 80, 80)         | constant        |     500000 | adam     |    0.826923 |   0.896552 | 1        | 0.905263 |
+| tanh         | (20, 80, 100)        | constant        |     500000 | adam     |    0.826923 |   0.896552 | 1        | 0.905263 |
+| tanh         | (40, 20, 60)         | invscaling      |     500000 | lbfgs    |    0.826923 |   0.896552 | 1        | 0.905263 |
+| tanh         | (40, 60, 60)         | constant        |     500000 | adam     |    0.826923 |   0.896552 | 1        | 0.905263 |
+| tanh         | (40, 60, 60)         | invscaling      |     500000 | adam     |    0.826923 |   0.896552 | 1        | 0.905263 |
+| tanh         | (40, 80, 20)         | adaptive        |     500000 | adam     |    0.826923 |   0.896552 | 1        | 0.905263 |
+| tanh         | (40, 100, 60)        | constant        |     500000 | adam     |    0.826923 |   0.896552 | 1        | 0.905263 |
+| tanh         | (60, 40, 20)         | constant        |     500000 | adam     |    0.826923 |   0.896552 | 1        | 0.905263 |
+| tanh         | (60, 40, 40)         | invscaling      |     500000 | adam     |    0.826923 |   0.896552 | 1        | 0.905263 |
+| tanh         | (60, 40, 80)         | constant        |     500000 | lbfgs    |    0.826923 |   0.896552 | 1        | 0.905263 |
+| tanh         | (60, 60, 20)         | constant        |     500000 | adam     |    0.826923 |   0.896552 | 1        | 0.905263 |
+| tanh         | (60, 80, 60)         | constant        |     500000 | adam     |    0.826923 |   0.896552 | 1        | 0.905263 |
+| tanh         | (60, 80, 80)         | invscaling      |     500000 | adam     |    0.826923 |   0.896552 | 1        | 0.905263 |
+| tanh         | (60, 100, 20)        | invscaling      |     500000 | adam     |    0.826923 |   0.896552 | 1        | 0.905263 |
+| tanh         | (60, 100, 40)        | adaptive        |     500000 | adam     |    0.826923 |   0.896552 | 1        | 0.905263 |
+| tanh         | (60, 100, 60)        | constant        |     500000 | adam     |    0.826923 |   0.896552 | 1        | 0.905263 |
+| tanh         | (60, 100, 60)        | adaptive        |     500000 | adam     |    0.826923 |   0.896552 | 1        | 0.905263 |
+| tanh         | (60, 100, 80)        | constant        |     500000 | adam     |    0.826923 |   0.896552 | 1        | 0.905263 |
+| tanh         | (80, 40, 40)         | constant        |     500000 | adam     |    0.826923 |   0.896552 | 1        | 0.905263 |
+
+## Random Forest (RF)
+
+| criterion   | class_weight       | max_features   |   precision |   accuracy |   recall |       f1 |
+|:------------|:-------------------|:---------------|------------:|-----------:|---------:|---------:|
+| gini        | balanced           | sqrt           |    0.836735 |   0.885057 | 0.953488 | 0.891304 |
+| entropy     | balanced           | sqrt           |    0.807692 |   0.873563 | 0.976744 | 0.884211 |
+| gini        | balanced_subsample | sqrt           |    0.807692 |   0.873563 | 0.976744 | 0.884211 |
+| entropy     | balanced_subsample | sqrt           |    0.807692 |   0.873563 | 0.976744 | 0.884211 |
+| gini        | balanced           | log2           |    0.82     |   0.873563 | 0.953488 | 0.88172  |
+| entropy     | balanced           | log2           |    0.82     |   0.873563 | 0.953488 | 0.88172  |
+| gini        | balanced_subsample | log2           |    0.803922 |   0.862069 | 0.953488 | 0.87234  |
+| entropy     | balanced_subsample | log2           |    0.803922 |   0.862069 | 0.953488 | 0.87234  |
+
+# Evaluation
+
+To evaluate the performance of each selected classifier, each model has trained 100 times by repeating a 5-fold
+cross validation procedure 20 times. A graphical and statistical analysis to compare the distribution of
+performance metrics (precision, recall and F1 score) follows.  
+
+Numeric tables, statistical analysis conclusions and the boxplot diagram shown in this section are obtained by
+running the script:
+
+```
+./evaluate_classifiers.py
+```
+
+## Output Distributions
+
+A boxplot chart to show the distribution of each of precision, recall, and F1 score
+for all classifiers (including the biased classifier) is shown in figure [1](#fig:boxplot){reference-type="ref" reference="fig:boxplot"}. Table [2](#tab:meanstd){reference-type="ref" reference="tab:meanstd"} is a numeric
+table summing up mean and standard deviation of each metric.
+
+![Precision, Recall and F1 score distribution for each classifier for 20-times cross validation.](../models/boxplot.svg){#fig:boxplot}
+
+::: {#tab:meanstd}
+| **Classifier**     | **Metric** | **Mean** | **Std. dev.** |
+|:-----------------------|:----------|---------:|-----------:|
+| BiasedClassifier       | f1        | 0.666238 | 0.00511791 |
+| BiasedClassifier       | precision | 0.49954  | 0.00575757 |
+| BiasedClassifier       | recall    | 1        | 0          |
+| DecisionTreeClassifier | f1        | 0.888104 | 0.0302085  |
+| DecisionTreeClassifier | precision | 0.832669 | 0.0425814  |
+| DecisionTreeClassifier | recall    | 0.953261 | 0.0358021  |
+| GaussianNB             | f1        | 0.54952  | 0.0912106  |
+| GaussianNB             | precision | 0.820866 | 0.066323   |
+| GaussianNB             | recall    | 0.418885 | 0.0950382  |
+| MLPClassifier          | f1        | 0.884807 | 0.0348063  |
+| MLPClassifier          | precision | 0.836507 | 0.0489391  |
+| MLPClassifier          | recall    | 0.941823 | 0.0454851  |
+| RandomForestClassifier | f1        | 0.91079  | 0.0301481  |
+| RandomForestClassifier | precision | 0.870685 | 0.0427383  |
+| RandomForestClassifier | recall    | 0.956665 | 0.0381288  |
+| SVC                    | f1        | 0.752656 | 0.0537908  |
+| SVC                    | precision | 0.7557   | 0.0515768  |
+| SVC                    | recall    | 0.754709 | 0.0819603  |
+
+  : Mean and standard deviation for each classifier for 20-times cross validation.
+:::
+
+
+## Comparison and Significance
+
+Given the distribution of metrics presented in the previous section, I perform a statistical analysis
+using the Wilixcon paired test to determine for each pair of classifiers which one performs better 
+according to each performance metric. When the *p-value* is too high, *ANOVA* power analysis (corrected
+by *alpha* = $= 0.05$) is performed to determine if the metrics are equally distributed or if the statistical
+test is inconclusive.
+
+# F1 Values
+
+- Mean *F1* for *DT*: 0.8881, mean *F1* for *NB*: 0.5495 $\Rightarrow$ *DT* is better than *NB* (*p-value* $= 0.0$)
+- Mean *F1* for *DT*: 0.8881, mean *F1* for *MLP*: 0.8848 $\Rightarrow$ statistical test inconclusive (*p-value* $= 0.4711$, *5% corrected ANOVA power* $= 0.2987$)
+- Mean *F1* for *DT*: 0.8881, mean *F1* for *RF*: 0.9108 $\Rightarrow$ *RF* is better than *DT* (*p-value* $= 0.0$)
+- Mean *F1* for *DT*: 0.8881, mean *F1* for *SVP*: 0.7527 $\Rightarrow$ *DT* is better than *SVP* (*p-value* $= 0.0$)
+- Mean *F1* for *NB*: 0.5495, mean *F1* for *MLP*: 0.8848 $\Rightarrow$ *MLP* is better than *NB* (*p-value* $= 0.0$)
+- Mean *F1* for *NB*: 0.5495, mean *F1* for *RF*: 0.9108 $\Rightarrow$ *RF* is better than *NB* (*p-value* $= 0.0$)
+- Mean *F1* for *NB*: 0.5495, mean *F1* for *SVP*: 0.7527 $\Rightarrow$ *SVP* is better than *NB* (*p-value* $= 0.0$)
+- Mean *F1* for *MLP*: 0.8848, mean *F1* for *RF*: 0.9108 $\Rightarrow$ *RF* is better than *MLP* (*p-value* $= 0.0$)
+- Mean *F1* for *MLP*: 0.8848, mean *F1* for *SVP*: 0.7527 $\Rightarrow$ *MLP* is better than *SVP* (*p-value* $= 0.0$)
+- Mean *F1* for *RF*: 0.9108, mean *F1* for *SVP*: 0.7527 $\Rightarrow$ *RF* is better than *SVP* (*p-value* $= 0.0$)
+- Mean *F1* for *Biased*: 0.6662, mean *F1* for *DT*: 0.8881 $\Rightarrow$ *DT* is better than *Biased* (*p-value* $= 0.0$)
+- Mean *F1* for *Biased*: 0.6662, mean *F1* for *NB*: 0.5495 $\Rightarrow$ *Biased* is better than *NB* (*p-value* $= 0.0$)
+- Mean *F1* for *Biased*: 0.6662, mean *F1* for *MLP*: 0.8848 $\Rightarrow$ *MLP* is better than *Biased* (*p-value* $= 0.0$)
+- Mean *F1* for *Biased*: 0.6662, mean *F1* for *RF*: 0.9108 $\Rightarrow$ *RF* is better than *Biased* (*p-value* $= 0.0$)
+- Mean *F1* for *Biased*: 0.6662, mean *F1* for *SVP*: 0.7527 $\Rightarrow$ *SVP* is better than *Biased* (*p-value* $= 0.0$)
+
+### Precision
+
+- Mean *precision* for *DT*: 0.8327, mean *precision* for *NB*: 0.8209 $\Rightarrow$ *DT* is as effective as *NB* (*p-value* $= 0.0893$, *5% corrected ANOVA power* $= 0.8498$)
+- Mean *precision* for *DT*: 0.8327, mean *precision* for *MLP*: 0.8365 $\Rightarrow$ statistical test inconclusive (*p-value* $= 0.4012$, *5% corrected ANOVA power* $= 0.2196$)
+- Mean *precision* for *DT*: 0.8327, mean *precision* for *RF*: 0.8707 $\Rightarrow$ *RF* is better than *DT* (*p-value* $= 0.0$)
+- Mean *precision* for *DT*: 0.8327, mean *precision* for *SVP*: 0.7557 $\Rightarrow$ *DT* is better than *SVP* (*p-value* $= 0.0$)
+- Mean *precision* for *NB*: 0.8209, mean *precision* for *MLP*: 0.8365 $\Rightarrow$ *MLP* is better than *NB* (*p-value* $= 0.0348$)
+- Mean *precision* for *NB*: 0.8209, mean *precision* for *RF*: 0.8707 $\Rightarrow$ *RF* is better than *NB* (*p-value* $= 0.0$)
+- Mean *precision* for *NB*: 0.8209, mean *precision* for *SVP*: 0.7557 $\Rightarrow$ *NB* is better than *SVP* (*p-value* $= 0.0$)
+- Mean *precision* for *MLP*: 0.8365, mean *precision* for *RF*: 0.8707 $\Rightarrow$ *RF* is better than *MLP* (*p-value* $= 0.0$)
+- Mean *precision* for *MLP*: 0.8365, mean *precision* for *SVP*: 0.7557 $\Rightarrow$ *MLP* is better than *SVP* (*p-value* $= 0.0$)
+- Mean *precision* for *RF*: 0.8707, mean *precision* for *SVP*: 0.7557 $\Rightarrow$ *RF* is better than *SVP* (*p-value* $= 0.0$)
+- Mean *precision* for *Biased*: 0.4995, mean *precision* for *DT*: 0.8327 $\Rightarrow$ *DT* is better than *Biased* (*p-value* $= 0.0$)
+- Mean *precision* for *Biased*: 0.4995, mean *precision* for *NB*: 0.8209 $\Rightarrow$ *NB* is better than *Biased* (*p-value* $= 0.0$)
+- Mean *precision* for *Biased*: 0.4995, mean *precision* for *MLP*: 0.8365 $\Rightarrow$ *MLP* is better than *Biased* (*p-value* $= 0.0$)
+- Mean *precision* for *Biased*: 0.4995, mean *precision* for *RF*: 0.8707 $\Rightarrow$ *RF* is better than *Biased* (*p-value* $= 0.0$)
+- Mean *precision* for *Biased*: 0.4995, mean *precision* for *SVP*: 0.7557 $\Rightarrow$ *SVP* is better than *Biased* (*p-value* $= 0.0$)
+
+### Recall
+
+- Mean *recall* for *DT*: 0.9533, mean *recall* for *NB*: 0.4189 $\Rightarrow$ *DT* is better than *NB* (*p-value* $= 0.0$)
+- Mean *recall* for *DT*: 0.9533, mean *recall* for *MLP*: 0.9418 $\Rightarrow$ *DT* is better than *MLP* (*p-value* $= 0.0118$)
+- Mean *recall* for *DT*: 0.9533, mean *recall* for *RF*: 0.9567 $\Rightarrow$ statistical test inconclusive (*p-value* $= 0.3276$, *5% corrected ANOVA power* $= 0.2558$)
+- Mean *recall* for *DT*: 0.9533, mean *recall* for *SVP*: 0.7547 $\Rightarrow$ *DT* is better than *SVP* (*p-value* $= 0.0$)
+- Mean *recall* for *NB*: 0.4189, mean *recall* for *MLP*: 0.9418 $\Rightarrow$ *MLP* is better than *NB* (*p-value* $= 0.0$)
+- Mean *recall* for *NB*: 0.4189, mean *recall* for *RF*: 0.9567 $\Rightarrow$ *RF* is better than *NB* (*p-value* $= 0.0$)
+- Mean *recall* for *NB*: 0.4189, mean *recall* for *SVP*: 0.7547 $\Rightarrow$ *SVP* is better than *NB* (*p-value* $= 0.0$)
+- Mean *recall* for *MLP*: 0.9418, mean *recall* for *RF*: 0.9567 $\Rightarrow$ *RF* is better than *MLP* (*p-value* $= 0.0001$)
+- Mean *recall* for *MLP*: 0.9418, mean *recall* for *SVP*: 0.7547 $\Rightarrow$ *MLP* is better than *SVP* (*p-value* $= 0.0$)
+- Mean *recall* for *RF*: 0.9567, mean *recall* for *SVP*: 0.7547 $\Rightarrow$ *RF* is better than *SVP* (*p-value* $= 0.0$)
+- Mean *recall* for *Biased*: 1.0, mean *recall* for *DT*: 0.9533 $\Rightarrow$ *Biased* is better than *DT* (*p-value* $= 0.0$)
+- Mean *recall* for *Biased*: 1.0, mean *recall* for *NB*: 0.4189 $\Rightarrow$ *Biased* is better than *NB* (*p-value* $= 0.0$)
+- Mean *recall* for *Biased*: 1.0, mean *recall* for *MLP*: 0.9418 $\Rightarrow$ *Biased* is better than *MLP* (*p-value* $= 0.0$)
+- Mean *recall* for *Biased*: 1.0, mean *recall* for *RF*: 0.9567 $\Rightarrow$ *Biased* is better than *RF* (*p-value* $= 0.0$)
+- Mean *recall* for *Biased*: 1.0, mean *recall* for *SVP*: 0.7547 $\Rightarrow$ *Biased* is better than *SVP* (*p-value* $= 0.0$)
+
+## Practical Usefulness
+
+The evaluation shows that all classifiers but the Naive Bayes classifier outperform the biased classifier in overall performance
+(represented by the F1 score). Given the evaluation was not performed on a dedicated test set, all classifiers may be subjected to
+training bias, which might yield better metrics than a similar evaluation performed on completely new data. However, given the 
+evaluation sample size (i.e. the number of training runs) this effect is minimal.
+
+Given this premise, I can say with reasonable confidence the *DT*, *MLP*, *RP* and *SVP* 
+classifiers would be useful in predicting potential bugs in the Google JSComp project, i.e. the source of the used dataset. Given the 
+literature presented during lecture, it is not certain if the same trained classifier would yield acceptable results for source
+code from another project. This is because differences in coding conventions, the bug tracking process, or simply the definition of 
+what constitutes a bug may vary from project to project. A solution to this problem might be to simply train the classifiers on a dataset
+coming from the project where bug prediction is needed. 

requirements.txt

@@ -1,4 +1,7 @@
 javalang==0.13.0
 pandas==1.5.2
 scikit_learn==1.2.1
-tabulate==0.9.0
+tabulate==0.9.0
+scipy==1.24.2
+seaborn==0.12.2
+statsmodels==0.14.0

train_classifiers.py

@@ -150,6 +150,11 @@ def find_best_and_save(df: pd.DataFrame):
       
     metrics = ['precision', 'accuracy', 'recall', 'f1']
     df_best.loc[:, metrics] = df_best.loc[:, metrics].round(decimals=4)
+    df_best = df_best.reindex(
+        ['classifier', 'params'] + \
+        [x for x in df_best.columns if x in metrics], \
+    axis=1)
+    
     print(df_best.to_markdown(index=False))
 
 
@@ -165,6 +170,27 @@ def main():
     else:
         df = pd.read_csv(OUT_DIR + '/models.csv')
 
+    for clazz in set(df['classifier']):
+        dfc = df.loc[df.classifier == clazz, :].copy()
+        dfc = dfc[dfc.columns.drop(list(df.filter(regex='^(mean_)|(std_)|(rank_)|(params$)|(classifier$)')))]
+        dfc = dfc.rename(columns={
+            "split0_test_precision": "precision",
+            "split0_test_accuracy": "accuracy",
+            "split0_test_recall": "recall",
+            "split0_test_f1": "f1"
+        })
+
+        dfc = dfc.reindex(
+            [x for x in dfc.columns if x.startswith('param_')] + \
+            [x for x in dfc.columns if not x.startswith('param_')], \
+            axis=1)
+        dfc = dfc.rename(columns=dict([(c, c.replace('param_', '')) for c in dfc.columns]))
+        dfc = dfc.loc[:, dfc.notna().any(axis=0)]
+
+        print(clazz)
+        print(dfc.head(100).to_markdown(index=False))
+        print()
+
     find_best_and_save(df)