Github-Issue-Classifier/github_issue_models_metrics.py at main · 2170chm/Github-Issue-Classifier · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
import numpy as np
import joblib
from sklearn.metrics import accuracy_score, f1_score, classification_report

def load_test_data(representation):
    X_test = np.load(f'text_representation/multiclass/X_test_{representation}.npy')
    y_test = np.load(f'text_representation/multiclass/y_test.npy')

    return X_test, y_test

def load_models(representation, model_type, one_vs_all):
    if one_vs_all:
        model_paths = [f'models/{model_type}_{representation}_one_vs_all_class_{i}.joblib' for i in range(3)]
    else:
        model_paths = [f'models/{model_type}_{representation}_multiclass.joblib']

    models = [joblib.load(path) for path in model_paths]

    if one_vs_all:
        return models
    else:
        return models[0]

def vote_predictions(models, X_test):
    num_samples = X_test.shape[0]

    voted_predictions = []

    for i in range(num_samples):
        sample_predictions = [model.predict(X_test[i].reshape(1, -1)) for model in models]
        sample_predictions = np.array(sample_predictions).flatten()

        voted_prediction = np.argmax(np.bincount(sample_predictions))
        voted_predictions.append(voted_prediction)

    voted_predictions = np.array(voted_predictions)

    return voted_predictions

def evaluate_model(model, X_test, y_test, model_name, representation):
    print(f"Metrics for model {model_name} with {representation} representation")
    predictions = model.predict(X_test)
    print(f"Accuracy: {accuracy_score(y_test, predictions)}")
    print(f"F1 Score: {f1_score(y_test, predictions, average='weighted')}")
    print(classification_report(y_test, predictions))

if __name__ == "__main__":
    representations = ['tfidf', 'lda', 'embeddings']
    model_types = ['rf', 'nb', 'nn', 'svm']

    for representation in representations:
        X_test, y_test = load_test_data(representation)

        for model_type in model_types:
            if model_type == 'svm':
                svm_models = load_models(representation, 'svm', True)
                voted_predictions = vote_predictions(svm_models, X_test)
                print(f"Metrics for SVM with {representation} representation (one-vs-all):")
                print(f"Accuracy: {accuracy_score(y_test, voted_predictions)}")
                print(f"F1 Score: {f1_score(y_test, voted_predictions, average='weighted')}")
                print(classification_report(y_test, voted_predictions))
            else:
                model = load_models(representation, model_type, False)
                evaluate_model(model, X_test, y_test, model_type, representation)