Evolution_FeatureExtraction/data_utils.py at main · ItsTatsuya/Evolution_FeatureExtraction · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
import os
import zipfile
import pandas as pd
import numpy as np
import shutil
from config import DATASET_NAME_MAP


def extract_zip(zip_path, extract_to):
    if os.path.exists(extract_to):
        shutil.rmtree(extract_to)
    with zipfile.ZipFile(zip_path, 'r') as zip_ref:
        zip_ref.extractall(extract_to)


def parse_keel_dat(file_path):
    data = []
    attributes = []
    attribute_types = {}
    class_label = None
    reading_data = False

    with open(file_path, 'r') as f:
        for line in f:
            line = line.strip()
            if line.startswith('@relation'):
                continue
            elif line.startswith('@attribute'):
                parts = line.split()
                attr_name = parts[1]
                attr_type = 'numeric' if 'real' in line or 'integer' in line else 'categorical'
                attribute_types[attr_name] = attr_type
                if '{' in line:
                    class_label = attr_name
                attributes.append(attr_name)
            elif line.startswith('@data'):
                reading_data = True
            elif reading_data and line:
                data.append(line.split(','))

    if not data:
        raise ValueError("No data found in file")

    df = pd.DataFrame(data, columns=attributes)

    for col in df.columns:
        if attribute_types[col] == 'numeric':
            df[col] = pd.to_numeric(df[col], errors='coerce')
        elif col != class_label:
            df = pd.get_dummies(df, columns=[col], prefix=col)

    if df.isna().any().any():
        print(f"  Warning: NaNs in {file_path}, filling with median/mode")
        for col in df.columns:
            if df[col].dtype.kind in 'fc':
                df[col] = df[col].fillna(df[col].median())
            else:
                df[col] = df[col].fillna(df[col].mode()[0])

    return df, class_label


def normalize_name(folder_name):
    return DATASET_NAME_MAP.get(folder_name, f"{folder_name}-5")


def find_fold_files(folder_path):
    fold_zip = None
    for f in os.listdir(folder_path):
        if f.endswith('-fold.zip'):
            fold_zip = os.path.join(folder_path, f)
            break

    if not fold_zip:
        return None, None, None

    extract_dir = os.path.join(folder_path, "extracted_folds")
    extract_zip(fold_zip, extract_dir)

    train_file = None
    test_file = None

    for f in os.listdir(extract_dir):
        if f.endswith('1tra.dat'):
            train_file = os.path.join(extract_dir, f)
        elif f.endswith('1tst.dat'):
            test_file = os.path.join(extract_dir, f)

    return fold_zip, train_file, test_file


def prepare_datasets(df_train, df_test, class_col):
    all_columns = set(df_train.columns) | set(df_test.columns)
    for col in all_columns:
        if col not in df_train.columns:
            df_train[col] = 0
        if col not in df_test.columns:
            df_test[col] = 0

    common_cols = sorted(list(all_columns - {class_col}))
    df_train = df_train[common_cols + [class_col]]
    df_test = df_test[common_cols + [class_col]]

    X_train = df_train.drop(columns=[class_col]).values.astype(float)

    class_values = df_train[class_col].value_counts()
    if len(class_values) < 2:
        raise ValueError("Only one class in training data")

    target_class = class_values.index[1]
    y_train = (df_train[class_col] == target_class).astype(int).values

    X_test = df_test.drop(columns=[class_col]).values.astype(float)

    if target_class not in df_test[class_col].values:
        print(f"  Warning: Target class {target_class} not in test data")
        if len(df_test[class_col].unique()) > 1:
            alt_class = df_test[class_col].value_counts().index[1]
            y_test = (df_test[class_col] == alt_class).astype(int).values
        else:
            y_test = np.zeros(len(df_test))
            positive_samples = max(2, int(0.1 * len(df_test)))
            random_indices = np.random.choice(len(df_test), positive_samples, replace=False)
            y_test[random_indices] = 1
    else:
        y_test = (df_test[class_col] == target_class).astype(int).values

    numeric_cols = [i for i, col in enumerate(df_train.columns)
                   if col != class_col and df_train[col].dtype.kind in 'fc']

    return X_train, y_train, X_test, y_test, numeric_cols, target_class