code.py

# -*- coding: utf-8 -*-
"""s4730672.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1x5XcUim29pQdVx6KE1s-rMHKvAYH8dC0
"""

import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.impute import SimpleImputer
from sklearn.preprocessing import StandardScaler
from sklearn.ensemble import RandomForestClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import classification_report, f1_score
from sklearn.metrics import accuracy_score


# Load the two training datasets
train_data_1 = pd.read_csv("train.csv")
train_data_2 = pd.read_csv("add_train.csv")

# Combine the two datasets into one
combined_train_data = pd.concat([train_data_1, train_data_2], ignore_index=True)

# Load the test data
test_data = pd.read_csv("test.csv")

combined_train_data.head()

test_data.head()

# Separate features (X) and labels (y) for training data
X_train = combined_train_data.drop(columns=["Label"])
y_train = combined_train_data["Label"]

# Split the data into training (80%) and validation (20%)
X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.2, random_state=42)

# Preprocessing for Training Data
imputer = SimpleImputer(strategy="mean")
X_train = imputer.fit_transform(X_train)

scaler = StandardScaler()
X_train = scaler.fit_transform(X_train)

# Handle Outliers (if needed)
def handle_outliers(data, threshold=1.5):
    for i in range(data.shape[1]):  # Loop through columns
        Q1 = np.percentile(data[:, i], 25)
        Q3 = np.percentile(data[:, i], 75)
        IQR = Q3 - Q1
        lower_bound = Q1 - threshold * IQR
        upper_bound = Q3 + threshold * IQR
        data[:, i] = np.where(data[:, i] < lower_bound, lower_bound, data[:, i])
        data[:, i] = np.where(data[:, i] > upper_bound, upper_bound, data[:, i])

# Preprocessing for Validation and Test Data
X_val = imputer.transform(X_val)
X_val = scaler.transform(X_val)

X_test = imputer.transform(test_data)
X_test = scaler.transform(X_test)

# Handle Outliers (if needed) - Use the same function as for the training data
for i in range(X_test.shape[1]):  # Loop through columns
    threshold = 1.5  # Adjust the threshold as needed
    Q1 = np.percentile(X_test[:, i], 25)
    Q3 = np.percentile(X_test[:, i], 75)
    IQR = Q3 - Q1
    lower_bound = Q1 - threshold * IQR
    upper_bound = Q3 + threshold * IQR
    X_test[:, i] = np.where(X_test[:, i] < lower_bound, lower_bound, X_test[:, i])
    X_test[:, i] = np.where(X_test[:, i] > upper_bound, upper_bound, X_test[:, i])

print(X_train)

print(y_train)

from sklearn.decomposition import PCA

# Perform PCA for dimensionality reduction
pca = PCA(n_components=10)  # Customize the number of components as needed
X_train_pca = pca.fit_transform(X_train)
X_val_pca = pca.transform(X_val)

# Model Training and Prediction with PCA
model_with_pca = RandomForestClassifier(n_estimators=100, random_state=42)
model_with_pca.fit(X_train_pca, y_train)
y_val_predictions_pca = model_with_pca.predict(X_val_pca)

# Calculate macro-F1 for the model with PCA
macro_f1_pca = f1_score(y_val, y_val_predictions_pca, average='macro')
print("Macro-F1 for Model with PCA:", macro_f1_pca)

from sklearn.experimental import enable_hist_gradient_boosting
from sklearn.ensemble import HistGradientBoostingClassifier
# Model Training for RandomForest
rf_model = RandomForestClassifier(n_estimators=100, random_state=42)
rf_model.fit(X_train, y_train)

# Model Prediction on Validation Data for RandomForest
y_val_predictions_rf = rf_model.predict(X_val)

# Model Prediction on Test Data for RandomForest
test_predictions_rf = rf_model.predict(X_test)

# Calculate macro-F1 for Random Forest
macro_f1_rf = f1_score(y_val, y_val_predictions_rf, average='macro')
print("Macro-F1 for Random Forest:", macro_f1_rf)

# Model Training for Decision Trees
dt_model = DecisionTreeClassifier(random_state=42)
dt_model.fit(X_train, y_train)

# Model Prediction on Validation Data for Decision Trees
y_val_predictions_dt = dt_model.predict(X_val)

# Model Prediction on Test Data for Decision Trees
test_predictions_dt = dt_model.predict(X_test)

# Calculate macro-F1 for Decision Trees
macro_f1_dt = f1_score(y_val, y_val_predictions_dt, average='macro')
print("Macro-F1 for Decision Trees:", macro_f1_dt)

# Model Training for K-Nearest Neighbors (K-NN)
knn_model = KNeighborsClassifier()
knn_model.fit(X_train, y_train)

# Model Prediction on Validation Data for K-NN
y_val_predictions_knn = knn_model.predict(X_val)

# Model Prediction on Test Data for K-NN
test_predictions_knn = knn_model.predict(X_test)

# Calculate macro-F1 for K-Nearest Neighbors
macro_f1_knn = f1_score(y_val, y_val_predictions_knn, average='macro')
print("Macro-F1 for K-Nearest Neighbors:", macro_f1_knn)

# Model Training for Naïve Bayes
naive_bayes = GaussianNB()
naive_bayes.fit(X_train, y_train)

# Model Prediction on Validation Data for Naïve Bayes
y_val_predictions_nb = naive_bayes.predict(X_val)

# Model Prediction on Test Data for Naïve Bayes
test_predictions_nb = naive_bayes.predict(X_test)

# Calculate macro-F1 for Naïve Bayes
macro_f1_nb = f1_score(y_val, y_val_predictions_nb, average='macro')
print("Macro-F1 for Naïve Bayes:", macro_f1_nb)


# Model Training for HistGradientBoosting
hgb_model = HistGradientBoostingClassifier(random_state=42)
hgb_model.fit(X_train, y_train)

# Model Prediction on Validation Data for HistGradientBoosting
y_val_predictions_hgb = hgb_model.predict(X_val)

# Model Prediction on Test Data for HistGradientBoosting
test_predictions_hgb = hgb_model.predict(X_test)

# Hyperparameter Tuning for Decision Trees
dt_param_grid = {
    'max_depth': [None, 10, 20, 30],
    'min_samples_split': [2, 5, 10],
}

# Create a grid search with cross-validation for Decision Trees
dt_grid_search = GridSearchCV(DecisionTreeClassifier(random_state=42), dt_param_grid, cv=5, scoring='accuracy')
dt_grid_search.fit(X_train, y_train)

# Get the best hyperparameters for Decision Trees
best_decision_tree = dt_grid_search.best_estimator_
y_val_predicted_dt = best_decision_tree.predict(X_val)

# Evaluate and compare results for Decision Trees
print("Hyperparameter Tuning for Decision Trees")
print("Best Hyperparameters:", dt_grid_search.best_params_)
print("Accuracy:", best_decision_tree.score(X_val, y_val))
print("Classification Report:")
print(classification_report(y_val, y_val_predicted_dt))

# Hyperparameter Tuning for Random Forest
rf_param_grid = {
    'n_estimators': [100, 200, 300],
    'max_depth': [None, 10, 20],
    'min_samples_split': [2, 5, 10],
}

# Create a grid search with cross-validation for Random Forest
rf_grid_search = GridSearchCV(RandomForestClassifier(random_state=42), rf_param_grid, cv=5, scoring='accuracy')
rf_grid_search.fit(X_train, y_train)

# Get the best hyperparameters for Random Forest
best_random_forest = rf_grid_search.best_estimator_
y_val_predicted_rf = best_random_forest.predict(X_val)

# Evaluate and compare results for Random Forest
print("Hyperparameter Tuning for Random Forest")
print("Best Hyperparameters:", rf_grid_search.best_params_)
print("Accuracy:", best_random_forest.score(X_val, y_val))
print("Classification Report:")
print(classification_report(y_val, y_val_predicted_rf))

# Hyperparameter Tuning for K-Nearest Neighbors (K-NN)
knn_param_grid = {
    'n_neighbors': [3, 5, 7],
    'weights': ['uniform', 'distance'],
}

# Create a grid search with cross-validation for K-NN
knn_grid_search = GridSearchCV(KNeighborsClassifier(), knn_param_grid, cv=5, scoring='accuracy')
knn_grid_search.fit(X_train, y_train)

# Get the best hyperparameters for K-NN
best_knn = knn_grid_search.best_estimator_
y_val_predicted_knn = best_knn.predict(X_val)

# Evaluate and compare results for K-NN
print("Hyperparameter Tuning for K-Nearest Neighbors (K-NN)")
print("Best Hyperparameters:", knn_grid_search.best_params_)
print("Accuracy:", best_knn.score(X_val, y_val))
print("Classification Report:")
print(classification_report(y_val, y_val_predicted_knn))

# Hyperparameter Tuning for Naïve Bayes
param_grid = {
    'priors': [None, [0.3, 0.7], [0.5, 0.5], [0.7, 0.3]]  # Example values, customize as needed
}

# Create a grid search with cross-validation for Naïve Bayes
nb_grid_search = GridSearchCV(naive_bayes, param_grid, cv=5, scoring='f1_macro')
nb_grid_search.fit(X_train, y_train)

# Get the best hyperparameters for Naïve Bayes
best_naive_bayes = nb_grid_search.best_estimator_
y_val_predicted_nb = best_naive_bayes.predict(X_val)

# No hyperparameter tuning is required for Naïve Bayes
naive_bayes = GaussianNB()
naive_bayes.fit(X_train, y_train)
y_val_predicted_nb = naive_bayes.predict(X_val)

# Evaluate and compare results for Naïve Bayes
print("Results for Naïve Bayes")
print("Best Hyperparameters:", nb_grid_search.best_params_)
print("Macro-F1 Score:", f1_score(y_val, y_val_predicted_nb, average='macro'))
print("Accuracy:", naive_bayes.score(X_val, y_val))
print("Classification Report:")
print(classification_report(y_val, y_val_predicted_nb))

from sklearn.ensemble import VotingClassifier

# Create a Voting Classifier with different models
ensemble = VotingClassifier(estimators=[
    ('Random Forest', best_random_forest),
    ('Decision Trees', best_decision_tree),
    ('K-Nearest Neighbors', best_knn),
    ('Naïve Bayes', best_naive_bayes)
], voting='soft')

# Train the ensemble
ensemble.fit(X_train, y_train)

# Make predictions with the ensemble model
y_val_predictions_ensemble = ensemble.predict(X_val)

# Calculate macro-F1 for the ensemble
macro_f1_ensemble = f1_score(y_val, y_val_predictions_ensemble, average='macro')
print("Macro-F1 for Ensemble of Ensembles:", macro_f1_ensemble)

# Calculate accuracy for the ensemble
accuracy_ensemble = accuracy_score(y_val, y_val_predictions_ensemble)
print("Accuracy for Ensemble of Ensembles:", accuracy_ensemble)

# Generate predictions for all tuned models
test_predictions_dt = best_decision_tree.predict(X_test)
test_predictions_rf = best_random_forest.predict(X_test)
test_predictions_knn = best_knn.predict(X_test)
test_predictions_nb = naive_bayes.predict(X_test)
test_predictions_ensemble = ensemble.predict(X_test)

# Save predictions for all models to separate CSV files
predictions_dt_df = pd.DataFrame({'Label': test_predictions_dt})
predictions_dt_df.to_csv("predictions_dt.csv", index=False)

predictions_rf_df = pd.DataFrame({'Label': test_predictions_rf})
predictions_rf_df.to_csv("predictions_rf.csv", index=False)

predictions_knn_df = pd.DataFrame({'Label': test_predictions_knn})
predictions_knn_df.to_csv("predictions_knn.csv", index=False)

predictions_nb_df = pd.DataFrame({'Label': test_predictions_nb})
predictions_nb_df.to_csv("predictions_nb.csv", index=False)

predictions_ensemble_df = pd.DataFrame({'pred': test_predictions_ensemble})
predictions_ensemble_df.to_csv("predictions_ensemble.csv", index=False)





# # Calculate the macro-F1 for each model
# macro_f1_dt = f1_score(y_val, y_val_predicted_dt, average='macro')
# macro_f1_rf = f1_score(y_val, y_val_predicted_rf, average='macro')
# macro_f1_knn = f1_score(y_val, y_val_predicted_knn, average='macro')
# macro_f1_nb = f1_score(y_val, y_val_predicted_nb, average='macro')

# # Calculate the corresponding marks for each model
# mark_dt = calculate_mark(macro_f1_dt)
# mark_rf = calculate_mark(macro_f1_rf)
# mark_knn = calculate_mark(macro_f1_knn)
# mark_nb = calculate_mark(macro_f1_nb)

# # Print marks for each model
# print("Marks for Decision Trees:", mark_dt)
# print("Marks for Random Forest:", mark_rf)
# print("Marks for K-Nearest Neighbors:", mark_knn)
# print("Marks for Naïve Bayes:", mark_nb)

# # Assuming you have a test dataset named "test.csv"
# # Preprocess the test data (impute, scale, encode categorical variables) as shown earlier

# # Generate predictions
# test_predictions = best_random_forest.predict(X_test)

# # Save predictions to a CSV file
# predictions_df = pd.DataFrame({'Label': test_predictions})
# predictions_df.to_csv("predictions.csv", index=False)