assignment2.py

import cvxpy as cp
import numpy as np
import csv
from cvxopt import matrix, solvers


def read_csv_train(file_path="./train.csv"):
    data = []
    labels = []

    with open(file_path, "r") as file:
        csv_reader = csv.reader(file)

        for line in csv_reader:
            label = float(line[0])
            if label == 0:
                label = -1.00
            labels.append(label)

            datas = [float(val) for val in line[1:]]
            data.append(datas)

    data_train = np.array(data[:4000])
    label_train = np.array(labels[:4000])

    return data_train, label_train

def read_csv_test(file_path="./test.csv"):
    data_test = []
    label_test = []

    with open(file_path, "r") as file:
        csv_reader = csv.reader(file)

        for line in csv_reader:
            label = float(line[0])
            if label == 0:
                label = -1.00
            label_test.append(label)

            datas = [float(val) for val in line[1:]]
            data_test.append(datas)

    return np.array(data_test), np.array(label_test)

def svm_train_primal(data_train, label_train, regularisation_para_C):
    num_samples = data_train.shape[0]
    num_features = data_train.shape[1]
    
    w = cp.Variable(num_features)  # Weight vector
    b = cp.Variable()  # Bias term
    xi = cp.Variable(num_samples)  # Slack variables
    
    # Defining the objective function
    objective = cp.Minimize((1/2) * cp.norm(w, 2)**2 + (regularisation_para_C/num_samples) * cp.sum(xi))
    
    # Defining the constraints
    constraints = [cp.multiply(label_train, data_train @ w + b) >= 1 - xi, xi >= 0]
    
    # Creating and solving the optimization problem
    problem = cp.Problem(objective, constraints)
    problem.solve()
    
    return {"w": w.value, "b": b.value}

def svm_predict_primal(data, label, svm_model):
    # Getting predictions
    predictions = data @ svm_model["w"] + svm_model["b"]
    # Assigning labels based on sign of predictions
    predicted_labels = np.sign(predictions)
    # Calculating accuracy
    accuracy = np.mean(predicted_labels == label)
    return accuracy

def svm_train_dual(data_train, label_train, regularisation_para_C):
    num_samples = data_train.shape[0]
    
    # Calculating the Gram matrix with a slightly different approach
    gram_matrix = np.zeros((num_samples, num_samples))
    for i in range(num_samples):
        for j in range(num_samples):
            gram_matrix[i, j] = label_train[i] * label_train[j] * np.dot(data_train[i], data_train[j])

    # Setting up the Quadratic Programming problem with different variable names
    P_matrix = matrix(gram_matrix)
    linear_terms = matrix(-np.ones(num_samples))
    inequality_matrix = matrix(np.vstack((-np.eye(num_samples), np.eye(num_samples))))
    bounds = matrix(np.hstack((np.zeros(num_samples), regularisation_para_C / num_samples * np.ones(num_samples))))
    equality_matrix = matrix(label_train.reshape(1, -1))
    equality_bound = matrix(0.0)  

    solution = solvers.qp(P_matrix, linear_terms, inequality_matrix, bounds, equality_matrix, equality_bound)
    alphas_result = np.array(solution['x'])

    return alphas_result

def get_primal_from_dual(data_train, label_train, alpha, regularisation_para_C):
    # Calculate w using matrix operations
    w_star = np.sum((alpha * label_train).reshape(-1, 1) * data_train)
    
    # Define a threshold for determining support vectors
    threshold = 1e-5
    
    # Get support vectors. We consider alphas that are not close to 0 or C.
    sv_indices = np.where((alpha > threshold) & (alpha < regularisation_para_C - threshold))[0]
    
    # Calculate b using the average of the support vectors
    b_values = label_train[sv_indices] - np.dot(data_train[sv_indices], w_star)
    b_star = np.mean(b_values)
    
    return w_star, b_star

if __name__ == "__main__":
    data_train, label_train = read_csv_train()
    data_test , label_test = read_csv_test()
    # Primal SVM
    svm_model = svm_train_primal(data_train, label_train, 100)
    test_accuracy = svm_predict_primal(data_test , label_test, svm_model)
    print("b:", svm_model["b"])
    print("sum of w:", np.sum(svm_model["w"]))
    print("Test Accuracy:", test_accuracy)

    # Dual SVM
    alpha = svm_train_dual(data_train, label_train, 100)
    print("Sum of alpha:", np.sum(alpha))