diff --git a/classifier/linear_model.py b/classifier/linear_model.py index 6c067b2..1fc4bdf 100644 --- a/classifier/linear_model.py +++ b/classifier/linear_model.py @@ -189,6 +189,36 @@ class MultiClassPerceptron: return y_test + def get_class_scores(self, X_test: list): + """ + This function takes testing instances as parameters and returns the probability for each + predicted label. + + + :param X_test: list of test data instances + :return: list of predicted label probabilities + """ + + if X_test is None or len(X_test) <= 0: + raise Exception('Testing Data cannot be empty') + + print('Predicting..... ') + + y_test = [] + labels = list(self.perceptron_dict.keys()) + for test_inst in X_test: + perceptron_scores = [] # list for storing perceptron scores for each label + for label in labels: + perceptron_scores.append(self.perceptron_dict[label].score(test_inst.features)) + # find the max score from the list of scores + #max_score = max(perceptron_scores) + + #label_max_score = labels[perceptron_scores.index(max_score)] + y_test.append(perceptron_scores) + + return y_test + + def get_sample_weights_with_features(theta_bias: float = 0.0, random_state: int = 42): """ This function creates a dictionary with feature as a key and a random floating number (feature weight) as value. diff --git a/classifier/nn_ff.py b/classifier/nn_ff.py index 1177c51..21b050c 100644 --- a/classifier/nn_ff.py +++ b/classifier/nn_ff.py @@ -4,14 +4,16 @@ Date: July 5th, 2020 """ import torch -from utils.nn_reader import read_csv_nn - +from utils.nn_reader import read_csv_nn, +from utils.nn_reader2 import read_csv_nn_dev +from sklearn.metrics import confusion_matrix +import pandas as pd class Feedforward(torch.nn.Module): """ Creates and trains a basic feedforward neural network. """ - + # def __init__(self, input_size, hidden_size, output_size): """ Sets up all basic elements of NN. """ super(Feedforward, self).__init__() @@ -19,16 +21,19 @@ class Feedforward(torch.nn.Module): self.hidden_size = hidden_size self.output_size = output_size self.fc1 = torch.nn.Linear(self.input_size, self.hidden_size) - self.relu = torch.nn.ReLU() + #self.relu = torch.nn.ReLU() + self.tanh = torch.nn.Tanh() self.fc2 = torch.nn.Linear(self.hidden_size, self.output_size) self.sigmoid = torch.nn.Sigmoid() self.softmax = torch.nn.Softmax(dim=1) - + # def forward(self, x): """ Computes output from a given input x. """ hidden = self.fc1(x) - relu = self.relu(hidden) - output = self.fc2(relu) + #relu = self.relu(hidden) + tanh = self.tanh(hidden) + #output = self.fc2(relu) + output = self.fc2(tanh) output = self.softmax(output) return output @@ -52,24 +57,77 @@ if __name__=='__main__': y_pred = model(X_train) before_train = criterion(y_pred, y_train) print('Test loss before training' , before_train.item()) + + l = X_train.shape[0] + batch_indices = list(zip(list(range(0,l,16))[:-1], list(range(16,l,16))))# + [(l-l%16,l)] + batch_indices[-1] = (batch_indices[-1][0], l) + train model model.train() - epoch = 2000 - for epoch in range(epoch): - optimizer.zero_grad() - # forward pass - y_pred = model(X_train) - loss = criterion(y_pred, y_train) - - print('Epoch {}: train loss: {}'.format(epoch, loss.item())) - # backward pass - loss.backward() - optimizer.step() + epochs = 50 + for epoch in range(epochs): + batch = 0 + for a,b in batch_indices: + optimizer.zero_grad() + # forward pass + y_pred = model(X_train[a:b]) + loss = criterion(y_pred, y_train[a:b]) + # + print('Epoch {}, batch {}: train loss: {}'.format(epoch, batch, loss.item())) + # backward pass + loss.backward() + optimizer.step() + batch += 1 + y_pred = model(X_train) + loss = criterion(y_pred, y_train) + print('Epoch {}: train loss: {}'.format(epoch, loss.item())) + # shuffle dataset + p = torch.randperm(l) + X_train = X_train[p] + y_train = y_train[p] model.eval() - y_pred = model(X_train) + y_pred = model.forward(X_train) after_train = criterion(y_pred, y_train) print('Training loss after training' , after_train.item()) + ## reload the data to get original order + #X_train, y_train, X_test = read_csv_nn() + #X_train = torch.FloatTensor(X_train) + #X_test = torch.FloatTensor(X_test) + #y_train_ = torch.FloatTensor(y_train) + #y_train = torch.max(torch.FloatTensor(y_train_),1)[1] + + # get dev set to make predictions + X_dev, y_dev = read_csv_nn_dev() + X_dev = torch.FloatTensor(X_dev) + y_dev_pre = torch.FloatTensor(y_dev) + y_dev = torch.max(torch.FloatTensor(y_dev_pre),1)[1] + + # post-process to get predictions & get back to np format + y_pred = model.forward(X_dev) + y_pred_np = y_pred.detach().numpy() + predmax = np.amax(y_pred_np, axis=1) + preds = 1*(y_pred_np[:,1]==predmax) + 2*(y_pred_np[:,2]==predmax) + y_dev_ = y_dev.detach().numpy() + + # create confusion matrix + cm = confusion_matrix(y_dev_, preds) + print(cm) + + # save predictions + df = pd.DataFrame() + df['preds'] = preds + df['true'] = y_dev_ + probs = y_pred.detach().numpy() + df['pr0'] = probs[:,0] + df['pr1'] = probs[:,1] + df['pr2'] = probs[:,2] + df['correct'] = df.preds==df.true + df.to_csv('/Users/iriley/code/machine_learning/lab2020/preds_ffnn.csv') + + + + diff --git a/testing/save_dev_results.py b/testing/save_dev_results.py new file mode 100644 index 0000000..9e91d6c --- /dev/null +++ b/testing/save_dev_results.py @@ -0,0 +1,62 @@ +import os +from classifier.linear_model import MultiClassPerceptron +from sklearn.metrics import confusion_matrix as cm +from utils.csv import read_csv_file +from eval.metrics import f1_score +import utils.constants as const +import pandas as pd +import numpy as np + + +train_file_path = '/Users/iriley/code/citation-analysis/data/tsv/train.tsv' +dev_file_path = '/Users/iriley/code/citation-analysis/data/tsv/dev.tsv' + + +# Read the training dataset +X_train_inst = read_csv_file(train_file_path, '\t') + +# set of labels from Training data +labels = set([inst.true_label for inst in X_train_inst]) + +# Read test data set +X_dev_inst = read_csv_file(dev_file_path, '\t') + +# number of training iterations +epochs = 50 + +# create MultiClassPerceptron classifier object +clf = MultiClassPerceptron(epochs=epochs, learning_rate=0.5, random_state=101) + +# train the model +clf.fit(X_train=X_train_inst, labels=list(labels)) + +# predict +y_pred = clf.predict(X_dev_inst) +y_scores = np.array(clf.get_class_scores(X_dev_inst)) + +y_true = [inst.true_label for inst in X_dev_inst] + +labeldict = {'background': 0, 'method': 1, 'result': 2} +y_pred = np.array([labeldict[x] for x in y_pred]) +y_true = np.array([labeldict[x] for x in y_true]) + +conmat = cm(y_true, y_pred) + +df = pd.DataFrame() +df['pred'] = y_pred +df['true'] = y_true +df['correct'] = y_pred==y_true +df['score0'] = np.round(y_scores[:,0],3) +df['score1'] = np.round(y_scores[:,1],3) +df['score2'] = np.round(y_scores[:,2],3) + +df.to_csv('/Users/iriley/code/machine_learning/lab2020/preds_perceptron.csv', index=False) + +## Model Evaluation +#f1_score_micro = f1_score(y_true, y_pred, labels, const.AVG_MICRO) +#f1_score_macro = f1_score(y_true, y_pred, labels, const.AVG_MACRO) +#f1_score_none = f1_score(y_true, y_pred, labels, None) + +## Print F1 Score +#for result in f1_score_micro + f1_score_macro + f1_score_none: +# result.print_result() \ No newline at end of file diff --git a/utils/nn_reader.py b/utils/nn_reader.py index 647444d..77ed33e 100644 --- a/utils/nn_reader.py +++ b/utils/nn_reader.py @@ -47,3 +47,4 @@ def read_csv_nn(scicite_dir=None): return X_train, y_train, X_test + diff --git a/utils/nn_reader2.py b/utils/nn_reader2.py new file mode 100644 index 0000000..4cd1d36 --- /dev/null +++ b/utils/nn_reader2.py @@ -0,0 +1,33 @@ +import numpy as np +from itertools import chain +from utils.csv import read_csv_file + + +def read_csv_nn_dev(scicite_dir=None): + + dev_file_path = 'data/tsv/dev.tsv' + dev_raw = read_csv_file(dev_file_path, '\t') + + features = [x.features for x in dev_raw] + features_unique = list(set(chain.from_iterable(features))) + nobs = len(features) + nfeats = len(features_unique) + + X_dev = np.zeros((nobs, nfeats)) + + for j in range(nfeats): + f = features_unique[j] + for i in range(nobs): + if f in features[i]: + X_dev[i,j] = 1 + + y_dev_raw = np.array([x.true_label for x in dev_raw]) + y_unique = sorted(list(set(y_dev_raw))) + y_dim = len(y_unique) + y_dev = np.zeros((nobs,y_dim)) + + for j in range(y_dim): + y_dev[:,j] = y_dev_raw == y_unique[j] + + return X_dev, y_dev +