added files for paper results

classifier/linear_model.py

@@ -189,6 +189,36 @@ class MultiClassPerceptron:
         return y_test
 
 
+    def get_class_scores(self, X_test: list):
+        """
+        This function takes testing instances as parameters and returns the probability for each
+         predicted label.
+
+        
+        :param X_test: list of test data instances
+        :return: list of predicted label probabilities
+        """
+
+        if X_test is None or len(X_test) <= 0:
+            raise Exception('Testing Data cannot be empty')
+
+        print('Predicting..... ')
+
+        y_test = []
+        labels = list(self.perceptron_dict.keys())
+        for test_inst in X_test:
+            perceptron_scores = []  # list for storing perceptron scores for each label
+            for label in labels:
+                perceptron_scores.append(self.perceptron_dict[label].score(test_inst.features))
+            # find the max score from the list of scores
+            #max_score = max(perceptron_scores)
+
+            #label_max_score = labels[perceptron_scores.index(max_score)]
+            y_test.append(perceptron_scores)
+
+        return y_test
+
+
 def get_sample_weights_with_features(theta_bias: float = 0.0, random_state: int = 42):
     """
     This function creates a dictionary with feature as a key and a random floating number (feature weight) as value.

classifier/nn_ff.py

@@ -4,14 +4,16 @@ Date: July 5th, 2020
 """
 
 import torch
-from utils.nn_reader import read_csv_nn
-
+from utils.nn_reader import read_csv_nn,
+from utils.nn_reader2 import read_csv_nn_dev
+from sklearn.metrics import confusion_matrix
+import pandas as pd
 
 class Feedforward(torch.nn.Module):
         """
         Creates and trains a basic feedforward neural network.
         """
-
+        #
         def __init__(self, input_size, hidden_size, output_size):
             """ Sets up all basic elements of NN. """
             super(Feedforward, self).__init__()
@@ -19,16 +21,19 @@ class Feedforward(torch.nn.Module):
             self.hidden_size  = hidden_size
             self.output_size = output_size
             self.fc1 = torch.nn.Linear(self.input_size, self.hidden_size)
-            self.relu = torch.nn.ReLU()
+            #self.relu = torch.nn.ReLU()
+            self.tanh = torch.nn.Tanh()
             self.fc2 = torch.nn.Linear(self.hidden_size, self.output_size)
             self.sigmoid = torch.nn.Sigmoid()
             self.softmax = torch.nn.Softmax(dim=1)
-        
+        #
         def forward(self, x):
             """ Computes output from a given input x. """
             hidden = self.fc1(x)
-            relu = self.relu(hidden)
-            output = self.fc2(relu)
+            #relu = self.relu(hidden)
+            tanh = self.tanh(hidden)
+            #output = self.fc2(relu)
+            output = self.fc2(tanh)
             output = self.softmax(output)
             return output
 
@@ -52,24 +57,77 @@ if __name__=='__main__':
     y_pred = model(X_train)
     before_train = criterion(y_pred, y_train)
     print('Test loss before training' , before_train.item())
+    
+    l = X_train.shape[0]
+    batch_indices = list(zip(list(range(0,l,16))[:-1], list(range(16,l,16))))# + [(l-l%16,l)]
+    batch_indices[-1] = (batch_indices[-1][0], l)
 
+    train model
     model.train()
-    epoch = 2000
-    for epoch in range(epoch):
-        optimizer.zero_grad()
-        # forward pass
-        y_pred = model(X_train)
-        loss = criterion(y_pred, y_train)
-       
-        print('Epoch {}: train loss: {}'.format(epoch, loss.item()))
-        # backward pass
-        loss.backward()
-        optimizer.step()
+    epochs = 50
+    for epoch in range(epochs):
+        batch = 0
+    	for a,b in batch_indices:
+	        optimizer.zero_grad()
+	        # forward pass
+	        y_pred = model(X_train[a:b])
+	        loss = criterion(y_pred, y_train[a:b])
+	        #
+	        print('Epoch {}, batch {}: train loss: {}'.format(epoch, batch, loss.item()))
+	        # backward pass
+	        loss.backward()
+	        optimizer.step()
+	        batch += 1
+	    y_pred = model(X_train)
+	    loss = criterion(y_pred, y_train)
+	    print('Epoch {}: train loss: {}'.format(epoch, loss.item()))
+	    # shuffle dataset
+	    p = torch.randperm(l)
+	    X_train = X_train[p]
+	    y_train = y_train[p]
 
     model.eval()
-    y_pred = model(X_train)
+    y_pred = model.forward(X_train)
     after_train = criterion(y_pred, y_train) 
     print('Training loss after training' , after_train.item())
 
+    ## reload the data to get original order
+    #X_train, y_train, X_test = read_csv_nn()
+    #X_train = torch.FloatTensor(X_train)
+    #X_test = torch.FloatTensor(X_test)
+    #y_train_ = torch.FloatTensor(y_train)
+    #y_train = torch.max(torch.FloatTensor(y_train_),1)[1]
+    
+    # get dev set to make predictions
+    X_dev, y_dev = read_csv_nn_dev()
+    X_dev = torch.FloatTensor(X_dev)
+    y_dev_pre = torch.FloatTensor(y_dev)
+    y_dev = torch.max(torch.FloatTensor(y_dev_pre),1)[1]
+
+    # post-process to get predictions & get back to np format
+    y_pred = model.forward(X_dev)
+    y_pred_np = y_pred.detach().numpy()
+    predmax = np.amax(y_pred_np, axis=1)
+    preds = 1*(y_pred_np[:,1]==predmax) + 2*(y_pred_np[:,2]==predmax)
+    y_dev_ = y_dev.detach().numpy()
+    
+    # create confusion matrix
+    cm = confusion_matrix(y_dev_, preds)
+    print(cm)
+
+    # save predictions
+    df = pd.DataFrame()
+    df['preds'] = preds
+    df['true']  = y_dev_
+    probs = y_pred.detach().numpy()
+    df['pr0']  = probs[:,0]
+    df['pr1']  = probs[:,1]
+    df['pr2']  = probs[:,2]
+    df['correct'] = df.preds==df.true
+    df.to_csv('/Users/iriley/code/machine_learning/lab2020/preds_ffnn.csv')
+
+
+
+
 
 

testing/save_dev_results.py

@@ -0,0 +1,62 @@
+import os
+from classifier.linear_model import MultiClassPerceptron
+from sklearn.metrics import confusion_matrix as cm
+from utils.csv import read_csv_file
+from eval.metrics import f1_score
+import utils.constants as const
+import pandas as pd
+import numpy as np
+
+
+train_file_path = '/Users/iriley/code/citation-analysis/data/tsv/train.tsv'
+dev_file_path   = '/Users/iriley/code/citation-analysis/data/tsv/dev.tsv'
+
+
+# Read the training dataset
+X_train_inst = read_csv_file(train_file_path, '\t')
+
+# set of labels from Training data
+labels = set([inst.true_label for inst in X_train_inst])
+
+# Read test data set
+X_dev_inst = read_csv_file(dev_file_path, '\t')
+
+# number of training iterations
+epochs = 50
+
+# create MultiClassPerceptron classifier object
+clf = MultiClassPerceptron(epochs=epochs, learning_rate=0.5, random_state=101)
+
+# train the model
+clf.fit(X_train=X_train_inst, labels=list(labels))
+
+# predict
+y_pred = clf.predict(X_dev_inst)
+y_scores = np.array(clf.get_class_scores(X_dev_inst))
+
+y_true = [inst.true_label for inst in X_dev_inst]
+
+labeldict = {'background': 0, 'method': 1, 'result': 2}
+y_pred = np.array([labeldict[x] for x in y_pred])
+y_true = np.array([labeldict[x] for x in y_true])
+
+conmat = cm(y_true, y_pred)
+
+df = pd.DataFrame()
+df['pred'] = y_pred
+df['true'] = y_true
+df['correct'] = y_pred==y_true
+df['score0'] = np.round(y_scores[:,0],3)
+df['score1'] = np.round(y_scores[:,1],3)
+df['score2'] = np.round(y_scores[:,2],3)
+
+df.to_csv('/Users/iriley/code/machine_learning/lab2020/preds_perceptron.csv', index=False)
+
+## Model Evaluation
+#f1_score_micro = f1_score(y_true, y_pred, labels, const.AVG_MICRO)
+#f1_score_macro = f1_score(y_true, y_pred, labels, const.AVG_MACRO)
+#f1_score_none  = f1_score(y_true, y_pred, labels, None)
+
+## Print F1 Score
+#for result in f1_score_micro + f1_score_macro + f1_score_none:
+#    result.print_result()

utils/nn_reader.py

@@ -47,3 +47,4 @@ def read_csv_nn(scicite_dir=None):
 	return X_train, y_train, X_test
 
 
+

utils/nn_reader2.py

@@ -0,0 +1,33 @@
+import numpy as np
+from itertools import chain
+from utils.csv import read_csv_file
+
+
+def read_csv_nn_dev(scicite_dir=None):
+
+	dev_file_path = 'data/tsv/dev.tsv'
+	dev_raw = read_csv_file(dev_file_path, '\t')
+
+	features = [x.features for x in dev_raw]
+	features_unique = list(set(chain.from_iterable(features)))
+	nobs = len(features)
+	nfeats = len(features_unique)
+
+	X_dev = np.zeros((nobs, nfeats))
+
+	for j in range(nfeats):
+		f = features_unique[j]
+		for i in range(nobs):
+			if f in features[i]:
+				X_dev[i,j] = 1
+
+	y_dev_raw = np.array([x.true_label for x in dev_raw])
+	y_unique = sorted(list(set(y_dev_raw)))
+	y_dim = len(y_unique)
+	y_dev = np.zeros((nobs,y_dim))
+
+	for j in range(y_dim):
+		y_dev[:,j] = y_dev_raw == y_unique[j]
+
+	return X_dev, y_dev
+