Fetch scores of CRF assigned tags from LSTM last layer

src/brat_to_conll.py

@@ -15,7 +15,6 @@ def get_start_and_end_offset_of_token_from_spacy(token):
 
 def get_sentences_and_tokens_from_spacy(text, spacy_nlp):
     document = spacy_nlp(text)
-    # sentences
     sentences = []
     for span in document.sents:
         sentence = [document[i] for i in range(span.start, span.end)]
@@ -66,10 +65,40 @@ def get_sentences_and_tokens_from_stanford(text, core_nlp):
         sentences.append(tokens)
     return sentences
 
-def get_entities_from_brat(text_filepath, annotation_filepath, verbose=False):
+def get_entities_from_brat(text_filepath, annotation_filepath, verbose=False, scores = False, indices = False):
     # load text
     with codecs.open(text_filepath, 'r', 'UTF-8') as f:
         text =f.read()
+
+    index_score_map = {}
+
+    if scores:
+        num_scores = len(scores)
+        num_indices = len(indices)
+        assert(num_scores == num_indices), "scores are not in sync with text!"
+
+        cum_scores = [-1]*num_scores
+        cum_scores[0] = scores[0]
+        for i in range(1, num_scores):
+            cum_scores[i] = cum_scores[i-1] + scores[i]
+
+        index_score_map = {}
+        i = 0
+        while i < num_scores:
+            st = indices[i][0]
+            j = i
+            while j < num_scores:
+                ed = indices[j][1]
+                try:
+                    tag_score = float(cum_scores[j] - cum_scores[i]  + scores[i]) / (j-i+1) # tag score, considering avg.
+                except:
+                    import pdb; pdb.set_trace()
+                key = "%s-%s" % (st, ed)
+                index_score_map[key] = tag_score
+                j += 1
+            i += 1            
+
+
     if verbose: print("\ntext:\n{0}\n".format(text))
 
     # parse annotation file
@@ -86,6 +115,11 @@ def get_entities_from_brat(text_filepath, annotation_filepath, verbose=False):
                 entity['start'] = int(anno[2])
                 entity['end'] = int(anno[3])
                 entity['text'] = ' '.join(anno[4:])
+                key = "%s-%s" % (anno[2], anno[3])
+                entity_score = index_score_map.get(key, "no score from model")
+                if entity_score != "no score from model":
+                    entity_score = float("{0:.3f}".format(entity_score))
+                entity['score'] = entity_score
                 if verbose:
                     print("entity: {0}".format(entity))
                 # Check compatibility between brat text and anootation

src/neuroner.py

@@ -1,5 +1,7 @@
 import matplotlib
+import gc
 matplotlib.use('Agg')
+from distutils import util
 import train
 import dataset as ds
 import tensorflow as tf
@@ -440,7 +442,6 @@ def fit(self):
 
     def predict(self, text):
         self.prediction_count += 1        
-
         if self.prediction_count == 1:
             self.parameters['dataset_text_folder'] = os.path.join('..', 'data', 'temp')
             self.stats_graph_folder, _ = self._create_stats_graph_folder(self.parameters)
@@ -468,14 +469,14 @@ def predict(self, text):
 
         # Predict labels and output brat
         output_filepaths = {}
-        prediction_output = train.prediction_step(self.sess, self.dataset, dataset_type, self.model, self.transition_params_trained, self.stats_graph_folder, self.prediction_count, self.parameters, self.dataset_filepaths)
-        _, _, output_filepaths[dataset_type] = prediction_output
+        prediction_output = train.prediction_step(self.sess, self.dataset, dataset_type, self.model, self.transition_params_trained, self.stats_graph_folder, self.prediction_count, self.parameters, self.dataset_filepaths,prediction_flag=True)
+        _, _, output_filepaths[dataset_type], _scores, _indices = prediction_output
         conll_to_brat.output_brat(output_filepaths, self.dataset_brat_folders, self.stats_graph_folder, overwrite=True)
 
         # Print and output result
         text_filepath = os.path.join(self.stats_graph_folder, 'brat', 'deploy', os.path.basename(dataset_brat_deploy_filepath))
         annotation_filepath = os.path.join(self.stats_graph_folder, 'brat', 'deploy', '{0}.ann'.format(utils.get_basename_without_extension(dataset_brat_deploy_filepath)))
-        text2, entities = brat_to_conll.get_entities_from_brat(text_filepath, annotation_filepath, verbose=True)
+        text2, entities = brat_to_conll.get_entities_from_brat(text_filepath, annotation_filepath, verbose=True, scores=_scores, indices=_indices)
         assert(text == text2)
         return entities
 
@@ -488,4 +489,4 @@ def close(self):
     def __del__(self):
         self.sess.close()
 
-
+gc.collect()

src/train.py

@@ -7,6 +7,8 @@
 import utils_tf
 import codecs
 import utils_nlp
+from sklearn.preprocessing import minmax_scale
+from sklearn.preprocessing import normalize
 #from tensorflow.python.tools.inspect_checkpoint import print_tensors_in_checkpoint_file
 
 def train_step(sess, dataset, sequence_number, model, parameters):
@@ -28,17 +30,21 @@ def train_step(sess, dataset, sequence_number, model, parameters):
                     feed_dict)
     return transition_params_trained
 
-def prediction_step(sess, dataset, dataset_type, model, transition_params_trained, stats_graph_folder, epoch_number, parameters, dataset_filepaths):
+def prediction_step(sess, dataset, dataset_type, model, transition_params_trained, stats_graph_folder, epoch_number, parameters, dataset_filepaths,prediction_flag=False):
     if dataset_type == 'deploy':
         print('Predict labels for the {0} set'.format(dataset_type))
     else:
         print('Evaluate model on the {0} set'.format(dataset_type))
     all_predictions = []
+    all_scores = []
+    all_indices = []
     all_y_true = []
     output_filepath = os.path.join(stats_graph_folder, '{1:03d}_{0}.txt'.format(dataset_type,epoch_number))
     output_file = codecs.open(output_filepath, 'w', 'UTF-8')
     original_conll_file = codecs.open(dataset_filepaths[dataset_type], 'r', 'UTF-8')
-
+    minmax_unary_scores = []
+    normalize_unary_scores = []
+    #assert(len(dataset.token_indices[dataset_type]) == 1),"lines more than 1!"
     for i in range(len(dataset.token_indices[dataset_type])):
         feed_dict = {
           model.input_token_indices: dataset.token_indices[dataset_type][i],
@@ -48,26 +54,38 @@ def prediction_step(sess, dataset, dataset_type, model, transition_params_traine
           model.dropout_keep_prob: 1.
         }
         unary_scores, predictions = sess.run([model.unary_scores, model.predictions], feed_dict)
+        nn_predictions = predictions.tolist()
         if parameters['use_crf']:
             predictions, _ = tf.contrib.crf.viterbi_decode(unary_scores, transition_params_trained)
             predictions = predictions[1:-1]
         else:
             predictions = predictions.tolist()
 
-        assert(len(predictions) == len(dataset.tokens[dataset_type][i]))
+        #assert(len(predictions) == len(dataset.tokens[dataset_type][i]))
         output_string = ''
         prediction_labels = [dataset.index_to_label[prediction] for prediction in predictions]
+        #minmax_unary_scores = minmax_scale(np.exp(unary_scores[1:-1]), axis=1)
+        normalize_unary_scores = normalize(np.exp(unary_scores[1:-1]), norm='l1', axis=1)
+        #normalize_unary_scores = normalize(minmax_unary_scores, norm='l1', axis=1)
+        #prediction_scores = np.amax(normalize_unary_scores, axis=1)
+        #my_predictions = np.argmax(normalize_unary_scores, axis=1)
+        prediction_scores = []
+        for ind, pred in enumerate(predictions):
+            prediction_scores.append(normalize_unary_scores[ind][pred])
+
         gold_labels = dataset.labels[dataset_type][i]
         if parameters['tagging_format'] == 'bioes':
             prediction_labels = utils_nlp.bioes_to_bio(prediction_labels)
             gold_labels = utils_nlp.bioes_to_bio(gold_labels)
+        indices = []
         for prediction, token, gold_label in zip(prediction_labels, dataset.tokens[dataset_type][i], gold_labels):
             while True:
                 line = original_conll_file.readline()
                 split_line = line.strip().split(' ')
                 if '-DOCSTART-' in split_line[0] or len(split_line) == 0 or len(split_line[0]) == 0:
                     continue
                 else:
+                    indices.append((int(split_line[2]), int(split_line[3])))
                     token_original = split_line[0]
                     if parameters['tagging_format'] == 'bioes':
                         split_line.pop()
@@ -79,6 +97,8 @@ def prediction_step(sess, dataset, dataset_type, model, transition_params_traine
         output_file.write(output_string+'\n')
 
         all_predictions.extend(predictions)
+        all_scores.extend(prediction_scores)
+        all_indices.extend(indices)
         all_y_true.extend(dataset.label_indices[dataset_type][i])
 
     output_file.close()
@@ -97,7 +117,10 @@ def prediction_step(sess, dataset, dataset_type, model, transition_params_traine
             new_y_pred, new_y_true, new_label_indices, new_label_names, _, _ = remap_labels(all_predictions, all_y_true, dataset, parameters['main_evaluation_mode'])
             print(sklearn.metrics.classification_report(new_y_true, new_y_pred, digits=4, labels=new_label_indices, target_names=new_label_names))
 
-    return all_predictions, all_y_true, output_filepath
+    if prediction_flag:
+    	return all_predictions, all_y_true, output_filepath, all_scores, all_indices
+    else:
+    	return all_predictions, all_y_true, output_filepath
 
 
 def predict_labels(sess, model, transition_params_trained, parameters, dataset, epoch_number, stats_graph_folder, dataset_filepaths):

src/utils_plots.py

@@ -33,7 +33,7 @@ def show_values(pc, fmt="%.2f", **kw):
     By HYRY
     '''
     pc.update_scalarmappable()
-    ax = pc.get_axes()
+    ax = pc.axes
     for p, color, value in zip(pc.get_paths(), pc.get_facecolors(), pc.get_array()):
         x, y = p.vertices[:-2, :].mean(0)
         if np.all(color[:3] > 0.5):