ensemble_predictions.py

"""
Given a set of validation predictions, this script computes the optimal linear weights on the validation set.
It computes the weighted blend of test predictions, where some models are replaced by their bagged versions.
"""

import os
import numpy as np
import sys

import theano
import theano.tensor as T
import scipy

import data
import utils
import nn_plankton


CONFIGS = ['convroll4_doublescale_fs5', 'cp8', 'convroll4_big_wd_maxout512',
           'triplescale_fs2_fs5', 'cr4_ds', 'convroll5_preinit_resume_drop@420',
           'doublescale_fs5_latemerge_2233', 'convroll_all_broaden_7x7_weightdecay', 'convroll4_1024_lesswd',
           'convroll4_big_weightdecay']

BAGGED_CONFIGS = ['convroll4_doublescale_fs5', 'cp8', 'convroll4_big_wd_maxout512',
                  'cr4_ds', 'convroll5_preinit_resume_drop@420',
                  'convroll_all_broaden_7x7_weightdecay', 'convroll4_big_weightdecay']

# creating and checking the paths
n_models = len(CONFIGS)
valid_predictions_paths = []
for config in CONFIGS:
    p = 'predictions/valid--blend_featblend_%s--featblend_%s--avg-prob.npy' % (config, config)
    valid_predictions_paths.append(p)

test_predictions_paths = [p.replace('valid--', 'test--', 1) for p in valid_predictions_paths]
test_bagged_prediction_paths = []
for bagged_config in BAGGED_CONFIGS:
    bagged_p = 'predictions/bagged--test--blend_featblend_bagged_%s--avg-prob.npy' % bagged_config
    test_bagged_prediction_paths.append(bagged_p)
    for i in xrange(n_models):
        if bagged_config in test_predictions_paths[i]:
            test_predictions_paths[i] = bagged_p

test_unbagged_prediction_paths = [p for p in test_predictions_paths if 'bagged' not in p]

missing_predictions = []
for path in valid_predictions_paths + test_bagged_prediction_paths + test_unbagged_prediction_paths:
    if not os.path.isfile(path):
        missing_predictions.append(path)

if missing_predictions:
    print '\tPlease generate the following predictions:\n\t%s' % '\n\t'.join(missing_predictions)
    sys.exit(0)

# loading validation predictions
s = np.load("validation_split_v1.pkl")
t_valid = data.labels_train[s['indices_valid']]

predictions_list = [np.load(path) for path in valid_predictions_paths]
predictions_stack = np.array(predictions_list).astype(theano.config.floatX)  # num_sources x num_datapoints x 121

print "Individual prediction errors"
individual_prediction_errors = [utils.log_loss(p, t_valid) for p in predictions_list]
del predictions_list
for i in xrange(n_models):
    print individual_prediction_errors[i], os.path.basename(valid_predictions_paths[i])
print

# optimizing weights
X = theano.shared(predictions_stack)  # source predictions
t = theano.shared(utils.one_hot(t_valid))  # targets
W = T.vector('W')

s = T.nnet.softmax(W).reshape((W.shape[0], 1, 1))
weighted_avg_predictions = T.sum(X * s, axis=0)  # T.tensordot(X, s, [[0], [0]])
error = nn_plankton.log_loss(weighted_avg_predictions, t)
grad = T.grad(error, W)

f = theano.function([W], error)
g = theano.function([W], grad)

w_init = np.zeros(n_models, dtype=theano.config.floatX)
out, loss, _ = scipy.optimize.fmin_l_bfgs_b(f, w_init, fprime=g, pgtol=1e-09, epsilon=1e-08, maxfun=10000)

weights = np.exp(out)
weights /= weights.sum()

print 'Optimal weights'
for i in xrange(n_models):
    print weights[i], os.path.basename(valid_predictions_paths[i])
print

print 'Generating test set predictions'
predictions_list = [np.load(path) for path in test_predictions_paths]
predictions_stack = np.array(predictions_list)  # num_sources x num_datapoints x 121
del predictions_list

target_path = 'predictions/weighted_blend.npy'
weighted_predictions = np.sum(predictions_stack * weights.reshape((weights.shape[0], 1, 1)), axis=0)
np.save(target_path, weighted_predictions)
print ' stored in %s' % target_path