From 58aeea18a268734f94f480b5c87afd3ccec876f2 Mon Sep 17 00:00:00 2001 From: jasonkuen Date: Tue, 31 May 2016 13:41:47 +0800 Subject: [PATCH] Channel-Wise RReLU --- lib/THNN/generic/RReLU.c | 42 +++++++++++++++++-------------- test.lua | 53 +++++++++++++++++++++------------------- 2 files changed, 52 insertions(+), 43 deletions(-) diff --git a/lib/THNN/generic/RReLU.c b/lib/THNN/generic/RReLU.c index 6410ee1c7..a4a13efa1 100644 --- a/lib/THNN/generic/RReLU.c +++ b/lib/THNN/generic/RReLU.c @@ -17,7 +17,7 @@ void THNN_(RReLU_updateOutput)( if (channelwise && train) { long bs, ks; - THIndex_t nOutputPlane; + long nOutputPlane; { long input_ndim = THTensor_(nDimension)(input); switch (input_ndim) @@ -39,7 +39,7 @@ void THNN_(RReLU_updateOutput)( ks = input->size[2] * input->size[3]; break; } - nOutputPlane = input->size[(input_ndim + 1) % 2] + nOutputPlane = input->size[(input_ndim + 1) % 2]; } // get default random generator if (inplace) @@ -51,14 +51,17 @@ void THNN_(RReLU_updateOutput)( real *input_data = THTensor_(data)(input); real *noise_data = THTensor_(data)(noise); if (!inplace) + { + THTensor_(resizeAs)(output, input); output_data = THTensor_(data)(output); + } THTensor *channel_noise = THTensor_(newWithSize1d)(nOutputPlane); real *channel_noise_data = THTensor_(data)(channel_noise); THIndex_t i, j, k; #pragma omp parallel for private(j) for (j = 0; j < nOutputPlane; ++j) - channel_noise_data[j] = (real)THRandom_uniform(generator, lower, upper) + channel_noise_data[j] = (real)THRandom_uniform(generator, lower, upper); #pragma omp parallel for private(j,k) for (i = 0; i < bs; ++i) { @@ -72,17 +75,22 @@ void THNN_(RReLU_updateOutput)( for (j = 0; j < nOutputPlane; ++j) { const real r = channel_noise_data[j]; - for (k = 0; k < ks; ++k) - if (inplace) - if n_input_data[k] <= 0 - { - n_input_data[k] = r * n_input_data[k]; - n_noise_data[k] = r; - } - else - n_noise_data[k] = 1; + for (k = 0; k < ks; ++k) + if (inplace) + if (n_input_data[k] <= 0) + { + n_input_data[k] = r * n_input_data[k]; + n_noise_data[k] = r; + } else - n_output_data[k] = (n_input_data[k] > 0) ? n_input_data[k] : r * n_input_data[k]; + n_noise_data[k] = 1; + else + n_output_data[k] = (n_input_data[k] > 0) ? n_input_data[k] : r * n_input_data[k]; + n_input_data += ks; + if (inplace) + n_noise_data += ks; + else + n_output_data += ks; } } if (inplace) @@ -172,7 +180,7 @@ void THNN_(RReLU_updateGradInput)( if (channelwise && !inplace) { long bs, ks; - THIndex_t nOutputPlane; + long nOutputPlane; { long input_ndim = THTensor_(nDimension)(input); switch (input_ndim) @@ -194,12 +202,12 @@ void THNN_(RReLU_updateGradInput)( ks = input->size[2] * input->size[3]; break; } - nOutputPlane = input->size[(input_ndim + 1) % 2] + nOutputPlane = input->size[(input_ndim + 1) % 2]; } - const real *output_data = output_data = THTensor_(data)(output); const real *input_data = THTensor_(data)(input); const real *gradOutput_data = THTensor_(data)(gradOutput); + THTensor_(resizeAs)(gradInput, input); real *gradInput_data = THTensor_(data)(gradInput); const real *noise_data = THTensor_(data)(noise); @@ -215,12 +223,10 @@ void THNN_(RReLU_updateGradInput)( { const real r = noise_data[j]; for (k = 0; k < ks; ++k) - { if (n_input_data[k] > 0) n_gradInput_data[k] = n_gradOutput_data[k]; else n_gradInput_data[k] = n_gradOutput_data[k] * r; - } n_input_data += ks; n_gradInput_data += ks; n_gradOutput_data += ks; diff --git a/test.lua b/test.lua index 3847166ed..0beba2ab8 100644 --- a/test.lua +++ b/test.lua @@ -484,32 +484,35 @@ function nntest.RReLU() for _,train in ipairs({true,false}) do -- test with separate output buffer and inplace for _,inplace in ipairs({false,true}) do - module = nn.RReLU(l, u, inplace) - if train then - module:training() - else - module:evaluate() - end - input = torch.rand(nframe, size, kW, kH) - 0.5 - input:storage()[1] = -1 - local original_input = input:clone() - local output = module:forward(input) - mytester:assert(output:sign():eq(original_input:sign()):all(), 'sign flipped forward ') - local gradOutput = torch.ones(output:size()) - local gradInput = module:backward(input, gradOutput) - mytester:assert(gradInput:gt(0):eq(input:ne(0)):all(), 'gradient ') - mytester:assert(gradInput:lt(1):eq(input:le(0)):all(), 'backward negative inputs ') - mytester:assert(gradInput:eq(1):eq(input:gt(0)):all(), 'backward positive inputs ') - if not train then - local err = gradInput[input:le(0)]:mean()-(module.lower+module.upper)/2 - mytester:assertlt(err, precision, 'error on gradient ') - end + -- test with channel-wise + for _,cw in ipairs({true,false}) do + module = nn.RReLU(l, u, inplace, cw) + if train then + module:training() + else + module:evaluate() + end + input = torch.rand(nframe, size, kW, kH) - 0.5 + input:storage()[1] = -1 + local original_input = input:clone() + local output = module:forward(input) + mytester:assert(output:sign():eq(original_input:sign()):all(), 'sign flipped forward ') + local gradOutput = torch.ones(output:size()) + local gradInput = module:backward(input, gradOutput) + mytester:assert(gradInput:gt(0):eq(input:ne(0)):all(), 'gradient ') + mytester:assert(gradInput:lt(1):eq(input:le(0)):all(), 'backward negative inputs ') + mytester:assert(gradInput:eq(1):eq(input:gt(0)):all(), 'backward positive inputs ') + if not train then + local err = gradInput[input:le(0)]:mean()-(module.lower+module.upper)/2 + mytester:assertlt(err, precision, 'error on gradient ') + end - input = -torch.rand(1000) - module:forward(input) -- fill internal noise tensor - local g = module:backward(input, torch.ones(1000)) - local err = math.abs(g[input:le(0)]:mean()-(module.lower+module.upper)/2) - mytester:assertlt(err, 0.05, 'mean deviation of gradient for negative inputs ') + input = -torch.rand(1000) + module:forward(input) -- fill internal noise tensor + local g = module:backward(input, torch.ones(1000)) + local err = math.abs(g[input:le(0)]:mean()-(module.lower+module.upper)/2) + mytester:assertlt(err, 0.05, 'mean deviation of gradient for negative inputs ') + end end end end