cavis/libnd4j/tests_cpu/layers_tests/ConvolutionTests1.cpp

/*******************************************************************************
 * Copyright (c) 2015-2018 Skymind, Inc.
 *
 * This program and the accompanying materials are made available under the
 * terms of the Apache License, Version 2.0 which is available at
 * https://www.apache.org/licenses/LICENSE-2.0.
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations
 * under the License.
 *
 * SPDX-License-Identifier: Apache-2.0
 ******************************************************************************/

//
// @author raver119@gmail.com
//

#ifndef LIBND4J_CONVOLUTIONTESTS1_H
#define LIBND4J_CONVOLUTIONTESTS1_H

#include "testlayers.h"
#include <NDArray.h>
#include <Context.h>
#include <Node.h>
#include <graph/Variable.h>
#include <graph/VariableSpace.h>
#include <ops/declarable/CustomOperations.h>
#include <ops/declarable/helpers/convolutions.h>
#include <ops/declarable/helpers/col2im.h>
#include <PointersManager.h>

using namespace nd4j;
using namespace nd4j::graph;

class ConvolutionTests1 : public testing::Test {
public:

};

template <typename T>
class TypedConvolutionTests1 : public testing::Test {
public:

};

typedef ::testing::Types<double, float> TestingTypes;
TYPED_TEST_CASE(TypedConvolutionTests1, TestingTypes);

//////////////////////////////////////////////////////////////////////
TYPED_TEST(TypedConvolutionTests1, conv2d_1) {

    int bS=1, iH=5,iW=4,  iC=2,oC=3,  kH=2,kW=2,  sH=1,sW=1,  pH=0,pW=0,  dH=1,dW=1;
    TypeParam _expB[]{664.0, 700.0, 736.0, 344.0, 808.0, 844.0, 880.0, 408.0, 952.0, 988.0, 1024.0, 472.0, 1096.0, 1132.0, 1168.0, 536.0, 466.0, 480.0, 494.0, 220.0, 1528.0, 1628.0, 1728.0, 856.0, 1928.0, 2028.0, 2128.0, 1048.0, 2328.0, 2428.0, 2528.0, 1240.0, 2728.0, 2828.0, 2928.0, 1432.0, 1346.0, 1392.0, 1438.0, 700.0, 2392.0, 2556.0, 2720.0, 1368.0, 3048.0, 3212.0, 3376.0, 1688.0, 3704.0, 3868.0, 4032.0, 2008.0, 4360.0, 4524.0, 4688.0, 2328.0, 2226.0, 2304.0, 2382.0, 1180.0};
    Nd4jLong _expS[]{4, 1, 3, 5, 4, 60, 20, 4, 1, typeid(TypeParam) == typeid(float) ? 8192 : 16384, 1, 99};
    auto input = NDArrayFactory::create_<TypeParam>('c', {bS, iC, iH, iW});
    auto weights = NDArrayFactory::create_<TypeParam>('c', {oC, iC, kH, kW});
    for (int e = 0; e < input->lengthOf(); e++)
        input->p(e, e + 1);

    for (int e = 0; e < weights->lengthOf(); e++)
        weights->p(e, e + 1);
    weights->permutei({2,3,1,0});

    // weights->printShapeInfo("weights");

    ArrayOptions::setDataType(_expS, input->dataType());
    auto exp = new NDArray(_expB, _expS);

    auto variableSpace = new VariableSpace();
    variableSpace->putVariable(-1, input);
    variableSpace->putVariable(-2, weights);

    auto block = new Context(1, variableSpace, false);  // not-in-place
    block->fillInputs({-1, -2});
    // 5,5 kernel
    block->getIArguments()->push_back(kH);
    block->getIArguments()->push_back(kW);

    // 1,1 stride
    block->getIArguments()->push_back(sH);
    block->getIArguments()->push_back(sW);

    // 0,0 padding
    block->getIArguments()->push_back(pH);
    block->getIArguments()->push_back(pW);

    // 1,1 dilation
    block->getIArguments()->push_back(dH);
    block->getIArguments()->push_back(dW);

    // same mode
    block->getIArguments()->push_back(1);

    // is NHWC
    block->getIArguments()->push_back(0);

    nd4j::ops::conv2d op;

    Nd4jStatus status = op.execute(block);
    ASSERT_EQ(ND4J_STATUS_OK, status);

    auto res = variableSpace->getVariable(1)->getNDArray();


    // checking output shape
    ASSERT_EQ(1, res->sizeAt(0));
    ASSERT_EQ(3, res->sizeAt(1));
    ASSERT_EQ(5, res->sizeAt(2));
    ASSERT_EQ(4, res->sizeAt(3));

    // basically the same as above
    ASSERT_TRUE(res->isSameShape(exp));
    // just for visual validation
    // exp->printIndexedBuffer("Expected");
    // res->printIndexedBuffer("Actual  ");
    // res->printShapeInfo("Result shape");
    // final check
    ASSERT_TRUE(res->equalsTo(exp));

    delete block;
    delete variableSpace;
    delete exp;
}

//////////////////////////////////////////////////////////////////////
TYPED_TEST(TypedConvolutionTests1, conv2d_2) {
    auto input = NDArrayFactory::create<TypeParam>('c', {1, 1, 1, 4});
    auto weights = NDArrayFactory::create<TypeParam>('c', {1, 1, 1, 4});
    auto exp = NDArrayFactory::create<TypeParam>('c', {1, 4, 1, 4}, {2., 4., 6., 8., 2., 4., 6., 8., 2., 4., 6., 8., 2., 4., 6., 8.});

    weights.assign(2.0);
    input.linspace(1);

    nd4j::ops::conv2d op;
    auto result = op.execute({&input, &weights}, {}, {1, 1, 1, 1, 0, 0, 1, 1, 0, 0});
    ASSERT_EQ(ND4J_STATUS_OK, result->status());

    auto z = result->at(0);

    ASSERT_TRUE(exp.isSameShape(z));
    ASSERT_TRUE(exp.equalsTo(z));

    delete result;
}

//////////////////////////////////////////////////////////////////////
TYPED_TEST(TypedConvolutionTests1, conv2d_3) {

    int bS=2, iH=4,iW=3,  iC=4,oC=3,  kH=3,kW=2,  sH=1,sW=1,  pH=0,pW=0,  dH=1,dW=1;
    int       oH=4,oW=3;
    int paddingMode = 1;             // 1-SAME, 0-VALID;
    int dataFormat  = 1;             // 1-NHWC, 0-NCHW

    auto input = NDArrayFactory::create<TypeParam>('c', {bS, iH, iW, iC});
    auto weights = NDArrayFactory::create<TypeParam>('c', {kH, kW, iC, oC});
    auto bias = NDArrayFactory::create<TypeParam>('c', {oC}, {1,2,3});


    auto expOutput = NDArrayFactory::create<TypeParam>('c', {bS, oH, oW, oC},{ 152. , 155.2, 158.4,152. , 155.2, 158.4, 66.4,  68. ,  69.6,170.4, 175.2, 180. ,170.4, 175.2, 180. , 70.8,  73.2,  75.6,
                                                      170.4, 175.2, 180. ,170.4, 175.2, 180. , 70.8,  73.2,  75.6, 75.2,  78.4,  81.6, 75.2,  78.4,  81.6, 28. ,  29.6,  31.2,
                                                      152. , 155.2, 158.4,152. , 155.2, 158.4, 66.4,  68. ,  69.6,170.4, 175.2, 180. ,170.4, 175.2, 180. , 70.8,  73.2,  75.6,
                                                      170.4, 175.2, 180. ,170.4, 175.2, 180. , 70.8,  73.2,  75.6, 75.2,  78.4,  81.6, 75.2,  78.4,  81.6, 28. ,  29.6,  31.2});
    input = 2.;
    weights.linspace(0.1, 0.1);

    nd4j::ops::conv2d op;
    auto results = op.execute({&input, &weights}, {}, {kH,kW,  sH,sW,  pH,pW,  dH,dW, paddingMode, dataFormat});
    auto output = results->at(0);

    ASSERT_EQ(Status::OK(), results->status());

    ASSERT_TRUE(expOutput.isSameShape(output));
    ASSERT_TRUE(expOutput.equalsTo(output));

    delete results;
}

//////////////////////////////////////////////////////////////////////
TYPED_TEST(TypedConvolutionTests1, conv2d_4) {

    int bS=2, iH=4,iW=3,  iC=4,oC=3,  kH=3,kW=2,  sH=1,sW=1,  pH=0,pW=0,  dH=1,dW=1;
    int       oH=2,oW=2;
    int paddingMode = 0;             // 1-SAME, 0-VALID;
    int dataFormat  = 1;             // 1-NHWC, 0-NCHW

    auto input = NDArrayFactory::create<TypeParam>('c', {bS, iH, iW, iC});
    auto weights = NDArrayFactory::create<TypeParam>('c', {kH, kW, iC, oC});
    auto bias = NDArrayFactory::create<TypeParam>('c', {oC}, {1,2,3});

    auto expOutput = NDArrayFactory::create<TypeParam>('c', {bS, oH, oW, oC},{ 170.4,175.20001,180.,170.4,175.20001,180.,170.4,175.20001,180.,170.4,175.20001,180.,170.4,175.20001,180.,170.4,175.20001,180.,170.4,175.20001,180.,170.4,175.20001,180.});

    input = 2.;
    weights.linspace(0.1, 0.1);

    nd4j::ops::conv2d op;
    auto results = op.execute({&input, &weights}, {}, {kH,kW,  sH,sW,  pH,pW,  dH,dW, paddingMode, dataFormat});
    auto output = results->at(0);

    ASSERT_EQ(Status::OK(), results->status());

    ASSERT_TRUE(expOutput.isSameShape(output));
    ASSERT_TRUE(expOutput.equalsTo(output));

    delete results;
}

//////////////////////////////////////////////////////////////////////
TYPED_TEST(TypedConvolutionTests1, conv2d_5) {

    int bS=2, iH=4,iW=3,  iC=4,oC=3,  kH=3,kW=2,  sH=1,sW=1,  pH=0,pW=0,  dH=1,dW=1;
    int       oH=2,oW=2;
    int paddingMode = 0;             // 1-SAME, 0-VALID;
    int dataFormat  = 0;             // 1-NHWC, 0-NCHW

    auto input    = NDArrayFactory::create<TypeParam>('c', {bS, iC, iH, iW});
    auto weights  = NDArrayFactory::create<TypeParam>('c', {oC, iC, kH, kW});
    auto bias     = NDArrayFactory::create<TypeParam>('c', {oC}, {1,2,3});

    auto expOutput = NDArrayFactory::create<TypeParam>('c', {bS, oC, oH, oW}, {61. ,  61. , 61. ,  61. ,177.2, 177.2,177.2, 177.2,293.4, 293.4,293.4, 293.4, 61. ,  61. , 61. ,  61. ,177.2, 177.2,177.2, 177.2,293.4, 293.4,293.4, 293.4});

    input = 2.;
    weights.linspace(0.1, 0.1);
    weights.permutei({2,3,1,0});

    nd4j::ops::conv2d op;
    auto results = op.execute({&input, &weights, &bias}, {}, {kH,kW,  sH,sW,  pH,pW,  dH,dW, paddingMode, dataFormat});
    auto output = results->at(0);

    // output->printIndexedBuffer();

    ASSERT_EQ(Status::OK(), results->status());

    ASSERT_TRUE(expOutput.isSameShape(output));
    ASSERT_TRUE(expOutput.equalsTo(output));

    delete results;
}

//////////////////////////////////////////////////////////////////////
TYPED_TEST(TypedConvolutionTests1, conv2d_6) {
    auto input = NDArrayFactory::create<TypeParam>('c', {54, 1, 12, 12});
    auto weights = NDArrayFactory::create<TypeParam>('c', {1, 2, 12, 2});

    nd4j::ops::conv2d op;
    auto result = op.execute({&input, &weights}, {}, {-1,-1,  1,1,  0,0,  1,1,  1,1});
    ASSERT_EQ(Status::OK(), result->status());

    delete result;
}

TYPED_TEST(TypedConvolutionTests1, TestAvgFF_TF) {
    auto input = NDArrayFactory::create<TypeParam>('c', {4, 10, 10, 3}, {9.37125111, 2.20166993,             2.91434479,             5.43639755,             -2.10573769,             4.08528662,             5.86908436,             -4.46203756,             2.21057916,             5.35849190,             0.01394637,             4.40566349,             7.07982206,             -0.09633455,             2.42429352,             3.97301817,             -1.89553940,             1.99690318,             6.33141708,             0.55401880,             1.70707977,             5.55204201,             -0.03513752,             1.60011971,             2.62700319,             -2.74582434,             3.06697464,             1.06277943,             -1.16075921,             -0.78095782,             9.72352791,             -1.22686064,             1.99644792,             7.35571337,             1.40607321,             0.11390255,             9.53334427,             2.28303599,             -1.66728830,             6.16678810,             -0.04532295,             -1.97708666,             9.74906158,             1.46223176,             -1.46734393,             4.30761862,             -1.23790228,             1.24823606,             6.13938427,             -3.83689475,             -1.19625473,             7.91535568,             6.05868721,             -3.22946382,             8.81633949,             -0.19967777,             0.66053957,             2.30919123,             0.74543846,             -0.39347672,             11.11058044,             0.53720862,             1.52645731,             5.70012379,             -1.15213466,             1.16451406,             7.00526333,             1.57362783,             -2.44384766,             5.54213285,             -1.98828590,             -0.70483637,             7.88281822,             -3.59875536,             0.80745387,             13.41578484,             -1.55507684,             -0.65855008,             9.32583523,             -0.14544789,             0.73436141,             3.61176538,             -1.71268058,             -2.58490300,             9.09280205,             -3.27405524,             -2.04569697,             4.44761324,             -0.62955856,             -2.61917663,             8.04890442,             0.54579324,             0.85929775,             9.82259560,             -1.93825579,             0.77703512,             4.67090321,             -4.79267597,             -2.38906908,             9.31265545,             0.96026313,             -1.14109385,             11.54231834,             -0.01417295,             -0.39500344,             8.49191666,             0.55300158,             2.79490185,             6.92466164,             1.72254205,             2.82222271,             8.83112717,             2.95033407,             2.18054962,             6.73509789,             -2.22272944,             0.51127720,             -1.04563558,             2.15747333,             -2.30959272,             9.55441570,             1.50396204,             1.77370787,             7.38146257,             -1.79076433,             3.20961165,             7.18864202,             2.91217351,             0.43018937,             7.11078024,             -1.17386127,             -0.16817921,             6.12327290,             -2.82205725,             3.30696845,             13.51291752,             -1.30856836,             -2.38332748,             11.09487438,             -1.47190213,             -0.53050828,             4.38285351,             -5.07309771,             1.50714362,             5.72274446,             -2.85825086,             -0.89673209,             3.73791552,             -0.67708802,             -4.13149452,             -0.00671843,             -0.26566532,             0.32961160,             7.14501762,             -1.41608179,             -4.96590328,             12.26205540,             -0.65158135,             -0.88641000,             6.95777559,             -0.79058206,             -0.10260171,             7.87169170,             1.35921454,             1.11759663,             5.461
    auto exp = NDArrayFactory::create<TypeParam>('c', {4, 4, 4, 3}, {7.97172260, 0.06878620,             2.27749538,             7.29276514,             -0.14074677,             0.65480286,             5.70313978,             -0.06546132,             0.35443667,             3.70382833,             -0.84020567,             0.63826996,             8.60301399,             -0.38236514,             1.55177069,             7.37542057,             -0.99374938,             -0.29971302,             8.84352493,             -0.67121059,             0.43132120,             4.78175592,             -1.25070143,             -1.91523600,             6.03855371,             -0.00292124,             -1.11214364,             7.90158176,             -0.57949901,             -0.96735370,             7.81192017,             -0.53255427,             -0.48009714,             3.16953635,             0.08353355,             -1.54299748,             3.74821687,             1.69396687,             0.72724354,             5.42915201,             -1.13686812,             -0.71793109,             5.78376389,             -0.72239977,             -0.60055625,             2.53636408,             0.56777251,             -2.07892323,             6.08064651,             0.68620735,             2.54017019,             5.65828180,             -0.68255502,             1.47283304,             6.10842514,             -0.39655915,             0.28380761,             1.96707797,             -1.98206317,             0.94027776,             4.71811438,             0.32104525,             -0.92409706,             8.34588146,             -1.05581069,             -0.55217457,             9.58440876,             -0.96549922,             0.45820439,             5.65453672,             -2.50953507,             -0.71441835,             8.03059578,             -0.21281289,             0.92125505,             9.26900673,             -0.35963219,             -0.70039093,             8.59924412,             -1.22358346,             0.81318003,             3.85920119,             -0.01305223,             -1.09234154,             6.33158875,             1.28094780,             -1.48926139,             4.94969177,             -0.77126902,             -1.97033751,             5.64381838,             -0.16285487,             -1.31277227,             2.39893222,             -1.32902908,             -1.39609122,             6.47572327,             -0.45267010,             1.55727172,             6.70965624,             -1.68735468,             -0.05672536,             7.25092363,             -0.64613032,             0.67050058,             3.60789680,             -2.05948973,             2.22687531,             8.15202713,             -0.70148355,             1.28314006,             8.14842319,             -1.88807654,             -1.04808438,             8.45500565,             -0.76425624,             0.94542569,             4.56179953,             -0.28786001,             -2.04502511,             8.46278095,             -0.31019822,             0.07339200,             9.34214592,             -0.61948007,             0.52481830,             8.32515621,             -1.52418160,             0.49678251,             5.11082315,             -1.09908783,             -0.52969611,             5.27806664,             0.88632923,             0.66754371,             4.75839233,             0.48928693,             -0.68036932,             6.56925392,             -0.02949905,             -2.99189186,             4.46320581,             -0.64534980,             -0.29516968,             8.60809517,             -1.13120568,             3.41720533,             5.84243155,             -1.24109328,             0.89566326,             5.99578333,             -0.42496428,             2.07076764,             3.17812920,             -0.81566459,             -0.14363396,             6.55184317,             0.39633346,             -0.43852386,             8.70214558,             -2.24613595,             0.30708700,             8.73882294,             -0.53545928,             1.54409575,             4.4945

    nd4j::ops::avgpool2d op;
    auto result = op.execute({&input}, {}, {3,3,  3,3,  0,0,  1,1,1,  0,1}, {});
    ASSERT_EQ(Status::OK(), result->status());

    auto z = result->at(0);

    // z->printIndexedBuffer("z");
    // exp.printIndexedBuffer("e");

    ASSERT_TRUE(exp.isSameShape(z));
    ASSERT_TRUE(exp.equalsTo(z));

    delete result;
}

//////////////////////////////////////////////////////////////////////
TYPED_TEST(TypedConvolutionTests1, conv2d_7) {

    int bS=1, iH=256,iW=256,  iC=1,oC=1,  kH=4,kW=3,  sH=1,sW=1,  pH=0,pW=0,  dH=1,dW=1;
    // int       oH=256,oW=256;
    int paddingMode = 1;             // 1-SAME, 0-VALID;
    int dataFormat  = 0;             // 1-NHWC, 0-NCHW

    auto input    = NDArrayFactory::create<TypeParam>('c', {bS, iC, iH, iW});
    auto weights  = NDArrayFactory::create<TypeParam>('c', {kH, kW, iC, oC});

    input = 5.;
    weights = 3.;

    nd4j::ops::conv2d op;
    auto results = op.execute({&input, &weights}, {}, {kH,kW,  sH,sW,  pH,pW,  dH,dW, paddingMode, dataFormat});
    auto* output = results->at(0);

    ASSERT_EQ(Status::OK(), results->status());

    delete results;
}

//////////////////////////////////////////////////////////////////////
TEST_F(ConvolutionTests1, sconv2d_1) {
    float _expB[] = {10025.0f,    10350.0f,    10675.0f,    11000.0f,    11325.0f,    11650.0f,    13275.0f,    13600.0f,    13925.0f,    14250.0f,    14575.0f,    14900.0f,    16525.0f,    16850.0f,    17175.0f,    17500.0f,    17825.0f,    18150.0f,    19775.0f,    20100.0f,    20425.0f,    20750.0f,    21075.0f,    21400.0f,    23025.0f,    23350.0f,    23675.0f,    24000.0f,    24325.0f,    24650.0f,    26275.0f,    26600.0f,    26925.0f,    27250.0f,    27575.0f,    27900.0f,    38775.0f,    40350.0f,    41925.0f,    43500.0f,    45075.0f,    46650.0f,    54525.0f,    56100.0f,    57675.0f,    59250.0f,    60825.0f,    62400.0f,    70275.0f,    71850.0f,    73425.0f,    75000.0f,    76575.0f,    78150.0f,    86025.0f,    87600.0f,    89175.0f,    90750.0f,    92325.0f,    93900.0f,   101775.0f,   103350.0f,   104925.0f,    106500.0f,   108075.0f,   109650.0f,   117525.0f,   119100.0f,   120675.0f,   122250.0f,    123825.0f,   125400.0f,    67525.0f,    70350.0f,    73175.0f,    76000.0f,    78825.0f,    81650.0f,    95775.0f,    98600.0f,   101425.0f,   104250.0f,   107075.0f,   109900.0f,    124025.0f,   126850.0f,   129675.0f,   132500.0f,   135325.0f,   138150.0f,   152275.0f,    155100.0f,   157925.0f,   160750.0f,   163575.0f,   166400.0f,   180525.0f,   183350.0f,    186175.0f,   189000.0f,   191825.0f,   194650.0f,   208775.0f,   211600.0f,   214425.0f,    217250.0f,   220075.0f,   222900.0f,   119400.0f,   120350.0f,   121300.0f,   122250.0f,    123200.0f,   124150.0f,   128900.0f,   129850.0f,   130800.0f,   131750.0f,   132700.0f,    133650.0f,   138400.0f,   139350.0f,   140300.0f,   141250.0f,   142200.0f,   143150.0f,    147900.0f,   148850.0f,   149800.0f,   150750.0f,   151700.0f,   152650.0f,   157400.0f,    158350.0f,   159300.0f,   160250.0f,   161200.0f,   162150.0f,   166900.0f,   167850.0f,    168800.0f,   169750.0f,   170700.0f,   171650.0f,   273150.0f,   275350.0f,   277550.0f,    279750.0f,   281950.0f,   284150.0f,   295150.0f,   297350.0f,   299550.0f,   301750.0f,    303950.0f,   306150.0f,   317150.0f,   319350.0f,   321550.0f,   323750.0f,   325950.0f,    328150.0f,   339150.0f,   341350.0f,   343550.0f,   345750.0f,   347950.0f,   350150.0f,    361150.0f,   363350.0f,   365550.0f,   367750.0f,   369950.0f,   372150.0f,   383150.0f,    385350.0f,   387550.0f,   389750.0f,   391950.0f,   394150.0f,   426900.0f,   430350.0f,    433800.0f,   437250.0f,   440700.0f,   444150.0f,   461400.0f,   464850.0f,   468300.0f,    471750.0f,   475200.0f,   478650.0f,   495900.0f,   499350.0f,   502800.0f,   506250.0f,    509700.0f,   513150.0f,   530400.0f,   533850.0f,   537300.0f,   540750.0f,   544200.0f,    547650.0f,   564900.0f,   568350.0f,   571800.0f,   575250.0f,   578700.0f,   582150.0f,    599400.0f,   602850.0f,   606300.0f,   609750.0f,   613200.0f,   616650.0f,    75025.0f,    75350.0f,    75675.0f,    76000.0f,    76325.0f,    76650.0f,    78275.0f,    78600.0f,    78925.0f,    79250.0f,    79575.0f,    79900.0f,    81525.0f,    81850.0f,    82175.0f,    82500.0f,    82825.0f,    83150.0f,    84775.0f,    85100.0f,    85425.0f,    85750.0f,    86075.0f,    86400.0f,    88025.0f,    88350.0f,    88675.0f,    89000.0f,    89325.0f,    89650.0f,    91275.0f,    91600.0f,    91925.0f,    92250.0f,    92575.0f,    92900.0f,    353775.0f,   355350.0f,   356925.0f,   358500.0f,   360075.0f,   361650.0f,   369525.0f,    371100.0f,   372675.0f,   374250.0f,   375825.0f,   377400.0f,   385275.0f,   386850.0f,    388425.0f,   390000.0f,   391575.0f,   393150.0f,   401025.0f,   402600.0f,   404175.0f,    405750.0f,   407325.0f,   408900.0f,   416775.0f,   418350.0f,   419925.0f,   421500.0f,    423075.0f,   424650.0f,   432525.0f,   434100.0f,   435675.0f,   437250.0f,   438825.0f,    440400.0f,   632525.0f,   635350.0f,   638175.0f,   641000.0f,   643825.0f,   646650.0f,    660775.0f,   663600.0f,   666425.0f,   669250.0f,   672075.0f,   674900.0f,   689025.0f,    691850.0f,   694675.0f,   697500.0f,   700325.0f,   703150.0f,   717275.0f,   720100.0f,    722925.0f,   725750.0f,   728575.0f,   73
    Nd4jLong _expS[] = {4, 2, 6, 6, 6, 144, 36, 6, 1, 8192, 1, 99};
    NDArray exp(_expB, _expS);

    int sY = 1;
    int sX = 1;
    int pY = 0;
    int pX = 0;
    int iC = 2;
    int oC = 3;
    int kY = 5;
    int kX = 5;
    int iY = 10;
    int iX = 10;
    int B = 2;

    auto input = NDArrayFactory::create_<float>('c', {B, iC, iY, iX});
    for (int e = 0; e < input->lengthOf(); e++)
        input->p(e, e+1);

    auto weights = NDArrayFactory::create_<float>('c', {oC, iC, kY, kX});
    for (int e = 0; e < weights->lengthOf(); e++)
        weights->p(e, e+1);
    weights->permutei({2,3,1,0});

    auto variableSpace = new VariableSpace();
    variableSpace->putVariable(-1, input);
    variableSpace->putVariable(-2, weights);

    auto block = new Context(1, variableSpace, false);
    block->fillInputs({-1, -2});

    block->getIArguments()->push_back(kY);
    block->getIArguments()->push_back(kX);

    block->getIArguments()->push_back(sY);
    block->getIArguments()->push_back(sX);

    block->getIArguments()->push_back(pY);
    block->getIArguments()->push_back(pX);

    // dilation
    block->getIArguments()->push_back(1);
    block->getIArguments()->push_back(1);

    // NOT same mode
    block->getIArguments()->push_back(0);

    nd4j::ops::sconv2d op;

    Nd4jStatus status = op.execute(block);

    ASSERT_EQ(ND4J_STATUS_OK, status);
    auto output = variableSpace->getVariable(1)->getNDArray();

    //exp.printShapeInfo("Expected shape");
    //output->printShapeInfo("Result shape");
    ASSERT_TRUE(exp.isSameShape(output));

        //exp.printBuffer("Expctd buffer");
    //output->printBuffer("Result buffer");
    ASSERT_TRUE(exp.equalsTo(output));

    delete block;
    delete variableSpace;
}

//////////////////////////////////////////////////////////////////////
TYPED_TEST(TypedConvolutionTests1, sconv2d_2) {
    TypeParam _expBFF[] = {108.9405008,  109.5920008,  110.2435008,  110.8950008,   111.5465008,  112.1980008,  115.4555008,  116.1070008,   116.7585008,  117.410000,   118.061500,   118.7130009,   121.9705009,  122.6220009,  123.2735009,  123.9250009,   124.5765009,  125.2280009,  128.4855009,  129.1370009,   129.7885009,  130.4400009,  131.09150,    131.74300,   135.0005010,  135.6520010,  136.3035010,  136.9550010,   137.6065010,  138.2580010,  141.5155010,  142.1670010,   142.8185010,  143.4700010,  144.1215010,  144.7730010,   248.9617514,  250.670751,   252.3797515,  254.0887515,   255.7977515,  257.5067515,  266.0517515,  267.7607515,   269.469751,   271.1787516,  272.8877516,  274.5967516,   283.1417516,  284.8507516,  286.5597516,  288.268751,   289.9777517,  291.6867517,  300.2317517,  301.9407517,   303.6497517,  305.3587517,  307.067751,   308.7767518,   317.3217518,  319.0307518,  320.7397518,  322.4487518,   324.157751,   325.866751,   334.4117519,  336.1207519,   337.8297519,  339.5387519,  341.2477519,  342.95675,   388.9829964,  391.7494964,  394.5159964,  397.2824964,   400.048996,   402.8154963,  416.647996,   419.4144962,   422.1809962,  424.9474962,  427.7139962,  430.4804962,   444.3129961,  447.0794961,  449.8459961,  452.6124960,   455.3789960,  458.1454960,  471.9779959,  474.7444959,   477.5109959,  480.2774959,  483.0439959,  485.8104958,   499.6429958,  502.4094957,  505.1759957,  507.9424957,   510.7089957,  513.4754957,  527.3079956,  530.0744956,   532.8409956,  535.607495,   538.3739955,  541.1404955,   529.0042487,  532.8282487,  536.6522487,  540.4762487,   544.3002487,  548.1242487,  567.2442487,  571.068248,   574.892248,   578.716248,   582.540248,   586.3642486,   605.4842486,  609.3082486,  613.1322486,  616.9562486,   620.7802486,  624.6042486,  643.7242486,  647.5482486,   651.3722486,  655.1962486,  659.0202486,  662.8442486,   681.9642486,  685.7882486,  689.6122486,  693.4362486,   697.2602486,  701.0842486,  720.2042486,  724.0282486,   727.852248,   731.676248,   735.500248,   739.324248,   669.0255044,  673.9070044,  678.7885044,  683.6700044,   688.5515044,  693.4330044,  717.8405044,  722.7220044,   727.6035044,  732.4850044,  737.3665044,  742.2480044,   766.6555043,  771.5370043,  776.4185043,  781.3000043,   786.1815043,  791.0630043,  815.4705043,  820.3520043,   825.2335043,  830.1150043,  834.9965043,  839.8780043,   864.2855042,  869.1670042,  874.0485042,  878.9300042,   883.8115042,  888.6930042,  913.1005042,  917.9820042,   922.8635042,  927.7450042,  932.6265042,  937.5080042,   809.0467424,  814.9857424,  820.9247424,  826.8637423,   832.8027423,  838.7417423,  868.4367421,  874.3757421,   880.3147420,  886.2537420,  892.1927420,  898.13174,   927.8267418,  933.7657418,  939.7047417,  945.6437417,   951.5827417,  957.5217416,  987.2167415,  993.155741,   999.0947414, 1005.0337414, 1010.972741,  1016.9117413,   1046.6067412, 1052.5457411, 1058.4847411, 1064.4237411,   1070.3627410, 1076.3017410, 1105.996740,  1111.9357408,   1117.8747408, 1123.8137408, 1129.7527407, 1135.6917407,   949.0679815,  956.0644814,  963.060981,   970.0574813,   977.0539812,  984.0504811, 1019.0329807, 1026.0294807,   1033.0259806, 1040.0224805, 1047.0189804, 1054.0154804,   1088.9979800, 1095.9944799, 1102.9909798, 1109.987479,   1116.9839797, 1123.9804796, 1158.9629792, 1165.9594791,   1172.9559791, 1179.9524790, 1186.9489789, 1193.9454788,   1228.9279785, 1235.9244784, 1242.9209783, 1249.9174782,   1256.913978,  1263.9104781, 1298.8929777, 1305.8894776,   1312.8859775, 1319.8824775, 1326.8789774, 1333.8754773,   1089.0892560, 1097.1432561, 1105.1972562, 1113.251256,   1121.3052563, 1129.3592564, 1169.6292568, 1177.6832568,   1185.7372569, 1193.7912570, 1201.845257,  1209.8992571,   1250.1692575, 1258.2232576, 1266.2772576, 1274.3312577,   1282.3852578, 1290.4392579, 1330.7092582, 1338.7632583,   1346.8172584, 1354.8712584, 1362.9252585, 1370.9792586,   1411.24925,   1419.3032590, 1427.3572591, 1435.4112592,   1443.465259,  1451.5192593, 1491.7892597, 1499.8432598,   1507.8972598, 1
    Nd4jLong _expSFF[] = {4, 2, 10, 6, 6, 360, 36, 6, 1, typeid(TypeParam) == typeid(float) ? 8192 : 16384, 1, 99,};
    NDArray expFF(_expBFF, _expSFF);

    auto input = NDArrayFactory::create<TypeParam>('c', {2, 3, 10, 10});
    auto weightsD = NDArrayFactory::create<TypeParam>('c', {5, 3, 5, 5});
    auto weightsP = NDArrayFactory::create<TypeParam>('c', {10, 15, 1, 1});

    input.linspace(1);
    weightsD.linspace(1);
    weightsP.linspace(1);
    weightsD.permutei({2,3,1,0});
    weightsP.permutei({2,3,1,0});

    input.applyScalar(scalar::Divide, 100.0);
    weightsD.applyScalar(scalar::Divide, 100.0);
    weightsP.applyScalar(scalar::Divide, 100.0);

    nd4j::ops::sconv2d op;

    auto resultFF = op.execute({&input, &weightsD, &weightsP}, {}, {5, 5, 1, 1, 0, 0, 1, 1, 0, 0}, {});

    auto z = resultFF->at(0);
    //z->printShapeInfo("FF shape");


    ASSERT_TRUE(z->isSameShape(&expFF));

    //expFF.printBuffer("e");
    //z->printBuffer("z");
    ASSERT_TRUE(z->equalsTo(&expFF, 1e-3));

    delete resultFF;
}

TYPED_TEST(TypedConvolutionTests1, sconv2d_3) {
    auto input = NDArrayFactory::create<TypeParam>('c', {3, 3, 8, 8});
    auto weightsD = NDArrayFactory::create<TypeParam>('c', {1, 3, 1, 1});
    auto weightsP = NDArrayFactory::create<TypeParam>('c', {2, 3, 1, 1});
    auto bias = NDArrayFactory::create<TypeParam>('c', {1, 2});
    auto output = NDArrayFactory::create<TypeParam>('c', {3, 2, 8, 8});
    output.assign(0.0);

    input.linspace(1);
    weightsD.linspace(1);
    weightsP.linspace(1);
    bias.linspace(1);
    weightsD.permutei({2,3,1,0});
    weightsP.permutei({2,3,1,0});

    auto expOutput = NDArrayFactory::create<TypeParam>('c', {3, 2, 8, 8});

    nd4j::ops::sconv2d op;
    Nd4jStatus status = op.execute({&input, &weightsD, &weightsP, &bias}, {&output}, {},  {1, 1, 1, 1, 0, 0, 1, 1, 0}, {});
    auto result = op.execute({&input, &weightsD, &weightsP, &bias}, {},  {1, 1, 1, 1, 0, 0, 1, 1, 0}, {});

    auto z = result->at(0);

    //printf("\n");
    //output.printBuffer("output");
    //z->printBuffer("z");


    //ASSERT_TRUE(expOutput.isSameShape(z));

    delete result;
}

TYPED_TEST(TypedConvolutionTests1, deconv2D_FF_NoBias_1) {
    Nd4jLong _expS[] = {4, 2, 3, 8, 8, 192, 64, 8, 1, typeid(TypeParam) == typeid(float) ? 8192 : 16384, 1, 99};
    TypeParam _expB[] = {6276.0,   12831.0,   19668.0,   26790.0,   27012.0,   20703.0,   14100.0,    7200.0,    13719.0,   28023.0,   42918.0,   58410.0,   58902.0,   45105.0,   30693.0,   15660.0,    22389.0,   45696.0,   69930.0,   95100.0,   95910.0,   73386.0,   49899.0,   25440.0,    32346.0,   65970.0,  100884.0,  137100.0,  138276.0,  105726.0,   71838.0,   36600.0,    33726.0,   68790.0,  105204.0,  142980.0,  144156.0,  110226.0,   74898.0,   38160.0,    27555.0,   56154.0,   85806.0,  116520.0,  117474.0,   89748.0,   60933.0,   31020.0,    19917.0,   40557.0,   61926.0,   84030.0,   84714.0,   64671.0,   43875.0,   22320.0,    10752.0,   21879.0,   33384.0,   45270.0,   45636.0,   34815.0,   23604.0,   12000.0,    7551.0,   15456.0,   23718.0,   32340.0,   32562.0,   24978.0,   17025.0,    8700.0,    16569.0,   33873.0,   51918.0,   70710.0,   71202.0,   54555.0,   37143.0,   18960.0,    27114.0,   55371.0,   84780.0,  115350.0,  116160.0,   88911.0,   60474.0,   30840.0,    39246.0,   80070.0,  122484.0,  166500.0,  167676.0,  128226.0,   87138.0,   44400.0,    40626.0,   82890.0,  126804.0,  172380.0,  173556.0,  132726.0,   90198.0,   45960.0,    33180.0,   67629.0,  103356.0,  140370.0,  141324.0,  107973.0,   73308.0,   37320.0,    23967.0,   48807.0,   74526.0,  101130.0,  101814.0,   77721.0,   52725.0,   26820.0,    12927.0,   26304.0,   40134.0,   54420.0,   54786.0,   41790.0,   28329.0,   14400.0,    8826.0,   18081.0,   27768.0,   37890.0,   38112.0,   29253.0,   19950.0,   10200.0,    19419.0,   39723.0,   60918.0,   83010.0,   83502.0,   64005.0,   43593.0,   22260.0,    31839.0,   65046.0,   99630.0,  135600.0,  136410.0,  104436.0,   71049.0,   36240.0,    46146.0,   94170.0,  144084.0,  195900.0,  197076.0,  150726.0,  102438.0,   52200.0,    47526.0,   96990.0,  148404.0,  201780.0,  202956.0,  155226.0,  105498.0,   53760.0,    38805.0,   79104.0,  120906.0,  164220.0,  165174.0,  126198.0,   85683.0,   43620.0,    28017.0,   57057.0,   87126.0,  118230.0,  118914.0,   90771.0,   61575.0,   31320.0,    15102.0,   30729.0,   46884.0,   63570.0,   63936.0,   48765.0,   33054.0,   16800.0,    17220.0,   34863.0,   52932.0,   71430.0,   72228.0,   54831.0,   36996.0,   18720.0,    36327.0,   73527.0,  111606.0,  150570.0,  152214.0,  115521.0,   77925.0,   39420.0,    57381.0,  116112.0,  176202.0,  237660.0,  240198.0,  182250.0,  122907.0,   62160.0,    80442.0,  162738.0,  246900.0,  332940.0,  336420.0,  255198.0,  172062.0,   87000.0,    84702.0,  171318.0,  259860.0,  350340.0,  353820.0,  268338.0,  180882.0,   91440.0,    66867.0,  135210.0,  205038.0,  276360.0,  279042.0,  211572.0,  142581.0,   72060.0,    46845.0,   94701.0,  143574.0,  193470.0,  195306.0,  148047.0,   99747.0,   50400.0,    24576.0,   49671.0,   75288.0,  101430.0,  102372.0,   77583.0,   52260.0,   26400.0,    22095.0,   44688.0,   67782.0,   91380.0,   92178.0,   69906.0,   47121.0,   23820.0,    46377.0,   93777.0,  142206.0,  191670.0,  193314.0,  146571.0,   98775.0,   49920.0,    72906.0,  147387.0,  223452.0,  301110.0,  303648.0,  230175.0,  155082.0,   78360.0,    101742.0,  205638.0,  311700.0,  419940.0,  423420.0,  320898.0,  216162.0,  109200.0,    106002.0,  214218.0,  324660.0,  437340.0,  440820.0,  334038.0,  224982.0,  113640.0,    83292.0,  168285.0,  254988.0,  343410.0,  346092.0,  262197.0,  176556.0,   89160.0,    58095.0,  117351.0,  177774.0,  239370.0,  241206.0,  182697.0,  122997.0,   62100.0,    30351.0,   61296.0,   92838.0,  124980.0,  125922.0,   95358.0,   64185.0,   32400.0,    26970.0,   54513.0,   82632.0,  111330.0,  112128.0,   84981.0,   57246.0,   28920.0,    56427.0,  114027.0,  172806.0,  232770.0,  234414.0,  177621.0,  119625.0,   60420.0,    88431.0,  178662.0,  270702.0,  364560.0,  367098.0,  278100.0,  187257.0,   94560.0,    123042.0,  248538.0,  376500.0,  506940.0,  510420.0,  386598.0,  260262.0,  131400.0,    127302.0,  257118.0,  389460.0,  524340.0,  527820.0,  399738.0,  269082.0,  135840.0,    99717.0,  201360.0,  304938.0,  410460.0,  413142.0,  312822.0, 
    NDArray exp(_expB, _expS);

    auto input = NDArrayFactory::create_<TypeParam>('c', {2, 3, 4, 4});
    auto weights = NDArrayFactory::create_<TypeParam>('c', {3, 3, 5, 5});

    input->linspace(1);
    weights->linspace(1);
    weights->permutei({2,3,1,0});

    auto variableSpace = new VariableSpace();
    variableSpace->putVariable(-1, input);
    variableSpace->putVariable(-2, weights);

    auto block = new Context(1, variableSpace, false);
    block->fillInputs({-1, -2});

    block->getIArguments()->push_back(5);
    block->getIArguments()->push_back(5);

    block->getIArguments()->push_back(1);
    block->getIArguments()->push_back(1);

    block->getIArguments()->push_back(0);
    block->getIArguments()->push_back(0);

    // dilation
    block->getIArguments()->push_back(1);
    block->getIArguments()->push_back(1);

    // NOT same mode
    block->getIArguments()->push_back(0);

    block->getIArguments()->push_back(0);

    nd4j::ops::deconv2d op;

    Nd4jStatus status = op.execute(block);

    ASSERT_EQ(ND4J_STATUS_OK, status);

    auto output = variableSpace->getVariable(1)->getNDArray();

    ASSERT_TRUE(exp.isSameShape(output));

    //    exp.printBuffer("Expctd buffer");
    //output->printBuffer("Result buffer");
    ASSERT_TRUE(exp.equalsTo(output));

    delete variableSpace;
    delete block;
}

TYPED_TEST(TypedConvolutionTests1, conv2D_BP_Bias_1) {
    TypeParam _expWGradB[] = {9312.0, 12580.0, 9528.0, 13168.0, 17712.0, 13360.0, 9960.0, 13348.0, 10032.0, 13344.0, 18148.0, 13848.0, 19312.0, 26160.0, 19888.0, 15144.0, 20452.0, 15504.0};
    Nd4jLong _expWGradS[] = {4, 2, 1, 3, 3, 9, 9, 3, 1, typeid(TypeParam) == typeid(float) ? 8192 : 16384, 1, 99};
    NDArray expWGrad(_expWGradB, _expWGradS);
    expWGrad.permutei({2,3,1,0});

    TypeParam _expBGradB[] = {784.0, 1296.0};
    Nd4jLong _expBGradS[] = {2, 2, 1, 1, 1, typeid(TypeParam) == typeid(float) ? 8192 : 16384, 1, 99};

    NDArray expBGrad(_expBGradB, _expBGradS);

    auto input = NDArrayFactory::create<TypeParam>('c', {2, 1, 4, 4});
    auto weights = NDArrayFactory::create<TypeParam>('c', {2, 1, 3, 3});
    auto bias = NDArrayFactory::create<TypeParam>('c', {2, 1});
    auto epsilonNext = NDArrayFactory::create<TypeParam>('c', {2, 2, 4, 4});


    TypeParam _expEpsB[] = {952.0, 1540.0, 1636.0, 1180.0, 1791.0, 2886.0, 3057.0, 2193.0, 2223.0, 3570.0, 3741.0, 2673.0, 1900.0, 3028.0, 3160.0, 2240.0, 2872.0, 4612.0, 4708.0, 3356.0, 5247.0, 8358.0, 8529.0, 6033.0, 5679.0, 9042.0, 9213.0, 6513.0, 4588.0, 7252.0, 7384.0, 5184.0};
    NDArray expEps(_expEpsB, input.getShapeInfo());

    input.linspace(1);
    weights.linspace(1);
    epsilonNext.linspace(1);
    weights.permutei({2,3,1,0});

    nd4j::ops::conv2d_bp op;

    auto results = op.execute({&input, &weights, &bias, &epsilonNext}, {},  {3, 3, 1, 1, 0, 0, 1, 1, 1}, {});

    ASSERT_TRUE(results->size() == 3);

    auto epsilon = results->at(0);
    auto gradW = results->at(1);
    auto gradB = results->at(2);

    ASSERT_TRUE(expWGrad.isSameShape(gradW));

    //expWGrad.printBuffer("Expctd buffer");
    //  gradW->printBuffer("Result buffer");
    ASSERT_TRUE(expWGrad.equalsTo(gradW));


    ASSERT_TRUE(input.isSameShape(epsilon));

    //  expEps.printBuffer("Expctd buffer");
    //epsilon->printBuffer("Result buffer");
    ASSERT_TRUE(expEps.equalsTo(epsilon));

    ASSERT_TRUE(expBGrad.isSameShape(gradB));

    ASSERT_TRUE(expBGrad.equalsTo(gradB));

    delete results;
}


TYPED_TEST(TypedConvolutionTests1, conv2D_BP_NoBias_1) {
    TypeParam _expWGradB[] = {9312.0, 12580.0, 9528.0, 13168.0, 17712.0, 13360.0, 9960.0, 13348.0, 10032.0, 13344.0, 18148.0, 13848.0, 19312.0, 26160.0, 19888.0, 15144.0, 20452.0, 15504.0};
    Nd4jLong _expWGradS[] = {4, 2, 1, 3, 3, 9, 9, 3, 1, typeid(TypeParam) == typeid(float) ? 8192 : 16384, 1, 99};
    NDArray expWGrad(_expWGradB, _expWGradS);
    expWGrad.permutei({2,3,1,0});

    auto input = NDArrayFactory::create<TypeParam>('c', {2, 1, 4, 4});
    auto weights = NDArrayFactory::create<TypeParam>('c', {2, 1, 3, 3});
    auto epsilonNext = NDArrayFactory::create<TypeParam>('c', {2, 2, 4, 4});


    TypeParam _expEpsB[] = {952.0, 1540.0, 1636.0, 1180.0, 1791.0, 2886.0, 3057.0, 2193.0, 2223.0, 3570.0, 3741.0, 2673.0, 1900.0, 3028.0, 3160.0, 2240.0, 2872.0, 4612.0, 4708.0, 3356.0, 5247.0, 8358.0, 8529.0, 6033.0, 5679.0, 9042.0, 9213.0, 6513.0, 4588.0, 7252.0, 7384.0, 5184.0};
    NDArray expEps(_expEpsB, input.getShapeInfo());

    input.linspace(1);
    weights.linspace(1);
    epsilonNext.linspace(1);
    weights.permutei({2,3,1,0});

    nd4j::ops::conv2d_bp op;

    auto results = op.execute({&input, &weights, &epsilonNext}, {},  {3, 3, 1, 1, 0, 0, 1, 1, 1}, {});

    ASSERT_TRUE(results->size() == 2);

    auto epsilon = results->at(0);
    auto gradW = results->at(1);

    ASSERT_TRUE(expWGrad.isSameShape(gradW));

    //expWGrad.printBuffer("Expctd buffer");
    //  gradW->printBuffer("Result buffer");
    ASSERT_TRUE(expWGrad.equalsTo(gradW));


    ASSERT_TRUE(input.isSameShape(epsilon));

    //  expEps.printBuffer("Expctd buffer");
    //epsilon->printBuffer("Result buffer");
    ASSERT_TRUE(expEps.equalsTo(epsilon));

    delete results;
}

TYPED_TEST(TypedConvolutionTests1, sconv2d_conv2d_1) {
    TypeParam _expBFF[] = {10025.0f,   10350.0f,   10675.0f,   11000.0f,   11325.0f,   11650.0f,   13275.0f,   13600.0f,   13925.0f,   14250.0f,   14575.0f,   14900.0f,   16525.0f,   16850.0f,   17175.0f,   17500.0f,   17825.0f,   18150.0f,   19775.0f,   20100.0f,   20425.0f,   20750.0f,   21075.0f,   21400.0f,   23025.0f,   23350.0f,   23675.0f,   24000.0f,   24325.0f,   24650.0f,   26275.0f,   26600.0f,   26925.0f,   27250.0f,   27575.0f,   27900.0f,   53150.0f,   55350.0f,   57550.0f,   59750.0f,   61950.0f,   64150.0f,   75150.0f,   77350.0f,   79550.0f,   81750.0f,   83950.0f,   86150.0f,   97150.0f,   99350.0f,  101550.0f,  103750.0f,  105950.0f,  108150.0f,   119150.0f,  121350.0f,  123550.0f,  125750.0f,  127950.0f,  130150.0f,   141150.0f,  143350.0f,  145550.0f,  147750.0f,  149950.0f,  152150.0f,   163150.0f,  165350.0f,  167550.0f,  169750.0f,  171950.0f,  174150.0f,   119400.0f,  120350.0f,  121300.0f,  122250.0f,  123200.0f,  124150.0f,   128900.0f,  129850.0f,  130800.0f,  131750.0f,  132700.0f,  133650.0f,   138400.0f,  139350.0f,  140300.0f,  141250.0f,  142200.0f,  143150.0f,   147900.0f,  148850.0f,  149800.0f,  150750.0f,  151700.0f,  152650.0f,   157400.0f,  158350.0f,  159300.0f,  160250.0f,  161200.0f,  162150.0f,   166900.0f,  167850.0f,  168800.0f,  169750.0f,  170700.0f,  171650.0f,   350025.0f,  352850.0f,  355675.0f,  358500.0f,  361325.0f,  364150.0f,   378275.0f,  381100.0f,  383925.0f,  386750.0f,  389575.0f,  392400.0f,   406525.0f,  409350.0f,  412175.0f,  415000.0f,  417825.0f,  420650.0f,   434775.0f,  437600.0f,  440425.0f,  443250.0f,  446075.0f,  448900.0f,   463025.0f,  465850.0f,  468675.0f,  471500.0f,  474325.0f,  477150.0f,   491275.0f,  494100.0f,  496925.0f,  499750.0f,  502575.0f,  505400.0f,   353775.0f,  355350.0f,  356925.0f,  358500.0f,  360075.0f,  361650.0f,   369525.0f,  371100.0f,  372675.0f,  374250.0f,  375825.0f,  377400.0f,   385275.0f,  386850.0f,  388425.0f,  390000.0f,  391575.0f,  393150.0f,   401025.0f,  402600.0f,  404175.0f,  405750.0f,  407325.0f,  408900.0f,   416775.0f,  418350.0f,  419925.0f,  421500.0f,  423075.0f,  424650.0f,   432525.0f,  434100.0f,  435675.0f,  437250.0f,  438825.0f,  440400.0f,   771900.0f,  775350.0f,  778800.0f,  782250.0f,  785700.0f,  789150.0f,   806400.0f,  809850.0f,  813300.0f,  816750.0f,  820200.0f,  823650.0f,   840900.0f,  844350.0f,  847800.0f,  851250.0f,  854700.0f,  858150.0f,   875400.0f,  878850.0f,  882300.0f,  885750.0f,  889200.0f,  892650.0f,   909900.0f,  913350.0f,  916800.0f,  920250.0f,  923700.0f,  927150.0f,   944400.0f,  947850.0f,  951300.0f,  954750.0f,  958200.0f,  961650.0f,   107525.0f,  107850.0f,  108175.0f,  108500.0f,  108825.0f,  109150.0f,   110775.0f,  111100.0f,  111425.0f,  111750.0f,  112075.0f,  112400.0f,   114025.0f,  114350.0f,  114675.0f,  115000.0f,  115325.0f,  115650.0f,   117275.0f,  117600.0f,  117925.0f,  118250.0f,  118575.0f,  118900.0f,   120525.0f,  120850.0f,  121175.0f,  121500.0f,  121825.0f,  122150.0f,   123775.0f,  124100.0f,  124425.0f,  124750.0f,  125075.0f,  125400.0f,   713150.0f,  715350.0f,  717550.0f,  719750.0f,  721950.0f,  724150.0f,   735150.0f,  737350.0f,  739550.0f,  741750.0f,  743950.0f,  746150.0f,   757150.0f,  759350.0f,  761550.0f,  763750.0f,  765950.0f,  768150.0f,   779150.0f,  781350.0f,  783550.0f,  785750.0f,  787950.0f,  790150.0f,   801150.0f,  803350.0f,  805550.0f,  807750.0f,  809950.0f,  812150.0f,   823150.0f,  825350.0f,  827550.0f,  829750.0f,  831950.0f,  834150.0f,   404400.0f,  405350.0f,  406300.0f,  407250.0f,  408200.0f,  409150.0f,   413900.0f,  414850.0f,  415800.0f,  416750.0f,  417700.0f,  418650.0f,   423400.0f,  424350.0f,  425300.0f,  426250.0f,  427200.0f,  428150.0f,   432900.0f,  433850.0f,  434800.0f,  435750.0f,  436700.0f,  437650.0f,   442400.0f,  443350.0f,  444300.0f,  445250.0f,  446200.0f,  447150.0f,   451900.0f,  452850.0f,  453800.0f,  454750.0f,  455700.0f,  456650.0f,   1197525.0f, 1200350.0f, 1203175.0f, 1206000.0f, 1208825.0f, 1211650.0f,   1225775.0f, 1228600.0f, 1231425.0f, 1234250.0f, 1237075.0f, 1
    Nd4jLong _expSFF[] = {4, 2, 6, 6, 6, 216, 36, 6, 1, typeid(TypeParam) == typeid(float) ? 8192 : 16384, 1, 99,};
    NDArray expFF(_expBFF, _expSFF);
    TypeParam _exp2BFF[] = {827.4900282f,   832.2350283f,   836.9800284f,   841.725028f,   846.4700287f,   851.2150288f,   874.9400293f,   879.6850294f,   884.4300295f,   889.1750296f,   893.9200297f,   898.665029f,   922.3900304f,   927.1350305f,   931.8800306f,   936.6250307f,   941.3700308f,   946.1150309f,   969.8400315f,   974.5850316f,   979.3300317f,   984.0750318f,   988.8200319f,   993.5650320f,   1017.2900326f,  1022.0350327f,  1026.7800328f,  1031.5250329f,   1036.2700330f,  1041.0150331f,  1064.7400337f,  1069.4850338f,   1074.2300339f,  1078.9750340f,  1083.7200341f,  1088.4650342f,   1822.4550553f,  1833.995055f,   1845.5350558f,  1857.075056f,   1868.6150563f,  1880.1550566f,  1937.8550578f,  1949.3950581f,   1960.9350583f,  1972.4750586f,  1984.015058f,   1995.5550591f,   2053.2550604f,  2064.7950606f,  2076.3350609f,  2087.8750611f,   2099.4150614f,  2110.955061f,   2168.6550629f,  2180.1950632f,   2191.7350634f,  2203.2750637f,  2214.8150639f,  2226.3550642f,   2284.0550655f,  2295.5950657f,  2307.1350660f,  2318.6750662f,   2330.2150665f,  2341.7550667f,  2399.4550680f,  2410.9950683f,   2422.5350685f,  2434.0750688f,  2445.6150690f,  2457.1550693f,   2817.419968f,   2835.7549686f,  2854.0899683f,  2872.4249680f,   2890.7599677f,  2909.0949674f,  3000.7699660f,  3019.104965f,   3037.4399655f,  3055.7749652f,  3074.1099649f,  3092.4449646f,   3184.1199632f,  3202.4549629f,  3220.789962f,   3239.1249624f,   3257.4599621f,  3275.7949618f,  3367.4699604f,  3385.8049601f,   3404.1399598f,  3422.474959f,   3440.8099593f,  3459.1449590f,   3550.8199576f,  3569.1549573f,  3587.4899570f,  3605.8249567f,   3624.1599565f,  3642.4949562f,  3734.1699548f,  3752.5049545f,   3770.8399542f,  3789.1749539f,  3807.5099536f,  3825.8449534f,   3812.385098f,   3837.5150988f,  3862.6450994f,  3887.7751000f,   3912.9051006f,  3938.0351012f,  4063.6851041f,  4088.8151047f,   4113.9451053f,  4139.0751059f,  4164.2051065f,  4189.3351071f,   4314.9851100f,  4340.1151106f,  4365.2451112f,  4390.3751118f,   4415.5051124f,  4440.6351130f,  4566.2851159f,  4591.4151165f,   4616.5451171f,  4641.6751177f,  4666.805118f,   4691.9351188f,   4817.5851218f,  4842.7151224f,  4867.8451230f,  4892.975123f,   4918.1051241f,  4943.2351247f,  5068.8851277f,  5094.0151283f,   5119.1451288f,  5144.2751294f,  5169.4051300f,  5194.5351306f,   4807.3499803f,  4839.2749801f,  4871.1999799f,  4903.1249797f,   4935.0499795f,  4966.9749793f,  5126.5999784f,  5158.5249782f,   5190.4499780f,  5222.3749778f,  5254.2999777f,  5286.2249775f,   5445.8499765f,  5477.774976f,   5509.6999762f,  5541.6249760f,   5573.5499758f,  5605.4749756f,  5765.0999747f,  5797.0249745f,   5828.9499743f,  5860.8749741f,  5892.7999739f,  5924.724973f,   6084.3499728f,  6116.2749726f,  6148.1999724f,  6180.1249723f,   6212.0499721f,  6243.9749719f,  6403.59997f,    6435.5249708f,   6467.4499706f,  6499.3749704f,  6531.2999702f,  6563.2249700f,   5802.3150007f,  5841.0350006f,  5879.7550005f,  5918.4750004f,   5957.195000f,   5995.9150003f,  6189.5149999f,  6228.2349998f,   6266.9549997f,  6305.6749996f,  6344.3949995f,  6383.114999f,   6576.7149990f,  6615.4349990f,  6654.1549989f,  6692.8749988f,   6731.5949987f,  6770.3149986f,  6963.9149982f,  7002.6349981f,   7041.3549981f,  7080.0749980f,  7118.7949979f,  7157.5149978f,   7351.1149974f,  7389.8349973f,  7428.5549972f,  7467.2749972f,   7505.9949971f,  7544.7149970f,  7738.3149966f,  7777.0349965f,   7815.7549964f,  7854.4749963f,  7893.1949963f,  7931.9149962f,   6797.2799488f,  6842.794948f,   6888.3099489f,  6933.8249490f,   6979.3399491f,  7024.8549492f,  7252.4299497f,  7297.9449498f,   7343.4599499f,  7388.9749500f,  7434.489950f,   7480.0049501f,   7707.5799506f,  7753.0949507f,  7798.6099508f,  7844.1249509f,   7889.6399510f,  7935.1549511f,  8162.7299515f,  8208.2449516f,   8253.7599517f,  8299.2749518f,  8344.7899519f,  8390.3049520f,   8617.8799525f,  8663.394952f,   8708.9099526f,  8754.4249527f,   8799.9399528f,  8845.4549529f,  9073.0299534f,  9118.5449535f,   9164.0599536f,  9209.5749537f,  9255.089953f,   93
    Nd4jLong _exp2SFF[] = {4, 2, 10, 6, 6, 360, 36, 6, 1, typeid(TypeParam) == typeid(float) ? 8192 : 16384, 1, 99,};
    NDArray exp2FF(_exp2BFF, _exp2SFF);

    auto input = NDArrayFactory::create<TypeParam>('c', {2, 3, 10, 10});
    auto weightsD = NDArrayFactory::create<TypeParam>('c', {2, 3, 5, 5});
    auto weightsP = NDArrayFactory::create<TypeParam>('c', {10, 6, 1, 1});


    input.linspace(1);
    weightsD.linspace(1);
    weightsP.linspace(1);
    weightsD.permutei({2,3,1,0});
    weightsP.permutei({2,3,1,0});

    weightsP.applyScalar(scalar::Divide, 10000.0);

    nd4j::ops::sconv2d op;
    auto resultFF = op.execute({&input, &weightsD}, {}, {5, 5, 1, 1, 0, 0, 1, 1, 0}, {});

    auto z = resultFF->at(0);

    ASSERT_TRUE(z->isSameShape(&expFF));
    ASSERT_TRUE(z->equalsTo(&expFF, 1));


    nd4j::ops::conv2d op2d;
    // weightsP.printShapeInfo();
    auto result2D = op2d.execute({z, &weightsP}, {}, {1, 1, 1, 1, 0, 0, 1, 1, 0, 0}, {});

    auto z2d = result2D->at(0);

    ASSERT_TRUE(z2d->isSameShape(&exp2FF));
    ASSERT_TRUE(z2d->equalsTo(&exp2FF));

    delete resultFF;
    delete result2D;
}



TEST_F(ConvolutionTests1, Test_im2col_col2im_1) {
    int kY = 5;
    int kX = 5;
    int sY = 1;
    int sX = 1;
    int pY = 0;
    int pX = 0;
    int dY = 1;
    int dX = 1;
    int inY = 28;
    int inX = 28;
    int channels = 3;

    bool isSameMode = true;

    auto x = NDArrayFactory::create<double>('c', {2, channels, inY, inX});
    x.linspace(1);

    int oY, oX;
    x.syncToDevice();
    //ASSERT_TRUE(x.isActualOnDeviceSide());
    ASSERT_TRUE(x.isActualOnHostSide());
    //x.printBuffer("x", 64);

    nd4j::ops::ConvolutionUtils::calcOutSizePool2D(oY, oX, kY, kX, sY, sX, pY, pX, dY, dX, inY, inX, isSameMode);

    if (isSameMode)
        nd4j::ops::ConvolutionUtils::calcPadding2D(pY, pX, oY, oX, inY, inX, kY, kX, sY, sX, dY, dX);

    auto im2col0 = NDArrayFactory::create<double>('c', {2, channels, kY, kX, oY, oX});

    ExtraArguments args({(double) kY, (double) kX, (double) sY, (double) sX, (double) pY, (double) pX, (double) dY, (double) dX, isSameMode ? (double) 1 : (double) 0, (double)0.0, (double) 0.});
    x.applyTransform(transform::Im2col, &im2col0, &args);

    nd4j::ops::im2col op;
    auto result2col = op.execute({&x}, {}, {kY, kX, sY, sX, pY, pX, dY, dX, isSameMode ? 1 : 0});

    auto im2col1 = result2col->at(0);

    //im2col0.printBuffer("transformed");
    //im2col1->printBuffer("customized", 64);

    ASSERT_TRUE(im2col1->isSameShape(&im2col0));
    ASSERT_TRUE(im2col1->equalsTo(&im2col0));


    ExtraArguments args2({ (double) sY, (double) sX, (double) pY, (double) pX, (double) inY, (double) inX, (double) dY, (double) dX, isSameMode ? (double) 1 : (double) 0});
    auto col2im0 = NDArrayFactory::create<double>('c', {2, channels, inY, inX});
    im2col0.applyTransform(transform::Col2Im, &col2im0, &args2);

    nd4j::ops::col2im op2im;
    auto result2im = op2im.execute({im2col1}, {}, {sY, sX, pY, pX, inY, inX, dY, dX, isSameMode ? 1 : 0});
    auto col2im1 = result2im->at(0);

    ASSERT_TRUE(col2im1->isSameShape(&col2im0));
    ASSERT_TRUE(col2im1->equalsTo(&col2im0));

    delete result2col;
    delete result2im;
}


TEST_F(ConvolutionTests1, Test_im2col_col2im_2) {
    int kY = 5;
    int kX = 5;
    int sY = 1;
    int sX = 1;
    int pY = 0;
    int pX = 0;
    int dY = 1;
    int dX = 1;
    int inY = 28;
    int inX = 28;
    int channels = 3;

    bool isSameMode = true;

    auto x = NDArrayFactory::create<double>('c', {2, channels, inY, inX});
    x.linspace(1);

    int oY, oX;

    nd4j::ops::ConvolutionUtils::calcOutSizePool2D(oY, oX, kY, kX, sY, sX, pY, pX, dY, dX, inY, inX, isSameMode);

    if (isSameMode)
        nd4j::ops::ConvolutionUtils::calcPadding2D(pY, pX, oY, oX, inY, inX, kY, kX, sY, sX, dY, dX);

    auto im2col0 = NDArrayFactory::create<double>('c', {2, channels, oY, oX, kY, kX});
    im2col0.permutei({0, 1, 4, 5, 2, 3});

    ExtraArguments args2col({(double) kY, (double) kX, (double) sY, (double) sX, (double) pY, (double) pX, (double) dY, (double) dX, isSameMode ? (double) 1 : (double) 0, (double)0.0, (double) 0.});
    x.applyTransform(transform::Im2col, &im2col0, &args2col);

    nd4j::ops::im2col op;
    auto result2col = op.execute({&x}, {}, {kY, kX, sY, sX, pY, pX, dY, dX, isSameMode ? 1 : 0});

    auto im2col1 = result2col->at(0);

    ASSERT_TRUE(im2col1->isSameShape(&im2col0));
    ASSERT_TRUE(im2col1->equalsTo(&im2col0));


    ExtraArguments args2im({ (double) sY, (double) sX, (double) pY, (double) pX, (double) inY, (double) inX, (double) dY, (double) dX, isSameMode ? (double) 1 : (double) 0});
    auto col2im0 = NDArrayFactory::create<double>('c', {2, channels, inY, inX});
    im2col0.applyTransform(transform::Col2Im, &col2im0, &args2im);

    nd4j::ops::col2im op2im;
    auto result2im = op2im.execute({im2col1}, {}, {sY, sX, pY, pX, inY, inX, dY, dX, isSameMode ? 1 : 0});
    auto col2im1 = result2im->at(0);

    ASSERT_TRUE(col2im1->isSameShape(&col2im0));
    ASSERT_TRUE(col2im1->equalsTo(&col2im0));

    delete result2col;
    delete result2im;
}

TEST_F(ConvolutionTests1, Test_im2col_col2im_3) {
    int kY = 5;
    int kX = 5;
    int sY = 1;
    int sX = 1;
    int pY = 0;
    int pX = 0;
    int dY = 1;
    int dX = 1;
    int inY = 28;
    int inX = 28;
    int channels = 3;

    bool isSameMode = true;

    auto x = NDArrayFactory::create<double>('c', {2, channels, inY, inX});
    x.linspace(1);

    int oY, oX;

    nd4j::ops::ConvolutionUtils::calcOutSizePool2D(oY, oX, kY, kX, sY, sX, pY, pX, dY, dX, inY, inX, isSameMode);

    if (isSameMode)
        nd4j::ops::ConvolutionUtils::calcPadding2D(pY, pX, oY, oX, inY, inX, kY, kX, sY, sX, dY, dX);

    auto im2col0 = NDArrayFactory::create<double>('c', {2, channels, oY, oX, kY, kX});
    im2col0.permutei({0, 1, 4, 5, 2, 3});

    auto im2col1 = NDArrayFactory::create<double>('c', {2, channels, oY, oX, kY, kX});
    im2col1.permutei({0, 1, 4, 5, 2, 3});

    ExtraArguments args2col({(double) kY, (double) kX, (double) sY, (double) sX, (double) pY, (double) pX, (double) dY, (double) dX, isSameMode ? (double) 1 : (double) 0, (double)0.0, (double) 0.});
    x.applyTransform(transform::Im2col, &im2col0, &args2col);

    nd4j::ops::im2col op;
    auto status = op.execute({&x}, {&im2col1}, {}, {kY, kX, sY, sX, pY, pX, dY, dX, isSameMode ? 1 : 0}, {});
    ASSERT_EQ(Status::OK(), status);

    ASSERT_TRUE(im2col1.isSameShape(&im2col0));
    ASSERT_TRUE(im2col1.equalsTo(&im2col0));


    ExtraArguments args2im({ (double) sY, (double) sX, (double) pY, (double) pX, (double) inY, (double) inX, (double) dY, (double) dX, isSameMode ? (double) 1 : (double) 0});
    auto col2im0 = NDArrayFactory::create<double>('c', {2, channels, inY, inX});
    im2col0.applyTransform(transform::Col2Im, &col2im0, &args2im);

    nd4j::ops::col2im op2im;
    auto result2im = op2im.execute({&im2col1}, {}, {sY, sX, pY, pX, inY, inX, dY, dX, isSameMode ? 1 : 0});
    auto col2im1 = result2im->at(0);

    ASSERT_TRUE(col2im1->isSameShape(&col2im0));
    ASSERT_TRUE(col2im1->equalsTo(&col2im0));

    delete result2im;
}


TEST_F(ConvolutionTests1, TestDeconv_bp_1) {


    double _expb[] = {  35.f,   38.f,   41.f,   44.f,   47.f,   50.f,   53.f,   56.f,   59.f,   62.f,   65.f,    68.f,   71.f,   74.f,   77.f,   80.f,   71.f,   78.f,   85.f,   92.f,   99.f,  106.f,    113.f,  120.f,  127.f,  134.f,  141.f,  148.f,  155.f,  162.f,  169.f,  176.f,  107.f,    118.f,  129.f,  140.f,  151.f,  162.f,  173.f,  184.f,  195.f,  206.f,  217.f,  228.f,    239.f,  250.f,  261.f,  272.f,  131.f,  134.f,  137.f,  140.f,  143.f,  146.f,  149.f,    152.f,  155.f,  158.f,  161.f,  164.f,  167.f,  170.f,  173.f,  176.f,  295.f,  302.f,    309.f,  316.f,  323.f,  330.f,  337.f,  344.f,  351.f,  358.f,  365.f,  372.f,  379.f,    386.f,  393.f,  400.f,  459.f,  470.f,  481.f,  492.f,  503.f,  514.f,  525.f,  536.f,    547.f,  558.f,  569.f,  580.f,  591.f,  602.f,  613.f,  624.f,  227.f,  230.f,  233.f,    236.f,  239.f,  242.f,  245.f,  248.f,  251.f,  254.f,  257.f,  260.f,  263.f,  266.f,    269.f,  272.f,  519.f,  526.f,  533.f,  540.f,  547.f,  554.f,  561.f,  568.f,  575.f,    582.f,  589.f,  596.f,  603.f,  610.f,  617.f,  624.f,  811.f,  822.f,  833.f,  844.f,    855.f,  866.f,  877.f,  888.f,  899.f,  910.f,  921.f,  932.f,  943.f,  954.f,  965.f,    976.f};
    std::shared_ptr<DataBuffer> pBuffer1 = std::make_shared<DataBuffer>(_expb, sizeof(_expb), nd4j::DataType::DOUBLE, false);
    NDArray expEpsilon(pBuffer1, 'c', {3, 3, 4, 4});

    double _expwb[] = { 160008.f,  203400.f,  191112.f,  246792.f,  222216.f,  290184.f};
    std::shared_ptr<DataBuffer> pBuffer2 = std::make_shared<DataBuffer>(_expwb, sizeof(_expwb), nd4j::DataType::DOUBLE, false);
    NDArray expGradW(pBuffer2, 'c', {3, 2, 1, 1});
    expGradW.permutei({2,3,1,0});

    double _expbb[] = {1944.f,  2712.f};
    std::shared_ptr<DataBuffer> pBuffer3 = std::make_shared<DataBuffer>(_expbb, sizeof(_expbb), nd4j::DataType::DOUBLE, false);
    NDArray expGradB(pBuffer3, 'c', {1, 2});

    auto input = NDArrayFactory::create<double>('c', {3, 3, 4, 4});
    auto bias = NDArrayFactory::create<double>('c', {1, 2});
    auto weights = NDArrayFactory::create<double>('c',{3, 2, 1, 1});
    auto epsilon = NDArrayFactory::create<double>('c', {3, 2, 4, 4});

    /*
        Input shape (3, 3, 4, 4)
        Weights shape (3, 2, 1, 1)
        Epsilon shape (3, 2, 4, 4)
     */

    input.linspace(1);
    weights.linspace(1);
    bias.linspace(1);
    epsilon.linspace(1);
    weights.permutei({2,3,1,0});

    nd4j::ops::deconv2d_bp op;

    auto result = op.execute({&input, &weights, &bias, &epsilon}, {}, {1, 1, 1, 1, 0, 0, 1, 1, 1, 0});

    ASSERT_EQ(ND4J_STATUS_OK, result->status());

    auto expNext = result->at(0);

    ASSERT_TRUE(expEpsilon.isSameShape(expNext));
    ASSERT_TRUE(expEpsilon.equalsTo(expNext));

    auto gradW = result->at(1);

    ASSERT_TRUE(expGradW.isSameShape(gradW));
    ASSERT_TRUE(expGradW.equalsTo(gradW));

    auto gradB = result->at(2);

    ASSERT_TRUE(expGradB.isSameShape(gradB));
    ASSERT_TRUE(expGradB.equalsTo(gradB));

    delete result;
}

TEST_F(ConvolutionTests1, TestDeconv_bp_2) {
    /*
     Input shape:
    [3, 3, 14, 14]
    Output shape:
    [3, 2, 15, 15]
    Weights shape:
    [3, 2, 2, 2]
    Bias shape:
    [1, 2]
    weight shape:
    [3, 2, 2, 2]
    weight grad shape:
    [3, 2, 2, 2]
    bias grad shape:
    [2]
    input epsilon shape:
    [3, 2, 15, 15]
    output epsilon shape:
    [3, 3, 14, 14]
     */
    /*
    auto input('c', {3, 3, 14, 14});
    auto bias('c', {2});
    auto weights('c',{3, 2, 2, 2});
    auto epsilon('c', {3, 2, 15, 15});


    input.linspace(1);
    weights.linspace(1);
    bias.linspace(1);
    epsilon.linspace(1);

    nd4j::ops::deconv2d_bp<double> op;

    auto result = op.execute({&input, &weights, &bias, &epsilon}, {}, {2, 2, 1, 1, 0, 0, 2, 2, 0});
    ASSERT_EQ(ND4J_STATUS_OK, result->status());


    delete result;*/
}

TEST_F(ConvolutionTests1, TestDeconv_ff_2) {

    NDArray exp('c', {3, 2, 4, 4}, {218., 227., 236., 245., 254., 263., 272., 281., 290., 299.,  308., 317., 326., 335., 344., 353., 270., 282., 294., 306.,  318., 330., 342., 354., 366., 378., 390., 402., 414., 426.,  438., 450., 650., 659., 668., 677., 686., 695., 704., 713.,  722., 731., 740., 749., 758., 767., 776., 785., 846., 858.,  870., 882., 894., 906., 918., 930., 942., 954., 966., 978.,  990., 1002., 1014., 1026., 1082., 1091., 1100., 1109., 1118., 1127.,  1136., 1145., 1154., 1163., 1172., 1181., 1190., 1199., 1208., 1217.,  1422., 1434., 1446., 1458., 1470., 1482., 1494., 1506., 1518., 1530.,  1542., 1554., 1566., 1578., 1590., 1602.});

    auto input = NDArrayFactory::create<double>('c', {3, 3, 4, 4});
    auto weights = NDArrayFactory::create<double>('c',{3, 2, 1, 1});
    auto bias = NDArrayFactory::create<double>('c', {1, 2});

    input.linspace(1);
    weights.linspace(1);
    bias.linspace(1);
    weights.permutei({2,3,1,0});

    nd4j::ops::deconv2d op;

    auto result = op.execute({&input, &weights, &bias}, {}, {1, 1, 1, 1, 0, 0, 1, 1, 1, 0});

    ASSERT_EQ(ND4J_STATUS_OK, result->status());

    auto output = result->at(0);

    ASSERT_TRUE(exp.isSameShape(output));
    ASSERT_TRUE(exp.equalsTo(output));

    delete result;
}

TYPED_TEST(TypedConvolutionTests1, Test_Conv1D_ff_1) {
    auto input = NDArrayFactory::create<TypeParam>('c', {2, 2, 6});
    auto weights = NDArrayFactory::create<TypeParam>('c', {2, 2, 3}, {1,5,9,3,7,11,2,6,10,4,8,12});
    auto bias = NDArrayFactory::create<TypeParam>('c', {1, 3});
    auto expFF = NDArrayFactory::create<TypeParam>('c', {2, 3, 5}, {59.0, 69.0, 79.0, 89.0, 99.0, 132.0, 158.0, 184.0, 210.0, 236.0, 205.0, 247.0, 289.0, 331.0, 373.0, 179.0, 189.0, 199.0, 209.0, 219.0, 444.0, 470.0, 496.0, 522.0, 548.0, 709.0, 751.0, 793.0, 835.0, 877.0});
    auto expEps = NDArrayFactory::create<TypeParam>('c', {2, 2, 6}, {130.0, 293.0, 326.0, 359.0, 392.0, 220.0, 166.0, 371.0, 416.0, 461.0, 506.0, 280.0, 355.0, 788.0, 821.0, 854.0, 887.0, 490.0, 481.0, 1046.0, 1091.0, 1136.0, 1181.0, 640.0});
    auto expGW = NDArrayFactory::create<TypeParam>('c', {3, 2, 2}, {1415.0, 1520.0, 2045.0, 2150.0, 1865.0, 2020.0, 2795.0, 2950.0, 2315.0, 2520.0, 3545.0, 3750.0});
    auto expGB = NDArrayFactory::create<TypeParam>('c', {1, 3}, {105.0, 155.0, 205.0});

    expGW.permutei({2,1,0});
    input.linspace(1);
    bias.linspace(1);

    nd4j::ops::conv1d op;
    auto result_FF = op.execute({&input, &weights, &bias}, {}, {2, 1, 0, 0});

    ASSERT_EQ(ND4J_STATUS_OK, result_FF->status());

    auto z = result_FF->at(0);

    ASSERT_TRUE(expFF.isSameShape(z));
    ASSERT_TRUE(expFF.equalsTo(z));

    nd4j::ops::conv1d_bp op_bp;

    auto epsilonNxt = z->dup();
    epsilonNxt->linspace(1);

    auto result_BP = op_bp.execute({&input, &weights, &bias, epsilonNxt}, {}, {2, 1, 0, 0});
    ASSERT_EQ(ND4J_STATUS_OK, result_BP->status());

    auto eps = result_BP->at(0);
    auto gradW = result_BP->at(1);
    auto gradB = result_BP->at(2);

    ASSERT_TRUE(expEps.isSameShape(eps));
    ASSERT_TRUE(expGW.isSameShape(gradW));
    ASSERT_TRUE(expGB.isSameShape(gradB));

    ASSERT_TRUE(expEps.equalsTo(eps));
    ASSERT_TRUE(expGW.equalsTo(gradW));
    ASSERT_TRUE(expGB.equalsTo(gradB));

    delete result_FF;
    delete result_BP;
    delete epsilonNxt;
}


TYPED_TEST(TypedConvolutionTests1, Test_Conv1D_ff_2) {
    auto input = NDArrayFactory::create<TypeParam>('c', {2, 2, 6});
    auto weights = NDArrayFactory::create<TypeParam>('c', {2, 2, 3}, {1,5,9,3,7,11,2,6,10,4,8,12});

    input.linspace(1);

    nd4j::ops::conv1d op;
    auto result = op.execute({&input, &weights}, {}, {2, 1, 0, 1});

    ASSERT_EQ(ND4J_STATUS_OK, result->status());

    auto z = result->at(0);

    delete result;
}

TEST_F(ConvolutionTests1, Test_Dilation2D_1) {
    auto input = NDArrayFactory::create<double>('c', {2, 6, 6, 3});
    auto weights = NDArrayFactory::create<double>('c', {3, 2, 3});
    auto exp = NDArrayFactory::create<double>('c', {2, 3, 3, 3}, {77,   79,   81,   83,   85,   87,   80,   82,   84,  113,  115,  117, 119,  121,  123,  116,  118,  120,  107,  109,  111,  113,  115,  117, 110,  112,  114,  185,  187,  189,  191,  193,  195,  188,  190,  192, 221,  223,  225,  227,  229,  231,  224,  226,  228,  215,  217,  219, 221,  223,  225,  218,  220,  222,});

    input.linspace(1);
    weights.linspace(1);

    nd4j::ops::dilation2d op;
    auto result = op.execute({&input, &weights}, {}, {1, 1,2,2,1, 1,2,2,1});
    ASSERT_EQ(Status::OK(), result->status());

    auto z = result->at(0);

    ASSERT_TRUE(exp.isSameShape(z));
    ASSERT_TRUE(exp.equalsTo(z));

    delete result;
}

TEST_F(ConvolutionTests1, Test_Dilation2D_2) {
    auto input = NDArrayFactory::create<double>('c', {2, 6, 6, 3});
    auto weights = NDArrayFactory::create<double>('c', {3, 2, 3});
    auto exp = NDArrayFactory::create<double>('c', {2, 1, 2, 3}, {95, 97, 99, 101, 103, 105, 203, 205, 207, 209, 211, 213});

    input.linspace(1);
    weights.linspace(1);

    nd4j::ops::dilation2d op;
    auto result = op.execute({&input, &weights}, {}, {0, 1,2,2,1, 1,2,2,1});
    ASSERT_EQ(Status::OK(), result->status());

    auto z = result->at(0);

    ASSERT_TRUE(exp.isSameShape(z));
    ASSERT_TRUE(exp.equalsTo(z));

    delete result;
}

//////////////////////////////////////////////////////////////////////
TYPED_TEST(TypedConvolutionTests1, conv2d_bp_test1) {

    int bS=2, iH=4,iW=3,  iC=4,oC=3,  kH=3,kW=2,  sH=1,sW=1,  pH=0,pW=0,  dH=1,dW=1;
    int       oH=4,oW=3;
    int paddingMode = 1;             // 1-SAME, 0-VALID;
    int dataFormat  = 1;             // 1-NHWC, 0-NCHW

    auto input = NDArrayFactory::create<TypeParam>('c', {bS, iH, iW, iC});
    auto weights = NDArrayFactory::create<TypeParam>('c', {kH, kW, iC, oC});
    auto bias = NDArrayFactory::create<TypeParam>('c', {oC}, {1,2,3});
    auto gradO = NDArrayFactory::create<TypeParam>('c', {bS, oH, oW, oC});

    auto expGradI = NDArrayFactory::create<TypeParam>('c', {bS, iH, iW, iC},{ 0.226,  0.343,  0.46 ,  0.577, 1.172,  1.46 ,  1.748,  2.036, 1.892,  2.288,  2.684,  3.08 , 1.284,  1.581,  1.878,  2.175, 4.458,  5.133,  5.808,  6.483, 6.186,  7.023,  7.86 ,  8.697,
                                                     3.39 ,  3.93 ,  4.47 ,  5.01 , 9.642, 10.803, 11.964, 13.125,11.37 , 12.693, 14.016, 15.339, 5.266,  5.707,  6.148,  6.589,12.98 , 13.916, 14.852, 15.788,14.564, 15.608, 16.652, 17.696,
                                                     3.25 ,  4.015,  4.78 ,  5.545, 9.812, 11.396, 12.98 , 14.564,10.532, 12.224, 13.916, 15.608, 9.708, 10.977, 12.246, 13.515,25.194, 27.813, 30.432, 33.051,26.922, 29.703, 32.484, 35.265,
                                                    11.814, 13.326, 14.838, 16.35 ,30.378, 33.483, 36.588, 39.693,32.106, 35.373, 38.64 , 41.907,13.474, 14.563, 15.652, 16.741,31.988, 34.22 , 36.452, 38.684,33.572, 35.912, 38.252, 40.592});

    auto expGradW = NDArrayFactory::create<TypeParam>('c', {kH, kW, iC, oC},{14.4 , 14.76, 15.12,14.4 , 14.76, 15.12,14.4 , 14.76, 15.12,14.4 , 14.76, 15.12, 9.24,  9.48,  9.72, 9.24,  9.48,  9.72, 9.24,  9.48,  9.72, 9.24,  9.48,  9.72,
                                                    17.04, 17.52, 18.  ,17.04, 17.52, 18.  ,17.04, 17.52, 18.  ,17.04, 17.52, 18.  ,10.88, 11.2 , 11.52,10.88, 11.2 , 11.52,10.88, 11.2 , 11.52,10.88, 11.2 , 11.52,
                                                    11.16, 11.52, 11.88,11.16, 11.52, 11.88,11.16, 11.52, 11.88,11.16, 11.52, 11.88, 7.08,  7.32,  7.56, 7.08,  7.32,  7.56, 7.08,  7.32,  7.56, 7.08,  7.32,  7.56});
    // auto expGradB('c', {oC},{});

    input = 2.;
    weights.linspace(0.1, 0.1);
    gradO.linspace(0.01, 0.01);

    nd4j::ops::conv2d_bp op;
    auto results = op.execute({&input, &weights, &bias, &gradO}, {}, {kH,kW,  sH,sW,  pH,pW,  dH,dW, paddingMode, dataFormat});
    auto gradI = results->at(0);
    auto gradW = results->at(1);

    ASSERT_EQ(Status::OK(), results->status());

    ASSERT_TRUE(expGradI.isSameShape(gradI));
    ASSERT_TRUE(expGradI.equalsTo(gradI));

    ASSERT_TRUE(expGradW.isSameShape(gradW));
    ASSERT_TRUE(expGradW.equalsTo(gradW));

    delete results;
}

//////////////////////////////////////////////////////////////////////
TYPED_TEST(TypedConvolutionTests1, conv2d_bp_test2) {

    int bS=2, iH=4,iW=3,  iC=4,oC=3,  kH=3,kW=2,  sH=1,sW=1,  pH=0,pW=0,  dH=1,dW=1;
    int       oH=2,oW=2;
    int paddingMode = 0;             // 1-SAME, 0-VALID;
    int dataFormat  = 1;             // 1-NHWC, 0-NCHW

    auto input = NDArrayFactory::create<TypeParam>('c', {bS, iH, iW, iC});
    auto weights = NDArrayFactory::create<TypeParam>('c', {kH, kW, iC, oC});
    auto bias = NDArrayFactory::create<TypeParam>('c', {oC}, {1,2,3});
    auto gradO = NDArrayFactory::create<TypeParam>('c', {bS, oH, oW, oC});

    auto expGradI = NDArrayFactory::create<TypeParam>('c', {bS, iH, iW, iC},{ 0.014,0.032, 0.05 , 0.068,0.118,0.181, 0.244, 0.307,0.212,0.257, 0.302, 0.347,0.208,0.298, 0.388, 0.478,1.028,1.262, 1.496, 1.73 ,1.036,1.18 , 1.324, 1.468,
                                                     0.928,1.018, 1.108, 1.198,2.9  ,3.134, 3.368, 3.602,2.188,2.332, 2.476, 2.62 ,1.202,1.274, 1.346, 1.418,3.142,3.313, 3.484, 3.655,2.048,2.147, 2.246, 2.345,
                                                     0.086,0.212, 0.338, 0.464,0.694,0.973, 1.252, 1.531,0.716,0.869, 1.022, 1.175,1.216,1.522, 1.828, 2.134,3.908,4.574, 5.24 , 5.906,2.908,3.268, 3.628, 3.988,
                                                     3.664,3.97 , 4.276, 4.582,9.236,9.902,10.568,11.234,5.788,6.148, 6.508, 6.868,3.002,3.182, 3.362, 3.542,7.174,7.561, 7.948, 8.335,4.28 ,4.487, 4.694, 4.901});

    auto expGradW = NDArrayFactory::create<TypeParam>('c', {kH, kW, iC, oC},{1.84, 2., 2.16,1.84, 2., 2.16,1.84, 2., 2.16,1.84, 2., 2.16,1.84, 2., 2.16,1.84, 2., 2.16,1.84, 2., 2.16,1.84, 2., 2.16,
                                                    1.84, 2., 2.16,1.84, 2., 2.16,1.84, 2., 2.16,1.84, 2., 2.16,1.84, 2., 2.16,1.84, 2., 2.16,1.84, 2., 2.16,1.84, 2., 2.16,
                                                    1.84, 2., 2.16,1.84, 2., 2.16,1.84, 2., 2.16,1.84, 2., 2.16,1.84, 2., 2.16,1.84, 2., 2.16,1.84, 2., 2.16,1.84, 2., 2.16});
    // auto expGradB('c', {oC},{});

    input = 2.;
    weights.linspace(0.1, 0.1);
    gradO.linspace(0.01, 0.01);

    nd4j::ops::conv2d_bp op;
    auto results = op.execute({&input, &weights, &bias, &gradO}, {}, {kH,kW,  sH,sW,  pH,pW,  dH,dW, paddingMode, dataFormat});
    auto gradI = results->at(0);
    auto gradW = results->at(1);

    ASSERT_EQ(Status::OK(), results->status());

    ASSERT_TRUE(expGradI.isSameShape(gradI));
    ASSERT_TRUE(expGradI.equalsTo(gradI));

    ASSERT_TRUE(expGradW.isSameShape(gradW));
    ASSERT_TRUE(expGradW.equalsTo(gradW));

    delete results;
}

//////////////////////////////////////////////////////////////////////
TYPED_TEST(TypedConvolutionTests1, conv2d_bp_test3) {

    int bS=2, iH=4,iW=3,  iC=4,oC=3,  kH=3,kW=2,  sH=1,sW=1,  pH=0,pW=0,  dH=1,dW=1;
    int       oH=2,oW=2;
    int paddingMode = 0;             // 1-SAME, 0-VALID;
    int dataFormat  = 0;             // 1-NHWC, 0-NCHW

    auto input = NDArrayFactory::create<TypeParam>('c', {bS, iC, iH, iW});
    auto weights = NDArrayFactory::create<TypeParam>('c', {oC, iC, kH, kW});
    auto bias = NDArrayFactory::create<TypeParam>('c', {oC}, {1,2,3});
    auto gradO = NDArrayFactory::create<TypeParam>('c', {bS, oC, oH, oW});

    auto expGradI = NDArrayFactory::create<TypeParam>('c', {bS, iC, iH, iW},{ 0.567, 1.224,0.66 ,1.314, 2.82 ,1.512,1.386, 2.976,1.596,0.801, 1.71 ,0.912,0.657, 1.422,0.768,1.53 , 3.288,1.764,1.602, 3.444,1.848,0.927, 1.98 ,1.056,
                                                     0.747, 1.62 ,0.876,1.746, 3.756,2.016,1.818, 3.912,2.1  ,1.053, 2.25 ,1.2  ,0.837, 1.818,0.984,1.962, 4.224,2.268,2.034, 4.38 ,2.352,1.179, 2.52 ,1.344,
                                                     1.467, 3.06 ,1.596,3.186, 6.636,3.456,3.402, 7.08 ,3.684,1.845, 3.834,1.992,1.773, 3.69 ,1.92 ,3.834, 7.968,4.14 ,4.05 , 8.412,4.368,2.187, 4.536,2.352,
                                                     2.079, 4.32 ,2.244,4.482, 9.3  ,4.824,4.698, 9.744,5.052,2.529, 5.238,2.712,2.385, 4.95 ,2.568,5.13 ,10.632,5.508,5.346,11.076,5.736,2.871, 5.94 ,3.072});

    auto expGradW = NDArrayFactory::create<TypeParam>('c', {oC, iC, kH, kW},{1.3600e+00,1.3600e+00,1.3600e+00,1.3600e+00,1.3600e+00,1.3600e+00,1.3600e+00,1.3600e+00,1.3600e+00,1.3600e+00,1.3600e+00,1.3600e+00,1.3600e+00,1.3600e+00,1.3600e+00,1.3600e+00,1.3600e+00,1.3600e+00,
                                                    1.3600e+00,1.3600e+00,1.3600e+00,1.3600e+00,1.3600e+00,1.3600e+00,2.0000e+00,2.0000e+00,2.0000e+00,2.0000e+00,2.0000e+00,2.0000e+00,2.0000e+00,2.0000e+00,2.0000e+00,2.0000e+00,2.0000e+00,2.0000e+00,
                                                    2.0000e+00,2.0000e+00,2.0000e+00,2.0000e+00,2.0000e+00,2.0000e+00,2.0000e+00,2.0000e+00,2.0000e+00,2.0000e+00,2.0000e+00,2.0000e+00,2.6400e+00,2.6400e+00,2.6400e+00,2.6400e+00,2.6400e+00,2.6400e+00,
                                                    2.6400e+00,2.6400e+00,2.6400e+00,2.6400e+00,2.6400e+00,2.6400e+00,2.6400e+00,2.6400e+00,2.6400e+00,2.6400e+00,2.6400e+00,2.6400e+00,2.6400e+00,2.6400e+00,2.6400e+00,2.6400e+00,2.6400e+00,2.6400e+00});
    auto expGradB = NDArrayFactory::create<TypeParam>('c', {oC},{0.68, 1., 1.32});

    input = 2.;
    weights.linspace(0.1, 0.1);
    gradO.linspace(0.01, 0.01);
    weights.permutei({2,3,1,0});
    expGradW.permutei({2,3,1,0});

    nd4j::ops::conv2d_bp op;
    auto results = op.execute({&input, &weights, &bias, &gradO}, {}, {kH,kW,  sH,sW,  pH,pW,  dH,dW, paddingMode, dataFormat});
    auto gradI = results->at(0);
    auto gradW = results->at(1);
    auto gradB = results->at(2);

    ASSERT_EQ(Status::OK(), results->status());

    ASSERT_TRUE(expGradI.isSameShape(gradI));
    ASSERT_TRUE(expGradI.equalsTo(gradI));

    ASSERT_TRUE(expGradW.isSameShape(gradW));
    ASSERT_TRUE(expGradW.equalsTo(gradW));

    ASSERT_TRUE(expGradB.isSameShape(gradB));
    ASSERT_TRUE(expGradB.equalsTo(gradB));

    delete results;
}

//////////////////////////////////////////////////////////////////////
TEST_F(ConvolutionTests1, conv2d_bp_4) {

    int bS=1, iH=7,iW=1,  iC=2,oC=3,  kH=2,kW=1,  sH=1,sW=1,  pH=0,pW=0,  dH=1,dW=1;
    int       oH=7,oW=1;
    int paddingMode = 1;             // 1-SAME, 0-VALID;
    int dataFormat  = 0;             // 1-NHWC, 0-NCHW

    NDArray input('c', {bS, iC, iH, iW}, nd4j::DataType::FLOAT32);
    NDArray weights('c', {kH, kW, iC, oC}, nd4j::DataType::FLOAT32);
    NDArray bias('c', {oC}, {1,2,3}, nd4j::DataType::FLOAT32);
    NDArray gradO('c', {bS, oC, oH, oW}, nd4j::DataType::FLOAT32);

    NDArray gradI('c', {bS, iC, iH, iW}, nd4j::DataType::FLOAT32);
    NDArray gradW('c', {kH, kW, iC, oC}, nd4j::DataType::FLOAT32);
    NDArray gradB('c', {oC}, nd4j::DataType::FLOAT32);


    input = 2.;
    weights.linspace(0.1, 0.1);
    gradO.linspace(0.01, 0.01);

    nd4j::ops::conv2d_bp op;
    auto status = op.execute({&input, &weights, &bias, &gradO}, {&gradI, &gradW, &gradB}, {}, {kH,kW,  sH,sW,  pH,pW,  dH,dW, paddingMode, dataFormat}, {});

    ASSERT_EQ(Status::OK(), status);
}

////////////////////////////////////////////////////////////////////
TYPED_TEST(TypedConvolutionTests1, conv3d_bp_test1) {

    int bS=2, iD=3,iH=4,iW=3,  iC=4,oC=3,  kD=2,kH=3,kW=2,  sD=1,sH=1,sW=1,  pD=0,pH=0,pW=0,  dD=1,dH=1,dW=1;
    int       oD=3,oH=4,oW=3;
    int paddingMode = 1;             // 1-SAME,  0-VALID;
    int dataFormat  = 1;             // 1-NDHWC, 0-NCDHW

    auto input    = NDArrayFactory::create<TypeParam>('c', {bS, iD, iH, iW, iC});
    auto weights  = NDArrayFactory::create<TypeParam>('c', {kD, kH, kW, iC, oC});
    auto bias     = NDArrayFactory::create<TypeParam>('c', {oC}, {1,2,3});
    auto gradO    = NDArrayFactory::create<TypeParam>('c', {bS, oD, oH, oW, oC});

    auto expGradI = NDArrayFactory::create<TypeParam>('c', {bS, iD, iH, iW, iC},{0.226,   0.343,   0.46 ,   0.577,  1.172,   1.46 ,   1.748,   2.036,  1.892,   2.288,   2.684,   3.08 ,  1.284,   1.581,   1.878,   2.175,  4.458,   5.133,   5.808,   6.483,  6.186,   7.023,   7.86 ,   8.697,  3.39 ,   3.93 ,   4.47 ,   5.01 ,  9.642,  10.803,  11.964,  13.125, 11.37 ,  12.693,  14.016,  15.339,
                                                        5.266,   5.707,   6.148,   6.589, 12.98 ,  13.916,  14.852,  15.788, 14.564,  15.608,  16.652,  17.696,  6.284,   7.166,   8.048,   8.93 , 17.896,  19.768,  21.64 ,  23.512, 21.928,  24.016,  26.104,  28.192, 18.12 ,  19.686,  21.252,  22.818, 45.852,  49.146,  52.44 ,  55.734, 53.196,  56.814,  60.432,  64.05 ,
                                                       28.164,  30.216,  32.268,  34.32 , 67.884,  72.15 ,  76.416,  80.682, 75.228,  79.818,  84.408,  88.998, 29.324,  30.854,  32.384,  33.914, 67.432,  70.6  ,  73.768,  76.936, 73.192,  76.576,  79.96 ,  83.344, 27.884,  30.062,  32.24 ,  34.418, 66.28 ,  70.744,  75.208,  79.672, 70.312,  74.992,  79.672,  84.352,
                                                       58.296,  61.806,  65.316,  68.826,133.98 , 141.162, 148.344, 155.526,141.324, 148.83 , 156.336, 163.842, 68.34 ,  72.336,  76.332,  80.328,156.012, 164.166, 172.32 , 180.474,163.356, 171.834, 180.312, 188.79 , 61.292,  64.118,  66.944,  69.77 ,136.552, 142.312, 148.072, 153.832,142.312, 148.288, 154.264, 160.24 ,
                                                        9.298,  11.359,  13.42 ,  15.481, 27.092,  31.268,  35.444,  39.62 , 27.812,  32.096,  36.38 ,  40.664, 26.556,  29.769,  32.982,  36.195, 66.666,  73.173,  79.68 ,  86.187, 68.394,  75.063,  81.732,  88.401, 28.662,  32.118,  35.574,  39.03 , 71.85 ,  78.843,  85.836,  92.829, 73.578,  80.733,  87.888,  95.043,
                                                       29.89 ,  32.275,  34.66 ,  37.045, 70.004,  74.828,  79.652,  84.476, 71.588,  76.52 ,  81.452,  86.384, 71.084,  75.854,  80.624,  85.394,163.048, 172.696, 182.344, 191.992,167.08 , 176.944, 186.808, 196.672,138.648, 146.046, 153.444, 160.842,310.236, 325.194, 340.152, 355.11 ,317.58 , 332.862, 348.144, 363.426,
                                                      148.692, 156.576, 164.46 , 172.344,332.268, 348.198, 364.128, 380.058,339.612, 355.866, 372.12 , 388.374,125.228, 130.646, 136.064, 141.482,274.792, 285.736, 296.68 , 307.624,280.552, 291.712, 302.872, 314.032, 92.684,  98.75 , 104.816, 110.882,211.432, 223.672, 235.912, 248.152,215.464, 227.92 , 240.376, 252.832,
                                                      178.824, 188.166, 197.508, 206.85 ,398.364, 417.21 , 436.056, 454.902,405.708, 424.878, 444.048, 463.218,188.868, 198.696, 208.524, 218.352,420.396, 440.214, 460.032, 479.85 ,427.74 , 447.882, 468.024, 488.166,157.196, 163.91 , 170.624, 177.338,343.912, 357.448, 370.984, 384.52 ,349.672, 363.424, 377.176, 390.928});

    auto expGradW = NDArrayFactory::create<TypeParam>('c', {kD, kH, kW, iC, oC},{120.96, 122.04, 123.12,120.96, 122.04, 123.12,120.96, 122.04, 123.12,120.96, 122.04, 123.12, 79.56,  80.28,  81.  , 79.56,  80.28,  81.  , 79.56,  80.28,  81.  , 79.56,  80.28,  81.  ,
                                                        154.8 , 156.24, 157.68,154.8 , 156.24, 157.68,154.8 , 156.24, 157.68,154.8 , 156.24, 157.68,101.76, 102.72, 103.68,101.76, 102.72, 103.68,101.76, 102.72, 103.68,101.76, 102.72, 103.68,
                                                        111.24, 112.32, 113.4 ,111.24, 112.32, 113.4 ,111.24, 112.32, 113.4 ,111.24, 112.32, 113.4 , 73.08,  73.8 ,  74.52, 73.08,  73.8 ,  74.52, 73.08,  73.8 ,  74.52, 73.08,  73.8 ,  74.52,
                                                         67.68,  68.4 ,  69.12, 67.68,  68.4 ,  69.12, 67.68,  68.4 ,  69.12, 67.68,  68.4 ,  69.12, 44.4 ,  44.88,  45.36, 44.4 ,  44.88,  45.36, 44.4 ,  44.88,  45.36, 44.4 ,  44.88,  45.36,
                                                         85.92,  86.88,  87.84, 85.92,  86.88,  87.84, 85.92,  86.88,  87.84, 85.92,  86.88,  87.84, 56.32,  56.96,  57.6 , 56.32,  56.96,  57.6 , 56.32,  56.96,  57.6 , 56.32,  56.96,  57.6 ,
                                                         61.2 ,  61.92,  62.64, 61.2 ,  61.92,  62.64, 61.2 ,  61.92,  62.64, 61.2 ,  61.92,  62.64, 40.08,  40.56,  41.04, 40.08,  40.56,  41.04, 40.08,  40.56,  41.04, 40.08,  40.56,  41.04});
    // auto expGradB('c', {oC},{});

    input = 2.;
    weights.linspace(0.1, 0.1);
    gradO.linspace(0.01, 0.01);

    nd4j::ops::conv3dnew_bp op;
    auto results = op.execute({&input, &weights, &bias, &gradO}, {}, {kD,kH,kW,  sD,sH,sW,  pD,pH,pW,  dD,dH,dW, paddingMode, dataFormat});
    auto* gradI = results->at(0);
    auto* gradW = results->at(1);

    ASSERT_EQ(Status::OK(), results->status());
    ASSERT_TRUE(expGradI.isSameShape(gradI));
    ASSERT_TRUE(expGradI.equalsTo(gradI));

    ASSERT_TRUE(expGradW.isSameShape(gradW));
    ASSERT_TRUE(expGradW.equalsTo(gradW));

    delete results;
}


////////////////////////////////////////////////////////////////////
TYPED_TEST(TypedConvolutionTests1, conv3d_bp_test2) {

    int bS=2, iD=3,iH=4,iW=3,  iC=4,oC=3,  kD=2,kH=3,kW=2,  sD=1,sH=1,sW=1,  pD=0,pH=0,pW=0,  dD=1,dH=1,dW=1;
    int       oD=2,oH=2,oW=2;
    int paddingMode = 0;             // 1-SAME,  0-VALID;
    int dataFormat  = 1;             // 1-NDHWC, 0-NCDHW

    auto input    = NDArrayFactory::create<TypeParam>('c', {bS, iD, iH, iW, iC});
    auto weights  = NDArrayFactory::create<TypeParam>('c', {kD, kH, kW, iC, oC});
    auto bias     = NDArrayFactory::create<TypeParam>('c', {oC}, {1,2,3});
    auto gradO    = NDArrayFactory::create<TypeParam>('c', {bS, oD, oH, oW, oC});

    auto expGradI = NDArrayFactory::create<TypeParam>('c', {bS, iD, iH, iW, iC},{ 0.014, 0.032, 0.05 , 0.068, 0.118, 0.181, 0.244, 0.307, 0.212, 0.257, 0.302, 0.347, 0.208, 0.298, 0.388, 0.478, 1.028, 1.262, 1.496, 1.73 , 1.036, 1.18 , 1.324, 1.468, 0.928, 1.018, 1.108, 1.198, 2.9  , 3.134, 3.368, 3.602, 2.188, 2.332, 2.476, 2.62 ,
                                                         1.202, 1.274, 1.346, 1.418, 3.142, 3.313, 3.484, 3.655, 2.048, 2.147, 2.246, 2.345, 0.532, 0.676, 0.82 , 0.964, 2.324, 2.666, 3.008, 3.35 , 2.008, 2.206, 2.404, 2.602, 3.584, 3.98 , 4.376, 4.772,10.552,11.452,12.352,13.252, 7.4  , 7.904, 8.408, 8.912,
                                                         6.752, 7.148, 7.544, 7.94 ,17.752,18.652,19.552,20.452,11.432,11.936,12.44 ,12.944, 5.932, 6.184, 6.436, 6.688,14.42 ,14.978,15.536,16.094, 8.704, 9.01 , 9.316, 9.622, 3.11 , 3.236, 3.362, 3.488, 7.39 , 7.669, 7.948, 8.227, 4.388, 4.541, 4.694, 4.847,
                                                         8.56 , 8.866, 9.172, 9.478,19.892,20.558,21.224,21.89 ,11.548,11.908,12.268,12.628,11.008,11.314,11.62 ,11.926,25.22 ,25.886,26.552,27.218,14.428,14.788,15.148,15.508, 7.322, 7.502, 7.682, 7.862,16.462,16.849,17.236,17.623, 9.248, 9.455, 9.662, 9.869,
                                                         0.158, 0.392, 0.626, 0.86 , 1.27 , 1.765, 2.26 , 2.755, 1.22 , 1.481, 1.742, 2.003, 2.224, 2.746, 3.268, 3.79 , 6.788, 7.886, 8.984,10.082, 4.78 , 5.356, 5.932, 6.508, 6.4  , 6.922, 7.444, 7.966,15.572,16.67 ,17.768,18.866, 9.388, 9.964,10.54 ,11.116,
                                                         4.802, 5.09 , 5.378, 5.666,11.206,11.809,12.412,13.015, 6.512, 6.827, 7.142, 7.457, 6.004, 6.58 , 7.156, 7.732,14.996,16.202,17.408,18.614, 9.208, 9.838,10.468,11.098,17.984,19.244,20.504,21.764,42.808,45.436,48.064,50.692,25.256,26.624,27.992,29.36 ,
                                                        28.064,29.324,30.584,31.844,63.832,66.46 ,69.088,71.716,36.2  ,37.568,38.936,40.304,18.316,19.   ,19.684,20.368,40.916,42.338,43.76 ,45.182,22.816,23.554,24.292,25.03 , 8.438, 8.78 , 9.122, 9.464,18.91 ,19.621,20.332,21.043,10.58 ,10.949,11.318,11.687,
                                                        20.944,21.682,22.42 ,23.158,46.388,47.918,49.448,50.978,25.66 ,26.452,27.244,28.036,26.848,27.586,28.324,29.062,58.628,60.158,61.688,63.218,31.996,32.788,33.58 ,34.372,16.106,16.502,16.898,17.294,34.894,35.713,36.532,37.351,18.896,19.319,19.742,20.165});

    auto expGradW = NDArrayFactory::create<TypeParam>('c', {kD, kH, kW, iC, oC},{7.52, 7.84, 8.16,7.52, 7.84, 8.16,7.52, 7.84, 8.16,7.52, 7.84, 8.16,7.52, 7.84, 8.16,7.52, 7.84, 8.16,7.52, 7.84, 8.16,7.52, 7.84, 8.16,7.52, 7.84, 8.16,7.52, 7.84, 8.16,7.52, 7.84, 8.16,7.52, 7.84, 8.16,
                                                        7.52, 7.84, 8.16,7.52, 7.84, 8.16,7.52, 7.84, 8.16,7.52, 7.84, 8.16,7.52, 7.84, 8.16,7.52, 7.84, 8.16,7.52, 7.84, 8.16,7.52, 7.84, 8.16,7.52, 7.84, 8.16,7.52, 7.84, 8.16,7.52, 7.84, 8.16,7.52, 7.84, 8.16,
                                                        7.52, 7.84, 8.16,7.52, 7.84, 8.16,7.52, 7.84, 8.16,7.52, 7.84, 8.16,7.52, 7.84, 8.16,7.52, 7.84, 8.16,7.52, 7.84, 8.16,7.52, 7.84, 8.16,7.52, 7.84, 8.16,7.52, 7.84, 8.16,7.52, 7.84, 8.16,7.52, 7.84, 8.16,
                                                        7.52, 7.84, 8.16,7.52, 7.84, 8.16,7.52, 7.84, 8.16,7.52, 7.84, 8.16,7.52, 7.84, 8.16,7.52, 7.84, 8.16,7.52, 7.84, 8.16,7.52, 7.84, 8.16,7.52, 7.84, 8.16,7.52, 7.84, 8.16,7.52, 7.84, 8.16,7.52, 7.84, 8.16});
    // auto expGradB('c', {oC},{});

    input = 2.;
    weights.linspace(0.1, 0.1);
    gradO.linspace(0.01, 0.01);

    nd4j::ops::conv3dnew_bp op;
    auto results = op.execute({&input, &weights, &bias, &gradO}, {}, {kD,kH,kW,  sD,sH,sW,  pD,pH,pW,  dD,dH,dW, paddingMode, dataFormat});
    auto gradI = results->at(0);
    auto gradW = results->at(1);

    ASSERT_EQ(Status::OK(), results->status());
    ASSERT_TRUE(expGradI.isSameShape(gradI));
    ASSERT_TRUE(expGradI.equalsTo(gradI));

    ASSERT_TRUE(expGradW.isSameShape(gradW));
    ASSERT_TRUE(expGradW.equalsTo(gradW));

    delete results;
}


////////////////////////////////////////////////////////////////////
TYPED_TEST(TypedConvolutionTests1, conv3d_bp_test3) {

    int bS=2, iD=3,iH=4,iW=3,  iC=4,oC=3,  kD=2,kH=3,kW=2,  sD=1,sH=1,sW=1,  pD=0,pH=0,pW=0,  dD=1,dH=1,dW=1;
    int       oD=2,oH=2,oW=2;
    int paddingMode = 0;             // 1-SAME,  0-VALID;
    int dataFormat  = 0;             // 1-NDHWC, 0-NCDHW

    auto input    = NDArrayFactory::create<TypeParam>('c', {bS, iC, iD, iH, iW});
    auto weights  = NDArrayFactory::create<TypeParam>('c', {oC, iC, kD, kH, kW});
    auto bias     = NDArrayFactory::create<TypeParam>('c', {oC}, {1,2,3});
    auto gradO    = NDArrayFactory::create<TypeParam>('c', {bS, oC, oD, oH, oW});

    auto expGradI = NDArrayFactory::create<TypeParam>('c', {bS, iC, iD, iH, iW},{2.091, 4.356, 2.268, 4.53 , 9.42 , 4.896, 4.65 , 9.672, 5.028, 2.517, 5.226, 2.712, 4.932,10.242, 5.316,10.62 ,22.02 ,11.412,10.908,22.62 ,11.724, 5.868,12.15 , 6.288, 2.913, 6.03 , 3.12 , 6.234,12.888, 6.66 , 6.402,13.236, 6.84 , 3.423, 7.068, 3.648,
                                                        2.415, 5.04 , 2.628, 5.25 ,10.932, 5.688, 5.37 ,11.184, 5.82 , 2.913, 6.054, 3.144, 5.724,11.898, 6.18 ,12.348,25.62 ,13.284,12.636,26.22 ,13.596, 6.804,14.094, 7.296, 3.381, 7.002, 3.624, 7.242,14.976, 7.74 , 7.41 ,15.324, 7.92 , 3.963, 8.184, 4.224,
                                                        2.739, 5.724, 2.988, 5.97 ,12.444, 6.48 , 6.09 ,12.696, 6.612, 3.309, 6.882, 3.576, 6.516,13.554, 7.044,14.076,29.22 ,15.156,14.364,29.82 ,15.468, 7.74 ,16.038, 8.304, 3.849, 7.974, 4.128, 8.25 ,17.064, 8.82 , 8.418,17.412, 9.   , 4.503, 9.3  , 4.8  ,
                                                        3.063, 6.408, 3.348, 6.69 ,13.956, 7.272, 6.81 ,14.208, 7.404, 3.705, 7.71 , 4.008, 7.308,15.21 , 7.908,15.804,32.82 ,17.028,16.092,33.42 ,17.34 , 8.676,17.982, 9.312, 4.317, 8.946, 4.632, 9.258,19.152, 9.9  , 9.426,19.5  ,10.08 , 5.043,10.416, 5.376,
                                                        5.619,11.484, 5.868,11.73 ,23.964,12.24 ,12.138,24.792,12.66 , 6.333,12.93 , 6.6  ,12.42 ,25.362,12.948,25.884,52.836,26.964,26.748,54.588,27.852,13.932,28.422,14.496, 6.873,14.022, 7.152,14.298,29.16 ,14.868,14.754,30.084,15.336, 7.671,15.636, 7.968,
                                                        6.807,13.896, 7.092,14.178,28.932,14.76 ,14.586,29.76 ,15.18 , 7.593,15.486, 7.896,14.94 ,30.474,15.54 ,31.068,63.348,32.292,31.932,65.1  ,33.18 ,16.596,33.822,17.232, 8.205,16.722, 8.52 ,17.034,34.704,17.676,17.49 ,35.628,18.144, 9.075,18.48 , 9.408,
                                                        7.995,16.308, 8.316,16.626,33.9  ,17.28 ,17.034,34.728,17.7  , 8.853,18.042, 9.192,17.46 ,35.586,18.132,36.252,73.86 ,37.62 ,37.116,75.612,38.508,19.26 ,39.222,19.968, 9.537,19.422, 9.888,19.77 ,40.248,20.484,20.226,41.172,20.952,10.479,21.324,10.848,
                                                        9.183,18.72 , 9.54 ,19.074,38.868,19.8  ,19.482,39.696,20.22 ,10.113,20.598,10.488,19.98 ,40.698,20.724,41.436,84.372,42.948,42.3  ,86.124,43.836,21.924,44.622,22.704,10.869,22.122,11.256,22.506,45.792,23.292,22.962,46.716,23.76 ,11.883,24.168,12.288});

    auto expGradW = NDArrayFactory::create<TypeParam>('c', {oC, iC, kD, kH, kW},{5.28, 5.28, 5.28, 5.28, 5.28, 5.28, 5.28, 5.28, 5.28, 5.28, 5.28, 5.28, 5.28, 5.28, 5.28, 5.28, 5.28, 5.28, 5.28, 5.28, 5.28, 5.28, 5.28, 5.28,
                                                        5.28, 5.28, 5.28, 5.28, 5.28, 5.28, 5.28, 5.28, 5.28, 5.28, 5.28, 5.28, 5.28, 5.28, 5.28, 5.28, 5.28, 5.28, 5.28, 5.28, 5.28, 5.28, 5.28, 5.28,
                                                        7.84, 7.84, 7.84, 7.84, 7.84, 7.84, 7.84, 7.84, 7.84, 7.84, 7.84, 7.84, 7.84, 7.84, 7.84, 7.84, 7.84, 7.84, 7.84, 7.84, 7.84, 7.84, 7.84, 7.84,
                                                        7.84, 7.84, 7.84, 7.84, 7.84, 7.84, 7.84, 7.84, 7.84, 7.84, 7.84, 7.84, 7.84, 7.84, 7.84, 7.84, 7.84, 7.84, 7.84, 7.84, 7.84, 7.84, 7.84, 7.84,
                                                        10.4, 10.4, 10.4, 10.4, 10.4, 10.4, 10.4, 10.4, 10.4, 10.4, 10.4, 10.4, 10.4, 10.4, 10.4, 10.4, 10.4, 10.4, 10.4, 10.4, 10.4, 10.4, 10.4, 10.4,
                                                        10.4, 10.4, 10.4, 10.4, 10.4, 10.4, 10.4, 10.4, 10.4, 10.4, 10.4, 10.4, 10.4, 10.4, 10.4, 10.4, 10.4, 10.4, 10.4, 10.4, 10.4, 10.4, 10.4, 10.4});

    auto expGradB = NDArrayFactory::create<TypeParam>('c', {oC},{2.64, 3.92, 5.2 });

    input = 2.;
    weights.linspace(0.1, 0.1);
    gradO.linspace(0.01, 0.01);
    weights.permutei({2, 3, 4, 1, 0});
    expGradW.permutei({2, 3, 4, 1, 0});

    nd4j::ops::conv3dnew_bp op;
    auto results = op.execute({&input, &weights, &bias, &gradO}, {}, {kD,kH,kW,  sD,sH,sW,  pD,pH,pW,  dD,dH,dW, paddingMode, dataFormat});
    auto* gradI = results->at(0);
    auto* gradW = results->at(1);
    auto* gradB = results->at(2);

    ASSERT_EQ(Status::OK(), results->status());

    ASSERT_TRUE(expGradI.isSameShape(gradI));
    ASSERT_TRUE(expGradI.equalsTo(gradI));

    ASSERT_TRUE(expGradW.isSameShape(gradW));
    ASSERT_TRUE(expGradW.equalsTo(gradW));

    ASSERT_TRUE(expGradB.isSameShape(gradB));
    ASSERT_TRUE(expGradB.equalsTo(gradB));

    delete results;
}

//////////////////////////////////////////////////////////////////////
TYPED_TEST(TypedConvolutionTests1, depthwise_conv2d_1) {

    int bS=2, iH=4,iW=3,  iC=2,mC=2,  kH=3,kW=2,  sH=1,sW=1,  pH=0,pW=0,  dH=1,dW=1;
    int       oC=iC*mC;
    int       oH=4,oW=3;
    int paddingMode = 1;             // 1-SAME, 0-VALID;
    int dataFormat  = 1;             // 1-NHWC, 0-NCHW

    auto input    = NDArrayFactory::create<TypeParam>('c', {bS, iH, iW, iC});
    auto weights  = NDArrayFactory::create<TypeParam>('c', {kH, kW, iC, mC});


    auto expOutput = NDArrayFactory::create<TypeParam>('c', {bS, oH, oW, oC},{12. , 12.8, 13.6, 14.4,12. , 12.8, 13.6, 14.4, 5.2,  5.6,  6. ,  6.4,13.2, 14.4, 15.6, 16.8,13.2, 14.4, 15.6, 16.8, 5.4,  6. ,  6.6,  7.2,
                                                     13.2, 14.4, 15.6, 16.8,13.2, 14.4, 15.6, 16.8, 5.4,  6. ,  6.6,  7.2, 5.6,  6.4,  7.2,  8. , 5.6,  6.4,  7.2,  8. , 2. ,  2.4,  2.8,  3.2,
                                                     12. , 12.8, 13.6, 14.4,12. , 12.8, 13.6, 14.4, 5.2,  5.6,  6. ,  6.4,13.2, 14.4, 15.6, 16.8,13.2, 14.4, 15.6, 16.8, 5.4,  6. ,  6.6,  7.2,
                                                     13.2, 14.4, 15.6, 16.8,13.2, 14.4, 15.6, 16.8, 5.4,  6. ,  6.6,  7.2, 5.6,  6.4,  7.2,  8. , 5.6,  6.4,  7.2,  8. , 2. ,  2.4,  2.8,  3.2});
    input = 2.;
    weights.linspace(0.1, 0.1);

    nd4j::ops::depthwise_conv2d op;
    auto results = op.execute({&input, &weights}, {}, {kH,kW,  sH,sW,  pH,pW,  dH,dW, paddingMode, dataFormat});
    auto* output = results->at(0);

    ASSERT_EQ(Status::OK(), results->status());

    ASSERT_TRUE(expOutput.isSameShape(output));
    ASSERT_TRUE(expOutput.equalsTo(output));

    delete results;
}

//////////////////////////////////////////////////////////////////////
TEST_F(ConvolutionTests1, depthwise_conv2d_2) {

    int bS=2, iH=4,iW=3,  iC=2,mC=2,  kH=3,kW=2,  sH=1,sW=1,  pH=0,pW=0,  dH=1,dW=1;
    int       oC=iC*mC;
    int       oH=2,oW=2;
    int paddingMode = 0;             // 1-SAME, 0-VALID;
    int dataFormat  = 1;             // 1-NHWC, 0-NCHW

    auto input    = NDArrayFactory::create<double>('c', {bS, iH, iW, iC});
    auto weights  = NDArrayFactory::create<double>('c', {kH, kW, iC, mC});


    auto expOutput = NDArrayFactory::create<double>('c', {bS, oH, oW, oC},{13.2, 14.4, 15.6, 16.8,13.2, 14.4, 15.6, 16.8,13.2, 14.4, 15.6, 16.8,13.2, 14.4, 15.6, 16.8,
                                                     13.2, 14.4, 15.6, 16.8,13.2, 14.4, 15.6, 16.8,13.2, 14.4, 15.6, 16.8,13.2, 14.4, 15.6, 16.8});
    input = 2.;
    weights.linspace(0.1, 0.1);

    nd4j::ops::depthwise_conv2d op;
    auto results = op.execute({&input, &weights}, {}, {kH,kW,  sH,sW,  pH,pW,  dH,dW, paddingMode, dataFormat});
    auto* output = results->at(0);

    ASSERT_EQ(Status::OK(), results->status());

    ASSERT_TRUE(expOutput.isSameShape(output));
    ASSERT_TRUE(expOutput.equalsTo(output));

    delete results;
}


//////////////////////////////////////////////////////////////////////
TEST_F(ConvolutionTests1, depthwise_conv2d_3) {

    int bS=2, iH=4,iW=3,  iC=2,mC=2,  kH=3,kW=2,  sH=1,sW=1,  pH=0,pW=0,  dH=1,dW=1;
    int       oC=iC*mC;
    int       oH=2,oW=2;
    int paddingMode = 0;             // 1-SAME, 0-VALID;
    int dataFormat  = 0;             // 1-NHWC, 0-NCHW

    auto input    = NDArrayFactory::create<double>('c', {bS, iC, iH, iW});
    auto weights  = NDArrayFactory::create<double>('c', {mC, iC, kH, kW});
    auto biases   = NDArrayFactory::create<double>('c', {iC*mC}, {1,2,3,4});


    auto expOutput = NDArrayFactory::create<double>('c', {bS, oC, oH, oW},{5.2, 5.2, 5.2, 5.2,20.6,20.6,20.6,20.6,14.4,14.4,14.4,14.4,29.8,29.8,29.8,29.8, 5.2, 5.2, 5.2, 5.2,20.6,20.6,20.6,20.6,14.4,14.4,14.4,14.4,29.8,29.8,29.8,29.8});
    input = 2.;
    weights.linspace(0.1, 0.1);
    weights.permutei({2,3,1,0});

    nd4j::ops::depthwise_conv2d op;
    auto results = op.execute({&input, &weights, &biases}, {}, {kH,kW,  sH,sW,  pH,pW,  dH,dW, paddingMode, dataFormat});
    auto* output = results->at(0);

    // output->printIndexedBuffer();

    ASSERT_EQ(Status::OK(), results->status());

    ASSERT_TRUE(expOutput.isSameShape(output));
    ASSERT_TRUE(expOutput.equalsTo(output));

    delete results;
}

//////////////////////////////////////////////////////////////////////
TEST_F(ConvolutionTests1, depthwise_conv2d_4) {

    int bS=1, iH=111,iW=111,  iC=32,mC=1,  kH=7,kW=7,  sH=2,sW=2,  pH=0,pW=0,  dH=1,dW=1;
    int       oC=iC*mC;
    int       oH=56,oW=56;

    int paddingMode = 1;             // 1-SAME, 0-VALID;
    int dataFormat  = 1;             // 1-NHWC, 0-NCHW

    const float unique = -1000000;

    NDArray input('c', {bS, iH, iW, iC}, nd4j::DataType::FLOAT32);
    NDArray weights('c', {kH, kW, iC, mC}, nd4j::DataType::FLOAT32);
    NDArray output('c', {bS, oH, oW, oC}, nd4j::DataType::FLOAT32);
    input.linspace(0.1, 0.0001);
    weights = 0.5;
    output = unique;

    nd4j::ops::depthwise_conv2d op;
    Nd4jStatus status = op.execute({&input, &weights}, {&output} , {}, {kH,kW,  sH,sW,  pH,pW,  dH,dW, paddingMode, dataFormat}, {});

    ASSERT_EQ(Status::OK(), status);

    for(Nd4jLong i=output.lengthOf()/1.5; i < output.lengthOf(); ++i)
        ASSERT_EQ(output.e<float>(i) != unique, true);
}

//////////////////////////////////////////////////////////////////////
TEST_F(ConvolutionTests1, depthwise_conv2d_5) {

    int bS=1, iH=3,iW=3,  iC=2,mC=1,  kH=2,kW=2,  sH=1,sW=1,  pH=0,pW=0,  dH=1,dW=1;
    int       oC=iC*mC;
    int       oH=3,oW=3;
    int paddingMode = 1;             // 1-SAME, 0-VALID;
    int dataFormat  = 1;             // 1-NHWC, 0-NCHW

    auto input   = NDArrayFactory::create<double>('c', {bS, iH, iW, iC});
    auto weights = NDArrayFactory::create<double>('c', {kH, kW, iC, mC});


    auto expOutput = NDArrayFactory::create<double>('c', {bS, oH, oW, oC}, {20., 24.,28., 32.,16., 18.,44., 48.,52., 56.,28., 30.,28., 30.,32., 34.,17., 18.});
    input.linspace(1.);
    weights = 1.;

    nd4j::ops::depthwise_conv2d op;
    auto results = op.execute({&input, &weights}, {}, {kH,kW,  sH,sW,  pH,pW,  dH,dW, paddingMode, dataFormat});
    auto output = results->at(0);
    // output->printIndexedBuffer();

    ASSERT_EQ(Status::OK(), results->status());

    ASSERT_TRUE(expOutput.isSameShape(output));
    ASSERT_TRUE(expOutput.equalsTo(output));

    delete results;
}

//////////////////////////////////////////////////////////////////////
TEST_F(ConvolutionTests1, depthwise_conv2d_6) {

    int bS=1, iH=3,iW=3,  iC=2,mC=1,  kH=2,kW=2,  sH=1,sW=1,  pH=0,pW=0,  dH=1,dW=1;
    int       oC=iC*mC;
    int       oH=3,oW=3;
    int paddingMode = 1;             // 1-SAME, 0-VALID;
    int dataFormat  = 1;             // 1-NHWC, 0-NCHW

    NDArray input('c', {bS, iH, iW, iC}, nd4j::DataType::DOUBLE);
    NDArray weights('c', {kH, kW, iC, mC}, nd4j::DataType::DOUBLE);

    NDArray expOutput('c', {bS, oH, oW, oC}, {20., 24.,28., 32.,16., 18.,44., 48.,52., 56.,28., 30.,28., 30.,32., 34.,17., 18.});
    input.linspace(1.);
    weights = 1.;

    nd4j::ops::depthwise_conv2d op;
    ResultSet* results = op.execute({&input, &weights}, {}, {kH,kW,  sH,sW,  pH,pW,  dH,dW, paddingMode, dataFormat});
    NDArray* output = results->at(0);
    // output.printIndexedBuffer();

    ASSERT_EQ(Status::OK(), results->status());

    ASSERT_TRUE(expOutput.isSameShape(output));
    ASSERT_TRUE(expOutput.equalsTo(output));

    delete results;
}

//////////////////////////////////////////////////////////////////////
TEST_F(ConvolutionTests1, depthwise_conv2d_bp_test1) {

    int bS=2, iH=4,iW=3,  iC=2,mC=2,  kH=3,kW=2,  sH=1,sW=1,  pH=0,pW=0,  dH=1,dW=1;
    int       oH=4,oW=3;
    int       oC=iC*mC;
    int paddingMode = 1;             // 1-SAME, 0-VALID;
    int dataFormat  = 1;             // 1-NHWC, 0-NCHW

    auto input    = NDArrayFactory::create<double>('c', {bS, iH, iW, iC});
    auto weights  = NDArrayFactory::create<double>('c', {kH, kW, iC, mC});
    auto bias     = NDArrayFactory::create<double>('c', {oC}, {1,2,3,4});
    auto gradO    = NDArrayFactory::create<double>('c', {bS, oH, oW, oC});

    auto expGradI = NDArrayFactory::create<double>('c', {bS, iH, iW, iC},{0.07 ,  0.19 , 0.348,  0.652, 0.588,  0.956, 0.387,  0.687, 1.326,  2.022, 1.878,  2.67 , 1.071,  1.515, 2.982,  3.966, 3.534,  4.614, 1.606,  1.982, 3.932,  4.748, 4.428,  5.308,
                                                    1.126,  1.63 , 3.228,  4.3  , 3.468,  4.604, 3.123,  3.999, 7.95 ,  9.798, 8.502, 10.446, 3.807,  4.827, 9.606, 11.742,10.158, 12.39 , 4.198,  4.958, 9.884, 11.468,10.38 , 12.028});

    auto expGradW = NDArrayFactory::create<double>('c', {kH, kW, iC, mC},{19.08, 19.44,19.8 , 20.16,12.24, 12.48,12.72, 12.96,22.56, 23.04,23.52, 24. ,14.4 , 14.72,15.04, 15.36,14.76, 15.12,15.48, 15.84, 9.36,  9.6 , 9.84, 10.08});

    input = 2.;
    weights.linspace(0.1, 0.1);
    gradO.linspace(0.01, 0.01);

    nd4j::ops::depthwise_conv2d_bp op;
    auto results = op.execute({&input, &weights, &bias, &gradO}, {}, {kH,kW,  sH,sW,  pH,pW,  dH,dW, paddingMode, dataFormat});
    auto* gradI = results->at(0);
    auto* gradW = results->at(1);

    ASSERT_EQ(Status::OK(), results->status());

    ASSERT_TRUE(expGradI.isSameShape(gradI));
    ASSERT_TRUE(expGradI.equalsTo(gradI));

    ASSERT_TRUE(expGradW.isSameShape(gradW));
    ASSERT_TRUE(expGradW.equalsTo(gradW));

    delete results;
}

//////////////////////////////////////////////////////////////////////
TEST_F(ConvolutionTests1, depthwise_conv2d_bp_test2) {

    int bS=2, iH=4,iW=3,  iC=2,mC=2,  kH=3,kW=2,  sH=1,sW=1,  pH=0,pW=0,  dH=1,dW=1;
    int       oH=2,oW=2;
    int       oC=iC*mC;
    int paddingMode = 0;             // 1-SAME, 0-VALID;
    int dataFormat  = 1;             // 1-NHWC, 0-NCHW

    auto input    = NDArrayFactory::create<double>('c', {bS, iH, iW, iC});
    auto weights  = NDArrayFactory::create<double>('c', {kH, kW, iC, mC});
    auto bias     = NDArrayFactory::create<double>('c', {oC}, {1,2,3,4});
    auto gradO    = NDArrayFactory::create<double>('c', {bS, oH, oW, oC});

    auto expGradI = NDArrayFactory::create<double>('c', {bS, iH, iW, iC},{0.005, 0.025,0.034, 0.106,0.061, 0.113,0.058, 0.162,0.292, 0.564,0.298, 0.466,0.234, 0.402,0.772, 1.172,0.602, 0.834,0.333, 0.449,0.882, 1.146,0.581, 0.729,
                                                    0.053, 0.137,0.258, 0.458,0.237, 0.353,0.41 , 0.642,1.252, 1.78 ,0.906, 1.202,1.098, 1.394,2.756, 3.412,1.722, 2.082,0.893, 1.073,2.13 , 2.522,1.269, 1.481});
    auto expGradW = NDArrayFactory::create<double>('c', {kH, kW, iC, mC},{2.4 , 2.56,2.72, 2.88,2.4 , 2.56,2.72, 2.88,2.4 , 2.56,2.72, 2.88,2.4 , 2.56,2.72, 2.88,2.4 , 2.56,2.72, 2.88,2.4 , 2.56,2.72, 2.88});

    input = 2.;
    weights.linspace(0.1, 0.1);
    gradO.linspace(0.01, 0.01);

    nd4j::ops::depthwise_conv2d_bp op;
    auto results = op.execute({&input, &weights, &bias, &gradO}, {}, {kH,kW,  sH,sW,  pH,pW,  dH,dW, paddingMode, dataFormat});
    auto* gradI = results->at(0);
    auto* gradW = results->at(1);

    ASSERT_EQ(Status::OK(), results->status());

    ASSERT_TRUE(expGradI.isSameShape(gradI));
    ASSERT_TRUE(expGradI.equalsTo(gradI));

    ASSERT_TRUE(expGradW.isSameShape(gradW));
    ASSERT_TRUE(expGradW.equalsTo(gradW));

    delete results;
}

//////////////////////////////////////////////////////////////////////
TYPED_TEST(TypedConvolutionTests1, conv3d_test1) {

    int bS=2, iD=3,iH=4,iW=3,  iC=4,oC=3,  kD=2,kH=3,kW=2,  sD=1,sH=1,sW=1,  pD=0,pH=0,pW=0,  dD=1,dH=1,dW=1;
    int paddingMode = 1;             // 1-SAME,  0-VALID;
    int dataFormat  = 1;             // 1-NDHWC, 0-NCDHW

    auto input    = NDArrayFactory::create<TypeParam>('c', {bS, iD, iH, iW, iC});
    auto weights  = NDArrayFactory::create<TypeParam>('c', {kD, kH, kW, iC, oC});
    auto expected = NDArrayFactory::create<TypeParam>('c', {2, 3, 4, 3, 3}, {64.,64.,64.,64.,64.,64.,32.,32.,32.,96.,96.,96.,96.,96.,96.,48.,48.,48.,96.,96.,96.,96.,96.,96.,48.,48.,48.,
                                                   64.,64.,64.,64.,64.,64.,32.,32.,32.,64.,64.,64.,64.,64.,64.,32.,32.,32.,96.,96.,96.,96.,96.,96.,48.,48.,48.,
                                                   96.,96.,96.,96.,96.,96.,48.,48.,48.,64.,64.,64.,64.,64.,64.,32.,32.,32.,32.,32.,32.,32.,32.,32.,16.,16.,16.,
                                                   48.,48.,48.,48.,48.,48.,24.,24.,24.,48.,48.,48.,48.,48.,48.,24.,24.,24.,32.,32.,32.,32.,32.,32.,16.,16.,16.,
                                                   64.,64.,64.,64.,64.,64.,32.,32.,32.,96.,96.,96.,96.,96.,96.,48.,48.,48.,96.,96.,96.,96.,96.,96.,48.,48.,48.,
                                                   64.,64.,64.,64.,64.,64.,32.,32.,32.,64.,64.,64.,64.,64.,64.,32.,32.,32.,96.,96.,96.,96.,96.,96.,48.,48.,48.,
                                                   96.,96.,96.,96.,96.,96.,48.,48.,48.,64.,64.,64.,64.,64.,64.,32.,32.,32.,32.,32.,32.,32.,32.,32.,16.,16.,16.,
                                                   48.,48.,48.,48.,48.,48.,24.,24.,24.,48.,48.,48.,48.,48.,48.,24.,24.,24.,32.,32.,32.,32.,32.,32.,16.,16.,16.});
    input = 2.;
    weights = 1.;

    nd4j::ops::conv3dnew op;
    auto results = op.execute({&input, &weights}, {}, {kD,kH,kW,  sD,sH,sW,  pD,pH,pW,  dD,dH,dW, paddingMode, dataFormat});
    auto* output = results->at(0);

    ASSERT_EQ(Status::OK(), results->status());
    ASSERT_TRUE(expected.isSameShape(output));
    ASSERT_TRUE(expected.equalsTo(output));

    delete results;
}


//////////////////////////////////////////////////////////////////////
TYPED_TEST(TypedConvolutionTests1, conv3d_test2) {

    int bS=2, iD=3,iH=4,iW=3,  iC=4,oC=3,  kD=2,kH=3,kW=2,  sD=1,sH=1,sW=1,  pD=0,pH=0,pW=0,  dD=1,dH=1,dW=1;
    int paddingMode = 1;             // 1-SAME,  0-VALID;
    int dataFormat  = 1;             // 1-NDHWC, 0-NCDHW

    auto input    = NDArrayFactory::create<TypeParam>('c', {bS, iD, iH, iW, iC});
    auto weights  = NDArrayFactory::create<TypeParam>('c', {kD, kH, kW, iC, oC});
    auto expected = NDArrayFactory::create<TypeParam>('c', {2, 3, 4, 3, 3}, {534.4,540.8,547.2,534.4,540.8,547.2,248. ,251.2,254.4,686.4,696. ,705.6,686.4,696. ,705.6,314.4,319.2,324. ,686.4,696. ,705.6,686.4,696. ,705.6,314.4,319.2,324. ,
                                                   380.8,387.2,393.6,380.8,387.2,393.6,171.2,174.4,177.6,534.4,540.8,547.2,534.4,540.8,547.2,248. ,251.2,254.4,686.4,696. ,705.6,686.4,696. ,705.6,314.4,319.2,324. ,
                                                   686.4,696. ,705.6,686.4,696. ,705.6,314.4,319.2,324. ,380.8,387.2,393.6,380.8,387.2,393.6,171.2,174.4,177.6,152. ,155.2,158.4,152. ,155.2,158.4, 66.4, 68. , 69.6,
                                                   170.4,175.2,180. ,170.4,175.2,180. , 70.8, 73.2, 75.6,170.4,175.2,180. ,170.4,175.2,180. , 70.8, 73.2, 75.6, 75.2, 78.4, 81.6, 75.2, 78.4, 81.6, 28. , 29.6, 31.2,
                                                   534.4,540.8,547.2,534.4,540.8,547.2,248. ,251.2,254.4,686.4,696. ,705.6,686.4,696. ,705.6,314.4,319.2,324. ,686.4,696. ,705.6,686.4,696. ,705.6,314.4,319.2,324. ,
                                                   380.8,387.2,393.6,380.8,387.2,393.6,171.2,174.4,177.6,534.4,540.8,547.2,534.4,540.8,547.2,248. ,251.2,254.4,686.4,696. ,705.6,686.4,696. ,705.6,314.4,319.2,324. ,
                                                   686.4,696. ,705.6,686.4,696. ,705.6,314.4,319.2,324. ,380.8,387.2,393.6,380.8,387.2,393.6,171.2,174.4,177.6,152. ,155.2,158.4,152. ,155.2,158.4, 66.4, 68. , 69.6,
                                                   170.4,175.2,180. ,170.4,175.2,180. , 70.8, 73.2, 75.6,170.4,175.2,180. ,170.4,175.2,180. , 70.8, 73.2, 75.6, 75.2, 78.4, 81.6, 75.2, 78.4, 81.6, 28. , 29.6, 31.2});
    input = 2.;
    weights.linspace(0.1, 0.1);

    nd4j::ops::conv3dnew op;
    auto results = op.execute({&input, &weights}, {}, {kD,kH,kW,  sD,sH,sW,  pD,pH,pW,  dD,dH,dW, paddingMode, dataFormat});
    auto* output = results->at(0);

    ASSERT_EQ(Status::OK(), results->status());
    ASSERT_TRUE(expected.isSameShape(output));
    ASSERT_TRUE(expected.equalsTo(output));

    delete results;
}

//////////////////////////////////////////////////////////////////////
TYPED_TEST(TypedConvolutionTests1, conv3d_test3) {

    int bS=2, iD=3,iH=4,iW=3,  iC=4,oC=3,  kD=2,kH=3,kW=2,  sD=1,sH=1,sW=1,  pD=0,pH=0,pW=0,  dD=1,dH=1,dW=1;
    int paddingMode = 0;             // 1-SAME,  0-VALID;
    int dataFormat  = 1;             // 1-NDHWC, 0-NCDHW

    auto input    = NDArrayFactory::create<TypeParam>('c', {bS, iD, iH, iW, iC});
    auto weights  = NDArrayFactory::create<TypeParam>('c', {kD, kH, kW, iC, oC});
    auto expected = NDArrayFactory::create<TypeParam>('c', {2, 2, 2, 2, 3},  {686.4,696.,705.6,686.4,696.,705.6,686.4,696.,705.6,686.4,696.,705.6,
                                                    686.4,696.,705.6,686.4,696.,705.6,686.4,696.,705.6,686.4,696.,705.6,
                                                    686.4,696.,705.6,686.4,696.,705.6,686.4,696.,705.6,686.4,696.,705.6,
                                                    686.4,696.,705.6,686.4,696.,705.6,686.4,696.,705.6,686.4,696.,705.6});
    input = 2.;
    weights.linspace(0.1, 0.1);

    nd4j::ops::conv3dnew op;
    auto results = op.execute({&input, &weights}, {}, {kD,kH,kW,  sD,sH,sW,  pD,pH,pW,  dD,dH,dW, paddingMode, dataFormat});
    auto* output = results->at(0);

    ASSERT_EQ(Status::OK(), results->status());
    ASSERT_TRUE(expected.isSameShape(output));
    ASSERT_TRUE(expected.equalsTo(output));

    delete results;
}


//////////////////////////////////////////////////////////////////////
TYPED_TEST(TypedConvolutionTests1, conv3d_test4) {

    int bS=2, iD=3,iH=4,iW=3,  iC=4,oC=3,  kD=2,kH=3,kW=2,  sD=1,sH=1,sW=1,  pD=0,pH=0,pW=0,  dD=1,dH=1,dW=1;
    int paddingMode = 0;             // 1-SAME,  0-VALID;
    int dataFormat =  0;             // 1-NDHWC, 0-NCDHW

    auto input    = NDArrayFactory::create<TypeParam>('c', {bS, iC, iD, iH, iW});
    auto weights  = NDArrayFactory::create<TypeParam>('c', {kD, kH, kW, iC, oC});
    auto expected = NDArrayFactory::create<TypeParam>('c', {2, 3, 2, 2, 2});
    input = 2.;
    weights = 0.5;
    expected = 48.;

    nd4j::ops::conv3dnew op;
    auto results = op.execute({&input, &weights}, {}, {kD,kH,kW,  sD,sH,sW,  pD,pH,pW,  dD,dH,dW, paddingMode, dataFormat});
    auto* output = results->at(0);

    ASSERT_EQ(Status::OK(), results->status());
    ASSERT_TRUE(expected.isSameShape(output));
    ASSERT_TRUE(expected.equalsTo(output));

    delete results;
}

////////////////////////////////////////////////////////////////////
TYPED_TEST(TypedConvolutionTests1, conv3d_test5) {

    int bS=2, iD=3,iH=4,iW=3,  iC=4,oC=3,  kD=2,kH=3,kW=2,  sD=1,sH=1,sW=1,  pD=0,pH=0,pW=0,  dD=1,dH=1,dW=1;
    int paddingMode = 0;             // 1-SAME,  0-VALID;
    int dataFormat  = 0;             // 1-NDHWC, 0-NCDHW

    auto input    = NDArrayFactory::create<TypeParam>('c', {bS, iC, iD, iH, iW});
    auto weights  = NDArrayFactory::create<TypeParam>('c', {kD, kH, kW, iC, oC});
    auto bias     = NDArrayFactory::create<TypeParam>('c', {oC});
    auto expected = NDArrayFactory::create<TypeParam>('c', {2, 3, 2, 2, 2});

    input = 2.;
    weights = 0.5;
    expected = 49.;
    bias = 1.;

    nd4j::ops::conv3dnew op;
    auto results = op.execute({&input, &weights, &bias}, {}, {kD,kH,kW,  sD,sH,sW,  pD,pH,pW,  dD,dH,dW, paddingMode, dataFormat});
    auto* output = results->at(0);

    // output->printIndexedBuffer();

    ASSERT_EQ(Status::OK(), results->status());
    ASSERT_TRUE(expected.isSameShape(output));
    ASSERT_TRUE(expected.equalsTo(output));

    delete results;
}

////////////////////////////////////////////////////////////////////
TYPED_TEST(TypedConvolutionTests1, conv3d_test6) {

    int bS=2, iD=3,iH=4,iW=3,  iC=4,oC=3,  kD=2,kH=3,kW=2,  sD=1,sH=1,sW=1,  pD=0,pH=0,pW=0,  dD=1,dH=1,dW=1;
    int paddingMode = 0;             // 1-SAME,  0-VALID;
    int dataFormat  = 0;             // 1-NDHWC, 0-NCDHW

    auto input    = NDArrayFactory::create<TypeParam>('c', {bS, iC, iD, iH, iW});
    auto weights  = NDArrayFactory::create<TypeParam>('c', {kD, kH, kW, iC, oC});
    auto bias     = NDArrayFactory::create<TypeParam>('c', {oC},{1,2,3});
    auto expected = NDArrayFactory::create<TypeParam>('c', {2, 3, 2, 2, 2},{49., 49.,49., 49., 49., 49.,49., 49., 50., 50.,50., 50., 50., 50.,50., 50.,
                                                  51., 51.,51., 51., 51., 51.,51., 51., 49., 49.,49., 49., 49., 49.,49., 49.,
                                                  50., 50.,50., 50., 50., 50.,50., 50., 51., 51.,51., 51., 51., 51.,51., 51.});
    input = 2.;
    weights = 0.5;

    nd4j::ops::conv3dnew op;
    auto results = op.execute({&input, &weights, &bias}, {}, {kD,kH,kW,  sD,sH,sW,  pD,pH,pW,  dD,dH,dW, paddingMode, dataFormat});
    auto* output = results->at(0);

    // output->printIndexedBuffer();

    ASSERT_EQ(Status::OK(), results->status());
    ASSERT_TRUE(expected.isSameShape(output));
    ASSERT_TRUE(expected.equalsTo(output));

    delete results;
}

////////////////////////////////////////////////////////////////////
TYPED_TEST(TypedConvolutionTests1, conv3d_test7) {

    int bS=2, iD=3,iH=4,iW=3,  iC=4,oC=3,  kD=2,kH=3,kW=2,  sD=1,sH=1,sW=1,  pD=0,pH=0,pW=0,  dD=1,dH=1,dW=1;
    int paddingMode = 0;             // 1-SAME,  0-VALID;
    int dataFormat  = 0;             // 1-NDHWC, 0-NCDHW

    auto input    = NDArrayFactory::create<TypeParam>('c', {bS, iC, iD, iH, iW});
    auto weights  = NDArrayFactory::create<TypeParam>('c', {oC, iC, kD, kH, kW});
    auto bias     = NDArrayFactory::create<TypeParam>('c', {oC},{1,2,3});
    auto expected = NDArrayFactory::create<TypeParam>('c', {2, 3, 2, 2, 2},{236.2, 236.2, 236.2, 236.2, 236.2, 236.2, 236.2, 236.2, 698. , 698. , 698. , 698. ,
                                                  698. , 698. , 698. , 698. ,1159.8,1159.8,1159.8,1159.8,1159.8,1159.8,1159.8,1159.8,
                                                  236.2, 236.2, 236.2, 236.2, 236.2, 236.2, 236.2, 236.2, 698. , 698. , 698. , 698. ,
                                                  698. , 698. , 698. , 698. ,1159.8,1159.8,1159.8,1159.8,1159.8,1159.8,1159.8,1159.8});
    input = 2.;
    weights.linspace(0.1, 0.1);
    weights.permutei({2, 3, 4, 1, 0});

    nd4j::ops::conv3dnew op;
    auto results = op.execute({&input, &weights, &bias}, {}, {kD,kH,kW,  sD,sH,sW,  pD,pH,pW,  dD,dH,dW, paddingMode, dataFormat});
    auto* output = results->at(0);

    // output->printIndexedBuffer();

    ASSERT_EQ(Status::OK(), results->status());
    ASSERT_TRUE(expected.isSameShape(output));
    ASSERT_TRUE(expected.equalsTo(output));

    delete results;
}

////////////////////////////////////////////////////////////////////
TYPED_TEST(TypedConvolutionTests1, conv3d_test8) {

    int bS=2, iD=3,iH=4,iW=3,  iC=4,oC=3,  kD=2,kH=3,kW=2,  sD=1,sH=1,sW=1,  pD=0,pH=0,pW=0,  dD=1,dH=1,dW=1;
    int paddingMode = 0;             // 1-SAME,  0-VALID;
    int dataFormat  = 0;             // 1-NDHWC, 0-NCDHW

    auto input    = NDArrayFactory::create<TypeParam>('c', {bS, iC, iD, iH, iW});
    auto weights  = NDArrayFactory::create<TypeParam>('c', {oC, iC, kD, kH, kW});
    auto expected = NDArrayFactory::create<TypeParam>('c', {2, 3, 2, 2, 2},{235.2, 235.2, 235.2, 235.2, 235.2, 235.2, 235.2, 235.2, 696. , 696. , 696. , 696. , 696. , 696. , 696. , 696. ,
                                                  1156.8,1156.8,1156.8,1156.8,1156.8,1156.8,1156.8,1156.8, 235.2, 235.2, 235.2, 235.2, 235.2, 235.2, 235.2, 235.2,
                                                  696. , 696. , 696. , 696. , 696. , 696. , 696. , 696. ,1156.8,1156.8,1156.8,1156.8,1156.8,1156.8,1156.8,1156.8});
    input = 2.;
    weights.linspace(0.1, 0.1);
    weights.permutei({2, 3, 4, 1, 0});

    nd4j::ops::conv3dnew op;
    auto results = op.execute({&input, &weights}, {}, {kD,kH,kW,  sD,sH,sW,  pD,pH,pW,  dD,dH,dW, paddingMode, dataFormat});
    auto* output = results->at(0);

    ASSERT_EQ(Status::OK(), results->status());
    ASSERT_TRUE(expected.isSameShape(output));
    ASSERT_TRUE(expected.equalsTo(output));

    delete results;
}

//////////////////////////////////////////////////////////////////////
TYPED_TEST(TypedConvolutionTests1, conv3d_test9) {
    auto x = NDArrayFactory::create<TypeParam>('c', {4, 2, 28, 28, 3});
    auto y = NDArrayFactory::create<TypeParam>('c', {2, 5, 5, 3, 4});
    auto e = NDArrayFactory::create<TypeParam>('c', {4, 1, 7, 10, 4});

    nd4j::ops::conv3dnew op;
    auto result = op.execute({&x, &y}, {}, {2,5,5, 5,4,3, 0,0,0, 1,1,1, 1,1});
    ASSERT_EQ(Status::OK(), result->status());

    auto z = result->at(0);

    ASSERT_TRUE(e.isSameShape(z));

    delete result;
}

TYPED_TEST(TypedConvolutionTests1, conv3d_test10) {
    auto x = NDArrayFactory::create<TypeParam>('c', {4, 2, 28, 28, 3});
    auto w = NDArrayFactory::create<TypeParam>('c', {2, 5, 5, 3, 4});
    auto exp = NDArrayFactory::create<TypeParam>('c', {4, 1, 7, 10, 4});

    nd4j::ops::conv3dnew op;
    auto result = op.execute({&x, &w}, {}, {2,5,5, 5,4,3, 0,0,0, 1,1,1, 1,1});
    ASSERT_EQ(Status::OK(), result->status());

    ShapeList shapeList({x.shapeInfo(), w.shapeInfo()});
    ContextPrototype proto;
    Context ctx(1);
    ctx.getIArguments()->push_back(2);
    ctx.getIArguments()->push_back(5);
    ctx.getIArguments()->push_back(5);

    ctx.getIArguments()->push_back(5);
    ctx.getIArguments()->push_back(4);
    ctx.getIArguments()->push_back(3);

    ctx.getIArguments()->push_back(0);
    ctx.getIArguments()->push_back(0);
    ctx.getIArguments()->push_back(0);

    ctx.getIArguments()->push_back(1);
    ctx.getIArguments()->push_back(1);
    ctx.getIArguments()->push_back(1);

    ctx.getIArguments()->push_back(0);
    ctx.getIArguments()->push_back(1);  // previous variant was "ctx.getIArguments()->push_back(0)" and this caused fail

    auto shapes = op.calculateOutputShape(&shapeList, ctx);
    ASSERT_EQ(1, shapes->size());

    auto s = shapes->at(0);

    auto z = result->at(0);
    // z->printShapeInfo("z shape");

    ASSERT_TRUE(exp.isSameShape(z));

    delete result;

    shapes->destroy();
    delete shapes;
}

//////////////////////////////////////////////////////////////////////
TYPED_TEST(TypedConvolutionTests1, pointwise_conv2d_test1) {

    int bS=2, iH=4,iW=3,  iC=4,oC=3;

    int dataFormat = 1;           // 1-NHWC, 0-NCHW

    auto input    = NDArrayFactory::create<TypeParam>('c', {bS, iH, iW, iC});
    auto weights  = NDArrayFactory::create<TypeParam>('c', {1,   1, iC, oC});
    auto bias     = NDArrayFactory::create<TypeParam>('c', {oC});


    auto expOutput = NDArrayFactory::create<TypeParam>('c', {bS, iH, iW, oC},{ 5.4, 6.2, 7.0, 5.4, 6.2, 7.0, 5.4, 6.2, 7.0, 5.4, 6.2, 7.0, 5.4, 6.2, 7.0, 5.4, 6.2, 7.0, 5.4, 6.2,
                                                      7.0, 5.4, 6.2, 7.0, 5.4, 6.2, 7.0, 5.4, 6.2, 7.0, 5.4, 6.2, 7.0, 5.4, 6.2, 7.0, 5.4, 6.2, 7.0, 5.4,
                                                      6.2, 7.0, 5.4, 6.2, 7.0, 5.4, 6.2, 7.0, 5.4, 6.2, 7.0, 5.4, 6.2, 7.0, 5.4, 6.2, 7.0, 5.4, 6.2, 7.0,
                                                      5.4, 6.2, 7.0, 5.4, 6.2, 7.0, 5.4, 6.2, 7.0, 5.4, 6.2, 7.0});
    input = 2.;
    weights.linspace(0.1, 0.1);
    bias = 1.;

    nd4j::ops::pointwise_conv2d op;
    auto results = op.execute({&input, &weights, &bias}, {}, {dataFormat});
    auto* output = results->at(0);

    ASSERT_EQ(Status::OK(), results->status());
    ASSERT_TRUE(expOutput.isSameShape(output));
    ASSERT_TRUE(expOutput.equalsTo(output));

    delete results;
}

//////////////////////////////////////////////////////////////////////
TYPED_TEST(TypedConvolutionTests1, conv3d_test11) {

    int bS=1, iD=2,iH=2,iW=2,  iC=1,oC=1,  kD=2,kH=2,kW=2,  sD=1,sH=1,sW=1,  pD=0,pH=0,pW=0,  dD=1,dH=1,dW=1;
    int paddingMode = 1;             // 1-SAME,  0-VALID;
    int dataFormat  = 0;             // 1-NDHWC, 0-NCDHW

    auto input    = NDArrayFactory::create<TypeParam>('c', {bS, iC, iD, iH, iW});
    auto weights  = NDArrayFactory::create<TypeParam>('c', {kD, kH, kW, iC, oC});

    input = 2.;
    weights = 1.;

    nd4j::ops::conv3dnew op;
    auto results = op.execute({&input, &weights}, {}, {kD,kH,kW,  sD,sH,sW,  pD,pH,pW,  dD,dH,dW, paddingMode, dataFormat});
    auto* output = results->at(0);

    ASSERT_EQ(Status::OK(), results->status());

    delete results;
}

//////////////////////////////////////////////////////////////////////
TYPED_TEST(TypedConvolutionTests1, conv3d_test12) {

    int bS=5, iD=4,iH=14,iW=14,  iC=1,oC=1,  kD=2,kH=2,kW=2,  sD=1,sH=1,sW=1,  pD=0,pH=0,pW=0,  dD=1,dH=1,dW=1;
    int       oD=3,oH=13,oW=13;
    int paddingMode = 0;             // 1-SAME,  0-VALID;
    int dataFormat  = 0;             // 1-NDHWC, 0-NCDHW

    auto input    = NDArrayFactory::create<TypeParam>('c', {bS, iC, iD, iH, iW});
    auto weights  = NDArrayFactory::create<TypeParam>('c', {kD, kH, kW, iC, oC});
    auto expected = NDArrayFactory::create<TypeParam>('c', {bS, oC, oD, oH, oW});

    input = 2.;
    weights = 1.;

    nd4j::ops::conv3dnew op;
    auto results = op.execute({&input, &weights}, {}, {kD,kH,kW,  sD,sH,sW,  pD,pH,pW,  dD,dH,dW, paddingMode, dataFormat});
    auto* output = results->at(0);

    ASSERT_EQ(Status::OK(), results->status());
    ASSERT_TRUE(output->isSameShape(&expected));

    delete results;
}

//////////////////////////////////////////////////////////////////////
TEST_F(ConvolutionTests1, vol2col_test1) {

    int bS=2, iD=2,iH=3,iW=2,  iC=3,oC=2,  kD=2,kH=3,kW=2,  sD=1,sH=1,sW=1,  pD=0,pH=0,pW=0,  dD=1,dH=1,dW=1;
    int       oD=2,oH=3,oW=2;

    NDArray volume('c', {bS, iC, iD, iH, iW}, nd4j::DataType::FLOAT32);
    NDArray columns('c', {bS, iC, kD, kH, kW, oD, oH, oW}, nd4j::DataType::FLOAT32);

    columns = -1.;
    volume.linspace(1);

    NDArray columnsExpected('c', {bS, iC, kD, kH, kW, oD, oH, oW}, {1., 2., 3., 4., 5., 6., 7., 8., 9., 10., 11., 12., 2., 0., 4., 0., 6.,0., 8., 0., 10., 0., 12., 0., 3., 4., 5., 6., 0., 0., 9., 10., 11., 12., 0., 0., 4., 0., 6., 0., 0., 0., 10., 0., 12., 0., 0., 0., 5., 6.,
0., 0., 0., 0., 11., 12., 0., 0., 0., 0., 6., 0., 0., 0., 0., 0., 12., 0., 0., 0., 0., 0., 7., 8., 9., 10., 11., 12., 0., 0., 0., 0., 0., 0., 8., 0., 10., 0., 12., 0., 0., 0., 0., 0., 0., 0., 9., 10., 11., 12., 0., 0., 0., 0., 0., 0., 0., 0., 10., 0., 12., 0., 0., 0., 0., 0.,
0., 0., 0., 0., 11., 12., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 12., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 13., 14., 15., 16., 17.,18., 19., 20., 21., 22., 23., 24., 14., 0., 16., 0., 18., 0., 20., 0., 22., 0., 24., 0., 15., 16., 17., 18., 0., 0., 21., 22., 23., 24., 0.,
0., 16., 0., 18., 0., 0., 0., 22., 0., 24., 0., 0., 0., 17., 18., 0., 0., 0., 0., 23., 24., 0., 0., 0., 0., 18., 0., 0., 0., 0., 0., 24., 0., 0., 0., 0., 0., 19., 20., 21., 22., 23., 24., 0., 0., 0., 0., 0., 0., 20., 0., 22., 0., 24., 0., 0., 0., 0., 0., 0., 0., 21., 22., 23.,
24., 0., 0., 0., 0., 0., 0., 0., 0., 22., 0., 24., 0., 0., 0., 0., 0., 0., 0., 0., 0., 23., 24., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 24.,0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 25., 26., 27., 28., 29., 30., 31., 32., 33., 34., 35., 36., 26., 0., 28., 0., 30., 0., 32., 0.,
34., 0., 36., 0., 27., 28., 29., 30., 0., 0., 33., 34., 35., 36., 0., 0., 28., 0., 30., 0., 0., 0., 34., 0., 36., 0., 0., 0., 29., 30., 0., 0., 0., 0., 35., 36., 0., 0., 0., 0., 30., 0., 0., 0., 0., 0., 36., 0., 0., 0., 0., 0., 31., 32., 33., 34., 35., 36., 0., 0., 0., 0., 0.,
0., 32., 0., 34., 0., 36., 0., 0., 0., 0., 0., 0., 0., 33., 34., 35., 36., 0., 0., 0., 0., 0., 0., 0., 0., 34., 0., 36., 0., 0., 0., 0., 0., 0., 0., 0., 0., 35., 36., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 36., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 37., 38., 39., 40.,
41., 42., 43., 44., 45., 46., 47., 48., 38., 0., 40., 0., 42., 0., 44., 0., 46., 0., 48., 0., 39., 40., 41., 42., 0., 0., 45., 46., 47., 48., 0., 0., 40., 0., 42., 0., 0., 0., 46., 0., 48., 0., 0., 0., 41., 42., 0., 0., 0., 0., 47., 48., 0., 0., 0., 0., 42., 0., 0., 0., 0.,
0., 48., 0., 0., 0., 0., 0., 43., 44., 45., 46., 47., 48., 0., 0., 0., 0., 0., 0., 44., 0., 46., 0., 48., 0., 0., 0., 0., 0., 0., 0., 45., 46., 47., 48., 0., 0., 0., 0., 0., 0., 0., 0., 46., 0., 48., 0., 0., 0., 0., 0., 0., 0., 0., 0., 47., 48., 0., 0., 0., 0., 0., 0., 0., 0.,
0., 0., 48., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 49., 50., 51., 52., 53., 54., 55., 56., 57., 58., 59., 60., 50., 0., 52., 0., 54.,0., 56., 0., 58., 0., 60., 0., 51., 52., 53., 54., 0., 0., 57., 58., 59., 60., 0., 0., 52., 0., 54., 0., 0., 0., 58., 0., 60., 0., 0., 0.,
53., 54., 0., 0., 0., 0., 59., 60., 0., 0., 0., 0., 54., 0., 0., 0., 0., 0., 60., 0., 0., 0., 0., 0., 55., 56., 57., 58., 59., 60., 0., 0.,0., 0., 0., 0., 56., 0., 58., 0., 60., 0., 0., 0., 0., 0., 0., 0., 57., 58., 59., 60., 0., 0., 0., 0., 0., 0., 0., 0., 58., 0., 60., 0.,
0., 0., 0., 0., 0., 0., 0., 0., 59., 60., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 60., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 61., 62., 63., 64., 65., 66., 67., 68., 69., 70., 71., 72., 62., 0., 64., 0., 66., 0., 68., 0., 70., 0., 72., 0., 63., 64., 65., 66., 0., 0., 69.,
70., 71., 72., 0., 0., 64., 0., 66., 0., 0., 0., 70., 0., 72., 0., 0., 0., 65., 66., 0., 0., 0., 0., 71., 72., 0., 0., 0., 0., 66., 0., 0., 0., 0., 0., 72., 0., 0., 0., 0., 0., 67., 68., 69., 70., 71., 72., 0., 0., 0., 0., 0., 0., 68., 0., 70., 0., 72., 0., 0., 0., 0., 0., 0.,
0., 69., 70., 71., 72., 0., 0., 0., 0., 0., 0., 0., 0., 70., 0., 72., 0., 0., 0., 0., 0., 0., 0., 0., 0., 71., 72., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 72., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.}, nd4j::DataType::FLOAT32);

    graph::Context context(1);
    nd4j::ops::ConvolutionUtils::vol2col(context, volume, columns, sD, sH, sW, pD, pH, pW, dD, dH, dW);
    // columns.printBuffer();

    ASSERT_TRUE(columns.equalsTo(columnsExpected));
}

//////////////////////////////////////////////////////////////////////
TEST_F(ConvolutionTests1, vol2col_test2) {

    int bS=2, iD=2,iH=3,iW=2,  iC=3,oC=2,  kD=2,kH=3,kW=2,  sD=1,sH=1,sW=1,  pD=0,pH=0,pW=0,  dD=1,dH=1,dW=1;
    int       oD=2,oH=3,oW=2;

    auto volume  = NDArrayFactory::create<float>('c', {iD, bS, iH, iC, iW});
    volume.permutei({1, 3, 0, 2, 4});
    volume.linspace(1);

    auto columns = NDArrayFactory::create<float>('c', {kD, iC, kH, oW, kW, bS, oD, oH});
    columns.permutei({5, 1, 0, 2, 4, 6, 7, 3});
    columns = -1.;
    auto columnsExpected = NDArrayFactory::create<float>('c', {bS, iC, kD, kH, kW, oD, oH, oW}, {1., 2., 3., 4., 5., 6., 7., 8., 9.,
10., 11., 12., 2., 0., 4., 0., 6., 0., 8., 0., 10., 0., 12., 0., 3., 4., 5., 6., 0., 0., 9., 10., 11., 12., 0., 0., 4., 0., 6., 0., 0.,0., 10., 0., 12., 0., 0., 0., 5., 6., 0., 0., 0., 0., 11., 12., 0., 0., 0., 0., 6., 0., 0., 0., 0., 0., 12., 0., 0., 0., 0., 0., 7., 8.,
9., 10., 11., 12., 0., 0., 0., 0., 0., 0., 8., 0., 10., 0., 12., 0., 0., 0., 0., 0., 0., 0., 9., 10., 11., 12., 0., 0., 0., 0., 0., 0., 0., 0., 10., 0., 12., 0., 0., 0., 0., 0., 0., 0., 0., 0., 11., 12., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 12., 0., 0., 0., 0., 0.,
0., 0., 0., 0., 0., 0., 13., 14., 15., 16., 17., 18., 19., 20., 21., 22., 23., 24., 14., 0., 16., 0., 18., 0., 20., 0., 22., 0., 24., 0., 15., 16., 17., 18., 0., 0., 21., 22., 23., 24., 0., 0., 16., 0., 18., 0., 0., 0., 22., 0., 24., 0., 0., 0., 17., 18., 0., 0., 0., 0.,
23., 24., 0., 0., 0., 0., 18., 0., 0., 0., 0., 0., 24., 0., 0., 0., 0., 0., 19., 20., 21., 22., 23., 24., 0., 0., 0., 0., 0., 0., 20., 0., 22., 0., 24., 0., 0., 0., 0., 0., 0., 0., 21., 22., 23., 24., 0., 0., 0., 0., 0., 0., 0., 0., 22., 0., 24., 0., 0., 0., 0., 0., 0.,
0., 0., 0., 23., 24., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 24., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 25., 26., 27., 28., 29., 30., 31., 32., 33., 34., 35., 36., 26., 0., 28., 0., 30., 0., 32., 0., 34., 0., 36., 0., 27., 28., 29., 30., 0., 0., 33., 34., 35., 36.,
0., 0., 28., 0., 30., 0., 0., 0., 34., 0., 36., 0., 0., 0., 29., 30., 0., 0., 0., 0., 35., 36., 0., 0., 0., 0., 30., 0., 0., 0., 0., 0., 36., 0., 0., 0., 0., 0., 31., 32., 33., 34., 35., 36., 0., 0., 0., 0., 0., 0., 32., 0., 34., 0., 36., 0., 0., 0., 0., 0., 0., 0., 33.,
34., 35., 36., 0., 0., 0., 0., 0., 0., 0., 0., 34., 0., 36., 0., 0., 0., 0., 0., 0., 0., 0., 0., 35., 36., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 36., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 37., 38., 39., 40., 41., 42., 43., 44., 45., 46., 47., 48., 38., 0., 40.,
0., 42., 0., 44., 0., 46., 0., 48., 0., 39., 40., 41., 42., 0., 0., 45., 46., 47., 48., 0., 0., 40., 0., 42., 0., 0., 0., 46., 0., 48., 0., 0., 0., 41., 42., 0., 0., 0., 0., 47., 48., 0., 0., 0., 0., 42., 0., 0., 0., 0., 0., 48., 0., 0., 0., 0., 0., 43., 44., 45., 46., 47.,
48., 0., 0., 0., 0., 0., 0., 44., 0., 46., 0., 48., 0., 0., 0., 0., 0., 0., 0., 45., 46., 47., 48., 0., 0., 0., 0., 0., 0., 0., 0., 46., 0., 48., 0., 0., 0., 0., 0., 0., 0., 0., 0., 47., 48., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 48., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
0., 0., 49., 50., 51., 52., 53., 54., 55., 56., 57., 58., 59., 60., 50., 0., 52., 0., 54., 0., 56., 0., 58., 0., 60., 0., 51., 52., 53., 54., 0., 0., 57., 58., 59., 60., 0., 0., 52., 0., 54., 0., 0., 0., 58., 0., 60., 0., 0., 0., 53., 54., 0., 0., 0., 0., 59., 60., 0., 0.,
0., 0., 54., 0., 0., 0., 0., 0., 60., 0., 0., 0., 0., 0., 55., 56., 57., 58., 59., 60., 0., 0., 0., 0., 0., 0., 56., 0., 58., 0., 60., 0., 0., 0., 0., 0., 0., 0., 57., 58., 59., 60., 0., 0., 0., 0., 0., 0., 0., 0., 58., 0., 60., 0., 0., 0., 0., 0., 0., 0., 0., 0., 59., 60.,
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 60., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 61., 62., 63., 64., 65., 66., 67., 68., 69., 70., 71., 72., 62., 0., 64., 0., 66., 0., 68., 0., 70., 0., 72., 0., 63., 64., 65., 66., 0., 0., 69., 70., 71., 72., 0., 0., 64., 0., 66.,
0., 0., 0., 70., 0., 72., 0., 0., 0., 65., 66., 0., 0., 0., 0., 71., 72., 0., 0., 0., 0., 66., 0., 0., 0., 0., 0., 72., 0., 0., 0., 0., 0., 67., 68., 69., 70., 71., 72., 0., 0., 0., 0., 0., 0., 68., 0., 70., 0., 72., 0., 0., 0., 0., 0., 0., 0., 69., 70., 71., 72., 0., 0.,
0., 0., 0., 0., 0., 0., 70., 0., 72., 0., 0., 0., 0., 0., 0., 0., 0., 0., 71., 72., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 72., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.});

    graph::Context context(1);
    nd4j::ops::ConvolutionUtils::vol2col(context, volume, columns, sD, sH, sW, pD, pH, pW, dD, dH, dW);
        // columns.printBuffer();

    ASSERT_TRUE(columns.equalsTo(columnsExpected));
}

//////////////////////////////////////////////////////////////////////
TEST_F(ConvolutionTests1, col2im_test1) {

    int bS=2, iH=2,iW=2,  iC=2,   kH=2,kW=2,  sD=1,sH=1,sW=1,  pD=0,pH=0,pW=0,  dD=1,dH=1,dW=1;
    int       oH=2,oW=2;

    auto image  = NDArrayFactory::create<float>('c', {bS, iC, iH, iW});
    image = -2.;

    auto columns = NDArrayFactory::create<float>('c', {bS, iC, kH, kW, oH, oW});
    columns.linspace(1);

    auto imageExpected = NDArrayFactory::create<float>('c', {bS, iC, iH, iW}, {1., 7., 12., 34., 17., 39., 44., 98., 33., 71., 76., 162., 49., 103., 108., 226.});

    LaunchContext ctx;
    nd4j::ops::helpers::col2im(ctx, columns, image, sH, sW, pH, pW, iH, iW, dH, dW);

    ASSERT_TRUE(image.equalsTo(imageExpected));
}


//////////////////////////////////////////////////////////////////////
TEST_F(ConvolutionTests1, upsampling2d_test1) {

    const int bS=3,  iH=2,iW=2,  iC=3;
    const int factorH=2, factorW=3;
    const int isNCHW = 0;                    // data format, default is NCHW

    auto input  = NDArrayFactory::create<float>('c', {bS, iH, iW, iC});
    input.linspace(1);

    auto expOutput = NDArrayFactory::create<float>('c', {bS, iH*factorH, iW*factorW, iC}, {1.,  2.,  3., 1.,  2.,  3., 1.,  2.,  3., 4.,  5.,  6., 4.,  5.,  6., 4.,  5.,  6., 1.,  2.,  3., 1.,  2.,  3., 1.,  2.,  3., 4.,  5.,  6., 4.,  5.,  6., 4.,  5.,  6.,
                                         7.,  8.,  9., 7.,  8.,  9., 7.,  8.,  9.,10., 11., 12.,10., 11., 12.,10., 11., 12., 7.,  8.,  9., 7.,  8.,  9., 7.,  8.,  9.,10., 11., 12.,10., 11., 12.,10., 11., 12.,
                                        13., 14., 15.,13., 14., 15.,13., 14., 15.,16., 17., 18.,16., 17., 18.,16., 17., 18.,13., 14., 15.,13., 14., 15.,13., 14., 15.,16., 17., 18.,16., 17., 18.,16., 17., 18.,
                                        19., 20., 21.,19., 20., 21.,19., 20., 21.,22., 23., 24.,22., 23., 24.,22., 23., 24.,19., 20., 21.,19., 20., 21.,19., 20., 21.,22., 23., 24.,22., 23., 24.,22., 23., 24.,
                                        25., 26., 27.,25., 26., 27.,25., 26., 27.,28., 29., 30.,28., 29., 30.,28., 29., 30.,25., 26., 27.,25., 26., 27.,25., 26., 27.,28., 29., 30.,28., 29., 30.,28., 29., 30.,
                                        31., 32., 33.,31., 32., 33.,31., 32., 33.,34., 35., 36.,34., 35., 36.,34., 35., 36.,31., 32., 33.,31., 32., 33.,31., 32., 33.,34., 35., 36.,34., 35., 36.,34., 35., 36.});

    nd4j::ops::upsampling2d op;
    auto results = op.execute({&input}, {}, {factorH, factorW, isNCHW});
    auto* output = results->at(0);

    ASSERT_EQ(Status::OK(), results->status());
    ASSERT_TRUE(expOutput.isSameShape(output));
    ASSERT_TRUE(expOutput.equalsTo(output));

    delete results;
}

//////////////////////////////////////////////////////////////////////
TEST_F(ConvolutionTests1, upsampling2d_test2) {

    const int bS=3,  iH=2,iW=2,  iC=3;
    const int factorH=2, factorW=3;
    const int isNCHW = 1;                    // data format, default is NCHW

    auto input  = NDArrayFactory::create<float>('c', {bS, iC, iH, iW});
    input.linspace(1);

    auto expOutput = NDArrayFactory::create<float>('c', {bS, iC, iH*factorH, iW*factorW}, {1.,  1.,  1.,  2.,  2.,  2., 1.,  1.,  1.,  2.,  2.,  2., 3.,  3.,  3.,  4.,  4.,  4., 3.,  3.,  3.,  4.,  4.,  4.,
                                 5.,  5.,  5.,  6.,  6.,  6., 5.,  5.,  5.,  6.,  6.,  6., 7.,  7.,  7.,  8.,  8.,  8., 7.,  7.,  7.,  8.,  8.,  8., 9.,  9.,  9., 10., 10., 10., 9.,  9.,  9., 10., 10., 10.,11., 11., 11., 12., 12., 12.,11., 11., 11., 12., 12., 12.,
                                13., 13., 13., 14., 14., 14.,13., 13., 13., 14., 14., 14.,15., 15., 15., 16., 16., 16.,15., 15., 15., 16., 16., 16.,17., 17., 17., 18., 18., 18.,17., 17., 17., 18., 18., 18.,19., 19., 19., 20., 20., 20.,19., 19., 19., 20., 20., 20.,
                                21., 21., 21., 22., 22., 22.,21., 21., 21., 22., 22., 22.,23., 23., 23., 24., 24., 24.,23., 23., 23., 24., 24., 24.,25., 25., 25., 26., 26., 26.,25., 25., 25., 26., 26., 26.,27., 27., 27., 28., 28., 28.,27., 27., 27., 28., 28., 28.,
                                29., 29., 29., 30., 30., 30.,29., 29., 29., 30., 30., 30.,31., 31., 31., 32., 32., 32.,31., 31., 31., 32., 32., 32.,
                                33., 33., 33., 34., 34., 34.,33., 33., 33., 34., 34., 34.,35., 35., 35., 36., 36., 36.,35., 35., 35., 36., 36., 36.});

    nd4j::ops::upsampling2d op;
    auto results = op.execute({&input}, {}, {factorH, factorW, isNCHW});
    auto* output = results->at(0);

    ASSERT_EQ(Status::OK(), results->status());

    ASSERT_TRUE(expOutput.isSameShape(output));
    ASSERT_TRUE(expOutput.equalsTo(output));

    delete results;
}

//////////////////////////////////////////////////////////////////////
TEST_F(ConvolutionTests1, upsampling3d_test1) {

    const int bS=3,  iD=2,iH=2,iW=2,  iC=3;
    const int factorD=2,factorH=3,factorW=2;
    const int isNCDHW = 0;                    // data format, default is NCHW

    auto input  = NDArrayFactory::create<float>('c', {bS, iD, iH, iW, iC});
    input.linspace(1);

    auto expOutput = NDArrayFactory::create<float>('c', {bS, iD*factorD, iH*factorH, iW*factorW, iC}, {1.,  2.,  3., 1.,  2.,  3., 4.,  5.,  6., 4.,  5.,  6., 1.,  2.,  3., 1.,  2.,  3., 4.,  5.,  6., 4.,  5.,  6., 1.,  2.,  3., 1.,  2.,  3., 4.,  5.,  6., 4.,  5.,  6., 7.,  8.,  9., 7.,  8.,  9.,10., 11., 12.,10., 11., 12., 7.,  8.,  9., 7.,  8.,  9.,10., 11., 12.,10., 11., 12.,
             7.,  8.,  9., 7.,  8.,  9.,10., 11., 12.,10., 11., 12., 1.,  2.,  3., 1.,  2.,  3., 4.,  5.,  6., 4.,  5.,  6., 1.,  2.,  3., 1.,  2.,  3., 4.,  5.,  6., 4.,  5.,  6., 1.,  2.,  3., 1.,  2.,  3., 4.,  5.,  6., 4.,  5.,  6., 7.,  8.,  9., 7.,  8.,  9.,10., 11., 12.,10., 11., 12.,
             7.,  8.,  9., 7.,  8.,  9.,10., 11., 12.,10., 11., 12., 7.,  8.,  9., 7.,  8.,  9.,10., 11., 12.,10., 11., 12.,13., 14., 15.,13., 14., 15.,16., 17., 18.,16., 17., 18.,13., 14., 15.,13., 14., 15.,16., 17., 18.,16., 17., 18.,13., 14., 15.,13., 14., 15.,16., 17., 18.,16., 17., 18.,
            19., 20., 21.,19., 20., 21.,22., 23., 24.,22., 23., 24.,19., 20., 21.,19., 20., 21.,22., 23., 24.,22., 23., 24.,19., 20., 21.,19., 20., 21.,22., 23., 24.,22., 23., 24.,13., 14., 15.,13., 14., 15.,16., 17., 18.,16., 17., 18.,13., 14., 15.,13., 14., 15.,16., 17., 18.,16., 17., 18.,
            13., 14., 15.,13., 14., 15.,16., 17., 18.,16., 17., 18.,19., 20., 21.,19., 20., 21.,22., 23., 24.,22., 23., 24.,19., 20., 21.,19., 20., 21.,22., 23., 24.,22., 23., 24.,19., 20., 21.,19., 20., 21.,22., 23., 24.,22., 23., 24.,25., 26., 27.,25., 26., 27.,28., 29., 30.,28., 29., 30.,
            25., 26., 27.,25., 26., 27.,28., 29., 30.,28., 29., 30.,25., 26., 27.,25., 26., 27.,28., 29., 30.,28., 29., 30.,31., 32., 33.,31., 32., 33.,34., 35., 36.,34., 35., 36.,31., 32., 33.,31., 32., 33.,34., 35., 36.,34., 35., 36.,31., 32., 33.,31., 32., 33.,34., 35., 36.,34., 35., 36.,
            25., 26., 27.,25., 26., 27.,28., 29., 30.,28., 29., 30.,25., 26., 27.,25., 26., 27.,28., 29., 30.,28., 29., 30.,25., 26., 27.,25., 26., 27.,28., 29., 30.,28., 29., 30.,31., 32., 33.,31., 32., 33.,34., 35., 36.,34., 35., 36.,31., 32., 33.,31., 32., 33.,34., 35., 36.,34., 35., 36.,
            31., 32., 33.,31., 32., 33.,34., 35., 36.,34., 35., 36.,37., 38., 39.,37., 38., 39.,40., 41., 42.,40., 41., 42.,37., 38., 39.,37., 38., 39.,40., 41., 42.,40., 41., 42.,37., 38., 39.,37., 38., 39.,40., 41., 42.,40., 41., 42.,43., 44., 45.,43., 44., 45.,46., 47., 48.,46., 47., 48.,
            43., 44., 45.,43., 44., 45.,46., 47., 48.,46., 47., 48.,43., 44., 45.,43., 44., 45.,46., 47., 48.,46., 47., 48.,37., 38., 39.,37., 38., 39.,40., 41., 42.,40., 41., 42.,37., 38., 39.,37., 38., 39.,40., 41., 42.,40., 41., 42.,37., 38., 39.,37., 38., 39.,40., 41., 42.,40., 41., 42.,
            43., 44., 45.,43., 44., 45.,46., 47., 48.,46., 47., 48.,43., 44., 45.,43., 44., 45.,46., 47., 48.,46., 47., 48.,43., 44., 45.,43., 44., 45.,46., 47., 48.,46., 47., 48.,49., 50., 51.,49., 50., 51.,52., 53., 54.,52., 53., 54.,49., 50., 51.,49., 50., 51.,52., 53., 54.,52., 53., 54.,
            49., 50., 51.,49., 50., 51.,52., 53., 54.,52., 53., 54.,55., 56., 57.,55., 56., 57.,58., 59., 60.,58., 59., 60.,55., 56., 57.,55., 56., 57.,58., 59., 60.,58., 59., 60.,55., 56., 57.,55., 56., 57.,58., 59., 60.,58., 59., 60.,49., 50., 51.,49., 50., 51.,52., 53., 54.,52., 53., 54.,
            49., 50., 51.,49., 50., 51.,52., 53., 54.,52., 53., 54.,49., 50., 51.,49., 50., 51.,52., 53., 54.,52., 53., 54.,55., 56., 57.,55., 56., 57.,58., 59., 60.,58., 59., 60.,55., 56., 57.,55., 56., 57.,58., 59., 60.,58., 59., 60.,55., 56., 57.,55., 56., 57.,58., 59., 60.,58., 59., 60.,
            61., 62., 63.,61., 62., 63.,64., 65., 66.,64., 65., 66.,61., 62., 63.,61., 62., 63.,64., 65., 66.,64., 65., 66.,61., 62., 63.,61., 62., 63.,64., 65., 66.,64., 65., 66.,67., 68., 69.,67., 68., 69.,70., 71., 72.,70., 71., 72.,67., 68., 69.,67., 68., 69.,70., 71., 72.,70., 71., 72.,
            67., 68., 69.,67., 68., 69.,70., 71., 72.,70., 71., 72.,61., 62., 63.,61., 62., 63.,64., 65., 66.,64., 65., 66.,61., 62., 63.,61., 62., 63.,64., 65., 66.,64., 65., 66.,61., 62., 63.,61., 62., 63.,64., 65., 66.,64., 65., 66.,67., 68., 69.,67., 68., 69.,70., 71., 72.,70., 71., 72.,
            67., 68., 69.,67., 68., 69.,70., 71., 72.,70., 71., 72.,67., 68., 69.,67., 68., 69.,70., 71., 72.,70., 71., 72.});

    nd4j::ops::upsampling3d op;
    auto results = op.execute({&input}, {}, {factorD, factorH, factorW, isNCDHW});
    auto* output = results->at(0);

    ASSERT_EQ(Status::OK(), results->status());
    ASSERT_TRUE(expOutput.isSameShape(output));
    ASSERT_TRUE(expOutput.equalsTo(output));

    delete results;
}

//////////////////////////////////////////////////////////////////////
TEST_F(ConvolutionTests1, upsampling3d_test2) {

    const int bS=3,  iD=2,iH=2,iW=2,  iC=3;
    const int factorD=2,factorH=3,factorW=2;
    const int isNCDHW = 1;                    // data format, default is NCHW

    auto input = NDArrayFactory::create<float>('c', {bS, iC, iD, iH, iW});
    input.linspace(1);

    auto expOutput = NDArrayFactory::create<float>('c', {bS, iC, iD*factorD, iH*factorH, iW*factorW}, { 1.,  1.,  2.,  2., 1.,  1.,  2.,  2., 1.,  1.,  2.,  2., 3.,  3.,  4.,  4., 3.,  3.,  4.,  4., 3.,  3.,  4.,  4., 1.,  1.,  2.,  2., 1.,  1.,  2.,  2., 1.,  1.,  2.,  2., 3.,  3.,  4.,  4., 3.,  3.,  4.,  4., 3.,  3.,  4.,  4., 5.,  5.,  6.,  6., 5.,  5.,  6.,  6., 5.,  5.,  6.,  6., 7.,  7.,  8.,  8., 7.,  7.,  8.,  8., 7.,  7.,  8.,  8.,
             5.,  5.,  6.,  6., 5.,  5.,  6.,  6., 5.,  5.,  6.,  6., 7.,  7.,  8.,  8., 7.,  7.,  8.,  8., 7.,  7.,  8.,  8., 9.,  9., 10., 10., 9.,  9., 10., 10., 9.,  9., 10., 10.,11., 11., 12., 12.,11., 11., 12., 12.,11., 11., 12., 12., 9.,  9., 10., 10., 9.,  9., 10., 10., 9.,  9., 10., 10.,11., 11., 12., 12.,11., 11., 12., 12.,11., 11., 12., 12.,
            13., 13., 14., 14.,13., 13., 14., 14.,13., 13., 14., 14.,15., 15., 16., 16.,15., 15., 16., 16.,15., 15., 16., 16.,13., 13., 14., 14.,13., 13., 14., 14.,13., 13., 14., 14.,15., 15., 16., 16.,15., 15., 16., 16.,15., 15., 16., 16.,17., 17., 18., 18.,17., 17., 18., 18.,17., 17., 18., 18.,19., 19., 20., 20.,19., 19., 20., 20.,19., 19., 20., 20.,
            17., 17., 18., 18.,17., 17., 18., 18.,17., 17., 18., 18.,19., 19., 20., 20.,19., 19., 20., 20.,19., 19., 20., 20.,21., 21., 22., 22.,21., 21., 22., 22.,21., 21., 22., 22.,23., 23., 24., 24.,23., 23., 24., 24.,23., 23., 24., 24.,21., 21., 22., 22.,21., 21., 22., 22.,21., 21., 22., 22.,23., 23., 24., 24.,23., 23., 24., 24.,23., 23., 24., 24.,
            25., 25., 26., 26.,25., 25., 26., 26.,25., 25., 26., 26.,27., 27., 28., 28.,27., 27., 28., 28.,27., 27., 28., 28.,25., 25., 26., 26.,25., 25., 26., 26.,25., 25., 26., 26.,27., 27., 28., 28.,27., 27., 28., 28.,27., 27., 28., 28.,29., 29., 30., 30.,29., 29., 30., 30.,29., 29., 30., 30.,31., 31., 32., 32.,31., 31., 32., 32.,31., 31., 32., 32.,
            29., 29., 30., 30.,29., 29., 30., 30.,29., 29., 30., 30.,31., 31., 32., 32.,31., 31., 32., 32.,31., 31., 32., 32.,33., 33., 34., 34.,33., 33., 34., 34.,33., 33., 34., 34.,35., 35., 36., 36.,35., 35., 36., 36.,35., 35., 36., 36.,33., 33., 34., 34.,33., 33., 34., 34.,33., 33., 34., 34.,35., 35., 36., 36.,35., 35., 36., 36.,35., 35., 36., 36.,
            37., 37., 38., 38.,37., 37., 38., 38.,37., 37., 38., 38.,39., 39., 40., 40.,39., 39., 40., 40.,39., 39., 40., 40.,37., 37., 38., 38.,37., 37., 38., 38.,37., 37., 38., 38.,39., 39., 40., 40.,39., 39., 40., 40.,39., 39., 40., 40.,41., 41., 42., 42.,41., 41., 42., 42.,41., 41., 42., 42.,43., 43., 44., 44.,43., 43., 44., 44.,43., 43., 44., 44.,
            41., 41., 42., 42.,41., 41., 42., 42.,41., 41., 42., 42.,43., 43., 44., 44.,43., 43., 44., 44.,43., 43., 44., 44.,45., 45., 46., 46.,45., 45., 46., 46.,45., 45., 46., 46.,47., 47., 48., 48.,47., 47., 48., 48.,47., 47., 48., 48.,45., 45., 46., 46.,45., 45., 46., 46.,45., 45., 46., 46.,47., 47., 48., 48.,47., 47., 48., 48.,47., 47., 48., 48.,
            49., 49., 50., 50.,49., 49., 50., 50.,49., 49., 50., 50.,51., 51., 52., 52.,51., 51., 52., 52.,51., 51., 52., 52.,49., 49., 50., 50.,49., 49., 50., 50.,49., 49., 50., 50.,51., 51., 52., 52.,51., 51., 52., 52.,51., 51., 52., 52.,53., 53., 54., 54.,53., 53., 54., 54.,53., 53., 54., 54.,55., 55., 56., 56.,55., 55., 56., 56.,55., 55., 56., 56.,
            53., 53., 54., 54.,53., 53., 54., 54.,53., 53., 54., 54.,55., 55., 56., 56.,55., 55., 56., 56.,55., 55., 56., 56.,57., 57., 58., 58.,57., 57., 58., 58.,57., 57., 58., 58.,59., 59., 60., 60.,59., 59., 60., 60.,59., 59., 60., 60.,57., 57., 58., 58.,57., 57., 58., 58.,57., 57., 58., 58.,59., 59., 60., 60.,59., 59., 60., 60.,59., 59., 60., 60.,
            61., 61., 62., 62.,61., 61., 62., 62.,61., 61., 62., 62.,63., 63., 64., 64.,63., 63., 64., 64.,63., 63., 64., 64.,61., 61., 62., 62.,61., 61., 62., 62.,61., 61., 62., 62.,63., 63., 64., 64.,63., 63., 64., 64.,63., 63., 64., 64.,65., 65., 66., 66.,65., 65., 66., 66.,65., 65., 66., 66.,67., 67., 68., 68.,67., 67., 68., 68.,67., 67., 68., 68.,
            65., 65., 66., 66.,65., 65., 66., 66.,65., 65., 66., 66.,67., 67., 68., 68.,67., 67., 68., 68.,67., 67., 68., 68.,69., 69., 70., 70.,69., 69., 70., 70.,69., 69., 70., 70.,71., 71., 72., 72.,71., 71., 72., 72.,71., 71., 72., 72.,69., 69., 70., 70.,69., 69., 70., 70.,69., 69., 70., 70.,71., 71., 72., 72.,71., 71., 72., 72.,71., 71., 72., 72.});

    nd4j::ops::upsampling3d op;
    auto results = op.execute({&input}, {}, {factorD, factorH, factorW, isNCDHW});
    auto* output = results->at(0);

    ASSERT_EQ(Status::OK(), results->status());

    ASSERT_TRUE(expOutput.isSameShape(output));
    ASSERT_TRUE(expOutput.equalsTo(output));

    delete results;
}

//////////////////////////////////////////////////////////////////////
TEST_F(ConvolutionTests1, upsampling2d_bp_test1) {

    const int bS=1,  iH=2,iW=2,  iC=1;
    const int factorH=2, factorW=2;
    const int isNCHW = 1;                    // data format, default is NCHW

    auto input  = NDArrayFactory::create<float>('c', {bS, iC, iH, iW});
    auto gradO  = NDArrayFactory::create<float>('c', {bS, iC, iH*factorH, iW*factorW});
    gradO = 1.;

    auto expGradI = NDArrayFactory::create<float>('c', {bS, iC, iH, iW});
    expGradI = 4.;

    nd4j::ops::upsampling2d_bp op;
    auto results = op.execute({&input, &gradO}, {}, {isNCHW});
    auto* gradI = results->at(0);

    ASSERT_EQ(Status::OK(), results->status());
    ASSERT_TRUE(expGradI.isSameShape(gradI));
    ASSERT_TRUE(expGradI.equalsTo(gradI));

    delete results;
}

//////////////////////////////////////////////////////////////////////
TEST_F(ConvolutionTests1, upsampling2d_bp_test2) {

    const int bS=1,  iH=2,iW=2,  iC=1;
    const int factorH=2, factorW=2;
    const int isNCHW = 0;                    // data format, default is NCHW

    auto input  = NDArrayFactory::create<float>('c', {bS, iH, iW, iC});
    auto gradO  = NDArrayFactory::create<float>('c', {bS, iH*factorH, iW*factorW, iC});
    gradO = 1.;

    auto expGradI = NDArrayFactory::create<float>('c', {bS, iH, iW, iC});
    expGradI = 4.;

    nd4j::ops::upsampling2d_bp op;
    auto results = op.execute({&input, &gradO}, {}, {isNCHW});
    auto* gradI = results->at(0);

    ASSERT_EQ(Status::OK(), results->status());
    ASSERT_TRUE(expGradI.isSameShape(gradI));
    ASSERT_TRUE(expGradI.equalsTo(gradI));

    delete results;
}

//////////////////////////////////////////////////////////////////////
TEST_F(ConvolutionTests1, upsampling3d_bp_test1) {

    const int bS=1,  iD=2,iH=2,iW=2,  iC=1;
    const int factorD=2, factorH=2, factorW=2;
    const int isNCDHW = 1;                    // data format, default is NCHW

    auto input  = NDArrayFactory::create<float>('c', {bS, iC, iD, iH, iW});
    auto gradO  = NDArrayFactory::create<float>('c', {bS, iC, iD*factorD, iH*factorH, iW*factorW});
    gradO = 1.;

    auto expGradI = NDArrayFactory::create<float>('c', {bS, iC, iD, iH, iW});
    expGradI = 8.;

    nd4j::ops::upsampling3d_bp op;
    auto results = op.execute({&input, &gradO}, {}, {isNCDHW});
    auto* gradI = results->at(0);

    ASSERT_EQ(Status::OK(), results->status());
    ASSERT_TRUE(expGradI.isSameShape(gradI));
    ASSERT_TRUE(expGradI.equalsTo(gradI));

    delete results;
}

TYPED_TEST(TypedConvolutionTests1, conv2D_input_BP_test1) {

    auto inputShape = NDArrayFactory::create<TypeParam>('c', {4}, {2, 1, 4, 4});
    auto weights = NDArrayFactory::create<TypeParam>('c', {2, 1, 3, 3});
    auto epsilonNext = NDArrayFactory::create<TypeParam>('c', {2, 2, 4, 4});
    auto shapeArr = NDArrayFactory::create<TypeParam>('c', {2, 1, 4, 4});


    TypeParam _expEpsB[] = {952.0, 1540.0, 1636.0, 1180.0, 1791.0, 2886.0, 3057.0, 2193.0, 2223.0, 3570.0, 3741.0, 2673.0, 1900.0, 3028.0, 3160.0, 2240.0, 2872.0, 4612.0, 4708.0, 3356.0, 5247.0, 8358.0, 8529.0, 6033.0, 5679.0, 9042.0, 9213.0, 6513.0, 4588.0, 7252.0, 7384.0, 5184.0};
    NDArray expEps(_expEpsB, shapeArr.getShapeInfo());

    weights.linspace(1);
    epsilonNext.linspace(1);
    weights.permutei({2,3,1,0});

    nd4j::ops::conv2d_input_bp op;

    auto results = op.execute({&inputShape, &weights, &epsilonNext}, {},  {3, 3, 1, 1, 0, 0, 1, 1, 1});

    ASSERT_TRUE(results->size() == 1);

    auto epsilon = results->at(0);

    ASSERT_TRUE(shapeArr.isSameShape(epsilon));
    ASSERT_TRUE(expEps.equalsTo(epsilon));

    delete results;
}

//////////////////////////////////////////////////////////////////////
TEST_F(ConvolutionTests1, deconv2d_test1) {

    int bS=2, iH=4,iW=4,  iC=5,oC=10,  kH=2,kW=2,  sH=1,sW=1,  pH=0,pW=0,  dH=1,dW=1;
    int       oH=3,oW=3;
    int paddingMode = 0;             // 1-SAME, 0-VALID;
    int dataFormat  = 1;             // 1-NHWC, 0-NCHW

    auto input    = NDArrayFactory::create<double>('c', {bS, oH, oW, oC});
    auto weights  = NDArrayFactory::create<double>('c', {kH, kW, iC, oC});
    auto exp = NDArrayFactory::create<double>('c', {bS, iH, iW, iC}, {  2.75,   7.75,  12.75,  17.75,  22.75, 30.5 ,  40.5 ,  50.5 ,  60.5 ,  70.5 , 30.5 ,  40.5 ,  50.5 ,  60.5 ,  70.5 , 27.75,  32.75,  37.75,  42.75,  47.75,
                                                  55.5 ,  65.5 ,  75.5 ,  85.5 ,  95.5 ,161.  , 181.  , 201.  , 221.  , 241.  ,161.  , 181.  , 201.  , 221.  , 241.  ,105.5 , 115.5 , 125.5 , 135.5 , 145.5 ,
                                                  55.5 ,  65.5 ,  75.5 ,  85.5 ,  95.5 ,161.  , 181.  , 201.  , 221.  , 241.  ,161.  , 181.  , 201.  , 221.  , 241.  ,105.5 , 115.5 , 125.5 , 135.5 , 145.5 ,
                                                  52.75,  57.75,  62.75,  67.75,  72.75,130.5 , 140.5 , 150.5 , 160.5 , 170.5 ,130.5 , 140.5 , 150.5 , 160.5 , 170.5 , 77.75,  82.75,  87.75,  92.75,  97.75,
                                                   2.75,   7.75,  12.75,  17.75,  22.75, 30.5 ,  40.5 ,  50.5 ,  60.5 ,  70.5 , 30.5 ,  40.5 ,  50.5 ,  60.5 ,  70.5 , 27.75,  32.75,  37.75,  42.75,  47.75,
                                                  55.5 ,  65.5 ,  75.5 ,  85.5 ,  95.5 ,161.  , 181.  , 201.  , 221.  , 241.  ,161.  , 181.  , 201.  , 221.  , 241.  ,105.5 , 115.5 , 125.5 , 135.5 , 145.5 ,
                                                  55.5 ,  65.5 ,  75.5 ,  85.5 ,  95.5 ,161.  , 181.  , 201.  , 221.  , 241.  ,161.  , 181.  , 201.  , 221.  , 241.  ,105.5 , 115.5 , 125.5 , 135.5 , 145.5 ,
                                                  52.75,  57.75,  62.75,  67.75,  72.75,130.5 , 140.5 , 150.5 , 160.5 , 170.5 ,130.5 , 140.5 , 150.5 , 160.5 , 170.5 , 77.75,  82.75,  87.75,  92.75,  97.75});
    input = 0.5;
    weights.linspace(0.1, 0.1);

    nd4j::ops::deconv2d op;
    auto results = op.execute({&input, &weights}, {}, {kH,kW,  sH,sW,  pH,pW,  dH,dW, paddingMode, dataFormat});
    ASSERT_EQ(Status::OK(), results->status());

    auto output = results->at(0);

    ASSERT_TRUE(exp.isSameShape(output));
    ASSERT_TRUE(exp.equalsTo(output));

    delete results;
}

//////////////////////////////////////////////////////////////////////
TEST_F(ConvolutionTests1, deconv2d_test2) {

    int bS=2, iH=4,iW=4,  iC=5,oC=10,  kH=2,kW=2,  sH=1,sW=1,  pH=0,pW=0,  dH=1,dW=1;
    int       oH=4,oW=4;
    int paddingMode = 1;             // 1-SAME, 0-VALID;
    int dataFormat  = 1;             // 1-NHWC, 0-NCHW

    auto input    = NDArrayFactory::create<double>('c', {bS, oH, oW, oC});
    auto weights  = NDArrayFactory::create<double>('c', {kH, kW, iC, oC});
    auto exp = NDArrayFactory::create<double>('c', {bS, iH, iW, iC}, {2.75,   7.75,  12.75,  17.75,  22.75, 30.5 ,  40.5 ,  50.5 ,  60.5 ,  70.5 , 30.5 ,  40.5 ,  50.5 ,  60.5 ,  70.5 , 30.5 ,  40.5 ,  50.5 ,  60.5 ,  70.5 ,
                                                55.5 ,  65.5 ,  75.5 ,  85.5 ,  95.5 ,161.  , 181.  , 201.  , 221.  , 241.  ,161.  , 181.  , 201.  , 221.  , 241.  ,161.  , 181.  , 201.  , 221.  , 241.  ,
                                                55.5 ,  65.5 ,  75.5 ,  85.5 ,  95.5 ,161.  , 181.  , 201.  , 221.  , 241.  ,161.  , 181.  , 201.  , 221.  , 241.  ,161.  , 181.  , 201.  , 221.  , 241.  ,
                                                55.5 ,  65.5 ,  75.5 ,  85.5 ,  95.5 ,161.  , 181.  , 201.  , 221.  , 241.  ,161.  , 181.  , 201.  , 221.  , 241.  ,161.  , 181.  , 201.  , 221.  , 241.  ,
                                                 2.75,   7.75,  12.75,  17.75,  22.75, 30.5 ,  40.5 ,  50.5 ,  60.5 ,  70.5 , 30.5 ,  40.5 ,  50.5 ,  60.5 ,  70.5 , 30.5 ,  40.5 ,  50.5 ,  60.5 ,  70.5 ,
                                                55.5 ,  65.5 ,  75.5 ,  85.5 ,  95.5 ,161.  , 181.  , 201.  , 221.  , 241.  ,161.  , 181.  , 201.  , 221.  , 241.  ,161.  , 181.  , 201.  , 221.  , 241.  ,
                                                55.5 ,  65.5 ,  75.5 ,  85.5 ,  95.5 ,161.  , 181.  , 201.  , 221.  , 241.  ,161.  , 181.  , 201.  , 221.  , 241.  ,161.  , 181.  , 201.  , 221.  , 241.  ,
                                                55.5 ,  65.5 ,  75.5 ,  85.5 ,  95.5 ,161.  , 181.  , 201.  , 221.  , 241.  ,161.  , 181.  , 201.  , 221.  , 241.  ,161.  , 181.  , 201.  , 221.  , 241.  });
    input = 0.5;
    weights.linspace(0.1, 0.1);

    nd4j::ops::deconv2d op;
    auto results = op.execute({&input, &weights}, {}, {kH,kW,  sH,sW,  pH,pW,  dH,dW, paddingMode, dataFormat});
    auto output = results->at(0);

    ASSERT_EQ(Status::OK(), results->status());
    ASSERT_TRUE(exp.isSameShape(output));
    ASSERT_TRUE(exp.equalsTo(output));

    delete results;
}

//////////////////////////////////////////////////////////////////////
TYPED_TEST(TypedConvolutionTests1, deconv2d_tf_test1) {

    int bS=2, iH=4,iW=4,  iC=5,oC=10,  kH=2,kW=2,  sH=1,sW=1,  pH=0,pW=0,  dH=1,dW=1;
    int       oH=3,oW=3;
    int paddingMode = 0;             // 1-SAME, 0-VALID;
    int dataFormat  = 1;             // 1-NHWC, 0-NCHW

    auto input    = NDArrayFactory::create<TypeParam>('c', {bS, oH, oW, oC});
    auto weights  = NDArrayFactory::create<TypeParam>('c', {kH, kW, iC, oC});
    auto outShape = NDArrayFactory::create<TypeParam>('c', {4}, {static_cast<TypeParam>(bS), static_cast<TypeParam>(iH), static_cast<TypeParam>(iW), static_cast<TypeParam>(iC)});
    auto exp = NDArrayFactory::create<TypeParam>('c', {bS, iH, iW, iC}, {  2.75,   7.75,  12.75,  17.75,  22.75, 30.5 ,  40.5 ,  50.5 ,  60.5 ,  70.5 , 30.5 ,  40.5 ,  50.5 ,  60.5 ,  70.5 , 27.75,  32.75,  37.75,  42.75,  47.75,
                                                  55.5 ,  65.5 ,  75.5 ,  85.5 ,  95.5 ,161.  , 181.  , 201.  , 221.  , 241.  ,161.  , 181.  , 201.  , 221.  , 241.  ,105.5 , 115.5 , 125.5 , 135.5 , 145.5 ,
                                                  55.5 ,  65.5 ,  75.5 ,  85.5 ,  95.5 ,161.  , 181.  , 201.  , 221.  , 241.  ,161.  , 181.  , 201.  , 221.  , 241.  ,105.5 , 115.5 , 125.5 , 135.5 , 145.5 ,
                                                  52.75,  57.75,  62.75,  67.75,  72.75,130.5 , 140.5 , 150.5 , 160.5 , 170.5 ,130.5 , 140.5 , 150.5 , 160.5 , 170.5 , 77.75,  82.75,  87.75,  92.75,  97.75,
                                                   2.75,   7.75,  12.75,  17.75,  22.75, 30.5 ,  40.5 ,  50.5 ,  60.5 ,  70.5 , 30.5 ,  40.5 ,  50.5 ,  60.5 ,  70.5 , 27.75,  32.75,  37.75,  42.75,  47.75,
                                                  55.5 ,  65.5 ,  75.5 ,  85.5 ,  95.5 ,161.  , 181.  , 201.  , 221.  , 241.  ,161.  , 181.  , 201.  , 221.  , 241.  ,105.5 , 115.5 , 125.5 , 135.5 , 145.5 ,
                                                  55.5 ,  65.5 ,  75.5 ,  85.5 ,  95.5 ,161.  , 181.  , 201.  , 221.  , 241.  ,161.  , 181.  , 201.  , 221.  , 241.  ,105.5 , 115.5 , 125.5 , 135.5 , 145.5 ,
                                                  52.75,  57.75,  62.75,  67.75,  72.75,130.5 , 140.5 , 150.5 , 160.5 , 170.5 ,130.5 , 140.5 , 150.5 , 160.5 , 170.5 , 77.75,  82.75,  87.75,  92.75,  97.75});
    input = 0.5;
    weights.linspace(0.1, 0.1);

    nd4j::ops::deconv2d_tf op;
    auto results = op.execute({&outShape, &weights, &input}, {}, {kH,kW,  sH,sW,  pH,pW,  dH,dW, paddingMode, dataFormat});
    auto output = results->at(0);

    ASSERT_EQ(Status::OK(), results->status());
    ASSERT_TRUE(exp.isSameShape(output));
    ASSERT_TRUE(exp.equalsTo(output));

    delete results;
}


#endif //LIBND4J_CONVOLUTIONTESTS1_H